From cf01cf6d9b0fbdc383c8364ab1d28240dfdc7aef Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 20:01:10 +0200 Subject: [PATCH 001/135] docs: preserve admin instance management design and plans --- .../2026-10-23-admin-01-service-control.md | 167 ++++++ .../plans/2026-10-23-admin-02-durable-jobs.md | 176 ++++++ .../2026-10-23-admin-03-account-policy.md | 174 ++++++ ...-23-admin-04-integrations-and-migration.md | 200 +++++++ .../2026-10-23-admin-05-transit-and-data.md | 194 +++++++ .../2026-10-23-admin-06-artifact-transfer.md | 249 +++++++++ .../2026-10-23-admin-07-privacy-workflows.md | 179 ++++++ ...6-10-23-admin-08-platform-configuration.md | 240 ++++++++ ...026-10-23-admin-09-maintenance-controls.md | 168 ++++++ .../2026-10-23-admin-instance-management.md | 175 ++++++ .../plans/2026-10-23-admin-plan-review.md | 113 ++++ .../2026-10-23-admin-control-protocols.md | 524 ++++++++++++++++++ ...-10-23-admin-instance-management-design.md | 242 ++++++++ 13 files changed, 2801 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-23-admin-01-service-control.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-02-durable-jobs.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-03-account-policy.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-05-transit-and-data.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-06-artifact-transfer.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-07-privacy-workflows.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-instance-management.md create mode 100644 docs/superpowers/plans/2026-10-23-admin-plan-review.md create mode 100644 docs/superpowers/specs/2026-10-23-admin-control-protocols.md create mode 100644 docs/superpowers/specs/2026-10-23-admin-instance-management-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md new file mode 100644 index 000000000..ac013ae2a --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -0,0 +1,167 @@ +# Service Control Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement S01–S08: effective updates, shared preparation, truthful batch outcomes, owned-service capabilities, safe core maintenance, and usable outage troubleshooting. + +**Architecture:** The agent derives action decisions and owned-container inventory from trusted authority, then enforces those decisions again at execution. API jobs retain preparation/effect identities and project agent outcomes; the generated SDK drives Services controls. Preserve existing release replacement and configuration-generation workflows instead of adding a command endpoint. + +**Tech Stack:** TypeScript, Fastify, Zod, PostgreSQL/Drizzle job records, Docker ops-agent, Commander CLI, React/MUI, generated SDK, Vitest. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md` and binding `docs/superpowers/specs/2026-10-23-admin-control-protocols.md` (protocol sections cited below); master: `docs/superpowers/plans/2026-10-23-admin-instance-management.md`. Audit base: `ef896ceae876959180116cf61ffcf643b88ebffe`. + +## Global Constraints + +- All product work stays in this worktree; preserve unrelated work and existing public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source policy boundaries. Community integrations remain declarative; no uploaded code execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain behind typed ops-agent operations. No generic command, arbitrary path or raw environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth declarations, strict schemas, audit events, explicit mutation identities and appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, durable operation identities, retained results, unknown outcomes and cancellation fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database and runtime checks, independent review, production builds and PR CI. Report any unavailable environment gate distinctly; do not replace runtime evidence with mocked assertions. + +## Review Focus + +1. A deselected container absent from the new Compose file must still be stoppable/readable only by verified ownership; forged labels or changed container identity must fail (Task 1). +2. A new generation or release pin between preparation and recreation must prevent stale application, including repeated job recovery (Task 2). +3. Failure after one completed service, fail-fast, or cancellation must preserve completed/failed/skipped identities without a success event (Task 3). +4. Replacing the API or edge while the browser disconnects must retain an externally observable result; the executing agent must never terminate itself (Task 4). +5. An agent outage, including the first server render, must show unknown state and leave full-admin troubleshooting navigation reachable (Tasks 5–6). + +--- + +## File ownership and integration order + +- Tasks 1/4 own new `apps/ops-agent/src/service-action-policy.ts`, `owned-services.ts`, `core-service-runtime.ts`, and `production-runtime.ts`; `index.ts` becomes the caller of the extracted production assembly, without changing validated startup/secret acquisition order. +- Task 2 owns new `apps/api/src/services/{service-preparation,prospective-configuration,administrative-configuration-writer,configuration-maintenance}.ts`, agent `configuration-maintenance-{runtime,store}.ts` and core `ops/configuration-maintenance.ts`, strict service/integration snapshot adapters, `packages/cli/src/lib/service-lifecycle.ts`, and `packages/core/src/services/service-lifecycle-receipt.ts`; existing lifecycle handlers remain thin adapters. This early producer owns strict reads, finite overlays, source authority and full prospective budget validation before04/05/08 consume them.04.2 owns visible ownership projection and all direct-route writer guards. Task 3 owns new `apps/api/src/services/service-batch-result.ts`. +- Controller serializes `packages/core/src/ops/contract.ts`, `packages/core/src/ops/index.ts`, `packages/core/src/api/contracts/service-admin/{services,workflows,index}.ts`, `scripts/ops-authority-inventory.json`, API route registration, and generated artifacts. The inventory filename was verified against the tree and `scripts/check-ops-authority.ts`; also update `scripts/ops-authority-inventory.test.ts` for exhaustive typed-role coverage. +- Shared edits requiring sequential ownership: `apps/ops-agent/src/{administrative-runtime,docker-runtime,policy,runtime,index,server,journal}.ts`; `apps/api/src/services/{admin-ops,admin-job-handlers,job-runner,ops-client}.ts`; `apps/api/src/routes/admin-services.ts`; CLI `commands/{services,data,compose}.ts`; Services UI files listed below. +- Follow the master's binding exact execution order: 02.1, 02.2, 06.1, 02.3, then 01 Tasks 1–5; 02 Task 4 consumes preparation from 01 Task 2, then 01 Task 6 follows controller generation. Each task has an exclusive integration/review window; no product edits overlap. No migration is needed solely for the S-series semantics (02 owns coordinated durable-job schema changes): failed batches retain the existing `failed` status with typed partial evidence. +- Plans 04/08 own config/credential fields and adoption; they consume preparation without replacing its CAS. Plan 05 owns transit policy/catalog changes. Plan 09 owns M01–M04 form/catalog/option additions; this plan reserves the interfaces below. Shared files are integrated by the controller, never concurrently rewritten. + +## Cross-plan interfaces + +- `ServiceAction = "start" | "stop" | "restart" | "update" | "build" | "logs" | "pull" | "apply"`; `ServiceActionDecision = { available: boolean; execution: "direct" | "workflow" | "host"; reason: "disabled" | "not_owned" | "not_running" | "control_plane" | "host_only" | "unavailable" | "not_buildable" | null; workflow: "core-restart" | "system-update" | null }`. A denied decision has a non-null reason; a direct available decision has null reason/workflow. An available workflow decision has null reason and a non-null workflow; `execution: "host"` is unavailable with `reason: "host_only"`. API/edge Restart selects `core-restart`; available application Update selects `system-update` only when existing checked-release eligibility permits it. Protected dependency lifecycle stays host-only. Never advertise a workflow solely because its route exists. +- New typed `services.inspect` is API-role synchronous, 15 seconds, maximum 256 services: `{ services: Array<{ serviceId: string; state: existingStackState; actions: Record }> }`. It returns only trusted registered services; acquisition failure throws. API adds `"unknown"` to its service status enum and returns `acquisition: { state: "available" | "unavailable"; reason: "ops_unavailable" | null }` plus `summary.unknown`. +- Move/re-export `HardlinkApplySummary` from `apps/api/src/services/admin-ops.ts` to controller-owned `packages/core/src/ops/service-results.ts` so the CLI and API import the same portable type; API preserves its current re-export for callers. +- `prepareServiceGeneration(ctx: JobContext, scope: string, options: { coordinatorMode: CoordinatorMode }): Promise` uses the exact strict candidate and publication/follow-up split in protocol§5–6. Portable `CoordinatorMode = "ensure-running" | "require-running"`; `ServicePreparationResult = { revisionId: string; hardlinks: HardlinkApplySummary; coordinatorMode: CoordinatorMode }`. Start/Update selects ensure-running;09.6 Compose Render selects require-running. Mode, original revisions, sealed candidate identity and both preparation child identities are retained under `servicePreparation[scope]` before effects. Replays observe those identities; they never reconstruct a candidate or silently overwrite newer configuration. +- Export `prepareProspectiveConfiguration({actorId,expectedCurrentRevisionId,overlay,operationKey,signal?}): Promise`, `persistAndApplyProspectiveConfiguration(candidate,{coordinatorMode,signal?}): Promise` and `withAdministrativeConfigurationWriter` exactly as protocol§5. All callers enter the API global writer before sorted integration locks; host effects then follow target→nonblocking transit→host writer→release store. Reentrancy requires the same branded live connection/context, never a boolean. Candidate payload, sealed bytes and source fingerprint are server-private; no later snapshot collector may replace them. +- Extend `service.start`, `service.recreate`, `service.recreateIsolated`, and `service.update` with `expectedConfigurationRevisionId: string` (the `cfg1_` schema). Every API-role call, including preparation's data-manager Start, supplies it; omission is a schema error, not a bypass. Agent checks it under shared target-lease and configuration writer ownership before effect. `service.update` returns `{ changed: true, revisionId: string }`, the resulting generation rather than its input condition; the API checkpoints this before any subsequent target. Only Update advances this condition; Start/recreate checks the exact prepared revision and never creates a fresh generation implicitly. +- M01 presets resolve to explicit canonical `serviceIds` at admission and store both `presetIds?: string[]` and expanded IDs in the job. Plan 09 owns the finite preset enum and resolver; agents never interpret a preset as shell text or silently broaden it on resume. +- M02 update uses `noDeps?: boolean` (default false); M03 build uses `motisImport?: boolean` (default false, meaningful only for `motis`); M04 pull is a distinct `action: "pull"`, never update with a suppressed recreate. Public log controls use `lines: number` (1–1000, default 200), `follow: boolean` (default true), `maxDurationSeconds: number` (1–900, default 900); only the agent adapter maps `lines` to ops `tail`, also bounded at 1000. Plan 09 owns public option exposure; reliability code preserves these exact fields in fingerprints, checkpoints and fixed argv. + +## Source findings and causal test rules + +Current `administrative-runtime.ts` installs build/release/data handlers but no `service.update`; `policy.ts` blocks disabled lifecycle and all core logs; `docker-runtime.ts` uses current Compose membership for Stop/Logs. `admin-job-handlers.ts` bulk Start skips the single-Start render/hardlink path, and build continuation returns failures as an ordinary success. `docker-compose.ts` maps unknown to not-running and catches acquisition failure as `[]`; `serverDeployment.ts` maps read failure to hosted false. These are current source observations, not test receipts. + +New test files first exercise the existing exported call path with valid authenticated fixtures. Record behavioral failure after setup succeeds; absent imports/exports, malformed test stubs or an unavailable dependency install are not causal RED. Each GREEN repeats the same assertion, then adds restart/authorization cases. For example, the current build-all runtime must fail `expect(status.state).toBe("failed")` after A succeeds/B fails/C succeeds, and outage inventory must fail `expect(body.summary.stopped).toBe(0)` with two known services. Every task includes unauthenticated/ordinary-user/privacy-admin denial at the real route (401/403, no enqueue or Docker call), plus stale authority at agent execution. Runtime acceptance uses the actual factory called by `index.ts` with only external I/O substituted. + +### Shared artifact target guard consumed by lifecycle + +Consume protocol§3's exact `acquire/delegate/adopt/settle/reconcile/releaseArtifactTargetLease` and `withArtifactTargetLease({dataRoot,serviceIds,owner,inherited?,signal?},run)` from `packages/core/src/build-artifacts/target-lease.ts`, implemented by06.1. Owner is `VerifiedExecutionAuthority`, inherited is `BorrowedArtifactTargetLease`, and callback receives `TargetLeaseAuthority`. No operationId-only constructor or `parentArtifactLeaseId` field survives. Ownership is anchored to one physical shared mount, role-verifier certificate, signed ledger and retained child intent; actual role credentials, fixed headers/environment, finite graph and no-steal settlement rules are in the binding spec. Borrowers cannot release the root or widen targets. `finally` retains the fence on attempted/unknown children. + +The agent's direct Start/recreate/core-restart/pull-with-recreate and hardlink adapters acquire before revision/ownership checks. For§5.4 configuration maintenance children they instead adopt the exact frozen signed subset of the retained root; no second root acquisition or target expansion. Agent Update/build acquires once and delegates to its fixed CLI; host CLI acquires its own root. CLI `withOwnership`/`buildServices` consumes this authority before the host writer. Derive the full manifest producer/consumer closure, not only the requested ID. `configuration.hardlinks.apply` acquires target→writer, validates exact revision/mode, and delegates to the fixed durable data-manager generation-hardlinks leaf; no Start or host writer reacquisition inside that leaf. Required coordinator Start is a separate settled child before entering this host writer. If transit reservation is busy, return busy and settle/release only the authority this execution owns. Host lock order and5s budget are unchanged. + +Task2's separate-process probes race CLI Start, single/bulk API Start, Update/build and import publication on the same physical fixture mount, including aliases/inverse target order. Assert one owner or explicit busy/conflict, no self-acquisition, no release after observer abort while a child lives, and denial of stale/foreign/replayed/widened grants.06.1 provides the signature/adoption foundation before lifecycle lands.08.3 uses the same target protocol, including its finite agent→external configuration-helper continuation; no old host writer survives across that detached handoff. + +### Task 1: Owned inventory and one enforced action policy — S04, S05 + +**Files:** Create `apps/ops-agent/src/{owned-services,service-action-policy,production-runtime}.ts` and their `.test.ts` files. Modify `apps/ops-agent/src/{index,policy,docker-runtime,trusted-config-runtime}.ts`; controller edits the ops contract and exhaustive operation inventories. + +**Interfaces:** `inspectOwnedServices(signal: AbortSignal): Promise`; `OwnedServiceSnapshot = { serviceId: string; containerId: string; state: "running" | "stopped" | "created" | "restarting" | "paused" | "unknown"; provenance: { project: string; composeWorkingDirectory: string; composeConfigFiles: string[]; imageId: string } }`. `ServiceAuthority = { revisionId: string; enabledServiceIds: ReadonlySet; services: readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[] }`. `decideServiceAction(authority: ServiceAuthority, snapshot: OwnedServiceSnapshot | undefined, action: ServiceAction): ServiceActionDecision`. `createProductionOpsRuntime(options: ProductionOpsRuntimeOptions): Promise` composes the existing installers in current production order; options carry the existing stack paths, authority loader, repository store and injected external I/O adapters, not replacement lifecycle handlers. + +- [ ] **RED test:** Add `deselected owned service retains stop and logs but cannot start` to `policy.test.ts` and `owned-services.test.ts`. Assert available stop/logs, denied start/recreate/pull/update/build; post-stop logs remain readable while the owned container exists. Include unknown registry ID, same name in another project, forged service label, replaced ID and community removal cases. +- [ ] **RED command:** `pnpm exec vitest run --project node apps/ops-agent/src/policy.test.ts apps/ops-agent/src/owned-services.test.ts`. At RED the new ownership test calls existing `createDockerRuntime`/`dispatchOpsOperation`; do not import the uncreated extractor. The existing policy must fail the available-stop/log assertions for disabled/core fixtures, and the existing Compose adapter must fail the immutable-ID argv assertion. After extraction switch the same assertions to the shared resolver. +- [ ] **Implement ownership:** Enumerate all containers for the fixed `services.STACK_PROJECT`, including stopped/orphaned containers; do not depend on current Compose membership. Verify `com.docker.compose.project`, `com.docker.compose.service`, `com.docker.compose.project.working_dir` and `com.docker.compose.project.config_files` against the fixed stack project, trusted registry and canonical infrastructure/current-or-retained generation paths; verify container image against the trusted effective image reference. Paths remain internal. Unverifiable/ambiguous provenance is denied; matching a caller's name or one label is insufficient. Capture immutable container IDs in the claim; re-inspect labels/identity at the effect boundary. Stop/log use fixed `docker stop`/`docker logs` argv on those IDs after verification, never a request-supplied name/path. A host principal with Docker authority is outside this web authorization boundary; do not promise that arbitrary hostile Docker metadata is cryptographic ownership. Oversized/malformed inventory fails acquisition instead of returning a truncated authoritative list. +- [ ] **Implement authority:** Keep current configured authority for start/build/recreate/pull/update. On deselection retain only stop/log access for a still-registered, proven owned container; no implicit automatic stop. Protect `ops-agent` from stop/restart/recreate/update, while permitting verified read-only logs. Tasks 4/6 consume the same policy decisions; routes are not the final authorization boundary. Remove `admin-ops.ts:assertKnownService`'s unconditional enabled check from Stop and log routes: retain known-ID checks, then consult the shared agent decision. Selection commits preserve registered disabled manifests and retained generations until owned cleanup; installed community removal must refuse while verified containers still need cleanup, rather than discarding their authority. +- [ ] **Implement assembly:** Move existing runtime composition into `production-runtime.ts`, call it from `index.ts`, and expose `services.inspect` there. Preserve trusted-config authority refresh, community recovery and all privacy/transit installers. `policy.ts` consumes the same resolver for admission; revalidation remains mandatory during execution. +- [ ] **GREEN command:** Run the RED command plus `pnpm exec vitest run --project node apps/ops-agent/src/production-runtime.test.ts apps/ops-agent/src/trusted-config-ownership.test.ts apps/ops-agent/src/startup-authority.test.ts`. +- [ ] **Review/commit:** Assert the assembled runtime, rather than a substituted `OpsRuntime`, executes stop/log against a deselected fixture. Commit only this task's owned files as `feat(admin): authorize owned service cleanup and logs`; controller includes coordinated contract changes in the same reviewed checkpoint. + +### Task 2: Wire update and share generation preparation — S01, S02 + +**Files:** Create `apps/api/src/services/{service-preparation,prospective-configuration,administrative-configuration-writer,configuration-maintenance}.ts`, agent `apps/ops-agent/src/configuration-maintenance-{runtime,store}.ts`, core `packages/core/src/ops/configuration-maintenance.ts`, API `__tests__/{configuration-maintenance-postgres,prospective-configuration-postgres}.test.ts` and agent `configuration-maintenance.integration.test.ts` plus focused module tests, `packages/cli/src/lib/service-lifecycle.ts`, `packages/cli/__tests__/cli-service-lifecycle.test.ts`, `packages/core/src/services/service-lifecycle-receipt.ts` and its test. Modify API `services/{admin-ops,admin-job-handlers,service-config-resolver,service-secrets,secrets,trusted-config-operations,trusted-config-publisher}.ts`, `integration-config.ts`, `server.ts`; agent `administrative-runtime.ts`, `production-runtime.ts`, `trusted-config-runtime.ts`, `trusted-config-source.ts`; CLI `commands/{services,compose}.ts`. Controller integrates core ops contracts/exports, finite candidate/maintenance types, source/preview/maintenance operations, safe failure projection and existing publisher/journal parsers in this exclusive window. Tests include existing trusted-config publisher/source, agent production runtime/writer concurrency, API admin-job-handlers/admin-ops-hardlinks and CLI update tests. No new migration; existing admin_job metadata holds the durable non-secret candidate fence, with bounded private filesystem candidate receipts. + +**Interfaces:** Implement protocol§5.4's exact `ConfigurationMaintenanceBinding/Prepare/Identity/Decision/Settle/Step/Receipt`, API `prepare/inspect/decide/settleConfigurationMaintenance` and typed `configurationMaintenance.prepare/inspect/decide/settle`. Consume02.3's `DataConfigurationFencePrepare/Receipt/Settle` and generated service-auth prepare/inspect/settle adapters;06.1 owns authority verification. Implement protocol§5's writer, strict service/integration readers, `ProspectiveConfigurationOverlay`, opaque `ValidatedConfigurationCandidate`, `ProspectiveConfigurationReceipt`, and source inspection; implement protocol§6's exact lifecycle helper/effects/receipt signatures. Consume02's `ExecutionRef`/void `FixedCliRunner` and06.1's nominal lease authority. Host-local optional expected revision is captured once under writer; agent wire requires it. `prepare` always renders the full selection. `ServiceLifecycleEffects.withOwnership` supplies the actual `ConfigurationGenerationWriter` and `TargetLeaseAuthority` to the inner helpers; no recursive lock acquisition. Export portable `CoordinatorMode`/`ServicePreparationResult` for04/06/08/09 and `ServiceLifecycleReceipt` for agent/CLI/API evidence. Existing `HardlinkApplySummary` moves to core service-results with API compatibility re-export. + +- [ ] **RED test:** `production service.update pulls then prepares and recreates` dispatches through `createProductionOpsRuntime` and `dispatchOpsOperation`, using fake process/Docker I/O and a real temporary generation/release store. Assert pin validation → successful pull → full-generation preparation/hardlinks → `up -d --force-recreate`; baseline dispatch throws `OpsNotWiredError`. At RED the test assembles the exported Docker/trusted-config/community/transit/admin installers exactly as current `index.ts` does, imports no new factory, and fails the successful-dispatch assertion; Task 1 later makes this probe use the extracted production factory. `ProductionOpsRuntimeOptions` injects process/filesystem/clock/transport edges, never a fake `service.update` or substituted runtime handler. +- [ ] **RED tests:** In `admin-job-handlers.test.ts`, Start one and bulk Start use one shared render/link path, full selected generation, deterministic per-service effect keys, and no target starts after preparation/link failure. Replay does not rerender; concurrent CAS change fails before start. Add ordinary CLI Start and bulk/single API traces asserting full-selection render → prune-enabled hardlinks → target-only `up -d`, including data-manager helper Start carrying the prepared revision. +- [ ] **RED command:** `pnpm exec vitest run --project node apps/ops-agent/src/production-runtime.test.ts apps/api/src/services/__tests__/admin-job-handlers.test.ts packages/cli/__tests__/cli-services-update.test.ts packages/cli/__tests__/cli-service-lifecycle.test.ts`. New lifecycle tests exercise current `registerServicesCommands` at RED, then the extracted helper at GREEN. +- [ ] **RED strict candidate/source tests:** Through current `buildConfigurationInput` and real save/import entry paths, fail one service DB read while vault/registry/other service reads succeed; assert no generation publication or mutation. Test a retained multi-byte/escaped feed map and multiple ordinary migration values whose merged envelope exceeds524288 bytes; existing write-before-apply path must fail the zero-persistence assertion. Preserve unchanged/orphan map entries. Test shadowed ordinary/resources through both namespaced SERVICE_* and actual host resource interpolation. These are behavioral RED, not missing new imports. Later04.2 adds its direct HTTP guard coverage;01.2 proves the shared producer now. +- [ ] **Implement strict producer before lifecycle:** Implement protocol§5 on the actual resolver/publisher paths. Acquire API global advisory then sorted integration locks using one dedicated connection/context. Strictly read unchanged services/integrations/vault/platform/roots, distinguish absent from failed reads, merge finite overlay, check schema/effective ownership/tree limits, canonical feed map≤64KiB/1000keys/512-code-point keys/8192-byte values and **exact sealed** envelope≤512KiB before any desired-row/proof save, effect enqueue or consumable publication; only after full candidate/queue/private-receipt preflight may§5.4 admit ownership-only maintenance. Preserve public fallback resolvers. `configuration.sources.inspect` reports bounded host ownership without values and binds expectedSourceRevision to typed commit. Candidate is frozen/private, durably retained and consumed verbatim, never reconstructed after save. Route existing full-snapshot helpers through this producer;04.2/05.6/08 later add their surface-specific guards/proofs. Strict integration reader is created here, not as a missing later dependency. +- [ ] **Implement candidate ownership/recovery:** Extend publisher to accept the validated presealed candidate and explicitly account its fixed `.candidates` receipts in current queue bounds. Retain original revision/source/body/execution identities, save-overlay row versions and attempted admission in existing admin_job metadata. Next writer checks unresolved candidate fences under the advisory lock; restart reads the exact retained DB candidate/decision and only looks up attempted operations; an unattempted retained decision can be delivered once per§5.4. Nested feed/transfer/credential writers reuse the proven context, never reacquire their own guard. Keep exact saved-row ownership/`previous_applied_not_proven` behavior; no automatic rollback. Permit an oversized legacy state only through a complete valid size-reducing candidate. Wire a real two-connection PostgreSQL lock/CAS test without adding or editing a migration. +- [ ] **Produce finite read/commit participation:** Export protocol§5.1 `withAdministrativeConfigurationRead`, `withConfigurationCommitParticipant`, `AdministrativeConfigurationReadContext`, `ConfigurationWriteTransaction` and closed nominal `ConfigurationCommitParticipant` now. Read-only contexts/reload acquire no write guard.04/05 later install only the named binding/migration/import/transit adapters; preview participants cannot persist, apply participants authorize/verify before preparation and verify/commit once inside the candidate transaction. Managed feed writes/proof consumption/ordinary overlay/job receipt are atomic on this exact connection; reject a caller purpose/boolean as authorization. Add real rollback-after-proof and nested-context/lock-order tests when03/04/05 supply those adapters, and producer tests now for foreign/expired/conflicting context and preview rejection. No new generic callback registry. +- [ ] **Implement preparation modes:** Split strict candidate generation publication from conditioned hardlink follow-up. Bulk Start prepares once; single Start, Apply and Update select ensure-running.09.6 selects require-running. Preflight the prospective plan through read-only `configuration.hardlinks.preview` after validation and before commit: nonempty require-running needs proven selected/running/ready coordinator; empty plan needs none. Recheck revision/mode at effect boundary. Coordinator exit after commit preserves committed/follow-up-failed or unknown and invokes zero lifecycle fallback. Checkpoint mode and exact candidate/preview/child identities before admission. `serviceStart`, `serviceApply` and bulk Start use `executeAdminJobOperation`, not the exit-code-only Docker wrapper. Adapt `applyTrustedConfiguration` to consume a candidate or prepare under this guard; its optional pre-observed revision uses an own-property check, and every authoritative path is strict. +- [ ] **Implement durable hardlink child:** Add API-role async `configuration.hardlinks.apply` (10-minute execution maximum, clamped to parent) with exact revision and coordinatorMode. Ensure-running's coordinator Start is a separate revision-checked settled child before host writer acquisition. Apply adopts the maintenance root when present (otherwise acquires its root), then holds target→writer and delegates only the fixed durable data-manager `generation-hardlinks` leaf from02.3/protocol§4. Persist intent before POST; lost response/cancel observes that child and retains the fence. Data-manager validates the confined plan/revision immediately before prune/publication and never starts a container or reacquires host writer. Browser cannot supply plan/path. Keep read-only inspect for diagnostics. +- [ ] **Remote maintenance foundation (R2-02):** Implement protocol§5.4 in the actual candidate persister and agent assembly. After whole-candidate/schema/source/size/queue/private-receipt preflight, checkpoint API job/candidate/prepare binding, admit ownership-only `configurationMaintenance.prepare`, and await its terminal prepared receipt. Agent acquires the full frozen closure then02.3's nonblocking durable DM fence; no target config/data/image/proof mutation occurs here. Under the prepared fence recheck pin/source/map/config/source-witness/proofs, then commit all desired rows/proofs/invalidation and the exact continue decision in the ONE writer transaction. Known rollback alone permits exact unused abandon; unknown DB/POST remains fenced and lookup-only. Deliver the retained decision after checkpointing its attempt; existing generation/hardlink/recreate operations borrow that root. Keep fixed plan/body/deadline identities, no new root, no target expansion and no reconsumed proof. Use the fixed private agent store, existing admin_job.result and02.3's jobs guard/mirror; no API data/private-key mount, new table or migration. Terminal preparations are excluded from live drain; DM starts fenced before cron/data admission and no DM maintenance worker must survive recreation. Settle only exact commit/containment plus runtime mount/readiness/all-child evidence, retaining unknown.08.3 uses this same guard and finite helper subset. +- [ ] **Implement update and exact receipt:** Install real `runtime["service.update"]`; fixed CLI argv remains `services update --expected-configuration-revision ` with optional no-deps. Identity/grant uses only fixed internal environment from02/06, never argv/output. API strict candidate publication is the checkpointed seed, not successful service application; no hardlinks yet. Agent initializes `ServiceLifecycleReceipt` under fixed `dataRoot/ops-agent/lifecycle/.json` from that seed and immutable execution, then delegates to the fixed CLI. CLI pins→successful pull→final preparation/hardlinks→recreate checkpoints exact phases/commit/follow-up/revisions under writer+lease ownership. `renderComposeWithWriter(opts:RenderRepoOptions,writer:ConfigurationGenerationWriter):Promise` avoids reacquiring. `FixedCliRunner` stays void; agent validates the exact bounded signed receipt after exit and returns its final revision only for complete. Seed/final commit followed by failure stays typed failure evidence; lost receipt is unknown. Never read a competing post-exit current pointer as this update's result. Bulk carries this returned final revision and stops on a newer external generation. +- [ ] **Pin/race rules:** Preserve an existing validated release overlay; resolve a complete release only when absent; never clear/reselect a pin implicitly. Hold the existing release-store lock while validating the selected overlay and invoking recreation, with lock order shared target lease → nonblocking transit single-flight reservation (when needed) → configuration writer → release store consistently across these paths. Release resolution/pull failures always stop recreation, including local/unpinned images: replace the two current CLI tests expecting failed-pull continuation, as the binding spec requires. No cached-image success fallback. +- [ ] **GREEN command:** Run the RED command plus `pnpm exec vitest run --project node apps/api/src/services/__tests__/service-preparation.test.ts apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts apps/ops-agent/src/configuration-writer-concurrency.test.ts`. Assert `noDeps` only changes recreation argv and persisted full selection is never narrowed to the update targets. +- [ ] **Causal acceptance extension:** Add and run focused strict resolver/candidate/publisher/receipt tests and the registered disposable PostgreSQL writer test. Boundary tests assert524288 accepted/524289 rejected only after full envelope serialization, failed query produces zero effects, all changed+unchanged maps count and reduced legacy candidate is valid. Real default runner/CLI entry test commits a competing generation after CLI exit before receipt read: returned revision is the CLI's own, next target conflicts. Include failure/crash after seed/final, active borrowed child cancellation and both coordinator modes with stopped/racing/empty-plan fixtures. API→production render-only probe asserts zero Start/restart/recreate calls across helper service too. Commands/receipts are recorded by the implementer; this plan correction executes none. +- [ ] **Remote-maintenance acceptance:** Add real API→agent→DM plus disposable PostgreSQL probes for ownership-only prepare, zero effects/proof consumption before decision, lost prepare/decision/settle POST, API death before/after save, DM recreation/startup admissions, all-child containment and retained unknown. Assert one root/decision, no live-parent drain wait, no nested target acquire and no release on missing receipts.05.6/04.6 add their actual proof/activation race consumers after those modules land;08.3 adds helper/DB outage cases. These are planned tests, not current results. +- [ ] **Review/commit:** Cover pin change, generation change and abort after pull before recreate; retain last-known effect evidence. Commit `fix(admin): wire prepared service updates and bulk start` after controller checks the new internal CLI flags cannot bypass CAS. + +### Task 3: Preserve batch failures across journal, API and UI — S03 + +**Files:** Create `apps/api/src/services/service-batch-result.ts` and `apps/api/src/services/__tests__/service-batch-result.test.ts`. Modify `apps/ops-agent/src/administrative-runtime.ts`, `apps/api/src/services/admin-job-handlers.ts`, `packages/cli/src/lib/service-builds.ts`; tests: `apps/ops-agent/src/administrative-runtime.test.ts`, `apps/api/src/services/__tests__/admin-job-handlers.test.ts`, `packages/cli/__tests__/cli-service-builds.test.ts`. 02 owns generic server/journal/client failure transport and job finalization. + +**Interfaces:** `ServiceBatchResult` is controller-exported from new `packages/core/src/ops/service-results.ts` (API helper imports it), with shape `{ outcome: "success" | "partial" | "failed"; completedServiceIds: string[]; failedServiceIds: string[]; skippedServiceIds: string[] }`, each list unique, disjoint and their combined membership bounded by the admitted target count, at most 256. Preserve canonical build order and raise the build-all success/failure result budget to 64 KiB; 256 maximum-length service IDs do not fit the current 16 KiB budget. `finishServiceBatch(ctx: JobContext, result: ServiceBatchResult): Promise` checkpoints `serviceBatch`; returns only success, otherwise throws 02's `AdminJobFailure` with the retained result. Ops build-all failures use 02's `OpsExecutionFailure` and `failureResult.serviceBatch`. + +- [ ] **RED test:** `continued build failure is terminal failure with retained partial result`: A succeeds, B fails, C succeeds; assert `status === "failed"`, outcome partial, exact lists, no success toast/SSE end-as-success. Add fail-fast A/B/C, all-failed, empty-plan and cancellation between B/C cases. Empty plan is failed with no effect, not “completed”. +- [ ] **RED command:** `pnpm exec vitest run --project node apps/api/src/services/__tests__/admin-job-handlers.test.ts apps/ops-agent/src/administrative-runtime.test.ts packages/cli/__tests__/cli-service-builds.test.ts`. +- [ ] **Implement aggregation:** Preserve manifest build ordering; accumulate receipts after every item before continuing. Default build-all continuation remains true, explicit selected-build continuation remains false. Unknown outcome/abort never counts as an ordinary item failure that permits the next mutation. Preserve completed IDs on fail-fast instead of throwing an unstructured first error. +- [ ] **Implement propagation:** Any failed item yields typed failed/partial evidence and a failed operation/job. The agent must not journal build-all as `succeeded` with nonempty failures. For API loops, persist results in the checkpoint before throwing; failure transport must retain them after restart. Build consumers retain existing stopped-consumer/provenance checks and pass `motisImport` unchanged for Plan 09. +- [ ] **GREEN command:** Run RED plus `pnpm exec vitest run --project node apps/api/src/services/__tests__/service-batch-result.test.ts apps/api/src/services/__tests__/job-runner-outcomes.test.ts` (the latter is created by 02 Task 2). +- [ ] **Review/commit:** Verify failed result serialization via generated SDK and Activity consumer test added in Task 6. Commit `fix(admin): retain partial service batch failures`; do not introduce a new database status just to display partial work. + +### Task 4: Safe core API/edge controls — S06 + +**Files:** Create `apps/ops-agent/src/core-service-runtime.ts`, `apps/ops-agent/src/core-service-runtime.test.ts`, `apps/api/src/services/core-service-maintenance.ts` and its `__tests__/core-service-maintenance.test.ts`. Modify `apps/ops-agent/src/{production-runtime,policy}.ts`, `apps/api/src/{server.ts,services/job-runner.ts,routes/admin-services.ts}`; reuse `apps/api/src/services/{system-maintenance,system-update-state}.ts` and current `appApi.replace`/agent handoff tests. + +**Interfaces:** New API-role asynchronous operation `coreService.restart = { kind: "coreService.restart"; serviceId: "app-api" | "traefik"; expectedConfigurationRevisionId: string }`, result `{ changed: true; serviceId: "app-api" | "traefik" }`, lifecycle timeout from 02. `runCoreServiceMaintenance(ctx: JobContext): Promise>` uses deterministic agent admission and retained status. Existing `runAdminServiceAction` with `{ action: "restart" }` dispatches eligible app-api/traefik decisions into job type `service.core-restart`; `server.ts` registers its handler and 02 adds this type to restart recovery. The regular action remains a direct generic restart for eligible non-core IDs. Generic core updates remain unavailable; capabilities direct users to the checked System Update workflow. + +- [ ] **RED test:** In real assembly, `core API restart survives loss of API observer`: accept dedicated job, checkpoint identity, drop observer before Docker effect, restart API runner and recover the same agent result without a second restart. Repeat for traefik and retained failure. Generic `service.stop/restart/update` on core IDs remains denied; core logs succeed. +- [ ] **RED command:** `pnpm exec vitest run --project node apps/ops-agent/src/core-service-runtime.test.ts apps/api/src/services/__tests__/core-service-maintenance.test.ts apps/api/src/services/__tests__/job-runner-initialize.test.ts`. +- [ ] **Implement core workflow:** Agent executes a fixed restart against verified container IDs, independent of the API/browser request lifecycle. Check current generation and current selection. Never offer web Stop for API/edge/agent: these remove the recovery channel. Protect all platform dependencies that would destroy the coordinating API's DB/queue availability (notably postgis/redis) from generic restart/stop/update too; return host guidance unless an existing dedicated checked workflow covers them. Task 1's policy uses a fixed trusted core-policy table, not just the current three-ID denylist. A service may still expose read-only logs while all lifecycle decisions are host-only. Never use this operation for image replacement or pin changes. +- [ ] **Reuse replacement:** API Update routes to existing `system.update` release validation/migrations/`appApi.replace` and pre-restart checkpoint. Agent Update routes to existing external `agent.update` helper/handoff only, with busy/checked-release guards. Keep agent ordinary start/restart/apply/remove disabled. Edge configuration apply belongs to Plan 08's guarded generation workflow, not generic lifecycle. +- [ ] **GREEN command:** Run RED plus `pnpm exec vitest run --project node apps/ops-agent/src/agent-update.test.ts apps/ops-agent/src/agent-handoff.test.ts apps/api/src/services/__tests__/system-maintenance.test.ts apps/ops-agent/src/production-runtime.test.ts`. +- [ ] **Review/commit:** Confirm every available workflow capability has a real route/handler; the dedicated job is in restart-safe observation recovery. Commit `feat(admin): add durable core service restart controls`; retain host instructions for whole-stack shutdown and volume destruction. + +### Task 5: Truthful inventory and stable deployment identity — S07, S08 + +**Files:** Create `apps/api/src/services/service-inventory.ts` and `__tests__/service-inventory.test.ts`. Modify `apps/api/src/utils/docker-compose.ts`, `apps/api/src/routes/{admin-services,admin-overview,admin-system}.ts`, `apps/api/src/services/dawarich/provisioning.ts`, `apps/web/src/lib/admin/serverDeployment.ts`, `apps/web/src/lib/admin/serverDeployment.test.tsx`, `apps/web/src/app/admin/{layout.tsx,layout.test.tsx}`. Controller owns contract extensions. + +**Interfaces:** `readServiceInventory(): Promise<{ acquisition: { state: "available" | "unavailable"; reason: "ops_unavailable" | null }; services: AdminServiceSummary[]; summary: { running: number; stopped: number; unknown: number; total: number } }`. Deployment `selfHosted` means configured instance intent, derived from `OPS_AGENT_URL`/token-file configuration or applied-services evidence, never a live Docker probe. `hostControlConfigured` means agent transport configuration, not API access to the host checkout. Deployment metadata is produced in `apps/api/src/routes/admin-services.ts:getAdminDeployment`; `admin-system.ts` also projects maintenance readiness. Update both together with overview and preserve Plan 08's configuration ownership fields. Unauthenticated/forbidden SSR reads still cannot grant navigation authority: role comes from the separately authenticated session. + +- [ ] **RED test:** Fail the agent read with known services present: all statuses unknown, unknown count equals total, stopped count zero, all effect capabilities unavailable. A successful empty container inventory instead marks known services not-running. An unfamiliar Docker state remains unknown. Check detail, inventory, health summary and Dawarich provisioning together. +- [ ] **RED command:** `pnpm exec vitest run --project node apps/api/src/routes/__tests__/admin-services.test.ts apps/api/src/services/__tests__/service-inventory.test.ts apps/api/src/routes/__tests__/admin-dawarich.test.ts`. +- [ ] **Implement acquisition:** Remove `dockerComposePs`'s `catch => []`; use explicit unavailable evidence in inventory consumers. Operational callers fail closed; Dawarich never treats an unavailable observation as permission to provision/change a running deployment. Overview distinguishes unknown from stopped. API list/detail serialize acquisition and all action decisions through canonical response schemas. +- [ ] **Implement identity:** Remove host-path probing as deployment authority and expose no new paths. Preserve compatibility fields pending Plan 08 contract cleanup, but never treat them as prerequisites for agent control. Change server deployment acquisition to `boolean | null`; null means unknown, not hosted. Full admins retain Services/Maintenance navigation for selfHosted true or unknown; explicitly hosted false still hides them. Privacy-admin navigation remains restricted. +- [ ] **GREEN command:** Run RED plus `pnpm exec vitest run --project web apps/web/src/lib/admin/serverDeployment.test.tsx apps/web/src/app/admin/layout.test.tsx`; add real serialization coverage in `apps/api/src/routes/__tests__/admin-services-sdk.test.ts` after controller generation. +- [ ] **Review/commit:** Verify first-load outage and recovered deployment classification; no cross-request cookie client caching. Commit `fix(admin): preserve unknown service state and troubleshooting access`. + +### Task 6: Capability-aware Services and outcome presentation — S01–S08 acceptance + +**Files:** Modify `apps/web/src/components/admin/services/{ServiceCatalog,ServiceDetail,ServiceLogsDrawer}.tsx`, `apps/web/src/components/admin/{AdminLayout,AdminSidebar}.tsx`, `apps/web/src/components/admin/shared/ServiceStatusChip.tsx`, `apps/web/src/components/admin/activity/JobDetail.tsx`, `apps/web/src/lib/admin/serviceAdmin.ts`, `apps/web/src/lib/admin/serviceAdminJobs.tsx`, `apps/web/src/lib/admin/adminJobEffects.tsx`. Tests: existing `ServiceAdministration.test.tsx`, `ServiceDetail.test.tsx`, `ServiceWorkflows.test.tsx`, `activity/__tests__/ActivityConsumers.test.tsx`; create `apps/web/src/components/admin/AdminSidebar.test.tsx`. + +**Interfaces:** Consume generated decisions/acquisition and `result.serviceBatch`; presentation derives “Partially completed” from failed + partial evidence, never from success. Use existing origin/service/job-owned mutation and observation providers. Plan 09 adds preset/noDeps/motisImport/pull/log form controls after this foundation. + +- [ ] **RED test:** Render enabled, deselected-running, core and unknown services. Assert disabled buttons with reasons, accessible owned logs, selected bulk action disabled when any target is ineligible (list the denied IDs), and no admission toast claiming application. Failed partial job displays completed/failed/skipped IDs; acceptance registers one observer and terminal invalidation. +- [ ] **RED command:** `pnpm exec vitest run --project web apps/web/src/components/admin/services/ServiceAdministration.test.tsx apps/web/src/components/admin/services/ServiceDetail.test.tsx apps/web/src/components/admin/activity/__tests__/ActivityConsumers.test.tsx apps/web/src/components/admin/AdminSidebar.test.tsx`. +- [ ] **Implement controls:** Derive all per-service/bulk actions from server decisions, not `enabled`/running booleans alone. Dedicated core Restart queues its guarded workflow; Update opens checked System Maintenance. Preserve selection/config/credentials tabs and display unknown acquisition with Retry. Read logs capability before opening a drawer; routes still enforce it. +- [ ] **GREEN command:** Run RED plus `pnpm exec vitest run --project web apps/web/src/components/admin/services/ServiceWorkflows.test.tsx apps/web/src/lib/admin/serviceAdmin.test.tsx apps/web/src/lib/admin/serviceAdminJobs.test.tsx` and node `apps/api/src/routes/__tests__/admin-services-sdk.test.ts`. +- [ ] **Rendered verification:** Controller starts the actual app with synthetic service fixtures and inspects desktop/mobile enabled, deselected-running, failed/partial, unavailable and restored states. Reconnect after dedicated core restart; verify observer recovery and persistent navigation. Record receipts locally; no PR screenshots. +- [ ] **Review/commit:** Fresh spec/quality review checks all S01–S08 receipts, real runtime assembly and generated serialization. Commit `feat(admin): expose service capabilities and reliable outcomes`. + +## Controller verification and self-review + +- Generate only under controller ownership: `pnpm openapi:generate`, `pnpm sdk:generate`; then `pnpm check-openapi`, `pnpm check-api-sdk`, `pnpm check-ops-authority`. +- Affected types: `pnpm --filter @openmapx/ops-agent check-types`, `pnpm --filter @openmapx/api check-types`, `pnpm --filter @openmapx/cli check-types`, `pnpm --filter @openmapx/web check-types`. Master owns builds, full suite, disposable Docker/database acceptance and CI. +- Self-review mapping: S01→2/6; S02→2; S03→3/6; S04→1/6; S05→1/6; S06→4/6; S07→5/6; S08→5/6. Every Review Focus item has a causal test. Shared files and contract generation are explicitly serialized. +- Controller ruling confirmed: enforce the spec's failed-pull prohibition for local/unpinned CLI updates too (existing tests deliberately permit it); document this changed CLI behavior in the PR. Local/third-party services may still Start from existing images, while Update cannot claim success after a failed pull. Core Stop remains host-only; safe Restart and release replacement are delivered, not deferred. +- Planning receipt: source and contracts inspected; plan self-reviewed. No tests, generators, dependency installs, builds, product edits or commits were run by the plan author. Plan 02 supplies the named failure/reconciliation interfaces; Plan 09 must copy the exact log defaults/ranges above. Review both plans together before implementation, with a fresh controller-appointed reviewer after root self-review. diff --git a/docs/superpowers/plans/2026-10-23-admin-02-durable-jobs.md b/docs/superpowers/plans/2026-10-23-admin-02-durable-jobs.md new file mode 100644 index 000000000..92734007c --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-02-durable-jobs.md @@ -0,0 +1,176 @@ +# Durable Administrative Jobs Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement J01–J04: live bounded diagnostics, operation-specific multi-hour deadlines, durable cancellation/reconciliation, and Full Update completion only after its transit child and dependent builds settle. + +**Architecture:** Keep the ops journal authoritative for host effects and PostgreSQL authoritative for data-manager work. Process output retention, observer lifetimes, backend cancellation and execution deadlines have distinct owners. Checkpoint each admission identity before effects, observe existing operations after disconnect/restart, and preserve failed/partial/unknown evidence rather than rerunning mutations. + +**Tech Stack:** TypeScript, Node child processes/POSIX process groups, Fastify, Zod, PostgreSQL/Drizzle, Commander CLI, generated ops/data-manager/admin SDKs, Vitest. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md` and binding `docs/superpowers/specs/2026-10-23-admin-control-protocols.md`; master: `docs/superpowers/plans/2026-10-23-admin-instance-management.md`. Audit base: `ef896ceae876959180116cf61ffcf643b88ebffe`. + +## Global Constraints + +- All product work stays in this worktree; preserve unrelated work and existing public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source policy boundaries. Community integrations remain declarative; no uploaded code execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain behind typed ops-agent operations. No generic command, arbitrary path or raw environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth declarations, strict schemas, audit events, explicit mutation identities and appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, durable operation identities, retained results, unknown outcomes and cancellation fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database and runtime checks, independent review, production builds and PR CI. Report any unavailable environment gate distinctly; do not replace runtime evidence with mocked assertions. + +## Review Focus + +1. A noisy process, one enormous unterminated line or split UTF-8/secret must not exhaust memory, leak a credential or kill a useful build (Task 1). +2. Lost admission/result/checkpoint responses and API/agent restarts must retain the original identity and unknown outcome, without a second mutation (Task 2). +3. Cancel before admission, during backend execution, after publication and during restart must settle from durable evidence; an observer disappearing proves none of these (Tasks 2–3). +4. Transit partial/error/interrupted/cancel-pending or a changed catalog/source revision must prevent dependent graph builds and parent success (Task 4). +5. Old handler completion racing cancellation/reconciliation cannot overwrite retained results or turn an unresolved cancel into canceled/success (Tasks 2–4). + +--- + +## Source map, ownership and shared contracts + +Current source findings (inspection, not executed tests): + +- `apps/ops-agent/src/administrative-runtime.ts:createDefaultFixedCli` invokes buffered `runContainedProcess`, emits output only after exit, and uses 1 MiB/30 minutes for every CLI call. `docker-runtime.ts:runContainedProcess` terminates the group when that byte limit is exceeded. Its log-follow runner is for disposable observers, not durable builds. +- `packages/core/src/ops/contract.ts:OPS_KIND_POLICIES` caps builds/data jobs at 30 minutes; `apps/api/src/services/ops-client.ts` independently clamps waits to 30 minutes + 5 seconds. The server already keeps cancellation pending until runtime settlement; preserve that containment boundary. +- The server retains only 64 events/32 KiB in memory; the journal retains terminal results but no output tail. The API projection stops after 2000 events/1 MiB. Neither limit is an execution deadline or backend success signal. +- `admin-job-ops.ts` checkpoints an async operation only after submission. `job-runner.ts` marks any fulfilled handler successful and any aborted handler canceled, omits single service jobs from restart recovery, and reconciles pending cancellation only during startup/cancel requests. +- `packages/cli/src/commands/data.ts:runTransitSync` returns a 202 child ID; Full Update proceeds to fonts/builds without observing the child. `services/data-manager/src/api.ts:launchReservedTransitSync` runs independently, has no cancellation controller, and `pipeline.ts` can break on abort and aggregate prior successful stages as `ok`. Restart reconciliation changes running jobs to interrupted, not proof that an external child process or candidate publication was contained. + +Controller owns edits to `packages/core/src/ops/{contract,index,client}.ts`, new `job-evidence.ts`, `packages/core/src/api/contracts/{activity-admin,service-admin/services,service-admin/data,service-admin/index}.ts`, `packages/core/src/services/data-manager/contracts/{operations,transit,index}.ts`, new `contracts/executions.ts`, server-only export maps, migrations, `scripts/ops-authority-inventory.json`, `scripts/ops-authority-inventory.test.ts`, `scripts/database-test-groups.json`, registration indexes and every generated file. The actual inventory filename was verified; its gate is `scripts/check-ops-authority.ts`. + +Follow the master's binding exact execution order: 02.1, 02.2, 06.1, 02.3, then 01.1–01.5, 02.4 and 01.6. 06.1 supplies the shared filesystem lease foundation. Each task has an exclusive integrated canonical-file/review window; no product edits overlap. 02 Task 4 consumes 01 Task 2 preparation. Shared `administrative-runtime.ts`, `docker-runtime.ts`, `server.ts`, `journal.ts`, API `job-runner.ts`, `admin-job-handlers.ts`, `admin-ops.ts`, CLI `commands/data.ts`, and data-manager `api.ts` are integrated sequentially by the controller. Plan 05 owns catalog/source/Overture product policy; it consumes these durable execution wrappers. Plan 06 owns `withArtifactTargetLease` in `packages/core/src/build-artifacts/target-lease.ts`; its shared filesystem authority must precede all managed Start/build/recreate/link/clean/promotion writes. Lock order: target lease → nonblocking transit single-flight reservation (when needed) → configuration writer → release store. Never replace it with a process-only mutex. If the transit reservation is busy, release the target lease and return busy; never wait for new transit admission while holding the target. Parent Full Update holds no target lease while awaiting transit; that child acquires its own. + +Plan 09 owns public preset/noDeps/MOTIS-import/log controls. Preserve its fields in identities and receipts: canonical expanded `serviceIds`, optional `presetIds`, update `noDeps`, build `motisImport`. Public logs are `{ lines: 1..1000 = 200, follow: boolean = true, maxDurationSeconds: 1..900 = 900 }`; only the adapter maps `lines` to internal `tail`. Closing an interactive `service.logs.follow` cancels that disposable reader; closing Activity never cancels a durable job. + +### Exact interfaces supplied to other plans + +Controller exports from `packages/core/src/ops/job-evidence.ts`: + +```ts +// 02.1 creates portable ServiceBatchResult in ops/service-results.ts;01.3 owns aggregation. +type ProcessDiagnostics = { + tail: Array<{ stream: "stdout" | "stderr"; message: string }>; + truncated: boolean; +}; +type DataWorkflowReceipt = { + version: 1; + phase: "osm" | "transit" | "fonts" | "build" | "link" | "complete"; + children: Array<{ + step: string; + operationKey: string; + jobId: string; + fingerprint: string; + deadlineAt: number; + status: string; + }>; + completedBuildIds: string[]; + failedBuildIds: string[]; + skippedBuildIds: string[]; + revisionId?: string; +}; +type OpsFailureResult = { + serviceBatch?: ServiceBatchResult; + dataWorkflow?: DataWorkflowReceipt; + diagnostics?: ProcessDiagnostics; + serviceLifecycle?: ServiceLifecycleFailureEvidence; +}; +``` + +Strict validation: at most 16 backend child receipts; step names from this plan's finite workflow enum; job IDs UUIDs; keys/fingerprints use existing ops validators; no host paths, raw request bodies, credentials, stack traces or unvalidated command output. The combined redacted diagnostics tail is ≤16 KiB, each message ≤4096 UTF-8 bytes, with a stable truncation marker. Failed evidence is ≤64 KiB, additionally constrained by the operation's result budget; build-all raises that budget to 64 KiB. Increase no global journal bound: retain its 256-entry/24 MiB limits and refuse admission before effects when capacity is unavailable. + +- Add independent `diagnostics?: ProcessDiagnostics` to status/journal records for retained tails in any state. Add `failureResult?: OpsFailureResult` only to failed/timed-out ops status/journal records, separate from successful `result`. Update strict parsers, per-kind client status/lookup/cancel parsing and persisted restart validation together. Old records lacking it remain readable. Never relax successful-result schemas to accept arbitrary failure objects. +- `OpsExecutionFailure extends Error` in `apps/ops-agent/src/runtime.ts`, constructor `(errorClass: OpsErrorClass, failureResult: OpsFailureResult)`. Server redacts/validates evidence before persistence and status publication; arbitrary exceptions get the existing fixed public error. +- `ApiOpsError` gains readonly `failureResult?: OpsFailureResult` without exposing raw exception text. `OpsObservationPending extends Error`, constructor `(identity: { kind: OpsOperationKind; operationKey: string; operationId?: string }, reason: "transport" | "observer_deadline" | "recovery_required")`, means no authoritative terminal result. Only observation is retried. +- `AdminJobFailure extends Error` in new `apps/api/src/services/admin-job-failure.ts`, constructor `(message: string, result: Record)`; runner merges retained checkpoint/result and emits the same result in failed SSE/detail. Plan 01's `finishServiceBatch` throws this. Do not add a database `partial` status for application jobs: failed + `result.serviceBatch.outcome === "partial"` is partial completion. +- `FixedCliRunner` remains `(args: readonly string[], options: FixedCliOptions): Promise`. Export `FixedCliOptions = { signal: AbortSignal; emitLog(stream: "stdout" | "stderr", message: string): void; operationKind?: OpsOperationKind; execution?: ExecutionRef; controlAssertion?: string; parentAssertion?: string }`. Managed lifecycle/build/data operations require `execution` from protocol§2; legacy short release-helper/scheduled callers may omit it only under their explicit existing policy. Assertions are private fixed environment, never argv/journal/output; clear inherited control variables for unrelated subprocesses.01.2 reads its exact `ServiceLifecycleReceipt`,02.4 its workflow receipt; void runner success carries no revision/child result. +- 02.1 creates `ExecutionRef`/normalization and body-versus-execution fingerprint split in core `ops/execution-identity.ts`, extends trusted execution context and version2 admission/journal/status contracts.02.1 declares the opaque `VerifiedParentExecution` type;06.1 later supplies its cryptographic verifier and nominal lease/authentication constructors;02.1 can test normalization with its server-internal verifier seam but must reject unverified wire parent context.02.3 enables the authenticated both-direction graph using06.1; no interim bare parent ID path. +- 01.2 supplies `ServiceLifecycleFailureEvidence` as the safe projection of protocol§6 receipt: `{expectedRevisionId,seedRevisionId,finalRevisionId,phase,commit,followUp,failure?}` with the same nullable cfg1 revisions/finite enums and≤256 unique failedServiceIds.02.1's generic failure schema reserves this exact closed shape in core `ops/service-results.ts` before01 consumes it; full execution/lease/private signature fields stay internal. Combined failure evidence remains≤64KiB. `DataWorkflowReceipt.children` additionally retains normalized deadline and execution fingerprint; public receipt≤16 children, private attempts use separate bounded records. + +### Task 1: Live bounded output and aligned execution deadlines — J01, J02 + +**Files:** Create `apps/ops-agent/src/process-output.ts`, `.test.ts`, `packages/core/src/ops/execution-identity.ts` and test; modify `docker-runtime.ts`, `administrative-runtime.ts`, `runtime.ts`, `server.ts`, `journal.ts`, core ops `contract.ts`, `client.ts`, `fingerprint.ts` and API `ops-client.ts`. Tests include these existing files plus API `__tests__/ops-client.test.ts`. Controller integrates strict version2 schemas/exports/policies in this exclusive task window. Do not change bounded JSON diagnostic subprocesses into lossy text parsers. + +**Interfaces:** Add `runContainedStreamingProcess(file: string, args: readonly string[], options: { signal: AbortSignal; timeout: number; killGraceMs?: number; onLine(stream: "stdout" | "stderr", message: string): void; redact: (message: string) => string }): Promise` in `docker-runtime.ts`. New `createBoundedProcessOutput(options: { onLine; redact; maxTailBytes?: number }): { write(stream, chunk: Uint8Array): void; end(): ProcessDiagnostics }` owns decoding/framing/tail, default 16 KiB. `onLine`/`redact` are explicitly typed as above. Nonzero exit/abort/deadline/containment failure throws `OpsExecutionFailure` carrying safe diagnostics; stdout volume alone never terminates a durable process. Keep `runContainedProcess`'s strict buffer failure for finite Docker inspect/JSON reads. + +- [ ] **Write causal RED:** Add a real Node child fixture to `administrative-runtime.test.ts` that emits `phase-one\n`, waits on a test-controlled pipe/file, emits >1 MiB including `last-diagnostic\n`, then exits. Invoke the existing default CLI runner with a temporary fixed `cli.mjs` next to the entry path. Assert `expect(seen).toContain("phase-one")` before releasing the child and `await expect(completion).resolves.toBeUndefined()`. Current runner fails pre-exit visibility and kills for volume; no missing new import is involved. Clean up/release the fixture in `finally` even on failure. +- [ ] **Run RED:** `pnpm exec vitest run --project node apps/ops-agent/src/administrative-runtime.test.ts apps/ops-agent/src/docker-runtime.test.ts -t 'live|output|contained|stream'`. Record behavioral assertion failures with valid fixtures, not timeouts caused by a blocked test handshake. +- [ ] **Implement framing/retention:** Decode each stream incrementally; frame LF and carriage-return progress, cap incomplete data at 4096 bytes, and drain remaining output after retention fills. Apply the existing agent/public diagnostic redaction policy before callbacks, persistence and errors; add bounded carry for sensitive patterns split across chunks, withholding ambiguous trailing credential text. Reuse `services/data-manager/src/utils/scrub-secrets.ts` behavior through a shared server-only helper rather than importing a service into core. Tests include split UTF-8, URL userinfo/query tokens, authorization/password assignments, ANSI/control characters, no newline and a credential crossing a frame boundary. The marker and final diagnostic survive overflow; no unbounded arrays/line strings or async log callback queue. +- [ ] **Implement real runner:** Use the existing `monitorPosixProcessGroup` for SIGTERM → 2-second grace → SIGKILL and closed-group proof. Callback failure detaches diagnostic projection and records truncation; it does not kill backend work. Spawn/containment failure remains failure. `createDefaultFixedCli` uses streaming; fixed Docker pull/lifecycle output used by administrative jobs also streams when no machine-readable output is needed. `createDefaultReleaseEffects` retains strict bounded collection for JSON/image/identity parsing; its pull path may stream without losing the existing fixed failure classification. +- [ ] **Align and retain budgets:** Implement protocol§2 `normalizeExecutionAdmission` and `ExecutionRef`; export `opsExecutionBudget(kind): {processMs:number;settlementMs:30000}`. Preserve exact12h build/24h buildAll/48h update/24h data budgets and later72h artifact(default)/7day ceiling/72h catalog validation/1h activation/24h advanced Overture/1h platform API-job policies. Diagnostics keep10min, logs≤900s; other existing kinds retain their deadlines. First admission clamps to requested parent deadline, **authenticated retained parent deadline**, and per-kind deployment budget from original admittedAt. Persist immutable admittedAt/deadlineAt and normalized execution fingerprint before effects; retain existing body fingerprint separately for sealed claims. Duplicate normalization uses original admittedAt, identical input observes, changed normalized deadline conflicts409. Agent context/CLI/nested runner use stored remaining time; restart never renews it. Version1 journal readers remain explicit compatibility readers, never infer parent authority. +- [ ] **Separate four lifetimes:** Short HTTP admission/read15s; independently chosen observerDeadlineAt; retained execution deadlineAt;30s containment/settlement after cancel/expiry. Remove API's30min clamp using the full policy/deployment maximum, not a48h ceiling. Observer loss throws pending and never cancels work. Execution expiry persists cancel intent and starts containment;30s expiry with live/unobservable child remains pending/unknown and retains locks. No fresh recovery restart/rollback phase may spend another execution budget after expiry. Fake-clock tests cross both old clamps, admit a late child near parent expiry, restart across deadline and attempt same-key extension; prove observation abort does not stop backend and true expiry does not fabricate containment. +- [ ] **GREEN:** Repeat RED, then `pnpm exec vitest run --project node apps/ops-agent/src/process-output.test.ts apps/ops-agent/src/server.test.ts apps/api/src/services/__tests__/ops-client.test.ts`. Prove output overflow does not signal/terminate the child, explicit abort/deadline still contains descendants, and diagnostics remain short. Controller records runtime platform gaps separately; no change to EPERM/process-group containment rules. +- [ ] **Review/checkpoint:** Fresh review of fixed executable selection, redaction, policy/runtime deadline agreement and surviving descendant behavior. Controller checkpoints `fix(ops): stream bounded diagnostics without ending jobs`. + +### Task 2: Retained failure evidence and restart-safe observation — J01, J03 foundation + +**Files:** Create `apps/api/src/services/admin-job-failure.ts`, `apps/api/src/services/__tests__/job-runner-outcomes.test.ts`, `apps/api/src/services/job-reconciliation.ts`, its `__tests__/job-reconciliation.test.ts`; modify agent `server.ts`, `journal.ts`, `runtime.ts`, API `ops-client.ts`, `admin-job-ops.ts`, `job-runner.ts`, `job-events.ts`, `activity-jobs.ts`, `server.ts`, `apps/web/src/components/admin/activity/JobDetail.tsx`. Tests: agent `server.test.ts`, `journal.test.ts`, API `__tests__/{admin-job-ops,job-runner-cancel,job-runner-initialize,ops-client}.test.ts`, routes `__tests__/admin-activity.test.ts`, web `activity/__tests__/ActivityConsumers.test.tsx`. Controller adds failure/observation contracts and serialized SDK tests. + +**Interfaces:** Extend the durable API projection to version 2: entries contain `kind`, `operationKey`, canonical non-secret typed `operation`, `operationId?: string`, phase `"admitting" | "observing" | "terminal"`, cursor/counters and `outcome?: "unknown"`. Parse v1 admitted entries compatibly; never invent a missing operation/identity. `reconcileAdminJobs(options?: { limit?: number }): Promise` handles at most 32 due rows per pass. `resumeAdminJobObservation(ctx: JobContext): Promise | undefined>` reuses original admission/operation/result checkpoints, not rebuilt desired state. Use existing `admin_job.result` for internal `_execution: { owner: string; version: number }` metadata: owner is a UUID, version a nonnegative integer. Each runner/reconciliation claim changes both atomically through PostgreSQL JSON-field CAS; `JobContext` carries them and checkpoints/finalizers match both plus admissible status. Checkpoint merges preserve this internal member, and serialized public results omit it. This requires no schema change in 02.2; no later migration is edited retroactively. Reconciliation uses these compare-and-set row claims to avoid duplicate local observers; no worker may publish a stale completion over a newer terminal/cancel-pending row. + +- [ ] **Write causal RED:** Existing `executeAdminJobOperation` test drops the first admission response after the agent accepted it, then simulates API restart; assert one execution, same key/fingerprint/ID, no rebuilt operation. Add fulfilled handler with `AdminJobFailure` evidence and assert failed DB result and SSE contain completed/failed IDs. Add abort without backend containment and assert it does not become canceled. Keep tests on existing imports until extracting helpers; new fixtures are not RED merely because the new classes do not exist. +- [ ] **Run RED:** `pnpm exec vitest run --project node apps/api/src/services/__tests__/admin-job-ops.test.ts apps/api/src/services/__tests__/job-runner-cancel.test.ts apps/api/src/services/__tests__/job-runner-initialize.test.ts apps/ops-agent/src/journal.test.ts`. +- [ ] **Implement write-ahead observation:** Persist the exact safe operation/key and `admitting` phase before first POST; then store operation ID before following output. On resumed `admitting`, use authenticated lookup only. A missing/expired/unavailable lookup after a prior attempt is unknown, not permission to resubmit. Fix configuration preparation the same way with its original revision condition (01 owns that adapter); never regenerate a fingerprint from newer state. Only a fresh checkpoint proving no admission was attempted can submit once. +- [ ] **Implement transport/evidence:** Wire `failureResult` end to end through status, lookup, cancel, journal reload and `ApiOpsError`; keep successful and failed variants strict. Persist a redacted diagnostic tail in the agent journal, coalesced at most once per second and flushed before terminal state (a crash may lose at most that last interval). Keep the current in-memory event ring; a restarted ring/cursor reports truncation and exposes retained diagnostics, not invented replay. API retained log projection remains ≤2000 events/1 MiB; on exhaustion persist marker/cursor and stop mirroring while result observation continues. Terminal detail supplies the independent tail. Logging/checkpoint projection failure is not backend cancellation. +- [ ] **Implement runner outcomes:** `AdminJobFailure` merges retained evidence on failed DB/SSE publication. `OpsObservationPending` keeps status running with `result.observation = { state: "unknown", reason, nextCheckAt }`, releases the local concurrency slot and schedules reconciliation, without emitting terminal/end/success. Reconcile at startup and every 5 seconds while listening; cap backoff at 60 seconds, preserve exact identities, and stop the scheduler during server shutdown. Transient transport may retry reads; unauthorized/invalid-identity reads remain unknown/unavailable with a fixed reason and no mutation retry. Agent `recovery_required` is explicit retained uncertainty requiring reconciliation; do not silently replay or declare that the underlying effect never happened. +- [ ] **Implement CAS/cancellation precedence:** Handler success/failure/checkpoint writes use row state/version ownership. Once cancel intent is durable, a late handler result is retained and reconciled, not blindly published over cancel-pending. `findActive` includes cancel-pending. Register single `service.start`, `.stop`, `.restart`, `.apply` and `compose.render` with explicit observation-only recovery;01.2 supplies their deterministic generation/mode/receipt checkpoints and09.6 later advertises compose.render. Add `service.core-restart` when01.4 registers it. A render receipt retains require-running, exact candidate/source/revision and hardlink child identity, committed/follow-up-failed/unknown; recovery never calls a fresh render or starts the coordinator. Do not replay arbitrary handlers. Retain exact-image/migration proof for system.update/appApi.replace.08.3's platform API DB job observes separate stack.render/ordinary child/configurationHandoff.prepare and inspect records, never a live agent platformConfig.apply ancestor; prepared handoff is terminal preparation, not completed configuration apply. +- [ ] **GREEN:** Repeat RED plus `pnpm exec vitest run --project node apps/api/src/services/__tests__/job-runner-outcomes.test.ts apps/api/src/services/__tests__/job-reconciliation.test.ts apps/ops-agent/src/server.test.ts apps/api/src/routes/__tests__/admin-activity.test.ts`; web `pnpm exec vitest run --project web apps/web/src/components/admin/activity/__tests__/ActivityConsumers.test.tsx`. Assert no terminal SSE for unknown/cancel-pending, no second mutation after API restart, and fresh status/SDK decoding of retained failure evidence after agent restart. +- [ ] **Review/checkpoint:** Authenticate route/status/event/cancel lookups for ordinary user/privacy-admin/wrong agent role; assert no backend call after 401/403 and no foreign job/origin result adoption. Controller checkpoints `fix(admin): reconcile durable jobs without replaying effects`. + +### Task 3: Durable data-manager execution and real cancellation — J03 + +**Files:** Create `services/data-manager/src/jobs/execution/{store,runtime,api,ops-children,configuration-maintenance}.ts` and corresponding tests; core `services/data-manager/executions.ts`, `.test.ts`; CLI `lib/data-execution.ts`, its test. Modify data-manager `api.ts`, `auth.ts`, `ops-client.ts`, `service.ts`, `cron.ts`, `jobs/reconcile.ts`, `jobs/persistence.ts`, `jobs/transitous/{runtime,single-flight,pipeline,types,persistence,promote,proxy-transaction,motis-import}.ts`. Own **all** promotion/stop/restart/proxy stage/commit/restore and API/cron one-off call sites, not only the top worker's AbortController. Wire existing cancellation consumers in `jobs/{atomic-download,download-osm,download-fonts,convert-overpass}.ts`, `jobs/overture/{sync,pull,ingest,extract-osm-pois,rebuild-links,duckdb}.ts`, `jobs/search-index/{build,extract}.ts`, `jobs/notable-places/build.ts` and transit process/script/publication boundaries. Modify API `admin-job-ops.ts`, `activity-jobs.ts`, `routes/admin-activity.ts`; coordinate `routes/data-manager.ts` with05. Controller integrates `packages/db-schema/src/data-manager.ts`, reserved migration `apps/api/src/db/migrations/0042_admin_data_execution.sql`, Drizzle metadata, ingestion test registration and data-manager/API contracts here.0042 includes immutable deadlines/parent binding/child attempt JSONB and fencing, plus§5.4's versioned `data_manager.jobs.metadata.configurationMaintenance` guard schema/unique identity/partial singleton index; no later task edits this applied migration. Other reservations and exclusive task windows stay unchanged. + +**Early closed exports (R2-01/R2-02):** This task creates core `ops/catalog-pin-approval.ts` with§4.1's `PairedPinApprovalInput/Result`, adds `transitousLock.approve` to `DataEffectKind` and `paired-pin-commit` to `DataOpsChildPhase` with async120s policy. Ref-only approval is disabled now;05.3 later installs the validation-bound runtime (no forward import of05.7 or permissive interim handler). It also creates core `services/data-manager/contracts/configuration-maintenance.ts` and `services/data-manager/configuration-maintenance.ts`: exact§5.4 `DataConfigurationFencePrepare/Receipt/Settle`, `prepareDataConfigurationFence`, `inspectDataConfigurationFence`, `settleDataConfigurationFence`. These service-only operations are separate from public DataExecutionInput;01.2 consumes them without moving the master order. Controller integrates canonical registration, policy/inventory and migration metadata in this same task. + +**Interfaces:** Finite `DataExecutionInput` union: `download-osm {region}`, `download-fonts {}`, `convert-overpass {region}`, `transit-sync {countries: string[]}`, `overture-pull|overture-ingest|overture-extract|overture-conflate|overture-sync {region, restart?: boolean}` (restart only conflate), `search-index-build {region}`, `notable-places-build {}`; protocol§4 adds service-internal `generation-hardlinks {revisionId,planDigest,coordinatorMode}` and `transit-maintenance {action:"restart-primary"}` / `{action:"reload-feed-proxy",candidateId:string}` for existing API/cron actions. These internal variants are not exposed as arbitrary browser ops/plan input. No URL/path/argv fields. Existing country/region/pin/ownership policy remains mandatory. `DataExecutionStatus = { id: string; operationKey: string; fingerprint: string; admittedAt: number; deadlineAt: number; kind: DataExecutionInput["kind"]; status: "queued" | "running" | "cancel_pending" | "ok" | "partial" | "error" | "canceled" | "interrupted"; cancellationRequestedAt: string | null; progress: number | null; diagnostics: ProcessDiagnostics; publication: "not_started" | "committed" | "unknown"; result?: DataExecutionResult }`; fingerprint is normalized execution fingerprint. Result is a strict per-kind union, no host paths/credentials,≤32KiB. Internal parent/delegation/child-attempt fields never leak through this status. + +Export `DataExecutionMetrics = { processed?: number; inserted?: number; updated?: number; skipped?: number; sizeBytes?: number }` (safe nonnegative integers). Export `DataExecutionResult` as `{ kind: "download-osm"; region: string; sizeBytes: number; md5?: string } | { kind: "download-fonts"; completed: true } | { kind: "convert-overpass"; region: string; sizeBytes: number } | { kind: "transit-sync"; finalStatus: "ok" | "partial" | "error"; sourceRevision: string; countries: string[] } | { kind: "overture-pull" | "overture-ingest" | "overture-extract" | "overture-conflate" | "overture-sync"; region: string; release?: string; metrics: DataExecutionMetrics } | { kind: "search-index-build"; region: string; metrics: DataExecutionMetrics } | { kind: "notable-places-build"; metrics: DataExecutionMetrics }`. `DataExecutionStatus.result?: DataExecutionResult` is discriminated/validated against the admitted kind; preserve original richer results only inside each worker's existing trusted store, not in this public receipt. Progress/tail and status retain partial/error detail. Add exact internal result variants `{kind:"generation-hardlinks",hardlinks:HardlinkApplySummary}` and `{kind:"transit-maintenance",action:"restart-primary"|"reload-feed-proxy",changed:true}`; no arbitrary operation result union. + +Add generated service-auth operations `startDataExecution` (`POST /executions`, body `{operationKey,input,requestedDeadlineAt?}` plus fixed internal parent/delegation headers), `getDataExecution` (`GET /executions/:id`), `lookupDataExecution` (`GET /executions/by-key/:operationKey`) and `cancelDataExecution` (`POST /executions/:id/cancel`, body `{operationKey,fingerprint}`). Auth adapter verifies protocol§3 role certificate/retained parent bound alongside existing service bearer. Server computes normalized fingerprint/deadline; caller cannot choose fingerprint or assert a role.202 status includes immutable admittedAt/deadlineAt. Same key normalized with original admittedAt+same input/parent observes, changed normalized deadline conflicts409. Reads404/410 after attempted admission mean unknown and never permit resubmission. Static lookup path precedes `/:id`; public proxy/Activity cancel still requires full admin/audit/identity/rate limits. Add the separately registered service-auth `startGenerationHardlinksExecution` at `POST /executions/generation-hardlinks`, exact body/plan bounds and signed digest from protocol§4, before dynamic ID routes. Ordinary startDataExecution rejects generation-hardlinks; the private leaf requires its verified grant and durable plan. + +SDK helper `observeDataExecution(identity: {id:string;operationKey:string;fingerprint:string}, options: {transport:DataManagerTransport;signal?:AbortSignal;observerDeadlineAt:number;onProgress?:(status:DataExecutionStatus)=>void}): Promise` only GETs. Observer abort/deadline/lost response throws existing planned `DataExecutionObservationPending(identity,reason:"transport"|"observer_deadline"|"interrupted")`; never POSTs cancellation. `requestDataExecutionCancellation(identity,options:{transport:DataManagerTransport;signal?:AbortSignal}):Promise` is separate. CLI `executeDurableDataStep(input,options:{operationKey:string;transport:DataManagerTransport;signal?:AbortSignal;requestedDeadlineAt:number;parentAssertion?:string;observerDeadlineAt:number;checkpointAttempt:(attempt:DataChildAttempt)=>Promise;checkpoint:(status:DataExecutionStatus)=>Promise}):Promise` persists normalized input/parent/deadline/key before POST, then adopts only matching status. `DataChildAttempt` is protocol's bounded pre-admission identity, with optional accepted ID; no accepted-only callback can stand in for it. Consume existing `DataManagerTransport`; service-internal headers never originate in browser parameters. Export protocol§4's exact `DataExecutionContext`, `DataOpsChildAttempt`, `runOwnedDataOpsOperation`, lookup and cancellation/settlement helpers from the execution module. + +- [ ] **Write causal RED:** Extend existing `services/data-manager/__tests__/transitous/pipeline.test.ts` with a first successful stage, abort before next stage and assert the result is never `ok`. Exercise current `registerApi` with an accepted transit job: destroy the observer, assert backend still running; request actual cancellation through the new adapter after scaffolding, block worker settlement, assert cancel-pending and no final timestamp until it releases. Baseline no cancellation or early `ok` is the behavioral RED; absent routes alone do not qualify. +- [ ] **Run RED:** `pnpm exec vitest run --project node services/data-manager/__tests__/transitous/pipeline.test.ts services/data-manager/__tests__/job-persistence.test.ts apps/api/src/services/__tests__/job-runner-cancel.test.ts`. +- [ ] **Implement durable store:** Reuse data_manager.jobs/stages and single-flight locks.0042 adds nullable versioned execution identity/body+execution fingerprints/admittedAt/deadlineAt/parent binding/cancellation, integer ownership-version fence, bounded private children JSONB and unique retained execution key. Transactionally admit once; server normalization uses original admission time on duplicates. Store normalized safe input/progress/result plus protocol§4 child attempts (≤32,≤8KiB each,≤128KiB total), never assertions/private keys. Active/cancel-pending/unknown rows do not expire; terminal identities remain72h, expired attempted identities are unknown and cannot be recreated. Register real admission-key races, deadline/parent duplicate conflicts, stale-version CAS and child checkpoint races in store-postgres.test.ts under ingestion; apply0042 once before that suite. +- [ ] **Durable configuration fence before01.2:** Implement protocol§5.4's three fixed DM endpoints, bearer plus pinned ops-agent grant/witness validation, same-job/key/plan CAS and signed private startup mirror. Prepared guard uses an existing jobs row with terminal preparation status and separate guardState; it is not a running worker or target owner. Serialize guard transitions and every data mutation admission using the fixed nonblocking DB advisory transaction key `(0x4f4d58,2)`; check/insert on the SAME connection/transaction. Include single-flight, source mutation, legacy API/cron/CLI, offline/traffic and other existing data writers, not only the new /executions endpoint. Startup reads guard/mirror/owner before cron or mutations; DB/mirror uncertainty stays closed. Only signed exact generation-hardlinks continuation may write while fenced; markSyncFinished/expiry/reconcile cannot clear it. Reserve the schema/index in0042 now; later01/05/08 never edit an applied migration. Add real store/admission/startup tests here;01.2 adds remote candidate orchestration. +- [ ] **Implement runtime:** `createDataExecutionRuntime(deps:{store:DataExecutionStore;dispatch(input:DataExecutionInput,context:DataExecutionContext):Promise;now?:()=>number}):DataExecutionRuntime`. `admit(input,{operationKey,requestedDeadlineAt?,parent?:VerifiedParentExecution}):Promise` computes identity; `inspect`, `lookup`, exact `requestCancel`, `reconcile`, `close` keep the signatures above. `DataExecutionContext` from protocol§4 owns execution/version, signal, target authority and checkpoint/progress/publication/child CAS. Store `checkpoint(id,expectedVersion,patch)` accepts only mutable status/progress/diagnostics/publication/result fields, never identity/deadline, and `checkpointChild(id,expectedVersion,attempt)` preserves immutable phase/key/fingerprint; stale CAS returns null and prohibits effects. Persist cancel intent before abort. Subprocess/DB/runner containment and pre-publication cancellation checks remain required at rename/schema/MOTIS/proxy/readiness boundaries. Already committed is committed; unobservable work is cancel-pending/unknown. +- [ ] **Own every nested host effect:** Implement `runOwnedDataOpsOperation` on actual services/data-manager/src/ops-client.ts transport and new ops-children store. Update stopContainer, stopBeforeSwap recovery, restartPrimary, motis-import staging restart, proxy validate/reload/commit/restore, API restartMotis and cron feed reload. Own the paired-pin-commit adapter as the same retained child path;05.3 wires activation after promotion to this exact operation/phase with the existing root continuously held. Raw approve callers are disabled until validation is installed. Persist phase/key/body+execution fingerprint/deadline/grant before POST; do not retain the random-key default for these effects. A lost response does lookup only, accepted ID is checkpointed before next effect. Explicit cancellation is separately sent with retained fingerprint then observed on a settlement signal independent of the aborted worker. No target release, proxy rollback or duplicate promotion while an earlier child is unknown. `restartPrimary`/proxy catches must propagate pending/unknown, not ordinary errors that authorize recovery. Replace recovery's fresh timeout with remaining retained execution budget; elapsed deadline cannot start new compensating work. One-off existing API/cron effects get a durable root and the same lease protocol, preserving their semantics without an anonymous mutation path. +- [ ] **Adapt current paths:** Service startup wires the real execution runtime before readiness/cron. Transit `launchReservedTransitSync`, source-triggered sync and cron register their children/controllers with the same owner; preserve deployment scope and proposal/active lock semantics. Legacy NDJSON endpoints become observers of these workers, preserving their current event shapes; their request close stops writes/reader ownership only. Do not attach backend abort to `reply.raw.close`. CLI admin data commands use generated durable admission/observation, avoiding an unowned long-running NDJSON request. Existing standalone `data sync`/`download gtfs` may remain admission-only, but print queued child ID and status instructions, not completed. Plan 05 adds the same cancel/status semantics to newly exposed advanced Overture operations. +- [ ] **Implement cancellation/restart reconciliation:** API cancellation enumerates exact agent projections and registered backend children. It persists application cancel-pending before sending requests and keeps reconciling across API restart; zero agent projections is not proof that no backend child exists. Only all contained/terminal receipts can finalize. Success that preceded cancel remains success; partial/error remains failure; unresolved work remains pending. DM startup reconciles running/cancel-pending work before accepting new jobs, inspecting its known child/process/publication evidence and retaining interrupted/unknown where containment is not proved. Never restart old builds automatically. Agent restart similarly retains `recovery_required`; use the Task 4 workflow store to discover child IDs even if the CLI disappeared. No generic “kill job ID” operation. +- [ ] **GREEN:** Repeat RED plus `pnpm exec vitest run --project node services/data-manager/src/jobs/execution/store.test.ts services/data-manager/src/jobs/execution/runtime.test.ts services/data-manager/src/jobs/execution/api.test.ts packages/core/src/services/data-manager/executions.test.ts packages/cli/__tests__/cli-data-execution.test.ts`. Test cancel during transfer, blocked DB commit, MOTIS promotion and cleanup; duplicate/foreign key, unauthorized service token, stale pin, lost cancel response, crash before/after durable intent and completed-before-cancel. Controller also runs migrated `pnpm test:database ingestion`; ordinary Vitest skips are not a database pass. +- [ ] **Separate-process child acceptance:** Add ops-children tests using authenticated loopback data-manager→actual assembled agent promotion/reload adapters and shared target mount; substitute only external Docker/download I/O. Drop admission and cancel responses, crash parent/agent around adoption/store CAS, hold child through local cancellation, and release before/after commit. Assert same retained identity, no second dispatch or premature lease release, truthful committed/cancel-pending/unknown publication and no hidden proxy recovery. Add fake-clock late child, restart across deadline and duplicate normalized-budget conflict; no multi-hour sleep. Durable generation-hardlinks also proves lost response reconciliation without repeat pruning and zero container lifecycle effects in require-running mode.02.3 tests the closed approval adapter/async lookup/cancel contract;05.3 adds actual proposal/promote/pointer-commit loss/crash acceptance. For the DM fence, race guard preparation against a real data admission on two DB connections; pause before insert, restart DM with a prepared or half-written guard, and verify all API/cron/CLI admissions remain blocked and a terminal preparation does not count as a live drain worker. +- [ ] **Review/checkpoint:** Fresh review follows every backend effect to its real cancellation/publication consumer. Update Activity data-manager active filters for cancel-pending, canceled terminal display and supported-only cancellation eligibility; existing unsupported historical jobs remain noncancelable. Controller checkpoints `feat(data): retain durable execution and cancellation ownership`. + +### Task 4: Full Update owns and awaits transit before builds — J04, J03 integration + +**Files:** Create `packages/cli/src/lib/data-update.ts`, `packages/cli/__tests__/cli-data-update.test.ts`, `packages/core/src/services/admin-data-workflow.ts` and its `__tests__/admin-data-workflow.test.ts`, `apps/ops-agent/src/data-workflow-runtime.ts`, `.test.ts`; modify CLI `commands/data.ts`, `lib/service-builds.ts`, agent `administrative-runtime.ts`, API `admin-job-handlers.ts`/`admin-job-ops.ts`; controller wires the server-only core export and operation evidence. Plan 01 owns the production assembler and shared Start/preparation. Extend its `production-runtime.test.ts` rather than inventing a second production assembly. + +**Interfaces:** `runDataUpdate(options:{region:string;countries:string[];failFast:boolean;execution:ExecutionRef;signal?:AbortSignal},effects:DataUpdateEffects):Promise`. `DataUpdateEffects` has `checkpoint(receipt):Promise`, `checkpointAttempt(attempt:DataChildAttempt):Promise`, `runDataStep(step:"osm"|"transit"|"fonts",input:DataExecutionInput):Promise`, `build(options:{region:string;continueOnError:boolean}):Promise`, `prepare(options:{coordinatorMode:"ensure-running"}):Promise` from01.2. Data step uses retained parent execution and signed deadline-only assertion, never an arbitrary caller fingerprint/deadline. Build retains canonical order/dependency/consumer/motisImport policy; Full Update does not implicitly enable MOTIS import. + +`openAdminDataWorkflow(options: { dataRoot: string; operationKey: string; fingerprint: string }): Promise<{ read(): DataWorkflowReceipt | null; checkpoint(receipt: DataWorkflowReceipt): Promise }>` is host-only in `admin-data-workflow.ts`, under fixed `dataRoot/ops-agent/workflows/.json`; filename/path is never an API input. Store immutable key/fingerprint in a versioned ≤64 KiB owner-only record using no-follow regular-file reads, exclusive creation, fsync/atomic rename/directory sync and shared cross-process writer ownership. Add admission-attempt states to each child receipt (`attempted` before POST, then returned ID); a write failure stops before the next effect. Retain unknown admitted children by key for read-only lookup, not a guessed child ID. The exact final `DataWorkflowReceipt` contains only accepted child IDs; the internal store also retains these pending identities. + +- [ ] **Write causal RED:** Add a Full Update command test through existing `registerDataCommands` with a valid OSM response, transit 202 job ID and a controlled running child. Assert `expect(buildServices).not.toHaveBeenCalled()` and `expect(exitSuccess).not.toHaveBeenCalled()` until terminal `ok`. Current command reaches builds immediately. Release the child as partial/error/interrupted and assert no dependent build/link and a non-success parent. Fresh scopes include a terminal child from another key/country fingerprint; it must not be adopted. +- [ ] **Run RED:** `pnpm exec vitest run --project node packages/cli/__tests__/cli-data-update.test.ts packages/cli/__tests__/cli-service-builds.test.ts apps/api/src/services/__tests__/admin-job-handlers.test.ts`. +- [ ] **Implement sequencing:** Extract the actual Full Update action into `runDataUpdate`; OSM → accepted-and-observed transit child → fonts → ordered graphs → generation/hardlinks → complete. Admission of transit is never completion. Only child `ok` permits graph work; partial/error/interrupted/canceled stops dependent builds regardless of `failFast` (that option controls independent build continuation, not prerequisite failure). Retain completed/failed/skipped build IDs and child receipts in every error. A continued graph failure produces a partial failed parent using 01's batch helper; it never prints/returns successful completion. +- [ ] **Implement identities/checkpoints:** Fixed runner supplies ExecutionRef and verified parent assertion only through protocol's fixed environment; remove provisional internal identity argv. CLI derives protocol§3.5 stable phase/ordinal keys, persists exact normalized input/requested deadline/parent binding before first POST and accepted child before proceeding. Standalone host CLI allocates and retains its own signed execution authority, server-clamped per kind, and prints only safe status/recovery identity. Agent reads the same workflow receipt for progress/failure/cancel/restart, never stdout. On API restart observe original agent key; on agent/CLI death reconcile recorded child intents including lost admission, mark unfinished host phases recovery-required and never rerun OSM/build/link. A late child ok does not prove remaining phases completed. +- [ ] **Implement publication guards:** Bind countries/catalog/overlay/source revisions at admission and recheck before graph publication. Consume exact protocol§3 nominal acquire/delegate/adopt/settle functions, role verifier/transport and bounded finite graph, not parentArtifactLeaseId. Full Update holds no target while awaiting transit; its signed deadline-only parent assertion authorizes the budget, while transit acquires its own root targets/nonblocking reservation. Build/link then acquires its own phase authority in target→nonblocking transit→host writer→release store order. The CLI may delegate only its allowed hardlink leaf under a narrower subset. No process-local lock, TTL stealing or host writer held through a reacquiring child. Cancel/lost observation cannot release before descendant settlement; retain unknown under the original deadline. +- [ ] **GREEN and assembled proof:** Repeat RED plus `pnpm exec vitest run --project node packages/core/src/services/__tests__/admin-data-workflow.test.ts apps/ops-agent/src/data-workflow-runtime.test.ts apps/ops-agent/src/production-runtime.test.ts`. Dispatch a real authenticated `data.operation/update` through the production administrative runtime and default CLI using the real source CLI entry/command registration (or controller-built bundle), a loopback authenticated data-manager execution API and controlled external Docker/download I/O; do not replace `runFixedCli` or the data/update handlers in this acceptance probe; hold transit running and verify no build argv, then release exact `ok` and verify builds and hardlinks follow. Test dropped Activity SSE, lost child admission/cancel responses, API restart, agent restart, stale generation, unauthorized cancellation and parent cancel between transit success/build. A mocked handler returning a child ID is insufficient runtime evidence. +- [ ] **Review/checkpoint:** Fresh reviewer checks phase order, durable store ownership, original-key lookup, child/parent cancellation evidence, source/revision/target leases and terminal serialization. Controller checkpoints `fix(data): await Full Update transit child before graph builds`. + +## Controller verification and self-review + +- Source checks and planned commands above are not execution receipts. This author ran no tests, generators, installs, builds, product edits or commits. +- Generate serially after canonical definitions settle: `pnpm data-manager:openapi:generate`, `pnpm data-manager:sdk:generate`, `pnpm openapi:generate`, `pnpm sdk:generate`; inspect serialization and generated diffs. Then `pnpm check-data-manager-openapi`, `pnpm check-data-manager-sdk`, `pnpm check-openapi`, `pnpm check-api-sdk`, `pnpm check-ops-authority`. +- Affected types: `pnpm --filter @openmapx/ops-agent check-types`, `pnpm --filter @openmapx/api check-types`, `pnpm --filter @openmapx/cli check-types`, `pnpm --filter @openmapx/data-manager check-types`, `pnpm --filter @openmapx/web check-types`. Master owns frozen installs, full suite/builds, disposable runtime/PostgreSQL acceptance, rendered desktop/mobile checks, whole-branch review and the one PR/CI. No screenshots in PR. +- Self-review coverage: J01→1/2 (live output and retained tail); J02→1 (every effective timeout layer); J03→2/3/4 (identities, durable intent, all backend owners, crash/unknown/CAS); J04→4 (actual parent/child production order). Every Review Focus condition is pinned to its owning task. New APIs use canonical service-auth/admin guards and generation is controller-serialized. +- Corrections required in neighboring plans: 01 consumes `OpsExecutionFailure`, `AdminJobFailure`, `OpsObservationPending`, required generation conditions and the shared target lease; 05's source/advanced Overture actions register with the durable runtime and pass catalog fingerprints/cancellation signals; 06 supplies cross-process leases before any caller lands; 09 states log default 200 explicitly and uses the exact fields/ranges above. Core restart's new recoverable type must be registered before it is advertised. Root self-review and a fresh reviewer precede product implementation. Migration reservation is exactly `apps/api/src/db/migrations/0042_admin_data_execution.sql`; target guard is exactly `build-artifacts/target-lease.ts`, in the final shared lock order recorded above. Migration reservation is exactly `apps/api/src/db/migrations/0042_admin_data_execution.sql`; target guard is exactly `build-artifacts/target-lease.ts`, in the final shared lock order recorded above. diff --git a/docs/superpowers/plans/2026-10-23-admin-03-account-policy.md b/docs/superpowers/plans/2026-10-23-admin-03-account-policy.md new file mode 100644 index 000000000..ead5ecf98 --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-03-account-policy.md @@ -0,0 +1,174 @@ +# Account Policy and Provisioning Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make account policy effective and provide safe, auditable account provisioning and recovery, covering A01–A10. + +**Architecture:** Resolve one immutable policy per auth operation and build Better Auth instances from that policy. A transaction-bound auth adapter and database invariant protect every remote account mutation; focused account services provide reauthentication and provisioning without enlarging the general admin route. + +**Tech Stack:** TypeScript, Better Auth 1.7.7, Fastify, Drizzle/PostgreSQL, Zod, React/MUI, Vitest. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md`; execution/controller gates: `docs/superpowers/plans/2026-10-23-admin-instance-management.md`. + +## Global Constraints + +- All product work stays in this worktree; preserve unrelated work and existing + public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source + policy boundaries. Community integrations remain declarative; no uploaded code + execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain + behind typed ops-agent operations. No generic command, arbitrary path or raw + environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth + declarations, strict schemas, audit events, explicit mutation identities and + appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never + hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, + durable operation identities, retained results, unknown outcomes and cancellation + fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, + place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation + and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful + admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots + to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database + and runtime checks, independent review, production builds and PR CI. Report any + unavailable environment gate distinctly; do not replace runtime evidence with + mocked assertions. + +Additional execution boundary: baseline `ef896ceae876959180116cf61ffcf643b88ebffe`; this document is planning only. No installation, generator, build, test execution, or commit occurred while writing it. Read root/web `AGENTS.md`, `CONTRIBUTING.md#quality-bar`, and `docs/docs/developer/api-surface.md`; read installed Next.js docs before implementation. + +## Review Focus + +1. A policy update races an in-flight login: each operation sees exactly one snapshot, including nested server API calls (Task 1). +2. Disabling enrollment with an existing TOTP account must retain the second-factor challenge, including recovery and reauthentication (Tasks 1, 3). +3. Two remaining admins concurrently demote/ban/delete one another: only one capacity-reducing transaction commits; rejected deletion never starts erasure cleanup (Task 2). +4. A mail provider accepts a request but the connection fails: account creation is retained and delivery is unknown, never automatically repeated or reported delivered (Task 4). +5. A user/detail view changes while a mutation settles: errors and password fields stay with the execution owner; only originating query keys invalidate (Tasks 5, 6). + +--- + +## Ownership, dependencies, and source findings + +- `apps/api/src/auth.ts` currently fixes verification to `true`, always installs `twoFactor()`, and exports one global instance. Settings in `apps/api/src/services/settings.ts` currently default verification/enrollment to `false` and are not consumed by auth. Preserve verification/enrollment `true` on upgrade and new installations. Legacy stored values remain pending; only an explicit full-admin validated atomic policy apply may record activation version `1` and make them effective. Environment ownership is disclosed and validated, and a pre-activation false value cannot silently weaken the baseline. +- The installed Better Auth supports `session.disableSessionRefresh`, `emailAndPassword.minPasswordLength/maxPasswordLength`, and immutable construction options. Use these supported seams; never mutate a shared `$context` per request. +- `apps/api/src/services/user-erasure.ts` starts journal/filesystem cleanup in the user-delete **before** hook. A database trigger alone fires too late to prevent destructive cleanup on a denied deletion. +- List uses `admin.setRole`, detail uses `admin.updateUser({data:{role}})`, and privacy roles use separate routes. Better Auth client results resolve `{data,error}`. Normalize these paths, preserving deletion receipts and privacy-role session revocation. +- Plan owns new `apps/api/src/account/` services and `apps/api/src/routes/admin-accounts.ts`, account UI, and the named auth hooks. Controller serializes edits to `apps/api/src/routes/index.ts`, `apps/api/src/db/schema.ts`, `apps/api/src/routes/admin-settings.ts`, `apps/api/src/services/settings.ts`, `packages/core/src/api/contracts/index.ts`, `packages/core/src/api/contracts/admin.ts`, `packages/core/src/api/admin-sdk.ts`, `scripts/api-sdk/config.mjs`, `scripts/database-test-groups.json`, and `packages/i18n/locales/en.json` / `de.json`. +- Controller-reserved append-only migrations are `0043_admin_auth_policy.sql` (Task1), `0044_admin_account_invariant.sql` (Task2), `0045_admin_reauthentication.sql` (Task3), and `0046_admin_account_provisioning.sql` (Task4), under `apps/api/src/db/migrations/` with corresponding Drizzle snapshot/journal changes. The master allocation/order is binding. Never modify an earlier applied migration; each task appends its own schema delta. No worker independently allocates migration indices. +- Plan 04 consumes Task 3's `AdminActionReauthentication`, atomic `consumeMany` and `AdminReauthenticationDialog`; plan 05 Task 6 consumes purpose `transit.credentials` with `targetId=catalogRevisionId`; plan 08.3 consumes `platform.domain.apply` with target equal to the exact actor-bound domain preview fingerprint; plan 07 keeps its existing artifact-specific ceremony and imports only the existing assurance primitives. Plan 08 shares settings handlers/UI and must compose this plan's validator/transaction path. No dependency on plan 07 UI. +- Controller registers account/reauth contracts and generates OpenAPI/SDK after producer schemas settle, before Task 6's generated consumers. New contracts live in `packages/core/src/api/contracts/account-admin/{policy,reauth,users,operations,index}.ts`; controller owns aggregation and generated files. + +## Task 1: Effective coherent auth policy (A01, A02, A03, A04) + +**Files:** Create `apps/api/src/db/account-administration-schema.ts`, `apps/api/src/account/auth-policy.ts`, `apps/api/src/account/auth-runtime.ts`, `apps/api/src/account/auth-database.ts`, `apps/api/src/account/auth-policy.test.ts`, `apps/api/src/account/auth-policy-postgres.test.ts`, `apps/api/src/account/password-verification-policy.ts`, `apps/api/src/account/password-verification-policy.test.ts`; modify `apps/api/src/auth.ts`, `apps/api/src/services/settings.ts`, `apps/api/src/routes/admin-settings.ts`, `apps/api/src/routes/__tests__/admin-settings.test.ts`, `packages/core/src/api/contracts/account-admin/policy.ts`. + +**Interfaces:** + +- `AuthPolicySnapshot = Readonly<{activationVersion: 0|1; revision: string; emailVerificationRequired: boolean; sessionDurationHours: number; twoFaEnabled: boolean; passwordMinLength: number}>`; revision is a stable hash of the validated effective tuple, with no secret input. +- `loadAuthPolicy(database?: AuthDatabase): Promise` reads singleton `auth_policy_state` (`id=1`, `activation_version=1`, complete four-setting JSON snapshot, revision, activated_by/at). Absent activation returns version `0` and verification/enrollment `true`, lifetime `168`, minimum `8`, ignoring inert legacy drafts. Once activated, valid `AUTH_EMAIL_VERIFICATION` (`true|false|1|0`) overrides the activated DB snapshot, with declared ownership. Invalid active policy fails closed with `AUTH_POLICY_INVALID`; database failure is `AUTH_POLICY_UNAVAILABLE`, never interpreted as unactivated defaults. +- `applyAuthPolicy({settings:{emailVerificationRequired,sessionDurationHours,twoFaEnabled,passwordMinLength},expectedRevision,confirmed:true,idempotencyKey,actorId}): Promise` requires a real full-admin session, validates all four effective values, locks the singleton and atomically writes settings, activation version `1`, revision and audit. GET `/api/admin/accounts/policy` returns `{active,pending,ownership}`; POST `/api/admin/accounts/policy/apply` is the only activation operation. Generic settings PATCH/import only stage validated pending auth drafts, even after activation; they never activate or replace the live tuple. +- `withAuthOperation(run: (instance: PolicyAuth) => Promise): Promise` uses AsyncLocalStorage to pin policy and transaction context. `auth.handler` and **every** `auth.api` method delegate through this facade, preserving inferred overloads, `asResponse`, headers, and `$Infer` types. `PolicyAuth` is `ReturnType`; no consumer receives an unguarded raw instance. +- Installed-source constraint: `better-auth/dist/plugins/admin/routes.mjs:createUser` spreads caller `data` into the user and checks only maximum password length; `plugins/email-otp/routes.mjs:resetPasswordEmailOTP` checks both bounds but then calls `updateUser(id,{emailVerified:true})`. These are actual installed 1.7.7 paths, not hypothetical upstream behavior. Own the HTTP and `auth.api.createUser/resetPasswordEmailOTP` paths in this task. +- `applyPasswordVerificationPolicy` in `account/password-verification-policy.ts` supplies typed Better Auth `databaseHooks.user.create.before/update.before` using the immutable operation context. On `/admin/create-user`, force `{data:{...user,emailVerified:false}}` before insertion (including a nested `data.emailVerified:true` and server API calls). The common facade enforces the current minimum for admin create before password hashing/insert. On `/email-otp/reset-password`, return `false` only for the installed plugin's exact verification-only update `{emailVerified:true}`; the installed `db/with-hooks.mjs:updateWithHooks` then performs no user update and the reset endpoint continues. Do not omit the field in `{data}`: Better Auth merges that patch back into the original. Any changed/unrecognized reset verification patch fails closed inside the transaction. This no-op preserves both false and true and cannot overwrite a concurrent legitimate email verification. Keep explicit verify-email/email-OTP verification and permitted assisted verification working; never patch node_modules. +- `AuthDatabase` is the exported Drizzle database/transaction structural subset used by the adapter and auth hooks; `currentAuthDatabase(): AuthDatabase` resolves the active transaction or ordinary DB. Cache only the latest immutable non-transactional factory result; in-flight calls retain their captured instance. Never cache an instance bound to a completed transaction. + +- [ ] **Step 1 — causal tests:** `legacy_policy_requires_explicit_activation` seeds old false verification/enrollment and custom numeric values, boots the real handler and asserts secure defaults plus pending view until explicit full-admin version-1 apply; ordinary user/privacy-admin/loopback and generic PATCH/import cannot activate. `policy_update_is_atomic_for_signin_and_server_api` begins login under verification=true, explicitly activates false, and asserts old login remains blocked while next HTTP and server API login succeed. Assert invalid booleans, fractional hours, hours `0/8761`, minimum `7/129`, and mixed valid/invalid settings reject the whole write. Assert sign-up/reset/change-password/admin-create/admin-set-password and email-OTP reset enforce the current minimum and maximum `128`. Through the installed handler AND server facade, admin create with `data.emailVerified:true` still stores/returns false, and a 7-character admin-created password is rejected before user/account writes. Email-OTP reset with a valid reset OTP changes the password but leaves initially false/true verification unchanged; invalid/expired/replayed OTP changes neither. Race a genuine verify-email commit with reset and assert reset never reverses it. Test explicit verify-email and email-OTP verification still set true. Execute these against installed plugin endpoints, not a fake provisioning return. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/account/auth-policy.test.ts apps/api/src/routes/__tests__/admin-settings.test.ts`; then `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/account/auth-policy-postgres.test.ts`. Expected causal failure: saved policy does not change the assembled auth handler, or invalid range is accepted; a missing dependency is not RED evidence. +- [ ] **Step 3 — minimal implementation:** extract construction options without changing OAuth/passkey/mobile/custom-session behavior. Set `requireEmailVerification` and password bounds from the snapshot; always retain `twoFactor()` but deny `/two-factor/enable` when `twoFaEnabled=false`. Existing verification, backup-code, disable, and sign-in challenges remain functional. Audit all password-writing endpoints of installed 1.7.7; add the same minimum hook wherever a plugin path omits it, specifically `/admin/create-user`. Install `applyPasswordVerificationPolicy` before persistence in both entry modes, composed with existing deletion/avatar/assurance hooks. Enforce minimum only on new passwords, never retroactively reject an existing password at sign-in. +- [ ] **Step 4 — lifetime and atomic application:** choose absolute session expiry at issuance: `expiresIn=hours*3600`, `disableSessionRefresh=true`, cookie-cache disabled. Existing sessions keep stored expiry; policy changes never lengthen them. Update settings copy to say “new sessions” and “absolute lifetime”. Settings PATCH/import stage drafts atomically using the same validator, and return `authPolicyApplication:"pending"`. Explicit policy Apply atomically stores the complete tuple plus activation version `1`; its response identifies `authPolicyRevision`, `activationVersion:1` and `appliesTo:"new_auth_operations"`. Failed activation leaves the entire previous active tuple/marker/audit unchanged. Other settings behavior remains intact. +- [ ] **Step 5 — GREEN:** rerun both RED commands plus `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/auth-postgres.test.ts`. Assert exact session expiry/cookie max-age, unchanged old expiry after policy increase and refresh, enrollment denial, retained TOTP challenge, and configured-MFA reauth denial for password-only login. Fake time must not mock Better Auth consumers. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(auth): apply coherent account policy to every auth operation`. Review snapshot lifetime, no auth-instance recursion, no policy-read fallback, all password entry points, and mobile/OAuth/session-projection compatibility. Record database unavailable separately. + +## Task 2: Atomic last-admin protection and one role authority (A09, A10) + +**Files:** Create `apps/api/src/account/admin-invariant.ts`, `apps/api/src/account/account-mutations.ts`, `apps/api/src/account/admin-invariant-postgres.test.ts`, `apps/api/src/account/account-mutations.test.ts`; modify `apps/api/src/db/account-administration-schema.ts`, `apps/api/src/account/auth-database.ts`, `apps/api/src/account/auth-runtime.ts`, `apps/api/src/auth.ts`, `apps/api/src/routes/privacy-roles.ts`, `apps/api/src/utils/auth-audit-hook.ts`, `apps/api/src/utils/auth-audit-hook.test.ts`, `apps/api/src/services/user-erasure.ts`; controller applies reserved migration and schema export. + +**Interfaces:** + +- `AccountRole = "user" | "admin" | "privacy_admin"`; `isActiveFullAdmin({role,banned})` uses the existing full-admin authorization predicate and `banned !== true`. An expired ban is still banned until Better Auth clears it. Reject new remote role arrays/comma roles. Under the migration write lock normalize historical combinations of recognized roles without losing effective full-admin authority; unknown role tokens abort with a bounded diagnostic. Test existing admin,user and admin,privacy_admin combinations explicitly, matching actual runtime authorization semantics. +- Database `account_admin_guard` has one row `id=1, active_count>=0`. Seed under a user-table write lock; transactional INSERT/UPDATE/DELETE triggers adjust the counter using OLD/NEW predicates. A negative delta that would leave zero raises named `LAST_ADMIN_REQUIRED`; a zero-admin bootstrap database permits positive transitions. No remote bypass flag or session-variable bypass exists. +- `withAccountMutation(intent: AccountMutationIntent, run: (tx: AuthDatabase) => Promise): Promise` locks the guard row **before target/user rows or erasure hooks**, revalidates actor/target, rejects unsafe projected loss, and runs Better Auth plus hooks against the same transaction via Task 1's adapter. `AccountMutationIntent` names actor, target, action (`role|ban|unban|delete|create`), expected role when supplied, and mutation key. +- `changeAccountRole({actorId,userId,role,expectedRole,idempotencyKey}, tx): Promise<{userId:string;role:AccountRole;changed:boolean}>` updates, revokes sessions, and writes `user.role.change` with `previousRole` and `role` in one transaction. Existing privacy-role grant/revoke URLs retain their prohibition on downgrading full admins and delegate to this service. + +- [ ] **Step 1 — causal tests:** launch two actual Fastify/Better Auth requests with separate connections and a barrier, each removing one of two admins, for role/ban/delete and mixed pairs. Assert exactly one succeeds, one usable full admin remains, counter equals query count, and rejected deletion invokes zero erasure request/cleanup calls. Cover self-service `/delete-user`, `/delete-user/callback`, direct `/admin/remove-user`, `/admin/ban-user`, `/admin/set-role`, and attempted `/admin/update-user` sensitive-field writes. +- [ ] **Step 2 — RED:** `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/account/admin-invariant-postgres.test.ts`. Expected baseline fails concurrency or invokes cleanup before last-admin rejection. Run against production migrations on an owned disposable DB, never a developer account database. +- [ ] **Step 3 — minimal implementation:** wrap all identified auth HTTP **and server API** mutation entry points; reject `role`, `banned`, `emailVerified`, `twoFactorEnabled`, email and credential fields in generic `/admin/update-user` (allow name/image only). Direct set-role delegates to the canonical role service. Use the DB trigger as the backstop, not a pre-check followed by an unlocked mutation. Failed/non-2xx Better Auth Responses cause transaction rollback before the facade returns them; never commit swallowed endpoint errors. +- [ ] **Step 4 — audit/rollback tests:** force audit failure, erasure-hook failure, ban-session-revoke failure and direct SQL counter failure; assert DB mutations/counter/audit/session changes roll back. A denied operation leaves no success audit. A role change through list, detail, legacy privacy route, or direct auth emits one `user.role.change`; prevent duplicate hook audit. Preserve pseudonymous deletion receipts and report existing external-erasure side effects honestly if cleanup fails after an admitted deletion; do not invent file rollback. +- [ ] **Step 5 — GREEN:** rerun RED and `pnpm exec vitest run --project node apps/api/src/account/account-mutations.test.ts apps/api/src/utils/auth-audit-hook.test.ts apps/api/src/services/__tests__/user-erasure.test.ts`; controller adds new DB test to `scripts/database-test-groups.json`. Verify local initial promotion still works, ordinary users cannot invoke administration, and neither `privacy_admin` nor a synthetic loopback principal gains new recovery authority. +- [ ] **Step 6 — commit/review gate:** proposed commit `fix(auth): serialize account authority and preserve the last administrator`. Review guard lock ordering, transaction-aware **all** erasure DB calls, direct auth bypass inventory, rollback, and trigger initialization. A route-mock-only test cannot qualify this task. + +## Task 3: Purpose-bound administrator reauthentication (dependency for A07/A08 and I06) + +**Files:** Create `apps/api/src/account/admin-reauthentication.ts`, `apps/api/src/routes/admin-reauthentication.ts`, `apps/api/src/account/admin-reauthentication-postgres.test.ts`, `packages/core/src/api/contracts/account-admin/reauth.ts`, `apps/api/src/account/domain-recovery-assurance.ts`, `apps/api/src/account/domain-recovery-assurance.test.ts`; modify `apps/api/src/privacy/session-assurance.ts`, `apps/api/src/db/privacy-auth-schema.ts` and its forward schema delta in reserved **0045**; create `apps/web/src/components/admin/users/AdminReauthenticationDialog.tsx`, `apps/web/src/components/admin/users/AdminReauthenticationDialog.test.tsx`, `apps/web/src/app/auth/admin/page.tsx`; extend the administration schema only through reserved append-only `0045_admin_reauthentication.sql` and controller-owned metadata. + +**Interfaces:** + +- `AdminReauthPurpose = "account.verify" | "account.password.set" | "account.reset.send" | "account.invite" | "credentials.export" | "credentials.import" | "transit.credentials" | "platform.domain.apply"`. For `transit.credentials`, bind `targetId` to the exact active `catalogRevisionId`; plan 05 rechecks that active revision before consumption and mutation. A changed catalog requires a new ceremony; this purpose never authorizes catalog activation. +- Domain proof is the narrower supported password flow: `DomainRecoveryMethod = "password"|"password_totp"|"password_recovery"`; `isDomainRecoveryAssuranceSufficient(session:ReauthenticationSession, originIndependentPasswordLogin:boolean):boolean` requires the server-recorded login flag AND `isAssuranceSufficient`, rejecting passkey, federated and unknown methods. `platform.domain.apply` binds `targetId` to plan08's opaque 64-hex domain preview fingerprint (including actor, old/new origins, expected configuration revision, effective auth-policy revision and acknowledged prerequisites); any changed input requires a new preview/proof. Only a fresh successful login after challenge start qualifies, never credential-row existence, a password reset alone, an unknown provisioned password or the current session. +- The server-produced `originIndependentPasswordLogin:boolean` column on `session_auth_assurance` defaults false, with its forward migration in **0045** (never modify the existing privacy migration). `recordSessionFromAuthContext` sets it for successful `/sign-in/email` or `/sign-in/username`; for `/two-factor/verify-totp` or `/two-factor/verify-backup-code` it sets it only for the actual pending sign-in path with no existing usable session. Before dispatching a two-factor verification HTTP/server-API operation, Task1's facade calls its captured immutable raw `PolicyAuth.api.getSession({headers,query:{disableCookieCache:true,disableRefresh:true}})` (read only, same policy/DB; not recursive facade dispatch) and records `factorEntryHadSession:boolean` in its private AsyncLocalStorage context. A thrown/error response denies proof; a valid null response means no current session. Do not use `getSessionFromCtx/getAuthoritativeSessionFromCtx` here: installed `api/routes/session.mjs` converts get-session errors to null in those helpers, whereas the direct getSession endpoint propagates them. The installed `verifyTwoFactor` requires/consumes a signed pending challenge before it creates a new session when no session existed; the configured producers are password/username only (phone plugin is not installed). Session create.after sets the flag only for a new successful factor-sign-in with `factorEntryHadSession===false`; missing context denies. Never import the plugin's private helper or infer the branch from request fields. A caller body/header cannot supply the flag. Enrollment/session-rotation verification sets false even though the old method mapper calls it `password_totp`. Missing/legacy flag fails domain proof. Keep existing privacy assurance behavior unchanged. Re-read live user role/ban, session, flag and current `twoFactorEnabled` at completion and consumption. Domain admission share-locks the policy singleton while checking its revision and consuming the challenge;08 rechecks the bound policy before the first host effect. +- `assertDomainRecoveryPreview({actorId:string,previewFingerprint:string,expectedCurrentRevisionId:string|null,expectedAuthPolicyRevision:string},tx:AuthDatabase):Promise` is a required plan08 export: checks its actor-bound unexpired preview under its admission transaction and returns no secret material. Plan03 only validates the fingerprint form and captures `authPolicyRevision` on the challenge, avoiding an early import of a later08 module. Plan08 calls its validator and `consume` with `purpose:"platform.domain.apply",targetId:previewFingerprint` atomically with durable platform admission after the same preview/policy check. Domain completion/consumption rejects changed live policy revision even if the method would still be sufficient; obtain a new preview and ceremony. The consumer is the API `platform.apply` job admission, with later `stack.render` commit and a separate terminal handoff-prepare before helper drain as08.3 specifies. Evidence stored in the challenge is method/session/time/fingerprint, never a password/hash. +- Email-OTP login is deliberately ineligible for this purpose: installed `plugins/two-factor/index.mjs` matches password/username/phone sign-in, not `/sign-in/email-otp`, and current `session-assurance.ts` records no email-OTP assurance. Supporting it would require a new audited assurance/MFA producer; do not weaken existing policy to pretend it works. Passkey-only operators first establish a known password through the supported verified/reset channel, then prove password plus configured TOTP/recovery on the old origin. Preview still requires DNS/TLS readiness, OAuth callback updates and passkey reenrollment acknowledgments; proof does not establish those external prerequisites. +- `AdminActionReauthentication.start({actorId,startingSessionId,purpose,targetId,intentFingerprint?:string,nonce?:Buffer}): Promise<{challengeId:string;nonce:Buffer;expiresAt:Date}>`; `complete({challengeId,nonce,session}): Promise`; `consume({challengeId,nonce,sessionId,purpose,targetId,intentFingerprint?:string}, tx): Promise`. Reuse `ReauthenticationSession`, `getSessionAuthAssurance`, and `isAssuranceSufficient` from privacy; keep separate storage and cookie `__Host-openmapx-admin-reauth` (HttpOnly, Secure, SameSite=Strict, Path=/). +- `AdminReauthProofRef={challengeId:string}` is a public reference, never authority. Plan05's `transitProofToken`/`importProofToken` are exactly these challenge IDs, not bearer secrets; server code supplies the matching HttpOnly nonce and current session. Required `intentFingerprint` for transit mutations and credential apply is a server-issued opaque intent fingerprint stored at start and matched at consumption; it includes the preview identity/selected merge/deletes and revision bindings without publishing a secret hash. Domain intent equals its target preview fingerprint. Ordinary credential preview/export may omit intent because they do not mutate a target. Add `consumeMany({proofs:ReadonlyArray<{challengeId:string;purpose:AdminReauthPurpose;targetId:string;intentFingerprint:string}>,sessionId:string,nonce:Buffer|string},tx):Promise` for the finite two-proof credential-import/feed-map combination; exactly two distinct matching challenges, same actor/session/nonce-family, both live and unconsumed, all-or-none in the caller transaction. Existing single `consume` delegates to the same checks. The one HttpOnly cookie holds a 32-byte nonce for a bounded ceremony family (at most two active challenges, 600 seconds); the route passes the existing validated cookie nonce through the private optional `start.nonce` parameter when joining the permitted pending pair, and starting the second distinct purpose retains that nonce and first proof, while restarting the same purpose supersedes only that purpose. The dialog starts both challenges before a single interactive login, then completes both against that one new session. Starting a second purpose after the first has already completed returns `REAUTH_GROUP_RESTART_REQUIRED`; explicit restart creates a new pair before login. Do not rebind a completed proof to a different session. Never let a second start silently invalidate the first proof or consume only one on failure. +- Export read-only `check` and `checkMany` with the exact argument shapes of `consume` / `consumeMany` and transaction context; they validate the same live proofs without marking consumed.04's finite candidate participant uses check before prospective prepare and consume inside01.2's one transaction. A preview/checked reference alone is never mutation authority; commit rechecks every predicate. +- `POST /api/admin/reauth/start` body `{purpose,targetId,intentFingerprint?:string}` returns `{challengeId,expiresAt,loginPath:"/auth/admin"}`; `/complete` body `{challengeId}` returns `{completed:true}`. Full real admin only, no loopback/impersonation; 600-second lifetime, 32-byte random nonce stored only as digest, same actor, **new session authenticated after start**, MFA assurance if configured, one-use target/purpose consumption. Start validates the domain target fingerprint form; complete/consume validate captured policy and the strict recovery predicate, while08 alone validates the live preview for `platform.domain.apply`; other purposes keep the existing assurance predicate. +- `AdminReauthenticationDialog` accepts either `{purpose,targetId,intentFingerprint?,onCompleted,onClose}` for one action or `{purposes:[{purpose:"credentials.import",targetId, intentFingerprint},{purpose:"transit.credentials",targetId,intentFingerprint}],onCompleted,onClose}` for the finite import pair; `onCompleted` receives the ordered challenge-ID references. It retains only challenge IDs and allowlisted return intent in tab-scoped sessionStorage; no token, nonce, password, or email. `/auth/admin` has a fixed `/admin` return surface and resumes only allowlisted local account/settings destinations, never arbitrary `returnTo` URLs. + +- [ ] **Step 1 — tests:** reject same session, old assurance, MFA downgrade, cross-actor, cross-purpose, cross-target, changed transit catalog revision, expired/superseded/replayed challenges and concurrent consumes. Assert rollback after mutation failure permits safe retry with the same mutation identity without emitting duplicate audit; successful consumption is atomic with the target DB mutation or durable admission. Domain cases: passkey-only and federated login fail, random unknown provisioned password gives no proof, failed password gives no proof, fresh known password succeeds without MFA and requires TOTP/recovery with MFA; enrollment rotation/old login/trusted-device password-only shortcut and session-read failures fail. Alter preview/origin/revision/actor, revoke session, change effective auth-policy revision or enable MFA between complete and consume: no admission. Dual-proof tests start/complete both ceremonies, consume them in one import transaction and assert stale pin/candidate rollback consumes neither; lone import proof, lone transit proof and swapped targets fail. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/account/domain-recovery-assurance.test.ts`; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/account/admin-reauthentication-postgres.test.ts`; expected missing guarded ceremony/incorrect acceptance, not skipped DB tests. +- [ ] **Step 3 — minimal implementation:** mirror the proven privacy nonce/session-assurance rules with separate storage; do not repurpose artifact FKs or weaken `PrivacyReauthenticationService`. Add per-actor start/complete rate limits (5/minute); mutation idempotency and response logging must redact challenges/cookies. Task 4 and plan 04 use this service, not independent password prompts. +- [ ] **Step 4 — GREEN:** repeat RED and `pnpm exec vitest run --project web apps/web/src/components/admin/users/AdminReauthenticationDialog.test.tsx`; assert fixed-origin return, fresh login, resumed confirmation, and canceled/expired ceremonies leave target unchanged. Controller captures real rendered login-return behavior. +- [ ] **Step 5 — commit/review gate:** proposed commit `feat(admin): add purpose-bound action reauthentication`. Review cookie configuration, assurance reuse, new-session test, unique consumption, and shared interface compatibility with plan 04 before dispatching it. + +## Task 4: Invitations, verification, and assisted recovery (A06, A07, A08) + +**Files:** Create `apps/api/src/account/account-provisioning.ts`, `apps/api/src/account/account-mail-outbox.ts`, `apps/api/src/routes/admin-accounts.ts`, `apps/api/src/account/account-provisioning-postgres.test.ts`, `apps/api/src/account/account-provisioning.test.ts`, `packages/core/src/api/contracts/account-admin/users.ts`, `packages/core/src/api/contracts/account-admin/operations.ts`, `packages/core/src/api/contracts/account-admin/index.ts`; modify `apps/api/src/utils/emailTemplates.ts`, `apps/api/src/utils/auth-audit-hook.ts`, the administration schema only through reserved append-only `0046_admin_account_provisioning.sql` and controller-owned metadata. + +**Interfaces:** + +- `provisionAccount({name,email,role,password?,sendInvitation,idempotencyKey,actor}): Promise<{userId:string;created:boolean;verificationDelivery:MailOutcome;passwordSetupDelivery:MailOutcome}>`, `MailOutcome="not_requested"|"queued"|"sent"|"failed"|"unknown"`. No password input means an unpredictable, undisclosed initial credential and a password-setup invitation; `emailVerified=false` regardless of invitation or password reset. +- Account routes under `/api/admin/accounts`: POST root (create), POST `/:userId/invitation`, `/:userId/verification-email`, `/:userId/verify`, `/:userId/reset-password`, `/:userId/password`, PATCH `/:userId/role`. Verification body `{confirmed:true,reasonCode,expectedEmail}`; password body `{confirmed:true,reasonCode,password}`; reset/invitation/resend body `{confirmed:true}`. Role consumes Task 2 input. Every mutation requires `Idempotency-Key`; sensitive actions consume the matching Task 3 challenge. +- Persist unique `(actorId,idempotencyKey)` request receipts and mail intents; retain password/token payloads only encrypted in outbox, never receipts/audit. Use existing Better Auth verification/reset token issuance with local callbacks/outbox instead of custom reset tokens; record distinct verification and password-setup outcomes. A failed or unknown mail handoff does not roll back an already-created account. + +- [ ] **Step 1 — tests:** create with checked invitation, replay creation, explicit resend, expired prior links, mail failure and acceptance-then-disconnect. Assert one account, no automatic resend, correct partial outcomes, and no password in mail/logs/audit. Installed HTTP/server-API admin create with `data.emailVerified:true` cannot bypass Task1's forced-false hook. Reset (including `/email-otp/reset-password`) and admin password set leave `emailVerified` unchanged; assisted verify requires reason, current email, real admin and valid action reauth, increments audit and revokes stale sessions. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/account/account-provisioning.test.ts`; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/account/account-provisioning-postgres.test.ts`. Expected missing invitation behavior/unprotected recovery, not a synthetic fake-return mismatch. +- [ ] **Step 3 — minimal implementation:** orchestrate Better Auth through the policy/transaction facade. Token issuance enqueues encrypted mail before external send; background dispatch treats interrupted sending as unknown and requires an explicit new resend action. Verification and setup are separate ownership proofs (two clearly labeled mail links/messages); completing reset alone must not verify email. Restrict direct Better Auth set-password to the same reauth admission; reject generic verified/credential flag edits. Rate-limit mail actions to 3 per target/hour and 20 per actor/hour; preserve Better Auth's public anti-enumeration behavior. +- [ ] **Step 4 — GREEN:** repeat RED, exercising actual generated-response serialization after controller generation. Assert obsolete reset token cannot be used twice, failure recovery never consumes arbitrary verification links, and successful role/verification/password transitions create exactly one bounded audit receipt with no secret fields. +- [ ] **Step 5 — commit/review gate:** proposed commit `feat(admin): add audited account provisioning and assisted recovery`. Review crash boundaries between admission, token issuance and delivery; an unknown send remains visible and is not claimed sent/delivered. Reauthentication proof binds the selected account and action through confirmation. + +## Task 5: Honest Better Auth reads and mutations (A05, A10 consumer reliability) + +**Files:** Create `apps/web/src/lib/admin/accountAdmin.ts`, `apps/web/src/lib/admin/accountAdmin.test.ts`, `apps/web/src/components/admin/users/UserActions.test.tsx`; modify `apps/web/src/components/admin/users/UserList.tsx`, `UserDetail.tsx`, `CreateUserDialog.tsx`, `BanUserDialog.tsx` in that same directory. + +**Interfaces:** `unwrapAuthResult(result:{data:T|null;error:{code?:string;message?:string;status?:number}|null}):T` throws a bounded typed `AccountAdminError` on error or missing data. `useAccountAdmin()` exposes read/mutation helpers with query keys starting `['admin','accounts',apiOrigin,actorId]` and target IDs; retains invalidation of legacy list/detail keys until migration completes. Operations explicitly use `retry:false`. + +- [ ] **Step 1 — tests:** every existing list/get/count/create/update/ban/unban/remove/impersonate/session-list/revoke mutation receives resolved `{data:null,error}`. Assert no success toast, closed modal, navigation, optimistic role, or empty-list success; retained good rows show a refetch error. A deletion error leaves row/selection intact. List and detail role controls call the same canonical service with expected role. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project web apps/web/src/lib/admin/accountAdmin.test.ts apps/web/src/components/admin/users/UserActions.test.tsx`; baseline must show a false success on a resolved Better Auth error. +- [ ] **Step 3 — minimal implementation:** unwrap inside query/mutation functions before React Query sees success, normalize bounded codes, use execution ownership tied to API origin/actor/target and mounted view. Do not use client admin-count as authority; server `LAST_ADMIN_REQUIRED` is authoritative and the count is advisory. Switch role writes away from generic `updateUser`. +- [ ] **Step 4 — GREEN:** repeat RED; deferred A-to-B selection/remount cases assert A's completion neither clears B's input nor navigates B, while A's cache may invalidate. Verify transient failures do not erase a confirmed session/user list. +- [ ] **Step 5 — commit/review gate:** proposed commit `fix(admin): unwrap account operations before reporting success`. Inventory every Better Auth call in the four files, not just the changed handlers; review ordinary-user and privacy-admin navigation boundaries. + +## Task 6: Provisioning/policy controls and integrated handoff (A01–A10 UI coverage) + +**Files:** Create `apps/web/src/components/admin/users/AccountRecoveryActions.tsx`, `AccountRecoveryActions.test.tsx`, `apps/web/src/components/admin/settings/AuthPolicySettings.tsx`, `AuthPolicySettings.test.tsx`; modify `apps/web/src/components/admin/users/CreateUserDialog.tsx`, `UserDetail.tsx`, `apps/web/src/components/admin/settings/SystemSettings.tsx`, `apps/web/src/lib/admin/accountAdmin.ts`, `docs/docs/administration/users-and-access.md`. Controller owns locale merges and SDK output. + +**Interfaces:** recovery controls consume Task 4's exact operation results and Task 3's dialog; `sendInvitation:boolean` is controlled form state and is actually sent. Policy UI displays active versus legacy pending values, activation version/revision, source, exact ranges, new-session expiry semantics, and “Allow new 2FA enrollment”; verification remains read-only display plus explicit assisted action, never a raw checkbox mutation. + +- [ ] **Step 1 — tests:** checkbox off/on changes request; create-success/mail-failed renders existing account plus retryable handoff, never a duplicate create; resend and assisted verification/password/reset each require confirmation and appropriate reauth; clear password only for that execution owner. Invalid policy bounds stop submission and server validation matches them. An upgrade fixture with legacy false values displays enabled active verification/enrollment and pending drafts; only explicit full-admin Apply changes the active view, and any failed partial apply preserves all four active values. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project web apps/web/src/components/admin/users/AccountRecoveryActions.test.tsx apps/web/src/components/admin/settings/AuthPolicySettings.test.tsx`; assert missing controls and ignored invitation state fail behaviorally. +- [ ] **Step 3 — minimal implementation:** compose small components into existing detail/settings surfaces, use generated account SDK and error unwrapping, present action unavailability before submission, and document verification versus reset ownership semantics and local bootstrap recovery. Controller composes the `SystemSettings.tsx` insertion with plans 04/08/09; no worker rewrites that shared file wholesale. +- [ ] **Step 4 — GREEN:** repeat RED; `pnpm -C apps/api check-types`; `pnpm -C packages/core check-types`; `pnpm -C apps/web check-types`. Controller runs actual desktop/mobile user list/detail/create, failed mail, fresh-login return, and last-admin rejection views using synthetic accounts. Do not upload screenshots. +- [ ] **Step 5 — commit/review gate:** proposed commit `feat(admin): expose effective account policy and recovery controls`. Full qualification stays with master Q01–Q05: new DB tests registered, existing privacy/assurance/OAuth suites, serialized `pnpm openapi:generate` then `pnpm sdk:generate`, `pnpm check-openapi`, `pnpm check-api-sdk`, builds and CI. No portable pass substitutes for migrated auth/runtime proof. + +## Coverage and planning self-review + +A01 verification → Tasks 1/6; A02 lifetime → Tasks 1/6; A03 enrollment → Tasks 1/3/6; A04 password limits → Tasks 1/4/6; A05 result handling → Task 5; A06 invitations → Tasks 4/6; A07 verification → Tasks 3/4/6; A08 recovery → Tasks 3/4/6; A09 invariant → Task 2; A10 role/audit → Tasks 2/5. All Review Focus entries have owning tests. + +Self-review completed against spec and actual source: fixed the pre-erasure timing gap, server-API policy bypass, existing-session extension risk, MFA enrollment bypass, and ignored mail outcomes in the design. Controller finalized the master's append-only migration allocation and secure version-1 policy activation ruling. Remaining coordination is shared-file sequencing, generated family registration and the documented absolute-session choice; transit credential reauthentication is catalog-revision-bound, managed-map imports consume both proofs atomically, domain recovery binds an actually proven password/MFA session to the exact preview, and privacy artifact reauthentication stays separate. Installed Better Auth create/reset overrides have explicit causal tests. R1 correction consumes the binding rulings; plan08 must supply `assertDomainRecoveryPreview` and use `platform.domain.apply` before durable admission. This author performed planning only; the user's implementation authorization continues after the root review gate. No tests were run by this author. diff --git a/docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md b/docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md new file mode 100644 index 000000000..f16aa1643 --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md @@ -0,0 +1,200 @@ +# Integration Ownership and Migration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Activate binding/configuration changes honestly, manage standalone declarative installations, and provide bounded configuration/credential migration, covering I01–I06. + +**Architecture:** Extend the existing configuration ownership, revision-CAS, extension journal and staged integration reload paths. Separate desired database state, effective runtime state and installation provenance; migration inventories are safe projections, while vault values travel only in an explicitly reauthenticated encrypted envelope. + +**Tech Stack:** TypeScript, Fastify, Zod, Drizzle/PostgreSQL, typed ops-agent, integration-framework installer, Node crypto/Web Crypto, React/MUI, Vitest. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md` and binding `docs/superpowers/specs/2026-10-23-admin-control-protocols.md` §5; controller: `docs/superpowers/plans/2026-10-23-admin-instance-management.md`. + +## Global Constraints + +- All product work stays in this worktree; preserve unrelated work and existing + public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source + policy boundaries. Community integrations remain declarative; no uploaded code + execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain + behind typed ops-agent operations. No generic command, arbitrary path or raw + environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth + declarations, strict schemas, audit events, explicit mutation identities and + appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never + hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, + durable operation identities, retained results, unknown outcomes and cancellation + fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, + place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation + and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful + admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots + to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database + and runtime checks, independent review, production builds and PR CI. Report any + unavailable environment gate distinctly; do not replace runtime evidence with + mocked assertions. + +Planning boundary: no product edits, dependency installation, generators, builds, tests, or commits while authoring. Read root/web `AGENTS.md`, API surface, integration system, built-in/external integration guides, service-manifest ownership contract, and Contributing quality bar. Baseline: `ef896ceae876959180116cf61ffcf643b88ebffe`. + +## Review Focus + +1. A binding reload reads stale/unavailable authority or overlaps another edit: never report the saved binding as active without matching runtime evidence (Task 1). +2. A file/environment value shadows a database save or vault rotation: reject the ineffective write before persistence, disclose ownership, and never reveal the secret (Task 2). +3. A legacy directory is replaced or claimed between preview and adoption/removal: fingerprint/ownership conflict prevents deleting another install (Tasks 3, 4). +4. Migration targets differ in manifests, selected services, or host overrides: preview names omissions/prerequisites and stale preview cannot apply (Task 5). +5. A hostile envelope requests huge KDF work or has altered metadata/ciphertext: reject before expensive work or any database/host mutation (Task 6). + +--- + +## Ownership, existing behavior, and controller coordination + +- Binding handlers in `apps/api/src/routes/admin-capability-bindings.ts` only persist. `integration-host.ts` captures `requiresMap` during setup. Its detached reload already atomically publishes a complete generation, but currently catches binding-read failure and falls back to auto-selection; an administration apply must fail closed here. +- `apps/api/src/integration-config.ts` resolves `default < database < vault < config.json < env`; retain wire source value `"config.json"` and display “file” in UI. Credentials currently reload after set/delete, but still report success when a higher source wins. Existing `persistIntegrationConfigPatch`, `runIntegrationImport`, `saved.owns()` and trusted configuration evidence remain authoritative; do not replace their guarded recovery with blind rollback. +- `packages/integration-framework/src/installer.ts` implements both source and artifact install with `allowLocalSources:false` for remote use, checksum/protocol/declarative checks. `packages/cli/src/commands/integrations.ts` can install/remove without `installed_extension` inventory. Legacy files must be inspected as their own provenance, not assigned to a catalog record by matching ID. +- Plan owns focused new modules under `apps/api/src/services/standalone-integrations/`, `apps/api/src/services/configuration-migration/`, and corresponding route/UI modules. Controller serializes `apps/api/src/routes/index.ts`, `apps/api/src/services/admin-job-handlers.ts`, `apps/api/src/services/job-runner.ts`, `apps/api/src/db/schema.ts`, `apps/ops-agent/src/runtime.ts`, `apps/ops-agent/src/server.ts`, `apps/ops-agent/src/index.ts`, `apps/ops-agent/package.json`, `pnpm-lock.yaml`, `packages/core/src/ops/{contract,index}.ts`, `scripts/check-ops-authority.ts`, SDK registrations and all generated files. +- Use append-only migrations reserved by the master: `0047_admin_integration_application.sql` (Task1 revisions/receipts), `0048_admin_integration_ownership.sql` (Task3 provenance), `0049_admin_configuration_migration.sql` (Task5 migration receipts), and `0050_admin_credential_transfer.sql` (Task6 encrypted short-lived staging), all under `apps/api/src/db/migrations/`. Controller serializes Drizzle metadata. Later tasks never edit an earlier applied migration. +- Shared files requiring serial edits: `apps/api/src/integration-host.ts` (Tasks 1/2/4), `apps/api/src/services/extension-installer.ts` and `extension-recovery.ts` (Tasks 3/4), `apps/api/src/routes/admin-settings.ts` / `services/settings.ts` / `apps/web/src/components/admin/settings/SystemSettings.tsx` (plans 03/08/09), `packages/core/src/api/contracts/integration-admin/{common,credentials,operations}.ts`, extension contracts and `apps/web/src/components/admin/shared/SchemaConfigForm.tsx` (plan 08). Controller merges locale keys in `packages/i18n/locales/en.json` and `de.json`. +- Task 6 consumes plan 03 Task 3's `AdminActionReauthentication` for `credentials.export/import` and the same dialog. Plan 08 owns service/platform fields and generation storage. This plan owns migration adapters over the existing service config writer and trusted configuration helpers; no undefined plan-08 vault/collector API is assumed. Plan 06 owns large staged transport; the standalone URL/source path works without it. Uploaded standalone artifacts are required scope: Task3 consumes plan06.2's `integration-artifact` stage, canonical binary upload and durable claim protocol. Encrypted credential envelopes alone retain bounded canonical JSON (≤1 MiB); no second large-upload service. +- Canonical definitions precede generation. New API modules: `packages/core/src/api/contracts/integration-admin/application.ts`, `standalone-integrations.ts`, `configuration-migration.ts`, `credential-transfer.ts`; aggregate operations/SDK changes are controller-owned. New typed ops schemas are in `packages/core/src/ops/standalone-integrations.ts`; operation-kind unions, fingerprints, inventory and assembled dispatch registration are serialized. + +## Required finite candidate transaction participation (04 →01.2/03/05) + +The binding01.2 `persistAndApplyProspectiveConfiguration` signature remains unchanged. Its `AdministrativeConfigurationWriter` is the context owner, and its persisted candidate lifecycle owns the one DB transaction. These exact additional **server-private** exports are required from that producer when04 wires its consumers; they are not public API fields or a second writer: + +- `withAdministrativeConfigurationRead({signal?:AbortSignal},run:(context:AdministrativeConfigurationReadContext)=>Promise):Promise` produces a coherent read-only transaction/registry/host-source context without global or integration write locks. Ownership GET/inventory/strict reload consumers use it; it never converts failed acquisition into absent state. Candidate preparation continues to use the writer-owned read context. +- `withConfigurationCommitParticipant(writer:AdministrativeConfigurationWriter,participant:ConfigurationCommitParticipant,run:()=>Promise):Promise` scopes one finite participant to that same live writer/operation and restores the scope on return. It neither acquires locks nor starts a transaction. `ConfigurationCommitParticipant` is a nominal closed kind `"integration-binding"|"configuration-migration"|"credential-import"|"transit-credentials"`, constructed only by the owning modules. Reject foreign/expired/second/conflicting participants; nested helpers reuse the same participant.01.2's prepare uses its validated scope as the non-consuming managed-secret authorization, and persistAndApply invokes it once in the candidate's own transaction before generic overlay persistence. Read-only preview participants have `mode:"preview"` and cannot enter persistAndApply; apply requires `mode:"apply"` and current proof/pin/revision checks. No caller boolean or purpose string creates one. Preview may seal/budget an inert candidate but cannot save a target, consume proof, enqueue or publish; unadmitted private candidate material is discarded under01.2 TTL/queue rules. +- The fixed participant protocol is `{kind,mode,actorId,operationKey,managedServiceSecretKeys:readonly [{serviceId:"data-manager";key:"TRANSITOUS_API_KEYS"}]|readonly [],authorize(writer:AdministrativeConfigurationWriter):Promise,verify(candidate:ValidatedConfigurationCandidate,writer:AdministrativeConfigurationWriter):Promise,commit(candidate:ValidatedConfigurationCandidate,writer:AdministrativeConfigurationWriter,tx:ConfigurationWriteTransaction):Promise}` behind that nominal brand. `ConfigurationWriteTransaction` is the writer connection's Drizzle transaction subset including select/insert/update/delete/execute;04/03/05 adapters must use it, never the default DB pool. authorize is read-only and runs before candidate preparation; verify is read-only and matches the prepared candidate/source/intent before persistence and again inside its commit transaction; commit runs only after all prospective schema/size/source checks succeed, in the same transaction as generic overlay writes/job checkpoint. The generic persister excludes only the exact managed key when claimed by the finite transit/import participant, so05.6 writes it once. A participant failure rolls back all desired rows, proofs and invalidation; no target-mutating continuation is authorized. If§5.4 already prepared ownership-only maintenance after full candidate preflight, only an exact known-unused abandon decision can settle it; unknown rollback keeps it fenced. The same transaction checkpoints participant kind, actor, operation/preview IDs and completed mutation identity (no proof nonce, secret or callback) in the existing bounded admin_job receipt. Restart after commit uses that retained candidate/receipt and lookup-only admission recovery; it never repeats proof consumption or feed persistence. Before-commit process loss does not prove absence of a remote reservation or DB commit: inspect retained candidate/decision and exact job row under§5.4 before choosing continuation/known-unused abandon. Discard only unadmitted, unowned candidate material per01.2 rules. +- 04.1 creates a binding participant holding the exact expected-prior binding and retained receipt;04.5 creates a migration participant holding validated pending auth drafts plus selected binding CAS changes;04.6 creates a credential-import participant holding the exact encrypted preview and03 proof references plus optional05 `PreparedFeedCredentialPatch`.03 exports non-consuming `checkMany` with the same proof/session/nonce/purpose/target/intent checks as `consumeMany`; preview context verifies actor/decrypt-stage ownership without acquiring mutation authority, while apply context uses checkMany before preparation and consumeMany inside commit.04.6 commit invokes05.6 `persistPreparedFeedCredentialPatch` with the current writer/transaction context and performs proof consumption/ordinary row writes atomically through the candidate persister. For a single proof, add `check` mirroring `consume`. All raw nonce/session context stays server-local. +- Required causal tests use two DB connections and the actual candidate coordinator: bind/import inverse-lock schedules do not deadlock; a changed feed pin/source/map or proof after verify fails the transaction with no ordinary-secret commit; a thrown participant after proof consume rolls back proof and every row; preview participant cannot persist. There is no arbitrary registered callback, generic command or remote participant registry.01.2 lands the closed dispatcher/context seam;04/05 install their fixed typed adapters in their exclusive task windows before advertising those operations. Root must mirror these exports in A's§5 and05.6 before the fresh review; they close a real signature gap rather than leaving the implementer to invent a transaction hook. + +## Task 1: Binding application with runtime evidence (I01) + +**Files:** Create `apps/api/src/services/integration-application.ts`, `apps/api/src/services/integration-application.test.ts`, `apps/api/src/services/integration-application-postgres.test.ts`, `apps/api/src/integration-host.binding-application.test.ts`, `apps/api/src/db/integration-administration-schema.ts`, `packages/core/src/api/contracts/integration-admin/application.ts`; modify `apps/api/src/services/capability-bindings.ts`, `apps/api/src/routes/admin-capability-bindings.ts`, `apps/api/src/integration-host.ts`, `apps/web/src/lib/admin/integrationAdmin.ts`, `apps/web/src/components/admin/integrations/IntegrationDetail.tsx`, existing integration contract operations/common. + +**Interfaces:** + +- `IntegrationApplyReceipt = {operationId:string; integrationId:string; desiredRevision:string; database:"retained"|"restored"|"conflict"|"unknown"; runtime:"applied"|"pending"|"failed"|"unknown"; runtimeGenerationId:string|null; reason?:"integration_disabled"|"reload_failed"|"ownership_changed"|"authority_unavailable"|"reconciliation_required"}`. Existing trusted host apply outcome remains a separate nested `hostApply` field where relevant; do not collapse committed-but-incomplete into failure/no-effect. +- `applyBindingChange({integrationId,capability,serviceId:string|null,expectedServiceId:string|null,operationId,actorId}): Promise`. POST/DELETE keep their URLs, require caller-retained `Idempotency-Key` and expected prior service ID, and return the receipt instead of bare `{ok:true}`. GET includes desired binding and live effective binding/generation independently. +- `inspectIntegrationRuntime(integrationId): IntegrationRuntimeEvidence | null`, where evidence is `{generationId:string;bindings:Record;configRevision:string}`; fingerprints/revisions contain no values. `withIntegrationAdministration({actorId:string,operationKey:string,integrationIds:readonly string[],signal?:AbortSignal},run:(writer:AdministrativeConfigurationWriter)=>Promise):Promise` delegates to01.2 `withAdministrativeConfigurationWriter(options,run)`, obtaining its global advisory key `(0x4f4d58,1)` then sorted per-integration locks on one dedicated connection across persistence/host apply/reload; all API config, import, vault and binding writers join this boundary, including delete/reinsert ABA checks. Nested import/feed/vault calls receive the exact opaque live connection/context, never a caller boolean; no path holds an integration lock before requesting global ownership. Strict reads and the reload publisher do not acquire a write guard. Host work follows global writer → sorted integration locks → host target → nonblocking transit reservation → host configuration writer → release store;The exact nominal `AdministrativeConfigurationWriter`/`AdministrativeConfigurationReadContext` come from `apps/api/src/services/administrative-configuration-writer.ts`; no competing context type. Nested adapters obtain the same ALS writer and verify the requested ID subset; they do not call an acquiring wrapper again or widen the locked set. + +- [ ] **Step 1 — tests:** set A→B, invoke a synthetic trusted integration provider that captured `ctx.getRequiredService`, and assert it now uses B; delete binding restores automatic resolution. Force binding query failure during reload and assert old provider stays active and receipt is failed/pending, never applied. Race two API writers; second expected-prior request conflicts rather than undoing the first. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/services/integration-application.test.ts apps/api/src/integration-host.binding-application.test.ts`; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/services/integration-application-postgres.test.ts`. Baseline persists without provider activation; assert that causal mismatch before editing. +- [ ] **Step 3 — minimal implementation:** validate integration declares the capability and target is installed/enabled/provides it; atomically CAS desired state, retain operation receipt, and await serialized reload. Make binding and configuration reads used by administrative reload strict; preserve the last live generation on unavailable authority. Capture generation evidence at atomic swap, then compare the operation's revision and live bindings before claiming applied. Disabled integrations return pending with `integration_disabled`. +- [ ] **Step 4 — failure ownership:** serialize imports/config/credentials through the same lock namespace and hold dedicated connection ownership across commits. Release physical advisory locks on scope exit only after retaining any unresolved candidate/admission fence in01.2's durable receipt; a later writer must reconcile that fence before starting. An observation failure cannot erase durable ownership. Failed reload retains desired state with explicit pending/failed receipt and last-live binding; retry is an explicit new apply action using current revision. Unknown receipt transport is reconciled by operation ID. Do not silently restore another writer's data or rebuild an unproven previous runtime. +- [ ] **Step 5 — GREEN:** repeat RED plus `pnpm exec vitest run --project web apps/web/src/lib/admin/integrationAdmin.test.tsx apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx`. Assert saved/applied/error copy and retained old binding, exact Fastify response serialization and generated-client fields after controller generation. +- [ ] **Step 6 — commit/review gate:** proposed commit `fix(admin): apply capability bindings through verified reloads`. Review coalesced/trailing reload schedules, no runtime success from database equality alone, sorted lock ordering, and strict-read failure behavior. + +## Task 2: Source-aware editing and effective vault rotation (I02) + +**Files:** Create `apps/api/src/services/configuration-ownership-view.ts`, `apps/api/src/services/configuration-ownership-view.test.ts`, `apps/api/src/services/integration-credential-application.ts`, `apps/api/src/services/integration-credential-application.test.ts`, `apps/api/src/routes/__tests__/admin-service-ownership.test.ts`; modify `apps/api/src/services/service-config-resolver.ts`, `apps/api/src/services/service-config-writer.ts`, `apps/api/src/services/service-secrets.ts`, `apps/api/src/services/trusted-config-operations.ts`, `apps/api/src/routes/admin-services.ts`, `packages/core/src/api/contracts/service-admin/{services,operations}.ts`, `apps/web/src/components/admin/services/{ServiceConfigForm,ServiceCredentials,ServiceDetail}.tsx`, `apps/api/src/integration-config.ts`, `apps/api/src/services/secrets.ts`, `apps/api/src/services/integration-credential-status.ts`, `apps/api/src/services/integration-config-import.ts`, `apps/api/src/routes/admin-integrations.ts`, `apps/api/src/routes/admin-integration-credentials.ts`, `apps/web/src/components/admin/shared/SchemaConfigForm.tsx`, `apps/web/src/components/admin/integrations/ConfigSchemaForm.tsx`, `SetCredentialDialog.tsx`, `IntegrationDetail.tsx` (last two in same directory), existing integration contracts. + +**Interfaces:** + +- `ConfigurationOwnership = {source:"default"|"database"|"vault"|"config.json"|"env"; writable:boolean; secret:boolean; adoption:"direct"|"host_action_required"; reason:"host_environment"|"host_file"|"vault_only"|"managed_workflow"|null; envVar?:string; managedBy?:"transit.credentials"|"dawarich-provisioning"}`. No host path, plaintext secret, secret digest or provider response is included. Service/integration forms use this descriptor. Plan 08's distinct `PlatformField={source:"host"}|{source:"database",value:T}` is preserved; migration adapts it to inventory metadata without pretending it contains a file/env distinction. +- `inspectIntegrationOwnership(integrationId): Promise>` resolves strictly for administrative operations. Non-secret default/database keys adopt directly by an explicit Save; secret default/database/vault keys route only to vault. File/env keys have `writable:false`, `adoption:"host_action_required"`; instructions identify the integration/key or env variable, never expose file contents. +- Plan01.2 owns the early strict service resolver and prospective validator. Consume `resolveServiceConfigWithSourcesStrict(manifest:ResolveServiceConfigInput,context:AdministrativeConfigurationReadContext):Promise>` / `resolveAllServiceConfigsStrict(manifests:ResolveServiceConfigInput[],context):Promise`, `resolveIntegrationConfigWithSourcesStrict(manifest,directory,context)` and strict vault access. `prepareProspectiveConfiguration({actorId,expectedCurrentRevisionId,overlay,operationKey,signal?}):Promise` requires the matching active writer; `persistAndApplyProspectiveConfiguration(candidate,{coordinatorMode,signal?}):Promise` owns persistence/publication. `CoordinatorMode="ensure-running"|"require-running"` is01.2's portable type; forms/imports requiring lifecycle preparation choose ensure-running, render-only flows require-running. Import candidate/overlay/receipt types from `services/prospective-configuration.ts`, not a second local shape; do not call fallback `resolveAllServiceConfigs` for administration. `inspectServiceOwnership(serviceId:string):Promise>` in `configuration-ownership-view.ts` projects that same strict source snapshot for GET config/credentials and all service forms. No separate source collector; a missing DB row is empty, a failed DB/vault/registry/source read is unavailable and prevents full-snapshot publication. +- Service precedence remains default < database < nonempty `SERVICE_${ID_UPPER_UNDERSCORE}_${KEY_UPPER}`; unlike integration resolution, service empty env values are unset (`config-resolver.ts:94-97`). Preserve manifest baseline suppression and proxy-host special projection. For resource fields also inspect the actual supported interpolation owner: `BROUTER_MEMORY`, `LOCAL_AI_MEMORY`, `MOTIS_SLOT_MEMORY` (both motis slots), `POSTGIS_MEMORY`, plus their namespaced `SERVICE_*_OPENMAPX_MEMORY_MIB` / `SERVICE_*_OPENMAPX_CPU_LIMIT` controls. A nonempty winning host knob makes the corresponding field host-owned and blocks save/import, even if an explicit DB resource override could replace its rendered interpolation. No raw env map or interpolation evaluator is added. Consume01.2's bounded read-only `configuration.sources.inspect {}` result for host knobs unavailable in API process.env: `{revisionId,sourceRevision,services:[{serviceId,fields:[{key,source:"host"|"none",envVar?}]}]}`, no values (≤128KiB,≤256 services/128 fields). Its private sourceRevision is bound into the candidate/typed operation as expectedSourceRevision and rechecked by agent at commit. Never read host .env in API. Plan08's resource resolver must consume this same policy; absent host variables retain literal/symbolic manifest defaults. Report env variable names only, never values. +- All direct service config POST, generic service-secret PUT/DELETE, integration config PATCH/import, binding changes affecting trusted snapshots, and integration vault set/delete join the same 01.2 administrative-writer boundary. Validate the complete strict merged candidate (all unchanged rows, tree bounds and **524,288-byte actual sealed envelope including MAC/newline/escaping**) before any desired row write, revision increment, audit-success, generation publish or effect admission. After complete schema/source/size/queue/private-receipt preflight,§5.4 permits only its fixed ownership-only maintenance admission before the DB transaction; this cannot change target configuration/data/images or consume proof. Hold revision/source ownership through candidate commit/apply, including asynchronous admission/reconciliation; later workers consume the retained candidate, never reconstruct it from today's rows. `persistServiceConfigPatch`/vault helpers are low-level transaction-bound adapters, not independent mutation entry points. Failed preflight writes nothing. A legacy-invalid snapshot may receive a correction only when the whole resulting candidate passes every limit; no truncation or partly-invalid persisted reduction. +- Managed secret integration: consume plan05.6 `isManagedTransitCredential`, `TRANSIT_CREDENTIAL_MANAGED_BY="transit.credentials"`, `TRANSIT_CREDENTIAL_MANAGED_CODE="transit_credentials_managed"` from `managed-service-credentials.ts`. Generic PUT **and DELETE** reject409 before parsing/mutation in `beforeValidation` and handler, including a retained undeclared row. Keep Dawarich's existing managed guard. Credential metadata projects `managedBy` and UI links to the feed workflow; Task6 calls the specialized import adapter, never the generic setter. Task04.2 establishes the guarded writer/projection now;05.6 registers the managed transit predicate before its declaration can expose a bypass. +- `applyIntegrationCredential({integrationId,key,value?:string,action:"set"|"delete",expectedRevision,operationId,actorId}): Promise` uses Task 1 lock/receipt and strict vault access; `value` is write-only. Reject shadowed set/delete with `409 CONFIGURATION_SOURCE_OWNED` and ownership descriptor before any DB mutation. All config PATCH/import paths enforce the same source check. + +- [ ] **Step 1 — tests:** matrix `default/database/vault/config.json/env` for ordinary and secret fields, including service-empty-env-as-unset versus integration-empty-env semantics, `***` preservation marker, undecryptable vault and failed DB reads. Assert file/env-owned writes cause zero persist/reload; direct adoption yields database/vault ownership; rotating a vault key changes the synthetic provider's captured credential without returning it. Concurrent file/authority change must prevent applied classification. Real service config/credential endpoints reject env-owned ordinary and resource fields with zero DB write; two real requests, one binding change and one mixed import, acquire global-before-integration in the same order and finish/conflict without deadlock; nested credential/feed calls reuse the exact live writer context while a forged/foreign/closed context fails; removing the override then explicit refresh/adoption succeeds without changing unrelated fields. Fail exactly one service DB query while registry/integration/vault reads succeed: full trusted assembly must reject with zero render/admission. Include oversized merged ordinary secrets and managed generic PUT/DELETE, not only projection mocks. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/services/configuration-ownership-view.test.ts apps/api/src/services/integration-credential-application.test.ts apps/api/src/integration-config.test.ts apps/api/src/routes/__tests__/admin-service-ownership.test.ts`; existing baseline falsely succeeds under file override. +- [ ] **Step 3 — minimal implementation:** consume 01.2 strict administrative resolution and candidate persistence/apply; preserve public-provider fallback behavior. Expand strict integration DB/file/vault acquisition at the same boundary: malformed/unreadable config.json is unavailable, not absent. Recheck ownership under Task 1 lock before persistence and before effective classification; annotate shadowing instead of weakening precedence. Preserve source absence versus retrieval failure. Use01.2's strict integration-vault reader parallel to `resolveServiceVaultSecretsStrict`; no swallowed decrypt errors on migration/apply. Validate secret fields without logging their invalid values (including `warnInvalidConfig` paths). +- [ ] **Step 4 — UI and host handoff:** lock both file and environment winners, render ownership and manual removal/restart instructions, show vault rotate/delete only when effective, and show previous/live versus desired revision if apply fails. Removing a host override is an explicit host action followed by refresh and Save/Rotate; there is no raw host-file editor or automatic secret copying. Service-vault publication retains existing `service.apply`/`configurationKind:"vault"` and `serviceApply` intent, but01.2 extends admission to carry the retained validated candidate reference and expected revision; the worker applies that exact candidate then hardlinks/recreates. Calling the old observe/build-after-save helper would violate candidate ownership. Protected targets use08.3's configuration handoff or truthful host prerequisite rather than generic recreation. Serialize modifications with plans 01/02/08; no new vault operation is invented. +- [ ] **Step 5 — GREEN:** repeat RED and `pnpm exec vitest run --project node apps/api/src/services/integration-config-import.test.ts apps/api/src/services/integration-config-store.test.ts`; `pnpm exec vitest run --project web apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx apps/web/src/components/admin/shared/__tests__/SchemaConfigForm.test.ts`. Preserve `previous_applied_not_proven`, fresh `saved.owns()` classification, CAS and bounded diagnostics. +- [ ] **Step 6 — commit/review gate:** proposed commit `fix(admin): enforce effective configuration ownership`. Review provider receives rotated vault value, all writers share policy, no host credential read/export, and UI/API cannot report a shadowed write as success. + +## Task 3: Source/artifact standalone installation (I03) + +**Files:** Create `packages/core/src/ops/standalone-integrations.ts`, `apps/ops-agent/src/standalone-integration-runtime.ts`, `apps/ops-agent/src/standalone-integration-runtime.test.ts`, `apps/api/src/services/standalone-integrations/install.ts`, `install.test.ts`, `apps/api/src/routes/admin-standalone-integrations.ts`, `packages/core/src/api/contracts/integration-admin/standalone-integrations.ts`, `apps/web/src/components/admin/extensions/StandaloneInstallDialog.tsx`, `StandaloneInstallDialog.test.tsx`; modify `packages/integration-framework/src/installer.ts`, `apps/api/src/services/extension-installer.ts`, `apps/api/src/services/extension-install-journal.ts`, `apps/api/src/db/installed-integration-schema.ts`, `installed-extension-schema.ts` in the same schema directory. Controller registers ops/runtime/jobs/contracts/migration and the integration-framework workspace dependency in `apps/ops-agent/package.json` if the focused runtime imports that installer; only controller updates the lockfile. + +**Interfaces:** + +- `StandaloneInstallSource = {kind:"artifact";url:string;sha256:string} | {kind:"source";repository:string;ref?:string} | {kind:"uploaded-artifact";stageId:string;expectedSha256:string}`. Uploaded stage ID is UUID, digest lowercase 64-hex, purpose is exactly `integration-artifact`, and compressed/expanded limits are each **200*1024*1024 bytes**. Browser inputs never include `ownerId`, paths or arbitrary acquisition headers. HTTPS allowlisted repository acquisition, immutable 40-hex resolved commit in preview; remote artifact requires 64-hex SHA-256. No local path, shell fragment, browser build toolchain or executable entry point. +- Typed ops `integration.inspectInstall` consumes `{source:StandaloneInstallSource,ownerId:string}` (owner is added by authenticated API admission, never copied from browser body) and returns `StandaloneInstallInspection={inspectionId:string,sourceKind:StandaloneInstallSource["kind"],integrationId:string,version:string,contentDigest:string,commit:string|null,stageId:string|null,archiveSha256:string|null,expiresAt:string,denials:string[]}`; `integration.install` consumes `{inspectionId,ownerId:string,expectedContentDigest:string,operationId}` and returns an opaque transaction receipt with previous/current digests. `integration.settle` consumes `{transactionId,decision:"commit"|"restore",expectedContentDigest}`. Retain transaction journal and completed result; stage paths never leave agent. +- An inspection is an immutable private record binding authenticated principal, source variant/stage UUID/archive digest, content digest, protocol/manifest identity, resolved source commit if applicable, destination ownership and expiry (10 minutes, capped by stage expiry). Install revalidates all bindings; its fingerprint includes them and cannot accept a substitute stage/digest. API POST preview body is `{source:StandaloneInstallSource}`; install body `{inspectionId,expectedContentDigest}`. Authenticated API injects actor/owner into the typed ops envelope, and the agent requires API role. Persist exact child phase/key/fingerprint/deadline before each agent admission via02's child protocol; lost responses use lookup only. +- Agent-side `inspectUploadedIntegration({store:ArtifactTransferStore,stageId:string,ownerId:string,expectedSha256:string,identity:StageClaimIdentity,signal:AbortSignal}):Promise` uses06.2's `claimStage({stageId,ownerId,purpose:"integration-artifact",use:"inspect-integration",identity,expectedSha256})` and `withClaimedIntegrationArtifact({claim,signal},consume)` against the registered `integration-artifact` validator, never turns a stage into a URL/local source string. Refactor installer artifact preparation to accept an already-open immutable byte source: `prepareIntegrationArtifact({source:AsyncIterable,expectedSha256:string,maxArtifactBytes:number,maxExpandedBytes:number,signal:AbortSignal}, context:IntegrationArtifactPreparationContext):Promise` in `packages/integration-framework/src/installer.ts`; context and result are server-local branded handles holding existing fixed install-root/stage/journal ownership. No caller-selected extraction path crosses HTTP. The store adapter owns the descriptor and supplies `{source,signal,bytes,sha256}`; the agent awaits full validation inside that callback; stream gzip/tar decoding preserves installer path/PAX/header/member/manifest/declarative limits (4096 headers,256 PAX records,1024-byte paths,32 components,255-byte components). This removes the current whole-file `gunzipSync(readFileSync(...))` allocation for uploaded artifacts; CLI/URL paths call the same validator, retaining their existing source policies. +- Claim settlement consumes06.2's exact `ArtifactStageClaim`, `StageClaimIdentity={executionId,operationKey,fingerprint,executionDeadlineAt}` and `StageInspectionBinding={inspectionId,stageId,sha256,bytes,policyRevision,validationFingerprint,expiresAt}` from `packages/core/src/ops/artifact-transfer.ts`. Identity comes from02's admitted context, never browser input. Private standalone inspection retains that binding. Preview's uploaded-byte claim settles `settleStageClaim({claim,settlement:{outcome:"validated",inspection}})` to ready; invalid bytes settle `{outcome:"failed",receiptId}`; proven no-effect cancellation settles `{outcome:"released",evidence:"no_effect"}`. Stage bytes are immutable after upload, and later PUT is forbidden. +- Install calls `claimStage` with `use:"install-integration"`, its own retained identity, exact owner/purpose/digest and the retained `inspection` binding. It holds that claim through directory publish, API reload, DB bookkeeping and `integration.settle`. Confirmed commit/reload/bookkeeping settles `{outcome:"consumed",receiptId}`; exact-owner proven restoration settles `{outcome:"released",evidence:"restored"}` (store permits reuse only if unexpired); irrecoverable known failure settles `{outcome:"failed",receiptId}`. Lost admission/settlement response, process death or ambiguous restore settles/retains `{outcome:"unknown",receiptId}` and remains claimed. No release in finally or deletion on deadline. Recover the same identity through typed `artifactTransfer.inspectClaim {stageId,ownerId,identity}` and retained integration child receipts; a missing response is not missing work. Keep quota/rollback evidence pinned through authoritative settlement, including past the execution deadline. +- API `POST /api/admin/standalone-integrations/preview` and `/install` use caller-retained mutation keys; install returns `{jobId,operationId}`. A `standalone.install` durable job coordinates agent publication, API reload and bookkeeping. Preserve existing extension trust/admission and component uniqueness; source acquisitions are declarative metadata, never a trust promotion. + +- [ ] **Step 1 — tests:** actual assembled ops dispatch with synthetic artifact/source fixtures validates checksums, pinned commit, protocol major/minor, manifest ID and no-code contract. Reject traversal, symlink/hardlink, duplicate archive entries, bombs, executable frontend/backend files, local paths, off-allowlist/private-network redirects, stale inspection and checksum mismatch; no target change on any refusal. Exercise the real06 upload→purpose validation→immutable inspection→install→reload→settle flow. Wrong owner/purpose/digest,expired/replayed stage and bytes changed after inspection produce no destination effect. Pause after publish and lose the response: the claim remains pinned across restart until exact receipt reconciliation; verify commit consumes once and proven restore alone permits retry. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/ops-agent/src/standalone-integration-runtime.test.ts apps/api/src/services/standalone-integrations/install.test.ts`; behavioral admission test fails because the existing admin surface cannot install a standalone source/artifact through typed ops. +- [ ] **Step 3 — minimal implementation:** factor reusable prepare/validate/publish installer stages so preview pins the exact artifact or source snapshot, never re-resolves a mutable ref at execution. Preserve same-filesystem stage/rename, descriptor bounds, install journal and conditional rollback. Agent owns filesystem acquisition/publication; API owns DB/reload. Use bounded acquisition limits already enforced by installer, with retained output/deadline/cancel support from plan 02. +- [ ] **Step 4 — ownership and recovery:** set `installed_extension.installationKind="standalone"`, `sourceTrust="community"`, unique `id="local-"+UUID`; create component link and installed-integration row in one transaction only after validated publish/reload. Store provenance as the strict union `{kind:"artifact",url,artifactSha256,contentDigest}|{kind:"source",url,commit,contentDigest}|{kind:"uploaded-artifact",stageId,artifactSha256,contentDigest}`. The latter keeps archive provenance after stage cleanup without inventing a URL; stageId is an opaque historical reference, never a reusable download capability. Existing catalog rows default `installationKind="catalog"`; never write invented artifact URLs or substitute catalog versions. Make legacy extension install and CLI installer share the same per-component filesystem lock/journal boundary so they cannot race new ops. +- [ ] **Step 5 — GREEN:** repeat RED, existing `apps/api/src/services/__tests__/extension-installer.test.ts`, and `pnpm exec vitest run --project web apps/web/src/components/admin/extensions/StandaloneInstallDialog.test.tsx`. Fault-inject DB commit, reload, agent settlement, disconnect and cancellation; only restore under exact current digest/transaction ownership, otherwise keep reconciliation required. Source-to-artifact packaging stays declarative and does not execute package scripts. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(admin): install standalone declarative integrations`. Controller checks assembled runtime operation inventory and authority gate, journal restart recovery and 06 stage compatibility before generated-contract integration. + +## Task 4: Legacy inventory, explicit adoption and safe removal (I04) + +**Files:** Create `apps/api/src/services/standalone-integrations/inventory.ts`, `ownership.ts`, `ownership.test.ts`, `ownership-postgres.test.ts`, `apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.tsx`, `StandaloneIntegrationsPanel.test.tsx`; extend Task 3 agent/runtime/contracts/routes, `apps/api/src/services/extension-recovery.ts`, `apps/api/src/services/extension-component-ownership.ts`, and `apps/web/src/components/admin/extensions/ExtensionStorePage.tsx`. + +**Interfaces:** + +- `StandaloneInventoryEntry={integrationId,version:string|null,ownership:"extension"|"standalone"|"unmanaged"|"conflict",extensionId:string|null,contentDigest:string|null,provenance:"artifact"|"source"|"uploaded-artifact"|"legacy_unknown",adoptable:boolean,removable:boolean,denialReason:string|null}`. Metadata inventory includes invalid/quarantined entries by opaque ID with bounded reasons, no file paths. Catalog outage does not hide local inventory. +- Ops `integration.inventory`, `integration.inspectOwnership`, `integration.remove` accept only IDs/opaque inspection IDs plus expected content digest. API `POST /api/admin/standalone-integrations/:id/adopt` and `/remove` admit jobs against exact inspected digest/ownership; adoption records `legacy_unknown` when no authenticated source receipt exists. +- `adoptStandaloneIntegration(input,tx)` creates a new standalone extension owner/component/provenance row atomically, never attaches to an unrelated store record. Such rows have `manifest:null` until supplied a verified distributable manifest; update action is unavailable with `source_unknown`, removal still works through stored receipt. + +- [ ] **Step 1 — tests:** inventory unrecorded CLI directory, DB-only stale row, invalid legacy code directory, genuine owned component and catalog outage. Race directory replacement, foreign extension claim, delete/reinsert and repeated removal; reject stale digest/owner, preserve unrelated files/rows and retain valid removal controls despite failed catalog enrichment. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/services/standalone-integrations/ownership.test.ts apps/ops-agent/src/standalone-integration-runtime.test.ts`; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/services/standalone-integrations/ownership-postgres.test.ts`. Baseline misses filesystem-only rows and cannot adopt/remove them with truthful ownership. +- [ ] **Step 3 — minimal implementation:** merge independent filesystem/DB inventories; inspect only trusted declarative manifests/assets, record content digest and source receipt when proven. Adoption never downloads/runs code. Unmanaged valid entries offer explicit adopt or confirmed remove; conflicting/built-in/foreign-owned entries deny both. Invalid executable legacy entries remain inert and offer bounded host cleanup guidance until a safely confined removal inspection can prove ownership. +- [ ] **Step 4 — removal transaction:** stage owned directory into journaled quarantine under the common install lock, reload and verify absence, then remove component/bookkeeping and settle quarantine. Rollback requires unchanged owner/digest and proven previous generation; DB/host unknown outcomes keep receipt and reconciliation action. Repeated same operation returns retained result; no browser retry on unknown. +- [ ] **Step 5 — GREEN:** repeat RED and `pnpm exec vitest run --project web apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.test.tsx apps/web/src/components/admin/extensions/ExtensionStorePage.test.tsx`. Test `hasUpdate:null` remains “unknown”, no fake source/version, and A-to-B view changes cannot receive old completion feedback. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(admin): adopt and remove legacy standalone installations`. Review catalog-owned collision, restart recovery, safe null-manifest uninstall, CLI common lock, and absence of privileged file operations in API/browser code. + +## Task 5: Configuration migration inventory, preview and apply (I05) + +**Files:** Create `apps/api/src/services/configuration-migration/inventory.ts`, `preview.ts`, `apply.ts`, `migration.test.ts`, `migration-postgres.test.ts`, `apps/api/src/routes/admin-configuration-migration.ts`, `packages/core/src/api/contracts/integration-admin/configuration-migration.ts`, `apps/web/src/components/admin/settings/ConfigurationMigrationPanel.tsx`, `ConfigurationMigrationPanel.test.tsx`; modify `apps/api/src/routes/admin-settings.ts`, `apps/web/src/components/admin/settings/SystemSettings.tsx`, `apps/api/src/services/integration-config-import.ts`, `docs/docs/administration/integrations-administration.md` via controller's shared-file queue. + +**Interfaces:** + +- `ConfigurationMigrationDocument={format:"openmapx-configuration",version:1,exportedAt,settings:Record,platform?:PlatformConfiguration,integrations:AdminIntegrationConfigEntry[],serviceConfigs:Array<{serviceId,config:Record}>,bindings:Array<{integrationId,capability,serviceId}>,installations:StandaloneInventoryEntry[],prerequisites:MigrationPrerequisite[]}`. Only validated declared non-secret values; no masking sentinel, raw env/file values, credentials or arbitrary unknown DB metadata. +- `MigrationPrerequisite={scope:"integration"|"service"|"platform"|"account",id,key?:string,source:ConfigurationOwnership["source"]|"missing",reason:"host_owned"|"credential_required"|"installation_required"|"unsupported"|"protected_bootstrap"}`. Inventory includes all declared fields and effective sources; export includes only portable DB-owned non-secrets and effective default metadata, never presents itself as full backup. +- `previewConfigurationMigration(document):Promise<{previewId,fingerprint,expectedCurrentRevisionId,changes,prerequisites,denials,expiresAt}>`; `applyConfigurationMigration({previewId,fingerprint,idempotencyKey,actorId}):Promise<{operationId,steps:Array<{scope,id,status:"applied"|"failed"|"pending"|"unknown"}>,outcome:"applied"|"partial"|"failed"|"unknown"}>`. Preview is actor-bound, expires after 10 minutes, max JSON 1 MiB/1,000 fields; confirmation revalidates schema versions, input/authority fingerprint and current revisions. + +- [ ] **Step 1 — tests:** export synthetic settings/integrations/service configs/bindings plus file/env/vault-owned fields; assert explicit omissions and required credentials, no values from host sources, no `***`, keys/paths/tokens or unknown metadata. Import against missing services, changed schema, unknown integration, secret injection, stale preview and changed active configuration; zero mutation on denied/stale preview. Feed a 1 MiB legal migration document whose individually legal fields and unchanged vault rows exceed the 512 KiB sealed envelope only after UTF-8/JSON escaping; assert no selected row, binding,auth draft,receipt-success or job admission changes. Fail one service config read while every other read succeeds and assert no incomplete generation. Race env ownership/current revision between preview and apply; assert conflict and no write. After each rejected candidate, unrelated preparation remains healthy. Test a legacy-invalid snapshot correction succeeds only when its complete replacement is valid. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/services/configuration-migration/migration.test.ts`; baseline exports fragmented JSON without a complete inventory/preview or prerequisites. +- [ ] **Step 3 — minimal implementation:** add GET `/api/admin/configuration-migration/inventory`, GET `/export`, POST `/preview`, POST `/apply`. This task defines `collectPortableServiceConfiguration():Promise` in `inventory.ts` from declared DB rows and01.2 strict service ownership. Import constructs01.2 `ProspectiveConfigurationOverlay={purpose:"migration",changes}`: service/integration config set/unset variants, selection and platform variants only; ≤512 change records,≤128 keys per set/unset. Auth drafts and binding changes are the finite transaction participants below, not undeclared overlay kinds. The1MiB/1000-field input ceiling does not waive these tighter candidate bounds; preview reports exact limits before confirmation. Preview invokes `prepareProspectiveConfiguration` without persistence and stores the actor-bound input/source fingerprint privately. Apply reacquires01.2 ownership, strictly revalidates the exact expected revision and all source identities, prepares the **whole** candidate, then persists through01.2's candidate transaction. Do not loop over independently committing `persistServiceConfigPatch` or reconstruct a payload after rows changed. Platform projection uses plan 08's `PlatformConfiguration`/`resolvePlatformConfiguration` and preserves unrelated generation fields. Core auth settings reuse plan 03 validation but stage pending values only; explicit policy Apply records activation version 1 separately. One bounded durable coordinator owns plan, identity and per-step receipts, not arbitrary client JSON execution. Prepare/validate before writing any portable DB row or pending auth draft, persist the complete selected desired batch under the one candidate/writer boundary, then apply its exact sealed generation; apply binding reload at its documented boundary and label staged auth policy as pending until its separate explicit Apply. +- [ ] **Step 4 — honest effects:** prerequisites must be resolved before dependent changes; do not silently skip conflicts. A cross-domain partial or unknown result lists exact scopes/IDs without secrets; use existing conditional restore only where DB ownership and host noncommit are proven. Never claim an all-or-nothing rollback spanning auth, service recreation and external effects. Existing raw settings/import routes call the same validation/application functions so they cannot bypass effective auth/config validation. +- [ ] **Step 5 — GREEN:** repeat RED; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/services/configuration-migration/migration-postgres.test.ts`; `pnpm exec vitest run --project web apps/web/src/components/admin/settings/ConfigurationMigrationPanel.test.tsx apps/web/src/components/admin/settings/__tests__/IntegrationTransfer.test.tsx`. Assert preview explicit included/excluded values and partial/unknown labels; actual API→serialized SDK result, not canned import success. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(admin): preview portable configuration migrations`. Review dependency order with plan 08 and account settings, preservation of unrelated keys, no silent defaulting after read failure, and 1 MiB bounded parse. + +## Task 6: Protected encrypted vault credential transfer (I06) + +**Files:** Create `apps/api/src/services/configuration-migration/credential-transfer.ts`, `credential-transfer.test.ts`, `credential-transfer-postgres.test.ts`, `apps/api/src/routes/admin-credential-transfer.ts`, `packages/core/src/api/contracts/integration-admin/credential-transfer.ts`, `apps/web/src/components/admin/settings/CredentialTransferDialog.tsx`, `CredentialTransferDialog.test.tsx`; extend Task 5 inventory/preview and Task 2 strict vault functions; controller owns ops/schema/settings integration. + +**Interfaces:** + +- `CredentialTransferEnvelope = {format:"openmapx-vault-transfer",version:1,kdf:{name:"scrypt",N:32768,r:8,p:1,salt:string},cipher:{name:"aes-256-gcm",iv:string,tag:string},ciphertext:string}` with base64url salt (16 bytes), IV (12 bytes), tag (16 bytes). Authenticate canonical format/version/KDF/cipher metadata as AAD (excluding ciphertext/tag). Reject all other KDF parameters before derivation; native async `scrypt` has `maxmem=64*1024*1024`, at most one derivation per actor and two per API process. Envelope and decrypted payload each ≤1 MiB, ≤1,000 entries, each ordinary value ≤16 KiB UTF-8, the sole managed `service/data-manager/TRANSITOUS_API_KEYS` value ≤64 KiB UTF-8 canonical JSON (plan05.6 validator:≤1000 keys,keys≤512 Unicode code points with invalid Unicode rejected,each value≤8192 UTF-8 bytes), passphrase 12–1,024 UTF-8 bytes. +- Plaintext (only transient encrypted-transfer memory) `{version:1,entries:Array<{scope:"integration"|"service",id,key,value}>}`. Export **only declared effective vault-owned rows** via strict decryption, including service vault rows; shadowed stored credentials remain excluded/manual prerequisites, never call a host secret reader or copy auth, Redis, bootstrap/master keys, environment/config.json secrets or plaintext settings secrets. Inventory marks these as manual/protected prerequisites. +- POST `/api/admin/credential-transfer/export` body `{selection:Array<{scope,id,key}>,passphrase,confirmed:true}` consumes purpose `credentials.export`, target `"vault-transfer"`, and returns the strict `CredentialTransferEnvelope` JSON object with `Content-Type:application/json` and no-store. The serialized envelope itself is≤1 MiB; generated canonical JSON consumers validate the schema, never decode encrypted bytes through SSE/text. UI may create a small encrypted-file Blob from this bounded JSON, filename `openmapx-vault-transfer.json.enc`, then revoke its object URL. File input checks `File.size≤1 MiB` before bounded `.text()`/JSON parse; this exception never covers large integration/prepared/privacy artifacts. POST `/preview` body `{envelope,passphrase}` consumes `credentials.import` target `"vault-transfer"` and returns field IDs/conflicts only, storing a short-lived encrypted staging record under the destination vault key; POST `/apply` consumes a fresh `credentials.import` proof with target `previewId`, expected revisions and idempotency key. Export retry after a lost response requires a new explicit ceremony; no server replay store of plaintext/passphrase. + +- Canonical integration-admin operation IDs are `exportAdminCredentialTransfer`, `previewAdminCredentialTransfer`, `chooseAdminCredentialTransferFields`, `applyAdminCredentialTransfer`; export uses `adminJson(credentialTransferEnvelopeSchema)` and the generated `exportAdminCredentialTransfer(body:CredentialExportInput,options?:AdminSdkRequestOptions):Promise`. `CredentialExportInput` is the exact export body above. The dialog unwraps the status/data union and passes `{client,signal,maxJsonBytes:1048576,maxErrorBytes:65536}` through existing ApiClient transport; it never calls raw fetch. Responses, parser and file-size checks measure UTF-8 bytes, including JSON/base64 expansion; a payload within1MiB that yields an envelope over1MiB is rejected without producing a downloadable file. All other transfer operations are canonical bounded JSON with the input/projection shapes below. +- `CredentialImportChoices={entries:Array<{scope:"integration"|"service";id:string;key:string;action:"import"|"skip"}>,feedMap?:{catalogRevisionId:string;expectedCredentialsRevision:string;setKeys:string[];deleteKeys:string[]}}`. POST `/preview` accepts `{envelope,passphrase,choices?:CredentialImportChoices}`; omitted choices produce inventory-only review, not an applyable preview. POST `/preview/:previewId/choices` accepts `{choices,expectedIntentFingerprint}` and explicit mutation identity; it CAS-checks the old actor-bound intent, revalidates the encrypted retained stage and returns a new opaque fingerprint, no passphrase needed. A changed choice invalidates all old intent proofs; it never rewrites an admitted preview. Response `CredentialImportPreview={previewId:string,intentFingerprint:string,expectedCurrentRevisionId:string|null,expiresAt:string,entries:Array<{scope,id,key,status:"selected"|"skipped"|"denied",reason?:string}>,feedMap?:FeedCredentialChangePreview,applyable:boolean}` contains IDs/counts/ownership/revisions only. Entry counts≤1000; combined feed set/delete choices≤1000 (the direct05.6 HTTP editor keeps its separate100-change request bound); ordinary overwrite requires explicit import choice; duplicate/unknown choices reject. Preview/choices is actor-bound and expires10 minutes. Request parser allows only bounded envelope+passphrase framing (1 MiB+8 KiB); enforce each envelope/plaintext limit separately. +- Managed transfer merge is specialized: `decodeTransferredFeedCredentialMap(text:string):Readonly>` in `credential-transfer.ts` rejects duplicate JSON member names, unsafe keys, invalid Unicode, arrays/non-string values and over64KiB input before returning a record; pass it to05.6 `validateFeedCredentialMap`. Canonical final limits use binding§5.3:≤512 Unicode code points per key,≤8192 UTF-8 bytes per value,≤1000 keys and≤65536 UTF-8 bytes including escaping; do not silently use B's earlier JS-code-unit wording; `setKeys` select imported values, absent keys are preserved, and only explicit `deleteKeys` delete current entries. A key cannot be both set/delete; unknown-to-catalog keys may only modify/delete already-retained keys per05.6 policy, not introduce foreign credential IDs. Call05.6 `prepareFeedCredentialImportPatch({actorId,catalogRevisionId,expectedCredentialsRevision,expectedCurrentRevisionId,changes,importPreviewId,operationKey,signal?})`. Its `PreparedFeedCredentialPatch` is private; combine its service-secret overlay with the ordinary selected overlays for **one**01.2 candidate, not one candidate per entry. `PreparedFeedCredentialPatch` must expose readonly private `overlayChange:Extract` to this coordinator; it is checked against the candidate by05.6 persistence, never serialized. Combine it into `{purpose:"migration",changes}` only while the validated import authorization context exists; direct feed uses `purpose:"transit-credentials"`. The finite secret variant is `{kind:"service-secret",serviceId:"data-manager",key:"TRANSITOUS_API_KEYS",action:"set",value:canonicalJson}`, including encrypted `{}` on last-key deletion. This required05.6 export and the commit participant below prevent an untyped whole-map bypass. +- Apply body is `{previewId,intentFingerprint,expectedCurrentRevisionId,importProofToken:string,transitProofToken?:string}` plus `Idempotency-Key`. Proof tokens are03.3 challenge IDs with current-session+HttpOnly nonce verification, never standalone bearer authority. For an import with the managed map, `AdminActionReauthentication.consumeMany` requires BOTH `{purpose:"credentials.import",targetId:previewId,intentFingerprint}` AND `{purpose:"transit.credentials",targetId:catalogRevisionId,intentFingerprint}` in the same target-write transaction. Start both before one login via03.3's bounded two-challenge ceremony. Mere export/decrypt preview does not mutate feed credentials; apply cannot reuse its preview-import proof. Ordinary imports consume only fresh `credentials.import`. Public keys/fingerprints never contain secret-derived hashes; retain secret comparison/private source material only encrypted/server-side. +- Apply order: acquire01.2's global configuration writer then sorted integration locks on its dedicated connection; construct the managed `overlayChange` and ordinary patches under the SAME participant. Non-consuming `checkMany`, whole-candidate schema/source/size/exact-seal/queue/private-receipt preflight all precede remote admission. If the managed map is selected, invoke01.2's§5.4 maintenance protocol through `persistAndApplyProspectiveConfiguration`: agent owns the frozen target closure, obtains02.3's durable nonblocking DM fence, and returns terminal prepared. API neither acquires local target authority nor mounts its root/private keys. Recheck pair/map/source/config/source-witness/proofs **under this remote fence**, then atomically consume both proofs, persist ordinary selected rows plus05.6 `persistPreparedFeedCredentialPatch({patch,candidate,writer})`, invalidate catalog validation and checkpoint the exact continue decision in ONE transaction. Managed revision advances on equal-value writes/last deletion to encrypted `{}`. Persist bindings/decisions/attempts before POST. Known rollback permits exact unused abandon only; unknown save/transport stays fenced and lookup-only, with no proof replay or optimistic release. API restart derives its decision from the retained checkpoint/private candidate, not new desired rows. +- Continue the exact existing stack.render/hardlink/DM-recreation plan under that SAME maintenanceId and borrowed root; no new target root/second maintenance flag. Guard survives DM recreation and blocks unrelated API/cron/CLI/data admissions at startup. Complete requires exact generation/mount/readiness and all child settlement; partial/unknown remains visible. Generic `setServiceSecret`/DELETE never writes the managed map. Ordinary-only imports use the same candidate producer and invoke maintenance when their frozen follow-up requires DM/protected recreation. No added table/migration:01.2 uses admin_job.result/private candidate,02.3 reserves jobs guard metadata/index in0042;0050 remains credential staging. + +- [ ] **Step 1 — tests:** cross-instance roundtrip with different vault master keys; wrong passphrase, tampered metadata/tag/ciphertext, oversized KDF/input, duplicates/unknown scope/key, file/env-owned target, changed vault revision, missing vault key, exhausted crypto concurrency, expired stage and repeated apply. Assert rejection before mutation/host read, existing target values retained, no secret/passphrase in receipts/logs/audit/browser storage and no partial desired-row import. Real route tests import a valid20 KiB managed map with both proofs, preserve an unrelated/orphaned existing credential, then explicitly delete it; missing/swapped/replayed proofs and stale catalog/map/config/source yield zero writes and zero invalidation. A 64 KiB map boundary includes UTF-8/punctuation/escaping and retained keys. Mixed ordinary+managed imports that exceed the final512 KiB envelope fail before either scope writes or consumes proof; a failed specialized CAS rolls back all ordinary rows and both proof consumptions. Changing credentials invalidates previously valid catalog evidence. +- [ ] **Remote maintenance acceptance (R2-02):** Use real API/agent/DM adapters and PostgreSQL for a mixed ordinary+managed import racing catalog activation. Pause after final verification/before DB commit; assert exclusive valid commit or zero selected-row/proof/invalidation mutation. Lose prepare/decision POST and kill API before/after save; recreate DM with retained guard. Assert one root/decision, no nested root/self-conflict, startup admission denial, no second generation/pin transition or proof double-consume, and no release on unknown. These extend01.2/05.6 acceptance; they are planned, not executed here. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/services/configuration-migration/credential-transfer.test.ts`; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/services/configuration-migration/credential-transfer-postgres.test.ts`. Existing unprotected JSON export must not be treated as credential transfer. +- [ ] **Step 3 — minimal implementation:** use built-in crypto, authenticated envelope and strict bounds; no new cryptographic package. Decrypt only after successful purpose-bound reauth and size validation. Destination validates declarations/ownership and whole prospective candidate, previews explicit choices through the contracts above, and commits selected vault entries only through01.2 and05.6 transaction-bound adapters. All target writes are atomic; later host activation can be partial/unknown and is separately reported. Passphrases stay only in request/component memory and are cleared on completion/close; best-effort zero byte buffers, no promise of erasing JS strings. +- [ ] **Step 4 — apply and cleanup:** integration runtime activation uses Tasks1/2 against the retained candidate revision. Service activation retains `service.apply` with `configurationKind:"vault"` and the exact candidate reference, expected revision and durable operation identity, then01.2 hardlinks/recreation under the retained maintenance identity when present; protected targets consume08.3's same guard/root, never an independent acquisition. Existing `vault.apply` commits serviceSecrets but does not recreate containers. For a combined batch,01.2 publishes the already-sealed full candidate as `stack.render`; do not call old `applyTrustedConfigurationWithEvidence/buildConfigurationInput` after saving, and do not reseal from later rows. Preserve `TrustedConfigurationApplyOutcome` plus per-scope applied/pending/unknown. Keep encrypted import stage10 minutes for unadmitted previews; active/unknown admissions retain candidate/evidence until authoritative settlement. Cleanup deletes ciphertext only after proven apply/cancel/expiry with no active ownership. Passphrases/plaintext are never journaled; no automatic request/mutation retry. +- [ ] **Step 5 — GREEN:** repeat RED and `pnpm exec vitest run --project web apps/web/src/components/admin/settings/CredentialTransferDialog.test.tsx`; `pnpm -C apps/api check-types`; `pnpm -C apps/ops-agent check-types`; `pnpm -C packages/core check-types`; `pnpm -C apps/web check-types`. Controller runs real synthetic source→destination transfer and provider/service activation plus local desktop/mobile inspection. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(admin): transfer vault credentials with bounded authenticated encryption`. Controller must review crypto/AAD framing, action reauth, stage expiry, concurrent overwrite, apply outcomes and absence of host credential reads before full gates/codegen/builds/CI. + +## Coverage and self-review + +I01 → Task 1; I02 → Task 2; I03 → Task 3; I04 → Task 4; I05 → Task 5; I06 → Task 6. Each is a planned independently verifiable deliverable, not future work. All five Review Focus cases have causal tests. + +Planning self-review fixed: treating reload count as effective binding proof, shadowed rotation success, catalog-ID adoption of unrelated files, null-manifest removal, import source leakage, and weak/unbounded credential envelopes. Controller finalized the master's append-only migration allocation and shared stage adapter: uploaded standalone artifacts use Plan06 purpose integration-artifact, while encrypted envelopes <=1MiB use API-owned encrypted staging. Shared install-lock/runtime ownership and serialized contract generation remain binding. No tests or runtime checks were executed during planning. diff --git a/docs/superpowers/plans/2026-10-23-admin-05-transit-and-data.md b/docs/superpowers/plans/2026-10-23-admin-05-transit-and-data.md new file mode 100644 index 000000000..9b016a21c --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-05-transit-and-data.md @@ -0,0 +1,194 @@ +# Transit and Data Administration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Deliver D01-D09: reversible sources, complete discovery, scoped sync, reviewed paired pins, supported Overture recovery, evidence-based inventory and safe feed credentials. + +**Architecture:** Extend the existing desired-source overlay, data-manager pipeline, typed ops authority and server-authored operation catalogue. Persist a reviewed Transitous/GBFS generation separately from activation, bind validation to the exact inactive-slot candidate, and expose server decisions through canonical generated clients. + +**Tech Stack:** TypeScript, Fastify, Zod, React/MUI, TanStack Query, Drizzle/PostgreSQL, typed ops-agent, Vitest; existing MOTIS A/B pipeline and credential vault. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md`; controller: `docs/superpowers/plans/2026-10-23-admin-instance-management.md`. Audit base `ef896ceae876959180116cf61ffcf643b88ebffe`. + +## Global Constraints + +- All product work stays in this worktree; preserve unrelated work and existing public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source policy boundaries. Community integrations remain declarative; no uploaded code execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain behind typed ops-agent operations. No generic command, arbitrary path or raw environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth declarations, strict schemas, audit events, explicit mutation identities and appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, durable operation identities, retained results, unknown outcomes and cancellation fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database and runtime checks, independent review, production builds and PR CI. Report any unavailable environment gate distinctly; do not replace runtime evidence with mocked assertions. +- Read root/web `AGENTS.md`, `CONTRIBUTING.md#quality-bar`, and developer `api-surface.md`, `service-manifest.md`, `integration-system.md`, `writing-an-integration.md`, `building-an-external-extension.md`. Resolve bundled Next.js docs before UI implementation. + +## Review Focus + +1. Disable followed by Enable must retain an operator source's exact URL/license; removal is a distinct confirmed action (Task 1). +2. A 501st source and a discovery duplicate must remain searchable/actionable under the installed catalog ID, including origin changes during requests (Task 2). +3. A replaced proposal, changed policy/credentials/image, interrupted activation or cron run must never approve stale or unvalidated pins (Task 3). +4. Empty output directories, multiple OSM extracts and stale build records must not imply readiness (Task 5). +5. Catalog disappearance, corrupt credentials and file/environment ownership must not erase secrets or report a successful vault save (Task 6). + +--- + +## Ownership, dependencies and fixed interface decisions + +- **Controller-owned, serialized edits:** `packages/core/src/ops/{contract,contract.test,index}.ts`, `packages/core/src/api/contracts/{index.ts,data-manager-admin/operations.ts,data-manager-admin/transit.ts,service-admin/operations.ts,service-admin/data.ts}`, `packages/core/src/services/data-manager/contracts/{operations,transit}.ts`, `apps/api/src/routes/{index,data-manager}.ts`, `apps/ops-agent/src/{index,policy,administrative-runtime}.ts`, `services/data-manager/src/{api,cron}.ts`, and both SDK/OpenAPI families. Tasks below specify their required deltas; workers supply them to the controller rather than racing shared files. +- **Additional overlap:** `apps/api/src/services/admin-operation-catalog.ts` with Plans 02/09; `packages/cli/src/commands/data.ts` with Plans 02/06/09; `packages/cli/src/lib/service-builds.ts` with Plans 01/09; `services/data-manager/service.json` with Plans 04/08; `DataWorkflowsPage.tsx` with Plan 06. Controller assigns sequential landing windows. No migration index is allocated here; vault changes reuse the existing table. +- **Plan 02 prerequisite:** retain `OpsExecutionContext.signal`, `emitLog(stream: "stdout" | "stderr", message: string): void` and durable operation/job IDs. This plan needs cancellable multi-hour validation and Overture jobs, terminal failure/partial/unknown visibility and observer-independent execution. Do not implement another runner or await acceptance as completion. +- **Plan 06 Task 1 prerequisite for Tasks 1/3/5:** Its shared filesystem target-lease foundation precedes transit mutation/validation and its artifact definitions precede inventory. `@openmapx/core/build-artifacts` owns `BuildRecord`, `BundleManifest`, `buildArtifactSpec(serviceId)` and manifest/checksum validation. Task 5 owns inventory projection and writing source fingerprints, not duplicate artifact definitions. Controller may land that foundation ahead of the remainder of Plan 06. +- **Plan 03 prerequisite for Task 6:** Consume its reauthentication purpose exactly as `transit.credentials`, bound to the current `catalogRevisionId` and credential mutation intent; Plan 03 owns token issuance/verification, expiry and replay prevention. Test that a generic or other-purpose reauthentication receipt and one bound to an older catalog revision are rejected. Do not implement a second reauthentication flow. + +- **Plan 01.2 prerequisite for Task 6:** consume `withAdministrativeConfigurationWriter`, `prepareProspectiveConfiguration`, `persistAndApplyProspectiveConfiguration`, `AdministrativeConfigurationWriter`, `ValidatedConfigurationCandidate`, `ProspectiveConfigurationOverlay` and `ProspectiveConfigurationReceipt` from `docs/superpowers/specs/2026-10-23-admin-control-protocols.md`§5, using strict full-state reads and the same serialized administrative writer through persistence/apply. Existing `setServiceSecret` is not a CAS or a prospective validator. Plans 04.2/04.6 consume the managed-secret guard and coordinated feed-map adapter exported by Task 6; Plan 08 consumes the same prospective validator. No second vault, snapshot collector or writer.04.2 lands its generic guard/projection support first;05.6 adds the transit predicate and wires it in the same task as the new declaration, so04.2 has no forward runtime import of a not-yet-created module. +- **R1 ordering:** Task 7 is the paired-pin foundation and runs after06.1, before Tasks 2/6 and before any paired-pin-dependent preflight. Task 6 supplies credential revision/invalidation before Task 3 validation/activation. Task 3 consumes Task 7; it no longer creates that foundation. Preserve all seven tasks and the append-only migration allocation; this filesystem foundation adds no DB migration. +- Each task's final commit is local, after causal RED/GREEN and fresh spec/quality review; stage only its enumerated files. Root owns aggregate commits, integration checks, fresh whole-branch review and PR. Commands below are execution instructions, not receipts. + +## Task 1: Reversible desired sources and honest actions — D01 + +**Files:** Modify `services/data-manager/src/transit-sources.ts`, `services/data-manager/src/jobs/transitous-feeds-overlay.ts`, `apps/web/src/components/admin/services/TransitSourcesSection.tsx`; add `services/data-manager/__tests__/transit-source-actions.test.ts`; extend `services/data-manager/__tests__/transitous-feeds-overlay.test.ts`, `apps/api/src/routes/__tests__/data-manager-sources.test.ts`, `apps/web/src/components/admin/services/TransitSourcesSection.workflow.test.tsx`. Shared API/contract registrations are controller-owned above. + +**Interfaces:** Produce `prepareSetTransitSourceEnabled(options: {catalogDir: string; overlayPath: string; sourceId: string; enabled: boolean; countries?: string[]}): {sourceId: string; persist(): void}`. Keep overlay `version: 3`: existing `{sourceId, skip}` patches also target operator records; no format migration is needed. Add `POST /transit/sources/:sourceId/disable` internally and `/api/data-manager/transit/sources/:sourceId/disable` publicly; existing Enable delegates with `enabled: true`. DELETE removes only operator additions; catalog removal returns 409 `source_action_not_allowed`. + +**Wire:** Extend `TransitSourceRow` with `actions: {enable: boolean; disable: boolean; remove: boolean; reason: string | null}` and `desiredState: "enabled" | "disabled" | "removed"`; pending lifecycle remains distinct from active evidence. Mutation bodies retain idempotency identity and add `expectedSourceRevision: string`; list rows carry this digest of canonical desired metadata. The reservation rechecks it before persisting. + +- [ ] **RED test:** Through existing routes disable a synthetic operator feed, sync successfully, then enable it. Assert exact URL, attribution and license remain and both jobs are observable; current DELETE loses the record and Enable rejects it. Assert actual Remove erases record and associated patch/quarantine, and catalog Remove is refused without mutation. +- [ ] **Run RED:** `pnpm exec vitest run services/data-manager/__tests__/transit-source-actions.test.ts apps/api/src/routes/__tests__/data-manager-sources.test.ts` — expect the lifecycle/route assertions to fail, not fixture/import setup. +- [ ] **Implement:** Acquire the shared filesystem target lease, then reuse `reserveSourceMutation` and its single-flight-before-persist order; extend action metadata with `disable`. Reserve and revision-check atomically before `persist()`. Failed acquisition/promotion leaves desired state visible and the active source unchanged. Permit disabling/removing a retained out-of-scope source, but reject enabling it outside configured scope. Reject unknown or active-only orphan IDs instead of fabricating restorable metadata. +- [ ] **UI:** Use server action flags/reasons; Disable calls disable, Enable calls enable, Remove uses `ConfirmDialog` with the exact source name. Add pending/failed/removed labels, disable mutations during owned submission, and set `retry: false`. Test repeated clicks and desired-disabled/active-true after pipeline failure. +- [ ] **GREEN:** Run RED command plus `pnpm exec vitest run services/data-manager/__tests__/transitous-feeds-overlay.test.ts apps/web/src/components/admin/services/TransitSourcesSection.workflow.test.tsx` — all selected tests pass. +- [ ] **Commit/review:** `fix(admin): make transit source disabling reversible`; reviewer traces request → reservation → overlay → generated materialization → active manifest and checks no source-policy/relay bypass. + +## Task 2: Complete discovery and country-scoped preflight — D02, D03, D04 + +**Files:** Modify `services/data-manager/src/transit-sources.ts`, `apps/api/src/services/transit-catalog/{index.ts,types.ts,index.test.ts}`, `apps/web/src/components/admin/services/TransitSourcesSection.tsx`; create `services/data-manager/src/jobs/transitous/admin-preflight.ts`, `services/data-manager/__tests__/transitous/admin-preflight.test.ts`, `apps/web/src/components/admin/services/TransitSyncDialog.tsx`, `apps/web/src/components/admin/services/TransitSyncDialog.test.tsx`; extend `apps/api/src/routes/__tests__/data-manager-sources.test.ts`, `apps/api/src/routes/__tests__/data-manager-sync.test.ts` and source workflow tests. + +**Interfaces:** Source/catalog queries use `{search?: string; limit?: number; offset?: number; country?: string}` with default 50, max 500; response `{sources, total, limit, offset}`. Preserve source-specific filters. Stable sort by `(name, id)` before paging. Discovery rows add `installedSourceId: string | null`, `discoveryIds: string[]`; installed ID, URL, licensing and `source: "transitous"` win URL dedup. Optional discovery enrichment cannot replace provenance or erase installed records. + +**Preflight:** Produce `inspectTransitPreflight({countries, signal}: {countries: string[]; signal?: AbortSignal}): Promise` where `TransitSyncPreflight = {preflightId: string; expiresAt: string; fingerprint: string; countries: string[]; allowedCountries: string[]; result: MotisPreflightResult}`. New internal `POST /transit/sync/preflight` and browser `POST /api/data-manager/transit/sync/preflight` take only `{countries: string[]}`. Add `{preflightId, fingerprint}` to public sync admission. Internal measured-estimate diagnostic endpoint remains separate. + +- [ ] **RED test:** Seed 601 sources; fetching offset 550 returns those identities, searching for source 600 returns it with total 1, and clearing search resets offset. A matching Mobility Database URL enriches but never replaces `catalog:de:Demo`; a provider outage still returns installed rows. Test origin changes and out-of-order search responses cannot restore the old page. +- [ ] **Run RED:** `pnpm exec vitest run apps/api/src/services/transit-catalog/index.test.ts apps/api/src/routes/__tests__/data-manager-sources.test.ts apps/web/src/components/admin/services/TransitSourcesSection.workflow.test.tsx` — assert current dedup loses installed ID and current browser cannot reach source 600. +- [ ] **Implement paging:** Replace client-side filtering/pagination with origin-scoped server query keys and `AdminTablePagination` driven by returned total. Add catalog paging after dedup and use its real installed ID for Enable; discovery-only rows fill Add fields and retain declared attribution. No provider network acquisition is added outside its host context. +- [ ] **RED preflight:** Test `['DE','de']` normalizes to `['de']`, empty selection rejects, `['de','us']` rejects when only DE/AT are authorized, unavailable capacity blocks, and changing overlay/pins/policy after preview rejects sync as 409 `preflight_stale`. Assert client-supplied free-space/feed counts are rejected, not trusted. +- [ ] **Run preflight RED:** `pnpm exec vitest run services/data-manager/__tests__/transitous/admin-preflight.test.ts apps/api/src/routes/__tests__/data-manager-sync.test.ts apps/web/src/components/admin/services/TransitSyncDialog.test.tsx` — expect missing server-derived preflight/admission assertions to fail. +- [ ] **Implement preflight:** Compute selected desired-feed counts/known sizes, OSM availability and actual capacity using existing `preflight.ts` inputs; bind five-minute evidence to country set, paired pins, overlay, source-policy revision and capacity observation. Recheck scope and current resource capacity at execution. Unknown measurements stay conservative/blocked. Dialog shows selected countries, blockers, estimate basis and warnings; submit exact reviewed subset. Omitted CLI/cron scope still uses deployment defaults and runtime preflight. +- [ ] **GREEN:** Repeat both RED commands; assert queued copy and no sync call when blocked. **Commit/review:** `feat(admin): page transit sources and preview scoped sync`; review all records remain reachable and browser estimates never grant authority. + +## Task 3: Reviewed paired proposals and validation-bound activation — D05 + +**Files:** Consume/extend Task 7’s `packages/core/src/catalog-pins/{types,store,index}.ts` and `store.test.ts`; create `packages/transitous-core/src/catalog-diff.ts`, `packages/transitous-core/__tests__/catalog-diff.test.ts`, `services/data-manager/src/jobs/transitous/catalog-review.ts`, `services/data-manager/__tests__/transitous/catalog-review.test.ts`, `apps/api/src/routes/admin-transit-pins.ts`, `apps/api/src/routes/__tests__/admin-transit-pins.test.ts`. Modify `apps/ops-agent/src/transitous-lock-runtime.ts` and its test, `packages/cli/src/commands/transitous-bump.ts` and `packages/cli/__tests__/transitous-bump.test.ts`, `services/data-manager/src/{transitous-lock,gbfs-catalog-lock}.ts`, `services/data-manager/src/jobs/transitous/{catalog-bump,prepare,compile-gbfs,pipeline,types,promote,proxy-transaction,single-flight}.ts`, `apps/web/src/app/admin/(full-admin)/transit/LockfileCard.tsx`, `apps/web/src/lib/admin/transitHooks.ts`; add `apps/web/src/app/admin/(full-admin)/transit/__tests__/LockfileCard.test.tsx`. Controller wires exports, cron and contracts, including paired read access for API/data-manager and mutation authorization only for the intended caller. Extend `packages/core/src/api/contracts/data-manager-admin/transit.ts` with the pin/review/preflight schemas and `packages/core/src/services/data-manager/contracts/transit.ts` with internal equivalents; no UI-only response types. + +**Consumes Task 7:** `CatalogPinSet`, `readCatalogPinSet`, `catalogRevisionId`, `publishCatalogPinSet` and `catalogPins.inspect`. Both proposal pins are captured once in a job’s immutable inputs and passed to prepare/compile; neither stage re-reads a separate legacy lock. `CatalogProposal = {proposalId: string; baseGenerationId: string | null; pins: CatalogPinSet; diffDigest: string; sourcePolicyRevision: string; state: "proposed" | "validating" | "validated" | "rejected" | "activating" | "activated"}`. Task 6’s `credentialsRevision` is mandatory evidence; no placeholder/fixed revision. No activation capability is advertised until all credential/source/pin invalidation consumers and recovery are wired. + +**R2-01 exact leaf:** Consume control-protocol§4.1: `PairedPinApprovalInput/Result` from02.3's core `ops/catalog-pin-approval.ts`, `DataEffectKind="transitousLock.approve"` and phase `paired-pin-commit` (ordinal0). This task installs its validation-bound agent runtime and makes it available only to the signed DM-transit-root borrower after promotion. Policy is durable async/data-manager/120s clamped to the retained activation deadline; status/by-key lookup/cancel use both retained fingerprints and signed target subset. Keep the root and reservation continuously through promotion, pair CAS and all child settlement. No local DM catalog-store call, fresh root or raw-ref approval. + +**Review/evidence:** `sourcePolicyRevision` is SHA-256 of the canonical effective operations profile plus current source allow/deny policy snapshot; policy acquisition failure blocks validation. Define `resolveCatalogProposal({branch: string, signal?: AbortSignal}): Promise`, `validateCatalogProposal({proposalId: string, jobId: string, signal: AbortSignal}): Promise` and `activateCatalogProposal({proposalId: string, validationId: string, expectedGenerationId: string | null, diffDigest: string, acknowledgeLicenseChanges: boolean, jobId: string, signal: AbortSignal}): Promise<{outcome: "activated" | "not_activated" | "unknown"; generationId: string | null; reason: string | null}>` in `catalog-review.ts`; API/cron use these functions, with typed host effects delegated to the agent. `CatalogValidation = {validationId: string; proposalId: string; pinDigest: string; diffDigest: string; sourcePolicyRevision: string; overlayDigest: string; credentialsRevision: string; countries: string[]; slot: "A" | "B"; epoch: string; manifestHash: string; imageDigest: string; completedAt: string; expiresAt: string; status: "passed" | "failed"}`. Evidence is server-authored, stored under fixed `data/motis/proposals//` and reverified; validity is 24 hours, shortened by any bound-input change. Missing/skipped critical stages never produce `passed`. + +**Routes/ops:** Browser GET `/api/admin/transit/pins` consumes the foundation paired read and adds proposal/evidence projection in this task; add POST `/api/admin/transit/pins/proposals`, POST `/api/admin/transit/pins/proposals/:proposalId/validate`, POST `/api/admin/transit/pins/proposals/:proposalId/activate`; mutations require retained `Idempotency-Key`. Activation input `{validationId, expectedGenerationId: string | null, diffDigest, acknowledgeLicenseChanges: boolean}` returns accepted `{jobId}`. Keep `catalogPins.inspect` as the sole paired authority; extend typed `transitousLock.propose` to paired payloads; `transitousLock.approve` takes exactly `{kind,proposalId,validationId,expectedGenerationId,diffDigest,acknowledgeLicenseChanges}` and returns§4.1 `PairedPinApprovalResult`, never a caller assertion that validation passed. The agent verifies fixed signed proposal/validation/activation records and its own successful promotion receipt; it alone invokes the attempt-bound CAS. Remove ref-only approval and force activation; old bump delegates to proposal creation. + +- [ ] **RED tests:** Proposing a changed GBFS commit with unchanged Transitous SHA creates a proposal; UI says “Proposal created”, never “Catalog updated”. Direct ref-only approve, failed/skipped canary, mismatched license acknowledgment and replaced proposals cannot change either pin. A crash between staging the two files leaves the previous pointer and pair readable. +- [ ] **Run RED:** `pnpm exec vitest run apps/ops-agent/src/transitous-lock-runtime.test.ts packages/cli/__tests__/transitous-bump.test.ts services/data-manager/__tests__/transitous/catalog-review.test.ts apps/api/src/routes/__tests__/admin-transit-pins.test.ts` — capture current one-pin/ref-only activation failure causally. +- [ ] **Implement proposal/diff:** Extract CLI's paired resolution and feed-diff behavior into shared helpers; use pinned commit/CSV digest and fail closed on unreadable metadata instead of silently skipping a diff. Diff source additions/removals, URL/origin changes, full structured license/attribution changes, country coverage, GBFS conflicts/quarantines and policy eligibility. Paginate detail at 100 rows with total/digest; require full review acknowledgment, not fetching every row into the browser. Apply existing safe-download and source-policy gates before any candidate acquisition; unknown licensing stays explicitly unknown. +- [ ] **Implement validation:** Under the cross-process artifact target lease, then the shared transit reservation, select the exact proposal and both pins (no proposed-to-active fallback), build inactive slot and stop at `motis-health`. Preserve the active slot. Before reusing an inactive slot that is `previousHealthySlot`, retain its complete immutable generation or refuse validation for insufficient retention capacity; never overwrite rollback evidence while claiming rollback remains available. Record which retained generation the slot previously held. Stop staging after tests, restore/reload the owned proxy transaction in `finally`, retain immutable candidate files, then release reservation. Because stage-proxy temporarily changes shared routes, hold its ownership through restoration and make restoration failure visible; never record passed evidence when restoration is unknown. Normal sync invalidates a retained validation before reusing its slot. +- [ ] **Implement activation:** Reacquire Plan 06 target lease, then reservation; re-read/re-hash all bound inputs and candidate data. Persist each child intent before submission through Plan02.3 `runOwnedDataOpsOperation(operation,{context:DataExecutionContext,phase:DataOpsChildPhase,ordinal,serviceIds})`, including key/kind/fingerprint, deadline and delegated lease; invoke guarded promotion with fresh staging probes and primary health, then call that SAME adapter with `PairedPinApprovalInput`, phase `paired-pin-commit`, ordinal0 and the retained MOTIS target subset. Agent alone CAS-selects through Task7 with `CatalogPinCommitIdentity`; no release between these two leaves. Use `lookupOwnedDataOpsChildren(context)` for lost admission and `cancelAndSettleOwnedDataOpsChildren(context)` for cancellation; cancel must settle promotion/proxy/restart/paired-pin-commit children before releasing the lease. Journal completion and cleanup separately. Reconcile an interrupted promote/pin step before any new sync/start; prove current epoch/manifest and ownership before completing pins or restoring prior state. If proof is unavailable, return unknown and retain journal/old pins, block further mutation, and expose recovery; do not assert rollback. Pin-pair publication is atomic; dataset/pin cross-service activation is journaled, not a fictitious multi-file transaction. +- [ ] **Exact publication evidence:** Implement§4.1's fixed agent proposal path, signed DM validation/activation records, independent policy/manifest/effective-mount verification and `CatalogPinCommitReceipt`. Before pointer rename persist commitId+execution+proposal/validation/diff and exact predecessor identity; atomic pointer carries this attempt's marker and committedAt. Reconcile marker+retained intent after crash; same generation or an arbitrary later pointer is never this attempt's receipt. Unknown means lookup-only and fenced, not replay/second pin transition. +- [ ] **Cron/CLI/UI:** CLI `bump` and opt-in auto-bump create paired proposals and may validate, but never activate without administrator review. Replace cron's current full-pipeline-plus-approve behavior. CLI show reads shared authority. Lock card shows active pair, proposed pair, diff, blockers/evidence and explicit Validate/Activate; old active dataset remains labeled during validation. Invalidate origin-scoped queries only from the owning execution. +- [ ] **GREEN:** Repeat RED command plus `pnpm exec vitest run packages/core/src/catalog-pins/store.test.ts packages/transitous-core/__tests__/catalog-diff.test.ts services/data-manager/__tests__/transitous/promote.test.ts services/data-manager/__tests__/transitous/proxy-transaction.test.ts 'apps/web/src/app/admin/(full-admin)/transit/__tests__/LockfileCard.test.tsx'`. Add real separate DM/agent adapter failpoints after promotion, before pointer commit, and after pointer rename before receipt sync/delivery. Lose approval admission/result/cancel responses and kill each process; recover the exact attempt/result with at most one pair transition and one root, no intervening release, no replay and no subsequent sync/start while unknown. Store-only CAS tests cannot substitute for this. Restart must reconcile, and concurrent source/pin/credential changes must invalidate evidence. Extend `services/data-manager/__tests__/transitous/prepare.test.ts`, `services/data-manager/__tests__/transitous/compile-gbfs.test.ts` and `services/data-manager/__tests__/cron.test.ts` to prove proposal validation uses both proposal pins and cron never reaches approval. +- [ ] **Commit/review:** `feat(admin): review and activate validated catalog pin pairs`; fresh reviewer traces browser, CLI and cron entrypoints plus post-crash recovery. Disposable real-MOTIS acceptance remains controller-owned and required. + +## Task 4: Overture restart and advanced recovery — D06 + +**Files:** Modify `apps/web/src/components/admin/services/OvertureMaintenance.tsx`, its `OvertureMaintenance.test.tsx`, `apps/api/src/services/admin-operation-catalog.ts`, `apps/api/src/services/__tests__/admin-operation-catalog.test.ts`, `apps/ops-agent/src/administrative-runtime.test.ts`, `packages/cli/src/commands/data.ts`; controller extends operation enums/runtime dispatch. Reuse `services/data-manager/src/jobs/overture/{pull,ingest,extract-osm-pois,operation-lock}.ts` without a second pipeline. + +**Interfaces:** Preserve `overture-conflate {region: string, restart: boolean = false}` while advancing its catalogue version to 2 and marking the operation destructive so both resume/restart pass an explicit preview confirmation; add catalogue IDs `overture-pull`, `overture-ingest`, `overture-extract` mapping to typed `data.overturePull`, `data.overtureIngest`, `data.overtureExtract`, each `{regionId: string}`. All use existing release-selected/region-validated data-manager methods and Plan 02 streaming/deadlines. No caller file paths or arbitrary release URLs. + +- [ ] **RED:** UI test confirms “Restart links from scratch” sends `{region, restart: true}` only after explicit destructive preview; Resume sends false. Advanced pull/ingest/extract controls dispatch exact fixed commands and refuse missing downloaded release/OSM prerequisites; ordinary Full sync stays primary. Busy lease disables all conflicting actions with a reason. +- [ ] **Run RED:** `pnpm exec vitest run apps/web/src/components/admin/services/OvertureMaintenance.test.tsx apps/api/src/services/__tests__/admin-operation-catalog.test.ts apps/ops-agent/src/administrative-runtime.test.ts` — current UI has no restart/advanced recovery actions. +- [ ] **Implement:** Put three lower-level actions in an Advanced recovery disclosure, use catalogue preview/confirmation and `useAdminMutationOwner`/service-job registration. Advertise prerequisites from current data-manager evidence and enforce them again at runtime. Restart clears only the selected resumable conflation workspace under its existing lease; it does not delete active Places publication. Every NDJSON terminal failure remains failed in Activity. +- [ ] **GREEN:** Repeat RED command plus `pnpm exec vitest run services/data-manager/__tests__/overture/sync.test.ts services/data-manager/__tests__/overture/rebuild-links.test.ts services/data-manager/__tests__/overture/operation-lock.test.ts`. +- [ ] **Commit/review:** `feat(admin): expose guarded overture recovery`; verify restart reaches the lease-protected runtime and cannot bypass the common catalogue version/schema guard. + +## Task 5: Artifact-backed full inventory — D07 + +**Files:** Modify `apps/ops-agent/src/data-inventory.ts`, `apps/ops-agent/src/administrative-runtime.test.ts`, `packages/cli/src/lib/service-builds.ts`, `apps/web/src/components/admin/services/DataWorkflowsPage.tsx`; create `apps/ops-agent/src/data-inventory.test.ts`, `apps/web/src/components/admin/services/DataInventorySection.tsx`, `apps/web/src/components/admin/services/DataInventorySection.test.tsx`; controller extends `data.inspect` and service-admin projection. Modify the actual download writer `services/data-manager/src/jobs/download-osm.ts`; add `services/data-manager/src/jobs/osm-fingerprints.ts` and `services/data-manager/__tests__/osm-fingerprints.test.ts` for atomic source evidence. Compute SHA-256 alongside its existing checksum pass and persist a bounded per-extract sidecar after successful publication under the same publish lock. Sidecar schema is `{schemaVersion: 1; artifactId: string; region: string; sha256: string; sizeBytes: number; device: string; inode: string; modifiedAtMs: number; changedAtMs: number}` at `data/osm/.openmapx.json`; device/inode remain internal, while the public ID is a digest of canonical source ID. Inventory reads it; status polling never initiates hashing. A failed sidecar publication reports missing evidence rather than verified readiness, and retries hash only in the downloader/build workflow. Shared artifact manifest definitions belong to Plan 06 Task 1. + +**Interfaces:** Replace first-file OSM projection with `osm: {items: OsmArtifact[]; total: number; limit: number; offset: number; unavailable: boolean}`, `OsmArtifact = {artifactId: string; filename: string; region: string | null; sizeBytes: number; modifiedAt: string; sourceFingerprint: string | null; fingerprintState: "verified" | "missing" | "stale"}`. `data.inspect` accepts `{osmLimit?: number; osmOffset?: number}` (50/500 bounds). Build rows keep legacy target IDs and add `{readiness: "ready" | "missing" | "incomplete" | "stale" | "unknown"; reason: string | null; sizeBytes: number | null; sourceFingerprints: Array<{id: string; sha256: string}>; buildRecordVersion: 1 | 2 | null; region: string | null; runtimeImages: Record}`; derive legacy `built` only from `readiness === "ready"` during consumer migration. + +- [ ] **RED:** Empty osrm directory is not ready; missing graph.obj/OSRM companions/tile MBTiles/MOTIS verified manifest is incomplete; a v1 record without source hashes is unknown, not freshly verified. Two OSM extracts appear separately regardless of alphabetical order; stale source fingerprint changes readiness. Permission/read failures are unavailable/unknown, never empty success. +- [ ] **Run RED:** `pnpm exec vitest run apps/ops-agent/src/data-inventory.test.ts apps/ops-agent/src/administrative-runtime.test.ts apps/web/src/components/admin/services/DataInventorySection.test.tsx` — directory-as-built and first-file assumptions fail. +- [ ] **Implement:** Reuse descriptor-anchored filesystem reads and Plan 06's allowed service layouts/record decoder. Record SHA-256 inputs and output manifests at successful build/import publication, not during every status request; link fingerprint evidence to recorded identity/size/mtime and invalidate on change. Inventory never hashes terabytes synchronously. Scan bounded indexed metadata and page all OSM records; a ceiling/read error is explicit incomplete inventory, never silent truncation. List every existing build target; managed database-backed Nominatim/Photon/Overpass status needs its authoritative readiness evidence, otherwise return unknown with reason rather than treating a directory as success. Valhalla uses existing data-manager dataset state and artifact metadata. +- [ ] **GREEN:** Repeat RED command plus `pnpm exec vitest run packages/cli/__tests__/cli-build-artifacts.test.ts services/data-manager/__tests__/osm-fingerprints.test.ts`; test same-size replaced input invalidates descriptor/evidence and imported artifacts carry recorded origin fingerprint. UI displays readiness reason, region, size, images and source fingerprints, with page controls. +- [ ] **Commit/review:** `fix(admin): derive data readiness from artifact evidence`; verify bounded reads, honest unknown states and no newly exported host paths. Real producer builds and database readiness are separate controller gates. + +## Task 6: Pinned, masked, non-destructive feed credentials — D08, D09 + +**Files:** Modify01.2’s `apps/api/src/services/prospective-configuration.ts` finite transit participant in the controller window (no second writer); modify `apps/api/src/routes/admin-services.ts` and its credential route tests under the controller’s exclusive window; create `apps/api/src/services/managed-service-credentials.ts` and `apps/api/src/services/__tests__/managed-service-credentials.test.ts`; modify `packages/cli/src/lib/transitous-api-keys.ts`, `packages/cli/__tests__/cli-transitous-api-keys.test.ts`, `services/data-manager/src/jobs/transitous/internal.ts`, `services/data-manager/service.json`; create `packages/transitous-core/src/feed-credentials.ts`, `packages/transitous-core/__tests__/feed-credentials.test.ts`, `apps/api/src/services/transit-feed-credentials.ts`, `apps/api/src/services/__tests__/transit-feed-credentials.test.ts`, `apps/api/src/routes/admin-transit-credentials.ts`, `apps/api/src/routes/__tests__/admin-transit-credentials.test.ts`, `apps/web/src/components/admin/services/TransitCredentialsSection.tsx`, `apps/web/src/components/admin/services/TransitCredentialsSection.test.tsx`. Controller wires routes, manifest/render exports and catalogue/runtime replacement for `generate-api-keys`. + +**Interfaces:** `collectRequiredFeedCredentialKeys(catalogDir: string): string[]` uses an isolated checkout at the active pair's exact Transitous/atlas SHAs. `mergeFeedCredentialTemplate(existing: Record, required: string[]): {values: Record; requiredCount: number; preservedCount: number; orphanedKeys: string[]}` preserves all existing values verbatim; obsolete keys become masked orphaned entries, never implicit deletions. Keep `droppedCount: 0` for CLI compatibility. + +**Wire/ownership:** Controller adds credential schemas in `packages/core/src/api/contracts/data-manager-admin/transit.ts` and registers GET/POST-preview/PATCH with explicit admin metadata. Typed `transitCredentials.inspect {catalogRevisionId: string}` returns only required key IDs and host ownership/configured booleans; the API combines this with strict vault metadata, never reading host secret values. GET `/api/admin/transit/credentials` returns `{catalogRevisionId, credentialsRevision, owner: "vault" | "file" | "environment" | "unset", editable, reason, entries: Array<{key: string; required: boolean; configured: boolean; orphaned: boolean}>}`. POST `/api/admin/transit/credentials/preview` takes `{catalogRevisionId, expectedCredentialsRevision, expectedCurrentRevisionId: string | null, changes: Array<{key: string; action: "set"; value: string} | {key: string; action: "delete"}>, operationKey: string}`; PATCH `/api/admin/transit/credentials` takes `{previewId, intentFingerprint, operationKey, transitProofToken}` against that immutable preview and cannot substitute values. No masked sentinel writes. Limit 100 changes with unique keys, keys1–512 Unicode code points (valid Unicode only), values at most8192 UTF-8 bytes, body1MiB. These request bounds do not replace the final-map/candidate budgets below. Require Plan 03 purpose `transit.credentials`, bound to `catalogRevisionId`, for modification/deletion; audit key IDs/action only. + +**Managed map exports (Task05.6 consumers are Plan04.2/04.6):** `isManagedTransitCredential(serviceId: string, key: string): boolean` matches only `data-manager/TRANSITOUS_API_KEYS`; export `TRANSIT_CREDENTIAL_MANAGED_BY = "transit.credentials"` and `TRANSIT_CREDENTIAL_MANAGED_CODE = "transit_credentials_managed"` from `managed-service-credentials.ts`. Generic service PUT/DELETE reject409 with that code before parsing/persistence in both `beforeValidation` and handler (including a retained row after declaration removal). Credential metadata adds `managedBy`, and generic UI links to the transit feed panel. The declaration is never generic-write authority. Plan04.2 retains its other managed-field guards and incorporates this exact predicate; Plan04.6 must call the adapter below rather than `setServiceSecret`/whole-map DELETE. + +**Canonical map validator:** In `packages/transitous-core/src/feed-credentials.ts` export `FeedCredentialChange = {key: string; action: "set"; value: string} | {key: string; action: "delete"}`, `ValidatedFeedCredentialMap = {values: Readonly>; canonicalJson: string; utf8Bytes: number; entryCount: number}`, `canonicalFeedCredentialMap(values: Readonly>): ValidatedFeedCredentialMap`, `parseFeedCredentialMapJson(json:string): Readonly>`, and `validateFeedCredentialMap(existing: Readonly>, changes: readonly FeedCredentialChange[]): ValidatedFeedCredentialMap`. The last function merges then calls `canonicalFeedCredentialMap`, matching control-protocol§5.3. `parseFeedCredentialMapJson` rejects duplicate raw JSON keys before ordinary object materialization; all paths reject unpaired surrogates/invalid Unicode. Sort keys by code-unit order and serialize one JSON object without whitespace; preserve every unchanged value exactly, including whitespace, orphaned keys and escaping. Reject arrays/non-string values, duplicate input changes, unsafe prototype keys and unknown mutation keys unless already retained or required by the bound catalog. The caller supplies that last catalog authorization check. Maximum final canonical JSON is65,536 UTF-8 bytes and1000 entries, key length512 Unicode code points, individual value8192 UTF-8 bytes, including retained/orphaned entries. Never apply string `.length` as a byte budget. The common trusted envelope remains524,288 bytes including JSON re-escaping of this string, revision/auth fields, MAC and newline. No splitting undeclared secret keys or raising limits. + +**Coordinated adapter:** `previewFeedCredentialChanges({actorId, catalogRevisionId, expectedCredentialsRevision, expectedCurrentRevisionId, changes, operationKey, signal?}): Promise` and `applyFeedCredentialChanges({actorId, previewId, intentFingerprint, operationKey, authorization, signal?}): Promise` in `transit-feed-credentials.ts`. All IDs are strings, current revision nullable, signal `AbortSignal`; changes is `FeedCredentialChange[]`. `FeedCredentialChangePreview = {previewId: string; intentFingerprint: string; catalogRevisionId: string; credentialsRevision: string; expectedCurrentRevisionId: string | null; expiresAt: string; changes: Array<{key: string; action: "set" | "delete"}>; preservedCount: number; mapBytes: number}`; ten-minute preview holds encrypted server-private changes and map/source revisions (no config admission or seal yet), never values or secret hashes in public metadata. Public intentFingerprint hashes the opaque random previewId, actor, catalog/map/config revisions and key/action selections; exact secret values are bound by the immutable encrypted preview record, not a public unsalted digest. The record rejects any value substitution at apply. `authorization` is the discriminated input `{kind: "direct"; transitProofToken: string} | {kind: "credential-import"; transitProofToken: string; importProofToken: string; importPreviewId: string; importFingerprint: string}`; these strings are Plan03.3 `AdminReauthProofRef.challengeId` references, never bearer authority; the authenticated server supplies the HttpOnly nonce and session. Consume direct proof with `AdminActionReauthentication.consume` or the import pair atomically with `consumeMany` in the persistence transaction before mutation. Direct PATCH uses this same prepare/apply path with its proof bound to the exact canonical intent; transfer preview calls it for the explicit selected per-key merge/deletes. Both require the current pin and map revision and configuration CAS. `FeedCredentialApplyResult = {credentialsRevision: string; configuration: TrustedConfigurationApplyOutcome}` uses the existing `trusted-config-outcome.ts` type; no invented success-only result. + +**Transfer/multi-secret participation:** Plan04.6 decrypts a managed map of at most64KiB (ordinary values stay16KiB; encrypted envelope and decrypted plaintext each≤1MiB), derives candidate set actions and shows existing-only keys as preserved. Deletion requires an explicit `delete` selection; missing keys in an attachment never delete. For atomic mixed imports, export `prepareFeedCredentialImportPatch({actorId, catalogRevisionId, expectedCredentialsRevision, expectedCurrentRevisionId, changes, importPreviewId, operationKey, signal?}): Promise`. `PreparedFeedCredentialPatch` is a nominal server-private immutable handle with readonly `{serviceId:"data-manager",key:"TRANSITOUS_API_KEYS",catalogRevisionId:string,expectedCredentialsRevision:string,expectedCurrentRevisionId:string|null,importPreviewId:string,operationKey:string,overlayChange:Extract}` plus private source/proof bindings; `overlayChange` is always set of the canonical map (including `{}`), never whole-row delete. This server-private patch holds the normalized overlay, opaque expected row revision, pin/source fingerprint and required `transit.credentials` intent; it performs no writes or lease acquisition. Under the _same_ administrative writer, Plan04.6 merges `patch.overlayChange` with ordinary credential patches into `ProspectiveConfigurationOverlay{purpose:"migration",changes}`, or direct apply uses purpose `transit-credentials`. After live proof validation below, call01.2 `prepareProspectiveConfiguration` once for the entire candidate and then `persistAndApplyProspectiveConfiguration`. Task05.6 adds one closed transit participant to that function’s finite service-secret persistence dispatch: it invokes `persistPreparedFeedCredentialPatch({patch, candidate, writer}): Promise<{credentialsRevision: string}>` inside the SAME transaction as all other selected changes/proof consumption/job checkpoint. Use01.2’s nominal `ConfigurationCommitParticipant` kind `transit-credentials` for direct changes or04.6’s single `credential-import` participant for a mixed batch; authorize/check and verify are read-only, commit owns the exact managed key and uses the current `ConfigurationWriteTransaction`. Never nest a second participant or start a second transaction.04.6 must not prewrite ordinary rows or call this helper outside that dispatcher, and the dispatcher must not also write this map with the generic secret adapter. `candidate:ValidatedConfigurationCandidate`/`writer:AdministrativeConfigurationWriter` use01.2’s exact exported nominal types; this function rejects a changed actor/key/candidate/row/pin/source and is callable only inside that writer. Both direct and transfer adapters enter this one dispatcher/persistence primitive, advance the same revision and invalidate the same catalog evidence. No nested independent writer or separate apply for the managed subset. + +**Proof timing/export required from03.3:** Preview issues the immutable random preview/intent first and performs no whole-candidate admission; `mapBytes` is the canonical map size, not a promise of full-envelope acceptance. Before `prepareProspectiveConfiguration`, verify the live proof reference without consuming it via03.3 `AdminActionReauthentication.check` for one direct transit proof or `checkMany` for the finite two-proof import pair. These read-only APIs have the exact argument shapes of `consume`/`consumeMany`, including session and server cookie nonce, and reuse their actor/session/nonce/expiry/purpose/intent checks without writes. It is not a new assurance path. The05.6 server adapter attaches that verified scope context to the nominal writer/candidate; a raw purpose field cannot construct it. Inside01.2’s common persistence transaction, recheck and call03.3 consume/consumeMany atomically with _all_ selected writes. Failure between validation and consumption rolls back the entire batch. Thus full candidate sealing precedes row mutation, purpose proof precedes prepare, and failed size/CAS validation does not burn the proof or save secrets. Consume03.3’s shared checks; no second proof implementation in05/04. + +**Persistence/order (R2-02):** Enter01.2's `withAdministrativeConfigurationWriter` on its dedicated connection then sorted integration locks (empty for feed-only). Read-only `check/checkMany` and `prepareFeedCredentialImportPatch` perform no host acquisition. Prepare and seal the **whole** candidate, validate schema/source/aggregate size/queue capacity and retain the private candidate receipt FIRST. Only then01.2's `persistAndApplyProspectiveConfiguration` uses control-protocol§5.4 `prepareConfigurationMaintenance(candidate)`: agent owns the frozen target closure and obtains the durable nonblocking DM admission guard. API has no target root/private signer and does not call local acquire/delegate APIs. A busy guard returns without desired-row/proof/invalidation mutation; an uncertain prepare remains fenced and lookup-only. + +Under the terminal prepared reservation recheck the exact paired pin, policy/overlay/source, credential row, full configuration, host-source witness and live proofs. Commit `consume/consumeMany`, the one managed map write, ordinary mixed-import rows, revision/invalidation and exact continue decision/checkpoint in ONE writer transaction. Retain binding/decision identity before POST. Known rollback alone permits exact unused abandon; unknown save/transport or API death retains candidate/root/DM/writer fences. Restart reads the retained DB checkpoint/candidate, never current desired rows, and does not reconsume proof or replay attempted effects. This is§5.4's one `configurationMaintenance.prepare/inspect/decide/settle` protocol, not a feed-specific remote lock. + +Apply the same candidate with its frozen maintenance identity: existing vault.apply (direct) or stack.render (mixed import), generation-hardlinks and guarded same-image DM recreation all borrow/resume the original root. No new root, release between save/apply or later target expansion. The DM guard is a terminal prepared logical record, survives restart and blocks every unrelated API/cron/CLI/data admission before startup readiness; no live DM settlement worker dies during recreation. All required children and exact desired configuration/mount/readiness must prove commit or containment before settlement; unknown cannot release.05.3 validation requires the credential revision and effective mount actually consumed, never a newer desired map with an old mount. + +Map revision remains an opaque digest of immutable row ID and encrypted row version (ciphertext/IV/tag); absent initial row has one explicit absent revision. Retain encrypted `{}` on last deletion; identical-value rotation advances revision; failed read is unavailable. Legacy correction must yield a fully valid final candidate. Direct and managed transfer use `persistAndApplyProspectiveConfiguration(candidate,{coordinatorMode:"require-running",signal?})`; `.application` supplies `FeedCredentialApplyResult.configuration`. Feed apply never starts a stopped DM; the explicit frozen follow-up may recreate an already-running DM to consume the new mount. Keep `overlayChange`, closed participant, `check/checkMany` and one transaction unchanged. + +- [ ] **RED:** CLI generator at active commit A with upstream B must inspect only A and retain an unrelated existing key byte-for-byte; malformed existing JSON aborts with no write. Tests assert file/environment-owned credentials reject Save/Delete, vault read/decryption failures do not appear unset, stale pin/revision rejects, and no value appears in response/log/audit/job data or browser cache. +- [ ] **Run RED:** `pnpm exec vitest run packages/cli/__tests__/cli-transitous-api-keys.test.ts packages/transitous-core/__tests__/feed-credentials.test.ts apps/api/src/routes/__tests__/admin-transit-credentials.test.ts` — current generator pulls/reset upstream and drops non-required keys. +- [ ] **Implement generation:** Remove mutable upstream `ensureCatalog` from key discovery; use isolated exact-pin checkout and fail closed on missing pins/catalog/atlas. Serialize with the catalog writer, recheck generation before publication, and use existing private-file no-follow/regular-single-link/0600 atomic-write patterns. Never regenerate the active shared catalog in place. Retire the misleading browser `generate-api-keys` catalogue item and its `data.generateApiKeys` ops path (including exhaustive kind/policy fixtures); the credential panel replaces it with pinned required-key discovery and deliberate writes. Keep the local CLI command, which operates under explicit local file ownership. Remove `transitous-fixed-v1`; do not allow the old fixed CLI subprocess to overwrite host-owned credentials from a UI request. +- [ ] **Implement ownership/apply:** Store the encrypted declared managed map as `data-manager/TRANSITOUS_API_KEYS`, materialized as `TRANSITOUS_API_KEYS_FILE`. Preserve explicit `TRANSITOUS_API_KEYS_PATH` ownership first, then managed file, then existing `/config/transitous/api-keys.json`; an existing default file remains file-owned, never silently adopted. Implement the exact validator/adapters/order above with strict vault/source reads and01.2’s candidate. Generic routes refuse the managed key; UI discloses ownership and provides host instructions. Persist/retain evidence through apply and invalidate Task3 validation on every credential revision change, including transfer. The local CLI file workflow remains explicitly local and must advance its source fingerprint used by validation; it does not bypass agent pin/target serialization. +- [ ] **RED budgets/bypass/transfer:** Exercise authenticated generic PUT and DELETE, not only the predicate/UI; both return409 and unchanged ciphertext/config revisions. Exercise malformed JSON, foreign catalog, stale map/config revision and file/env ownership through direct and transfer routes; assert zero DB mutation, admission and validation invalidation. Test final JSON at65,536/65,537 bytes, 1000/1001 keys, multibyte/escaping expansion, eight8192-byte values, and a one-key PATCH over a nearly-full retained/orphaned map. Reject a valid-size map when the full sealed envelope exceeds524,288 bytes only after encoding; an unrelated subsequent preparation remains healthy. Test a20KiB map encrypted transfer round-trip with both proofs, reject either missing/wrong-purpose proof, preserve unrelated entries unless explicit deletes were selected, and reject a mixed multi-service oversized import atomically. Legacy oversize→fully valid correction succeeds; remaining oversized candidate cannot partially persist. Leave an old managed secret mount running after a new desired generation commits: UI retains pending application and catalog validation cannot claim the new credential revision until guarded follow-up proves the effective mount. Pause after preview and race another writer/pin/source change; rejected CAS leaves the newer row intact. +- [ ] **Remote maintenance race/crash acceptance:** Through real API/agent/DM transports and disposable PostgreSQL, race direct feed apply against05.3 activation, pausing after final verification but before DB commit. Assert exclusive valid commit or zero credential/proof/invalidation writes. Drop prepare/decision/settle responses, kill API before/after save and recreate DM during follow-up; prove one root/decision, startup admission block, no self-conflict/reconsumed proof/second pin transition and no release while unknown.04.6 repeats this with ordinary+managed imports;08.3 uses the same guard. +- [ ] **Run causal budget/bypass RED, then GREEN:** `pnpm exec vitest run apps/api/src/routes/__tests__/admin-transit-credentials.test.ts apps/api/src/services/__tests__/transit-feed-credentials.test.ts apps/api/src/services/__tests__/managed-service-credentials.test.ts` plus Plan04.6’s real credential-transfer route suite; existing persist-first/generic declared-secret acceptance is the counterexample. Extend disposable DB tests to prove mixed-import atomicity/row ABA protection; do not call those database receipts satisfied by unit mocks. +- [ ] **UI/GREEN:** Show configured/required/orphaned and owner only; blank password input means no change, Delete is separate confirmation. Clear value state after submission/failure/disposal. Repeat RED and run `pnpm exec vitest run apps/api/src/services/__tests__/transit-feed-credentials.test.ts apps/web/src/components/admin/services/TransitCredentialsSection.test.tsx`. +- [ ] **Commit/review:** `feat(admin): manage pinned transit feed credentials safely`; reviewer traces ownership, revision CAS, secret materialization and runtime consumption, including malformed and stale files. + +## Task 7: Paired-pin authority foundation — D05/D08 prerequisite (R1-08) + +**Files:** Create `packages/core/src/catalog-pins/{types,store,index}.ts` and `store.test.ts`; modify `apps/ops-agent/src/transitous-lock-runtime.ts`/test, `services/data-manager/src/{transitous-lock,gbfs-catalog-lock}.ts`, `services/data-manager/src/jobs/transitous/{prepare,compile-gbfs,types}.ts`, `packages/cli/src/commands/transitous-bump.ts`/test. Controller owns focused core package exports and `catalogPins.inspect` ops schema/policy/result plus consumer registration. Task3 later extends proposals/activation; this task performs no dataset promotion and exposes no unvalidated activation. + +**Portable exports:** Move the existing exact `TransitousLock` and `GbfsCatalogLock` structures from data-manager into `types.ts`; keep re-exports for callers. `TransitousLock = {ref: string; submodules: Record; lockedAt: string; lockedBy: string; comment?: string}`; `GbfsCatalogLock = {schemaVersion: 1; source: "mobilitydata-gbfs"; commit: string; url: string; sha256: string; lockedAt: string; lockedBy: string}`. Export `CatalogPinSet = {schemaVersion: 1; generationId: string; transitous: TransitousLock; gbfs: GbfsCatalogLock}` and `catalogRevisionId(pins: CatalogPinSet): string` (exactly `pins.generationId`). Generation ID is lowercase SHA-256 of UTF-8 canonical sorted-key `{schemaVersion:1,transitous,gbfs}` without generationId; reconstruct and compare at every read. No independent credential catalog revision algorithm. Structural types have a browser-safe focused export; filesystem `store.ts` remains server-only. + +**Store interfaces:** `readCatalogPinSet(rootDir: string): CatalogPinSet | null`, `initializeCatalogPinSet({rootDir, lease}): Promise`, `publishCatalogPinSet({rootDir, lease, pins, expectedGenerationId, commit:CatalogPinCommitIdentity}): Promise` and `readCatalogPinCommit({rootDir,commit:CatalogPinCommitIdentity}):Promise`; rootDir is local repository root only, `lease` is06.1/A’s verified `TargetLeaseAuthority` (owner or borrowed) covering the MOTIS targets, pins is `CatalogPinSet`, expectedGenerationId is `string | null`. Export§4.1's exact `CatalogPinCommitIdentity/Receipt` from core catalog-pins types in this task; `ExecutionRef` already exists from02.1. Foundation tests cover attempt-bound intent/pointer/receipt recovery;05.3 alone enables validation-bound activation. Publication is an internal CAS primitive; only Task3’s validation-bound activation may call it for a new pair. It verifies lease ownership, expected current and pins hash before mutation. Immutable generations live at `infra/docker/catalog-pins/generations/.json`; one fsynced atomic `current.json` pointer selects both and carries§4.1's exact commitId/key/fingerprint/committedAt marker. The private write-ahead receipt recovers that attempt after loss, never samples arbitrary current state. No remote path inputs. Agent owns that tree; data-manager/API consume typed `catalogPins.inspect {}` → `{active: CatalogPinSet | null; catalogRevisionId: string | null}` in one authenticated response (not a direct new checkout bind mount). One captured pair flows through prepare and GBFS compilation for the entire execution; CLI reads the same store. Declare `catalogPins.inspect` as sync/api,30s,768KiB maxResultBytes (within existing1MiB result ceiling); extend the existing `authorizeOpsOperation(role,kind)` with the sole extra read allowance for data-manager on this exact kind. No role expansion for propose/approve or any other effect. Keep a512KiB canonical generation budget so wrapper/header overhead fits, and test serialized maximum result through the real ops client. Existing separate inspect APIs become projections of the same pair for compatibility; no consumer may assemble two independent responses as authority. + +**Strict legacy/bootstrap:** The assembled agent startup performs `initializeCatalogPinSet` once before advertising pin-dependent operations; managed standalone CLI mutation entrypoints call it under their existing verified lease if needed. Read-only inspect/show never silently write and report unavailable until initialization succeeds. Busy/crash-held initialization leaves those operations unavailable with retained evidence, not a legacy fallback. Initialization under the shared lease opens both existing fixed lockfiles no-follow, regular/single-link, bounds each to256KiB, validates the entire schema and pinned identities, then rechecks descriptors/content before publishing the generation/pointer. Both absent→null; one absent, parse/read error, changed descriptor or malformed pair→unavailable without a pointer. Strip only the known `$schema` compatibility key. Transitous branch ref reuses the current ops grammar (safe branch prefix, total≤256 chars, full40 lowercase hex commit) and submodules use40-hex commits; GBFS retains its exact40-hex commit,64-hex digest and commit-bound raw GitHub URL. Bound submodules128, submodule key256 chars, lockedBy128 chars, lockedAt valid ISO timestamp, comment512 chars and combined canonical generation512KiB; reject rather than truncate. Existing checked-in pins meet this contract. Interrupted writes without a committed pointer reconcile under the same lease; readers never infer authority from a stray generation. Once a pointer exists, never fall back to legacy files on corruption/missing generation. Old lock files are compatibility projections only; all managed writers/readers move to this pair and stale ref-only `approve` is disabled until Task3’s replacement lands. Unsupported abbreviated legacy refs require deliberate host correction to full immutable commits; no mutable network resolution during read. + +- [ ] **RED:** Separate legacy readers accept mismatched/partial state today. Test valid legacy import produces one stable generation, both absent, one missing, malformed GBFS digest/ref, symlink, read failure and replacement between two descriptor reads; denied imports leave no pointer. Inject crashes before/after fsync/rename; readers see previous complete pair or the new complete pair, never mixed or a fallback to corrupt legacy. Competing CLI/agent writers and stale expectedGenerationId cannot overwrite the winner. +- [ ] **Run RED/GREEN:** `pnpm exec vitest run packages/core/src/catalog-pins/store.test.ts apps/ops-agent/src/transitous-lock-runtime.test.ts packages/cli/__tests__/transitous-bump.test.ts services/data-manager/__tests__/transitous/prepare.test.ts services/data-manager/__tests__/transitous/compile-gbfs.test.ts` — causal failure is old separate/ref-only authority; after implementation every consumer uses the captured pair. Real typed inspect response serialization retains both pins and generation. No activation route/capability appears yet. +- [ ] **Implement/review:** Land exports/store/readers first, then Task6 discovery/revision, then Task3 proposals/validation/activation. Test credential discovery returns this exact generation ID; changing either pin or credentials invalidates later evidence in Task3. Local commit after implementation/review: `refactor(transit): establish paired catalog pin authority`. No new migration; master task count increases49→50. + +## Integration verification and self-review + +- Controller serializes contract edits then runs `pnpm data-manager:openapi:generate`, `pnpm data-manager:sdk:generate`, `pnpm openapi:generate`, `pnpm sdk:generate`; inspect wire serialization and generated diff. Follow with `pnpm check-data-manager-openapi`, `pnpm check-data-manager-sdk`, `pnpm check-openapi`, `pnpm check-api-sdk`, `pnpm check-ops-authority`. +- Affected type checks: `pnpm -C apps/api check-types`, `pnpm -C apps/ops-agent check-types`, `pnpm -C services/data-manager check-types`, `pnpm -C apps/web check-types`, `pnpm -C packages/cli check-types`, `pnpm -C packages/core check-types`. Controller owns `pnpm test:database ingestion`, real MOTIS inactive-slot/promote/failure recovery, full verification/builds and rendered desktop/mobile coverage. No duplicate dependency installation. +- Coverage self-review: D01→1; D02/D03/D04→2; D05→7/3; D06→4; D07→5; D08/D09→6. All five Review Focus cases have task assertions. Names and paired pin/evidence fields are consistent across CLI/API/agent/UI. +- **Root coordination rulings before execution:** accept paired-generation authority plus explicit-only activation (including opt-in cron); reserve atomic publication/recovery and target-lease integration with Plans 01/06 (shared filesystem lock/journal across CLI, agent and data-manager, no TTL stealing; target lease → configuration writer → release store); sequence Plan 06 target-lease/manifest foundation before Tasks 1/3/5; fold feed ownership/materialization into Plan 04/08 without a second writer. These are implementation coordination decisions, not user questions. + +### Review handoff + +- Reliability ordering: Tasks 1/2 precede their new UI controls; Task 7's paired authority precedes Tasks2/6; Task6 invalidation precedes Task3 validation/activation. Master must place05.7 after06.1 and before05.2/05.6; retain50 tasks. +- Inventory is a projection of retained evidence. Legacy records may remain unknown with a precise reason; a fresh supported build/import must produce v2 evidence and a ready row. +- The rollback retention check in Task 3 applies before touching the inactive slot, not after discovering an import failure. +- Every UI mutation carries server revision evidence and is registered with the existing execution owner; rendering a button is not evidence of runtime support. +- Controller ruling applied: target leases use shared filesystem locks/journals across CLI, agent and data-manager; the process-local handle only represents already-acquired authority. Lock order is target lease → configuration writer → release store, with no TTL stealing. +- The controller must read both subsystem plans together before assigning common target leases or `BuildRecord` ownership. + +- Planning self-review only: no tests, generators, builds, installations or runtime verification executed while writing this plan. diff --git a/docs/superpowers/plans/2026-10-23-admin-06-artifact-transfer.md b/docs/superpowers/plans/2026-10-23-admin-06-artifact-transfer.md new file mode 100644 index 000000000..280a16cee --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-06-artifact-transfer.md @@ -0,0 +1,249 @@ +# Prepared Artifact Transfer Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Deliver T01-T04: authenticated prepared-artifact export/upload/import with bounded streaming, verified compatibility, stopped consumers, recoverable atomic publication and cleanup. + +**Architecture:** Extract the existing CLI bundle contract and integrity checks into a shared server-only module, harden archive admission before extraction, and reuse that code in CLI and typed ops-agent workflows. The API brokers opaque, principal-bound stages and durable jobs; bulk bytes stream through a dedicated authenticated transport rather than JSON job envelopes or browser buffers. + +**Tech Stack:** TypeScript, Node streams/filesystem, Fastify, Zod, typed ops-agent, React/MUI, generated admin SDK, Vitest; existing build records, service registry and configuration generations. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md`; master `docs/superpowers/plans/2026-10-23-admin-instance-management.md`. Audit base `ef896ceae876959180116cf61ffcf643b88ebffe`. + +## Global Constraints + +- All product work stays in this worktree; preserve unrelated work and existing public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source policy boundaries. Community integrations remain declarative; no uploaded code execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain behind typed ops-agent operations. No generic command, arbitrary path or raw environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth declarations, strict schemas, audit events, explicit mutation identities and appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, durable operation identities, retained results, unknown outcomes and cancellation fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database and runtime checks, independent review, production builds and PR CI. Report any unavailable environment gate distinctly; do not replace runtime evidence with mocked assertions. +- Read root/web `AGENTS.md`, `CONTRIBUTING.md#quality-bar` and developer API/service/integration contracts. User authorized planning only for plan authors; implementation begins after controller review. No duplicate installation, generators, builds or commits during planning. + +## Review Focus + +1. Malicious member names, duplicate entries, PAX/long-name headers, links, sparse files and decompression bombs cannot write outside a private stage or exceed budgets (Task 1). +2. Chunked length/digest lies, foreign-principal stage IDs and disconnects cannot finalize usable stages (Task 2). +3. A target starting, image/config revision changing, cancellation or crash around publication cannot expose a partial artifact as ready (Task 3). +4. Exporting an aliased MOTIS root, concurrently mutating a build or delivering a multi-hundred-GB file cannot expose secrets or buffer it in memory (Task 4). +5. Expiry/cleanup racing a live upload, inspection, import or download must preserve owned work and report failed cleanup honestly (Task 5). + +--- + +## Ownership and controller contracts + +**Exclusive new modules:** `packages/core/src/build-artifacts/{manifest,archive,target-lease,control-authority,publication,index}.ts`, `packages/core/src/ops/artifact-transfer.ts`, `apps/ops-agent/src/artifact-transfer/{policy,store,routes,runtime}.ts`, `apps/api/src/services/artifact-transfers.ts`, `apps/api/src/routes/admin-artifact-transfers.ts`, and the focused UI/client modules listed below. Core archive/publication modules are server-only through `@openmapx/core/build-artifacts`; never re-export through the React/client barrel. + +**Serialized controller files:** `packages/core/{package.json,src/ops/contract.ts,src/ops/contract.test.ts,src/ops/client.ts,src/ops/index.ts,src/api/contracts/index.ts,src/api/contracts/service-admin/operations.ts}`, `apps/ops-agent/src/{index,server,policy,administrative-runtime}.ts`, `apps/api/src/routes/index.ts`, `packages/cli/src/commands/data.ts`, `apps/web/src/components/admin/services/DataWorkflowsPage.tsx`, `services/ops-agent/service.json`, `services/traefik/config/traefik.yml`, `packages/core/src/services/traefik-renderer.ts`, `packages/core/src/__tests__/services-traefik-renderer.test.ts`, and `pnpm-lock.yaml`. Controller adds `packages/core/src/api/contracts/service-admin/artifact-transfers.ts` for canonical JSON/binary schemas and stable operation IDs under the existing `service-admin` family (new focused `apps/api/src/routes/admin-artifact-transfer-contract.ts` registers canonical metadata/serialization); update `scripts/api-sdk/config.mjs` only where its binary transport declarations require it. Controller owns route-family/generator metadata and all generated OpenAPI/SDK outputs; any DB migration/index remains controller-assigned. This design uses agent stage metadata and the existing operation journal, not a new application stage table. + +**Plan 02:** Retain `OpsExecutionContext.signal`, `emitLog("stdout" | "stderr", message)` and durable job identity/result retention. Transfer preparation/validation/publication jobs have a 72-hour default wall deadline, a 7-day deployment ceiling, bounded log retention and no observer-disconnect cancellation. Request byte streams have separate idle/deadline limits below. Root reconciles this operation policy with Plan02 before wiring; no second job registry. Plan08.3 configuration handoff consumes this store’s active byte-transfer ownership: while its agent maintenance/drain gate is closed, create/new byte-stream/claim admissions return503 with a bounded retry reason. Already-open streams remain counted until body/callback/file cleanup settles, including abort; the helper cannot recreate the agent while an uncontained transfer still owns descriptors. Its deadline may cancel an owned stream through the same transport abort path, never by dropping counters. Unknown installation/publication claims retain their independent durable fences; a drain cannot erase them. Use08/A’s existing handoff gate, not a second maintenance flag/service. + +**Plans 01/02/05/08/09 — binding lease authority:** Task06.1 implements the exact shared protocol in `docs/superpowers/specs/2026-10-23-admin-control-protocols.md` (PlanA owner), including its acquire/delegate/adopt/settle/reconcile signatures, owner/grant/child records and authenticated fixed-CLI/peer adapters. It does not define a competing lease or accept `parentArtifactLeaseId` as authority. `ArtifactTargetLease` is an opaque verified local handle backed by the common physical data mount, persistent ownership and child journal. Private owner recovery material, signed bounded child grants and actual peer-role admission remain internal; neither bare IDs nor browser-supplied parent context grant rights. No general lock service/shared signing-secret substitution. Host order is target lease → nonblocking transit reservation → host configuration writer → release store; API-origin config work first holds the API global administrative writer and sorted integration locks per§5.1, before acquiring host targets; sorted target acquisition,5s acquisition timeout, no TTL stealing. A child may only borrow the retained signed subset and may not release a parent. Unknown/lost child admission, parent death, cancellation or deadlines retain the fence until authoritative settlement/reconciliation. Full Update owns no target while awaiting a child that needs it. All managed Start/build/clean/recreate/promotion/import paths consume this protocol before advertising transfer support. + +**Reconciled06.1 exact exports (control-protocol§§2–3, read after publication):** Produce `ArtifactTargetLease` (owner), `BorrowedArtifactTargetLease` and their union `TargetLeaseAuthority`; all artifact/publication/catalog store consumers accept the union where a managed fixed CLI may be a borrower. `withArtifactTargetLease({dataRoot:string,serviceIds:readonly string[],owner:VerifiedExecutionAuthority,inherited?:BorrowedArtifactTargetLease,signal?:AbortSignal},run:(lease:TargetLeaseAuthority)=>Promise):Promise`. Implement `acquireArtifactTargetLease`, `delegateArtifactTargetLease`, `adoptArtifactTargetLease`, `settleArtifactTargetLeaseChild`, `releaseArtifactTargetLease`, `reconcileArtifactTargetLease` with the spec’s exact signatures; no operationId-only overload. `VerifiedExecutionAuthority`, `VerifiedControlPeer`, `VerifiedChildSettlement`, `VerifiedRecoveryAuthority`, `ArtifactDelegationAssertion`, `ControlChildPhase` and `ControlRecipient` come only from fixed `control-authority.ts` constructors. Use02’s `ExecutionRef` with both body `operationFingerprint` and normalized execution `fingerprint` plus epoch-ms `deadlineAt`; never substitute one for the other in seals/receipts. Internal assertion header is `x-openmapx-control-delegation`, fixed CLI env `OPENMAPX_CONTROL_DELEGATION`; deadline-only parent header/env are exactly§3.4’s. Callbacks do not unconditionally release in finally. + +**06.1 provisioning/limits are part of this foundation:** Add `control-authority.ts` and its tests; extend `packages/cli/src/commands/compose.ts`, built-in agent/data-manager manifests, fixed CLI adapter and startup verification in the task’s exclusive window. Consume the three installation role keypairs and fixed verifier bundle/private recovery mounts in§3.1, outside shared `/data` because both services run the same UID. Per-lease ephemeral private keys stay outside sibling mounts. This is the chosen narrow alternative to a shared-data-only owner key: fixed installation trust is necessary to prevent same-UID sibling forgery. No new network service and no user-supplied credential paths. Preserve8KiB decoded assertions, depth2/chain3,256 targets,64 child intents/128KiB ledger/256 unsettled leases,5s acquisition and72h settled receipt retention. Stage/archive quotas remain separate. Stage operations and catalog workflows are named root owners, except§4.1's explicit DM-transit-root→agent `paired-pin-commit` approval leaf; no child-kind wildcard. Implement§3.5/§5.4's exact configuration-maintenance root→agent frozen commit/hardlinks/recreate/handoff edges and agent→DM `configuration-fence`/`configuration-fence-settle` logical leaves. Terminal prepared root/guard are retained authority, not live drain workers. Hardlink and helper descendants remain within depth2; helper has no further edge.01.2 owns orchestration,02.3 owns early closed DM contracts/adapters,05.3 owns approval consumer. No new API mount or signer. Tests must include role-key substitution, sibling-readable mount regression, external helper surviving agent death and prepared-terminal-before-drain ordering, besides the existing separate-process cases. + +**Plan 05:** Owns source fingerprint creation and inventory UI. Task 1 supplies the shared target-lease foundation before Plan 05 Task 3; the MOTIS-specific import adapter in Task 3 follows Plan 05 pin/credential evidence, avoiding a circular dependency. This plan owns shared `BuildRecord`/`BundleManifest` decoding, allowed artifact layouts, archive safety and publication. Land Task 1 before Plan 05 Task 5; its `data.inspect` projection must not invent independent readiness/checksum rules. + +**Plan 04 reuse, offered for root to align without editing that plan:** `TransferPurpose = "prepared-artifact" | "instance-migration" | "integration-artifact"`; `TransferDirection = "upload" | "download"`; `TransferStage = {stageId: string; purpose: TransferPurpose; direction: TransferDirection; state: "created" | "uploading" | "uploaded" | "validating" | "ready" | "claimed" | "consumed" | "failed" | "expired"; bytes: number; sha256: string | null; expiresAt: string; cleanup: "pending" | "complete" | "failed"}`. IDs are random UUIDs and never encode host paths. Metadata retains initiating principal, operation key, creation time and policy revision internally. + +**Stage input types:** All IDs/owner IDs/operation keys/digests are strings; byte counts are nonnegative safe integers, timestamps ISO strings, `now: Date`, and cancellation is `AbortSignal`. `expectedSha256`/fingerprints are64 lowercase hex; stage/claim/inspection IDs are UUIDs, owner IDs nonempty≤128 chars, operationKey uses existing `opk1_[A-Za-z0-9_-]{16,64}` and agent execution IDs the existing `job1_[A-Za-z0-9_-]{16,64}` journal identity. StageClaimIdentity is derived from that admitted agent operation even when an API job/preview is its parent; `fingerprint` is02’s execution fingerprint and `executionDeadlineAt` is the ISO projection of immutable `ExecutionRef.deadlineAt` (convert once and verify equality on reads), never substitute a browser preview UUID for an agent execution. Timestamps parse as ISO instants and deadlines are validated by02 admission. Stage methods below are methods of the single agent `ArtifactTransferStore`, not exported stateless functions with hidden storage. + +**Shared stage methods and exact server exports (06.2):** `ArtifactTransferStore` in `apps/ops-agent/src/artifact-transfer/store.ts` owns `createStage({purpose, direction, ownerId, declaredBytes?, expectedSha256?, operationKey}): Promise`, `getStage({stageId, ownerId, purpose}): Promise`, `inspectClaim({stageId, ownerId, identity}): Promise<{claim: ArtifactStageClaim; settlement: StageClaimSettlement | null} | null>`, `deleteStage({stageId, ownerId}): Promise` and the claims below. Browser-create has no ownerId; the API derives it from the session. Agent accepts it only from the authenticated API role/capability; CLI ownership uses its separately authenticated execution principal. All purpose lookups and stream opens check ownership as well as the stage ID. + +```ts +type StageClaimUse = + | "inspect-integration" + | "install-integration" + | "validate-build" + | "import-build" + | "read-migration"; +type StageClaimIdentity = { + executionId: string; + operationKey: string; + fingerprint: string; + executionDeadlineAt: string; +}; +type StageInspectionBinding = { + inspectionId: string; + stageId: string; + sha256: string; + bytes: number; + policyRevision: string; + validationFingerprint: string; + expiresAt: string; +}; +// Server-private, not the public stage projection or bearer authority; validated against its journal/context. +type ArtifactStageClaim = { + claimId: string; + stageId: string; + ownerId: string; + purpose: TransferPurpose; + use: StageClaimUse; + identity: StageClaimIdentity; + sha256: string; + bytes: number; +}; +type StageClaimSettlement = + | { outcome: "validated"; inspection: StageInspectionBinding } + | { outcome: "released"; evidence: "no_effect" | "restored" } + | { outcome: "consumed"; receiptId: string } + | { outcome: "failed"; receiptId: string } + | { outcome: "unknown"; receiptId: string }; +``` + +`claimStage({stageId, ownerId, purpose, use, identity, expectedSha256, inspection?}): Promise` atomically journals intent/claim before bytes leave the store; `settleStageClaim({claim, settlement}): Promise` replaces the old unconditional release method. `withStageFile({claim, signal}, run: (file: FileHandle, signal: AbortSignal) => Promise): Promise` is agent-internal only, opens the exact immutable descriptor with no-follow/inode+digest verification, and closes it after callback completion/abort containment. All portable types live in `packages/core/src/ops/artifact-transfer.ts`; export them from the focused ops surface. Claim methods are authenticated typed `artifactTransfer.claim`/`.settle` ops for API only, with exact schema/purpose-use matching. Settlement strings/receipt IDs alone are not proof: the store verifies the claim’s fixed purpose-consumer receipt/journal, execution identity and owned transaction/inspection against the requested transition. `released/no_effect` requires no publication intent or proved contained pre-effect termination; `released/restored` requires the matching integration/publication restore receipt. An API assertion cannot release unknown work. API persists durable intent before POST and recovers by `artifactTransfer.inspectClaim {stageId, ownerId, identity}`; an absent response never causes a fresh claim identity or automatic install retry. Duplicate exact identity returns its retained claim/result, changed fingerprint conflicts. Deadline is inherited/clamped via Plan02; observation cannot extend it. + +**Finite claim/state transitions:** Complete digest-verified upload becomes `uploaded`. `inspect-integration` may claim only `integration-artifact`/uploaded (or re-inspect unclaimed ready); successful consumer validation stores an immutable inspection binding (10min from inspection, capped by stage expiry) and moves to `ready`. Inspection ID/validation fingerprint comes from04.3’s retained agent inspection and is submitted only by that authenticated runtime consumer, never directly by the browser. `install-integration` claims only integration-artifact/ready and requires that exact inspection binding plus digest, principal and unexpired policy. `validate-build`/`import-build` similarly require prepared-artifact uploaded/ready; migration uses its registered encrypted consumer. Every claim pins prior state, immutable bytes and its execution identity. Durable queued validation may show `validating`; once a worker claims bytes state is `claimed` with its private use/phase, and successful validation changes it to ready. Public views may label that claimed inspection as validation but never imply it is installable. Validation→ready, proven no-effect/restored→prior ready/uploaded state only while still unexpired, committed install/import→consumed, proved irrecoverable failure→failed. Unknown retains `claimed` with recovery-required evidence; neither timeout nor DELETE nor a missing live process releases it. An expired unclaimed inspection/stage returns410 and cannot be newly claimed; an already admitted claim remains pinned through authoritative settlement even after its deadline. Reconciliation requests containment at deadline, retains unknown evidence/quota, and never turns deadline expiry into settlement proof. No install reuse after consumed; exact execution replay returns only its retained outcome. + +**Uploaded integration adapter (same agent as installer):** `withClaimedIntegrationArtifact({claim, signal}, consume: (input: {source: AsyncIterable; signal: AbortSignal; bytes: number; sha256: string}) => Promise): Promise` is a method exported by `ArtifactTransferStore` in `apps/ops-agent/src/artifact-transfer/store.ts`. It accepts only its persisted owner-matched claim of purpose integration-artifact/use inspect/install, wraps `withStageFile` with `file.createReadStream({autoClose:false})`, verifies descriptor/record before reading and incremental actual size/digest through EOF, and awaits full callback/stream cleanup. Early consumer return cancels the source and fails validation rather than treating unread bytes as verified. Plan04.3’s `inspectUploadedIntegration`/install runtime consumes this adapter and `prepareIntegrationArtifact` in the integration framework. This avoids unnecessary agent→API→agent byte transport: source shows installer publication is a host effect, and04.3 already places it behind typed ops. No path, URL, FileHandle or owner chosen by the browser crosses that boundary. The API persists intent and injects authenticated ownerId into `integration.inspectInstall/install`; the admitted agent runtime obtains its own `StageClaimIdentity` from02’s retained execution, not from public fields. The store additionally permits those exact authenticated API-role runtime consumers to call claim/settle locally with their context; it is not a general role bypass. + +Installer preparation validates the immutable stream into its own bounded private workspace without converting stageId into a hidden source string. The agent retains claim through publish, API registry reload/DB bookkeeping and `integration.settle`; callback completion is not settlement. Successful inspection calls `settleStageClaim` with `validated` and stores the same `StageInspectionBinding` in04.3’s private inspection. Install references that exact binding. Proven preview cancellation uses `released/no_effect`; failed validation uses `failed` with its retained inspection receipt. Commit consumes only after the04.3 transaction’s durable commit/reload/bookkeeping receipt; failed reload permits `released/restored` only after exact-owner restoration, otherwise unknown retains claimed. Lost inspect/install/settle responses use the same retained child/claim identity lookup; no new operation or premature claim release. Preview observer cancellation stops and settles only its owned inspection; a disconnected admitted install remains running. Remote prepared-artifact/migration byte consumers, if needed, still use `consumeArtifactStage` with the same active claim and body lifetime; they receive no host handle. + +Purpose discrimination is mandatory at every lookup/claim. Plan04 owns migration encryption/KDF and integration manifest/trust/checksum validation; neither is a build bundle. Register only these finite consumers, not a general upload service. Integration-artifact wire and expanded limits each209,715,200 bytes (200MiB), including source adapter staging; preserve the installer’s4096 tar-header/256 PAX-record/1024-byte path/32-component/255-byte component limits and declarative-only validation. Encrypted instance-migration wire cap64MiB, with smaller Plan04 document limits still binding. Credential envelopes≤1MiB stay Plan04 API-owned encrypted staging and canonical bounded JSON; plaintext secrets never enter generic staging/logs. A small encrypted attachment Blob in04.6 is allowed under that1MiB cap and does not relax any large-artifact memory rule. + +## Deployment transfer policy + +Defaults are ceilings, not preallocation; one multi-hour country build must be practical: + +- `maxUploadBytes = 1_099_511_627_776` (1 TiB); `maxExpandedBytes = 2_199_023_255_552` (2 TiB); `maxStageBytes = 4_398_046_511_104` (4 TiB aggregate reserved/occupied space); `maxEntryBytes = maxExpandedBytes`. +- `maxEntries = 250_000`; `maxManifestBytes = 67_108_864` (64 MiB); `maxPathBytes = 4096`; `maxPathDepth = 64`; extended tar header payload <=16 KiB; gzip expansion ratio <=1000 after the first 64 MiB expanded. Allow regular files/directories only, with normalized mode 0600/0700. +- `maxConcurrentArchiveJobs = 1`, `maxConcurrentTransfers = 2`, `maxConcurrentTransfersPerPrincipal = 1`, `maxRetainedStagesPerPrincipal = 4`; aggregate stream queues <=8 MiB per transfer; metadata independently bounded by manifest/entry ceilings. Never claim the whole process uses only the stream-buffer budget. The current ops-agent container has a 256m limit; controller raises its supported default to 1g for this bounded metadata work and verifies peak RSS below that limit. Deployments retaining a smaller budget must lower entry/manifest/concurrency limits or get an explicit unavailable preflight. Larger overrides require a matching declared memory budget; they cannot silently retain the 1g assumption. +- Byte-stream idle timeout 5 minutes; total upload/download budget 72 hours; stage lifetime 96 hours from creation; completed-download retention 24 hours; cleanup scan every 15 minutes. Active claims/streams have recorded absolute deadlines; reaching one initiates cancellation/containment, never ownership release. Unknown effect settlement pins stage/evidence/quota beyond that deadline until reconciliation. Expiry cannot delete a still-owned stream/claim; crash reconciliation decides state first. +- Policy ceilings are never per-operation reservations. Upload admission reserves the declared/admitted compressed byte count only, plus bounded metadata/inodes and headroom. For an unknown-length CLI stream, acquire an initial 1 GiB grant (or the remaining lower policy allowance) and extend the reservation in 1 GiB increments before writing more bytes; reject safely if extension fails. Browser File uploads declare `file.size`; actual counts must match. +- After reading and validating the bounded first-member manifest, reserve its summed expanded file sizes and directory/file inodes before extracting any payload. Count already-occupied uploaded bytes once; export reserves the calculated tar/output requirement from the source manifest. Reject manifest totals above policy ceilings. Enforce aggregate reservations/occupancy plus `max(1 GiB, 5% of this operation’s additional requiredBytes)` free headroom, current available inodes and retained rollback bytes. Same-filesystem rename does not reserve a duplicate of already-occupied old artifacts. A tiny bundle must not require 2 TiB free. Unknown capacity blocks; recheck during extraction/publication and stop safely on ENOSPC. +- Load a strict `artifactTransfer` object from the ops-agent's supported deployment policy, supplied through `OPENMAPX_ARTIFACT_TRANSFER_POLICY_FILE` at `/run/openmapx/artifact-transfer-policy.json`, mounted read-only from `@infra:artifact-transfer-policy.json`. Missing file uses documented defaults; the renderer treats this as an optional supported mount, never an arbitrary path. Only the deployment operator controls it. Positive safe integers, cross-field consistency and ceilings are validated at startup; invalid policy disables transfer with an actionable reason. Do not accept policy overrides from browser requests. +- Supported upper bounds: upload 8 TiB, expanded 16 TiB, aggregate 32 TiB, entries 1 million, manifest 256 MiB, concurrency 4, transfer deadline 7 days and stage lifetime 8 days. Stream queue ceiling stays 8 MiB. Surface effective limits before file selection; document lowering them for small hosts and raising them for large builds. Plan 08 owns any later UI for deployment policy. + +## Task 1: Shared CLI-compatible bundle validation — T02 foundation + +**Files:** Create `packages/core/src/build-artifacts/{manifest,archive,target-lease,control-authority,index}.ts` and `packages/core/src/build-artifacts/{manifest,archive,target-lease,control-authority}.test.ts`; include the fixed provisioning/admission adapters assigned above; modify `packages/cli/src/lib/build-artifacts.ts`, `packages/cli/__tests__/cli-build-artifacts.test.ts`; controller adds server-only core export and a direct `tar-stream` dependency pinned to the existing lockfile's 3.2.2, plus types if necessary. Do not depend on an undeclared transitive package. + +**Interfaces:** Implement/export A’s binding target-lease protocol and `ArtifactTargetLease` exactly and preserve `buildArtifactSpec(serviceId: string): BuildArtifactSpec`, `BUILD_ARTIFACTS`, `BundleFile`, v1 `BuildRecord` and v1 `BundleManifest`. Add v2 record `{schemaVersion: 2; service: string; builtAt: string; region?: string; revision?: string; runtimeImages: Record; toolImages: Record; sourceFingerprints: Array<{id: string; sha256: string; sizeBytes: number}>; artifactFingerprint: string}`; v2 bundle adds the existing `dirs/directories/files`. `artifactFingerprint` hashes canonical sorted file path/size/SHA-256 tuples, not a self-reference. + +`validateBuildBundle({source: Readable, stagingRoot: string, limits: ArtifactArchiveLimits, signal: AbortSignal}): Promise` is server-local; `ValidatedBuildBundle = {manifest: BundleManifest; archiveSha256: string; expandedBytes: number; manifestSha256: string; artifactFingerprint: string}`. `stagingRoot` never crosses HTTP/ops. `writeBuildBundle({dataRoot, serviceId, record, target: Writable, limits, signal}): Promise` shares manifest creation and byte-digest verification. Define `ArtifactArchiveLimits` with the archive policy fields above, all numeric positive safe integers. Export `ImageMismatch = {service: string; bundle: string; local: string}` from `manifest.ts`. Preserve v1 sourcePbf metadata; v2 retains it optionally and adds the new source hashes without implying historical hashes exist. + +- [ ] **RED tests:** Existing valid v1 CLI bundles still import; changed file bytes, mismatched runtime images or running consumers fail. New malicious fixtures contain `../escape`, absolute/drive/backslash paths, duplicate normalized paths, symlink→file and hardlink entries, FIFO/device, sparse/PAX size tricks, missing/extra members and gzip expansion overflow. Assert no file outside staging exists even when extraction fails. Current extract-first `tar -xf` is the causal counterexample. +- [ ] **Run RED:** `pnpm exec vitest run packages/cli/__tests__/cli-build-artifacts.test.ts packages/core/src/build-artifacts/archive.test.ts packages/core/src/build-artifacts/target-lease.test.ts` — failures must identify unsafe extraction or missing bounded validation, not unrelated tool availability. +- [ ] **Implement shared lease:** Implement A’s filesystem-backed acquire/delegate/adopt/settle/reconcile contract with exact fixed-CLI environment and authenticated peer headers. Spawn separate fixture processes representing CLI/agent/data-manager on one physical mount with different path aliases. Test both delegation directions, permitted depth/subsets, forged/stale/foreign/replayed grants, wrong peer principal/kind/key/fingerprint, durable duplicate admission, owner death and child cancellation/lost responses. Include§4.1's approval leaf and§5.4's terminal maintenance/fence preparation, exact one-use continuation grants and final settlement. Assert no nested reacquisition, no widened continuation/helper subset, no unauthorized delegation and no release before every child settles; check actual inode/journal verifier and ordered target/reservation/config/release acquisition. This is the prerequisite shared module for Plan 05, not an in-memory stand-in. +- [ ] **Implement:** Move existing `BUILD_ARTIFACTS`, manifest decoding, artifact walking, per-file SHA-256 and `runtimeImageMismatches` policy into shared modules; keep CLI wrappers/signatures. Borrow the tar/PAX/path limits and malicious fixtures from `packages/integration-framework/src/installer.ts` and its package-contract tests, but not its `gunzipSync(readFileSync(...))` extraction strategy; Plan 04 retains ownership of that installer. Strengthen strict manifest shape/types, unique names/counts and exact set equality. Preserve the current five portable services: tileserver, osrm, otp, pelias, motis; unsupported database services get a reason, not a pretend portable bundle. +- [ ] **Archive handling:** Accept plain tar and gzip tar by magic bytes only. A bounded 512-byte framing guard rejects unsupported/sparse/link types and oversized extended headers before the tar parser buffers them; effective PAX/GNU path/size values receive identical checks. Require the manifest as the first regular member (matching CLI exports), bounded metadata and exact declared files. No shell `tar -xf` on uploads; create each output under descriptor-anchored no-follow parents with exclusive regular-file creation. Verify streamed byte counts/checksums, EOF/trailing structure and complete membership before marking validated. Reject compression errors/concatenated unexplained data. +- [ ] **Export hygiene:** Pin trusted aliases inside the data root and reject links elsewhere, including hardlinked credential files. MOTIS bundles omit generated `feed-proxy-vars.json`, credential-bearing nginx routing and injected API keys: create a sanitized portable candidate from retained source metadata, include required credential IDs without values, and rebuild proxy/config through the target's credential workflow before readiness. Treat sanitized MOTIS payloads as v2 with `credentialRequirements: string[]` and `runtimeMaterialization: "motis-target-credentials"`; the target records original bundle digest separately from its newly materialized candidate fingerprint. A legacy credential-bearing MOTIS bundle fails with explicit rebuild guidance; CLI and web enforce the same boundary. Other safe v1 bundles remain readable. +- [ ] **GREEN:** Repeat RED plus `pnpm exec vitest run packages/core/src/build-artifacts/manifest.test.ts`; round-trip representative synthetic fixtures for all five services, v1/v2 records, gzip/plain input and real filesystem extraction. Verify malformed size metadata aborts before writing payload. +- [ ] **Commit/review:** `refactor(data): share bounded build artifact validation`; reviewer checks compatibility, exact runtime image pins, secret exclusion and validation-before-write. Do not claim live build qualification from fixtures. + +## Task 2: Opaque authenticated stage transport — T01 upload, T02 capacity/digest + +**Files:** Create `packages/core/src/ops/artifact-transfer.ts`, its test, `apps/ops-agent/src/artifact-transfer/{policy,store,routes}.ts` and matching `.test.ts`; create `apps/api/src/services/artifact-transfers.ts`, `apps/api/src/routes/admin-artifact-transfers.ts`, `apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts`. In the controller’s exclusive window modify `packages/core/src/api/{client,admin-sdk-transport}.ts`, `packages/core/src/api/contracts/admin.ts`, `apps/api/src/routes/admin-contract.ts`, `scripts/api-sdk/config.mjs` and their existing tests; extend `scripts/api-sdk/codegen.test.ts`. Controller wires ops client/server/auth, route families, exports and generated artifacts. A helper alone without registrar/generator/consumer adoption is not this task’s deliverable. + +**Ops bytes:** Add `artifactTransfer.create/inspect/delete/claim/inspectClaim/settle` typed operations (API role only) with the contracts above. Dedicated `PUT /v1/artifact-transfers/:stageId/content` and `GET /v1/artifact-transfers/:stageId/content` carry bytes; JSON operation envelopes never do. Server-only `OpsClient` exposes `uploadArtifactStage({stageId, ownerId, purpose, body: ReadableStream, contentLength?: number, signal?, executionDeadlineAt}): Promise` and `consumeArtifactStage({stageId, ownerId, purpose, claim?: ArtifactStageClaim, signal?, executionDeadlineAt}, consume: (input: {body: ReadableStream; signal: AbortSignal; headers: Headers}) => Promise): Promise`. executionDeadlineAt is ISO string converted from02’s epoch-ms retained `deadlineAt`; requested streams clamp to admission/policy deadline and remaining time. Download stage reads require ready/download state; uploaded bytes may be read only by the exact active purpose/use claim. Internal raw transport owns authorization, stream-attempt capability, idle timer, total timer, abort propagation and response-body cancellation _through callback completion_. Unread/errored/aborted bodies are canceled and descriptor/stream counters settled before resolving/rejecting; no `Promise` escape. API pipes to Fastify with backpressure and awaits downstream finish/abort inside that callback. Upload responses/errors are bounded JSON (stage success64KiB, errors64KiB); non-UTF8 payload never passes a TextDecoder. Do not reuse the ops JSON30-minute request timeout as the multi-hour transfer deadline. + +**Canonical binary request/response contract:** Extend `AdminOperation` with optional `requestBody: {kind: "bytes"; mediaType: "application/octet-stream"}` mutually exclusive with `request.body` Zod JSON. Add `AdminResponseContract` variant `{kind: "bytes"; mediaType: string; headers?: z.ZodType; nativeDownload?: true}` and helper `adminBytes(mediaType: string, headers?: z.ZodType, nativeDownload?: true)`. `kind:"stream"` keeps existing text/SSE semantics. Registry validation rejects ambiguous binary+JSON requests and nativeDownload on anything except a GET byte success. The route registrar emits binary requestBody/response schemas as `{type:"string",format:"binary"}` with their exact media types in OpenAPI, without Zod-string parsing or JSON serialization of bytes. Transfer parser is scoped and authenticates before consumption. Raw/streamed byte replies validate declared success status/media/headers before writing headers and complete through one byte-send adapter; errors use ordinary canonical JSON serialization while headers are unsent, otherwise abort the stream. Never `JSON.stringify` a stream or send a raw response outside guard/serialization metadata. + +**Exact canonical client interfaces:** Retain the existing `ApiClient.request(path, init, consume, options): Promise` and `ApiClient.send` lifecycle; all binary acquisition and consumption run inside its callback with `acceptedStatuses:"all"`. Extend the transport in `admin-sdk-transport.ts` with: + +```ts +type AdminBinaryUploadRequestOptions = Omit< + AdminSdkRequestOptions, + "body" | "onTextChunk" | "keepalive" +>; +type AdminByteResponseRequestOptions = Omit< + AdminSdkRequestOptions, + "body" | "onTextChunk" | "keepalive" +> & { + onByteChunk: (chunk: Uint8Array, signal: AbortSignal) => void | Promise; + onByteHeaders?: (headers: Headers, signal: AbortSignal) => void | Promise; +}; +type AdminNativeDownloadTarget = { url: string; ownership: "browser" }; +type AdminNativeDownloadTargetOptions = { + client?: ApiClient; + signal?: AbortSignal; + method?: "GET"; + headers?: Readonly>; +}; +``` + +Export `bindAdminByteOperation(operationId)` for byte success and `bindAdminBinaryUploadOperation(operationId)` for byte requests, with operation-ID unions derived from canonical registries, not arbitrary URLs. They retain ordinary `{status,headers,data}` generated unions; successful byte response has `data: undefined` only after EOF and every callback settles; JSON errors retain their canonical typed `data` and never call byte callbacks. Validate media/headers before callbacks. Error JSON is bounded to64KiB regardless of advertised length (caller may lower, never raise), with existing `ApiContractError` for malformed/unknown status/media and Retry-After behavior retained. Upload File/Blob is sent unchanged as `body` via the supplied client; no JSON stringify, arrayBuffer, XHR or reconstruction of private auth options. Set octet-stream explicitly even when File.type differs, and validate effective transport headers after the existing header-interceptor merge so an interceptor cannot silently change the declared media type. Preserve configured authentication/cache/options and existing credential default (`omit` when unspecified on an explicit client). + +For byte requests default client timeout72h, maximum7days and5min idle budget; a caller may shorten them and the server retains its earlier absolute deadline. Extend `ApiResponseOptions` with `expectedRequestMediaType?: "application/octet-stream"`; `send` validates the final merged Content-Type against it before fetch, after the interceptor. Binary upload binders set it and the canonical Accept header; ordinary JSON/text callers retain existing behavior. The byte reader obtains one chunk, awaits `onByteChunk(chunk,effectiveSignal)`, then pulls the next; no accumulating arrays or text decoding. Callbacks must settle on signal abort and finish their sink cleanup; the request stays owned until the active callback and reader cancellation finish. Timeout/caller abort after headers still rejects `ApiRequestAbortedError`, aborts the body/sink and removes timers/listeners only afterward. A callback throw cancels upstream; success requires EOF, verified declared length when present and all callback work. Headers callback participates in the same lifetime. Invalid success headers are rejected before any destination bytes are written. The server owns upload byte counts; the browser polls stage inspect for `{bytes,state}` and labels those as bytes received, not guessed network progress. + +**Generated signatures/operation inventory:** In the existing service-admin family define `uploadAdminArtifactTransferContent(stageId: string, body: Blob, options?: AdminBinaryUploadRequestOptions): Promise` and `downloadAdminArtifactTransferContent(stageId: string, options: AdminByteResponseRequestOptions): Promise`. The former returns the canonical200 `TransferStage`/JSON-error union; the latter200 `{status:200;headers:Headers;data:undefined}`/JSON-error union. `getDownloadAdminArtifactTransferContentTarget(stageId: string, options?: AdminNativeDownloadTargetOptions): Promise` is generated separately from that same native-eligible GET; it does not perform/pretend a download. Extend `adminOperationBindings` and the family fetch emitters by canonical request/response byte metadata, ahead of text/SSE handling; generated public functions must require the byte callback and accept Blob uploads, never Orval’s default text/Blob-buffering response. Add the same byte bindings to Plan07’s privacy attachment/artifact download GETs and any native-eligible route there; retain privacy family permission/assurance guards. Plan04.6 stays canonical bounded JSON and creates only its encrypted≤1MiB local Blob, so it is not forced through a large-byte path. Explicit export maps and generator regression fixtures cover every adopted byte operation/family; root reconciles07’s exact operation IDs before generation. + +**Native target capability:** Add `ApiClient.downloadTarget(path: string): AdminNativeDownloadTarget` and the target/error types in `client.ts`; `admin-sdk-transport.ts` re-exports those types (client.ts must not import the administration registry/transport). Only browser runtime, a same-origin HTTP(S) API URL relative to actual `window.location.origin`, no userinfo/hash/protocol-relative input, `credentials:"same-origin"` or `"include"`, and _no configured headerInterceptor_ may construct a target. Omitted/`omit` credentials, any custom request header, interceptor (even currently empty), alternate API origin, unsupported options or non-browser runtime fail explicitly with `ApiNativeDownloadUnavailableError` carrying one finite reason (`"runtime"|"origin"|"credentials"|"headers"|"interceptor"`). `bindAdminDownloadTarget(operationId)` validates the canonical native-eligible GET and params/query and rejects all unsupported options before calling this method. Signal checks construction only; native navigation is subsequently browser-owned, with no claim to custom headers, deadline/progress or programmatic cancellation. This deliberately narrower same-origin capability avoids pretending native navigation can reproduce an arbitrary configured cross-origin/auth client. Alternate-origin and header/omit clients use the normal generated byte callback into a supported browser file sink; if unavailable, show the precise limitation and supported reconnect/download guidance. Never switch their credentials or origin silently, and never fall back to fetch→Blob for large files. + +**Browser API:** `/api/admin/artifact-transfers` GET lists the owner’s stages with `{limit: 50, offset: 0}` (max 100), POST creates, `/:stageId` GET inspects/DELETE cancels a stage, `/:stageId/content` PUT uploads/GET downloads. All require full admin; owner ID is derived from session, never browser JSON. Cookie mutation Origin/CSRF protection, no-store, nosniff and safe constant download names remain enforced. 401/403 auth, 404 foreign/missing stage, 409 state/purpose/stale, 410 expired, 413 size, 422 digest/archive, 429 concurrency, 507 capacity and 503 unavailable are canonical schemas. Rate-limit admission to 10/minute/principal plus active-stream quotas. Byte PUT success is200 TransferStage; byte GET success is200 application/octet-stream with Content-Disposition attachment, no-store, nosniff, exact Content-Length and independently computed digest header. CORS exposes the required length/disposition/digest headers for authenticated separate-origin byte consumers; native targets remain same-origin only. Register415 for unsupported request media; error bodies never contain supplied secret/path data. + +**Streaming authorization:** Agent token plus a one-use, short-lived capability bound to stage ID, owner, purpose, direction, declared byte cap, exact claim ID/use/identity when reading claimed uploads, and stream attempt UUID. API issues the capability only after stage ownership/auth checks; reuse the existing ops capability/HMAC implementation pattern with an explicit `artifact-transfer-v1` domain and a key derived by HKDF from the API-agent shared token; never reuse a privacy capability or add an unmanaged secret file. Secrets stay in headers over the trusted agent channel and out of browser URLs/journals. At stream open atomically claim the attempt before consuming any bytes. + +- [ ] **RED tests:** A valid admin upload streams to the agent; unauthenticated/cross-owner/cross-purpose requests consume zero body bytes. Chunked input exceeds the declared cap, advertises the wrong digest, disconnects or sends a second simultaneous PUT: no ready stage exists and the accepted writer remains protected. Unknown free capacity refuses before upload. A 2 MiB bundle on a host with 2 GiB free succeeds under the default 2 TiB expansion ceiling; reservations equal admitted compressed bytes, then actual manifest-derived expansion, not that ceiling. +- [ ] **RED canonical bytes/lifetime/stage consumer:** Through real Fastify registration and generated functions upload/download non-UTF8 bytes (`00 ff c0 80`) and compare exact digest. Wrong media fails415 before consumption; missing/malformed success headers and >64KiB/non-JSON errors abort boundedly. Stall a sink after headers, abort caller/timeout and assert upstream cancel, no further chunks, callback cleanup before completion and timer/listener cleanup. A slow sink bounds pull count/queues. Test explicit omit/header-interceptor clients retain auth on byte requests and native-target construction refuses; test separate API origin with real CORS and the same configured client. Through04.3 exercise upload→claimed inspect→immutable ready binding→install/reload receipt; wrong owner/purpose/checksum, expired/replayed stage, policy/inspection changes and lost claim/admission responses produce no install. Crash after publication or failed restoration retains unknown claim until authoritative reconciliation. +- [ ] **Run RED:** `pnpm exec vitest run packages/core/src/ops/artifact-transfer.test.ts apps/ops-agent/src/artifact-transfer/routes.test.ts apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts` — missing routes/guards and unsupported transport are the intended failures. +- [ ] **Implement:** Persist stages under agent-controlled `infra/docker/data/.openmapx-transfers//`, directory 0700/files 0600, with atomic fsynced metadata. Stream request→API→agent→exclusive file with backpressure, incremental SHA-256 and actual counters. Apply limits before/during write; Fastify auth must run before octet-stream parsing, and the parser returns a stream rather than Buffer. Scope parsers to transfer routes and leave JSON limits intact. Verify the shipped Traefik route has no request-buffering middleware; retain streaming. Coordinate static ingress read/write timeout settings (72 hours) with API byte-route idle/total deadlines; Traefik entrypoint timeouts are not falsely described as route-local. Keep ordinary API parser/body/time limits bounded independently. Test the configured ingress with a long throttled stream; a direct localhost upload does not prove this gate. +- [ ] **Finalize:** Require complete upload and matching optional declared digest/length; record independently computed SHA-256 and `uploaded`, never `ready` before purpose validation. Unknown outcomes reconcile with inspect rather than retrying PUT automatically. Staging consumes both occupied/reserved disk budget, releases reservations only when deletion is proven, and rejects symlinked/replaced stage roots. A stream capability expires after 60 seconds if unopened; admission consumes it once and the admitted stream then uses its 5-minute idle/72-hour total budget. The API may mint a new attempt only after inspect proves the previous attempt never claimed or is terminal. +- [ ] **GREEN:** Repeat RED and existing `packages/core/src/api/client.test.ts`, `packages/core/src/api/admin-sdk-transport.test.ts`, `scripts/api-sdk/codegen.test.ts` plus `pnpm exec vitest run apps/ops-agent/src/artifact-transfer/policy.test.ts apps/ops-agent/src/artifact-transfer/store.test.ts`; test a generated 2 GiB stream without materializing it, peak queued bytes <=8 MiB, and policy admitting a declared 600 GiB bundle on a simulated sufficiently large host. Real capacity/stream runtime proof remains Task 5. +- [ ] **Commit/review:** `feat(admin): stream authenticated artifact stages`; review capability replay/expiry, CSRF timing, opaque IDs, parser limits, principal quotas and cleanup on abort. + +## Task 3: Stopped-target preview and journaled publication — T03 + +**Files:** Create `apps/ops-agent/src/artifact-transfer/motis.ts`/test; extract the existing data-manager `jobs/transitous/candidate.ts` manifest types/decoding/integrity rules into `packages/transitous-core/src/candidate-manifest.ts`/test with `verifyCandidateManifest(stagingDir:string,signal?:AbortSignal):Promise` and incremental hashing, updating existing callers to await it; agent `motis.ts` owns offline target-credential materialization using existing `@openmapx/motis-feed-proxy-config` normalizer/renderer and05 credential inputs; controller owns package exports/dependencies. Create `packages/core/src/build-artifacts/publication.ts`, `packages/core/src/build-artifacts/publication.test.ts`, `apps/ops-agent/src/artifact-transfer/runtime.ts`, `apps/ops-agent/src/artifact-transfer/runtime.test.ts`; modify `packages/cli/src/lib/{build-artifacts,build-staging}.ts` and CLI artifact tests. Controller wires target-lease checks into managed service/data entrypoints, ops operations and trusted configuration follow-up. + +**Operations:** `artifactTransfer.validate {stageId, ownerId, purpose: "prepared-artifact"}` returns a durable job; `artifactTransfer.previewImport {stageId, ownerId}` returns `ArtifactImportPreview = {stageId: string; sha256: string; serviceId: string; region: string | null; bytes: number; requiredFreeBytes: number; requiredCredentials: string[]; runtimeImageMismatches: ImageMismatch[]; runningConsumers: string[]; blockers: string[]; fingerprint: string; expiresAt: string}`. Preview lasts 10 minutes. `artifactTransfer.import {stageId, ownerId, expectedSha256, previewFingerprint, expectedCurrentRevisionId: string | null}` returns a durable job, not synchronous success. + +**HTTP control mappings:** POST `/:stageId/validate` returns 202 `{jobId}`; POST `/:stageId/preview-import` returns 200 `ArtifactImportPreview`; POST `/:stageId/import` takes `{expectedSha256, previewFingerprint, expectedCurrentRevisionId}` and returns 202 `{jobId}`. Paths are relative to `/api/admin/artifact-transfers`; mutations require `Idempotency-Key`. GET `/capabilities` returns effective limits, eligible targets and per-target availability/reasons; register static paths ahead of `/:stageId`. + +**Publication result:** Export `ArtifactPublicationResult = {outcome: "committed"; artifactFingerprint: string; serviceId: string; followUp: "complete" | "failed"; cleanupFailed: boolean}` or `{outcome: "not_committed"; reason: "conflict" | "busy" | "invalid" | "failed"; cleanupFailed: boolean}`; transport/lost journal is `unknown` in the API adapter. `publishValidatedBuild({bundle, stageId, lease, expectedFingerprint, signal}): Promise` uses `lease:TargetLeaseAuthority` and is local-only and requires a claimed stage plus target lease. + +- [ ] **RED tests:** Preview a valid stopped OSRM target, then change runtime image, start a consumer, alter current configuration or replace stage bytes. Import must conflict without renames. Inject failures after each directory move, build-record write and hardlink apply; no partial generation becomes ready, and committed-follow-up-failed remains committed. Two users importing the same target serialize/conflict. +- [ ] **Run RED:** `pnpm exec vitest run packages/core/src/build-artifacts/publication.test.ts packages/cli/__tests__/cli-build-artifacts.test.ts apps/ops-agent/src/artifact-transfer/runtime.test.ts` — existing multi-rename/late-record code lacks durable recovery and stale-preview fencing. +- [ ] **Validate/preview:** Claim and verify complete archive with Task 1, recheck exact runtimeImages via the registry, current configuration revision, required target credentials, source/artifact fingerprints and live owned Docker consumers. Preview hashes these inputs and stage digest. Reject running consumers rather than implicitly stopping them; name the guarded Stop workflow. Never accept a caller's `consumersStopped: true`. +- [ ] **Publication protocol:** Under the shared target lease and configuration writer (target lease → configuration writer → release store whenever release-store authority is also needed), recheck all fences and stopped consumers. Write/fsync a journal containing old/new identities and paths derived only from allowlisted service layout; stage on the target filesystem. Reuse `promoteStagedBuildDirs` semantics with journaled phases and atomic record replacement. The transaction is atomic to managed consumers because Start/readiness and other writers are fenced until all dirs and record commit; do not describe multiple renames as one filesystem atomic action. +- [ ] **Recovery/cancellation:** Recover journals before accepting relevant Start/import/build; identity-check each owned path before resume/restore. Pre-commit cancellation restores only proven owned changes; post-commit cancellation retains commit evidence and completes/reports follow-up. An unprovable restoration becomes unknown, blocks Start and retains evidence; never delete previous copies on uncertain commit. Commit point is durable `committed` journal after all new dirs/build record are installed; crashes before it recover the old complete set while consumers remain stopped. +- [ ] **MOTIS/follow-up:** Preserve `motis/live` and `motis/staging` aliases and slot-state semantics; stage imported data through the existing inactive candidate verifier, regenerate secret-backed proxy configuration at destination, and require exact image/config/license/source manifest readiness before publication. Hold the MOTIS/transit lease shared with Plan 05. After publication use01.2’s preparation with `coordinatorMode:"require-running"` (never implicitly start data-manager/consumers); preflight a nonempty hardlink plan before commit and retain committed/follow-up-failed if the coordinator disappears. Empty plan requires no coordinator. Prepared import is a named agent/host-CLI root. MOTIS import performs offline candidate verification/materialization under that root; it never launches `motis.primary.promote` (which actually recreates), proxy reload or a nested transit pipeline, since consumers must remain stopped. Factor the existing `candidate.ts` manifest/hash validation and feed-proxy config generation into the shared bounded validator/materializer consumed by data-manager and this agent adapter; no imported health-success claim replaces a future explicit Start/probe. Its only remote follow-up is§3.5’s permitted generation-hardlinks leaf with the borrowed target authority. Normal catalog activation remains a data-manager root and uses02.3 `runOwnedDataOpsOperation`/lookup/cancel settlement for promotion/proxy/restart; unknown child outcomes retain target/stage ownership. Then apply hardlinks; consumers remain stopped for an explicit Start. Follow-up failure surfaces recovery instructions without pretending import was rolled back. +- [ ] **GREEN:** Repeat RED; crash/restart fixtures check every journal phase, missing/changed old inode, failed restoration, canceled-before/after commit and recovered replay with the same operation ID. A valid duplicate returns retained result and does not publish again. +- [ ] **Commit/review:** `feat(data): publish imported artifacts with recovery fences`; fresh reviewer verifies all consumers/writers honor the target lease and separates atomic visibility from crash recovery. Do not ship until controller integrates those shared guards. + +## Task 4: Export and usable browser/CLI transfer flows — T01 export, T03 UI + +**Files:** Extend artifact runtime/routes and API tests; create `apps/web/src/lib/admin/artifactTransferClient.ts`, `apps/web/src/lib/admin/artifactTransferClient.test.ts`, `apps/web/src/components/admin/services/ArtifactTransfersSection.tsx`, `apps/web/src/components/admin/services/ArtifactTransfersSection.test.tsx`; modify `packages/cli/src/commands/data.ts`, `packages/cli/__tests__/cli-build-artifacts.test.ts`; controller mounts the section in `DataWorkflowsPage.tsx` and generates control-plane SDK bindings. + +**Export:** Browser POST `/api/admin/artifact-transfers/exports` takes `{serviceId: string, expectedArtifactFingerprint: string}` plus `Idempotency-Key`, returns 202 `{jobId}`; final stage metadata comes from the retained job result. `artifactTransfer.export {serviceId, ownerId, expectedArtifactFingerprint, operationKey}` runs durably under target lease with stopped consumers and source/build identity fencing. It streams a sanitized Task 1 bundle into a private download stage, validates its final digest and size, then returns `{stageId, sha256, bytes, expiresAt}` as the retained job result. Recheck source descriptors before/during/after reading; any mutation fails export without a downloadable stage. + +**Browser transport:** JSON controls and all byte requests use06.2’s generated clients. `uploadArtifactFile({client, stageId, file, signal, onProgress}): Promise` in `artifactTransferClient.ts` takes `client: ApiClient`, `stageId: string`, `signal?: AbortSignal`, `file: File`, `onProgress?: (progress: {receivedBytes: number; state: TransferStage["state"]}) => void`. It forwards the File unchanged to `uploadAdminArtifactTransferContent`, polls the same client/origin’s stage metadata at1s while active, and reconciles lost response by inspect rather than resending the body. Stop polling/owned upload on abort/disposal; never retain File in queries/storage. Server-computed digest is authoritative. + +Download first asks the generated native-target constructor; supported same-origin cookie clients use a browser-owned link with attachment headers and honest lack of progress/cancel. When native target is unavailable, feature-detect a user-selected file sink (e.g. browser `showSaveFilePicker` capability); select it directly in the user gesture before network awaits. `downloadArtifactToSink({client, stageId, sink, signal}): Promise` accepts `sink: {write(chunk: Uint8Array): Promise; close(): Promise; abort(reason?: unknown): Promise}` and invokes `downloadAdminArtifactTransferContent` with awaited writes under the effective signal. On success EOF close once; on non2xx/abort/write failure abort once and retain the download stage for retry. Never let an old origin/job close or mutate a newly selected sink. An unsupported browser gets an explicit availability reason and same-origin reconnect or supported-browser/CLI guidance. No memory Blob/objectURL fallback, no XHR and no copying private ApiClient config into a second network stack. + +- [ ] **RED tests:** Export is disabled with reason for running/unsupported/missing artifacts; changes during export invalidate output. UI Upload reports bytes, then validation, then preview and explicit Import; queued does not say imported. Import conflict retains stage/preview error for inspection. Spy on `File.arrayBuffer`/Response.blob and fail if called; changing origin or unmounting must not apply old responses to the new view. +- [ ] **Run RED:** `pnpm exec vitest run apps/ops-agent/src/artifact-transfer/runtime.test.ts apps/web/src/lib/admin/artifactTransferClient.test.ts apps/web/src/components/admin/services/ArtifactTransfersSection.test.tsx packages/cli/__tests__/cli-build-artifacts.test.ts` — absent workflows and old helper boundaries fail. +- [ ] **Implement UI:** Add Export and Import tabs with eligible service selection, effective limits/free capacity, native file picker, byte progress, computed digest, compatibility/credential/stopped-consumer preview, explicit confirmation and Activity link. Use execution-owned mutation registration and `retry: false`. Abort only the owned active byte upload on Cancel; leaving a page does not cancel a queued validation/import/export job. Never persist File objects or contents in query/local storage. +- [ ] **CLI parity:** Keep `data export ` / `data import ` and `-` streaming. Route both through the hardened shared engine/target lease, same readiness/image guards, recovered journal and truthful render/hardlink result. For stdin, stream to private bounded staging first; no `readFileSync`/concatenation of bulk content. CLI exposes the same effective deployment limits and redacts secret-bearing MOTIS content. +- [ ] **GREEN:** Repeat RED; validate no whole-file browser buffers, origin-isolated query keys, masked credential prerequisites, retry/manual-cancel distinctions and download headers. Integration route test consumes a streamed response and checks final digest, not just metadata JSON. Controller runs an actual supported same-origin cookie browser navigation and verifies downloaded bytes/attachment filename; separately verify alternate-origin/custom-header/omit clients use their authenticated sink path, and unavailable sink/browser cases show guidance. No PR screenshots are needed. +- [ ] **Commit/review:** `feat(admin): expose prepared artifact transfer workflows`; inspect actual rendered desktop/mobile Upload/Validate/Blocked/Ready/Queued/Failed/Unknown states locally after controller starts the app. No screenshots uploaded to PR. + +## Task 5: Expiry, crash cleanup and runtime qualification — T04 + +**Files:** Extend `apps/ops-agent/src/artifact-transfer/{store,routes,runtime}.ts` and their tests; create `apps/ops-agent/src/artifact-transfer/lifecycle.integration.test.ts`; create `docs/docs/administration/artifact-transfers.md`; update `docs/docs/developer/api-surface.md` and `docs/docs/install/managing-services.md` with controller coordination. No separate cleanup daemon, cron service or migration is required. + +**Lifecycle:** Keep `ArtifactPublicationResult` distinct from `TransferStage.state` so successful file transfer cannot imply publication. `reconcileTransferStages({now, lookupExecution, signal}): Promise<{expired: number; removed: number; cleanupFailed: number}>` runs at startup and every15 minutes under a single sweep lock. `lookupExecution(identity: StageClaimIdentity): Promise<{state:"running"}|{state:"settled";settlement:StageClaimSettlement}|{state:"unknown"}>` is a typed adapter over02’s retained execution/child evidence and the purpose consumer’s own receipt, not a live-process-ID set; uncertain reads return unknown. Store verifies settled proof as above. Metadata/files stay private; scan at most 1000 stages per pass with a persisted cursor. Reconcile orphan uploading/validating/claimed states against owned stream/job records before expiry. Unknown job outcome keeps publication evidence pinned and consumes quota until resolved. + +- [ ] **RED tests:** Kill upload halfway; restart marks failed and removes partial bytes. Expiry during a healthy stream/claim does not delete it; foreign DELETE cannot cancel another owner. Client disconnect during download releases only stream ownership; prepared download remains until expiry. Import observer disconnect leaves job running. Race an08.3 prepared handoff against stream admission and abort after headers; no new stream enters after gate closure, and drain observes zero only after actual descriptor/body cleanup. Simulated rm/permission failure records cleanup failed, retains disk accounting and is retried on the next sweep. +- [ ] **Run RED:** `pnpm exec vitest run apps/ops-agent/src/artifact-transfer/store.test.ts apps/ops-agent/src/artifact-transfer/routes.test.ts apps/ops-agent/src/artifact-transfer/lifecycle.integration.test.ts` — missing recovery/claim-aware expiry behavior fails. +- [ ] **Implement:** Wire shutdown cancellation for active byte streams, awaited file closure before unlink, durable terminal status, startup reconciliation and idempotent tombstone cleanup. DELETE a claimed stage returns 409 with its job ID; cancellation uses the existing durable job workflow and cleanup follows terminal evidence. Cap retained terminal metadata separately at 7 days/10,000 records and never prune an unresolved publication journal. Cleanup must target the stage's verified UUID directory, not caller paths or recursively followed aliases. +- [ ] **GREEN:** Repeat RED with real localhost API→agent streams, temporary disk roots and injected capacity/clock; tests use synthetic payloads. Assert resource counters return to baseline after cancel/error and queued bytes stay bounded. Use process kill/restart in the integration fixture; mocked `finally` execution does not prove crash cleanup. +- [ ] **Runtime acceptance:** Controller runs disposable Docker stopped/running consumers plus real filesystem rename/restore/hardlink publication, configured ingress upload/download, small valid bundles for every supported service, and an opt-in long stream near deployment scale with measured RSS/disk/inodes/timeouts. Record achieved bytes and actual memory, not inferred country-scale acceptance. Skip only with explicit unavailable-gate report; no production instance/data. +- [ ] **Documentation/commit:** Explain supported formats/services, credential prerequisites, policy override bounds, stage expiry, partial/unknown recovery and host-owned stopped-consumer procedure. Commit `test(admin): verify artifact transfer interruption and recovery` after fresh review of T01-T04 receipts. + +## Integration verification and self-review + +- Controller generates `pnpm openapi:generate` then `pnpm sdk:generate`; extend the binary endpoint contract/transport mode rather than forcing bytes through generated JSON validation. Run `pnpm check-openapi`, `pnpm check-api-sdk`, `pnpm check-ops-authority`, focused suites above, then affected package `check-types` in core/cli/api/ops-agent/web. Policy/export/package changes join master full verification and production builds. +- Scope map: T01→2/4; T02→1/2/3; T03→3/4; T04→5, with abort paths pinned in 2/3/4. All five Review Focus lines have explicit RED/GREEN assertions. Shared stage names are identical for Plan 04 reuse; data-manager's pinned source evidence comes from Plan 05 rather than a second inventory. +- Self-review correction: the existing CLI helper extracts with `tar -xf` before validation and its directory swaps are sequential. Reuse its manifest/integrity/compatibility rules but strengthen pre-extraction guards and durable publication; no plan claims existing helpers already prove safe upload or crash atomicity. +- **Controller rulings applied:** cross-process filesystem target lock/journal and target → configuration → release order; admitted-byte/manifest-derived reservations rather than policy-ceiling reservations; exact stage/purpose transport for Plan 04 standalone integration artifacts within installer limits; <=1 MiB encrypted credential envelopes may use API-owned storage, with no plaintext generic stage. +- **Remaining root coordination before execution:** Plan04.3 consumes the exact claim/inspection/byte adapter; Plan07 download routes consume06.2 byte/native interfaces and declare their operation IDs; Plans01/02/05/08/09 consume A’s shared protocol (06.1 implements it). Serialize policy/manifest/registrar/SDK files; reconcile admission-clamped72-hour transfer deadlines with02 and protected handoffs with08; preserve05.7→05.6→05.3 order and sanitized MOTIS destination credential rebuild. These are controller coordination gates, not additional user questions. +- No product code, generators, builds, tests or installations ran during this planning task. Full runtime, UI, builds and CI remain future implementation receipts. diff --git a/docs/superpowers/plans/2026-10-23-admin-07-privacy-workflows.md b/docs/superpowers/plans/2026-10-23-admin-07-privacy-workflows.md new file mode 100644 index 000000000..a02f94459 --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-07-privacy-workflows.md @@ -0,0 +1,179 @@ +# Guarded Privacy Case Workflows Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Expose the existing guarded identity, task, decision, attachment, backup and assisted-delivery workflows in the privacy administration UI, covering P01–P06. + +**Architecture:** Keep privacy decisions in the current request/identity/reauthentication services and add canonical transport schemas plus small case-bound UI components. Split touched route groups into focused plugins as needed; preserve privacy-admin authorization, request versions, durable idempotency, redaction, readiness and independent human release approvals. + +**Tech Stack:** TypeScript, Fastify, Zod, Drizzle/PostgreSQL, generated API SDK, React/MUI/React Query, Vitest, existing encrypted artifact storage. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md`; controller: `docs/superpowers/plans/2026-10-23-admin-instance-management.md`. + +## Global Constraints + +- All product work stays in this worktree; preserve unrelated work and existing + public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source + policy boundaries. Community integrations remain declarative; no uploaded code + execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain + behind typed ops-agent operations. No generic command, arbitrary path or raw + environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth + declarations, strict schemas, audit events, explicit mutation identities and + appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never + hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, + durable operation identities, retained results, unknown outcomes and cancellation + fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, + place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation + and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful + admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots + to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database + and runtime checks, independent review, production builds and PR CI. Report any + unavailable environment gate distinctly; do not replace runtime evidence with + mocked assertions. + +Planning-only boundary: baseline `ef896ceae876959180116cf61ffcf643b88ebffe`; no code, generator, build, installation, test run or commit performed. Read root/web `AGENTS.md`, API surface and Contributing quality bar; implementation reads the installed Next.js guide before adding navigation behavior. + +## Review Focus + +1. Selecting case B while A's read/upload/decision finishes must not put A's evidence, draft, error or reauth into B (all UI tasks, especially Tasks 1/4/6). +2. “Verified” cannot be asserted by selecting a login/email method; it needs its existing proof-producing channel, with independent representative authority (Task 1). +3. A stale version, reused mutation key with changed payload, or already-terminal request must not overwrite another caseworker's decision (Tasks 2/3/4). +4. A corrupt/unavailable backup or unreviewed supplement must not become an empty, approved source or enable generation (Tasks 4/5). +5. Reauthentication expires or identity/authority is revoked after artifact verification: zero artifact bytes leave the server and no successful-delivery event is written (Task 6). + +--- + +## Read-first findings and ownership + +- Read `apps/api/src/routes/privacy-admin.ts`, `privacy/request-contracts.ts`, `privacy/request-service.ts`, `privacy/reauthentication.ts`, `privacy/session-assurance.ts`, `privacy/attachments.ts`, `privacy/backup-review.ts`, `privacy/backup-omission-review.ts`, `packages/core/src/privacy/{contracts,codes}.ts`, `packages/core/src/api/privacy.ts`, and existing guarded PostgreSQL tests before implementation. +- Current page `apps/web/src/app/admin/privacy/page.tsx` has safe case-sequence checks on detail fetches, email challenge forms and backup-warning acceptance, but generic mutation callbacks are not execution-owned. Tasks offer only `operator_review` and artifacts are text rows. Extend these existing semantics instead of inventing a new privacy state machine. +- Existing identity schemas use `version`, `party`, `outcome`, `method`, `reasonableDoubtCode`, `evidenceAttachmentId`, `authorityOutcome`, `authorityAttachmentId`, `deliveryAuthorized`. Task mutations use `version`; backup omission acceptance uses **`requestVersion`**. Assisted reauth start uses `{channel:"assisted"|"representative",reasonCode}` and complete uses `{challengeId}`. Preserve these names and status/error codes. +- Existing privacy admin authorization accepts real `admin` or `privacy_admin` sessions and explicitly rejects loopback. Release approvals remain full-admin-only and reject implementation actors; no UI task creates approvals automatically. Plan 03 owns account roles and auth-policy changes; this plan must never broaden that authority. +- This plan alone owns `apps/api/src/routes/privacy-admin.ts`, privacy workflow service changes, the privacy page, and new components under `apps/web/src/components/admin/privacy/`. New route plugins receive `PrivacyAdminRouteOptions` and existing services; they register beneath the guarded parent and retain no-store headers. Do not move shared auth/identity logic into browser components. +- Controller owns `apps/api/src/routes/index.ts`, `packages/core/src/api/contracts/{admin,index}.ts`, `packages/core/src/api/admin-sdk.ts`, `scripts/api-sdk/config.mjs`, `scripts/database-test-groups.json`, `apps/api/openapi.json`, generated SDK files and locale merges `packages/i18n/locales/en.json` / `de.json`. Add `privacy_admin` to the registrar's supported auth metadata without altering other route guards; family `privacyAdmin`, tag `privacy-admin`, model directory `privacy-models`. +- Proposed canonical domain schemas: `packages/core/src/privacy/admin-contracts.ts`; operations/projections: `packages/core/src/api/contracts/privacy-admin/{common,identity,tasks,decisions,attachments,backups,delivery,operations,index}.ts`. Move/reexport existing identical schemas from API modules; core must not import API/database code. Keep date coercion at service input boundary and ISO date strings on the wire. +- Plan06.2 is the binary transport producer: consume `AdminByteResponseRequestOptions`, `AdminNativeDownloadTargetOptions`, `AdminNativeDownloadTarget`, `bindAdminByteOperation`, `bindAdminDownloadTarget`, `ApiClient.downloadTarget` and `ApiNativeDownloadUnavailableError` exactly. This plan owns privacy operation IDs/headers/permissions and generated consumers below; no independent fetch/download URL builder. Account plan03 supplies assurance only; artifact challenge storage/cookie remains privacy-owned. +- No migration is planned for ordinary UI coverage. Idempotency uses existing request events/unique keys and version columns. If implementation proves new persistence necessary, the controller reserves the exact migration path before coding it; never independently consume account/integration plan slots0042–0050 or earlier applied privacy migrations. + +## Task 1: Canonical case client and identity/representative decisions (P01) + +**Files:** Create `packages/core/src/privacy/admin-contracts.ts`, privacy contract `common.ts`, `identity.ts`, `operations.ts`, `index.ts`, `apps/api/src/routes/privacy-admin-identity.ts`, `apps/api/src/routes/privacy-admin-contract.ts`, `apps/api/src/privacy/admin-contracts.test.ts`, `apps/web/src/lib/admin/privacyAdmin.ts`, `privacyAdmin.test.ts`, `apps/web/src/components/admin/privacy/PrivacyIdentityReview.tsx`, `PrivacyIdentityReview.test.tsx`; modify `apps/api/src/routes/privacy-admin.ts`, `apps/api/src/privacy/request-contracts.ts`, `apps/api/src/privacy/assisted-identity-route.test.ts`, `apps/web/src/app/admin/privacy/page.tsx` and its test. + +**Interfaces:** + +- `PrivacyAdminDetail={request:PrivacyCaseView,tasks:PrivacyOperatorTask[],events:PrivacyCaseEvent[],identities:PrivacyIdentityView[],attachments:PrivacyAttachmentView[],artifacts:PrivacyArtifactView[],capabilities:PrivacyCaseCapabilities}`. Project only the existing safe fields; expose no raw locators, encrypted fields, storage keys, DEKs, account email or unbounded event payload. `PrivacyCaseView` includes receivedAt/dueAt/extension/timeZone/version; identities retain numeric `deliveryAuthorized:0|1` on current wire shape. +- `PrivacyCaseCapabilities` is a server-derived allowlist `{identityReview:boolean,taskDecision:boolean,taskAssignment:boolean,extendDeadline:boolean,caseDecision:boolean,attachmentWrite:boolean,backupReview:boolean,generate:boolean,assistedDelivery:boolean,representativeDelivery:boolean,reasons:Record}`. Derive it from the same state/identity/readiness predicates as handlers; it is advice, never authorization. Detail/readiness errors fail closed, not default-true. +- `usePrivacyAdmin()` exposes typed generated operations and query keys `['admin','privacy',apiOrigin,actorId,'case',requestId]`. A `PrivacyMutationIntent={key:string;requestId:string;version:number;kind:string}` is created once per explicit confirmation, survives pending response in QueryClient state, uses `retry:false`, and binds callbacks to the current origin/actor/case. After a version conflict, preserve draft, reload authority and require a new confirmation/key. +- `PrivacyIdentityReview({detail,client})` submits existing `/api/privacy/admin/requests/:requestId/identity` payloads. Manual verified review only uses `method:"exceptional_evidence"` with reasonable-doubt code and same-case live identity attachment; account-login proof and verified-email-challenge proof remain their existing guarded flows. Representative approval requires authority attachment and explicit delivery authorization, separately from identity. + +- [ ] **Step 1 — causal tests:** render pending subject/representative cases and assert exceptional-evidence/authority controls exist; verify refuses absent/wrong-case/wrong-purpose/deleted/expired evidence and missing reasonable doubt. Email challenge completion is required before an email verified label; a submitted `method:"account_login"` or `"verified_email_challenge"` still yields `ACCOUNT_LOGIN_PROOF_REQUIRED` / `EMAIL_CHALLENGE_PROOF_REQUIRED` from manual review. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project web apps/web/src/components/admin/privacy/PrivacyIdentityReview.test.tsx apps/web/src/lib/admin/privacyAdmin.test.ts`; `pnpm exec vitest run --project node apps/api/src/privacy/assisted-identity-route.test.ts apps/api/src/privacy/admin-contracts.test.ts`. Baseline lacks the full review form and typed safe boundary; require behavior failures rather than only missing imports. +- [ ] **Step 3 — minimal implementation:** extract identity endpoints unchanged into guarded plugin; canonicalize schema/projection and register operations via controller. Keep `/verify` returning 410 `IDENTITY_METHOD_REQUIRED`. Preserve subject-owned `/api/privacy/assisted-requests/:requestId/identity/account-login`; caseworker cannot impersonate that proof. Add explicit expiry checks for evidence references alongside existing purpose/case/deletion checks. +- [ ] **Step 4 — ownership/idempotency:** use one execution owner for reads and all mutations; selecting another case clears identity code, evidence selection and consent drafts immediately. Replayed identity challenge issue/complete must not rotate challenge or duplicate email silently: persist actor/case/party-bound operation fingerprints using existing event/outbox storage, and require explicit resend with a new key. Returned challenge delivery remains `sent|queued`, never fabricated verified. +- [ ] **Step 5 — GREEN:** repeat RED; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/privacy/assisted-workflow-postgres.test.ts apps/api/src/privacy/email-challenge-postgres.test.ts`. Assert stale version→409, same key/different party→409, role/loopback rejection, redacted serialization and case-switch race isolation. Controller generates contracts before executing browser SDK consumers. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(privacy): expose guarded identity and representative review`. Review proof-channel separation, capabilities vs actual guards, same-case attachment lifetime, no new personal-data projection, and full-admin approval separation unchanged. + +## Task 2: Task assignment and meaningful completion (P02) + +**Files:** Create privacy contract `tasks.ts`, `apps/api/src/routes/privacy-admin-tasks.ts`, `apps/api/src/privacy/operator-task-actions.ts`, `operator-task-actions.test.ts`, `apps/web/src/components/admin/privacy/PrivacyTaskActions.tsx`, `PrivacyTaskActions.test.tsx`; modify `apps/api/src/privacy/request-service.ts`, `apps/api/src/privacy/assisted-workflow-postgres.test.ts`, `apps/api/src/routes/privacy-admin.ts`, `apps/web/src/app/admin/privacy/page.tsx`. + +**Interfaces:** + +- Keep POST `/api/privacy/admin/requests/:requestId/tasks/:taskId/assignment` body `{version,assignedTo:string|null}` and `/decision` body `{version,status:"complete"|"not_applicable"|"operator_review"|"retryable"|"canceled",reasonCode?,redactionCode?,recordCount?}`. Use the existing schema's positive version, stable-code ≤128 characters and recordCount `0..10000000`. +- `getPrivacyAssignableOperators({limit=50,cursor?}:{limit?:number;cursor?:string}):Promise<{operators:Array<{id:string;name:string}>;nextCursor:string|null}>` backs GET `/api/privacy/admin/operators`; bounded paginated query (`limit` 1–100, default 50, opaque cursor) returns active real admin/privacy-admin IDs/names only. No broad Better Auth list-users access for privacy admins. Assignment revalidates selected operator still has authority and is not banned. +- `deriveOperatorTaskActions(request,task,registrations):{actions:TaskDecisionStatus[];reason?:string}` is shared by capabilities and mutation validation. Human terminal decisions cannot manufacture collector completion: active claimed/automated tasks deny manual complete; operator/manual-review tasks require evidence/record counts where their registration needs them. Use existing `assertAssemblyReady` as the final gate. + +- [ ] **Step 1 — tests:** assign/unassign self and another authorized operator; revoked/banned assignee fails. Complete a required operator task with recordCount=0 and valid evidence; not_applicable/canceled/operator_review require reason. Assert stale version, task from another request, terminal case and attempt to complete an actively collecting automated source all fail without updating task/event/version. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project web apps/web/src/components/admin/privacy/PrivacyTaskActions.test.tsx`; `pnpm exec vitest run --project node apps/api/src/privacy/operator-task-actions.test.ts`. Baseline UI exposes only operator_review; existing broad service mutation must not permit bypassing collector evidence. +- [ ] **Step 3 — minimal implementation:** mount focused task forms, read assignment from detail and submit exact case version/intent. Keep state transitions and case/task update+event in one transaction; derive allowable actions from manifest registration/task status, do not conflate “assigned” with complete. Refetch after `{ok:true}` decision response before using a new version; avoid guessing version increments. +- [ ] **Step 4 — GREEN:** repeat RED; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/privacy/assisted-workflow-postgres.test.ts`. Two concurrent assignments/decisions on one version yield one commit; same operation replay yields one event; different payload with same key is `IDEMPOTENCY_KEY_REUSED`. Test unresolved required task still blocks generation. +- [ ] **Step 5 — commit/review gate:** proposed commit `feat(privacy): add guarded task assignment and completion`. Review least-privilege assignee projection, collector claim preservation, evidence conditions and retained current case draft on conflict. + +## Task 3: Deadlines, clarification and refusal (P03) + +**Files:** Create privacy contract `decisions.ts`, `apps/api/src/routes/privacy-admin-decisions.ts`, `apps/api/src/privacy/case-decisions.ts`, `case-decisions.test.ts`, `case-decisions-postgres.test.ts`, `apps/web/src/components/admin/privacy/PrivacyCaseDecisionForms.tsx`, `PrivacyCaseDecisionForms.test.tsx`; modify `apps/api/src/privacy/request-service.ts`, `apps/api/src/privacy/deadline.test.ts`, `apps/api/src/routes/privacy-admin.ts`, privacy page. + +**Interfaces:** + +- Preserve `/extension` body `{version,extendedUntil:ISODateTime,extensionNotifiedAt:ISODateTime,reasonCode}` and `/decision` body `{version,decision:"refused"|"clarification_needed"|"operator_review",reasonCode}`. Add strict shared schemas instead of unchecked `Number(body.version)`. Responses remain `{request:PrivacyCaseView}`; expose current original and extended due dates in UI without accepting client-computed authoritative dates. +- `originalDueAt=calculateSubjectRequestDueAt(receivedAt,timeZone)`; `maximumExtendedUntil=addCalendarMonths(originalDueAt,2,timeZone)`. Extend only an active case, within that fixed original bound, notified no later than original due date and not in the future; reject repeated cumulative extensions beyond the original maximum. Keep stored actual notification timestamp supplied by the caseworker; never default it to “now” or silently assert notification happened. +- `recordCaseDecision(input)` / `recordCaseExtension(input)` reuse existing service transitions but store/check full normalized operation fingerprints. Same key/different decision, dates or reason→409; same-state decisions still require version CAS and audit rather than bypassing optimistic concurrency. + +- [ ] **Step 1 — tests:** month-end/leap-year/DST dates, forged/invalid/future notification, missing reason, repeated extension, stale version, terminal case and same-key different payload. UI displays original due date, selected zone and explicit confirmation for refusal; cancel leaves case unchanged. Clarification and refusal preserve real server states and existing artifact-revocation semantics. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project node apps/api/src/privacy/case-decisions.test.ts apps/api/src/privacy/deadline.test.ts`; `pnpm exec vitest run --project web apps/web/src/components/admin/privacy/PrivacyCaseDecisionForms.test.tsx`. Capture current repeated-extension bound and idempotency mismatch behavior before patching. +- [ ] **Step 3 — minimal implementation:** use focused decision service/route module; keep `request-service.ts` thin delegators where extraction is necessary. Implement fixed-original deadline bound and payload-bound idempotency within case row locks/transactions, preserving notification outbox and exact reason-only audit projections. No generic “set state” endpoint and no legal prose in audit payloads. +- [ ] **Step 4 — GREEN:** repeat RED; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/privacy/case-decisions-postgres.test.ts apps/api/src/privacy/lifecycle-postgres.test.ts`. Assert one winner on simultaneous decisions, exact replay returns retained result, failed transitions do not send notification or create success audit, and removed/refused case still displays its retained receipt. +- [ ] **Step 5 — commit/review gate:** proposed commit `feat(privacy): add versioned deadline and case decisions`. Review fixed original deadline versus already-extended dueAt, no fabricated notification evidence, no automatic retries, and server transition semantics preserved. + +## Task 4: Encrypted attachments, rights review and passive download (P04) + +**Files:** Create privacy contract `attachments.ts`, `apps/api/src/routes/privacy-admin-attachments.ts`, `apps/api/src/privacy/attachment-review.ts`, `attachment-review.test.ts`, `attachment-review-postgres.test.ts`, `apps/web/src/components/admin/privacy/PrivacyAttachments.tsx`, `PrivacyAttachments.test.tsx`; modify `apps/api/src/privacy/attachments.ts`, `apps/api/src/privacy/request-service.ts`, `apps/api/src/routes/privacy-admin.ts`, privacy page. Preserve `apps/api/src/privacy/collectors/attachments.ts` contract; add integration assertions to its existing collector tests. + +**Interfaces:** + +- Existing upload body remains `{purpose,filename,mediaType,contentBase64,metadata?}` plus required positive integer `version` in the canonical admin contract; retain HTTP parser limit `ceil(25*1024*1024*4/3)+16*1024`. Supported purposes: `identity_evidence|representative_authority|processor_response|operator_supplement`; MIME: PDF, plain text, CSV, JSON, PNG, JPEG. Decoded limit is **25 MiB**; one active upload per case/browser, no large unbounded conversion. +- Existing DELETE/download paths remain; DELETE gets required body `{version}` via canonical contract. Add POST `/api/privacy/admin/requests/:requestId/attachments/:attachmentId/review` body `{version,decision:"approved"|"rejected",reasonCode}`. Case-version CAS, same-case live attachment, active case, real privacy actor and receipt fingerprint are mandatory. Upload starts `rightsReviewState:"pending"` and client cannot set it. +- Download operation ID `downloadPrivacyAdminAttachment` is GET `/api/privacy/admin/requests/:requestId/attachments/:attachmentId/download`; canonical200 uses06.2 `adminBytes("application/octet-stream",privacyDownloadHeaders,true)`, normalizing the passive download Content-Type while retaining validated original MIME in attachment metadata. `privacyDownloadHeaders` validates Content-Disposition attachment/sanitized filename, optional valid nonnegative Content-Length, private no-store, nosniff and no-referrer; no paths/locators. Preserve canonical JSON errors (including401/403/404/416 and actual bounded privacy codes) and no Range. Register under `privacyAdmin`, not the full-admin service family. +- Generated `downloadPrivacyAdminAttachment(requestId:string,attachmentId:string,options:AdminByteResponseRequestOptions):Promise` delivers raw Uint8Array chunks through `onByteChunk(chunk,signal)` and optional `onByteHeaders`;200 is `{status:200;headers:Headers;data:undefined}` only at EOF/callback settlement, errors remain typed JSON and bounded64KiB. Generated `getDownloadPrivacyAdminAttachmentTarget(requestId:string,attachmentId:string,options?:AdminNativeDownloadTargetOptions):Promise` only constructs06.2's capability. IDs are encoded by canonical binders, never concatenated by UI. +- `reviewCaseAttachment(input,tx):Promise<{request:PrivacyCaseView;attachment:PrivacyAttachmentView}>`; only processor_response/operator_supplement can be approved for export. To supply redacted material, operator uploads a separately redacted replacement with provenance/redactionCode, explicitly rejects the original and reviews the replacement. No flag claims the server transformed bytes; identity/authority documents never enter the export collector. + +- [ ] **Step 1 — tests:** 25 MiB boundary and +1, malformed base64, false MIME signature, active/archive extension, path filename, missing supplement provenance, stale case, wrong case and duplicate submission. Assert rejected upload leaves no DB row/storage leak, valid upload stays pending, and approval requires fresh version/reason. Only approved live supplements appear in actual export; evidence/authority files never do. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project web apps/web/src/components/admin/privacy/PrivacyAttachments.test.tsx`; `pnpm exec vitest run --project node apps/api/src/privacy/attachment-review.test.ts`. Baseline has no upload/review/download/delete UI and no guarded rights-review route. +- [ ] **Step 3 — minimal implementation:** extract existing attachment handlers into guarded plugin. Validate version, case and mutation fingerprint before storage writes; serialize concurrent same-key uploads on case ownership and clean only storage created by that attempt if transaction fails. Add rights review with bounded event metadata, no attachment contents. Use06.2's generated native target only when available; otherwise use the generated byte callback into a supported user-selected browser file sink. No `fetch().blob()`, raw Response, text decoder or XHR for downloads. Keep upload on its existing explicitly bounded25MiB canonical JSON/base64 contract, with size checked before conversion; do not introduce a competing bulk transport. +- [ ] **Step 4 — retain delivery guards:** attachment download already requires session assurance within 10 minutes, safe Content-Disposition, no Range and a final case-state check. Also require `isAssuranceSufficient(method,currentUser.twoFactorEnabled)` for configured MFA, not merely recent timestamp. Keep fresh-login return separate from artifact challenge consumption; no URL secrets, inline HTML/PDF/image evidence rendering, or browser object URLs retained after case switch. +- [ ] **Step 5 — GREEN:** repeat RED; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/privacy/attachment-review-postgres.test.ts apps/api/src/privacy/assisted-workflow-postgres.test.ts`; `pnpm exec vitest run --project node apps/api/src/privacy/collectors.test.ts apps/api/src/privacy/artifact-storage.test.ts`. Include concurrent upload/delete/review, revoked MFA/session, storage failure cleanup, stale file selection and case-switch error isolation. Test non-UTF8 attachment bytes through actual route and generated binder, typed≤64KiB errors, and abort after headers while the file sink is blocked: upstream/sink cancellation completes before request cleanup and no bytes enter a different case. Actual native navigation is tested only for the supported same-origin credential configuration. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(privacy): manage encrypted case evidence and supplements`. Review receipt/row/storage ordering, all bounds, original/replacement evidence, no raw secret/PDF contents in logs and no automatic approval/readiness shortcut. + +## Task 5: Trusted backup review and omission acceptance (P05) + +**Files:** Create privacy contract `backups.ts`, `apps/api/src/routes/privacy-admin-backups.ts`, `apps/api/src/privacy/admin-backup-review.test.ts`, `apps/web/src/components/admin/privacy/PrivacyBackupReview.tsx`, `PrivacyBackupReview.test.tsx`; modify `apps/api/src/routes/privacy-admin.ts`, `apps/api/src/privacy/backup-review.ts`, `apps/api/src/privacy/backup-omission-review.ts`, `apps/api/src/privacy/backup-omission-review-postgres.test.ts`, privacy page. + +**Interfaces:** + +- GET `/api/privacy/admin/backups` retains descriptor-verified path-free inventory. POST `/api/privacy/admin/requests/:requestId/backup-reviews` body remains **`{backupId}`**, not caller-supplied provenance/decision; service rechecks active case and immutable backup review before storing. Add GET on that case path returning safe reviews for refresh/reopen (IDs, verified manifest digest, decision/reason, reviewer/time; never volumes' host paths). +- Preserve GET `/:requestId/backup-omissions` and POST `/backup-omissions/accept` body `{warningsDigest,reasonCode,requestVersion}` exactly. A warning digest is bound to review ID, manifest digest and warning codes; receipt is immutable. UI shows server-derived `not_applicable|no_material_difference|extract|unavailable` and reasons; it never allows operators to type a manifest digest or force extract. + +- [ ] **Step 1 — tests:** trusted backup review sends only backupId; rejected injected digest/decision proves strict input. Corrupt/unverified/unsupported/missing inventory remains unavailable with reason, API outage is an error with retained good data, not an empty backup list. Same-key same backup replays one immutable review; same key different backup fails. Omission acknowledgment requires displayed digest/reason/current requestVersion and cannot approve unseen changed warnings. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project web apps/web/src/components/admin/privacy/PrivacyBackupReview.test.tsx`; `pnpm exec vitest run --project node apps/api/src/privacy/admin-backup-review.test.ts`. Baseline silently turns inventory error into `[]` and has no selected-case trusted-review submission. +- [ ] **Step 3 — minimal implementation:** extract existing backup review handlers and expose safe review list; check request existence/active state under a row lock before recording review. Preserve server-side ops inventory hashing and `BACKUP_REVIEW_IMMUTABLE`; do not change `backup.list` authority or accept host paths. Keep warning review on existing service, without adding generic task completion around omitted backup sources. +- [ ] **Step 4 — GREEN:** repeat RED; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/privacy/backup-omission-review-postgres.test.ts`; `pnpm exec vitest run --project node apps/api/src/privacy/backup-review.test.ts apps/api/src/privacy/backup-omission-review.test.ts`. Assert case/refusal race cannot record a new affirmative review and unaccepted warnings keep assembly blocked. +- [ ] **Step 5 — commit/review gate:** proposed commit `feat(privacy): expose trusted backup review workflows`. Review no fake empty inventory, immutable review provenance, warning digest freshness, no forced extraction and existing readiness policy preserved. Actual retained-backup/export-restore tests remain a separate controller gate, not a mocked ops-list pass. + +## Task 6: Reauthenticated assisted artifact delivery and complete UI qualification (P06) + +**Files:** Create privacy contract `delivery.ts`, `apps/api/src/routes/privacy-admin-delivery.ts`, `apps/api/src/privacy/admin-delivery-route.test.ts`, `apps/web/src/components/admin/privacy/PrivacyArtifactDelivery.tsx`, `PrivacyArtifactDelivery.test.tsx`, `apps/web/src/lib/admin/privacyReauthentication.ts`, `privacyReauthentication.test.ts`, `apps/web/src/lib/admin/privacyDownload.ts`, `privacyDownload.test.ts`, `apps/web/src/app/auth/privacy/admin/page.tsx`; modify `apps/api/src/routes/privacy-admin.ts`, `apps/api/src/privacy/delivery-postgres.test.ts`, `apps/web/src/app/admin/privacy/page.tsx`, `apps/web/src/app/admin/privacy/page.test.tsx`, `docs/docs/administration/data-subject-requests.md`. Preserve self-service `/auth/privacy` behavior. + +**Interfaces:** + +- Keep POST `/:requestId/artifacts/:artifactId/reauth/start` body `{channel:"assisted"|"representative",reasonCode}`, response `{challengeId,expiresAt,loginPath}`; complete body `{challengeId}`, response `{completed:true}`; GET `/download` consumes cookie/challenge and streams bytes. Only admin start changes `loginPath` to fixed `/auth/privacy/admin`; self-service still returns `/auth/privacy`. Use existing `PrivacyReauthenticationService` and `__Host-openmapx-export-reauth`, not plan 03's unrelated administrator-action challenge. +- `PendingPrivacyAdminDelivery={apiOrigin,actorId,requestId,artifactId,challengeId,expiresAt,channel}` is the only tab-scoped resume record; no nonce, cookie, token, locator or recipient address. `PrivacyArtifactDelivery({detail,client})` explicitly confirms artifact, channel and reason, starts reauth, navigates to interactive AuthDialog and resumes the selected case after login. `/auth/privacy/admin` has fixed callback `/admin/privacy`; case selection comes from the validated resume record, not arbitrary URL redirect data. +- Canonical operation `downloadPrivacyAdminArtifact`: GET `/api/privacy/admin/requests/:requestId/artifacts/:artifactId/download`,200 `adminBytes("application/octet-stream",privacyDownloadHeaders,true)` plus canonical bounded JSON errors and416 Range denial. Normalize passive download type to octet-stream; retain artifact filename/original format metadata. Generated `downloadPrivacyAdminArtifact(requestId:string,artifactId:string,options:AdminByteResponseRequestOptions):Promise` and `getDownloadPrivacyAdminArtifactTarget(requestId:string,artifactId:string,options?:AdminNativeDownloadTargetOptions):Promise` mirror the exact attachment signatures. Replace the old hand-built string target. No query token, whole-archive Blob, mail attachment or export manipulation. Transport success is not server delivery receipt; only `recordSuccessfulArtifactDelivery` after server stream finish means delivered. +- Shared `PrivacyDownloadSink` in `privacyDownload.ts` is `{write(chunk:Uint8Array,signal:AbortSignal):Promise;close():Promise;abort(reason:unknown):Promise}`. `consumePrivacyDownload({kind:"attachment"|"artifact",requestId:string,itemId:string,client:ApiClient,sink:PrivacyDownloadSink,signal:AbortSignal}):Promise` calls the respective generated byte operation with awaited sink writes, `retry:false` in owning mutation logic, and typed status unwrapping; on200/EOF it awaits close, on any error/abort it awaits abort and preserves error. Sink adapter uses an actually supported `showSaveFilePicker`/`createWritable` capability from an explicit user click, handles cancellation, installs an AbortSignal handler that aborts the underlying writable, and awaits both the pending write and abort cleanup before settling. It closes/aborts only owned handles. No in-memory fallback and no File/handle/bytes in QueryClient or sessionStorage. Backpressure permits at most one pending chunk/write. Acquire the sink on explicit Download before issuing the one-use GET, not during an automatic resume. +- Native target capability is06.2's same-origin HTTP(S), credentials include/same-origin, no headerInterceptor/custom request headers, browser-only contract. An explicit `credentials:"omit"`, alternate API origin, any interceptor (even returning empty now) or custom-header client cannot construct a native target. Preserve the given client on the callback-to-sink path; never copy private config into XHR, change credentials or silently navigate to another origin. If no supported sink exists, expose `ApiNativeDownloadUnavailableError.reason` with reconnect-to-supported-origin/cookie-session guidance before starting a delivery ceremony. A sink does not bypass privacy's cookie/reauth guards: omit clients lacking those credentials still receive the actual401/403 and zero bytes. Separate-origin callback tests use a genuinely permitted same-site cookie/CORS deployment; do not claim cross-site Strict cookies are supported. +- Generated byte transport owns auth/interceptors/timeout/abort/body through every awaited chunk callback and EOF/cancel cleanup (06.2), not merely headers. A sink abort/case switch cancels that owned byte request; native navigation has `{ownership:"browser"}` and cannot promise programmatic progress/cancellation. No automatic GET retry, HEAD probe or prefetch may consume the one-use artifact challenge. After any uncertain/interrupted delivery, refresh server status and require a new explicit ceremony for another GET. + +- [ ] **Step 1 — tests:** ready artifact gains an action, expired/revoked/not-ready/readiness-denied rows show reasons, and representative delivery requires verified subject+representative, approved authority and `deliveryAuthorized=1`. Assert ceremony start→new sufficiently-assured login→complete→one-use download; same session, wrong actor/artifact/case/channel, old login, password-only with MFA, superseded/expired nonce and repeated GET yield no bytes. Register both exact privacy byte operation IDs in canonical registries, transport family bindings and generator exports; test real response headers/serialization and non-UTF8 equality. Same-origin include/no-interceptor target construction plus actual browser navigation succeeds; alternate-origin/omit/custom-header/interceptor targets explicitly refuse. Normal callback transport retains the provided client and awaits a slow sink; abort/timeout after headers cancels upstream and sink before timers/listeners are released. Unsupported sink/native combination disables Download with accurate guidance instead of buffering the archive. +- [ ] **Step 2 — RED:** `pnpm exec vitest run --project web apps/web/src/components/admin/privacy/PrivacyArtifactDelivery.test.tsx apps/web/src/lib/admin/privacyReauthentication.test.ts apps/web/src/lib/admin/privacyDownload.test.ts`; `pnpm exec vitest run --project node apps/api/src/privacy/admin-delivery-route.test.ts`. Baseline only renders artifact text and returns self-service login path for admin ceremony. +- [ ] **Step 3 — minimal implementation:** extract delivery routes preserving verification-before-authorize stream and final identity/case checks. Tighten existing event idempotency: start replay can return the same active challenge only with its still-matching nonce cookie; otherwise return bounded `REAUTH_RESTART_REQUIRED`, never silently supersede a live challenge under the same key. Complete replay checks its fingerprint/session and emits no duplicate event. New deliberate ceremony uses new key; no auto retry of start/complete/download. +- [ ] **Step 4 — guarded navigation:** validate resume IDs/origin/expiry and actor against fresh server session; a changed actor/case clears resume state, never autoselects another artifact. Preflight supported target/sink capability before ceremony; completion offers one explicit Download and then clears resume intent; browser stream interruption shows “check delivery status” and requires a new ceremony for another attempt. Refresh state from server; never mark delivered on click or headers alone. +- [ ] **Step 5 — GREEN:** repeat RED; `OPENMAPX_RUN_DATABASE_TESTS=1 pnpm exec vitest run --project node --maxWorkers=1 apps/api/src/privacy/delivery-postgres.test.ts apps/api/src/privacy/assisted-workflow-postgres.test.ts`; `pnpm exec vitest run --project node apps/api/src/privacy/reauthentication.test.ts apps/api/src/privacy/delivery-regression.test.ts`; `pnpm exec vitest run --project web apps/web/src/app/admin/privacy/page.test.tsx`. Add a barrier after artifact integrity verification, revoke representative authority/withdraw case, then assert no response bytes and no delivered event; preserve disconnect/no-finish delivery behavior. +- [ ] **Step 6 — commit/review gate:** proposed commit `feat(privacy): enable guarded assisted artifact delivery`. Run affected package types (`pnpm -C apps/api check-types`, `pnpm -C packages/core check-types`, `pnpm -C apps/web check-types`). Controller verifies real desktop/mobile queue, identity/representative forms, task completion, stale-version error, deadline/refusal, upload/review, backup outage, auth return and streamed download with synthetic data. No screenshots uploaded and no human release approval fabricated. + +## Controller qualification and self-review + +P01 → Task 1; P02 → Task 2; P03 → Task 3; P04 → Task 4; P05 → Task 5; P06 → Task 6. All five Review Focus conditions have explicit tests. Task order permits component work after contracts, but `privacy-admin.ts` extraction and `request-service.ts` service changes execute serially under one owner. + +Controller serially aggregates privacy-admin contracts/auth metadata, runs `pnpm openapi:generate` then `pnpm sdk:generate`, inspects exact JSON/binary response serialization and validates `pnpm check-openapi` / `pnpm check-api-sdk`. Register the new case-decisions/attachment-review PostgreSQL tests in api-privacy group. Run `pnpm test:database api-privacy`; run `pnpm test:database export-restore` only with its owned retained-backup fixture, reporting unavailable gates separately. Master plan owns full verification, production builds, independent final review and CI. + +Self-review completed: source schemas and routes were read before choosing forms; verified method selection cannot fabricate proof; existing `version` versus `requestVersion` distinction is retained; supplement approval and artifact delivery cannot skip evidence/MFA/readiness; bounded attachment upload retains its current transport; both attachment/artifact downloads use06.2's canonical byte callbacks or native capability with body-lifetime ownership. Exact operation IDs and generated signatures above are the required06.2 generator/privacy family exports, not standalone URL helpers. Found source gaps in same-state decision CAS, cumulative extension bounds, replay fingerprinting and attachment review are assigned concrete tasks above. No tests ran during planning. Controller rulings are shared registrar/SDK-family registration and serialization order, not unresolved user-facing design questions. diff --git a/docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md b/docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md new file mode 100644 index 000000000..d22da277e --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md @@ -0,0 +1,240 @@ +# Platform and Service Configuration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox syntax for tracking. + +**Goal:** Make supported service resources and platform/domain/TLS/release settings manageable through an honest, coordinated admin workflow. + +**Architecture:** Keep non-secret platform ownership in the applied configuration generation and resolve it identically in CLI/API/agent consumers. Extend manifest-backed service forms with explicit safe settings; privileged application remains behind sealed typed operations and existing revision CAS. + +**Tech Stack:** TypeScript, Zod, Fastify, React/MUI, configuration generations, generated service-admin SDK, Vitest. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md`. +Shared execution/lease/candidate contracts are defined by +`docs/superpowers/specs/2026-10-23-admin-control-protocols.md`; use those exact +interfaces once finalized, without inventing a competing coordinator. + +## Global Constraints + +All specification constraints apply. Cover C01-C07. Do not expose arbitrary +environment editing, shell commands, registry credentials, executable JVM/Node +options, bind paths, Docker privilege/network settings or platform bootstrap keys. +Preserve current memory/command defaults until the operator saves an override. +Environment ownership changes require an explicit previewed adoption action. + +## Review Focus + +1. A no-op save must retain byte-equivalent runtime behavior, including capitalized Valhalla booleans and symbolic MOTIS memory defaults. +2. A stale revision, pointer commit followed by failure, or lost result cannot restore unrelated configuration or falsely report non-commit. +3. Domain/TLS changes must update routing, API/auth origins, web runtime environment and reconnect guidance coherently. +4. Resource settings must change Docker resources rather than become inert container environment variables. +5. Host-owned values and bootstrap credentials must not be disclosed or silently superseded by a database save. + +## Task C1: Canonical non-secret platform state and ownership + +**Files:** Create `packages/core/src/services/platform-configuration.ts` and +`packages/core/src/services/__tests__/platform-configuration.test.ts`. Modify +`packages/core/src/ops/trusted-config.ts`, `packages/core/src/services/configuration-generation.ts`, +`packages/core/src/services/index.ts`, `packages/cli/src/commands/compose.ts`, +`apps/api/src/services/trusted-config-operations.ts`. Extend existing +configuration-generation, trusted-config and CLI-compose tests. + +**Interfaces:** Export `PlatformField = { source: "host" } | { source: +"database"; value: T }`; `PlatformConfiguration` has version `1` and optional +`domain`, `acmeEmail`, `releaseSelector`, `defaultRegion`, `backupRetentionDays` +fields. Export `resolvePlatformConfiguration({ saved, env, overrides })` returning +`{ values, sources, warnings }`. Values are non-secret. Add optional `platform` +to `TrustedConfigurationPayload`; older generations without it keep host behavior. +The existing required `domain` equals the resolved domain. Explicit CLI arguments +override for that invocation; absent flags do not replace persisted adoption. + +Resolved values have exact keys `{domain:string,acmeEmail:string, +releaseSelector:string,defaultRegion:string|null,backupRetentionDays:number}`. +Host mappings/defaults are DOMAIN/localhost, ACME_EMAIL/admin@example.com, +OPENMAPX_RELEASE/main, OPENMAPX_REGION/null and +BACKUP_RETENTION_DAYS/30. Explicit platform adoption previews that +these known generated slots will use UI ownership instead of host values until +Return to host is applied; this is not permission to adopt arbitrary env keys. +Service/integration fields retain file/env precedence and manual host handoff. + +- [ ] Add tests: host defaults win without adoption; explicit database adoption survives later CLI render; explicit CLI override applies; returning ownership to host clears the database value; malformed domains/email/selector/region/retention reject; old generations remain readable; secrets never appear in inspection/export. +- [ ] Run `pnpm test packages/core/src/services/__tests__/platform-configuration.test.ts packages/core/src/services/__tests__/configuration-generation.test.ts packages/core/src/ops/trusted-config.test.ts packages/cli/__tests__/cli-compose.test.ts --maxWorkers=3`; verify causal RED for missing state/resolution. +- [ ] Implement the resolver and generation persistence/readers. Domain is a hostname or localhost (no URL/port/path/credentials); ACME email <=320 characters and syntactically validated; selector uses `normalizeReleaseSelection`; region uses existing allowed resource identifiers; retention is integer 1-36500 days. Host/DB ownership is metadata, never an unvalidated env map. +- [ ] Thread state through CLI/API preparation under existing writer/CAS semantics. Do not read the database from core/CLI. Preserve the platform projection when applying unrelated service/integration changes. +- [ ] Route default-region consumers through this resolver: CLI default region lookup, agent fixed CLI invocation and generated data-manager OPENMAPX_REGION. Explicit per-operation regions remain allowed only by the agent deployment allowlist. A saved default cannot broaden deployment source/region authority. +- [ ] Re-run the named tests to GREEN and commit the complete canonical-state task. + +## Task C2: Safe manifest settings and actual resource rendering + +**Files:** Modify all 30 current `services/*/service.json` built-in manifests +(`app-api`, `app-web`, `brouter`, `data-manager`, `dawarich-app`, +`dawarich-postgis`, `dawarich-redis`, `dawarich-sidekiq`, `elasticsearch`, `local-ai`, +`martin`, `motis`, `motis-feed-proxy`, `motis-staging`, `nominatim`, `ops-agent`, +`osrm`, `otp`, `overpass`, `pelias`, `pelias-pip`, `pelias-placeholder`, `photon`, +`postgis`, `redis`, `tileserver`, `traefik`, `transitous-runner`, `valhalla`, +`well-known`). Assert this exact inventory before editing and explicitly reconcile +any added or removed manifest rather than silently omitting it. Modify +`packages/core/src/services/compose-renderer.ts`, `config-resolver.ts`, +`apps/api/src/services/service-config-resolver.ts`. Create focused +`packages/core/src/services/runtime-resources.ts` and tests; extend manifest and +renderer tests. Also modify `apps/web/src/integration-api/runtime/{env.ts,EnvProvider.tsx}`, +`apps/web/src/integration-api/overlay/overlayZoomGate.ts`, +`apps/web/src/lib/serverClientEnv.ts` and their tests; extend the actual +`integrations/overlay-traffic-tomtom` layer tests. No new untrusted manifest execution capability. + +**Interfaces:** Reserved numeric fields `OPENMAPX_MEMORY_MIB` (integer16-1048576) +and `OPENMAPX_CPU_LIMIT` (number0.125-256) target Docker `deploy.resources.limits`, +not environment. Absence preserves the manifest's limit, including interpolation. +`resolveServiceRuntimeResources(manifest, values)` returns `{ memory?: string; +cpus?: string }`. Strip reserved keys from emitted environment. Use the same +strict numeric bounds in schema and renderer; community manifests cannot use +these reserved fields to escape their existing sandbox/resource authority. + +- [ ] Add a manifest inventory test proving every current built-in has operator resource metadata and existing declared fields remain intact. Add render tests proving actual memory/CPU placement, no reserved environment keys, omitted overrides preserve defaults, invalid values reject and unrelated service output stays unchanged. +- [ ] Run `pnpm test packages/core/src/services/__tests__/manifest-schema.test.ts packages/core/src/services/__tests__/compose-renderer.test.ts packages/core/src/services/__tests__/community-service-config.test.ts --maxWorkers=3` plus the new resource tests; verify RED before changing production rendering. +- [ ] Implement exactly the supported fields in the table below, plus common resource controls for all30 built-ins. Keep all five existing schemas. Preserve scalar serialization required by each image (Valhalla booleans are `True`/`False`). Add a field-to-runtime-consumer table in administration docs. Do not invent settings for images without a supported knob. +- [ ] For heap controls expose bounded numeric MiB, never free-form `JAVA_TOOL_OPTIONS`, `ES_JAVA_OPTS` or `NODE_OPTIONS`; synthesize only the known heap flag for the specific built-in consumer. Do not expose shell-containing preprocessing/command fields. Treat MOTIS primary/staging resource changes as coordinated operations under current capacity policy. +- [ ] Ensure managed web environment reads occur on the server at runtime, avoiding build-time `NEXT_PUBLIC_*` substitution for fields promised as live configurable. Verify with rendered-server/production-build tests, not only source object assertions. +- [ ] Complete the existing raster traffic zoom knob: `ClientEnv.trafficMinZoom?:number` reads validated integer0-24 from NEXT_PUBLIC_TRAFFIC_MIN_ZOOM, with invalid/unset omitted. Add `useOptionalEnv():ClientEnv|null` beside existing strict `useEnv`; `useOverlayMinZoom('traffic')` uses the configured value or the registry manifest's default10. Other overlays retain their manifest policy, including traffic-flow default6. Both the raster layer and layer-selector gate already consume this shared hook; test they agree at changed zoom8, malformed input retains10, and unrelated gates remain unchanged. This is a raster traffic control; do not silently alter the separate vector-flow integration. +- [ ] Share a bounded URL parser for API/UI/migration: empty means default; allow absolute http(s) with no userinfo, or a same-origin root-relative path beginning one `/`, never `//`, backslashes or controls. Ordinary base/TileJSON/attribution URLs accept no placeholders; tile-template fields permit only `{z}`,`{x}`,`{y}`,`{s}` supported by their consumer. Preserve `/local-dem/tiles.json`, `/local-contours/tiles.json` and existing same-origin tiles/glyph-base paths. Map style legacy key denotes the glyph server base; do not mislabel it as a complete style JSON URL. +- [ ] All service forms/imports use plan01's strict `prepareProspectiveConfiguration` candidate and plan04's effective-source guard before persistence; a failed service read or total sealed snapshot overflow is a refusal, never a default generation. Resource ownership detects both SERVICE_* overrides and supported host variables referenced by the manifest's resource expression. +- [ ] Run focused schema/resolver/renderer tests, `pnpm check-ops-authority` and affected types to GREEN; commit the field/resource task. + +## Task C3: Typed platform preview/apply and recovery + +**Files:** Create `packages/core/src/api/contracts/service-admin/platform.ts`, +`apps/api/src/routes/admin-platform.ts`, `apps/api/src/services/platform-configuration.ts`, +`apps/ops-agent/src/platform-configuration-runtime.ts` and focused tests. Modify +service-admin `index.ts`, core ops contract/policy/trusted-config handlers, +`apps/api/src/routes/index.ts`, `apps/ops-agent/src/index.ts`, and generation +rendering adapters. Create `apps/ops-agent/src/configuration-handoff.ts` and +`configuration-handoff-main.ts` plus tests, reusing the existing confined +`agent-handoff.ts` Docker/helper machinery. Modify `server.ts`, +`administrative-runtime.ts`, build-runtime entrypoints and API job recovery in +the same task window. These shared files are controller-serialized with plans01/02. + +**Interfaces:** Service-admin operation IDs `getAdminPlatformConfiguration`, +`previewAdminPlatformConfiguration`, `applyAdminPlatformConfiguration` at +`GET /api/admin/platform/configuration`, `POST .../preview`, `POST .../apply`. +Preview input is `{ expectedCurrentRevisionId: string|null, changes: PlatformConfiguration }`; +output includes bounded effective values/source metadata, affected service IDs, +old/new public origin, warnings, preview fingerprint and required confirmation. +Apply binds the exact revision/fingerprint to durable API job +`platform.apply`. Consume01.2/control-protocol§5.4: whole-candidate/schema/source/size/queue/private-receipt preflight FIRST, then ownership-only +`configurationMaintenance.prepare` and its durable DM fence before desired rows/proofs commit. +Under that remote fence recheck proof/policy/source/config and atomically save desired rows, +consume proof and checkpoint the exact continue decision. Only then publish the retained +snapshot using existing typed `stack.render`/`applyTrustedConfigurationSnapshot`; +ordinary follow-up and protected handoff borrow that SAME frozen maintenance root. +API cannot construct host authority or acquire a second root. Do not create a long-lived `platformConfig.apply` agent ancestor +that waits while the helper tries to drain it. The API job survives/reconciles +through child/handoff identities; result retains committed revision, follow-up +status and recovery instructions. + +`ProtectedConfigurationServiceId = 'postgis'|'redis'|'app-web'|'app-api'|'traefik'|'ops-agent'`. +API-role typed `configurationHandoff.prepare` input is `{handoffId,maintenanceId,candidateId,planDigest,serviceIds, +expectedCurrentRevisionId,expectedImageIdentities,previewFingerprint}`. +These fields match01.2's frozen plan; expectedCurrentRevisionId is the desired generation +already committed by its exact stack.render receipt. The previous generation remains +`ConfigurationMaintenanceBinding.expectedCurrentRevisionId`, never a new pointer sample. IDs are a +nonempty unique subset of that finite set; image expectations must match the +currently owned containers and trusted current pins, never a request to select a +new image. Result is only `{handoffId,phase:'prepared'}`, not applied. Typed +`configurationHandoff.inspect {handoffId}` returns safe retained +`{phase:'prepared'|'draining'|'applying'|'verifying'|'applied'|'rolled_back'|'failed'|'recovery_required', +expectedRevisionId,appliedRevisionId:string|null,completedServiceIds,failedServiceIds,recoveryCodes}`. +Both use strict schemas/operation identity and existing journal bounds; status +inspection remains allowed during handoff. Same-image configuration and release +replacement are distinct modes; do not call the current updateAgent no-op branch. + +The helper record is a bounded versioned configuration-handoff variant with +parent operation key/fingerprint, exact old/new generation identities, verified +image/container identities, target subset, deadline, lease delegation and +launch-attempt/phase evidence. It lives at a fixed agent-owned location; no path +crosses the API. Retain completed evidence for the existing journal retention. +The helper adopts only§3.5's finite agent→configuration-helper grant, at depth2 +under the maintenance root/handoff child, and cannot broaden target/image authority. +It retains maintenanceId/candidateId/planDigest and02.3's exact DM fence receipt; +no second maintenance flag, DM reservation or target root is created. No configuration writer lock survives process +replacement: acquire/revalidate it in the helper after adopting the target lease. + +For a domain change, consume plan03's `platform.domain.apply` proof bound to the +actor and exact preview fingerprint. It requires a freshly successful supported +password login plus the user's effective TOTP/recovery factor. Email-OTP is +ineligible because the installed plugin does not produce the required MFA +assurance. Neither a password row nor a domain-bound passkey/OAuth session qualifies by itself. Preview +and confirmation retain the policy revision and external DNS/TLS/OAuth +acknowledgments. A passkey-only/unknown-password administrator must establish and +prove an alternate supported login before apply; no security-policy weakening. +Export `assertDomainRecoveryPreview({actorId:string,previewFingerprint:string, +expectedCurrentRevisionId:string|null,expectedAuthPolicyRevision:string}, +tx:AuthDatabase):Promise` from `platform-configuration.ts`. It checks the +actor-bound unexpired stored preview, expected configuration and live policy +revision in the same desired-row transaction that consumes the proof and checkpoints +the retained `platform.apply` continue decision, after whole-candidate preflight and +remote ownership-only preparation. Initial job/prepare admission never consumes it. +Recheck the retained policy/preview before the first host effect.03's challenge +stores the preview/policy binding without importing this later module. + +- [ ] Add API/agent integration tests: unauthorized caller/unknown field/stale revision reject before writes; concurrent service config retains unrelated fields; commit-follow-up failure preserves commit evidence; changed preview cannot apply; hostname/TLS/web/auth values agree; helper/API replacement resumes the same job after disconnect. +- [ ] Run the new platform API/runtime tests plus trusted-config tests to expected RED. +- [ ] Implement platform preview and durable apply using plan01's supported action capabilities and plan02's retained outcomes. Materialize adopted domain/ACME values in known renderer slots so stale host `${DOMAIN}`/`${ACME_EMAIL}` interpolation cannot undo the chosen values. Never perform general template evaluation or rewrite `.env`. +- [ ] Preview exact origin consumers: routing, CORS_ORIGIN, BETTER_AUTH_URL, PASSKEY_RP_ID/PASSKEY_ORIGIN, app-web API URL and tileserver public URL. Require the proven alternate-login ceremony above, explicit DNS/TLS/OAuth/passkey acknowledgment and retained reconnect/host recovery evidence. Tests deny passkey-only and random-unknown-password rows, accept a newly proven password+requiredMFA, and reject a proof for another preview or policy revision. +- [ ] Prepare the exact prospective snapshot and all budgets/private receipts before remote ownership admission. Consume01.2's `prepare/inspect/decide/settleConfigurationMaintenance`: agent acquires frozen targets then02.3's durable DM guard; under prepared ownership recheck candidate/pin/source/config/proofs, commit desired rows/proofs/decision in one transaction, and authorize only the retained continuation. Known rollback may abandon only the exact unused reservation; unknown save/transport remains fenced/lookup-only. Ordinary generation/hardlink/recreation children and protected handoff use the same authority; do not reacquire targets after DB save. Before helper drain `configurationHandoff.prepare` must have a durable terminal prepared result and no held parent writer; the API job observes the handoff independently. The launched helper waits for that original terminal record and drains blocking host activities (mutating operations and active byte transfers), refusing new mutations/byte streams while allowing inspection/logs. Read-only log observations cannot indefinitely block drain. It then recreates in fixed dependency order: requested postgis/redis (health first), web, API, edge and agent last. Actor/job/intent stays retained while API/DB/origin is unavailable. The DM guard is a terminal prepared logical record, never a live worker owning a lease to be drained. DM recreation preserves its private startup mirror and DB guard; the new process admits no cron/API/CLI/data work before loading the fence. Only the frozen signed generation-hardlinks leaf may execute, and it settles before recreation. A target with unresolved backend work or unsupported health/recovery prerequisites is blocked before committing; do not queue a generic apply that cuts its coordinator. +- [ ] The helper uses fixed same-image Compose recreation and verifies actual environment/resource/mount identities against the exact desired generation, plus normal health and image identity. Secret-backed comparisons remain private/keyed; no public raw values or guessable secret hashes. Record each service's verified phase before proceeding. Resource-only agent changes must really change Docker memory/CPU even when image identity is unchanged. The helper survives the old agent's exit; newly started API/agent recovers the same handoff rather than relaunching it. +- [ ] Persist helper launch intent before Docker create and use a deterministic owned helper identity. Lost launch response reconciles Docker/journal evidence, never blindly creates another helper. Unprovable launch/application is recovery_required with the target fence retained. Automatic rollback may restore the exact previous generation/target configuration only when no competing revision/owner exists and old identities are proven; restoring a release overlay alone is insufficient. A newer writer or unprovable state yields unknown recovery, not a claimed rollback. +- [ ] Settle through01.2 only after exact desired generation/environment/resource/mount/readiness and every child/helper receipt prove commit or containment. DB/cache loss keeps DM startup admission closed via its private mirror; missing/unknown evidence never clears it. The same maintenanceId keys the existing helper drain/transfer gate; terminal maintenance/decision/handoff preparations do not count as live ancestors. Inspect existing receipts after API/agent replacement, never replay attempted phases or consume proofs again. +- [ ] Test production assembly with actual helper process boundaries: same-image agent limit changes, terminal-before-drain, API observer loss/restart, edge env recreation, DB/cache coordinator loss/health recovery, changed configuration during apply, launch-response loss and failure after each target. Add API death before/after desired-row save, lost maintenance prepare/decision response, DM recreation/startup cron blocking, and protected postgis outage with a retained DM guard. Assert no nested target root/self-conflict, no drain waiting on its own terminal preparation, and no release while a child/guard is unknown. Unsupported legacy agent/helper capability returns an actionable host-update prerequisite before ownership admission. +- [ ] Re-run integration tests GREEN; inspect serialized responses and regenerate canonical API/SDK outputs through the controller before committing. + +## Task C4: Platform form, release/retention consumers and documentation + +**Files:** Create `apps/web/src/components/admin/settings/PlatformConfiguration.tsx` +and tests. Modify `SystemSettings.tsx`, `ServiceConfigForm.tsx`, +`packages/cli/src/lib/release.ts`, `packages/core/src/services/release-manifest.ts`, +`apps/ops-agent/src/administrative-runtime.ts`, `apps/ops-agent/src/index.ts`, +`packages/cli/src/commands/backup.ts`, and affected configuration/administration docs. +Release/retention consumers are serialized with plan09. + +**Interfaces:** All release/backup consumers call the C1 resolver using current +applied state; no module independently snapshots a retention/selector env value +that would make an applied UI setting inert. Readonly origin/ownership/pending/ +recovery state is part of the generated platform response. Form edits do not +automatically apply, and confirmation binds the preview. + +- [ ] Add UI behavior tests for host-owned locked values, adoption preview, pending edits, conflict, partial/unknown outcome, public-origin reconnect and copyable recovery. Add consumer tests proving changed selector drives release resolution and changed retention drives create/prune/restore/scheduled janitor consistently. +- [ ] Run new platform UI tests and CLI release/backup plus agent release/retention tests to causal RED. +- [ ] Implement the form with existing schema/shared admin components and explicit source ownership. Add field documentation for supported settings and host-only bootstrap keys; document restart/rebuild/application timing and origin recovery. +- [ ] Run focused tests GREEN; inspect actual desktop/mobile rendering with synthetic state. Record local verification without PR screenshots. Commit the complete task after task review. + +## Self-review + +C01-C04 map to C1-C3; C05-C07 map to C1/C3/C4. Review Focus1/4 map to C2, +Focus2/3 to C3 and Focus5 to C1/C4. Cross-plan lifecycle and job consumers must +use the finalized plan01/02 interfaces. Plan09 consumes C1's resolver for +retention/selector and must not implement a competing settings store. + +Exact service/field inventory follows; shared method names are reconciled against +all subsystem plans before execution. No executable options are authorized. + +## Safe field inventory + +These are explicit additions to existing schemas. Integer bounds reject rather +than clamp. Resource-only manifests still receive useful memory/CPU controls. + +| Service | Fields and validation | Runtime consumer | +| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| app-api | LOG_LEVEL finite Pino levels; RATE_LIMIT_IMAGE_PROXY_MAX_BYTES 1048576-10737418240; WINDOW_MS 1000-3600000; MAX_CONCURRENT_PER_CLIENT 1-256; MAX_CONCURRENT_GLOBAL 1-4096 (full RATE_LIMIT_IMAGE_PROXY_ prefixes) | API logger and routes/image-proxy-budget.ts | +| app-web | NEXT_PUBLIC_STYLE_PROVIDER openmapx/maptiler; NEXT_PUBLIC_MAP_STYLE_URL (glyph base), NEXT_PUBLIC_TILES_URL, NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE, NEXT_PUBLIC_CYCLOSM_TILE_URL_TEMPLATE, NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL, NEXT_PUBLIC_TERRAIN_CONTOUR_TILEJSON_URL optional validated http(s) or root-relative values; attribution name <=200 and URL same ordinary URL grammar; NEXT_PUBLIC_TRAFFIC_MIN_ZOOM optional integer0-24, unset uses raster manifest10 | dynamic server env projection into browser; map style/glyph base consumer, actual terrain/tile consumers and shared raster traffic layer/selector zoom hook | +| data-manager | LOG_LEVEL finite Pino levels; OVERTURE_DUCKDB_MEMORY_MB 256-1048576; OVERTURE_DUCKDB_THREADS 1-64; OVERTURE_DUCKDB_TEMP_MB 1024-10485760 | existing data-manager runtime parser, enforce its stricter bounds if present | +| local-ai | OLLAMA_KEEP_ALIVE bounded duration or0, at most24h; OLLAMA_MAX_LOADED_MODELS 1-32 | Ollama image environment | +| elasticsearch | OPENMAPX_JAVA_HEAP_MIB 128-262144 | synthesize only -Xms/-Xmx in ES_JAVA_OPTS; never accept arbitrary JVM flags | +| nominatim | THREADS 1-256; REPLICATION_UPDATE_INTERVAL 60-604800; IMPORT_STYLE full/admin/street/address; REVERSE_ONLY, IMPORT_WIKIPEDIA, IMPORT_US_POSTCODES, IMPORT_GB_POSTCODES boolean | existing image environment; annotate import-time settings requiring rebuild | +| photon | REGION existing resource-region validator; UPDATE_STRATEGY PARALLEL/SEQUENTIAL; UPDATE_INTERVAL bounded duration1h-8760h; LOG_LEVEL TRACE/DEBUG/INFO/WARN/ERROR | existing Photon image environment | +| overpass | OVERPASS_SPACE 1073741824-10995116277760; OVERPASS_FASTCGI_PROCESSES 1-128; OVERPASS_UPDATE_SLEEP 60-604800; OVERPASS_RULES_LOAD 1-100 | existing image environment; exclude preprocessing shell fields | +| postgis | POSTGIS_DIAGNOSTICS, POSTGIS_TRACK_IO_TIMING boolean | existing entrypoint diagnostics settings | +| valhalla | serve_tiles, build_time_zones, use_tiles_ignore_pbf boolean in addition to existing schema | existing image environment, capitalized boolean adapter | + +Duration validation accepts only a finite numeric value with documented unit; +no shell substitutions. Image capability tests verify enum support; if an upstream +image lacks an enumerated value, restrict to its supported set and record the +source-grounded correction in the review ledger before implementation. diff --git a/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md b/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md new file mode 100644 index 000000000..d923cf45f --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md @@ -0,0 +1,168 @@ +# Maintenance Controls Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox syntax for tracking. + +**Goal:** Complete normal CLI option parity for grouped services, build/update/log controls, release staging, backup retention and scoped cache maintenance. + +**Architecture:** Reuse the reliable action policy/preparation/jobs from plans01/02 and applied platform ownership from plan08. Expose bounded typed inputs through the canonical service-admin family; use host guidance for operations that remove their own control plane. + +**Tech Stack:** TypeScript, Zod, Fastify, generated SDK, React/MUI, ops-agent, Vitest. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md`. + +## Global Constraints + +Cover M01-M07. No new generic command runner, arbitrary log argv, unrestricted +Redis pattern, FLUSHDB/FLUSHALL endpoint or destructive stack shutdown operation. +Do not create a competing platform/release/retention store. Coordinate shared +contracts and giant existing UI/runtime modules with the controller. + +## Review Focus + +1. Presets expand identically in CLI and UI, including missing services and dependency roots, without bypassing per-action capabilities. +2. `noDeps` and MOTIS import reach their runtime consumers; unsupported combinations fail before side effects. +3. A log query change cancels exactly its owned stream and cannot append stale service/origin data or grow a partial line without bound. +4. Backup preview cannot delete a new/replaced/unpreviewed archive, and retention changes cannot resurrect an already expired backup. +5. Cache previews/clears remain within `int:` and `cache:` prefixes, preserve invalidation epochs and use bounded memory under large scans. + +## Task M1: Shared presets and complete service action options + +**Files:** Create `packages/core/src/services/presets.ts` and tests; turn +`packages/cli/src/lib/presets.ts` into compatible re-exports. Modify +`packages/core/src/services/index.ts`, canonical service-admin `services.ts`, +`data.ts`, `index.ts`, `apps/api/src/routes/admin-services.ts`, +`apps/api/src/services/admin-job-handlers.ts`, +`apps/web/src/components/admin/services/ServiceCatalog.tsx`, and focused tests. +Agent runtime build/update changes consume the finalized plans01/02 interfaces. + +**Interfaces:** Export existing `PRESETS`, `expandPresets` and `UnknownPresetError` +from core without changing preset IDs/root order. Add `listAdminServicePresets` +(`GET /api/admin/services/presets`) returning `{ presets: [{ id, rootIds, +effectiveIds, missingIds }] }`. Bulk input accepts optional `presetIds` or +`serviceIds`, requires a nonempty resolved selection, deduplicates server-side, +and rejects unknown presets. Update-only `noDeps?: boolean`; build-only +`motisImport?: boolean`, valid only when the build includes MOTIS. Never persist +an action's temporary preset selection unless the explicit root editor is saved. + +- [ ] Add CLI/core tests preserving every preset expansion and UI/API tests proving preset expansion uses server authority, missing IDs are visible, mixed selections deduplicate, and forbidden actions remain forbidden. +- [ ] Add request-to-runtime tests proving `noDeps:true` reaches the isolated update path and `motisImport:true` emits the MOTIS import step only for MOTIS; invalid combinations reject before jobs/commands. +- [ ] Run `pnpm test packages/cli/__tests__/cli-services.test.ts packages/cli/__tests__/cli-compose.test.ts apps/api/src/routes/__tests__/admin-services.test.ts apps/api/src/services/__tests__/admin-job-handlers.test.ts apps/web/src/components/admin/services/ServiceWorkflows.test.tsx --maxWorkers=3` and new preset tests; record causal RED. +- [ ] Implement shared presets, strict action unions and UI controls using plan01 capabilities. Pass exact typed fields; do not hide unsupported flags in an unvalidated payload. Preserve build continuation/fail-fast defaults. +- [ ] Complete catalogue selection/discovery parity in this task: add enabled/all/disabled and exact capability filters, plus a capability inventory distinguishing recognized vocabulary from unrecognized manifest entries. Extend canonical list metadata with bounded `capabilityInventory:{recognized:string[],unrecognized:Array<{capability:string,serviceIds:string[]}>}` using the existing core vocabulary. Add per-service Enable/Disable root actions through the existing selection API, preserving dependency explanations and host-owned selection locks; a dependency cannot be silently removed while another root requires it. Tests cover root/dependency rows, unknown vocabulary, locked selection and failed generation apply. +- [ ] Re-run focused checks GREEN, verify actual runtime argument/preparation ordering, and commit after review. + +## Task M2: Bounded log retrieval options + +**Files:** Modify `packages/core/src/api/contracts/service-admin/services.ts`, +`apps/api/src/routes/admin-services.ts`, `apps/api/src/utils/docker-compose.ts`, +`apps/web/src/components/admin/services/ServiceLogsDrawer.tsx`; create its +behavior test and extend API/ops log tests. Coordinate policy with plan01. + +**Interfaces:** Logs query `{ lines?: number|string; follow?: boolean; +maxDurationSeconds?: number }`; `lines` 1-1000, follow default true, duration +1-900 seconds. Snapshot uses typed `service.logs`; follow uses +`service.logs.follow`. UI offers tail100/200/500/1000, snapshot/follow, pause and +reconnect; reconnect uses a fresh durable stream identity. Retain at most1000 +lines and256KiB, including a bounded incomplete line and truncation indicator. + +- [ ] Add tests for snapshot versus follow dispatch, strict malformed/unknown query rejection, byte and partial-line limits, core-service read capability, origin/service/query switches, close/cancel races and stale-chunk suppression. +- [ ] Run new drawer behavior tests and `pnpm test apps/api/src/routes/__tests__/admin-services.test.ts apps/ops-agent/src/docker-runtime.test.ts --maxWorkers=3`; verify RED. +- [ ] Implement the controls and typed dispatch, preserving redaction and owner fencing. Distinguish stream timeout/end from process/service failure; do not retry mutations automatically. +- [ ] Run focused tests GREEN, inspect rendered small/large viewports and commit after review. + +## Task M3: Pull-only staging and release pin clearing + +**Files:** Create focused `apps/api/src/routes/admin-release-controls.ts`, +`apps/api/src/services/admin-release-controls.ts` and tests. Modify service-admin +contracts, core ops contract/policy, `apps/ops-agent/src/administrative-runtime.ts`, +`apps/api/src/routes/index.ts`, `apps/web/src/components/admin/system/SystemMaintenance.tsx` +and relevant service catalog action adapters. Reuse +`packages/cli/src/lib/release.ts:clearReleaseSelection` semantics, factoring a +core helper if needed without introducing API direct filesystem access. + +**Interfaces:** `pullAdminServiceImages` (`POST /api/admin/services/images/pull`) +accepts `{ serviceIds }`, queues a typed image-stage job and never starts or +recreates containers. Known release-pinned images use the exact resolved release. +`clearAdminReleaseSelection` (`POST /api/admin/system/release/clear`) accepts +`{ expectedReleaseId, confirmation: "CLEAR RELEASE PIN" }`; typed `release.clear` +removes only the matching selected overlay under its existing release-store lock. +Running containers, evidence and running state remain unchanged. Plan08's selector +remains independently configured; clearing is not switching to local images. + +- [ ] Add route-to-agent tests proving pulls do not render/recreate/start, failed pull retains the current release, clear requires admin/confirmation/current release, concurrent overlay replacement conflicts, and local/automatic selector behavior remains explicit. +- [ ] Run new release-control tests, `pnpm test packages/cli/__tests__/cli-release.test.ts apps/api/src/routes/__tests__/admin-system.test.ts apps/ops-agent/src/administrative-runtime.test.ts --maxWorkers=3`; verify RED. +- [ ] Implement strict typed operations and UI preview/confirmation with effective selector and expected pin. Use the plan01 distinction between image staging and destructive lifecycle authority for protected core services. +- [ ] Verify GREEN and actual serialized response, regenerate through controller, then commit after review. + +## Task M4: Manual backup prune and effective retention policy + +**Files:** Create `apps/api/src/routes/admin-backup-maintenance.ts`, +`apps/api/src/services/admin-backup-maintenance.ts` and focused tests. Modify +service-admin `data.ts`/`index.ts`, core ops contract, agent backup runtime, +`packages/cli/src/commands/backup.ts`, +`apps/web/src/components/admin/services/BackupsPage.tsx` and tests. Retention +ownership/resolution belongs to plan08's C1/C4, not a second settings entry. + +**Interfaces:** `previewAdminBackupPrune` (`POST /api/admin/services/backups/prune/preview`) +returns candidates, sizes, cutoff, effective retention/source, inventory digest +and a15-minute expiry. `pruneAdminServiceBackups` (`POST .../prune`) binds that +preview and confirmation `PRUNE EXPIRED BACKUPS` in a durable job. Typed +`backup.prune.preview` and `backup.prune` run under the existing backup-store lock; +the mutation revalidates exactly the previewed candidates and policy revision. +It never deletes newly eligible or replaced archives without a fresh preview. + +- [ ] Add tests for preview-only no mutation, expired/changed preview, concurrently created/replaced backup, malformed/corrupt entry, lock contention and correct deleted/retained counts. Test policy changes across create/prune/restore/janitor and no resurrection of prior expiry. +- [ ] Run new backup-maintenance API/runtime tests and `pnpm test packages/cli/__tests__/cli-backup.test.ts apps/web/src/components/admin/services/__tests__/BackupsPage.test.tsx --maxWorkers=3`; verify RED. +- [ ] Factor a shared read-only prune plan from current CLI retention logic; keep existing integrity/path/link checks and backup-store locking. Bind inventory identities, not arbitrary directories. +- [ ] Stamp an immutable latest-allowed expiry for new backups; migrate unstamped local entries under the store lock using the pre-change effective policy. A shorter policy may shorten existing expiry; a longer policy applies to new snapshots and never reauthorizes an expired one. Portable manifests/readers accept the additive expiry with validation. +- [ ] Add the preview/prune and source-aware retention editor using plan08, retain create/restore subset/stop-running behavior, and surface corrupt entries separately. +- [ ] Run GREEN, serialized contract tests and appropriate export-restore database fixtures; commit after review. + +## Task M5: Scoped cache pattern preview/dry-run + +**Files:** Modify `apps/api/src/routes/admin-cache.ts`, +`apps/api/src/utils/cache-policy.ts`, +`apps/web/src/components/admin/cache/CacheManager.tsx`; create focused cache +contract helpers under `packages/core/src/api/contracts/` and extend API/UI plus +cache invalidation tests. Keep existing namespace and clear-all clients compatible. + +**Interfaces:** Strict clear body `{ namespace?: string; pattern?: string; +dryRun?: boolean }`; namespace/pattern are mutually exclusive. Pattern <=256 +characters and must begin literally with `int:` or `cache:`; reject NUL/control +characters and unrestricted `*`/other prefixes. Dry-run returns `{ dryRun:true, +estimatedCount, patterns }` and never calls UNLINK or changes invalidation epochs. +Real clear returns existing deleted count and includes resolved patterns. Omitted +target continues to mean the two application globs, never the entire Redis DB. + +- [ ] Add tests rejecting `*`, session/job/foreign prefixes, malformed/unknown fields and ambiguous targets. Prove dry-run has no deletion or epoch changes and real targeted deletion retains non-app keys and concurrent invalidation guarantees. +- [ ] Run `pnpm test apps/api/src/routes/__tests__/admin-cache.test.ts apps/web/src/components/admin/cache/CacheManager.test.tsx apps/api/src/services/__tests__/cache-invalidation.test.ts --maxWorkers=3`; verify RED. +- [ ] Count/clear with streaming SCAN and bounded batches; never accumulate an unbounded complete key list. Counts are explicitly estimates under concurrent writes/SCAN duplication; no key values are returned. Add pattern/count preview and deliberate clear confirmation to the UI. +- [ ] Verify GREEN and optional disposable Valkey integration fixture, regenerate API description through controller and commit after review. + +## Task M6: Honest host boundaries and administration documentation + +**Files:** Modify `apps/web/src/components/admin/services/ComposePreview.tsx`, +its tests, `docs/docs/administration/admin-panel.md`, +`services-administration.md`, `backup-and-restore.md`, `monitoring.md`, +`docs/docs/install/managing-services.md`, `configuration.md`. +Modify canonical service-admin/workflow contracts, `apps/api/src/routes/admin-compose.ts` +and `apps/api/src/services/admin-job-handlers.ts` in the same exclusive window. + +**Interfaces:** The Compose Down surface displays an explicit explanation and +copyable `pnpm openmapx compose down` instructions; it does not send a mutation +known to return503. Destructive `--volumes` remains a documented host procedure, +not the default copied command. + +- [ ] Add a UI test asserting no stop request is sent, instructions are actionable and Up still works; run to RED. +- [ ] Replace the doomed button with bounded host guidance, show retained remote diagnostics/log access and recovery links, and document all supported new options plus their verification/application limits. +- [ ] Add render-only generation commit through canonical `renderAdminComposeConfiguration`, POST `/api/admin/compose/render`, body `{expectedCurrentRevisionId:string|null,confirmed:true}`. Queue a durable `compose.render` job using current desired configuration, exact revision CAS and plan01's explicit `require-running` coordinator preparation mode, without container lifecycle effects. The mode belongs to the admitted identity/checkpoint, not a UI hint. Nonempty hardlink plans require an already-running authorized data-manager before commit and revalidation at execution; an empty plan needs no coordinator. A stopped/concurrently stopping coordinator is never started. UI distinguishes Preview, Generate configuration, and Up; generated-only result says container restart/apply is still pending. Domain/roots/presets are edited through supported forms before generation. Test the real API→preparation→agent call graph for zero lifecycle effects, running/stopped/racing coordinator, empty/nonempty plan, stale revision and post-commit link failure. Register `compose.render` in02.2 exact durable recovery/result dispatch; retain committed-follow-up-failed/unknown evidence. +- [ ] Run the Compose preview tests GREEN, verify documentation formatting/build and inspect rendered guidance locally. Commit after review. + +## Self-review + +M01-M03 map to M1/M2, M04 to M3, M05 to M4, M06 to M5, M07 to M6. +Every Review Focus condition has assertions in its owning task. Shared changes +to bulk contracts/handler/catalog and agent runtime are sequential with plan01; +retention/release resolution is sequential with plan08. Generated artifacts are +controller-owned, never concurrent worker output. No task changes host bootstrap +secrets or grants generic self-stop authority. diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md new file mode 100644 index 000000000..9184c56e3 --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -0,0 +1,175 @@ +# Admin Instance Management Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox syntax for tracking. + +**Goal:** Complete the reliability and operator coverage recommendations from the CLI/admin audit in one verified pull request. + +**Architecture:** Extend canonical typed ops-agent and admin API contracts, reuse configuration generations and durable jobs, then expose missing browser workflows. Independent subsystem plans share this specification, scope ledger and integration gate. + +**Tech Stack:** TypeScript, pnpm, Fastify, Next.js/React/MUI, Zod, generated SDKs, Drizzle/PostgreSQL, Docker ops-agent, Vitest. + +**Spec:** `docs/superpowers/specs/2026-10-23-admin-instance-management-design.md` +and its binding supplement `docs/superpowers/specs/2026-10-23-admin-control-protocols.md`. + +## Global Constraints + +All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Codex-only agents; local rendered verification, no screenshots attached to PR. + +## Review Focus + +1. Accepted jobs must reach the real production runtime and retain failed/partial/unknown outcomes. +2. Deselecting a running service must retain bounded shutdown/log authority without granting start authority. +3. A saved auth/config/binding value must affect the documented runtime consumer or explicitly remain pending/host-owned. +4. A disconnected observer, changed pin, expired stage or concurrent configuration change cannot publish stale work. +5. Credentials, archive paths and privacy artifacts must retain source ownership, bounded transfer and reauthentication guards. + +## Subsystem plans and sequence + +Reliability: 01 service control, 02 durable jobs, 03 account policy, +04 integrations and migration (reliability tasks), 05 transit and data (reliability tasks). +Coverage: remaining 04/05, 06 artifact transfer, 07 privacy workflows, +08 platform configuration, 09 maintenance controls. +Final: generated-contract integration, full qualification, whole-branch review, PR and CI. + +Subsystem plans are authored and self-reviewed before any product edits. The +controller records cross-plan shared-file/interface ownership before dispatch. + +## Task Q0: Review and baseline + +- [ ] Complete all nine subsystem plans with exact file ownership, interfaces and test-first steps. +- [ ] Self-review every scope ID against a task, record decisions and resolve shared-file conflicts. +- [ ] Obtain a fresh Codex builder/plan review and resolve every material objection. +- [ ] Install both frozen dependency sets and run the focused audit baseline in this worktree. +- [ ] Commit reviewed design/plans and record the starting implementation SHA in the execution ledger. + +## Task Q1: Integrated contract/runtime proof + +- [ ] Generate API/OpenAPI and SDK artifacts once shared definitions settle; inspect serialization and diff. +- [ ] Add real production runtime-composition dispatch probes for advertised actions and policy denials. +- [ ] Add multi-layer auth, configuration, data parent/child, artifact, extension and privacy workflow probes. +- [ ] Run focused tests and affected package types; record exact commands and outcomes. + +## Task Q2: Qualification and rendered verification + +- [ ] Run `pnpm verify:full` with both frozen dependency sets installed. +- [ ] Run affected production builds and appropriate disposable database suites. +- [ ] Inspect the actual rendered admin workflows using synthetic/public fixtures at desktop/mobile sizes. +- [ ] Review the complete diff with fresh Codex reviewers; fix all material findings and re-review fixes. +- [ ] Reconcile every specification scope ID with implementation and fresh verification receipts. + +## Task Q3: One PR and green CI + +- [ ] Commit the complete implementation and push the isolated branch. +- [ ] Create one concise PR against main using `--body-file`; omit screenshots per user request. +- [ ] Immediately register the PR with `link_pull_request`; verify the final PR description and linked state. +- [ ] Watch with `watch_pull_request`; fix failed checks on wake and repeat until latest-head CI passes. +- [ ] Stop watching, verify head SHA/check state and report the PR, checks and remaining host boundaries. + +## Self-review and execution state + +Controller self-review has mapped all scope IDs to subsystem tasks. Product +implementation starts only after the fresh independent plan review is resolved. + +## Binding integration rulings + +- Execute one implementation task at a time, with a fresh task reviewer before + the next task. The controller grants that worker an exclusive window for all + required canonical definitions/registration files; “controller-serialized” does + not mean leaving an unintegrated module for someone to wire later. Only the + controller runs aggregate generation/full checks, and no product edits overlap. +- Cross-process target leases follow the exact anchored role-signed protocol in + the control supplement, including fixed role-key provisioning, private recovery + directories, finite delegation graph, durable attempts and settlement. Host + order is target lease → nonblocking transit reservation → host configuration + writer → release store. API configuration work first acquires its global + administrative writer and sorted integration locks on one owned connection. + Full Update holds no target while awaiting a child that needs it. A bare ID, + caller boolean, observer abort or expired deadline cannot release authority. +- Task01.2 owns strict prospective configuration for all administrative writers. + Prepare and budget the exact sealed complete candidate before target writes; + use its closed transaction participant for binding/migration/credential effects, + then persist proofs, selected rows and job checkpoint atomically. Read-only + proof APIs are03.3's `check`/`checkMany`; the feed import patch exposes + `overlayChange`. No competing validator or managed-key generic persistence. +- Configuration/feed maintenance follows control-protocol§5.4: successful whole + candidate preflight permits one ownership-only remote preparation before saving. + Agent owns the retained root and durable DM admission fence; the API commits + rows/proofs and one continue decision atomically, then uses the frozen borrowed + continuation. It never acquires a local host lease or another apply root. + Task02.3 owns the early DM guard/index in0042,01.2 the orchestration, and + 04.6/05.6/08.3 consume that same protocol. +- Paired-pin activation retains its DM root through promotion and the explicit + asynchronous `transitousLock.approve`/`paired-pin-commit` leaf.02.3 supplies its + closed contract,05.7 the exact pointer-attempt receipt, and05.3 the validated + runtime. Lost publication is lookup-only; no ref-only or fresh-root fallback. +- Shared portable `ServiceBatchResult` and `HardlinkApplySummary` types live in + `packages/core/src/ops/service-results.ts`. Task02.1 creates any type required + by its failure-evidence contract; Task01.3 owns aggregation behavior and + consumes that type. Do not leave a forward import into a not-yet-created API + module or define a competing batch shape. +- Public log query is `lines`1–1000/default200, `follow`true and + `maxDurationSeconds`1–900/default900. Only the Docker adapter calls it `tail`. +- Old inactive auth settings are pending until atomic explicit version1 Apply; + existing runtime verification/enrollment defaults remain true. Enrollment + disable never bypasses an enrolled account's second factor. +- Common resource controls cover the exact30 built-ins. Resource values never + leak into container environment. Domain adoption is a previewed transition of + known platform slots, while ordinary service/integration host overrides retain + host ownership. Domain changes disclose passkey/OAuth/DNS recovery prerequisites. +- Artifact ceilings are not per-operation disk reservations. Reserve uploaded + bytes, then manifest-derived extraction/output bytes and inodes. One archive + job at a time plus explicit metadata/RSS budgets protect the agent. Plan06's + current1g default must pass measured worst-case metadata acceptance; reduce + limits or increase supported budget if measurement disproves that allowance. +- Prepared artifacts and uploaded standalone integrations share Plan06's opaque, + purpose-bound transport. Credential envelopes ≤1MiB may use API-owned encrypted + staging. Host-owned credentials remain explicit prerequisites. +- New long operation policies are explicit: artifact validation/export/import72h + (deployment ceiling7days), catalog validation72h, catalog activation1h, + advanced Overture acquisition/ingest/extract24h, platform apply1h. Short + inspect/create/delete calls keep bounded admission/read deadlines. Every layer + uses the remaining absolute deadline; no phase renews a job budget. +- Updates never recreate after a failed pull, including explicitly local/unpinned + CLI mode. Document this behavior change; Start remains the action for already + present images. No false successful “update” through a cached-image fallback. +- Account role migration preserves existing effective administrator authority; + do not require a host normalization merely because historical Better Auth + allowed multiple known roles. Normalize recognized role combinations under a + write lock using the existing authorization predicate; unknown roles fail + closed with a migration diagnostic. Remote new role writes use the finite set. + +## Task execution order and append-only migrations + +Run02.1,02.2,06.1,02.3,01.1–01.5,02.4,01.6, +03.1,03.2,03.5,03.3,03.4,03.6,04.1,04.2,05.1,05.7,05.2, +08.1,05.6,05.3–05.5,06.2,04.3,04.4,08.2–08.4, +09.1–09.6,06.3–06.5,04.5,04.6,07.1–07.6,thenQ1–Q3. +The early06.1 shared lease/manifest foundation is required for reliable writers; +its browser transfer workflows land after reliability. No task is deferred. +The nine subsystem plans contain50 implementation tasks. Task05.7 is the +paired-pin authority foundation;05.2 and05.6 consume its identity, +then05.3 adds reviewed validation/activation using the credential revision. +Public activation stays unavailable until the complete evidence chain is wired. + +Migration allocation supersedes the provisional shared0042/0043 suggestions in +early planner drafts. Each landed task adds a new forward migration; never edit +an earlier migration after a test database/commit has applied it. The controller +serializes Drizzle snapshots/journal and verifies the fresh migrated schema. + +| Task | Migration | Contents | +| ---- | -------------------------------------- | -------------------------------------------------------------- | +| 02.3 | 0042_admin_data_execution.sql | durable data-manager execution identity/cancel/result metadata | +| 03.1 | 0043_admin_auth_policy.sql | activated policy singleton and version/revision | +| 03.2 | 0044_admin_account_invariant.sql | guard counter/triggers and atomic role receipts | +| 03.3 | 0045_admin_reauthentication.sql | purpose-bound administrator challenges | +| 03.4 | 0046_admin_account_provisioning.sql | idempotent provisioning and encrypted mail outbox | +| 04.1 | 0047_admin_integration_application.sql | administrative revisions and retained apply receipts | +| 04.3 | 0048_admin_integration_ownership.sql | installation provenance/standalone ownership | +| 04.5 | 0049_admin_configuration_migration.sql | bounded migration preview/apply receipts | +| 04.6 | 0050_admin_credential_transfer.sql | encrypted short-lived import stages | + +All paths are beneath `apps/api/src/db/migrations/`; paired shared schema lives +in the current owning package, including `@openmapx/db-schema` for data-manager. +If a task proves its persistence fits a previously existing table without losing +the stated guarantees, document that finding and omit an empty migration rather +than adding no-op schema churn. Subsequent names stay reserved and append-only. diff --git a/docs/superpowers/plans/2026-10-23-admin-plan-review.md b/docs/superpowers/plans/2026-10-23-admin-plan-review.md new file mode 100644 index 000000000..8fad69ef5 --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-admin-plan-review.md @@ -0,0 +1,113 @@ +# Controller plan self-review + +Reviewed the specification, all nine subsystem plans and actual producer/consumer +boundaries before product implementation. Baseline remains +`ef896ceae876959180116cf61ffcf643b88ebffe`; no product edits have occurred. + +## Scope coverage + +| Scope | Owning tasks | +| ------- | ---------------------------------------------------- | +| S01,S02 | 01.2,01.6 | +| S03 | 01.3,01.6 | +| S04,S05 | 01.1,01.6 | +| S06 | 01.4,01.6 | +| S07,S08 | 01.5,01.6 | +| J01,J02 | 02.1,02.2 | +| J03 | 02.2–02.4 | +| J04 | 02.4 | +| A01–A04 | 03.1,03.6 | +| A05 | 03.5 | +| A06–A08 | 03.3,03.4,03.6 | +| A09,A10 | 03.2,03.5 | +| I01,I02 | 04.1,04.2 | +| I03,I04 | 04.3,04.4 with06.2 transport | +| I05,I06 | 04.5,04.6 with03.3 assurance | +| D01 | 05.1 | +| D02–D04 | 05.2 | +| D05 | 05.7 foundation,05.3 with06.1 lease | +| D06,D07 | 05.4,05.5 | +| D08,D09 | 05.7/05.6 with03.3/04.2 | +| T01–T04 | 06.1–06.5 | +| P01–P06 | 07.1–07.6 | +| C01–C07 | 08.1–08.4 | +| M01–M03 | 09.1,09.2 including catalogue filters/root controls | +| M04–M06 | 09.3–09.5 | +| M07,C04 | 09.6 host guidance and actual render-only generation | +| Q01–Q05 | masterQ1–Q3 integrated qualification/one green PR | + +## Rulings from self-review + +The master records binding exact interfaces, exclusive implementation windows, +execution order and append-only migration allocation. Specific corrections: + +- Preserve secure auth upgrade defaults; require explicit version1 activation + instead of turning formerly inert false database values into live policy. +- Protect the last full administrator before destructive erasure hooks and at + the database boundary. Normalize historical recognized role combinations + without accidentally removing effective administrator authority. +- Separate observer loss, cancellation intent, proven containment, failed work + and unknown effects. Bounded retained output never kills a useful build. +- Share one cross-process target lease and lock order across CLI/agent/data + manager; do not hold a parent lease while awaiting a child that needs it. +- Create shared portable result types before their consumers; require exact + generation conditions on API-role lifecycle requests. Keep canonical schemas, + actual serializers, SDKs and production assembly in the same task window. +- Reserve disk by actual admitted/manifest sizes and separately bound metadata; + verify the agent's supported memory budget with measured archive tests. +- Cover all30 built-ins, resource placement and boolean adapters. Known platform + adoption is explicit; service/integration host-owned values stay locked. +- Domain change previews include DNS/TLS/OAuth/passkey prerequisites, alternate + admin recovery, and retained reconnect evidence. Agent resource application + uses its external helper, never generic self-recreation. +- Add missing routine catalogue/root controls and render-only generation rather + than equating preview or queued admission with application. + +## Conflict and test scan + +The durable SDD ledger contains a row for each of50 tasks and774 shared +file/interface pairs in `preflight-scan.md`. Shared files receive one exclusive +task window; producer signatures and the master's binding rulings govern the +consumer. Controller-only generation/full qualification remains separate from +worker checks. No concurrent product edits or migrations are permitted. + +All tasks specify causal RED/GREEN checks, authorization/refusal cases and their +own Review Focus assertions. Full qualification includes real assembled runtime +dispatch, migrated synthetic PostGIS, actual byte streams, interrupted work, +rendered desktop/mobile admin states and latest-head CI. Mocked/source checks +are not represented as runtime acceptance. + +Fresh worktree baseline:510 passed,5 skipped across17 focused files. Both frozen +installs completed. Docker and GitHub access verified; an owned synthetic +PostGIS18/PostGIS3.6 container has successfully applied baseline migrations. +These are baseline receipts, not evidence that future changes pass. + +## Independent review + +Round1 fresh Codex review found0Critical/13Important. All13 were source-verified and accepted, with a source-grounded correction to the raster traffic consumer forR1-13. Corrected contracts live in the binding control-protocol supplement and plans01–09; responses are retained in the SDD workspace. No finding is silently deferred. + +A new independent round2 verifies the original findings and new breakage in their corrections before product edits. No user questions or approval pause is needed or requested. + +## Round1 correction self-review + +- R1-05: original admission clamp, body/execution fingerprints, authenticated parent deadline, no renewal; four separate clocks. +- R1-01: role keys/private mounts, anchored finite grants, both-direction adoption, durable attempts and settlement-before-release. +- R1-02: retained key per child phase; POST loss lookup; cancel intent/containment before rollback/release. +- R1-03/10: global API→integration→host; strict complete source reads, exact sealed512KiB budget before persistence; same transaction participant and durable unknown fence. +- R1-06: void runner reads exact private signed seed/final receipt, never stdout or competing current generation. +- R1-07: require-running binds preflight/identity/checkpoint, empty plan needs none; no Start fallback. +- R1-08: foundation precedes paired-pin consumers; credential invalidation precedes activation. +- R1-09: check/checkMany before prepare; consume/consumeMany atomically with ordinary+managed writes; sole overlayChange seam. +- R1-03/09: final64KiB UTF8,1000keys,512codepoints/key,8192bytes/value; duplicate/invalid Unicode rejected; generic PUT/DELETE blocked. +- R1-11/12: callback owns byte lifetime/auth; limited native target; uploaded-artifact immutable claim through install/settle; no detached Response/finally-release. +- R1-04: fresh supported password+effectiveMFA bound to actor/preview/policy; unknown password/OTP/federated/passkey alone denied. +- R1-04: API parent, terminal helper prepare before drain, actual same-image runtime proof; no live agent ancestor or blind relaunch/rollback. +- R1-13: raster traffic shared layer/selector zoom, dynamic server projection, supported relative URL grammar. + +Baseline migrated suites completed:api-privacy23files/129tests; ingestion13files/65tests; export-restore2files/7tests with synthetic retained backup. These remain baseline receipts only. + +## Round2 correction review and recovery + +Round2 cleared11/13 original findings and identified two unique Important seams: the delegated agent-owned paired-pin CAS and remote pre-save feed/configuration ownership. Corrected control-protocol§4.1/§5.4 and plans01/02/04/05/06/08 now define the finite publication leaf, exact attempt receipt, agent-owned terminal prepared maintenance root, durable DM admission guard, atomic DB continue decision and borrowed apply/recreation/settlement. No generic lock endpoint or new API mount is introduced. The existing50-task order and0042–0050 forward migrations remain. + +The correction worker hit a provider usage limit after writing the substantive protocol/report. Its files were recovered from exact historical document-write inputs after ignored files disappeared. The controller checked those recovery errors against the original tool receipts, regenerated50 tasks/774 pair rows, completed the explicit hardlinkPlanDigest fields and reformatted the documents. Prior product/database checks remain historical; no product change or runtime acceptance is claimed. A NEW independent round3 must verify both open findings and correction-caused material breakage before implementation. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md new file mode 100644 index 000000000..e4b248e1f --- /dev/null +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -0,0 +1,524 @@ +# Administrative control protocols + +Date: 2026-10-23. Binding supplement to `2026-10-23-admin-instance-management-design.md` and the controller's `plan-fix-rulings-r1.md`. This is an implementation contract, not an implementation or test receipt. Tasks 01.2 and 02.1–02.4 produce the interfaces assigned below; 06.1 implements the shared filesystem protocol. Later plans consume these names rather than inventing parallel locks, command runners, secret stores or upload services. + +## 1. Source constraints and ownership + +- `configuration-ownership.ts:acquireConfigurationOwnership` already uses exclusive mkdir, a 5-second wait, inode checks and no stealing after a crash. `configuration-generation.ts:withConfigurationGenerationWriter` owns the current pointer and commit evidence. Extend this discipline; PID age and TTL are not ownership. +- `services/{ops-agent,data-manager}/service.json` run with the same configured UID/GID. Data-manager mounts `/data`; the agent/host CLI see that same physical directory beneath the checkout. A key inside shared `/data` with mode0600 is **not** private from the sibling service. A pathname, UID, signature over an attacker-chosen public key, or lease UUID alone cannot authenticate a parent. +- Agent `server.ts:resolveRole` distinguishes the API and data-manager bearer tokens. Data-manager `auth.ts:registerAuth` has one shared bearer credential and does not identify which service presented it. Both directions therefore require a verified role-specific signature in addition to their existing transport authentication. +- Agent `FixedCliRunner` returns `Promise`. Final revisions must cross this process boundary through a fixed durable receipt, not stdout or the current pointer after exit. +- API `service-config-resolver.ts` currently swallows database failures; `integration-config.ts` also has intentional public fallback paths. Administrative snapshot resolution must be separate and strict. `trusted-config.ts` caps the complete sealed envelope at512KiB, independently of its tree and per-secret limits. +- Data-manager `ops-client.ts` currently creates random keys and aborts observation without containing the remote effect. `promote.ts`, `proxy-transaction.ts`, `motis-import.ts`, API restart and cron callers must acquire a durable child owner before dispatch. + +Product file allocation: 06.1 owns `packages/core/src/build-artifacts/{target-lease,control-authority}.ts`, their server-only exports, protocol credential provisioning in `packages/cli/src/commands/compose.ts`, and the corresponding fixed built-in mounts/startup verifiers. 02.1 owns portable `ops/execution-identity.ts` plus deadline/admission wiring. 02.3 owns data execution authentication/store/child adapters, early paired-approval contracts and DM configuration-fence contracts/adapters/0042 metadata. 01.2 owns API `services/{prospective-configuration,administrative-configuration-writer,configuration-maintenance}.ts`, agent configuration-maintenance runtime/store, portable ops configuration-maintenance types, strict resolver adapters and `packages/core/src/services/service-lifecycle-receipt.ts`. Shared files land in each task's exclusive controller window. No migration is needed for filesystem receipts or the API writer: use existing `admin_job.result` for **non-secret** ownership/attempt metadata. 02.3's data schema remains migration0042; later reserved migrations remain append-only and unchanged. + +## 2. Execution identity and time + +Export from server-capable core `ops/execution-identity.ts`: + +```ts +type ControlIssuer = "ops-agent" | "data-manager" | "host-cli"; +type ExecutionRef = { + executor: ControlIssuer; + principal: "api" | "data-manager" | "host-cli"; + executionId: string; + operationKey: string; + operationFingerprint: string; + fingerprint: string; + deadlineAt: number; +}; +type ExecutionAdmission = { + requestedDeadlineAt?: number; + parent?: VerifiedParentExecution; // opaque, server-local, never a parsed body +}; +normalizeExecutionAdmission(input: { + operationFingerprint: string; operationKey: string; + kind: string; admittedAt: number; requestedDeadlineAt?: number; + parent?: VerifiedParentExecution; +}): { admittedAt: number; deadlineAt: number; fingerprint: string }; +``` + +IDs use their actual producer's validator: agent `job1_…`, data-manager UUID, host CLI random UUID persisted before any effect. Keys retain `^opk1_[A-Za-z0-9_-]{16,64}$`; both fingerprints are64 lower-case hex. `operationFingerprint` is the existing `opsOperationFingerprint(operation)` (or canonical data input hash). `fingerprint` is the **execution** hash of version2, kind, body fingerprint, normalized absolute deadline and immutable parent identity, using domain-separated SHA-256 canonical JSON. Do not replace the body hash inside existing trusted-configuration HMACs or `OpsTrustedClaim.fingerprint`: retain both fields and validate their distinct roles. Admission/status/cancel/lookup and all new child records expose/use the execution fingerprint; v1 journals stay readable with explicit version1 semantics and never acquire invented parent authority. Version2 journal parsing recomputes both hashes. + +All millisecond times are finite safe integers since epoch. At first admission compute `deadlineAt = min(requestedDeadlineAt ?? admittedAt + kindBudget, verifiedParent.deadlineAt ?? Infinity, admittedAt + deploymentClampedKindBudget)`. Reject an already elapsed deadline before effects. A child must supply its parent assertion and requested absolute deadline; omission on a parented call is invalid. Store `admittedAt`, requested and normalized deadline before dispatch. On a duplicate key, compare the same normalized input using **original admittedAt**, not today's clock: a changed normalized deadline/body/parent conflicts409; an exactly identical identity returns the retained operation without extending time. Missing/expired retained identity following an attempted POST is unknown, never permission to retry that mutation. Unparented supported host CLI is authenticated as host-cli and gets the same server kind/deployment clamp. + +02.1 extends the strict **ops request envelope**, not arbitrary operation bodies, with optional `execution:{requestedDeadlineAt:number}`. `OpsClient.execute(operation,options)` adds `execution?:{requestedDeadlineAt:number}`, `controlAssertion?:string`, `parentAssertion?:string` to its existing operationKey/signal options; the latter two only set the fixed internal headers. Existing issuedAt/expiresAt remain short request authentication timing and never become the backend execution deadline. Version2 async admission/status/lookup/cancel all return `{admittedAt,deadlineAt,fingerprint}` with the retained identity. Sync operations also clamp runtime to remaining budget. All six existing MOTIS/proxy effect kinds in this graph are already async in `contract.ts:189–194`; preserve that durable admission. Change the existing sync data-manager `transitousLock.approve` to the durable async paired commit in§4.1; do not add a second approval command. Cancellation's existing fingerprint header carries the normalized execution fingerprint for version2. Public API SDK requests cannot inject these internal fields/headers. + +02.1 retains the specified budgets: build12h, buildAll24h, Full Update48h, existing large data operations24h; later artifact jobs72h/default,7days ceiling, catalog validation72h, activation1h, advanced Overture24h, platform API job1h. A platform API job owns one absolute deadline in its DB checkpoint; each agent child is clamped to the remaining value, with no long-lived `platformConfig.apply` ancestor. API-parent assertions use the existing API bearer boundary toward agent and durable API job metadata; the agent re-attests its admitted child for any downstream delegation. The browser never supplies a trusted parent or arbitrary expanded deadline. + +Keep four clocks separate: ordinary HTTP admission/read timeout15s; each observer's independently chosen `observerDeadlineAt`; retained execution `deadlineAt`; containment/settlement window30s **after** expiry/cancel. Runtime starts no new forward phase after execution expiry. The settlement window allows cancellation, process-group containment and evidence reads, not a fresh restart/promotion/rollback budget. Existing `promote.ts` recovery that grants a fresh restart budget must instead use the remaining retained deadline; if expired it records recovery-required and holds the fence. At30s unresolved containment remains pending/unknown and reconciliation continues; elapsed grace does not release ownership. HTTP close/observer expiry never calls cancellation. Execution expiry and explicit cancellation persist intent, then signal all owned children. + +`OpsExecutionContext` gains readonly `execution: ExecutionRef`, `artifactLease?: BorrowedArtifactTargetLease` and trusted internal parent metadata; the server constructs them after authentication/admission and keeps them out of `capability.values`. `FixedCliOptions` retains signal/log callbacks and adds required `execution` for managed operations, plus internal `controlAssertion?: string` and `parentAssertion?:string`. The fixed runner serializes execution as bounded≤2KiB `OPENMAPX_CONTROL_EXECUTION`, checked against its signed assertion and retained spawn intent; that unsigned environment field alone grants nothing. Legacy short scheduled callers may omit execution only where their existing bounded policy is explicitly retained. Never synthesize operation identity from output.02.1 defines all portable identity/opaque parent types before consumers;06.1 adds the signature verifier constructors, not a forward import into an absent module during02.1. + +## 3. Anchored filesystem authority + +### 3.1 Fixed trust material, not a generic authority service + +06.1 extends the existing local Compose secret preparation with exactly three installation Ed25519 role keypairs: `ops-agent`, `data-manager`, `host-cli`. Private role keys live at `infra/docker/secrets/control-.key` (0600, owner UID); public verifier bundle is `infra/docker/secrets/control-verifiers.json` (0600). No browser API can rotate/select them. Create missing keys with exclusive durable creation; never overwrite an existing key, silently regenerate after a missing verifier, or rotate while active protocol records exist. Validate key kind, installation UUID, role, owner, mode, regular-file/no-symlink/single-link and bounded size. Bundle≤4KiB contains only installation UUID and the three public keys. Preserve old records across ordinary restart; a host key rotation first settles every old record or requires explicit offline recovery. + +Mount only the appropriate role key read-only at `/run/secrets/openmapx-control-role-key` and the verifier bundle read-only at `/run/secrets/openmapx-control-verifiers` in each service. Give data-manager its own0700 recovery bind `@infra:control-private/data-manager` → `/var/lib/openmapx/control-private`; the agent uses `@infra:control-private/ops-agent`. These directories are **outside** `/data` and outside the sibling's mounts. Host CLI uses fixed `infra/docker/control-private/host-cli`; fixed CLI spawned inside the agent uses the agent recovery directory and a delegated ephemeral key, not an exported role private key. The external configuration helper uses the agent private directory through its existing fixed helper mount and survives the agent process. Host/Docker administrators remain trusted. A compromised data-manager can deny shared-disk service, but its role key cannot impersonate ops-agent or widen agent per-kind policy. This added provisioning is required because same-UID/shared-data permissions alone cannot provide authenticated adoption; it is a narrow deployment cost, not a new network lock/secret service. + +### 3.2 Files and records + +Open the trusted local data root once by descriptor; refuse symlink root or unsafe ownership/modes. Use physical descriptor dev/inode identity so host/container aliases meet in the same directory. Under fixed `.admin-target-control/` create `locks//` by exclusive mkdir and `leases//` for protocol state. Service IDs are canonical registered IDs≤64 characters, sorted and unique, maximum256. Never hash path spellings as lock identity. The shared namespace uses the same configured UID/GID,0700 directories,0600 regular single-link files, O_NOFOLLOW and before/after fstat/lstat checks. Persist records by fsync+atomic rename+parent-directory sync; cap a single lease ledger at128KiB, at most64 child intents, and at most256 unsettled leases. Refuse new work before its first effect if capacity is unavailable; unsettled records are never evicted. Terminal receipts remain72h and keys are never reused by the producer. + +An immutable role-signed owner certificate binds `{version:1,installationId,leaseId,dataRootIdentity,lockIdentities,owner:ExecutionRef,ownerPublicKey,serviceIds,createdAt}`. Dev/inode are decimal strings, no local paths. Its ephemeral per-lease private key is stored only in the issuer's fixed private recovery directory. The mutable `LeaseOwnerRecord` is signed by that key and contains `{certificate,sequence,phase,grantDigests}`; phase is `held|cancel_pending|recovery_required|settled`. Separate `grants/.json` records contain the parent-signed immutable grant and issuer-signed prepared/attempted transition; separate `children/.json` records contain the recipient-signed adoption/settlement transitions. Each node has its own monotonic sequence. Child intent binds grant digest, parent node, finite phase, execution identity and target subset, plus accepted ID/settlement when available. A borrower signs only its own child record and permitted descendant grants with its certified child key; it cannot rewrite the root-signed record or needs the root private key. Root terminal/release aggregates the complete anchored grant tree rather than trusting a mutable child list supplied by a borrower. Sensitive assertions/private keys are absent from ops/API journals, argv, events and public status; private recovery files alone retain resumable assertion bytes. Public projection is only busy/conflict/unknown plus permitted operation identity. + +There is one no-steal `.journal-writer` mkdir guard **per lease** for record CAS, never a global lock broker. Signed `sequence` increments under that guard; verify current directory identity, owner certificate, every held target inode and parent phase before mutating. A crashed metadata writer blocks until supervised reconciliation with all relevant writers quiescent. Never remove an unexplained nonempty or replaced lock. Save each executor's private expected certificate digest/sequence high-water mark with its durable operation receipt; stale/replaced/shared-file rollback conflicts. The ledger is protocol admission authority, tied to the real durable job through `ExecutionRef`, not a process-local cache. Before publishing a job's cancel/terminal state, its adapter first writes the corresponding ledger transition; after a crash, startup reconciles ledger↔journal before admitting new work. Terminal preparing operations may retain only the configuration-maintenance continuation (§5.4) and configuration-helper continuation (§3.5); neither is a live worker for drain accounting. + +### 3.3 Exact library surface + +Export from `build-artifacts/target-lease.ts` (server-only): + +```ts +type ArtifactTargetLease = { readonly leaseId: string; readonly owner: ExecutionRef; + readonly serviceIds: readonly string[]; readonly mode: "owner" }; +type BorrowedArtifactTargetLease = { readonly leaseId: string; + readonly owner: ExecutionRef; readonly execution: ExecutionRef; + readonly serviceIds: readonly string[]; readonly mode: "borrowed" }; +type TargetLeaseAuthority = ArtifactTargetLease | BorrowedArtifactTargetLease; +type ChildSettlement = { + identity: ExecutionRef; + outcome: "contained" | "committed" | "failed_without_effect" | "unknown"; + publication: "not_started" | "committed" | "unknown"; + terminalReceiptDigest: string; // private, non-secret receipt digest +}; +acquireArtifactTargetLease(options: { dataRoot: string; serviceIds: readonly string[]; + owner: VerifiedExecutionAuthority; signal?: AbortSignal }): Promise; +delegateArtifactTargetLease(lease: TargetLeaseAuthority, child: { + execution: ExecutionRef; phase: ControlChildPhase; serviceIds: readonly string[]; + recipient: ControlRecipient; childPublicKey: string; +}): Promise; +adoptArtifactTargetLease(options: { dataRoot: string; assertion: string; + peer: VerifiedControlPeer; execution: ExecutionRef; + serviceIds: readonly string[] }): Promise; +settleArtifactTargetLeaseChild(lease: TargetLeaseAuthority, + evidence: VerifiedChildSettlement): Promise; +releaseArtifactTargetLease(lease: ArtifactTargetLease): Promise; +reconcileArtifactTargetLease(options: { dataRoot: string; leaseId: string; + authority: VerifiedRecoveryAuthority }): Promise<"released" | "fenced">; +withArtifactTargetLease(options: { dataRoot: string; + serviceIds: readonly string[]; owner: VerifiedExecutionAuthority; + inherited?: BorrowedArtifactTargetLease; signal?: AbortSignal }, + run: (lease: TargetLeaseAuthority) => Promise): Promise; +``` + +All `Verified*` types are nominal, constructor-private values minted by the relevant fixed adapter after descriptor/signature/role/job checks; no caller boolean or cast from a request is an authorization mechanism. `VerifiedExecutionAuthority` binds the local admitted durable record and role signer. `VerifiedRecoveryAuthority` additionally proves exact durable record ownership and containment observations; a matching UUID is insufficient. `VerifiedChildSettlement` is built from authenticated status plus durable effect evidence, never arbitrary caller text. `ControlRecipient` is `"fixed-cli"|"data-manager"|"ops-agent"|"configuration-helper"`. `ControlChildPhase` is the finite phases in §3.5, not free text. `VerifiedControlPeer` comes from the fixed HTTP/CLI adapters in §3.4. These constructors live in `control-authority.ts`; service-specific adapters supply their existing typed journal/store lookup, never a generic user-pluggable authority callback. + +Acquisition attempts all sorted targets under one5s budget, releasing only successfully acquired empty locks if acquisition fails before ledger publication. Once published, release checks current certificate/inodes, no unsettled intent/borrower/descendant, durable own-effect settlement and no unknown publication. A normal root writes its signed settled receipt after the effect/containment proof, then releases; its outer agent handler may still be returning, so do not require an impossible already-terminal outer journal before callback return. Every attempted child must already have its own verified settlement. The special helper alone additionally waits for its prepare parent's terminal journal before drain. `withArtifactTargetLease` does **not** unconditionally unlock in `finally`: failure/abort with an attempted child or unknown effect marks recovery-required and retains the fence. A borrower can settle its own child/node, never call root release; the library rejects forged/mixed handles. An inherited lease must contain the exact required target closure or fail before work, never acquire extra targets while borrowed. No target waits while retaining a partially acquired set beyond the shared budget. + +### 3.4 Both-direction verification and admission transport + +`ArtifactDelegationAssertion` is a base64url canonical signed envelope≤4096 encoded header bytes, carried in exactly one `x-openmapx-control-delegation` internal HTTP header or the fixed CLI environment variable `OPENMAPX_CONTROL_DELEGATION`. Reject duplicate raw headers, invalid encoding, extra fields and over-limit chains. Its compact fields are `{version,installationId,leaseId,ownerCertificateDigest,delegationId,grantDigest,ancestorDelegationIds,signature}`. It names only UUID/digest-derived fixed records, never paths. The referenced parent-signed grant binds parent execution, child kind/key/body+execution fingerprints, target subset, deadline, one-use delegation UUID, recipient, child public key, permitted further edges and depth. Load and verify that exact bounded grant and root certificate from the shared ledger; do not inline256 target IDs/certificates into an oversized HTTP header. No bearer/private key is inside it. The parent persists the grant and **attempted** intent before first POST/spawn, in both execution receipt and lease ledger. Signature chain length≤3; full grant nodes/targets remain within the128KiB ledger budget. Bound both optional control headers together to8192 encoded bytes. + +Verification location is real admission: `apps/ops-agent/src/server.ts` for data-manager→agent; `services/data-manager/src/auth.ts` plus execution/link admission in `api.ts` for agent/CLI→data-manager; `packages/cli/src/lib/control-context.ts` for fixed child entry; existing external helper entry for the configuration continuation. Keep the data-manager bearer and the agent's distinct role token guards. Agent additionally requires the signer/peer principal to be data-manager for data-manager-origin grants. Data-manager resolves the origin role from the pinned role certificate, not its shared bearer token or a caller `role` header. CLI verifies the anchored certificate, spawn intent and own fresh execution challenge/key; the default runner removes inherited control variables from unrelated subprocesses and supplies them only to its fixed command. Never forward a parent assertion to an unrelated request or integration. + +The recipient checks: configured installation/verifier, full signature chain, current immutable lock identities, current signed ledger sequence/phase, durable grant entry and parent job binding, exact child body/execution fingerprints, audience, target subset and deadline. Existing policy still authorizes operation and resources independently. HTTP recipients use their already-pinned role public key as childPublicKey. For fixed CLI/helper children the parent allocates a fresh keypair before spawn in its private recovery directory at the fixed delegation-ID-derived location; the child reads only that certified key through its fixed adapter, with no caller-selected path or key in argv/environment. The signed grant allocates a random proposed receiver execution ID using the receiver's actual job1/UUID format. Admission accepts that ID only on this authenticated internal path, checks uniqueness with key/fingerprint/parent, and otherwise creates its usual server-assigned ID; public callers cannot select another job's ID. To avoid a cross-store atomicity gap, first create the receiver's durable `pending-adoption` record with that predetermined ID and fingerprint; then under lease journal guard exclusively create the recipient-signed child record claiming that delegation UUID; then persist `adopted/ready` before any effect. A crash in either gap is fenced and reconciled; no effect occurs until both agree. Duplicate exact admission observes this one retained receiver; a different identity or second claimant conflicts. These duplicate observations are allowed after grant deadline but cannot authorize fresh effects. Borrowers can sign narrower descendants only along the enumerated edge and within the inherited deadline/target subset; they never receive the root private key. For a new grant the parent computes a requested absolute child deadline no later than its own and the recipient's advertised kind budget, retaining its expected execution fingerprint. Admission recomputes the clamp; a stricter changed deployment policy that changes the signed identity refuses before adoption rather than silently changing the grant. No second mutation is issued from the same attempted intent. + +Deadline-only parents (not holding targets), notably Full Update→data execution, use `x-openmapx-parent-execution`/`OPENMAPX_PARENT_EXECUTION` with the same fixed signature/recipient/one-use/retained-attempt validation, but an execution certificate rather than a lease certificate. `attestParentExecution(authority, child)` and `verifyParentExecution(assertion, peer, retainedAttempt)` in `control-authority.ts` produce/consume `VerifiedParentExecution`. Such an assertion grants **no** target ownership; the child must acquire its own target lease. The local execution authority receipt is under the same issuer-private recovery root, keyed by operation key,≤64KiB and includes current phase/deadline/child intents. It provides the retained parent bound checked by the verifier, without a general network authority API. Signed shared execution witnesses under `.admin-target-control/executions//.json` expose only those safe fields; validation/CAS/crash fencing is identical to the lease ledger. API→agent uses the existing API authenticated admission plus exact retained API job identity; data-manager does not trust an unsigned API identity forwarded by a CLI. + +For the API→agent edge specifically,02.1's API adapter creates a purpose-bound HMAC assertion using the existing API ops token (no fourth role key). Domain is `openmapx-api-job-parent-v1`; strict signed fields are `{version:1,apiJobId,apiOwnerVersion,parentOperationKey,parentFingerprint,parentDeadlineAt,childOperationKey,childOperationFingerprint,requestedDeadlineAt,attemptId}`. It is minted only after the corresponding API job/child-attempt checkpoint CAS; no proof or hash of user secrets enters it. `apiJobId`/owner version come from the durable DB runner, not HTTP input. Agent verifies the HMAC with its existing API token, requires resolved bearer role api, uses this retained attestation as parent deadline authority and stores its digest/attemptId/parent fields in version2 admission. Same child key with any changed bound parent conflicts. It cannot be forwarded to data-manager; agent/CLI must issue its own permitted role-signed execution witness. Cancel/restart cannot mint a replacement parent deadline for an already attempted child. This is one fixed API job attestation, not a general shared signing service. + +### 3.5 Finite effect graph, targets and continuation + +No arbitrary recipient/kind pair is permitted. Maximum borrowed depth is2; deadline-only ancestry can add one Full Update node but cannot raise borrowed depth. A phase key is derived once as `opk1_ + base64url(SHA256("openmapx-control-child-v1\0" + parent.operationKey + "\0" + phase + "\0" + canonicalOrdinal)).slice(0,32)`; canonical ordinal is the index in the admission's immutable sorted plan, not a retry count. Persist before use. Effect attempts never get a new key on transport failure. + +| Parent → recipient | Allowed phase / child operation | Authority and target rule | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Agent → fixed CLI | `lifecycle` (`services start/update`), `build` (`services build`/build-all) | Exact admitted service closure; depth1; CLI may delegate only `hardlinks` to data-manager at depth2. | +| Agent → fixed CLI | `data-link` (`data link`), `data-clean` (`data clean`) | Exact manifest-derived data targets; depth1; link may delegate only the hardlinks leaf at depth2; clean is a local confined leaf. | +| Agent → fixed CLI | `data-workflow` (`data update`, OSM/fonts download, Overpass conversion, Overture sync/conflate, search-index/notable-places build) | Execution/deadline assertion only, no target lease while awaiting data-manager. This is the existing fixed administrative-runtime command map; CLI may call only its matching data execution, or osm/transit/fonts for Full Update. | +| Agent → data-manager | `hardlinks` (fixed durable generation-link execution) | Exact generation-derived target closure and revision; depth1 (depth2 only beneath `maintenance-hardlinks`), leaf; data-manager never calls agent Start under this grant. | +| Fixed CLI → data-manager | `hardlinks` | Same subset/revision, depth2 leaf, or depth1 for a host CLI root. Host CLI can apply links locally under its owned lease where its existing confined adapter does so; it must not independently reacquire. | +| Data-manager transit root → agent | `stop-primary`, `stop-staging`, `restart-staging`, `promote-primary`, `recover-primary`, `proxy-stage`, `proxy-commit`, `proxy-restore` | Respectively fixed `motis.primary.stop`, `motis.staging.stop`, `motis.staging.restart`, `motis.primary.promote`, `motis.primary.restart`, `feedProxy.validateAndReload`. Roots hold manifest-derived union including motis, motis-staging, motis-feed-proxy and their data producers/consumers; each grant narrows to exact affected targets. All leaves; recovery/restore require proven unchanged ownership and unexpired deadline, never blind compensation. | +| Data-manager transit root → agent | `paired-pin-commit` (`transitousLock.approve`) | Depth1 leaf under the SAME activation root as promotion; exact proposal/validation/expected-generation/diff/acknowledgment and MOTIS closure, durable async identity and attempt-bound pointer receipt (§4.1). No root reacquisition or intervening release. | +| Agent configuration-maintenance root → data-manager | `configuration-fence`, `configuration-fence-settle` | Fixed prepare/settle logical guard in§5.4, exact maintenance/candidate/root/plan binding. Terminal prepared guard has no live worker or retained DM target borrower; it authorizes no data write. | +| Agent configuration-maintenance root → agent | `configuration-commit`, `maintenance-hardlinks`, `maintenance-recreate`, `maintenance-handoff` | Only the frozen§5.4 plan: one existing typed generation commit, one hardlink operation, exact same-image ordinary recreations and one protected handoff prepare. Depth1 borrowers, original root/target closure/deadline; never acquire a second root. | +| Agent/host CLI Full Update → data-manager | `osm`, `transit`, `fonts` data executions | Deadline-only parent. Parent holds no targets while awaiting this child; child acquires its own lease, then nonblocking transit reservation where applicable. | +| Fixed CLI single data command → data-manager | `data-step`, exact matching `download-osm`, `download-fonts`, `convert-overpass`, `overture-sync`, `overture-conflate`, `search-index-build`, `notable-places-build` | Deadline-only parent, fixed kind/normalized region/countries; child acquires its root lease.05's advanced pull/ingest/extract API jobs admit directly, without adding a generic CLI command edge. | +| Agent → fixed external configuration helper | `configuration-handoff` | Exact `configurationHandoff.prepare` terminal receipt, handoff UUID, expected/desired config revisions, current verified image and target set; depth1 normally, depth2 beneath `maintenance-handoff`. It borrows the existing maintenance root when present, survives preparing agent death, and is a leaf with no widening or further delegation. | + +06/05 may add their named artifact/catalog operations as **root** owners under these same rules; paired-pin approval is exclusively the explicit borrowed leaf above. They must not add a generic child-kind wildcard. API/cron one-off MOTIS restart/proxy actions use a durable data-manager execution root and the same finite child phases, not anonymous `runOpsOperation`. Any additional real nested edge discovered by implementation must be explicitly added to this table and reviewed before use; it cannot fall back to an ID bypass. + +Host order is target lease → **nonblocking** transit reservation → host configuration writer → release store. Busy transit admission drops an unborrowed unused target lease immediately and returns busy; a borrower cannot drop its parent's lease, so it reports busy and settles without effect. Never hold the host configuration writer while awaiting a child which reacquires that writer. `configuration.hardlinks.apply` retains it while its leaf data-manager link writes; that child only uses the borrowed target authority and generation-bound plan and acquires no host writer. + +For configuration handoff, the overall platform apply is an API DB job. First use§5.4's pre-save ownership-only preparation, then its durable DB decision. `stack.render`, ordinary follow-up and `configurationHandoff.prepare` are the frozen children of that retained root. Handoff preparation records the exact durable handoff/helper grant, starts the fixed helper, and journals terminal `{handoffId,phase:"prepared"}`. The helper may adopt but starts drain only after reading that exact terminal receipt; if preparation dies before terminal, it performs no recreation and retains the fence for reconciliation. No old process holds a host configuration writer across the handoff. Terminal preparation does not release the target root or DM logical guard. The helper reacquires only the host writer under its borrowed subset, verifies revisions/image/mount identity and performs08.3's same-image workflow. Depth stays≤2; the helper never delegates further. Agent recovery settles the exact handoff/children and then§5.4's guard/root. API reconnect observes existing receipts, never reconstructs or resubmits an attempted phase. + +The agent drain gate is the existing configuration-handoff gate keyed by this same maintenanceId, not a second flag. It excludes the terminal maintenance/decision/handoff preparation records and allows their exact inspect/settlement controls; it blocks new unrelated mutating operations and byte admissions, awaits active mutations/streams, and excludes read-only logs/inspect. The DM maintenance guard is§5.4's durable logical admission reservation, not a live DM worker whose stop/lease release is needed to recreate DM. It rejects unrelated or unresolved work nonblockingly before desired-row commit, survives recreation, and permits only the signed, frozen hardlink continuation.08.3 owns exact same-image runtime proof and protected target policy. + +### 3.6 Settlement and recovery + +Cancellation marks root+descendant intent first, prevents new forward grants and sends explicit authenticated child cancellation using retained IDs/keys/fingerprints. Observation uses a separate signal. Parent completion/release awaits verified terminal/contained evidence from **every attempted child**, including attempts whose response was lost. Lookup404/410, transport errors, a dead parent PID, expired deadline or a local AbortError are not containment. Unknown child/publication keeps the lease fenced. A child already committed records committed, never fabricated canceled; later follow-up failure remains visible. + +Startup reads fixed private receipts and matching shared ledgers, then performs authenticated lookup-only reconciliation; never redispatch an attempted mutation. Agent journal recovery and data-manager row versions prevent old local completion overwriting the new owner. Reconciliation can release only after effects/descendants and process groups are proved settled and immutable lock identities still match. If journal/ledger or required key is missing/malformed, fail closed and provide fixed host recovery instructions: stop **all** relevant writers/helpers, inspect retained receipts and publication state, then explicitly repair the exact owned record. No age-based removal and no automatic claimed rollback. The same rule covers parent death while a borrower is active and a helper continuing after the agent exits. + +## 4. Durable data-manager children + +02.3 creates `services/data-manager/src/jobs/execution/ops-children.ts` and a private `children` JSONB column on existing data-manager jobs in reserved migration0042. Maximum32 attempts per execution, each≤8KiB and aggregate≤128KiB; sanitized public child summaries are separately bounded to16. Internal attempts contain phase/ordinal, exact safe typed operation, key, body/execution fingerprint, normalized deadline, delegation UUID/digest, intended/accepted receiver ID, state `prepared|attempted|observing|terminal|unknown`, cancellation intent and settlement/publication evidence. No key/assertion/secret/private path is serialized in public `DataExecutionStatus`. + +`runOwnedDataOpsOperation(operation: Extract, options: {context: DataExecutionContext; phase: DataOpsChildPhase; ordinal: number; serviceIds: readonly string[]}): Promise>` is the only effectful administrative/transit adapter. `DataEffectKind` and phases are the finite data-manager→agent table above; `DataExecutionContext = {execution:ExecutionRef;executionVersion:number;signal:AbortSignal;targetAuthority:TargetLeaseAuthority;checkpoint(patch:DataExecutionProgressPatch):Promise;checkpointChild(attempt:DataOpsChildAttempt):Promise}`. `DataExecutionProgressPatch` is `Partial>`. ExecutionRef supplies the retained deadline. `lookupOwnedDataOpsChildren(context): Promise` and `cancelAndSettleOwnedDataOpsChildren(context): Promise<"settled"|"pending">` recover the same attempts. Read-only inspect/capacity calls may retain `runOpsOperation`; effectful managed calls may not take its random-key default. Existing one-off API/cron callers use only `DataExecutionInput` variants `{kind:"transit-maintenance",action:"restart-primary"}` or `{kind:"transit-maintenance",action:"reload-feed-proxy",candidateId:string}` with the existing bounded candidate-ID validator; no arbitrary ops operation is accepted. Their result is `{kind:"transit-maintenance",action:"restart-primary"|"reload-feed-proxy",changed:true}`. The cron owner starts before writing its proxy candidate and retains that candidate identity/phase through reload. + +Persist attempt+grant before POST, return only a validated result matching the retained identity, and checkpoint before the next local rename/proxy write/publication. Lost POST response invokes lookup only. Cancellation sends the remote cancel mutation with its own short transport timeout then observes under a settlement signal; an aborted worker signal must not abort these cancellation/read requests. Lost cancellation response is reconciled by reads; exact idempotent cancellation may be re-requested from durable cancel intent, never the original effect. No local proxy rollback or second promotion while an earlier agent child is unknown. `restartPrimary` must propagate unknown/pending as such instead of collapsing it into an ordinary failure string that triggers rollback. Update stop/recovery/import-restart/proxy stage/commit/restore and one-off API/cron MOTIS/proxy paths together. A cron/API one-off gets its own retained root; all effects within a transit execution inherit that execution. + +Data admission becomes `POST /executions {operationKey,input,requestedDeadlineAt?}` plus internal parent assertion header. `DataExecutionStatus` includes immutable `admittedAt` and `deadlineAt` milliseconds and its normalized fingerprint. Store parent binding/deadline/child attempts before work. `DataExecutionRuntime.admit(input, {operationKey,requestedDeadlineAt?,parent?:VerifiedParentExecution}): Promise` computes identity itself; never trust a caller-supplied fingerprint. Dispatch receives `DataExecutionContext`. Helpers use separate `{observerDeadlineAt,signal}` and `{requestedDeadlineAt,parentAssertion}` options. `DataChildAttempt = {phase:"osm"|"transit"|"fonts"|"data-step";operationKey:string;input:DataExecutionInput;operationFingerprint:string;requestedDeadlineAt:number;parentExecution:ExecutionRef;state:"prepared"|"attempted"|"observing"|"terminal"|"unknown";id?:string;fingerprint?:string;deadlineAt?:number}` is the private pre-admission workflow receipt,≤8KiB, at most16 per parent. Store assertion digest/one-use ID separately in its private authority receipt, not in this public projection. + +The fixed generation-link child additionally uses internal `DataExecutionInput` kind `generation-hardlinks` with `{revisionId,planDigest,coordinatorMode}`. It is admitted by a distinct fixed service-auth endpoint `POST /executions/generation-hardlinks` (operation ID `startGenerationHardlinksExecution`) with body `{operationKey,requestedDeadlineAt,input,plan}`; input has the preceding exact kind and existing confined `linkRequestSchema` validation, narrowed to the host `configuration.hardlinks.inspect` limit256 entries, source/target≤1024 characters and its existing bounded ID fields; prune is fixed true. Whole body≤1MiB UTF-8. Bind canonical plan bytes into planDigest and the signed child body fingerprint; derive targets only from the named generation. Agent supplies the plan under its host writer and leaf delegation; data-manager stores the private plan at its fixed recovery-root `hardlinks/.json` before ready,≤1MiB, signed digest and no-follow/fsync ownership. Child attempt records contain only the digest/identity, not the full plan. Browser endpoints expose no plan/path field. Public `/executions` rejects generation-hardlinks input to avoid a missing-plan path. Result is `{kind:"generation-hardlinks",hardlinks:HardlinkApplySummary}`. The leaf rechecks signed generation binding immediately before prune/apply and retains its result for lost-response recovery. + +### 4.1 Paired approval is one durable borrowed leaf (R2-01) + +02.3 creates core `ops/catalog-pin-approval.ts` for `PairedPinApprovalInput/Result` and owns the early closed `DataEffectKind` member `"transitousLock.approve"`, `DataOpsChildPhase`/`ControlChildPhase` member `"paired-pin-commit"`, portable input/result schemas and async policy;05.3 installs the validation-bound runtime and activation consumer. Until then approval is unavailable (including legacy ref-only callers), never a permissive placeholder. Extend the existing operation, with **data-manager role only**, maximum execution120000ms clamped to the activation's retained1h deadline; admission/read15s and settlement30s remain independent. Request/result each≤4096 UTF-8 bytes: + +```ts +type PairedPinApprovalInput = { + kind: "transitousLock.approve"; + proposalId: string; + validationId: string; + expectedGenerationId: string | null; + diffDigest: string; + acknowledgeLicenseChanges: boolean; +}; +type PairedPinApprovalResult = { + outcome: "activated"; + proposalId: string; + validationId: string; + previousGenerationId: string | null; + generationId: string; + commitId: string; + committedAt: string; +}; +type CatalogPinCommitIdentity = { + commitId: string; + execution: ExecutionRef; + proposalId: string; + validationId: string; + diffDigest: string; +}; +``` + +Proposal/validation/commit IDs are UUIDs; generation/diff IDs64 lower-case hex; committedAt ISO. No ref, raw pins, caller-supplied passed flag or actor label can authorize approval. `approve` requires§3.4's authenticated DM bearer **and** anchored one-use borrowed grant from the exact transit activation root, not merely the role token. Body and normalized execution fingerprints include every input/parent/deadline. Stable phase key uses `paired-pin-commit`, ordinal0; persist the child attempt/grant/receiver ID before POST via `runOwnedDataOpsOperation`. Standard version2 async status, by-key lookup and fingerprint-bound cancel retain this exact identity/result. Unknown POST/cancel/status is lookup-only and keeps the parent target fence; no second pair transition or fresh approval key. Cancellation before pointer commit can prove no effect; after commit retain activated evidence; unprovable pointer/containment stays unknown. + +Agent `transitous-lock-runtime.ts` is the sole authorization/store adapter.05.3's paired `transitousLock.propose` retains an immutable agent-signed proposal at fixed `infra/docker/catalog-pins/proposals/.json` (≤768KiB, including the≤512KiB pair). DM writes role-signed immutable validation at `data/motis/proposals//validation-.json` and activation evidence at `.../activation-.json`, each≤64KiB. Agent derives these paths from validated IDs under its already-open roots; descriptors are no-follow/regular/single-link, rechecked before commit. It verifies the pinned DM role signature, proposal signature/hash/base, passed unexpired24h evidence, exact diff/license acknowledgment, source-policy/overlay/credential revisions, countries, slot/epoch/manifest/image and the exact successful `promote-primary` child receipt under this parent. Activation evidence binds its immutable parent, both child identities and the current effective config/credential mount witness; it is not an assertion from the approval body. Rehash candidate manifest and compare owned runtime promotion evidence; source acquisition errors block. The parent keeps its target lease and nonblocking transit reservation continuously from final validation through promotion, approval and child settlement. The§5.4 guard prevents managed credential changes during this interval; fixed host-file/source witnesses still require before-commit identity checks. API/DM never mount the agent catalog tree writable. + +05.7 extends its local-only CAS to `publishCatalogPinSet({rootDir,lease,pins,expectedGenerationId,commit:CatalogPinCommitIdentity}):Promise` and `readCatalogPinCommit({rootDir,commit:CatalogPinCommitIdentity}):Promise`. `CatalogPinCommitReceipt = {version:1,identity:CatalogPinCommitIdentity,previousGenerationId:string|null,generationId:string,phase:"prepared"|"committed"|"not_committed"|"unknown",committedAt:string|null}`. Store intent/receipt in the agent's fixed private recovery `catalog-commits/.json` (≤64KiB), using§3 durability/signature/high-water rules; retain terminal72h, never evict unresolved. Allocate commitId once and fsync the expected pointer identity, intended generation and full attempt binding **before** replacing the pointer. The atomic pointer becomes `{version:1,generationId,commitId,operationKey,fingerprint,committedAt}` (≤4096 bytes); initialization uses its own explicit bootstrap identity. Generation content/hash remains unchanged. Rename plus directory sync selects the pair; receipt stores that exact marker, not a later arbitrary current generation. Reconciliation validates the retained intent and exact marker: matching commitId/key/fingerprint+generation proves this attempt; exact unchanged predecessor plus contained writer proves noncommit; anything else remains unknown. If the rename survived but receipt delivery did not, recover the original result and committedAt from this marker. Never infer success from generation equality alone, replay the pointer write, or infer noncommit from a competing current pointer. No managed writer can advance it while unresolved; once settled the retained receipt remains authoritative even after later generations. Identical already-current pins without this attempt's marker do not constitute a newly activated pair. Bootstrap initialization remains separate and cannot manufacture a validation receipt. + +Required planned acceptance in05.3 uses real separate DM/agent transports and this CAS: promote→approve under one root; lose approval POST/terminal response; kill each process before pointer commit and after rename before receipt sync/delivery. Assert one phase key, exact recovered attempt/result, at most one pointer transition, no root reacquisition/release between promotion and publication, no cancel success while unknown, and no subsequent sync/start until reconciliation. Store-only tests are insufficient. + +## 5. Strict prospective configuration + +### 5.1 Writer order and exact preparation surface + +01.2 introduces one API administrative writer boundary using a **dedicated PostgreSQL connection** and session advisory key `(0x4f4d58,1)`,5s acquisition budget. Order is API global configuration writer → sorted per-integration advisory locks → host target lease → nonblocking transit reservation → host configuration writer → release store. The integration lock namespace stays the one assigned by04.1; it is entered only after this global guard. Strict reads and runtime reload do not acquire write locks. Host/data-manager effects never call back into an API configuration writer while holding host ownership. No global/integration reverse acquisition is permitted. + +```ts +withAdministrativeConfigurationWriter(options: { + actorId: string; operationKey: string; integrationIds: readonly string[]; + signal?: AbortSignal; +}, run: (writer: AdministrativeConfigurationWriter) => Promise): Promise; + +prepareProspectiveConfiguration(input: { + actorId: string; expectedCurrentRevisionId: string | null; + overlay: ProspectiveConfigurationOverlay; operationKey: string; + signal?: AbortSignal; +}): Promise; + +persistAndApplyProspectiveConfiguration(candidate: ValidatedConfigurationCandidate, + options: { coordinatorMode: CoordinatorMode; signal?: AbortSignal } +): Promise; +``` + +The writer is a nominal context backed by that same live connection, job owner/version and AsyncLocalStorage scope. `prepare` requires this context and matching actor/key; no implicit acquisition. Nested helpers reuse that exact context/connection, verify that requested integration IDs are within its already locked set and cannot expand/reorder locks. No `alreadyLocked:true` parameter.04.1 binding edits,04.2 direct config/secret guards,04.5 imports,04.6 encrypted transfer,05.6 feed mutations and08 platform writes all enter here **before** their per-integration locks. Reload receives the captured immutable candidate and acquires no writer. Ordinary public provider reads keep their existing fallback policy. + +01.2 also exports the exact finite transaction/read seams required by04 (from `administrative-configuration-writer.ts`, with candidate dispatcher in `prospective-configuration.ts`): + +```ts +withAdministrativeConfigurationRead(options:{signal?:AbortSignal}, + run:(context:AdministrativeConfigurationReadContext)=>Promise):Promise; +withConfigurationCommitParticipant(writer:AdministrativeConfigurationWriter, + participant:ConfigurationCommitParticipant, run:()=>Promise):Promise; +``` + +`AdministrativeConfigurationReadContext` contains a coherent read-only transaction/registry/host-source observation; it never acquires global/integration write locks, and cannot persist. Candidate preparation derives the corresponding context from its existing writer connection. `ConfigurationWriteTransaction` is the writer connection's Drizzle transaction subset (`select/insert/update/delete/execute`), not the global pool. `ConfigurationCommitParticipant` is nominal and closed to `integration-binding|configuration-migration|credential-import|transit-credentials`. Its fixed shape is `{kind,mode:"preview"|"apply",actorId,operationKey,managedServiceSecretKeys:readonly [{serviceId:"data-manager",key:"TRANSITOUS_API_KEYS"}]|readonly [],authorize(writer):Promise,verify(candidate,writer):Promise,commit(candidate,writer,tx:ConfigurationWriteTransaction):Promise}`. Only the owning built-in04/05 module can construct its branded participant; no arbitrary registered callback/remote plugin participant. Foreign/expired/conflicting/second participants reject. Nested adapters reuse the exact scoped participant and live writer without acquiring locks/transactions again. + +`authorize` is read-only before prepare; `verify` is read-only and binds actor/proof/source/preview/pin/map revisions to the candidate before persistence and again inside the write transaction. Only after all prospective budgets/schema/ownership checks pass, `persistAndApplyProspectiveConfiguration` invokes `commit` once inside its own transaction, alongside generic overlay and job checkpoint writes. Exclude only the exact managed feed key from generic persistence when its participant claims it, so05.6 writes it once. Commit failure rolls back ordinary+managed rows, credential revision/invalidation and consumed03 proofs; no target-mutating continuation is authorized. The already-prepared ownership-only reservation, when required by§5.4, can be abandoned only with exact known-unused/rollback evidence. Persist only participant kind/actor/operation+preview IDs/completed identity in the private job metadata, never callback/proof nonce/session/secret. Restart after commit observes original candidate/identity and never repeats proof consumption. Preview participant cannot enter persistAndApply, consume proof, save targets, enqueue or publish; discard its inert candidate by bounded TTL/queue cleanup. + +04.1 owns binding participant expected-prior CAS/receipt;04.5 owns pending auth drafts and binding CAS;04.6 owns decrypt-stage/import intent and03 `check/checkMany` before prepare versus `consume/consumeMany` in transaction, with optional05 `PreparedFeedCredentialPatch`.05.6 owns `persistPreparedFeedCredentialPatch` on this **same** writer/transaction.01.2 lands the closed context/dispatcher and rejects uninstalled kinds until their owner task wires its adapter; it does not invent early mock proof/pin policy or expose those workflows prematurely. Acceptance includes two real connections with inverse caller lock schedules, nested credential/feed apply on one context, changed pin/proof after verify, failure after proof consumption (all rows/proofs roll back), and preview persist rejection. This is the finite extension seam for existing administrative invariants, not a general transaction callback service. + +Final proof naming is exclusively C03's `check` (single, exact consume argument shape) and `checkMany` (exactly the finite two-proof consumeMany argument shape), both read-only on the supplied transaction context. Direct transit uses check/consume; credential-import plus transit uses checkMany/consumeMany. Do not introduce validateProofs, a variable-size proof batch or a separate validator implementation. Root reconciles B05 to these names and the common prepared-patch `overlayChange` name; this producer accepts only the actual finite overlay variant. + +`ProspectiveConfigurationOverlay` is strict `{purpose,coordinatorMode,changes}` with required `CoordinatorMode` from§6. Purpose is `render|selection|service-config|service-secret|integration-config|integration-secret|migration|platform|transit-credentials`. Seal the mode into the typed configuration operation and candidate; persistAndApply's mode must match or conflict before persistence. Changes≤512 are discriminated variants: + +- `{kind:"service-config",serviceId,set:Record,unset:string[]}`; +- `{kind:"integration-config",integrationId,set:Record,unset:string[]}`; +- `{kind:"service-secret",serviceId,key,action:"set",value:string}` or corresponding `action:"delete"` without value; +- the same set/delete shapes for `kind:"integration-secret",integrationId`; +- `{kind:"selection",selectedRoots:string[]}`; +- `{kind:"platform",patch:PlatformConfigurationPatch}`. + +`BoundedConfigValue` follows `trustedConfigurationPayloadSchema`'s bounded JSON; schema-declared ordinary keys only, no raw env map. Disallow duplicate/contradictory changes to one field and limit each set/unset collection to128 keys; IDs use existing schema validators. `PlatformConfigurationPatch` has only optional `domain`, `acmeEmail`, `releaseSelector`, `defaultRegion`, `backupRetentionDays`, each `PlatformField = {source:"host"}|{source:"database",value:T}`; exact value validators are08.1's hostname/email/release-selection/allowed-region/1–36500 bounds.01.2 declares the closed patch type and rejects platform purpose until08.1 installs its typed resolver; it preserves existing platform state in every unrelated candidate once installed. Managed `data-manager/TRANSITOUS_API_KEYS` is accepted only with purpose transit-credentials and the validated05.6 authorization context, or04.6's adapter proving both credential-import and catalog-bound transit.credentials; generic secret purpose rejects. The scope consumer supplies its purpose-specific proof before calling prepare; no public caller can opt into a purpose to bypass proof. + +Choose the typed apply operation deterministically: render/migration/platform→stack.render; selection→serviceSelection.apply (one selection change); service-config→serviceConfig.apply (one service ID); service-secret/transit-credentials→vault.apply (one service ID, managed map fixed data-manager); integration-config/integration-secret→integrationConfig.apply (one integration ID). Multi-target imports use migration and its finite participant. Every kind binds expectedCurrentRevisionId, expectedSourceRevision, coordinatorMode and the sealed revision in its body fingerprint. Integration vault changes remain in the API's private overlay/runtime reload context, not in the trusted envelope's existing projected integrationConfigs; do not leak integration secrets into host config by flattening effective resolution. + +`ValidatedConfigurationCandidate` is an opaque frozen server-private object with readonly internal metadata `{candidateId:UUID,operationKey,expectedCurrentRevisionId,revisionId,sourceFingerprint,sealedBytes,affectedServiceIds}`. `sealedBytes` is a **count**, not byte content. It privately retains merged row values, source versions/ownership, payload, exact sealed envelope bytes, nonce/issue/expiry and typed operation. `ProspectiveConfigurationReceipt` is `{candidateId,revisionId,saved:"unchanged"|"committed"|"conflict"|"unknown",application:TrustedConfigurationApplyOutcome}`. Public routes project only approved metadata; never serialize candidate, source fingerprint, secret content or a public digest of a secret. Source fingerprint is HMAC-SHA256 over complete observed source/revision data using a domain-separated subkey derived from the API's existing32-byte ops token (`HMAC(token,"openmapx-prospective-source-v1")`); keep this server-private. This token is already required for sealing and avoids requiring a vault key when the instance has no stored vault secrets. Agent host sourceRevision uses its own role key to sign the private witness; public projection contains only an opaque random witness ID, never a public hash of host values. No new user-managed secret service. + +### 5.2 Strict reads, authority and ownership + +Produce `resolveServiceConfigWithSourcesStrict(manifest:ResolveServiceConfigInput, context:AdministrativeConfigurationReadContext): Promise>` and `resolveAllServiceConfigsStrict(manifests:ResolveServiceConfigInput[],context): Promise`. Context carries the current writer's read transaction, exact registry generation, captured host observation and overlay. Preserve current core type imports and env-key/reference behavior, including the proxy host exception. Add `resolveIntegrationConfigWithSourcesStrict(manifest,directory,context)` and strict integration-vault access for this producer;04.2 consumes them for projection rather than creating a second collector. Failed query, malformed row/file, unreadable file, undecryptable vault or missing decryption capability with existing rows throws a fixed sanitized acquisition error. Only confirmed ENOENT/zero-row is absent. Read all unchanged registered service/integration config, service/integration vault and adopted platform/root state inside one coherent DB transaction; don't skip disabled services, unchanged secret maps or orphaned map keys. Orphan **credential map entries** are preserved; undeclared/unregistered incompatible configuration rows receive bounded diagnostics and require explicit correction, never silent removal from the candidate. + +Source authority matches actual resolution: service `default < database < env`, integration `default < database < vault < config.json < env`; empty host strings follow existing unset semantics. Reject proposed set/delete that is shadowed by file/env before persistence.01.2 owns core policy;04.2 owns direct-route enforcement and visible `ConfigurationOwnership`. Resource ownership additionally includes namespaced `SERVICE__OPENMAPX_MEMORY_MIB`/`…CPU_LIMIT` and actual host interpolation tokens in the manifest's resource field (e.g. `MOTIS_SLOT_MEMORY` in primary/staging), plus08.2's declared mapped heap/slot knobs. Use the same closed field-to-runtime mapping as the renderer; do not infer permission from an absent schema field or parse arbitrary shell. Numeric UI resource values cannot silently override a present supported host knob. Removing that override and deliberately adopting is a fresh intent. Platform adoption is the explicit finite exception defined in08.1; it never grants arbitrary service/env adoption. + +Host interpolation not visible in API `process.env` must be supplied by a new **read-only** API-role sync15s `configuration.sources.inspect` agent operation owned by01.2. Request has no path/env names; response is bounded≤128KiB `{revisionId,sourceRevision,services:[{serviceId,fields:[{key,source:"host"|"none",envVar?:string}]}]}` with≤256 services/128 fields per service, no values; exceeding total bound is unavailable, never truncation. Agent derives the finite field mapping from trusted manifests and supported renderer mappings, descriptor-reads actual host interpolation inputs, and verifies source identities/current revision before and after that read (at most3 attempts within15s, otherwise unavailable). This read acquires no API/integration write guard or host writer. Retain an owner-private signed source witness indexed by opaque random sourceRevision,≤128 entries,≤1MiB aggregate,5min lifetime; active admitted candidate pins its exact witness until settlement within the existing job budget. Candidate typed config operation carries `expectedSourceRevision`; agent rechecks the witness/source observations under host writer immediately before commit and refuses drift. Existing seal body fingerprint binds this addition. A failed source inspection blocks prepare. Public inspection only shows safe names/ownership; no secret hash/value. This closes the API-versus-host-env gap without importing host `.env` into the API. + +### 5.3 Prospective budgets and persistence + +Before any desired-row/vault/proof mutation, effect enqueue or consumable snapshot publication, and before the ownership-only maintenance admission in§5.4: merge the full overlay into strict unchanged state; enforce manifest schema/ownership; run the exact trusted payload parser; call `sealTrustedConfigurationSnapshot` in memory using the real token/operation/revision/nonce/timestamps; enforce its actual UTF-8 byte count≤524288. Count escaping, identifiers, MAC and newline, not string length estimates. Retain existing≤256 service/config/secret lists,≤128 top-level config/secret keys,≤64 nested record/array children, depth≤6 and≤2048 nodes per config tree. Current ordinary secret limit65536 JS code units remains an independent upper limit; ordinary credential-transfer values stay≤16KiB UTF-8 per04.6. Feed map has additional `canonicalFeedCredentialMap` limits:≤1000 keys, each key≤512 Unicode code points, each value≤8192 UTF-8 bytes, complete sorted compact JSON≤65536 UTF-8 bytes. Reject invalid Unicode, duplicate JSON keys, non-string values and unsafe keys. Count all retained/orphan keys and JSON escaping. Never truncate.05.6 owns its catalog/pin/credential revision validation and uses this exact canonicalizer. + +The same in-memory seal is the candidate that is persisted and admitted; no subsequent unrelated-row reconstruction. Extend publisher with `publishPreparedTrustedConfigurationSnapshot(candidate)` accepting this nominal candidate, validates exact bytes/identity/TTL and reserves existing queue capacity. Persist prepared private bytes under the fixed `OPS_TRUSTED_CONFIG_DIR/.candidates/.json` receipt namespace (0700/0600, no-follow,≤768KiB per record including metadata; new explicit bounded scanner branch). Account candidates/reservations in the existing128-entry/16MiB queue budget, so moving one into the queue/claim cannot double count or bypass quota. This directory contains private sealed content just as the existing trusted snapshot queue does; it is never a public file endpoint. Failure of sealing/capacity/private durable write stops before database mutation. The existing5-minute seal TTL must still be valid at the DB decision and initial apply-claim admission; do not reseal an attempted identity. Unadmitted expired candidates may be discarded. A candidate attached to a prepared/unknown maintenance reservation is pinned until exact abandonment/settlement, even after expiry; expiry prevents forward apply but does not remove its fence. A new explicit intent requires proof that the old reservation/effects are settled. + +Under the same API global+integration ownership, use§5.4 after successful whole-candidate/private-receipt preflight whenever the candidate needs the DM/configuration maintenance boundary. Agent ownership-only prepare precedes final rechecks and desired-row persistence; generation/effect admission follows the durable DB decision. Recheck exact source versions and candidate TTL under that remote fence, then transactionally write the finite overlay plus existing `admin_job` ownership/checkpoint row containing candidate ID, operation identity, source version, maintenance/decision identities and phase **only**, no plaintext/cipher material. The writer's dedicated connection is reused, never another pool connection that accidentally bypasses its locks. Once DB save commits, publication/target-mutating admission uses the retained candidate, original expected revisions and exact authorized maintenance continuation (when present). Persist each binding/decision/attempt before POST; never reacquire a new target root for apply. Unknown outcome retains both candidate and active administrative-writer fence through that job row; every later writer checks for an unresolved fence while holding global advisory ownership and refuses, even after process death released its session lock. Read-only views continue. Session locks may release after a durable pending outcome, but that durable fence remains until reconciliation proves result. Startup/next writer performs lookup-only reconciliation first. Do not erase a candidate or accept a new configuration writer simply because an observer aborted. + +If legacy unchanged state is already over a budget, allow an explicit size-reducing correction only when its **complete resulting candidate passes all schema, ownership and envelope limits**. No bypass for an intermediate still-oversize save; return sanitized limit diagnostics to permit a larger corrective batch. After persistence failure/unknown apply retain `saved` and host commit evidence; do not blindly restore. Existing04 saved-row ownership and `previous_applied_not_proven` recovery rules still apply. Compensation itself must prepare a strict candidate under the same context and exact revision/owned-row proof, never reconstruct unrelated newer state. + +### 5.4 Remote configuration/feed maintenance (R2-02) + +**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. + +**Preflight before ownership admission.** Under the API global→sorted integration writer, first perform non-consuming `check/checkMany`, strict full prospective schema/source/size validation, exact seal, queue-capacity reservation and durable private candidate receipt. No remote ownership admission is allowed if any of these fails. Read-only host source/hardlink/health inspection is permitted during preflight. The inert candidate may be staged privately for those reads; it is not a consumable apply claim. Derive all eventual targets/effects before prepare: changed service closures, full prospective hardlink producer/consumer closure, DM if needed, MOTIS/motis-staging/motis-feed-proxy closure for feed credentials, and the protected helper subset. Include dependencies required for recreation, never just caller IDs. Candidate retains the exact safe follow-up intent, current image/container expectations and plan preview. Agent independently derives and freezes the closure from trusted manifests, candidate and source/preview witnesses. Reject unavailable prerequisites, >256 sorted targets, >32 effect children or changed plan before prepared; no partial plan or late target expansion. `require-running` never starts DM; its feed flow may explicitly recreate an already-running DM for the new mount, whereas `render-only` permits no lifecycle at all. These distinct actions are frozen before save. Before sealing, allocate candidateId/maintenanceId and the prepare key once using the existing durable API key helper with namespace `configuration-maintenance.prepare` and durable identity `apiJobId + ":" + candidateId`. Derive applyOperationKey from that prepare key with§3.5 phase `configuration-commit`, ordinal0, so the sealed apply identity already equals the frozen child key. Preallocate all safe effect inputs; no post-prepare reseal is required. + +Portable strict contracts (all IDs UUID unless an existing producer has its own validator; revisions use cfg1/null and fingerprints64 lowercase hex): + +```ts +type ConfigurationMaintenanceBinding = { + maintenanceId: string; + candidateId: string; + applyOperationKey: string; + applyOperationFingerprint: string; + expectedCurrentRevisionId: string | null; + revisionId: string; + expectedSourceRevision: string; + coordinatorMode: CoordinatorMode; + followUp: "render-only" | "service-apply" | "platform-apply"; +}; +type ConfigurationMaintenancePrepare = ConfigurationMaintenanceBinding & { + kind: "configurationMaintenance.prepare"; +}; +type ConfigurationMaintenanceIdentity = { + maintenanceId: string; + candidateId: string; + prepareOperationKey: string; + prepareFingerprint: string; + planDigest: string; +}; +type ConfigurationMaintenanceDecision = ConfigurationMaintenanceIdentity & { + kind: "configurationMaintenance.decide"; + decisionId: string; + decision: "continue" | "abandon"; + expectedSequence: number; + checkpointVersion: number; + checkpointDigest: string; +}; +type ConfigurationMaintenanceSettle = ConfigurationMaintenanceIdentity & { + kind: "configurationMaintenance.settle"; + decisionId: string; + expectedSequence: number; + evidenceDigest: string; +}; +type ConfigurationMaintenanceEffect = Extract< + OpsOperation, + { + kind: + | "stack.render" + | "serviceSelection.apply" + | "serviceConfig.apply" + | "integrationConfig.apply" + | "vault.apply" + | "configuration.hardlinks.apply" + | "service.recreate" + | "service.recreateIsolated" + | "configurationHandoff.prepare"; + } +>; +type ConfigurationMaintenanceStep = { + phase: + | "configuration-commit" + | "maintenance-hardlinks" + | "maintenance-recreate" + | "maintenance-handoff"; + ordinal: number; + operation: ConfigurationMaintenanceEffect; + operationKey: string; + operationFingerprint: string; + serviceIds: string[]; + prerequisites: number[]; +}; +type ConfigurationMaintenanceReceipt = { + version: 1; + binding: ConfigurationMaintenanceBinding; + prepare: ExecutionRef; + apiJobId: string; + parentOperationKey: string; + parentFingerprint: string; + parentDeadlineAt: number; + planDigest: string; + serviceIds: string[]; + steps: ConfigurationMaintenanceStep[]; + preflight: "current" | "drifted" | "unavailable"; + sequence: number; + state: + | "preparing" + | "prepared" + | "authorized" + | "applying" + | "settling" + | "settled" + | "recovery_required"; + decision: { + decisionId: string; + decision: "continue" | "abandon"; + checkpointVersion: number; + checkpointDigest: string; + } | null; + dmFence: DataConfigurationFenceReceipt | null; + generation: "not_started" | "committed" | "not_committed" | "unknown"; + appliedRevisionId: string | null; + followUp: "not_started" | "complete" | "contained" | "unknown"; +}; +``` + +Step ordinals are integers0–31, target IDs are unique sorted subsets of the root closure, prerequisites are unique earlier plan indices≤31, and the graph is acyclic with exactly one first generation commit. Whole receipt≤64KiB and private plan≤128KiB remain aggregate caps; exceeding either refuses preparation, never truncates. + +`configurationMaintenance.inspect {maintenanceId,candidateId}` returns this receipt or explicit missing/unavailable; no authority is created by a UUID. All four ops are API-role only and enforce the original API job binding. Inspect refreshes `preflight` by checking the retained source witness/current config and frozen prerequisite identities; drift/unavailable blocks the API save rather than presenting a cached prepared check as current. Steps are internal API-to-agent data, not browser authority; do not project their private plan to public job responses. Prepare is async,1h max clamped to the retained API job deadline (acquisition itself5s, transport15s); its terminal result is a receipt in `prepared`, not applied or released. Decide/settle are async,60s bounded control attempts, each keyed and journaled before mutation; inspect is sync15s. Request≤8KiB; receipt≤64KiB, sequences nonnegative safe integers. Forward continue/effect admission uses§2 normalization and rejects an expired root. No-effect abandon and settle are cancellation/reconciliation controls: authenticate the original expired binding using `VerifiedRecoveryAuthority`, keep its immutable forward deadline in every receipt, and bound only the control attempt to60s (request15s), as for cancellation. They may contain/read/release proved work after expiry, never issue a forward grant or extend the root budget. All status/lookup/cancel paths retain the usual body/execution fingerprint distinction. Cancel or expiry fences new forward phases; it cannot automatically choose abandon, release an unknown reservation or undo saved rows. + +API module exports `prepareConfigurationMaintenance(candidate:ValidatedConfigurationCandidate):Promise`, `inspectConfigurationMaintenance(identity:Pick):Promise`, `decideConfigurationMaintenance(decision:ConfigurationMaintenanceDecision):Promise` and `settleConfigurationMaintenance(request:ConfigurationMaintenanceSettle):Promise`. They require the same live nominal writer/job context, persist attempts before POST, use retained-key lookup after any attempted unknown call, and are called inside `persistAndApplyProspectiveConfiguration`;05/04 do not acquire host leases themselves. The inspect input deliberately needs no returned planDigest/prepareFingerprint, so it is usable after a lost prepare response; verify any discovered binding against the persisted original attempt before adopting it. Recovery gets the writer/job claim first and reconstructs these inputs solely from the retained private candidate and `admin_job.result` checkpoint. Original API job ID/key/fingerprint/deadline are immutable; a new runner owner/version may observe or deliver an **unattempted retained** decision, not invent a new candidate/deadline. The existing§3.4 `openmapx-api-job-parent-v1` HMAC covers each exact typed child body fingerprint, including binding/decision fields. Persist those fields under API job CAS before signing/POST. Prepare verifies original apply operation HMAC/body and source witness from the fixed private candidate; subsequent controls require the same authenticated API parent and stored binding. No proof nonce or secret hash enters this wire receipt. + +**Durable root and finite continuation.** Agent stores a signed≤128KiB record at its existing private recovery `configuration-maintenance/.json`, with immutable binding, API attestation digest, root certificate, frozen plan and per-phase attempts/results. Its≤32 effect children plus fence controls stay within§3's64 intents/128KiB ledger bounds; refuse capacity before effects. Private candidate bytes remain in the existing reserved128-entry/16MiB trusted queue budget, pinned until settled, with no duplicate accounting. Keep terminal identities72h; unresolved records never expire/evict. A crash after root acquisition/before DM fence admission or after fence admission/before prepared result keeps `preparing/recovery_required` and uses exact lookup, never another root/prepare POST. + +The frozen effect plan contains one original typed `stack.render|serviceSelection.apply|serviceConfig.apply|integrationConfig.apply|vault.apply`, then at most one `configuration.hardlinks.apply`, exact ordinary `service.recreate|service.recreateIsolated` children, and at most one `configurationHandoff.prepare`. Compute planDigest from canonical version1 binding, target closure and safe normalized step intents **excluding** their planDigest fields and derived fingerprints/grant IDs, then fill those fields and compute each operation fingerprint; no self-referential hash. Derive stable keys from§3.5 finite phases and sorted ordinal; each record fixes kind/body hash/deadline/target subset and prerequisites. The generation body includes maintenanceId/candidateId before sealing; follow-ups include maintenanceId, desired revision and planDigest. 01.2 owns the closed protected step intent with the exact08.3 target enum; it refuses platform/protected continuation until08.3 installs its fixed adapter.08.3 adds the typed operation schema and then materializes the frozen target/image/preview intent before save; no absent-module forward import, generic adapter registry or change of closure. Each API request still carries its purpose-bound parent HMAC; the agent matches it to the frozen step and issues/adopts the one-use signed grant **locally from its retained owner**, supplying a borrowed `OpsExecutionContext.artifactLease`. No API-made lease/assertion and no later root acquire. The fixed verifier recognizes a terminal prepare journal only with its same retained maintenance record and terminal authorized decision, then only for the next frozen unattempted step; terminal status alone cannot issue a grant. Internal grant/admission intent is persisted before any effect, using§3.4's normal one-use pending-adoption sequence. Unknown child admission is lookup-only. Hardlinks may delegate only to the generation-hardlinks leaf at depth2. The helper may be depth2 below terminal handoff preparation, with no descendant. Generation commit obtains the host writer under this authority, rechecks source/current revision and commits the exact retained seal; it releases the host writer before a child needing that writer. Runtime data/images cannot change during prepare/decide. Continue decision is terminally journaled **before** any child can start; no live maintenance or decide operation waits for its children and blocks helper drain. API job coordinates the frozen children; the external helper/agent recovery can settle already-authorized work after API replacement by inspecting receipts, never resubmitting attempted children. + +**Fixed DM logical guard —02.3 produces before01.2.** Add `packages/core/src/services/data-manager/contracts/configuration-maintenance.ts` (portable types/canonical service-auth operations), `packages/core/src/services/data-manager/configuration-maintenance.ts` (typed client), and `services/data-manager/src/jobs/execution/configuration-maintenance.ts` plus tests. Define: + +```ts +type DataConfigurationFencePrepare = { + maintenanceId: string; + candidateId: string; + ownerPrepare: ExecutionRef; + planDigest: string; + serviceIds: string[]; + revisionId: string; + hardlinkPlanDigest: string | null; + coordinatorMode: CoordinatorMode; +}; +type DataConfigurationFenceReceipt = { + fenceId: string; + maintenanceId: string; + candidateId: string; + ownerPrepare: ExecutionRef; + planDigest: string; + revisionId: string; + hardlinkPlanDigest: string | null; + sequence: number; + state: "prepared" | "settled" | "recovery_required"; +}; +type DataConfigurationFenceSettle = { + maintenanceId: string; + candidateId: string; + planDigest: string; + decisionId: string; + expectedSequence: number; + outcome: "unused" | "applied" | "contained"; + ownerReceiptDigest: string; +}; +``` + +`planDigest` here identifies the entire frozen maintenance plan; `hardlinkPlanDigest` is the distinct existing§4 digest of canonical generation-link bytes, or null for no link work. Never compare that leaf's `DataExecutionInput.planDigest` to the whole maintenance digest. The signed root/grant supplies the maintenance identity, and the guard matches the leaf's revision/hash to its retained hardlinkPlanDigest. + +Register `prepareDataConfigurationFence` POST `/executions/configuration-maintenance/prepare`, `inspectDataConfigurationFence` GET `/executions/configuration-maintenance/:maintenanceId`, and `settleDataConfigurationFence` POST `/executions/configuration-maintenance/:maintenanceId/settle`, ahead of dynamic execution IDs. Prepare returns200 only after durable preparation, not202 worker admission; request/receipt≤32KiB,≤256 IDs,15s request limit. The typed client signatures are `prepareDataConfigurationFence(input:DataConfigurationFencePrepare,options:{transport:DataManagerTransport;controlAssertion:string;signal?:AbortSignal}):Promise`, `inspectDataConfigurationFence(maintenanceId:string,options:{transport:DataManagerTransport;witness:string;signal?:AbortSignal}):Promise` and `settleDataConfigurationFence(input:DataConfigurationFenceSettle,options:{transport:DataManagerTransport;controlAssertion:string;signal?:AbortSignal}):Promise`; adapters unwrap canonical response unions and throw bounded missing/busy/unavailable errors. Internal headers are set only by these service adapters; no browser parameter can supply them. Exact retries observe the retained record; changed owner/candidate/plan conflicts409. Lost requests use inspect only. These are **service-only**, never browser/proxy DataExecutionInput variants. Require existing DM bearer plus pinned ops-agent signature:§3.4 compact assertion references an anchored grant phase `configuration-fence`/`configuration-fence-settle`, exact body/owner/root/plan/target subset and one-use ID. Inspect verifies a purpose-bound ops-agent signed execution witness naming the same maintenance/candidate/root; the shared bearer alone cannot impersonate this role.02.3 defines the strict inspect witness domain `openmapx-configuration-fence-inspect-v1`, fields `{installationId,maintenanceId,candidateId,ownerCertificateDigest,issuedAt,expiresAt}`,≤4096 encoded bytes, lifetime≤15s; repeated reads grant no mutation. No arbitrary operation, path or callback is accepted. The DM preparation adopter settles its short borrowed node with committed **logical guard** evidence before returning prepared, retaining no live target borrower; future hardlinks adopt their own exact authorized grant under the original agent root. + +Use existing `data_manager.jobs` with kind `configuration-maintenance`, status `ok` and finishedAt marking completion of preparation, not release; `metadata.configurationMaintenance` stores the versioned receipt and `guardState:"preparing"|"prepared"|"settling"|"settled"|"recovery_required"`. Reserve this finite≤64KiB metadata schema, unique maintenance/key identity and partial unique index `data_manager_jobs_configuration_guard_uq` on `kind` WHERE `kind = 'configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'` in **forward0042**, alongside its already-planned execution metadata; do not modify0042 after application or allocate0051. `admin_job.result` holds API binding/decision/attempt metadata; it needs no new table/migration. Generic job expiry/reconcile/markSyncFinished must not erase or finish this guard. It is explicitly excluded from live worker/drain queries, while its guardState still denies admissions. + +Serialize **every DM mutation admission** and guard transition under transaction-scoped `pg_try_advisory_xact_lock(0x4f4d58,2)` on that transaction's connection; no waiting for another reservation while holding targets. Busy returns409 `configuration_maintenance_busy`. Under the same guard transaction, reject a prepared/uncertain guard before inserting a job or authorizing a filesystem/source mutation. Preparation checks all active/cancel-pending/unknown executions, outstanding child/publication receipts, current transit reservation and other data/transfer consumers; any unrelated unsettled work returns busy without changing it. One admitted job wins or one guard wins; a status check followed by an unlocked insert is invalid. Job insertion/reservation commits before its effects; unfinished admissions count as busy. Extend `SingleFlightController.tryStartSync` to this common DB admission adapter; `markSyncFinished` clears only its own transit execution, never maintenance. All API, cron and CLI data/transit paths (including source-overlay writes, proposal/validation/activation, downloads, links, Overture, search/notable-place, offline preparation and mutating traffic tasks) enter the same guarded adapter before effects. Existing specialized job tables keep their own rows; their admission transaction also takes this fixed guard and checks it. No generic remote locking endpoint or optional bypass. + +Persist a signed mirror at the existing DM-private recovery root `configuration-maintenance/.json` (≤64KiB; at most one unsettled guard, terminal72h). `fenceId` is the UUID receiver execution ID preallocated by the signed grant and becomes the jobs row ID, so adoption/row recovery refers to one identity. After the busy checks under the SAME DB admission transaction guard, write a preparing mirror before committing the DB reservation; after commit finalize prepared and settle the short adoption node before responding. Either half missing/uncertain blocks admission, never looks idle. Known pre-reservation rollback removes only its exact unused preparing mirror after adoption/DB noncommit proof; an unknown write/commit retains it for reconciliation. No other guard is erased on busy or rollback. Startup loads/reconciles private mirror, shared signed owner/guard and DB state **before cron/startup mutations or worker readiness**; DB unavailable, corrupt/missing expected mirror or mismatched identities keeps admissions closed. Read-only health/status/inspect/cancel containment remains usable. DM recreation preserves this private mount and same identity. A new DM can report control/health ready with data admission fenced; exact maintenance-bound generation-hardlinks is the only admitted write exception, requiring the frozen step/grant, exact revision/hardlinkPlanDigest and authorizing decision witnessed by the agent. It receives no new root. Its own child settlement must finish before DM recreation. Stop old DM only after unrelated work is absent and authorized link children settle; there is no maintenance worker to kill or lease to wait on. The prepared guard survives postgis outage too; never infer openness from a failed DB read. + +**DB decision and failure windows.** With root+DM prepared guard held, API rechecks current paired pin, source policy/overlay, credential row revision, full unchanged configuration versions, host source witness/current revision, candidate TTL and live `check/checkMany` (or08 domain proof checks). Final checks precede and repeat inside the existing one connection/transaction, including a fresh maintenance inspect with `preflight:"current"`; failed inspection is not current authority. Retain candidate/maintenance/prepare identity in `admin_job.result` before prepare POST. In the **one** desired-write transaction, consume proofs, commit ordinary+managed rows/credential revision/invalidation, and write `{candidateId,maintenanceId,prepareOperationKey,prepareFingerprint,planDigest,decisionId,decision:"continue",checkpointVersion,checkpointDigest,attempt:"not_attempted"}`. checkpointDigest is a domain-separated hash of this non-secret tuple **excluding checkpointDigest and attempt**, plus exact desired row-version receipts, never values; signer binds it to the existing job. No new DB row-save service or second participant. API then durably CAS-marks the decision attempted and POSTs once. Agent accepts only one decisionId/value for the prepared binding; a changed same-key body conflicts. It verifies the authenticated DB-decision attestation and its candidate/plan, stores `authorized`, and only then permits the frozen effects. No proof is consumed again during apply/restart. + +Known transaction rollback/no save permits only an explicit retained `abandon` decision for this exact unused reservation, under the same job/writer CAS. Agent verifies no continue decision, apply claim, effect attempt or child publication exists, then settles unused guard/root. A failed COMMIT response, lost prepare/decision POST, API death or unavailable checkpoint is **unknown**, keeps private candidate/target/DM/API-writer fences and is lookup-only. No TTL steal, automatic abort, optimistic release, proof replay or compensation. After API restart, reclaim the job under02 ownership CAS and inspect the original candidate/checkpoint on a transaction that locks the job row (thereby waiting out an earlier in-flight save transaction); exact committed checkpoint determines continue, exact known rolled-back/pre-save state plus no attempted decision/effect permits recording abandon. Absence/unavailable/ambiguous evidence alone never authorizes it. A retained decision marked not_attempted may be delivered once; any attempted decision/effect is only inspected. New desired state is never used to reconstruct the old decision. Expired unconsumed seals cannot be renewed; report saved/pending and retain or settle only from proven containment. + +**Settlement/recreation.** `configurationMaintenance.settle` is a verification request, not caller-supplied success authority. Agent reads the exact plan's generation commit, hardlink, recreate and helper receipts, checks all attempts/descendants terminal or contained, immutable root ownership and exact desired runtime configuration/resource/image/mount/readiness evidence (private keyed comparisons for secret mounts). For applied, appliedRevisionId must be this candidate and all required follow-up complete. A known failed follow-up can settle as contained only with no live/unknown effect and truthful saved/committed/failed evidence; it never reports applied. Before release, compute evidenceDigest from the exact retained non-secret child/commit identities, write `settling` with that digest, send the signed finite DM settle leaf carrying that exact owner receipt, and reconcile its DB/mirror settlement. Agent attests the verified private owner receipt in the signed settle grant's exact body (`decisionId`, `outcome`, `ownerReceiptDigest`); DM verifies this pinned-role attestation, its anchored root/settling phase and owned decision binding plus no active authorized link child. It never tries to read the agent-private directory or trusts an unsigned digest. DM commits settled under the same admission guard and fsyncs its mirror/tombstone; admissions reopen only when both agree and no unresolved owner evidence remains. Agent releases root only after exact DM settled receipt plus every child agree. Response loss between guard settlement and release retains the root until lookup proves settlement; fresh target-bound work still conflicts. Unknown generation/helper/DM settlement retains all fences.08's helper uses this same maintenance identity/guard/frozen subset and terminal-before-drain rule; it creates neither a second reservation nor a second root. + +Required acceptance is future work:01.2 real API/agent/DM preparation/DB transaction/recreation probes;02.3 real DB admission races/startup guards;05.6 direct feed and04.6 mixed-import activation races paused **after final verify and before DB commit**, asserting one exclusive valid commit or zero desired/proof/invalidation writes. Drop prepare/decision/settle POST responses; kill API before and after save; recreate DM and exercise startup cron/CLI/data denial before readiness. Assert one retained root, one decision, no nested acquisition/self-conflict, no proof double consumption, no repeated generation/pin transition, no release while any outcome is unknown, and precise applied versus contained/failed classification.08.3 adds protected helper/DB outage coverage using the same guard; no product execution is claimed here. + +## 6. Lifecycle receipts and coordinator policy + +Export portable `CoordinatorMode = "ensure-running" | "require-running"` from `ops/service-results.ts`. Make mode mandatory on preparation/hardlink typed operations, execution fingerprints and checkpoints. `prepareServiceGeneration(ctx:JobContext,scope:string,options:{coordinatorMode:CoordinatorMode}): Promise`; `ServicePreparationResult = {revisionId:string,hardlinks:HardlinkApplySummary,coordinatorMode:CoordinatorMode}`. Produce `publishServiceGeneration(...)` inside01.2 as the candidate publication stage and `applyGenerationHardlinks({revisionId,operationIdentity,coordinatorMode,signal?}): Promise` as the conditioned follow-up. The public wrapper retains all commit/follow-up evidence from `TrustedConfigurationApplyOutcome`; unknown cannot return an ordinary completed result. + +`ensure-running` is Start/Update's policy: derive the full hardlink plan, obtain required coordinator readiness with an exactly revision-checked, durably owned `service.start`, then run `configuration.hardlinks.apply` under target→writer with the signed leaf delegation. Never call Start from inside the link leaf or hold a host writer while awaiting another agent operation that requires it. CLI uses its existing local Docker adapter under its owned writer where possible, not a recursive agent request. + +`require-running` is Compose Render: before any generation publication, validate the **prospective** confined hardlink plan. Empty plan needs no data-manager and produces empty applied summary. Nonempty plan requires selected/verified-running data-manager plus readiness; stopped/unavailable is a prerequisite refusal before commit. Recheck under target/writer immediately before linking; coordinator exit after preflight/commit returns committed/follow-up-failed (or unknown if unobservable), with zero Start/restart/recreate/pull effects across the whole call graph. Link failure never activates ensure-running as fallback.01.2 adds API-role sync15s `configuration.hardlinks.preview` input `{candidateId,revisionId,expectedCurrentRevisionId,expectedSourceRevision,coordinatorMode}`. Agent reads only the fixed private candidate receipt, uses existing `inspectTrustedConfigurationSnapshot` HMAC verification plus embedded original typed apply-operation/body-fingerprint validation, then renders a plan in memory without committing/consuming the sealed apply claim. Output is `{entries:number,requiredServiceIds:string[],coordinatorState:"not_required"|"running"|"stopped"|"disabled"|"unavailable",previewDigest:string}` with≤256 entries/IDs and≤32KiB result. Browser cannot submit a plan. Candidate budget validation and private prepared publication precede this read-only preview; DB save/generation publication follow it. Bind preview digest+mode to the apply execution/receipt, revalidate under host target/writer before commit, and verify candidates do not accidentally authorize a different typed kind.02.2 registers `compose.render` for checkpoint/lookup-only recovery;09.6 consumes this mode. + +01.2's host helper signatures become: + +```ts +prepareServiceLifecycle(request: { + serviceIds: readonly string[]; expectedConfigurationRevisionId?: string; + coordinatorMode: CoordinatorMode; execution: ExecutionRef; + inheritedLease?: BorrowedArtifactTargetLease; signal?: AbortSignal; +}, effects: ServiceLifecycleEffects): Promise; +runServiceUpdate(request: { + serviceIds: readonly string[]; noDeps?: boolean; + expectedConfigurationRevisionId?: string; execution: ExecutionRef; + inheritedLease?: BorrowedArtifactTargetLease; signal?: AbortSignal; +}, effects: ServiceLifecycleEffects): Promise<{changed:true;revisionId:string}>; +``` + +Optional expected revision is **host-local only**: absence captures it once under writer ownership before effects. API agent calls require the explicit cfg1 revision. `ServiceLifecycleEffects` passes nominal `VerifiedLifecycleOwnership` containing the existing `ConfigurationGenerationWriter` and `TargetLeaseAuthority` to its fixed prepare adapter, as fully typed below. Inner `renderComposeWithWriter` never reacquires its own writer. Update always uses ensure-running and pins→successful pull→full preparation/hardlinks→recreate, with seed configuration publication separately disclosed before the CLI phase. + +The exact effect signature is `ServiceLifecycleEffects = {withOwnership(request:LifecycleOwnershipRequest,run:(ownership:VerifiedLifecycleOwnership)=>Promise):Promise;validatePins(ids:readonly string[]):Promise;pull(ids:readonly string[],signal?:AbortSignal):Promise;prepare(options:{ownership:VerifiedLifecycleOwnership;coordinatorMode:CoordinatorMode;signal?:AbortSignal}):Promise;recreate(ids:readonly string[],noDeps:boolean,signal?:AbortSignal):Promise}`. `LifecycleOwnershipRequest = {execution:ExecutionRef;serviceIds:readonly string[];expectedConfigurationRevisionId?:string;inheritedLease?:BorrowedArtifactTargetLease;signal?:AbortSignal}`. `VerifiedLifecycleOwnership` is nominal and contains readonly `{writer:ConfigurationGenerationWriter,lease:TargetLeaseAuthority}` plus private receipt ownership; produced only inside withOwnership and invalid after its callback. Its writer commits return the exact revision used to checkpoint this operation's receipt, not an independently read pointer. + +`packages/core/src/services/service-lifecycle-receipt.ts` exports: + +```ts +type ServiceLifecycleReceipt = { + version: 1; execution: ExecutionRef; serviceIds: string[]; + action: "start" | "update"; noDeps: boolean; coordinatorMode: CoordinatorMode; + expectedRevisionId: string | null; seedRevisionId: string | null; + finalRevisionId: string | null; + phase: "prepared" | "pins_validated" | "pulled" | "generation_committed" + | "hardlinks_applied" | "recreated" | "complete" | "failed" | "unknown"; + commit: "not_committed" | "seed_committed" | "final_committed" | "unknown"; + followUp: "not_started" | "complete" | "failed" | "unknown"; + hardlinks?: HardlinkApplySummary; + failure?: {reason:"pins"|"pull"|"revision_conflict"|"hardlinks"|"recreate" + |"deadline"|"canceled"|"containment"|"receipt_io"; + failedServiceIds:string[]}; + sequence: number; +}; +openServiceLifecycleReceipt(options: { dataRoot:string; execution:ExecutionRef; + serviceIds:readonly string[]; action:"start"|"update"; noDeps:boolean; + coordinatorMode:CoordinatorMode; expectedRevisionId:string|null; + seedRevisionId:string|null }): Promise; +// store.read(): Promise +// store.checkpoint(expectedSequence:number, receipt:ServiceLifecycleReceipt, +// ownership:VerifiedLifecycleOwnership): Promise +readServiceLifecycleReceipt(options: { dataRoot:string; + execution:ExecutionRef }): Promise; +``` + +Fixed path is `dataRoot/ops-agent/lifecycle/.json`,≤64KiB,0700 directory/0600 no-follow single-link file, fixed nonrecursive store and inode-CAS durability rules from§3. Root/receipt signatures and private high-water marks protect it from shared-data impersonation. A checkpoint verifies exact operation/fingerprint/action/targets/mode and live target+configuration writer ownership; only source helpers can construct `VerifiedLifecycleOwnership`. Seed receipt is initialized by the agent from **its checkpointed seed**, not a new current-pointer read. CLI checkpoints before/after each effect while holding ownership; config commit callback records its returned final revision atomically with its own receipt transition before follow-up. If failure occurs after commit but before receipt flush, preserve unknown with generation commit evidence for reconciliation, never classify not_committed. Terminal receipt validation enforces: success requires final revision, hardlinks settled, all targets recreated and followUp complete; failure after seed/final retains those revisions and commit state. Paths, raw stderr and secret material are forbidden. + +The default runner remains void and supplies only fixed identity/delegation environment. After it settles, agent reads this exact receipt, validates it and returns `{changed:true,revisionId:receipt.finalRevisionId}` only for complete. A competing current generation cannot replace that result. Next bulk target conditions on this returned final revision and conflicts if another writer advanced it. Missing/malformed/unfinished receipt is unknown/recovery-required;02 failure evidence adds safe `serviceLifecycle` projection with revisions/phase/commit/followUp/failure. Agent/CLI death reconciles exact receipt/children, no automatic rerun. Retention72h and no eviction of unsettled receipts. Root08 consumes commit evidence for protected follow-up, but its external helper has its own dedicated configuration-handoff receipt; do not label prepared handoff as completed lifecycle recreation. + +## 7. Required causal planned proof and consumers + +All are future tests; this design run executes none. + +- **06.1 / R1-01:** real separate CLI, agent and data-manager processes sharing one physical mount, including aliases and inverse target order. Show one owner and no self-reacquisition. Exercise both signature directions, wrong bearer role, substituted verifier/key, forged UUID, widened subset/kind, duplicate grant under a different identity, invalid phase, journal/ledger gap crashes and dead parent with live borrowed child. Assert no root release until every child settles; same-key lost response observes original admission. Include actual helper surviving agent death and refusing drain before its exact prepare terminal receipt. A process-only mock cannot establish these properties. +- **R2-01:**§4.1 and05.3 require the real DM→agent promotion→paired approval crash/response-loss schedules, exact pointer-attempt recovery, one root/transition and no release while unknown. +- **R2-02:**§5.4 and01.2/02.3/05.6/04.6/08.3 require whole-candidate-first preparation, real DB/admission race, exact retained decision, API death, DM startup/recreation fence and same-root helper settlement. +- **02.3 / R1-02:** authenticated loopback production agent promotion/reload; drop admission and cancel responses, crash/restart data-manager and agent, pause before/after local rename and proxy reload. Assert one retained child key, no blind rollback/second effect on unknown, cancel-pending while remote child lives, truthful committed result and a retained target fence. Cover recovery restart and one-off cron/API effects as well as primary promotion. +- **02.1/02.3 / R1-05:** fake clock admits a late child near parent expiry, restarts across expiry, attempts same-key budget extension and observes a short HTTP timeout. Parent bounds persist, extension conflicts, observer timeout leaves backend alive, execution expiry contains or remains pending;30s cannot create fabricated completion. +- **01.2 / R1-03/R1-10:** existing actual trusted snapshot assembly fails one service DB query while others succeed: no seal publication, saved row, enqueue or agent mutation. UTF-8/escaping feed maps,1001st/orphan key, full envelope at524288/524289 bytes, tree limits, multi-service migration and ordinary transfer values exercise prospective whole-state budgets before any persistence. An oversize legacy state can receive one complete valid reduction. Test direct service/import writes shadowed by namespaced or real manifest host resource knobs, host source drift between preview/commit, and successful explicit adoption after removing the override. Include two real DB connections entering global→integration locks in opposite caller order, nested credential/feed transfer on the proven same context, and crash fence recovery; no deadlock or lost unrelated value. +- **01.2 / R1-06:** default runner and real CLI registration, retained seed/final commit, failure after each commit and CLI crash. Commit a competing generation after CLI exit but before agent receipt read: agent returns its own final revision, next bulk target refuses newer current state. Do not use a fake void runner as receipt proof. +- **01.2/02.2/09.6 / R1-07:** API→production preparation→data-manager with require-running, stopped, concurrently exiting and empty-plan coordinator states. Assert zero container lifecycle calls including helper service; failure after commit retains commit/follow-up status; restart reuses exact mode/operation identity, never renders again. + +Cross-plan exports are mandatory dependencies, not deferred interfaces:04.1 enters the global writer before integration locks;04.2 consumes strict resolution and guards all direct writers;04.5/04.6 consume the full prospective budget/candidate and managed feed adapter;05.6 uses canonical map+same writer+credential revision;05.3 consumes durable children/deadlines/borrowed targets;06.1 implements§2's verifier integration and§3;06.3 uses the same lease and preparation mode;08.1 extends the finite platform resolver;08.3 consumes source-bound candidate and detached helper continuation;09.6 selects require-running and02.2 recovery. Root reconciles these consumers, the updated05.7→05.6→05.3 order and50-task total, then obtains a **new** independent review before product code. Existing migration allocation and single exclusive task windows remain binding. diff --git a/docs/superpowers/specs/2026-10-23-admin-instance-management-design.md b/docs/superpowers/specs/2026-10-23-admin-instance-management-design.md new file mode 100644 index 000000000..4dfd59b02 --- /dev/null +++ b/docs/superpowers/specs/2026-10-23-admin-instance-management-design.md @@ -0,0 +1,242 @@ +# Reliable administration and CLI coverage + +Date: 2026-10-23. Baseline: `ef896ceae876959180116cf61ffcf643b88ebffe`. +Branch: `feat/admin-instance-management-parity`. Delivery: one pull request. + +## Authority and intent + +The operator should manage an already installed OpenMapX instance through the +admin panel, including maintenance, configuration, credentials, datasets, +accounts, extensions, backups and privacy cases. The user authorized every +recommendation from the preceding CLI/admin audit, reliability before coverage, +autonomous design decisions, Codex-only delegation, local rendered verification, +and a single PR with passing CI. The user explicitly waived questions, design +approval pauses and PR screenshots. This specification records the controller's +decisions; it does not attribute those decisions to the user. + +The original audit's 510 passing focused tests and five skipped tests are +historical baseline evidence, not qualification of this branch. No product code +has been changed at specification creation. + +## Approaches considered and chosen + +1. Mirror CLI execution behind a generic command runner. Rejected: command, + path, secret and resource authority would escape the existing typed boundary; + accepted jobs would still lack effective-result guarantees. +2. Add isolated screens over current APIs. Rejected as the primary approach: + several current API/runtime paths are incomplete and would remain misleading. +3. Extend the existing typed operations, canonical contracts, generated SDKs, + configuration generations and durable jobs. Chosen: share policy and + preparation, make outcomes observable, then expose missing operator workflows. + +## Global constraints + +- All product work stays in this worktree; preserve unrelated work and existing + public-provider behavior. One branch/PR; no merge or production deployment. +- Use Codex provider instances only for delegated work. +- Retain `IntegrationContext`, HTTP/upstream acquisition, attribution and source + policy boundaries. Community integrations remain declarative; no uploaded code + execution and no browser developer toolchain. +- Docker, registry credentials, privileged host files and host data writes remain + behind typed ops-agent operations. No generic command, arbitrary path or raw + environment-file editor is introduced. +- Core API routes use `registerCoreRoutes`, runtime admin guards beside auth + declarations, strict schemas, audit events, explicit mutation identities and + appropriate rate limits. Privacy roles retain their narrower authority. +- Contracts precede generated outputs. Serialize OpenAPI/SDK generation; never + hand-edit generated files. Check actual response serialization. +- Preserve configuration writer ownership, revision CAS, admission fingerprints, + durable operation identities, retained results, unknown outcomes and cancellation + fences. No automatic mutation retries or unproven rollback/compensation. +- Use synthetic/public data in tests and UI verification. Never print secrets, + place secrets in argv or logs, or copy private operator data into artifacts. +- Stream large transfers with bounded memory, disk budgets, timeouts, cancellation + and cleanup. Opaque IDs cross trust boundaries; host paths do not. +- UI controls expose availability and reasons before submission. A successful + admission means queued, not applied. Failed/partial/unknown work stays visible. +- Inspect the actual rendered desktop/mobile UI locally. Do not upload screenshots + to the PR, as explicitly requested. +- Complete focused checks, full local verification, relevant disposable database + and runtime checks, independent review, production builds and PR CI. Report any + unavailable environment gate distinctly; do not replace runtime evidence with + mocked assertions. + +## Product and security decisions + +### Operation availability and lifecycle + +Return operation-specific capabilities and denial reasons from service inventory +and detail reads, derived from the same action policy the agent enforces. Start, +build and recreate require current configured authority. Narrow stop/read-log +authority remains available for known, owned running containers after deselection. +Never infer ownership from a caller-provided container name alone. Read-only logs +must not be denied merely because a service is part of the control plane. + +Ordinary Start and bulk Start share generation preparation and hardlink apply. +Updates validate pins, pull, prepare, then recreate; failed pull never recreates. +Safe API/edge-service maintenance uses durable dedicated workflows where a generic +action would cut its own coordinator. The executing agent cannot stop itself; +agent replacement retains the existing external helper/handoff protocol. Whole +stack shutdown and volume destruction remain explicit host instructions. + +Deselection must either complete owned deactivation before retiring action +authority or retain the narrowly bounded stop/log authority until cleanup. A +control-plane acquisition error produces unknown/unavailable status, never a +fabricated empty inventory. Deployment kind does not change during an outage. + +### Durable jobs and data execution + +Stream redacted output during execution. Limit retained output independently of +process lifetime; retain a truncation marker and bounded diagnostic tail. Deadlines +are operation-specific: diagnostics remain short; prepared builds/full refresh and +large dataset transfer permit multi-hour work with durable identity and explicit +cancellation. An observer disconnect does not imply backend completion or cancel a +durable data-manager job. Parent Full Update records and awaits its transit child +before dependent graph builds and successful completion. + +Continuing a batch after an item failure does not make the batch successful. Keep +completed/failed IDs and distinguish failed or partial terminal outcomes in all +SDK/UI consumers. A timed-out or lost result remains unknown until reconciled. + +### Account policy and provisioning + +Authentication settings are effective policy, not decorative database entries. +Validate password minimum 8-128 characters, session lifetime 1-8760 hours, and +boolean verification/2FA enrollment settings. New requests use a coherent policy +snapshot; policy changes must not extend existing sessions. Disabling new 2FA +enrollment never bypasses an enrolled user's second-factor challenge. Document +the policy application point and test sign-in/reset/admin-create paths. + +Upgrade behavior preserves the existing runtime defaults: verification and new +2FA enrollment remain enabled. Previously inert saved authentication values are +shown as pending until a full administrator explicitly applies a validated policy +with an activation marker. That apply atomically validates all four settings and +records policy version1; subsequent auth operations use that activated snapshot. +Explicit supported environment ownership is disclosed, with no silent weakening +from legacy database values. New installations use the same secure defaults. + +Unwrap every Better Auth read/mutation result before React Query success. Provide +an explicit invitation/verification handoff and audited administrator-assisted +verification/password recovery with reauthentication and confirmation. A password +reset is not proof of email ownership. Enforce the last non-banned full-admin +invariant atomically for remote role changes, banning and deletion, including +direct auth endpoints. Initial administrator bootstrap remains a local recovery +boundary. Use consistent role mutations and audit events from list/detail. + +### Configuration ownership and application + +Service/integration fields disclose default/database/vault/file/environment +ownership. File/environment owners cannot receive a misleading successful Save or +Rotate; provide explicit supported adoption into UI ownership or actionable host +instructions. Binding changes activate through reload with an honest apply result. + +Expand operator-facing service schemas from actual supported manifest settings, +including resource controls. Define platform/domain/TLS/release-selection forms +over bounded supported fields and coordinated configuration-generation apply. +Preview affected services, preserve last-good configuration, and provide recovery +guidance when origin/TLS changes disconnect the browser. Platform-owned bootstrap +keys and Redis authentication remain explicit host procedures; do not pretend a +live in-app form can safely rotate the authentication dependencies it requires. + +Migration has an inventory/preview describing included values, source ownership, +manual prerequisites and unresolved credentials. Non-secret JSON is not presented +as a full instance backup. Vault-owned credentials may travel only through an +explicit reauthenticated encrypted transfer with authenticated encryption and +bounded KDF parameters. Host-owned credentials remain unresolved prerequisites; +they are not silently read or disclosed from host files. + +### Transit and datasets + +Custom-source disabling is reversible and distinct from removal. Only show valid +actions for an origin/state. Server-side pagination/search covers every source; +discovery dedup preserves installed catalog IDs and provenance. Standalone sync +can select a deployment-authorized country subset and preview eligibility. + +Show active/proposed/validated pin state and a source/license diff before explicit +approval. Preserve paired Transitous/GBFS proposal/activation rather than silently +updating one member. Do not claim a proposal changed the active catalog. Do not +add an unsafe force-activation path around validation/promotion. + +Expose Overture resume and confirmed restart-from-scratch. Ordinary combined sync +remains the default; lower-level pull/ingest/extract become supported advanced +recovery actions. Inventory uses artifacts/build records, sizes, regions, +fingerprints and readiness, not directory existence. Feed-key management is +masked, write-only and tied to the active pinned catalog; regeneration preserves +existing credentials and never drops values from an unrelated upstream revision. + +### Portability and installed integrations + +Prepared-artifact transfers preserve existing checksums, compatibility checks, +stopped-consumer preconditions and atomic replacement. UI upload/export/import +uses bounded streaming, opaque staged IDs, digest verification, preflight capacity +and progress. Reuse existing archive validation; reject traversal, links, duplicate +entries, archive bombs, mismatched images and stale confirmations. Publish no +partially validated artifact. Handle interruption and stale stage cleanup. + +Standalone integration installs retain the source/artifact checksum and trust +restrictions of the CLI. Prefer artifact distribution and explicit adoption into +extension ownership; never claim a filesystem-only install belongs to an unrelated +store record. Provide guarded adoption/removal and reload/rollback semantics. + +### Maintenance and privacy + +Add manual backup pruning with preview and policy editing; retain scheduled +retention and restore safeguards. Cache glob/count/dry-run stays confined to +application prefixes; a web endpoint never runs FLUSHDB/FLUSHALL. Add service +presets, isolated update choice, MOTIS import, pull-only staging, release-pin +clearing and bounded tail/follow controls. + +Privacy case forms expose existing guarded identity/representative decisions, +assignment and task completion, deadlines, clarification/refusal, attachments, +backup review and reauthenticated assisted artifact delivery. Preserve expected +versions, idempotency, case-specific permissions, redaction, reauthentication and +human approval semantics. No UI shortcut fabricates evidence or bypasses readiness. + +## Scope ledger and plan owners + +Every row must acquire implementation and verification receipts before completion. +The implementation plans expand these IDs into test-first tasks and exact files. + +| IDs | Required outcome | Plan | +| ------- | ------------------------------------------------------------------------------- | ----------------------------- | +| S01-S03 | wired updates, shared Start preparation, truthful partial build results | 01 service control | +| S04-S06 | action capabilities, deselected stop/log access, safe core controls | 01 service control | +| S07-S08 | unavailable status and persistent troubleshooting navigation | 01 service control | +| J01-J03 | streaming bounded logs, deadlines, cancellation/reconciliation | 02 durable jobs | +| J04 | Full Update awaits transit child and preserves failure | 02 durable jobs | +| A01-A04 | effective verification/session/2FA/password policy | 03 account policy | +| A05-A08 | auth errors, invitations, verification and assisted recovery | 03 account policy | +| A09-A10 | atomic last-admin invariant and consistent role audit | 03 account policy | +| I01-I02 | binding activation and config/vault ownership | 04 integrations and migration | +| I03-I05 | standalone install/adoption/removal and migration inventory | 04 integrations and migration | +| I06 | explicit protected vault credential transfer | 04 integrations and migration | +| D01-D04 | reversible sources, pagination, discovery identity, scoped preflight | 05 transit and data | +| D05-D07 | reviewed paired pins, Overture recovery, real full inventory | 05 transit and data | +| D08-D09 | masked feed credentials and pinned non-destructive generation | 05 transit and data | +| T01-T04 | prepared-artifact upload/export/import, validation and cleanup | 06 artifact transfer | +| P01-P06 | complete guarded privacy-case operator workflows and delivery | 07 privacy workflows | +| C01-C04 | supported service/resource/platform/domain/TLS configuration | 08 platform configuration | +| C05-C07 | release selector, ownership and coordinated apply/recovery | 08 platform configuration | +| M01-M03 | presets, no-deps, MOTIS import and log controls | 09 maintenance controls | +| M04-M06 | pull-only/clear pin, backup prune/policy, cache glob/dry-run | 09 maintenance controls | +| M07 | honest Compose Down host guidance and recovery documentation | 09 maintenance controls | +| Q01-Q05 | contract generation, integrated tests, rendered UI, builds/full gates, green CI | master plan | + +## Execution and qualification + +All plans are reviewed before product implementation. The controller performs a +spec-to-task coverage scan, interface/file conflict scan, self-review and fresh +Codex builder review, resolving objections without user questions. Reliability +tasks land before their coverage consumers. Independent file owners may execute +in parallel; shared ops/contracts, generated SDKs, route indexes and migrations +are serialized by the controller. Each task has causal RED/GREEN evidence and a +fresh spec/quality review; a whole-branch review follows integration. + +Use disposable synthetic runtime/database fixtures. Exercise actual assembled +ops-runtime dispatch rather than only injected handler mocks; auth tests exercise +actual policy consumers; data tests observe child completion and activation; +privacy tests exercise guarded transitions and delivery. Run `pnpm verify:full`, +relevant database groups, affected production builds, policy/codegen freshness, +and actual rendered UI. Keep skipped/unavailable checks separate. Open/link the +single PR, monitor through T3's watcher, fix CI failures, and finish only with +the latest-head checks passing. Do not merge. From cef25c9e5a0e4e935bc0610a58e1c04fb7f5f61e Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 20:07:57 +0200 Subject: [PATCH 002/135] docs: record completed admin parity plan review --- .../plans/2026-10-23-admin-instance-management.md | 10 +++++----- docs/superpowers/plans/2026-10-23-admin-plan-review.md | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index 9184c56e3..e4b48afe9 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -36,11 +36,11 @@ controller records cross-plan shared-file/interface ownership before dispatch. ## Task Q0: Review and baseline -- [ ] Complete all nine subsystem plans with exact file ownership, interfaces and test-first steps. -- [ ] Self-review every scope ID against a task, record decisions and resolve shared-file conflicts. -- [ ] Obtain a fresh Codex builder/plan review and resolve every material objection. -- [ ] Install both frozen dependency sets and run the focused audit baseline in this worktree. -- [ ] Commit reviewed design/plans and record the starting implementation SHA in the execution ledger. +- [x] Complete all nine subsystem plans with exact file ownership, interfaces and test-first steps. +- [x] Self-review every scope ID against a task, record decisions and resolve shared-file conflicts. +- [x] Obtain a fresh Codex builder/plan review and resolve every material objection. +- [x] Install both frozen dependency sets and run the focused audit baseline in this worktree. +- [x] Commit reviewed design/plans and record the starting implementation SHA in the execution ledger. ## Task Q1: Integrated contract/runtime proof diff --git a/docs/superpowers/plans/2026-10-23-admin-plan-review.md b/docs/superpowers/plans/2026-10-23-admin-plan-review.md index 8fad69ef5..cbec1c9e4 100644 --- a/docs/superpowers/plans/2026-10-23-admin-plan-review.md +++ b/docs/superpowers/plans/2026-10-23-admin-plan-review.md @@ -111,3 +111,7 @@ Baseline migrated suites completed:api-privacy23files/129tests; ingestion13files Round2 cleared11/13 original findings and identified two unique Important seams: the delegated agent-owned paired-pin CAS and remote pre-save feed/configuration ownership. Corrected control-protocol§4.1/§5.4 and plans01/02/04/05/06/08 now define the finite publication leaf, exact attempt receipt, agent-owned terminal prepared maintenance root, durable DM admission guard, atomic DB continue decision and borrowed apply/recreation/settlement. No generic lock endpoint or new API mount is introduced. The existing50-task order and0042–0050 forward migrations remain. The correction worker hit a provider usage limit after writing the substantive protocol/report. Its files were recovered from exact historical document-write inputs after ignored files disappeared. The controller checked those recovery errors against the original tool receipts, regenerated50 tasks/774 pair rows, completed the explicit hardlinkPlanDigest fields and reformatted the documents. Prior product/database checks remain historical; no product change or runtime acceptance is claimed. A NEW independent round3 must verify both open findings and correction-caused material breakage before implementation. + +## Final review disposition + +Fresh independent round3 at cf01cf6d9 returned READY: R2-01 and R2-02 ADDRESSED,0Critical/0Important, no correction-caused material breakage. Together with round2’s11 cleared contracts, this resolves the full original plan review. The report is retained in the SDD workspace; it performed source/design review only. Product implementation begins at Task02.1; all product/runtime/full qualification remains ahead. From 8552f840da3a918ce5a40ccd6572c02a678345e3 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 20:32:09 +0200 Subject: [PATCH 003/135] fix(ops): stream bounded diagnostics without ending jobs --- .../src/services/__tests__/ops-client.test.ts | 171 ++++++-- apps/api/src/services/ops-client.ts | 134 +++++-- apps/ops-agent/src/__fixtures__/execution.ts | 6 + .../src/administrative-runtime.test.ts | 148 +++++++ apps/ops-agent/src/administrative-runtime.ts | 123 ++++-- apps/ops-agent/src/agent-update.test.ts | 2 + apps/ops-agent/src/docker-runtime.test.ts | 152 +++++++- apps/ops-agent/src/docker-runtime.ts | 170 ++++++-- apps/ops-agent/src/journal.test.ts | 66 +++- apps/ops-agent/src/journal.ts | 89 ++++- apps/ops-agent/src/process-output.test.ts | 98 +++++ apps/ops-agent/src/process-output.ts | 180 +++++++++ apps/ops-agent/src/runtime.test.ts | 3 + apps/ops-agent/src/runtime.ts | 14 + apps/ops-agent/src/server-foundation.test.ts | 20 +- apps/ops-agent/src/server.test.ts | 367 +++++++++++++++++- apps/ops-agent/src/server.ts | 317 +++++++++++++-- .../src/transitous-lock-runtime.test.ts | 2 + .../src/trusted-config-ownership.test.ts | 5 + .../src/trusted-config-runtime.test.ts | 26 ++ packages/core/package.json | 4 + packages/core/src/ops/client.test.ts | 18 +- packages/core/src/ops/client.ts | 101 ++++- packages/core/src/ops/contract.test.ts | 6 +- packages/core/src/ops/contract.ts | 69 +++- .../core/src/ops/execution-identity.test.ts | 100 +++++ packages/core/src/ops/execution-identity.ts | 238 ++++++++++++ packages/core/src/ops/fingerprint.ts | 2 +- packages/core/src/ops/index.ts | 3 + packages/core/src/ops/job-evidence.ts | 60 +++ packages/core/src/ops/service-results.ts | 59 +++ packages/core/src/scrub-secrets.ts | 179 +++++++++ .../data-manager/src/utils/scrub-secrets.ts | 180 +-------- 33 files changed, 2698 insertions(+), 414 deletions(-) create mode 100644 apps/ops-agent/src/__fixtures__/execution.ts create mode 100644 apps/ops-agent/src/process-output.test.ts create mode 100644 apps/ops-agent/src/process-output.ts create mode 100644 packages/core/src/ops/execution-identity.test.ts create mode 100644 packages/core/src/ops/execution-identity.ts create mode 100644 packages/core/src/ops/job-evidence.ts create mode 100644 packages/core/src/ops/service-results.ts create mode 100644 packages/core/src/scrub-secrets.ts diff --git a/apps/api/src/services/__tests__/ops-client.test.ts b/apps/api/src/services/__tests__/ops-client.test.ts index 3b7d0f4ad..2d8c2c89d 100644 --- a/apps/api/src/services/__tests__/ops-client.test.ts +++ b/apps/api/src/services/__tests__/ops-client.test.ts @@ -1,3 +1,4 @@ +import { getEventListeners } from "node:events"; import type { OpsEventBatch, OpsJobStatusFor, @@ -13,6 +14,7 @@ import { executeAndWait, executeTrustedConfigurationAndWait, followOpsEvents, + OpsObservationPending, waitForOpsResult, } from "../ops-client.js"; @@ -94,7 +96,7 @@ describe("bounded async ops helpers", () => { checkpoints.push(now); }, }), - ).rejects.toMatchObject({ errorClass: "timeout" }); + ).rejects.toMatchObject({ name: "OpsObservationPending" }); expect(checkpoints).toContain(630_000); expect(now).toBe(635_000); expect(client.execute).not.toHaveBeenCalled(); @@ -143,7 +145,7 @@ describe("bounded async ops helpers", () => { recoveryTimeoutMs: 5, }, ), - ).rejects.toMatchObject({ errorClass: "timeout" }); + ).rejects.toMatchObject({ name: "OpsObservationPending" }); expect(execute).not.toHaveBeenCalled(); expect(lookup).toHaveBeenCalled(); }); @@ -342,7 +344,10 @@ describe("bounded async ops helpers", () => { now += ms; }, }); - await expect(timedOut).rejects.toEqual(new ApiOpsError("timeout")); + await expect(timedOut).rejects.toMatchObject({ + name: "OpsObservationPending", + reason: "observer_deadline", + }); }); it("rejects a status bound to a different operation key", async () => { @@ -374,8 +379,6 @@ describe("bounded async ops helpers", () => { it("removes the sleep abort listener after every normal poll", async () => { const controller = new AbortController(); - const add = vi.spyOn(controller.signal, "addEventListener"); - const remove = vi.spyOn(controller.signal, "removeEventListener"); const admission = { execution: "async" as const, operationId: "job1_abcdefghijklmnop", @@ -383,36 +386,35 @@ describe("bounded async ops helpers", () => { kind: "service.stop" as const, state: "queued" as const, }; - const status = vi - .fn() - .mockResolvedValueOnce({ - version: 1, - operationId: admission.operationId, - operationKey: admission.operationKey, - kind: admission.kind, - resourceId: "redis", - state: "running", - submittedAt: "2026-08-23T10:00:00.000Z", - updatedAt: "2026-08-23T10:00:01.000Z", - }) - .mockResolvedValueOnce({ - version: 1, - operationId: admission.operationId, - operationKey: admission.operationKey, - kind: admission.kind, - resourceId: "redis", - state: "succeeded", - submittedAt: "2026-08-23T10:00:00.000Z", - updatedAt: "2026-08-23T10:00:02.000Z", - result: { changed: true }, - }); - await waitForOpsResult(fakeClient({ status }), admission.kind, admission, { - signal: controller.signal, - pollIntervalMs: 1, - timeoutMs: 100, + let effectiveSignal: AbortSignal | undefined; + let reads = 0; + const client = fakeClient({ + status: async (_kind: unknown, _id: unknown, options: { signal: AbortSignal }) => { + effectiveSignal = options.signal; + reads++; + return { + version: 1, + operationId: admission.operationId, + operationKey: admission.operationKey, + kind: admission.kind, + resourceId: "redis", + state: reads === 1 ? "running" : "succeeded", + submittedAt: "2026-08-23T10:00:00.000Z", + updatedAt: "2026-08-23T10:00:01.000Z", + ...(reads === 1 ? {} : { result: { changed: true } }), + }; + }, }); - expect(add).toHaveBeenCalledTimes(1); - expect(remove).toHaveBeenCalledTimes(1); + await expect( + waitForOpsResult(client, admission.kind, admission, { + signal: controller.signal, + pollIntervalMs: 1, + timeoutMs: 100, + }), + ).resolves.toEqual({ changed: true }); + if (!effectiveSignal) throw new Error("Status did not receive the observer signal"); + expect(getEventListeners(effectiveSignal, "abort")).toHaveLength(0); + expect(reads).toBe(2); }); it("streams bounded event pages and stops promptly on caller cancellation", async () => { @@ -626,3 +628,104 @@ describe("configuration outcome evidence", () => { ).toEqual({ outcome: "unknown", reason: "transport_or_result" }); }); }); + +describe("independent observation deadlines", () => { + it("observes beyond thirty minutes without extending or canceling backend work", async () => { + let now = 0; + const admission = { + execution: "async" as const, + kind: "service.build" as const, + operationId: "job1_abcdefghijklmnop", + operationKey: "opk1_abcdefghijklmnop", + state: "running" as const, + admittedAt: 0, + deadlineAt: 12 * 3600_000, + fingerprint: "a".repeat(64), + }; + let polls = 0; + const client = fakeClient({ + status: async () => ({ + version: 2, + ...admission, + resourceId: "valhalla", + submittedAt: new Date(0).toISOString(), + updatedAt: new Date(now).toISOString(), + state: ++polls === 1 ? "running" : "succeeded", + ...(polls === 1 ? {} : { result: { changed: true } }), + }), + }); + await expect( + waitForOpsResult(client, admission.kind, admission, { + now: () => now, + sleep: async () => { + now = 31 * 60_000; + }, + }), + ).resolves.toEqual({ changed: true }); + expect(client.cancel).not.toHaveBeenCalled(); + expect(client.execute).not.toHaveBeenCalled(); + }); + it("allows a separately selected observer beyond forty-eight hours and reports loss as pending", async () => { + const controller = new AbortController(); + const admission = { + execution: "async" as const, + kind: "system.update" as const, + operationId: "job1_abcdefghijklmnop", + operationKey: "opk1_abcdefghijklmnop", + state: "running" as const, + }; + const client = fakeClient({ + status: async () => { + controller.abort(); + throw new DOMException("Aborted", "AbortError"); + }, + }); + await expect( + waitForOpsResult(client, admission.kind, admission, { + signal: controller.signal, + timeoutMs: 72 * 3600_000, + }), + ).rejects.toBeInstanceOf(OpsObservationPending); + expect(client.cancel).not.toHaveBeenCalled(); + }); +}); + +it("bounds an inflight status read by the observer deadline without canceling execution", async () => { + const admission = { + execution: "async" as const, + kind: "service.build" as const, + operationId: "job1_abcdefghijklmnop", + operationKey: "opk1_abcdefghijklmnop", + state: "running" as const, + }; + const client = fakeClient({ + status: async (_kind: unknown, _id: unknown, options: { signal: AbortSignal }) => + new Promise((resolve, reject) => { + const timer = setTimeout( + () => + resolve({ + version: 1, + ...admission, + resourceId: "valhalla", + state: "succeeded", + result: { completed: true }, + submittedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }), + 200, + ); + options.signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }, + { once: true }, + ); + }), + }); + await expect( + waitForOpsResult(client, admission.kind, admission, { observerDeadlineAt: Date.now() + 20 }), + ).rejects.toMatchObject({ name: "OpsObservationPending", reason: "observer_deadline" }); + expect(client.cancel).not.toHaveBeenCalled(); +}); diff --git a/apps/api/src/services/ops-client.ts b/apps/api/src/services/ops-client.ts index 42b93e450..81354f2e5 100644 --- a/apps/api/src/services/ops-client.ts +++ b/apps/api/src/services/ops-client.ts @@ -8,11 +8,13 @@ import { OPS_KIND_POLICIES, OPS_MAX_EVENT_BATCH, OPS_MAX_EVENT_MESSAGE_BYTES, + OPS_MAX_EXECUTION_MS, OPS_PUBLIC_ERROR_MESSAGES, OpsClient, OpsClientError, type OpsErrorClass, type OpsEventBatch, + type OpsFailureResult, type OpsJobStatusFor, type OpsOperation, type OpsOperationKind, @@ -29,7 +31,7 @@ import { } from "@openmapx/core/privacy"; const DEFAULT_POLL_INTERVAL_MS = 250; -const MAX_WAIT_TIMEOUT_MS = 30 * 60_000 + 5_000; +const MAX_WAIT_TIMEOUT_MS = OPS_MAX_EXECUTION_MS + 30_000; const MAX_STREAM_EVENTS = 2_000; const MAX_STREAM_BYTES = 1024 * 1024; const MAX_TRANSIENT_STATUS_ATTEMPTS = 3; @@ -38,7 +40,10 @@ const AMBIGUOUS_ADMISSION_RECOVERY_MS = 2_000; export type ApiOpsClient = Pick; export class ApiOpsError extends Error { - constructor(readonly errorClass: OpsErrorClass) { + constructor( + readonly errorClass: OpsErrorClass, + readonly failureResult?: OpsFailureResult, + ) { super( errorClass === "host_update_required" ? OPS_PUBLIC_ERROR_MESSAGES.host_update_required @@ -48,6 +53,17 @@ export class ApiOpsError extends Error { } } +/** Observation ended without authoritative settlement. Only status/lookup may be retried. */ +export class OpsObservationPending extends Error { + constructor( + readonly identity: { kind: OpsOperationKind; operationKey: string; operationId?: string }, + readonly reason: "transport" | "observer_deadline" | "recovery_required", + ) { + super("Operations observation is pending"); + this.name = "OpsObservationPending"; + } +} + export class DawarichExportError extends Error { constructor( readonly code: string, @@ -293,6 +309,9 @@ async function defaultSleep(ms: number, signal?: AbortSignal): Promise { export interface WaitForOpsResultOptions { signal?: AbortSignal; timeoutMs?: number; + observerDeadlineAt?: number; + execution?: { requestedDeadlineAt: number }; + parentAssertion?: string; pollIntervalMs?: number; now?: () => number; sleep?: (ms: number, signal?: AbortSignal) => Promise; @@ -305,41 +324,76 @@ export async function waitForOpsResult( options: WaitForOpsResultOptions = {}, ): Promise> { if (admission.kind !== kind) throw new ApiOpsError("runtime"); - const timeoutMs = - options.timeoutMs ?? Math.min(OPS_KIND_POLICIES[kind].timeoutMs + 5_000, MAX_WAIT_TIMEOUT_MS); + const timeoutMs = options.timeoutMs ?? OPS_KIND_POLICIES[kind].timeoutMs + 5_000; const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; validatePositiveBounded(timeoutMs, MAX_WAIT_TIMEOUT_MS, "operations wait timeout"); validatePositiveBounded(pollIntervalMs, 10_000, "operations poll interval"); const now = options.now ?? Date.now; const sleep = options.sleep ?? defaultSleep; - const deadline = now() + timeoutMs; + const deadline = options.observerDeadlineAt ?? now() + timeoutMs; + if (!Number.isSafeInteger(deadline) || deadline < 0 || deadline - now() > MAX_WAIT_TIMEOUT_MS) + throw new Error("Invalid observer deadline"); + const identity = { + kind, + operationKey: admission.operationKey, + operationId: admission.operationId, + }; + const pending = (reason: "transport" | "observer_deadline" | "recovery_required"): never => { + throw new OpsObservationPending(identity, reason); + }; + const observer = new AbortController(); + const observerTimer = setTimeout(() => observer.abort(), Math.max(1, deadline - now())); + observerTimer.unref?.(); + const signal = options.signal + ? AbortSignal.any([options.signal, observer.signal]) + : observer.signal; + const observerExpired = () => signal.aborted || now() >= deadline; + const observeSleep = async (ms: number) => { + try { + await sleep(ms, signal); + } catch { + pending(observerExpired() ? "observer_deadline" : "transport"); + } + }; let transientStatusAttempts = 0; - while (true) { - throwIfAborted(options.signal); - if (now() >= deadline) throw new ApiOpsError("timeout"); - let status: OpsJobStatusFor; - try { - status = await client.status(kind, admission.operationId, { signal: options.signal }); - transientStatusAttempts = 0; - } catch (error) { - if (isTransientTransportError(error) && !options.signal?.aborted) { - transientStatusAttempts += 1; - if (transientStatusAttempts < MAX_TRANSIENT_STATUS_ATTEMPTS && now() < deadline) { - await sleep(Math.min(pollIntervalMs, Math.max(1, deadline - now())), options.signal); - continue; + try { + while (true) { + if (observerExpired()) pending("observer_deadline"); + let status: OpsJobStatusFor; + try { + status = await client.status(kind, admission.operationId, { signal }); + transientStatusAttempts = 0; + } catch (error) { + if (isTransientTransportError(error) && !observerExpired()) { + transientStatusAttempts += 1; + if (transientStatusAttempts < MAX_TRANSIENT_STATUS_ATTEMPTS && now() < deadline) { + await observeSleep(Math.min(pollIntervalMs, Math.max(1, deadline - now()))); + continue; + } } + if (isTransientTransportError(error)) + pending(observerExpired() ? "observer_deadline" : "transport"); + translateError(error); } - translateError(error); - } - if (status.operationKey !== admission.operationKey) throw new ApiOpsError("runtime"); - if (status.state === "succeeded") return status.result; - if (status.state === "failed" || status.state === "timed_out") { - throw new ApiOpsError(status.errorClass); + if (observerExpired()) pending("observer_deadline"); + if ( + status.operationKey !== admission.operationKey || + (admission.fingerprint !== undefined && status.fingerprint !== admission.fingerprint) || + (admission.deadlineAt !== undefined && status.deadlineAt !== admission.deadlineAt) + ) + throw new ApiOpsError("runtime"); + if (status.state === "succeeded") return status.result; + if (status.state === "failed" || status.state === "timed_out") { + if (status.errorClass === "recovery_required") pending("recovery_required"); + throw new ApiOpsError(status.errorClass, status.failureResult); + } + const remaining = deadline - now(); + if (remaining <= 0) pending("observer_deadline"); + await observeSleep(Math.min(pollIntervalMs, remaining)); } - const remaining = deadline - now(); - if (remaining <= 0) throw new ApiOpsError("timeout"); - await sleep(Math.min(pollIntervalMs, remaining), options.signal); + } finally { + clearTimeout(observerTimer); } } @@ -480,8 +534,9 @@ export async function executeAndWait( try { const submitted = await submitOpsOperationWithRecovery(client, operation, operationKey, { signal: options.signal, + execution: options.execution, + parentAssertion: options.parentAssertion, }); - throwIfAborted(options.signal); if (submitted.execution === "sync") return submitted.value; return await waitForOpsResult(client, operation.kind, submitted, options); } catch (error) { @@ -497,13 +552,20 @@ export async function submitOpsOperationWithRecovery signal?: AbortSignal; admissionAlreadyAttempted?: boolean; recoveryTimeoutMs?: number; + execution?: { requestedDeadlineAt: number }; + parentAssertion?: string; } = {}, ): Promise> { let firstError: unknown; if (!options.admissionAlreadyAttempted) { throwIfAborted(options.signal); try { - return await client.execute(operation, { operationKey, signal: options.signal }); + return await client.execute(operation, { + operationKey, + signal: options.signal, + execution: options.execution, + parentAssertion: options.parentAssertion, + }); } catch (error) { firstError = error; } @@ -529,24 +591,34 @@ export async function submitOpsOperationWithRecovery try { const status = await client.lookup(operation, operationKey, { signal: recoverySignal, + execution: options.execution, }); return { execution: "async", + version: status.version, operationId: status.operationId, operationKey: status.operationKey, kind: operation.kind, state: status.state, + ...(status.version === 2 + ? { + admittedAt: status.admittedAt, + deadlineAt: status.deadlineAt, + fingerprint: status.fingerprint, + } + : {}), } as OpsSubmitResult; } catch (lookupError) { const lookupClass = errorClass(lookupError); if (lookupClass !== "not_found" && !isTransientTransportError(lookupError)) { translateError(lookupError); } - if (recoverySignal.aborted) throw new ApiOpsError("timeout"); + if (recoverySignal.aborted) + throw new OpsObservationPending({ kind: operation.kind, operationKey }, "transport"); try { await defaultSleep(250, recoverySignal); } catch { - throw new ApiOpsError("timeout"); + throw new OpsObservationPending({ kind: operation.kind, operationKey }, "transport"); } } } diff --git a/apps/ops-agent/src/__fixtures__/execution.ts b/apps/ops-agent/src/__fixtures__/execution.ts new file mode 100644 index 000000000..aa3495023 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/execution.ts @@ -0,0 +1,6 @@ +import type { ExecutionRef } from "@openmapx/core/ops"; + +/** Synthetic context for adapter tests; the real server supplies the retained admission. */ +export function testExecution(): ExecutionRef { + return { executor: "ops-agent", principal: "api", executionId: "job1_testabcdefghijkl", operationKey: "opk1_testabcdefghijkl", operationFingerprint: "a".repeat(64), fingerprint: "b".repeat(64), deadlineAt: Date.now() + 3600_000 }; +} diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index d35326913..fa3746e43 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -11,8 +11,10 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { OpsOperation } from "@openmapx/core/ops"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; import { createAdministrativeRuntime, + createDefaultFixedCli, createDefaultReleaseEffects, inspectBackupAuthority, inspectDataTypeAuthority, @@ -44,6 +46,7 @@ function temporaryRoot(): string { function context(kind: string) { return { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: { @@ -60,6 +63,48 @@ afterEach(async () => { await Promise.all(temporaryRoots.splice(0).map((root) => rm(root, { recursive: true }))); }); +describe("default fixed CLI live output", () => { + it.each(["visibility", "volume"])("streams live bounded output: %s", async (check) => { + const root = temporaryRoot(); + const release = join(root, "release"); + const ready = join(root, "ready"); + writeFileSync( + join(root, "cli.mjs"), + ` + import { existsSync, writeFileSync } from 'node:fs'; + import { setTimeout } from 'node:timers/promises'; + console.log('phase-one'); + writeFileSync(${JSON.stringify(ready)}, 'ready'); + while (!existsSync(${JSON.stringify(release)})) await setTimeout(5); + for (let i = 0; i < 2048; i++) { + if (!process.stdout.write('x'.repeat(1024) + '\\n')) + await new Promise(resolve => process.stdout.once('drain', resolve)); + } + console.log('last-diagnostic'); + `, + ); + const seen: string[] = []; + const completion = createDefaultFixedCli(root, join(root, "agent.mjs"))([], { + signal: new AbortController().signal, + emitLog: (_stream, message) => seen.push(message), + }); + // Attach immediately: the volume regression may reject before the assertion. + void completion.catch(() => undefined); + try { + await vi.waitFor(() => expect(existsSync(ready)).toBe(true), { timeout: 2000 }); + if (check === "visibility") { + await vi.waitFor(() => expect(seen).toContain("phase-one"), { timeout: 2000 }); + } + writeFileSync(release, "go"); + await expect(completion).resolves.toBeUndefined(); + expect(seen).toContain("last-diagnostic"); + } finally { + writeFileSync(release, "go"); + await completion.catch(() => undefined); + } + }); +}); + describe("administrative backup runtime", () => { it("runs scheduled retention through the fixed CLI under validated bounds", async () => { const calls: string[][] = []; @@ -1479,3 +1524,106 @@ describe("administrative release runtime", () => { expect(apply).not.toHaveBeenCalled(); }); }); + +describe("fixed CLI execution context", () => { + it("uses the original long deadline and clears inherited control authority", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const root = temporaryRoot(); + const ready = join(root, "ready"); + const release = join(root, "release"); + const execution = testExecution(); + execution.deadlineAt = Date.now() + 12 * 3600_000; + writeFileSync( + join(root, "cli.mjs"), + `import{writeFileSync,existsSync}from'node:fs';import{setTimeout}from'node:timers/promises';if(process.env.OPENMAPX_CONTROL_DELEGATION||process.env.OPENMAPX_PARENT_EXECUTION)process.exit(2);const execution=JSON.parse(process.env.OPENMAPX_CONTROL_EXECUTION);writeFileSync(${JSON.stringify(ready)},JSON.stringify(execution));while(!existsSync(${JSON.stringify(release)}))await setTimeout(5);console.log('completed');`, + ); + vi.stubEnv("OPENMAPX_CONTROL_DELEGATION", "untrusted-inherited"); + vi.stubEnv("OPENMAPX_PARENT_EXECUTION", "untrusted-parent"); + let settled = false; + const completion = createDefaultFixedCli(root, join(root, "agent.mjs"))([], { + execution, + operationKind: "service.build", + signal: new AbortController().signal, + emitLog: () => undefined, + }); + void completion + .finally(() => { + settled = true; + }) + .catch(() => undefined); + try { + await vi.waitFor(() => expect(existsSync(ready)).toBe(true), { timeout: 2000 }); + expect(JSON.parse(readFileSync(ready, "utf8"))).toEqual(execution); + await vi.advanceTimersByTimeAsync(31 * 60_000); + expect(settled).toBe(false); + writeFileSync(release, "go"); + await expect(completion).resolves.toBeUndefined(); + await expect( + createDefaultFixedCli(root, join(root, "agent.mjs"))([], { + execution: { ...execution, deadlineAt: Date.now() - 1 }, + signal: new AbortController().signal, + emitLog: () => undefined, + }), + ).rejects.toMatchObject({ errorClass: "timeout" }); + } finally { + writeFileSync(release, "go"); + vi.useRealTimers(); + vi.unstubAllEnvs(); + await completion.catch(() => undefined); + } + }); +}); + +it("never gives release recovery a fresh execution budget after restart expiry", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const rootDir = temporaryRoot(); + const releaseId = "release-expired-recovery"; + const directory = join(rootDir, "infra", "docker", ".ops-agent-releases"); + mkdirSync(directory, { mode: 0o700 }); + writeFileSync( + join(directory, `${releaseId}.json`), + JSON.stringify({ + schemaVersion: 1, + release: releaseId, + images: Object.fromEntries( + [ + "api", + "web", + "data-manager", + "ops-agent", + "privacy-backup", + "transitous-runner", + "transitous-tools", + ].map((name) => [name, `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`]), + ), + privacyReleaseValidation, + }), + { mode: 0o600 }, + ); + const execution = { ...testExecution(), deadlineAt: Date.now() + 1000 }; + const runFixedCli = vi.fn(async () => undefined); + try { + const first = createDefaultReleaseEffects(rootDir, runFixedCli, { + afterReleasePhase: (phase) => { + if (phase === "overlay_written") throw Error("crash"); + }, + }); + await expect( + first.apply(releaseId, ["app-api"], { + signal: new AbortController().signal, + emitLog: () => undefined, + execution, + }), + ).rejects.toThrow("crash"); + vi.setSystemTime(Date.now() + 2000); + const recovered = createDefaultReleaseEffects(rootDir, runFixedCli); + await expect(recovered.initialize?.()).rejects.toMatchObject({ + name: "OpsExecutionFailure", + errorClass: "timeout", + }); + expect(runFixedCli).not.toHaveBeenCalled(); + expect(existsSync(join(directory, "transaction.json"))).toBe(true); + } finally { + vi.useRealTimers(); + } +}); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 5bda1ac5a..ec803cb30 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -18,9 +18,12 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { + type ExecutionRef, + executionRefSchema, OPS_MAX_BACKUP_ID_LENGTH, OPS_MAX_BACKUP_INVENTORY_ENTRIES, OPS_PUBLIC_ERROR_MESSAGES, + type OpsOperationKind, type OpsResultFor, } from "@openmapx/core/ops"; import { atomicWrite, durableUnlink, fsyncDirectory } from "@openmapx/core/release-files"; @@ -40,12 +43,14 @@ import { listDescriptorAnchoredDirectory, readDescriptorAnchoredUtf8, } from "./descriptor-file"; -import { runContainedProcess } from "./docker-runtime"; +import { runContainedProcess, runContainedStreamingProcess } from "./docker-runtime"; import { strictReadFile } from "./durable-files"; +import { safeProcessDiagnostic } from "./process-output"; import { createImageInspection } from "./release-images"; import { OpsAgentBusyError, type OpsExecutionContext, + OpsExecutionFailure, OpsHostUpdateRequiredError, type OpsRuntime, } from "./runtime"; @@ -94,7 +99,11 @@ const DATA_TYPE_ALIASES: Readonly> = { type BackupKind = "backup.create" | "backup.restore" | "backup.delete"; type BackupInventoryEntry = OpsResultFor<"backup.list">["backups"][number]; -interface FixedCliOptions { +export interface FixedCliOptions { + operationKind?: OpsOperationKind; + execution?: ExecutionRef; + controlAssertion?: string; + parentAssertion?: string; signal: AbortSignal; emitLog(stream: "stdout" | "stderr", message: string): void; } @@ -656,7 +665,7 @@ function emitLines( contents: string, ): void { for (const line of contents.split(/\r?\n/).filter(Boolean)) { - emitLog(stream, line); + emitLog(stream, safeProcessDiagnostic(line)); } } @@ -922,7 +931,7 @@ export function createDefaultReleaseEffects( (async (args: readonly string[], context: FixedCliOptions) => { const result = await runContainedProcess("docker", args, { signal: context.signal, - timeout: MAX_CLI_DURATION_MS, + timeout: remainingCliTime(context), maxBuffer: MAX_CLI_OUTPUT_BYTES, }); emitLines(context.emitLog, "stdout", result.stdout); @@ -1071,6 +1080,7 @@ export function createDefaultReleaseEffects( digest: string; serviceIds: string[]; updateJobId?: string; + execution?: ExecutionRef; previousOverlay: string | null; previousState: string | null; } @@ -1127,6 +1137,7 @@ export function createDefaultReleaseEffects( ) { throw new Error("Release transaction rejected"); } + if (value.execution !== undefined) executionRefSchema.parse(value.execution); const transaction = value as unknown as ReleaseTransaction; const manifest = loadManifest(transaction.releaseId); if (releaseDigest(coreServices.canonicalReleaseManifest(manifest)) !== transaction.digest) { @@ -1184,6 +1195,8 @@ export function createDefaultReleaseEffects( transaction: ReleaseTransaction, context: FixedCliOptions, ): Promise => { + if (transaction.execution) context = { ...context, execution: transaction.execution }; + remainingCliTime(context); await writeTransaction({ ...transaction, phase: "rollback_overlay" }); if (transaction.previousOverlay === null) durableUnlink(paths.overlay); else atomicWrite(paths.overlay, transaction.previousOverlay); @@ -1197,6 +1210,8 @@ export function createDefaultReleaseEffects( transaction: ReleaseTransaction, context: FixedCliOptions, ): Promise => { + if (transaction.execution) context = { ...context, execution: transaction.execution }; + remainingCliTime(context); const manifest = loadManifest(transaction.releaseId); // The transaction's digest was taken from the exact bytes authorized when // the apply was admitted. Reopening by release ID would otherwise let a @@ -1388,7 +1403,14 @@ export function createDefaultReleaseEffects( ] as const; for (const [index, name] of names.entries()) { context.emitLog("stdout", `Downloading ${name} image (${index + 1}/${names.length})`); - await runDocker(["pull", manifest.images[name]], context); + if (runtimeOptions.runDocker) await runDocker(["pull", manifest.images[name]], context); + else + await runContainedStreamingProcess("docker", ["pull", manifest.images[name]], { + signal: context.signal, + timeout: remainingCliTime(context), + onLine: context.emitLog, + redact: safeProcessDiagnostic, + }); } }, inspect: async () => { @@ -1542,6 +1564,7 @@ export function createDefaultReleaseEffects( ...(updateJobId ? { updateJobId } : {}), previousOverlay, previousState, + ...(context.execution ? { execution: context.execution } : {}), }; await writeTransaction(transaction); if (afterHandoff) { @@ -1578,7 +1601,7 @@ export function createDefaultReleaseEffects( await reconcileHandoff(); await validateCandidate(request.releaseId, request.expectedDigest); const manifest = loadManifest(request.releaseId); - const silent = { signal: context.signal, emitLog: () => undefined }; + const silent = { ...context, emitLog: () => undefined }; const existing = readAgentHandoff(rootDir); if (existing?.handoffId === request.handoffId) { return { handoffId: request.handoffId, replaced: true }; @@ -1648,6 +1671,18 @@ export function createDefaultReleaseEffects( * `cli.mjs` sits next to the agent's entry; in development the agent runs from * `src/` and the CLI runs from source under tsx. */ +function remainingCliTime(options: FixedCliOptions): number { + if (options.signal.aborted) throw new OpsExecutionFailure("runtime", {}); + if (!options.execution) { + if (options.operationKind) throw new Error("Managed execution context is required"); + return MAX_CLI_DURATION_MS; + } + const execution = executionRefSchema.parse(options.execution); + const remaining = execution.deadlineAt - Date.now(); + if (remaining <= 0) throw new OpsExecutionFailure("timeout", {}); + return remaining; +} + export function createDefaultFixedCli( _rootDir: string, entry: string = process.argv[1] ?? "", @@ -1664,13 +1699,29 @@ export function createDefaultFixedCli( join(runtimeRoot, "packages", "cli", "src", "index.ts"), ]; return async (args, options) => { - const result = await runContainedProcess(executable as string, [...prefix, ...args], { + const timeout = remainingCliTime(options); + const env = { ...process.env }; + for (const key of [ + "OPENMAPX_CONTROL_EXECUTION", + "OPENMAPX_CONTROL_DELEGATION", + "OPENMAPX_PARENT_EXECUTION", + ]) + delete env[key]; + if (options.execution) { + const encoded = JSON.stringify(executionRefSchema.parse(options.execution)); + if (Buffer.byteLength(encoded) > 2048) throw new Error("Execution context exceeds budget"); + env.OPENMAPX_CONTROL_EXECUTION = encoded; + if (options.controlAssertion || options.parentAssertion) + throw new Error("Fixed CLI control verifier is unavailable"); + } else if (options.controlAssertion || options.parentAssertion) + throw new Error("Execution context is required"); + await runContainedStreamingProcess(executable as string, [...prefix, ...args], { + env, signal: options.signal, - timeout: MAX_CLI_DURATION_MS, - maxBuffer: MAX_CLI_OUTPUT_BYTES, + timeout, + onLine: options.emitLog, + redact: safeProcessDiagnostic, }); - emitLines(options.emitLog, "stdout", result.stdout); - emitLines(options.emitLog, "stderr", result.stderr); }; } @@ -1736,8 +1787,8 @@ export function createAdministrativeRuntime( const lock = await acquireBackupStoreLock(options.rootDir); try { await runFixedCli(["backup", "create", "--name", operation.backupId], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); } finally { lock.release(); @@ -1767,7 +1818,7 @@ export function createAdministrativeRuntime( ...(operation.serviceIds?.length ? ["--services", ...operation.serviceIds] : []), ...(operation.stopRunning ? ["--stop-running"] : []), ], - { signal: context.signal, emitLog: context.emitLog }, + { ...context, operationKind: context.claim.operation.kind }, ); if (inspectBackupManifestDigest(options.rootDir, operation.backupId) !== digest) { throw new Error("Backup authority rejected"); @@ -1781,8 +1832,8 @@ export function createAdministrativeRuntime( const lock = await acquireBackupStoreLock(options.rootDir); try { await runFixedCli(["backup", "delete", operation.backupId], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); } finally { lock.release(); @@ -1793,14 +1844,14 @@ export function createAdministrativeRuntime( runtime["data.downloadOsm"] = async (operation, context) => { await runFixedCli( ["data", "download", "osm", ...(operation.regionId ? [operation.regionId] : [])], - { signal: context.signal, emitLog: context.emitLog }, + { ...context, operationKind: context.claim.operation.kind }, ); return { completed: true, ...(operation.regionId ? { resourceId: operation.regionId } : {}) }; }; runtime["data.downloadFonts"] = async (_operation, context) => { await runFixedCli(["data", "download", "fonts"], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { completed: true }; }; @@ -1815,42 +1866,42 @@ export function createAdministrativeRuntime( : []), ...(operation.failFast ? ["--fail-fast"] : []), ], - { signal: context.signal, emitLog: context.emitLog }, + { ...context, operationKind: context.claim.operation.kind }, ); return { completed: true, ...(operation.regionId ? { resourceId: operation.regionId } : {}) }; }; runtime["data.convertOverpass"] = async (operation, context) => { await runFixedCli( ["data", "convert", "overpass", ...(operation.regionId ? [operation.regionId] : [])], - { signal: context.signal, emitLog: context.emitLog }, + { ...context, operationKind: context.claim.operation.kind }, ); return { completed: true, ...(operation.regionId ? { resourceId: operation.regionId } : {}) }; }; runtime["data.link"] = async (_operation, context) => { await runFixedCli(["data", "link"], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { completed: true }; }; runtime["data.clean"] = async (operation, context) => { await runFixedCli(["data", "clean", operation.dataTypeId], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { completed: true, resourceId: operation.dataTypeId }; }; runtime["data.generateApiKeys"] = async (_operation, context) => { await runFixedCli(["data", "generate-api-keys"], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { completed: true }; }; runtime["data.overtureSync"] = async (operation, context) => { await runFixedCli(["data", "overture-sync", operation.regionId], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { completed: true, resourceId: operation.regionId }; }; @@ -1862,28 +1913,28 @@ export function createAdministrativeRuntime( operation.regionId, ...(operation.restart ? ["--restart"] : []), ], - { signal: context.signal, emitLog: context.emitLog }, + { ...context, operationKind: context.claim.operation.kind }, ); return { completed: true, resourceId: operation.regionId }; }; runtime["data.searchIndexBuild"] = async (operation, context) => { await runFixedCli(["data", "search-index", "build", operation.regionId], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { completed: true, resourceId: operation.regionId }; }; runtime["data.notablePlacesBuild"] = async (_operation, context) => { await runFixedCli(["data", "notable-places", "build"], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { completed: true }; }; runtime["system.diagnostics"] = async (_operation, context) => { await runFixedCli(["check"], { - signal: context.signal, - emitLog: context.emitLog, + ...context, + operationKind: context.claim.operation.kind, }); return { ok: true, checks: [] }; }; diff --git a/apps/ops-agent/src/agent-update.test.ts b/apps/ops-agent/src/agent-update.test.ts index bbd2c8954..3c3b6d293 100644 --- a/apps/ops-agent/src/agent-update.test.ts +++ b/apps/ops-agent/src/agent-update.test.ts @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; import { createAdministrativeRuntime, createDefaultReleaseEffects } from "./administrative-runtime"; import { type AgentHandoffRecord, readAgentHandoff, writeAgentHandoff } from "./agent-handoff"; import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; @@ -243,6 +244,7 @@ describe("operations agent update", () => { runtime, { kind: "agent.update", releaseId: release, expectedDigest: digest, handoffId: "handoff-1" }, { + execution: testExecution(), signal: new AbortController().signal, emitLog: () => {}, otherActiveOperations: () => 2, diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index 2096bedb7..e9812cb4c 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -1,13 +1,28 @@ import { randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { delimiter, join } from "node:path"; import type { OpsOperation } from "@openmapx/core/ops"; import { describe, expect, it, vi } from "vitest"; -import { createDockerRuntime, runContainedLogProcess, runContainedProcess } from "./docker-runtime"; +import { testExecution } from "./__fixtures__/execution"; +import { + createDockerRuntime, + runContainedLogProcess, + runContainedProcess, + runContainedStreamingProcess, +} from "./docker-runtime"; import { dispatchOpsOperation } from "./runtime"; const context = () => ({ + execution: testExecution(), signal: new AbortController().signal, emitLog: () => undefined, claim: { @@ -1781,7 +1796,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done } }); - it.each(["action", "follow"] as const)( + it.each(["action", "follow", "stream"] as const)( "contains a residual TERM-ignoring descendant for %s authority", async (kind) => { const root = mkdtempSync(join(tmpdir(), "openmapx-contained-group-")); @@ -1806,14 +1821,22 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done maxBuffer: 1_024, killGraceMs: 20, }) - : runContainedLogProcess(process.execPath, ["-e", parentScript], { - signal: new AbortController().signal, - timeout: 5_000, - maxBuffer: 1_024, - onLine: () => undefined, - }); + : kind === "stream" + ? runContainedStreamingProcess(process.execPath, ["-e", parentScript], { + signal: new AbortController().signal, + timeout: 5000, + killGraceMs: 20, + onLine: () => undefined, + redact: (x) => x, + }) + : runContainedLogProcess(process.execPath, ["-e", parentScript], { + signal: new AbortController().signal, + timeout: 5_000, + maxBuffer: 1_024, + onLine: () => undefined, + }); - await expect(operation).rejects.toThrow(/Contained .*process failed/); + await expect(operation).rejects.toThrow(/Contained .*process failed|Operation failed/); await new Promise((resolve) => setTimeout(resolve, 2_500)); expect(existsSync(marker)).toBe(false); rmSync(root, { recursive: true, force: true }); @@ -1821,3 +1844,110 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done 12_000, ); }); + +describe("durable streaming process containment", () => { + it("does not kill a noisy successful child when projection fails", async () => { + const diagnostics = await runContainedStreamingProcess( + process.execPath, + [ + "-e", + "for(let i=0;i<2048;i++) console.log('x'.repeat(1024));console.error('last-diagnostic');", + ], + { + signal: new AbortController().signal, + timeout: 5000, + onLine: () => { + throw Error("projection failed"); + }, + redact: (x) => x, + }, + ); + expect(diagnostics).toMatchObject({ + truncated: true, + tail: expect.arrayContaining([{ stream: "stderr", message: "last-diagnostic" }]), + }); + expect(Buffer.byteLength(JSON.stringify(diagnostics))).toBeLessThanOrEqual(16 * 1024); + }); + it.each(["abort", "deadline"] as const)( + "contains descendants on explicit %s with short safe errors", + async (mode) => { + const root = mkdtempSync(join(tmpdir(), "openmapx-stream-containment-")); + const ready = join(root, "ready"); + const marker = join(root, "marker"); + const script = `const {spawn}=require('node:child_process');process.on('SIGTERM',()=>{});spawn(process.execPath,['-e', ${JSON.stringify("process.on('SIGTERM',()=>{});require('node:fs').writeFileSync(process.argv[1],'ready');setTimeout(()=>require('node:fs').writeFileSync(process.argv[2],'late'),5000);setInterval(()=>{},1000)")},process.argv[1],process.argv[2]],{stdio:'ignore'});console.error('password=private');setInterval(()=>{},1000);`; + const controller = new AbortController(); + const completion = runContainedStreamingProcess( + process.execPath, + ["-e", script, ready, marker], + { + signal: controller.signal, + timeout: mode === "deadline" ? 3500 : 10_000, + killGraceMs: 20, + onLine: () => undefined, + redact: (x) => x, + }, + ); + const rejected = expect(completion).rejects.toMatchObject({ + name: "OpsExecutionFailure", + errorClass: mode === "deadline" ? "timeout" : "runtime", + }); + try { + await vi.waitFor(() => expect(existsSync(ready)).toBe(true), { timeout: 3000 }); + if (mode === "abort") controller.abort(); + await rejected; + const failure = await completion.catch((x) => x); + expect(JSON.stringify(failure.failureResult)).not.toContain("private"); + await new Promise((resolve) => setTimeout(resolve, 5100)); + expect(existsSync(marker)).toBe(false); + } finally { + controller.abort(); + await rejected; + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); + +it("streams the actual default fixed Docker human output without the JSON collector limit", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-fixed-docker-")); + const ready = join(root, "ready"); + const release = join(root, "release"); + writeFileSync( + join(root, "docker"), + `#!${process.execPath}\nconst{writeFileSync,existsSync}=require('node:fs');console.log('phase-one');writeFileSync(${JSON.stringify(ready)},'ready');const timer=setInterval(()=>{if(existsSync(${JSON.stringify(release)})){clearInterval(timer);process.stdout.write('x'.repeat(2*1024*1024)+'\\nlast-diagnostic\\n');}},5);`, + ); + chmodSync(join(root, "docker"), 0o700); + vi.stubEnv("PATH", root + delimiter + process.env.PATH); + const seen: string[] = []; + const runtime = createDockerRuntime({ + stack: { + infraDir: root, + composePath: join(root, "compose.yml"), + composeReleasePath: join(root, "release.yml"), + }, + fileExists: () => false, + }); + const completion = dispatchOpsOperation( + runtime, + { kind: "service.pull", serviceId: "redis" }, + { + ...context(), + emitLog: (_s, line) => { + seen.push(line); + }, + }, + ); + void completion.catch(() => undefined); + try { + await vi.waitFor(() => expect(existsSync(ready)).toBe(true), { timeout: 2000 }); + await vi.waitFor(() => expect(seen).toContain("phase-one"), { timeout: 2000 }); + writeFileSync(release, "go"); + await expect(completion).resolves.toEqual({ changed: true }); + expect(seen).toContain("last-diagnostic"); + } finally { + writeFileSync(release, "go"); + await completion.catch(() => undefined); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 5b0e2b447..bfb613282 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -7,13 +7,20 @@ import { OPS_MAX_EVENT_MESSAGE_BYTES, OPS_MAX_FOLLOW_LOG_EVENTS, type OpsResultFor, + type ProcessDiagnostics, } from "@openmapx/core/ops"; import { assertPosixProcessGroupsSupported, monitorPosixProcessGroup, services, } from "@openmapx/core/server"; -import { createUnavailableRuntime, type OpsRuntime } from "./runtime"; +import { createBoundedProcessOutput, safeProcessDiagnostic } from "./process-output"; +import { + createUnavailableRuntime, + type OpsExecutionContext, + OpsExecutionFailure, + type OpsRuntime, +} from "./runtime"; const DEFAULT_TIMEOUT_MS = 120_000; const LIFECYCLE_TIMEOUT_MS = 10 * 60_000; @@ -73,6 +80,85 @@ export interface ContainedProcessOptions { killGraceMs?: number; } +export interface StreamingProcessOptions { + signal: AbortSignal; + timeout: number; + killGraceMs?: number; + onLine(stream: "stdout" | "stderr", message: string): void; + redact: (message: string) => string; + /** Only the fixed CLI adapter supplies a private managed context. */ + env?: NodeJS.ProcessEnv; +} + +function childEnvironment(): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const key of [ + "OPENMAPX_CONTROL_EXECUTION", + "OPENMAPX_CONTROL_DELEGATION", + "OPENMAPX_PARENT_EXECUTION", + ]) + delete env[key]; + return env; +} + +export async function runContainedStreamingProcess( + file: string, + args: readonly string[], + options: StreamingProcessOptions, +): Promise { + assertPosixProcessGroupsSupported(); + if (options.signal.aborted || options.timeout <= 0) + throw new OpsExecutionFailure("timeout", { diagnostics: { tail: [], truncated: false } }); + if ( + !Number.isSafeInteger(options.timeout) || + (options.killGraceMs !== undefined && + (!Number.isSafeInteger(options.killGraceMs) || options.killGraceMs < 1)) + ) + throw new OpsExecutionFailure("runtime", { diagnostics: { tail: [], truncated: false } }); + const output = createBoundedProcessOutput(options); + let child: ReturnType; + try { + child = spawn(file, [...args], { + detached: true, + stdio: ["ignore", "pipe", "pipe"], + env: options.env ?? childEnvironment(), + }); + } catch { + throw new OpsExecutionFailure("runtime", { diagnostics: output.end() }); + } + const group = monitorPosixProcessGroup(child, { + killGraceMs: options.killGraceMs ?? KILL_GRACE_MS, + }); + let timedOut = false; + const abort = () => group.terminate(); + const timer = setTimeout(() => { + timedOut = true; + group.terminate(); + }, options.timeout); + options.signal.addEventListener("abort", abort, { once: true }); + if (options.signal.aborted) abort(); + child.stdout?.on("data", (chunk: Buffer) => output.write("stdout", chunk)); + child.stderr?.on("data", (chunk: Buffer) => output.write("stderr", chunk)); + try { + const exit = await group.closed; + const diagnostics = output.end(); + if ( + timedOut || + options.signal.aborted || + exit.spawnFailed || + exit.residualDescendants || + exit.containmentFailed || + exit.code !== 0 || + exit.signal !== null + ) + throw new OpsExecutionFailure(timedOut ? "timeout" : "runtime", { diagnostics }); + return diagnostics; + } finally { + clearTimeout(timer); + options.signal.removeEventListener("abort", abort); + } +} + // The agent runs as the data-owning user, so its own identity is the default // owner for artifacts produced inside a container on the shared mount. const defaultDataMountOwner = { @@ -120,10 +206,13 @@ export function runContainedProcess( options: ContainedProcessOptions, ): Promise<{ stdout: string; stderr: string }> { assertPosixProcessGroupsSupported(); + if (options.signal.aborted || options.timeout <= 0) + return Promise.reject(new Error("Contained process failed")); return new Promise((resolve, reject) => { const child = spawn(file, [...args], { detached: true, stdio: ["ignore", "pipe", "pipe"], + env: childEnvironment(), }); const group = monitorPosixProcessGroup(child, { killGraceMs: options.killGraceMs ?? KILL_GRACE_MS, @@ -151,8 +240,8 @@ export function runContainedProcess( const timeout = setTimeout(terminate, options.timeout); options.signal.addEventListener("abort", onAbort, { once: true }); if (options.signal.aborted) terminate(); - child.stdout.on("data", onData(stdout)); - child.stderr.on("data", onData(stderr)); + child.stdout?.on("data", onData(stdout)); + child.stderr?.on("data", onData(stderr)); child.once("error", () => { failed = true; }); @@ -188,6 +277,7 @@ export function runContainedLogProcess( const child = spawn(file, [...args], { detached: true, stdio: ["ignore", "pipe", "pipe"], + env: childEnvironment(), }); const group = monitorPosixProcessGroup(child, { killGraceMs: KILL_GRACE_MS }); let bytes = 0; @@ -325,8 +415,21 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { services.stackComposeArgs(options.stack, options.fileExists); const run = (args: readonly string[], signal: AbortSignal, timeout = DEFAULT_TIMEOUT_MS) => execFile("docker", args, { signal, timeout, maxBuffer: MAX_OUTPUT_BYTES }); - const changed = async (args: readonly string[], signal: AbortSignal, timeout?: number) => { - await run(args, signal, timeout); + const changed = async ( + args: readonly string[], + context: OpsExecutionContext, + timeout = DEFAULT_TIMEOUT_MS, + ) => { + const remaining = context.execution.deadlineAt - Date.now(); + if (remaining <= 0 || context.signal.aborted) throw new OpsExecutionFailure("timeout", {}); + if (options.execFile) await run(args, context.signal, Math.min(timeout, remaining)); + else + await runContainedStreamingProcess("docker", args, { + signal: context.signal, + timeout: Math.min(timeout, remaining), + onLine: context.emitLog, + redact: safeProcessDiagnostic, + }); return { changed: true } as const; }; const motisState = async (name: "motis" | "motis-staging", signal: AbortSignal) => { @@ -356,12 +459,12 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { }; const stopMotis = async ( name: "motis" | "motis-staging", - signal: AbortSignal, + context: OpsExecutionContext, ): Promise> => { - const before = await motisState(name, signal); + const before = await motisState(name, context.signal); if (before !== "running") return { changed: false, state: before }; - await run(["stop", name], signal); - const after = await motisState(name, signal); + await changed(["stop", name], context); + const after = await motisState(name, context.signal); if (after === "running") throw new Error("MOTIS container remains running after stop"); return { changed: true, state: after }; }; @@ -377,45 +480,29 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { return parseStackStatus(result.stdout); }; runtime["stack.start"] = (_operation, context) => - changed([...composePrefix(), "up", "-d"], context.signal, LIFECYCLE_TIMEOUT_MS); + changed([...composePrefix(), "up", "-d"], context, LIFECYCLE_TIMEOUT_MS); runtime["service.start"] = (operation, context) => - changed( - [...composePrefix(), "up", "-d", operation.serviceId], - context.signal, - LIFECYCLE_TIMEOUT_MS, - ); + changed([...composePrefix(), "up", "-d", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.stop"] = (operation, context) => - changed( - [...composePrefix(), "stop", operation.serviceId], - context.signal, - LIFECYCLE_TIMEOUT_MS, - ); + changed([...composePrefix(), "stop", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.restart"] = (operation, context) => - changed( - [...composePrefix(), "restart", operation.serviceId], - context.signal, - LIFECYCLE_TIMEOUT_MS, - ); + changed([...composePrefix(), "restart", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.recreate"] = (operation, context) => changed( [...composePrefix(), "up", "-d", "--force-recreate", operation.serviceId], - context.signal, + context, LIFECYCLE_TIMEOUT_MS, ); runtime["service.recreateIsolated"] = (operation, context) => changed( [...composePrefix(), "up", "-d", "--force-recreate", "--no-deps", operation.serviceId], - context.signal, + context, LIFECYCLE_TIMEOUT_MS, ); runtime["service.pull"] = (operation, context) => - changed([...composePrefix(), "pull", operation.serviceId], context.signal, PULL_TIMEOUT_MS); + changed([...composePrefix(), "pull", operation.serviceId], context, PULL_TIMEOUT_MS); runtime["service.remove"] = (operation, context) => - changed( - [...composePrefix(), "rm", "-sf", operation.serviceId], - context.signal, - LIFECYCLE_TIMEOUT_MS, - ); + changed([...composePrefix(), "rm", "-sf", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.logs"] = async (operation, context) => { const result = await run( [...composePrefix(), "logs", "--no-color", `--tail=${operation.tail}`, operation.serviceId], @@ -505,8 +592,8 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { runtime["feedProxy.validateAndReload"] = async (operation, context) => { // Validate before reloading: a reload against a bad configuration would // take the proxy down rather than reject the candidate. - await run(["exec", FEED_PROXY_CONTAINER, "nginx", "-t"], context.signal, 30_000); - await run(["exec", FEED_PROXY_CONTAINER, "nginx", "-s", "reload"], context.signal, 30_000); + await changed(["exec", FEED_PROXY_CONTAINER, "nginx", "-t"], context, 30_000); + await changed(["exec", FEED_PROXY_CONTAINER, "nginx", "-s", "reload"], context, 30_000); return { candidateId: operation.candidateId, reloaded: true as const }; }; // Valhalla traffic effects. The whole sequence lives here because each step @@ -766,25 +853,24 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { }; runtime["motis.staging.restart"] = (_operation, context) => - changed(["restart", "motis-staging"], context.signal); - runtime["motis.staging.stop"] = (_operation, context) => - stopMotis("motis-staging", context.signal); + changed(["restart", "motis-staging"], context); + runtime["motis.staging.stop"] = (_operation, context) => stopMotis("motis-staging", context); runtime["motis.primary.restart"] = (_operation, context) => - changed(["restart", "motis"], context.signal); + changed(["restart", "motis"], context); runtime["motis.primary.promote"] = async (operation, context) => { // Recreate rather than restart: Docker resolves a bind-mount symlink when // the container is created, so a plain restart can keep pointing at the old // A/B target. `--no-deps` stops Compose from cascading into the feed proxy // and recreating MOTIS a second time, which would race the caller's // post-swap health probe. - await run( + await changed( [...composePrefix(), "up", "-d", "--force-recreate", "--no-deps", "motis"], - context.signal, + context, 10 * 60_000, ); // The caller performed the data swap; the agent reports the run it activated. return { activeRunId: operation.preparedRunId }; }; - runtime["motis.primary.stop"] = (_operation, context) => stopMotis("motis", context.signal); + runtime["motis.primary.stop"] = (_operation, context) => stopMotis("motis", context); return runtime; } diff --git a/apps/ops-agent/src/journal.test.ts b/apps/ops-agent/src/journal.test.ts index dfb3ac663..6e9ba93a3 100644 --- a/apps/ops-agent/src/journal.test.ts +++ b/apps/ops-agent/src/journal.test.ts @@ -51,7 +51,7 @@ describe("durable ops job journal", () => { it("durably probes a missing journal before open succeeds", async () => { const path = join(root(), "journal", "jobs-v1.json"); await openOpsJobJournal(path); - expect(JSON.parse(readFileSync(path, "utf8"))).toEqual({ version: 1, jobs: [] }); + expect(JSON.parse(readFileSync(path, "utf8"))).toEqual({ version: 2, jobs: [] }); }); it("fails open when an existing readable journal directory is not writable", async () => { @@ -210,3 +210,67 @@ describe("durable ops job journal", () => { } }); }); + +describe("version2 execution journal", () => { + it("recomputes both hashes and never renews a running deadline on restart", async () => { + const { normalizeExecutionAdmission, opsOperationFingerprint } = await import( + "@openmapx/core/ops" + ); + const path = join(root(), "jobs-v1.json"); + const operation = { kind: "service.build" as const, serviceId: "valhalla" }; + const operationFingerprint = opsOperationFingerprint(operation); + const admittedAt = Date.parse("2026-08-23T18:00:00.000Z"); + const operationKey = "opk1_abcdefghijklmnop"; + const normalized = normalizeExecutionAdmission({ + operationFingerprint, + operationKey, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: admittedAt + 1000, + }); + const record = { + executionVersion: 2 as const, + ...normalized, + operationFingerprint, + requestedDeadlineAt: admittedAt + 1000, + deploymentBudgetMs: 12 * 3600_000, + operation, + operationKey, + operationId: "job1_abcdefghijklmnop", + role: "api" as const, + resourceId: "valhalla", + state: "running" as const, + submittedAt: new Date(admittedAt).toISOString(), + updatedAt: new Date(admittedAt).toISOString(), + }; + const journal = await openOpsJobJournal(path); + await journal.replace([record]); + const recovered = await openOpsJobJournal(path, { now: () => new Date(admittedAt + 2000) }); + expect(recovered.records()[0]).toMatchObject({ + admittedAt, + deadlineAt: admittedAt + 1000, + fingerprint: normalized.fingerprint, + state: "failed", + errorClass: "recovery_required", + terminationRequestedAt: new Date(admittedAt + 1000).toISOString(), + }); + const retained = recovered.records()[0]; + if (!retained) throw new Error("Missing retained execution"); + const extended = normalizeExecutionAdmission({ + operationFingerprint, + operationKey, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: admittedAt + 5000, + }); + await expect( + recovered.replace([{ ...retained, ...extended, requestedDeadlineAt: admittedAt + 5000 }]), + ).rejects.toThrow("Retained execution identity is immutable"); + await expect( + recovered.replace([{ ...retained, deadlineAt: admittedAt + 5000 }]), + ).rejects.toThrow(); + await expect( + recovered.replace([{ ...retained, operationFingerprint: "c".repeat(64) }]), + ).rejects.toThrow(); + }); +}); diff --git a/apps/ops-agent/src/journal.ts b/apps/ops-agent/src/journal.ts index ff32f287d..a9acd243f 100644 --- a/apps/ops-agent/src/journal.ts +++ b/apps/ops-agent/src/journal.ts @@ -3,15 +3,22 @@ import { constants } from "node:fs"; import { mkdir, open, opendir, rename, unlink } from "node:fs/promises"; import { basename, dirname, join } from "node:path"; import { + OPS_KIND_POLICIES, OPS_PUBLIC_ERROR_MESSAGES, type OpsErrorClass, + type OpsFailureResult, type OpsJobState, type OpsOperation, type OpsRole, + opsFailureResultSchema, opsOperationFingerprint, opsOperationSchema, opsResourceId, + type ProcessDiagnostics, parseBoundedOpsResult, + processDiagnosticsSchema, + type RetainedExecutionAdmission, + validateRetainedExecutionAdmission, } from "@openmapx/core/ops"; const OPS_JOB_JOURNAL_MAX_ENTRIES = 256; @@ -33,6 +40,15 @@ const ERROR_CLASSES = new Set( ); export interface PersistedOpsJob { + executionVersion?: 2; + admittedAt?: number; + deadlineAt?: number; + operationFingerprint?: string; + requestedDeadlineAt?: number; + deploymentBudgetMs?: number; + parent?: RetainedExecutionAdmission["parent"]; + diagnostics?: ProcessDiagnostics; + failureResult?: OpsFailureResult; role: OpsRole; operation: OpsOperation; operationId: string; @@ -79,6 +95,15 @@ function parseJob(value: unknown): PersistedOpsJob { const raw = value as Record; if ( !exactKeys(raw, [ + "executionVersion", + "admittedAt", + "deadlineAt", + "operationFingerprint", + "requestedDeadlineAt", + "deploymentBudgetMs", + "parent", + "diagnostics", + "failureResult", "role", "operation", "operationId", @@ -114,11 +139,39 @@ function parseJob(value: unknown): PersistedOpsJob { } const operation = opsOperationSchema.parse(raw.operation); if ( - opsOperationFingerprint(operation) !== raw.fingerprint || + opsOperationFingerprint(operation) !== + (raw.executionVersion === 2 ? raw.operationFingerprint : raw.fingerprint) || opsResourceId(operation) !== raw.resourceId ) { throw new Error("invalid"); } + if (raw.executionVersion === 2) { + validateRetainedExecutionAdmission( + operation.kind, + raw as unknown as RetainedExecutionAdmission, + ); + if (raw.admittedAt !== Date.parse(raw.submittedAt as string)) throw new Error("invalid"); + } else if ( + [ + "executionVersion", + "admittedAt", + "deadlineAt", + "operationFingerprint", + "requestedDeadlineAt", + "deploymentBudgetMs", + "parent", + ].some((key) => raw[key] !== undefined) + ) + throw new Error("invalid"); + if (raw.diagnostics !== undefined) processDiagnosticsSchema.parse(raw.diagnostics); + if (raw.failureResult !== undefined) { + if (!["failed", "timed_out"].includes(raw.state as string)) throw new Error("invalid"); + const result = opsFailureResultSchema.parse(raw.failureResult); + if ( + Buffer.byteLength(JSON.stringify(result)) > OPS_KIND_POLICIES[operation.kind].maxResultBytes + ) + throw new Error("invalid"); + } const state = raw.state as OpsJobState; if (state === "succeeded") { if (raw.result === undefined || raw.errorClass !== undefined || raw.terminalAt === undefined) @@ -144,13 +197,15 @@ function parseFile(bytes: Uint8Array, maxEntries: number): PersistedOpsJob[] { const raw = decoded as Record; if ( Object.keys(raw).sort().join(",") !== "jobs,version" || - raw.version !== 1 || + (raw.version !== 1 && raw.version !== 2) || !Array.isArray(raw.jobs) || raw.jobs.length > maxEntries ) { throw new Error("invalid"); } const jobs = raw.jobs.map(parseJob); + if (raw.version === 1 && jobs.some((job) => job.executionVersion !== undefined)) + throw new Error("invalid journal version"); const ids = new Set(jobs.map((entry) => entry.operationId)); const keys = new Set(jobs.map((entry) => `${entry.role}:${entry.operationKey}`)); if (ids.size !== jobs.length || keys.size !== jobs.length) throw new Error("invalid"); @@ -270,12 +325,34 @@ export async function openOpsJobJournal( const writeSnapshot = async (records: readonly PersistedOpsJob[]): Promise => { if (records.length > maxEntries) throw new Error("Ops job journal capacity exceeded"); const validated = records.map(parseJob); + for (const record of validated) { + const retained = current.find( + (prior) => + prior.operationId === record.operationId || + (prior.role === record.role && prior.operationKey === record.operationKey), + ); + if ( + retained && + (retained.operationId !== record.operationId || + retained.operationKey !== record.operationKey || + retained.role !== record.role || + retained.fingerprint !== record.fingerprint || + retained.executionVersion !== record.executionVersion || + retained.admittedAt !== record.admittedAt || + retained.deadlineAt !== record.deadlineAt || + retained.operationFingerprint !== record.operationFingerprint || + retained.requestedDeadlineAt !== record.requestedDeadlineAt || + retained.deploymentBudgetMs !== record.deploymentBudgetMs || + JSON.stringify(retained.parent) !== JSON.stringify(record.parent)) + ) + throw Error("Retained execution identity is immutable"); + } const ids = new Set(validated.map((entry) => entry.operationId)); const keys = new Set(validated.map((entry) => `${entry.role}:${entry.operationKey}`)); if (ids.size !== validated.length || keys.size !== validated.length) { throw new Error("Ops job journal contains duplicate identity"); } - const bytes = Buffer.from(JSON.stringify({ version: 1, jobs: validated }), "utf8"); + const bytes = Buffer.from(JSON.stringify({ version: 2, jobs: validated }), "utf8"); if (bytes.byteLength > maxBytes) throw new Error("Ops job journal capacity exceeded"); const directory = dirname(path); const temporary = join(directory, `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); @@ -322,6 +399,12 @@ export async function openOpsJobJournal( ["queued", "running", "termination_pending"].includes(entry.state) ? { ...entry, + ...(entry.deadlineAt !== undefined && entry.deadlineAt <= now().getTime() + ? { + terminationRequestedAt: + entry.terminationRequestedAt ?? new Date(entry.deadlineAt).toISOString(), + } + : {}), state: "failed" as const, updatedAt: timestamp, terminalAt: timestamp, diff --git a/apps/ops-agent/src/process-output.test.ts b/apps/ops-agent/src/process-output.test.ts new file mode 100644 index 000000000..510f7d1b7 --- /dev/null +++ b/apps/ops-agent/src/process-output.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import { createBoundedProcessOutput } from "./process-output"; + +describe("bounded live process output", () => { + it("decodes split UTF8 and frames CR/LF before exit", () => { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_, line) => seen.push(line), + redact: (x) => x, + }); + output.write("stdout", Buffer.from([0xf0, 0x9f])); + output.write("stdout", Buffer.from([0x98, 0x80, 13, 10])); + output.write("stderr", Buffer.from("progress\rnext\ntail")); + expect(seen).toEqual(["😀", "progress", "next"]); + expect(output.end().tail.map((x) => x.message)).toEqual(["😀", "progress", "next", "tail"]); + }); + it("scrubs credentials across chunks, controls and frame boundaries", () => { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_, line) => seen.push(line), + redact: (x) => x, + }); + for (const chunk of [ + "https://user:pas", + "s@host/path?token=private\nAuthor", + "ization: Bearer hidden\npass", + "word=", + "topsecret\n\u001b[31mhello\u001b[0m\u0000\npassword=\n", + "continuedsecret\n", + `${"x".repeat(4090)} token=`, + "secret-crossing-boundary\nlast-diagnostic\n", + ]) + output.write("stdout", Buffer.from(chunk)); + const result = output.end(); + const text = JSON.stringify({ seen, result }); + for (const secret of [ + "private", + "user:pass", + "hidden", + "topsecret", + "continuedsecret", + "secret-crossing-boundary", + "\\u001b", + "\\u0000", + ]) + expect(text).not.toContain(secret); + expect(seen).toContain("hello"); + expect(seen).toContain("last-diagnostic"); + expect(result.truncated).toBe(true); + }); + it("drains noisy and unterminated output and retains the final meaningful tail", () => { + const output = createBoundedProcessOutput({ onLine: () => undefined, redact: (x) => x }); + for (let i = 0; i < 2048; i++) output.write("stdout", Buffer.from(`${"x".repeat(1024)}\n`)); + output.write("stderr", Buffer.from("y".repeat(1024 * 1024))); + output.write("stderr", Buffer.from("\nlast-diagnostic\n")); + const result = output.end(); + expect(result.truncated).toBe(true); + expect(result.tail[0]?.message).toBe("[output truncated]"); + expect(result.tail.at(-1)?.message).toBe("last-diagnostic"); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThanOrEqual(16 * 1024); + expect(result.tail.every((x) => Buffer.byteLength(x.message) <= 4096)).toBe(true); + }); + it("detaches a failed callback and continues safe diagnostics", () => { + let calls = 0; + const output = createBoundedProcessOutput({ + onLine: () => { + calls++; + throw Error("sink"); + }, + redact: (x) => x, + }); + output.write("stdout", Buffer.from("one\ntwo\n")); + expect(output.end()).toMatchObject({ + truncated: true, + tail: expect.arrayContaining([{ stream: "stdout", message: "two" }]), + }); + expect(calls).toBe(1); + }); +}); + +it("withholds an unterminated quoted credential across multiple real frames", () => { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_s, line) => seen.push(line), + redact: (x) => x, + }); + for (const chunk of [ + 'password="first-secret\n', + "second-secret\r", + 'third-secret"\n', + "last-diagnostic\n", + ]) + output.write("stdout", Buffer.from(chunk)); + const text = JSON.stringify({ seen, diagnostics: output.end() }); + for (const secret of ["first-secret", "second-secret", "third-secret"]) + expect(text).not.toContain(secret); + expect(seen).toContain("last-diagnostic"); +}); diff --git a/apps/ops-agent/src/process-output.ts b/apps/ops-agent/src/process-output.ts new file mode 100644 index 000000000..fd2bd5102 --- /dev/null +++ b/apps/ops-agent/src/process-output.ts @@ -0,0 +1,180 @@ +import { StringDecoder } from "node:string_decoder"; +import { stripVTControlCharacters } from "node:util"; +import { PROCESS_OUTPUT_TRUNCATED, type ProcessDiagnostics } from "@openmapx/core/ops"; +import { scrubSecrets } from "@openmapx/core/scrub-secrets"; + +type Stream = "stdout" | "stderr"; +interface OutputOptions { + onLine(stream: Stream, message: string): void; + redact: (message: string) => string; + maxTailBytes?: number; +} +const QUOTED_CREDENTIAL = + /\b((?:authorization|api[-_]?key|access[-_]?token|token|secret|password|passwd|auth|key)\s*[:=]\s*)(["']).*$/gi; +const QUOTED_CREDENTIAL_START = + /\b(?:authorization|api[-_]?key|access[-_]?token|token|secret|password|passwd|auth|key)\s*[:=]\s*(["'])/i; +export function safeProcessDiagnostic(message: string): string { + return scrubSecrets( + // biome-ignore lint/suspicious/noControlCharactersInRegex: diagnostic sanitization intentionally strips C0/C1 controls. + stripVTControlCharacters(message).replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/g, ""), + ).replace(QUOTED_CREDENTIAL, "$1[redacted]"); +} + +/** Fixed-size framing, incremental decoding and a rolling redacted tail. Never owns termination. */ +export function createBoundedProcessOutput(options: OutputOptions): { + write(stream: Stream, chunk: Uint8Array): void; + end(): ProcessDiagnostics; +} { + const maximum = options.maxTailBytes ?? 16 * 1024; + if (!Number.isSafeInteger(maximum) || maximum < 256 || maximum > 16 * 1024) + throw Error("Invalid diagnostic budget"); + const streams = { + stdout: { + decoder: new StringDecoder("utf8"), + text: "", + bytes: 0, + dropping: false, + sensitive: false, + credentialQuote: undefined as string | undefined, + cr: false, + }, + stderr: { + decoder: new StringDecoder("utf8"), + text: "", + bytes: 0, + dropping: false, + sensitive: false, + credentialQuote: undefined as string | undefined, + cr: false, + }, + }; + const tail: ProcessDiagnostics["tail"] = []; + let truncated = false; + let detached = false; + let ended = false; + // Bound serialized storage as well as payload bytes, including tiny-line floods. + let retainedBytes = 0; + const streamBytes = { stdout: 0, stderr: 0 }; + const emit = (stream: Stream, progress = false) => { + const state = streams[stream]; + const raw = state.text; + const carriedQuote = state.credentialQuote; + if (carriedQuote) { + if (raw.includes(carriedQuote)) state.credentialQuote = undefined; + } else { + const start = QUOTED_CREDENTIAL_START.exec(raw); + if (start && !raw.slice(start.index + start[0].length).includes(start[1] as string)) + state.credentialQuote = start[1]; + } + const sensitiveNext = + !!state.credentialQuote || + /(?:\b(?:authorization\s*[:=]\s*(?:(?:bearer|basic|token)\s*)?|(?:password|passwd|token|secret|api[-_]?key|access[-_]?token|auth|key)\s*[:=]\s*)|\b(?:https?|ftps?):\/\/[^\s]*|AGE-ENCRYPTED:)$/i.test( + raw, + ) || + (progress && + /(?:authorization|password|passwd|token|secret|api[-_]?key|auth)\s*[:=]\s*\S*$/i.test(raw)); + if (raw) { + let message: string; + try { + message = safeProcessDiagnostic( + options.redact( + safeProcessDiagnostic(state.sensitive || carriedQuote ? "[redacted]" : raw), + ), + ); + } catch { + message = "[redacted]"; + detached = true; + truncated = true; + } + // A custom projector can expand text. Bound its returned message as well. + if (Buffer.byteLength(message) > 4096) { + message = "[output truncated]"; + truncated = true; + } + const entry = { stream, message }; + const bytes = Buffer.byteLength(JSON.stringify(entry)) + 1; + tail.push(entry); + retainedBytes += bytes; + streamBytes[stream] += bytes; + while (tail.length > 255 || retainedBytes + 128 > maximum) { + // A stdout flood must not evict the last stderr diagnostic (and vice versa). + const evictStream = streamBytes.stdout >= streamBytes.stderr ? "stdout" : "stderr"; + const index = tail.findIndex((entry) => entry.stream === evictStream); + const [removed] = tail.splice(index, 1); + if (!removed) break; + const removedBytes = Buffer.byteLength(JSON.stringify(removed)) + 1; + retainedBytes -= removedBytes; + streamBytes[removed.stream] -= removedBytes; + truncated = true; + } + if (!detached) { + try { + options.onLine(stream, message); + } catch { + detached = true; + truncated = true; + } + } + } + state.sensitive = sensitiveNext; + state.text = ""; + state.bytes = 0; + }; + const consume = (stream: Stream, text: string) => { + const state = streams[stream]; + for (const character of text) { + if (character === "\n" || character === "\r") { + if (!(character === "\n" && state.cr)) { + if (!state.dropping) emit(stream, character === "\r"); + else { + state.text = ""; + state.bytes = 0; + state.sensitive = !!state.credentialQuote; + } + state.dropping = false; + } + state.cr = character === "\r"; + continue; + } + state.cr = false; + if (state.dropping) { + if (character === state.credentialQuote) state.credentialQuote = undefined; + continue; + } + const size = Buffer.byteLength(character); + if (state.bytes + size > 4096) { + // Never project an artificial continuation: it may be the middle of a credential. + emit(stream); + state.dropping = true; + truncated = true; + continue; + } + state.text += character; + state.bytes += size; + } + }; + return { + write(stream, chunk) { + if (ended) return; + // Decode bounded slices even when a caller supplies one enormous chunk. + for (let i = 0; i < chunk.length; i += 4096) + consume(stream, streams[stream].decoder.write(Buffer.from(chunk.subarray(i, i + 4096)))); + }, + end() { + if (!ended) { + for (const stream of ["stdout", "stderr"] as const) { + consume(stream, streams[stream].decoder.end()); + if (!streams[stream].dropping) emit(stream); + } + ended = true; + } + return { + tail: [ + ...(truncated ? [{ stream: "stderr" as const, message: PROCESS_OUTPUT_TRUNCATED }] : []), + ...tail, + ], + truncated, + }; + }, + }; +} diff --git a/apps/ops-agent/src/runtime.test.ts b/apps/ops-agent/src/runtime.test.ts index d031e8eee..5cdc20e36 100644 --- a/apps/ops-agent/src/runtime.test.ts +++ b/apps/ops-agent/src/runtime.test.ts @@ -1,8 +1,10 @@ import { OPS_OPERATION_KINDS } from "@openmapx/core/ops"; import { describe, expect, it } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; const context = { + execution: testExecution(), signal: new AbortController().signal, emitLog: () => undefined, claim: { @@ -34,6 +36,7 @@ describe("typed ops runtime registry", () => { runtime, { kind: "docker.status" }, { + execution: testExecution(), signal: new AbortController().signal, emitLog: () => undefined, claim: context.claim, diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index 4ee81b80c..ef1d95c62 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -1,13 +1,26 @@ import type { TrustedConfigurationPayload } from "@openmapx/core/ops"; import { + type ExecutionRef, OPS_OPERATION_KINDS, OPS_PUBLIC_ERROR_MESSAGES, + type OpsErrorClass, + type OpsFailureResult, type OpsOperation, type OpsOperationKind, type OpsResultFor, parseOpsResult, } from "@openmapx/core/ops"; +export class OpsExecutionFailure extends Error { + constructor( + readonly errorClass: OpsErrorClass, + readonly failureResult: OpsFailureResult, + ) { + super(OPS_PUBLIC_ERROR_MESSAGES[errorClass]); + this.name = "OpsExecutionFailure"; + } +} + export class OpsNotWiredError extends Error { constructor() { super("Typed adapter is not wired"); @@ -31,6 +44,7 @@ export class OpsAgentBusyError extends Error { } export interface OpsExecutionContext { + readonly execution: ExecutionRef; signal: AbortSignal; emitLog(stream: "stdout" | "stderr", message: string): void; claim: OpsTrustedClaim; diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index 80153109b..6a3ac70e3 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -314,7 +314,7 @@ describe("ops-agent async foundation", () => { const canceled = await app.inject({ method: "DELETE", url: `/v1/operations/${operationId}`, - headers: auth(), + headers: { ...auth(), "x-ops-operation-fingerprint": admitted.json().result.fingerprint }, }); expect(canceled.statusCode).toBe(200); expect(canceled.json().result).toMatchObject({ @@ -367,7 +367,10 @@ describe("ops-agent async foundation", () => { const migratedCancel = await app.inject({ method: "DELETE", url: `/v1/operations/${restartId}`, - headers: auth(), + headers: { + ...auth(), + "x-ops-operation-fingerprint": restartAdmission.json().result.fingerprint, + }, }); expect(migratedCancel.statusCode).toBe(200); expect(migratedCancel.json().result.state).toBe("termination_pending"); @@ -445,7 +448,7 @@ describe("ops-agent async foundation", () => { const canceled = await app.inject({ method: "DELETE", url: `/v1/operations/${operationId}`, - headers: auth(), + headers: { ...auth(), "x-ops-operation-fingerprint": admitted.json().result.fingerprint }, }); expect(canceled.statusCode).toBe(500); expect(aborted).toBe(true); @@ -597,7 +600,7 @@ describe("ops-agent async foundation", () => { await app.close(); }); - it("rejects validated sync and async results that exceed their per-kind byte budgets", async () => { + it("rejects oversized sync output and retains the maximum build-all receipt", async () => { const app = buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, @@ -640,12 +643,15 @@ describe("ops-agent async foundation", () => { ), }); const operationId = admitted.json().result.operationId as string; - const failed = await eventually( + const completed = await eventually( async () => app.inject({ method: "GET", url: `/v1/operations/${operationId}`, headers: auth() }), - (response) => response.json().result.state === "failed", + (response) => response.json().result.state === "succeeded", + ); + expect(completed.json().result.result.completedServiceIds).toHaveLength(256); + expect(Buffer.byteLength(JSON.stringify(completed.json().result.result))).toBeGreaterThan( + 16 * 1024, ); - expect(failed.json().result.errorClass).toBe("runtime"); await app.close(); }); diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index d5865d4b9..a599fbe65 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -228,7 +228,8 @@ describe("ops-agent lifecycle deadlines", () => { ["service.stop", [299_999]], ["service.recreate", [240_000, 220_000]], ] as const)("allows bounded shutdown and dependency work for %s", async (kind, durations) => { - vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + vi.setSystemTime(new Date(nowIso)); let aborted = false; const completed: number[] = []; const runtime = createDockerRuntime({ @@ -303,7 +304,8 @@ describe("ops-agent lifecycle deadlines", () => { it.each(["deadline", "cancellation"] as const)( "settles lifecycle %s without retrying", async (mode) => { - vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + vi.setSystemTime(new Date(nowIso)); let signal: AbortSignal | undefined; const dispatch = vi.fn>( (_operation, _runtime, context) => @@ -330,7 +332,7 @@ describe("ops-agent lifecycle deadlines", () => { payload: envelope({ kind: "service.stop", serviceId: "postgis" }), }); expect(response.statusCode).toBe(202); - const { operationId } = response.json().result; + const { operationId, fingerprint } = response.json().result; await vi.advanceTimersByTimeAsync(mode === "deadline" ? 629_999 : 1_000); expect(signal?.aborted).toBe(false); expect( @@ -347,7 +349,7 @@ describe("ops-agent lifecycle deadlines", () => { await app.inject({ method: "DELETE", url: `/v1/operations/${operationId}`, - headers: auth(apiToken), + headers: { ...auth(apiToken), "x-ops-operation-fingerprint": fingerprint }, }); expect(signal?.aborted).toBe(true); const status = ( @@ -371,3 +373,360 @@ describe("ops-agent lifecycle deadlines", () => { }, ); }); + +describe("retained execution deadlines", () => { + it("retains original admission across duplicate, extension attempt and old 30 minute limit", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + vi.setSystemTime(new Date(nowIso)); + let signal: AbortSignal | undefined; + let finish!: (x: unknown) => void; + let calls = 0; + const records: import("./journal").PersistedOpsJob[][] = []; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + journal: { + records: () => [], + replace: async (x) => { + records.push(structuredClone([...x])); + }, + }, + dispatch: async (_op, _runtime, ctx) => { + calls++; + signal = ctx.signal; + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + try { + const first = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload: { + ...envelope({ kind: "service.build", serviceId: "valhalla" }), + execution: { requestedDeadlineAt: Date.now() + 4 * 3600_000 }, + }, + }); + const admitted = first.json().result; + expect(admitted.deadlineAt).toBe(Date.now() + 4 * 3600_000); + expect(records.at(-1)?.[0]).toMatchObject({ + admittedAt: Date.now(), + deadlineAt: admitted.deadlineAt, + operationFingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + await vi.advanceTimersByTimeAsync(31 * 60_000); + expect(signal?.aborted).toBe(false); + const duplicate = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload: { + ...envelope( + { kind: "service.build", serviceId: "valhalla" }, + "ops1_duplicateabcdefgh", + undefined, + new Date().toISOString(), + new Date(Date.now() + 20_000).toISOString(), + ), + execution: { requestedDeadlineAt: admitted.deadlineAt }, + }, + }); + expect(duplicate.json().result).toMatchObject({ + operationId: admitted.operationId, + admittedAt: admitted.admittedAt, + deadlineAt: admitted.deadlineAt, + fingerprint: admitted.fingerprint, + }); + const changed = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload: { + ...envelope( + { kind: "service.build", serviceId: "valhalla" }, + "ops1_changedabcdefghij", + undefined, + new Date().toISOString(), + new Date(Date.now() + 20_000).toISOString(), + ), + execution: { requestedDeadlineAt: admitted.deadlineAt + 1 }, + }, + }); + expect(changed.statusCode).toBe(409); + expect(calls).toBe(1); + } finally { + finish?.({ changed: true }); + await vi.advanceTimersByTimeAsync(0); + await app.close(); + vi.useRealTimers(); + } + }); + it("persists true expiry intent while unresolved containment stays pending after settlement grace", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + vi.setSystemTime(new Date(nowIso)); + let signal: AbortSignal | undefined; + let finish!: (x: unknown) => void; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + dispatch: async (_o, _r, c) => { + signal = c.signal; + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + try { + const response = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload: { + ...envelope({ kind: "service.build", serviceId: "valhalla" }), + execution: { requestedDeadlineAt: Date.now() + 100 }, + }, + }); + const id = response.json().result.operationId; + await vi.advanceTimersByTimeAsync(30_101); + expect(signal?.aborted).toBe(true); + const status = ( + await app.inject({ method: "GET", url: `/v1/operations/${id}`, headers: auth(apiToken) }) + ).json().result; + expect(status).toMatchObject({ + state: "termination_pending", + terminationRequestedAt: expect.any(String), + }); + expect(status.errorClass).toBeUndefined(); + } finally { + finish?.({ changed: true }); + await vi.advanceTimersByTimeAsync(0); + await app.close(); + vi.useRealTimers(); + } + }); + it("rejects unsigned wire parent and delegation headers before effects", async () => { + const dispatch = vi.fn(); + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + now: () => new Date(nowIso), + dispatch, + }); + try { + for (const name of ["x-openmapx-parent-execution", "x-openmapx-control-delegation"]) { + const r = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { ...auth(apiToken), [name]: "unverified" }, + payload: envelope( + { kind: "service.build", serviceId: "valhalla" }, + `ops1_${name.replaceAll("-", "")}`, + ), + }); + expect(r.statusCode).toBe(403); + } + expect(dispatch).not.toHaveBeenCalled(); + } finally { + await app.close(); + } + }); +}); + +describe("execution identity at the agent boundary", () => { + it("clamps an authenticated late API child and keeps a duplicate after expiry immutable", async () => { + const { createHmac } = await import("node:crypto"); + const { opsOperationFingerprint } = await import("@openmapx/core/ops"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + vi.setSystemTime(new Date(nowIso)); + const operation = { kind: "service.build" as const, serviceId: "valhalla" }; + const operationKey = "opk1_parentBoundChild"; + const fields = { + version: 1, + apiJobId: "123e4567-e89b-42d3-a456-426614174000", + apiOwnerVersion: 1, + parentOperationKey: "opk1_parentabcdefghij", + parentFingerprint: "a".repeat(64), + parentDeadlineAt: Date.now() + 1000, + childOperationKey: operationKey, + childOperationFingerprint: opsOperationFingerprint(operation), + requestedDeadlineAt: Date.now() + 12 * 3600_000, + attemptId: "123e4567-e89b-42d3-a456-426614174001", + }; + const payload = Buffer.from(JSON.stringify(fields)).toString("base64url"); + const assertion = + payload + + "." + + createHmac("sha256", apiToken) + .update("openmapx-api-job-parent-v1\0") + .update(payload) + .digest("base64url"); + let finish!: (x: unknown) => void; + let execution: import("@openmapx/core/ops").ExecutionRef | undefined; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + dispatch: async (_o, _r, c) => { + execution = c.execution; + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + try { + const send = (id: string) => + app.inject({ + method: "POST", + url: "/v1/operations", + headers: { ...auth(apiToken), "x-openmapx-parent-execution": assertion }, + payload: { + ...envelope( + operation, + id, + operationKey, + new Date().toISOString(), + new Date(Date.now() + 20_000).toISOString(), + ), + execution: { requestedDeadlineAt: fields.requestedDeadlineAt }, + }, + }); + const first = (await send("ops1_parentFirstChild000")).json().result; + expect(first.deadlineAt).toBe(fields.parentDeadlineAt); + expect(execution).toMatchObject({ + deadlineAt: first.deadlineAt, + fingerprint: first.fingerprint, + operationFingerprint: fields.childOperationFingerprint, + }); + await vi.advanceTimersByTimeAsync(31_001); + const duplicate = (await send("ops1_parentDuplicate000")).json().result; + expect(duplicate).toMatchObject({ + operationId: first.operationId, + deadlineAt: first.deadlineAt, + admittedAt: first.admittedAt, + fingerprint: first.fingerprint, + state: "termination_pending", + }); + } finally { + finish?.({ changed: true }); + await vi.advanceTimersByTimeAsync(0); + await app.close(); + vi.useRealTimers(); + } + }); + it("retains a redacted live tail and independent failure evidence through the journal", async () => { + const { OpsExecutionFailure } = await import("./runtime"); + let fail!: (x: unknown) => void; + const snapshots: import("./journal").PersistedOpsJob[][] = []; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + now: () => new Date(nowIso), + journal: { + records: () => [], + replace: async (records) => { + snapshots.push(structuredClone([...records])); + }, + }, + dispatch: async (_o, _r, c) => { + c.emitLog("stderr", "password=hidden"); + for (let i = 0; i < 100; i++) c.emitLog("stdout", "x".repeat(4096)); + c.emitLog("stderr", "last-diagnostic"); + return new Promise((_resolve, reject) => { + fail = reject; + }); + }, + }); + try { + const response = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload: envelope({ kind: "service.build", serviceId: "valhalla" }), + }); + const id = response.json().result.operationId; + const live = ( + await app.inject({ method: "GET", url: `/v1/operations/${id}`, headers: auth(apiToken) }) + ).json().result; + expect(live.diagnostics.truncated).toBe(true); + expect(live.diagnostics.tail[0].message).toBe("[output truncated]"); + expect(JSON.stringify(live)).not.toContain("hidden"); + fail( + new OpsExecutionFailure("runtime", { + diagnostics: { + tail: [{ stream: "stderr", message: "password=private" }], + truncated: false, + }, + }), + ); + await vi.waitFor(() => expect(snapshots.at(-1)?.[0]?.state).toBe("failed")); + const retained = snapshots.at(-1)?.[0]; + expect(retained?.failureResult?.diagnostics?.tail[0]?.message).toBe("password=[redacted]"); + expect(retained?.result).toBeUndefined(); + } finally { + await app.close(); + } + }); +}); + +it("observer abort leaves the admitted backend running until its real completion", async () => { + const { waitForOpsResult, OpsObservationPending } = await import( + "../../api/src/services/ops-client" + ); + const { parseOpsJobStatusForKind } = await import("@openmapx/core/ops"); + const observer = new AbortController(); + let backendSignal: AbortSignal | undefined; + let finish!: (x: unknown) => void; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + now: () => new Date(nowIso), + dispatch: async (_o, _r, c) => { + backendSignal = c.signal; + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + try { + const response = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload: envelope({ kind: "service.build", serviceId: "valhalla" }), + }); + const admission = response.json().result; + const status = async () => + parseOpsJobStatusForKind( + "service.build", + ( + await app.inject({ + method: "GET", + url: `/v1/operations/${admission.operationId}`, + headers: auth(apiToken), + }) + ).json().result, + ); + const cancel = vi.fn(); + await expect( + waitForOpsResult( + { status, cancel } as unknown as import("../../api/src/services/ops-client").ApiOpsClient, + "service.build", + admission, + { + signal: observer.signal, + sleep: async () => { + observer.abort(); + }, + }, + ), + ).rejects.toBeInstanceOf(OpsObservationPending); + expect(backendSignal?.aborted).toBe(false); + expect(cancel).not.toHaveBeenCalled(); + finish({ completed: true }); + await vi.waitFor(async () => expect((await status()).state).toBe("succeeded")); + } finally { + finish?.({ completed: true }); + await app.close(); + } +}); diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index ab30bdc2b..1a0fbc791 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -2,6 +2,9 @@ import { createHash, randomBytes as nodeRandomBytes, timingSafeEqual } from "nod import { type ConfigurationApplyResult, configurationApplyResultSchema, + type ExecutionRef, + executionFingerprint, + normalizeExecutionAdmission, OPS_KIND_POLICIES, OPS_MAX_CLOCK_SKEW_MS, OPS_MAX_EVENT_BATCH, @@ -13,18 +16,24 @@ import { OpsContractError, type OpsErrorClass, type OpsEventBatch, + type OpsFailureResult, type OpsJobState, type OpsJobStatus, type OpsOperation, type OpsOperationKind, type OpsResourcePolicy, type OpsRole, + opsFailureResultSchema, opsOperationFingerprint, opsResourceId, opsSuccess, + PROCESS_OUTPUT_TRUNCATED, parseBoundedOpsResult, parseOpsRequest, + processDiagnosticsSchema, redactedOpsError, + type VerifiedParentExecution, + verifyApiJobParentExecution, } from "@openmapx/core/ops"; import Fastify, { type FastifyInstance } from "fastify"; import { @@ -37,11 +46,13 @@ import { type PrivacyBackupExtractionRouteOptions, registerPrivacyBackupExtractionRoute, } from "./privacy-backup-extraction"; +import { safeProcessDiagnostic } from "./process-output"; import { createUnavailableRuntime, dispatchOpsOperation, OpsAgentBusyError, type OpsExecutionContext, + OpsExecutionFailure, OpsHostUpdateRequiredError, OpsNotWiredError, type OpsRuntime, @@ -53,7 +64,7 @@ const DEFAULT_CONCURRENCY = 4; const DEFAULT_REPLAY_ENTRIES = 10_000; const DEFAULT_JOB_ENTRIES = 256; const DEFAULT_JOB_RETENTION_MS = 60 * 60_000; -const DEFAULT_ADMISSION_TIMEOUT_MS = 30_000; +const DEFAULT_ADMISSION_TIMEOUT_MS = 15_000; const DEFAULT_CLAIM_TIMEOUT_MS = 5_000; const MAX_JOB_ID_ATTEMPTS = 8; const MAX_RETAINED_EVENTS_PER_JOB = 64; @@ -117,7 +128,7 @@ export interface BuildOpsAgentServerOptions { }>; } -interface JobRecord { +interface JobRecord extends Omit { role: OpsRole; operation: OpsOperation; operationId: string; @@ -240,6 +251,8 @@ function classify(error: unknown): { errorClass: OpsErrorClass; statusCode: numb if (error instanceof RequestGuardError) { return { errorClass: error.errorClass, statusCode: error.statusCode }; } + if (error instanceof OpsExecutionFailure) + return { errorClass: error.errorClass, statusCode: error.errorClass === "timeout" ? 504 : 500 }; if (error instanceof OpsHostUpdateRequiredError) return { errorClass: "host_update_required", statusCode: 409 }; if (error instanceof OpsAgentBusyError) return { errorClass: "busy", statusCode: 409 }; @@ -292,6 +305,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const configurationResultsSettling = new Set(); let active = 0; let admissionTail = Promise.resolve(); + let diagnosticTimer: ReturnType | undefined; + let diagnosticWriteActive = false; + const dirtyDiagnostics = new Set(); const withAdmissionLock = async (work: () => Promise): Promise => { const previous = admissionTail; @@ -347,6 +363,19 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }; const persisted = (job: JobRecord): PersistedOpsJob => ({ + ...(job.executionVersion === 2 + ? { + executionVersion: 2 as const, + admittedAt: job.admittedAt, + deadlineAt: job.deadlineAt, + operationFingerprint: job.operationFingerprint, + requestedDeadlineAt: job.requestedDeadlineAt, + deploymentBudgetMs: job.deploymentBudgetMs, + parent: job.parent, + } + : {}), + ...(job.diagnostics ? { diagnostics: job.diagnostics } : {}), + ...(job.failureResult ? { failureResult: job.failureResult } : {}), role: job.role, operation: job.operation, operationId: job.operationId, @@ -364,6 +393,32 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif : { terminationRequestedAt: job.terminationRequestedAt }), }); + const flushDiagnostics = async () => { + if (diagnosticWriteActive || dirtyDiagnostics.size === 0) return; + diagnosticWriteActive = true; + dirtyDiagnostics.clear(); + try { + await withAdmissionLock(async () => journal.replace([...jobsById.values()].map(persisted))); + } catch { + /* Projection persistence failure never terminates owned backend work. */ + } finally { + diagnosticWriteActive = false; + } + }; + const scheduleDiagnostics = (job: JobRecord) => { + dirtyDiagnostics.add(job.operationId); + if (diagnosticTimer !== undefined) return; + diagnosticTimer = setTimeout(() => { + diagnosticTimer = undefined; + void flushDiagnostics(); + }, 1000); + diagnosticTimer.unref?.(); + }; + app.addHook("onClose", async () => { + if (diagnosticTimer !== undefined) clearTimeout(diagnosticTimer); + await flushDiagnostics(); + }); + const pruneJobs = async (nowMs: number) => { await withAdmissionLock(async () => { const retained = [...jobsById.values()].filter( @@ -422,7 +477,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const transition = async ( job: JobRecord, state: OpsJobState, - updates: Partial> = {}, + updates: Partial< + Pick< + JobRecord, + "result" | "errorClass" | "terminalAt" | "terminationRequestedAt" | "failureResult" + > + > = {}, ): Promise => { await withAdmissionLock(async () => { const updatedAt = now().toISOString(); @@ -453,6 +513,21 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif pushEvent(job, { cursor: job.cursor, type: "state", state: "failed" }); }; + const identityFor = (job: JobRecord) => + job.executionVersion === 2 + ? { admittedAt: job.admittedAt, deadlineAt: job.deadlineAt, fingerprint: job.fingerprint } + : {}; + const executionFor = (job: JobRecord): ExecutionRef => + Object.freeze({ + executor: "ops-agent", + principal: job.role, + executionId: job.operationId, + operationKey: job.operationKey, + operationFingerprint: job.operationFingerprint ?? job.fingerprint, + fingerprint: job.fingerprint, + // Compatibility records never resume a forward execution or gain a new budget. + deadlineAt: job.deadlineAt ?? Date.parse(job.submittedAt), + }); const contextFor = ( job: JobRecord, signal: AbortSignal, @@ -460,6 +535,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ): OpsExecutionContext => ({ signal, claim, + execution: executionFor(job), otherActiveOperations: () => [...jobsById.values()].filter( (other) => @@ -469,7 +545,43 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif other.state === "termination_pending"), ).length, emitLog: (stream, message) => { - const boundedMessage = truncateUtf8(message, OPS_MAX_EVENT_MESSAGE_BYTES); + const boundedMessage = truncateUtf8( + safeProcessDiagnostic(message), + OPS_MAX_EVENT_MESSAGE_BYTES, + ); + const tail = [ + ...(job.diagnostics?.tail ?? []).filter( + (entry) => entry.message !== PROCESS_OUTPUT_TRUNCATED, + ), + { stream, message: boundedMessage }, + ]; + let truncated = job.diagnostics?.truncated ?? false; + while ( + tail.length > 255 || + Buffer.byteLength(JSON.stringify({ tail, truncated })) > 16 * 1024 - 128 + ) { + const stdoutBytes = tail + .filter((entry) => entry.stream === "stdout") + .reduce((bytes, entry) => bytes + Buffer.byteLength(JSON.stringify(entry)), 0); + const stderrBytes = tail + .filter((entry) => entry.stream === "stderr") + .reduce((bytes, entry) => bytes + Buffer.byteLength(JSON.stringify(entry)), 0); + tail.splice( + tail.findIndex( + (entry) => entry.stream === (stdoutBytes >= stderrBytes ? "stdout" : "stderr"), + ), + 1, + ); + truncated = true; + } + job.diagnostics = { + tail: [ + ...(truncated ? [{ stream: "stderr" as const, message: PROCESS_OUTPUT_TRUNCATED }] : []), + ...tail, + ], + truncated, + }; + scheduleDiagnostics(job); job.cursor += 1; pushEvent(job, { cursor: job.cursor, @@ -481,7 +593,10 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }); const statusFor = (job: JobRecord): OpsJobStatus => ({ - version: 1, + version: job.executionVersion === 2 ? 2 : 1, + ...(job.executionVersion === 2 ? identityFor(job) : {}), + ...(job.diagnostics ? { diagnostics: job.diagnostics } : {}), + ...(job.failureResult ? { failureResult: job.failureResult } : {}), operationId: job.operationId, operationKey: job.operationKey, kind: job.operation.kind, @@ -516,23 +631,28 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } const controller = new AbortController(); activeControllers.set(job.operationId, controller); - const timeoutMs = Math.min( - OPS_KIND_POLICIES[job.operation.kind].timeoutMs, - options.operationTimeoutMs ?? Number.POSITIVE_INFINITY, - ); + const timeoutMs = Math.max(0, (job.deadlineAt ?? now().getTime()) - now().getTime()); let timedOut = false; let terminalDurable = false; let claimReleased = false; let configurationEvidence: ConfigurationApplyResult | undefined; const timer = setTimeout(() => { timedOut = true; - void transition(job, "termination_pending").then( + void transition(job, "termination_pending", { + terminationRequestedAt: job.terminationRequestedAt ?? now().toISOString(), + }).then( () => controller.abort(), () => controller.abort(), ); }, timeoutMs); const settlement = Promise.resolve() - .then(() => dispatch(job.operation, runtime, contextFor(job, controller.signal, claim))) + .then(() => { + if ((job.deadlineAt ?? 0) <= now().getTime()) { + timedOut = true; + throw new OpsExecutionFailure("timeout", {}); + } + return dispatch(job.operation, runtime, contextFor(job, controller.signal, claim)); + }) .then( async (rawResult) => { clearTimeout(timer); @@ -681,8 +801,29 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const classification = timedOut ? { errorClass: "timeout" as const } : classify(error); + let failureResult: OpsFailureResult | undefined; + if (error instanceof OpsExecutionFailure) { + const safe = opsFailureResultSchema.parse(error.failureResult); + if (safe.diagnostics) + safe.diagnostics = { + ...safe.diagnostics, + tail: safe.diagnostics.tail.map((entry) => ({ + ...entry, + message: safeProcessDiagnostic(entry.message), + })), + }; + failureResult = opsFailureResultSchema.parse(safe); + if ( + Buffer.byteLength(JSON.stringify(failureResult)) > + OPS_KIND_POLICIES[job.operation.kind].maxResultBytes + ) + failureResult = undefined; + if (failureResult?.diagnostics) + job.diagnostics = processDiagnosticsSchema.parse(failureResult.diagnostics); + } await transition(job, timedOut ? "timed_out" : "failed", { result: undefined, + failureResult, errorClass: classification.errorClass, terminalAt: now().getTime(), }); @@ -765,13 +906,70 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const timing = validateTiming(parsed.issuedAt, parsed.expiresAt, nowMs); replay.accept(parsed.requestId, timing.expiresAtMs, nowMs); const policy = OPS_KIND_POLICIES[operation.kind]; - const fingerprint = opsOperationFingerprint(operation); + const operationFingerprint = opsOperationFingerprint(operation); + let parent: VerifiedParentExecution | undefined; + const parentHeader = request.headers["x-openmapx-parent-execution"]; + const delegationHeader = request.headers["x-openmapx-control-delegation"]; + const controlHeaders = request.raw.rawHeaders + .filter((_, i) => i % 2 === 0) + .map((x) => x.toLowerCase()); + if ( + delegationHeader !== undefined || + controlHeaders.filter((x) => x === "x-openmapx-parent-execution").length > 1 + ) + throw new RequestGuardError("authorization", 403); + if (parentHeader !== undefined) { + if (role !== "api" || typeof parentHeader !== "string") + throw new RequestGuardError("authorization", 403); + try { + parent = verifyApiJobParentExecution(parentHeader, options.tokens.api, { + operationKey: parsed.operationKey, + operationFingerprint, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + }); + } catch { + throw new RequestGuardError("authorization", 403); + } + } + const normalize = ( + originalAdmittedAt: number, + deploymentBudgetMs = Math.min( + policy.timeoutMs, + options.operationTimeoutMs ?? policy.timeoutMs, + ), + ) => { + try { + return normalizeExecutionAdmission({ + kind, + operationKey: parsed.operationKey, + operationFingerprint, + admittedAt: originalAdmittedAt, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + parent, + deploymentBudgetMs, + }); + } catch { + throw new RequestGuardError("validation", 400); + } + }; + let normalized: ReturnType; + const fingerprint = operationFingerprint; // Existing sealed claims bind the body, never execution timing. + const matchesRetained = (job: JobRecord): boolean => { + try { + return job.executionVersion === 2 + ? job.fingerprint === + normalize(job.admittedAt as number, job.deploymentBudgetMs).fingerprint + : job.fingerprint === operationFingerprint && parsed.execution === undefined && !parent; + } catch { + return false; + } + }; if (policy.execution === "async") { await pruneJobs(nowMs); const ownerKey = `${role}:${parsed.operationKey}`; const existing = jobsByKey.get(ownerKey); if (existing) { - if (existing.fingerprint !== fingerprint) throw new RequestGuardError("conflict", 409); + if (!matchesRetained(existing)) throw new RequestGuardError("conflict", 409); return reply.code(202).send( opsSuccess(parsed.requestId, { execution: "async", @@ -779,6 +977,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif operationKey: existing.operationKey, kind: existing.operation.kind, state: outwardStateFor(existing), + version: existing.executionVersion === 2 ? 2 : 1, + ...identityFor(existing), }), ); } @@ -790,6 +990,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("busy", 409); } } + normalized = normalize(nowMs); if (active >= maxConcurrency) throw new RequestGuardError("busy", 429); active += 1; const claimController = new AbortController(); @@ -872,13 +1073,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const admission = await withAdmissionLock(async () => { const retained = jobsByKey.get(ownerKey); if (retained) { - if (retained.fingerprint !== fingerprint) { + if (!matchesRetained(retained)) { throw new RequestGuardError("conflict", 409); } return { type: "retained", job: retained } as const; } if (jobsById.size >= maxJobEntries) throw new RequestGuardError("busy", 429); - const timestamp = now().toISOString(); + if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); + const timestamp = new Date(normalized.admittedAt).toISOString(); let operationId: string | undefined; try { for (let attempt = 0; attempt < MAX_JOB_ID_ATTEMPTS; attempt += 1) { @@ -897,7 +1099,15 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif operation, operationId, operationKey: parsed.operationKey, - fingerprint, + executionVersion: 2, + ...normalized, + operationFingerprint, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + parent, + deploymentBudgetMs: Math.min( + policy.timeoutMs, + options.operationTimeoutMs ?? policy.timeoutMs, + ), resourceId, state: "queued", submittedAt: timestamp, @@ -926,11 +1136,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif operationKey: admission.job.operationKey, kind: admission.job.operation.kind, state: outwardStateFor(admission.job), + version: admission.job.executionVersion === 2 ? 2 : 1, + ...identityFor(admission.job), }), ); } const { job } = admission; try { + if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); await claimAdmission?.commit(); } catch { stickyRecoveryRequired.add(job.operationId); @@ -951,27 +1164,37 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif operationKey: job.operationKey, kind: job.operation.kind, state: outwardStateFor(job), + version: job.executionVersion === 2 ? 2 : 1, + ...identityFor(job), }), ); } const controller = new AbortController(); try { + if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); await claimAdmission?.commit(); } catch { active -= 1; throw new RequestGuardError("runtime", 500); } - const timeoutMs = Math.min( - policy.timeoutMs, - options.operationTimeoutMs ?? Number.POSITIVE_INFINITY, - ); - const execution = Promise.resolve().then(() => - dispatch(operation, runtime, { + const timeoutMs = Math.max(0, normalized.deadlineAt - now().getTime()); + const execution = Promise.resolve().then(() => { + if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); + return dispatch(operation, runtime, { signal: controller.signal, + execution: Object.freeze({ + executor: "ops-agent", + principal: role, + executionId: `job1_${Buffer.from(randomBytes(18)).toString("base64url")}`, + operationKey: parsed.operationKey, + operationFingerprint, + fingerprint: normalized.fingerprint, + deadlineAt: normalized.deadlineAt, + }), emitLog: () => undefined, claim, - }), - ); + }); + }); let timeout: ReturnType | undefined; try { const timeoutResult = new Promise((_resolve, reject) => { @@ -1037,11 +1260,42 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const job = jobsByKey.get(`${role}:${operationKey}`); if (!job) return reply.code(404).send(redactedOpsError(requestId, "not_found")); const fingerprint = request.headers["x-ops-operation-fingerprint"]; - if (typeof fingerprint !== "string" || !/^[a-f0-9]{64}$/.test(fingerprint)) { - return reply.code(400).send(redactedOpsError(requestId, "validation")); - } - if (fingerprint !== job.fingerprint) { - return reply.code(409).send(redactedOpsError(requestId, "conflict")); + const bodyFingerprint = request.headers["x-ops-body-fingerprint"]; + // Explicit compatibility: a v1 lookup used the body hash in the sole header. + if (bodyFingerprint !== undefined) { + if (bodyFingerprint !== (job.operationFingerprint ?? job.fingerprint)) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); + if (fingerprint !== undefined && fingerprint !== job.fingerprint) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); + const requested = request.headers["x-ops-requested-deadline"]; + if (requested !== undefined) { + const requestedDeadlineAt = Number(requested); + if ( + typeof requested !== "string" || + !Number.isSafeInteger(requestedDeadlineAt) || + requestedDeadlineAt < 0 + ) + return reply.code(400).send(redactedOpsError(requestId, "validation")); + const deadlineAt = Math.min( + requestedDeadlineAt, + job.parent?.deadlineAt ?? Infinity, + (job.admittedAt ?? 0) + (job.deploymentBudgetMs ?? 0), + ); + if ( + executionFingerprint({ + kind: job.operation.kind, + operationFingerprint: bodyFingerprint as string, + deadlineAt, + ...(job.parent ? { parent: job.parent } : {}), + }) !== job.fingerprint + ) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); + } + } else { + if (typeof fingerprint !== "string" || !/^[a-f0-9]{64}$/.test(fingerprint)) + return reply.code(400).send(redactedOpsError(requestId, "validation")); + if (fingerprint !== job.fingerprint && fingerprint !== job.operationFingerprint) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); } return reply.code(200).send(opsSuccess(requestId, statusFor(job))); }); @@ -1075,6 +1329,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (!job || job.role !== role) { return reply.code(404).send(redactedOpsError(requestId, "not_found")); } + if ( + job.executionVersion === 2 && + request.headers["x-ops-operation-fingerprint"] !== job.fingerprint + ) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); if (job.terminalAt !== undefined) { return reply.code(200).send(opsSuccess(requestId, statusFor(job))); } diff --git a/apps/ops-agent/src/transitous-lock-runtime.test.ts b/apps/ops-agent/src/transitous-lock-runtime.test.ts index 99c82f984..31236ceb9 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.test.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.test.ts @@ -2,6 +2,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; import { createTransitousLockRuntime } from "./transitous-lock-runtime"; @@ -18,6 +19,7 @@ function root(): string { function context() { return { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: { diff --git a/apps/ops-agent/src/trusted-config-ownership.test.ts b/apps/ops-agent/src/trusted-config-ownership.test.ts index c91c87a72..34577fa75 100644 --- a/apps/ops-agent/src/trusted-config-ownership.test.ts +++ b/apps/ops-agent/src/trusted-config-ownership.test.ts @@ -10,6 +10,7 @@ import { } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; import { afterEach, describe, expect, it } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; import type { PersistedOpsJob } from "./journal"; import { createRegistryResourceClaimer } from "./policy"; import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; @@ -79,6 +80,7 @@ function fixture() { ...(expected !== undefined ? { expectedCurrentRevisionId: expected } : {}), } as OpsOperation & { kind: "stack.render" }; return dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog() {}, claim: { @@ -147,6 +149,7 @@ describe("trusted configuration ownership and outcomes", () => { ...(kind === "integrationConfig.apply" ? { integrationId: "demo" } : {}), } as OpsOperation; return dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog() {}, claim: { @@ -209,6 +212,7 @@ describe("trusted configuration ownership and outcomes", () => { const operation = { kind: "configuration.inspect" as const }; let settled = false; const observation = dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog() {}, claim: { @@ -309,6 +313,7 @@ describe("trusted configuration ownership and outcomes", () => { const operation = { ...claimed, expectedCurrentRevisionId: revision("b") }; await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog() {}, claim: { diff --git a/apps/ops-agent/src/trusted-config-runtime.test.ts b/apps/ops-agent/src/trusted-config-runtime.test.ts index e80dad420..7aab0acbc 100644 --- a/apps/ops-agent/src/trusted-config-runtime.test.ts +++ b/apps/ops-agent/src/trusted-config-runtime.test.ts @@ -20,6 +20,7 @@ import { } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; import { createUnavailableRuntime, dispatchOpsOperation, type OpsTrustedClaim } from "./runtime"; import { initializeTrustedConfigurationRuntime, @@ -155,6 +156,7 @@ describe("ops-agent trusted configuration runtime", () => { }), ); const result = await dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: trusted, @@ -209,6 +211,7 @@ describe("ops-agent trusted configuration runtime", () => { }), ); await dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: trusted, @@ -283,6 +286,7 @@ describe("ops-agent trusted configuration runtime", () => { revisionId, }, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -318,6 +322,7 @@ describe("ops-agent trusted configuration runtime", () => { select(paths.infraDir, ["app-api"]); await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -352,6 +357,7 @@ describe("ops-agent trusted configuration runtime", () => { }; select(paths.infraDir, ["app-api", "alpha"]); await dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -426,6 +432,7 @@ describe("ops-agent trusted configuration runtime", () => { }; await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -466,6 +473,7 @@ describe("ops-agent trusted configuration runtime", () => { select(paths.infraDir, ["dawarich-app"]); await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -507,6 +515,7 @@ describe("ops-agent trusted configuration runtime", () => { }; await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -552,6 +561,7 @@ describe("ops-agent trusted configuration runtime", () => { }; await expect( dispatchOpsOperation(runtime, selection, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -587,6 +597,7 @@ describe("ops-agent trusted configuration runtime", () => { }; await expect( dispatchOpsOperation(runtime, config, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -613,6 +624,7 @@ describe("ops-agent trusted configuration runtime", () => { }; await expect( dispatchOpsOperation(runtime, removed, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -650,6 +662,7 @@ describe("ops-agent trusted configuration runtime", () => { revisionId, }, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -690,6 +703,7 @@ describe("ops-agent trusted configuration runtime", () => { revisionId, }, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -727,6 +741,7 @@ describe("ops-agent trusted configuration runtime", () => { }); await expect( dispatchOpsOperation(crashing, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ integrationConfigs: [] })), @@ -745,6 +760,7 @@ describe("ops-agent trusted configuration runtime", () => { }); await expect( dispatchOpsOperation(retry, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ integrationConfigs: [] })), @@ -780,6 +796,7 @@ describe("ops-agent trusted configuration runtime", () => { infraDir: paths.infraDir, }); await dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ integrationConfigs: [] })), @@ -793,6 +810,7 @@ describe("ops-agent trusted configuration runtime", () => { await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ integrationConfigs: [] })), @@ -820,6 +838,7 @@ describe("ops-agent trusted configuration runtime", () => { }); await expect( dispatchOpsOperation(crashing, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ integrationConfigs: [] })), @@ -833,6 +852,7 @@ describe("ops-agent trusted configuration runtime", () => { }); await expect( dispatchOpsOperation(retry, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ integrationConfigs: [] })), @@ -860,6 +880,7 @@ describe("ops-agent trusted configuration runtime", () => { revisionId: firstRevision, }; await dispatchOpsOperation(runtime, first, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -882,6 +903,7 @@ describe("ops-agent trusted configuration runtime", () => { revisionId: secondRevision, }; await dispatchOpsOperation(runtime, second, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim( @@ -928,6 +950,7 @@ describe("ops-agent trusted configuration runtime", () => { }; await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ integrationConfigs: [] })), @@ -962,6 +985,7 @@ describe("ops-agent trusted configuration runtime", () => { }; await expect( dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: claim(operation, payload({ selectedRoots: ["beta"], integrationConfigs: [] })), @@ -974,6 +998,7 @@ describe("ops-agent trusted configuration runtime", () => { runtime, { kind: "serviceSelection.inspect" }, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: { @@ -1028,6 +1053,7 @@ describe("ops-agent trusted configuration runtime", () => { revisionId: inspectRevision, }; return dispatchOpsOperation(runtime, operation, { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), claim: { diff --git a/packages/core/package.json b/packages/core/package.json index a40f59a95..cd4f9eb5e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -14,6 +14,10 @@ "types": "./src/index.ts", "default": "./src/index.ts" }, + "./scrub-secrets": { + "types": "./src/scrub-secrets.ts", + "default": "./src/scrub-secrets.ts" + }, "./release-files": { "types": "./src/release-files.ts", "default": "./src/release-files.ts" diff --git a/packages/core/src/ops/client.test.ts b/packages/core/src/ops/client.test.ts index eedafe66f..71189dbc8 100644 --- a/packages/core/src/ops/client.test.ts +++ b/packages/core/src/ops/client.test.ts @@ -21,8 +21,8 @@ function tokenFile(contents = Buffer.alloc(32, 7).toString("base64url")): string describe("ops client", () => { it("derives synchronous transport timeouts from the per-kind execution budget", () => { - expect(opsRequestTimeoutMs("docker.status")).toBe(10_000); - expect(opsRequestTimeoutMs("service.logs")).toBe(35_000); + expect(opsRequestTimeoutMs("docker.status")).toBe(15_000); + expect(opsRequestTimeoutMs("service.logs")).toBe(15_000); expect(opsRequestTimeoutMs("service.logs", 7)).toBe(7); expect(opsRequestTimeoutMs("service.pull")).toBe(15_000); }); @@ -41,7 +41,7 @@ describe("ops client", () => { const fetch = vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { const body = JSON.parse(String(init?.body)) as Record; expect(body).toMatchObject({ - version: 1, + version: 2, requestId: "ops1_AQEBAQEBAQEBAQEBAQEBAQEB", operationKey: "opk1_AQEBAQEBAQEBAQEBAQEBAQEB", issuedAt: "2026-08-23T18:00:00.000Z", @@ -248,6 +248,10 @@ describe("ops client", () => { requestId: body.requestId, ok: true, result: { + version: 2, + admittedAt: 0, + deadlineAt: 1000, + fingerprint: "a".repeat(64), execution: "async", operationId: "job1_reusableOperation1", operationKey: body.operationKey, @@ -281,7 +285,7 @@ describe("ops client", () => { const token = Buffer.alloc(32, 17).toString("base64url"); const fetch = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { expect(String(input)).toBe(`http://ops-agent:4300/v1/operations/${operationId}`); - expect(init?.method).toBe("DELETE"); + expect(["GET", "DELETE"]).toContain(init?.method); expect(new Headers(init?.headers).get("authorization")).toBe(`Bearer ${token}`); const requestId = new Headers(init?.headers).get("x-ops-request-id"); return new Response( @@ -326,7 +330,7 @@ describe("ops client", () => { const fingerprints: string[] = []; const fetch = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { expect(String(input)).toBe(`http://ops-agent:4300/v1/operation-keys/${operationKey}`); - const fingerprint = new Headers(init?.headers).get("x-ops-operation-fingerprint"); + const fingerprint = new Headers(init?.headers).get("x-ops-body-fingerprint"); expect(fingerprint).toMatch(/^[a-f0-9]{64}$/); fingerprints.push(fingerprint as string); const requestId = new Headers(init?.headers).get("x-ops-request-id"); @@ -442,6 +446,10 @@ describe("ops client", () => { requestId: body.requestId, ok: true, result: { + version: 2, + admittedAt: 0, + deadlineAt: 1000, + fingerprint: "a".repeat(64), execution: "async", operationId: `job1_retainedClient${index}00`, operationKey: body.operationKey, diff --git a/packages/core/src/ops/client.ts b/packages/core/src/ops/client.ts index a36341cfd..faef81583 100644 --- a/packages/core/src/ops/client.ts +++ b/packages/core/src/ops/client.ts @@ -26,9 +26,8 @@ import { opsOperationFingerprint } from "./fingerprint"; const TOKEN_BYTES = 43; const DEFAULT_REQUEST_TIMEOUT_MS = 15_000; const DEFAULT_RESPONSE_BYTES = OPS_MAX_HTTP_RESPONSE_BYTES; -const MAX_REQUEST_TIMEOUT_MS = 30 * 60_000; +const MAX_REQUEST_TIMEOUT_MS = 15_000; const MAX_RESPONSE_BYTES = OPS_MAX_HTTP_RESPONSE_BYTES; -const TRANSPORT_MARGIN_MS = 5_000; const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/; const ERROR_CLASSES = new Set([ "authentication", @@ -160,7 +159,7 @@ function parseResponse(raw: string, requestId: string, status: number): OpsRespo } const envelope = value as Record; if ( - envelope.version !== OPS_PROTOCOL_VERSION || + envelope.version !== 1 || envelope.requestId !== requestId || typeof envelope.ok !== "boolean" ) { @@ -201,11 +200,9 @@ export interface OpsClientOptions { randomBytes?: (size: number) => Uint8Array; } -export function opsRequestTimeoutMs(kind: OpsOperationKind, configuredTimeoutMs?: number): number { +export function opsRequestTimeoutMs(_kind: OpsOperationKind, configuredTimeoutMs?: number): number { if (configuredTimeoutMs !== undefined) return configuredTimeoutMs; - const policy = OPS_KIND_POLICIES[kind]; - if (policy.execution === "async") return DEFAULT_REQUEST_TIMEOUT_MS; - return Math.min(policy.timeoutMs + TRANSPORT_MARGIN_MS, MAX_REQUEST_TIMEOUT_MS); + return DEFAULT_REQUEST_TIMEOUT_MS; } export class OpsClient { @@ -247,7 +244,13 @@ export class OpsClient { private async get( path: string, - options: { method?: "GET" | "DELETE"; signal?: AbortSignal; fingerprint?: string } = {}, + options: { + method?: "GET" | "DELETE"; + signal?: AbortSignal; + fingerprint?: string; + bodyFingerprint?: string; + requestedDeadlineAt?: number; + } = {}, ): Promise { const token = await readOpsTokenFile(this.tokenFile); const requestId = `ops1_${Buffer.from(this.randomBytes(18)).toString("base64url")}`; @@ -265,6 +268,10 @@ export class OpsClient { authorization: `Bearer ${token}`, "x-ops-request-id": requestId, ...(options.fingerprint ? { "x-ops-operation-fingerprint": options.fingerprint } : {}), + ...(options.bodyFingerprint ? { "x-ops-body-fingerprint": options.bodyFingerprint } : {}), + ...(options.requestedDeadlineAt === undefined + ? {} + : { "x-ops-requested-deadline": String(options.requestedDeadlineAt) }), }, signal: signals.length === 1 ? signals[0] : AbortSignal.any(signals), }); @@ -335,18 +342,32 @@ export class OpsClient { } } - async cancel(operationId: string, options: { signal?: AbortSignal } = {}): Promise { + async cancel( + operationId: string, + options: { signal?: AbortSignal; fingerprint?: string } = {}, + ): Promise { if (!/^job1_[A-Za-z0-9_-]{16,64}$/.test(operationId)) { throw new OpsClientError("validation"); } try { + const retained = options.fingerprint + ? undefined + : parseOpsJobStatus( + await this.get(`/v1/operations/${operationId}`, { signal: options.signal }), + ); const status = parseOpsJobStatus( await this.get(`/v1/operations/${operationId}`, { method: "DELETE", signal: options.signal, + fingerprint: options.fingerprint ?? retained?.fingerprint, }), ); if (status.operationId !== operationId) throw new Error("Operation status ID mismatch"); + if ( + status.version === 2 && + status.fingerprint !== (options.fingerprint ?? retained?.fingerprint) + ) + throw new Error("Operation cancellation identity mismatch"); return status; } catch (error) { if (error instanceof OpsClientError) throw error; @@ -358,7 +379,11 @@ export class OpsClient { async lookup( operation: Extract, operationKey: string, - options: { signal?: AbortSignal } = {}, + options: { + signal?: AbortSignal; + fingerprint?: string; + execution?: { requestedDeadlineAt: number }; + } = {}, ): Promise> { if (!/^opk1_[A-Za-z0-9_-]{16,64}$/.test(operationKey)) throw new OpsClientError("validation"); try { @@ -366,7 +391,9 @@ export class OpsClient { operation.kind, await this.get(`/v1/operation-keys/${operationKey}`, { signal: options.signal, - fingerprint: opsOperationFingerprint(operation), + bodyFingerprint: opsOperationFingerprint(operation), + fingerprint: options.fingerprint, + requestedDeadlineAt: options.execution?.requestedDeadlineAt, }), {}, ); @@ -383,7 +410,13 @@ export class OpsClient { async execute( operation: Extract, - options: { operationKey?: string; signal?: AbortSignal } = {}, + options: { + operationKey?: string; + signal?: AbortSignal; + execution?: { requestedDeadlineAt: number }; + controlAssertion?: string; + parentAssertion?: string; + } = {}, ): Promise> { if (OPS_KIND_POLICIES[operation.kind].execution === "async" && !options.operationKey) { throw new Error("Async operation key is required"); @@ -401,6 +434,7 @@ export class OpsClient { issued.getTime() + Math.min(20_000, OPS_MAX_REQUEST_TTL_MS), ).toISOString(), operation, + ...(options.execution ? { execution: options.execution } : {}), }; const timeoutSignal = AbortSignal.timeout( opsRequestTimeoutMs(operation.kind, this.configuredRequestTimeoutMs), @@ -415,6 +449,12 @@ export class OpsClient { headers: { authorization: `Bearer ${token}`, "content-type": "application/json", + ...(options.controlAssertion + ? { "x-openmapx-control-delegation": options.controlAssertion } + : {}), + ...(options.parentAssertion + ? { "x-openmapx-parent-execution": options.parentAssertion } + : {}), }, body: JSON.stringify(request), signal: signals.length === 1 ? signals[0] : AbortSignal.any(signals), @@ -446,7 +486,27 @@ export class OpsClient { } as OpsSubmitResult; } if ( - !exactKeys(submit, ["execution", "operationId", "operationKey", "kind", "state"]) || + !(submit.version === 2 + ? exactKeys(submit, [ + "version", + "execution", + "operationId", + "operationKey", + "kind", + "state", + "admittedAt", + "deadlineAt", + "fingerprint", + ]) + : submit.version === 1 && + exactKeys(submit, [ + "version", + "execution", + "operationId", + "operationKey", + "kind", + "state", + ])) || submit.execution !== "async" || submit.kind !== operation.kind || submit.operationKey !== operationKey || @@ -458,6 +518,21 @@ export class OpsClient { ) { throw new OpsClientError("runtime", response.status); } + if ( + submit.version === 2 || + submit.admittedAt !== undefined || + submit.deadlineAt !== undefined || + submit.fingerprint !== undefined + ) { + if ( + !Number.isSafeInteger(submit.admittedAt) || + !Number.isSafeInteger(submit.deadlineAt) || + (submit.deadlineAt as number) <= (submit.admittedAt as number) || + typeof submit.fingerprint !== "string" || + !/^[a-f0-9]{64}$/.test(submit.fingerprint) + ) + throw new OpsClientError("runtime"); + } return submit as unknown as OpsSubmitResult; } catch (error) { if (error instanceof OpsClientError) throw error; diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index ad2d27129..dc82977cf 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -236,7 +236,7 @@ describe("ops contract", () => { }); }); - it("preserves budgets outside the wired lifecycle operations", () => { + it("retains short operation budgets and expands long build budgets", () => { for (const [kind, timeoutMs] of [ ["docker.status", 5_000], ["stack.status", 15_000], @@ -244,8 +244,8 @@ describe("ops contract", () => { ["service.logs", 30_000], ["service.logs.follow", 900_000], ["service.pull", 900_000], - ["service.build", 1_800_000], - ["services.buildAll", 1_800_000], + ["service.build", 43_200_000], + ["services.buildAll", 86_400_000], ["stack.stop", 300_000], ["service.update", 900_000], ] as const) diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 95a8e4cf9..a9ec5ee02 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -1,12 +1,13 @@ import z from "zod/v4"; +import { opsFailureResultSchema, processDiagnosticsSchema } from "./job-evidence"; -export const OPS_PROTOCOL_VERSION = 1 as const; +export const OPS_PROTOCOL_VERSION = 2 as const; export const OPS_MAX_LOG_TAIL = 2_000; export const OPS_MAX_EVENT_MESSAGE_BYTES = 4_096; export const OPS_MAX_EVENT_BATCH = 100; export const OPS_MAX_FOLLOW_LOG_EVENTS = 2_000; export const OPS_MAX_RESULT_BYTES = 1024 * 1024; -export const OPS_MAX_HTTP_RESPONSE_BYTES = OPS_MAX_RESULT_BYTES + 16 * 1024; +export const OPS_MAX_HTTP_RESPONSE_BYTES = OPS_MAX_RESULT_BYTES + 32 * 1024; export const OPS_MAX_REQUEST_TTL_MS = 30_000; export const OPS_MAX_CLOCK_SKEW_MS = 5_000; export const OPS_MAX_BACKUP_ID_LENGTH = 128; @@ -144,8 +145,8 @@ export const OPS_KIND_POLICIES = { "service.pull": asyncEffect("api", 15 * 60_000), "service.remove": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "service.update": asyncEffect("api", 15 * 60_000), - "service.build": asyncEffect("api", 30 * 60_000), - "services.buildAll": asyncEffect("api", 30 * 60_000, 16 * 1024), + "service.build": asyncEffect("api", 12 * 3600_000), + "services.buildAll": asyncEffect("api", 24 * 3600_000, 64 * 1024), "service.logs": sync("api", 30_000, 1024 * 1024), "service.logs.follow": asyncEffect("api", 15 * 60_000, 64 * 1024), "dawarich.provisioning.inspect": sync("api", 30_000), @@ -163,7 +164,7 @@ export const OPS_KIND_POLICIES = { "backup.delete": asyncEffect("api", 5 * 60_000), "system.diagnostics": asyncEffect("api", 10 * 60_000, 256 * 1024), "system.inspect": sync("api", 30_000, 256 * 1024), - "system.update": asyncEffect("api", 30 * 60_000), + "system.update": asyncEffect("api", 48 * 3600_000), "serviceRepository.inspect": asyncEffect("api", 5 * 60_000, 1024 * 1024), "serviceRepository.publish": asyncEffect("api", 10 * 60_000), "serviceRepository.settle": asyncEffect("api", 5 * 60_000), @@ -175,17 +176,17 @@ export const OPS_KIND_POLICIES = { "integrationConfig.apply": asyncEffect("api", 2 * 60_000), "vault.apply": asyncEffect("api", 2 * 60_000), "data.inspect": sync("api", 30_000, 256 * 1024), - "data.downloadOsm": asyncEffect("api", 30 * 60_000), - "data.downloadFonts": asyncEffect("api", 15 * 60_000), - "data.update": asyncEffect("api", 30 * 60_000), - "data.convertOverpass": asyncEffect("api", 30 * 60_000), + "data.downloadOsm": asyncEffect("api", 24 * 3600_000), + "data.downloadFonts": asyncEffect("api", 24 * 3600_000), + "data.update": asyncEffect("api", 24 * 3600_000), + "data.convertOverpass": asyncEffect("api", 24 * 3600_000), "data.link": asyncEffect("api", 10 * 60_000), "data.clean": asyncEffect("api", 10 * 60_000), "data.generateApiKeys": asyncEffect("api", 10 * 60_000), - "data.overtureSync": asyncEffect("api", 30 * 60_000), - "data.overtureConflate": asyncEffect("api", 30 * 60_000), - "data.searchIndexBuild": asyncEffect("api", 30 * 60_000), - "data.notablePlacesBuild": asyncEffect("api", 30 * 60_000), + "data.overtureSync": asyncEffect("api", 24 * 3600_000), + "data.overtureConflate": asyncEffect("api", 24 * 3600_000), + "data.searchIndexBuild": asyncEffect("api", 24 * 3600_000), + "data.notablePlacesBuild": asyncEffect("api", 24 * 3600_000), "motis.staging.restart": asyncEffect("data-manager", 2 * 60_000), "motis.staging.stop": asyncEffect("data-manager", 2 * 60_000), "motis.primary.restart": asyncEffect("data-manager", 2 * 60_000), @@ -485,12 +486,15 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ export type OpsOperation = z.infer; const opsRequestSchema = z.strictObject({ - version: z.literal(OPS_PROTOCOL_VERSION), + version: z.union([z.literal(1), z.literal(OPS_PROTOCOL_VERSION)]), requestId: requestIdSchema, operationKey: operationKeySchema, issuedAt: timestampSchema, expiresAt: timestampSchema, operation: opsOperationSchema, + execution: z + .strictObject({ requestedDeadlineAt: z.number().int().nonnegative().safe() }) + .optional(), }); export type OpsRequest = z.infer; @@ -1192,7 +1196,15 @@ const errorClass = z.enum( Object.keys(OPS_PUBLIC_ERROR_MESSAGES) as [OpsErrorClass, ...OpsErrorClass[]], ); const jobStatus = z.strictObject({ - version: z.literal(1), + version: z.union([z.literal(1), z.literal(2)]), + admittedAt: z.number().int().nonnegative().safe().optional(), + deadlineAt: z.number().int().nonnegative().safe().optional(), + fingerprint: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + diagnostics: processDiagnosticsSchema.optional(), + failureResult: opsFailureResultSchema.optional(), operationId: operationIdSchema, operationKey: operationKeySchema, kind: operationKindSchema, @@ -1211,6 +1223,29 @@ const jobStatus = z.strictObject({ export type OpsJobStatus = z.infer; export function parseOpsJobStatus(input: unknown): OpsJobStatus { const status = jobStatus.parse(input); + if ( + status.version === 1 && + (status.admittedAt !== undefined || + status.deadlineAt !== undefined || + status.fingerprint !== undefined) + ) + throw new Error("Invalid compatibility identity"); + if ( + status.version === 2 && + (status.admittedAt === undefined || + status.deadlineAt === undefined || + status.fingerprint === undefined || + status.deadlineAt <= status.admittedAt) + ) + throw new Error("Invalid execution identity"); + if (status.failureResult !== undefined && !["failed", "timed_out"].includes(status.state)) + throw new Error("Invalid failure evidence"); + if ( + status.failureResult && + new TextEncoder().encode(JSON.stringify(status.failureResult)).byteLength > + OPS_KIND_POLICIES[status.kind].maxResultBytes + ) + throw new Error("Failure evidence exceeds budget"); if (status.state === "succeeded") { if (status.result === undefined || status.errorClass !== undefined) throw new Error("Invalid operation status"); @@ -1302,6 +1337,10 @@ export function parseOpsEventBatch( } export interface OpsAdmission { + version?: 1 | 2; + admittedAt?: number; + deadlineAt?: number; + fingerprint?: string; execution: "async"; operationId: string; operationKey: string; diff --git a/packages/core/src/ops/execution-identity.test.ts b/packages/core/src/ops/execution-identity.test.ts new file mode 100644 index 000000000..bc13a5167 --- /dev/null +++ b/packages/core/src/ops/execution-identity.test.ts @@ -0,0 +1,100 @@ +import { createHmac } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { OPS_KIND_POLICIES } from "./contract"; +import { + normalizeExecutionAdmission, + opsExecutionBudget, + verifyApiJobParentExecution, +} from "./execution-identity"; + +const admittedAt = 1_000_000; +const base = { + kind: "service.build", + operationKey: "opk1_abcdefghijklmnop", + operationFingerprint: "a".repeat(64), + admittedAt, +}; +describe("immutable execution admission", () => { + it("retains long per-kind budgets without the old CLI/observer clamps", () => { + expect(OPS_KIND_POLICIES["service.build"].timeoutMs).toBe(12 * 3600_000); + expect(OPS_KIND_POLICIES["services.buildAll"].timeoutMs).toBe(24 * 3600_000); + expect(OPS_KIND_POLICIES["system.update"].timeoutMs).toBe(48 * 3600_000); + expect(opsExecutionBudget("artifactTransfer.import")).toEqual({ + processMs: 72 * 3600_000, + settlementMs: 30_000, + }); + expect(opsExecutionBudget("catalog.validate").processMs).toBe(72 * 3600_000); + expect(opsExecutionBudget("platform.config.apply").processMs).toBe(3600_000); + expect( + normalizeExecutionAdmission({ ...base, kind: "artifactTransfer.import" }).deadlineAt, + ).toBe(admittedAt + 72 * 3600_000); + expect( + normalizeExecutionAdmission({ + ...base, + kind: "artifactTransfer.import", + requestedDeadlineAt: admittedAt + 8 * 24 * 3600_000, + deploymentBudgetMs: 7 * 24 * 3600_000, + }).deadlineAt, + ).toBe(admittedAt + 7 * 24 * 3600_000); + }); + it("normalizes from original admission, body, deadline and parent, never today's clock", () => { + const first = normalizeExecutionAdmission(base); + expect(first.deadlineAt).toBe(admittedAt + 12 * 3600_000); + expect(normalizeExecutionAdmission(base)).toEqual(first); + expect( + normalizeExecutionAdmission({ ...base, requestedDeadlineAt: first.deadlineAt + 1 }), + ).toEqual(first); + expect( + normalizeExecutionAdmission({ ...base, requestedDeadlineAt: first.deadlineAt - 1 }) + .fingerprint, + ).not.toBe(first.fingerprint); + expect(normalizeExecutionAdmission({ ...base, deploymentBudgetMs: 100 }).deadlineAt).toBe( + admittedAt + 100, + ); + expect(() => + normalizeExecutionAdmission({ ...base, requestedDeadlineAt: admittedAt }), + ).toThrow(); + }); + it("clamps a late child to the authenticated parent and rejects invented authority", () => { + const fields = { + version: 1, + apiJobId: "123e4567-e89b-42d3-a456-426614174000", + apiOwnerVersion: 1, + parentOperationKey: "opk1_parentabcdefghij", + parentFingerprint: "b".repeat(64), + parentDeadlineAt: admittedAt + 100, + childOperationKey: base.operationKey, + childOperationFingerprint: base.operationFingerprint, + requestedDeadlineAt: admittedAt + 1000, + attemptId: "123e4567-e89b-42d3-a456-426614174001", + }; + const payload = Buffer.from(JSON.stringify(fields)).toString("base64url"); + const signature = createHmac("sha256", "test-token") + .update("openmapx-api-job-parent-v1\0") + .update(payload) + .digest("base64url"); + const assertion = `${payload}.${signature}`; + const parent = verifyApiJobParentExecution(assertion, "test-token", { + operationKey: base.operationKey, + operationFingerprint: base.operationFingerprint, + requestedDeadlineAt: fields.requestedDeadlineAt, + }); + const result = normalizeExecutionAdmission({ + ...base, + requestedDeadlineAt: fields.requestedDeadlineAt, + parent, + }); + expect(result.deadlineAt).toBe(admittedAt + 100); + expect(() => normalizeExecutionAdmission({ ...base, parent })).toThrow(); + expect(() => + normalizeExecutionAdmission({ ...base, parent: { deadlineAt: admittedAt + 10 } as never }), + ).toThrow(); + expect(() => + verifyApiJobParentExecution(assertion, "other-token", { + operationKey: base.operationKey, + operationFingerprint: base.operationFingerprint, + requestedDeadlineAt: fields.requestedDeadlineAt, + }), + ).toThrow(); + }); +}); diff --git a/packages/core/src/ops/execution-identity.ts b/packages/core/src/ops/execution-identity.ts new file mode 100644 index 000000000..bf252cca8 --- /dev/null +++ b/packages/core/src/ops/execution-identity.ts @@ -0,0 +1,238 @@ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { z } from "zod"; +import { OPS_KIND_POLICIES, type OpsOperationKind } from "./contract"; +import { canonicalJson } from "./fingerprint"; + +export type ControlIssuer = "ops-agent" | "data-manager" | "host-cli"; +export const executionRefSchema = z + .strictObject({ + executor: z.enum(["ops-agent", "data-manager", "host-cli"]), + principal: z.enum(["api", "data-manager", "host-cli"]), + executionId: z.string(), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + operationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + }) + .refine((x) => + x.executor === "ops-agent" + ? /^job1_[A-Za-z0-9_-]{16,64}$/.test(x.executionId) + : z.uuid().safeParse(x.executionId).success, + ); +export type ExecutionRef = z.infer; + +const parentIdentitySchema = z.strictObject({ + issuer: z.literal("api"), + apiJobId: z.uuid(), + apiOwnerVersion: z.number().int().positive().safe(), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + attemptId: z.uuid(), + assertionDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +/** Persisted identity is evidence only; parsing it does not authenticate a parent. */ +export type RetainedParentExecution = z.infer; +declare const verifiedParent: unique symbol; +export type VerifiedParentExecution = Readonly & { + readonly [verifiedParent]: true; +}; +const verifiedParents = new WeakSet(); +export type ExecutionAdmission = { requestedDeadlineAt?: number; parent?: VerifiedParentExecution }; +export const retainedParentExecutionSchema = parentIdentitySchema; +export const OPS_MAX_EXECUTION_MS = 7 * 24 * 3600_000; +export const OPS_SETTLEMENT_MS = 30_000 as const; +const futureBudgets: Readonly> = { + "artifactTransfer.import": 72 * 3600_000, + "artifactTransfer.export": 72 * 3600_000, + "artifactTransfer.validate": 72 * 3600_000, + "catalog.validate": 72 * 3600_000, + "catalog.activate": 3600_000, + "data.overturePull": 24 * 3600_000, + "data.overtureIngest": 24 * 3600_000, + "data.overtureExtract": 24 * 3600_000, + "platform.config.apply": 3600_000, + "download-osm": 24 * 3600_000, + "download-fonts": 24 * 3600_000, + "convert-overpass": 24 * 3600_000, + "transit-sync": 24 * 3600_000, + "overture-pull": 24 * 3600_000, + "overture-ingest": 24 * 3600_000, + "overture-extract": 24 * 3600_000, + "overture-conflate": 24 * 3600_000, + "overture-sync": 24 * 3600_000, + "search-index-build": 24 * 3600_000, + "notable-places-build": 24 * 3600_000, + "generation-hardlinks": 10 * 60_000, + "transit-maintenance": 120_000, +}; +/** Future policies are budgets only; they register no executable operations. */ +export function opsExecutionBudget(kind: string): { processMs: number; settlementMs: 30000 } { + const policy = Object.hasOwn(OPS_KIND_POLICIES, kind) + ? OPS_KIND_POLICIES[kind as OpsOperationKind] + : undefined; + const processMs = + policy?.timeoutMs ?? (Object.hasOwn(futureBudgets, kind) ? futureBudgets[kind] : undefined); + if (!processMs) throw Error("Unknown execution kind"); + return { processMs, settlementMs: OPS_SETTLEMENT_MS }; +} +export function opsExecutionMaximumMs(kind: string): number { + return [ + "artifactTransfer.import", + "artifactTransfer.export", + "artifactTransfer.validate", + ].includes(kind) + ? OPS_MAX_EXECUTION_MS + : opsExecutionBudget(kind).processMs; +} +export function executionFingerprint(input: { + kind: string; + operationFingerprint: string; + deadlineAt: number; + parent?: RetainedParentExecution; +}): string { + return createHash("sha256") + .update("openmapx-execution-v2\0") + .update(canonicalJson({ version: 2, ...input })) + .digest("hex"); +} +function epoch(value: number): boolean { + return Number.isSafeInteger(value) && value >= 0; +} +export function normalizeExecutionAdmission(input: { + operationFingerprint: string; + operationKey: string; + kind: string; + admittedAt: number; + requestedDeadlineAt?: number; + parent?: VerifiedParentExecution; + deploymentBudgetMs?: number; +}): { admittedAt: number; deadlineAt: number; fingerprint: string } { + const budget = opsExecutionBudget(input.kind).processMs; + const deploymentBudget = input.deploymentBudgetMs ?? budget; + if ( + !epoch(input.admittedAt) || + !/^[a-f0-9]{64}$/.test(input.operationFingerprint) || + !/^opk1_[A-Za-z0-9_-]{16,64}$/.test(input.operationKey) || + !Number.isSafeInteger(deploymentBudget) || + deploymentBudget <= 0 || + (input.requestedDeadlineAt !== undefined && !epoch(input.requestedDeadlineAt)) || + (input.parent && + (!verifiedParents.has(input.parent) || input.requestedDeadlineAt === undefined)) + ) + throw Error("Invalid execution admission"); + const deadlineAt = Math.min( + input.requestedDeadlineAt ?? input.admittedAt + budget, + input.parent?.deadlineAt ?? Infinity, + input.admittedAt + Math.min(opsExecutionMaximumMs(input.kind), deploymentBudget), + ); + if (!epoch(deadlineAt) || deadlineAt <= input.admittedAt) + throw Error("Elapsed execution deadline"); + return { + admittedAt: input.admittedAt, + deadlineAt, + fingerprint: executionFingerprint({ + kind: input.kind, + operationFingerprint: input.operationFingerprint, + deadlineAt, + ...(input.parent ? { parent: input.parent } : {}), + }), + }; +} + +const apiAssertionSchema = z.strictObject({ + version: z.literal(1), + apiJobId: z.uuid(), + apiOwnerVersion: z.number().int().positive().safe(), + parentOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + parentFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + parentDeadlineAt: z.number().int().nonnegative().safe(), + childOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + childOperationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + attemptId: z.uuid(), +}); +/** Fixed API bearer/HMAC edge only. Role-signed parent/lease verification belongs to 06.1. */ +export function verifyApiJobParentExecution( + assertion: string, + token: string, + child: { operationKey: string; operationFingerprint: string; requestedDeadlineAt?: number }, +): VerifiedParentExecution { + if (assertion.length > 4096 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{43}$/.test(assertion)) + throw Error("Invalid parent assertion"); + const [payload, signature] = assertion.split(".") as [string, string]; + const expected = createHmac("sha256", token) + .update("openmapx-api-job-parent-v1\0") + .update(payload) + .digest(); + const supplied = Buffer.from(signature, "base64url"); + if (supplied.length !== expected.length || !timingSafeEqual(expected, supplied)) + throw Error("Invalid parent assertion"); + const fields = apiAssertionSchema.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.from(payload, "base64url"))), + ); + if ( + Buffer.from(JSON.stringify(fields)).toString("base64url") !== payload || + fields.childOperationKey !== child.operationKey || + fields.childOperationFingerprint !== child.operationFingerprint || + fields.requestedDeadlineAt !== child.requestedDeadlineAt + ) + throw Error("Invalid parent binding"); + const parent = Object.freeze({ + issuer: "api" as const, + apiJobId: fields.apiJobId, + apiOwnerVersion: fields.apiOwnerVersion, + operationKey: fields.parentOperationKey, + fingerprint: fields.parentFingerprint, + deadlineAt: fields.parentDeadlineAt, + attemptId: fields.attemptId, + assertionDigest: createHash("sha256").update(assertion).digest("hex"), + }); + verifiedParents.add(parent); + return parent as VerifiedParentExecution; +} + +export interface RetainedExecutionAdmission { + executionVersion: 2; + admittedAt: number; + deadlineAt: number; + operationFingerprint: string; + fingerprint: string; + requestedDeadlineAt?: number; + parent?: RetainedParentExecution; + deploymentBudgetMs: number; +} +/** Restart validation recomputes evidence; it never manufactures verified parent authority. */ +export function validateRetainedExecutionAdmission( + kind: string, + input: RetainedExecutionAdmission, +): void { + const budget = opsExecutionBudget(kind).processMs; + const parent = + input.parent === undefined ? undefined : retainedParentExecutionSchema.parse(input.parent); + if ( + input.executionVersion !== 2 || + !epoch(input.admittedAt) || + !epoch(input.deadlineAt) || + !Number.isSafeInteger(input.deploymentBudgetMs) || + input.deploymentBudgetMs <= 0 || + input.deploymentBudgetMs > opsExecutionMaximumMs(kind) || + (input.requestedDeadlineAt !== undefined && !epoch(input.requestedDeadlineAt)) || + (parent && input.requestedDeadlineAt === undefined) || + input.deadlineAt <= input.admittedAt || + input.deadlineAt !== + Math.min( + input.requestedDeadlineAt ?? input.admittedAt + budget, + parent?.deadlineAt ?? Infinity, + input.admittedAt + input.deploymentBudgetMs, + ) || + input.fingerprint !== + executionFingerprint({ + kind, + operationFingerprint: input.operationFingerprint, + deadlineAt: input.deadlineAt, + ...(parent ? { parent } : {}), + }) + ) + throw Error("Invalid retained execution"); +} diff --git a/packages/core/src/ops/fingerprint.ts b/packages/core/src/ops/fingerprint.ts index b8b3c4b31..79532e7e9 100644 --- a/packages/core/src/ops/fingerprint.ts +++ b/packages/core/src/ops/fingerprint.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { type OpsOperation, opsOperationSchema } from "./contract"; -function canonicalJson(value: unknown): string { +export function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; if (value && typeof value === "object") { return `{${Object.entries(value as Record) diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index ff29f9dcc..54bb89a67 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -1,6 +1,9 @@ export * from "./client"; export * from "./contract"; +export * from "./execution-identity"; export * from "./fingerprint"; +export * from "./job-evidence"; export * from "./privacy-backup"; +export * from "./service-results"; export * from "./trusted-config"; export * from "./trusted-config-queue-lock"; diff --git a/packages/core/src/ops/job-evidence.ts b/packages/core/src/ops/job-evidence.ts new file mode 100644 index 000000000..bd2573fd6 --- /dev/null +++ b/packages/core/src/ops/job-evidence.ts @@ -0,0 +1,60 @@ +import { z } from "zod"; +import { serviceBatchResultSchema, serviceLifecycleFailureEvidenceSchema } from "./service-results"; + +export const PROCESS_OUTPUT_TRUNCATED = "[output truncated]"; +export const processDiagnosticsSchema = z + .strictObject({ + tail: z + .array( + z.strictObject({ + stream: z.enum(["stdout", "stderr"]), + message: z.string().refine((x) => Buffer.byteLength(x) <= 4096), + }), + ) + .max(256), + truncated: z.boolean(), + }) + .refine((x) => Buffer.byteLength(JSON.stringify(x)) <= 16 * 1024); +export type ProcessDiagnostics = z.infer; +export const dataWorkflowReceiptSchema = z.strictObject({ + version: z.literal(1), + phase: z.enum(["osm", "transit", "fonts", "build", "link", "complete"]), + children: z + .array( + z.strictObject({ + step: z.enum(["osm", "transit", "fonts", "build", "link"]), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + jobId: z.uuid(), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + status: z.enum([ + "queued", + "running", + "succeeded", + "failed", + "interrupted", + "cancel_pending", + "canceled", + "unknown", + ]), + }), + ) + .max(16), + completedBuildIds: z.array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)).max(256), + failedBuildIds: z.array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)).max(256), + skippedBuildIds: z.array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)).max(256), + revisionId: z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .optional(), +}); +export type DataWorkflowReceipt = z.infer; +export const opsFailureResultSchema = z + .strictObject({ + serviceBatch: serviceBatchResultSchema.optional(), + dataWorkflow: dataWorkflowReceiptSchema.optional(), + diagnostics: processDiagnosticsSchema.optional(), + serviceLifecycle: serviceLifecycleFailureEvidenceSchema.optional(), + }) + .refine((x) => Buffer.byteLength(JSON.stringify(x)) <= 64 * 1024); +export type OpsFailureResult = z.infer; diff --git a/packages/core/src/ops/service-results.ts b/packages/core/src/ops/service-results.ts new file mode 100644 index 000000000..6eb80d4e6 --- /dev/null +++ b/packages/core/src/ops/service-results.ts @@ -0,0 +1,59 @@ +import { z } from "zod"; + +export type CoordinatorMode = "ensure-running" | "require-running"; +const serviceId = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const ids = z + .array(serviceId) + .max(256) + .refine((x) => new Set(x).size === x.length); +export const serviceBatchResultSchema = z + .strictObject({ + outcome: z.enum(["success", "partial", "failed"]), + completedServiceIds: ids, + failedServiceIds: ids, + skippedServiceIds: ids, + }) + .refine((x) => { + const all = [...x.completedServiceIds, ...x.failedServiceIds, ...x.skippedServiceIds]; + return all.length <= 256 && new Set(all).size === all.length; + }); +export type ServiceBatchResult = z.infer; +const revision = z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .nullable(); +export const serviceLifecycleFailureEvidenceSchema = z.strictObject({ + expectedRevisionId: revision, + seedRevisionId: revision, + finalRevisionId: revision, + phase: z.enum([ + "prepared", + "pins_validated", + "pulled", + "generation_committed", + "hardlinks_applied", + "recreated", + "complete", + "failed", + "unknown", + ]), + commit: z.enum(["not_committed", "seed_committed", "final_committed", "unknown"]), + followUp: z.enum(["not_started", "complete", "failed", "unknown"]), + failure: z + .strictObject({ + reason: z.enum([ + "pins", + "pull", + "revision_conflict", + "hardlinks", + "recreate", + "deadline", + "canceled", + "containment", + "receipt_io", + ]), + failedServiceIds: ids, + }) + .optional(), +}); +export type ServiceLifecycleFailureEvidence = z.infer; diff --git a/packages/core/src/scrub-secrets.ts b/packages/core/src/scrub-secrets.ts new file mode 100644 index 000000000..1ff0815f8 --- /dev/null +++ b/packages/core/src/scrub-secrets.ts @@ -0,0 +1,179 @@ +/** + * Failure text from the transit pipeline is republished to an operator-configured + * external issue tracker. Anything crossing that boundary must pass through here. + */ + +const URL_PATTERN = /\b(?:https?|ftps?):\/\/[^\s"'`<>\\]+/gi; +const PATH_SECRET_SEGMENTS = new Set([ + "key", + "keys", + "apikey", + "api-key", + "api_key", + "token", + "access-token", + "access_token", + "secret", + "auth", + "operator-feed", +]); +const BARE_CREDENTIAL_PATTERN = + /((? { + return scrubUrl(raw); + }); + + scrubbed = scrubbed.replace( + /\b(Authorization\s*[:=]\s*)(?:Bearer|Basic|Token)\s+\S+/gi, + "$1[redacted]", + ); + scrubbed = scrubbed.replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/g, "$1 [redacted]"); + scrubbed = scrubbed.replace(BARE_CREDENTIAL_PATTERN, "$1$2[redacted]$2"); + scrubbed = scrubbed.replace(/AGE-ENCRYPTED:[A-Za-z0-9+/=]+/g, "AGE-ENCRYPTED:[redacted]"); + + return scrubbed; +} + +export function scrubSecretsOptional(message: string | undefined): string | undefined { + return message === undefined ? undefined : scrubSecrets(message); +} + +const EXACT_SECRET_FIELD_NAMES = new Set([ + "auth", + "authorization", + "cookie", + "setcookie", + "proxyauthorization", + "accesskeyid", + "privatekey", +]); + +function isSecretFieldName(fieldName: string): boolean { + const normalized = fieldName.replaceAll(/[-_]/g, "").toLowerCase(); + return ( + EXACT_SECRET_FIELD_NAMES.has(normalized) || + /(?:password|passwd|secret|token|apikey)$/.test(normalized) + ); +} + +function scrubDiagnosticValueInner( + value: unknown, + seen: WeakMap, + fieldName?: string, +): unknown { + if (fieldName && isSecretFieldName(fieldName)) return "[redacted]"; + if (typeof value === "string") { + const normalizedFieldName = fieldName?.replaceAll(/[-_]/g, "").toLowerCase(); + return normalizedFieldName?.endsWith("url") ? scrubUrl(value) : scrubSecrets(value); + } + if (value === null || typeof value !== "object") return value; + + const existing = seen.get(value); + if (existing !== undefined) return existing; + + if (value instanceof URL) return scrubSecrets(value.toString()); + if (value instanceof Date || ArrayBuffer.isView(value)) return value; + + // Fastify/Pino request serializers rely on object identity and hidden + // symbols. Preserve the instance and redact the serializer's output later. + if ("raw" in value && "id" in value && "log" in value) return value; + + if (value instanceof Error) { + const scrubbed = new Error(scrubSecrets(value.message)); + seen.set(value, scrubbed); + scrubbed.name = value.name; + if (value.stack) scrubbed.stack = scrubSecrets(value.stack); + if (value.cause !== undefined) { + scrubbed.cause = scrubDiagnosticValueInner(value.cause, seen, "cause"); + } + for (const [key, nested] of Object.entries(value)) { + (scrubbed as unknown as Record)[key] = scrubDiagnosticValueInner( + nested, + seen, + key, + ); + } + return scrubbed; + } + + if (Array.isArray(value)) { + const scrubbed: unknown[] = []; + seen.set(value, scrubbed); + for (const nested of value) scrubbed.push(scrubDiagnosticValueInner(nested, seen)); + return scrubbed; + } + + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return value; + + const scrubbed: Record = {}; + seen.set(value, scrubbed); + for (const [key, nested] of Object.entries(value)) { + scrubbed[key] = scrubDiagnosticValueInner(nested, seen, key); + } + return scrubbed; +} + +/** + * Return a scrubbed copy of JSON-shaped diagnostics without mutating the + * caller's object. Error instances remain Error instances so Pino retains + * their non-enumerable message, stack, and type. + */ +export function scrubDiagnosticValue(value: unknown): unknown { + return scrubDiagnosticValueInner(value, new WeakMap()); +} + +export interface SafeUrlDiagnostic { + protocol: string; + host: string; + path: string; +} + +/** Strip URL credentials, query, and fragment while retaining safe routing context. */ +export function safeUrlDiagnostic(rawUrl: string): SafeUrlDiagnostic | undefined { + try { + const parsed = new URL(scrubUrl(rawUrl)); + return { + protocol: parsed.protocol, + host: parsed.host, + path: parsed.pathname, + }; + } catch { + return undefined; + } +} diff --git a/services/data-manager/src/utils/scrub-secrets.ts b/services/data-manager/src/utils/scrub-secrets.ts index 1ff0815f8..2b6fa8a5f 100644 --- a/services/data-manager/src/utils/scrub-secrets.ts +++ b/services/data-manager/src/utils/scrub-secrets.ts @@ -1,179 +1 @@ -/** - * Failure text from the transit pipeline is republished to an operator-configured - * external issue tracker. Anything crossing that boundary must pass through here. - */ - -const URL_PATTERN = /\b(?:https?|ftps?):\/\/[^\s"'`<>\\]+/gi; -const PATH_SECRET_SEGMENTS = new Set([ - "key", - "keys", - "apikey", - "api-key", - "api_key", - "token", - "access-token", - "access_token", - "secret", - "auth", - "operator-feed", -]); -const BARE_CREDENTIAL_PATTERN = - /((? { - return scrubUrl(raw); - }); - - scrubbed = scrubbed.replace( - /\b(Authorization\s*[:=]\s*)(?:Bearer|Basic|Token)\s+\S+/gi, - "$1[redacted]", - ); - scrubbed = scrubbed.replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/g, "$1 [redacted]"); - scrubbed = scrubbed.replace(BARE_CREDENTIAL_PATTERN, "$1$2[redacted]$2"); - scrubbed = scrubbed.replace(/AGE-ENCRYPTED:[A-Za-z0-9+/=]+/g, "AGE-ENCRYPTED:[redacted]"); - - return scrubbed; -} - -export function scrubSecretsOptional(message: string | undefined): string | undefined { - return message === undefined ? undefined : scrubSecrets(message); -} - -const EXACT_SECRET_FIELD_NAMES = new Set([ - "auth", - "authorization", - "cookie", - "setcookie", - "proxyauthorization", - "accesskeyid", - "privatekey", -]); - -function isSecretFieldName(fieldName: string): boolean { - const normalized = fieldName.replaceAll(/[-_]/g, "").toLowerCase(); - return ( - EXACT_SECRET_FIELD_NAMES.has(normalized) || - /(?:password|passwd|secret|token|apikey)$/.test(normalized) - ); -} - -function scrubDiagnosticValueInner( - value: unknown, - seen: WeakMap, - fieldName?: string, -): unknown { - if (fieldName && isSecretFieldName(fieldName)) return "[redacted]"; - if (typeof value === "string") { - const normalizedFieldName = fieldName?.replaceAll(/[-_]/g, "").toLowerCase(); - return normalizedFieldName?.endsWith("url") ? scrubUrl(value) : scrubSecrets(value); - } - if (value === null || typeof value !== "object") return value; - - const existing = seen.get(value); - if (existing !== undefined) return existing; - - if (value instanceof URL) return scrubSecrets(value.toString()); - if (value instanceof Date || ArrayBuffer.isView(value)) return value; - - // Fastify/Pino request serializers rely on object identity and hidden - // symbols. Preserve the instance and redact the serializer's output later. - if ("raw" in value && "id" in value && "log" in value) return value; - - if (value instanceof Error) { - const scrubbed = new Error(scrubSecrets(value.message)); - seen.set(value, scrubbed); - scrubbed.name = value.name; - if (value.stack) scrubbed.stack = scrubSecrets(value.stack); - if (value.cause !== undefined) { - scrubbed.cause = scrubDiagnosticValueInner(value.cause, seen, "cause"); - } - for (const [key, nested] of Object.entries(value)) { - (scrubbed as unknown as Record)[key] = scrubDiagnosticValueInner( - nested, - seen, - key, - ); - } - return scrubbed; - } - - if (Array.isArray(value)) { - const scrubbed: unknown[] = []; - seen.set(value, scrubbed); - for (const nested of value) scrubbed.push(scrubDiagnosticValueInner(nested, seen)); - return scrubbed; - } - - const prototype = Object.getPrototypeOf(value); - if (prototype !== Object.prototype && prototype !== null) return value; - - const scrubbed: Record = {}; - seen.set(value, scrubbed); - for (const [key, nested] of Object.entries(value)) { - scrubbed[key] = scrubDiagnosticValueInner(nested, seen, key); - } - return scrubbed; -} - -/** - * Return a scrubbed copy of JSON-shaped diagnostics without mutating the - * caller's object. Error instances remain Error instances so Pino retains - * their non-enumerable message, stack, and type. - */ -export function scrubDiagnosticValue(value: unknown): unknown { - return scrubDiagnosticValueInner(value, new WeakMap()); -} - -export interface SafeUrlDiagnostic { - protocol: string; - host: string; - path: string; -} - -/** Strip URL credentials, query, and fragment while retaining safe routing context. */ -export function safeUrlDiagnostic(rawUrl: string): SafeUrlDiagnostic | undefined { - try { - const parsed = new URL(scrubUrl(rawUrl)); - return { - protocol: parsed.protocol, - host: parsed.host, - path: parsed.pathname, - }; - } catch { - return undefined; - } -} +export * from "@openmapx/core/scrub-secrets"; From 6d511a6ede04517a00f4d4a4730fb64ccafbf145 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 21:09:55 +0200 Subject: [PATCH 004/135] fix(ops): close output and deadline compatibility gaps --- .../src/administrative-runtime.test.ts | 164 +++++++++++- apps/ops-agent/src/administrative-runtime.ts | 14 +- apps/ops-agent/src/agent-update.test.ts | 6 +- apps/ops-agent/src/bootstrap.test.ts | 159 +++++++++++ apps/ops-agent/src/bootstrap.ts | 26 ++ apps/ops-agent/src/index.ts | 133 +++++----- apps/ops-agent/src/journal.test.ts | 5 + apps/ops-agent/src/journal.ts | 6 + apps/ops-agent/src/process-output.test.ts | 61 +++++ apps/ops-agent/src/process-output.ts | 194 ++++++++++---- apps/ops-agent/src/server-foundation.test.ts | 4 +- apps/ops-agent/src/server.test.ts | 251 +++++++++++++++++- apps/ops-agent/src/server.ts | 129 +++++---- .../__tests__/cli-execution-deadline.test.ts | 149 +++++++++++ packages/cli/src/commands/backup.ts | 27 +- packages/cli/src/commands/compose.ts | 4 + packages/cli/src/commands/data.ts | 12 +- packages/cli/src/index.ts | 8 +- packages/cli/src/lib/build-artifacts.ts | 6 +- packages/cli/src/lib/build-staging.ts | 3 + packages/cli/src/lib/docker.ts | 5 +- packages/cli/src/lib/execution-deadline.ts | 110 ++++++++ packages/cli/src/lib/hardlinks.ts | 12 +- packages/cli/src/lib/motis-data.ts | 7 +- packages/cli/src/lib/osrm-graph.ts | 7 +- packages/cli/src/lib/otp-graph.ts | 7 +- packages/cli/src/lib/pelias-data.ts | 7 +- packages/cli/src/lib/release.ts | 3 +- packages/cli/src/lib/service-builds.ts | 4 + packages/cli/src/lib/tile-mbtiles.ts | 7 +- packages/cli/src/lib/transitous-api-keys.ts | 7 +- packages/core/src/ops/client.test.ts | 95 +++++-- packages/core/src/ops/client.ts | 96 ++++--- packages/core/src/ops/contract.test.ts | 23 ++ packages/core/src/ops/contract.ts | 50 +++- 35 files changed, 1520 insertions(+), 281 deletions(-) create mode 100644 apps/ops-agent/src/bootstrap.test.ts create mode 100644 apps/ops-agent/src/bootstrap.ts create mode 100644 packages/cli/__tests__/cli-execution-deadline.test.ts create mode 100644 packages/cli/src/lib/execution-deadline.ts diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index fa3746e43..bfb7ab084 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -85,6 +85,7 @@ describe("default fixed CLI live output", () => { ); const seen: string[] = []; const completion = createDefaultFixedCli(root, join(root, "agent.mjs"))([], { + execution: testExecution(), signal: new AbortController().signal, emitLog: (_stream, message) => seen.push(message), }); @@ -625,7 +626,11 @@ describe("administrative release runtime", () => { } return ""; }; - const context = { signal: new AbortController().signal, emitLog: vi.fn() }; + const context = { + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }; const first = createDefaultReleaseEffects(rootDir, async () => undefined, { runDocker: docker, }); @@ -710,7 +715,11 @@ describe("administrative release runtime", () => { return ""; }, }); - const context = { signal: new AbortController().signal, emitLog: vi.fn() }; + const context = { + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }; expect(await runtime.resolve(context)).toBe(`${sourceRevision}-123-1`); attempt = 2; expect(await runtime.resolve(context)).toBe(`${sourceRevision}-123-2`); @@ -836,8 +845,16 @@ describe("administrative release runtime", () => { }; const results = await Promise.allSettled([ - resolveFor("release-a").resolve({ signal: new AbortController().signal, emitLog: vi.fn() }), - resolveFor("release-b").resolve({ signal: new AbortController().signal, emitLog: vi.fn() }), + resolveFor("release-a").resolve({ + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }), + resolveFor("release-b").resolve({ + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }), ]); const stayed = readdirSync(directory).filter((name) => name.endsWith(".json")).length; @@ -895,7 +912,7 @@ describe("administrative release runtime", () => { first.apply( releaseId, ["app-api"], - { signal: new AbortController().signal, emitLog: vi.fn() }, + { execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn() }, "job-recovery", ), ).rejects.toThrow("injected crash"); @@ -961,7 +978,7 @@ describe("administrative release runtime", () => { first.apply( releaseId, ["app-api"], - { signal: new AbortController().signal, emitLog: vi.fn() }, + { execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn() }, "job-recovery", ), ).rejects.toThrow("injected crash"); @@ -1011,6 +1028,7 @@ describe("administrative release runtime", () => { }); await expect( first.apply(releaseId, ["app-api"], { + execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), }), @@ -1074,7 +1092,7 @@ describe("administrative release runtime", () => { first.apply( releaseId, ["app-api"], - { signal: new AbortController().signal, emitLog: vi.fn() }, + { execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn() }, `job-${crashPhase}`, ), ).rejects.toThrow("injected crash"); @@ -1150,7 +1168,11 @@ describe("administrative release runtime", () => { }); await expect( - effects.inspectSystem({ signal: new AbortController().signal, emitLog: vi.fn() }), + effects.inspectSystem({ + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }), ).resolves.toMatchObject({ dockerReachable: true, services: [ @@ -1209,7 +1231,11 @@ describe("administrative release runtime", () => { return createDefaultReleaseEffects(rootDir, async () => undefined, { runDocker }); } - const context = () => ({ signal: new AbortController().signal, emitLog: vi.fn() }); + const context = () => ({ + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }); it("reports unknown rather than stopped when a per-service observation throws", async () => { const effects = systemFixture("file", async (args) => { @@ -1346,7 +1372,7 @@ describe("administrative release runtime", () => { effects.apply( releaseId, ["app-api"], - { signal: new AbortController().signal, emitLog: vi.fn() }, + { execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn() }, "job-1", ), ).rejects.toThrow("injected update failure"); @@ -1627,3 +1653,121 @@ it("never gives release recovery a fresh execution budget after restart expiry", vi.useRealTimers(); } }); + +describe("fix1 legacy release recovery", () => { + it.each(["overlay_written", "rollback_services"])( + "refuses legacy %s effects without a retained deadline", + async (phase) => { + const rootDir = temporaryRoot(); + const releaseId = "release-legacy-unknown"; + const directory = join(rootDir, "infra", "docker", ".ops-agent-releases"); + mkdirSync(directory, { mode: 0o700 }); + writeFileSync( + join(directory, `${releaseId}.json`), + JSON.stringify({ + schemaVersion: 1, + release: releaseId, + images: Object.fromEntries( + [ + "api", + "web", + "data-manager", + "ops-agent", + "privacy-backup", + "transitous-runner", + "transitous-tools", + ].map((name) => [name, `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`]), + ), + privacyReleaseValidation, + }), + { mode: 0o600 }, + ); + const first = createDefaultReleaseEffects(rootDir, async () => undefined, { + afterReleasePhase: (p) => { + if (p === "overlay_written") throw Error("fixture crash"); + }, + }); + await expect( + first.apply(releaseId, ["app-api"], { + signal: new AbortController().signal, + emitLog: () => undefined, + }), + ).rejects.toThrow("fixture crash"); + const transaction = join(directory, "transaction.json"); + const record = JSON.parse(readFileSync(transaction, "utf8")); + record.phase = phase; + writeFileSync(transaction, JSON.stringify(record)); + const overlay = join(rootDir, "infra", "docker", "docker-compose.release.yml"); + writeFileSync(overlay, "operator-state\n"); + const run = vi.fn(async () => undefined); + const recovered = createDefaultReleaseEffects(rootDir, run, { + verifyAppliedRelease: async () => true, + }); + await expect(recovered.initialize?.()).rejects.toMatchObject({ + name: "OpsExecutionFailure", + errorClass: "recovery_required", + }); + expect(run).not.toHaveBeenCalled(); + expect(readFileSync(overlay, "utf8")).toBe("operator-state\n"); + expect(existsSync(transaction)).toBe(true); + }, + ); +}); + +it("fix1 recovery cannot publish after verification consumes the remaining deadline", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const rootDir = temporaryRoot(); + const releaseId = "release-verification-expiry"; + const directory = join(rootDir, "infra/docker/.ops-agent-releases"); + mkdirSync(directory, { mode: 0o700 }); + writeFileSync( + join(directory, `${releaseId}.json`), + JSON.stringify({ + schemaVersion: 1, + release: releaseId, + images: Object.fromEntries( + [ + "api", + "web", + "data-manager", + "ops-agent", + "privacy-backup", + "transitous-runner", + "transitous-tools", + ].map((name) => [name, `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`]), + ), + privacyReleaseValidation, + }), + { mode: 0o600 }, + ); + const execution = { ...testExecution(), deadlineAt: Date.now() + 1000 }; + try { + const first = createDefaultReleaseEffects(rootDir, async () => undefined, { + afterReleasePhase: (phase) => { + if (phase === "overlay_written") throw Error("fixture crash"); + }, + }); + await expect( + first.apply(releaseId, ["app-api"], { + execution, + signal: new AbortController().signal, + emitLog: () => undefined, + }), + ).rejects.toThrow("fixture crash"); + const path = join(directory, "transaction.json"); + const record = JSON.parse(readFileSync(path, "utf8")); + record.phase = "services_applied"; + writeFileSync(path, JSON.stringify(record)); + const recovered = createDefaultReleaseEffects(rootDir, async () => undefined, { + verifyAppliedRelease: async () => { + vi.setSystemTime(Date.now() + 2000); + return true; + }, + }); + await expect(recovered.initialize?.()).rejects.toMatchObject({ errorClass: "timeout" }); + expect(existsSync(join(directory, "current.json"))).toBe(false); + expect(existsSync(path)).toBe(true); + } finally { + vi.useRealTimers(); + } +}); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index ec803cb30..c228ef3ff 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -1195,13 +1195,17 @@ export function createDefaultReleaseEffects( transaction: ReleaseTransaction, context: FixedCliOptions, ): Promise => { - if (transaction.execution) context = { ...context, execution: transaction.execution }; + if (!transaction.execution) throw new OpsExecutionFailure("recovery_required", {}); + context = { ...context, execution: transaction.execution }; remainingCliTime(context); await writeTransaction({ ...transaction, phase: "rollback_overlay" }); + remainingCliTime(context); if (transaction.previousOverlay === null) durableUnlink(paths.overlay); else atomicWrite(paths.overlay, transaction.previousOverlay); await writeTransaction({ ...transaction, phase: "rollback_services" }); + remainingCliTime(context); await runFixedCli(["services", "update", "--no-deps", ...transaction.serviceIds], context); + remainingCliTime(context); if (transaction.previousState === null) durableUnlink(paths.state); else atomicWrite(paths.state, transaction.previousState); durableUnlink(paths.transaction); @@ -1210,7 +1214,8 @@ export function createDefaultReleaseEffects( transaction: ReleaseTransaction, context: FixedCliOptions, ): Promise => { - if (transaction.execution) context = { ...context, execution: transaction.execution }; + if (!transaction.execution) throw new OpsExecutionFailure("recovery_required", {}); + context = { ...context, execution: transaction.execution }; remainingCliTime(context); const manifest = loadManifest(transaction.releaseId); // The transaction's digest was taken from the exact bytes authorized when @@ -1224,12 +1229,15 @@ export function createDefaultReleaseEffects( await requireCurrentAgent(manifest, context); } if (transaction.phase === "prepared") { + remainingCliTime(context); coreServices.writeReleaseComposeArtifacts(manifest, paths.overlay); transaction = { ...transaction, phase: "overlay_written" }; await writeTransaction(transaction); } if (transaction.phase === "overlay_written") { + remainingCliTime(context); coreServices.writeReleaseComposeArtifacts(manifest, paths.overlay); + remainingCliTime(context); await runFixedCli(["services", "update", "--no-deps", ...transaction.serviceIds], context); if (!(await verifyApplied(manifest, transaction.serviceIds, context))) { throw new Error("Release recovery verification failed"); @@ -1246,6 +1254,7 @@ export function createDefaultReleaseEffects( if (!(await verifyApplied(manifest, transaction.serviceIds, context))) { throw new Error("Release recovery verification failed"); } + remainingCliTime(context); publishState(transaction); transaction = { ...transaction, phase: "state_published" }; await writeTransaction(transaction); @@ -1259,6 +1268,7 @@ export function createDefaultReleaseEffects( if (!(await verifyApplied(manifest, transaction.serviceIds, context))) { throw new Error("Release recovery verification failed"); } + remainingCliTime(context); durableUnlink(paths.transaction); return; } diff --git a/apps/ops-agent/src/agent-update.test.ts b/apps/ops-agent/src/agent-update.test.ts index 3c3b6d293..aa1340a77 100644 --- a/apps/ops-agent/src/agent-update.test.ts +++ b/apps/ops-agent/src/agent-update.test.ts @@ -88,7 +88,11 @@ function fixture(options: { agent: "old" | "current"; helperStarts?: boolean }) runDocker: docker, verifyAppliedRelease: async () => true, }); - const context = { signal: new AbortController().signal, emitLog: vi.fn() }; + const context = { + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }; return { rootDir, docker, calls, cli, effects, context }; } diff --git a/apps/ops-agent/src/bootstrap.test.ts b/apps/ops-agent/src/bootstrap.test.ts new file mode 100644 index 000000000..259d7e704 --- /dev/null +++ b/apps/ops-agent/src/bootstrap.test.ts @@ -0,0 +1,159 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; +import { expect, it, vi } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; +import { createDefaultReleaseEffects } from "./administrative-runtime"; +import { startOpsAgentServer } from "./bootstrap"; + +it.each(["legacy", "expired"])( + "fix1 production bootstrap retains health and observation with %s release recovery", + async (mode) => { + const root = mkdtempSync(join(tmpdir(), "openmapx-bootstrap-")); + const directory = join(root, "infra/docker/.ops-agent-releases"); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + const manifest = { + schemaVersion: 1, + release: "release-bootstrap", + images: Object.fromEntries( + [ + "api", + "web", + "data-manager", + "ops-agent", + "privacy-backup", + "transitous-runner", + "transitous-tools", + ].map((name) => [name, `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`]), + ), + privacyReleaseValidation: { + version: 1, + sourceBuildFingerprint: "a".repeat(64), + validatedAt: "2026-09-05T12:00:00.000Z", + checks: { translationsConsistent: true, openApiConsistent: true, policyConsistent: true }, + }, + }; + writeFileSync(join(directory, "release-bootstrap.json"), JSON.stringify(manifest), { + mode: 0o600, + }); + writeFileSync( + join(directory, "transaction.json"), + JSON.stringify({ + version: 1, + phase: "overlay_written", + releaseId: manifest.release, + digest: createHash("sha256") + .update( + services.canonicalReleaseManifest( + services.parseReleaseManifest(JSON.stringify(manifest)), + ), + ) + .digest("hex"), + serviceIds: ["app-api"], + previousOverlay: null, + previousState: null, + ...(mode === "expired" + ? { execution: { ...testExecution(), deadlineAt: Date.now() - 1 } } + : {}), + }), + { mode: 0o600 }, + ); + const overlay = join(root, "infra/docker/docker-compose.release.yml"); + writeFileSync(overlay, "operator-state\n"); + const run = vi.fn(async () => undefined); + const dispatch = vi.fn(async () => ({ reachable: true, version: "test" })); + const token = Buffer.alloc(32, 1).toString("base64url"); + const retained = { + role: "api" as const, + operation: { kind: "service.pull" as const, serviceId: "redis" }, + operationId: "job1_0123456789abcdef", + operationKey: "opk1_0123456789abcdef", + fingerprint: (await import("@openmapx/core/ops")).opsOperationFingerprint({ + kind: "service.pull", + serviceId: "redis", + }), + resourceId: "redis", + state: "succeeded" as const, + submittedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + terminalAt: new Date().toISOString(), + result: { completed: true }, + }; + const ready = vi.fn(); + let app: Awaited> | undefined; + try { + app = await startOpsAgentServer( + { + tokens: { api: token, "data-manager": Buffer.alloc(32, 2).toString("base64url") }, + resourcePolicy: { allowGlobal: () => true, allowService: () => true }, + journal: { records: () => [retained], replace: async () => undefined }, + dispatch, + }, + createDefaultReleaseEffects(root, run), + { host: "127.0.0.1", port: 0 }, + ready, + ); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("listener missing"); + const health = await (await fetch(`http://127.0.0.1:${address.port}/health`)).json(); + expect(health).toMatchObject({ + ok: true, + releaseRecovery: { ready: false, reason: "recovery_required" }, + }); + const headers = { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }; + expect( + (await app.inject({ method: "GET", url: "/v1/operations/job1_0123456789abcdef", headers })) + .statusCode, + ).toBe(200); + expect( + ( + await app.inject({ + method: "GET", + url: "/v1/operation-keys/opk1_0123456789abcdef", + headers: { ...headers, "x-ops-operation-fingerprint": retained.fingerprint }, + }) + ).statusCode, + ).toBe(200); + const now = Date.now(); + const envelope = (operation: unknown, requestId: string) => ({ + version: 2, + requestId, + operationKey: "opk1_bootstrapnew0000", + issuedAt: new Date(now).toISOString(), + expiresAt: new Date(now + 15000).toISOString(), + operation, + }); + expect( + ( + await app.inject({ + method: "POST", + url: "/v1/operations", + headers, + payload: envelope( + { kind: "service.restart", serviceId: "redis" }, + "ops1_0123456789abcdef", + ), + }) + ).json().error.class, + ).toBe("recovery_required"); + expect( + ( + await app.inject({ + method: "POST", + url: "/v1/operations", + headers, + payload: envelope({ kind: "docker.status" }, "ops1_abcdef0123456789"), + }) + ).statusCode, + ).toBe(200); + expect(ready).not.toHaveBeenCalled(); + expect(run).not.toHaveBeenCalled(); + expect(readFileSync(overlay, "utf8")).toBe("operator-state\n"); + } finally { + await app?.close(); + rmSync(root, { recursive: true, force: true }); + } + }, +); diff --git a/apps/ops-agent/src/bootstrap.ts b/apps/ops-agent/src/bootstrap.ts new file mode 100644 index 000000000..3d54bcd7d --- /dev/null +++ b/apps/ops-agent/src/bootstrap.ts @@ -0,0 +1,26 @@ +import type { FastifyListenOptions } from "fastify"; +import type { AdministrativeReleaseEffects } from "./administrative-runtime"; +import { type BuildOpsAgentServerOptions, buildOpsAgentServer } from "./server"; + +/** Production assembly, after token and configuration authority validation. */ +export async function startOpsAgentServer( + options: BuildOpsAgentServerOptions, + releaseEffects: Pick, + listen: FastifyListenOptions, + onReleaseReady?: () => void, +) { + let ready = false; + try { + await releaseEffects.initialize?.(); + ready = true; + } catch { + // Retain the transaction for explicit reconciliation; never retry unknown effects here. + } + const app = buildOpsAgentServer({ + ...options, + releaseRecovery: { ready, ...(!ready ? { reason: "recovery_required" as const } : {}) }, + }); + await app.listen(listen); + if (ready) onReleaseReady?.(); + return app; +} diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index f346fe616..64fa10524 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -14,6 +14,7 @@ import { pruneBackupRetention, } from "./administrative-runtime"; import { isTerminalHandoff, readAgentHandoff } from "./agent-handoff"; +import { startOpsAgentServer } from "./bootstrap"; import { CommunityRepositoryStore, installCommunityRepositoryRuntime, @@ -32,7 +33,6 @@ import { janitorPrivacyBackupScratch, probePrivacyBackupRuntime, } from "./privacy-backup-runtime"; -import { buildOpsAgentServer } from "./server"; import { afterValidatedServiceAuthority, resolveBootstrapEnabledServiceIds, @@ -173,7 +173,6 @@ async function main(): Promise { : undefined; const administrativeCli = createDefaultFixedCli(config.rootDir); const releaseEffects = createDefaultReleaseEffects(config.rootDir, administrativeCli); - await releaseEffects.initialize?.(); createTransitousLockRuntime(runtime, { rootDir: config.rootDir }); createAdministrativeRuntime(runtime, { rootDir: config.rootDir, @@ -195,8 +194,6 @@ async function main(): Promise { void pruneBackupRetention(config.rootDir, retentionDays, administrativeCli).catch(() => { process.stderr.write("Scheduled backup retention failed\n"); }); - runBackupRetention(); - setInterval(runBackupRetention, 24 * 60 * 60 * 1_000); const policyAuthority = async () => { const authority = await authorityLoader(); const resources = loadConfiguredResourceAuthority(); @@ -212,68 +209,78 @@ async function main(): Promise { }; }; const initialPolicyAuthority = await policyAuthority(); - const app = buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, - resourceClaimer: createProductionRegistryResourceClaimer({ - services: initialPolicyAuthority.services, - integrationIds: initialPolicyAuthority.integrationIds, - loadAuthority: policyAuthority, - enabledServiceIds: (authorityServices) => { - const trusted = readTrustedEnabledServiceIds( - paths.infraDir, - authorityServices.map((service) => service.selectionAuthority), - ); - return trusted ?? bootstrapEnabledServiceIds; - }, - trustedData: createFileTrustedOpsDataSource({ - directory: config.trustedConfigDirectory, - token: apiToken, - allowedUids: [uid], - expectedGid: gid, + await startOpsAgentServer( + { + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourceClaimer: createProductionRegistryResourceClaimer({ + services: initialPolicyAuthority.services, + integrationIds: initialPolicyAuthority.integrationIds, + loadAuthority: policyAuthority, + enabledServiceIds: (authorityServices) => { + const trusted = readTrustedEnabledServiceIds( + paths.infraDir, + authorityServices.map((service) => service.selectionAuthority), + ); + return trusted ?? bootstrapEnabledServiceIds; + }, + trustedData: createFileTrustedOpsDataSource({ + directory: config.trustedConfigDirectory, + token: apiToken, + allowedUids: [uid], + expectedGid: gid, + }), + resourceAuthority: { + allowBackup: (kind, backupId, serviceIds) => + inspectBackupAuthority(config.rootDir, kind, backupId, serviceIds), + allowRegion: (_kind, regionId) => + loadConfiguredResourceAuthority().regions.has(regionId), + allowCountry: (_kind, countryCode) => + loadConfiguredResourceAuthority().countries.has(countryCode), + allowDataType: async (dataTypeId) => + inspectDataTypeAuthority((await authorityLoader()).services, dataTypeId), + allowCatalogRevision: (kind, revisionId) => + kind === "data.generateApiKeys" && revisionId === "transitous-fixed-v1", + allowRelease: (_kind, releaseId) => inspectReleaseAuthority(config.rootDir, releaseId), + allowUpdateJobId: (updateJobId) => + /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(updateJobId), + }, }), - resourceAuthority: { - allowBackup: (kind, backupId, serviceIds) => - inspectBackupAuthority(config.rootDir, kind, backupId, serviceIds), - allowRegion: (_kind, regionId) => loadConfiguredResourceAuthority().regions.has(regionId), - allowCountry: (_kind, countryCode) => - loadConfiguredResourceAuthority().countries.has(countryCode), - allowDataType: async (dataTypeId) => - inspectDataTypeAuthority((await authorityLoader()).services, dataTypeId), - allowCatalogRevision: (kind, revisionId) => - kind === "data.generateApiKeys" && revisionId === "transitous-fixed-v1", - allowRelease: (_kind, releaseId) => inspectReleaseAuthority(config.rootDir, releaseId), - allowUpdateJobId: (updateJobId) => /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(updateJobId), + journal, + runtime, + replacementInProgress: () => { + try { + const handoff = readAgentHandoff(config.rootDir); + return handoff !== null && !isTerminalHandoff(handoff.phase); + } catch { + // An unreadable record fails release operations, not every operation. + return false; + } }, - }), - journal, - runtime, - replacementInProgress: () => { - try { - const handoff = readAgentHandoff(config.rootDir); - return handoff !== null && !isTerminalHandoff(handoff.phase); - } catch { - // An unreadable record fails release operations, not every operation. - return false; - } + audit: (event) => process.stdout.write(`${JSON.stringify(event)}\n`), + ...(privacyBackupExtraction ? { privacyBackupExtraction } : {}), + privacyBackupHealth: () => + probePrivacyBackupRuntime({ + ...privacyBackupRuntimeOptions, + enabled: privacyBackupExtraction?.enabled === true, + }), + ...(dawarichCollector && dawarichInspector + ? { + dawarichExport: { + enabled: true, + inspectRuntime: dawarichInspector, + runCollector: dawarichCollector, + }, + } + : {}), }, - audit: (event) => process.stdout.write(`${JSON.stringify(event)}\n`), - ...(privacyBackupExtraction ? { privacyBackupExtraction } : {}), - privacyBackupHealth: () => - probePrivacyBackupRuntime({ - ...privacyBackupRuntimeOptions, - enabled: privacyBackupExtraction?.enabled === true, - }), - ...(dawarichCollector && dawarichInspector - ? { - dawarichExport: { - enabled: true, - inspectRuntime: dawarichInspector, - runCollector: dawarichCollector, - }, - } - : {}), - }); - await app.listen({ host: config.host, port: config.port }); + releaseEffects, + { host: config.host, port: config.port }, + () => { + runBackupRetention(); + const retentionTimer = setInterval(runBackupRetention, 24 * 60 * 60 * 1_000); + retentionTimer.unref(); + }, + ); }); } diff --git a/apps/ops-agent/src/journal.test.ts b/apps/ops-agent/src/journal.test.ts index 6e9ba93a3..8f53c17ce 100644 --- a/apps/ops-agent/src/journal.test.ts +++ b/apps/ops-agent/src/journal.test.ts @@ -230,6 +230,7 @@ describe("version2 execution journal", () => { }); const record = { executionVersion: 2 as const, + requestVersion: 1 as const, ...normalized, operationFingerprint, requestedDeadlineAt: admittedAt + 1000, @@ -250,12 +251,16 @@ describe("version2 execution journal", () => { admittedAt, deadlineAt: admittedAt + 1000, fingerprint: normalized.fingerprint, + requestVersion: 1, state: "failed", errorClass: "recovery_required", terminationRequestedAt: new Date(admittedAt + 1000).toISOString(), }); const retained = recovered.records()[0]; if (!retained) throw new Error("Missing retained execution"); + await expect(recovered.replace([{ ...retained, requestVersion: 2 }])).rejects.toThrow( + "Retained execution identity is immutable", + ); const extended = normalizeExecutionAdmission({ operationFingerprint, operationKey, diff --git a/apps/ops-agent/src/journal.ts b/apps/ops-agent/src/journal.ts index a9acd243f..364a036b8 100644 --- a/apps/ops-agent/src/journal.ts +++ b/apps/ops-agent/src/journal.ts @@ -41,6 +41,7 @@ const ERROR_CLASSES = new Set( export interface PersistedOpsJob { executionVersion?: 2; + requestVersion?: 1 | 2; admittedAt?: number; deadlineAt?: number; operationFingerprint?: string; @@ -96,6 +97,7 @@ function parseJob(value: unknown): PersistedOpsJob { if ( !exactKeys(raw, [ "executionVersion", + "requestVersion", "admittedAt", "deadlineAt", "operationFingerprint", @@ -150,10 +152,13 @@ function parseJob(value: unknown): PersistedOpsJob { operation.kind, raw as unknown as RetainedExecutionAdmission, ); + if (raw.requestVersion !== undefined && raw.requestVersion !== 1 && raw.requestVersion !== 2) + throw new Error("invalid request version"); if (raw.admittedAt !== Date.parse(raw.submittedAt as string)) throw new Error("invalid"); } else if ( [ "executionVersion", + "requestVersion", "admittedAt", "deadlineAt", "operationFingerprint", @@ -338,6 +343,7 @@ export async function openOpsJobJournal( retained.role !== record.role || retained.fingerprint !== record.fingerprint || retained.executionVersion !== record.executionVersion || + retained.requestVersion !== record.requestVersion || retained.admittedAt !== record.admittedAt || retained.deadlineAt !== record.deadlineAt || retained.operationFingerprint !== record.operationFingerprint || diff --git a/apps/ops-agent/src/process-output.test.ts b/apps/ops-agent/src/process-output.test.ts index 510f7d1b7..5661a2312 100644 --- a/apps/ops-agent/src/process-output.test.ts +++ b/apps/ops-agent/src/process-output.test.ts @@ -96,3 +96,64 @@ it("withholds an unterminated quoted credential across multiple real frames", () expect(text).not.toContain(secret); expect(seen).toContain("last-diagnostic"); }); + +describe("fix1 credential boundary regressions", () => { + it.each([ + 'pass\x1b(Bword="synthetic-first\n', + 'pass\x90control\x9cword="synthetic-first\n', + 'password=closed;token="synthetic-first\n', + 'password="closed" token="synthetic-first\n', + 'pass\x1b[31mword="synthetic-first\n', + ])("tracks every normalized open assignment: %j", (first) => { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_s, line) => seen.push(line), + redact: (x) => x, + }); + for (const frame of [ + first, + 'synthetic-continuation\\"still-open\n', + 'synthetic-final"\n', + "last-diagnostic\n", + ]) + output.write("stdout", Buffer.from(frame)); + const all = JSON.stringify({ seen, diagnostics: output.end() }); + for (const secret of ["synthetic-first", "synthetic-continuation", "synthetic-final"]) + expect(all).not.toContain(secret); + expect(seen).toContain("last-diagnostic"); + }); + it.each([" https://synthetic-user", " https://synthetic-user:443"])( + "withholds ambiguous URL userinfo at an artificial boundary: %s", + (suffix) => { + const prefix = "x".repeat(4096 - Buffer.byteLength(suffix)) + suffix; + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_s, line) => seen.push(line), + redact: (x) => x, + }); + output.write("stdout", Buffer.from(prefix)); + output.write("stdout", Buffer.from("@host.invalid/path\nlast-diagnostic\n")); + expect(JSON.stringify({ seen, diagnostics: output.end() })).not.toContain("synthetic-user"); + expect(seen).toContain("last-diagnostic"); + }, + ); +}); + +it("fix1 withholds a quoted assignment whose delimiter follows a real frame boundary", () => { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_s, line) => seen.push(line), + redact: (x) => x, + }); + for (const frame of [ + "password\n", + '="synthetic-key-continuation\n', + 'synthetic-value-continuation"\n', + "last-diagnostic\n", + ]) + output.write("stdout", Buffer.from(frame)); + const all = JSON.stringify({ seen, result: output.end() }); + expect(all).not.toContain("synthetic-key-continuation"); + expect(all).not.toContain("synthetic-value-continuation"); + expect(seen).toContain("last-diagnostic"); +}); diff --git a/apps/ops-agent/src/process-output.ts b/apps/ops-agent/src/process-output.ts index fd2bd5102..fe8c947a8 100644 --- a/apps/ops-agent/src/process-output.ts +++ b/apps/ops-agent/src/process-output.ts @@ -11,8 +11,114 @@ interface OutputOptions { } const QUOTED_CREDENTIAL = /\b((?:authorization|api[-_]?key|access[-_]?token|token|secret|password|passwd|auth|key)\s*[:=]\s*)(["']).*$/gi; -const QUOTED_CREDENTIAL_START = - /\b(?:authorization|api[-_]?key|access[-_]?token|token|secret|password|passwd|auth|key)\s*[:=]\s*(["'])/i; +const CREDENTIAL_KEYS = new Set([ + "authorization", + "apikey", + "api-key", + "api_key", + "access-token", + "access_token", + "token", + "secret", + "password", + "passwd", + "auth", + "key", +]); +interface CredentialScan { + phase: "word" | "key" | "value" | "bare" | "quote"; + word: string; + quote: string; + escaped: boolean; +} +function scanCredential(scan: CredentialScan, character: string): void { + if (scan.phase === "quote") { + if (scan.escaped) scan.escaped = false; + else if (character === "\\") scan.escaped = true; + else if (character === scan.quote) scan.phase = "word"; + return; + } + if (scan.phase === "bare") { + if (/[\s'"&,;)\]}]/.test(character)) scan.phase = "word"; + else return; + } + if (scan.phase === "value") { + if (/\s/.test(character)) return; + if (character === '"' || character === "'") { + scan.phase = "quote"; + scan.quote = character; + scan.escaped = false; + } else scan.phase = "bare"; + return; + } + if (scan.phase === "key") { + if (/\s/.test(character)) return; + scan.phase = character === ":" || character === "=" ? "value" : "word"; + if (scan.phase === "value") return; + } + if (/[\w-]/.test(character)) { + scan.word = (scan.word + character.toLowerCase()).slice(-32); + return; + } + const key = CREDENTIAL_KEYS.has(scan.word.replace(/^--/, "")); + scan.word = ""; + if (key && (character === ":" || character === "=")) scan.phase = "value"; + else if (key && /\s/.test(character)) scan.phase = "key"; +} + +// Normalize before both state tracking and projection, including split ANSI sequences. +function controlFree(state: { control: string }, character: string): string { + if (state.control === "osc" || state.control === "string") { + if (character === "\x07" || character === "\x9c") state.control = ""; + else if (character === "\x1b") state.control += "-escape"; + return ""; + } + if (state.control.endsWith("-escape")) { + state.control = character === "\\" ? "" : state.control.replace("-escape", ""); + return ""; + } + if (state.control === "escape") { + state.control = + character === "[" + ? "csi" + : character === "]" + ? "osc" + : ["P", "^", "_"].includes(character) + ? "string" + : character >= " " && character <= "/" + ? "escape-intermediate" + : ""; + return ""; + } + if (state.control === "escape-intermediate") { + if (character < " " || character > "/") state.control = ""; + return ""; + } + if (state.control === "csi") { + if (character >= "@" && character <= "~") state.control = ""; + if (character !== "\n" && character !== "\r") return ""; + state.control = ""; + } + const code = character.codePointAt(0) ?? 0; + if (code === 27) { + state.control = "escape"; + return ""; + } + if (code === 155) { + state.control = "csi"; + return ""; + } + if ([144, 152, 158, 159].includes(code)) { + state.control = "string"; + return ""; + } + if (code === 157) { + state.control = "osc"; + return ""; + } + if ((code < 32 && ![9, 10, 13].includes(code)) || (code >= 127 && code <= 159)) return ""; + return character; +} export function safeProcessDiagnostic(message: string): string { return scrubSecrets( // biome-ignore lint/suspicious/noControlCharactersInRegex: diagnostic sanitization intentionally strips C0/C1 controls. @@ -28,26 +134,17 @@ export function createBoundedProcessOutput(options: OutputOptions): { const maximum = options.maxTailBytes ?? 16 * 1024; if (!Number.isSafeInteger(maximum) || maximum < 256 || maximum > 16 * 1024) throw Error("Invalid diagnostic budget"); - const streams = { - stdout: { - decoder: new StringDecoder("utf8"), - text: "", - bytes: 0, - dropping: false, - sensitive: false, - credentialQuote: undefined as string | undefined, - cr: false, - }, - stderr: { - decoder: new StringDecoder("utf8"), - text: "", - bytes: 0, - dropping: false, - sensitive: false, - credentialQuote: undefined as string | undefined, - cr: false, - }, - }; + const streamState = () => ({ + decoder: new StringDecoder("utf8"), + text: "", + bytes: 0, + dropping: false, + protected: false, + cr: false, + control: "", + scan: { phase: "word", word: "", quote: "", escaped: false } as CredentialScan, + }); + const streams = { stdout: streamState(), stderr: streamState() }; const tail: ProcessDiagnostics["tail"] = []; let truncated = false; let detached = false; @@ -55,31 +152,14 @@ export function createBoundedProcessOutput(options: OutputOptions): { // Bound serialized storage as well as payload bytes, including tiny-line floods. let retainedBytes = 0; const streamBytes = { stdout: 0, stderr: 0 }; - const emit = (stream: Stream, progress = false) => { + const emit = (stream: Stream) => { const state = streams[stream]; const raw = state.text; - const carriedQuote = state.credentialQuote; - if (carriedQuote) { - if (raw.includes(carriedQuote)) state.credentialQuote = undefined; - } else { - const start = QUOTED_CREDENTIAL_START.exec(raw); - if (start && !raw.slice(start.index + start[0].length).includes(start[1] as string)) - state.credentialQuote = start[1]; - } - const sensitiveNext = - !!state.credentialQuote || - /(?:\b(?:authorization\s*[:=]\s*(?:(?:bearer|basic|token)\s*)?|(?:password|passwd|token|secret|api[-_]?key|access[-_]?token|auth|key)\s*[:=]\s*)|\b(?:https?|ftps?):\/\/[^\s]*|AGE-ENCRYPTED:)$/i.test( - raw, - ) || - (progress && - /(?:authorization|password|passwd|token|secret|api[-_]?key|auth)\s*[:=]\s*\S*$/i.test(raw)); if (raw) { let message: string; try { message = safeProcessDiagnostic( - options.redact( - safeProcessDiagnostic(state.sensitive || carriedQuote ? "[redacted]" : raw), - ), + options.redact(safeProcessDiagnostic(state.protected ? "[redacted]" : raw)), ); } catch { message = "[redacted]"; @@ -116,35 +196,40 @@ export function createBoundedProcessOutput(options: OutputOptions): { } } } - state.sensitive = sensitiveNext; state.text = ""; state.bytes = 0; }; const consume = (stream: Stream, text: string) => { const state = streams[stream]; - for (const character of text) { + for (const decoded of text) { + const character = controlFree(state, decoded); + if (!character) continue; + const progressSecret = character === "\r" && state.scan.phase === "bare"; + scanCredential(state.scan, character); if (character === "\n" || character === "\r") { if (!(character === "\n" && state.cr)) { - if (!state.dropping) emit(stream, character === "\r"); + if (!state.dropping) emit(stream); else { state.text = ""; state.bytes = 0; - state.sensitive = !!state.credentialQuote; } + state.protected = + state.scan.phase === "quote" || + state.scan.phase === "value" || + state.scan.phase === "key" || + progressSecret; state.dropping = false; } state.cr = character === "\r"; continue; } state.cr = false; - if (state.dropping) { - if (character === state.credentialQuote) state.credentialQuote = undefined; - continue; - } + if (state.dropping) continue; const size = Buffer.byteLength(character); if (state.bytes + size > 4096) { - // Never project an artificial continuation: it may be the middle of a credential. - emit(stream); + // A valid-looking URL prefix can still be userinfo. Never publish an artificial frame. + state.text = ""; + state.bytes = 0; state.dropping = true; truncated = true; continue; @@ -167,6 +252,13 @@ export function createBoundedProcessOutput(options: OutputOptions): { if (!streams[stream].dropping) emit(stream); } ended = true; + if (truncated && !detached) { + try { + options.onLine("stderr", PROCESS_OUTPUT_TRUNCATED); + } catch { + detached = true; + } + } } return { tail: [ diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index 6a3ac70e3..0d5175b44 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -26,12 +26,12 @@ const allowAllResources: OpsResourcePolicy = { }; function auth(token = apiToken) { - return { authorization: `Bearer ${token}` }; + return { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }; } function request(operation: OpsOperation, requestId: string, operationKey: string) { return { - version: 1, + version: 2, requestId, operationKey, issuedAt: new Date(baseTime).toISOString(), diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index a599fbe65..70b7a26c4 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -26,11 +26,11 @@ function envelope( issuedAt = nowIso, expiresAt = "2026-08-23T18:00:20.000Z", ) { - return { version: 1, requestId, operationKey, issuedAt, expiresAt, operation }; + return { version: 2, requestId, operationKey, issuedAt, expiresAt, operation }; } function auth(token: string) { - return { authorization: `Bearer ${token}` }; + return { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }; } describe("ops-agent request boundary", () => { @@ -730,3 +730,250 @@ it("observer abort leaves the admitted backend running until its real completion await app.close(); } }); + +it.each([0, 1])( + "fix1 retains safe truncation and final diagnostics through the real CLI/server boundary (exit %s)", + async (exitCode) => { + const { mkdtempSync, writeFileSync, rmSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { createDefaultFixedCli, createAdministrativeRuntime } = await import( + "./administrative-runtime" + ); + const { createUnavailableRuntime } = await import("./runtime"); + const root = mkdtempSync(join(tmpdir(), "openmapx-fix1-diagnostics-")); + const suffix = " https://synthetic-user:443"; + const frames = [ + 'password="closed" token="synthetic-first\n', + 'synthetic-continuation\\"still-open\n', + 'synthetic-final"\n', + 'pass\x1b[31mword="ansi-first\n', + 'ansi-continuation"\n', + `${"x".repeat(4096 - Buffer.byteLength(suffix))}${suffix}@host.invalid/path\n`, + `${"x".repeat(6000)}\n`, + "last-diagnostic\n", + ]; + writeFileSync( + join(root, "cli.mjs"), + `for(const frame of ${JSON.stringify(frames)})process.stdout.write(frame);process.exitCode=${exitCode};`, + ); + const runtime = createUnavailableRuntime(); + createAdministrativeRuntime(runtime, { + rootDir: root, + runFixedCli: createDefaultFixedCli(root, join(root, "agent.mjs")), + loadBuildAuthority: async () => [ + { + serviceId: "osrm", + enabled: true, + isBuiltIn: true, + buildCommand: "openmapx services build osrm", + }, + ], + }); + const retained: import("./journal").PersistedOpsJob[][] = []; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + runtime, + journal: { + records: () => [], + replace: async (x) => { + retained.push(structuredClone([...x])); + }, + }, + }); + try { + const payload = { + ...envelope({ kind: "service.build", serviceId: "osrm" }), + version: 2, + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20000).toISOString(), + }; + const response = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload, + }); + const id = response.json().result.operationId; + await vi.waitFor( + () => expect(retained.at(-1)?.[0]?.state).toBe(exitCode === 0 ? "succeeded" : "failed"), + { timeout: 3000 }, + ); + const status = ( + await app.inject({ + method: "GET", + url: `/v1/operations/${id}`, + headers: { ...auth(apiToken), "x-ops-protocol-version": "2" }, + }) + ).json().result; + expect(status.diagnostics.truncated).toBe(true); + expect(status.diagnostics.tail).toContainEqual({ + stream: "stdout", + message: "last-diagnostic", + }); + expect(retained.at(-1)?.[0]?.diagnostics?.truncated).toBe(true); + const events = ( + await app.inject({ + method: "GET", + url: `/v1/operations/${id}/events`, + headers: auth(apiToken), + }) + ).json(); + const all = JSON.stringify({ status, events, retained }); + for (const secret of [ + "synthetic-first", + "synthetic-continuation", + "synthetic-final", + "ansi-first", + "ansi-continuation", + "synthetic-user", + ]) + expect(all).not.toContain(secret); + if (exitCode !== 0) + expect(status.failureResult.diagnostics.tail).toContainEqual({ + stream: "stdout", + message: "last-diagnostic", + }); + } finally { + await app.close(); + rmSync(root, { recursive: true, force: true }); + } + }, +); + +describe("fix1 baseline protocol compatibility", () => { + const legacyEnvelope = (...args: Parameters) => ({ + ...envelope(...args), + version: 1, + }); + it("projects the exact historical admission/status/lookup/cancel shapes for a v1 caller", async () => { + const { opsOperationFingerprint } = await import("@openmapx/core/ops"); + let finish!: (x: unknown) => void; + let signal: AbortSignal | undefined; + let dispatches = 0; + const operation = { kind: "service.build" as const, serviceId: "osrm" }; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + now: () => new Date(nowIso), + dispatch: async (_o, _r, c) => { + dispatches += 1; + signal = c.signal; + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + try { + const response = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${apiToken}` }, + payload: legacyEnvelope(operation), + }); + const admission = response.json().result; + expect(Object.keys(admission).sort()).toEqual( + ["execution", "operationId", "operationKey", "kind", "state"].sort(), + ); + const duplicate = ( + await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${apiToken}` }, + payload: legacyEnvelope(operation, "ops1_legacyretry00000"), + }) + ).json().result; + expect(duplicate).toEqual(admission); + expect(dispatches).toBe(1); + for (const url of [ + `/v1/operations/${admission.operationId}`, + `/v1/operation-keys/${admission.operationKey}`, + ]) { + const status = ( + await app.inject({ + method: "GET", + url, + headers: { + ...{ authorization: `Bearer ${apiToken}` }, + "x-ops-operation-fingerprint": opsOperationFingerprint(operation), + }, + }) + ).json().result; + expect(status.version).toBe(1); + expect(Object.keys(status).sort()).toEqual( + [ + "version", + "operationId", + "operationKey", + "kind", + "resourceId", + "state", + "submittedAt", + "updatedAt", + ].sort(), + ); + } + const cancel = app.inject({ + method: "DELETE", + url: `/v1/operations/${admission.operationId}`, + headers: { authorization: `Bearer ${apiToken}` }, + }); + await vi.waitFor(() => expect(signal?.aborted).toBe(true)); + finish({ completed: true }); + expect((await cancel).statusCode).toBe(200); + const modern = ( + await app.inject({ + method: "GET", + url: `/v1/operations/${admission.operationId}`, + headers: { ...{ authorization: `Bearer ${apiToken}` }, "x-ops-protocol-version": "2" }, + }) + ).json().result; + expect(modern.version).toBe(2); + expect(modern.fingerprint).toMatch(/^[a-f0-9]{64}$/); + } finally { + finish?.({ completed: true }); + await app.close(); + } + }); +}); + +it("fix1 cannot downgrade modern-origin cancellation to the legacy fence", async () => { + let finish!: (x: unknown) => void; + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAll, + now: () => new Date(nowIso), + dispatch: async () => + new Promise((resolve) => { + finish = resolve; + }), + }); + try { + const admission = ( + await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(apiToken), + payload: envelope({ kind: "service.build", serviceId: "osrm" }), + }) + ).json().result; + const rejected = await app.inject({ + method: "DELETE", + url: `/v1/operations/${admission.operationId}`, + headers: { authorization: `Bearer ${apiToken}` }, + }); + expect(rejected.statusCode).toBe(409); + const cancellation = app.inject({ + method: "DELETE", + url: `/v1/operations/${admission.operationId}`, + headers: { ...auth(apiToken), "x-ops-operation-fingerprint": admission.fingerprint }, + }); + await vi.waitFor(() => expect(finish).toBeTypeOf("function")); + finish({ completed: true }); + expect((await cancellation).statusCode).toBe(200); + } finally { + finish?.({ completed: true }); + await app.close(); + } +}); diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 1a0fbc791..83f8ac352 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -77,6 +77,28 @@ export const OPS_AGENT_RETENTION_LIMITS = { maxAggregateJournalBytes: 24 * 1024 * 1024, maxAggregateEventBytes: DEFAULT_JOB_ENTRIES * MAX_RETAINED_EVENT_BYTES_PER_JOB, } as const; +// Conservative read-only surface while a retained release transaction needs reconciliation. +const RELEASE_RECOVERY_READ_KINDS = new Set([ + "docker.status", + "stack.status", + "configuration.inspect", + "configuration.hardlinks.inspect", + "service.logs", + "service.logs.follow", + "dawarich.provisioning.inspect", + "release.inspect", + "appApi.runtime.inspect", + "agent.handoff.inspect", + "backup.list", + "system.inspect", + "communityServices.inspect", + "serviceSelection.inspect", + "data.inspect", + "valhalla.traffic.inspect", + "postgis.capacity.inspect", + "transitousLock.inspect", + "gbfsCatalogLock.inspect", +]); const FALLBACK_REQUEST_ID = "ops1_unavailable000000"; const OPERATION_KIND_SET = new Set(OPS_OPERATION_KINDS); const REQUEST_ID_PATTERN = /^ops1_[A-Za-z0-9_-]{16,64}$/; @@ -93,6 +115,7 @@ export interface OpsAuditEvent { export interface BuildOpsAgentServerOptions { tokens: Record; + releaseRecovery?: { ready: boolean; reason?: "recovery_required" }; resourcePolicy?: OpsResourcePolicy; resourceClaimer?: OpsResourceClaimer; journal?: OpsJobJournal; @@ -366,6 +389,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ...(job.executionVersion === 2 ? { executionVersion: 2 as const, + requestVersion: job.requestVersion, admittedAt: job.admittedAt, deadlineAt: job.deadlineAt, operationFingerprint: job.operationFingerprint, @@ -555,7 +579,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ), { stream, message: boundedMessage }, ]; - let truncated = job.diagnostics?.truncated ?? false; + let truncated = + (job.diagnostics?.truncated ?? false) || boundedMessage === PROCESS_OUTPUT_TRUNCATED; while ( tail.length > 255 || Buffer.byteLength(JSON.stringify({ tail, truncated })) > 16 * 1024 - 128 @@ -592,11 +617,25 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }, }); - const statusFor = (job: JobRecord): OpsJobStatus => ({ - version: job.executionVersion === 2 ? 2 : 1, - ...(job.executionVersion === 2 ? identityFor(job) : {}), - ...(job.diagnostics ? { diagnostics: job.diagnostics } : {}), - ...(job.failureResult ? { failureResult: job.failureResult } : {}), + const projectionVersion = (request: { headers: Record }): 1 | 2 => + request.headers["x-ops-protocol-version"] === "2" ? 2 : 1; + const admissionFor = (job: JobRecord, version: 1 | 2) => ({ + execution: "async", + operationId: job.operationId, + operationKey: job.operationKey, + kind: job.operation.kind, + state: outwardStateFor(job), + ...(version === 2 && job.executionVersion === 2 ? { version: 2, ...identityFor(job) } : {}), + }); + const statusFor = (job: JobRecord, version: 1 | 2): OpsJobStatus => ({ + version: version === 2 && job.executionVersion === 2 ? 2 : 1, + ...(version === 2 && job.executionVersion === 2 ? identityFor(job) : {}), + ...(version === 2 && job.executionVersion === 2 && job.diagnostics + ? { diagnostics: job.diagnostics } + : {}), + ...(version === 2 && job.executionVersion === 2 && job.failureResult + ? { failureResult: job.failureResult } + : {}), operationId: job.operationId, operationKey: job.operationKey, kind: job.operation.kind, @@ -876,6 +915,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif .code(statusCode) .send(redactedOpsError(requestIdFromBody(request.body), "validation", error)); }); + app.get("/v1/protocol", async (request, reply) => { + const requestId = validRequestId(request.headers["x-ops-request-id"]); + if (!resolveRole(request.headers.authorization, options.tokens)) + return reply.code(401).send(redactedOpsError(requestId, "authentication")); + return reply.send(opsSuccess(requestId, { versions: [1, 2] })); + }); app.get("/health", async () => { let privacyBackup = { ready: false, @@ -887,7 +932,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } catch { // Health evidence is fail-closed and never includes Docker/inventory errors. } - return { ok: true, privacyBackup }; + return { + ok: true, + privacyBackup, + ...(options.releaseRecovery ? { releaseRecovery: options.releaseRecovery } : {}), + }; }); app.post("/v1/operations", async (request, reply) => { @@ -918,6 +967,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif controlHeaders.filter((x) => x === "x-openmapx-parent-execution").length > 1 ) throw new RequestGuardError("authorization", 403); + if (parsed.version === 1 && (parentHeader !== undefined || delegationHeader !== undefined)) + throw new RequestGuardError("authorization", 403); if (parentHeader !== undefined) { if (role !== "api" || typeof parentHeader !== "string") throw new RequestGuardError("authorization", 403); @@ -970,17 +1021,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const existing = jobsByKey.get(ownerKey); if (existing) { if (!matchesRetained(existing)) throw new RequestGuardError("conflict", 409); - return reply.code(202).send( - opsSuccess(parsed.requestId, { - execution: "async", - operationId: existing.operationId, - operationKey: existing.operationKey, - kind: existing.operation.kind, - state: outwardStateFor(existing), - version: existing.executionVersion === 2 ? 2 : 1, - ...identityFor(existing), - }), - ); + return reply + .code(202) + .send(opsSuccess(parsed.requestId, admissionFor(existing, parsed.version))); } if (jobsById.size >= maxJobEntries) { throw new RequestGuardError("busy", 429); @@ -990,6 +1033,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("busy", 409); } } + if (options.releaseRecovery?.ready === false && !RELEASE_RECOVERY_READ_KINDS.has(kind)) + throw new RequestGuardError("recovery_required", 409); normalized = normalize(nowMs); if (active >= maxConcurrency) throw new RequestGuardError("busy", 429); active += 1; @@ -1100,6 +1145,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif operationId, operationKey: parsed.operationKey, executionVersion: 2, + requestVersion: parsed.version, ...normalized, operationFingerprint, requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, @@ -1129,17 +1175,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (admission.type === "retained") { await claimAdmission?.rollback().catch(() => undefined); active -= 1; - return reply.code(202).send( - opsSuccess(parsed.requestId, { - execution: "async", - operationId: admission.job.operationId, - operationKey: admission.job.operationKey, - kind: admission.job.operation.kind, - state: outwardStateFor(admission.job), - version: admission.job.executionVersion === 2 ? 2 : 1, - ...identityFor(admission.job), - }), - ); + return reply + .code(202) + .send(opsSuccess(parsed.requestId, admissionFor(admission.job, parsed.version))); } const { job } = admission; try { @@ -1157,17 +1195,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("runtime", 500); } await runAsync(job, startedAt, claim, claimAdmission); - return reply.code(202).send( - opsSuccess(parsed.requestId, { - execution: "async", - operationId: job.operationId, - operationKey: job.operationKey, - kind: job.operation.kind, - state: outwardStateFor(job), - version: job.executionVersion === 2 ? 2 : 1, - ...identityFor(job), - }), - ); + return reply + .code(202) + .send(opsSuccess(parsed.requestId, admissionFor(job, parsed.version))); } const controller = new AbortController(); try { @@ -1265,7 +1295,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (bodyFingerprint !== undefined) { if (bodyFingerprint !== (job.operationFingerprint ?? job.fingerprint)) return reply.code(409).send(redactedOpsError(requestId, "conflict")); - if (fingerprint !== undefined && fingerprint !== job.fingerprint) + if ( + fingerprint !== undefined && + fingerprint !== job.fingerprint && + fingerprint !== bodyFingerprint + ) return reply.code(409).send(redactedOpsError(requestId, "conflict")); const requested = request.headers["x-ops-requested-deadline"]; if (requested !== undefined) { @@ -1297,7 +1331,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (fingerprint !== job.fingerprint && fingerprint !== job.operationFingerprint) return reply.code(409).send(redactedOpsError(requestId, "conflict")); } - return reply.code(200).send(opsSuccess(requestId, statusFor(job))); + return reply.code(200).send(opsSuccess(requestId, statusFor(job, projectionVersion(request)))); }); app.get("/v1/operations/:operationId", async (request, reply) => { @@ -1313,7 +1347,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (!job || job.role !== role) { return reply.code(404).send(redactedOpsError(requestId, "not_found")); } - return reply.code(200).send(opsSuccess(requestId, statusFor(job))); + return reply.code(200).send(opsSuccess(requestId, statusFor(job, projectionVersion(request)))); }); app.delete("/v1/operations/:operationId", async (request, reply) => { @@ -1331,11 +1365,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } if ( job.executionVersion === 2 && - request.headers["x-ops-operation-fingerprint"] !== job.fingerprint + request.headers["x-ops-operation-fingerprint"] !== job.fingerprint && + !(projectionVersion(request) === 1 && job.requestVersion === 1) ) return reply.code(409).send(redactedOpsError(requestId, "conflict")); if (job.terminalAt !== undefined) { - return reply.code(200).send(opsSuccess(requestId, statusFor(job))); + return reply + .code(200) + .send(opsSuccess(requestId, statusFor(job, projectionVersion(request)))); } const controller = activeControllers.get(operationId); if (!controller) { @@ -1378,7 +1415,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (waitTimer !== undefined) clearTimeout(waitTimer); }); } - return reply.code(200).send(opsSuccess(requestId, statusFor(job))); + return reply.code(200).send(opsSuccess(requestId, statusFor(job, projectionVersion(request)))); }); app.get("/v1/operations/:operationId/events", async (request, reply) => { diff --git a/packages/cli/__tests__/cli-execution-deadline.test.ts b/packages/cli/__tests__/cli-execution-deadline.test.ts new file mode 100644 index 000000000..d4f82dc1e --- /dev/null +++ b/packages/cli/__tests__/cli-execution-deadline.test.ts @@ -0,0 +1,149 @@ +import { execFile } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +const exec = promisify(execFile); +it("fix1 actual CLI entry refuses an expired retained deadline before its nested Docker runner", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-cli-deadline-")); + const marker = join(root, "docker-started"); + mkdirSync(join(root, "infra/docker"), { recursive: true }); + mkdirSync(join(root, "bin")); + writeFileSync( + join(root, "bin/docker"), + `#!${process.execPath}\nrequire('node:fs').writeFileSync(${JSON.stringify(marker)},'started');\n`, + { mode: 0o755 }, + ); + const execution = { + executor: "ops-agent", + principal: "api", + executionId: "job1_0123456789abcdef", + operationKey: "opk1_0123456789abcdef", + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() - 1000, + }; + try { + let failure: unknown; + try { + await exec( + resolve("packages/cli/node_modules/.bin/tsx"), + [ + "--tsconfig", + resolve("packages/cli/tsconfig.json"), + resolve("packages/cli/src/index.ts"), + "services", + "logs", + "redis", + ], + { + env: { + ...process.env, + OPENMAPX_ROOT_DIR: root, + OPENMAPX_CONTROL_EXECUTION: JSON.stringify(execution), + PATH: `${join(root, "bin")}:${process.env.PATH}`, + }, + timeout: 10000, + }, + ); + } catch (error) { + failure = error; + } + expect(existsSync(marker)).toBe(false); + expect(String(failure)).toContain("Execution deadline elapsed"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +it("fix1 nested default build runner spends the stored remaining budget across phases and never publishes after expiry", async () => { + const { buildOsrmGraph } = await import("../src/lib/osrm-graph"); + const { withCliExecutionDeadline } = await import("../src/lib/execution-deadline"); + const { readFileSync } = await import("node:fs"); + const root = mkdtempSync(join(tmpdir(), "openmapx-nested-deadline-")); + const previousPath = process.env.PATH, + previousExecution = process.env.OPENMAPX_CONTROL_EXECUTION; + const marker = join(root, "phases"); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(join(root, "services")); + mkdirSync(join(root, "infra/docker/data/osm"), { recursive: true }); + mkdirSync(join(root, "bin")); + writeFileSync(join(root, "infra/docker/data/osm/region.osm.pbf"), "synthetic-pbf"); + mkdirSync(join(root, "infra/docker/data/osrm-graph")); + writeFileSync(join(root, "infra/docker/data/osrm-graph/region.osrm"), "previous-live"); + writeFileSync( + join(root, "bin/docker"), + `#!${process.execPath}\nconst fs=require('node:fs');fs.appendFileSync(${JSON.stringify(marker)},'phase\\n');const mount=process.argv.find(a=>a.endsWith(':/data')).slice(0,-6);setTimeout(()=>{fs.writeFileSync(mount+'/region.osrm','new');},400);\n`, + { mode: 0o755 }, + ); + try { + process.env.PATH = `${join(root, "bin")}:${previousPath}`; + process.env.OPENMAPX_CONTROL_EXECUTION = JSON.stringify({ + executor: "ops-agent", + principal: "api", + executionId: "job1_0123456789abcdef", + operationKey: "opk1_0123456789abcdef", + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 650, + }); + await expect( + withCliExecutionDeadline(["services", "build", "osrm"], () => + buildOsrmGraph({ rootDir: root, image: "synthetic/image:1" }), + ), + ).rejects.toThrow(); + expect(readFileSync(marker, "utf8").trim().split("\n").length).toBeLessThanOrEqual(2); + expect(readFileSync(join(root, "infra/docker/data/osrm-graph/region.osrm"), "utf8")).toBe( + "previous-live", + ); + } finally { + if (previousExecution === undefined) delete process.env.OPENMAPX_CONTROL_EXECUTION; + else process.env.OPENMAPX_CONTROL_EXECUTION = previousExecution; + process.env.PATH = previousPath; + rmSync(root, { recursive: true, force: true }); + } +}); + +it("fix1 unsigned metadata can only clamp policy and cannot authorize a remote mutation", async () => { + const { vi } = await import("vitest"); + const { + withCliExecutionDeadline, + cliProcessOptions, + cliAcquisitionOptions, + constrainCliDataManagerTransport, + } = await import("../src/lib/execution-deadline"); + const { opsExecutionBudget } = await import("@openmapx/core/ops"); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.stubEnv("OPENMAPX_CLI_DEADLINE_AT", String(Date.now() + 100 * 24 * 3600_000)); + try { + const request = vi.fn(async () => ({})); + await withCliExecutionDeadline(["services", "build", "osrm"], async () => { + const original = cliProcessOptions(); + expect(original.timeout).toBe(opsExecutionBudget("service.build").processMs); + expect(original.env?.OPENMAPX_CONTROL_EXECUTION).toBeUndefined(); + if (original.timeout === undefined) throw Error("Missing deadline clamp"); + vi.setSystemTime(Date.now() + original.timeout - 5); + expect(cliProcessOptions().timeout).toBe(5); + await withCliExecutionDeadline(["services", "update"], async () => { + expect(cliProcessOptions().timeout).toBe(5); + }); + const transport = constrainCliDataManagerTransport({ request } as never); + expect(() => transport.request("link", "/link", { method: "POST" })).toThrow( + "deadline-aware receiver upgrade", + ); + expect(request).not.toHaveBeenCalled(); + expect(cliAcquisitionOptions().requestTimeoutMs).toBe(5); + vi.setSystemTime(Date.now() + 6); + expect(() => cliProcessOptions()).toThrow("Execution deadline elapsed"); + }); + vi.stubEnv("OPENMAPX_PARENT_EXECUTION", "unsigned-parent"); + await expect( + withCliExecutionDeadline(["services", "build"], async () => undefined), + ).rejects.toThrow("CLI control verifier is unavailable"); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + } +}); diff --git a/packages/cli/src/commands/backup.ts b/packages/cli/src/commands/backup.ts index ab8904de8..cc90c20b4 100644 --- a/packages/cli/src/commands/backup.ts +++ b/packages/cli/src/commands/backup.ts @@ -36,6 +36,7 @@ import { PLATFORM_VERSION } from "@openmapx/integration-framework"; import type { Command } from "commander"; import { execa } from "execa"; import kleur from "kleur"; +import { cliProcessOptions } from "../lib/execution-deadline"; import { log, table } from "../lib/output"; import { repoPaths } from "../lib/paths"; import { applyServiceSelection } from "../lib/service-selection"; @@ -393,6 +394,7 @@ function ctxFromRepo(rootDir?: string): ComposeContext { async function dockerCompose(ctx: ComposeContext, args: string[]): Promise { const result = await execa("docker", [...ctx.composeArgs, ...args], { + ...cliProcessOptions(), cwd: ctx.cwd, reject: false, }); @@ -484,7 +486,7 @@ async function waitForPostgres( "-t", "1", ], - { cwd: ctx.cwd, reject: false, timeout: 5_000 }, + { ...cliProcessOptions(5_000), cwd: ctx.cwd, reject: false }, ); if (result.exitCode === 0) return; if (attempt < 30) await new Promise((resolve) => setTimeout(resolve, 1_000)); @@ -800,7 +802,13 @@ async function pgDumpToFile( "--no-privileges", db, ], - { cwd: ctx.cwd, reject: false, stderr: "pipe", buffer: { stdout: false } }, + { + ...cliProcessOptions(), + cwd: ctx.cwd, + reject: false, + stderr: "pipe", + buffer: { stdout: false }, + }, ); if (!sub.stdout) { @@ -841,7 +849,7 @@ async function tarVolumeToFile( "/source", ".", ], - { reject: false }, + { ...cliProcessOptions(), reject: false }, ); if (result.exitCode !== 0) { throw new Error( @@ -1106,7 +1114,7 @@ async function listRestoredUsers( "-c", `SELECT json_build_object('id', id, 'email', email)::text FROM "user"`, ], - { cwd: ctx.cwd, reject: false, maxBuffer: 16 * 1024 * 1024 }, + { ...cliProcessOptions(), cwd: ctx.cwd, reject: false, maxBuffer: 16 * 1024 * 1024 }, ); if (result.exitCode !== 0) throw new Error(`Unable to enumerate restored users: ${result.stderr}`); @@ -1227,7 +1235,7 @@ COMMIT; "-v", "ON_ERROR_STOP=1", ], - { cwd: ctx.cwd, reject: false, input: statement }, + { ...cliProcessOptions(), cwd: ctx.cwd, reject: false, input: statement }, ); if (result.exitCode !== 0) { throw new Error(`Unable to replay user erasure (psql exit ${result.exitCode})`); @@ -1504,7 +1512,7 @@ async function pgRestoreFromFile( const drop = await execa( "docker", [...ctx.composeArgs, "exec", "-T", serviceId, "dropdb", "-U", user, "--if-exists", db], - { cwd: ctx.cwd, reject: false }, + { ...cliProcessOptions(), cwd: ctx.cwd, reject: false }, ); if (drop.exitCode !== 0) { throw new Error(`dropdb failed: ${drop.stderr ?? ""}`); @@ -1513,7 +1521,7 @@ async function pgRestoreFromFile( const create = await execa( "docker", [...ctx.composeArgs, "exec", "-T", serviceId, "createdb", "-U", user, db], - { cwd: ctx.cwd, reject: false }, + { ...cliProcessOptions(), cwd: ctx.cwd, reject: false }, ); if (create.exitCode !== 0) { throw new Error(`createdb failed: ${create.stderr ?? ""}`); @@ -1521,6 +1529,7 @@ async function pgRestoreFromFile( // gunzip | docker compose exec -T psql -U const gunzip = execa("gunzip", ["-c", gzFile], { + ...cliProcessOptions(), cwd: ctx.cwd, reject: false, stdout: "pipe", @@ -1528,7 +1537,7 @@ async function pgRestoreFromFile( const psql = execa( "docker", [...ctx.composeArgs, "exec", "-T", serviceId, "psql", "-U", user, "-v", "ON_ERROR_STOP=1", db], - { cwd: ctx.cwd, reject: false, input: gunzip.stdout ?? undefined }, + { ...cliProcessOptions(), cwd: ctx.cwd, reject: false, input: gunzip.stdout ?? undefined }, ); const [gunzipRes, psqlRes] = await Promise.all([gunzip, psql]); @@ -1564,7 +1573,7 @@ async function tarRestoreFromFile( `cd /target && find . -mindepth 1 -delete && ` + `tar -xzf /backup/${shellEscape(fileName)} -C /target`, ], - { reject: false }, + { ...cliProcessOptions(), reject: false }, ); if (result.exitCode !== 0) { throw new Error( diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index d8e6961ce..8adca3113 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -5,6 +5,7 @@ import { services as coreServices } from "@openmapx/core/server"; import { type Command, Option } from "commander"; import { assertCliDeploymentSecrets } from "../lib/deployment-secret-policy"; import { dockerComposeStream } from "../lib/docker"; +import { assertCliExecutionAvailable } from "../lib/execution-deadline"; import { applyGeneratedHardlinks } from "../lib/hardlinks"; import { log } from "../lib/output"; import { repoPaths } from "../lib/paths"; @@ -190,10 +191,12 @@ export function rotateRedisPasswordForRepo( } export async function renderComposeForRepo(opts: RenderRepoOptions): Promise { + assertCliExecutionAvailable(); coreServices.releaseSelection(); const paths = repoPaths(opts.rootDir); assertCliDeploymentSecrets(); return withConfigurationGenerationWriter(paths.infraDir, async (writer) => { + assertCliExecutionAvailable(); ensurePlatformPrivateDirectory(join(paths.infraDir, "data", "ops-agent", "trusted-config")); const erasureDirectory = join(paths.infraDir, "data", "erasure"); ensurePlatformPrivateDirectory(erasureDirectory); @@ -280,6 +283,7 @@ export async function renderComposeForRepo(opts: RenderRepoOptions): Promise 0 ? { envKeys } : {}) }]; }); + assertCliExecutionAvailable(); const committed = await writer.commit({ services, integrationSchemas, diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index 160d14e0c..ba2c70a94 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -24,6 +24,7 @@ import { resolveTransitousCountries, TRANSITOUS_COUNTRIES_ENV, } from "../lib/env-defaults"; +import { cliAcquisitionOptions, constrainCliDataManagerTransport } from "../lib/execution-deadline"; import { applyGeneratedHardlinks } from "../lib/hardlinks"; import { log, routeLogsToStderr, table } from "../lib/output"; import { buildServices, resolveDataBuildServiceId } from "../lib/service-builds"; @@ -31,10 +32,13 @@ import { generateTransitousApiKeys } from "../lib/transitous-api-keys"; import { renderComposeForRepo } from "./compose"; function createCliTransport() { - return dm.createDataManagerTransport({ - baseUrl: dataManagerUrl(), - authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), - }); + return constrainCliDataManagerTransport( + dm.createDataManagerTransport({ + ...cliAcquisitionOptions(), + baseUrl: dataManagerUrl(), + authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), + }), + ); } function formatBytes(n: number): string { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 2c48cf662..3c9dc0e36 100755 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -11,12 +11,12 @@ import { registerServicesCommands } from "./commands/services"; import { registerTransitousCommands } from "./commands/transitous-bump"; import { registerTransportApisPinCommands } from "./commands/transport-apis-pin"; import { registerUsersCommands } from "./commands/users"; +import { assertCliExecutionAvailable, withCliExecutionDeadline } from "./lib/execution-deadline"; import { loadInfraEnv } from "./lib/infra-env"; // Mirror Docker Compose's behaviour: auto-load infra/docker/.env so that // `--domain` and `SERVICE__` overrides resolve consistently between // `pnpm openmapx compose render` (CLI) and `docker compose up` (compose). -loadInfraEnv(); const program = new Command(); @@ -39,4 +39,8 @@ registerCacheCommands(program); registerTransportApisPinCommands(program); registerTransitousCommands(program); -await program.parseAsync(process.argv); +await withCliExecutionDeadline(process.argv.slice(2), async () => { + loadInfraEnv(); + program.hook("preAction", () => assertCliExecutionAvailable()); + await program.parseAsync(process.argv); +}); diff --git a/packages/cli/src/lib/build-artifacts.ts b/packages/cli/src/lib/build-artifacts.ts index 6fd2c5f6d..77f33760f 100644 --- a/packages/cli/src/lib/build-artifacts.ts +++ b/packages/cli/src/lib/build-artifacts.ts @@ -16,6 +16,7 @@ import { dirname, isAbsolute, join } from "node:path"; import { services as coreServices } from "@openmapx/core/server"; import { execa } from "execa"; import { promoteStagedBuildDirs } from "./build-staging"; +import { cliProcessOptions } from "./execution-deadline"; import { repoPaths, resolveInvocationPath } from "./paths"; /** @@ -121,6 +122,7 @@ export function readBuildRecord(serviceId: string, rootDir?: string): BuildRecor export async function currentRevision(rootDir?: string): Promise { const result = await execa("git", ["rev-parse", "HEAD"], { + ...cliProcessOptions(), cwd: repoPaths(rootDir).root, reject: false, }); @@ -216,10 +218,12 @@ async function defaultTar( args: string[], opts: { cwd?: string; stdin?: "inherit"; stdout?: "inherit" }, ): Promise { + const executionOptions = cliProcessOptions(); await execa("tar", args, { + ...executionOptions, cwd: opts.cwd, // Stops macOS tar from adding `._*` AppleDouble members. - env: { COPYFILE_DISABLE: "1" }, + env: { ...executionOptions.env, COPYFILE_DISABLE: "1" }, stdin: opts.stdin ?? "ignore", stdout: opts.stdout ?? "inherit", stderr: "inherit", diff --git a/packages/cli/src/lib/build-staging.ts b/packages/cli/src/lib/build-staging.ts index 7743c9992..431bb4d8d 100644 --- a/packages/cli/src/lib/build-staging.ts +++ b/packages/cli/src/lib/build-staging.ts @@ -1,4 +1,5 @@ import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs"; +import { assertCliExecutionAvailable } from "./execution-deadline"; import { log } from "./output"; /** @@ -20,6 +21,7 @@ function previousBuildDir(liveDir: string): string { * that can't be removed fails the build up front rather than after it. */ export function prepareStagedBuildDir(liveDir: string): string { + assertCliExecutionAvailable(); const nextDir = stagedBuildDir(liveDir); rmSync(previousBuildDir(liveDir), { recursive: true, force: true }); rmSync(nextDir, { recursive: true, force: true }); @@ -39,6 +41,7 @@ export interface StagedSwap { * artifact is already live by then. */ export function promoteStagedBuildDirs(swaps: readonly StagedSwap[]): void { + assertCliExecutionAvailable(); const moved: StagedSwap[] = []; const installed: StagedSwap[] = []; try { diff --git a/packages/cli/src/lib/docker.ts b/packages/cli/src/lib/docker.ts index 3f68b86fe..a2768f886 100644 --- a/packages/cli/src/lib/docker.ts +++ b/packages/cli/src/lib/docker.ts @@ -1,5 +1,6 @@ import { services as coreServices } from "@openmapx/core/server"; import { execa } from "execa"; +import { cliProcessOptions } from "./execution-deadline"; import { repoPaths } from "./paths"; export async function dockerCompose( @@ -7,6 +8,7 @@ export async function dockerCompose( ): Promise<{ stdout: string; stderr: string; exitCode: number }> { const paths = repoPaths(); const result = await execa("docker", [...coreServices.stackComposeArgs(paths), ...args], { + ...cliProcessOptions(), cwd: paths.infraDir, reject: false, }); @@ -26,7 +28,7 @@ export async function dockerCompose( export async function dockerRun( args: string[], ): Promise<{ stdout: string; stderr: string; exitCode: number }> { - const result = await execa("docker", ["run", ...args], { reject: false }); + const result = await execa("docker", ["run", ...args], { ...cliProcessOptions(), reject: false }); return { stdout: result.stdout ?? "", stderr: result.stderr ?? "", @@ -37,6 +39,7 @@ export async function dockerRun( export async function dockerComposeStream(args: string[]): Promise { const paths = repoPaths(); const sub = execa("docker", [...coreServices.stackComposeArgs(paths), ...args], { + ...cliProcessOptions(), cwd: paths.infraDir, stdio: "inherit", reject: false, diff --git a/packages/cli/src/lib/execution-deadline.ts b/packages/cli/src/lib/execution-deadline.ts new file mode 100644 index 000000000..f9c94208f --- /dev/null +++ b/packages/cli/src/lib/execution-deadline.ts @@ -0,0 +1,110 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type { DataManagerTransport } from "@openmapx/core/data-manager-sdk"; +import { executionRefSchema, opsExecutionBudget } from "@openmapx/core/ops"; + +// Restriction only. These values authenticate no principal, parent, role or lease. +const deadlines = new AsyncLocalStorage<{ deadlineAt: number; signal: AbortSignal }>(); +function localBudget(args: readonly string[]): number { + const [group, action] = args; + const kind = + group === "services" && action === "build" + ? "service.build" + : group === "services" && action === "build-all" + ? "services.buildAll" + : group === "services" && action === "logs" + ? "service.logs.follow" + : group === "services" && action === "update" + ? "system.update" + : group === "data" + ? "data.update" + : group === "check" + ? "system.diagnostics" + : "backup.create"; + return opsExecutionBudget(kind).processMs; +} +export async function withCliExecutionDeadline( + args: readonly string[], + run: () => Promise, +): Promise { + const encoded = process.env.OPENMAPX_CONTROL_EXECUTION; + const inherited = process.env.OPENMAPX_CLI_DEADLINE_AT; + if (process.env.OPENMAPX_CONTROL_DELEGATION || process.env.OPENMAPX_PARENT_EXECUTION) + throw Error("CLI control verifier is unavailable"); + if (encoded === undefined && inherited === undefined) return run(); + if (encoded !== undefined && Buffer.byteLength(encoded) > 2048) + throw Error("Invalid execution deadline"); + const requested = + encoded === undefined ? Infinity : executionRefSchema.parse(JSON.parse(encoded)).deadlineAt; + if ( + inherited !== undefined && + (!/^\d{1,16}$/.test(inherited) || !Number.isSafeInteger(Number(inherited))) + ) + throw Error("Invalid execution deadline"); + const deadlineAt = Math.min( + requested, + inherited === undefined ? Infinity : Number(inherited), + Date.now() + localBudget(args), + deadlines.getStore()?.deadlineAt ?? Infinity, + ); + if (deadlineAt <= Date.now()) throw Error("Execution deadline elapsed"); + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(Error("Execution deadline elapsed")), + deadlineAt - Date.now(), + ); + timer.unref(); + try { + return await deadlines.run({ deadlineAt, signal: controller.signal }, run); + } finally { + clearTimeout(timer); + } +} +export function assertCliExecutionAvailable(): void { + const context = deadlines.getStore(); + if (context && (context.signal.aborted || Date.now() >= context.deadlineAt)) + throw Error("Execution deadline elapsed"); +} +export function cliProcessOptions(phaseTimeoutMs?: number) { + assertCliExecutionAvailable(); + const context = deadlines.getStore(); + if (!context) return phaseTimeoutMs === undefined ? {} : { timeout: phaseTimeoutMs }; + return { + timeout: Math.max(1, Math.min(phaseTimeoutMs ?? Infinity, context.deadlineAt - Date.now())), + cancelSignal: context.signal, + forceKillAfterDelay: 2000, + env: { + OPENMAPX_CONTROL_EXECUTION: undefined, + OPENMAPX_CONTROL_DELEGATION: undefined, + OPENMAPX_PARENT_EXECUTION: undefined, + OPENMAPX_CLI_DEADLINE_AT: String(context.deadlineAt), + }, + }; +} +export function cliAcquisitionOptions(): { signal?: AbortSignal; requestTimeoutMs?: number } { + assertCliExecutionAvailable(); + const context = deadlines.getStore(); + return context + ? { signal: context.signal, requestTimeoutMs: Math.min(15000, context.deadlineAt - Date.now()) } + : {}; +} +/** Remote mutations need a receiver-retained deadline. Unsigned CLI metadata cannot supply it. */ +export function constrainCliDataManagerTransport( + transport: DataManagerTransport, +): DataManagerTransport { + return { + request(operationId, url, options) { + assertCliExecutionAvailable(); + if ( + deadlines.getStore() && + !["GET", "HEAD"].includes((options?.method ?? "GET").toUpperCase()) + ) + throw Error("Managed data-manager execution requires a deadline-aware receiver upgrade"); + return transport.request(operationId, url, options); + }, + }; +} +export function assertCliRemoteMutationSupported(): void { + assertCliExecutionAvailable(); + if (deadlines.getStore()) + throw Error("Managed data-manager execution requires a deadline-aware receiver upgrade"); +} diff --git a/packages/cli/src/lib/hardlinks.ts b/packages/cli/src/lib/hardlinks.ts index c9e7ee072..4b5ce7480 100644 --- a/packages/cli/src/lib/hardlinks.ts +++ b/packages/cli/src/lib/hardlinks.ts @@ -1,3 +1,8 @@ +import { + assertCliExecutionAvailable, + assertCliRemoteMutationSupported, + cliAcquisitionOptions, +} from "./execution-deadline"; // CLI-side hardlink apply. Two paths: // // 1. Preferred — POST the plan to the data-manager's `/link` endpoint. @@ -106,8 +111,9 @@ async function dataManagerReachable(baseUrl: string, timeoutMs: number): Promise const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { + const { signal } = cliAcquisitionOptions(); const res = await fetch(`${baseUrl.replace(/\/+$/, "")}/status`, { - signal: controller.signal, + signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal, }); return res.ok; } catch { @@ -120,6 +126,7 @@ async function dataManagerReachable(baseUrl: string, timeoutMs: number): Promise export async function applyGeneratedHardlinks( opts: ApplyGeneratedHardlinkOptions = {}, ): Promise { + assertCliExecutionAvailable(); const paths = repoPaths(opts.rootDir); const planPath = hardlinkPlanPath(paths.infraDir); if (!existsSync(planPath)) { @@ -146,7 +153,9 @@ export async function applyGeneratedHardlinks( const baseUrl = opts.dataManagerUrl ?? dataManagerUrl(); const timeout = opts.reachabilityTimeoutMs ?? 1500; if (await dataManagerReachable(baseUrl, timeout)) { + assertCliRemoteMutationSupported(); const transport = createDataManagerTransport({ + ...cliAcquisitionOptions(), baseUrl, authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), }); @@ -165,6 +174,7 @@ export async function applyGeneratedHardlinks( } } + assertCliExecutionAvailable(); const result = applyHardlinkPlan(plan, { rootDir: dataRoot, prune: opts.prune }); return { applied: true, diff --git a/packages/cli/src/lib/motis-data.ts b/packages/cli/src/lib/motis-data.ts index 31551e14e..d43982f51 100644 --- a/packages/cli/src/lib/motis-data.ts +++ b/packages/cli/src/lib/motis-data.ts @@ -37,6 +37,7 @@ import { } from "@openmapx/transitous-core"; import { execa } from "execa"; import { TRANSITOUS_COUNTRIES_ENV } from "./env-defaults"; +import { cliProcessOptions } from "./execution-deadline"; import { readFeedProxyVars, renderFeedProxyNginxConfig } from "./motis-feed-proxy"; import { resolveOsmPbf } from "./osm-pbf"; import { repoPaths } from "./paths"; @@ -125,7 +126,11 @@ async function defaultRunner( args: string[], opts: { cwd?: string; stdio?: "inherit" | "pipe" }, ): Promise { - await execa(command, args, { cwd: opts.cwd, stdio: opts.stdio ?? "inherit" }); + await execa(command, args, { + ...cliProcessOptions(), + cwd: opts.cwd, + stdio: opts.stdio ?? "inherit", + }); } function linkOrCopy(source: string, target: string): void { diff --git a/packages/cli/src/lib/osrm-graph.ts b/packages/cli/src/lib/osrm-graph.ts index d0c2198dd..729c6fb1e 100644 --- a/packages/cli/src/lib/osrm-graph.ts +++ b/packages/cli/src/lib/osrm-graph.ts @@ -12,6 +12,7 @@ import { import { basename, dirname, join, resolve } from "node:path"; import { execa } from "execa"; import { withStagedBuildDir } from "./build-staging"; +import { cliProcessOptions } from "./execution-deadline"; import { resolveOsmPbf } from "./osm-pbf"; import { repoPaths } from "./paths"; @@ -81,7 +82,11 @@ async function defaultRunner( args: string[], opts: { cwd?: string; stdio?: "inherit" }, ): Promise { - await execa(command, args, { cwd: opts.cwd, stdio: opts.stdio ?? "inherit" }); + await execa(command, args, { + ...cliProcessOptions(), + cwd: opts.cwd, + stdio: opts.stdio ?? "inherit", + }); } function dockerOsrmArgs(graphDir: string, image: string, command: string[]): string[] { diff --git a/packages/cli/src/lib/otp-graph.ts b/packages/cli/src/lib/otp-graph.ts index 379b219a1..f721eae93 100644 --- a/packages/cli/src/lib/otp-graph.ts +++ b/packages/cli/src/lib/otp-graph.ts @@ -2,6 +2,7 @@ import { copyFileSync, existsSync, linkSync, readdirSync, statSync } from "node: import { basename, extname, join, resolve } from "node:path"; import { execa } from "execa"; import { withStagedBuildDir } from "./build-staging"; +import { cliProcessOptions } from "./execution-deadline"; import { resolveOsmPbf } from "./osm-pbf"; import { repoPaths } from "./paths"; @@ -38,7 +39,11 @@ async function defaultRunner( args: string[], opts: { cwd?: string; stdio?: "inherit" }, ): Promise { - await execa(command, args, { cwd: opts.cwd, stdio: opts.stdio ?? "inherit" }); + await execa(command, args, { + ...cliProcessOptions(), + cwd: opts.cwd, + stdio: opts.stdio ?? "inherit", + }); } function linkOrCopy(source: string, target: string): void { diff --git a/packages/cli/src/lib/pelias-data.ts b/packages/cli/src/lib/pelias-data.ts index 9274c1873..a4c527f02 100644 --- a/packages/cli/src/lib/pelias-data.ts +++ b/packages/cli/src/lib/pelias-data.ts @@ -10,6 +10,7 @@ import { import { join, resolve } from "node:path"; import { execa } from "execa"; import { withStagedBuildDir } from "./build-staging"; +import { cliProcessOptions } from "./execution-deadline"; import { resolveOsmPbf } from "./osm-pbf"; import { repoPaths } from "./paths"; @@ -59,7 +60,11 @@ async function defaultRunner( args: string[], opts: { cwd?: string; stdio?: "inherit" | "pipe" }, ): Promise { - await execa(command, args, { cwd: opts.cwd, stdio: opts.stdio ?? "inherit" }); + await execa(command, args, { + ...cliProcessOptions(), + cwd: opts.cwd, + stdio: opts.stdio ?? "inherit", + }); } function linkOrCopy(source: string, target: string): void { diff --git a/packages/cli/src/lib/release.ts b/packages/cli/src/lib/release.ts index fdb4c2454..95ca4964b 100644 --- a/packages/cli/src/lib/release.ts +++ b/packages/cli/src/lib/release.ts @@ -16,6 +16,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { services as coreServices } from "@openmapx/core/server"; import { execa } from "execa"; +import { cliProcessOptions } from "./execution-deadline"; import { repoPaths } from "./paths"; const { @@ -34,7 +35,7 @@ export type ReleaseDockerRunner = ( ) => Promise<{ stdout: string; exitCode: number; stderr: string }>; async function defaultDocker(args: string[]) { - const result = await execa("docker", args, { reject: false }); + const result = await execa("docker", args, { ...cliProcessOptions(), reject: false }); return { stdout: result.stdout ?? "", stderr: result.stderr ?? "", diff --git a/packages/cli/src/lib/service-builds.ts b/packages/cli/src/lib/service-builds.ts index 482cb76ef..e7b11a238 100644 --- a/packages/cli/src/lib/service-builds.ts +++ b/packages/cli/src/lib/service-builds.ts @@ -9,6 +9,7 @@ import { } from "./build-artifacts"; import { runningComposeServices } from "./docker"; import { resolveBuildRegion } from "./env-defaults"; +import { assertCliExecutionAvailable } from "./execution-deadline"; import { buildMotisData, importMotisData } from "./motis-data"; import { buildOsrmGraph } from "./osrm-graph"; import { buildOtpGraph } from "./otp-graph"; @@ -210,7 +211,9 @@ async function recordBuild( region: string | undefined, artifact: ServiceBuildArtifact, ): Promise { + assertCliExecutionAvailable(); const revision = await currentRevision(rootDir); + assertCliExecutionAvailable(); writeBuildRecord( { schemaVersion: 1, @@ -363,6 +366,7 @@ export async function buildServices( } for (const item of plan) { + assertCliExecutionAvailable(); log.dim(`Manifest command → ${item.buildCommand}`); try { const resolvedRegion = resolveBuildRegion(item.id, opts.region); diff --git a/packages/cli/src/lib/tile-mbtiles.ts b/packages/cli/src/lib/tile-mbtiles.ts index a20ffb830..e9b1801ae 100644 --- a/packages/cli/src/lib/tile-mbtiles.ts +++ b/packages/cli/src/lib/tile-mbtiles.ts @@ -2,6 +2,7 @@ import { existsSync, mkdirSync, rmSync, statSync } from "node:fs"; import { basename, join, resolve } from "node:path"; import { execa } from "execa"; import { withStagedBuildDir } from "./build-staging"; +import { cliProcessOptions } from "./execution-deadline"; import { resolveOsmPbf } from "./osm-pbf"; import { repoPaths, resolveInvocationPath } from "./paths"; @@ -44,7 +45,11 @@ async function defaultRunner( args: string[], opts: { cwd?: string; stdio?: "inherit" }, ): Promise { - await execa(command, args, { cwd: opts.cwd, stdio: opts.stdio ?? "inherit" }); + await execa(command, args, { + ...cliProcessOptions(), + cwd: opts.cwd, + stdio: opts.stdio ?? "inherit", + }); } function isPlanet(sourcePbf: string): boolean { diff --git a/packages/cli/src/lib/transitous-api-keys.ts b/packages/cli/src/lib/transitous-api-keys.ts index 0914a66ee..ba44cf544 100644 --- a/packages/cli/src/lib/transitous-api-keys.ts +++ b/packages/cli/src/lib/transitous-api-keys.ts @@ -8,6 +8,7 @@ import { type TransitousFeedFile, } from "@openmapx/transitous-core"; import { execa } from "execa"; +import { cliProcessOptions } from "./execution-deadline"; import { repoPaths } from "./paths"; export type { CommandRunner }; @@ -42,7 +43,11 @@ async function defaultRunner( args: string[], opts: { cwd?: string; stdio?: "inherit" | "pipe" }, ): Promise { - await execa(command, args, { cwd: opts.cwd, stdio: opts.stdio ?? "pipe" }); + await execa(command, args, { + ...cliProcessOptions(), + cwd: opts.cwd, + stdio: opts.stdio ?? "pipe", + }); } /** Defense-in-depth: refuse to write outside the monorepo root. */ diff --git a/packages/core/src/ops/client.test.ts b/packages/core/src/ops/client.test.ts index 71189dbc8..bbfed494f 100644 --- a/packages/core/src/ops/client.test.ts +++ b/packages/core/src/ops/client.test.ts @@ -19,6 +19,28 @@ function tokenFile(contents = Buffer.alloc(32, 7).toString("base64url")): string return path; } +// Existing operation tests use a negotiated v2 peer; negotiation itself is tested below. +class ProtocolTestClient extends OpsClient { + constructor(options: ConstructorParameters[0]) { + const transport = options.fetch ?? globalThis.fetch; + super({ + ...options, + fetch: async (url, init) => { + if (new URL(String(url)).pathname === "/v1/protocol") + return new Response( + JSON.stringify({ + version: 1, + requestId: new Headers(init?.headers).get("x-ops-request-id"), + ok: true, + result: { versions: [1, 2] }, + }), + ); + return transport(url, init); + }, + }); + } +} + describe("ops client", () => { it("derives synchronous transport timeouts from the per-kind execution budget", () => { expect(opsRequestTimeoutMs("docker.status")).toBe(15_000); @@ -64,7 +86,7 @@ describe("ops client", () => { { status: 200 }, ); }); - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(token), fetch, @@ -86,7 +108,7 @@ describe("ops client", () => { ])( "accepts only canonical host-update errors (canonical: %s)", async (canonical, expectedCode) => { - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), fetch: async (_url, init) => @@ -120,11 +142,11 @@ describe("ops client", () => { it("bounds response bodies and never includes token, file path, URL, or remote body in errors", async () => { const token = Buffer.alloc(32, 11).toString("base64url"); const path = tokenFile(token); - const secretBody = `remote docker stderr ${token}`; - const client = new OpsClient({ + const secretBody = `remote docker stderr ${token}`.repeat(100); + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: path, - maxResponseBytes: 32, + maxResponseBytes: 256, fetch: async () => new Response(secretBody, { status: 500 }), }); @@ -145,7 +167,7 @@ describe("ops client", () => { }); }), ); - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), fetch, @@ -159,7 +181,7 @@ describe("ops client", () => { it("rejects non-private destinations without echoing the URL", () => { for (const baseUrl of ["http://evil.example/steal?token=x", "https://evil.example"]) { - expect(() => new OpsClient({ baseUrl, tokenFile: tokenFile() })).toThrow( + expect(() => new ProtocolTestClient({ baseUrl, tokenFile: tokenFile() })).toThrow( "Invalid ops agent destination", ); } @@ -198,7 +220,7 @@ describe("ops client", () => { ]) { expect( () => - new OpsClient({ + new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), ...options, @@ -207,13 +229,13 @@ describe("ops client", () => { } expect( () => - new OpsClient({ + new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), maxResponseBytes: OPS_MAX_HTTP_RESPONSE_BYTES, }), ).not.toThrow(); - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), randomBytes: () => Buffer.alloc(18, 4), @@ -262,7 +284,7 @@ describe("ops client", () => { { status: 202 }, ); }); - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), randomBytes: () => Buffer.alloc(18, 8), @@ -306,7 +328,7 @@ describe("ops client", () => { }), ); }); - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(token), fetch, @@ -352,7 +374,7 @@ describe("ops client", () => { }), ); }); - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), fetch, @@ -370,7 +392,7 @@ describe("ops client", () => { it("preserves caller AbortError for an in-flight event request", async () => { const controller = new AbortController(); - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), requestTimeoutMs: 500, @@ -392,7 +414,7 @@ describe("ops client", () => { it("carries caller AbortSignal through admission and retained status requests", async () => { const signals: AbortSignal[] = []; - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), requestTimeoutMs: 500, @@ -431,7 +453,7 @@ describe("ops client", () => { "timed_out", ] as const; let index = 0; - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), randomBytes: () => Buffer.alloc(18, 12), @@ -472,7 +494,7 @@ describe("ops client", () => { }); it("strictly parses typed retained status and bounded event pages", async () => { - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), randomBytes: () => Buffer.alloc(18, 9), @@ -520,7 +542,7 @@ describe("ops client", () => { }); it("rejects retained responses for another operation and incoherent event cursors", async () => { - const client = new OpsClient({ + const client = new ProtocolTestClient({ baseUrl: "http://ops-agent:4300", tokenFile: tokenFile(), randomBytes: () => Buffer.alloc(18, 10), @@ -556,3 +578,40 @@ describe("ops client", () => { ).rejects.toMatchObject({ errorClass: "runtime" }); }); }); + +it("fix1 refuses an older agent before any version2 mutation admission", async () => { + let posts = 0; + const client = new OpsClient({ + baseUrl: "http://ops-agent:4300", + tokenFile: tokenFile(), + fetch: async (_url, init) => { + if (init?.method !== "POST") + return new Response(JSON.stringify({ error: "Not Found" }), { status: 404 }); + posts++; + const body = JSON.parse(String(init.body)); + return new Response( + JSON.stringify({ + version: 1, + requestId: body.requestId, + ok: true, + result: { + execution: "async", + operationId: "job1_abcdefghijklmnop", + operationKey: body.operationKey, + kind: "service.build", + state: "running", + }, + }), + { status: 202 }, + ); + }, + }); + const error = await client + .execute( + { kind: "service.build", serviceId: "osrm" }, + { operationKey: "opk1_abcdefghijklmnop" }, + ) + .catch((error) => error); + expect(posts).toBe(0); + expect(error).toMatchObject({ name: "OpsProtocolUpgradeRequiredError" }); +}); diff --git a/packages/core/src/ops/client.ts b/packages/core/src/ops/client.ts index faef81583..c93ffe4d6 100644 --- a/packages/core/src/ops/client.ts +++ b/packages/core/src/ops/client.ts @@ -16,6 +16,7 @@ import { type OpsRequest, type OpsResponseEnvelope, type OpsSubmitResult, + parseOpsAdmission, parseOpsEventBatch, parseOpsJobStatus, parseOpsJobStatusForKind, @@ -62,6 +63,14 @@ export class OpsClientError extends Error { } } +export class OpsProtocolUpgradeRequiredError extends OpsClientError { + constructor() { + super("host_update_required", 409); + this.name = "OpsProtocolUpgradeRequiredError"; + this.message = "Update the ops-agent before submitting version2 operations"; + } +} + export async function readOpsTokenFile(path: string): Promise { let handle: Awaited> | undefined; try { @@ -206,6 +215,7 @@ export function opsRequestTimeoutMs(_kind: OpsOperationKind, configuredTimeoutMs } export class OpsClient { + private protocolReady?: Promise; private readonly baseUrl: string; private readonly tokenFile: string; private readonly fetchImpl: typeof globalThis.fetch; @@ -267,6 +277,7 @@ export class OpsClient { headers: { authorization: `Bearer ${token}`, "x-ops-request-id": requestId, + "x-ops-protocol-version": "2", ...(options.fingerprint ? { "x-ops-operation-fingerprint": options.fingerprint } : {}), ...(options.bodyFingerprint ? { "x-ops-body-fingerprint": options.bodyFingerprint } : {}), ...(options.requestedDeadlineAt === undefined @@ -392,7 +403,7 @@ export class OpsClient { await this.get(`/v1/operation-keys/${operationKey}`, { signal: options.signal, bodyFingerprint: opsOperationFingerprint(operation), - fingerprint: options.fingerprint, + fingerprint: options.fingerprint ?? opsOperationFingerprint(operation), requestedDeadlineAt: options.execution?.requestedDeadlineAt, }), {}, @@ -408,6 +419,37 @@ export class OpsClient { } } + private async requireProtocol2(signal?: AbortSignal): Promise { + this.protocolReady ??= (async () => { + let value: unknown; + try { + value = await this.get("/v1/protocol", { signal }); + } catch (error) { + if ( + isAbortError(error) || + (error instanceof OpsClientError && + ["authentication", "authorization", "timeout"].includes(error.errorClass)) + ) + throw error; + throw new OpsProtocolUpgradeRequiredError(); + } + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + !exactKeys(value as Record, ["versions"]) || + JSON.stringify((value as { versions?: unknown }).versions) !== "[1,2]" + ) + throw new OpsProtocolUpgradeRequiredError(); + })(); + try { + await this.protocolReady; + } catch (error) { + this.protocolReady = undefined; + throw error; + } + } + async execute( operation: Extract, options: { @@ -421,6 +463,7 @@ export class OpsClient { if (OPS_KIND_POLICIES[operation.kind].execution === "async" && !options.operationKey) { throw new Error("Async operation key is required"); } + await this.requireProtocol2(options.signal); const token = await readOpsTokenFile(this.tokenFile); const issued = this.now(); const requestId = `ops1_${Buffer.from(this.randomBytes(18)).toString("base64url")}`; @@ -485,55 +528,10 @@ export class OpsClient { value: parseOpsResult(operation.kind, submit.value), } as OpsSubmitResult; } - if ( - !(submit.version === 2 - ? exactKeys(submit, [ - "version", - "execution", - "operationId", - "operationKey", - "kind", - "state", - "admittedAt", - "deadlineAt", - "fingerprint", - ]) - : submit.version === 1 && - exactKeys(submit, [ - "version", - "execution", - "operationId", - "operationKey", - "kind", - "state", - ])) || - submit.execution !== "async" || - submit.kind !== operation.kind || - submit.operationKey !== operationKey || - typeof submit.operationId !== "string" || - !/^job1_[A-Za-z0-9_-]{16,64}$/.test(submit.operationId) || - !["queued", "running", "succeeded", "failed", "termination_pending", "timed_out"].includes( - String(submit.state), - ) - ) { + const admission = parseOpsAdmission(submit); + if (admission.kind !== operation.kind || admission.operationKey !== operationKey) throw new OpsClientError("runtime", response.status); - } - if ( - submit.version === 2 || - submit.admittedAt !== undefined || - submit.deadlineAt !== undefined || - submit.fingerprint !== undefined - ) { - if ( - !Number.isSafeInteger(submit.admittedAt) || - !Number.isSafeInteger(submit.deadlineAt) || - (submit.deadlineAt as number) <= (submit.admittedAt as number) || - typeof submit.fingerprint !== "string" || - !/^[a-f0-9]{64}$/.test(submit.fingerprint) - ) - throw new OpsClientError("runtime"); - } - return submit as unknown as OpsSubmitResult; + return admission as OpsSubmitResult; } catch (error) { if (error instanceof OpsClientError) throw error; if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError"); diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index dc82977cf..9869b9643 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -718,3 +718,26 @@ describe("ops contract", () => { ).toThrow(); }); }); + +it("fix1 reads only the actual historical or exact v2 admission projection", async () => { + const { parseOpsAdmission } = await import("./contract"); + const historical = { + execution: "async", + operationId: "job1_0123456789abcdef", + operationKey: "opk1_0123456789abcdef", + kind: "service.build", + state: "queued", + }; + expect(parseOpsAdmission(historical)).toEqual(historical); + for (const extra of [{ version: 1 }, { deadlineAt: 10 }, { parent: {} }, { unexpected: true }]) + expect(() => parseOpsAdmission({ ...historical, ...extra })).toThrow(); + expect( + parseOpsAdmission({ + ...historical, + version: 2, + admittedAt: 1, + deadlineAt: 2, + fingerprint: "a".repeat(64), + }), + ).toMatchObject({ version: 2 }); +}); diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index a9ec5ee02..42004da47 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -485,17 +485,22 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ ]); export type OpsOperation = z.infer; -const opsRequestSchema = z.strictObject({ - version: z.union([z.literal(1), z.literal(OPS_PROTOCOL_VERSION)]), - requestId: requestIdSchema, - operationKey: operationKeySchema, - issuedAt: timestampSchema, - expiresAt: timestampSchema, - operation: opsOperationSchema, - execution: z - .strictObject({ requestedDeadlineAt: z.number().int().nonnegative().safe() }) - .optional(), -}); +const opsRequestSchema = z + .strictObject({ + version: z.union([z.literal(1), z.literal(OPS_PROTOCOL_VERSION)]), + requestId: requestIdSchema, + operationKey: operationKeySchema, + issuedAt: timestampSchema, + expiresAt: timestampSchema, + operation: opsOperationSchema, + execution: z + .strictObject({ requestedDeadlineAt: z.number().int().nonnegative().safe() }) + .optional(), + }) + .refine( + (request) => request.version === 2 || request.execution === undefined, + "Version1 cannot carry execution authority", + ); export type OpsRequest = z.infer; export function authorizeOpsOperation(role: OpsRole, kind: OpsOperationKind): boolean { @@ -1227,7 +1232,9 @@ export function parseOpsJobStatus(input: unknown): OpsJobStatus { status.version === 1 && (status.admittedAt !== undefined || status.deadlineAt !== undefined || - status.fingerprint !== undefined) + status.fingerprint !== undefined || + status.diagnostics !== undefined || + status.failureResult !== undefined) ) throw new Error("Invalid compatibility identity"); if ( @@ -1347,6 +1354,25 @@ export interface OpsAdmission { kind: OpsOperationKind; state: OpsJobState; } +const legacyAdmission = z.strictObject({ + execution: z.literal("async"), + operationId: operationIdSchema, + operationKey: operationKeySchema, + kind: operationKindSchema, + state: jobState, +}); +const executionAdmission = legacyAdmission + .extend({ + version: z.literal(2), + admittedAt: z.number().int().nonnegative().safe(), + deadlineAt: z.number().int().nonnegative().safe(), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + }) + .refine((value) => value.deadlineAt > value.admittedAt); +/** The historical admission had no inner version. Both forms remain exact, disjoint schemas. */ +export function parseOpsAdmission(input: unknown): OpsAdmission { + return z.union([legacyAdmission, executionAdmission]).parse(input); +} export interface OpsSyncResult { execution: "sync"; kind: K; From 4de1551c3630e40798dcf70d50c465556832ae1b Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 21:29:36 +0200 Subject: [PATCH 005/135] fix(ops): retain credential continuations across frames --- apps/ops-agent/src/process-output.test.ts | 101 +++++++++++++++++++++ apps/ops-agent/src/process-output.ts | 106 +++++++++++++++++----- apps/ops-agent/src/server.test.ts | 18 +++- packages/core/src/scrub-secrets.ts | 19 ++-- 4 files changed, 213 insertions(+), 31 deletions(-) diff --git a/apps/ops-agent/src/process-output.test.ts b/apps/ops-agent/src/process-output.test.ts index 5661a2312..5f6a20ce7 100644 --- a/apps/ops-agent/src/process-output.test.ts +++ b/apps/ops-agent/src/process-output.test.ts @@ -157,3 +157,104 @@ it("fix1 withholds a quoted assignment whose delimiter follows a real frame boun expect(all).not.toContain("synthetic-value-continuation"); expect(seen).toContain("last-diagnostic"); }); + +describe("fix2 credential continuation parity", () => { + it.each(["accesstoken", "accessToken", "ACCESSTOKEN"])( + "withholds quoted %s continuations with every real line ending and byte split", + (key) => { + const observed: unknown[] = []; + for (const ending of ["\n", "\r\n", "\r"]) { + for (const splitBytes of [false, true]) { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_stream, line) => seen.push(line), + redact: (x) => x, + }); + for (const frame of [ + `${key}="synthetic-access-first${ending}`, + `synthetic-access-second"${ending}`, + `last-diagnostic${ending}`, + ]) { + const bytes = Buffer.from(frame); + if (splitBytes) for (const byte of bytes) output.write("stdout", Uint8Array.of(byte)); + else output.write("stdout", bytes); + } + const diagnostics = output.end(); + expect(seen).toContain("last-diagnostic"); + expect(diagnostics.tail.at(-1)?.message).toBe("last-diagnostic"); + observed.push({ ending, splitBytes, seen, diagnostics }); + } + } + const all = JSON.stringify(observed); + expect(all).not.toContain("synthetic-access-first"); + expect(all).not.toContain("synthetic-access-second"); + }, + ); + + it.each(["Bearer", "Basic", "Token"])( + "retains Authorization %s awaiting its value across every real line ending and byte split", + (scheme) => { + const observed: unknown[] = []; + for (const ending of ["\n", "\r\n", "\r"]) { + for (const splitBytes of [false, true]) { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_stream, line) => seen.push(line), + redact: (x) => x, + }); + for (const frame of [ + `Authorization: ${scheme}${ending}`, + `synthetic-authorization-value${ending}`, + `last-diagnostic${ending}`, + ]) { + const bytes = Buffer.from(frame); + if (splitBytes) for (const byte of bytes) output.write("stdout", Uint8Array.of(byte)); + else output.write("stdout", bytes); + } + const diagnostics = output.end(); + expect(seen).toContain("last-diagnostic"); + expect(diagnostics.tail.at(-1)?.message).toBe("last-diagnostic"); + observed.push({ ending, splitBytes, seen, diagnostics }); + } + } + expect(JSON.stringify(observed)).not.toContain("synthetic-authorization-value"); + }, + ); +}); + +it("fix2 preserves inline authorization CR continuation protection", () => { + for (const scheme of ["Bearer", "Basic", "Token"]) { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_stream, line) => seen.push(line), + redact: (x) => x, + }); + for (const frame of [ + `Authorization: ${scheme} synthetic-inline-value\r`, + "synthetic-inline-continuation\r", + "last-diagnostic\n", + ]) + output.write("stdout", Buffer.from(frame)); + const diagnostics = output.end(); + expect(JSON.stringify({ seen, diagnostics })).not.toContain("synthetic-inline-"); + expect(seen).toContain("last-diagnostic"); + } +}); + +it("fix2 still tracks a quoted assignment used as an authorization value", () => { + for (const prefix of ["Authorization: Bearer ", "Authorization: Bearer\n"]) { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_stream, line) => seen.push(line), + redact: (x) => x, + }); + for (const frame of [ + `${prefix}token="synthetic-nested-first\n`, + 'synthetic-nested-second"\n', + "last-diagnostic\n", + ]) + output.write("stdout", Buffer.from(frame)); + expect(JSON.stringify({ seen, diagnostics: output.end() })).not.toContain("synthetic-nested-"); + expect(seen).toContain("last-diagnostic"); + } +}); diff --git a/apps/ops-agent/src/process-output.ts b/apps/ops-agent/src/process-output.ts index fe8c947a8..aee0b2599 100644 --- a/apps/ops-agent/src/process-output.ts +++ b/apps/ops-agent/src/process-output.ts @@ -1,7 +1,11 @@ import { StringDecoder } from "node:string_decoder"; import { stripVTControlCharacters } from "node:util"; import { PROCESS_OUTPUT_TRUNCATED, type ProcessDiagnostics } from "@openmapx/core/ops"; -import { scrubSecrets } from "@openmapx/core/scrub-secrets"; +import { + DIAGNOSTIC_AUTHORIZATION_SCHEMES, + DIAGNOSTIC_CREDENTIAL_KEY_SOURCE, + scrubSecrets, +} from "@openmapx/core/scrub-secrets"; type Stream = "stdout" | "stderr"; interface OutputOptions { @@ -9,45 +13,87 @@ interface OutputOptions { redact: (message: string) => string; maxTailBytes?: number; } -const QUOTED_CREDENTIAL = - /\b((?:authorization|api[-_]?key|access[-_]?token|token|secret|password|passwd|auth|key)\s*[:=]\s*)(["']).*$/gi; -const CREDENTIAL_KEYS = new Set([ - "authorization", - "apikey", - "api-key", - "api_key", - "access-token", - "access_token", - "token", - "secret", - "password", - "passwd", - "auth", - "key", -]); +const CREDENTIAL_KEY_SOURCE = `(?:authorization|${DIAGNOSTIC_CREDENTIAL_KEY_SOURCE})`; +const CREDENTIAL_KEY = new RegExp(`^${CREDENTIAL_KEY_SOURCE}$`, "i"); +const QUOTED_CREDENTIAL = new RegExp( + String.raw`\b(${CREDENTIAL_KEY_SOURCE}\s*[:=]\s*)(["']).*$`, + "gi", +); +const AUTHORIZATION_SCHEMES = DIAGNOSTIC_AUTHORIZATION_SCHEMES.map((scheme) => + scheme.toLowerCase(), +); interface CredentialScan { - phase: "word" | "key" | "value" | "bare" | "quote"; + phase: + | "word" + | "key" + | "value" + | "scheme" + | "schemeValue" + | "framedSchemeValue" + | "bare" + | "quote"; + authorization: boolean; + inlineAuthorization: boolean; word: string; quote: string; escaped: boolean; } function scanCredential(scan: CredentialScan, character: string): void { + if (/\s/.test(character)) scan.inlineAuthorization = false; if (scan.phase === "quote") { if (scan.escaped) scan.escaped = false; else if (character === "\\") scan.escaped = true; else if (character === scan.quote) scan.phase = "word"; return; } + if (scan.phase === "scheme") { + if (/\s/.test(character)) { + // A complete scheme followed by a real frame boundary still awaits its credential. + scan.phase = AUTHORIZATION_SCHEMES.includes(scan.word) ? "schemeValue" : "word"; + scan.word = ""; + } else { + const candidate = scan.word + character.toLowerCase(); + if (AUTHORIZATION_SCHEMES.some((scheme) => scheme.startsWith(candidate))) { + scan.word = candidate; // At most the longest known scheme; never accumulates a raw value. + return; + } + scan.word = ""; + scan.phase = "bare"; + } + } if (scan.phase === "bare") { if (/[\s'"&,;)\]}]/.test(character)) scan.phase = "word"; else return; } - if (scan.phase === "value") { - if (/\s/.test(character)) return; + if ( + scan.phase === "value" || + scan.phase === "schemeValue" || + scan.phase === "framedSchemeValue" + ) { + if (/\s/.test(character)) { + if (scan.phase === "schemeValue" && (character === "\r" || character === "\n")) + scan.phase = "framedSchemeValue"; + return; + } + const schemeValue = scan.phase === "schemeValue" || scan.phase === "framedSchemeValue"; + const inlineAuthorization = scan.phase === "schemeValue"; + const authorization = scan.authorization; + scan.authorization = false; if (character === '"' || character === "'") { scan.phase = "quote"; scan.quote = character; scan.escaped = false; + } else if (schemeValue) { + // Keep recognizing later assignments rather than swallowing them as an opaque token. + scan.phase = "word"; + scan.word = /[\w-]/.test(character) ? character.toLowerCase() : ""; + scan.inlineAuthorization = inlineAuthorization; + } else if ( + authorization && + AUTHORIZATION_SCHEMES.some((scheme) => scheme.startsWith(character.toLowerCase())) + ) { + scan.phase = "scheme"; + scan.word = character.toLowerCase(); } else scan.phase = "bare"; return; } @@ -55,12 +101,16 @@ function scanCredential(scan: CredentialScan, character: string): void { if (/\s/.test(character)) return; scan.phase = character === ":" || character === "=" ? "value" : "word"; if (scan.phase === "value") return; + scan.authorization = false; } if (/[\w-]/.test(character)) { scan.word = (scan.word + character.toLowerCase()).slice(-32); return; } - const key = CREDENTIAL_KEYS.has(scan.word.replace(/^--/, "")); + const name = scan.word.replace(/^--/, ""); + const key = CREDENTIAL_KEY.test(name); + scan.authorization = key && name === "authorization"; + if (key) scan.inlineAuthorization = false; scan.word = ""; if (key && (character === ":" || character === "=")) scan.phase = "value"; else if (key && /\s/.test(character)) scan.phase = "key"; @@ -142,7 +192,14 @@ export function createBoundedProcessOutput(options: OutputOptions): { protected: false, cr: false, control: "", - scan: { phase: "word", word: "", quote: "", escaped: false } as CredentialScan, + scan: { + phase: "word", + authorization: false, + inlineAuthorization: false, + word: "", + quote: "", + escaped: false, + } as CredentialScan, }); const streams = { stdout: streamState(), stderr: streamState() }; const tail: ProcessDiagnostics["tail"] = []; @@ -204,7 +261,8 @@ export function createBoundedProcessOutput(options: OutputOptions): { for (const decoded of text) { const character = controlFree(state, decoded); if (!character) continue; - const progressSecret = character === "\r" && state.scan.phase === "bare"; + const progressSecret = + character === "\r" && (state.scan.phase === "bare" || state.scan.inlineAuthorization); scanCredential(state.scan, character); if (character === "\n" || character === "\r") { if (!(character === "\n" && state.cr)) { @@ -216,6 +274,8 @@ export function createBoundedProcessOutput(options: OutputOptions): { state.protected = state.scan.phase === "quote" || state.scan.phase === "value" || + state.scan.phase === "schemeValue" || + state.scan.phase === "framedSchemeValue" || state.scan.phase === "key" || progressSecret; state.dropping = false; diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index 70b7a26c4..c35b90691 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -732,7 +732,7 @@ it("observer abort leaves the admitted backend running until its real completion }); it.each([0, 1])( - "fix1 retains safe truncation and final diagnostics through the real CLI/server boundary (exit %s)", + "fix2 retains safe credential continuations, truncation and final diagnostics through the real CLI/server boundary (exit %s)", async (exitCode) => { const { mkdtempSync, writeFileSync, rmSync } = await import("node:fs"); const { tmpdir } = await import("node:os"); @@ -749,13 +749,24 @@ it.each([0, 1])( 'synthetic-final"\n', 'pass\x1b[31mword="ansi-first\n', 'ansi-continuation"\n', + ...["accesstoken", "accessToken", "ACCESSTOKEN"].flatMap((key, index) => { + const ending = ["\n", "\r\n", "\r"][index]; + return [ + `${key}="synthetic-access-first-${index}${ending}`, + `synthetic-access-second-${index}"${ending}`, + ]; + }), + ...["Bearer", "Basic", "Token"].flatMap((scheme, index) => { + const ending = ["\n", "\r\n", "\r"][index]; + return [`Authorization: ${scheme}${ending}`, `synthetic-auth-value-${index}${ending}`]; + }), `${"x".repeat(4096 - Buffer.byteLength(suffix))}${suffix}@host.invalid/path\n`, `${"x".repeat(6000)}\n`, "last-diagnostic\n", ]; writeFileSync( join(root, "cli.mjs"), - `for(const frame of ${JSON.stringify(frames)})process.stdout.write(frame);process.exitCode=${exitCode};`, + `for(const frame of ${JSON.stringify(frames)})for(const byte of Buffer.from(frame))process.stdout.write(Buffer.from([byte]));process.exitCode=${exitCode};`, ); const runtime = createUnavailableRuntime(); createAdministrativeRuntime(runtime, { @@ -828,6 +839,9 @@ it.each([0, 1])( "ansi-first", "ansi-continuation", "synthetic-user", + "synthetic-access-first-", + "synthetic-access-second-", + "synthetic-auth-value-", ]) expect(all).not.toContain(secret); if (exitCode !== 0) diff --git a/packages/core/src/scrub-secrets.ts b/packages/core/src/scrub-secrets.ts index 1ff0815f8..4062b19c8 100644 --- a/packages/core/src/scrub-secrets.ts +++ b/packages/core/src/scrub-secrets.ts @@ -17,8 +17,18 @@ const PATH_SECRET_SEGMENTS = new Set([ "auth", "operator-feed", ]); -const BARE_CREDENTIAL_PATTERN = - /((? Date: Fri, 23 Oct 2026 21:40:15 +0200 Subject: [PATCH 006/135] fix(ops): protect CR-split authorization schemes --- apps/ops-agent/src/process-output.test.ts | 52 +++++++++++++++++++++++ apps/ops-agent/src/process-output.ts | 6 ++- apps/ops-agent/src/server.test.ts | 7 ++- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/apps/ops-agent/src/process-output.test.ts b/apps/ops-agent/src/process-output.test.ts index 5f6a20ce7..820e6cec1 100644 --- a/apps/ops-agent/src/process-output.test.ts +++ b/apps/ops-agent/src/process-output.test.ts @@ -258,3 +258,55 @@ it("fix2 still tracks a quoted assignment used as an authorization value", () => expect(seen).toContain("last-diagnostic"); } }); + +it.each(["Bearer", "Basic", "Token"])( + "fix3 protects every nonempty proper %s prefix at CR and CRLF and releases following diagnostics", + (scheme) => { + const failures: string[] = []; + const secret = "synthetic-review-r3-continuation"; + for (let split = 1; split < scheme.length; split++) { + for (const ending of ["\r", "\r\n"]) { + for (const splitBytes of [false, true]) { + for (const stream of ["stdout", "stderr"] as const) { + const otherStream = stream === "stdout" ? "stderr" : "stdout"; + const seen: { stream: "stdout" | "stderr"; message: string }[] = []; + const output = createBoundedProcessOutput({ + onLine: (stream, message) => seen.push({ stream, message }), + redact: (x) => x, + }); + const write = (target: "stdout" | "stderr", frame: string) => { + const bytes = Buffer.from(frame); + if (splitBytes) for (const byte of bytes) output.write(target, Uint8Array.of(byte)); + else output.write(target, bytes); + }; + write(stream, `Authorization: ${scheme.slice(0, split)}${ending}`); + write(otherStream, "independent-diagnostic\n"); + write(stream, `${scheme.slice(split)} ${secret}\n`); + write(stream, "last-diagnostic\n"); + const diagnostics = output.end(); + const callbackWithheld = !seen.some((entry) => entry.message.includes(secret)); + const tailWithheld = !diagnostics.tail.some((entry) => entry.message.includes(secret)); + const finalEntry = { stream, message: "last-diagnostic" }; + const independentEntry: typeof finalEntry = { + stream: otherStream, + message: "independent-diagnostic", + }; + const includes = (entries: typeof seen, expected: typeof finalEntry) => + entries.some( + (entry) => entry.stream === expected.stream && entry.message === expected.message, + ); + const finalVisible = + includes(seen, finalEntry) && includes(diagnostics.tail, finalEntry); + const independentVisible = + includes(seen, independentEntry) && includes(diagnostics.tail, independentEntry); + if (!callbackWithheld || !tailWithheld || !finalVisible || !independentVisible) + failures.push( + `${scheme.slice(0, split)} + ${scheme.slice(split)}, ${JSON.stringify(ending)}, bytes=${splitBytes}, ${stream}: callbackWithheld=${callbackWithheld}, tailWithheld=${tailWithheld}, finalVisible=${finalVisible}, independentVisible=${independentVisible}`, + ); + } + } + } + } + expect(failures).toEqual([]); + }, +); diff --git a/apps/ops-agent/src/process-output.ts b/apps/ops-agent/src/process-output.ts index aee0b2599..d9a7bd4ef 100644 --- a/apps/ops-agent/src/process-output.ts +++ b/apps/ops-agent/src/process-output.ts @@ -261,8 +261,12 @@ export function createBoundedProcessOutput(options: OutputOptions): { for (const decoded of text) { const character = controlFree(state, decoded); if (!character) continue; + // Preserve CR carry before scheme scanning can discard an incomplete prefix. const progressSecret = - character === "\r" && (state.scan.phase === "bare" || state.scan.inlineAuthorization); + character === "\r" && + (state.scan.phase === "bare" || + state.scan.phase === "scheme" || + state.scan.inlineAuthorization); scanCredential(state.scan, character); if (character === "\n" || character === "\r") { if (!(character === "\n" && state.cr)) { diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index c35b90691..94d00ef10 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -732,7 +732,7 @@ it("observer abort leaves the admitted backend running until its real completion }); it.each([0, 1])( - "fix2 retains safe credential continuations, truncation and final diagnostics through the real CLI/server boundary (exit %s)", + "fix3 retains safe credential continuations, truncation and final diagnostics through the real CLI/server boundary (exit %s)", async (exitCode) => { const { mkdtempSync, writeFileSync, rmSync } = await import("node:fs"); const { tmpdir } = await import("node:os"); @@ -760,6 +760,10 @@ it.each([0, 1])( const ending = ["\n", "\r\n", "\r"][index]; return [`Authorization: ${scheme}${ending}`, `synthetic-auth-value-${index}${ending}`]; }), + "Authorization: Bea\r", + "rer synthetic-review-r3-continuation\n", + "Authorization: To\r\n", + "ken synthetic-review-r3-crlf-continuation\n", `${"x".repeat(4096 - Buffer.byteLength(suffix))}${suffix}@host.invalid/path\n`, `${"x".repeat(6000)}\n`, "last-diagnostic\n", @@ -842,6 +846,7 @@ it.each([0, 1])( "synthetic-access-first-", "synthetic-access-second-", "synthetic-auth-value-", + "synthetic-review-r3-", ]) expect(all).not.toContain(secret); if (exitCode !== 0) From 47549f6448d1b0e24f7ecb7fe0e4476d86a22352 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 21:50:41 +0200 Subject: [PATCH 007/135] fix(ops): retain authorization prefixes across progress frames --- apps/ops-agent/src/process-output.test.ts | 130 ++++++++++++++++++++++ apps/ops-agent/src/process-output.ts | 26 +++-- apps/ops-agent/src/server.test.ts | 13 ++- 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/apps/ops-agent/src/process-output.test.ts b/apps/ops-agent/src/process-output.test.ts index 820e6cec1..b840bf399 100644 --- a/apps/ops-agent/src/process-output.test.ts +++ b/apps/ops-agent/src/process-output.test.ts @@ -310,3 +310,133 @@ it.each(["Bearer", "Basic", "Token"])( expect(failures).toEqual([]); }, ); + +describe("fix4 bounded multi-CR authorization recognition", () => { + it("withholds the credential after a second CR while Bearer is still resuming", () => { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_stream, message) => seen.push(message), + redact: (x) => x, + }); + for (const frame of [ + "Authorization: B\r", + "ea\r", + "rer synthetic-review-r4-continuation\n", + "last-diagnostic\n", + ]) + output.write("stdout", Buffer.from(frame)); + const diagnostics = output.end(); + expect(JSON.stringify({ seen, diagnostics })).not.toContain("synthetic-review-r4-continuation"); + expect(seen).toContain("last-diagnostic"); + expect(diagnostics.tail).toContainEqual({ stream: "stdout", message: "last-diagnostic" }); + }); + + it.each(["Bearer", "Basic", "Token"])( + "withholds every nonempty %s partition across CR/CRLF and releases subsequent diagnostics", + (scheme) => { + const failures: string[] = []; + let failureCount = 0; + // Each bit cuts one character boundary; include every partition with at least one cut. + for (let cuts = 1; cuts < 2 ** (scheme.length - 1); cuts++) { + const parts: string[] = []; + let start = 0; + for (let index = 1; index < scheme.length; index++) { + if (cuts & (1 << (index - 1))) { + parts.push(scheme.slice(start, index)); + start = index; + } + } + parts.push(scheme.slice(start)); + for (const endings of [["\r"], ["\r\n"], ["\r", "\r\n"]]) { + for (const valueEnding of ["\n", "\r", "\r\n"]) { + for (const pendingValue of [false, true]) { + for (const splitBytes of [false, true]) { + for (const stream of ["stdout", "stderr"] as const) { + const otherStream = stream === "stdout" ? "stderr" : "stdout"; + const seen: { stream: "stdout" | "stderr"; message: string }[] = []; + const projected: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (stream, message) => seen.push({ stream, message }), + redact: (message) => { + projected.push(message); + return message; + }, + }); + const write = (target: "stdout" | "stderr", frame: string) => { + const bytes = Buffer.from(frame); + if (splitBytes) + for (const byte of bytes) output.write(target, Uint8Array.of(byte)); + else output.write(target, bytes); + }; + for (let index = 0; index < parts.length; index++) { + const prefix = index === 0 ? "Authorization: " : ""; + const ending = endings[index % endings.length]; + if (index < parts.length - 1) { + write(stream, `${prefix}${parts[index]}${ending}`); + write(otherStream, "independent-diagnostic\n"); + } else { + const separator = pendingValue ? ending : " "; + write( + stream, + `${prefix}${parts[index]}${separator}synthetic-review-r4-value${valueEnding}`, + ); + } + } + write(stream, "last-diagnostic\n"); + const diagnostics = output.end(); + const withheld = !JSON.stringify({ seen, diagnostics, projected }).includes( + "synthetic-review-r4-value", + ); + const includes = ( + entries: typeof seen, + target: "stdout" | "stderr", + message: string, + ) => + entries.some((entry) => entry.stream === target && entry.message === message); + const finalVisible = + includes(seen, stream, "last-diagnostic") && + includes(diagnostics.tail, stream, "last-diagnostic"); + const independentVisible = + includes(seen, otherStream, "independent-diagnostic") && + includes(diagnostics.tail, otherStream, "independent-diagnostic"); + if (!withheld || !finalVisible || !independentVisible) { + failureCount++; + if (failures.length < 10) + failures.push( + `${parts.join(" + ")}, ${JSON.stringify(endings)}, valueEnding=${JSON.stringify(valueEnding)}, pending=${pendingValue}, bytes=${splitBytes}, ${stream}: withheld=${withheld}, finalVisible=${finalVisible}, independentVisible=${independentVisible}`, + ); + } + } + } + } + } + } + } + expect({ failureCount, examples: failures }).toEqual({ failureCount: 0, examples: [] }); + }, + ); + + it.each(["\r", "\r\n"])( + "releases a rejected scheme prefix instead of latching over later diagnostics (%j)", + (ending) => { + const seen: string[] = []; + const output = createBoundedProcessOutput({ + onLine: (_stream, message) => seen.push(message), + redact: (x) => x, + }); + for (const frame of [ + `Authorization: B${ending}`, + `ea${ending}`, + "not-a-scheme\n", + "last-diagnostic\n", + "another-diagnostic\n", + ]) + output.write("stdout", Buffer.from(frame)); + const diagnostics = output.end(); + for (const message of ["last-diagnostic", "another-diagnostic"]) { + expect(seen).toContain(message); + expect(diagnostics.tail).toContainEqual({ stream: "stdout", message }); + } + }, + ); +}); diff --git a/apps/ops-agent/src/process-output.ts b/apps/ops-agent/src/process-output.ts index d9a7bd4ef..c8d5361c5 100644 --- a/apps/ops-agent/src/process-output.ts +++ b/apps/ops-agent/src/process-output.ts @@ -28,6 +28,7 @@ interface CredentialScan { | "key" | "value" | "scheme" + | "framedScheme" | "schemeValue" | "framedSchemeValue" | "bare" @@ -46,10 +47,20 @@ function scanCredential(scan: CredentialScan, character: string): void { else if (character === scan.quote) scan.phase = "word"; return; } - if (scan.phase === "scheme") { + if (scan.phase === "scheme" || scan.phase === "framedScheme") { if (/\s/.test(character)) { + const complete = AUTHORIZATION_SCHEMES.includes(scan.word); + if (character === "\r" && !complete) { + // Retain only the known prefix while it resumes across progress frames. + scan.phase = "framedScheme"; + return; + } // A complete scheme followed by a real frame boundary still awaits its credential. - scan.phase = AUTHORIZATION_SCHEMES.includes(scan.word) ? "schemeValue" : "word"; + scan.phase = complete + ? scan.phase === "framedScheme" + ? "framedSchemeValue" + : "schemeValue" + : "word"; scan.word = ""; } else { const candidate = scan.word + character.toLowerCase(); @@ -261,13 +272,11 @@ export function createBoundedProcessOutput(options: OutputOptions): { for (const decoded of text) { const character = controlFree(state, decoded); if (!character) continue; - // Preserve CR carry before scheme scanning can discard an incomplete prefix. + // Bare/inline values need CR carry captured before scanning the delimiter. const progressSecret = - character === "\r" && - (state.scan.phase === "bare" || - state.scan.phase === "scheme" || - state.scan.inlineAuthorization); - scanCredential(state.scan, character); + character === "\r" && (state.scan.phase === "bare" || state.scan.inlineAuthorization); + // Coalesce CRLF for recognition as well as emission, even across byte writes. + if (!(character === "\n" && state.cr)) scanCredential(state.scan, character); if (character === "\n" || character === "\r") { if (!(character === "\n" && state.cr)) { if (!state.dropping) emit(stream); @@ -278,6 +287,7 @@ export function createBoundedProcessOutput(options: OutputOptions): { state.protected = state.scan.phase === "quote" || state.scan.phase === "value" || + state.scan.phase === "framedScheme" || state.scan.phase === "schemeValue" || state.scan.phase === "framedSchemeValue" || state.scan.phase === "key" || diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index 94d00ef10..5f836967e 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -732,7 +732,7 @@ it("observer abort leaves the admitted backend running until its real completion }); it.each([0, 1])( - "fix3 retains safe credential continuations, truncation and final diagnostics through the real CLI/server boundary (exit %s)", + "fix4 retains safe credential continuations, truncation and final diagnostics through the real CLI/server boundary (exit %s)", async (exitCode) => { const { mkdtempSync, writeFileSync, rmSync } = await import("node:fs"); const { tmpdir } = await import("node:os"); @@ -764,6 +764,16 @@ it.each([0, 1])( "rer synthetic-review-r3-continuation\n", "Authorization: To\r\n", "ken synthetic-review-r3-crlf-continuation\n", + "Authorization: B\r", + "ea\r", + "rer synthetic-review-r4-continuation\n", + "Authorization: B\r\n", + "a\r\n", + "sic synthetic-review-r4-basic\r\n", + "Authorization: T\r", + "o\r\n", + "ken\r", + "synthetic-review-r4-token\r", `${"x".repeat(4096 - Buffer.byteLength(suffix))}${suffix}@host.invalid/path\n`, `${"x".repeat(6000)}\n`, "last-diagnostic\n", @@ -847,6 +857,7 @@ it.each([0, 1])( "synthetic-access-second-", "synthetic-auth-value-", "synthetic-review-r3-", + "synthetic-review-r4-", ]) expect(all).not.toContain(secret); if (exitCode !== 0) From 92a33e616511b0bbabc167a4235933ac097e9d2a Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 22:20:54 +0200 Subject: [PATCH 008/135] fix(admin): reconcile durable jobs without replaying effects --- .../routes/__tests__/admin-activity.test.ts | 69 ++ apps/api/src/server.ts | 6 + .../__tests__/admin-job-handlers.test.ts | 21 + .../services/__tests__/admin-job-ops.test.ts | 431 ++++++++++++- .../__tests__/admin-ops-authority.test.ts | 29 +- .../__tests__/job-reconciliation.test.ts | 39 ++ .../__tests__/job-runner-cancel.test.ts | 82 ++- .../__tests__/job-runner-initialize.test.ts | 26 +- .../__tests__/job-runner-outcomes.test.ts | 183 ++++++ .../src/services/__tests__/ops-client.test.ts | 6 +- .../__tests__/system-maintenance.test.ts | 6 +- .../trusted-config-operations.test.ts | 96 ++- apps/api/src/services/activity-jobs.ts | 2 + apps/api/src/services/admin-job-failure.ts | 24 + apps/api/src/services/admin-job-handlers.ts | 12 + apps/api/src/services/admin-job-ops.ts | 549 +++++++++++++--- apps/api/src/services/admin-job-parent.ts | 89 +++ apps/api/src/services/admin-job-state.ts | 66 ++ apps/api/src/services/admin-ops.ts | 21 +- apps/api/src/services/job-events.ts | 3 + apps/api/src/services/job-reconciliation.ts | 32 + apps/api/src/services/job-runner.ts | 589 ++++++++++-------- apps/api/src/services/ops-client.ts | 25 +- apps/api/src/services/system-maintenance.ts | 7 + .../src/services/trusted-config-operations.ts | 54 +- apps/ops-agent/src/journal.test.ts | 56 ++ .../components/admin/activity/JobDetail.tsx | 44 +- .../__tests__/ActivityConsumers.test.tsx | 32 + .../core/src/api/contracts/activity-admin.ts | 36 +- packages/core/src/ops/job-evidence.ts | 6 +- 30 files changed, 2236 insertions(+), 405 deletions(-) create mode 100644 apps/api/src/services/__tests__/job-reconciliation.test.ts create mode 100644 apps/api/src/services/__tests__/job-runner-outcomes.test.ts create mode 100644 apps/api/src/services/admin-job-failure.ts create mode 100644 apps/api/src/services/admin-job-parent.ts create mode 100644 apps/api/src/services/admin-job-state.ts create mode 100644 apps/api/src/services/job-reconciliation.ts diff --git a/apps/api/src/routes/__tests__/admin-activity.test.ts b/apps/api/src/routes/__tests__/admin-activity.test.ts index b3b5e49c0..7cc883106 100644 --- a/apps/api/src/routes/__tests__/admin-activity.test.ts +++ b/apps/api/src/routes/__tests__/admin-activity.test.ts @@ -300,3 +300,72 @@ describe("actual generated Activity SDK through Fastify serialization", () => { } }); }); + +it("serializes retained failure and observation evidence through the actual SDK and rejects internal ownership", async () => { + const { getAdminActivityJob } = await import("@openmapx/core/api-admin-sdk"); + const app = await createAdminTestApp(adminActivityRoute); + try { + const evidence = { + serviceBatch: { + outcome: "partial", + completedServiceIds: ["tileserver"], + failedServiceIds: ["valhalla"], + skippedServiceIds: [], + }, + diagnostics: { truncated: false, tail: [{ stream: "stderr", message: "final diagnostic" }] }, + observation: { + state: "unknown", + reason: "recovery_required", + nextCheckAt: "2026-10-23T20:00:00.000Z", + }, + }; + detail.mockResolvedValue({ ...activityDetail(), status: "cancel_pending", result: evidence }); + const response = await app.inject("/admin/jobs/job-1"); + expect(response.statusCode).toBe(200); + expect(response.json().result).toEqual(evidence); + // The generated JSON result decoder must preserve this same producer shape. + vi.stubGlobal( + "fetch", + async () => + new Response(response.body, { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + const client = createApiClient({ baseUrl: "http://api.test" }); + expect(await getAdminActivityJob("job-1", {}, { client })).toMatchObject({ + data: { result: evidence }, + }); + detail.mockResolvedValue({ + ...activityDetail(), + result: { _execution: { owner: "private", version: 1 } }, + }); + const rejected = await app.inject("/admin/jobs/job-1"); + expect(rejected.statusCode).toBe(500); + expect(rejected.body).not.toContain("private"); + } finally { + await app.close(); + } +}); + +it.each([401, 403])( + "denies detail/cancel before any domain call for guard status %s", + async (statusCode) => { + const app = await createAdminTestApp(adminActivityRoute); + try { + for (const [method, url] of [ + ["GET", "/admin/jobs/job-1"], + ["POST", "/admin/jobs/job-1/cancel"], + ] as const) { + mocks.requireAdmin.mockRejectedValueOnce( + Object.assign(new Error("Denied"), { statusCode }), + ); + expect((await app.inject({ method, url })).statusCode).toBe(statusCode); + } + expect(detail).not.toHaveBeenCalled(); + expect(cancel).not.toHaveBeenCalled(); + } finally { + await app.close(); + } + }, +); diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 445afb5b6..66f72aa50 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -110,6 +110,10 @@ import { withExtensionChangeLock, } from "./services/extension-recovery"; import { pruneOldRecords } from "./services/health-history"; +import { + startAdminJobReconciliation, + stopAdminJobReconciliation, +} from "./services/job-reconciliation"; import { jobRunner } from "./services/job-runner"; import { readOfflinePackagePrincipalKeyFile } from "./services/offline-package-principal.js"; import { getServiceRegistry, initServiceRegistry } from "./services/service-registry"; @@ -724,6 +728,7 @@ await startDataUsePolicyRefresh(); const port = envInt("PORT", 3000); const host = envString("HOST", "0.0.0.0"); +server.addHook("onClose", stopAdminJobReconciliation); try { await server.listen({ port, host }); @@ -736,6 +741,7 @@ try { // genuinely listening, migrations succeeded, and Docker confirms that this // process is running the exact image pulled by the update job. await jobRunner.initialize(); +startAdminJobReconciliation(); // Graceful shutdown for (const signal of ["SIGINT", "SIGTERM"] as const) { diff --git a/apps/api/src/services/__tests__/admin-job-handlers.test.ts b/apps/api/src/services/__tests__/admin-job-handlers.test.ts index 89c394f83..87e572ce5 100644 --- a/apps/api/src/services/__tests__/admin-job-handlers.test.ts +++ b/apps/api/src/services/__tests__/admin-job-handlers.test.ts @@ -344,6 +344,27 @@ describe("admin job handlers", () => { }); describe("bulk service operations", () => { + it("does not continue a batch after unknown admission even with continueOnError", async () => { + const { OpsObservationPending } = await import("../ops-client"); + const pending = new OpsObservationPending( + { kind: "service.build", operationKey: "opk1_abcdefghijklmnop" }, + "transport", + ); + runAdminOperationMock.mockClear(); + runAdminOperationMock.mockRejectedValueOnce(pending); + await expect( + handleServiceBulkJob({ + jobId: "job", + payload: { action: "build", serviceIds: ["valhalla", "osrm"], continueOnError: true }, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn(), + }), + ).rejects.toBe(pending); + expect(runAdminOperationMock).toHaveBeenCalledTimes(1); + }); + it("submits one typed lifecycle operation per exact service ID", async () => { runAdminOperationMock.mockClear(); const ctx = { diff --git a/apps/api/src/services/__tests__/admin-job-ops.test.ts b/apps/api/src/services/__tests__/admin-job-ops.test.ts index 027f6dcf6..f02911c0b 100644 --- a/apps/api/src/services/__tests__/admin-job-ops.test.ts +++ b/apps/api/src/services/__tests__/admin-job-ops.test.ts @@ -1,6 +1,10 @@ -import type { OpsOperation } from "@openmapx/core/ops"; +import { OpsClientError, type OpsOperation } from "@openmapx/core/ops"; import { describe, expect, it, vi } from "vitest"; -import { cancelAdminJobOperations, executeAdminJobOperation } from "../admin-job-ops"; +import { + cancelAdminJobOperations, + executeAdminJobOperation, + resumeAdminJobObservation, +} from "../admin-job-ops"; import type { JobContext } from "../job-runner"; function context(): JobContext { @@ -38,6 +42,85 @@ describe("administrative job operations broker", () => { }); }); + it("retains the original attempted body before a lost admission and restart", async () => { + const operation = { kind: "backup.delete", backupId: "original" } as const; + const operationKey = "opk1_abcdefghijklmnop"; + const operationId = "job1_abcdefghijklmnop"; + const ctx = context(); + let durable: Record | undefined; + ctx.checkpoint = vi.fn(async (value) => { + durable = structuredClone(value); + }); + const client = { + execute: vi.fn(async () => { + throw new OpsClientError("timeout"); + }), + lookup: vi.fn(async (..._args: unknown[]) => { + throw new OpsClientError("authorization"); + }), + status: vi.fn(), + events: vi.fn(), + cancel: vi.fn(), + }; + await executeAdminJobOperation(ctx, operation, "admin-job.backup.delete", { + client: client as never, + operationKey, + pollIntervalMs: 1, + }).catch(() => undefined); + expect(durable).toMatchObject({ + opsProjection: { + version: 2, + operations: { + [operationKey]: { operation, phase: "admitting", operationKey }, + }, + }, + }); + const restarted = context(); + restarted.checkpointResult = durable; + const status = { + version: 1, + operationId, + operationKey, + kind: operation.kind, + resourceId: "original", + state: "succeeded", + result: { backupId: "original" }, + submittedAt: "2026-10-23T00:00:00.000Z", + updatedAt: "2026-10-23T00:00:01.000Z", + }; + client.lookup.mockResolvedValue(status as never); + client.status.mockResolvedValue(status); + client.events.mockResolvedValue({ + version: 1, + operationId, + terminal: true, + events: [], + nextCursor: 0, + truncated: false, + }); + await expect( + executeAdminJobOperation( + restarted, + { ...operation, backupId: "newer-desired-state" }, + "admin-job.backup.delete", + { + client: client as never, + operationKey, + pollIntervalMs: 1, + }, + ), + ).resolves.toEqual({ backupId: "original" }); + expect(client.execute).toHaveBeenCalledTimes(1); + expect(client.lookup.mock.calls.at(-1)?.[0]).toEqual(operation); + expect(restarted.checkpointResult).toMatchObject({ + opsProjection: { + operations: { + [operationKey]: { operation, operationId, phase: "terminal" }, + }, + }, + }); + }); + it("retains the host-update failure code in the durable admin job", async () => { const ctx = context(); const operationKey = "opk1_abcdefghijklmnop"; @@ -146,13 +229,13 @@ describe("administrative job operations broker", () => { expect(client.events).toHaveBeenCalledWith(operationId, { after: 0, limit: 100, - signal: ctx.signal, + signal: expect.any(AbortSignal), }); expect(cancel).not.toHaveBeenCalled(); expect(ctx.checkpoint).toHaveBeenCalledWith( expect.objectContaining({ opsProjection: expect.objectContaining({ - version: 1, + version: 2, eventTotal: 1, byteTotal: 4, operations: expect.objectContaining({ @@ -236,7 +319,7 @@ describe("administrative job operations broker", () => { expect(events).toHaveBeenCalledWith(operationId, { after: 1, limit: 100, - signal: ctx.signal, + signal: expect.any(AbortSignal), }); expect(ctx.log).toHaveBeenCalledTimes(1); expect(ctx.log).toHaveBeenCalledWith("next", "stdout", `ops:${operationId}:2`, 2); @@ -349,6 +432,8 @@ describe("administrative job operations broker", () => { status: vi.fn(), }; + client.status.mockImplementation((_kind: string, id: string) => client.cancel(id)); + // A succeeded sibling must not mask the operation this request contained: // reporting "completed" would restore the job to running. await expect( @@ -394,6 +479,7 @@ describe("administrative job operations broker", () => { status: vi.fn(), }; + client.status.mockImplementation(client.cancel); await expect( cancelAdminJobOperations(projection, client as never, { pollIntervalMs: 1, @@ -402,3 +488,338 @@ describe("administrative job operations broker", () => { ).resolves.toBe("already_terminal"); }); }); + +it("refuses admission when the write-ahead checkpoint fails", async () => { + const ctx = context(); + ctx.checkpoint = vi.fn().mockRejectedValue(new Error("CAS lost")); + const execute = vi.fn(); + await expect( + executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "nightly" }, + "admin-job.backup.delete", + { client: { execute } as never }, + ), + ).rejects.toThrow("CAS lost"); + expect(execute).not.toHaveBeenCalled(); +}); + +it("retains a service workflow parent budget beyond its short render child", async () => { + const { prepareApiJobChildAttempt } = await import("../admin-job-parent"); + const { opsExecutionBudget } = await import("@openmapx/core/ops"); + const ctx = context(); + ctx.jobType = "service.start"; + ctx.execution = { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 1 }; + const now = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); + try { + const render = prepareApiJobChildAttempt( + ctx, + { + kind: "stack.render", + revisionId: "generation", + expectedCurrentRevisionId: null, + }, + "opk1_renderAttempt12345", + ); + now.mockReturnValue(1_800_000_000_000 + opsExecutionBudget("stack.render").processMs); + const start = prepareApiJobChildAttempt( + ctx, + { kind: "service.start", serviceId: "tileserver" }, + "opk1_serviceAttempt1234", + ); + expect(start?.parentDeadlineAt).toBe(render?.parentDeadlineAt); + expect(start?.requestedDeadlineAt).toBe( + Date.now() + opsExecutionBudget("service.start").processMs, + ); + } finally { + now.mockRestore(); + } +}); + +it("validates a resumed operation identity before reading its event stream", async () => { + const ctx = context(); + const operationKey = "opk1_abcdefghijklmnop"; + const operationId = "job1_abcdefghijklmnop"; + ctx.checkpointResult = { + opsProjection: { + version: 1, + eventTotal: 0, + byteTotal: 0, + truncated: false, + operations: { + [operationKey]: { + kind: "backup.delete", + operationKey, + operationId, + cursor: 0, + events: 0, + bytes: 0, + truncated: false, + terminal: false, + }, + }, + }, + }; + const client = { + execute: vi.fn(), + cancel: vi.fn(), + lookup: vi.fn(), + status: vi.fn().mockResolvedValue({ + kind: "backup.delete", + operationId, + operationKey: "opk1_foreignOperation12", + state: "succeeded", + result: { backupId: "foreign" }, + }), + events: vi.fn().mockResolvedValue({ + version: 1, + operationId, + nextCursor: 1, + terminal: true, + truncated: false, + events: [{ cursor: 1, type: "log", stream: "stdout", message: "foreign log" }], + }), + }; + await expect( + executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "nightly" }, + "admin-job.backup.delete", + { client: client as never, operationKey }, + ), + ).rejects.toMatchObject({ reason: "invalid_identity" }); + expect(client.events).not.toHaveBeenCalled(); + expect(ctx.log).not.toHaveBeenCalled(); + expect(client.execute).not.toHaveBeenCalled(); +}); + +it.each([false, true])( + "observes one real journal admission across lost response and both restarts (failure=%s)", + async (failed) => { + const { mkdtemp, writeFile, rm } = await import("node:fs/promises"); + const { join } = await import("node:path"); + const { tmpdir } = await import("node:os"); + const { OpsClient } = await import("@openmapx/core/ops"); + const { buildOpsAgentServer } = await import("../../../../ops-agent/src/server"); + const { openOpsJobJournal } = await import("../../../../ops-agent/src/journal"); + const { OpsExecutionFailure } = await import("../../../../ops-agent/src/runtime"); + const root = await mkdtemp(join(tmpdir(), "task022-admission-")); + const token = Buffer.alloc(32, 7).toString("base64url"); + const tokenFile = join(root, "token"); + await writeFile(tokenFile, token, { mode: 0o600 }); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); + const path = join(root, "journal", "jobs-v1.json"); + const journal = await openOpsJobJournal(path); + const dispatch = vi.fn(async () => { + if (failed) + throw new OpsExecutionFailure("runtime", { + diagnostics: { + truncated: false, + tail: [{ stream: "stderr", message: "final diagnostic" }], + }, + }); + return { backupId: "original" }; + }); + let app = buildOpsAgentServer({ + tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, + journal, + dispatch, + resourcePolicy: { allowBackup: () => true }, + }); + const controller = new AbortController(); + let loseResponse = true; + let posts = 0; + const responses: unknown[] = []; + const fetcher: typeof fetch = async (input, init) => { + const url = new URL(String(input)); + const response = await app.inject({ + method: (init?.method as "GET" | "POST" | "DELETE") ?? "GET", + url: url.pathname + url.search, + headers: Object.fromEntries(new Headers(init?.headers)), + payload: init?.body as string, + }); + responses.push({ path: url.pathname, status: response.statusCode, body: response.body }); + if (init?.method === "POST") { + posts++; + if (loseResponse) { + controller.abort(); + throw new TypeError("Response dropped after admission"); + } + } + return new Response(response.body, { + status: response.statusCode, + headers: { "content-type": "application/json" }, + }); + }; + const client = () => + new OpsClient({ baseUrl: "http://ops-agent:4300", tokenFile, fetch: fetcher }); + let durable: Record | undefined; + const first = context(); + first.signal = controller.signal; + first.execution = { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 3 }; + first.checkpoint = async (value) => { + durable = structuredClone(value); + }; + const key = "opk1_restartAccepted1234"; + try { + const initial = await executeAdminJobOperation( + first, + { kind: "backup.delete", backupId: "original" }, + "admin-job.backup.delete", + { client: client(), operationKey: key, pollIntervalMs: 1 }, + ).catch((error) => error); + expect(initial, JSON.stringify(responses)).toMatchObject({ name: "OpsObservationPending" }); + await vi.waitFor(() => + expect(journal.records()[0]?.state).toBe(failed ? "failed" : "succeeded"), + ); + const admitted = journal.records()[0]; + if (!admitted) throw new Error("Missing retained admission"); + expect(admitted.parent?.apiJobId).toBe(first.jobId); + expect(admitted.parent?.apiOwnerVersion).toBe(3); + expect(admitted.operation).toEqual({ kind: "backup.delete", backupId: "original" }); + await app.close(); + const reloaded = await openOpsJobJournal(path); + app = buildOpsAgentServer({ + tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, + journal: reloaded, + dispatch, + }); + loseResponse = false; + const restarted = context(); + restarted.checkpointResult = durable; + restarted.execution = { owner: "5c0c9623-252c-41b6-8f99-41915ad38b12", version: 4 }; + const observation = executeAdminJobOperation( + restarted, + { kind: "backup.delete", backupId: "newer-state" }, + "admin-job.backup.delete", + { client: client(), operationKey: key, pollIntervalMs: 1 }, + ); + if (failed) + await expect(observation).rejects.toMatchObject({ + name: "ApiOpsError", + failureResult: { + diagnostics: { tail: [{ stream: "stderr", message: "final diagnostic" }] }, + }, + }); + else await expect(observation).resolves.toEqual({ backupId: "original" }); + vi.stubEnv("OPS_AGENT_URL", "http://ops-agent:4300"); + vi.stubGlobal("fetch", fetcher); + const reconciled = context(); + reconciled.jobType = "backup.operation"; + reconciled.observationOnly = true; + reconciled.payload = { name: "newer-state" }; + reconciled.checkpointResult = durable; + const recovery = resumeAdminJobObservation(reconciled); + if (failed) + await expect(recovery).rejects.toMatchObject({ + name: "AdminJobFailure", + result: { diagnostics: { tail: [{ stream: "stderr", message: "final diagnostic" }] } }, + }); + else await expect(recovery).resolves.toMatchObject({ backupId: "original" }); + expect(posts).toBe(1); + expect(dispatch).toHaveBeenCalledTimes(1); + expect(reloaded.records()[0]?.fingerprint).toBe(admitted.fingerprint); + expect(restarted.checkpointResult).toMatchObject({ + opsProjection: { + operations: { + [key]: { + operationId: admitted.operationId, + fingerprint: admitted.fingerprint, + phase: "terminal", + attempt: { apiOwnerVersion: 3 }, + }, + }, + }, + }); + const events = await client().events(admitted.operationId, { after: 100, limit: 10 }); + expect(events.truncated).toBe(true); + expect(events.events).toEqual([]); + for (const path of [ + `/v1/operations/${admitted.operationId}`, + `/v1/operations/${admitted.operationId}/events`, + `/v1/operation-keys/${key}`, + ]) { + const denied = await app.inject({ + method: "GET", + url: path, + headers: { authorization: `Bearer ${Buffer.alloc(32, 8).toString("base64url")}` }, + }); + expect(denied.statusCode).toBe(404); + } + expect( + ( + await app.inject({ + method: "DELETE", + url: `/v1/operations/${admitted.operationId}`, + headers: { authorization: `Bearer ${Buffer.alloc(32, 8).toString("base64url")}` }, + }) + ).statusCode, + ).toBe(404); + expect(dispatch).toHaveBeenCalledTimes(1); + } finally { + await app.close(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + await rm(root, { recursive: true, force: true }); + } + }, +); + +it("does not read or cancel an attempted child belonging to another API job", async () => { + const ctx = context(); + const key = "opk1_foreignAttempt1234"; + const { opsOperationFingerprint } = await import("@openmapx/core/ops"); + ctx.checkpointResult = { + opsProjection: { + version: 2, + eventTotal: 0, + byteTotal: 0, + truncated: false, + operations: { + [key]: { + kind: "backup.delete", + operationKey: key, + operationId: "job1_foreignAttempt1234", + operation: { kind: "backup.delete", backupId: "nightly" }, + phase: "observing", + cursor: 0, + events: 0, + bytes: 0, + terminal: false, + truncated: false, + attempt: { + version: 1, + apiJobId: "4c0c9623-252c-41b6-8f99-41915ad38b12", + apiOwnerVersion: 1, + parentOperationKey: "opk1_parentAttempt12345", + parentFingerprint: "a".repeat(64), + parentDeadlineAt: 2000000000000, + childOperationKey: key, + childOperationFingerprint: opsOperationFingerprint({ + kind: "backup.delete", + backupId: "nightly", + }), + requestedDeadlineAt: 2000000000000, + attemptId: "5c0c9623-252c-41b6-8f99-41915ad38b12", + }, + }, + }, + }, + }; + const client = { + status: vi.fn().mockRejectedValue(new Error("unavailable")), + execute: vi.fn(), + lookup: vi.fn(), + events: vi.fn(), + cancel: vi.fn(), + }; + await executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "nightly" }, + "admin-job.backup.delete", + { client: client as never, operationKey: key }, + ).catch(() => undefined); + expect(client.status).not.toHaveBeenCalled(); + expect(client.events).not.toHaveBeenCalled(); + expect(client.cancel).not.toHaveBeenCalled(); +}); diff --git a/apps/api/src/services/__tests__/admin-ops-authority.test.ts b/apps/api/src/services/__tests__/admin-ops-authority.test.ts index 7fe85fdca..db211ddff 100644 --- a/apps/api/src/services/__tests__/admin-ops-authority.test.ts +++ b/apps/api/src/services/__tests__/admin-ops-authority.test.ts @@ -7,6 +7,10 @@ process.env.BETTER_AUTH_SECRET ||= "admin-ops-authority-test-stub-secret"; const dockerComposeAction = vi.fn().mockResolvedValue({ exitCode: 0, stdout: "", stderr: "" }); const registryGet = vi.fn(); +const executeAdminJobOperation = vi.fn().mockResolvedValue({ changed: true }); +vi.mock("../admin-job-ops", () => ({ + executeAdminJobOperation: (...args: unknown[]) => executeAdminJobOperation(...args), +})); vi.mock("../../utils/docker-compose.js", () => ({ dockerComposeAction: (...args: unknown[]) => dockerComposeAction(...args), @@ -43,14 +47,13 @@ describe("admin service authority split", () => { it("keeps enabled built-ins on typed ops-agent lifecycle", async () => { registryGet.mockReturnValue({ manifest: { id: "redis" }, enabled: true, isBuiltIn: true }); await serviceStop("redis", context()); - expect(dockerComposeAction).toHaveBeenCalledWith( - "redis", - "stop", - expect.objectContaining({ - operationKey: expect.stringMatching(/^opk1_/), - signal: expect.anything(), - }), + expect(executeAdminJobOperation).toHaveBeenCalledWith( + expect.objectContaining({ jobId: "job-123" }), + { kind: "service.stop", serviceId: "redis" }, + "admin-job.service.stop", + { durableIdentity: "redis" }, ); + expect(dockerComposeAction).not.toHaveBeenCalled(); }); it("routes enabled community lifecycle through the same typed operation", async () => { @@ -63,11 +66,14 @@ describe("admin service authority split", () => { await serviceRestart("community-weather", ctx); // No direct-Docker adapter remains: the agent authorizes the community id // against its own trusted selection. - expect(dockerComposeAction).toHaveBeenCalledWith( - "community-weather", - "restart", - expect.objectContaining({ signal: ctx.signal }), + expect(executeAdminJobOperation).toHaveBeenCalledWith( + ctx, + { kind: "service.restart", serviceId: "community-weather" }, + "admin-job.service.restart", + { durableIdentity: "community-weather" }, ); + expect(ctx.checkpoint).toHaveBeenCalledBefore(executeAdminJobOperation); + expect(dockerComposeAction).not.toHaveBeenCalled(); }); it("denies disabled services before either authority boundary", async () => { @@ -78,5 +84,6 @@ describe("admin service authority split", () => { }); await expect(serviceStop("community-weather", context())).rejects.toThrow("is disabled"); expect(dockerComposeAction).not.toHaveBeenCalled(); + expect(executeAdminJobOperation).not.toHaveBeenCalled(); }); }); diff --git a/apps/api/src/services/__tests__/job-reconciliation.test.ts b/apps/api/src/services/__tests__/job-reconciliation.test.ts new file mode 100644 index 000000000..ab07cd00a --- /dev/null +++ b/apps/api/src/services/__tests__/job-reconciliation.test.ts @@ -0,0 +1,39 @@ +import { afterEach, expect, it, vi } from "vitest"; + +const reconcile = vi.hoisted(() => vi.fn()); +vi.mock("../job-runner", () => ({ jobRunner: { reconcile } })); +const { reconcileAdminJobs, startAdminJobReconciliation, stopAdminJobReconciliation } = + await import("../job-reconciliation"); +afterEach(async () => { + await stopAdminJobReconciliation(); + vi.useRealTimers(); + reconcile.mockReset(); +}); +it("runs at startup/every five seconds, coalesces passes, and stops on shutdown", async () => { + vi.useFakeTimers(); + let finish!: () => void; + reconcile + .mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ) + .mockResolvedValue(undefined); + startAdminJobReconciliation(); + expect(reconcile).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(15_000); + expect(reconcile).toHaveBeenCalledTimes(1); + finish(); + await vi.advanceTimersByTimeAsync(5_000); + expect(reconcile).toHaveBeenCalledTimes(2); + await stopAdminJobReconciliation(); + await vi.advanceTimersByTimeAsync(60_000); + expect(reconcile).toHaveBeenCalledTimes(2); +}); +it("clears a failed read pass for the next attempt and preserves the requested bound", async () => { + reconcile.mockRejectedValueOnce(new Error("read unavailable")).mockResolvedValue(undefined); + await expect(reconcileAdminJobs({ limit: 8 })).rejects.toThrow("read unavailable"); + await reconcileAdminJobs({ limit: 8 }); + expect(reconcile).toHaveBeenLastCalledWith({ limit: 8 }, expect.any(AbortSignal)); +}); diff --git a/apps/api/src/services/__tests__/job-runner-cancel.test.ts b/apps/api/src/services/__tests__/job-runner-cancel.test.ts index f089e67ab..b6a1449cc 100644 --- a/apps/api/src/services/__tests__/job-runner-cancel.test.ts +++ b/apps/api/src/services/__tests__/job-runner-cancel.test.ts @@ -1,3 +1,5 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; import { beforeEach, describe, expect, it, vi } from "vitest"; const cancelOperations = vi.hoisted(() => vi.fn()); @@ -21,7 +23,13 @@ const database = vi.hoisted(() => { updates.push(value); return { where: vi.fn(() => ({ - returning: vi.fn().mockResolvedValue([]), + returning: vi + .fn() + .mockResolvedValue( + value.status === "canceled" && updates.length === 1 + ? [] + : [{ id: "job-1", status: value.status ?? "running", result }], + ), // biome-ignore lint/suspicious/noThenProperty: the Drizzle query builder is thenable. then: (resolve: (value: unknown[]) => unknown) => Promise.resolve([]).then(resolve), })), @@ -55,8 +63,12 @@ describe("admin job cancellation", () => { await expect(jobRunner.cancel("job-1")).resolves.toBe(true); - expect(cancelOperations).toHaveBeenCalledWith(database.result); - expect(database.updates).toContainEqual({ status: "cancel_pending" }); + expect(cancelOperations).toHaveBeenCalledWith( + database.result, + undefined, + expect.objectContaining({ onEvidence: expect.any(Function) }), + ); + expect(database.updates).toContainEqual(expect.objectContaining({ status: "cancel_pending" })); expect(database.updates).toContainEqual( expect.objectContaining({ status: "canceled", finishedAt: expect.any(Date) }), ); @@ -66,6 +78,68 @@ describe("admin job cancellation", () => { cancelOperations.mockResolvedValue("pending"); await expect(jobRunner.cancel("job-1")).resolves.toBe(false); - expect(database.updates.at(-1)).toEqual({ status: "cancel_pending" }); + expect(database.updates.at(-1)).toMatchObject({ status: "cancel_pending" }); + }); +}); + +it("observer abort without backend containment never publishes canceled", async () => { + const controller = new AbortController(); + jobRunner.register("test.observer-abort", async () => { + controller.abort(); + throw new DOMException("Aborted", "AbortError"); + }); + const execute = jobRunner as unknown as { + executeJob( + id: string, + type: string, + payload: Record, + result: Record | null, + signal: AbortSignal, + ): Promise; + }; + await execute.executeJob("job-1", "test.observer-abort", {}, null, controller.signal); + expect(database.updates).not.toContainEqual(expect.objectContaining({ status: "canceled" })); + const patch = new PgDialect().sqlToQuery(database.updates.at(-1)?.result as SQL); + expect(JSON.parse(patch.params[0] as string)).toMatchObject({ + observation: { state: "unknown" }, + }); +}); + +it("retains a fulfilled backend's partial failure evidence in failed DB and SSE results", async () => { + const { ApiOpsError } = await import("../ops-client"); + const { jobEventBus } = await import("../job-events"); + const evidence = { + serviceBatch: { + outcome: "partial" as const, + completedServiceIds: ["tileserver"], + failedServiceIds: ["valhalla"], + skippedServiceIds: [], + }, + }; + const events: unknown[] = []; + const unsubscribe = jobEventBus.subscribe("job-evidence", ({ event }) => events.push(event)); + jobRunner.register("test.partial", async () => { + const result = await Promise.resolve(evidence); + throw new ApiOpsError("runtime", result); }); + await (jobRunner as unknown as { executeJob(...args: unknown[]): Promise }).executeJob( + "job-evidence", + "test.partial", + {}, + { phase: "build" }, + new AbortController().signal, + ); + unsubscribe(); + expect(database.updates).toContainEqual( + expect.objectContaining({ + status: "failed", + result: expect.objectContaining({ phase: "build", ...evidence }), + }), + ); + expect(events).toContainEqual( + expect.objectContaining({ + status: "failed", + result: expect.objectContaining(evidence), + }), + ); }); diff --git a/apps/api/src/services/__tests__/job-runner-initialize.test.ts b/apps/api/src/services/__tests__/job-runner-initialize.test.ts index 23322d641..604d2585e 100644 --- a/apps/api/src/services/__tests__/job-runner-initialize.test.ts +++ b/apps/api/src/services/__tests__/job-runner-initialize.test.ts @@ -19,7 +19,11 @@ const dbMock = vi.hoisted(() => { set: vi.fn((values: Record) => { updates.push(values); return { - where: vi.fn(() => Object.assign(emptyResult(), { returning: emptyResult })), + where: vi.fn(() => + Object.assign(emptyResult(), { + returning: async () => rows.map((row) => ({ ...row, status: "running" })), + }), + ), }; }), })), @@ -54,18 +58,21 @@ afterEach(() => { }); describe("job runner restart initialization", () => { - it("requeues only typed agent-backed administrative jobs for durable replay", async () => { + it("keeps typed agent-backed jobs for observation without handler replay", async () => { expect(AGENT_RECOVERABLE_ADMIN_JOB_TYPES).toEqual([ "backup.operation", "data.operation", "service.bulk", + "service.start", + "service.stop", + "service.restart", + "service.apply", + "compose.render", "system.diagnostics", "system.update", ]); await jobRunner.initialize({ completeRestartedUpdates: true }); - expect(dbMock.updates).toContainEqual( - expect.objectContaining({ status: "queued", error: null }), - ); + expect(dbMock.updates).not.toContainEqual(expect.objectContaining({ status: "queued" })); }); it("finalizes a checkpointed update after successful migrations", async () => { @@ -82,7 +89,7 @@ describe("job runner restart initialization", () => { expect.objectContaining({ status: "success", progress: 100, - result: { phase: "complete", completedAfterRestart: true }, + result: expect.objectContaining({ phase: "complete", completedAfterRestart: true }), }), ); }); @@ -153,12 +160,7 @@ describe("job runner restart initialization", () => { resolveReplacementOutcome: appliedOutcome, }); - expect(dbMock.updates).toContainEqual( - expect.objectContaining({ - status: "failed", - error: "Job interrupted by api restart — re-run if still needed", - }), - ); + expect(dbMock.updates).not.toContainEqual(expect.objectContaining({ status: "failed" })); expect(dbMock.updates).not.toContainEqual(expect.objectContaining({ status: "success" })); }); }); diff --git a/apps/api/src/services/__tests__/job-runner-outcomes.test.ts b/apps/api/src/services/__tests__/job-runner-outcomes.test.ts new file mode 100644 index 000000000..5fe176c15 --- /dev/null +++ b/apps/api/src/services/__tests__/job-runner-outcomes.test.ts @@ -0,0 +1,183 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { beforeEach, expect, it, vi } from "vitest"; +import { AdminJobFailure } from "../admin-job-failure"; +import { claimAdminJob, jobOwnership } from "../admin-job-state"; +import { jobEventBus } from "../job-events"; +import { OpsObservationPending } from "../ops-client"; + +const store = vi.hoisted(() => ({ + writes: [] as Array<{ values: Record; where: unknown }>, + accept: true, + status: "running", + rows: [] as unknown[], +})); +vi.mock("../../db", () => ({ + db: { + update: () => ({ + set: (values: Record) => ({ + where: (where: unknown) => { + store.writes.push({ values, where }); + const result = store.accept + ? [{ id: "1d2b29cd-23de-4b19-8c32-86c196833b79", status: store.status }] + : []; + return Object.assign(Promise.resolve(result), { returning: async () => result }); + }, + }), + }), + select: () => ({ + from: () => ({ + where: () => ({ + limit: async () => store.rows, + orderBy: () => ({ limit: async () => store.rows }), + }), + }), + }), + }, +})); +const { jobRunner } = await import("../job-runner"); +const jobId = "1d2b29cd-23de-4b19-8c32-86c196833b79"; +const execution = { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 4 }; +const run = (type: string, result: Record = {}) => + (jobRunner as unknown as { executeJob(...args: unknown[]): Promise }).executeJob( + jobId, + type, + {}, + { ...result, _execution: execution }, + new AbortController().signal, + ); +beforeEach(() => { + store.writes.length = 0; + store.accept = true; + store.status = "running"; + store.rows = []; +}); + +it("merges AdminJobFailure evidence into DB/SSE without publishing ownership or admission proofs", async () => { + const result = { + serviceBatch: { + outcome: "partial", + completedServiceIds: ["tileserver"], + failedServiceIds: ["valhalla"], + skippedServiceIds: [], + }, + }; + const events: unknown[] = []; + const unsubscribe = jobEventBus.subscribe(jobId, ({ event }) => events.push(event)); + jobRunner.register("test.evidence", async () => { + throw new AdminJobFailure("Build failed", await Promise.resolve(result)); + }); + await run("test.evidence", { phase: "build", opsProjection: { private: "proof" } }); + unsubscribe(); + expect(store.writes.at(-1)?.values).toMatchObject({ + status: "failed", + result: { phase: "build", _execution: execution, ...result }, + }); + expect(events.at(-1)).toMatchObject({ + type: "status", + status: "failed", + result: { phase: "build", ...result }, + }); + expect(JSON.stringify(events)).not.toContain("_execution"); + expect(JSON.stringify(events)).not.toContain("proof"); +}); + +it("publishes unknown observation without a terminal event and retains the original identity", async () => { + const identity = { + kind: "service.build" as const, + operationKey: "opk1_abcdefghijklmnop", + operationId: "job1_abcdefghijklmnop", + }; + jobRunner.register("test.unknown", async () => { + throw new OpsObservationPending(identity, "transport"); + }); + const events: Array<{ status?: string }> = []; + const stop = jobEventBus.subscribe(jobId, ({ event }) => { + if (event.type === "status") events.push(event); + }); + await run("test.unknown", { receipt: identity }); + stop(); + const patch = new PgDialect().sqlToQuery(store.writes.at(-1)?.values.result as SQL); + expect(JSON.parse(patch.params[0] as string)).toMatchObject({ + observation: { state: "unknown", reason: "transport", nextCheckAt: expect.any(String) }, + }); + expect(patch.sql).toContain("coalesce("); + expect( + events.some((event) => ["success", "failed", "canceled"].includes(event.status ?? "")), + ).toBe(false); +}); + +it("does not announce an old owner's completion when PostgreSQL rejects its CAS", async () => { + store.accept = false; + jobRunner.register("test.stale", async () => ({ completed: true })); + const events: Array<{ status?: string }> = []; + const stop = jobEventBus.subscribe(jobId, ({ event }) => { + if (event.type === "status") events.push(event); + }); + await run("test.stale"); + stop(); + expect(events.some((event) => event.status === "success")).toBe(false); + const late = new PgDialect().sqlToQuery(store.writes.at(-1)?.values.result as SQL); + expect(JSON.parse(late.params[0] as string)).toEqual({ + _settled: { status: "success", result: { completed: true } }, + }); + for (const write of store.writes) { + const query = new PgDialect().sqlToQuery(write.where as SQL); + expect(query.sql).toContain("->'_execution'"); + expect(query.params).toContain(JSON.stringify(execution)); + expect(query.params).toContain(jobId); + } + expect(new PgDialect().sqlToQuery(store.writes.at(-1)?.where as SQL).params).toContain( + "cancel_pending", + ); +}); + +it("rejects stale checkpoint writes before another child can be admitted", async () => { + store.accept = false; + const effect = vi.fn(); + jobRunner.register("test.checkpoint", async (ctx) => { + await ctx.checkpoint({ _execution: { owner: "forged", version: 999 }, prepared: true }); + effect(); + return {}; + }); + await run("test.checkpoint"); + expect(effect).not.toHaveBeenCalled(); + expect(store.writes[0]?.values.result).toMatchObject({ _execution: execution }); +}); + +it("changes owner and version together while retaining the complete result in PostgreSQL", async () => { + await claimAdminJob({ id: jobId, status: "running", result: { _execution: execution } }); + const write = store.writes[0]; + if (!write) throw new Error("Missing claim"); + const assignment = new PgDialect().sqlToQuery(write.values.result as SQL); + const next = JSON.parse(assignment.params[0] as string)._execution; + expect(next.owner).not.toBe(execution.owner); + expect(next.version).toBe(5); + expect(assignment.sql).toContain("coalesce("); + expect(assignment.sql).toContain("||"); + const where = new PgDialect().sqlToQuery(jobOwnership(jobId, execution) as SQL); + expect(where.params).toEqual([jobId, "running", JSON.stringify(execution)]); + await expect(jobRunner.reconcile({ limit: 33 })).rejects.toThrow("Invalid reconciliation limit"); +}); + +it("CAS binds a fresh child attempt to an absent key and the retained parent before signing", async () => { + const key = "opk1_abcdefghijklmnop"; + jobRunner.register("test.child-intent", async (ctx) => { + await ctx.checkpoint( + { + _parentExecution: { operationKey: "opk1_parentidentity1234" }, + opsProjection: { operations: { [key]: { phase: "admitting" } } }, + }, + undefined, + key, + ); + return {}; + }); + await run("test.child-intent"); + const query = new PgDialect().sqlToQuery(store.writes[0]?.where as SQL); + expect(query.sql).toContain("->'opsProjection'->'operations'->"); + expect(query.sql).toContain("IS NULL"); + expect(query.sql).toContain("->'_parentExecution'"); + expect(query.params).toContain(key); + expect(query.params).toContain(JSON.stringify(execution)); +}); diff --git a/apps/api/src/services/__tests__/ops-client.test.ts b/apps/api/src/services/__tests__/ops-client.test.ts index 2d8c2c89d..fd33dac03 100644 --- a/apps/api/src/services/__tests__/ops-client.test.ts +++ b/apps/api/src/services/__tests__/ops-client.test.ts @@ -247,7 +247,7 @@ describe("bounded async ops helpers", () => { const denied = vi.fn().mockRejectedValue(new ApiOpsError("authorization")); await expect( waitForOpsResult(fakeClient({ status: denied }), admission.kind, admission), - ).rejects.toMatchObject({ errorClass: "authorization" }); + ).rejects.toMatchObject({ name: "OpsObservationPending", reason: "unavailable" }); expect(denied).toHaveBeenCalledTimes(1); }); @@ -372,8 +372,8 @@ describe("bounded async ops helpers", () => { })), }); await expect(waitForOpsResult(client, admission.kind, admission)).rejects.toMatchObject({ - name: "ApiOpsError", - errorClass: "runtime", + name: "OpsObservationPending", + reason: "invalid_identity", }); }); diff --git a/apps/api/src/services/__tests__/system-maintenance.test.ts b/apps/api/src/services/__tests__/system-maintenance.test.ts index 65acd201b..ec8983afa 100644 --- a/apps/api/src/services/__tests__/system-maintenance.test.ts +++ b/apps/api/src/services/__tests__/system-maintenance.test.ts @@ -120,7 +120,11 @@ describe("agent-owned system maintenance", () => { }, "admin-job.system.update", ); - expect(ctx.checkpoint).not.toHaveBeenCalled(); + expect(ctx.checkpoint).toHaveBeenCalledWith( + expect.objectContaining({ + _completion: expect.objectContaining({ operationKey: expect.stringMatching(/^opk1_/) }), + }), + ); expect( executeAdminOperation.mock.calls.some((call) => call[1].kind === "release.resolve"), ).toBe(false); diff --git a/apps/api/src/services/__tests__/trusted-config-operations.test.ts b/apps/api/src/services/__tests__/trusted-config-operations.test.ts index c5fa889bb..63b06f31a 100644 --- a/apps/api/src/services/__tests__/trusted-config-operations.test.ts +++ b/apps/api/src/services/__tests__/trusted-config-operations.test.ts @@ -30,7 +30,8 @@ vi.mock("../service-config-resolver", () => ({ vi.mock("../service-secrets", () => ({ resolveServiceVaultSecretsStrict: async () => ({}), })); -vi.mock("../ops-client", () => ({ +vi.mock("../ops-client", async (importOriginal) => ({ + ...(await importOriginal()), createApiOpsClient: vi.fn(), createDurableOpsKey: () => "opk1_inspect123456789", executeTrustedConfigurationAndWait: vi.fn(), @@ -274,3 +275,96 @@ describe("applyTrustedConfiguration", () => { ).rejects.toThrow("Trusted configuration unavailable"); }); }); + +it("resumes a retained render without inspecting or rebuilding newer configuration", async () => { + prepared.mockClear(); + inspect.mockReset().mockResolvedValue({ revisionId: null, integrationConfigDigests: [] }); + published.length = 0; + process.env.OPS_TRUSTED_CONFIG_DIR = "/var/lib/openmapx/trusted-config"; + process.env.OPS_AGENT_TOKEN_FILE = "/run/secrets/ops-agent-api-token"; + process.env.OPS_TRUSTED_CONFIG_UID = "1000"; + process.env.OPS_TRUSTED_CONFIG_GID = "1000"; + vi.mocked(executeTrustedConfigurationAndWait).mockResolvedValue({ + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + }); + const { createApiOpsClient } = await import("../ops-client"); + const key = "opk1_retainedRender12345"; + const operationId = "job1_retainedRender12345"; + const revisionId = `cfg1_${"r".repeat(43)}`; + const operation = { kind: "stack.render", revisionId, expectedCurrentRevisionId: null }; + const result = { + outcome: "committed", + revisionId, + enabledServiceIds: [], + followUp: "complete", + cleanupFailed: false, + }; + const client = { + execute: vi.fn(), + lookup: vi.fn(), + cancel: vi.fn(), + status: vi.fn().mockResolvedValue({ + version: 1, + kind: operation.kind, + operationId, + operationKey: key, + state: "succeeded", + result, + submittedAt: "2026-10-23T00:00:00Z", + updatedAt: "2026-10-23T00:00:00Z", + }), + events: vi.fn().mockResolvedValue({ + version: 1, + operationId, + nextCursor: 0, + terminal: true, + truncated: false, + events: [], + }), + }; + vi.mocked(createApiOpsClient).mockReturnValue(client as never); + const ctx = { + jobId: "1d2b29cd-23de-4b19-8c32-86c196833b79", + payload: {}, + signal: new AbortController().signal, + log: vi.fn(), + checkpoint: vi.fn(), + setProgress: vi.fn(), + checkpointResult: { + opsProjection: { + version: 2, + eventTotal: 0, + byteTotal: 0, + truncated: false, + operations: { + [key]: { + kind: operation.kind, + operation, + operationId, + operationKey: key, + phase: "observing", + cursor: 0, + events: 0, + bytes: 0, + terminal: false, + truncated: false, + }, + }, + }, + }, + }; + await expect( + applyTrustedConfigurationWithEvidence({ + kind: "stack.render", + operationKey: key, + ctx, + } as never), + ).rejects.toMatchObject({ reason: "recovery_required" }); + expect(prepared).not.toHaveBeenCalled(); + expect(inspect).not.toHaveBeenCalled(); + expect(published).toEqual([]); + expect(client.execute).not.toHaveBeenCalled(); + expect(client.status).toHaveBeenCalled(); +}); diff --git a/apps/api/src/services/activity-jobs.ts b/apps/api/src/services/activity-jobs.ts index d78885a0d..ed91d2be5 100644 --- a/apps/api/src/services/activity-jobs.ts +++ b/apps/api/src/services/activity-jobs.ts @@ -3,6 +3,7 @@ import { unionAll } from "drizzle-orm/pg-core"; import { db } from "../db"; import { adminJob, adminJobLog, dataManagerJobStages, dataManagerJobs } from "../db/schema"; import { type ActorSummary, resolveActors } from "../utils/resolve-actor"; +import { publicAdminJobResult } from "./admin-job-failure"; export type ActivityJobSource = "application" | "data-manager"; export type ActivityJobFilter = "active" | "completed" | "failed"; @@ -205,6 +206,7 @@ export async function getActivityJob(id: string, source: ActivityJobSource) { .orderBy(adminJobLog.seq); return { ...job, + result: publicAdminJobResult(job.result), source, actorLabel: job.createdBy ? "Administrator" : "System", cancelable: job.status === "queued" || job.status === "running", diff --git a/apps/api/src/services/admin-job-failure.ts b/apps/api/src/services/admin-job-failure.ts new file mode 100644 index 000000000..e62b4dfa6 --- /dev/null +++ b/apps/api/src/services/admin-job-failure.ts @@ -0,0 +1,24 @@ +/** A settled failure whose safe evidence must survive both DB and live publication. */ +export class AdminJobFailure extends Error { + constructor( + message: string, + readonly result: Record, + ) { + super(message); + this.name = "AdminJobFailure"; + } +} + +/** Internal admission bodies, ownership and proofs never cross the activity boundary. */ +export function publicAdminJobResult(result: T): T { + if (!result || typeof result !== "object" || Array.isArray(result)) return result; + return Object.fromEntries( + Object.entries(result).filter(([key]) => !key.startsWith("_") && key !== "opsProjection"), + ) as T; +} + +export class JobObservationPending extends Error { + constructor(readonly reason: "transport" | "recovery_required" | "invalid_identity") { + super("Administrative job observation is pending"); + } +} diff --git a/apps/api/src/services/admin-job-handlers.ts b/apps/api/src/services/admin-job-handlers.ts index 3dc79e1cc..e551eb65b 100644 --- a/apps/api/src/services/admin-job-handlers.ts +++ b/apps/api/src/services/admin-job-handlers.ts @@ -1,8 +1,11 @@ import { assertValidBackupName } from "./admin-cli"; +import { JobObservationPending } from "./admin-job-failure"; import { executeAdminJobOperation } from "./admin-job-ops"; +import { JobOwnershipLost } from "./admin-job-state"; import { getAdminOperation, parseAdminOperationInput } from "./admin-operation-catalog"; import { assertKnownServiceIds, assertRegion } from "./admin-operation-input"; import type { JobContext } from "./job-runner"; +import { OpsObservationPending } from "./ops-client"; type BackupOperation = "create" | "restore" | "delete"; type BulkServiceAction = "start" | "stop" | "restart" | "update" | "build"; @@ -159,6 +162,15 @@ export async function handleServiceBulkJob(ctx: JobContext): Promise).opsProjection; if (candidate === undefined) return emptyProjection(); if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) { - throw new Error("Invalid durable operations projection"); + throw new JobObservationPending("invalid_identity"); } const value = candidate as Record; if ( - value.version !== 1 || + (value.version !== 1 && value.version !== 2) || !boundedInteger(value.eventTotal, ADMIN_EVENT_LIMIT) || !boundedInteger(value.byteTotal, ADMIN_EVENT_BYTE_LIMIT) || typeof value.truncated !== "boolean" || @@ -69,21 +100,22 @@ function readProjection(result: unknown): OpsProjection { typeof value.operations !== "object" || Array.isArray(value.operations) ) { - throw new Error("Invalid durable operations projection"); + throw new JobObservationPending("invalid_identity"); } const entries = Object.entries(value.operations as Record); if (entries.length > MAX_PROJECTED_OPERATIONS) - throw new Error("Invalid durable operations projection"); + throw new JobObservationPending("invalid_identity"); const operations: Record = {}; for (const [key, raw] of entries) { if (!OPERATION_KEY.test(key) || !raw || typeof raw !== "object" || Array.isArray(raw)) { - throw new Error("Invalid durable operations projection"); + throw new JobObservationPending("invalid_identity"); } const operation = raw as Record; if ( operation.operationKey !== key || - typeof operation.operationId !== "string" || - !OPERATION_ID.test(operation.operationId) || + (operation.operationId !== undefined && + (typeof operation.operationId !== "string" || !OPERATION_ID.test(operation.operationId))) || + (value.version === 1 && operation.operationId === undefined) || typeof operation.kind !== "string" || !boundedInteger(operation.cursor, Number.MAX_SAFE_INTEGER) || !boundedInteger(operation.events, ADMIN_EVENT_LIMIT) || @@ -91,12 +123,62 @@ function readProjection(result: unknown): OpsProjection { typeof operation.truncated !== "boolean" || typeof operation.terminal !== "boolean" ) { - throw new Error("Invalid durable operations projection"); + throw new JobObservationPending("invalid_identity"); + } + if (!Object.hasOwn(OPS_KIND_POLICIES, operation.kind as string)) + throw new JobObservationPending("invalid_identity"); + if ( + value.version === 2 && + !["admitting", "observing", "terminal"].includes(operation.phase as string) + ) + throw new JobObservationPending("invalid_identity"); + const checked = + operation.operation === undefined + ? undefined + : opsOperationSchema.safeParse(operation.operation); + if (checked && !checked.success) throw new JobObservationPending("invalid_identity"); + const parsed = checked?.data; + if ( + operation.fingerprint !== undefined && + (typeof operation.fingerprint !== "string" || !/^[a-f0-9]{64}$/.test(operation.fingerprint)) + ) + throw new JobObservationPending("invalid_identity"); + if ( + operation.deadlineAt !== undefined && + !boundedInteger(operation.deadlineAt, Number.MAX_SAFE_INTEGER) + ) + throw new JobObservationPending("invalid_identity"); + if ( + operation.failureResult !== undefined && + !opsFailureResultSchema.safeParse(operation.failureResult).success + ) + throw new JobObservationPending("invalid_identity"); + if ( + operation.diagnostics !== undefined && + !processDiagnosticsSchema.safeParse(operation.diagnostics).success + ) + throw new JobObservationPending("invalid_identity"); + if (operation.attempt !== undefined) { + const attempt = apiJobChildAttemptSchema.safeParse(operation.attempt); + if ( + !attempt.success || + !parsed || + attempt.data.childOperationKey !== key || + attempt.data.childOperationFingerprint !== opsOperationFingerprint(parsed) + ) + throw new JobObservationPending("invalid_identity"); } - operations[key] = operation as unknown as ProjectedOperation; + if (parsed && parsed.kind !== operation.kind) + throw new JobObservationPending("invalid_identity"); + operations[key] = { + ...operation, + ...(parsed ? { operation: parsed } : {}), + phase: + value.version === 1 ? (operation.terminal ? "terminal" : "observing") : operation.phase, + } as ProjectedOperation; } return { - version: 1, + version: 2, eventTotal: value.eventTotal, byteTotal: value.byteTotal, truncated: value.truncated, @@ -104,9 +186,14 @@ function readProjection(result: unknown): OpsProjection { }; } -async function persistProjection(context: JobContext, projection: OpsProjection): Promise { +async function persistProjection( + context: JobContext, + projection: OpsProjection, + admissionKey?: string, +): Promise { const value = { ...(context.checkpointResult ?? {}), opsProjection: projection }; - await context.checkpoint(value); + if (admissionKey) await context.checkpoint(value, undefined, admissionKey); + else await context.checkpoint(value); context.checkpointResult = value; } @@ -129,100 +216,315 @@ export async function executeAdminJobOperation( const projection = readProjection(context.checkpointResult); const restored = projection.operations[operationKey]; if (restored && restored.kind !== operation.kind) - throw new Error("Invalid durable operations projection"); + throw new JobObservationPending("invalid_identity"); - const admission = await submitOpsOperationWithRecovery(client, operation, operationKey, { - signal: context.signal, - admissionAlreadyAttempted: restored !== undefined, - }); - if (admission.execution === "sync") return admission.value; - if (restored && restored.operationId !== admission.operationId) { - throw new Error("Invalid durable operations projection"); - } - if (!restored && Object.keys(projection.operations).length >= MAX_PROJECTED_OPERATIONS) { + if (!restored && Object.keys(projection.operations).length >= MAX_PROJECTED_OPERATIONS) throw new Error("Durable operations projection limit exceeded"); - } - - let projected: ProjectedOperation = - restored ?? - ({ - kind: operation.kind, - operationKey, - operationId: admission.operationId, - cursor: 0, - events: projection.eventTotal, - bytes: projection.byteTotal, - truncated: false, - terminal: false, - } satisfies ProjectedOperation); + // A retained typed body wins over any newer desired state. Legacy v1 entries + // have only an ID and are observed directly; no body/identity is invented. + const exactOperation = + restored?.operation ?? (restored ? undefined : opsOperationSchema.parse(operation)); + const projected: ProjectedOperation = restored ?? { + kind: operation.kind, + operationKey, + operation: exactOperation, + phase: "admitting", + cursor: 0, + events: projection.eventTotal, + bytes: projection.byteTotal, + truncated: false, + terminal: false, + attempt: prepareApiJobChildAttempt(context, operation, operationKey), + }; projection.operations[operationKey] = projected; - await persistProjection(context, projection); + if (!restored) await persistProjection(context, projection, operationKey); // CAS BEFORE first POST or signing + return observeOperation(context, projection, projected, client, options, !restored) as Promise< + OpsResultFor + >; +} - const eventMirror = followOpsEvents(client, admission.operationId, { - signal: context.signal, - maxEvents: ADMIN_EVENT_LIMIT, - maxBytes: ADMIN_EVENT_BYTE_LIMIT, - pollIntervalMs: options.pollIntervalMs, - cancelOnExit: false, - initial: { - cursor: projected.cursor, - events: projection.eventTotal, - bytes: projection.byteTotal, - truncated: projection.truncated, - }, - onLog: async (stream, message, cursor) => { - try { - await context.log(message, stream, `ops:${admission.operationId}:${cursor}`, cursor); - } catch { - // Logging is a projection into the API database. Losing that - // projection must not change the durable agent operation. - } - }, - onProgress: async (state) => { - projected = { ...projected, ...state }; - projection.eventTotal = state.events; - projection.byteTotal = state.bytes; - projection.truncated = state.truncated; - projection.operations[operationKey] = projected; - await persistProjection(context, projection); - }, - }).catch(() => undefined); +export function retainedAdminJobOperation(ctx: JobContext, key: string): OpsOperation | undefined { + return readProjection(ctx.checkpointResult).operations[key]?.operation; +} +export function hasRetainedAdminJobOperation(ctx: JobContext, key: string): boolean { + return readProjection(ctx.checkpointResult).operations[key] !== undefined; +} +export async function observeRetainedAdminJobOperation( + ctx: JobContext, + key: string, +): Promise { + const projection = readProjection(ctx.checkpointResult); + const retained = projection.operations[key]; + if (!retained) throw new JobObservationPending("recovery_required"); + return observeOperation(ctx, projection, retained, createApiOpsClient(), {}); +} - // The mirror's returned state is authoritative for its final cursor and - // counters: its limit and byte-budget exits are reported here rather than - // through an intermediate progress callback. - const adoptMirrorState = (state: FollowOpsEventsState | undefined): void => { - if (!state) return; - projected = { ...projected, ...state }; - projection.eventTotal = state.events; - projection.byteTotal = state.bytes; - projection.truncated = state.truncated; +async function observeOperation( + context: JobContext, + projection: OpsProjection, + projected: ProjectedOperation, + client: ApiOpsClient, + options: ExecuteAdminJobOperationOptions, + fresh = false, +): Promise { + const identity = { + kind: projected.kind, + operationKey: projected.operationKey, + operationId: projected.operationId, + }; + if (projected.attempt && projected.attempt.apiJobId !== context.jobId) + throw new OpsObservationPending(identity, "invalid_identity"); + if (projected.terminal && (projected.result !== undefined || projected.errorClass)) { + if (projected.errorClass) + throw new ApiOpsError( + projected.errorClass as ApiOpsError["errorClass"], + projected.failureResult, + ); + return projected.result; + } + const safeCheckpoint = async () => { + try { + await persistProjection(context, projection); + } catch { + /* Observation never cancels effects. */ + } }; - try { - const result = await waitForOpsResult(client, operation.kind, admission, { - signal: context.signal, - pollIntervalMs: options.pollIntervalMs, - }); - adoptMirrorState(await eventMirror); - projected = { ...projected, terminal: true }; - projection.operations[operationKey] = projected; - await persistProjection(context, projection); - return result; + let admission: OpsSubmitResult; + if (projected.operationId) { + // A retained ID is not authority to adopt another operation's logs. + // Authenticate and match its complete identity before opening the stream. + const status = await client + .status(projected.kind, projected.operationId, { + signal: AbortSignal.any([context.signal, AbortSignal.timeout(2_000)]), + }) + .catch(() => { + throw new OpsObservationPending( + identity, + context.signal.aborted ? "observer_deadline" : "unavailable", + ); + }); + if ( + status.kind !== projected.kind || + status.operationId !== projected.operationId || + status.operationKey !== projected.operationKey || + (projected.fingerprint !== undefined && status.fingerprint !== projected.fingerprint) || + (projected.deadlineAt !== undefined && status.deadlineAt !== projected.deadlineAt) + ) + throw new OpsObservationPending(identity, "invalid_identity"); + admission = { + execution: "async" as const, + kind: projected.kind, + operationId: projected.operationId, + operationKey: projected.operationKey, + state: "queued" as const, + fingerprint: projected.fingerprint, + deadlineAt: projected.deadlineAt, + }; + } else { + if (!projected.operation) throw new OpsObservationPending(identity, "invalid_identity"); + admission = await submitOpsOperationWithRecovery( + client, + projected.operation, + projected.operationKey, + { + signal: context.signal, + admissionAlreadyAttempted: !fresh, + recoveryTimeoutMs: 2_000, + ...(projected.attempt + ? { execution: { requestedDeadlineAt: projected.attempt.requestedDeadlineAt } } + : {}), + ...(fresh && projected.attempt + ? { parentAssertion: await issueApiJobChildAssertion(projected.attempt) } + : {}), + }, + ); + if (admission.execution === "sync") { + projected.result = admission.value; + projected.terminal = true; + projected.phase = "terminal"; + await persistProjection(context, projection); + return admission.value; + } + projected.operationId = admission.operationId; + projected.fingerprint = admission.fingerprint; + projected.deadlineAt = admission.deadlineAt; + projected.phase = "observing"; + // If this checkpoint fails, the admitting intent still supports lookup-only recovery. + try { + await persistProjection(context, projection); + } catch { + throw new OpsObservationPending(identity, "transport"); + } + } + const mirrorController = new AbortController(); + const mirrorSignal = AbortSignal.any([context.signal, mirrorController.signal]); + const mirror = projected.mirrorStopped + ? Promise.resolve() + : followOpsEvents(client, admission.operationId, { + signal: mirrorSignal, + maxEvents: ADMIN_EVENT_LIMIT, + maxBytes: ADMIN_EVENT_BYTE_LIMIT, + pollIntervalMs: options.pollIntervalMs, + cancelOnExit: false, + initial: { + cursor: projected.cursor, + events: projection.eventTotal, + bytes: projection.byteTotal, + truncated: projection.truncated, + }, + onLog: async (stream, message, cursor) => { + try { + await context.log(message, stream, `ops:${admission.operationId}:${cursor}`, cursor); + } catch { + projected.truncated = true; + projection.truncated = true; + } + }, + onProgress: async (state) => { + Object.assign(projected, state); + projection.eventTotal = state.events; + projection.byteTotal = state.bytes; + projection.truncated ||= state.truncated; + await safeCheckpoint(); + }, + }) + .then(async (state) => { + if (state) { + Object.assign(projected, state); + projected.mirrorStopped = true; + projection.truncated ||= state.truncated; + await safeCheckpoint(); + } + }) + .catch(() => undefined); + try { + const result = await waitForOpsResult(client, projected.kind, admission, { + signal: context.signal, + pollIntervalMs: options.pollIntervalMs, + ...(context.observationOnly ? { timeoutMs: 2_000 } : {}), + onStatus: (status) => { + if (status.diagnostics) projected.diagnostics = status.diagnostics; + if (status.failureResult) projected.failureResult = status.failureResult; + }, + }); + projected.result = result; + const configuration = configurationApplyResultSchema.safeParse(result); + if (configuration.success) + context.checkpointResult = { + ...context.checkpointResult, + configuration: configuration.data, + }; + projected.terminal = true; + projected.phase = "terminal"; + delete projected.outcome; + return result; + } finally { + mirrorController.abort(); + await mirror; + } } catch (error) { - if (error instanceof ApiOpsError && error.errorClass === "host_update_required") { - context.checkpointResult = { ...context.checkpointResult, errorCode: error.errorClass }; + if (error instanceof ApiOpsError) { + projected.failureResult = error.failureResult; + projected.errorClass = error.errorClass; + projected.terminal = true; + projected.phase = "terminal"; + context.checkpointResult = { + ...context.checkpointResult, + ...error.failureResult, + ...(error.errorClass === "host_update_required" ? { errorCode: error.errorClass } : {}), + }; + } else { + projected.outcome = "unknown"; } - adoptMirrorState(await eventMirror); - projection.operations[operationKey] = projected; - await persistProjection(context, projection); throw error; + } finally { + context.checkpointResult = { + ...context.checkpointResult, + ...projected.failureResult, + ...(projection.truncated ? { logTruncated: true } : {}), + }; + if (projected.diagnostics) + context.checkpointResult = { + ...context.checkpointResult, + diagnostics: projected.diagnostics, + }; + await safeCheckpoint(); } } +/** Reconciliation observes retained attempts only. It never invokes the original handler. */ +export async function resumeAdminJobObservation( + ctx: JobContext, +): Promise | undefined> { + const projection = readProjection(ctx.checkpointResult); + const entries = Object.values(projection.operations); + const firstEntry = entries[0]; + if (!firstEntry) throw new JobObservationPending("recovery_required"); + let client: ApiOpsClient; + try { + client = createApiOpsClient(); + } catch { + throw new JobObservationPending("transport"); + } + let failure: ApiOpsError | undefined; + for (const operation of entries) { + try { + await observeOperation(ctx, projection, operation, client, {}); + } catch (error) { + if (!(error instanceof ApiOpsError)) throw error; + failure ??= error; + } + } + const evidence = { ...ctx.checkpointResult }; + if (failure) + throw new AdminJobFailure(failure.message, { ...evidence, ...failure.failureResult }); + const settled = evidence._settled as + | { status: string; result?: Record; error?: string } + | undefined; + if (settled?.status === "failed") + throw new AdminJobFailure(settled.error ?? "Operation failed", { + ...evidence, + ...settled.result, + }); + if (settled?.status === "success") return { ...evidence, ...settled.result }; + const completion = evidence._completion as + | { operationKey?: string; result?: Record } + | undefined; + if ( + completion?.operationKey && + entries.some((entry) => entry.operationKey === completion.operationKey && entry.terminal) + ) + return { ...evidence, ...completion.result }; + // Single-effect jobs have a complete retained receipt. Multi-step workflows + // require the original completion checkpoint; observing a render is not apply. + if ( + [ + "service.stop", + "service.restart", + "backup.operation", + "data.operation", + "system.diagnostics", + ].includes(ctx.jobType ?? "") && + entries.length === 1 + ) + return { ...evidence, ...((entries[0]?.result as Record) ?? {}) }; + throw new OpsObservationPending( + { + kind: firstEntry.kind, + operationKey: firstEntry.operationKey, + operationId: firstEntry.operationId, + }, + "recovery_required", + ); +} + export interface CancelAdminJobOperationsOptions { pollIntervalMs?: number; sleep?: (ms: number) => Promise; + signal?: AbortSignal; + jobId?: string; + onEvidence?: (evidence: Record) => Promise; } /** Per-operation reconciliation outcome. */ @@ -241,12 +543,12 @@ export type AdminJobCancellationOutcome = "canceled" | "completed" | "already_te export async function cancelAdminJobOperations( result: unknown, - client: Pick = createApiOpsClient(), + client: Pick = createApiOpsClient(), options: CancelAdminJobOperationsOptions = {}, ): Promise { const projection = readProjection(result); - const pending = Object.values(projection.operations).filter((operation) => !operation.terminal); - if (pending.length === 0) return "completed"; + const pending = Object.values(projection.operations); + if (pending.length === 0) return "pending"; const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); const pollIntervalMs = options.pollIntervalMs ?? 250; @@ -259,22 +561,62 @@ export async function cancelAdminJobOperations( // and a failure that predates this request must not be reported as a // cancellation it did not cause. const outcomes: AdminOperationCancellation[] = []; + const readSignal = () => + options.signal + ? AbortSignal.any([options.signal, AbortSignal.timeout(2_000)]) + : AbortSignal.timeout(2_000); for (const operation of pending) { + if (options.signal?.aborted) return "pending"; + if (options.jobId && operation.attempt && operation.attempt.apiJobId !== options.jobId) + return "pending"; + if (!operation.operationId) { + if (!operation.operation) return "pending"; + try { + const found = await client.lookup(operation.operation, operation.operationKey, { + signal: readSignal(), + ...(operation.attempt + ? { execution: { requestedDeadlineAt: operation.attempt.requestedDeadlineAt } } + : {}), + }); + if (found.kind !== operation.kind || found.operationKey !== operation.operationKey) + return "pending"; + operation.operationId = found.operationId; + operation.fingerprint = found.fingerprint; + operation.deadlineAt = found.deadlineAt; + } catch { + return "pending"; + } + } // The agent enters `termination_pending` only from a cancellation request, // so observing it here is equivalent evidence to the durable marker. let sawTerminationPending = false; let status: Awaited> | undefined; + const matches = (value: Awaited>) => + value && + value.operationId === operation.operationId && + value.operationKey === operation.operationKey && + value.kind === operation.kind && + (operation.fingerprint === undefined || value.fingerprint === operation.fingerprint) && + (operation.deadlineAt === undefined || value.deadlineAt === operation.deadlineAt); try { - status = await client.cancel(operation.operationId, { signal: AbortSignal.timeout(2_000) }); + status = await client.status(operation.kind, operation.operationId, { signal: readSignal() }); + if (!matches(status)) return "pending"; + sawTerminationPending = status.state === "termination_pending"; + if (!["succeeded", "failed", "timed_out"].includes(status.state)) + status = await client.cancel(operation.operationId, { + signal: readSignal(), + ...(operation.fingerprint ? { fingerprint: operation.fingerprint } : {}), + }); } catch { // The cancel request may have reached the agent. Reconcile through its // authenticated journal status rather than treating response loss as success. } for (let attempt = 0; attempt < CANCEL_POLL_ATTEMPTS; attempt += 1) { + if (options.signal?.aborted) return "pending"; if (!status || !["succeeded", "failed", "timed_out"].includes(status.state)) { try { status = await client.status(operation.kind, operation.operationId, { - signal: AbortSignal.timeout(2_000), + signal: readSignal(), }); } catch { status = undefined; @@ -284,10 +626,17 @@ export async function cancelAdminJobOperations( if ( status.operationId !== operation.operationId || status.operationKey !== operation.operationKey || + (operation.fingerprint !== undefined && status.fingerprint !== operation.fingerprint) || + (operation.deadlineAt !== undefined && status.deadlineAt !== operation.deadlineAt) || status.kind !== operation.kind ) { throw new Error("Invalid operations cancellation status"); } + await options.onEvidence?.({ + ...status.failureResult, + ...(status.diagnostics ? { diagnostics: status.diagnostics } : {}), + }); + if (status.errorClass === "recovery_required") return "pending"; if (status.state === "termination_pending") sawTerminationPending = true; if (status.state === "succeeded") break; if (status.state === "failed" || status.state === "timed_out") break; diff --git a/apps/api/src/services/admin-job-parent.ts b/apps/api/src/services/admin-job-parent.ts new file mode 100644 index 000000000..500932451 --- /dev/null +++ b/apps/api/src/services/admin-job-parent.ts @@ -0,0 +1,89 @@ +import { createHash, createHmac, randomUUID } from "node:crypto"; +import { + canonicalJson, + type OpsOperation, + opsExecutionBudget, + opsOperationFingerprint, + readOpsTokenFile, +} from "@openmapx/core/ops"; +import { z } from "zod"; +import type { JobContext } from "./job-runner"; +import { createDurableOpsKey } from "./ops-client"; + +export const apiJobChildAttemptSchema = z.strictObject({ + version: z.literal(1), + apiJobId: z.uuid(), + apiOwnerVersion: z.number().int().positive().safe(), + parentOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + parentFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + parentDeadlineAt: z.number().int().nonnegative().safe(), + childOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + childOperationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + attemptId: z.uuid(), +}); +export type ApiJobChildAttempt = z.infer; + +function parentJobBudget(ctx: JobContext, operation: OpsOperation): number { + if (ctx.jobType === "system.update") return opsExecutionBudget("system.update").processMs; + if (ctx.jobType === "service.start" || ctx.jobType === "service.apply") { + // Fixed current workflow: render, inspect, start the coordinator, await its + // one-minute readiness window, apply hardlinks, then start/recreate service. + return [ + "stack.render", + "configuration.hardlinks.inspect", + "service.start", + "generation-hardlinks", + ctx.jobType === "service.apply" ? "service.recreate" : "service.start", + ].reduce((sum, kind) => sum + opsExecutionBudget(kind).processMs, 60_000); + } + if (ctx.jobType === "service.bulk") return opsExecutionBudget("services.buildAll").processMs; + return opsExecutionBudget(operation.kind).processMs; +} +/** These non-secret fields are checkpointed before the assertion is minted. */ +export function prepareApiJobChildAttempt( + ctx: JobContext, + operation: OpsOperation, + key: string, +): ApiJobChildAttempt | undefined { + if (!ctx.execution) return undefined; + const retained = ctx.checkpointResult?._parentExecution as + | { operationKey: string; fingerprint: string; deadlineAt: number } + | undefined; + const parent = retained ?? { + operationKey: createDurableOpsKey("admin-job.parent", ctx.jobId), + // Job identity, not its potentially secret payload, binds the parent. + fingerprint: createHash("sha256") + .update(canonicalJson({ jobId: ctx.jobId })) + .digest("hex"), + deadlineAt: Date.now() + parentJobBudget(ctx, operation), + }; + ctx.checkpointResult = { ...ctx.checkpointResult, _parentExecution: parent }; + return { + version: 1, + apiJobId: ctx.jobId, + apiOwnerVersion: ctx.execution.version, + parentOperationKey: parent.operationKey, + parentFingerprint: parent.fingerprint, + parentDeadlineAt: parent.deadlineAt, + childOperationKey: key, + childOperationFingerprint: opsOperationFingerprint(operation), + requestedDeadlineAt: Math.min( + parent.deadlineAt, + Date.now() + opsExecutionBudget(operation.kind).processMs, + ), + attemptId: randomUUID(), + }; +} +export async function issueApiJobChildAssertion(attempt: ApiJobChildAttempt): Promise { + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + // Exact schema order and NUL-terminated domain used by verifyApiJobParentExecution. + const payload = Buffer.from(JSON.stringify(apiJobChildAttemptSchema.parse(attempt))).toString( + "base64url", + ); + const signature = createHmac("sha256", token) + .update("openmapx-api-job-parent-v1\0") + .update(payload) + .digest("base64url"); + return `${payload}.${signature}`; +} diff --git a/apps/api/src/services/admin-job-state.ts b/apps/api/src/services/admin-job-state.ts new file mode 100644 index 000000000..ab41648f5 --- /dev/null +++ b/apps/api/src/services/admin-job-state.ts @@ -0,0 +1,66 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { db } from "../db"; +import { adminJob } from "../db/schema"; + +export interface JobExecution { + owner: string; + version: number; +} +export class JobOwnershipLost extends Error { + constructor() { + super("Administrative job ownership changed"); + } +} +export function readJobExecution(result: unknown): JobExecution | undefined { + if ( + result !== null && + result !== undefined && + (typeof result !== "object" || Array.isArray(result)) + ) + throw new JobOwnershipLost(); + const value = (result as { _execution?: JobExecution } | null)?._execution; + if (value === undefined) return undefined; + if ( + !value || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + value.owner, + ) || + !Number.isSafeInteger(value.version) || + value.version < 0 + ) + throw new JobOwnershipLost(); + return value; +} +export function jobOwnership( + id: string, + execution: JobExecution | undefined, + statuses = ["running"], +) { + return and( + eq(adminJob.id, id), + inArray(adminJob.status, statuses), + execution + ? sql`${adminJob.result}->'_execution' = ${JSON.stringify(execution)}::jsonb` + : sql`${adminJob.result}->'_execution' IS NULL`, + ); +} +/** Both fields change in one PostgreSQL statement; the old JSON member is the CAS token. */ +export async function claimAdminJob( + job: { id: string; result: unknown; status: string }, + patch: Partial = {}, + observation = false, +) { + const previous = readJobExecution(job.result); + if (previous?.version === Number.MAX_SAFE_INTEGER) throw new JobOwnershipLost(); + const execution = { owner: randomUUID(), version: (previous?.version ?? 0) + 1 }; + const [claimed] = await db + .update(adminJob) + .set({ + ...patch, + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _execution: execution, ...(observation ? { _observationLeaseUntil: new Date(Date.now() + 60_000).toISOString() } : {}) })}::jsonb`, + }) + .where(jobOwnership(job.id, previous, [job.status])) + .returning(); + return claimed; +} diff --git a/apps/api/src/services/admin-ops.ts b/apps/api/src/services/admin-ops.ts index 349fcaa19..200b695e6 100644 --- a/apps/api/src/services/admin-ops.ts +++ b/apps/api/src/services/admin-ops.ts @@ -12,6 +12,7 @@ import { dockerComposeAction, dockerStatus, } from "../utils/docker-compose"; +import { executeAdminJobOperation } from "./admin-job-ops"; import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; import { getServiceRegistry } from "./service-registry"; @@ -86,6 +87,7 @@ export function resetDockerCache(): void { export async function renderAndPersistCompose(options: { operationKey: string; signal?: AbortSignal; + ctx?: JobContext; }): Promise<{ revisionId: string }> { const { revisionId } = await applyTrustedConfiguration({ kind: "stack.render", ...options }); return { revisionId }; @@ -213,16 +215,27 @@ async function serviceComposeAction( // Community and built-in services now take the same typed path: the agent // authorizes the id against its own trusted selection, so the API no longer // needs a direct-Docker adapter for community lifecycle. - return dockerComposeAction(service, action, { - operationKey: createDurableOpsKey(`admin-job.service.${action}`, `${ctx.jobId}:${service}`), - signal: ctx.signal, + await ctx.checkpoint({ + ...ctx.checkpointResult, + _completion: { + operationKey: createDurableOpsKey(`admin-job.service.${action}`, `${ctx.jobId}:${service}`), + result: { service, action }, + }, }); + await executeAdminJobOperation( + ctx, + { kind: `service.${action}`, serviceId: service }, + `admin-job.service.${action}`, + { durableIdentity: service }, + ); + return { exitCode: 0 }; } export async function serviceStart(service: string, ctx: JobContext): Promise { assertKnownService(service); await ctx.log(`Rendering compose for latest config...`); const { revisionId } = await renderAndPersistCompose({ + ctx, operationKey: createDurableOpsKey("admin-job.config.render", `${ctx.jobId}:${service}`), signal: ctx.signal, }); @@ -248,12 +261,14 @@ export async function serviceApply(service: string, ctx: JobContext): Promise & { @@ -90,6 +91,8 @@ export class JobEventBus { } publish(jobId: string, event: JobEvent): StoredJobEvent { + if (event.type === "status" && event.result !== undefined) + event = { ...event, result: publicAdminJobResult(event.result) }; const state = this.state(jobId); const stored: StoredJobEvent = { cursor: state.nextCursor, publishedAtMs: this.nowMs(), event }; state.nextCursor += 1; diff --git a/apps/api/src/services/job-reconciliation.ts b/apps/api/src/services/job-reconciliation.ts new file mode 100644 index 000000000..65beceafc --- /dev/null +++ b/apps/api/src/services/job-reconciliation.ts @@ -0,0 +1,32 @@ +import { jobRunner } from "./job-runner"; + +let timer: ReturnType | undefined; +let pass: Promise | undefined; +let observer: AbortController | undefined; +export async function reconcileAdminJobs(options?: { limit?: number }): Promise { + if (pass) return pass; + observer = new AbortController(); + pass = jobRunner.reconcile(options, observer.signal); + try { + await pass; + } finally { + pass = undefined; + } +} +export function startAdminJobReconciliation(): void { + if (timer) return; + const run = () => { + void reconcileAdminJobs().catch(() => { + /* Next bounded pass retries reads. */ + }); + }; + timer = setInterval(run, 5_000); + timer.unref?.(); + run(); +} +export async function stopAdminJobReconciliation(): Promise { + if (timer) clearInterval(timer); + timer = undefined; + observer?.abort(); + await pass; +} diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index e6ce6736f..8b34fdeb0 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -1,10 +1,19 @@ import { randomUUID } from "node:crypto"; -import { and, asc, eq, inArray, notInArray } from "drizzle-orm"; +import { and, asc, eq, inArray, sql } from "drizzle-orm"; import { db } from "../db"; import { adminJob, adminJobLog } from "../db/schema"; import { dbActorId } from "../utils/actor"; -import { cancelAdminJobOperations } from "./admin-job-ops"; +import { AdminJobFailure, JobObservationPending, publicAdminJobResult } from "./admin-job-failure"; +import { cancelAdminJobOperations, resumeAdminJobObservation } from "./admin-job-ops"; +import { + claimAdminJob, + type JobExecution, + JobOwnershipLost, + jobOwnership, + readJobExecution, +} from "./admin-job-state"; import { type JobStatusEvent, jobEventBus, TERMINAL_JOB_STATUSES } from "./job-events"; +import { ApiOpsError, OpsObservationPending } from "./ops-client"; import { appApiRestartCheckpoint } from "./system-update-state"; type AppApiReplacementOutcome = "applied" | "rolled-back" | "failed"; @@ -16,6 +25,9 @@ interface AppApiRuntimeInfo { export interface JobContext { jobId: string; + jobType?: string; + execution?: JobExecution; + observationOnly?: boolean; payload: Record; signal: AbortSignal; log( @@ -26,7 +38,11 @@ export interface JobContext { ): Promise; setProgress(progress: number): Promise; /** Persist restart-safe state before an operation intentionally replaces app-api. */ - checkpoint(result: Record, progress?: number): Promise; + checkpoint( + result: Record, + progress?: number, + admissionKey?: string, + ): Promise; /** Last durable checkpoint loaded with the job and updated after each checkpoint. */ checkpointResult?: Record | null; } @@ -37,6 +53,11 @@ export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ "backup.operation", "data.operation", "service.bulk", + "service.start", + "service.stop", + "service.restart", + "service.apply", + "compose.render", "system.diagnostics", "system.update", ] as const; @@ -92,42 +113,45 @@ class AdminJobRunner { } const [job] = await db - .select({ status: adminJob.status, result: adminJob.result }) - .from(adminJob) - .where(and(eq(adminJob.id, jobId), eq(adminJob.status, "running"))) - .limit(1); - if (!job) return false; - await db .update(adminJob) - .set({ status: "cancel_pending" }) - .where(and(eq(adminJob.id, jobId), eq(adminJob.status, "running"))); + .set({ + status: "cancel_pending", + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || jsonb_build_object('_cancelRequestedAt', coalesce(${adminJob.result}->>'_cancelRequestedAt', ${new Date().toISOString()}::text))`, + }) + .where(and(eq(adminJob.id, jobId), inArray(adminJob.status, ["running", "cancel_pending"]))) + .returning(); + if (!job) return false; this.announce(jobId, "cancel_pending"); - let outcome: Awaited>; + // Stops only this observer/forward handler. Backend containment is independently verified. + this.active.get(jobId)?.abort(); + let outcome: Awaited> = "pending"; try { - outcome = await cancelAdminJobOperations(job.result); + outcome = await cancelAdminJobOperations(job.result, undefined, { + jobId, + onEvidence: async (evidence) => { + await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify(evidence)}::jsonb`, + }) + .where(jobOwnership(jobId, readJobExecution(job.result), ["cancel_pending"])); + }, + }); } catch { - return false; + /* Retain intent. */ } - if (outcome === "pending") return false; - // `already_terminal` means the agent work had ended before this request - // reached it, so this cancellation contained nothing. The still-running - // local execution settles the row; claiming a cancellation here would - // record containment that never happened. - if (outcome === "completed" || outcome === "already_terminal") { - await db - .update(adminJob) - .set({ status: "running" }) - .where(and(eq(adminJob.id, jobId), eq(adminJob.status, "cancel_pending"))); - this.announce(jobId, "running"); - return false; - } - this.active.get(jobId)?.abort(); + if (outcome !== "canceled") return false; const finishedAt = new Date(); - await db + const rows = await db .update(adminJob) .set({ status: "canceled", finishedAt }) - .where(and(eq(adminJob.id, jobId), eq(adminJob.status, "cancel_pending"))); - this.announce(jobId, "canceled", { finishedAt: finishedAt.toISOString() }); + .where(jobOwnership(jobId, readJobExecution(job.result), ["cancel_pending"])) + .returning({ id: adminJob.id, result: adminJob.result }); + if (rows.length === 0) return false; + this.announce(jobId, "canceled", { + finishedAt: finishedAt.toISOString(), + result: publicAdminJobResult(rows[0]?.result as Record | null), + }); return true; } @@ -135,7 +159,12 @@ class AdminJobRunner { const [job] = await db .select({ id: adminJob.id, status: adminJob.status }) .from(adminJob) - .where(and(eq(adminJob.type, type), inArray(adminJob.status, ["queued", "running"]))) + .where( + and( + eq(adminJob.type, type), + inArray(adminJob.status, ["queued", "running", "cancel_pending"]), + ), + ) .orderBy(asc(adminJob.createdAt)) .limit(1); return job ?? null; @@ -149,175 +178,88 @@ class AdminJobRunner { cleanupReplacementOutcome?: (outcomeFile: string) => Promise; } = {}, ): Promise { - const interruptedCancellations = await db - .select({ id: adminJob.id, status: adminJob.status, result: adminJob.result }) - .from(adminJob) - .where(eq(adminJob.status, "cancel_pending")); - for (const job of interruptedCancellations) { - if (job.status !== "cancel_pending") continue; - let outcome: Awaited> = "pending"; - try { - outcome = await cancelAdminJobOperations(job.result); - } catch { - // Keep the durable pending state until the authoritative agent can be reached. - } - if (outcome === "canceled") { - await db - .update(adminJob) - .set({ status: "canceled", finishedAt: new Date() }) - .where(and(eq(adminJob.id, job.id), eq(adminJob.status, "cancel_pending"))); - } else if (outcome === "completed") { - await db - .update(adminJob) - .set({ status: "queued", startedAt: null }) - .where(and(eq(adminJob.id, job.id), eq(adminJob.status, "cancel_pending"))); - } else if (outcome === "already_terminal") { - // The agent operations had already failed before the cancel request, - // and no local execution survived the restart to settle this row. - await db - .update(adminJob) - .set({ - status: "failed", - finishedAt: new Date(), - error: "Operation failed before the cancellation request was applied", - }) - .where(and(eq(adminJob.id, job.id), eq(adminJob.status, "cancel_pending"))); - } - } // An application update deliberately replaces app-api as its final step. // The server calls this only after it is listening. Finalize only jobs that // wrote the explicit pre-restart checkpoint, passed the migration gate, // and are now running the exact image that the update pulled. Any other - // interrupted job remains a failure (including an update that died before - // the checkpoint). + // interrupted job retains uncertainty until its original effect is observed. const interruptedUpdates = await db - .select({ id: adminJob.id, result: adminJob.result }) + .select({ id: adminJob.id, result: adminJob.result, status: adminJob.status }) .from(adminJob) .where(and(eq(adminJob.status, "running"), eq(adminJob.type, "system.update"))); const checkpointedUpdates = interruptedUpdates.flatMap((job) => { const checkpoint = appApiRestartCheckpoint(job.result); - return checkpoint ? [{ id: job.id, checkpoint }] : []; + return checkpoint ? [{ ...job, checkpoint }] : []; }); - const checkpointedUpdateIds = checkpointedUpdates.map((job) => job.id); - const resolveOutcome = options.resolveReplacementOutcome ?? (async () => null); - const evaluatedUpdates = await Promise.all( - checkpointedUpdates.map(async (job) => ({ - ...job, - outcome: await resolveOutcome(job.checkpoint.outcomeFile), - })), - ); - const runtime = options.currentAppApiRuntime; - const cleanupOutcome = options.cleanupReplacementOutcome ?? (async () => undefined); - const completedUpdateIds = - options.completeRestartedUpdates === false - ? [] - : evaluatedUpdates - .filter( - (job) => - job.outcome === "applied" && - runtime?.imageId === job.checkpoint.expectedImageId && - runtime.containerId !== job.checkpoint.previousContainerId && - runtime.updateJobId === job.id, - ) - .map((job) => job.id); - if (completedUpdateIds.length > 0) { - await db - .update(adminJob) - .set({ - status: "success", - finishedAt: new Date(), - progress: 100, - result: { phase: "complete", completedAfterRestart: true }, - }) - .where(inArray(adminJob.id, completedUpdateIds)); - await Promise.all( - evaluatedUpdates - .filter((job) => completedUpdateIds.includes(job.id)) - .map((job) => cleanupOutcome(job.checkpoint.outcomeFile)), + for (const candidate of checkpointedUpdates) { + const claimed = await claimAdminJob({ ...candidate, status: candidate.status ?? "running" }); + if (!claimed) continue; + const execution = readJobExecution(claimed.result); + const outcome = await (options.resolveReplacementOutcome ?? (async () => null))( + candidate.checkpoint.outcomeFile, ); - } - if (options.completeRestartedUpdates === false && checkpointedUpdateIds.length > 0) { - await db + const runtime = options.currentAppApiRuntime; + const migrationsFailed = options.completeRestartedUpdates === false; + if (!migrationsFailed && (outcome === null || (outcome === "applied" && !runtime))) continue; + const complete = + !migrationsFailed && + outcome === "applied" && + runtime?.imageId === candidate.checkpoint.expectedImageId && + runtime.containerId !== candidate.checkpoint.previousContainerId && + runtime.updateJobId === candidate.id; + const error = migrationsFailed + ? "Update replaced app-api, but database migrations did not complete" + : outcome === "rolled-back" + ? "Update failed; the previous app-api image was restored" + : outcome === "failed" + ? "Update failed and app-api rollback did not complete" + : "Update restarted app-api with an unexpected container identity or image"; + const rows = await db .update(adminJob) .set({ - status: "failed", + status: complete ? "success" : "failed", finishedAt: new Date(), - error: "Update replaced app-api, but database migrations did not complete", + ...(complete + ? { + progress: 100, + result: { + ...(claimed.result as Record), + phase: "complete", + completedAfterRestart: true, + }, + } + : { error }), }) - .where(inArray(adminJob.id, checkpointedUpdateIds)); - await Promise.all(evaluatedUpdates.map((job) => cleanupOutcome(job.checkpoint.outcomeFile))); - } - - if (options.completeRestartedUpdates !== false) { - const failedUpdates = evaluatedUpdates.filter((job) => !completedUpdateIds.includes(job.id)); - for (const job of failedUpdates) { - const error = - job.outcome === "rolled-back" - ? "Update failed; the previous app-api image was restored" - : job.outcome === "failed" - ? "Update failed and app-api rollback did not complete" - : job.outcome === null - ? "Update helper did not report a durable outcome" - : "Update restarted app-api with an unexpected container identity or image"; - await db - .update(adminJob) - .set({ - status: "failed", - finishedAt: new Date(), - error, - }) - .where(eq(adminJob.id, job.id)); - await cleanupOutcome(job.checkpoint.outcomeFile); - } + .where(jobOwnership(candidate.id, execution)) + .returning({ id: adminJob.id }); + if (rows.length) await options.cleanupReplacementOutcome?.(candidate.checkpoint.outcomeFile); } - // Agent-backed jobs use deterministic operation keys. Re-running their API - // projection performs a journal lookup/replay and cannot duplicate the - // underlying host effect, so an API restart safely requeues them. + // Startup marks interrupted observers due; no handler is requeued/replayed. await db .update(adminJob) .set({ - status: "queued", - startedAt: null, - finishedAt: null, - error: null, + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || jsonb_build_object('observation', jsonb_build_object('state', 'unknown', 'reason', 'transport', 'nextCheckAt', ${new Date().toISOString()}::text))`, }) .where( and( - eq(adminJob.status, "running"), + inArray(adminJob.status, ["running", "cancel_pending"]), inArray(adminJob.type, [...AGENT_RECOVERABLE_ADMIN_JOB_TYPES]), ), ); - // Mark every other job that was running when the api process died as failed. - // - // Earlier this re-queued them, but that turned a job that crashed - // the api once into a crash loop: the job re-runs immediately on - // startup, hits the same condition that killed the api before - // (commonly `service.bulk` recreating postgis while the api was - // logging to it), and dies again before clearing its own row. - // - // Failing the row instead is safe: the operator can re-trigger the - // intent from the admin UI, and they'll see in the activity log that - // the previous attempt was aborted by an api restart. + // Unknown handlers are never replayed. Interruption alone proves neither + // failure nor containment, so retain their evidence for operator recovery. await db .update(adminJob) .set({ - status: "failed", - finishedAt: new Date(), - error: "Job interrupted by api restart — re-run if still needed", + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || jsonb_build_object('observation', jsonb_build_object('state', 'unknown', 'reason', 'recovery_required', 'nextCheckAt', ${new Date().toISOString()}::text))`, }) .where( - checkpointedUpdateIds.length > 0 - ? and( - eq(adminJob.status, "running"), - notInArray(adminJob.type, [...AGENT_RECOVERABLE_ADMIN_JOB_TYPES]), - notInArray(adminJob.id, checkpointedUpdateIds), - ) - : and( - eq(adminJob.status, "running"), - notInArray(adminJob.type, [...AGENT_RECOVERABLE_ADMIN_JOB_TYPES]), - ), + and( + inArray(adminJob.status, ["running", "cancel_pending"]), + sql`${adminJob.result}->'observation' IS NULL`, + ), ); void this.pump(); } @@ -335,28 +277,16 @@ class AdminJobRunner { do { this.pumpAgain = false; while (this.active.size < this.maxConcurrent) { - // Atomically claim a queued job by updating its status to running - const [job] = await db - .update(adminJob) - .set({ status: "running", startedAt: new Date() }) - .where( - and( - eq(adminJob.status, "queued"), - inArray( - adminJob.id, - db - .select({ id: adminJob.id }) - .from(adminJob) - .where(eq(adminJob.status, "queued")) - .orderBy(asc(adminJob.createdAt)) - .limit(1), - ), - ), - ) - .returning(); - - if (!job) break; - this.launchJob(job); + const [candidate] = await db + .select() + .from(adminJob) + .where(eq(adminJob.status, "queued")) + .orderBy(asc(adminJob.createdAt)) + .limit(1); + if (!candidate) break; + const job = await claimAdminJob(candidate, { status: "running", startedAt: new Date() }); + if (!job) continue; + void this.launchJob(job); } } while (this.pumpAgain && this.active.size < this.maxConcurrent); } finally { @@ -364,17 +294,28 @@ class AdminJobRunner { } } - private launchJob(job: typeof adminJob.$inferSelect): void { + private launchJob( + job: typeof adminJob.$inferSelect, + observe = false, + observationSignal?: AbortSignal, + ): Promise { const controller = new AbortController(); this.active.set(job.id, controller); this.logSeq.set(job.id, 0); - void this.executeJob( + return this.executeJob( job.id, job.type, (job.payload as Record) ?? {}, (job.result as Record | null) ?? null, - controller.signal, + observe + ? AbortSignal.any([ + controller.signal, + AbortSignal.timeout(3_000), + ...(observationSignal ? [observationSignal] : []), + ]) + : controller.signal, + observe, ) .finally(() => { this.active.delete(job.id); @@ -395,22 +336,16 @@ class AdminJobRunner { payload: Record, checkpointResult: Record | null, signal: AbortSignal, + observe = false, ): Promise { - const handler = this.handlers.get(type); - if (!handler) { - await db - .update(adminJob) - .set({ - status: "failed", - error: `No handler registered for job type "${type}"`, - finishedAt: new Date(), - }) - .where(eq(adminJob.id, jobId)); - return; - } - + const handler = observe ? resumeAdminJobObservation : this.handlers.get(type); + const execution = readJobExecution(checkpointResult); + let checkpointTail = Promise.resolve(); const ctx: JobContext = { jobId, + jobType: type, + execution, + observationOnly: observe, payload, signal, checkpointResult, @@ -435,63 +370,215 @@ class AdminJobRunner { }); }, setProgress: async (progress) => { - await db.update(adminJob).set({ progress }).where(eq(adminJob.id, jobId)); + const rows = await db + .update(adminJob) + .set({ progress }) + .where(jobOwnership(jobId, execution)) + .returning({ id: adminJob.id }); + if (!rows.length) throw new JobOwnershipLost(); jobEventBus.publish(jobId, { type: "progress", progress }); }, - checkpoint: async (result, progress) => { - await db - .update(adminJob) - .set({ result, ...(progress === undefined ? {} : { progress }) }) - .where(eq(adminJob.id, jobId)); - ctx.checkpointResult = result; - if (progress !== undefined) jobEventBus.publish(jobId, { type: "progress", progress }); + checkpoint: async (result, progress, admissionKey) => { + const write = checkpointTail.then(async () => { + // A handler cannot replace the runner's ownership member. + const merged: Record = { + ...ctx.checkpointResult, + ...result, + ...(execution ? { _execution: execution } : {}), + }; + const rows = await db + .update(adminJob) + .set({ result: merged, ...(progress === undefined ? {} : { progress }) }) + .where( + and( + jobOwnership(jobId, execution), + ...(admissionKey + ? [ + sql`${adminJob.result}->'opsProjection'->'operations'->${admissionKey} IS NULL`, + sql`(${adminJob.result}->'_parentExecution' IS NULL OR ${adminJob.result}->'_parentExecution' = ${JSON.stringify(merged._parentExecution ?? null)}::jsonb)`, + ] + : []), + ), + ) + .returning({ id: adminJob.id }); + if (!rows.length) throw new JobOwnershipLost(); + ctx.checkpointResult = merged; + if (progress !== undefined) jobEventBus.publish(jobId, { type: "progress", progress }); + }); + checkpointTail = write.catch(() => undefined); + return write; }, }; - this.announce(jobId, "running", { startedAt: new Date().toISOString() }); + if (!observe) this.announce(jobId, "running", { startedAt: new Date().toISOString() }); const startedAtMs = Date.now(); - try { - if (signal.aborted) throw new DOMException("Aborted", "AbortError"); - const result = await handler(ctx); - jobEventBus.recordHandlerDuration(Date.now() - startedAtMs); + const pending = async (reason: OpsObservationPending["reason"]) => { + const attempts = Math.min(12, Number(ctx.checkpointResult?._observationAttempts ?? 0) + 1); + const observationPatch = { + _observationAttempts: attempts, + _observationLeaseUntil: null, + observation: { + state: "unknown", + reason, + nextCheckAt: new Date( + Date.now() + Math.min(60_000, 5_000 * 2 ** (attempts - 1)), + ).toISOString(), + }, + }; + const rows = await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify(observationPatch)}::jsonb`, + }) + .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])) + .returning({ status: adminJob.status, result: adminJob.result }); + if (rows[0]) + this.announce(jobId, rows[0].status, { + result: publicAdminJobResult(rows[0].result as Record | null), + }); + }; + const settle = async ( + status: "success" | "failed", + result: Record | null, + error?: string, + ) => { + await checkpointTail; + const merged: Record = { + ...ctx.checkpointResult, + ...result, + ...(execution ? { _execution: execution } : {}), + }; + delete merged.observation; const finishedAt = new Date(); - if (signal.aborted) { - await db - .update(adminJob) - .set({ status: "canceled", finishedAt }) - .where(eq(adminJob.id, jobId)); - this.announce(jobId, "canceled", { finishedAt: finishedAt.toISOString() }); + const rows = await db + .update(adminJob) + .set({ + status, + result: merged, + error: error ?? null, + finishedAt, + ...(status === "success" ? { progress: 100 } : {}), + }) + .where(jobOwnership(jobId, execution)) + .returning({ id: adminJob.id }); + if (rows.length) { + this.announce(jobId, status, { + result: publicAdminJobResult(merged), + error: error ?? null, + ...(status === "success" ? { progress: 100 } : {}), + finishedAt: finishedAt.toISOString(), + }); } else { - const mergedResult = - result === undefined - ? (ctx.checkpointResult ?? null) - : { ...(ctx.checkpointResult ?? {}), ...result }; + // Cancel intent wins; retain late evidence without publishing a terminal event. await db .update(adminJob) - .set({ status: "success", result: mergedResult, finishedAt, progress: 100 }) - .where(eq(adminJob.id, jobId)); - this.announce(jobId, "success", { - result: mergedResult, - progress: 100, - finishedAt: finishedAt.toISOString(), - }); + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _settled: { status, result: publicAdminJobResult(merged), error } })}::jsonb`, + }) + .where(jobOwnership(jobId, execution, ["cancel_pending"])); + } + }; + try { + if (signal.aborted) throw new DOMException("Aborted", "AbortError"); + if (!handler) throw new Error(`No handler registered for job type "${type}"`); + if (observe) { + const [current] = await db + .select() + .from(adminJob) + .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])) + .limit(1); + if (!current) return; + if (current.status === "cancel_pending") { + const outcome = await cancelAdminJobOperations(current.result, undefined, { + jobId, + signal, + onEvidence: async (evidence) => { + ctx.checkpointResult = { ...ctx.checkpointResult, ...evidence }; + await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify(evidence)}::jsonb`, + }) + .where(jobOwnership(jobId, execution, ["cancel_pending"])); + }, + }); + if (outcome === "pending") { + await pending("transport"); + return; + } + if (outcome === "canceled") { + const finishedAt = new Date(); + const rows = await db + .update(adminJob) + .set({ status: "canceled", finishedAt }) + .where(jobOwnership(jobId, execution, ["cancel_pending"])) + .returning({ id: adminJob.id }); + if (rows.length) + this.announce(jobId, "canceled", { + finishedAt: finishedAt.toISOString(), + result: publicAdminJobResult(ctx.checkpointResult), + }); + return; + } + // Containment was not performed. Observe the original outcome under this claim. + const rows = await db + .update(adminJob) + .set({ status: "running" }) + .where(jobOwnership(jobId, execution, ["cancel_pending"])) + .returning({ id: adminJob.id }); + if (!rows.length) return; + } } + const result = await handler(ctx); + await settle("success", result ?? null); } catch (err) { + if (err instanceof OpsObservationPending || err instanceof JobObservationPending) + await pending(err.reason); + else if ( + err instanceof JobOwnershipLost || + signal.aborted || + (err instanceof Error && err.name === "AbortError") + ) + await pending("observer_deadline"); + else + await settle( + "failed", + err instanceof AdminJobFailure + ? err.result + : err instanceof ApiOpsError + ? (err.failureResult ?? null) + : null, + err instanceof Error ? err.message : "Administrative job failed", + ); + } finally { jobEventBus.recordHandlerDuration(Date.now() - startedAtMs); - const isAbort = signal.aborted || (err instanceof DOMException && err.name === "AbortError"); - const finishedAt = new Date(); - const error = err instanceof Error ? err.message : String(err); - await db - .update(adminJob) - .set(isAbort ? { status: "canceled", finishedAt } : { status: "failed", error, finishedAt }) - .where(eq(adminJob.id, jobId)); - this.announce( - jobId, - isAbort ? "canceled" : "failed", - isAbort - ? { finishedAt: finishedAt.toISOString() } - : { error, finishedAt: finishedAt.toISOString() }, - ); + } + } + + /** Bounded, database-fenced acquisition; no arbitrary handler is dispatched. */ + async reconcile(options: { limit?: number } = {}, signal?: AbortSignal): Promise { + const limit = options.limit ?? 32; + if (!Number.isInteger(limit) || limit < 1 || limit > 32) + throw new Error("Invalid reconciliation limit"); + const due = await db + .select() + .from(adminJob) + .where( + and( + inArray(adminJob.status, ["running", "cancel_pending"]), + inArray(adminJob.type, [...AGENT_RECOVERABLE_ADMIN_JOB_TYPES]), + sql`(${adminJob.status} = 'cancel_pending' OR ${adminJob.result}->'observation'->>'state' = 'unknown')`, + sql`coalesce(${adminJob.result}->'observation'->>'nextCheckAt', '') <= ${new Date().toISOString()}`, + sql`coalesce(${adminJob.result}->>'_observationLeaseUntil', '') <= ${new Date().toISOString()}`, + ), + ) + .orderBy(asc(adminJob.createdAt)) + .limit(limit); + for (const job of due) { + if (signal?.aborted) break; + if (this.active.has(job.id)) continue; + const claimed = await claimAdminJob(job, {}, true); + if (claimed) await this.launchJob(claimed, true, signal); } } } diff --git a/apps/api/src/services/ops-client.ts b/apps/api/src/services/ops-client.ts index 81354f2e5..058939244 100644 --- a/apps/api/src/services/ops-client.ts +++ b/apps/api/src/services/ops-client.ts @@ -57,7 +57,12 @@ export class ApiOpsError extends Error { export class OpsObservationPending extends Error { constructor( readonly identity: { kind: OpsOperationKind; operationKey: string; operationId?: string }, - readonly reason: "transport" | "observer_deadline" | "recovery_required", + readonly reason: + | "transport" + | "observer_deadline" + | "recovery_required" + | "unavailable" + | "invalid_identity", ) { super("Operations observation is pending"); this.name = "OpsObservationPending"; @@ -310,6 +315,7 @@ export interface WaitForOpsResultOptions { signal?: AbortSignal; timeoutMs?: number; observerDeadlineAt?: number; + onStatus?: (status: OpsJobStatusFor) => void | Promise; execution?: { requestedDeadlineAt: number }; parentAssertion?: string; pollIntervalMs?: number; @@ -338,7 +344,7 @@ export async function waitForOpsResult( operationKey: admission.operationKey, operationId: admission.operationId, }; - const pending = (reason: "transport" | "observer_deadline" | "recovery_required"): never => { + const pending = (reason: OpsObservationPending["reason"]): never => { throw new OpsObservationPending(identity, reason); }; const observer = new AbortController(); @@ -374,15 +380,18 @@ export async function waitForOpsResult( } if (isTransientTransportError(error)) pending(observerExpired() ? "observer_deadline" : "transport"); - translateError(error); + return pending("unavailable"); } if (observerExpired()) pending("observer_deadline"); if ( + status.kind !== kind || + status.operationId !== admission.operationId || status.operationKey !== admission.operationKey || (admission.fingerprint !== undefined && status.fingerprint !== admission.fingerprint) || (admission.deadlineAt !== undefined && status.deadlineAt !== admission.deadlineAt) ) - throw new ApiOpsError("runtime"); + pending("invalid_identity"); + await options.onStatus?.(status); if (status.state === "succeeded") return status.result; if (status.state === "failed" || status.state === "timed_out") { if (status.errorClass === "recovery_required") pending("recovery_required"); @@ -593,6 +602,11 @@ export async function submitOpsOperationWithRecovery signal: recoverySignal, execution: options.execution, }); + if (status.kind !== operation.kind || status.operationKey !== operationKey) + throw new OpsObservationPending( + { kind: operation.kind, operationKey }, + "invalid_identity", + ); return { execution: "async", version: status.version, @@ -609,9 +623,10 @@ export async function submitOpsOperationWithRecovery : {}), } as OpsSubmitResult; } catch (lookupError) { + if (lookupError instanceof OpsObservationPending) throw lookupError; const lookupClass = errorClass(lookupError); if (lookupClass !== "not_found" && !isTransientTransportError(lookupError)) { - translateError(lookupError); + throw new OpsObservationPending({ kind: operation.kind, operationKey }, "unavailable"); } if (recoverySignal.aborted) throw new OpsObservationPending({ kind: operation.kind, operationKey }, "transport"); diff --git a/apps/api/src/services/system-maintenance.ts b/apps/api/src/services/system-maintenance.ts index c2924b765..f98701dda 100644 --- a/apps/api/src/services/system-maintenance.ts +++ b/apps/api/src/services/system-maintenance.ts @@ -69,6 +69,13 @@ export async function handleSystemUpdateJob( await replaceAgentIfNeeded(ctx, payload.releaseId, payload.expectedDigest, dependencies); await ctx.setProgress(60); const createBackup = payload.createBackup !== false; + await ctx.checkpoint({ + ...ctx.checkpointResult, + _completion: { + operationKey: createDurableOpsKey("admin-job.system.update", ctx.jobId), + result: { operation: "apply", release: payload.releaseId, phase: "complete" }, + }, + }); const result = await executeAdminJobOperation( ctx, { diff --git a/apps/api/src/services/trusted-config-operations.ts b/apps/api/src/services/trusted-config-operations.ts index 62528801e..ac9cf1e60 100644 --- a/apps/api/src/services/trusted-config-operations.ts +++ b/apps/api/src/services/trusted-config-operations.ts @@ -1,4 +1,5 @@ import { + configurationApplyResultSchema, configurationGenerationRevisionSchema, type OpsOperation, type OpsResultFor, @@ -10,7 +11,14 @@ import { envString } from "@openmapx/core/server-env"; import { db } from "../db"; import { integrationConfig } from "../db/schema"; import { getAllIntegrations } from "../integration-host"; +import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; +import { + executeAdminJobOperation, + hasRetainedAdminJobOperation, + observeRetainedAdminJobOperation, +} from "./admin-job-ops"; import { projectIntegrationConfig } from "./integration-config-projection"; +import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, @@ -43,6 +51,7 @@ export interface ApplyTrustedConfigurationOptions { integrationId?: string; selectedRoots?: string[]; signal?: AbortSignal; + ctx?: JobContext; } function exactInteger(env: NodeJS.ProcessEnv, name: string): number { @@ -110,6 +119,7 @@ export async function applyTrustedConfiguration( options: ApplyTrustedConfigurationOptions, ): Promise<{ revisionId: string; enabledServiceIds: string[] }> { const { outcome } = await applyTrustedConfigurationWithEvidence(options); + if (options.ctx && outcome.outcome === "unknown") throw new JobObservationPending("transport"); if ( outcome.outcome !== "committed" || outcome.followUp !== "complete" || @@ -117,7 +127,9 @@ export async function applyTrustedConfiguration( outcome.publisherCleanupFailed || outcome.registryUpdateFailed ) - throw new TrustedConfigurationApplyError(outcome); + if (options.ctx) + throw new AdminJobFailure("Configuration did not complete", { configuration: outcome }); + else throw new TrustedConfigurationApplyError(outcome); return { revisionId: outcome.revisionId, enabledServiceIds: outcome.enabledServiceIds }; } @@ -129,6 +141,21 @@ export async function applyTrustedConfigurationWithEvidence( previousApplied?: OpsResultFor<"configuration.inspect">; outcome: TrustedConfigurationApplyOutcome; }> { + if (options.ctx && hasRetainedAdminJobOperation(options.ctx, options.operationKey)) { + const outcome = configurationApplyResultSchema.parse( + await observeRetainedAdminJobOperation(options.ctx, options.operationKey), + ); + const retained = ( + options.ctx.checkpointResult?._configurationOutcomes as + | Record + | undefined + )?.[options.operationKey]; + if (!retained) { + await options.ctx.checkpoint({ ...options.ctx.checkpointResult, configuration: outcome }); + throw new JobObservationPending("recovery_required"); + } + return { outcome: retained }; + } let payload: TrustedConfigurationPayload; let previousApplied: OpsResultFor<"configuration.inspect">; try { @@ -242,6 +269,10 @@ async function applySnapshot( const consumed = await consumePublishedTrustedConfiguration(sealed, () => { const client = createApiOpsClient(); submitted = true; + if (options.ctx) + return executeAdminJobOperation(options.ctx, sealed.operation, "admin-job.configuration", { + operationKey: options.operationKey, + }).then((result) => configurationApplyResultSchema.parse(result)); return executeTrustedConfigurationAndWait( client, sealed.operation as Extract, @@ -249,6 +280,7 @@ async function applySnapshot( { signal: options.signal }, ); }); + if (options.ctx && consumed.submission.status === "rejected") throw consumed.submission.reason; const result = consumed.submission.status === "fulfilled" ? consumed.submission.value @@ -263,8 +295,24 @@ async function applySnapshot( registryUpdateFailed = true; } } - return { ...result, publisherCleanupFailed: consumed.cleanupFailed, registryUpdateFailed }; - } catch { + const outcome = { + ...result, + publisherCleanupFailed: consumed.cleanupFailed, + registryUpdateFailed, + }; + if (options.ctx) + await options.ctx.checkpoint({ + ...options.ctx.checkpointResult, + configuration: outcome, + _configurationOutcomes: { + ...((options.ctx.checkpointResult?._configurationOutcomes as Record) ?? + {}), + [options.operationKey]: outcome, + }, + }); + return outcome; + } catch (error) { + if (options.ctx && submitted) throw error; // Publication/preparation failed before execute, or an unexpected error after submission. return { ...(submitted diff --git a/apps/ops-agent/src/journal.test.ts b/apps/ops-agent/src/journal.test.ts index 8f53c17ce..388e07e00 100644 --- a/apps/ops-agent/src/journal.test.ts +++ b/apps/ops-agent/src/journal.test.ts @@ -279,3 +279,59 @@ describe("version2 execution journal", () => { ).rejects.toThrow(); }); }); + +it("reloads strict v2 failed evidence and rejects it on a successful record", async () => { + const { normalizeExecutionAdmission } = await import("@openmapx/core/ops"); + const operation = { kind: "service.build", serviceId: "valhalla" } as const; + const operationKey = "opk1_retainedFailure123"; + const operationFingerprint = opsOperationFingerprint(operation); + const admittedAt = Date.parse("2026-10-23T12:00:00.000Z"); + const normalized = normalizeExecutionAdmission({ + kind: operation.kind, + operationKey, + operationFingerprint, + admittedAt, + }); + const evidence = { + serviceBatch: { + outcome: "partial" as const, + completedServiceIds: ["tileserver"], + failedServiceIds: ["valhalla"], + skippedServiceIds: [], + }, + diagnostics: { + truncated: false, + tail: [{ stream: "stderr" as const, message: "retained diagnostic" }], + }, + }; + const record = job({ + operation, + operationKey, + operationFingerprint, + resourceId: "valhalla", + submittedAt: new Date(admittedAt).toISOString(), + ...normalized, + executionVersion: 2, + requestVersion: 2, + deploymentBudgetMs: 12 * 3600_000, + state: "failed", + errorClass: "runtime", + terminalAt: "2026-10-23T12:00:01.000Z", + failureResult: evidence, + diagnostics: evidence.diagnostics, + }); + const path = join(root(), "journal", "jobs-v1.json"); + const journal = await openOpsJobJournal(path); + await journal.replace([record]); + const reopened = await openOpsJobJournal(path); + expect(reopened.records()[0]).toMatchObject({ + state: "failed", + failureResult: evidence, + diagnostics: evidence.diagnostics, + }); + await expect( + reopened.replace([ + { ...record, state: "succeeded", errorClass: undefined, result: { completed: true } }, + ]), + ).rejects.toThrow(); +}); diff --git a/apps/web/src/components/admin/activity/JobDetail.tsx b/apps/web/src/components/admin/activity/JobDetail.tsx index a56156a3c..baeb94990 100644 --- a/apps/web/src/components/admin/activity/JobDetail.tsx +++ b/apps/web/src/components/admin/activity/JobDetail.tsx @@ -9,7 +9,10 @@ import LinearProgress from "@mui/material/LinearProgress"; import Stack from "@mui/material/Stack"; import Typography from "@mui/material/Typography"; import { cancelAdminActivityJob, unwrapAdminResponse } from "@openmapx/core/api-admin-sdk"; -import type { AdminActivityJobSource } from "@openmapx/core/api-contracts"; +import { + type AdminActivityJobSource, + adminJobObservationSchema, +} from "@openmapx/core/api-contracts"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useEffect, useRef } from "react"; import { DataManagerJobStages } from "../shared/DataManagerJobStages"; @@ -126,6 +129,27 @@ export function JobDetail({ jobId, source }: { jobId: string; source: AdminActiv ); } + const result = + data.result && typeof data.result === "object" && !Array.isArray(data.result) + ? data.result + : {}; + const observation = adminJobObservationSchema.safeParse(result.observation); + const diagnostics = + result.diagnostics && + typeof result.diagnostics === "object" && + !Array.isArray(result.diagnostics) + ? result.diagnostics + : null; + const tail = Array.isArray(diagnostics?.tail) + ? diagnostics.tail.filter( + (entry): entry is { stream: string; message: string } => + !!entry && + typeof entry === "object" && + !Array.isArray(entry) && + typeof entry.message === "string" && + typeof entry.stream === "string", + ) + : []; return ( )} + {observation.success && ( + + Outcome unknown; checking retained operation status. + + )} + {tail.length > 0 && ( + + + Retained diagnostics + + + {tail.map((entry) => entry.message).join("\n")} + + + )} {data.error && ( { ); }); }); + +it("shows unknown observation and independent retained diagnostics while cancellation is pending", async () => { + vi.stubGlobal("fetch", async () => + Response.json( + job({ + status: "cancel_pending", + cancelable: false, + result: { + observation: { state: "unknown", reason: "transport", nextCheckAt: timestamp }, + serviceBatch: { + outcome: "partial", + completedServiceIds: ["tileserver"], + failedServiceIds: ["valhalla"], + skippedServiceIds: [], + }, + diagnostics: { + truncated: true, + tail: [{ stream: "stderr", message: "retained final diagnostic" }], + }, + }, + logs: [], + }), + ), + ); + mount(); + expect( + await screen.findByText("Outcome unknown; checking retained operation status."), + ).toBeInTheDocument(); + expect(screen.getByText("Retained diagnostics")).toBeInTheDocument(); + expect(screen.getByText("retained final diagnostic")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Cancel" })).not.toBeInTheDocument(); +}); diff --git a/packages/core/src/api/contracts/activity-admin.ts b/packages/core/src/api/contracts/activity-admin.ts index 822d61b9c..1b8060542 100644 --- a/packages/core/src/api/contracts/activity-admin.ts +++ b/packages/core/src/api/contracts/activity-admin.ts @@ -1,4 +1,21 @@ import z from "zod/v4"; +import { opsFailureResultSchema, processDiagnosticsSchema } from "../../ops/job-evidence"; +export const adminJobObservationSchema = z.strictObject({ + state: z.literal("unknown"), + reason: z.enum([ + "transport", + "observer_deadline", + "recovery_required", + "unavailable", + "invalid_identity", + ]), + nextCheckAt: z.iso.datetime(), +}); +export const adminJobEvidenceSchema = opsFailureResultSchema.safeExtend({ + observation: adminJobObservationSchema.optional(), + diagnostics: processDiagnosticsSchema.optional(), +}); + import { type AdminOperation, adminCommonResponses, @@ -78,8 +95,23 @@ export const adminActivityJobStageSchema = z.strictObject({ error: adminJsonValueSchema, artifacts: adminJsonValueSchema, }); +// Historical JSON results remain supported; known evidence members are strict. +export const adminActivityJobResultSchema = adminJsonValueSchema.superRefine((value, ctx) => { + if (!value || typeof value !== "object" || Array.isArray(value)) return; + if (Object.keys(value).some((key) => key.startsWith("_") || key === "opsProjection")) + ctx.addIssue({ code: "custom", message: "Internal administrative job state" }); + const evidence = Object.fromEntries( + Object.entries(value).filter(([key]) => + ["serviceBatch", "dataWorkflow", "serviceLifecycle", "diagnostics", "observation"].includes( + key, + ), + ), + ); + if (!adminJobEvidenceSchema.safeParse(evidence).success) + ctx.addIssue({ code: "custom", message: "Invalid administrative job evidence" }); +}); const detail = { - result: adminJsonValueSchema, + result: adminActivityJobResultSchema, logs: z.array(adminActivityJobLogSchema), stages: z.array(adminActivityJobStageSchema), }; @@ -118,7 +150,7 @@ export const adminJobStreamEventSchemas = { status: adminApplicationJobStatusSchema, progress: z.number().finite().nullable().optional(), error: z.string().nullable().optional(), - result: adminJsonValueSchema.optional(), + result: adminActivityJobResultSchema.optional(), startedAt: adminTimestampSchema.nullable().optional(), finishedAt: adminTimestampSchema.nullable().optional(), }), diff --git a/packages/core/src/ops/job-evidence.ts b/packages/core/src/ops/job-evidence.ts index bd2573fd6..2b5e0eedf 100644 --- a/packages/core/src/ops/job-evidence.ts +++ b/packages/core/src/ops/job-evidence.ts @@ -8,13 +8,13 @@ export const processDiagnosticsSchema = z .array( z.strictObject({ stream: z.enum(["stdout", "stderr"]), - message: z.string().refine((x) => Buffer.byteLength(x) <= 4096), + message: z.string().refine((x) => new TextEncoder().encode(x).byteLength <= 4096), }), ) .max(256), truncated: z.boolean(), }) - .refine((x) => Buffer.byteLength(JSON.stringify(x)) <= 16 * 1024); + .refine((x) => new TextEncoder().encode(JSON.stringify(x)).byteLength <= 16 * 1024); export type ProcessDiagnostics = z.infer; export const dataWorkflowReceiptSchema = z.strictObject({ version: z.literal(1), @@ -56,5 +56,5 @@ export const opsFailureResultSchema = z diagnostics: processDiagnosticsSchema.optional(), serviceLifecycle: serviceLifecycleFailureEvidenceSchema.optional(), }) - .refine((x) => Buffer.byteLength(JSON.stringify(x)) <= 64 * 1024); + .refine((x) => new TextEncoder().encode(JSON.stringify(x)).byteLength <= 64 * 1024); export type OpsFailureResult = z.infer; From 8705502c42d5e0ae3d31d294caa68993d96d4a73 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 22:45:58 +0200 Subject: [PATCH 009/135] fix(admin): fence observers and recover abandoned jobs --- .../services/__tests__/admin-job-ops.test.ts | 74 ++- .../__tests__/admin-ops-hardlinks.test.ts | 91 +++- .../__tests__/job-runner-outcomes.test.ts | 6 +- .../__tests__/job-runner-postgres.test.ts | 428 ++++++++++++++++++ apps/api/src/services/admin-job-ops.ts | 101 +++-- apps/api/src/services/admin-job-state.ts | 53 ++- apps/api/src/services/admin-ops.ts | 91 +++- apps/api/src/services/job-runner.ts | 231 ++++++---- 8 files changed, 941 insertions(+), 134 deletions(-) create mode 100644 apps/api/src/services/__tests__/job-runner-postgres.test.ts diff --git a/apps/api/src/services/__tests__/admin-job-ops.test.ts b/apps/api/src/services/__tests__/admin-job-ops.test.ts index f02911c0b..b01724138 100644 --- a/apps/api/src/services/__tests__/admin-job-ops.test.ts +++ b/apps/api/src/services/__tests__/admin-job-ops.test.ts @@ -38,7 +38,7 @@ describe("administrative job operations broker", () => { expect(execute.mock.calls[0]?.[0]).toEqual(operation); expect(execute.mock.calls[0]?.[1]).toEqual({ operationKey: expect.stringMatching(/^opk1_[A-Za-z0-9_-]+$/), - signal: ctx.signal, + signal: expect.any(AbortSignal), }); }); @@ -823,3 +823,75 @@ it("does not read or cancel an attempted child belonging to another API job", as expect(client.events).not.toHaveBeenCalled(); expect(client.cancel).not.toHaveBeenCalled(); }); + +it.each(["log", "checkpoint", "output"])( + "retires ownership loss but tolerates ordinary %s projection failure without backend cancel", + async (edge) => { + const { JobOwnershipLost } = await import("../admin-job-state"); + const ctx = context(); + const entered = Promise.withResolvers(); + const resume = Promise.withResolvers(); + let writes = 0; + ctx.checkpoint = async () => { + writes++; + if (edge === "checkpoint" && writes >= 3) { + entered.resolve(); + await resume.promise; + throw new JobOwnershipLost(); + } + }; + if (edge === "log" || edge === "output") + ctx.log = async () => { + entered.resolve(); + await resume.promise; + throw edge === "log" ? new JobOwnershipLost() : new Error("output unavailable"); + }; + const key = "opk1_displacedObserver1"; + const id = "job1_displacedObserver1"; + const client = { + execute: vi.fn().mockResolvedValue({ + execution: "async", + kind: "backup.delete", + operationKey: key, + operationId: id, + state: "queued", + }), + status: vi.fn(async () => { + await resume.promise; + return { + kind: "backup.delete", + operationId: id, + operationKey: key, + state: "succeeded", + result: { backupId: "nightly" }, + }; + }), + events: vi.fn().mockResolvedValue({ + version: 1, + operationId: id, + terminal: true, + truncated: false, + nextCursor: 1, + events: [{ cursor: 1, type: "log", stream: "stdout", message: "old owner" }], + }), + cancel: vi.fn(), + lookup: vi.fn(), + }; + const result = executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "nightly" }, + "admin-job.backup.delete", + { client: client as never, operationKey: key }, + ); + const outcome = result.then( + () => "fulfilled", + (error) => error, + ); + await entered.promise; + resume.resolve(); + if (edge === "output") expect(await outcome).toBe("fulfilled"); + else expect(await outcome).toBeInstanceOf(JobOwnershipLost); + expect(client.cancel).not.toHaveBeenCalled(); + expect(client.execute).toHaveBeenCalledTimes(1); + }, +); diff --git a/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts b/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts index ec14dbf2a..eefc6e3cd 100644 --- a/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts +++ b/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts @@ -16,6 +16,8 @@ const plan = [ const executeAndWait = vi.fn(); const dockerComposeAction = vi.fn(); const link = vi.fn(); +const getStatus = vi.fn(); +const broker = vi.fn(); const registryGet = vi.fn(); vi.mock("../ops-client.js", () => ({ @@ -29,7 +31,7 @@ vi.mock("../../utils/docker-compose.js", () => ({ })); vi.mock("@openmapx/core/data-manager-sdk", () => ({ createDataManagerTransport: () => ({}), - getStatus: vi.fn(async () => ({ data: {}, status: 200 })), + getStatus: (...args: unknown[]) => getStatus(...args), link: (...args: unknown[]) => link(...args), unwrapDataManagerResponse: (response: unknown) => response, })); @@ -37,7 +39,15 @@ vi.mock("../service-registry.js", () => ({ getServiceRegistry: () => ({ get: registryGet }), })); -const { applyGenerationHardlinks } = await import("../admin-ops.js"); +vi.mock("../admin-job-ops", () => ({ + executeAdminJobOperation: (...args: unknown[]) => broker(...args), +})); + +vi.mock("../trusted-config-operations", () => ({ + applyTrustedConfiguration: async () => ({ revisionId: REVISION }), +})); + +const { applyGenerationHardlinks, serviceStart } = await import("../admin-ops.js"); beforeEach(() => { vi.clearAllMocks(); @@ -45,6 +55,8 @@ beforeEach(() => { dockerComposeAction.mockResolvedValue({ exitCode: 0, stdout: "", stderr: "" }); link.mockResolvedValue({ linked: 1, skipped: 0, pruned: 0 }); registryGet.mockReturnValue({ enabled: true }); + getStatus.mockResolvedValue({ data: {}, status: 200 }); + broker.mockResolvedValue({ changed: true }); }); describe("applyGenerationHardlinks", () => { @@ -84,3 +96,78 @@ describe("applyGenerationHardlinks", () => { expect(link).not.toHaveBeenCalled(); }); }); + +it("does not link or start a later lifecycle effect after canceled readiness resolves", async () => { + const ready = Promise.withResolvers(); + getStatus.mockReturnValue(ready.promise); + const abort = new AbortController(); + const ctx = { + jobId: "fix1-service", + payload: {}, + signal: abort.signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn(), + checkpointResult: {}, + }; + const work = serviceStart("valhalla", ctx); + const outcome = work.then( + () => "fulfilled", + (error) => error, + ); + await vi.waitFor(() => expect(getStatus).toHaveBeenCalled()); + abort.abort(); + ready.resolve({ data: {}, status: 200 }); + expect(await outcome).toBeInstanceOf(Error); + expect(link).not.toHaveBeenCalled(); + expect(broker.mock.calls.map((call) => call[1])).toEqual([ + { kind: "service.start", serviceId: "data-manager" }, + ]); +}); + +it("fences the link attempt against current ownership after readiness", async () => { + const { JobOwnershipLost } = await import("../admin-job-state"); + const ctx = { + jobId: "fix1-owner", + payload: {}, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn().mockRejectedValue(new JobOwnershipLost()), + checkpointResult: {}, + }; + await expect( + applyGenerationHardlinks({ + revisionId: REVISION, + operationIdentity: "fix1-owner:valhalla", + ctx, + }), + ).rejects.toBeInstanceOf(JobOwnershipLost); + expect(link).not.toHaveBeenCalled(); +}); + +it("retains an unknown link attempt and refuses any coordinator/link replay", async () => { + const { JobObservationPending } = await import("../admin-job-failure"); + const ctx: import("../job-runner").JobContext = { + jobId: "fix1-link", + payload: {}, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: async (value) => { + ctx.checkpointResult = structuredClone(value); + }, + checkpointResult: {}, + }; + link.mockImplementation(async () => { + expect(ctx.checkpointResult).toMatchObject({ + _hardlinkAttempts: { "fix1-link:valhalla": { revisionId: REVISION, phase: "attempted" } }, + }); + throw new Error("response lost after remote acceptance"); + }); + const options = { revisionId: REVISION, operationIdentity: "fix1-link:valhalla", ctx }; + await expect(applyGenerationHardlinks(options)).rejects.toBeInstanceOf(JobObservationPending); + await expect(applyGenerationHardlinks(options)).rejects.toBeInstanceOf(JobObservationPending); + expect(link).toHaveBeenCalledTimes(1); + expect(broker).toHaveBeenCalledTimes(1); +}); diff --git a/apps/api/src/services/__tests__/job-runner-outcomes.test.ts b/apps/api/src/services/__tests__/job-runner-outcomes.test.ts index 5fe176c15..6b8409040 100644 --- a/apps/api/src/services/__tests__/job-runner-outcomes.test.ts +++ b/apps/api/src/services/__tests__/job-runner-outcomes.test.ts @@ -142,7 +142,11 @@ it("rejects stale checkpoint writes before another child can be admitted", async }); await run("test.checkpoint"); expect(effect).not.toHaveBeenCalled(); - expect(store.writes[0]?.values.result).toMatchObject({ _execution: execution }); + expect( + JSON.parse( + new PgDialect().sqlToQuery(store.writes[0]?.values.result as SQL).params[0] as string, + ), + ).toMatchObject({ _execution: execution }); }); it("changes owner and version together while retaining the complete result in PostgreSQL", async () => { diff --git a/apps/api/src/services/__tests__/job-runner-postgres.test.ts b/apps/api/src/services/__tests__/job-runner-postgres.test.ts new file mode 100644 index 000000000..dec393746 --- /dev/null +++ b/apps/api/src/services/__tests__/job-runner-postgres.test.ts @@ -0,0 +1,428 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { eq, inArray } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import * as schema from "../../db/schema"; +import type { JobContext } from "../job-runner"; +import type { ApiOpsClient } from "../ops-client"; + +const state = vi.hoisted(() => ({ + db: null as unknown as ReturnType>, + client: null as unknown as ApiOpsClient, +})); +vi.mock("../../db", () => ({ + get db() { + return state.db; + }, +})); +vi.mock("../ops-client", async (original) => ({ + ...(await original()), + createApiOpsClient: () => state.client, +})); +const { jobRunner } = await import("../job-runner"); +const { claimAdminJob, renewAdminJobClaim, readJobExecution, JobOwnershipLost } = await import( + "../admin-job-state" +); +const { executeAdminJobOperation } = await import("../admin-job-ops"); +const { jobEventBus } = await import("../job-events"); +const { OpsClientError } = await import("@openmapx/core/ops"); +const { adminJob, adminJobLog } = schema; +const runner = jobRunner as unknown as { + executeJob(...args: unknown[]): Promise; + launchJob(...args: unknown[]): Promise; +}; + +// Explicit opt-in connection supplied only after the task launcher verifies the +// synthetic container's exact ID/owner. No migrations or table-wide cleanup. +describe.skipIf(!process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL)( + "admin job PostgreSQL ownership/recovery", + () => { + let client: ReturnType; + let root: string; + const ids: string[] = []; + beforeAll(async () => { + const connectionUrl = process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL; + if (!connectionUrl) throw new Error("Explicit synthetic DB required"); + client = postgres(connectionUrl, { + max: 4, + connection: { statement_timeout: 5000 }, + }); + state.db = drizzle(client, { schema }); + const active = await state.db + .select() + .from(adminJob) + .where(inArray(adminJob.status, ["queued", "running", "cancel_pending"])); + expect(active).toEqual([]); // Never adopt another fixture's work. + root = await mkdtemp(join(tmpdir(), "task022-fix1-")); + const tokenFile = join(root, "api-token"); + await writeFile(tokenFile, Buffer.alloc(32, 7).toString("base64url"), { mode: 0o600 }); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); + }); + afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); + if (ids.length) await state.db.delete(adminJob).where(inArray(adminJob.id, ids.splice(0))); + }); + afterAll(async () => { + await client?.end({ timeout: 2 }); + if (root) await rm(root, { recursive: true }); + vi.unstubAllEnvs(); + }); + async function seed( + type = "backup.operation", + result: Record = {}, + status = "running", + ) { + const id = randomUUID(); + ids.push(id); + const execution = { owner: randomUUID(), version: 1 }; + const [job] = await state.db + .insert(adminJob) + .values({ id, type, status, payload: {}, result: { ...result, _execution: execution } }) + .returning(); + return job; + } + const row = async (id: string) => + (await state.db.select().from(adminJob).where(eq(adminJob.id, id)))[0]; + const run = (job: typeof adminJob.$inferSelect, observe = false) => + runner.executeJob(job.id, job.type, {}, job.result, new AbortController().signal, observe); + + it("rejects a resumed old owner's log before insert/publication and stops forward work", async () => { + const job = await seed("test.stale-log"); + const ready = Promise.withResolvers(); + const resume = Promise.withResolvers(); + const after = vi.fn(); + jobRunner.register(job.type, async (ctx) => { + ready.resolve(ctx); + await resume.promise; + await ctx.log("stale"); + after(); + return {}; + }); + const logs: unknown[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "log") logs.push(event); + }); + const running = run(job); + const ctx = await ready.promise; + await claimAdminJob(job); + resume.resolve(); + await running; + stop(); + expect( + await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), + ).toEqual([]); + expect(logs).toEqual([]); + expect(after).not.toHaveBeenCalled(); + expect(ctx.signal.aborted).toBe(true); + }); + + it("publishes a source event once when its atomic insert is deduplicated", async () => { + const job = await seed("test.duplicate-log"); + jobRunner.register(job.type, async (ctx) => { + await ctx.log("once", "stdout", `${job.id}:event`, 1); + await ctx.log("once", "stdout", `${job.id}:event`, 1); + return {}; + }); + const logs: unknown[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "log") logs.push(event); + }); + await run(job); + stop(); + expect( + await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), + ).toHaveLength(1); + expect(logs).toHaveLength(1); + }); + + it("rechecks ownership after waiting behind a concurrent replacement's row lock", async () => { + const job = await seed("test.blocked-log"); + const ready = Promise.withResolvers(); + const resume = Promise.withResolvers(); + jobRunner.register(job.type, async (ctx) => { + ready.resolve(ctx); + await resume.promise; + await ctx.log("blocked stale log"); + return {}; + }); + const running = run(job); + await ready.promise; + const locked = Promise.withResolvers(); + const commit = Promise.withResolvers(); + const replacement = client.begin(async (tx) => { + await tx`UPDATE admin_job SET result = result || ${JSON.stringify({ _execution: { owner: randomUUID(), version: 2 } })}::jsonb WHERE id = ${job.id}`; + const [backend] = await tx`SELECT pg_backend_pid() AS pid`; + locked.resolve(backend.pid); + await commit.promise; + }); + const pid = await locked.promise; + try { + resume.resolve(); + await vi.waitFor( + async () => { + const [waiting] = + await client`SELECT count(*)::int AS n FROM pg_stat_activity WHERE ${pid} = ANY(pg_blocking_pids(pid))`; + expect(waiting.n).toBe(1); + }, + { timeout: 2000, interval: 10 }, + ); + } finally { + commit.resolve(); + await replacement; + await running; + } + expect( + await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), + ).toEqual([]); + }); + + it("preserves a renewed live lease through checkpoints and rejects a stale due-row claim", async () => { + const job = await seed("test.live-lease"); + const claimed = await claimAdminJob(job); + const ready = Promise.withResolvers(); + const resume = Promise.withResolvers(); + jobRunner.register(job.type, async (ctx) => { + ready.resolve(ctx); + await resume.promise; + return {}; + }); + if (!claimed) throw new Error("Missing claim"); + const work = run(claimed); + const ctx = await ready.promise; + vi.useFakeTimers({ toFake: ["Date"] }); + const start = Date.now(); + try { + vi.setSystemTime(start + 40_000); + await renewAdminJobClaim(job.id, ctx.execution); + const live = (await row(job.id)).result as Record; + await ctx.checkpoint({ phase: "still working" }); + expect((await row(job.id)).result).toHaveProperty( + "_observationLeaseUntil", + live._observationLeaseUntil, + ); + vi.setSystemTime(start + 65_000); // Original snapshot expired, renewed owner is live. + expect(await claimAdminJob(claimed, {}, true)).toBeUndefined(); + expect(readJobExecution((await row(job.id)).result)).toEqual(ctx.execution); + vi.setSystemTime(start + 105_000); + await expect(renewAdminJobClaim(job.id, ctx.execution)).rejects.toBeInstanceOf( + JobOwnershipLost, + ); + } finally { + resume.resolve(); + await work; + } + }); + + it.each([ + ["pending", false], + ["finalization", false], + ["pending", true], + ["finalization", true], + ] as const)( + "recovers a %s write outage without restart or another POST (backendFailed=%s)", + async (failure, backendFailed) => { + const job = await seed(); + const claimed = await claimAdminJob(job); + let offline = false; + const update = state.db.update.bind(state.db); + vi.spyOn(state.db, "update").mockImplementation((...args) => { + if (offline) throw new Error("synthetic database unavailable"); + return update(...args); + }); + const key = "opk1_fixround1recover12"; + const opId = "job1_fixround1recover12"; + const terminal = { + version: 1, + operationId: opId, + operationKey: key, + kind: "backup.delete", + resourceId: "nightly", + ...(backendFailed + ? { + state: "failed", + errorClass: "runtime", + failureResult: { + diagnostics: { + truncated: false, + tail: [{ stream: "stderr", message: "retained failure" }], + }, + }, + } + : { state: "succeeded", result: { backupId: "nightly" } }), + submittedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; + const execute = vi.fn().mockResolvedValue({ + execution: "async", + kind: "backup.delete", + operationId: opId, + operationKey: key, + state: "queued", + }); + const status = vi.fn(async () => { + offline = true; + if (failure === "pending") throw new OpsClientError("authorization"); + return terminal; + }); + state.client = { + execute, + status, + events: vi.fn().mockResolvedValue({ + version: 1, + operationId: opId, + events: [], + nextCursor: 0, + terminal: true, + truncated: false, + }), + cancel: vi.fn(), + lookup: vi.fn().mockResolvedValue(terminal), + } as unknown as ApiOpsClient; + jobRunner.register(job.type, async (ctx) => { + await executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "nightly" }, + "admin-job.backup.delete", + { client: state.client, operationKey: key }, + ); + return { backupId: "nightly" }; + }); + const outcomes: string[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "status") outcomes.push(event.status); + }); + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); + await runner.launchJob(claimed); + offline = false; + expect((await row(job.id)).status).toBe("running"); + expect(outcomes).not.toContain("failed"); + expect(outcomes).not.toContain("canceled"); + status.mockResolvedValue(terminal); + // The abandoned claim must become due without initialize() or an unknown write. + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 65_000); + await jobRunner.reconcile(); + expect((await row(job.id)).status).toBe(backendFailed ? "failed" : "success"); + if (backendFailed) + expect((await row(job.id)).result).toHaveProperty( + "diagnostics.tail.0.message", + "retained failure", + ); + expect(execute).toHaveBeenCalledTimes(1); + expect(state.client.cancel).not.toHaveBeenCalled(); + if (!backendFailed) expect(outcomes).not.toContain("failed"); + expect(outcomes).not.toContain("canceled"); + stop(); + errors.mockRestore(); + }, + ); + + it("never converts repeated success-finalization outages into a handler failure", async () => { + const job = await seed(); + const claimed = await claimAdminJob(job); + let failures = 0; + const update = state.db.update.bind(state.db); + vi.spyOn(state.db, "update").mockImplementation((...args) => { + if (failures-- > 0) throw new Error("synthetic DB outage"); + return update(...args); + }); + jobRunner.register(job.type, async () => { + failures = 2; + return { completed: true }; + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + await runner.launchJob(claimed); + expect((await row(job.id)).status).toBe("running"); + }); + + it.each([ + [false, false], + [true, false], + [false, true], + [true, true], + ])( + "settles cancellation only with containment (reconcile=%s, unknownLink=%s)", + async (reconcile, unknownLink) => { + const key = "opk1_fixround1cancel123"; + const operationId = "job1_fixround1cancel123"; + const diagnostics = { truncated: false, tail: [{ stream: "stderr", message: "retained" }] }; + const job = await seed( + "backup.operation", + { + ...(unknownLink + ? { + _hardlinkAttempts: { + synthetic: { revisionId: "cfg1_synthetic", phase: "attempted" }, + }, + } + : {}), + observation: { + state: "unknown", + reason: "transport", + nextCheckAt: new Date(0).toISOString(), + }, + diagnostics, + opsProjection: { + version: 1, + eventTotal: 0, + byteTotal: 0, + truncated: false, + operations: { + [key]: { + kind: "backup.delete", + operationKey: key, + operationId, + cursor: 0, + events: 0, + bytes: 0, + truncated: false, + terminal: false, + }, + }, + }, + }, + reconcile ? "cancel_pending" : "running", + ); + const status = { + version: 1, + kind: "backup.delete", + operationKey: key, + operationId, + state: "failed", + errorClass: "runtime", + terminationRequestedAt: new Date().toISOString(), + diagnostics, + }; + state.client = { + status: vi.fn().mockResolvedValue(status), + cancel: vi.fn(), + lookup: vi.fn(), + } as unknown as ApiOpsClient; + const events: unknown[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "status" && event.status === "canceled") events.push(event.result); + }); + if (reconcile) await jobRunner.reconcile(); + else await jobRunner.cancel(job.id); + stop(); + const finished = await row(job.id); + expect(finished.result).toHaveProperty("diagnostics", diagnostics); + if (unknownLink) { + expect(finished.status).toBe("cancel_pending"); + expect(finished.result).toHaveProperty("observation.state", "unknown"); + expect(events).toEqual([]); + } else { + expect(finished.status).toBe("canceled"); + expect(finished.result).not.toHaveProperty("observation"); + expect(events).toEqual([expect.objectContaining({ diagnostics })]); + expect(events[0]).not.toHaveProperty("observation"); + } + }, + ); + }, +); diff --git a/apps/api/src/services/admin-job-ops.ts b/apps/api/src/services/admin-job-ops.ts index 104363ab1..321d6bbe0 100644 --- a/apps/api/src/services/admin-job-ops.ts +++ b/apps/api/src/services/admin-job-ops.ts @@ -19,6 +19,7 @@ import { issueApiJobChildAssertion, prepareApiJobChildAttempt, } from "./admin-job-parent"; +import { JobOwnershipLost } from "./admin-job-state"; import type { JobContext } from "./job-runner"; import { type ApiOpsClient, @@ -267,6 +268,17 @@ async function observeOperation( options: ExecuteAdminJobOperationOptions, fresh = false, ): Promise { + context.signal.throwIfAborted(); + const observer = new AbortController(); + const signal = AbortSignal.any([context.signal, observer.signal]); + let ownershipLost: JobOwnershipLost | undefined; + const retireIfLost = (error: unknown) => { + if (error instanceof JobOwnershipLost) { + ownershipLost = error; + observer.abort(error); + throw error; + } + }; const identity = { kind: projected.kind, operationKey: projected.operationKey, @@ -285,8 +297,9 @@ async function observeOperation( const safeCheckpoint = async () => { try { await persistProjection(context, projection); - } catch { - /* Observation never cancels effects. */ + } catch (error) { + retireIfLost(error); + /* Ordinary projection failure never cancels effects. */ } }; try { @@ -296,7 +309,7 @@ async function observeOperation( // Authenticate and match its complete identity before opening the stream. const status = await client .status(projected.kind, projected.operationId, { - signal: AbortSignal.any([context.signal, AbortSignal.timeout(2_000)]), + signal: AbortSignal.any([signal, AbortSignal.timeout(2_000)]), }) .catch(() => { throw new OpsObservationPending( @@ -328,7 +341,7 @@ async function observeOperation( projected.operation, projected.operationKey, { - signal: context.signal, + signal, admissionAlreadyAttempted: !fresh, recoveryTimeoutMs: 2_000, ...(projected.attempt @@ -353,12 +366,13 @@ async function observeOperation( // If this checkpoint fails, the admitting intent still supports lookup-only recovery. try { await persistProjection(context, projection); - } catch { + } catch (error) { + retireIfLost(error); throw new OpsObservationPending(identity, "transport"); } } const mirrorController = new AbortController(); - const mirrorSignal = AbortSignal.any([context.signal, mirrorController.signal]); + const mirrorSignal = AbortSignal.any([signal, mirrorController.signal]); const mirror = projected.mirrorStopped ? Promise.resolve() : followOpsEvents(client, admission.operationId, { @@ -376,7 +390,8 @@ async function observeOperation( onLog: async (stream, message, cursor) => { try { await context.log(message, stream, `ops:${admission.operationId}:${cursor}`, cursor); - } catch { + } catch (error) { + retireIfLost(error); projected.truncated = true; projection.truncated = true; } @@ -397,32 +412,42 @@ async function observeOperation( await safeCheckpoint(); } }) - .catch(() => undefined); - try { - const result = await waitForOpsResult(client, projected.kind, admission, { - signal: context.signal, - pollIntervalMs: options.pollIntervalMs, - ...(context.observationOnly ? { timeoutMs: 2_000 } : {}), - onStatus: (status) => { - if (status.diagnostics) projected.diagnostics = status.diagnostics; - if (status.failureResult) projected.failureResult = status.failureResult; - }, - }); - projected.result = result; - const configuration = configurationApplyResultSchema.safeParse(result); - if (configuration.success) - context.checkpointResult = { - ...context.checkpointResult, - configuration: configuration.data, - }; - projected.terminal = true; - projected.phase = "terminal"; - delete projected.outcome; - return result; - } finally { - mirrorController.abort(); - await mirror; - } + .catch((error) => { + // Observe this rejection immediately; propagate after joining the + // result observer so it cannot become an unhandled rejection. + if (error instanceof JobOwnershipLost) { + ownershipLost = error; + observer.abort(error); + } + }); + const observed = await waitForOpsResult(client, projected.kind, admission, { + signal, + pollIntervalMs: options.pollIntervalMs, + ...(context.observationOnly ? { timeoutMs: 2_000 } : {}), + onStatus: (status) => { + if (status.diagnostics) projected.diagnostics = status.diagnostics; + if (status.failureResult) projected.failureResult = status.failureResult; + }, + }).then( + (result) => ({ ok: true as const, result }), + (error: unknown) => ({ ok: false as const, error }), + ); + mirrorController.abort(); + await mirror; + if (ownershipLost) throw ownershipLost; + if (!observed.ok) throw observed.error; + const result = observed.result; + projected.result = result; + const configuration = configurationApplyResultSchema.safeParse(result); + if (configuration.success) + context.checkpointResult = { + ...context.checkpointResult, + configuration: configuration.data, + }; + projected.terminal = true; + projected.phase = "terminal"; + delete projected.outcome; + return result; } catch (error) { if (error instanceof ApiOpsError) { projected.failureResult = error.failureResult; @@ -449,7 +474,7 @@ async function observeOperation( ...context.checkpointResult, diagnostics: projected.diagnostics, }; - await safeCheckpoint(); + if (!ownershipLost) await safeCheckpoint(); } } @@ -477,6 +502,7 @@ export async function resumeAdminJobObservation( } } const evidence = { ...ctx.checkpointResult }; + if (hasUnsettledHardlinks(evidence)) throw new JobObservationPending("recovery_required"); if (failure) throw new AdminJobFailure(failure.message, { ...evidence, ...failure.failureResult }); const settled = evidence._settled as @@ -519,6 +545,12 @@ export async function resumeAdminJobObservation( ); } +function hasUnsettledHardlinks(result: unknown): boolean { + const attempts = (result as { _hardlinkAttempts?: Record } | null) + ?._hardlinkAttempts; + return !!attempts && Object.values(attempts).some((attempt) => attempt?.phase !== "complete"); +} + export interface CancelAdminJobOperationsOptions { pollIntervalMs?: number; sleep?: (ms: number) => Promise; @@ -652,6 +684,7 @@ export async function cancelAdminJobOperations( : "already_terminal", ); } + if (hasUnsettledHardlinks(result)) return "pending"; if (outcomes.includes("contained")) return "canceled"; return outcomes.every((outcome) => outcome === "succeeded") ? "completed" : "already_terminal"; } diff --git a/apps/api/src/services/admin-job-state.ts b/apps/api/src/services/admin-job-state.ts index ab41648f5..720949b45 100644 --- a/apps/api/src/services/admin-job-state.ts +++ b/apps/api/src/services/admin-job-state.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, inArray, sql } from "drizzle-orm"; import { db } from "../db"; -import { adminJob } from "../db/schema"; +import { adminJob, adminJobLog } from "../db/schema"; export interface JobExecution { owner: string; @@ -45,6 +45,48 @@ export function jobOwnership( : sql`${adminJob.result}->'_execution' IS NULL`, ); } +export const ADMIN_JOB_LEASE_MS = 60_000; +export function adminJobObservationDue(now = new Date().toISOString()) { + return and( + sql`(${adminJob.status} = 'cancel_pending' OR ${adminJob.result}->'observation'->>'state' = 'unknown' OR ${adminJob.result}->>'_observationLeaseUntil' IS NOT NULL)`, + sql`coalesce(${adminJob.result}->'observation'->>'nextCheckAt', '') <= ${now}`, + sql`coalesce(${adminJob.result}->>'_observationLeaseUntil', '') <= ${now}`, + ); +} +export async function renewAdminJobClaim(id: string, execution: JobExecution | undefined) { + const now = new Date().toISOString(); + const rows = await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _observationLeaseUntil: new Date(Date.now() + ADMIN_JOB_LEASE_MS).toISOString() })}::jsonb`, + }) + .where( + and(jobOwnership(id, execution), sql`${adminJob.result}->>'_observationLeaseUntil' > ${now}`), + ) + .returning({ id: adminJob.id }); + if (!rows.length) throw new JobOwnershipLost(); +} + +/** Row locking serializes the insert with claims/cancel/settlement. The returned + * insertion, not a preflight SELECT, is the event publication authority. */ +export async function insertOwnedJobLog( + execution: JobExecution | undefined, + entry: typeof adminJobLog.$inferInsert, +): Promise { + const [result] = await db.execute<{ owned: boolean; inserted: boolean }>(sql` + WITH owner AS MATERIALIZED ( + SELECT ${adminJob.id} FROM ${adminJob} WHERE ${jobOwnership(entry.jobId, execution)} FOR UPDATE + ), inserted AS ( + INSERT INTO ${adminJobLog} (id, job_id, seq, stream, line, created_at) + SELECT ${entry.id}, ${entry.jobId}, ${entry.seq}, ${entry.stream}, ${entry.line}, ${entry.createdAt?.toISOString()}::timestamp + FROM owner ON CONFLICT (id) DO NOTHING RETURNING id + ) + SELECT EXISTS (SELECT 1 FROM owner) AS owned, EXISTS (SELECT 1 FROM inserted) AS inserted + `); + if (!result?.owned) throw new JobOwnershipLost(); + return result.inserted; +} + /** Both fields change in one PostgreSQL statement; the old JSON member is the CAS token. */ export async function claimAdminJob( job: { id: string; result: unknown; status: string }, @@ -58,9 +100,14 @@ export async function claimAdminJob( .update(adminJob) .set({ ...patch, - result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _execution: execution, ...(observation ? { _observationLeaseUntil: new Date(Date.now() + 60_000).toISOString() } : {}) })}::jsonb`, + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _execution: execution, _observationLeaseUntil: new Date(Date.now() + ADMIN_JOB_LEASE_MS).toISOString() })}::jsonb`, }) - .where(jobOwnership(job.id, previous, [job.status])) + .where( + and( + jobOwnership(job.id, previous, [job.status]), + ...(observation ? [adminJobObservationDue()] : []), + ), + ) .returning(); return claimed; } diff --git a/apps/api/src/services/admin-ops.ts b/apps/api/src/services/admin-ops.ts index 200b695e6..6c57f60aa 100644 --- a/apps/api/src/services/admin-ops.ts +++ b/apps/api/src/services/admin-ops.ts @@ -1,4 +1,5 @@ import { dirname, join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { createDataManagerTransport, @@ -12,7 +13,9 @@ import { dockerComposeAction, dockerStatus, } from "../utils/docker-compose"; +import { JobObservationPending } from "./admin-job-failure"; import { executeAdminJobOperation } from "./admin-job-ops"; +import { JobOwnershipLost } from "./admin-job-state"; import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; import { getServiceRegistry } from "./service-registry"; @@ -112,7 +115,7 @@ function dataManagerUrlCandidates(): string[] { return [...new Set(out)]; } -async function waitForDataManagerTransport(): Promise<{ +async function waitForDataManagerTransport(signal?: AbortSignal): Promise<{ transport: ReturnType; url: string; }> { @@ -122,18 +125,21 @@ async function waitForDataManagerTransport(): Promise<{ while (Date.now() < deadline) { for (const url of urls) { + signal?.throwIfAborted(); const transport = createDataManagerTransport({ baseUrl: url, authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), }); try { - unwrapDataManagerResponse(await getStatus({ transport })); + unwrapDataManagerResponse(await getStatus({ transport, signal })); + signal?.throwIfAborted(); return { transport, url }; } catch (err) { + signal?.throwIfAborted(); lastErr = err; } } - await new Promise((resolve) => setTimeout(resolve, DATA_MANAGER_POLL_INTERVAL_MS)); + await delay(DATA_MANAGER_POLL_INTERVAL_MS, undefined, { signal }); } const detail = (lastErr as Error | undefined)?.message ?? String(lastErr ?? "unknown error"); @@ -153,12 +159,25 @@ export async function applyGenerationHardlinks(opts: { operationIdentity: string; log?: (msg: string) => Promise | void; signal?: AbortSignal; + ctx?: JobContext; }): Promise { + const signal = opts.ctx?.signal ?? opts.signal; + signal?.throwIfAborted(); + const attempts = opts.ctx?.checkpointResult?._hardlinkAttempts as + | Record + | undefined; + const prior = attempts?.[opts.operationIdentity]; + if (prior) { + if (prior.revisionId !== opts.revisionId || prior.phase !== "complete" || !prior.result) + throw new JobObservationPending("recovery_required"); + return prior.result; + } + if (opts.ctx?.observationOnly) throw new JobObservationPending("recovery_required"); const { plan } = await executeAndWait( createApiOpsClient(), { kind: "configuration.hardlinks.inspect", revisionId: opts.revisionId }, createDurableOpsKey("admin-job.hardlinks.inspect", opts.operationIdentity), - { signal: opts.signal }, + { signal }, ); if (plan.length === 0) { return { applied: true, linked: 0, skipped: 0, pruned: 0, entries: 0 }; @@ -170,21 +189,61 @@ export async function applyGenerationHardlinks(opts: { } await opts.log?.("Ensuring data-manager is running for hardlink apply..."); - const dmStart = await dockerComposeAction("data-manager", "start", { - operationKey: createDurableOpsKey( + signal?.throwIfAborted(); + const coordinatorKey = createDurableOpsKey( + "admin-job.hardlinks.start", + `${opts.operationIdentity}:data-manager`, + ); + if (opts.ctx) { + await executeAdminJobOperation( + opts.ctx, + { kind: "service.start", serviceId: "data-manager" }, "admin-job.hardlinks.start", - `${opts.operationIdentity}:data-manager`, - ), - signal: opts.signal, - }); - if (dmStart.exitCode !== 0) { - throw new Error(`docker compose up data-manager exited with ${dmStart.exitCode}`); + { operationKey: coordinatorKey }, + ); + } else { + const dmStart = await dockerComposeAction("data-manager", "start", { + operationKey: coordinatorKey, + signal, + }); + if (dmStart.exitCode !== 0) + throw new Error(`docker compose up data-manager exited with ${dmStart.exitCode}`); } await opts.log?.(`Applying hardlink plan (${plan.length} entries, prune enabled)...`); - const { transport, url } = await waitForDataManagerTransport(); - const result = unwrapDataManagerResponse(await link({ plan, prune: true }, { transport })); - return { applied: true, entries: plan.length, via: url, ...result }; + const { transport, url } = await waitForDataManagerTransport(signal); + signal?.throwIfAborted(); + const checkpoint = async (phase: "attempted" | "complete", result?: HardlinkApplySummary) => { + if (!opts.ctx) return; + await opts.ctx.checkpoint({ + ...opts.ctx.checkpointResult, + _hardlinkAttempts: { + ...(opts.ctx.checkpointResult?._hardlinkAttempts as Record), + [opts.operationIdentity]: { + revisionId: opts.revisionId, + coordinatorKey, + phase, + ...(result ? { result } : {}), + }, + }, + }); + }; + // This records only the current caller's exact attempt, not a receiver lease + // or containment proof. Unknown remote work is never submitted again. + await checkpoint("attempted"); + signal?.throwIfAborted(); + try { + const result = unwrapDataManagerResponse( + await link({ plan, prune: true }, { transport, signal }), + ); + const summary = { applied: true, entries: plan.length, via: url, ...result }; + await checkpoint("complete", summary); + return summary; + } catch (error) { + if (error instanceof JobOwnershipLost) throw error; + if (opts.ctx) throw new JobObservationPending("recovery_required"); + throw error; + } } // Service control (job handlers) — registry-authorized + routed through the @@ -240,6 +299,7 @@ export async function serviceStart(service: string, ctx: JobContext): Promise ctx.log(m), @@ -273,6 +333,7 @@ export async function serviceApply(service: string, ctx: JobContext): Promise ctx.log(m), diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 8b34fdeb0..7f9aebc4c 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -1,16 +1,19 @@ import { randomUUID } from "node:crypto"; import { and, asc, eq, inArray, sql } from "drizzle-orm"; import { db } from "../db"; -import { adminJob, adminJobLog } from "../db/schema"; +import { adminJob } from "../db/schema"; import { dbActorId } from "../utils/actor"; import { AdminJobFailure, JobObservationPending, publicAdminJobResult } from "./admin-job-failure"; import { cancelAdminJobOperations, resumeAdminJobObservation } from "./admin-job-ops"; import { + adminJobObservationDue, claimAdminJob, + insertOwnedJobLog, type JobExecution, JobOwnershipLost, jobOwnership, readJobExecution, + renewAdminJobClaim, } from "./admin-job-state"; import { type JobStatusEvent, jobEventBus, TERMINAL_JOB_STATUSES } from "./job-events"; import { ApiOpsError, OpsObservationPending } from "./ops-client"; @@ -62,6 +65,8 @@ export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ "system.update", ] as const; +class JobPersistenceUnavailable extends Error {} + class AdminJobRunner { private readonly handlers = new Map(); private readonly active = new Map(); @@ -144,7 +149,11 @@ class AdminJobRunner { const finishedAt = new Date(); const rows = await db .update(adminJob) - .set({ status: "canceled", finishedAt }) + .set({ + status: "canceled", + finishedAt, + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) - 'observation'`, + }) .where(jobOwnership(jobId, readJobExecution(job.result), ["cancel_pending"])) .returning({ id: adminJob.id, result: adminJob.result }); if (rows.length === 0) return false; @@ -303,6 +312,25 @@ class AdminJobRunner { this.active.set(job.id, controller); this.logSeq.set(job.id, 0); + let closed = false; + let heartbeat: ReturnType | undefined; + const renew = async () => { + if (closed || controller.signal.aborted) return; + try { + await renewAdminJobClaim(job.id, readJobExecution(job.result)); + } catch (error) { + controller.abort(error); + return; + } + if (!closed) { + heartbeat = setTimeout(() => void renew(), 10_000); + heartbeat.unref?.(); + } + }; + if (!observe) { + heartbeat = setTimeout(() => void renew(), 10_000); + heartbeat.unref?.(); + } return this.executeJob( job.id, job.type, @@ -318,6 +346,8 @@ class AdminJobRunner { observe, ) .finally(() => { + closed = true; + clearTimeout(heartbeat); this.active.delete(job.id); this.logSeq.delete(job.id); setTimeout(() => void this.pump(), 0); @@ -338,6 +368,13 @@ class AdminJobRunner { signal: AbortSignal, observe = false, ): Promise { + const retirement = new AbortController(); + signal = AbortSignal.any([signal, retirement.signal]); + const lost = (): never => { + const error = new JobOwnershipLost(); + retirement.abort(error); + throw error; + }; const handler = observe ? resumeAdminJobObservation : this.handlers.get(type); const execution = readJobExecution(checkpointResult); let checkpointTail = Promise.resolve(); @@ -355,11 +392,23 @@ class AdminJobRunner { this.logSeq.set(jobId, Math.max(localSeq + 1, seq + 1)); const id = sourceEventId ?? randomUUID(); const createdAt = new Date(); - const insertion = db - .insert(adminJobLog) - .values({ id, jobId, seq, stream, line, createdAt }); - if (sourceEventId) await insertion.onConflictDoNothing(); - else await insertion; + signal.throwIfAborted(); + let inserted: boolean; + try { + inserted = await insertOwnedJobLog(execution, { + id, + jobId, + seq, + stream, + line, + createdAt, + }); + } catch (error) { + if (error instanceof JobOwnershipLost) lost(); + throw error; + } + signal.throwIfAborted(); + if (!inserted) return; jobEventBus.publish(jobId, { type: "log", id, @@ -370,16 +419,21 @@ class AdminJobRunner { }); }, setProgress: async (progress) => { + signal.throwIfAborted(); const rows = await db .update(adminJob) .set({ progress }) .where(jobOwnership(jobId, execution)) - .returning({ id: adminJob.id }); - if (!rows.length) throw new JobOwnershipLost(); + .returning({ id: adminJob.id }) + .catch(() => { + throw new JobPersistenceUnavailable("Administrative job progress unavailable"); + }); + if (!rows.length) lost(); jobEventBus.publish(jobId, { type: "progress", progress }); }, checkpoint: async (result, progress, admissionKey) => { const write = checkpointTail.then(async () => { + signal.throwIfAborted(); // A handler cannot replace the runner's ownership member. const merged: Record = { ...ctx.checkpointResult, @@ -388,7 +442,10 @@ class AdminJobRunner { }; const rows = await db .update(adminJob) - .set({ result: merged, ...(progress === undefined ? {} : { progress }) }) + .set({ + result: sql`${JSON.stringify(merged)}::jsonb || jsonb_build_object('_observationLeaseUntil', ${adminJob.result}->'_observationLeaseUntil')`, + ...(progress === undefined ? {} : { progress }), + }) .where( and( jobOwnership(jobId, execution), @@ -401,81 +458,92 @@ class AdminJobRunner { ), ) .returning({ id: adminJob.id }); - if (!rows.length) throw new JobOwnershipLost(); + if (!rows.length) lost(); ctx.checkpointResult = merged; if (progress !== undefined) jobEventBus.publish(jobId, { type: "progress", progress }); }); checkpointTail = write.catch(() => undefined); - return write; + return write.catch((error) => { + if (error instanceof JobOwnershipLost || signal.aborted) throw error; + throw new JobPersistenceUnavailable("Administrative job checkpoint unavailable"); + }); }, }; if (!observe) this.announce(jobId, "running", { startedAt: new Date().toISOString() }); const startedAtMs = Date.now(); const pending = async (reason: OpsObservationPending["reason"]) => { - const attempts = Math.min(12, Number(ctx.checkpointResult?._observationAttempts ?? 0) + 1); - const observationPatch = { - _observationAttempts: attempts, - _observationLeaseUntil: null, - observation: { - state: "unknown", - reason, - nextCheckAt: new Date( - Date.now() + Math.min(60_000, 5_000 * 2 ** (attempts - 1)), - ).toISOString(), - }, - }; - const rows = await db - .update(adminJob) - .set({ - result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify(observationPatch)}::jsonb`, - }) - .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])) - .returning({ status: adminJob.status, result: adminJob.result }); - if (rows[0]) - this.announce(jobId, rows[0].status, { - result: publicAdminJobResult(rows[0].result as Record | null), - }); + try { + const attempts = Math.min(12, Number(ctx.checkpointResult?._observationAttempts ?? 0) + 1); + const observationPatch = { + _observationAttempts: attempts, + _observationLeaseUntil: null, + observation: { + state: "unknown", + reason, + nextCheckAt: new Date( + Date.now() + Math.min(60_000, 5_000 * 2 ** (attempts - 1)), + ).toISOString(), + }, + }; + const rows = await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify(observationPatch)}::jsonb`, + }) + .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])) + .returning({ status: adminJob.status, result: adminJob.result }); + if (rows[0]) + this.announce(jobId, rows[0].status, { + result: publicAdminJobResult(rows[0].result as Record | null), + }); + } catch { + throw new JobPersistenceUnavailable("Administrative job persistence unavailable"); + } }; const settle = async ( status: "success" | "failed", result: Record | null, error?: string, ) => { - await checkpointTail; - const merged: Record = { - ...ctx.checkpointResult, - ...result, - ...(execution ? { _execution: execution } : {}), - }; - delete merged.observation; - const finishedAt = new Date(); - const rows = await db - .update(adminJob) - .set({ - status, - result: merged, - error: error ?? null, - finishedAt, - ...(status === "success" ? { progress: 100 } : {}), - }) - .where(jobOwnership(jobId, execution)) - .returning({ id: adminJob.id }); - if (rows.length) { - this.announce(jobId, status, { - result: publicAdminJobResult(merged), - error: error ?? null, - ...(status === "success" ? { progress: 100 } : {}), - finishedAt: finishedAt.toISOString(), - }); - } else { - // Cancel intent wins; retain late evidence without publishing a terminal event. - await db + try { + await checkpointTail; + const merged: Record = { + ...ctx.checkpointResult, + ...result, + ...(execution ? { _execution: execution } : {}), + }; + delete merged.observation; + const finishedAt = new Date(); + const rows = await db .update(adminJob) .set({ - result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _settled: { status, result: publicAdminJobResult(merged), error } })}::jsonb`, + status, + result: merged, + error: error ?? null, + finishedAt, + ...(status === "success" ? { progress: 100 } : {}), }) - .where(jobOwnership(jobId, execution, ["cancel_pending"])); + .where(jobOwnership(jobId, execution)) + .returning({ id: adminJob.id }); + if (rows.length) { + this.announce(jobId, status, { + result: publicAdminJobResult(merged), + error: error ?? null, + ...(status === "success" ? { progress: 100 } : {}), + finishedAt: finishedAt.toISOString(), + }); + } else { + // Cancel intent wins; retain late evidence without publishing a terminal event. + await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _settled: { status, result: publicAdminJobResult(merged), error } })}::jsonb`, + }) + .where(jobOwnership(jobId, execution, ["cancel_pending"])); + } + } catch { + throw new JobPersistenceUnavailable("Administrative job persistence unavailable"); } }; try { @@ -510,13 +578,17 @@ class AdminJobRunner { const finishedAt = new Date(); const rows = await db .update(adminJob) - .set({ status: "canceled", finishedAt }) + .set({ + status: "canceled", + finishedAt, + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) - 'observation'`, + }) .where(jobOwnership(jobId, execution, ["cancel_pending"])) - .returning({ id: adminJob.id }); + .returning({ id: adminJob.id, result: adminJob.result }); if (rows.length) this.announce(jobId, "canceled", { finishedAt: finishedAt.toISOString(), - result: publicAdminJobResult(ctx.checkpointResult), + result: publicAdminJobResult(rows[0]?.result as Record | null), }); return; } @@ -530,16 +602,21 @@ class AdminJobRunner { } } const result = await handler(ctx); - await settle("success", result ?? null); + // A failed terminal write is uncertainty, not a handler failure. Never + // turn an accepted successful effect into failed because PostgreSQL failed. + try { + await settle("success", result ?? null); + } catch { + await pending("transport"); + } } catch (err) { + if (err instanceof JobOwnershipLost || err instanceof JobPersistenceUnavailable) return; if (err instanceof OpsObservationPending || err instanceof JobObservationPending) await pending(err.reason); - else if ( - err instanceof JobOwnershipLost || - signal.aborted || - (err instanceof Error && err.name === "AbortError") - ) + else if (signal.aborted || (err instanceof Error && err.name === "AbortError")) await pending("observer_deadline"); + else if (observe && !(err instanceof AdminJobFailure) && !(err instanceof ApiOpsError)) + await pending("transport"); else await settle( "failed", @@ -567,9 +644,7 @@ class AdminJobRunner { and( inArray(adminJob.status, ["running", "cancel_pending"]), inArray(adminJob.type, [...AGENT_RECOVERABLE_ADMIN_JOB_TYPES]), - sql`(${adminJob.status} = 'cancel_pending' OR ${adminJob.result}->'observation'->>'state' = 'unknown')`, - sql`coalesce(${adminJob.result}->'observation'->>'nextCheckAt', '') <= ${new Date().toISOString()}`, - sql`coalesce(${adminJob.result}->>'_observationLeaseUntil', '') <= ${new Date().toISOString()}`, + adminJobObservationDue(), ), ) .orderBy(asc(adminJob.createdAt)) From 5233d5f40f94dee8d48b4234398e83ee0d4c452b Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 23 Oct 2026 23:42:53 +0200 Subject: [PATCH 010/135] refactor(data): share bounded build artifact validation --- .dockerignore | 3 + .gitignore | 3 + apps/ops-agent/src/administrative-runtime.ts | 24 +- apps/ops-agent/src/agent-handoff.test.ts | 8 +- apps/ops-agent/src/agent-handoff.ts | 23 +- apps/ops-agent/src/control-admission.test.ts | 194 ++ apps/ops-agent/src/index.ts | 9 + apps/ops-agent/src/journal.ts | 7 + apps/ops-agent/src/runtime.ts | 6 + apps/ops-agent/src/server.ts | 188 +- .../src/transitous-lock-runtime.test.ts | 44 +- apps/ops-agent/src/transitous-lock-runtime.ts | 31 +- .../docs/developer/build-artifacts-control.md | 116 ++ .../cli/__tests__/cli-build-artifacts.test.ts | 90 +- .../cli/__tests__/cli-control-context.test.ts | 107 + .../__tests__/cli-execution-deadline.test.ts | 2 +- packages/cli/src/commands/compose.ts | 4 + packages/cli/src/commands/data.ts | 67 +- packages/cli/src/commands/services.ts | 49 +- packages/cli/src/lib/build-artifacts.ts | 446 +---- packages/cli/src/lib/control-context.ts | 140 ++ packages/cli/src/lib/execution-deadline.ts | 13 +- packages/core/package.json | 10 +- .../core/src/build-artifacts/archive.test.ts | 287 +++ packages/core/src/build-artifacts/archive.ts | 720 +++++++ .../build-artifacts/control-authority.test.ts | 359 ++++ .../src/build-artifacts/control-authority.ts | 1758 +++++++++++++++++ .../core/src/build-artifacts/filesystem.ts | 200 ++ packages/core/src/build-artifacts/index.ts | 46 + .../core/src/build-artifacts/manifest.test.ts | 45 + packages/core/src/build-artifacts/manifest.ts | 191 ++ .../src/build-artifacts/peer-admission.ts | 80 + .../src/build-artifacts/peer-process.test.ts | 692 +++++++ .../src/build-artifacts/target-lease.test.ts | 371 ++++ .../core/src/build-artifacts/target-lease.ts | 1348 +++++++++++++ .../test-fixtures/peer-process.ts | 145 ++ packages/core/src/ops/execution-identity.ts | 21 +- .../core/src/ops/verified-parent-internal.ts | 3 + pnpm-lock.yaml | 13 + services/data-manager/__tests__/auth.test.ts | 29 + services/data-manager/service.json | 68 +- services/data-manager/src/auth.ts | 11 + services/data-manager/src/service.ts | 7 + services/ops-agent/service.json | 45 +- 44 files changed, 7489 insertions(+), 534 deletions(-) create mode 100644 apps/ops-agent/src/control-admission.test.ts create mode 100644 docs/docs/developer/build-artifacts-control.md create mode 100644 packages/cli/__tests__/cli-control-context.test.ts create mode 100644 packages/cli/src/lib/control-context.ts create mode 100644 packages/core/src/build-artifacts/archive.test.ts create mode 100644 packages/core/src/build-artifacts/archive.ts create mode 100644 packages/core/src/build-artifacts/control-authority.test.ts create mode 100644 packages/core/src/build-artifacts/control-authority.ts create mode 100644 packages/core/src/build-artifacts/filesystem.ts create mode 100644 packages/core/src/build-artifacts/index.ts create mode 100644 packages/core/src/build-artifacts/manifest.test.ts create mode 100644 packages/core/src/build-artifacts/manifest.ts create mode 100644 packages/core/src/build-artifacts/peer-admission.ts create mode 100644 packages/core/src/build-artifacts/peer-process.test.ts create mode 100644 packages/core/src/build-artifacts/target-lease.test.ts create mode 100644 packages/core/src/build-artifacts/target-lease.ts create mode 100644 packages/core/src/build-artifacts/test-fixtures/peer-process.ts create mode 100644 packages/core/src/ops/verified-parent-internal.ts diff --git a/.dockerignore b/.dockerignore index d86227de5..c3d82dc14 100644 --- a/.dockerignore +++ b/.dockerignore @@ -61,3 +61,6 @@ yarn-error.log* *.pem **/*.pem docs/ + +# Role-private control signing and recovery material +infra/docker/control-private/ diff --git a/.gitignore b/.gitignore index 48bef7481..f75e181b9 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,6 @@ dist/mobile/ # Generated Traefik file-provider routing (produced by `openmapx compose render`). services/traefik/config/dynamic/generated-routes.yml + +# Role-private control signing and recovery material +infra/docker/control-private/ diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index c228ef3ff..e50aedac5 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -17,6 +17,11 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { + closeControlInstallation, + openFixedChildInstallation, + readFixedCliSpawnIntent, +} from "@openmapx/core/build-artifacts"; import { type ExecutionRef, executionRefSchema, @@ -1721,8 +1726,23 @@ export function createDefaultFixedCli( const encoded = JSON.stringify(executionRefSchema.parse(options.execution)); if (Buffer.byteLength(encoded) > 2048) throw new Error("Execution context exceeds budget"); env.OPENMAPX_CONTROL_EXECUTION = encoded; - if (options.controlAssertion || options.parentAssertion) - throw new Error("Fixed CLI control verifier is unavailable"); + if (options.controlAssertion || options.parentAssertion) { + if (options.controlAssertion && options.parentAssertion) + throw Error("Ambiguous CLI authority"); + const assertion = (options.controlAssertion ?? options.parentAssertion)!; + const installation = openFixedChildInstallation({ + infraDir: join(_rootDir, "infra/docker"), + assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + readFixedCliSpawnIntent(installation, options.execution, args); + } finally { + closeControlInstallation(installation); + } + if (options.controlAssertion) env.OPENMAPX_CONTROL_DELEGATION = assertion; + else env.OPENMAPX_PARENT_EXECUTION = assertion; + } } else if (options.controlAssertion || options.parentAssertion) throw new Error("Execution context is required"); await runContainedStreamingProcess(executable as string, [...prefix, ...args], { diff --git a/apps/ops-agent/src/agent-handoff.test.ts b/apps/ops-agent/src/agent-handoff.test.ts index 19bf2e348..96776c922 100644 --- a/apps/ops-agent/src/agent-handoff.test.ts +++ b/apps/ops-agent/src/agent-handoff.test.ts @@ -366,7 +366,7 @@ describe("agent handoff helper container", () => { rootSource: "/srv/openmapx", }; - it("runs the agent image with only the socket and checkout, no network and no privileges", () => { + it("runs the agent image with isolated control material, no network and no privileges", () => { const args = agentHandoffRunArgs(facts, "/srv/openmapx", "handoff-1"); expect(args.slice(0, 3)).toEqual(["run", "--detach", "--rm"]); expect(args).toEqual( @@ -386,7 +386,11 @@ describe("agent handoff helper container", () => { "DOCKER_CONFIG=/tmp/docker", ]), ); - expect(args.filter((arg) => arg === "--mount")).toHaveLength(2); + expect(args.filter((arg) => arg === "--mount")).toHaveLength(9); + for (const role of ["ops-agent", "data-manager", "host-cli"]) + expect(args).toContain( + `type=bind,source=/srv/openmapx/infra/docker/control-private/hidden-key,target=/srv/openmapx/infra/docker/secrets/control-${role}.key,readonly`, + ); expect(args).not.toContain("HOSTNAME"); expect(args).not.toContain("DOCKER_CONFIG"); // Only names: Docker copies each value from the agent's own environment. diff --git a/apps/ops-agent/src/agent-handoff.ts b/apps/ops-agent/src/agent-handoff.ts index 11dff1984..25c152f12 100644 --- a/apps/ops-agent/src/agent-handoff.ts +++ b/apps/ops-agent/src/agent-handoff.ts @@ -192,7 +192,7 @@ export function agentHandoffContainerName(handoffId: string): string { return `openmapx-ops-agent-handoff-${handoffId.slice(0, 64)}`; } -/** `docker run` arguments for the helper: Docker socket and checkout only, no network. */ +/** Fixed helper mounts keep sibling role keys and private receipts hidden; no network. */ export function agentHandoffRunArgs( facts: AgentContainerFacts, rootDir: string, @@ -206,7 +206,14 @@ export function agentHandoffRunArgs( const inherited = facts.envNames.filter( (name) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) && - !["HOSTNAME", "DOCKER_CONFIG", "OPS_AGENT_HANDOFF_ID"].includes(name), + ![ + "HOSTNAME", + "DOCKER_CONFIG", + "OPS_AGENT_HANDOFF_ID", + "OPENMAPX_CONTROL_EXECUTION", + "OPENMAPX_CONTROL_DELEGATION", + "OPENMAPX_PARENT_EXECUTION", + ].includes(name), ); return [ "run", @@ -236,6 +243,18 @@ export function agentHandoffRunArgs( `type=bind,source=${facts.dockerSocketSource},target=/var/run/docker.sock`, "--mount", `type=bind,source=${facts.rootSource},target=${rootDir}`, + ...["ops-agent", "data-manager", "host-cli"].flatMap((role) => [ + "--mount", + `type=bind,source=${facts.rootSource}/infra/docker/control-private/hidden-key,target=${rootDir}/infra/docker/secrets/control-${role}.key,readonly`, + ]), + ...["data-manager", "host-cli"].flatMap((role) => [ + "--mount", + `type=bind,source=${facts.rootSource}/infra/docker/control-private/empty,target=${rootDir}/infra/docker/control-private/${role},readonly`, + ]), + "--mount", + `type=bind,source=${facts.rootSource}/infra/docker/control-private/ops-agent,target=/var/lib/openmapx/control-private`, + "--mount", + `type=bind,source=${facts.rootSource}/infra/docker/secrets/control-verifiers.json,target=/run/secrets/openmapx-control-verifiers,readonly`, ...inherited.flatMap((name) => ["--env", name]), "--env", "DOCKER_CONFIG=/tmp/docker", diff --git a/apps/ops-agent/src/control-admission.test.ts b/apps/ops-agent/src/control-admission.test.ts new file mode 100644 index 000000000..6a12c3d0d --- /dev/null +++ b/apps/ops-agent/src/control-admission.test.ts @@ -0,0 +1,194 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + type ExecutionRef, + executionFingerprint, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { type OpsJobJournal, openOpsJobJournal } from "./journal"; +import { buildOpsAgentServer } from "./server"; + +it.each([false, true])( + "adopts at authenticated HTTP admission without replay (ready-write failure: %s)", + async (failReadyWrite) => { + const root = mkdtempSync(join(tmpdir(), "omx-control-http-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis"], + }); + const agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const operation = { kind: "transit-sync" }; + const hash = opsOperationFingerprint({ kind: "motis.primary.stop" }); + const deadlineAt = Date.now() + 60000; + // The source root uses its canonical data-input digest. + const { createHash } = await import("node:crypto"); + const rootHash = createHash("sha256").update(JSON.stringify(operation)).digest("hex"); + const owner: ExecutionRef = { + executor: "data-manager", + principal: "data-manager", + executionId: randomUUID(), + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint: rootHash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: rootHash, + deadlineAt, + }), + }; + const child: ExecutionRef = { + executor: "ops-agent", + principal: "data-manager", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: control.controlChildOperationKey(owner.operationKey, "stop-primary", 0), + operationFingerprint: hash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: "motis.primary.stop", + operationFingerprint: hash, + deadlineAt, + parent: control.retainedControlParent(owner), + }), + }; + const plan = { + phase: "stop-primary" as const, + ordinal: 0, + kind: "motis.primary.stop", + execution: child, + serviceIds: ["motis"], + recipient: "ops-agent" as const, + }; + const authority = control.retainAdmittedControlExecution(dm, { + execution: owner, + kind: operation.kind, + operation, + admittedAt: deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await control.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const assertion = await control.delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: agent.publicKey, + }); + let calls = 0; + const token = "b".repeat(32); + const journalFile = join(root, "data/ops-agent/jobs-v1.json"); + const options = { + tokens: { api: "a".repeat(32), "data-manager": token }, + controlInstallation: agent, + resourcePolicy: { allowGlobal: () => true, allowService: () => true }, + dispatch: async () => { + calls++; + return { changed: true, state: "stopped" }; + }, + }; + const journal = await openOpsJobJournal(journalFile); + let writes = 0; + const controlledJournal: OpsJobJournal = { + records: () => journal.records(), + replace: async (records) => { + if (failReadyWrite && ++writes === 2) throw Error("synthetic ready journal failure"); + await journal.replace(records); + }, + }; + const app = buildOpsAgentServer({ ...options, journal: controlledJournal }); + const request = () => + app.inject({ + method: "POST", + url: "/v1/operations", + headers: { + authorization: `Bearer ${token}`, + "x-ops-protocol-version": "2", + "x-openmapx-control-delegation": assertion, + }, + payload: { + version: 2, + requestId: `ops1_${randomUUID().replaceAll("-", "")}`, + operationKey: child.operationKey, + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 10000).toISOString(), + operation: { kind: "motis.primary.stop" }, + execution: { requestedDeadlineAt: deadlineAt }, + }, + }); + try { + const result = await request(); + if (failReadyWrite) { + expect(result.statusCode).toBe(500); + expect(calls).toBe(0); + expect((await request()).statusCode).toBe(202); + expect(journal.records()[0]).toMatchObject({ + state: "failed", + errorClass: "recovery_required", + controlAdoption: { phase: "recovery_required" }, + }); + await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); + return; + } + expect(result.statusCode, result.body).toBe(202); + await expect + .poll( + async () => + ( + await app.inject({ + url: `/v1/operations/${child.executionId}`, + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }, + }) + ).json().result.state, + ) + .toBe("succeeded"); + expect((await request()).statusCode).toBe(202); + expect(calls).toBe(1); + await app.close(); + const replacement = buildOpsAgentServer({ + ...options, + journal: await openOpsJobJournal(journalFile), + }); + await replacement.ready(); + expect(calls).toBe(1); + await replacement.close(); + control.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await control.settleArtifactTargetLeaseChild( + lease, + control.verifyControlEffectSettlement(dm, owner), + ); + await control.releaseArtifactTargetLease(lease); + const releasedRestart = buildOpsAgentServer({ + ...options, + journal: await openOpsJobJournal(journalFile), + }); + const status = await releasedRestart.inject({ + url: `/v1/operations/${child.executionId}`, + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }, + }); + expect(status.json().result.state).toBe("succeeded"); + await releasedRestart.close(); + } finally { + await app.close(); + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agent); + control.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + } + }, +); diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 64fa10524..65ee93813 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -1,6 +1,7 @@ import { constants, existsSync } from "node:fs"; import { open } from "node:fs/promises"; import { join } from "node:path"; +import { loadControlStartup } from "@openmapx/core/build-artifacts"; import { readOpsTokenFile } from "@openmapx/core/ops"; import { repoPaths, services } from "@openmapx/core/server"; import { @@ -130,6 +131,13 @@ async function main(): Promise { service.enabled = bootstrapEnabledServiceIds.has(service.manifest.id); } const journal = await openOpsJobJournal(config.journalFile); + const controlStartup = loadControlStartup({ + infraDir: paths.infraDir, + role: "ops-agent", + serviceIds: [...builtInServices, ...communityServices].map((s) => s.manifest.id), + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + if (controlStartup.reason) console.error(controlStartup.reason); await initializeTrustedSnapshotDirectory(config.trustedConfigDirectory, { allowedUids: [uid], expectedGid: gid, @@ -246,6 +254,7 @@ async function main(): Promise { }, }), journal, + controlInstallation: controlStartup.installation, runtime, replacementInProgress: () => { try { diff --git a/apps/ops-agent/src/journal.ts b/apps/ops-agent/src/journal.ts index 364a036b8..d8fc3c06a 100644 --- a/apps/ops-agent/src/journal.ts +++ b/apps/ops-agent/src/journal.ts @@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto"; import { constants } from "node:fs"; import { mkdir, open, opendir, rename, unlink } from "node:fs/promises"; import { basename, dirname, join } from "node:path"; +import { type ControlAdoption, controlAdoptionSchema } from "@openmapx/core/build-artifacts"; import { OPS_KIND_POLICIES, OPS_PUBLIC_ERROR_MESSAGES, @@ -40,6 +41,7 @@ const ERROR_CLASSES = new Set( ); export interface PersistedOpsJob { + controlAdoption?: ControlAdoption; executionVersion?: 2; requestVersion?: 1 | 2; admittedAt?: number; @@ -97,6 +99,7 @@ function parseJob(value: unknown): PersistedOpsJob { if ( !exactKeys(raw, [ "executionVersion", + "controlAdoption", "requestVersion", "admittedAt", "deadlineAt", @@ -139,6 +142,10 @@ function parseJob(value: unknown): PersistedOpsJob { ) { throw new Error("invalid"); } + if (raw.controlAdoption !== undefined) { + controlAdoptionSchema.parse(raw.controlAdoption); + if (raw.executionVersion !== 2) throw Error("invalid control adoption"); + } const operation = opsOperationSchema.parse(raw.operation); if ( opsOperationFingerprint(operation) !== diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index ef1d95c62..431a27309 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -1,3 +1,7 @@ +import type { + BorrowedArtifactTargetLease, + VerifiedExecutionAuthority, +} from "@openmapx/core/build-artifacts"; import type { TrustedConfigurationPayload } from "@openmapx/core/ops"; import { type ExecutionRef, @@ -45,6 +49,8 @@ export class OpsAgentBusyError extends Error { export interface OpsExecutionContext { readonly execution: ExecutionRef; + readonly artifactLease?: BorrowedArtifactTargetLease; + readonly controlAuthority?: VerifiedExecutionAuthority; signal: AbortSignal; emitLog(stream: "stdout" | "stderr", message: string): void; claim: OpsTrustedClaim; diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 83f8ac352..1ca5054a9 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -1,4 +1,22 @@ import { createHash, randomBytes as nodeRandomBytes, timingSafeEqual } from "node:crypto"; +import { + assertArtifactTargetLeaseForward, + type BorrowedArtifactTargetLease, + beginControlEffect, + type ControlInstallation, + cancelArtifactTargetLease, + closeArtifactTargetLeaseHandle, + inspectControlAdmission, + controlHeaders as readControlHeaders, + restoreAdmittedControlExecution, + retainAdmittedControlExecution, + retainControlEffectReceipt, + retainedControlAdoptionAssertion, + settleArtifactTargetLeaseChild, + type VerifiedExecutionAuthority, + verifyControlEffectSettlement, + verifyRetainedArtifactTargetSettlement, +} from "@openmapx/core/build-artifacts"; import { type ConfigurationApplyResult, configurationApplyResultSchema, @@ -114,6 +132,7 @@ export interface OpsAuditEvent { } export interface BuildOpsAgentServerOptions { + controlInstallation?: ControlInstallation; tokens: Record; releaseRecovery?: { ready: boolean; reason?: "recovery_required" }; resourcePolicy?: OpsResourcePolicy; @@ -385,7 +404,15 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } }; + const controlLeases = new Map(); + const controlAuthorities = new Map(); + app.addHook("onClose", async () => { + for (const lease of controlLeases.values()) closeArtifactTargetLeaseHandle(lease); + controlLeases.clear(); + controlAuthorities.clear(); + }); const persisted = (job: JobRecord): PersistedOpsJob => ({ + ...(job.controlAdoption ? { controlAdoption: job.controlAdoption } : {}), ...(job.executionVersion === 2 ? { executionVersion: 2 as const, @@ -510,6 +537,24 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ): Promise => { await withAdmissionLock(async () => { const updatedAt = now().toISOString(); + const lease = controlLeases.get(job.operationId); + const authority = controlAuthorities.get(job.operationId); + if (lease && state === "termination_pending") await cancelArtifactTargetLease(lease); + if (lease && authority && ["succeeded", "failed", "timed_out"].includes(state)) { + // The fixed runtime's validated result is own-effect evidence. Failure alone is not containment. + const committed = state === "succeeded" && updates.result !== undefined; + retainControlEffectReceipt(authority, { + outcome: committed ? "committed" : "unknown", + publication: committed ? "committed" : "unknown", + receipt: committed + ? { kind: "ops-result", result: updates.result } + : { kind: "unresolved" }, + }); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(options.controlInstallation!, authority.execution), + ); + } const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; await journal.replace( [...jobsById.values()].map((entry) => persisted(entry === job ? candidate : entry)), @@ -517,6 +562,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif Object.assign(job, updates); job.updatedAt = updatedAt; job.state = state; + if (lease && ["succeeded", "failed", "timed_out"].includes(state)) { + closeArtifactTargetLeaseHandle(lease); + controlLeases.delete(job.operationId); + controlAuthorities.delete(job.operationId); + } job.cursor += 1; pushEvent(job, { cursor: job.cursor, @@ -560,6 +610,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif signal, claim, execution: executionFor(job), + artifactLease: controlLeases.get(job.operationId), + controlAuthority: controlAuthorities.get(job.operationId), otherActiveOperations: () => [...jobsById.values()].filter( (other) => @@ -690,6 +742,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif timedOut = true; throw new OpsExecutionFailure("timeout", {}); } + const authority = controlAuthorities.get(job.operationId); + if (authority) { + const lease = controlLeases.get(job.operationId); + if (lease) assertArtifactTargetLeaseForward(lease); + beginControlEffect(authority); + } return dispatch(job.operation, runtime, contextFor(job, controller.signal, claim)); }) .then( @@ -939,6 +997,56 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }; }); + // No replay during restart. Rejoin only the original signed adoption and settle + // from an already retained own-effect receipt; absent evidence keeps its fence. + app.addHook("onReady", async () => { + let changed = false; + for (const job of jobsById.values()) { + if (!job.controlAdoption) continue; + try { + if (!options.controlInstallation) throw Error("Control trust unavailable"); + const execution = executionFor(job); + if (job.state === "succeeded") { + verifyRetainedArtifactTargetSettlement( + options.controlInstallation, + job.controlAdoption, + execution, + ); + continue; + } + const assertion = retainedControlAdoptionAssertion( + options.controlInstallation, + job.controlAdoption, + execution, + ); + const admission = inspectControlAdmission(options.controlInstallation, { + rawHeaders: ["x-openmapx-control-delegation", assertion], + transportPrincipal: "data-manager", + kind: job.operation.kind, + operationKey: job.operationKey, + operationFingerprint: execution.operationFingerprint, + requestedDeadlineAt: execution.deadlineAt, + }); + const lease = await admission.adopt(execution); + const authority = restoreAdmittedControlExecution(options.controlInstallation, execution); + controlLeases.set(job.operationId, lease); + controlAuthorities.set(job.operationId, authority); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(options.controlInstallation, execution), + ); + } catch { + job.controlAdoption = { ...job.controlAdoption, phase: "recovery_required" }; + job.state = "failed"; + job.errorClass = "recovery_required"; + job.result = undefined; + stickyRecoveryRequired.add(job.operationId); + changed = true; + } + } + if (changed) await journal.replace([...jobsById.values()].map(persisted)); + }); + app.post("/v1/operations", async (request, reply) => { const startedAt = Date.now(); const requestId = requestIdFromBody(request.body); @@ -957,16 +1065,36 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const policy = OPS_KIND_POLICIES[operation.kind]; const operationFingerprint = opsOperationFingerprint(operation); let parent: VerifiedParentExecution | undefined; + let controlAdmission: ReturnType | undefined; const parentHeader = request.headers["x-openmapx-parent-execution"]; const delegationHeader = request.headers["x-openmapx-control-delegation"]; - const controlHeaders = request.raw.rawHeaders - .filter((_, i) => i % 2 === 0) - .map((x) => x.toLowerCase()); - if ( - delegationHeader !== undefined || - controlHeaders.filter((x) => x === "x-openmapx-parent-execution").length > 1 - ) + try { + readControlHeaders(request.raw.rawHeaders); + } catch { throw new RequestGuardError("authorization", 403); + } + if (delegationHeader !== undefined) { + if ( + !options.controlInstallation || + role !== "data-manager" || + parsed.version !== 2 || + parentHeader !== undefined + ) + throw new RequestGuardError("authorization", 403); + try { + controlAdmission = inspectControlAdmission(options.controlInstallation, { + rawHeaders: request.raw.rawHeaders, + transportPrincipal: "data-manager", + kind, + operationKey: parsed.operationKey, + operationFingerprint, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + }); + parent = controlAdmission.parent; + } catch { + throw new RequestGuardError("authorization", 403); + } + } if (parsed.version === 1 && (parentHeader !== undefined || delegationHeader !== undefined)) throw new RequestGuardError("authorization", 403); if (parentHeader !== undefined) { @@ -1126,9 +1254,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (jobsById.size >= maxJobEntries) throw new RequestGuardError("busy", 429); if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); const timestamp = new Date(normalized.admittedAt).toISOString(); - let operationId: string | undefined; + let operationId: string | undefined = controlAdmission?.execution.executionId; + if (operationId && jobsById.has(operationId)) + throw new RequestGuardError("conflict", 409); try { - for (let attempt = 0; attempt < MAX_JOB_ID_ATTEMPTS; attempt += 1) { + for (let attempt = 0; !operationId && attempt < MAX_JOB_ID_ATTEMPTS; attempt += 1) { const candidate = `job1_${Buffer.from(randomBytes(18)).toString("base64url")}`; if (OPERATION_ID_PATTERN.test(candidate) && !jobsById.has(candidate)) { operationId = candidate; @@ -1140,6 +1270,15 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } if (!operationId) throw new RequestGuardError("busy", 429); const job: JobRecord = { + ...(controlAdmission + ? { + controlAdoption: { + leaseId: controlAdmission.leaseId, + delegationId: controlAdmission.delegationId, + phase: "pending-adoption" as const, + }, + } + : {}), role, operation, operationId, @@ -1166,6 +1305,37 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); jobsById.set(job.operationId, job); jobsByKey.set(ownerKey, job); + if (controlAdmission) { + try { + const lease = await controlAdmission.adopt(executionFor(job)); + const authority = retainAdmittedControlExecution(options.controlInstallation!, { + execution: executionFor(job), + kind: job.operation.kind, + operation: job.operation, + admittedAt: job.admittedAt!, + parent: job.parent, + serviceIds: lease.serviceIds, + }); + controlLeases.set(job.operationId, lease); + controlAuthorities.set(job.operationId, authority); + job.controlAdoption = { ...job.controlAdoption!, phase: "ready" }; + await journal.replace([...jobsById.values()].map(persisted)); + } catch (error) { + // The receiver may already have adopted. This identity remains + // observable as recovery-required and is never dispatched again. + job.controlAdoption = { ...job.controlAdoption!, phase: "recovery_required" }; + forceRecoveryRequired(job); + try { + await journal.replace([...jobsById.values()].map(persisted)); + } catch (persistence) { + throw new AggregateError( + [error, persistence], + "Control adoption requires reconciliation", + ); + } + throw error; + } + } return { type: "new", job } as const; }).catch(async (error) => { await claimAdmission?.rollback().catch(() => undefined); diff --git a/apps/ops-agent/src/transitous-lock-runtime.test.ts b/apps/ops-agent/src/transitous-lock-runtime.test.ts index 31236ceb9..b246a4a7e 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.test.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.test.ts @@ -81,7 +81,7 @@ describe("agent-owned Transitous catalog lock", () => { }); }); - it("activates a proposal only when the approved ref matches exactly", async () => { + it("keeps approval closed until durable paired-pin publication is installed", async () => { const rootDir = root(); const runtime = runtimeFor(rootDir); await dispatchOpsOperation( @@ -89,40 +89,16 @@ describe("agent-owned Transitous catalog lock", () => { { kind: "transitousLock.propose", ref: REF, submodules: {}, lockedBy: "auto-bump" }, context(), ); - - // Approving a different ref than the one proposed must not activate. - await expect( - dispatchOpsOperation( - runtime, - { kind: "transitousLock.approve", ref: OTHER_REF, approvedBy: "admin" }, - context(), - ), - ).rejects.toThrow(/does not match/i); + for (const ref of [REF, OTHER_REF]) + await expect( + dispatchOpsOperation( + runtime, + { kind: "transitousLock.approve", ref, approvedBy: "admin" }, + context(), + ), + ).rejects.toThrow(/paired-pin/); expect(existsSync(lockPath(rootDir))).toBe(false); - - const approved = await dispatchOpsOperation( - runtime, - { kind: "transitousLock.approve", ref: REF, approvedBy: "admin" }, - context(), - ); - expect(approved).toMatchObject({ ref: REF }); - expect(JSON.parse(readFileSync(lockPath(rootDir), "utf-8"))).toMatchObject({ - ref: REF, - lockedBy: "admin", - }); - // The consumed proposal is cleared. - expect(existsSync(proposedPath(rootDir))).toBe(false); - }); - - it("refuses to approve when there is no proposal", async () => { - const rootDir = root(); - await expect( - dispatchOpsOperation( - runtimeFor(rootDir), - { kind: "transitousLock.approve", ref: REF, approvedBy: "admin" }, - context(), - ), - ).rejects.toThrow(/no transitous lock proposal/i); + expect(existsSync(proposedPath(rootDir))).toBe(true); }); it("reads back an existing active lock", async () => { diff --git a/apps/ops-agent/src/transitous-lock-runtime.ts b/apps/ops-agent/src/transitous-lock-runtime.ts index 735071b96..17e6b745b 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.ts @@ -13,7 +13,7 @@ import { } from "node:fs"; import { dirname, join } from "node:path"; import type { OpsResultFor } from "@openmapx/core/ops"; -import type { OpsRuntime } from "./runtime"; +import { OpsNotWiredError, type OpsRuntime } from "./runtime"; // The catalog lock lives in the repository's `infra/docker/`, which only the // operations agent may write. Data-manager and the API previously bind-mounted @@ -134,28 +134,13 @@ export function createTransitousLockRuntime( return { ref: operation.ref, proposed: true as const }; }; - runtime["transitousLock.approve"] = async (operation) => { - const proposal = readLockAt(proposedPath); - if (!proposal) throw new Error("No Transitous lock proposal to approve"); - if (proposal.ref !== operation.ref) { - // The reviewer approved a different ref than the one now proposed. - throw new Error("Transitous lock proposal does not match the approved ref"); - } - const lockedAt = now().toISOString(); - atomicWriteJson(lockPath, { - ref: proposal.ref, - submodules: proposal.submodules, - lockedAt, - lockedBy: operation.approvedBy, - comment: "Approved after compatibility review and inactive-slot validation.", - }); - try { - unlinkSync(proposedPath); - } catch { - // The activation already committed; a leftover proposal is inert because - // `approve` re-reads and re-matches it. - } - return { ref: proposal.ref, lockedAt }; + // The legacy ref-only approval cannot prove paired generation/pointer ownership. + // Tasks02.3/05.3 replace this same operation with durable paired-pin admission. + runtime["transitousLock.approve"] = async () => { + const error = new OpsNotWiredError(); + error.message = + "Transitous approval requires the durable paired-pin publication upgrade; retain the proposal and inspect current status"; + throw error; }; return runtime; diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md new file mode 100644 index 000000000..549b13bdf --- /dev/null +++ b/docs/docs/developer/build-artifacts-control.md @@ -0,0 +1,116 @@ +# Build bundles and target authority + +`@openmapx/core/build-artifacts` is a server-only surface. It is intentionally absent +from the browser barrel. CLI bundle commands use the same bounded validator and +writer; uploaded archives never invoke shell extraction. + +## Bundle contract + +The five portable layouts are tileserver, OSRM, OTP, Pelias and MOTIS. Other services +must use their database or service-specific backup workflow. Safe version 1 records +remain readable, including optional `sourcePbf` metadata. Version 2 adds source +hashes and a SHA-256 over sorted `[path, sizeBytes, sha256]` tuples. Historical +version 1 records do not acquire invented source hashes. + +`validateBuildBundle` takes a readable stream, an empty private staging directory, +`ArtifactArchiveLimits`, and an abort signal. It returns the manifest, archive and +manifest digests, expanded file bytes and artifact fingerprint only after exact +membership and streamed digest verification. It accepts plain tar and gzip by +magic, bounds tar/PAX/GNU metadata before payload creation, and rejects links, +special files, duplicate/unsafe names, inconsistent sizes and trailing streams. +Files are created exclusively through descriptor-anchored parents with 0600/0700 +modes. Linux uses `/proc/self/fd`; macOS uses the held descriptor's `/.vol/dev/ino` +anchor. Unsupported platforms fail closed. + +The defaults are 1 TiB upload, 2 TiB expanded/entry bytes, 4 TiB stage policy, +250,000 entries, 64 MiB manifest, 4,096 path bytes, depth 64, 16 KiB extended +headers and expansion ratio 1,000 after 64 MiB. Aggregate stage reservations, +free-space/inode preflight, concurrent transfer admission and process memory +qualification belong to the transfer store; the archive module does not substitute +its byte limits for that store's reservations. + +MOTIS export requires `motis/live/portable-sources.json`, a strict object containing +`version: 1`, `sourceFingerprints: [{id, sha256, sizeBytes}]` and +`credentialRequirements: string[]`. This retained producer metadata contains no +credential values. Export retains source archives and the data subtree, omits +generated configuration/proxy files, and writes a version 2 manifest with +`runtimeMaterialization: "motis-target-credentials"`. Credential-bearing legacy +bundles fail with rebuild guidance. Import cannot declare readiness until the +future target credential materializer is installed; it must retain the original +bundle digest separately from the materialized fingerprint. + +## Installation and process boundaries + +Host `openmapx compose render` provisions exactly three Ed25519 role keys and one +public verifier bundle in `infra/docker/secrets`. Existing keys are validated and +reused; partial trust, changed roles, unsafe owners/modes, links and foreign +installation records are rejected. Keys and recovery records are never rotated +implicitly. `infra/docker/control-private/` stays outside shared data mounts. +The agent's checkout mount masks sibling keys and recovery directories. The +external helper masks all role keys and uses the agent recovery directory plus a +fresh certified child key. Docker/host administrators remain trusted. + +Services read their fixed role key and verifier mounts on startup. Missing or +conflicting trust disables signed-control admission with a fixed host recovery +prerequisite, while status/recovery access remains available. Recovery instructions +require stopping all relevant writers/helpers and inspecting exact retained +receipts and publication state; age or a missing PID never permits lock removal. + +## Execution and lease lifecycle + +Fixed adapters retain the complete admitted operation and immutable `ExecutionRef` +before effects. Both body and execution fingerprints remain distinct. A role-signed +owner certificate binds the physical data root and sorted target lock inodes; a +private ephemeral key signs its mutable journal. Assertions identify bounded signed +records, never paths or private keys. Each recipient independently verifies the +installation, role, complete chain, current tree, immutable identity, subset and +absolute deadline. Per-lease journal guards and private sequence high-water marks +reject conflicting writers and shared-record rollback. + +Use `acquireArtifactTargetLease`, `delegateArtifactTargetLease`, +`adoptArtifactTargetLease`, `settleArtifactTargetLeaseChild`, +`releaseArtifactTargetLease` and `reconcileArtifactTargetLease`. All verified +contexts are nominal. An inherited borrower cannot expand targets or acquire a +second root. `withArtifactTargetLease` checks forward permission and retains fences +on failure; its callback must produce durable own-effect evidence before release. +Acquisition has one five-second budget. The limits are 256 registered targets, +64 child intents, 128 KiB per lease ledger and 256 unsettled leases. Settled +receipts remain retained; this foundation performs no age-based garbage collection. + +Host ordering is target lease, nonblocking transit reservation, configuration +writer, then release store. A busy reservation must settle an unused owner without +effects; a borrower settles its own node and never releases the parent. No writer +is held while awaiting a child that needs that same writer. + +Only the fixed protocol graph is accepted, with borrowed depth at most two. Phase +keys derive from the original operation key and immutable plan ordinal. A lost +response never changes the phase key. Both peer directions and fixed CLI children +use the original identity/deadline. Fixed CLI spawn intents bind exact command +arguments and a fresh private child key; entry consumes a one-use start marker +before adoption can lead to effects. Unrelated subprocesses receive no authority +variables. Deadline-only Full Update forwarding owns no target lease. + +Settlement requires signed retained structured evidence from the fixed executor: +a parsed typed result, exact fixed-command completion, or an execution that has +never started effects. Errors alone produce unknown evidence. Arbitrary text, +caller booleans, IDs, expired deadlines and dead processes cannot release targets. +Private effect bodies remain outside shared journals. Known committed evidence +cannot be erased by a later response/journal failure. Startup restores original +admissions/adoptions and performs settlement from retained evidence, never dispatch. + +Terminal maintenance and handoff preparation retain ownership. The preparer can +certify preparation only by reading its exact succeeded journal entry. The helper +may adopt early but `verifyConfigurationHelperReady` refuses drain until that +signed terminal preparation exists. Maintenance continuations remain frozen, +one-use subsets; helper descendants have no further edge. + +## Deferred product consumers + +Task02.3 owns durable data-manager HTTP/row admission and containment. The existing +managed CLI mutation gate and data-manager control-header gate remain closed until +that receiver lands. Task01.2 owns maintenance decisions and orchestration; +Task05.3 owns durable paired-pin approval; Task08.3 owns configuration helper drain +and same-image recreation. Legacy ref-only approval now returns not-wired instead +of bypassing paired publication. These future consumers must use the fixed +constructors and original retained identities; named graph slots do not register +new executable operations or advertise transfer readiness. diff --git a/packages/cli/__tests__/cli-build-artifacts.test.ts b/packages/cli/__tests__/cli-build-artifacts.test.ts index 4e1e4fe96..035fa0acc 100644 --- a/packages/cli/__tests__/cli-build-artifacts.test.ts +++ b/packages/cli/__tests__/cli-build-artifacts.test.ts @@ -162,7 +162,11 @@ describe("build bundles", () => { execFileSync("tar", ["-xf", bundle, "-C", unpacked]); writeFileSync(join(unpacked, "osrm-graph", "region.osrm"), "TAMPERED"); const tampered = join(tmp, "tampered.tar"); - execFileSync("tar", ["-cf", tampered, "-C", unpacked, BUNDLE_MANIFEST_DIR, "osrm-graph"]); + execFileSync( + "tar", + ["--no-xattrs", "-cf", tampered, "-C", unpacked, BUNDLE_MANIFEST_DIR, "osrm-graph"], + { env: { ...process.env, COPYFILE_DISABLE: "1" } }, + ); await expect( importBuildBundle({ bundle: tampered, rootDir: serveHost, runningServices: noneRunning }), @@ -179,7 +183,11 @@ describe("build bundles", () => { execFileSync("tar", ["-xf", bundle, "-C", unpacked]); writeFileSync(join(unpacked, "osrm-graph", "extra"), "X"); const padded = join(tmp, "padded.tar"); - execFileSync("tar", ["-cf", padded, "-C", unpacked, BUNDLE_MANIFEST_DIR, "osrm-graph"]); + execFileSync( + "tar", + ["--no-xattrs", "-cf", padded, "-C", unpacked, BUNDLE_MANIFEST_DIR, "osrm-graph"], + { env: { ...process.env, COPYFILE_DISABLE: "1" } }, + ); await expect( importBuildBundle({ bundle: padded, rootDir: serveHost, runningServices: noneRunning }), @@ -209,11 +217,15 @@ describe("build bundles", () => { ); symlinkSync(outside, join(crafted, "osrm-graph")); const bundle = join(tmp, "crafted.tar"); - execFileSync("tar", ["-cf", bundle, "-C", crafted, BUNDLE_MANIFEST_DIR, "osrm-graph"]); + execFileSync( + "tar", + ["--no-xattrs", "-cf", bundle, "-C", crafted, BUNDLE_MANIFEST_DIR, "osrm-graph"], + { env: { ...process.env, COPYFILE_DISABLE: "1" } }, + ); await expect( importBuildBundle({ bundle, rootDir: serveHost, runningServices: noneRunning }), - ).rejects.toThrow(/Refusing symlink in build artifact: osrm-graph/); + ).rejects.toThrow(/Unsupported tar type/); expect(existsSync(join(dataDir(serveHost), "osrm-graph"))).toBe(false); }); @@ -262,46 +274,56 @@ describe("build bundles", () => { ).rejects.toThrow(/Refusing symlink in build artifact: osrm-graph\/link/); }); - it("exports an aliased MOTIS live dir as the directory it points at", async () => { + it("requires rebuilding legacy MOTIS output before exporting generated credentials", async () => { const data = dataDir(buildHost); - mkdirSync(join(data, "motis", "slots", "A", "data"), { recursive: true }); - writeFileSync(join(data, "motis", "slots", "A", "config.yml"), "osm: x.osm.pbf\n"); - writeFileSync(join(data, "motis", "slots", "A", "data", "tt.bin"), "TT"); - symlinkSync(join(data, "motis", "slots", "A"), join(data, "motis", "live")); - mkdirSync(join(data, "motis-feed-proxy", "conf"), { recursive: true }); - writeFileSync(join(data, "motis-feed-proxy", "conf", "default.conf"), "server {}\n"); + mkdirSync(join(data, "motis/slots/A/data"), { recursive: true }); + symlinkSync("slots/A", join(data, "motis/live")); + mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); + writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "secret-api-key"); writeBuildRecord(motisRecord(), buildHost); - - const bundle = join(tmp, "motis.tar"); - await exportBuildBundle({ serviceId: "motis", bundle, rootDir: buildHost }); - await importBuildBundle({ bundle, rootDir: serveHost, runningServices: noneRunning }); - - const live = join(dataDir(serveHost), "motis", "live"); - expect(readFileSync(join(live, "data", "tt.bin"), "utf-8")).toBe("TT"); - expect(readFileSync(join(live, "config.yml"), "utf-8")).toBe("osm: x.osm.pbf\n"); - expect(existsSync(join(dataDir(serveHost), "motis-feed-proxy", "conf", "default.conf"))).toBe( - true, - ); + await expect( + exportBuildBundle({ serviceId: "motis", bundle: join(tmp, "motis.tar"), rootDir: buildHost }), + ).rejects.toThrow(/retained portable source metadata/); }); - it("imports MOTIS into the active slot when the serving host uses the slot layout", async () => { + it("keeps the active MOTIS slot unchanged until target credential materialization exists", async () => { const source = dataDir(buildHost); - mkdirSync(join(source, "motis", "live", "data"), { recursive: true }); - writeFileSync(join(source, "motis", "live", "data", "tt.bin"), "NEW"); + mkdirSync(join(source, "motis/live/data"), { recursive: true }); mkdirSync(join(source, "motis-feed-proxy"), { recursive: true }); + writeFileSync(join(source, "motis/live/data/tt.bin"), "NEW"); + writeFileSync( + join(source, "motis/live/portable-sources.json"), + JSON.stringify({ version: 1, sourceFingerprints: [], credentialRequirements: [] }), + ); writeBuildRecord(motisRecord(), buildHost); const bundle = join(tmp, "motis.tar"); await exportBuildBundle({ serviceId: "motis", bundle, rootDir: buildHost }); - const target = dataDir(serveHost); - const slotA = join(target, "motis", "slots", "A"); + const slotA = join(target, "motis/slots/A"); mkdirSync(join(slotA, "data"), { recursive: true }); - writeFileSync(join(slotA, "data", "tt.bin"), "OLD"); - symlinkSync(slotA, join(target, "motis", "live")); - - await importBuildBundle({ bundle, rootDir: serveHost, runningServices: noneRunning }); - - expect(lstatSync(join(target, "motis", "live")).isSymbolicLink()).toBe(true); - expect(readFileSync(join(slotA, "data", "tt.bin"), "utf-8")).toBe("NEW"); + writeFileSync(join(slotA, "data/tt.bin"), "OLD"); + symlinkSync(slotA, join(target, "motis/live")); + await expect( + importBuildBundle({ bundle, rootDir: serveHost, runningServices: noneRunning }), + ).rejects.toThrow(/target credential materialization/); + expect(lstatSync(join(target, "motis/live")).isSymbolicLink()).toBe(true); + expect(readFileSync(join(slotA, "data/tt.bin"), "utf8")).toBe("OLD"); }); }); + +it("validates the manifest before invoking any legacy extraction hook", async () => { + const bundle = join(tmp, "invalid.tar"); + writeFileSync(bundle, Buffer.alloc(1024)); + let extracted = false; + await expect( + importBuildBundle({ + bundle, + rootDir: serveHost, + runningServices: noneRunning, + tar: async () => { + extracted = true; + }, + }), + ).rejects.toThrow(); + expect(extracted, "untrusted archive reached extract-first tar hook").toBe(false); +}); diff --git a/packages/cli/__tests__/cli-control-context.test.ts b/packages/cli/__tests__/cli-control-context.test.ts new file mode 100644 index 000000000..df5d4563c --- /dev/null +++ b/packages/cli/__tests__/cli-control-context.test.ts @@ -0,0 +1,107 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as c from "@openmapx/core/build-artifacts"; +import { executionFingerprint, opsOperationFingerprint } from "@openmapx/core/ops"; +import { expect, it, vi } from "vitest"; +import { completeCliCommand, withCliControlContext } from "../src/lib/control-context"; + +it("settles successful explicit CLI exits once through the real control context", async () => { + const repo = mkdtempSync(join(tmpdir(), "omx-fixed-cli-entry-")); + const infra = join(repo, "infra/docker"); + mkdirSync(join(infra, "data"), { recursive: true, mode: 0o700 }); + await c.provisionControlInstallation(infra); + const installation = c.openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["motis"], + }); + const operation = { kind: "service.start" as const, serviceId: "motis" }; + const operationFingerprint = opsOperationFingerprint(operation); + const deadlineAt = Date.now() + 60000; + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const child = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: c.controlChildOperationKey(execution.operationKey, "lifecycle", 0), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + parent: c.retainedControlParent(execution), + }), + }; + const plan = { + phase: "lifecycle" as const, + ordinal: 0, + kind: operation.kind, + execution: child, + serviceIds: ["motis"], + recipient: "fixed-cli" as const, + }; + const authority = c.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt: deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await c.acquireArtifactTargetLease({ + dataRoot: join(infra, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const delegationId = randomUUID(); + const publicKey = c.prepareFixedControlChild(authority, delegationId); + const assertion = await c.delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: publicKey, + }); + const args = c.fixedCliArguments(operation); + c.retainFixedCliSpawnIntent(authority, { delegationId, execution: child, operation, args }); + vi.stubEnv("OPENMAPX_ROOT_DIR", repo); + vi.stubEnv("OPENMAPX_CONTROL_EXECUTION", JSON.stringify(child)); + vi.stubEnv("OPENMAPX_CONTROL_DELEGATION", assertion); + vi.stubEnv("OPENMAPX_PARENT_EXECUTION", undefined); + let calls = 0; + try { + await withCliControlContext(args, async () => { + calls++; + completeCliCommand(0); + }); + expect(calls).toBe(1); + await expect( + withCliControlContext(args, async () => { + calls++; + }), + ).rejects.toThrow(); + expect(calls).toBe(1); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await c.settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, execution), + ); + await c.releaseArtifactTargetLease(lease); + } finally { + vi.unstubAllEnvs(); + c.closeControlInstallation(installation); + rmSync(repo, { recursive: true, force: true }); + } +}); diff --git a/packages/cli/__tests__/cli-execution-deadline.test.ts b/packages/cli/__tests__/cli-execution-deadline.test.ts index d4f82dc1e..73b95583d 100644 --- a/packages/cli/__tests__/cli-execution-deadline.test.ts +++ b/packages/cli/__tests__/cli-execution-deadline.test.ts @@ -141,7 +141,7 @@ it("fix1 unsigned metadata can only clamp policy and cannot authorize a remote m vi.stubEnv("OPENMAPX_PARENT_EXECUTION", "unsigned-parent"); await expect( withCliExecutionDeadline(["services", "build"], async () => undefined), - ).rejects.toThrow("CLI control verifier is unavailable"); + ).rejects.toThrow(); } finally { vi.useRealTimers(); vi.unstubAllEnvs(); diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index 8adca3113..f9b4c3a38 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -1,5 +1,7 @@ import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; import { join } from "node:path"; +import { provisionControlInstallation } from "@openmapx/core/build-artifacts"; import { initializeErasureJournal } from "@openmapx/core/erasure-journal"; import { services as coreServices } from "@openmapx/core/server"; import { type Command, Option } from "commander"; @@ -195,6 +197,8 @@ export async function renderComposeForRepo(opts: RenderRepoOptions): Promise { assertCliExecutionAvailable(); ensurePlatformPrivateDirectory(join(paths.infraDir, "data", "ops-agent", "trusted-config")); diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index ba2c70a94..bc5579a27 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -9,6 +9,7 @@ import { exportBuildBundle, importBuildBundle, } from "../lib/build-artifacts"; +import { completeCliCommand } from "../lib/control-context"; import { applyDataCleanup, collectOfflineDataStatus, @@ -280,14 +281,14 @@ export function registerDataCommands(program: Command): void { await runFontDownload(transport); } else { log.err(`Unknown kind: ${kind} (use osm | gtfs | fonts)`); - process.exit(1); + completeCliCommand(1); } } catch (err) { log.err(`download failed: ${(err as Error).message}`); if (!(err as Error).message.startsWith("region required")) { dataManagerHint(); } - process.exit(1); + completeCliCommand(1); } }); @@ -301,7 +302,7 @@ export function registerDataCommands(program: Command): void { const serviceId = resolveDataBuildServiceId(kind); if (!serviceId) { log.err(`Unknown kind: ${kind} (use: motis | osrm | otp | pelias | tileserver)`); - process.exit(1); + completeCliCommand(1); } const result = await buildServices({ mode: "explicit", @@ -312,7 +313,7 @@ export function registerDataCommands(program: Command): void { await renderAndApplyHardlinks(); } catch (err) { log.err(`build failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }); @@ -338,7 +339,7 @@ export function registerDataCommands(program: Command): void { ); } catch (err) { log.err(`export failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }); @@ -353,7 +354,7 @@ export function registerDataCommands(program: Command): void { manifest = await importBuildBundle({ bundle, runningServices: runningComposeServices }); } catch (err) { log.err(`import failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } log.ok( `Imported ${manifest.service} (${manifest.files.length} files, built ${manifest.builtAt}${manifest.revision ? ` at ${manifest.revision.slice(0, 12)}` : ""})`, @@ -364,7 +365,7 @@ export function registerDataCommands(program: Command): void { log.err( `The artifact is in place, but rendering and linking failed: ${(err as Error).message}. Fix that, then run \`openmapx data link\`.`, ); - process.exit(1); + completeCliCommand(1); } log.info( `Start the consumers with \`openmapx services start ${buildArtifactSpec(manifest.service).consumers.join(" ")}\``, @@ -394,7 +395,7 @@ export function registerDataCommands(program: Command): void { if (!(err as Error).message.startsWith("region required")) { dataManagerHint(); } - process.exit(1); + completeCliCommand(1); } try { @@ -410,13 +411,13 @@ export function registerDataCommands(program: Command): void { log.err( `update completed with ${buildResult.failures.length} build failure${buildResult.failures.length === 1 ? "" : "s"}`, ); - process.exit(1); + completeCliCommand(1); } log.ok("Update complete"); } catch (err) { log.err(`update failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }, ); @@ -447,12 +448,12 @@ export function registerDataCommands(program: Command): void { log.ok(`Converted → ${r.targetBz2} (${(r.sizeBytes / 1e9).toFixed(2)} GB)`); } else { log.err(`Unknown kind: ${kind} (use: overpass)`); - process.exit(1); + completeCliCommand(1); } } catch (err) { log.err(`convert failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -474,7 +475,7 @@ export function registerDataCommands(program: Command): void { ); } catch (err) { log.err(`link failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }); @@ -491,7 +492,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`sync failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -511,7 +512,7 @@ export function registerDataCommands(program: Command): void { log.err( `search-index build requires a region (for example europe/germany) or $${OPENMAPX_REGION_ENV}`, ); - process.exit(1); + completeCliCommand(1); return; } try { @@ -526,7 +527,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`search-index build failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -562,7 +563,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`search-index status failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -586,7 +587,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`notable-places build failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -620,7 +621,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`notable-places status failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -658,7 +659,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`source list failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -683,7 +684,7 @@ export function registerDataCommands(program: Command): void { ) => { if (!options.licenseSpdx && !options.licenseUrl) { log.err("source add requires --license-spdx or --license-url"); - process.exit(1); + completeCliCommand(1); } try { const result = dm.unwrapDataManagerResponse( @@ -705,7 +706,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`source add failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }, ); @@ -722,7 +723,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`source remove failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -738,7 +739,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`source enable failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -800,7 +801,7 @@ export function registerDataCommands(program: Command): void { } } catch (err) { log.err(`clean failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }); @@ -831,7 +832,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`overture-sync failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -854,7 +855,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`overture-pull failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -877,7 +878,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`overture-ingest failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -901,7 +902,7 @@ export function registerDataCommands(program: Command): void { log.err( `overture-conflate failed: ${result.message ?? result.status ?? "unknown error"}`, ); - process.exit(1); + completeCliCommand(1); } if (result.status === "already_running") { log.warn( @@ -926,7 +927,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`overture-conflate failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -941,7 +942,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`overture-status failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -964,7 +965,7 @@ export function registerDataCommands(program: Command): void { } catch (err) { log.err(`overture-extract failed: ${(err as Error).message}`); dataManagerHint(); - process.exit(1); + completeCliCommand(1); } }); @@ -1000,7 +1001,7 @@ export function registerDataCommands(program: Command): void { log.dim("Fill in missing values, then run: openmapx data sync"); } catch (err) { log.err(`generate-api-keys failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }); diff --git a/packages/cli/src/commands/services.ts b/packages/cli/src/commands/services.ts index e3a0abef0..31aad23fc 100644 --- a/packages/cli/src/commands/services.ts +++ b/packages/cli/src/commands/services.ts @@ -1,6 +1,7 @@ import { services as coreServices } from "@openmapx/core/server"; import type { Command } from "commander"; import kleur from "kleur"; +import { completeCliCommand } from "../lib/control-context"; import { dockerComposeStream } from "../lib/docker"; import { applyGeneratedHardlinks } from "../lib/hardlinks"; import { log, table } from "../lib/output"; @@ -181,7 +182,7 @@ function mergeIdsWithPresets(ids: string[], preset: string | undefined): string[ } catch (err) { if (err instanceof UnknownPresetError) { log.err(err.message); - process.exit(1); + completeCliCommand(1); } throw err; } @@ -202,7 +203,7 @@ export function registerServicesCommands(program: Command): void { for (const warning of applied.selection.warnings) log.warn(warning); } catch (err) { log.err((err as Error).message); - process.exit(1); + completeCliCommand(1); } }); @@ -221,7 +222,7 @@ export function registerServicesCommands(program: Command): void { if ((err as Error).message.includes(SERVICE_SELECTION_ENV)) { log.dim(`Selection file was not changed.`); } - process.exit(1); + completeCliCommand(1); } }); @@ -240,7 +241,7 @@ export function registerServicesCommands(program: Command): void { if ((err as Error).message.includes(SERVICE_SELECTION_ENV)) { log.dim(`Selection file was not changed.`); } - process.exit(1); + completeCliCommand(1); } }); @@ -273,7 +274,7 @@ export function registerServicesCommands(program: Command): void { const svc = list.find((s) => s.manifest.id === id); if (!svc) { log.err(`Service not found: ${id}`); - process.exit(1); + completeCliCommand(1); } console.log(JSON.stringify(svc.manifest, null, 2)); }); @@ -307,11 +308,11 @@ export function registerServicesCommands(program: Command): void { log.err( `build-all completed with ${result.failures.length} failure${result.failures.length === 1 ? "" : "s"}`, ); - process.exit(1); + completeCliCommand(1); } } catch (err) { log.err(`build-all failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }); @@ -357,11 +358,11 @@ export function registerServicesCommands(program: Command): void { log.err( `build completed with ${result.failures.length} failure${result.failures.length === 1 ? "" : "s"}`, ); - process.exit(1); + completeCliCommand(1); } } catch (err) { log.err(`build failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } }, ); @@ -379,7 +380,7 @@ export function registerServicesCommands(program: Command): void { const allIds = mergeIdsWithPresets(ids, options.preset); if (allIds.length === 0) { log.err("No services selected. Pass or --preset."); - process.exit(1); + completeCliCommand(1); } try { // Don't pass `services: allIds` — that would narrow the persisted @@ -397,7 +398,7 @@ export function registerServicesCommands(program: Command): void { ); } catch (err) { log.err(`prepare/start failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } if (touchesReleasePinnedServices(allIds)) { const overlay = await ensureReleaseOverlay(); @@ -407,13 +408,13 @@ export function registerServicesCommands(program: Command): void { for (const line of releaseStatusLines(overlay.path)) log.dim(line); } else if (overlay.status === "unpinned") { log.err(unpinnedReleaseWarning(overlay.reason)); - process.exit(1); + completeCliCommand(1); } else if (overlay.status === "disabled") { log.dim("Release pinning disabled (OPENMAPX_RELEASE=local); using manifest image tags."); } } const code = await dockerComposeStream(["up", "-d", ...allIds]); - process.exit(code); + completeCliCommand(code); }); services @@ -424,10 +425,10 @@ export function registerServicesCommands(program: Command): void { const allIds = mergeIdsWithPresets(ids, options.preset); if (allIds.length === 0) { log.err("No services selected. Pass or --preset."); - process.exit(1); + completeCliCommand(1); } const code = await dockerComposeStream(["stop", ...allIds]); - process.exit(code); + completeCliCommand(code); }); services @@ -440,10 +441,10 @@ export function registerServicesCommands(program: Command): void { const allIds = mergeIdsWithPresets(ids, options.preset); if (allIds.length === 0) { log.err("No services selected. Pass or --preset."); - process.exit(1); + completeCliCommand(1); } const code = await dockerComposeStream(["restart", ...allIds]); - process.exit(code); + completeCliCommand(code); }); services @@ -457,7 +458,7 @@ export function registerServicesCommands(program: Command): void { const allIds = mergeIdsWithPresets(ids, options.preset); if (allIds.length === 0) { log.err("No services selected. Pass or --preset."); - process.exit(1); + completeCliCommand(1); } try { // Render the operator's selection (env / service-selection.json), NOT @@ -473,7 +474,7 @@ export function registerServicesCommands(program: Command): void { ); } catch (err) { log.err(`prepare/update failed: ${(err as Error).message}`); - process.exit(1); + completeCliCommand(1); } let releasePinned = false; @@ -487,7 +488,7 @@ export function registerServicesCommands(program: Command): void { for (const line of releaseStatusLines(overlay.path)) log.dim(line); } else if (overlay.status === "unpinned") { log.err(unpinnedReleaseWarning(overlay.reason)); - process.exit(1); + completeCliCommand(1); } else if (overlay.status === "disabled") { log.warn( "Release pinning is disabled (OPENMAPX_RELEASE=local); this update uses the manifest image tags instead of a digest-pinned release.", @@ -499,7 +500,7 @@ export function registerServicesCommands(program: Command): void { if (pullCode !== 0) { if (releasePinned) { log.err("Could not pull the pinned release images; refusing to recreate services."); - process.exit(pullCode); + completeCliCommand(pullCode); } log.warn("Some images could not be pulled (service may be locally built). Continuing."); } @@ -511,7 +512,7 @@ export function registerServicesCommands(program: Command): void { ...(options.deps === false ? ["--no-deps"] : []), ...allIds, ]); - process.exit(code); + completeCliCommand(code); }); services @@ -520,7 +521,7 @@ export function registerServicesCommands(program: Command): void { .action(async (id?: string) => { const args = id ? ["ps", id] : ["ps"]; const code = await dockerComposeStream(args); - process.exit(code); + completeCliCommand(code); }); services @@ -533,7 +534,7 @@ export function registerServicesCommands(program: Command): void { if (options.follow) args.push("-f"); args.push(id); const code = await dockerComposeStream(args); - process.exit(code); + completeCliCommand(code); }); services diff --git a/packages/cli/src/lib/build-artifacts.ts b/packages/cli/src/lib/build-artifacts.ts index 77f33760f..fb16ae2a0 100644 --- a/packages/cli/src/lib/build-artifacts.ts +++ b/packages/cli/src/lib/build-artifacts.ts @@ -1,103 +1,50 @@ -import { createHash } from "node:crypto"; import { createReadStream, + createWriteStream, existsSync, lstatSync, mkdirSync, mkdtempSync, - readdirSync, readFileSync, realpathSync, rmSync, writeFileSync, } from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, isAbsolute, join } from "node:path"; -import { services as coreServices } from "@openmapx/core/server"; +import { dirname, join, relative } from "node:path"; +import { Writable } from "node:stream"; +import { + BUILD_RECORD_DIR, + type BuildRecord, + type BuildRecordV2, + type BundleManifest, + buildArtifactSpec, + DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + safeArtifactPath, + runtimeImageMismatches as sharedRuntimeImageMismatches, + validateBuildBundle, + writeBuildBundle, +} from "@openmapx/core/build-artifacts"; import { execa } from "execa"; import { promoteStagedBuildDirs } from "./build-staging"; -import { cliProcessOptions } from "./execution-deadline"; +import { + assertCliExecutionAvailable, + cliAcquisitionOptions, + cliProcessOptions, +} from "./execution-deadline"; import { repoPaths, resolveInvocationPath } from "./paths"; -/** - * What a `services build ` run leaves behind, and what has to travel when - * the build ran on a different host than the one serving it. - */ -export interface BuildArtifactSpec { - /** Output dirs relative to `infra/docker/data/`. */ - dirs: string[]; - /** - * Build inputs staged into the output dir that the runtime never reads. - * Matched against the path relative to the data root: a trailing `/` marks - * a directory prefix, a leading `*` a filename suffix. - */ - exclude: string[]; - /** Compose services that read the output; none may run during a swap. */ - consumers: string[]; -} - -export const BUILD_ARTIFACTS: Readonly> = { - tileserver: { dirs: ["tile-mbtiles"], exclude: [], consumers: ["tileserver"] }, - osrm: { - dirs: ["osrm-graph"], - exclude: ["osrm-graph/region.osm.pbf"], - consumers: ["osrm"], - }, - otp: { dirs: ["otp-graph"], exclude: ["*.pbf", "*.zip"], consumers: ["otp"] }, - pelias: { - dirs: ["pelias"], - exclude: ["pelias/openstreetmap/"], - consumers: ["pelias", "pelias-pip", "pelias-placeholder", "elasticsearch"], - }, - // `motis import` re-verifies every input named in config.yml on start, so - // the timetables and PBF travel with the import output. `motis-staging` can - // repoint or write the slot `motis/live` aliases, so it must be down too. - motis: { - dirs: ["motis/live", "motis-feed-proxy"], - exclude: [], - consumers: ["motis", "motis-staging"], - }, -}; - -export function buildArtifactSpec(serviceId: string): BuildArtifactSpec { - const spec = Object.hasOwn(BUILD_ARTIFACTS, serviceId) ? BUILD_ARTIFACTS[serviceId] : undefined; - if (!spec) throw new Error(`Service "${serviceId}" has no exportable build artifact`); - return spec; -} - -export const BUILD_RECORD_DIR = ".openmapx-builds"; -const BUNDLE_MANIFEST_DIR = ".openmapx-bundle"; -export const BUNDLE_MANIFEST_PATH = `${BUNDLE_MANIFEST_DIR}/manifest.json`; -const IMPORT_STAGING_DIR = ".openmapx-import"; - -export interface BuildRecord { - schemaVersion: 1; - service: string; - region?: string; - sourcePbf?: { name: string; sizeBytes: number }; - builtAt: string; - /** Git commit of the OpenMapX checkout that ran the build, when known. */ - revision?: string; - /** - * Runtime images whose on-disk format the artifact is bound to, keyed by - * compose service id. A serving host must run exactly these to load it. - */ - runtimeImages: Record; - /** Helper images the build ran; informational. */ - toolImages: Record; -} - -export interface BundleFile { - path: string; - sizeBytes: number; - sha256: string; -} - -export interface BundleManifest extends BuildRecord { - dirs: string[]; - directories: string[]; - files: BundleFile[]; -} +export { + BUILD_ARTIFACTS, + BUILD_RECORD_DIR, + BUNDLE_MANIFEST_PATH, + type BuildArtifactSpec, + type BuildRecord, + type BuildRecordV2, + type BundleFile, + type BundleManifest, + buildArtifactSpec, + type ImageMismatch, +} from "@openmapx/core/build-artifacts"; function dataRootFor(rootDir?: string): string { return join(repoPaths(rootDir).infraDir, "data"); @@ -107,17 +54,20 @@ function buildRecordPath(dataRoot: string, serviceId: string): string { return join(dataRoot, BUILD_RECORD_DIR, `${serviceId}.json`); } -export function writeBuildRecord(record: BuildRecord, rootDir?: string): string { +export function writeBuildRecord(record: BuildRecord | BuildRecordV2, rootDir?: string): string { const path = buildRecordPath(dataRootFor(rootDir), record.service); mkdirSync(join(dataRootFor(rootDir), BUILD_RECORD_DIR), { recursive: true }); writeFileSync(path, `${JSON.stringify(record, null, 2)}\n`, "utf-8"); return path; } -export function readBuildRecord(serviceId: string, rootDir?: string): BuildRecord | undefined { +export function readBuildRecord( + serviceId: string, + rootDir?: string, +): BuildRecord | BuildRecordV2 | undefined { const path = buildRecordPath(dataRootFor(rootDir), serviceId); if (!existsSync(path)) return undefined; - return JSON.parse(readFileSync(path, "utf-8")) as BuildRecord; + return JSON.parse(readFileSync(path, "utf-8")) as BuildRecord | BuildRecordV2; } export async function currentRevision(rootDir?: string): Promise { @@ -130,314 +80,108 @@ export async function currentRevision(rootDir?: string): Promise { - if (pattern.startsWith("*")) return !isDir && relPath.endsWith(pattern.slice(1)); - if (pattern.endsWith("/")) return asDir.startsWith(pattern); - return relPath === pattern; - }); -} - -interface WalkedTree { - directories: string[]; - files: string[]; -} - -/** - * List every directory and regular file under `dirs` (relative to `base`), - * applying the spec's excludes. Anything else (symlinks, sockets, devices) is - * refused: a bundle carries plain files only, so it can't plant a link. - */ -function walkArtifact( - base: string, - dirs: readonly string[], - exclude: readonly string[], - opts: { allowAliasedRoot: boolean }, -): WalkedTree { - const tree: WalkedTree = { directories: [], files: [] }; - const visit = (rel: string, abs: string) => { - const stat = lstatSync(abs); - if (stat.isSymbolicLink()) throw new Error(`Refusing symlink in build artifact: ${rel}`); - if (stat.isDirectory()) { - if (isExcluded(rel, true, exclude)) return; - tree.directories.push(rel); - for (const name of readdirSync(abs).sort()) visit(`${rel}/${name}`, join(abs, name)); - return; - } - if (!stat.isFile()) throw new Error(`Refusing non-regular file in build artifact: ${rel}`); - if (!isExcluded(rel, false, exclude)) tree.files.push(rel); - }; - for (const dir of dirs) { - const abs = join(base, dir); - if (!existsSync(abs)) throw new Error(`Build artifact dir is missing: ${dir}`); - // On the exporting host the artifact root may be an alias (the MOTIS slot - // layout points `motis/live` at the active slot). An extracted bundle has - // no such excuse: every segment down to the root must be a real dir. - if (opts.allowAliasedRoot) { - visit(dir, realpathSync(abs)); - continue; - } - let segmentPath = base; - for (const segment of dir.split("/")) { - segmentPath = join(segmentPath, segment); - if (lstatSync(segmentPath).isSymbolicLink()) { - throw new Error(`Refusing symlink in build artifact: ${dir}`); - } - } - visit(dir, abs); - } - return tree; -} - -function sha256OfFile(path: string): Promise { - return new Promise((resolveHash, rejectHash) => { - const hash = createHash("sha256"); - createReadStream(path) - .on("data", (chunk) => hash.update(chunk)) - .on("end", () => resolveHash(hash.digest("hex"))) - .on("error", rejectHash); - }); -} - -function isSafeRelPath(rel: string): boolean { - if (!rel || isAbsolute(rel) || rel.includes("\\")) return false; - return rel.split("/").every((segment) => segment !== "" && segment !== "." && segment !== ".."); -} - -function bundlePath(bundle: string): string { - return bundle === "-" ? bundle : resolveInvocationPath(bundle); -} - +/** @deprecated Archive subprocesses are no longer used; kept for source compatibility. */ export type BundleCommandRunner = ( args: string[], opts: { cwd?: string; stdin?: "inherit"; stdout?: "inherit" }, ) => Promise; - -async function defaultTar( - args: string[], - opts: { cwd?: string; stdin?: "inherit"; stdout?: "inherit" }, -): Promise { - const executionOptions = cliProcessOptions(); - await execa("tar", args, { - ...executionOptions, - cwd: opts.cwd, - // Stops macOS tar from adding `._*` AppleDouble members. - env: { ...executionOptions.env, COPYFILE_DISABLE: "1" }, - stdin: opts.stdin ?? "ignore", - stdout: opts.stdout ?? "inherit", - stderr: "inherit", - }); -} - export interface ExportBuildBundleOptions { serviceId: string; - /** Destination file, or `-` for stdout. */ bundle: string; rootDir?: string; tar?: BundleCommandRunner; } - export async function exportBuildBundle(opts: ExportBuildBundleOptions): Promise { - const spec = buildArtifactSpec(opts.serviceId); const dataRoot = dataRootFor(opts.rootDir); const record = readBuildRecord(opts.serviceId, opts.rootDir); - if (!record) { - throw new Error( - `No build record for "${opts.serviceId}". Run \`openmapx services build ${opts.serviceId}\` on this host first.`, + if (!record) + throw Error( + `No build record for "${opts.serviceId}". Run openmapx services build ${opts.serviceId} first.`, ); - } if ( opts.serviceId === "motis" && !record.runtimeImages.motis && - existsSync(join(dataRoot, "motis", "live", "data")) - ) { - throw new Error( - "motis/live holds import output, but the last build ran without --import, so it isn't tied to a MOTIS image. Rebuild with `openmapx services build motis --import` before exporting.", + existsSync(join(dataRoot, "motis/live/data")) + ) + throw Error( + "MOTIS build ran without --import; rebuild with openmapx services build motis --import", ); - } - const tree = walkArtifact(dataRoot, spec.dirs, spec.exclude, { allowAliasedRoot: true }); - const files: BundleFile[] = []; - for (const rel of tree.files) { - const abs = join(dataRoot, rel); - files.push({ path: rel, sizeBytes: lstatSync(abs).size, sha256: await sha256OfFile(abs) }); - } - const manifest: BundleManifest = { - ...record, - dirs: spec.dirs, - directories: tree.directories, - files, - }; - - // bsdtar stops reading options at the first operand, so everything is - // archived from the data root in one `-C`, manifest included. - const manifestDir = join(dataRoot, BUNDLE_MANIFEST_DIR); - const scratch = mkdtempSync(join(tmpdir(), "openmapx-export-")); + const target = + opts.bundle === "-" + ? new Writable({ + write(chunk, encoding, done) { + process.stdout.write(chunk, encoding, done); + }, + }) + : createWriteStream(resolveInvocationPath(opts.bundle), { flags: "wx", mode: 0o600 }); + target.on("error", () => undefined); try { - mkdirSync(manifestDir, { recursive: true }); - writeFileSync(join(dataRoot, BUNDLE_MANIFEST_PATH), `${JSON.stringify(manifest)}\n`); - const listPath = join(scratch, "members.txt"); - // Manifest first so an importer meets it before the payload; members are - // listed explicitly (no recursion) so the archive matches it exactly. - const members = [BUNDLE_MANIFEST_PATH, ...tree.directories, ...tree.files]; - writeFileSync(listPath, `${members.join("\n")}\n`); - const tar = opts.tar ?? defaultTar; - // `-h` stores an aliased artifact root as the directory it points at. - await tar( - [ - "-chf", - bundlePath(opts.bundle), - "--no-recursion", - "--no-xattrs", - "-C", + return ( + await writeBuildBundle({ dataRoot, - "-T", - listPath, - ], - { stdout: opts.bundle === "-" ? "inherit" : undefined }, - ); - } finally { - rmSync(scratch, { recursive: true, force: true }); - rmSync(manifestDir, { recursive: true, force: true }); + serviceId: opts.serviceId, + record, + target, + limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: cliAcquisitionOptions().signal ?? new AbortController().signal, + }) + ).manifest; + } catch (e) { + if (target !== process.stdout) target.destroy(); + throw e; } - return manifest; } - -function parseBundleManifest(text: string): BundleManifest { - const parsed = JSON.parse(text) as Partial; - if ( - parsed.schemaVersion !== 1 || - typeof parsed.service !== "string" || - !Array.isArray(parsed.dirs) || - !Array.isArray(parsed.directories) || - !Array.isArray(parsed.files) || - typeof parsed.runtimeImages !== "object" || - parsed.runtimeImages === null - ) { - throw new Error("Bundle manifest is malformed"); - } - return parsed as BundleManifest; +export async function runtimeImageMismatches(images: Record, rootDir?: string) { + return sharedRuntimeImageMismatches(images, repoPaths(rootDir).root); } - -/** - * The extracted tree must be exactly what the manifest lists: same dirs, same - * files, same bytes. Anything extra, missing, or altered fails the import. - */ -async function verifyExtractedBundle( - stagingRoot: string, - manifest: BundleManifest, - spec: BuildArtifactSpec, -): Promise { - if (JSON.stringify([...manifest.dirs].sort()) !== JSON.stringify([...spec.dirs].sort())) { - throw new Error( - `Bundle dirs [${manifest.dirs.join(", ")}] do not match "${manifest.service}" artifact dirs [${spec.dirs.join(", ")}]`, - ); - } - const inArtifact = (rel: string) => - isSafeRelPath(rel) && spec.dirs.some((dir) => rel === dir || rel.startsWith(`${dir}/`)); - for (const rel of [...manifest.directories, ...manifest.files.map((f) => f.path)]) { - if (!inArtifact(rel)) throw new Error(`Bundle lists a path outside its artifact: ${rel}`); - } - - const tree = walkArtifact(stagingRoot, spec.dirs, [], { allowAliasedRoot: false }); - const extractedDirs = new Set(tree.directories); - const missingDir = manifest.directories.find((rel) => !extractedDirs.has(rel)); - if (missingDir) throw new Error(`Bundle is missing ${missingDir}/`); - const expectedDirs = new Set(manifest.directories); - const expectedFiles = new Map(manifest.files.map((file) => [file.path, file])); - const unexpected = [ - ...tree.directories.filter((rel) => !expectedDirs.has(rel)), - ...tree.files.filter((rel) => !expectedFiles.has(rel)), - ]; - if (unexpected.length > 0) { - throw new Error(`Bundle contains entries its manifest does not list: ${unexpected[0]}`); - } - const extracted = new Set(tree.files); - for (const file of manifest.files) { - if (!extracted.has(file.path)) throw new Error(`Bundle is missing ${file.path}`); - const abs = join(stagingRoot, file.path); - if (lstatSync(abs).size !== file.sizeBytes || (await sha256OfFile(abs)) !== file.sha256) { - throw new Error(`Bundle file failed verification: ${file.path}`); - } - } -} - -export interface ImageMismatch { - service: string; - bundle: string; - local: string; -} - -export async function runtimeImageMismatches( - runtimeImages: Record, - rootDir?: string, -): Promise { - const registry = new coreServices.ServiceRegistry({ rootDir: repoPaths(rootDir).root }); - await registry.load(); - const mismatches: ImageMismatch[] = []; - for (const [service, bundle] of Object.entries(runtimeImages)) { - const loaded = registry.get(service); - const local = loaded - ? coreServices.serviceContainerImageReference(loaded.manifest.container) - : "(not installed)"; - if (local !== bundle) mismatches.push({ service, bundle, local }); - } - return mismatches; -} - export interface ImportBuildBundleOptions { - /** Source file, or `-` for stdin. */ bundle: string; rootDir?: string; tar?: BundleCommandRunner; - /** Compose services currently running; injected by the CLI. */ runningServices: (ids: string[]) => Promise; } - export async function importBuildBundle(opts: ImportBuildBundleOptions): Promise { const dataRoot = dataRootFor(opts.rootDir); - const stagingRoot = join(dataRoot, IMPORT_STAGING_DIR); - // Extract inside the data root so the final swap is a same-filesystem rename. - rmSync(stagingRoot, { recursive: true, force: true }); - mkdirSync(stagingRoot, { recursive: true }); + mkdirSync(dataRoot, { recursive: true }); + const stagingRoot = mkdtempSync(join(dataRoot, ".openmapx-import-")); try { - const tar = opts.tar ?? defaultTar; - await tar(["-xf", bundlePath(opts.bundle), "-C", stagingRoot, "--no-same-owner"], { - stdin: opts.bundle === "-" ? "inherit" : undefined, + const { manifest } = await validateBuildBundle({ + source: + opts.bundle === "-" ? process.stdin : createReadStream(resolveInvocationPath(opts.bundle)), + stagingRoot, + limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: cliAcquisitionOptions().signal ?? new AbortController().signal, }); - const manifestPath = join(stagingRoot, BUNDLE_MANIFEST_PATH); - if (!existsSync(manifestPath)) throw new Error(`Bundle has no ${BUNDLE_MANIFEST_PATH}`); - const manifest = parseBundleManifest(readFileSync(manifestPath, "utf-8")); const spec = buildArtifactSpec(manifest.service); - await verifyExtractedBundle(stagingRoot, manifest, spec); - const mismatches = await runtimeImageMismatches(manifest.runtimeImages, opts.rootDir); - if (mismatches.length > 0) { - const detail = mismatches - .map((m) => `${m.service}: bundle ${m.bundle}, this host ${m.local}`) - .join("; "); - throw new Error( - `Bundle was built for different runtime images (${detail}). Build and serve from the same OpenMapX revision.`, + if (mismatches.length) + throw Error( + `Bundle was built for different runtime images (${mismatches.map((m) => `${m.service}: bundle ${m.bundle}, this host ${m.local}`).join("; ")}). Build and serve from the same OpenMapX revision.`, + ); + if (manifest.schemaVersion === 2 && manifest.runtimeMaterialization) + throw Error( + "MOTIS portable bundle requires the target credential materialization workflow before readiness; rebuild locally until that workflow is available", ); - } - const running = await opts.runningServices(spec.consumers); - if (running.length > 0) { - throw new Error( - `Refusing to import: ${running.join(", ")} ${running.length === 1 ? "is" : "are"} running. ` + - `Stop first with \`openmapx services stop ${running.join(" ")}\`.`, + if (running.length) + throw Error( + `Refusing to import: ${running.join(", ")} ${running.length === 1 ? "is" : "are"} running. Stop first with openmapx services stop ${running.join(" ")}.`, ); - } - const swaps = spec.dirs.map((dir) => { const live = join(dataRoot, dir); mkdirSync(dirname(live), { recursive: true }); - // Swap the aliased directory, not the alias: the MOTIS slot layout keeps - // `motis/live` pointing at the active slot and won't adopt a real dir. const liveDir = existsSync(live) && lstatSync(live).isSymbolicLink() ? realpathSync(live) : live; + const physical = relative( + realpathSync(dataRoot), + existsSync(liveDir) + ? realpathSync(liveDir) + : join(realpathSync(dirname(liveDir)), dir.split("/").at(-1)!), + ); + if (!safeArtifactPath(physical)) throw Error("Artifact target alias leaves data root"); return { liveDir, nextDir: join(stagingRoot, dir) }; }); + assertCliExecutionAvailable(); promoteStagedBuildDirs(swaps); const { dirs: _dirs, directories: _directories, files: _files, ...record } = manifest; writeBuildRecord(record, opts.rootDir); diff --git a/packages/cli/src/lib/control-context.ts b/packages/cli/src/lib/control-context.ts new file mode 100644 index 000000000..a5c7523f9 --- /dev/null +++ b/packages/cli/src/lib/control-context.ts @@ -0,0 +1,140 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { + adoptArtifactTargetLease, + assertArtifactTargetLeaseForward, + type BorrowedArtifactTargetLease, + beginFixedCliExecution, + bindBorrowedControlExecution, + closeArtifactTargetLeaseHandle, + closeControlInstallation, + openFixedChildInstallation, + readFixedCliSpawnIntent, + retainAdmittedControlExecution, + retainControlEffectReceipt, + settleArtifactTargetLeaseChild, + type VerifiedExecutionAuthority, + verifiedParentForArtifactDelegation, + verifyControlEffectSettlement, + verifyFixedCliControlPeer, + verifyParentExecution, +} from "@openmapx/core/build-artifacts"; +import { executionRefSchema } from "@openmapx/core/ops"; +import { repoPaths } from "./paths"; + +const contexts = new AsyncLocalStorage<{ + authority: VerifiedExecutionAuthority; + lease?: BorrowedArtifactTargetLease; +}>(); +class FixedCliCommandExit extends Error { + constructor(readonly code: number) { + super(`Fixed CLI command exited with code ${code}`); + } +} +/** Managed commands unwind through durable child settlement before the process exits. */ +export function completeCliCommand(code = 0): never { + if (contexts.getStore()) throw new FixedCliCommandExit(code); + process.exit(code); +} +export function cliControlContext() { + return contexts.getStore(); +} +export async function withCliControlContext( + args: readonly string[], + run: () => Promise, +): Promise { + const delegation = process.env.OPENMAPX_CONTROL_DELEGATION, + parentAssertion = process.env.OPENMAPX_PARENT_EXECUTION; + if (!delegation && !parentAssertion) return run(); + if (delegation && parentAssertion) throw Error("Ambiguous fixed CLI control authority"); + const encoded = process.env.OPENMAPX_CONTROL_EXECUTION; + if (!encoded || Buffer.byteLength(encoded) > 2048) + throw Error("Missing bounded fixed CLI execution"); + const execution = executionRefSchema.parse(JSON.parse(encoded)); + const assertion = (delegation ?? parentAssertion)!; + if (assertion.length > 4096) throw Error("Fixed CLI assertion exceeds header budget"); + const infraDir = repoPaths().infraDir; + const installation = openFixedChildInstallation({ + infraDir, + assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + let lease: BorrowedArtifactTargetLease | undefined; + try { + const intent = readFixedCliSpawnIntent(installation, execution, args); + const peer = verifyFixedCliControlPeer(installation, { + assertion, + execution, + kind: intent.operation.kind, + }); + let parent: ReturnType | undefined; + if (delegation) { + parent = verifiedParentForArtifactDelegation(installation, assertion, execution); + beginFixedCliExecution(installation, execution); + lease = await adoptArtifactTargetLease({ + dataRoot: join(infraDir, "data"), + assertion, + peer, + execution, + serviceIds: intent.serviceIds, + }); + } else { + parent = verifyParentExecution(assertion, peer, { + execution, + kind: intent.operation.kind, + requestedDeadlineAt: execution.deadlineAt, + }); + beginFixedCliExecution(installation, execution); + } + // The lease grant's parent is verified in admission; role fingerprints were checked by the fixed runner. + const authority = retainAdmittedControlExecution(installation, { + execution, + kind: intent.operation.kind, + operation: intent.operation, + admittedAt: intent.admittedAt, + ...(parent ? { parent } : {}), + serviceIds: intent.serviceIds, + plan: intent.plan, + }); + if (lease) bindBorrowedControlExecution(lease, authority); + if (lease) assertArtifactTargetLeaseForward(lease); + try { + let result: T; + try { + result = await contexts.run({ authority, lease }, run); + } catch (e) { + if (e instanceof FixedCliCommandExit && e.code === 0) result = undefined as T; + else throw e; + } + if (process.exitCode && Number(process.exitCode) !== 0) + throw Error("Fixed CLI command failed"); + retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "fixed-cli-exit", args: [...args], exitCode: 0 }, + }); + if (lease) + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, execution), + ); + return result; + } catch (e) { + retainControlEffectReceipt(authority, { + outcome: "unknown", + publication: "unknown", + receipt: { kind: "unresolved" }, + }); + if (lease) + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, execution), + ); + throw e; + } + } finally { + if (lease) closeArtifactTargetLeaseHandle(lease); + closeControlInstallation(installation); + } +} diff --git a/packages/cli/src/lib/execution-deadline.ts b/packages/cli/src/lib/execution-deadline.ts index f9c94208f..2abdb9d1a 100644 --- a/packages/cli/src/lib/execution-deadline.ts +++ b/packages/cli/src/lib/execution-deadline.ts @@ -1,6 +1,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import type { DataManagerTransport } from "@openmapx/core/data-manager-sdk"; import { executionRefSchema, opsExecutionBudget } from "@openmapx/core/ops"; +import { withCliControlContext } from "./control-context"; // Restriction only. These values authenticate no principal, parent, role or lease. const deadlines = new AsyncLocalStorage<{ deadlineAt: number; signal: AbortSignal }>(); @@ -28,8 +29,12 @@ export async function withCliExecutionDeadline( ): Promise { const encoded = process.env.OPENMAPX_CONTROL_EXECUTION; const inherited = process.env.OPENMAPX_CLI_DEADLINE_AT; - if (process.env.OPENMAPX_CONTROL_DELEGATION || process.env.OPENMAPX_PARENT_EXECUTION) - throw Error("CLI control verifier is unavailable"); + if ( + (process.env.OPENMAPX_CONTROL_DELEGATION || process.env.OPENMAPX_PARENT_EXECUTION) && + encoded === undefined + ) + throw Error("Missing bounded fixed CLI execution"); + if (encoded === undefined && inherited === undefined) return run(); if (encoded !== undefined && Buffer.byteLength(encoded) > 2048) throw Error("Invalid execution deadline"); @@ -54,7 +59,9 @@ export async function withCliExecutionDeadline( ); timer.unref(); try { - return await deadlines.run({ deadlineAt, signal: controller.signal }, run); + return await deadlines.run({ deadlineAt, signal: controller.signal }, () => + withCliControlContext(args, run), + ); } finally { clearTimeout(timer); } diff --git a/packages/core/package.json b/packages/core/package.json index cd4f9eb5e..7dd5ed717 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -193,6 +193,10 @@ "./api-admin-sdk": { "types": "./src/api/admin-sdk.ts", "default": "./src/api/admin-sdk.ts" + }, + "./build-artifacts": { + "types": "./src/build-artifacts/index.ts", + "default": "./src/build-artifacts/index.ts" } }, "scripts": { @@ -207,6 +211,7 @@ "@types/node": "^26.6.5", "@types/react": "^19.3.0", "@types/react-dom": "^19.3.0", + "@types/tar-stream": "3.1.4", "esbuild": "^0.28.2", "postgres": "^3.4.9", "react-dom": "19.3.0", @@ -230,6 +235,7 @@ "@better-auth/oauth-provider": "1.7.7", "@better-auth/passkey": "1.7.7", "@openmapx/air-quality": "workspace:*", + "@openmapx/extension-contracts": "workspace:^", "@openmapx/mobility-core": "workspace:^", "@tanstack/react-query": "^5.104.1", "@turf/helpers": "^7.4.0", @@ -242,11 +248,11 @@ "opening_hours": "^3.15.0", "parse5": "8.0.1", "react": "19.3.0", + "tar-stream": "3.2.2", "tz-lookup": "^6.1.25", "undici": "8.11.2", "zod": "^4.6.5", - "zustand": "^5.0.15", - "@openmapx/extension-contracts": "workspace:^" + "zustand": "^5.0.15" }, "private": true } diff --git a/packages/core/src/build-artifacts/archive.test.ts b/packages/core/src/build-artifacts/archive.test.ts new file mode 100644 index 000000000..e6f2454aa --- /dev/null +++ b/packages/core/src/build-artifacts/archive.test.ts @@ -0,0 +1,287 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { gzipSync } from "node:zlib"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +function tarMember(name: string, bytes = Buffer.alloc(0), type = "0", size = bytes.length): Buffer { + const header = Buffer.alloc(512); + header.write(name, 0, 100); + header.write("0000600\0", 100); + header.write("0000000\0", 108); + header.write("0000000\0", 116); + header.write(`${size.toString(8).padStart(11, "0")}\0`, 124); + header.write("00000000000\0", 136); + header.fill(32, 148, 156); + header.write(type, 156); + header.write("ustar\0", 257); + header.write("00", 263); + header.write( + `${[...header] + .reduce((a, b) => a + b, 0) + .toString(8) + .padStart(6, "0")}\0 `, + 148, + ); + return Buffer.concat([header, bytes, Buffer.alloc((512 - (bytes.length % 512)) % 512)]); +} +const sha = (b: Buffer) => createHash("sha256").update(b).digest("hex"); +const payload = Buffer.from("GRAPH"); +const fixtureManifest = () => ({ + schemaVersion: 1, + service: "osrm", + builtAt: "2026-10-23T00:00:00.000Z", + runtimeImages: { osrm: "image:1" }, + toolImages: {}, + dirs: ["osrm-graph"], + directories: ["osrm-graph"], + files: [{ path: "osrm-graph/region.osrm", sizeBytes: payload.length, sha256: sha(payload) }], +}); +function archiveFixture(members?: Buffer[], manifest = fixtureManifest()): Buffer { + return Buffer.concat([ + tarMember(".openmapx-bundle/manifest.json", Buffer.from(JSON.stringify(manifest))), + ...(members ?? [ + tarMember("osrm-graph/", Buffer.alloc(0), "5"), + tarMember("osrm-graph/region.osrm", payload), + ]), + Buffer.alloc(1024), + ]); +} +let root: string; +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "omx-archive-test-")); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); +async function validate(bytes: Buffer, overrides = {}) { + const sourcePath = join(import.meta.dirname, "archive.ts"); + expect(existsSync(sourcePath), "missing shared bounded archive validation").toBe(true); + const api = await import("./archive"); + const stagingRoot = join(root, "stage"); + rmSync(stagingRoot, { recursive: true, force: true }); + mkdirSync(stagingRoot, { recursive: true, mode: 0o700 }); + return api.validateBuildBundle({ + source: Readable.from([bytes]), + stagingRoot, + limits: { ...api.DEFAULT_ARTIFACT_ARCHIVE_LIMITS, ...overrides }, + signal: new AbortController().signal, + }); +} +describe("bounded build archive", () => { + it.each([false, true])("streams safe v1 bytes from plain/gzip (%s) tar", async (gzip) => { + const bytes = gzip ? gzipSync(archiveFixture()) : archiveFixture(); + const result = await validate(bytes); + expect(result.archiveSha256).toBe(sha(bytes)); + expect(result.expandedBytes).toBe(5); + expect(readFileSync(join(root, "stage/osrm-graph/region.osrm"), "utf8")).toBe("GRAPH"); + }); + it.each(["../escape", "/escape", "C:/escape", "osrm-graph\\escape", "osrm-graph/../escape"])( + "rejects unsafe path %s before write", + async (name) => { + await expect(validate(archiveFixture([tarMember(name, payload)]))).rejects.toThrow( + /path|member|artifact/, + ); + expect(existsSync(join(root, "escape"))).toBe(false); + }, + ); + it.each(["1", "2", "3", "4", "6", "S"])("rejects link/device/sparse type %s", async (type) => { + await expect( + validate(archiveFixture([tarMember("osrm-graph/region.osrm", payload, type)])), + ).rejects.toThrow(/type|link|sparse/); + expect(existsSync(join(root, "stage/osrm-graph/region.osrm"))).toBe(false); + }); + it("rejects duplicate directory spellings", async () => { + await expect( + validate( + archiveFixture([ + tarMember("osrm-graph", Buffer.alloc(0), "5"), + tarMember("osrm-graph/", Buffer.alloc(0), "5"), + ]), + ), + ).rejects.toThrow(/duplicate/i); + }); + it("rejects missing and extra members", async () => { + await expect(validate(archiveFixture([]))).rejects.toThrow(/missing/i); + await expect( + validate(archiveFixture([tarMember("osrm-graph/extra", payload)])), + ).rejects.toThrow(/does not list/); + }); + it("checks metadata sizes before opening payload", async () => { + await expect( + validate(archiveFixture([tarMember("osrm-graph/region.osrm", payload, "0", 100)])), + ).rejects.toThrow(/verification|size/); + expect(existsSync(join(root, "stage/osrm-graph/region.osrm"))).toBe(false); + }); + it("bounds extended headers before reading them", async () => { + await expect( + validate(archiveFixture([tarMember("PaxHeader", Buffer.alloc(0), "x", 16385)])), + ).rejects.toThrow(/extended|header/); + }); + it("rejects changed bytes, truncated EOF and nonzero trailing structure", async () => { + await expect( + validate(archiveFixture([tarMember("osrm-graph/region.osrm", Buffer.from("OTHER"))])), + ).rejects.toThrow(/verification/); + await expect(validate(archiveFixture().subarray(0, -512))).rejects.toThrow( + /EOF|truncated|terminator/, + ); + await expect(validate(Buffer.concat([archiveFixture(), Buffer.from("extra")]))).rejects.toThrow( + /trailing/, + ); + }); + it("enforces actual expansion and single gzip member", async () => { + await expect(validate(gzipSync(archiveFixture()), { maxExpandedBytes: 4 })).rejects.toThrow( + /expanded|limit|budget/, + ); + await expect( + validate(Buffer.concat([gzipSync(archiveFixture()), gzipSync(Buffer.alloc(0))])), + ).rejects.toThrow(/gzip|trailing|concatenated/); + }); +}); + +it("exports a sanitized MOTIS candidate with credential IDs and no generated runtime credentials", async () => { + const { writeBuildBundle, DEFAULT_ARTIFACT_ARCHIVE_LIMITS } = await import("./archive"); + const { Writable } = await import("node:stream"); + const { writeFileSync, symlinkSync } = await import("node:fs"); + const data = join(root, "data"); + mkdirSync(join(data, "motis/slots/A/data"), { recursive: true }); + mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); + symlinkSync("slots/A", join(data, "motis/live")); + writeFileSync(join(data, "motis/slots/A/data/tt.bin"), "TT"); + writeFileSync(join(data, "motis/slots/A/config.yml"), "secret-api-key"); + writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "secret-api-key"); + writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "secret-api-key"); + writeFileSync( + join(data, "motis/slots/A/portable-sources.json"), + JSON.stringify({ + version: 1, + sourceFingerprints: [], + credentialRequirements: ["synthetic-feed"], + }), + ); + const chunks: Buffer[] = []; + const target = new Writable({ + write(chunk, _encoding, done) { + chunks.push(Buffer.from(chunk)); + done(); + }, + }); + const result = await writeBuildBundle({ + dataRoot: data, + serviceId: "motis", + record: { + schemaVersion: 1, + service: "motis", + builtAt: "2026-10-23T00:00:00.000Z", + runtimeImages: { motis: "image:1" }, + toolImages: {}, + }, + target, + limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: new AbortController().signal, + }); + expect(result.manifest).toMatchObject({ + schemaVersion: 2, + credentialRequirements: ["synthetic-feed"], + runtimeMaterialization: "motis-target-credentials", + }); + expect(Buffer.concat(chunks).includes(Buffer.from("secret-api-key"))).toBe(false); + const validated = await validate(Buffer.concat(chunks)); + expect(validated.artifactFingerprint).toBe(result.artifactFingerprint); +}); +it.each(["tileserver", "osrm", "otp", "pelias", "motis"])( + "round-trips the portable %s layout on the real filesystem", + async (service) => { + const { writeBuildBundle, DEFAULT_ARTIFACT_ARCHIVE_LIMITS } = await import("./archive"); + const { Writable } = await import("node:stream"); + const { writeFileSync } = await import("node:fs"); + const dirs: Record = { + tileserver: ["tile-mbtiles"], + osrm: ["osrm-graph"], + otp: ["otp-graph"], + pelias: ["pelias"], + motis: ["motis/live/data", "motis-feed-proxy"], + }; + const data = join(root, "data"); + const serviceDirs = dirs[service]; + if (!serviceDirs?.[0]) throw Error("Missing fixture artifact directory"); + for (const dir of serviceDirs) mkdirSync(join(data, dir), { recursive: true }); + writeFileSync(join(data, serviceDirs[0], "artifact.bin"), "DATA"); + if (service === "motis") + writeFileSync( + join(data, "motis/live/portable-sources.json"), + JSON.stringify({ version: 1, sourceFingerprints: [], credentialRequirements: [] }), + ); + const chunks: Buffer[] = []; + const result = await writeBuildBundle({ + dataRoot: data, + serviceId: service, + record: { + schemaVersion: 1, + service, + builtAt: "2026-10-23T00:00:00.000Z", + runtimeImages: { [service]: "image:1" }, + toolImages: {}, + }, + target: new Writable({ + write(b, _e, done) { + chunks.push(Buffer.from(b)); + done(); + }, + }), + limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: new AbortController().signal, + }); + const checked = await validate(gzipSync(Buffer.concat(chunks))); + expect(checked.manifest.service).toBe(service); + expect(checked.artifactFingerprint).toBe(result.artifactFingerprint); + }, +); +it("rejects effective PAX escape/size and sparse metadata before payload", async () => { + function paxRecord(key: string, value: string) { + const tail = ` ${key}=${value}\n`; + let size = tail.length + 1; + while (size !== tail.length + String(size).length) size = tail.length + String(size).length; + return Buffer.from(`${size}${tail}`); + } + for (const [key, value] of [ + ["path", "../escape"], + ["size", "-1"], + ["GNU.sparse.size", "5"], + ]) + await expect( + validate( + archiveFixture([ + tarMember("PaxHeader", paxRecord(key!, value!), "x"), + tarMember("osrm-graph/region.osrm", payload), + ]), + ), + ).rejects.toThrow(/path|size|sparse|PAX/); + expect(existsSync(join(root, "escape"))).toBe(false); +}); +it("rejects expansion-ratio overflow and corrupt compression", async () => { + await expect( + validate(gzipSync(archiveFixture()), { maxExpansionRatio: 1, expansionRatioThresholdBytes: 1 }), + ).rejects.toThrow(/ratio/); + const corrupt = gzipSync(archiveFixture()); + corrupt[corrupt.length - 8] ^= 0xff; + await expect(validate(corrupt)).rejects.toThrow(/gzip|checksum/); +}); + +it("validates fragmented tar trailing blocks independently of input chunk boundaries", async () => { + const a = await import("./archive"); + const bytes = Buffer.concat([archiveFixture(), Buffer.alloc(1024)]); + mkdirSync(join(root, "stage"), { mode: 0o700 }); + const chunks = []; + for (let i = 0; i < bytes.length; i += 137) chunks.push(bytes.subarray(i, i + 137)); + await expect( + a.validateBuildBundle({ + source: Readable.from(chunks), + stagingRoot: join(root, "stage"), + limits: a.DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: new AbortController().signal, + }), + ).resolves.toMatchObject({ expandedBytes: 5 }); +}); diff --git a/packages/core/src/build-artifacts/archive.ts b/packages/core/src/build-artifacts/archive.ts new file mode 100644 index 000000000..606c90618 --- /dev/null +++ b/packages/core/src/build-artifacts/archive.ts @@ -0,0 +1,720 @@ +import { createHash } from "node:crypto"; +import { + constants as C, + closeSync, + createReadStream, + fstatSync, + fsyncSync, + lstatSync, + openSync, + realpathSync, + type Stats, + writeSync, +} from "node:fs"; +import { join, relative } from "node:path"; +import type { Readable, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { crc32, createInflateRaw } from "node:zlib"; +import { pack } from "tar-stream"; +import { AnchoredDirectory, identity, sameInode, withDirectories } from "./filesystem"; +import { + artifactFingerprint, + BUNDLE_MANIFEST_PATH, + type BuildRecord, + type BuildRecordV2, + type BundleFile, + type BundleManifest, + buildArtifactSpec, + parseBundleManifest, + portableMotisSourcesSchema, + safeArtifactPath, +} from "./manifest"; +export interface ArtifactArchiveLimits { + maxUploadBytes: number; + maxExpandedBytes: number; + maxStageBytes: number; + maxEntryBytes: number; + maxEntries: number; + maxManifestBytes: number; + maxPathBytes: number; + maxPathDepth: number; + maxExtendedHeaderBytes: number; + maxExpansionRatio: number; + expansionRatioThresholdBytes: number; +} +export const DEFAULT_ARTIFACT_ARCHIVE_LIMITS: Readonly = Object.freeze({ + maxUploadBytes: 1_099_511_627_776, + maxExpandedBytes: 2_199_023_255_552, + maxStageBytes: 4_398_046_511_104, + maxEntryBytes: 2_199_023_255_552, + maxEntries: 250_000, + maxManifestBytes: 67_108_864, + maxPathBytes: 4096, + maxPathDepth: 64, + maxExtendedHeaderBytes: 16384, + maxExpansionRatio: 1000, + expansionRatioThresholdBytes: 67_108_864, +}); +export type ValidatedBuildBundle = { + manifest: BundleManifest; + archiveSha256: string; + expandedBytes: number; + manifestSha256: string; + artifactFingerprint: string; +}; +function checkLimits(limits: ArtifactArchiveLimits) { + if ( + Object.keys(DEFAULT_ARTIFACT_ARCHIVE_LIMITS).some( + (k) => + !Number.isSafeInteger(limits[k as keyof ArtifactArchiveLimits]) || + limits[k as keyof ArtifactArchiveLimits] <= 0, + ) + ) + throw Error("Archive limits must be positive safe integers"); + if ( + limits.maxEntries > 1_000_000 || + limits.maxManifestBytes > 268_435_456 || + limits.maxPathBytes > 4096 || + limits.maxPathDepth > 64 || + limits.maxExtendedHeaderBytes > 16384 || + limits.maxExpansionRatio > 1000 + ) + throw Error("Archive limits exceed supported ceiling"); +} +class Bytes { + private pending = Buffer.alloc(0); + private done = false; + private readonly iterator: AsyncIterator; + constructor(source: AsyncIterable) { + this.iterator = source[Symbol.asyncIterator](); + } + async take(maximum: number): Promise { + while (!this.pending.length && !this.done) { + const next = await this.iterator.next(); + this.done = !!next.done; + if (!next.done) this.pending = Buffer.from(next.value); + } + const bytes = this.pending.subarray(0, maximum); + this.pending = this.pending.subarray(bytes.length); + return bytes; + } + async exact(size: number): Promise { + const chunks: Buffer[] = []; + let count = 0; + while (count < size) { + const b = await this.take(Math.min(65536, size - count)); + if (!b.length) throw Error("Truncated archive EOF"); + chunks.push(b); + count += b.length; + } + return Buffer.concat(chunks, size); + } + prepend(bytes: Buffer) { + this.pending = Buffer.concat([bytes, this.pending]); + } + async *rest() { + for (;;) { + const b = await this.take(65536); + if (!b.length) return; + yield b; + } + } +} +async function* decode(source: Bytes, limits: ArtifactArchiveLimits): AsyncGenerator { + const magic = await source.exact(2); + if (magic[0] !== 0x1f || magic[1] !== 0x8b) { + source.prepend(magic); + yield* source.rest(); + return; + } + const header = await source.exact(8); + if (header[0] !== 8 || header[1]! & 0xe0) throw Error("Unsupported gzip header"); + const flags = header[1]!; + let headerBytes = 10; + if (flags & 4) { + const size = (await source.exact(2)).readUInt16LE(); + headerBytes += 2 + size; + if (headerBytes > limits.maxExtendedHeaderBytes) throw Error("gzip header limit"); + await source.exact(size); + } + for (const flag of [8, 16]) + if (flags & flag) { + for (;;) { + if (++headerBytes > limits.maxExtendedHeaderBytes) throw Error("gzip header limit"); + if ((await source.exact(1))[0] === 0) break; + } + } + // Header CRC is deliberately unsupported; accepting it without checking is unsafe. + if (flags & 2) throw Error("Unsupported gzip header CRC"); + const inflate = createInflateRaw({ chunkSize: 65536 }); + let crc = 0, + total = 0; + let footer: Buffer | undefined; + const feed = (async () => { + try { + for (;;) { + const chunk = await source.take(65536); + if (!chunk.length) break; + const before = inflate.bytesWritten; + await new Promise((resolve, reject) => + inflate.write(chunk, (e) => (e ? reject(e) : resolve())), + ); + const consumed = inflate.bytesWritten - before; + if (consumed < chunk.length) { + source.prepend(chunk.subarray(consumed)); + break; + } + } + inflate.end(); + footer = await source.exact(8); + if ((await source.take(1)).length) throw Error("Concatenated or trailing gzip data"); + } catch (e) { + inflate.destroy(e as Error); + throw e; + } + })(); + feed.catch(() => undefined); + try { + for await (const raw of inflate) { + const b = Buffer.from(raw); + crc = crc32(b, crc); + total = (total + b.length) >>> 0; + yield b; + } + await feed; + if (!footer || footer.readUInt32LE() !== crc || footer.readUInt32LE(4) !== total) + throw Error("gzip checksum or size mismatch"); + } finally { + inflate.destroy(); + } +} +function number(header: Buffer, start: number, length: number): number { + const field = header.subarray(start, start + length); + let value = 0; + if (field[0]! & 0x80) { + if (field[0] !== 0x80) throw Error("Invalid tar size"); + for (const byte of field.subarray(1)) { + value = value * 256 + byte; + if (!Number.isSafeInteger(value)) throw Error("Tar size exceeds safe integer"); + } + } else { + const text = field.toString("ascii").replace(/\0.*$/s, "").trim(); + if (!/^[0-7]*$/.test(text)) throw Error("Invalid tar numeric metadata"); + value = parseInt(text || "0", 8); + } + if (!Number.isSafeInteger(value) || value < 0) throw Error("Invalid tar size"); + return value; +} +function utf8(bytes: Buffer): string { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); +} +function tarString(header: Buffer, start: number, length: number): string { + const field = header.subarray(start, start + length); + const nul = field.indexOf(0); + return utf8(nul < 0 ? field : field.subarray(0, nul)); +} +function pax(data: Buffer): Record { + const result: Record = Object.create(null); + let offset = 0, + count = 0; + while (offset < data.length) { + const space = data.indexOf(32, offset); + if (space < 0 || space - offset > 10) throw Error("Invalid PAX length"); + const str = data.subarray(offset, space).toString("ascii"); + if (!/^[1-9][0-9]*$/.test(str)) throw Error("Invalid PAX length"); + const length = Number(str); + if ( + length <= space - offset + 2 || + offset + length > data.length || + data[offset + length - 1] !== 10 || + ++count > 256 + ) + throw Error("Invalid PAX bounds"); + const text = utf8(data.subarray(space + 1, offset + length - 1)); + const equal = text.indexOf("="); + const key = text.slice(0, equal); + if (equal < 1 || Object.hasOwn(result, key) || /sparse|linkpath/i.test(key)) + throw Error("Unsupported or duplicate PAX metadata"); + if ( + ![ + "path", + "size", + "mtime", + "atime", + "ctime", + "uid", + "gid", + "uname", + "gname", + "charset", + "comment", + "SCHILY.fflags", + "LIBARCHIVE.creationtime", + ].includes(key) + ) + throw Error("Unsupported PAX metadata"); + result[key] = text.slice(equal + 1); + offset += length; + } + return result; +} +export async function validateBuildBundle(options: { + source: Readable; + stagingRoot: string; + limits: ArtifactArchiveLimits; + signal: AbortSignal; +}): Promise { + const { source, limits, signal } = options; + checkLimits(limits); + signal.throwIfAborted(); + const root = new AnchoredDirectory(options.stagingRoot); + if (root.list().length) { + root.close(); + throw Error("Archive staging root must be empty"); + } + const archiveHash = createHash("sha256"); + let wireBytes = 0, + framedBytes = 0; + const abort = () => source.destroy(Error("Archive validation aborted")); + signal.addEventListener("abort", abort, { once: true }); + async function* wire() { + for await (const raw of source) { + signal.throwIfAborted(); + const b = Buffer.from(raw); + wireBytes += b.length; + if (wireBytes > limits.maxUploadBytes || wireBytes > limits.maxStageBytes) + throw Error("Archive upload limit exceeded"); + archiveHash.update(b); + yield b; + } + } + const decoded = decode(new Bytes(wire()), limits); + async function* bounded() { + for await (const b of decoded) { + signal.throwIfAborted(); + framedBytes += b.length; + if ( + framedBytes > + limits.maxExpandedBytes + + limits.maxManifestBytes + + limits.maxEntries * 1024 + + limits.maxExtendedHeaderBytes * limits.maxEntries || + (framedBytes > limits.expansionRatioThresholdBytes && + framedBytes > Math.max(1, wireBytes) * limits.maxExpansionRatio) + ) + throw Error("Archive expanded bytes or gzip ratio limit exceeded"); + yield b; + } + } + const reader = new Bytes(bounded()); + let manifest: BundleManifest | undefined, + manifestSha256 = "", + expandedBytes = 0, + headers = 0; + let extended: Record | undefined; + const seen = new Set(); + let expectedFiles = new Map(), + expectedDirs = new Set(); + try { + for (;;) { + signal.throwIfAborted(); + const header = await reader.exact(512); + if (header.every((b) => b === 0)) { + const end = await reader.exact(512); + if (end.some((b) => b !== 0) || extended) throw Error("Invalid tar terminator"); + let trailingBytes = 0; + for await (const b of reader.rest()) { + trailingBytes += b.length; + if (b.some((v) => v !== 0)) throw Error("Unexpected trailing tar data"); + } + if (trailingBytes % 512 !== 0) throw Error("Truncated trailing tar block"); + break; + } + if (++headers > limits.maxEntries + 1) throw Error("Archive entry limit exceeded"); + let checksum = 0; + for (let i = 0; i < 512; i++) checksum += i >= 148 && i < 156 ? 32 : header[i]!; + if (checksum !== number(header, 148, 8)) throw Error("Invalid tar header checksum"); + const type = String.fromCharCode(header[156] || 48); + const rawSize = number(header, 124, 12); + if (!["0", "5", "x", "L"].includes(type)) + throw Error("Unsupported tar type (links, devices and sparse members are forbidden)"); + if (type === "x" || type === "L") { + if (extended || rawSize > limits.maxExtendedHeaderBytes) + throw Error("Extended tar header limit or repeated metadata"); + const bytes = await reader.exact(rawSize); + await reader.exact((512 - (rawSize % 512)) % 512); + extended = type === "x" ? pax(bytes) : { path: utf8(bytes).replace(/\0$/, "") }; + continue; + } + const prefix = tarString(header, 345, 155); + let name = extended?.path ?? [prefix, tarString(header, 0, 100)].filter(Boolean).join("/"); + let size = rawSize; + if (extended?.size !== undefined) { + if (!/^(0|[1-9][0-9]*)$/.test(extended.size)) throw Error("Invalid PAX size"); + size = Number(extended.size); + if (!Number.isSafeInteger(size)) throw Error("Invalid PAX size"); + } + extended = undefined; + if (type === "5") name = name.replace(/\/$/, ""); + if (!safeArtifactPath(name, limits.maxPathBytes, limits.maxPathDepth)) + throw Error("Unsafe archive member path"); + if (seen.has(name)) throw Error("Duplicate normalized archive path"); + seen.add(name); + if (type === "5" && size !== 0) throw Error("Invalid directory size"); + if (size > limits.maxEntryBytes) throw Error("Archive entry size limit exceeded"); + if (!manifest) { + if (type === "5" && name === ".openmapx-bundle") continue; + if (type !== "0" || name !== BUNDLE_MANIFEST_PATH) + throw Error("Manifest must be first regular archive member"); + if (size > limits.maxManifestBytes) throw Error("Manifest size limit exceeded"); + const bytes = await reader.exact(size); + manifest = parseBundleManifest(utf8(bytes)); + manifestSha256 = createHash("sha256").update(bytes).digest("hex"); + if (manifest.files.length + manifest.directories.length > limits.maxEntries) + throw Error("Manifest entry limit exceeded"); + let total = 0; + for (const f of manifest.files) { + total += f.sizeBytes; + if ( + f.sizeBytes > limits.maxEntryBytes || + !Number.isSafeInteger(total) || + total > limits.maxExpandedBytes + ) + throw Error("Manifest expanded byte limit exceeded"); + } + for (const p of [...manifest.directories, ...manifest.files.map((f) => f.path)]) + if (!safeArtifactPath(p, limits.maxPathBytes, limits.maxPathDepth)) + throw Error("Manifest path limit exceeded"); + expectedFiles = new Map(manifest.files.map((f) => [f.path, f])); + expectedDirs = new Set(manifest.directories); + } else if (type === "5") { + if (!expectedDirs.delete(name)) + throw Error(`Bundle contains entries its manifest does not list: ${name}`); + withDirectories(root, name.split("/"), true, () => undefined); + } else { + const file = expectedFiles.get(name); + if (!file) throw Error(`Bundle contains entries its manifest does not list: ${name}`); + if (file.sizeBytes !== size) throw Error(`Bundle file failed verification: ${name} (size)`); + const parts = name.split("/"); + const leaf = parts.pop()!; + const dirs: AnchoredDirectory[] = []; + let parent = root; + let fd: number | undefined; + try { + for (const part of parts) { + parent = parent.directory(part, true); + dirs.push(parent); + } + fd = openSync(parent.name(leaf), C.O_WRONLY | C.O_CREAT | C.O_EXCL | C.O_NOFOLLOW, 0o600); + const stat = fstatSync(fd, { bigint: true }); + const hash = createHash("sha256"); + let remaining = size; + while (remaining) { + const b = await reader.exact(Math.min(65536, remaining)); + signal.throwIfAborted(); + hash.update(b); + let offset = 0; + while (offset < b.length) offset += writeSync(fd, b, offset, b.length - offset); + remaining -= b.length; + expandedBytes += b.length; + if (expandedBytes > limits.maxExpandedBytes) + throw Error("Expanded byte limit exceeded"); + } + fsyncSync(fd); + parent.check(); + if ( + !sameInode(identity(stat), identity(lstatSync(parent.name(leaf), { bigint: true }))) || + fstatSync(fd).nlink !== 1 || + hash.digest("hex") !== file.sha256 + ) + throw Error(`Bundle file failed verification: ${name}`); + expectedFiles.delete(name); + } finally { + if (fd !== undefined) closeSync(fd); + for (const d of dirs.reverse()) d.close(); + } + } + const pad = await reader.exact((512 - (size % 512)) % 512); + if (pad.some((b) => b !== 0)) throw Error("Nonzero tar member padding"); + } + if (!manifest) throw Error("Bundle has no manifest"); + if (expectedFiles.size || expectedDirs.size) throw Error("Bundle is missing declared members"); + if (manifest.service === "motis" && manifest.schemaVersion === 2) { + if ( + manifest.runtimeMaterialization !== "motis-target-credentials" || + !manifest.credentialRequirements + ) + throw Error("MOTIS target materialization declaration required"); + const portable = withDirectories(root, ["motis", "live"], false, (d) => + portableMotisSourcesSchema.parse( + JSON.parse(d.read("portable-sources.json", 1048576, false).toString("utf8")), + ), + ); + if ( + JSON.stringify(portable.sourceFingerprints) !== + JSON.stringify(manifest.sourceFingerprints) || + JSON.stringify(portable.credentialRequirements) !== + JSON.stringify(manifest.credentialRequirements) + ) + throw Error("MOTIS portable source/credential metadata mismatch"); + } + root.check(); + return { + manifest, + archiveSha256: archiveHash.digest("hex"), + expandedBytes, + manifestSha256, + artifactFingerprint: artifactFingerprint(manifest.files), + }; + } finally { + signal.removeEventListener("abort", abort); + source.destroy(); + await decoded.return(undefined); + root.close(); + } +} +function excluded(path: string, dir: boolean, patterns: readonly string[]) { + return patterns.some((p) => + p.startsWith("*") + ? !dir && path.endsWith(p.slice(1)) + : p.endsWith("/") + ? (dir ? `${path}/` : path).startsWith(p) + : path === p, + ); +} +export async function writeBuildBundle(options: { + dataRoot: string; + serviceId: string; + record: BuildRecord | BuildRecordV2; + target: Writable; + limits: ArtifactArchiveLimits; + signal: AbortSignal; +}): Promise { + const { limits, signal, target } = options; + checkLimits(limits); + signal.throwIfAborted(); + const root = new AnchoredDirectory(options.dataRoot); + const spec = buildArtifactSpec(options.serviceId); + let record = options.record; + let portableSources: ReturnType | undefined; + if (options.serviceId === "motis") { + try { + const real = realpathSync(join(options.dataRoot, "motis/live")); + const physical = relative(realpathSync(options.dataRoot), real); + if (!safeArtifactPath(physical)) throw Error("Unsafe MOTIS alias"); + portableSources = withDirectories(root, physical.split("/"), false, (d) => + portableMotisSourcesSchema.parse( + JSON.parse(d.read("portable-sources.json", 1048576, false).toString("utf8")), + ), + ); + } catch { + root.close(); + throw Error( + "MOTIS export requires retained portable source metadata; rebuild the candidate before exporting", + ); + } + } + const files: BundleFile[] = []; + const directories: string[] = []; + const sources = new Map(); + const fileRead = async (path: string, read: (fd: number, stat: Stats) => Promise) => { + const parts = path.split("/"); + const leaf = parts.pop()!; + const opened: AnchoredDirectory[] = []; + let parent = root; + let fd: number | undefined; + try { + for (const part of parts) { + parent = parent.directory(part); + opened.push(parent); + } + const name = parent.name(leaf); + fd = openSync(name, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1) + throw Error("Refusing non-regular or hardlinked build artifact"); + await read(fd, before); + const after = fstatSync(fd); + parent.check(); + const current = lstatSync(name); + if ( + before.dev !== current.dev || + before.ino !== current.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + after.nlink !== 1 + ) + throw Error("Build artifact changed during export"); + } finally { + if (fd !== undefined) closeSync(fd); + for (const d of opened.reverse()) d.close(); + } + }; + async function visit(rel: string, physical: string) { + signal.throwIfAborted(); + if (!safeArtifactPath(rel, limits.maxPathBytes, limits.maxPathDepth)) + throw Error("Export path limit"); + if ( + options.serviceId === "motis" && + (rel.startsWith("motis-feed-proxy/") || + (rel.startsWith("motis/live/") && + !(rel === "motis/live/data" || rel.startsWith("motis/live/data/")) && + !/\.(osm\.pbf|gtfs\.zip|netex\.zip)$/.test(rel) && + rel !== "motis/live/portable-sources.json")) + ) + return; + const stat = lstatSync(join(options.dataRoot, physical)); + if (stat.isSymbolicLink()) throw Error(`Refusing symlink in build artifact: ${rel}`); + if (stat.isDirectory()) { + if (excluded(rel, true, spec.exclude)) return; + directories.push(rel); + const entries = withDirectories(root, physical.split("/"), false, (d) => d.list().sort()); + for (const name of entries) await visit(`${rel}/${name}`, `${physical}/${name}`); + } else { + if (!stat.isFile() || stat.nlink !== 1) + throw Error(`Refusing non-regular or hardlinked build artifact: ${rel}`); + if (excluded(rel, false, spec.exclude)) return; + let sha256 = ""; + await fileRead(physical, async (fd, s) => { + if (s.size > limits.maxEntryBytes) throw Error("Archive entry size limit"); + const hash = createHash("sha256"); + for await (const b of createReadStream("", { + fd, + autoClose: false, + start: 0, + end: s.size, + })) { + signal.throwIfAborted(); + hash.update(b); + } + sha256 = hash.digest("hex"); + }); + files.push({ path: rel, sizeBytes: stat.size, sha256 }); + sources.set(rel, physical); + } + if (files.length + directories.length > limits.maxEntries) throw Error("Archive entry limit"); + } + try { + for (const dir of spec.dirs) { + const real = realpathSync(join(options.dataRoot, dir)); + const physical = relative(realpathSync(options.dataRoot), real); + if (!safeArtifactPath(physical) || physical.startsWith("../")) + throw Error("Artifact alias leaves trusted data root"); + if ( + physical !== dir && + !( + options.serviceId === "motis" && + dir === "motis/live" && + /^motis\/slots\/[AB]$/.test(physical) + ) + ) + throw Error("Untrusted artifact alias"); + await visit(dir, physical); + } + if (portableSources) + record = { + ...record, + schemaVersion: 2, + sourceFingerprints: portableSources.sourceFingerprints, + credentialRequirements: portableSources.credentialRequirements, + runtimeMaterialization: "motis-target-credentials", + artifactFingerprint: artifactFingerprint(files), + }; + const manifest = parseBundleManifest( + JSON.stringify({ + ...record, + ...(record.schemaVersion === 2 ? { artifactFingerprint: artifactFingerprint(files) } : {}), + dirs: spec.dirs, + directories, + files, + }), + ); + if (manifest.service !== options.serviceId) throw Error("Build record service mismatch"); + const manifestBytes = Buffer.from(`${JSON.stringify(manifest)}\n`); + if (manifestBytes.length > limits.maxManifestBytes) throw Error("Manifest size limit"); + const expandedBytes = files.reduce((sum, f) => sum + f.sizeBytes, 0); + if (!Number.isSafeInteger(expandedBytes) || expandedBytes > limits.maxExpandedBytes) + throw Error("Expanded byte limit"); + const archive = pack(); + let written = 0; + const hash = createHash("sha256"); + const output = pipeline( + archive, + async function* (source) { + for await (const raw of source) { + const b = Buffer.from(raw as Uint8Array); + written += b.length; + if (written > limits.maxUploadBytes) throw Error("Archive upload limit"); + hash.update(b); + yield b; + } + }, + target, + { signal }, + ); + output.catch(() => undefined); + const entry = (name: string, body: Buffer, type: "file" | "directory" = "file") => + new Promise((resolve, reject) => { + archive.entry( + { + name, + size: body.length, + type, + mode: type === "file" ? 0o600 : 0o700, + mtime: new Date(0), + uid: 0, + gid: 0, + }, + body, + (e) => (e ? reject(e) : resolve()), + ); + }); + try { + await entry(BUNDLE_MANIFEST_PATH, manifestBytes); + for (const d of directories) await entry(d, Buffer.alloc(0), "directory"); + for (const file of files) + await fileRead(sources.get(file.path)!, async (fd, stat) => { + if (stat.size !== file.sizeBytes) throw Error("Build artifact size changed"); + const digest = createHash("sha256"); + const member = archive.entry({ + name: file.path, + size: file.sizeBytes, + type: "file", + mode: 0o600, + mtime: new Date(0), + uid: 0, + gid: 0, + }); + await pipeline( + createReadStream("", { fd, autoClose: false, start: 0 }), + async function* (source) { + for await (const b of source) { + digest.update(b); + yield b; + } + }, + member, + { signal }, + ); + if (digest.digest("hex") !== file.sha256) + throw Error("Build artifact digest changed during export"); + }); + archive.finalize(); + await output; + } catch (e) { + archive.destroy(e as Error); + await output.catch(() => undefined); + throw e; + } + return { + manifest, + archiveSha256: hash.digest("hex"), + expandedBytes, + manifestSha256: createHash("sha256").update(manifestBytes).digest("hex"), + artifactFingerprint: artifactFingerprint(files), + }; + } finally { + root.close(); + } +} diff --git a/packages/core/src/build-artifacts/control-authority.test.ts b/packages/core/src/build-artifacts/control-authority.test.ts new file mode 100644 index 000000000..06ba11fd4 --- /dev/null +++ b/packages/core/src/build-artifacts/control-authority.test.ts @@ -0,0 +1,359 @@ +import { + chmodSync, + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +let root: string; +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "omx-control-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); +}); +afterEach(() => rmSync(root, { recursive: true, force: true })); +async function api() { + expect( + existsSync(join(import.meta.dirname, "control-authority.ts")), + "missing role-private certified control authority", + ).toBe(true); + return import("./control-authority"); +} +it("provisions three distinct role keys durably and reuses the exact installation", async () => { + const a = await api(); + await a.provisionControlInstallation(root); + const before = readFileSync(join(root, "secrets/control-ops-agent.key")); + await a.provisionControlInstallation(root); + expect(readFileSync(join(root, "secrets/control-ops-agent.key"))).toEqual(before); + const bundle = JSON.parse(readFileSync(join(root, "secrets/control-verifiers.json"), "utf8")); + expect(new Set(Object.values(bundle.publicKeys)).size).toBe(3); + expect(existsSync(join(root, "data/control-private"))).toBe(false); +}); +it("never silently regenerates missing trust or accepts a linked role key", async () => { + const a = await api(); + await a.provisionControlInstallation(root); + rmSync(join(root, "secrets/control-verifiers.json")); + await expect(a.provisionControlInstallation(root)).rejects.toThrow(/trust|verifier|installation/); +}); +it.each(["permissions", "hardlink", "symlink"])("rejects unsafe role key %s", async (mode) => { + const a = await api(); + await a.provisionControlInstallation(root); + const path = join(root, "secrets/control-ops-agent.key"); + if (mode === "permissions") chmodSync(path, 0o644); + if (mode === "hardlink") linkSync(path, join(root, "copy")); + if (mode === "symlink") { + const bytes = readFileSync(path); + rmSync(path); + writeFileSync(join(root, "copy"), bytes, { mode: 0o600 }); + symlinkSync(join(root, "copy"), path); + } + expect(() => a.openControlInstallation({ infraDir: root, role: "ops-agent" })).toThrow(); +}); +it("rejects role-key substitution even with the same UID and valid Ed25519 key", async () => { + const a = await api(); + await a.provisionControlInstallation(root); + writeFileSync( + join(root, "secrets/control-ops-agent.key"), + readFileSync(join(root, "secrets/control-data-manager.key")), + ); + expect(() => a.openControlInstallation({ infraDir: root, role: "ops-agent" })).toThrow( + /role|key|trust/, + ); +}); +it("provides deadline-only role attestations without target ownership", async () => { + const module = await api(); + expect(module).toHaveProperty("attestParentExecution"); + expect(module).toHaveProperty("verifyParentExecution"); +}); +it("rejects duplicate control headers and the aggregate header budget", async () => { + const a = await api(); + expect(() => + a.controlHeaders([ + "X-OpenMapx-Control-Delegation", + "one", + "x-openmapx-control-delegation", + "two", + ]), + ).toThrow(/Duplicate/); + expect(() => a.controlHeaders(["x-openmapx-parent-execution", "x".repeat(4097)])).toThrow( + /limit/, + ); +}); +it("retains deadline-only attempts separately from execution witnesses", async () => { + const a = await api(); + const { randomUUID } = await import("node:crypto"); + const { executionFingerprint } = await import("../ops/execution-identity"); + const { opsOperationFingerprint } = await import("../ops/fingerprint"); + await a.provisionControlInstallation(root); + const installation = a.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["osrm"], + }); + const operation = { kind: "data.update" as const }; + const deadlineAt = Date.now() + 60000; + const operationFingerprint = opsOperationFingerprint(operation); + const parent = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const body = { kind: "download-osm", regionId: "europe/germany" }; + const hash = a.controlDigest(body); + const child = { + executor: "data-manager" as const, + principal: "data-manager" as const, + executionId: randomUUID(), + operationKey: a.controlChildOperationKey(parent.operationKey, "osm", 0), + operationFingerprint: hash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: body.kind, + operationFingerprint: hash, + deadlineAt, + parent: a.retainedControlParent(parent), + }), + }; + const plan = { + phase: "osm" as const, + ordinal: 0, + kind: body.kind, + execution: child, + serviceIds: ["osrm"], + recipient: "data-manager" as const, + }; + const authority = a.retainAdmittedControlExecution(installation, { + execution: parent, + operation, + kind: operation.kind, + admittedAt: deadlineAt - 60000, + serviceIds: ["osrm"], + plan: [plan], + }); + const receiver = a.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["osrm"], + }); + const assertion = await a.attestParentExecution(authority, { + ...plan, + childPublicKey: receiver.publicKey, + }); + const peer = a.verifyControlPeer(receiver, { + rawHeaders: ["x-openmapx-parent-execution", assertion], + transportPrincipal: "shared-data-manager-bearer", + execution: child, + kind: body.kind, + }); + expect( + a.verifyParentExecution(assertion, peer, { + execution: child, + kind: body.kind, + requestedDeadlineAt: deadlineAt, + }), + ).toMatchObject({ operationKey: parent.operationKey }); + await expect( + a.attestParentExecution(authority, { ...plan, childPublicKey: receiver.publicKey }), + ).rejects.toThrow(/replay/); + a.closeControlInstallation(receiver); + a.closeControlInstallation(installation); +}); +it("service mounts keep same-UID sibling role material outside the shared mount", () => { + const repo = join(import.meta.dirname, "../../../.."); + const agent = JSON.parse(readFileSync(join(repo, "services/ops-agent/service.json"), "utf8")); + const dm = JSON.parse(readFileSync(join(repo, "services/data-manager/service.json"), "utf8")); + for (const [role, manifest] of [ + ["ops-agent", agent], + ["data-manager", dm], + ] as const) { + expect(manifest.bindMounts).toContainEqual({ + source: `@infra:secrets/control-${role}.key`, + target: "/run/secrets/openmapx-control-role-key", + readOnly: true, + }); + expect(manifest.bindMounts).toContainEqual({ + source: `@infra:control-private/${role}`, + target: "/var/lib/openmapx/control-private", + readOnly: false, + }); + expect( + manifest.bindMounts.some( + (m: { source: string }) => + m.source === + `@infra:control-private/${role === "ops-agent" ? "data-manager" : "ops-agent"}`, + ), + ).toBe(false); + } + for (const role of ["data-manager", "host-cli"]) { + expect( + agent.bindMounts.some( + (m: { source: string; target: string; readOnly: boolean }) => + m.source === "@infra:control-private/hidden-key" && + m.target.endsWith(`/secrets/control-${role}.key`) && + m.readOnly, + ), + ).toBe(true); + expect( + agent.bindMounts.some( + (m: { source: string; target: string; readOnly: boolean }) => + m.source === "@infra:control-private/empty" && + m.target.endsWith(`/control-private/${role}`) && + m.readOnly, + ), + ).toBe(true); + } +}); +it("verifies the certified ephemeral fixed-CLI deadline chain without exporting a role key", async () => { + const c = await api(); + const { randomUUID } = await import("node:crypto"); + const { executionFingerprint } = await import("../ops/execution-identity"); + const { opsOperationFingerprint } = await import("../ops/fingerprint"); + await c.provisionControlInstallation(root); + const agent = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["osrm"], + }); + const dm = c.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["osrm"], + }); + const deadlineAt = Date.now() + 60000; + const operation = { kind: "data.update" as const }; + const hash = opsOperationFingerprint(operation); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint: hash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: hash, + deadlineAt, + }), + }; + const cli = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: c.controlChildOperationKey(execution.operationKey, "data-workflow", 0), + operationFingerprint: hash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: hash, + deadlineAt, + parent: c.retainedControlParent(execution), + }), + }; + const body = { kind: "download-osm", regionId: "europe/germany" }; + const childHash = c.controlDigest(body); + const child = { + executor: "data-manager" as const, + principal: "data-manager" as const, + executionId: randomUUID(), + operationKey: c.controlChildOperationKey(cli.operationKey, "osm", 0), + operationFingerprint: childHash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: body.kind, + operationFingerprint: childHash, + deadlineAt, + parent: c.retainedControlParent(cli), + }), + }; + const planned = { + phase: "data-workflow" as const, + ordinal: 0, + kind: operation.kind, + execution: cli, + serviceIds: ["osrm"], + recipient: "fixed-cli" as const, + }; + const leaf = { + phase: "osm" as const, + ordinal: 0, + kind: body.kind, + execution: child, + serviceIds: ["osrm"], + recipient: "data-manager" as const, + }; + const authority = c.retainAdmittedControlExecution(agent, { + execution, + kind: operation.kind, + operation, + admittedAt: deadlineAt - 60000, + serviceIds: ["osrm"], + plan: [planned], + }); + const id = randomUUID(); + const publicKey = c.prepareFixedControlChild(authority, id); + const assertion = await c.attestParentExecution(authority, { + ...planned, + childPublicKey: publicKey, + }); + const args = c.fixedCliArguments(operation); + c.retainFixedCliSpawnIntent(authority, { + delegationId: id, + execution: cli, + operation, + args, + plan: [leaf], + }); + const fixed = c.openFixedChildInstallation({ infraDir: root, assertion }); + expect(fixed.publicKey).not.toBe(agent.publicKey); + const intent = c.readFixedCliSpawnIntent(fixed, cli, args); + const peer = c.verifyFixedCliControlPeer(fixed, { + assertion, + execution: cli, + kind: operation.kind, + }); + const parent = c.verifyParentExecution(assertion, peer, { + execution: cli, + kind: operation.kind, + requestedDeadlineAt: deadlineAt, + }); + const cliAuthority = c.retainAdmittedControlExecution(fixed, { + execution: cli, + kind: operation.kind, + operation, + admittedAt: intent.admittedAt, + parent, + serviceIds: intent.serviceIds, + plan: intent.plan, + }); + const next = await c.attestParentExecution(cliAuthority, { + ...leaf, + childPublicKey: dm.publicKey, + }); + const dmPeer = c.verifyControlPeer(dm, { + rawHeaders: ["x-openmapx-parent-execution", next], + transportPrincipal: "shared-data-manager-bearer", + execution: child, + kind: body.kind, + }); + expect( + c.verifyParentExecution(next, dmPeer, { + execution: child, + kind: body.kind, + requestedDeadlineAt: deadlineAt, + }), + ).toMatchObject({ operationKey: cli.operationKey }); + c.closeControlInstallation(fixed); + c.closeControlInstallation(dm); + c.closeControlInstallation(agent); +}); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts new file mode 100644 index 000000000..3031c1058 --- /dev/null +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -0,0 +1,1758 @@ +import { + createHash, + createPrivateKey, + createPublicKey, + generateKeyPairSync, + type KeyObject, + randomUUID, + sign, + verify, +} from "node:crypto"; +import { existsSync, fsyncSync, mkdirSync, realpathSync } from "node:fs"; +import { join, relative } from "node:path"; +import { z } from "zod"; +import { type OpsOperationKind, opsOperationSchema, parseBoundedOpsResult } from "../ops/contract"; +import { + type ControlIssuer, + type ExecutionRef, + executionFingerprint, + executionRefSchema, + opsExecutionMaximumMs, + type RetainedParentExecution, +} from "../ops/execution-identity"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { verifiedParents } from "../ops/verified-parent-internal"; +import { AnchoredDirectory, identity } from "./filesystem"; +export const CONTROL_ROLES = ["ops-agent", "data-manager", "host-cli"] as const; +export type ControlRecipient = "fixed-cli" | "data-manager" | "ops-agent" | "configuration-helper"; +export type ControlChildPhase = + | "lifecycle" + | "build" + | "data-link" + | "data-clean" + | "data-workflow" + | "hardlinks" + | "stop-primary" + | "stop-staging" + | "restart-staging" + | "promote-primary" + | "recover-primary" + | "proxy-stage" + | "proxy-commit" + | "proxy-restore" + | "paired-pin-commit" + | "configuration-fence" + | "configuration-fence-settle" + | "configuration-commit" + | "maintenance-hardlinks" + | "maintenance-recreate" + | "maintenance-handoff" + | "osm" + | "transit" + | "fonts" + | "data-step" + | "configuration-handoff"; +declare const authorityBrand: unique symbol, + peerBrand: unique symbol, + settlementBrand: unique symbol, + recoveryBrand: unique symbol, + installationBrand: unique symbol, + assertionBrand: unique symbol; +export type VerifiedExecutionAuthority = Readonly<{ + execution: ExecutionRef; + [authorityBrand]: true; +}>; +export type VerifiedControlPeer = Readonly<{ [peerBrand]: true }>; +export type VerifiedChildSettlement = Readonly<{ [settlementBrand]: true }>; +export type VerifiedRecoveryAuthority = Readonly<{ [recoveryBrand]: true }>; +export type ArtifactDelegationAssertion = string & { readonly [assertionBrand]: true }; +export type ControlInstallation = Readonly<{ + installationId: string; + role: ControlIssuer; + publicKey: string; + [installationBrand]: true; +}>; +export type ChildSettlement = { + identity: ExecutionRef; + outcome: "contained" | "committed" | "failed_without_effect" | "unknown"; + publication: "not_started" | "committed" | "unknown"; + terminalReceiptDigest: string; +}; +export const uuid = z.uuid(); +export const digest = z.string().regex(/^[a-f0-9]{64}$/); +const publicKey = z.string().max(256); +const bundleSchema = z.strictObject({ + installationId: uuid, + publicKeys: z.strictObject({ + "ops-agent": publicKey, + "data-manager": publicKey, + "host-cli": publicKey, + }), +}); +const roleKeySchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + role: z.enum(CONTROL_ROLES), + privateKey: z.string().max(512), +}); +export const targetSchema = z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine((x) => x.every((v, i) => i === 0 || x[i - 1]! < v), "Targets must be sorted and unique"); +export function controlDigest(value: unknown): string { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} +export function signControl( + key: KeyObject, + domain: string, + payload: T, +): { payload: T; signature: string } { + return { + payload, + signature: sign(null, Buffer.from(`${domain}\0${canonicalJson(payload)}`), key).toString( + "base64url", + ), + }; +} +export function verifyControl( + key: string, + domain: string, + envelope: unknown, + schema: z.ZodType, +): T { + const e = z + .strictObject({ payload: schema, signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }) + .parse(envelope); + const k = createPublicKey({ key: Buffer.from(key, "base64url"), format: "der", type: "spki" }); + if ( + k.asymmetricKeyType !== "ed25519" || + !verify( + null, + Buffer.from(`${domain}\0${canonicalJson(e.payload)}`), + k, + Buffer.from(e.signature, "base64url"), + ) + ) + throw Error("Invalid control signature"); + return e.payload; +} +export function newControlKey() { + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + return { + privateKey, + publicKey: publicKey.export({ type: "spki", format: "der" }).toString("base64url"), + }; +} +function parsePrivate(encoded: string) { + const key = createPrivateKey({ + key: Buffer.from(encoded, "base64url"), + format: "der", + type: "pkcs8", + }); + if (key.asymmetricKeyType !== "ed25519") throw Error("Invalid control key kind"); + return key; +} +export const encodePrivate = (key: KeyObject) => + key.export({ type: "pkcs8", format: "der" }).toString("base64url"); +export async function provisionControlInstallation(infraDir: string): Promise { + const infra = new AnchoredDirectory(infraDir); + let secrets: AnchoredDirectory | undefined, privateRoot: AnchoredDirectory | undefined; + try { + secrets = infra.directory("secrets", true); + const names = secrets.list(); + const existing = names.filter( + (n) => + n === "control-verifiers.json" || + /^control-(ops-agent|data-manager|host-cli)\.key$/.test(n), + ); + if (existing.length && existing.length !== 4) + throw Error( + "Incomplete control installation trust; stop all writers/helpers and recover existing keys/verifiers, never regenerate", + ); + if (!existing.length) { + if (existsSync(join(infraDir, "data/.admin-target-control"))) + throw Error("Existing control records require original installation trust"); + const installationId = randomUUID(); + const keys = Object.fromEntries(CONTROL_ROLES.map((role) => [role, newControlKey()])); + for (const role of CONTROL_ROLES) { + secrets.write( + `control-${role}.key`, + canonicalJson({ + version: 1, + installationId, + role, + privateKey: encodePrivate(keys[role]?.privateKey), + }), + true, + ); + } + secrets.write( + "control-verifiers.json", + canonicalJson({ + installationId, + publicKeys: Object.fromEntries( + CONTROL_ROLES.map((role) => [role, keys[role]?.publicKey]), + ), + }), + true, + ); + } + const bundle = bundleSchema.parse( + JSON.parse(secrets.read("control-verifiers.json", 4096).toString("utf8")), + ); + for (const role of CONTROL_ROLES) { + const saved = roleKeySchema.parse( + JSON.parse(secrets.read(`control-${role}.key`, 2048).toString("utf8")), + ); + if ( + saved.role !== role || + saved.installationId !== bundle.installationId || + createPublicKey(parsePrivate(saved.privateKey)) + .export({ type: "spki", format: "der" }) + .toString("base64url") !== bundle.publicKeys[role] + ) + throw Error("Control role key does not match fixed trust"); + } + privateRoot = infra.directory("control-private", true, true); + privateRoot.directory("empty", true, true).close(); + if (!privateRoot.list().includes("hidden-key")) privateRoot.write("hidden-key", "", true); + for (const role of CONTROL_ROLES) privateRoot.directory(role, true, true).close(); + } finally { + privateRoot?.close(); + secrets?.close(); + infra.close(); + } +} +export interface InstallationState { + fixedChildId?: string; + fixedParentAssertion?: string; + installation: ControlInstallation; + privateKey: KeyObject; + publicKeys: Record; + privateRoot: AnchoredDirectory; + dataRoot: string; + registeredTargets: ReadonlySet; +} +const installations = new WeakMap(); +export function installationState(value: ControlInstallation): InstallationState { + const state = installations.get(value); + if (!state) throw Error("Unverified control installation"); + state.privateRoot.check(); + return state; +} +/** Called only at fixed host/service startup; infraDir is the configured installation, never a request field. */ +export function openControlInstallation(options: { + infraDir: string; + role: ControlIssuer; + serviceIds?: readonly string[]; + container?: boolean; +}): ControlInstallation { + if (options.serviceIds?.length) targetSchema.parse([...new Set(options.serviceIds)].sort()); + const infra = options.container ? undefined : new AnchoredDirectory(options.infraDir); + const secrets = options.container + ? new AnchoredDirectory("/run/secrets", undefined, false, true) + : (() => { + if (!infra) throw Error("Missing fixed installation root"); + return infra.directory("secrets"); + })(); + let privateRoot: AnchoredDirectory | undefined; + try { + const bundle = bundleSchema.parse( + JSON.parse( + secrets + .read(options.container ? "openmapx-control-verifiers" : "control-verifiers.json", 4096) + .toString("utf8"), + ), + ); + const saved = roleKeySchema.parse( + JSON.parse( + secrets + .read( + options.container ? "openmapx-control-role-key" : `control-${options.role}.key`, + 2048, + ) + .toString("utf8"), + ), + ); + const key = parsePrivate(saved.privateKey); + if ( + saved.role !== options.role || + saved.installationId !== bundle.installationId || + createPublicKey(key).export({ type: "spki", format: "der" }).toString("base64url") !== + bundle.publicKeys[options.role] + ) + throw Error("Control role key/trust mismatch"); + privateRoot = new AnchoredDirectory( + options.container + ? "/var/lib/openmapx/control-private" + : join(options.infraDir, "control-private", options.role), + undefined, + true, + ); + const dataRoot = + options.container && options.role === "data-manager" + ? "/data" + : join(options.infraDir, "data"); + const privateRelative = relative(realpathSync(dataRoot), realpathSync(privateRoot.path)); + if (!privateRelative || (!privateRelative.startsWith("../") && privateRelative !== "..")) + throw Error("Role recovery material is inside the shared data mount"); + const data = new AnchoredDirectory(dataRoot); + try { + if ( + identity(data.stat).dev === identity(privateRoot.stat).dev && + identity(data.stat).ino === identity(privateRoot.stat).ino + ) + throw Error("Role recovery material must be outside shared data"); + } finally { + data.close(); + } + const installation = Object.freeze({ + installationId: bundle.installationId, + role: options.role, + publicKey: bundle.publicKeys[options.role], + }) as ControlInstallation; + installations.set(installation, { + installation, + privateKey: key, + publicKeys: bundle.publicKeys, + privateRoot, + dataRoot, + registeredTargets: new Set(options.serviceIds ?? []), + }); + privateRoot = undefined; + return installation; + } finally { + privateRoot?.close(); + secrets.close(); + infra?.close(); + } +} +export function closeControlInstallation(installation: ControlInstallation) { + const s = installationState(installation); + s.privateRoot.close(); + installations.delete(installation); +} +export const childPhaseKinds: Readonly> = { + lifecycle: ["service.start", "system.update"], + build: ["service.build", "services.buildAll"], + "data-link": ["data.link", "configuration.hardlinks.apply"], + "data-clean": ["data.clean"], + "data-workflow": [ + "data.update", + "data.downloadOsm", + "data.downloadFonts", + "data.convertOverpass", + "data.overtureSync", + "data.overtureConflate", + "data.searchIndexBuild", + "data.notablePlacesBuild", + ], + hardlinks: ["generation-hardlinks"], + "stop-primary": ["motis.primary.stop"], + "stop-staging": ["motis.staging.stop"], + "restart-staging": ["motis.staging.restart"], + "promote-primary": ["motis.primary.promote"], + "recover-primary": ["motis.primary.restart"], + "proxy-stage": ["feedProxy.validateAndReload"], + "proxy-commit": ["feedProxy.validateAndReload"], + "proxy-restore": ["feedProxy.validateAndReload"], + "paired-pin-commit": ["transitousLock.approve"], + "configuration-fence": ["configuration-maintenance.prepare"], + "configuration-fence-settle": ["configuration-maintenance.settle"], + "configuration-commit": [ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ], + "maintenance-hardlinks": ["configuration.hardlinks.apply"], + "maintenance-recreate": ["service.recreate", "service.recreateIsolated"], + "maintenance-handoff": ["configurationHandoff.prepare"], + osm: ["download-osm"], + transit: ["transit-sync"], + fonts: ["download-fonts"], + "data-step": [ + "download-osm", + "download-fonts", + "convert-overpass", + "overture-sync", + "overture-conflate", + "search-index-build", + "notable-places-build", + ], + "configuration-handoff": ["configuration-helper"], +}; +export function controlChildOperationKey( + parentKey: string, + phase: ControlChildPhase, + ordinal: number, +): string { + if ( + !/^opk1_[A-Za-z0-9_-]{16,64}$/.test(parentKey) || + !Number.isSafeInteger(ordinal) || + ordinal < 0 || + ordinal > 63 || + !Object.hasOwn(childPhaseKinds, phase) + ) + throw Error("Invalid fixed phase key"); + return `opk1_${createHash("sha256").update(`openmapx-control-child-v1\0${parentKey}\0${phase}\0${ordinal}`).digest("base64url").slice(0, 32)}`; +} +export interface ControlPlannedChild { + phase: ControlChildPhase; + ordinal: number; + kind: string; + execution: ExecutionRef; + serviceIds: readonly string[]; + recipient: ControlRecipient; +} +const plannedChildSchema = z.strictObject({ + phase: z.enum(Object.keys(childPhaseKinds) as [ControlChildPhase, ...ControlChildPhase[]]), + ordinal: z.number().int().min(0).max(63), + kind: z.string().max(128), + execution: executionRefSchema, + serviceIds: targetSchema, + recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), +}); +export interface AuthorityState { + installation: InstallationState; + execution: ExecutionRef; + kind: string; + serviceIds: readonly string[]; + plan: readonly ControlPlannedChild[]; + receiptDigest: string; +} +const authorities = new WeakMap(); +export function executionAuthorityState(value: VerifiedExecutionAuthority) { + const s = authorities.get(value); + if (!s) throw Error("Unverified execution authority"); + s.installation.privateRoot.check(); + const current = readPrivateRecord( + s.installation, + `executions`, + `${s.execution.operationKey}.json`, + 65536, + ); + if (controlDigest(current) !== s.receiptDigest) + throw Error("Durable execution authority changed"); + return s; +} +export function writePrivateRecord( + state: InstallationState, + namespace: string, + name: string, + value: unknown, + exclusive = false, +) { + const dir = state.privateRoot.directory(namespace, true, true); + try { + const bytes = canonicalJson(value); + if (Buffer.byteLength(bytes) > 131072) throw Error("Private receipt limit"); + dir.write(name, bytes, exclusive); + } finally { + dir.close(); + } +} +export function readPrivateRecord( + state: InstallationState, + namespace: string, + name: string, + max = 131072, +): unknown { + const dir = state.privateRoot.directory(namespace); + try { + return JSON.parse(dir.read(name, max).toString("utf8")); + } finally { + dir.close(); + } +} +/** Fixed adapters call after their own durable admission, passing the complete typed operation, never only its IDs. */ +export function retainAdmittedControlExecution( + installation: ControlInstallation, + input: { + execution: ExecutionRef; + kind: string; + operation: unknown; + admittedAt: number; + parent?: RetainedParentExecution; + serviceIds: readonly string[]; + plan?: readonly ControlPlannedChild[]; + }, +): VerifiedExecutionAuthority { + const state = installationState(installation); + const execution = Object.freeze(executionRefSchema.parse(input.execution)); + const serviceIds = targetSchema.parse(input.serviceIds); + if ( + (execution.executor !== installation.role && + !(state.fixedChildId && execution.executor === "host-cli")) || + serviceIds.some((id) => !state.registeredTargets.has(id)) + ) + throw Error("Unregistered execution role/targets"); + const parsed = opsOperationSchema.safeParse(input.operation); + if (parsed.success) { + if ( + parsed.data.kind !== input.kind || + opsOperationFingerprint(parsed.data) !== execution.operationFingerprint + ) + throw Error("Admitted operation body mismatch"); + } else if ( + ![ + "transit-sync", + "download-osm", + "download-fonts", + "convert-overpass", + "overture-sync", + "overture-conflate", + "search-index-build", + "notable-places-build", + "catalog.activate", + "catalog.validate", + "artifactTransfer.import", + "artifactTransfer.export", + "artifactTransfer.validate", + "transit-maintenance", + "generation-hardlinks", + "configuration-maintenance.prepare", + "configuration-maintenance.settle", + "configurationMaintenance.prepare", + "configurationHandoff.prepare", + "configuration-helper", + ].includes(input.kind) || + controlDigest(input.operation) !== execution.operationFingerprint + ) + throw Error("Unsupported fixed control operation"); + if ( + !Number.isSafeInteger(input.admittedAt) || + execution.deadlineAt <= input.admittedAt || + execution.fingerprint !== + executionFingerprint({ + kind: input.kind, + operationFingerprint: execution.operationFingerprint, + deadlineAt: execution.deadlineAt, + ...(input.parent ? { parent: input.parent } : {}), + }) + ) + throw Error("Immutable execution identity mismatch"); + const plan = z + .array(plannedChildSchema) + .max(64) + .parse(input.plan ?? []); + const budget = (() => { + try { + return opsExecutionMaximumMs(input.kind); + } catch { + if ( + [ + "configurationMaintenance.prepare", + "configurationHandoff.prepare", + "configuration-helper", + ].includes(input.kind) + ) + return 3600000; + if ( + ["configuration-maintenance.prepare", "configuration-maintenance.settle"].includes( + input.kind, + ) + ) + return 60000; + throw Error("Unknown control kind budget"); + } + })(); + if (execution.deadlineAt > input.admittedAt + budget) + throw Error("Control execution exceeds kind budget"); + if (plan.length > 64) throw Error("Control child intent capacity"); + const keys = new Set(); + for (const [ordinal, child] of plan.entries()) { + executionRefSchema.parse(child.execution); + targetSchema.parse(child.serviceIds); + if ( + child.ordinal !== ordinal || + keys.has(child.execution.operationKey) || + child.execution.operationKey !== + controlChildOperationKey(execution.operationKey, child.phase, child.ordinal) || + !childPhaseKinds[child.phase]?.includes(child.kind) || + child.execution.deadlineAt > execution.deadlineAt || + child.serviceIds.some((id) => !serviceIds.includes(id)) + ) + throw Error("Invalid fixed child plan"); + keys.add(child.execution.operationKey); + } + const receipt = signControl(state.privateKey, "openmapx-control-execution-v1", { + version: 1, + installationId: installation.installationId, + ...input, + execution, + serviceIds, + plan, + }); + if (Buffer.byteLength(canonicalJson(receipt)) > 65536) + throw Error("Execution authority receipt capacity"); + const name = `${execution.operationKey}.json`; + try { + const old = readPrivateRecord(state, "executions", name, 65536); + if (controlDigest(old) !== controlDigest(receipt)) + throw Error("Conflicting retained execution identity"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + writePrivateRecord(state, "executions", name, receipt, true); + } + publishControlExecutionWitness( + state, + { execution, kind: input.kind, serviceIds }, + controlDigest(receipt), + ); + const authority = Object.freeze({ execution }) as VerifiedExecutionAuthority; + authorities.set(authority, { + installation: state, + execution, + kind: input.kind, + serviceIds, + plan, + receiptDigest: controlDigest(receipt), + }); + return authority; +} +export interface PeerState { + installation: InstallationState; + role: ControlIssuer; + local?: boolean; + recipient: ControlRecipient; + execution: ExecutionRef; + kind: string; + assertion: string; + childPrivateKey: KeyObject; + childPublicKey: string; +} +const peers = new WeakMap(); +export function controlPeerState(peer: VerifiedControlPeer) { + const s = peers.get(peer); + if (!s) throw Error("Unverified control peer"); + return s; +} +export const assertionSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + leaseId: uuid, + ownerCertificateDigest: digest, + delegationId: uuid, + grantDigest: digest, + ancestorDelegationIds: z.array(uuid).max(2), + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), +}); +export function parseControlAssertion(assertion: string) { + if (Buffer.byteLength(assertion) > 4096 || !/^[A-Za-z0-9_-]+$/.test(assertion)) + throw Error("Invalid control assertion encoding/limit"); + const bytes = Buffer.from(assertion, "base64url"); + if (bytes.toString("base64url") !== assertion) throw Error("Invalid control assertion encoding"); + const parsed = assertionSchema.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), + ); + if (Buffer.from(canonicalJson(parsed)).toString("base64url") !== assertion) + throw Error("Noncanonical control assertion"); + return parsed; +} +export function controlHeaders(rawHeaders: readonly string[]): { + delegation?: string; + parent?: string; +} { + const found = new Map(); + for (let i = 0; i < rawHeaders.length; i += 2) { + const name = rawHeaders[i]?.toLowerCase(); + if (!["x-openmapx-control-delegation", "x-openmapx-parent-execution"].includes(name)) continue; + if (found.has(name) || typeof rawHeaders[i + 1] !== "string") + throw Error("Duplicate control header"); + found.set(name, rawHeaders[i + 1]!); + } + const delegation = found.get("x-openmapx-control-delegation"), + parent = found.get("x-openmapx-parent-execution"); + if ( + (delegation?.length ?? 0) + (parent?.length ?? 0) > 8192 || + (delegation?.length ?? 0) > 4096 || + (parent?.length ?? 0) > 4096 + ) + throw Error("Control header limit"); + return { delegation, parent }; +} +/** Transport authentication is required by the fixed caller. Signature authentication finishes during anchored adoption. */ +export function verifyControlPeer( + installation: ControlInstallation, + input: { + rawHeaders: readonly string[]; + transportPrincipal: "data-manager" | "host-cli" | "shared-data-manager-bearer"; + execution: ExecutionRef; + kind: string; + }, +): VerifiedControlPeer { + const state = installationState(installation); + const headers = controlHeaders(input.rawHeaders); + const assertionText = headers.delegation ?? headers.parent; + if (!assertionText) throw Error("Missing control assertion"); + const assertion = headers.delegation + ? parseControlAssertion(assertionText) + : parseParentAssertion(assertionText); + if (assertion.installationId !== installation.installationId) + throw Error("Foreign control installation"); + if (installation.role === "ops-agent" && input.transportPrincipal !== "data-manager") + throw Error("Wrong peer transport principal"); + if (installation.role === "host-cli") throw Error("Host CLI is not an HTTP receiver"); + if (installation.role === "data-manager" && input.transportPrincipal === "data-manager") + throw Error("Wrong peer transport principal"); + const peer = Object.freeze({}) as VerifiedControlPeer; + peers.set(peer, { + installation: state, + role: installation.role === "ops-agent" ? "data-manager" : "ops-agent", + recipient: installation.role, + execution: executionRefSchema.parse(input.execution), + kind: input.kind, + assertion: assertionText, + childPrivateKey: state.privateKey, + childPublicKey: installation.publicKey, + }); + return peer; +} +/** Same-agent maintenance continuations must match this role's private attempted intent. */ +export function verifyRetainedLocalControlPeer( + installation: ControlInstallation, + input: { assertion: string; execution: ExecutionRef; kind: string }, +): VerifiedControlPeer { + const state = installationState(installation); + const assertion = parseControlAssertion(input.assertion); + if (installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Local agent continuation required"); + const saved = z + .object({ + assertion: z.string(), + grant: z.object({ payload: z.object({ child: executionRefSchema, childKind: z.string() }) }), + }) + .parse(readPrivateRecord(state, "attempts", `${assertion.delegationId}.json`)); + if ( + saved.assertion !== input.assertion || + saved.grant.payload.childKind !== input.kind || + controlDigest(saved.grant.payload.child) !== controlDigest(input.execution) + ) + throw Error("Local continuation intent mismatch"); + const peer = Object.freeze({}) as VerifiedControlPeer; + peers.set(peer, { + installation: state, + role: "ops-agent", + local: true, + recipient: "ops-agent", + execution: input.execution, + kind: input.kind, + assertion: input.assertion, + childPrivateKey: state.privateKey, + childPublicKey: installation.publicKey, + }); + return peer; +} +export function prepareFixedControlChild( + authority: VerifiedExecutionAuthority, + delegationId: string, +): string { + uuid.parse(delegationId); + const s = executionAuthorityState(authority); + if (s.installation.installation.role !== "ops-agent") + throw Error("Only agent prepares fixed children"); + const key = newControlKey(); + writePrivateRecord( + s.installation, + "children", + `${delegationId}.key`, + { publicKey: key.publicKey, privateKey: encodePrivate(key.privateKey) }, + true, + ); + return key.publicKey; +} +export function verifyFixedCliControlPeer( + installation: ControlInstallation, + input: { + assertion: string; + execution: ExecutionRef; + kind: string; + recipient?: "fixed-cli" | "configuration-helper"; + }, +): VerifiedControlPeer { + const state = installationState(installation); + const encoded = JSON.parse(Buffer.from(input.assertion, "base64url").toString("utf8")) as { + leaseId?: unknown; + }; + const assertion = encoded.leaseId + ? parseControlAssertion(input.assertion) + : parseParentAssertion(input.assertion); + const fixedId = "delegationId" in assertion ? assertion.delegationId : assertion.attemptId; + if (installation.role !== "ops-agent" || assertion.installationId !== installation.installationId) + throw Error("Invalid fixed CLI installation"); + const saved = z + .strictObject({ publicKey, privateKey: z.string().max(512) }) + .parse(readPrivateRecord(state, "children", `${fixedId}.key`, 2048)); + const key = parsePrivate(saved.privateKey); + if ( + createPublicKey(key).export({ type: "spki", format: "der" }).toString("base64url") !== + saved.publicKey + ) + throw Error("Fixed child challenge mismatch"); + const peer = Object.freeze({}) as VerifiedControlPeer; + peers.set(peer, { + installation: state, + role: "ops-agent", + recipient: input.recipient ?? "fixed-cli", + execution: executionRefSchema.parse(input.execution), + kind: input.kind, + assertion: input.assertion, + childPrivateKey: key, + childPublicKey: saved.publicKey, + }); + return peer; +} +const settlements = new WeakMap(); +export function childSettlementState(evidence: VerifiedChildSettlement) { + const result = settlements.get(evidence); + if (!result) throw Error("Unverified child settlement"); + return result; +} +const settlementSchema = z.strictObject({ + identity: executionRefSchema, + outcome: z.enum(["contained", "committed", "failed_without_effect", "unknown"]), + publication: z.enum(["not_started", "committed", "unknown"]), + terminalReceiptDigest: digest, +}); +const effectObservationSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("ops-result"), result: z.unknown() }), + z.strictObject({ + kind: z.literal("fixed-cli-exit"), + args: z.array(z.string().max(256)).max(64), + exitCode: z.literal(0), + }), + z.strictObject({ kind: z.literal("not-started") }), + z.strictObject({ kind: z.literal("unresolved") }), +]); +function withPrivateEffectWriter(s: AuthorityState, write: () => void): void { + const guards = s.installation.privateRoot.directory("effect-writers", true, true); + try { + const name = s.execution.operationKey; + mkdirSync(guards.name(name), { mode: 0o700 }); + fsyncSync(guards.fd); + const guard = guards.directory(name); + const held = identity(guard.stat); + try { + write(); + } finally { + guard.close(); + guards.removeDirectory(name, held); + } + } finally { + guards.close(); + } +} +export function beginControlEffect(authority: VerifiedExecutionAuthority): void { + const s = executionAuthorityState(authority); + withPrivateEffectWriter(s, () => { + try { + readPrivateRecord(s.installation, "effects", `${s.execution.operationKey}.json`); + throw Error("Settled execution cannot restart effects"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + writePrivateRecord( + s.installation, + "effect-started", + `${s.execution.operationKey}.json`, + { execution: s.execution, receiptDigest: s.receiptDigest }, + true, + ); + }); +} +/** Fixed executor adapters supply their parsed result or exact fixed-command exit; arbitrary text is never proof. */ +export function retainControlEffectReceipt( + authority: VerifiedExecutionAuthority, + effect: { + outcome: ChildSettlement["outcome"]; + publication: ChildSettlement["publication"]; + receipt: unknown; + }, +): void { + const s = executionAuthorityState(authority); + withPrivateEffectWriter(s, () => { + const parsed = effectObservationSchema.safeParse(effect.receipt); + if (!parsed.success) throw Error("Unsupported effect receipt evidence"); + const receipt = parsed.data; + let outcome: ChildSettlement["outcome"] = "unknown", + publication: ChildSettlement["publication"] = "unknown"; + if (receipt.kind === "ops-result") { + receipt.result = parseBoundedOpsResult(s.kind as OpsOperationKind, receipt.result); + const result = receipt.result as { outcome?: string }; + if (!result.outcome || ["applied", "unchanged", "committed"].includes(result.outcome)) { + outcome = "committed"; + publication = "committed"; + } + } else if (receipt.kind === "fixed-cli-exit") { + if (!s.installation.fixedChildId) + throw Error("Fixed command receipt requires certified child"); + readFixedCliSpawnIntent(s.installation.installation, s.execution, receipt.args); + readPrivateRecord(s.installation, "spawn-started", `${s.installation.fixedChildId}.json`); + outcome = "committed"; + publication = "committed"; + } else if (receipt.kind === "not-started") { + try { + readPrivateRecord(s.installation, "effect-started", `${s.execution.operationKey}.json`); + throw Error("Started effects require containment evidence"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (s.installation.fixedChildId) throw Error("Fixed child cannot assert unused execution"); + outcome = "failed_without_effect"; + publication = "not_started"; + } + if (effect.outcome !== outcome || effect.publication !== publication) + throw Error("Effect evidence does not prove requested settlement"); + const payload = { + identity: s.execution, + outcome, + publication, + terminalReceiptDigest: controlDigest(receipt), + receipt, + }; + const signed = signControl( + s.installation.privateKey, + "openmapx-control-settlement-v1", + payload, + ); + const name = `${s.execution.operationKey}.json`; + try { + const prior = verifyControl( + s.installation.installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord(s.installation, "effects", name, 65536), + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if (controlDigest(prior) === controlDigest(payload)) return; + // A later transport/journal failure must never erase proved committed effects. + if (prior.outcome !== "unknown") return; + writePrivateRecord(s.installation, "effects", name, signed); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + writePrivateRecord(s.installation, "effects", name, signed, true); + } + }); +} +export function verifyControlEffectSettlement( + installation: ControlInstallation, + execution: ExecutionRef, +): VerifiedChildSettlement { + const s = installationState(installation); + const evidence = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord(s, "effects", `${execution.operationKey}.json`, 65536), + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if (evidence.terminalReceiptDigest !== controlDigest(evidence.receipt)) + throw Error("Effect receipt digest mismatch"); + if (controlDigest(evidence.identity) !== controlDigest(execution)) + throw Error("Settlement immutable identity mismatch"); + const verified = Object.freeze({}) as VerifiedChildSettlement; + const { receipt: _privateReceipt, ...publicEvidence } = evidence; + settlements.set(verified, { state: s, evidence: publicEvidence }); + return verified; +} +const recovery = new WeakMap(); +export function verifyControlRecoveryAuthority( + authority: VerifiedExecutionAuthority, +): VerifiedRecoveryAuthority { + const s = executionAuthorityState(authority); + const evidence = childSettlementState( + verifyControlEffectSettlement(s.installation.installation, s.execution), + ).evidence; + const result = Object.freeze({}) as VerifiedRecoveryAuthority; + recovery.set(result, { authority: s, evidence }); + return result; +} +export function recoveryAuthorityState(value: VerifiedRecoveryAuthority) { + const s = recovery.get(value); + if (!s) throw Error("Unverified recovery authority"); + return s; +} + +/** Exact role parent identity is independent of the assertion that binds its child (no hash cycle). */ +export function retainedControlParent(execution: ExecutionRef): RetainedParentExecution { + const retained = executionRefSchema.parse(execution); + return { + issuer: retained.executor, + execution: retained, + operationKey: retained.operationKey, + fingerprint: retained.fingerprint, + deadlineAt: retained.deadlineAt, + }; +} + +export const CONTROL_RECOVERY_INSTRUCTIONS = + "Run openmapx compose render on the installation host to provision control trust. For missing or conflicting retained records: stop all relevant writers and helpers, inspect the original private receipts and publication state, and repair only the exact owned record; never delete locks by age."; +/** Availability is not authority. An unavailable signer leaves inspection/recovery HTTP routes intact. */ +export function loadControlStartup(options: Parameters[0]): { + installation?: ControlInstallation; + reason?: string; +} { + try { + return { installation: openControlInstallation(options) }; + } catch { + return { reason: CONTROL_RECOVERY_INSTRUCTIONS }; + } +} +export const controlAdoptionSchema = z.strictObject({ + leaseId: uuid, + delegationId: uuid, + phase: z.enum(["pending-adoption", "ready", "recovery_required"]), +}); +export type ControlAdoption = z.infer; + +const parentAssertionSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + issuer: z.enum(CONTROL_ROLES), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + attemptId: uuid, + grantDigest: digest, + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), +}); +const parentGrantSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + parent: executionRefSchema, + child: executionRefSchema, + kind: z.string().max(128), + phase: z.enum(["data-workflow", "osm", "transit", "fonts", "data-step"]), + recipient: z.enum(["fixed-cli", "data-manager"]), + childPublicKey: z.string().max(256), + attemptId: uuid, + parentAssertion: z.string().max(4096).optional(), + ordinal: z.number().int().min(0).max(63), +}); +function parseParentAssertion(assertion: string) { + if (assertion.length > 4096 || !/^[A-Za-z0-9_-]+$/.test(assertion)) + throw Error("Invalid parent assertion"); + const parsed = parentAssertionSchema.parse( + JSON.parse(Buffer.from(assertion, "base64url").toString("utf8")), + ); + if (Buffer.from(canonicalJson(parsed)).toString("base64url") !== assertion) + throw Error("Noncanonical parent assertion"); + return parsed; +} +function executionWitnessDirectory( + state: InstallationState, + issuer: ControlIssuer, + create = false, +) { + const data = new AnchoredDirectory(state.dataRoot), + control = data.directory(".admin-target-control", create, true), + executions = control.directory("executions", create, true), + role = executions.directory(issuer, create, true); + return { + role, + close() { + role.close(); + executions.close(); + control.close(); + data.close(); + }, + }; +} +export async function attestParentExecution( + authority: VerifiedExecutionAuthority, + child: { + execution: ExecutionRef; + phase: ControlChildPhase; + serviceIds: readonly string[]; + recipient: ControlRecipient; + childPublicKey: string; + }, +): Promise { + const s = executionAuthorityState(authority); + if (Date.now() >= s.execution.deadlineAt) throw Error("Expired parent execution"); + const plan = s.plan.find( + (p) => + controlDigest(p.execution) === controlDigest(child.execution) && + p.phase === child.phase && + p.recipient === child.recipient, + ); + if (!plan || !childPhaseKinds[child.phase].includes(plan.kind)) + throw Error("Child absent from retained plan"); + try { + readPrivateRecord(s.installation, "execution-leases", `${s.execution.operationKey}.json`); + throw Error("Deadline-only parent must not hold targets"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if ( + child.recipient === "fixed-cli" + ? child.phase !== "data-workflow" || s.installation.installation.role !== "ops-agent" + : child.recipient !== "data-manager" || + !["osm", "transit", "fonts", "data-step"].includes(child.phase) + ) + throw Error("Unauthorized deadline-only edge"); + if (["osm", "transit", "fonts"].includes(child.phase) && s.kind !== "data.update") + throw Error("Only Full Update may delegate data phases"); + if ( + child.recipient === "data-manager" && + child.childPublicKey !== s.installation.publicKeys["data-manager"] + ) + throw Error("Wrong child role key"); + verifyDeadlineEdge(s.kind, child.phase, child.recipient, plan.kind); + let attemptId: string = randomUUID(); + if (child.recipient === "fixed-cli") { + const d = s.installation.privateRoot.directory("children"); + try { + const matches = d + .list() + .filter( + (n) => + n.endsWith(".key") && + (JSON.parse(d.read(n, 2048).toString("utf8")) as { publicKey: string }).publicKey === + child.childPublicKey, + ); + if (matches.length !== 1) throw Error("Missing fixed child key"); + attemptId = uuid.parse(matches[0]?.slice(0, -4)); + } finally { + d.close(); + } + } + const grant = parentGrantSchema.parse({ + version: 1, + installationId: s.installation.installation.installationId, + parent: s.execution, + child: child.execution, + kind: plan.kind, + phase: child.phase, + recipient: child.recipient, + childPublicKey: child.childPublicKey, + attemptId, + ordinal: plan.ordinal, + ...(s.installation.fixedParentAssertion + ? { parentAssertion: s.installation.fixedParentAssertion } + : {}), + }); + const signed = signControl(s.installation.privateKey, "openmapx-parent-grant-v1", grant); + const fields = { + version: 1 as const, + installationId: grant.installationId, + issuer: s.execution.executor, + operationKey: s.execution.operationKey, + attemptId, + grantDigest: controlDigest(signed), + }; + const assertion = Buffer.from( + canonicalJson({ + ...fields, + signature: signControl(s.installation.privateKey, "openmapx-parent-assertion-v1", fields) + .signature, + }), + ).toString("base64url"); + parseParentAssertion(assertion); + const dir = executionWitnessDirectory(s.installation, s.execution.executor, true); + try { + const name = `${s.execution.operationKey}.children.json`; + const guardName = `${s.execution.operationKey}.writer`; + mkdirSync(dir.role.name(guardName), { mode: 0o700 }); + const guard = dir.role.directory(guardName); + const guardIdentity = identity(guard.stat); + try { + let witness: Record = {}; + try { + witness = z + .record(uuid, z.unknown()) + .parse(JSON.parse(dir.role.read(name, 65536).toString("utf8"))); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + for (const value of Object.values(witness)) { + const previous = verifyControl( + s.installation.installation.publicKey, + "openmapx-parent-grant-v1", + value, + parentGrantSchema, + ); + if (previous.child.operationKey === child.execution.operationKey) + throw Error("Attempted parent child cannot be replayed"); + } + if (Object.keys(witness).length >= 64) throw Error("Parent attempt capacity"); + witness[attemptId] = signed; + const bytes = canonicalJson(witness); + if (Buffer.byteLength(bytes) > 65536) throw Error("Parent witness capacity"); + writePrivateRecord( + s.installation, + "parent-attempts", + `${attemptId}.json`, + { grant: signed, assertion }, + true, + ); + dir.role.write(name, bytes); + return assertion; + } finally { + guard.close(); + dir.role.removeDirectory(guardName, guardIdentity); + } + } finally { + dir.close(); + } +} +function readParentGrant( + state: InstallationState, + assertion: string, + depth = 0, +): z.infer { + if (depth > 1) throw Error("Parent assertion chain limit"); + const parsed = parseParentAssertion(assertion); + if (parsed.installationId !== state.installation.installationId) + throw Error("Foreign parent installation"); + const d = executionWitnessDirectory(state, parsed.issuer); + let signed: unknown; + try { + const attempts = z + .record(uuid, z.unknown()) + .parse( + JSON.parse(d.role.read(`${parsed.operationKey}.children.json`, 65536).toString("utf8")), + ); + signed = attempts[parsed.attemptId]; + } finally { + d.close(); + } + if (!signed || controlDigest(signed) !== parsed.grantDigest) + throw Error("Missing exact retained parent attempt"); + const candidate = z.object({ payload: parentGrantSchema }).parse(signed).payload; + let verifier = state.publicKeys[parsed.issuer]; + if (candidate.parentAssertion) { + const ancestor = readParentGrant(state, candidate.parentAssertion, depth + 1); + if ( + ancestor.recipient !== "fixed-cli" || + controlDigest(ancestor.child) !== controlDigest(candidate.parent) || + candidate.recipient !== "data-manager" + ) + throw Error("Invalid fixed parent certification"); + verifier = ancestor.childPublicKey; + } + const { signature, ...fields } = parsed; + verifyControl( + verifier, + "openmapx-parent-assertion-v1", + { payload: fields, signature }, + parentAssertionSchema.omit({ signature: true }), + ); + const grant = verifyControl(verifier, "openmapx-parent-grant-v1", signed, parentGrantSchema); + const parent = readControlExecutionWitness(state, grant.parent, verifier); + if ( + grant.parent.executor !== parsed.issuer || + grant.parent.operationKey !== parsed.operationKey || + grant.attemptId !== parsed.attemptId || + grant.child.operationKey !== + controlChildOperationKey(grant.parent.operationKey, grant.phase, grant.ordinal) || + !childPhaseKinds[grant.phase].includes(grant.kind) + ) + throw Error("Parent immutable phase binding mismatch"); + verifyDeadlineEdge(parent.kind, grant.phase, grant.recipient, grant.kind); + return grant; +} +function verifyDeadlineEdge( + parentKind: string, + phase: ControlChildPhase, + recipient: ControlRecipient, + childKind: string, +) { + const dataKinds: Record = { + "data.downloadOsm": "download-osm", + "data.downloadFonts": "download-fonts", + "data.convertOverpass": "convert-overpass", + "data.overtureSync": "overture-sync", + "data.overtureConflate": "overture-conflate", + "data.searchIndexBuild": "search-index-build", + "data.notablePlacesBuild": "notable-places-build", + }; + if ( + recipient === "fixed-cli" && + phase === "data-workflow" && + childKind === parentKind && + childPhaseKinds[phase].includes(parentKind) + ) + return; + if ( + recipient === "data-manager" && + ((["osm", "transit", "fonts"].includes(phase) && parentKind === "data.update") || + (phase === "data-step" && dataKinds[parentKind] === childKind)) + ) + return; + throw Error("Unauthorized deadline-only child kind"); +} +export function verifyParentExecution( + assertion: string, + peer: VerifiedControlPeer, + retainedAttempt: { execution: ExecutionRef; kind: string; requestedDeadlineAt: number }, +): import("../ops/execution-identity").VerifiedParentExecution { + const p = controlPeerState(peer); + const parsed = parseParentAssertion(assertion); + if ( + assertion !== p.assertion || + parsed.installationId !== p.installation.installation.installationId + ) + throw Error("Foreign parent assertion"); + const grant = readParentGrant(p.installation, assertion); + if ( + grant.parent.operationKey !== parsed.operationKey || + grant.attemptId !== parsed.attemptId || + grant.childPublicKey !== p.childPublicKey || + grant.recipient !== p.recipient || + grant.kind !== retainedAttempt.kind || + controlDigest(grant.child) !== controlDigest(retainedAttempt.execution) || + grant.child.deadlineAt !== retainedAttempt.requestedDeadlineAt || + grant.child.deadlineAt > grant.parent.deadlineAt + ) + throw Error("Parent child identity mismatch"); + const name = `${parsed.attemptId}.json`; + let retained = false; + try { + if ( + controlDigest(readPrivateRecord(p.installation, "parent-adoptions", name)) !== + controlDigest(grant) + ) + throw Error("Parent attempt replay conflict"); + retained = true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (!retained) { + if (Date.now() >= grant.child.deadlineAt) throw Error("Expired parent grant"); + writePrivateRecord(p.installation, "parent-adoptions", name, grant, true); + } + const parent = Object.freeze( + retainedControlParent(grant.parent), + ) as import("../ops/execution-identity").VerifiedParentExecution; + verifiedParents.add(parent); + return parent; +} + +/** The child opens its ephemeral key, never the installation role private key. */ +export function openFixedChildInstallation(options: { + infraDir: string; + assertion: string; + container?: boolean; +}): ControlInstallation { + const encoded = JSON.parse(Buffer.from(options.assertion, "base64url").toString("utf8")) as { + leaseId?: unknown; + }; + const envelope = encoded.leaseId + ? parseControlAssertion(options.assertion) + : parseParentAssertion(options.assertion); + const id = "delegationId" in envelope ? envelope.delegationId : envelope.attemptId; + const infra = options.container ? undefined : new AnchoredDirectory(options.infraDir); + const secrets = options.container + ? new AnchoredDirectory("/run/secrets", undefined, false, true) + : (() => { + if (!infra) throw Error("Missing fixed installation root"); + return infra.directory("secrets"); + })(); + let recovery: AnchoredDirectory | undefined; + try { + const bundle = bundleSchema.parse( + JSON.parse( + secrets + .read(options.container ? "openmapx-control-verifiers" : "control-verifiers.json", 4096) + .toString("utf8"), + ), + ); + if (bundle.installationId !== envelope.installationId) + throw Error("Foreign fixed child installation"); + recovery = new AnchoredDirectory( + options.container + ? "/var/lib/openmapx/control-private" + : join(options.infraDir, "control-private/ops-agent"), + undefined, + true, + ); + const privateRelative = relative( + realpathSync(join(options.infraDir, "data")), + realpathSync(recovery.path), + ); + if (!privateRelative || (!privateRelative.startsWith("../") && privateRelative !== "..")) + throw Error("Fixed child recovery material is inside shared data"); + const children = recovery.directory("children"); + let saved: { publicKey: string; privateKey: string }; + try { + saved = z + .strictObject({ publicKey, privateKey: z.string().max(512) }) + .parse(JSON.parse(children.read(`${id}.key`, 2048).toString("utf8"))); + } finally { + children.close(); + } + const key = parsePrivate(saved.privateKey); + if ( + createPublicKey(key).export({ type: "spki", format: "der" }).toString("base64url") !== + saved.publicKey + ) + throw Error("Fixed child key mismatch"); + const installation = Object.freeze({ + installationId: bundle.installationId, + role: "ops-agent", + publicKey: saved.publicKey, + }) as ControlInstallation; + installations.set(installation, { + installation, + privateKey: key, + publicKeys: bundle.publicKeys, + privateRoot: recovery, + dataRoot: join(options.infraDir, "data"), + registeredTargets: new Set(), + fixedChildId: id, + ...("attemptId" in envelope ? { fixedParentAssertion: options.assertion } : {}), + }); + recovery = undefined; + return installation; + } finally { + recovery?.close(); + secrets.close(); + infra?.close(); + } +} +export function retainFixedCliSpawnIntent( + authority: VerifiedExecutionAuthority, + input: { + delegationId: string; + execution: ExecutionRef; + operation: unknown; + args: readonly string[]; + plan?: readonly ControlPlannedChild[]; + }, +): void { + const s = executionAuthorityState(authority); + const id = uuid.parse(input.delegationId); + const operation = opsOperationSchema.parse(input.operation); + const plan = s.plan.find( + (p) => + controlDigest(p.execution) === controlDigest(input.execution) && p.recipient === "fixed-cli", + ); + if ( + !plan || + opsOperationFingerprint(operation) !== input.execution.operationFingerprint || + operation.kind !== plan.kind || + canonicalJson(fixedCliArguments(operation)) !== canonicalJson(input.args) + ) + throw Error("Fixed command does not match retained child plan"); + writePrivateRecord( + s.installation, + "spawns", + `${id}.json`, + { + execution: input.execution, + operation, + args: input.args, + serviceIds: plan.serviceIds, + admittedAt: Date.now(), + plan: z + .array(plannedChildSchema) + .max(64) + .parse(input.plan ?? []), + }, + true, + ); +} +export function fixedCliArguments(operation: import("../ops/contract").OpsOperation): string[] { + switch (operation.kind) { + case "service.build": + return [ + "services", + "build", + operation.serviceId, + ...(operation.regionId ? ["--region", operation.regionId] : []), + ]; + case "services.buildAll": + return [ + "services", + "build-all", + ...(operation.regionId ? ["--region", operation.regionId] : []), + ]; + case "service.start": + return ["services", "start", operation.serviceId]; + case "data.link": + return ["data", "link"]; + case "data.clean": + return ["data", "clean", operation.dataTypeId]; + case "system.update": + return ["services", "update"]; + case "data.update": + return [ + "data", + "update", + ...(operation.regionId ? [operation.regionId] : []), + ...(operation.countryCodes?.length + ? ["--countries", operation.countryCodes.join(",")] + : []), + ...(operation.failFast ? ["--fail-fast"] : []), + ]; + case "data.overtureConflate": + return [ + "data", + "overture-conflate", + operation.regionId, + ...(operation.restart ? ["--restart"] : []), + ]; + case "data.downloadOsm": + return ["data", "download", "osm", ...(operation.regionId ? [operation.regionId] : [])]; + case "data.downloadFonts": + return ["data", "download", "fonts"]; + case "data.convertOverpass": + return ["data", "convert", "overpass", ...(operation.regionId ? [operation.regionId] : [])]; + case "data.overtureSync": + return ["data", "overture-sync", operation.regionId]; + case "data.searchIndexBuild": + return ["data", "search-index", "build", operation.regionId]; + case "data.notablePlacesBuild": + return ["data", "notable-places", "build"]; + default: + throw Error("This fixed CLI control consumer is unavailable"); + } +} +export function readFixedCliSpawnIntent( + installation: ControlInstallation, + execution: ExecutionRef, + args: readonly string[], +) { + const s = installationState(installation); + if (!s.fixedChildId) throw Error("Fixed child authority required"); + const saved = z + .strictObject({ + plan: z.array(plannedChildSchema).max(64), + execution: executionRefSchema, + operation: opsOperationSchema, + args: z.array(z.string().max(4096)).max(64), + serviceIds: targetSchema, + admittedAt: z.number().int().nonnegative().safe(), + }) + .parse(readPrivateRecord(s, "spawns", `${s.fixedChildId}.json`, 65536)); + if ( + controlDigest(execution) !== controlDigest(saved.execution) || + canonicalJson(args) !== canonicalJson(saved.args) || + canonicalJson(fixedCliArguments(saved.operation)) !== canonicalJson(saved.args) || + opsOperationFingerprint(saved.operation) !== execution.operationFingerprint + ) + throw Error("Fixed CLI spawn identity/command mismatch"); + s.registeredTargets = new Set(saved.serviceIds); + return saved; +} +export function beginFixedCliExecution( + installation: ControlInstallation, + execution: ExecutionRef, +): void { + const s = installationState(installation); + if (!s.fixedChildId) throw Error("Fixed CLI authority required"); + writePrivateRecord(s, "spawn-started", `${s.fixedChildId}.json`, { execution }, true); +} + +const executionWitnessSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + execution: executionRefSchema, + kind: z.string().max(128), + serviceIds: targetSchema, + authorityDigest: digest, +}); +export function readControlExecutionWitness( + state: InstallationState, + execution: ExecutionRef, + publicVerifier: string, +) { + const d = executionWitnessDirectory(state, execution.executor); + try { + const witness = verifyControl( + publicVerifier, + "openmapx-control-execution-witness-v1", + JSON.parse(d.role.read(`${execution.operationKey}.json`, 65536).toString("utf8")), + executionWitnessSchema, + ); + if ( + witness.installationId !== state.installation.installationId || + controlDigest(witness.execution) !== controlDigest(execution) + ) + throw Error("Durable parent execution witness mismatch"); + return witness; + } finally { + d.close(); + } +} +function publishControlExecutionWitness( + state: InstallationState, + input: { execution: ExecutionRef; kind: string; serviceIds: readonly string[] }, + authorityDigest: string, +) { + const d = executionWitnessDirectory(state, input.execution.executor, true); + try { + const witness = signControl(state.privateKey, "openmapx-control-execution-witness-v1", { + version: 1 as const, + installationId: state.installation.installationId, + execution: input.execution, + kind: input.kind, + serviceIds: [...input.serviceIds], + authorityDigest, + }); + const name = `${input.execution.operationKey}.json`; + try { + const prior = JSON.parse(d.role.read(name, 65536).toString("utf8")); + if (controlDigest(prior) !== controlDigest(witness)) + throw Error("Replaced execution witness"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + d.role.write(name, canonicalJson(witness), true); + } + } finally { + d.close(); + } +} + +/** Restore only the original signed durable admission; this grants no permission to redispatch. */ +export function restoreAdmittedControlExecution( + installation: ControlInstallation, + execution: ExecutionRef, +): VerifiedExecutionAuthority { + const state = installationState(installation); + const payload = verifyControl( + installation.publicKey, + "openmapx-control-execution-v1", + readPrivateRecord(state, "executions", `${execution.operationKey}.json`, 65536), + z + .object({ version: z.literal(1), installationId: uuid, execution: executionRefSchema }) + .passthrough(), + ); + if ( + payload.installationId !== installation.installationId || + controlDigest(payload.execution) !== controlDigest(execution) + ) + throw Error("Recovery execution ownership mismatch"); + const { version: _version, installationId: _installationId, ...input } = payload; + return retainAdmittedControlExecution( + installation, + input as Parameters[1], + ); +} +export function retainedControlAdoptionAssertion( + installation: ControlInstallation, + adoption: ControlAdoption, + execution: ExecutionRef, +): string { + const state = installationState(installation); + const saved = z + .object({ assertion: z.string().max(4096), execution: executionRefSchema }) + .parse(readPrivateRecord(state, "adoptions", `${uuid.parse(adoption.delegationId)}.json`)); + const assertion = parseControlAssertion(saved.assertion); + if ( + assertion.leaseId !== adoption.leaseId || + assertion.delegationId !== adoption.delegationId || + controlDigest(saved.execution) !== controlDigest(execution) + ) + throw Error("Recovery adoption ownership mismatch"); + return saved.assertion; +} + +const preparedTerminalSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + execution: executionRefSchema, + kind: z.enum(["configurationMaintenance.prepare", "configurationHandoff.prepare"]), + receiptDigest: digest, +}); +/** Fixed preparing adapter calls only after publishing its terminal journal. This performs the read itself. */ +export function retainControlPreparedTerminal(authority: VerifiedExecutionAuthority): void { + const s = executionAuthorityState(authority); + if ( + s.installation.fixedChildId || + s.execution.executor !== "ops-agent" || + !["configurationMaintenance.prepare", "configurationHandoff.prepare"].includes(s.kind) + ) + throw Error("Unsupported terminal preparation"); + const data = new AnchoredDirectory(s.installation.dataRoot); + let jobs: AnchoredDirectory | undefined; + try { + jobs = data.directory("ops-agent"); + const journal = z + .object({ version: z.literal(2), jobs: z.array(z.unknown()).max(256) }) + .parse(JSON.parse(jobs.read("jobs-v1.json", 24 * 1024 * 1024).toString("utf8"))); + const matches = journal.jobs.filter( + (j) => + j && + typeof j === "object" && + (j as { operationId?: string }).operationId === s.execution.executionId, + ); + if (matches.length !== 1) throw Error("Missing exact terminal preparation journal"); + const job = z + .object({ + operationId: z.string(), + operationKey: z.string(), + operationFingerprint: digest, + fingerprint: digest, + deadlineAt: z.number().int().safe(), + state: z.literal("succeeded"), + operation: z.object({ kind: z.string() }).passthrough(), + result: z + .object({ + phase: z.literal("prepared").optional(), + state: z.literal("prepared").optional(), + }) + .passthrough(), + terminalAt: z.iso.datetime(), + }) + .parse(matches[0]); + if ( + job.operationKey !== s.execution.operationKey || + job.operationFingerprint !== s.execution.operationFingerprint || + job.fingerprint !== s.execution.fingerprint || + job.deadlineAt !== s.execution.deadlineAt || + job.operation.kind !== s.kind || + controlDigest(job.operation) !== s.execution.operationFingerprint || + !(job.result.phase === "prepared" || job.result.state === "prepared") + ) + throw Error("Terminal preparation identity mismatch"); + const signed = signControl( + s.installation.privateKey, + "openmapx-control-prepared-v1", + preparedTerminalSchema.parse({ + version: 1, + installationId: s.installation.installation.installationId, + execution: s.execution, + kind: s.kind, + receiptDigest: controlDigest(job), + }), + ); + writePrivateRecord( + s.installation, + "prepared", + `${s.execution.operationKey}.json`, + signed, + true, + ); + const d = executionWitnessDirectory(s.installation, "ops-agent", true); + try { + d.role.write(`${s.execution.operationKey}.prepared.json`, canonicalJson(signed), true); + } finally { + d.close(); + } + } finally { + jobs?.close(); + data.close(); + } +} +export function verifyControlPreparedTerminal( + state: InstallationState, + execution: ExecutionRef, +): void { + const d = executionWitnessDirectory(state, "ops-agent"); + try { + const receipt = verifyControl( + state.publicKeys["ops-agent"], + "openmapx-control-prepared-v1", + JSON.parse(d.role.read(`${execution.operationKey}.prepared.json`, 8192).toString("utf8")), + preparedTerminalSchema, + ); + if ( + receipt.installationId !== state.installation.installationId || + controlDigest(receipt.execution) !== controlDigest(execution) + ) + throw Error("Foreign terminal preparation"); + } finally { + d.close(); + } +} diff --git a/packages/core/src/build-artifacts/filesystem.ts b/packages/core/src/build-artifacts/filesystem.ts new file mode 100644 index 000000000..b47ca3f6d --- /dev/null +++ b/packages/core/src/build-artifacts/filesystem.ts @@ -0,0 +1,200 @@ +/** Server-local descriptor anchoring. Never accepts archive paths without component validation. */ +import { randomUUID } from "node:crypto"; +import { + type BigIntStats, + constants as C, + closeSync, + fstatSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readdirSync, + readSync, + renameSync, + rmdirSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { isAbsolute, join } from "node:path"; +export type FileIdentity = { dev: string; ino: string }; +export const identity = (s: BigIntStats): FileIdentity => ({ + dev: String(s.dev), + ino: String(s.ino), +}); +export const sameInode = (a: FileIdentity, b: FileIdentity) => a.dev === b.dev && a.ino === b.ino; +function component(name: string) { + if ( + !name || + name === "." || + name === ".." || + name.includes("/") || + name.includes("\\") || + [...name].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127) || + Buffer.byteLength(name) > 255 + ) + throw Error("Unsafe filesystem component"); +} +export class AnchoredDirectory { + readonly fd: number; + readonly stat: BigIntStats; + readonly anchor: string; + private closed = false; + constructor( + readonly path: string, + readonly parent?: AnchoredDirectory, + private strict = false, + private allowSystemOwner = false, + ) { + if (!isAbsolute(path)) throw Error("Absolute trusted root required"); + this.fd = openSync(path, C.O_RDONLY | C.O_DIRECTORY | C.O_NOFOLLOW); + try { + this.stat = fstatSync(this.fd, { bigint: true }); + this.anchor = + process.platform === "linux" + ? `/proc/self/fd/${this.fd}` + : process.platform === "darwin" + ? `/.vol/${this.stat.dev}/${this.stat.ino}` + : ""; + if ( + !this.anchor || + !sameInode(identity(this.stat), identity(statSync(this.anchor, { bigint: true }))) + ) + throw Error("Descriptor filesystem anchoring unavailable"); + this.check(); + } catch (e) { + closeSync(this.fd); + throw e; + } + } + check() { + if (this.closed) throw Error("Closed directory authority"); + this.parent?.check(); + const current = fstatSync(this.fd, { bigint: true }); + const path = lstatSync(this.path, { bigint: true }); + if ( + !current.isDirectory() || + !path.isDirectory() || + (current.uid !== BigInt(process.getuid?.() ?? -1) && + !(this.allowSystemOwner && current.uid === 0n)) || + (current.mode & BigInt(this.strict ? 0o077 : 0o022)) !== 0n || + !sameInode(identity(this.stat), identity(current)) || + !sameInode(identity(current), identity(path)) + ) + throw Error("Directory authority changed or unsafe"); + } + name(name: string) { + component(name); + this.check(); + return join(this.anchor, name); + } + directory(name: string, create = false, strict = this.strict): AnchoredDirectory { + const path = this.name(name); + if (create) { + try { + mkdirSync(path, { mode: 0o700 }); + fsyncSync(this.fd); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + } + } + return new AnchoredDirectory(path, this, strict); + } + list() { + this.check(); + return readdirSync(this.anchor); + } + read(name: string, maximum: number, privateMode = true): Buffer { + const path = this.name(name); + const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + try { + const before = fstatSync(fd, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== BigInt(process.getuid?.() ?? -1) || + (before.mode & BigInt(privateMode ? 0o077 : 0o022)) !== 0n || + before.size > BigInt(maximum) + ) + throw Error("Unsafe or oversized regular file"); + const buffer = Buffer.alloc(Number(before.size) + 1); + let count = 0; + while (count < buffer.length) { + const n = readSync(fd, buffer, count, buffer.length - count, null); + if (!n) break; + count += n; + } + const bytes = buffer.subarray(0, count); + const after = fstatSync(fd, { bigint: true }); + if ( + BigInt(bytes.length) !== before.size || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + !sameInode(identity(before), identity(lstatSync(path, { bigint: true }))) + ) + throw Error("Regular file changed during read"); + this.check(); + return bytes; + } finally { + closeSync(fd); + } + } + write(name: string, bytes: string | Buffer, exclusive = false) { + const target = this.name(name); + const temporary = exclusive ? name : `.write-${randomUUID()}`; + const path = this.name(temporary); + const fd = openSync(path, C.O_WRONLY | C.O_CREAT | C.O_EXCL | C.O_NOFOLLOW, 0o600); + try { + writeFileSync(fd, bytes); + fsyncSync(fd); + this.check(); + } finally { + closeSync(fd); + } + if (!exclusive) { + renameSync(path, target); + } + fsyncSync(this.fd); + } + removeFile(name: string) { + unlinkSync(this.name(name)); + fsyncSync(this.fd); + } + removeDirectory(name: string, expected: FileIdentity) { + const child = this.directory(name); + try { + if (!sameInode(identity(child.stat), expected) || child.list().length) + throw Error("Refusing replaced or nonempty lock"); + rmdirSync(this.name(name)); + fsyncSync(this.fd); + } finally { + child.close(); + } + } + close() { + if (!this.closed) { + this.closed = true; + closeSync(this.fd); + } + } +} +export function withDirectories( + root: AnchoredDirectory, + components: readonly string[], + create: boolean, + run: (parent: AnchoredDirectory) => T, +): T { + const dirs: AnchoredDirectory[] = []; + let current = root; + try { + for (const name of components) { + current = current.directory(name, create); + dirs.push(current); + } + return run(current); + } finally { + for (const dir of dirs.reverse()) dir.close(); + } +} diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts new file mode 100644 index 000000000..798c51abd --- /dev/null +++ b/packages/core/src/build-artifacts/index.ts @@ -0,0 +1,46 @@ +// Server-only surface: intentionally absent from core's browser barrels. + +export * from "./archive"; +export { + type ArtifactDelegationAssertion, + attestParentExecution, + beginControlEffect, + beginFixedCliExecution, + CONTROL_RECOVERY_INSTRUCTIONS, + type ControlAdoption, + type ControlChildPhase, + type ControlInstallation, + type ControlPlannedChild, + type ControlRecipient, + closeControlInstallation, + controlAdoptionSchema, + controlChildOperationKey, + controlHeaders, + fixedCliArguments, + loadControlStartup, + openControlInstallation, + openFixedChildInstallation, + prepareFixedControlChild, + provisionControlInstallation, + readFixedCliSpawnIntent, + restoreAdmittedControlExecution, + retainAdmittedControlExecution, + retainControlEffectReceipt, + retainControlPreparedTerminal, + retainedControlAdoptionAssertion, + retainedControlParent, + retainFixedCliSpawnIntent, + type VerifiedChildSettlement, + type VerifiedControlPeer, + type VerifiedExecutionAuthority, + type VerifiedRecoveryAuthority, + verifyControlEffectSettlement, + verifyControlPeer, + verifyControlRecoveryAuthority, + verifyFixedCliControlPeer, + verifyParentExecution, + verifyRetainedLocalControlPeer, +} from "./control-authority"; +export * from "./manifest"; +export { inspectControlAdmission, verifiedParentForArtifactDelegation } from "./peer-admission"; +export * from "./target-lease"; diff --git a/packages/core/src/build-artifacts/manifest.test.ts b/packages/core/src/build-artifacts/manifest.test.ts new file mode 100644 index 000000000..866508c2c --- /dev/null +++ b/packages/core/src/build-artifacts/manifest.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { artifactFingerprint, parseBundleManifest } from "./manifest"; + +const manifest = { + schemaVersion: 1, + service: "osrm", + builtAt: "2026-10-23T00:00:00.000Z", + runtimeImages: { osrm: "image:1" }, + toolImages: {}, + dirs: ["osrm-graph"], + directories: ["osrm-graph"], + files: [], +}; +describe("manifest shape and immutable content identity", () => { + it("preserves v1 provenance without inventing source hashes", () => { + const m = parseBundleManifest( + JSON.stringify({ ...manifest, sourcePbf: { name: "source.pbf", sizeBytes: 1 } }), + ); + expect(m).not.toHaveProperty("sourceFingerprints"); + }); + it.each([ + { extra: true }, + { files: [{ path: "osrm-graph/x", sizeBytes: -1, sha256: "a".repeat(64) }] }, + { directories: ["osrm-graph", "osrm-graph"] }, + { runtimeImages: {} }, + { toolImages: [] }, + ])("rejects invalid manifest %j", (patch) => + expect(() => parseBundleManifest(JSON.stringify({ ...manifest, ...patch }))).toThrow(), + ); + it("v2 hashes canonical tuples, independent of metadata and file order", () => { + expect(artifactFingerprint([])).toBe( + "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + ); + expect( + parseBundleManifest( + JSON.stringify({ + ...manifest, + schemaVersion: 2, + sourceFingerprints: [], + artifactFingerprint: artifactFingerprint([]), + }), + ).schemaVersion, + ).toBe(2); + }); +}); diff --git a/packages/core/src/build-artifacts/manifest.ts b/packages/core/src/build-artifacts/manifest.ts new file mode 100644 index 000000000..24d749d3b --- /dev/null +++ b/packages/core/src/build-artifacts/manifest.ts @@ -0,0 +1,191 @@ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { ServiceRegistry } from "../services/registry"; +import { serviceContainerImageReference } from "../services/types"; +export interface BuildArtifactSpec { + dirs: string[]; + exclude: string[]; + consumers: string[]; +} +export const BUILD_ARTIFACTS: Readonly> = { + tileserver: { dirs: ["tile-mbtiles"], exclude: [], consumers: ["tileserver"] }, + osrm: { dirs: ["osrm-graph"], exclude: ["osrm-graph/region.osm.pbf"], consumers: ["osrm"] }, + otp: { dirs: ["otp-graph"], exclude: ["*.pbf", "*.zip"], consumers: ["otp"] }, + pelias: { + dirs: ["pelias"], + exclude: ["pelias/openstreetmap/"], + consumers: ["pelias", "pelias-pip", "pelias-placeholder", "elasticsearch"], + }, + motis: { + dirs: ["motis/live", "motis-feed-proxy"], + exclude: [], + consumers: ["motis", "motis-staging"], + }, +}; +export function buildArtifactSpec(serviceId: string): BuildArtifactSpec { + const spec = Object.hasOwn(BUILD_ARTIFACTS, serviceId) ? BUILD_ARTIFACTS[serviceId] : undefined; + if (!spec) + throw Error( + `Service "${serviceId}" has no exportable build artifact; database services require their supported backup workflow`, + ); + return structuredClone(spec); +} +export const BUILD_RECORD_DIR = ".openmapx-builds"; +export const BUNDLE_MANIFEST_PATH = ".openmapx-bundle/manifest.json"; +export interface BuildRecord { + schemaVersion: 1; + service: string; + builtAt: string; + region?: string; + revision?: string; + sourcePbf?: { name: string; sizeBytes: number }; + runtimeImages: Record; + toolImages: Record; +} +export interface BuildRecordV2 extends Omit { + schemaVersion: 2; + sourceFingerprints: Array<{ id: string; sha256: string; sizeBytes: number }>; + artifactFingerprint: string; + credentialRequirements?: string[]; + runtimeMaterialization?: "motis-target-credentials"; +} +export interface BundleFile { + path: string; + sizeBytes: number; + sha256: string; +} +export type BundleManifest = (BuildRecord | BuildRecordV2) & { + dirs: string[]; + directories: string[]; + files: BundleFile[]; +}; +export type ImageMismatch = { service: string; bundle: string; local: string }; +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const string = z.string().min(1).max(4096); +const size = z.number().int().nonnegative().safe(); +const images = z.record(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), string); +const common = { + service: z.string().max(64), + builtAt: z.iso.datetime(), + region: string.optional(), + revision: string.optional(), + sourcePbf: z.strictObject({ name: string, sizeBytes: size }).optional(), + runtimeImages: images, + toolImages: images, +}; +const v2 = { + sourceFingerprints: z.array(z.strictObject({ id: string, sha256: digest, sizeBytes: size })), + artifactFingerprint: digest, + credentialRequirements: z.array(string).optional(), + runtimeMaterialization: z.literal("motis-target-credentials").optional(), +}; +const layout = { + dirs: z.array(string), + directories: z.array(string), + files: z.array(z.strictObject({ path: string, sizeBytes: size, sha256: digest })), +}; +const schema = z.discriminatedUnion("schemaVersion", [ + z.strictObject({ schemaVersion: z.literal(1), ...common, ...layout }), + z.strictObject({ schemaVersion: z.literal(2), ...common, ...v2, ...layout }), +]); +export function safeArtifactPath(path: string, maxBytes = 4096, maxDepth = 64): boolean { + return ( + !!path && + Buffer.byteLength(path) <= maxBytes && + !/^[A-Za-z]:/.test(path) && + !path.includes("\\") && + ![...path].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127) && + path.split("/").length <= maxDepth && + path.split("/").every((p) => !!p && p !== "." && p !== ".." && Buffer.byteLength(p) <= 255) + ); +} +export function artifactFingerprint(files: readonly BundleFile[]): string { + return createHash("sha256") + .update( + JSON.stringify( + [...files] + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + .map((f) => [f.path, f.sizeBytes, f.sha256]), + ), + ) + .digest("hex"); +} +export function parseBundleManifest(text: string): BundleManifest { + const result = schema.safeParse(JSON.parse(text)); + if (!result.success) throw Error("Bundle manifest is malformed"); + const m = result.data; + const spec = buildArtifactSpec(m.service); + if (JSON.stringify([...m.dirs].sort()) !== JSON.stringify([...spec.dirs].sort())) + throw Error("Bundle artifact dirs do not match service"); + const names = new Set(); + for (const name of [...m.directories, ...m.files.map((f) => f.path)]) { + if (!safeArtifactPath(name) || !spec.dirs.some((d) => name === d || name.startsWith(`${d}/`))) + throw Error("Bundle lists a path outside its artifact"); + if (names.has(name)) throw Error("Duplicate bundle manifest path"); + names.add(name); + } + const dirs = new Set(m.directories); + for (const d of spec.dirs) if (!dirs.has(d)) throw Error("Bundle is missing artifact directory"); + for (const name of names) { + const parent = name.split("/").slice(0, -1).join("/"); + if ( + parent && + spec.dirs.some((d) => parent === d || parent.startsWith(`${d}/`)) && + !dirs.has(parent) + ) + throw Error("Bundle missing parent directory"); + } + if (!m.runtimeImages[m.service]) + throw Error("Bundle has no runtime image binding; rebuild with the runtime image"); + if (m.schemaVersion === 2) { + if ( + new Set(m.sourceFingerprints.map((f) => f.id)).size !== m.sourceFingerprints.length || + new Set(m.credentialRequirements).size !== (m.credentialRequirements?.length ?? 0) + ) + throw Error("Duplicate source or credential ID"); + if (m.artifactFingerprint !== artifactFingerprint(m.files)) + throw Error("Bundle artifact fingerprint failed verification"); + if ((m.runtimeMaterialization || m.credentialRequirements) && m.service !== "motis") + throw Error("Invalid credential materialization service"); + } + if ( + m.service === "motis" && + m.files.some( + (f) => + f.path.startsWith("motis-feed-proxy/") || + /(?:^|\/)(?:feed-proxy-vars\.json|api-keys\.json|config\.yml)$/.test(f.path), + ) + ) + throw Error( + "Legacy credential-bearing MOTIS bundle: rebuild a portable candidate from retained source metadata and materialize target credentials before readiness", + ); + return m; +} +export async function runtimeImageMismatches( + runtimeImages: Record, + rootDir: string, +): Promise { + const registry = new ServiceRegistry({ rootDir }); + await registry.load(); + return Object.entries(runtimeImages).flatMap(([service, bundle]) => { + const loaded = registry.get(service); + const local = loaded + ? serviceContainerImageReference(loaded.manifest.container) + : "(not installed)"; + return local === bundle ? [] : [{ service, bundle, local }]; + }); +} + +export const portableMotisSourcesSchema = z.strictObject({ + version: z.literal(1), + sourceFingerprints: z + .array( + z.strictObject({ + id: z.string().min(1).max(512), + sha256: digest, + sizeBytes: size, + }), + ) + .max(10000), + credentialRequirements: z.array(z.string().min(1).max(512)).max(1000), +}); diff --git a/packages/core/src/build-artifacts/peer-admission.ts b/packages/core/src/build-artifacts/peer-admission.ts new file mode 100644 index 000000000..f759bd510 --- /dev/null +++ b/packages/core/src/build-artifacts/peer-admission.ts @@ -0,0 +1,80 @@ +import type { ExecutionRef, VerifiedParentExecution } from "../ops/execution-identity"; +import { verifiedParents } from "../ops/verified-parent-internal"; +import { + type ControlInstallation, + controlDigest, + controlHeaders, + installationState, + retainedControlParent, + verifyControlPeer, +} from "./control-authority"; +import { adoptArtifactTargetLease, inspectArtifactDelegation } from "./target-lease"; +/** Both receivers keep transport auth and resource policy. This verifies only the exact signed internal attempt. */ +export function inspectControlAdmission( + installation: ControlInstallation, + input: { + rawHeaders: readonly string[]; + transportPrincipal: "data-manager" | "shared-data-manager-bearer" | "host-cli"; + kind: string; + operationKey: string; + operationFingerprint: string; + requestedDeadlineAt?: number; + }, +) { + const state = installationState(installation); + const { delegation } = controlHeaders(input.rawHeaders); + if (!delegation) throw Error("Missing control assertion"); + const inspected = inspectArtifactDelegation({ + dataRoot: state.dataRoot, + assertion: delegation, + installation, + }); + if ( + inspected.kind !== input.kind || + inspected.execution.operationKey !== input.operationKey || + inspected.execution.operationFingerprint !== input.operationFingerprint || + inspected.execution.deadlineAt !== input.requestedDeadlineAt || + inspected.execution.executor !== installation.role + ) + throw Error("Signed immutable admission mismatch"); + const peer = verifyControlPeer(installation, { + rawHeaders: input.rawHeaders, + transportPrincipal: input.transportPrincipal, + execution: inspected.execution, + kind: input.kind, + }); + const parent = Object.freeze(retainedControlParent(inspected.parent)) as VerifiedParentExecution; + verifiedParents.add(parent); + return Object.freeze({ + ...inspected, + parent, + async adopt(execution: ExecutionRef) { + if (controlDigest(execution) !== controlDigest(inspected.execution)) + throw Error("Normalized execution identity differs from grant"); + return adoptArtifactTargetLease({ + dataRoot: state.dataRoot, + assertion: delegation, + peer, + execution, + serviceIds: inspected.serviceIds, + }); + }, + }); +} +export function verifiedParentForArtifactDelegation( + installation: ControlInstallation, + assertion: string, + execution: ExecutionRef, +): VerifiedParentExecution { + const state = installationState(installation); + const inspected = inspectArtifactDelegation({ + dataRoot: state.dataRoot, + assertion, + installation, + }); + if (controlDigest(inspected.execution) !== controlDigest(execution)) + throw Error("Wrong delegated execution"); + const parent = Object.freeze(retainedControlParent(inspected.parent)) as VerifiedParentExecution; + verifiedParents.add(parent); + return parent; +} diff --git a/packages/core/src/build-artifacts/peer-process.test.ts b/packages/core/src/build-artifacts/peer-process.test.ts new file mode 100644 index 000000000..f34bff397 --- /dev/null +++ b/packages/core/src/build-artifacts/peer-process.test.ts @@ -0,0 +1,692 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { build } from "esbuild"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + type ControlIssuer, + type ExecutionRef, + executionFingerprint, +} from "../ops/execution-identity"; +import { opsOperationFingerprint } from "../ops/fingerprint"; +import { + type ControlChildPhase, + type ControlPlannedChild, + controlChildOperationKey, + controlDigest, + openControlInstallation, + provisionControlInstallation, + retainAdmittedControlExecution, + retainControlEffectReceipt, + retainedControlParent, + verifyControlEffectSettlement, +} from "./control-authority"; +import { + acquireArtifactTargetLease, + delegateArtifactTargetLease, + releaseArtifactTargetLease, + settleArtifactTargetLeaseChild, +} from "./target-lease"; + +let root: string, fixture: string; +beforeAll(async () => { + const result = await build({ + entryPoints: [join(import.meta.dirname, "test-fixtures/peer-process.ts")], + bundle: true, + write: false, + platform: "node", + format: "esm", + target: "node24", + }); + fixture = result.outputFiles[0]?.text; +}); +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-peer-process-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + symlinkSync(root, join(root, "host-alias")); +}); +afterEach(() => rmSync(root, { recursive: true, force: true })); +function ref( + role: ControlIssuer, + kind: string, + operation: unknown, + parent?: ExecutionRef, + phase?: ControlChildPhase, +): ExecutionRef { + const operationFingerprint = + role === "ops-agent" && + !["configurationHandoff.prepare", "configurationMaintenance.prepare"].includes(kind) + ? opsOperationFingerprint(operation as Parameters[0]) + : controlDigest(operation); + const deadlineAt = parent?.deadlineAt ?? Date.now() + 60000; + return { + executor: role, + principal: role === "ops-agent" ? (parent ? "data-manager" : "api") : role, + executionId: role === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), + operationKey: parent + ? controlChildOperationKey(parent.operationKey, phase!, 0) + : `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ + kind, + operationFingerprint, + deadlineAt, + ...(parent ? { parent: retainedControlParent(parent) } : {}), + }), + }; +} +async function child(input: unknown) { + const p = spawn(process.execPath, ["--input-type=module", "-e", fixture], { + stdio: ["pipe", "pipe", "pipe"], + }); + let out = "", + err = ""; + p.stdout.on("data", (b) => (out += b)); + p.stderr.on("data", (b) => (err += b)); + p.stdin.end(`${JSON.stringify(input)}\n`); + await new Promise((resolve, reject) => { + p.once("error", reject); + p.once("exit", (code) => (code === 0 ? resolve() : reject(Error(err)))); + }); + return JSON.parse(out) as { ok: boolean; retained?: boolean; error?: string; leaseId?: string }; +} +describe("separate-process physical-mount delegation", () => { + it.each(["data-manager", "ops-agent"] as const)( + "retains one owner and one receiver across %s direction and path aliases", + async (role) => { + const receiver = role === "data-manager" ? "ops-agent" : "data-manager"; + const phase = role === "data-manager" ? "stop-primary" : "hardlinks"; + const operation = + role === "data-manager" + ? { kind: "transit-sync" } + : { kind: "service.start", serviceId: "motis" }; + const kind = operation.kind; + const rootRef = ref(role, kind, operation); + const childOperation = + receiver === "ops-agent" + ? { kind: "motis.primary.stop" } + : { + kind: "generation-hardlinks", + revisionId: "cfg1_test", + planDigest: "a".repeat(64), + coordinatorMode: "require-running", + }; + const childRef = ref(receiver, childOperation.kind, childOperation, rootRef, phase); + const installation = openControlInstallation({ + infraDir: root, + role, + serviceIds: ["motis", "osrm"], + }); + const plan: ControlPlannedChild = { + phase, + ordinal: 0, + kind: childOperation.kind, + execution: childRef, + serviceIds: ["motis"], + recipient: receiver, + }; + const authority = retainAdmittedControlExecution(installation, { + execution: rootRef, + kind, + operation, + admittedAt: rootRef.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "host-alias/data"), + serviceIds: ["motis"], + owner: authority, + }); + const verifiers = JSON.parse( + readFileSync(join(root, "secrets/control-verifiers.json"), "utf8"), + ); + const assertion = await delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: verifiers.publicKeys[receiver], + }); + const input = { + infraDir: root, + role: receiver, + assertion, + execution: childRef, + kind: childOperation.kind, + operation: childOperation, + admittedAt: childRef.deadlineAt - 60000, + }; + const adopted = await child(input); + expect(adopted, JSON.stringify(adopted)).toMatchObject({ + ok: true, + retained: false, + leaseId: lease.leaseId, + }); + await expect(releaseArtifactTargetLease(lease)).rejects.toThrow(); + expect( + await child({ + ...input, + execution: { ...childRef, operationKey: `opk1_${"z".repeat(32)}` }, + }), + ).toMatchObject({ ok: false }); + expect(await child({ ...input, settle: true })).toMatchObject({ ok: true, retained: true }); + await expect( + delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: verifiers.publicKeys[receiver], + }), + ).rejects.toThrow(/replay/); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, rootRef), + ); + await releaseArtifactTargetLease(lease); + }, + ); +}); +it("keeps owner-death recovery fenced until a separately admitted child settles", async () => { + const { verifyControlRecoveryAuthority } = await import("./control-authority"); + const { reconcileArtifactTargetLease } = await import("./target-lease"); + const role = "data-manager"; + const operation = { kind: "transit-sync" }; + const execution = ref(role, operation.kind, operation); + const childOperation = { kind: "motis.primary.stop" }; + const childRef = ref("ops-agent", childOperation.kind, childOperation, execution, "stop-primary"); + const installation = openControlInstallation({ infraDir: root, role, serviceIds: ["motis"] }); + const plan: ControlPlannedChild = { + phase: "stop-primary", + ordinal: 0, + kind: childOperation.kind, + execution: childRef, + serviceIds: ["motis"], + recipient: "ops-agent", + }; + const admitted = { + execution, + kind: operation.kind, + operation, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }; + const authority = retainAdmittedControlExecution(installation, admitted); + const lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const publicKeys = JSON.parse( + readFileSync(join(root, "secrets/control-verifiers.json"), "utf8"), + ).publicKeys; + const assertion = await delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: publicKeys["ops-agent"], + }); + const input = { + infraDir: root, + role: "ops-agent", + assertion, + execution: childRef, + kind: childOperation.kind, + operation: childOperation, + admittedAt: childRef.deadlineAt - 60000, + }; + expect(await child(input)).toMatchObject({ ok: true }); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + // Reopen role-private receipts exactly as a replacement owner does; no live handle is passed to recovery. + const replacement = openControlInstallation({ infraDir: root, role, serviceIds: ["motis"] }); + const recovered = retainAdmittedControlExecution(replacement, admitted); + const recovery = verifyControlRecoveryAuthority(recovered); + expect( + await reconcileArtifactTargetLease({ + dataRoot: join(root, "host-alias/data"), + leaseId: lease.leaseId, + authority: recovery, + }), + ).toBe("fenced"); + expect(await child({ ...input, settle: true })).toMatchObject({ ok: true }); + expect( + await reconcileArtifactTargetLease({ + dataRoot: join(root, "data"), + leaseId: lease.leaseId, + authority: recovery, + }), + ).toBe("released"); +}); +it("certifies a one-use fixed CLI and its depth-two hardlink leaf across three processes", async () => { + const c = await import("./control-authority"); + const operation = { kind: "service.start" as const, serviceId: "motis" }; + const execution = ref("ops-agent", operation.kind, operation); + const cli = ref("host-cli", operation.kind, operation, execution, "lifecycle"); + const plan: ControlPlannedChild = { + phase: "lifecycle", + ordinal: 0, + kind: operation.kind, + execution: cli, + serviceIds: ["motis"], + recipient: "fixed-cli", + }; + const installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis", "osrm"], + }); + const authority = c.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const id = randomUUID(); + const key = c.prepareFixedControlChild(authority, id); + const assertion = await delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }); + const args = c.fixedCliArguments(operation); + c.retainFixedCliSpawnIntent(authority, { delegationId: id, execution: cli, operation, args }); + const hardlinks = { + kind: "generation-hardlinks", + revisionId: "cfg1_test", + planDigest: "a".repeat(64), + coordinatorMode: "require-running", + }; + const leaf = ref("data-manager", hardlinks.kind, hardlinks, cli, "hardlinks"); + const childPublicKey = JSON.parse( + readFileSync(join(root, "secrets/control-verifiers.json"), "utf8"), + ).publicKeys["data-manager"]; + const fixed = { mode: "fixed", infraDir: root, assertion, execution: cli, args }; + const leafInput = { + execution: leaf, + phase: "hardlinks", + recipient: "data-manager", + serviceIds: ["motis"], + childPublicKey, + }; + expect(await child({ ...fixed, child: { ...leafInput, serviceIds: ["osrm"] } })).toMatchObject({ + ok: false, + }); + const started = (await child({ ...fixed, action: "run", child: leafInput })) as { + ok: boolean; + assertion: string; + error?: string; + }; + expect(started, JSON.stringify(started)).toMatchObject({ ok: true }); + expect(await child({ ...fixed, action: "run" })).toMatchObject({ ok: false }); + expect(await child({ ...fixed, action: "settle" })).toMatchObject({ ok: false }); + expect( + await child({ + infraDir: root, + role: "data-manager", + assertion: started.assertion, + execution: leaf, + kind: hardlinks.kind, + operation: hardlinks, + admittedAt: leaf.deadlineAt - 60000, + settle: true, + }), + ).toMatchObject({ ok: true }); + expect(await child({ ...fixed, action: "settle" })).toMatchObject({ ok: true }); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, execution), + ); + await releaseArtifactTargetLease(lease); + c.closeControlInstallation(installation); +}); +it("keeps an attempted child fenced after SIGKILL of its actual owner process", async () => { + const c = await import("./control-authority"); + const l = await import("./target-lease"); + const operation = { kind: "transit-sync" }; + const execution = ref("data-manager", operation.kind, operation); + const body = { kind: "motis.primary.stop" }; + const receiver = ref("ops-agent", body.kind, body, execution, "stop-primary"); + const plan: ControlPlannedChild = { + phase: "stop-primary", + ordinal: 0, + kind: body.kind, + execution: receiver, + serviceIds: ["motis"], + recipient: "ops-agent", + }; + const admitted = { + execution, + kind: operation.kind, + operation, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }; + const childPublicKey = JSON.parse( + readFileSync(join(root, "secrets/control-verifiers.json"), "utf8"), + ).publicKeys["ops-agent"]; + const p = spawn(process.execPath, ["--input-type=module", "-e", fixture], { + stdio: ["pipe", "pipe", "pipe"], + }); + const response = new Promise<{ ok: boolean; assertion: string; leaseId: string }>( + (resolve, reject) => { + let text = ""; + p.stdout.on("data", (b) => { + text += b; + if (text.includes("\n")) resolve(JSON.parse(text.split("\n")[0]!)); + }); + p.once("error", reject); + }, + ); + p.stdin.end( + `${JSON.stringify({ mode: "owner", infraDir: root, role: "data-manager", admitted, child: { ...plan, childPublicKey } })}\n`, + ); + const owned = await response; + expect(owned.ok).toBe(true); + const exited = new Promise((resolve) => p.once("exit", () => resolve())); + p.kill("SIGKILL"); + await exited; + const installation = c.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis"], + }); + const authority = c.retainAdmittedControlExecution(installation, admitted); + const recovery = c.verifyControlRecoveryAuthority(authority); + expect( + await l.reconcileArtifactTargetLease({ + dataRoot: join(root, "data"), + leaseId: owned.leaseId, + authority: recovery, + }), + ).toBe("fenced"); + const input = { + infraDir: root, + role: "ops-agent", + assertion: owned.assertion, + execution: receiver, + kind: body.kind, + operation: body, + admittedAt: receiver.deadlineAt - 60000, + }; + expect(await child({ ...input, cancel: true })).toMatchObject({ ok: true }); + expect( + await l.reconcileArtifactTargetLease({ + dataRoot: join(root, "data"), + leaseId: owned.leaseId, + authority: recovery, + }), + ).toBe("fenced"); + expect(await child({ ...input, settle: true })).toMatchObject({ ok: true }); + expect( + await l.reconcileArtifactTargetLease({ + dataRoot: join(root, "data"), + leaseId: owned.leaseId, + authority: recovery, + }), + ).toBe("released"); + c.closeControlInstallation(installation); +}); +it.each([false, true])( + "external helper survives owner death and requires terminal preparation (%s)", + async (prepared) => { + const operation = { + kind: "configurationHandoff.prepare", + handoffId: randomUUID(), + expectedRevisionId: "cfg1_old", + desiredRevisionId: "cfg1_new", + image: "sha256:" + "1".repeat(64), + }; + const execution = ref("ops-agent", operation.kind, operation); + const body = { kind: "configuration-helper", handoffId: operation.handoffId }; + const helper = ref("host-cli", body.kind, body, execution, "configuration-handoff"); + const plan: ControlPlannedChild = { + phase: "configuration-handoff", + ordinal: 0, + kind: body.kind, + execution: helper, + serviceIds: ["motis"], + recipient: "configuration-helper", + }; + const admitted = { + execution, + kind: operation.kind, + operation, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }; + const { writeFileSync } = await import("node:fs"); + mkdirSync(join(root, "data/ops-agent"), { mode: 0o700 }); + writeFileSync( + join(root, "data/ops-agent/jobs-v1.json"), + JSON.stringify({ + version: 2, + jobs: [ + { + operationId: execution.executionId, + operationKey: execution.operationKey, + operationFingerprint: execution.operationFingerprint, + fingerprint: execution.fingerprint, + deadlineAt: execution.deadlineAt, + state: prepared ? "succeeded" : "running", + operation, + result: { phase: "prepared", handoffId: operation.handoffId }, + terminalAt: new Date().toISOString(), + }, + ], + }), + { mode: 0o600 }, + ); + const p = spawn(process.execPath, ["--input-type=module", "-e", fixture], { + stdio: ["pipe", "pipe", "pipe"], + }); + const response = new Promise<{ ok: boolean; assertion: string; error?: string }>( + (resolve, reject) => { + let out = ""; + p.stdout.on("data", (b) => { + out += b; + if (out.includes("\n")) resolve(JSON.parse(out.split("\n")[0]!)); + }); + p.once("error", reject); + }, + ); + p.stdin.end( + JSON.stringify({ + mode: "owner", + infraDir: root, + role: "ops-agent", + admitted, + child: { ...plan, childPublicKey: "" }, + fixedId: randomUUID(), + prepared, + }) + "\n", + ); + const owned = await response; + try { + expect(owned, JSON.stringify(owned)).toMatchObject({ ok: true }); + } finally { + const exited = new Promise((resolve) => { + if (p.exitCode !== null) resolve(); + else p.once("exit", () => resolve()); + }); + p.kill("SIGKILL"); + await exited; + } + const observed = await child({ + mode: "helper", + infraDir: root, + assertion: owned.assertion, + execution: helper, + }); + expect(observed, JSON.stringify(observed)).toMatchObject({ ok: prepared }); + }, +); +it("keeps paired-pin approval a one-use leaf under its original activation root", async () => { + const c = await import("./control-authority"); + const operation = { kind: "catalog.activate", validationId: randomUUID() }; + const execution = ref("data-manager", operation.kind, operation); + const body = { + kind: "transitousLock.approve" as const, + ref: "main@" + "a".repeat(40), + approvedBy: "synthetic-reviewer", + }; + const childRef = ref("ops-agent", body.kind, body, execution, "paired-pin-commit"); + // The current legacy kind has a 30 s budget and stays closed in the real adapter. + childRef.deadlineAt = Date.now() + 20000; + childRef.fingerprint = executionFingerprint({ + kind: body.kind, + operationFingerprint: childRef.operationFingerprint, + deadlineAt: childRef.deadlineAt, + parent: c.retainedControlParent(execution), + }); + const plan: ControlPlannedChild = { + phase: "paired-pin-commit", + ordinal: 0, + kind: body.kind, + execution: childRef, + serviceIds: ["motis"], + recipient: "ops-agent", + }; + const installation = c.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis"], + }); + const authority = c.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const key = JSON.parse(readFileSync(join(root, "secrets/control-verifiers.json"), "utf8")) + .publicKeys["ops-agent"]; + const assertion = await delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }); + const input = { + infraDir: root, + role: "ops-agent", + assertion, + execution: childRef, + kind: body.kind, + operation: body, + admittedAt: Date.now(), + }; + for (const patch of [ + { kind: "motis.primary.stop" }, + { transportPrincipal: "host-cli" }, + { execution: { ...childRef, fingerprint: "0".repeat(64) } }, + ]) + expect(await child({ ...input, ...patch })).toMatchObject({ ok: false }); + expect(await child({ ...input, settle: true })).toMatchObject({ ok: true }); + await expect( + delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }), + ).rejects.toThrow(/replay/); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, execution), + ); + await releaseArtifactTargetLease(lease); + c.closeControlInstallation(installation); +}); +it("settles the logical configuration-fence leaf without releasing the maintenance owner", async () => { + const c = await import("./control-authority"); + const operation = { + kind: "configurationMaintenance.prepare", + maintenanceId: randomUUID(), + candidateId: randomUUID(), + }; + const execution = ref("ops-agent", operation.kind, operation); + const body = { + kind: "configuration-maintenance.prepare", + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + }; + const childRef = ref("data-manager", body.kind, body, execution, "configuration-fence"); + const plan: ControlPlannedChild = { + phase: "configuration-fence", + ordinal: 0, + kind: body.kind, + execution: childRef, + serviceIds: ["motis"], + recipient: "data-manager", + }; + const installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const authority = c.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const key = JSON.parse(readFileSync(join(root, "secrets/control-verifiers.json"), "utf8")) + .publicKeys["data-manager"]; + const assertion = await delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }); + const envelope = JSON.parse(Buffer.from(assertion, "base64url").toString("utf8")); + const forged = Buffer.from(JSON.stringify({ ...envelope, signature: "a".repeat(86) })).toString( + "base64url", + ); + const foreign = Buffer.from( + JSON.stringify({ ...envelope, installationId: randomUUID() }), + ).toString("base64url"); + const input = { + infraDir: root, + role: "data-manager", + assertion, + execution: childRef, + kind: body.kind, + operation: body, + admittedAt: childRef.deadlineAt - 60000, + }; + expect(await child({ ...input, assertion: forged })).toMatchObject({ ok: false }); + expect(await child({ ...input, assertion: foreign })).toMatchObject({ ok: false }); + expect(await child({ ...input, settle: true })).toMatchObject({ ok: true }); + await expect(releaseArtifactTargetLease(lease)).rejects.toThrow(/settlement/); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, execution), + ); + await releaseArtifactTargetLease(lease); + c.closeControlInstallation(installation); +}); diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts new file mode 100644 index 000000000..ea9e3cd46 --- /dev/null +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -0,0 +1,371 @@ +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { executionFingerprint } from "../ops/execution-identity"; +import { opsOperationFingerprint } from "../ops/fingerprint"; +import { + type ControlInstallation, + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, + retainAdmittedControlExecution, + retainControlEffectReceipt, + verifyControlEffectSettlement, +} from "./control-authority"; + +let root: string; +let installation: ControlInstallation; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-target-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + installation = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["osrm", "motis"], + }); +}); +afterEach(() => { + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); +}); +function owner() { + const operation = { kind: "service.build" as const, serviceId: "osrm" }; + const operationFingerprint = opsOperationFingerprint(operation); + const deadlineAt = Date.now() + 60000; + return retainAdmittedControlExecution(installation, { + execution: { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + deadlineAt, + }, + kind: operation.kind, + operation, + admittedAt: deadlineAt - 60000, + serviceIds: ["osrm"], + }); +} +async function api() { + expect( + existsSync(join(import.meta.dirname, "target-lease.ts")), + "missing certified filesystem target authority", + ).toBe(true); + return import("./target-lease"); +} +describe("certified target leases", () => { + it("requires nominal durable ownership, then fences unknown own effects", async () => { + const a = await api(); + const authority = owner(); + await expect( + a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: { execution: authority.execution } as typeof authority, + }), + ).rejects.toThrow(/authority/); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/settlement|effect/); + expect(existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(true); + }); + it("requires the exact unchanged inode and durable settlement before release", async () => { + const a = await api(); + const authority = owner(); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, authority.execution), + ); + await a.releaseArtifactTargetLease(lease); + expect(existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(false); + }); + it("rejects replacement lock inodes without removing the replacement", async () => { + const a = await api(); + const authority = owner(); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + const path = join(root, "data/.admin-target-control/locks/osrm"); + renameSync(path, `${path}.old`); + mkdirSync(path, { mode: 0o700 }); + await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/inode|identity|changed/); + expect(existsSync(path)).toBe(true); + }); + it("keeps the fence when the callback throws; elapsed time is never containment", async () => { + const a = await api(); + await expect( + a.withArtifactTargetLease( + { dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: owner() }, + async () => { + throw Error("lost response"); + }, + ), + ).rejects.toThrow("lost response"); + expect(existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(true); + }); +}); +it("never reacquires or expands a retained execution's root", async () => { + const a = await api(); + const authority = owner(); + await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + await expect( + a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }), + ).rejects.toThrow(/already acquired/); + await expect( + a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }), + ).rejects.toThrow(/closure/); +}); +it("detects rollback of a signed shared journal using private high-water evidence", async () => { + const a = await api(); + const authority = owner(); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + const path = join(root, `data/.admin-target-control/leases/${lease.leaseId}/owner.json`); + const previous = readFileSync(path); + await a.cancelArtifactTargetLease(lease); + const { writeFileSync } = await import("node:fs"); + writeFileSync(path, previous); + await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/rollback|replacement/); +}); +it("rejects arbitrary caller text as effect evidence", async () => { + const authority = owner(); + expect(() => + retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: "trust me", + }), + ).toThrow(/evidence|receipt/); +}); +it("retains terminal maintenance ownership and permits only frozen one-use continuations", async () => { + const c = await import("./control-authority"); + const a = await api(); + const { writeFileSync } = await import("node:fs"); + const agent = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis", "osrm"], + }); + const operation = { + kind: "configurationMaintenance.prepare", + maintenanceId: randomUUID(), + candidateId: randomUUID(), + }; + const operationFingerprint = c.controlDigest(operation); + const deadlineAt = Date.now() + 60000; + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const body = { kind: "service.recreate" as const, serviceId: "motis" }; + const hash = opsOperationFingerprint(body); + const child = { + ...execution, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: c.controlChildOperationKey(execution.operationKey, "maintenance-recreate", 0), + operationFingerprint: hash, + fingerprint: executionFingerprint({ + kind: body.kind, + operationFingerprint: hash, + deadlineAt, + parent: c.retainedControlParent(execution), + }), + }; + const plan = { + phase: "maintenance-recreate" as const, + ordinal: 0, + kind: body.kind, + execution: child, + serviceIds: ["motis"], + recipient: "ops-agent" as const, + }; + const authority = c.retainAdmittedControlExecution(agent, { + execution, + kind: operation.kind, + operation, + admittedAt: deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + await expect( + a.delegateArtifactTargetLease(lease, { ...plan, childPublicKey: agent.publicKey }), + ).rejects.toThrow(); + mkdirSync(join(root, "data/ops-agent"), { mode: 0o700 }); + writeFileSync( + join(root, "data/ops-agent/jobs-v1.json"), + JSON.stringify({ + version: 2, + jobs: [ + { + operationId: execution.executionId, + operationKey: execution.operationKey, + operationFingerprint: execution.operationFingerprint, + fingerprint: execution.fingerprint, + deadlineAt, + state: "succeeded", + operation, + result: { state: "prepared" }, + terminalAt: new Date().toISOString(), + }, + ], + }), + { mode: 0o600 }, + ); + c.retainControlPreparedTerminal(authority); + await expect( + a.delegateArtifactTargetLease(lease, { + ...plan, + serviceIds: ["osrm"], + childPublicKey: agent.publicKey, + }), + ).rejects.toThrow(); + const assertion = await a.delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: agent.publicKey, + }); + await expect( + a.delegateArtifactTargetLease(lease, { ...plan, childPublicKey: agent.publicKey }), + ).rejects.toThrow(/replay/); + const peer = c.verifyRetainedLocalControlPeer(agent, { + assertion, + execution: child, + kind: body.kind, + }); + const borrowed = await a.adoptArtifactTargetLease({ + dataRoot: join(root, "data"), + assertion, + peer, + execution: child, + serviceIds: ["motis"], + }); + const childAuthority = c.retainAdmittedControlExecution(agent, { + execution: child, + kind: body.kind, + operation: body, + parent: c.retainedControlParent(execution), + admittedAt: deadlineAt - 60000, + serviceIds: ["motis"], + }); + await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(); + c.retainControlEffectReceipt(childAuthority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild(borrowed, c.verifyControlEffectSettlement(agent, child)); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild(lease, c.verifyControlEffectSettlement(agent, execution)); + await a.releaseArtifactTargetLease(lease); + c.closeControlInstallation(agent); +}); +it("orders physical target ownership before a nonblocking reservation, configuration writer and release store", async () => { + const { acquireConfigurationOwnership } = await import("../services/configuration-ownership"); + const { acquireReleaseStoreLock } = await import("../services/release-store-lock"); + const { rmdirSync } = await import("node:fs"); + const a = await api(); + const c = await import("./control-authority"); + const authority = owner(); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + const order = ["target"]; + // The future DM durable reservation is a closed consumer. This fixture uses + // an actual exclusive directory, not an in-memory success flag. + const reservation = join(root, "data/transit-reservation-fixture"); + mkdirSync(reservation, { mode: 0o700 }); + order.push("reservation"); + expect(() => mkdirSync(reservation, { mode: 0o700 })).toThrow(); + expect(existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(true); + const releaseWriter = await acquireConfigurationOwnership(root); + order.push("configuration"); + const store = await acquireReleaseStoreLock(root, {}, { failIfBusy: true }); + order.push("release-store"); + expect(existsSync(join(root, ".trusted-config-writer"))).toBe(true); + expect(order).toEqual(["target", "reservation", "configuration", "release-store"]); + store.release(); + releaseWriter(); + rmdirSync(reservation); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, authority.execution), + ); + await a.releaseArtifactTargetLease(lease); +}); +it("does not turn an already-started execution into no-effect evidence", async () => { + const c = await import("./control-authority"); + const authority = owner(); + c.beginControlEffect(authority); + expect(() => + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }), + ).toThrow(/Started/); +}); +it("does not restart effects after retaining unused-execution evidence", async () => { + const c = await import("./control-authority"); + const authority = owner(); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + expect(() => c.beginControlEffect(authority)).toThrow(/cannot restart/); +}); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts new file mode 100644 index 000000000..4bff8890e --- /dev/null +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -0,0 +1,1348 @@ +import { createPrivateKey, type KeyObject, randomUUID } from "node:crypto"; +import { fsyncSync, mkdirSync } from "node:fs"; +import { setTimeout as delay } from "node:timers/promises"; +import { z } from "zod"; +import { + type ExecutionRef, + executionFingerprint, + executionRefSchema, +} from "../ops/execution-identity"; +import { canonicalJson } from "../ops/fingerprint"; +import { + type ArtifactDelegationAssertion, + type AuthorityState, + assertionSchema, + type ChildSettlement, + type ControlChildPhase, + type ControlPlannedChild, + type ControlRecipient, + childPhaseKinds, + childSettlementState, + controlChildOperationKey, + controlDigest, + controlPeerState, + digest, + encodePrivate, + executionAuthorityState, + type InstallationState, + installationState, + newControlKey, + parseControlAssertion, + readControlExecutionWitness, + readPrivateRecord, + recoveryAuthorityState, + retainedControlParent, + signControl, + targetSchema, + uuid, + type VerifiedChildSettlement, + type VerifiedControlPeer, + type VerifiedExecutionAuthority, + type VerifiedRecoveryAuthority, + verifyControl, + verifyControlEffectSettlement, + verifyControlPreparedTerminal, + writePrivateRecord, +} from "./control-authority"; +import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; + +export type { ChildSettlement } from "./control-authority"; +export type ArtifactTargetLease = { + readonly leaseId: string; + readonly owner: ExecutionRef; + readonly serviceIds: readonly string[]; + readonly mode: "owner"; +}; +export type BorrowedArtifactTargetLease = { + readonly leaseId: string; + readonly owner: ExecutionRef; + readonly execution: ExecutionRef; + readonly serviceIds: readonly string[]; + readonly mode: "borrowed"; +}; +export type TargetLeaseAuthority = ArtifactTargetLease | BorrowedArtifactTargetLease; +const inode = z.strictObject({ dev: z.string().regex(/^\d+$/), ino: z.string().regex(/^\d+$/) }); +const certificateSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + leaseId: uuid, + dataRootIdentity: inode, + lockIdentities: z.record(z.string(), inode), + owner: executionRefSchema, + ownerPublicKey: z.string().max(256), + serviceIds: targetSchema, + createdAt: z.number().int().nonnegative().safe(), +}); +type Certificate = z.infer; +const signedCertificateSchema = z.strictObject({ + payload: certificateSchema, + signature: z.string(), +}); +const outcomeSchema = z.strictObject({ + identity: executionRefSchema, + outcome: z.enum(["contained", "committed", "failed_without_effect", "unknown"]), + publication: z.enum(["not_started", "committed", "unknown"]), + terminalReceiptDigest: digest, +}); +const phaseSchema = z.enum(["held", "cancel_pending", "recovery_required", "settled"]); +const rootSchema = z.strictObject({ + certificate: signedCertificateSchema, + sequence: z.number().int().nonnegative().safe(), + phase: phaseSchema, + grantDigests: z.record(uuid, digest), + settlement: outcomeSchema.nullable(), +}); +type RootRecord = z.infer; +const grantSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + leaseId: uuid, + ownerCertificateDigest: digest, + delegationId: uuid, + parentNode: uuid.nullable(), + parent: executionRefSchema, + child: executionRefSchema, + childKind: z.string().max(128), + phase: z.string().max(64), + ordinal: z.number().int().nonnegative().max(63), + serviceIds: targetSchema, + recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), + childPublicKey: z.string().max(256), + permittedFurtherEdges: z.array(z.string()).max(1), + depth: z.number().int().min(1).max(2), +}); +type Grant = z.infer; +const childSchema = z.strictObject({ + delegationId: uuid, + grantDigest: digest, + execution: executionRefSchema, + sequence: z.number().int().nonnegative().safe(), + phase: phaseSchema, + grantDigests: z.record(uuid, digest), + settlement: outcomeSchema.nullable(), +}); +type ChildRecord = z.infer; +const transitionSchema = z.strictObject({ + delegationId: uuid, + grantDigest: digest, + sequence: z.literal(1), + state: z.literal("attempted"), +}); +interface State { + data: AnchoredDirectory; + control: AnchoredDirectory; + locks: AnchoredDirectory; + leases: AnchoredDirectory; + ledger: AnchoredDirectory; + installation: InstallationState; + certificate: Certificate; + certificateEnvelope: ReturnType>; + key: KeyObject; + node: string | null; + execution: ExecutionRef; + authority?: AuthorityState; + closed: boolean; +} +const handles = new WeakMap(); +function stateOf(lease: TargetLeaseAuthority): State { + const state = handles.get(lease); + if (!state || state.closed || (lease.mode === "owner") !== (state.node === null)) + throw Error("Unverified or mixed target lease authority"); + verifyAnchors(state); + return state; +} +function verifyAnchors(s: State) { + s.data.check(); + s.ledger.check(); + if (!sameInode(identity(s.data.stat), s.certificate.dataRootIdentity)) + throw Error("Data root physical identity changed"); + for (const id of s.certificate.serviceIds) { + const lock = s.locks.directory(id); + try { + if (!sameInode(identity(lock.stat), s.certificate.lockIdentities[id]!)) + throw Error("Target lock inode identity changed"); + if (lock.list().length) throw Error("Unexplained nonempty target lock"); + } finally { + lock.close(); + } + } +} +function json(dir: AnchoredDirectory, name: string, max = 131072): unknown { + return JSON.parse(dir.read(name, max).toString("utf8")); +} +function readRoot(s: State): RootRecord { + return verifyControl( + s.certificate.ownerPublicKey, + "openmapx-lease-owner-v1", + json(s.ledger, "owner.json"), + rootSchema, + ); +} +function childDirectory(s: State, kind: "grants" | "children") { + return s.ledger.directory(kind); +} +function readGrant(s: State, id: string, parentKey: string): Grant { + const dir = childDirectory(s, "grants"); + try { + const envelope = z + .strictObject({ grant: z.unknown(), transition: z.unknown() }) + .parse(json(dir, `${uuid.parse(id)}.json`)); + const grant = verifyControl(parentKey, "openmapx-lease-grant-v1", envelope.grant, grantSchema); + const transition = verifyControl( + parentKey, + "openmapx-lease-intent-v1", + envelope.transition, + transitionSchema, + ); + if ( + grant.delegationId !== id || + transition.delegationId !== id || + transition.grantDigest !== controlDigest(envelope.grant) || + grant.ownerCertificateDigest !== controlDigest(s.certificateEnvelope) || + grant.installationId !== s.installation.installation.installationId || + grant.leaseId !== s.certificate.leaseId + ) + throw Error("Anchored grant identity mismatch"); + return grant; + } finally { + dir.close(); + } +} +function readChild(s: State, grant: Grant): ChildRecord | undefined { + const dir = childDirectory(s, "children"); + try { + let raw: unknown; + try { + raw = json(dir, `${grant.delegationId}.json`); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === "ENOENT") return; + throw e; + } + const child = verifyControl(grant.childPublicKey, "openmapx-lease-child-v1", raw, childSchema); + if ( + child.delegationId !== grant.delegationId || + controlDigest(child.execution) !== controlDigest(grant.child) + ) + throw Error("Child durable identity mismatch"); + return child; + } finally { + dir.close(); + } +} +function highWater(s: State, nodes: Record) { + const name = `${s.certificate.leaseId}.json`; + let prior: Record = {}; + try { + prior = z + .record( + z.string(), + z.strictObject({ sequence: z.number().int().nonnegative().safe(), digest }), + ) + .parse(readPrivateRecord(s.installation, "observations", name)); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + for (const [key, old] of Object.entries(prior)) { + const current = nodes[key]; + if ( + !current || + current.sequence < old.sequence || + (current.sequence === old.sequence && current.digest !== old.digest) + ) + throw Error("Shared ledger rollback or replacement"); + } + writePrivateRecord(s.installation, "observations", name, nodes); +} +function tree(s: State) { + const root = readRoot(s); + if (controlDigest(root.certificate) !== controlDigest(s.certificateEnvelope)) + throw Error("Owner certificate replacement"); + const nodes: Record = { + owner: { sequence: root.sequence, digest: controlDigest(root) }, + }; + const grants = new Map(); + const children = new Map(); + let count = 0; + function walk( + parent: ExecutionRef, + node: string | null, + subset: readonly string[], + publicKey: string, + digests: Record, + depth: number, + edges?: readonly string[], + ) { + for (const [id, expected] of Object.entries(digests)) { + if (++count > 64) throw Error("Child intent capacity exceeded"); + const grant = readGrant(s, id, publicKey); + const dir = childDirectory(s, "grants"); + try { + const raw = json(dir, `${id}.json`) as { grant: unknown }; + if (controlDigest(raw.grant) !== expected) throw Error("Grant digest changed"); + } finally { + dir.close(); + } + if ( + grant.child.fingerprint !== + executionFingerprint({ + kind: grant.childKind, + operationFingerprint: grant.child.operationFingerprint, + deadlineAt: grant.child.deadlineAt, + parent: retainedControlParent(grant.parent), + }) || + grant.parentNode !== node || + controlDigest(grant.parent) !== controlDigest(parent) || + grant.depth !== depth || + grant.child.deadlineAt > parent.deadlineAt || + grant.serviceIds.some((id) => !subset.includes(id)) || + (edges && !edges.includes(grant.phase)) + ) + throw Error("Grant ancestry/deadline/subset mismatch"); + const witness = readControlExecutionWitness( + s.installation, + parent, + node === null ? s.installation.publicKeys[parent.executor] : publicKey, + ); + const allowedKinds = childPhaseKinds[grant.phase as ControlChildPhase]; + if ( + !allowedKinds?.includes(grant.childKind) || + grant.child.operationKey !== + controlChildOperationKey( + parent.operationKey, + grant.phase as ControlChildPhase, + grant.ordinal, + ) + ) + throw Error("Grant kind or stable phase key mismatch"); + const parentRecipient = + node === null ? s.certificate.owner.executor : grants.get(node)?.recipient; + if (!parentRecipient) throw Error("Missing parent recipient"); + const expectedEdges = verifyFiniteEdge( + parentRecipient, + witness.kind, + grant.phase as ControlChildPhase, + grant.recipient, + ); + if ( + canonicalJson(expectedEdges) !== canonicalJson(grant.permittedFurtherEdges) || + grant.serviceIds.some((id) => !witness.serviceIds.includes(id)) + ) + throw Error("Grant widens certified parent policy"); + grants.set(id, grant); + const child = readChild(s, grant); + if (child) { + if (child.grantDigest !== expected) throw Error("Adoption grant digest mismatch"); + children.set(id, child); + nodes[id] = { sequence: child.sequence, digest: controlDigest(child) }; + walk( + grant.child, + id, + grant.serviceIds, + grant.childPublicKey, + child.grantDigests, + depth + 1, + grant.permittedFurtherEdges, + ); + } + } + } + walk( + s.certificate.owner, + null, + s.certificate.serviceIds, + s.certificate.ownerPublicKey, + root.grantDigests, + 1, + ); + const grantDir = childDirectory(s, "grants"), + childDir = childDirectory(s, "children"); + try { + if ( + grantDir.list().some((n) => !grants.has(n.replace(/\.json$/, ""))) || + childDir.list().some((n) => !children.has(n.replace(/\.json$/, ""))) + ) + throw Error("Unexplained protocol record; recovery required"); + } finally { + grantDir.close(); + childDir.close(); + } + highWater(s, nodes); + return { root, grants, children }; +} +function ledgerBytes(s: State) { + let bytes = s.ledger.read("owner.json", 131072).length; + for (const kind of ["grants", "children"] as const) { + const dir = childDirectory(s, kind); + try { + if (dir.list().length > 64) throw Error("Protocol record capacity"); + for (const name of dir.list()) bytes += dir.read(name, 131072).length; + } finally { + dir.close(); + } + } + return bytes; +} +function writeBounded( + s: State, + kind: "owner" | "grants" | "children", + name: string, + value: unknown, + exclusive = false, +) { + const dir = kind === "owner" ? s.ledger : childDirectory(s, kind); + try { + const bytes = canonicalJson(value); + let old = 0; + try { + old = dir.read(name, 131072).length; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (ledgerBytes(s) - old + Buffer.byteLength(bytes) > 131072) + throw Error("Lease ledger capacity exceeded"); + dir.write(name, bytes, exclusive); + } finally { + if (dir !== s.ledger) dir.close(); + } +} +async function guarded(s: State, run: () => T): Promise { + verifyAnchors(s); + const path = s.ledger.name(".journal-writer"); + try { + mkdirSync(path, { mode: 0o700 }); + fsyncSync(s.ledger.fd); + } catch { + throw Error("Lease journal writer is busy or requires supervised recovery"); + } + const guard = s.ledger.directory(".journal-writer"); + const expected = identity(guard.stat); + try { + return run(); + } finally { + guard.close(); + s.ledger.removeDirectory(".journal-writer", expected); + } +} +function writeNode(s: State, node: RootRecord | ChildRecord) { + const domain = s.node === null ? "openmapx-lease-owner-v1" : "openmapx-lease-child-v1"; + writeBounded( + s, + s.node === null ? "owner" : "children", + s.node === null ? "owner.json" : `${s.node}.json`, + signControl(s.key, domain, node), + ); +} +function closeState(s: State) { + if (s.closed) return; + s.closed = true; + s.ledger.close(); + s.leases.close(); + s.locks.close(); + s.control.close(); + s.data.close(); +} +function knownOutcome(e: ChildSettlement | null | undefined) { + return ( + !!e && + e.outcome !== "unknown" && + e.publication !== "unknown" && + !(e.outcome === "failed_without_effect" && e.publication !== "not_started") + ); +} +export async function acquireArtifactTargetLease(options: { + dataRoot: string; + serviceIds: readonly string[]; + owner: VerifiedExecutionAuthority; + signal?: AbortSignal; +}): Promise { + const authority = executionAuthorityState(options.owner); + if (authority.installation.fixedChildId) + throw Error("Fixed children must borrow and cannot acquire a root"); + const serviceIds = targetSchema.parse(options.serviceIds); + if ( + serviceIds.some((id) => !authority.serviceIds.includes(id)) || + Date.now() >= authority.execution.deadlineAt + ) + throw Error("Target closure or execution deadline mismatch"); + options.signal?.throwIfAborted(); + try { + readPrivateRecord( + authority.installation, + "execution-leases", + `${authority.execution.operationKey}.json`, + ); + throw Error("Execution already acquired a root lease; reconcile its original identity"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + const data = new AnchoredDirectory(options.dataRoot); + const trusted = new AnchoredDirectory(authority.installation.dataRoot); + try { + if (!sameInode(identity(data.stat), identity(trusted.stat))) + throw Error("Foreign physical data root"); + } finally { + trusted.close(); + } + const control = data.directory(".admin-target-control", true, true), + locks = control.directory("locks", true, true), + leases = control.directory("leases", true, true); + const acquired: Record = {}; + let published = false; + let ledger: AnchoredDirectory | undefined; + try { + let unsettled = 0; + for (const id of leases.list()) { + uuid.parse(id); + const d = leases.directory(id); + try { + const raw = json(d, "owner.json"); + const preliminary = z.object({ payload: rootSchema }).parse(raw); + const certificate = preliminary.payload.certificate; + const cert = verifyControl( + authority.installation.publicKeys[certificate.payload.owner.executor], + "openmapx-lease-certificate-v1", + certificate, + certificateSchema, + ); + if (cert.installationId !== authority.installation.installation.installationId) + throw Error("Foreign retained lease"); + const record = verifyControl( + cert.ownerPublicKey, + "openmapx-lease-owner-v1", + raw, + rootSchema, + ); + if (record.phase !== "settled") unsettled++; + } finally { + d.close(); + } + } + if (unsettled >= 256) throw Error("Unsettled lease capacity exhausted"); + const until = Math.min(Date.now() + 5000, authority.execution.deadlineAt); + for (const id of serviceIds) { + for (;;) { + options.signal?.throwIfAborted(); + if (Date.now() >= until) throw Error("Target lease acquisition deadline elapsed"); + try { + mkdirSync(locks.name(id), { mode: 0o700 }); + fsyncSync(locks.fd); + const lock = locks.directory(id); + acquired[id] = identity(lock.stat); + lock.close(); + break; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + if (Date.now() >= until) + throw Error("Target lease busy; existing ownership cannot be stolen"); + await delay(Math.min(50, until - Date.now()), undefined, { signal: options.signal }); + } + } + } + const leaseId = randomUUID(); + ledger = leases.directory(leaseId, true, true); + ledger.directory("grants", true).close(); + ledger.directory("children", true).close(); + const key = newControlKey(); + const certificate: Certificate = { + version: 1, + installationId: authority.installation.installation.installationId, + leaseId, + dataRootIdentity: identity(data.stat), + lockIdentities: acquired, + owner: authority.execution, + ownerPublicKey: key.publicKey, + serviceIds, + createdAt: Date.now(), + }; + const certificateEnvelope = signControl( + authority.installation.privateKey, + "openmapx-lease-certificate-v1", + certificate, + ); + writePrivateRecord( + authority.installation, + "leases", + `${leaseId}.json`, + { + certificate: certificateEnvelope, + privateKey: encodePrivate(key.privateKey), + executionReceiptDigest: authority.receiptDigest, + }, + true, + ); + const record: RootRecord = { + certificate: certificateEnvelope, + sequence: 0, + phase: "held", + grantDigests: {}, + settlement: null, + }; + writePrivateRecord( + authority.installation, + "execution-leases", + `${authority.execution.operationKey}.json`, + { leaseId, certificateDigest: controlDigest(certificateEnvelope) }, + true, + ); + published = true; + ledger.write( + "owner.json", + canonicalJson(signControl(key.privateKey, "openmapx-lease-owner-v1", record)), + true, + ); + const state: State = { + data, + control, + locks, + leases, + ledger, + installation: authority.installation, + certificate, + certificateEnvelope, + key: key.privateKey, + node: null, + execution: authority.execution, + authority, + closed: false, + }; + const lease = Object.freeze({ + leaseId, + owner: certificate.owner, + serviceIds: Object.freeze([...serviceIds]), + mode: "owner" as const, + }); + handles.set(lease, state); + tree(state); + return lease; + } catch (e) { + if (!published) + for (const [id, expected] of Object.entries(acquired)) locks.removeDirectory(id, expected); + ledger?.close(); + leases.close(); + locks.close(); + control.close(); + data.close(); + throw e; + } +} +function verifyFiniteEdge( + role: ControlRecipient | "host-cli", + kind: string, + phase: ControlChildPhase, + recipient: ControlRecipient, +): string[] { + if (role === "data-manager") { + if ( + !["transit-sync", "catalog.activate", "transit-maintenance"].includes(kind) || + recipient !== "ops-agent" || + ![ + "stop-primary", + "stop-staging", + "restart-staging", + "promote-primary", + "recover-primary", + "proxy-stage", + "proxy-commit", + "proxy-restore", + "paired-pin-commit", + ].includes(phase) || + (phase === "paired-pin-commit" && kind !== "catalog.activate") + ) + throw Error("Unauthorized data-manager child edge"); + return []; + } + if ( + role === "ops-agent" && + recipient === "fixed-cli" && + ["lifecycle", "build", "data-link", "data-clean"].includes(phase) && + childPhaseKinds[phase].includes(kind) + ) + return phase === "data-clean" ? [] : ["hardlinks"]; + if ( + ["ops-agent", "fixed-cli", "host-cli"].includes(role) && + recipient === "data-manager" && + phase === "hardlinks" && + [ + "service.start", + "system.update", + "service.build", + "services.buildAll", + "data.link", + "configuration.hardlinks.apply", + ].includes(kind) + ) + return []; + if ( + role === "ops-agent" && + recipient === "configuration-helper" && + phase === "configuration-handoff" && + kind === "configurationHandoff.prepare" + ) + return []; + if (role === "ops-agent" && kind === "configurationMaintenance.prepare") { + if ( + recipient === "data-manager" && + ["configuration-fence", "configuration-fence-settle"].includes(phase) + ) + return []; + if ( + recipient === "ops-agent" && + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(phase) + ) + return phase === "maintenance-hardlinks" + ? ["hardlinks"] + : phase === "maintenance-handoff" + ? ["configuration-handoff"] + : []; + } + throw Error("Unauthorized finite control edge"); +} +function permitted(s: State, child: ControlPlannedChild, parentGrant?: Grant): string[] { + if ( + parentGrant && + (!parentGrant.permittedFurtherEdges.includes(child.phase) || parentGrant.depth >= 2) + ) + throw Error("Unauthorized descendant edge"); + const key = parentGrant?.childPublicKey ?? s.certificate.ownerPublicKey; + // Root execution witnesses use the pinned role key; borrowed witnesses use the certified child key. + const witness = readControlExecutionWitness( + s.installation, + s.execution, + s.node === null ? s.installation.publicKeys[s.execution.executor] : key, + ); + return verifyFiniteEdge( + parentGrant?.recipient ?? s.installation.installation.role, + witness.kind, + child.phase, + child.recipient, + ); +} +export async function delegateArtifactTargetLease( + lease: TargetLeaseAuthority, + child: { + execution: ExecutionRef; + phase: ControlChildPhase; + serviceIds: readonly string[]; + recipient: ControlRecipient; + childPublicKey: string; + }, +): Promise { + const s = stateOf(lease); + return guarded(s, () => { + const current = tree(s); + const node = s.node === null ? current.root : current.children.get(s.node)!; + if ( + node.phase !== "held" || + current.root.phase !== "held" || + Date.now() >= s.execution.deadlineAt + ) + throw Error("Lease does not permit forward delegation"); + let plan = s.authority?.plan.find( + (p) => + controlDigest(p.execution) === controlDigest(child.execution) && + p.phase === child.phase && + p.recipient === child.recipient && + canonicalJson(p.serviceIds) === canonicalJson(child.serviceIds), + ); + if (!plan && s.node !== null) { + const inherited = current.grants.get(s.node)!; + const kinds = childPhaseKinds[child.phase]; + if ( + inherited.permittedFurtherEdges.includes(child.phase) && + kinds.length === 1 && + child.execution.operationKey === + controlChildOperationKey(s.execution.operationKey, child.phase, 0) + ) + plan = { ...child, kind: kinds[0]!, ordinal: 0 }; + } + if (!plan) throw Error("Child is absent from immutable admitted plan"); + if (current.grants.size >= 64) throw Error("Child intent capacity exhausted"); + if ( + [...current.grants.values()].some( + (g) => g.child.operationKey === child.execution.operationKey, + ) + ) + throw Error("Attempted child cannot be replayed"); + if (child.serviceIds.some((id) => !lease.serviceIds.includes(id))) + throw Error("Widened target subset"); + const parentGrant = s.node === null ? undefined : current.grants.get(s.node); + const edges = permitted(s, plan, parentGrant); + if ( + s.authority?.kind === "configurationMaintenance.prepare" && + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(child.phase) + ) + verifyControlPreparedTerminal(s.installation, s.execution); + let delegationId: string = randomUUID(); + if (child.recipient === "fixed-cli" || child.recipient === "configuration-helper") { + const dir = s.installation.privateRoot.directory("children"); + try { + const matches = dir.list().filter((name) => { + if (!name.endsWith(".key")) return false; + const saved = JSON.parse(dir.read(name, 2048).toString("utf8")); + return saved.publicKey === child.childPublicKey; + }); + if (matches.length !== 1) throw Error("Missing fixed child private challenge"); + delegationId = uuid.parse(matches[0]?.slice(0, -4)); + } finally { + dir.close(); + } + } else if (child.childPublicKey !== s.installation.publicKeys[child.recipient]) + throw Error("Child role key mismatch"); + const grant: Grant = { + version: 1, + installationId: s.certificate.installationId, + leaseId: lease.leaseId, + ownerCertificateDigest: controlDigest(s.certificateEnvelope), + delegationId, + parentNode: s.node, + parent: s.execution, + child: child.execution, + childKind: plan.kind, + phase: child.phase, + ordinal: plan.ordinal, + serviceIds: [...child.serviceIds], + recipient: child.recipient, + childPublicKey: child.childPublicKey, + permittedFurtherEdges: edges, + depth: (parentGrant?.depth ?? 0) + 1, + }; + const envelope = signControl(s.key, "openmapx-lease-grant-v1", grant); + const grantDigest = controlDigest(envelope); + const transition = signControl(s.key, "openmapx-lease-intent-v1", { + delegationId, + grantDigest, + sequence: 1 as const, + state: "attempted" as const, + }); + const ancestors: string[] = []; + let ancestor = s.node; + while (ancestor) { + ancestors.unshift(ancestor); + ancestor = current.grants.get(ancestor)?.parentNode ?? null; + } + const fields = { + version: 1 as const, + installationId: s.certificate.installationId, + leaseId: lease.leaseId, + ownerCertificateDigest: controlDigest(s.certificateEnvelope), + delegationId, + grantDigest, + ancestorDelegationIds: ancestors, + }; + const signature = signControl(s.key, "openmapx-lease-assertion-v1", fields).signature; + const assertion = Buffer.from(canonicalJson({ ...fields, signature })).toString( + "base64url", + ) as ArtifactDelegationAssertion; + parseControlAssertion(assertion); + writePrivateRecord( + s.installation, + "attempts", + `${delegationId}.json`, + { grant: envelope, transition, assertion }, + true, + ); + writeBounded(s, "grants", `${delegationId}.json`, { grant: envelope, transition }, true); + node.grantDigests[delegationId] = grantDigest; + node.sequence++; + writeNode(s, node); + tree(s); + return assertion; + }); +} +function openState( + dataRoot: string, + leaseId: string, + installation: InstallationState, + terminalObservation = false, +): State { + uuid.parse(leaseId); + const data = new AnchoredDirectory(dataRoot); + const control = data.directory(".admin-target-control"), + locks = control.directory("locks"), + leases = control.directory("leases"), + ledger = leases.directory(leaseId); + try { + const raw = z.object({ payload: rootSchema }).parse(json(ledger, "owner.json")); + const envelope = raw.payload.certificate; + const cert = certificateSchema.parse(envelope.payload); + const certificate = verifyControl( + installation.publicKeys[cert.owner.executor], + "openmapx-lease-certificate-v1", + envelope, + certificateSchema, + ); + if ( + certificate.installationId !== installation.installation.installationId || + certificate.leaseId !== leaseId + ) + throw Error("Foreign lease installation"); + const s = { + data, + control, + locks, + leases, + ledger, + installation, + certificate, + certificateEnvelope: envelope, + key: installation.privateKey, + node: null, + execution: certificate.owner, + closed: false, + } satisfies State; + if (terminalObservation && readRoot(s).phase === "settled") { + s.data.check(); + s.ledger.check(); + if (!sameInode(identity(s.data.stat), s.certificate.dataRootIdentity)) + throw Error("Changed physical root"); + } else verifyAnchors(s); + return s; + } catch (e) { + ledger.close(); + leases.close(); + locks.close(); + control.close(); + data.close(); + throw e; + } +} +export async function adoptArtifactTargetLease(options: { + dataRoot: string; + assertion: string; + peer: VerifiedControlPeer; + execution: ExecutionRef; + serviceIds: readonly string[]; +}): Promise { + const peer = controlPeerState(options.peer); + if ( + options.assertion !== peer.assertion || + controlDigest(options.execution) !== controlDigest(peer.execution) + ) + throw Error("Peer execution binding mismatch"); + const assertion = parseControlAssertion(options.assertion); + const s = openState(options.dataRoot, assertion.leaseId, peer.installation); + try { + return await guarded(s, () => { + const t = tree(s); + const grant = t.grants.get(assertion.delegationId); + if ( + !grant || + controlDigest(s.certificateEnvelope) !== assertion.ownerCertificateDigest || + controlDigest(grant.child) !== controlDigest(options.execution) || + grant.childKind !== peer.kind || + grant.recipient !== peer.recipient || + grant.childPublicKey !== peer.childPublicKey || + canonicalJson(grant.serviceIds) !== canonicalJson(targetSchema.parse(options.serviceIds)) + ) + throw Error("Wrong peer principal, kind, key, fingerprint, or subset"); + const parentGrant = grant.parentNode ? t.grants.get(grant.parentNode) : undefined; + const parentKey = parentGrant?.childPublicKey ?? s.certificate.ownerPublicKey; + const { signature, ...fields } = assertion; + verifyControl( + parentKey, + "openmapx-lease-assertion-v1", + { payload: fields, signature }, + assertionSchema.omit({ signature: true }), + ); + const parent = t.children.get(grant.parentNode ?? "") ?? t.root; + const expectedAncestors: string[] = []; + let ancestor = grant.parentNode; + while (ancestor) { + expectedAncestors.unshift(ancestor); + ancestor = t.grants.get(ancestor)?.parentNode ?? null; + } + if ( + canonicalJson(expectedAncestors) !== canonicalJson(assertion.ancestorDelegationIds) || + parent.grantDigests[grant.delegationId] !== assertion.grantDigest + ) + throw Error("Wrong grant ancestry/digest"); + const issuer = parentGrant?.recipient ?? s.certificate.owner.executor; + if ( + peer.recipient === "ops-agent" && + issuer !== "data-manager" && + !( + issuer === "ops-agent" && + peer.local && + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(grant.phase) + ) + ) + throw Error("Wrong peer principal"); + if ( + peer.recipient === "data-manager" && + !["ops-agent", "fixed-cli", "host-cli"].includes(issuer) + ) + throw Error("Wrong peer principal"); + const retained = t.children.get(grant.delegationId); + if ( + !retained && + (t.root.phase !== "held" || parent.phase !== "held" || Date.now() >= grant.child.deadlineAt) + ) + throw Error("Stale or canceled delegation"); + const admissionName = `${grant.delegationId}.json`; + const pending = { + version: 1, + grantDigest: assertion.grantDigest, + execution: grant.child, + state: "pending-adoption", + assertion: options.assertion, + }; + let local: unknown; + try { + local = readPrivateRecord(peer.installation, "adoptions", admissionName); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (local) { + const old = z + .object({ + grantDigest: digest, + execution: executionRefSchema, + state: z.enum(["pending-adoption", "ready"]), + }) + .parse(local); + if ( + old.grantDigest !== assertion.grantDigest || + controlDigest(old.execution) !== controlDigest(grant.child) + ) + throw Error("Durable duplicate admission conflict"); + } else writePrivateRecord(peer.installation, "adoptions", admissionName, pending, true); + s.key = peer.childPrivateKey; + s.node = grant.delegationId; + s.execution = grant.child; + if (!retained) + writeBounded( + s, + "children", + `${s.node}.json`, + signControl(s.key, "openmapx-lease-child-v1", { + delegationId: s.node, + grantDigest: assertion.grantDigest, + execution: grant.child, + sequence: 0, + phase: "held", + grantDigests: {}, + settlement: null, + } satisfies ChildRecord), + true, + ); + writePrivateRecord(peer.installation, "adoptions", admissionName, { + ...pending, + state: "ready", + }); + tree(s); + const handle = Object.freeze({ + leaseId: assertion.leaseId, + owner: s.certificate.owner, + execution: grant.child, + serviceIds: Object.freeze([...grant.serviceIds]), + mode: "borrowed" as const, + }); + handles.set(handle, s); + return handle; + }); + } catch (e) { + closeState(s); + throw e; + } +} +/** A fixed receiver supplies its own durable execution authority only after adoption. */ +export function bindBorrowedControlExecution( + lease: BorrowedArtifactTargetLease, + authority: VerifiedExecutionAuthority, +): void { + const s = stateOf(lease), + a = executionAuthorityState(authority); + if ( + controlDigest(a.execution) !== controlDigest(lease.execution) || + a.installation.installation.installationId !== s.certificate.installationId + ) + throw Error("Borrower execution authority mismatch"); + s.authority = a; +} +export async function settleArtifactTargetLeaseChild( + lease: TargetLeaseAuthority, + evidence: VerifiedChildSettlement, +): Promise { + const s = stateOf(lease); + const proof = childSettlementState(evidence); + if ( + proof.state !== s.installation || + controlDigest(proof.evidence.identity) !== controlDigest(s.execution) + ) + throw Error("Foreign settlement identity"); + await guarded(s, () => { + const t = tree(s); + const node = s.node === null ? t.root : t.children.get(s.node)!; + for (const grant of t.grants.values()) { + if (grant.parentNode === s.node) { + const child = t.children.get(grant.delegationId); + if (child?.phase !== "settled" || !knownOutcome(child.settlement)) + throw Error("Unsettled attempted child retains target fence"); + } + } + node.settlement = proof.evidence; + node.phase = knownOutcome(proof.evidence) ? "settled" : "recovery_required"; + node.sequence++; + writeNode(s, node); + tree(s); + }); +} +export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Promise { + const s = stateOf(lease); + if (s.node !== null) throw Error("Borrower cannot release root"); + await guarded(s, () => { + const t = tree(s); + if (t.root.phase !== "settled" || !knownOutcome(t.root.settlement)) + throw Error("Durable own-effect settlement is required"); + for (const id of t.grants.keys()) { + const child = t.children.get(id); + if (child?.phase !== "settled" || !knownOutcome(child.settlement)) + throw Error("Every attempted descendant must settle"); + } + }); + for (const id of s.certificate.serviceIds) + s.locks.removeDirectory(id, s.certificate.lockIdentities[id]!); + closeState(s); +} +export async function reconcileArtifactTargetLease(options: { + dataRoot: string; + leaseId: string; + authority: VerifiedRecoveryAuthority; +}): Promise<"released" | "fenced"> { + const recovery = recoveryAuthorityState(options.authority); + const s = openState(options.dataRoot, options.leaseId, recovery.authority.installation); + try { + if (controlDigest(s.execution) !== controlDigest(recovery.authority.execution)) + throw Error("Foreign recovery execution"); + const saved = z + .object({ + certificate: signedCertificateSchema, + privateKey: z.string(), + executionReceiptDigest: digest, + }) + .parse(readPrivateRecord(s.installation, "leases", `${options.leaseId}.json`)); + if ( + controlDigest(saved.certificate) !== controlDigest(s.certificateEnvelope) || + saved.executionReceiptDigest !== recovery.authority.receiptDigest + ) + throw Error("Recovery certificate mismatch"); + s.key = createPrivateKey({ + key: Buffer.from(saved.privateKey, "base64url"), + type: "pkcs8", + format: "der", + }); + const t = tree(s); + if ( + !knownOutcome(recovery.evidence) || + [...t.grants.keys()].some( + (id) => + t.children.get(id)?.phase !== "settled" || !knownOutcome(t.children.get(id)?.settlement), + ) + ) + return "fenced"; + await guarded(s, () => { + const root = readRoot(s); + root.settlement = recovery.evidence; + root.phase = "settled"; + root.sequence++; + writeNode(s, root); + }); + const lease = Object.freeze({ + leaseId: options.leaseId, + owner: s.execution, + serviceIds: s.certificate.serviceIds, + mode: "owner" as const, + }); + handles.set(lease, s); + await releaseArtifactTargetLease(lease); + return "released"; + } finally { + closeState(s); + } +} +export async function withArtifactTargetLease( + options: { + dataRoot: string; + serviceIds: readonly string[]; + owner: VerifiedExecutionAuthority; + inherited?: BorrowedArtifactTargetLease; + signal?: AbortSignal; + }, + run: (lease: TargetLeaseAuthority) => Promise, +): Promise { + if (options.inherited) { + const state = stateOf(options.inherited); + const requestedRoot = new AnchoredDirectory(options.dataRoot); + try { + if (!sameInode(identity(requestedRoot.stat), state.certificate.dataRootIdentity)) + throw Error("Inherited physical root mismatch"); + } finally { + requestedRoot.close(); + } + const authority = executionAuthorityState(options.owner); + if ( + controlDigest(state.execution) !== controlDigest(authority.execution) || + options.serviceIds.some((id) => !options.inherited?.serviceIds.includes(id)) + ) + throw Error("Inherited target closure cannot widen or reacquire"); + assertArtifactTargetLeaseForward(options.inherited); + const inheritedGrant = tree(state).grants.get(state.node!); + if (inheritedGrant?.recipient === "configuration-helper") + verifyConfigurationHelperReady(options.inherited); + return run(options.inherited); + } + const lease = await acquireArtifactTargetLease(options); + try { + assertArtifactTargetLeaseForward(lease); + const result = await run(lease); + await releaseArtifactTargetLease(lease); + return result; + } catch (e) { + const s = stateOf(lease); + try { + await guarded(s, () => { + const root = readRoot(s); + if (root.phase !== "settled") { + root.phase = "recovery_required"; + root.sequence++; + writeNode(s, root); + } + }); + } finally { + closeState(s); + } + throw e; + } +} + +/** Verification-only preview for fixed admission adapters. No adoption or effect authority. */ +export function inspectArtifactDelegation(options: { + dataRoot: string; + assertion: string; + installation: import("./control-authority").ControlInstallation; +}) { + const parsed = parseControlAssertion(options.assertion); + const s = openState(options.dataRoot, parsed.leaseId, installationState(options.installation)); + try { + const t = tree(s); + const grant = t.grants.get(parsed.delegationId); + if (!grant || parsed.ownerCertificateDigest !== controlDigest(s.certificateEnvelope)) + throw Error("Unanchored delegation"); + const parent = grant.parentNode ? t.grants.get(grant.parentNode)! : undefined; + const parentKey = parent?.childPublicKey ?? s.certificate.ownerPublicKey; + const { signature, ...fields } = parsed; + verifyControl( + parentKey, + "openmapx-lease-assertion-v1", + { payload: fields, signature }, + assertionSchema.omit({ signature: true }), + ); + const parentNode = grant.parentNode ? t.children.get(grant.parentNode)! : t.root; + if (parentNode.grantDigests[grant.delegationId] !== parsed.grantDigest) + throw Error("Grant digest mismatch"); + return Object.freeze({ + leaseId: parsed.leaseId, + parent: grant.parent, + execution: grant.child, + kind: grant.childKind, + serviceIds: Object.freeze([...grant.serviceIds]), + recipient: grant.recipient, + phase: grant.phase, + delegationId: grant.delegationId, + retained: !!t.children.get(grant.delegationId), + }); + } finally { + closeState(s); + } +} + +export async function cancelArtifactTargetLease(lease: TargetLeaseAuthority): Promise { + const s = stateOf(lease); + await guarded(s, () => { + const t = tree(s); + const node = s.node === null ? t.root : t.children.get(s.node)!; + if (node.phase === "settled") return; + node.phase = "cancel_pending"; + node.sequence++; + writeNode(s, node); + tree(s); + }); +} + +/** A helper can adopt before its preparer exits, but may not enter drain/writer work before terminal preparation. */ +export function verifyConfigurationHelperReady(lease: BorrowedArtifactTargetLease): void { + const s = stateOf(lease); + const t = tree(s); + const grant = t.grants.get(s.node!); + if ( + grant?.recipient !== "configuration-helper" || + grant.phase !== "configuration-handoff" || + t.root.phase !== "held" || + t.children.get(s.node!)?.phase !== "held" || + Date.now() >= grant.child.deadlineAt + ) + throw Error("Configuration helper cannot enter drain"); + verifyControlPreparedTerminal(s.installation, grant.parent); +} + +/** Observation/adoption of an old identity is not permission to start another forward effect. */ +export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): void { + const s = stateOf(lease); + const t = tree(s); + const node = s.node === null ? t.root : t.children.get(s.node); + if ( + !node || + t.root.phase !== "held" || + node.phase !== "held" || + Date.now() >= s.execution.deadlineAt + ) + throw Error("Target lease permits observation/settlement only"); +} + +/** Drop local descriptors only. Durable locks, ownership and all unknown fences remain untouched. */ +export function closeArtifactTargetLeaseHandle(lease: TargetLeaseAuthority): void { + const s = handles.get(lease); + if (!s) throw Error("Unverified target lease handle"); + closeState(s); + handles.delete(lease); +} + +/** Read-only terminal evidence after root release: never mints a reusable lease handle. */ +export function verifyRetainedArtifactTargetSettlement( + installation: import("./control-authority").ControlInstallation, + adoption: import("./control-authority").ControlAdoption, + execution: ExecutionRef, +): void { + const local = installationState(installation); + const s = openState(local.dataRoot, adoption.leaseId, local, true); + try { + const t = tree(s); + const child = t.children.get(adoption.delegationId); + const proof = childSettlementState(verifyControlEffectSettlement(installation, execution)); + if ( + !child || + child.phase !== "settled" || + !knownOutcome(child.settlement) || + controlDigest(child.execution) !== controlDigest(execution) || + controlDigest(child.settlement) !== controlDigest(proof.evidence) + ) + throw Error("Terminal child settlement is not proved"); + } finally { + closeState(s); + } +} diff --git a/packages/core/src/build-artifacts/test-fixtures/peer-process.ts b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts new file mode 100644 index 000000000..0fe64eb98 --- /dev/null +++ b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts @@ -0,0 +1,145 @@ +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import * as c from "../control-authority"; +import { inspectControlAdmission, verifiedParentForArtifactDelegation } from "../peer-admission"; +import * as l from "../target-lease"; + +for await (const line of createInterface({ input: process.stdin })) { + try { + const input = JSON.parse(line); + if (input.mode === "owner") { + const installation = c.openControlInstallation({ + infraDir: input.infraDir, + role: input.role, + serviceIds: ["motis", "osrm"], + }); + const authority = c.retainAdmittedControlExecution(installation, input.admitted); + const lease = await l.acquireArtifactTargetLease({ + dataRoot: join(input.infraDir, "data"), + serviceIds: input.admitted.serviceIds, + owner: authority, + }); + if (input.fixedId) + input.child.childPublicKey = c.prepareFixedControlChild(authority, input.fixedId); + const assertion = await l.delegateArtifactTargetLease(lease, input.child); + if (input.prepared) c.retainControlPreparedTerminal(authority); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + process.stdout.write(`${JSON.stringify({ ok: true, assertion, leaseId: lease.leaseId })}\n`); + await new Promise(() => setInterval(() => {}, 60000)); + } else if (input.mode === "helper") { + const installation = c.openFixedChildInstallation({ + infraDir: input.infraDir, + assertion: input.assertion, + }); + const peer = c.verifyFixedCliControlPeer(installation, { + assertion: input.assertion, + execution: input.execution, + kind: "configuration-helper", + recipient: "configuration-helper", + }); + const lease = await l.adoptArtifactTargetLease({ + dataRoot: join(input.infraDir, "data"), + assertion: input.assertion, + peer, + execution: input.execution, + serviceIds: ["motis"], + }); + l.verifyConfigurationHelperReady(lease); + process.stdout.write(JSON.stringify({ ok: true, drainPermitted: true }) + "\n"); + } else if (input.mode === "fixed") { + const installation = c.openFixedChildInstallation({ + infraDir: input.infraDir, + assertion: input.assertion, + }); + const intent = c.readFixedCliSpawnIntent(installation, input.execution, input.args); + const peer = c.verifyFixedCliControlPeer(installation, { + assertion: input.assertion, + execution: input.execution, + kind: intent.operation.kind, + }); + const parent = verifiedParentForArtifactDelegation( + installation, + input.assertion, + input.execution, + ); + const lease = await l.adoptArtifactTargetLease({ + dataRoot: join(input.infraDir, "data"), + assertion: input.assertion, + peer, + execution: input.execution, + serviceIds: intent.serviceIds, + }); + const authority = c.retainAdmittedControlExecution(installation, { + execution: input.execution, + kind: intent.operation.kind, + operation: intent.operation, + admittedAt: intent.admittedAt, + parent, + serviceIds: intent.serviceIds, + }); + l.bindBorrowedControlExecution(lease, authority); + if (input.action === "run") c.beginFixedCliExecution(installation, input.execution); + let assertion: string | undefined; + if (input.child) assertion = await l.delegateArtifactTargetLease(lease, input.child); + if (input.action === "settle") { + c.retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "fixed-cli-exit", args: input.args, exitCode: 0 }, + }); + await l.settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, input.execution), + ); + } + process.stdout.write(`${JSON.stringify({ ok: true, leaseId: lease.leaseId, assertion })}\n`); + } else { + const installation = c.openControlInstallation({ + infraDir: input.infraDir, + role: input.role, + serviceIds: ["motis", "osrm"], + }); + const admission = inspectControlAdmission(installation, { + rawHeaders: ["x-openmapx-control-delegation", input.assertion], + transportPrincipal: + input.transportPrincipal ?? + (input.role === "ops-agent" ? "data-manager" : "shared-data-manager-bearer"), + kind: input.kind, + operationKey: input.execution.operationKey, + operationFingerprint: input.execution.operationFingerprint, + requestedDeadlineAt: input.execution.deadlineAt, + }); + const lease = await admission.adopt(input.execution); + const authority = c.retainAdmittedControlExecution(installation, { + execution: input.execution, + kind: input.kind, + operation: input.operation, + admittedAt: input.admittedAt, + parent: admission.parent, + serviceIds: lease.serviceIds, + }); + if (input.settle) { + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await l.settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, input.execution), + ); + } + if (input.cancel) await l.cancelArtifactTargetLease(lease); + process.stdout.write( + `${JSON.stringify({ ok: true, retained: admission.retained, leaseId: lease.leaseId })}\n`, + ); + } + } catch (error) { + process.stdout.write(`${JSON.stringify({ ok: false, error: (error as Error).message })}\n`); + } + break; +} diff --git a/packages/core/src/ops/execution-identity.ts b/packages/core/src/ops/execution-identity.ts index bf252cca8..0e5b225b2 100644 --- a/packages/core/src/ops/execution-identity.ts +++ b/packages/core/src/ops/execution-identity.ts @@ -21,7 +21,7 @@ export const executionRefSchema = z ); export type ExecutionRef = z.infer; -const parentIdentitySchema = z.strictObject({ +const apiParentIdentitySchema = z.strictObject({ issuer: z.literal("api"), apiJobId: z.uuid(), apiOwnerVersion: z.number().int().positive().safe(), @@ -31,13 +31,30 @@ const parentIdentitySchema = z.strictObject({ attemptId: z.uuid(), assertionDigest: z.string().regex(/^[a-f0-9]{64}$/), }); +const roleParentIdentitySchema = z + .strictObject({ + issuer: z.enum(["ops-agent", "data-manager", "host-cli"]), + execution: executionRefSchema, + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + }) + .refine( + (p) => + p.issuer === p.execution.executor && + p.operationKey === p.execution.operationKey && + p.fingerprint === p.execution.fingerprint && + p.deadlineAt === p.execution.deadlineAt, + ); +const parentIdentitySchema = z.union([apiParentIdentitySchema, roleParentIdentitySchema]); /** Persisted identity is evidence only; parsing it does not authenticate a parent. */ export type RetainedParentExecution = z.infer; declare const verifiedParent: unique symbol; export type VerifiedParentExecution = Readonly & { readonly [verifiedParent]: true; }; -const verifiedParents = new WeakSet(); + +import { verifiedParents } from "./verified-parent-internal"; export type ExecutionAdmission = { requestedDeadlineAt?: number; parent?: VerifiedParentExecution }; export const retainedParentExecutionSchema = parentIdentitySchema; export const OPS_MAX_EXECUTION_MS = 7 * 24 * 3600_000; diff --git a/packages/core/src/ops/verified-parent-internal.ts b/packages/core/src/ops/verified-parent-internal.ts new file mode 100644 index 000000000..31d93bb42 --- /dev/null +++ b/packages/core/src/ops/verified-parent-internal.ts @@ -0,0 +1,3 @@ +// Internal mint registry shared only by the fixed API and role-signature verifiers. +// Deliberately absent from every package export. +export const verifiedParents = new WeakSet(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f0b933518..8e6299ce5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2569,6 +2569,9 @@ importers: react: specifier: 19.3.0 version: 19.3.0 + tar-stream: + specifier: 3.2.2 + version: 3.2.2 tz-lookup: specifier: ^6.1.25 version: 6.1.25 @@ -2597,6 +2600,9 @@ importers: '@types/react-dom': specifier: ^19.3.0 version: 19.3.0(@types/react@19.3.0) + '@types/tar-stream': + specifier: 3.1.4 + version: 3.1.4 esbuild: specifier: '>=0.28.1' version: 0.28.2 @@ -7079,6 +7085,9 @@ packages: '@types/stack-utils@2.0.3': resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} + '@types/tar-stream@3.1.4': + resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==} + '@types/tough-cookie@4.0.5': resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} @@ -16962,6 +16971,10 @@ snapshots: '@types/stack-utils@2.0.3': {} + '@types/tar-stream@3.1.4': + dependencies: + '@types/node': 26.6.5 + '@types/tough-cookie@4.0.5': {} '@types/trusted-types@2.0.7': {} diff --git a/services/data-manager/__tests__/auth.test.ts b/services/data-manager/__tests__/auth.test.ts index 8d9316589..026bca3e8 100644 --- a/services/data-manager/__tests__/auth.test.ts +++ b/services/data-manager/__tests__/auth.test.ts @@ -298,3 +298,32 @@ describe("resolveAuthToken", () => { expect(token).toMatch(/^[0-9a-f]{64}$/); }); }); + +it("retains the durable-receiver prerequisite for internal control headers", async () => { + const app = await buildApp(); + try { + for (const name of ["x-openmapx-control-delegation", "x-openmapx-parent-execution"]) { + for (const value of ["", "not-yet-admitted"]) { + const response = await app.inject({ + url: "/protected", + headers: { authorization: `Bearer ${TOKEN}`, [name]: value }, + }); + expect(response.statusCode).toBe(503); + } + expect( + ( + await app.inject({ + url: "/protected", + headers: { authorization: `Bearer ${TOKEN}`, [name]: "x".repeat(4097) }, + }) + ).statusCode, + ).toBe(403); + } + expect( + (await app.inject({ url: "/protected", headers: { authorization: `Bearer ${TOKEN}` } })) + .statusCode, + ).toBe(200); + } finally { + await app.close(); + } +}); diff --git a/services/data-manager/service.json b/services/data-manager/service.json index 8a5180b87..d4a7c4bc6 100644 --- a/services/data-manager/service.json +++ b/services/data-manager/service.json @@ -140,17 +140,38 @@ "startPeriod": "120s" }, "dependsOn": [ - { "service": "postgis", "condition": "service_healthy" }, - { "service": "ops-agent", "condition": "service_healthy" }, - { "service": "transitous-runner", "condition": "service_healthy" } + { + "service": "postgis", + "condition": "service_healthy" + }, + { + "service": "ops-agent", + "condition": "service_healthy" + }, + { + "service": "transitous-runner", + "condition": "service_healthy" + } ] }, "provides": ["osm-data", "gtfs-data", "tile-asset-data"], "produces": [ - { "type": "osm-pbf", "sourceDir": "data/osm" }, - { "type": "osm-pbf-bz2", "sourceDir": "data/osm-bz2" }, - { "type": "gtfs", "sourceDir": "data/gtfs" }, - { "type": "tile-fonts", "sourceDir": "data/tile-fonts" } + { + "type": "osm-pbf", + "sourceDir": "data/osm" + }, + { + "type": "osm-pbf-bz2", + "sourceDir": "data/osm-bz2" + }, + { + "type": "gtfs", + "sourceDir": "data/gtfs" + }, + { + "type": "tile-fonts", + "sourceDir": "data/tile-fonts" + } ], "bindMounts": [ { @@ -163,7 +184,11 @@ "target": "/run/secrets/transitous-runner-capability", "readOnly": true }, - { "source": "@infra:data", "target": "/data", "readOnly": false }, + { + "source": "@infra:data", + "target": "/data", + "readOnly": false + }, { "source": "@service:motis:tools/transitous", "target": "/config/transitous", @@ -174,10 +199,33 @@ "target": "/secrets/transitous-feed-proxy.age", "readOnly": true, "optional": true + }, + { + "source": "@infra:secrets/control-data-manager.key", + "target": "/run/secrets/openmapx-control-role-key", + "readOnly": true + }, + { + "source": "@infra:secrets/control-verifiers.json", + "target": "/run/secrets/openmapx-control-verifiers", + "readOnly": true + }, + { + "source": "@infra:control-private/data-manager", + "target": "/var/lib/openmapx/control-private", + "readOnly": false } ], "exposure": { - "hostPorts": [{ "container": 4000, "host": 4000, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 4000, + "host": 4000, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + } } diff --git a/services/data-manager/src/auth.ts b/services/data-manager/src/auth.ts index 6b0882e69..adb8858e2 100644 --- a/services/data-manager/src/auth.ts +++ b/services/data-manager/src/auth.ts @@ -1,5 +1,6 @@ import { randomBytes, timingSafeEqual } from "node:crypto"; import rateLimit, { normalizeIP } from "@fastify/rate-limit"; +import { controlHeaders } from "@openmapx/core/build-artifacts"; import { assertProductionApplicationSecret } from "@openmapx/core/deployment-secret-policy"; import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; @@ -99,5 +100,15 @@ export function registerAuth(app: FastifyInstance, expectedToken: string): void .send({ error: "Too many authentication attempts" }); return reply.code(401).send({ error: "Unauthorized" }); } + // Task02.3 owns durable receiver admission. Never silently ignore an internal authority header. + try { + const headers = controlHeaders(req.raw.rawHeaders); + if (headers.delegation !== undefined || headers.parent !== undefined) + return reply + .code(503) + .send({ error: "Managed data-manager execution requires the durable receiver upgrade" }); + } catch { + return reply.code(403).send({ error: "Invalid control authority" }); + } }); } diff --git a/services/data-manager/src/service.ts b/services/data-manager/src/service.ts index 2c7346050..d0f2fe71f 100644 --- a/services/data-manager/src/service.ts +++ b/services/data-manager/src/service.ts @@ -1,5 +1,6 @@ import { accessSync, constants } from "node:fs"; import { join } from "node:path"; +import { loadControlStartup } from "@openmapx/core/build-artifacts"; import { createFatalProcessHandler } from "@openmapx/core/server"; import Fastify from "fastify"; import { registerApi } from "./api.js"; @@ -29,6 +30,12 @@ import { StateStore } from "./state.js"; const app = Fastify({ loggerInstance: rootLogger }); registerAuth(app, resolveAuthToken(app)); const readiness = new DataManagerReadiness(); +const controlStartup = loadControlStartup({ + infraDir: "/data", + role: "data-manager", + container: true, +}); +if (controlStartup.reason) app.log.warn(controlStartup.reason); const dataDir = process.env.DATA_DIR ?? "/data"; const offlinePackageStorage = new OfflinePackageStorage(join(dataDir, "offline-packages")); diff --git a/services/ops-agent/service.json b/services/ops-agent/service.json index c0634011a..53a6e0090 100644 --- a/services/ops-agent/service.json +++ b/services/ops-agent/service.json @@ -77,7 +77,11 @@ "target": "/run/secrets/privacy-backup-capability-key", "readOnly": true }, - { "source": "@docker-socket", "target": "/var/run/docker.sock", "readOnly": false }, + { + "source": "@docker-socket", + "target": "/var/run/docker.sock", + "readOnly": false + }, { "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set — ops-agent renders and applies trusted local stack configuration}", "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}", @@ -87,7 +91,44 @@ "source": "${OPENMAPX_DOCKER_CONFIG_DIR:-/root/.docker}", "target": "/home/node/.docker", "readOnly": true + }, + { + "source": "@infra:secrets/control-ops-agent.key", + "target": "/run/secrets/openmapx-control-role-key", + "readOnly": true + }, + { + "source": "@infra:secrets/control-verifiers.json", + "target": "/run/secrets/openmapx-control-verifiers", + "readOnly": true + }, + { + "source": "@infra:control-private/ops-agent", + "target": "/var/lib/openmapx/control-private", + "readOnly": false + }, + { + "source": "@infra:control-private/hidden-key", + "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/secrets/control-data-manager.key", + "readOnly": true + }, + { + "source": "@infra:control-private/empty", + "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/control-private/data-manager", + "readOnly": true + }, + { + "source": "@infra:control-private/hidden-key", + "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/secrets/control-host-cli.key", + "readOnly": true + }, + { + "source": "@infra:control-private/empty", + "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/control-private/host-cli", + "readOnly": true } ], - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + } } From 0f14ec06cf151549c1a2bb2dc8ef7ce4b5e86de3 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 08:15:08 +0100 Subject: [PATCH 011/135] fix(ops): enforce retained control authority invariants --- apps/ops-agent/src/control-admission.test.ts | 24 +- apps/ops-agent/src/server.ts | 19 +- .../docs/developer/build-artifacts-control.md | 72 ++- packages/cli/src/lib/control-context.ts | 3 +- .../build-artifacts/control-authority.test.ts | 231 ++++++++ .../src/build-artifacts/control-authority.ts | 524 +++++++++++++++--- .../src/build-artifacts/control-retention.ts | 74 +++ .../core/src/build-artifacts/filesystem.ts | 52 +- packages/core/src/build-artifacts/index.ts | 3 + .../src/build-artifacts/peer-admission.ts | 30 + .../src/build-artifacts/peer-process.test.ts | 300 ++++++---- .../src/build-artifacts/target-lease.test.ts | 148 +++++ .../core/src/build-artifacts/target-lease.ts | 315 ++++++++--- .../test-fixtures/peer-process.ts | 94 +++- 14 files changed, 1604 insertions(+), 285 deletions(-) create mode 100644 packages/core/src/build-artifacts/control-retention.ts diff --git a/apps/ops-agent/src/control-admission.test.ts b/apps/ops-agent/src/control-admission.test.ts index 6a12c3d0d..fd195986d 100644 --- a/apps/ops-agent/src/control-admission.test.ts +++ b/apps/ops-agent/src/control-admission.test.ts @@ -12,21 +12,23 @@ import { expect, it } from "vitest"; import { type OpsJobJournal, openOpsJobJournal } from "./journal"; import { buildOpsAgentServer } from "./server"; -it.each([false, true])( - "adopts at authenticated HTTP admission without replay (ready-write failure: %s)", - async (failReadyWrite) => { +it.each(["normal", "ready-failure", "wrong-target"])( + "adopts at authenticated HTTP admission without replay (%s)", + async (scenario) => { + const failReadyWrite = scenario === "ready-failure"; + const serviceIds = scenario === "wrong-target" ? ["osrm"] : ["motis"]; const root = mkdtempSync(join(tmpdir(), "omx-control-http-")); mkdirSync(join(root, "data"), { mode: 0o700 }); await control.provisionControlInstallation(root); const dm = control.openControlInstallation({ infraDir: root, role: "data-manager", - serviceIds: ["motis"], + serviceIds: ["motis", "osrm"], }); const agent = control.openControlInstallation({ infraDir: root, role: "ops-agent", - serviceIds: ["motis"], + serviceIds: ["motis", "osrm"], }); const operation = { kind: "transit-sync" }; const hash = opsOperationFingerprint({ kind: "motis.primary.stop" }); @@ -66,7 +68,7 @@ it.each([false, true])( ordinal: 0, kind: "motis.primary.stop", execution: child, - serviceIds: ["motis"], + serviceIds, recipient: "ops-agent" as const, }; const authority = control.retainAdmittedControlExecution(dm, { @@ -74,12 +76,12 @@ it.each([false, true])( kind: operation.kind, operation, admittedAt: deadlineAt - 60000, - serviceIds: ["motis"], + serviceIds, plan: [plan], }); const lease = await control.acquireArtifactTargetLease({ dataRoot: join(root, "data"), - serviceIds: ["motis"], + serviceIds, owner: authority, }); const assertion = await control.delegateArtifactTargetLease(lease, { @@ -129,6 +131,12 @@ it.each([false, true])( }); try { const result = await request(); + if (scenario === "wrong-target") { + expect(result.statusCode, result.body).toBe(403); + expect(calls).toBe(0); + expect(journal.records()).toHaveLength(0); + return; + } if (failReadyWrite) { expect(result.statusCode).toBe(500); expect(calls).toBe(0); diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 1ca5054a9..0b58131f1 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -541,6 +541,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const authority = controlAuthorities.get(job.operationId); if (lease && state === "termination_pending") await cancelArtifactTargetLease(lease); if (lease && authority && ["succeeded", "failed", "timed_out"].includes(state)) { + if (!options.controlInstallation) throw Error("Missing control settlement installation"); // The fixed runtime's validated result is own-effect evidence. Failure alone is not containment. const committed = state === "succeeded" && updates.result !== undefined; retainControlEffectReceipt(authority, { @@ -552,7 +553,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }); await settleArtifactTargetLeaseChild( lease, - verifyControlEffectSettlement(options.controlInstallation!, authority.execution), + verifyControlEffectSettlement(options.controlInstallation, authority.execution), ); } const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; @@ -1023,6 +1024,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif rawHeaders: ["x-openmapx-control-delegation", assertion], transportPrincipal: "data-manager", kind: job.operation.kind, + operation: job.operation, operationKey: job.operationKey, operationFingerprint: execution.operationFingerprint, requestedDeadlineAt: execution.deadlineAt, @@ -1086,6 +1088,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif rawHeaders: request.raw.rawHeaders, transportPrincipal: "data-manager", kind, + operation, operationKey: parsed.operationKey, operationFingerprint, requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, @@ -1306,24 +1309,30 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif jobsById.set(job.operationId, job); jobsByKey.set(ownerKey, job); if (controlAdmission) { + const adoption = { + leaseId: controlAdmission.leaseId, + delegationId: controlAdmission.delegationId, + }; try { + if (!options.controlInstallation || job.admittedAt === undefined) + throw Error("Missing durable control admission context"); const lease = await controlAdmission.adopt(executionFor(job)); - const authority = retainAdmittedControlExecution(options.controlInstallation!, { + const authority = retainAdmittedControlExecution(options.controlInstallation, { execution: executionFor(job), kind: job.operation.kind, operation: job.operation, - admittedAt: job.admittedAt!, + admittedAt: job.admittedAt, parent: job.parent, serviceIds: lease.serviceIds, }); controlLeases.set(job.operationId, lease); controlAuthorities.set(job.operationId, authority); - job.controlAdoption = { ...job.controlAdoption!, phase: "ready" }; + job.controlAdoption = { ...adoption, phase: "ready" }; await journal.replace([...jobsById.values()].map(persisted)); } catch (error) { // The receiver may already have adopted. This identity remains // observable as recovery-required and is never dispatched again. - job.controlAdoption = { ...job.controlAdoption!, phase: "recovery_required" }; + job.controlAdoption = { ...adoption, phase: "recovery_required" }; forceRecoveryRequired(job); try { await journal.replace([...jobsById.values()].map(persisted)); diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index 549b13bdf..0e4918220 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -66,6 +66,17 @@ records, never paths or private keys. Each recipient independently verifies the installation, role, complete chain, current tree, immutable identity, subset and absolute deadline. Per-lease journal guards and private sequence high-water marks reject conflicting writers and shared-record rollback. +Inspection and forward checks acquire that same per-lease guard before reading +and publishing their private high-water observations. A busy/crashed writer fails +closed; read-only inspection cannot overwrite a newer process's observation. + +The data-manager-to-agent HTTP receiver derives the required targets from the +parsed operation: primary stop/restart/promotion require `motis`, staging +stop/restart require `motis-staging`, and proxy validation/reload requires +`motis-feed-proxy`. Restart does not start dependencies and promotion uses +`--no-deps`. The signed grant must contain those targets, independently of its +body fingerprint and resource policy. Other operation families remain unavailable +through this receiver until their fixed manifest/generation resolver exists. Use `acquireArtifactTargetLease`, `delegateArtifactTargetLease`, `adoptArtifactTargetLease`, `settleArtifactTargetLeaseChild`, @@ -75,7 +86,7 @@ second root. `withArtifactTargetLease` checks forward permission and retains fen on failure; its callback must produce durable own-effect evidence before release. Acquisition has one five-second budget. The limits are 256 registered targets, 64 child intents, 128 KiB per lease ledger and 256 unsettled leases. Settled -receipts remain retained; this foundation performs no age-based garbage collection. +receipts remain retained under the capacity policy below. Host ordering is target lease, nonblocking transit reservation, configuration writer, then release store. A busy reservation must settle an unused owner without @@ -89,6 +100,65 @@ use the original identity/deadline. Fixed CLI spawn intents bind exact command arguments and a fresh private child key; entry consumes a one-use start marker before adoption can lead to effects. Unrelated subprocesses receive no authority variables. Deadline-only Full Update forwarding owns no target lease. +Its separate signed lifecycle binds the original execution, monotonic sequence, +phase and at most 64 child-intent digests. Each transition persists its private +fence before shared publication under the execution writer guard. Terminal effect +receipts fence new grants; `cancelControlExecution` persists cancellation and +returns the original attempted identities for the future runtime's authenticated +cancellation/lookup adapter. `assertControlExecutionForward` and +`beginControlEffect` validate the complete deadline-only ancestry. New adoption +and effect-start intent share the lifecycle guard with cancellation. Exact +already-adopted duplicates remain usable for observation/settlement after cancel, +terminal state or expiry, and do not renew forward permission. +Acquiring a child's independent target lease and forward use of its bound handle +also check this lifecycle. Terminal maintenance/handoff preparation has only its +explicit frozen delegation exception: current signed terminal preparation, +unexpired identity, original admitted plan and held lease ancestry are all still +required. It cannot restart ordinary effects. + +Target borrowers also check every intermediate ancestor. Canceling an intermediate +borrower fences its retained descendants and returns their original attempted +identities without deleting or settling any descendant evidence. + +### Authority storage capacity and retirement + +`CONTROL_RETENTION_LIMITS` fixes the installation lifetime policy: + +- At most 1,024 admitted executions per role recovery root, including fixed + children sharing the agent root. A signed immutable admission reserves capacity + before witness publication or effects. Existing identities and settlement remain + accessible when admission is full. +- At most 1,024 lease histories across the physical installation and 256 active + leases. Exclusive fixed admission slots reserve both before target acquisition; + only authoritative root release removes an active slot. Terminal histories and + partial published histories keep their retained slot. Acquisition starts its + five-second budget before admission work and probes at most 1,024 history slots + plus 256 active slots, without listing or opening historical ledgers. +- Exactly 18 fixed private receipt namespaces, each capped at 65,536 records + (1,024 executions times 64 intents); the execution namespace has the stricter + 1,024 limit. Each private record is at most 128 KiB, so even the conservative + aggregate private record ceiling is finite, below 145 GiB per role, excluding + filesystem overhead. Actual typed key/admission/effect records have tighter + schema limits. Durable private counters reserve creation before publication; + bounded streaming enumeration is needed only to initialize a missing counter. + Failed reservations consume capacity rather than allowing undercount/reuse. +- Fixed-child keys use a direct public-key-digest index plus exact challenge read, + never a historical key-directory scan. Each execution may allocate at most 64 + keys and no more than its admitted plan length. Shared witnesses are bounded by + the admitted execution count: immutable witness, lifecycle and attempted-child + map each at most 64 KiB, plus at most 4 KiB of key identities and a bounded + prepared witness. Lease ledgers remain capped at 128 KiB/64 children. + +There is **no online retirement** of terminal records or key/identity slots. +Terminal records therefore survive at least 72 hours; unresolved records survive +regardless of age, deadline or PID. This deliberately trades lifetime throughput +for exact one-use/replay evidence: exhaustion refuses new work and requires +supervised offline capacity planning/recovery after all relevant writers/helpers +stop. No constructor resets counters, deletes aged evidence or rotates trust to +make room. Automatic bounded retirement with sustained throughput is not supplied +by this foundation. Older lease layouts without the admission index require +supervised indexing; startup never silently excludes their fences. A crashed +metadata guard likewise requires supervised recovery. Settlement requires signed retained structured evidence from the fixed executor: a parsed typed result, exact fixed-command completion, or an execution that has diff --git a/packages/cli/src/lib/control-context.ts b/packages/cli/src/lib/control-context.ts index a5c7523f9..5279f0ab6 100644 --- a/packages/cli/src/lib/control-context.ts +++ b/packages/cli/src/lib/control-context.ts @@ -52,7 +52,8 @@ export async function withCliControlContext( if (!encoded || Buffer.byteLength(encoded) > 2048) throw Error("Missing bounded fixed CLI execution"); const execution = executionRefSchema.parse(JSON.parse(encoded)); - const assertion = (delegation ?? parentAssertion)!; + const assertion = delegation ?? parentAssertion; + if (!assertion) throw Error("Missing fixed CLI control assertion"); if (assertion.length > 4096) throw Error("Fixed CLI assertion exceeds header budget"); const infraDir = repoPaths().infraDir; const installation = openFixedChildInstallation({ diff --git a/packages/core/src/build-artifacts/control-authority.test.ts b/packages/core/src/build-artifacts/control-authority.test.ts index 06ba11fd4..d07c38e73 100644 --- a/packages/core/src/build-artifacts/control-authority.test.ts +++ b/packages/core/src/build-artifacts/control-authority.test.ts @@ -302,6 +302,14 @@ it("verifies the certified ephemeral fixed-CLI deadline chain without exporting }); const id = randomUUID(); const publicKey = c.prepareFixedControlChild(authority, id); + // Unrelated retained keys must never be enumerated or parsed to find this + // exact certified challenge. These unreadable historical fixtures expose a scan. + for (let i = 0; i < 128; i++) + writeFileSync( + join(root, `control-private/ops-agent/children/retained-${i}.key`), + "historical", + { mode: 0o600 }, + ); const assertion = await c.attestParentExecution(authority, { ...planned, childPublicKey: publicKey, @@ -353,7 +361,230 @@ it("verifies the certified ephemeral fixed-CLI deadline chain without exporting requestedDeadlineAt: deadlineAt, }), ).toMatchObject({ operationKey: cli.operationKey }); + c.cancelControlExecution(authority); + expect(() => c.assertControlExecutionForward(cliAuthority)).toThrow(/ancestor|observation/); + expect( + c.verifyParentExecution(next, dmPeer, { + execution: child, + kind: body.kind, + requestedDeadlineAt: deadlineAt, + }), + ).toMatchObject({ operationKey: cli.operationKey }); c.closeControlInstallation(fixed); c.closeControlInstallation(dm); c.closeControlInstallation(agent); }); + +async function deadlineFamily() { + const c = await api(); + const { randomUUID } = await import("node:crypto"); + const { executionFingerprint } = await import("../ops/execution-identity"); + const { opsOperationFingerprint } = await import("../ops/fingerprint"); + await c.provisionControlInstallation(root); + const agent = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["osrm"], + }); + const dm = c.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["osrm"], + }); + const operation = { kind: "data.update" as const }; + const deadlineAt = Date.now() + 60000; + const operationFingerprint = opsOperationFingerprint(operation); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const hash = c.controlDigest({ kind: "download-osm", regionId: "europe/germany" }); + const child = { + executor: "data-manager" as const, + principal: "data-manager" as const, + executionId: randomUUID(), + operationKey: c.controlChildOperationKey(execution.operationKey, "osm", 0), + operationFingerprint: hash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: "download-osm", + operationFingerprint: hash, + deadlineAt, + parent: c.retainedControlParent(execution), + }), + }; + const plan = { + phase: "osm" as const, + ordinal: 0, + kind: "download-osm", + execution: child, + serviceIds: ["osrm"], + recipient: "data-manager" as const, + }; + const authority = c.retainAdmittedControlExecution(agent, { + execution, + operation, + kind: operation.kind, + admittedAt: deadlineAt - 60000, + serviceIds: ["osrm"], + plan: [plan], + }); + const grant = () => c.attestParentExecution(authority, { ...plan, childPublicKey: dm.publicKey }); + const admit = (assertion: string) => + c.verifyParentExecution( + assertion, + c.verifyControlPeer(dm, { + rawHeaders: ["x-openmapx-parent-execution", assertion], + transportPrincipal: "shared-data-manager-bearer", + execution: child, + kind: plan.kind, + }), + { execution: child, kind: plan.kind, requestedDeadlineAt: deadlineAt }, + ); + return { c, agent, dm, authority, execution, child, grant, admit }; +} +it("terminal deadline-only parent cannot mint an unattempted child", async () => { + const f = await deadlineFamily(); + try { + f.c.retainControlEffectReceipt(f.authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await expect(f.grant()).rejects.toThrow(/terminal|settled|forward/); + } finally { + f.c.closeControlInstallation(f.dm); + f.c.closeControlInstallation(f.agent); + } +}); +it("terminal deadline-only parent fences a previously attempted unadopted grant", async () => { + const f = await deadlineFamily(); + try { + const assertion = await f.grant(); + f.c.retainControlEffectReceipt(f.authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + expect(() => f.admit(assertion)).toThrow(/terminal|observation/); + expect(existsSync(join(root, "data/.admin-target-control/locks"))).toBe(false); + } finally { + f.c.closeControlInstallation(f.dm); + f.c.closeControlInstallation(f.agent); + } +}); +it.each([false, true])( + "canceled deadline-only parent fences unadopted attempts (attempted: %s)", + async (attempted) => { + const f = await deadlineFamily(); + try { + const assertion = attempted ? await f.grant() : undefined; + const children = f.c.cancelControlExecution(f.authority); + expect(children).toHaveLength(attempted ? 1 : 0); + if (assertion) expect(() => f.admit(assertion)).toThrow(/cancel|forward|observation/); + else await expect(f.grant()).rejects.toThrow(/cancel|forward/); + expect(() => f.c.beginControlEffect(f.authority)).toThrow(/cancel|forward/); + } finally { + f.c.closeControlInstallation(f.dm); + f.c.closeControlInstallation(f.agent); + } + }, +); +it("retained deadline-only duplicates observe after cancellation but cannot start work", async () => { + const f = await deadlineFamily(); + try { + const assertion = await f.grant(); + const parent = f.admit(assertion); + f.c.cancelControlExecution(f.authority); + expect(f.admit(assertion)).toEqual(parent); + const body = { kind: "download-osm", regionId: "europe/germany" }; + const authority = f.c.retainAdmittedControlExecution(f.dm, { + execution: f.child, + operation: body, + kind: body.kind, + admittedAt: f.execution.deadlineAt - 60000, + parent, + serviceIds: ["osrm"], + }); + expect(() => f.c.beginControlEffect(authority)).toThrow(/cancel|forward|observation/); + const { acquireArtifactTargetLease } = await import("./target-lease"); + await expect( + acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }), + ).rejects.toThrow(/cancel|ancestor|observation/); + } finally { + f.c.closeControlInstallation(f.dm); + f.c.closeControlInstallation(f.agent); + } +}); + +it("refuses exhausted private admission capacity before publishing a witness or effect", async () => { + const f = await deadlineFamily(); + const { CONTROL_RETENTION_LIMITS } = await import("./control-retention"); + try { + // A durable consumed-reservation fixture, not a mocked authorization decision. + const count = join(root, "control-private/ops-agent/executions/.retention-count"); + writeFileSync(count, String(CONTROL_RETENTION_LIMITS.executionsPerRole), { mode: 0o600 }); + const replacement = { ...f.execution, operationKey: `opk1_${"z".repeat(32)}` }; + expect(() => + f.c.retainAdmittedControlExecution(f.agent, { + execution: replacement, + operation: { kind: "data.update" }, + kind: "data.update", + admittedAt: replacement.deadlineAt - 60000, + serviceIds: ["osrm"], + }), + ).toThrow(/capacity/); + expect( + existsSync( + join( + root, + `data/.admin-target-control/executions/ops-agent/${replacement.operationKey}.json`, + ), + ), + ).toBe(false); + // Capacity is admission-only: existing exact identity and settlement stay available. + f.c.retainControlEffectReceipt(f.authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + expect(() => f.c.verifyControlEffectSettlement(f.agent, f.execution)).not.toThrow(); + } finally { + f.c.closeControlInstallation(f.dm); + f.c.closeControlInstallation(f.agent); + } +}); +it("expired exact deadline-only adoption stays observable without renewing one-use work", async () => { + const f = await deadlineFamily(); + const { vi } = await import("vitest"); + try { + const assertion = await f.grant(); + const admitted = f.admit(assertion); + const clock = vi + .spyOn(Date, "now") + .mockReturnValue(f.execution.deadlineAt + 96 * 60 * 60 * 1000); + try { + expect(f.admit(assertion)).toEqual(admitted); + await expect(f.grant()).rejects.toThrow(/Expired/); + expect( + existsSync( + join(root, `control-private/ops-agent/executions/${f.execution.operationKey}.json`), + ), + ).toBe(true); + } finally { + clock.mockRestore(); + } + } finally { + f.c.closeControlInstallation(f.dm); + f.c.closeControlInstallation(f.agent); + } +}); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 3031c1058..d24b74494 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -22,6 +22,7 @@ import { } from "../ops/execution-identity"; import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; import { verifiedParents } from "../ops/verified-parent-internal"; +import { CONTROL_RETENTION_LIMITS, writeRetainedRecord } from "./control-retention"; import { AnchoredDirectory, identity } from "./filesystem"; export const CONTROL_ROLES = ["ops-agent", "data-manager", "host-cli"] as const; export type ControlRecipient = "fixed-cli" | "data-manager" | "ops-agent" | "configuration-helper"; @@ -99,7 +100,10 @@ export const targetSchema = z .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) .min(1) .max(256) - .refine((x) => x.every((v, i) => i === 0 || x[i - 1]! < v), "Targets must be sorted and unique"); + .refine( + (x) => x.every((v, i) => i === 0 || (x[i - 1] ?? v) < v), + "Targets must be sorted and unique", + ); export function controlDigest(value: unknown): string { return createHash("sha256").update(canonicalJson(value)).digest("hex"); } @@ -449,7 +453,38 @@ export function writePrivateRecord( try { const bytes = canonicalJson(value); if (Buffer.byteLength(bytes) > 131072) throw Error("Private receipt limit"); - dir.write(name, bytes, exclusive); + if ( + ![ + "executions", + "children", + "child-keys", + "effects", + "effect-started", + "observations", + "leases", + "execution-leases", + "attempts", + "adoptions", + "parent-attempts", + "parent-adoptions", + "execution-parents", + "execution-lifecycle", + "lifecycle-observations", + "spawns", + "spawn-started", + "prepared", + ].includes(namespace) + ) + throw Error("Unknown fixed authority namespace"); + writeRetainedRecord( + dir, + name, + bytes, + exclusive, + namespace === "executions" + ? CONTROL_RETENTION_LIMITS.executionsPerRole + : CONTROL_RETENTION_LIMITS.recordsPerNamespace, + ); } finally { dir.close(); } @@ -589,6 +624,7 @@ export function retainAdmittedControlExecution( if (Buffer.byteLength(canonicalJson(receipt)) > 65536) throw Error("Execution authority receipt capacity"); const name = `${execution.operationKey}.json`; + let newlyAdmitted = false; try { const old = readPrivateRecord(state, "executions", name, 65536); if (controlDigest(old) !== controlDigest(receipt)) @@ -596,12 +632,14 @@ export function retainAdmittedControlExecution( } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; writePrivateRecord(state, "executions", name, receipt, true); + newlyAdmitted = true; } publishControlExecutionWitness( state, { execution, kind: input.kind, serviceIds }, controlDigest(receipt), ); + initializeLifecycle(state, execution, newlyAdmitted); const authority = Object.freeze({ execution }) as VerifiedExecutionAuthority; authorities.set(authority, { installation: state, @@ -662,7 +700,9 @@ export function controlHeaders(rawHeaders: readonly string[]): { if (!["x-openmapx-control-delegation", "x-openmapx-parent-execution"].includes(name)) continue; if (found.has(name) || typeof rawHeaders[i + 1] !== "string") throw Error("Duplicate control header"); - found.set(name, rawHeaders[i + 1]!); + const value = rawHeaders[i + 1]; + if (value === undefined) throw Error("Missing control header value"); + found.set(name, value); } const delegation = found.get("x-openmapx-control-delegation"), parent = found.get("x-openmapx-parent-execution"); @@ -754,6 +794,21 @@ export function prepareFixedControlChild( const s = executionAuthorityState(authority); if (s.installation.installation.role !== "ops-agent") throw Error("Only agent prepares fixed children"); + withExecutionWriter(s.installation, s.execution, (dir) => { + const name = `${s.execution.operationKey}.keys.json`; + let ids: string[] = []; + try { + ids = z + .array(uuid) + .max(64) + .parse(JSON.parse(dir.read(name, 4096).toString("utf8"))); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (ids.length >= Math.min(64, s.plan.length) || ids.includes(delegationId)) + throw Error("Fixed child key capacity/replay"); + dir.write(name, canonicalJson([...ids, delegationId])); + }); const key = newControlKey(); writePrivateRecord( s.installation, @@ -762,8 +817,31 @@ export function prepareFixedControlChild( { publicKey: key.publicKey, privateKey: encodePrivate(key.privateKey) }, true, ); + writePrivateRecord( + s.installation, + "child-keys", + `${controlDigest(key.publicKey)}.json`, + { delegationId, publicKey: key.publicKey, execution: s.execution }, + true, + ); return key.publicKey; } +export function fixedControlChildId( + state: InstallationState, + key: string, + parent: ExecutionRef, +): string { + const saved = z + .strictObject({ delegationId: uuid, publicKey, execution: executionRefSchema }) + .parse(readPrivateRecord(state, "child-keys", `${controlDigest(key)}.json`, 2048)); + if (saved.publicKey !== key || controlDigest(saved.execution) !== controlDigest(parent)) + throw Error("Fixed child key index mismatch"); + const challenge = z + .object({ publicKey }) + .parse(readPrivateRecord(state, "children", `${saved.delegationId}.key`, 2048)); + if (challenge.publicKey !== key) throw Error("Fixed child key challenge mismatch"); + return saved.delegationId; +} export function verifyFixedCliControlPeer( installation: ControlInstallation, input: { @@ -847,21 +925,23 @@ function withPrivateEffectWriter(s: AuthorityState, write: () => void): void { } export function beginControlEffect(authority: VerifiedExecutionAuthority): void { const s = executionAuthorityState(authority); - withPrivateEffectWriter(s, () => { - try { - readPrivateRecord(s.installation, "effects", `${s.execution.operationKey}.json`); - throw Error("Settled execution cannot restart effects"); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - writePrivateRecord( - s.installation, - "effect-started", - `${s.execution.operationKey}.json`, - { execution: s.execution, receiptDigest: s.receiptDigest }, - true, - ); - }); + withControlExecutionForward(authority, () => + withPrivateEffectWriter(s, () => { + try { + readPrivateRecord(s.installation, "effects", `${s.execution.operationKey}.json`); + throw Error("Settled execution cannot restart effects"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + writePrivateRecord( + s.installation, + "effect-started", + `${s.execution.operationKey}.json`, + { execution: s.execution, receiptDigest: s.receiptDigest }, + true, + ); + }), + ); } /** Fixed executor adapters supply their parsed result or exact fixed-command exit; arbitrary text is never proof. */ export function retainControlEffectReceipt( @@ -906,6 +986,7 @@ export function retainControlEffectReceipt( } if (effect.outcome !== outcome || effect.publication !== publication) throw Error("Effect evidence does not prove requested settlement"); + settleExecutionLifecycle(s); const payload = { identity: s.execution, outcome, @@ -1057,6 +1138,297 @@ function executionWitnessDirectory( }, }; } +const lifecycleSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + execution: executionRefSchema, + sequence: z.number().int().nonnegative().safe(), + phase: z.enum(["held", "cancel_pending", "recovery_required", "settled"]), + childIntents: z.record(uuid, digest), +}); +type ExecutionLifecycle = z.infer; +function withExecutionWriter( + state: InstallationState, + execution: ExecutionRef, + run: (dir: AnchoredDirectory) => T, +): T { + const d = executionWitnessDirectory(state, execution.executor, true); + const name = `${execution.operationKey}.writer`; + let guard: AnchoredDirectory | undefined; + try { + mkdirSync(d.role.name(name), { mode: 0o700 }); + fsyncSync(d.role.fd); + guard = d.role.directory(name); + return run(d.role); + } finally { + if (guard) { + const held = identity(guard.stat); + guard.close(); + d.role.removeDirectory(name, held); + } + d.close(); + } +} +function readLifecycle( + state: InstallationState, + dir: AnchoredDirectory, + execution: ExecutionRef, + verifier: string, +): ExecutionLifecycle { + const raw = JSON.parse( + dir.read(`${execution.operationKey}.lifecycle.json`, 65536).toString("utf8"), + ); + const current = verifyControl(verifier, "openmapx-execution-lifecycle-v1", raw, lifecycleSchema); + if ( + current.installationId !== state.installation.installationId || + controlDigest(current.execution) !== controlDigest(execution) || + Object.keys(current.childIntents).length > 64 + ) + throw Error("Execution lifecycle identity/capacity mismatch"); + const observationName = `${execution.executor}-${execution.operationKey}.json`; + function compare(old: ExecutionLifecycle) { + if ( + current.sequence < old.sequence || + (current.sequence === old.sequence && controlDigest(current) !== controlDigest(old)) + ) + throw Error("Execution lifecycle rollback or replacement"); + } + if (verifier === state.installation.publicKey) { + const local = verifyControl( + verifier, + "openmapx-execution-lifecycle-v1", + readPrivateRecord(state, "execution-lifecycle", `${execution.operationKey}.json`, 65536), + lifecycleSchema, + ); + compare(local); + } + try { + compare( + lifecycleSchema.parse( + readPrivateRecord(state, "lifecycle-observations", observationName, 65536), + ), + ); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + writePrivateRecord(state, "lifecycle-observations", observationName, current); + return current; +} +function writeLifecycle( + state: InstallationState, + dir: AnchoredDirectory, + lifecycle: ExecutionLifecycle, + exclusive = false, +) { + const signed = signControl(state.privateKey, "openmapx-execution-lifecycle-v1", lifecycle); + const bytes = canonicalJson(signed); + if (Buffer.byteLength(bytes) > 65536) throw Error("Execution lifecycle capacity"); + // Persist the private fence first. A crash before shared publication fails closed. + writePrivateRecord( + state, + "execution-lifecycle", + `${lifecycle.execution.operationKey}.json`, + signed, + exclusive, + ); + dir.write(`${lifecycle.execution.operationKey}.lifecycle.json`, bytes, exclusive); + readLifecycle(state, dir, lifecycle.execution, state.installation.publicKey); +} +function initializeLifecycle( + state: InstallationState, + execution: ExecutionRef, + newlyAdmitted: boolean, +) { + withExecutionWriter(state, execution, (dir) => { + try { + readLifecycle(state, dir, execution, state.installation.publicKey); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + if (!newlyAdmitted) + throw Error("Missing retained execution lifecycle requires supervised recovery"); + // Never replace a private fence after partial publication. + try { + readPrivateRecord(state, "execution-lifecycle", `${execution.operationKey}.json`); + throw Error("Incomplete execution lifecycle requires supervised recovery"); + } catch (missing) { + if ((missing as NodeJS.ErrnoException).code !== "ENOENT") throw missing; + } + writeLifecycle( + state, + dir, + { + version: 1, + installationId: state.installation.installationId, + execution, + sequence: 0, + phase: "held", + childIntents: {}, + }, + true, + ); + } + }); +} +function withParentLifecycles( + state: InstallationState, + assertion: string, + run: (forward: boolean) => T, +): T { + const chain: Array<{ + assertion: string; + grant: z.infer; + verifier: string; + }> = []; + function collect(value: string) { + const grant = readParentGrant(state, value); + if (grant.parentAssertion) collect(grant.parentAssertion); + const verifier = grant.parentAssertion + ? readParentGrant(state, grant.parentAssertion).childPublicKey + : state.publicKeys[grant.parent.executor]; + chain.push({ assertion: value, grant, verifier }); + } + collect(assertion); + function lock(index: number, forward: boolean): T { + const item = chain[index]; + if (!item) return run(forward); + return withExecutionWriter(state, item.grant.parent, (dir) => { + const lifecycle = readLifecycle(state, dir, item.grant.parent, item.verifier); + const parsed = parseParentAssertion(item.assertion); + if (lifecycle.childIntents[parsed.attemptId] !== parsed.grantDigest) + throw Error("Missing durable lifecycle child intent"); + return lock( + index + 1, + forward && lifecycle.phase === "held" && Date.now() < lifecycle.execution.deadlineAt, + ); + }); + } + return lock(0, true); +} +/** Narrow local runtime fence; exact duplicate admission alone never grants forward permission. */ +function withControlExecutionForward(authority: VerifiedExecutionAuthority, run: () => T): T { + const s = executionAuthorityState(authority); + const check = () => + withExecutionWriter(s.installation, s.execution, (dir) => { + const lifecycle = readLifecycle( + s.installation, + dir, + s.execution, + s.installation.installation.publicKey, + ); + if (lifecycle.phase !== "held" || Date.now() >= s.execution.deadlineAt) + throw Error( + "Execution cannot restart effects; observation/settlement only; forward phase fenced", + ); + return run(); + }); + const assertion = retainedParentAssertion(s); + return assertion + ? withParentLifecycles(s.installation, assertion, (forward) => { + if (!forward) throw Error("Canceled/terminal ancestor permits observation only"); + return check(); + }) + : check(); +} +export function assertControlExecutionForward(authority: VerifiedExecutionAuthority): void { + withControlExecutionForward(authority, () => {}); +} +/** Only the protocol's frozen terminal preparation edges may outlive their local worker. */ +export function assertControlDelegationForward( + authority: VerifiedExecutionAuthority, + phase: ControlChildPhase, +): void { + try { + assertControlExecutionForward(authority); + } catch (error) { + const s = executionAuthorityState(authority); + if ( + !( + (s.kind === "configurationMaintenance.prepare" && + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(phase)) || + (s.kind === "configurationHandoff.prepare" && phase === "configuration-handoff") + ) + ) + throw error; + withExecutionWriter(s.installation, s.execution, (dir) => { + const lifecycle = readLifecycle( + s.installation, + dir, + s.execution, + s.installation.installation.publicKey, + ); + if (lifecycle.phase !== "settled" || Date.now() >= s.execution.deadlineAt) throw error; + verifyControlPreparedTerminal(s.installation, s.execution); + }); + } +} +function retainedParentAssertion(s: AuthorityState): string | undefined { + if (s.installation.fixedParentAssertion) return s.installation.fixedParentAssertion; + // Retain the exact adopted assertion with the execution, never search history. + try { + return z + .object({ assertion: z.string() }) + .parse( + readPrivateRecord(s.installation, "execution-parents", `${s.execution.operationKey}.json`), + ).assertion; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + return undefined; +} +/** Persist before a runtime publishes canceled state; return original attempted child identities for cancellation/lookup. */ +export function cancelControlExecution( + authority: VerifiedExecutionAuthority, +): readonly ExecutionRef[] { + const s = executionAuthorityState(authority); + return withExecutionWriter(s.installation, s.execution, (dir) => { + const lifecycle = readLifecycle( + s.installation, + dir, + s.execution, + s.installation.installation.publicKey, + ); + if (lifecycle.phase !== "settled" && lifecycle.phase !== "cancel_pending") { + lifecycle.phase = "cancel_pending"; + lifecycle.sequence++; + writeLifecycle(s.installation, dir, lifecycle); + } + return Object.freeze( + Object.keys(lifecycle.childIntents).map((id) => { + const saved = z + .object({ grant: z.unknown() }) + .parse(readPrivateRecord(s.installation, "parent-attempts", `${id}.json`)); + return Object.freeze( + verifyControl( + s.installation.installation.publicKey, + "openmapx-parent-grant-v1", + saved.grant, + parentGrantSchema, + ).child, + ); + }), + ); + }); +} +function settleExecutionLifecycle(s: AuthorityState) { + withExecutionWriter(s.installation, s.execution, (dir) => { + const lifecycle = readLifecycle( + s.installation, + dir, + s.execution, + s.installation.installation.publicKey, + ); + if (lifecycle.phase !== "settled") { + lifecycle.phase = "settled"; + lifecycle.sequence++; + writeLifecycle(s.installation, dir, lifecycle); + } + }); +} export async function attestParentExecution( authority: VerifiedExecutionAuthority, child: { @@ -1100,21 +1472,7 @@ export async function attestParentExecution( verifyDeadlineEdge(s.kind, child.phase, child.recipient, plan.kind); let attemptId: string = randomUUID(); if (child.recipient === "fixed-cli") { - const d = s.installation.privateRoot.directory("children"); - try { - const matches = d - .list() - .filter( - (n) => - n.endsWith(".key") && - (JSON.parse(d.read(n, 2048).toString("utf8")) as { publicKey: string }).publicKey === - child.childPublicKey, - ); - if (matches.length !== 1) throw Error("Missing fixed child key"); - attemptId = uuid.parse(matches[0]?.slice(0, -4)); - } finally { - d.close(); - } + attemptId = fixedControlChildId(s.installation, child.childPublicKey, s.execution); } const grant = parentGrantSchema.parse({ version: 1, @@ -1148,19 +1506,22 @@ export async function attestParentExecution( }), ).toString("base64url"); parseParentAssertion(assertion); - const dir = executionWitnessDirectory(s.installation, s.execution.executor, true); - try { - const name = `${s.execution.operationKey}.children.json`; - const guardName = `${s.execution.operationKey}.writer`; - mkdirSync(dir.role.name(guardName), { mode: 0o700 }); - const guard = dir.role.directory(guardName); - const guardIdentity = identity(guard.stat); - try { + const issue = () => + withExecutionWriter(s.installation, s.execution, (dir) => { + const lifecycle = readLifecycle( + s.installation, + dir, + s.execution, + s.installation.installation.publicKey, + ); + if (lifecycle.phase !== "held" || Date.now() >= s.execution.deadlineAt) + throw Error("Canceled/terminal execution cannot authorize forward children"); + const name = `${s.execution.operationKey}.children.json`; let witness: Record = {}; try { witness = z .record(uuid, z.unknown()) - .parse(JSON.parse(dir.role.read(name, 65536).toString("utf8"))); + .parse(JSON.parse(dir.read(name, 65536).toString("utf8"))); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } @@ -1185,16 +1546,21 @@ export async function attestParentExecution( { grant: signed, assertion }, true, ); - dir.role.write(name, bytes); + lifecycle.childIntents[attemptId] = fields.grantDigest; + lifecycle.sequence++; + writeLifecycle(s.installation, dir, lifecycle); + dir.write(name, bytes); return assertion; - } finally { - guard.close(); - dir.role.removeDirectory(guardName, guardIdentity); - } - } finally { - dir.close(); - } + }); + const inherited = retainedParentAssertion(s); + return inherited + ? withParentLifecycles(s.installation, inherited, (forward) => { + if (!forward) throw Error("Canceled ancestor prevents forward delegation"); + return issue(); + }) + : issue(); } + function readParentGrant( state: InstallationState, assertion: string, @@ -1305,22 +1671,39 @@ export function verifyParentExecution( grant.child.deadlineAt > grant.parent.deadlineAt ) throw Error("Parent child identity mismatch"); - const name = `${parsed.attemptId}.json`; - let retained = false; - try { - if ( - controlDigest(readPrivateRecord(p.installation, "parent-adoptions", name)) !== - controlDigest(grant) - ) - throw Error("Parent attempt replay conflict"); - retained = true; - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - if (!retained) { - if (Date.now() >= grant.child.deadlineAt) throw Error("Expired parent grant"); - writePrivateRecord(p.installation, "parent-adoptions", name, grant, true); - } + withParentLifecycles(p.installation, assertion, (forward) => { + const name = `${parsed.attemptId}.json`; + let retained = false; + try { + if ( + controlDigest(readPrivateRecord(p.installation, "parent-adoptions", name)) !== + controlDigest(grant) + ) + throw Error("Parent attempt replay conflict"); + retained = true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (!retained) { + if (!forward) throw Error("Canceled/terminal parent permits observation only"); + if (Date.now() >= grant.child.deadlineAt) throw Error("Expired parent grant"); + writePrivateRecord( + p.installation, + "execution-parents", + `${grant.child.operationKey}.json`, + { assertion }, + true, + ); + writePrivateRecord(p.installation, "parent-adoptions", name, grant, true); + } + const binding = z + .strictObject({ assertion: z.string() }) + .parse( + readPrivateRecord(p.installation, "execution-parents", `${grant.child.operationKey}.json`), + ); + if (binding.assertion !== assertion) + throw Error("Incomplete/conflicting parent adoption binding"); + }); const parent = Object.freeze( retainedControlParent(grant.parent), ) as import("../ops/execution-identity").VerifiedParentExecution; @@ -1541,7 +1924,14 @@ export function beginFixedCliExecution( ): void { const s = installationState(installation); if (!s.fixedChildId) throw Error("Fixed CLI authority required"); - writePrivateRecord(s, "spawn-started", `${s.fixedChildId}.json`, { execution }, true); + const begin = () => + writePrivateRecord(s, "spawn-started", `${s.fixedChildId}.json`, { execution }, true); + if (s.fixedParentAssertion) + withParentLifecycles(s, s.fixedParentAssertion, (forward) => { + if (!forward) throw Error("Canceled/terminal parent prevents fixed CLI start"); + begin(); + }); + else begin(); } const executionWitnessSchema = z.strictObject({ diff --git a/packages/core/src/build-artifacts/control-retention.ts b/packages/core/src/build-artifacts/control-retention.ts new file mode 100644 index 000000000..313f4dcbc --- /dev/null +++ b/packages/core/src/build-artifacts/control-retention.ts @@ -0,0 +1,74 @@ +import { fsyncSync, mkdirSync } from "node:fs"; +import { type AnchoredDirectory, identity } from "./filesystem"; + +// No online retirement: receipts and consumed identity/key slots remain for the +// installation lifetime, including >=72h terminal history and every unknown. +// Exhaustion closes admission, never settlement of already reserved executions. +export const CONTROL_RETENTION_LIMITS = { + executionsPerRole: 1024, + leases: 1024, + activeLeases: 256, + childrenPerExecution: 64, + recordsPerNamespace: 65536, + privateRecordBytes: 131072, + minimumTerminalRetentionMs: 72 * 60 * 60 * 1000, +} as const; + +/** Private, fixed-namespace creation accounting. Count reservations before publishing. */ +export function writeRetainedRecord( + dir: AnchoredDirectory, + name: string, + bytes: string, + exclusive: boolean, + maximum: number, +) { + try { + dir.read(name, CONTROL_RETENTION_LIMITS.privateRecordBytes); + dir.write(name, bytes, exclusive); + return; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + mkdirSync(dir.name(".retention-writer"), { mode: 0o700 }); + fsyncSync(dir.fd); + const guard = dir.directory(".retention-writer"); + const held = identity(guard.stat); + try { + let count: number; + try { + count = JSON.parse(dir.read(".retention-count", 64).toString("utf8")); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + count = dir.listBounded(maximum + 1).filter((n) => n !== ".retention-writer").length; + } + if (!Number.isSafeInteger(count) || count < 0 || count >= maximum) + throw Error("Retained authority capacity exhausted; preserve receipts and stop new work"); + dir.write(".retention-count", String(count + 1)); + dir.write(name, bytes, exclusive); + } finally { + guard.close(); + dir.removeDirectory(".retention-writer", held); + } +} + +/** Fixed slots bound admission without listing or parsing historical records. */ +export function reserveAuthoritySlot(dir: AnchoredDirectory, maximum: number, until: number) { + for (let slot = 0; slot < maximum; slot++) { + if (Date.now() >= until) throw Error("Target lease acquisition deadline elapsed"); + const name = `slot-${slot}`; + try { + mkdirSync(dir.name(name), { mode: 0o700 }); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === "EEXIST") continue; + throw e; + } + fsyncSync(dir.fd); + const child = dir.directory(name); + try { + return { name, identity: identity(child.stat) }; + } finally { + child.close(); + } + } + throw Error("Retained authority capacity exhausted; no ownership may be stolen"); +} diff --git a/packages/core/src/build-artifacts/filesystem.ts b/packages/core/src/build-artifacts/filesystem.ts index b47ca3f6d..928747079 100644 --- a/packages/core/src/build-artifacts/filesystem.ts +++ b/packages/core/src/build-artifacts/filesystem.ts @@ -8,6 +8,7 @@ import { fsyncSync, lstatSync, mkdirSync, + opendirSync, openSync, readdirSync, readSync, @@ -105,6 +106,20 @@ export class AnchoredDirectory { this.check(); return readdirSync(this.anchor); } + listBounded(maximum: number): string[] { + this.check(); + const dir = opendirSync(this.anchor); + const names: string[] = []; + try { + for (let entry = dir.readSync(); entry; entry = dir.readSync()) { + if (names.length >= maximum) throw Error("Authority directory capacity exceeded"); + names.push(entry.name); + } + return names; + } finally { + dir.closeSync(); + } + } read(name: string, maximum: number, privateMode = true): Buffer { const path = this.name(name); const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); @@ -146,17 +161,34 @@ export class AnchoredDirectory { const temporary = exclusive ? name : `.write-${randomUUID()}`; const path = this.name(temporary); const fd = openSync(path, C.O_WRONLY | C.O_CREAT | C.O_EXCL | C.O_NOFOLLOW, 0o600); + const expected = identity(fstatSync(fd, { bigint: true })); try { - writeFileSync(fd, bytes); - fsyncSync(fd); - this.check(); - } finally { - closeSync(fd); - } - if (!exclusive) { - renameSync(path, target); + try { + writeFileSync(fd, bytes); + fsyncSync(fd); + this.check(); + } finally { + closeSync(fd); + } + if (!exclusive) renameSync(path, target); + fsyncSync(this.fd); + } catch (error) { + // Only remove this call's unpublished temporary inode. Published or + // exclusive evidence, and unexplained replacements, remain fenced. + if (!exclusive) { + try { + const current = lstatSync(path, { bigint: true }); + if (!current.isFile() || current.nlink !== 1n || !sameInode(identity(current), expected)) + throw Error("Refusing replaced authority temporary"); + unlinkSync(path); + fsyncSync(this.fd); + } catch (cleanup) { + if ((cleanup as NodeJS.ErrnoException).code !== "ENOENT") + throw new AggregateError([error, cleanup], "Authority write requires recovery"); + } + } + throw error; } - fsyncSync(this.fd); } removeFile(name: string) { unlinkSync(this.name(name)); @@ -165,7 +197,7 @@ export class AnchoredDirectory { removeDirectory(name: string, expected: FileIdentity) { const child = this.directory(name); try { - if (!sameInode(identity(child.stat), expected) || child.list().length) + if (!sameInode(identity(child.stat), expected) || child.listBounded(1).length) throw Error("Refusing replaced or nonempty lock"); rmdirSync(this.name(name)); fsyncSync(this.fd); diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 798c51abd..4dbefe0c0 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -3,6 +3,7 @@ export * from "./archive"; export { type ArtifactDelegationAssertion, + assertControlExecutionForward, attestParentExecution, beginControlEffect, beginFixedCliExecution, @@ -12,6 +13,7 @@ export { type ControlInstallation, type ControlPlannedChild, type ControlRecipient, + cancelControlExecution, closeControlInstallation, controlAdoptionSchema, controlChildOperationKey, @@ -41,6 +43,7 @@ export { verifyParentExecution, verifyRetainedLocalControlPeer, } from "./control-authority"; +export { CONTROL_RETENTION_LIMITS } from "./control-retention"; export * from "./manifest"; export { inspectControlAdmission, verifiedParentForArtifactDelegation } from "./peer-admission"; export * from "./target-lease"; diff --git a/packages/core/src/build-artifacts/peer-admission.ts b/packages/core/src/build-artifacts/peer-admission.ts index f759bd510..e9b059e3d 100644 --- a/packages/core/src/build-artifacts/peer-admission.ts +++ b/packages/core/src/build-artifacts/peer-admission.ts @@ -1,4 +1,6 @@ +import { opsOperationSchema } from "../ops/contract"; import type { ExecutionRef, VerifiedParentExecution } from "../ops/execution-identity"; +import { opsOperationFingerprint } from "../ops/fingerprint"; import { verifiedParents } from "../ops/verified-parent-internal"; import { type ControlInstallation, @@ -16,12 +18,38 @@ export function inspectControlAdmission( rawHeaders: readonly string[]; transportPrincipal: "data-manager" | "shared-data-manager-bearer" | "host-cli"; kind: string; + operation: unknown; operationKey: string; operationFingerprint: string; requestedDeadlineAt?: number; }, ) { const state = installationState(installation); + const operation = opsOperationSchema.parse(input.operation); + // These are the fixed container effects implemented by docker-runtime. Restart + // does not start dependencies; promotion explicitly uses --no-deps. Other + // adapters need their own manifest/generation resolver before being enabled. + const targets = (() => { + switch (operation.kind) { + case "motis.primary.stop": + case "motis.primary.restart": + case "motis.primary.promote": + return ["motis"]; + case "motis.staging.stop": + case "motis.staging.restart": + return ["motis-staging"]; + case "feedProxy.validateAndReload": + return ["motis-feed-proxy"]; + default: + throw Error("Receiving target resolver unavailable for this control operation"); + } + })(); + if ( + installation.role !== "ops-agent" || + operation.kind !== input.kind || + opsOperationFingerprint(operation) !== input.operationFingerprint + ) + throw Error("Receiving typed operation mismatch"); const { delegation } = controlHeaders(input.rawHeaders); if (!delegation) throw Error("Missing control assertion"); const inspected = inspectArtifactDelegation({ @@ -37,6 +65,8 @@ export function inspectControlAdmission( inspected.execution.executor !== installation.role ) throw Error("Signed immutable admission mismatch"); + if (targets.some((id) => !state.registeredTargets.has(id) || !inspected.serviceIds.includes(id))) + throw Error("Grant does not contain operation-derived target closure"); const peer = verifyControlPeer(installation, { rawHeaders: input.rawHeaders, transportPrincipal: input.transportPrincipal, diff --git a/packages/core/src/build-artifacts/peer-process.test.ts b/packages/core/src/build-artifacts/peer-process.test.ts index f34bff397..e1c5beb74 100644 --- a/packages/core/src/build-artifacts/peer-process.test.ts +++ b/packages/core/src/build-artifacts/peer-process.test.ts @@ -1,10 +1,10 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { build } from "esbuild"; -import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { type ControlIssuer, type ExecutionRef, @@ -30,7 +30,7 @@ import { settleArtifactTargetLeaseChild, } from "./target-lease"; -let root: string, fixture: string; +let root: string, fixture: string, fixtureRoot: string; beforeAll(async () => { const result = await build({ entryPoints: [join(import.meta.dirname, "test-fixtures/peer-process.ts")], @@ -40,8 +40,13 @@ beforeAll(async () => { format: "esm", target: "node24", }); - fixture = result.outputFiles[0]?.text; + const output = result.outputFiles[0]; + if (!output) throw Error("Missing peer fixture bundle"); + fixtureRoot = mkdtempSync(join(tmpdir(), "omx-peer-bundle-")); + fixture = join(fixtureRoot, "peer.mjs"); + writeFileSync(fixture, output.text); }); +afterAll(() => rmSync(fixtureRoot, { recursive: true, force: true })); beforeEach(async () => { root = mkdtempSync(join(tmpdir(), "omx-peer-process-")); mkdirSync(join(root, "data"), { mode: 0o700 }); @@ -56,6 +61,7 @@ function ref( parent?: ExecutionRef, phase?: ControlChildPhase, ): ExecutionRef { + if (parent && !phase) throw Error("Missing child phase in fixture"); const operationFingerprint = role === "ops-agent" && !["configurationHandoff.prepare", "configurationMaintenance.prepare"].includes(kind) @@ -67,7 +73,7 @@ function ref( principal: role === "ops-agent" ? (parent ? "data-manager" : "api") : role, executionId: role === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), operationKey: parent - ? controlChildOperationKey(parent.operationKey, phase!, 0) + ? controlChildOperationKey(parent.operationKey, phase ?? "hardlinks", 0) : `opk1_${randomUUID().replaceAll("-", "")}`, operationFingerprint, deadlineAt, @@ -80,7 +86,7 @@ function ref( }; } async function child(input: unknown) { - const p = spawn(process.execPath, ["--input-type=module", "-e", fixture], { + const p = spawn(process.execPath, [fixture], { stdio: ["pipe", "pipe", "pipe"], }); let out = "", @@ -264,73 +270,74 @@ it("keeps owner-death recovery fenced until a separately admitted child settles" }), ).toBe("released"); }); -it("certifies a one-use fixed CLI and its depth-two hardlink leaf across three processes", async () => { - const c = await import("./control-authority"); - const operation = { kind: "service.start" as const, serviceId: "motis" }; - const execution = ref("ops-agent", operation.kind, operation); - const cli = ref("host-cli", operation.kind, operation, execution, "lifecycle"); - const plan: ControlPlannedChild = { - phase: "lifecycle", - ordinal: 0, - kind: operation.kind, - execution: cli, - serviceIds: ["motis"], - recipient: "fixed-cli", - }; - const installation = c.openControlInstallation({ - infraDir: root, - role: "ops-agent", - serviceIds: ["motis", "osrm"], - }); - const authority = c.retainAdmittedControlExecution(installation, { - execution, - kind: operation.kind, - operation, - admittedAt: execution.deadlineAt - 60000, - serviceIds: ["motis"], - plan: [plan], - }); - const lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["motis"], - owner: authority, - }); - const id = randomUUID(); - const key = c.prepareFixedControlChild(authority, id); - const assertion = await delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }); - const args = c.fixedCliArguments(operation); - c.retainFixedCliSpawnIntent(authority, { delegationId: id, execution: cli, operation, args }); - const hardlinks = { - kind: "generation-hardlinks", - revisionId: "cfg1_test", - planDigest: "a".repeat(64), - coordinatorMode: "require-running", - }; - const leaf = ref("data-manager", hardlinks.kind, hardlinks, cli, "hardlinks"); - const childPublicKey = JSON.parse( - readFileSync(join(root, "secrets/control-verifiers.json"), "utf8"), - ).publicKeys["data-manager"]; - const fixed = { mode: "fixed", infraDir: root, assertion, execution: cli, args }; - const leafInput = { - execution: leaf, - phase: "hardlinks", - recipient: "data-manager", - serviceIds: ["motis"], - childPublicKey, - }; - expect(await child({ ...fixed, child: { ...leafInput, serviceIds: ["osrm"] } })).toMatchObject({ - ok: false, - }); - const started = (await child({ ...fixed, action: "run", child: leafInput })) as { - ok: boolean; - assertion: string; - error?: string; - }; - expect(started, JSON.stringify(started)).toMatchObject({ ok: true }); - expect(await child({ ...fixed, action: "run" })).toMatchObject({ ok: false }); - expect(await child({ ...fixed, action: "settle" })).toMatchObject({ ok: false }); - expect( - await child({ +it.each([false, true])( + "certifies a depth-two hardlink leaf across processes (cancel intermediate: %s)", + async (cancelIntermediate) => { + const c = await import("./control-authority"); + const operation = { kind: "service.start" as const, serviceId: "motis" }; + const execution = ref("ops-agent", operation.kind, operation); + const cli = ref("host-cli", operation.kind, operation, execution, "lifecycle"); + const plan: ControlPlannedChild = { + phase: "lifecycle", + ordinal: 0, + kind: operation.kind, + execution: cli, + serviceIds: ["motis"], + recipient: "fixed-cli", + }; + const installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis", "osrm"], + }); + const authority = c.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const id = randomUUID(); + const key = c.prepareFixedControlChild(authority, id); + const assertion = await delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }); + const args = c.fixedCliArguments(operation); + c.retainFixedCliSpawnIntent(authority, { delegationId: id, execution: cli, operation, args }); + const hardlinks = { + kind: "generation-hardlinks", + revisionId: "cfg1_test", + planDigest: "a".repeat(64), + coordinatorMode: "require-running", + }; + const leaf = ref("data-manager", hardlinks.kind, hardlinks, cli, "hardlinks"); + const childPublicKey = JSON.parse( + readFileSync(join(root, "secrets/control-verifiers.json"), "utf8"), + ).publicKeys["data-manager"]; + const fixed = { mode: "fixed", infraDir: root, assertion, execution: cli, args }; + const leafInput = { + execution: leaf, + phase: "hardlinks", + recipient: "data-manager", + serviceIds: ["motis"], + childPublicKey, + }; + expect(await child({ ...fixed, child: { ...leafInput, serviceIds: ["osrm"] } })).toMatchObject({ + ok: false, + }); + const started = (await child({ ...fixed, action: "run", child: leafInput })) as { + ok: boolean; + assertion: string; + error?: string; + }; + expect(started, JSON.stringify(started)).toMatchObject({ ok: true }); + expect(await child({ ...fixed, action: "run" })).toMatchObject({ ok: false }); + expect(await child({ ...fixed, action: "settle" })).toMatchObject({ ok: false }); + const hardlinkInput = { infraDir: root, role: "data-manager", assertion: started.assertion, @@ -338,22 +345,40 @@ it("certifies a one-use fixed CLI and its depth-two hardlink leaf across three p kind: hardlinks.kind, operation: hardlinks, admittedAt: leaf.deadlineAt - 60000, - settle: true, - }), - ).toMatchObject({ ok: true }); - expect(await child({ ...fixed, action: "settle" })).toMatchObject({ ok: true }); - c.retainControlEffectReceipt(authority, { - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - await settleArtifactTargetLeaseChild( - lease, - c.verifyControlEffectSettlement(installation, execution), - ); - await releaseArtifactTargetLease(lease); - c.closeControlInstallation(installation); -}); + }; + if (cancelIntermediate) { + expect(await child({ ...hardlinkInput, forward: true })).toMatchObject({ ok: true }); + expect(await child({ ...fixed, action: "cancel" })).toMatchObject({ ok: true }); + expect(await child(hardlinkInput)).toMatchObject({ ok: true, retained: true }); + expect(await child({ ...hardlinkInput, forward: true })).toMatchObject({ ok: false }); + await expect(releaseArtifactTargetLease(lease)).rejects.toThrow(); + } + expect( + await child({ + infraDir: root, + role: "data-manager", + assertion: started.assertion, + execution: leaf, + kind: hardlinks.kind, + operation: hardlinks, + admittedAt: leaf.deadlineAt - 60000, + settle: true, + }), + ).toMatchObject({ ok: true }); + expect(await child({ ...fixed, action: "settle" })).toMatchObject({ ok: true }); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, execution), + ); + await releaseArtifactTargetLease(lease); + c.closeControlInstallation(installation); + }, +); it("keeps an attempted child fenced after SIGKILL of its actual owner process", async () => { const c = await import("./control-authority"); const l = await import("./target-lease"); @@ -380,7 +405,7 @@ it("keeps an attempted child fenced after SIGKILL of its actual owner process", const childPublicKey = JSON.parse( readFileSync(join(root, "secrets/control-verifiers.json"), "utf8"), ).publicKeys["ops-agent"]; - const p = spawn(process.execPath, ["--input-type=module", "-e", fixture], { + const p = spawn(process.execPath, [fixture], { stdio: ["pipe", "pipe", "pipe"], }); const response = new Promise<{ ok: boolean; assertion: string; leaseId: string }>( @@ -388,7 +413,7 @@ it("keeps an attempted child fenced after SIGKILL of its actual owner process", let text = ""; p.stdout.on("data", (b) => { text += b; - if (text.includes("\n")) resolve(JSON.parse(text.split("\n")[0]!)); + if (text.includes("\n")) resolve(JSON.parse(text.split("\n")[0] ?? "")); }); p.once("error", reject); }, @@ -450,7 +475,7 @@ it.each([false, true])( handoffId: randomUUID(), expectedRevisionId: "cfg1_old", desiredRevisionId: "cfg1_new", - image: "sha256:" + "1".repeat(64), + image: `sha256:${"1".repeat(64)}`, }; const execution = ref("ops-agent", operation.kind, operation); const body = { kind: "configuration-helper", handoffId: operation.handoffId }; @@ -493,7 +518,7 @@ it.each([false, true])( }), { mode: 0o600 }, ); - const p = spawn(process.execPath, ["--input-type=module", "-e", fixture], { + const p = spawn(process.execPath, [fixture], { stdio: ["pipe", "pipe", "pipe"], }); const response = new Promise<{ ok: boolean; assertion: string; error?: string }>( @@ -501,13 +526,13 @@ it.each([false, true])( let out = ""; p.stdout.on("data", (b) => { out += b; - if (out.includes("\n")) resolve(JSON.parse(out.split("\n")[0]!)); + if (out.includes("\n")) resolve(JSON.parse(out.split("\n")[0] ?? "")); }); p.once("error", reject); }, ); p.stdin.end( - JSON.stringify({ + `${JSON.stringify({ mode: "owner", infraDir: root, role: "ops-agent", @@ -515,7 +540,7 @@ it.each([false, true])( child: { ...plan, childPublicKey: "" }, fixedId: randomUUID(), prepared, - }) + "\n", + })}\n`, ); const owned = await response; try { @@ -543,7 +568,7 @@ it("keeps paired-pin approval a one-use leaf under its original activation root" const execution = ref("data-manager", operation.kind, operation); const body = { kind: "transitousLock.approve" as const, - ref: "main@" + "a".repeat(40), + ref: `main@${"a".repeat(40)}`, approvedBy: "synthetic-reviewer", }; const childRef = ref("ops-agent", body.kind, body, execution, "paired-pin-commit"); @@ -585,6 +610,7 @@ it("keeps paired-pin approval a one-use leaf under its original activation root" .publicKeys["ops-agent"]; const assertion = await delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }); const input = { + primitive: true, infraDir: root, role: "ops-agent", assertion, @@ -690,3 +716,83 @@ it("settles the logical configuration-fence leaf without releasing the maintenan await releaseArtifactTargetLease(lease); c.closeControlInstallation(installation); }); + +it("serializes a paused observer and a newer writer across actual processes", async () => { + const c = await import("./control-authority"); + const l = await import("./target-lease"); + const fs = await import("node:fs"); + const operation = { kind: "transit-sync" }; + const execution = ref("data-manager", operation.kind, operation); + const body = { kind: "motis.primary.stop" }; + const receiver = ref("ops-agent", body.kind, body, execution, "stop-primary"); + const installation = c.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis"], + }); + const plan: ControlPlannedChild = { + phase: "stop-primary", + ordinal: 0, + kind: body.kind, + execution: receiver, + serviceIds: ["motis"], + recipient: "ops-agent", + }; + const authority = c.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt: execution.deadlineAt - 60000, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await l.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const publicKey = JSON.parse(readFileSync(join(root, "secrets/control-verifiers.json"), "utf8")) + .publicKeys["ops-agent"]; + const assertion = await l.delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: publicKey, + }); + const ready = join(root, "observer-ready"), + resume = join(root, "observer-resume"); + const observing = child({ + mode: "observe-interleaving", + infraDir: root, + assertion, + ready, + resume, + }); + let busy = false; + try { + await expect.poll(() => fs.existsSync(ready)).toBe(true); + try { + await l.cancelArtifactTargetLease(lease); + } catch (error) { + busy = /writer.*busy/.test(String(error)); + } + } finally { + fs.writeFileSync(resume, "resume"); + } + expect(await observing).toMatchObject({ ok: true }); + expect(busy, "observer must hold the same CAS guard as the writer").toBe(true); + const ownerPath = join(root, `data/.admin-target-control/leases/${lease.leaseId}/owner.json`); + const old = readFileSync(ownerPath); + await l.cancelArtifactTargetLease(lease); + const mark = JSON.parse( + readFileSync( + join(root, `control-private/data-manager/observations/${lease.leaseId}.json`), + "utf8", + ), + ); + expect(mark.owner.sequence).toBe(JSON.parse(readFileSync(ownerPath, "utf8")).payload.sequence); + fs.writeFileSync(ownerPath, old); + expect(() => + l.inspectArtifactDelegation({ dataRoot: join(root, "data"), assertion, installation }), + ).toThrow(/rollback/); + l.closeArtifactTargetLeaseHandle(lease); + c.closeControlInstallation(installation); +}); diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index ea9e3cd46..6d8791c23 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -257,6 +257,11 @@ it("retains terminal maintenance ownership and permits only frozen one-use conti { mode: 0o600 }, ); c.retainControlPreparedTerminal(authority); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); await expect( a.delegateArtifactTargetLease(lease, { ...plan, @@ -369,3 +374,146 @@ it("does not restart effects after retaining unused-execution evidence", async ( }); expect(() => c.beginControlEffect(authority)).toThrow(/cannot restart/); }); + +it("admits from a bounded active index without opening retained terminal ledgers", async () => { + const a = await api(); + const { AnchoredDirectory } = await import("./filesystem"); + const { vi } = await import("vitest"); + // Real retained terminal histories, with valid signatures and released locks. + for (let i = 0; i < 12; i++) { + const authority = owner(); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, authority.execution), + ); + await a.releaseArtifactTargetLease(lease); + } + const authority = owner(); + const read = AnchoredDirectory.prototype.read; + const historicalReads: string[] = []; + const spy = vi.spyOn(AnchoredDirectory.prototype, "read").mockImplementation(function ( + this: InstanceType, + name, + max, + mode, + ) { + if (name === "owner.json") historicalReads.push(this.path); + return read.call(this, name, max, mode); + }); + let lease: import("./target-lease").ArtifactTargetLease | undefined; + try { + lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + } finally { + spy.mockRestore(); + } + expect(new Set(historicalReads).size, "new acquisition must read only its new ledger").toBe(1); + if (lease) a.closeArtifactTargetLeaseHandle(lease); +}); + +it("retains terminal and unresolved records past 72h and refuses at retained capacity before effects", async () => { + const a = await api(); + const { CONTROL_RETENTION_LIMITS } = await import("./control-retention"); + const { vi } = await import("vitest"); + const fs = await import("node:fs"); + const authority = owner(); + const terminal = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild( + terminal, + verifyControlEffectSettlement(installation, authority.execution), + ); + await a.releaseArtifactTargetLease(terminal); + const unsettledAuthority = owner(); + const unresolved = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: unsettledAuthority, + }); + const retainedFile = join( + root, + `data/.admin-target-control/leases/${unresolved.leaseId}/owner.json`, + ); + const original = readFileSync(retainedFile); + const history = join(root, "data/.admin-target-control/admission-index-v1/history"); + for (let i = 0; i < CONTROL_RETENTION_LIMITS.leases; i++) { + try { + mkdirSync(join(history, `slot-${i}`), { mode: 0o700 }); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + } + } + const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 60 * 60 * 1000); + let effects = 0; + try { + await expect( + a.withArtifactTargetLease( + { dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: owner() }, + async () => { + effects++; + }, + ), + ).rejects.toThrow(/capacity/); + expect(effects).toBe(0); + expect(readFileSync(retainedFile)).toEqual(original); + expect( + fs.existsSync(join(root, `data/.admin-target-control/leases/${terminal.leaseId}/owner.json`)), + ).toBe(true); + expect( + fs.existsSync(join(root, `control-private/host-cli/leases/${terminal.leaseId}.json`)), + ).toBe(true); + expect(fs.existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(true); + expect(() => a.assertArtifactTargetLeaseForward(unresolved)).toThrow(/observation/); + } finally { + clock.mockRestore(); + a.closeArtifactTargetLeaseHandle(unresolved); + } +}); + +it("does not accumulate temporary authority records after failed atomic updates", async () => { + const { AnchoredDirectory } = await import("./filesystem"); + const { vi } = await import("vitest"); + const { readdirSync } = await import("node:fs"); + const dir = new AnchoredDirectory(root); + try { + dir.write("receipt.json", "original", true); + for (let i = 0; i < 3; i++) { + const check = dir.check.bind(dir); + let calls = 0; + const fault = vi.spyOn(dir, "check").mockImplementation(() => { + if (++calls === 3) throw Error("synthetic post-write authority failure"); + check(); + }); + try { + expect(() => dir.write("receipt.json", "replacement")).toThrow(/synthetic/); + } finally { + fault.mockRestore(); + } + } + expect(dir.read("receipt.json", 64).toString()).toBe("original"); + expect(readdirSync(root).filter((name) => name.startsWith(".write-"))).toEqual([]); + } finally { + dir.close(); + } +}); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index 4bff8890e..ebd55c71f 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -11,6 +11,8 @@ import { canonicalJson } from "../ops/fingerprint"; import { type ArtifactDelegationAssertion, type AuthorityState, + assertControlDelegationForward, + assertControlExecutionForward, assertionSchema, type ChildSettlement, type ControlChildPhase, @@ -24,6 +26,7 @@ import { digest, encodePrivate, executionAuthorityState, + fixedControlChildId, type InstallationState, installationState, newControlKey, @@ -44,6 +47,7 @@ import { verifyControlPreparedTerminal, writePrivateRecord, } from "./control-authority"; +import { CONTROL_RETENTION_LIMITS, reserveAuthoritySlot } from "./control-retention"; import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; export type { ChildSettlement } from "./control-authority"; @@ -74,6 +78,17 @@ const certificateSchema = z.strictObject({ createdAt: z.number().int().nonnegative().safe(), }); type Certificate = z.infer; +const slotSchema = z.strictObject({ name: z.string().regex(/^slot-\d{1,4}$/), identity: inode }); +const admissionSlotSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + leaseId: uuid, + certificateDigest: digest, + active: slotSchema, + history: slotSchema, +}); +type AdmissionSlots = z.infer; + const signedCertificateSchema = z.strictObject({ payload: certificateSchema, signature: z.string(), @@ -141,9 +156,16 @@ interface State { node: string | null; execution: ExecutionRef; authority?: AuthorityState; + verifiedAuthority?: VerifiedExecutionAuthority; closed: boolean; + journalGuardHeld?: boolean; + slots?: AdmissionSlots; } const handles = new WeakMap(); +function required(value: T | undefined, message: string): T { + if (value === undefined) throw Error(message); + return value; +} function stateOf(lease: TargetLeaseAuthority): State { const state = handles.get(lease); if (!state || state.closed || (lease.mode === "owner") !== (state.node === null)) @@ -159,9 +181,14 @@ function verifyAnchors(s: State) { for (const id of s.certificate.serviceIds) { const lock = s.locks.directory(id); try { - if (!sameInode(identity(lock.stat), s.certificate.lockIdentities[id]!)) + if ( + !sameInode( + identity(lock.stat), + required(s.certificate.lockIdentities[id], "Missing certified target inode"), + ) + ) throw Error("Target lock inode identity changed"); - if (lock.list().length) throw Error("Unexplained nonempty target lock"); + if (lock.listBounded(1).length) throw Error("Unexplained nonempty target lock"); } finally { lock.close(); } @@ -253,7 +280,13 @@ function highWater(s: State, nodes: Record; + children: Map; +}; +function tree(s: State): LeaseTree { + if (!s.journalGuardHeld) return guarded(s, () => tree(s), false); const root = readRoot(s); if (controlDigest(root.certificate) !== controlDigest(s.certificateEnvelope)) throw Error("Owner certificate replacement"); @@ -358,8 +391,8 @@ function tree(s: State) { childDir = childDirectory(s, "children"); try { if ( - grantDir.list().some((n) => !grants.has(n.replace(/\.json$/, ""))) || - childDir.list().some((n) => !children.has(n.replace(/\.json$/, ""))) + grantDir.listBounded(64).some((n) => !grants.has(n.replace(/\.json$/, ""))) || + childDir.listBounded(64).some((n) => !children.has(n.replace(/\.json$/, ""))) ) throw Error("Unexplained protocol record; recovery required"); } finally { @@ -370,12 +403,13 @@ function tree(s: State) { return { root, grants, children }; } function ledgerBytes(s: State) { - let bytes = s.ledger.read("owner.json", 131072).length; + let bytes = + s.ledger.read("owner.json", 131072).length + s.ledger.read("admission.json", 4096).length; for (const kind of ["grants", "children"] as const) { const dir = childDirectory(s, kind); try { - if (dir.list().length > 64) throw Error("Protocol record capacity"); - for (const name of dir.list()) bytes += dir.read(name, 131072).length; + if (dir.listBounded(64).length > 64) throw Error("Protocol record capacity"); + for (const name of dir.listBounded(64)) bytes += dir.read(name, 131072).length; } finally { dir.close(); } @@ -405,8 +439,13 @@ function writeBounded( if (dir !== s.ledger) dir.close(); } } -async function guarded(s: State, run: () => T): Promise { - verifyAnchors(s); +function guarded(s: State, run: () => T, checkTargets = true): T { + if (s.journalGuardHeld) throw Error("Nested lease journal writer"); + if (checkTargets) verifyAnchors(s); + else { + s.data.check(); + s.ledger.check(); + } const path = s.ledger.name(".journal-writer"); try { mkdirSync(path, { mode: 0o700 }); @@ -417,8 +456,10 @@ async function guarded(s: State, run: () => T): Promise { const guard = s.ledger.directory(".journal-writer"); const expected = identity(guard.stat); try { + s.journalGuardHeld = true; return run(); } finally { + s.journalGuardHeld = false; guard.close(); s.ledger.removeDirectory(".journal-writer", expected); } @@ -449,12 +490,60 @@ function knownOutcome(e: ChildSettlement | null | undefined) { !(e.outcome === "failed_without_effect" && e.publication !== "not_started") ); } +function admissionIndex( + control: AnchoredDirectory, + leases: AnchoredDirectory, + installation: InstallationState, +) { + let created = false; + try { + mkdirSync(control.name("admission-index-v1"), { mode: 0o700 }); + fsyncSync(control.fd); + created = true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + } + const index = control.directory("admission-index-v1"); + try { + if (created) { + // Existing installations require supervised indexing; never silently omit an old fence. + if (leases.listBounded(1).length) + throw Error("Existing leases require supervised admission-index recovery"); + index.directory("active", true).close(); + index.directory("history", true).close(); + index.write( + "ready.json", + canonicalJson({ installationId: installation.installation.installationId }), + true, + ); + } + const ready = z.strictObject({ installationId: uuid }).parse(json(index, "ready.json", 1024)); + if (ready.installationId !== installation.installation.installationId) + throw Error("Foreign admission index"); + return index; + } catch (e) { + index.close(); + throw e; + } +} +function releaseActiveSlot(s: State) { + if (!s.slots) throw Error("Missing retained admission slots"); + const index = s.control.directory("admission-index-v1"), + active = index.directory("active"); + try { + active.removeDirectory(s.slots.active.name, s.slots.active.identity); + } finally { + active.close(); + index.close(); + } +} export async function acquireArtifactTargetLease(options: { dataRoot: string; serviceIds: readonly string[]; owner: VerifiedExecutionAuthority; signal?: AbortSignal; }): Promise { + const beganAt = Date.now(); const authority = executionAuthorityState(options.owner); if (authority.installation.fixedChildId) throw Error("Fixed children must borrow and cannot acquire a root"); @@ -464,6 +553,7 @@ export async function acquireArtifactTargetLease(options: { Date.now() >= authority.execution.deadlineAt ) throw Error("Target closure or execution deadline mismatch"); + assertControlExecutionForward(options.owner); options.signal?.throwIfAborted(); try { readPrivateRecord( @@ -489,36 +579,21 @@ export async function acquireArtifactTargetLease(options: { const acquired: Record = {}; let published = false; let ledger: AnchoredDirectory | undefined; + let activeSlot: z.infer | undefined, + historySlot: z.infer | undefined; + let index: AnchoredDirectory | undefined; try { - let unsettled = 0; - for (const id of leases.list()) { - uuid.parse(id); - const d = leases.directory(id); - try { - const raw = json(d, "owner.json"); - const preliminary = z.object({ payload: rootSchema }).parse(raw); - const certificate = preliminary.payload.certificate; - const cert = verifyControl( - authority.installation.publicKeys[certificate.payload.owner.executor], - "openmapx-lease-certificate-v1", - certificate, - certificateSchema, - ); - if (cert.installationId !== authority.installation.installation.installationId) - throw Error("Foreign retained lease"); - const record = verifyControl( - cert.ownerPublicKey, - "openmapx-lease-owner-v1", - raw, - rootSchema, - ); - if (record.phase !== "settled") unsettled++; - } finally { - d.close(); - } + const until = Math.min(beganAt + 5000, authority.execution.deadlineAt); + index = admissionIndex(control, leases, authority.installation); + const history = index.directory("history"), + active = index.directory("active"); + try { + historySlot = reserveAuthoritySlot(history, CONTROL_RETENTION_LIMITS.leases, until); + activeSlot = reserveAuthoritySlot(active, CONTROL_RETENTION_LIMITS.activeLeases, until); + } finally { + active.close(); + history.close(); } - if (unsettled >= 256) throw Error("Unsettled lease capacity exhausted"); - const until = Math.min(Date.now() + 5000, authority.execution.deadlineAt); for (const id of serviceIds) { for (;;) { options.signal?.throwIfAborted(); @@ -559,6 +634,21 @@ export async function acquireArtifactTargetLease(options: { "openmapx-lease-certificate-v1", certificate, ); + const slots: AdmissionSlots = { + version: 1, + installationId: certificate.installationId, + leaseId, + certificateDigest: controlDigest(certificateEnvelope), + active: activeSlot, + history: historySlot, + }; + ledger.write( + "admission.json", + canonicalJson( + signControl(authority.installation.privateKey, "openmapx-admission-slots-v1", slots), + ), + true, + ); writePrivateRecord( authority.installation, "leases", @@ -603,6 +693,8 @@ export async function acquireArtifactTargetLease(options: { node: null, execution: authority.execution, authority, + verifiedAuthority: options.owner, + slots, closed: false, }; const lease = Object.freeze({ @@ -613,8 +705,29 @@ export async function acquireArtifactTargetLease(options: { }); handles.set(lease, state); tree(state); + if (Date.now() >= until) + throw Error( + "Target lease acquisition deadline elapsed after publication; reconcile original identity", + ); + index.close(); return lease; } catch (e) { + if (!published && index) { + for (const [kind, slot] of [ + ["active", activeSlot], + ["history", historySlot], + ] as const) { + if (!slot) continue; + if (kind === "history" && ledger) continue; // Partial records consume retained capacity too. + const dir = index.directory(kind); + try { + dir.removeDirectory(slot.name, slot.identity); + } finally { + dir.close(); + } + } + } + index?.close(); if (!published) for (const [id, expected] of Object.entries(acquired)) locks.removeDirectory(id, expected); ledger?.close(); @@ -722,6 +835,16 @@ function permitted(s: State, child: ControlPlannedChild, parentGrant?: Grant): s child.recipient, ); } +function forwardAncestry(t: ReturnType, node: string | null): boolean { + if (t.root.phase !== "held") return false; + while (node !== null) { + const grant = t.grants.get(node); + if (!grant || t.children.get(node)?.phase !== "held" || Date.now() >= grant.child.deadlineAt) + return false; + node = grant.parentNode; + } + return Date.now() < t.root.certificate.payload.owner.deadlineAt; +} export async function delegateArtifactTargetLease( lease: TargetLeaseAuthority, child: { @@ -733,14 +856,14 @@ export async function delegateArtifactTargetLease( }, ): Promise { const s = stateOf(lease); + if (s.verifiedAuthority) assertControlDelegationForward(s.verifiedAuthority, child.phase); return guarded(s, () => { const current = tree(s); - const node = s.node === null ? current.root : current.children.get(s.node)!; - if ( - node.phase !== "held" || - current.root.phase !== "held" || - Date.now() >= s.execution.deadlineAt - ) + const node = + s.node === null + ? current.root + : required(current.children.get(s.node), "Missing retained child node"); + if (!forwardAncestry(current, s.node) || Date.now() >= s.execution.deadlineAt) throw Error("Lease does not permit forward delegation"); let plan = s.authority?.plan.find( (p) => @@ -750,7 +873,7 @@ export async function delegateArtifactTargetLease( canonicalJson(p.serviceIds) === canonicalJson(child.serviceIds), ); if (!plan && s.node !== null) { - const inherited = current.grants.get(s.node)!; + const inherited = required(current.grants.get(s.node), "Missing retained child grant"); const kinds = childPhaseKinds[child.phase]; if ( inherited.permittedFurtherEdges.includes(child.phase) && @@ -758,7 +881,7 @@ export async function delegateArtifactTargetLease( child.execution.operationKey === controlChildOperationKey(s.execution.operationKey, child.phase, 0) ) - plan = { ...child, kind: kinds[0]!, ordinal: 0 }; + plan = { ...child, kind: required(kinds[0], "Missing finite child kind"), ordinal: 0 }; } if (!plan) throw Error("Child is absent from immutable admitted plan"); if (current.grants.size >= 64) throw Error("Child intent capacity exhausted"); @@ -784,18 +907,7 @@ export async function delegateArtifactTargetLease( verifyControlPreparedTerminal(s.installation, s.execution); let delegationId: string = randomUUID(); if (child.recipient === "fixed-cli" || child.recipient === "configuration-helper") { - const dir = s.installation.privateRoot.directory("children"); - try { - const matches = dir.list().filter((name) => { - if (!name.endsWith(".key")) return false; - const saved = JSON.parse(dir.read(name, 2048).toString("utf8")); - return saved.publicKey === child.childPublicKey; - }); - if (matches.length !== 1) throw Error("Missing fixed child private challenge"); - delegationId = uuid.parse(matches[0]?.slice(0, -4)); - } finally { - dir.close(); - } + delegationId = fixedControlChildId(s.installation, child.childPublicKey, s.execution); } else if (child.childPublicKey !== s.installation.publicKeys[child.recipient]) throw Error("Child role key mismatch"); const grant: Grant = { @@ -886,7 +998,20 @@ function openState( certificate.leaseId !== leaseId ) throw Error("Foreign lease installation"); + const slots = verifyControl( + installation.publicKeys[certificate.owner.executor], + "openmapx-admission-slots-v1", + json(ledger, "admission.json", 4096), + admissionSlotSchema, + ); + if ( + slots.leaseId !== leaseId || + slots.installationId !== certificate.installationId || + slots.certificateDigest !== controlDigest(envelope) + ) + throw Error("Admission slot identity mismatch"); const s = { + slots, data, control, locks, @@ -990,7 +1115,7 @@ export async function adoptArtifactTargetLease(options: { const retained = t.children.get(grant.delegationId); if ( !retained && - (t.root.phase !== "held" || parent.phase !== "held" || Date.now() >= grant.child.deadlineAt) + (!forwardAncestry(t, grant.parentNode) || Date.now() >= grant.child.deadlineAt) ) throw Error("Stale or canceled delegation"); const admissionName = `${grant.delegationId}.json`; @@ -1073,6 +1198,7 @@ export function bindBorrowedControlExecution( ) throw Error("Borrower execution authority mismatch"); s.authority = a; + s.verifiedAuthority = authority; } export async function settleArtifactTargetLeaseChild( lease: TargetLeaseAuthority, @@ -1087,7 +1213,8 @@ export async function settleArtifactTargetLeaseChild( throw Error("Foreign settlement identity"); await guarded(s, () => { const t = tree(s); - const node = s.node === null ? t.root : t.children.get(s.node)!; + const node = + s.node === null ? t.root : required(t.children.get(s.node), "Missing retained child node"); for (const grant of t.grants.values()) { if (grant.parentNode === s.node) { const child = t.children.get(grant.delegationId); @@ -1116,7 +1243,11 @@ export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Pr } }); for (const id of s.certificate.serviceIds) - s.locks.removeDirectory(id, s.certificate.lockIdentities[id]!); + s.locks.removeDirectory( + id, + required(s.certificate.lockIdentities[id], "Missing certified target inode"), + ); + releaseActiveSlot(s); closeState(s); } export async function reconcileArtifactTargetLease(options: { @@ -1186,6 +1317,7 @@ export async function withArtifactTargetLease( run: (lease: TargetLeaseAuthority) => Promise, ): Promise { if (options.inherited) { + assertControlExecutionForward(options.owner); const state = stateOf(options.inherited); const requestedRoot = new AnchoredDirectory(options.dataRoot); try { @@ -1201,7 +1333,7 @@ export async function withArtifactTargetLease( ) throw Error("Inherited target closure cannot widen or reacquire"); assertArtifactTargetLeaseForward(options.inherited); - const inheritedGrant = tree(state).grants.get(state.node!); + const inheritedGrant = state.node === null ? undefined : tree(state).grants.get(state.node); if (inheritedGrant?.recipient === "configuration-helper") verifyConfigurationHelperReady(options.inherited); return run(options.inherited); @@ -1243,7 +1375,9 @@ export function inspectArtifactDelegation(options: { const grant = t.grants.get(parsed.delegationId); if (!grant || parsed.ownerCertificateDigest !== controlDigest(s.certificateEnvelope)) throw Error("Unanchored delegation"); - const parent = grant.parentNode ? t.grants.get(grant.parentNode)! : undefined; + const parent = grant.parentNode + ? required(t.grants.get(grant.parentNode), "Missing parent grant") + : undefined; const parentKey = parent?.childPublicKey ?? s.certificate.ownerPublicKey; const { signature, ...fields } = parsed; verifyControl( @@ -1252,7 +1386,9 @@ export function inspectArtifactDelegation(options: { { payload: fields, signature }, assertionSchema.omit({ signature: true }), ); - const parentNode = grant.parentNode ? t.children.get(grant.parentNode)! : t.root; + const parentNode = grant.parentNode + ? required(t.children.get(grant.parentNode), "Missing parent adoption") + : t.root; if (parentNode.grantDigests[grant.delegationId] !== parsed.grantDigest) throw Error("Grant digest mismatch"); return Object.freeze({ @@ -1271,16 +1407,35 @@ export function inspectArtifactDelegation(options: { } } -export async function cancelArtifactTargetLease(lease: TargetLeaseAuthority): Promise { +export async function cancelArtifactTargetLease( + lease: TargetLeaseAuthority, +): Promise { const s = stateOf(lease); - await guarded(s, () => { + return guarded(s, () => { const t = tree(s); - const node = s.node === null ? t.root : t.children.get(s.node)!; - if (node.phase === "settled") return; - node.phase = "cancel_pending"; - node.sequence++; - writeNode(s, node); - tree(s); + const node = + s.node === null ? t.root : required(t.children.get(s.node), "Missing retained child node"); + if (node.phase !== "settled") { + node.phase = "cancel_pending"; + node.sequence++; + writeNode(s, node); + tree(s); + } + // These immutable attempted identities are the adapter's cancellation + // outbox, including lost admissions. No descendant evidence is released. + return Object.freeze( + [...t.grants.values()] + .filter((grant) => { + let parent = grant.parentNode; + if (s.node === null) return true; + while (parent !== null) { + if (parent === s.node) return true; + parent = t.grants.get(parent)?.parentNode ?? null; + } + return false; + }) + .map((grant) => Object.freeze({ ...grant.child })), + ); }); } @@ -1288,12 +1443,11 @@ export async function cancelArtifactTargetLease(lease: TargetLeaseAuthority): Pr export function verifyConfigurationHelperReady(lease: BorrowedArtifactTargetLease): void { const s = stateOf(lease); const t = tree(s); - const grant = t.grants.get(s.node!); + const grant = s.node === null ? undefined : t.grants.get(s.node); if ( grant?.recipient !== "configuration-helper" || grant.phase !== "configuration-handoff" || - t.root.phase !== "held" || - t.children.get(s.node!)?.phase !== "held" || + !forwardAncestry(t, s.node) || Date.now() >= grant.child.deadlineAt ) throw Error("Configuration helper cannot enter drain"); @@ -1303,14 +1457,10 @@ export function verifyConfigurationHelperReady(lease: BorrowedArtifactTargetLeas /** Observation/adoption of an old identity is not permission to start another forward effect. */ export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): void { const s = stateOf(lease); + if (s.verifiedAuthority) assertControlExecutionForward(s.verifiedAuthority); const t = tree(s); const node = s.node === null ? t.root : t.children.get(s.node); - if ( - !node || - t.root.phase !== "held" || - node.phase !== "held" || - Date.now() >= s.execution.deadlineAt - ) + if (!node || !forwardAncestry(t, s.node) || Date.now() >= s.execution.deadlineAt) throw Error("Target lease permits observation/settlement only"); } @@ -1335,8 +1485,7 @@ export function verifyRetainedArtifactTargetSettlement( const child = t.children.get(adoption.delegationId); const proof = childSettlementState(verifyControlEffectSettlement(installation, execution)); if ( - !child || - child.phase !== "settled" || + child?.phase !== "settled" || !knownOutcome(child.settlement) || controlDigest(child.execution) !== controlDigest(execution) || controlDigest(child.settlement) !== controlDigest(proof.evidence) diff --git a/packages/core/src/build-artifacts/test-fixtures/peer-process.ts b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts index 0fe64eb98..c1222ba3d 100644 --- a/packages/core/src/build-artifacts/test-fixtures/peer-process.ts +++ b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts @@ -1,13 +1,42 @@ +import { existsSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { createInterface } from "node:readline"; import * as c from "../control-authority"; +import { AnchoredDirectory } from "../filesystem"; import { inspectControlAdmission, verifiedParentForArtifactDelegation } from "../peer-admission"; import * as l from "../target-lease"; for await (const line of createInterface({ input: process.stdin })) { try { const input = JSON.parse(line); - if (input.mode === "owner") { + if (input.mode === "observe-interleaving") { + const installation = c.openControlInstallation({ + infraDir: input.infraDir, + role: "data-manager", + serviceIds: ["motis"], + }); + const read = AnchoredDirectory.prototype.read; + let paused = false; + AnchoredDirectory.prototype.read = function (name, max, privateMode) { + const bytes = read.call(this, name, max, privateMode); + if (!paused && this.path.endsWith("/observations")) { + paused = true; + writeFileSync(input.ready, "ready"); + const until = Date.now() + 10000; + while (!existsSync(input.resume)) { + if (Date.now() >= until) throw Error("Interleaving resume timed out"); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); + } + } + return bytes; + }; + l.inspectArtifactDelegation({ + dataRoot: join(input.infraDir, "data"), + assertion: input.assertion, + installation, + }); + process.stdout.write(`${JSON.stringify({ ok: true })}\n`); + } else if (input.mode === "owner") { const installation = c.openControlInstallation({ infraDir: input.infraDir, role: input.role, @@ -49,7 +78,7 @@ for await (const line of createInterface({ input: process.stdin })) { serviceIds: ["motis"], }); l.verifyConfigurationHelperReady(lease); - process.stdout.write(JSON.stringify({ ok: true, drainPermitted: true }) + "\n"); + process.stdout.write(`${JSON.stringify({ ok: true, drainPermitted: true })}\n`); } else if (input.mode === "fixed") { const installation = c.openFixedChildInstallation({ infraDir: input.infraDir, @@ -82,6 +111,7 @@ for await (const line of createInterface({ input: process.stdin })) { serviceIds: intent.serviceIds, }); l.bindBorrowedControlExecution(lease, authority); + if (input.action === "cancel") await l.cancelArtifactTargetLease(lease); if (input.action === "run") c.beginFixedCliExecution(installation, input.execution); let assertion: string | undefined; if (input.child) assertion = await l.delegateArtifactTargetLease(lease, input.child); @@ -103,17 +133,53 @@ for await (const line of createInterface({ input: process.stdin })) { role: input.role, serviceIds: ["motis", "osrm"], }); - const admission = inspectControlAdmission(installation, { - rawHeaders: ["x-openmapx-control-delegation", input.assertion], - transportPrincipal: - input.transportPrincipal ?? - (input.role === "ops-agent" ? "data-manager" : "shared-data-manager-bearer"), - kind: input.kind, - operationKey: input.execution.operationKey, - operationFingerprint: input.execution.operationFingerprint, - requestedDeadlineAt: input.execution.deadlineAt, - }); + const admission = + input.role === "ops-agent" && !input.primitive + ? inspectControlAdmission(installation, { + rawHeaders: ["x-openmapx-control-delegation", input.assertion], + transportPrincipal: + input.transportPrincipal ?? + (input.role === "ops-agent" ? "data-manager" : "shared-data-manager-bearer"), + kind: input.kind, + operation: input.operation, + operationKey: input.execution.operationKey, + operationFingerprint: input.execution.operationFingerprint, + requestedDeadlineAt: input.execution.deadlineAt, + }) + : (() => { + // Primitive protocol fixture, not the future generation-bound DM HTTP adapter. + const inspected = l.inspectArtifactDelegation({ + dataRoot: join(input.infraDir, "data"), + assertion: input.assertion, + installation, + }); + const peer = c.verifyControlPeer(installation, { + rawHeaders: ["x-openmapx-control-delegation", input.assertion], + transportPrincipal: + input.transportPrincipal ?? + (input.role === "ops-agent" ? "data-manager" : "shared-data-manager-bearer"), + execution: input.execution, + kind: input.kind, + }); + return { + ...inspected, + parent: verifiedParentForArtifactDelegation( + installation, + input.assertion, + input.execution, + ), + adopt: (execution: typeof input.execution) => + l.adoptArtifactTargetLease({ + dataRoot: join(input.infraDir, "data"), + assertion: input.assertion, + peer, + execution, + serviceIds: inspected.serviceIds, + }), + }; + })(); const lease = await admission.adopt(input.execution); + if (input.forward) l.assertArtifactTargetLeaseForward(lease); const authority = c.retainAdmittedControlExecution(installation, { execution: input.execution, kind: input.kind, @@ -139,7 +205,9 @@ for await (const line of createInterface({ input: process.stdin })) { ); } } catch (error) { - process.stdout.write(`${JSON.stringify({ ok: false, error: (error as Error).message })}\n`); + process.stdout.write( + `${JSON.stringify({ ok: false, error: (error as Error).message, stack: (error as Error).stack })}\n`, + ); } break; } From 28353cdff6d6765c4264b3436d8311a0a3f98665 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 08:37:45 +0100 Subject: [PATCH 012/135] docs(ops): record renewable retention identity constraint --- .../docs/developer/build-artifacts-control.md | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index 0e4918220..44c437c46 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -122,7 +122,11 @@ identities without deleting or settling any descendant evidence. ### Authority storage capacity and retirement -`CONTROL_RETENTION_LIMITS` fixes the installation lifetime policy: +The lifetime limits below are an interim fail-closed bound. They are not an +accepted policy for routine administration: renewable retention remains +unimplemented pending the control-identity contract described below. + +`CONTROL_RETENTION_LIMITS` currently fixes these installation lifetime limits: - At most 1,024 admitted executions per role recovery root, including fixed children sharing the agent root. A signed immutable admission reserves capacity @@ -160,6 +164,41 @@ by this foundation. Older lease layouts without the admission index require supervised indexing; startup never silently excludes their fences. A crashed metadata guard likewise requires supervised recovery. +#### Renewable retention and operation identity + +The current protocol accepts opaque `opk1_` keys and permits an unparented root +to omit its requested deadline. On first admission that root receives a deadline +based on the current admission time. A duplicate instead needs its original +admission time; an expired missing attempt must remain unknown and must not be +retried as a new mutation (control protocol section 2). Section 3.2 also requires +producers never to reuse keys. + +After the per-key admission details are reclaimed, the same root operation/key +without a requested deadline is indistinguishable from a first admission. Fresh +normalization gives it a new deadline and fingerprint. A timestamp retirement +floor cannot identify that old key: both newly assigned times exceed the floor. +Keeping every spent key would recreate unbounded history; a bounded spent-key +cache eventually loses this exact recognition. Deadline expiry still rejects +the original unchanged signed child grant, but does not solve this root case. + +The proposed minimal protocol amendment is an issuer-controlled monotonic birth +sequence encoded in control root keys, with that origin certified through the +existing child authority chains. A durable private retirement floor, with bounded +exceptions for unresolved original authorities, can then reject retired origins +without remembering every old key. The fixed child phase-key hash can remain +unchanged if signed origin evidence accompanies derived child keys. Legacy +undated roots need an explicit cutover rule: retained originals remain readable, +and missing old roots cannot be normalized into fresh authority. Ordinary +transport timestamps are not this birth evidence. + +This is a proposed amendment, not an implemented protocol or a renewal guarantee. +The protocol owner must define the root issuance/cutover and signed origin +binding before retirement can reclaim terminal details and keys. Any resulting +collector must still retain terminal evidence for at least 72 hours and through +authority validity, preserve every unresolved dependency, and coordinate signed +cross-role settlement and crash-safe counters/indexes before reuse. No expiry, +PID check or counter reset is settlement proof. + Settlement requires signed retained structured evidence from the fixed executor: a parsed typed result, exact fixed-command completion, or an execution that has never started effects. Errors alone produce unknown evidence. Arbitrary text, From 1f0824b5052197914b814cfc2ce8c65a1711dabc Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 09:35:54 +0100 Subject: [PATCH 013/135] docs(ops): define renewable control receipt retention --- .../2026-10-23-admin-instance-management.md | 6 +- ...10-25-admin-control-retention-amendment.md | 41 +++++ .../2026-10-23-admin-control-protocols.md | 2 + ...10-25-admin-control-retention-amendment.md | 149 ++++++++++++++++++ 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/plans/2026-10-25-admin-control-retention-amendment.md create mode 100644 docs/superpowers/specs/2026-10-25-admin-control-retention-amendment.md diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index e4b48afe9..d35918011 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -13,7 +13,11 @@ and its binding supplement `docs/superpowers/specs/2026-10-23-admin-control-prot ## Global Constraints -All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Codex-only agents; local rendered verification, no screenshots attached to PR. +All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Delegated work uses only Codex6.1-Sol or ClaudeOpus, preferablyCodex; Astra is prohibited by the latest user instruction. Local rendered verification, no screenshots attached to PR. + +## Approved Task06.1 retention correction + +Task06.1 consumes the [approved retention amendment](../specs/2026-10-25-admin-control-retention-amendment.md) and its [implementation plan](2026-10-25-admin-control-retention-amendment.md). Its fresh root key/origin, bounded renewable family GC, API receipt delivery and atomic stable-step reservation supersede conflicting earlier provisional interfaces;50-task order/scope remains unchanged. Follow-on02.3/02.4/01.2 use its actual exports. Earlier source clearance for02.2 does not qualify the new projectionv3/ack changes: those are reviewed and tested in this06.1 correction window. Agent-first then currentAPI/CLI producer cutover preserves legacy observations while rejecting fresh unsupported authority roots before effects. ## Review Focus diff --git a/docs/superpowers/plans/2026-10-25-admin-control-retention-amendment.md b/docs/superpowers/plans/2026-10-25-admin-control-retention-amendment.md new file mode 100644 index 000000000..409067521 --- /dev/null +++ b/docs/superpowers/plans/2026-10-25-admin-control-retention-amendment.md @@ -0,0 +1,41 @@ +# Renewable Control Retention Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans; steps use checkbox syntax. + +**Goal:** Close Task06.1 I5 with renewable bounded retention, retaining I1–I4 fixes. +**Architecture:** Expiry-bearing root keys plus existing signed origin continuity; bounded per-origin retirement under current role-private CAS. No key-issuing service or lifetime quota. +**Tech Stack:** TypeScript, Node filesystem/crypto, existing Fastify ops and API durable job adapters, Vitest. +**Spec:** docs/superpowers/specs/2026-10-25-admin-control-retention-amendment.md; existing control supplement remains binding elsewhere. + +## Global Constraints + +- Same worktree/one PR, no merge/deploy. Only6.1Sol orClaudeOpus workers, preferCodex; noAstra or nestedsubagents. +- Newrootkey scheme is restrictive expiry metadata, not role/target authority. No operation deadline renewal or mutation replay. +- At least72h terminal retention, every unresolved family protected; no permanent1024 lifetime cap. +- Roleprivate mounts, physical inode/signature/CAS/highwater and fixed graph/depth unchanged. +- No hand-edited generated artifacts; source-first canonical contracts, appropriate runtime/serialization checks. + +## Review Focus + +1. Expired root resurrection after deleting all historical key details (steps1/2). +2. Derived hash accepted as fresh root or unsigned origin gaining authority (step2). +3. Partial family retirement loses another role's only containment receipt (step3). +4. Slot/index/counter crash releases capacity before durable retirement (step3). +5. Old producer sends undated roots or changes keys/deadlines on observer recovery (step4). + +## Task06.1 continuation: dated root origin and renewable receipt families + +**Files:** Create core `src/ops/control-root-key.ts` and test; extend build-artifacts `control-authority.ts`, `control-retention.ts`, `target-lease.ts`, `peer-admission.ts`, server-only exports and focused process fixtures/tests. Modify API `admin-job-ops.ts`/`admin-job-parent.ts`/`ops-client.ts`/`trusted-config-operations.ts`/`job-runner.ts`/`admin-job-state.ts`/`admin-job-failure.ts` and their focused tests; include current job cleanup/delete predicates and `job-runner-postgres.test.ts` only as needed for new root checkpoint/key construction; CLI `control-context.ts` and actual local root producer, agent `server.ts`/admission tests. Keep publication/receiver/product consumers of02.3/01.2/05.3 unavailable until implemented. Developer build-artifacts-control.md records actual supported cutover/retention. + +**Interfaces:** Produce exact `createControlRootOperationKey`/`parseControlRootOperationKey` and internal ControlExecutionOrigin from spec. Extend nominal parent verification with inherited origin internally; do not widen public HTTP input into authority. Add exact API-only `controlRetention.acknowledge` typed operation, core policy/client, agent verifier/store and API post-CAS acknowledgement, as specified in revision2. Keep its generated/public API footprint scoped; it exposes no browser path. Expose a bounded `reconcileControlRetention(installation:ControlInstallation,options?:{signal?:AbortSignal;maxFamilies?:number}):Promise<{retired:number,pending:number}>` for startup/admission maintenance; maxFamilies defaults16, range1–64. Actual per-family manifests/cursors stay private/signed fixed records. Same expiry/origin validation applies when a derived deadline-only child acquires its root lease. + +- [ ] **1 RED:** Convert the retained renewal-replay probe into focused tests in control-authority/retention and pure key tests: exact expired timed root after history deletion rejects; changed deadline conflicts; malformed/noncanonical/over7day keys reject; a caller-supplied origin is not nominal authority. Run `pnpm exec vitest run packages/core/src/ops/control-root-key.test.ts packages/core/src/build-artifacts/control-authority.test.ts packages/core/src/build-artifacts/target-lease.test.ts`. Record causal behavior separately from absent exports. +- [ ] **2 GREEN identity:** Implement pure timed keys, immutable explicit root deadline, signed root/child origin continuity and duplicate/legacy observation rules. Add depth-two/deadline-only independent-root cases with actual processes. No signed issuer counters or new issuance endpoint. Add signed version2 origin records and explicit API-HMAC/new-agent-origin boundary; verify the private retirement-clock before fresh authority. +- [ ] **3 RED/GREEN retirement:** Replace lifetime slots/counters with bounded retained-window family accounting and journaled role-local reclamation. Add terminal72h+expiry renewal, unresolved early family with later retirements, actual role-private cleanup/ack, fully reclaim then roll wall clock back across restart and replay exact key with zero effects, missing/rolled-back private clock, kill/fault between journal/delete/counter/slot phases, stale inode/index/highwater and expired replay after full cleanup. Exercise actual separate processes/filesystem. Rootunknown retains capacity; unrelated terminal families can renew independently. Implement revision2 prepared/ready/committed/clean/final/released states and exact signed participant/API acknowledgements, preallocated2MiB family workspaces/index pages and fixed quota transaction cell. Prove cleanup at full ordinary capacity with no spare record, API lost-terminal response with no checkpoint-copy ack, participant crash/offline return, duplicated/mismatched ack, namespace unlink/count crash and generation reuse. Bounded maxFamilies/admission work with round-robin cursor; no history-sized list/parse. +- [ ] **4 Producers/cutover:** Implement private projectionv3 timeless stepAddress mapping and helper resolution before time/config preparation; persist mapping+finalrootkey+deadline+body+attempt+parent in one CAS before signing/POST, exact recovery remains lookup-only. Test explicit address, response-loss/restart after desired config changes, legacyv1/v2 entries and ambiguous pre-checkpoint outcome. Persist terminal evidence copy then send/reconcile exact API acknowledgement; unresolved API parents retain their copy. Existing undated retained jobs only observe/cancel; actionable upgrade for new authority roots. Adapt current CLI/local root factories; inherited fixed children keep original phase hash and verified origin. Focused actualFastify/APIcheckpoint/CLItests prove zero effects for unsupported roots and no stale deadline refresh. Implement revision3 pending/confirmed receiptDelivery, cached-terminal ensure helper, job terminalization SQL gate and pending-copy cleanup guard. Use exact-proof acknowledgement retry and post-GC expired/no-dependency discharge; no new outbox table. Add terminal-copy digest/identity, lost request, persisted response loss, cached restart, canceled/failure completion and pending-cleanup tests. Implement fourth checkpoint stepAddress parameter, atomic owner/parent/step-absence reservation, currentDB JSON merge, pure attempt preparation and distinct JobStepAllocationConflict behavior. Add real PostgreSQL same-owner/same-address/different-deadline race, ambiguous first checkpoint and distinct-address preservation; at most one fresh mutation POST. Add current-DB atomic capacity predicates for both maps and JobStepCapacityExceeded classification. RealPG races start at63 entries with same-owner distinct addresses/retained parent; at most64 entries and one last-slot POST, preserving all existing receipts. Cover full rows, mixedlegacy/v3 capacity and same-address winner observation at capacity. No caller-snapshot-only bound or post-write rejection. No original mutation POST may occur during receipt delivery recovery. No broad SDK migration beyond this root producer seam. +- [ ] **5 Qualification/selfreview:** Covering core/CLI/agent/API focused tests and affected package types/policy checks, then selfreview exact changes. Save exact command/output/exits and platform limitations; no fullsuite/runtime acceptance from mocks. Aggregate DB/Linux/Docker/build/browser/fullSDK/CI remain controller gates. Append existing task06.1 report and local commits only with normal hooks. +- [ ] **6 Review:** Controller generates full5233d5f40..FINALHEAD fix package and NEW scoped independent reviewer against I1–I5, M1, this amendment and fix-caused breakage. No downstream02.3 until all material foundation issues clear. + +## Self-review + +Revision2 maps clock rollback, explicit retirement/API participant states, reserved cleanup quota and timeless producer lookup to steps2–4 with named counterexamples; revision3 decides the remaining API pending-ack completion/cleanup and database stable-step CAS/race contracts; scoped independent design review completed with0openfindings; code still requires the task-scoped independent review after implementation. New fields stay inside signed control metadata; no public ExecutionRef shape or unlimited tombstone store. Original task66-scope ledger and50-task order remain unchanged: this is Task06.1 fix1, with no silent deferral. The root acceptance rule/legacy compatibility change is explicit and independently reviewed before code. Implementation may propose a smaller equivalent finite retirement layout, but cannot change the expiry/unknown/role/72h guarantees without a recorded controller ruling. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index e4b248e1f..8fa3cadb1 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -2,6 +2,8 @@ Date: 2026-10-23. Binding supplement to `2026-10-23-admin-instance-management-design.md` and the controller's `plan-fix-rulings-r1.md`. This is an implementation contract, not an implementation or test receipt. Tasks 01.2 and 02.1–02.4 produce the interfaces assigned below; 06.1 implements the shared filesystem protocol. Later plans consume these names rather than inventing parallel locks, command runners, secret stores or upload services. +The approved [renewable control retention amendment](2026-10-25-admin-control-retention-amendment.md) supersedes this supplement only for fresh authority root keys, signed origin propagation, renewable receipt-family retention, API receipt acknowledgement and stable-step checkpoint reservation. Apply all revision2–4 decisions together. Legacy undated identities remain observation/settlement-only; do not use the earlier optional-root-deadline language to mint new managed authority. All other contracts below remain binding. + ## 1. Source constraints and ownership - `configuration-ownership.ts:acquireConfigurationOwnership` already uses exclusive mkdir, a 5-second wait, inode checks and no stealing after a crash. `configuration-generation.ts:withConfigurationGenerationWriter` owns the current pointer and commit evidence. Extend this discipline; PID age and TTL are not ownership. diff --git a/docs/superpowers/specs/2026-10-25-admin-control-retention-amendment.md b/docs/superpowers/specs/2026-10-25-admin-control-retention-amendment.md new file mode 100644 index 000000000..206b86406 --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-admin-control-retention-amendment.md @@ -0,0 +1,149 @@ +# Renewable control retention amendment + +Status: APPROVED for implementation on2026-10-25. Controller self-review and independent review rounds1–4 resolved all material findings: R2 closed clock/quota, R3 closed API delivery/stable-step CAS, R4 closed atomic projection capacity;0 open Critical/Important/Minor. Scope is Task06.1 I5. This is design/buildability approval, not code or runtime acceptance. User explicitly delegated design decisions and waived question/approval gates. + +This amendment supersedes only the opaque fresh control-root key/capacity paragraphs in the2026-10-23 control supplement. Original typed effect graph, role-private trust, body/execution fingerprints, immutable deadlines, ownership/settlement and no-replay rules remain. Existing undated records never receive invented origin metadata. + +## Intent and decision + +An installation must renew terminal evidence storage through ordinary successful use. A permanent1024-operation lifetime quota does not meet remote administration. At least72h terminal evidence and every unresolved/unknown family remain protected. Reclaimed old authority must never become a fresh effect. + +Three approaches were evaluated: + +1. Delete old opaque identities behind a time watermark: rejected. The real sequential probe in task-06.1-report.md demonstrates same-key/body admission receiving a fresh deadline after deletion. +2. Certified issuer sequences and retirement floors: feasible in principle, but require new issuance, cutover and cross-role sequence machinery. A counter assigned only at admission does not identify replayed old opaque keys. +3. Self-describing immutable root expiry, bound by existing authenticated/signed identity: selected. The key itself retains its last admissible time, independent of stored historical membership. Changing that time changes the key. A caller-chosen expiry is only a restrictive validity input, never role/target/privilege authority; server policy and existing signatures/bearers still enforce access and actual execution budget. This adds no key-minting network service. + +## Root key contract and admission + +`createControlRootOperationKey({notAfter:number,nonce:Uint8Array}):string` and `parseControlRootOperationKey(key:string):{notAfter:number,nonce:string}` live in core ops/control-root-key.ts (browser-safe pure format/hash helpers; no private trust export). Format is `opk1_t1_<14 lowercase hex expiry digits>_<22 canonical base64url nonce characters>`. Nonce is exactly16bytes, encoded without padding. Expiry is a positive safe epoch-ms integer, canonically zero-padded to14hex digits. Root suffix length40 fits the existing opk1 validator; the exact32-character phase-key hash format remains distinct and unchanged. No expiry is derived from HTTP issuedAt/expiresAt. + +A fixed producer chooses the root absolute requested deadline and key BEFORE its first mutation submission and persists both before dispatch. API uses a deterministic16byte nonce from domain-separated SHA256(namespace,durableAPIjob/step identity); host CLI/cron uses cryptographic16byte randomness persisted with its local attempt. Observers reuse the exact checkpoint. No renewal, same-key fresh deadline, or prepare-after-lost-mutation fallback exists. + +For a previously unseen authority-bearing root, require a timed root key, explicit `requestedDeadlineAt === key.notAfter`, and `0 < notAfter-now <= 7days`. Normalized runtime deadline remains the minimum of requested, deployment/kind and verified API-parent bounds; the key cannot enlarge it. An elapsed key is expired/unknown before any admission reservation/effect, even if its historical details are gone. Duplicate retained reads/cancel/settlement remain possible after expiry; they never authorize new forward work. Existing exact retained normalization/conflict rules remain. + +A derived fixed child keeps the existing phase-key formula and must carry verified inherited origin through its role-signed parent/lease/fixedCLI chain. An undated hash is not admitted as a new standalone root. A deadline-only child acquiring its independent target lease inherits the same origin, even though its executor differs. Its local durable execution and key are still its original distinct identity. + +`ControlExecutionOrigin = {version:1,rootKey:string,issuer:ControlIssuer,notAfter:number}` is internal non-secret metadata, signed within existing execution admissions/witnesses, lease certificates and child grants. Root construction derives it from the authenticated fixed executor and parsed key; child construction derives it solely from the verified parent's retained signed origin. No request boolean, JSON origin object, unsigned env field or cast creates nominal origin authority. Every child/retained duplicate checks exact origin continuity and deadline<=origin.notAfter. Public ExecutionRef shape need not change; origin is separate signed immutable authority evidence and a bounded private retention index key. + +## Renewable bounded retirement + +Keep bounded active and retained capacities, with before-effect refusal if capacity genuinely cannot be reclaimed yet. Capacity is retained-window/unresolved pressure, never a lifetime operation counter. Roots still valid, families terminal for less than72h, and any unresolved/attempted/unknown effect or descendant cannot be reclaimed. An unadmitted issued root is not a mutation attempt, but its partial private admission must be conservatively contained before retirement. + +Use existing per-role private roots, shared signed ledgers and writer guards; no generic service. A bounded per-origin family index records the exact fixed namespaces/basenames/keys/slot identities owned by that family (no arbitrary paths), plus durable GC phase/cursors. New writes reserve index/counter capacity before publication. Size/count limits and maximum fixed descendants remain enforced. Admission looks only at bounded active slots/index pages; retirement is incremental and must not turn its5s admission budget into historical synchronous scans. + +An origin family is eligible only after: (a) `now >= origin.notAfter`; (b) at least72h since its last authoritative terminal transition; (c) every local effect, lease and attempted descendant has authenticated terminal/contained evidence; (d) root and all shared target nodes are authoritatively settled with exact physical ownership, and no helper/stream/worker handle can still perform effects; and (e) cross-role evidence/settlement needed by any still-unresolved parent has been retained or acknowledged. Age/deadline/PID alone never supplies c–e. + +Retirement uses a bounded, signed journal and exact ownership/CAS. First persist the irrevocable expired/no-forward decision and complete evidence manifest; then reclaim only listed settled private/shared records and keys, with role-local ownership; finally return index/namespace/history slots. Crashes leave resumable exact progress or safe refusal, never a free slot without durable no-replay/containment evidence. Other roles acknowledge their own private cleanup; no role writes or reads sibling-private material. Before removing the final shared evidence, each participating role must either hold the exact signed terminal/retirement proof it needs or acknowledge that it no longer needs it. A missing/corrupt participant remains fenced; it is not treated as cleaned. A bounded pending-retirement family may consume capacity until the role returns and reconciles, without requiring ordinary successful families to stop the host. + +A retired timed root can be recognized as expired directly from its unchanged key. Retired undated child hashes cannot become roots and cannot be newly adopted without the original valid signed chain; expired/missing chain produces unknown/expired. Reclamation does not require one permanent tombstone per old key, issuer sequencing, a lossy membership filter or reset of installation trust. Preserve at least72h exact lookup; afterward return expired/unknown without manufacturing not-started/canceled success. + +## Producer and compatibility cutover + +Task06.1 owns pure key/origin/retirement foundation, current host/fixedCLI constructors, actual agent receiving adapters, and current API root operation-key producer where necessary for coherent use. Task02.3/02.4 and01.2 consume the same helpers in their new durable data/config workflows; do not invent competing keys or origin stores. API CAS checkpoints both deadline and final timed key before signing or submitting. No root issuance HTTP endpoint is added. + +Existing v1/v2 status, lookup, cancel, result/failure wire shapes stay readable. Already-retained undated operations remain observation/settlement-only and are never replayed or stamped with a guessed origin. Missing old undated roots cannot enter the new authority path: reject before mutation with a fixed actionable client-upgrade prerequisite. Ordinary non-authority read operations remain compatible. This explicitly narrows the earlier old-client/new-agent promise for fresh authority-bearing mutations; the upgrade must install agent verification first, then current API/CLI producers, before those actions are offered. Preserve legacy wire parsing rather than silently normalizing old keys as fresh authority. An unsupported root-key scheme is a prerequisite rejection, not backend success or an unknown attempted effect. + +Existing pre-index/private protocol records are experimental/unreleased on this branch. Retain them fail-closed for exact lookup/settlement and supervised reconciliation; never silently migrate them into forward authority. New retention cannot delete them merely because they lack origin. Test fixtures for new authority use timed keys or verified inherited origin. Production checks must explicitly distinguish legacy reading from current managed admission; no claim of transparent old undated mutation support. + +## Self-review and review focus + +This removes the exact-membership contradiction by changing the fresh root identity acceptance rule, rather than pretending a timestamp authenticates a role. Exact old key expiry is immutable in the request key and rejected against the durable retirement floor below; changing it is a different authenticated operation. New kind/deployment clamps remain independent. Derived descendants cannot escape through an undated root fallback. Family GC must not remove another unresolved role's only evidence. No remaining audit scope is deferred. + +Review/test counterexamples: delete expired terminal details then re-normalize exact root key; change a requested deadline while retaining key; take an old phase hash as a fresh root; recreate a child through a deadline-only parent after root retirement; keep one unresolved early family while many later families expire; kill each role between signed retirement and private deletion/slot reuse; offline participant returns; duplicate observation during retirement; stale index/high-water/slot inode; missing private evidence; prepared helper still active. All must reject or retain the original fenced state without duplicate effects or unbounded history growth. + +## Revision2 binding decisions — amendment review I1–I4 + +The following concrete contracts resolve the review; they supersede any less specific language above. This is still a reviewed-before-code amendment, not approval of an implementer's future collector. + +### Durable expiry barrier (I1) + +Each role has one installation-bound signed private `retention-clock.json`, preallocated during verified empty-store bootstrap: `{version:1,installationId,role,sequence,lastRetirementWallTime,rejectNotAfterThrough}`. Sequence is a safe nonnegative monotonic integer; the time fields are safe epoch-ms integers. The private writer/physical identity and private high-water/CAS discipline are the same as existing execution state, including bounded read, no-follow, single link and fsync. Missing/corrupt/rolled-back clock state in a nonempty store disables fresh control authority and retirement; it never silently bootstraps a zero floor. A simultaneous rollback of the trusted host's entire private installation/key backup is outside online recovery and requires the existing supervised restore boundary. + +Before a role deletes any replay evidence for an origin, it durably advances `rejectNotAfterThrough` to at least that origin.notAfter and `lastRetirementWallTime` to at least the current retirement wall time, under the same journal transaction; other participant roles do so before their private deletion too. Every fresh root admission/retained descendant forward authorization rejects origin.notAfter<=the relevant durable floor. While wallNow:`. Maximum128 executions and256 total retained child intents per origin (existing tighter per-node64/depth2 limits still apply). Max3 role participants plus1 API parent. A larger prospective family is refused before its extra effect/admission. TerminalAt is the latest authoritative terminal transition, not first completion or observer close. + +The API dependency is satisfied only after the API durably CAS-copies this operation's exact terminal identity, normalized deadline, safe result/failure/publication evidence and digest into its private job checkpoint. An unresolved API parent retains that copy; terminal parent cleanup preserves it at least72h afterward. Missing/uncertain DB write sends no acknowledgement. Add fixed API-only sync15s typed operation `controlRetention.acknowledge` with `{originRootKey,operationKey,fingerprint,terminalDigest,proof}`; request<=16KiB, result `{acknowledged:true}`. `proof` is the existing API ops-token HMAC with domain `openmapx-api-retention-ack-v1\0`, over strict canonical fields `{version:1,apiJobId,originalApiOwnerVersion,attemptId,checkpointOwnerVersion,terminalCopyId,operationKey,fingerprint,terminalDigest}`. TerminalCopyId UUID and owner version are persisted in the same successful checkpoint CAS before minting. Agent validates API role, HMAC, exact original parent/attempt/operation and its current terminal digest, and persists the ack in the preallocated family cell. Different identity/digest conflicts. Response loss is safe to observe/repeat this exact proof; it cannot dispatch a host effect. No browser route accepts it. + +Core server-capable ops exports `controlTerminalReceiptDigest(status:OpsJobStatus):string`. It validates version2 terminal `succeeded|failed|timed_out` with the existing strict per-kind result/failure parser, then domain-separated SHA256 of canonical `{version:2,operationId,operationKey,kind,admittedAt,deadlineAt,fingerprint,state,terminationRequestedAt:null|string,errorClass:null|string,result:null|typedResult,failureResult:null|OpsFailureResult}`. Missing optional fields use null; progress/diagnostics/update time are excluded. API and agent use this identical helper. The private copied terminal receipt is bounded128KiB, retains safe publication evidence inside the existing result/failure contract, and is not sent in the acknowledgement. + +The API ack may arrive before the GC manifest exists: it binds exact operation+terminal digest and original attempt, and the later manifest includes that same executionReceiptDigest. It does not claim knowledge of an unfrozen manifest. An API parent still unknown with no durable receipt copy prevents retirement of that family, even if all three role workers are locally terminal. This is independent of unrelated eligible families. API observer recovery retries acknowledgement from the retained checkpoint; it never reissues the original mutation to obtain it. + +### Retirement state machine and acknowledgement authority (I2/I3) + +Every participating role writes only its own role-private progress and its separately signed shared acknowledgement cell. Coordinator writes the shared decision cell. Paths derive solely from validated origin/family/role IDs under fixed roots; sibling-private access remains forbidden. Shared participant ack payloads are `{version:1,installationId,familyId,generation,originDigest,manifestDigest,role,phase,terminalSequenceDigest,recordSetDigest,proofCopyDigest,sequence}`, signed by that role's fixed key. Phase is `ready|clean|released`; sequence is monotonic. Exact manifest/generation/digests must match; an ack for another family, state or prior slot occupant is invalid. + +| State | Owner and durable action | Recovery / next condition | +| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| active | Each role's index reserves workspace before its first family write/effect; grants enlist before exposure. | No GC while forward-valid, young, uncontained or external API dependency unacknowledged. | +| prepared | Coordinator closes membership, verifies all terminal/contained evidence, persists signed manifest and irreversible expiry barrier; shared prepared decision references exact manifest. | Missing/uncertain proof remains pending; no private/shared deletion. | +| ready | Each enlisted role verifies manifest/local terminal records, copies needed signed terminal/manifest proof to its preallocated private GC cell, then publishes signed ready ack. | Copied proof cannot be removed while the role depends on shared state. Unused offline configured roles are not participants. | +| committed | Coordinator verifies every role ready plus exact API ack if present, then persists/publishes signed commit referencing all ready digests. | This decision is immutable. A process restarted with only prepared state cannot assume commit. | +| clean | Each role copies commit locally, advances its private expiry barrier, journals/deletes only its exact listed settled records/keys and updates quotas under the fixed transaction below; retains its GC proof cell. Publishes clean ack after durable completion. | Shared semantic evidence stays until every role clean. Any uncertainty remains pending with slot reserved. | +| final | Coordinator verifies every clean ack, writes signed final decision, then journals/removes the exact now-unneeded shared semantic records/history; final decision stays until released acknowledgements. | All participants already own proof independent of semantic shared files. | +| released | Each role copies final locally, completes its private accounting and publishes released ack. Only after that can its local GC workspace be freed; interrupted local cleanup resumes from its local final copy. | Coordinator reuses its shared decision/family slot only after every role released. An offline enlisted role delays only its retained family. | + +The API ack is evidence-copy-before-ack, not a permission to erase an unresolved API parent's DB copy. Its digest participates in prepared/commit proof; a fourth role-private mount is unnecessary. After all released a role's GC proof has no remaining dependent parent: API holds its own copy, role children/root are terminal and retired, and the expired self-describing key plus durable floor prevents fresh replay. Therefore retirement cells and keys can be reclaimed without a permanent tombstone. A role that crashes after released but before local workspace removal uses its already-private final copy and slot generation; it needs no deleted shared evidence. + +Observers participating before membership closes retain exact proof via existing admission/attempt identity; ordinary human read requests hold a transient descriptor guard, not a permanent new participant. Deletion waits for those descriptor callbacks to settle. Late reads after the72h/expiry retirement boundary return expired/unknown, never a fabricated terminal result. Missing enlisted roles/acknowledgements, attempted unaccepted children and corrupt records remain pending; PID age is not cleanup authority. + +### Reserved collector workspace and exact accounting (I3) + +Ordinary capacity remains1024 retained families per role and256 active leases, with256 retained child intents/128 executions per family as above. Before exposing its first family assertion/admission, each role reserves a family slot and preallocates its GC progress/proof cell outside ordinary namespace counts. Shared coordinator and max3 participant ack cells are preallocated at root admission/enlistment. Reserving the last ordinary record cannot consume these cells. Slot inode+generation UUID is retained and checked on every reuse. + +Per-role/per-family index records at most4096 owned records/keys in64 pages of64 entries, each page<=16KiB; each entry is a fixed namespace+validated basename+record digest/inode+ordinal. Total reserved GC/index metadata is capped2MiB per role/family, including one<=128KiB progress/proof cell and prebudgeted ack/decision records. These bounds are separate from ordinary128KiB receipt limits. Index page capacity is reserved at family admission; publishing a new ordinary record requires its ownership entry/creation reservation first. Only the enumerated fixed namespaces are allowed. GC reads a page at a time and maxFamilies defaults16/range1–64. A durable round-robin slot cursor advances past pending families so an early unresolved family cannot starve later eligible families. No admission lists/parses lifetime history. + +Retirement creation/updates use preallocated cells, even at full ordinary quota. Physical IO failure/ENOSPC still causes safe pending/refusal, not a false freed slot; deployment disk-resource qualification remains required. No new retirement record per completed operation survives final workspace release. + +Namespace quota changes use one fixed preallocated transaction cell under that namespace's existing owned writer guard. Payload identifies family slot/generation, manifest digest, record ordinal/inode, before/after count and phase `planned|unlinked|counted|item_done`. Persist planned before unlink. Unlink only exact inode/owner and fsync directory. Persist unlinked, then counted with the exact decremented count; keep that same transaction identity until the family progress cell durably records item_done. Clear it only afterward, before another adjustment. A crash after unlink is accepted as this authorized deletion only with the matching planned transaction, not generic ENOENT. A crash after decrement cannot decrement twice because counted transaction is retained until item_done. Replaced inode/digest or mismatched counter/version fences recovery. Counter decrease never precedes physical deletion. + +History/namespace/family slot release similarly journals the exact slot inode/generation and returned-capacity phase before reuse; a new occupant has a different generation and cannot be deleted or decremented by an old continuation. Uncertain writer containment cannot steal its guard; existing verified recovery authority must prove the exact writer contained before finishing its retained transaction. This may require the documented offline failure-recovery boundary after a real uncontained control-plane failure; ordinary success/full quota does not. Tests distinguish contained crash resumption from unknown ownership. A restart uses only this exact journal, never recomputed current counts or unconditional counter reset. + +### Stable API step address and checkpoint version3 (I4) + +Keep `createDurableOpsKey(namespace,durableIdentity)` as a stable logical step address. `ExecuteAdminJobOperationOptions.operationKey`, when supplied by existing API callers, is that address as well; it is not directly translated at the transport after lookup. Add private `OpsProjection` version3 with `steps:Record` and `operations:Record`, at most64 entries in each. Existing operation/result/event bounds remain. Mapping and complete typed operation/attempt/parent checkpoint are committed in ONE existing owner-fenced CAS before signing/POST. + +`executeAdminJobOperation` first computes the timeless stepAddress and reads/resolves the mapping before reading time or creating input. A retained mapping wins: reuse exact final key/deadline/body/attempt and observe only. `retainedAdminJobOperation`, `hasRetainedAdminJobOperation`, `observeRetainedAdminJobOperation`, resume and cancellation helpers all resolve stable address first; old literal final-key lookups remain usable internally. Trusted configuration's pre-preparation gate at trusted-config-operations.ts uses this resolver before rebuilding desired config or taking a fresh snapshot. Its `_configurationOutcomes` can remain keyed by the stable address. + +For an absent address only: choose/persist the parent budget and requested absolute child deadline, derive the timed key using that deadline plus deterministic16byte nonce from jobID+stable address, prepare its exact API attempt, then CAS-save mapping+operation+attempt+parent together. A failed/unknown checkpoint never signs or POSTs; DB reconciliation must resolve the original row before another owner treats the address as absent. After any attempted POST, recovery never creates a replacement mapping or re-prepares desired input. A caller's changed supplied address is a different explicit logical step; the same address cannot renew a deadline. + +Legacy projections versions1/2 resolve their existing `operations[address]` directly; never invent a mapping/deadline/origin for them. Already-attempted old entries are lookup-only. A retained pre-admission legacy entry without conclusive not-submitted proof remains unknown/prerequisite; it is not normalized as a new timed root. Fresh v3 entries need a real durable JobContext owner; mocks must supply that context rather than bypassing it. Signed fixed control children outside this API-root broker retain their exact phase key; this address mapping does not re-key delegated children. + +## Revision3 binding decisions — remaining amendment I2/I4 + +### Recoverable API receipt delivery obligation (I2) + +Choose completion gating in the existing durable API job, not a new outbox table/service. Each fresh v3 projected operation stores `receiptDelivery: {state:"pending"|"confirmed", terminalCopyId:UUID, checkpointOwnerVersion:number, terminalDigest:string, terminalReceipt:OpsJobStatus}`. The original attempt/API parent plus the exact checkpoint owner version/copy ID are retained for deterministic re-signing. A terminal receipt and pending delivery obligation are CAS-written together before sending any acknowledgement. The terminal status is validated and bounded128KiB as above. Public projection strips this private metadata; existing safe operation failure/result remains truthful. + +EVERY terminal consumption path calls `ensureAdminJobReceiptDelivery` before returning/rethrowing a cached result/failure: newly observed terminal, cached terminal on restart, terminal-before-cancel, and failure/cancellation reconciliation. Exact pending proof may be resent; the original mutating operation may not. Ordinary observer/worker abort does not abort the durable delivery obligation; its network attempt uses a separate bounded15s observation signal, and failure schedules the existing bounded job reconciliation/backoff. Parent stays `running` or `cancel_pending` with `JobObservationPending("receipt_delivery")` while delivery or its confirmation checkpoint is uncertain. Do not misclassify receipt delivery as unknown backend effects: the authenticated terminal operation result is already in the checkpoint. Its final outcome is preserved and returned/rethrown only after delivery is durably confirmed. Success that preceded cancellation remains success; a known failure remains failure. + +`job-runner.ts` enforces this completion gate on every success/failure/canceled finalization in BOTH code and its owner-fenced SQL predicate: no terminal write while any v3 projected receiptDelivery.state is pending. The check uses current DB JSON, not only the handler's stale result. Existing reconcile selects this nonterminal job and resolves only retained operations/receipt obligations; it never repeats a handler mutation. `ensureAdminJobReceiptDelivery` executes on cached terminal entries before their current early-return branch. On confirmed agent response, CAS-mark confirmed; a lost confirmation write leaves pending for exact retry. All admin-job cleanup/expiry/delete predicates prohibit deleting the last terminal copy while pending, regardless of parent status. Correct new code cannot create a terminal parent with pending delivery; existing terminal v1/v2 jobs do not acquire an invented acknowledgement obligation. A missing/invalid pending copy is a recovery-required job, not a fresh action. + +Lost response after agent persistence must not strand an API job after GC. `controlRetention.acknowledge` reaccepts the exact valid HMAC proof in these finite cases: (1) retained family/operation exists: exact API parent/attempt/terminal digest match, persist or observe identical ack; (2) no retained family or partial-index/GC entry exists, root key is syntactically timed and originRootKey==operationKey, and its expiry is at/below the durable role rejection floor: return `{acknowledged:true}` as DISCHARGE OF A RECEIPT DEPENDENCY ONLY, creating no retained record or authority. The proof must still validate against the fixed API token and its exact canonical fields. The API already owns the authenticated terminal copy. This response asserts neither operation success nor that an unknown job ran; it only states the agent has no retained dependency to await. If any index/GC/semantic record suggests missing/corrupt/partial retained state, return pending/recovery-required instead. No absence-only discharge for an unexpired key or unsupported legacy key. Never reconstruct terminal effects from404/410. + +Thus an ack persisted but response lost can finish after full GC without one ack tombstone per historical key. If the request never persisted, the retained family cannot have retired because its API dependency is still pending; the resend compares and writes that exact acknowledgement. The API clears its delivery obligation only on the authenticated success response followed by confirmed checkpoint CAS. After confirmed delivery, unresolved API parents still retain their copied terminal evidence; terminal parents retain it at least72h. Cleanup cannot race a pending obligation. This is routine retry of a no-host-effect acknowledgement, never replay of the original command. + +### Atomic stable-address reservation and same-owner races (I4) + +Extend `JobContext.checkpoint(result,progress?,admissionKey?,stepAddress?)` with the fourth optional stable step address; existing legacy three-argument behavior remains for v1/v2 records. For fresh v3 allocation, admissionKey is the candidate final timed key and stepAddress is the timeless logical address. `job-runner.ts` performs ONE owner/version/active-status fenced DB update with all predicates: (a) `opsProjection.steps[stepAddress] IS NULL`; (b) `opsProjection.operations[stepAddress] IS NULL` (legacy address protection); (c) `opsProjection.operations[admissionKey] IS NULL`; (d) retained parent is absent or exactly the candidate parent; (e) fresh-admission cancellation guard still permits new work. Validate the complete v3 mapping/body/attempt/deadline before this update. It is not the old final-key-only null test. + +The DB update merges only this reserved step/mapping and its operation into the CURRENT DB JSON; it does not replace a caller's whole stale opsProjection. Preserve other addresses, operations, event counters, observer lease/owner state and unrelated checkpoint members. Return the full updated checkpoint from DB and assign ctx.checkpointResult only after confirmed success. Parent-attempt preparation is pure until this CAS: `prepareApiJobChildAttempt` must not mutate ctx.checkpointResult or publish a parent before the allocation wins. Candidate parent fields are included in the same atomic update. + +If no row updates: distinguish actual owner loss from stable-step/parent conflict by a fenced read of the owned row. Owner loss retains existing JobOwnershipLost behavior. Same-owner losing allocation refreshes ctx.checkpointResult from the winning DB mapping and raises distinct `JobStepAllocationConflict`, which the broker resolves by observing the winning original key/deadline/body with fresh=false (or JobObservationPending if the winner is not yet observable). The loser neither signs nor POSTs and never chooses another deadline. A failed/ambiguous DB write is persistence uncertainty, not a winner or permission for another allocation; reconcile that same address from DB. An overwritten winner is impossible because the stable-address null predicate is atomic. If the first commit is ambiguous, zero submissions is allowed; later lookup remains unknown rather than resubmitting. + +Non-admission v3 projection writes also preserve mappings and unrelated entries using identity-conditioned per-entry updates; a stale whole snapshot cannot erase another address, reverse confirmed delivery to pending, replace immutable input/attempt/deadline or turn terminal back into observing. New entries/mappings are permitted only by the reservation path. Existing serialized checkpointTail is useful local sequencing but is not the database uniqueness proof. Legacy entries are still resolved by literal address before allocation, and final-key internal lookups are read-only aliases. Include concurrent distinct-address preservation as a covering test alongside same-address one-winner tests. + +## Revision4 binding decision — atomic projection capacity (R3-N1) + +The SAME current-DB reservation transaction/update that enforces stable-address uniqueness must validate the POST-MERGE cardinality of BOTH maps: steps<=64 and operations<=64, including every retained legacy operation. It must use current row state under its write/row lock, not the caller's earlier snapshot; candidate validation or post-write parsing alone is insufficient. An equivalent row-locked transaction must check these limits before committing the mapping. No invalid65-entry row may ever be persisted. The no-overwrite/owner/parent/cancellation predicates above remain. + +On a failed reservation, first resolve whether the SAME stable address already has a retained winner; if so observe that exact winner even when the row is full. If that address remains absent and the resulting current-DB map would exceed either limit, raise distinct `JobStepCapacityExceeded` before signing/POST, preserve the current checkpoint byte-for-byte except unrelated valid concurrent writes, and do not allocate a fresh address/deadline, evict evidence or mistake exhaustion for owner loss. This is a known admission refusal, not an unknown submitted effect. Parent finalization still awaits containment and receipt delivery for any prior accepted children. Concurrent distinct-address allocators competing for the64th slot may grant only one reservation; a losing address has no winning mapping to observe. From 2086c92259a863277016abf0acf36f5c702da569 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 12:56:49 +0100 Subject: [PATCH 014/135] fix(ops): renew bounded control authority retention --- .../services/__tests__/admin-job-ops.test.ts | 146 +- .../__tests__/admin-job-renewal.test.ts | 112 + .../__tests__/job-runner-postgres.test.ts | 587 +++- .../src/services/__tests__/ops-client.test.ts | 17 +- .../trusted-config-operations.test.ts | 85 +- apps/api/src/services/activity-retention.ts | 4 + apps/api/src/services/admin-job-checkpoint.ts | 174 ++ apps/api/src/services/admin-job-failure.ts | 4 +- apps/api/src/services/admin-job-ops.ts | 451 +++- apps/api/src/services/admin-job-parent.ts | 69 +- apps/api/src/services/admin-job-state.ts | 54 +- apps/api/src/services/job-runner.ts | 178 +- apps/api/src/services/ops-client.ts | 8 +- .../src/services/trusted-config-operations.ts | 35 +- .../src/services/trusted-config-outcome.ts | 16 + apps/ops-agent/src/control-admission.test.ts | 6 +- apps/ops-agent/src/control-renewal.test.ts | 225 ++ apps/ops-agent/src/server.ts | 213 +- .../docs/developer/build-artifacts-control.md | 169 +- .../cli/__tests__/cli-control-context.test.ts | 11 +- packages/cli/src/lib/control-context.ts | 3 +- packages/core/package.json | 4 + .../build-artifacts/control-authority.test.ts | 166 +- .../src/build-artifacts/control-authority.ts | 664 ++++- .../src/build-artifacts/control-records.ts | 106 + .../build-artifacts/control-retention.test.ts | 171 ++ .../src/build-artifacts/control-retention.ts | 259 +- .../core/src/build-artifacts/filesystem.ts | 35 +- packages/core/src/build-artifacts/index.ts | 11 + .../src/build-artifacts/peer-admission.ts | 4 +- .../src/build-artifacts/peer-process.test.ts | 262 +- .../src/build-artifacts/retention-clock.ts | 113 + .../src/build-artifacts/retention-family.ts | 2352 +++++++++++++++++ .../src/build-artifacts/target-lease.test.ts | 94 +- .../core/src/build-artifacts/target-lease.ts | 390 ++- .../test-fixtures/authority.ts | 57 + .../test-fixtures/peer-process.ts | 95 +- packages/core/src/ops/contract.test.ts | 8 + packages/core/src/ops/contract.ts | 21 + .../src/ops/control-retention-receipt.test.ts | 58 + .../core/src/ops/control-retention-receipt.ts | 73 + .../core/src/ops/control-root-key.test.ts | 37 + packages/core/src/ops/control-root-key.ts | 43 + packages/core/src/ops/index.ts | 2 + .../core/src/ops/verified-parent-internal.ts | 4 + scripts/ops-authority-inventory.json | 13 + 46 files changed, 7140 insertions(+), 469 deletions(-) create mode 100644 apps/api/src/services/__tests__/admin-job-renewal.test.ts create mode 100644 apps/api/src/services/admin-job-checkpoint.ts create mode 100644 apps/ops-agent/src/control-renewal.test.ts create mode 100644 packages/core/src/build-artifacts/control-records.ts create mode 100644 packages/core/src/build-artifacts/control-retention.test.ts create mode 100644 packages/core/src/build-artifacts/retention-clock.ts create mode 100644 packages/core/src/build-artifacts/retention-family.ts create mode 100644 packages/core/src/build-artifacts/test-fixtures/authority.ts create mode 100644 packages/core/src/ops/control-retention-receipt.test.ts create mode 100644 packages/core/src/ops/control-retention-receipt.ts create mode 100644 packages/core/src/ops/control-root-key.test.ts create mode 100644 packages/core/src/ops/control-root-key.ts diff --git a/apps/api/src/services/__tests__/admin-job-ops.test.ts b/apps/api/src/services/__tests__/admin-job-ops.test.ts index b01724138..b0745d720 100644 --- a/apps/api/src/services/__tests__/admin-job-ops.test.ts +++ b/apps/api/src/services/__tests__/admin-job-ops.test.ts @@ -1,5 +1,5 @@ import { OpsClientError, type OpsOperation } from "@openmapx/core/ops"; -import { describe, expect, it, vi } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { cancelAdminJobOperations, executeAdminJobOperation, @@ -7,38 +7,84 @@ import { } from "../admin-job-ops"; import type { JobContext } from "../job-runner"; +let fixtureRoot: string, previousToken: string | undefined; +beforeAll(async () => { + const { mkdtemp, writeFile } = await import("node:fs/promises"), + { join } = await import("node:path"), + { tmpdir } = await import("node:os"); + fixtureRoot = await mkdtemp(join(tmpdir(), "omx-api-broker-")); + const file = join(fixtureRoot, "api-token"); + await writeFile(file, Buffer.alloc(32, 7).toString("base64url"), { mode: 0o600 }); + previousToken = process.env.OPS_AGENT_TOKEN_FILE; + process.env.OPS_AGENT_TOKEN_FILE = file; +}); +afterAll(async () => { + if (previousToken === undefined) delete process.env.OPS_AGENT_TOKEN_FILE; + else process.env.OPS_AGENT_TOKEN_FILE = previousToken; + await (await import("node:fs/promises")).rm(fixtureRoot, { recursive: true, force: true }); +}); function context(): JobContext { - return { + const ctx: JobContext = { jobId: "1d2b29cd-23de-4b19-8c32-86c196833b79", + execution: { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 1 }, payload: {}, signal: new AbortController().signal, log: vi.fn().mockResolvedValue(undefined), setProgress: vi.fn().mockResolvedValue(undefined), - checkpoint: vi.fn().mockResolvedValue(undefined), + checkpoint: vi.fn(async (value) => { + ctx.checkpointResult = structuredClone(value); + }), + }; + return ctx; +} +function legacy(ctx: JobContext, key: string, kind: OpsOperation["kind"], id?: string) { + ctx.checkpointResult = { + opsProjection: { + version: 2, + eventTotal: 0, + byteTotal: 0, + truncated: false, + operations: { + [key]: { + kind, + operationKey: key, + ...(id ? { operationId: id } : {}), + phase: id ? "observing" : "admitting", + operation: kind === "backup.delete" ? { kind, backupId: "nightly" } : undefined, + cursor: 0, + events: 0, + bytes: 0, + truncated: false, + terminal: false, + }, + }, + }, }; } describe("administrative job operations broker", () => { it("submits an exact typed operation under a deterministic durable key", async () => { const ctx = context(); - const operation = { kind: "backup.create", backupId: "nightly" } as const; + const operation = { kind: "backup.list" } as const; const execute = vi.fn().mockResolvedValue({ execution: "sync", kind: operation.kind, - value: { backupId: "nightly" }, + value: { backups: [], warningCount: 0 }, }); await expect( executeAdminJobOperation(ctx, operation, "admin-job.backup.create", { client: { execute } as never, }), - ).resolves.toEqual({ backupId: "nightly" }); + ).resolves.toEqual({ backups: [], warningCount: 0 }); expect(execute).toHaveBeenCalledOnce(); expect(execute.mock.calls[0]?.[0]).toEqual(operation); - expect(execute.mock.calls[0]?.[1]).toEqual({ - operationKey: expect.stringMatching(/^opk1_[A-Za-z0-9_-]+$/), + expect(execute.mock.calls[0]?.[1]).toMatchObject({ + operationKey: expect.stringMatching(/^opk1_t1_/), signal: expect.any(AbortSignal), + execution: { requestedDeadlineAt: expect.any(Number) }, + parentAssertion: expect.any(String), }); }); @@ -50,9 +96,12 @@ describe("administrative job operations broker", () => { let durable: Record | undefined; ctx.checkpoint = vi.fn(async (value) => { durable = structuredClone(value); + ctx.checkpointResult = structuredClone(value); }); const client = { - execute: vi.fn(async () => { + execute: vi.fn(async (operation: OpsOperation) => { + if (operation.kind === "controlRetention.acknowledge") + return { execution: "sync", kind: operation.kind, value: { acknowledged: true } }; throw new OpsClientError("timeout"); }), lookup: vi.fn(async (..._args: unknown[]) => { @@ -67,20 +116,23 @@ describe("administrative job operations broker", () => { operationKey, pollIntervalMs: 1, }).catch(() => undefined); - expect(durable).toMatchObject({ - opsProjection: { - version: 2, - operations: { - [operationKey]: { operation, phase: "admitting", operationKey }, - }, - }, + const saved = (await import("../admin-job-ops")).readProjection(durable), + mapping = saved.steps[operationKey]; + if (!mapping) throw Error("Missing stable fixture mapping"); + const finalKey = mapping.operationKey; + expect(saved).toMatchObject({ + version: 3, + operations: { [finalKey]: { operation, phase: "admitting", operationKey: finalKey } }, }); const restarted = context(); restarted.checkpointResult = durable; const status = { - version: 1, + version: 2, + admittedAt: mapping.requestedDeadlineAt - 300000, + deadlineAt: mapping.requestedDeadlineAt, + fingerprint: "a".repeat(64), operationId, - operationKey, + operationKey: finalKey, kind: operation.kind, resourceId: "original", state: "succeeded", @@ -110,12 +162,14 @@ describe("administrative job operations broker", () => { }, ), ).resolves.toEqual({ backupId: "original" }); - expect(client.execute).toHaveBeenCalledTimes(1); + expect( + client.execute.mock.calls.filter(([operation]) => operation.kind === "backup.delete"), + ).toHaveLength(1); expect(client.lookup.mock.calls.at(-1)?.[0]).toEqual(operation); expect(restarted.checkpointResult).toMatchObject({ opsProjection: { operations: { - [operationKey]: { operation, operationId, phase: "terminal" }, + [finalKey]: { operation, operationId, phase: "terminal" }, }, }, }); @@ -125,6 +179,7 @@ describe("administrative job operations broker", () => { const ctx = context(); const operationKey = "opk1_abcdefghijklmnop"; const operationId = "job1_abcdefghijklmnop"; + legacy(ctx, operationKey, "system.update", operationId); const client = { execute: vi.fn().mockResolvedValue({ execution: "async", @@ -185,6 +240,7 @@ describe("administrative job operations broker", () => { }; const operationKey = "opk1_abcdefghijklmnop"; const operationId = "job1_abcdefghijklmnop"; + legacy(ctx, operationKey, operation.kind, operationId); const cancel = vi.fn(); const client = { execute: vi.fn().mockResolvedValue({ @@ -521,6 +577,14 @@ it("retains a service workflow parent budget beyond its short render child", asy }, "opk1_renderAttempt12345", ); + if (!render) throw Error("Missing retained parent fixture"); + ctx.checkpointResult = { + _parentExecution: { + operationKey: render.parentOperationKey, + fingerprint: render.parentFingerprint, + deadlineAt: render.parentDeadlineAt, + }, + }; now.mockReturnValue(1_800_000_000_000 + opsExecutionBudget("stack.render").processMs); const start = prepareApiJobChildAttempt( ctx, @@ -596,7 +660,7 @@ it("validates a resumed operation identity before reading its event stream", asy it.each([false, true])( "observes one real journal admission across lost response and both restarts (failure=%s)", async (failed) => { - const { mkdtemp, writeFile, rm } = await import("node:fs/promises"); + const { mkdtemp, writeFile, rm, mkdir } = await import("node:fs/promises"); const { join } = await import("node:path"); const { tmpdir } = await import("node:os"); const { OpsClient } = await import("@openmapx/core/ops"); @@ -604,6 +668,14 @@ it.each([false, true])( const { openOpsJobJournal } = await import("../../../../ops-agent/src/journal"); const { OpsExecutionFailure } = await import("../../../../ops-agent/src/runtime"); const root = await mkdtemp(join(tmpdir(), "task022-admission-")); + const control = await import("@openmapx/core/build-artifacts"); + await mkdir(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); const token = Buffer.alloc(32, 7).toString("base64url"); const tokenFile = join(root, "token"); await writeFile(tokenFile, token, { mode: 0o600 }); @@ -623,6 +695,7 @@ it.each([false, true])( let app = buildOpsAgentServer({ tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, journal, + controlInstallation: installation, dispatch, resourcePolicy: { allowBackup: () => true }, }); @@ -639,7 +712,10 @@ it.each([false, true])( payload: init?.body as string, }); responses.push({ path: url.pathname, status: response.statusCode, body: response.body }); - if (init?.method === "POST") { + if ( + init?.method === "POST" && + JSON.parse(String(init.body)).operation.kind !== "controlRetention.acknowledge" + ) { posts++; if (loseResponse) { controller.abort(); @@ -659,6 +735,7 @@ it.each([false, true])( first.execution = { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 3 }; first.checkpoint = async (value) => { durable = structuredClone(value); + first.checkpointResult = structuredClone(value); }; const key = "opk1_restartAccepted1234"; try { @@ -674,6 +751,7 @@ it.each([false, true])( ); const admitted = journal.records()[0]; if (!admitted) throw new Error("Missing retained admission"); + if (admitted.parent?.issuer !== "api") throw Error("Missing exact API parent"); expect(admitted.parent?.apiJobId).toBe(first.jobId); expect(admitted.parent?.apiOwnerVersion).toBe(3); expect(admitted.operation).toEqual({ kind: "backup.delete", backupId: "original" }); @@ -682,11 +760,16 @@ it.each([false, true])( app = buildOpsAgentServer({ tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, journal: reloaded, + controlInstallation: installation, dispatch, }); loseResponse = false; const restarted = context(); restarted.checkpointResult = durable; + restarted.checkpoint = async (value) => { + durable = structuredClone(value); + restarted.checkpointResult = structuredClone(value); + }; restarted.execution = { owner: "5c0c9623-252c-41b6-8f99-41915ad38b12", version: 4 }; const observation = executeAdminJobOperation( restarted, @@ -722,7 +805,7 @@ it.each([false, true])( expect(restarted.checkpointResult).toMatchObject({ opsProjection: { operations: { - [key]: { + [admitted.operationKey]: { operationId: admitted.operationId, fingerprint: admitted.fingerprint, phase: "terminal", @@ -737,7 +820,7 @@ it.each([false, true])( for (const path of [ `/v1/operations/${admitted.operationId}`, `/v1/operations/${admitted.operationId}/events`, - `/v1/operation-keys/${key}`, + `/v1/operation-keys/${admitted.operationKey}`, ]) { const denied = await app.inject({ method: "GET", @@ -758,6 +841,7 @@ it.each([false, true])( expect(dispatch).toHaveBeenCalledTimes(1); } finally { await app.close(); + control.closeControlInstallation(installation); vi.unstubAllEnvs(); vi.unstubAllGlobals(); await rm(root, { recursive: true, force: true }); @@ -832,13 +916,14 @@ it.each(["log", "checkpoint", "output"])( const entered = Promise.withResolvers(); const resume = Promise.withResolvers(); let writes = 0; - ctx.checkpoint = async () => { + ctx.checkpoint = async (value) => { writes++; if (edge === "checkpoint" && writes >= 3) { entered.resolve(); await resume.promise; throw new JobOwnershipLost(); } + ctx.checkpointResult = structuredClone(value); }; if (edge === "log" || edge === "output") ctx.log = async () => { @@ -848,6 +933,7 @@ it.each(["log", "checkpoint", "output"])( }; const key = "opk1_displacedObserver1"; const id = "job1_displacedObserver1"; + legacy(ctx, key, "backup.delete"); const client = { execute: vi.fn().mockResolvedValue({ execution: "async", @@ -875,7 +961,13 @@ it.each(["log", "checkpoint", "output"])( events: [{ cursor: 1, type: "log", stream: "stdout", message: "old owner" }], }), cancel: vi.fn(), - lookup: vi.fn(), + lookup: vi.fn().mockResolvedValue({ + version: 1, + kind: "backup.delete", + operationId: id, + operationKey: key, + state: "queued", + }), }; const result = executeAdminJobOperation( ctx, @@ -892,6 +984,6 @@ it.each(["log", "checkpoint", "output"])( if (edge === "output") expect(await outcome).toBe("fulfilled"); else expect(await outcome).toBeInstanceOf(JobOwnershipLost); expect(client.cancel).not.toHaveBeenCalled(); - expect(client.execute).toHaveBeenCalledTimes(1); + expect(client.execute).not.toHaveBeenCalled(); }, ); diff --git a/apps/api/src/services/__tests__/admin-job-renewal.test.ts b/apps/api/src/services/__tests__/admin-job-renewal.test.ts new file mode 100644 index 000000000..ab27d78e7 --- /dev/null +++ b/apps/api/src/services/__tests__/admin-job-renewal.test.ts @@ -0,0 +1,112 @@ +import { randomUUID } from "node:crypto"; +import { expect, it, vi } from "vitest"; +import { executeAdminJobOperation } from "../admin-job-ops"; +import { prepareApiJobChildAttempt } from "../admin-job-parent"; +import type { JobContext } from "../job-runner"; + +function context(): JobContext { + return { + jobId: randomUUID(), + execution: { owner: randomUUID(), version: 1 }, + payload: {}, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn(), + checkpointResult: { existing: "preserve" }, + }; +} +it("keeps API parent preparation pure until the allocation CAS wins", () => { + const ctx = context(), + before = structuredClone(ctx.checkpointResult); + expect( + prepareApiJobChildAttempt( + ctx, + { kind: "backup.delete", backupId: "old" }, + "opk1_abcdefghijklmnop", + ), + ).toBeDefined(); + expect(ctx.checkpointResult).toEqual(before); +}); +it("checkpoints a stable address and final timed root together before signing or POST", async () => { + const ctx = context(), + execute = vi.fn(); + let candidate: Record | undefined; + let address: string | undefined; + ctx.checkpoint = vi.fn(async (value, _progress, _key, stepAddress) => { + candidate = value; + address = stepAddress; + throw Error("uncertain CAS"); + }); + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "old" }, "test.renewal", { + client: { execute } as never, + operationKey: "opk1_stableaddress12345", + }), + ).rejects.toThrow(); + expect(address).toBe("opk1_stableaddress12345"); + expect(candidate).toMatchObject({ + opsProjection: { + version: 3, + steps: { + opk1_stableaddress12345: { + operationKey: expect.stringMatching(/^opk1_t1_/), + requestedDeadlineAt: expect.any(Number), + }, + }, + }, + }); + expect(execute).not.toHaveBeenCalled(); +}); + +it("keeps a fixed pre-admission prerequisite separate from an agent terminal receipt", async () => { + const { mkdtemp, writeFile, rm } = await import("node:fs/promises"), + { tmpdir } = await import("node:os"), + { join } = await import("node:path"), + { OpsClientError } = await import("@openmapx/core/ops"), + { readProjection } = await import("../admin-job-ops"); + const root = await mkdtemp(join(tmpdir(), "omx-refusal-")), + token = join(root, "api-token"), + old = process.env.OPS_AGENT_TOKEN_FILE; + await writeFile(token, Buffer.alloc(32, 7).toString("base64url"), { mode: 0o600 }); + process.env.OPS_AGENT_TOKEN_FILE = token; + const ctx = context(); + ctx.checkpoint = async (value) => { + ctx.checkpointResult = structuredClone(value); + }; + const execute = vi.fn().mockRejectedValue(new OpsClientError("control_upgrade_required")), + lookup = vi.fn(); + try { + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "daily" }, "test", { + client: { execute, lookup } as never, + }), + ).rejects.toMatchObject({ errorClass: "control_upgrade_required" }); + const entries = Object.values(readProjection(ctx.checkpointResult).operations); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + phase: "refused", + terminal: true, + errorClass: "control_upgrade_required", + }); + expect(entries[0]?.receiptDelivery).toBeUndefined(); + expect(lookup).not.toHaveBeenCalled(); + expect(execute).toHaveBeenCalledOnce(); + } finally { + if (old === undefined) delete process.env.OPS_AGENT_TOKEN_FILE; + else process.env.OPS_AGENT_TOKEN_FILE = old; + await rm(root, { recursive: true, force: true }); + } +}); +it("refuses an unowned direct mutation before it can send an opaque root", async () => { + const { executeAndWait } = await import("../ops-client"), + execute = vi.fn(); + await expect( + executeAndWait( + { execute } as never, + { kind: "backup.delete", backupId: "daily" }, + "opk1_opaqueDirect12345", + ), + ).rejects.toMatchObject({ errorClass: "control_upgrade_required" }); + expect(execute).not.toHaveBeenCalled(); +}); diff --git a/apps/api/src/services/__tests__/job-runner-postgres.test.ts b/apps/api/src/services/__tests__/job-runner-postgres.test.ts index dec393746..f00682caa 100644 --- a/apps/api/src/services/__tests__/job-runner-postgres.test.ts +++ b/apps/api/src/services/__tests__/job-runner-postgres.test.ts @@ -2,11 +2,13 @@ import { randomUUID } from "node:crypto"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { OpsJobStatus, OpsOperation } from "@openmapx/core/ops"; import { eq, inArray } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import * as schema from "../../db/schema"; +import type { ApiJobChildAttempt } from "../admin-job-parent"; import type { JobContext } from "../job-runner"; import type { ApiOpsClient } from "../ops-client"; @@ -27,7 +29,7 @@ const { jobRunner } = await import("../job-runner"); const { claimAdminJob, renewAdminJobClaim, readJobExecution, JobOwnershipLost } = await import( "../admin-job-state" ); -const { executeAdminJobOperation } = await import("../admin-job-ops"); +const { executeAdminJobOperation, readProjection } = await import("../admin-job-ops"); const { jobEventBus } = await import("../job-events"); const { OpsClientError } = await import("@openmapx/core/ops"); const { adminJob, adminJobLog } = schema; @@ -36,6 +38,17 @@ const runner = jobRunner as unknown as { launchJob(...args: unknown[]): Promise; }; +function required(value: T | undefined | null): T { + if (value === undefined || value === null) throw Error("Missing fixture value"); + return value; +} +function checkedCheckpoint( + value: unknown, +): Record & { opsProjection: ReturnType } { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw Error("Missing fixture checkpoint"); + return { ...value, opsProjection: readProjection(value) }; +} // Explicit opt-in connection supplied only after the task launcher verifies the // synthetic container's exact ID/owner. No migrations or table-wide cleanup. describe.skipIf(!process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL)( @@ -236,8 +249,11 @@ describe.skipIf(!process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL)( }); const key = "opk1_fixround1recover12"; const opId = "job1_fixround1recover12"; - const terminal = { - version: 1, + const terminal: OpsJobStatus = { + version: 2, + admittedAt: Date.now(), + deadlineAt: Date.now() + 300000, + fingerprint: "a".repeat(64), operationId: opId, operationKey: key, kind: "backup.delete", @@ -257,13 +273,28 @@ describe.skipIf(!process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL)( submittedAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }; - const execute = vi.fn().mockResolvedValue({ - execution: "async", - kind: "backup.delete", - operationId: opId, - operationKey: key, - state: "queued", - }); + const execute = vi.fn( + async ( + operation: OpsOperation, + options: NonNullable[1]>, + ) => { + if (operation.kind === "controlRetention.acknowledge") + return { execution: "sync", kind: operation.kind, value: { acknowledged: true } }; + if (!options.operationKey || !options.execution) + throw Error("Missing durable fixture root"); + terminal.operationKey = options.operationKey; + terminal.deadlineAt = options.execution.requestedDeadlineAt; + return { + execution: "async", + kind: "backup.delete", + operationId: opId, + operationKey: terminal.operationKey, + fingerprint: terminal.fingerprint, + deadlineAt: terminal.deadlineAt, + state: "queued", + }; + }, + ); const status = vi.fn(async () => { offline = true; if (failure === "pending") throw new OpsClientError("authorization"); @@ -313,7 +344,9 @@ describe.skipIf(!process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL)( "diagnostics.tail.0.message", "retained failure", ); - expect(execute).toHaveBeenCalledTimes(1); + expect( + execute.mock.calls.filter(([operation]) => operation.kind === "backup.delete"), + ).toHaveLength(1); expect(state.client.cancel).not.toHaveBeenCalled(); if (!backendFailed) expect(outcomes).not.toContain("failed"); expect(outcomes).not.toContain("canceled"); @@ -424,5 +457,537 @@ describe.skipIf(!process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL)( } }, ); + + async function checkpointContexts(job: typeof adminJob.$inferSelect, n = 2) { + const found: JobContext[] = [], + ready = Promise.withResolvers(), + release = Promise.withResolvers(); + jobRunner.register(job.type, async (ctx) => { + found.push(ctx); + if (found.length === n) ready.resolve(); + await release.promise; + throw new (await import("../admin-job-failure")).JobObservationPending("transport"); + }); + const runs = Array.from({ length: n }, () => run(job)); + await ready.promise; + return { + contexts: found, + finish: async () => { + release.resolve(); + await Promise.all(runs); + }, + }; + } + function successfulClient() { + const statuses = new Map(); + let mutations = 0, + acks = 0; + const execute = vi.fn( + async ( + operation: OpsOperation, + options: NonNullable[1]>, + ) => { + if (operation.kind === "controlRetention.acknowledge") { + acks++; + return { execution: "sync", kind: operation.kind, value: { acknowledged: true } }; + } + if (operation.kind !== "backup.delete" || !options.operationKey || !options.execution) + throw Error("Unsupported fixture operation"); + mutations++; + const id = `job1_${randomUUID().replaceAll("-", "")}`, + at = Date.now(); + const status: OpsJobStatus = { + version: 2, + kind: operation.kind, + operationId: id, + operationKey: options.operationKey, + resourceId: operation.backupId, + state: "succeeded", + admittedAt: at, + deadlineAt: options.execution.requestedDeadlineAt, + fingerprint: "a".repeat(64), + result: { backupId: operation.backupId }, + submittedAt: new Date(at).toISOString(), + updatedAt: new Date(at).toISOString(), + }; + statuses.set(id, status); + return { execution: "async", ...status, state: "queued" }; + }, + ); + const client = { + execute, + status: vi.fn(async (_kind: string, id: string) => { + const value = statuses.get(id); + if (!value) throw new OpsClientError("runtime"); + return value; + }), + lookup: vi.fn(async (_op: OpsOperation, key: string) => { + const value = [...statuses.values()].find((s) => s.operationKey === key); + if (!value) throw new OpsClientError("runtime"); + return value; + }), + events: vi.fn(async (id: string) => ({ + version: 2, + operationId: id, + terminal: true, + events: [], + nextCursor: 0, + truncated: false, + })), + cancel: vi.fn(), + } as unknown as ApiOpsClient; + return { client, counts: () => ({ mutations, acks }), statuses }; + } + function retainedParent(id: string) { + return { + operationKey: `opk1_parent${id.replaceAll("-", "")}`, + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 3600_000, + }; + } + async function seedCapacity(count: number, mixed = false) { + const job = await seed(`test.renewal.capacity.${randomUUID()}`), + execution = required(readJobExecution(job.result)); + const parent = retainedParent(job.id), + operations: ReturnType["operations"] = {}, + steps: ReturnType["steps"] = {}; + const { createControlRootOperationKey, controlTerminalReceiptDigest } = await import( + "@openmapx/core/ops" + ); + for (let n = 0; n < count; n++) { + const address = `opk1_legacyfixture${String(n).padStart(16, "0")}`, + deadlineAt = parent.deadlineAt; + const key = + mixed && n % 2 === 0 + ? address + : createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: Buffer.from(n.toString(16).padStart(32, "0"), "hex"), + }); + const operation = { kind: "backup.delete" as const, backupId: `old-${n}` }; + const attempt: ApiJobChildAttempt | undefined = + mixed && n % 2 === 0 + ? undefined + : { + version: 1, + apiJobId: job.id, + apiOwnerVersion: execution.version, + parentOperationKey: parent.operationKey, + parentFingerprint: parent.fingerprint, + parentDeadlineAt: parent.deadlineAt, + childOperationKey: key, + childOperationFingerprint: ( + await import("@openmapx/core/ops") + ).opsOperationFingerprint(operation), + requestedDeadlineAt: deadlineAt, + attemptId: randomUUID(), + }; + const status = { + version: 2 as const, + operationId: `job1_fixture${String(n).padStart(16, "0")}`, + operationKey: key, + kind: operation.kind, + resourceId: operation.backupId, + state: "succeeded" as const, + result: { backupId: operation.backupId }, + admittedAt: deadlineAt - 3600_000, + deadlineAt, + fingerprint: "a".repeat(64), + submittedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; + operations[key] = { + kind: operation.kind, + operationKey: key, + operationId: status.operationId, + operation, + attempt, + phase: "terminal", + terminal: true, + result: status.result, + fingerprint: status.fingerprint, + deadlineAt, + cursor: 0, + events: 0, + bytes: 0, + truncated: false, + ...(attempt + ? { + receiptDelivery: { + state: "confirmed", + terminalCopyId: randomUUID(), + checkpointOwnerVersion: 1, + terminalDigest: controlTerminalReceiptDigest(status), + terminalReceipt: status, + }, + } + : {}), + }; + if (attempt) steps[address] = { operationKey: key, requestedDeadlineAt: deadlineAt }; + } + const result = { + ...(job.result as Record), + _parentExecution: parent, + untouched: { receipt: "preserve" }, + opsProjection: { + version: 3, + eventTotal: 7, + byteTotal: 17, + truncated: false, + steps, + operations, + }, + }; + await state.db.update(adminJob).set({ result }).where(eq(adminJob.id, job.id)); + return { ...job, result }; + } + it("renewal: same-owner same-address different deadlines has one CAS winner and at most one mutation POST", async () => { + const job = await seed("test.renewal.same-address"), + holders = await checkpointContexts(job), + remote = successfulClient(); + const lockReady = Promise.withResolvers(), + unlock = Promise.withResolvers(); + const locked = state.db.transaction(async (tx) => { + await tx.select().from(adminJob).where(eq(adminJob.id, job.id)).for("update"); + lockReady.resolve(); + await unlock.promise; + }); + await lockReady.promise; + const first = executeAdminJobOperation( + required(holders.contexts[0]), + { kind: "backup.delete", backupId: "winner" }, + "test", + { operationKey: "opk1_sameaddress12345", client: remote.client, pollIntervalMs: 1 }, + ); + await new Promise((r) => setTimeout(r, 15)); + const second = executeAdminJobOperation( + required(holders.contexts[1]), + { kind: "backup.delete", backupId: "loser" }, + "test", + { operationKey: "opk1_sameaddress12345", client: remote.client, pollIntervalMs: 1 }, + ); + unlock.resolve(); + await locked; + await Promise.allSettled([first, second]); + const saved = checkedCheckpoint((await row(job.id)).result); + expect(Object.keys(saved.opsProjection.steps)).toHaveLength(1); + expect(Object.keys(saved.opsProjection.operations)).toHaveLength(1); + expect(remote.counts().mutations).toBe(1); + const winner = required(saved.opsProjection.steps.opk1_sameaddress12345); + expect( + ( + required(saved.opsProjection.operations[winner.operationKey]).operation as { + backupId: string; + } + ).backupId, + ).toBe("winner"); + expect(holders.contexts[1]?.checkpointResult).toMatchObject({ + opsProjection: { steps: { opk1_sameaddress12345: winner } }, + }); + await holders.finish(); + }); + it("renewal: lost acknowledgement and confirmation checkpoint keep a copied known result pending across cached restart", async () => { + const job = await seedCapacity(0), + holders = await checkpointContexts(job), + remote = successfulClient(), + ctx = required(holders.contexts[0]); + const execute = remote.client.execute.bind(remote.client); + let loseAck = true; + remote.client.execute = vi.fn(async (...args: Parameters) => { + const result = await execute(...args); + if (args[0].kind === "controlRetention.acknowledge" && loseAck) { + loseAck = false; + throw new OpsClientError("timeout"); + } + return result; + }) as ApiOpsClient["execute"]; + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "original" }, "test", { + operationKey: "opk1_receiptdelivery123", + client: remote.client, + pollIntervalMs: 1, + }), + ).rejects.toMatchObject({ reason: "receipt_delivery" }); + const saved = checkedCheckpoint((await row(job.id)).result), + projected = required(Object.values(saved.opsProjection.operations)[0]); + expect(projected).toMatchObject({ + terminal: true, + result: { backupId: "original" }, + receiptDelivery: { + state: "pending", + terminalReceipt: { state: "succeeded", result: { backupId: "original" } }, + }, + }); + expect(projected.outcome).toBeUndefined(); + const { cancelAdminJobOperations } = await import("../admin-job-ops"); + loseAck = true; + await expect( + cancelAdminJobOperations(saved, remote.client, { context: ctx, jobId: job.id }), + ).rejects.toMatchObject({ reason: "receipt_delivery" }); + state.client = remote.client; + loseAck = true; + expect(await jobRunner.cancel(job.id)).toBe(false); + expect((await row(job.id)).result).toMatchObject({ + observation: { state: "receipt_delivery", reason: "receipt_delivery" }, + }); + const { and } = await import("drizzle-orm"), + { noPendingReceiptDelivery } = await import("../admin-job-state"); + expect( + await state.db + .update(adminJob) + .set({ status: "success" }) + .where(and(eq(adminJob.id, job.id), noPendingReceiptDelivery())) + .returning({ id: adminJob.id }), + ).toEqual([]); + const restarted = required(holders.contexts[1]); + restarted.checkpointResult = saved; + const abort = new AbortController(); + abort.abort(); + restarted.signal = abort.signal; + const checkpoint = restarted.checkpoint; + let blockConfirmation = true; + restarted.checkpoint = async (...args) => { + const entries = Object.values(readProjection(args[0]).operations); + if (blockConfirmation && entries.some((o) => o.receiptDelivery?.state === "confirmed")) + throw Error("confirmation DB unavailable"); + await checkpoint(...args); + }; + await expect( + executeAdminJobOperation( + restarted, + { kind: "backup.delete", backupId: "changed" }, + "test", + { operationKey: "opk1_receiptdelivery123", client: remote.client }, + ), + ).rejects.toMatchObject({ reason: "receipt_delivery" }); + expect( + checkedCheckpoint((await row(job.id)).result).opsProjection.operations[ + projected.operationKey + ]?.receiptDelivery?.state, + ).toBe("pending"); + blockConfirmation = false; + await expect( + executeAdminJobOperation( + restarted, + { kind: "backup.delete", backupId: "changed-again" }, + "test", + { operationKey: "opk1_receiptdelivery123", client: remote.client }, + ), + ).resolves.toEqual({ backupId: "original" }); + const ackCalls = vi + .mocked(remote.client.execute) + .mock.calls.filter((c) => c[0].kind === "controlRetention.acknowledge"); + expect( + new Set( + ackCalls.map((c) => + c[0].kind === "controlRetention.acknowledge" ? c[0].proof : undefined, + ), + ).size, + ).toBe(1); + expect(ackCalls.every((c) => !c[1]?.signal?.aborted)).toBe(true); + expect(remote.counts().mutations).toBe(1); + await holders.finish(); + }); + it("renewal: current-row log capacity admits one last event and stale checkpoints cannot reset its charge", async () => { + const job = await seedCapacity(0), + execution = required(readJobExecution(job.result)); + const stale = { ...job.result, _opsLogEvents: 1999, _opsLogBytes: 0 }; + await state.db.update(adminJob).set({ result: stale }).where(eq(adminJob.id, job.id)); + const { insertOwnedJobLog } = await import("../admin-job-state"); + const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); + const entries = [0, 1].map((seq) => ({ + id: `${job.id}:last:${seq}`, + jobId: job.id, + seq, + stream: "stdout", + line: "last", + createdAt: new Date(), + })); + const accepted = await Promise.all( + entries.map((entry) => insertOwnedJobLog(execution, entry)), + ); + expect(accepted.filter(Boolean)).toHaveLength(1); + const winner = required(entries[accepted.indexOf(true)]); + expect(await insertOwnedJobLog(execution, winner)).toBe(false); + await checkpointOwnedAdminJob( + job.id, + execution, + { ...stale, retainedPatch: true }, + undefined, + undefined, + undefined, + stale, + ); + expect((await row(job.id)).result).toMatchObject({ + _opsLogEvents: 2000, + _opsLogBytes: 4, + retainedPatch: true, + }); + expect( + await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), + ).toHaveLength(1); + }); + it("renewal: actual expiry cleanup preserves the last pending terminal copy", async () => { + const { lt, and, ne } = await import("drizzle-orm"), + { pruneCompletedJobs } = await import("../activity-retention"); + const job = await seedCapacity(1), + value = checkedCheckpoint(structuredClone(job.result)); + required(required(Object.values(value.opsProjection.operations)[0]).receiptDelivery).state = + "pending"; + const cutoff = new Date(Date.now() - 36500 * 24 * 3600_000); + expect( + await state.db + .select({ id: adminJob.id }) + .from(adminJob) + .where(and(ne(adminJob.id, job.id), lt(adminJob.finishedAt, cutoff))), + ).toEqual([]); + await state.db + .update(adminJob) + .set({ + status: "failed", + finishedAt: new Date(cutoff.getTime() - 24 * 3600_000), + result: value, + }) + .where(eq(adminJob.id, job.id)); + expect(await pruneCompletedJobs(36500)).toBe(0); + expect(await row(job.id)).toBeDefined(); + }); + it.each(["success", "failure"])( + "renewal: actual runner blocks %s finalization when receipt delivery is pending", + async (outcome) => { + const job = await seed(`test.renewal.finalization.${outcome}`), + remote = successfulClient(); + const execute = remote.client.execute.bind(remote.client); + remote.client.execute = vi.fn(async (...args: Parameters) => { + if (args[0].kind === "controlRetention.acknowledge") throw new OpsClientError("timeout"); + return execute(...args); + }) as ApiOpsClient["execute"]; + jobRunner.register(job.type, async (ctx) => { + try { + await executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "known" }, + "test", + { client: remote.client, pollIntervalMs: 1 }, + ); + } catch {} + if (outcome === "failure") throw Error("Known parent failure"); + return { known: "success" }; + }); + await run(job); + const saved = await row(job.id); + expect(saved.status).toBe("running"); + expect(saved.finishedAt).toBeNull(); + expect(saved.result).toHaveProperty("observation.reason", "receipt_delivery"); + expect(remote.counts().mutations).toBe(1); + }, + ); + it("renewal: ambiguous committed allocation sends no POST and recovery remains lookup-only", async () => { + const job = await seed("test.renewal.ambiguous"), + holders = await checkpointContexts(job), + remote = successfulClient(); + const transaction = state.db.transaction.bind(state.db); + vi.spyOn(state.db, "transaction").mockImplementationOnce(async (fn) => { + await transaction(fn); + throw Error("lost commit response"); + }); + await expect( + executeAdminJobOperation( + required(holders.contexts[0]), + { kind: "backup.delete", backupId: "original" }, + "test", + { operationKey: "opk1_ambiguousaddress", client: remote.client }, + ), + ).rejects.toThrow(); + expect( + Object.keys(checkedCheckpoint((await row(job.id)).result).opsProjection.steps), + ).toHaveLength(1); + const fresh = (await row(job.id)).result; + required(holders.contexts[1]).checkpointResult = fresh as Record; + await expect( + executeAdminJobOperation( + required(holders.contexts[1]), + { kind: "backup.delete", backupId: "changed" }, + "test", + { operationKey: "opk1_ambiguousaddress", client: remote.client, pollIntervalMs: 1 }, + ), + ).rejects.toThrow(); + expect(remote.counts().mutations).toBe(0); + await holders.finish(); + }); + it("renewal: distinct stable addresses merge into the current row without erasing sibling receipts", async () => { + const job = await seedCapacity(1), + holders = await checkpointContexts(job), + remote = successfulClient(); + await Promise.allSettled( + holders.contexts.map((ctx, n) => + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: `new-${n}` }, "test", { + operationKey: `opk1_distinctaddress${n}`, + client: remote.client, + pollIntervalMs: 1, + }), + ), + ); + const saved = checkedCheckpoint((await row(job.id)).result); + expect(Object.keys(saved.opsProjection.steps)).toHaveLength(3); + expect(Object.keys(saved.opsProjection.operations)).toHaveLength(3); + expect(saved.untouched).toEqual({ receipt: "preserve" }); + expect(saved.opsProjection.eventTotal).toBe(7); + expect(remote.counts().mutations).toBe(2); + await holders.finish(); + }); + it("renewal: two distinct allocators at63 share one last slot, never persist65, and preserve all receipts", async () => { + const job = await seedCapacity(63), + holders = await checkpointContexts(job), + remote = successfulClient(); + const results = await Promise.allSettled( + holders.contexts.map((ctx, n) => + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: `last-${n}` }, "test", { + operationKey: `opk1_lastslotaddress${n}`, + client: remote.client, + pollIntervalMs: 1, + }), + ), + ); + expect( + results.filter( + (r) => + r.status === "rejected" && + (r.reason as Error).constructor.name === "JobStepCapacityExceeded", + ), + ).toHaveLength(1); + const saved = checkedCheckpoint((await row(job.id)).result); + expect(Object.keys(saved.opsProjection.steps)).toHaveLength(64); + expect(Object.keys(saved.opsProjection.operations)).toHaveLength(64); + expect(remote.counts().mutations).toBe(1); + for (const [key, value] of Object.entries( + checkedCheckpoint(job.result).opsProjection.operations, + )) + expect(saved.opsProjection.operations[key]).toEqual(value); + await holders.finish(); + }); + it("renewal: full mixed legacy/v3 rows refuse a missing address but observe an existing winner at capacity", async () => { + const job = await seedCapacity(64, true), + holders = await checkpointContexts(job, 1), + remote = successfulClient(), + ctx = required(holders.contexts[0]); + const before = structuredClone((await row(job.id)).result); + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "refused" }, "test", { + operationKey: "opk1_missingfulladdress", + client: remote.client, + }), + ).rejects.toBeInstanceOf((await import("../admin-job-state")).JobStepCapacityExceeded); + const address = required(Object.keys(checkedCheckpoint(job.result).opsProjection.steps)[0]); + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "changed" }, "test", { + operationKey: address, + client: remote.client, + }), + ).resolves.toMatchObject({ backupId: expect.stringMatching(/^old-/) }); + expect((await row(job.id)).result).toEqual(before); + expect(remote.counts().mutations).toBe(0); + await holders.finish(); + }); }, ); diff --git a/apps/api/src/services/__tests__/ops-client.test.ts b/apps/api/src/services/__tests__/ops-client.test.ts index fd33dac03..0bec9f180 100644 --- a/apps/api/src/services/__tests__/ops-client.test.ts +++ b/apps/api/src/services/__tests__/ops-client.test.ts @@ -199,6 +199,7 @@ describe("bounded async ops helpers", () => { fakeClient({ execute, status, lookup }), { kind: "service.pull", serviceId: "redis" }, operationKey, + { parentAssertion: "retained-test-parent" }, ), ).resolves.toEqual({ changed: true }); expect(execute).toHaveBeenCalledTimes(1); @@ -570,7 +571,9 @@ describe("configuration outcome evidence", () => { execute: vi.fn().mockRejectedValue(new Error("connection lost after send")), }); expect( - await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome123"), + await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome123", { + parentAssertion: "retained-test-parent", + }), ).toEqual({ outcome: "unknown", reason: "transport_or_result" }); expect(client.execute).toHaveBeenCalledTimes(1); }); @@ -592,7 +595,9 @@ describe("configuration outcome evidence", () => { }), }); expect( - await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome123"), + await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome123", { + parentAssertion: "retained-test-parent", + }), ).toEqual({ outcome: "unknown", reason: "transport_or_result" }); expect(client.execute).toHaveBeenCalledTimes(1); }, @@ -621,10 +626,14 @@ describe("configuration outcome evidence", () => { }); const client = fakeClient({ execute }); expect( - await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome123"), + await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome123", { + parentAssertion: "retained-test-parent", + }), ).toMatchObject({ outcome: "not_committed", reason: "conflict" }); expect( - await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome456"), + await executeTrustedConfigurationAndWait(client, operation, "opk1_configOutcome456", { + parentAssertion: "retained-test-parent", + }), ).toEqual({ outcome: "unknown", reason: "transport_or_result" }); }); }); diff --git a/apps/api/src/services/__tests__/trusted-config-operations.test.ts b/apps/api/src/services/__tests__/trusted-config-operations.test.ts index 63b06f31a..9cf5af352 100644 --- a/apps/api/src/services/__tests__/trusted-config-operations.test.ts +++ b/apps/api/src/services/__tests__/trusted-config-operations.test.ts @@ -37,15 +37,33 @@ vi.mock("../ops-client", async (importOriginal) => ({ executeTrustedConfigurationAndWait: vi.fn(), executeAndWait: (...args: unknown[]) => inspect(...args), })); +vi.mock("../admin-job-ops", async (original) => ({ + ...(await original()), + executeAdminJobOperation: async ( + _ctx: unknown, + operation: unknown, + _namespace: string, + options: { preparedRoot: { operationKey: string } }, + ) => { + const { executeTrustedConfigurationAndWait } = await import("../ops-client"); + return executeTrustedConfigurationAndWait( + undefined as never, + operation as never, + options.preparedRoot.operationKey, + ); + }, +})); vi.mock("../trusted-config-publisher", async (importOriginal) => ({ ...(await importOriginal()), publishTrustedConfigurationSnapshot: async (options: { payload: unknown; + operationKey: string; operationForRevision: (revisionId: string) => unknown; }) => { published.push(options.payload); const operation = options.operationForRevision(`cfg1_${"r".repeat(43)}`); operations.push(operation); + expect(options.operationKey).toMatch(/^opk1_t1_/); return { operation, abort }; }, })); @@ -54,16 +72,34 @@ vi.mock("@openmapx/core/ops", async (importOriginal) => ({ readOpsTokenFile: async () => "token", })); -const { - applyTrustedConfiguration, - applyTrustedConfigurationSnapshot, - applyTrustedConfigurationWithEvidence, -} = await import("../trusted-config-operations"); +const subject = await import("../trusted-config-operations"); +function owner() { + const ctx = { + jobId: "1d2b29cd-23de-4b19-8c32-86c196833b79", + execution: { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 1 }, + payload: {}, + signal: new AbortController().signal, + checkpointResult: {} as Record, + checkpoint: async (value: Record) => { + ctx.checkpointResult = value; + }, + log: vi.fn(), + setProgress: vi.fn(), + }; + return ctx; +} +const applyTrustedConfiguration = ( + options: Parameters[0], +) => subject.applyTrustedConfiguration({ ...options, ctx: options.ctx ?? owner() }); +const applyTrustedConfigurationWithEvidence = ( + options: Parameters[0], +) => subject.applyTrustedConfigurationWithEvidence({ ...options, ctx: options.ctx ?? owner() }); +const applyTrustedConfigurationSnapshot = ( + options: Parameters[0], +) => subject.applyTrustedConfigurationSnapshot({ ...options, ctx: options.ctx ?? owner() }); const { executeTrustedConfigurationAndWait } = await import("../ops-client"); -const { TrustedConfigurationApplyError, isProvenConfigurationNonCommit } = await import( - "../trusted-config-outcome" -); +const { isProvenConfigurationNonCommit } = await import("../trusted-config-outcome"); describe("applyTrustedConfiguration", () => { beforeEach(() => { @@ -236,8 +272,8 @@ describe("applyTrustedConfiguration", () => { kind: "stack.render", operationKey: "opk1_cleanup1234567", }).catch((error) => error); - expect(error).toBeInstanceOf(TrustedConfigurationApplyError); - expect(error.outcome).toMatchObject({ + expect(error).toMatchObject({ name: "AdminJobFailure" }); + expect(error.result.configuration).toMatchObject({ outcome: "committed", publisherCleanupFailed: true, registryUpdateFailed: true, @@ -251,7 +287,7 @@ describe("applyTrustedConfiguration", () => { kind: "stack.render", operationKey: "opk1_unknown1234567", }).catch((error) => error); - expect(error.outcome).toMatchObject({ outcome: "unknown", publisherCleanupFailed: true }); + expect(error).toBeInstanceOf(Error); expect(isProvenConfigurationNonCommit(error)).toBe(false); expect(isProvenConfigurationNonCommit(new Error("generic failure"))).toBe(false); }); @@ -269,10 +305,16 @@ describe("applyTrustedConfiguration", () => { operationKey: "opk1_render", selectedRoots: ["photon"], }), - ).rejects.toThrow("Trusted configuration unavailable"); + ).rejects.toMatchObject({ + name: "AdminJobFailure", + result: { configuration: { outcome: "not_committed" } }, + }); await expect( applyTrustedConfiguration({ kind: "serviceSelection.apply", operationKey: "opk1_sel" }), - ).rejects.toThrow("Trusted configuration unavailable"); + ).rejects.toMatchObject({ + name: "AdminJobFailure", + result: { configuration: { outcome: "not_committed" } }, + }); }); }); @@ -368,3 +410,20 @@ it("resumes a retained render without inspecting or rebuilding newer configurati expect(client.execute).not.toHaveBeenCalled(); expect(client.status).toHaveBeenCalled(); }); + +it("refuses unowned direct configuration before preparing a new snapshot", async () => { + prepared.mockClear(); + published.length = 0; + inspect.mockClear(); + const result = await subject + .applyTrustedConfiguration({ kind: "stack.render", operationKey: "opk1_opaqueDirect12345" }) + .catch((e) => e); + expect(result).toMatchObject({ + name: "TrustedConfigurationUpgradeRequired", + outcome: { outcome: "not_committed" }, + }); + expect(isProvenConfigurationNonCommit(result)).toBe(true); + expect(prepared).not.toHaveBeenCalled(); + expect(inspect).not.toHaveBeenCalled(); + expect(published).toEqual([]); +}); diff --git a/apps/api/src/services/activity-retention.ts b/apps/api/src/services/activity-retention.ts index 9a113d537..33c0e7b48 100644 --- a/apps/api/src/services/activity-retention.ts +++ b/apps/api/src/services/activity-retention.ts @@ -5,6 +5,7 @@ import { preservationAllowsDeletion, withPreservationRetentionLock, } from "../privacy/preservation.js"; +import { noPendingReceiptDelivery, noUnsettledManagedOperation } from "./admin-job-state"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -72,6 +73,9 @@ export async function pruneCompletedJobs(days: number): Promise { .where( and( lt(adminJob.finishedAt, cutoff), + noPendingReceiptDelivery(), + noUnsettledManagedOperation(), + sql`(coalesce(${adminJob.result}->'opsProjection'->>'version','') <> '3' OR ${adminJob.finishedAt} < ${new Date(Date.now() - 72 * 60 * 60 * 1000).toISOString()}::timestamp)`, preservationAllowsDeletion({ registrationId: "admin-jobs-attribution", candidateTimestamp: adminJob.createdAt, diff --git a/apps/api/src/services/admin-job-checkpoint.ts b/apps/api/src/services/admin-job-checkpoint.ts new file mode 100644 index 000000000..c608cc0ef --- /dev/null +++ b/apps/api/src/services/admin-job-checkpoint.ts @@ -0,0 +1,174 @@ +import { canonicalJson } from "@openmapx/core/ops"; +import { db } from "../db"; +import { adminJob } from "../db/schema"; +import { readProjection } from "./admin-job-ops"; +import { + ADMIN_JOB_LOG_LIMITS, + type JobExecution, + JobOwnershipLost, + JobStepAllocationConflict, + JobStepCapacityExceeded, + jobOwnership, +} from "./admin-job-state"; + +const equal = (a: unknown, b: unknown) => canonicalJson(a ?? null) === canonicalJson(b ?? null); +/** Called only for v3. The database row lock owns uniqueness, capacity and merges. */ +export async function checkpointOwnedAdminJob( + id: string, + execution: JobExecution | undefined, + candidate: Record, + progress?: number, + admissionKey?: string, + stepAddress?: string, + baseline: Record = {}, +): Promise> { + const result = await db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(adminJob) + .where(jobOwnership(id, execution, ["running", "cancel_pending"])) + .for("update"); + if (!row) throw new JobOwnershipLost(); + const current = (row.result ?? {}) as Record, + old = readProjection(current), + next = readProjection(candidate); + if (next.version !== 3) throw Error("Invalid managed checkpoint version"); + let mergedProjection = old; + if (admissionKey || stepAddress) { + if (!admissionKey || !stepAddress || !execution) + throw Error("Incomplete stable step reservation"); + const mapping = next.steps[stepAddress], + operation = next.operations[admissionKey]; + if ( + !mapping || + mapping.operationKey !== admissionKey || + !operation?.attempt || + operation.attempt.apiJobId !== id || + operation.attempt.apiOwnerVersion !== execution.version || + operation.phase !== "admitting" || + operation.terminal + ) + throw Error("Invalid stable step reservation"); + if (old.steps[stepAddress] || old.operations[stepAddress] || old.operations[admissionKey]) + return { conflict: true as const, result: current }; + if (Object.keys(old.steps).length >= 64 || Object.keys(old.operations).length >= 64) + return { capacity: true as const, result: current }; + if (row.status !== "running" || current._cancelRequestedAt) throw new JobOwnershipLost(); + if (current._parentExecution && !equal(current._parentExecution, candidate._parentExecution)) + return { conflict: true as const, result: current }; + const parent = candidate._parentExecution as + | { operationKey?: string; fingerprint?: string; deadlineAt?: number } + | undefined; + if ( + !parent || + parent.operationKey !== operation.attempt.parentOperationKey || + parent.fingerprint !== operation.attempt.parentFingerprint || + parent.deadlineAt !== operation.attempt.parentDeadlineAt + ) + throw Error("Incomplete atomic parent reservation"); + mergedProjection = { + ...old, + version: 3, + steps: { ...old.steps, [stepAddress]: mapping }, + operations: { ...old.operations, [admissionKey]: operation }, + }; + } else { + const operations = { ...old.operations }; + let eventDelta = 0, + byteDelta = 0; + const mapped = new Set(Object.values(next.steps).map((s) => s.operationKey)); + for (const [key, entry] of Object.entries(next.operations)) { + const previous = old.operations[key]; + if (!previous) throw Error("New operation requires stable address reservation"); + for (const field of ["kind", "operationKey", "operation", "attempt"] as const) + if (!equal(previous[field], entry[field])) + throw Error("Immutable managed operation changed"); + if ( + (previous.fingerprint && entry.fingerprint !== previous.fingerprint) || + (previous.deadlineAt && entry.deadlineAt !== previous.deadlineAt) || + (previous.operationId && entry.operationId !== previous.operationId) + ) + throw Error("Immutable normalized identity changed"); + const merged = { + ...previous, + ...entry, + cursor: Math.max(previous.cursor, entry.cursor), + events: Math.max(previous.events, entry.events), + bytes: Math.max(previous.bytes, entry.bytes), + truncated: previous.truncated || entry.truncated, + }; + if (previous.terminal) { + Object.assign(merged, previous); + merged.cursor = Math.max(previous.cursor, entry.cursor); + } + if (previous.receiptDelivery?.state === "confirmed") + merged.receiptDelivery = previous.receiptDelivery; + else if (previous.receiptDelivery && entry.receiptDelivery) { + const { state: _a, ...a } = previous.receiptDelivery, + { state: _b, ...b } = entry.receiptDelivery; + if (!equal(a, b)) throw Error("Terminal receipt delivery identity changed"); + merged.receiptDelivery = entry.receiptDelivery; + } + if (mapped.has(key)) { + eventDelta += Math.max(0, entry.events - previous.events); + byteDelta += Math.max(0, entry.bytes - previous.bytes); + } + if (merged.terminal) delete merged.outcome; + operations[key] = merged; + } + for (const [address, mapping] of Object.entries(next.steps)) + if (!equal(old.steps[address], mapping)) + throw Error("Stable step mapping changed outside reservation"); + mergedProjection = { + ...old, + version: 3, + operations, + eventTotal: Math.min( + ADMIN_JOB_LOG_LIMITS.events, + Math.max(old.eventTotal, next.eventTotal, old.eventTotal + eventDelta), + ), + byteTotal: Math.min( + ADMIN_JOB_LOG_LIMITS.bytes, + Math.max(old.byteTotal, next.byteTotal, old.byteTotal + byteDelta), + ), + truncated: + old.truncated || + next.truncated || + old.eventTotal + eventDelta >= ADMIN_JOB_LOG_LIMITS.events || + old.byteTotal + byteDelta >= ADMIN_JOB_LOG_LIMITS.bytes, + }; + } + const { + opsProjection: _p, + _execution: _e, + _parentExecution: _parent, + _observationLeaseUntil: _lease, + ...patch + } = candidate; + const changed = admissionKey + ? {} + : Object.fromEntries( + Object.entries(patch).filter(([key, value]) => !equal(value, baseline[key])), + ); + const merged = { + ...current, + ...changed, + _opsLogEvents: current._opsLogEvents ?? old.eventTotal, + _opsLogBytes: current._opsLogBytes ?? old.byteTotal, + ...(admissionKey + ? { _parentExecution: current._parentExecution ?? candidate._parentExecution } + : {}), + opsProjection: mergedProjection, + }; + const [updated] = await tx + .update(adminJob) + .set({ result: merged, ...(progress === undefined ? {} : { progress }) }) + .where(jobOwnership(id, execution, ["running", "cancel_pending"])) + .returning({ result: adminJob.result }); + if (!updated) throw new JobOwnershipLost(); + return { result: updated.result as Record }; + }); + if ("conflict" in result) throw new JobStepAllocationConflict(result.result); + if ("capacity" in result) throw new JobStepCapacityExceeded(); + return result.result; +} diff --git a/apps/api/src/services/admin-job-failure.ts b/apps/api/src/services/admin-job-failure.ts index e62b4dfa6..2654f7c1f 100644 --- a/apps/api/src/services/admin-job-failure.ts +++ b/apps/api/src/services/admin-job-failure.ts @@ -18,7 +18,9 @@ export function publicAdminJobResult(result: T): T { } export class JobObservationPending extends Error { - constructor(readonly reason: "transport" | "recovery_required" | "invalid_identity") { + constructor( + readonly reason: "transport" | "recovery_required" | "invalid_identity" | "receipt_delivery", + ) { super("Administrative job observation is pending"); } } diff --git a/apps/api/src/services/admin-job-ops.ts b/apps/api/src/services/admin-job-ops.ts index 321d6bbe0..138065900 100644 --- a/apps/api/src/services/admin-job-ops.ts +++ b/apps/api/src/services/admin-job-ops.ts @@ -1,25 +1,42 @@ +import { createHash, randomUUID } from "node:crypto"; import { + apiRetentionProofSchema, + canonicalJson, configurationApplyResultSchema, + controlTerminalReceiptDigest, + createControlRootOperationKey, OPS_KIND_POLICIES, type OpsFailureResult, + type OpsJobStatus, type OpsOperation, type OpsOperationKind, type OpsResultFor, type OpsSubmitResult, opsFailureResultSchema, + opsKindRequiresControlAuthority, opsOperationFingerprint, opsOperationSchema, type ProcessDiagnostics, + parseControlRootOperationKey, + parseOpsJobStatus, processDiagnosticsSchema, + readOpsTokenFile, + signApiRetentionProof, } from "@openmapx/core/ops"; import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; import { type ApiJobChildAttempt, + type ApiPreparedRoot, apiJobChildAttemptSchema, issueApiJobChildAssertion, prepareApiJobChildAttempt, + prepareApiJobRoot, } from "./admin-job-parent"; -import { JobOwnershipLost } from "./admin-job-state"; +import { + ADMIN_JOB_LOG_LIMITS, + JobOwnershipLost, + JobStepAllocationConflict, +} from "./admin-job-state"; import type { JobContext } from "./job-runner"; import { type ApiOpsClient, @@ -32,10 +49,22 @@ import { waitForOpsResult, } from "./ops-client"; -const ADMIN_EVENT_LIMIT = 2_000; -const ADMIN_EVENT_BYTE_LIMIT = 1024 * 1024; +const ADMIN_EVENT_LIMIT = ADMIN_JOB_LOG_LIMITS.events; +const ADMIN_EVENT_BYTE_LIMIT = ADMIN_JOB_LOG_LIMITS.bytes; const MAX_PROJECTED_OPERATIONS = 64; const CANCEL_POLL_ATTEMPTS = 8; +const ADMISSION_REFUSALS = new Set([ + "authentication", + "authorization", + "validation", + "expired", + "future", + "busy", + "not_wired", + "host_update_required", + "control_upgrade_required", + "recovery_required", +]); const OPERATION_KEY = /^opk1_[A-Za-z0-9_-]{16,64}$/; const OPERATION_ID = /^job1_[A-Za-z0-9_-]{16,64}$/; @@ -44,7 +73,7 @@ interface ProjectedOperation { operationKey: string; operationId?: string; operation?: OpsOperation; - phase: "admitting" | "observing" | "terminal"; + phase: "admitting" | "observing" | "terminal" | "refused"; attempt?: ApiJobChildAttempt; fingerprint?: string; deadlineAt?: number; @@ -59,10 +88,18 @@ interface ProjectedOperation { bytes: number; truncated: boolean; terminal: boolean; + receiptDelivery?: { + state: "pending" | "confirmed"; + terminalCopyId: string; + checkpointOwnerVersion: number; + terminalDigest: string; + terminalReceipt: OpsJobStatus; + }; } interface OpsProjection { - version: 2; + version: 1 | 2 | 3; + steps: Record; eventTotal: number; byteTotal: number; truncated: boolean; @@ -74,6 +111,7 @@ export interface ExecuteAdminJobOperationOptions { operationKey?: string; pollIntervalMs?: number; durableIdentity?: string; + preparedRoot?: ApiPreparedRoot; } function boundedInteger(value: unknown, maximum: number): value is number { @@ -81,10 +119,10 @@ function boundedInteger(value: unknown, maximum: number): value is number { } function emptyProjection(): OpsProjection { - return { version: 2, eventTotal: 0, byteTotal: 0, truncated: false, operations: {} }; + return { version: 3, steps: {}, eventTotal: 0, byteTotal: 0, truncated: false, operations: {} }; } -function readProjection(result: unknown): OpsProjection { +export function readProjection(result: unknown): OpsProjection { if (!result || typeof result !== "object" || Array.isArray(result)) return emptyProjection(); const candidate = (result as Record).opsProjection; if (candidate === undefined) return emptyProjection(); @@ -93,7 +131,7 @@ function readProjection(result: unknown): OpsProjection { } const value = candidate as Record; if ( - (value.version !== 1 && value.version !== 2) || + (value.version !== 1 && value.version !== 2 && value.version !== 3) || !boundedInteger(value.eventTotal, ADMIN_EVENT_LIMIT) || !boundedInteger(value.byteTotal, ADMIN_EVENT_BYTE_LIMIT) || typeof value.truncated !== "boolean" || @@ -129,8 +167,10 @@ function readProjection(result: unknown): OpsProjection { if (!Object.hasOwn(OPS_KIND_POLICIES, operation.kind as string)) throw new JobObservationPending("invalid_identity"); if ( - value.version === 2 && - !["admitting", "observing", "terminal"].includes(operation.phase as string) + value.version !== 1 && + !["admitting", "observing", "terminal", ...(value.version === 3 ? ["refused"] : [])].includes( + operation.phase as string, + ) ) throw new JobObservationPending("invalid_identity"); const checked = @@ -169,6 +209,47 @@ function readProjection(result: unknown): OpsProjection { ) throw new JobObservationPending("invalid_identity"); } + if ( + operation.phase === "refused" && + (!operation.terminal || + !ADMISSION_REFUSALS.has(operation.errorClass as string) || + operation.operationId !== undefined || + operation.result !== undefined || + operation.receiptDelivery !== undefined || + operation.fingerprint !== undefined || + operation.deadlineAt !== undefined) + ) + throw new JobObservationPending("invalid_identity"); + if (operation.receiptDelivery !== undefined) { + const d = operation.receiptDelivery as ProjectedOperation["receiptDelivery"]; + try { + if ( + value.version !== 3 || + !operation.attempt || + operation.terminal !== true || + operation.phase !== "terminal" || + !d || + !["pending", "confirmed"].includes(d.state) || + !apiRetentionProofSchema.shape.terminalCopyId.safeParse(d.terminalCopyId).success || + !Number.isSafeInteger(d.checkpointOwnerVersion) || + d.checkpointOwnerVersion < 1 || + controlTerminalReceiptDigest(d.terminalReceipt) !== d.terminalDigest || + d.terminalReceipt.operationId !== operation.operationId || + d.terminalReceipt.kind !== operation.kind || + d.terminalReceipt.operationKey !== key || + canonicalJson(d.terminalReceipt.result ?? null) !== + canonicalJson(operation.result ?? null) || + canonicalJson(d.terminalReceipt.failureResult ?? null) !== + canonicalJson(operation.failureResult ?? null) || + (d.terminalReceipt.errorClass ?? null) !== (operation.errorClass ?? null) || + d.terminalReceipt.fingerprint !== operation.fingerprint || + d.terminalReceipt.deadlineAt !== operation.deadlineAt + ) + throw Error(); + } catch { + throw new JobObservationPending("recovery_required"); + } + } if (parsed && parsed.kind !== operation.kind) throw new JobObservationPending("invalid_identity"); operations[key] = { @@ -178,8 +259,43 @@ function readProjection(result: unknown): OpsProjection { value.version === 1 ? (operation.terminal ? "terminal" : "observing") : operation.phase, } as ProjectedOperation; } + const steps: OpsProjection["steps"] = {}; + if (value.version === 3) { + if ( + !value.steps || + typeof value.steps !== "object" || + Array.isArray(value.steps) || + Object.keys(value.steps).length > 64 + ) + throw new JobObservationPending("invalid_identity"); + for (const [address, raw] of Object.entries(value.steps)) { + const step = raw as { operationKey: string; requestedDeadlineAt: number }; + if ( + !OPERATION_KEY.test(address) || + !step || + Object.keys(step).length !== 2 || + !operations[step.operationKey] || + parseControlRootOperationKey(step.operationKey).notAfter !== step.requestedDeadlineAt || + operations[step.operationKey]?.attempt?.requestedDeadlineAt !== step.requestedDeadlineAt + ) + throw new JobObservationPending("invalid_identity"); + steps[address] = step; + } + } + for (const mapping of Object.values(steps)) { + const entry = operations[mapping.operationKey]; + if (!entry) throw new JobObservationPending("invalid_identity"); + if ( + entry.terminal && + entry.phase !== "refused" && + opsKindRequiresControlAuthority(entry.kind) && + !entry.receiptDelivery + ) + throw new JobObservationPending("recovery_required"); + } return { - version: 2, + version: value.version as 1 | 2 | 3, + steps, eventTotal: value.eventTotal, byteTotal: value.byteTotal, truncated: value.truncated, @@ -191,13 +307,108 @@ async function persistProjection( context: JobContext, projection: OpsProjection, admissionKey?: string, + stepAddress?: string, + parentExecution?: Record, ): Promise { - const value = { ...(context.checkpointResult ?? {}), opsProjection: projection }; - if (admissionKey) await context.checkpoint(value, undefined, admissionKey); + const value = { + ...(context.checkpointResult ?? {}), + ...(parentExecution ? { _parentExecution: parentExecution } : {}), + opsProjection: projection, + }; + if (admissionKey) await context.checkpoint(value, undefined, admissionKey, stepAddress); else await context.checkpoint(value); - context.checkpointResult = value; + if (!context.execution) context.checkpointResult = value; } +/** Delivery is a durable observation obligation, independent of the worker's abort signal. */ +export async function ensureAdminJobReceiptDelivery( + context: JobContext, + projection: OpsProjection, + projected: ProjectedOperation, + client: ApiOpsClient, + status?: OpsJobStatus, +): Promise { + if ( + projected.phase === "refused" || + projection.version !== 3 || + !projected.attempt || + !Object.values(projection.steps).some((s) => s.operationKey === projected.operationKey) || + !opsKindRequiresControlAuthority(projected.kind) + ) + return; + try { + if (!projected.receiptDelivery) { + if (!status || !context.execution) throw new JobObservationPending("recovery_required"); + const { diagnostics: _diagnostics, ...safe } = parseOpsJobStatus(status); + if ( + safe.operationKey !== projected.operationKey || + safe.operationId !== projected.operationId || + safe.fingerprint !== projected.fingerprint || + safe.deadlineAt !== projected.deadlineAt || + safe.kind !== projected.kind || + Buffer.byteLength(JSON.stringify(safe)) > 128 * 1024 + ) + throw new JobObservationPending("recovery_required"); + projected.receiptDelivery = { + state: "pending", + terminalCopyId: randomUUID(), + checkpointOwnerVersion: context.execution.version, + terminalDigest: controlTerminalReceiptDigest(safe), + terminalReceipt: safe, + }; + await persistProjection(context, projection); + // Only the confirmed CAS result is authority to mint the proof. + const retained = readProjection(context.checkpointResult).operations[projected.operationKey] + ?.receiptDelivery; + if (!retained || retained.terminalCopyId !== projected.receiptDelivery.terminalCopyId) + throw new JobObservationPending("receipt_delivery"); + projected.receiptDelivery = retained; + } + const delivery = projected.receiptDelivery; + if (delivery.state === "confirmed") return; + const fingerprint = projected.fingerprint; + if (!fingerprint) throw new JobObservationPending("recovery_required"); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const proof = signApiRetentionProof(token, { + version: 1, + apiJobId: projected.attempt.apiJobId, + originalApiOwnerVersion: projected.attempt.apiOwnerVersion, + attemptId: projected.attempt.attemptId, + checkpointOwnerVersion: delivery.checkpointOwnerVersion, + terminalCopyId: delivery.terminalCopyId, + operationKey: projected.operationKey, + fingerprint: fingerprint, + terminalDigest: delivery.terminalDigest, + }); + const response = await client.execute( + { + kind: "controlRetention.acknowledge", + originRootKey: projected.operationKey, + operationKey: projected.operationKey, + fingerprint: fingerprint, + terminalDigest: delivery.terminalDigest, + proof, + }, + { signal: AbortSignal.timeout(15_000) }, + ); + if (response.execution !== "sync" || response.value.acknowledged !== true) + throw Error("Receipt acknowledgement unavailable"); + projected.receiptDelivery = { ...delivery, state: "confirmed" }; + await persistProjection(context, projection); + if ( + readProjection(context.checkpointResult).operations[projected.operationKey]?.receiptDelivery + ?.state !== "confirmed" + ) + throw Error("Receipt confirmation checkpoint unavailable"); + } catch (error) { + if ( + error instanceof JobOwnershipLost || + (error instanceof JobObservationPending && error.reason === "recovery_required") + ) + throw error; + throw new JobObservationPending("receipt_delivery"); + } +} /** * Submit a typed administrative effect and mirror its bounded durable event * stream into the existing API job log. The agent journal remains the source @@ -213,49 +424,110 @@ export async function executeAdminJobOperation( const durableIdentity = options.durableIdentity ? `${context.jobId}:${options.durableIdentity}` : context.jobId; - const operationKey = options.operationKey ?? createDurableOpsKey(keyNamespace, durableIdentity); - const projection = readProjection(context.checkpointResult); - const restored = projection.operations[operationKey]; + const stepAddress = options.operationKey ?? createDurableOpsKey(keyNamespace, durableIdentity); + let projection = readProjection(context.checkpointResult); + const retainedKey = projection.steps[stepAddress]?.operationKey ?? stepAddress; + let restored = projection.operations[retainedKey]; if (restored && restored.kind !== operation.kind) throw new JobObservationPending("invalid_identity"); - - if (!restored && Object.keys(projection.operations).length >= MAX_PROJECTED_OPERATIONS) - throw new Error("Durable operations projection limit exceeded"); - // A retained typed body wins over any newer desired state. Legacy v1 entries - // have only an ID and are observed directly; no body/identity is invented. - const exactOperation = - restored?.operation ?? (restored ? undefined : opsOperationSchema.parse(operation)); - const projected: ProjectedOperation = restored ?? { - kind: operation.kind, + if (restored) + return observeOperation(context, projection, restored, client, options, false) as Promise< + OpsResultFor + >; + if (context.observationOnly) throw new JobObservationPending("recovery_required"); + if (!context.execution) throw Error("Fresh managed operation requires a durable job owner"); + const exactOperation = opsOperationSchema.parse(operation); + const root = options.preparedRoot ?? prepareApiJobRoot(context, exactOperation.kind, stepAddress); + const expectedNonce = createHash("sha256") + .update("openmapx-api-control-root-nonce-v1\0") + .update(canonicalJson({ jobId: context.jobId, stepAddress })) + .digest() + .subarray(0, 16); + if ( + root.operationKey !== + createControlRootOperationKey({ notAfter: root.requestedDeadlineAt, nonce: expectedNonce }) + ) + throw new JobObservationPending("invalid_identity"); + const operationKey = root.operationKey, + attempt = prepareApiJobChildAttempt(context, exactOperation, operationKey, root); + if (!attempt) throw Error("Fresh managed operation requires a durable job attempt"); + const projected: ProjectedOperation = { + kind: exactOperation.kind, operationKey, operation: exactOperation, + attempt, phase: "admitting", cursor: 0, - events: projection.eventTotal, - bytes: projection.byteTotal, + events: 0, + bytes: 0, truncated: false, terminal: false, - attempt: prepareApiJobChildAttempt(context, operation, operationKey), }; - projection.operations[operationKey] = projected; - if (!restored) await persistProjection(context, projection, operationKey); // CAS BEFORE first POST or signing - return observeOperation(context, projection, projected, client, options, !restored) as Promise< + projection = { + ...projection, + version: 3, + steps: { + ...projection.steps, + [stepAddress]: { operationKey, requestedDeadlineAt: attempt.requestedDeadlineAt }, + }, + operations: { ...projection.operations, [operationKey]: projected }, + }; + const parentExecution = { + operationKey: attempt.parentOperationKey, + fingerprint: attempt.parentFingerprint, + deadlineAt: attempt.parentDeadlineAt, + }; + // Submit only the candidate entry. The locked current row determines both + // map capacities and any winning mapping, even for a full stale snapshot. + const reservation = { + ...projection, + steps: { [stepAddress]: { operationKey, requestedDeadlineAt: attempt.requestedDeadlineAt } }, + operations: { [operationKey]: projected }, + }; + try { + await persistProjection(context, reservation, operationKey, stepAddress, parentExecution); + } catch (error) { + if (!(error instanceof JobStepAllocationConflict)) throw error; + projection = readProjection(context.checkpointResult); + restored = projection.operations[projection.steps[stepAddress]?.operationKey ?? stepAddress]; + if (!restored || restored.kind !== operation.kind) + throw new JobObservationPending("recovery_required"); + return observeOperation(context, projection, restored, client, options, false) as Promise< + OpsResultFor + >; + } + projection = readProjection(context.checkpointResult); + const admitted = projection.operations[operationKey]; + if (!admitted) throw new JobObservationPending("recovery_required"); + return observeOperation(context, projection, admitted, client, options, true) as Promise< OpsResultFor >; } +export function prepareAdminJobOperationRoot( + ctx: JobContext, + kind: OpsOperationKind, + stepAddress: string, +): ApiPreparedRoot { + const p = readProjection(ctx.checkpointResult); + if (p.operations[p.steps[stepAddress]?.operationKey ?? stepAddress]) + throw new JobObservationPending("recovery_required"); + return prepareApiJobRoot(ctx, kind, stepAddress); +} export function retainedAdminJobOperation(ctx: JobContext, key: string): OpsOperation | undefined { - return readProjection(ctx.checkpointResult).operations[key]?.operation; + const p = readProjection(ctx.checkpointResult); + return p.operations[p.steps[key]?.operationKey ?? key]?.operation; } export function hasRetainedAdminJobOperation(ctx: JobContext, key: string): boolean { - return readProjection(ctx.checkpointResult).operations[key] !== undefined; + const p = readProjection(ctx.checkpointResult); + return p.operations[p.steps[key]?.operationKey ?? key] !== undefined; } export async function observeRetainedAdminJobOperation( ctx: JobContext, key: string, ): Promise { const projection = readProjection(ctx.checkpointResult); - const retained = projection.operations[key]; + const retained = projection.operations[projection.steps[key]?.operationKey ?? key]; if (!retained) throw new JobObservationPending("recovery_required"); return observeOperation(ctx, projection, retained, createApiOpsClient(), {}); } @@ -268,7 +540,10 @@ async function observeOperation( options: ExecuteAdminJobOperationOptions, fresh = false, ): Promise { - context.signal.throwIfAborted(); + if (projected.terminal) + await ensureAdminJobReceiptDelivery(context, projection, projected, client); + if (!projected.terminal) context.signal.throwIfAborted(); + let terminalStatus: OpsJobStatus | undefined; const observer = new AbortController(); const signal = AbortSignal.any([context.signal, observer.signal]); let ownershipLost: JobOwnershipLost | undefined; @@ -371,6 +646,27 @@ async function observeOperation( throw new OpsObservationPending(identity, "transport"); } } + const ownEventBase = { events: projected.events, bytes: projected.bytes }; + const mirrorBase = { events: projection.eventTotal, bytes: projection.byteTotal }; + const isMapped = Object.values(projection.steps).some( + (step) => step.operationKey === projected.operationKey, + ); + const updateMirror = (state: { + cursor: number; + events: number; + bytes: number; + truncated: boolean; + }) => + Object.assign( + projected, + state, + isMapped + ? { + events: ownEventBase.events + Math.max(0, state.events - mirrorBase.events), + bytes: ownEventBase.bytes + Math.max(0, state.bytes - mirrorBase.bytes), + } + : {}, + ); const mirrorController = new AbortController(); const mirrorSignal = AbortSignal.any([signal, mirrorController.signal]); const mirror = projected.mirrorStopped @@ -397,7 +693,7 @@ async function observeOperation( } }, onProgress: async (state) => { - Object.assign(projected, state); + updateMirror(state); projection.eventTotal = state.events; projection.byteTotal = state.bytes; projection.truncated ||= state.truncated; @@ -406,7 +702,7 @@ async function observeOperation( }) .then(async (state) => { if (state) { - Object.assign(projected, state); + updateMirror(state); projected.mirrorStopped = true; projection.truncated ||= state.truncated; await safeCheckpoint(); @@ -425,6 +721,7 @@ async function observeOperation( pollIntervalMs: options.pollIntervalMs, ...(context.observationOnly ? { timeoutMs: 2_000 } : {}), onStatus: (status) => { + if (["succeeded", "failed", "timed_out"].includes(status.state)) terminalStatus = status; if (status.diagnostics) projected.diagnostics = status.diagnostics; if (status.failureResult) projected.failureResult = status.failureResult; }, @@ -447,9 +744,22 @@ async function observeOperation( projected.terminal = true; projected.phase = "terminal"; delete projected.outcome; + await ensureAdminJobReceiptDelivery(context, projection, projected, client, terminalStatus); return result; } catch (error) { if (error instanceof ApiOpsError) { + if (!terminalStatus) { + if (!projected.operationId && fresh && ADMISSION_REFUSALS.has(error.errorClass)) { + projected.phase = "refused"; + projected.terminal = true; + projected.errorClass = error.errorClass; + delete projected.outcome; + await persistProjection(context, projection); + throw error; + } + projected.outcome = "unknown"; + throw new JobObservationPending("recovery_required"); + } projected.failureResult = error.failureResult; projected.errorClass = error.errorClass; projected.terminal = true; @@ -459,7 +769,8 @@ async function observeOperation( ...error.failureResult, ...(error.errorClass === "host_update_required" ? { errorCode: error.errorClass } : {}), }; - } else { + await ensureAdminJobReceiptDelivery(context, projection, projected, client, terminalStatus); + } else if (!(error instanceof JobObservationPending && error.reason === "receipt_delivery")) { projected.outcome = "unknown"; } throw error; @@ -517,9 +828,15 @@ export async function resumeAdminJobObservation( const completion = evidence._completion as | { operationKey?: string; result?: Record } | undefined; + const completionAddress = completion?.operationKey; if ( - completion?.operationKey && - entries.some((entry) => entry.operationKey === completion.operationKey && entry.terminal) + completionAddress && + entries.some( + (entry) => + entry.operationKey === + (projection.steps[completionAddress]?.operationKey ?? completionAddress) && + entry.terminal, + ) ) return { ...evidence, ...completion.result }; // Single-effect jobs have a complete retained receipt. Multi-step workflows @@ -552,6 +869,7 @@ function hasUnsettledHardlinks(result: unknown): boolean { } export interface CancelAdminJobOperationsOptions { + context?: JobContext; pollIntervalMs?: number; sleep?: (ms: number) => Promise; signal?: AbortSignal; @@ -598,6 +916,33 @@ export async function cancelAdminJobOperations( ? AbortSignal.any([options.signal, AbortSignal.timeout(2_000)]) : AbortSignal.timeout(2_000); for (const operation of pending) { + if (operation.phase === "refused") { + outcomes.push("already_terminal"); + continue; + } + if (operation.terminal && operation.receiptDelivery) { + if (!options.context) return "pending"; + try { + await ensureAdminJobReceiptDelivery( + options.context, + projection, + operation, + client as ApiOpsClient, + ); + } catch (error) { + if (error instanceof JobObservationPending) throw error; + return "pending"; + } + const receipt = operation.receiptDelivery.terminalReceipt; + outcomes.push( + receipt.state === "succeeded" + ? "succeeded" + : receipt.terminationRequestedAt !== undefined + ? "contained" + : "already_terminal", + ); + continue; + } if (options.signal?.aborted) return "pending"; if (options.jobId && operation.attempt && operation.attempt.apiJobId !== options.jobId) return "pending"; @@ -676,6 +1021,30 @@ export async function cancelAdminJobOperations( if (attempt + 1 < CANCEL_POLL_ATTEMPTS) await sleep(pollIntervalMs); } if (!status || !["succeeded", "failed", "timed_out"].includes(status.state)) return "pending"; + operation.terminal = true; + operation.phase = "terminal"; + operation.fingerprint = status.fingerprint; + operation.deadlineAt = status.deadlineAt; + if (status.state === "succeeded") operation.result = status.result; + else { + operation.errorClass = status.errorClass; + operation.failureResult = status.failureResult; + } + if (projection.version === 3 && opsKindRequiresControlAuthority(operation.kind)) { + if (!options.context) return "pending"; + try { + await ensureAdminJobReceiptDelivery( + options.context, + projection, + operation, + client as ApiOpsClient, + status, + ); + } catch (error) { + if (error instanceof JobObservationPending) throw error; + return "pending"; + } + } outcomes.push( status.state === "succeeded" ? "succeeded" diff --git a/apps/api/src/services/admin-job-parent.ts b/apps/api/src/services/admin-job-parent.ts index 500932451..91b12faa0 100644 --- a/apps/api/src/services/admin-job-parent.ts +++ b/apps/api/src/services/admin-job-parent.ts @@ -1,7 +1,9 @@ import { createHash, createHmac, randomUUID } from "node:crypto"; import { canonicalJson, + createControlRootOperationKey, type OpsOperation, + type OpsOperationKind, opsExecutionBudget, opsOperationFingerprint, readOpsTokenFile, @@ -24,7 +26,7 @@ export const apiJobChildAttemptSchema = z.strictObject({ }); export type ApiJobChildAttempt = z.infer; -function parentJobBudget(ctx: JobContext, operation: OpsOperation): number { +function parentJobBudget(ctx: JobContext, kind: OpsOperationKind): number { if (ctx.jobType === "system.update") return opsExecutionBudget("system.update").processMs; if (ctx.jobType === "service.start" || ctx.jobType === "service.apply") { // Fixed current workflow: render, inspect, start the coordinator, await its @@ -38,42 +40,65 @@ function parentJobBudget(ctx: JobContext, operation: OpsOperation): number { ].reduce((sum, kind) => sum + opsExecutionBudget(kind).processMs, 60_000); } if (ctx.jobType === "service.bulk") return opsExecutionBudget("services.buildAll").processMs; - return opsExecutionBudget(operation.kind).processMs; + return opsExecutionBudget(kind).processMs; } -/** These non-secret fields are checkpointed before the assertion is minted. */ -export function prepareApiJobChildAttempt( +export interface ApiPreparedRoot { + operationKey: string; + requestedDeadlineAt: number; + parent: { operationKey: string; fingerprint: string; deadlineAt: number }; +} +/** Pure preparation. Sealed configuration claims need the final key before + * preparing their body; the complete candidate is still committed in one CAS. */ +export function prepareApiJobRoot( ctx: JobContext, - operation: OpsOperation, - key: string, -): ApiJobChildAttempt | undefined { - if (!ctx.execution) return undefined; - const retained = ctx.checkpointResult?._parentExecution as - | { operationKey: string; fingerprint: string; deadlineAt: number } - | undefined; + kind: OpsOperationKind, + stepAddress: string, +): ApiPreparedRoot { + if (!ctx.execution) throw Error("A durable administrative job owner is required"); + const retained = ctx.checkpointResult?._parentExecution as ApiPreparedRoot["parent"] | undefined; const parent = retained ?? { operationKey: createDurableOpsKey("admin-job.parent", ctx.jobId), - // Job identity, not its potentially secret payload, binds the parent. fingerprint: createHash("sha256") .update(canonicalJson({ jobId: ctx.jobId })) .digest("hex"), - deadlineAt: Date.now() + parentJobBudget(ctx, operation), + deadlineAt: Date.now() + parentJobBudget(ctx, kind), }; - ctx.checkpointResult = { ...ctx.checkpointResult, _parentExecution: parent }; + const requestedDeadlineAt = Math.min( + parent.deadlineAt, + Date.now() + opsExecutionBudget(kind).processMs, + ); + const nonce = createHash("sha256") + .update("openmapx-api-control-root-nonce-v1\0") + .update(canonicalJson({ jobId: ctx.jobId, stepAddress })) + .digest() + .subarray(0, 16); return { + operationKey: createControlRootOperationKey({ notAfter: requestedDeadlineAt, nonce }), + requestedDeadlineAt, + parent, + }; +} +/** These non-secret fields are checkpointed before the assertion is minted. */ +export function prepareApiJobChildAttempt( + ctx: JobContext, + operation: OpsOperation, + key: string, + prepared?: ApiPreparedRoot, +): ApiJobChildAttempt | undefined { + if (!ctx.execution) return undefined; + const root = prepared ?? prepareApiJobRoot(ctx, operation.kind, key); + return apiJobChildAttemptSchema.parse({ version: 1, apiJobId: ctx.jobId, apiOwnerVersion: ctx.execution.version, - parentOperationKey: parent.operationKey, - parentFingerprint: parent.fingerprint, - parentDeadlineAt: parent.deadlineAt, + parentOperationKey: root.parent.operationKey, + parentFingerprint: root.parent.fingerprint, + parentDeadlineAt: root.parent.deadlineAt, childOperationKey: key, childOperationFingerprint: opsOperationFingerprint(operation), - requestedDeadlineAt: Math.min( - parent.deadlineAt, - Date.now() + opsExecutionBudget(operation.kind).processMs, - ), + requestedDeadlineAt: root.requestedDeadlineAt, attemptId: randomUUID(), - }; + }); } export async function issueApiJobChildAssertion(attempt: ApiJobChildAttempt): Promise { const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); diff --git a/apps/api/src/services/admin-job-state.ts b/apps/api/src/services/admin-job-state.ts index 720949b45..e44d126c3 100644 --- a/apps/api/src/services/admin-job-state.ts +++ b/apps/api/src/services/admin-job-state.ts @@ -3,6 +3,7 @@ import { and, eq, inArray, sql } from "drizzle-orm"; import { db } from "../db"; import { adminJob, adminJobLog } from "../db/schema"; +export const ADMIN_JOB_LOG_LIMITS = { events: 2000, bytes: 1024 * 1024 } as const; export interface JobExecution { owner: string; version: number; @@ -12,6 +13,44 @@ export class JobOwnershipLost extends Error { super("Administrative job ownership changed"); } } +export class JobStepAllocationConflict extends Error { + constructor(readonly checkpoint: Record) { + super("Stable administrative step already reserved"); + } +} +export class JobStepCapacityExceeded extends Error { + constructor() { + super("Administrative step capacity exhausted"); + } +} +export function hasPendingReceiptDelivery(result: unknown): boolean { + const projection = ( + result as { + opsProjection?: { + version?: number; + operations?: Record; + }; + } | null + )?.opsProjection; + return ( + projection?.version === 3 && + Object.values(projection.operations ?? {}).some((o) => o.receiptDelivery?.state === "pending") + ); +} +export function hasUnsettledManagedOperation(result: unknown): boolean { + const p = ( + result as { + opsProjection?: { version?: number; operations?: Record }; + } | null + )?.opsProjection; + return p?.version === 3 && Object.values(p.operations ?? {}).some((o) => o.terminal !== true); +} +export function noUnsettledManagedOperation() { + return sql`NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS(SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS active_entry WHERE coalesce(active_entry.value->>'terminal','false') <> 'true'))`; +} +export function noPendingReceiptDelivery() { + return sql`NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS (SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS receipt_entry WHERE receipt_entry.value->'receiptDelivery'->>'state' = 'pending'))`; +} export function readJobExecution(result: unknown): JobExecution | undefined { if ( result !== null && @@ -48,7 +87,7 @@ export function jobOwnership( export const ADMIN_JOB_LEASE_MS = 60_000; export function adminJobObservationDue(now = new Date().toISOString()) { return and( - sql`(${adminJob.status} = 'cancel_pending' OR ${adminJob.result}->'observation'->>'state' = 'unknown' OR ${adminJob.result}->>'_observationLeaseUntil' IS NOT NULL)`, + sql`(${adminJob.status} = 'cancel_pending' OR ${adminJob.result}->'observation'->>'state' IN ('unknown','receipt_delivery') OR ${adminJob.result}->>'_observationLeaseUntil' IS NOT NULL)`, sql`coalesce(${adminJob.result}->'observation'->>'nextCheckAt', '') <= ${now}`, sql`coalesce(${adminJob.result}->>'_observationLeaseUntil', '') <= ${now}`, ); @@ -75,11 +114,20 @@ export async function insertOwnedJobLog( ): Promise { const [result] = await db.execute<{ owned: boolean; inserted: boolean }>(sql` WITH owner AS MATERIALIZED ( - SELECT ${adminJob.id} FROM ${adminJob} WHERE ${jobOwnership(entry.jobId, execution)} FOR UPDATE + SELECT ${adminJob.id}, ${adminJob.result} FROM ${adminJob} WHERE ${jobOwnership(entry.jobId, execution)} FOR UPDATE ), inserted AS ( INSERT INTO ${adminJobLog} (id, job_id, seq, stream, line, created_at) SELECT ${entry.id}, ${entry.jobId}, ${entry.seq}, ${entry.stream}, ${entry.line}, ${entry.createdAt?.toISOString()}::timestamp - FROM owner ON CONFLICT (id) DO NOTHING RETURNING id + FROM owner WHERE coalesce(owner.result->'opsProjection'->>'version','') <> '3' OR ( + coalesce((owner.result->>'_opsLogEvents')::integer,0) < ${ADMIN_JOB_LOG_LIMITS.events} + AND coalesce((owner.result->>'_opsLogBytes')::integer,0) + ${Buffer.byteLength(entry.line, "utf8")} <= ${ADMIN_JOB_LOG_LIMITS.bytes} + ) ON CONFLICT (id) DO NOTHING RETURNING job_id + ), charged AS ( + UPDATE ${adminJob} SET result=coalesce(${adminJob.result},'{}'::jsonb) || jsonb_build_object( + '_opsLogEvents',coalesce((owner.result->>'_opsLogEvents')::integer,0)+1, + '_opsLogBytes',coalesce((owner.result->>'_opsLogBytes')::integer,0)+${Buffer.byteLength(entry.line, "utf8")}) + FROM owner,inserted WHERE ${adminJob.id}=owner.id AND inserted.job_id=owner.id AND owner.result->'opsProjection'->>'version'='3' + RETURNING ${adminJob.id} ) SELECT EXISTS (SELECT 1 FROM owner) AS owned, EXISTS (SELECT 1 FROM inserted) AS inserted `); diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 7f9aebc4c..0b20bdef7 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -3,15 +3,22 @@ import { and, asc, eq, inArray, sql } from "drizzle-orm"; import { db } from "../db"; import { adminJob } from "../db/schema"; import { dbActorId } from "../utils/actor"; +import { checkpointOwnedAdminJob } from "./admin-job-checkpoint"; import { AdminJobFailure, JobObservationPending, publicAdminJobResult } from "./admin-job-failure"; import { cancelAdminJobOperations, resumeAdminJobObservation } from "./admin-job-ops"; import { adminJobObservationDue, claimAdminJob, + hasPendingReceiptDelivery, + hasUnsettledManagedOperation, insertOwnedJobLog, type JobExecution, JobOwnershipLost, + JobStepAllocationConflict, + JobStepCapacityExceeded, jobOwnership, + noPendingReceiptDelivery, + noUnsettledManagedOperation, readJobExecution, renewAdminJobClaim, } from "./admin-job-state"; @@ -45,11 +52,36 @@ export interface JobContext { result: Record, progress?: number, admissionKey?: string, + stepAddress?: string, ): Promise; /** Last durable checkpoint loaded with the job and updated after each checkpoint. */ checkpointResult?: Record | null; } +function cancellationReceiptContext(job: typeof adminJob.$inferSelect): JobContext { + const context: JobContext = { + jobId: job.id, + jobType: job.type, + execution: readJobExecution(job.result), + checkpointResult: job.result as Record | null, + payload: {}, + signal: new AbortController().signal, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (value) => { + context.checkpointResult = await checkpointOwnedAdminJob( + job.id, + context.execution, + value, + undefined, + undefined, + undefined, + context.checkpointResult ?? {}, + ); + }, + }; + return context; +} type JobHandler = (ctx: JobContext) => Promise | undefined>; export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ @@ -110,7 +142,14 @@ class AdminJobRunner { const queued = await db .update(adminJob) .set({ status: "canceled", finishedAt: canceledAt }) - .where(and(eq(adminJob.id, jobId), eq(adminJob.status, "queued"))) + .where( + and( + eq(adminJob.id, jobId), + eq(adminJob.status, "queued"), + noPendingReceiptDelivery(), + noUnsettledManagedOperation(), + ), + ) .returning({ id: adminJob.id }); if (queued.length > 0) { this.announce(jobId, "canceled", { finishedAt: canceledAt.toISOString() }); @@ -132,6 +171,7 @@ class AdminJobRunner { let outcome: Awaited> = "pending"; try { outcome = await cancelAdminJobOperations(job.result, undefined, { + context: cancellationReceiptContext(job), jobId, onEvidence: async (evidence) => { await db @@ -142,7 +182,20 @@ class AdminJobRunner { .where(jobOwnership(jobId, readJobExecution(job.result), ["cancel_pending"])); }, }); - } catch { + } catch (error) { + if (error instanceof JobObservationPending && error.reason === "receipt_delivery") { + const observation = { + state: "receipt_delivery", + reason: "receipt_delivery", + nextCheckAt: new Date(Date.now() + 5000).toISOString(), + }; + await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ observation })}::jsonb`, + }) + .where(jobOwnership(jobId, readJobExecution(job.result), ["cancel_pending"])); + } /* Retain intent. */ } if (outcome !== "canceled") return false; @@ -154,7 +207,13 @@ class AdminJobRunner { finishedAt, result: sql`coalesce(${adminJob.result}, '{}'::jsonb) - 'observation'`, }) - .where(jobOwnership(jobId, readJobExecution(job.result), ["cancel_pending"])) + .where( + and( + jobOwnership(jobId, readJobExecution(job.result), ["cancel_pending"]), + noPendingReceiptDelivery(), + noUnsettledManagedOperation(), + ), + ) .returning({ id: adminJob.id, result: adminJob.result }); if (rows.length === 0) return false; this.announce(jobId, "canceled", { @@ -378,6 +437,7 @@ class AdminJobRunner { const handler = observe ? resumeAdminJobObservation : this.handlers.get(type); const execution = readJobExecution(checkpointResult); let checkpointTail = Promise.resolve(); + let checkpointBaseline = structuredClone((checkpointResult ?? {}) as Record); const ctx: JobContext = { jobId, jobType: type, @@ -431,8 +491,30 @@ class AdminJobRunner { if (!rows.length) lost(); jobEventBus.publish(jobId, { type: "progress", progress }); }, - checkpoint: async (result, progress, admissionKey) => { + checkpoint: async (result, progress, admissionKey, stepAddress) => { const write = checkpointTail.then(async () => { + const isV3 = (result.opsProjection as { version?: number } | undefined)?.version === 3; + if (isV3) { + if (admissionKey) signal.throwIfAborted(); + try { + ctx.checkpointResult = await checkpointOwnedAdminJob( + jobId, + execution, + result, + progress, + admissionKey, + stepAddress, + checkpointBaseline, + ); + checkpointBaseline = structuredClone(ctx.checkpointResult); + } catch (error) { + if (error instanceof JobStepAllocationConflict) + ctx.checkpointResult = error.checkpoint; + throw error; + } + if (progress !== undefined) jobEventBus.publish(jobId, { type: "progress", progress }); + return; + } signal.throwIfAborted(); // A handler cannot replace the runner's ownership member. const merged: Record = { @@ -464,7 +546,13 @@ class AdminJobRunner { }); checkpointTail = write.catch(() => undefined); return write.catch((error) => { - if (error instanceof JobOwnershipLost || signal.aborted) throw error; + if ( + error instanceof JobOwnershipLost || + error instanceof JobStepAllocationConflict || + error instanceof JobStepCapacityExceeded || + signal.aborted + ) + throw error; throw new JobPersistenceUnavailable("Administrative job checkpoint unavailable"); }); }, @@ -472,14 +560,14 @@ class AdminJobRunner { if (!observe) this.announce(jobId, "running", { startedAt: new Date().toISOString() }); const startedAtMs = Date.now(); - const pending = async (reason: OpsObservationPending["reason"]) => { + const pending = async (reason: OpsObservationPending["reason"] | "receipt_delivery") => { try { const attempts = Math.min(12, Number(ctx.checkpointResult?._observationAttempts ?? 0) + 1); const observationPatch = { _observationAttempts: attempts, _observationLeaseUntil: null, observation: { - state: "unknown", + state: reason === "receipt_delivery" ? "receipt_delivery" : "unknown", reason, nextCheckAt: new Date( Date.now() + Math.min(60_000, 5_000 * 2 ** (attempts - 1)), @@ -505,6 +593,7 @@ class AdminJobRunner { status: "success" | "failed", result: Record | null, error?: string, + knownFailure = false, ) => { try { await checkpointTail; @@ -514,17 +603,42 @@ class AdminJobRunner { ...(execution ? { _execution: execution } : {}), }; delete merged.observation; + if (hasUnsettledManagedOperation(ctx.checkpointResult)) { + if (status === "success" || knownFailure) + await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _settled: { status, result: publicAdminJobResult(merged), error } })}::jsonb`, + }) + .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])); + throw new JobObservationPending("recovery_required"); + } + if (hasPendingReceiptDelivery(ctx.checkpointResult)) { + await db + .update(adminJob) + .set({ + result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _settled: { status, result: publicAdminJobResult(merged), error } })}::jsonb`, + }) + .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])); + throw new JobObservationPending("receipt_delivery"); + } const finishedAt = new Date(); const rows = await db .update(adminJob) .set({ status, - result: merged, + result: sql`(coalesce(${adminJob.result}, '{}'::jsonb) - 'observation') || ${JSON.stringify(Object.fromEntries(Object.entries(merged).filter(([key]) => !["opsProjection", "_opsLogEvents", "_opsLogBytes"].includes(key))))}::jsonb`, error: error ?? null, finishedAt, ...(status === "success" ? { progress: 100 } : {}), }) - .where(jobOwnership(jobId, execution)) + .where( + and( + jobOwnership(jobId, execution), + noPendingReceiptDelivery(), + noUnsettledManagedOperation(), + ), + ) .returning({ id: adminJob.id }); if (rows.length) { this.announce(jobId, status, { @@ -540,9 +654,11 @@ class AdminJobRunner { .set({ result: sql`coalesce(${adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _settled: { status, result: publicAdminJobResult(merged), error } })}::jsonb`, }) - .where(jobOwnership(jobId, execution, ["cancel_pending"])); + .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])); + await pending("receipt_delivery"); } - } catch { + } catch (error) { + if (error instanceof JobObservationPending) throw error; throw new JobPersistenceUnavailable("Administrative job persistence unavailable"); } }; @@ -558,6 +674,7 @@ class AdminJobRunner { if (!current) return; if (current.status === "cancel_pending") { const outcome = await cancelAdminJobOperations(current.result, undefined, { + context: ctx, jobId, signal, onEvidence: async (evidence) => { @@ -583,7 +700,13 @@ class AdminJobRunner { finishedAt, result: sql`coalesce(${adminJob.result}, '{}'::jsonb) - 'observation'`, }) - .where(jobOwnership(jobId, execution, ["cancel_pending"])) + .where( + and( + jobOwnership(jobId, execution, ["cancel_pending"]), + noPendingReceiptDelivery(), + noUnsettledManagedOperation(), + ), + ) .returning({ id: adminJob.id, result: adminJob.result }); if (rows.length) this.announce(jobId, "canceled", { @@ -606,8 +729,8 @@ class AdminJobRunner { // turn an accepted successful effect into failed because PostgreSQL failed. try { await settle("success", result ?? null); - } catch { - await pending("transport"); + } catch (error) { + await pending(error instanceof JobObservationPending ? error.reason : "transport"); } } catch (err) { if (err instanceof JobOwnershipLost || err instanceof JobPersistenceUnavailable) return; @@ -617,16 +740,23 @@ class AdminJobRunner { await pending("observer_deadline"); else if (observe && !(err instanceof AdminJobFailure) && !(err instanceof ApiOpsError)) await pending("transport"); - else - await settle( - "failed", - err instanceof AdminJobFailure - ? err.result - : err instanceof ApiOpsError - ? (err.failureResult ?? null) - : null, - err instanceof Error ? err.message : "Administrative job failed", - ); + else { + try { + await settle( + "failed", + err instanceof AdminJobFailure + ? err.result + : err instanceof ApiOpsError + ? (err.failureResult ?? null) + : null, + err instanceof Error ? err.message : "Administrative job failed", + err instanceof AdminJobFailure || err instanceof ApiOpsError, + ); + } catch (error) { + if (error instanceof JobObservationPending) await pending(error.reason); + else if (!(error instanceof JobPersistenceUnavailable)) throw error; + } + } } finally { jobEventBus.recordHandlerDuration(Date.now() - startedAtMs); } diff --git a/apps/api/src/services/ops-client.ts b/apps/api/src/services/ops-client.ts index 058939244..713f0f13d 100644 --- a/apps/api/src/services/ops-client.ts +++ b/apps/api/src/services/ops-client.ts @@ -20,6 +20,7 @@ import { type OpsOperationKind, type OpsResultFor, type OpsSubmitResult, + opsKindRequiresControlAuthority, PRIVACY_BACKUP_TAR_MEDIA_TYPE, type PrivacyBackupSubjectExportRequest, privacyBackupSubjectExportRequestSchema, @@ -45,8 +46,8 @@ export class ApiOpsError extends Error { readonly failureResult?: OpsFailureResult, ) { super( - errorClass === "host_update_required" - ? OPS_PUBLIC_ERROR_MESSAGES.host_update_required + errorClass === "host_update_required" || errorClass === "control_upgrade_required" + ? OPS_PUBLIC_ERROR_MESSAGES[errorClass] : "Operations request failed", ); this.name = "ApiOpsError"; @@ -540,6 +541,9 @@ export async function executeAndWait( options: WaitForOpsResultOptions = {}, ): Promise> { throwIfAborted(options.signal); + if (opsKindRequiresControlAuthority(operation.kind) && !options.parentAssertion) { + throw new ApiOpsError("control_upgrade_required"); + } try { const submitted = await submitOpsOperationWithRecovery(client, operation, operationKey, { signal: options.signal, diff --git a/apps/api/src/services/trusted-config-operations.ts b/apps/api/src/services/trusted-config-operations.ts index ac9cf1e60..7803c9b17 100644 --- a/apps/api/src/services/trusted-config-operations.ts +++ b/apps/api/src/services/trusted-config-operations.ts @@ -16,6 +16,7 @@ import { executeAdminJobOperation, hasRetainedAdminJobOperation, observeRetainedAdminJobOperation, + prepareAdminJobOperationRoot, } from "./admin-job-ops"; import { projectIntegrationConfig } from "./integration-config-projection"; import type { JobContext } from "./job-runner"; @@ -31,6 +32,7 @@ import { resolveServiceVaultSecretsStrict } from "./service-secrets"; import { TrustedConfigurationApplyError, type TrustedConfigurationApplyOutcome, + TrustedConfigurationUpgradeRequired, } from "./trusted-config-outcome"; import { consumePublishedTrustedConfiguration, @@ -156,6 +158,12 @@ export async function applyTrustedConfigurationWithEvidence( } return { outcome: retained }; } + if (!options.ctx) throw new TrustedConfigurationUpgradeRequired(); + const preparedRoot = prepareAdminJobOperationRoot( + options.ctx, + options.kind, + options.operationKey, + ); let payload: TrustedConfigurationPayload; let previousApplied: OpsResultFor<"configuration.inspect">; try { @@ -186,7 +194,7 @@ export async function applyTrustedConfigurationWithEvidence( return { payload, previousApplied, - outcome: await applySnapshot(options, payload, previousApplied.revisionId), + outcome: await applySnapshot(options, payload, previousApplied.revisionId, preparedRoot), }; } @@ -196,6 +204,7 @@ export async function applyTrustedConfigurationSnapshot(options: { expectedCurrentRevisionId: string | null; operationKey: string; signal?: AbortSignal; + ctx?: JobContext; }): Promise { if ( !configurationGenerationRevisionSchema.nullable().safeParse(options.expectedCurrentRevisionId) @@ -210,7 +219,12 @@ export async function applyTrustedConfigurationSnapshot(options: { }; } return applySnapshot( - { kind: "stack.render", operationKey: options.operationKey, signal: options.signal }, + { + kind: "stack.render", + operationKey: options.operationKey, + signal: options.signal, + ctx: options.ctx, + }, options.payload, options.expectedCurrentRevisionId, ); @@ -220,7 +234,21 @@ async function applySnapshot( options: ApplyTrustedConfigurationOptions, input: TrustedConfigurationPayload, expectedCurrentRevisionId: string | null, + rootCandidate?: import("./admin-job-parent").ApiPreparedRoot, ): Promise { + if (!options.ctx) throw new TrustedConfigurationUpgradeRequired(); + if (hasRetainedAdminJobOperation(options.ctx, options.operationKey)) { + await observeRetainedAdminJobOperation(options.ctx, options.operationKey); + const outcome = ( + options.ctx.checkpointResult?._configurationOutcomes as + | Record + | undefined + )?.[options.operationKey]; + if (!outcome) throw new JobObservationPending("recovery_required"); + return outcome; + } + const preparedRoot = + rootCandidate ?? prepareAdminJobOperationRoot(options.ctx, options.kind, options.operationKey); let submitted = false; try { const payload = trustedConfigurationPayloadSchema.parse(input); @@ -262,7 +290,7 @@ async function applySnapshot( token, ownerUid: exactInteger(process.env, "OPS_TRUSTED_CONFIG_UID"), ownerGid: exactInteger(process.env, "OPS_TRUSTED_CONFIG_GID"), - operationKey: options.operationKey, + operationKey: preparedRoot.operationKey, operationForRevision, payload, }); @@ -272,6 +300,7 @@ async function applySnapshot( if (options.ctx) return executeAdminJobOperation(options.ctx, sealed.operation, "admin-job.configuration", { operationKey: options.operationKey, + preparedRoot, }).then((result) => configurationApplyResultSchema.parse(result)); return executeTrustedConfigurationAndWait( client, diff --git a/apps/api/src/services/trusted-config-outcome.ts b/apps/api/src/services/trusted-config-outcome.ts index 7400d0ee4..018fa066c 100644 --- a/apps/api/src/services/trusted-config-outcome.ts +++ b/apps/api/src/services/trusted-config-outcome.ts @@ -16,6 +16,22 @@ export class TrustedConfigurationApplyError extends Error { } } +/** A known local refusal before snapshot publication or mutation dispatch. */ +export class TrustedConfigurationUpgradeRequired extends TrustedConfigurationApplyError { + constructor() { + super({ + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }); + this.name = "TrustedConfigurationUpgradeRequired"; + this.message = + "Update this configuration producer to use a durable administrative job before submitting the mutation."; + } +} + /** Only explicit local/host evidence permits callers to consider DB restoration. */ export function isProvenConfigurationNonCommit(error: unknown): boolean { return ( diff --git a/apps/ops-agent/src/control-admission.test.ts b/apps/ops-agent/src/control-admission.test.ts index fd195986d..2fdaf19b2 100644 --- a/apps/ops-agent/src/control-admission.test.ts +++ b/apps/ops-agent/src/control-admission.test.ts @@ -1,9 +1,10 @@ -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { + createControlRootOperationKey, type ExecutionRef, executionFingerprint, opsOperationFingerprint, @@ -40,7 +41,7 @@ it.each(["normal", "ready-failure", "wrong-target"])( executor: "data-manager", principal: "data-manager", executionId: randomUUID(), - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint: rootHash, deadlineAt, fingerprint: executionFingerprint({ @@ -72,6 +73,7 @@ it.each(["normal", "ready-failure", "wrong-target"])( recipient: "ops-agent" as const, }; const authority = control.retainAdmittedControlExecution(dm, { + requestedDeadlineAt: owner.deadlineAt, execution: owner, kind: operation.kind, operation, diff --git a/apps/ops-agent/src/control-renewal.test.ts b/apps/ops-agent/src/control-renewal.test.ts new file mode 100644 index 000000000..d9c82602e --- /dev/null +++ b/apps/ops-agent/src/control-renewal.test.ts @@ -0,0 +1,225 @@ +import { createHmac, randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as c from "@openmapx/core/build-artifacts"; +import { + controlTerminalReceiptDigest, + createControlRootOperationKey, + type OpsJobStatus, + opsOperationFingerprint, + parseOpsJobStatus, + signApiRetentionProof, +} from "@openmapx/core/ops"; +import { expect, it, vi } from "vitest"; +import { openOpsJobJournal } from "./journal"; +import { buildOpsAgentServer } from "./server"; + +it("retains the exact API terminal dependency until ack, retires it, and discharges lost ack responses after GC without replay", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-agent-renewal-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await c.provisionControlInstallation(root); + const installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const token = "a".repeat(32), + dmToken = "b".repeat(32), + journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")), + dispatch = vi.fn(async () => ({ backupId: "daily" })); + const app = buildOpsAgentServer({ + tokens: { api: token, "data-manager": dmToken }, + controlInstallation: installation, + journal, + dispatch, + now: () => new Date(Date.now()), + resourcePolicy: { allowBackup: () => true, allowGlobal: () => true }, + }); + const operation = { kind: "backup.delete" as const, backupId: "daily" }, + notAfter = Date.now() + 60000, + key = createControlRootOperationKey({ notAfter, nonce: randomBytes(16) }), + apiJobId = randomUUID(), + attemptId = randomUUID(); + const fields = { + version: 1, + apiJobId, + apiOwnerVersion: 1, + parentOperationKey: "opk1_parentabcdefghijkl", + parentFingerprint: "b".repeat(64), + parentDeadlineAt: notAfter, + childOperationKey: key, + childOperationFingerprint: opsOperationFingerprint(operation), + requestedDeadlineAt: notAfter, + attemptId, + }; + const payload = Buffer.from(JSON.stringify(fields)).toString("base64url"), + parent = `${payload}.${createHmac("sha256", token).update("openmapx-api-job-parent-v1\0").update(payload).digest("base64url")}`; + const request = ( + op: unknown, + operationKey = key, + headers: Record = {}, + execution?: { requestedDeadlineAt: number }, + ) => + app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2", ...headers }, + payload: { + version: 2, + requestId: `ops1_${randomUUID().replaceAll("-", "")}`, + operationKey, + issuedAt: new Date(Date.now()).toISOString(), + expiresAt: new Date(Date.now() + 10000).toISOString(), + operation: op, + ...(execution ? { execution } : {}), + }, + }); + try { + expect((await request(operation, "opk1_oldclientabcdefghijkl")).statusCode).toBe(409); + expect(dispatch).not.toHaveBeenCalled(); + expect(journal.records()).toHaveLength(0); + const admitted = await request( + operation, + key, + { "x-openmapx-parent-execution": parent }, + { requestedDeadlineAt: notAfter }, + ); + expect(admitted.statusCode, admitted.body).toBe(202); + const id = admitted.json().result.operationId; + let terminal: OpsJobStatus | undefined; + await vi.waitFor( + async () => { + const response = await app.inject({ + url: `/v1/operations/${id}`, + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }, + }); + terminal = parseOpsJobStatus(response.json().result); + expect(terminal?.state).toBe("succeeded"); + }, + { timeout: 15000 }, + ); + await vi.waitFor( + () => { + const p: { executions: Record } = JSON.parse( + readFileSync( + join(root, "control-private/ops-agent/retention-families/slot-0/progress.json"), + "utf8", + ), + ); + expect(Object.values(p.executions).every((e) => e.journalContained)).toBe(true); + }, + { timeout: 15000 }, + ); + if (!terminal?.fingerprint) throw Error("Missing synthetic terminal identity"); + const digest = controlTerminalReceiptDigest(terminal), + proofFields = { + version: 1 as const, + apiJobId, + originalApiOwnerVersion: 1, + attemptId, + checkpointOwnerVersion: 1, + terminalCopyId: randomUUID(), + operationKey: key, + fingerprint: terminal.fingerprint, + terminalDigest: digest, + }; + const ack = { + kind: "controlRetention.acknowledge", + originRootKey: key, + operationKey: key, + fingerprint: terminal.fingerprint, + terminalDigest: digest, + proof: signApiRetentionProof(token, proofFields), + }; + const wall = Date.now(), + clock = vi.spyOn(Date, "now").mockReturnValue(notAfter + 96 * 3600_000); + expect(await c.reconcileControlRetention(installation, { maxFamilies: 1 })).toMatchObject({ + retired: 0, + pending: 1, + }); + expect(journal.records()).toHaveLength(1); + const familyHash = JSON.parse( + readFileSync( + join(root, "control-private/ops-agent/retention-families/slot-0/identity.json"), + "utf8", + ), + ).originHash; + const ackCell = join( + root, + `data/.admin-target-control/retention-families/${familyHash}/api-ack.json`, + ); + writeFileSync( + ackCell, + JSON.stringify({ + operationKey: key, + fingerprint: terminal.fingerprint, + terminalDigest: digest, + proofDigest: "c".repeat(64), + }), + ); + expect(await c.reconcileControlRetention(installation, { maxFamilies: 1 })).toMatchObject({ + retired: 0, + pending: 1, + }); + expect(journal.records()).toHaveLength(1); + writeFileSync(ackCell, "null"); + + expect( + ( + await request( + operation, + key, + { "x-openmapx-parent-execution": parent }, + { requestedDeadlineAt: notAfter }, + ) + ).statusCode, + ).toBe(202); + expect(dispatch).toHaveBeenCalledTimes(1); + const wrong = { + ...ack, + proof: signApiRetentionProof(token, { ...proofFields, attemptId: randomUUID() }), + }; + expect((await request(wrong, "opk1_ackrequest123456")).statusCode).toBe(409); + expect((await request(ack, "opk1_ackrequest123456")).statusCode).toBe(200); + expect((await request(ack, "opk1_ackrequest123456")).statusCode).toBe(200); + expect( + ( + await request( + { + ...ack, + proof: signApiRetentionProof(token, { ...proofFields, terminalCopyId: randomUUID() }), + }, + "opk1_ackrequest123456", + ) + ).statusCode, + ).toBe(409); + expect( + (await request(ack, "opk1_ackrequest123456", { authorization: `Bearer ${dmToken}` })) + .statusCode, + ).toBe(403); + expect(await c.reconcileControlRetention(installation, { maxFamilies: 1 })).toMatchObject({ + retired: 1, + pending: 0, + }); + expect(journal.records()).toHaveLength(0); + clock.mockReturnValue(wall); + expect((await request(ack, "opk1_ackrequest123456")).statusCode).toBe(200); + expect( + ( + await request( + operation, + key, + { "x-openmapx-parent-execution": parent }, + { requestedDeadlineAt: notAfter }, + ) + ).statusCode, + ).toBe(409); + expect(dispatch).toHaveBeenCalledTimes(1); + } finally { + vi.restoreAllMocks(); + await app.close(); + c.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } +}, 60000); diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 0b58131f1..cbe19e3b6 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -1,16 +1,24 @@ import { createHash, randomBytes as nodeRandomBytes, timingSafeEqual } from "node:crypto"; import { + acknowledgeControlRetention, assertArtifactTargetLeaseForward, + assertFreshControlRoot, type BorrowedArtifactTargetLease, beginControlEffect, type ControlInstallation, cancelArtifactTargetLease, closeArtifactTargetLeaseHandle, + completeControlJournalExecution, + controlRegisteredServiceIds, inspectControlAdmission, controlHeaders as readControlHeaders, + reconcileControlRetention, + registerControlRetentionJournal, + requireControlJournalRetention, restoreAdmittedControlExecution, retainAdmittedControlExecution, retainControlEffectReceipt, + retainControlTerminalStatus, retainedControlAdoptionAssertion, settleArtifactTargetLeaseChild, type VerifiedExecutionAuthority, @@ -20,6 +28,7 @@ import { import { type ConfigurationApplyResult, configurationApplyResultSchema, + controlTerminalReceiptDigest, type ExecutionRef, executionFingerprint, normalizeExecutionAdmission, @@ -42,11 +51,13 @@ import { type OpsResourcePolicy, type OpsRole, opsFailureResultSchema, + opsKindRequiresControlAuthority, opsOperationFingerprint, opsResourceId, opsSuccess, PROCESS_OUTPUT_TRUNCATED, parseBoundedOpsResult, + parseControlRootOperationKey, parseOpsRequest, processDiagnosticsSchema, redactedOpsError, @@ -333,7 +344,10 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const replay = new ReplayCache(options.maxReplayEntries ?? DEFAULT_REPLAY_ENTRIES); const maxConcurrency = options.maxConcurrency ?? DEFAULT_CONCURRENCY; const maxJobEntries = Math.min(options.maxJobEntries ?? DEFAULT_JOB_ENTRIES, DEFAULT_JOB_ENTRIES); - const jobRetentionMs = options.jobRetentionMs ?? DEFAULT_JOB_RETENTION_MS; + const jobRetentionMs = Math.max( + options.jobRetentionMs ?? DEFAULT_JOB_RETENTION_MS, + 72 * 60 * 60_000, + ); const journal: OpsJobJournal = options.journal ?? ({ records: () => [], replace: async () => undefined } satisfies OpsJobJournal); @@ -477,6 +491,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif (job.state === "failed" && job.errorClass === "recovery_required") || configurationResultsSettling.has(job.operationId) || job.terminalAt === undefined || + (options.controlInstallation && opsKindRequiresControlAuthority(job.operation.kind)) || job.terminalAt + jobRetentionMs > nowMs, ); if (retained.length === jobsById.size) return; @@ -485,6 +500,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if ( !(job.state === "failed" && job.errorClass === "recovery_required") && !configurationResultsSettling.has(job.operationId) && + !(options.controlInstallation && opsKindRequiresControlAuthority(job.operation.kind)) && job.terminalAt !== undefined && job.terminalAt + jobRetentionMs <= nowMs ) { @@ -540,7 +556,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const lease = controlLeases.get(job.operationId); const authority = controlAuthorities.get(job.operationId); if (lease && state === "termination_pending") await cancelArtifactTargetLease(lease); - if (lease && authority && ["succeeded", "failed", "timed_out"].includes(state)) { + if (authority && ["succeeded", "failed", "timed_out"].includes(state)) { if (!options.controlInstallation) throw Error("Missing control settlement installation"); // The fixed runtime's validated result is own-effect evidence. Failure alone is not containment. const committed = state === "succeeded" && updates.result !== undefined; @@ -551,10 +567,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ? { kind: "ops-result", result: updates.result } : { kind: "unresolved" }, }); - await settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(options.controlInstallation, authority.execution), - ); + if (lease) + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(options.controlInstallation, authority.execution), + ); } const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; await journal.replace( @@ -566,7 +583,6 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (lease && ["succeeded", "failed", "timed_out"].includes(state)) { closeArtifactTargetLeaseHandle(lease); controlLeases.delete(job.operationId); - controlAuthorities.delete(job.operationId); } job.cursor += 1; pushEvent(job, { @@ -707,6 +723,49 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif : { errorClass: job.errorClass }), }); + const unregisterRetention = options.controlInstallation + ? registerControlRetentionJournal(options.controlInstallation, { + contained: (execution, digest) => { + const job = jobsById.get(execution.executionId); + return ( + !!job && + !activeControllers.has(job.operationId) && + ["succeeded", "failed", "timed_out"].includes(job.state) && + !!digest && + controlTerminalReceiptDigest(statusFor(job, 2)) === digest + ); + }, + retire: async (executions, _manifestDigest) => + withAdmissionLock(async () => { + const remove = new Set(); + for (const { execution, digest } of executions) { + const job = jobsById.get(execution.executionId); + if (!job) continue; + if ( + activeControllers.has(job.operationId) || + job.operationKey !== execution.operationKey || + job.fingerprint !== execution.fingerprint || + controlTerminalReceiptDigest(statusFor(job, 2)) !== digest + ) + throw Error("Agent terminal journal changed during retirement"); + remove.add(job.operationId); + } + await journal.replace( + [...jobsById.values()].filter((job) => !remove.has(job.operationId)).map(persisted), + ); + for (const id of remove) { + const job = jobsById.get(id); + if (!job) throw Error("Missing retiring operation journal"); + jobsById.delete(id); + jobsByKey.delete(`${job.role}:${job.operationKey}`); + controlAuthorities.delete(id); + } + }), + }) + : undefined; + app.addHook("onClose", async () => { + unregisterRetention?.(); + }); const runAsync = async ( job: JobRecord, startedAt: number, @@ -726,7 +785,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const timeoutMs = Math.max(0, (job.deadlineAt ?? now().getTime()) - now().getTime()); let timedOut = false; let terminalDurable = false; - let claimReleased = false; + let claimReleased = !claimAdmission; let configurationEvidence: ConfigurationApplyResult | undefined; const timer = setTimeout(() => { timedOut = true; @@ -769,10 +828,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif terminalAt: now().getTime(), }); terminalDurable = true; - claimReleased = true; try { await claimAdmission?.release(); + claimReleased = true; } catch { + claimReleased = false; configurationEvidence = { ...result, cleanupFailed: true }; await transition(job, "succeeded", { result: configurationEvidence, @@ -870,15 +930,29 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif pushEvent(job, { cursor: job.cursor, type: "state", state: "succeeded" }); } if (terminalDurable && !claimReleased && !configurationEvidence) { - await claimAdmission?.release().catch(async () => { - stickyRecoveryRequired.add(job.operationId); - await transition(job, "failed", { - result: undefined, - errorClass: "recovery_required", - terminalAt: now().getTime(), - }).catch(() => forceRecoveryRequired(job)); - }); + await claimAdmission + ?.release() + .then(() => { + claimReleased = true; + }) + .catch(async () => { + stickyRecoveryRequired.add(job.operationId); + await transition(job, "failed", { + result: undefined, + errorClass: "recovery_required", + terminalAt: now().getTime(), + }).catch(() => forceRecoveryRequired(job)); + }); } + const authority = controlAuthorities.get(job.operationId); + if (terminalDurable && authority && options.controlInstallation) + retainControlTerminalStatus( + options.controlInstallation, + authority.execution, + statusFor(job, 2), + ); + if (terminalDurable && claimReleased && authority) + completeControlJournalExecution(authority); activeControllers.delete(job.operationId); active -= 1; } @@ -947,15 +1021,29 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif .catch(() => forceRecoveryRequired(job)); } finally { if (terminalDurable && !claimReleased) { - await claimAdmission?.release().catch(async () => { - stickyRecoveryRequired.add(job.operationId); - await transition(job, "failed", { - result: undefined, - errorClass: "recovery_required", - terminalAt: now().getTime(), - }).catch(() => forceRecoveryRequired(job)); - }); + await claimAdmission + ?.release() + .then(() => { + claimReleased = true; + }) + .catch(async () => { + stickyRecoveryRequired.add(job.operationId); + await transition(job, "failed", { + result: undefined, + errorClass: "recovery_required", + terminalAt: now().getTime(), + }).catch(() => forceRecoveryRequired(job)); + }); } + const authority = controlAuthorities.get(job.operationId); + if (terminalDurable && authority && options.controlInstallation) + retainControlTerminalStatus( + options.controlInstallation, + authority.execution, + statusFor(job, 2), + ); + if (terminalDurable && claimReleased && authority) + completeControlJournalExecution(authority); activeControllers.delete(job.operationId); active -= 1; } @@ -1047,6 +1135,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } } if (changed) await journal.replace([...jobsById.values()].map(persisted)); + if (options.controlInstallation) + await reconcileControlRetention(options.controlInstallation).catch(() => undefined); }); app.post("/v1/operations", async (request, reply) => { @@ -1066,6 +1156,28 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif replay.accept(parsed.requestId, timing.expiresAtMs, nowMs); const policy = OPS_KIND_POLICIES[operation.kind]; const operationFingerprint = opsOperationFingerprint(operation); + if (operation.kind === "controlRetention.acknowledge") { + if (role !== "api" || !options.controlInstallation) + throw new RequestGuardError("authorization", 403); + try { + const existing = jobsByKey.get(`api:${operation.operationKey}`); + if ( + existing && + controlTerminalReceiptDigest(statusFor(existing, 2)) !== operation.terminalDigest + ) + throw Error("Current terminal digest conflict"); + const value = acknowledgeControlRetention( + options.controlInstallation, + operation, + options.tokens.api, + ); + return reply + .code(200) + .send(opsSuccess(parsed.requestId, { execution: "sync", kind: operation.kind, value })); + } catch { + throw new RequestGuardError("recovery_required", 409); + } + } let parent: VerifiedParentExecution | undefined; let controlAdmission: ReturnType | undefined; const parentHeader = request.headers["x-openmapx-parent-execution"]; @@ -1156,6 +1268,13 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif .code(202) .send(opsSuccess(parsed.requestId, admissionFor(existing, parsed.version))); } + if (options.controlInstallation) { + try { + await reconcileControlRetention(options.controlInstallation, { maxFamilies: 1 }); + } catch { + throw new RequestGuardError("recovery_required", 409); + } + } if (jobsById.size >= maxJobEntries) { throw new RequestGuardError("busy", 429); } @@ -1166,6 +1285,24 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } if (options.releaseRecovery?.ready === false && !RELEASE_RECOVERY_READ_KINDS.has(kind)) throw new RequestGuardError("recovery_required", 409); + if (opsKindRequiresControlAuthority(operation.kind) && !controlAdmission) { + if (!options.controlInstallation) throw new RequestGuardError("recovery_required", 409); + try { + const key = parseControlRootOperationKey(parsed.operationKey); + if (key.notAfter !== parsed.execution?.requestedDeadlineAt) throw Error(); + } catch { + throw new RequestGuardError("control_upgrade_required", 409); + } + try { + assertFreshControlRoot( + options.controlInstallation, + parsed.operationKey, + parsed.execution?.requestedDeadlineAt, + ); + } catch { + throw new RequestGuardError("recovery_required", 409); + } + } normalized = normalize(nowMs); if (active >= maxConcurrency) throw new RequestGuardError("busy", 429); active += 1; @@ -1308,6 +1445,31 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); jobsById.set(job.operationId, job); jobsByKey.set(ownerKey, job); + if (!controlAdmission && opsKindRequiresControlAuthority(operation.kind)) { + try { + if (!options.controlInstallation || job.admittedAt === undefined) + throw Error("Control trust unavailable"); + const targets = + "serviceId" in operation + ? [operation.serviceId] + : controlRegisteredServiceIds(options.controlInstallation); + const authority = retainAdmittedControlExecution(options.controlInstallation, { + execution: executionFor(job), + operation: job.operation, + kind: job.operation.kind, + admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, + parent: job.parent, + serviceIds: targets, + }); + requireControlJournalRetention(authority); + controlAuthorities.set(job.operationId, authority); + } catch (error) { + forceRecoveryRequired(job); + await journal.replace([...jobsById.values()].map(persisted)); + throw error; + } + } if (controlAdmission) { const adoption = { leaseId: controlAdmission.leaseId, @@ -1325,6 +1487,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif parent: job.parent, serviceIds: lease.serviceIds, }); + requireControlJournalRetention(authority); controlLeases.set(job.operationId, lease); controlAuthorities.set(job.operationId, authority); job.controlAdoption = { ...adoption, phase: "ready" }; diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index 44c437c46..cc46a8fc4 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -122,82 +122,99 @@ identities without deleting or settling any descendant evidence. ### Authority storage capacity and retirement -The lifetime limits below are an interim fail-closed bound. They are not an -accepted policy for routine administration: renewable retention remains -unimplemented pending the control-identity contract described below. - -`CONTROL_RETENTION_LIMITS` currently fixes these installation lifetime limits: - -- At most 1,024 admitted executions per role recovery root, including fixed - children sharing the agent root. A signed immutable admission reserves capacity - before witness publication or effects. Existing identities and settlement remain - accessible when admission is full. -- At most 1,024 lease histories across the physical installation and 256 active - leases. Exclusive fixed admission slots reserve both before target acquisition; - only authoritative root release removes an active slot. Terminal histories and - partial published histories keep their retained slot. Acquisition starts its - five-second budget before admission work and probes at most 1,024 history slots - plus 256 active slots, without listing or opening historical ledgers. -- Exactly 18 fixed private receipt namespaces, each capped at 65,536 records - (1,024 executions times 64 intents); the execution namespace has the stricter - 1,024 limit. Each private record is at most 128 KiB, so even the conservative - aggregate private record ceiling is finite, below 145 GiB per role, excluding - filesystem overhead. Actual typed key/admission/effect records have tighter - schema limits. Durable private counters reserve creation before publication; - bounded streaming enumeration is needed only to initialize a missing counter. - Failed reservations consume capacity rather than allowing undercount/reuse. -- Fixed-child keys use a direct public-key-digest index plus exact challenge read, - never a historical key-directory scan. Each execution may allocate at most 64 - keys and no more than its admitted plan length. Shared witnesses are bounded by - the admitted execution count: immutable witness, lifecycle and attempted-child - map each at most 64 KiB, plus at most 4 KiB of key identities and a bounded - prepared witness. Lease ledgers remain capped at 128 KiB/64 children. - -There is **no online retirement** of terminal records or key/identity slots. -Terminal records therefore survive at least 72 hours; unresolved records survive -regardless of age, deadline or PID. This deliberately trades lifetime throughput -for exact one-use/replay evidence: exhaustion refuses new work and requires -supervised offline capacity planning/recovery after all relevant writers/helpers -stop. No constructor resets counters, deletes aged evidence or rotates trust to -make room. Automatic bounded retirement with sustained throughput is not supplied -by this foundation. Older lease layouts without the admission index require -supervised indexing; startup never silently excludes their fences. A crashed -metadata guard likewise requires supervised recovery. - -#### Renewable retention and operation identity - -The current protocol accepts opaque `opk1_` keys and permits an unparented root -to omit its requested deadline. On first admission that root receives a deadline -based on the current admission time. A duplicate instead needs its original -admission time; an expired missing attempt must remain unknown and must not be -retried as a new mutation (control protocol section 2). Section 3.2 also requires -producers never to reuse keys. - -After the per-key admission details are reclaimed, the same root operation/key -without a requested deadline is indistinguishable from a first admission. Fresh -normalization gives it a new deadline and fingerprint. A timestamp retirement -floor cannot identify that old key: both newly assigned times exceed the floor. -Keeping every spent key would recreate unbounded history; a bounded spent-key -cache eventually loses this exact recognition. Deadline expiry still rejects -the original unchanged signed child grant, but does not solve this root case. - -The proposed minimal protocol amendment is an issuer-controlled monotonic birth -sequence encoded in control root keys, with that origin certified through the -existing child authority chains. A durable private retirement floor, with bounded -exceptions for unresolved original authorities, can then reject retired origins -without remembering every old key. The fixed child phase-key hash can remain -unchanged if signed origin evidence accompanies derived child keys. Legacy -undated roots need an explicit cutover rule: retained originals remain readable, -and missing old roots cannot be normalized into fresh authority. Ordinary -transport timestamps are not this birth evidence. - -This is a proposed amendment, not an implemented protocol or a renewal guarantee. -The protocol owner must define the root issuance/cutover and signed origin -binding before retirement can reclaim terminal details and keys. Any resulting -collector must still retain terminal evidence for at least 72 hours and through -authority validity, preserve every unresolved dependency, and coordinate signed -cross-role settlement and crash-safe counters/indexes before reuse. No expiry, -PID check or counter reset is settlement proof. +New root authority uses `opk1_t1_<14 lower-case hex expiry>_<22 canonical +base64url nonce>` keys. The nonce contains 16 random bytes; the expiry is an exact +millisecond deadline, explicitly supplied before first submission and no more +than seven days ahead. The browser-safe `@openmapx/core/ops/control-root-key` +entry point formats and parses these keys without Node imports. The runtime may +clamp its execution budget independently. Existing undated admissions remain +available for observation, cancellation and settlement; a missing undated root +returns `control_upgrade_required` before effects. Retrying an opaque intent with +a newly minted timed key is not recovery. + +A version 2 signed admission, witness, lease certificate or grant binds the +original `{version, rootKey, issuer, notAfter}` origin. Descendants, including a +child's independent lease, inherit this origin through verified parent authority. +Raw caller JSON cannot supply it. Each role holds a signed private retirement +clock and high-water record. A zero floor is provisioned only with a verified new +installation's trust bundle. Missing, corrupt or rolled-back state in an existing +installation requires supervised recovery; provisioning never resets its floor. +A backward wall clock blocks new forward work and collection while retained +observation and settlement stay available. + +`CONTROL_RETENTION_LIMITS` bounds retained state and admission work: + +- Each role has 1,024 reusable origin-family slots. The physical installation has + 256 active lease slots and 1,024 retained lease-history slots. Slot identities + include both inode and generation; partial reservations consume capacity. +- An origin permits 128 executions and 256 attempted intents, with at most 64 + children per node and the fixed protocol depth of two. Fixed-child keys have a + direct digest index, not a scan through historical keys. +- Each role-family has at most 4,096 indexed records in 64 pages of 64 entries, + each page at most 16 KiB. Its ordinary index, proof, acknowledgement and progress + metadata together are capped at 2 MiB. The 19 fixed private receipt namespaces + each permit at most 65,536 records of at most 128 KiB. Namespace creation + counters and high-water records reserve capacity before publication. +- Retirement cells are reserved before effects and remain writable outside the + ordinary record quota. A capacity refusal occurs before effects. Reconciliation + examines at most 16 families by default, accepts 1–64, saves a round-robin cursor + past pending families, and reads index pages incrementally. + +A family becomes eligible only after its origin expires, at least 72 hours after +its latest authoritative terminal receipt, and after every attempted execution, +child and independent lease is known contained. Target locks must have been +physically released. Live helpers, journal workers, effect writers and streams +keep it pending. An unaccepted attempted child, unknown effect, corrupt index, +offline participant, missing proof or unknown writer keeps the original evidence +regardless of age or PID. + +The origin's coordinator freezes a signed manifest. The coordinator raises its private durable expiry floor before preparing the +manifest. Each actual participant retains its exact proof copy, then signs ready. All ready acknowledgements and any API terminal-copy acknowledgement +are required before commit. Each role raises its private durable expiry floor before deleting its own private +indexed records and signing clean. Shared semantic evidence is removed only after every +participant is clean and a signed final proof exists. Each participant retains a +private final proof before release; the shared family slot is reusable only after +all required releases. Participants never open sibling-private directories. +Namespace deletion records `planned`, `unlinked`, `counted` and `item_done` in a +preallocated exact-identity transaction. Only an already retained matching plan +can resume an absent unlink. Counter decrements and slot reuse check the original +family, generation, manifest, ordinal and inode. A killed unknown writer is never +stolen automatically. + +### API parent checkpoints and terminal receipt delivery + +Owned administrative jobs use `OpsProjection` version 3. A timeless logical step +address resolves its retained entry before time, body preparation or configuration +publication. A new step chooses its final timed key and requested deadline, then +atomically checkpoints that mapping, full operation, attempt and API parent under +the current database owner/version, active status, cancellation and absence +fences. Both current maps have a 64-entry capacity limit. Same-address contenders +observe the durable winner even at capacity; distinct contenders cannot erase +siblings or exceed the current row's limit. Parent preparation is pure until the +checkpoint succeeds. Unknown checkpoint outcome sends no POST. + +Before consuming any authority-bearing terminal result, failure or cancellation, +the API checkpoints a bounded copy and `receiptDelivery: pending`. It then sends +the API-only, no-effect `controlRetention.acknowledge` operation with a canonical +HMAC proof using the existing API token. The terminal digest covers exact typed +version 2 terminal identity and result/failure fields, excluding changing progress +and diagnostic fields. Acknowledgement has a separate 15-second signal so a +stopped worker cannot strand a copied receipt. Only a confirmed checkpoint allows +return or throw of the terminal outcome. Pending copies keep the parent running +or `cancel_pending` with `receipt_delivery`; both completion SQL and actual expiry +cleanup preserve them. Terminal version 3 parents are retained for at least 72 +hours. After full family collection, an exact valid acknowledgement retry is +accepted without creating records only when no retained/index/semantic evidence +remains and the timed root's expiry is covered by the private floor. + +Current owned-job adapters persist the new root identity. Fixed CLI entry retains +its original phase key and verified inherited origin; it does not issue a new root. +Unowned direct mutation calls through `executeAndWait`, including direct +configuration mutation routes, return the explicit client-upgrade prerequisite +before POST or configuration preparation. These older producers need a durable +owned-job handoff before they can mutate; they do not receive a replacement root +from an opaque intent. Sync reads and retained legacy observations remain +available. The future receiver prerequisites below stay closed. Settlement requires signed retained structured evidence from the fixed executor: a parsed typed result, exact fixed-command completion, or an execution that has diff --git a/packages/cli/__tests__/cli-control-context.test.ts b/packages/cli/__tests__/cli-control-context.test.ts index df5d4563c..210891e13 100644 --- a/packages/cli/__tests__/cli-control-context.test.ts +++ b/packages/cli/__tests__/cli-control-context.test.ts @@ -1,9 +1,13 @@ -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import * as c from "@openmapx/core/build-artifacts"; -import { executionFingerprint, opsOperationFingerprint } from "@openmapx/core/ops"; +import { + createControlRootOperationKey, + executionFingerprint, + opsOperationFingerprint, +} from "@openmapx/core/ops"; import { expect, it, vi } from "vitest"; import { completeCliCommand, withCliControlContext } from "../src/lib/control-context"; @@ -24,7 +28,7 @@ it("settles successful explicit CLI exits once through the real control context" executor: "ops-agent" as const, principal: "api" as const, executionId: `job1_${randomUUID().replaceAll("-", "")}`, - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint, deadlineAt, fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), @@ -52,6 +56,7 @@ it("settles successful explicit CLI exits once through the real control context" recipient: "fixed-cli" as const, }; const authority = c.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, diff --git a/packages/cli/src/lib/control-context.ts b/packages/cli/src/lib/control-context.ts index 5279f0ab6..84780b211 100644 --- a/packages/cli/src/lib/control-context.ts +++ b/packages/cli/src/lib/control-context.ts @@ -72,7 +72,6 @@ export async function withCliControlContext( let parent: ReturnType | undefined; if (delegation) { parent = verifiedParentForArtifactDelegation(installation, assertion, execution); - beginFixedCliExecution(installation, execution); lease = await adoptArtifactTargetLease({ dataRoot: join(infraDir, "data"), assertion, @@ -86,7 +85,6 @@ export async function withCliControlContext( kind: intent.operation.kind, requestedDeadlineAt: execution.deadlineAt, }); - beginFixedCliExecution(installation, execution); } // The lease grant's parent is verified in admission; role fingerprints were checked by the fixed runner. const authority = retainAdmittedControlExecution(installation, { @@ -98,6 +96,7 @@ export async function withCliControlContext( serviceIds: intent.serviceIds, plan: intent.plan, }); + beginFixedCliExecution(installation, execution); if (lease) bindBorrowedControlExecution(lease, authority); if (lease) assertArtifactTargetLeaseForward(lease); try { diff --git a/packages/core/package.json b/packages/core/package.json index 7dd5ed717..31fd9a126 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -197,6 +197,10 @@ "./build-artifacts": { "types": "./src/build-artifacts/index.ts", "default": "./src/build-artifacts/index.ts" + }, + "./ops/control-root-key": { + "types": "./src/ops/control-root-key.ts", + "default": "./src/ops/control-root-key.ts" } }, "scripts": { diff --git a/packages/core/src/build-artifacts/control-authority.test.ts b/packages/core/src/build-artifacts/control-authority.test.ts index d07c38e73..f900f34b5 100644 --- a/packages/core/src/build-artifacts/control-authority.test.ts +++ b/packages/core/src/build-artifacts/control-authority.test.ts @@ -1,3 +1,4 @@ +import { randomBytes } from "node:crypto"; import { chmodSync, existsSync, @@ -11,6 +12,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; import { afterEach, beforeEach, expect, it } from "vitest"; let root: string; @@ -18,6 +20,55 @@ beforeEach(() => { root = mkdtempSync(join(tmpdir(), "omx-control-")); mkdirSync(join(root, "data"), { mode: 0o700 }); }); +it("renewal refuses an expired timed root after all original identity details were reclaimed", async () => { + const c = await api(); + const { randomUUID } = await import("node:crypto"); + const { executionFingerprint } = await import("../ops/execution-identity"); + const { opsOperationFingerprint } = await import("../ops/fingerprint"); + await c.provisionControlInstallation(root); + const installation = c.openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["osrm"], + }); + const operation = { kind: "service.build" as const, serviceId: "osrm" }; + const hash = opsOperationFingerprint(operation); + const expiredAt = Date.now() - 96 * 3600_000; + const operationKey = `opk1_t1_${expiredAt.toString(16).padStart(14, "0")}_AAAAAAAAAAAAAAAAAAAAAA`; + const currentDeadline = Date.now() + 60000; + // The old implementation accepts this exact expired timed key with a freshly + // normalized deadline, even with no historical receipt left to find. + const execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey, + operationFingerprint: hash, + deadlineAt: currentDeadline, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: hash, + deadlineAt: currentDeadline, + }), + }; + try { + expect(() => + c.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt: currentDeadline - 60000, + requestedDeadlineAt: expiredAt, + serviceIds: ["osrm"], + }), + ).toThrow(/expired|expiry|deadline/i); + expect(existsSync(join(root, `control-private/host-cli/executions/${operationKey}.json`))).toBe( + false, + ); + } finally { + c.closeControlInstallation(installation); + } +}); afterEach(() => rmSync(root, { recursive: true, force: true })); async function api() { expect( @@ -104,7 +155,7 @@ it("retains deadline-only attempts separately from execution witnesses", async ( executor: "ops-agent" as const, principal: "api" as const, executionId: `job1_${randomUUID().replaceAll("-", "")}`, - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint, deadlineAt, fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), @@ -134,6 +185,7 @@ it("retains deadline-only attempts separately from execution witnesses", async ( recipient: "data-manager" as const, }; const authority = a.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: parent.deadlineAt, execution: parent, operation, kind: operation.kind, @@ -237,7 +289,7 @@ it("verifies the certified ephemeral fixed-CLI deadline chain without exporting executor: "ops-agent" as const, principal: "api" as const, executionId: `job1_${randomUUID().replaceAll("-", "")}`, - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint: hash, deadlineAt, fingerprint: executionFingerprint({ @@ -293,6 +345,7 @@ it("verifies the certified ephemeral fixed-CLI deadline chain without exporting recipient: "data-manager" as const, }; const authority = c.retainAdmittedControlExecution(agent, { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -336,6 +389,7 @@ it("verifies the certified ephemeral fixed-CLI deadline chain without exporting requestedDeadlineAt: deadlineAt, }); const cliAuthority = c.retainAdmittedControlExecution(fixed, { + requestedDeadlineAt: cli.deadlineAt, execution: cli, kind: operation.kind, operation, @@ -398,7 +452,7 @@ async function deadlineFamily() { executor: "ops-agent" as const, principal: "api" as const, executionId: `job1_${randomUUID().replaceAll("-", "")}`, - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint, deadlineAt, fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), @@ -427,6 +481,7 @@ async function deadlineFamily() { recipient: "data-manager" as const, }; const authority = c.retainAdmittedControlExecution(agent, { + requestedDeadlineAt: execution.deadlineAt, execution, operation, kind: operation.kind, @@ -504,6 +559,7 @@ it("retained deadline-only duplicates observe after cancellation but cannot star expect(f.admit(assertion)).toEqual(parent); const body = { kind: "download-osm", regionId: "europe/germany" }; const authority = f.c.retainAdmittedControlExecution(f.dm, { + requestedDeadlineAt: f.child.deadlineAt, execution: f.child, operation: body, kind: body.kind, @@ -532,10 +588,25 @@ it("refuses exhausted private admission capacity before publishing a witness or try { // A durable consumed-reservation fixture, not a mocked authorization decision. const count = join(root, "control-private/ops-agent/executions/.retention-count"); - writeFileSync(count, String(CONTROL_RETENTION_LIMITS.executionsPerRole), { mode: 0o600 }); - const replacement = { ...f.execution, operationKey: `opk1_${"z".repeat(32)}` }; + const full = { + version: 1, + sequence: 100000, + count: CONTROL_RETENTION_LIMITS.recordsPerNamespace, + }; + writeFileSync(count, JSON.stringify(full), { mode: 0o600 }); + writeFileSync(count.replace(".retention-count", ".retention-highwater"), JSON.stringify(full), { + mode: 0o600, + }); + const replacement = { + ...f.execution, + operationKey: createControlRootOperationKey({ + notAfter: f.execution.deadlineAt, + nonce: randomBytes(16), + }), + }; expect(() => f.c.retainAdmittedControlExecution(f.agent, { + requestedDeadlineAt: replacement.deadlineAt, execution: replacement, operation: { kind: "data.update" }, kind: "data.update", @@ -588,3 +659,88 @@ it("expired exact deadline-only adoption stays observable without renewing one-u f.c.closeControlInstallation(f.agent); } }); + +it("keeps a same-role unaccepted fixed child even if its shared intent ticket disappears", async () => { + const c = await api(), + { vi } = await import("vitest"), + { randomUUID } = await import("node:crypto"), + { executionFingerprint } = await import("../ops/execution-identity"), + { opsOperationFingerprint } = await import("../ops/fingerprint"); + await c.provisionControlInstallation(root); + const installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["osrm"], + }); + const operation = { kind: "data.update" as const }, + operationFingerprint = opsOperationFingerprint(operation), + deadlineAt = Date.now() + 60000; + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const child = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: c.controlChildOperationKey(execution.operationKey, "data-workflow", 0), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + parent: c.retainedControlParent(execution), + }), + }; + const plan = { + phase: "data-workflow" as const, + ordinal: 0, + kind: operation.kind, + execution: child, + serviceIds: ["osrm"], + recipient: "fixed-cli" as const, + }; + try { + const authority = c.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt: deadlineAt - 60000, + requestedDeadlineAt: deadlineAt, + serviceIds: ["osrm"], + plan: [plan], + }); + const id = randomUUID(), + publicKey = c.prepareFixedControlChild(authority, id); + c.retainFixedCliSpawnIntent(authority, { + delegationId: id, + execution: child, + operation, + args: c.fixedCliArguments(operation), + }); + await c.attestParentExecution(authority, { ...plan, childPublicKey: publicKey }); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + const origin = c.executionAuthorityState(authority).origin, + hash = c.controlDigest(origin); + rmSync(join(root, `data/.admin-target-control/retention-families/${hash}/intents/slot-0.json`)); + vi.spyOn(Date, "now").mockReturnValue(deadlineAt + 96 * 3600_000); + expect(await c.reconcileControlRetention(installation, { maxFamilies: 1 })).toEqual({ + retired: 0, + pending: 1, + }); + expect(existsSync(join(root, `control-private/ops-agent/children/${id}.key`))).toBe(true); + } finally { + vi.restoreAllMocks(); + c.closeControlInstallation(installation); + } +}, 60000); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index d24b74494..1670709c8 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1,3 +1,31 @@ +import { + type ControlInstallation, + controlDigest, + controlOriginSchema, + digest, + type InstallationState, + originRecordDomain, + sameControlOrigin, + signControl, + uuid, + validateOriginRecord, + verifyControl, +} from "./control-records"; + +export { + type ControlInstallation, + controlDigest, + controlOriginSchema, + digest, + type InstallationState, + originRecordDomain, + sameControlOrigin, + signControl, + uuid, + validateOriginRecord, + verifyControl, +} from "./control-records"; + import { createHash, createPrivateKey, @@ -5,13 +33,21 @@ import { generateKeyPairSync, type KeyObject, randomUUID, - sign, - verify, } from "node:crypto"; import { existsSync, fsyncSync, mkdirSync, realpathSync } from "node:fs"; import { join, relative } from "node:path"; import { z } from "zod"; -import { type OpsOperationKind, opsOperationSchema, parseBoundedOpsResult } from "../ops/contract"; +import { + type OpsJobStatus, + type OpsOperationKind, + opsOperationSchema, + parseBoundedOpsResult, +} from "../ops/contract"; +import { + controlTerminalReceiptDigest, + verifyApiRetentionProof, +} from "../ops/control-retention-receipt"; +import { type ControlExecutionOrigin, parseControlRootOperationKey } from "../ops/control-root-key"; import { type ControlIssuer, type ExecutionRef, @@ -21,9 +57,27 @@ import { type RetainedParentExecution, } from "../ops/execution-identity"; import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; -import { verifiedParents } from "../ops/verified-parent-internal"; +import { verifiedParentOrigins, verifiedParents } from "../ops/verified-parent-internal"; import { CONTROL_RETENTION_LIMITS, writeRetainedRecord } from "./control-retention"; import { AnchoredDirectory, identity } from "./filesystem"; +import { + assertOriginClockForward, + bootstrapRetentionClock, + readRetentionClock, +} from "./retention-clock"; +import { + acknowledgeRetentionDependency, + completeFamilyJournal, + enlistFamilyIntent, + familyRecordWrite, + hasLocalRetentionFamily, + hasRetentionEvidence, + markFamilyTerminal, + reconcileRetentionFamilies, + registerFamilyExecution, + requireFamilyJournal, + retentionApiDependency, +} from "./retention-family"; export const CONTROL_ROLES = ["ops-agent", "data-manager", "host-cli"] as const; export type ControlRecipient = "fixed-cli" | "data-manager" | "ops-agent" | "configuration-helper"; export type ControlChildPhase = @@ -57,7 +111,6 @@ declare const authorityBrand: unique symbol, peerBrand: unique symbol, settlementBrand: unique symbol, recoveryBrand: unique symbol, - installationBrand: unique symbol, assertionBrand: unique symbol; export type VerifiedExecutionAuthority = Readonly<{ execution: ExecutionRef; @@ -67,20 +120,14 @@ export type VerifiedControlPeer = Readonly<{ [peerBrand]: true }>; export type VerifiedChildSettlement = Readonly<{ [settlementBrand]: true }>; export type VerifiedRecoveryAuthority = Readonly<{ [recoveryBrand]: true }>; export type ArtifactDelegationAssertion = string & { readonly [assertionBrand]: true }; -export type ControlInstallation = Readonly<{ - installationId: string; - role: ControlIssuer; - publicKey: string; - [installationBrand]: true; -}>; export type ChildSettlement = { identity: ExecutionRef; outcome: "contained" | "committed" | "failed_without_effect" | "unknown"; publication: "not_started" | "committed" | "unknown"; terminalReceiptDigest: string; }; -export const uuid = z.uuid(); -export const digest = z.string().regex(/^[a-f0-9]{64}$/); +export type { ControlExecutionOrigin } from "../ops/control-root-key"; + const publicKey = z.string().max(256); const bundleSchema = z.strictObject({ installationId: uuid, @@ -104,43 +151,6 @@ export const targetSchema = z (x) => x.every((v, i) => i === 0 || (x[i - 1] ?? v) < v), "Targets must be sorted and unique", ); -export function controlDigest(value: unknown): string { - return createHash("sha256").update(canonicalJson(value)).digest("hex"); -} -export function signControl( - key: KeyObject, - domain: string, - payload: T, -): { payload: T; signature: string } { - return { - payload, - signature: sign(null, Buffer.from(`${domain}\0${canonicalJson(payload)}`), key).toString( - "base64url", - ), - }; -} -export function verifyControl( - key: string, - domain: string, - envelope: unknown, - schema: z.ZodType, -): T { - const e = z - .strictObject({ payload: schema, signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }) - .parse(envelope); - const k = createPublicKey({ key: Buffer.from(key, "base64url"), format: "der", type: "spki" }); - if ( - k.asymmetricKeyType !== "ed25519" || - !verify( - null, - Buffer.from(`${domain}\0${canonicalJson(e.payload)}`), - k, - Buffer.from(e.signature, "base64url"), - ) - ) - throw Error("Invalid control signature"); - return e.payload; -} export function newControlKey() { const { privateKey, publicKey } = generateKeyPairSync("ed25519"); return { @@ -174,7 +184,17 @@ export async function provisionControlInstallation(infraDir: string): Promise; - privateRoot: AnchoredDirectory; - dataRoot: string; - registeredTargets: ReadonlySet; -} const installations = new WeakMap(); export function installationState(value: ControlInstallation): InstallationState { const state = installations.get(value); @@ -426,6 +452,7 @@ export interface AuthorityState { serviceIds: readonly string[]; plan: readonly ControlPlannedChild[]; receiptDigest: string; + origin?: ControlExecutionOrigin; } const authorities = new WeakMap(); export function executionAuthorityState(value: VerifiedExecutionAuthority) { @@ -448,6 +475,7 @@ export function writePrivateRecord( name: string, value: unknown, exclusive = false, + origin?: ControlExecutionOrigin, ) { const dir = state.privateRoot.directory(namespace, true, true); try { @@ -473,22 +501,228 @@ export function writePrivateRecord( "spawns", "spawn-started", "prepared", + "execution-adoptions", ].includes(namespace) ) throw Error("Unknown fixed authority namespace"); - writeRetainedRecord( - dir, - name, - bytes, - exclusive, - namespace === "executions" - ? CONTROL_RETENTION_LIMITS.executionsPerRole - : CONTROL_RETENTION_LIMITS.recordsPerNamespace, - ); + const write = (beforePublish?: (inode: import("./filesystem").FileIdentity) => void) => + writeRetainedRecord( + dir, + name, + bytes, + exclusive, + CONTROL_RETENTION_LIMITS.recordsPerNamespace, + beforePublish, + ); + if (origin) + familyRecordWrite( + state, + origin, + { scope: "private", namespace, name, container: "" }, + bytes, + write, + exclusive, + ); + else write(); } finally { dir.close(); } } +export function retainedExecutionOrigin( + state: InstallationState, + execution: ExecutionRef, +): ControlExecutionOrigin | undefined { + const raw = readPrivateRecord(state, "executions", `${execution.operationKey}.json`, 65536); + const admitted = verifyControl( + state.installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z + .object({ + version: z.union([z.literal(1), z.literal(2)]), + execution: executionRefSchema, + origin: controlOriginSchema.optional(), + }) + .passthrough(), + ); + if ( + !validateOriginRecord(admitted) || + controlDigest(execution) !== controlDigest(admitted.execution) + ) + throw Error("Execution origin identity mismatch"); + return admitted.origin; +} +export function reconcileControlRetention( + installation: ControlInstallation, + options?: { signal?: AbortSignal; maxFamilies?: number }, +): Promise<{ retired: number; pending: number }> { + return reconcileRetentionFamilies(installationState(installation), options); +} +/** No effect authority: binds the agent's retained API receipt dependency only. */ +export function acknowledgeControlRetention( + installation: ControlInstallation, + input: { + originRootKey: string; + operationKey: string; + fingerprint: string; + terminalDigest: string; + proof: string; + }, + token: string, +): { acknowledged: true } { + const state = installationState(installation); + if (installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Fixed API receipt adapter required"); + if (Buffer.byteLength(canonicalJson(input)) > 16384) + throw Error("Retention acknowledgement exceeds capacity"); + const proof = verifyApiRetentionProof(token, input.proof); + if ( + proof.operationKey !== input.operationKey || + proof.fingerprint !== input.fingerprint || + proof.terminalDigest !== input.terminalDigest + ) + throw Error("Receipt proof identity conflict"); + const parsed = parseControlRootOperationKey(input.originRootKey); + const origin: ControlExecutionOrigin = { + version: 1, + rootKey: input.originRootKey, + issuer: "ops-agent", + notAfter: parsed.notAfter, + }; + if (!hasRetentionEvidence(state, origin)) { + if ( + input.originRootKey !== input.operationKey || + parsed.notAfter > readRetentionClock(state).rejectNotAfterThrough + ) + throw Error("Receipt dependency recovery required"); + // This discharges no surviving dependency; it asserts no operation outcome. + return { acknowledged: true }; + } + const dependency = retentionApiDependency(state, origin); + if ( + !dependency || + dependency.operationKey !== input.operationKey || + dependency.fingerprint !== input.fingerprint || + dependency.parent.apiJobId !== proof.apiJobId || + dependency.parent.apiOwnerVersion !== proof.originalApiOwnerVersion || + dependency.parent.attemptId !== proof.attemptId + ) + throw Error("Receipt parent/attempt conflict"); + acknowledgeRetentionDependency(state, origin, { + operationKey: input.operationKey, + fingerprint: input.fingerprint, + terminalDigest: input.terminalDigest, + proofDigest: controlDigest(proof), + }); + return { acknowledged: true }; +} +export function retainControlTerminalStatus( + installation: ControlInstallation, + execution: ExecutionRef, + status: OpsJobStatus, +): void { + const authority = restoreAdmittedControlExecution(installation, execution), + s = executionAuthorityState(authority); + if ( + !s.origin || + status.operationId !== execution.executionId || + status.operationKey !== execution.operationKey || + status.fingerprint !== execution.fingerprint || + status.deadlineAt !== execution.deadlineAt || + status.kind !== s.kind + ) + throw Error("Terminal control receipt identity conflict"); + const digest = controlTerminalReceiptDigest(status); + const settlement = childSettlementState(verifyControlEffectSettlement(installation, execution)); + markFamilyTerminal( + s.installation, + s.origin, + execution, + settlement.evidence.outcome !== "unknown" && + (status.state === "succeeded" || settlement.evidence.publication !== "committed"), + digest, + ); +} +/** Fixed agent adapter, installed before startup reconciliation; no request carries this capability. */ +export function registerControlRetentionJournal( + installation: ControlInstallation, + journal: NonNullable, +): () => void { + const s = installationState(installation); + if (s.fixedChildId || installation.role !== "ops-agent" || s.retentionJournal) + throw Error("Invalid journal retention adapter"); + s.retentionJournal = journal; + return () => { + if (s.retentionJournal === journal) s.retentionJournal = undefined; + }; +} +export function completeControlJournalExecution(authority: VerifiedExecutionAuthority): void { + const s = executionAuthorityState(authority); + if (!s.origin) throw Error("Missing journal origin"); + completeFamilyJournal(s.installation, s.origin, s.execution); +} +export function requireControlJournalRetention(authority: VerifiedExecutionAuthority): void { + const s = executionAuthorityState(authority); + if (!s.origin) throw Error("Legacy origin is observation-only"); + requireFamilyJournal(s.installation, s.origin, s.execution); +} +export function hasRetainedControlOrigin( + installation: ControlInstallation, + rootKey: string, +): boolean { + try { + const parsed = parseControlRootOperationKey(rootKey); + return hasRetentionEvidence(installationState(installation), { + version: 1, + rootKey, + issuer: installation.role, + notAfter: parsed.notAfter, + }); + } catch { + return true; + } // Legacy, malformed, corrupt or partial evidence is never age-pruned by this adapter. +} +export function assertFreshControlRoot( + installation: ControlInstallation, + key: string, + requestedDeadlineAt: number | undefined, +): void { + const state = installationState(installation), + parsed = parseControlRootOperationKey(key); + if (requestedDeadlineAt !== parsed.notAfter || parsed.notAfter - Date.now() > 7 * 24 * 3600_000) + throw Error( + "Upgrade prerequisite: a persisted timed control root and exact deadline are required", + ); + assertOriginClockForward(state, { + version: 1, + rootKey: key, + issuer: installation.role, + notAfter: parsed.notAfter, + }); +} +export function controlRegisteredServiceIds(installation: ControlInstallation): string[] { + return [...installationState(installation).registeredTargets].sort(); +} +function writeExecutionShared( + state: InstallationState, + execution: ExecutionRef, + dir: AnchoredDirectory, + name: string, + bytes: string, + exclusive = false, + origin = retainedExecutionOrigin(state, execution), +) { + if (origin) + familyRecordWrite( + state, + origin, + { scope: "shared", namespace: "execution", container: execution.executor, name }, + bytes, + (beforePublish) => dir.write(name, bytes, exclusive, beforePublish), + exclusive, + ); + else dir.write(name, bytes, exclusive); +} export function readPrivateRecord( state: InstallationState, namespace: string, @@ -510,6 +744,7 @@ export function retainAdmittedControlExecution( kind: string; operation: unknown; admittedAt: number; + requestedDeadlineAt?: number; parent?: RetainedParentExecution; serviceIds: readonly string[]; plan?: readonly ControlPlannedChild[]; @@ -613,16 +848,75 @@ export function retainAdmittedControlExecution( throw Error("Invalid fixed child plan"); keys.add(child.execution.operationKey); } - const receipt = signControl(state.privateKey, "openmapx-control-execution-v1", { - version: 1, + if (Object.hasOwn(input, "origin")) + throw Error("Caller-supplied origin is not verified authority"); + const existingName = `${execution.operationKey}.json`; + let existing: unknown; + try { + existing = readPrivateRecord(state, "executions", existingName, 65536); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (existing) { + const stored = verifyControl( + installation.publicKey, + originRecordDomain("openmapx-control-execution", existing), + existing, + z + .object({ + version: z.union([z.literal(1), z.literal(2)]), + installationId: uuid, + origin: controlOriginSchema.optional(), + }) + .passthrough(), + ); + if (!validateOriginRecord(stored)) throw Error("Legacy origin cannot be invented"); + const { + version: _version, + installationId: _installationId, + origin: _origin, + ...oldInput + } = stored; + if (controlDigest(oldInput) !== controlDigest({ ...input, execution, serviceIds, plan })) + throw Error("Conflicting retained execution identity"); + return restoreAdmittedControlExecution(installation, execution); + } + let origin: ControlExecutionOrigin; + const inherited = input.parent && verifiedParentOrigins.get(input.parent); + if (inherited) { + if (!input.parent || !verifiedParents.has(input.parent)) + throw Error("Unverified parent origin"); + origin = controlOriginSchema.parse(inherited); + } else { + if (input.parent && (input.parent.issuer !== "api" || !verifiedParents.has(input.parent))) + throw Error("Missing signed inherited origin; legacy parent is observation-only"); + const parsed = parseControlRootOperationKey(execution.operationKey); + const now = Date.now(); + if (parsed.notAfter <= now) throw Error("Expired control root expiry"); + if (parsed.notAfter - now > 7 * 24 * 3600_000 || input.requestedDeadlineAt !== parsed.notAfter) + throw Error("Control root requested deadline must equal its expiry within seven days"); + origin = { + version: 1, + rootKey: execution.operationKey, + issuer: execution.executor, + notAfter: parsed.notAfter, + }; + } + if (execution.deadlineAt > origin.notAfter) + throw Error("Control execution widens origin deadline"); + assertOriginClockForward(state, origin); + const receipt = signControl(state.privateKey, "openmapx-control-execution-v2", { + version: 2, installationId: installation.installationId, ...input, execution, serviceIds, plan, + origin, }); if (Buffer.byteLength(canonicalJson(receipt)) > 65536) throw Error("Execution authority receipt capacity"); + registerFamilyExecution(state, origin, execution, controlDigest(receipt), input.parent); const name = `${execution.operationKey}.json`; let newlyAdmitted = false; try { @@ -631,12 +925,14 @@ export function retainAdmittedControlExecution( throw Error("Conflicting retained execution identity"); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - writePrivateRecord(state, "executions", name, receipt, true); + writePrivateRecord(state, "executions", name, receipt, true, origin); + writePrivateRecord(state, "effects", name, null, true, origin); + writePrivateRecord(state, "effect-started", name, null, true, origin); newlyAdmitted = true; } publishControlExecutionWitness( state, - { execution, kind: input.kind, serviceIds }, + { execution, kind: input.kind, serviceIds, origin }, controlDigest(receipt), ); initializeLifecycle(state, execution, newlyAdmitted); @@ -648,6 +944,7 @@ export function retainAdmittedControlExecution( serviceIds, plan, receiptDigest: controlDigest(receipt), + origin, }); return authority; } @@ -807,7 +1104,13 @@ export function prepareFixedControlChild( } if (ids.length >= Math.min(64, s.plan.length) || ids.includes(delegationId)) throw Error("Fixed child key capacity/replay"); - dir.write(name, canonicalJson([...ids, delegationId])); + writeExecutionShared( + s.installation, + s.execution, + dir, + name, + canonicalJson([...ids, delegationId]), + ); }); const key = newControlKey(); writePrivateRecord( @@ -816,6 +1119,7 @@ export function prepareFixedControlChild( `${delegationId}.key`, { publicKey: key.publicKey, privateKey: encodePrivate(key.privateKey) }, true, + s.origin, ); writePrivateRecord( s.installation, @@ -823,6 +1127,7 @@ export function prepareFixedControlChild( `${controlDigest(key.publicKey)}.json`, { delegationId, publicKey: key.publicKey, execution: s.execution }, true, + s.origin, ); return key.publicKey; } @@ -928,17 +1233,23 @@ export function beginControlEffect(authority: VerifiedExecutionAuthority): void withControlExecutionForward(authority, () => withPrivateEffectWriter(s, () => { try { - readPrivateRecord(s.installation, "effects", `${s.execution.operationKey}.json`); - throw Error("Settled execution cannot restart effects"); + if ( + readPrivateRecord(s.installation, "effects", `${s.execution.operationKey}.json`) !== null + ) + throw Error("Settled execution cannot restart effects"); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } + const name = `${s.execution.operationKey}.json`; + if (readPrivateRecord(s.installation, "effect-started", name) !== null) + throw Error("Execution effects already started"); writePrivateRecord( s.installation, "effect-started", - `${s.execution.operationKey}.json`, + name, { execution: s.execution, receiptDigest: s.receiptDigest }, - true, + false, + s.origin, ); }), ); @@ -975,8 +1286,14 @@ export function retainControlEffectReceipt( publication = "committed"; } else if (receipt.kind === "not-started") { try { - readPrivateRecord(s.installation, "effect-started", `${s.execution.operationKey}.json`); - throw Error("Started effects require containment evidence"); + if ( + readPrivateRecord( + s.installation, + "effect-started", + `${s.execution.operationKey}.json`, + ) !== null + ) + throw Error("Started effects require containment evidence"); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } @@ -1001,20 +1318,41 @@ export function retainControlEffectReceipt( ); const name = `${s.execution.operationKey}.json`; try { + const raw = readPrivateRecord(s.installation, "effects", name, 65536); + if (raw === null) { + writePrivateRecord(s.installation, "effects", name, signed, false, s.origin); + if (s.origin) + markFamilyTerminal( + s.installation, + s.origin, + s.execution, + outcome !== "unknown", + controlDigest(payload), + ); + return; + } const prior = verifyControl( s.installation.installation.publicKey, "openmapx-control-settlement-v1", - readPrivateRecord(s.installation, "effects", name, 65536), + raw, settlementSchema.extend({ receipt: effectObservationSchema }), ); if (controlDigest(prior) === controlDigest(payload)) return; // A later transport/journal failure must never erase proved committed effects. if (prior.outcome !== "unknown") return; - writePrivateRecord(s.installation, "effects", name, signed); + writePrivateRecord(s.installation, "effects", name, signed, false, s.origin); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - writePrivateRecord(s.installation, "effects", name, signed, true); + writePrivateRecord(s.installation, "effects", name, signed, true, s.origin); } + if (s.origin) + markFamilyTerminal( + s.installation, + s.origin, + s.execution, + outcome !== "unknown", + controlDigest(payload), + ); }); } export function verifyControlEffectSettlement( @@ -1097,7 +1435,8 @@ const parentAssertionSchema = z.strictObject({ signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), }); const parentGrantSchema = z.strictObject({ - version: z.literal(1), + version: z.union([z.literal(1), z.literal(2)]), + origin: controlOriginSchema.optional(), installationId: uuid, parent: executionRefSchema, child: executionRefSchema, @@ -1211,7 +1550,9 @@ function readLifecycle( } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } - writePrivateRecord(state, "lifecycle-observations", observationName, current); + const origin = readControlExecutionWitness(state, execution, verifier).origin; + if (!origin || hasLocalRetentionFamily(state, origin)) + writePrivateRecord(state, "lifecycle-observations", observationName, current, false, origin); return current; } function writeLifecycle( @@ -1230,8 +1571,16 @@ function writeLifecycle( `${lifecycle.execution.operationKey}.json`, signed, exclusive, + retainedExecutionOrigin(state, lifecycle.execution), + ); + writeExecutionShared( + state, + lifecycle.execution, + dir, + `${lifecycle.execution.operationKey}.lifecycle.json`, + bytes, + exclusive, ); - dir.write(`${lifecycle.execution.operationKey}.lifecycle.json`, bytes, exclusive); readLifecycle(state, dir, lifecycle.execution, state.installation.publicKey); } function initializeLifecycle( @@ -1307,6 +1656,11 @@ function withParentLifecycles( /** Narrow local runtime fence; exact duplicate admission alone never grants forward permission. */ function withControlExecutionForward(authority: VerifiedExecutionAuthority, run: () => T): T { const s = executionAuthorityState(authority); + if (!s.origin) + throw Error( + "Legacy undated control authority is observation/settlement only; upgrade producer", + ); + assertOriginClockForward(s.installation, s.origin); const check = () => withExecutionWriter(s.installation, s.execution, (dir) => { const lifecycle = readLifecycle( @@ -1405,7 +1759,7 @@ export function cancelControlExecution( return Object.freeze( verifyControl( s.installation.installation.publicKey, - "openmapx-parent-grant-v1", + originRecordDomain("openmapx-parent-grant", saved.grant), saved.grant, parentGrantSchema, ).child, @@ -1474,8 +1828,11 @@ export async function attestParentExecution( if (child.recipient === "fixed-cli") { attemptId = fixedControlChildId(s.installation, child.childPublicKey, s.execution); } + if (!s.origin) throw Error("Legacy parent origin is observation-only"); + assertOriginClockForward(s.installation, s.origin); const grant = parentGrantSchema.parse({ - version: 1, + version: 2, + origin: s.origin, installationId: s.installation.installation.installationId, parent: s.execution, child: child.execution, @@ -1489,7 +1846,7 @@ export async function attestParentExecution( ? { parentAssertion: s.installation.fixedParentAssertion } : {}), }); - const signed = signControl(s.installation.privateKey, "openmapx-parent-grant-v1", grant); + const signed = signControl(s.installation.privateKey, "openmapx-parent-grant-v2", grant); const fields = { version: 1 as const, installationId: grant.installationId, @@ -1528,7 +1885,7 @@ export async function attestParentExecution( for (const value of Object.values(witness)) { const previous = verifyControl( s.installation.installation.publicKey, - "openmapx-parent-grant-v1", + originRecordDomain("openmapx-parent-grant", value), value, parentGrantSchema, ); @@ -1539,17 +1896,26 @@ export async function attestParentExecution( witness[attemptId] = signed; const bytes = canonicalJson(witness); if (Buffer.byteLength(bytes) > 65536) throw Error("Parent witness capacity"); + if (!s.origin) throw Error("Legacy origin is observation-only"); + enlistFamilyIntent( + s.installation, + s.origin, + attemptId, + child.execution, + child.recipient === "data-manager" ? "data-manager" : "ops-agent", + ); writePrivateRecord( s.installation, "parent-attempts", `${attemptId}.json`, { grant: signed, assertion }, true, + s.origin, ); lifecycle.childIntents[attemptId] = fields.grantDigest; lifecycle.sequence++; writeLifecycle(s.installation, dir, lifecycle); - dir.write(name, bytes); + writeExecutionShared(s.installation, s.execution, dir, name, bytes); return assertion; }); const inherited = retainedParentAssertion(s); @@ -1603,9 +1969,16 @@ function readParentGrant( { payload: fields, signature }, parentAssertionSchema.omit({ signature: true }), ); - const grant = verifyControl(verifier, "openmapx-parent-grant-v1", signed, parentGrantSchema); + const grant = verifyControl( + verifier, + originRecordDomain("openmapx-parent-grant", signed), + signed, + parentGrantSchema, + ); const parent = readControlExecutionWitness(state, grant.parent, verifier); if ( + !validateOriginRecord(grant) || + !sameControlOrigin(grant.origin, parent.origin) || grant.parent.executor !== parsed.issuer || grant.parent.operationKey !== parsed.operationKey || grant.attemptId !== parsed.attemptId || @@ -1685,6 +2058,8 @@ export function verifyParentExecution( if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } if (!retained) { + if (!grant.origin) throw Error("Legacy parent is observation-only; upgrade producer"); + assertOriginClockForward(p.installation, grant.origin); if (!forward) throw Error("Canceled/terminal parent permits observation only"); if (Date.now() >= grant.child.deadlineAt) throw Error("Expired parent grant"); writePrivateRecord( @@ -1693,8 +2068,9 @@ export function verifyParentExecution( `${grant.child.operationKey}.json`, { assertion }, true, + grant.origin, ); - writePrivateRecord(p.installation, "parent-adoptions", name, grant, true); + writePrivateRecord(p.installation, "parent-adoptions", name, grant, true, grant.origin); } const binding = z .strictObject({ assertion: z.string() }) @@ -1708,6 +2084,7 @@ export function verifyParentExecution( retainedControlParent(grant.parent), ) as import("../ops/execution-identity").VerifiedParentExecution; verifiedParents.add(parent); + if (grant.origin) verifiedParentOrigins.set(parent, Object.freeze(grant.origin)); return parent; } @@ -1833,6 +2210,7 @@ export function retainFixedCliSpawnIntent( .parse(input.plan ?? []), }, true, + s.origin, ); } export function fixedCliArguments(operation: import("../ops/contract").OpsOperation): string[] { @@ -1925,7 +2303,14 @@ export function beginFixedCliExecution( const s = installationState(installation); if (!s.fixedChildId) throw Error("Fixed CLI authority required"); const begin = () => - writePrivateRecord(s, "spawn-started", `${s.fixedChildId}.json`, { execution }, true); + writePrivateRecord( + s, + "spawn-started", + `${s.fixedChildId}.json`, + { execution }, + true, + retainedExecutionOrigin(s, execution), + ); if (s.fixedParentAssertion) withParentLifecycles(s, s.fixedParentAssertion, (forward) => { if (!forward) throw Error("Canceled/terminal parent prevents fixed CLI start"); @@ -1935,7 +2320,8 @@ export function beginFixedCliExecution( } const executionWitnessSchema = z.strictObject({ - version: z.literal(1), + version: z.union([z.literal(1), z.literal(2)]), + origin: controlOriginSchema.optional(), installationId: uuid, execution: executionRefSchema, kind: z.string().max(128), @@ -1951,11 +2337,15 @@ export function readControlExecutionWitness( try { const witness = verifyControl( publicVerifier, - "openmapx-control-execution-witness-v1", + originRecordDomain( + "openmapx-control-execution-witness", + JSON.parse(d.role.read(`${execution.operationKey}.json`, 65536).toString("utf8")), + ), JSON.parse(d.role.read(`${execution.operationKey}.json`, 65536).toString("utf8")), executionWitnessSchema, ); if ( + !validateOriginRecord(witness) || witness.installationId !== state.installation.installationId || controlDigest(witness.execution) !== controlDigest(execution) ) @@ -1967,13 +2357,19 @@ export function readControlExecutionWitness( } function publishControlExecutionWitness( state: InstallationState, - input: { execution: ExecutionRef; kind: string; serviceIds: readonly string[] }, + input: { + execution: ExecutionRef; + kind: string; + serviceIds: readonly string[]; + origin: ControlExecutionOrigin; + }, authorityDigest: string, ) { const d = executionWitnessDirectory(state, input.execution.executor, true); try { - const witness = signControl(state.privateKey, "openmapx-control-execution-witness-v1", { - version: 1 as const, + const witness = signControl(state.privateKey, "openmapx-control-execution-witness-v2", { + version: 2 as const, + origin: input.origin, installationId: state.installation.installationId, execution: input.execution, kind: input.kind, @@ -1987,7 +2383,15 @@ function publishControlExecutionWitness( throw Error("Replaced execution witness"); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - d.role.write(name, canonicalJson(witness), true); + writeExecutionShared( + state, + input.execution, + d.role, + name, + canonicalJson(witness), + true, + input.origin, + ); } } finally { d.close(); @@ -2000,24 +2404,54 @@ export function restoreAdmittedControlExecution( execution: ExecutionRef, ): VerifiedExecutionAuthority { const state = installationState(installation); + const raw = readPrivateRecord(state, "executions", `${execution.operationKey}.json`, 65536); const payload = verifyControl( installation.publicKey, - "openmapx-control-execution-v1", - readPrivateRecord(state, "executions", `${execution.operationKey}.json`, 65536), + originRecordDomain("openmapx-control-execution", raw), + raw, z - .object({ version: z.literal(1), installationId: uuid, execution: executionRefSchema }) + .object({ + version: z.union([z.literal(1), z.literal(2)]), + installationId: uuid, + execution: executionRefSchema, + kind: z.string(), + serviceIds: targetSchema, + origin: controlOriginSchema.optional(), + plan: z.array(plannedChildSchema).max(64), + }) .passthrough(), ); if ( + !validateOriginRecord(payload) || payload.installationId !== installation.installationId || controlDigest(payload.execution) !== controlDigest(execution) ) throw Error("Recovery execution ownership mismatch"); - const { version: _version, installationId: _installationId, ...input } = payload; - return retainAdmittedControlExecution( - installation, - input as Parameters[1], - ); + const authority = Object.freeze({ execution: payload.execution }) as VerifiedExecutionAuthority; + authorities.set(authority, { + installation: state, + execution: payload.execution, + kind: payload.kind, + serviceIds: payload.serviceIds, + plan: payload.plan, + receiptDigest: controlDigest(raw), + origin: payload.origin, + }); + return authority; +} +/** Fixed local continuation adapter: private signed admission, never a bare request identity. */ +export function verifiedRetainedControlParent( + installation: ControlInstallation, + execution: ExecutionRef, +): import("../ops/execution-identity").VerifiedParentExecution { + const authority = restoreAdmittedControlExecution(installation, execution); + const s = executionAuthorityState(authority); + const parent = Object.freeze( + retainedControlParent(execution), + ) as import("../ops/execution-identity").VerifiedParentExecution; + verifiedParents.add(parent); + if (s.origin) verifiedParentOrigins.set(parent, s.origin); + return parent; } export function retainedControlAdoptionAssertion( installation: ControlInstallation, @@ -2113,10 +2547,18 @@ export function retainControlPreparedTerminal(authority: VerifiedExecutionAuthor `${s.execution.operationKey}.json`, signed, true, + s.origin, ); const d = executionWitnessDirectory(s.installation, "ops-agent", true); try { - d.role.write(`${s.execution.operationKey}.prepared.json`, canonicalJson(signed), true); + writeExecutionShared( + s.installation, + s.execution, + d.role, + `${s.execution.operationKey}.prepared.json`, + canonicalJson(signed), + true, + ); } finally { d.close(); } diff --git a/packages/core/src/build-artifacts/control-records.ts b/packages/core/src/build-artifacts/control-records.ts new file mode 100644 index 000000000..b861d595d --- /dev/null +++ b/packages/core/src/build-artifacts/control-records.ts @@ -0,0 +1,106 @@ +/** Signed record primitives; contains no admission or retirement constructors. */ +import { createHash, createPublicKey, type KeyObject, sign, verify } from "node:crypto"; +import { z } from "zod"; +import { type ControlExecutionOrigin, parseControlRootOperationKey } from "../ops/control-root-key"; +import type { ControlIssuer, ExecutionRef } from "../ops/execution-identity"; +import { canonicalJson } from "../ops/fingerprint"; +import type { AnchoredDirectory } from "./filesystem"; + +declare const installationBrand: unique symbol; +export type ControlInstallation = Readonly<{ + installationId: string; + role: ControlIssuer; + publicKey: string; + [installationBrand]: true; +}>; +export interface InstallationState { + fixedChildId?: string; + fixedParentAssertion?: string; + installation: ControlInstallation; + privateKey: KeyObject; + publicKeys: Record; + privateRoot: AnchoredDirectory; + dataRoot: string; + registeredTargets: ReadonlySet; + retentionJournal?: { + contained(execution: ExecutionRef, digest: string | undefined): boolean; + retire( + executions: readonly { execution: ExecutionRef; digest: string }[], + manifestDigest: string, + ): Promise; + }; +} +export const controlOriginSchema = z + .strictObject({ + version: z.literal(1), + rootKey: z.string().max(69), + issuer: z.enum(["ops-agent", "data-manager", "host-cli"]), + notAfter: z.number().int().positive().safe(), + }) + .refine((o) => { + try { + return parseControlRootOperationKey(o.rootKey).notAfter === o.notAfter; + } catch { + return false; + } + }); +export function sameControlOrigin( + a: ControlExecutionOrigin | undefined, + b: ControlExecutionOrigin | undefined, +): boolean { + return a === undefined || b === undefined + ? a === b + : a.version === b.version && + a.rootKey === b.rootKey && + a.issuer === b.issuer && + a.notAfter === b.notAfter; +} +export function originRecordDomain(base: string, envelope: unknown): string { + const version = (envelope as { payload?: { version?: number } } | null)?.payload?.version; + return `${base}-v${version === 2 ? 2 : 1}`; +} +export function validateOriginRecord(value: { + version?: number; + origin?: ControlExecutionOrigin; +}): boolean { + return value.version === 2 ? !!value.origin : value.origin === undefined; +} +export const uuid = z.uuid(); +export const digest = z.string().regex(/^[a-f0-9]{64}$/); +export function controlDigest(value: unknown): string { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} +export function signControl( + key: KeyObject, + domain: string, + payload: T, +): { payload: T; signature: string } { + return { + payload, + signature: sign(null, Buffer.from(`${domain}\0${canonicalJson(payload)}`), key).toString( + "base64url", + ), + }; +} +export function verifyControl( + key: string, + domain: string, + envelope: unknown, + schema: z.ZodType, +): T { + const e = z + .strictObject({ payload: schema, signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }) + .parse(envelope); + const k = createPublicKey({ key: Buffer.from(key, "base64url"), format: "der", type: "spki" }); + if ( + k.asymmetricKeyType !== "ed25519" || + !verify( + null, + Buffer.from(`${domain}\0${canonicalJson(e.payload)}`), + k, + Buffer.from(e.signature, "base64url"), + ) + ) + throw Error("Invalid control signature"); + return e.payload; +} diff --git a/packages/core/src/build-artifacts/control-retention.test.ts b/packages/core/src/build-artifacts/control-retention.test.ts new file mode 100644 index 000000000..e32b00691 --- /dev/null +++ b/packages/core/src/build-artifacts/control-retention.test.ts @@ -0,0 +1,171 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import * as c from "./control-authority"; +import { advanceRetentionClock } from "./retention-clock"; +import { admittedRoot, terminalWithoutEffect } from "./test-fixtures/authority"; + +let root: string; +const installations: c.ControlInstallation[] = []; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-renewable-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await c.provisionControlInstallation(root); +}); +afterEach(() => { + vi.restoreAllMocks(); + for (const installation of installations.splice(0)) c.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); +}); +function owner() { + const f = admittedRoot(root); + installations.push(f.installation); + return f; +} +it("renews ordinary capacity from an eligible terminal family instead of exhausting lifetime receipts", async () => { + const f = owner(); + terminalWithoutEffect(f.authority); + const state = c.installationState(f.installation); + const count = join(root, "control-private/host-cli/executions/.retention-count"); + const full = { version: 1, sequence: 100000, count: 65536 }; + writeFileSync(count, JSON.stringify(full)); + writeFileSync(count.replace(".retention-count", ".retention-highwater"), JSON.stringify(full)); // Consumed creation reservations at ordinary capacity. + const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 3600_000); + const api = await import("./index"); + // Before implementation, no collector runs and real fresh admission still + // fails the old lifetime counter. Do not use a missing export as the RED. + if ("reconcileControlRetention" in api) { + let retired = 0; + for (let n = 0; n < 32 && !retired; n++) + retired += (await api.reconcileControlRetention(f.installation, { maxFamilies: 1 })).retired; + expect(retired).toBe(1); + } + expect(() => owner()).not.toThrow(); + expect( + existsSync(join(root, `control-private/host-cli/executions/${f.execution.operationKey}.json`)), + ).toBe(false); + expect(state.privateRoot.hasEntries()).toBe(true); + clock.mockRestore(); +}, 60000); +it("rejects old exact roots after durable retirement clock advance and wall-clock rollback across restart", () => { + const f = owner(); + terminalWithoutEffect(f.authority); + const original = Date.now(); + const clock = vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); + advanceRetentionClock(c.installationState(f.installation), f.execution.deadlineAt); + clock.mockReturnValue(original); + const replacement = c.openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["motis", "osrm"], + }); + installations.push(replacement); + expect(() => admittedRoot(root)).toThrow(/clock.*recovery|clock.*backward/i); + const authority = c.restoreAdmittedControlExecution(replacement, f.execution); + expect(() => c.beginControlEffect(authority)).toThrow(/clock|expired|observation/i); + expect(() => c.verifyControlEffectSettlement(replacement, f.execution)).not.toThrow(); +}); +it("does not recreate a missing or rolled-back private clock in a nonempty store", () => { + const f = owner(); + const path = join(root, "control-private/host-cli/retention-clock.json"); + const old = readFileSync(path); + vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); + advanceRetentionClock(c.installationState(f.installation), f.execution.deadlineAt); + writeFileSync(path, old); + expect(() => owner()).toThrow(/clock.*rollback|clock.*recovery/i); + rmSync(path); + expect(() => owner()).toThrow(/clock.*missing|clock.*recovery/i); +}); + +it("does not reset the expiry floor by reprovisioning an emptied private role store", async () => { + const f = owner(); + terminalWithoutEffect(f.authority); + vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); + advanceRetentionClock(c.installationState(f.installation), f.execution.deadlineAt); + c.closeControlInstallation(f.installation); + installations.splice(installations.indexOf(f.installation), 1); + rmSync(join(root, "control-private/host-cli"), { recursive: true }); + await c.provisionControlInstallation(root); + expect(() => owner()).toThrow(/clock.*missing|clock.*recovery/i); + expect(existsSync(join(root, "control-private/host-cli/retention-clock.json"))).toBe(false); +}); + +it("does not retire a young family or a replaced indexed receipt", async () => { + const f = owner(); + terminalWithoutEffect(f.authority); + expect(await c.reconcileControlRetention(f.installation, { maxFamilies: 1 })).toEqual({ + retired: 0, + pending: 1, + }); + const path = join(root, `control-private/host-cli/executions/${f.execution.operationKey}.json`), + bytes = readFileSync(path); + const { renameSync } = await import("node:fs"); + writeFileSync(`${path}.replacement`, bytes, { mode: 0o600 }); + renameSync(`${path}.replacement`, path); + vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 96 * 3600_000); + expect(await c.reconcileControlRetention(f.installation, { maxFamilies: 1 })).toEqual({ + retired: 0, + pending: 1, + }); + expect(existsSync(path)).toBe(true); +}, 60000); + +it("never creates a new deletion plan for an already absent record or decrements a sibling", async () => { + const { AnchoredDirectory } = await import("./filesystem"), + q = await import("./control-retention"), + { randomUUID } = await import("node:crypto"); + const anchor = new AnchoredDirectory(root), + directory = anchor.directory("quota-fixture", true); + try { + q.writeRetainedRecord(directory, "first.json", "first", true); + q.writeRetainedRecord(directory, "second.json", "second", true); + const item = { + familyId: randomUUID(), + generation: randomUUID(), + manifestDigest: "a".repeat(64), + ordinal: 0, + name: "first.json", + inode: directory.fileIdentity("first.json"), + digest: c.controlDigest("first"), + }; + q.deleteRetainedRecord(directory, item, () => {}); + expect(() => q.deleteRetainedRecord(directory, item, () => {})).toThrow(); + expect(JSON.parse(directory.read(".retention-count", 4096).toString()).count).toBe(1); + expect(directory.read("second.json", 4096).toString()).toBe("second"); + } finally { + directory.close(); + anchor.close(); + } +}, 60000); + +it("keeps a family pending when a shared semantic file disappears before its first deletion plan", async () => { + const { AnchoredDirectory } = await import("./filesystem"); + const f = owner(); + terminalWithoutEffect(f.authority); + const witness = join( + root, + `data/.admin-target-control/executions/host-cli/${f.execution.operationKey}.json`, + ); + const original = AnchoredDirectory.prototype.write; + let removed = false; + vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(function ( + this: import("./filesystem").AnchoredDirectory, + ...args + ) { + original.apply(this, args); + if (args[0] !== "decision.json" || removed) return; + const payload = JSON.parse(args[1].toString()).payload; + if (payload?.phase === "final" && payload.semanticClean !== true) { + rmSync(witness); + removed = true; + } + }); + vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 96 * 3600_000); + let retired = 0; + for (let n = 0; n < 4; n++) + retired += (await c.reconcileControlRetention(f.installation, { maxFamilies: 1 })).retired; + expect(removed).toBe(true); + expect(retired).toBe(0); + expect(existsSync(join(root, "control-private/host-cli/retention-families/slot-0"))).toBe(true); +}, 60000); diff --git a/packages/core/src/build-artifacts/control-retention.ts b/packages/core/src/build-artifacts/control-retention.ts index 313f4dcbc..71dda2fdf 100644 --- a/packages/core/src/build-artifacts/control-retention.ts +++ b/packages/core/src/build-artifacts/control-retention.ts @@ -1,57 +1,248 @@ +import { randomUUID } from "node:crypto"; import { fsyncSync, mkdirSync } from "node:fs"; -import { type AnchoredDirectory, identity } from "./filesystem"; +import { controlDigest } from "./control-records"; +import { type AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; -// No online retirement: receipts and consumed identity/key slots remain for the -// installation lifetime, including >=72h terminal history and every unknown. -// Exhaustion closes admission, never settlement of already reserved executions. export const CONTROL_RETENTION_LIMITS = { - executionsPerRole: 1024, + familiesPerRole: 1024, leases: 1024, activeLeases: 256, + executionsPerFamily: 128, + intentsPerFamily: 256, childrenPerExecution: 64, recordsPerNamespace: 65536, + recordsPerFamily: 4096, + indexPages: 64, + pageEntries: 64, + indexPageBytes: 16384, + familyMetadataBytes: 2 * 1024 * 1024, privateRecordBytes: 131072, minimumTerminalRetentionMs: 72 * 60 * 60 * 1000, } as const; - -/** Private, fixed-namespace creation accounting. Count reservations before publishing. */ +interface Counter { + version: 1; + sequence: number; + count: number; +} +export interface RetentionDeletion { + familyId: string; + generation: string; + manifestDigest: string; + ordinal: number; + name: string; + inode: FileIdentity; + digest: string; +} +interface Transaction extends RetentionDeletion { + phase: "planned" | "unlinked" | "counted" | "item_done"; + before: Counter; + after: Counter; +} +function absent(e: unknown) { + return (e as NodeJS.ErrnoException).code === "ENOENT"; +} +function readJson(dir: AnchoredDirectory, name: string, max = 4096): unknown { + return JSON.parse(dir.read(name, max).toString("utf8")); +} +function counter(value: unknown): Counter { + if (!value || typeof value !== "object") throw Error("Invalid retention counter"); + const c = value as Counter; + if ( + c.version !== 1 || + !Number.isSafeInteger(c.sequence) || + c.sequence < 0 || + !Number.isSafeInteger(c.count) || + c.count < 0 + ) + throw Error("Invalid retention counter"); + return c; +} +function writeCounter(dir: AnchoredDirectory, c: Counter) { + counter(c); + dir.write(".retention-count", JSON.stringify(c)); + dir.write(".retention-highwater", JSON.stringify(c)); +} +function readCounter(dir: AnchoredDirectory): Counter { + const current = counter(readJson(dir, ".retention-count")); + const prior = counter(readJson(dir, ".retention-highwater")); + if ( + current.sequence < prior.sequence || + (current.sequence === prior.sequence && current.count !== prior.count) + ) + throw Error("Retention counter rollback; supervised recovery required"); + if (current.sequence > prior.sequence) dir.write(".retention-highwater", JSON.stringify(current)); + return current; +} +function writeTransaction( + dir: AnchoredDirectory, + value: Transaction | null, + exclusive = false, +): void { + const bytes = JSON.stringify(value); + if (Buffer.byteLength(bytes, "utf8") > 4096) + throw Error("Retention transaction exceeds reserved cell"); + dir.write(".retention-transaction", bytes.padEnd(4096), exclusive); +} +function transaction(dir: AnchoredDirectory): Transaction | null { + const value = readJson(dir, ".retention-transaction") as Transaction | null; + if (value === null) return null; + if ( + !value || + !["planned", "unlinked", "counted", "item_done"].includes(value.phase) || + !Number.isSafeInteger(value.ordinal) || + !value.inode || + typeof value.digest !== "string" + ) + throw Error("Corrupt quota transaction"); + counter(value.before); + counter(value.after); + if ( + value.after.sequence !== value.before.sequence + 1 || + value.after.count !== value.before.count - 1 + ) + throw Error("Corrupt quota decrement"); + return value; +} +function initialize(dir: AnchoredDirectory) { + try { + readCounter(dir); + transaction(dir); + } catch (e) { + if (!absent(e)) throw e; + const names = dir.listBounded(CONTROL_RETENTION_LIMITS.recordsPerNamespace + 5); + if (names.some((n) => ![".retention-writer"].includes(n))) + throw Error("Missing namespace accounting in nonempty store; supervised recovery required"); + writeCounter(dir, { version: 1, sequence: 0, count: 0 }); + writeTransaction(dir, null, true); + } +} +export function withRetentionNamespaceWriter(dir: AnchoredDirectory, run: () => T): T { + mkdirSync(dir.name(".retention-writer"), { mode: 0o700 }); + fsyncSync(dir.fd); + const guard = dir.directory(".retention-writer"), + held = identity(guard.stat); + try { + initialize(dir); + return run(); + } finally { + guard.close(); + dir.removeDirectory(".retention-writer", held); + } +} +/** Creation reservations precede ordinary publication; cleanup metadata is outside this quota. */ export function writeRetainedRecord( dir: AnchoredDirectory, name: string, bytes: string, exclusive: boolean, - maximum: number, + maximum: number = CONTROL_RETENTION_LIMITS.recordsPerNamespace, + beforePublish?: (inode: FileIdentity) => void, ) { try { dir.read(name, CONTROL_RETENTION_LIMITS.privateRecordBytes); - dir.write(name, bytes, exclusive); + dir.write(name, bytes, exclusive, beforePublish); return; } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + if (!absent(e)) throw e; } - mkdirSync(dir.name(".retention-writer"), { mode: 0o700 }); - fsyncSync(dir.fd); - const guard = dir.directory(".retention-writer"); - const held = identity(guard.stat); - try { - let count: number; - try { - count = JSON.parse(dir.read(".retention-count", 64).toString("utf8")); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - count = dir.listBounded(maximum + 1).filter((n) => n !== ".retention-writer").length; + withRetentionNamespaceWriter(dir, () => { + if (transaction(dir)) + throw Error("Pending quota transaction; reconcile its exact family before admission"); + const prior = readCounter(dir); + if (prior.count >= maximum) + throw Error("Retained authority capacity exhausted; await terminal retention or containment"); + writeCounter(dir, { version: 1, sequence: prior.sequence + 1, count: prior.count + 1 }); + dir.write(name, bytes, exclusive, beforePublish); + }); +} +export function finishRetainedDeletion(dir: AnchoredDirectory, item: RetentionDeletion): void { + withRetentionNamespaceWriter(dir, () => { + const t = transaction(dir); + if (!t) return; + if ( + t.familyId !== item.familyId || + t.generation !== item.generation || + t.ordinal !== item.ordinal || + t.manifestDigest !== item.manifestDigest || + !["counted", "item_done"].includes(t.phase) + ) + throw Error("Quota item_done identity mismatch"); + writeTransaction(dir, { ...t, phase: "item_done" }); + writeTransaction(dir, null); + }); +} +/** The caller durably records item_done before this transaction can be cleared. */ +export function deleteRetainedRecord( + dir: AnchoredDirectory, + item: RetentionDeletion, + itemDone: () => void, +): void { + withRetentionNamespaceWriter(dir, () => { + let t = transaction(dir); + if ( + t && + (t.familyId !== item.familyId || + t.generation !== item.generation || + t.ordinal !== item.ordinal || + t.manifestDigest !== item.manifestDigest || + t.name !== item.name || + t.digest !== item.digest || + !sameInode(t.inode, item.inode)) + ) + throw Error("Another exact quota transaction remains pending"); + if (!t) { + // An absent item is resumable only through its already retained plan. + // A new plan cannot turn an old continuation into a second decrement. + if ( + !sameInode(dir.fileIdentity(item.name), item.inode) || + controlDigest( + dir.read(item.name, CONTROL_RETENTION_LIMITS.privateRecordBytes).toString("utf8"), + ) !== item.digest + ) + throw Error("Retirement record inode/digest replacement"); + const before = readCounter(dir); + if (before.count < 1) throw Error("Retention counter underflow"); + t = { + ...item, + before, + after: { version: 1, sequence: before.sequence + 1, count: before.count - 1 }, + phase: "planned", + }; + writeTransaction(dir, t); } - if (!Number.isSafeInteger(count) || count < 0 || count >= maximum) - throw Error("Retained authority capacity exhausted; preserve receipts and stop new work"); - dir.write(".retention-count", String(count + 1)); - dir.write(name, bytes, exclusive); - } finally { - guard.close(); - dir.removeDirectory(".retention-writer", held); - } + if (t.phase === "planned") { + if (controlDigest(readCounter(dir)) !== controlDigest(t.before)) + throw Error("Quota planned counter/version mismatch"); + try { + if ( + !sameInode(dir.fileIdentity(item.name), item.inode) || + controlDigest( + dir.read(item.name, CONTROL_RETENTION_LIMITS.privateRecordBytes).toString("utf8"), + ) !== item.digest + ) + throw Error("Retirement record inode/digest replacement"); + dir.removeFile(item.name); + } catch (e) { + if (!absent(e)) + throw e; /* matching planned transaction alone authorizes an already-unlinked item */ + } + t = { ...t, phase: "unlinked" }; + writeTransaction(dir, t); + } + if (t.phase === "unlinked") { + const current = readCounter(dir); + if (controlDigest(current) === controlDigest(t.before)) writeCounter(dir, t.after); + else if (controlDigest(current) !== controlDigest(t.after)) + throw Error("Quota decrement/version mismatch"); + t = { ...t, phase: "counted" }; + writeTransaction(dir, t); + } + if (t.phase !== "counted") throw Error("Unfinished retention transaction phase"); + itemDone(); + writeTransaction(dir, { ...t, phase: "item_done" }); + writeTransaction(dir, null); + }); } - -/** Fixed slots bound admission without listing or parsing historical records. */ export function reserveAuthoritySlot(dir: AnchoredDirectory, maximum: number, until: number) { for (let slot = 0; slot < maximum; slot++) { if (Date.now() >= until) throw Error("Target lease acquisition deadline elapsed"); @@ -65,10 +256,12 @@ export function reserveAuthoritySlot(dir: AnchoredDirectory, maximum: number, un fsyncSync(dir.fd); const child = dir.directory(name); try { - return { name, identity: identity(child.stat) }; + const generation = randomUUID(); + child.write("generation.json", JSON.stringify({ generation }), true); + return { name, identity: identity(child.stat), generation }; } finally { child.close(); } } - throw Error("Retained authority capacity exhausted; no ownership may be stolen"); + throw Error("Retained authority capacity exhausted; await terminal retention or containment"); } diff --git a/packages/core/src/build-artifacts/filesystem.ts b/packages/core/src/build-artifacts/filesystem.ts index 928747079..1567c56fc 100644 --- a/packages/core/src/build-artifacts/filesystem.ts +++ b/packages/core/src/build-artifacts/filesystem.ts @@ -120,6 +120,15 @@ export class AnchoredDirectory { dir.closeSync(); } } + hasEntries(): boolean { + this.check(); + const dir = opendirSync(this.anchor); + try { + return dir.readSync() !== null; + } finally { + dir.closeSync(); + } + } read(name: string, maximum: number, privateMode = true): Buffer { const path = this.name(name); const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); @@ -156,7 +165,30 @@ export class AnchoredDirectory { closeSync(fd); } } - write(name: string, bytes: string | Buffer, exclusive = false) { + fileIdentity(name: string): FileIdentity { + const path = this.name(name); + const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + try { + const value = fstatSync(fd, { bigint: true }); + if ( + !value.isFile() || + value.nlink !== 1n || + value.uid !== BigInt(process.getuid?.() ?? -1) || + (value.mode & 0o077n) !== 0n || + !sameInode(identity(value), identity(lstatSync(path, { bigint: true }))) + ) + throw Error("Unsafe retained record inode"); + return identity(value); + } finally { + closeSync(fd); + } + } + write( + name: string, + bytes: string | Buffer, + exclusive = false, + beforePublish?: (inode: FileIdentity) => void, + ) { const target = this.name(name); const temporary = exclusive ? name : `.write-${randomUUID()}`; const path = this.name(temporary); @@ -166,6 +198,7 @@ export class AnchoredDirectory { try { writeFileSync(fd, bytes); fsyncSync(fd); + beforePublish?.(expected); this.check(); } finally { closeSync(fd); diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 4dbefe0c0..87ae98d6d 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -3,32 +3,42 @@ export * from "./archive"; export { type ArtifactDelegationAssertion, + acknowledgeControlRetention, assertControlExecutionForward, + assertFreshControlRoot, attestParentExecution, beginControlEffect, beginFixedCliExecution, CONTROL_RECOVERY_INSTRUCTIONS, type ControlAdoption, type ControlChildPhase, + type ControlExecutionOrigin, type ControlInstallation, type ControlPlannedChild, type ControlRecipient, cancelControlExecution, closeControlInstallation, + completeControlJournalExecution, controlAdoptionSchema, controlChildOperationKey, controlHeaders, + controlRegisteredServiceIds, fixedCliArguments, + hasRetainedControlOrigin, loadControlStartup, openControlInstallation, openFixedChildInstallation, prepareFixedControlChild, provisionControlInstallation, readFixedCliSpawnIntent, + reconcileControlRetention, + registerControlRetentionJournal, + requireControlJournalRetention, restoreAdmittedControlExecution, retainAdmittedControlExecution, retainControlEffectReceipt, retainControlPreparedTerminal, + retainControlTerminalStatus, retainedControlAdoptionAssertion, retainedControlParent, retainFixedCliSpawnIntent, @@ -36,6 +46,7 @@ export { type VerifiedControlPeer, type VerifiedExecutionAuthority, type VerifiedRecoveryAuthority, + verifiedRetainedControlParent, verifyControlEffectSettlement, verifyControlPeer, verifyControlRecoveryAuthority, diff --git a/packages/core/src/build-artifacts/peer-admission.ts b/packages/core/src/build-artifacts/peer-admission.ts index e9b059e3d..2d1c6fb91 100644 --- a/packages/core/src/build-artifacts/peer-admission.ts +++ b/packages/core/src/build-artifacts/peer-admission.ts @@ -1,7 +1,7 @@ import { opsOperationSchema } from "../ops/contract"; import type { ExecutionRef, VerifiedParentExecution } from "../ops/execution-identity"; import { opsOperationFingerprint } from "../ops/fingerprint"; -import { verifiedParents } from "../ops/verified-parent-internal"; +import { verifiedParentOrigins, verifiedParents } from "../ops/verified-parent-internal"; import { type ControlInstallation, controlDigest, @@ -75,6 +75,7 @@ export function inspectControlAdmission( }); const parent = Object.freeze(retainedControlParent(inspected.parent)) as VerifiedParentExecution; verifiedParents.add(parent); + if (inspected.origin) verifiedParentOrigins.set(parent, inspected.origin); return Object.freeze({ ...inspected, parent, @@ -106,5 +107,6 @@ export function verifiedParentForArtifactDelegation( throw Error("Wrong delegated execution"); const parent = Object.freeze(retainedControlParent(inspected.parent)) as VerifiedParentExecution; verifiedParents.add(parent); + if (inspected.origin) verifiedParentOrigins.set(parent, inspected.origin); return parent; } diff --git a/packages/core/src/build-artifacts/peer-process.test.ts b/packages/core/src/build-artifacts/peer-process.test.ts index e1c5beb74..2e3237ad7 100644 --- a/packages/core/src/build-artifacts/peer-process.test.ts +++ b/packages/core/src/build-artifacts/peer-process.test.ts @@ -1,10 +1,11 @@ import { spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; import { build } from "esbuild"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { type ControlIssuer, type ExecutionRef, @@ -14,6 +15,7 @@ import { opsOperationFingerprint } from "../ops/fingerprint"; import { type ControlChildPhase, type ControlPlannedChild, + closeControlInstallation, controlChildOperationKey, controlDigest, openControlInstallation, @@ -74,7 +76,7 @@ function ref( executionId: role === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), operationKey: parent ? controlChildOperationKey(parent.operationKey, phase ?? "hardlinks", 0) - : `opk1_${randomUUID().replaceAll("-", "")}`, + : createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint, deadlineAt, fingerprint: executionFingerprint({ @@ -98,7 +100,16 @@ async function child(input: unknown) { p.once("error", reject); p.once("exit", (code) => (code === 0 ? resolve() : reject(Error(err)))); }); - return JSON.parse(out) as { ok: boolean; retained?: boolean; error?: string; leaseId?: string }; + return JSON.parse(out) as { + ok: boolean; + retired?: number; + pending?: number; + origin?: { rootKey: string; issuer: string; notAfter: number }; + injected?: boolean; + retained?: boolean; + error?: string; + leaseId?: string; + }; } describe("separate-process physical-mount delegation", () => { it.each(["data-manager", "ops-agent"] as const)( @@ -136,6 +147,7 @@ describe("separate-process physical-mount delegation", () => { recipient: receiver, }; const authority = retainAdmittedControlExecution(installation, { + requestedDeadlineAt: rootRef.deadlineAt, execution: rootRef, kind, operation, @@ -215,6 +227,7 @@ it("keeps owner-death recovery fenced until a separately admitted child settles" recipient: "ops-agent", }; const admitted = { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -291,6 +304,7 @@ it.each([false, true])( serviceIds: ["motis", "osrm"], }); const authority = c.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -378,6 +392,7 @@ it.each([false, true])( await releaseArtifactTargetLease(lease); c.closeControlInstallation(installation); }, + 60000, ); it("keeps an attempted child fenced after SIGKILL of its actual owner process", async () => { const c = await import("./control-authority"); @@ -395,6 +410,7 @@ it("keeps an attempted child fenced after SIGKILL of its actual owner process", recipient: "ops-agent", }; const admitted = { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -489,6 +505,7 @@ it.each([false, true])( recipient: "configuration-helper", }; const admitted = { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -594,6 +611,7 @@ it("keeps paired-pin approval a one-use leaf under its original activation root" serviceIds: ["motis"], }); const authority = c.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -669,6 +687,7 @@ it("settles the logical configuration-fence leaf without releasing the maintenan serviceIds: ["motis"], }); const authority = c.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -739,6 +758,7 @@ it("serializes a paused observer and a newer writer across actual processes", as recipient: "ops-agent", }; const authority = c.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: execution.deadlineAt, execution, operation, kind: operation.kind, @@ -796,3 +816,237 @@ it("serializes a paused observer and a newer writer across actual processes", as l.closeArtifactTargetLeaseHandle(lease); c.closeControlInstallation(installation); }); + +describe("renewable retention with independent processes", () => { + it("inherits origin through a deadline-only independent lease and retains an offline participant while unrelated families renew", async () => { + const c = await import("./control-authority"), + { admittedRoot, terminalWithoutEffect } = await import("./test-fixtures/authority"); + const operation = { kind: "data.update" as const }, + execution = ref("ops-agent", operation.kind, operation), + childOperation = { kind: "download-osm", regionId: "europe/germany" }; + const leaf = ref("data-manager", childOperation.kind, childOperation, execution, "osm"), + installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["osrm"], + }); + const plan = { + phase: "osm" as const, + ordinal: 0, + kind: childOperation.kind, + execution: leaf, + serviceIds: ["osrm"], + recipient: "data-manager" as const, + }; + const authority = c.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt: execution.deadlineAt - 60000, + requestedDeadlineAt: execution.deadlineAt, + serviceIds: ["osrm"], + plan: [plan], + }); + const key = JSON.parse(readFileSync(join(root, "secrets/control-verifiers.json"), "utf8")) + .publicKeys["data-manager"]; + const assertion = await c.attestParentExecution(authority, { ...plan, childPublicKey: key }); + const adopted = await child({ + mode: "deadline-lease", + infraDir: root, + role: "data-manager", + assertion, + execution: leaf, + operation: childOperation, + admittedAt: leaf.deadlineAt - 60000, + }); + expect(adopted, JSON.stringify(adopted)).toMatchObject({ + ok: true, + origin: { + rootKey: execution.operationKey, + issuer: "ops-agent", + notAfter: execution.deadlineAt, + }, + }); + terminalWithoutEffect(authority); + c.closeControlInstallation(installation); + const unrelated = admittedRoot(root, "ops-agent"); + terminalWithoutEffect(unrelated.authority); + c.closeControlInstallation(unrelated.installation); + const now = unrelated.execution.deadlineAt + 96 * 3600_000, + gc = (role: string) => + child({ + mode: "retention", + infraDir: root, + role, + now, + maxFamilies: 1, + assertPrivateRole: true, + }); + expect(await gc("ops-agent")).toMatchObject({ retired: 0, pending: 1 }); + expect(await gc("ops-agent")).toMatchObject({ retired: 1, pending: 0 }); + expect( + readFileSync( + join(root, `control-private/data-manager/executions/${leaf.operationKey}.json`), + "utf8", + ), + ).toContain(leaf.operationKey); + let retired = 0; + for (let n = 0; n < 12 && retired < 2; n++) { + const a = await gc("data-manager"), + b = await gc("ops-agent"); + retired += (a.retired ?? 0) + (b.retired ?? 0); + } + expect(retired).toBe(2); + expect( + (await import("node:fs")).existsSync( + join(root, `control-private/data-manager/executions/${leaf.operationKey}.json`), + ), + ).toBe(false); + expect( + (await import("node:fs")).existsSync( + join(root, `control-private/ops-agent/executions/${execution.operationKey}.json`), + ), + ).toBe(false); + }, 120000); + it.each(["planned", "unlinked", "counted", "item_done"])( + "resumes contained failure after durable %s without double decrement", + async (phase) => { + const { admittedRoot, terminalWithoutEffect } = await import("./test-fixtures/authority"); + const f = admittedRoot(root); + terminalWithoutEffect(f.authority); + closeControlInstallation(f.installation); + const now = f.execution.deadlineAt + 96 * 3600_000; + const first = await child({ mode: "retention", infraDir: root, now, fault: phase }); + expect(first.injected).toBe(true); + expect(first.retired).toBe(0); + const again = await child({ mode: "retention", infraDir: root, now }); + expect( + again.retired, + again.retired === 1 + ? undefined + : readFileSync( + join(root, "control-private/host-cli/retention-families/slot-0/progress.json"), + "utf8", + ).trim(), + ).toBe(1); + const counter = JSON.parse( + readFileSync(join(root, "control-private/host-cli/executions/.retention-count"), "utf8"), + ); + expect(counter.count).toBe(0); + const restarted = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["motis", "osrm"], + }); + expect(() => + retainAdmittedControlExecution(restarted, { + execution: f.execution, + operation: f.operation, + kind: f.operation.kind, + admittedAt: f.execution.deadlineAt - 60000, + requestedDeadlineAt: f.execution.deadlineAt, + serviceIds: ["osrm"], + }), + ).toThrow(/clock|expired/i); + closeControlInstallation(restarted); + }, + 60000, + ); + it("cannot release a reused family slot through its old inode/generation transaction", async () => { + const { admittedRoot, terminalWithoutEffect } = await import("./test-fixtures/authority"), + c = await import("./control-authority"); + const first = admittedRoot(root); + terminalWithoutEffect(first.authority); + c.closeControlInstallation(first.installation); + const now = first.execution.deadlineAt + 96 * 3600_000; + expect( + await child({ mode: "retention", infraDir: root, now, fault: "slot_planned" }), + ).toMatchObject({ retired: 0, injected: true }); + const journal = readFileSync( + join(root, "control-private/host-cli/retention-slot-transaction.json"), + "utf8", + ); + expect(await child({ mode: "retention", infraDir: root, now })).toMatchObject({ retired: 1 }); + const clock = vi.spyOn(Date, "now").mockReturnValue(now), + replacement = admittedRoot(root); + try { + writeFileSync( + join(root, "control-private/host-cli/retention-slot-transaction.json"), + journal, + ); + await expect(c.reconcileControlRetention(replacement.installation)).rejects.toThrow( + /slot.*(inode|generation).*replaced/i, + ); + expect( + readFileSync( + join( + root, + `control-private/host-cli/executions/${replacement.execution.operationKey}.json`, + ), + "utf8", + ), + ).toContain(replacement.execution.operationKey); + expect( + JSON.parse( + readFileSync(join(root, "control-private/host-cli/executions/.retention-count"), "utf8"), + ).count, + ).toBe(1); + } finally { + clock.mockRestore(); + c.closeControlInstallation(replacement.installation); + } + }, 60000); + it("does not steal an unknown killed namespace writer or release its family", async () => { + const { admittedRoot, terminalWithoutEffect } = await import("./test-fixtures/authority"); + const f = admittedRoot(root); + terminalWithoutEffect(f.authority); + closeControlInstallation(f.installation); + const now = f.execution.deadlineAt + 96 * 3600_000, + p = spawn(process.execPath, [fixture], { stdio: ["pipe", "pipe", "pipe"] }); + let output = ""; + p.stdout.on("data", (b) => (output += b)); + p.stdin.end( + `${JSON.stringify({ mode: "retention", infraDir: root, now, fault: "unlinked", kill: true })}\n`, + ); + await vi.waitFor(() => expect(output).toContain('"ready":true'), { timeout: 30000 }); + p.kill("SIGKILL"); + await new Promise((resolve) => p.once("exit", resolve)); + const again = await child({ mode: "retention", infraDir: root, now }); + expect(again).toMatchObject({ retired: 0, pending: 1 }); + expect( + readFileSync( + join(root, "control-private/host-cli/executions/.retention-transaction"), + "utf8", + ), + ).toContain('"phase":"unlinked"'); + }, 60000); + it("advances a bounded round-robin cursor beyond an older unresolved family", async () => { + const { admittedRoot, terminalWithoutEffect } = await import("./test-fixtures/authority"); + const unknown = admittedRoot(root), + done = admittedRoot(root); + terminalWithoutEffect(done.authority); + closeControlInstallation(unknown.installation); + closeControlInstallation(done.installation); + const now = done.execution.deadlineAt + 96 * 3600_000; + expect(await child({ mode: "retention", infraDir: root, now })).toMatchObject({ + retired: 0, + pending: 1, + }); + const next = await child({ mode: "retention", infraDir: root, now }); + expect( + next, + next.retired === 1 + ? undefined + : readFileSync( + join(root, "control-private/host-cli/retention-families/slot-1/progress.json"), + "utf8", + ).trim(), + ).toMatchObject({ retired: 1, pending: 0 }); + expect( + readFileSync( + join(root, `control-private/host-cli/executions/${unknown.execution.operationKey}.json`), + "utf8", + ), + ).toContain(unknown.execution.operationKey); + }, 60000); +}); diff --git a/packages/core/src/build-artifacts/retention-clock.ts b/packages/core/src/build-artifacts/retention-clock.ts new file mode 100644 index 000000000..0be187b6a --- /dev/null +++ b/packages/core/src/build-artifacts/retention-clock.ts @@ -0,0 +1,113 @@ +import type { KeyObject } from "node:crypto"; +import { fsyncSync, mkdirSync } from "node:fs"; +import { z } from "zod"; +import type { ControlExecutionOrigin } from "../ops/control-root-key"; +import { controlDigest, signControl, verifyControl } from "./control-records"; +import { type AnchoredDirectory, identity } from "./filesystem"; + +interface ClockHost { + installation: { installationId: string; role: "ops-agent" | "data-manager" | "host-cli" }; + privateRoot: AnchoredDirectory; + privateKey: KeyObject; + publicKeys: Record<"ops-agent" | "data-manager" | "host-cli", string>; +} +const schema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + role: z.enum(["ops-agent", "data-manager", "host-cli"]), + sequence: z.number().int().nonnegative().safe(), + lastRetirementWallTime: z.number().int().nonnegative().safe(), + rejectNotAfterThrough: z.number().int().nonnegative().safe(), +}); +type Clock = z.infer; +function writer(s: ClockHost, run: () => T): T { + const root = s.privateRoot; + mkdirSync(root.name(".retention-clock-writer"), { mode: 0o700 }); + fsyncSync(root.fd); + const guard = root.directory(".retention-clock-writer"), + held = identity(guard.stat); + try { + return run(); + } finally { + guard.close(); + root.removeDirectory(".retention-clock-writer", held); + } +} +function read(s: ClockHost): Clock { + const parse = (name: string) => + verifyControl( + s.publicKeys[s.installation.role], + "openmapx-retention-clock-v1", + JSON.parse(s.privateRoot.read(name, 2048).toString("utf8")), + schema, + ); + let current: Clock, prior: Clock; + try { + current = parse("retention-clock.json"); + prior = parse("retention-clock-highwater.json"); + } catch { + throw Error("Control retention clock missing/corrupt; supervised clock recovery required"); + } + if ( + current.installationId !== s.installation.installationId || + current.role !== s.installation.role || + current.sequence < prior.sequence || + current.rejectNotAfterThrough < prior.rejectNotAfterThrough || + current.lastRetirementWallTime < prior.lastRetirementWallTime || + (current.sequence === prior.sequence && controlDigest(current) !== controlDigest(prior)) + ) + throw Error("Control retention clock rollback; supervised clock recovery required"); + return current; +} +export function bootstrapRetentionClock(s: ClockHost): void { + // Only verified empty-store provisioning can establish the zero barrier. + if (s.privateRoot.hasEntries()) return; + writer(s, () => { + const payload: Clock = { + version: 1, + installationId: s.installation.installationId, + role: s.installation.role, + sequence: 0, + lastRetirementWallTime: 0, + rejectNotAfterThrough: 0, + }; + const bytes = JSON.stringify(signControl(s.privateKey, "openmapx-retention-clock-v1", payload)); + s.privateRoot.write("retention-clock.json", bytes.padEnd(2048), true); + s.privateRoot.write("retention-clock-highwater.json", bytes.padEnd(2048), true); + s.privateRoot.write("retention-cursor.json", "0".padEnd(64), true); + s.privateRoot.write("retention-slot-transaction.json", "null".padEnd(4096), true); + }); +} +export function assertOriginClockForward(s: ClockHost, origin: ControlExecutionOrigin): void { + writer(s, () => { + const clock = read(s), + now = Date.now(); + if (now < clock.lastRetirementWallTime) + throw Error("Wall clock moved backward; control clock recovery prerequisite"); + if (origin.notAfter <= clock.rejectNotAfterThrough || now >= origin.notAfter) + throw Error("Expired control origin; observation/settlement only"); + }); +} +export function advanceRetentionClock(s: ClockHost, notAfter: number): Clock { + return writer(s, () => { + const prior = read(s), + now = Date.now(); + if (now < prior.lastRetirementWallTime || now < notAfter) + throw Error("Control retirement clock recovery prerequisite"); + const current: Clock = { + ...prior, + sequence: prior.sequence + 1, + lastRetirementWallTime: Math.max(now, prior.lastRetirementWallTime), + rejectNotAfterThrough: Math.max(notAfter, prior.rejectNotAfterThrough), + }; + schema.parse(current); + const bytes = JSON.stringify(signControl(s.privateKey, "openmapx-retention-clock-v1", current)); + // High-water first: partial publication is a safe clock prerequisite. + s.privateRoot.write("retention-clock-highwater.json", bytes.padEnd(2048)); + s.privateRoot.write("retention-clock.json", bytes.padEnd(2048)); + return current; + }); +} +export function readRetentionClock(s: ClockHost): Clock { + return writer(s, () => read(s)); +} diff --git a/packages/core/src/build-artifacts/retention-family.ts b/packages/core/src/build-artifacts/retention-family.ts new file mode 100644 index 000000000..d21443d0f --- /dev/null +++ b/packages/core/src/build-artifacts/retention-family.ts @@ -0,0 +1,2352 @@ +/** Fixed, role-local receipt ownership and preallocated per-origin retirement cells. */ +import { createHash, randomUUID } from "node:crypto"; +import { fsyncSync, mkdirSync } from "node:fs"; +import { z } from "zod"; +import type { ControlExecutionOrigin } from "../ops/control-root-key"; +import type { + ControlIssuer, + ExecutionRef, + RetainedParentExecution, +} from "../ops/execution-identity"; +import { canonicalJson } from "../ops/fingerprint"; +import { + controlDigest, + controlOriginSchema, + type InstallationState, + signControl, + verifyControl, +} from "./control-records"; +import { + deleteRetainedRecord, + finishRetainedDeletion, + CONTROL_RETENTION_LIMITS as LIMITS, +} from "./control-retention"; +import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; +import { advanceRetentionClock, readRetentionClock } from "./retention-clock"; + +const ROLES = ["ops-agent", "data-manager", "host-cli"] as const; +const PRIVATE = new Set([ + "executions", + "children", + "child-keys", + "effects", + "effect-started", + "observations", + "leases", + "execution-leases", + "attempts", + "adoptions", + "parent-attempts", + "parent-adoptions", + "execution-parents", + "execution-lifecycle", + "lifecycle-observations", + "spawns", + "spawn-started", + "prepared", + "execution-adoptions", +]); +const SHARED = new Set([ + "execution", + "lease-owner", + "lease-grants", + "lease-children", + "lease-directory", + "history-slot", +]); +const originSchema = controlOriginSchema; +const identitySchema = z.strictObject({ + dev: z.string().regex(/^\d+$/), + ino: z.string().regex(/^\d+$/), +}); +const sharedIdentitySchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + origin: originSchema, + familyId: z.uuid(), + generation: z.uuid(), + dataRootIdentity: identitySchema, + sharedIdentity: identitySchema, +}); +type SharedIdentity = z.infer; +type Phase = "active" | "prepared" | "ready" | "committed" | "clean" | "final" | "released"; +export interface FamilyRecord { + scope: "private" | "shared"; + namespace: string; + name: string; + container: string; + digest: string; + inode: FileIdentity; + ordinal: number; + state: "reserved" | "prepared" | "written"; + counted: boolean; + directory: boolean; + generation?: string; +} +interface ExecutionFact { + execution: ExecutionRef; + verifier: string; + receiptDigest: string; + terminalDigest?: string; + terminalAt?: number; + known: boolean; + fixedChildId?: string; + requiresJournal?: boolean; + journalContained?: boolean; +} +interface IntentFact { + id: string; + childDigest: string; + recipient: ControlIssuer; +} +interface LeaseFact { + id: string; + certificateDigest: string; + ownerRole: ControlIssuer; + released: boolean; +} +interface ApiDependency { + parent: Extract; + operationKey: string; + fingerprint: string; +} +interface Progress { + version: 1; + familyId: string; + generation: string; + sequence: number; + phase: Phase; + records: number; + sharedRecords: number; + metadataBytes: number; + executions: Record; + intents: Record; + leases: Record; + participants: ControlIssuer[]; + terminalAt: number; + api?: ApiDependency; + apiAck?: { terminalDigest: string; proofDigest: string }; + cursor: number; + sharedCursor: number; + manifestDigest?: string; + lastDone?: { + namespace: string; + ordinal: number; + name: string; + digest: string; + inode: FileIdentity; + }; + sharedTransaction?: { ordinal: number; phase: "planned" | "unlinked"; record: FamilyRecord }; + sharedRelease?: boolean; + journalCleanupPlanned?: boolean; + journalClean?: boolean; + lastError?: string; +} +interface Candidate { + version: 1; + installationId: string; + familyId: string; + generation: string; + role: ControlIssuer; + originDigest: string; + terminalAt: number; + terminalSequenceDigest: string; + executionReceiptDigests: Record; + localRecordSetDigest: string; + sharedRecordSetDigest: string; + leaseDigests: Record; + intentDigests: Record; +} +const apiCopySchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + familyId: z.uuid(), + generation: z.uuid(), + originDigest: z.string().regex(/^[a-f0-9]{64}$/), + parentDigest: z.string().regex(/^[a-f0-9]{64}$/), + operationKey: z.string(), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + terminalDigest: z.string().regex(/^[a-f0-9]{64}$/), + proofDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +function apiCopy(f: Family, p: Progress) { + const raw = json(f.shared, "api-ack.json", 4096); + if (raw === null) return null; + const value = verifyControl( + f.state.publicKeys[f.meta.origin.issuer], + "openmapx-retention-api-copy-v1", + raw, + apiCopySchema, + ); + sameFamily(f, value); + if ( + value.installationId !== f.meta.installationId || + !p.api || + value.parentDigest !== controlDigest(p.api.parent) || + value.operationKey !== p.api.operationKey || + value.fingerprint !== p.api.fingerprint + ) + throw Error("Invalid API retirement acknowledgement"); + return value; +} +interface Manifest { + version: 1; + installationId: string; + origin: ControlExecutionOrigin; + familyId: string; + generation: string; + sequence: number; + terminalAt: number; + terminalSequenceDigest: string; + participants: string[]; + executionReceiptDigests: Record; + sharedRecordSetDigest: string; + localRecordSetDigests: Record; +} +interface Decision { + version: 1; + familyId: string; + generation: string; + phase: "active" | "prepared" | "committed" | "final"; + manifestDigest?: string; + readyDigests?: Record; + cleanDigests?: Record; + sequence: number; +} +interface Ack { + version: 1; + installationId: string; + familyId: string; + generation: string; + originDigest: string; + manifestDigest: string; + role: ControlIssuer; + phase: "ready" | "clean" | "released"; + terminalSequenceDigest: string; + recordSetDigest: string; + proofCopyDigest: string; + sequence: number; +} +interface Family { + state: InstallationState; + root: AnchoredDirectory; + slots: AnchoredDirectory; + slot: AnchoredDirectory; + sharedRoot: AnchoredDirectory; + shared: AnchoredDirectory; + meta: SharedIdentity; + name: string; + slotIdentity: FileIdentity; + slotGeneration: string; + originHash: string; + sharedGone?: boolean; + close(): void; +} +const DIGEST = /^[a-f0-9]{64}$/; +const safeName = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,159}$/; +const slotName = /^slot-\d{1,4}$/; +function missing(e: unknown): boolean { + return (e as NodeJS.ErrnoException).code === "ENOENT"; +} +function cell( + dir: AnchoredDirectory, + name: string, + value: unknown, + maximum: number, + exclusive = false, +) { + const bytes = canonicalJson(value); + if (Buffer.byteLength(bytes) > maximum) + throw Error("Reserved retention workspace capacity exceeded"); + dir.write(name, bytes + " ".repeat(maximum - Buffer.byteLength(bytes)), exclusive); +} +function json(dir: AnchoredDirectory, name: string, max = 131072): T { + return JSON.parse(dir.read(name, max).toString("utf8")) as T; +} +function signed(state: InstallationState, domain: string, value: T) { + return signControl(state.privateKey, domain, value); +} +function verified( + state: InstallationState, + role: ControlIssuer, + domain: string, + raw: unknown, +): T { + return verifyControl(state.publicKeys[role], domain, raw, z.unknown()) as T; +} +function originHash(origin: ControlExecutionOrigin) { + return controlDigest(origin); +} +function tuple(e: FamilyRecord): unknown[] { + return [ + e.scope === "private" ? "p" : "s", + e.namespace, + e.name, + e.container, + e.digest, + e.inode.dev, + e.inode.ino, + e.ordinal, + e.state, + e.counted ? 1 : 0, + e.directory ? 1 : 0, + e.generation ?? "", + ]; +} +function entry(v: unknown): FamilyRecord { + if ( + !Array.isArray(v) || + v.length !== 12 || + !["p", "s"].includes(v[0]) || + !safeName.test(v[2]) || + !DIGEST.test(v[4]) || + !Number.isSafeInteger(v[7]) || + !["reserved", "prepared", "written"].includes(v[8]) + ) + throw Error("Corrupt retention index entry"); + const e: FamilyRecord = { + scope: v[0] === "p" ? "private" : "shared", + namespace: v[1], + name: v[2], + container: v[3], + digest: v[4], + inode: identitySchema.parse({ dev: v[5], ino: v[6] }), + ordinal: v[7], + state: v[8], + counted: v[9] === 1, + directory: v[10] === 1, + ...(v[11] ? { generation: z.uuid().parse(v[11]) } : {}), + }; + if (!(e.scope === "private" ? PRIVATE : SHARED).has(e.namespace)) + throw Error("Unknown fixed retention namespace"); + return e; +} +function readProgress(f: Family): Progress { + const p = json(f.slot, "progress.json"); + if ( + p.version !== 1 || + p.familyId !== f.meta.familyId || + p.generation !== f.meta.generation || + !Number.isSafeInteger(p.sequence) || + p.records > LIMITS.recordsPerFamily || + !Number.isSafeInteger(p.sharedRecords) || + p.sharedRecords < 0 || + p.sharedRecords > 2048 || + !Number.isSafeInteger(p.metadataBytes) || + p.metadataBytes > LIMITS.familyMetadataBytes || + Object.keys(p.executions).length > LIMITS.executionsPerFamily || + Object.keys(p.intents).length > LIMITS.intentsPerFamily || + p.participants.some((r) => !ROLES.includes(r)) + ) + throw Error("Retention family progress identity/capacity mismatch"); + const prior = json<{ sequence: number; digest: string }>(f.slot, "progress-highwater.json", 1024); + if ( + p.sequence < prior.sequence || + (p.sequence === prior.sequence && controlDigest(p) !== prior.digest) + ) + throw Error("Retention family progress rollback"); + return p; +} +function progress(f: Family, p: Progress, reserveAdmission = false) { + if (json(f.slot, "progress.json").sequence !== p.sequence) + throw Error("Family progress compare-and-swap conflict"); + if (p.sequence >= Number.MAX_SAFE_INTEGER) throw Error("Family sequence exhausted"); + // Reserve space for retirement transactions and post-effect timestamps. + // Ordinary admission may never consume the collector's last 8KiB. + if (reserveAdmission && p.phase === "active" && Buffer.byteLength(canonicalJson(p)) > 120 * 1024) + throw Error("Family progress capacity exhausted before exposure"); + p.sequence++; + cell(f.slot, "progress.json", p, 131072); + cell(f.slot, "progress-highwater.json", { sequence: p.sequence, digest: controlDigest(p) }, 1024); +} +function guard(dir: AnchoredDirectory, name: string, run: () => T): T { + mkdirSync(dir.name(name), { mode: 0o700 }); + fsyncSync(dir.fd); + const held = dir.directory(name), + expected = identity(held.stat); + try { + return run(); + } finally { + held.close(); + dir.removeDirectory(name, expected); + } +} +function decision(f: Family): Decision { + const value = verified( + f.state, + f.meta.origin.issuer, + "openmapx-retention-decision-v1", + json(f.shared, "decision.json", 8192), + ); + if ( + value.version !== 1 || + value.familyId !== f.meta.familyId || + value.generation !== f.meta.generation || + !["active", "prepared", "committed", "final"].includes(value.phase) + ) + throw Error("Retention decision identity mismatch"); + return value; +} +function writeDecision(f: Family, value: Decision) { + if (f.state.fixedChildId || f.state.installation.role !== f.meta.origin.issuer) + throw Error("Only the origin coordinator may publish a retirement decision"); + cell(f.shared, "decision.json", signed(f.state, "openmapx-retention-decision-v1", value), 8192); +} +function checkLocalIdentity(f: Family) { + f.slot.check(); + f.slots.check(); + if (!sameInode(identity(f.slot.stat), f.slotIdentity)) throw Error("Replaced family slot inode"); + const local = json<{ familyId: string; generation: string; slotGeneration: string }>( + f.slot, + "identity.json", + 2048, + ); + if ( + local.familyId !== f.meta.familyId || + local.generation !== f.meta.generation || + local.slotGeneration !== f.slotGeneration + ) + throw Error("Reused family slot generation"); +} +function openFamily(state: InstallationState, origin: ControlExecutionOrigin): Family { + const hash = originHash(origin), + opened: AnchoredDirectory[] = []; + const own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + const close = () => { + for (const directory of [...opened].reverse()) directory.close(); + }; + try { + const locations = own(state.privateRoot.directory("retention-locations")); + const local = json<{ + name: string; + inode: FileIdentity; + slotGeneration: string; + identity: unknown; + }>(locations, `${hash}.json`, 4096); + if (!slotName.test(local.name)) throw Error("Invalid family slot name"); + const meta = verifyControl( + state.publicKeys[origin.issuer], + "openmapx-retention-family-v1", + local.identity, + sharedIdentitySchema, + ); + if ( + controlDigest(meta.origin) !== hash || + meta.installationId !== state.installation.installationId + ) + throw Error("Foreign retained origin"); + const slots = own(state.privateRoot.directory("retention-families")), + slot = own(slots.directory(local.name)), + root = own(new AnchoredDirectory(state.dataRoot)), + control = own(root.directory(".admin-target-control")), + sharedRoot = own(control.directory("retention-families")); + if (!sameInode(identity(root.stat), meta.dataRootIdentity)) + throw Error("Family physical data root replaced"); + let shared: AnchoredDirectory, + sharedGone = false; + try { + shared = own(sharedRoot.directory(hash)); + } catch (error) { + const p = json(slot, "progress.json"); + if (!missing(error) || p.phase !== "released") throw error; + const final = verified( + state, + origin.issuer, + "openmapx-retention-decision-v1", + json(slot, "final.json", 8192), + ); + if ( + final.phase !== "final" || + !final.semanticClean || + final.familyId !== meta.familyId || + final.generation !== meta.generation + ) + throw Error("Missing private final retirement proof"); + sharedGone = true; + shared = own(state.privateRoot.directory("retention-families")); + } + const f: Family = { + state, + root, + slots, + slot, + sharedRoot, + shared, + meta, + name: local.name, + slotIdentity: local.inode, + slotGeneration: local.slotGeneration, + originHash: hash, + sharedGone, + close, + }; + checkLocalIdentity(f); + if (!sharedGone && !sameInode(identity(shared.stat), meta.sharedIdentity)) + throw Error("Shared family slot inode replaced"); + return f; + } catch (error) { + close(); + throw error; + } +} +export function hasLocalRetentionFamily( + state: InstallationState, + origin: ControlExecutionOrigin, +): boolean { + try { + const f = openFamily(state, origin); + f.close(); + return true; + } catch (e) { + if (missing(e)) return false; + throw e; + } +} +function preallocateSharedRole(shared: AnchoredDirectory, role: ControlIssuer) { + for (const [name, size] of [ + [`ack-${role}.json`, 4096], + [`candidate-${role}.json`, 65536], + ] as const) + try { + cell(shared, name, null, size, true); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + } + for (let n = 0; n < 32; n++) + try { + cell(shared, `shared-${role}-${n}.json`, [], 16384, true); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + } +} +/** Called by fixed admission/enlistment before any family record or assertion is exposed. */ +export function reserveControlFamily( + state: InstallationState, + origin: ControlExecutionOrigin, +): void { + try { + const f = openFamily(state, origin); + f.close(); + return; + } catch (e) { + if (!missing(e)) throw e; + } + if (state.fixedChildId) throw Error("A fixed child requires its role's already reserved family"); + const root = new AnchoredDirectory(state.dataRoot), + control = root.directory(".admin-target-control", true, true), + sharedRoot = control.directory("retention-families", true, true); + const slots = state.privateRoot.directory("retention-families", true, true), + locations = state.privateRoot.directory("retention-locations", true, true); + const hash = originHash(origin); + let shared: AnchoredDirectory | undefined, slot: AnchoredDirectory | undefined; + try { + if (json(state.privateRoot, "retention-slot-transaction.json", 4096) !== null) + throw Error("Family slot release pending; reconcile before admission"); + let name: string | undefined; + for (let n = 0; n < LIMITS.familiesPerRole; n++) { + const candidate = `slot-${n}`; + try { + mkdirSync(slots.name(candidate), { mode: 0o700 }); + name = candidate; + fsyncSync(slots.fd); + break; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + } + } + if (!name) throw Error("Retained family capacity exhausted; await retention/containment"); + slot = slots.directory(name); + try { + shared = sharedRoot.directory(hash); + } catch (e) { + if (!missing(e) || origin.issuer !== state.installation.role) throw e; + shared = sharedRoot.directory(hash, true, true); + const meta: SharedIdentity = { + version: 1, + installationId: state.installation.installationId, + origin, + familyId: randomUUID(), + generation: randomUUID(), + dataRootIdentity: identity(root.stat), + sharedIdentity: identity(shared.stat), + }; + cell( + shared, + "identity.json", + signed(state, "openmapx-retention-family-v1", meta), + 2048, + true, + ); + cell( + shared, + "decision.json", + signed(state, "openmapx-retention-decision-v1", { + version: 1, + familyId: meta.familyId, + generation: meta.generation, + phase: "active", + sequence: 0, + }), + 8192, + true, + ); + cell(shared, "manifest.json", null, 32768, true); + cell(shared, "api-ack.json", null, 4096, true); + shared.directory("executions", true).close(); + shared.directory("intents", true).close(); + shared.directory("leases", true).close(); + } + const rawIdentity = json(shared, "identity.json", 2048); + const meta = verifyControl( + state.publicKeys[origin.issuer], + "openmapx-retention-family-v1", + rawIdentity, + sharedIdentitySchema, + ); + if ( + controlDigest(meta.origin) !== hash || + !sameInode(meta.dataRootIdentity, identity(root.stat)) || + !sameInode(meta.sharedIdentity, identity(shared.stat)) + ) + throw Error("Origin physical identity mismatch"); + preallocateSharedRole(shared, state.installation.role); + const slotGeneration = randomUUID(); + cell( + slot, + "identity.json", + { familyId: meta.familyId, generation: meta.generation, slotGeneration, originHash: hash }, + 2048, + true, + ); + for (let n = 0; n < LIMITS.indexPages; n++) + cell(slot, `page-${n}.json`, [], LIMITS.indexPageBytes, true); + for (const [file, size] of [ + ["manifest.json", 32768], + ["commit.json", 8192], + ["final.json", 8192], + ["proof.json", 65536], + ] as const) + cell(slot, file, null, size, true); + const p: Progress = { + version: 1, + familyId: meta.familyId, + generation: meta.generation, + sequence: 0, + phase: "active", + records: 0, + sharedRecords: 0, + metadataBytes: 1942528, + executions: {}, + intents: {}, + leases: {}, + participants: [origin.issuer, state.installation.role].filter( + (r, i, a) => a.indexOf(r) === i, + ), + terminalAt: 0, + cursor: 0, + sharedCursor: 0, + }; + cell(slot, "progress.json", p, 131072, true); + cell(slot, "progress-highwater.json", { sequence: 0, digest: controlDigest(p) }, 1024, true); + locations.write( + `${hash}.json`, + canonicalJson({ name, inode: identity(slot.stat), slotGeneration, identity: rawIdentity }), + true, + ); + } finally { + slot?.close(); + locations.close(); + slots.close(); + shared?.close(); + sharedRoot.close(); + control.close(); + root.close(); + } +} +function* records(f: Family): Generator { + const p = readProgress(f); + let ordinal = 0; + for (let n = 0; n < Math.ceil(p.records / LIMITS.pageEntries); n++) { + const page = json(f.slot, `page-${n}.json`, LIMITS.indexPageBytes); + if (!Array.isArray(page) || page.length > 64) throw Error("Invalid retained index page"); + for (const value of page) { + const e = entry(value); + if (e.ordinal !== ordinal++) throw Error("Retention index ordinal mismatch"); + yield e; + } + } + if (ordinal !== p.records) throw Error("Retention index count mismatch"); +} +function recordSetDigest(values: Iterable): string { + const h = createHash("sha256"); + h.update("["); + let first = true; + for (const e of values) { + if (!first) h.update(","); + first = false; + h.update(canonicalJson(tuple(e))); + } + h.update("]"); + return h.digest("hex"); +} +function findRecord( + f: Family, + address: Pick, +): FamilyRecord | undefined { + for (const e of records(f)) if (equalAddress(e, address)) return e; + return undefined; +} +function saveEntry(f: Family, e: FamilyRecord) { + const n = Math.floor(e.ordinal / LIMITS.pageEntries), + page = json(f.slot, `page-${n}.json`, LIMITS.indexPageBytes); + page[e.ordinal % LIMITS.pageEntries] = tuple(e); + cell(f.slot, `page-${n}.json`, page, LIMITS.indexPageBytes); +} +function equalAddress( + a: Pick, + b: Pick, +) { + return ( + a.scope === b.scope && + a.namespace === b.namespace && + a.name === b.name && + a.container === b.container + ); +} +export function familyRecordWrite( + state: InstallationState, + origin: ControlExecutionOrigin, + address: Pick, + bytes: string, + write: (beforePublish: (inode: FileIdentity) => void) => void, + exclusive = false, +): void { + reserveControlFamily(state, origin); + const f = openFamily(state, origin); + try { + let ref: FamilyRecord; + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + old = findRecord(f, address); + if (exclusive && old?.state === "written") + throw Object.assign(Error("Exclusive retained record already exists"), { code: "EEXIST" }); + const nextDigest = controlDigest(bytes); + if (old?.state === "written" && old.digest === nextDigest) { + const d = recordDirectory(f, old); + try { + if ( + sameInode(d.dir.fileIdentity(old.name), old.inode) && + controlDigest(d.dir.read(old.name, LIMITS.privateRecordBytes).toString("utf8")) === + old.digest + ) { + ref = old; + return; + } + } finally { + d.close(); + } + } + if (p.phase !== "active" || decision(f).phase !== "active") + throw Error("Retiring family is observation-only"); + if ( + !old && + (p.records >= LIMITS.recordsPerFamily || + (address.scope === "shared" && p.sharedRecords >= 2048)) + ) + throw Error("Family record capacity exhausted"); + ref = { + ...address, + digest: nextDigest, + inode: old?.inode ?? { dev: "0", ino: "0" }, + ordinal: old?.ordinal ?? p.records, + state: "written", + counted: address.scope === "private", + directory: false, + }; + // The durable count reserves this ordinal before ordinary publication. + // The inode-bound page is fsynced by the publication callback while the + // same family writer is held. An interrupted rename leaves a pinned + // mismatch, never an unindexed exposed record. + if (!old) { + p.records++; + if (address.scope === "shared") p.sharedRecords++; + progress(f, p, true); + } + write((inode) => { + ref = { ...ref, inode }; + saveEntry(f, ref); + }); + }); + } finally { + f.close(); + } +} +export function trackFamilyDirectory( + state: InstallationState, + origin: ControlExecutionOrigin, + address: Pick, + inode: FileIdentity, + generation?: string, +) { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + old = findRecord(f, { scope: "shared", ...address }); + if (old) { + if (!sameInode(old.inode, inode)) throw Error("Replaced indexed directory"); + return; + } + if ( + p.phase !== "active" || + decision(f).phase !== "active" || + p.records >= LIMITS.recordsPerFamily || + p.sharedRecords >= 2048 + ) + throw Error("Retained directory capacity/fence"); + const e: FamilyRecord = { + scope: "shared", + ...address, + inode, + generation, + digest: controlDigest({ inode, generation: generation ?? null }), + ordinal: p.records, + state: "written", + counted: false, + directory: true, + }; + saveEntry(f, e); + p.records++; + p.sharedRecords++; + progress(f, p, true); + }); + } finally { + f.close(); + } +} +function ticket( + f: Family, + p: Progress, + kind: "executions" | "intents" | "leases", + id: string, + maximum: number, + value: unknown, +) { + const dir = f.shared.directory(kind); + try { + for (let n = 0; n < maximum; n++) { + const name = `slot-${n}`; + try { + const existing = json<{ id: string; generation: string }>(dir, `${name}.json`, 4096); + if (existing.id === id) { + if (existing.generation !== f.meta.generation) throw Error("Ticket generation mismatch"); + return; + } + } catch (e) { + if (!missing(e)) throw e; + } + const bytes = canonicalJson({ id, generation: f.meta.generation, value }); + if (p.metadataBytes + Buffer.byteLength(bytes) > LIMITS.familyMetadataBytes) + throw Error("Origin retention metadata capacity exhausted before exposure"); + p.metadataBytes += Buffer.byteLength(bytes); + progress(f, p, true); + try { + dir.write(`${name}.json`, bytes, true); + return; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; + } + } + throw Error(`Origin ${kind} capacity exhausted before effect`); + } finally { + dir.close(); + } +} +export function registerFamilyExecution( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, + receiptDigest: string, + parent?: RetainedParentExecution, +) { + reserveControlFamily(state, origin); + const f = openFamily(state, origin); + try { + guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + id = controlDigest(execution); + if (p.executions[id]) { + if (p.executions[id].receiptDigest !== receiptDigest) + throw Error("Family execution conflict"); + return; + } + if (p.phase !== "active" || decision(f).phase !== "active") + throw Error("Closed origin membership"); + ticket(f, p, "executions", id, LIMITS.executionsPerFamily, execution); + p.executions[id] = { + execution, + receiptDigest, + verifier: state.installation.publicKey, + known: false, + terminalAt: 0, + terminalDigest: "0".repeat(64), + requiresJournal: false, + journalContained: false, + ...(state.fixedChildId ? { fixedChildId: state.fixedChildId } : {}), + }; + if (parent?.issuer === "api") { + const api = { + parent, + operationKey: execution.operationKey, + fingerprint: execution.fingerprint, + }; + if (p.api && controlDigest(p.api) !== controlDigest(api)) + throw Error("Origin API parent conflict"); + p.api = api; + } + progress(f, p, true); + }), + ); + } finally { + f.close(); + } +} +export function enlistFamilyIntent( + state: InstallationState, + origin: ControlExecutionOrigin, + id: string, + child: ExecutionRef, + recipient: ControlIssuer, +) { + const f = openFamily(state, origin); + try { + guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + const p = readProgress(f); + if (p.phase !== "active" || decision(f).phase !== "active") + throw Error("Closed origin intent membership"); + if (state.installation.role !== origin.issuer && !p.participants.includes(recipient)) + throw Error("Finite noncoordinator edge cannot enlist a third role"); + ticket(f, p, "intents", id, LIMITS.intentsPerFamily, { + childDigest: controlDigest(child), + recipient, + }); + p.intents[id] = { id, childDigest: controlDigest(child), recipient }; + if (!p.participants.includes(recipient)) p.participants.push(recipient); + preallocateSharedRole(f.shared, recipient); + progress(f, p, true); + }), + ); + } finally { + f.close(); + } +} +export function registerFamilyLease( + state: InstallationState, + origin: ControlExecutionOrigin, + id: string, + certificateDigest: string, + ownerRole: ControlIssuer, +) { + const f = openFamily(state, origin); + try { + guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + const p = readProgress(f); + if (p.leases[id]) return; + if (p.phase !== "active" || decision(f).phase !== "active") + throw Error("Closed origin lease membership"); + ticket(f, p, "leases", id, LIMITS.executionsPerFamily, { certificateDigest, ownerRole }); + p.leases[id] = { id, certificateDigest, ownerRole, released: false }; + progress(f, p, true); + }), + ); + } finally { + f.close(); + } +} +export function requireFamilyJournal( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, +): void { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + fact = p.executions[controlDigest(execution)]; + if (!fact || p.phase !== "active") throw Error("Invalid journal dependency"); + fact.requiresJournal = true; + fact.journalContained = false; + progress(f, p); + }); + } finally { + f.close(); + } +} +export function completeFamilyJournal( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, +): void { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + fact = p.executions[controlDigest(execution)]; + if (!fact?.requiresJournal || p.phase !== "active") + throw Error("Invalid joined journal worker"); + fact.journalContained = true; + p.terminalAt = Math.max(p.terminalAt, Date.now()); + progress(f, p); + }); + } finally { + f.close(); + } +} +export function markFamilyTerminal( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, + known: boolean, + terminalDigest?: string, +) { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + e = p.executions[controlDigest(execution)]; + if (!e) throw Error("Missing indexed terminal execution"); + if (p.phase !== "active") { + if (e.known && known && (!terminalDigest || e.terminalDigest === terminalDigest)) return; + throw Error("Terminal transition after membership closed"); + } + const changed = + e.known !== known || (terminalDigest !== undefined && e.terminalDigest !== terminalDigest); + e.known = known; + if (terminalDigest) e.terminalDigest = terminalDigest; + if (changed || e.terminalAt === undefined) { + e.terminalAt = Date.now(); + p.terminalAt = Math.max(p.terminalAt, e.terminalAt); + } + progress(f, p); + }); + } finally { + f.close(); + } +} +export function markFamilyLeaseReleased( + state: InstallationState, + origin: ControlExecutionOrigin, + id: string, + certificateDigest: string, +) { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + lease = p.leases[id]; + if ( + !lease || + lease.certificateDigest !== certificateDigest || + lease.ownerRole !== state.installation.role + ) + throw Error("Lease release family mismatch"); + if (lease.released) return; + const value = { + version: 1, + familyId: f.meta.familyId, + generation: f.meta.generation, + originDigest: originHash(origin), + leaseId: id, + certificateDigest, + releasedAt: Date.now(), + role: state.installation.role, + }; + lease.released = true; + p.terminalAt = Math.max(p.terminalAt, value.releasedAt); + progress(f, p); + }); + } finally { + f.close(); + } +} +function recordDirectory(f: Family, e: FamilyRecord): { dir: AnchoredDirectory; close(): void } { + if (!safeName.test(e.name) || !(e.scope === "private" ? PRIVATE : SHARED).has(e.namespace)) + throw Error("Invalid fixed record address"); + const opened: AnchoredDirectory[] = []; + let dir: AnchoredDirectory; + if (e.scope === "private") dir = f.state.privateRoot.directory(e.namespace); + else { + let current = f.root.directory(".admin-target-control"); + opened.push(current); + if (e.namespace === "execution") { + if (!ROLES.includes(e.container as ControlIssuer)) throw Error("Invalid witness issuer"); + current = current.directory("executions"); + opened.push(current); + dir = current.directory(e.container); + } else if (e.namespace === "history-slot") { + current = current.directory("admission-index-v1"); + opened.push(current); + dir = current.directory("history"); + } else { + if (!z.uuid().safeParse(e.container).success) throw Error("Invalid lease record container"); + current = current.directory("leases"); + opened.push(current); + if (e.namespace === "lease-directory" && e.name === e.container) dir = current; + else { + current = current.directory(e.container); + opened.push(current); + dir = + e.namespace === "lease-grants" + ? current.directory("grants") + : e.namespace === "lease-children" + ? current.directory("children") + : current; + if (dir === current) opened.pop(); + } + } + } + return { + dir, + close() { + dir.close(); + for (const d of opened.reverse()) d.close(); + }, + }; +} +function readAllTickets( + f: Family, + kind: "executions" | "intents" | "leases", + maximum: number, +): Array<{ id: string; value: unknown }> { + const d = f.shared.directory(kind), + result: Array<{ id: string; value: unknown }> = []; + try { + for (let n = 0; n < maximum; n++) + try { + const t = json<{ id: string; generation: string; value: unknown }>( + d, + `slot-${n}.json`, + 4096, + ); + if (t.generation !== f.meta.generation) throw Error("Stale family ticket generation"); + result.push(t); + } catch (e) { + if (!missing(e)) throw e; + } + return result; + } finally { + d.close(); + } +} +function directoryPresent(parent: AnchoredDirectory, name: string): boolean { + try { + const dir = parent.directory(name); + dir.close(); + return true; + } catch (error) { + if (missing(error)) return false; + throw error; + } +} +function executionWriterPresent(f: Family, execution: ExecutionRef): boolean { + const opened: AnchoredDirectory[] = []; + const own = (d: AnchoredDirectory) => { + opened.push(d); + return d; + }; + try { + try { + const effects = own(f.state.privateRoot.directory("effect-writers")); + if (directoryPresent(effects, execution.operationKey)) return true; + } catch (error) { + if (!missing(error)) throw error; + } + const control = own(f.root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + role = own(executions.directory(execution.executor)); + return directoryPresent(role, `${execution.operationKey}.writer`); + } finally { + for (const dir of opened.reverse()) dir.close(); + } +} +function localProof(f: Family, p: Progress): boolean { + if ( + !Object.keys(p.executions).length || + Object.values(p.executions).some((e) => !e.known || e.terminalAt === undefined) + ) + return false; + for (const fact of Object.values(p.executions)) { + if (executionWriterPresent(f, fact.execution)) return false; + if ( + fact.requiresJournal && + (!fact.journalContained || + !f.state.retentionJournal?.contained(fact.execution, fact.terminalDigest)) + ) + return false; + const executions = f.state.privateRoot.directory("executions"), + effects = f.state.privateRoot.directory("effects"), + lifecycle = f.state.privateRoot.directory("execution-lifecycle"); + try { + const raw = json(executions, `${fact.execution.operationKey}.json`, 65536); + if (controlDigest(raw) !== fact.receiptDigest) return false; + const admitted = verifyControl( + fact.verifier, + "openmapx-control-execution-v2", + raw, + z + .object({ version: z.literal(2), execution: z.unknown(), origin: originSchema }) + .passthrough(), + ); + if ( + controlDigest(admitted.execution) !== controlDigest(fact.execution) || + controlDigest(admitted.origin) !== f.originHash + ) + return false; + const terminal = verifyControl( + fact.verifier, + "openmapx-control-settlement-v1", + json(effects, `${fact.execution.operationKey}.json`, 65536), + z + .object({ + identity: z.unknown(), + outcome: z.enum(["contained", "committed", "failed_without_effect"]), + publication: z.enum(["not_started", "committed"]), + terminalReceiptDigest: z.string(), + receipt: z.unknown(), + }) + .passthrough(), + ); + if ( + controlDigest(terminal.identity) !== controlDigest(fact.execution) || + terminal.terminalReceiptDigest !== controlDigest(terminal.receipt) + ) + return false; + const phase = verifyControl( + fact.verifier, + "openmapx-execution-lifecycle-v1", + json(lifecycle, `${fact.execution.operationKey}.json`, 65536), + z.object({ execution: z.unknown(), phase: z.literal("settled") }).passthrough(), + ); + if (controlDigest(phase.execution) !== controlDigest(fact.execution)) return false; + } finally { + lifecycle.close(); + effects.close(); + executions.close(); + } + } + for (const lease of Object.values(p.leases)) { + const control = f.root.directory(".admin-target-control"), + leases = control.directory("leases"), + ledger = leases.directory(lease.id); + try { + if (directoryPresent(ledger, ".journal-writer")) return false; + const released = verified<{ + leaseId: string; + certificateDigest: string; + releasedAt?: number; + }>( + f.state, + lease.ownerRole, + "openmapx-admission-slots-v1", + json(ledger, "admission.json", 4096), + ); + if ( + released.leaseId !== lease.id || + released.certificateDigest !== lease.certificateDigest || + !released.releasedAt + ) + return false; + } finally { + ledger.close(); + leases.close(); + control.close(); + } + } + return true; +} +function sameFamily(f: Family, v: { familyId: string; generation: string; originDigest?: string }) { + if ( + v.familyId !== f.meta.familyId || + v.generation !== f.meta.generation || + (v.originDigest !== undefined && v.originDigest !== f.originHash) + ) + throw Error("Retirement family/generation mismatch"); +} +function ack(f: Family, role: ControlIssuer): Ack | null { + const raw = json(f.shared, `ack-${role}.json`, 4096); + if (raw === null) return null; + const a = verified(f.state, role, "openmapx-retention-ack-v1", raw); + sameFamily(f, a); + if ( + a.role !== role || + !DIGEST.test(a.manifestDigest) || + !["ready", "clean", "released"].includes(a.phase) || + !Number.isSafeInteger(a.sequence) || + a.sequence < 1 + ) + throw Error("Invalid retirement acknowledgement"); + return a; +} +function manifest(f: Family): Manifest { + const m = verified( + f.state, + f.meta.origin.issuer, + "openmapx-retention-manifest-v1", + json(f.shared, "manifest.json", 32768), + ); + sameFamily(f, m); + if ( + m.version !== 1 || + m.installationId !== f.state.installation.installationId || + controlDigest(m.origin) !== f.originHash || + Object.keys(m.executionReceiptDigests).length > LIMITS.executionsPerFamily || + m.participants.length > 4 + ) + throw Error("Invalid retirement manifest"); + return m; +} +function publishAck(f: Family, p: Progress, m: Manifest, phase: Ack["phase"]) { + const old = ack(f, f.state.installation.role); + if (old && old.manifestDigest !== controlDigest(m)) + throw Error("Conflicting retirement acknowledgement"); + if (old && old.sequence >= Number.MAX_SAFE_INTEGER) + throw Error("Retirement acknowledgement sequence exhausted"); + const value: Ack = { + version: 1, + installationId: f.state.installation.installationId, + familyId: f.meta.familyId, + generation: f.meta.generation, + originDigest: f.originHash, + manifestDigest: controlDigest(m), + role: f.state.installation.role, + phase, + terminalSequenceDigest: m.terminalSequenceDigest, + recordSetDigest: m.localRecordSetDigests[f.state.installation.role] ?? "", + proofCopyDigest: controlDigest({ + proof: json(f.slot, "proof.json", 65536), + manifest: json(f.slot, "manifest.json", 32768), + commit: json(f.slot, "commit.json", 8192), + final: json(f.slot, "final.json", 8192), + }), + sequence: (old?.sequence ?? 0) + 1, + }; + cell( + f.shared, + `ack-${f.state.installation.role}.json`, + signed(f.state, "openmapx-retention-ack-v1", value), + 4096, + ); + p.phase = phase; + progress(f, p); +} +function publishCandidate(f: Family, p: Progress): Candidate | null { + if ( + Date.now() < f.meta.origin.notAfter || + Date.now() - p.terminalAt < LIMITS.minimumTerminalRetentionMs || + !localProof(f, p) + ) + return null; + let sharedPage: unknown[] = [], + sharedPageNumber = 0, + sharedCount = 0; + const flush = () => { + cell( + f.shared, + `shared-${f.state.installation.role}-${sharedPageNumber++}.json`, + sharedPage, + 16384, + ); + sharedPage = []; + }; + for (const e of records(f)) { + if (e.state !== "written") return null; + const d = recordDirectory(f, e); + try { + if (e.directory) { + const child = d.dir.directory(e.name); + try { + if (!sameInode(identity(child.stat), e.inode)) + throw Error("Retirement directory replaced"); + } finally { + child.close(); + } + } else if ( + !sameInode(d.dir.fileIdentity(e.name), e.inode) || + controlDigest(d.dir.read(e.name, LIMITS.privateRecordBytes).toString("utf8")) !== e.digest + ) + throw Error("Retirement record replaced"); + } finally { + d.close(); + } + if (e.scope === "shared") { + if (++sharedCount > 2048) throw Error("Shared retirement record workspace exhausted"); + sharedPage.push(tuple(e)); + if (sharedPage.length === 64) flush(); + } + } + if (sharedPage.length) flush(); + while (sharedPageNumber < 32) flush(); + const executionReceiptDigests = Object.fromEntries( + Object.entries(p.executions).map(([id, e]) => [id, e.terminalDigest ?? e.receiptDigest]), + ); + const candidate: Candidate = { + version: 1, + installationId: f.state.installation.installationId, + familyId: f.meta.familyId, + generation: f.meta.generation, + role: f.state.installation.role, + originDigest: f.originHash, + terminalAt: p.terminalAt, + terminalSequenceDigest: controlDigest( + Object.values(p.executions).map((e) => ({ + execution: e.execution, + terminalAt: e.terminalAt, + digest: e.terminalDigest ?? e.receiptDigest, + })), + ), + executionReceiptDigests, + leaseDigests: Object.fromEntries( + Object.values(p.leases).map((l) => [l.id, l.certificateDigest]), + ), + intentDigests: Object.fromEntries(Object.values(p.intents).map((i) => [i.id, i.childDigest])), + localRecordSetDigest: recordSetDigest(records(f)), + sharedRecordSetDigest: recordSetDigest(readSharedRecords(f, f.state.installation.role)), + }; + cell( + f.shared, + `candidate-${f.state.installation.role}.json`, + signed(f.state, "openmapx-retention-candidate-v1", candidate), + 65536, + ); + return candidate; +} +function* readSharedRecords(f: Family, role: ControlIssuer): Generator { + for (let page = 0; page < 32; page++) { + const values = json(f.shared, `shared-${role}-${page}.json`, 16384); + if (!Array.isArray(values) || values.length > 64) throw Error("Corrupt shared ownership page"); + for (const value of values) yield entry(value); + } +} +function prepare(f: Family, p: Progress): boolean { + if (f.state.installation.role !== f.meta.origin.issuer || f.state.fixedChildId) return false; + return guard(f.shared, ".membership-writer", () => { + if (decision(f).phase !== "active") return true; + const candidates: Candidate[] = []; + for (const role of p.participants) { + const raw = json(f.shared, `candidate-${role}.json`, 65536); + if (raw === null) return false; + const c = verified(f.state, role, "openmapx-retention-candidate-v1", raw); + sameFamily(f, c); + if ( + c.role !== role || + c.installationId !== f.state.installation.installationId || + Date.now() - c.terminalAt < LIMITS.minimumTerminalRetentionMs + ) + return false; + if (recordSetDigest(readSharedRecords(f, role)) !== c.sharedRecordSetDigest) + throw Error("Shared ownership digest conflict"); + candidates.push(c); + } + const receipts = Object.assign( + {}, + ...candidates.map((c) => c.executionReceiptDigests), + ) as Record; + const executions = readAllTickets(f, "executions", LIMITS.executionsPerFamily); + const intents = readAllTickets(f, "intents", LIMITS.intentsPerFamily); + if ( + Object.keys(receipts).length !== executions.length || + executions.some((t) => !receipts[t.id]) || + intents.some((t) => !receipts[(t.value as { childDigest: string }).childDigest]) + ) + return false; + const intentDigests = Object.assign({}, ...candidates.map((c) => c.intentDigests)) as Record< + string, + string + >; + if ( + Object.keys(intentDigests).length !== intents.length || + Object.values(intentDigests).some((id) => !receipts[id]) || + intents.some((t) => intentDigests[t.id] !== (t.value as { childDigest: string }).childDigest) + ) + return false; + const leaseDigests = Object.assign({}, ...candidates.map((c) => c.leaseDigests)) as Record< + string, + string + >, + leases = readAllTickets(f, "leases", LIMITS.executionsPerFamily); + if ( + Object.keys(leaseDigests).length !== leases.length || + leases.some( + (t) => leaseDigests[t.id] !== (t.value as { certificateDigest: string }).certificateDigest, + ) + ) + return false; + const external: string[] = []; + if (p.api) { + const a = apiCopy(f, p); + const root = Object.values(p.executions).find( + (e) => e.execution.operationKey === p.api?.operationKey, + ); + if ( + !a || + !root?.terminalDigest || + a.operationKey !== p.api.operationKey || + a.fingerprint !== p.api.fingerprint || + a.terminalDigest !== root.terminalDigest + ) + return false; + external.push(`api:${p.api.parent.apiJobId}:${p.api.parent.attemptId}`); + } + const m: Manifest = { + version: 1, + installationId: f.state.installation.installationId, + origin: f.meta.origin, + familyId: f.meta.familyId, + generation: f.meta.generation, + sequence: 1, + terminalAt: Math.max(...candidates.map((c) => c.terminalAt)), + terminalSequenceDigest: controlDigest({ + roles: candidates.map((c) => c.terminalSequenceDigest), + api: p.api ? apiCopy(f, p) : null, + }), + participants: [...p.participants, ...external], + executionReceiptDigests: receipts, + sharedRecordSetDigest: controlDigest(candidates.map((c) => c.sharedRecordSetDigest)), + localRecordSetDigests: Object.fromEntries( + candidates.map((c) => [c.role, c.localRecordSetDigest]), + ), + }; + advanceRetentionClock(f.state, f.meta.origin.notAfter); + const envelope = signed(f.state, "openmapx-retention-manifest-v1", m); + cell(f.slot, "manifest.json", envelope, 32768); + cell(f.shared, "manifest.json", envelope, 32768); + writeDecision(f, { + version: 1, + familyId: f.meta.familyId, + generation: f.meta.generation, + phase: "prepared", + manifestDigest: controlDigest(m), + sequence: 1, + }); + p.phase = "prepared"; + p.manifestDigest = controlDigest(m); + progress(f, p); + return true; + }); +} +function roleIds(m: Manifest): ControlIssuer[] { + return m.participants.filter((p): p is ControlIssuer => ROLES.includes(p as ControlIssuer)); +} +function validateAck(_f: Family, m: Manifest, a: Ack | null, phase: Ack["phase"]): a is Ack { + return ( + !!a && + a.phase === phase && + a.manifestDigest === controlDigest(m) && + a.terminalSequenceDigest === m.terminalSequenceDigest && + a.recordSetDigest === m.localRecordSetDigests[a.role] + ); +} +function unlinkShared(f: Family, p: Progress, e: FamilyRecord) { + const previous = p.sharedTransaction; + if ( + previous && + (previous.ordinal !== e.ordinal || controlDigest(previous.record) !== controlDigest(e)) + ) + throw Error("Shared retirement transaction conflict"); + const d = recordDirectory(f, e); + try { + if (!previous) { + // Absence is resumable only after this exact item has a durable plan. + if (e.directory) { + const child = d.dir.directory(e.name); + try { + if (!sameInode(identity(child.stat), e.inode)) + throw Error("Replaced retirement directory"); + if ( + e.generation && + json<{ generation: string }>(child, "generation.json", 1024).generation !== e.generation + ) + throw Error("Reused history slot generation"); + } finally { + child.close(); + } + } else if ( + !sameInode(d.dir.fileIdentity(e.name), e.inode) || + controlDigest(d.dir.read(e.name, LIMITS.privateRecordBytes).toString("utf8")) !== e.digest + ) + throw Error("Replaced shared retirement record"); + p.sharedTransaction = { ordinal: e.ordinal, phase: "planned", record: e }; + progress(f, p); + } + if (p.sharedTransaction?.phase === "planned") { + try { + if (e.directory) { + const child = d.dir.directory(e.name); + try { + if (!sameInode(identity(child.stat), e.inode)) + throw Error("Replaced retirement directory"); + } finally { + child.close(); + } + if (e.generation) { + const child = d.dir.directory(e.name); + try { + try { + const g = json<{ generation: string }>(child, "generation.json", 1024); + if (g.generation !== e.generation) throw Error("Reused history slot generation"); + child.removeFile("generation.json"); + } catch (error) { + if (!missing(error)) throw error; + } + } finally { + child.close(); + } + } + d.dir.removeDirectory(e.name, e.inode); + } else { + if ( + !sameInode(d.dir.fileIdentity(e.name), e.inode) || + controlDigest(d.dir.read(e.name, LIMITS.privateRecordBytes).toString("utf8")) !== + e.digest + ) + throw Error("Replaced shared retirement record"); + d.dir.removeFile(e.name); + } + } catch (error) { + if (!missing(error)) throw error; + } + p.sharedTransaction = { ordinal: e.ordinal, phase: "unlinked", record: e }; + progress(f, p); + } + p.sharedCursor++; + delete p.sharedTransaction; + progress(f, p); + } finally { + d.close(); + } +} +function cleanupLocalRecords(f: Family, p: Progress, m: Manifest): boolean { + if (p.lastDone) { + const dir = f.state.privateRoot.directory(p.lastDone.namespace); + try { + finishRetainedDeletion(dir, { + ...p.lastDone, + familyId: f.meta.familyId, + generation: f.meta.generation, + manifestDigest: controlDigest(m), + }); + } finally { + dir.close(); + } + delete p.lastDone; + progress(f, p); + } + let remaining = 64; + while (p.cursor < p.records && remaining--) { + const page = json(f.slot, `page-${Math.floor(p.cursor / 64)}.json`, 16384); + const e = entry(page[p.cursor % 64]); + if (e.ordinal !== p.cursor) throw Error("Missing local cleanup ordinal"); + if (e.scope !== "private") { + p.cursor++; + progress(f, p); + continue; + } + const d = recordDirectory(f, e); + try { + deleteRetainedRecord( + d.dir, + { + familyId: f.meta.familyId, + generation: f.meta.generation, + manifestDigest: controlDigest(m), + ordinal: e.ordinal, + name: e.name, + inode: e.inode, + digest: e.digest, + }, + () => { + p.cursor++; + p.lastDone = { + namespace: e.namespace, + ordinal: e.ordinal, + name: e.name, + inode: e.inode, + digest: e.digest, + }; + progress(f, p); + }, + ); + delete p.lastDone; + progress(f, p); + } finally { + d.close(); + } + } + return p.cursor === p.records; +} +function cleanupSharedRecords(f: Family, p: Progress, m: Manifest): boolean { + // Four fixed passes preserve physical containment: files, ledger subdirs, + // ledger roots, then history slots. Resume uses the same fixed ordinal space + // and reads one <=16KiB ownership page at a time. + const roles = roleIds(m), + perPass = roles.length * 2048, + total = perPass * 4; + let budget = 64, + dirty = false; + const passFor = (e: FamilyRecord) => + !e.directory ? 0 : e.namespace === "history-slot" ? 3 : e.name === e.container ? 2 : 1; + let cached = "", + page: unknown[] = []; + while (p.sharedCursor < total && budget > 0) { + const cursor = p.sharedCursor, + pass = Math.floor(cursor / perPass), + within = cursor % perPass, + role = roles[Math.floor(within / 2048)], + offset = within % 2048, + pageNumber = Math.floor(offset / 64), + key = `${role}:${pageNumber}`; + if (!role) throw Error("Invalid shared cleanup role cursor"); + if (cached !== key) { + page = json(f.shared, `shared-${role}-${pageNumber}.json`, 16384); + if (!Array.isArray(page) || page.length > 64) throw Error("Corrupt shared cleanup page"); + cached = key; + } + const value = page[offset % 64]; + if (value === undefined) { + p.sharedCursor += 64 - (offset % 64); + dirty = true; + continue; + } + const e = entry(value); + if (passFor(e) !== pass) { + p.sharedCursor++; + dirty = true; + continue; + } + if (dirty) { + progress(f, p); + dirty = false; + } + unlinkShared(f, p, { ...e, ordinal: cursor }); + budget--; + } + if (dirty) progress(f, p); + return p.sharedCursor === total; +} +const SLOT_FILES = [ + ...Array.from({ length: 64 }, (_, n) => `page-${n}.json`), + "manifest.json", + "commit.json", + "final.json", + "proof.json", + "progress.json", + "progress-highwater.json", + "identity.json", +]; +interface SlotRelease { + familyId: string; + generation: string; + slotGeneration: string; + name: string; + inode: FileIdentity; + originHash: string; + role: ControlIssuer; + proof: unknown; + phase: "planned" | "item_done"; +} +function resumeSlotRelease(state: InstallationState): boolean { + return guard(state.privateRoot, ".retention-slot-writer", () => { + const t = json(state.privateRoot, "retention-slot-transaction.json", 4096); + if (t === null) return false; + if ( + !slotName.test(t.name) || + !DIGEST.test(t.originHash) || + !ROLES.includes(t.role) || + !z.uuid().safeParse(t.slotGeneration).success + ) + throw Error("Corrupt family slot transaction"); + const proof = verified( + state, + t.role, + "openmapx-retention-decision-v1", + t.proof, + ); + if ( + proof.familyId !== t.familyId || + proof.generation !== t.generation || + proof.phase !== "final" || + !proof.semanticClean + ) + throw Error("Invalid final slot-release proof"); + const slots = state.privateRoot.directory("retention-families"); + try { + let slot: AnchoredDirectory | undefined; + try { + slot = slots.directory(t.name); + } catch (e) { + if (!missing(e)) throw e; + } + if (slot) { + try { + if (!sameInode(identity(slot.stat), t.inode)) + throw Error("Slot transaction inode replaced"); + try { + const id = json<{ familyId: string; generation: string; slotGeneration: string }>( + slot, + "identity.json", + 2048, + ); + if ( + id.familyId !== t.familyId || + id.generation !== t.generation || + id.slotGeneration !== t.slotGeneration + ) + throw Error("Slot transaction generation replaced"); + } catch (e) { + if (!missing(e)) throw e; + } + for (const name of SLOT_FILES) + try { + slot.removeFile(name); + } catch (e) { + if (!missing(e)) throw e; + } + } finally { + slot.close(); + } + slots.removeDirectory(t.name, t.inode); + } + const loc = state.privateRoot.directory("retention-locations"); + try { + try { + loc.removeFile(`${t.originHash}.json`); + } catch (e) { + if (!missing(e)) throw e; + } + } finally { + loc.close(); + } + cell( + state.privateRoot, + "retention-slot-transaction.json", + { ...t, phase: "item_done" }, + 4096, + ); + cell(state.privateRoot, "retention-slot-transaction.json", null, 4096); + return true; + } finally { + slots.close(); + } + }); +} +function freeFamily(f: Family, _p: Progress): void { + guard(f.state.privateRoot, ".retention-slot-writer", () => { + if (json(f.state.privateRoot, "retention-slot-transaction.json", 4096) !== null) + throw Error("Other slot transaction pending"); + checkLocalIdentity(f); + const t: SlotRelease = { + familyId: f.meta.familyId, + generation: f.meta.generation, + slotGeneration: f.slotGeneration, + name: f.name, + inode: f.slotIdentity, + originHash: f.originHash, + role: f.meta.origin.issuer, + proof: json(f.slot, "final.json", 8192), + phase: "planned", + }; + cell(f.state.privateRoot, "retention-slot-transaction.json", t, 4096); + }); + resumeSlotRelease(f.state); +} +function freeSharedWorkspace(f: Family, p: Progress, m: Manifest): void { + if (f.sharedGone) return; + p.sharedRelease = true; + progress(f, p); + const names = [ + "identity.json", + "decision.json", + "manifest.json", + "api-ack.json", + ...roleIds(m).flatMap((role) => [ + `ack-${role}.json`, + `candidate-${role}.json`, + ...Array.from({ length: 32 }, (_, n) => `shared-${role}-${n}.json`), + ]), + ]; + for (const kind of ["executions", "intents", "leases"] as const) { + let dir: AnchoredDirectory; + try { + dir = f.shared.directory(kind); + } catch (e) { + if (missing(e)) continue; + throw e; + } + try { + const max = kind === "intents" ? 256 : 128; + for (let n = 0; n < max; n++) + try { + dir.removeFile(`slot-${n}.json`); + } catch (e) { + if (!missing(e)) throw e; + } + const inode = identity(dir.stat); + dir.close(); + f.shared.removeDirectory(kind, inode); + } finally { + dir.close(); + } + } + for (const name of names) + try { + f.shared.removeFile(name); + } catch (e) { + if (!missing(e)) throw e; + } + try { + f.sharedRoot.removeDirectory(f.originHash, identity(f.shared.stat)); + } catch (e) { + if (!missing(e)) throw e; + } +} +function assertPrivateRetirementProof(f: Family, m: Manifest): void { + const local = verifyControl( + f.state.publicKeys[f.meta.origin.issuer], + "openmapx-retention-manifest-v1", + json(f.slot, "manifest.json", 32768), + z.unknown(), + ); + if (controlDigest(local) !== controlDigest(m)) + throw Error("Private retirement manifest copy changed"); + const proof = verified( + f.state, + f.state.installation.role, + "openmapx-retention-candidate-v1", + json(f.slot, "proof.json", 65536), + ); + sameFamily(f, proof); + if ( + proof.role !== f.state.installation.role || + proof.localRecordSetDigest !== m.localRecordSetDigests[proof.role] || + Object.entries(proof.executionReceiptDigests).some( + ([id, digest]) => m.executionReceiptDigests[id] !== digest, + ) + ) + throw Error("Private terminal proof copy changed"); +} +async function step(f: Family): Promise { + if (f.state.fixedChildId) throw Error("Fixed children do not coordinate role-private retirement"); + let p = readProgress(f); + checkLocalIdentity(f); + if (p.phase === "released" && f.state.installation.role !== f.meta.origin.issuer) { + freeFamily(f, p); + return true; + } + if (p.phase === "released" && p.sharedRelease) { + const m = verified( + f.state, + f.meta.origin.issuer, + "openmapx-retention-manifest-v1", + json(f.slot, "manifest.json", 32768), + ); + freeSharedWorkspace(f, p, m); + freeFamily(f, p); + return true; + } + const d = decision(f); + if (d.phase === "active") { + guard(f.slot, ".family-writer", () => publishCandidate(f, p)); + if (!prepare(f, p)) return false; + p = readProgress(f); + } + const m = manifest(f), + md = controlDigest(m); + if (decision(f).manifestDigest !== md) throw Error("Manifest/decision digest conflict"); + if (p.phase === "active" || p.phase === "prepared") { + for (const e of records(f)) if (e.state !== "written") return false; + if ( + recordSetDigest(records(f)) !== m.localRecordSetDigests[f.state.installation.role] || + !localProof(f, p) + ) + return false; + cell(f.slot, "manifest.json", json(f.shared, "manifest.json", 32768), 32768); + cell( + f.slot, + "proof.json", + json(f.shared, `candidate-${f.state.installation.role}.json`, 65536), + 65536, + ); + p.manifestDigest = md; + publishAck(f, p, m, "ready"); + } + assertPrivateRetirementProof(f, m); + if (f.state.installation.role === f.meta.origin.issuer && decision(f).phase === "prepared") { + const readyDigests: Record = {}; + for (const role of roleIds(m)) { + const a = ack(f, role); + if (!validateAck(f, m, a, "ready")) return false; + readyDigests[role] = controlDigest(a); + } + writeDecision(f, { + version: 1, + familyId: f.meta.familyId, + generation: f.meta.generation, + phase: "committed", + manifestDigest: md, + readyDigests, + sequence: 2, + }); + } + if (decision(f).phase === "prepared") return false; + if (p.phase === "ready" || p.phase === "committed") { + const commit = decision(f); + if (commit.phase !== "committed" && commit.phase !== "final") return false; + cell(f.slot, "commit.json", json(f.shared, "decision.json", 8192), 8192); + advanceRetentionClock(f.state, f.meta.origin.notAfter); + p.phase = "committed"; + progress(f, p); + if (!cleanupLocalRecords(f, p, m)) return false; + if (Object.values(p.executions).some((e) => e.requiresJournal) && !p.journalClean) { + if (!f.state.retentionJournal) return false; + p.journalCleanupPlanned = true; + progress(f, p); + await f.state.retentionJournal.retire( + Object.values(p.executions) + .filter((e) => e.requiresJournal) + .map((e) => { + if (!e.terminalDigest) throw Error("Missing retiring terminal digest"); + return { execution: e.execution, digest: e.terminalDigest }; + }), + md, + ); + p = readProgress(f); + p.journalClean = true; + progress(f, p); + } + publishAck(f, p, m, "clean"); + } + if (f.state.installation.role === f.meta.origin.issuer && decision(f).phase === "committed") { + const cleanDigests: Record = {}; + for (const role of roleIds(m)) { + const a = ack(f, role); + if (!validateAck(f, m, a, "clean")) return false; + cleanDigests[role] = controlDigest(a); + } + writeDecision(f, { + version: 1, + familyId: f.meta.familyId, + generation: f.meta.generation, + phase: "final", + manifestDigest: md, + cleanDigests, + sequence: 3, + }); + } + if (decision(f).phase !== "final") return false; + if (f.state.installation.role === f.meta.origin.issuer && !cleanupSharedRecords(f, p, m)) + return false; + // Participants can release only after coordinator's final semantic cleanup. + if (f.state.installation.role === f.meta.origin.issuer) { + cell( + f.shared, + "decision.json", + signed(f.state, "openmapx-retention-decision-v1", { ...decision(f), semanticClean: true }), + 8192, + ); + } + const final = decision(f) as Decision & { semanticClean?: boolean }; + if (!final.semanticClean) return false; + cell(f.slot, "final.json", json(f.shared, "decision.json", 8192), 8192); + if (p.phase !== "released") publishAck(f, p, m, "released"); + if (f.state.installation.role !== f.meta.origin.issuer) { + freeFamily(f, p); + return true; + } + for (const role of roleIds(m)) if (!validateAck(f, m, ack(f, role), "released")) return false; + freeSharedWorkspace(f, p, m); + freeFamily(f, p); + return true; +} +export async function reconcileRetentionFamilies( + state: InstallationState, + options: { signal?: AbortSignal; maxFamilies?: number } = {}, +): Promise<{ retired: number; pending: number }> { + const max = options.maxFamilies ?? 16; + if (!Number.isSafeInteger(max) || max < 1 || max > 64) + throw Error("Invalid retention maxFamilies"); + options.signal?.throwIfAborted(); + try { + mkdirSync(state.privateRoot.name(".retention-reconcile-writer"), { mode: 0o700 }); + fsyncSync(state.privateRoot.fd); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return { retired: 0, pending: 1 }; + throw error; + } + const held = state.privateRoot.directory(".retention-reconcile-writer"), + expected = identity(held.stat); + try { + return await reconcileRetentionFamiliesLocked(state, options); + } finally { + held.close(); + state.privateRoot.removeDirectory(".retention-reconcile-writer", expected); + } +} +async function reconcileRetentionFamiliesLocked( + state: InstallationState, + options: { signal?: AbortSignal; maxFamilies?: number } = {}, +): Promise<{ retired: number; pending: number }> { + const max = options.maxFamilies ?? 16; + if (!Number.isSafeInteger(max) || max < 1 || max > 64) + throw Error("Invalid retention maxFamilies"); + const clock = readRetentionClock(state); + if (Date.now() < clock.lastRetirementWallTime) + throw Error("Wall clock moved backward; retention recovery prerequisite"); + const recovered = resumeSlotRelease(state); + const cursorName = "retention-cursor.json"; + let cursor = json(state.privateRoot, cursorName, 64); + if (!Number.isInteger(cursor) || cursor < 0 || cursor >= 1024) + throw Error("Invalid retention cursor"); + let retired = recovered ? 1 : 0, + pending = 0, + seen = 0; + let slots: AnchoredDirectory; + try { + slots = state.privateRoot.directory("retention-families"); + } catch (e) { + if (missing(e)) return { retired, pending }; + throw e; + } + try { + for (let scanned = 0; scanned < 1024 && seen < max; scanned++) { + options.signal?.throwIfAborted(); + const name = `slot-${cursor}`; + cursor = (cursor + 1) % 1024; + let slot: AnchoredDirectory; + try { + slot = slots.directory(name); + } catch (e) { + if (missing(e)) continue; + throw e; + } + let hash: string | undefined; + try { + const local = json<{ originHash?: string }>(slot, "identity.json", 2048); + hash = local.originHash; + } catch { + seen++; + pending++; + } finally { + slot.close(); + } + if (!hash) { + if (hash === undefined && seen === 0) { + /* old partial index stays reserved */ + } + continue; + } + seen++; + try { + const locations = state.privateRoot.directory("retention-locations"); + let meta: SharedIdentity; + try { + const loc = json<{ identity: unknown }>(locations, `${hash}.json`, 4096); + const candidate = (loc.identity as { payload: SharedIdentity }).payload; + meta = verifyControl( + state.publicKeys[candidate.origin.issuer], + "openmapx-retention-family-v1", + loc.identity, + sharedIdentitySchema, + ); + } finally { + locations.close(); + } + const f = openFamily(state, meta.origin); + try { + if (await step(f)) retired++; + else pending++; + } catch (error) { + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f); + p.lastError = String(error).slice(0, 512); + progress(f, p); + }); + } catch {} + throw error; + } finally { + f.close(); + } + } catch { + pending++; + } + } + } finally { + cell(state.privateRoot, cursorName, cursor, 64); + slots.close(); + } + return { retired, pending }; +} +/** Exact retained API dependency lookup; absence proof is bounded by the active family slots. */ +export function retentionApiDependency( + state: InstallationState, + origin: ControlExecutionOrigin, +): ApiDependency | undefined { + const f = openFamily(state, origin); + try { + return readProgress(f).api; + } finally { + f.close(); + } +} +export function acknowledgeRetentionDependency( + state: InstallationState, + origin: ControlExecutionOrigin, + input: { operationKey: string; fingerprint: string; terminalDigest: string; proofDigest: string }, +) { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + dep = p.api, + fact = Object.values(p.executions).find( + (e) => e.execution.operationKey === input.operationKey, + ); + if ( + !dep || + dep.operationKey !== input.operationKey || + dep.fingerprint !== input.fingerprint || + !fact?.terminalDigest || + !fact.terminalAt || + fact.terminalDigest !== input.terminalDigest + ) + throw Error("API terminal dependency conflict"); + const payload = apiCopySchema.parse({ + version: 1, + installationId: f.meta.installationId, + familyId: f.meta.familyId, + generation: f.meta.generation, + originDigest: f.originHash, + parentDigest: controlDigest(dep.parent), + ...input, + }); + const old = apiCopy(f, p); + if (old) { + if (controlDigest(old) !== controlDigest(payload)) + throw Error("Conflicting API receipt acknowledgement"); + return; + } + if (decision(f).phase !== "active") + throw Error("Missing committed API acknowledgement proof"); + p.apiAck = { terminalDigest: input.terminalDigest, proofDigest: input.proofDigest }; + progress(f, p); + cell( + f.shared, + "api-ack.json", + signed(state, "openmapx-retention-api-copy-v1", payload), + 4096, + ); + }); + } finally { + f.close(); + } +} +export function hasRetentionEvidence( + state: InstallationState, + origin: ControlExecutionOrigin, +): boolean { + const hash = originHash(origin), + transaction = json<{ originHash?: string } | null>( + state.privateRoot, + "retention-slot-transaction.json", + 4096, + ); + if (transaction && (transaction.originHash === hash || !transaction.originHash)) return true; + let locations: AnchoredDirectory | undefined; + try { + locations = state.privateRoot.directory("retention-locations"); + locations.fileIdentity(`${hash}.json`); + return true; + } catch (error) { + if (!missing(error)) throw error; + } finally { + locations?.close(); + } + let slots: AnchoredDirectory | undefined; + try { + slots = state.privateRoot.directory("retention-families"); + for (let n = 0; n < 1024; n++) { + let slot: AnchoredDirectory; + try { + slot = slots.directory(`slot-${n}`); + } catch (error) { + if (missing(error)) continue; + throw error; + } + try { + let id: { originHash: string }; + try { + id = json(slot, "identity.json", 2048); + } catch { + throw Error("Partial family index requires recovery"); + } + if (!DIGEST.test(id.originHash)) throw Error("Corrupt partial family index"); + if (id.originHash === hash) return true; + } finally { + slot.close(); + } + } + } catch (error) { + if (!missing(error)) throw error; + } finally { + slots?.close(); + } + const opened: AnchoredDirectory[] = []; + const own = (dir: AnchoredDirectory) => { + opened.push(dir); + return dir; + }; + try { + const root = own(new AnchoredDirectory(state.dataRoot)), + control = own(root.directory(".admin-target-control")); + try { + const families = own(control.directory("retention-families")), + shared = own(families.directory(hash)); + shared.check(); + return true; + } catch (error) { + if (!missing(error)) throw error; + } + try { + const executions = own(control.directory("executions")), + role = own(executions.directory(origin.issuer)); + for (const suffix of [ + ".json", + ".lifecycle.json", + ".children.json", + ".keys.json", + ".prepared.json", + ]) + try { + role.fileIdentity(`${origin.rootKey}${suffix}`); + return true; + } catch (error) { + if (!missing(error)) throw error; + } + } catch (error) { + if (!missing(error)) throw error; + } + return false; + } catch (error) { + if (missing(error)) return false; + throw error; + } finally { + for (const dir of opened.reverse()) dir.close(); + } +} diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index 6d8791c23..b928de01c 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -1,7 +1,8 @@ -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { executionFingerprint } from "../ops/execution-identity"; import { opsOperationFingerprint } from "../ops/fingerprint"; @@ -36,11 +37,20 @@ function owner() { const operationFingerprint = opsOperationFingerprint(operation); const deadlineAt = Date.now() + 60000; return retainAdmittedControlExecution(installation, { + requestedDeadlineAt: { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + deadlineAt, + }.deadlineAt, execution: { executor: "host-cli", principal: "host-cli", executionId: randomUUID(), - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint, fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), deadlineAt, @@ -162,6 +172,78 @@ it("detects rollback of a signed shared journal using private high-water evidenc writeFileSync(path, previous); await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/rollback|replacement/); }); +it("preserves private high-water observations for retained legacy lease certificates", async () => { + const c = await import("./control-authority"), + a = await api(); + const { createPrivateKey } = await import("node:crypto"), + { writeFileSync } = await import("node:fs"); + const authority = owner(), + lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + a.closeArtifactTargetLeaseHandle(lease); + c.retainControlEffectReceipt(authority, { + outcome: "unknown", + publication: "unknown", + receipt: { kind: "unresolved" }, + }); + const state = c.installationState(installation), + ledger = join(root, `data/.admin-target-control/leases/${lease.leaseId}`); + const privatePath = join(root, `control-private/host-cli/leases/${lease.leaseId}.json`), + saved = JSON.parse(readFileSync(privatePath, "utf8")); + const { origin: _origin, ...certificate } = saved.certificate.payload; + saved.certificate = c.signControl(state.privateKey, "openmapx-lease-certificate-v1", { + ...certificate, + version: 1, + }); + writeFileSync(privatePath, JSON.stringify(saved)); + const ownerKey = createPrivateKey({ + key: Buffer.from(saved.privateKey, "base64url"), + format: "der", + type: "pkcs8", + }); + const path = join(ledger, "owner.json"), + { + version: _version, + origin: _rootOrigin, + ...rootRecord + } = JSON.parse(readFileSync(path, "utf8")).payload; + const legacy = { ...rootRecord, certificate: saved.certificate }; + const old = c.signControl(ownerKey, "openmapx-lease-owner-v1", legacy); + writeFileSync(path, JSON.stringify(old)); + const slotsPath = join(ledger, "admission.json"), + slots = JSON.parse(readFileSync(slotsPath, "utf8")).payload; + writeFileSync( + slotsPath, + JSON.stringify( + c.signControl(state.privateKey, "openmapx-admission-slots-v1", { + ...slots, + certificateDigest: c.controlDigest(saved.certificate), + }), + ), + ); + const observation = join(root, `control-private/host-cli/observations/${lease.leaseId}.json`); + rmSync(observation); + const recovery = c.verifyControlRecoveryAuthority(authority), + input = { dataRoot: join(root, "data"), leaseId: lease.leaseId, authority: recovery }; + expect(await a.reconcileArtifactTargetLease(input)).toBe("fenced"); + expect(existsSync(observation)).toBe(true); + writeFileSync( + path, + JSON.stringify( + c.signControl(ownerKey, "openmapx-lease-owner-v1", { + ...legacy, + sequence: 1, + phase: "cancel_pending", + }), + ), + ); + expect(await a.reconcileArtifactTargetLease(input)).toBe("fenced"); + writeFileSync(path, JSON.stringify(old)); + await expect(a.reconcileArtifactTargetLease(input)).rejects.toThrow(/rollback|replacement/); +}); it("rejects arbitrary caller text as effect evidence", async () => { const authority = owner(); expect(() => @@ -192,7 +274,7 @@ it("retains terminal maintenance ownership and permits only frozen one-use conti executor: "ops-agent" as const, principal: "api" as const, executionId: `job1_${randomUUID().replaceAll("-", "")}`, - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), operationFingerprint, deadlineAt, fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), @@ -220,6 +302,7 @@ it("retains terminal maintenance ownership and permits only frozen one-use conti recipient: "ops-agent" as const, }; const authority = c.retainAdmittedControlExecution(agent, { + requestedDeadlineAt: execution.deadlineAt, execution, kind: operation.kind, operation, @@ -289,10 +372,11 @@ it("retains terminal maintenance ownership and permits only frozen one-use conti serviceIds: ["motis"], }); const childAuthority = c.retainAdmittedControlExecution(agent, { + requestedDeadlineAt: child.deadlineAt, execution: child, kind: body.kind, operation: body, - parent: c.retainedControlParent(execution), + parent: c.verifiedRetainedControlParent(agent, execution), admittedAt: deadlineAt - 60000, serviceIds: ["motis"], }); @@ -422,7 +506,7 @@ it("admits from a bounded active index without opening retained terminal ledgers } expect(new Set(historicalReads).size, "new acquisition must read only its new ledger").toBe(1); if (lease) a.closeArtifactTargetLeaseHandle(lease); -}); +}, 60000); it("retains terminal and unresolved records past 72h and refuses at retained capacity before effects", async () => { const a = await api(); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index ebd55c71f..75551b4d1 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -22,6 +22,7 @@ import { childSettlementState, controlChildOperationKey, controlDigest, + controlOriginSchema, controlPeerState, digest, encodePrivate, @@ -30,11 +31,13 @@ import { type InstallationState, installationState, newControlKey, + originRecordDomain, parseControlAssertion, readControlExecutionWitness, readPrivateRecord, recoveryAuthorityState, retainedControlParent, + sameControlOrigin, signControl, targetSchema, uuid, @@ -42,6 +45,7 @@ import { type VerifiedControlPeer, type VerifiedExecutionAuthority, type VerifiedRecoveryAuthority, + validateOriginRecord, verifyControl, verifyControlEffectSettlement, verifyControlPreparedTerminal, @@ -49,6 +53,16 @@ import { } from "./control-authority"; import { CONTROL_RETENTION_LIMITS, reserveAuthoritySlot } from "./control-retention"; import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; +import { assertOriginClockForward } from "./retention-clock"; +import { + enlistFamilyIntent, + familyRecordWrite, + hasLocalRetentionFamily, + markFamilyLeaseReleased, + registerFamilyLease, + reserveControlFamily, + trackFamilyDirectory, +} from "./retention-family"; export type { ChildSettlement } from "./control-authority"; export type ArtifactTargetLease = { @@ -67,7 +81,8 @@ export type BorrowedArtifactTargetLease = { export type TargetLeaseAuthority = ArtifactTargetLease | BorrowedArtifactTargetLease; const inode = z.strictObject({ dev: z.string().regex(/^\d+$/), ino: z.string().regex(/^\d+$/) }); const certificateSchema = z.strictObject({ - version: z.literal(1), + version: z.union([z.literal(1), z.literal(2)]), + origin: controlOriginSchema.optional(), installationId: uuid, leaseId: uuid, dataRootIdentity: inode, @@ -78,7 +93,11 @@ const certificateSchema = z.strictObject({ createdAt: z.number().int().nonnegative().safe(), }); type Certificate = z.infer; -const slotSchema = z.strictObject({ name: z.string().regex(/^slot-\d{1,4}$/), identity: inode }); +const slotSchema = z.strictObject({ + name: z.string().regex(/^slot-\d{1,4}$/), + identity: inode, + generation: uuid.optional(), +}); const admissionSlotSchema = z.strictObject({ version: z.literal(1), installationId: uuid, @@ -86,6 +105,16 @@ const admissionSlotSchema = z.strictObject({ certificateDigest: digest, active: slotSchema, history: slotSchema, + release: z + .object({ + phase: z.enum(["planned", "released"]), + locks: z.array(z.string()).max(16), + active: z.boolean(), + pending: z.string().optional(), + activePlanned: z.boolean().optional(), + }) + .optional(), + releasedAt: z.number().int().positive().safe().optional(), }); type AdmissionSlots = z.infer; @@ -101,6 +130,8 @@ const outcomeSchema = z.strictObject({ }); const phaseSchema = z.enum(["held", "cancel_pending", "recovery_required", "settled"]); const rootSchema = z.strictObject({ + version: z.literal(2).optional(), + origin: controlOriginSchema.optional(), certificate: signedCertificateSchema, sequence: z.number().int().nonnegative().safe(), phase: phaseSchema, @@ -109,7 +140,8 @@ const rootSchema = z.strictObject({ }); type RootRecord = z.infer; const grantSchema = z.strictObject({ - version: z.literal(1), + version: z.union([z.literal(1), z.literal(2)]), + origin: controlOriginSchema.optional(), installationId: uuid, leaseId: uuid, ownerCertificateDigest: digest, @@ -128,6 +160,8 @@ const grantSchema = z.strictObject({ }); type Grant = z.infer; const childSchema = z.strictObject({ + version: z.literal(2).optional(), + origin: controlOriginSchema.optional(), delegationId: uuid, grantDigest: digest, execution: executionRefSchema, @@ -200,7 +234,7 @@ function json(dir: AnchoredDirectory, name: string, max = 131072): unknown { function readRoot(s: State): RootRecord { return verifyControl( s.certificate.ownerPublicKey, - "openmapx-lease-owner-v1", + originRecordDomain("openmapx-lease-owner", json(s.ledger, "owner.json")), json(s.ledger, "owner.json"), rootSchema, ); @@ -214,7 +248,12 @@ function readGrant(s: State, id: string, parentKey: string): Grant { const envelope = z .strictObject({ grant: z.unknown(), transition: z.unknown() }) .parse(json(dir, `${uuid.parse(id)}.json`)); - const grant = verifyControl(parentKey, "openmapx-lease-grant-v1", envelope.grant, grantSchema); + const grant = verifyControl( + parentKey, + originRecordDomain("openmapx-lease-grant", envelope.grant), + envelope.grant, + grantSchema, + ); const transition = verifyControl( parentKey, "openmapx-lease-intent-v1", @@ -222,6 +261,8 @@ function readGrant(s: State, id: string, parentKey: string): Grant { transitionSchema, ); if ( + !validateOriginRecord(grant) || + !sameControlOrigin(grant.origin, s.certificate.origin) || grant.delegationId !== id || transition.delegationId !== id || transition.grantDigest !== controlDigest(envelope.grant) || @@ -245,8 +286,15 @@ function readChild(s: State, grant: Grant): ChildRecord | undefined { if ((e as NodeJS.ErrnoException).code === "ENOENT") return; throw e; } - const child = verifyControl(grant.childPublicKey, "openmapx-lease-child-v1", raw, childSchema); + const child = verifyControl( + grant.childPublicKey, + originRecordDomain("openmapx-lease-child", raw), + raw, + childSchema, + ); if ( + !validateOriginRecord(child) || + !sameControlOrigin(child.origin, grant.origin) || child.delegationId !== grant.delegationId || controlDigest(child.execution) !== controlDigest(grant.child) ) @@ -278,7 +326,8 @@ function highWater(s: State, nodes: Record tree(s), false); const root = readRoot(s); - if (controlDigest(root.certificate) !== controlDigest(s.certificateEnvelope)) + if ( + !validateOriginRecord(root) || + !sameControlOrigin(root.origin, s.certificate.origin) || + controlDigest(root.certificate) !== controlDigest(s.certificateEnvelope) + ) throw Error("Owner certificate replacement"); const nodes: Record = { owner: { sequence: root.sequence, digest: controlDigest(root) }, @@ -338,6 +391,7 @@ function tree(s: State): LeaseTree { ); const allowedKinds = childPhaseKinds[grant.phase as ControlChildPhase]; if ( + !sameControlOrigin(witness.origin, grant.origin) || !allowedKinds?.includes(grant.childKind) || grant.child.operationKey !== controlChildOperationKey( @@ -434,7 +488,21 @@ function writeBounded( } if (ledgerBytes(s) - old + Buffer.byteLength(bytes) > 131072) throw Error("Lease ledger capacity exceeded"); - dir.write(name, bytes, exclusive); + if (s.certificate.origin) + familyRecordWrite( + s.installation, + s.certificate.origin, + { + scope: "shared", + namespace: kind === "owner" ? "lease-owner" : `lease-${kind}`, + container: s.certificate.leaseId, + name, + }, + bytes, + (before) => dir.write(name, bytes, exclusive, before), + exclusive, + ); + else dir.write(name, bytes, exclusive); } finally { if (dir !== s.ledger) dir.close(); } @@ -465,7 +533,7 @@ function guarded(s: State, run: () => T, checkTargets = true): T { } } function writeNode(s: State, node: RootRecord | ChildRecord) { - const domain = s.node === null ? "openmapx-lease-owner-v1" : "openmapx-lease-child-v1"; + const domain = `${s.node === null ? "openmapx-lease-owner" : "openmapx-lease-child"}-v${node.version === 2 ? 2 : 1}`; writeBounded( s, s.node === null ? "owner" : "children", @@ -526,12 +594,29 @@ function admissionIndex( throw e; } } -function releaseActiveSlot(s: State) { +function removeSlot(dir: AnchoredDirectory, slot: z.infer) { + const child = dir.directory(slot.name); + try { + if (!sameInode(identity(child.stat), slot.identity)) + throw Error("Replaced admission slot inode"); + if (slot.generation) { + const value = z + .strictObject({ generation: uuid }) + .parse(json(child, "generation.json", 1024)); + if (value.generation !== slot.generation) throw Error("Reused admission slot generation"); + child.removeFile("generation.json"); + } + } finally { + child.close(); + } + dir.removeDirectory(slot.name, slot.identity); +} +function _releaseActiveSlot(s: State) { if (!s.slots) throw Error("Missing retained admission slots"); const index = s.control.directory("admission-index-v1"), active = index.directory("active"); try { - active.removeDirectory(s.slots.active.name, s.slots.active.identity); + removeSlot(active, s.slots.active); } finally { active.close(); index.close(); @@ -619,7 +704,8 @@ export async function acquireArtifactTargetLease(options: { ledger.directory("children", true).close(); const key = newControlKey(); const certificate: Certificate = { - version: 1, + version: 2, + origin: authority.origin, installationId: authority.installation.installation.installationId, leaseId, dataRootIdentity: identity(data.stat), @@ -631,7 +717,7 @@ export async function acquireArtifactTargetLease(options: { }; const certificateEnvelope = signControl( authority.installation.privateKey, - "openmapx-lease-certificate-v1", + "openmapx-lease-certificate-v2", certificate, ); const slots: AdmissionSlots = { @@ -642,11 +728,49 @@ export async function acquireArtifactTargetLease(options: { active: activeSlot, history: historySlot, }; - ledger.write( - "admission.json", - canonicalJson( - signControl(authority.installation.privateKey, "openmapx-admission-slots-v1", slots), - ), + if (!authority.origin) throw Error("Legacy root cannot acquire a fresh lease"); + registerFamilyLease( + authority.installation, + authority.origin, + leaseId, + controlDigest(certificateEnvelope), + authority.execution.executor, + ); + trackFamilyDirectory( + authority.installation, + authority.origin, + { namespace: "lease-directory", container: leaseId, name: leaseId }, + identity(ledger.stat), + ); + for (const name of ["grants", "children"]) { + const dir = ledger.directory(name); + try { + trackFamilyDirectory( + authority.installation, + authority.origin, + { namespace: "lease-directory", container: leaseId, name }, + identity(dir.stat), + ); + } finally { + dir.close(); + } + } + trackFamilyDirectory( + authority.installation, + authority.origin, + { namespace: "history-slot", container: leaseId, name: historySlot.name }, + historySlot.identity, + historySlot.generation, + ); + const admissionBytes = canonicalJson( + signControl(authority.installation.privateKey, "openmapx-admission-slots-v1", slots), + ); + familyRecordWrite( + authority.installation, + authority.origin, + { scope: "shared", namespace: "lease-owner", container: leaseId, name: "admission.json" }, + admissionBytes, + (before) => ledger?.write("admission.json", admissionBytes, true, before), true, ); writePrivateRecord( @@ -659,8 +783,11 @@ export async function acquireArtifactTargetLease(options: { executionReceiptDigest: authority.receiptDigest, }, true, + authority.origin, ); const record: RootRecord = { + version: 2, + origin: authority.origin, certificate: certificateEnvelope, sequence: 0, phase: "held", @@ -673,11 +800,18 @@ export async function acquireArtifactTargetLease(options: { `${authority.execution.operationKey}.json`, { leaseId, certificateDigest: controlDigest(certificateEnvelope) }, true, + authority.origin, ); published = true; - ledger.write( - "owner.json", - canonicalJson(signControl(key.privateKey, "openmapx-lease-owner-v1", record)), + const ownerBytes = canonicalJson( + signControl(key.privateKey, "openmapx-lease-owner-v2", record), + ); + familyRecordWrite( + authority.installation, + authority.origin, + { scope: "shared", namespace: "lease-owner", container: leaseId, name: "owner.json" }, + ownerBytes, + (before) => ledger?.write("owner.json", ownerBytes, true, before), true, ); const state: State = { @@ -721,7 +855,7 @@ export async function acquireArtifactTargetLease(options: { if (kind === "history" && ledger) continue; // Partial records consume retained capacity too. const dir = index.directory(kind); try { - dir.removeDirectory(slot.name, slot.identity); + removeSlot(dir, slot); } finally { dir.close(); } @@ -910,8 +1044,11 @@ export async function delegateArtifactTargetLease( delegationId = fixedControlChildId(s.installation, child.childPublicKey, s.execution); } else if (child.childPublicKey !== s.installation.publicKeys[child.recipient]) throw Error("Child role key mismatch"); + if (!s.certificate.origin) throw Error("Legacy lease is observation-only; upgrade producer"); + assertOriginClockForward(s.installation, s.certificate.origin); const grant: Grant = { - version: 1, + version: 2, + origin: s.certificate.origin, installationId: s.certificate.installationId, leaseId: lease.leaseId, ownerCertificateDigest: controlDigest(s.certificateEnvelope), @@ -928,7 +1065,7 @@ export async function delegateArtifactTargetLease( permittedFurtherEdges: edges, depth: (parentGrant?.depth ?? 0) + 1, }; - const envelope = signControl(s.key, "openmapx-lease-grant-v1", grant); + const envelope = signControl(s.key, "openmapx-lease-grant-v2", grant); const grantDigest = controlDigest(envelope); const transition = signControl(s.key, "openmapx-lease-intent-v1", { delegationId, @@ -956,12 +1093,20 @@ export async function delegateArtifactTargetLease( "base64url", ) as ArtifactDelegationAssertion; parseControlAssertion(assertion); + enlistFamilyIntent( + s.installation, + s.certificate.origin, + delegationId, + child.execution, + child.recipient === "data-manager" ? "data-manager" : "ops-agent", + ); writePrivateRecord( s.installation, "attempts", `${delegationId}.json`, { grant: envelope, transition, assertion }, true, + s.certificate.origin, ); writeBounded(s, "grants", `${delegationId}.json`, { grant: envelope, transition }, true); node.grantDigests[delegationId] = grantDigest; @@ -989,11 +1134,12 @@ function openState( const cert = certificateSchema.parse(envelope.payload); const certificate = verifyControl( installation.publicKeys[cert.owner.executor], - "openmapx-lease-certificate-v1", + originRecordDomain("openmapx-lease-certificate", envelope), envelope, certificateSchema, ); if ( + !validateOriginRecord(certificate) || certificate.installationId !== installation.installation.installationId || certificate.leaseId !== leaseId ) @@ -1118,6 +1264,11 @@ export async function adoptArtifactTargetLease(options: { (!forwardAncestry(t, grant.parentNode) || Date.now() >= grant.child.deadlineAt) ) throw Error("Stale or canceled delegation"); + if (!retained) { + if (!grant.origin) throw Error("Legacy lease grant is observation-only; upgrade producer"); + assertOriginClockForward(peer.installation, grant.origin); + reserveControlFamily(peer.installation, grant.origin); + } const admissionName = `${grant.delegationId}.json`; const pending = { version: 1, @@ -1145,7 +1296,15 @@ export async function adoptArtifactTargetLease(options: { controlDigest(old.execution) !== controlDigest(grant.child) ) throw Error("Durable duplicate admission conflict"); - } else writePrivateRecord(peer.installation, "adoptions", admissionName, pending, true); + } else + writePrivateRecord( + peer.installation, + "adoptions", + admissionName, + pending, + true, + grant.origin, + ); s.key = peer.childPrivateKey; s.node = grant.delegationId; s.execution = grant.child; @@ -1154,7 +1313,9 @@ export async function adoptArtifactTargetLease(options: { s, "children", `${s.node}.json`, - signControl(s.key, "openmapx-lease-child-v1", { + signControl(s.key, "openmapx-lease-child-v2", { + version: 2, + origin: grant.origin, delegationId: s.node, grantDigest: assertion.grantDigest, execution: grant.child, @@ -1165,10 +1326,17 @@ export async function adoptArtifactTargetLease(options: { } satisfies ChildRecord), true, ); - writePrivateRecord(peer.installation, "adoptions", admissionName, { - ...pending, - state: "ready", - }); + writePrivateRecord( + peer.installation, + "adoptions", + admissionName, + { + ...pending, + state: "ready", + }, + false, + grant.origin, + ); tree(s); const handle = Object.freeze({ leaseId: assertion.leaseId, @@ -1194,6 +1362,7 @@ export function bindBorrowedControlExecution( a = executionAuthorityState(authority); if ( controlDigest(a.execution) !== controlDigest(lease.execution) || + !sameControlOrigin(a.origin, s.certificate.origin) || a.installation.installation.installationId !== s.certificate.installationId ) throw Error("Borrower execution authority mismatch"); @@ -1229,25 +1398,140 @@ export async function settleArtifactTargetLeaseChild( tree(s); }); } +function finishOwnedRelease(s: State): void { + guarded( + s, + () => { + const t = tree(s); + if (t.root.phase !== "settled" || !knownOutcome(t.root.settlement)) + throw Error("Durable own-effect settlement is required"); + for (const id of t.grants.keys()) { + const child = t.children.get(id); + if (child?.phase !== "settled" || !knownOutcome(child.settlement)) + throw Error("Every attempted descendant must settle"); + } + if (!s.slots) throw Error("Missing admission slots"); + const persist = () => + writeBounded( + s, + "owner", + "admission.json", + signControl(s.installation.privateKey, "openmapx-admission-slots-v1", s.slots), + ); + const release = s.slots.release ?? { phase: "planned" as const, locks: [], active: false }; + s.slots.release = release; + if ( + release.locks.some((id) => !s.certificate.serviceIds.includes(id)) || + (release.pending && !s.certificate.serviceIds.includes(release.pending)) + ) + throw Error("Invalid release journal target"); + for (const id of s.certificate.serviceIds) { + if (release.locks.includes(id)) continue; + if (release.pending && release.pending !== id) + throw Error("Conflicting physical release step"); + if (release.pending !== id) { + const lock = s.locks.directory(id); + try { + if ( + !sameInode( + identity(lock.stat), + required(s.certificate.lockIdentities[id], "Missing certified inode"), + ) || + lock.hasEntries() + ) + throw Error("Changed unplanned target release"); + } finally { + lock.close(); + } + release.pending = id; + persist(); + } + try { + s.locks.removeDirectory( + id, + required(s.certificate.lockIdentities[id], "Missing certified target inode"), + ); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + release.locks.push(id); + delete release.pending; + persist(); + } + if (!release.active) { + const previouslyPlanned = release.activePlanned === true; + const index = s.control.directory("admission-index-v1"), + active = index.directory("active"); + try { + if (!previouslyPlanned) { + const current = active.directory(s.slots.active.name); + try { + if (!sameInode(identity(current.stat), s.slots.active.identity)) + throw Error("Replaced unplanned active slot"); + if ( + s.slots.active.generation && + z.strictObject({ generation: uuid }).parse(json(current, "generation.json", 1024)) + .generation !== s.slots.active.generation + ) + throw Error("Reused unplanned active slot"); + } finally { + current.close(); + } + release.activePlanned = true; + persist(); + } + let slot: AnchoredDirectory | undefined; + try { + slot = active.directory(s.slots.active.name); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + if (slot) { + try { + if (!sameInode(identity(slot.stat), s.slots.active.identity)) + throw Error("Replaced active slot inode"); + if (s.slots.active.generation) { + try { + const g = z + .strictObject({ generation: uuid }) + .parse(json(slot, "generation.json", 1024)); + if (g.generation !== s.slots.active.generation) + throw Error("Reused active slot generation"); + slot.removeFile("generation.json"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + } + } finally { + slot.close(); + } + active.removeDirectory(s.slots.active.name, s.slots.active.identity); + } + } finally { + active.close(); + index.close(); + } + release.active = true; + persist(); + } + release.phase = "released"; + s.slots.releasedAt ??= Date.now(); + persist(); + if (s.certificate.origin) + markFamilyLeaseReleased( + s.installation, + s.certificate.origin, + s.certificate.leaseId, + controlDigest(s.certificateEnvelope), + ); + }, + !s.slots?.release, + ); +} export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Promise { const s = stateOf(lease); if (s.node !== null) throw Error("Borrower cannot release root"); - await guarded(s, () => { - const t = tree(s); - if (t.root.phase !== "settled" || !knownOutcome(t.root.settlement)) - throw Error("Durable own-effect settlement is required"); - for (const id of t.grants.keys()) { - const child = t.children.get(id); - if (child?.phase !== "settled" || !knownOutcome(child.settlement)) - throw Error("Every attempted descendant must settle"); - } - }); - for (const id of s.certificate.serviceIds) - s.locks.removeDirectory( - id, - required(s.certificate.lockIdentities[id], "Missing certified target inode"), - ); - releaseActiveSlot(s); + finishOwnedRelease(s); closeState(s); } export async function reconcileArtifactTargetLease(options: { @@ -1256,7 +1540,7 @@ export async function reconcileArtifactTargetLease(options: { authority: VerifiedRecoveryAuthority; }): Promise<"released" | "fenced"> { const recovery = recoveryAuthorityState(options.authority); - const s = openState(options.dataRoot, options.leaseId, recovery.authority.installation); + const s = openState(options.dataRoot, options.leaseId, recovery.authority.installation, true); try { if (controlDigest(s.execution) !== controlDigest(recovery.authority.execution)) throw Error("Foreign recovery execution"); @@ -1286,6 +1570,10 @@ export async function reconcileArtifactTargetLease(options: { ) ) return "fenced"; + if (s.slots?.release) { + finishOwnedRelease(s); + return "released"; + } await guarded(s, () => { const root = readRoot(s); root.settlement = recovery.evidence; @@ -1393,6 +1681,7 @@ export function inspectArtifactDelegation(options: { throw Error("Grant digest mismatch"); return Object.freeze({ leaseId: parsed.leaseId, + origin: grant.origin, parent: grant.parent, execution: grant.child, kind: grant.childKind, @@ -1457,6 +1746,9 @@ export function verifyConfigurationHelperReady(lease: BorrowedArtifactTargetLeas /** Observation/adoption of an old identity is not permission to start another forward effect. */ export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): void { const s = stateOf(lease); + if (!s.certificate.origin) + throw Error("Legacy lease is observation/settlement only; upgrade producer"); + assertOriginClockForward(s.installation, s.certificate.origin); if (s.verifiedAuthority) assertControlExecutionForward(s.verifiedAuthority); const t = tree(s); const node = s.node === null ? t.root : t.children.get(s.node); diff --git a/packages/core/src/build-artifacts/test-fixtures/authority.ts b/packages/core/src/build-artifacts/test-fixtures/authority.ts new file mode 100644 index 000000000..93c81ae07 --- /dev/null +++ b/packages/core/src/build-artifacts/test-fixtures/authority.ts @@ -0,0 +1,57 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { createControlRootOperationKey } from "../../ops/control-root-key"; +import { type ControlIssuer, executionFingerprint } from "../../ops/execution-identity"; +import { opsOperationFingerprint } from "../../ops/fingerprint"; +import * as c from "../control-authority"; + +export function admittedRoot(root: string, role: ControlIssuer = "host-cli", lifetime = 60000) { + const installation = c.openControlInstallation({ + infraDir: root, + role, + serviceIds: ["motis", "osrm"], + }); + const operation = + role === "data-manager" + ? { kind: "transit-sync" as const } + : { kind: "service.build" as const, serviceId: "osrm" }; + const hash = + operation.kind === "transit-sync" + ? c.controlDigest(operation) + : opsOperationFingerprint(operation); + const admittedAt = Date.now(), + deadlineAt = admittedAt + lifetime; + const execution = { + executor: role, + principal: role === "ops-agent" ? ("api" as const) : role, + executionId: role === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint: hash, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: hash, + deadlineAt, + }), + }; + try { + const authority = c.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["osrm"], + }); + return { installation, execution, authority, operation }; + } catch (error) { + c.closeControlInstallation(installation); + throw error; + } +} +export function terminalWithoutEffect(authority: c.VerifiedExecutionAuthority) { + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); +} diff --git a/packages/core/src/build-artifacts/test-fixtures/peer-process.ts b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts index c1222ba3d..c8180f4b9 100644 --- a/packages/core/src/build-artifacts/test-fixtures/peer-process.ts +++ b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts @@ -9,7 +9,98 @@ import * as l from "../target-lease"; for await (const line of createInterface({ input: process.stdin })) { try { const input = JSON.parse(line); - if (input.mode === "observe-interleaving") { + if (input.mode === "retention") { + if (input.now) Date.now = () => input.now; + if (input.assertPrivateRole) { + const read = AnchoredDirectory.prototype.read; + AnchoredDirectory.prototype.read = function (name, max, privateMode) { + const role = this.path.match( + /control-private\/(ops-agent|data-manager|host-cli)(?:\/|$)/, + )?.[1]; + if (role && role !== (input.role ?? "host-cli")) + throw Error("Sibling private read forbidden"); + return read.call(this, name, max, privateMode); + }; + } + const installation = c.openControlInstallation({ + infraDir: input.infraDir, + role: input.role ?? "host-cli", + serviceIds: ["motis", "osrm"], + }); + const write = AnchoredDirectory.prototype.write; + let injected = false; + AnchoredDirectory.prototype.write = function (name, bytes, exclusive, beforePublish) { + write.call(this, name, bytes, exclusive, beforePublish); + if ( + input.fault && + !injected && + (name === ".retention-transaction" || name === "retention-slot-transaction.json") + ) { + const value = JSON.parse(bytes.toString()); + if ( + (name === ".retention-transaction" && value?.phase === input.fault) || + (name === "retention-slot-transaction.json" && + input.fault === "slot_planned" && + value?.phase === "planned") + ) { + injected = true; + if (input.kill) { + process.stdout.write(`${JSON.stringify({ ready: true })}\n`); + while (true) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000); + } + throw Error("Contained synthetic fault after durable quota phase"); + } + } + }; + const result = await c.reconcileControlRetention(installation, { + maxFamilies: input.maxFamilies ?? 1, + }); + c.closeControlInstallation(installation); + process.stdout.write(`${JSON.stringify({ ok: true, ...result, injected })}\n`); + } else if (input.mode === "deadline-lease") { + const installation = c.openControlInstallation({ + infraDir: input.infraDir, + role: input.role, + serviceIds: ["motis", "osrm"], + }); + const peer = c.verifyControlPeer(installation, { + rawHeaders: ["x-openmapx-parent-execution", input.assertion], + transportPrincipal: "shared-data-manager-bearer", + execution: input.execution, + kind: input.operation.kind, + }); + const parent = c.verifyParentExecution(input.assertion, peer, { + execution: input.execution, + kind: input.operation.kind, + requestedDeadlineAt: input.execution.deadlineAt, + }); + const authority = c.retainAdmittedControlExecution(installation, { + execution: input.execution, + operation: input.operation, + kind: input.operation.kind, + admittedAt: input.admittedAt, + parent, + serviceIds: ["osrm"], + }); + const lease = await l.acquireArtifactTargetLease({ + dataRoot: join(input.infraDir, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await l.settleArtifactTargetLeaseChild( + lease, + c.verifyControlEffectSettlement(installation, input.execution), + ); + await l.releaseArtifactTargetLease(lease); + const origin = c.retainedExecutionOrigin(c.installationState(installation), input.execution); + c.closeControlInstallation(installation); + process.stdout.write(`${JSON.stringify({ ok: true, leaseId: lease.leaseId, origin })}\n`); + } else if (input.mode === "observe-interleaving") { const installation = c.openControlInstallation({ infraDir: input.infraDir, role: "data-manager", @@ -103,6 +194,7 @@ for await (const line of createInterface({ input: process.stdin })) { serviceIds: intent.serviceIds, }); const authority = c.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: input.execution.deadlineAt, execution: input.execution, kind: intent.operation.kind, operation: intent.operation, @@ -181,6 +273,7 @@ for await (const line of createInterface({ input: process.stdin })) { const lease = await admission.adopt(input.execution); if (input.forward) l.assertArtifactTargetLeaseForward(lease); const authority = c.retainAdmittedControlExecution(installation, { + requestedDeadlineAt: input.execution.deadlineAt, execution: input.execution, kind: input.kind, operation: input.operation, diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 9869b9643..d870d701b 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -30,6 +30,14 @@ function request(operation: unknown, extra: Record = {}) { } const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { + "controlRetention.acknowledge": { + kind: "controlRetention.acknowledge", + originRootKey: "opk1_abcdefghijklmnop", + operationKey: "opk1_abcdefghijklmnop", + fingerprint: "a".repeat(64), + terminalDigest: "b".repeat(64), + proof: "test", + }, "docker.status": { kind: "docker.status" }, "stack.status": { kind: "stack.status" }, "configuration.inspect": { kind: "configuration.inspect" }, diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 42004da47..fdd79e125 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -14,6 +14,7 @@ export const OPS_MAX_BACKUP_ID_LENGTH = 128; export const OPS_MAX_BACKUP_INVENTORY_ENTRIES = 128; export const OPS_OPERATION_KINDS = [ + "controlRetention.acknowledge", "docker.status", "stack.status", "stack.render", @@ -130,6 +131,7 @@ const asyncEffect = ( const LIFECYCLE_TIMEOUT_MS = 10 * 60_000 + 30_000; export const OPS_KIND_POLICIES = { + "controlRetention.acknowledge": sync("api", 15_000), "docker.status": sync("api", 5_000), "stack.status": sync("api", 15_000, 256 * 1024), "configuration.inspect": sync("api", 15_000), @@ -204,6 +206,13 @@ export const OPS_KIND_POLICIES = { "gbfsCatalogLock.inspect": sync("data-manager", 30_000), } as const satisfies Record; +/** Read streams/diagnostics do not acquire control-root authority or a retention dependency. */ +export function opsKindRequiresControlAuthority(kind: OpsOperationKind): boolean { + return ( + OPS_KIND_POLICIES[kind].execution === "async" && + !["service.logs.follow", "system.diagnostics", "serviceRepository.inspect"].includes(kind) + ); +} const operationKindSchema = z.enum(OPS_OPERATION_KINDS); const serviceIdSchema = z .string() @@ -289,6 +298,14 @@ const serviceOperation = (kind: T) => export const configurationGenerationRevisionSchema = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); export const opsOperationSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("controlRetention.acknowledge"), + originRootKey: operationKeySchema, + operationKey: operationKeySchema, + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + terminalDigest: z.string().regex(/^[a-f0-9]{64}$/), + proof: z.string().min(1).max(4096), + }), kindOnly("docker.status"), kindOnly("stack.status"), kindOnly("configuration.inspect"), @@ -519,6 +536,7 @@ export type OpsErrorClass = | "conflict" | "not_found" | "not_wired" + | "control_upgrade_required" | "recovery_required" | "host_update_required" | "runtime"; @@ -534,6 +552,8 @@ export const OPS_PUBLIC_ERROR_MESSAGES: Record = { conflict: "Operation key conflicts with an existing operation", not_found: "Operation was not found", not_wired: "Operation is not available", + control_upgrade_required: + "Update the API or CLI client to persist a timed control root before submitting this mutation.", recovery_required: "Operation requires an explicit retry", host_update_required: "Host update required: the operations agent or its backup helper could not be updated from the admin panel. Run `pnpm openmapx compose release`, then `pnpm openmapx services update app-api app-web data-manager ops-agent transitous-runner` on the host.", @@ -867,6 +887,7 @@ const pinnedImageReference = z .regex(/^[a-z0-9][a-z0-9._:/-]*@sha256:[a-f0-9]{64}$/); export const OPS_RESULT_SCHEMAS = { + "controlRetention.acknowledge": z.strictObject({ acknowledged: z.literal(true) }), "docker.status": z.strictObject({ reachable: z.boolean(), version: z.string().max(128).optional(), diff --git a/packages/core/src/ops/control-retention-receipt.test.ts b/packages/core/src/ops/control-retention-receipt.test.ts new file mode 100644 index 000000000..0650e6bad --- /dev/null +++ b/packages/core/src/ops/control-retention-receipt.test.ts @@ -0,0 +1,58 @@ +import { randomUUID } from "node:crypto"; +import { expect, it } from "vitest"; +import type { OpsJobStatus } from "./contract"; +import { + controlTerminalReceiptDigest, + signApiRetentionProof, + verifyApiRetentionProof, +} from "./control-retention-receipt"; + +const status: OpsJobStatus = { + version: 2, + operationId: "job1_abcdefghijklmnop", + operationKey: "opk1_abcdefghijklmnop", + kind: "backup.delete", + resourceId: "daily", + state: "succeeded", + result: { backupId: "daily" }, + admittedAt: 1, + deadlineAt: 60001, + fingerprint: "a".repeat(64), + submittedAt: "2026-10-25T00:00:00.000Z", + updatedAt: "2026-10-25T00:00:01.000Z", +}; +it("binds exact safe terminal evidence while excluding observation-only timestamps", () => { + expect(controlTerminalReceiptDigest({ ...status, updatedAt: "2026-10-26T00:00:00.000Z" })).toBe( + controlTerminalReceiptDigest(status), + ); + expect(controlTerminalReceiptDigest({ ...status, result: { backupId: "different" } })).not.toBe( + controlTerminalReceiptDigest(status), + ); + expect(() => + controlTerminalReceiptDigest({ ...status, state: "running", result: undefined }), + ).toThrow(); + expect(() => + controlTerminalReceiptDigest({ + ...status, + result: { backupId: "daily", unexpected: "secret" }, + }), + ).toThrow(); + expect(() => controlTerminalReceiptDigest({ ...status, version: 1 })).toThrow(); +}); +it("requires the fixed token, canonical fields and distinct acknowledgement domain", () => { + const fields = { + version: 1 as const, + apiJobId: randomUUID(), + originalApiOwnerVersion: 1, + attemptId: randomUUID(), + checkpointOwnerVersion: 2, + terminalCopyId: randomUUID(), + operationKey: status.operationKey, + fingerprint: "a".repeat(64), + terminalDigest: controlTerminalReceiptDigest(status), + }; + const proof = signApiRetentionProof("token", fields); + expect(verifyApiRetentionProof("token", proof)).toEqual(fields); + expect(() => verifyApiRetentionProof("wrong", proof)).toThrow(); + expect(() => verifyApiRetentionProof("token", `${proof}=`)).toThrow(); +}); diff --git a/packages/core/src/ops/control-retention-receipt.ts b/packages/core/src/ops/control-retention-receipt.ts new file mode 100644 index 000000000..d3bcba1ac --- /dev/null +++ b/packages/core/src/ops/control-retention-receipt.ts @@ -0,0 +1,73 @@ +/** Server-capable receipt digests and the existing fixed API token proof. */ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { z } from "zod"; +import { type OpsJobStatus, parseBoundedOpsResult, parseOpsJobStatus } from "./contract"; +import { canonicalJson } from "./fingerprint"; + +export function controlTerminalReceiptDigest(input: OpsJobStatus): string { + const s = parseOpsJobStatus(input); + if (s.version !== 2 || !["succeeded", "failed", "timed_out"].includes(s.state)) + throw Error("A version2 terminal receipt is required"); + const result = s.result === undefined ? null : parseBoundedOpsResult(s.kind, s.result); + const fields = { + version: 2, + operationId: s.operationId, + operationKey: s.operationKey, + kind: s.kind, + admittedAt: s.admittedAt, + deadlineAt: s.deadlineAt, + fingerprint: s.fingerprint, + state: s.state, + terminationRequestedAt: s.terminationRequestedAt ?? null, + errorClass: s.errorClass ?? null, + result, + failureResult: s.failureResult ?? null, + }; + if (Buffer.byteLength(canonicalJson(fields)) > 128 * 1024) + throw Error("Terminal receipt exceeds retention capacity"); + return createHash("sha256") + .update("openmapx-control-terminal-receipt-v1\0") + .update(canonicalJson(fields)) + .digest("hex"); +} +export const apiRetentionProofSchema = z + .strictObject({ + version: z.literal(1), + apiJobId: z.uuid(), + originalApiOwnerVersion: z.number().int().positive().safe(), + attemptId: z.uuid(), + checkpointOwnerVersion: z.number().int().positive().safe(), + terminalCopyId: z.uuid(), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + terminalDigest: z.string().regex(/^[a-f0-9]{64}$/), + }) + .refine( + (v) => v.checkpointOwnerVersion >= v.originalApiOwnerVersion, + "Receipt owner version predates its original attempt", + ); +export type ApiRetentionProof = z.infer; +export function signApiRetentionProof(token: string, input: ApiRetentionProof): string { + const payload = Buffer.from(canonicalJson(apiRetentionProofSchema.parse(input))).toString( + "base64url", + ); + return `${payload}.${createHmac("sha256", token).update("openmapx-api-retention-ack-v1\0").update(payload).digest("base64url")}`; +} +export function verifyApiRetentionProof(token: string, proof: string): ApiRetentionProof { + if (proof.length > 4096 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{43}$/.test(proof)) + throw Error("Invalid API retention proof"); + const [payload, signature] = proof.split(".") as [string, string]; + const expected = createHmac("sha256", token) + .update("openmapx-api-retention-ack-v1\0") + .update(payload) + .digest(), + supplied = Buffer.from(signature, "base64url"); + if (supplied.length !== expected.length || !timingSafeEqual(expected, supplied)) + throw Error("Invalid API retention proof"); + const fields = apiRetentionProofSchema.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.from(payload, "base64url"))), + ); + if (Buffer.from(canonicalJson(fields)).toString("base64url") !== payload) + throw Error("Noncanonical API retention proof"); + return fields; +} diff --git a/packages/core/src/ops/control-root-key.test.ts b/packages/core/src/ops/control-root-key.test.ts new file mode 100644 index 000000000..36dfa1f4e --- /dev/null +++ b/packages/core/src/ops/control-root-key.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { createControlRootOperationKey, parseControlRootOperationKey } from "./control-root-key"; + +describe("self-describing control root keys", () => { + it("round trips an exact canonical expiry and 128-bit nonce", () => { + const key = createControlRootOperationKey({ notAfter: 1, nonce: new Uint8Array(16) }); + expect(key).toBe("opk1_t1_00000000000001_AAAAAAAAAAAAAAAAAAAAAA"); + expect(parseControlRootOperationKey(key)).toEqual({ + notAfter: 1, + nonce: "AAAAAAAAAAAAAAAAAAAAAA", + }); + expect(key).toMatch(/^opk1_[A-Za-z0-9_-]{16,64}$/); + }); + it.each([0, -1, Infinity, NaN, 1.5, Number.MAX_SAFE_INTEGER + 1])( + "rejects invalid expiry %s", + (notAfter) => { + expect(() => + createControlRootOperationKey({ notAfter, nonce: new Uint8Array(16) }), + ).toThrow(); + }, + ); + it.each([0, 15, 17])("requires exactly sixteen nonce bytes (%s)", (length) => { + expect(() => + createControlRootOperationKey({ notAfter: 1, nonce: new Uint8Array(length) }), + ).toThrow(); + }); + it.each([ + "opk1_t1_00000000000000_AAAAAAAAAAAAAAAAAAAAAA", + "opk1_t1_0000000000000A_AAAAAAAAAAAAAAAAAAAAAA", + "opk1_t1_00000000000001_AAAAAAAAAAAAAAAAAAAAAB", + "opk1_t1_00000000000001_AAAAAAAAAAAAAAAAAAAAAA=", + "opk1_t1_20000000000000_AAAAAAAAAAAAAAAAAAAAAA", + "opk1_abcdefghijklmnopqrstuvwxyz012345", + ])("rejects noncanonical or unsupported root %s", (key) => { + expect(() => parseControlRootOperationKey(key)).toThrow(); + }); +}); diff --git a/packages/core/src/ops/control-root-key.ts b/packages/core/src/ops/control-root-key.ts new file mode 100644 index 000000000..1ddf33f5d --- /dev/null +++ b/packages/core/src/ops/control-root-key.ts @@ -0,0 +1,43 @@ +/** Browser-safe format only. A timed key restricts validity; it grants no authority. */ +export interface ControlExecutionOrigin { + version: 1; + rootKey: string; + issuer: "ops-agent" | "data-manager" | "host-cli"; + notAfter: number; +} +function encode(bytes: Uint8Array): string { + return btoa(String.fromCharCode(...bytes)) + .replaceAll("+", "-") + .replaceAll("/", "_") + .replace(/=+$/, ""); +} +export function createControlRootOperationKey(input: { + notAfter: number; + nonce: Uint8Array; +}): string { + if ( + !Number.isSafeInteger(input.notAfter) || + input.notAfter <= 0 || + !(input.nonce instanceof Uint8Array) || + input.nonce.byteLength !== 16 + ) + throw Error("Invalid control root expiry or nonce"); + return `opk1_t1_${input.notAfter.toString(16).padStart(14, "0")}_${encode(input.nonce)}`; +} +export function parseControlRootOperationKey(key: string): { notAfter: number; nonce: string } { + const match = /^opk1_t1_([0-9a-f]{14})_([A-Za-z0-9_-]{22})$/.exec(key); + if (!match?.[1] || !match[2]) + throw Error("Unsupported control root key; upgrade the control producer"); + const notAfter = Number.parseInt(match[1], 16); + let bytes: Uint8Array; + try { + bytes = Uint8Array.from(atob(match[2].replaceAll("-", "+").replaceAll("_", "/")), (c) => + c.charCodeAt(0), + ); + } catch { + throw Error("Noncanonical control root nonce"); + } + if (createControlRootOperationKey({ notAfter, nonce: bytes }) !== key) + throw Error("Noncanonical control root key"); + return { notAfter, nonce: match[2] }; +} diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index 54bb89a67..a7a0ca7b7 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -1,5 +1,7 @@ export * from "./client"; export * from "./contract"; +export * from "./control-retention-receipt"; +export * from "./control-root-key"; export * from "./execution-identity"; export * from "./fingerprint"; export * from "./job-evidence"; diff --git a/packages/core/src/ops/verified-parent-internal.ts b/packages/core/src/ops/verified-parent-internal.ts index 31d93bb42..210b39375 100644 --- a/packages/core/src/ops/verified-parent-internal.ts +++ b/packages/core/src/ops/verified-parent-internal.ts @@ -1,3 +1,7 @@ // Internal mint registry shared only by the fixed API and role-signature verifiers. // Deliberately absent from every package export. export const verifiedParents = new WeakSet(); +export const verifiedParentOrigins = new WeakMap< + object, + import("./control-root-key").ControlExecutionOrigin +>(); diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 5ab7b6259..724518a51 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -38,6 +38,7 @@ "communityServices.inspect", "configuration.hardlinks.inspect", "configuration.inspect", + "controlRetention.acknowledge", "data.clean", "data.convertOverpass", "data.downloadFonts", @@ -144,6 +145,14 @@ ] }, "businessEffects": [ + { + "id": "api.control-retention.acknowledge", + "callSiteIds": [], + "semanticSources": [ + "packages/core/src/build-artifacts/control-authority.ts#acknowledgeControlRetention" + ], + "typedKinds": ["controlRetention.acknowledge"] + }, { "id": "api.admin-cli.data", "callSiteIds": [], @@ -300,6 +309,10 @@ } ], "mounts": [ + "ops-agent|@infra:control-private/empty|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/control-private/host-cli|true", + "ops-agent|@infra:control-private/hidden-key|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/secrets/control-host-cli.key|true", + "ops-agent|@infra:control-private/empty|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/control-private/data-manager|true", + "ops-agent|@infra:control-private/hidden-key|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/secrets/control-data-manager.key|true", "app-api|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set \u2014 community integrations are stored under the host checkout}/custom_integrations|${OPENMAPX_CUSTOM_INTEGRATIONS_DIR:-/app/custom_integrations}|false", "ops-agent|@docker-socket|/var/run/docker.sock|false", "ops-agent|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set \u2014 ops-agent renders and applies trusted local stack configuration}|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}|false", From d03beff9b94ddd0cf734fae4644f22156ba677e8 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 14:06:20 +0100 Subject: [PATCH 015/135] fix(ops): repair bounded release and terminal recovery --- .../src/control-terminal-restart.test.ts | 472 ++++++++++++++++++ apps/ops-agent/src/index.ts | 18 +- apps/ops-agent/src/server.ts | 126 +++-- .../terminal-retention-process.ts | 235 +++++++++ apps/ops-agent/src/trusted-config-source.ts | 38 ++ .../docs/developer/build-artifacts-control.md | 17 +- .../src/build-artifacts/control-authority.ts | 74 +++ packages/core/src/build-artifacts/index.ts | 1 + .../src/build-artifacts/retention-family.ts | 16 +- .../src/build-artifacts/target-lease.test.ts | 171 ++++++- .../core/src/build-artifacts/target-lease.ts | 89 +++- 11 files changed, 1183 insertions(+), 74 deletions(-) create mode 100644 apps/ops-agent/src/control-terminal-restart.test.ts create mode 100644 apps/ops-agent/src/test-fixtures/terminal-retention-process.ts diff --git a/apps/ops-agent/src/control-terminal-restart.test.ts b/apps/ops-agent/src/control-terminal-restart.test.ts new file mode 100644 index 000000000..5403eaa1b --- /dev/null +++ b/apps/ops-agent/src/control-terminal-restart.test.ts @@ -0,0 +1,472 @@ +import { spawn } from "node:child_process"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as c from "@openmapx/core/build-artifacts"; +import { + controlTerminalReceiptDigest, + createControlRootOperationKey, + type ExecutionRef, + executionFingerprint, + opsOperationFingerprint, + parseOpsJobStatus, + signApiRetentionProof, +} from "@openmapx/core/ops"; +import { build } from "esbuild"; +import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { openOpsJobJournal, type PersistedOpsJob } from "./journal"; +import { buildOpsAgentServer } from "./server"; +import { initializeTerminalClaimCleanup } from "./trusted-config-source"; + +let bundle: string, bundleRoot: string; +beforeAll(async () => { + bundleRoot = mkdtempSync(join(tmpdir(), "omx-terminal-process-")); + bundle = join(bundleRoot, "agent.mjs"); + await build({ + entryPoints: [join(import.meta.dirname, "test-fixtures/terminal-retention-process.ts")], + bundle: true, + platform: "node", + format: "esm", + target: "node24", + outfile: bundle, + banner: { + js: `import {createRequire} from "node:module"; const require=createRequire(${JSON.stringify(join(import.meta.dirname, "../package.json"))});`, + }, + }); +}); +afterAll(() => rmSync(bundleRoot, { recursive: true, force: true })); +interface Interrupted { + job: PersistedOpsJob; + apiJobId: string; + attemptId: string; +} +async function interrupt( + root: string, + boundary: string, + outcome: string, + borrowed?: { assertion: string; execution: ExecutionRef }, +): Promise { + const child = spawn(process.execPath, [bundle], { stdio: ["pipe", "pipe", "pipe"] }); + let stdout = "", + stderr = ""; + const exited = new Promise((resolve) => child.once("close", () => resolve())); + const stopped = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => + reject(Error(`Fixture exited ${code}/${signal}: ${stderr}`)), + ); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (stdout.includes("\n")) { + try { + resolve(JSON.parse(stdout.split("\n")[0] ?? "")); + } catch (error) { + reject(error); + } + } + }); + }); + const timeout = setTimeout(() => child.kill("SIGKILL"), 30000); + child.stdin.end(JSON.stringify({ root, boundary, outcome, borrowed })); + try { + return await stopped; + } finally { + clearTimeout(timeout); + child.kill("SIGKILL"); + await exited; + } +} +function progress(root: string) { + return JSON.parse( + readFileSync( + join(root, "control-private/ops-agent/retention-families/slot-0/progress.json"), + "utf8", + ), + ) as { + terminalAt: number; + executions: Record< + string, + { known: boolean; terminalAt: number; terminalDigest: string; journalContained: boolean } + >; + }; +} +const cases = [ + { boundary: "terminal", outcome: "success", known: true }, + { boundary: "digest", outcome: "success", known: true }, + { boundary: "terminal", outcome: "not-started", known: true }, + { boundary: "digest", outcome: "not-started", known: true }, + { boundary: "terminal", outcome: "unknown", known: false }, + { boundary: "terminal", outcome: "configuration", known: true }, + { boundary: "terminal", outcome: "success", known: false, tamper: "missing-proof" }, + { boundary: "terminal", outcome: "success", known: false, tamper: "mismatched-journal" }, + { boundary: "terminal", outcome: "success", known: false, tamper: "admission-time" }, +] as const; +it.each( + cases.map((scenario) => ({ + ...scenario, + label: "tamper" in scenario ? scenario.tamper : "original", + })), +)( + "repairs only proven terminal retention across actual restart ($boundary/$outcome/$label)", + async (scenario) => { + const { boundary, outcome, known } = scenario; + const root = mkdtempSync(join(tmpdir(), "omx-terminal-restart-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await c.provisionControlInstallation(root); + let app: ReturnType | undefined; + let installation: c.ControlInstallation | undefined; + const dispatch = vi.fn(async () => { + throw Error("Restart must never dispatch"); + }); + try { + const stopped = await interrupt(root, boundary, outcome), + original = stopped.job; + if ("tamper" in scenario) { + if (scenario.tamper === "missing-proof") + rmSync(join(root, `control-private/ops-agent/effects/${original.operationKey}.json`)); + else if (scenario.tamper === "admission-time") { + if (original.admittedAt === undefined) throw Error("Missing fixture admission timestamp"); + original.admittedAt += 1; + original.submittedAt = new Date(original.admittedAt).toISOString(); + writeFileSync( + join(root, "data/ops-agent/jobs-v1.json"), + JSON.stringify({ version: 2, jobs: [original] }), + ); + } else { + original.result = { backupId: "different-result" }; + const changedJournal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + await changedJournal.replace([original]); + } + } + expect( + existsSync( + join(root, "control-private/ops-agent/retention-families/slot-0/.family-writer"), + ), + ).toBe(false); + expect( + existsSync(join(root, `control-private/ops-agent/effect-writers/${original.operationKey}`)), + ).toBe(false); + if (!original.terminalAt) throw Error("Missing actual terminal journal timestamp"); + const initial = progress(root); + expect(Object.values(initial.executions)[0]?.journalContained).toBe(false); + const originalEligibility = initial.terminalAt; + const now = vi + .spyOn(Date, "now") + .mockReturnValue(Date.parse(original.terminalAt) + 71 * 3600_000); + const start = async () => { + installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const terminalClaimCleanup = + "revisionId" in original.operation + ? await initializeTerminalClaimCleanup(join(root, "trusted-config"), { + allowedUids: [process.getuid?.() ?? -1], + expectedGid: process.getgid?.() ?? -1, + token: Buffer.alloc(32, 7).toString("base64url"), + journalRecords: journal.records(), + }) + : undefined; + app = buildOpsAgentServer({ + terminalClaimCleanup, + tokens: { + api: Buffer.alloc(32, 7).toString("base64url"), + "data-manager": "b".repeat(32), + }, + controlInstallation: installation, + journal, + dispatch, + now: () => new Date(Date.now()), + resourcePolicy: { allowBackup: () => true }, + }); + await app.ready(); + expect(journal.records()).toEqual([original]); + return journal; + }; + if ("revisionId" in original.operation) { + installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const unproved = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const claimed = join(root, `trusted-config/.claimed/${original.operation.revisionId}.json`); + expect(existsSync(claimed)).toBe(true); + app = buildOpsAgentServer({ + tokens: { + api: Buffer.alloc(32, 7).toString("base64url"), + "data-manager": "b".repeat(32), + }, + controlInstallation: installation, + journal: unproved, + dispatch, + terminalClaimCleanup: Object.freeze( + {}, + ) as import("./trusted-config-source").VerifiedTerminalClaimCleanup, + }); + await app.ready(); + expect(Object.values(progress(root).executions)[0]?.journalContained).toBe(false); + expect(existsSync(claimed)).toBe(true); + await app.close(); + app = undefined; + c.closeControlInstallation(installation); + installation = undefined; + } + let journal = await start(); + if ("revisionId" in original.operation) + expect( + existsSync(join(root, `trusted-config/.claimed/${original.operation.revisionId}.json`)), + ).toBe(false); + const get = async () => { + if (!app) throw Error("Missing restarted server"); + const response = await app.inject({ + url: `/v1/operations/${original.operationId}`, + headers: { + authorization: `Bearer ${Buffer.alloc(32, 7).toString("base64url")}`, + "x-ops-protocol-version": "2", + }, + }); + expect(response.statusCode).toBe(200); + return parseOpsJobStatus(response.json().result); + }; + const status = await get(); + expect(status).toMatchObject({ + operationId: original.operationId, + operationKey: original.operationKey, + admittedAt: original.admittedAt, + deadlineAt: original.deadlineAt, + fingerprint: original.fingerprint, + state: original.state, + updatedAt: original.updatedAt, + }); + const digest = controlTerminalReceiptDigest(status); + if (!("tamper" in scenario)) + expect(Object.values(initial.executions)[0]?.terminalDigest === digest).toBe( + boundary === "digest", + ); + const ack = { + kind: "controlRetention.acknowledge", + originRootKey: original.operationKey, + operationKey: original.operationKey, + fingerprint: original.fingerprint, + terminalDigest: digest, + proof: signApiRetentionProof(Buffer.alloc(32, 7).toString("base64url"), { + version: 1, + apiJobId: stopped.apiJobId, + originalApiOwnerVersion: 1, + attemptId: stopped.attemptId, + checkpointOwnerVersion: 1, + terminalCopyId: randomUUID(), + operationKey: original.operationKey, + fingerprint: original.fingerprint, + terminalDigest: digest, + }), + }; + const acknowledge = async () => { + if (!app) throw Error("Missing restarted server"); + return app.inject({ + method: "POST", + url: "/v1/operations", + headers: { + authorization: `Bearer ${Buffer.alloc(32, 7).toString("base64url")}`, + "x-ops-protocol-version": "2", + }, + payload: { + version: 2, + requestId: `ops1_${randomUUID().replaceAll("-", "")}`, + operationKey: "opk1_ackrequest123456", + issuedAt: new Date(Date.now()).toISOString(), + expiresAt: new Date(Date.now() + 10000).toISOString(), + operation: ack, + }, + }); + }; + if (known) { + expect((await acknowledge()).statusCode).toBe(200); + const repaired = progress(root); + expect(Object.values(repaired.executions)[0]).toMatchObject({ + known: true, + terminalDigest: digest, + journalContained: true, + }); + expect(repaired.terminalAt).toBeLessThanOrEqual( + Math.max(originalEligibility, Date.parse(original.terminalAt)), + ); + await app?.close(); + app = undefined; + if (installation) c.closeControlInstallation(installation); + installation = undefined; + now.mockReturnValue(Date.parse(original.terminalAt) + 71.5 * 3600_000); + journal = await start(); + expect(await get()).toEqual(status); + expect(progress(root)).toEqual(repaired); + expect((await acknowledge()).statusCode).toBe(200); + now.mockReturnValue( + Math.max(originalEligibility, Date.parse(original.terminalAt)) + 73 * 3600_000, + ); + if (!installation) throw Error("Missing reopened installation"); + expect(await c.reconcileControlRetention(installation, { maxFamilies: 1 })).toEqual({ + retired: 1, + pending: 0, + }); + expect(journal.records()).toHaveLength(0); + expect(existsSync(join(root, "control-private/ops-agent/retention-families/slot-0"))).toBe( + false, + ); + } else { + expect((await acknowledge()).statusCode).toBe(409); + expect(Object.values(progress(root).executions)[0]?.journalContained).toBe(false); + now.mockReturnValue(Date.parse(original.terminalAt) + 96 * 3600_000); + if (!installation) throw Error("Missing reopened installation"); + expect(await c.reconcileControlRetention(installation, { maxFamilies: 1 })).toEqual({ + retired: 0, + pending: 1, + }); + expect(journal.records()).toEqual([original]); + } + expect(dispatch).not.toHaveBeenCalled(); + const effects = join(root, "effect-calls"); + expect( + existsSync(effects) ? readFileSync(effects, "utf8").split("\n").filter(Boolean).length : 0, + ).toBe(outcome === "not-started" ? 0 : 1); + } finally { + vi.restoreAllMocks(); + await app?.close(); + if (installation) c.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }, + 90000, +); + +it("repairs a borrowed terminal worker after the actual canonical-digest restart boundary without replay or releasing its parent", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-borrowed-terminal-restart-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await c.provisionControlInstallation(root); + const dm = c.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis"], + }); + let installation: c.ControlInstallation | undefined, + app: ReturnType | undefined; + const dispatch = vi.fn(async () => { + throw Error("Restart must never dispatch"); + }); + try { + const deadlineAt = Date.now() + 120000, + operation = { kind: "transit-sync" }; + const operationFingerprint = createHash("sha256") + .update(JSON.stringify(operation)) + .digest("hex"); + const owner: ExecutionRef = { + executor: "data-manager", + principal: "data-manager", + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + deadlineAt, + }; + const childFingerprint = opsOperationFingerprint({ kind: "motis.primary.stop" }); + const execution: ExecutionRef = { + executor: "ops-agent", + principal: "data-manager", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: c.controlChildOperationKey(owner.operationKey, "stop-primary", 0), + operationFingerprint: childFingerprint, + fingerprint: executionFingerprint({ + kind: "motis.primary.stop", + operationFingerprint: childFingerprint, + deadlineAt, + parent: c.retainedControlParent(owner), + }), + deadlineAt, + }; + const plan = { + phase: "stop-primary" as const, + ordinal: 0, + kind: "motis.primary.stop", + execution, + serviceIds: ["motis"], + recipient: "ops-agent" as const, + }; + const authority = c.retainAdmittedControlExecution(dm, { + execution: owner, + operation, + kind: operation.kind, + admittedAt: deadlineAt - 120000, + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis"], + plan: [plan], + }); + const lease = await c.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + const peer = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const assertion = await c.delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: peer.publicKey, + }); + c.closeControlInstallation(peer); + const stopped = await interrupt(root, "digest", "success", { assertion, execution }); + const initial = progress(root); + expect(Object.values(initial.executions)[0]?.journalContained).toBe(false); + installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + app = buildOpsAgentServer({ + tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32) }, + controlInstallation: installation, + journal, + dispatch, + resourcePolicy: { allowGlobal: () => true }, + }); + await app.ready(); + expect(journal.records()).toEqual([stopped.job]); + expect(Object.values(progress(root).executions)[0]).toMatchObject({ + journalContained: true, + known: true, + }); + expect(progress(root).terminalAt).toBe(initial.terminalAt); + expect(existsSync(join(root, "data/.admin-target-control/locks/motis"))).toBe(true); + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await c.settleArtifactTargetLeaseChild(lease, c.verifyControlEffectSettlement(dm, owner)); + await c.releaseArtifactTargetLease(lease); + vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 3600_000); + let retired = 0; + for (let n = 0; n < 12 && retired < 2; n++) { + retired += (await c.reconcileControlRetention(installation, { maxFamilies: 1 })).retired; + retired += (await c.reconcileControlRetention(dm, { maxFamilies: 1 })).retired; + } + expect(retired).toBe(2); + expect(journal.records()).toHaveLength(0); + expect(readFileSync(join(root, "effect-calls"), "utf8")).toBe("effect\n"); + expect(dispatch).not.toHaveBeenCalled(); + } finally { + vi.restoreAllMocks(); + await app?.close(); + if (installation) c.closeControlInstallation(installation); + c.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + } +}, 90000); diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 65ee93813..0c8e26dd2 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -46,7 +46,7 @@ import { } from "./trusted-config-runtime"; import { createFileTrustedOpsDataSource, - initializeTrustedSnapshotDirectory, + initializeTerminalClaimCleanup, } from "./trusted-config-source"; import { createTrustedConfigurationAuthorityLoader } from "./trusted-configuration-authority"; @@ -138,12 +138,15 @@ async function main(): Promise { container: existsSync("/run/secrets/openmapx-control-verifiers"), }); if (controlStartup.reason) console.error(controlStartup.reason); - await initializeTrustedSnapshotDirectory(config.trustedConfigDirectory, { - allowedUids: [uid], - expectedGid: gid, - token: apiToken, - journalRecords: journal.records(), - }); + const terminalClaimCleanup = await initializeTerminalClaimCleanup( + config.trustedConfigDirectory, + { + allowedUids: [uid], + expectedGid: gid, + token: apiToken, + journalRecords: journal.records(), + }, + ); await initializeTrustedConfigurationRuntime(paths.infraDir); const authorityLoader = createTrustedConfigurationAuthorityLoader({ ...authorityOptions, @@ -255,6 +258,7 @@ async function main(): Promise { }), journal, controlInstallation: controlStartup.installation, + terminalClaimCleanup, runtime, replacementInProgress: () => { try { diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index cbe19e3b6..48bb8f6b8 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -14,6 +14,7 @@ import { controlHeaders as readControlHeaders, reconcileControlRetention, registerControlRetentionJournal, + repairControlJournalTerminal, requireControlJournalRetention, restoreAdmittedControlExecution, retainAdmittedControlExecution, @@ -88,6 +89,11 @@ import { type OpsTrustedClaim, } from "./runtime"; +import { + type VerifiedTerminalClaimCleanup, + verifiesTerminalClaimCleanup, +} from "./trusted-config-source"; + const DEFAULT_BODY_LIMIT = 32 * 1024; const DEFAULT_CONCURRENCY = 4; const DEFAULT_REPLAY_ENTRIES = 10_000; @@ -144,6 +150,7 @@ export interface OpsAuditEvent { export interface BuildOpsAgentServerOptions { controlInstallation?: ControlInstallation; + terminalClaimCleanup?: VerifiedTerminalClaimCleanup; tokens: Record; releaseRecovery?: { ready: boolean; reason?: "recovery_required" }; resourcePolicy?: OpsResourcePolicy; @@ -1086,52 +1093,93 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }; }); - // No replay during restart. Rejoin only the original signed adoption and settle - // from an already retained own-effect receipt; absent evidence keeps its fence. + // No replay during restart. Rejoin original signed admissions/adoptions and + // retained own-effect proof; absent containment evidence keeps its fence. app.addHook("onReady", async () => { let changed = false; for (const job of jobsById.values()) { - if (!job.controlAdoption) continue; - try { - if (!options.controlInstallation) throw Error("Control trust unavailable"); - const execution = executionFor(job); - if (job.state === "succeeded") { - verifyRetainedArtifactTargetSettlement( + if (job.controlAdoption) { + try { + if (!options.controlInstallation) throw Error("Control trust unavailable"); + const execution = executionFor(job); + if (job.state === "succeeded") { + verifyRetainedArtifactTargetSettlement( + options.controlInstallation, + job.controlAdoption, + execution, + ); + } else { + const assertion = retainedControlAdoptionAssertion( + options.controlInstallation, + job.controlAdoption, + execution, + ); + const admission = inspectControlAdmission(options.controlInstallation, { + rawHeaders: ["x-openmapx-control-delegation", assertion], + transportPrincipal: "data-manager", + kind: job.operation.kind, + operation: job.operation, + operationKey: job.operationKey, + operationFingerprint: execution.operationFingerprint, + requestedDeadlineAt: execution.deadlineAt, + }); + const lease = await admission.adopt(execution); + const authority = restoreAdmittedControlExecution( + options.controlInstallation, + execution, + ); + controlLeases.set(job.operationId, lease); + controlAuthorities.set(job.operationId, authority); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(options.controlInstallation, execution), + ); + } + } catch { + job.controlAdoption = { ...job.controlAdoption, phase: "recovery_required" }; + job.state = "failed"; + job.errorClass = "recovery_required"; + job.result = undefined; + stickyRecoveryRequired.add(job.operationId); + changed = true; + } + } + if ( + job.executionVersion === 2 && + opsKindRequiresControlAuthority(job.operation.kind) && + ["succeeded", "failed", "timed_out"].includes(job.state) + ) { + try { + if (!options.controlInstallation || job.terminalAt === undefined) + throw Error("Original terminal authority unavailable"); + if ( + "revisionId" in job.operation && + !verifiesTerminalClaimCleanup( + options.terminalClaimCleanup, + job.operation, + new Date(job.terminalAt).toISOString(), + ) + ) + throw Error("Terminal claim cleanup is not proved"); + const execution = executionFor(job); + if (job.controlAdoption) + verifyRetainedArtifactTargetSettlement( + options.controlInstallation, + job.controlAdoption, + execution, + ); + const authority = repairControlJournalTerminal( options.controlInstallation, - job.controlAdoption, execution, + statusFor(job, 2), + job.terminalAt, ); - continue; + controlAuthorities.set(job.operationId, authority); + } catch { + // The loaded terminal state, elapsed time and absence of a PID never + // prove containment. Keep its exact outcome and original time intact. + stickyRecoveryRequired.add(job.operationId); } - const assertion = retainedControlAdoptionAssertion( - options.controlInstallation, - job.controlAdoption, - execution, - ); - const admission = inspectControlAdmission(options.controlInstallation, { - rawHeaders: ["x-openmapx-control-delegation", assertion], - transportPrincipal: "data-manager", - kind: job.operation.kind, - operation: job.operation, - operationKey: job.operationKey, - operationFingerprint: execution.operationFingerprint, - requestedDeadlineAt: execution.deadlineAt, - }); - const lease = await admission.adopt(execution); - const authority = restoreAdmittedControlExecution(options.controlInstallation, execution); - controlLeases.set(job.operationId, lease); - controlAuthorities.set(job.operationId, authority); - await settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(options.controlInstallation, execution), - ); - } catch { - job.controlAdoption = { ...job.controlAdoption, phase: "recovery_required" }; - job.state = "failed"; - job.errorClass = "recovery_required"; - job.result = undefined; - stickyRecoveryRequired.add(job.operationId); - changed = true; } } if (changed) await journal.replace([...jobsById.values()].map(persisted)); diff --git a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts new file mode 100644 index 000000000..8fc303d94 --- /dev/null +++ b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts @@ -0,0 +1,235 @@ +import { createHmac, randomBytes, randomUUID } from "node:crypto"; +import { appendFileSync, mkdirSync, writeFileSync, writeSync } from "node:fs"; +import { join } from "node:path"; +import * as c from "@openmapx/core/build-artifacts"; +import { + controlTerminalReceiptDigest, + createControlRootOperationKey, + type ExecutionRef, + type OpsJobStatus, + type OpsOperation, + opsOperationFingerprint, + sealTrustedConfigurationSnapshot, +} from "@openmapx/core/ops"; +import { AnchoredDirectory } from "../../../../packages/core/src/build-artifacts/filesystem"; +import { openOpsJobJournal, type PersistedOpsJob } from "../journal"; +import { createRegistryResourceClaimer } from "../policy"; +import { buildOpsAgentServer } from "../server"; +import { + createFileTrustedOpsDataSource, + initializeTrustedSnapshotDirectory, +} from "../trusted-config-source"; + +let inputText = ""; +for await (const chunk of process.stdin) inputText += chunk; +const input = JSON.parse(inputText) as { + root: string; + boundary: "terminal" | "digest"; + outcome: "success" | "unknown" | "not-started" | "configuration"; + borrowed?: { assertion: string; execution: ExecutionRef }; +}; +const token = Buffer.alloc(32, 7).toString("base64url"), + notAfter = input.borrowed?.execution.deadlineAt ?? Date.now() + 120000; +const key = + input.borrowed?.execution.operationKey ?? + createControlRootOperationKey({ notAfter, nonce: randomBytes(16) }); +let operation: OpsOperation = input.borrowed + ? { kind: "motis.primary.stop" } + : { kind: "backup.delete", backupId: "daily" }; +let resourceClaimer: ReturnType | undefined; +if (input.outcome === "configuration") { + const directory = join(input.root, "trusted-config"); + mkdirSync(directory, { mode: 0o700 }); + const policy = { + allowedUids: [process.getuid?.() ?? -1], + expectedGid: process.getgid?.() ?? -1, + token, + }; + await initializeTrustedSnapshotDirectory(directory, policy); + const snapshot = sealTrustedConfigurationSnapshot({ + role: "api", + operationKey: key, + token, + issuedAtMs: Date.now(), + nonce: "nonce_0123456789abcdef", + operationForRevision: (revisionId) => ({ + kind: "stack.render", + revisionId, + expectedCurrentRevisionId: null, + }), + payload: { + domain: "maps.example.test", + selectedRoots: ["motis"], + serviceConfigs: [{ serviceId: "motis", values: {} }], + integrationConfigs: [], + serviceSecrets: [], + }, + }); + operation = { + kind: "stack.render", + revisionId: snapshot.revisionId, + expectedCurrentRevisionId: null, + }; + writeFileSync(join(directory, `${snapshot.revisionId}.json`), snapshot.bytes, { mode: 0o600 }); + resourceClaimer = createRegistryResourceClaimer({ + serviceIds: ["motis"], + integrationIds: [], + trustedData: createFileTrustedOpsDataSource({ directory, ...policy }), + }); +} +const apiJobId = randomUUID(), + attemptId = randomUUID(); +const installation = c.openControlInstallation({ + infraDir: input.root, + role: "ops-agent", + serviceIds: ["motis"], +}); +const journal = await openOpsJobJournal(join(input.root, "data/ops-agent/jobs-v1.json")); +let terminal: PersistedOpsJob | undefined; +function wire(job: PersistedOpsJob): OpsJobStatus { + return { + version: 2, + operationId: job.operationId, + operationKey: job.operationKey, + kind: job.operation.kind, + resourceId: job.resourceId, + state: job.state, + submittedAt: job.submittedAt, + updatedAt: job.updatedAt, + admittedAt: job.admittedAt, + deadlineAt: job.deadlineAt, + fingerprint: job.fingerprint, + ...(job.result === undefined ? {} : { result: job.result }), + ...(job.errorClass === undefined ? {} : { errorClass: job.errorClass }), + ...(job.failureResult === undefined ? {} : { failureResult: job.failureResult }), + ...(job.terminationRequestedAt === undefined + ? {} + : { terminationRequestedAt: job.terminationRequestedAt }), + } as OpsJobStatus; +} +function stop(job: PersistedOpsJob): never { + writeSync(1, `${JSON.stringify({ job, apiJobId, attemptId })}\n`); + process.kill(process.pid, "SIGSTOP"); + throw Error("Stopped fixture resumed unexpectedly"); +} +// This is an actual pre-publication filesystem failure, not a startup-helper mock. +const write = AnchoredDirectory.prototype.write; +AnchoredDirectory.prototype.write = function (...args) { + if (input.outcome === "not-started" && args[0] === `${key}.json`) { + const value = JSON.parse(args[1].toString()); + if (value?.execution && value.receiptDigest) throw Error("effect intent publication failed"); + } + return write.apply(this, args); +}; +const remove = AnchoredDirectory.prototype.removeDirectory; +AnchoredDirectory.prototype.removeDirectory = function (...args) { + const result = remove.apply(this, args); + // Stop after the real canonical-digest transaction AND guard removal. No + // unknown metadata writer is left for startup to steal. + if (input.boundary === "digest" && terminal && args[0] === ".family-writer") { + const progress = JSON.parse(this.read("progress.json", 131072).toString()); + const facts = Object.values(progress.executions) as Array<{ + execution: { executionId: string }; + terminalDigest?: string; + journalContained?: boolean; + }>; + if ( + facts.some( + (f) => + f.execution.executionId === terminal?.operationId && + f.terminalDigest === controlTerminalReceiptDigest(wire(terminal)) && + !f.journalContained, + ) + ) + stop(terminal); + } + return result; +}; +const replace = journal.replace; +journal.replace = async (records) => { + await replace(records); + terminal = records.find((job) => ["succeeded", "failed", "timed_out"].includes(job.state)); + if (!terminal) return; + if (input.outcome === "not-started") { + if ( + terminal.admittedAt === undefined || + terminal.deadlineAt === undefined || + !terminal.operationFingerprint + ) + throw Error("Missing exact admission"); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: terminal.operationId, + operationKey: terminal.operationKey, + operationFingerprint: terminal.operationFingerprint, + fingerprint: terminal.fingerprint, + deadlineAt: terminal.deadlineAt, + }; + const authority = c.restoreAdmittedControlExecution(installation, execution); + // The fixed executor signs actual never-started evidence only after the + // failed intent write and joined failure handler; no application IO ran. + c.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + } + if (input.boundary === "terminal") stop(terminal); +}; +const app = buildOpsAgentServer({ + tokens: { api: token, "data-manager": "b".repeat(32) }, + controlInstallation: installation, + journal, + resourcePolicy: { allowBackup: () => true, allowGlobal: () => true, allowService: () => true }, + resourceClaimer, + dispatch: async () => { + appendFileSync(join(input.root, "effect-calls"), "effect\n", { mode: 0o600 }); + if (input.outcome === "unknown") throw Error("uncontained application failure"); + if (input.borrowed) return { changed: true, state: "stopped" }; + return "revisionId" in operation + ? { + outcome: "committed", + revisionId: operation.revisionId, + enabledServiceIds: ["motis"], + followUp: "complete", + cleanupFailed: false, + } + : { backupId: "daily" }; + }, +}); +const fields = { + version: 1, + apiJobId, + apiOwnerVersion: 1, + parentOperationKey: "opk1_parentabcdefghijkl", + parentFingerprint: "b".repeat(64), + parentDeadlineAt: notAfter, + childOperationKey: key, + childOperationFingerprint: opsOperationFingerprint(operation), + requestedDeadlineAt: notAfter, + attemptId, +}; +const payload = Buffer.from(JSON.stringify(fields)).toString("base64url"); +const parent = `${payload}.${createHmac("sha256", token).update("openmapx-api-job-parent-v1\0").update(payload).digest("base64url")}`; +const admitted = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { + authorization: `Bearer ${input.borrowed ? "b".repeat(32) : token}`, + "x-ops-protocol-version": "2", + ...(input.borrowed + ? { "x-openmapx-control-delegation": input.borrowed.assertion } + : { "x-openmapx-parent-execution": parent }), + }, + payload: { + version: 2, + requestId: `ops1_${randomUUID().replaceAll("-", "")}`, + operationKey: key, + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 10000).toISOString(), + operation, + execution: { requestedDeadlineAt: notAfter }, + }, +}); +if (admitted.statusCode !== 202) throw Error(`Fixture admission failed ${admitted.body}`); diff --git a/apps/ops-agent/src/trusted-config-source.ts b/apps/ops-agent/src/trusted-config-source.ts index 4a9300224..fa1e80f65 100644 --- a/apps/ops-agent/src/trusted-config-source.ts +++ b/apps/ops-agent/src/trusted-config-source.ts @@ -21,6 +21,7 @@ import { OPS_TRUSTED_CONFIG_MAX_BYTES, OPS_TRUSTED_CONFIG_QUEUE_LOCK_NAME, openTrustedConfigurationSnapshot, + opsOperationFingerprint, trustedConfigurationQueueFits, } from "@openmapx/core/ops"; import type { OpsClaimOwner, TrustedOpsDataSource } from "./policy"; @@ -36,6 +37,21 @@ const TEMPORARY_NAME = /^\.cfg1_[A-Za-z0-9_-]{43}\.nonce_[A-Za-z0-9_-]{16,96}\.t const ABORT_NAME = /^\.cfg1_[A-Za-z0-9_-]{43}\.[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.abort$/; const HARDLINK_ABORT_NAME = /^\.(cfg1_[A-Za-z0-9_-]{43})\.(terminal|rollback)\.abort$/; +declare const terminalCleanupBrand: unique symbol; +export type VerifiedTerminalClaimCleanup = Readonly<{ [terminalCleanupBrand]: true }>; +const terminalCleanups = new WeakMap>(); +/** Nominal proof from the fixed authenticated, inode-checked startup cleanup. */ +export function verifiesTerminalClaimCleanup( + proof: VerifiedTerminalClaimCleanup | undefined, + operation: OpsOperation, + terminalAt: string, +): boolean { + return ( + !!proof && + terminalCleanups.get(proof)?.has(`${opsOperationFingerprint(operation)}:${terminalAt}`) === true + ); +} + export interface TrustedSnapshotDirectoryPolicy { allowedUids: readonly number[]; expectedGid: number; @@ -435,6 +451,13 @@ export async function initializeTrustedSnapshotDirectory( directory: string, policy: TrustedSnapshotDirectoryPolicy, ): Promise { + await initializeTerminalClaimCleanup(directory, policy); +} +/** Fixed agent startup additionally retains the verified terminal-cleanup capability. */ +export async function initializeTerminalClaimCleanup( + directory: string, + policy: TrustedSnapshotDirectoryPolicy, +): Promise { let queueLock: Awaited> | undefined; try { if (!directoryIsSafe(directory, policy)) throw new Error(DIRECTORY_REJECTED); @@ -489,6 +512,21 @@ export async function initializeTrustedSnapshotDirectory( syncDirectoryNow(plan.second.parent); } } + const proof = Object.freeze({}) as VerifiedTerminalClaimCleanup; + terminalCleanups.set( + proof, + new Set( + (policy.journalRecords ?? []) + .filter( + (record) => + "revisionId" in record.operation && + ["succeeded", "failed", "timed_out"].includes(record.state) && + record.terminalAt !== undefined, + ) + .map((record) => `${opsOperationFingerprint(record.operation)}:${record.terminalAt}`), + ), + ); + return proof; } catch { throw new Error(DIRECTORY_REJECTED); } finally { diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index cc46a8fc4..7d3c3e8ae 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -179,7 +179,22 @@ Namespace deletion records `planned`, `unlinked`, `counted` and `item_done` in a preallocated exact-identity transaction. Only an already retained matching plan can resume an absent unlink. Counter decrements and slot reuse check the original family, generation, manifest, ordinal and inode. A killed unknown writer is never -stolen automatically. +stolen automatically. Target release progress uses a certified sorted-target +prefix count and a pending target index, so all 256 maximum-length target IDs fit +within the signed admission journal's 4 KiB read/write budget. Writes validate the +actual schema and byte limit. Retained signed name-list progress within that +budget is checked against the exact certified prefix before compact recovery. + +On agent startup, terminal retention repair rejoins the original signed admission +and exact authenticated own-effect proof. A known result must match the signed +result body; a known failure needs signed never-started evidence. Borrowed workers +also need their exact retained settled lease proof. Configuration workers require +the fixed authenticated, inode-checked startup claim cleanup capability. Loaded +terminal state, age and missing processes do not prove containment. Missing, +mismatched or unknown proof keeps the fence. Repair publishes only the missing +canonical terminal digest and journal completion, preserving original admission, +result/failure, deadline and terminal time; duplicate restarts do not restart work +or postpone the 72-hour retention window. ### API parent checkpoints and terminal receipt delivery diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 1670709c8..2052a1a4a 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -42,6 +42,7 @@ import { type OpsOperationKind, opsOperationSchema, parseBoundedOpsResult, + parseOpsJobStatus, } from "../ops/contract"; import { controlTerminalReceiptDigest, @@ -643,6 +644,79 @@ export function retainControlTerminalStatus( digest, ); } +/** Rejoin a terminal worker from its original signed admission and known effect + * receipt. The fixed startup adapter separately proves borrowed/claim cleanup. */ +export function repairControlJournalTerminal( + installation: ControlInstallation, + execution: ExecutionRef, + status: OpsJobStatus, + terminalAt: number, +): VerifiedExecutionAuthority { + const authority = restoreAdmittedControlExecution(installation, execution), + s = executionAuthorityState(authority); + const terminal = parseOpsJobStatus(status); + if ( + !s.origin || + installation.role !== "ops-agent" || + s.installation.fixedChildId || + terminal.version !== 2 || + terminal.admittedAt === undefined || + terminal.operationId !== execution.executionId || + terminal.operationKey !== execution.operationKey || + terminal.fingerprint !== execution.fingerprint || + terminal.deadlineAt !== execution.deadlineAt || + terminal.kind !== s.kind || + !Number.isSafeInteger(terminalAt) || + terminalAt < terminal.admittedAt + ) + throw Error("Original terminal journal identity required"); + const origin = s.origin; + withPrivateEffectWriter(s, () => { + const admittedRaw = readPrivateRecord( + s.installation, + "executions", + `${execution.operationKey}.json`, + 65536, + ); + const admitted = verifyControl( + installation.publicKey, + originRecordDomain("openmapx-control-execution", admittedRaw), + admittedRaw, + z.object({ admittedAt: z.number().int().nonnegative().safe() }).passthrough(), + ); + if (admitted.admittedAt !== terminal.admittedAt) throw Error("Original admission time changed"); + verifyControlEffectSettlement(installation, execution); + const proof = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord(s.installation, "effects", `${execution.operationKey}.json`, 65536), + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + const succeeded = + terminal.state === "succeeded" && + proof.outcome === "committed" && + proof.publication === "committed" && + proof.receipt.kind === "ops-result" && + controlDigest(proof.receipt.result) === controlDigest(terminal.result); + const failedWithoutEffect = + ["failed", "timed_out"].includes(terminal.state) && + proof.outcome === "failed_without_effect" && + proof.publication === "not_started" && + proof.receipt.kind === "not-started"; + if (!succeeded && !failedWithoutEffect) + throw Error("Terminal worker containment is not proved"); + markFamilyTerminal( + s.installation, + origin, + execution, + true, + controlTerminalReceiptDigest(terminal), + terminalAt, + ); + completeFamilyJournal(s.installation, origin, execution, terminalAt); + }); + return authority; +} /** Fixed agent adapter, installed before startup reconciliation; no request carries this capability. */ export function registerControlRetentionJournal( installation: ControlInstallation, diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 87ae98d6d..67dbcafea 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -33,6 +33,7 @@ export { readFixedCliSpawnIntent, reconcileControlRetention, registerControlRetentionJournal, + repairControlJournalTerminal, requireControlJournalRetention, restoreAdmittedControlExecution, retainAdmittedControlExecution, diff --git a/packages/core/src/build-artifacts/retention-family.ts b/packages/core/src/build-artifacts/retention-family.ts index d21443d0f..e1ae52297 100644 --- a/packages/core/src/build-artifacts/retention-family.ts +++ b/packages/core/src/build-artifacts/retention-family.ts @@ -993,16 +993,20 @@ export function completeFamilyJournal( state: InstallationState, origin: ControlExecutionOrigin, execution: ExecutionRef, + terminalAt?: number, ): void { const f = openFamily(state, origin); try { guard(f.slot, ".family-writer", () => { const p = readProgress(f), fact = p.executions[controlDigest(execution)]; - if (!fact?.requiresJournal || p.phase !== "active") - throw Error("Invalid joined journal worker"); + if (!fact?.requiresJournal) throw Error("Invalid joined journal worker"); + if (fact.journalContained) return; + if (p.phase !== "active") throw Error("Invalid joined journal worker"); + if (terminalAt !== undefined && (!Number.isSafeInteger(terminalAt) || terminalAt <= 0)) + throw Error("Invalid original terminal time"); fact.journalContained = true; - p.terminalAt = Math.max(p.terminalAt, Date.now()); + p.terminalAt = Math.max(p.terminalAt, terminalAt ?? Date.now()); progress(f, p); }); } finally { @@ -1015,6 +1019,7 @@ export function markFamilyTerminal( execution: ExecutionRef, known: boolean, terminalDigest?: string, + terminalAt?: number, ) { const f = openFamily(state, origin); try { @@ -1028,10 +1033,13 @@ export function markFamilyTerminal( } const changed = e.known !== known || (terminalDigest !== undefined && e.terminalDigest !== terminalDigest); + if (terminalAt !== undefined && (!Number.isSafeInteger(terminalAt) || terminalAt <= 0)) + throw Error("Invalid original terminal time"); + if (!changed && e.terminalAt !== undefined) return; e.known = known; if (terminalDigest) e.terminalDigest = terminalDigest; if (changed || e.terminalAt === undefined) { - e.terminalAt = Date.now(); + e.terminalAt = Math.max(e.terminalAt ?? 0, terminalAt ?? Date.now()); p.terminalAt = Math.max(p.terminalAt, e.terminalAt); } progress(f, p); diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index b928de01c..63068eb38 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync } import { tmpdir } from "node:os"; import { join } from "node:path"; import { createControlRootOperationKey } from "@openmapx/core/ops"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { executionFingerprint } from "../ops/execution-identity"; import { opsOperationFingerprint } from "../ops/fingerprint"; import { @@ -29,9 +29,178 @@ beforeEach(async () => { }); }); afterEach(() => { + vi.restoreAllMocks(); closeControlInstallation(installation); rmSync(root, { recursive: true, force: true }); }); + +it.each([18, 256])( + "recovers release after target17 for %i maximum-length certified targets within the admission byte budget", + async (count) => { + const c = await import("./control-authority"), + a = await api(); + const { AnchoredDirectory } = await import("./filesystem"); + const targets = Array.from( + { length: count }, + (_, n) => `svc-${String(n).padStart(3, "0")}${"x".repeat(57)}`, + ); + closeControlInstallation(installation); + installation = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: targets, + }); + const serviceId = targets[0]; + if (!serviceId) throw Error("Missing test target"); + const operation = { kind: "service.build" as const, serviceId }, + operationFingerprint = opsOperationFingerprint(operation), + deadlineAt = Date.now() + 120000; + const execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + deadlineAt, + }; + const authority = retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt: deadlineAt - 120000, + requestedDeadlineAt: deadlineAt, + serviceIds: targets, + }); + const admissionPath = join( + root, + `control-private/host-cli/executions/${execution.operationKey}.json`, + ), + admitted = readFileSync(admissionPath); + const lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: targets, + owner: authority, + }); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, execution), + ); + const original = AnchoredDirectory.prototype.write; + let interrupted = false, + maximumBytes = 0; + const fault = vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(function ( + this: import("./filesystem").AnchoredDirectory, + ...args + ) { + original.apply(this, args); + if (args[0] !== "admission.json") return; + maximumBytes = Math.max(maximumBytes, Buffer.byteLength(args[1])); + const release = JSON.parse(args[1].toString()).payload.release; + if ( + !interrupted && + (release?.completedTargets ?? release?.locks?.length) === 17 && + release.pending === undefined + ) { + interrupted = true; + throw Error("contained stop after target17 publication"); + } + }); + await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow( + "contained stop after target17", + ); + expect(interrupted).toBe(true); + expect( + targets + .slice(0, 17) + .every((id) => !existsSync(join(root, `data/.admin-target-control/locks/${id}`))), + ).toBe(true); + expect(existsSync(join(root, `data/.admin-target-control/locks/${targets[17]}`))).toBe(true); + if (count === 18) { + // Reopen an existing signed name-list checkpoint too. Keep its actual + // family index/inode accounting consistent with the fixture publication. + const state = c.installationState(installation), + origin = c.executionAuthorityState(authority).origin; + if (!origin) throw Error("Missing fixture origin"); + const ledger = new AnchoredDirectory( + join(root, `data/.admin-target-control/leases/${lease.leaseId}`), + ); + try { + const payload = JSON.parse(ledger.read("admission.json", 4096).toString()).payload; + payload.release = { phase: "planned", locks: targets.slice(0, 17), active: false }; + const bytes = JSON.stringify( + c.signControl(state.privateKey, "openmapx-admission-slots-v1", payload), + ); + const { familyRecordWrite } = await import("./retention-family"); + familyRecordWrite( + state, + origin, + { + scope: "shared", + namespace: "lease-owner", + container: lease.leaseId, + name: "admission.json", + }, + bytes, + (before) => ledger.write("admission.json", bytes, false, before), + ); + } finally { + ledger.close(); + } + } + a.closeArtifactTargetLeaseHandle(lease); + closeControlInstallation(installation); + installation = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: targets, + }); + const restored = c.restoreAdmittedControlExecution(installation, execution), + recovery = c.verifyControlRecoveryAuthority(restored); + const input = { dataRoot: join(root, "data"), leaseId: lease.leaseId, authority: recovery }; + await expect(a.reconcileArtifactTargetLease(input)).resolves.toBe("released"); + await expect(a.reconcileArtifactTargetLease(input)).resolves.toBe("released"); + fault.mockRestore(); + expect(maximumBytes).toBeLessThanOrEqual(4096); + const journalBytes = readFileSync( + join(root, `data/.admin-target-control/leases/${lease.leaseId}/admission.json`), + ); + expect(journalBytes.byteLength).toBeLessThanOrEqual(4096); + const finalJournal = JSON.parse(journalBytes.toString()).payload; + expect(finalJournal.release).toMatchObject({ + version: 2, + phase: "released", + completedTargets: count, + active: true, + }); + expect( + existsSync( + join( + root, + `data/.admin-target-control/admission-index-v1/active/${finalJournal.active.name}`, + ), + ), + ).toBe(false); + expect( + targets.every((id) => !existsSync(join(root, `data/.admin-target-control/locks/${id}`))), + ).toBe(true); + expect(readFileSync(admissionPath)).toEqual(admitted); + expect( + JSON.parse( + readFileSync( + join(root, `control-private/host-cli/effect-started/${execution.operationKey}.json`), + "utf8", + ), + ), + ).toBeNull(); + }, + 120000, +); function owner() { const operation = { kind: "service.build" as const, serviceId: "osrm" }; const operationFingerprint = opsOperationFingerprint(operation); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index 75551b4d1..621ad8433 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -98,6 +98,22 @@ const slotSchema = z.strictObject({ identity: inode, generation: uuid.optional(), }); +const releaseProgressSchema = z.strictObject({ + version: z.literal(2), + phase: z.enum(["planned", "released"]), + completedTargets: z.number().int().min(0).max(256), + pending: z.number().int().min(0).max(255).optional(), + active: z.boolean(), + activePlanned: z.boolean().optional(), +}); +// Previously signed name-list progress remains readable within the same byte bound. +const legacyReleaseProgressSchema = z.strictObject({ + phase: z.enum(["planned", "released"]), + locks: z.array(z.string()).max(256), + active: z.boolean(), + pending: z.string().optional(), + activePlanned: z.boolean().optional(), +}); const admissionSlotSchema = z.strictObject({ version: z.literal(1), installationId: uuid, @@ -105,15 +121,7 @@ const admissionSlotSchema = z.strictObject({ certificateDigest: digest, active: slotSchema, history: slotSchema, - release: z - .object({ - phase: z.enum(["planned", "released"]), - locks: z.array(z.string()).max(16), - active: z.boolean(), - pending: z.string().optional(), - activePlanned: z.boolean().optional(), - }) - .optional(), + release: z.union([releaseProgressSchema, legacyReleaseProgressSchema]).optional(), releasedAt: z.number().int().positive().safe().optional(), }); type AdmissionSlots = z.infer; @@ -480,6 +488,8 @@ function writeBounded( const dir = kind === "owner" ? s.ledger : childDirectory(s, kind); try { const bytes = canonicalJson(value); + if (name === "admission.json" && Buffer.byteLength(bytes) > 4096) + throw Error("Admission release journal exceeds byte capacity"); let old = 0; try { old = dir.read(name, 131072).length; @@ -763,8 +773,14 @@ export async function acquireArtifactTargetLease(options: { historySlot.generation, ); const admissionBytes = canonicalJson( - signControl(authority.installation.privateKey, "openmapx-admission-slots-v1", slots), + signControl( + authority.installation.privateKey, + "openmapx-admission-slots-v1", + admissionSlotSchema.parse(slots), + ), ); + if (Buffer.byteLength(admissionBytes) > 4096) + throw Error("Admission journal exceeds byte capacity"); familyRecordWrite( authority.installation, authority.origin, @@ -1416,20 +1432,49 @@ function finishOwnedRelease(s: State): void { s, "owner", "admission.json", - signControl(s.installation.privateKey, "openmapx-admission-slots-v1", s.slots), + signControl( + s.installation.privateKey, + "openmapx-admission-slots-v1", + admissionSlotSchema.parse(s.slots), + ), ); - const release = s.slots.release ?? { phase: "planned" as const, locks: [], active: false }; - s.slots.release = release; + const saved = s.slots.release; + let release: z.infer; + if (saved && !("version" in saved)) { + if ( + saved.locks.some((id, i) => id !== s.certificate.serviceIds[i]) || + (saved.pending !== undefined && + saved.pending !== s.certificate.serviceIds[saved.locks.length]) + ) + throw Error("Invalid legacy release prefix"); + release = { + version: 2, + phase: saved.phase, + completedTargets: saved.locks.length, + active: saved.active, + ...(saved.pending === undefined ? {} : { pending: saved.locks.length }), + ...(saved.activePlanned === undefined ? {} : { activePlanned: saved.activePlanned }), + }; + } else + release = saved ?? { version: 2, phase: "planned", completedTargets: 0, active: false }; + releaseProgressSchema.parse(release); if ( - release.locks.some((id) => !s.certificate.serviceIds.includes(id)) || - (release.pending && !s.certificate.serviceIds.includes(release.pending)) + release.completedTargets > s.certificate.serviceIds.length || + (release.pending !== undefined && + (release.pending !== release.completedTargets || + release.pending >= s.certificate.serviceIds.length)) || + ((release.active || release.activePlanned) && + (release.completedTargets !== s.certificate.serviceIds.length || + release.pending !== undefined)) || + (release.phase === "released" && !release.active) ) - throw Error("Invalid release journal target"); - for (const id of s.certificate.serviceIds) { - if (release.locks.includes(id)) continue; - if (release.pending && release.pending !== id) + throw Error("Invalid release journal certified prefix"); + s.slots.release = release; + for (let index = release.completedTargets; index < s.certificate.serviceIds.length; index++) { + const id = required(s.certificate.serviceIds[index], "Missing certified target"); + if (release.pending !== undefined && release.pending !== index) throw Error("Conflicting physical release step"); - if (release.pending !== id) { + if (release.pending === undefined) { const lock = s.locks.directory(id); try { if ( @@ -1443,7 +1488,7 @@ function finishOwnedRelease(s: State): void { } finally { lock.close(); } - release.pending = id; + release.pending = index; persist(); } try { @@ -1454,7 +1499,7 @@ function finishOwnedRelease(s: State): void { } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } - release.locks.push(id); + release.completedTargets = index + 1; delete release.pending; persist(); } From 5079201b88c878476b706ece33708d1b9ad398ca Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 15:02:42 +0100 Subject: [PATCH 016/135] fix(ops): authenticate immutable terminal receipts --- .../src/control-terminal-restart.test.ts | 189 ++++++++++++++++ apps/ops-agent/src/server.ts | 22 +- .../terminal-retention-process.ts | 26 ++- .../docs/developer/build-artifacts-control.md | 19 +- .../src/build-artifacts/control-authority.ts | 208 ++++++++++++++++-- .../build-artifacts/control-retention.test.ts | 182 +++++++++++++++ packages/core/src/build-artifacts/index.ts | 1 + .../src/build-artifacts/retention-family.ts | 94 +++++++- 8 files changed, 704 insertions(+), 37 deletions(-) diff --git a/apps/ops-agent/src/control-terminal-restart.test.ts b/apps/ops-agent/src/control-terminal-restart.test.ts index 5403eaa1b..615a91de1 100644 --- a/apps/ops-agent/src/control-terminal-restart.test.ts +++ b/apps/ops-agent/src/control-terminal-restart.test.ts @@ -9,6 +9,7 @@ import { createControlRootOperationKey, type ExecutionRef, executionFingerprint, + type OpsJobStatus, opsOperationFingerprint, parseOpsJobStatus, signApiRetentionProof, @@ -94,6 +95,186 @@ function progress(root: string) { >; }; } +function terminalStatus(job: PersistedOpsJob): OpsJobStatus { + return parseOpsJobStatus({ + version: 2, + operationId: job.operationId, + operationKey: job.operationKey, + kind: job.operation.kind, + resourceId: job.resourceId, + state: job.state, + submittedAt: job.submittedAt, + updatedAt: job.updatedAt, + admittedAt: job.admittedAt, + deadlineAt: job.deadlineAt, + fingerprint: job.fingerprint, + ...(job.result === undefined ? {} : { result: job.result }), + ...(job.errorClass === undefined ? {} : { errorClass: job.errorClass }), + ...(job.failureResult === undefined ? {} : { failureResult: job.failureResult }), + ...(job.terminationRequestedAt === undefined + ? {} + : { terminationRequestedAt: job.terminationRequestedAt }), + }); +} + +const tupleChanges = [ + "failure-error", + "failure-result", + "failure-cancel", + "failure-time", + "success-cancel", + "success-time", +] as const; +it.each( + ["terminal", "digest"].flatMap((boundary) => + tupleChanges.map((change) => ({ boundary, change })), + ), +)( + "rejects altered original terminal tuple across actual restart ($boundary/$change)", + async ({ boundary, change }) => { + const root = mkdtempSync(join(tmpdir(), "omx-terminal-tuple-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await c.provisionControlInstallation(root); + let app: ReturnType | undefined, + installation: c.ControlInstallation | undefined; + const token = Buffer.alloc(32, 7).toString("base64url"), + dispatch = vi.fn(async () => { + throw Error("Restart must never dispatch"); + }); + try { + const failure = change.startsWith("failure"), + stopped = await interrupt(root, boundary, failure ? "not-started" : "success"); + const original = stopped.job; + if (!original.terminalAt) throw Error("Missing original terminal timestamp"); + const originalTime = Date.parse(original.terminalAt), + originalDigest = controlTerminalReceiptDigest(terminalStatus(original)); + const before = progress(root), + altered = structuredClone(original); + const commitmentPath = join( + root, + `control-private/ops-agent/observations/terminal-${original.operationKey}.json`, + ); + const commitment = readFileSync(commitmentPath); + const admission = readFileSync( + join(root, `control-private/ops-agent/executions/${original.operationKey}.json`), + ); + if (change === "failure-error") altered.errorClass = "timeout"; + else if (change === "failure-result") + altered.failureResult = { + diagnostics: { + tail: [{ stream: "stderr", message: "different valid failure evidence" }], + truncated: false, + }, + }; + else if (change.endsWith("cancel")) + altered.terminationRequestedAt = new Date(originalTime).toISOString(); + else altered.terminalAt = new Date(originalTime + 1000).toISOString(); + const file = join(root, "data/ops-agent/jobs-v1.json"), + changedJournal = await openOpsJobJournal(file); + await changedJournal.replace([altered]); // Actual parser accepts each schema-valid change. + const clock = vi.spyOn(Date, "now").mockReturnValue(originalTime + 71 * 3600_000); + const start = async (expected: PersistedOpsJob) => { + installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const journal = await openOpsJobJournal(file); + app = buildOpsAgentServer({ + tokens: { api: token, "data-manager": "b".repeat(32) }, + controlInstallation: installation, + journal, + dispatch, + now: () => new Date(Date.now()), + resourcePolicy: { allowBackup: () => true }, + }); + await app.ready(); + expect(journal.records()).toEqual([expected]); + return { journal, server: app }; + }; + await start(altered); + expect(Object.values(progress(root).executions)[0]?.terminalDigest).toBe( + Object.values(before.executions)[0]?.terminalDigest, + ); + expect(Object.values(progress(root).executions)[0]?.journalContained).toBe(false); + expect(progress(root).terminalAt).toBe(before.terminalAt); + expect(readFileSync(commitmentPath)).toEqual(commitment); + await app?.close(); + app = undefined; + if (installation) c.closeControlInstallation(installation); + installation = undefined; + const restoredJournal = await openOpsJobJournal(file); + await restoredJournal.replace([original]); + clock.mockReturnValue(originalTime + 71.5 * 3600_000); + const { journal, server } = await start(original); + const ack = { + kind: "controlRetention.acknowledge", + originRootKey: original.operationKey, + operationKey: original.operationKey, + fingerprint: original.fingerprint, + terminalDigest: originalDigest, + proof: signApiRetentionProof(token, { + version: 1, + apiJobId: stopped.apiJobId, + originalApiOwnerVersion: 1, + attemptId: stopped.attemptId, + checkpointOwnerVersion: 1, + terminalCopyId: randomUUID(), + operationKey: original.operationKey, + fingerprint: original.fingerprint, + terminalDigest: originalDigest, + }), + }; + for (let retry = 0; retry < 2; retry++) { + const response = await server.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }, + payload: { + version: 2, + requestId: `ops1_${randomUUID().replaceAll("-", "")}`, + operationKey: "opk1_ackrequest123456", + issuedAt: new Date(Date.now()).toISOString(), + expiresAt: new Date(Date.now() + 10000).toISOString(), + operation: ack, + }, + }); + expect(response.statusCode, response.body).toBe(200); + } + expect(Object.values(progress(root).executions)[0]).toMatchObject({ + journalContained: true, + terminalDigest: originalDigest, + }); + expect(progress(root).terminalAt).toBeLessThanOrEqual( + Math.max(before.terminalAt, originalTime), + ); + expect( + readFileSync( + join(root, `control-private/ops-agent/executions/${original.operationKey}.json`), + ), + ).toEqual(admission); + clock.mockReturnValue(Math.max(before.terminalAt, originalTime) + 73 * 3600_000); + if (!installation) throw Error("Missing restored installation"); + expect(await c.reconcileControlRetention(installation, { maxFamilies: 1 })).toEqual({ + retired: 1, + pending: 0, + }); + expect(journal.records()).toHaveLength(0); + expect(dispatch).not.toHaveBeenCalled(); + expect( + existsSync(join(root, "effect-calls")) + ? readFileSync(join(root, "effect-calls"), "utf8") + : "", + ).toBe(failure ? "" : "effect\n"); + } finally { + vi.restoreAllMocks(); + await app?.close(); + if (installation) c.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }, + 90000, +); const cases = [ { boundary: "terminal", outcome: "success", known: true }, { boundary: "digest", outcome: "success", known: true }, @@ -102,6 +283,7 @@ const cases = [ { boundary: "terminal", outcome: "unknown", known: false }, { boundary: "terminal", outcome: "configuration", known: true }, { boundary: "terminal", outcome: "success", known: false, tamper: "missing-proof" }, + { boundary: "terminal", outcome: "success", known: false, tamper: "missing-terminal-commitment" }, { boundary: "terminal", outcome: "success", known: false, tamper: "mismatched-journal" }, { boundary: "terminal", outcome: "success", known: false, tamper: "admission-time" }, ] as const; @@ -128,6 +310,13 @@ it.each( if ("tamper" in scenario) { if (scenario.tamper === "missing-proof") rmSync(join(root, `control-private/ops-agent/effects/${original.operationKey}.json`)); + else if (scenario.tamper === "missing-terminal-commitment") + rmSync( + join( + root, + `control-private/ops-agent/observations/terminal-${original.operationKey}.json`, + ), + ); else if (scenario.tamper === "admission-time") { if (original.admittedAt === undefined) throw Error("Missing fixture admission timestamp"); original.admittedAt += 1; diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 48bb8f6b8..35aef496d 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -19,6 +19,7 @@ import { restoreAdmittedControlExecution, retainAdmittedControlExecution, retainControlEffectReceipt, + retainControlTerminalCommitment, retainControlTerminalStatus, retainedControlAdoptionAssertion, settleArtifactTargetLeaseChild, @@ -581,6 +582,15 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ); } const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; + if (authority && ["succeeded", "failed", "timed_out"].includes(state)) { + if (candidate.terminalAt === undefined) throw Error("Missing terminal time"); + candidate.terminalAt = retainControlTerminalCommitment( + authority, + statusFor(candidate, 2, true), + candidate.terminalAt, + ); + updates = { ...updates, terminalAt: candidate.terminalAt }; + } await journal.replace( [...jobsById.values()].map((entry) => persisted(entry === job ? candidate : entry)), ); @@ -703,7 +713,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif state: outwardStateFor(job), ...(version === 2 && job.executionVersion === 2 ? { version: 2, ...identityFor(job) } : {}), }); - const statusFor = (job: JobRecord, version: 1 | 2): OpsJobStatus => ({ + const statusFor = (job: JobRecord, version: 1 | 2, internalTerminal = false): OpsJobStatus => ({ version: version === 2 && job.executionVersion === 2 ? 2 : 1, ...(version === 2 && job.executionVersion === 2 ? identityFor(job) : {}), ...(version === 2 && job.executionVersion === 2 && job.diagnostics @@ -716,16 +726,18 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif operationKey: job.operationKey, kind: job.operation.kind, resourceId: job.resourceId, - state: outwardStateFor(job), + state: internalTerminal ? job.state : outwardStateFor(job), submittedAt: job.submittedAt, updatedAt: job.updatedAt, ...(job.terminationRequestedAt === undefined ? {} : { terminationRequestedAt: job.terminationRequestedAt }), - ...(job.result === undefined || configurationResultsSettling.has(job.operationId) + ...(job.result === undefined || + (!internalTerminal && configurationResultsSettling.has(job.operationId)) ? {} : { result: job.result }), - ...(job.errorClass === undefined || configurationResultsSettling.has(job.operationId) + ...(job.errorClass === undefined || + (!internalTerminal && configurationResultsSettling.has(job.operationId)) ? {} : { errorClass: job.errorClass }), }); @@ -957,6 +969,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif options.controlInstallation, authority.execution, statusFor(job, 2), + job.terminalAt, ); if (terminalDurable && claimReleased && authority) completeControlJournalExecution(authority); @@ -1048,6 +1061,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif options.controlInstallation, authority.execution, statusFor(job, 2), + job.terminalAt, ); if (terminalDurable && claimReleased && authority) completeControlJournalExecution(authority); diff --git a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts index 8fc303d94..68e307ba4 100644 --- a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts +++ b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts @@ -1,5 +1,12 @@ import { createHmac, randomBytes, randomUUID } from "node:crypto"; -import { appendFileSync, mkdirSync, writeFileSync, writeSync } from "node:fs"; +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + writeFileSync, + writeSync, +} from "node:fs"; import { join } from "node:path"; import * as c from "@openmapx/core/build-artifacts"; import { @@ -126,8 +133,21 @@ AnchoredDirectory.prototype.removeDirectory = function (...args) { const result = remove.apply(this, args); // Stop after the real canonical-digest transaction AND guard removal. No // unknown metadata writer is left for startup to steal. - if (input.boundary === "digest" && terminal && args[0] === ".family-writer") { - const progress = JSON.parse(this.read("progress.json", 131072).toString()); + if ( + input.boundary === "digest" && + terminal && + [".family-writer", key].includes(args[0]) && + !existsSync(join(input.root, `control-private/ops-agent/effect-writers/${key}`)) && + !existsSync( + join(input.root, "control-private/ops-agent/retention-families/slot-0/.family-writer"), + ) + ) { + const progress = JSON.parse( + readFileSync( + join(input.root, "control-private/ops-agent/retention-families/slot-0/progress.json"), + "utf8", + ), + ); const facts = Object.values(progress.executions) as Array<{ execution: { executionId: string }; terminalDigest?: string; diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index 7d3c3e8ae..ba48e70de 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -185,8 +185,20 @@ within the signed admission journal's 4 KiB read/write budget. Writes validate t actual schema and byte limit. Retained signed name-list progress within that budget is checked against the exact certified prefix before compact recovery. -On agent startup, terminal retention repair rejoins the original signed admission -and exact authenticated own-effect proof. A known result must match the signed +Before an agent publishes a terminal journal row, its fixed live producer signs +the complete canonical terminal tuple digest and original terminal time in a +preallocated private `observations` record. This uses the existing role key, +namespace quota, family index and collection path. Own-effect settlement and +canonical terminal publication have separate digests. Configuration cleanup can +refine a prepared result while public status remains running, preserving its +first terminal time; sealing the canonical receipt freezes its tuple and time. +An already canonical digest cannot be replaced by either settlement or repair. + +On agent startup, terminal retention repair rejoins the original signed admission, +exact authenticated own-effect proof and existing signed terminal commitment. +Every canonical tuple field and the original terminal time must match that +commitment; startup cannot sign a new tuple from the loaded public row. +A known result must match the signed result body; a known failure needs signed never-started evidence. Borrowed workers also need their exact retained settled lease proof. Configuration workers require the fixed authenticated, inode-checked startup claim cleanup capability. Loaded @@ -195,6 +207,9 @@ mismatched or unknown proof keeps the fence. Repair publishes only the missing canonical terminal digest and journal completion, preserving original admission, result/failure, deadline and terminal time; duplicate restarts do not restart work or postpone the 72-hour retention window. +Legacy families without the terminal authentication record or canonical +publication reservation remain observation/recovery-required; startup does not +upgrade them by trusting their public journal. ### API parent checkpoints and terminal receipt delivery diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 2052a1a4a..33163938c 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -68,12 +68,14 @@ import { } from "./retention-clock"; import { acknowledgeRetentionDependency, + assertFamilyCanonicalCompatibility, completeFamilyJournal, enlistFamilyIntent, familyRecordWrite, hasLocalRetentionFamily, hasRetentionEvidence, markFamilyTerminal, + publishFamilyCanonicalTerminal, reconcileRetentionFamilies, registerFamilyExecution, requireFamilyJournal, @@ -617,32 +619,180 @@ export function acknowledgeControlRetention( }); return { acknowledged: true }; } +const terminalCommitmentSchema = z.strictObject({ + version: z.literal(1), + phase: z.enum(["prepared", "canonical"]), + installationId: uuid, + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: digest, + terminalDigest: digest, + terminalAt: z.number().int().positive().safe(), +}); +const terminalCommitmentName = (execution: ExecutionRef) => + `terminal-${execution.operationKey}.json`; +function terminalIdentity(s: AuthorityState, status: OpsJobStatus) { + const terminal = parseOpsJobStatus(status), + execution = s.execution; + const raw = readPrivateRecord( + s.installation, + "executions", + `${execution.operationKey}.json`, + 65536, + ); + const admitted = verifyControl( + s.installation.installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z + .object({ admittedAt: z.number().int().nonnegative().safe(), operation: opsOperationSchema }) + .passthrough(), + ); + if ( + !s.origin || + s.installation.installation.role !== "ops-agent" || + s.installation.fixedChildId || + terminal.version !== 2 || + terminal.admittedAt !== admitted.admittedAt || + terminal.operationId !== execution.executionId || + terminal.operationKey !== execution.operationKey || + terminal.fingerprint !== execution.fingerprint || + terminal.deadlineAt !== execution.deadlineAt || + terminal.kind !== s.kind + ) + throw Error("Original terminal identity required"); + return { terminal, admitted, digest: controlTerminalReceiptDigest(terminal) }; +} +function readTerminalCommitment(s: AuthorityState) { + const proof = verifyControl( + s.installation.installation.publicKey, + "openmapx-control-terminal-v1", + readPrivateRecord(s.installation, "observations", terminalCommitmentName(s.execution), 4096), + terminalCommitmentSchema, + ); + if ( + proof.installationId !== s.installation.installation.installationId || + proof.admissionDigest !== s.receiptDigest || + controlDigest(proof.execution) !== controlDigest(s.execution) || + !sameControlOrigin(proof.origin, s.origin) + ) + throw Error("Original terminal commitment identity changed"); + return proof; +} +/** Fixed live producer only: sign the full canonical tuple digest and original + * time before its public journal row. This record makes no containment claim. */ +export function retainControlTerminalCommitment( + authority: VerifiedExecutionAuthority, + status: OpsJobStatus, + terminalAt: number, +): number { + const s = executionAuthorityState(authority), + value = terminalIdentity(s, status), + origin = s.origin; + if (!origin || !Number.isSafeInteger(terminalAt) || terminalAt < value.admitted.admittedAt) + throw Error("Invalid original terminal time"); + return withPrivateEffectWriter(s, () => { + const raw = readPrivateRecord( + s.installation, + "observations", + terminalCommitmentName(s.execution), + 4096, + ); + const prior = raw === null ? undefined : readTerminalCommitment(s); + const originalTime = prior?.terminalAt ?? terminalAt; + assertFamilyCanonicalCompatibility( + s.installation, + origin, + s.execution, + value.digest, + originalTime, + ); + if (prior) { + if (prior.terminalDigest === value.digest) return originalTime; + // Configuration results stay masked while confidential claim cleanup + // refines their prepared result. A canonical receipt is never replaceable. + if (prior.phase !== "prepared" || !("revisionId" in value.admitted.operation)) + throw Error("Original terminal tuple conflict"); + } + const payload = terminalCommitmentSchema.parse({ + version: 1, + phase: "prepared", + installationId: s.installation.installation.installationId, + origin, + execution: s.execution, + admissionDigest: s.receiptDigest, + terminalDigest: value.digest, + terminalAt: originalTime, + }); + writePrivateRecord( + s.installation, + "observations", + terminalCommitmentName(s.execution), + signControl(s.installation.privateKey, "openmapx-control-terminal-v1", payload), + false, + origin, + ); + return originalTime; + }); +} +function verifyAndSealTerminalCommitment( + s: AuthorityState, + status: OpsJobStatus, + terminalAt?: number, +) { + const value = terminalIdentity(s, status), + proof = readTerminalCommitment(s), + origin = s.origin; + if ( + !origin || + proof.terminalDigest !== value.digest || + (terminalAt !== undefined && proof.terminalAt !== terminalAt) + ) + throw Error("Original canonical terminal tuple/time changed"); + assertFamilyCanonicalCompatibility( + s.installation, + origin, + s.execution, + value.digest, + proof.terminalAt, + ); + if (proof.phase === "prepared") + writePrivateRecord( + s.installation, + "observations", + terminalCommitmentName(s.execution), + signControl(s.installation.privateKey, "openmapx-control-terminal-v1", { + ...proof, + phase: "canonical" as const, + }), + false, + origin, + ); + return proof; +} export function retainControlTerminalStatus( installation: ControlInstallation, execution: ExecutionRef, status: OpsJobStatus, + terminalAt?: number, ): void { const authority = restoreAdmittedControlExecution(installation, execution), s = executionAuthorityState(authority); - if ( - !s.origin || - status.operationId !== execution.executionId || - status.operationKey !== execution.operationKey || - status.fingerprint !== execution.fingerprint || - status.deadlineAt !== execution.deadlineAt || - status.kind !== s.kind - ) - throw Error("Terminal control receipt identity conflict"); - const digest = controlTerminalReceiptDigest(status); - const settlement = childSettlementState(verifyControlEffectSettlement(installation, execution)); - markFamilyTerminal( - s.installation, - s.origin, - execution, - settlement.evidence.outcome !== "unknown" && - (status.state === "succeeded" || settlement.evidence.publication !== "committed"), - digest, - ); + withPrivateEffectWriter(s, () => { + const proof = verifyAndSealTerminalCommitment(s, status, terminalAt), + origin = s.origin; + if (!origin) throw Error("Missing terminal origin"); + const settlement = childSettlementState(verifyControlEffectSettlement(installation, execution)); + publishFamilyCanonicalTerminal( + s.installation, + origin, + execution, + settlement.evidence.outcome !== "unknown" && + (status.state === "succeeded" || settlement.evidence.publication !== "committed"), + proof.terminalDigest, + proof.terminalAt, + ); + }); } /** Rejoin a terminal worker from its original signed admission and known effect * receipt. The fixed startup adapter separately proves borrowed/claim cleanup. */ @@ -705,13 +855,14 @@ export function repairControlJournalTerminal( proof.receipt.kind === "not-started"; if (!succeeded && !failedWithoutEffect) throw Error("Terminal worker containment is not proved"); - markFamilyTerminal( + const committed = verifyAndSealTerminalCommitment(s, terminal, terminalAt); + publishFamilyCanonicalTerminal( s.installation, origin, execution, true, - controlTerminalReceiptDigest(terminal), - terminalAt, + committed.terminalDigest, + committed.terminalAt, ); completeFamilyJournal(s.installation, origin, execution, terminalAt); }); @@ -738,6 +889,15 @@ export function completeControlJournalExecution(authority: VerifiedExecutionAuth export function requireControlJournalRetention(authority: VerifiedExecutionAuthority): void { const s = executionAuthorityState(authority); if (!s.origin) throw Error("Legacy origin is observation-only"); + // Reserve the immutable terminal authentication cell before any effect. + writePrivateRecord( + s.installation, + "observations", + terminalCommitmentName(s.execution), + null, + true, + s.origin, + ); requireFamilyJournal(s.installation, s.origin, s.execution); } export function hasRetainedControlOrigin( @@ -1284,7 +1444,7 @@ const effectObservationSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("not-started") }), z.strictObject({ kind: z.literal("unresolved") }), ]); -function withPrivateEffectWriter(s: AuthorityState, write: () => void): void { +function withPrivateEffectWriter(s: AuthorityState, write: () => T): T { const guards = s.installation.privateRoot.directory("effect-writers", true, true); try { const name = s.execution.operationKey; @@ -1293,7 +1453,7 @@ function withPrivateEffectWriter(s: AuthorityState, write: () => void): void { const guard = guards.directory(name); const held = identity(guard.stat); try { - write(); + return write(); } finally { guard.close(); guards.removeDirectory(name, held); diff --git a/packages/core/src/build-artifacts/control-retention.test.ts b/packages/core/src/build-artifacts/control-retention.test.ts index e32b00691..8898d96bd 100644 --- a/packages/core/src/build-artifacts/control-retention.test.ts +++ b/packages/core/src/build-artifacts/control-retention.test.ts @@ -1,9 +1,16 @@ +import { randomBytes, randomUUID } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import type { OpsJobStatus } from "../ops/contract"; +import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; +import { createControlRootOperationKey } from "../ops/control-root-key"; +import { executionFingerprint } from "../ops/execution-identity"; +import { opsOperationFingerprint } from "../ops/fingerprint"; import * as c from "./control-authority"; import { advanceRetentionClock } from "./retention-clock"; +import { markFamilyTerminal, publishFamilyCanonicalTerminal } from "./retention-family"; import { admittedRoot, terminalWithoutEffect } from "./test-fixtures/authority"; let root: string; @@ -23,6 +30,181 @@ function owner() { installations.push(f.installation); return f; } +it("keeps canonical terminal publication immutable and distinct from settlement evidence", () => { + const f = admittedRoot(root, "ops-agent"); + installations.push(f.installation); + c.requireControlJournalRetention(f.authority); + terminalWithoutEffect(f.authority); + const admittedAt = f.execution.deadlineAt - 60000, + terminalAt = Date.now(); + const status: OpsJobStatus = { + version: 2, + operationId: f.execution.executionId, + operationKey: f.execution.operationKey, + kind: "service.build", + resourceId: "osrm", + state: "failed", + errorClass: "runtime", + admittedAt, + deadlineAt: f.execution.deadlineAt, + fingerprint: f.execution.fingerprint, + submittedAt: new Date(admittedAt).toISOString(), + updatedAt: new Date(terminalAt).toISOString(), + }; + const unsettled = JSON.parse( + readFileSync( + join(root, "control-private/ops-agent/retention-families/slot-0/progress.json"), + "utf8", + ), + ); + expect(Object.values(unsettled.executions)[0]).toMatchObject({ + terminalDigest: "0".repeat(64), + canonicalDigest: "0".repeat(64), + }); + expect( + (Object.values(unsettled.executions)[0] as { settlementDigest: string }).settlementDigest, + ).not.toBe("0".repeat(64)); + c.retainControlTerminalCommitment(f.authority, status, terminalAt); + c.retainControlTerminalStatus(f.installation, f.execution, status); + const path = join(root, "control-private/ops-agent/retention-families/slot-0/progress.json"), + canonical = readFileSync(path); + expect(() => + c.retainControlTerminalStatus(f.installation, f.execution, { + ...status, + errorClass: "timeout", + }), + ).toThrow(); + expect(readFileSync(path)).toEqual(canonical); + c.retainControlTerminalStatus(f.installation, f.execution, status); + expect(readFileSync(path)).toEqual(canonical); + const record = JSON.parse(canonical.toString()); + expect(Object.values(record.executions)[0]).toMatchObject({ + canonicalDigest: controlTerminalReceiptDigest(status), + }); + const state = c.installationState(f.installation), + origin = c.executionAuthorityState(f.authority).origin; + if (!origin) throw Error("Missing canonical test origin"); + expect(() => + publishFamilyCanonicalTerminal(state, origin, f.execution, true, "e".repeat(64), terminalAt), + ).toThrow(/canonical.*conflict/i); + expect(() => + publishFamilyCanonicalTerminal( + state, + origin, + f.execution, + true, + controlTerminalReceiptDigest(status), + terminalAt + 1, + ), + ).toThrow(/canonical.*conflict/i); + const settlement = c.readPrivateRecord(state, "effects", `${f.execution.operationKey}.json`) as { + payload: unknown; + }; + markFamilyTerminal(state, origin, f.execution, true, c.controlDigest(settlement.payload)); + expect(readFileSync(path)).toEqual(canonical); +}, 60000); + +it("does not mint a missing original terminal commitment from a public status", () => { + const f = admittedRoot(root, "ops-agent"); + installations.push(f.installation); + c.requireControlJournalRetention(f.authority); + terminalWithoutEffect(f.authority); + const admittedAt = f.execution.deadlineAt - 60000; + const status: OpsJobStatus = { + version: 2, + operationId: f.execution.executionId, + operationKey: f.execution.operationKey, + kind: "service.build", + resourceId: "osrm", + state: "failed", + errorClass: "runtime", + admittedAt, + deadlineAt: f.execution.deadlineAt, + fingerprint: f.execution.fingerprint, + submittedAt: new Date(admittedAt).toISOString(), + updatedAt: new Date(admittedAt).toISOString(), + }; + expect(() => c.retainControlTerminalStatus(f.installation, f.execution, status)).toThrow(); +}, 60000); + +it("allows only prepared configuration refinement and freezes its canonical tuple and original time", () => { + const installation = c.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["osrm"], + }); + installations.push(installation); + const operation = { + kind: "stack.render" as const, + revisionId: `cfg1_${"a".repeat(43)}`, + expectedCurrentRevisionId: null, + }; + const admittedAt = Date.now(), + deadlineAt = admittedAt + 60000, + operationFingerprint = opsOperationFingerprint(operation); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + deadlineAt, + }; + const authority = c.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["osrm"], + }); + c.requireControlJournalRetention(authority); + const result = { + outcome: "committed" as const, + revisionId: operation.revisionId, + enabledServiceIds: ["osrm"], + followUp: "complete" as const, + cleanupFailed: false, + }; + c.retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "ops-result", result }, + }); + const first: OpsJobStatus = { + version: 2, + operationId: execution.executionId, + operationKey: execution.operationKey, + kind: operation.kind, + resourceId: "global", + state: "succeeded", + result, + admittedAt, + deadlineAt, + fingerprint: execution.fingerprint, + submittedAt: new Date(admittedAt).toISOString(), + updatedAt: new Date(admittedAt).toISOString(), + }; + expect(c.retainControlTerminalCommitment(authority, first, admittedAt)).toBe(admittedAt); + const final: OpsJobStatus = { ...first, result: { ...result, cleanupFailed: true } }; + expect(c.retainControlTerminalCommitment(authority, final, admittedAt + 1000)).toBe(admittedAt); + c.retainControlTerminalStatus(installation, execution, final, admittedAt); + const path = join( + root, + `control-private/ops-agent/observations/terminal-${execution.operationKey}.json`, + ), + sealed = readFileSync(path); + expect(JSON.parse(sealed.toString()).payload).toMatchObject({ + phase: "canonical", + terminalAt: admittedAt, + terminalDigest: controlTerminalReceiptDigest(final), + }); + expect(() => c.retainControlTerminalCommitment(authority, first, admittedAt + 2000)).toThrow( + /canonical.*conflict/i, + ); + expect(readFileSync(path)).toEqual(sealed); +}, 60000); it("renews ordinary capacity from an eligible terminal family instead of exhausting lifetime receipts", async () => { const f = owner(); terminalWithoutEffect(f.authority); diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 67dbcafea..59d1a343e 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -39,6 +39,7 @@ export { retainAdmittedControlExecution, retainControlEffectReceipt, retainControlPreparedTerminal, + retainControlTerminalCommitment, retainControlTerminalStatus, retainedControlAdoptionAssertion, retainedControlParent, diff --git a/packages/core/src/build-artifacts/retention-family.ts b/packages/core/src/build-artifacts/retention-family.ts index e1ae52297..b8b2096c0 100644 --- a/packages/core/src/build-artifacts/retention-family.ts +++ b/packages/core/src/build-artifacts/retention-family.ts @@ -88,6 +88,9 @@ interface ExecutionFact { verifier: string; receiptDigest: string; terminalDigest?: string; + settlementDigest?: string; + canonicalDigest?: string; + canonicalTerminalAt?: number; terminalAt?: number; known: boolean; fixedChildId?: string; @@ -895,6 +898,9 @@ export function registerFamilyExecution( known: false, terminalAt: 0, terminalDigest: "0".repeat(64), + settlementDigest: "0".repeat(64), + canonicalDigest: "0".repeat(64), + canonicalTerminalAt: 0, requiresJournal: false, journalContained: false, ...(state.fixedChildId ? { fixedChildId: state.fixedChildId } : {}), @@ -1028,16 +1034,26 @@ export function markFamilyTerminal( e = p.executions[controlDigest(execution)]; if (!e) throw Error("Missing indexed terminal execution"); if (p.phase !== "active") { - if (e.known && known && (!terminalDigest || e.terminalDigest === terminalDigest)) return; + if ( + e.known && + known && + (!terminalDigest || (e.settlementDigest ?? e.terminalDigest) === terminalDigest) + ) + return; throw Error("Terminal transition after membership closed"); } const changed = - e.known !== known || (terminalDigest !== undefined && e.terminalDigest !== terminalDigest); + e.known !== known || + (terminalDigest !== undefined && + (e.settlementDigest ?? e.terminalDigest) !== terminalDigest); if (terminalAt !== undefined && (!Number.isSafeInteger(terminalAt) || terminalAt <= 0)) throw Error("Invalid original terminal time"); if (!changed && e.terminalAt !== undefined) return; e.known = known; - if (terminalDigest) e.terminalDigest = terminalDigest; + if (terminalDigest) { + e.settlementDigest = terminalDigest; + if (!e.requiresJournal) e.terminalDigest = terminalDigest; + } if (changed || e.terminalAt === undefined) { e.terminalAt = Math.max(e.terminalAt ?? 0, terminalAt ?? Date.now()); p.terminalAt = Math.max(p.terminalAt, e.terminalAt); @@ -1048,6 +1064,72 @@ export function markFamilyTerminal( f.close(); } } +function canonicalCompatible( + p: Progress, + execution: ExecutionRef, + digest: string, + terminalAt: number, +): ExecutionFact { + const e = p.executions[controlDigest(execution)]; + if (!e?.requiresJournal || e.canonicalDigest === undefined || e.canonicalTerminalAt === undefined) + throw Error("Missing canonical publication reservation; recovery required"); + if (!DIGEST.test(digest) || !Number.isSafeInteger(terminalAt) || terminalAt <= 0) + throw Error("Invalid canonical terminal publication"); + if (e.canonicalDigest !== "0".repeat(64)) { + if ( + e.canonicalDigest !== digest || + e.terminalDigest !== digest || + e.canonicalTerminalAt !== terminalAt + ) + throw Error("Canonical terminal receipt/time conflict"); + } else if (p.phase !== "active") + throw Error("Canonical publication after retirement preparation"); + return e; +} +export function assertFamilyCanonicalCompatibility( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, + digest: string, + terminalAt: number, +): void { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + canonicalCompatible(readProgress(f), execution, digest, terminalAt); + }); + } finally { + f.close(); + } +} +/** One-way canonical publication, independent of the immutable own-effect proof. */ +export function publishFamilyCanonicalTerminal( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, + known: boolean, + digest: string, + terminalAt: number, +): void { + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + const p = readProgress(f), + e = canonicalCompatible(p, execution, digest, terminalAt); + if (e.canonicalDigest !== "0".repeat(64) && (e.known || !known)) return; + if (p.phase !== "active") throw Error("Canonical containment after retirement preparation"); + e.canonicalDigest = digest; + e.canonicalTerminalAt = terminalAt; + e.terminalDigest = digest; + e.known = known; + e.terminalAt = Math.max(e.terminalAt ?? 0, terminalAt); + p.terminalAt = Math.max(p.terminalAt, e.terminalAt); + progress(f, p); + }); + } finally { + f.close(); + } +} export function markFamilyLeaseReleased( state: InstallationState, origin: ControlExecutionOrigin, @@ -1194,7 +1276,10 @@ function localProof(f: Family, p: Progress): boolean { if (executionWriterPresent(f, fact.execution)) return false; if ( fact.requiresJournal && - (!fact.journalContained || + (!fact.canonicalDigest || + fact.canonicalDigest === "0".repeat(64) || + fact.canonicalDigest !== fact.terminalDigest || + !fact.journalContained || !f.state.retentionJournal?.contained(fact.execution, fact.terminalDigest)) ) return false; @@ -2231,6 +2316,7 @@ export function acknowledgeRetentionDependency( dep.operationKey !== input.operationKey || dep.fingerprint !== input.fingerprint || !fact?.terminalDigest || + fact.canonicalDigest !== fact.terminalDigest || !fact.terminalAt || fact.terminalDigest !== input.terminalDigest ) From 022ff5997397c04e60b450f13f9e888339f66d51 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 16:07:00 +0100 Subject: [PATCH 017/135] docs(admin): define late child phase binding --- .../2026-10-23-admin-instance-management.md | 4 + .../2026-10-25-admin-late-child-binding.md | 37 ++++++ .../2026-10-23-admin-control-protocols.md | 2 + ...6-10-25-admin-late-child-binding-design.md | 115 ++++++++++++++++++ 4 files changed, 158 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-25-admin-late-child-binding.md create mode 100644 docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index d35918011..d7906b11b 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -19,6 +19,10 @@ All Global Constraints and Product and security decisions in the specification a Task06.1 consumes the [approved retention amendment](../specs/2026-10-25-admin-control-retention-amendment.md) and its [implementation plan](2026-10-25-admin-control-retention-amendment.md). Its fresh root key/origin, bounded renewable family GC, API receipt delivery and atomic stable-step reservation supersede conflicting earlier provisional interfaces;50-task order/scope remains unchanged. Follow-on02.3/02.4/01.2 use its actual exports. Earlier source clearance for02.2 does not qualify the new projectionv3/ack changes: those are reviewed and tested in this06.1 correction window. Agent-first then currentAPI/CLI producer cutover preserves legacy observations while rejecting fresh unsupported authority roots before effects. +## Approved Task02.3 late-child correction + +Task02.3 implements the [approved late-child binding correction](../specs/2026-10-25-admin-late-child-binding-design.md) and [additional steps](2026-10-25-admin-late-child-binding.md). Root admission reserves finite slots with fixed typed bodies/candidate IDs; exact child deadlines/fingerprints bind once at phase readiness. Current-row staged CAS, signed publication references and scope-held cancellation-visible grant issuance preserve original ownership, no replay and existing budgets. This corrects a concrete late120s-child contradiction within02.3; the50-task order and applied0042 remain unchanged. Later01.2/02.4/05/08 consumers use its actual reviewed exports. + ## Review Focus 1. Accepted jobs must reach the real production runtime and retain failed/partial/unknown outcomes. diff --git a/docs/superpowers/plans/2026-10-25-admin-late-child-binding.md b/docs/superpowers/plans/2026-10-25-admin-late-child-binding.md new file mode 100644 index 000000000..0786ae688 --- /dev/null +++ b/docs/superpowers/plans/2026-10-25-admin-late-child-binding.md @@ -0,0 +1,37 @@ +# Late Child Binding Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans. These are additional steps inside existing Task02.3, not a new independent task. + +**Goal:** Admit late short-budget children without replacing or extending their original root. + +**Architecture:** Immutable finite root reservations plus one issuer-signed exact phase binding. Existing grant, admission, cancellation and retention mechanisms consume that binding. + +**Tech Stack:** TypeScript, existing Ed25519/private filesystem protocol, PostgreSQL child CAS, Vitest/process fixtures. + +**Spec:** `docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md`. + +## Global Constraints + +All accepted control/retention contracts remain binding except future child exact identity may be late-bound through this fixed reservation. Combined64 plan slots,8KiB cell/128KiB aggregate, existing64KiB admission/128KiB lease and family/index limits remain. Never edit applied0042. Only Codex6.1Sol; no user questions, no additional delegated writer or reviewer from the implementer. + +## Review Focus + +1. Hours-late phase preserves original root ownership and gives only the existing child budget. +2. Concurrent or restarted bind cannot change body/deadline/phase or dispatch twice. +3. Cancellation between binding and exposure cannot lose an attempted descendant. +4. Missing private/DB/grant checkpoints do not become new forward authority. +5. Slot allocation and family collection stay bounded and recoverable at capacity. + +## Task02.3 additional steps: finite phase bindings + +**Files:** Create focused `packages/core/src/build-artifacts/child-phase.ts` and `child-phase.test.ts`; modify `control-authority.ts`, `target-lease.ts`, `index.ts`, fixed spawn/parent adapters and their named tests. Reuse existing indexed/private-record constructors; no new key or generic plugin. Modify02.3's `services/data-manager/src/jobs/execution/ops-children.ts`, store/runtime tests and existing process fixtures; deadline-only CLI/agent callers use the same resolver. Keep new responsibility in the focused module instead of growing another authority monolith. + +**Interfaces:** Export exact `ControlChildReservation`, `bindControlChildPhase` and `lookupControlChildPhase`, plus nominal `FreshControlChildDispatch`/`issueControlChildPhase`/`consumeControlChildDispatch` from the spec through the server-only barrel. Root admission/restoration accepts immutable optional reservations. Existing consumers resolve both exact and bound plans; no receiver wire relaxation. + +- [ ] Write causal late120s root/delegation regression before core implementation; capture missing/expired exact-plan failure against real authority/lease state. Run `pnpm exec vitest run packages/core/src/build-artifacts/child-phase.test.ts --maxWorkers=1`. +- [ ] Add immutable reservation parsing and preallocated bounded indexed cells; implement signed one-time bind/lookup and unchanged duplicate restoration, finite graph and wholly fixed body policy, root-owned preallocated candidate IDs and original normalization. Preserve raw old exact receipt identity when reservations are absent. Same-slot race and clock/expiry tests use actual filesystem processes. +- [ ] Integrate exact resolver into lease and deadline-only grant/spawn paths. Implement module-internal publishReservedLeaseGrant holding lease→root-first ancestry→local execution guards through publication; register finite lease-attempt records in the execution cancellation index and reconcile the union with shared lease grants. Add actual paused-after-precheck local/ancestor cancel races, opposite-order partial publication and deadline-only cases, not just binder races. Pin no-post-on-ambiguous-checkpoint and existing attempted-descendant behavior with focused core/process tests. Run the new file plus affected authority/admission test selections, not every unchanged suite. +- [ ] Wire real DM child store CAS and typed adapters: confirmed reserved full-body CAS → immutable binding → confirmed bound CAS → scope-held grant/reference/outbox → confirmed attempted/dispatch-claimed CAS + fresh nominal one-use ticket → one POST. Model reserved/bound/attempted/observing/terminal/unknown as a strict JSON union with stage-specific real fields; update store immutable-transition checks, no dummy grant values. Recover original body and grant via fixed slot publication reference; no fresh issuance ticket or mutation replay. Add crashes after every body/binding/DB/outbox/shared/node/linkage step, two same-slot callers, ambiguous/stale CAS and exact state assertions. Add real PostgreSQL child races and actual separate-process DM→assembled-agent late-phase/admission/loss/cancel tests. Use existing owned synthetic DB verification and applied0042. +- [ ] Run affected package types and host-authority policy once stable; self-review changed interfaces and consumers. Append exact RED/GREEN/time/exit/limits to Task02.3 report/evidence and include changes in its local normal-hook commit. Controller reviews whole02.3 afterward; no independent commit/PR/deploy requirement for this substep. + +Self-review: all five risks have named tests; interfaces match the spec, all body values are root-fixed and IDs preallocated, old exact plans remain valid, and no generator/DB migration is required by this correction. Independent scoped R2 review is READY with I1–I4 addressed and0 open; execute these steps in02.3. Design readiness does not qualify implementation. Binding envelope is version2/openmapx-control-phase-binding-v2; reservation/origin metadata version1 and legacy verifier remain unchanged. Test wrong version/domain/changed origin and capacity/unbound allocation/reclamation. Explicitly test original Full Update→fixedCLI→sequentialDM reservation transport and no target hold by its waiting parent. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 8fa3cadb1..dee83e2b3 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -4,6 +4,8 @@ Date: 2026-10-23. Binding supplement to `2026-10-23-admin-instance-management-de The approved [renewable control retention amendment](2026-10-25-admin-control-retention-amendment.md) supersedes this supplement only for fresh authority root keys, signed origin propagation, renewable receipt-family retention, API receipt acknowledgement and stable-step checkpoint reservation. Apply all revision2–4 decisions together. Legacy undated identities remain observation/settlement-only; do not use the earlier optional-root-deadline language to mint new managed authority. All other contracts below remain binding. +The approved [late-child binding correction](2026-10-25-admin-late-child-binding-design.md) and its [Task02.3 implementation steps](../plans/2026-10-25-admin-late-child-binding.md) additionally permit immutable finite phase reservations with one exact late identity binding. Every body field and candidate ID remains root-fixed. The original root/deadline, recipient normalization, finite graph, cancellation and retention boundaries remain binding; grant publication now holds the specified ancestry/execution guards through cancellation-visible outbox publication. + ## 1. Source constraints and ownership - `configuration-ownership.ts:acquireConfigurationOwnership` already uses exclusive mkdir, a 5-second wait, inode checks and no stealing after a crash. `configuration-generation.ts:withConfigurationGenerationWriter` owns the current pointer and commit evidence. Extend this discipline; PID age and TTL are not ownership. diff --git a/docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md b/docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md new file mode 100644 index 000000000..987b4f116 --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md @@ -0,0 +1,115 @@ +# Late child binding under an immutable control root + +Status: approved after independent scoped R2 review (R1-I1–I4 addressed;0 open). This is a narrow Task02.3 correction, not a new task or a replacement for the accepted retention amendment. + +## Problem and decision + +The accepted root receipt fixes complete future child `ExecutionRef`s. A transit root can spend hours downloading/building before a child with a120s kind budget is ready. A root-time deadline either expires before that phase or is changed by the recipient's actual-admission clamp. The exact grant identity check correctly rejects that change. Root admission, target ownership and immutable root time cannot be postponed. + +Controller decision: retain the immutable root and reserve a finite child slot at root admission; bind that slot exactly once when its phase is ready. Do not extend the child's policy budget, invent a new root, rewrite the admitted root plan or relax recipient equality checks. The alternative of an early deadline makes normal long transit work impossible; using a root-sized child budget bypasses existing policy. Root-time slots with late exact binding preserve both. + +## Immutable reservation and interfaces + +Keep existing exact `ControlPlannedChild` plans readable and usable. Add optional `reservations: readonly ControlChildReservation[]` to the signed admitted receipt, with an empty default for old records. It is immutable on duplicate admission. Existing exact entries and reservations share one canonical ordinal space and the existing combined64-entry cap; no phase can be appended later. + +```ts +type ControlChildReservation = { + version: 1; + phase: ControlChildPhase; + ordinal: number; + kind: string; + recipient: ControlRecipient; + serviceIds: readonly string[]; + execution: Pick; + operationFingerprint: string; +}; +bindControlChildPhase(authority: VerifiedExecutionAuthority, input: { + phase: ControlChildPhase; + ordinal: number; +}): Promise; +lookupControlChildPhase(authority: VerifiedExecutionAuthority, input: { + phase: ControlChildPhase; + ordinal: number; +}): ControlPlannedChild | null; +``` + +The reservation fixes the existing derived phase key, receiver-valid proposed execution ID, executor/principal, finite graph edge, kind, recipient, exact sorted target subset AND exact whole typed-operation fingerprint. There is no null/wildcard body or late operation field. The binder accepts only phase/ordinal and reads every identity field from the signed reservation. Root input/receipt owns all body values; changing any field conflicts before binding, grant or POST. + +### Root-fixed bodies and owned candidate identifiers (R1-I1) + +Use fixed DM adapter `reserveTransitChildOperations` in `services/data-manager/src/jobs/execution/ops-children.ts` at admission, before any build/proxy write. It allocates the safe typed operations and candidate identities into the original durable job metadata/child reservations; fingerprints use the existing canonical operation helper. For promotion, `preparedRunId` is the original DM execution UUID (the prepared-run writer must use that ID). For each proxy stage/commit/restore, `candidateId = "ctrl-" + SHA256("openmapx-transit-candidate-v1\0" + root.operationKey + "\0" + phase + "\0" + canonicalOrdinal).hex.slice(0,32)`. Replace mtime-derived candidate IDs on these owned paths. Candidate bytes/checksums can be produced later; their identity cannot. + +`assertOwnedTransitChildCandidate` is the fixed local adapter before binding/dispatch. It reads the SAME retained job's phase checkpoint and its existing confined candidate/run marker, requiring parent execution identity, phase, reserved ID and current recorded byte digest to agree. It cannot accept a caller boolean, another root's candidate, or a freshly reconstructed candidate after an attempted child. The prepared/proxy writers retain this original parent/phase/ID provenance in their existing private/job state before exposing candidate data. Grant policy/resource checks remain independent. No general callback or authority constructor is exposed to HTTP. + +For paired pin approval, ALL six fields `{kind,proposalId,validationId,expectedGenerationId,diffDigest,acknowledgeLicenseChanges}` are fixed by the original `catalog.activate` input. The accepted05.3 activation API already requires validationId before activation admission. The fixed adapter rechecks that exact retained proposal/validation/current-generation policy; it does not accept validation produced by another activation in place of the selected evidence. Only DM `catalog.activate` may reserve `paired-pin-commit`; neither transit-sync nor transit-maintenance gains that edge.02.3 retains this leaf closed;05.3 supplies the specified real verification. No future runtime fallback permits null bodies. + +Other exact ops and deadline-only data inputs likewise persist their entire safe typed body in their owning store before first binding. Reserve a fixed child's own reservations through its existing private spawn intent and signed admission, not only the immediate CLI child. Validate all reservations against the SAME existing finite parent-kind/phase/recipient/depth policy. They grant no new edge or widened target subset. Existing exact plans without the reservations member keep their raw bytes/signature/digest; normalize absent to empty only in comparison views so exact old duplicate admission/restoration does not conflict merely because the new parser knows that member. + +## One binding, original budget + +Lookup resolves an existing binding BEFORE reading time. An identical bind returns its original exact child; a changed body is rejected by the signed reservation/owned checkpoint before binding. Read-only lookup remains possible after cancellation/expiry; it grants no forward authority. A missing/corrupt expected binding is recovery-required, never an unbound slot. + +For an actually unbound reserved slot, under the original execution's non-stealable private writer, verify its signed root receipt, current complete forward ancestry, origin clock and unexpired root. Capture `boundAt = Date.now()`. Choose `requestedDeadlineAt = min(root.deadlineAt, boundAt + opsExecutionBudget(kind).processMs)`; use the existing verified parent identity and normalization to compute the exact execution fingerprint. The recipient STILL recomputes its own kind/deployment clamp at actual admission; any stricter policy/clock mismatch refuses before adoption. It does not mutate the grant or authorize a replacement attempt. No transport delay can extend the stored child deadline; if elapsed, no fresh child effect is permitted. + +Persist an issuer-signed binding with payload version2/domain `openmapx-control-phase-binding-v2`, installation/origin, original parent execution and signed admission digest, reservation digest, complete `ControlPlannedChild`, boundAt and requestedDeadlineAt. Its identity/time are write-once. Reservation schema version1 and origin metadata version1 remain distinct from this version2 signed origin-bearing envelope; validateOriginRecord is unchanged and rejects origin-bearing version1 bindings. Root execution/deadline/fingerprint and signed admission bytes are unchanged. No new signing key, issuer endpoint, browser parameter or generic callback is added. + +Preallocate one issuer-signed version2 `unbound` binding cell for each reservation at admission, using the existing role-private `observations` namespace and family index. Derive the basename solely from parent key and canonical ordinal. Each cell is at most8KiB; combined per-node binding payload at most128KiB. Account the actual worst-case signed size determined by fixed identities/target lengths, including the bounded publication reference below, rather than promising64 maximum-size cells fit128KiB. Reserve its accounting/index space before root forward readiness/effects; refuse an oversized prospective plan then. Keep existing64KiB admission receipt,128KiB lease ledger,64 node intents,128 family executions/256 intents and2MiB family index/GC limits. Binding/grant duplication still must fit those bounds; never silently raise them. Reclamation uses the existing exact indexed namespace transaction, not an additional collector or lifetime scan. + +## Recoverable binding and child checkpoints (R1-I2) + +The safe typed body is durably stored BEFORE binding. Replace the current all-fields-required child JSON with a strict staged union (no dummy grant UUID/digest): + +| State | Required immutable fields | Mutation/dispatch permission | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | +| reserved | original parent, phase/ordinal, full safe typed operation + fingerprint, reserved key/proposed ID/targets, cancellation intent | confirmed current owner CAS only; no child identity/grant yet | +| bound | reserved fields + exact ExecutionRef, requested deadline, binding digest | only the live confirmed reserved-CAS caller may proceed; duplicate/restart is observation | +| attempted | bound fields + actual delegation UUID/grant digest; dispatchState `pending` or `claimed` | grant is already conservatively attempted; only fresh live issuance ticket + confirmed claimed-CAS permits ONE POST | +| observing / terminal / unknown | preceding immutable fields + existing accepted/terminal/containment data | lookup/cancel/settle only; never the original effect POST | + +The existing32-child/8KiB-each/128KiB-total JSON limits and0042 columns remain. Store CAS validates permitted one-way transitions and current owner version, status/cancel/parent identity. Only explicit current `reserved` creation returns a local forward continuation; same-slot losers receive observation. No caller can claim a winner's continuation from parsed JSON. + +Ordering: (1) current owned-row CAS stores reserved body and returns local continuation; (2) binder signs its original exact identity; (3) confirmed owned CAS attaches binding to bound row; (4) scope-held issuer below chooses and persists real grant identity/reference then its existing outbox/attempt publications; (5) confirmed owned CAS attaches exact grant and claims dispatch, before first POST. The fresh grant return includes module-private nominal one-use `FreshControlChildDispatch`; lookup cannot mint it. The fixed DM adapter consumes it only after the confirmed claimed-CAS. A stale/failed/ambiguous CAS or same-slot loser authorizes zero POST. Do not hold DB locks across filesystem guards or HTTP. + +The binding cell has an immutable signed binding section plus a monotonic signed publication section: `unissued` or `grant_prepared` with `{delegationId,grantDigest,leaseId:null|UUID}`. Reserve its maximum size initially. Under issuance guards, write this exact reference BEFORE private/shared grant publication. Grant body/assertion remain only in the existing bounded private outbox/parent-attempt records, not duplicated into the cell. That fixed parent-key/ordinal address restores the original grant even if DB grant fields were never attached. Once grant_prepared exists, no new UUID/grant can be issued from that slot; partial/missing outbox evidence is recovery-required. + +Crash recovery verifies the original reserved body in DB, signed reservation/binding/publication reference and exact outbox/shared grant; a mismatch or missing expected cell stays fenced. It may copy the same binding/grant into the DB under current CAS and observe it, but cannot obtain a FreshControlChildDispatch or resend an attempted effect. A crash after reserved body but before binding may report interrupted/no-child according to actual containment; startup does not run an old build/phase. A still-live original caller may use a conclusively unbound slot after its own confirmed reservation CAS, within the unchanged root. Binding and all later stored deadlines are never recomputed on recovery. + +The live adapter uses this additional server-only fixed surface, whose nominal ticket stays process-local: + +```ts +type FreshControlChildDispatch = Readonly<{ /* private nominal brand */ }>; +issueControlChildPhase(authority: VerifiedExecutionAuthority, input: { + phase: ControlChildPhase; + ordinal: number; + childPublicKey: string; + lease?: TargetLeaseAuthority; +}): Promise<{ + channel: "delegation" | "parent"; + assertion: string; + dispatchTicket: FreshControlChildDispatch; +}>; +consumeControlChildDispatch(ticket: FreshControlChildDispatch): void; +``` + +Issuance requires an already-bound reserved slot, verifies lease execution/subset or the fixed deadline-only edge, and returns only after all attempt publication. Duplicate/partial issuance is observation-required, never another ticket. `consume` irreversibly retires its WeakMap-backed ticket before POST; parsed/stored/foreign/reused tickets fail. Existing exact-plan public grant signatures remain compatible and internally share the guarded issuer; they do not gain a new retry path. The new DM adapter uses the ticket-bearing surface, not the older string-only helper as a dispatch permission. + +## Scope-held grant publication and cancellation (R1-I3) + +Lease-backed and deadline-only issuers resolve an exact child from either the original exact plan or authenticated bound reservation. Fixed spawn validation uses that resolver. A plain returned child object is not authority. Add fixed module-internal `publishReservedLeaseGrant` in `target-lease.ts` (not an exported callback API); route ALL existing exact and reserved lease issuance through that same critical section. + +It acquires the lease journal guard, then deadline ancestry execution guards root-first, then the local execution writer (deduplicate any identical ancestor/local execution guard); holds all required guards through signed binding/ancestry validation, publication-reference/outbox reservation, cancellation-visible child-intent publication, shared grant write and parent lease-node update. Recheck all local/ancestor/lease phases and deadlines INSIDE that scope. Preserve the existing phase-restricted prepared/settled continuation exceptions; the guard-lifetime correction must not replace them with an unconditional held-only policy. The current assertControlDelegationForward precheck is insufficient and is not the authority for publication. A paused precheck followed by durable cancel must fail inside the actual issuance scope. + +Register each lease grant in the execution lifecycle's cancellation-visible child-intent index before shared exposure. Add a finite signed lease-attempt variant referencing exact original execution, lease/delegation IDs, child, outbox/grant digest and origin; the existing deadline-only parent-attempt variant remains separately verified by its proper domain. Execution cancellation enumerates the union of these exact private attempt records; lease cancellation enumerates its shared grant outbox. Startup reconciles the UNION, including prepared/private-only and shared-before-parent-node publication. No arbitrary grant plugin or sibling-private read is introduced. If publication crashes at any point after its reference is retained, no fresh ticket is returned and the original attempt remains fenced/lookup-only; physical absence or writer death alone is not no-effect proof. + +Both cancellation paths persist their respective intent and return exact descendants. Cancellation adapters acquire/release execution guards before acquiring a lease guard: NEVER wait for lease/DB while holding an execution writer. Binder takes only its execution/ancestry guards and waits for no lease/DB. Deadline-only issuance preserves its existing scope-held ancestor/local guard protection and uses the same reserved-slot publication reference/one-use rule. If grant issuance wins, durable cancellation finds its exact child even when DB linkage/response is lost; if cancel wins, no new grant/assertion/ticket becomes issuable. Unknown publication/writer containment retains fences; no TTL/PID stealing. + +Grant/execution signature wire shapes and receiver equality/adoption checks remain exact. Existing old admissions with no reservations cannot acquire one. Signed legacy records retain their existing observation/settlement restrictions. Parent/root restart never recomputes a slot's deadline. Family membership/enlistment and original expiry-floor retirement remain unchanged. + +## Required proof and scope + +Task02.3 owns the correction and its actual transit/deadline-only consumers in the existing exclusive window. Migration0042 is already applied to the owned synthetic database; this correction uses its bounded child JSON plus private protocol cells and MUST NOT edit that migration or allocate a new schema migration. + +Tests must causally reproduce late120s delegation failure, then prove successful actual child admission after hours of simulated root work under the original root and target inode. Real separate processes/loopback validate exact identity, unchanged root bytes and one effect. Include same-slot concurrent binding, altered root-fixed body/phase/ordinal/target/recipient (including every paired approval field), delayed/stricter-policy rejection, bind-versus-cancel, crash between binding/DB/grant/admission checkpoints, lost response lookup-only, duplicate after expiry without renewal, missing/corrupt binding, borrowed depth/subset and deadline-only Full Update. Use finite fake-clock advances, not multi-hour waits. Prove retention/index/capacity and no extra generation/key authority. + +Self-review: this addresses late normalization only, preserves exact admission and all accepted authority/retention boundaries, and assigns every new interface to02.3. R1-I1 is addressed by removing late body fields and preallocating owned candidate IDs; I2 by body-first staged CAS/reference/ticket states; I3 by a named scope-held issuer and union cancellation outboxes; I4 by the version2 binding domain. Independent R2 review checked these contracts and found0 open design issues; code and runtime proofs remain required in Task02.3. Review report: .superpowers/sdd/2026-10-23-admin-instance-management/late-child-amendment-review-r2.md (local evidence, not production acceptance). From f545f40a3f9c23ae94ef767394afbbb837f63c2b Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 21:35:59 +0100 Subject: [PATCH 018/135] docs(admin): define fixed proxy validation settlement --- .../2026-10-23-admin-instance-management.md | 4 ++ ...26-10-25-admin-proxy-validation-failure.md | 37 +++++++++++ .../2026-10-23-admin-control-protocols.md | 2 + ...5-admin-proxy-validation-failure-design.md | 62 +++++++++++++++++++ 4 files changed, 105 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-25-admin-proxy-validation-failure.md create mode 100644 docs/superpowers/specs/2026-10-25-admin-proxy-validation-failure-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index d7906b11b..2911ec689 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -23,6 +23,10 @@ Task06.1 consumes the [approved retention amendment](../specs/2026-10-25-admin-c Task02.3 implements the [approved late-child binding correction](../specs/2026-10-25-admin-late-child-binding-design.md) and [additional steps](2026-10-25-admin-late-child-binding.md). Root admission reserves finite slots with fixed typed bodies/candidate IDs; exact child deadlines/fingerprints bind once at phase readiness. Current-row staged CAS, signed publication references and scope-held cancellation-visible grant issuance preserve original ownership, no replay and existing budgets. This corrects a concrete late120s-child contradiction within02.3; the50-task order and applied0042 remain unchanged. Later01.2/02.4/05/08 consumers use its actual reviewed exports. +## Approved Task02.3 fixed proxy validation failure + +Task02.3 implements the [fixed proxy validation-failure contract](../specs/2026-10-25-admin-proxy-validation-failure-design.md) and [additional steps](2026-10-25-admin-proxy-validation-failure.md). Only the actual fixed nginx read-validation producer can prove a completed no-reload failure; generic failures remain unknown. Parent recovery obligations survive failed restore, cancellation and restart and gate terminal publication/root release. This preserves the existing automatic restore behavior without changing public schemas, graph, keys, deadlines, migration0042 or the50-task count. Fresh design R2 is readiness only; causal/process/PG/task review remains required. + ## Review Focus 1. Accepted jobs must reach the real production runtime and retain failed/partial/unknown outcomes. diff --git a/docs/superpowers/plans/2026-10-25-admin-proxy-validation-failure.md b/docs/superpowers/plans/2026-10-25-admin-proxy-validation-failure.md new file mode 100644 index 000000000..b2d4bd534 --- /dev/null +++ b/docs/superpowers/plans/2026-10-25-admin-proxy-validation-failure.md @@ -0,0 +1,37 @@ +# Fixed Proxy Validation Failure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans. These steps extend existing Task02.3; no new task is added. + +**Goal:** Settle a conclusively completed read-validation failure so the existing owned parent can recover safely. + +**Architecture:** Fixed nonce-bound nginx wrapper, private mutually exclusive validation/reload commitment, exact signed failed-child settlement and existing parent recovery. Unknown failures remain fenced. + +**Tech Stack:** TypeScript, existing Docker/contained-process adapter, role-signed indexed records, Vitest/process/real PostgreSQL fixtures. + +**Spec:** `docs/superpowers/specs/2026-10-25-admin-proxy-validation-failure-design.md`. + +## Global Constraints + +Only feedProxy.validateAndReload validation phase gains this private proof. No public schema/operation/graph/key/migration changes; applied0042 remains immutable. Existing deadlines, cancellation, private single-link records, bounded4096-byte cell/index/retirement and exact canonical terminal commitment remain binding. Use only Codex6.1Sol; no subagents from the implementer or user questions. + +## Review Focus + +1. Docker transport failure cannot masquerade as an nginx validation failure. +2. Reload-attempted or altered candidate cannot acquire no-publication proof. +3. A canceled/expired parent never starts restore despite a known failed child. +4. Crash recovery never invents a terminal tuple, repeats effects or renews time. +5. Real mounted hardlinks and actual registry claims are exercised, with private trust checks unchanged. + +## Additional Task02.3 steps + +**Files:** Create focused `apps/ops-agent/src/feed-proxy-validation.ts` and `.test.ts` for the constant wrapper/marker parser; modify `docker-runtime.ts`, `runtime.ts`, `server.ts`, relevant actual process fixtures/tests. Add finite private proof in `packages/core/src/build-artifacts/control-authority.ts` (use a focused internal helper if needed), barrel/retention consumers and affected core tests. Modify DM `jobs/execution/{ops-children,store,runtime,worker-scope}.ts`, `jobs/transitous/proxy-transaction.ts`, core `data-execution-journal.ts` and corresponding process/PostgreSQL tests to enforce the finite parent recovery obligation and authenticated settlement; do not add a new graph edge. No SDK generator is required. + +- [ ] Capture causal RED in the actual DM→production resource claimer→agent validation/restore chain; ordinary mock-stage assertions are insufficient. +- [ ] Implement fixed wrapper/strict marker handling using existing real contained runner; reserve bounded original-authority phase cell before effects, publish the typed producer proof and reload-start marker through purpose-bound context methods. +- [ ] Preserve verified failure settlement through failed transition/borrowed settlement/canonical terminal and exact restart repair; accept no generic failed status or arbitrary error as proof. +- [ ] Persist original restore reserved child before any restore/relink writes; implement one finite assertOwnedProxyRecoverySettled policy using current original-plan/DB/private child evidence. Gate live/terminal SQL/private worker completion/root settlement/release AND restart/cleanup/retirement. Do not relabel a known failed child unknown. +- [ ] Add causal stage-validation-failure→restore-validation-failure→restart proof with parent publication unknown/no final timestamp, original physical target retained and no extra POST. Pair with successful-restore case releasing once but preserving failed parent, plus reservation/file/validation crash gaps and cancel/expiry/mismatched evidence. +- [ ] Add actual invalid-validation→restore GREEN plus all negative/cancel/expiry/crash/hardlink cases specified above. Run only the named new/affected core/agent/DM selections while iterating; final affected types/host-authority and scoped style once stable. +- [ ] Include exact command/output/exit/substitutions and remaining platform limits in02.3 evidence/report, self-review and normal-hook local task commit. Controller's independent whole02.3 task review covers this extension; implementer does not spawn another reviewer. + +Self-review: each review risk is pinned to tests; existing primitive/result/continuation semantics remain fixed. Scoped R2 is READY with R1-I1 addressed and0 open; implement these steps in02.3. Design readiness does not qualify implementation/runtime. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index dee83e2b3..390dbf507 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -6,6 +6,8 @@ The approved [renewable control retention amendment](2026-10-25-admin-control-re The approved [late-child binding correction](2026-10-25-admin-late-child-binding-design.md) and its [Task02.3 implementation steps](../plans/2026-10-25-admin-late-child-binding.md) additionally permit immutable finite phase reservations with one exact late identity binding. Every body field and candidate ID remains root-fixed. The original root/deadline, recipient normalization, finite graph, cancellation and retention boundaries remain binding; grant publication now holds the specified ancestry/execution guards through cancellation-visible outbox publication. +The approved [fixed proxy validation-failure contract](2026-10-25-admin-proxy-validation-failure-design.md) and [Task02.3 steps](../plans/2026-10-25-admin-proxy-validation-failure.md) add only a private actual-producer proof for conclusively completed nginx validation before reload. The failed child stays failed/contained; a durable parent recovery obligation prevents live/restart release without exact original successful restore evidence. Other failure/transport uncertainty retains existing fences; no public operation, graph edge, signing key or migration changes. + ## 1. Source constraints and ownership - `configuration-ownership.ts:acquireConfigurationOwnership` already uses exclusive mkdir, a 5-second wait, inode checks and no stealing after a crash. `configuration-generation.ts:withConfigurationGenerationWriter` owns the current pointer and commit evidence. Extend this discipline; PID age and TTL are not ownership. diff --git a/docs/superpowers/specs/2026-10-25-admin-proxy-validation-failure-design.md b/docs/superpowers/specs/2026-10-25-admin-proxy-validation-failure-design.md new file mode 100644 index 000000000..01559a0d7 --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-admin-proxy-validation-failure-design.md @@ -0,0 +1,62 @@ +# Fixed proxy validation failure settlement + +Status: approved after scoped R1/R2 builder review; R1-I1 addressed,0 open. Additional Task02.3 contract; no new task, operation, graph edge, public schema, migration or signing key. + +## Problem and choice + +The actual agent calls `nginx -t` before `nginx -s reload`. A conclusive validation failure currently becomes signed `unresolved` because every begun failed operation has that outcome. The DM parent cannot settle this child or run its already-permitted proxy-restore phase. A public failed status or caller-supplied failure result cannot prove containment. + +Choose a fixed producer commitment for this one read-validation failure. Do not classify every failed Docker command as contained: a Docker transport error does not establish what executed. Reload failure remains unknown. Keep the existing finite graph, original root/deadline and conditional recovery policy. + +## Actual command completion, not error inference + +Only `feedProxy.validateAndReload` may use this producer, with the already authorized fixed proxy container and existing candidate claim. Replace its validation invocation with a fixed container shell wrapper equivalent to: + +```sh +nginx -t +validation_status=$? +printf '\nOPENMAPX_NGINX_TEST_V1:%s:%s\n' "$1" "$validation_status" +exit 0 +``` + +The agent invokes fixed argv `docker exec /bin/sh -c openmapx-nginx-test `. Nonce is fresh128 random bits encoded base64url, generated only by this invocation. It is data in a positional argument, never interpolated shell source or a credential. No request supplies container/path/argv/script. The shell waits for nginx and exits0 only after emitting its result. Use the existing actual contained-process runner; a returned success proves normal zero Docker exec completion, no timeout/abort/spawn error/residual or uncontained local process group. The last stdout record must match this nonce exactly once and contain an integer0–255; remove the marker from diagnostics. Missing, duplicate, malformed or mismatched marker is unknown. A nonzero Docker/wrapper exit, rejected process runner, truncation, cancellation during the command or lost transport is unknown even if diagnostics look like a validation error. Test doubles that merely throw an ordinary Error cannot mint proof. + +Validation code0 follows the existing candidate recheck and reload. Nonzero validation code plus normal wrapper completion is the sole known failure here. Capture/recheck the original confined candidate snapshot, including the actual mounted hardlink inode/bytes. Data-only snapshot reads may allow verified expected hardlinks; private authority records retain their existing single-link policy. An unexpected replacement/changed snapshot cannot produce this commitment. This check is settlement-only after the read and grants no new forward permission after cancellation. + +## Fixed private state and producer surface + +Preallocate one bounded indexed `observations` cell for this operation before effects, derived only from its validated operation key (maximum4096 bytes). Use existing family/index accounting and deletion; no completion-time allocation or separate collector. Signed origin-bearing payloads are version2/domain `openmapx-feed-proxy-validation-v2` and bind installation/origin, original execution/admission/body fingerprint, candidate ID and snapshot digest, phase and original timestamps. + +Phases are `validation_pending`, `validation_failed`, `reload_started`. The first is reserved before dispatch. Failure and reload transitions are mutually exclusive and write-once under the existing private effect writer. Failure binds validation exit1–255 and a domain-separated completion digest of original execution, candidate digest, nonce and exit code. Reload-start is persisted BEFORE invoking reload; it never authorizes a failure commitment afterward. Identical repetition observes the retained proof; changed data conflicts. Missing/corrupt expected state is unknown. + +The server constructs two purpose-bound internal context methods ONLY for this exact admitted operation: `commitFeedProxyValidationFailure({candidateDigest,validationExitCode,completionDigest}):Promise` and `markFeedProxyReloadAttempted():Promise`. They close over its nominal original authority/typed operation and private record. They are not network fields, injected user callbacks, a generic containment factory or a parsed boolean capability. The fixed Docker adapter calls commit only after verifying the actual wrapper/process/candidate observations above, immediately before throwing its ordinary runtime failure. It calls mark only after successful validation and forward/candidate checks, immediately before reload. A fixed core publisher writes/verifies this closed proof; arbitrary `OpsFailureResult` or public status cannot create one. + +The signed effect receipt has one new finite private variant `feed-proxy-validation-failure` referencing the authenticated private commitment digest. Core derives `outcome:"contained", publication:"not_started"` ONLY from that exact proof for the original feedProxy operation in validation_failed phase. The failed status remains failed, not succeeded. Failure does not imply that no process ever started; it proves the validation command ended and the reload/publication phase did not start. + +## Terminal, restart and parent consumption + +The server failure transition preserves this exact known signed settlement instead of overwriting it with unresolved. Other failed/timed-out begun leaves keep their current conservative outcome. Borrowed settlement remains authenticated through the existing child record, canonical terminal tuple commitment and exact receipt delivery. Update terminal-retention repair to accept this closed contained/not_started variant only with its authenticated original proof AND unchanged complete canonical terminal commitment/time; do not upgrade a public failed row. + +A crash before durable producer proof remains unknown. A proof persisted before canonical failed-terminal publication does not by itself invent a public result/time or authorize replay: reconcile the existing authenticated records or retain recovery-required. A crash in/after canonical terminal publication uses the original exact commitment. Never rerun validation/reload or renew a deadline to recover evidence. Original lease, candidate identity, journal/index and retirement limits remain. + +DM observation accepts the known failure only after the actual signed shared settlement matches the exact attempted child. It can then execute the existing separately reserved proxy-restore child if original root/targets/candidate ownership and remaining deadline permit it. Canceled/expired/unknown roots cannot start restore. A restored proxy does not convert the failed pipeline into success; retain its error/partial outcome and child receipts. A failed/unknown restore remains fenced. + +## Durable parent proxy recovery obligation (R1-I1) + +The fixed DM proxy transaction and owned-child store own this parent obligation; a contained failed child cannot discharge it. Before the first restore/relink filesystem write, persist the original `proxy-restore` child in its existing body-first `reserved` state under current owned-row CAS. It identifies the original parent, immutable restore slot/ordinal/key/body/candidate and backup provenance. A stale/failed/ambiguous CAS authorizes zero restore writes. Pass the same live reserved continuation into the existing owned-child path; never allocate a second restore attempt. + +Use one finite shared predicate `assertOwnedProxyRecoverySettled` in the server-only data execution journal policy, consumed by DM live runtime and restart. It reads complete current retained child/phase records and the original signed phase plan, not an in-memory transaction phase, caught exception or supplied success flag. Any activated/reserved/bound/attempted `proxy-restore` creates the obligation. A known failed `proxy-stage`/`proxy-commit` validation after its local candidate publication also requires the original planned restore even if cancellation prevented starting it; lack of a restore row is not successful recovery. Bind triggers and expected restore slot to the same original proxy transaction/root/targets, never a different ordinal/root or a different candidate's successful receipt. Existing reservation entries that have never been activated are not automatically attempted obligations. + +The obligation is satisfied ONLY by authenticated canonical succeeded `feedProxy.validateAndReload` status `{candidateId:,reloaded:true}` and matching signed committed settlement for that exact restore execution. A known contained/not_started validation failure of restore is NOT satisfaction; keep that child's authentic known outcome. Unaccepted, missing, unknown, canceled, expired, failed or mismatched restore remains pending. If no recovery was required, the predicate permits ordinary completion. Historical proof for a failed stage remains valid after legitimate restore bytes replace its candidate: verify its frozen original snapshot/commitment, not the current live candidate as if it were the old invocation. + +This predicate gates ALL parent success/failure/canceled terminal SQL writes, private contained-worker completion proof, root effect settlement, release, terminal cleanup and retirement. Check the current DB child union and original signed worker evidence before each finalization/release, including the startup path for an already terminal SQL row and restart from a completed worker receipt. A pending obligation retains parent `publication:"unknown"`/recovery-required ownership, no final timestamp or successful cleanup, and the ORIGINAL target lease. Factual local-worker containment and already committed local writes may still be durably recorded, but with the unresolved recovery obligation and unknown aggregate publication; they cannot assert known parent completion or erase committed-write evidence. Do not mint a known root settlement from a completed local file-write observation while recovery is pending. Older terminal rows or inconsistent DB/private records remain fenced rather than released. Store CAS cannot erase or replace the triggering/restore child record; private completion's child digest and current records must agree. + +Restart may reconcile these original child receipts and copy the same verified evidence, but never executes another restore/reload or chooses a new key/deadline. Cancel/expiry can prohibit forward restore without clearing the obligation. Only actual original successful restore evidence permits the original failed parent to settle/release truthfully; the parent still reports its original pipeline failure. Unresolved recovery is outside72h terminal retirement and remains visible for recovery. + +## Required proof + +Use the actual production resource claimer, real hardlinked candidate layout and actual DM runtime/owned-child adapter. Substitute only external Docker IO; use a real controlled process for the wrapper/containment mechanism. Causal RED shows known invalid nginx candidate leaves failed child unresolved and prevents restore. GREEN proves one completed validation, zero reloads for that failed child, exact contained/not_started signed settlement, original-root restore once and truthful failed parent. + +Negative cases: timeout/abort/transport exception, missing/forged/repeated/wrong nonce or bad exit, output truncation/uncontained process, foreign execution/candidate or replaced inode/bytes, failure publication after reload_started, reused commitment and reload failure. These remain unknown, with no automatic restore/release. Test stage-validation failure followed by restore-validation failure, then restart: both failed children remain known/contained, parent remains recovery-required with no final timestamp, same target inode remains held and no new dispatch occurs. Also test stage failure plus successful original restore permits one release while parent remains failed. Test crash after restore reservation/before files and after files/before validation, missing/mismatched restore evidence, cancel/expiry before restore, crash before proof and around canonical publication, exact restart/ack/retirement, no replay/time renewal and no generic-error containment. Existing successful validation→reload and all other operations retain behavior. + +Controller self-review: this is a finite actual-producer proof, not general failure settlement. The fixed wrapper separates nginx status from Docker transport completion while retaining current process containment. Private and public evidence remain distinct. Code/runtime/platform qualification is still required. From 54cbe85916d93c35838677b864eeb71ec471107e Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 22:20:50 +0100 Subject: [PATCH 019/135] feat(data): retain durable execution and cancellation ownership --- .../migrations/0042_admin_data_execution.sql | 18 + .../src/db/migrations/meta/0042_snapshot.json | 8122 ++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + .../routes/__tests__/admin-activity.test.ts | 30 +- apps/api/src/routes/admin-activity.ts | 46 +- .../__tests__/admin-data-children.test.ts | 208 + .../__tests__/job-runner-cancel.test.ts | 10 +- .../__tests__/job-runner-postgres.test.ts | 1755 +- apps/api/src/services/activity-jobs.ts | 85 +- apps/api/src/services/admin-data-children.ts | 231 + apps/api/src/services/admin-job-ops.ts | 65 +- apps/api/src/services/admin-job-state.ts | 19 +- apps/ops-agent/src/administrative-runtime.ts | 110 +- apps/ops-agent/src/control-admission.test.ts | 102 +- .../src/data-transit-resource-source.test.ts | 503 + .../src/data-transit-resource-source.ts | 39 + apps/ops-agent/src/docker-runtime.ts | 78 +- .../src/feed-proxy-validation.test.ts | 107 + apps/ops-agent/src/feed-proxy-validation.ts | 46 + apps/ops-agent/src/index.ts | 16 +- apps/ops-agent/src/runtime.ts | 8 + apps/ops-agent/src/server.ts | 161 +- .../test-fixtures/data-owned-http-process.ts | 169 + .../src/traffic-target-ownership.test.ts | 154 + .../src/transitous-lock-runtime.test.ts | 15 +- apps/transitous-runner/src/runner.test.ts | 4 +- apps/transitous-runner/src/runner.ts | 94 +- apps/transitous-runner/src/server.ts | 14 + .../2026-10-25-admin-late-child-binding.md | 2 +- ...6-10-25-admin-late-child-binding-design.md | 4 + .../cli/__tests__/cli-data-execution.test.ts | 25 + .../fixtures/single-data-cli-process.ts | 15 + packages/cli/src/commands/data.ts | 186 +- packages/cli/src/lib/control-context.ts | 14 +- packages/cli/src/lib/data-execution.ts | 302 + packages/cli/src/lib/execution-deadline.ts | 3 +- .../src/build-artifacts/child-phase.test.ts | 776 + .../core/src/build-artifacts/child-phase.ts | 562 + .../src/build-artifacts/control-authority.ts | 666 +- .../src/build-artifacts/control-records.ts | 12 +- .../src/build-artifacts/control-retention.ts | 12 +- .../data-configuration-fence.ts | 935 + .../src/build-artifacts/data-database-time.ts | 18 + .../data-execution-candidate.ts | 566 + .../build-artifacts/data-execution-journal.ts | 1075 + .../build-artifacts/data-execution-outcome.ts | 14 + .../build-artifacts/feed-proxy-validation.ts | 419 + .../core/src/build-artifacts/filesystem.ts | 4 +- .../build-artifacts/fixed-data-execution.ts | 282 + packages/core/src/build-artifacts/index.ts | 55 +- .../src/build-artifacts/peer-admission.ts | 90 + .../src/build-artifacts/peer-process.test.ts | 9 +- .../src/build-artifacts/retention-family.ts | 60 +- .../core/src/build-artifacts/target-lease.ts | 542 +- .../test-fixtures/child-phase-process.ts | 30 + .../core/src/ops/catalog-pin-approval.test.ts | 77 + packages/core/src/ops/catalog-pin-approval.ts | 30 + packages/core/src/ops/contract.test.ts | 7 +- packages/core/src/ops/contract.ts | 19 +- packages/core/src/ops/execution-identity.ts | 19 +- packages/core/src/ops/execution-ref.ts | 19 + packages/core/src/ops/index.ts | 1 + packages/core/src/ops/service-results.ts | 10 + .../data-manager/configuration-maintenance.ts | 63 + .../contracts/configuration-maintenance.ts | 44 + .../data-manager/contracts/executions.ts | 205 + .../services/data-manager/contracts/index.ts | 2 + .../data-manager/contracts/operations.ts | 168 +- .../services/data-manager/executions.test.ts | 142 + .../src/services/data-manager/executions.ts | 120 + .../generated/data-manager.generated.ts | 1256 ++ .../generated/models/addTransitSource503.ts | 20 + .../generated/models/approveTransitBump409.ts | 20 + .../generated/models/approveTransitBump503.ts | 20 + .../models/bakePredictedTraffic503.ts | 20 + .../generated/models/buildAmbientPlaces503.ts | 20 + .../generated/models/buildNotablePlaces409.ts | 20 + .../generated/models/buildNotablePlaces503.ts | 20 + .../generated/models/buildSearchIndex409.ts | 20 + .../generated/models/buildSearchIndex503.ts | 20 + .../models/cancelDataExecution200.ts | 33 + .../cancelDataExecution200Diagnostics.ts | 9 + ...ncelDataExecution200DiagnosticsTailItem.ts | 8 + ...taExecution200DiagnosticsTailItemStream.ts | 10 + .../models/cancelDataExecution200Kind.ts | 21 + .../cancelDataExecution200Publication.ts | 11 + .../models/cancelDataExecution200Result.ts | 355 + .../models/cancelDataExecution200Status.ts | 16 + .../models/cancelDataExecution400.ts | 20 + .../models/cancelDataExecution401.ts | 20 + .../models/cancelDataExecution404.ts | 20 + .../models/cancelDataExecution409.ts | 20 + .../models/cancelDataExecution413.ts | 20 + .../models/cancelDataExecution415.ts | 20 + .../models/cancelDataExecution429.ts | 20 + .../models/cancelDataExecution500.ts | 20 + .../models/cancelDataExecution503.ts | 20 + .../models/cancelDataExecutionBody.ts | 9 + .../generated/models/conflateOverture409.ts | 20 + .../generated/models/conflateOverture503.ts | 20 + .../generated/models/convertOverpass409.ts | 20 + .../generated/models/convertOverpass503.ts | 20 + .../models/discardAmbientPlaces503.ts | 20 + .../generated/models/downloadFonts409.ts | 20 + .../generated/models/downloadFonts503.ts | 20 + .../generated/models/downloadOsm409.ts | 20 + .../generated/models/downloadOsm503.ts | 20 + .../models/enableTransitSource503.ts | 20 + .../generated/models/extractOverture409.ts | 20 + .../generated/models/extractOverture503.ts | 20 + .../generated/models/getDataExecution200.ts | 33 + .../models/getDataExecution200Diagnostics.ts | 9 + .../getDataExecution200DiagnosticsTailItem.ts | 8 + ...taExecution200DiagnosticsTailItemStream.ts | 10 + .../models/getDataExecution200Kind.ts | 21 + .../models/getDataExecution200Publication.ts | 11 + .../models/getDataExecution200Result.ts | 355 + .../models/getDataExecution200Status.ts | 16 + .../generated/models/getDataExecution400.ts | 20 + .../generated/models/getDataExecution401.ts | 20 + .../generated/models/getDataExecution404.ts | 20 + .../generated/models/getDataExecution410.ts | 20 + .../generated/models/getDataExecution413.ts | 20 + .../generated/models/getDataExecution415.ts | 20 + .../generated/models/getDataExecution429.ts | 20 + .../generated/models/getDataExecution500.ts | 20 + .../generated/models/getDataExecution503.ts | 20 + .../generated/models/getOfflineGlyph409.ts | 20 + .../generated/models/getOfflineGlyph503.ts | 20 + .../models/getOfflineGlyphCatalog409.ts | 20 + .../models/getOfflineGlyphCatalog503.ts | 20 + .../models/getOfflinePackageArchive409.ts | 20 + .../models/getOfflinePackageArchive503.ts | 20 + .../models/getOfflinePackageJob409.ts | 20 + .../models/getOfflinePackageJob503.ts | 20 + .../models/getOfflinePackageManifest409.ts | 20 + .../models/getOfflinePackageManifest503.ts | 20 + .../data-manager/generated/models/index.ts | 195 +- .../generated/models/ingestOverture409.ts | 20 + .../generated/models/ingestOverture503.ts | 20 + .../inspectDataConfigurationFence200.ts | 25 + ...ctDataConfigurationFence200OwnerPrepare.ts | 21 + ...nfigurationFence200OwnerPrepareExecutor.ts | 11 + ...figurationFence200OwnerPreparePrincipal.ts | 11 + .../inspectDataConfigurationFence200State.ts | 11 + .../inspectDataConfigurationFence400.ts | 20 + .../inspectDataConfigurationFence401.ts | 20 + .../inspectDataConfigurationFence403.ts | 20 + .../inspectDataConfigurationFence404.ts | 20 + .../inspectDataConfigurationFence413.ts | 20 + .../inspectDataConfigurationFence415.ts | 20 + .../inspectDataConfigurationFence429.ts | 20 + .../inspectDataConfigurationFence500.ts | 20 + .../inspectDataConfigurationFence503.ts | 20 + .../data-manager/generated/models/link409.ts | 20 + .../data-manager/generated/models/link503.ts | 20 + .../models/lookupDataExecution200.ts | 33 + .../lookupDataExecution200Diagnostics.ts | 9 + ...okupDataExecution200DiagnosticsTailItem.ts | 8 + ...taExecution200DiagnosticsTailItemStream.ts | 10 + .../models/lookupDataExecution200Kind.ts | 21 + .../lookupDataExecution200Publication.ts | 11 + .../models/lookupDataExecution200Result.ts | 355 + .../models/lookupDataExecution200Status.ts | 16 + .../models/lookupDataExecution400.ts | 20 + .../models/lookupDataExecution401.ts | 20 + .../models/lookupDataExecution404.ts | 20 + .../models/lookupDataExecution410.ts | 20 + .../models/lookupDataExecution413.ts | 20 + .../models/lookupDataExecution415.ts | 20 + .../models/lookupDataExecution429.ts | 20 + .../models/lookupDataExecution500.ts | 20 + .../models/lookupDataExecution503.ts | 20 + .../generated/models/preflightTransit409.ts | 20 + .../generated/models/preflightTransit503.ts | 20 + .../prepareDataConfigurationFence200.ts | 25 + ...reDataConfigurationFence200OwnerPrepare.ts | 21 + ...nfigurationFence200OwnerPrepareExecutor.ts | 11 + ...figurationFence200OwnerPreparePrincipal.ts | 11 + .../prepareDataConfigurationFence200State.ts | 11 + .../prepareDataConfigurationFence400.ts | 20 + .../prepareDataConfigurationFence401.ts | 20 + .../prepareDataConfigurationFence403.ts | 20 + .../prepareDataConfigurationFence409.ts | 20 + .../prepareDataConfigurationFence413.ts | 20 + .../prepareDataConfigurationFence415.ts | 20 + .../prepareDataConfigurationFence429.ts | 20 + .../prepareDataConfigurationFence500.ts | 20 + .../prepareDataConfigurationFence503.ts | 20 + .../prepareDataConfigurationFenceBody.ts | 24 + ...taConfigurationFenceBodyCoordinatorMode.ts | 10 + ...eDataConfigurationFenceBodyOwnerPrepare.ts | 21 + ...figurationFenceBodyOwnerPrepareExecutor.ts | 11 + ...igurationFenceBodyOwnerPreparePrincipal.ts | 11 + .../generated/models/proposeTransitBump503.ts | 20 + .../generated/models/pullOverture409.ts | 20 + .../generated/models/pullOverture503.ts | 20 + .../generated/models/reloadDatasets409.ts | 20 + .../generated/models/reloadDatasets503.ts | 20 + .../models/removeTransitSource503.ts | 20 + .../generated/models/restartMotis409.ts | 20 + .../models/resumeAmbientPlaces503.ts | 20 + .../models/rollbackAmbientPlaces503.ts | 20 + .../models/setAmbientPlacesEnabled503.ts | 20 + .../models/settleDataConfigurationFence200.ts | 25 + ...leDataConfigurationFence200OwnerPrepare.ts | 21 + ...nfigurationFence200OwnerPrepareExecutor.ts | 11 + ...figurationFence200OwnerPreparePrincipal.ts | 11 + .../settleDataConfigurationFence200State.ts | 11 + .../models/settleDataConfigurationFence400.ts | 20 + .../models/settleDataConfigurationFence401.ts | 20 + .../models/settleDataConfigurationFence403.ts | 20 + .../models/settleDataConfigurationFence404.ts | 20 + .../models/settleDataConfigurationFence409.ts | 20 + .../models/settleDataConfigurationFence413.ts | 20 + .../models/settleDataConfigurationFence415.ts | 20 + .../models/settleDataConfigurationFence429.ts | 20 + .../models/settleDataConfigurationFence500.ts | 20 + .../models/settleDataConfigurationFence503.ts | 20 + .../settleDataConfigurationFenceBody.ts | 22 + ...settleDataConfigurationFenceBodyOutcome.ts | 11 + .../generated/models/startDataExecution202.ts | 33 + .../startDataExecution202Diagnostics.ts | 9 + ...tartDataExecution202DiagnosticsTailItem.ts | 8 + ...taExecution202DiagnosticsTailItemStream.ts | 10 + .../models/startDataExecution202Kind.ts | 21 + .../startDataExecution202Publication.ts | 11 + .../models/startDataExecution202Result.ts | 355 + .../models/startDataExecution202Status.ts | 16 + .../generated/models/startDataExecution400.ts | 20 + .../generated/models/startDataExecution401.ts | 20 + .../generated/models/startDataExecution403.ts | 20 + .../generated/models/startDataExecution409.ts | 20 + .../generated/models/startDataExecution413.ts | 20 + .../generated/models/startDataExecution415.ts | 20 + .../generated/models/startDataExecution429.ts | 20 + .../generated/models/startDataExecution500.ts | 20 + .../generated/models/startDataExecution503.ts | 20 + .../models/startDataExecutionBody.ts | 14 + .../models/startDataExecutionBodyInput.ts | 91 + .../startGenerationHardlinksExecution202.ts | 33 + ...erationHardlinksExecution202Diagnostics.ts | 9 + ...ardlinksExecution202DiagnosticsTailItem.ts | 8 + ...ksExecution202DiagnosticsTailItemStream.ts | 10 + ...tartGenerationHardlinksExecution202Kind.ts | 21 + ...erationHardlinksExecution202Publication.ts | 11 + ...rtGenerationHardlinksExecution202Result.ts | 355 + ...rtGenerationHardlinksExecution202Status.ts | 16 + .../startGenerationHardlinksExecution400.ts | 20 + .../startGenerationHardlinksExecution401.ts | 20 + .../startGenerationHardlinksExecution403.ts | 20 + .../startGenerationHardlinksExecution409.ts | 20 + .../startGenerationHardlinksExecution413.ts | 20 + .../startGenerationHardlinksExecution415.ts | 20 + .../startGenerationHardlinksExecution429.ts | 20 + .../startGenerationHardlinksExecution500.ts | 20 + .../startGenerationHardlinksExecution503.ts | 20 + .../startGenerationHardlinksExecutionBody.ts | 16 + ...rtGenerationHardlinksExecutionBodyInput.ts | 13 + ...dlinksExecutionBodyInputCoordinatorMode.ts | 10 + ...nerationHardlinksExecutionBodyInputKind.ts | 9 + ...artGenerationHardlinksExecutionBodyPlan.ts | 9 + ...ationHardlinksExecutionBodyPlanPlanItem.ts | 23 + .../generated/models/syncOverture409.ts | 20 + .../generated/models/syncOverture503.ts | 20 + .../generated/models/syncTransit409.ts | 30 +- .../generated/models/syncTransit409Reason.ts | 9 - .../generated/models/syncTransit503.ts | 20 + .../generated/operation-bindings.generated.ts | 62 + .../core/src/services/data-manager/index.ts | 19 + .../core/src/transitous-runner/contract.ts | 10 + packages/core/src/transitous-runner/index.ts | 1 + packages/db-schema/src/data-manager.ts | 79 +- .../durable-data-execution-storage.md | 23 + scripts/database-ci-contract.test.ts | 4 + scripts/database-test-groups.json | 8 + scripts/test-database.mjs | 13 +- .../ambient-places/api-postgres.test.ts | 3 + services/data-manager/__tests__/api.test.ts | 181 +- .../__tests__/atomic-download-cancel.test.ts | 72 + services/data-manager/__tests__/cron.test.ts | 160 +- .../helpers/data-admission-fixture.ts | 42 + .../__tests__/helpers/execution-receiver.ts | 40 + .../__tests__/helpers/owned-transit-unit.ts | 44 + .../__tests__/job-persistence.test.ts | 40 + .../__tests__/notable-places/schedule.test.ts | 16 +- ...ffline-package-postgres-accounting.test.ts | 4 + .../__tests__/ops-client-ownership.test.ts | 53 + .../__tests__/overture/overture-cron.test.ts | 491 +- .../data-manager/__tests__/reconcile.test.ts | 10 +- .../__tests__/transitous/motis-import.test.ts | 7 + .../__tests__/transitous/pipeline.test.ts | 49 + .../__tests__/transitous/promote.test.ts | 136 +- .../transitous/proxy-transaction.test.ts | 7 + services/data-manager/openapi.json | 16820 ++++++++++++++-- services/data-manager/src/api.ts | 550 +- services/data-manager/src/auth.ts | 22 +- services/data-manager/src/cron.ts | 515 +- services/data-manager/src/db/index.ts | 4 +- .../src/jobs/ambient-places/api.ts | 40 +- .../data-manager/src/jobs/atomic-download.ts | 38 +- .../data-manager/src/jobs/convert-overpass.ts | 28 +- .../data-manager/src/jobs/download-fonts.ts | 46 +- .../data-manager/src/jobs/download-osm.ts | 24 +- .../src/jobs/execution/admission.ts | 17 + .../src/jobs/execution/api.test.ts | 51 + .../data-manager/src/jobs/execution/api.ts | 109 + .../configuration-maintenance.test.ts | 686 + .../execution/configuration-maintenance.ts | 22 + .../src/jobs/execution/diagnostics.test.ts | 16 + .../src/jobs/execution/diagnostics.ts | 19 + .../src/jobs/execution/dispatch.ts | 349 + .../src/jobs/execution/fixed-cli.test.ts | 251 + .../src/jobs/execution/hardlinks.test.ts | 310 + .../src/jobs/execution/hardlinks.ts | 30 + .../data-manager/src/jobs/execution/index.ts | 23 + .../src/jobs/execution/legacy-mutation.ts | 93 + .../src/jobs/execution/ops-children.test.ts | 1124 ++ .../src/jobs/execution/ops-children.ts | 645 + .../src/jobs/execution/receiver.ts | 117 + .../execution/runner-cancellation.test.ts | 130 + .../src/jobs/execution/runtime.test.ts | 618 + .../src/jobs/execution/runtime.ts | 542 + .../src/jobs/execution/store-postgres.test.ts | 271 + .../src/jobs/execution/store.test.ts | 29 + .../data-manager/src/jobs/execution/store.ts | 645 + .../src/jobs/execution/worker-scope.ts | 180 + .../src/jobs/notable-places/build.ts | 42 +- .../src/jobs/notable-places/schedule.ts | 16 +- .../src/jobs/overture/conflate.ts | 17 +- .../data-manager/src/jobs/overture/duckdb.ts | 3 +- .../src/jobs/overture/extract-osm-pois.ts | 42 +- .../src/jobs/overture/feed-state.ts | 48 + .../data-manager/src/jobs/overture/ingest.ts | 43 +- .../src/jobs/overture/operation-lock.ts | 7 +- .../data-manager/src/jobs/overture/pull.ts | 15 +- .../src/jobs/overture/retention.ts | 5 +- services/data-manager/src/jobs/persistence.ts | 40 +- services/data-manager/src/jobs/reconcile.ts | 46 +- .../src/jobs/search-index/build.ts | 31 +- .../src/jobs/search-index/extract.ts | 3 + .../src/jobs/search-index/operation-lock.ts | 7 +- .../src/jobs/search-index/state.ts | 11 +- .../src/jobs/transitous/functional-probes.ts | 14 +- .../src/jobs/transitous/internal.ts | 7 +- .../src/jobs/transitous/motis-import.ts | 7 +- .../src/jobs/transitous/motis-probe.ts | 19 +- .../src/jobs/transitous/owned-ops.ts | 67 + .../src/jobs/transitous/pipeline.ts | 13 +- .../src/jobs/transitous/promote.ts | 135 +- .../src/jobs/transitous/proxy-transaction.ts | 81 +- .../src/jobs/transitous/runtime.ts | 4 +- .../src/jobs/transitous/script-runner.ts | 131 +- .../src/jobs/transitous/single-flight.ts | 109 +- .../data-manager/src/jobs/transitous/types.ts | 1 + .../offline-packages/postgres-accounting.ts | 2 + services/data-manager/src/ops-client.ts | 166 +- services/data-manager/src/service.ts | 94 +- 358 files changed, 48400 insertions(+), 5390 deletions(-) create mode 100644 apps/api/src/db/migrations/0042_admin_data_execution.sql create mode 100644 apps/api/src/db/migrations/meta/0042_snapshot.json create mode 100644 apps/api/src/services/__tests__/admin-data-children.test.ts create mode 100644 apps/api/src/services/admin-data-children.ts create mode 100644 apps/ops-agent/src/data-transit-resource-source.test.ts create mode 100644 apps/ops-agent/src/data-transit-resource-source.ts create mode 100644 apps/ops-agent/src/feed-proxy-validation.test.ts create mode 100644 apps/ops-agent/src/feed-proxy-validation.ts create mode 100644 apps/ops-agent/src/test-fixtures/data-owned-http-process.ts create mode 100644 apps/ops-agent/src/traffic-target-ownership.test.ts create mode 100644 packages/cli/__tests__/cli-data-execution.test.ts create mode 100644 packages/cli/__tests__/fixtures/single-data-cli-process.ts create mode 100644 packages/cli/src/lib/data-execution.ts create mode 100644 packages/core/src/build-artifacts/child-phase.test.ts create mode 100644 packages/core/src/build-artifacts/child-phase.ts create mode 100644 packages/core/src/build-artifacts/data-configuration-fence.ts create mode 100644 packages/core/src/build-artifacts/data-database-time.ts create mode 100644 packages/core/src/build-artifacts/data-execution-candidate.ts create mode 100644 packages/core/src/build-artifacts/data-execution-journal.ts create mode 100644 packages/core/src/build-artifacts/data-execution-outcome.ts create mode 100644 packages/core/src/build-artifacts/feed-proxy-validation.ts create mode 100644 packages/core/src/build-artifacts/fixed-data-execution.ts create mode 100644 packages/core/src/build-artifacts/test-fixtures/child-phase-process.ts create mode 100644 packages/core/src/ops/catalog-pin-approval.test.ts create mode 100644 packages/core/src/ops/catalog-pin-approval.ts create mode 100644 packages/core/src/ops/execution-ref.ts create mode 100644 packages/core/src/services/data-manager/configuration-maintenance.ts create mode 100644 packages/core/src/services/data-manager/contracts/configuration-maintenance.ts create mode 100644 packages/core/src/services/data-manager/contracts/executions.ts create mode 100644 packages/core/src/services/data-manager/executions.test.ts create mode 100644 packages/core/src/services/data-manager/executions.ts create mode 100644 packages/core/src/services/data-manager/generated/models/addTransitSource503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/approveTransitBump409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/approveTransitBump503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/bakePredictedTraffic503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/buildAmbientPlaces503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/buildNotablePlaces409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/buildNotablePlaces503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/buildSearchIndex409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/buildSearchIndex503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200Diagnostics.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItemStream.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200Publication.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution200Status.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution404.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecution503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/cancelDataExecutionBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/conflateOverture409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/conflateOverture503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/convertOverpass409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/convertOverpass503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/discardAmbientPlaces503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/downloadFonts409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/downloadFonts503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/downloadOsm409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/downloadOsm503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/enableTransitSource503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/extractOverture409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/extractOverture503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200Diagnostics.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItemStream.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200Publication.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution200Status.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution404.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution410.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getDataExecution503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflineGlyph409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflineGlyph503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflinePackageJob409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflinePackageJob503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/ingestOverture409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/ingestOverture503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepare.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepareExecutor.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPreparePrincipal.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200State.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence403.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence404.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/link409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/link503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200Diagnostics.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItemStream.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200Publication.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution200Status.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution404.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution410.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/lookupDataExecution503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/preflightTransit409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/preflightTransit503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepare.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepareExecutor.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPreparePrincipal.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200State.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence403.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyCoordinatorMode.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepare.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepareExecutor.ts create mode 100644 packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPreparePrincipal.ts create mode 100644 packages/core/src/services/data-manager/generated/models/proposeTransitBump503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/pullOverture409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/pullOverture503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/reloadDatasets409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/reloadDatasets503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/removeTransitSource503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/restartMotis409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/resumeAmbientPlaces503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/rollbackAmbientPlaces503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/setAmbientPlacesEnabled503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepare.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepareExecutor.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPreparePrincipal.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200State.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence403.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence404.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202Diagnostics.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItemStream.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202Publication.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution202Status.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution403.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecution503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecutionBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Diagnostics.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItemStream.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Publication.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Status.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution403.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInput.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputCoordinatorMode.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputKind.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlan.ts create mode 100644 packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlanPlanItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/syncOverture409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/syncOverture503.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/syncTransit409Reason.ts create mode 100644 packages/core/src/services/data-manager/generated/models/syncTransit503.ts create mode 100644 release-notes/durable-data-execution-storage.md create mode 100644 services/data-manager/__tests__/atomic-download-cancel.test.ts create mode 100644 services/data-manager/__tests__/helpers/data-admission-fixture.ts create mode 100644 services/data-manager/__tests__/helpers/execution-receiver.ts create mode 100644 services/data-manager/__tests__/helpers/owned-transit-unit.ts create mode 100644 services/data-manager/__tests__/ops-client-ownership.test.ts create mode 100644 services/data-manager/src/jobs/execution/admission.ts create mode 100644 services/data-manager/src/jobs/execution/api.test.ts create mode 100644 services/data-manager/src/jobs/execution/api.ts create mode 100644 services/data-manager/src/jobs/execution/configuration-maintenance.test.ts create mode 100644 services/data-manager/src/jobs/execution/configuration-maintenance.ts create mode 100644 services/data-manager/src/jobs/execution/diagnostics.test.ts create mode 100644 services/data-manager/src/jobs/execution/diagnostics.ts create mode 100644 services/data-manager/src/jobs/execution/dispatch.ts create mode 100644 services/data-manager/src/jobs/execution/fixed-cli.test.ts create mode 100644 services/data-manager/src/jobs/execution/hardlinks.test.ts create mode 100644 services/data-manager/src/jobs/execution/hardlinks.ts create mode 100644 services/data-manager/src/jobs/execution/index.ts create mode 100644 services/data-manager/src/jobs/execution/legacy-mutation.ts create mode 100644 services/data-manager/src/jobs/execution/ops-children.test.ts create mode 100644 services/data-manager/src/jobs/execution/ops-children.ts create mode 100644 services/data-manager/src/jobs/execution/receiver.ts create mode 100644 services/data-manager/src/jobs/execution/runner-cancellation.test.ts create mode 100644 services/data-manager/src/jobs/execution/runtime.test.ts create mode 100644 services/data-manager/src/jobs/execution/runtime.ts create mode 100644 services/data-manager/src/jobs/execution/store-postgres.test.ts create mode 100644 services/data-manager/src/jobs/execution/store.test.ts create mode 100644 services/data-manager/src/jobs/execution/store.ts create mode 100644 services/data-manager/src/jobs/execution/worker-scope.ts create mode 100644 services/data-manager/src/jobs/overture/feed-state.ts create mode 100644 services/data-manager/src/jobs/transitous/owned-ops.ts diff --git a/apps/api/src/db/migrations/0042_admin_data_execution.sql b/apps/api/src/db/migrations/0042_admin_data_execution.sql new file mode 100644 index 000000000..b24995b08 --- /dev/null +++ b/apps/api/src/db/migrations/0042_admin_data_execution.sql @@ -0,0 +1,18 @@ +ALTER TABLE "data_manager"."jobs" ADD COLUMN "execution_version" integer;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "operation_key" text;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "operation_fingerprint" text;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "execution_fingerprint" text;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "admitted_at" bigint;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "deadline_at" bigint;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "requested_deadline_at" bigint;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "execution_parent" jsonb;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "execution_input" jsonb;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "execution_state" jsonb;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "execution_children" jsonb DEFAULT '[]'::jsonb NOT NULL;--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD COLUMN "ownership_version" integer DEFAULT 0 NOT NULL;--> statement-breakpoint +CREATE UNIQUE INDEX "data_manager_jobs_execution_key_uq" ON "data_manager"."jobs" USING btree ("operation_key") WHERE "data_manager"."jobs"."operation_key" IS NOT NULL;--> statement-breakpoint +CREATE UNIQUE INDEX "data_manager_jobs_configuration_maintenance_uq" ON "data_manager"."jobs" USING btree (("metadata"->'configurationMaintenance'->'receipt'->>'maintenanceId')) WHERE "data_manager"."jobs"."kind" = 'configuration-maintenance';--> statement-breakpoint +CREATE UNIQUE INDEX "data_manager_jobs_configuration_guard_uq" ON "data_manager"."jobs" USING btree ("kind") WHERE "data_manager"."jobs"."kind" = 'configuration-maintenance' AND coalesce("data_manager"."jobs"."metadata"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled';--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD CONSTRAINT "data_manager_jobs_execution_bounds" CHECK ("data_manager"."jobs"."ownership_version" >= 0 AND jsonb_typeof("data_manager"."jobs"."execution_children") = 'array' AND jsonb_array_length("data_manager"."jobs"."execution_children") <= 32 AND octet_length("data_manager"."jobs"."execution_children"::text) <= 131072);--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD CONSTRAINT "data_manager_jobs_execution_shape" CHECK ("data_manager"."jobs"."execution_version" IS NULL OR coalesce(("data_manager"."jobs"."execution_version" = 2 AND "data_manager"."jobs"."operation_key" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND "data_manager"."jobs"."operation_fingerprint" ~ '^[a-f0-9]{64}$' AND "data_manager"."jobs"."execution_fingerprint" ~ '^[a-f0-9]{64}$' AND "data_manager"."jobs"."admitted_at" >= 0 AND "data_manager"."jobs"."deadline_at" > "data_manager"."jobs"."admitted_at" AND "data_manager"."jobs"."execution_input" IS NOT NULL AND "data_manager"."jobs"."execution_state" IS NOT NULL), false));--> statement-breakpoint +ALTER TABLE "data_manager"."jobs" ADD CONSTRAINT "data_manager_jobs_configuration_guard_shape" CHECK ("data_manager"."jobs"."kind" <> 'configuration-maintenance' OR coalesce((jsonb_typeof("data_manager"."jobs"."metadata"->'configurationMaintenance') = 'object' AND "data_manager"."jobs"."metadata"->'configurationMaintenance'->>'version' = '1' AND "data_manager"."jobs"."metadata"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof("data_manager"."jobs"."metadata"->'configurationMaintenance'->'receipt') = 'object' AND octet_length(("data_manager"."jobs"."metadata"->'configurationMaintenance')::text) <= 65536), false)); \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0042_snapshot.json b/apps/api/src/db/migrations/meta/0042_snapshot.json new file mode 100644 index 000000000..99a185e7b --- /dev/null +++ b/apps/api/src/db/migrations/meta/0042_snapshot.json @@ -0,0 +1,8122 @@ +{ + "id": "ea53844f-257b-4be6-aefc-34ee5145df6c", + "prevId": "20f25e34-cac1-4d8d-aebc-ddf2c05a6b72", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 30fc5470e..28d0c3b15 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -295,6 +295,13 @@ "when": 1792617704664, "tag": "0041_community_repository_transactions", "breakpoints": true + }, + { + "idx": 42, + "version": "7", + "when": 1792939089650, + "tag": "0042_admin_data_execution", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/routes/__tests__/admin-activity.test.ts b/apps/api/src/routes/__tests__/admin-activity.test.ts index 7cc883106..6ce932355 100644 --- a/apps/api/src/routes/__tests__/admin-activity.test.ts +++ b/apps/api/src/routes/__tests__/admin-activity.test.ts @@ -11,9 +11,11 @@ const mocks = installAdminRouteMocks(); const list = vi.fn(); const detail = vi.fn(); const cancel = vi.fn(); +const cancelData = vi.fn(); vi.mock("../../services/activity-jobs", () => ({ listActivityJobs: (...args: unknown[]) => list(...args), getActivityJob: (...args: unknown[]) => detail(...args), + cancelDataManagerActivityJob: (...args: unknown[]) => cancelData(...args), })); vi.mock("../../services/job-runner", () => ({ jobRunner: { cancel: (...args: unknown[]) => cancel(...args) }, @@ -65,6 +67,7 @@ beforeEach(() => { list.mockReset().mockResolvedValue({ jobs: [], total: 0 }); detail.mockReset().mockResolvedValue(null); cancel.mockReset().mockResolvedValue(true); + cancelData.mockReset().mockResolvedValue(false); }); afterEach(() => vi.unstubAllGlobals()); @@ -237,6 +240,28 @@ describe("actual generated Activity SDK through Fastify serialization", () => { await app.close(); } }); + it("sends supported data cancellation through the authenticated receiver and audits its distinct identity", async () => { + cancelData.mockResolvedValueOnce(true); + const app = await createAdminTestApp(adminActivityRoute); + try { + const response = await app.inject({ + method: "POST", + url: "/admin/jobs/durable-data-id/cancel?source=data-manager", + }); + expect(response.statusCode).toBe(200); + expect(cancelData).toHaveBeenCalledWith("durable-data-id"); + expect(cancel).not.toHaveBeenCalled(); + expect(mocks.writeAuditLog).toHaveBeenCalledWith( + expect.objectContaining({ + targetId: "durable-data-id", + targetType: "data-manager-job", + action: "job.cancel", + }), + ); + } finally { + await app.close(); + } + }); it("validates actual 400/401/403/404/429/500 JSON and Retry-After", async () => { const sdk = await import("@openmapx/core/api-admin-sdk"); expect(sdk.listAdminActivityJobs).toBeTypeOf("function"); @@ -256,7 +281,10 @@ describe("actual generated Activity SDK through Fastify serialization", () => { } expect( await sdk.cancelAdminActivityJob("job-1", { source: "data-manager" }, { client }), - ).toMatchObject({ status: 400, data: { error: "Data-manager jobs cannot be canceled" } }); + ).toMatchObject({ + status: 400, + data: { error: "Job cannot be canceled (not running or queued)" }, + }); expect(await sdk.getAdminActivityJob("missing", undefined, { client })).toMatchObject({ status: 404, data: { error: "Job not found" }, diff --git a/apps/api/src/routes/admin-activity.ts b/apps/api/src/routes/admin-activity.ts index a9abe5430..8d6f3cc1a 100644 --- a/apps/api/src/routes/admin-activity.ts +++ b/apps/api/src/routes/admin-activity.ts @@ -1,8 +1,13 @@ import { activityAdminOperations } from "@openmapx/core/api-contracts"; import type { FastifyInstance } from "fastify"; -import { getActivityJob, listActivityJobs } from "../services/activity-jobs"; +import { + cancelDataManagerActivityJob, + getActivityJob, + listActivityJobs, +} from "../services/activity-jobs"; import { jobRunner } from "../services/job-runner"; import { writeAuditLog } from "../utils/audit-log"; +import { RateLimiter } from "../utils/rate-limit"; import { getAdminSession, requireAdmin } from "../utils/require-admin"; import { declareRouteAuth } from "../utils/route-auth"; import { createAdminRouteRegistrar } from "./admin-contract"; @@ -12,6 +17,8 @@ export const registerActivityAdminRoute = createAdminRouteRegistrar(activityAdmi validationPhase: "preHandler", }); +const activityCancellationLimit = new RateLimiter({ max: 20, windowMs: 60000 }); + export async function adminActivityRoute(app: FastifyInstance): Promise { declareRouteAuth(app, "admin"); app.addHook("preHandler", async (request) => { @@ -29,19 +36,26 @@ export async function adminActivityRoute(app: FastifyInstance): Promise { if (!job) return reply.status(404).send({ error: "Job not found" }); return job; }); - registerActivityAdminRoute(app, "cancelAdminActivityJob", async (request, reply) => { - if (request.query.source === "data-manager") - return reply.status(400).send({ error: "Data-manager jobs cannot be canceled" }); - const adminSession = getAdminSession(request); - if (!(await jobRunner.cancel(request.params.id))) - return reply.status(400).send({ error: "Job cannot be canceled (not running or queued)" }); - await writeAuditLog({ - actorId: adminSession.user.id, - targetId: request.params.id, - targetType: "job", - action: "job.cancel", - request, - }); - return { ok: true }; - }); + registerActivityAdminRoute( + app, + "cancelAdminActivityJob", + async (request, reply) => { + const adminSession = getAdminSession(request); + const canceled = + request.query.source === "data-manager" + ? await cancelDataManagerActivityJob(request.params.id) + : await jobRunner.cancel(request.params.id); + if (!canceled) + return reply.status(400).send({ error: "Job cannot be canceled (not running or queued)" }); + await writeAuditLog({ + actorId: adminSession.user.id, + targetId: request.params.id, + targetType: request.query.source === "data-manager" ? "data-manager-job" : "job", + action: "job.cancel", + request, + }); + return { ok: true }; + }, + { preHandler: activityCancellationLimit.preHandler() }, + ); } diff --git a/apps/api/src/services/__tests__/admin-data-children.test.ts b/apps/api/src/services/__tests__/admin-data-children.test.ts new file mode 100644 index 000000000..4130f1d64 --- /dev/null +++ b/apps/api/src/services/__tests__/admin-data-children.test.ts @@ -0,0 +1,208 @@ +import { controlChildOperationKey } from "@openmapx/core/build-artifacts"; +import { afterEach, expect, it, vi } from "vitest"; +import type { JobContext } from "../job-runner"; + +const fixture = vi.hoisted(() => ({ rows: [] as Record[] })); +vi.mock("../../db", () => ({ + db: { select: () => ({ from: () => ({ where: () => ({ limit: async () => fixture.rows }) }) }) }, +})); + +import { cancelDataManagerActivityJob } from "../activity-jobs"; +import { cancelAdminDataChildren } from "../admin-data-children"; +import { hasUnsettledManagedOperation } from "../admin-job-state"; + +const key = "opk1_abcdefghijklmnop", + at = Date.now(), + deadline = at + 60000; +function child(index: number) { + return { + id: `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`, + operationKey: `opk1_abcdefghijklmnop${index}`, + fingerprint: String(index).repeat(64), + parentAgentOperationKey: key, + kind: "download-fonts" as const, + admittedAt: at, + deadlineAt: deadline, + cancellationRequestedAt: null, + state: "observing" as const, + status: "running" as const, + publication: "not_started" as const, + }; +} +function context(result: Record): JobContext { + return { + jobId: "fixture-job", + payload: {}, + signal: new AbortController().signal, + checkpointResult: result, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async function (value) { + this.checkpointResult = value; + }, + }; +} +afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + fixture.rows = []; +}); +it("keeps an interrupted unknown durable Activity execution explicitly cancelable", async () => { + vi.stubEnv("DATA_MANAGER_URL", "https://synthetic-dm.test"); + const owned = child(1); + fixture.rows = [ + { + id: owned.id, + kind: owned.kind, + executionVersion: 2, + status: "interrupted", + finishedAt: null, + executionState: { publication: "unknown" }, + executionInput: { kind: "download-fonts" }, + operationKey: owned.operationKey, + executionFingerprint: owned.fingerprint, + }, + ]; + const sent = vi.fn(async () => + Response.json({ + id: owned.id, + operationKey: owned.operationKey, + fingerprint: owned.fingerprint, + kind: owned.kind, + status: "cancel_pending", + admittedAt: at, + deadlineAt: deadline, + cancellationRequestedAt: new Date().toISOString(), + progress: null, + diagnostics: { tail: [], truncated: false }, + publication: "unknown", + }), + ); + vi.stubGlobal("fetch", sent); + expect(await cancelDataManagerActivityJob(owned.id)).toBe(true); + expect(sent).toHaveBeenCalledTimes(1); + fixture.rows[0]!.executionVersion = 1; + expect(await cancelDataManagerActivityJob(owned.id)).toBe(false); + expect(sent).toHaveBeenCalledTimes(1); +}); +it("cancels every registered backend with zero agent projections and retains pending after lost responses", async () => { + vi.stubEnv("DATA_MANAGER_URL", "https://synthetic-dm.test"); + const children = [child(1), child(2)], + ctx = context({ _cancelRequestedAt: new Date().toISOString(), _dataExecutions: children }), + sent: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, options?: RequestInit) => { + const id = url.split("/").at(-1) === "cancel" ? url.split("/").at(-2) : url.split("/").at(-1), + owned = children.find((child) => child.id === id); + if (!owned) throw Error("Foreign backend"); + if (options?.method === "POST") { + expect( + (ctx.checkpointResult?._dataExecutions as typeof children).find( + (child) => child.id === id, + )?.cancellationRequestedAt, + ).not.toBeNull(); + sent.push(id!); + throw Error("Lost real cancellation response"); + } + return Response.json({ + ...owned, + status: sent.includes(id!) ? "cancel_pending" : "running", + cancellationRequestedAt: sent.includes(id!) ? new Date().toISOString() : null, + progress: null, + diagnostics: { tail: [], truncated: false }, + parentAgentOperationKey: undefined, + state: undefined, + }); + }), + ); + expect( + await cancelAdminDataChildren(ctx.checkpointResult, [], { + context: ctx, + sleep: async () => {}, + }), + ).toBe("pending"); + expect(sent).toEqual(children.map((child) => child.id)); + expect(hasUnsettledManagedOperation(ctx.checkpointResult)).toBe(true); +}); +it("discovers the exact fixed CLI descendant from its durable DM row before cancellation", async () => { + vi.stubEnv("DATA_MANAGER_URL", "https://synthetic-dm.test"); + const cliKey = controlChildOperationKey(key, "data-workflow", 0), + dataKey = controlChildOperationKey(cliKey, "data-step", 0), + owned = { ...child(1), operationKey: dataKey }, + ctx = context({ _cancelRequestedAt: new Date().toISOString() }); + fixture.rows = [ + { + id: owned.id, + kind: owned.kind, + status: "running", + operationKey: dataKey, + executionFingerprint: owned.fingerprint, + admittedAt: at, + deadlineAt: deadline, + executionVersion: 2, + executionParent: { + issuer: "host-cli", + execution: { executor: "host-cli", operationKey: cliKey }, + }, + executionState: { + cancellationRequestedAt: null, + progress: null, + diagnostics: { tail: [], truncated: false }, + publication: "not_started", + }, + }, + ]; + let canceled = false; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, options?: RequestInit) => { + if (options?.method === "POST") { + expect((ctx.checkpointResult?._dataExecutions as (typeof owned)[])[0]).toMatchObject({ + id: owned.id, + operationKey: dataKey, + parentAgentOperationKey: key, + cancellationRequestedAt: expect.any(String), + }); + canceled = true; + } + return Response.json({ + id: owned.id, + operationKey: dataKey, + fingerprint: owned.fingerprint, + admittedAt: at, + deadlineAt: deadline, + kind: owned.kind, + status: canceled ? "canceled" : "running", + cancellationRequestedAt: canceled ? new Date().toISOString() : null, + progress: null, + diagnostics: { tail: [], truncated: false }, + publication: "not_started", + }); + }), + ); + expect( + await cancelAdminDataChildren( + ctx.checkpointResult, + [{ kind: "data.downloadFonts", operationKey: key }], + { context: ctx, sleep: async () => {} }, + ), + ).toBe("contained"); + expect(canceled).toBe(true); + expect(hasUnsettledManagedOperation(ctx.checkpointResult)).toBe(false); +}); + +it("keeps malformed or unknown backend terminal metadata behind the finalization fence", () => { + expect( + hasUnsettledManagedOperation({ + _dataExecutions: [{ ...child(1), state: "terminal", publication: undefined }], + }), + ).toBe(true); + expect( + hasUnsettledManagedOperation({ + _dataExecutions: [ + { ...child(1), state: "terminal", publication: "committed", status: "running" }, + ], + }), + ).toBe(true); +}); diff --git a/apps/api/src/services/__tests__/job-runner-cancel.test.ts b/apps/api/src/services/__tests__/job-runner-cancel.test.ts index b6a1449cc..ab065e976 100644 --- a/apps/api/src/services/__tests__/job-runner-cancel.test.ts +++ b/apps/api/src/services/__tests__/job-runner-cancel.test.ts @@ -130,12 +130,10 @@ it("retains a fulfilled backend's partial failure evidence in failed DB and SSE new AbortController().signal, ); unsubscribe(); - expect(database.updates).toContainEqual( - expect.objectContaining({ - status: "failed", - result: expect.objectContaining({ phase: "build", ...evidence }), - }), - ); + const failed = database.updates.find((update) => update.status === "failed"); + expect(failed).toBeDefined(); + const stored = new PgDialect().sqlToQuery(failed!.result as SQL); + expect(JSON.parse(stored.params[0] as string)).toMatchObject({ phase: "build", ...evidence }); expect(events).toContainEqual( expect.objectContaining({ status: "failed", diff --git a/apps/api/src/services/__tests__/job-runner-postgres.test.ts b/apps/api/src/services/__tests__/job-runner-postgres.test.ts index f00682caa..77c8d6a05 100644 --- a/apps/api/src/services/__tests__/job-runner-postgres.test.ts +++ b/apps/api/src/services/__tests__/job-runner-postgres.test.ts @@ -51,943 +51,938 @@ function checkedCheckpoint( } // Explicit opt-in connection supplied only after the task launcher verifies the // synthetic container's exact ID/owner. No migrations or table-wide cleanup. -describe.skipIf(!process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL)( - "admin job PostgreSQL ownership/recovery", - () => { - let client: ReturnType; - let root: string; - const ids: string[] = []; - beforeAll(async () => { - const connectionUrl = process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL; - if (!connectionUrl) throw new Error("Explicit synthetic DB required"); - client = postgres(connectionUrl, { - max: 4, - connection: { statement_timeout: 5000 }, - }); - state.db = drizzle(client, { schema }); - const active = await state.db - .select() - .from(adminJob) - .where(inArray(adminJob.status, ["queued", "running", "cancel_pending"])); - expect(active).toEqual([]); // Never adopt another fixture's work. - root = await mkdtemp(join(tmpdir(), "task022-fix1-")); - const tokenFile = join(root, "api-token"); - await writeFile(tokenFile, Buffer.alloc(32, 7).toString("base64url"), { mode: 0o600 }); - vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); +describe.skipIf( + !process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)("admin job PostgreSQL ownership/recovery", () => { + let client: ReturnType; + let root: string; + const ids: string[] = []; + beforeAll(async () => { + const connectionUrl = process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL; + if (!connectionUrl) throw new Error("Explicit synthetic DB required"); + client = postgres(connectionUrl, { + max: 4, + connection: { statement_timeout: 5000 }, }); - afterEach(async () => { - vi.restoreAllMocks(); - vi.useRealTimers(); - if (ids.length) await state.db.delete(adminJob).where(inArray(adminJob.id, ids.splice(0))); + state.db = drizzle(client, { schema }); + const active = await state.db + .select() + .from(adminJob) + .where(inArray(adminJob.status, ["queued", "running", "cancel_pending"])); + expect(active).toEqual([]); // Never adopt another fixture's work. + root = await mkdtemp(join(tmpdir(), "task022-fix1-")); + const tokenFile = join(root, "api-token"); + await writeFile(tokenFile, Buffer.alloc(32, 7).toString("base64url"), { mode: 0o600 }); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); + }); + afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); + if (ids.length) await state.db.delete(adminJob).where(inArray(adminJob.id, ids.splice(0))); + }); + afterAll(async () => { + await client?.end({ timeout: 2 }); + if (root) await rm(root, { recursive: true }); + vi.unstubAllEnvs(); + }); + async function seed( + type = "backup.operation", + result: Record = {}, + status = "running", + ) { + const id = randomUUID(); + ids.push(id); + const execution = { owner: randomUUID(), version: 1 }; + const [job] = await state.db + .insert(adminJob) + .values({ id, type, status, payload: {}, result: { ...result, _execution: execution } }) + .returning(); + return job; + } + const row = async (id: string) => + (await state.db.select().from(adminJob).where(eq(adminJob.id, id)))[0]; + const run = (job: typeof adminJob.$inferSelect, observe = false) => + runner.executeJob(job.id, job.type, {}, job.result, new AbortController().signal, observe); + + it("rejects a resumed old owner's log before insert/publication and stops forward work", async () => { + const job = await seed("test.stale-log"); + const ready = Promise.withResolvers(); + const resume = Promise.withResolvers(); + const after = vi.fn(); + jobRunner.register(job.type, async (ctx) => { + ready.resolve(ctx); + await resume.promise; + await ctx.log("stale"); + after(); + return {}; }); - afterAll(async () => { - await client?.end({ timeout: 2 }); - if (root) await rm(root, { recursive: true }); - vi.unstubAllEnvs(); + const logs: unknown[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "log") logs.push(event); }); - async function seed( - type = "backup.operation", - result: Record = {}, - status = "running", - ) { - const id = randomUUID(); - ids.push(id); - const execution = { owner: randomUUID(), version: 1 }; - const [job] = await state.db - .insert(adminJob) - .values({ id, type, status, payload: {}, result: { ...result, _execution: execution } }) - .returning(); - return job; - } - const row = async (id: string) => - (await state.db.select().from(adminJob).where(eq(adminJob.id, id)))[0]; - const run = (job: typeof adminJob.$inferSelect, observe = false) => - runner.executeJob(job.id, job.type, {}, job.result, new AbortController().signal, observe); + const running = run(job); + const ctx = await ready.promise; + await claimAdminJob(job); + resume.resolve(); + await running; + stop(); + expect(await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id))).toEqual( + [], + ); + expect(logs).toEqual([]); + expect(after).not.toHaveBeenCalled(); + expect(ctx.signal.aborted).toBe(true); + }); - it("rejects a resumed old owner's log before insert/publication and stops forward work", async () => { - const job = await seed("test.stale-log"); - const ready = Promise.withResolvers(); - const resume = Promise.withResolvers(); - const after = vi.fn(); - jobRunner.register(job.type, async (ctx) => { - ready.resolve(ctx); - await resume.promise; - await ctx.log("stale"); - after(); - return {}; - }); - const logs: unknown[] = []; - const stop = jobEventBus.subscribe(job.id, ({ event }) => { - if (event.type === "log") logs.push(event); - }); - const running = run(job); - const ctx = await ready.promise; - await claimAdminJob(job); - resume.resolve(); - await running; - stop(); - expect( - await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), - ).toEqual([]); - expect(logs).toEqual([]); - expect(after).not.toHaveBeenCalled(); - expect(ctx.signal.aborted).toBe(true); + it("publishes a source event once when its atomic insert is deduplicated", async () => { + const job = await seed("test.duplicate-log"); + jobRunner.register(job.type, async (ctx) => { + await ctx.log("once", "stdout", `${job.id}:event`, 1); + await ctx.log("once", "stdout", `${job.id}:event`, 1); + return {}; }); - - it("publishes a source event once when its atomic insert is deduplicated", async () => { - const job = await seed("test.duplicate-log"); - jobRunner.register(job.type, async (ctx) => { - await ctx.log("once", "stdout", `${job.id}:event`, 1); - await ctx.log("once", "stdout", `${job.id}:event`, 1); - return {}; - }); - const logs: unknown[] = []; - const stop = jobEventBus.subscribe(job.id, ({ event }) => { - if (event.type === "log") logs.push(event); - }); - await run(job); - stop(); - expect( - await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), - ).toHaveLength(1); - expect(logs).toHaveLength(1); + const logs: unknown[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "log") logs.push(event); }); + await run(job); + stop(); + expect( + await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), + ).toHaveLength(1); + expect(logs).toHaveLength(1); + }); - it("rechecks ownership after waiting behind a concurrent replacement's row lock", async () => { - const job = await seed("test.blocked-log"); - const ready = Promise.withResolvers(); - const resume = Promise.withResolvers(); - jobRunner.register(job.type, async (ctx) => { - ready.resolve(ctx); - await resume.promise; - await ctx.log("blocked stale log"); - return {}; - }); - const running = run(job); - await ready.promise; - const locked = Promise.withResolvers(); - const commit = Promise.withResolvers(); - const replacement = client.begin(async (tx) => { - await tx`UPDATE admin_job SET result = result || ${JSON.stringify({ _execution: { owner: randomUUID(), version: 2 } })}::jsonb WHERE id = ${job.id}`; - const [backend] = await tx`SELECT pg_backend_pid() AS pid`; - locked.resolve(backend.pid); - await commit.promise; - }); - const pid = await locked.promise; - try { - resume.resolve(); - await vi.waitFor( - async () => { - const [waiting] = - await client`SELECT count(*)::int AS n FROM pg_stat_activity WHERE ${pid} = ANY(pg_blocking_pids(pid))`; - expect(waiting.n).toBe(1); - }, - { timeout: 2000, interval: 10 }, - ); - } finally { - commit.resolve(); - await replacement; - await running; - } - expect( - await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), - ).toEqual([]); + it("rechecks ownership after waiting behind a concurrent replacement's row lock", async () => { + const job = await seed("test.blocked-log"); + const ready = Promise.withResolvers(); + const resume = Promise.withResolvers(); + jobRunner.register(job.type, async (ctx) => { + ready.resolve(ctx); + await resume.promise; + await ctx.log("blocked stale log"); + return {}; }); - - it("preserves a renewed live lease through checkpoints and rejects a stale due-row claim", async () => { - const job = await seed("test.live-lease"); - const claimed = await claimAdminJob(job); - const ready = Promise.withResolvers(); - const resume = Promise.withResolvers(); - jobRunner.register(job.type, async (ctx) => { - ready.resolve(ctx); - await resume.promise; - return {}; - }); - if (!claimed) throw new Error("Missing claim"); - const work = run(claimed); - const ctx = await ready.promise; - vi.useFakeTimers({ toFake: ["Date"] }); - const start = Date.now(); - try { - vi.setSystemTime(start + 40_000); - await renewAdminJobClaim(job.id, ctx.execution); - const live = (await row(job.id)).result as Record; - await ctx.checkpoint({ phase: "still working" }); - expect((await row(job.id)).result).toHaveProperty( - "_observationLeaseUntil", - live._observationLeaseUntil, - ); - vi.setSystemTime(start + 65_000); // Original snapshot expired, renewed owner is live. - expect(await claimAdminJob(claimed, {}, true)).toBeUndefined(); - expect(readJobExecution((await row(job.id)).result)).toEqual(ctx.execution); - vi.setSystemTime(start + 105_000); - await expect(renewAdminJobClaim(job.id, ctx.execution)).rejects.toBeInstanceOf( - JobOwnershipLost, - ); - } finally { - resume.resolve(); - await work; - } + const running = run(job); + await ready.promise; + const locked = Promise.withResolvers(); + const commit = Promise.withResolvers(); + const replacement = client.begin(async (tx) => { + await tx`UPDATE admin_job SET result = result || ${JSON.stringify({ _execution: { owner: randomUUID(), version: 2 } })}::jsonb WHERE id = ${job.id}`; + const [backend] = await tx`SELECT pg_backend_pid() AS pid`; + locked.resolve(backend.pid); + await commit.promise; }); - - it.each([ - ["pending", false], - ["finalization", false], - ["pending", true], - ["finalization", true], - ] as const)( - "recovers a %s write outage without restart or another POST (backendFailed=%s)", - async (failure, backendFailed) => { - const job = await seed(); - const claimed = await claimAdminJob(job); - let offline = false; - const update = state.db.update.bind(state.db); - vi.spyOn(state.db, "update").mockImplementation((...args) => { - if (offline) throw new Error("synthetic database unavailable"); - return update(...args); - }); - const key = "opk1_fixround1recover12"; - const opId = "job1_fixround1recover12"; - const terminal: OpsJobStatus = { - version: 2, - admittedAt: Date.now(), - deadlineAt: Date.now() + 300000, - fingerprint: "a".repeat(64), - operationId: opId, - operationKey: key, - kind: "backup.delete", - resourceId: "nightly", - ...(backendFailed - ? { - state: "failed", - errorClass: "runtime", - failureResult: { - diagnostics: { - truncated: false, - tail: [{ stream: "stderr", message: "retained failure" }], - }, - }, - } - : { state: "succeeded", result: { backupId: "nightly" } }), - submittedAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }; - const execute = vi.fn( - async ( - operation: OpsOperation, - options: NonNullable[1]>, - ) => { - if (operation.kind === "controlRetention.acknowledge") - return { execution: "sync", kind: operation.kind, value: { acknowledged: true } }; - if (!options.operationKey || !options.execution) - throw Error("Missing durable fixture root"); - terminal.operationKey = options.operationKey; - terminal.deadlineAt = options.execution.requestedDeadlineAt; - return { - execution: "async", - kind: "backup.delete", - operationId: opId, - operationKey: terminal.operationKey, - fingerprint: terminal.fingerprint, - deadlineAt: terminal.deadlineAt, - state: "queued", - }; - }, - ); - const status = vi.fn(async () => { - offline = true; - if (failure === "pending") throw new OpsClientError("authorization"); - return terminal; - }); - state.client = { - execute, - status, - events: vi.fn().mockResolvedValue({ - version: 1, - operationId: opId, - events: [], - nextCursor: 0, - terminal: true, - truncated: false, - }), - cancel: vi.fn(), - lookup: vi.fn().mockResolvedValue(terminal), - } as unknown as ApiOpsClient; - jobRunner.register(job.type, async (ctx) => { - await executeAdminJobOperation( - ctx, - { kind: "backup.delete", backupId: "nightly" }, - "admin-job.backup.delete", - { client: state.client, operationKey: key }, - ); - return { backupId: "nightly" }; - }); - const outcomes: string[] = []; - const stop = jobEventBus.subscribe(job.id, ({ event }) => { - if (event.type === "status") outcomes.push(event.status); - }); - const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); - await runner.launchJob(claimed); - offline = false; - expect((await row(job.id)).status).toBe("running"); - expect(outcomes).not.toContain("failed"); - expect(outcomes).not.toContain("canceled"); - status.mockResolvedValue(terminal); - // The abandoned claim must become due without initialize() or an unknown write. - vi.useFakeTimers({ toFake: ["Date"] }); - vi.setSystemTime(Date.now() + 65_000); - await jobRunner.reconcile(); - expect((await row(job.id)).status).toBe(backendFailed ? "failed" : "success"); - if (backendFailed) - expect((await row(job.id)).result).toHaveProperty( - "diagnostics.tail.0.message", - "retained failure", - ); - expect( - execute.mock.calls.filter(([operation]) => operation.kind === "backup.delete"), - ).toHaveLength(1); - expect(state.client.cancel).not.toHaveBeenCalled(); - if (!backendFailed) expect(outcomes).not.toContain("failed"); - expect(outcomes).not.toContain("canceled"); - stop(); - errors.mockRestore(); - }, + const pid = await locked.promise; + try { + resume.resolve(); + await vi.waitFor( + async () => { + const [waiting] = + await client`SELECT count(*)::int AS n FROM pg_stat_activity WHERE ${pid} = ANY(pg_blocking_pids(pid))`; + expect(waiting.n).toBe(1); + }, + { timeout: 2000, interval: 10 }, + ); + } finally { + commit.resolve(); + await replacement; + await running; + } + expect(await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id))).toEqual( + [], ); + }); + + it("preserves a renewed live lease through checkpoints and rejects a stale due-row claim", async () => { + const job = await seed("test.live-lease"); + const claimed = await claimAdminJob(job); + const ready = Promise.withResolvers(); + const resume = Promise.withResolvers(); + jobRunner.register(job.type, async (ctx) => { + ready.resolve(ctx); + await resume.promise; + return {}; + }); + if (!claimed) throw new Error("Missing claim"); + const work = run(claimed); + const ctx = await ready.promise; + vi.useFakeTimers({ toFake: ["Date"] }); + const start = Date.now(); + try { + vi.setSystemTime(start + 40_000); + await renewAdminJobClaim(job.id, ctx.execution); + const live = (await row(job.id)).result as Record; + await ctx.checkpoint({ phase: "still working" }); + expect((await row(job.id)).result).toHaveProperty( + "_observationLeaseUntil", + live._observationLeaseUntil, + ); + vi.setSystemTime(start + 65_000); // Original snapshot expired, renewed owner is live. + expect(await claimAdminJob(claimed, {}, true)).toBeUndefined(); + expect(readJobExecution((await row(job.id)).result)).toEqual(ctx.execution); + vi.setSystemTime(start + 105_000); + await expect(renewAdminJobClaim(job.id, ctx.execution)).rejects.toBeInstanceOf( + JobOwnershipLost, + ); + } finally { + resume.resolve(); + await work; + } + }); - it("never converts repeated success-finalization outages into a handler failure", async () => { + it.each([ + ["pending", false], + ["finalization", false], + ["pending", true], + ["finalization", true], + ] as const)( + "recovers a %s write outage without restart or another POST (backendFailed=%s)", + async (failure, backendFailed) => { const job = await seed(); const claimed = await claimAdminJob(job); - let failures = 0; + let offline = false; const update = state.db.update.bind(state.db); vi.spyOn(state.db, "update").mockImplementation((...args) => { - if (failures-- > 0) throw new Error("synthetic DB outage"); + if (offline) throw new Error("synthetic database unavailable"); return update(...args); }); - jobRunner.register(job.type, async () => { - failures = 2; - return { completed: true }; - }); - vi.spyOn(console, "error").mockImplementation(() => undefined); - await runner.launchJob(claimed); - expect((await row(job.id)).status).toBe("running"); - }); - - it.each([ - [false, false], - [true, false], - [false, true], - [true, true], - ])( - "settles cancellation only with containment (reconcile=%s, unknownLink=%s)", - async (reconcile, unknownLink) => { - const key = "opk1_fixround1cancel123"; - const operationId = "job1_fixround1cancel123"; - const diagnostics = { truncated: false, tail: [{ stream: "stderr", message: "retained" }] }; - const job = await seed( - "backup.operation", - { - ...(unknownLink - ? { - _hardlinkAttempts: { - synthetic: { revisionId: "cfg1_synthetic", phase: "attempted" }, - }, - } - : {}), - observation: { - state: "unknown", - reason: "transport", - nextCheckAt: new Date(0).toISOString(), - }, - diagnostics, - opsProjection: { - version: 1, - eventTotal: 0, - byteTotal: 0, - truncated: false, - operations: { - [key]: { - kind: "backup.delete", - operationKey: key, - operationId, - cursor: 0, - events: 0, - bytes: 0, + const key = "opk1_fixround1recover12"; + const opId = "job1_fixround1recover12"; + const terminal: OpsJobStatus = { + version: 2, + admittedAt: Date.now(), + deadlineAt: Date.now() + 300000, + fingerprint: "a".repeat(64), + operationId: opId, + operationKey: key, + kind: "backup.delete", + resourceId: "nightly", + ...(backendFailed + ? { + state: "failed", + errorClass: "runtime", + failureResult: { + diagnostics: { truncated: false, - terminal: false, + tail: [{ stream: "stderr", message: "retained failure" }], }, }, - }, - }, - reconcile ? "cancel_pending" : "running", - ); - const status = { - version: 1, - kind: "backup.delete", - operationKey: key, - operationId, - state: "failed", - errorClass: "runtime", - terminationRequestedAt: new Date().toISOString(), - diagnostics, - }; - state.client = { - status: vi.fn().mockResolvedValue(status), - cancel: vi.fn(), - lookup: vi.fn(), - } as unknown as ApiOpsClient; - const events: unknown[] = []; - const stop = jobEventBus.subscribe(job.id, ({ event }) => { - if (event.type === "status" && event.status === "canceled") events.push(event.result); - }); - if (reconcile) await jobRunner.reconcile(); - else await jobRunner.cancel(job.id); - stop(); - const finished = await row(job.id); - expect(finished.result).toHaveProperty("diagnostics", diagnostics); - if (unknownLink) { - expect(finished.status).toBe("cancel_pending"); - expect(finished.result).toHaveProperty("observation.state", "unknown"); - expect(events).toEqual([]); - } else { - expect(finished.status).toBe("canceled"); - expect(finished.result).not.toHaveProperty("observation"); - expect(events).toEqual([expect.objectContaining({ diagnostics })]); - expect(events[0]).not.toHaveProperty("observation"); - } - }, - ); - - async function checkpointContexts(job: typeof adminJob.$inferSelect, n = 2) { - const found: JobContext[] = [], - ready = Promise.withResolvers(), - release = Promise.withResolvers(); - jobRunner.register(job.type, async (ctx) => { - found.push(ctx); - if (found.length === n) ready.resolve(); - await release.promise; - throw new (await import("../admin-job-failure")).JobObservationPending("transport"); - }); - const runs = Array.from({ length: n }, () => run(job)); - await ready.promise; - return { - contexts: found, - finish: async () => { - release.resolve(); - await Promise.all(runs); - }, + } + : { state: "succeeded", result: { backupId: "nightly" } }), + submittedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }; - } - function successfulClient() { - const statuses = new Map(); - let mutations = 0, - acks = 0; const execute = vi.fn( async ( operation: OpsOperation, options: NonNullable[1]>, ) => { - if (operation.kind === "controlRetention.acknowledge") { - acks++; + if (operation.kind === "controlRetention.acknowledge") return { execution: "sync", kind: operation.kind, value: { acknowledged: true } }; - } - if (operation.kind !== "backup.delete" || !options.operationKey || !options.execution) - throw Error("Unsupported fixture operation"); - mutations++; - const id = `job1_${randomUUID().replaceAll("-", "")}`, - at = Date.now(); - const status: OpsJobStatus = { - version: 2, - kind: operation.kind, - operationId: id, - operationKey: options.operationKey, - resourceId: operation.backupId, - state: "succeeded", - admittedAt: at, - deadlineAt: options.execution.requestedDeadlineAt, - fingerprint: "a".repeat(64), - result: { backupId: operation.backupId }, - submittedAt: new Date(at).toISOString(), - updatedAt: new Date(at).toISOString(), + if (!options.operationKey || !options.execution) + throw Error("Missing durable fixture root"); + terminal.operationKey = options.operationKey; + terminal.deadlineAt = options.execution.requestedDeadlineAt; + return { + execution: "async", + kind: "backup.delete", + operationId: opId, + operationKey: terminal.operationKey, + fingerprint: terminal.fingerprint, + deadlineAt: terminal.deadlineAt, + state: "queued", }; - statuses.set(id, status); - return { execution: "async", ...status, state: "queued" }; }, ); - const client = { + const status = vi.fn(async () => { + offline = true; + if (failure === "pending") throw new OpsClientError("authorization"); + return terminal; + }); + state.client = { execute, - status: vi.fn(async (_kind: string, id: string) => { - const value = statuses.get(id); - if (!value) throw new OpsClientError("runtime"); - return value; - }), - lookup: vi.fn(async (_op: OpsOperation, key: string) => { - const value = [...statuses.values()].find((s) => s.operationKey === key); - if (!value) throw new OpsClientError("runtime"); - return value; - }), - events: vi.fn(async (id: string) => ({ - version: 2, - operationId: id, - terminal: true, + status, + events: vi.fn().mockResolvedValue({ + version: 1, + operationId: opId, events: [], nextCursor: 0, + terminal: true, truncated: false, - })), + }), cancel: vi.fn(), + lookup: vi.fn().mockResolvedValue(terminal), } as unknown as ApiOpsClient; - return { client, counts: () => ({ mutations, acks }), statuses }; - } - function retainedParent(id: string) { - return { - operationKey: `opk1_parent${id.replaceAll("-", "")}`, - fingerprint: "b".repeat(64), - deadlineAt: Date.now() + 3600_000, - }; - } - async function seedCapacity(count: number, mixed = false) { - const job = await seed(`test.renewal.capacity.${randomUUID()}`), - execution = required(readJobExecution(job.result)); - const parent = retainedParent(job.id), - operations: ReturnType["operations"] = {}, - steps: ReturnType["steps"] = {}; - const { createControlRootOperationKey, controlTerminalReceiptDigest } = await import( - "@openmapx/core/ops" - ); - for (let n = 0; n < count; n++) { - const address = `opk1_legacyfixture${String(n).padStart(16, "0")}`, - deadlineAt = parent.deadlineAt; - const key = - mixed && n % 2 === 0 - ? address - : createControlRootOperationKey({ - notAfter: deadlineAt, - nonce: Buffer.from(n.toString(16).padStart(32, "0"), "hex"), - }); - const operation = { kind: "backup.delete" as const, backupId: `old-${n}` }; - const attempt: ApiJobChildAttempt | undefined = - mixed && n % 2 === 0 - ? undefined - : { - version: 1, - apiJobId: job.id, - apiOwnerVersion: execution.version, - parentOperationKey: parent.operationKey, - parentFingerprint: parent.fingerprint, - parentDeadlineAt: parent.deadlineAt, - childOperationKey: key, - childOperationFingerprint: ( - await import("@openmapx/core/ops") - ).opsOperationFingerprint(operation), - requestedDeadlineAt: deadlineAt, - attemptId: randomUUID(), - }; - const status = { - version: 2 as const, - operationId: `job1_fixture${String(n).padStart(16, "0")}`, - operationKey: key, - kind: operation.kind, - resourceId: operation.backupId, - state: "succeeded" as const, - result: { backupId: operation.backupId }, - admittedAt: deadlineAt - 3600_000, - deadlineAt, - fingerprint: "a".repeat(64), - submittedAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }; - operations[key] = { - kind: operation.kind, - operationKey: key, - operationId: status.operationId, - operation, - attempt, - phase: "terminal", - terminal: true, - result: status.result, - fingerprint: status.fingerprint, - deadlineAt, - cursor: 0, - events: 0, - bytes: 0, - truncated: false, - ...(attempt + jobRunner.register(job.type, async (ctx) => { + await executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "nightly" }, + "admin-job.backup.delete", + { client: state.client, operationKey: key }, + ); + return { backupId: "nightly" }; + }); + const outcomes: string[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "status") outcomes.push(event.status); + }); + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); + await runner.launchJob(claimed); + offline = false; + expect((await row(job.id)).status).toBe("running"); + expect(outcomes).not.toContain("failed"); + expect(outcomes).not.toContain("canceled"); + status.mockResolvedValue(terminal); + // The abandoned claim must become due without initialize() or an unknown write. + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 65_000); + await jobRunner.reconcile(); + expect((await row(job.id)).status).toBe(backendFailed ? "failed" : "success"); + if (backendFailed) + expect((await row(job.id)).result).toHaveProperty( + "diagnostics.tail.0.message", + "retained failure", + ); + expect( + execute.mock.calls.filter(([operation]) => operation.kind === "backup.delete"), + ).toHaveLength(1); + expect(state.client.cancel).not.toHaveBeenCalled(); + if (!backendFailed) expect(outcomes).not.toContain("failed"); + expect(outcomes).not.toContain("canceled"); + stop(); + errors.mockRestore(); + }, + ); + + it("never converts repeated success-finalization outages into a handler failure", async () => { + const job = await seed(); + const claimed = await claimAdminJob(job); + let failures = 0; + const update = state.db.update.bind(state.db); + vi.spyOn(state.db, "update").mockImplementation((...args) => { + if (failures-- > 0) throw new Error("synthetic DB outage"); + return update(...args); + }); + jobRunner.register(job.type, async () => { + failures = 2; + return { completed: true }; + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + await runner.launchJob(claimed); + expect((await row(job.id)).status).toBe("running"); + }); + + it.each([ + [false, false], + [true, false], + [false, true], + [true, true], + ])( + "settles cancellation only with containment (reconcile=%s, unknownLink=%s)", + async (reconcile, unknownLink) => { + const key = "opk1_fixround1cancel123"; + const operationId = "job1_fixround1cancel123"; + const diagnostics = { truncated: false, tail: [{ stream: "stderr", message: "retained" }] }; + const job = await seed( + "backup.operation", + { + ...(unknownLink ? { - receiptDelivery: { - state: "confirmed", - terminalCopyId: randomUUID(), - checkpointOwnerVersion: 1, - terminalDigest: controlTerminalReceiptDigest(status), - terminalReceipt: status, + _hardlinkAttempts: { + synthetic: { revisionId: "cfg1_synthetic", phase: "attempted" }, }, } : {}), - }; - if (attempt) steps[address] = { operationKey: key, requestedDeadlineAt: deadlineAt }; - } - const result = { - ...(job.result as Record), - _parentExecution: parent, - untouched: { receipt: "preserve" }, - opsProjection: { - version: 3, - eventTotal: 7, - byteTotal: 17, - truncated: false, - steps, - operations, + observation: { + state: "unknown", + reason: "transport", + nextCheckAt: new Date(0).toISOString(), + }, + diagnostics, + opsProjection: { + version: 1, + eventTotal: 0, + byteTotal: 0, + truncated: false, + operations: { + [key]: { + kind: "backup.delete", + operationKey: key, + operationId, + cursor: 0, + events: 0, + bytes: 0, + truncated: false, + terminal: false, + }, + }, + }, }, - }; - await state.db.update(adminJob).set({ result }).where(eq(adminJob.id, job.id)); - return { ...job, result }; - } - it("renewal: same-owner same-address different deadlines has one CAS winner and at most one mutation POST", async () => { - const job = await seed("test.renewal.same-address"), - holders = await checkpointContexts(job), - remote = successfulClient(); - const lockReady = Promise.withResolvers(), - unlock = Promise.withResolvers(); - const locked = state.db.transaction(async (tx) => { - await tx.select().from(adminJob).where(eq(adminJob.id, job.id)).for("update"); - lockReady.resolve(); - await unlock.promise; - }); - await lockReady.promise; - const first = executeAdminJobOperation( - required(holders.contexts[0]), - { kind: "backup.delete", backupId: "winner" }, - "test", - { operationKey: "opk1_sameaddress12345", client: remote.client, pollIntervalMs: 1 }, + reconcile ? "cancel_pending" : "running", ); - await new Promise((r) => setTimeout(r, 15)); - const second = executeAdminJobOperation( - required(holders.contexts[1]), - { kind: "backup.delete", backupId: "loser" }, - "test", - { operationKey: "opk1_sameaddress12345", client: remote.client, pollIntervalMs: 1 }, - ); - unlock.resolve(); - await locked; - await Promise.allSettled([first, second]); - const saved = checkedCheckpoint((await row(job.id)).result); - expect(Object.keys(saved.opsProjection.steps)).toHaveLength(1); - expect(Object.keys(saved.opsProjection.operations)).toHaveLength(1); - expect(remote.counts().mutations).toBe(1); - const winner = required(saved.opsProjection.steps.opk1_sameaddress12345); - expect( - ( - required(saved.opsProjection.operations[winner.operationKey]).operation as { - backupId: string; - } - ).backupId, - ).toBe("winner"); - expect(holders.contexts[1]?.checkpointResult).toMatchObject({ - opsProjection: { steps: { opk1_sameaddress12345: winner } }, + const status = { + version: 1, + kind: "backup.delete", + operationKey: key, + operationId, + state: "failed", + errorClass: "runtime", + terminationRequestedAt: new Date().toISOString(), + diagnostics, + }; + state.client = { + status: vi.fn().mockResolvedValue(status), + cancel: vi.fn(), + lookup: vi.fn(), + } as unknown as ApiOpsClient; + const events: unknown[] = []; + const stop = jobEventBus.subscribe(job.id, ({ event }) => { + if (event.type === "status" && event.status === "canceled") events.push(event.result); }); - await holders.finish(); + if (reconcile) await jobRunner.reconcile(); + else await jobRunner.cancel(job.id); + stop(); + const finished = await row(job.id); + expect(finished.result).toHaveProperty("diagnostics", diagnostics); + if (unknownLink) { + expect(finished.status).toBe("cancel_pending"); + expect(finished.result).toHaveProperty("observation.state", "unknown"); + expect(events).toEqual([]); + } else { + expect(finished.status).toBe("canceled"); + expect(finished.result).not.toHaveProperty("observation"); + expect(events).toEqual([expect.objectContaining({ diagnostics })]); + expect(events[0]).not.toHaveProperty("observation"); + } + }, + ); + + async function checkpointContexts(job: typeof adminJob.$inferSelect, n = 2) { + const found: JobContext[] = [], + ready = Promise.withResolvers(), + release = Promise.withResolvers(); + jobRunner.register(job.type, async (ctx) => { + found.push(ctx); + if (found.length === n) ready.resolve(); + await release.promise; + throw new (await import("../admin-job-failure")).JobObservationPending("transport"); }); - it("renewal: lost acknowledgement and confirmation checkpoint keep a copied known result pending across cached restart", async () => { - const job = await seedCapacity(0), - holders = await checkpointContexts(job), - remote = successfulClient(), - ctx = required(holders.contexts[0]); - const execute = remote.client.execute.bind(remote.client); - let loseAck = true; - remote.client.execute = vi.fn(async (...args: Parameters) => { - const result = await execute(...args); - if (args[0].kind === "controlRetention.acknowledge" && loseAck) { - loseAck = false; - throw new OpsClientError("timeout"); + const runs = Array.from({ length: n }, () => run(job)); + await ready.promise; + return { + contexts: found, + finish: async () => { + release.resolve(); + await Promise.all(runs); + }, + }; + } + function successfulClient() { + const statuses = new Map(); + let mutations = 0, + acks = 0; + const execute = vi.fn( + async ( + operation: OpsOperation, + options: NonNullable[1]>, + ) => { + if (operation.kind === "controlRetention.acknowledge") { + acks++; + return { execution: "sync", kind: operation.kind, value: { acknowledged: true } }; } - return result; - }) as ApiOpsClient["execute"]; - await expect( - executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "original" }, "test", { - operationKey: "opk1_receiptdelivery123", - client: remote.client, - pollIntervalMs: 1, - }), - ).rejects.toMatchObject({ reason: "receipt_delivery" }); - const saved = checkedCheckpoint((await row(job.id)).result), - projected = required(Object.values(saved.opsProjection.operations)[0]); - expect(projected).toMatchObject({ + if (operation.kind !== "backup.delete" || !options.operationKey || !options.execution) + throw Error("Unsupported fixture operation"); + mutations++; + const id = `job1_${randomUUID().replaceAll("-", "")}`, + at = Date.now(); + const status: OpsJobStatus = { + version: 2, + kind: operation.kind, + operationId: id, + operationKey: options.operationKey, + resourceId: operation.backupId, + state: "succeeded", + admittedAt: at, + deadlineAt: options.execution.requestedDeadlineAt, + fingerprint: "a".repeat(64), + result: { backupId: operation.backupId }, + submittedAt: new Date(at).toISOString(), + updatedAt: new Date(at).toISOString(), + }; + statuses.set(id, status); + return { execution: "async", ...status, state: "queued" }; + }, + ); + const client = { + execute, + status: vi.fn(async (_kind: string, id: string) => { + const value = statuses.get(id); + if (!value) throw new OpsClientError("runtime"); + return value; + }), + lookup: vi.fn(async (_op: OpsOperation, key: string) => { + const value = [...statuses.values()].find((s) => s.operationKey === key); + if (!value) throw new OpsClientError("runtime"); + return value; + }), + events: vi.fn(async (id: string) => ({ + version: 2, + operationId: id, terminal: true, - result: { backupId: "original" }, - receiptDelivery: { - state: "pending", - terminalReceipt: { state: "succeeded", result: { backupId: "original" } }, - }, - }); - expect(projected.outcome).toBeUndefined(); - const { cancelAdminJobOperations } = await import("../admin-job-ops"); - loseAck = true; - await expect( - cancelAdminJobOperations(saved, remote.client, { context: ctx, jobId: job.id }), - ).rejects.toMatchObject({ reason: "receipt_delivery" }); - state.client = remote.client; - loseAck = true; - expect(await jobRunner.cancel(job.id)).toBe(false); - expect((await row(job.id)).result).toMatchObject({ - observation: { state: "receipt_delivery", reason: "receipt_delivery" }, - }); - const { and } = await import("drizzle-orm"), - { noPendingReceiptDelivery } = await import("../admin-job-state"); - expect( - await state.db - .update(adminJob) - .set({ status: "success" }) - .where(and(eq(adminJob.id, job.id), noPendingReceiptDelivery())) - .returning({ id: adminJob.id }), - ).toEqual([]); - const restarted = required(holders.contexts[1]); - restarted.checkpointResult = saved; - const abort = new AbortController(); - abort.abort(); - restarted.signal = abort.signal; - const checkpoint = restarted.checkpoint; - let blockConfirmation = true; - restarted.checkpoint = async (...args) => { - const entries = Object.values(readProjection(args[0]).operations); - if (blockConfirmation && entries.some((o) => o.receiptDelivery?.state === "confirmed")) - throw Error("confirmation DB unavailable"); - await checkpoint(...args); + events: [], + nextCursor: 0, + truncated: false, + })), + cancel: vi.fn(), + } as unknown as ApiOpsClient; + return { client, counts: () => ({ mutations, acks }), statuses }; + } + function retainedParent(id: string) { + return { + operationKey: `opk1_parent${id.replaceAll("-", "")}`, + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 3600_000, + }; + } + async function seedCapacity(count: number, mixed = false) { + const job = await seed(`test.renewal.capacity.${randomUUID()}`), + execution = required(readJobExecution(job.result)); + const parent = retainedParent(job.id), + operations: ReturnType["operations"] = {}, + steps: ReturnType["steps"] = {}; + const { createControlRootOperationKey, controlTerminalReceiptDigest } = await import( + "@openmapx/core/ops" + ); + for (let n = 0; n < count; n++) { + const address = `opk1_legacyfixture${String(n).padStart(16, "0")}`, + deadlineAt = parent.deadlineAt; + const key = + mixed && n % 2 === 0 + ? address + : createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: Buffer.from(n.toString(16).padStart(32, "0"), "hex"), + }); + const operation = { kind: "backup.delete" as const, backupId: `old-${n}` }; + const attempt: ApiJobChildAttempt | undefined = + mixed && n % 2 === 0 + ? undefined + : { + version: 1, + apiJobId: job.id, + apiOwnerVersion: execution.version, + parentOperationKey: parent.operationKey, + parentFingerprint: parent.fingerprint, + parentDeadlineAt: parent.deadlineAt, + childOperationKey: key, + childOperationFingerprint: ( + await import("@openmapx/core/ops") + ).opsOperationFingerprint(operation), + requestedDeadlineAt: deadlineAt, + attemptId: randomUUID(), + }; + const status = { + version: 2 as const, + operationId: `job1_fixture${String(n).padStart(16, "0")}`, + operationKey: key, + kind: operation.kind, + resourceId: operation.backupId, + state: "succeeded" as const, + result: { backupId: operation.backupId }, + admittedAt: deadlineAt - 3600_000, + deadlineAt, + fingerprint: "a".repeat(64), + submittedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }; - await expect( - executeAdminJobOperation( - restarted, - { kind: "backup.delete", backupId: "changed" }, - "test", - { operationKey: "opk1_receiptdelivery123", client: remote.client }, - ), - ).rejects.toMatchObject({ reason: "receipt_delivery" }); - expect( - checkedCheckpoint((await row(job.id)).result).opsProjection.operations[ - projected.operationKey - ]?.receiptDelivery?.state, - ).toBe("pending"); - blockConfirmation = false; - await expect( - executeAdminJobOperation( - restarted, - { kind: "backup.delete", backupId: "changed-again" }, - "test", - { operationKey: "opk1_receiptdelivery123", client: remote.client }, - ), - ).resolves.toEqual({ backupId: "original" }); - const ackCalls = vi - .mocked(remote.client.execute) - .mock.calls.filter((c) => c[0].kind === "controlRetention.acknowledge"); - expect( - new Set( - ackCalls.map((c) => - c[0].kind === "controlRetention.acknowledge" ? c[0].proof : undefined, - ), - ).size, - ).toBe(1); - expect(ackCalls.every((c) => !c[1]?.signal?.aborted)).toBe(true); - expect(remote.counts().mutations).toBe(1); - await holders.finish(); - }); - it("renewal: current-row log capacity admits one last event and stale checkpoints cannot reset its charge", async () => { - const job = await seedCapacity(0), - execution = required(readJobExecution(job.result)); - const stale = { ...job.result, _opsLogEvents: 1999, _opsLogBytes: 0 }; - await state.db.update(adminJob).set({ result: stale }).where(eq(adminJob.id, job.id)); - const { insertOwnedJobLog } = await import("../admin-job-state"); - const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); - const entries = [0, 1].map((seq) => ({ - id: `${job.id}:last:${seq}`, - jobId: job.id, - seq, - stream: "stdout", - line: "last", - createdAt: new Date(), - })); - const accepted = await Promise.all( - entries.map((entry) => insertOwnedJobLog(execution, entry)), - ); - expect(accepted.filter(Boolean)).toHaveLength(1); - const winner = required(entries[accepted.indexOf(true)]); - expect(await insertOwnedJobLog(execution, winner)).toBe(false); - await checkpointOwnedAdminJob( - job.id, - execution, - { ...stale, retainedPatch: true }, - undefined, - undefined, - undefined, - stale, - ); - expect((await row(job.id)).result).toMatchObject({ - _opsLogEvents: 2000, - _opsLogBytes: 4, - retainedPatch: true, - }); - expect( - await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), - ).toHaveLength(1); + operations[key] = { + kind: operation.kind, + operationKey: key, + operationId: status.operationId, + operation, + attempt, + phase: "terminal", + terminal: true, + result: status.result, + fingerprint: status.fingerprint, + deadlineAt, + cursor: 0, + events: 0, + bytes: 0, + truncated: false, + ...(attempt + ? { + receiptDelivery: { + state: "confirmed", + terminalCopyId: randomUUID(), + checkpointOwnerVersion: 1, + terminalDigest: controlTerminalReceiptDigest(status), + terminalReceipt: status, + }, + } + : {}), + }; + if (attempt) steps[address] = { operationKey: key, requestedDeadlineAt: deadlineAt }; + } + const result = { + ...(job.result as Record), + _parentExecution: parent, + untouched: { receipt: "preserve" }, + opsProjection: { + version: 3, + eventTotal: 7, + byteTotal: 17, + truncated: false, + steps, + operations, + }, + }; + await state.db.update(adminJob).set({ result }).where(eq(adminJob.id, job.id)); + return { ...job, result }; + } + it("renewal: same-owner same-address different deadlines has one CAS winner and at most one mutation POST", async () => { + const job = await seed("test.renewal.same-address"), + holders = await checkpointContexts(job), + remote = successfulClient(); + const lockReady = Promise.withResolvers(), + unlock = Promise.withResolvers(); + const locked = state.db.transaction(async (tx) => { + await tx.select().from(adminJob).where(eq(adminJob.id, job.id)).for("update"); + lockReady.resolve(); + await unlock.promise; }); - it("renewal: actual expiry cleanup preserves the last pending terminal copy", async () => { - const { lt, and, ne } = await import("drizzle-orm"), - { pruneCompletedJobs } = await import("../activity-retention"); - const job = await seedCapacity(1), - value = checkedCheckpoint(structuredClone(job.result)); - required(required(Object.values(value.opsProjection.operations)[0]).receiptDelivery).state = - "pending"; - const cutoff = new Date(Date.now() - 36500 * 24 * 3600_000); - expect( - await state.db - .select({ id: adminJob.id }) - .from(adminJob) - .where(and(ne(adminJob.id, job.id), lt(adminJob.finishedAt, cutoff))), - ).toEqual([]); - await state.db - .update(adminJob) - .set({ - status: "failed", - finishedAt: new Date(cutoff.getTime() - 24 * 3600_000), - result: value, - }) - .where(eq(adminJob.id, job.id)); - expect(await pruneCompletedJobs(36500)).toBe(0); - expect(await row(job.id)).toBeDefined(); + await lockReady.promise; + const first = executeAdminJobOperation( + required(holders.contexts[0]), + { kind: "backup.delete", backupId: "winner" }, + "test", + { operationKey: "opk1_sameaddress12345", client: remote.client, pollIntervalMs: 1 }, + ); + await new Promise((r) => setTimeout(r, 15)); + const second = executeAdminJobOperation( + required(holders.contexts[1]), + { kind: "backup.delete", backupId: "loser" }, + "test", + { operationKey: "opk1_sameaddress12345", client: remote.client, pollIntervalMs: 1 }, + ); + unlock.resolve(); + await locked; + await Promise.allSettled([first, second]); + const saved = checkedCheckpoint((await row(job.id)).result); + expect(Object.keys(saved.opsProjection.steps)).toHaveLength(1); + expect(Object.keys(saved.opsProjection.operations)).toHaveLength(1); + expect(remote.counts().mutations).toBe(1); + const winner = required(saved.opsProjection.steps.opk1_sameaddress12345); + expect( + ( + required(saved.opsProjection.operations[winner.operationKey]).operation as { + backupId: string; + } + ).backupId, + ).toBe("winner"); + expect(holders.contexts[1]?.checkpointResult).toMatchObject({ + opsProjection: { steps: { opk1_sameaddress12345: winner } }, }); - it.each(["success", "failure"])( - "renewal: actual runner blocks %s finalization when receipt delivery is pending", - async (outcome) => { - const job = await seed(`test.renewal.finalization.${outcome}`), - remote = successfulClient(); - const execute = remote.client.execute.bind(remote.client); - remote.client.execute = vi.fn(async (...args: Parameters) => { - if (args[0].kind === "controlRetention.acknowledge") throw new OpsClientError("timeout"); - return execute(...args); - }) as ApiOpsClient["execute"]; - jobRunner.register(job.type, async (ctx) => { - try { - await executeAdminJobOperation( - ctx, - { kind: "backup.delete", backupId: "known" }, - "test", - { client: remote.client, pollIntervalMs: 1 }, - ); - } catch {} - if (outcome === "failure") throw Error("Known parent failure"); - return { known: "success" }; - }); - await run(job); - const saved = await row(job.id); - expect(saved.status).toBe("running"); - expect(saved.finishedAt).toBeNull(); - expect(saved.result).toHaveProperty("observation.reason", "receipt_delivery"); - expect(remote.counts().mutations).toBe(1); + await holders.finish(); + }); + it("renewal: lost acknowledgement and confirmation checkpoint keep a copied known result pending across cached restart", async () => { + const job = await seedCapacity(0), + holders = await checkpointContexts(job), + remote = successfulClient(), + ctx = required(holders.contexts[0]); + const execute = remote.client.execute.bind(remote.client); + let loseAck = true; + remote.client.execute = vi.fn(async (...args: Parameters) => { + const result = await execute(...args); + if (args[0].kind === "controlRetention.acknowledge" && loseAck) { + loseAck = false; + throw new OpsClientError("timeout"); + } + return result; + }) as ApiOpsClient["execute"]; + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "original" }, "test", { + operationKey: "opk1_receiptdelivery123", + client: remote.client, + pollIntervalMs: 1, + }), + ).rejects.toMatchObject({ reason: "receipt_delivery" }); + const saved = checkedCheckpoint((await row(job.id)).result), + projected = required(Object.values(saved.opsProjection.operations)[0]); + expect(projected).toMatchObject({ + terminal: true, + result: { backupId: "original" }, + receiptDelivery: { + state: "pending", + terminalReceipt: { state: "succeeded", result: { backupId: "original" } }, }, - ); - it("renewal: ambiguous committed allocation sends no POST and recovery remains lookup-only", async () => { - const job = await seed("test.renewal.ambiguous"), - holders = await checkpointContexts(job), - remote = successfulClient(); - const transaction = state.db.transaction.bind(state.db); - vi.spyOn(state.db, "transaction").mockImplementationOnce(async (fn) => { - await transaction(fn); - throw Error("lost commit response"); - }); - await expect( - executeAdminJobOperation( - required(holders.contexts[0]), - { kind: "backup.delete", backupId: "original" }, - "test", - { operationKey: "opk1_ambiguousaddress", client: remote.client }, - ), - ).rejects.toThrow(); - expect( - Object.keys(checkedCheckpoint((await row(job.id)).result).opsProjection.steps), - ).toHaveLength(1); - const fresh = (await row(job.id)).result; - required(holders.contexts[1]).checkpointResult = fresh as Record; - await expect( - executeAdminJobOperation( - required(holders.contexts[1]), - { kind: "backup.delete", backupId: "changed" }, - "test", - { operationKey: "opk1_ambiguousaddress", client: remote.client, pollIntervalMs: 1 }, - ), - ).rejects.toThrow(); - expect(remote.counts().mutations).toBe(0); - await holders.finish(); }); - it("renewal: distinct stable addresses merge into the current row without erasing sibling receipts", async () => { - const job = await seedCapacity(1), - holders = await checkpointContexts(job), - remote = successfulClient(); - await Promise.allSettled( - holders.contexts.map((ctx, n) => - executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: `new-${n}` }, "test", { - operationKey: `opk1_distinctaddress${n}`, - client: remote.client, - pollIntervalMs: 1, - }), + expect(projected.outcome).toBeUndefined(); + const { cancelAdminJobOperations } = await import("../admin-job-ops"); + loseAck = true; + await expect( + cancelAdminJobOperations(saved, remote.client, { context: ctx, jobId: job.id }), + ).rejects.toMatchObject({ reason: "receipt_delivery" }); + state.client = remote.client; + loseAck = true; + expect(await jobRunner.cancel(job.id)).toBe(false); + expect((await row(job.id)).result).toMatchObject({ + observation: { state: "receipt_delivery", reason: "receipt_delivery" }, + }); + const { and } = await import("drizzle-orm"), + { noPendingReceiptDelivery } = await import("../admin-job-state"); + expect( + await state.db + .update(adminJob) + .set({ status: "success" }) + .where(and(eq(adminJob.id, job.id), noPendingReceiptDelivery())) + .returning({ id: adminJob.id }), + ).toEqual([]); + const restarted = required(holders.contexts[1]); + restarted.checkpointResult = saved; + const abort = new AbortController(); + abort.abort(); + restarted.signal = abort.signal; + const checkpoint = restarted.checkpoint; + let blockConfirmation = true; + restarted.checkpoint = async (...args) => { + const entries = Object.values(readProjection(args[0]).operations); + if (blockConfirmation && entries.some((o) => o.receiptDelivery?.state === "confirmed")) + throw Error("confirmation DB unavailable"); + await checkpoint(...args); + }; + await expect( + executeAdminJobOperation(restarted, { kind: "backup.delete", backupId: "changed" }, "test", { + operationKey: "opk1_receiptdelivery123", + client: remote.client, + }), + ).rejects.toMatchObject({ reason: "receipt_delivery" }); + expect( + checkedCheckpoint((await row(job.id)).result).opsProjection.operations[projected.operationKey] + ?.receiptDelivery?.state, + ).toBe("pending"); + blockConfirmation = false; + await expect( + executeAdminJobOperation( + restarted, + { kind: "backup.delete", backupId: "changed-again" }, + "test", + { operationKey: "opk1_receiptdelivery123", client: remote.client }, + ), + ).resolves.toEqual({ backupId: "original" }); + const ackCalls = vi + .mocked(remote.client.execute) + .mock.calls.filter((c) => c[0].kind === "controlRetention.acknowledge"); + expect( + new Set( + ackCalls.map((c) => + c[0].kind === "controlRetention.acknowledge" ? c[0].proof : undefined, ), - ); - const saved = checkedCheckpoint((await row(job.id)).result); - expect(Object.keys(saved.opsProjection.steps)).toHaveLength(3); - expect(Object.keys(saved.opsProjection.operations)).toHaveLength(3); - expect(saved.untouched).toEqual({ receipt: "preserve" }); - expect(saved.opsProjection.eventTotal).toBe(7); - expect(remote.counts().mutations).toBe(2); - await holders.finish(); + ).size, + ).toBe(1); + expect(ackCalls.every((c) => !c[1]?.signal?.aborted)).toBe(true); + expect(remote.counts().mutations).toBe(1); + await holders.finish(); + }); + it("renewal: current-row log capacity admits one last event and stale checkpoints cannot reset its charge", async () => { + const job = await seedCapacity(0), + execution = required(readJobExecution(job.result)); + const stale = { ...job.result, _opsLogEvents: 1999, _opsLogBytes: 0 }; + await state.db.update(adminJob).set({ result: stale }).where(eq(adminJob.id, job.id)); + const { insertOwnedJobLog } = await import("../admin-job-state"); + const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); + const entries = [0, 1].map((seq) => ({ + id: `${job.id}:last:${seq}`, + jobId: job.id, + seq, + stream: "stdout", + line: "last", + createdAt: new Date(), + })); + const accepted = await Promise.all(entries.map((entry) => insertOwnedJobLog(execution, entry))); + expect(accepted.filter(Boolean)).toHaveLength(1); + const winner = required(entries[accepted.indexOf(true)]); + expect(await insertOwnedJobLog(execution, winner)).toBe(false); + await checkpointOwnedAdminJob( + job.id, + execution, + { ...stale, retainedPatch: true }, + undefined, + undefined, + undefined, + stale, + ); + expect((await row(job.id)).result).toMatchObject({ + _opsLogEvents: 2000, + _opsLogBytes: 4, + retainedPatch: true, }); - it("renewal: two distinct allocators at63 share one last slot, never persist65, and preserve all receipts", async () => { - const job = await seedCapacity(63), - holders = await checkpointContexts(job), + expect( + await state.db.select().from(adminJobLog).where(eq(adminJobLog.jobId, job.id)), + ).toHaveLength(1); + }); + it("renewal: actual expiry cleanup preserves the last pending terminal copy", async () => { + const { lt, and, ne } = await import("drizzle-orm"), + { pruneCompletedJobs } = await import("../activity-retention"); + const job = await seedCapacity(1), + value = checkedCheckpoint(structuredClone(job.result)); + required(required(Object.values(value.opsProjection.operations)[0]).receiptDelivery).state = + "pending"; + const cutoff = new Date(Date.now() - 36500 * 24 * 3600_000); + expect( + await state.db + .select({ id: adminJob.id }) + .from(adminJob) + .where(and(ne(adminJob.id, job.id), lt(adminJob.finishedAt, cutoff))), + ).toEqual([]); + await state.db + .update(adminJob) + .set({ + status: "failed", + finishedAt: new Date(cutoff.getTime() - 24 * 3600_000), + result: value, + }) + .where(eq(adminJob.id, job.id)); + expect(await pruneCompletedJobs(36500)).toBe(0); + expect(await row(job.id)).toBeDefined(); + }); + it.each(["success", "failure"])( + "renewal: actual runner blocks %s finalization when receipt delivery is pending", + async (outcome) => { + const job = await seed(`test.renewal.finalization.${outcome}`), remote = successfulClient(); - const results = await Promise.allSettled( - holders.contexts.map((ctx, n) => - executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: `last-${n}` }, "test", { - operationKey: `opk1_lastslotaddress${n}`, - client: remote.client, - pollIntervalMs: 1, - }), - ), - ); - expect( - results.filter( - (r) => - r.status === "rejected" && - (r.reason as Error).constructor.name === "JobStepCapacityExceeded", - ), - ).toHaveLength(1); - const saved = checkedCheckpoint((await row(job.id)).result); - expect(Object.keys(saved.opsProjection.steps)).toHaveLength(64); - expect(Object.keys(saved.opsProjection.operations)).toHaveLength(64); + const execute = remote.client.execute.bind(remote.client); + remote.client.execute = vi.fn(async (...args: Parameters) => { + if (args[0].kind === "controlRetention.acknowledge") throw new OpsClientError("timeout"); + return execute(...args); + }) as ApiOpsClient["execute"]; + jobRunner.register(job.type, async (ctx) => { + try { + await executeAdminJobOperation( + ctx, + { kind: "backup.delete", backupId: "known" }, + "test", + { client: remote.client, pollIntervalMs: 1 }, + ); + } catch {} + if (outcome === "failure") throw Error("Known parent failure"); + return { known: "success" }; + }); + await run(job); + const saved = await row(job.id); + expect(saved.status).toBe("running"); + expect(saved.finishedAt).toBeNull(); + expect(saved.result).toHaveProperty("observation.reason", "receipt_delivery"); expect(remote.counts().mutations).toBe(1); - for (const [key, value] of Object.entries( - checkedCheckpoint(job.result).opsProjection.operations, - )) - expect(saved.opsProjection.operations[key]).toEqual(value); - await holders.finish(); + }, + ); + it("renewal: ambiguous committed allocation sends no POST and recovery remains lookup-only", async () => { + const job = await seed("test.renewal.ambiguous"), + holders = await checkpointContexts(job), + remote = successfulClient(); + const transaction = state.db.transaction.bind(state.db); + vi.spyOn(state.db, "transaction").mockImplementationOnce(async (fn) => { + await transaction(fn); + throw Error("lost commit response"); }); - it("renewal: full mixed legacy/v3 rows refuse a missing address but observe an existing winner at capacity", async () => { - const job = await seedCapacity(64, true), - holders = await checkpointContexts(job, 1), - remote = successfulClient(), - ctx = required(holders.contexts[0]); - const before = structuredClone((await row(job.id)).result); - await expect( - executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "refused" }, "test", { - operationKey: "opk1_missingfulladdress", + await expect( + executeAdminJobOperation( + required(holders.contexts[0]), + { kind: "backup.delete", backupId: "original" }, + "test", + { operationKey: "opk1_ambiguousaddress", client: remote.client }, + ), + ).rejects.toThrow(); + expect( + Object.keys(checkedCheckpoint((await row(job.id)).result).opsProjection.steps), + ).toHaveLength(1); + const fresh = (await row(job.id)).result; + required(holders.contexts[1]).checkpointResult = fresh as Record; + await expect( + executeAdminJobOperation( + required(holders.contexts[1]), + { kind: "backup.delete", backupId: "changed" }, + "test", + { operationKey: "opk1_ambiguousaddress", client: remote.client, pollIntervalMs: 1 }, + ), + ).rejects.toThrow(); + expect(remote.counts().mutations).toBe(0); + await holders.finish(); + }); + it("renewal: distinct stable addresses merge into the current row without erasing sibling receipts", async () => { + const job = await seedCapacity(1), + holders = await checkpointContexts(job), + remote = successfulClient(); + await Promise.allSettled( + holders.contexts.map((ctx, n) => + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: `new-${n}` }, "test", { + operationKey: `opk1_distinctaddress${n}`, client: remote.client, + pollIntervalMs: 1, }), - ).rejects.toBeInstanceOf((await import("../admin-job-state")).JobStepCapacityExceeded); - const address = required(Object.keys(checkedCheckpoint(job.result).opsProjection.steps)[0]); - await expect( - executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "changed" }, "test", { - operationKey: address, + ), + ); + const saved = checkedCheckpoint((await row(job.id)).result); + expect(Object.keys(saved.opsProjection.steps)).toHaveLength(3); + expect(Object.keys(saved.opsProjection.operations)).toHaveLength(3); + expect(saved.untouched).toEqual({ receipt: "preserve" }); + expect(saved.opsProjection.eventTotal).toBe(7); + expect(remote.counts().mutations).toBe(2); + await holders.finish(); + }); + it("renewal: two distinct allocators at63 share one last slot, never persist65, and preserve all receipts", async () => { + const job = await seedCapacity(63), + holders = await checkpointContexts(job), + remote = successfulClient(); + const results = await Promise.allSettled( + holders.contexts.map((ctx, n) => + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: `last-${n}` }, "test", { + operationKey: `opk1_lastslotaddress${n}`, client: remote.client, + pollIntervalMs: 1, }), - ).resolves.toMatchObject({ backupId: expect.stringMatching(/^old-/) }); - expect((await row(job.id)).result).toEqual(before); - expect(remote.counts().mutations).toBe(0); - await holders.finish(); - }); - }, -); + ), + ); + expect( + results.filter( + (r) => + r.status === "rejected" && + (r.reason as Error).constructor.name === "JobStepCapacityExceeded", + ), + ).toHaveLength(1); + const saved = checkedCheckpoint((await row(job.id)).result); + expect(Object.keys(saved.opsProjection.steps)).toHaveLength(64); + expect(Object.keys(saved.opsProjection.operations)).toHaveLength(64); + expect(remote.counts().mutations).toBe(1); + for (const [key, value] of Object.entries( + checkedCheckpoint(job.result).opsProjection.operations, + )) + expect(saved.opsProjection.operations[key]).toEqual(value); + await holders.finish(); + }); + it("renewal: full mixed legacy/v3 rows refuse a missing address but observe an existing winner at capacity", async () => { + const job = await seedCapacity(64, true), + holders = await checkpointContexts(job, 1), + remote = successfulClient(), + ctx = required(holders.contexts[0]); + const before = structuredClone((await row(job.id)).result); + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "refused" }, "test", { + operationKey: "opk1_missingfulladdress", + client: remote.client, + }), + ).rejects.toBeInstanceOf((await import("../admin-job-state")).JobStepCapacityExceeded); + const address = required(Object.keys(checkedCheckpoint(job.result).opsProjection.steps)[0]); + await expect( + executeAdminJobOperation(ctx, { kind: "backup.delete", backupId: "changed" }, "test", { + operationKey: address, + client: remote.client, + }), + ).resolves.toMatchObject({ backupId: expect.stringMatching(/^old-/) }); + expect((await row(job.id)).result).toEqual(before); + expect(remote.counts().mutations).toBe(0); + await holders.finish(); + }); +}); diff --git a/apps/api/src/services/activity-jobs.ts b/apps/api/src/services/activity-jobs.ts index ed91d2be5..a462f7185 100644 --- a/apps/api/src/services/activity-jobs.ts +++ b/apps/api/src/services/activity-jobs.ts @@ -1,4 +1,13 @@ -import { count, desc, eq, inArray, sql } from "drizzle-orm"; +import { + dataExecutionIdentitySchema, + dataExecutionInputSchema, +} from "@openmapx/core/data-manager-contracts"; +import { + createDataManagerTransport, + requestDataExecutionCancellation, +} from "@openmapx/core/data-manager-sdk"; +import { envString } from "@openmapx/core/server-env"; +import { and, count, desc, eq, inArray, or, sql } from "drizzle-orm"; import { unionAll } from "drizzle-orm/pg-core"; import { db } from "../db"; import { adminJob, adminJobLog, dataManagerJobStages, dataManagerJobs } from "../db/schema"; @@ -48,8 +57,8 @@ const APPLICATION_STATUSES: Record = { }; const DATA_MANAGER_STATUSES: Record = { - active: ["running"], - completed: ["ok", "partial"], + active: ["queued", "running", "cancel_pending"], + completed: ["ok", "partial", "canceled"], failed: ["error", "interrupted"], }; @@ -93,7 +102,33 @@ function applicationWhere(filter?: ActivityJobFilter) { } function dataManagerWhere(filter?: ActivityJobFilter) { - return filter ? inArray(dataManagerJobs.status, DATA_MANAGER_STATUSES[filter]) : undefined; + if (!filter) return undefined; + const ordinary = inArray(dataManagerJobs.status, DATA_MANAGER_STATUSES[filter]); + return filter === "active" + ? or(ordinary, dataManagerRecoveryPending()) + : filter === "failed" + ? and(ordinary, sql`NOT (${dataManagerRecoveryPending()})`) + : ordinary; +} +function dataManagerRecoveryPending() { + return sql`coalesce(${dataManagerJobs.executionVersion}=2 AND ${dataManagerJobs.status}='interrupted' AND ${dataManagerJobs.finishedAt} IS NULL AND ${dataManagerJobs.executionState}->>'publication'='unknown',false)`; +} +function currentDataManagerMutation(job: { + executionVersion: number | null; + status: string; + finishedAt: Date | null; + executionState: unknown; +}) { + return ( + job.executionVersion === 2 && + (["queued", "running", "cancel_pending"].includes(job.status) || + (job.status === "interrupted" && + job.finishedAt === null && + typeof job.executionState === "object" && + job.executionState !== null && + "publication" in job.executionState && + job.executionState.publication === "unknown")) + ); } function applicationSelect(filter?: ActivityJobFilter) { @@ -128,7 +163,10 @@ function dataManagerSelect(filter?: ActivityJobFilter) { when ${dataManagerJobs.status} = 'error' then 'failed' else ${dataManagerJobs.status} end`.as("status"), - payload: dataManagerJobs.metadata, + payload: + sql`CASE WHEN ${dataManagerJobs.executionVersion}=2 THEN ${dataManagerJobs.executionInput} ELSE ${dataManagerJobs.metadata} END`.as( + "payload", + ), error: sql`null::text`.as("error"), progress: sql`null::integer`.as("progress"), createdBy: sql`null::text`.as("created_by"), @@ -136,7 +174,10 @@ function dataManagerSelect(filter?: ActivityJobFilter) { createdAt: dataManagerJobs.startedAt, startedAt: dataManagerJobs.startedAt, finishedAt: dataManagerJobs.finishedAt, - cancelable: sql`false`.as("cancelable"), + cancelable: + sql`${dataManagerJobs.executionVersion}=2 AND (${dataManagerJobs.status} IN ('queued','running','cancel_pending') OR ${dataManagerRecoveryPending()}) AND ${dataManagerJobs.kind} IN ('download-osm','download-fonts','convert-overpass','transit-sync','overture-pull','overture-ingest','overture-extract','overture-conflate','overture-sync','search-index-build','notable-places-build','generation-hardlinks','transit-maintenance') AND ${dataManagerJobs.operationKey} IS NOT NULL AND ${dataManagerJobs.executionFingerprint} IS NOT NULL`.as( + "cancelable", + ), }) .from(dataManagerJobs) .where(dataManagerWhere(filter)); @@ -228,7 +269,7 @@ export async function getActivityJob(id: string, source: ActivityJobSource) { id: String(job.id), type: job.kind, status: normalizeDataManagerJobStatus(job.status), - payload: job.metadata, + payload: job.executionVersion === 2 ? job.executionInput : job.metadata, result: null, error: null, progress: null, @@ -237,8 +278,36 @@ export async function getActivityJob(id: string, source: ActivityJobSource) { createdAt: job.startedAt, startedAt: job.startedAt, finishedAt: job.finishedAt, - cancelable: false, + cancelable: + currentDataManagerMutation(job) && + dataExecutionInputSchema.safeParse(job.executionInput).success, logs: [], stages, }; } + +/** Load the receiver-owned immutable tuple; browser parameters select only the + * visible job. The receiver persists cancellation before signaling its worker. */ +export async function cancelDataManagerActivityJob(id: string): Promise { + const [job] = await db.select().from(dataManagerJobs).where(eq(dataManagerJobs.id, id)).limit(1); + if ( + !job || + !currentDataManagerMutation(job) || + !dataExecutionInputSchema.safeParse(job.executionInput).success + ) + return false; + const identity = dataExecutionIdentitySchema.parse({ + id: String(job.id), + operationKey: job.operationKey, + fingerprint: job.executionFingerprint, + }); + await requestDataExecutionCancellation(identity, { + transport: createDataManagerTransport({ + baseUrl: envString("DATA_MANAGER_URL", "http://localhost:4000"), + authToken: envString("DATA_MANAGER_AUTH_TOKEN", ""), + requestTimeoutMs: 15000, + }), + signal: AbortSignal.timeout(15000), + }); + return true; +} diff --git a/apps/api/src/services/admin-data-children.ts b/apps/api/src/services/admin-data-children.ts new file mode 100644 index 000000000..b49f2e16e --- /dev/null +++ b/apps/api/src/services/admin-data-children.ts @@ -0,0 +1,231 @@ +/** Finite existing single-data command descendants. Addresses come from the + * retained agent root, including before a CLI/DM admission response exists. */ + +import { controlChildOperationKey, isFixedSingleDataCommand } from "@openmapx/core/build-artifacts"; +import { + dataExecutionIdentitySchema, + dataExecutionStatusSchema, + isDataExecutionTerminal, +} from "@openmapx/core/data-manager-contracts"; +import { + createDataManagerTransport, + getDataExecution, + requestDataExecutionCancellation, + unwrapDataManagerResponse, +} from "@openmapx/core/data-manager-sdk"; +import { envString } from "@openmapx/core/server-env"; +import { eq } from "drizzle-orm"; +import z from "zod/v4"; +import { db } from "../db"; +import { dataManagerJobs } from "../db/schema"; +import { JobObservationPending } from "./admin-job-failure"; +import type { JobContext } from "./job-runner"; + +const childSchema = z.strictObject({ + ...dataExecutionIdentitySchema.shape, + parentAgentOperationKey: dataExecutionIdentitySchema.shape.operationKey, + kind: dataExecutionStatusSchema.shape.kind, + admittedAt: dataExecutionStatusSchema.shape.admittedAt, + deadlineAt: dataExecutionStatusSchema.shape.deadlineAt, + cancellationRequestedAt: z.iso.datetime().nullable(), + state: z.enum(["observing", "terminal"]), + status: dataExecutionStatusSchema.shape.status, + publication: dataExecutionStatusSchema.shape.publication, +}); +const childrenSchema = z + .array(childSchema) + .max(32) + .refine( + (children) => + Buffer.byteLength(JSON.stringify(children)) <= 128 * 1024 && + new Set(children.map((child) => child.operationKey)).size === children.length, + ); +export type DataChildrenCancellation = + | "none" + | "succeeded" + | "contained" + | "already_terminal" + | "pending"; +export async function cancelAdminDataChildren( + result: unknown, + roots: readonly { kind: string; operationKey: string }[], + options: { + context?: JobContext; + signal?: AbortSignal; + sleep?: (ms: number) => Promise; + pollIntervalMs?: number; + } = {}, +): Promise { + const known = childrenSchema.safeParse( + (result as { _dataExecutions?: unknown } | null)?._dataExecutions ?? [], + ); + if (!known.success) throw new JobObservationPending("invalid_identity"); + const children = known.data; + const candidates = roots.filter((root) => isFixedSingleDataCommand(root.kind)); + if (!children.length && !candidates.length) return "none"; + if (!options.context) return "pending"; + const persist = async () => { + await options.context!.checkpoint({ + ...options.context!.checkpointResult, + _dataExecutions: children, + }); + }; + // Actual DM rows can outlive both its CLI and the API observer. The verified + // DM receiver wrote this parent binding; an absent row remains unknown to its + // agent-root reconciliation and never authorizes another POST. + for (const root of candidates) { + const cliKey = controlChildOperationKey(root.operationKey, "data-workflow", 0), + key = controlChildOperationKey(cliKey, "data-step", 0); + const rows = await db + .select() + .from(dataManagerJobs) + .where(eq(dataManagerJobs.operationKey, key)) + .limit(2); + if (rows.length > 1) throw new JobObservationPending("invalid_identity"); + const row = rows[0]; + if (!row) continue; + const parent = z + .object({ + issuer: z.literal("host-cli"), + execution: z.object({ executor: z.literal("host-cli"), operationKey: z.literal(cliKey) }), + }) + .safeParse(row.executionParent); + if (row.executionVersion !== 2 || !parent.success) + throw new JobObservationPending("invalid_identity"); + const state = dataExecutionStatusSchema.parse({ + id: String(row.id), + kind: row.kind, + status: row.status, + operationKey: row.operationKey, + fingerprint: row.executionFingerprint, + admittedAt: Number(row.admittedAt), + deadlineAt: Number(row.deadlineAt), + ...z + .object({ + cancellationRequestedAt: z.string().nullable(), + progress: z.number().nullable(), + diagnostics: z.unknown(), + publication: z.string(), + result: z.unknown().optional(), + }) + .parse(row.executionState), + }); + const child = childSchema.parse({ + id: state.id, + operationKey: key, + fingerprint: state.fingerprint, + parentAgentOperationKey: root.operationKey, + kind: state.kind, + admittedAt: state.admittedAt, + deadlineAt: state.deadlineAt, + cancellationRequestedAt: null, + state: isDataExecutionTerminal(state) ? "terminal" : "observing", + status: state.status, + publication: state.publication, + }), + old = children.find((item) => item.operationKey === key); + if (old) { + if ( + old.id !== child.id || + old.fingerprint !== child.fingerprint || + old.kind !== child.kind || + old.admittedAt !== child.admittedAt || + old.deadlineAt !== child.deadlineAt || + old.parentAgentOperationKey !== root.operationKey + ) + throw new JobObservationPending("invalid_identity"); + } else children.push(child); + } + childrenSchema.parse(children); + if (!children.length) return "none"; + await persist(); + const transport = createDataManagerTransport({ + baseUrl: envString("DATA_MANAGER_URL", "http://localhost:4000"), + authToken: envString("DATA_MANAGER_AUTH_TOKEN", ""), + requestTimeoutMs: 2000, + }); + const outcomes: DataChildrenCancellation[] = []; + let unresolved = false; + const readSignal = () => + options.signal + ? AbortSignal.any([options.signal, AbortSignal.timeout(2000)]) + : AbortSignal.timeout(2000), + sleep = + options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + for (const child of children) { + const match = (value: unknown) => { + const status = dataExecutionStatusSchema.parse(value); + if ( + status.id !== child.id || + status.operationKey !== child.operationKey || + status.fingerprint !== child.fingerprint || + status.kind !== child.kind || + status.admittedAt !== child.admittedAt || + status.deadlineAt !== child.deadlineAt + ) + throw new JobObservationPending("invalid_identity"); + return status; + }; + let status: ReturnType; + try { + status = match( + unwrapDataManagerResponse( + await getDataExecution(child.id, { transport, signal: readSignal() }), + ), + ); + } catch { + unresolved = true; + continue; + } + if (!isDataExecutionTerminal(status)) { + child.cancellationRequestedAt ??= new Date().toISOString(); + await persist(); + try { + status = match( + await requestDataExecutionCancellation( + { id: child.id, operationKey: child.operationKey, fingerprint: child.fingerprint }, + { transport, signal: readSignal() }, + ), + ); + } catch { + /* Receiver may have persisted intent; exact GET observation follows. */ + } + for (let attempt = 0; attempt < 8 && !isDataExecutionTerminal(status); attempt++) { + await sleep(options.pollIntervalMs ?? 250); + try { + status = match( + unwrapDataManagerResponse( + await getDataExecution(child.id, { transport, signal: readSignal() }), + ), + ); + } catch { + unresolved = true; + break; + } + } + } + if (!isDataExecutionTerminal(status)) { + unresolved = true; + continue; + } + Object.assign(child, { + state: "terminal", + status: status.status, + publication: status.publication, + }); + await persist(); + outcomes.push( + status.status === "ok" + ? "succeeded" + : status.status === "canceled" + ? "contained" + : "already_terminal", + ); + } + if (unresolved) return "pending"; + return outcomes.includes("contained") + ? "contained" + : outcomes.every((outcome) => outcome === "succeeded") + ? "succeeded" + : "already_terminal"; +} diff --git a/apps/api/src/services/admin-job-ops.ts b/apps/api/src/services/admin-job-ops.ts index 138065900..aafc50825 100644 --- a/apps/api/src/services/admin-job-ops.ts +++ b/apps/api/src/services/admin-job-ops.ts @@ -23,6 +23,7 @@ import { readOpsTokenFile, signApiRetentionProof, } from "@openmapx/core/ops"; +import { cancelAdminDataChildren } from "./admin-data-children"; import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; import { type ApiJobChildAttempt, @@ -898,6 +899,11 @@ export async function cancelAdminJobOperations( ): Promise { const projection = readProjection(result); const pending = Object.values(projection.operations); + const backend = await cancelAdminDataChildren(result, pending, options); + // Reconcile every known root even when a backend sibling remains pending. + // No agent projection is not proof that no registered backend remains. + // Known descendants have been explicitly canceled/observed above; missing + // original parent containment still keeps the application pending. if (pending.length === 0) return "pending"; const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); @@ -906,6 +912,51 @@ export async function cancelAdminJobOperations( throw new Error("Invalid cancellation poll interval"); } + const cancellationSent = new Set(); + // Send exact retained cancellation to all roots before any held sibling can + // cause observation to return pending. Adoption is checkpointed first. + if (pending.length > 1 || backend !== "none") + for (const operation of pending) { + if (operation.phase === "refused" || operation.terminal || options.signal?.aborted) continue; + try { + const status = operation.operationId + ? await client.status(operation.kind, operation.operationId, { + signal: options.signal + ? AbortSignal.any([options.signal, AbortSignal.timeout(2000)]) + : AbortSignal.timeout(2000), + }) + : operation.operation + ? await client.lookup(operation.operation, operation.operationKey, { + signal: AbortSignal.timeout(2000), + ...(operation.attempt + ? { execution: { requestedDeadlineAt: operation.attempt.requestedDeadlineAt } } + : {}), + }) + : undefined; + if ( + !status || + status.kind !== operation.kind || + status.operationKey !== operation.operationKey || + (operation.operationId && status.operationId !== operation.operationId) || + (operation.fingerprint && status.fingerprint !== operation.fingerprint) || + (operation.deadlineAt && status.deadlineAt !== operation.deadlineAt) + ) + continue; + operation.operationId = status.operationId; + operation.fingerprint = status.fingerprint; + operation.deadlineAt = status.deadlineAt; + if (options.context) await persistProjection(options.context, projection); + if (!["succeeded", "failed", "timed_out"].includes(status.state)) { + cancellationSent.add(operation.operationKey); + await client.cancel(status.operationId, { + signal: AbortSignal.timeout(2000), + ...(status.fingerprint ? { fingerprint: status.fingerprint } : {}), + }); + } + } catch { + /* Every sibling is still attempted; response loss is observed below. */ + } + } // Each operation is reconciled on its own. A sibling that finished // successfully must not mask another that this request actually contained, // and a failure that predates this request must not be reported as a @@ -979,7 +1030,10 @@ export async function cancelAdminJobOperations( status = await client.status(operation.kind, operation.operationId, { signal: readSignal() }); if (!matches(status)) return "pending"; sawTerminationPending = status.state === "termination_pending"; - if (!["succeeded", "failed", "timed_out"].includes(status.state)) + if ( + !["succeeded", "failed", "timed_out"].includes(status.state) && + !cancellationSent.has(operation.operationKey) + ) status = await client.cancel(operation.operationId, { signal: readSignal(), ...(operation.fingerprint ? { fingerprint: operation.fingerprint } : {}), @@ -1053,6 +1107,15 @@ export async function cancelAdminJobOperations( : "already_terminal", ); } + const latestBackend = await cancelAdminDataChildren( + options.context?.checkpointResult ?? result, + pending, + options, + ); + if (backend === "pending" || latestBackend === "pending") return "pending"; + if (backend === "contained" || latestBackend === "contained") outcomes.push("contained"); + else if (backend === "already_terminal" || latestBackend === "already_terminal") + outcomes.push("already_terminal"); if (hasUnsettledHardlinks(result)) return "pending"; if (outcomes.includes("contained")) return "canceled"; return outcomes.every((outcome) => outcome === "succeeded") ? "completed" : "already_terminal"; diff --git a/apps/api/src/services/admin-job-state.ts b/apps/api/src/services/admin-job-state.ts index e44d126c3..e02802acb 100644 --- a/apps/api/src/services/admin-job-state.ts +++ b/apps/api/src/services/admin-job-state.ts @@ -43,10 +43,25 @@ export function hasUnsettledManagedOperation(result: unknown): boolean { opsProjection?: { version?: number; operations?: Record }; } | null )?.opsProjection; - return p?.version === 3 && Object.values(p.operations ?? {}).some((o) => o.terminal !== true); + const data = (result as { _dataExecutions?: unknown } | null)?._dataExecutions; + return ( + (p?.version === 3 && Object.values(p.operations ?? {}).some((o) => o.terminal !== true)) || + (data !== undefined && + (!Array.isArray(data) || + data.some( + (child) => + !child || + child.state !== "terminal" || + !["not_started", "committed"].includes(child.publication) || + !["ok", "partial", "error", "canceled", "interrupted"].includes(child.status) || + typeof child.id !== "string" || + typeof child.operationKey !== "string" || + typeof child.fingerprint !== "string", + ))) + ); } export function noUnsettledManagedOperation() { - return sql`NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS(SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS active_entry WHERE coalesce(active_entry.value->>'terminal','false') <> 'true'))`; + return sql`NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS(SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS active_entry WHERE coalesce(active_entry.value->>'terminal','false') <> 'true')) AND CASE WHEN ${adminJob.result}->'_dataExecutions' IS NULL THEN true WHEN jsonb_typeof(${adminJob.result}->'_dataExecutions') <> 'array' THEN false ELSE NOT EXISTS(SELECT 1 FROM jsonb_array_elements(${adminJob.result}->'_dataExecutions') AS backend_entry WHERE backend_entry->>'state' IS DISTINCT FROM 'terminal' OR coalesce(backend_entry->>'publication','unknown') NOT IN ('not_started','committed') OR coalesce(backend_entry->>'status','') NOT IN ('ok','partial','error','canceled','interrupted') OR coalesce(backend_entry->>'id','') !~ '^[a-f0-9-]{36}$' OR coalesce(backend_entry->>'operationKey','') !~ '^opk1_[A-Za-z0-9_-]{16,64}$' OR coalesce(backend_entry->>'fingerprint','') !~ '^[a-f0-9]{64}$') END`; } export function noPendingReceiptDelivery() { return sql`NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS (SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS receipt_entry WHERE receipt_entry.value->'receiptDelivery'->>'state' = 'pending'))`; diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index e50aedac5..594c29afe 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -18,9 +18,18 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { + bindControlChildPhase, closeControlInstallation, + consumeControlChildDispatch, + fixedCliArguments as coreFixedCliArguments, + isFixedSingleDataCommand, + issueControlChildPhase, openFixedChildInstallation, + prepareFixedControlChild, readFixedCliSpawnIntent, + readFixedSingleDataPlan, + retainFixedCliSpawnIntent, + verifyControlEffectSettlement, } from "@openmapx/core/build-artifacts"; import { type ExecutionRef, @@ -1871,40 +1880,60 @@ export function createAdministrativeRuntime( return { backupId: operation.backupId }; }; runtime["data.inspect"] = async () => inspectDataInventory(options.rootDir); + async function runMatchingDataCommand( + operation: import("@openmapx/core/ops").OpsOperation, + context: import("./runtime").OpsExecutionContext, + ): Promise { + if (!isFixedSingleDataCommand(operation.kind) || !context.controlAuthority) + throw new OpsHostUpdateRequiredError(); + const plan = readFixedSingleDataPlan(context.controlAuthority), + bound = await bindControlChildPhase(context.controlAuthority, plan.rootReservation); + const publicKey = prepareFixedControlChild(context.controlAuthority, plan.delegationId); + retainFixedCliSpawnIntent(context.controlAuthority, { + delegationId: plan.delegationId, + execution: bound.execution, + operation, + args: coreFixedCliArguments(operation), + reservations: plan.cliReservations, + }); + const issued = await issueControlChildPhase(context.controlAuthority, { + phase: "data-workflow", + ordinal: 0, + childPublicKey: publicKey, + }); + consumeControlChildDispatch(issued.dispatchTicket); + await runFixedCli(coreFixedCliArguments(operation), { + ...context, + execution: bound.execution, + parentAssertion: issued.assertion, + operationKind: operation.kind, + }); + const child = openFixedChildInstallation({ + infraDir: join(options.rootDir, "infra/docker"), + assertion: issued.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + verifyControlEffectSettlement(child, bound.execution); + } finally { + closeControlInstallation(child); + } + } runtime["data.downloadOsm"] = async (operation, context) => { - await runFixedCli( - ["data", "download", "osm", ...(operation.regionId ? [operation.regionId] : [])], - { ...context, operationKind: context.claim.operation.kind }, - ); + await runMatchingDataCommand(operation, context); return { completed: true, ...(operation.regionId ? { resourceId: operation.regionId } : {}) }; }; - runtime["data.downloadFonts"] = async (_operation, context) => { - await runFixedCli(["data", "download", "fonts"], { - ...context, - operationKind: context.claim.operation.kind, - }); + runtime["data.downloadFonts"] = async (operation, context) => { + await runMatchingDataCommand(operation, context); return { completed: true }; }; - runtime["data.update"] = async (operation, context) => { - await runFixedCli( - [ - "data", - "update", - ...(operation.regionId ? [operation.regionId] : []), - ...(operation.countryCodes?.length - ? ["--countries", operation.countryCodes.join(",")] - : []), - ...(operation.failFast ? ["--fail-fast"] : []), - ], - { ...context, operationKind: context.claim.operation.kind }, - ); - return { completed: true, ...(operation.regionId ? { resourceId: operation.regionId } : {}) }; + // Full Update build/link finalization is the reviewed Task02.4 owner. Until + // then reject before any download rather than report partial workflow success. + runtime["data.update"] = async () => { + throw new OpsHostUpdateRequiredError(); }; runtime["data.convertOverpass"] = async (operation, context) => { - await runFixedCli( - ["data", "convert", "overpass", ...(operation.regionId ? [operation.regionId] : [])], - { ...context, operationKind: context.claim.operation.kind }, - ); + await runMatchingDataCommand(operation, context); return { completed: true, ...(operation.regionId ? { resourceId: operation.regionId } : {}) }; }; runtime["data.link"] = async (_operation, context) => { @@ -1929,36 +1958,19 @@ export function createAdministrativeRuntime( return { completed: true }; }; runtime["data.overtureSync"] = async (operation, context) => { - await runFixedCli(["data", "overture-sync", operation.regionId], { - ...context, - operationKind: context.claim.operation.kind, - }); + await runMatchingDataCommand(operation, context); return { completed: true, resourceId: operation.regionId }; }; runtime["data.overtureConflate"] = async (operation, context) => { - await runFixedCli( - [ - "data", - "overture-conflate", - operation.regionId, - ...(operation.restart ? ["--restart"] : []), - ], - { ...context, operationKind: context.claim.operation.kind }, - ); + await runMatchingDataCommand(operation, context); return { completed: true, resourceId: operation.regionId }; }; runtime["data.searchIndexBuild"] = async (operation, context) => { - await runFixedCli(["data", "search-index", "build", operation.regionId], { - ...context, - operationKind: context.claim.operation.kind, - }); + await runMatchingDataCommand(operation, context); return { completed: true, resourceId: operation.regionId }; }; - runtime["data.notablePlacesBuild"] = async (_operation, context) => { - await runFixedCli(["data", "notable-places", "build"], { - ...context, - operationKind: context.claim.operation.kind, - }); + runtime["data.notablePlacesBuild"] = async (operation, context) => { + await runMatchingDataCommand(operation, context); return { completed: true }; }; runtime["system.diagnostics"] = async (_operation, context) => { diff --git a/apps/ops-agent/src/control-admission.test.ts b/apps/ops-agent/src/control-admission.test.ts index 2fdaf19b2..3177cf85c 100644 --- a/apps/ops-agent/src/control-admission.test.ts +++ b/apps/ops-agent/src/control-admission.test.ts @@ -11,9 +11,61 @@ import { } from "@openmapx/core/ops"; import { expect, it } from "vitest"; import { type OpsJobJournal, openOpsJobJournal } from "./journal"; +import type { OpsExecutionContext } from "./runtime"; import { buildOpsAgentServer } from "./server"; -it.each(["normal", "ready-failure", "wrong-target"])( +it("refuses paired approval as a bearer-only root before reserving a worker", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-closed-approval-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const app = buildOpsAgentServer({ + controlInstallation: installation, + journal, + tokens: { api: "a".repeat(32), "data-manager": "b".repeat(32) }, + resourcePolicy: { allowGlobal: () => true }, + }); + try { + const requestedDeadlineAt = Date.now() + 60000; + const response = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${"b".repeat(32)}`, "x-ops-protocol-version": "2" }, + payload: { + version: 2, + requestId: `ops1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 10000).toISOString(), + execution: { requestedDeadlineAt }, + operation: { + kind: "transitousLock.approve", + proposalId: randomUUID(), + validationId: randomUUID(), + expectedGenerationId: null, + diffDigest: "a".repeat(64), + acknowledgeLicenseChanges: false, + }, + }, + }); + expect(response.statusCode, response.body).toBe(403); + expect(journal.records()).toEqual([]); + } finally { + await app.close(); + control.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each(["normal", "ready-failure", "wrong-target", "canceled", "cancel-journal-failure"])( "adopts at authenticated HTTP admission without replay (%s)", async (scenario) => { const failReadyWrite = scenario === "ready-failure"; @@ -91,14 +143,21 @@ it.each(["normal", "ready-failure", "wrong-target"])( childPublicKey: agent.publicKey, }); let calls = 0; + let releaseWorker = () => {}; + const worker = new Promise((resolve) => { + releaseWorker = resolve; + }); + let workerContext: OpsExecutionContext | undefined; const token = "b".repeat(32); const journalFile = join(root, "data/ops-agent/jobs-v1.json"); const options = { tokens: { api: "a".repeat(32), "data-manager": token }, controlInstallation: agent, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, - dispatch: async () => { + dispatch: async (_operation: unknown, _runtime: unknown, context: OpsExecutionContext) => { calls++; + workerContext = context; + if (scenario === "canceled" || scenario === "cancel-journal-failure") await worker; return { changed: true, state: "stopped" }; }, }; @@ -108,6 +167,11 @@ it.each(["normal", "ready-failure", "wrong-target"])( records: () => journal.records(), replace: async (records) => { if (failReadyWrite && ++writes === 2) throw Error("synthetic ready journal failure"); + if ( + scenario === "cancel-journal-failure" && + records.some((record) => record.state === "termination_pending") + ) + throw Error("synthetic cancellation journal failure"); await journal.replace(records); }, }; @@ -152,6 +216,39 @@ it.each(["normal", "ready-failure", "wrong-target"])( return; } expect(result.statusCode, result.body).toBe(202); + if (scenario === "canceled" || scenario === "cancel-journal-failure") { + await expect.poll(() => calls).toBe(1); + const cancellation = app.inject({ + method: "DELETE", + url: `/v1/operations/${child.executionId}`, + headers: { + authorization: `Bearer ${token}`, + "x-ops-protocol-version": "2", + "x-ops-operation-fingerprint": child.fingerprint, + }, + }); + await expect.poll(() => workerContext?.signal.aborted).toBe(true); + if (!workerContext?.artifactLease || !workerContext.controlAuthority) + throw Error("Missing real adopted authority"); + const { artifactLease, controlAuthority } = workerContext; + expect(() => control.assertArtifactTargetLeaseForward(artifactLease)).toThrow(); + expect(() => control.assertControlExecutionForward(controlAuthority)).toThrow(); + await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); + if (scenario === "canceled") + expect(journal.records()[0]).toMatchObject({ state: "termination_pending" }); + expect(journal.records()[0]?.terminalAt).toBeUndefined(); + if (scenario === "cancel-journal-failure") { + expect((await cancellation).statusCode).toBe(500); + const status = await app.inject({ + url: `/v1/operations/${child.executionId}`, + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }, + }); + expect(status.json().result.state).toBe("termination_pending"); + } + releaseWorker(); + await cancellation; + return; + } await expect .poll( async () => @@ -194,6 +291,7 @@ it.each(["normal", "ready-failure", "wrong-target"])( expect(status.json().result.state).toBe("succeeded"); await releasedRestart.close(); } finally { + releaseWorker(); await app.close(); control.closeArtifactTargetLeaseHandle(lease); control.closeControlInstallation(agent); diff --git a/apps/ops-agent/src/data-transit-resource-source.test.ts b/apps/ops-agent/src/data-transit-resource-source.test.ts new file mode 100644 index 000000000..9cbf09b8a --- /dev/null +++ b/apps/ops-agent/src/data-transit-resource-source.test.ts @@ -0,0 +1,503 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + createControlRootOperationKey, + type ExecutionRef, + executionFingerprint, + OpsClient, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { applyHardlinkPlan } from "../../../packages/hardlinks/src/index"; +import { createDataTransitTrustedOpsDataSource } from "./data-transit-resource-source"; +import { createDockerRuntime, runContainedProcess } from "./docker-runtime"; +import { openOpsJobJournal } from "./journal"; +import { createProductionRegistryResourceClaimer } from "./policy"; +import { buildOpsAgentServer } from "./server"; + +async function fixture( + replaceAfterValidation = false, + admissionPolicy?: "stricter" | "late", + fault?: "validation" | "transport" | "reload" | "marker", +) { + const root = mkdtempSync(join(tmpdir(), "omx-real-candidate-policy-")), + data = join(root, "data"), + proxy = join(data, "motis-feed-proxy"); + mkdirSync(join(proxy, "conf"), { recursive: true }); + writeFileSync(join(proxy, "conf/default.conf"), "server { listen 80; }\n"); + writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n"); + applyHardlinkPlan( + [ + { + source: "motis-feed-proxy/conf", + target: "motis-feed-proxy/motis-feed-proxy-config", + consumerService: "motis-feed-proxy", + dataType: "motis-feed-proxy-config", + }, + ], + { rootDir: data }, + ); + await control.provisionControlInstallation(root); + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis-feed-proxy"], + }), + agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis-feed-proxy"], + }), + deadlineAt = Date.now() + 60000, + input = { kind: "transit-sync", countries: ["de"] }, + bodyHash = createHash("sha256").update(canonicalJson(input)).digest("hex"), + key = createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + parent: ExecutionRef = { + executor: "data-manager", + principal: "data-manager", + executionId: randomUUID(), + operationKey: key, + operationFingerprint: bodyHash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: input.kind, + operationFingerprint: bodyHash, + deadlineAt, + }), + }, + phase = "proxy-stage" as const, + operation = { + kind: "feedProxy.validateAndReload" as const, + candidateId: "ctrl-" + "a".repeat(32), + }, + slot: control.ControlChildReservation = { + version: 1, + phase, + ordinal: 0, + kind: operation.kind, + recipient: "ops-agent", + serviceIds: ["motis-feed-proxy"], + operationFingerprint: opsOperationFingerprint(operation), + execution: { + executor: "ops-agent", + principal: "data-manager", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: control.controlChildOperationKey(key, phase, 0), + }, + }; + // The candidate identifier is fixed by the original root, before any grant. + operation.candidateId = `ctrl-${createHash("sha256").update(`openmapx-transit-candidate-v1\0${key}\0${phase}\0${0}`).digest("hex").slice(0, 32)}`; + slot.operationFingerprint = opsOperationFingerprint(operation); + const authority = control.retainAdmittedControlExecution(dm, { + execution: parent, + kind: input.kind, + operation: input, + admittedAt: Date.now(), + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis-feed-proxy"], + reservations: [slot], + }), + lease = await control.acquireArtifactTargetLease({ + dataRoot: data, + serviceIds: ["motis-feed-proxy"], + owner: authority, + }); + control.retainOwnedDataTransitCandidate(authority, slot); + const bound = await control.bindControlChildPhase(authority, slot), + issued = await control.issueControlChildPhase(authority, { + ...slot, + lease, + childPublicKey: agent.publicKey, + }); + const claimer = createProductionRegistryResourceClaimer({ + services: [{ serviceId: "motis-feed-proxy", isBuiltIn: true, enabled: true }], + integrationIds: [], + trustedData: createDataTransitTrustedOpsDataSource(agent, { claim: async () => null }), + }); + let calls = 0; + const runtime = createDockerRuntime({ + stack: { + infraDir: root, + composePath: join(root, "compose.yml"), + composeReleasePath: join(root, "release.yml"), + }, + fileExists: () => false, + execFile: async (_file, args, options) => { + calls++; + if (args[2] === "/bin/sh") { + const bin = join(root, "fixture-bin"); + mkdirSync(bin, { recursive: true }); + writeFileSync(join(bin, "nginx"), `#!/bin/sh\nexit ${fault === "validation" ? 1 : 0}\n`, { + mode: 0o700, + }); + const completed = await runContainedProcess( + "/bin/sh", + [ + "-c", + 'PATH="$1:$PATH"; export PATH; shift; exec "$@"', + "fixed-docker-substitute", + bin, + ...args.slice(2), + ], + options, + ); + if (replaceAfterValidation) + writeFileSync(join(proxy, "conf/default.conf"), "replaced after the resource claim\n"); + if (fault === "transport") throw Error("Lost Docker transport after validation"); + if (fault === "marker") + return { ...completed, stdout: completed.stdout.replace(/:[0-9]+\n$/, ":256\n") }; + return completed; + } + if (fault === "reload" && args.includes("reload")) throw Error("Lost reload completion"); + return { stdout: "", stderr: "" }; + }, + }), + journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")), + token = "b".repeat(43), + tokenFile = join(root, "token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + const app = buildOpsAgentServer({ + runtime, + journal, + ...(admissionPolicy === "stricter" ? { operationTimeoutMs: 1000 } : {}), + ...(admissionPolicy === "late" ? { now: () => new Date(Date.now() + 120000) } : {}), + controlInstallation: agent, + resourceClaimer: claimer, + tokens: { api: "a".repeat(43), "data-manager": token }, + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("Missing actual agent address"); + const client = new OpsClient({ baseUrl: `http://127.0.0.1:${address.port}`, tokenFile }), + published = join( + data, + ".admin-target-control/executions/data-manager", + `${slot.execution.operationKey}.data-candidate.json`, + ); + return { + root, + data, + proxy, + dm, + agent, + authority, + lease, + parent, + operation, + slot, + bound, + issued, + claimer, + published, + client, + calls: () => calls, + async close() { + await app.close(); + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agent); + control.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + }, + }; +} +it.each(["validation", "validation-replaced", "transport", "reload", "marker"] as const)( + "retains only the exact actual %s validation proof", + async (fault) => { + const f = await fixture( + fault === "validation-replaced", + undefined, + fault === "validation-replaced" ? "validation" : fault, + ); + try { + control.consumeControlChildDispatch(f.issued.dispatchTicket); + const accepted = await f.client.execute(f.operation, { + operationKey: f.slot.execution.operationKey, + execution: { requestedDeadlineAt: f.bound.execution.deadlineAt }, + controlAssertion: f.issued.assertion, + }); + if (accepted.execution !== "async") throw Error("Expected actual agent admission"); + await expect + .poll(async () => (await f.client.status(f.operation.kind, accepted.operationId)).state, { + timeout: 15000, + }) + .toBe("failed"); + const authority = control.restoreAdmittedControlExecution(f.agent, f.bound.execution); + if (fault === "validation") { + await expect + .poll( + () => control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.bound.execution), + { timeout: 15000 }, + ) + .toMatchObject({ outcome: "contained", publication: "not_started" }); + expect(control.inspectFeedProxyValidationFailure(authority)).toMatch(/^[a-f0-9]{64}$/); + await expect + .poll( + () => control.verifyFeedProxyCanonicalTerminal(f.authority, f.bound.execution).state, + { timeout: 15000 }, + ) + .toBe("failed"); + const raw = JSON.parse( + readFileSync( + join( + f.root, + "control-private/ops-agent/observations", + `proxy-validation-${f.bound.execution.operationKey}.json`, + ), + "utf8", + ), + ); + const input = { + candidateDigest: raw.payload.candidateDigest, + validationExitCode: raw.payload.validationExitCode, + completionDigest: raw.payload.completionDigest, + }; + // An original failed invocation remains known after a later legitimate + // restore replaces the active bytes. Observation grants no new effect. + writeFileSync(join(f.proxy, "conf/default.conf"), "restored previous proxy bytes\n"); + expect(() => control.commitFeedProxyValidationFailure(authority, input)).not.toThrow(); + expect(() => + control.commitFeedProxyValidationFailure(authority, { ...input, validationExitCode: 2 }), + ).toThrow(); + expect(() => control.markFeedProxyReloadAttempted(authority)).toThrow(); + const status = await f.client.status(f.operation.kind, accepted.operationId); + const terminal = JSON.parse( + readFileSync( + join( + f.root, + "control-private/ops-agent/observations", + `terminal-${f.bound.execution.operationKey}.json`, + ), + "utf8", + ), + ); + expect(() => + control.repairControlJournalTerminal( + f.agent, + f.bound.execution, + status, + terminal.payload.terminalAt, + ), + ).not.toThrow(); + expect(() => + control.repairControlJournalTerminal( + f.agent, + f.bound.execution, + status, + terminal.payload.terminalAt + 1, + ), + ).toThrow(); + raw.signature = "a".repeat(86); + writeFileSync( + join( + f.root, + "control-private/ops-agent/observations", + `proxy-validation-${f.bound.execution.operationKey}.json`, + ), + JSON.stringify(raw), + { mode: 0o600 }, + ); + expect(() => + control.repairControlJournalTerminal( + f.agent, + f.bound.execution, + status, + terminal.payload.terminalAt, + ), + ).toThrow(); + } else { + expect( + control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.bound.execution), + ).toBeNull(); + expect(control.inspectFeedProxyValidationFailure(authority)).toBeNull(); + expect(() => + control.commitFeedProxyValidationFailure(authority, { + candidateDigest: "a".repeat(64), + validationExitCode: 1, + completionDigest: "b".repeat(64), + }), + ).toThrow(); + } + expect(f.calls()).toBe(fault === "reload" ? 2 : 1); + } finally { + await f.close(); + } + }, + 30000, +); +it("accepts the production hardlinked proxy candidate through validation and reload", async () => { + const f = await fixture(); + try { + control.consumeControlChildDispatch(f.issued.dispatchTicket); + const accepted = await f.client.execute(f.operation, { + operationKey: f.slot.execution.operationKey, + execution: { requestedDeadlineAt: f.bound.execution.deadlineAt }, + controlAssertion: f.issued.assertion, + }); + if (accepted.execution !== "async") throw Error("Expected actual agent admission"); + await expect + .poll(async () => (await f.client.status(f.operation.kind, accepted.operationId)).state, { + timeout: 15000, + }) + .toBe("succeeded"); + expect(f.calls()).toBe(2); + } finally { + await f.close(); + } +}, 30000); +it.each([ + "missing", + "forged", + "foreign-root", + "foreign-phase", + "foreign-body", + "foreign-target", + "foreign-origin", + "inode", + "bytes", + "directory", + "consumer-inode", + "consumer-symlink", + "canceled", +] as const)( + "rejects %s provenance through the actual production registry before Docker effects", + async (mode) => { + const f = await fixture(); + try { + let key = f.slot.execution.operationKey, + operation = f.operation; + if (mode === "missing") rmSync(f.published); + if (mode === "forged" || mode === "foreign-target" || mode === "foreign-origin") { + const raw = JSON.parse(readFileSync(f.published, "utf8")); + if (mode === "forged") raw.signature = "a".repeat(86); + else if (mode === "foreign-target") raw.payload.serviceIds = ["motis"]; + else raw.payload.origin.notAfter++; + writeFileSync(f.published, JSON.stringify(raw), { mode: 0o600 }); + } + if (mode === "foreign-root" || mode === "foreign-phase") { + key = control.controlChildOperationKey( + mode === "foreign-root" + ? createControlRootOperationKey({ + notAfter: f.parent.deadlineAt, + nonce: randomBytes(16), + }) + : f.parent.operationKey, + mode === "foreign-phase" ? "proxy-commit" : "proxy-stage", + 0, + ); + copyFileSync( + f.published, + join( + f.data, + ".admin-target-control/executions/data-manager", + `${key}.data-candidate.json`, + ), + ); + } + if (mode === "foreign-body") + operation = { ...operation, candidateId: "ctrl-" + "b".repeat(32) }; + if (mode === "inode") { + const path = join(f.proxy, "conf/default.conf"), + bytes = readFileSync(path); + renameSync(path, `${path}.original`); + writeFileSync(path, bytes); + } + if (mode === "bytes") + writeFileSync(join(f.proxy, "feed-proxy-vars.json"), '{"foreign":true}\n'); + if (mode === "directory") { + renameSync(f.proxy, `${f.proxy}.original`); + mkdirSync(join(f.proxy, "conf"), { recursive: true }); + writeFileSync(join(f.proxy, "conf/default.conf"), "server { listen 80; }\n"); + writeFileSync(join(f.proxy, "feed-proxy-vars.json"), "{}\n"); + } + if (mode === "consumer-inode" || mode === "consumer-symlink") { + const path = join(f.proxy, "motis-feed-proxy-config/default.conf"), + bytes = readFileSync(path); + rmSync(path); + if (mode === "consumer-inode") writeFileSync(path, bytes); + else (await import("node:fs")).symlinkSync(join(f.proxy, "conf/default.conf"), path); + } + if (mode === "canceled") { + control.cancelControlExecution(f.authority); + await control.cancelArtifactTargetLease(f.lease); + } + if (mode !== "canceled") + expect( + await f.claimer.claim( + operation, + opsOperationFingerprint(operation), + new AbortController().signal, + { role: "data-manager", operationKey: key }, + ), + ).toBeNull(); + if (mode === "canceled") + expect(() => control.consumeControlChildDispatch(f.issued.dispatchTicket)).toThrow( + /fenced|cancel/i, + ); + else control.consumeControlChildDispatch(f.issued.dispatchTicket); + await expect( + f.client.execute(operation, { + operationKey: key, + execution: { requestedDeadlineAt: f.bound.execution.deadlineAt }, + controlAssertion: f.issued.assertion, + }), + ).rejects.toThrow(); + expect(f.calls()).toBe(0); + } finally { + await f.close(); + } + }, + 30000, +); +it("rechecks the retained snapshot after nginx validation and before reload", async () => { + const f = await fixture(true); + try { + control.consumeControlChildDispatch(f.issued.dispatchTicket); + const accepted = await f.client.execute(f.operation, { + operationKey: f.slot.execution.operationKey, + execution: { requestedDeadlineAt: f.bound.execution.deadlineAt }, + controlAssertion: f.issued.assertion, + }); + if (accepted.execution !== "async") throw Error("Expected actual asynchronous agent admission"); + await expect + .poll(async () => (await f.client.status(f.operation.kind, accepted.operationId)).state) + .toBe("failed"); + expect(f.calls()).toBe(1); + } finally { + await f.close(); + } +}, 30000); + +it.each(["stricter", "late"] as const)( + "retains the bound identity when the actual recipient is %s", + async (policy) => { + const f = await fixture(false, policy); + try { + const original = f.bound.execution; + control.consumeControlChildDispatch(f.issued.dispatchTicket); + await expect( + f.client.execute(f.operation, { + operationKey: original.operationKey, + execution: { requestedDeadlineAt: original.deadlineAt }, + controlAssertion: f.issued.assertion, + }), + ).rejects.toThrow(); + expect(f.calls()).toBe(0); + expect(control.lookupControlChildPhase(f.authority, f.slot)?.execution).toEqual(original); + } finally { + await f.close(); + } + }, + 30000, +); diff --git a/apps/ops-agent/src/data-transit-resource-source.ts b/apps/ops-agent/src/data-transit-resource-source.ts new file mode 100644 index 000000000..d71e16a46 --- /dev/null +++ b/apps/ops-agent/src/data-transit-resource-source.ts @@ -0,0 +1,39 @@ +import { + type ControlInstallation, + inspectDataTransitCandidate, +} from "@openmapx/core/build-artifacts"; +import { opsOperationFingerprint } from "@openmapx/core/ops"; +import type { TrustedOpsDataSource } from "./policy"; +/** The existing registry claimer still performs its kind/service policy. This + * finite source supplies only DM's authenticated, confined transit candidate. */ +export function createDataTransitTrustedOpsDataSource( + installation: ControlInstallation | undefined, + configuration: TrustedOpsDataSource, +): TrustedOpsDataSource { + return { + async claim(operation, fingerprint, signal, owner) { + if ( + operation.kind !== "motis.primary.promote" && + operation.kind !== "feedProxy.validateAndReload" + ) + return configuration.claim(operation, fingerprint, signal, owner); + if ( + !installation || + signal.aborted || + owner?.role !== "data-manager" || + fingerprint !== opsOperationFingerprint(operation) + ) + return null; + try { + const checked = inspectDataTransitCandidate(installation, owner.operationKey, operation); + signal.throwIfAborted(); + return { + revisionId: `data-candidate-${checked.digest}`, + values: { candidateDigest: checked.digest }, + }; + } catch { + return null; + } + }, + }; +} diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index bfb613282..aea3a8cf5 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -3,6 +3,11 @@ import { existsSync } from "node:fs"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { StringDecoder } from "node:string_decoder"; +import { + assertArtifactTargetLeaseForward, + assertControlExecutionForward, + assertDataTransitCandidateForExecution, +} from "@openmapx/core/build-artifacts"; import { OPS_MAX_EVENT_MESSAGE_BYTES, OPS_MAX_FOLLOW_LOG_EVENTS, @@ -14,6 +19,10 @@ import { monitorPosixProcessGroup, services, } from "@openmapx/core/server"; +import { + createFeedProxyValidationInvocation, + parseFeedProxyValidationCompletion, +} from "./feed-proxy-validation"; import { createBoundedProcessOutput, safeProcessDiagnostic } from "./process-output"; import { createUnavailableRuntime, @@ -589,10 +598,62 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { ); return { availableBytes: parsePosixDfAvailableBytes(result.stdout) }; }; + function assertTransitCandidateEffect( + operation: Parameters[1], + context: OpsExecutionContext, + ) { + context.signal.throwIfAborted(); + if ( + !context.controlAuthority || + !context.artifactLease || + typeof context.claim.capability.values.candidateDigest !== "string" + ) + throw Error("Original owned transit resource claim required"); + assertArtifactTargetLeaseForward(context.artifactLease); + assertDataTransitCandidateForExecution( + context.controlAuthority, + operation, + context.claim.capability.values.candidateDigest, + ); + } runtime["feedProxy.validateAndReload"] = async (operation, context) => { + assertTransitCandidateEffect(operation, context); // Validate before reloading: a reload against a bad configuration would // take the proxy down rather than reject the candidate. - await changed(["exec", FEED_PROXY_CONTAINER, "nginx", "-t"], context, 30_000); + if (!context.commitFeedProxyValidationFailure || !context.markFeedProxyReloadAttempted) + throw Error("Fixed validation producer is unavailable"); + const candidateDigest = context.claim.capability.values.candidateDigest; + if (typeof candidateDigest !== "string") throw Error("Original candidate digest missing"); + const invocation = createFeedProxyValidationInvocation(), + remaining = context.execution.deadlineAt - Date.now(); + if (remaining <= 0) throw new OpsExecutionFailure("timeout", {}); + // Unlike streaming mutation output, validation requires the complete bounded + // nonce marker from the strict normal-zero contained-process adapter. + const completed = await run( + ["exec", FEED_PROXY_CONTAINER, ...invocation.args], + context.signal, + Math.min(30_000, remaining), + ); + context.signal.throwIfAborted(); + const validation = parseFeedProxyValidationCompletion( + completed.stdout, + invocation.nonce, + context.execution, + candidateDigest, + ); + if (validation.output) context.emitLog("stdout", validation.output); + if (completed.stderr) context.emitLog("stderr", completed.stderr); + if (validation.validationExitCode !== 0) { + await context.commitFeedProxyValidationFailure({ + candidateDigest, + validationExitCode: validation.validationExitCode, + completionDigest: validation.completionDigest, + }); + throw new OpsExecutionFailure("runtime", {}); + } + assertTransitCandidateEffect(operation, context); + await context.markFeedProxyReloadAttempted(); + assertTransitCandidateEffect(operation, context); await changed(["exec", FEED_PROXY_CONTAINER, "nginx", "-s", "reload"], context, 30_000); return { candidateId: operation.candidateId, reloaded: true as const }; }; @@ -661,7 +722,13 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { return { state: tileMtime > tarMtime ? ("not_ready" as const) : ("ready" as const) }; }; + function assertTrafficForward(context: OpsExecutionContext): void { + context.signal.throwIfAborted(); + if (context.controlAuthority) assertControlExecutionForward(context.controlAuthority); + if (context.ownedTargetLease) assertArtifactTargetLeaseForward(context.ownedTargetLease); + } runtime["valhalla.traffic.maintain"] = async (operation, context) => { + assertTrafficForward(context); const dataRoot = options.trafficDataRoot; if (!dataRoot) throw new Error("Valhalla traffic maintenance data root is not configured"); const sharedDir = join(dataRoot, "valhalla", "osm-pbf"); @@ -685,7 +752,9 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { ); if (stopped.stdout.trim() !== "false") throw new Error("Valhalla serving could not be fenced"); + assertTrafficForward(context); await mkdir(sharedDir, { recursive: true }); + assertTrafficForward(context); await writeFile( fencePath, `${JSON.stringify({ @@ -773,9 +842,12 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { completedAt: new Date().toISOString(), }; const temporaryManifest = `${generationsPath}.${operationId}.tmp`; + assertTrafficForward(context); await writeFile(temporaryManifest, `${JSON.stringify(manifest)}\n`, "utf8"); + assertTrafficForward(context); await rename(temporaryManifest, generationsPath); + assertTrafficForward(context); await run(["start", VALHALLA_CONTAINER], context.signal, 30_000); const healthDeadline = Date.now() + 60_000; while (true) { @@ -789,6 +861,7 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { await new Promise((resolve) => setTimeout(resolve, 500)); } + assertTrafficForward(context); await rm(fencePath, { force: true }); return { changed: true, @@ -835,6 +908,7 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { }; runtime["valhalla.traffic.refreshWaysToEdges"] = async (_operation, context) => { + assertTrafficForward(context); // Writes `/way_edges.txt`, a hard-coded name with no output flag. // data-manager reads and filters it from the shared mount afterwards. // A whole-country graph takes minutes, far past the default command bound. @@ -845,6 +919,7 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { ); if (build.exitCode !== 0) throw new Error("valhalla_ways_to_edges failed"); const owner = options.dataMountOwner ?? defaultDataMountOwner; + assertTrafficForward(context); await valhallaExec( ["chown", `${owner.uid}:${owner.gid}`, `${VALHALLA_TILE_DIR}/way_edges.txt`], context.signal, @@ -858,6 +933,7 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { runtime["motis.primary.restart"] = (_operation, context) => changed(["restart", "motis"], context); runtime["motis.primary.promote"] = async (operation, context) => { + assertTransitCandidateEffect(operation, context); // Recreate rather than restart: Docker resolves a bind-mount symlink when // the container is created, so a plain restart can keep pointing at the old // A/B target. `--no-deps` stops Compose from cascading into the feed proxy diff --git a/apps/ops-agent/src/feed-proxy-validation.test.ts b/apps/ops-agent/src/feed-proxy-validation.test.ts new file mode 100644 index 000000000..a0abfa849 --- /dev/null +++ b/apps/ops-agent/src/feed-proxy-validation.test.ts @@ -0,0 +1,107 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { runContainedProcess } from "./docker-runtime"; +import { + createFeedProxyValidationInvocation, + parseFeedProxyValidationCompletion, +} from "./feed-proxy-validation"; + +const execution = { + executor: "ops-agent" as const, + principal: "data-manager" as const, + executionId: "job1_aaaaaaaaaaaaaaaa", + operationKey: "opk1_aaaaaaaaaaaaaaaa", + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 60000, +}; +it.each([0, 1, 255])( + "observes nginx exit %s only after the real fixed wrapper exits normally with its nonce", + async (code) => { + const root = mkdtempSync(join(tmpdir(), "omx-nginx-wrapper-")); + mkdirSync(join(root, "bin")); + writeFileSync( + join(root, "bin/nginx"), + `#!/bin/sh\nprintf 'validation diagnostics\\n' >&2\nexit ${code}\n`, + { mode: 0o700 }, + ); + try { + const invocation = createFeedProxyValidationInvocation(); + const result = await runContainedProcess( + "/bin/sh", + [ + "-c", + 'PATH="$1:$PATH"; export PATH; shift; exec "$@"', + "docker-io-substitute", + join(root, "bin"), + ...invocation.args, + ], + { signal: new AbortController().signal, timeout: 5000, maxBuffer: 4096 }, + ); + const parsed = parseFeedProxyValidationCompletion( + result.stdout, + invocation.nonce, + execution, + "c".repeat(64), + ); + expect(parsed.validationExitCode).toBe(code); + expect(parsed.output).not.toContain("OPENMAPX_NGINX_TEST"); + expect(parsed.completionDigest).toMatch(/^[a-f0-9]{64}$/); + expect(result.stderr).toContain("validation diagnostics"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, +); +it.each([ + "missing", + "duplicate", + "foreign", + "negative", + "overflow", + "malformed", + "truncated", + "trailing", +] as const)("refuses a %s completion marker", (mode) => { + const invocation = createFeedProxyValidationInvocation(), + marker = `OPENMAPX_NGINX_TEST_V1:${invocation.nonce}:1`; + const outputs = { + missing: "nginx failed\n", + duplicate: marker + "\n" + marker + "\n", + foreign: marker.replace(invocation.nonce, "a".repeat(22)) + "\n", + negative: marker.replace(":1", ":-1") + "\n", + overflow: marker.replace(":1", ":256") + "\n", + malformed: marker.replace(":1", ":01") + "\n", + truncated: marker, + trailing: marker + "\nuntrusted\n", + }; + expect(() => + parseFeedProxyValidationCompletion(outputs[mode], invocation.nonce, execution, "c".repeat(64)), + ).toThrow(); +}); +it.each(["abort", "timeout", "truncation", "nonzero", "residual"] as const)( + "the actual contained runner refuses %s instead of producing validation evidence", + async (mode) => { + const controller = new AbortController(); + if (mode === "abort") controller.abort(); + const script = + mode === "nonzero" + ? "process.exit(1)" + : mode === "truncation" + ? "process.stdout.write('x'.repeat(10000))" + : mode === "residual" + ? "require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'});process.exit(0)" + : "setInterval(()=>{},1000)"; + await expect( + runContainedProcess(process.execPath, ["-e", script], { + signal: controller.signal, + timeout: mode === "timeout" ? 30 : 3000, + maxBuffer: 128, + killGraceMs: 20, + }), + ).rejects.toThrow(); + }, + 10000, +); diff --git a/apps/ops-agent/src/feed-proxy-validation.ts b/apps/ops-agent/src/feed-proxy-validation.ts new file mode 100644 index 000000000..623c1581a --- /dev/null +++ b/apps/ops-agent/src/feed-proxy-validation.ts @@ -0,0 +1,46 @@ +import { randomBytes } from "node:crypto"; +import { feedProxyValidationCompletionDigest } from "@openmapx/core/build-artifacts"; +import type { ExecutionRef } from "@openmapx/core/ops"; + +export const FEED_PROXY_VALIDATION_SCRIPT = + 'nginx -t\nvalidation_status=$?\nprintf "\\nOPENMAPX_NGINX_TEST_V1:%s:%s\\n" "$1" "$validation_status"\nexit 0'; +const PREFIX = "OPENMAPX_NGINX_TEST_V1:"; +export function createFeedProxyValidationInvocation() { + const nonce = randomBytes(16).toString("base64url"); + return { + nonce, + args: ["/bin/sh", "-c", FEED_PROXY_VALIDATION_SCRIPT, "openmapx-nginx-test", nonce], + }; +} +/** Called only after the strict contained runner's normal zero completion. */ +export function parseFeedProxyValidationCompletion( + stdout: string, + nonce: string, + execution: ExecutionRef, + candidateDigest: string, +) { + if (!/^[A-Za-z0-9_-]{22}$/.test(nonce)) throw Error("Invalid validation invocation"); + const lines = stdout.split("\n"); + if (lines.at(-1) !== "") throw Error("Incomplete validation completion marker"); + lines.pop(); + const marker = lines.pop(); + const match = marker?.match(/^OPENMAPX_NGINX_TEST_V1:([A-Za-z0-9_-]{22}):(0|[1-9][0-9]{0,2})$/); + if ( + !match || + match[1] !== nonce || + Number(match[2]) > 255 || + lines.some((line) => line.includes(PREFIX)) + ) + throw Error("Missing, foreign or repeated validation completion marker"); + const validationExitCode = Number(match[2]); + return { + validationExitCode, + completionDigest: feedProxyValidationCompletionDigest( + execution, + candidateDigest, + nonce, + validationExitCode, + ), + output: lines.join("\n"), + }; +} diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 0c8e26dd2..171c16626 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -21,6 +21,7 @@ import { installCommunityRepositoryRuntime, } from "./community-repositories"; import { loadOpsAgentConfig } from "./config"; +import { createDataTransitTrustedOpsDataSource } from "./data-transit-resource-source"; import { createDockerDawarichCollector, createDockerDawarichRuntimeInspector, @@ -234,12 +235,15 @@ async function main(): Promise { ); return trusted ?? bootstrapEnabledServiceIds; }, - trustedData: createFileTrustedOpsDataSource({ - directory: config.trustedConfigDirectory, - token: apiToken, - allowedUids: [uid], - expectedGid: gid, - }), + trustedData: createDataTransitTrustedOpsDataSource( + controlStartup.installation, + createFileTrustedOpsDataSource({ + directory: config.trustedConfigDirectory, + token: apiToken, + allowedUids: [uid], + expectedGid: gid, + }), + ), resourceAuthority: { allowBackup: (kind, backupId, serviceIds) => inspectBackupAuthority(config.rootDir, kind, backupId, serviceIds), diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index 431a27309..b5588d3ee 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -1,4 +1,5 @@ import type { + ArtifactTargetLease, BorrowedArtifactTargetLease, VerifiedExecutionAuthority, } from "@openmapx/core/build-artifacts"; @@ -50,7 +51,14 @@ export class OpsAgentBusyError extends Error { export interface OpsExecutionContext { readonly execution: ExecutionRef; readonly artifactLease?: BorrowedArtifactTargetLease; + readonly ownedTargetLease?: ArtifactTargetLease; readonly controlAuthority?: VerifiedExecutionAuthority; + commitFeedProxyValidationFailure?(input: { + candidateDigest: string; + validationExitCode: number; + completionDigest: string; + }): Promise; + markFeedProxyReloadAttempted?(): Promise; signal: AbortSignal; emitLog(stream: "stdout" | "stderr", message: string): void; claim: OpsTrustedClaim; diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 35aef496d..7a2545a33 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -1,28 +1,39 @@ import { createHash, randomBytes as nodeRandomBytes, timingSafeEqual } from "node:crypto"; import { acknowledgeControlRetention, + acquireArtifactTargetLease, assertArtifactTargetLeaseForward, assertFreshControlRoot, - type BorrowedArtifactTargetLease, beginControlEffect, type ControlInstallation, cancelArtifactTargetLease, + cancelControlExecution, closeArtifactTargetLeaseHandle, + commitFeedProxyValidationFailure, completeControlJournalExecution, + controlInstallationDataRoot, controlRegisteredServiceIds, inspectControlAdmission, + inspectFeedProxyValidationFailure, + isFixedSingleDataCommand, + markFeedProxyReloadAttempted, + planFixedSingleDataCommand, controlHeaders as readControlHeaders, reconcileControlRetention, registerControlRetentionJournal, + releaseArtifactTargetLease, repairControlJournalTerminal, requireControlJournalRetention, + reserveFeedProxyValidation, restoreAdmittedControlExecution, retainAdmittedControlExecution, retainControlEffectReceipt, retainControlTerminalCommitment, retainControlTerminalStatus, retainedControlAdoptionAssertion, + retainFixedSingleDataPlan, settleArtifactTargetLeaseChild, + type TargetLeaseAuthority, type VerifiedExecutionAuthority, verifyControlEffectSettlement, verifyRetainedArtifactTargetSettlement, @@ -426,7 +437,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } }; - const controlLeases = new Map(); + const controlLeases = new Map(); const controlAuthorities = new Map(); app.addHook("onClose", async () => { for (const lease of controlLeases.values()) closeArtifactTargetLeaseHandle(lease); @@ -560,26 +571,40 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif > = {}, ): Promise => { await withAdmissionLock(async () => { + if (state === "termination_pending" && job.terminalAt !== undefined) return; const updatedAt = now().toISOString(); const lease = controlLeases.get(job.operationId); const authority = controlAuthorities.get(job.operationId); + if (authority && state === "termination_pending") cancelControlExecution(authority); if (lease && state === "termination_pending") await cancelArtifactTargetLease(lease); if (authority && ["succeeded", "failed", "timed_out"].includes(state)) { if (!options.controlInstallation) throw Error("Missing control settlement installation"); // The fixed runtime's validated result is own-effect evidence. Failure alone is not containment. const committed = state === "succeeded" && updates.result !== undefined; + let validationFailure: string | null = null; + if (state === "failed" && job.operation.kind === "feedProxy.validateAndReload") { + try { + validationFailure = inspectFeedProxyValidationFailure(authority); + } catch { + /* Missing/corrupt producer evidence stays unknown. */ + } + } retainControlEffectReceipt(authority, { - outcome: committed ? "committed" : "unknown", - publication: committed ? "committed" : "unknown", + outcome: committed ? "committed" : validationFailure ? "contained" : "unknown", + publication: committed ? "committed" : validationFailure ? "not_started" : "unknown", receipt: committed ? { kind: "ops-result", result: updates.result } - : { kind: "unresolved" }, + : validationFailure + ? { kind: "feed-proxy-validation-failure", commitmentDigest: validationFailure } + : { kind: "unresolved" }, }); - if (lease) + if (lease) { await settleArtifactTargetLeaseChild( lease, verifyControlEffectSettlement(options.controlInstallation, authority.execution), ); + if (lease.mode === "owner" && committed) await releaseArtifactTargetLease(lease); + } } const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; if (authority && ["succeeded", "failed", "timed_out"].includes(state)) { @@ -612,13 +637,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const forceRecoveryRequired = (job: JobRecord): void => { stickyRecoveryRequired.add(job.operationId); + const workerActive = activeControllers.has(job.operationId); job.result = undefined; - job.errorClass = "recovery_required"; - job.terminalAt = now().getTime(); + job.errorClass = workerActive ? undefined : "recovery_required"; + job.terminalAt = workerActive ? undefined : now().getTime(); job.updatedAt = now().toISOString(); - job.state = "failed"; + job.state = workerActive ? "termination_pending" : "failed"; job.cursor += 1; - pushEvent(job, { cursor: job.cursor, type: "state", state: "failed" }); + pushEvent(job, { cursor: job.cursor, type: "state", state: job.state }); }; const identityFor = (job: JobRecord) => @@ -636,6 +662,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif // Compatibility records never resume a forward execution or gain a new budget. deadlineAt: job.deadlineAt ?? Date.parse(job.submittedAt), }); + function borrowedLeaseFor(id: string) { + const lease = controlLeases.get(id); + return lease?.mode === "borrowed" ? lease : undefined; + } + function ownedLeaseFor(id: string) { + const lease = controlLeases.get(id); + return lease?.mode === "owner" ? lease : undefined; + } const contextFor = ( job: JobRecord, signal: AbortSignal, @@ -644,8 +678,27 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif signal, claim, execution: executionFor(job), - artifactLease: controlLeases.get(job.operationId), + artifactLease: borrowedLeaseFor(job.operationId), + ownedTargetLease: ownedLeaseFor(job.operationId), controlAuthority: controlAuthorities.get(job.operationId), + ...(job.operation.kind === "feedProxy.validateAndReload" + ? { + async commitFeedProxyValidationFailure(input: { + candidateDigest: string; + validationExitCode: number; + completionDigest: string; + }) { + const authority = controlAuthorities.get(job.operationId); + if (!authority) throw Error("Original validation authority missing"); + commitFeedProxyValidationFailure(authority, input); + }, + async markFeedProxyReloadAttempted() { + const authority = controlAuthorities.get(job.operationId); + if (!authority) throw Error("Original validation authority missing"); + markFeedProxyReloadAttempted(authority); + }, + } + : {}), otherActiveOperations: () => [...jobsById.values()].filter( (other) => @@ -825,6 +878,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (authority) { const lease = controlLeases.get(job.operationId); if (lease) assertArtifactTargetLeaseForward(lease); + if (job.operation.kind === "feedProxy.validateAndReload") { + const digest = claim.capability.values.candidateDigest; + if (typeof digest !== "string") throw Error("Original candidate claim missing"); + reserveFeedProxyValidation(authority, job.operation.candidateId, digest); + } beginControlEffect(authority); } return dispatch(job.operation, runtime, contextFor(job, controller.signal, claim)); @@ -867,6 +925,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif startedAt, ); } else if (stickyRecoveryRequired.has(job.operationId)) { + await transition(job, "failed", { + result: undefined, + errorClass: "recovery_required", + terminalAt: now().getTime(), + }); + terminalDurable = true; audit( job.role, job.operation.kind, @@ -875,6 +939,30 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif startedAt, "recovery_required", ); + } else if ( + (timedOut || job.state === "termination_pending") && + job.role === "data-manager" && + [ + "motis.primary.stop", + "motis.staging.stop", + "motis.staging.restart", + "motis.primary.restart", + "motis.primary.promote", + "feedProxy.validateAndReload", + "transitousLock.approve", + ].includes(job.operation.kind) + ) { + // These fixed leaf handlers return only after their host call + // settles. Cancellation cannot erase a validated commit receipt + // from an already-started call; no new forward phase is allowed. + const result = parseBoundedOpsResult(job.operation.kind, rawResult); + await transition(job, "succeeded", { + result, + errorClass: undefined, + terminalAt: now().getTime(), + }); + terminalDurable = true; + audit(job.role, job.operation.kind, job.resourceId, "late_completion", startedAt); } else if (timedOut) { await transition(job, "timed_out", { errorClass: "timeout", @@ -981,6 +1069,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif clearTimeout(timer); try { if (stickyRecoveryRequired.has(job.operationId)) { + await transition(job, "failed", { + result: undefined, + errorClass: "recovery_required", + terminalAt: now().getTime(), + }); + terminalDurable = true; audit( job.role, job.operation.kind, @@ -1244,6 +1338,10 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif let controlAdmission: ReturnType | undefined; const parentHeader = request.headers["x-openmapx-parent-execution"]; const delegationHeader = request.headers["x-openmapx-control-delegation"]; + // Approval is exclusively a borrowed activation leaf. Installing its + // future validation runtime must never enable a bearer-only root path. + if (operation.kind === "transitousLock.approve" && delegationHeader === undefined) + throw new RequestGuardError("authorization", 403); try { readControlHeaders(request.raw.rawHeaders); } catch { @@ -1515,6 +1613,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif "serviceId" in operation ? [operation.serviceId] : controlRegisteredServiceIds(options.controlInstallation); + const fixedDataPlan = isFixedSingleDataCommand(job.operation.kind) + ? planFixedSingleDataCommand( + executionFor(job), + job.operation, + targets, + process.env.OPENMAPX_REGION, + ) + : undefined; const authority = retainAdmittedControlExecution(options.controlInstallation, { execution: executionFor(job), operation: job.operation, @@ -1523,9 +1629,23 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif requestedDeadlineAt: job.requestedDeadlineAt, parent: job.parent, serviceIds: targets, + ...(fixedDataPlan ? { reservations: [fixedDataPlan.rootReservation] } : {}), }); + if (fixedDataPlan) retainFixedSingleDataPlan(authority, fixedDataPlan); requireControlJournalRetention(authority); controlAuthorities.set(job.operationId, authority); + if ( + ["valhalla.traffic.maintain", "valhalla.traffic.refreshWaysToEdges"].includes( + job.operation.kind, + ) + ) { + const lease = await acquireArtifactTargetLease({ + dataRoot: controlInstallationDataRoot(options.controlInstallation), + serviceIds: ["valhalla"], + owner: authority, + }); + controlLeases.set(job.operationId, lease); + } } catch (error) { forceRecoveryRequired(job); await journal.replace([...jobsById.values()].map(persisted)); @@ -1783,23 +1903,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif return reply.code(409).send(redactedOpsError(requestId, "conflict")); } try { - await withAdmissionLock(async () => { - if (job.terminalAt !== undefined || job.state === "termination_pending") return; - const updatedAt = now().toISOString(); - const candidate: JobRecord = { - ...job, - state: "termination_pending", - updatedAt, - terminationRequestedAt: job.terminationRequestedAt ?? updatedAt, - }; - await journal.replace( - [...jobsById.values()].map((entry) => persisted(entry === job ? candidate : entry)), - ); - job.state = "termination_pending"; - job.updatedAt = updatedAt; - job.terminationRequestedAt = candidate.terminationRequestedAt; - job.cursor += 1; - pushEvent(job, { cursor: job.cursor, type: "state", state: "termination_pending" }); + await transition(job, "termination_pending", { + terminationRequestedAt: job.terminationRequestedAt ?? now().toISOString(), }); } catch { forceRecoveryRequired(job); diff --git a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts new file mode 100644 index 000000000..67d258e60 --- /dev/null +++ b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts @@ -0,0 +1,169 @@ +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import { + assertControlExecutionForward, + closeControlInstallation, + openControlInstallation, + restoreAdmittedControlExecution, +} from "@openmapx/core/build-artifacts"; +import { createDataTransitTrustedOpsDataSource } from "../data-transit-resource-source"; +import { createDockerRuntime, runContainedProcess } from "../docker-runtime"; +import { openOpsJobJournal } from "../journal"; +import { createProductionRegistryResourceClaimer } from "../policy"; +import { buildOpsAgentServer } from "../server"; +import { createTransitousLockRuntime } from "../transitous-lock-runtime"; +import { createFileTrustedOpsDataSource } from "../trusted-config-source"; + +const lines = createInterface({ input: process.stdin }); +const messages = lines[Symbol.asyncIterator](); +const initial = await messages.next(); +if (initial.done) throw Error("Missing fixed fixture startup"); +const { root, token, validationFailure, crash } = JSON.parse(initial.value) as { + root: string; + token: string; + validationFailure?: boolean | "twice"; + crash?: "before-proof" | "before-terminal" | "after-terminal"; +}; +const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis", "motis-feed-proxy"], +}); +const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); +const replace = journal.replace.bind(journal); +journal.replace = async (records) => { + const terminal = records.some( + (record) => + record.operation.kind === "feedProxy.validateAndReload" && + record.state === "failed" && + record.errorClass === "runtime", + ); + if (terminal && crash === "before-terminal") process.exit(86); + await replace(records); + if (terminal && crash === "after-terminal") process.exit(86); +}; +let release = () => {}; +const held = new Promise((resolve) => { + release = resolve; +}); +let effects = 0; +let validations = 0; +const send = (value: unknown) => process.stdout.write(`${JSON.stringify(value)}\n`); +const runtime = createDockerRuntime({ + stack: { + infraDir: root, + composePath: join(root, "compose.yml"), + composeReleasePath: join(root, "release.yml"), + }, + fileExists: existsSync, + // Only external Docker I/O is substituted. Production handlers, admission, + // journal, HTTP auth, delegation and cancellation run in this process. + execFile: async (_file, args, options) => { + if (args[2] === "/bin/sh") { + validations++; + const invalid = validationFailure && (validations === 1 || validationFailure === "twice"); + if (invalid) send({ type: "validation-failed" }); + const bin = join(root, "fixture-bin"); + mkdirSync(bin, { recursive: true }); + writeFileSync(join(bin, "nginx"), `#!/bin/sh\nexit ${invalid ? 1 : 0}\n`, { mode: 0o700 }); + const completed = await runContainedProcess( + "/bin/sh", + [ + "-c", + 'PATH="$1:$PATH"; export PATH; shift; exec "$@"', + "fixed-docker-substitute", + bin, + ...args.slice(2), + ], + options, + ); + if (crash === "before-proof") process.exit(86); + return completed; + } + const isPromotion = args.includes("--force-recreate"), + isReload = args.includes("reload"); + if (isPromotion || isReload || (args[0] === "restart" && args[1] === "motis")) { + effects++; + send({ type: "effect", effects }); + const aborted = () => send({ type: "aborted" }); + options.signal.addEventListener("abort", aborted, { once: true }); + if (options.signal.aborted) aborted(); + try { + await held; + } finally { + options.signal.removeEventListener("abort", aborted); + } + } + return { stdout: "", stderr: "" }; + }, +}); +createTransitousLockRuntime(runtime, { rootDir: root }); +const app = buildOpsAgentServer({ + runtime, + journal, + controlInstallation: installation, + tokens: { api: "a".repeat(43), "data-manager": token }, + resourceClaimer: createProductionRegistryResourceClaimer({ + services: ["motis", "motis-feed-proxy"].map((serviceId) => ({ + serviceId, + enabled: true, + isBuiltIn: true, + })), + integrationIds: [], + trustedData: createDataTransitTrustedOpsDataSource( + installation, + createFileTrustedOpsDataSource({ + directory: join(root, "trusted-config"), + token: "a".repeat(43), + allowedUids: [process.getuid!()], + expectedGid: process.getgid!(), + }), + ), + }), +}); +await app.listen({ host: "127.0.0.1", port: 0 }); +const address = app.server.address(); +if (!address || typeof address === "string") throw Error("Missing actual agent address"); +send({ type: "ready", port: address.port }); +try { + for await (const line of { [Symbol.asyncIterator]: () => messages }) { + const command = JSON.parse(line) as { type: string }; + if (command.type === "release") release(); + if (command.type === "probe") { + const record = journal.records()[0]; + if (!record?.operationFingerprint || !record.deadlineAt) + throw Error("Missing original admission"); + const authority = restoreAdmittedControlExecution(installation, { + executor: "ops-agent", + principal: "data-manager", + executionId: record.operationId, + operationKey: record.operationKey, + operationFingerprint: record.operationFingerprint, + fingerprint: record.fingerprint, + deadlineAt: record.deadlineAt, + }); + let forward = true; + try { + assertControlExecutionForward(authority); + } catch { + forward = false; + } + send({ + type: "probe", + forward, + effects, + operationId: record.operationId, + state: record.state, + terminalAt: record.terminalAt ?? null, + }); + } + if (command.type === "close") break; + } +} finally { + release(); + await app.close(); + closeControlInstallation(installation); + lines.close(); + process.stdin.destroy(); +} diff --git a/apps/ops-agent/src/traffic-target-ownership.test.ts b/apps/ops-agent/src/traffic-target-ownership.test.ts new file mode 100644 index 000000000..c804664f1 --- /dev/null +++ b/apps/ops-agent/src/traffic-target-ownership.test.ts @@ -0,0 +1,154 @@ +import { randomBytes } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + createControlRootOperationKey, + OpsClient, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { createDockerRuntime } from "./docker-runtime"; +import { openOpsJobJournal } from "./journal"; +import { createProductionRegistryResourceClaimer, denyAllTrustedOpsData } from "./policy"; +import { buildOpsAgentServer } from "./server"; + +it.each(["succeeded", "canceled"] as const)( + "owns the actual traffic target across its Docker adapter and HTTP cancellation (%s)", + async (mode) => { + const root = mkdtempSync(join(tmpdir(), "omx-traffic-target-")), + data = join(root, "data"); + mkdirSync(data, { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["valhalla"], + }), + journal = await openOpsJobJournal(join(data, "ops-agent/jobs-v1.json")); + let ready = () => {}, + release = () => {}, + aborted = () => {}, + effects = 0; + const started = new Promise((resolve) => { + ready = resolve; + }), + held = new Promise((resolve) => { + release = resolve; + }), + canceled = new Promise((resolve) => { + aborted = resolve; + }); + const runtime = createDockerRuntime({ + fileExists: existsSync, + stack: { + infraDir: root, + composePath: join(root, "compose.yml"), + composeReleasePath: join(root, "release.yml"), + }, + execFile: async (_file, args, options) => { + if (args.includes("valhalla_ways_to_edges")) { + effects++; + ready(); + options.signal.addEventListener("abort", aborted, { once: true }); + if (options.signal.aborted) aborted(); + await held; + } + return { stdout: "", stderr: "" }; + }, + }), + token = "d".repeat(43), + tokenFile = join(root, "token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + const app = buildOpsAgentServer({ + runtime, + journal, + controlInstallation: installation, + tokens: { api: "a".repeat(43), "data-manager": token }, + resourceClaimer: createProductionRegistryResourceClaimer({ + services: [{ serviceId: "valhalla", enabled: true, isBuiltIn: true }], + integrationIds: [], + trustedData: denyAllTrustedOpsData, + }), + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("Actual agent endpoint missing"); + const baseUrl = `http://127.0.0.1:${address.port}`, + client = new OpsClient({ baseUrl, tokenFile }); + try { + const operation = { kind: "valhalla.traffic.refreshWaysToEdges" as const }, + deadline = Date.now() + 60000, + operationKey = createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + admitted = await client.execute(operation, { + operationKey, + execution: { requestedDeadlineAt: deadline }, + }); + if (admitted.execution !== "async") throw Error("Async traffic root required"); + await started; + expect(effects).toBe(1); + expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(true); + if (mode === "canceled") { + const status = await client.status(operation.kind, admitted.operationId), + lossy = new OpsClient({ + baseUrl, + tokenFile, + fetch: async (url, options) => { + const response = await fetch(url, options); + if (options?.method === "DELETE") { + await response.body?.cancel(); + throw Error("Synthetic lost cancel response"); + } + return response; + }, + }); + await expect( + lossy.cancel(admitted.operationId, { fingerprint: status.fingerprint }), + ).rejects.toThrow(); + await canceled; + expect((await client.status(operation.kind, admitted.operationId)).state).toBe( + "termination_pending", + ); + expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(true); + if (!status.fingerprint || !status.deadlineAt) + throw Error("Original dated traffic identity required"); + const authority = control.restoreAdmittedControlExecution(installation, { + executor: "ops-agent", + principal: "data-manager", + executionId: admitted.operationId, + operationKey, + operationFingerprint: opsOperationFingerprint(operation), + fingerprint: status.fingerprint, + deadlineAt: status.deadlineAt, + }); + expect(() => control.assertControlExecutionForward(authority)).toThrow(); + release(); + await expect + .poll(async () => (await client.status(operation.kind, admitted.operationId)).state, { + timeout: 10000, + }) + .toBe("failed"); + expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(true); + } else { + release(); + await expect + .poll(async () => (await client.status(operation.kind, admitted.operationId)).state, { + timeout: 10000, + }) + .toBe("succeeded"); + expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(false); + } + } finally { + release(); + app.server.closeAllConnections(); + await app.close(); + control.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }, + 30000, +); diff --git a/apps/ops-agent/src/transitous-lock-runtime.test.ts b/apps/ops-agent/src/transitous-lock-runtime.test.ts index b246a4a7e..493b4b713 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.test.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.test.ts @@ -8,7 +8,6 @@ import { createTransitousLockRuntime } from "./transitous-lock-runtime"; const roots: string[] = []; const REF = `main@${"a".repeat(40)}`; -const OTHER_REF = `main@${"b".repeat(40)}`; function root(): string { const directory = mkdtempSync(join(tmpdir(), "openmapx-lock-runtime-")); @@ -89,11 +88,21 @@ describe("agent-owned Transitous catalog lock", () => { { kind: "transitousLock.propose", ref: REF, submodules: {}, lockedBy: "auto-bump" }, context(), ); - for (const ref of [REF, OTHER_REF]) + for (const proposalId of [ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000002", + ]) await expect( dispatchOpsOperation( runtime, - { kind: "transitousLock.approve", ref, approvedBy: "admin" }, + { + kind: "transitousLock.approve", + proposalId, + validationId: "00000000-0000-4000-8000-000000000003", + expectedGenerationId: null, + diffDigest: "a".repeat(64), + acknowledgeLicenseChanges: false, + }, context(), ), ).rejects.toThrow(/paired-pin/); diff --git a/apps/transitous-runner/src/runner.test.ts b/apps/transitous-runner/src/runner.test.ts index 7668a8211..b8ebf34e2 100644 --- a/apps/transitous-runner/src/runner.test.ts +++ b/apps/transitous-runner/src/runner.test.ts @@ -223,7 +223,9 @@ describe("private Transitous runner", () => { const { catalogDir, stagingDir } = fixture(); const killed: number[] = []; const originalKill = process.kill; - (process as { kill: unknown }).kill = ((pid: number) => { + (process as { kill: unknown }).kill = ((pid: number, signal?: string | number) => { + if (signal === 0) + throw Object.assign(Error("Synthetic process group has exited"), { code: "ESRCH" }); killed.push(pid); }) as typeof process.kill; diff --git a/apps/transitous-runner/src/runner.ts b/apps/transitous-runner/src/runner.ts index b1f73dfc2..0af8fd19d 100644 --- a/apps/transitous-runner/src/runner.ts +++ b/apps/transitous-runner/src/runner.ts @@ -1,6 +1,8 @@ import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; import { readFileSync, realpathSync, statSync } from "node:fs"; import { isAbsolute, resolve, sep } from "node:path"; +import { setTimeout as processGroupDelay } from "node:timers/promises"; import { CAPABILITY_TTL_MS, OPERATOR_METADATA_DIR, @@ -119,7 +121,10 @@ export function createTransitousRunner(options: TransitousRunnerOptions) { spentNonces.set(verified.nonce, at); } - async function execute(run: TransitousRunnerScript): Promise { + async function execute( + run: TransitousRunnerScript, + signal: AbortSignal, + ): Promise { const argv = transitousRunnerArgv(run); if (run.script === "fetch") assertInsideCatalog(options.catalogDir, run.feedPath); if (run.script === "fetch-operator") { @@ -180,6 +185,9 @@ export function createTransitousRunner(options: TransitousRunnerOptions) { child.kill("SIGKILL"); } }; + const abort = () => terminate(); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); const timer = setTimeout(() => { truncated = true; terminate(); @@ -189,6 +197,7 @@ export function createTransitousRunner(options: TransitousRunnerOptions) { if (settled) return; settled = true; clearTimeout(timer); + signal.removeEventListener("abort", abort); // Child output is arbitrary bytes. Decode lossily, then bound the // resulting UTF-8 string too: a replacement character for a partial or // invalid sequence can occupy more bytes than the input it replaced. @@ -213,16 +222,91 @@ export function createTransitousRunner(options: TransitousRunnerOptions) { durationMs: Date.now() - startedAt, }); }; - child.on("close", (code) => settle(code ?? 1)); - child.on("error", () => settle(1)); + child.on("close", (code) => { + // A parent exit does not prove its detached descendants have exited. + // Poll only the original group; never report containment while it lives. + void (async () => { + for (;;) { + if (child.pid === undefined) break; + try { + process.kill(-child.pid, 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") break; + await processGroupDelay(100); + continue; + } + await processGroupDelay(100); + } + settle(signal.aborted ? 1 : (code ?? 1)); + })(); + }); + child.on("error", () => { + /* close follows spawn/stdio failure; await it */ + }); }); } + type OwnedRun = { + requestDigest: string; + controller: AbortController; + state: "running" | "cancel_pending" | "terminal"; + result?: TransitousRunnerResult; + finishedAt?: number; + }; + const retained = new Map(); + const address = (request: TransitousRunnerRequest) => + createHash("sha256").update(request.capability).digest("hex"); + const requestDigest = (request: TransitousRunnerRequest) => + createHash("sha256").update(JSON.stringify(request.run)).digest("hex"); + function find(request: TransitousRunnerRequest) { + const row = retained.get(address(request)); + if (!row || row.requestDigest !== requestDigest(request)) + throw new TransitousRunnerError("authorization", "Original runner identity is unknown"); + return row; + } + const snapshot = (row: OwnedRun) => ({ + version: 1 as const, + state: row.state, + ...(row.result ? { result: row.result } : {}), + }); return { - /** Validate, authorize, and run one request. */ + inspect(request: TransitousRunnerRequest) { + return snapshot(find(request)); + }, + acknowledge(request: TransitousRunnerRequest) { + const row = find(request); + if (row.state !== "terminal") throw Error("Runner remains active"); + retained.delete(address(request)); + return { acknowledged: true }; + }, + cancel(request: TransitousRunnerRequest) { + const row = find(request); + if (row.state !== "terminal") { + row.state = "cancel_pending"; + row.controller.abort(); + } + return snapshot(row); + }, async run(request: TransitousRunnerRequest): Promise { + for (const [key, row] of retained) + if (row.finishedAt !== undefined && now() - row.finishedAt > 24 * 3600000) + retained.delete(key); + if (retained.size >= 256) + throw new TransitousRunnerError("isolation", "Retained runner capacity is full"); consumeCapability(request.capability, request.run); - return await execute(request.run); + const row: OwnedRun = { + requestDigest: requestDigest(request), + controller: new AbortController(), + state: "running", + }; + retained.set(address(request), row); + // The retained identity precedes the actual spawn. Disconnecting the + // POST reader never owns this process group or its cancellation. + const result = await execute(request.run, row.controller.signal); + row.result = result; + row.state = "terminal"; + row.finishedAt = now(); + return result; }, }; } diff --git a/apps/transitous-runner/src/server.ts b/apps/transitous-runner/src/server.ts index 63f6a36dc..38ae586db 100644 --- a/apps/transitous-runner/src/server.ts +++ b/apps/transitous-runner/src/server.ts @@ -42,5 +42,19 @@ export function buildTransitousRunnerServer(options: TransitousRunnerOptions) { } }); + for (const action of ["inspect", "cancel", "acknowledge"] as const) + app.post(`/v1/run/${action}`, async (request, reply) => { + const parsed = transitousRunnerRequestSchema.safeParse(request.body); + if (!parsed.success) return reply.code(400).send({ ok: false, error: "validation" }); + try { + return action === "cancel" + ? runner.cancel(parsed.data) + : action === "acknowledge" + ? runner.acknowledge(parsed.data) + : runner.inspect(parsed.data); + } catch { + return reply.code(404).send({ ok: false, error: "unknown" }); + } + }); return app; } diff --git a/docs/superpowers/plans/2026-10-25-admin-late-child-binding.md b/docs/superpowers/plans/2026-10-25-admin-late-child-binding.md index 0786ae688..aa7999332 100644 --- a/docs/superpowers/plans/2026-10-25-admin-late-child-binding.md +++ b/docs/superpowers/plans/2026-10-25-admin-late-child-binding.md @@ -34,4 +34,4 @@ All accepted control/retention contracts remain binding except future child exac - [ ] Wire real DM child store CAS and typed adapters: confirmed reserved full-body CAS → immutable binding → confirmed bound CAS → scope-held grant/reference/outbox → confirmed attempted/dispatch-claimed CAS + fresh nominal one-use ticket → one POST. Model reserved/bound/attempted/observing/terminal/unknown as a strict JSON union with stage-specific real fields; update store immutable-transition checks, no dummy grant values. Recover original body and grant via fixed slot publication reference; no fresh issuance ticket or mutation replay. Add crashes after every body/binding/DB/outbox/shared/node/linkage step, two same-slot callers, ambiguous/stale CAS and exact state assertions. Add real PostgreSQL child races and actual separate-process DM→assembled-agent late-phase/admission/loss/cancel tests. Use existing owned synthetic DB verification and applied0042. - [ ] Run affected package types and host-authority policy once stable; self-review changed interfaces and consumers. Append exact RED/GREEN/time/exit/limits to Task02.3 report/evidence and include changes in its local normal-hook commit. Controller reviews whole02.3 afterward; no independent commit/PR/deploy requirement for this substep. -Self-review: all five risks have named tests; interfaces match the spec, all body values are root-fixed and IDs preallocated, old exact plans remain valid, and no generator/DB migration is required by this correction. Independent scoped R2 review is READY with I1–I4 addressed and0 open; execute these steps in02.3. Design readiness does not qualify implementation. Binding envelope is version2/openmapx-control-phase-binding-v2; reservation/origin metadata version1 and legacy verifier remain unchanged. Test wrong version/domain/changed origin and capacity/unbound allocation/reclamation. Explicitly test original Full Update→fixedCLI→sequentialDM reservation transport and no target hold by its waiting parent. +Self-review: all five risks have named tests; interfaces match the spec, all body values are root-fixed and IDs preallocated, old exact plans remain valid, and no generator/DB migration is required by this correction. Independent scoped R2 review is READY with I1–I4 addressed and0 open; execute these steps in02.3. Design readiness does not qualify implementation. Binding envelope is version2/openmapx-control-phase-binding-v2; reservation/origin metadata version1 and legacy verifier remain unchanged. Test wrong version/domain/changed origin and capacity/unbound allocation/reclamation. Test fixedCLI reservation transport and actual matching single data commands→durable DM, including protocol-level sequential phases and no target hold by a waiting deadline-only parent. The complete original Full Update command with build/link and actual production default-CLI acceptance is assigned to original02.4 after01.2; it must resolve the finite finalization graph before implementation and preserve all semantics. This is an execution allocation correction, not a new authority edge or waived final proof. diff --git a/docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md b/docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md index 987b4f116..d9c4429da 100644 --- a/docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md +++ b/docs/superpowers/specs/2026-10-25-admin-late-child-binding-design.md @@ -113,3 +113,7 @@ Task02.3 owns the correction and its actual transit/deadline-only consumers in t Tests must causally reproduce late120s delegation failure, then prove successful actual child admission after hours of simulated root work under the original root and target inode. Real separate processes/loopback validate exact identity, unchanged root bytes and one effect. Include same-slot concurrent binding, altered root-fixed body/phase/ordinal/target/recipient (including every paired approval field), delayed/stricter-policy rejection, bind-versus-cancel, crash between binding/DB/grant/admission checkpoints, lost response lookup-only, duplicate after expiry without renewal, missing/corrupt binding, borrowed depth/subset and deadline-only Full Update. Use finite fake-clock advances, not multi-hour waits. Prove retention/index/capacity and no extra generation/key authority. Self-review: this addresses late normalization only, preserves exact admission and all accepted authority/retention boundaries, and assigns every new interface to02.3. R1-I1 is addressed by removing late body fields and preallocating owned candidate IDs; I2 by body-first staged CAS/reference/ticket states; I3 by a named scope-held issuer and union cancellation outboxes; I4 by the version2 binding domain. Independent R2 review checked these contracts and found0 open design issues; code and runtime proofs remain required in Task02.3. Review report: .superpowers/sdd/2026-10-23-admin-instance-management/late-child-amendment-review-r2.md (local evidence, not production acceptance). + +## Execution allocation clarification + +Task02.3 proves reserved-plan transport, actual matching fixed CLI data commands through the durable DM receiver, and the real DM-owned host-child/cancellation/recovery chain. The complete real Full Update command (including build/link finalization) and its production CLI acceptance belong to original Task02.4 after Plan01 preparation.02.4 must first resolve the finite finalization phase contract under original ownership and the existing no-target-while-waiting rule; this amendment grants no such new edge. The full command remains a mandatory02.4/Q1 proof, not waived by02.3 protocol tests or an interim prerequisite refusal. Original update semantics cannot become downloads-only or permanently unavailable in the final PR. diff --git a/packages/cli/__tests__/cli-data-execution.test.ts b/packages/cli/__tests__/cli-data-execution.test.ts new file mode 100644 index 000000000..dbe7c58fe --- /dev/null +++ b/packages/cli/__tests__/cli-data-execution.test.ts @@ -0,0 +1,25 @@ +import { expect, it } from "vitest"; +import { executeDurableDataStep } from "../src/lib/data-execution"; + +it("refuses an unowned durable data step before any POST", async () => { + let effects = 0; + await expect( + executeDurableDataStep( + { kind: "download-fonts" }, + { + operationKey: "opk1_abcdefghijklmnop", + requestedDeadlineAt: Date.now() + 60000, + observerDeadlineAt: Date.now() + 60000, + transport: { + request: async () => { + effects++; + throw Error("Must not acquire"); + }, + }, + checkpointAttempt: async () => {}, + checkpoint: async () => {}, + }, + ), + ).rejects.toThrow("Retained data command owner required"); + expect(effects).toBe(0); +}); diff --git a/packages/cli/__tests__/fixtures/single-data-cli-process.ts b/packages/cli/__tests__/fixtures/single-data-cli-process.ts new file mode 100644 index 000000000..d0026d787 --- /dev/null +++ b/packages/cli/__tests__/fixtures/single-data-cli-process.ts @@ -0,0 +1,15 @@ +/** Actual existing data-command assembly; only unrelated command families are + * omitted to keep this separate-process acceptance fixture focused. */ +import { Command } from "commander"; +import { registerDataCommands } from "../../src/commands/data"; +import { + assertCliExecutionAvailable, + withCliExecutionDeadline, +} from "../../src/lib/execution-deadline"; + +const program = new Command(); +registerDataCommands(program); +await withCliExecutionDeadline(process.argv.slice(2), async () => { + program.hook("preAction", () => assertCliExecutionAvailable()); + await program.parseAsync(process.argv); +}); diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index bc5579a27..3431d5982 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -2,14 +2,14 @@ import { mkdirSync } from "node:fs"; import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts"; import * as dm from "@openmapx/core/data-manager-sdk"; import type { Command } from "commander"; -import kleur from "kleur"; import { BUILD_ARTIFACTS, buildArtifactSpec, exportBuildBundle, importBuildBundle, } from "../lib/build-artifacts"; -import { completeCliCommand } from "../lib/control-context"; +import { cliControlContext, completeCliCommand } from "../lib/control-context"; +import { runCliDataStep } from "../lib/data-execution"; import { applyDataCleanup, collectOfflineDataStatus, @@ -49,67 +49,6 @@ function formatBytes(n: number): string { return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`; } -function formatDuration(ms: number): string { - const s = Math.floor(ms / 1000); - if (s < 60) return `${s}s`; - const m = Math.floor(s / 60); - const rs = s % 60; - if (m < 60) return `${m}m ${rs}s`; - const h = Math.floor(m / 60); - return `${h}h ${m % 60}m`; -} - -interface ProgressRenderer { - update: (bytes: number, total: number | undefined, startedAtMs: number) => void; - done: () => void; -} - -/** - * Single-line progress bar that rewrites itself in-place via `\r`. Falls back - * to plain text (and never repeats a line) when stdout isn't a TTY — useful - * for CI logs and `tee`/pipe redirections. - */ -function progressRenderer(): ProgressRenderer { - const isTty = process.stdout.isTTY === true; - const barWidth = 24; - let lastLine = ""; - - const write = (line: string) => { - if (!isTty) { - if (line === lastLine) return; - process.stdout.write(`${line}\n`); - lastLine = line; - return; - } - process.stdout.write(`\r${" ".repeat(lastLine.length)}\r${line}`); - lastLine = line; - }; - - return { - update(bytes, total, startedAtMs) { - const elapsed = Date.now() - startedAtMs; - const rate = elapsed > 0 ? bytes / (elapsed / 1000) : 0; - const rateStr = `${formatBytes(rate)}/s`; - if (total && total > 0) { - const pct = Math.min(100, (bytes / total) * 100); - const filled = Math.round((pct / 100) * barWidth); - const bar = `${"█".repeat(filled)}${"░".repeat(barWidth - filled)}`; - const etaMs = rate > 0 ? ((total - bytes) / rate) * 1000 : 0; - const etaStr = rate > 0 ? ` · ETA ${formatDuration(etaMs)}` : ""; - write( - `${kleur.cyan(bar)} ${pct.toFixed(1).padStart(5)}% ${formatBytes(bytes)} / ${formatBytes(total)} ${rateStr}${etaStr}`, - ); - } else { - write(`${kleur.dim("downloading")} ${formatBytes(bytes)} ${rateStr}`); - } - }, - done() { - if (isTty && lastLine) process.stdout.write("\n"); - lastLine = ""; - }, - }; -} - function printDataManagerStatusTable(datasets: DatasetMetadata[]): void { if (datasets.length === 0) { log.info("(no datasets downloaded yet)"); @@ -204,17 +143,14 @@ async function runOsmDownload( ); } - const started = Date.now(); - const render = progressRenderer(); - const r = await dm.downloadOsmWithProgress( - { region: resolvedRegion.value }, - { - onProgress: ({ bytes, totalBytes }) => render.update(bytes, totalBytes, started), - transport, - }, + const status = await runCliDataStep( + { kind: "download-osm", region: resolvedRegion.value }, + transport, + ); + if (status.result?.kind !== "download-osm") throw Error("OSM execution result is unavailable"); + log.ok( + `Downloaded ${resolvedRegion.value} (${(status.result.sizeBytes / 1e9).toFixed(2)} GB); execution ${status.id}`, ); - render.done(); - log.ok(`Downloaded ${resolvedRegion.value} (${(r.sizeBytes / 1e9).toFixed(2)} GB) → ${r.path}`); return resolvedRegion.value; } @@ -238,12 +174,14 @@ async function runTransitSync( { transport }, ), ); - log.ok(`Transit sync started: jobId=${result.jobId}`); + log.info( + `Transit sync queued: ${result.jobId}; inspect this execution in Activity or GET /executions/${result.jobId}`, + ); return result.jobId; } async function runFontDownload(transport: ReturnType): Promise { - dm.unwrapDataManagerResponse(await dm.downloadFonts({ transport })); + await runCliDataStep({ kind: "download-fonts" }, transport); log.ok("Downloaded map glyphs"); } @@ -384,6 +322,8 @@ export function registerDataCommands(program: Command): void { .option("--fail-fast", "Stop build-all after the first build failure") .action( async (region: string | undefined, options: { countries?: string; failFast?: boolean }) => { + if (cliControlContext()) + throw Error("Managed Full Update requires the reviewed build/link workflow upgrade"); const transport = createCliTransport(); let resolvedRegion: string; try { @@ -435,17 +375,17 @@ export function registerDataCommands(program: Command): void { if (resolvedRegion.sourceEnv) { log.dim(`using region "${resolvedRegion.value}" from $${resolvedRegion.sourceEnv}`); } - const started = Date.now(); - const render = progressRenderer(); - const r = await dm.convertOverpassWithProgress( - { region: resolvedRegion.value }, - { - onProgress: ({ bytes, totalBytes }) => render.update(bytes, totalBytes, started), - transport, - }, + if (!resolvedRegion.value) + throw Error("Overpass conversion requires a configured region"); + const status = await runCliDataStep( + { kind: "convert-overpass", region: resolvedRegion.value }, + transport, + ); + if (status.result?.kind !== "convert-overpass") + throw Error("Conversion execution result is unavailable"); + log.ok( + `Converted ${(status.result.sizeBytes / 1e9).toFixed(2)} GB; execution ${status.id}`, ); - render.done(); - log.ok(`Converted → ${r.targetBz2} (${(r.sizeBytes / 1e9).toFixed(2)} GB)`); } else { log.err(`Unknown kind: ${kind} (use: overpass)`); completeCliCommand(1); @@ -516,14 +456,11 @@ export function registerDataCommands(program: Command): void { return; } try { - const result = await dm.buildSearchIndexWithProgress( - { region: resolvedRegion.value }, - { onProgress: ({ message }) => log.dim(message), transport: createCliTransport() }, - ); - log.ok( - `Search index published for ${resolvedRegion.value}: ${result.placeCount ?? 0} places, ${result.termCount ?? 0} terms` + - (result.epoch ? ` (epoch ${result.epoch})` : ""), + const status = await runCliDataStep( + { kind: "search-index-build", region: resolvedRegion.value }, + createCliTransport(), ); + log.ok(`Search index published for ${resolvedRegion.value}; execution ${status.id}`); } catch (err) { log.err(`search-index build failed: ${(err as Error).message}`); dataManagerHint(); @@ -576,14 +513,8 @@ export function registerDataCommands(program: Command): void { .description("Fetch the notable places from Wikidata and publish them for search") .action(async () => { try { - const result = await dm.buildNotablePlacesWithProgress({ - onProgress: ({ message }) => log.dim(message), - transport: createCliTransport(), - }); - log.ok( - `Notable places published: ${result.placeCount ?? 0} places, ${result.nameCount ?? 0} names` + - (result.epoch ? ` (epoch ${result.epoch})` : ""), - ); + const status = await runCliDataStep({ kind: "notable-places-build" }, createCliTransport()); + log.ok(`Notable places published; execution ${status.id}`); } catch (err) { log.err(`notable-places build failed: ${(err as Error).message}`); dataManagerHint(); @@ -813,22 +744,11 @@ export function registerDataCommands(program: Command): void { const resolvedRegion = region || process.env.OPENMAPX_REGION || "europe/germany/berlin"; try { log.dim(`Refreshing Overture places for "${resolvedRegion}"…`); - const result = await dm.syncOvertureWithProgress( - { region: resolvedRegion }, - { - onProgress: ({ message }) => log.dim(message), - transport, - }, + const status = await runCliDataStep( + { kind: "overture-sync", region: resolvedRegion }, + transport, ); - log.ok(`Overture ${result.release ?? "release"} active for "${resolvedRegion}"`); - if (result.conflation === "failed" || result.conflation === "waiting_for_osm") { - log.warn( - `OSM link rebuild is ${result.conflation.replaceAll("_", " ")} and will retry independently` + - (result.conflationError ? `: ${result.conflationError}` : ""), - ); - } else { - log.ok(`${result.linked ?? 0} OSM↔Overture links active`); - } + log.ok(`Overture active for ${resolvedRegion}; execution ${status.id}`); } catch (err) { log.err(`overture-sync failed: ${(err as Error).message}`); dataManagerHint(); @@ -891,39 +811,11 @@ export function registerDataCommands(program: Command): void { const resolvedRegion = region || process.env.OPENMAPX_REGION || "europe/germany/berlin"; try { log.dim(`Running Overture conflation for "${resolvedRegion}"…`); - const result = await dm.conflateOvertureWithProgress( - { region: resolvedRegion, restart: options.restart === true }, - { - onProgress: ({ message }) => log.dim(message), - transport, - }, + const status = await runCliDataStep( + { kind: "overture-conflate", region: resolvedRegion, restart: options.restart === true }, + transport, ); - if (!result.ok) { - log.err( - `overture-conflate failed: ${result.message ?? result.status ?? "unknown error"}`, - ); - completeCliCommand(1); - } - if (result.status === "already_running") { - log.warn( - result.message ?? - "Overture conflation is still running under another worker; no work was claimed", - ); - return; - } - const linked = result.linked ?? 0; - if (result.status === "already_completed") { - log.ok( - `Overture conflation was already complete: ${linked} link${linked === 1 ? "" : "s"} active`, - ); - } else { - log.ok( - `Overture conflation complete: ${linked} link${linked === 1 ? "" : "s"} written` + - ("extracted" in result && "candidates" in result - ? ` from ${result.extracted} OSM POIs and ${result.candidates} accepted edges` - : ""), - ); - } + log.ok(`Overture conflation complete for ${resolvedRegion}; execution ${status.id}`); } catch (err) { log.err(`overture-conflate failed: ${(err as Error).message}`); dataManagerHint(); diff --git a/packages/cli/src/lib/control-context.ts b/packages/cli/src/lib/control-context.ts index 84780b211..efb6cdaab 100644 --- a/packages/cli/src/lib/control-context.ts +++ b/packages/cli/src/lib/control-context.ts @@ -7,6 +7,7 @@ import { type BorrowedArtifactTargetLease, beginFixedCliExecution, bindBorrowedControlExecution, + type ControlInstallation, closeArtifactTargetLeaseHandle, closeControlInstallation, openFixedChildInstallation, @@ -25,6 +26,7 @@ import { repoPaths } from "./paths"; const contexts = new AsyncLocalStorage<{ authority: VerifiedExecutionAuthority; + installation: ControlInstallation; lease?: BorrowedArtifactTargetLease; }>(); class FixedCliCommandExit extends Error { @@ -95,6 +97,7 @@ export async function withCliControlContext( ...(parent ? { parent } : {}), serviceIds: intent.serviceIds, plan: intent.plan, + ...(intent.reservations ? { reservations: intent.reservations } : {}), }); beginFixedCliExecution(installation, execution); if (lease) bindBorrowedControlExecution(lease, authority); @@ -102,7 +105,7 @@ export async function withCliControlContext( try { let result: T; try { - result = await contexts.run({ authority, lease }, run); + result = await contexts.run({ authority, lease, installation }, run); } catch (e) { if (e instanceof FixedCliCommandExit && e.code === 0) result = undefined as T; else throw e; @@ -138,3 +141,12 @@ export async function withCliControlContext( closeControlInstallation(installation); } } + +export function withHostDataControlContext( + installation: ControlInstallation, + authority: VerifiedExecutionAuthority, + run: () => Promise, +): Promise { + if (contexts.getStore()) throw Error("Existing CLI control context"); + return contexts.run({ installation, authority }, run); +} diff --git a/packages/cli/src/lib/data-execution.ts b/packages/cli/src/lib/data-execution.ts new file mode 100644 index 000000000..b89de8984 --- /dev/null +++ b/packages/cli/src/lib/data-execution.ts @@ -0,0 +1,302 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { + bindControlChildPhase, + cliDataReservation, + closeControlInstallation, + consumeControlChildDispatch, + controlChildOperationKey, + controlRegisteredServiceIds, + controlRolePublicKey, + fixedDataBodyFingerprint, + issueControlChildPhase, + openControlInstallation, + readCliDataAttempt, + retainAdmittedControlExecution, + retainCliDataAttempt, + retainControlEffectReceipt, + verifyCliDataTerminal, +} from "@openmapx/core/build-artifacts"; +import { + type DataChildAttempt, + type DataExecutionInput, + type DataExecutionStatus, + dataExecutionStatusSchema, + normalizeDataExecutionInput, + ordinaryDataExecutionInputSchema, +} from "@openmapx/core/data-manager-contracts"; +import { + DataExecutionObservationPending, + type DataManagerTransport, + lookupDataExecution, + observeDataExecution, + startDataExecution, + unwrapDataManagerResponse, +} from "@openmapx/core/data-manager-sdk"; +import { + createControlRootOperationKey, + normalizeExecutionAdmission, + type OpsOperation, + opsExecutionBudget, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { cliControlContext, withHostDataControlContext } from "./control-context"; +import { repoPaths } from "./paths"; +export async function executeDurableDataStep( + input: DataExecutionInput, + options: { + operationKey: string; + transport: DataManagerTransport; + signal?: AbortSignal; + requestedDeadlineAt: number; + parentAssertion?: string; + observerDeadlineAt: number; + checkpointAttempt: (attempt: DataChildAttempt) => Promise; + checkpoint: (status: DataExecutionStatus) => Promise; + }, +): Promise { + const context = cliControlContext(); + if (!context) throw Error("Retained data command owner required"); + const normalized = ordinaryDataExecutionInputSchema.parse(normalizeDataExecutionInput(input)), + slot = cliDataReservation(context.authority), + bound = await bindControlChildPhase(context.authority, slot); + if ( + options.operationKey !== slot.execution.operationKey || + options.requestedDeadlineAt !== bound.execution.deadlineAt || + fixedDataBodyFingerprint(normalized) !== slot.operationFingerprint + ) + throw Error("Data step differs from retained command plan"); + const identity = { + id: bound.execution.executionId, + operationKey: bound.execution.operationKey, + fingerprint: bound.execution.fingerprint, + }; + const match = (value: unknown) => { + const status = dataExecutionStatusSchema.parse(value); + if ( + status.id !== identity.id || + status.operationKey !== identity.operationKey || + status.fingerprint !== identity.fingerprint || + status.deadlineAt !== bound.execution.deadlineAt || + status.kind !== normalized.kind + ) + throw new DataExecutionObservationPending(identity, "transport"); + return status; + }; + let attempt = readCliDataAttempt(context.authority, slot.execution.operationKey), + status: DataExecutionStatus | undefined; + if (!attempt) { + attempt = { + phase: "data-step", + operationKey: slot.execution.operationKey, + input: normalized, + operationFingerprint: slot.operationFingerprint, + requestedDeadlineAt: options.requestedDeadlineAt, + parentExecution: context.authority.execution, + state: "prepared", + }; + await options.checkpointAttempt(attempt); + if (!options.parentAssertion) throw Error("Original signed data parent assertion required"); + attempt = { ...attempt, state: "attempted" }; + await options.checkpointAttempt(attempt); + try { + status = match( + unwrapDataManagerResponse( + await startDataExecution( + { + operationKey: options.operationKey, + input: normalized, + requestedDeadlineAt: options.requestedDeadlineAt, + }, + { + transport: options.transport, + signal: options.signal, + headers: { "x-openmapx-parent-execution": options.parentAssertion }, + }, + ), + ), + ); + } catch { + /* An ambiguous POST is lookup-only, including 404/410. */ + } + } + if (!status) + try { + status = match( + unwrapDataManagerResponse( + await lookupDataExecution(options.operationKey, { + transport: options.transport, + signal: options.signal, + }), + ), + ); + } catch { + throw new DataExecutionObservationPending( + identity, + options.signal?.aborted ? "interrupted" : "transport", + ); + } + attempt = { + ...attempt, + state: "observing", + id: status.id, + fingerprint: status.fingerprint, + deadlineAt: status.deadlineAt, + }; + await options.checkpointAttempt(attempt); + await options.checkpoint(status); + let checkpoints: Promise = Promise.resolve(); + let checkpointError: unknown; + const terminal = await observeDataExecution(identity, { + transport: options.transport, + signal: options.signal, + observerDeadlineAt: options.observerDeadlineAt, + onProgress: (status) => { + checkpoints = checkpoints + .then(() => options.checkpoint(status)) + .catch((error) => { + checkpointError ??= error; + }); + }, + }); + await checkpoints; + if (checkpointError) throw checkpointError; + // The terminal GET precedes final physical target release by a small window. + // Wait for the actual DM-signed shared containment witness, never infer it. + for (;;) { + options.signal?.throwIfAborted(); + try { + verifyCliDataTerminal(context.authority, terminal); + break; + } catch { + if (Date.now() >= options.observerDeadlineAt) + throw new DataExecutionObservationPending(identity, "observer_deadline"); + await delay(100, undefined, { signal: options.signal }); + } + } + await options.checkpointAttempt({ ...attempt, state: "terminal" }); + return terminal; +} +function operationFor(input: DataExecutionInput): OpsOperation { + switch (input.kind) { + case "download-osm": + return { kind: "data.downloadOsm", regionId: input.region }; + case "download-fonts": + return { kind: "data.downloadFonts" }; + case "convert-overpass": + return { kind: "data.convertOverpass", regionId: input.region }; + case "overture-sync": + return { kind: "data.overtureSync", regionId: input.region }; + case "overture-conflate": + return { kind: "data.overtureConflate", regionId: input.region, restart: input.restart }; + case "search-index-build": + return { kind: "data.searchIndexBuild", regionId: input.region }; + case "notable-places-build": + return { kind: "data.notablePlacesBuild" }; + default: + throw Error("No existing single-command edge for this data input"); + } +} +async function runOwned(input: DataExecutionInput, transport: DataManagerTransport) { + const context = cliControlContext(); + if (!context) throw Error("Data command owner required"); + const slot = cliDataReservation(context.authority), + bound = await bindControlChildPhase(context.authority, slot); + const prior = readCliDataAttempt(context.authority, slot.execution.operationKey); + let assertion: string | undefined; + if (!prior) { + const issued = await issueControlChildPhase(context.authority, { + phase: "data-step", + ordinal: 0, + childPublicKey: controlRolePublicKey(context.installation, "data-manager"), + }); + assertion = issued.assertion; + consumeControlChildDispatch(issued.dispatchTicket); + } + const result = await executeDurableDataStep(input, { + operationKey: slot.execution.operationKey, + requestedDeadlineAt: bound.execution.deadlineAt, + parentAssertion: assertion, + transport, + observerDeadlineAt: context.authority.execution.deadlineAt, + checkpointAttempt: async (attempt) => retainCliDataAttempt(context.authority, attempt), + checkpoint: async () => {}, + }); + if (result.status !== "ok") throw Error(`Data execution ${result.id} finished ${result.status}`); + return result; +} +/** Host commands use their own dated, retained coordinator. Managed fixed CLI + * commands reuse their certified root-fixed slot and acquire no target root. */ +export async function runCliDataStep( + value: DataExecutionInput, + transport: DataManagerTransport, +): Promise { + const input = normalizeDataExecutionInput(value); + if (cliControlContext()) return runOwned(input, transport); + const operation = operationFor(input), + admittedAt = Date.now(), + requestedDeadlineAt = admittedAt + opsExecutionBudget(operation.kind).processMs, + operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }), + installation = openControlInstallation({ + infraDir: repoPaths().infraDir, + role: "host-cli", + serviceIds: ["data-manager"], + }); + const normalized = normalizeExecutionAdmission({ + kind: operation.kind, + operationKey, + operationFingerprint: opsOperationFingerprint(operation), + admittedAt, + requestedDeadlineAt, + }), + execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey, + operationFingerprint: opsOperationFingerprint(operation), + fingerprint: normalized.fingerprint, + deadlineAt: normalized.deadlineAt, + }; + try { + const authority = retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt, + serviceIds: controlRegisteredServiceIds(installation), + reservations: [ + { + version: 1, + phase: "data-step", + ordinal: 0, + kind: input.kind, + recipient: "data-manager", + serviceIds: ["data-manager"], + operationFingerprint: fixedDataBodyFingerprint(input), + execution: { + executor: "data-manager", + principal: "host-cli", + executionId: randomUUID(), + operationKey: controlChildOperationKey(operationKey, "data-step", 0), + }, + }, + ], + }); + const result = await withHostDataControlContext(installation, authority, () => + runOwned(input, transport), + ); + retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "data-command", status: result }, + }); + return result; + } finally { + closeControlInstallation(installation); + } +} diff --git a/packages/cli/src/lib/execution-deadline.ts b/packages/cli/src/lib/execution-deadline.ts index 2abdb9d1a..c745323ee 100644 --- a/packages/cli/src/lib/execution-deadline.ts +++ b/packages/cli/src/lib/execution-deadline.ts @@ -103,7 +103,8 @@ export function constrainCliDataManagerTransport( assertCliExecutionAvailable(); if ( deadlines.getStore() && - !["GET", "HEAD"].includes((options?.method ?? "GET").toUpperCase()) + !["GET", "HEAD"].includes((options?.method ?? "GET").toUpperCase()) && + !["startDataExecution", "cancelDataExecution"].includes(operationId) ) throw Error("Managed data-manager execution requires a deadline-aware receiver upgrade"); return transport.request(operationId, url, options); diff --git a/packages/core/src/build-artifacts/child-phase.test.ts b/packages/core/src/build-artifacts/child-phase.test.ts new file mode 100644 index 000000000..91a793e45 --- /dev/null +++ b/packages/core/src/build-artifacts/child-phase.test.ts @@ -0,0 +1,776 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { createControlRootOperationKey } from "../ops/control-root-key"; +import { + type ExecutionRef, + executionFingerprint, + normalizeExecutionAdmission, +} from "../ops/execution-identity"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { inspectControlChildPublication } from "./child-phase"; +import * as control from "./index"; + +afterEach(() => vi.restoreAllMocks()); +it("admits a120s fixed phase after hours of original-root work without a new root", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-late-phase-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis"], + }); + const agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const admittedAt = Date.now(), + deadlineAt = admittedAt + 24 * 3600000; + const operation = { kind: "transit-sync", countries: ["de"] }, + operationFingerprint = createHash("sha256").update(canonicalJson(operation)).digest("hex"); + const owner: ExecutionRef = { + executor: "data-manager", + principal: "data-manager", + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const body = { kind: "motis.primary.stop" as const }; + const child: ExecutionRef = { + executor: "ops-agent", + principal: "data-manager", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: control.controlChildOperationKey(owner.operationKey, "stop-primary", 0), + operationFingerprint: opsOperationFingerprint(body), + deadlineAt: admittedAt + 120000, + fingerprint: executionFingerprint({ + kind: body.kind, + operationFingerprint: opsOperationFingerprint(body), + deadlineAt: admittedAt + 120000, + parent: control.retainedControlParent(owner), + }), + }; + const reservation: control.ControlChildReservation = { + version: 1, + phase: "stop-primary", + ordinal: 0, + kind: body.kind, + recipient: "ops-agent", + serviceIds: ["motis"], + operationFingerprint: child.operationFingerprint, + execution: { + executor: child.executor, + principal: child.principal, + executionId: child.executionId, + operationKey: child.operationKey, + }, + }; + const authority = control.retainAdmittedControlExecution(dm, { + execution: owner, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis"], + reservations: [reservation], + }); + const lease = await control.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["motis"], + owner: authority, + }); + try { + const original = readFileSync( + join(root, "control-private/data-manager/executions", `${owner.operationKey}.json`), + ); + const held = statSync(join(root, "data/.admin-target-control/locks/motis")).ino; + const late = admittedAt + 4 * 3600000; + vi.spyOn(Date, "now").mockReturnValue(late); + const normalized = normalizeExecutionAdmission({ + kind: body.kind, + operationKey: child.operationKey, + operationFingerprint: child.operationFingerprint, + admittedAt: late, + requestedDeadlineAt: deadlineAt, + parent: control.verifiedRetainedControlParent(dm, owner), + }); + const bound = await control.bindControlChildPhase(authority, { + phase: "stop-primary", + ordinal: 0, + }); + expect(bound.execution).toEqual({ + ...child, + deadlineAt: normalized.deadlineAt, + fingerprint: normalized.fingerprint, + }); + const issued = await control.issueControlChildPhase(authority, { + phase: "stop-primary", + ordinal: 0, + childPublicKey: agent.publicKey, + lease, + }); + expect(issued.channel).toBe("delegation"); + control.consumeControlChildDispatch(issued.dispatchTicket); + expect(() => control.consumeControlChildDispatch(issued.dispatchTicket)).toThrow(/consumed/); + expect( + readFileSync( + join(root, "control-private/data-manager/executions", `${owner.operationKey}.json`), + ), + ).toEqual(original); + expect(statSync(join(root, "data/.admin-target-control/locks/motis")).ino).toBe(held); + expect(control.cancelControlExecution(authority)).toEqual([bound.execution]); + await expect( + control.issueControlChildPhase(authority, { + phase: "stop-primary", + ordinal: 0, + childPublicKey: agent.publicKey, + lease, + }), + ).rejects.toThrow(); + } finally { + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agent); + control.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + } +}); + +async function fixture(exact = false, services = ["motis", "motis-staging"]) { + const root = mkdtempSync(join(tmpdir(), "omx-phase-state-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: services, + }); + const admittedAt = Date.now(), + deadlineAt = admittedAt + 24 * 3600000, + operation = { kind: "transit-sync", countries: ["de"] }; + const operationFingerprint = createHash("sha256").update(canonicalJson(operation)).digest("hex"); + const execution: ExecutionRef = { + executor: "data-manager", + principal: "data-manager", + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const reservations: control.ControlChildReservation[] = ["stop-primary", "stop-staging"].map( + (_phase, index) => ({ + version: 1, + phase: index === 0 ? "stop-primary" : "stop-staging", + ordinal: index, + kind: index === 0 ? "motis.primary.stop" : "motis.staging.stop", + recipient: "ops-agent", + serviceIds: [services[index] ?? "motis"], + operationFingerprint: opsOperationFingerprint( + index === 0 ? { kind: "motis.primary.stop" } : { kind: "motis.staging.stop" }, + ), + execution: { + executor: "ops-agent", + principal: "data-manager", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: control.controlChildOperationKey( + execution.operationKey, + index === 0 ? "stop-primary" : "stop-staging", + index, + ), + }, + }), + ); + const plan = exact + ? reservations.map(({ version: _v, operationFingerprint: body, ...slot }) => ({ + ...slot, + execution: { + ...slot.execution, + operationFingerprint: body, + deadlineAt: admittedAt + 120000, + fingerprint: executionFingerprint({ + kind: slot.kind, + operationFingerprint: body, + deadlineAt: admittedAt + 120000, + parent: control.retainedControlParent(execution), + }), + }, + })) + : []; + const input = { + execution, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: services, + plan, + ...(exact ? {} : { reservations }), + }; + const authority = control.retainAdmittedControlExecution(dm, input); + const cell = join( + root, + "control-private/data-manager/observations", + `phase-${execution.operationKey}-0.json`, + ); + return { + root, + dm, + execution, + input, + authority, + cell, + close: () => { + control.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + }, + }; +} +it("returns the original binding after cancel/expiry and cannot bind another phase", async () => { + const f = await fixture(); + try { + const phase = { phase: "stop-primary" as const, ordinal: 0 }; + const first = await control.bindControlChildPhase(f.authority, phase); + control.cancelControlExecution(f.authority); + vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); + expect(await control.bindControlChildPhase(f.authority, phase)).toEqual(first); + await expect( + control.bindControlChildPhase(f.authority, { phase: "stop-staging", ordinal: 1 }), + ).rejects.toThrow(/fenced|clock|expir/i); + expect(control.lookupControlChildPhase(f.authority, phase)).toEqual(first); + } finally { + f.close(); + } +}); +it("observes one same-slot binding and rejects signed-cell rollback against the retained physical index", async () => { + const f = await fixture(); + try { + const unbound = readFileSync(f.cell), + phase = { phase: "stop-primary" as const, ordinal: 0 }; + const results = await Promise.all([ + control.bindControlChildPhase(f.authority, phase), + control.bindControlChildPhase(f.authority, phase), + ]); + expect(results[0]).toEqual(results[1]); + writeFileSync(f.cell, unbound); + expect(() => control.lookupControlChildPhase(f.authority, phase)).toThrow(/record.*changed/); + await expect(control.bindControlChildPhase(f.authority, phase)).rejects.toThrow( + /record.*changed/, + ); + } finally { + f.close(); + } +}); +it("keeps an old exact admission's signed bytes when a duplicate supplies empty reservations", async () => { + const f = await fixture(true); + try { + const path = join( + f.root, + "control-private/data-manager/executions", + `${f.execution.operationKey}.json`, + ), + original = readFileSync(path); + const repeated = control.retainAdmittedControlExecution(f.dm, { ...f.input, reservations: [] }); + expect(repeated.execution).toEqual(f.execution); + expect(readFileSync(path)).toEqual(original); + } finally { + f.close(); + } +}); + +import { AnchoredDirectory } from "./filesystem"; + +it.each([ + "reference", + "cancellation-index", + "lease-outbox", + "grant-outbox", + "shared-grant", + "parent-linkage", +] as const)( + "retains the exact phase and refuses another dispatch after a crash at %s", + async (point) => { + const f = await fixture(), + agent = control.openControlInstallation({ + infraDir: f.root, + role: "ops-agent", + serviceIds: ["motis", "motis-staging"], + }), + lease = await control.acquireArtifactTargetLease({ + dataRoot: join(f.root, "data"), + serviceIds: ["motis", "motis-staging"], + owner: f.authority, + }); + let fired = false; + try { + const phase = { phase: "stop-primary" as const, ordinal: 0 }, + bound = await control.bindControlChildPhase(f.authority, phase), + original = AnchoredDirectory.prototype.write; + const fault = vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(function ( + this: AnchoredDirectory, + ...args: Parameters + ) { + const [name, bytes] = args, + text = String(bytes), + path = this.path; + const selected = + !fired && + ((point === "reference" && + path.endsWith("/observations") && + name.startsWith("phase-") && + text.includes('"grant_prepared"')) || + (point === "cancellation-index" && + path.endsWith("/execution-lifecycle") && + text.includes('"leaseIntents":{')) || + (point === "lease-outbox" && + path.endsWith("/parent-attempts") && + text.includes('"kind":"lease"')) || + (point === "grant-outbox" && path.endsWith("/attempts")) || + (point === "shared-grant" && path.endsWith("/grants")) || + (point === "parent-linkage" && + path.endsWith(`/${lease.leaseId}`) && + name === "owner.json" && + text.includes('"grantDigests":{') && + !text.includes('"grantDigests":{}'))); + const value = Reflect.apply(original, this, args); + if (selected) { + fired = true; + throw Error(`Synthetic crash after ${point}`); + } + return value; + }); + await expect( + control.issueControlChildPhase(f.authority, { + ...phase, + lease, + childPublicKey: agent.publicKey, + }), + ).rejects.toThrow(); + fault.mockRestore(); + expect(fired).toBe(true); + // An ambiguous indexed write is recovery-required; a fully indexed reference + // is readable but never issues another nominal ticket or mutation. + try { + expect(control.lookupControlChildPhase(f.authority, phase)?.execution).toEqual( + bound.execution, + ); + } catch (error) { + expect(String(error)).toMatch(/retained|record|changed|identity/i); + } + await expect( + control.issueControlChildPhase(f.authority, { + ...phase, + lease, + childPublicKey: agent.publicKey, + }), + ).rejects.toThrow(); + try { + expect(control.cancelControlExecution(f.authority)).toContainEqual(bound.execution); + } catch { + /* Torn private/shared publication is itself a closed recovery fence. */ + } + expect(() => control.assertControlExecutionForward(f.authority)).toThrow(); + expect(statSync(join(f.root, "data/.admin-target-control/locks/motis")).isDirectory()).toBe( + true, + ); + await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); + } finally { + vi.restoreAllMocks(); + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agent); + f.close(); + } + }, +); + +it.each(["body", "phase", "ordinal", "target", "recipient"] as const)( + "rejects changed reserved %s before any binding or grant", + async (field) => { + const f = await fixture(); + try { + const reservations = structuredClone(f.input.reservations ?? []), + slot = reservations[0]; + if (!slot) throw Error("Fixture reservation missing"); + if (field === "body") slot.operationFingerprint = "f".repeat(64); + if (field === "phase") slot.phase = "recover-primary"; + if (field === "ordinal") slot.ordinal = 1; + if (field === "target") slot.serviceIds = ["motis-staging"]; + if (field === "recipient") slot.recipient = "data-manager"; + const original = readFileSync(f.cell); + expect(() => + control.retainAdmittedControlExecution(f.dm, { ...f.input, reservations }), + ).toThrow(); + expect( + control.lookupControlChildPhase(f.authority, { phase: "stop-primary", ordinal: 0 }), + ).toBeNull(); + expect(readFileSync(f.cell)).toEqual(original); + } finally { + f.close(); + } + }, +); +it.each(["version", "domain", "origin", "missing"] as const)( + "rejects %s phase evidence rather than binding a new identity", + async (field) => { + const f = await fixture(); + try { + if (field === "missing") rmSync(f.cell); + else { + const raw = JSON.parse(readFileSync(f.cell, "utf8")); + if (field === "version") raw.payload.version = 1; + if (field === "domain") raw.signature = "a".repeat(86); + if (field === "origin") raw.payload.origin.notAfter++; + writeFileSync(f.cell, JSON.stringify(raw)); + } + await expect( + control.bindControlChildPhase(f.authority, { phase: "stop-primary", ordinal: 0 }), + ).rejects.toThrow(); + } finally { + f.close(); + } + }, +); +it("rechecks durable cancellation inside publication after the outside identity read", async () => { + const f = await fixture(), + agent = control.openControlInstallation({ + infraDir: f.root, + role: "ops-agent", + serviceIds: ["motis", "motis-staging"], + }), + lease = await control.acquireArtifactTargetLease({ + dataRoot: join(f.root, "data"), + serviceIds: ["motis", "motis-staging"], + owner: f.authority, + }); + let fired = false; + try { + const phase = { phase: "stop-primary" as const, ordinal: 0 }; + await control.bindControlChildPhase(f.authority, phase); + const original = AnchoredDirectory.prototype.name; + vi.spyOn(AnchoredDirectory.prototype, "name").mockImplementation(function ( + this: AnchoredDirectory, + name: string, + ) { + if (!fired && name === ".journal-writer" && this.path.endsWith(`/${lease.leaseId}`)) { + fired = true; + control.cancelControlExecution(f.authority); + } + return original.call(this, name); + }); + await expect( + control.issueControlChildPhase(f.authority, { + ...phase, + lease, + childPublicKey: agent.publicKey, + }), + ).rejects.toThrow(/fenced|cancel/i); + expect(fired).toBe(true); + expect(inspectControlChildPublication(f.authority, phase).publication.state).toBe("unissued"); + expect(control.cancelControlExecution(f.authority)).toEqual([]); + } finally { + vi.restoreAllMocks(); + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agent); + f.close(); + } +}); +it("binds one same-slot identity across actual filesystem processes", async () => { + const f = await fixture(), + { build } = await import("esbuild"), + { spawn } = await import("node:child_process"), + { createInterface } = await import("node:readline"), + bundle = join(f.root, "phase-process.mjs"); + try { + await build({ + entryPoints: [join(import.meta.dirname, "test-fixtures/child-phase-process.ts")], + bundle: true, + platform: "node", + format: "esm", + target: "node24", + outfile: bundle, + }); + const run = async () => { + const process = spawn(globalThis.process.execPath, [bundle], { + stdio: ["pipe", "pipe", "pipe"], + }); + let stderr = ""; + process.stderr.on("data", (chunk) => (stderr += String(chunk))); + const lines = createInterface({ input: process.stdout }), + next = lines[Symbol.asyncIterator]().next(), + exited = new Promise((resolve) => process.once("close", () => resolve())); + process.stdin.end(JSON.stringify({ root: f.root, execution: f.execution }) + "\n"); + const result = await next; + await exited; + lines.close(); + if (result.done) throw Error(`Phase process exited without evidence: ${stderr}`); + return JSON.parse(result.value) as { + ok: boolean; + child?: control.ControlPlannedChild; + error?: string; + }; + }; + const results = await Promise.all([run(), run()]); + expect(results.some((result) => result.ok)).toBe(true); + const bound = control.lookupControlChildPhase(f.authority, { + phase: "stop-primary", + ordinal: 0, + }); + expect(bound).not.toBeNull(); + for (const result of results) + if (result.ok) expect(result.child).toEqual(bound); + else expect(result.error).toMatch(/busy|writer/); + expect( + await control.bindControlChildPhase(f.authority, { phase: "stop-primary", ordinal: 0 }), + ).toEqual(bound); + } finally { + f.close(); + } +}, 30000); +it("refuses a prospective phase plan above the combined64 slots before exposure", async () => { + const f = await fixture(); + try { + const reservations = Array.from({ length: 65 }, (_, ordinal) => ({ + ...f.input.reservations![0]!, + ordinal, + execution: { + ...f.input.reservations![0]!.execution, + operationKey: control.controlChildOperationKey( + f.execution.operationKey, + "stop-primary", + Math.min(ordinal, 63), + ), + }, + })); + expect(() => + control.retainAdmittedControlExecution(f.dm, { ...f.input, reservations }), + ).toThrow(); + expect( + control.lookupControlChildPhase(f.authority, { phase: "stop-primary", ordinal: 0 }), + ).toBeNull(); + } finally { + f.close(); + } +}); + +it.each(["cell", "aggregate"] as const)( + "reserves the signed %s capacity before any forward exposure", + async (limit) => { + const services = + limit === "cell" + ? Array.from( + { length: 256 }, + (_, n) => `svc-${String(n).padStart(3, "0")}-${"a".repeat(50)}`, + ) + : [ + "motis", + "motis-staging", + ...Array.from({ length: 8 }, (_, n) => `svc-${n}-${"a".repeat(50)}`), + ], + f = await fixture(false, services); + try { + const deadlineAt = f.execution.deadlineAt, + execution = { + ...f.execution, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + }, + base = f.input.reservations![0]!, + reservations = Array.from({ length: limit === "cell" ? 1 : 64 }, (_, ordinal) => ({ + ...base, + ordinal, + serviceIds: [...services].sort(), + execution: { + ...base.execution, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: control.controlChildOperationKey( + execution.operationKey, + "stop-primary", + ordinal, + ), + }, + })); + expect(() => + control.retainAdmittedControlExecution(f.dm, { + ...f.input, + execution, + reservations, + serviceIds: services, + }), + ).toThrow(limit === "cell" ? /cell capacity/ : /aggregate capacity/); + expect( + (await import("node:fs")).existsSync( + join( + f.root, + "data/.admin-target-control/executions/data-manager", + `${execution.operationKey}.json`, + ), + ), + ).toBe(false); + } finally { + f.close(); + } + }, + 30000, +); + +async function deadlineFixture() { + const root = mkdtempSync(join(tmpdir(), "omx-fixed-phase-ancestry-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const services = ["data-manager"], + agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: services, + }), + dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: services, + }), + operation = { kind: "data.downloadFonts" as const }, + operationFingerprint = opsOperationFingerprint(operation), + deadlineAt = Date.now() + 60000, + execution: ExecutionRef = { + executor: "ops-agent", + principal: "api", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }, + plan = control.planFixedSingleDataCommand(execution, operation, services), + authority = control.retainAdmittedControlExecution(agent, { + execution, + kind: operation.kind, + operation, + admittedAt: Date.now(), + requestedDeadlineAt: deadlineAt, + serviceIds: services, + reservations: [plan.rootReservation], + }); + control.retainFixedSingleDataPlan(authority, plan); + const bound = await control.bindControlChildPhase(authority, plan.rootReservation), + childPublicKey = control.prepareFixedControlChild(authority, plan.delegationId), + args = control.fixedCliArguments(operation); + control.retainFixedCliSpawnIntent(authority, { + delegationId: plan.delegationId, + execution: bound.execution, + operation, + args, + reservations: plan.cliReservations, + }); + const issued = await control.issueControlChildPhase(authority, { + ...plan.rootReservation, + childPublicKey, + }); + control.consumeControlChildDispatch(issued.dispatchTicket); + const cli = control.openFixedChildInstallation({ infraDir: root, assertion: issued.assertion }), + intent = control.readFixedCliSpawnIntent(cli, bound.execution, args), + peer = control.verifyFixedCliControlPeer(cli, { + assertion: issued.assertion, + execution: bound.execution, + kind: operation.kind, + }), + parent = control.verifyParentExecution(issued.assertion, peer, { + execution: bound.execution, + kind: operation.kind, + requestedDeadlineAt: bound.execution.deadlineAt, + }), + cliAuthority = control.retainAdmittedControlExecution(cli, { + execution: bound.execution, + kind: operation.kind, + operation, + admittedAt: intent.admittedAt, + requestedDeadlineAt: bound.execution.deadlineAt, + parent, + serviceIds: intent.serviceIds, + reservations: intent.reservations, + }); + control.beginFixedCliExecution(cli, bound.execution); + return { + root, + agent, + dm, + cli, + authority, + execution, + cliAuthority, + slot: plan.cliReservations[0]!, + close() { + control.closeControlInstallation(cli); + control.closeControlInstallation(dm); + control.closeControlInstallation(agent); + rmSync(root, { recursive: true, force: true }); + }, + }; +} +it.each(["cancel-first", "grant-first"] as const)( + "keeps fixed CLI deadline-only descendant publication ordered with ancestor cancellation (%s)", + async (order) => { + const f = await deadlineFixture(); + let fired = false; + try { + const bound = await control.bindControlChildPhase(f.cliAuthority, f.slot); + expect( + (await import("node:fs")).existsSync( + join(f.root, "data/.admin-target-control/locks/data-manager"), + ), + ).toBe(false); + if (order === "cancel-first") { + const original = AnchoredDirectory.prototype.name; + vi.spyOn(AnchoredDirectory.prototype, "name").mockImplementation(function ( + this: AnchoredDirectory, + name: string, + ) { + if ( + !fired && + name === `${f.execution.operationKey}.writer` && + this.path.endsWith("/ops-agent") + ) { + fired = true; + control.cancelControlExecution(f.authority); + } + return original.call(this, name); + }); + await expect( + control.issueControlChildPhase(f.cliAuthority, { + ...f.slot, + childPublicKey: f.dm.publicKey, + }), + ).rejects.toThrow(/cancel|fenced/i); + expect(fired).toBe(true); + expect(inspectControlChildPublication(f.cliAuthority, f.slot).publication.state).toBe( + "unissued", + ); + expect(control.cancelControlExecution(f.cliAuthority)).toEqual([]); + } else { + const issued = await control.issueControlChildPhase(f.cliAuthority, { + ...f.slot, + childPublicKey: f.dm.publicKey, + }); + control.cancelControlExecution(f.authority); + expect(control.cancelControlExecution(f.cliAuthority)).toEqual([bound.execution]); + expect(() => control.consumeControlChildDispatch(issued.dispatchTicket)).toThrow( + /cancel|fenced/i, + ); + expect(control.lookupControlChildPhaseAttempt(f.cliAuthority, f.slot)?.execution).toEqual( + bound.execution, + ); + } + } finally { + vi.restoreAllMocks(); + f.close(); + } + }, + 30000, +); diff --git a/packages/core/src/build-artifacts/child-phase.ts b/packages/core/src/build-artifacts/child-phase.ts new file mode 100644 index 000000000..4c0158832 --- /dev/null +++ b/packages/core/src/build-artifacts/child-phase.ts @@ -0,0 +1,562 @@ +/** Server-only fixed phase identities. An observation never mints a dispatch ticket. */ +import { z } from "zod"; +import { + type ExecutionRef, + executionFingerprint, + executionRefSchema, + normalizeExecutionAdmission, + opsExecutionBudget, +} from "../ops/execution-identity"; +import { canonicalJson } from "../ops/fingerprint"; +// biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { + assertControlDelegationForward, + assertControlPublicationScope, + attestParentExecution, + type ControlChildPhase, + type ControlPlannedChild, + type ControlRecipient, + childPhaseKinds, + controlChildOperationKey, + executionAuthorityState, + readPrivateRecord, + retainedControlParent, + type VerifiedExecutionAuthority, + verifiedRetainedControlParent, + verifyDeadlineEdge, + withControlDelegationPublication, + writePrivateRecord, +} from "./control-authority"; +// biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. +import { + controlDigest, + controlOriginSchema, + digest, + signControl, + uuid, + validateOriginRecord, + verifyControl, +} from "./control-records"; +import { verifyFamilyOwnedRecord } from "./retention-family"; +// biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { + delegateArtifactTargetLease, + inspectArtifactDelegation, + type TargetLeaseAuthority, + verifyFiniteControlEdge, +} from "./target-lease"; +// biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. + +export type ControlChildReservation = { + version: 1; + phase: ControlChildPhase; + ordinal: number; + kind: string; + recipient: ControlRecipient; + serviceIds: readonly string[]; + execution: Pick; + operationFingerprint: string; +}; +const address = z + .strictObject({ + executor: executionRefSchema.shape.executor, + principal: executionRefSchema.shape.principal, + executionId: z.string(), + operationKey: executionRefSchema.shape.operationKey, + }) + .refine((v) => + v.executor === "ops-agent" + ? /^job1_[A-Za-z0-9_-]{16,64}$/.test(v.executionId) + : z.uuid().safeParse(v.executionId).success, + ); +export const controlChildReservationSchema: z.ZodType = z.lazy(() => + z.strictObject({ + version: z.literal(1), + phase: z.enum(Object.keys(childPhaseKinds) as [ControlChildPhase, ...ControlChildPhase[]]), + ordinal: z.number().int().min(0).max(63), + kind: z.string().min(1).max(128), + recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), + serviceIds: z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine((a) => a.every((v, i) => i === 0 || (a[i - 1] ?? v) < v)), + execution: address, + operationFingerprint: digest, + }), +); +export function validateControlChildReservations(input: { + role: ControlRecipient | "host-cli"; + kind: string; + execution: ExecutionRef; + serviceIds: readonly string[]; + plan: readonly ControlPlannedChild[]; + reservations: readonly ControlChildReservation[]; +}): void { + const slots = [...input.plan, ...input.reservations].sort((a, b) => a.ordinal - b.ordinal); + if ( + slots.length > 64 || + new Set(slots.map((slot) => slot.execution.operationKey)).size !== slots.length + ) + throw Error("Control child reservation capacity or duplicate key"); + for (const [ordinal, slot] of slots.entries()) { + if ( + slot.ordinal !== ordinal || + slot.execution.operationKey !== + controlChildOperationKey(input.execution.operationKey, slot.phase, ordinal) || + !childPhaseKinds[slot.phase]?.includes(slot.kind) || + slot.serviceIds.some((id) => !input.serviceIds.includes(id)) + ) + throw Error("Invalid immutable child reservation"); + if ( + (slot.recipient === "fixed-cli" && slot.phase === "data-workflow") || + (slot.recipient === "data-manager" && + ["osm", "transit", "fonts", "data-step"].includes(slot.phase)) + ) + verifyDeadlineEdge(input.kind, slot.phase, slot.recipient, slot.kind); + else verifyFiniteControlEdge(input.role, input.kind, slot.phase, slot.recipient); + const executor = + slot.recipient === "ops-agent" + ? "ops-agent" + : slot.recipient === "data-manager" + ? "data-manager" + : "host-cli"; + if (slot.execution.executor !== executor) + throw Error("Reserved receiver execution address mismatch"); + } +} +const planSchema = z.lazy(() => + z.strictObject({ + phase: z.enum(Object.keys(childPhaseKinds) as [ControlChildPhase, ...ControlChildPhase[]]), + ordinal: z.number().int().min(0).max(63), + kind: z.string(), + recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), + serviceIds: z.array(z.string()).max(256), + execution: executionRefSchema, + }), +); +const publicationSchema = z.discriminatedUnion("state", [ + z.strictObject({ state: z.literal("unissued") }), + z.strictObject({ + state: z.literal("grant_prepared"), + delegationId: uuid, + grantDigest: digest, + leaseId: uuid.nullable(), + }), +]); +const cellSchema = z.strictObject({ + version: z.literal(2), + installationId: uuid, + origin: controlOriginSchema, + parent: executionRefSchema, + admissionDigest: digest, + reservationDigest: digest, + binding: z + .strictObject({ + child: planSchema, + boundAt: z.number().int().nonnegative().safe(), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + }) + .nullable(), + publication: publicationSchema, +}); +type Cell = z.infer; +const cellName = (authority: VerifiedExecutionAuthority, ordinal: number) => + `phase-${authority.execution.operationKey}-${ordinal}.json`; +function reservation( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +) { + const state = executionAuthorityState(authority); + const value = state.reservations.find( + (r) => r.phase === input.phase && r.ordinal === input.ordinal, + ); + if (!value) throw Error("Phase is absent from immutable reservations"); + return { state, value }; +} +function readCell( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +): Cell { + const { state, value } = reservation(authority, input); + const raw = readPrivateRecord( + state.installation, + "observations", + cellName(authority, input.ordinal), + 8192, + ); + const cell = verifyControl( + state.installation.installation.publicKey, + "openmapx-control-phase-binding-v2", + raw, + cellSchema, + ); + if (!state.origin) throw Error("Reserved phase origin is missing"); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { + scope: "private", + namespace: "observations", + name: cellName(authority, input.ordinal), + container: "", + }, + canonicalJson(raw), + ); + if ( + !validateOriginRecord(cell) || + cell.installationId !== state.installation.installation.installationId || + controlDigest(cell.origin) !== controlDigest(state.origin) || + controlDigest(cell.parent) !== controlDigest(state.execution) || + cell.admissionDigest !== state.receiptDigest || + cell.reservationDigest !== controlDigest(value) + ) + throw Error("Reserved phase binding identity changed"); + if (cell.binding) { + const { child, boundAt, requestedDeadlineAt } = cell.binding; + const expected = { + phase: value.phase, + ordinal: value.ordinal, + kind: value.kind, + recipient: value.recipient, + serviceIds: value.serviceIds, + execution: { + ...value.execution, + operationFingerprint: value.operationFingerprint, + deadlineAt: requestedDeadlineAt, + fingerprint: executionFingerprint({ + kind: value.kind, + operationFingerprint: value.operationFingerprint, + deadlineAt: requestedDeadlineAt, + parent: retainedControlParent(state.execution), + }), + }, + }; + if ( + requestedDeadlineAt <= boundAt || + requestedDeadlineAt > state.execution.deadlineAt || + controlDigest(child) !== controlDigest(expected) + ) + throw Error("Reserved phase exact identity mismatch"); + } else if (cell.publication.state !== "unissued") + throw Error("An unbound phase cannot have a grant"); + return cell; +} +function writeCell( + authority: VerifiedExecutionAuthority, + ordinal: number, + cell: Cell, + exclusive = false, +) { + const state = executionAuthorityState(authority); + const signed = signControl( + state.installation.privateKey, + "openmapx-control-phase-binding-v2", + cellSchema.parse(cell), + ); + if (Buffer.byteLength(canonicalJson(signed)) > 8192) + throw Error("Phase binding capacity exceeded"); + writePrivateRecord( + state.installation, + "observations", + cellName(authority, ordinal), + signed, + exclusive, + state.origin, + ); +} +/** Called by durable admission before any forward witness/effect becomes ready. */ +export function initializeControlChildReservations(authority: VerifiedExecutionAuthority): void { + const state = executionAuthorityState(authority); + if (!state.origin) throw Error("Missing reserved phase origin"); + const cells = state.reservations.map((r) => ({ + r, + cell: cellSchema.parse({ + version: 2, + installationId: state.installation.installation.installationId, + origin: state.origin, + parent: state.execution, + admissionDigest: state.receiptDigest, + reservationDigest: controlDigest(r), + binding: null, + publication: { state: "unissued" }, + }), + })); + // Bound exact identities, decimal epoch lengths and UUID/digest publication + // fields have fixed maxima. Budget their complete signed envelopes up front. + const maximum = cells.reduce((total, { r, cell }) => { + const largest = { + ...cell, + binding: { + boundAt: Number.MAX_SAFE_INTEGER, + requestedDeadlineAt: Number.MAX_SAFE_INTEGER, + child: { + phase: r.phase, + ordinal: r.ordinal, + kind: r.kind, + recipient: r.recipient, + serviceIds: r.serviceIds, + execution: { + ...r.execution, + operationFingerprint: r.operationFingerprint, + deadlineAt: Number.MAX_SAFE_INTEGER, + fingerprint: "f".repeat(64), + }, + }, + }, + publication: { + state: "grant_prepared", + delegationId: "f".repeat(36), + grantDigest: "f".repeat(64), + leaseId: "f".repeat(36), + }, + }; + const size = Buffer.byteLength(canonicalJson({ payload: largest, signature: "f".repeat(86) })); + if (size > 8192) throw Error("Phase binding cell capacity exceeded"); + return total + size; + }, 0); + if (maximum > 128 * 1024) throw Error("Phase binding aggregate capacity exceeded"); + for (const { r, cell } of cells) writeCell(authority, r.ordinal, cell, true); +} +export function lookupControlChildPhase( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +): ControlPlannedChild | null { + return readCell(authority, input).binding?.child ?? null; +} +export async function bindControlChildPhase( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +): Promise { + const retained = readCell(authority, input); + if (retained.binding) return retained.binding.child; + return withControlDelegationPublication(authority, input.phase, () => { + const current = readCell(authority, input); + if (current.binding) return current.binding.child; + const { state, value } = reservation(authority, input), + boundAt = Date.now(); + const requestedDeadlineAt = Math.min( + state.execution.deadlineAt, + boundAt + opsExecutionBudget(value.kind).processMs, + ); + const normalized = normalizeExecutionAdmission({ + kind: value.kind, + operationKey: value.execution.operationKey, + operationFingerprint: value.operationFingerprint, + admittedAt: boundAt, + requestedDeadlineAt, + parent: verifiedRetainedControlParent(state.installation.installation, state.execution), + }); + const child = { + phase: value.phase, + ordinal: value.ordinal, + kind: value.kind, + recipient: value.recipient, + serviceIds: [...value.serviceIds], + execution: { + ...value.execution, + operationFingerprint: value.operationFingerprint, + deadlineAt: normalized.deadlineAt, + fingerprint: normalized.fingerprint, + }, + }; + writeCell(authority, value.ordinal, { + ...current, + binding: { child, boundAt, requestedDeadlineAt }, + }); + return child; + }); +} +/** Internal fixed issuer hook, invoked while the actual publication guards are held. */ +export function prepareControlChildPublication( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, + reference: { delegationId: string; grantDigest: string; leaseId: string | null }, +): void { + assertControlPublicationScope(authority); + const state = executionAuthorityState(authority); + if (!state.reservations.some((r) => r.phase === input.phase && r.ordinal === input.ordinal)) + return; + const cell = readCell(authority, input); + if (!cell.binding || cell.publication.state !== "unissued") + throw Error("Reserved phase publication requires observation"); + writeCell(authority, input.ordinal, { + ...cell, + publication: { state: "grant_prepared", ...reference }, + }); +} +export function inspectControlChildPublication( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +) { + const cell = readCell(authority, input); + return { + binding: cell.binding, + publication: cell.publication, + bindingDigest: controlDigest(cell.binding), + }; +} +export function lookupControlChildPhaseBinding( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +) { + const cell = readCell(authority, input); + return cell.binding + ? Object.freeze({ ...cell.binding, bindingDigest: controlDigest(cell.binding) }) + : null; +} +/** Restore the original private outbox for observation/cancellation only. This + * returns no ticket and never repairs or republishes a missing attempt. */ +export function lookupControlChildPhaseAttempt( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +) { + const cell = readCell(authority, input), + s = executionAuthorityState(authority); + if (cell.publication.state === "unissued") return null; + if (!cell.binding) throw Error("Attempted phase has no exact binding"); + const reference = cell.publication, + channel = reference.leaseId === null ? ("parent" as const) : ("delegation" as const); + const saved = z + .object({ grant: z.unknown(), assertion: z.string().max(4096) }) + .parse( + readPrivateRecord( + s.installation, + channel === "parent" ? "parent-attempts" : "attempts", + `${reference.delegationId}.json`, + 65536, + ), + ); + if (controlDigest(saved.grant) !== reference.grantDigest) + throw Error("Original phase outbox digest changed"); + if (channel === "delegation") { + const observed = inspectArtifactDelegation({ + dataRoot: s.installation.dataRoot, + assertion: saved.assertion, + installation: s.installation.installation, + }); + if ( + observed.leaseId !== reference.leaseId || + observed.delegationId !== reference.delegationId || + observed.kind !== cell.binding.child.kind || + controlDigest(observed.execution) !== controlDigest(cell.binding.child.execution) || + controlDigest(observed.parent) !== controlDigest(s.execution) + ) + throw Error("Original phase grant identity changed"); + } else { + const observed = verifyControl( + s.installation.installation.publicKey, + "openmapx-parent-grant-v2", + saved.grant, + z + .object({ + version: z.literal(2), + origin: controlOriginSchema, + parent: executionRefSchema, + child: executionRefSchema, + kind: z.string(), + attemptId: uuid, + }) + .passthrough(), + ); + if ( + observed.attemptId !== reference.delegationId || + observed.kind !== cell.binding.child.kind || + controlDigest(observed.origin) !== controlDigest(s.origin) || + controlDigest(observed.child) !== controlDigest(cell.binding.child.execution) || + controlDigest(observed.parent) !== controlDigest(s.execution) + ) + throw Error("Original parent phase identity changed"); + } + return Object.freeze({ + channel, + assertion: saved.assertion, + execution: cell.binding.child.execution, + bindingDigest: controlDigest(cell.binding), + ...reference, + }); +} +export function resolveControlChildPlan( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; execution: ExecutionRef; recipient: ControlRecipient }, +): ControlPlannedChild | undefined { + const state = executionAuthorityState(authority); + const exact = state.plan.find( + (p) => + p.phase === input.phase && + p.recipient === input.recipient && + controlDigest(p.execution) === controlDigest(input.execution), + ); + if (exact) return exact; + const slot = state.reservations.find( + (r) => r.phase === input.phase && r.execution.operationKey === input.execution.operationKey, + ); + if (!slot) return undefined; + const bound = lookupControlChildPhase(authority, slot); + if ( + bound && + bound.recipient === input.recipient && + controlDigest(bound.execution) === controlDigest(input.execution) + ) + return bound; + return undefined; +} +const dispatchBrand = Symbol("fresh-control-child-dispatch"); +export type FreshControlChildDispatch = Readonly<{ [dispatchBrand]: true }>; +const tickets = new WeakMap< + FreshControlChildDispatch, + { + used: boolean; + authority: VerifiedExecutionAuthority; + phase: ControlChildPhase; + deadlineAt: number; + } +>(); +export async function issueControlChildPhase( + authority: VerifiedExecutionAuthority, + input: { + phase: ControlChildPhase; + ordinal: number; + childPublicKey: string; + lease?: TargetLeaseAuthority; + }, +) { + const child = lookupControlChildPhase(authority, input); + if (!child) throw Error("Child phase is not bound"); + const before = inspectControlChildPublication(authority, input); + if (before.publication.state !== "unissued") throw Error("Attempted child requires observation"); + if ( + input.lease && + controlDigest(input.lease.mode === "owner" ? input.lease.owner : input.lease.execution) !== + controlDigest(authority.execution) + ) + throw Error("Foreign phase target authority"); + const assertion = input.lease + ? await delegateArtifactTargetLease(input.lease, { + ...child, + childPublicKey: input.childPublicKey, + }) + : await attestParentExecution(authority, { ...child, childPublicKey: input.childPublicKey }); + const saved = inspectControlChildPublication(authority, input); + if (saved.publication.state !== "grant_prepared") + throw Error("Child publication was not retained"); + const dispatchTicket = Object.freeze({ [dispatchBrand]: true as const }); + tickets.set(dispatchTicket, { + used: false, + authority, + phase: input.phase, + deadlineAt: child.execution.deadlineAt, + }); + return { + channel: input.lease ? ("delegation" as const) : ("parent" as const), + assertion, + dispatchTicket, + }; +} +export function consumeControlChildDispatch(ticket: FreshControlChildDispatch): void { + const saved = tickets.get(ticket); + if (!saved || saved.used) throw Error("Missing or consumed child dispatch ticket"); + saved.used = true; + assertControlDelegationForward(saved.authority, saved.phase); + if (Date.now() >= saved.deadlineAt) throw Error("Child dispatch deadline elapsed"); +} diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 33163938c..8d571bcec 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -7,10 +7,13 @@ import { originRecordDomain, sameControlOrigin, signControl, + targetSchema, uuid, validateOriginRecord, verifyControl, } from "./control-records"; +// biome-ignore lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { verifyRetainedArtifactTargetRootRelease } from "./target-lease"; export { type ControlInstallation, @@ -21,6 +24,7 @@ export { originRecordDomain, sameControlOrigin, signControl, + targetSchema, uuid, validateOriginRecord, verifyControl, @@ -59,7 +63,28 @@ import { } from "../ops/execution-identity"; import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; import { verifiedParentOrigins, verifiedParents } from "../ops/verified-parent-internal"; +import { dataConfigurationFenceReceiptSchema } from "../services/data-manager/contracts/configuration-maintenance"; +import { dataExecutionStatusSchema } from "../services/data-manager/contracts/executions"; +// biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { + type ControlChildReservation, + controlChildReservationSchema, + initializeControlChildReservations, + inspectControlChildPublication, + lookupControlChildPhase, + prepareControlChildPublication, + resolveControlChildPlan, + validateControlChildReservations, +} from "./child-phase"; +// biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. import { CONTROL_RETENTION_LIMITS, writeRetainedRecord } from "./control-retention"; +// biome-ignore-start lint/suspicious/noImportCycles: Fixed proof verification uses this module's original nominal authority and effect writer after initialization. +import { + inspectFeedProxyValidationFailure, + publishFeedProxyCanonicalTerminal, + recordFeedProxyReloadAttempted, +} from "./feed-proxy-validation"; +// biome-ignore-end lint/suspicious/noImportCycles: Fixed live proof dependency described above. import { AnchoredDirectory, identity } from "./filesystem"; import { assertOriginClockForward, @@ -146,14 +171,7 @@ const roleKeySchema = z.strictObject({ role: z.enum(CONTROL_ROLES), privateKey: z.string().max(512), }); -export const targetSchema = z - .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) - .min(1) - .max(256) - .refine( - (x) => x.every((v, i) => i === 0 || (x[i - 1] ?? v) < v), - "Targets must be sorted and unique", - ); + export function newControlKey() { const { privateKey, publicKey } = generateKeyPairSync("ed25519"); return { @@ -454,6 +472,7 @@ export interface AuthorityState { kind: string; serviceIds: readonly string[]; plan: readonly ControlPlannedChild[]; + reservations: readonly ControlChildReservation[]; receiptDigest: string; origin?: ControlExecutionOrigin; } @@ -483,7 +502,8 @@ export function writePrivateRecord( const dir = state.privateRoot.directory(namespace, true, true); try { const bytes = canonicalJson(value); - if (Buffer.byteLength(bytes) > 131072) throw Error("Private receipt limit"); + if (Buffer.byteLength(bytes) > (namespace === "hardlinks" ? 1024 * 1024 : 131072)) + throw Error("Private receipt limit"); if ( ![ "executions", @@ -505,6 +525,8 @@ export function writePrivateRecord( "spawn-started", "prepared", "execution-adoptions", + "configuration-maintenance", + "hardlinks", ].includes(namespace) ) throw Error("Unknown fixed authority namespace"); @@ -516,6 +538,7 @@ export function writePrivateRecord( exclusive, CONTROL_RETENTION_LIMITS.recordsPerNamespace, beforePublish, + namespace === "hardlinks" ? 1024 * 1024 : CONTROL_RETENTION_LIMITS.privateRecordBytes, ); if (origin) familyRecordWrite( @@ -782,6 +805,8 @@ export function retainControlTerminalStatus( const proof = verifyAndSealTerminalCommitment(s, status, terminalAt), origin = s.origin; if (!origin) throw Error("Missing terminal origin"); + if (s.kind === "feedProxy.validateAndReload") + publishFeedProxyCanonicalTerminal(authority, status, proof.terminalAt); const settlement = childSettlementState(verifyControlEffectSettlement(installation, execution)); publishFamilyCanonicalTerminal( s.installation, @@ -853,9 +878,18 @@ export function repairControlJournalTerminal( proof.outcome === "failed_without_effect" && proof.publication === "not_started" && proof.receipt.kind === "not-started"; - if (!succeeded && !failedWithoutEffect) + const validationFailed = + terminal.state === "failed" && + s.kind === "feedProxy.validateAndReload" && + proof.outcome === "contained" && + proof.publication === "not_started" && + proof.receipt.kind === "feed-proxy-validation-failure" && + inspectFeedProxyValidationFailure(authority, proof.receipt.commitmentDigest) !== null; + if (!succeeded && !failedWithoutEffect && !validationFailed) throw Error("Terminal worker containment is not proved"); const committed = verifyAndSealTerminalCommitment(s, terminal, terminalAt); + if (s.kind === "feedProxy.validateAndReload") + publishFeedProxyCanonicalTerminal(authority, terminal, committed.terminalAt); publishFamilyCanonicalTerminal( s.installation, origin, @@ -934,9 +968,18 @@ export function assertFreshControlRoot( notAfter: parsed.notAfter, }); } +export function controlInstallationDataRoot(installation: ControlInstallation): string { + return installationState(installation).dataRoot; +} export function controlRegisteredServiceIds(installation: ControlInstallation): string[] { return [...installationState(installation).registeredTargets].sort(); } +export function controlRolePublicKey( + installation: ControlInstallation, + role: ControlIssuer, +): string { + return installationState(installation).publicKeys[role]; +} function writeExecutionShared( state: InstallationState, execution: ExecutionRef, @@ -982,6 +1025,7 @@ export function retainAdmittedControlExecution( parent?: RetainedParentExecution; serviceIds: readonly string[]; plan?: readonly ControlPlannedChild[]; + reservations?: readonly ControlChildReservation[]; }, ): VerifiedExecutionAuthority { const state = installationState(installation); @@ -1007,6 +1051,9 @@ export function retainAdmittedControlExecution( "download-fonts", "convert-overpass", "overture-sync", + "overture-pull", + "overture-ingest", + "overture-extract", "overture-conflate", "search-index-build", "notable-places-build", @@ -1042,6 +1089,18 @@ export function retainAdmittedControlExecution( .array(plannedChildSchema) .max(64) .parse(input.plan ?? []); + const reservations = z + .array(controlChildReservationSchema) + .max(64) + .parse(input.reservations ?? []); + validateControlChildReservations({ + role: state.fixedChildId ? "fixed-cli" : installation.role, + kind: input.kind, + execution, + serviceIds, + plan, + reservations, + }); const budget = (() => { try { return opsExecutionMaximumMs(input.kind); @@ -1065,13 +1124,13 @@ export function retainAdmittedControlExecution( })(); if (execution.deadlineAt > input.admittedAt + budget) throw Error("Control execution exceeds kind budget"); - if (plan.length > 64) throw Error("Control child intent capacity"); + if (plan.length + reservations.length > 64) throw Error("Control child intent capacity"); const keys = new Set(); - for (const [ordinal, child] of plan.entries()) { + for (const child of plan) { executionRefSchema.parse(child.execution); targetSchema.parse(child.serviceIds); if ( - child.ordinal !== ordinal || + !Number.isSafeInteger(child.ordinal) || keys.has(child.execution.operationKey) || child.execution.operationKey !== controlChildOperationKey(execution.operationKey, child.phase, child.ordinal) || @@ -1111,7 +1170,10 @@ export function retainAdmittedControlExecution( origin: _origin, ...oldInput } = stored; - if (controlDigest(oldInput) !== controlDigest({ ...input, execution, serviceIds, plan })) + if ( + controlDigest({ ...oldInput, reservations: oldInput.reservations ?? [] }) !== + controlDigest({ ...input, execution, serviceIds, plan, reservations }) + ) throw Error("Conflicting retained execution identity"); return restoreAdmittedControlExecution(installation, execution); } @@ -1164,12 +1226,6 @@ export function retainAdmittedControlExecution( writePrivateRecord(state, "effect-started", name, null, true, origin); newlyAdmitted = true; } - publishControlExecutionWitness( - state, - { execution, kind: input.kind, serviceIds, origin }, - controlDigest(receipt), - ); - initializeLifecycle(state, execution, newlyAdmitted); const authority = Object.freeze({ execution }) as VerifiedExecutionAuthority; authorities.set(authority, { installation: state, @@ -1177,9 +1233,17 @@ export function retainAdmittedControlExecution( kind: input.kind, serviceIds, plan, + reservations, receiptDigest: controlDigest(receipt), origin, }); + if (newlyAdmitted) initializeControlChildReservations(authority); + publishControlExecutionWitness( + state, + { execution, kind: input.kind, serviceIds, origin }, + controlDigest(receipt), + ); + initializeLifecycle(state, execution, newlyAdmitted); return authority; } export interface PeerState { @@ -1336,7 +1400,10 @@ export function prepareFixedControlChild( } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; } - if (ids.length >= Math.min(64, s.plan.length) || ids.includes(delegationId)) + if ( + ids.length >= Math.min(64, s.plan.length + s.reservations.length) || + ids.includes(delegationId) + ) throw Error("Fixed child key capacity/replay"); writeExecutionShared( s.installation, @@ -1435,7 +1502,25 @@ const settlementSchema = z.strictObject({ terminalReceiptDigest: digest, }); const effectObservationSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("data-configuration-fence-refused"), + maintenanceId: uuid, + reason: z.literal("busy"), + }), + z.strictObject({ + kind: z.literal("data-configuration-fence"), + receipt: dataConfigurationFenceReceiptSchema, + guardDigest: digest, + }), + z.strictObject({ + kind: z.literal("data-execution"), + status: dataExecutionStatusSchema, + workerId: uuid, + childrenDigest: digest, + }), + z.strictObject({ kind: z.literal("data-command"), status: dataExecutionStatusSchema }), z.strictObject({ kind: z.literal("ops-result"), result: z.unknown() }), + z.strictObject({ kind: z.literal("feed-proxy-validation-failure"), commitmentDigest: digest }), z.strictObject({ kind: z.literal("fixed-cli-exit"), args: z.array(z.string().max(256)).max(64), @@ -1444,7 +1529,7 @@ const effectObservationSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("not-started") }), z.strictObject({ kind: z.literal("unresolved") }), ]); -function withPrivateEffectWriter(s: AuthorityState, write: () => T): T { +export function withPrivateEffectWriter(s: AuthorityState, write: () => T): T { const guards = s.installation.privateRoot.directory("effect-writers", true, true); try { const name = s.execution.operationKey; @@ -1511,11 +1596,52 @@ export function retainControlEffectReceipt( outcome = "committed"; publication = "committed"; } + } else if (receipt.kind === "feed-proxy-validation-failure") { + if ( + s.kind !== "feedProxy.validateAndReload" || + inspectFeedProxyValidationFailure(authority, receipt.commitmentDigest) === null + ) + throw Error("Original contained validation failure required"); + outcome = "contained"; + publication = "not_started"; + } else if (receipt.kind === "data-command") { + const slots = s.reservations.filter( + (slot) => slot.recipient === "data-manager" && slot.phase === "data-step", + ); + if (slots.length !== 1) throw Error("Exact single data command required"); + const bound = lookupControlChildPhase(authority, slots[0]!); + if ( + !bound || + bound.execution.executionId !== receipt.status.id || + bound.execution.fingerprint !== receipt.status.fingerprint + ) + throw Error("Bound data command identity required"); + verifyDeadlineEdge(s.kind, "data-step", "data-manager", receipt.status.kind); + verifyDataControlTerminalWitness( + s.installation.installation, + bound.execution, + receipt.status, + ); + if (receipt.status.status === "ok") { + outcome = "committed"; + publication = "committed"; + } } else if (receipt.kind === "fixed-cli-exit") { if (!s.installation.fixedChildId) throw Error("Fixed command receipt requires certified child"); readFixedCliSpawnIntent(s.installation.installation, s.execution, receipt.args); readPrivateRecord(s.installation, "spawn-started", `${s.installation.fixedChildId}.json`); + for (const slot of s.reservations.filter( + (item) => item.recipient === "data-manager" && item.phase === "data-step", + )) { + const binding = lookupControlChildPhase(authority, slot); + if ( + !binding || + verifyDataControlTerminalWitness(s.installation.installation, binding.execution).status + .status !== "ok" + ) + throw Error("Actual backend data containment required before fixed command success"); + } outcome = "committed"; publication = "committed"; } else if (receipt.kind === "not-started") { @@ -1711,6 +1837,114 @@ function executionWitnessDirectory( }, }; } +const dataContainmentWitnessSchema = z.strictObject({ + version: z.literal(1), + installationId: uuid, + execution: executionRefSchema, + status: dataExecutionStatusSchema, + terminalAt: z.number().int().positive().safe(), + terminalDigest: digest, +}); +/** Fixed DM journal calls only after verifying its own DB tuple, private + * original terminal commitment, all children and actual target release. */ +export function publishDataControlTerminalWitness( + authority: VerifiedExecutionAuthority, + status: import("../services/data-manager/contracts/executions").DataExecutionStatus, + terminalAt: number, +): void { + const s = executionAuthorityState(authority); + if (s.installation.installation.role !== "data-manager" || s.installation.fixedChildId) + throw Error("Original data journal required"); + verifyRetainedArtifactTargetRootRelease(authority); + const expected = controlDigest({ version: 1, status: dataExecutionStatusSchema.parse(status) }); + const original = verifyControl( + s.installation.installation.publicKey, + "openmapx-data-terminal-v1", + readPrivateRecord( + s.installation, + "observations", + `terminal-${s.execution.operationKey}.json`, + 8192, + ), + z + .object({ + phase: z.literal("canonical"), + execution: executionRefSchema, + terminalDigest: digest, + terminalAt: z.number(), + admissionDigest: digest, + }) + .passthrough(), + ); + if ( + original.terminalDigest !== expected || + original.terminalAt !== terminalAt || + original.admissionDigest !== s.receiptDigest || + controlDigest(original.execution) !== controlDigest(s.execution) + ) + throw Error("Original canonical data commitment required"); + const payload = dataContainmentWitnessSchema.parse({ + version: 1, + installationId: s.installation.installation.installationId, + execution: s.execution, + status, + terminalAt, + terminalDigest: expected, + }), + directory = executionWitnessDirectory(s.installation, "data-manager"); + try { + const name = `${s.execution.operationKey}.data-terminal.json`, + bytes = canonicalJson( + signControl(s.installation.privateKey, "openmapx-data-contained-terminal-v1", payload), + ); + let prior: Buffer | undefined; + try { + prior = directory.role.read(name, 65536); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (prior) { + if (prior.toString("utf8") !== bytes) throw Error("Canonical data witness changed"); + return; + } + writeExecutionShared(s.installation, s.execution, directory.role, name, bytes, true); + } finally { + directory.close(); + } +} +/** A parent verifies the pinned DM signature and exact bound execution; a GET + * status or dead child process alone is never backend containment evidence. */ +export function verifyDataControlTerminalWitness( + installation: ControlInstallation, + execution: ExecutionRef, + status?: import("../services/data-manager/contracts/executions").DataExecutionStatus, +) { + const local = installationState(installation), + directory = executionWitnessDirectory(local, "data-manager"); + try { + const proof = verifyControl( + local.publicKeys["data-manager"], + "openmapx-data-contained-terminal-v1", + JSON.parse( + directory.role.read(`${execution.operationKey}.data-terminal.json`, 65536).toString("utf8"), + ), + dataContainmentWitnessSchema, + ); + if ( + proof.installationId !== installation.installationId || + controlDigest(proof.execution) !== controlDigest(execution) || + proof.terminalDigest !== controlDigest({ version: 1, status: proof.status }) || + (status && canonicalJson(status) !== canonicalJson(proof.status)) || + !["ok", "partial", "error", "canceled", "interrupted"].includes(proof.status.status) || + proof.status.publication === "unknown" + ) + throw Error("Exact canonical data containment witness required"); + readControlExecutionWitness(local, execution, local.publicKeys["data-manager"]); + return proof; + } finally { + directory.close(); + } +} const lifecycleSchema = z.strictObject({ version: z.literal(1), installationId: uuid, @@ -1718,6 +1952,17 @@ const lifecycleSchema = z.strictObject({ sequence: z.number().int().nonnegative().safe(), phase: z.enum(["held", "cancel_pending", "recovery_required", "settled"]), childIntents: z.record(uuid, digest), + leaseIntents: z + .record( + uuid, + z.strictObject({ + child: executionRefSchema, + leaseId: uuid, + grantDigest: digest, + attemptDigest: digest, + }), + ) + .optional(), }); type ExecutionLifecycle = z.infer; function withExecutionWriter( @@ -1755,7 +2000,7 @@ function readLifecycle( if ( current.installationId !== state.installation.installationId || controlDigest(current.execution) !== controlDigest(execution) || - Object.keys(current.childIntents).length > 64 + Object.keys(current.childIntents).length + Object.keys(current.leaseIntents ?? {}).length > 64 ) throw Error("Execution lifecycle identity/capacity mismatch"); const observationName = `${execution.executor}-${execution.operationKey}.json`; @@ -1888,6 +2133,10 @@ function withParentLifecycles( return lock(0, true); } /** Narrow local runtime fence; exact duplicate admission alone never grants forward permission. */ +export class ControlExecutionFencedError extends Error {} +export function markFeedProxyReloadAttempted(authority: VerifiedExecutionAuthority): void { + withControlExecutionForward(authority, () => recordFeedProxyReloadAttempted(authority)); +} function withControlExecutionForward(authority: VerifiedExecutionAuthority, run: () => T): T { const s = executionAuthorityState(authority); if (!s.origin) @@ -1904,7 +2153,7 @@ function withControlExecutionForward(authority: VerifiedExecutionAuthority, r s.installation.installation.publicKey, ); if (lifecycle.phase !== "held" || Date.now() >= s.execution.deadlineAt) - throw Error( + throw new ControlExecutionFencedError( "Execution cannot restart effects; observation/settlement only; forward phase fenced", ); return run(); @@ -1912,25 +2161,41 @@ function withControlExecutionForward(authority: VerifiedExecutionAuthority, r const assertion = retainedParentAssertion(s); return assertion ? withParentLifecycles(s.installation, assertion, (forward) => { - if (!forward) throw Error("Canceled/terminal ancestor permits observation only"); + if (!forward) + throw new ControlExecutionFencedError( + "Canceled/terminal ancestor permits observation only", + ); return check(); }) : check(); } -export function assertControlExecutionForward(authority: VerifiedExecutionAuthority): void { - withControlExecutionForward(authority, () => {}); +/** Internal publication scope. All filesystem writes complete before guards + * release; callers may not await HTTP/DB or acquire a lease from this scope. */ +const publicationScopes = new WeakMap(); +export function assertControlPublicationScope( + authority: VerifiedExecutionAuthority, +): AnchoredDirectory { + const directory = publicationScopes.get(authority); + if (!directory) throw Error("Control publication scope is not held"); + return directory; } -/** Only the protocol's frozen terminal preparation edges may outlive their local worker. */ -export function assertControlDelegationForward( +export function withControlDelegationPublication( authority: VerifiedExecutionAuthority, phase: ControlChildPhase, -): void { - try { - assertControlExecutionForward(authority); - } catch (error) { - const s = executionAuthorityState(authority); - if ( - !( + run: (directory: AnchoredDirectory) => T, +): T { + const s = executionAuthorityState(authority); + if (!s.origin) throw Error("Legacy control admission cannot publish grants"); + assertOriginClockForward(s.installation, s.origin); + const own = () => + withExecutionWriter(s.installation, s.execution, (directory) => { + const current = readLifecycle( + s.installation, + directory, + s.execution, + s.installation.installation.publicKey, + ); + const prepared = (s.kind === "configurationMaintenance.prepare" && [ "configuration-commit", @@ -1938,21 +2203,40 @@ export function assertControlDelegationForward( "maintenance-recreate", "maintenance-handoff", ].includes(phase)) || - (s.kind === "configurationHandoff.prepare" && phase === "configuration-handoff") + (s.kind === "configurationHandoff.prepare" && phase === "configuration-handoff"); + if ( + Date.now() >= s.execution.deadlineAt || + !(current.phase === "held" || (current.phase === "settled" && prepared)) ) - ) - throw error; - withExecutionWriter(s.installation, s.execution, (dir) => { - const lifecycle = readLifecycle( - s.installation, - dir, - s.execution, - s.installation.installation.publicKey, - ); - if (lifecycle.phase !== "settled" || Date.now() >= s.execution.deadlineAt) throw error; - verifyControlPreparedTerminal(s.installation, s.execution); + throw new ControlExecutionFencedError( + "Control grant publication is fenced; terminal or canceled execution prevents forward effects", + ); + if (current.phase === "settled") verifyControlPreparedTerminal(s.installation, s.execution); + if (publicationScopes.has(authority)) throw Error("Nested control publication scope"); + publicationScopes.set(authority, directory); + try { + return run(directory); + } finally { + publicationScopes.delete(authority); + } }); - } + const assertion = retainedParentAssertion(s); + return assertion + ? withParentLifecycles(s.installation, assertion, (forward) => { + if (!forward) throw Error("Canceled ancestor prevents grant publication"); + return own(); + }) + : own(); +} +export function assertControlExecutionForward(authority: VerifiedExecutionAuthority): void { + withControlExecutionForward(authority, () => {}); +} +/** Only the protocol's frozen terminal preparation edges may outlive their local worker. */ +export function assertControlDelegationForward( + authority: VerifiedExecutionAuthority, + phase: ControlChildPhase, +): void { + withControlDelegationPublication(authority, phase, () => {}); } function retainedParentAssertion(s: AuthorityState): string | undefined { if (s.installation.fixedParentAssertion) return s.installation.fixedParentAssertion; @@ -1985,24 +2269,127 @@ export function cancelControlExecution( lifecycle.sequence++; writeLifecycle(s.installation, dir, lifecycle); } - return Object.freeze( - Object.keys(lifecycle.childIntents).map((id) => { - const saved = z - .object({ grant: z.unknown() }) - .parse(readPrivateRecord(s.installation, "parent-attempts", `${id}.json`)); - return Object.freeze( - verifyControl( - s.installation.installation.publicKey, - originRecordDomain("openmapx-parent-grant", saved.grant), - saved.grant, - parentGrantSchema, - ).child, + const children = Object.keys(lifecycle.childIntents).map((id) => { + const saved = z + .object({ grant: z.unknown() }) + .parse(readPrivateRecord(s.installation, "parent-attempts", `${id}.json`)); + return Object.freeze( + verifyControl( + s.installation.installation.publicKey, + originRecordDomain("openmapx-parent-grant", saved.grant), + saved.grant, + parentGrantSchema, + ).child, + ); + }); + for (const [id, intent] of Object.entries(lifecycle.leaseIntents ?? {})) { + // The signed index is published before its private outbox. A crash in + // that gap retains this exact identity for cancellation/lookup. + let saved: unknown; + try { + saved = readPrivateRecord(s.installation, "parent-attempts", `${id}.json`, 8192); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (saved !== undefined && saved !== null) { + const parsed = z + .strictObject({ kind: z.literal("lease"), attempt: z.unknown() }) + .parse(saved); + const value = verifyControl( + s.installation.installation.publicKey, + "openmapx-lease-execution-attempt-v2", + parsed.attempt, + leaseExecutionAttemptSchema, ); - }), - ); + if ( + value.installationId !== s.installation.installation.installationId || + controlDigest(value.origin) !== controlDigest(s.origin) || + controlDigest(value.parent) !== controlDigest(s.execution) || + controlDigest(parsed.attempt) !== intent.attemptDigest || + controlDigest(value.child) !== controlDigest(intent.child) || + value.leaseId !== intent.leaseId || + value.grantDigest !== intent.grantDigest + ) + throw Error("Retained lease cancellation intent changed"); + } + children.push(Object.freeze({ ...intent.child })); + } + for (const reserved of s.reservations) { + const saved = inspectControlChildPublication(authority, reserved); + if (saved.publication.state === "grant_prepared" && saved.binding) + children.push(Object.freeze({ ...saved.binding.child.execution })); + } + const distinct = new Map(); + for (const child of children) { + const prior = distinct.get(child.operationKey); + if (prior && controlDigest(prior) !== controlDigest(child)) + throw Error("Cancellation child identity disagreement"); + distinct.set(child.operationKey, child); + } + return Object.freeze([...distinct.values()]); }); } -function settleExecutionLifecycle(s: AuthorityState) { +const leaseExecutionAttemptSchema = z.strictObject({ + version: z.literal(2), + installationId: uuid, + origin: controlOriginSchema, + parent: executionRefSchema, + child: executionRefSchema, + leaseId: uuid, + delegationId: uuid, + grantDigest: digest, +}); +/** Fixed lease issuer only; exact identities become cancellation-visible before + * outbox/shared exposure, including a crash before the parent node update. */ +export function recordLeaseControlAttempt( + authority: VerifiedExecutionAuthority, + input: { child: ExecutionRef; leaseId: string; delegationId: string; grantDigest: string }, +): void { + const s = executionAuthorityState(authority), + directory = assertControlPublicationScope(authority); + if (!s.origin) throw Error("Lease attempt origin is missing"); + const lifecycle = readLifecycle( + s.installation, + directory, + s.execution, + s.installation.installation.publicKey, + ); + const leases = lifecycle.leaseIntents ?? {}; + if ( + Object.keys(lifecycle.childIntents).length + Object.keys(leases).length >= 64 || + leases[input.delegationId] + ) + throw Error("Execution cancellation intent capacity/replay"); + const attempt = signControl( + s.installation.privateKey, + "openmapx-lease-execution-attempt-v2", + leaseExecutionAttemptSchema.parse({ + version: 2, + installationId: s.installation.installation.installationId, + origin: s.origin, + parent: s.execution, + ...input, + }), + ); + leases[input.delegationId] = { + child: input.child, + leaseId: input.leaseId, + grantDigest: input.grantDigest, + attemptDigest: controlDigest(attempt), + }; + lifecycle.leaseIntents = leases; + lifecycle.sequence++; + writeLifecycle(s.installation, directory, lifecycle); + writePrivateRecord( + s.installation, + "parent-attempts", + `${input.delegationId}.json`, + { kind: "lease", attempt }, + true, + s.origin, + ); +} +export function settleExecutionLifecycle(s: AuthorityState) { withExecutionWriter(s.installation, s.execution, (dir) => { const lifecycle = readLifecycle( s.installation, @@ -2029,12 +2416,7 @@ export async function attestParentExecution( ): Promise { const s = executionAuthorityState(authority); if (Date.now() >= s.execution.deadlineAt) throw Error("Expired parent execution"); - const plan = s.plan.find( - (p) => - controlDigest(p.execution) === controlDigest(child.execution) && - p.phase === child.phase && - p.recipient === child.recipient, - ); + const plan = resolveControlChildPlan(authority, child); if (!plan || !childPhaseKinds[child.phase].includes(plan.kind)) throw Error("Child absent from retained plan"); try { @@ -2097,71 +2479,69 @@ export async function attestParentExecution( }), ).toString("base64url"); parseParentAssertion(assertion); - const issue = () => - withExecutionWriter(s.installation, s.execution, (dir) => { - const lifecycle = readLifecycle( - s.installation, - dir, - s.execution, + return withControlDelegationPublication(authority, child.phase, (dir) => { + const lifecycle = readLifecycle( + s.installation, + dir, + s.execution, + s.installation.installation.publicKey, + ); + if (lifecycle.phase !== "held" || Date.now() >= s.execution.deadlineAt) + throw Error("Canceled/terminal execution cannot authorize forward children"); + const name = `${s.execution.operationKey}.children.json`; + let witness: Record = {}; + try { + witness = z + .record(uuid, z.unknown()) + .parse(JSON.parse(dir.read(name, 65536).toString("utf8"))); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + } + for (const value of Object.values(witness)) { + const previous = verifyControl( s.installation.installation.publicKey, + originRecordDomain("openmapx-parent-grant", value), + value, + parentGrantSchema, ); - if (lifecycle.phase !== "held" || Date.now() >= s.execution.deadlineAt) - throw Error("Canceled/terminal execution cannot authorize forward children"); - const name = `${s.execution.operationKey}.children.json`; - let witness: Record = {}; - try { - witness = z - .record(uuid, z.unknown()) - .parse(JSON.parse(dir.read(name, 65536).toString("utf8"))); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - for (const value of Object.values(witness)) { - const previous = verifyControl( - s.installation.installation.publicKey, - originRecordDomain("openmapx-parent-grant", value), - value, - parentGrantSchema, - ); - if (previous.child.operationKey === child.execution.operationKey) - throw Error("Attempted parent child cannot be replayed"); - } - if (Object.keys(witness).length >= 64) throw Error("Parent attempt capacity"); - witness[attemptId] = signed; - const bytes = canonicalJson(witness); - if (Buffer.byteLength(bytes) > 65536) throw Error("Parent witness capacity"); - if (!s.origin) throw Error("Legacy origin is observation-only"); - enlistFamilyIntent( - s.installation, - s.origin, - attemptId, - child.execution, - child.recipient === "data-manager" ? "data-manager" : "ops-agent", - ); - writePrivateRecord( - s.installation, - "parent-attempts", - `${attemptId}.json`, - { grant: signed, assertion }, - true, - s.origin, - ); - lifecycle.childIntents[attemptId] = fields.grantDigest; - lifecycle.sequence++; - writeLifecycle(s.installation, dir, lifecycle); - writeExecutionShared(s.installation, s.execution, dir, name, bytes); - return assertion; + if (previous.child.operationKey === child.execution.operationKey) + throw Error("Attempted parent child cannot be replayed"); + } + if (Object.keys(witness).length + Object.keys(lifecycle.leaseIntents ?? {}).length >= 64) + throw Error("Parent attempt capacity"); + witness[attemptId] = signed; + const bytes = canonicalJson(witness); + if (Buffer.byteLength(bytes) > 65536) throw Error("Parent witness capacity"); + if (!s.origin) throw Error("Legacy origin is observation-only"); + enlistFamilyIntent( + s.installation, + s.origin, + attemptId, + child.execution, + child.recipient === "data-manager" ? "data-manager" : "ops-agent", + ); + prepareControlChildPublication(authority, plan, { + delegationId: attemptId, + grantDigest: fields.grantDigest, + leaseId: null, }); - const inherited = retainedParentAssertion(s); - return inherited - ? withParentLifecycles(s.installation, inherited, (forward) => { - if (!forward) throw Error("Canceled ancestor prevents forward delegation"); - return issue(); - }) - : issue(); + writePrivateRecord( + s.installation, + "parent-attempts", + `${attemptId}.json`, + { grant: signed, assertion }, + true, + s.origin, + ); + lifecycle.childIntents[attemptId] = fields.grantDigest; + lifecycle.sequence++; + writeLifecycle(s.installation, dir, lifecycle); + writeExecutionShared(s.installation, s.execution, dir, name, bytes); + return assertion; + }); } -function readParentGrant( +export function readParentGrant( state: InstallationState, assertion: string, depth = 0, @@ -2224,7 +2604,7 @@ function readParentGrant( verifyDeadlineEdge(parent.kind, grant.phase, grant.recipient, grant.kind); return grant; } -function verifyDeadlineEdge( +export function verifyDeadlineEdge( parentKind: string, phase: ControlChildPhase, recipient: ControlRecipient, @@ -2235,6 +2615,9 @@ function verifyDeadlineEdge( "data.downloadFonts": "download-fonts", "data.convertOverpass": "convert-overpass", "data.overtureSync": "overture-sync", + "data.overturePull": "overture-pull", + "data.overtureIngest": "overture-ingest", + "data.overtureExtract": "overture-extract", "data.overtureConflate": "overture-conflate", "data.searchIndexBuild": "search-index-build", "data.notablePlacesBuild": "notable-places-build", @@ -2412,15 +2795,22 @@ export function retainFixedCliSpawnIntent( operation: unknown; args: readonly string[]; plan?: readonly ControlPlannedChild[]; + reservations?: readonly ControlChildReservation[]; }, ): void { const s = executionAuthorityState(authority); const id = uuid.parse(input.delegationId); const operation = opsOperationSchema.parse(input.operation); - const plan = s.plan.find( - (p) => - controlDigest(p.execution) === controlDigest(input.execution) && p.recipient === "fixed-cli", + const slot = [...s.plan, ...s.reservations].find( + (p) => p.execution.executionId === input.execution.executionId && p.recipient === "fixed-cli", ); + const plan = slot + ? resolveControlChildPlan(authority, { + phase: slot.phase, + execution: input.execution, + recipient: "fixed-cli", + }) + : undefined; if ( !plan || opsOperationFingerprint(operation) !== input.execution.operationFingerprint || @@ -2442,6 +2832,11 @@ export function retainFixedCliSpawnIntent( .array(plannedChildSchema) .max(64) .parse(input.plan ?? []), + ...(input.reservations === undefined + ? {} + : { + reservations: z.array(controlChildReservationSchema).max(64).parse(input.reservations), + }), }, true, s.origin, @@ -2513,6 +2908,7 @@ export function readFixedCliSpawnIntent( const saved = z .strictObject({ plan: z.array(plannedChildSchema).max(64), + reservations: z.array(controlChildReservationSchema).max(64).optional(), execution: executionRefSchema, operation: opsOperationSchema, args: z.array(z.string().max(4096)).max(64), @@ -2652,6 +3048,7 @@ export function restoreAdmittedControlExecution( serviceIds: targetSchema, origin: controlOriginSchema.optional(), plan: z.array(plannedChildSchema).max(64), + reservations: z.array(controlChildReservationSchema).max(64).optional(), }) .passthrough(), ); @@ -2668,6 +3065,7 @@ export function restoreAdmittedControlExecution( kind: payload.kind, serviceIds: payload.serviceIds, plan: payload.plan, + reservations: payload.reservations ?? [], receiptDigest: controlDigest(raw), origin: payload.origin, }); diff --git a/packages/core/src/build-artifacts/control-records.ts b/packages/core/src/build-artifacts/control-records.ts index b861d595d..9e5600547 100644 --- a/packages/core/src/build-artifacts/control-records.ts +++ b/packages/core/src/build-artifacts/control-records.ts @@ -23,9 +23,10 @@ export interface InstallationState { dataRoot: string; registeredTargets: ReadonlySet; retentionJournal?: { + prepare?(executions: readonly { execution: ExecutionRef; digest?: string }[]): Promise; contained(execution: ExecutionRef, digest: string | undefined): boolean; retire( - executions: readonly { execution: ExecutionRef; digest: string }[], + executions: readonly { execution: ExecutionRef; digest: string; terminalAt?: number }[], manifestDigest: string, ): Promise; }; @@ -104,3 +105,12 @@ export function verifyControl( throw Error("Invalid control signature"); return e.payload; } + +export const targetSchema = z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine( + (x) => x.every((v, i) => i === 0 || (x[i - 1] ?? v) < v), + "Targets must be sorted and unique", + ); diff --git a/packages/core/src/build-artifacts/control-retention.ts b/packages/core/src/build-artifacts/control-retention.ts index 71dda2fdf..18b0aad39 100644 --- a/packages/core/src/build-artifacts/control-retention.ts +++ b/packages/core/src/build-artifacts/control-retention.ts @@ -137,9 +137,10 @@ export function writeRetainedRecord( exclusive: boolean, maximum: number = CONTROL_RETENTION_LIMITS.recordsPerNamespace, beforePublish?: (inode: FileIdentity) => void, + maximumBytes: number = CONTROL_RETENTION_LIMITS.privateRecordBytes, ) { try { - dir.read(name, CONTROL_RETENTION_LIMITS.privateRecordBytes); + dir.read(name, maximumBytes); dir.write(name, bytes, exclusive, beforePublish); return; } catch (e) { @@ -176,6 +177,7 @@ export function deleteRetainedRecord( dir: AnchoredDirectory, item: RetentionDeletion, itemDone: () => void, + maximumBytes: number = CONTROL_RETENTION_LIMITS.privateRecordBytes, ): void { withRetentionNamespaceWriter(dir, () => { let t = transaction(dir); @@ -195,9 +197,7 @@ export function deleteRetainedRecord( // A new plan cannot turn an old continuation into a second decrement. if ( !sameInode(dir.fileIdentity(item.name), item.inode) || - controlDigest( - dir.read(item.name, CONTROL_RETENTION_LIMITS.privateRecordBytes).toString("utf8"), - ) !== item.digest + controlDigest(dir.read(item.name, maximumBytes).toString("utf8")) !== item.digest ) throw Error("Retirement record inode/digest replacement"); const before = readCounter(dir); @@ -216,9 +216,7 @@ export function deleteRetainedRecord( try { if ( !sameInode(dir.fileIdentity(item.name), item.inode) || - controlDigest( - dir.read(item.name, CONTROL_RETENTION_LIMITS.privateRecordBytes).toString("utf8"), - ) !== item.digest + controlDigest(dir.read(item.name, maximumBytes).toString("utf8")) !== item.digest ) throw Error("Retirement record inode/digest replacement"); dir.removeFile(item.name); diff --git a/packages/core/src/build-artifacts/data-configuration-fence.ts b/packages/core/src/build-artifacts/data-configuration-fence.ts new file mode 100644 index 000000000..cfde74944 --- /dev/null +++ b/packages/core/src/build-artifacts/data-configuration-fence.ts @@ -0,0 +1,935 @@ +/** Fixed service-only configuration/feed guard; signatures are verified against + * the anchored original agent grant, never against the shared bearer alone. */ +import { z } from "zod"; +import { executionFingerprint, executionRefSchema } from "../ops/execution-identity"; +import { canonicalJson } from "../ops/fingerprint"; +import { + type DataConfigurationFencePrepare, + type DataConfigurationFenceReceipt, + type DataConfigurationFenceSettle, + dataConfigurationFencePrepareSchema, + dataConfigurationFenceReceiptSchema, + dataConfigurationFenceSettleSchema, +} from "../services/data-manager/contracts/configuration-maintenance"; +import { startGenerationHardlinksExecutionSchema } from "../services/data-manager/contracts/executions"; +import { + beginControlEffect, + type ControlInstallation, + controlDigest, + controlHeaders, + executionAuthorityState, + installationState, + readPrivateRecord, + requireControlJournalRetention, + restoreAdmittedControlExecution, + retainAdmittedControlExecution, + retainedControlParent, + settleExecutionLifecycle, + verifyControlEffectSettlement, + verifyControlPeer, + withPrivateEffectWriter, + writePrivateRecord, +} from "./control-authority"; +import { controlOriginSchema, signControl, verifyControl } from "./control-records"; +import { retainedDataDatabaseTimestamp } from "./data-database-time"; +import { verifiedParentForArtifactDelegation } from "./peer-admission"; +import { + completeFamilyJournal, + markFamilyTerminal, + publishFamilyCanonicalTerminal, + verifyFamilyOwnedRecord, +} from "./retention-family"; +import { + adoptArtifactTargetLease, + assertArtifactTargetLeaseForward, + bindBorrowedControlExecution, + closeArtifactTargetLeaseHandle, + inspectArtifactDelegation, + settleArtifactTargetLeaseChild, + verifyRetainedArtifactTargetSettlement, +} from "./target-lease"; +export interface DataFenceSql { + unsafe( + query: string, + parameters?: (string | number | null)[], + ): Promise[]>; + transaction(run: (tx: Pick) => Promise): Promise; +} +const metadataSchema = z.strictObject({ + version: z.literal(1), + guardState: z.enum(["preparing", "prepared", "settling", "settled", "recovery_required"]), + receipt: dataConfigurationFenceReceiptSchema, + input: dataConfigurationFencePrepareSchema, + settlement: dataConfigurationFenceSettleSchema.optional(), +}); +const mirrorSchema = z.strictObject({ + preparedAt: z.number().int().positive().safe(), + ownerCertificateDigest: z.string().regex(/^[a-f0-9]{64}$/), + prepareAssertion: z.string().max(4096), + version: z.literal(1), + installationId: z.uuid(), + origin: controlOriginSchema, + metadata: metadataSchema, +}); +const fenceTerminalSchema = z.strictObject({ + version: z.literal(1), + phase: z.enum(["prepared", "canonical"]), + execution: executionRefSchema, + admissionDigest: z.string().regex(/^[a-f0-9]{64}$/), + terminalDigest: z.string().regex(/^[a-f0-9]{64}$/), + terminalAt: z.number().int().positive().safe(), +}); +const inspectWitnessSchema = z.strictObject({ + installationId: z.uuid(), + maintenanceId: z.uuid(), + candidateId: z.uuid(), + ownerCertificateDigest: z.string().regex(/^[a-f0-9]{64}$/), + issuedAt: z.number().int().nonnegative().safe(), + expiresAt: z.number().int().nonnegative().safe(), +}); +export class DataFenceBusy extends Error { + readonly statusCode = 409; + constructor() { + super("configuration_maintenance_busy"); + } +} +export function createDataConfigurationFenceStore( + installation: ControlInstallation, + sql: DataFenceSql, +) { + const local = installationState(installation); + if (installation.role !== "data-manager" || local.fixedChildId) + throw Error("Fixed DM fence receiver required"); + let startup = "closed" as "closed" | "open" | "fenced"; + const mirrors = new Map>(); + function mirror(id: string) { + const raw = readPrivateRecord( + local, + "configuration-maintenance", + `${z.uuid().parse(id)}.json`, + 65536, + ), + parsed = verifyControl( + installation.publicKey, + "openmapx-data-configuration-fence-v1", + raw, + mirrorSchema, + ); + verifyFamilyOwnedRecord( + local, + parsed.origin, + { + scope: "private", + namespace: "configuration-maintenance", + name: `${id}.json`, + container: "", + }, + canonicalJson(raw), + ); + if ( + parsed.installationId !== installation.installationId || + parsed.metadata.receipt.maintenanceId !== id + ) + throw Error("Configuration guard mirror identity changed"); + return parsed; + } + function write(value: z.infer) { + const parsed = mirrorSchema.parse(value); + writePrivateRecord( + local, + "configuration-maintenance", + `${parsed.metadata.receipt.maintenanceId}.json`, + signControl(local.privateKey, "openmapx-data-configuration-fence-v1", parsed), + false, + parsed.origin, + ); + mirrors.set(parsed.metadata.receipt.maintenanceId, parsed); + } + async function lock(tx: Pick) { + const rows = await tx.unsafe( + "SELECT pg_try_advisory_xact_lock($1::int,$2::int) AS held", + [0x4f4d58, 2], + ); + if (rows[0]?.held !== true) throw new DataFenceBusy(); + } + async function row(tx: Pick, id: string) { + const rows = await tx.unsafe( + "SELECT id,metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND metadata->'configurationMaintenance'->'receipt'->>'maintenanceId'=$1", + [id], + ); + if (rows.length > 1) throw Error("Duplicate configuration fence"); + if (!rows[0]) return null; + return metadataSchema.parse( + z.object({ configurationMaintenance: metadataSchema }).parse(rows[0].metadata) + .configurationMaintenance, + ); + } + function inspectGrant( + body: DataConfigurationFencePrepare | DataConfigurationFenceSettle, + rawHeaders: readonly string[], + kind: "configuration-maintenance.prepare" | "configuration-maintenance.settle", + owner?: DataConfigurationFencePrepare, + ) { + const headers = controlHeaders(rawHeaders); + if (!headers.delegation || headers.parent) throw Error("Pinned agent fence grant required"); + const grant = inspectArtifactDelegation({ + dataRoot: local.dataRoot, + assertion: headers.delegation, + installation, + }); + const operation = { kind, ...body }, + parent = "ownerPrepare" in body ? body.ownerPrepare : owner?.ownerPrepare; + if ( + grant.recipient !== "data-manager" || + grant.parent.executor !== "ops-agent" || + grant.kind !== kind || + grant.phase !== + (kind.endsWith(".prepare") ? "configuration-fence" : "configuration-fence-settle") || + !parent || + controlDigest(grant.parent) !== controlDigest(parent) || + grant.execution.operationFingerprint !== controlDigest(operation) || + grant.execution.executor !== "data-manager" || + grant.execution.fingerprint !== + executionFingerprint({ + kind, + operationFingerprint: controlDigest(operation), + deadlineAt: grant.execution.deadlineAt, + parent: retainedControlParent(grant.parent), + }) + ) + throw Error("Configuration fence grant identity conflict"); + if ("serviceIds" in body && canonicalJson(grant.serviceIds) !== canonicalJson(body.serviceIds)) + throw Error("Configuration fence target subset conflict"); + const peer = verifyControlPeer(installation, { + rawHeaders, + transportPrincipal: "shared-data-manager-bearer", + execution: grant.execution, + kind, + }); + return { + grant, + peer, + operation, + parent: verifiedParentForArtifactDelegation( + installation, + headers.delegation, + grant.execution, + ), + assertion: headers.delegation, + }; + } + async function withLeaf( + verified: ReturnType, + run: ( + authority: import("./control-authority").VerifiedExecutionAuthority, + ) => Promise, + ) { + const { grant, peer } = verified; + const lease = await adoptArtifactTargetLease({ + dataRoot: local.dataRoot, + assertion: verified.assertion, + peer, + execution: grant.execution, + serviceIds: grant.serviceIds, + }); + try { + const admittedAt = Date.now(); + let authority: ReturnType | undefined; + try { + authority = restoreAdmittedControlExecution(installation, grant.execution); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + authority ??= retainAdmittedControlExecution(installation, { + execution: grant.execution, + kind: grant.kind, + operation: verified.operation, + admittedAt, + requestedDeadlineAt: grant.execution.deadlineAt, + parent: verified.parent, + serviceIds: grant.serviceIds, + }); + bindBorrowedControlExecution(lease, authority); + let priorEffect: unknown; + try { + priorEffect = readPrivateRecord( + local, + "effects", + `${grant.execution.operationKey}.json`, + 65536, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (priorEffect !== undefined && priorEffect !== null) { + verifyRetainedArtifactTargetSettlement( + installation, + { leaseId: grant.leaseId, delegationId: grant.delegationId, phase: "ready" }, + grant.execution, + ); + const kind = z + .object({ payload: z.object({ receipt: z.object({ kind: z.string() }) }) }) + .parse(priorEffect).payload.receipt.kind; + if (kind === "data-configuration-fence-refused") { + verifyControlEffectSettlement(installation, grant.execution); + throw new DataFenceBusy(); + } + const original = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + priorEffect, + z + .object({ + receipt: z + .object({ + kind: z.literal("data-configuration-fence"), + receipt: dataConfigurationFenceReceiptSchema, + }) + .passthrough(), + }) + .passthrough(), + ); + const saved = await row(sql, original.receipt.receipt.maintenanceId); + if (!saved) throw new DataFenceBusy(); + return saved.receipt; + } + if (grant.kind === "configuration-maintenance.prepare") + requireControlJournalRetention(authority); + if (grant.kind === "configuration-maintenance.prepare") + assertArtifactTargetLeaseForward(lease); + beginControlEffect(authority); + let receipt: DataConfigurationFenceReceipt; + try { + receipt = await run(authority); + } catch (error) { + // Before any preparing mirror exists the fixed guard path has made + // no logical write. A busy result can settle this exact unused leaf. + if (error instanceof DataFenceBusy) { + let absent = false; + try { + readPrivateRecord( + local, + "configuration-maintenance", + `${z.object({ maintenanceId: z.uuid() }).parse(verified.operation).maintenanceId}.json`, + 65536, + ); + } catch (missing) { + absent = (missing as NodeJS.ErrnoException).code === "ENOENT"; + } + if ( + absent && + !(await row( + sql, + z.object({ maintenanceId: z.uuid() }).parse(verified.operation).maintenanceId, + )) + ) { + const state = executionAuthorityState(authority), + observation = { + kind: "data-configuration-fence-refused" as const, + maintenanceId: z.object({ maintenanceId: z.uuid() }).parse(verified.operation) + .maintenanceId, + reason: "busy" as const, + }, + effect = { + identity: grant.execution, + outcome: "failed_without_effect" as const, + publication: "not_started" as const, + terminalReceiptDigest: controlDigest(observation), + receipt: observation, + }; + const terminalAt = Date.now(), + terminalDigest = controlDigest({ version: 1, refused: observation }); + withPrivateEffectWriter(state, () => { + writePrivateRecord( + local, + "observations", + `terminal-${grant.execution.operationKey}.json`, + signControl( + local.privateKey, + "openmapx-data-fence-terminal-v1", + fenceTerminalSchema.parse({ + version: 1, + phase: "canonical", + execution: grant.execution, + admissionDigest: state.receiptDigest, + terminalDigest, + terminalAt, + }), + ), + false, + state.origin, + ); + settleExecutionLifecycle(state); + writePrivateRecord( + local, + "effects", + `${grant.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-control-settlement-v1", effect), + false, + state.origin, + ); + if (state.origin) + markFamilyTerminal( + local, + state.origin, + grant.execution, + true, + controlDigest(effect), + ); + }); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, grant.execution), + ); + if (state.origin) { + publishFamilyCanonicalTerminal( + local, + state.origin, + grant.execution, + true, + terminalDigest, + terminalAt, + ); + completeFamilyJournal(local, state.origin, grant.execution, terminalAt); + } + } + } + throw error; + } + const state = executionAuthorityState(authority), + stored = mirror(receipt.maintenanceId); + if (controlDigest(stored.metadata.receipt) !== controlDigest(receipt)) + throw Error("Durable fence receipt changed before settlement"); + withPrivateEffectWriter(state, () => { + const proof = { + kind: "data-configuration-fence" as const, + receipt, + guardDigest: controlDigest(stored), + }, + effect = { + identity: grant.execution, + outcome: "committed" as const, + publication: "committed" as const, + terminalReceiptDigest: controlDigest(proof), + receipt: proof, + }; + settleExecutionLifecycle(state); + writePrivateRecord( + local, + "effects", + `${grant.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-control-settlement-v1", effect), + false, + state.origin, + ); + if (state.origin) + markFamilyTerminal(local, state.origin, grant.execution, true, controlDigest(effect)); + }); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, grant.execution), + ); + if (grant.kind === "configuration-maintenance.prepare" && state.origin) { + const digest = dataFenceTerminalDigest(receipt), + original = verifyControl( + installation.publicKey, + "openmapx-data-fence-terminal-v1", + readPrivateRecord( + local, + "observations", + `terminal-${grant.execution.operationKey}.json`, + 8192, + ), + fenceTerminalSchema, + ); + if ( + original.terminalDigest !== digest || + original.terminalAt !== stored.preparedAt || + original.admissionDigest !== state.receiptDigest + ) + throw Error("Original logical guard terminal commitment changed"); + writePrivateRecord( + local, + "observations", + `terminal-${grant.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-data-fence-terminal-v1", { + ...original, + phase: "canonical", + }), + false, + state.origin, + ); + publishFamilyCanonicalTerminal( + local, + state.origin, + grant.execution, + true, + digest, + stored.preparedAt, + ); + completeFamilyJournal(local, state.origin, grant.execution, stored.preparedAt); + } + return receipt; + } finally { + closeArtifactTargetLeaseHandle(lease); + } + } + async function assertAdmission(tx: Pick) { + if ( + startup === "closed" || + [...mirrors.values()].some((value) => value.metadata.guardState !== "settled") + ) + throw new DataFenceBusy(); + const rows = await tx.unsafe( + "SELECT id FROM data_manager.jobs WHERE kind='configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled' LIMIT 1", + ); + if (rows.length) throw new DataFenceBusy(); + const unknown = await tx.unsafe( + "SELECT id FROM data_manager.jobs WHERE kind <> 'configuration-maintenance' AND (execution_state->>'publication'='unknown' OR (execution_version IS NULL AND finished_at IS NULL AND status='interrupted')) LIMIT 1", + ); + if (unknown.length) throw new DataFenceBusy(); + } + return { + async startup() { + startup = "closed"; + mirrors.clear(); + let directory: ReturnType | undefined; + try { + directory = local.privateRoot.directory("configuration-maintenance"); + for (const file of directory.listBounded(4097)) { + if ([".retention-count", ".retention-highwater", ".retention-transaction"].includes(file)) + continue; + if (!/^[a-f0-9-]{36}\.json$/.test(file)) + throw Error("Unexpected configuration fence file"); + const value = mirror(file.slice(0, -5)); + mirrors.set(value.metadata.receipt.maintenanceId, value); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } finally { + directory?.close(); + } + const rows = await sql.unsafe( + "SELECT metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled' LIMIT 2", + ); + if (rows.length > 1) throw Error("Conflicting configuration guards"); + for (const raw of rows) { + const value = z + .object({ configurationMaintenance: metadataSchema }) + .parse(raw.metadata).configurationMaintenance, + saved = mirrors.get(value.receipt.maintenanceId); + if (!saved || canonicalJson(saved.metadata) !== canonicalJson(value)) + throw Error("Configuration guard mirror/database recovery required"); + } + for (const value of mirrors.values()) + if (value.metadata.guardState !== "settled") { + const saved = await row(sql, value.metadata.receipt.maintenanceId); + if (!saved || canonicalJson(saved) !== canonicalJson(value.metadata)) + throw Error("Half-written configuration guard retains admission fence"); + } + startup = rows.length ? "fenced" : "open"; + }, + assertAdmission, + async assertGenerationAdmission(value: unknown, rawHeaders: readonly string[]): Promise { + const request = startGenerationHardlinksExecutionSchema.parse(value), + headers = controlHeaders(rawHeaders); + if (!headers.delegation || headers.parent) + throw Object.assign(Error("Signed generation grant required"), { statusCode: 403 }); + const grant = inspectArtifactDelegation({ + dataRoot: local.dataRoot, + assertion: headers.delegation, + installation, + }); + if ( + grant.recipient !== "data-manager" || + grant.kind !== "generation-hardlinks" || + grant.phase !== "hardlinks" || + grant.execution.operationKey !== request.operationKey || + grant.execution.operationFingerprint !== controlDigest(request.input) || + request.input.planDigest !== controlDigest(request.plan) || + request.plan.plan.some((entry) => !grant.serviceIds.includes(entry.consumerService)) + ) + throw Object.assign(Error("Generation plan/grant binding conflict"), { statusCode: 403 }); + if (startup === "closed") throw new DataFenceBusy(); + const rows = await sql.unsafe( + "SELECT metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled' LIMIT 2", + ); + if (rows.length > 1) throw new DataFenceBusy(); + const saved = rows[0] + ? z.object({ configurationMaintenance: metadataSchema }).parse(rows[0].metadata) + .configurationMaintenance + : undefined; + if (!saved) { + if ([...mirrors.values()].some((mirror) => mirror.metadata.guardState !== "settled")) + throw new DataFenceBusy(); + return; + } + const owned = mirror(saved.receipt.maintenanceId); + if (canonicalJson(owned.metadata) !== canonicalJson(saved)) throw new DataFenceBusy(); + if ( + grant.ownerCertificateDigest !== owned.ownerCertificateDigest || + canonicalJson(grant.owner) !== canonicalJson(saved.receipt.ownerPrepare) || + grant.owner.executor !== "ops-agent" || + request.input.revisionId !== saved.receipt.revisionId || + request.input.planDigest !== saved.receipt.hardlinkPlanDigest || + request.input.coordinatorMode !== saved.input.coordinatorMode || + grant.serviceIds.some((id) => !saved.input.serviceIds.includes(id)) + ) + throw Object.assign(Error("Generation maintenance owner/revision/hash conflict"), { + statusCode: 403, + }); + //01.2 owns the real frozen-step/DB-decision witness and positive + // continuation. A matching prepared receipt alone authorizes no write. + throw new DataFenceBusy(); + }, + async withAdmission(run: (tx: Pick) => Promise): Promise { + return sql.transaction(async (tx) => { + await lock(tx); + await assertAdmission(tx); + return run(tx); + }); + }, + async prepare(value: DataConfigurationFencePrepare, rawHeaders: readonly string[]) { + const input = dataConfigurationFencePrepareSchema.parse(value), + verified = inspectGrant(input, rawHeaders, "configuration-maintenance.prepare"); + const existing = await row(sql, input.maintenanceId); + if (existing) { + if ( + canonicalJson(existing.input) !== canonicalJson(input) || + existing.receipt.fenceId !== verified.grant.execution.executionId + ) + throw Object.assign(Error("Configuration fence conflict"), { statusCode: 409 }); + return withLeaf(verified, async () => existing.receipt); + } + if (startup === "closed") throw new DataFenceBusy(); + const receipt = dataConfigurationFenceReceiptSchema.parse({ + fenceId: verified.grant.execution.executionId, + maintenanceId: input.maintenanceId, + candidateId: input.candidateId, + ownerPrepare: input.ownerPrepare, + planDigest: input.planDigest, + revisionId: input.revisionId, + hardlinkPlanDigest: input.hardlinkPlanDigest, + sequence: 0, + state: "prepared", + }); + const metadata = metadataSchema.parse({ + version: 1, + guardState: "preparing", + receipt, + input, + }); + if (!verified.grant.origin) throw Error("Dated configuration guard required"); + const owned = { + preparedAt: Date.now(), + version: 1 as const, + installationId: installation.installationId, + origin: verified.grant.origin, + ownerCertificateDigest: verified.grant.ownerCertificateDigest, + prepareAssertion: verified.assertion, + metadata, + }; + return withLeaf(verified, async (authority) => { + await sql.transaction(async (tx) => { + await lock(tx); + await assertAdmission(tx); + const active = await tx.unsafe( + "SELECT id FROM data_manager.jobs WHERE kind <> 'configuration-maintenance' AND (finished_at IS NULL OR execution_state->>'publication'='unknown') LIMIT 1", + ); + const transfers = await tx.unsafe( + "SELECT id FROM data_manager.offline_package_jobs WHERE status='preparing' LIMIT 1", + ); + if (active.length || transfers.length) throw new DataFenceBusy(); + write(owned); + const state = executionAuthorityState(authority), + terminal = fenceTerminalSchema.parse({ + version: 1, + phase: "prepared", + execution: verified.grant.execution, + admissionDigest: state.receiptDigest, + terminalDigest: dataFenceTerminalDigest(receipt), + terminalAt: owned.preparedAt, + }); + writePrivateRecord( + local, + "observations", + `terminal-${verified.grant.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-data-fence-terminal-v1", terminal), + false, + state.origin, + ); + await tx.unsafe( + "INSERT INTO data_manager.jobs(id,kind,status,started_at,finished_at,operation_key,metadata) VALUES($1::uuid,'configuration-maintenance','ok',$4::text::timestamptz,$4::text::timestamptz,$2,$3::text::jsonb)", + [ + receipt.fenceId, + verified.grant.execution.operationKey, + canonicalJson({ configurationMaintenance: metadata }), + new Date(owned.preparedAt).toISOString(), + ], + ); + }); + metadata.guardState = "prepared"; + write(owned); + await sql.transaction(async (tx) => { + await lock(tx); + const current = await row(tx, input.maintenanceId); + if (!current || canonicalJson(current.receipt) !== canonicalJson(receipt)) + throw Error("Configuration fence preparation CAS conflict"); + await tx.unsafe( + "UPDATE data_manager.jobs SET metadata=$2::text::jsonb,ownership_version=ownership_version+1 WHERE id=$1::uuid", + [receipt.fenceId, canonicalJson({ configurationMaintenance: metadata })], + ); + }); + startup = "fenced"; + return receipt; + }); + }, + async inspect(maintenanceId: string, rawHeaders: readonly string[]) { + const header = rawHeaders.filter( + (_, index) => + index % 2 === 0 && + rawHeaders[index]?.toLowerCase() === "x-openmapx-configuration-fence-witness", + ); + if (header.length !== 1) throw Error("Purpose-bound agent fence witness required"); + const index = rawHeaders.findIndex( + (name, i) => + i % 2 === 0 && name.toLowerCase() === "x-openmapx-configuration-fence-witness", + ), + encoded = rawHeaders[index + 1]; + if (!encoded || encoded.length > 4096 || !/^[A-Za-z0-9_-]+$/.test(encoded)) + throw Error("Invalid fence witness"); + const witness = verifyControl( + local.publicKeys["ops-agent"], + "openmapx-configuration-fence-inspect-v1", + JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")), + inspectWitnessSchema, + ), + now = Date.now(); + if ( + witness.installationId !== installation.installationId || + witness.maintenanceId !== maintenanceId || + witness.issuedAt > now || + witness.expiresAt <= now || + witness.expiresAt - witness.issuedAt > 15000 + ) + throw Error("Expired or foreign fence witness"); + const saved = await row(sql, maintenanceId); + if (!saved) return null; + const proof = mirror(maintenanceId); + if ( + saved.receipt.candidateId !== witness.candidateId || + proof.ownerCertificateDigest !== witness.ownerCertificateDigest || + canonicalJson(proof.metadata) !== canonicalJson(saved) + ) + throw Error("Configuration guard recovery required"); + const anchored = inspectArtifactDelegation({ + dataRoot: local.dataRoot, + assertion: proof.prepareAssertion, + installation, + }); + if ( + anchored.ownerCertificateDigest !== witness.ownerCertificateDigest || + controlDigest(anchored.parent) !== controlDigest(saved.receipt.ownerPrepare) + ) + throw Error("Original owner certificate changed"); + // Verify its still-anchored original owner certificate without granting a mutation. + const owned = mirrors.get(maintenanceId); + if ( + !owned || + controlDigest(owned.metadata.receipt.ownerPrepare) !== + controlDigest(saved.receipt.ownerPrepare) + ) + throw Error("Original fence owner required"); + return saved.receipt; + }, + async settle(value: DataConfigurationFenceSettle, rawHeaders: readonly string[]) { + const input = dataConfigurationFenceSettleSchema.parse(value), + saved = await row(sql, input.maintenanceId); + if (!saved) throw Object.assign(Error("Unknown configuration fence"), { statusCode: 404 }); + const verified = inspectGrant( + input, + rawHeaders, + "configuration-maintenance.settle", + saved.input, + ); + if ( + saved.receipt.candidateId !== input.candidateId || + saved.receipt.planDigest !== input.planDigest + ) + throw Error("Configuration guard settlement identity conflict"); + if (saved.guardState === "settled") { + if (canonicalJson(saved.settlement) !== canonicalJson(input)) + throw Error("Configuration guard settlement conflict"); + return withLeaf(verified, async () => saved.receipt); + } + if (saved.receipt.sequence !== input.expectedSequence) + throw Error("Configuration guard sequence conflict"); + const receipt = { + ...saved.receipt, + state: "settled" as const, + sequence: saved.receipt.sequence + 1, + }, + metadata = metadataSchema.parse({ ...saved, guardState: "settling", settlement: input }); + const owned = mirror(input.maintenanceId); + return withLeaf(verified, async () => { + await sql.transaction(async (tx) => { + await lock(tx); + const current = await row(tx, input.maintenanceId); + if (canonicalJson(current) !== canonicalJson(saved)) + throw Error("Configuration guard settlement CAS conflict"); + const active = await tx.unsafe( + "SELECT id FROM data_manager.jobs WHERE kind <> 'configuration-maintenance' AND (finished_at IS NULL OR execution_state->>'publication'='unknown') LIMIT 1", + ); + if (active.length) throw new DataFenceBusy(); + write({ ...owned, metadata }); + metadata.guardState = "settled"; + metadata.receipt = receipt; + await tx.unsafe( + "UPDATE data_manager.jobs SET metadata=$2::text::jsonb,ownership_version=ownership_version+1 WHERE id=$1::uuid", + [receipt.fenceId, canonicalJson({ configurationMaintenance: metadata })], + ); + }); + write({ ...owned, metadata }); + startup = "open"; + return receipt; + }); + }, + }; +} + +export function dataFenceTerminalDigest(receipt: DataConfigurationFenceReceipt) { + return controlDigest({ version: 1, receipt: dataConfigurationFenceReceiptSchema.parse(receipt) }); +} +/** Fixed SQL-journal retention adapter. Only a fully settled logical guard may + * retire its original prepared canonical receipt and original terminal time. */ +export function verifyRetainedDataFenceTerminal( + installation: ControlInstallation, + execution: import("../ops/execution-ref").ExecutionRef, + raw: Record, +) { + const local = installationState(installation), + metadata = z + .object({ configurationMaintenance: metadataSchema }) + .parse(raw.metadata).configurationMaintenance; + const proof = verifyControl( + installation.publicKey, + "openmapx-data-configuration-fence-v1", + readPrivateRecord( + local, + "configuration-maintenance", + `${metadata.receipt.maintenanceId}.json`, + 65536, + ), + mirrorSchema, + ); + if ( + metadata.guardState !== "settled" || + canonicalJson(metadata) !== canonicalJson(proof.metadata) || + raw.id !== execution.executionId || + raw.operation_key !== execution.operationKey || + retainedDataDatabaseTimestamp(raw.finished_at).getTime() !== proof.preparedAt + ) + throw Error("Settled original logical guard required"); + const effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord(local, "effects", `${execution.operationKey}.json`, 65536), + z + .object({ + identity: executionRefSchema, + receipt: z.object({ + kind: z.literal("data-configuration-fence"), + receipt: dataConfigurationFenceReceiptSchema, + guardDigest: z.string(), + }), + }) + .passthrough(), + ); + const original = { + ...proof, + metadata: { + version: 1 as const, + guardState: "prepared" as const, + receipt: effect.receipt.receipt, + input: metadata.input, + }, + }; + if ( + canonicalJson(effect.identity) !== canonicalJson(execution) || + controlDigest(original) !== effect.receipt.guardDigest || + effect.receipt.receipt.state !== "prepared" || + effect.receipt.receipt.sequence !== 0 || + canonicalJson({ ...metadata.receipt, state: "prepared", sequence: 0 }) !== + canonicalJson(effect.receipt.receipt) + ) + throw Error("Original logical guard commitment changed"); + const commitment = verifyControl( + installation.publicKey, + "openmapx-data-fence-terminal-v1", + readPrivateRecord(local, "observations", `terminal-${execution.operationKey}.json`, 8192), + fenceTerminalSchema, + ), + authority = executionAuthorityState(restoreAdmittedControlExecution(installation, execution)); + if ( + commitment.phase !== "canonical" || + commitment.admissionDigest !== authority.receiptDigest || + canonicalJson(commitment.execution) !== canonicalJson(execution) || + commitment.terminalDigest !== dataFenceTerminalDigest(effect.receipt.receipt) || + commitment.terminalAt !== proof.preparedAt + ) + throw Error("Original logical guard terminal commitment required"); + verifyControlEffectSettlement(installation, execution); + return { digest: commitment.terminalDigest, terminalAt: commitment.terminalAt }; +} + +export function dataFenceRetirementTuple(raw: Record) { + const metadata = z + .object({ configurationMaintenance: metadataSchema }) + .parse(raw.metadata).configurationMaintenance; + if (metadata.guardState !== "settled") throw Error("Unsettled guard cannot retire"); + return { + digest: dataFenceTerminalDigest({ ...metadata.receipt, state: "prepared", sequence: 0 }), + terminalAt: retainedDataDatabaseTimestamp(raw.finished_at).getTime(), + }; +} + +export function verifyRetainedDataFenceRefusal( + installation: ControlInstallation, + execution: import("../ops/execution-ref").ExecutionRef, +) { + const local = installationState(installation), + effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord(local, "effects", `${execution.operationKey}.json`, 65536), + z + .object({ + identity: executionRefSchema, + outcome: z.literal("failed_without_effect"), + publication: z.literal("not_started"), + receipt: z.strictObject({ + kind: z.literal("data-configuration-fence-refused"), + maintenanceId: z.uuid(), + reason: z.literal("busy"), + }), + }) + .passthrough(), + ), + commitment = verifyControl( + installation.publicKey, + "openmapx-data-fence-terminal-v1", + readPrivateRecord(local, "observations", `terminal-${execution.operationKey}.json`, 8192), + fenceTerminalSchema, + ), + authority = executionAuthorityState(restoreAdmittedControlExecution(installation, execution)); + if ( + canonicalJson(effect.identity) !== canonicalJson(execution) || + canonicalJson(commitment.execution) !== canonicalJson(execution) || + commitment.phase !== "canonical" || + commitment.admissionDigest !== authority.receiptDigest || + commitment.terminalDigest !== controlDigest({ version: 1, refused: effect.receipt }) + ) + throw Error("Original configuration refusal commitment required"); + verifyControlEffectSettlement(installation, execution); + return { digest: commitment.terminalDigest, terminalAt: commitment.terminalAt }; +} diff --git a/packages/core/src/build-artifacts/data-database-time.ts b/packages/core/src/build-artifacts/data-database-time.ts new file mode 100644 index 000000000..a2961dd0b --- /dev/null +++ b/packages/core/src/build-artifacts/data-database-time.ts @@ -0,0 +1,18 @@ +/** PostgreSQL returns Date under the raw driver and dated strings under the + * shared Drizzle driver. Neither representation supplies a missing timestamp. */ +export function retainedDataDatabaseTimestamp(value: unknown): Date { + if ( + !(value instanceof Date) && + !( + typeof value === "string" && + /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}(?::?\d{2})?)$/.test( + value, + ) + ) + ) + throw Error("Original dated database timestamp required"); + const result = new Date(value); + if (!Number.isSafeInteger(result.getTime()) || result.getTime() < 0) + throw Error("Invalid retained database timestamp"); + return result; +} diff --git a/packages/core/src/build-artifacts/data-execution-candidate.ts b/packages/core/src/build-artifacts/data-execution-candidate.ts new file mode 100644 index 000000000..6230911ec --- /dev/null +++ b/packages/core/src/build-artifacts/data-execution-candidate.ts @@ -0,0 +1,566 @@ +/** Fixed candidate provenance for the data-manager transit phases. */ +import { createHash } from "node:crypto"; +import { + constants as C, + closeSync, + fstatSync, + lstatSync, + openSync, + readSync, + realpathSync, +} from "node:fs"; +import { join, sep } from "node:path"; +import { z } from "zod"; +import type { OpsOperation } from "../ops/contract"; +import { retainedParentExecutionSchema } from "../ops/execution-identity"; +import { executionRefSchema } from "../ops/execution-ref"; +import { canonicalJson } from "../ops/fingerprint"; +// biome-ignore lint/suspicious/noImportCycles: Candidate snapshots and fixed proof publication share the original phase/authority; calls occur after schema initialization. +import { inspectControlChildPublication } from "./child-phase"; +// biome-ignore-start lint/suspicious/noImportCycles: Nominal execution verification is reused by the fixed producer after module initialization. +import { + assertControlExecutionForward, + type ControlChildPhase, + type ControlInstallation, + controlChildOperationKey, + controlDigest, + executionAuthorityState, + installationState, + readPrivateRecord, + retainedControlParent, + type VerifiedExecutionAuthority, + writePrivateRecord, +} from "./control-authority"; +// biome-ignore-end lint/suspicious/noImportCycles: Shared original execution verification described above. +import { + controlOriginSchema, + originRecordDomain, + sameControlOrigin, + signControl, + targetSchema, + verifyControl, +} from "./control-records"; +import { AnchoredDirectory, identity, sameInode } from "./filesystem"; +import { familyRecordWrite, verifyFamilyOwnedRecord } from "./retention-family"; + +const schema = z.strictObject({ + version: z.literal(1), + execution: executionRefSchema, + phase: z.enum(["promote-primary", "proxy-stage", "proxy-commit", "proxy-restore"]), + ordinal: z.number().int().min(0).max(31), + candidateId: z.string(), + directory: z.strictObject({ dev: z.string(), ino: z.string() }), + consumer: z + .strictObject({ + directory: z.strictObject({ dev: z.string(), ino: z.string() }), + inode: z.strictObject({ dev: z.string(), ino: z.string() }), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .nullable(), + artifactsDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + files: z + .array( + z.strictObject({ + name: z.string(), + inode: z.strictObject({ dev: z.string(), ino: z.string() }), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }), + ) + .min(1) + .max(2), +}); +/** Data files intentionally have producer/consumer hardlinks. Authority record + * readers retain their stricter single-link policy. Read identity and bytes + * from one confined descriptor and detect replacement or mutation during it. */ +function readCandidateFile(parent: AnchoredDirectory, name: string) { + const path = parent.name(name), + fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + try { + const before = fstatSync(fd, { bigint: true }); + if ( + !before.isFile() || + before.nlink < 1n || + before.uid !== BigInt(process.getuid?.() ?? -1) || + (before.mode & 0o022n) !== 0n || + before.size > 32n * 1024n * 1024n + ) + throw Error("Unsafe or oversized candidate file"); + const bytes = Buffer.alloc(Number(before.size) + 1); + let count = 0; + while (count < bytes.length) { + const n = readSync(fd, bytes, count, bytes.length - count, null); + if (!n) break; + count += n; + } + const after = fstatSync(fd, { bigint: true }); + if ( + BigInt(count) !== before.size || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + !sameInode(identity(before), identity(lstatSync(path, { bigint: true }))) + ) + throw Error("Candidate inode/bytes changed during read"); + parent.check(); + return { + inode: identity(before), + sha256: createHash("sha256").update(bytes.subarray(0, count)).digest("hex"), + bytes: bytes.subarray(0, count), + }; + } finally { + closeSync(fd); + } +} +const artifactSchema = z.strictObject({ + path: z.string().min(1).max(255), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + sizeBytes: z.number().int().nonnegative().safe(), +}); +/** The existing candidate writer verifies artifact hashes before retaining its + * manifest. Bind all verified files' inode/change metadata into the signed + * snapshot as well, so subsequent byte/inode changes cannot hide behind an + * unchanged manifest. Paths are the finite manifest roles, never HTTP input. */ +function candidateArtifacts( + directory: AnchoredDirectory, + manifestBytes: Buffer, + executionId: string, +): string { + const manifest = z + .object({ + schemaVersion: z.literal(1), + epoch: z.uuid(), + artifacts: z.strictObject({ + config: artifactSchema, + license: artifactSchema, + proxyConfig: artifactSchema, + proxyVars: artifactSchema, + datasets: z.array(artifactSchema).min(1).max(16384), + sourceIndex: artifactSchema.optional(), + sourceManifest: artifactSchema, + }), + }) + .parse(JSON.parse(manifestBytes.toString("utf8"))); + if (manifest.epoch !== executionId) + throw Error("Prepared candidate belongs to another execution"); + const a = manifest.artifacts; + if ( + a.config.path !== "config.yml" || + a.license.path !== "license.json" || + a.proxyConfig.path !== ".openmapx-feed-proxy/conf/default.conf" || + a.proxyVars.path !== ".openmapx-feed-proxy/feed-proxy-vars.json" || + a.sourceManifest.path !== "transit-source-manifest.json" || + (a.sourceIndex && a.sourceIndex.path !== "gbfs-source-index.json") || + a.datasets.some((f) => !/^[^.][^/\\]*\.(gtfs|netex)\.zip$/i.test(f.path)) + ) + throw Error("Unexpected prepared candidate artifact role/path"); + const digest = createHash("sha256"); + for (const file of [ + a.config, + a.license, + a.proxyConfig, + a.proxyVars, + ...a.datasets, + ...(a.sourceIndex ? [a.sourceIndex] : []), + a.sourceManifest, + ]) { + const parts = file.path.split("/"), + owned: AnchoredDirectory[] = []; + let parent = directory; + try { + for (const part of parts.slice(0, -1)) { + parent = parent.directory(part); + owned.push(parent); + } + const path = parent.name(parts.at(-1)!); + const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + try { + const stat = fstatSync(fd, { bigint: true }); + if ( + !stat.isFile() || + stat.nlink < 1n || + stat.uid !== BigInt(process.getuid?.() ?? -1) || + (stat.mode & 0o022n) !== 0n || + stat.size !== BigInt(file.sizeBytes) || + !sameInode(identity(stat), identity(lstatSync(path, { bigint: true }))) + ) + throw Error("Prepared artifact inode/size/ownership changed"); + digest.update( + canonicalJson({ + path: file.path, + inode: identity(stat), + size: String(stat.size), + mtime: String(stat.mtimeNs), + ctime: String(stat.ctimeNs), + }), + ); + parent.check(); + } finally { + closeSync(fd); + } + } finally { + for (const dir of owned.reverse()) dir.close(); + } + } + return digest.digest("hex"); +} +function proxyConsumer( + directory: AnchoredDirectory, + producer: { inode: { dev: string; ino: string }; sha256: string }, +) { + const mounted = directory.directory("motis-feed-proxy-config"); + try { + const file = readCandidateFile(mounted, "default.conf"); + if (!sameInode(file.inode, producer.inode) || file.sha256 !== producer.sha256) + throw Error("Mounted proxy config differs from the original producer inode/bytes"); + return { directory: identity(mounted.stat), inode: file.inode, sha256: file.sha256 }; + } finally { + mounted.close(); + } +} +function snapshot( + authority: VerifiedExecutionAuthority, + phase: ControlChildPhase, + ordinal: number, + preparing = false, +) { + const s = executionAuthorityState(authority), + slot = + s.reservations.find((r) => r.phase === phase && r.ordinal === ordinal) ?? + s.plan.find((r) => r.phase === phase && r.ordinal === ordinal); + if ( + !slot || + s.installation.installation.role !== "data-manager" || + !["transit-sync", "transit-maintenance"].includes(s.kind) + ) + throw Error("Original data candidate reservation required"); + const proxy = ["proxy-stage", "proxy-commit", "proxy-restore"].includes(phase); + if (!proxy && phase !== "promote-primary") return null; + const relative = proxy ? "motis-feed-proxy" : `motis/${preparing ? "staging" : "live"}`; + const root = realpathSync(s.installation.dataRoot), + path = realpathSync(join(root, relative)); + if (!path.startsWith(root + sep)) throw Error("Candidate directory escaped data root"); + const directory = new AnchoredDirectory(path); + try { + const files = proxy + ? [ + { name: "conf/default.conf", parts: ["conf", "default.conf"] }, + { name: "feed-proxy-vars.json", parts: ["feed-proxy-vars.json"] }, + ] + : [{ name: "motis-candidate-manifest.json", parts: ["motis-candidate-manifest.json"] }]; + const retainedFiles = files.map(({ name, parts }) => { + const nested = parts.length === 2 ? directory.directory(parts[0]!) : undefined; + try { + return { name, ...readCandidateFile(nested ?? directory, parts.at(-1)!) }; + } finally { + nested?.close(); + } + }); + return { + version: 1 as const, + execution: s.execution, + phase, + ordinal, + candidateId: proxy + ? `ctrl-${createHash("sha256").update(`openmapx-transit-candidate-v1\0${s.execution.operationKey}\0${phase}\0${ordinal}`).digest("hex").slice(0, 32)}` + : s.execution.executionId, + directory: identity(directory.stat), + files: retainedFiles.map(({ name, inode, sha256 }) => ({ name, inode, sha256 })), + consumer: proxy ? proxyConsumer(directory, retainedFiles[0]!) : null, + artifactsDigest: proxy + ? null + : candidateArtifacts(directory, retainedFiles[0]!.bytes, s.execution.executionId), + }; + } finally { + directory.close(); + } +} +const name = (authority: VerifiedExecutionAuthority, ordinal: number) => + `candidate-${authority.execution.operationKey}-${ordinal}.json`; +export function retainOwnedDataTransitCandidate( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +): string | null { + const value = snapshot(authority, input.phase, input.ordinal, input.phase === "promote-primary"); + if (!value) return null; + const s = executionAuthorityState(authority), + reserved = s.reservations.some( + (slot) => slot.phase === input.phase && slot.ordinal === input.ordinal, + ), + publication = reserved ? inspectControlChildPublication(authority, input) : undefined; + let saved: unknown; + try { + saved = readPrivateRecord(s.installation, "observations", name(authority, input.ordinal), 8192); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (saved !== null && saved !== undefined) { + const prior = verifyControl( + s.installation.installation.publicKey, + "openmapx-data-candidate-v1", + saved, + schema, + ); + if (canonicalJson(prior) !== canonicalJson(value)) + throw Error("Original candidate bytes/owner changed"); + publishCandidate(authority, prior); + return controlDigest(prior); + } + if (publication && (publication.binding || publication.publication.state !== "unissued")) + throw Error("Attempted candidate cannot be reconstructed"); + assertControlExecutionForward(authority); + const parsed = schema.parse(value); + writePrivateRecord( + s.installation, + "observations", + name(authority, input.ordinal), + signControl(s.installation.privateKey, "openmapx-data-candidate-v1", parsed), + true, + s.origin, + ); + publishCandidate(authority, parsed); + return controlDigest(parsed); +} +export function assertOwnedDataTransitCandidate( + authority: VerifiedExecutionAuthority, + input: { phase: ControlChildPhase; ordinal: number }, +): string | null { + const value = snapshot(authority, input.phase, input.ordinal); + if (!value) return null; + const s = executionAuthorityState(authority), + saved = verifyControl( + s.installation.installation.publicKey, + "openmapx-data-candidate-v1", + readPrivateRecord(s.installation, "observations", name(authority, input.ordinal), 8192), + schema, + ); + if (canonicalJson(saved) !== canonicalJson(value)) + throw Error("Foreign or changed owned transit candidate"); + return controlDigest(saved); +} + +const publishedSchema = z.strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + candidate: schema, + childOperationKey: z.string(), + operationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + serviceIds: targetSchema, +}); +function sharedDirectory(local: ReturnType) { + const data = new AnchoredDirectory(local.dataRoot), + control = data.directory(".admin-target-control"), + executions = control.directory("executions"), + role = executions.directory("data-manager"); + return { + role, + close() { + role.close(); + executions.close(); + control.close(); + data.close(); + }, + }; +} +function publishCandidate( + authority: VerifiedExecutionAuthority, + candidate: z.infer, +): void { + const s = executionAuthorityState(authority), + slot = + s.reservations.find((r) => r.phase === candidate.phase && r.ordinal === candidate.ordinal) ?? + s.plan.find((r) => r.phase === candidate.phase && r.ordinal === candidate.ordinal); + if (!slot || !s.origin) throw Error("Dated original candidate slot required"); + assertControlExecutionForward(authority); + const payload = publishedSchema.parse({ + version: 2, + installationId: s.installation.installation.installationId, + origin: s.origin, + candidate, + childOperationKey: slot.execution.operationKey, + operationFingerprint: + "operationFingerprint" in slot + ? slot.operationFingerprint + : slot.execution.operationFingerprint, + serviceIds: [...slot.serviceIds], + }), + bytes = canonicalJson( + signControl(s.installation.privateKey, "openmapx-data-candidate-publication-v2", payload), + ), + directory = sharedDirectory(s.installation), + name = `${slot.execution.operationKey}.data-candidate.json`; + try { + let prior: Buffer | undefined; + try { + prior = directory.role.read(name, 8192); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (prior) { + if (prior.toString("utf8") !== bytes) throw Error("Original published candidate changed"); + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "shared", namespace: "execution", container: "data-manager", name }, + bytes, + ); + return; + } + familyRecordWrite( + s.installation, + s.origin, + { scope: "shared", namespace: "execution", container: "data-manager", name }, + bytes, + (before) => directory.role.write(name, bytes, true, before), + true, + ); + } finally { + directory.close(); + } +} +/** Fixed registry resource input. This reads shared signed evidence, never a + * sibling role's private state, and never authorizes execution by itself. */ +export function inspectDataTransitCandidate( + installation: ControlInstallation, + operationKey: string, + operation: OpsOperation, +) { + if ( + installation.role !== "ops-agent" || + (operation.kind !== "motis.primary.promote" && operation.kind !== "feedProxy.validateAndReload") + ) + throw Error("Fixed transit candidate required"); + const local = installationState(installation), + directory = sharedDirectory(local); + try { + const proof = verifyControl( + local.publicKeys["data-manager"], + "openmapx-data-candidate-publication-v2", + JSON.parse( + directory.role.read(`${operationKey}.data-candidate.json`, 8192).toString("utf8"), + ), + publishedSchema, + ), + candidate = proof.candidate; + if ( + proof.installationId !== installation.installationId || + proof.childOperationKey !== operationKey || + proof.operationFingerprint !== controlDigest(operation) || + operationKey !== + controlChildOperationKey( + candidate.execution.operationKey, + candidate.phase, + candidate.ordinal, + ) || + candidate.execution.executor !== "data-manager" + ) + throw Error("Original candidate parent/phase/body changed"); + const promotion = operation.kind === "motis.primary.promote"; + if ( + promotion + ? candidate.phase !== "promote-primary" || + candidate.candidateId !== operation.preparedRunId || + candidate.candidateId !== candidate.execution.executionId || + canonicalJson(proof.serviceIds) !== canonicalJson(["motis"]) + : !["proxy-stage", "proxy-commit", "proxy-restore"].includes(candidate.phase) || + candidate.candidateId !== operation.candidateId || + canonicalJson(proof.serviceIds) !== canonicalJson(["motis-feed-proxy"]) + ) + throw Error("Candidate identity/target mismatch"); + const root = realpathSync(local.dataRoot), + path = realpathSync(join(root, promotion ? "motis/live" : "motis-feed-proxy")); + if (!path.startsWith(root + sep)) throw Error("Candidate escaped data root"); + const confined = new AnchoredDirectory(path); + try { + if (canonicalJson(identity(confined.stat)) !== canonicalJson(candidate.directory)) + throw Error("Candidate directory replaced"); + const expected = promotion + ? ["motis-candidate-manifest.json"] + : ["conf/default.conf", "feed-proxy-vars.json"]; + if (canonicalJson(candidate.files.map((f) => f.name)) !== canonicalJson(expected)) + throw Error("Candidate files changed"); + let manifestBytes: Buffer | undefined; + for (const file of candidate.files) { + const parts = file.name.split("/"), + nested = parts.length === 2 ? confined.directory(parts[0]!) : undefined, + parent = nested ?? confined; + try { + const current = readCandidateFile(parent, parts.at(-1)!); + if ( + canonicalJson(current.inode) !== canonicalJson(file.inode) || + current.sha256 !== file.sha256 + ) + throw Error("Candidate inode/bytes replaced"); + if (promotion) manifestBytes = current.bytes; + } finally { + nested?.close(); + } + } + if (promotion) { + if ( + !manifestBytes || + candidate.consumer !== null || + candidateArtifacts(confined, manifestBytes, candidate.execution.executionId) !== + candidate.artifactsDigest + ) + throw Error("Prepared candidate artifacts changed"); + } else if ( + candidate.artifactsDigest !== null || + canonicalJson(proxyConsumer(confined, candidate.files[0]!)) !== + canonicalJson(candidate.consumer) + ) + throw Error("Mounted candidate inode/bytes replaced"); + } finally { + confined.close(); + } + return { digest: controlDigest(proof), proof }; + } finally { + directory.close(); + } +} +/** Recheck the actual admitted parent, target lease and snapshot at each fixed + * Docker effect boundary; a previously returned resource claim is insufficient. */ +export function verifyDataTransitCandidateForExecution( + authority: VerifiedExecutionAuthority, + operation: OpsOperation, + expectedDigest: string, +): void { + const s = executionAuthorityState(authority), + checked = inspectDataTransitCandidate( + s.installation.installation, + s.execution.operationKey, + operation, + ), + raw = readPrivateRecord( + s.installation, + "executions", + `${s.execution.operationKey}.json`, + 65536, + ), + admission = verifyControl( + s.installation.installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z.object({ parent: retainedParentExecutionSchema }).passthrough(), + ); + if ( + controlDigest(raw) !== s.receiptDigest || + checked.digest !== expectedDigest || + canonicalJson(admission.parent) !== + canonicalJson(retainedControlParent(checked.proof.candidate.execution)) || + !sameControlOrigin(checked.proof.origin, s.origin) || + canonicalJson(s.serviceIds) !== canonicalJson(checked.proof.serviceIds) + ) + throw Error("Candidate differs from original borrowed admission"); +} +export function assertDataTransitCandidateForExecution( + authority: VerifiedExecutionAuthority, + operation: OpsOperation, + expectedDigest: string, +): void { + verifyDataTransitCandidateForExecution(authority, operation, expectedDigest); + assertControlExecutionForward(authority); +} diff --git a/packages/core/src/build-artifacts/data-execution-journal.ts b/packages/core/src/build-artifacts/data-execution-journal.ts new file mode 100644 index 000000000..8c8480c08 --- /dev/null +++ b/packages/core/src/build-artifacts/data-execution-journal.ts @@ -0,0 +1,1075 @@ +/** Fixed data-manager database/worker journal. No public row or callback boolean + * can mint a completion or repair an absent original terminal commitment. */ +import { randomUUID } from "node:crypto"; +import { z } from "zod"; +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { canonicalJson } from "../ops/fingerprint"; +import { + type DataExecutionResult, + type DataExecutionStatus, + dataExecutionResultSchema, + dataExecutionStatusSchema, + generationHardlinksPlanSchema, + isDataExecutionTerminal, +} from "../services/data-manager/contracts/executions"; +import { + transitousRunnerRequestSchema, + transitousRunnerResultSchema, +} from "../transitous-runner/contract"; +import { + beginControlEffect, + type ControlInstallation, + controlDigest, + executionAuthorityState, + installationState, + publishDataControlTerminalWitness, + readPrivateRecord, + requireControlJournalRetention, + restoreAdmittedControlExecution, + settleExecutionLifecycle, + type VerifiedExecutionAuthority, + verifyControlEffectSettlement, + withPrivateEffectWriter, + writePrivateRecord, +} from "./control-authority"; +import { + controlOriginSchema, + sameControlOrigin, + signControl, + verifyControl, +} from "./control-records"; +import { + dataFenceRetirementTuple, + verifyRetainedDataFenceRefusal, + verifyRetainedDataFenceTerminal, +} from "./data-configuration-fence"; +import { retainedDataDatabaseTimestamp } from "./data-database-time"; +import { DataExecutionPartialResult } from "./data-execution-outcome"; +import { verifyFeedProxyCanonicalTerminal } from "./feed-proxy-validation"; +import { + assertFamilyCanonicalCompatibility, + completeFamilyJournal, + markFamilyTerminal, + publishFamilyCanonicalTerminal, +} from "./retention-family"; +import { + closeArtifactTargetLeaseHandle, + inspectArtifactTargetLeaseChildSettlement, + restoreArtifactTargetLeaseForSettlement, + restoreGenerationHardlinksLeaseForSettlement, + type TargetLeaseAuthority, + verifyRetainedArtifactTargetRootRelease, +} from "./target-lease"; + +export interface DataExecutionJournalDatabase { + unsafe( + query: string, + parameters?: (string | number | null)[], + ): Promise[]>; +} +const completionBrand = Symbol("contained-data-worker"); +export type ContainedDataExecution = Readonly<{ [completionBrand]: true }>; +const completions = new WeakMap< + ContainedDataExecution, + { + authority: VerifiedExecutionAuthority; + workerId: string; + publication: "not_started" | "committed" | "unknown"; + childrenDigest?: string; + result?: DataExecutionResult; + error?: unknown; + partial?: boolean; + lease?: TargetLeaseAuthority; + } +>(); +const workerSchema = z + .strictObject({ + version: z.literal(1), + installationId: z.uuid(), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: z.string().regex(/^[a-f0-9]{64}$/), + workerId: z.uuid(), + containedAt: z.number().int().positive().safe(), + outcome: z.enum(["returned", "partial", "failed"]), + publication: z.enum(["not_started", "committed", "unknown"]), + childrenDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + proxyRecovery: z.enum(["not_required", "settled", "pending"]).optional(), + result: dataExecutionResultSchema.optional(), + }) + .refine((value) => (value.outcome !== "failed") === (value.result !== undefined)); +const runnerJournalSchema = z.strictObject({ + version: z.literal(1), + execution: executionRefSchema, + completed: z.number().int().nonnegative().safe(), + completedDigest: z.string().regex(/^[a-f0-9]{64}$/), + attempts: z.array(transitousRunnerRequestSchema).max(32), +}); +const terminalSchema = z.strictObject({ + version: z.literal(1), + phase: z.enum(["prepared", "canonical"]), + installationId: z.uuid(), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: z.string().regex(/^[a-f0-9]{64}$/), + terminalDigest: z.string().regex(/^[a-f0-9]{64}$/), + terminalAt: z.number().int().positive().safe(), + workerId: z.uuid(), + childrenDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +const rowSchema = z.object({ + id: z.uuid(), + execution_version: z.literal(2), + operation_key: z.string(), + execution_fingerprint: z.string(), + admitted_at: z.union([z.string(), z.number()]), + deadline_at: z.union([z.string(), z.number()]), + kind: z.string(), + status: z.string(), + execution_state: z.unknown(), + execution_input: z.unknown(), + execution_children: z.array(z.unknown()), + finished_at: z + .unknown() + .transform((value) => (value === null ? null : retainedDataDatabaseTimestamp(value))), + ownership_version: z.number().int(), + metadata: z.unknown(), +}); +const mutableSchema = z.strictObject({ + cancellationRequestedAt: z.string().nullable(), + progress: z.number().nullable(), + diagnostics: z.unknown(), + publication: z.enum(["not_started", "committed", "unknown"]), + result: z.unknown().optional(), +}); +const childSchema = z + .object({ + state: z.string(), + execution: executionRefSchema.optional(), + settlement: z + .object({ + outcome: z.enum(["contained", "committed", "failed_without_effect"]), + terminalReceiptDigest: z.string().regex(/^[a-f0-9]{64}$/), + }) + .optional(), + }) + .passthrough(); +export const dataTerminalReceiptDigest = (status: DataExecutionStatus) => + controlDigest({ version: 1, status: dataExecutionStatusSchema.parse(status) }); +function statusOf(raw: unknown) { + const row = rowSchema.parse(raw), + state = mutableSchema.parse(row.execution_state); + const status = dataExecutionStatusSchema.parse({ + id: row.id, + operationKey: row.operation_key, + fingerprint: row.execution_fingerprint, + admittedAt: Number(row.admitted_at), + deadlineAt: Number(row.deadline_at), + kind: row.kind, + status: row.status, + ...state, + }); + const execution = z + .object({ dataExecution: z.object({ execution: executionRefSchema }) }) + .parse(row.metadata).dataExecution.execution; + if ( + execution.executionId !== status.id || + execution.operationKey !== status.operationKey || + execution.fingerprint !== status.fingerprint || + execution.deadlineAt !== status.deadlineAt || + execution.executor !== "data-manager" + ) + throw Error("Data journal identity conflict"); + return { row, status, execution }; +} +export function createDataExecutionControlJournal( + installation: ControlInstallation, + database: DataExecutionJournalDatabase, +) { + const local = installationState(installation); + if (installation.role !== "data-manager" || local.fixedChildId || local.retentionJournal) + throw Error("Fixed data-manager journal required"); + const contained = new Map(); + async function load(execution: ExecutionRef) { + const rows = await database.unsafe( + "SELECT * FROM data_manager.jobs WHERE execution_version=2 AND operation_key=$1", + [execution.operationKey], + ); + if (rows.length !== 1) throw Error("Original data journal row required"); + const value = statusOf(rows[0]); + if (controlDigest(value.execution) !== controlDigest(execution)) + throw Error("Original data journal execution changed"); + return value; + } + const recoverySchema = z.strictObject({ + version: z.literal(1), + execution: executionRefSchema, + admissionDigest: z.string(), + origin: controlOriginSchema, + restore: z.unknown(), + backupDigest: z.string().regex(/^[a-f0-9]{64}$/), + }); + function recoveryIntent(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority); + try { + const proof = verifyControl( + installation.publicKey, + "openmapx-data-proxy-recovery-v1", + readPrivateRecord( + local, + "observations", + `proxy-recovery-${s.execution.operationKey}.json`, + 8192, + ), + recoverySchema, + ); + if ( + proof.admissionDigest !== s.receiptDigest || + controlDigest(proof.execution) !== controlDigest(s.execution) || + !sameControlOrigin(proof.origin, s.origin) + ) + throw Error("Original proxy recovery identity changed"); + return proof; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + } + /** One fixed policy over the current actual row, original signed plan and + * authenticated shared child/canonical receipts. No status/result shortcut. */ + async function assertOwnedProxyRecoverySettled( + execution: ExecutionRef, + ): Promise<"not_required" | "settled"> { + const value = await load(execution), + authority = restoreAdmittedControlExecution(installation, execution), + s = executionAuthorityState(authority); + const restore = s.reservations.find((slot) => slot.phase === "proxy-restore"); + const children = value.row.execution_children.map((raw) => + z + .object({ + phase: z.string(), + ordinal: z.number(), + operationKey: z.string(), + operationFingerprint: z.string(), + serviceIds: z.array(z.string()), + parentExecution: executionRefSchema, + operation: z.unknown(), + state: z.string(), + execution: executionRefSchema.optional(), + proxyRecovery: z.object({ version: z.literal(1), backupDigest: z.string() }).optional(), + }) + .passthrough() + .parse(raw), + ); + const intent = recoveryIntent(authority); + let required = intent !== null || children.some((child) => child.phase === "proxy-restore"); + let lease: TargetLeaseAuthority | undefined; + try { + for (const child of children.filter((child) => + ["proxy-stage", "proxy-commit", "proxy-restore"].includes(child.phase), + )) { + const slot = s.reservations.find( + (slot) => slot.phase === child.phase && slot.ordinal === child.ordinal, + ); + if ( + !slot || + slot.execution.operationKey !== child.operationKey || + slot.operationFingerprint !== child.operationFingerprint || + controlDigest(child.operation) !== slot.operationFingerprint || + controlDigest(child.parentExecution) !== controlDigest(execution) || + controlDigest(child.serviceIds) !== controlDigest(slot.serviceIds) + ) + throw Error("Original proxy phase changed"); + if (child.state !== "terminal" || !child.execution) continue; + lease ??= restoreArtifactTargetLeaseForSettlement(authority); + const shared = inspectArtifactTargetLeaseChildSettlement(lease, child.execution); + const canonical = verifyFeedProxyCanonicalTerminal(authority, child.execution); + if ( + restore && + child.phase !== "proxy-restore" && + canonical.state === "failed" && + shared?.outcome === "contained" && + shared.publication === "not_started" + ) + required = true; + } + if (!required) return "not_required"; + if (!restore || !intent || controlDigest(intent.restore) !== controlDigest(restore)) + throw Error("Original proxy restore obligation remains pending"); + const child = children.find( + (child) => child.phase === "proxy-restore" && child.ordinal === restore.ordinal, + ); + if ( + !child || + child.state !== "terminal" || + !child.execution || + child.proxyRecovery?.backupDigest !== intent.backupDigest || + child.execution.executionId !== restore.execution.executionId || + child.execution.operationKey !== restore.execution.operationKey + ) + throw Error("Original proxy restore obligation remains pending"); + lease ??= restoreArtifactTargetLeaseForSettlement(authority); + const shared = inspectArtifactTargetLeaseChildSettlement(lease, child.execution), + canonical = verifyFeedProxyCanonicalTerminal(authority, child.execution); + const result = z + .strictObject({ candidateId: z.string(), reloaded: z.literal(true) }) + .parse(canonical.result); + const operation = z + .strictObject({ kind: z.literal("feedProxy.validateAndReload"), candidateId: z.string() }) + .parse(child.operation); + if ( + canonical.state !== "succeeded" || + shared?.outcome !== "committed" || + shared.publication !== "committed" || + result.candidateId !== operation.candidateId || + controlDigest({ kind: "ops-result", result }) !== shared.terminalReceiptDigest + ) + throw Error("Original successful proxy restore is not proved"); + return "settled"; + } finally { + if (lease) closeArtifactTargetLeaseHandle(lease); + } + } + function readProof(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority); + const proof = verifyControl( + installation.publicKey, + "openmapx-data-terminal-v1", + readPrivateRecord(local, "observations", `terminal-${s.execution.operationKey}.json`, 8192), + terminalSchema, + ); + if ( + proof.installationId !== installation.installationId || + proof.admissionDigest !== s.receiptDigest || + !sameControlOrigin(proof.origin, s.origin) || + controlDigest(proof.execution) !== controlDigest(s.execution) + ) + throw Error("Original data terminal commitment changed"); + return proof; + } + async function verifyStored(execution: ExecutionRef, digest?: string) { + await assertOwnedProxyRecoverySettled(execution); + const value = await load(execution), + authority = restoreAdmittedControlExecution(installation, execution), + proof = readProof(authority); + if ( + !isDataExecutionTerminal(value.status) || + !value.row.finished_at || + proof.phase !== "canonical" || + proof.terminalAt !== value.row.finished_at.getTime() || + proof.terminalDigest !== dataTerminalReceiptDigest(value.status) || + (digest !== undefined && proof.terminalDigest !== digest) || + value.row.execution_children.some((raw) => childSchema.parse(raw).state !== "terminal") + ) + throw Error("Data journal terminal containment is not proved"); + verifyControlEffectSettlement(installation, execution); + return { value, authority, proof }; + } + const journal = { + async prepare(executions: readonly { execution: ExecutionRef; digest?: string }[]) { + contained.clear(); + for (const item of executions) + try { + const rows = await database.unsafe( + "SELECT * FROM data_manager.jobs WHERE operation_key=$1", + [item.execution.operationKey], + ); + if (rows.length === 0) { + const proof = verifyRetainedDataFenceRefusal(installation, item.execution); + if (item.digest === proof.digest) + contained.set(item.execution.operationKey, proof.digest); + continue; + } + if (rows.length !== 1) continue; + if (rows[0]!.kind === "configuration-maintenance") { + const proof = verifyRetainedDataFenceTerminal(installation, item.execution, rows[0]!); + if (item.digest !== undefined && item.digest !== proof.digest) continue; + contained.set(item.execution.operationKey, proof.digest); + } else { + const { proof } = await verifyStored(item.execution, item.digest); + contained.set(item.execution.operationKey, proof.terminalDigest); + } + } catch { + /* Missing, unfinished or unknown guard/worker evidence is retained. */ + } + }, + contained(execution: ExecutionRef, digest: string | undefined) { + return digest !== undefined && contained.get(execution.operationKey) === digest; + }, + async retire( + executions: readonly { execution: ExecutionRef; digest: string; terminalAt?: number }[], + _manifestDigest: string, + ) { + // The signed family manifest has already committed the original tuple + // and time. Private records may already be removed on a resumed cleanup. + for (const item of executions) { + // A resumed, already committed family cleanup may have deleted its row. + // Absence here permits no effects; the caller owns the signed manifest. + const rows = await database.unsafe( + "SELECT * FROM data_manager.jobs WHERE execution_version=2 AND operation_key=$1", + [item.execution.operationKey], + ); + if (rows.length === 0) { + const guards = await database.unsafe( + "SELECT * FROM data_manager.jobs WHERE kind='configuration-maintenance' AND operation_key=$1", + [item.execution.operationKey], + ); + if (guards.length) { + const guard = guards[0]!, + tuple = dataFenceRetirementTuple(guard); + if ( + guards.length !== 1 || + guard.id !== item.execution.executionId || + tuple.digest !== item.digest || + tuple.terminalAt !== item.terminalAt + ) + throw Error("Retiring guard tuple/time changed"); + const deleted = await database.unsafe( + "DELETE FROM data_manager.jobs WHERE id=$1::uuid AND kind='configuration-maintenance' AND operation_key=$2 AND ownership_version=$3 RETURNING id", + [ + item.execution.executionId, + item.execution.operationKey, + z.number().int().parse(guard.ownership_version), + ], + ); + if (deleted.length !== 1) throw Error("Guard retirement CAS conflict"); + } + contained.delete(item.execution.operationKey); + continue; + } + const value = statusOf(rows[0]); + if ( + rows.length !== 1 || + controlDigest(value.execution) !== controlDigest(item.execution) || + !isDataExecutionTerminal(value.status) || + dataTerminalReceiptDigest(value.status) !== item.digest || + !item.terminalAt || + value.row.finished_at?.getTime() !== item.terminalAt + ) + throw Error("Retiring data terminal tuple/time changed"); + const removed = await database.unsafe( + "DELETE FROM data_manager.jobs WHERE id=$1::uuid AND execution_version=2 AND operation_key=$2 AND ownership_version=$3 AND finished_at=$4::text::timestamptz RETURNING id", + [ + value.status.id, + item.execution.operationKey, + value.row.ownership_version, + value.row.finished_at.toISOString(), + ], + ); + if (removed.length !== 1) throw Error("Data journal retirement CAS conflict"); + contained.delete(item.execution.operationKey); + } + }, + }; + local.retentionJournal = journal; + return { + installation, + dataRoot: local.dataRoot, + assertOwnedProxyRecoverySettled, + /** Repair only an unproved proxy root's public finality. Its original + * identity, local write evidence, child receipts and physical lease stay. */ + async fenceUnsettledProxyRecovery(execution: ExecutionRef): Promise { + try { + await assertOwnedProxyRecoverySettled(execution); + return; + } catch { + /* Below requires the original signed proxy plan and current row. */ + } + const authority = restoreAdmittedControlExecution(installation, execution), + s = executionAuthorityState(authority), + value = await load(execution); + if ( + !s.reservations.some((slot) => slot.phase === "proxy-restore") || + !value.row.execution_children.some((raw) => + z.object({ phase: z.string() }).parse(raw).phase.startsWith("proxy-"), + ) + ) + throw Error("Original proxy recovery root required"); + const status = { + ...value.status, + status: value.status.cancellationRequestedAt ? "cancel_pending" : "interrupted", + publication: "unknown", + }; + const rows = await database.unsafe( + "UPDATE data_manager.jobs SET status=$1,execution_state=$2::text::jsonb,finished_at=NULL,ownership_version=ownership_version+1 WHERE id=$3::uuid AND execution_version=2 AND operation_key=$4 AND ownership_version=$5 RETURNING id", + [ + status.status, + canonicalJson({ + ...mutableSchema.parse(value.row.execution_state), + publication: "unknown", + }), + execution.executionId, + execution.operationKey, + value.row.ownership_version, + ], + ); + if (rows.length !== 1) throw Error("Proxy recovery finality repair CAS lost"); + }, + async retainProxyRecoveryIntent(authority: VerifiedExecutionAuthority, backupDigest: string) { + const value = await load(authority.execution), + s = executionAuthorityState(authority), + restore = s.reservations.find((slot) => slot.phase === "proxy-restore"); + const child = value.row.execution_children + .map((raw) => + z + .object({ + phase: z.string(), + state: z.string(), + operationKey: z.string(), + proxyRecovery: z.object({ backupDigest: z.string() }).optional(), + }) + .parse(raw), + ) + .find((child) => child.phase === "proxy-restore"); + if ( + s.installation !== local || + !s.origin || + !restore || + child?.state !== "reserved" || + child.operationKey !== restore.execution.operationKey || + child.proxyRecovery?.backupDigest !== backupDigest + ) + throw Error("Confirmed original restore reservation required"); + const payload = recoverySchema.parse({ + version: 1, + execution: s.execution, + admissionDigest: s.receiptDigest, + origin: s.origin, + restore, + backupDigest, + }); + const prior = recoveryIntent(authority); + if (prior) { + if (controlDigest(prior) !== controlDigest(payload)) + throw Error("Original recovery backup changed"); + return; + } + writePrivateRecord( + local, + "observations", + `proxy-recovery-${s.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-data-proxy-recovery-v1", payload), + true, + s.origin, + ); + }, + retainRunnerAttempt(authority: VerifiedExecutionAuthority, request: unknown): void { + const state = executionAuthorityState(authority), + body = transitousRunnerRequestSchema.parse(request); + if (state.installation !== local || state.kind !== "transit-sync" || !state.origin) + throw Error("Original transit worker required"); + withPrivateEffectWriter(state, () => { + const name = `runners-${state.execution.operationKey}.json`; + let saved: z.infer = { + version: 1, + execution: state.execution, + completed: 0, + completedDigest: controlDigest([]), + attempts: [], + }; + try { + saved = verifyControl( + installation.publicKey, + "openmapx-data-runner-journal-v1", + readPrivateRecord(local, "observations", name, 65536), + runnerJournalSchema, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if ( + controlDigest(saved.execution) !== controlDigest(state.execution) || + saved.attempts.some((attempt) => attempt.capability === body.capability) || + saved.attempts.length >= 32 + ) + throw Error("Runner attempt replay/capacity conflict"); + const next = { ...saved, attempts: [...saved.attempts, body] }; + writePrivateRecord( + local, + "observations", + name, + signControl(local.privateKey, "openmapx-data-runner-journal-v1", next), + false, + state.origin, + ); + }); + }, + completeRunnerAttempt( + authority: VerifiedExecutionAuthority, + request: unknown, + result: unknown, + ): void { + const state = executionAuthorityState(authority), + body = transitousRunnerRequestSchema.parse(request), + terminal = transitousRunnerResultSchema.parse(result); + withPrivateEffectWriter(state, () => { + const name = `runners-${state.execution.operationKey}.json`, + saved = verifyControl( + installation.publicKey, + "openmapx-data-runner-journal-v1", + readPrivateRecord(local, "observations", name, 65536), + runnerJournalSchema, + ), + attempt = saved.attempts.find((attempt) => attempt.capability === body.capability); + if ( + controlDigest(saved.execution) !== controlDigest(state.execution) || + !attempt || + canonicalJson(attempt) !== canonicalJson(body) + ) + throw Error("Original runner attempt required"); + writePrivateRecord( + local, + "observations", + name, + signControl(local.privateKey, "openmapx-data-runner-journal-v1", { + ...saved, + completed: saved.completed + 1, + completedDigest: controlDigest({ + previous: saved.completedDigest, + attempt: controlDigest(body), + result: terminal, + }), + attempts: saved.attempts.filter((entry) => entry.capability !== body.capability), + }), + false, + state.origin, + ); + }); + }, + async retainHardlinksPlan(authority: VerifiedExecutionAuthority, plan: unknown) { + const s = executionAuthorityState(authority), + value = await load(authority.execution), + body = generationHardlinksPlanSchema.parse(plan); + if (value.status.kind !== "generation-hardlinks") throw Error("Generation leaf required"); + const input = z + .object({ + execution_input: z.object({ + kind: z.literal("generation-hardlinks"), + revisionId: z.string(), + planDigest: z.string(), + coordinatorMode: z.string(), + }), + }) + .parse(value.row).execution_input; + if ( + input.planDigest !== controlDigest(body) || + body.plan.some((entry) => !s.serviceIds.includes(entry.consumerService)) + ) + throw Error("Signed generation plan/target binding mismatch"); + const payload = { version: 1, execution: s.execution, input, plan: body }; + let saved: unknown; + try { + saved = readPrivateRecord( + local, + "hardlinks", + `${s.execution.executionId}.json`, + 1024 * 1024, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (saved) { + const original = verifyControl( + installation.publicKey, + "openmapx-data-hardlinks-v1", + saved, + z.unknown(), + ); + if (canonicalJson(original) !== canonicalJson(payload)) + throw Error("Original generation plan conflict"); + return; + } + writePrivateRecord( + local, + "hardlinks", + `${s.execution.executionId}.json`, + signControl(local.privateKey, "openmapx-data-hardlinks-v1", payload), + true, + s.origin, + ); + }, + async readHardlinksPlan(execution: ExecutionRef) { + const value = await load(execution), + payload = verifyControl( + installation.publicKey, + "openmapx-data-hardlinks-v1", + readPrivateRecord(local, "hardlinks", `${execution.executionId}.json`, 1024 * 1024), + z.strictObject({ + version: z.literal(1), + execution: executionRefSchema, + input: z.object({ + kind: z.literal("generation-hardlinks"), + revisionId: z.string(), + planDigest: z.string(), + coordinatorMode: z.string(), + }), + plan: generationHardlinksPlanSchema, + }), + ); + if ( + controlDigest(payload.execution) !== controlDigest(execution) || + controlDigest(payload.input) !== controlDigest(value.row.execution_input) || + payload.input.planDigest !== controlDigest(payload.plan) + ) + throw Error("Original generation plan changed"); + return payload.plan; + }, + async begin(authority: VerifiedExecutionAuthority) { + const value = await load(authority.execution); + if (value.status.status !== "queued" || value.status.cancellationRequestedAt) + throw Error("Only a newly admitted data worker may begin"); + requireControlJournalRetention(authority); + beginControlEffect(authority); + }, + /** The actual dispatcher promise must contain and await all of its process, + * transfer, transaction and remote-child continuations before settling. */ + async contain( + authority: VerifiedExecutionAuthority, + worker: Promise, + lease?: TargetLeaseAuthority, + ) { + const state = executionAuthorityState(authority); + if (state.installation !== local) throw Error("Foreign data worker"); + let result: DataExecutionResult | undefined, + error: unknown, + partial = false; + try { + result = await worker; + } catch (value) { + if (value instanceof DataExecutionPartialResult) { + result = value.result; + partial = true; + } else error = value; + } + if (result !== undefined) { + try { + result = dataExecutionResultSchema.parse(result); + if (result.kind !== state.kind) throw Error("Worker result kind changed"); + } catch (value) { + result = undefined; + error = value; + } + } + let remotePending = false; + try { + const remote = verifyControl( + installation.publicKey, + "openmapx-data-runner-journal-v1", + readPrivateRecord( + local, + "observations", + `runners-${state.execution.operationKey}.json`, + 65536, + ), + runnerJournalSchema, + ); + remotePending = remote.attempts.length > 0; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const value = await load(authority.execution), + workerId = randomUUID(); + let recovery: "not_required" | "settled" | "pending"; + try { + recovery = await assertOwnedProxyRecoverySettled(authority.execution); + } catch { + recovery = "pending"; + } + const childrenDigest = controlDigest(value.row.execution_children), + publication = + remotePending || recovery === "pending" + ? "unknown" + : result + ? "committed" + : value.status.publication; + if (remotePending) { + result = undefined; + error = Error("Remote runner containment remains unknown"); + } + if (!state.origin) throw Error("Dated contained worker origin required"); + const proof = workerSchema.parse({ + version: 1, + installationId: installation.installationId, + origin: state.origin, + execution: state.execution, + admissionDigest: state.receiptDigest, + workerId, + containedAt: Date.now(), + outcome: result ? (partial ? "partial" : "returned") : "failed", + publication, + childrenDigest, + proxyRecovery: recovery, + ...(result ? { result } : {}), + }); + // Only the live producer, after awaiting the actual dispatcher promise, + // creates this immutable completion. Recovery only reads its signature. + writePrivateRecord( + local, + "observations", + `worker-${state.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-data-worker-contained-v1", proof), + true, + state.origin, + ); + const completion = Object.freeze({ [completionBrand]: true as const }); + completions.set(completion, { + authority, + workerId, + publication, + childrenDigest, + partial, + result, + error, + ...(lease ? { lease } : {}), + }); + return { completion, result, error, partial }; + }, + async restoreCompletion(execution: ExecutionRef) { + const value = await load(execution), + authority = restoreAdmittedControlExecution(installation, execution), + state = executionAuthorityState(authority); + const proof = verifyControl( + installation.publicKey, + "openmapx-data-worker-contained-v1", + readPrivateRecord(local, "observations", `worker-${execution.operationKey}.json`, 65536), + workerSchema, + ); + if ( + proof.installationId !== installation.installationId || + proof.admissionDigest !== state.receiptDigest || + !sameControlOrigin(proof.origin, state.origin) || + controlDigest(proof.execution) !== controlDigest(execution) || + proof.containedAt < value.status.admittedAt || + (proof.result && proof.result.kind !== value.status.kind) + ) + throw Error("Original contained worker identity changed"); + if ( + (proof.childrenDigest && + proof.childrenDigest !== controlDigest(value.row.execution_children)) || + (!proof.childrenDigest && value.row.execution_children.length > 0) + ) + throw Error("Original worker child snapshot changed"); + await assertOwnedProxyRecoverySettled(execution); + const lease = + value.status.kind === "generation-hardlinks" + ? restoreGenerationHardlinksLeaseForSettlement(authority) + : restoreArtifactTargetLeaseForSettlement(authority), + error = proof.outcome === "failed" ? Error("Original contained worker failed") : undefined; + const completion = Object.freeze({ [completionBrand]: true as const }); + completions.set(completion, { + authority, + workerId: proof.workerId, + partial: proof.outcome === "partial", + publication: proof.publication, + childrenDigest: proof.childrenDigest, + result: proof.result, + error, + lease, + }); + return { + completion, + authority, + lease, + result: proof.result, + error, + publication: proof.publication, + partial: proof.outcome === "partial", + }; + }, + async prepareTerminal( + completion: ContainedDataExecution, + status: DataExecutionStatus, + terminalAt: number, + ) { + const owned = completions.get(completion); + if (!owned) throw Error("Actual contained data worker required"); + const { authority } = owned, + s = executionAuthorityState(authority), + value = await load(authority.execution), + terminal = dataExecutionStatusSchema.parse(status); + await assertOwnedProxyRecoverySettled(authority.execution); + if ( + owned.childrenDigest !== undefined && + owned.childrenDigest !== controlDigest(value.row.execution_children) + ) + throw Error("Contained worker/current child union changed"); + if ( + !s.origin || + !isDataExecutionTerminal(terminal) || + terminal.publication === "unknown" || + terminal.id !== value.status.id || + terminal.operationKey !== value.status.operationKey || + terminal.fingerprint !== value.status.fingerprint || + terminal.admittedAt !== value.status.admittedAt || + terminal.deadlineAt !== value.status.deadlineAt || + terminal.kind !== value.status.kind || + !Number.isSafeInteger(terminalAt) || + terminalAt < terminal.admittedAt + ) + throw Error("Invalid original data terminal tuple"); + if ( + terminal.result !== undefined && + canonicalJson(terminal.result) !== canonicalJson(owned.result) + ) + throw Error("Terminal result differs from actual dispatcher result"); + if ( + owned.result && + terminal.status !== + (owned.result.kind === "transit-sync" + ? owned.result.finalStatus + : owned.partial + ? "partial" + : "ok") + ) + throw Error("Terminal status differs from actual dispatcher outcome"); + if (terminal.status === "ok" && owned.error !== undefined) + throw Error("Failed dispatcher cannot succeed"); + const children = value.row.execution_children.map((raw) => childSchema.parse(raw)); + for (const child of children) { + if (child.state !== "terminal" || !child.execution || !child.settlement || !owned.lease) + throw Error("Unsettled data descendant"); + const proof = inspectArtifactTargetLeaseChildSettlement(owned.lease, child.execution); + if ( + !proof || + controlDigest(proof) !== + controlDigest({ + identity: child.execution, + ...child.settlement, + publication: child.publication, + }) + ) + throw Error("Original shared child settlement changed"); + } + const childrenDigest = controlDigest(children), + digest = dataTerminalReceiptDigest(terminal); + return withPrivateEffectWriter(s, () => { + const raw = readPrivateRecord( + local, + "observations", + `terminal-${s.execution.operationKey}.json`, + 8192, + ), + prior = raw === null ? undefined : readProof(authority); + const original = prior?.terminalAt ?? terminalAt; + if ( + prior && + (prior.terminalDigest !== digest || + prior.workerId !== owned.workerId || + prior.childrenDigest !== childrenDigest) + ) + throw Error("Original data terminal tuple is immutable"); + assertFamilyCanonicalCompatibility(local, s.origin!, s.execution, digest, original); + const payload = terminalSchema.parse({ + version: 1, + phase: "prepared", + installationId: installation.installationId, + origin: s.origin, + execution: s.execution, + admissionDigest: s.receiptDigest, + terminalDigest: digest, + terminalAt: original, + workerId: owned.workerId, + childrenDigest, + }); + writePrivateRecord( + local, + "observations", + `terminal-${s.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-data-terminal-v1", payload), + false, + s.origin, + ); + const receipt = { + kind: "data-execution" as const, + status: terminal, + workerId: owned.workerId, + childrenDigest, + }, + outcome = + terminal.publication === "committed" ? ("committed" as const) : ("contained" as const); + const effect = { + identity: s.execution, + outcome, + publication: terminal.publication, + terminalReceiptDigest: controlDigest(receipt), + receipt, + }; + settleExecutionLifecycle(s); + writePrivateRecord( + local, + "effects", + `${s.execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-control-settlement-v1", effect), + false, + s.origin, + ); + markFamilyTerminal(local, s.origin!, s.execution, true, controlDigest(effect)); + return original; + }); + }, + async publishTerminal(execution: ExecutionRef) { + await assertOwnedProxyRecoverySettled(execution); + const value = await load(execution), + authority = restoreAdmittedControlExecution(installation, execution), + s = executionAuthorityState(authority), + proof = readProof(authority); + if ( + !isDataExecutionTerminal(value.status) || + value.row.finished_at?.getTime() !== proof.terminalAt || + dataTerminalReceiptDigest(value.status) !== proof.terminalDigest || + controlDigest(value.row.execution_children) !== proof.childrenDigest + ) + throw Error("Original committed data terminal tuple/time required"); + verifyControlEffectSettlement(installation, execution); + withPrivateEffectWriter(s, () => { + if (proof.phase !== "canonical") + writePrivateRecord( + local, + "observations", + `terminal-${execution.operationKey}.json`, + signControl(local.privateKey, "openmapx-data-terminal-v1", { + ...proof, + phase: "canonical", + }), + false, + s.origin, + ); + publishFamilyCanonicalTerminal( + local, + proof.origin, + execution, + true, + proof.terminalDigest, + proof.terminalAt, + ); + completeFamilyJournal(local, proof.origin, execution, proof.terminalAt); + }); + contained.set(execution.operationKey, proof.terminalDigest); + }, + async publishContainedTerminal(execution: ExecutionRef) { + const { value, authority, proof } = await verifyStored(execution); + verifyRetainedArtifactTargetRootRelease(authority); + publishDataControlTerminalWitness(authority, value.status, proof.terminalAt); + }, + repairTerminal: async (execution: ExecutionRef) => { + await assertOwnedProxyRecoverySettled(execution); + // existing signed commitment only; never reconstruct one from a row + const authority = restoreAdmittedControlExecution(installation, execution); + readProof(authority); + await (async () => { + const value = await load(execution); + if (!isDataExecutionTerminal(value.status)) throw Error("Data terminal row required"); + })(); + // Use the same exact tuple/time verifier and publisher above via the fixed method. + return authority; + }, + close() { + if (local.retentionJournal === journal) local.retentionJournal = undefined; + contained.clear(); + }, + }; +} +export type DataExecutionControlJournal = ReturnType; diff --git a/packages/core/src/build-artifacts/data-execution-outcome.ts b/packages/core/src/build-artifacts/data-execution-outcome.ts new file mode 100644 index 000000000..d1cdfc5ad --- /dev/null +++ b/packages/core/src/build-artifacts/data-execution-outcome.ts @@ -0,0 +1,14 @@ +import { + type DataExecutionResult, + dataExecutionResultSchema, +} from "../services/data-manager/contracts/executions"; +/** A trusted worker reports a completed publication with an incomplete optional + * stage. This is an outcome, not containment authority: the journal still awaits + * that worker and every child before signing its original terminal commitment. */ +export class DataExecutionPartialResult extends Error { + readonly result: DataExecutionResult; + constructor(result: DataExecutionResult) { + super("Data publication completed with an incomplete stage"); + this.result = dataExecutionResultSchema.parse(result); + } +} diff --git a/packages/core/src/build-artifacts/feed-proxy-validation.ts b/packages/core/src/build-artifacts/feed-proxy-validation.ts new file mode 100644 index 000000000..866afb922 --- /dev/null +++ b/packages/core/src/build-artifacts/feed-proxy-validation.ts @@ -0,0 +1,419 @@ +/** Closed ops-agent producer proof for completed nginx read-validation only. */ +import { z } from "zod"; +import { type OpsJobStatus, parseOpsJobStatus } from "../ops/contract"; +import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; +import type { ExecutionRef } from "../ops/execution-ref"; +import { executionRefSchema } from "../ops/execution-ref"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +// biome-ignore-start lint/suspicious/noImportCycles: This fixed producer shares the existing nominal authority/effect writer; it is invoked only after module initialization. +import { + assertControlExecutionForward, + controlDigest, + executionAuthorityState, + readPrivateRecord, + type VerifiedExecutionAuthority, + withPrivateEffectWriter, + writePrivateRecord, +} from "./control-authority"; +// biome-ignore-end lint/suspicious/noImportCycles: Fixed live producer dependency described above. +import { + controlOriginSchema, + sameControlOrigin, + signControl, + verifyControl, +} from "./control-records"; +// biome-ignore lint/suspicious/noImportCycles: The finite producer rechecks the same retained candidate at invocation, after schemas initialize. +import { verifyDataTransitCandidateForExecution } from "./data-execution-candidate"; +import { AnchoredDirectory } from "./filesystem"; +import { familyRecordWrite, verifyFamilyOwnedRecord } from "./retention-family"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const schema = z + .strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: digest, + candidateId: z.string().max(64), + candidateDigest: digest, + reservedAt: z.number().int().nonnegative().safe(), + phase: z.enum(["validation_pending", "validation_failed", "reload_started"]), + completedAt: z.number().int().nonnegative().safe().nullable(), + validationExitCode: z.number().int().min(1).max(255).nullable(), + completionDigest: digest.nullable(), + padding: z.string().max(4096), + }) + .superRefine((p, ctx) => { + if ( + (p.phase === "validation_failed") !== + (p.validationExitCode !== null && p.completionDigest !== null) || + (p.phase === "validation_pending") !== (p.completedAt === null) + ) + ctx.addIssue({ code: "custom", message: "Invalid fixed validation phase" }); + }); +type Proof = z.infer; +const domain = "openmapx-feed-proxy-validation-v2"; +const name = (authority: VerifiedExecutionAuthority) => + `proxy-validation-${authority.execution.operationKey}.json`; +function state(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority); + if ( + s.installation.installation.role !== "ops-agent" || + s.kind !== "feedProxy.validateAndReload" || + !s.origin + ) + throw Error("Original fixed proxy authority required"); + return s; +} +function read(authority: VerifiedExecutionAuthority) { + const s = state(authority), + raw = readPrivateRecord(s.installation, "observations", name(authority), 4096); + const p = verifyControl(s.installation.installation.publicKey, domain, raw, schema); + if ( + p.installationId !== s.installation.installation.installationId || + !sameControlOrigin(p.origin, s.origin) || + p.admissionDigest !== s.receiptDigest || + controlDigest(p.execution) !== controlDigest(s.execution) || + opsOperationFingerprint({ kind: "feedProxy.validateAndReload", candidateId: p.candidateId }) !== + s.execution.operationFingerprint + ) + throw Error("Original validation proof identity changed"); + verifyFamilyOwnedRecord( + s.installation, + p.origin, + { scope: "private", namespace: "observations", container: "", name: name(authority) }, + canonicalJson(raw), + ); + return p; +} +function write(authority: VerifiedExecutionAuthority, value: Proof, exclusive = false) { + const s = state(authority), + p = { ...value, padding: "" }; + // Account the entire cell up front; transitions allocate no additional space. + const signed = signControl(s.installation.privateKey, domain, p); + const room = 4096 - Buffer.byteLength(canonicalJson(signed)); + if (room < 0) throw Error("Validation proof capacity exceeded"); + p.padding = " ".repeat(room); + const parsed = schema.parse(p); + writePrivateRecord( + s.installation, + "observations", + name(authority), + signControl(s.installation.privateKey, domain, parsed), + exclusive, + s.origin, + ); +} +export function feedProxyValidationCompletionDigest( + execution: ExecutionRef, + candidateDigest: string, + nonce: string, + validationExitCode: number, +): string { + return controlDigest({ + domain: "openmapx-feed-proxy-validation-completion-v1", + execution, + candidateDigest, + nonce, + validationExitCode, + }); +} +export function reserveFeedProxyValidation( + authority: VerifiedExecutionAuthority, + candidateId: string, + candidateDigest: string, +): void { + const s = state(authority); + assertControlExecutionForward(authority); + verifyDataTransitCandidateForExecution( + authority, + { kind: "feedProxy.validateAndReload", candidateId }, + candidateDigest, + ); + withPrivateEffectWriter(s, () => { + let raw: unknown; + try { + raw = readPrivateRecord(s.installation, "observations", name(authority), 4096); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (raw !== null && raw !== undefined) { + const prior = read(authority); + if (prior.candidateId !== candidateId || prior.candidateDigest !== candidateDigest) + throw Error("Validation candidate changed"); + const dir = witnessDirectory(s.installation); + try { + dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096); + } finally { + dir.close(); + } + return; + } + write( + authority, + schema.parse({ + version: 2, + installationId: s.installation.installation.installationId, + origin: s.origin, + execution: s.execution, + admissionDigest: s.receiptDigest, + candidateId, + candidateDigest, + reservedAt: Date.now(), + phase: "validation_pending", + completedAt: null, + validationExitCode: null, + completionDigest: null, + padding: "", + }), + true, + ); + reserveWitness(authority); + }); +} +export function commitFeedProxyValidationFailure( + authority: VerifiedExecutionAuthority, + input: { candidateDigest: string; validationExitCode: number; completionDigest: string }, +): void { + const s = state(authority); + withPrivateEffectWriter(s, () => { + const p = read(authority); + if (p.candidateDigest !== input.candidateDigest) throw Error("Foreign validation candidate"); + if (p.phase === "validation_failed") { + if ( + p.validationExitCode !== input.validationExitCode || + p.completionDigest !== input.completionDigest + ) + throw Error("Validation commitment changed"); + return; + } + if (p.phase !== "validation_pending") throw Error("Reload already attempted"); + // A new producer commitment verifies the actual invocation's bytes. An + // identical existing commitment above observes its frozen original proof, + // which remains valid after a later legitimate restore replaces files. + verifyDataTransitCandidateForExecution( + authority, + { kind: "feedProxy.validateAndReload", candidateId: p.candidateId }, + p.candidateDigest, + ); + write( + authority, + schema.parse({ ...p, ...input, phase: "validation_failed", completedAt: Date.now() }), + ); + }); +} +export function recordFeedProxyReloadAttempted(authority: VerifiedExecutionAuthority): void { + const s = state(authority); + withPrivateEffectWriter(s, () => { + const p = read(authority); + if (p.phase !== "validation_pending") throw Error("Validation phase already consumed"); + verifyDataTransitCandidateForExecution( + authority, + { kind: "feedProxy.validateAndReload", candidateId: p.candidateId }, + p.candidateDigest, + ); + write(authority, { ...p, phase: "reload_started", completedAt: Date.now() }); + }); +} +/** Frozen original proof stays valid after a legitimate restore replaces files. */ +export function inspectFeedProxyValidationFailure( + authority: VerifiedExecutionAuthority, + expectedDigest?: string, +): string | null { + const p = read(authority); + if (p.phase !== "validation_failed") return null; + const proofDigest = controlDigest(p); + if (expectedDigest !== undefined && expectedDigest !== proofDigest) + throw Error("Foreign validation commitment"); + return proofDigest; +} + +const terminalSchema = z.strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: digest, + phase: z.enum(["pending", "canonical"]), + status: z.unknown().nullable(), + terminalAt: z.number().int().positive().safe().nullable(), + terminalDigest: digest.nullable(), + padding: z.string().max(4096), +}); +function witnessDirectory(local: ReturnType["installation"]) { + const data = new AnchoredDirectory(local.dataRoot), + control = data.directory(".admin-target-control"), + executions = control.directory("executions"), + role = executions.directory("ops-agent"); + return { + role, + close() { + role.close(); + executions.close(); + control.close(); + data.close(); + }, + }; +} +function writeWitness( + authority: VerifiedExecutionAuthority, + payload: z.infer, + exclusive: boolean, +) { + const s = state(authority), + dir = witnessDirectory(s.installation), + name = `${s.execution.operationKey}.proxy-terminal.json`; + try { + const value = { ...payload, padding: "" }, + domain = "openmapx-proxy-canonical-terminal-v2"; + const room = + 4096 - + Buffer.byteLength(canonicalJson(signControl(s.installation.privateKey, domain, value))); + if (room < 0) throw Error("Proxy terminal cell capacity exceeded"); + value.padding = " ".repeat(room); + const bytes = canonicalJson(signControl(s.installation.privateKey, domain, value)); + familyRecordWrite( + s.installation, + payload.origin, + { scope: "shared", namespace: "execution", container: "ops-agent", name }, + bytes, + (before) => dir.role.write(name, bytes, exclusive, before), + exclusive, + ); + } finally { + dir.close(); + } +} +function reserveWitness(authority: VerifiedExecutionAuthority) { + const s = state(authority); + writeWitness( + authority, + terminalSchema.parse({ + version: 2, + installationId: s.installation.installation.installationId, + origin: s.origin, + execution: s.execution, + admissionDigest: s.receiptDigest, + phase: "pending", + status: null, + terminalAt: null, + terminalDigest: null, + padding: "", + }), + true, + ); +} +/** Only an already sealed original complete canonical tuple can publish this + * finite shared proxy witness. It creates no new terminal time or status. */ +export function publishFeedProxyCanonicalTerminal( + authority: VerifiedExecutionAuthority, + input: OpsJobStatus, + terminalAt: number, +) { + const s = state(authority), + status = parseOpsJobStatus(input), + terminalDigest = controlTerminalReceiptDigest(status); + const original = verifyControl( + s.installation.installation.publicKey, + "openmapx-control-terminal-v1", + readPrivateRecord( + s.installation, + "observations", + `terminal-${s.execution.operationKey}.json`, + 8192, + ), + z + .object({ + phase: z.literal("canonical"), + execution: executionRefSchema, + admissionDigest: digest, + terminalAt: z.number(), + terminalDigest: digest, + }) + .passthrough(), + ); + if ( + controlDigest(original.execution) !== controlDigest(s.execution) || + original.admissionDigest !== s.receiptDigest || + original.terminalAt !== terminalAt || + original.terminalDigest !== terminalDigest + ) + throw Error("Original canonical proxy terminal required"); + const dir = witnessDirectory(s.installation); + try { + const prior = verifyControl( + s.installation.installation.publicKey, + "openmapx-proxy-canonical-terminal-v2", + JSON.parse( + dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), + ), + terminalSchema, + ); + if ( + prior.installationId !== s.installation.installation.installationId || + prior.admissionDigest !== s.receiptDigest || + !sameControlOrigin(prior.origin, s.origin) || + controlDigest(prior.execution) !== controlDigest(s.execution) + ) + throw Error("Original reserved proxy terminal changed"); + if (prior.phase === "canonical") { + if (prior.terminalDigest !== terminalDigest || prior.terminalAt !== terminalAt) + throw Error("Proxy canonical terminal changed"); + return; + } + // Diagnostics are observational and deliberately excluded from the exact + // canonical digest. Keep the fixed shared tuple bounded. + writeWitness( + authority, + { + ...prior, + phase: "canonical", + status: { ...status, diagnostics: { tail: [], truncated: false } }, + terminalDigest, + terminalAt, + }, + false, + ); + } finally { + dir.close(); + } +} +export function verifyFeedProxyCanonicalTerminal( + parent: VerifiedExecutionAuthority, + execution: ExecutionRef, +): OpsJobStatus { + const s = executionAuthorityState(parent), + dir = witnessDirectory(s.installation); + try { + const proof = verifyControl( + s.installation.publicKeys["ops-agent"], + "openmapx-proxy-canonical-terminal-v2", + JSON.parse( + dir.role.read(`${execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), + ), + terminalSchema, + ); + if ( + proof.phase !== "canonical" || + proof.installationId !== s.installation.installation.installationId || + !sameControlOrigin(proof.origin, s.origin) || + controlDigest(proof.execution) !== controlDigest(execution) || + !proof.terminalAt + ) + throw Error("Original canonical proxy child missing"); + const status = parseOpsJobStatus(proof.status); + if ( + status.kind !== "feedProxy.validateAndReload" || + status.operationId !== execution.executionId || + status.operationKey !== execution.operationKey || + status.fingerprint !== execution.fingerprint || + status.deadlineAt !== execution.deadlineAt || + controlTerminalReceiptDigest(status) !== proof.terminalDigest + ) + throw Error("Canonical proxy status changed"); + return status; + } finally { + dir.close(); + } +} diff --git a/packages/core/src/build-artifacts/filesystem.ts b/packages/core/src/build-artifacts/filesystem.ts index 1567c56fc..276c2eacf 100644 --- a/packages/core/src/build-artifacts/filesystem.ts +++ b/packages/core/src/build-artifacts/filesystem.ts @@ -165,7 +165,7 @@ export class AnchoredDirectory { closeSync(fd); } } - fileIdentity(name: string): FileIdentity { + fileIdentity(name: string, privateMode = true): FileIdentity { const path = this.name(name); const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); try { @@ -174,7 +174,7 @@ export class AnchoredDirectory { !value.isFile() || value.nlink !== 1n || value.uid !== BigInt(process.getuid?.() ?? -1) || - (value.mode & 0o077n) !== 0n || + (value.mode & BigInt(privateMode ? 0o077 : 0o022)) !== 0n || !sameInode(identity(value), identity(lstatSync(path, { bigint: true }))) ) throw Error("Unsafe retained record inode"); diff --git a/packages/core/src/build-artifacts/fixed-data-execution.ts b/packages/core/src/build-artifacts/fixed-data-execution.ts new file mode 100644 index 000000000..7ad9e8c1a --- /dev/null +++ b/packages/core/src/build-artifacts/fixed-data-execution.ts @@ -0,0 +1,282 @@ +/** Fixed administrative data-command plans and private write-ahead receipts. */ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { type OpsOperation, opsOperationSchema } from "../ops/contract"; +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { + type DataChildAttempt, + type DataExecutionInput, + dataChildAttemptSchema, + normalizeDataExecutionInput, + ordinaryDataExecutionInputSchema, +} from "../services/data-manager/contracts/executions"; +import { + type ControlChildReservation, + controlChildReservationSchema, + lookupControlChildPhase, +} from "./child-phase"; +import { + controlChildOperationKey, + controlDigest, + executionAuthorityState, + readPrivateRecord, + signControl, + type VerifiedExecutionAuthority, + verifyControl, + verifyDataControlTerminalWitness, + withPrivateEffectWriter, + writePrivateRecord, +} from "./control-authority"; + +const fixedDataKinds = [ + "data.downloadOsm", + "data.downloadFonts", + "data.convertOverpass", + "data.overtureSync", + "data.overtureConflate", + "data.searchIndexBuild", + "data.notablePlacesBuild", +]; +export function isFixedSingleDataCommand(kind: string): boolean { + return fixedDataKinds.includes(kind); +} +function uuidFor(key: string, purpose: string) { + const bytes = createHash("sha256") + .update(`openmapx-fixed-data-address-v1\0${key}\0${purpose}`) + .digest() + .subarray(0, 16); + bytes[6] = ((bytes[6] ?? 0) & 15) | 64; + bytes[8] = ((bytes[8] ?? 0) & 63) | 128; + const hex = bytes.toString("hex"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} +export const fixedDataBodyFingerprint = (input: DataExecutionInput) => + controlDigest(normalizeDataExecutionInput(input)); +export function fixedDataCommandInput( + operation: OpsOperation, + defaultRegion?: string, +): DataExecutionInput { + const region = "regionId" in operation ? (operation.regionId ?? defaultRegion) : defaultRegion; + switch (operation.kind) { + case "data.downloadOsm": + return ordinaryDataExecutionInputSchema.parse({ kind: "download-osm", region }); + case "data.downloadFonts": + return { kind: "download-fonts" }; + case "data.convertOverpass": + return ordinaryDataExecutionInputSchema.parse({ kind: "convert-overpass", region }); + case "data.overtureSync": + return ordinaryDataExecutionInputSchema.parse({ kind: "overture-sync", region }); + case "data.overtureConflate": + return ordinaryDataExecutionInputSchema.parse({ + kind: "overture-conflate", + region, + restart: operation.restart, + }); + case "data.searchIndexBuild": + return ordinaryDataExecutionInputSchema.parse({ kind: "search-index-build", region }); + case "data.notablePlacesBuild": + return { kind: "notable-places-build" }; + default: + throw Error("Only an existing matching single data command is supported"); + } +} +const bundleSchema = z.strictObject({ + version: z.literal(1), + parent: executionRefSchema, + operation: opsOperationSchema, + input: ordinaryDataExecutionInputSchema, + rootReservation: controlChildReservationSchema, + cliReservations: z.array(controlChildReservationSchema).length(1), + delegationId: z.uuid(), +}); +export function planFixedSingleDataCommand( + parent: ExecutionRef, + operation: OpsOperation, + serviceIds: readonly string[], + defaultRegion?: string, +) { + const input = fixedDataCommandInput(operation, defaultRegion), + cliId = uuidFor(parent.operationKey, "cli"), + cliKey = controlChildOperationKey(parent.operationKey, "data-workflow", 0), + targets = [...serviceIds].sort(); + const rootReservation: ControlChildReservation = { + version: 1, + phase: "data-workflow", + ordinal: 0, + kind: operation.kind, + recipient: "fixed-cli", + serviceIds: targets, + operationFingerprint: opsOperationFingerprint(operation), + execution: { + executor: "host-cli", + principal: parent.principal, + executionId: cliId, + operationKey: cliKey, + }, + }; + const cliReservations: ControlChildReservation[] = [ + { + version: 1, + phase: "data-step", + ordinal: 0, + kind: input.kind, + recipient: "data-manager", + serviceIds: targets, + operationFingerprint: fixedDataBodyFingerprint(input), + execution: { + executor: "data-manager", + principal: "host-cli", + executionId: uuidFor(parent.operationKey, "dm"), + operationKey: controlChildOperationKey(cliKey, "data-step", 0), + }, + }, + ]; + return bundleSchema.parse({ + version: 1, + parent, + operation, + input, + rootReservation, + cliReservations, + delegationId: uuidFor(parent.operationKey, "challenge"), + }); +} +export function retainFixedSingleDataPlan( + authority: VerifiedExecutionAuthority, + plan: ReturnType, +): void { + const s = executionAuthorityState(authority), + body = bundleSchema.parse(plan); + if ( + s.installation.installation.role !== "ops-agent" || + s.installation.fixedChildId || + controlDigest(body.parent) !== controlDigest(s.execution) || + opsOperationFingerprint(body.operation) !== s.execution.operationFingerprint || + !s.reservations.some((slot) => canonicalJson(slot) === canonicalJson(body.rootReservation)) + ) + throw Error("Original fixed data admission plan required"); + writePrivateRecord( + s.installation, + "observations", + `data-plan-${s.execution.operationKey}.json`, + signControl(s.installation.privateKey, "openmapx-fixed-data-plan-v1", body), + true, + s.origin, + ); +} +export function readFixedSingleDataPlan(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority), + body = verifyControl( + s.installation.installation.publicKey, + "openmapx-fixed-data-plan-v1", + readPrivateRecord( + s.installation, + "observations", + `data-plan-${s.execution.operationKey}.json`, + 8192, + ), + bundleSchema, + ); + if ( + controlDigest(body.parent) !== controlDigest(s.execution) || + opsOperationFingerprint(body.operation) !== s.execution.operationFingerprint || + !s.reservations.some((slot) => canonicalJson(slot) === canonicalJson(body.rootReservation)) + ) + throw Error("Retained fixed command plan changed"); + return body; +} +export function readCliDataAttempt( + authority: VerifiedExecutionAuthority, + key: string, +): DataChildAttempt | null { + const s = executionAuthorityState(authority); + let raw: unknown; + try { + raw = readPrivateRecord(s.installation, "observations", `data-attempt-${key}.json`, 8192); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + return verifyControl( + s.installation.installation.publicKey, + "openmapx-cli-data-attempt-v1", + raw, + dataChildAttemptSchema, + ); +} +export function retainCliDataAttempt( + authority: VerifiedExecutionAuthority, + value: DataChildAttempt, +): void { + const s = executionAuthorityState(authority), + attempt = dataChildAttemptSchema.parse(value), + slot = s.reservations.find((item) => item.execution.operationKey === attempt.operationKey), + binding = slot ? lookupControlChildPhase(authority, slot) : null; + if ( + !slot || + !binding || + !isFixedSingleDataCommand(s.kind) || + slot.phase !== "data-step" || + slot.kind !== attempt.input.kind || + slot.operationFingerprint !== fixedDataBodyFingerprint(attempt.input) || + attempt.operationFingerprint !== slot.operationFingerprint || + canonicalJson(attempt.parentExecution) !== canonicalJson(s.execution) || + attempt.requestedDeadlineAt !== binding.execution.deadlineAt || + (attempt.id && attempt.id !== binding.execution.executionId) || + (attempt.fingerprint && attempt.fingerprint !== binding.execution.fingerprint) + ) + throw Error("Exact retained data command binding required"); + withPrivateEffectWriter(s, () => { + const prior = readCliDataAttempt(authority, attempt.operationKey); + if (prior) { + const { state: _old, id: _id, fingerprint: _fp, deadlineAt: _deadline, ...before } = prior, + { state: _next, id: _id2, fingerprint: _fp2, deadlineAt: _deadline2, ...after } = attempt; + if ( + canonicalJson(before) !== canonicalJson(after) || + (prior.state !== "prepared" && attempt.state === "prepared") || + (prior.id && prior.id !== attempt.id) || + (prior.fingerprint && prior.fingerprint !== attempt.fingerprint) || + (prior.state === "terminal" && canonicalJson(prior) !== canonicalJson(attempt)) + ) + throw Error("Retained CLI data attempt is immutable"); + } + writePrivateRecord( + s.installation, + "observations", + `data-attempt-${attempt.operationKey}.json`, + signControl(s.installation.privateKey, "openmapx-cli-data-attempt-v1", attempt), + !prior, + s.origin, + ); + }); +} +export function verifyCliDataTerminal( + authority: VerifiedExecutionAuthority, + status: import("../services/data-manager/contracts/executions").DataExecutionStatus, +) { + const s = executionAuthorityState(authority), + slot = s.reservations.find((item) => item.execution.operationKey === status.operationKey), + binding = slot ? lookupControlChildPhase(authority, slot) : null; + if ( + !binding || + binding.execution.executionId !== status.id || + binding.execution.fingerprint !== status.fingerprint + ) + throw Error("Original bound data command required"); + return verifyDataControlTerminalWitness(s.installation.installation, binding.execution, status); +} + +export function cliDataReservation(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority), + slots = s.reservations.filter( + (slot) => slot.recipient === "data-manager" && slot.phase === "data-step", + ); + if (!isFixedSingleDataCommand(s.kind) || slots.length !== 1) + throw Error("Matching data command reservation required"); + return { + ...slots[0]!, + serviceIds: [...slots[0]!.serviceIds], + execution: { ...slots[0]!.execution }, + }; +} diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 59d1a343e..26210f2ea 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -1,6 +1,17 @@ +export { DataExecutionPartialResult } from "./data-execution-outcome"; // Server-only surface: intentionally absent from core's browser barrels. export * from "./archive"; +export { + bindControlChildPhase, + type ControlChildReservation, + consumeControlChildDispatch, + type FreshControlChildDispatch, + issueControlChildPhase, + lookupControlChildPhase, + lookupControlChildPhaseAttempt, + lookupControlChildPhaseBinding, +} from "./child-phase"; export { type ArtifactDelegationAssertion, acknowledgeControlRetention, @@ -12,6 +23,7 @@ export { CONTROL_RECOVERY_INSTRUCTIONS, type ControlAdoption, type ControlChildPhase, + ControlExecutionFencedError, type ControlExecutionOrigin, type ControlInstallation, type ControlPlannedChild, @@ -22,10 +34,13 @@ export { controlAdoptionSchema, controlChildOperationKey, controlHeaders, + controlInstallationDataRoot, controlRegisteredServiceIds, + controlRolePublicKey, fixedCliArguments, hasRetainedControlOrigin, loadControlStartup, + markFeedProxyReloadAttempted, openControlInstallation, openFixedChildInstallation, prepareFixedControlChild, @@ -52,11 +67,49 @@ export { verifyControlEffectSettlement, verifyControlPeer, verifyControlRecoveryAuthority, + verifyDataControlTerminalWitness, verifyFixedCliControlPeer, verifyParentExecution, verifyRetainedLocalControlPeer, } from "./control-authority"; export { CONTROL_RETENTION_LIMITS } from "./control-retention"; +export { createDataConfigurationFenceStore, type DataFenceSql } from "./data-configuration-fence"; +export { retainedDataDatabaseTimestamp } from "./data-database-time"; +export { + assertDataTransitCandidateForExecution, + assertOwnedDataTransitCandidate, + inspectDataTransitCandidate, + retainOwnedDataTransitCandidate, +} from "./data-execution-candidate"; +export { + type ContainedDataExecution, + createDataExecutionControlJournal, + type DataExecutionControlJournal, + dataTerminalReceiptDigest, +} from "./data-execution-journal"; +export { + commitFeedProxyValidationFailure, + feedProxyValidationCompletionDigest, + inspectFeedProxyValidationFailure, + reserveFeedProxyValidation, + verifyFeedProxyCanonicalTerminal, +} from "./feed-proxy-validation"; +export { + cliDataReservation, + fixedDataBodyFingerprint, + fixedDataCommandInput, + isFixedSingleDataCommand, + planFixedSingleDataCommand, + readCliDataAttempt, + readFixedSingleDataPlan, + retainCliDataAttempt, + retainFixedSingleDataPlan, + verifyCliDataTerminal, +} from "./fixed-data-execution"; export * from "./manifest"; -export { inspectControlAdmission, verifiedParentForArtifactDelegation } from "./peer-admission"; +export { + inspectControlAdmission, + inspectDataExecutionAdmission, + verifiedParentForArtifactDelegation, +} from "./peer-admission"; export * from "./target-lease"; diff --git a/packages/core/src/build-artifacts/peer-admission.ts b/packages/core/src/build-artifacts/peer-admission.ts index 2d1c6fb91..8c24c7c20 100644 --- a/packages/core/src/build-artifacts/peer-admission.ts +++ b/packages/core/src/build-artifacts/peer-admission.ts @@ -7,8 +7,10 @@ import { controlDigest, controlHeaders, installationState, + readParentGrant, retainedControlParent, verifyControlPeer, + verifyParentExecution, } from "./control-authority"; import { adoptArtifactTargetLease, inspectArtifactDelegation } from "./target-lease"; /** Both receivers keep transport auth and resource policy. This verifies only the exact signed internal attempt. */ @@ -110,3 +112,91 @@ export function verifiedParentForArtifactDelegation( if (inspected.origin) verifiedParentOrigins.set(parent, inspected.origin); return parent; } + +/** Fixed DM receiver: the only proposed ID comes from the authenticated grant. + * Reading the grant is observation; normalized equality is checked at adoption. */ +export function inspectDataExecutionAdmission( + installation: ControlInstallation, + input: { + rawHeaders: readonly string[]; + input: import("../services/data-manager/contracts/executions").DataExecutionInput; + operationKey: string; + operationFingerprint: string; + requestedDeadlineAt?: number; + }, +) { + const local = installationState(installation), + headers = controlHeaders(input.rawHeaders); + if (installation.role !== "data-manager") throw Error("Data-manager receiver required"); + if (!headers.parent && !headers.delegation) return null; + if (headers.parent) { + const grant = readParentGrant(local, headers.parent), + assertion = headers.parent; + if ( + grant.recipient !== "data-manager" || + grant.child.executor !== "data-manager" || + grant.kind !== input.input.kind || + grant.child.operationKey !== input.operationKey || + grant.child.operationFingerprint !== input.operationFingerprint || + grant.child.deadlineAt !== input.requestedDeadlineAt + ) + throw Error("Signed data parent identity mismatch"); + const peer = verifyControlPeer(installation, { + rawHeaders: input.rawHeaders, + transportPrincipal: "shared-data-manager-bearer", + execution: grant.child, + kind: grant.kind, + }); + const parent = verifyParentExecution(assertion, peer, { + execution: grant.child, + kind: grant.kind, + requestedDeadlineAt: grant.child.deadlineAt, + }); + return Object.freeze({ + execution: grant.child, + parent, + serviceIds: [] as string[], + async adopt(execution: ExecutionRef) { + if (controlDigest(execution) !== controlDigest(grant.child)) + throw Error("Normalized data child differs from grant"); + return undefined; + }, + }); + } + const assertion = headers.delegation!; + const grant = inspectArtifactDelegation({ dataRoot: local.dataRoot, assertion, installation }); + if ( + input.input.kind !== "generation-hardlinks" || + grant.kind !== input.input.kind || + grant.execution.executor !== "data-manager" || + grant.execution.operationKey !== input.operationKey || + grant.execution.operationFingerprint !== input.operationFingerprint || + grant.execution.deadlineAt !== input.requestedDeadlineAt + ) + throw Error("Unsupported or mismatched borrowed data leaf"); + const peer = verifyControlPeer(installation, { + rawHeaders: input.rawHeaders, + transportPrincipal: "shared-data-manager-bearer", + execution: grant.execution, + kind: grant.kind, + }); + const parent = verifiedParentForArtifactDelegation(installation, assertion, grant.execution); + return Object.freeze({ + execution: grant.execution, + parent, + serviceIds: [] as string[], + async adopt(execution: ExecutionRef) { + if (controlDigest(execution) !== controlDigest(grant.execution)) + throw Error("Normalized data child differs from grant"); + const lease = await adoptArtifactTargetLease({ + dataRoot: local.dataRoot, + assertion, + peer, + execution, + serviceIds: grant.serviceIds, + }); + local.registeredTargets = new Set([...local.registeredTargets, ...lease.serviceIds]); + return lease; + }, + }); +} diff --git a/packages/core/src/build-artifacts/peer-process.test.ts b/packages/core/src/build-artifacts/peer-process.test.ts index 2e3237ad7..0263f19f1 100644 --- a/packages/core/src/build-artifacts/peer-process.test.ts +++ b/packages/core/src/build-artifacts/peer-process.test.ts @@ -585,11 +585,14 @@ it("keeps paired-pin approval a one-use leaf under its original activation root" const execution = ref("data-manager", operation.kind, operation); const body = { kind: "transitousLock.approve" as const, - ref: `main@${"a".repeat(40)}`, - approvedBy: "synthetic-reviewer", + proposalId: randomUUID(), + validationId: operation.validationId, + expectedGenerationId: null, + diffDigest: "a".repeat(64), + acknowledgeLicenseChanges: false, }; const childRef = ref("ops-agent", body.kind, body, execution, "paired-pin-commit"); - // The current legacy kind has a 30 s budget and stays closed in the real adapter. + // The exact async120s leaf stays closed in the production adapter until05.3. childRef.deadlineAt = Date.now() + 20000; childRef.fingerprint = executionFingerprint({ kind: body.kind, diff --git a/packages/core/src/build-artifacts/retention-family.ts b/packages/core/src/build-artifacts/retention-family.ts index b8b2096c0..011b53d78 100644 --- a/packages/core/src/build-artifacts/retention-family.ts +++ b/packages/core/src/build-artifacts/retention-family.ts @@ -45,7 +45,13 @@ const PRIVATE = new Set([ "spawn-started", "prepared", "execution-adoptions", + "configuration-maintenance", + "hardlinks", ]); +const recordMaximum = (record: { scope: string; namespace: string }) => + record.scope === "private" && record.namespace === "hardlinks" + ? 1024 * 1024 + : LIMITS.privateRecordBytes; const SHARED = new Set([ "execution", "lease-owner", @@ -719,6 +725,38 @@ function equalAddress( a.container === b.container ); } +/** Fixed protocol readers compare the current indexed digest and physical + * record before accepting an older still-valid signed payload. */ +export function verifyFamilyOwnedRecord( + state: InstallationState, + origin: ControlExecutionOrigin, + address: Pick, + bytes: string, +): void { + const family = openFamily(state, origin); + try { + guard(family.slot, ".family-writer", () => { + checkLocalIdentity(family); + const retained = findRecord(family, address); + if (!retained || retained.state !== "written" || retained.digest !== controlDigest(bytes)) + throw Error("Retained record digest changed"); + const opened = recordDirectory(family, retained); + try { + if ( + !sameInode(opened.dir.fileIdentity(retained.name), retained.inode) || + controlDigest( + opened.dir.read(retained.name, recordMaximum(retained)).toString("utf8"), + ) !== retained.digest + ) + throw Error("Retained record identity changed"); + } finally { + opened.close(); + } + }); + } finally { + family.close(); + } +} export function familyRecordWrite( state: InstallationState, origin: ControlExecutionOrigin, @@ -743,8 +781,7 @@ export function familyRecordWrite( try { if ( sameInode(d.dir.fileIdentity(old.name), old.inode) && - controlDigest(d.dir.read(old.name, LIMITS.privateRecordBytes).toString("utf8")) === - old.digest + controlDigest(d.dir.read(old.name, recordMaximum(old)).toString("utf8")) === old.digest ) { ref = old; return; @@ -1472,7 +1509,7 @@ function publishCandidate(f: Family, p: Progress): Candidate | null { } } else if ( !sameInode(d.dir.fileIdentity(e.name), e.inode) || - controlDigest(d.dir.read(e.name, LIMITS.privateRecordBytes).toString("utf8")) !== e.digest + controlDigest(d.dir.read(e.name, recordMaximum(e)).toString("utf8")) !== e.digest ) throw Error("Retirement record replaced"); } finally { @@ -1672,7 +1709,7 @@ function unlinkShared(f: Family, p: Progress, e: FamilyRecord) { } } else if ( !sameInode(d.dir.fileIdentity(e.name), e.inode) || - controlDigest(d.dir.read(e.name, LIMITS.privateRecordBytes).toString("utf8")) !== e.digest + controlDigest(d.dir.read(e.name, recordMaximum(e)).toString("utf8")) !== e.digest ) throw Error("Replaced shared retirement record"); p.sharedTransaction = { ordinal: e.ordinal, phase: "planned", record: e }; @@ -1706,8 +1743,7 @@ function unlinkShared(f: Family, p: Progress, e: FamilyRecord) { } else { if ( !sameInode(d.dir.fileIdentity(e.name), e.inode) || - controlDigest(d.dir.read(e.name, LIMITS.privateRecordBytes).toString("utf8")) !== - e.digest + controlDigest(d.dir.read(e.name, recordMaximum(e)).toString("utf8")) !== e.digest ) throw Error("Replaced shared retirement record"); d.dir.removeFile(e.name); @@ -1775,6 +1811,7 @@ function cleanupLocalRecords(f: Family, p: Progress, m: Manifest): boolean { }; progress(f, p); }, + recordMaximum(e), ); delete p.lastDone; progress(f, p); @@ -2037,6 +2074,11 @@ function assertPrivateRetirementProof(f: Family, m: Manifest): void { async function step(f: Family): Promise { if (f.state.fixedChildId) throw Error("Fixed children do not coordinate role-private retirement"); let p = readProgress(f); + await f.state.retentionJournal?.prepare?.( + Object.values(p.executions) + .filter((fact) => fact.requiresJournal) + .map((fact) => ({ execution: fact.execution, digest: fact.terminalDigest })), + ); checkLocalIdentity(f); if (p.phase === "released" && f.state.installation.role !== f.meta.origin.issuer) { freeFamily(f, p); @@ -2115,7 +2157,11 @@ async function step(f: Family): Promise { .filter((e) => e.requiresJournal) .map((e) => { if (!e.terminalDigest) throw Error("Missing retiring terminal digest"); - return { execution: e.execution, digest: e.terminalDigest }; + return { + execution: e.execution, + digest: e.terminalDigest, + terminalAt: e.canonicalTerminalAt, + }; }), md, ); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index 621ad8433..e6e17de1b 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -8,10 +8,12 @@ import { executionRefSchema, } from "../ops/execution-identity"; import { canonicalJson } from "../ops/fingerprint"; +// biome-ignore lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { prepareControlChildPublication, resolveControlChildPlan } from "./child-phase"; +// biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { type ArtifactDelegationAssertion, type AuthorityState, - assertControlDelegationForward, assertControlExecutionForward, assertionSchema, type ChildSettlement, @@ -21,36 +23,41 @@ import { childPhaseKinds, childSettlementState, controlChildOperationKey, - controlDigest, - controlOriginSchema, controlPeerState, - digest, encodePrivate, executionAuthorityState, fixedControlChildId, type InstallationState, installationState, newControlKey, - originRecordDomain, parseControlAssertion, readControlExecutionWitness, readPrivateRecord, + recordLeaseControlAttempt, recoveryAuthorityState, retainedControlParent, - sameControlOrigin, - signControl, - targetSchema, - uuid, type VerifiedChildSettlement, type VerifiedControlPeer, type VerifiedExecutionAuthority, type VerifiedRecoveryAuthority, - validateOriginRecord, - verifyControl, verifyControlEffectSettlement, verifyControlPreparedTerminal, + withControlDelegationPublication, writePrivateRecord, } from "./control-authority"; +// biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. +import { + controlDigest, + controlOriginSchema, + digest, + originRecordDomain, + sameControlOrigin, + signControl, + targetSchema, + uuid, + validateOriginRecord, + verifyControl, +} from "./control-records"; import { CONTROL_RETENTION_LIMITS, reserveAuthoritySlot } from "./control-retention"; import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; import { assertOriginClockForward } from "./retention-clock"; @@ -64,6 +71,7 @@ import { trackFamilyDirectory, } from "./retention-family"; +// biome-ignore lint/suspicious/noImportCycles: Type re-export for the existing shared settlement protocol. export type { ChildSettlement } from "./control-authority"; export type ArtifactTargetLease = { readonly leaseId: string; @@ -201,6 +209,7 @@ interface State { verifiedAuthority?: VerifiedExecutionAuthority; closed: boolean; journalGuardHeld?: boolean; + settlementOnly?: boolean; slots?: AdmissionSlots; } const handles = new WeakMap(); @@ -412,7 +421,7 @@ function tree(s: State): LeaseTree { const parentRecipient = node === null ? s.certificate.owner.executor : grants.get(node)?.recipient; if (!parentRecipient) throw Error("Missing parent recipient"); - const expectedEdges = verifyFiniteEdge( + const expectedEdges = verifyFiniteControlEdge( parentRecipient, witness.kind, grant.phase as ControlChildPhase, @@ -888,7 +897,7 @@ export async function acquireArtifactTargetLease(options: { throw e; } } -function verifyFiniteEdge( +export function verifyFiniteControlEdge( role: ControlRecipient | "host-cli", kind: string, phase: ControlChildPhase, @@ -978,7 +987,7 @@ function permitted(s: State, child: ControlPlannedChild, parentGrant?: Grant): s s.execution, s.node === null ? s.installation.publicKeys[s.execution.executor] : key, ); - return verifyFiniteEdge( + return verifyFiniteControlEdge( parentGrant?.recipient ?? s.installation.installation.role, witness.kind, child.phase, @@ -1006,131 +1015,156 @@ export async function delegateArtifactTargetLease( }, ): Promise { const s = stateOf(lease); - if (s.verifiedAuthority) assertControlDelegationForward(s.verifiedAuthority, child.phase); - return guarded(s, () => { - const current = tree(s); - const node = - s.node === null - ? current.root - : required(current.children.get(s.node), "Missing retained child node"); - if (!forwardAncestry(current, s.node) || Date.now() >= s.execution.deadlineAt) - throw Error("Lease does not permit forward delegation"); - let plan = s.authority?.plan.find( - (p) => - controlDigest(p.execution) === controlDigest(child.execution) && - p.phase === child.phase && - p.recipient === child.recipient && - canonicalJson(p.serviceIds) === canonicalJson(child.serviceIds), - ); - if (!plan && s.node !== null) { - const inherited = required(current.grants.get(s.node), "Missing retained child grant"); - const kinds = childPhaseKinds[child.phase]; - if ( - inherited.permittedFurtherEdges.includes(child.phase) && - kinds.length === 1 && - child.execution.operationKey === - controlChildOperationKey(s.execution.operationKey, child.phase, 0) - ) - plan = { ...child, kind: required(kinds[0], "Missing finite child kind"), ordinal: 0 }; - } - if (!plan) throw Error("Child is absent from immutable admitted plan"); - if (current.grants.size >= 64) throw Error("Child intent capacity exhausted"); - if ( - [...current.grants.values()].some( - (g) => g.child.operationKey === child.execution.operationKey, - ) - ) - throw Error("Attempted child cannot be replayed"); - if (child.serviceIds.some((id) => !lease.serviceIds.includes(id))) - throw Error("Widened target subset"); - const parentGrant = s.node === null ? undefined : current.grants.get(s.node); - const edges = permitted(s, plan, parentGrant); + if (s.settlementOnly) throw Error("Recovered lease permits settlement only"); + const authority = s.verifiedAuthority; + if (!authority) throw Error("A durable local execution must own child grant publication"); + return guarded(s, () => + withControlDelegationPublication(authority, child.phase, () => + publishReservedLeaseGrant(s, lease, child, authority), + ), + ); +} +/** Lease → ancestor execution scopes → local execution scope stay held through + * private reference/outbox, cancellation index and shared/node publication. */ +function publishReservedLeaseGrant( + s: State, + lease: TargetLeaseAuthority, + child: { + execution: ExecutionRef; + phase: ControlChildPhase; + serviceIds: readonly string[]; + recipient: ControlRecipient; + childPublicKey: string; + }, + authority: VerifiedExecutionAuthority, +): ArtifactDelegationAssertion { + const current = tree(s); + const node = + s.node === null + ? current.root + : required(current.children.get(s.node), "Missing retained child node"); + if (!forwardAncestry(current, s.node) || Date.now() >= s.execution.deadlineAt) + throw Error("Lease does not permit forward delegation"); + let plan = resolveControlChildPlan(authority, child); + if (plan && canonicalJson(plan.serviceIds) !== canonicalJson(child.serviceIds)) + throw Error("Reserved child target subset changed"); + if (!plan && s.node !== null) { + const inherited = required(current.grants.get(s.node), "Missing retained child grant"); + const kinds = childPhaseKinds[child.phase]; if ( - s.authority?.kind === "configurationMaintenance.prepare" && - [ - "configuration-commit", - "maintenance-hardlinks", - "maintenance-recreate", - "maintenance-handoff", - ].includes(child.phase) + inherited.permittedFurtherEdges.includes(child.phase) && + kinds.length === 1 && + child.execution.operationKey === + controlChildOperationKey(s.execution.operationKey, child.phase, 0) ) - verifyControlPreparedTerminal(s.installation, s.execution); - let delegationId: string = randomUUID(); - if (child.recipient === "fixed-cli" || child.recipient === "configuration-helper") { - delegationId = fixedControlChildId(s.installation, child.childPublicKey, s.execution); - } else if (child.childPublicKey !== s.installation.publicKeys[child.recipient]) - throw Error("Child role key mismatch"); - if (!s.certificate.origin) throw Error("Legacy lease is observation-only; upgrade producer"); - assertOriginClockForward(s.installation, s.certificate.origin); - const grant: Grant = { - version: 2, - origin: s.certificate.origin, - installationId: s.certificate.installationId, - leaseId: lease.leaseId, - ownerCertificateDigest: controlDigest(s.certificateEnvelope), - delegationId, - parentNode: s.node, - parent: s.execution, - child: child.execution, - childKind: plan.kind, - phase: child.phase, - ordinal: plan.ordinal, - serviceIds: [...child.serviceIds], - recipient: child.recipient, - childPublicKey: child.childPublicKey, - permittedFurtherEdges: edges, - depth: (parentGrant?.depth ?? 0) + 1, - }; - const envelope = signControl(s.key, "openmapx-lease-grant-v2", grant); - const grantDigest = controlDigest(envelope); - const transition = signControl(s.key, "openmapx-lease-intent-v1", { - delegationId, - grantDigest, - sequence: 1 as const, - state: "attempted" as const, - }); - const ancestors: string[] = []; - let ancestor = s.node; - while (ancestor) { - ancestors.unshift(ancestor); - ancestor = current.grants.get(ancestor)?.parentNode ?? null; - } - const fields = { - version: 1 as const, - installationId: s.certificate.installationId, - leaseId: lease.leaseId, - ownerCertificateDigest: controlDigest(s.certificateEnvelope), - delegationId, - grantDigest, - ancestorDelegationIds: ancestors, - }; - const signature = signControl(s.key, "openmapx-lease-assertion-v1", fields).signature; - const assertion = Buffer.from(canonicalJson({ ...fields, signature })).toString( - "base64url", - ) as ArtifactDelegationAssertion; - parseControlAssertion(assertion); - enlistFamilyIntent( - s.installation, - s.certificate.origin, - delegationId, - child.execution, - child.recipient === "data-manager" ? "data-manager" : "ops-agent", - ); - writePrivateRecord( - s.installation, - "attempts", - `${delegationId}.json`, - { grant: envelope, transition, assertion }, - true, - s.certificate.origin, - ); - writeBounded(s, "grants", `${delegationId}.json`, { grant: envelope, transition }, true); - node.grantDigests[delegationId] = grantDigest; - node.sequence++; - writeNode(s, node); - tree(s); - return assertion; + plan = { ...child, kind: required(kinds[0], "Missing finite child kind"), ordinal: 0 }; + } + if (!plan) throw Error("Child is absent from immutable admitted plan"); + if (current.grants.size >= 64) throw Error("Child intent capacity exhausted"); + if ( + [...current.grants.values()].some((g) => g.child.operationKey === child.execution.operationKey) + ) + throw Error("Attempted child cannot be replayed"); + if (child.serviceIds.some((id) => !lease.serviceIds.includes(id))) + throw Error("Widened target subset"); + const parentGrant = s.node === null ? undefined : current.grants.get(s.node); + const edges = permitted(s, plan, parentGrant); + if ( + s.authority?.kind === "configurationMaintenance.prepare" && + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(child.phase) + ) + verifyControlPreparedTerminal(s.installation, s.execution); + let delegationId: string = randomUUID(); + if (child.recipient === "fixed-cli" || child.recipient === "configuration-helper") { + delegationId = fixedControlChildId(s.installation, child.childPublicKey, s.execution); + } else if (child.childPublicKey !== s.installation.publicKeys[child.recipient]) + throw Error("Child role key mismatch"); + if (!s.certificate.origin) throw Error("Legacy lease is observation-only; upgrade producer"); + assertOriginClockForward(s.installation, s.certificate.origin); + const grant: Grant = { + version: 2, + origin: s.certificate.origin, + installationId: s.certificate.installationId, + leaseId: lease.leaseId, + ownerCertificateDigest: controlDigest(s.certificateEnvelope), + delegationId, + parentNode: s.node, + parent: s.execution, + child: child.execution, + childKind: plan.kind, + phase: child.phase, + ordinal: plan.ordinal, + serviceIds: [...child.serviceIds], + recipient: child.recipient, + childPublicKey: child.childPublicKey, + permittedFurtherEdges: edges, + depth: (parentGrant?.depth ?? 0) + 1, + }; + const envelope = signControl(s.key, "openmapx-lease-grant-v2", grant); + const grantDigest = controlDigest(envelope); + const transition = signControl(s.key, "openmapx-lease-intent-v1", { + delegationId, + grantDigest, + sequence: 1 as const, + state: "attempted" as const, + }); + const ancestors: string[] = []; + let ancestor = s.node; + while (ancestor) { + ancestors.unshift(ancestor); + ancestor = current.grants.get(ancestor)?.parentNode ?? null; + } + const fields = { + version: 1 as const, + installationId: s.certificate.installationId, + leaseId: lease.leaseId, + ownerCertificateDigest: controlDigest(s.certificateEnvelope), + delegationId, + grantDigest, + ancestorDelegationIds: ancestors, + }; + const signature = signControl(s.key, "openmapx-lease-assertion-v1", fields).signature; + const assertion = Buffer.from(canonicalJson({ ...fields, signature })).toString( + "base64url", + ) as ArtifactDelegationAssertion; + parseControlAssertion(assertion); + enlistFamilyIntent( + s.installation, + s.certificate.origin, + delegationId, + child.execution, + child.recipient === "data-manager" ? "data-manager" : "ops-agent", + ); + prepareControlChildPublication(authority, plan, { + delegationId, + grantDigest, + leaseId: lease.leaseId, + }); + recordLeaseControlAttempt(authority, { + child: child.execution, + leaseId: lease.leaseId, + delegationId, + grantDigest, }); + writePrivateRecord( + s.installation, + "attempts", + `${delegationId}.json`, + { grant: envelope, transition, assertion }, + true, + s.certificate.origin, + ); + writeBounded(s, "grants", `${delegationId}.json`, { grant: envelope, transition }, true); + node.grantDigests[delegationId] = grantDigest; + node.sequence++; + writeNode(s, node); + tree(s); + return assertion; } function openState( dataRoot: string, @@ -1384,6 +1418,137 @@ export function bindBorrowedControlExecution( throw Error("Borrower execution authority mismatch"); s.authority = a; s.verifiedAuthority = authority; + if (a.kind === "generation-hardlinks" && a.installation.installation.role === "data-manager") { + if (!s.node || !a.origin) throw Error("Original generation adoption required"); + const payload = { + version: 2, + origin: a.origin, + execution: a.execution, + admissionDigest: a.receiptDigest, + leaseId: s.certificate.leaseId, + delegationId: s.node, + certificateDigest: controlDigest(s.certificateEnvelope), + }; + const name = `${a.execution.operationKey}.json`; + let prior: unknown; + try { + prior = readPrivateRecord(a.installation, "execution-adoptions", name, 8192); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (prior !== undefined) { + const saved = verifyControl( + a.installation.installation.publicKey, + "openmapx-generation-adoption-v2", + prior, + z.unknown(), + ); + if (controlDigest(saved) !== controlDigest(payload)) + throw Error("Generation adoption changed"); + } else + writePrivateRecord( + a.installation, + "execution-adoptions", + name, + signControl(a.installation.privateKey, "openmapx-generation-adoption-v2", payload), + true, + a.origin, + ); + } +} +/** Reopen only the original authenticated generation leaf for settlement. + * There is no adopt, target acquire, deadline renewal or owner-release path. */ +export function restoreGenerationHardlinksLeaseForSettlement( + authority: VerifiedExecutionAuthority, +): BorrowedArtifactTargetLease { + const a = executionAuthorityState(authority); + if ( + a.kind !== "generation-hardlinks" || + a.installation.installation.role !== "data-manager" || + !a.origin + ) + throw Error("Original generation receiver required"); + const retained = verifyControl( + a.installation.installation.publicKey, + "openmapx-generation-adoption-v2", + readPrivateRecord( + a.installation, + "execution-adoptions", + `${a.execution.operationKey}.json`, + 8192, + ), + z.strictObject({ + version: z.literal(2), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: digest, + leaseId: uuid, + delegationId: uuid, + certificateDigest: digest, + }), + ); + if ( + controlDigest(retained.execution) !== controlDigest(a.execution) || + retained.admissionDigest !== a.receiptDigest || + !sameControlOrigin(retained.origin, a.origin) + ) + throw Error("Generation retained adoption identity changed"); + const s = openState(a.installation.dataRoot, retained.leaseId, a.installation); + try { + return guarded(s, () => { + const adoption = z + .object({ + state: z.literal("ready"), + execution: executionRefSchema, + assertion: z.string(), + grantDigest: digest, + }) + .parse( + readPrivateRecord(a.installation, "adoptions", `${retained.delegationId}.json`, 8192), + ); + const assertion = parseControlAssertion(adoption.assertion); + if ( + assertion.leaseId !== retained.leaseId || + assertion.delegationId !== retained.delegationId || + assertion.grantDigest !== adoption.grantDigest || + retained.certificateDigest !== controlDigest(s.certificateEnvelope) || + controlDigest(adoption.execution) !== controlDigest(a.execution) || + !sameControlOrigin(a.origin, s.certificate.origin) + ) + throw Error("Original ready generation grant required"); + s.node = retained.delegationId; + s.execution = a.execution; + s.key = a.installation.privateKey; + s.authority = a; + s.verifiedAuthority = authority; + s.settlementOnly = true; + const t = tree(s), + grant = t.grants.get(s.node), + child = t.children.get(s.node); + if ( + !grant || + !child || + grant.recipient !== "data-manager" || + grant.childKind !== "generation-hardlinks" || + grant.childPublicKey !== a.installation.installation.publicKey || + controlDigest(grant.child) !== controlDigest(a.execution) || + controlDigest(child.execution) !== controlDigest(a.execution) + ) + throw Error("Original generation child is missing or foreign"); + const handle = Object.freeze({ + leaseId: retained.leaseId, + owner: s.certificate.owner, + execution: a.execution, + serviceIds: Object.freeze([...grant.serviceIds]), + mode: "borrowed" as const, + }); + handles.set(handle, s); + return handle; + }); + } catch (error) { + closeState(s); + throw error; + } } export async function settleArtifactTargetLeaseChild( lease: TargetLeaseAuthority, @@ -1414,6 +1579,27 @@ export async function settleArtifactTargetLeaseChild( tree(s); }); } +/** Verify the exact agent-signed shared child node; no sibling private mount or + * caller status can stand in for actual target containment. */ +export function inspectArtifactTargetLeaseChildSettlement( + lease: TargetLeaseAuthority, + execution: ExecutionRef, +): ChildSettlement | null { + const s = stateOf(lease); + return guarded(s, () => { + const t = tree(s), + entries = [...t.grants.values()].filter( + (grant) => + grant.parentNode === s.node && controlDigest(grant.child) === controlDigest(execution), + ); + if (entries.length !== 1) throw Error("Exact retained child grant required"); + const child = t.children.get(entries[0]!.delegationId); + if (!child || child.phase !== "settled" || !knownOutcome(child.settlement)) return null; + if (controlDigest(child.execution) !== controlDigest(execution)) + throw Error("Shared child execution changed"); + return child.settlement; + }); +} function finishOwnedRelease(s: State): void { guarded( s, @@ -1579,6 +1765,87 @@ export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Pr finishOwnedRelease(s); closeState(s); } +/** Read-only original root release proof for a data command's parent. */ +export function verifyRetainedArtifactTargetRootRelease( + authority: VerifiedExecutionAuthority, +): void { + const a = executionAuthorityState(authority), + reference = z + .strictObject({ leaseId: uuid, certificateDigest: digest }) + .parse( + readPrivateRecord(a.installation, "execution-leases", `${a.execution.operationKey}.json`), + ); + const s = openState(a.installation.dataRoot, reference.leaseId, a.installation, true); + try { + const t = tree(s), + effect = childSettlementState( + verifyControlEffectSettlement(a.installation.installation, a.execution), + ).evidence; + if ( + controlDigest(s.execution) !== controlDigest(a.execution) || + controlDigest(s.certificateEnvelope) !== reference.certificateDigest || + s.slots?.release?.phase !== "released" || + t.root.phase !== "settled" || + controlDigest(t.root.settlement) !== controlDigest(effect) || + [...t.grants.keys()].some( + (id) => + t.children.get(id)?.phase !== "settled" || !knownOutcome(t.children.get(id)?.settlement), + ) + ) + throw Error("Original root release is not proved"); + } finally { + closeState(s); + } +} +/** Reopen the original owned lease for cancellation/settlement only. This never + * acquires a target, creates a certificate, renews a deadline or permits grants. */ +export function restoreArtifactTargetLeaseForSettlement( + authority: VerifiedExecutionAuthority, +): ArtifactTargetLease { + const a = executionAuthorityState(authority); + const retained = z + .strictObject({ leaseId: uuid, certificateDigest: digest }) + .parse( + readPrivateRecord(a.installation, "execution-leases", `${a.execution.operationKey}.json`), + ); + const s = openState(a.installation.dataRoot, retained.leaseId, a.installation); + try { + const saved = z + .strictObject({ + certificate: signedCertificateSchema, + privateKey: z.string(), + executionReceiptDigest: digest, + }) + .parse(readPrivateRecord(a.installation, "leases", `${retained.leaseId}.json`)); + if ( + controlDigest(s.execution) !== controlDigest(a.execution) || + retained.certificateDigest !== controlDigest(s.certificateEnvelope) || + controlDigest(saved.certificate) !== retained.certificateDigest || + saved.executionReceiptDigest !== a.receiptDigest || + !sameControlOrigin(a.origin, s.certificate.origin) + ) + throw Error("Original retained lease identity changed"); + s.key = createPrivateKey({ + key: Buffer.from(saved.privateKey, "base64url"), + type: "pkcs8", + format: "der", + }); + s.authority = a; + s.verifiedAuthority = authority; + s.settlementOnly = true; + const lease = Object.freeze({ + leaseId: retained.leaseId, + owner: s.execution, + serviceIds: s.certificate.serviceIds, + mode: "owner" as const, + }); + handles.set(lease, s); + return lease; + } catch (error) { + closeState(s); + throw error; + } +} export async function reconcileArtifactTargetLease(options: { dataRoot: string; leaseId: string; @@ -1726,6 +1993,8 @@ export function inspectArtifactDelegation(options: { throw Error("Grant digest mismatch"); return Object.freeze({ leaseId: parsed.leaseId, + ownerCertificateDigest: controlDigest(s.certificateEnvelope), + owner: s.certificate.owner, origin: grant.origin, parent: grant.parent, execution: grant.child, @@ -1791,6 +2060,7 @@ export function verifyConfigurationHelperReady(lease: BorrowedArtifactTargetLeas /** Observation/adoption of an old identity is not permission to start another forward effect. */ export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): void { const s = stateOf(lease); + if (s.settlementOnly) throw Error("Recovered lease permits settlement only"); if (!s.certificate.origin) throw Error("Legacy lease is observation/settlement only; upgrade producer"); assertOriginClockForward(s.installation, s.certificate.origin); diff --git a/packages/core/src/build-artifacts/test-fixtures/child-phase-process.ts b/packages/core/src/build-artifacts/test-fixtures/child-phase-process.ts new file mode 100644 index 000000000..67e0f132f --- /dev/null +++ b/packages/core/src/build-artifacts/test-fixtures/child-phase-process.ts @@ -0,0 +1,30 @@ +import { writeSync } from "node:fs"; +import { createInterface } from "node:readline"; +import { bindControlChildPhase } from "../child-phase"; +import { + closeControlInstallation, + openControlInstallation, + restoreAdmittedControlExecution, +} from "../control-authority"; + +const lines = createInterface({ input: process.stdin }); +for await (const line of lines) { + const input = JSON.parse(line); + let installation: ReturnType | undefined; + try { + installation = openControlInstallation({ + infraDir: input.root, + role: "data-manager", + serviceIds: ["motis", "motis-staging"], + }); + const authority = restoreAdmittedControlExecution(installation, input.execution); + const child = await bindControlChildPhase(authority, { phase: "stop-primary", ordinal: 0 }); + writeSync(1, JSON.stringify({ ok: true, child }) + "\n"); + } catch (error) { + writeSync(1, JSON.stringify({ ok: false, error: String(error) }) + "\n"); + } finally { + if (installation) closeControlInstallation(installation); + } + break; +} +lines.close(); diff --git a/packages/core/src/ops/catalog-pin-approval.test.ts b/packages/core/src/ops/catalog-pin-approval.test.ts new file mode 100644 index 000000000..c4712fa2c --- /dev/null +++ b/packages/core/src/ops/catalog-pin-approval.test.ts @@ -0,0 +1,77 @@ +import { expect, it } from "vitest"; +import { OPS_KIND_POLICIES, opsOperationSchema, parseBoundedOpsResult } from "./contract"; + +it("admits only the paired approval identity through the durable policy", () => { + const input = { + kind: "transitousLock.approve", + proposalId: "00000000-0000-4000-8000-000000000001", + validationId: "00000000-0000-4000-8000-000000000002", + expectedGenerationId: null, + diffDigest: "a".repeat(64), + acknowledgeLicenseChanges: false, + }; + expect(opsOperationSchema.parse(input)).toEqual(input); + expect( + opsOperationSchema.safeParse({ + kind: "transitousLock.approve", + ref: `main@${"a".repeat(40)}`, + approvedBy: "operator", + }).success, + ).toBe(false); + expect(OPS_KIND_POLICIES["transitousLock.approve"].execution).toBe("async"); + expect(() => + parseBoundedOpsResult("transitousLock.approve", { + ref: `main@${"a".repeat(40)}`, + lockedAt: new Date().toISOString(), + }), + ).toThrow(); +}); + +import { verifyFiniteControlEdge } from "../build-artifacts/target-lease"; +import { opsOperationFingerprint } from "./fingerprint"; + +it.each([ + "proposalId", + "validationId", + "expectedGenerationId", + "diffDigest", + "acknowledgeLicenseChanges", +] as const)("binds the exact paired approval %s into its original body identity", (field) => { + const body = { + kind: "transitousLock.approve" as const, + proposalId: "00000000-0000-4000-8000-000000000001", + validationId: "00000000-0000-4000-8000-000000000002", + expectedGenerationId: null, + diffDigest: "a".repeat(64), + acknowledgeLicenseChanges: false, + }; + const changed = { + ...body, + [field]: + field === "diffDigest" || field === "expectedGenerationId" + ? "b".repeat(64) + : field === "acknowledgeLicenseChanges" + ? true + : "00000000-0000-4000-8000-000000000003", + }; + expect(opsOperationFingerprint(changed)).not.toBe(opsOperationFingerprint(body)); + expect(opsOperationSchema.safeParse({ ...body, kind: "transitousLock.inspect" }).success).toBe( + false, + ); +}); +it("allows only catalog activation to reserve the closed paired approval phase", () => { + expect(() => + verifyFiniteControlEdge("data-manager", "transit-sync", "paired-pin-commit", "ops-agent"), + ).toThrow(); + expect(() => + verifyFiniteControlEdge( + "data-manager", + "transit-maintenance", + "paired-pin-commit", + "ops-agent", + ), + ).toThrow(); + expect(() => + verifyFiniteControlEdge("data-manager", "catalog.activate", "paired-pin-commit", "ops-agent"), + ).not.toThrow(); +}); diff --git a/packages/core/src/ops/catalog-pin-approval.ts b/packages/core/src/ops/catalog-pin-approval.ts new file mode 100644 index 000000000..dc3c62024 --- /dev/null +++ b/packages/core/src/ops/catalog-pin-approval.ts @@ -0,0 +1,30 @@ +import z from "zod/v4"; +import type { ExecutionRef } from "./execution-identity"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); +export const pairedPinApprovalInputSchema = z.strictObject({ + kind: z.literal("transitousLock.approve"), + proposalId: z.uuid(), + validationId: z.uuid(), + expectedGenerationId: digest.nullable(), + diffDigest: digest, + acknowledgeLicenseChanges: z.boolean(), +}); +export const pairedPinApprovalResultSchema = z.strictObject({ + outcome: z.literal("activated"), + proposalId: z.uuid(), + validationId: z.uuid(), + previousGenerationId: digest.nullable(), + generationId: digest, + commitId: z.uuid(), + committedAt: z.iso.datetime({ offset: true }), +}); +export type PairedPinApprovalInput = z.infer; +export type PairedPinApprovalResult = z.infer; +export type CatalogPinCommitIdentity = { + commitId: string; + execution: ExecutionRef; + proposalId: string; + validationId: string; + diffDigest: string; +}; diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index d870d701b..b411eeb74 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -205,8 +205,11 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { }, "transitousLock.approve": { kind: "transitousLock.approve", - ref: `main@${"a".repeat(40)}`, - approvedBy: "fixture-actor", + proposalId: "00000000-0000-4000-8000-000000000001", + validationId: "00000000-0000-4000-8000-000000000002", + expectedGenerationId: null, + diffDigest: "a".repeat(64), + acknowledgeLicenseChanges: false, }, "gbfsCatalogLock.inspect": { kind: "gbfsCatalogLock.inspect" }, }; diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index fdd79e125..83cb9b459 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -1,4 +1,8 @@ import z from "zod/v4"; +import { + pairedPinApprovalInputSchema, + pairedPinApprovalResultSchema, +} from "./catalog-pin-approval"; import { opsFailureResultSchema, processDiagnosticsSchema } from "./job-evidence"; export const OPS_PROTOCOL_VERSION = 2 as const; @@ -202,7 +206,7 @@ export const OPS_KIND_POLICIES = { "postgis.capacity.inspect": sync("data-manager", 30_000), "transitousLock.inspect": sync("data-manager", 30_000), "transitousLock.propose": sync("data-manager", 30_000), - "transitousLock.approve": sync("data-manager", 30_000), + "transitousLock.approve": asyncEffect("data-manager", 120_000, 4096), "gbfsCatalogLock.inspect": sync("data-manager", 30_000), } as const satisfies Record; @@ -491,13 +495,7 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ lockedBy: actorLabelSchema, comment: z.string().min(1).max(512).optional(), }), - z.strictObject({ - kind: z.literal("transitousLock.approve"), - // The exact ref being activated, so an approval cannot land on a proposal - // that changed after the reviewer read it. - ref: transitousRefSchema, - approvedBy: actorLabelSchema, - }), + pairedPinApprovalInputSchema, kindOnly("gbfsCatalogLock.inspect"), ]); export type OpsOperation = z.infer; @@ -1171,10 +1169,7 @@ export const OPS_RESULT_SCHEMAS = { proposed: transitousLockRecordSchema.nullable(), }), "transitousLock.propose": z.strictObject({ ref: transitousRefSchema, proposed: z.literal(true) }), - "transitousLock.approve": z.strictObject({ - ref: transitousRefSchema, - lockedAt: timestampSchema, - }), + "transitousLock.approve": pairedPinApprovalResultSchema, "gbfsCatalogLock.inspect": z.strictObject({ commit: z.string().regex(/^[0-9a-f]{40}$/), url: z.string().min(1).max(2_048), diff --git a/packages/core/src/ops/execution-identity.ts b/packages/core/src/ops/execution-identity.ts index 0e5b225b2..02fce97ba 100644 --- a/packages/core/src/ops/execution-identity.ts +++ b/packages/core/src/ops/execution-identity.ts @@ -1,25 +1,10 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import { z } from "zod"; import { OPS_KIND_POLICIES, type OpsOperationKind } from "./contract"; +import { executionRefSchema } from "./execution-ref"; import { canonicalJson } from "./fingerprint"; -export type ControlIssuer = "ops-agent" | "data-manager" | "host-cli"; -export const executionRefSchema = z - .strictObject({ - executor: z.enum(["ops-agent", "data-manager", "host-cli"]), - principal: z.enum(["api", "data-manager", "host-cli"]), - executionId: z.string(), - operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), - operationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), - fingerprint: z.string().regex(/^[a-f0-9]{64}$/), - deadlineAt: z.number().int().nonnegative().safe(), - }) - .refine((x) => - x.executor === "ops-agent" - ? /^job1_[A-Za-z0-9_-]{16,64}$/.test(x.executionId) - : z.uuid().safeParse(x.executionId).success, - ); -export type ExecutionRef = z.infer; +export { type ControlIssuer, type ExecutionRef, executionRefSchema } from "./execution-ref"; const apiParentIdentitySchema = z.strictObject({ issuer: z.literal("api"), diff --git a/packages/core/src/ops/execution-ref.ts b/packages/core/src/ops/execution-ref.ts new file mode 100644 index 000000000..2d8a751af --- /dev/null +++ b/packages/core/src/ops/execution-ref.ts @@ -0,0 +1,19 @@ +import { z } from "zod"; + +export type ControlIssuer = "ops-agent" | "data-manager" | "host-cli"; +export const executionRefSchema = z + .strictObject({ + executor: z.enum(["ops-agent", "data-manager", "host-cli"]), + principal: z.enum(["api", "data-manager", "host-cli"]), + executionId: z.string(), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + operationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + }) + .refine((x) => + x.executor === "ops-agent" + ? /^job1_[A-Za-z0-9_-]{16,64}$/.test(x.executionId) + : z.uuid().safeParse(x.executionId).success, + ); +export type ExecutionRef = z.infer; diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index a7a0ca7b7..a96281cc5 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -1,3 +1,4 @@ +export * from "./catalog-pin-approval"; export * from "./client"; export * from "./contract"; export * from "./control-retention-receipt"; diff --git a/packages/core/src/ops/service-results.ts b/packages/core/src/ops/service-results.ts index 6eb80d4e6..3721b5d4b 100644 --- a/packages/core/src/ops/service-results.ts +++ b/packages/core/src/ops/service-results.ts @@ -1,6 +1,16 @@ import { z } from "zod"; export type CoordinatorMode = "ensure-running" | "require-running"; +const count = z.number().int().nonnegative().safe(); +export const hardlinkApplySummarySchema = z.strictObject({ + applied: z.boolean(), + linked: count, + skipped: count, + pruned: count, + entries: count, + via: z.enum(["data-manager", "local", "none"]).optional(), +}); +export type HardlinkApplySummary = z.infer; const serviceId = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); const ids = z .array(serviceId) diff --git a/packages/core/src/services/data-manager/configuration-maintenance.ts b/packages/core/src/services/data-manager/configuration-maintenance.ts new file mode 100644 index 000000000..d51a3cedd --- /dev/null +++ b/packages/core/src/services/data-manager/configuration-maintenance.ts @@ -0,0 +1,63 @@ +import { + type DataConfigurationFencePrepare, + type DataConfigurationFenceReceipt, + type DataConfigurationFenceSettle, + dataConfigurationFencePrepareSchema, + dataConfigurationFenceReceiptSchema, + dataConfigurationFenceSettleSchema, +} from "./contracts/configuration-maintenance"; +import * as sdk from "./generated/data-manager.generated"; +import { type DataManagerTransport, unwrapDataManagerResponse } from "./transport"; + +type ControlOptions = { + transport: DataManagerTransport; + controlAssertion: string; + signal?: AbortSignal; +}; +function boundedHeader(value: string): string { + if (!/^[A-Za-z0-9_-]+$/.test(value) || value.length > 4096) + throw Error("Invalid bounded control header"); + return value; +} +function boundedBody(value: T): T { + if (new TextEncoder().encode(JSON.stringify(value)).length > 32 * 1024) + throw Error("Configuration fence request exceeds budget"); + return value; +} +export async function prepareDataConfigurationFence( + input: DataConfigurationFencePrepare, + options: ControlOptions, +): Promise { + const result = await sdk.prepareDataConfigurationFence( + boundedBody(dataConfigurationFencePrepareSchema.parse(input)), + { + transport: options.transport, + signal: options.signal, + headers: { "x-openmapx-control-delegation": boundedHeader(options.controlAssertion) }, + }, + ); + return dataConfigurationFenceReceiptSchema.parse(unwrapDataManagerResponse(result)); +} +export async function inspectDataConfigurationFence( + maintenanceId: string, + options: { transport: DataManagerTransport; witness: string; signal?: AbortSignal }, +): Promise { + const result = await sdk.inspectDataConfigurationFence(maintenanceId, { + transport: options.transport, + signal: options.signal, + headers: { "x-openmapx-configuration-fence-witness": boundedHeader(options.witness) }, + }); + return dataConfigurationFenceReceiptSchema.parse(unwrapDataManagerResponse(result)); +} +export async function settleDataConfigurationFence( + input: DataConfigurationFenceSettle, + options: ControlOptions, +): Promise { + const body = boundedBody(dataConfigurationFenceSettleSchema.parse(input)); + const result = await sdk.settleDataConfigurationFence(body.maintenanceId, body, { + transport: options.transport, + signal: options.signal, + headers: { "x-openmapx-control-delegation": boundedHeader(options.controlAssertion) }, + }); + return dataConfigurationFenceReceiptSchema.parse(unwrapDataManagerResponse(result)); +} diff --git a/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts new file mode 100644 index 000000000..c98c7539a --- /dev/null +++ b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts @@ -0,0 +1,44 @@ +import z from "zod/v4"; +import { executionRefSchema } from "../../../ops/execution-ref"; +import { + dataCoordinatorModeSchema, + dataEpochSchema, + dataFingerprintSchema, + dataRevisionSchema, +} from "./executions"; + +const binding = { + maintenanceId: z.uuid(), + candidateId: z.uuid(), + ownerPrepare: executionRefSchema, + planDigest: dataFingerprintSchema, + revisionId: dataRevisionSchema, + hardlinkPlanDigest: dataFingerprintSchema.nullable(), +}; +export const dataConfigurationFencePrepareSchema = z.strictObject({ + ...binding, + serviceIds: z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine((values) => values.every((value, i) => i === 0 || (values[i - 1] ?? value) < value)), + coordinatorMode: dataCoordinatorModeSchema, +}); +export const dataConfigurationFenceReceiptSchema = z.strictObject({ + ...binding, + fenceId: z.uuid(), + sequence: dataEpochSchema, + state: z.enum(["prepared", "settled", "recovery_required"]), +}); +export const dataConfigurationFenceSettleSchema = z.strictObject({ + maintenanceId: z.uuid(), + candidateId: z.uuid(), + planDigest: dataFingerprintSchema, + decisionId: z.uuid(), + expectedSequence: dataEpochSchema, + outcome: z.enum(["unused", "applied", "contained"]), + ownerReceiptDigest: dataFingerprintSchema, +}); +export type DataConfigurationFencePrepare = z.infer; +export type DataConfigurationFenceReceipt = z.infer; +export type DataConfigurationFenceSettle = z.infer; diff --git a/packages/core/src/services/data-manager/contracts/executions.ts b/packages/core/src/services/data-manager/contracts/executions.ts new file mode 100644 index 000000000..cce6f7138 --- /dev/null +++ b/packages/core/src/services/data-manager/contracts/executions.ts @@ -0,0 +1,205 @@ +import z from "zod/v4"; +import { executionRefSchema } from "../../../ops/execution-ref"; +import { processDiagnosticsSchema } from "../../../ops/job-evidence"; +import { hardlinkApplySummarySchema } from "../../../ops/service-results"; +import { regionSchema } from "./common"; +import { hardlinkEntrySchema } from "./datasets"; + +export const dataOperationKeySchema = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/); +export const dataFingerprintSchema = z.string().regex(/^[a-f0-9]{64}$/); +export const dataEpochSchema = z.number().int().nonnegative().safe(); +export const dataRevisionSchema = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +export const dataCoordinatorModeSchema = z.enum(["ensure-running", "require-running"]); +const regional = (kind: K) => + z.strictObject({ kind: z.literal(kind), region: regionSchema }); +const countries = z.array(z.string().regex(/^[A-Za-z]{2,3}$/)).max(256); +const ordinaryInputs = [ + regional("download-osm"), + z.strictObject({ kind: z.literal("download-fonts") }), + regional("convert-overpass"), + z.strictObject({ kind: z.literal("transit-sync"), countries }), + regional("overture-pull"), + regional("overture-ingest"), + regional("overture-extract"), + regional("overture-conflate").extend({ restart: z.boolean().optional() }), + regional("overture-sync"), + regional("search-index-build"), + z.strictObject({ kind: z.literal("notable-places-build") }), +] as const; +export const generationHardlinksInputSchema = z.strictObject({ + kind: z.literal("generation-hardlinks"), + revisionId: dataRevisionSchema, + planDigest: dataFingerprintSchema, + coordinatorMode: dataCoordinatorModeSchema, +}); +const maintenance = z.discriminatedUnion("action", [ + z.strictObject({ kind: z.literal("transit-maintenance"), action: z.literal("restart-primary") }), + z.strictObject({ + kind: z.literal("transit-maintenance"), + action: z.literal("reload-feed-proxy"), + candidateId: z + .string() + .max(128) + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/), + }), +]); +export const ordinaryDataExecutionInputSchema = z.discriminatedUnion("kind", ordinaryInputs); +export const dataExecutionInputSchema = z.union([ + ordinaryDataExecutionInputSchema, + generationHardlinksInputSchema, + maintenance, +]); +export type DataExecutionInput = z.output; +export function normalizeDataExecutionInput(value: unknown): DataExecutionInput { + const input = dataExecutionInputSchema.parse(value); + return input.kind === "transit-sync" + ? { + ...input, + countries: [...new Set(input.countries.map((country) => country.toLowerCase()))].sort(), + } + : input; +} +export const dataExecutionMetricsSchema = z.strictObject({ + processed: dataEpochSchema.optional(), + inserted: dataEpochSchema.optional(), + updated: dataEpochSchema.optional(), + skipped: dataEpochSchema.optional(), + sizeBytes: dataEpochSchema.optional(), +}); +export type DataExecutionMetrics = z.infer; +const metricsResult = (kind: K) => + regional(kind).extend({ + metrics: dataExecutionMetricsSchema, + release: z + .string() + .max(128) + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/) + .optional(), + }); +export const dataExecutionResultSchema = z + .discriminatedUnion("kind", [ + regional("download-osm").extend({ + sizeBytes: dataEpochSchema, + md5: z + .string() + .regex(/^[a-f0-9]{32}$/) + .optional(), + }), + z.strictObject({ kind: z.literal("download-fonts"), completed: z.literal(true) }), + regional("convert-overpass").extend({ sizeBytes: dataEpochSchema }), + z.strictObject({ + kind: z.literal("transit-sync"), + finalStatus: z.enum(["ok", "partial", "error"]), + sourceRevision: z.string().min(1).max(512), + countries, + }), + metricsResult("overture-pull"), + metricsResult("overture-ingest"), + metricsResult("overture-extract"), + metricsResult("overture-conflate"), + metricsResult("overture-sync"), + regional("search-index-build").extend({ metrics: dataExecutionMetricsSchema }), + z.strictObject({ + kind: z.literal("notable-places-build"), + metrics: dataExecutionMetricsSchema, + }), + z.strictObject({ + kind: z.literal("generation-hardlinks"), + hardlinks: hardlinkApplySummarySchema, + }), + z.strictObject({ + kind: z.literal("transit-maintenance"), + action: z.enum(["restart-primary", "reload-feed-proxy"]), + changed: z.literal(true), + }), + ]) + .refine((value) => new TextEncoder().encode(JSON.stringify(value)).length <= 32 * 1024); +export type DataExecutionResult = z.infer; +export const dataExecutionStatusSchema = z + .strictObject({ + id: z.uuid(), + operationKey: dataOperationKeySchema, + fingerprint: dataFingerprintSchema, + admittedAt: dataEpochSchema, + deadlineAt: dataEpochSchema, + kind: z.enum([ + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance", + ]), + status: z.enum([ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted", + ]), + cancellationRequestedAt: z.iso.datetime({ offset: true }).nullable(), + progress: z.number().finite().min(0).max(1).nullable(), + diagnostics: processDiagnosticsSchema, + publication: z.enum(["not_started", "committed", "unknown"]), + result: dataExecutionResultSchema.optional(), + }) + .refine( + (value) => + value.deadlineAt > value.admittedAt && (!value.result || value.result.kind === value.kind), + ); +export type DataExecutionStatus = z.infer; +export const startDataExecutionSchema = z.strictObject({ + operationKey: dataOperationKeySchema, + input: ordinaryDataExecutionInputSchema, + requestedDeadlineAt: dataEpochSchema.optional(), +}); +export const generationHardlinksPlanSchema = z.strictObject({ + plan: z.array(hardlinkEntrySchema).max(256), + prune: z.literal(true), +}); +export const startGenerationHardlinksExecutionSchema = z + .strictObject({ + operationKey: dataOperationKeySchema, + requestedDeadlineAt: dataEpochSchema, + input: generationHardlinksInputSchema, + plan: generationHardlinksPlanSchema, + }) + .refine((value) => new TextEncoder().encode(JSON.stringify(value)).length <= 1024 * 1024); +export const dataExecutionIdentitySchema = z.strictObject({ + id: z.uuid(), + operationKey: dataOperationKeySchema, + fingerprint: dataFingerprintSchema, +}); +export type DataExecutionIdentity = z.infer; +export const cancelDataExecutionSchema = dataExecutionIdentitySchema.omit({ id: true }); +export const dataChildAttemptSchema = z + .strictObject({ + phase: z.enum(["osm", "transit", "fonts", "data-step"]), + operationKey: dataOperationKeySchema, + input: dataExecutionInputSchema, + operationFingerprint: dataFingerprintSchema, + requestedDeadlineAt: dataEpochSchema, + parentExecution: executionRefSchema, + state: z.enum(["prepared", "attempted", "observing", "terminal", "unknown"]), + id: z.uuid().optional(), + fingerprint: dataFingerprintSchema.optional(), + deadlineAt: dataEpochSchema.optional(), + }) + .refine((value) => new TextEncoder().encode(JSON.stringify(value)).length <= 8192); +export type DataChildAttempt = z.infer; +export function isDataExecutionTerminal(status: DataExecutionStatus): boolean { + return ( + ["ok", "partial", "error", "canceled", "interrupted"].includes(status.status) && + status.publication !== "unknown" + ); +} diff --git a/packages/core/src/services/data-manager/contracts/index.ts b/packages/core/src/services/data-manager/contracts/index.ts index cacfa5a14..2c586e71a 100644 --- a/packages/core/src/services/data-manager/contracts/index.ts +++ b/packages/core/src/services/data-manager/contracts/index.ts @@ -1,6 +1,8 @@ export * from "./ambient"; export * from "./common"; +export * from "./configuration-maintenance"; export * from "./datasets"; +export * from "./executions"; export * from "./headers"; export * from "./indexes"; export * from "./offline"; diff --git a/packages/core/src/services/data-manager/contracts/operations.ts b/packages/core/src/services/data-manager/contracts/operations.ts index e4d99e8d6..d6611e9c7 100644 --- a/packages/core/src/services/data-manager/contracts/operations.ts +++ b/packages/core/src/services/data-manager/contracts/operations.ts @@ -14,7 +14,9 @@ import { regionSchema, serviceErrors, } from "./common"; +import * as cm from "./configuration-maintenance"; import * as d from "./datasets"; +import * as e from "./executions"; import * as h from "./headers"; import * as i from "./indexes"; import * as f from "./offline"; @@ -23,14 +25,19 @@ import { trafficApplicationSnapshotSchema } from "./traffic"; import * as t from "./transit"; const service = "service" as const; +const dataMutationErrors = { + ...serviceErrors, + 409: json(httpErrorSchema), + 503: json(httpErrorSchema), +}; const sourceParams = z.strictObject({ sourceId: z.string().min(1) }); const sourceMutationErrors = { - ...serviceErrors, + ...dataMutationErrors, 404: json(httpErrorSchema), 409: json(z.union([httpErrorSchema, t.transitConflictSchema])), }; const offlineErrors = { - ...serviceErrors, + ...dataMutationErrors, 400: json(z.union([httpErrorSchema, f.offlinePackageErrorSchema])), 404: json(httpErrorSchema), }; @@ -56,6 +63,114 @@ export const coverageEvidenceQuerySchema = z .refine((v) => !v.offset || !!v.snapshotId, "snapshotId is required for continuation pages"); /** The sole method/path/status authority for the internal service and its generated SDK. */ export const dataManagerOperations = { + startDataExecution: { + method: "POST", + path: "/executions", + auth: service, + request: { body: e.startDataExecutionSchema }, + responses: { + ...serviceErrors, + 202: json(e.dataExecutionStatusSchema), + 403: json(httpErrorSchema), + 409: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, + startGenerationHardlinksExecution: { + method: "POST", + path: "/executions/generation-hardlinks", + auth: service, + request: { body: e.startGenerationHardlinksExecutionSchema }, + responses: { + ...serviceErrors, + 202: json(e.dataExecutionStatusSchema), + 403: json(httpErrorSchema), + 409: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, + prepareDataConfigurationFence: { + method: "POST", + path: "/executions/configuration-maintenance/prepare", + auth: service, + request: { body: cm.dataConfigurationFencePrepareSchema }, + responses: { + ...serviceErrors, + 200: json(cm.dataConfigurationFenceReceiptSchema), + 403: json(httpErrorSchema), + 409: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, + inspectDataConfigurationFence: { + method: "GET", + path: "/executions/configuration-maintenance/:maintenanceId", + auth: service, + request: { params: z.strictObject({ maintenanceId: z.uuid() }) }, + responses: { + ...serviceErrors, + 200: json(cm.dataConfigurationFenceReceiptSchema), + 403: json(httpErrorSchema), + 404: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, + settleDataConfigurationFence: { + method: "POST", + path: "/executions/configuration-maintenance/:maintenanceId/settle", + auth: service, + request: { + params: z.strictObject({ maintenanceId: z.uuid() }), + body: cm.dataConfigurationFenceSettleSchema, + }, + responses: { + ...serviceErrors, + 200: json(cm.dataConfigurationFenceReceiptSchema), + 403: json(httpErrorSchema), + 404: json(httpErrorSchema), + 409: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, + lookupDataExecution: { + method: "GET", + path: "/executions/by-key/:operationKey", + auth: service, + request: { params: z.strictObject({ operationKey: e.dataOperationKeySchema }) }, + responses: { + ...serviceErrors, + 200: json(e.dataExecutionStatusSchema), + 404: json(httpErrorSchema), + 410: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, + getDataExecution: { + method: "GET", + path: "/executions/:id", + auth: service, + request: { params: z.strictObject({ id: z.uuid() }) }, + responses: { + ...serviceErrors, + 200: json(e.dataExecutionStatusSchema), + 404: json(httpErrorSchema), + 410: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, + cancelDataExecution: { + method: "POST", + path: "/executions/:id/cancel", + auth: service, + request: { params: z.strictObject({ id: z.uuid() }), body: e.cancelDataExecutionSchema }, + responses: { + ...serviceErrors, + 200: json(e.dataExecutionStatusSchema), + 404: json(httpErrorSchema), + 409: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, getLive: { method: "GET", path: "/live", @@ -82,20 +197,20 @@ export const dataManagerOperations = { method: "POST", path: "/datasets/reload", auth: service, - responses: { ...serviceErrors, 200: json(d.reloadDatasetsSchema) }, + responses: { ...dataMutationErrors, 200: json(d.reloadDatasetsSchema) }, }, downloadFonts: { method: "POST", path: "/download/fonts", auth: service, - responses: { ...serviceErrors, 200: json(okSchema) }, + responses: { ...dataMutationErrors, 200: json(okSchema) }, }, link: { method: "POST", path: "/link", auth: service, request: { body: d.linkRequestSchema }, - responses: { ...serviceErrors, 200: json(d.linkResultSchema) }, + responses: { ...dataMutationErrors, 200: json(d.linkResultSchema) }, }, downloadOsm: { method: "POST", @@ -103,7 +218,7 @@ export const dataManagerOperations = { auth: service, request: { body: regionBodySchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(d.downloadOsmEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -113,7 +228,7 @@ export const dataManagerOperations = { auth: service, request: { body: z.strictObject({ region: regionSchema.optional() }).optional() }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(d.convertOverpassEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -133,7 +248,7 @@ export const dataManagerOperations = { auth: service, request: { body: regionBodySchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(i.searchIndexEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -152,7 +267,7 @@ export const dataManagerOperations = { path: "/notable-places/build", auth: service, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(i.notablePlacesEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -168,7 +283,7 @@ export const dataManagerOperations = { auth: service, request: { body: regionBodySchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(o.overturePullEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -178,7 +293,7 @@ export const dataManagerOperations = { auth: service, request: { body: regionBodySchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(o.overtureSyncEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -188,7 +303,7 @@ export const dataManagerOperations = { auth: service, request: { body: regionBodySchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(o.overtureSimpleEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -198,7 +313,7 @@ export const dataManagerOperations = { auth: service, request: { body: regionBodySchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(o.overtureSimpleEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -208,7 +323,7 @@ export const dataManagerOperations = { auth: service, request: { body: regionBodySchema.extend({ restart: z.boolean().optional() }) }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: ndjson(o.overtureConflateEventSchema, h.ndjsonResponseHeadersSchema), }, }, @@ -269,7 +384,7 @@ export const dataManagerOperations = { auth: service, request: { body: t.transitPreflightRequestSchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: json(t.transitPreflightSchema.extend({ ok: z.literal(true) })), 422: json(t.transitPreflightSchema.extend({ ok: z.literal(false) })), }, @@ -280,9 +395,9 @@ export const dataManagerOperations = { auth: service, request: { body: t.transitSyncRequestSchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 202: json(t.transitSyncAcceptedSchema), - 409: json(t.transitConflictSchema), + 409: json(z.union([httpErrorSchema, t.transitConflictSchema])), 422: json(t.transitScopeRejectedSchema), }, }, @@ -291,7 +406,7 @@ export const dataManagerOperations = { path: "/transit/restart-motis", auth: service, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: json(z.strictObject({ ok: z.literal(true), status: z.literal("restart-initiated") })), 503: json(httpErrorSchema), }, @@ -302,7 +417,7 @@ export const dataManagerOperations = { auth: service, request: { body: t.transitBumpRequestSchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: json(t.transitBumpResultSchema), 409: json(httpErrorSchema), 502: json(httpErrorSchema), @@ -314,10 +429,11 @@ export const dataManagerOperations = { auth: service, request: { body: t.transitApproveRequestSchema }, responses: { - ...serviceErrors, + ...dataMutationErrors, 200: json(t.transitApprovalSchema), 404: json(httpErrorSchema), 422: json(t.transitApprovalMismatchSchema), + 503: json(httpErrorSchema), }, }, bakePredictedTraffic: { @@ -325,7 +441,7 @@ export const dataManagerOperations = { path: "/traffic/predicted/bake", auth: service, responses: { - ...serviceErrors, + ...dataMutationErrors, 202: json(acceptedSchema), 409: json(httpErrorSchema), 501: json(httpErrorSchema), @@ -364,34 +480,34 @@ export const dataManagerOperations = { path: "/ambient-places/build", auth: service, request: { body: a.ambientRegionSchema }, - responses: { ...serviceErrors, 202: json(acceptedSchema), 409: json(httpErrorSchema) }, + responses: { ...dataMutationErrors, 202: json(acceptedSchema), 409: json(httpErrorSchema) }, }, resumeAmbientPlaces: { method: "POST", path: "/ambient-places/resume", auth: service, request: { body: a.ambientGenerationRequestSchema }, - responses: { ...serviceErrors, 202: json(acceptedSchema), 409: json(httpErrorSchema) }, + responses: { ...dataMutationErrors, 202: json(acceptedSchema), 409: json(httpErrorSchema) }, }, discardAmbientPlaces: { method: "POST", path: "/ambient-places/discard", auth: service, request: { body: a.ambientGenerationRequestSchema }, - responses: { ...serviceErrors, 200: json(okSchema), 409: json(httpErrorSchema) }, + responses: { ...dataMutationErrors, 200: json(okSchema), 409: json(httpErrorSchema) }, }, setAmbientPlacesEnabled: { method: "POST", path: "/ambient-places/enabled", auth: service, request: { body: a.ambientEnabledRequestSchema }, - responses: { ...serviceErrors, 200: json(okSchema), 409: json(httpErrorSchema) }, + responses: { ...dataMutationErrors, 200: json(okSchema), 409: json(httpErrorSchema) }, }, rollbackAmbientPlaces: { method: "POST", path: "/ambient-places/rollback", auth: service, - responses: { ...serviceErrors, 200: json(okSchema), 409: json(httpErrorSchema) }, + responses: { ...dataMutationErrors, 200: json(okSchema), 409: json(httpErrorSchema) }, }, getOfflinePackageCapability: { method: "GET", diff --git a/packages/core/src/services/data-manager/executions.test.ts b/packages/core/src/services/data-manager/executions.test.ts new file mode 100644 index 000000000..dc2d16711 --- /dev/null +++ b/packages/core/src/services/data-manager/executions.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "vitest"; +import { + dataExecutionInputSchema, + dataExecutionStatusSchema, + normalizeDataExecutionInput, +} from "./contracts/executions"; +import { + DataExecutionObservationPending, + observeDataExecution, + requestDataExecutionCancellation, +} from "./executions"; +import { createDataManagerTransport } from "./transport"; + +describe("durable data execution receipts", () => { + it("normalizes country identity and rejects arbitrary effects", () => { + expect( + normalizeDataExecutionInput({ kind: "transit-sync", countries: ["DE", "ch", "de"] }), + ).toEqual({ kind: "transit-sync", countries: ["ch", "de"] }); + expect( + dataExecutionInputSchema.safeParse({ kind: "download-osm", region: "../private" }).success, + ).toBe(false); + expect( + dataExecutionInputSchema.safeParse({ kind: "download-fonts", url: "https://example.test" }) + .success, + ).toBe(false); + expect( + dataExecutionInputSchema.safeParse({ + kind: "overture-pull", + region: "europe/germany", + restart: true, + }).success, + ).toBe(false); + }); + it("rejects foreign results and private metadata in a public status", () => { + const status = { + id: "00000000-0000-4000-8000-000000000001", + operationKey: "opk1_abcdefghijklmnop", + fingerprint: "a".repeat(64), + admittedAt: 1000, + deadlineAt: 2000, + kind: "download-fonts", + status: "ok", + cancellationRequestedAt: null, + progress: 1, + diagnostics: { tail: [], truncated: false }, + publication: "committed", + result: { kind: "download-fonts", completed: true }, + }; + expect(dataExecutionStatusSchema.safeParse(status).success).toBe(true); + expect( + dataExecutionStatusSchema.safeParse({ + ...status, + result: { kind: "download-osm", region: "de", sizeBytes: 1 }, + }).success, + ).toBe(false); + expect(dataExecutionStatusSchema.safeParse({ ...status, children: [] }).success).toBe(false); + expect( + dataExecutionStatusSchema.safeParse({ + ...status, + result: { kind: "download-fonts", completed: true, path: "/host" }, + }).success, + ).toBe(false); + }); +}); + +it("retires a disconnected observer without sending backend cancellation", async () => { + const controller = new AbortController(); + const identity = { + id: "00000000-0000-4000-8000-000000000001", + operationKey: "opk1_abcdefghijklmnop", + fingerprint: "a".repeat(64), + }; + const methods: string[] = []; + const transport = createDataManagerTransport({ + baseUrl: "http://127.0.0.1", + fetch: async (_url, options) => { + methods.push(options?.method ?? "GET"); + return Response.json({ + ...identity, + admittedAt: 1000, + deadlineAt: 2000, + kind: "download-fonts", + status: "running", + cancellationRequestedAt: null, + progress: 0.2, + diagnostics: { tail: [], truncated: false }, + publication: "not_started", + }); + }, + }); + await expect( + observeDataExecution(identity, { + transport, + signal: controller.signal, + observerDeadlineAt: Date.now() + 10000, + onProgress: () => controller.abort(), + }), + ).rejects.toMatchObject({ + name: "DataExecutionObservationPending", + reason: "interrupted", + identity, + }); + expect(methods).toEqual(["GET"]); +}); +it("uses an independent explicit fingerprint-bound cancellation request", async () => { + const identity = { + id: "00000000-0000-4000-8000-000000000001", + operationKey: "opk1_abcdefghijklmnop", + fingerprint: "a".repeat(64), + }; + const requests: Array<{ method: string; body: string }> = []; + const transport = createDataManagerTransport({ + baseUrl: "http://127.0.0.1", + fetch: async (_url, options) => { + requests.push({ method: options?.method ?? "GET", body: String(options?.body ?? "") }); + return Response.json({ + ...identity, + admittedAt: 1000, + deadlineAt: 2000, + kind: "download-fonts", + status: "cancel_pending", + cancellationRequestedAt: "2026-10-25T00:00:00.000Z", + progress: 0.2, + diagnostics: { tail: [], truncated: false }, + publication: "unknown", + }); + }, + }); + expect((await requestDataExecutionCancellation(identity, { transport })).status).toBe( + "cancel_pending", + ); + expect(requests).toEqual([ + { + method: "POST", + body: JSON.stringify({ + operationKey: identity.operationKey, + fingerprint: identity.fingerprint, + }), + }, + ]); + expect(new DataExecutionObservationPending(identity, "transport").identity).toEqual(identity); +}); diff --git a/packages/core/src/services/data-manager/executions.ts b/packages/core/src/services/data-manager/executions.ts new file mode 100644 index 000000000..4d98ae9a0 --- /dev/null +++ b/packages/core/src/services/data-manager/executions.ts @@ -0,0 +1,120 @@ +import { + type DataExecutionIdentity, + type DataExecutionStatus, + dataExecutionIdentitySchema, + dataExecutionStatusSchema, + isDataExecutionTerminal, +} from "./contracts/executions"; +import { cancelDataExecution, getDataExecution } from "./generated/data-manager.generated"; +import { type DataManagerTransport, unwrapDataManagerResponse } from "./transport"; +export class DataExecutionObservationPending extends Error { + constructor( + readonly identity: DataExecutionIdentity, + readonly reason: "transport" | "observer_deadline" | "interrupted", + ) { + super(`Data execution observation pending (${reason})`); + this.name = "DataExecutionObservationPending"; + } +} +export function matchDataExecutionIdentity( + identity: DataExecutionIdentity, + value: unknown, +): DataExecutionStatus { + dataExecutionIdentitySchema.parse(identity); + const status = dataExecutionStatusSchema.parse(value); + if ( + status.id !== identity.id || + status.operationKey !== identity.operationKey || + status.fingerprint !== identity.fingerprint + ) + throw Error("Data execution receipt identity conflict"); + return status; +} +/** Observation owns only its GETs. Its deadline/abort never cancels work. */ +export async function observeDataExecution( + identity: DataExecutionIdentity, + options: { + transport: DataManagerTransport; + signal?: AbortSignal; + observerDeadlineAt: number; + onProgress?: (status: DataExecutionStatus) => void; + }, +): Promise { + dataExecutionIdentitySchema.parse(identity); + if (!Number.isSafeInteger(options.observerDeadlineAt) || options.observerDeadlineAt < 0) + throw Error("Invalid observer deadline"); + const remaining = options.observerDeadlineAt - Date.now(); + if (remaining <= 0) throw new DataExecutionObservationPending(identity, "observer_deadline"); + const deadline = new AbortController(); + const timer = setTimeout(() => deadline.abort(), Math.min(remaining, 2147483647)); + const signal = options.signal + ? AbortSignal.any([options.signal, deadline.signal]) + : deadline.signal; + try { + for (;;) { + signal.throwIfAborted(); + const status = matchDataExecutionIdentity( + identity, + unwrapDataManagerResponse( + await getDataExecution(identity.id, { transport: options.transport, signal }), + ), + ); + options.onProgress?.(status); + if (isDataExecutionTerminal(status)) return status; + await new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(wait); + signal.removeEventListener("abort", abort); + reject(signal.reason); + }; + const wait = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(); + }, 250); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + }); + } + } catch (error) { + if (error instanceof DataExecutionObservationPending) throw error; + throw new DataExecutionObservationPending( + identity, + options.signal?.aborted + ? "interrupted" + : deadline.signal.aborted || Date.now() >= options.observerDeadlineAt + ? "observer_deadline" + : "transport", + ); + } finally { + clearTimeout(timer); + } +} +/** Explicit backend intent, independent of any retired progress reader. */ +export async function requestDataExecutionCancellation( + identity: DataExecutionIdentity, + options: { transport: DataManagerTransport; signal?: AbortSignal }, +): Promise { + dataExecutionIdentitySchema.parse(identity); + try { + const result = await cancelDataExecution( + identity.id, + { operationKey: identity.operationKey, fingerprint: identity.fingerprint }, + options, + ); + return matchDataExecutionIdentity(identity, unwrapDataManagerResponse(result)); + } catch (error) { + if (error instanceof DataExecutionObservationPending) throw error; + throw new DataExecutionObservationPending( + identity, + options.signal?.aborted ? "interrupted" : "transport", + ); + } +} +export type { + DataChildAttempt, + DataExecutionIdentity, + DataExecutionInput, + DataExecutionMetrics, + DataExecutionResult, + DataExecutionStatus, +} from "./contracts/executions"; diff --git a/packages/core/src/services/data-manager/generated/data-manager.generated.ts b/packages/core/src/services/data-manager/generated/data-manager.generated.ts index e9addfcec..b0ea8f53e 100644 --- a/packages/core/src/services/data-manager/generated/data-manager.generated.ts +++ b/packages/core/src/services/data-manager/generated/data-manager.generated.ts @@ -10,16 +10,19 @@ import type { AddTransitSource415, AddTransitSource429, AddTransitSource500, + AddTransitSource503, AddTransitSourceBody, ApproveTransitBump200, ApproveTransitBump400, ApproveTransitBump401, ApproveTransitBump404, + ApproveTransitBump409, ApproveTransitBump413, ApproveTransitBump415, ApproveTransitBump422, ApproveTransitBump429, ApproveTransitBump500, + ApproveTransitBump503, ApproveTransitBumpBody, BakePredictedTraffic202, BakePredictedTraffic400, @@ -30,6 +33,7 @@ import type { BakePredictedTraffic429, BakePredictedTraffic500, BakePredictedTraffic501, + BakePredictedTraffic503, BuildAmbientPlaces202, BuildAmbientPlaces400, BuildAmbientPlaces401, @@ -38,37 +42,57 @@ import type { BuildAmbientPlaces415, BuildAmbientPlaces429, BuildAmbientPlaces500, + BuildAmbientPlaces503, BuildAmbientPlacesBody, BuildNotablePlaces200, BuildNotablePlaces400, BuildNotablePlaces401, + BuildNotablePlaces409, BuildNotablePlaces413, BuildNotablePlaces415, BuildNotablePlaces429, BuildNotablePlaces500, + BuildNotablePlaces503, BuildSearchIndex200, BuildSearchIndex400, BuildSearchIndex401, + BuildSearchIndex409, BuildSearchIndex413, BuildSearchIndex415, BuildSearchIndex429, BuildSearchIndex500, + BuildSearchIndex503, BuildSearchIndexBody, + CancelDataExecution200, + CancelDataExecution400, + CancelDataExecution401, + CancelDataExecution404, + CancelDataExecution409, + CancelDataExecution413, + CancelDataExecution415, + CancelDataExecution429, + CancelDataExecution500, + CancelDataExecution503, + CancelDataExecutionBody, ConflateOverture200, ConflateOverture400, ConflateOverture401, + ConflateOverture409, ConflateOverture413, ConflateOverture415, ConflateOverture429, ConflateOverture500, + ConflateOverture503, ConflateOvertureBody, ConvertOverpass200, ConvertOverpass400, ConvertOverpass401, + ConvertOverpass409, ConvertOverpass413, ConvertOverpass415, ConvertOverpass429, ConvertOverpass500, + ConvertOverpass503, ConvertOverpassBody, DiscardAmbientPlaces200, DiscardAmbientPlaces400, @@ -78,21 +102,26 @@ import type { DiscardAmbientPlaces415, DiscardAmbientPlaces429, DiscardAmbientPlaces500, + DiscardAmbientPlaces503, DiscardAmbientPlacesBody, DownloadFonts200, DownloadFonts400, DownloadFonts401, + DownloadFonts409, DownloadFonts413, DownloadFonts415, DownloadFonts429, DownloadFonts500, + DownloadFonts503, DownloadOsm200, DownloadOsm400, DownloadOsm401, + DownloadOsm409, DownloadOsm413, DownloadOsm415, DownloadOsm429, DownloadOsm500, + DownloadOsm503, DownloadOsmBody, EnableTransitSource202, EnableTransitSource400, @@ -103,14 +132,17 @@ import type { EnableTransitSource415, EnableTransitSource429, EnableTransitSource500, + EnableTransitSource503, EnableTransitSourceBody, ExtractOverture200, ExtractOverture400, ExtractOverture401, + ExtractOverture409, ExtractOverture413, ExtractOverture415, ExtractOverture429, ExtractOverture500, + ExtractOverture503, ExtractOvertureBody, GetAmbientPlacesStatus200, GetAmbientPlacesStatus400, @@ -137,6 +169,16 @@ import type { GetCoverageEvidence500, GetCoverageEvidence503, GetCoverageEvidenceParams, + GetDataExecution200, + GetDataExecution400, + GetDataExecution401, + GetDataExecution404, + GetDataExecution410, + GetDataExecution413, + GetDataExecution415, + GetDataExecution429, + GetDataExecution500, + GetDataExecution503, GetDatasets200, GetDatasets400, GetDatasets401, @@ -158,25 +200,31 @@ import type { GetOfflineGlyph400, GetOfflineGlyph401, GetOfflineGlyph404, + GetOfflineGlyph409, GetOfflineGlyph413, GetOfflineGlyph415, GetOfflineGlyph429, GetOfflineGlyph500, + GetOfflineGlyph503, GetOfflineGlyphCatalog200, GetOfflineGlyphCatalog400, GetOfflineGlyphCatalog401, GetOfflineGlyphCatalog404, + GetOfflineGlyphCatalog409, GetOfflineGlyphCatalog413, GetOfflineGlyphCatalog415, GetOfflineGlyphCatalog429, GetOfflineGlyphCatalog500, + GetOfflineGlyphCatalog503, GetOfflinePackageArchive400, GetOfflinePackageArchive401, GetOfflinePackageArchive404, + GetOfflinePackageArchive409, GetOfflinePackageArchive413, GetOfflinePackageArchive415, GetOfflinePackageArchive429, GetOfflinePackageArchive500, + GetOfflinePackageArchive503, GetOfflinePackageArchiveHeaders, GetOfflinePackageCapability200, GetOfflinePackageCapability400, @@ -189,19 +237,23 @@ import type { GetOfflinePackageJob400, GetOfflinePackageJob401, GetOfflinePackageJob404, + GetOfflinePackageJob409, GetOfflinePackageJob413, GetOfflinePackageJob415, GetOfflinePackageJob429, GetOfflinePackageJob500, + GetOfflinePackageJob503, GetOfflinePackageJobHeaders, GetOfflinePackageManifest200, GetOfflinePackageManifest400, GetOfflinePackageManifest401, GetOfflinePackageManifest404, + GetOfflinePackageManifest409, GetOfflinePackageManifest413, GetOfflinePackageManifest415, GetOfflinePackageManifest429, GetOfflinePackageManifest500, + GetOfflinePackageManifest503, GetOvertureStatus200, GetOvertureStatus400, GetOvertureStatus401, @@ -260,28 +312,65 @@ import type { IngestOverture200, IngestOverture400, IngestOverture401, + IngestOverture409, IngestOverture413, IngestOverture415, IngestOverture429, IngestOverture500, + IngestOverture503, IngestOvertureBody, + InspectDataConfigurationFence200, + InspectDataConfigurationFence400, + InspectDataConfigurationFence401, + InspectDataConfigurationFence403, + InspectDataConfigurationFence404, + InspectDataConfigurationFence413, + InspectDataConfigurationFence415, + InspectDataConfigurationFence429, + InspectDataConfigurationFence500, + InspectDataConfigurationFence503, Link200, Link400, Link401, + Link409, Link413, Link415, Link429, Link500, + Link503, LinkBody, + LookupDataExecution200, + LookupDataExecution400, + LookupDataExecution401, + LookupDataExecution404, + LookupDataExecution410, + LookupDataExecution413, + LookupDataExecution415, + LookupDataExecution429, + LookupDataExecution500, + LookupDataExecution503, PreflightTransit200, PreflightTransit400, PreflightTransit401, + PreflightTransit409, PreflightTransit413, PreflightTransit415, PreflightTransit422, PreflightTransit429, PreflightTransit500, + PreflightTransit503, PreflightTransitBody, + PrepareDataConfigurationFence200, + PrepareDataConfigurationFence400, + PrepareDataConfigurationFence401, + PrepareDataConfigurationFence403, + PrepareDataConfigurationFence409, + PrepareDataConfigurationFence413, + PrepareDataConfigurationFence415, + PrepareDataConfigurationFence429, + PrepareDataConfigurationFence500, + PrepareDataConfigurationFence503, + PrepareDataConfigurationFenceBody, PrepareOfflinePackage200, PrepareOfflinePackage202, PrepareOfflinePackage400, @@ -304,22 +393,27 @@ import type { ProposeTransitBump429, ProposeTransitBump500, ProposeTransitBump502, + ProposeTransitBump503, ProposeTransitBumpBody, PullOverture200, PullOverture400, PullOverture401, + PullOverture409, PullOverture413, PullOverture415, PullOverture429, PullOverture500, + PullOverture503, PullOvertureBody, ReloadDatasets200, ReloadDatasets400, ReloadDatasets401, + ReloadDatasets409, ReloadDatasets413, ReloadDatasets415, ReloadDatasets429, ReloadDatasets500, + ReloadDatasets503, RemoveTransitSource202, RemoveTransitSource400, RemoveTransitSource401, @@ -329,10 +423,12 @@ import type { RemoveTransitSource415, RemoveTransitSource429, RemoveTransitSource500, + RemoveTransitSource503, RemoveTransitSourceBody, RestartMotis200, RestartMotis400, RestartMotis401, + RestartMotis409, RestartMotis413, RestartMotis415, RestartMotis429, @@ -346,6 +442,7 @@ import type { ResumeAmbientPlaces415, ResumeAmbientPlaces429, ResumeAmbientPlaces500, + ResumeAmbientPlaces503, ResumeAmbientPlacesBody, RollbackAmbientPlaces200, RollbackAmbientPlaces400, @@ -355,6 +452,7 @@ import type { RollbackAmbientPlaces415, RollbackAmbientPlaces429, RollbackAmbientPlaces500, + RollbackAmbientPlaces503, SetAmbientPlacesEnabled200, SetAmbientPlacesEnabled400, SetAmbientPlacesEnabled401, @@ -363,14 +461,51 @@ import type { SetAmbientPlacesEnabled415, SetAmbientPlacesEnabled429, SetAmbientPlacesEnabled500, + SetAmbientPlacesEnabled503, SetAmbientPlacesEnabledBody, + SettleDataConfigurationFence200, + SettleDataConfigurationFence400, + SettleDataConfigurationFence401, + SettleDataConfigurationFence403, + SettleDataConfigurationFence404, + SettleDataConfigurationFence409, + SettleDataConfigurationFence413, + SettleDataConfigurationFence415, + SettleDataConfigurationFence429, + SettleDataConfigurationFence500, + SettleDataConfigurationFence503, + SettleDataConfigurationFenceBody, + StartDataExecution202, + StartDataExecution400, + StartDataExecution401, + StartDataExecution403, + StartDataExecution409, + StartDataExecution413, + StartDataExecution415, + StartDataExecution429, + StartDataExecution500, + StartDataExecution503, + StartDataExecutionBody, + StartGenerationHardlinksExecution202, + StartGenerationHardlinksExecution400, + StartGenerationHardlinksExecution401, + StartGenerationHardlinksExecution403, + StartGenerationHardlinksExecution409, + StartGenerationHardlinksExecution413, + StartGenerationHardlinksExecution415, + StartGenerationHardlinksExecution429, + StartGenerationHardlinksExecution500, + StartGenerationHardlinksExecution503, + StartGenerationHardlinksExecutionBody, SyncOverture200, SyncOverture400, SyncOverture401, + SyncOverture409, SyncOverture413, SyncOverture415, SyncOverture429, SyncOverture500, + SyncOverture503, SyncOvertureBody, SyncTransit202, SyncTransit400, @@ -381,6 +516,7 @@ import type { SyncTransit422, SyncTransit429, SyncTransit500, + SyncTransit503, SyncTransitBody, } from "./models"; import { @@ -390,6 +526,7 @@ import { buildAmbientPlacesRequest, buildNotablePlacesRequest, buildSearchIndexRequest, + cancelDataExecutionRequest, conflateOvertureRequest, convertOverpassRequest, discardAmbientPlacesRequest, @@ -400,6 +537,7 @@ import { getAmbientPlacesStatusRequest, getAppliedTrafficConditionsRequest, getCoverageEvidenceRequest, + getDataExecutionRequest, getDatasetsRequest, getLiveRequest, getNotablePlacesStatusRequest, @@ -419,8 +557,11 @@ import { headOfflineGlyphRequest, headOfflinePackageArchiveRequest, ingestOvertureRequest, + inspectDataConfigurationFenceRequest, linkRequest, + lookupDataExecutionRequest, preflightTransitRequest, + prepareDataConfigurationFenceRequest, prepareOfflinePackageRequest, proposeTransitBumpRequest, pullOvertureRequest, @@ -430,6 +571,9 @@ import { resumeAmbientPlacesRequest, rollbackAmbientPlacesRequest, setAmbientPlacesEnabledRequest, + settleDataConfigurationFenceRequest, + startDataExecutionRequest, + startGenerationHardlinksExecutionRequest, syncOvertureRequest, syncTransitRequest, } from "./operation-bindings.generated"; @@ -438,6 +582,818 @@ interface TypedResponse extends Response { json(): Promise; } +export type startDataExecutionResponse202 = { + data: StartDataExecution202; + status: 202; +}; + +export type startDataExecutionResponse400 = { + data: StartDataExecution400; + status: 400; +}; + +export type startDataExecutionResponse401 = { + data: StartDataExecution401; + status: 401; +}; + +export type startDataExecutionResponse403 = { + data: StartDataExecution403; + status: 403; +}; + +export type startDataExecutionResponse409 = { + data: StartDataExecution409; + status: 409; +}; + +export type startDataExecutionResponse413 = { + data: StartDataExecution413; + status: 413; +}; + +export type startDataExecutionResponse415 = { + data: StartDataExecution415; + status: 415; +}; + +export type startDataExecutionResponse429 = { + data: StartDataExecution429; + status: 429; +}; + +export type startDataExecutionResponse500 = { + data: StartDataExecution500; + status: 500; +}; + +export type startDataExecutionResponse503 = { + data: StartDataExecution503; + status: 503; +}; + +export type startDataExecutionResponseSuccess = startDataExecutionResponse202 & { + headers: Headers; +}; +export type startDataExecutionResponseError = ( + | startDataExecutionResponse400 + | startDataExecutionResponse401 + | startDataExecutionResponse403 + | startDataExecutionResponse409 + | startDataExecutionResponse413 + | startDataExecutionResponse415 + | startDataExecutionResponse429 + | startDataExecutionResponse500 + | startDataExecutionResponse503 +) & { + headers: Headers; +}; + +export type startDataExecutionResponse = + | startDataExecutionResponseSuccess + | startDataExecutionResponseError; + +export const getStartDataExecutionUrl = () => { + return `/executions`; +}; + +export const startDataExecution = async ( + startDataExecutionBody: StartDataExecutionBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return startDataExecutionRequest(getStartDataExecutionUrl(), { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(startDataExecutionBody), + }); +}; + +export type startGenerationHardlinksExecutionResponse202 = { + data: StartGenerationHardlinksExecution202; + status: 202; +}; + +export type startGenerationHardlinksExecutionResponse400 = { + data: StartGenerationHardlinksExecution400; + status: 400; +}; + +export type startGenerationHardlinksExecutionResponse401 = { + data: StartGenerationHardlinksExecution401; + status: 401; +}; + +export type startGenerationHardlinksExecutionResponse403 = { + data: StartGenerationHardlinksExecution403; + status: 403; +}; + +export type startGenerationHardlinksExecutionResponse409 = { + data: StartGenerationHardlinksExecution409; + status: 409; +}; + +export type startGenerationHardlinksExecutionResponse413 = { + data: StartGenerationHardlinksExecution413; + status: 413; +}; + +export type startGenerationHardlinksExecutionResponse415 = { + data: StartGenerationHardlinksExecution415; + status: 415; +}; + +export type startGenerationHardlinksExecutionResponse429 = { + data: StartGenerationHardlinksExecution429; + status: 429; +}; + +export type startGenerationHardlinksExecutionResponse500 = { + data: StartGenerationHardlinksExecution500; + status: 500; +}; + +export type startGenerationHardlinksExecutionResponse503 = { + data: StartGenerationHardlinksExecution503; + status: 503; +}; + +export type startGenerationHardlinksExecutionResponseSuccess = + startGenerationHardlinksExecutionResponse202 & { + headers: Headers; + }; +export type startGenerationHardlinksExecutionResponseError = ( + | startGenerationHardlinksExecutionResponse400 + | startGenerationHardlinksExecutionResponse401 + | startGenerationHardlinksExecutionResponse403 + | startGenerationHardlinksExecutionResponse409 + | startGenerationHardlinksExecutionResponse413 + | startGenerationHardlinksExecutionResponse415 + | startGenerationHardlinksExecutionResponse429 + | startGenerationHardlinksExecutionResponse500 + | startGenerationHardlinksExecutionResponse503 +) & { + headers: Headers; +}; + +export type startGenerationHardlinksExecutionResponse = + | startGenerationHardlinksExecutionResponseSuccess + | startGenerationHardlinksExecutionResponseError; + +export const getStartGenerationHardlinksExecutionUrl = () => { + return `/executions/generation-hardlinks`; +}; + +export const startGenerationHardlinksExecution = async ( + startGenerationHardlinksExecutionBody: StartGenerationHardlinksExecutionBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return startGenerationHardlinksExecutionRequest( + getStartGenerationHardlinksExecutionUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(startGenerationHardlinksExecutionBody), + }, + ); +}; + +export type prepareDataConfigurationFenceResponse200 = { + data: PrepareDataConfigurationFence200; + status: 200; +}; + +export type prepareDataConfigurationFenceResponse400 = { + data: PrepareDataConfigurationFence400; + status: 400; +}; + +export type prepareDataConfigurationFenceResponse401 = { + data: PrepareDataConfigurationFence401; + status: 401; +}; + +export type prepareDataConfigurationFenceResponse403 = { + data: PrepareDataConfigurationFence403; + status: 403; +}; + +export type prepareDataConfigurationFenceResponse409 = { + data: PrepareDataConfigurationFence409; + status: 409; +}; + +export type prepareDataConfigurationFenceResponse413 = { + data: PrepareDataConfigurationFence413; + status: 413; +}; + +export type prepareDataConfigurationFenceResponse415 = { + data: PrepareDataConfigurationFence415; + status: 415; +}; + +export type prepareDataConfigurationFenceResponse429 = { + data: PrepareDataConfigurationFence429; + status: 429; +}; + +export type prepareDataConfigurationFenceResponse500 = { + data: PrepareDataConfigurationFence500; + status: 500; +}; + +export type prepareDataConfigurationFenceResponse503 = { + data: PrepareDataConfigurationFence503; + status: 503; +}; + +export type prepareDataConfigurationFenceResponseSuccess = + prepareDataConfigurationFenceResponse200 & { + headers: Headers; + }; +export type prepareDataConfigurationFenceResponseError = ( + | prepareDataConfigurationFenceResponse400 + | prepareDataConfigurationFenceResponse401 + | prepareDataConfigurationFenceResponse403 + | prepareDataConfigurationFenceResponse409 + | prepareDataConfigurationFenceResponse413 + | prepareDataConfigurationFenceResponse415 + | prepareDataConfigurationFenceResponse429 + | prepareDataConfigurationFenceResponse500 + | prepareDataConfigurationFenceResponse503 +) & { + headers: Headers; +}; + +export type prepareDataConfigurationFenceResponse = + | prepareDataConfigurationFenceResponseSuccess + | prepareDataConfigurationFenceResponseError; + +export const getPrepareDataConfigurationFenceUrl = () => { + return `/executions/configuration-maintenance/prepare`; +}; + +export const prepareDataConfigurationFence = async ( + prepareDataConfigurationFenceBody: PrepareDataConfigurationFenceBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return prepareDataConfigurationFenceRequest( + getPrepareDataConfigurationFenceUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(prepareDataConfigurationFenceBody), + }, + ); +}; + +export type inspectDataConfigurationFenceResponse200 = { + data: InspectDataConfigurationFence200; + status: 200; +}; + +export type inspectDataConfigurationFenceResponse400 = { + data: InspectDataConfigurationFence400; + status: 400; +}; + +export type inspectDataConfigurationFenceResponse401 = { + data: InspectDataConfigurationFence401; + status: 401; +}; + +export type inspectDataConfigurationFenceResponse403 = { + data: InspectDataConfigurationFence403; + status: 403; +}; + +export type inspectDataConfigurationFenceResponse404 = { + data: InspectDataConfigurationFence404; + status: 404; +}; + +export type inspectDataConfigurationFenceResponse413 = { + data: InspectDataConfigurationFence413; + status: 413; +}; + +export type inspectDataConfigurationFenceResponse415 = { + data: InspectDataConfigurationFence415; + status: 415; +}; + +export type inspectDataConfigurationFenceResponse429 = { + data: InspectDataConfigurationFence429; + status: 429; +}; + +export type inspectDataConfigurationFenceResponse500 = { + data: InspectDataConfigurationFence500; + status: 500; +}; + +export type inspectDataConfigurationFenceResponse503 = { + data: InspectDataConfigurationFence503; + status: 503; +}; + +export type inspectDataConfigurationFenceResponseSuccess = + inspectDataConfigurationFenceResponse200 & { + headers: Headers; + }; +export type inspectDataConfigurationFenceResponseError = ( + | inspectDataConfigurationFenceResponse400 + | inspectDataConfigurationFenceResponse401 + | inspectDataConfigurationFenceResponse403 + | inspectDataConfigurationFenceResponse404 + | inspectDataConfigurationFenceResponse413 + | inspectDataConfigurationFenceResponse415 + | inspectDataConfigurationFenceResponse429 + | inspectDataConfigurationFenceResponse500 + | inspectDataConfigurationFenceResponse503 +) & { + headers: Headers; +}; + +export type inspectDataConfigurationFenceResponse = + | inspectDataConfigurationFenceResponseSuccess + | inspectDataConfigurationFenceResponseError; + +export const getInspectDataConfigurationFenceUrl = (maintenanceId: string) => { + return `/executions/configuration-maintenance/${encodeURIComponent(String(maintenanceId))}`; +}; + +export const inspectDataConfigurationFence = async ( + maintenanceId: string, + options?: Parameters[1], +): Promise => { + return inspectDataConfigurationFenceRequest( + getInspectDataConfigurationFenceUrl(maintenanceId), + { + ...options, + method: "GET", + }, + ); +}; + +export type settleDataConfigurationFenceResponse200 = { + data: SettleDataConfigurationFence200; + status: 200; +}; + +export type settleDataConfigurationFenceResponse400 = { + data: SettleDataConfigurationFence400; + status: 400; +}; + +export type settleDataConfigurationFenceResponse401 = { + data: SettleDataConfigurationFence401; + status: 401; +}; + +export type settleDataConfigurationFenceResponse403 = { + data: SettleDataConfigurationFence403; + status: 403; +}; + +export type settleDataConfigurationFenceResponse404 = { + data: SettleDataConfigurationFence404; + status: 404; +}; + +export type settleDataConfigurationFenceResponse409 = { + data: SettleDataConfigurationFence409; + status: 409; +}; + +export type settleDataConfigurationFenceResponse413 = { + data: SettleDataConfigurationFence413; + status: 413; +}; + +export type settleDataConfigurationFenceResponse415 = { + data: SettleDataConfigurationFence415; + status: 415; +}; + +export type settleDataConfigurationFenceResponse429 = { + data: SettleDataConfigurationFence429; + status: 429; +}; + +export type settleDataConfigurationFenceResponse500 = { + data: SettleDataConfigurationFence500; + status: 500; +}; + +export type settleDataConfigurationFenceResponse503 = { + data: SettleDataConfigurationFence503; + status: 503; +}; + +export type settleDataConfigurationFenceResponseSuccess = + settleDataConfigurationFenceResponse200 & { + headers: Headers; + }; +export type settleDataConfigurationFenceResponseError = ( + | settleDataConfigurationFenceResponse400 + | settleDataConfigurationFenceResponse401 + | settleDataConfigurationFenceResponse403 + | settleDataConfigurationFenceResponse404 + | settleDataConfigurationFenceResponse409 + | settleDataConfigurationFenceResponse413 + | settleDataConfigurationFenceResponse415 + | settleDataConfigurationFenceResponse429 + | settleDataConfigurationFenceResponse500 + | settleDataConfigurationFenceResponse503 +) & { + headers: Headers; +}; + +export type settleDataConfigurationFenceResponse = + | settleDataConfigurationFenceResponseSuccess + | settleDataConfigurationFenceResponseError; + +export const getSettleDataConfigurationFenceUrl = (maintenanceId: string) => { + return `/executions/configuration-maintenance/${encodeURIComponent(String(maintenanceId))}/settle`; +}; + +export const settleDataConfigurationFence = async ( + maintenanceId: string, + settleDataConfigurationFenceBody: SettleDataConfigurationFenceBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return settleDataConfigurationFenceRequest( + getSettleDataConfigurationFenceUrl(maintenanceId), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(settleDataConfigurationFenceBody), + }, + ); +}; + +export type lookupDataExecutionResponse200 = { + data: LookupDataExecution200; + status: 200; +}; + +export type lookupDataExecutionResponse400 = { + data: LookupDataExecution400; + status: 400; +}; + +export type lookupDataExecutionResponse401 = { + data: LookupDataExecution401; + status: 401; +}; + +export type lookupDataExecutionResponse404 = { + data: LookupDataExecution404; + status: 404; +}; + +export type lookupDataExecutionResponse410 = { + data: LookupDataExecution410; + status: 410; +}; + +export type lookupDataExecutionResponse413 = { + data: LookupDataExecution413; + status: 413; +}; + +export type lookupDataExecutionResponse415 = { + data: LookupDataExecution415; + status: 415; +}; + +export type lookupDataExecutionResponse429 = { + data: LookupDataExecution429; + status: 429; +}; + +export type lookupDataExecutionResponse500 = { + data: LookupDataExecution500; + status: 500; +}; + +export type lookupDataExecutionResponse503 = { + data: LookupDataExecution503; + status: 503; +}; + +export type lookupDataExecutionResponseSuccess = lookupDataExecutionResponse200 & { + headers: Headers; +}; +export type lookupDataExecutionResponseError = ( + | lookupDataExecutionResponse400 + | lookupDataExecutionResponse401 + | lookupDataExecutionResponse404 + | lookupDataExecutionResponse410 + | lookupDataExecutionResponse413 + | lookupDataExecutionResponse415 + | lookupDataExecutionResponse429 + | lookupDataExecutionResponse500 + | lookupDataExecutionResponse503 +) & { + headers: Headers; +}; + +export type lookupDataExecutionResponse = + | lookupDataExecutionResponseSuccess + | lookupDataExecutionResponseError; + +export const getLookupDataExecutionUrl = (operationKey: string) => { + return `/executions/by-key/${encodeURIComponent(String(operationKey))}`; +}; + +export const lookupDataExecution = async ( + operationKey: string, + options?: Parameters[1], +): Promise => { + return lookupDataExecutionRequest( + getLookupDataExecutionUrl(operationKey), + { + ...options, + method: "GET", + }, + ); +}; + +export type getDataExecutionResponse200 = { + data: GetDataExecution200; + status: 200; +}; + +export type getDataExecutionResponse400 = { + data: GetDataExecution400; + status: 400; +}; + +export type getDataExecutionResponse401 = { + data: GetDataExecution401; + status: 401; +}; + +export type getDataExecutionResponse404 = { + data: GetDataExecution404; + status: 404; +}; + +export type getDataExecutionResponse410 = { + data: GetDataExecution410; + status: 410; +}; + +export type getDataExecutionResponse413 = { + data: GetDataExecution413; + status: 413; +}; + +export type getDataExecutionResponse415 = { + data: GetDataExecution415; + status: 415; +}; + +export type getDataExecutionResponse429 = { + data: GetDataExecution429; + status: 429; +}; + +export type getDataExecutionResponse500 = { + data: GetDataExecution500; + status: 500; +}; + +export type getDataExecutionResponse503 = { + data: GetDataExecution503; + status: 503; +}; + +export type getDataExecutionResponseSuccess = getDataExecutionResponse200 & { + headers: Headers; +}; +export type getDataExecutionResponseError = ( + | getDataExecutionResponse400 + | getDataExecutionResponse401 + | getDataExecutionResponse404 + | getDataExecutionResponse410 + | getDataExecutionResponse413 + | getDataExecutionResponse415 + | getDataExecutionResponse429 + | getDataExecutionResponse500 + | getDataExecutionResponse503 +) & { + headers: Headers; +}; + +export type getDataExecutionResponse = + | getDataExecutionResponseSuccess + | getDataExecutionResponseError; + +export const getGetDataExecutionUrl = (id: string) => { + return `/executions/${encodeURIComponent(String(id))}`; +}; + +export const getDataExecution = async ( + id: string, + options?: Parameters[1], +): Promise => { + return getDataExecutionRequest(getGetDataExecutionUrl(id), { + ...options, + method: "GET", + }); +}; + +export type cancelDataExecutionResponse200 = { + data: CancelDataExecution200; + status: 200; +}; + +export type cancelDataExecutionResponse400 = { + data: CancelDataExecution400; + status: 400; +}; + +export type cancelDataExecutionResponse401 = { + data: CancelDataExecution401; + status: 401; +}; + +export type cancelDataExecutionResponse404 = { + data: CancelDataExecution404; + status: 404; +}; + +export type cancelDataExecutionResponse409 = { + data: CancelDataExecution409; + status: 409; +}; + +export type cancelDataExecutionResponse413 = { + data: CancelDataExecution413; + status: 413; +}; + +export type cancelDataExecutionResponse415 = { + data: CancelDataExecution415; + status: 415; +}; + +export type cancelDataExecutionResponse429 = { + data: CancelDataExecution429; + status: 429; +}; + +export type cancelDataExecutionResponse500 = { + data: CancelDataExecution500; + status: 500; +}; + +export type cancelDataExecutionResponse503 = { + data: CancelDataExecution503; + status: 503; +}; + +export type cancelDataExecutionResponseSuccess = cancelDataExecutionResponse200 & { + headers: Headers; +}; +export type cancelDataExecutionResponseError = ( + | cancelDataExecutionResponse400 + | cancelDataExecutionResponse401 + | cancelDataExecutionResponse404 + | cancelDataExecutionResponse409 + | cancelDataExecutionResponse413 + | cancelDataExecutionResponse415 + | cancelDataExecutionResponse429 + | cancelDataExecutionResponse500 + | cancelDataExecutionResponse503 +) & { + headers: Headers; +}; + +export type cancelDataExecutionResponse = + | cancelDataExecutionResponseSuccess + | cancelDataExecutionResponseError; + +export const getCancelDataExecutionUrl = (id: string) => { + return `/executions/${encodeURIComponent(String(id))}/cancel`; +}; + +export const cancelDataExecution = async ( + id: string, + cancelDataExecutionBody: CancelDataExecutionBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return cancelDataExecutionRequest(getCancelDataExecutionUrl(id), { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(cancelDataExecutionBody), + }); +}; + export type getAmbientPlacesStatusResponse200 = { data: GetAmbientPlacesStatus200; status: 200; @@ -547,6 +1503,11 @@ export type buildAmbientPlacesResponse500 = { status: 500; }; +export type buildAmbientPlacesResponse503 = { + data: BuildAmbientPlaces503; + status: 503; +}; + export type buildAmbientPlacesResponseSuccess = buildAmbientPlacesResponse202 & { headers: Headers; }; @@ -558,6 +1519,7 @@ export type buildAmbientPlacesResponseError = ( | buildAmbientPlacesResponse415 | buildAmbientPlacesResponse429 | buildAmbientPlacesResponse500 + | buildAmbientPlacesResponse503 ) & { headers: Headers; }; @@ -641,6 +1603,11 @@ export type resumeAmbientPlacesResponse500 = { status: 500; }; +export type resumeAmbientPlacesResponse503 = { + data: ResumeAmbientPlaces503; + status: 503; +}; + export type resumeAmbientPlacesResponseSuccess = resumeAmbientPlacesResponse202 & { headers: Headers; }; @@ -652,6 +1619,7 @@ export type resumeAmbientPlacesResponseError = ( | resumeAmbientPlacesResponse415 | resumeAmbientPlacesResponse429 | resumeAmbientPlacesResponse500 + | resumeAmbientPlacesResponse503 ) & { headers: Headers; }; @@ -735,6 +1703,11 @@ export type discardAmbientPlacesResponse500 = { status: 500; }; +export type discardAmbientPlacesResponse503 = { + data: DiscardAmbientPlaces503; + status: 503; +}; + export type discardAmbientPlacesResponseSuccess = discardAmbientPlacesResponse200 & { headers: Headers; }; @@ -746,6 +1719,7 @@ export type discardAmbientPlacesResponseError = ( | discardAmbientPlacesResponse415 | discardAmbientPlacesResponse429 | discardAmbientPlacesResponse500 + | discardAmbientPlacesResponse503 ) & { headers: Headers; }; @@ -829,6 +1803,11 @@ export type setAmbientPlacesEnabledResponse500 = { status: 500; }; +export type setAmbientPlacesEnabledResponse503 = { + data: SetAmbientPlacesEnabled503; + status: 503; +}; + export type setAmbientPlacesEnabledResponseSuccess = setAmbientPlacesEnabledResponse200 & { headers: Headers; }; @@ -840,6 +1819,7 @@ export type setAmbientPlacesEnabledResponseError = ( | setAmbientPlacesEnabledResponse415 | setAmbientPlacesEnabledResponse429 | setAmbientPlacesEnabledResponse500 + | setAmbientPlacesEnabledResponse503 ) & { headers: Headers; }; @@ -926,6 +1906,11 @@ export type rollbackAmbientPlacesResponse500 = { status: 500; }; +export type rollbackAmbientPlacesResponse503 = { + data: RollbackAmbientPlaces503; + status: 503; +}; + export type rollbackAmbientPlacesResponseSuccess = rollbackAmbientPlacesResponse200 & { headers: Headers; }; @@ -937,6 +1922,7 @@ export type rollbackAmbientPlacesResponseError = ( | rollbackAmbientPlacesResponse415 | rollbackAmbientPlacesResponse429 | rollbackAmbientPlacesResponse500 + | rollbackAmbientPlacesResponse503 ) & { headers: Headers; }; @@ -1262,6 +2248,11 @@ export type getOfflinePackageJobResponse404 = { status: 404; }; +export type getOfflinePackageJobResponse409 = { + data: GetOfflinePackageJob409; + status: 409; +}; + export type getOfflinePackageJobResponse413 = { data: GetOfflinePackageJob413; status: 413; @@ -1282,6 +2273,11 @@ export type getOfflinePackageJobResponse500 = { status: 500; }; +export type getOfflinePackageJobResponse503 = { + data: GetOfflinePackageJob503; + status: 503; +}; + export type getOfflinePackageJobResponseSuccess = getOfflinePackageJobResponse200 & { headers: Headers; }; @@ -1289,10 +2285,12 @@ export type getOfflinePackageJobResponseError = ( | getOfflinePackageJobResponse400 | getOfflinePackageJobResponse401 | getOfflinePackageJobResponse404 + | getOfflinePackageJobResponse409 | getOfflinePackageJobResponse413 | getOfflinePackageJobResponse415 | getOfflinePackageJobResponse429 | getOfflinePackageJobResponse500 + | getOfflinePackageJobResponse503 ) & { headers: Headers; }; @@ -1359,6 +2357,11 @@ export type getOfflinePackageManifestResponse404 = { status: 404; }; +export type getOfflinePackageManifestResponse409 = { + data: GetOfflinePackageManifest409; + status: 409; +}; + export type getOfflinePackageManifestResponse413 = { data: GetOfflinePackageManifest413; status: 413; @@ -1379,6 +2382,11 @@ export type getOfflinePackageManifestResponse500 = { status: 500; }; +export type getOfflinePackageManifestResponse503 = { + data: GetOfflinePackageManifest503; + status: 503; +}; + export type getOfflinePackageManifestResponseSuccess = getOfflinePackageManifestResponse200 & { headers: Headers; }; @@ -1386,10 +2394,12 @@ export type getOfflinePackageManifestResponseError = ( | getOfflinePackageManifestResponse400 | getOfflinePackageManifestResponse401 | getOfflinePackageManifestResponse404 + | getOfflinePackageManifestResponse409 | getOfflinePackageManifestResponse413 | getOfflinePackageManifestResponse415 | getOfflinePackageManifestResponse429 | getOfflinePackageManifestResponse500 + | getOfflinePackageManifestResponse503 ) & { headers: Headers; }; @@ -1435,6 +2445,11 @@ export type getOfflineGlyphCatalogResponse404 = { status: 404; }; +export type getOfflineGlyphCatalogResponse409 = { + data: GetOfflineGlyphCatalog409; + status: 409; +}; + export type getOfflineGlyphCatalogResponse413 = { data: GetOfflineGlyphCatalog413; status: 413; @@ -1455,6 +2470,11 @@ export type getOfflineGlyphCatalogResponse500 = { status: 500; }; +export type getOfflineGlyphCatalogResponse503 = { + data: GetOfflineGlyphCatalog503; + status: 503; +}; + export type getOfflineGlyphCatalogResponseSuccess = getOfflineGlyphCatalogResponse200 & { headers: Headers; }; @@ -1462,10 +2482,12 @@ export type getOfflineGlyphCatalogResponseError = ( | getOfflineGlyphCatalogResponse400 | getOfflineGlyphCatalogResponse401 | getOfflineGlyphCatalogResponse404 + | getOfflineGlyphCatalogResponse409 | getOfflineGlyphCatalogResponse413 | getOfflineGlyphCatalogResponse415 | getOfflineGlyphCatalogResponse429 | getOfflineGlyphCatalogResponse500 + | getOfflineGlyphCatalogResponse503 ) & { headers: Headers; }; @@ -1577,6 +2599,11 @@ export type getOfflineGlyphResponse404 = { status: 404; }; +export type getOfflineGlyphResponse409 = { + data: GetOfflineGlyph409; + status: 409; +}; + export type getOfflineGlyphResponse413 = { data: GetOfflineGlyph413; status: 413; @@ -1597,6 +2624,11 @@ export type getOfflineGlyphResponse500 = { status: 500; }; +export type getOfflineGlyphResponse503 = { + data: GetOfflineGlyph503; + status: 503; +}; + export type getOfflineGlyphResponseSuccess = getOfflineGlyphResponse200 & { headers: Headers; }; @@ -1604,10 +2636,12 @@ export type getOfflineGlyphResponseError = ( | getOfflineGlyphResponse400 | getOfflineGlyphResponse401 | getOfflineGlyphResponse404 + | getOfflineGlyphResponse409 | getOfflineGlyphResponse413 | getOfflineGlyphResponse415 | getOfflineGlyphResponse429 | getOfflineGlyphResponse500 + | getOfflineGlyphResponse503 ) & { headers: Headers; }; @@ -1757,6 +2791,11 @@ export type getOfflinePackageArchiveResponse404 = { status: 404; }; +export type getOfflinePackageArchiveResponse409 = { + data: GetOfflinePackageArchive409; + status: 409; +}; + export type getOfflinePackageArchiveResponse413 = { data: GetOfflinePackageArchive413; status: 413; @@ -1782,6 +2821,11 @@ export type getOfflinePackageArchiveResponse500 = { status: 500; }; +export type getOfflinePackageArchiveResponse503 = { + data: GetOfflinePackageArchive503; + status: 503; +}; + export type getOfflinePackageArchiveResponseSuccess = ( | getOfflinePackageArchiveResponse200 | getOfflinePackageArchiveResponse206 @@ -1792,11 +2836,13 @@ export type getOfflinePackageArchiveResponseError = ( | getOfflinePackageArchiveResponse400 | getOfflinePackageArchiveResponse401 | getOfflinePackageArchiveResponse404 + | getOfflinePackageArchiveResponse409 | getOfflinePackageArchiveResponse413 | getOfflinePackageArchiveResponse415 | getOfflinePackageArchiveResponse416 | getOfflinePackageArchiveResponse429 | getOfflinePackageArchiveResponse500 + | getOfflinePackageArchiveResponse503 ) & { headers: Headers; }; @@ -2274,6 +3320,11 @@ export type addTransitSourceResponse500 = { status: 500; }; +export type addTransitSourceResponse503 = { + data: AddTransitSource503; + status: 503; +}; + export type addTransitSourceResponseSuccess = addTransitSourceResponse202 & { headers: Headers; }; @@ -2286,6 +3337,7 @@ export type addTransitSourceResponseError = ( | addTransitSourceResponse415 | addTransitSourceResponse429 | addTransitSourceResponse500 + | addTransitSourceResponse503 ) & { headers: Headers; }; @@ -2452,6 +3504,11 @@ export type removeTransitSourceResponse500 = { status: 500; }; +export type removeTransitSourceResponse503 = { + data: RemoveTransitSource503; + status: 503; +}; + export type removeTransitSourceResponseSuccess = removeTransitSourceResponse202 & { headers: Headers; }; @@ -2464,6 +3521,7 @@ export type removeTransitSourceResponseError = ( | removeTransitSourceResponse415 | removeTransitSourceResponse429 | removeTransitSourceResponse500 + | removeTransitSourceResponse503 ) & { headers: Headers; }; @@ -2556,6 +3614,11 @@ export type enableTransitSourceResponse500 = { status: 500; }; +export type enableTransitSourceResponse503 = { + data: EnableTransitSource503; + status: 503; +}; + export type enableTransitSourceResponseSuccess = enableTransitSourceResponse202 & { headers: Headers; }; @@ -2568,6 +3631,7 @@ export type enableTransitSourceResponseError = ( | enableTransitSourceResponse415 | enableTransitSourceResponse429 | enableTransitSourceResponse500 + | enableTransitSourceResponse503 ) & { headers: Headers; }; @@ -2630,6 +3694,11 @@ export type preflightTransitResponse401 = { status: 401; }; +export type preflightTransitResponse409 = { + data: PreflightTransit409; + status: 409; +}; + export type preflightTransitResponse413 = { data: PreflightTransit413; status: 413; @@ -2655,17 +3724,24 @@ export type preflightTransitResponse500 = { status: 500; }; +export type preflightTransitResponse503 = { + data: PreflightTransit503; + status: 503; +}; + export type preflightTransitResponseSuccess = preflightTransitResponse200 & { headers: Headers; }; export type preflightTransitResponseError = ( | preflightTransitResponse400 | preflightTransitResponse401 + | preflightTransitResponse409 | preflightTransitResponse413 | preflightTransitResponse415 | preflightTransitResponse422 | preflightTransitResponse429 | preflightTransitResponse500 + | preflightTransitResponse503 ) & { headers: Headers; }; @@ -2724,6 +3800,11 @@ export type reloadDatasetsResponse401 = { status: 401; }; +export type reloadDatasetsResponse409 = { + data: ReloadDatasets409; + status: 409; +}; + export type reloadDatasetsResponse413 = { data: ReloadDatasets413; status: 413; @@ -2744,16 +3825,23 @@ export type reloadDatasetsResponse500 = { status: 500; }; +export type reloadDatasetsResponse503 = { + data: ReloadDatasets503; + status: 503; +}; + export type reloadDatasetsResponseSuccess = reloadDatasetsResponse200 & { headers: Headers; }; export type reloadDatasetsResponseError = ( | reloadDatasetsResponse400 | reloadDatasetsResponse401 + | reloadDatasetsResponse409 | reloadDatasetsResponse413 | reloadDatasetsResponse415 | reloadDatasetsResponse429 | reloadDatasetsResponse500 + | reloadDatasetsResponse503 ) & { headers: Headers; }; @@ -2818,6 +3906,11 @@ export type bakePredictedTrafficResponse501 = { status: 501; }; +export type bakePredictedTrafficResponse503 = { + data: BakePredictedTraffic503; + status: 503; +}; + export type bakePredictedTrafficResponseSuccess = bakePredictedTrafficResponse202 & { headers: Headers; }; @@ -2830,6 +3923,7 @@ export type bakePredictedTrafficResponseError = ( | bakePredictedTrafficResponse429 | bakePredictedTrafficResponse500 | bakePredictedTrafficResponse501 + | bakePredictedTrafficResponse503 ) & { headers: Headers; }; @@ -2941,6 +4035,11 @@ export type downloadOsmResponse401 = { status: 401; }; +export type downloadOsmResponse409 = { + data: DownloadOsm409; + status: 409; +}; + export type downloadOsmResponse413 = { data: DownloadOsm413; status: 413; @@ -2961,16 +4060,23 @@ export type downloadOsmResponse500 = { status: 500; }; +export type downloadOsmResponse503 = { + data: DownloadOsm503; + status: 503; +}; + export type downloadOsmResponseSuccess = downloadOsmResponse200 & { headers: Headers; }; export type downloadOsmResponseError = ( | downloadOsmResponse400 | downloadOsmResponse401 + | downloadOsmResponse409 | downloadOsmResponse413 | downloadOsmResponse415 | downloadOsmResponse429 | downloadOsmResponse500 + | downloadOsmResponse503 ) & { headers: Headers; }; @@ -3099,6 +4205,11 @@ export type buildSearchIndexResponse401 = { status: 401; }; +export type buildSearchIndexResponse409 = { + data: BuildSearchIndex409; + status: 409; +}; + export type buildSearchIndexResponse413 = { data: BuildSearchIndex413; status: 413; @@ -3119,16 +4230,23 @@ export type buildSearchIndexResponse500 = { status: 500; }; +export type buildSearchIndexResponse503 = { + data: BuildSearchIndex503; + status: 503; +}; + export type buildSearchIndexResponseSuccess = buildSearchIndexResponse200 & { headers: Headers; }; export type buildSearchIndexResponseError = ( | buildSearchIndexResponse400 | buildSearchIndexResponse401 + | buildSearchIndexResponse409 | buildSearchIndexResponse413 | buildSearchIndexResponse415 | buildSearchIndexResponse429 | buildSearchIndexResponse500 + | buildSearchIndexResponse503 ) & { headers: Headers; }; @@ -3262,6 +4380,11 @@ export type buildNotablePlacesResponse401 = { status: 401; }; +export type buildNotablePlacesResponse409 = { + data: BuildNotablePlaces409; + status: 409; +}; + export type buildNotablePlacesResponse413 = { data: BuildNotablePlaces413; status: 413; @@ -3282,16 +4405,23 @@ export type buildNotablePlacesResponse500 = { status: 500; }; +export type buildNotablePlacesResponse503 = { + data: BuildNotablePlaces503; + status: 503; +}; + export type buildNotablePlacesResponseSuccess = buildNotablePlacesResponse200 & { headers: Headers; }; export type buildNotablePlacesResponseError = ( | buildNotablePlacesResponse400 | buildNotablePlacesResponse401 + | buildNotablePlacesResponse409 | buildNotablePlacesResponse413 | buildNotablePlacesResponse415 | buildNotablePlacesResponse429 | buildNotablePlacesResponse500 + | buildNotablePlacesResponse503 ) & { headers: Headers; }; @@ -3328,6 +4458,11 @@ export type downloadFontsResponse401 = { status: 401; }; +export type downloadFontsResponse409 = { + data: DownloadFonts409; + status: 409; +}; + export type downloadFontsResponse413 = { data: DownloadFonts413; status: 413; @@ -3348,16 +4483,23 @@ export type downloadFontsResponse500 = { status: 500; }; +export type downloadFontsResponse503 = { + data: DownloadFonts503; + status: 503; +}; + export type downloadFontsResponseSuccess = downloadFontsResponse200 & { headers: Headers; }; export type downloadFontsResponseError = ( | downloadFontsResponse400 | downloadFontsResponse401 + | downloadFontsResponse409 | downloadFontsResponse413 | downloadFontsResponse415 | downloadFontsResponse429 | downloadFontsResponse500 + | downloadFontsResponse503 ) & { headers: Headers; }; @@ -3392,6 +4534,11 @@ export type linkResponse401 = { status: 401; }; +export type linkResponse409 = { + data: Link409; + status: 409; +}; + export type linkResponse413 = { data: Link413; status: 413; @@ -3412,16 +4559,23 @@ export type linkResponse500 = { status: 500; }; +export type linkResponse503 = { + data: Link503; + status: 503; +}; + export type linkResponseSuccess = linkResponse200 & { headers: Headers; }; export type linkResponseError = ( | linkResponse400 | linkResponse401 + | linkResponse409 | linkResponse413 | linkResponse415 | linkResponse429 | linkResponse500 + | linkResponse503 ) & { headers: Headers; }; @@ -3478,6 +4632,11 @@ export type convertOverpassResponse401 = { status: 401; }; +export type convertOverpassResponse409 = { + data: ConvertOverpass409; + status: 409; +}; + export type convertOverpassResponse413 = { data: ConvertOverpass413; status: 413; @@ -3498,16 +4657,23 @@ export type convertOverpassResponse500 = { status: 500; }; +export type convertOverpassResponse503 = { + data: ConvertOverpass503; + status: 503; +}; + export type convertOverpassResponseSuccess = convertOverpassResponse200 & { headers: Headers; }; export type convertOverpassResponseError = ( | convertOverpassResponse400 | convertOverpassResponse401 + | convertOverpassResponse409 | convertOverpassResponse413 | convertOverpassResponse415 | convertOverpassResponse429 | convertOverpassResponse500 + | convertOverpassResponse503 ) & { headers: Headers; }; @@ -3564,6 +4730,11 @@ export type pullOvertureResponse401 = { status: 401; }; +export type pullOvertureResponse409 = { + data: PullOverture409; + status: 409; +}; + export type pullOvertureResponse413 = { data: PullOverture413; status: 413; @@ -3584,16 +4755,23 @@ export type pullOvertureResponse500 = { status: 500; }; +export type pullOvertureResponse503 = { + data: PullOverture503; + status: 503; +}; + export type pullOvertureResponseSuccess = pullOvertureResponse200 & { headers: Headers; }; export type pullOvertureResponseError = ( | pullOvertureResponse400 | pullOvertureResponse401 + | pullOvertureResponse409 | pullOvertureResponse413 | pullOvertureResponse415 | pullOvertureResponse429 | pullOvertureResponse500 + | pullOvertureResponse503 ) & { headers: Headers; }; @@ -3722,6 +4900,11 @@ export type syncOvertureResponse401 = { status: 401; }; +export type syncOvertureResponse409 = { + data: SyncOverture409; + status: 409; +}; + export type syncOvertureResponse413 = { data: SyncOverture413; status: 413; @@ -3742,16 +4925,23 @@ export type syncOvertureResponse500 = { status: 500; }; +export type syncOvertureResponse503 = { + data: SyncOverture503; + status: 503; +}; + export type syncOvertureResponseSuccess = syncOvertureResponse200 & { headers: Headers; }; export type syncOvertureResponseError = ( | syncOvertureResponse400 | syncOvertureResponse401 + | syncOvertureResponse409 | syncOvertureResponse413 | syncOvertureResponse415 | syncOvertureResponse429 | syncOvertureResponse500 + | syncOvertureResponse503 ) & { headers: Headers; }; @@ -3808,6 +4998,11 @@ export type ingestOvertureResponse401 = { status: 401; }; +export type ingestOvertureResponse409 = { + data: IngestOverture409; + status: 409; +}; + export type ingestOvertureResponse413 = { data: IngestOverture413; status: 413; @@ -3828,16 +5023,23 @@ export type ingestOvertureResponse500 = { status: 500; }; +export type ingestOvertureResponse503 = { + data: IngestOverture503; + status: 503; +}; + export type ingestOvertureResponseSuccess = ingestOvertureResponse200 & { headers: Headers; }; export type ingestOvertureResponseError = ( | ingestOvertureResponse400 | ingestOvertureResponse401 + | ingestOvertureResponse409 | ingestOvertureResponse413 | ingestOvertureResponse415 | ingestOvertureResponse429 | ingestOvertureResponse500 + | ingestOvertureResponse503 ) & { headers: Headers; }; @@ -3894,6 +5096,11 @@ export type extractOvertureResponse401 = { status: 401; }; +export type extractOvertureResponse409 = { + data: ExtractOverture409; + status: 409; +}; + export type extractOvertureResponse413 = { data: ExtractOverture413; status: 413; @@ -3914,16 +5121,23 @@ export type extractOvertureResponse500 = { status: 500; }; +export type extractOvertureResponse503 = { + data: ExtractOverture503; + status: 503; +}; + export type extractOvertureResponseSuccess = extractOvertureResponse200 & { headers: Headers; }; export type extractOvertureResponseError = ( | extractOvertureResponse400 | extractOvertureResponse401 + | extractOvertureResponse409 | extractOvertureResponse413 | extractOvertureResponse415 | extractOvertureResponse429 | extractOvertureResponse500 + | extractOvertureResponse503 ) & { headers: Headers; }; @@ -3980,6 +5194,11 @@ export type conflateOvertureResponse401 = { status: 401; }; +export type conflateOvertureResponse409 = { + data: ConflateOverture409; + status: 409; +}; + export type conflateOvertureResponse413 = { data: ConflateOverture413; status: 413; @@ -4000,16 +5219,23 @@ export type conflateOvertureResponse500 = { status: 500; }; +export type conflateOvertureResponse503 = { + data: ConflateOverture503; + status: 503; +}; + export type conflateOvertureResponseSuccess = conflateOvertureResponse200 & { headers: Headers; }; export type conflateOvertureResponseError = ( | conflateOvertureResponse400 | conflateOvertureResponse401 + | conflateOvertureResponse409 | conflateOvertureResponse413 | conflateOvertureResponse415 | conflateOvertureResponse429 | conflateOvertureResponse500 + | conflateOvertureResponse503 ) & { headers: Headers; }; @@ -4098,6 +5324,11 @@ export type syncTransitResponse500 = { status: 500; }; +export type syncTransitResponse503 = { + data: SyncTransit503; + status: 503; +}; + export type syncTransitResponseSuccess = syncTransitResponse202 & { headers: Headers; }; @@ -4110,6 +5341,7 @@ export type syncTransitResponseError = ( | syncTransitResponse422 | syncTransitResponse429 | syncTransitResponse500 + | syncTransitResponse503 ) & { headers: Headers; }; @@ -4166,6 +5398,11 @@ export type restartMotisResponse401 = { status: 401; }; +export type restartMotisResponse409 = { + data: RestartMotis409; + status: 409; +}; + export type restartMotisResponse413 = { data: RestartMotis413; status: 413; @@ -4197,6 +5434,7 @@ export type restartMotisResponseSuccess = restartMotisResponse200 & { export type restartMotisResponseError = ( | restartMotisResponse400 | restartMotisResponse401 + | restartMotisResponse409 | restartMotisResponse413 | restartMotisResponse415 | restartMotisResponse429 @@ -4266,6 +5504,11 @@ export type proposeTransitBumpResponse502 = { status: 502; }; +export type proposeTransitBumpResponse503 = { + data: ProposeTransitBump503; + status: 503; +}; + export type proposeTransitBumpResponseSuccess = proposeTransitBumpResponse200 & { headers: Headers; }; @@ -4278,6 +5521,7 @@ export type proposeTransitBumpResponseError = ( | proposeTransitBumpResponse429 | proposeTransitBumpResponse500 | proposeTransitBumpResponse502 + | proposeTransitBumpResponse503 ) & { headers: Headers; }; @@ -4341,6 +5585,11 @@ export type approveTransitBumpResponse404 = { status: 404; }; +export type approveTransitBumpResponse409 = { + data: ApproveTransitBump409; + status: 409; +}; + export type approveTransitBumpResponse413 = { data: ApproveTransitBump413; status: 413; @@ -4366,6 +5615,11 @@ export type approveTransitBumpResponse500 = { status: 500; }; +export type approveTransitBumpResponse503 = { + data: ApproveTransitBump503; + status: 503; +}; + export type approveTransitBumpResponseSuccess = approveTransitBumpResponse200 & { headers: Headers; }; @@ -4373,11 +5627,13 @@ export type approveTransitBumpResponseError = ( | approveTransitBumpResponse400 | approveTransitBumpResponse401 | approveTransitBumpResponse404 + | approveTransitBumpResponse409 | approveTransitBumpResponse413 | approveTransitBumpResponse415 | approveTransitBumpResponse422 | approveTransitBumpResponse429 | approveTransitBumpResponse500 + | approveTransitBumpResponse503 ) & { headers: Headers; }; diff --git a/packages/core/src/services/data-manager/generated/models/addTransitSource503.ts b/packages/core/src/services/data-manager/generated/models/addTransitSource503.ts new file mode 100644 index 000000000..ce16272fb --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/addTransitSource503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type AddTransitSource503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/approveTransitBump409.ts b/packages/core/src/services/data-manager/generated/models/approveTransitBump409.ts new file mode 100644 index 000000000..1f9f95d67 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/approveTransitBump409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ApproveTransitBump409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/approveTransitBump503.ts b/packages/core/src/services/data-manager/generated/models/approveTransitBump503.ts new file mode 100644 index 000000000..76b1b14a7 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/approveTransitBump503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ApproveTransitBump503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/bakePredictedTraffic503.ts b/packages/core/src/services/data-manager/generated/models/bakePredictedTraffic503.ts new file mode 100644 index 000000000..936468b60 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/bakePredictedTraffic503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type BakePredictedTraffic503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/buildAmbientPlaces503.ts b/packages/core/src/services/data-manager/generated/models/buildAmbientPlaces503.ts new file mode 100644 index 000000000..b7155bf3d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/buildAmbientPlaces503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type BuildAmbientPlaces503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/buildNotablePlaces409.ts b/packages/core/src/services/data-manager/generated/models/buildNotablePlaces409.ts new file mode 100644 index 000000000..49f762334 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/buildNotablePlaces409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type BuildNotablePlaces409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/buildNotablePlaces503.ts b/packages/core/src/services/data-manager/generated/models/buildNotablePlaces503.ts new file mode 100644 index 000000000..d09013b0f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/buildNotablePlaces503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type BuildNotablePlaces503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/buildSearchIndex409.ts b/packages/core/src/services/data-manager/generated/models/buildSearchIndex409.ts new file mode 100644 index 000000000..6b5c72453 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/buildSearchIndex409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type BuildSearchIndex409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/buildSearchIndex503.ts b/packages/core/src/services/data-manager/generated/models/buildSearchIndex503.ts new file mode 100644 index 000000000..cdd6b9eaa --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/buildSearchIndex503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type BuildSearchIndex503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200.ts new file mode 100644 index 000000000..00f27d53f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { CancelDataExecution200Diagnostics } from "./cancelDataExecution200Diagnostics"; +import type { CancelDataExecution200Kind } from "./cancelDataExecution200Kind"; +import type { CancelDataExecution200Publication } from "./cancelDataExecution200Publication"; +import type { CancelDataExecution200Result } from "./cancelDataExecution200Result"; +import type { CancelDataExecution200Status } from "./cancelDataExecution200Status"; + +export type CancelDataExecution200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: CancelDataExecution200Kind; + status: CancelDataExecution200Status; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: CancelDataExecution200Diagnostics; + publication: CancelDataExecution200Publication; + result?: CancelDataExecution200Result; +}; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Diagnostics.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Diagnostics.ts new file mode 100644 index 000000000..dfe7aad2c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Diagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { CancelDataExecution200DiagnosticsTailItem } from "./cancelDataExecution200DiagnosticsTailItem"; + +export type CancelDataExecution200Diagnostics = { + /** @maxItems 256 */ + tail: CancelDataExecution200DiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItem.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItem.ts new file mode 100644 index 000000000..b4d71cbf8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { CancelDataExecution200DiagnosticsTailItemStream } from "./cancelDataExecution200DiagnosticsTailItemStream"; + +export type CancelDataExecution200DiagnosticsTailItem = { + stream: CancelDataExecution200DiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItemStream.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItemStream.ts new file mode 100644 index 000000000..cb9fadf0f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200DiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution200DiagnosticsTailItemStream = + (typeof CancelDataExecution200DiagnosticsTailItemStream)[keyof typeof CancelDataExecution200DiagnosticsTailItemStream]; + +export const CancelDataExecution200DiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts new file mode 100644 index 000000000..f3dd04605 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution200Kind = + (typeof CancelDataExecution200Kind)[keyof typeof CancelDataExecution200Kind]; + +export const CancelDataExecution200Kind = { + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Publication.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Publication.ts new file mode 100644 index 000000000..5b87d9104 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Publication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution200Publication = + (typeof CancelDataExecution200Publication)[keyof typeof CancelDataExecution200Publication]; + +export const CancelDataExecution200Publication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts new file mode 100644 index 000000000..9704165ad --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts @@ -0,0 +1,355 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution200Result = + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Status.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Status.ts new file mode 100644 index 000000000..7269126f0 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Status.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution200Status = + (typeof CancelDataExecution200Status)[keyof typeof CancelDataExecution200Status]; + +export const CancelDataExecution200Status = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution400.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution400.ts new file mode 100644 index 000000000..618dc033c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution401.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution401.ts new file mode 100644 index 000000000..4a65b4445 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution404.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution404.ts new file mode 100644 index 000000000..0d55df7df --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution409.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution409.ts new file mode 100644 index 000000000..d8ff7213b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution413.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution413.ts new file mode 100644 index 000000000..f5d02ba77 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution415.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution415.ts new file mode 100644 index 000000000..f634731d8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution429.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution429.ts new file mode 100644 index 000000000..f7daa8a2e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution500.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution500.ts new file mode 100644 index 000000000..5de11d71b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution503.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution503.ts new file mode 100644 index 000000000..c4b0cf666 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecution503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecutionBody.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecutionBody.ts new file mode 100644 index 000000000..499dd23f9 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecutionBody.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type CancelDataExecutionBody = { + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/conflateOverture409.ts b/packages/core/src/services/data-manager/generated/models/conflateOverture409.ts new file mode 100644 index 000000000..6fb4967d1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/conflateOverture409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ConflateOverture409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/conflateOverture503.ts b/packages/core/src/services/data-manager/generated/models/conflateOverture503.ts new file mode 100644 index 000000000..321f28341 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/conflateOverture503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ConflateOverture503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/convertOverpass409.ts b/packages/core/src/services/data-manager/generated/models/convertOverpass409.ts new file mode 100644 index 000000000..94acfc005 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/convertOverpass409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ConvertOverpass409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/convertOverpass503.ts b/packages/core/src/services/data-manager/generated/models/convertOverpass503.ts new file mode 100644 index 000000000..1f76680e9 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/convertOverpass503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ConvertOverpass503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/discardAmbientPlaces503.ts b/packages/core/src/services/data-manager/generated/models/discardAmbientPlaces503.ts new file mode 100644 index 000000000..160803165 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/discardAmbientPlaces503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DiscardAmbientPlaces503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/downloadFonts409.ts b/packages/core/src/services/data-manager/generated/models/downloadFonts409.ts new file mode 100644 index 000000000..25d519d1c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/downloadFonts409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DownloadFonts409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/downloadFonts503.ts b/packages/core/src/services/data-manager/generated/models/downloadFonts503.ts new file mode 100644 index 000000000..244bf8bd7 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/downloadFonts503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DownloadFonts503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/downloadOsm409.ts b/packages/core/src/services/data-manager/generated/models/downloadOsm409.ts new file mode 100644 index 000000000..79b102e8f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/downloadOsm409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DownloadOsm409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/downloadOsm503.ts b/packages/core/src/services/data-manager/generated/models/downloadOsm503.ts new file mode 100644 index 000000000..728193346 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/downloadOsm503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DownloadOsm503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/enableTransitSource503.ts b/packages/core/src/services/data-manager/generated/models/enableTransitSource503.ts new file mode 100644 index 000000000..999f59d2b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/enableTransitSource503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type EnableTransitSource503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/extractOverture409.ts b/packages/core/src/services/data-manager/generated/models/extractOverture409.ts new file mode 100644 index 000000000..73e214deb --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/extractOverture409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ExtractOverture409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/extractOverture503.ts b/packages/core/src/services/data-manager/generated/models/extractOverture503.ts new file mode 100644 index 000000000..d4158ccb6 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/extractOverture503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ExtractOverture503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200.ts new file mode 100644 index 000000000..777a03f4e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { GetDataExecution200Diagnostics } from "./getDataExecution200Diagnostics"; +import type { GetDataExecution200Kind } from "./getDataExecution200Kind"; +import type { GetDataExecution200Publication } from "./getDataExecution200Publication"; +import type { GetDataExecution200Result } from "./getDataExecution200Result"; +import type { GetDataExecution200Status } from "./getDataExecution200Status"; + +export type GetDataExecution200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: GetDataExecution200Kind; + status: GetDataExecution200Status; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: GetDataExecution200Diagnostics; + publication: GetDataExecution200Publication; + result?: GetDataExecution200Result; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200Diagnostics.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200Diagnostics.ts new file mode 100644 index 000000000..8c44899bc --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200Diagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { GetDataExecution200DiagnosticsTailItem } from "./getDataExecution200DiagnosticsTailItem"; + +export type GetDataExecution200Diagnostics = { + /** @maxItems 256 */ + tail: GetDataExecution200DiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItem.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItem.ts new file mode 100644 index 000000000..2281ee5f5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { GetDataExecution200DiagnosticsTailItemStream } from "./getDataExecution200DiagnosticsTailItemStream"; + +export type GetDataExecution200DiagnosticsTailItem = { + stream: GetDataExecution200DiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItemStream.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItemStream.ts new file mode 100644 index 000000000..015edf25b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200DiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution200DiagnosticsTailItemStream = + (typeof GetDataExecution200DiagnosticsTailItemStream)[keyof typeof GetDataExecution200DiagnosticsTailItemStream]; + +export const GetDataExecution200DiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts new file mode 100644 index 000000000..f432111b3 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution200Kind = + (typeof GetDataExecution200Kind)[keyof typeof GetDataExecution200Kind]; + +export const GetDataExecution200Kind = { + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200Publication.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200Publication.ts new file mode 100644 index 000000000..ebbaa43ee --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200Publication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution200Publication = + (typeof GetDataExecution200Publication)[keyof typeof GetDataExecution200Publication]; + +export const GetDataExecution200Publication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts new file mode 100644 index 000000000..cae13c06e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts @@ -0,0 +1,355 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution200Result = + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200Status.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200Status.ts new file mode 100644 index 000000000..6da8b6f86 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200Status.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution200Status = + (typeof GetDataExecution200Status)[keyof typeof GetDataExecution200Status]; + +export const GetDataExecution200Status = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution400.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution400.ts new file mode 100644 index 000000000..660d51b99 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution401.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution401.ts new file mode 100644 index 000000000..019754864 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution404.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution404.ts new file mode 100644 index 000000000..4b48e608e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution410.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution410.ts new file mode 100644 index 000000000..5a1a7d87f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution413.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution413.ts new file mode 100644 index 000000000..43241cc2f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution415.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution415.ts new file mode 100644 index 000000000..d1ab6afa1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution429.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution429.ts new file mode 100644 index 000000000..9d15246e4 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution500.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution500.ts new file mode 100644 index 000000000..1d19c2d43 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution503.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution503.ts new file mode 100644 index 000000000..69aa929e7 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetDataExecution503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflineGlyph409.ts b/packages/core/src/services/data-manager/generated/models/getOfflineGlyph409.ts new file mode 100644 index 000000000..0bf35ea96 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflineGlyph409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflineGlyph409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflineGlyph503.ts b/packages/core/src/services/data-manager/generated/models/getOfflineGlyph503.ts new file mode 100644 index 000000000..724b6f5e4 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflineGlyph503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflineGlyph503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog409.ts b/packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog409.ts new file mode 100644 index 000000000..9be10885b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflineGlyphCatalog409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog503.ts b/packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog503.ts new file mode 100644 index 000000000..08e4e39f5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflineGlyphCatalog503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflineGlyphCatalog503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive409.ts b/packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive409.ts new file mode 100644 index 000000000..e21475e58 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflinePackageArchive409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive503.ts b/packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive503.ts new file mode 100644 index 000000000..fd83919a2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflinePackageArchive503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflinePackageArchive503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflinePackageJob409.ts b/packages/core/src/services/data-manager/generated/models/getOfflinePackageJob409.ts new file mode 100644 index 000000000..4fb01e009 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflinePackageJob409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflinePackageJob409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflinePackageJob503.ts b/packages/core/src/services/data-manager/generated/models/getOfflinePackageJob503.ts new file mode 100644 index 000000000..466e89f02 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflinePackageJob503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflinePackageJob503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest409.ts b/packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest409.ts new file mode 100644 index 000000000..5a752d4be --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflinePackageManifest409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest503.ts b/packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest503.ts new file mode 100644 index 000000000..258fcfeba --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOfflinePackageManifest503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOfflinePackageManifest503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/index.ts b/packages/core/src/services/data-manager/generated/models/index.ts index 991ff89d9..822780d8e 100644 --- a/packages/core/src/services/data-manager/generated/models/index.ts +++ b/packages/core/src/services/data-manager/generated/models/index.ts @@ -11,18 +11,21 @@ export * from "./addTransitSource413"; export * from "./addTransitSource415"; export * from "./addTransitSource429"; export * from "./addTransitSource500"; +export * from "./addTransitSource503"; export * from "./addTransitSourceBody"; export * from "./addTransitSourceBodyLicense"; export * from "./approveTransitBump200"; export * from "./approveTransitBump400"; export * from "./approveTransitBump401"; export * from "./approveTransitBump404"; +export * from "./approveTransitBump409"; export * from "./approveTransitBump413"; export * from "./approveTransitBump415"; export * from "./approveTransitBump422"; export * from "./approveTransitBump422Error"; export * from "./approveTransitBump429"; export * from "./approveTransitBump500"; +export * from "./approveTransitBump503"; export * from "./approveTransitBumpBody"; export * from "./bakePredictedTraffic202"; export * from "./bakePredictedTraffic400"; @@ -33,6 +36,7 @@ export * from "./bakePredictedTraffic415"; export * from "./bakePredictedTraffic429"; export * from "./bakePredictedTraffic500"; export * from "./bakePredictedTraffic501"; +export * from "./bakePredictedTraffic503"; export * from "./buildAmbientPlaces202"; export * from "./buildAmbientPlaces400"; export * from "./buildAmbientPlaces401"; @@ -41,38 +45,65 @@ export * from "./buildAmbientPlaces413"; export * from "./buildAmbientPlaces415"; export * from "./buildAmbientPlaces429"; export * from "./buildAmbientPlaces500"; +export * from "./buildAmbientPlaces503"; export * from "./buildAmbientPlacesBody"; export * from "./buildAmbientPlacesBodyCoverage"; export * from "./buildNotablePlaces200"; export * from "./buildNotablePlaces400"; export * from "./buildNotablePlaces401"; +export * from "./buildNotablePlaces409"; export * from "./buildNotablePlaces413"; export * from "./buildNotablePlaces415"; export * from "./buildNotablePlaces429"; export * from "./buildNotablePlaces500"; +export * from "./buildNotablePlaces503"; export * from "./buildSearchIndex200"; export * from "./buildSearchIndex400"; export * from "./buildSearchIndex401"; +export * from "./buildSearchIndex409"; export * from "./buildSearchIndex413"; export * from "./buildSearchIndex415"; export * from "./buildSearchIndex429"; export * from "./buildSearchIndex500"; +export * from "./buildSearchIndex503"; export * from "./buildSearchIndexBody"; +export * from "./cancelDataExecution200"; +export * from "./cancelDataExecution200Diagnostics"; +export * from "./cancelDataExecution200DiagnosticsTailItem"; +export * from "./cancelDataExecution200DiagnosticsTailItemStream"; +export * from "./cancelDataExecution200Kind"; +export * from "./cancelDataExecution200Publication"; +export * from "./cancelDataExecution200Result"; +export * from "./cancelDataExecution200Status"; +export * from "./cancelDataExecution400"; +export * from "./cancelDataExecution401"; +export * from "./cancelDataExecution404"; +export * from "./cancelDataExecution409"; +export * from "./cancelDataExecution413"; +export * from "./cancelDataExecution415"; +export * from "./cancelDataExecution429"; +export * from "./cancelDataExecution500"; +export * from "./cancelDataExecution503"; +export * from "./cancelDataExecutionBody"; export * from "./conflateOverture200"; export * from "./conflateOverture400"; export * from "./conflateOverture401"; +export * from "./conflateOverture409"; export * from "./conflateOverture413"; export * from "./conflateOverture415"; export * from "./conflateOverture429"; export * from "./conflateOverture500"; +export * from "./conflateOverture503"; export * from "./conflateOvertureBody"; export * from "./convertOverpass200"; export * from "./convertOverpass400"; export * from "./convertOverpass401"; +export * from "./convertOverpass409"; export * from "./convertOverpass413"; export * from "./convertOverpass415"; export * from "./convertOverpass429"; export * from "./convertOverpass500"; +export * from "./convertOverpass503"; export * from "./convertOverpassBody"; export * from "./dataManagerGetTransitCatalogResponse200Schema0"; export * from "./dataManagerGetTransitSourcesResponse200Schema0"; @@ -84,21 +115,26 @@ export * from "./discardAmbientPlaces413"; export * from "./discardAmbientPlaces415"; export * from "./discardAmbientPlaces429"; export * from "./discardAmbientPlaces500"; +export * from "./discardAmbientPlaces503"; export * from "./discardAmbientPlacesBody"; export * from "./downloadFonts200"; export * from "./downloadFonts400"; export * from "./downloadFonts401"; +export * from "./downloadFonts409"; export * from "./downloadFonts413"; export * from "./downloadFonts415"; export * from "./downloadFonts429"; export * from "./downloadFonts500"; +export * from "./downloadFonts503"; export * from "./downloadOsm200"; export * from "./downloadOsm400"; export * from "./downloadOsm401"; +export * from "./downloadOsm409"; export * from "./downloadOsm413"; export * from "./downloadOsm415"; export * from "./downloadOsm429"; export * from "./downloadOsm500"; +export * from "./downloadOsm503"; export * from "./downloadOsmBody"; export * from "./enableTransitSource202"; export * from "./enableTransitSource202Status"; @@ -110,14 +146,17 @@ export * from "./enableTransitSource413"; export * from "./enableTransitSource415"; export * from "./enableTransitSource429"; export * from "./enableTransitSource500"; +export * from "./enableTransitSource503"; export * from "./enableTransitSourceBody"; export * from "./extractOverture200"; export * from "./extractOverture400"; export * from "./extractOverture401"; +export * from "./extractOverture409"; export * from "./extractOverture413"; export * from "./extractOverture415"; export * from "./extractOverture429"; export * from "./extractOverture500"; +export * from "./extractOverture503"; export * from "./extractOvertureBody"; export * from "./getAmbientPlacesStatus200"; export * from "./getAmbientPlacesStatus200Active"; @@ -219,6 +258,23 @@ export * from "./getCoverageEvidence429"; export * from "./getCoverageEvidence500"; export * from "./getCoverageEvidence503"; export * from "./getCoverageEvidenceParams"; +export * from "./getDataExecution200"; +export * from "./getDataExecution200Diagnostics"; +export * from "./getDataExecution200DiagnosticsTailItem"; +export * from "./getDataExecution200DiagnosticsTailItemStream"; +export * from "./getDataExecution200Kind"; +export * from "./getDataExecution200Publication"; +export * from "./getDataExecution200Result"; +export * from "./getDataExecution200Status"; +export * from "./getDataExecution400"; +export * from "./getDataExecution401"; +export * from "./getDataExecution404"; +export * from "./getDataExecution410"; +export * from "./getDataExecution413"; +export * from "./getDataExecution415"; +export * from "./getDataExecution429"; +export * from "./getDataExecution500"; +export * from "./getDataExecution503"; export * from "./getDatasets200"; export * from "./getDatasets200DatasetsItem"; export * from "./getDatasets200DatasetsItemType"; @@ -243,25 +299,31 @@ export * from "./getNotablePlacesStatus500"; export * from "./getOfflineGlyph400"; export * from "./getOfflineGlyph401"; export * from "./getOfflineGlyph404"; +export * from "./getOfflineGlyph409"; export * from "./getOfflineGlyph413"; export * from "./getOfflineGlyph415"; export * from "./getOfflineGlyph429"; export * from "./getOfflineGlyph500"; +export * from "./getOfflineGlyph503"; export * from "./getOfflineGlyphCatalog200"; export * from "./getOfflineGlyphCatalog400"; export * from "./getOfflineGlyphCatalog401"; export * from "./getOfflineGlyphCatalog404"; +export * from "./getOfflineGlyphCatalog409"; export * from "./getOfflineGlyphCatalog413"; export * from "./getOfflineGlyphCatalog415"; export * from "./getOfflineGlyphCatalog429"; export * from "./getOfflineGlyphCatalog500"; +export * from "./getOfflineGlyphCatalog503"; export * from "./getOfflinePackageArchive400"; export * from "./getOfflinePackageArchive401"; export * from "./getOfflinePackageArchive404"; +export * from "./getOfflinePackageArchive409"; export * from "./getOfflinePackageArchive413"; export * from "./getOfflinePackageArchive415"; export * from "./getOfflinePackageArchive429"; export * from "./getOfflinePackageArchive500"; +export * from "./getOfflinePackageArchive503"; export * from "./getOfflinePackageArchiveHeaders"; export * from "./getOfflinePackageCapability200"; export * from "./getOfflinePackageCapability400"; @@ -274,10 +336,12 @@ export * from "./getOfflinePackageJob200"; export * from "./getOfflinePackageJob400"; export * from "./getOfflinePackageJob401"; export * from "./getOfflinePackageJob404"; +export * from "./getOfflinePackageJob409"; export * from "./getOfflinePackageJob413"; export * from "./getOfflinePackageJob415"; export * from "./getOfflinePackageJob429"; export * from "./getOfflinePackageJob500"; +export * from "./getOfflinePackageJob503"; export * from "./getOfflinePackageJobHeaders"; export * from "./getOfflinePackageManifest200"; export * from "./getOfflinePackageManifest200Archive"; @@ -292,10 +356,12 @@ export * from "./getOfflinePackageManifest200SchemaVersion"; export * from "./getOfflinePackageManifest400"; export * from "./getOfflinePackageManifest401"; export * from "./getOfflinePackageManifest404"; +export * from "./getOfflinePackageManifest409"; export * from "./getOfflinePackageManifest413"; export * from "./getOfflinePackageManifest415"; export * from "./getOfflinePackageManifest429"; export * from "./getOfflinePackageManifest500"; +export * from "./getOfflinePackageManifest503"; export * from "./getOvertureStatus200"; export * from "./getOvertureStatus200Phase"; export * from "./getOvertureStatus200Status"; @@ -377,20 +443,55 @@ export * from "./headOfflinePackageArchiveHeaders"; export * from "./ingestOverture200"; export * from "./ingestOverture400"; export * from "./ingestOverture401"; +export * from "./ingestOverture409"; export * from "./ingestOverture413"; export * from "./ingestOverture415"; export * from "./ingestOverture429"; export * from "./ingestOverture500"; +export * from "./ingestOverture503"; export * from "./ingestOvertureBody"; +export * from "./inspectDataConfigurationFence200"; +export * from "./inspectDataConfigurationFence200OwnerPrepare"; +export * from "./inspectDataConfigurationFence200OwnerPrepareExecutor"; +export * from "./inspectDataConfigurationFence200OwnerPreparePrincipal"; +export * from "./inspectDataConfigurationFence200State"; +export * from "./inspectDataConfigurationFence400"; +export * from "./inspectDataConfigurationFence401"; +export * from "./inspectDataConfigurationFence403"; +export * from "./inspectDataConfigurationFence404"; +export * from "./inspectDataConfigurationFence413"; +export * from "./inspectDataConfigurationFence415"; +export * from "./inspectDataConfigurationFence429"; +export * from "./inspectDataConfigurationFence500"; +export * from "./inspectDataConfigurationFence503"; export * from "./link200"; export * from "./link400"; export * from "./link401"; +export * from "./link409"; export * from "./link413"; export * from "./link415"; export * from "./link429"; export * from "./link500"; +export * from "./link503"; export * from "./linkBody"; export * from "./linkBodyPlanItem"; +export * from "./lookupDataExecution200"; +export * from "./lookupDataExecution200Diagnostics"; +export * from "./lookupDataExecution200DiagnosticsTailItem"; +export * from "./lookupDataExecution200DiagnosticsTailItemStream"; +export * from "./lookupDataExecution200Kind"; +export * from "./lookupDataExecution200Publication"; +export * from "./lookupDataExecution200Result"; +export * from "./lookupDataExecution200Status"; +export * from "./lookupDataExecution400"; +export * from "./lookupDataExecution401"; +export * from "./lookupDataExecution404"; +export * from "./lookupDataExecution410"; +export * from "./lookupDataExecution413"; +export * from "./lookupDataExecution415"; +export * from "./lookupDataExecution429"; +export * from "./lookupDataExecution500"; +export * from "./lookupDataExecution503"; export * from "./preflightTransit200"; export * from "./preflightTransit200Capacity"; export * from "./preflightTransit200Estimate"; @@ -402,6 +503,7 @@ export * from "./preflightTransit200PolicyProfile"; export * from "./preflightTransit200PolicyResourceEnvelope"; export * from "./preflightTransit400"; export * from "./preflightTransit401"; +export * from "./preflightTransit409"; export * from "./preflightTransit413"; export * from "./preflightTransit415"; export * from "./preflightTransit422"; @@ -415,7 +517,27 @@ export * from "./preflightTransit422PolicyProfile"; export * from "./preflightTransit422PolicyResourceEnvelope"; export * from "./preflightTransit429"; export * from "./preflightTransit500"; +export * from "./preflightTransit503"; export * from "./preflightTransitBody"; +export * from "./prepareDataConfigurationFence200"; +export * from "./prepareDataConfigurationFence200OwnerPrepare"; +export * from "./prepareDataConfigurationFence200OwnerPrepareExecutor"; +export * from "./prepareDataConfigurationFence200OwnerPreparePrincipal"; +export * from "./prepareDataConfigurationFence200State"; +export * from "./prepareDataConfigurationFence400"; +export * from "./prepareDataConfigurationFence401"; +export * from "./prepareDataConfigurationFence403"; +export * from "./prepareDataConfigurationFence409"; +export * from "./prepareDataConfigurationFence413"; +export * from "./prepareDataConfigurationFence415"; +export * from "./prepareDataConfigurationFence429"; +export * from "./prepareDataConfigurationFence500"; +export * from "./prepareDataConfigurationFence503"; +export * from "./prepareDataConfigurationFenceBody"; +export * from "./prepareDataConfigurationFenceBodyCoordinatorMode"; +export * from "./prepareDataConfigurationFenceBodyOwnerPrepare"; +export * from "./prepareDataConfigurationFenceBodyOwnerPrepareExecutor"; +export * from "./prepareDataConfigurationFenceBodyOwnerPreparePrincipal"; export * from "./prepareOfflinePackage200"; export * from "./prepareOfflinePackage200ErrorCode"; export * from "./prepareOfflinePackage200Manifest"; @@ -485,22 +607,27 @@ export * from "./proposeTransitBump415"; export * from "./proposeTransitBump429"; export * from "./proposeTransitBump500"; export * from "./proposeTransitBump502"; +export * from "./proposeTransitBump503"; export * from "./proposeTransitBumpBody"; export * from "./pullOverture200"; export * from "./pullOverture400"; export * from "./pullOverture401"; +export * from "./pullOverture409"; export * from "./pullOverture413"; export * from "./pullOverture415"; export * from "./pullOverture429"; export * from "./pullOverture500"; +export * from "./pullOverture503"; export * from "./pullOvertureBody"; export * from "./reloadDatasets200"; export * from "./reloadDatasets400"; export * from "./reloadDatasets401"; +export * from "./reloadDatasets409"; export * from "./reloadDatasets413"; export * from "./reloadDatasets415"; export * from "./reloadDatasets429"; export * from "./reloadDatasets500"; +export * from "./reloadDatasets503"; export * from "./removeTransitSource202"; export * from "./removeTransitSource202Status"; export * from "./removeTransitSource400"; @@ -511,11 +638,13 @@ export * from "./removeTransitSource413"; export * from "./removeTransitSource415"; export * from "./removeTransitSource429"; export * from "./removeTransitSource500"; +export * from "./removeTransitSource503"; export * from "./removeTransitSourceBody"; export * from "./restartMotis200"; export * from "./restartMotis200Status"; export * from "./restartMotis400"; export * from "./restartMotis401"; +export * from "./restartMotis409"; export * from "./restartMotis413"; export * from "./restartMotis415"; export * from "./restartMotis429"; @@ -529,6 +658,7 @@ export * from "./resumeAmbientPlaces413"; export * from "./resumeAmbientPlaces415"; export * from "./resumeAmbientPlaces429"; export * from "./resumeAmbientPlaces500"; +export * from "./resumeAmbientPlaces503"; export * from "./resumeAmbientPlacesBody"; export * from "./rollbackAmbientPlaces200"; export * from "./rollbackAmbientPlaces400"; @@ -538,6 +668,7 @@ export * from "./rollbackAmbientPlaces413"; export * from "./rollbackAmbientPlaces415"; export * from "./rollbackAmbientPlaces429"; export * from "./rollbackAmbientPlaces500"; +export * from "./rollbackAmbientPlaces503"; export * from "./setAmbientPlacesEnabled200"; export * from "./setAmbientPlacesEnabled400"; export * from "./setAmbientPlacesEnabled401"; @@ -546,24 +677,86 @@ export * from "./setAmbientPlacesEnabled413"; export * from "./setAmbientPlacesEnabled415"; export * from "./setAmbientPlacesEnabled429"; export * from "./setAmbientPlacesEnabled500"; +export * from "./setAmbientPlacesEnabled503"; export * from "./setAmbientPlacesEnabledBody"; +export * from "./settleDataConfigurationFence200"; +export * from "./settleDataConfigurationFence200OwnerPrepare"; +export * from "./settleDataConfigurationFence200OwnerPrepareExecutor"; +export * from "./settleDataConfigurationFence200OwnerPreparePrincipal"; +export * from "./settleDataConfigurationFence200State"; +export * from "./settleDataConfigurationFence400"; +export * from "./settleDataConfigurationFence401"; +export * from "./settleDataConfigurationFence403"; +export * from "./settleDataConfigurationFence404"; +export * from "./settleDataConfigurationFence409"; +export * from "./settleDataConfigurationFence413"; +export * from "./settleDataConfigurationFence415"; +export * from "./settleDataConfigurationFence429"; +export * from "./settleDataConfigurationFence500"; +export * from "./settleDataConfigurationFence503"; +export * from "./settleDataConfigurationFenceBody"; +export * from "./settleDataConfigurationFenceBodyOutcome"; +export * from "./startDataExecution202"; +export * from "./startDataExecution202Diagnostics"; +export * from "./startDataExecution202DiagnosticsTailItem"; +export * from "./startDataExecution202DiagnosticsTailItemStream"; +export * from "./startDataExecution202Kind"; +export * from "./startDataExecution202Publication"; +export * from "./startDataExecution202Result"; +export * from "./startDataExecution202Status"; +export * from "./startDataExecution400"; +export * from "./startDataExecution401"; +export * from "./startDataExecution403"; +export * from "./startDataExecution409"; +export * from "./startDataExecution413"; +export * from "./startDataExecution415"; +export * from "./startDataExecution429"; +export * from "./startDataExecution500"; +export * from "./startDataExecution503"; +export * from "./startDataExecutionBody"; +export * from "./startDataExecutionBodyInput"; +export * from "./startGenerationHardlinksExecution202"; +export * from "./startGenerationHardlinksExecution202Diagnostics"; +export * from "./startGenerationHardlinksExecution202DiagnosticsTailItem"; +export * from "./startGenerationHardlinksExecution202DiagnosticsTailItemStream"; +export * from "./startGenerationHardlinksExecution202Kind"; +export * from "./startGenerationHardlinksExecution202Publication"; +export * from "./startGenerationHardlinksExecution202Result"; +export * from "./startGenerationHardlinksExecution202Status"; +export * from "./startGenerationHardlinksExecution400"; +export * from "./startGenerationHardlinksExecution401"; +export * from "./startGenerationHardlinksExecution403"; +export * from "./startGenerationHardlinksExecution409"; +export * from "./startGenerationHardlinksExecution413"; +export * from "./startGenerationHardlinksExecution415"; +export * from "./startGenerationHardlinksExecution429"; +export * from "./startGenerationHardlinksExecution500"; +export * from "./startGenerationHardlinksExecution503"; +export * from "./startGenerationHardlinksExecutionBody"; +export * from "./startGenerationHardlinksExecutionBodyInput"; +export * from "./startGenerationHardlinksExecutionBodyInputCoordinatorMode"; +export * from "./startGenerationHardlinksExecutionBodyInputKind"; +export * from "./startGenerationHardlinksExecutionBodyPlan"; +export * from "./startGenerationHardlinksExecutionBodyPlanPlanItem"; export * from "./syncOverture200"; export * from "./syncOverture400"; export * from "./syncOverture401"; +export * from "./syncOverture409"; export * from "./syncOverture413"; export * from "./syncOverture415"; export * from "./syncOverture429"; export * from "./syncOverture500"; +export * from "./syncOverture503"; export * from "./syncOvertureBody"; export * from "./syncTransit202"; export * from "./syncTransit202Status"; export * from "./syncTransit400"; export * from "./syncTransit401"; export * from "./syncTransit409"; -export * from "./syncTransit409Reason"; export * from "./syncTransit413"; export * from "./syncTransit415"; export * from "./syncTransit422"; export * from "./syncTransit429"; export * from "./syncTransit500"; +export * from "./syncTransit503"; export * from "./syncTransitBody"; diff --git a/packages/core/src/services/data-manager/generated/models/ingestOverture409.ts b/packages/core/src/services/data-manager/generated/models/ingestOverture409.ts new file mode 100644 index 000000000..5b4c3b71d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/ingestOverture409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type IngestOverture409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/ingestOverture503.ts b/packages/core/src/services/data-manager/generated/models/ingestOverture503.ts new file mode 100644 index 000000000..87dc98c27 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/ingestOverture503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type IngestOverture503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200.ts new file mode 100644 index 000000000..8b3c24f83 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200.ts @@ -0,0 +1,25 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectDataConfigurationFence200OwnerPrepare } from "./inspectDataConfigurationFence200OwnerPrepare"; +import type { InspectDataConfigurationFence200State } from "./inspectDataConfigurationFence200State"; + +export type InspectDataConfigurationFence200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + maintenanceId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + candidateId: string; + ownerPrepare: InspectDataConfigurationFence200OwnerPrepare; + /** @pattern ^[a-f0-9]{64}$ */ + planDigest: string; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + hardlinkPlanDigest: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + fenceId: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sequence: number; + state: InspectDataConfigurationFence200State; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepare.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepare.ts new file mode 100644 index 000000000..096c68c28 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepare.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectDataConfigurationFence200OwnerPrepareExecutor } from "./inspectDataConfigurationFence200OwnerPrepareExecutor"; +import type { InspectDataConfigurationFence200OwnerPreparePrincipal } from "./inspectDataConfigurationFence200OwnerPreparePrincipal"; + +export type InspectDataConfigurationFence200OwnerPrepare = { + executor: InspectDataConfigurationFence200OwnerPrepareExecutor; + principal: InspectDataConfigurationFence200OwnerPreparePrincipal; + executionId: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + operationFingerprint: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepareExecutor.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepareExecutor.ts new file mode 100644 index 000000000..a59472b60 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPrepareExecutor.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence200OwnerPrepareExecutor = + (typeof InspectDataConfigurationFence200OwnerPrepareExecutor)[keyof typeof InspectDataConfigurationFence200OwnerPrepareExecutor]; + +export const InspectDataConfigurationFence200OwnerPrepareExecutor = { + "ops-agent": "ops-agent", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPreparePrincipal.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPreparePrincipal.ts new file mode 100644 index 000000000..fdbb22927 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200OwnerPreparePrincipal.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence200OwnerPreparePrincipal = + (typeof InspectDataConfigurationFence200OwnerPreparePrincipal)[keyof typeof InspectDataConfigurationFence200OwnerPreparePrincipal]; + +export const InspectDataConfigurationFence200OwnerPreparePrincipal = { + api: "api", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200State.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200State.ts new file mode 100644 index 000000000..b0331b6a7 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence200State.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence200State = + (typeof InspectDataConfigurationFence200State)[keyof typeof InspectDataConfigurationFence200State]; + +export const InspectDataConfigurationFence200State = { + prepared: "prepared", + settled: "settled", + recovery_required: "recovery_required", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence400.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence400.ts new file mode 100644 index 000000000..ad0fccd84 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence401.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence401.ts new file mode 100644 index 000000000..f2eee2ed9 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence403.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence403.ts new file mode 100644 index 000000000..8f2a1173c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence404.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence404.ts new file mode 100644 index 000000000..b1fe88fc7 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence413.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence413.ts new file mode 100644 index 000000000..780b26f1f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence415.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence415.ts new file mode 100644 index 000000000..3f69001a4 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence429.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence429.ts new file mode 100644 index 000000000..b2adad52d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence500.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence500.ts new file mode 100644 index 000000000..975feba96 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence503.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence503.ts new file mode 100644 index 000000000..cd89216c1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFence503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFence503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/link409.ts b/packages/core/src/services/data-manager/generated/models/link409.ts new file mode 100644 index 000000000..a44ee6a98 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/link409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type Link409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/link503.ts b/packages/core/src/services/data-manager/generated/models/link503.ts new file mode 100644 index 000000000..35c06223e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/link503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type Link503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200.ts new file mode 100644 index 000000000..0921b0b72 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { LookupDataExecution200Diagnostics } from "./lookupDataExecution200Diagnostics"; +import type { LookupDataExecution200Kind } from "./lookupDataExecution200Kind"; +import type { LookupDataExecution200Publication } from "./lookupDataExecution200Publication"; +import type { LookupDataExecution200Result } from "./lookupDataExecution200Result"; +import type { LookupDataExecution200Status } from "./lookupDataExecution200Status"; + +export type LookupDataExecution200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: LookupDataExecution200Kind; + status: LookupDataExecution200Status; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: LookupDataExecution200Diagnostics; + publication: LookupDataExecution200Publication; + result?: LookupDataExecution200Result; +}; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Diagnostics.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Diagnostics.ts new file mode 100644 index 000000000..013d3a1a2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Diagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { LookupDataExecution200DiagnosticsTailItem } from "./lookupDataExecution200DiagnosticsTailItem"; + +export type LookupDataExecution200Diagnostics = { + /** @maxItems 256 */ + tail: LookupDataExecution200DiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItem.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItem.ts new file mode 100644 index 000000000..8584c20df --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { LookupDataExecution200DiagnosticsTailItemStream } from "./lookupDataExecution200DiagnosticsTailItemStream"; + +export type LookupDataExecution200DiagnosticsTailItem = { + stream: LookupDataExecution200DiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItemStream.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItemStream.ts new file mode 100644 index 000000000..b53075b00 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200DiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution200DiagnosticsTailItemStream = + (typeof LookupDataExecution200DiagnosticsTailItemStream)[keyof typeof LookupDataExecution200DiagnosticsTailItemStream]; + +export const LookupDataExecution200DiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts new file mode 100644 index 000000000..df4b40d12 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution200Kind = + (typeof LookupDataExecution200Kind)[keyof typeof LookupDataExecution200Kind]; + +export const LookupDataExecution200Kind = { + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Publication.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Publication.ts new file mode 100644 index 000000000..47b622656 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Publication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution200Publication = + (typeof LookupDataExecution200Publication)[keyof typeof LookupDataExecution200Publication]; + +export const LookupDataExecution200Publication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts new file mode 100644 index 000000000..84fdcb84d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts @@ -0,0 +1,355 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution200Result = + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Status.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Status.ts new file mode 100644 index 000000000..1d6189cdf --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Status.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution200Status = + (typeof LookupDataExecution200Status)[keyof typeof LookupDataExecution200Status]; + +export const LookupDataExecution200Status = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution400.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution400.ts new file mode 100644 index 000000000..f5f7b475d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution401.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution401.ts new file mode 100644 index 000000000..ba359b9f3 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution404.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution404.ts new file mode 100644 index 000000000..ceb3ccd8f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution410.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution410.ts new file mode 100644 index 000000000..7a315f4b2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution413.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution413.ts new file mode 100644 index 000000000..cf4c79ce8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution415.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution415.ts new file mode 100644 index 000000000..d5b7e09ad --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution429.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution429.ts new file mode 100644 index 000000000..4d3e2a2ae --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution500.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution500.ts new file mode 100644 index 000000000..c8004ea09 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution503.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution503.ts new file mode 100644 index 000000000..cdaf3bbbd --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type LookupDataExecution503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/preflightTransit409.ts b/packages/core/src/services/data-manager/generated/models/preflightTransit409.ts new file mode 100644 index 000000000..582e191f2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/preflightTransit409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PreflightTransit409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/preflightTransit503.ts b/packages/core/src/services/data-manager/generated/models/preflightTransit503.ts new file mode 100644 index 000000000..5335bfbe1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/preflightTransit503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PreflightTransit503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200.ts new file mode 100644 index 000000000..c948e3154 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200.ts @@ -0,0 +1,25 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { PrepareDataConfigurationFence200OwnerPrepare } from "./prepareDataConfigurationFence200OwnerPrepare"; +import type { PrepareDataConfigurationFence200State } from "./prepareDataConfigurationFence200State"; + +export type PrepareDataConfigurationFence200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + maintenanceId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + candidateId: string; + ownerPrepare: PrepareDataConfigurationFence200OwnerPrepare; + /** @pattern ^[a-f0-9]{64}$ */ + planDigest: string; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + hardlinkPlanDigest: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + fenceId: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sequence: number; + state: PrepareDataConfigurationFence200State; +}; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepare.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepare.ts new file mode 100644 index 000000000..074e81406 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepare.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { PrepareDataConfigurationFence200OwnerPrepareExecutor } from "./prepareDataConfigurationFence200OwnerPrepareExecutor"; +import type { PrepareDataConfigurationFence200OwnerPreparePrincipal } from "./prepareDataConfigurationFence200OwnerPreparePrincipal"; + +export type PrepareDataConfigurationFence200OwnerPrepare = { + executor: PrepareDataConfigurationFence200OwnerPrepareExecutor; + principal: PrepareDataConfigurationFence200OwnerPreparePrincipal; + executionId: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + operationFingerprint: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepareExecutor.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepareExecutor.ts new file mode 100644 index 000000000..b36d6155a --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPrepareExecutor.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence200OwnerPrepareExecutor = + (typeof PrepareDataConfigurationFence200OwnerPrepareExecutor)[keyof typeof PrepareDataConfigurationFence200OwnerPrepareExecutor]; + +export const PrepareDataConfigurationFence200OwnerPrepareExecutor = { + "ops-agent": "ops-agent", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPreparePrincipal.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPreparePrincipal.ts new file mode 100644 index 000000000..520ca0f50 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200OwnerPreparePrincipal.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence200OwnerPreparePrincipal = + (typeof PrepareDataConfigurationFence200OwnerPreparePrincipal)[keyof typeof PrepareDataConfigurationFence200OwnerPreparePrincipal]; + +export const PrepareDataConfigurationFence200OwnerPreparePrincipal = { + api: "api", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200State.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200State.ts new file mode 100644 index 000000000..83cdce215 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence200State.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence200State = + (typeof PrepareDataConfigurationFence200State)[keyof typeof PrepareDataConfigurationFence200State]; + +export const PrepareDataConfigurationFence200State = { + prepared: "prepared", + settled: "settled", + recovery_required: "recovery_required", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence400.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence400.ts new file mode 100644 index 000000000..8e9a7c9f5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence401.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence401.ts new file mode 100644 index 000000000..098e696b0 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence403.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence403.ts new file mode 100644 index 000000000..34020a34c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence409.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence409.ts new file mode 100644 index 000000000..d31e17899 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence413.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence413.ts new file mode 100644 index 000000000..70ed94469 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence415.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence415.ts new file mode 100644 index 000000000..357a1d51e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence429.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence429.ts new file mode 100644 index 000000000..45ffadc8b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence500.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence500.ts new file mode 100644 index 000000000..9838cea5f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence503.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence503.ts new file mode 100644 index 000000000..3ec706f4e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFence503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFence503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBody.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBody.ts new file mode 100644 index 000000000..04e32238b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBody.ts @@ -0,0 +1,24 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { PrepareDataConfigurationFenceBodyCoordinatorMode } from "./prepareDataConfigurationFenceBodyCoordinatorMode"; +import type { PrepareDataConfigurationFenceBodyOwnerPrepare } from "./prepareDataConfigurationFenceBodyOwnerPrepare"; + +export type PrepareDataConfigurationFenceBody = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + maintenanceId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + candidateId: string; + ownerPrepare: PrepareDataConfigurationFenceBodyOwnerPrepare; + /** @pattern ^[a-f0-9]{64}$ */ + planDigest: string; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + hardlinkPlanDigest: string | null; + /** + * @minItems 1 + * @maxItems 256 + * @items.pattern ^[a-z0-9][a-z0-9-]{0,63}$ + */ + serviceIds: string[]; + coordinatorMode: PrepareDataConfigurationFenceBodyCoordinatorMode; +}; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyCoordinatorMode.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyCoordinatorMode.ts new file mode 100644 index 000000000..d0e571320 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyCoordinatorMode.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFenceBodyCoordinatorMode = + (typeof PrepareDataConfigurationFenceBodyCoordinatorMode)[keyof typeof PrepareDataConfigurationFenceBodyCoordinatorMode]; + +export const PrepareDataConfigurationFenceBodyCoordinatorMode = { + "ensure-running": "ensure-running", + "require-running": "require-running", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepare.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepare.ts new file mode 100644 index 000000000..b94071a87 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepare.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { PrepareDataConfigurationFenceBodyOwnerPrepareExecutor } from "./prepareDataConfigurationFenceBodyOwnerPrepareExecutor"; +import type { PrepareDataConfigurationFenceBodyOwnerPreparePrincipal } from "./prepareDataConfigurationFenceBodyOwnerPreparePrincipal"; + +export type PrepareDataConfigurationFenceBodyOwnerPrepare = { + executor: PrepareDataConfigurationFenceBodyOwnerPrepareExecutor; + principal: PrepareDataConfigurationFenceBodyOwnerPreparePrincipal; + executionId: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + operationFingerprint: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepareExecutor.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepareExecutor.ts new file mode 100644 index 000000000..e30e48b1e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPrepareExecutor.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFenceBodyOwnerPrepareExecutor = + (typeof PrepareDataConfigurationFenceBodyOwnerPrepareExecutor)[keyof typeof PrepareDataConfigurationFenceBodyOwnerPrepareExecutor]; + +export const PrepareDataConfigurationFenceBodyOwnerPrepareExecutor = { + "ops-agent": "ops-agent", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPreparePrincipal.ts b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPreparePrincipal.ts new file mode 100644 index 000000000..db085cad6 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/prepareDataConfigurationFenceBodyOwnerPreparePrincipal.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PrepareDataConfigurationFenceBodyOwnerPreparePrincipal = + (typeof PrepareDataConfigurationFenceBodyOwnerPreparePrincipal)[keyof typeof PrepareDataConfigurationFenceBodyOwnerPreparePrincipal]; + +export const PrepareDataConfigurationFenceBodyOwnerPreparePrincipal = { + api: "api", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/proposeTransitBump503.ts b/packages/core/src/services/data-manager/generated/models/proposeTransitBump503.ts new file mode 100644 index 000000000..3b3fc4d23 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/proposeTransitBump503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ProposeTransitBump503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/pullOverture409.ts b/packages/core/src/services/data-manager/generated/models/pullOverture409.ts new file mode 100644 index 000000000..a218fcda2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/pullOverture409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PullOverture409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/pullOverture503.ts b/packages/core/src/services/data-manager/generated/models/pullOverture503.ts new file mode 100644 index 000000000..f44096237 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/pullOverture503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type PullOverture503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/reloadDatasets409.ts b/packages/core/src/services/data-manager/generated/models/reloadDatasets409.ts new file mode 100644 index 000000000..6da91ee57 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/reloadDatasets409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ReloadDatasets409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/reloadDatasets503.ts b/packages/core/src/services/data-manager/generated/models/reloadDatasets503.ts new file mode 100644 index 000000000..dee7d0392 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/reloadDatasets503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ReloadDatasets503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/removeTransitSource503.ts b/packages/core/src/services/data-manager/generated/models/removeTransitSource503.ts new file mode 100644 index 000000000..d93de1f4e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/removeTransitSource503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RemoveTransitSource503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/restartMotis409.ts b/packages/core/src/services/data-manager/generated/models/restartMotis409.ts new file mode 100644 index 000000000..c55da5e31 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/restartMotis409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RestartMotis409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/resumeAmbientPlaces503.ts b/packages/core/src/services/data-manager/generated/models/resumeAmbientPlaces503.ts new file mode 100644 index 000000000..f70e311ab --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/resumeAmbientPlaces503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type ResumeAmbientPlaces503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/rollbackAmbientPlaces503.ts b/packages/core/src/services/data-manager/generated/models/rollbackAmbientPlaces503.ts new file mode 100644 index 000000000..95c8a5171 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/rollbackAmbientPlaces503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RollbackAmbientPlaces503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/setAmbientPlacesEnabled503.ts b/packages/core/src/services/data-manager/generated/models/setAmbientPlacesEnabled503.ts new file mode 100644 index 000000000..652690c42 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/setAmbientPlacesEnabled503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SetAmbientPlacesEnabled503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200.ts new file mode 100644 index 000000000..0e6918f22 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200.ts @@ -0,0 +1,25 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { SettleDataConfigurationFence200OwnerPrepare } from "./settleDataConfigurationFence200OwnerPrepare"; +import type { SettleDataConfigurationFence200State } from "./settleDataConfigurationFence200State"; + +export type SettleDataConfigurationFence200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + maintenanceId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + candidateId: string; + ownerPrepare: SettleDataConfigurationFence200OwnerPrepare; + /** @pattern ^[a-f0-9]{64}$ */ + planDigest: string; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + hardlinkPlanDigest: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + fenceId: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sequence: number; + state: SettleDataConfigurationFence200State; +}; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepare.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepare.ts new file mode 100644 index 000000000..9b2030801 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepare.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { SettleDataConfigurationFence200OwnerPrepareExecutor } from "./settleDataConfigurationFence200OwnerPrepareExecutor"; +import type { SettleDataConfigurationFence200OwnerPreparePrincipal } from "./settleDataConfigurationFence200OwnerPreparePrincipal"; + +export type SettleDataConfigurationFence200OwnerPrepare = { + executor: SettleDataConfigurationFence200OwnerPrepareExecutor; + principal: SettleDataConfigurationFence200OwnerPreparePrincipal; + executionId: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + operationFingerprint: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepareExecutor.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepareExecutor.ts new file mode 100644 index 000000000..a0b081569 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPrepareExecutor.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence200OwnerPrepareExecutor = + (typeof SettleDataConfigurationFence200OwnerPrepareExecutor)[keyof typeof SettleDataConfigurationFence200OwnerPrepareExecutor]; + +export const SettleDataConfigurationFence200OwnerPrepareExecutor = { + "ops-agent": "ops-agent", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPreparePrincipal.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPreparePrincipal.ts new file mode 100644 index 000000000..af6350a58 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200OwnerPreparePrincipal.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence200OwnerPreparePrincipal = + (typeof SettleDataConfigurationFence200OwnerPreparePrincipal)[keyof typeof SettleDataConfigurationFence200OwnerPreparePrincipal]; + +export const SettleDataConfigurationFence200OwnerPreparePrincipal = { + api: "api", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200State.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200State.ts new file mode 100644 index 000000000..6c4346cca --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence200State.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence200State = + (typeof SettleDataConfigurationFence200State)[keyof typeof SettleDataConfigurationFence200State]; + +export const SettleDataConfigurationFence200State = { + prepared: "prepared", + settled: "settled", + recovery_required: "recovery_required", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence400.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence400.ts new file mode 100644 index 000000000..b9fd9ffd5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence401.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence401.ts new file mode 100644 index 000000000..ebfdfd464 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence403.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence403.ts new file mode 100644 index 000000000..88c2df306 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence404.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence404.ts new file mode 100644 index 000000000..e7325691a --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence409.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence409.ts new file mode 100644 index 000000000..164cc4ee5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence413.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence413.ts new file mode 100644 index 000000000..a62b7a4f2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence415.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence415.ts new file mode 100644 index 000000000..3357cfe9c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence429.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence429.ts new file mode 100644 index 000000000..bcf601b10 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence500.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence500.ts new file mode 100644 index 000000000..05132a28f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence503.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence503.ts new file mode 100644 index 000000000..05176d0d2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFence503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFence503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts new file mode 100644 index 000000000..77b947693 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { SettleDataConfigurationFenceBodyOutcome } from "./settleDataConfigurationFenceBodyOutcome"; + +export type SettleDataConfigurationFenceBody = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + maintenanceId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + candidateId: string; + /** @pattern ^[a-f0-9]{64}$ */ + planDigest: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + decisionId: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + expectedSequence: number; + outcome: SettleDataConfigurationFenceBodyOutcome; + /** @pattern ^[a-f0-9]{64}$ */ + ownerReceiptDigest: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts new file mode 100644 index 000000000..864bf0301 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFenceBodyOutcome = + (typeof SettleDataConfigurationFenceBodyOutcome)[keyof typeof SettleDataConfigurationFenceBodyOutcome]; + +export const SettleDataConfigurationFenceBodyOutcome = { + unused: "unused", + applied: "applied", + contained: "contained", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202.ts new file mode 100644 index 000000000..a78071318 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartDataExecution202Diagnostics } from "./startDataExecution202Diagnostics"; +import type { StartDataExecution202Kind } from "./startDataExecution202Kind"; +import type { StartDataExecution202Publication } from "./startDataExecution202Publication"; +import type { StartDataExecution202Result } from "./startDataExecution202Result"; +import type { StartDataExecution202Status } from "./startDataExecution202Status"; + +export type StartDataExecution202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: StartDataExecution202Kind; + status: StartDataExecution202Status; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: StartDataExecution202Diagnostics; + publication: StartDataExecution202Publication; + result?: StartDataExecution202Result; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202Diagnostics.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202Diagnostics.ts new file mode 100644 index 000000000..9d0410911 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202Diagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartDataExecution202DiagnosticsTailItem } from "./startDataExecution202DiagnosticsTailItem"; + +export type StartDataExecution202Diagnostics = { + /** @maxItems 256 */ + tail: StartDataExecution202DiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItem.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItem.ts new file mode 100644 index 000000000..2e9c34c08 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartDataExecution202DiagnosticsTailItemStream } from "./startDataExecution202DiagnosticsTailItemStream"; + +export type StartDataExecution202DiagnosticsTailItem = { + stream: StartDataExecution202DiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItemStream.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItemStream.ts new file mode 100644 index 000000000..80558c6e1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202DiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution202DiagnosticsTailItemStream = + (typeof StartDataExecution202DiagnosticsTailItemStream)[keyof typeof StartDataExecution202DiagnosticsTailItemStream]; + +export const StartDataExecution202DiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts new file mode 100644 index 000000000..ae2e69451 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution202Kind = + (typeof StartDataExecution202Kind)[keyof typeof StartDataExecution202Kind]; + +export const StartDataExecution202Kind = { + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202Publication.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202Publication.ts new file mode 100644 index 000000000..445976647 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202Publication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution202Publication = + (typeof StartDataExecution202Publication)[keyof typeof StartDataExecution202Publication]; + +export const StartDataExecution202Publication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts new file mode 100644 index 000000000..d907d3531 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts @@ -0,0 +1,355 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution202Result = + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202Status.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202Status.ts new file mode 100644 index 000000000..019ceaf64 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202Status.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution202Status = + (typeof StartDataExecution202Status)[keyof typeof StartDataExecution202Status]; + +export const StartDataExecution202Status = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution400.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution400.ts new file mode 100644 index 000000000..b9be80490 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution401.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution401.ts new file mode 100644 index 000000000..54fb4f4ae --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution403.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution403.ts new file mode 100644 index 000000000..85496c747 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution409.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution409.ts new file mode 100644 index 000000000..784015e97 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution413.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution413.ts new file mode 100644 index 000000000..bef4b2d97 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution415.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution415.ts new file mode 100644 index 000000000..e64172af8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution429.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution429.ts new file mode 100644 index 000000000..216a30bc2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution500.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution500.ts new file mode 100644 index 000000000..b515c7b53 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution503.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution503.ts new file mode 100644 index 000000000..38240bbe2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecution503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecutionBody.ts b/packages/core/src/services/data-manager/generated/models/startDataExecutionBody.ts new file mode 100644 index 000000000..f6f0bcbd9 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecutionBody.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartDataExecutionBodyInput } from "./startDataExecutionBodyInput"; + +export type StartDataExecutionBody = { + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + input: StartDataExecutionBodyInput; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + requestedDeadlineAt?: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts new file mode 100644 index 000000000..d42f11e8f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts @@ -0,0 +1,91 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartDataExecutionBodyInput = + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + } + | { + kind: "download-fonts"; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + } + | { + kind: "transit-sync"; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + restart?: boolean; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + } + | { + kind: "notable-places-build"; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202.ts new file mode 100644 index 000000000..31009daea --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartGenerationHardlinksExecution202Diagnostics } from "./startGenerationHardlinksExecution202Diagnostics"; +import type { StartGenerationHardlinksExecution202Kind } from "./startGenerationHardlinksExecution202Kind"; +import type { StartGenerationHardlinksExecution202Publication } from "./startGenerationHardlinksExecution202Publication"; +import type { StartGenerationHardlinksExecution202Result } from "./startGenerationHardlinksExecution202Result"; +import type { StartGenerationHardlinksExecution202Status } from "./startGenerationHardlinksExecution202Status"; + +export type StartGenerationHardlinksExecution202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: StartGenerationHardlinksExecution202Kind; + status: StartGenerationHardlinksExecution202Status; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: StartGenerationHardlinksExecution202Diagnostics; + publication: StartGenerationHardlinksExecution202Publication; + result?: StartGenerationHardlinksExecution202Result; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Diagnostics.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Diagnostics.ts new file mode 100644 index 000000000..d5f0ff820 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Diagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartGenerationHardlinksExecution202DiagnosticsTailItem } from "./startGenerationHardlinksExecution202DiagnosticsTailItem"; + +export type StartGenerationHardlinksExecution202Diagnostics = { + /** @maxItems 256 */ + tail: StartGenerationHardlinksExecution202DiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItem.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItem.ts new file mode 100644 index 000000000..8047db696 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartGenerationHardlinksExecution202DiagnosticsTailItemStream } from "./startGenerationHardlinksExecution202DiagnosticsTailItemStream"; + +export type StartGenerationHardlinksExecution202DiagnosticsTailItem = { + stream: StartGenerationHardlinksExecution202DiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItemStream.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItemStream.ts new file mode 100644 index 000000000..aefc74967 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202DiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution202DiagnosticsTailItemStream = + (typeof StartGenerationHardlinksExecution202DiagnosticsTailItemStream)[keyof typeof StartGenerationHardlinksExecution202DiagnosticsTailItemStream]; + +export const StartGenerationHardlinksExecution202DiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts new file mode 100644 index 000000000..59eca469b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution202Kind = + (typeof StartGenerationHardlinksExecution202Kind)[keyof typeof StartGenerationHardlinksExecution202Kind]; + +export const StartGenerationHardlinksExecution202Kind = { + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Publication.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Publication.ts new file mode 100644 index 000000000..f9ebd189d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Publication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution202Publication = + (typeof StartGenerationHardlinksExecution202Publication)[keyof typeof StartGenerationHardlinksExecution202Publication]; + +export const StartGenerationHardlinksExecution202Publication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts new file mode 100644 index 000000000..6fa714b92 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts @@ -0,0 +1,355 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution202Result = + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Status.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Status.ts new file mode 100644 index 000000000..0b4544189 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Status.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution202Status = + (typeof StartGenerationHardlinksExecution202Status)[keyof typeof StartGenerationHardlinksExecution202Status]; + +export const StartGenerationHardlinksExecution202Status = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution400.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution400.ts new file mode 100644 index 000000000..59ee57163 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution401.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution401.ts new file mode 100644 index 000000000..6039d11d2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution403.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution403.ts new file mode 100644 index 000000000..e81f7718e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution409.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution409.ts new file mode 100644 index 000000000..ddaf73d63 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution413.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution413.ts new file mode 100644 index 000000000..3f29b5fa0 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution415.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution415.ts new file mode 100644 index 000000000..7cfa89e85 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution429.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution429.ts new file mode 100644 index 000000000..4431c2c9b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution500.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution500.ts new file mode 100644 index 000000000..dea204b13 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution503.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution503.ts new file mode 100644 index 000000000..4e78955a2 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecution503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBody.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBody.ts new file mode 100644 index 000000000..f786696f8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBody.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartGenerationHardlinksExecutionBodyInput } from "./startGenerationHardlinksExecutionBodyInput"; +import type { StartGenerationHardlinksExecutionBodyPlan } from "./startGenerationHardlinksExecutionBodyPlan"; + +export type StartGenerationHardlinksExecutionBody = { + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + requestedDeadlineAt: number; + input: StartGenerationHardlinksExecutionBodyInput; + plan: StartGenerationHardlinksExecutionBodyPlan; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInput.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInput.ts new file mode 100644 index 000000000..cd3a7a3c6 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInput.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartGenerationHardlinksExecutionBodyInputCoordinatorMode } from "./startGenerationHardlinksExecutionBodyInputCoordinatorMode"; +import type { StartGenerationHardlinksExecutionBodyInputKind } from "./startGenerationHardlinksExecutionBodyInputKind"; + +export type StartGenerationHardlinksExecutionBodyInput = { + kind: StartGenerationHardlinksExecutionBodyInputKind; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** @pattern ^[a-f0-9]{64}$ */ + planDigest: string; + coordinatorMode: StartGenerationHardlinksExecutionBodyInputCoordinatorMode; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputCoordinatorMode.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputCoordinatorMode.ts new file mode 100644 index 000000000..d83f6345b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputCoordinatorMode.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecutionBodyInputCoordinatorMode = + (typeof StartGenerationHardlinksExecutionBodyInputCoordinatorMode)[keyof typeof StartGenerationHardlinksExecutionBodyInputCoordinatorMode]; + +export const StartGenerationHardlinksExecutionBodyInputCoordinatorMode = { + "ensure-running": "ensure-running", + "require-running": "require-running", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputKind.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputKind.ts new file mode 100644 index 000000000..5a2341484 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyInputKind.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecutionBodyInputKind = + (typeof StartGenerationHardlinksExecutionBodyInputKind)[keyof typeof StartGenerationHardlinksExecutionBodyInputKind]; + +export const StartGenerationHardlinksExecutionBodyInputKind = { + "generation-hardlinks": "generation-hardlinks", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlan.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlan.ts new file mode 100644 index 000000000..c558361b0 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlan.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { StartGenerationHardlinksExecutionBodyPlanPlanItem } from "./startGenerationHardlinksExecutionBodyPlanPlanItem"; + +export type StartGenerationHardlinksExecutionBodyPlan = { + /** @maxItems 256 */ + plan: StartGenerationHardlinksExecutionBodyPlanPlanItem[]; + prune: true; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlanPlanItem.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlanPlanItem.ts new file mode 100644 index 000000000..1ea23b004 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecutionBodyPlanPlanItem.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type StartGenerationHardlinksExecutionBodyPlanPlanItem = { + /** + * @minLength 1 + * @maxLength 1024 + */ + source: string; + /** + * @minLength 1 + * @maxLength 1024 + */ + target: string; + /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ + consumerService: string; + /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ + dataType: string; + /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ + instance?: string; + /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ + targetFilename?: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/syncOverture409.ts b/packages/core/src/services/data-manager/generated/models/syncOverture409.ts new file mode 100644 index 000000000..6c16c21f8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/syncOverture409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SyncOverture409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/syncOverture503.ts b/packages/core/src/services/data-manager/generated/models/syncOverture503.ts new file mode 100644 index 000000000..bf2ed62de --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/syncOverture503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SyncOverture503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/syncTransit409.ts b/packages/core/src/services/data-manager/generated/models/syncTransit409.ts index 44d34ce2e..d6ee57704 100644 --- a/packages/core/src/services/data-manager/generated/models/syncTransit409.ts +++ b/packages/core/src/services/data-manager/generated/models/syncTransit409.ts @@ -1,10 +1,26 @@ // biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. // Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { SyncTransit409Reason } from "./syncTransit409Reason"; -export type SyncTransit409 = { - ok: false; - reason: SyncTransit409Reason; - /** @nullable */ - existingJobId: string | null; -}; +export type SyncTransit409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + } + | { + ok: false; + reason: "in-flight" | "duplicate-idempotency-key"; + /** @nullable */ + existingJobId: string | null; + }; diff --git a/packages/core/src/services/data-manager/generated/models/syncTransit409Reason.ts b/packages/core/src/services/data-manager/generated/models/syncTransit409Reason.ts deleted file mode 100644 index 6a158bcb4..000000000 --- a/packages/core/src/services/data-manager/generated/models/syncTransit409Reason.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type SyncTransit409Reason = (typeof SyncTransit409Reason)[keyof typeof SyncTransit409Reason]; - -export const SyncTransit409Reason = { - "in-flight": "in-flight", - "duplicate-idempotency-key": "duplicate-idempotency-key", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/syncTransit503.ts b/packages/core/src/services/data-manager/generated/models/syncTransit503.ts new file mode 100644 index 000000000..7637b7735 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/syncTransit503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SyncTransit503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts index 0779ab553..6ed4cfd27 100644 --- a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts +++ b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts @@ -44,6 +44,13 @@ export async function buildSearchIndexRequest( ): Promise { return buildSearchIndexBound(url, options); } +const cancelDataExecutionBound = bindDataManagerOperation("cancelDataExecution"); +export async function cancelDataExecutionRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return cancelDataExecutionBound(url, options); +} const conflateOvertureBound = bindDataManagerOperation("conflateOverture"); export async function conflateOvertureRequest( url: string, @@ -114,6 +121,13 @@ export async function getCoverageEvidenceRequest( ): Promise { return getCoverageEvidenceBound(url, options); } +const getDataExecutionBound = bindDataManagerOperation("getDataExecution"); +export async function getDataExecutionRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return getDataExecutionBound(url, options); +} const getDatasetsBound = bindDataManagerOperation("getDatasets"); export async function getDatasetsRequest( url: string, @@ -247,10 +261,26 @@ export async function ingestOvertureRequest( ): Promise { return ingestOvertureBound(url, options); } +const inspectDataConfigurationFenceBound = bindDataManagerOperation( + "inspectDataConfigurationFence", +); +export async function inspectDataConfigurationFenceRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return inspectDataConfigurationFenceBound(url, options); +} const linkBound = bindDataManagerOperation("link"); export async function linkRequest(url: string, options?: DataManagerRequestOptions): Promise { return linkBound(url, options); } +const lookupDataExecutionBound = bindDataManagerOperation("lookupDataExecution"); +export async function lookupDataExecutionRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return lookupDataExecutionBound(url, options); +} const preflightTransitBound = bindDataManagerOperation("preflightTransit"); export async function preflightTransitRequest( url: string, @@ -258,6 +288,15 @@ export async function preflightTransitRequest( ): Promise { return preflightTransitBound(url, options); } +const prepareDataConfigurationFenceBound = bindDataManagerOperation( + "prepareDataConfigurationFence", +); +export async function prepareDataConfigurationFenceRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return prepareDataConfigurationFenceBound(url, options); +} const prepareOfflinePackageBound = bindDataManagerOperation("prepareOfflinePackage"); export async function prepareOfflinePackageRequest( url: string, @@ -321,6 +360,29 @@ export async function setAmbientPlacesEnabledRequest( ): Promise { return setAmbientPlacesEnabledBound(url, options); } +const settleDataConfigurationFenceBound = bindDataManagerOperation("settleDataConfigurationFence"); +export async function settleDataConfigurationFenceRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return settleDataConfigurationFenceBound(url, options); +} +const startDataExecutionBound = bindDataManagerOperation("startDataExecution"); +export async function startDataExecutionRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return startDataExecutionBound(url, options); +} +const startGenerationHardlinksExecutionBound = bindDataManagerOperation( + "startGenerationHardlinksExecution", +); +export async function startGenerationHardlinksExecutionRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return startGenerationHardlinksExecutionBound(url, options); +} const syncOvertureBound = bindDataManagerOperation("syncOverture"); export async function syncOvertureRequest( url: string, diff --git a/packages/core/src/services/data-manager/index.ts b/packages/core/src/services/data-manager/index.ts index d8550872d..07ccdcd00 100644 --- a/packages/core/src/services/data-manager/index.ts +++ b/packages/core/src/services/data-manager/index.ts @@ -1,3 +1,22 @@ +export { + inspectDataConfigurationFence, + prepareDataConfigurationFence, + settleDataConfigurationFence, +} from "./configuration-maintenance"; +export type { + DataChildAttempt, + DataExecutionIdentity, + DataExecutionInput, + DataExecutionMetrics, + DataExecutionResult, + DataExecutionStatus, +} from "./executions"; +export { + DataExecutionObservationPending, + matchDataExecutionIdentity, + observeDataExecution, + requestDataExecutionCancellation, +} from "./executions"; export * from "./generated/data-manager.generated"; export * as dataManagerSdk from "./generated/data-manager.generated"; export * from "./streams"; diff --git a/packages/core/src/transitous-runner/contract.ts b/packages/core/src/transitous-runner/contract.ts index d4f7fe963..4f2daa8e3 100644 --- a/packages/core/src/transitous-runner/contract.ts +++ b/packages/core/src/transitous-runner/contract.ts @@ -148,3 +148,13 @@ out = Path("out") out.mkdir(parents=True, exist_ok=True) (out / "feed-proxy-vars.json").write_text(json.dumps(feed_vars, indent=2, sort_keys=True) + "\\n") `; + +/** Authenticated lookup/cancel only returns the exact previously consumed + * capability's process. Absence after runner restart is unknown. */ +export const transitousRunnerStatusSchema = z + .strictObject({ + version: z.literal(1), + state: z.enum(["running", "cancel_pending", "terminal"]), + result: transitousRunnerResultSchema.optional(), + }) + .refine((value) => (value.state === "terminal") === (value.result !== undefined)); diff --git a/packages/core/src/transitous-runner/index.ts b/packages/core/src/transitous-runner/index.ts index a9beef920..3b5a430af 100644 --- a/packages/core/src/transitous-runner/index.ts +++ b/packages/core/src/transitous-runner/index.ts @@ -18,4 +18,5 @@ export { transitousRunnerRequestSchema, transitousRunnerResultSchema, transitousRunnerScriptSchema, + transitousRunnerStatusSchema, } from "./contract"; diff --git a/packages/db-schema/src/data-manager.ts b/packages/db-schema/src/data-manager.ts index 4c190d817..e53405092 100644 --- a/packages/db-schema/src/data-manager.ts +++ b/packages/db-schema/src/data-manager.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm"; import { bigint, + check, index, integer, jsonb, @@ -20,25 +21,65 @@ import { */ export const dataManager = pgSchema("data_manager"); -export const jobs = dataManager.table("jobs", { - id: uuid("id").primaryKey().defaultRandom(), - /** e.g. "transitous-sync" */ - kind: text("kind").notNull(), - startedAt: timestamp("started_at", { withTimezone: true }).notNull().defaultNow(), - finishedAt: timestamp("finished_at", { withTimezone: true }), - /** "running" | "ok" | "partial" | "error" */ - status: text("status").notNull(), - /** "cron" | "manual:" */ - triggeredBy: text("triggered_by"), - /** - * Client-supplied idempotency token. A POST /sync that replays the same - * key within a 24h window (E7 contract) returns 409 instead of starting a - * second pipeline. NULL when the caller did not supply a key (cron path). - */ - idempotencyKey: text("idempotency_key"), - /** { transitousRef, countries[], ... } */ - metadata: jsonb("metadata"), -}); +export const jobs = dataManager.table( + "jobs", + { + id: uuid("id").primaryKey().defaultRandom(), + /** e.g. "transitous-sync" */ + kind: text("kind").notNull(), + startedAt: timestamp("started_at", { withTimezone: true }).notNull().defaultNow(), + finishedAt: timestamp("finished_at", { withTimezone: true }), + /** "running" | "ok" | "partial" | "error" */ + status: text("status").notNull(), + /** "cron" | "manual:" */ + triggeredBy: text("triggered_by"), + /** + * Client-supplied idempotency token. A POST /sync that replays the same + * key within a 24h window (E7 contract) returns 409 instead of starting a + * second pipeline. NULL when the caller did not supply a key (cron path). + */ + idempotencyKey: text("idempotency_key"), + /** { transitousRef, countries[], ... } */ + metadata: jsonb("metadata"), + executionVersion: integer("execution_version"), + operationKey: text("operation_key"), + operationFingerprint: text("operation_fingerprint"), + executionFingerprint: text("execution_fingerprint"), + admittedAt: bigint("admitted_at", { mode: "number" }), + deadlineAt: bigint("deadline_at", { mode: "number" }), + requestedDeadlineAt: bigint("requested_deadline_at", { mode: "number" }), + executionParent: jsonb("execution_parent"), + executionInput: jsonb("execution_input"), + executionState: jsonb("execution_state"), + executionChildren: jsonb("execution_children").notNull().default([]), + ownershipVersion: integer("ownership_version").notNull().default(0), + }, + (table) => [ + uniqueIndex("data_manager_jobs_execution_key_uq") + .on(table.operationKey) + .where(sql`${table.operationKey} IS NOT NULL`), + uniqueIndex("data_manager_jobs_configuration_maintenance_uq") + .on(sql`(${table.metadata}->'configurationMaintenance'->'receipt'->>'maintenanceId')`) + .where(sql`${table.kind} = 'configuration-maintenance'`), + uniqueIndex("data_manager_jobs_configuration_guard_uq") + .on(table.kind) + .where( + sql`${table.kind} = 'configuration-maintenance' AND coalesce(${table.metadata}->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'`, + ), + check( + "data_manager_jobs_execution_bounds", + sql`${table.ownershipVersion} >= 0 AND jsonb_typeof(${table.executionChildren}) = 'array' AND jsonb_array_length(${table.executionChildren}) <= 32 AND octet_length(${table.executionChildren}::text) <= 131072`, + ), + check( + "data_manager_jobs_execution_shape", + sql`${table.executionVersion} IS NULL OR coalesce((${table.executionVersion} = 2 AND ${table.operationKey} ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND ${table.operationFingerprint} ~ '^[a-f0-9]{64}$' AND ${table.executionFingerprint} ~ '^[a-f0-9]{64}$' AND ${table.admittedAt} >= 0 AND ${table.deadlineAt} > ${table.admittedAt} AND ${table.executionInput} IS NOT NULL AND ${table.executionState} IS NOT NULL), false)`, + ), + check( + "data_manager_jobs_configuration_guard_shape", + sql`${table.kind} <> 'configuration-maintenance' OR coalesce((jsonb_typeof(${table.metadata}->'configurationMaintenance') = 'object' AND ${table.metadata}->'configurationMaintenance'->>'version' = '1' AND ${table.metadata}->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(${table.metadata}->'configurationMaintenance'->'receipt') = 'object' AND octet_length((${table.metadata}->'configurationMaintenance')::text) <= 65536), false)`, + ), + ], +); export const jobStages = dataManager.table("job_stages", { id: uuid("id").primaryKey().defaultRandom(), diff --git a/release-notes/durable-data-execution-storage.md b/release-notes/durable-data-execution-storage.md new file mode 100644 index 000000000..f1063893b --- /dev/null +++ b/release-notes/durable-data-execution-storage.md @@ -0,0 +1,23 @@ +--- +impacts: [migration, operator-action] +--- + +## What changes + +Migration 0042 adds retained execution identities, deadlines, ownership versions, +bounded child receipts and a configuration maintenance guard to data-manager jobs. +Existing historical jobs remain readable. Ref-only catalog approval is disabled +until paired validation and activation are available. + +## Who is affected + +Self-hosted installations upgrading the API, data manager and operations agent. + +## Required action + +Apply the normal API database migrations before enabling durable data execution. +Upgrade the control services together. Do not remove unresolved execution or +configuration maintenance records to retry work: uncertain outcomes retain their +original identity and require reconciliation. Durable mutations also require the +installation’s provisioned control trust; missing prerequisites keep mutation +admission closed while inspection remains available. diff --git a/scripts/database-ci-contract.test.ts b/scripts/database-ci-contract.test.ts index 36f106d42..208ac542d 100644 --- a/scripts/database-ci-contract.test.ts +++ b/scripts/database-ci-contract.test.ts @@ -269,6 +269,10 @@ describe("required database test gate", () => { process.execPath, [join(temp, "scripts/test-database.mjs"), "ingestion"], { + env: { + ...process.env, + DATABASE_URL: "postgresql://postgres:postgres@127.0.0.1:1/openmapx_test", + }, cwd: temp, encoding: "utf8", timeout: 5000, diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 0c5eec828..de6224b47 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -25,6 +25,14 @@ "apps/api/src/services/service-config-writer-postgres.test.ts" ], "ingestion": [ + "apps/api/src/services/__tests__/job-runner-postgres.test.ts", + "services/data-manager/src/jobs/execution/store-postgres.test.ts", + "services/data-manager/src/jobs/execution/runtime.test.ts", + "services/data-manager/src/jobs/execution/fixed-cli.test.ts", + "services/data-manager/src/jobs/execution/runner-cancellation.test.ts", + "services/data-manager/src/jobs/execution/hardlinks.test.ts", + "services/data-manager/src/jobs/execution/ops-children.test.ts", + "services/data-manager/src/jobs/execution/configuration-maintenance.test.ts", "services/data-manager/__tests__/ambient-places/api-postgres.test.ts", "services/data-manager/__tests__/ambient-places/extractor-postgres.test.ts", "services/data-manager/__tests__/ambient-places/germany-postgres.test.ts", diff --git a/scripts/test-database.mjs b/scripts/test-database.mjs index 0bfa784c6..2d040cccd 100644 --- a/scripts/test-database.mjs +++ b/scripts/test-database.mjs @@ -24,7 +24,14 @@ export function createDatabaseTestCommand(group = "all", env = process.env) { ); } } - if (group !== "ingestion" && !env.DATABASE_URL?.trim()) { + if ( + !env.DATABASE_URL?.trim() && + !( + group === "ingestion" && + env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL?.trim() && + env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL?.trim() + ) + ) { throw new Error("DATABASE_URL must identify an owned, migrated disposable database"); } return { @@ -38,6 +45,10 @@ export function createDatabaseTestCommand(group = "all", env = process.env) { env: { ...env, OPENMAPX_RUN_DATABASE_TESTS: "1", + OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL: + env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL ?? env.DATABASE_URL, + OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL: + env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL ?? env.DATABASE_URL, OPENMAPX_RUN_RESTORE_DATABASE_TESTS: requiresBackup ? "1" : "0", }, }; diff --git a/services/data-manager/__tests__/ambient-places/api-postgres.test.ts b/services/data-manager/__tests__/ambient-places/api-postgres.test.ts index ad6f024c5..7f75829ad 100644 --- a/services/data-manager/__tests__/ambient-places/api-postgres.test.ts +++ b/services/data-manager/__tests__/ambient-places/api-postgres.test.ts @@ -7,6 +7,7 @@ import { buildSearchIndexIndexesDDL, buildSearchIndexSchemaDDL, } from "../../src/jobs/search-index/schema.js"; +import { installDisposableDataAdmission } from "../helpers/data-admission-fixture.js"; import { startPostgis } from "../helpers/postgis-testcontainer.js"; const request = { @@ -19,6 +20,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( () => { it("rejects competing API instances without aborting the winning repeatable-read build", async () => { const pg = await startPostgis(); + const closeAdmission = await installDisposableDataAdmission(pg.sql); const barrier = postgres(pg.connectionString, { max: 1 }); const coldSql = postgres(pg.connectionString, { max: 2 }); const cold = Fastify(); @@ -88,6 +90,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await barrier.end({ timeout: 2 }); await a.close(); await b.close(); + closeAdmission(); await pg.stop(); } }, 120_000); diff --git a/services/data-manager/__tests__/api.test.ts b/services/data-manager/__tests__/api.test.ts index bb9b7487c..e79b3e773 100644 --- a/services/data-manager/__tests__/api.test.ts +++ b/services/data-manager/__tests__/api.test.ts @@ -1,3 +1,18 @@ +vi.mock("../src/jobs/execution/legacy-mutation.js", () => ({ + startLegacyDataMutation: async (_kind: string, run: () => Promise) => ({ + id: "unit-writer", + completion: run(), + }), + withLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + independentLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), +})); +vi.mock("../src/jobs/persistence.js", async () => ({ + ...(await vi.importActual( + "../src/jobs/persistence.js", + )), + finalizeJobRow: vi.fn(async () => {}), +})); + import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -8,6 +23,7 @@ import { registerApi } from "../src/api.js"; import { resolveOperationsProfile } from "../src/jobs/transitous/operations-profile.js"; import { OperatorFeedRelayStore } from "../src/jobs/transitous/operator-feed-relay.js"; import type { SingleFlightController } from "../src/jobs/transitous/single-flight.js"; +import { dataReceipt, receiverFixture } from "./helpers/execution-receiver.js"; const mockDownloadOsm = vi.hoisted(() => vi.fn(async ({ region }: { region: string }) => ({ @@ -19,6 +35,39 @@ const mockDownloadOsm = vi.hoisted(() => vi.mock("../src/jobs/download-osm.js", () => ({ downloadOsm: mockDownloadOsm })); +function observedUnitReceiver( + kind: import("@openmapx/core/data-manager-contracts").DataExecutionInput["kind"], + run: (progress: (event: Record) => void) => Promise>, +) { + let current = dataReceipt({ kind, status: "running", publication: "unknown" }), + done: Record | undefined, + listener: ((event: Record) => void) | undefined; + return receiverFixture({ + admitLocal: async () => { + setTimeout(() => { + void run((event) => listener?.({ event: "progress", ...event })).then( + (result) => { + done = { event: "done", ok: true, ...result }; + current = { ...current, status: "ok", publication: "committed" }; + }, + () => { + current = { ...current, status: "error", publication: "not_started" }; + }, + ); + }, 0); + return current; + }, + inspect: async () => current, + subscribe: (_id, onEvent) => { + listener = onEvent; + return () => { + listener = undefined; + }; + }, + legacyResult: () => done, + }); +} + describe("data-manager API", () => { let fixtureRoot: string; let apps: FastifyInstance[]; @@ -154,23 +203,54 @@ describe("data-manager API", () => { expect(res.json()).toEqual({ ok: false, error: "osm_search index not built" }); }); + it("continues a legacy observer through running unknown publication without canceling its worker", async () => { + const status = dataReceipt({ + kind: "download-fonts", + status: "running", + publication: "unknown", + }), + inspect = vi + .fn() + .mockResolvedValueOnce(status) + .mockResolvedValueOnce({ ...status, status: "ok", publication: "committed" }), + cancel = vi.fn(), + app = createTestApp(); + registerApi(app, { + dataDir: createDataDir(), + executionReceiver: receiverFixture({ + admitLocal: async () => status, + inspect, + requestCancel: cancel, + }), + }); + const response = await app.inject({ method: "POST", url: "/download/fonts" }); + expect(response.statusCode).toBe(200); + expect(inspect).toHaveBeenCalledTimes(2); + expect(response.json()).toEqual({ ok: true }); + expect(cancel).not.toHaveBeenCalled(); + }); + it("POST /search-index/build streams progress and the final snapshot", async () => { const dataDir = createDataDir(); const app = createTestApp(); - const buildSearchIndex = vi.fn(async (options: { onProgress?: (event: object) => void }) => { - options.onProgress?.({ stage: "extract", message: "streaming" }); - return { - region: "europe/germany", - epoch: "new-epoch", - sourceFingerprint: "md5:abc", - placeCount: 2, - termCount: 3, - }; - }); + const buildSearchIndex = vi.fn( + async (options: { onProgress?: (event: Record) => void }) => { + options.onProgress?.({ stage: "extract", message: "streaming" }); + return { + region: "europe/germany", + epoch: "new-epoch", + sourceFingerprint: "md5:abc", + placeCount: 2, + termCount: 3, + }; + }, + ); registerApi(app, { dataDir, searchIndexSql: { reserve: vi.fn() } as never, - buildSearchIndex: buildSearchIndex as never, + executionReceiver: observedUnitReceiver("search-index-build", (onProgress) => + buildSearchIndex({ onProgress }), + ), }); const res = await app.inject({ method: "POST", @@ -203,23 +283,27 @@ describe("data-manager API", () => { it("POST /notable-places/build streams progress and the published snapshot", async () => { const app = createTestApp(); - const build = vi.fn(async (options: { onProgress?: (event: object) => void }) => { - options.onProgress?.({ stage: "names", message: "Fetching labels in en" }); - return { - epoch: "e1", - source: "https://sparql.test", - minSitelinks: 8, - placeCount: 5, - nameCount: 9, - }; - }); + const build = vi.fn( + async (options: { onProgress?: (event: Record) => void }) => { + options.onProgress?.({ stage: "names", message: "Fetching labels in en" }); + return { + epoch: "e1", + source: "https://sparql.test", + minSitelinks: 8, + placeCount: 5, + nameCount: 9, + }; + }, + ); registerApi(app, { dataDir: join(fixtureRoot, "data"), searchIndexSql: { reserve: vi.fn() } as never, + executionReceiver: observedUnitReceiver("notable-places-build", (onProgress) => + build({ onProgress }), + ), notablePlaces: { runtimeState: { building: false, failure: null }, operationLock: { inFlight: false, run: (operation) => operation() }, - build: build as never, }, }); const res = await app.inject({ method: "POST", url: "/notable-places/build" }); @@ -233,7 +317,7 @@ describe("data-manager API", () => { ]); }); - it("keeps a successful OSM download when search-index fingerprint persistence fails", async () => { + it("preserves a successful owned OSM receipt in the legacy observer", async () => { const dataDir = createDataDir(); writeFileSync( join(dataDir, ".data-manager-state.json"), @@ -259,6 +343,9 @@ describe("data-manager API", () => { registerApi(app, { dataDir, searchIndexSql: { unsafe, reserve: vi.fn() } as never, + executionReceiver: observedUnitReceiver("download-osm", () => + mockDownloadOsm({ region: "europe/germany" }), + ), }); const res = await app.inject({ method: "POST", @@ -320,11 +407,12 @@ describe("data-manager API", () => { reason: "in-flight" as const, existingJobId: "running-job", })), + reserveOwned: vi.fn(() => true), markSyncFinished: vi.fn(), getInflight: () => ({ jobId: "running-job", startedAt: new Date() }), }; const app = createTestApp(); - registerApi(app, { dataDir, singleFlight, launchTransitSync: vi.fn() }); + registerApi(app, { dataDir, singleFlight, executionReceiver: receiverFixture() }); const res = await app.inject({ method: "POST", url: "/transit/sources", @@ -348,12 +436,16 @@ describe("data-manager API", () => { events.push("job-inserted"); return { ok: true as const, jobId: "job-visible" }; }), + reserveOwned: vi.fn(() => true), markSyncFinished: vi.fn(), getInflight: () => ({ jobId: "job-visible", startedAt: new Date() }), }; - const launchTransitSync = vi.fn(() => events.push("pipeline-launched")); + const admitLocal = vi.fn(async () => { + events.push("durable-admitted"); + return dataReceipt({ kind: "transit-sync" }); + }); const app = createTestApp(); - registerApi(app, { dataDir, singleFlight, launchTransitSync }); + registerApi(app, { dataDir, singleFlight, executionReceiver: receiverFixture({ admitLocal }) }); const res = await app.inject({ method: "POST", url: "/transit/sources", @@ -366,11 +458,11 @@ describe("data-manager API", () => { }); expect(res.statusCode).toBe(202); expect(res.json()).toEqual({ - jobId: "job-visible", + jobId: dataReceipt().id, sourceId: "operator:de:operator-feed", status: "started", }); - expect(events).toEqual(["job-inserted", "pipeline-launched"]); + expect(events).toEqual(["job-inserted", "durable-admitted"]); const overlay = JSON.parse( readFileSync(join(dataDir, "overrides", "feeds-overlay.json"), "utf-8"), ) as { version: number }; @@ -384,6 +476,7 @@ describe("data-manager API", () => { const dataDir = createDataDir(); const singleFlight: SingleFlightController = { tryStartSync: vi.fn(), + reserveOwned: vi.fn(() => true), markSyncFinished: vi.fn(), getInflight: () => null, }; @@ -392,7 +485,6 @@ describe("data-manager API", () => { dataDir, operationsPolicy: resolveOperationsProfile({ countries: ["de"] }), singleFlight, - launchTransitSync: vi.fn(), }); const res = await app.inject({ @@ -409,16 +501,17 @@ describe("data-manager API", () => { const dataDir = createDataDir(); const singleFlight: SingleFlightController = { tryStartSync: vi.fn(async () => ({ ok: true as const, jobId: "job-sync" })), + reserveOwned: vi.fn(() => true), markSyncFinished: vi.fn(), getInflight: () => ({ jobId: "job-sync", startedAt: new Date() }), }; - const launchTransitSync = vi.fn(); + const admitLocal = vi.fn(async () => dataReceipt({ kind: "transit-sync" })); const app = createTestApp(); registerApi(app, { dataDir, operationsPolicy: resolveOperationsProfile({ countries: ["de"] }), singleFlight, - launchTransitSync, + executionReceiver: receiverFixture({ admitLocal }), }); const res = await app.inject({ @@ -428,12 +521,8 @@ describe("data-manager API", () => { }); expect(res.statusCode).toBe(202); - expect(res.json()).toEqual({ ok: true, jobId: "job-sync", status: "started" }); - expect(launchTransitSync).toHaveBeenCalledWith({ - jobId: "job-sync", - countries: ["DE"], - trigger: "api", - }); + expect(res.json()).toEqual({ ok: true, jobId: dataReceipt().id, status: "started" }); + expect(admitLocal).toHaveBeenCalledWith({ kind: "transit-sync", countries: ["DE"] }); }); it.each(["../etc", "Europe/Germany"])( @@ -442,7 +531,10 @@ describe("data-manager API", () => { const dataDir = createDataDir(); mockDownloadOsm.mockClear(); const app = createTestApp(); - registerApi(app, { dataDir }); + registerApi(app, { + dataDir, + executionReceiver: observedUnitReceiver("download-osm", () => mockDownloadOsm({ region })), + }); const res = await app.inject({ method: "POST", @@ -461,7 +553,10 @@ describe("data-manager API", () => { const dataDir = createDataDir(); mockDownloadOsm.mockClear(); const app = createTestApp(); - registerApi(app, { dataDir }); + registerApi(app, { + dataDir, + executionReceiver: observedUnitReceiver("download-osm", () => mockDownloadOsm({ region })), + }); const res = await app.inject({ method: "POST", @@ -470,7 +565,7 @@ describe("data-manager API", () => { }); expect(res.statusCode).toBe(200); - expect(mockDownloadOsm).toHaveBeenCalledWith(expect.objectContaining({ region, dataDir })); + expect(mockDownloadOsm).toHaveBeenCalledWith(expect.objectContaining({ region })); expect(res.body).toContain('"event":"done"'); }, ); @@ -546,14 +641,14 @@ describe("data-manager API", () => { expect(res.statusCode).toBe(400); }); - it("POST /link accepts a well-formed empty plan", async () => { + it("POST /link closes a legacy unsigned plan before effects", async () => { const dataDir = createDataDir(); const app = createTestApp(); registerApi(app, { dataDir }); const res = await app.inject({ method: "POST", url: "/link", payload: { plan: [] } }); - expect(res.statusCode).toBe(200); - expect(JSON.parse(res.body).ok).toBe(true); + expect(res.statusCode).toBe(503); + expect(res.json()).toEqual({ error: "A signed generation-hardlinks execution is required" }); }); }); diff --git a/services/data-manager/__tests__/atomic-download-cancel.test.ts b/services/data-manager/__tests__/atomic-download-cancel.test.ts new file mode 100644 index 000000000..490d48b6f --- /dev/null +++ b/services/data-manager/__tests__/atomic-download-cancel.test.ts @@ -0,0 +1,72 @@ +import { mkdtemp, readdir, readFile, rm, utimes, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, it } from "vitest"; +import { curlAtomicWithResult } from "../src/jobs/atomic-download.js"; + +it("contains a real curl transfer before cancellation returns and preserves the prior target", async () => { + const dir = await mkdtemp(join(tmpdir(), "omx-download-cancel-")); + const target = join(dir, "snapshot"); + await writeFile(target, "previous"); + const controller = new AbortController(); + let observed = () => {}; + const requestSeen = new Promise((resolve) => { + observed = resolve; + }); + let release = () => {}; + const released = new Promise((resolve) => { + release = resolve; + }); + const server = createServer(async (_request, response) => { + response.write("partial"); + observed(); + await released; + response.end("complete"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address(); + if (!address || typeof address === "string") throw Error("Missing loopback server"); + const pending = curlAtomicWithResult(`http://127.0.0.1:${address.port}/fixture`, target, { + signal: controller.signal, + }); + await requestSeen; + controller.abort(new Error("execution canceled")); + release(); + await expect(pending).rejects.toThrow(); + expect(await readFile(target, "utf8")).toBe("previous"); + expect(await readdir(dir)).toEqual(["snapshot"]); + } finally { + release(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await rm(dir, { recursive: true, force: true }); + } +}); +it("rechecks cancellation after validation immediately before publication", async () => { + const dir = await mkdtemp(join(tmpdir(), "omx-download-publish-cancel-")); + const source = join(dir, "fixture"), + target = join(dir, "snapshot"); + await writeFile(source, "new"); + await writeFile(target, "previous"); + await utimes(target, new Date(0), new Date(0)); + const controller = new AbortController(); + let validated = false; + try { + await expect( + curlAtomicWithResult(pathToFileURL(source).href, target, { + signal: controller.signal, + beforePublish: async () => { + validated = true; + controller.abort(new Error("canceled at publication")); + }, + }), + ).rejects.toThrow(); + expect(validated).toBe(true); + expect(await readFile(target, "utf8")).toBe("previous"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/services/data-manager/__tests__/cron.test.ts b/services/data-manager/__tests__/cron.test.ts index bc03585db..87fad5208 100644 --- a/services/data-manager/__tests__/cron.test.ts +++ b/services/data-manager/__tests__/cron.test.ts @@ -10,7 +10,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // never resolves and the test hangs to the 15s timeout. Mock the db module to // a chainable thenable that resolves immediately — no query ever executes. // The catalog lock is agent-owned. This models the agent's slot behaviour so -// the cron's propose/approve sequence can be asserted without a repo mount. +// the cron's proposal retention can be asserted without a repo mount. const lockSlots: { active: Record | null; proposed: Record | null; @@ -28,15 +28,8 @@ vi.mock("../src/ops-client.js", () => ({ lockedBy: operation.lockedBy, }; return { ref: operation.ref, proposed: true }; - case "transitousLock.approve": { - if (!lockSlots.proposed) throw new Error("No Transitous lock proposal to approve"); - if (lockSlots.proposed.ref !== operation.ref) { - throw new Error("Transitous lock proposal does not match the approved ref"); - } - lockSlots.active = { ...lockSlots.proposed, lockedBy: operation.approvedBy }; - lockSlots.proposed = null; - return { ref: operation.ref, lockedAt: "2026-05-01T00:00:00.000Z" }; - } + case "transitousLock.approve": + throw new Error("Paired approval requires the validation-bound activation runtime"); default: return { changed: true }; } @@ -67,12 +60,14 @@ vi.mock("../src/db/index.js", () => { return { db: makeChain(), sql: { end: async () => {} } }; }); +import type { DataExecutionStatus } from "@openmapx/core/data-manager-contracts"; import { awaitInflightSync, type CronHandles, type CronSetupOptions, - setupCron, + setupCron as setupProductionCron, } from "../src/cron.js"; +import type { DataExecutionReceiver } from "../src/jobs/execution/api.js"; import * as trafficEvidence from "../src/jobs/traffic/evidence.js"; import * as spanEdges from "../src/jobs/traffic/span-edges.js"; import { @@ -80,6 +75,50 @@ import { type SingleFlightController, } from "../src/jobs/transitous/single-flight.js"; +vi.mock("../src/jobs/execution/legacy-mutation.js", () => ({ + withLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + independentLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), +})); +function receipt(patch: Partial = {}): DataExecutionStatus { + return { + id: "00000000-0000-4000-8000-000000000001", + operationKey: "opk1_unit_cron_original", + fingerprint: "a".repeat(64), + admittedAt: 1, + deadlineAt: 4102444800000, + kind: "transit-sync", + status: "ok", + publication: "committed", + cancellationRequestedAt: null, + progress: 1, + diagnostics: { tail: [], truncated: false }, + ...patch, + }; +} +function receiverFixture(patch: Partial = {}): DataExecutionReceiver { + const unsupported = async (): Promise => { + throw Error("Unused fixture receiver operation"); + }; + return { + admitLocal: async () => receipt(), + admitMaintenance: async () => receipt({ kind: "transit-maintenance" }), + subscribe: () => () => {}, + legacyResult: () => undefined, + admit: unsupported, + admitHardlinks: unsupported, + inspect: async () => receipt(), + lookup: unsupported, + requestCancel: unsupported, + prepareFence: unsupported, + inspectFence: unsupported, + settleFence: unsupported, + ...patch, + }; +} +function setupCron(options: CronSetupOptions) { + return setupProductionCron({ executionReceiver: receiverFixture(), ...options }); +} + interface JobRow { id: string; kind: string; @@ -220,7 +259,7 @@ describe("setupCron", () => { key: "syncCron", configure: (work) => ({ syncCronExpression: everySecond, - runPipeline: async (jobId) => ({ jobId, results: [], finalStatus: "ok" }), + executionReceiver: receiverFixture(), // Keep the post-sync check pending after the domain lock releases: // the scheduler must still own the complete callback lifetime. runStalenessCheck: work, @@ -230,7 +269,12 @@ describe("setupCron", () => { key: "feedProxyReloadCron", configure: (work) => ({ feedProxyReloadCronExpression: everySecond, - reloadFeedProxy: work, + executionReceiver: receiverFixture({ + admitMaintenance: async () => { + await work(); + return receipt({ kind: "transit-maintenance" }); + }, + }), }), }, { @@ -253,7 +297,6 @@ describe("setupCron", () => { transitlandAtlasSha: "b".repeat(40), }; }, - runBumpPipeline: async (jobId) => ({ jobId, results: [], finalStatus: "ok" }), }), }, { @@ -519,7 +562,11 @@ describe("setupCron", () => { logger: { info: () => {}, warn: () => {}, error: () => {} }, syncCronExpression: "0 0 1 1 *", // far-future; we trigger manually feedProxyReloadCronExpression: "disabled", - runPipeline, + executionReceiver: receiverFixture({ + admitLocal: async () => { + throw Error("configuration_maintenance_busy"); + }, + }), }); await handles.runSyncNow(); @@ -545,7 +592,17 @@ describe("setupCron", () => { syncCronExpression: "0 0 1 1 *", feedProxyReloadCronExpression: "disabled", stalenessCheckCronExpression: "disabled", - runPipeline, + executionReceiver: receiverFixture({ + admitLocal: async () => { + await runPipeline(); + return receipt({ status: "error" }); + }, + legacyResult: () => ({ + results: [ + { stage: "motis-health", status: "error", error: { message: "rentals empty" } }, + ], + }), + }), runStalenessCheck: async () => {}, githubIssueSink: { findOpenIssueByTitle: async () => null, createIssue }, }); @@ -559,6 +616,42 @@ describe("setupCron", () => { handles.stop(); }); + it("keeps observing running unknown publication and alerts on the original partial terminal", async () => { + const inspect = vi + .fn() + .mockResolvedValueOnce(receipt({ status: "running", publication: "unknown" })) + .mockResolvedValueOnce(receipt({ status: "partial" })), + cancel = vi.fn(), + createIssue = vi.fn(async () => "https://x/issues/2"), + staleness = vi.fn(async () => {}); + const handles = setupCron({ + dataDir, + countries: [], + store: {} as never, + singleFlight: makeController(), + logger: { info: () => {}, warn: () => {}, error: () => {} }, + syncCronExpression: "disabled", + feedProxyReloadCronExpression: "disabled", + stalenessCheckCronExpression: "disabled", + runStalenessCheck: staleness, + githubIssueSink: { createIssue }, + executionReceiver: receiverFixture({ + admitLocal: async () => receipt({ status: "running", publication: "unknown" }), + inspect, + requestCancel: cancel, + }), + }); + try { + await handles.runSyncNow(); + expect(inspect).toHaveBeenCalledTimes(2); + expect(createIssue).toHaveBeenCalledOnce(); + expect(staleness).toHaveBeenCalledOnce(); + expect(cancel).not.toHaveBeenCalled(); + } finally { + handles.stop(); + } + }); + describe("auto-bump cron", () => { const ACTIVE = "a".repeat(40); const CANDIDATE = "e".repeat(40); @@ -601,7 +694,7 @@ describe("setupCron", () => { handles.stop(); }); - it("activates the new pin when the canary passes", async () => { + it("retains the proposal without running a canary or unvalidated activation", async () => { seedLocks(); const runBumpPipeline = vi.fn(async (_jobId: string) => ({ jobId: "j", @@ -619,18 +712,17 @@ describe("setupCron", () => { stalenessCheckCronExpression: "disabled", autoBumpCronExpression: "0 0 1 1 *", resolveBumpCandidate: async () => newCandidate, - runBumpPipeline, }); await handles.runAutoBumpNow(); - expect(runBumpPipeline).toHaveBeenCalledTimes(1); + expect(runBumpPipeline).not.toHaveBeenCalled(); const active = activeLock(); - expect(active.ref).toBe(`main@${CANDIDATE}`); - expect(active.submodules["transitland-atlas"]).toBe(ATLAS_NEW); - expect(proposalExists()).toBe(false); + expect(active.ref).toBe(`main@${ACTIVE}`); + expect(active.submodules["transitland-atlas"]).toBe(ATLAS_OLD); + expect(proposalExists()).toBe(true); handles.stop(); }); - it("keeps the current pin, retains the proposal, and alerts when the canary rejects", async () => { + it("keeps the current pin and proposal while validation-bound activation is unavailable", async () => { seedLocks(); const createIssue = vi.fn(async (_t: string, _b: string) => "https://x/issues/1"); const runBumpPipeline = vi.fn(async (_jobId: string) => { @@ -647,15 +739,14 @@ describe("setupCron", () => { stalenessCheckCronExpression: "disabled", autoBumpCronExpression: "0 0 1 1 *", resolveBumpCandidate: async () => newCandidate, - runBumpPipeline, + githubIssueSink: { findOpenIssueByTitle: async () => null, createIssue }, }); await handles.runAutoBumpNow(); expect(activeLock().ref).toBe(`main@${ACTIVE}`); expect(proposalExists()).toBe(true); - expect(createIssue).toHaveBeenCalledTimes(1); - const [title] = createIssue.mock.calls[0] ?? []; - expect(title).toContain("auto-bump"); + expect(runBumpPipeline).not.toHaveBeenCalled(); + expect(createIssue).not.toHaveBeenCalled(); handles.stop(); }); @@ -682,7 +773,6 @@ describe("setupCron", () => { transitousSha: ACTIVE, transitlandAtlasSha: ATLAS_OLD, }), - runBumpPipeline, }); await handles.runAutoBumpNow(); expect(runBumpPipeline).not.toHaveBeenCalled(); @@ -707,7 +797,12 @@ describe("setupCron", () => { logger: { info: () => {}, warn: () => {}, error: () => {} }, syncCronExpression: "disabled", feedProxyReloadCronExpression: "0 0 1 1 *", - reloadFeedProxy: reload, + executionReceiver: receiverFixture({ + admitMaintenance: async () => { + await reload(); + return receipt({ kind: "transit-maintenance" }); + }, + }), }); // First run: file exists, no previous reload, so we should reload. @@ -737,7 +832,12 @@ describe("setupCron", () => { logger: { info: () => {}, warn: () => {}, error: () => {} }, syncCronExpression: "disabled", feedProxyReloadCronExpression: "0 0 1 1 *", - reloadFeedProxy: reload, + executionReceiver: receiverFixture({ + admitMaintenance: async () => { + await reload(); + return receipt({ kind: "transit-maintenance" }); + }, + }), }); await handles.runFeedProxyReloadNow(); expect(reload).not.toHaveBeenCalled(); diff --git a/services/data-manager/__tests__/helpers/data-admission-fixture.ts b/services/data-manager/__tests__/helpers/data-admission-fixture.ts new file mode 100644 index 000000000..de94c327a --- /dev/null +++ b/services/data-manager/__tests__/helpers/data-admission-fixture.ts @@ -0,0 +1,42 @@ +/** Admission prerequisites for the existing disposable DB suites. This runs + * only in those new owned fixtures, never against the retained Task02.3 DB. */ +import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import type postgres from "postgres"; +import { installDataMutationAdmission } from "../../src/jobs/execution/admission.js"; +import { createDataConfigurationMaintenance } from "../../src/jobs/execution/configuration-maintenance.js"; +export async function installDisposableDataAdmission(sql: postgres.Sql): Promise<() => void> { + const existing = await sql.unsafe("SELECT to_regclass('data_manager.jobs') AS table_name"); + if (existing[0]?.table_name) throw Error("Admission fixture requires its new disposable schema"); + const migrations = resolve(import.meta.dirname, "../../../../apps/api/src/db/migrations"); + for (const name of [ + "0004_data_manager_jobs_and_feed_state.sql", + "0042_admin_data_execution.sql", + ]) { + const body = readFileSync(join(migrations, name), "utf8") + .replace('CREATE SCHEMA "data_manager";', 'CREATE SCHEMA IF NOT EXISTS "data_manager";') + .replaceAll("--> statement-breakpoint", ""); + await sql.unsafe(body); + } + const root = mkdtempSync(join(tmpdir(), "omx-disposable-admission-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + const installation = openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }), + fence = createDataConfigurationMaintenance(sql, installation); + await fence.startup(); + installDataMutationAdmission(fence); + return () => { + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + }; +} diff --git a/services/data-manager/__tests__/helpers/execution-receiver.ts b/services/data-manager/__tests__/helpers/execution-receiver.ts new file mode 100644 index 000000000..c1ba12d61 --- /dev/null +++ b/services/data-manager/__tests__/helpers/execution-receiver.ts @@ -0,0 +1,40 @@ +import type { DataExecutionStatus } from "@openmapx/core/data-manager-contracts"; +import type { DataExecutionReceiver } from "../../src/jobs/execution/api.js"; +/** Unit observer collaborator only. Real SQL/authority/HTTP acceptance uses the + * owned fixtures under src/jobs/execution, never this adapter. */ +export function dataReceipt(patch: Partial = {}): DataExecutionStatus { + return { + id: "00000000-0000-4000-8000-000000000001", + operationKey: "opk1_unit_original_cron", + fingerprint: "a".repeat(64), + admittedAt: 1, + deadlineAt: 4102444800000, + kind: "overture-sync", + status: "ok", + publication: "committed", + cancellationRequestedAt: null, + progress: 1, + diagnostics: { tail: [], truncated: false }, + ...patch, + }; +} +export function receiverFixture(patch: Partial = {}): DataExecutionReceiver { + const unsupported = async (): Promise => { + throw Error("Unused unit receiver operation"); + }; + return { + admitLocal: async () => dataReceipt(), + admitMaintenance: unsupported, + subscribe: () => () => {}, + legacyResult: () => undefined, + admit: unsupported, + admitHardlinks: unsupported, + inspect: async () => dataReceipt(), + lookup: unsupported, + requestCancel: unsupported, + prepareFence: unsupported, + inspectFence: unsupported, + settleFence: unsupported, + ...patch, + }; +} diff --git a/services/data-manager/__tests__/helpers/owned-transit-unit.ts b/services/data-manager/__tests__/helpers/owned-transit-unit.ts new file mode 100644 index 000000000..05e6030dc --- /dev/null +++ b/services/data-manager/__tests__/helpers/owned-transit-unit.ts @@ -0,0 +1,44 @@ +/** Unit stage I/O collaborator only. This exercises directory/probe/rollback + * semantics; production authority is qualified by real SQL/separate processes. */ +import type { DataOpsChildPhase } from "../../src/jobs/execution/ops-children.js"; +import { publishDataChange } from "../../src/jobs/execution/worker-scope.js"; +import type { JobContext } from "../../src/jobs/transitous/types.js"; +import { runOpsOperation } from "../../src/ops-client.js"; +export function assertTransitForward(ctx: JobContext) { + ctx.abortSignal.throwIfAborted(); +} +export function retainTransitPhaseCandidate() { + return Promise.resolve(); +} +/** No dispatch ticket is fabricated by this I/O-only unit collaborator. */ +export function reserveTransitProxyRecovery() { + return Promise.resolve(); +} +export async function publishTransitChange( + ctx: JobContext, + run: () => T | Promise, +): Promise { + assertTransitForward(ctx); + return publishDataChange(run); +} +export async function runTransitOwnedPhase(ctx: JobContext, phase: DataOpsChildPhase) { + assertTransitForward(ctx); + switch (phase) { + case "stop-primary": + return runOpsOperation({ kind: "motis.primary.stop" }); + case "stop-staging": + return runOpsOperation({ kind: "motis.staging.stop" }); + case "restart-staging": + return runOpsOperation({ kind: "motis.staging.restart" }); + case "promote-primary": + return runOpsOperation({ kind: "motis.primary.promote", preparedRunId: ctx.jobId }); + case "recover-primary": + return runOpsOperation({ kind: "motis.primary.restart" }); + case "proxy-stage": + case "proxy-commit": + case "proxy-restore": + return runOpsOperation({ kind: "feedProxy.validateAndReload", candidateId: `unit-${phase}` }); + default: + throw Error("Closed unit phase"); + } +} diff --git a/services/data-manager/__tests__/job-persistence.test.ts b/services/data-manager/__tests__/job-persistence.test.ts index eb521a766..6a2d60c12 100644 --- a/services/data-manager/__tests__/job-persistence.test.ts +++ b/services/data-manager/__tests__/job-persistence.test.ts @@ -13,8 +13,36 @@ const insertCalls: InsertCall[] = []; const updateCalls: UpdateCall[] = []; let returnedRows: Array<{ id: string }> = []; let insertError: Error | null = null; +let admissionError: Error | null = null; +let admissionChecks = 0; +vi.mock("../src/jobs/execution/admission.js", () => ({ + assertDataMutationAdmission: async () => { + admissionChecks++; + if (admissionError) throw admissionError; + }, +})); vi.mock("../src/db/index.js", () => ({ + rawSql: { + begin: async (run: (tx: unknown) => Promise) => { + const tx = Object.assign( + async (_query: TemplateStringsArray, ...values: unknown[]) => { + insertCalls.push({ + values: { + kind: values[0], + status: "running", + triggeredBy: values[1], + metadata: typeof values[2] === "string" ? JSON.parse(values[2]) : values[2], + }, + }); + if (insertError) throw insertError; + return returnedRows; + }, + { json: (value: unknown) => value }, + ); + return run(tx); + }, + }, db: { insert: () => ({ values: (values: Record) => { @@ -54,6 +82,8 @@ beforeEach(() => { updateCalls.length = 0; returnedRows = []; insertError = null; + admissionError = null; + admissionChecks = 0; }); describe("job persistence", () => { @@ -68,6 +98,7 @@ describe("job persistence", () => { metadata: { sourceId: "example" }, }), ).resolves.toBe("job-1"); + expect(admissionChecks).toBe(1); expect(insertCalls[0]?.values).toEqual({ kind: "example-sync", status: "running", @@ -76,6 +107,15 @@ describe("job persistence", () => { }); }); + it("checks the durable maintenance admission before any legacy insertion", async () => { + const { createJobRow } = await import("../src/jobs/persistence.js"); + admissionError = Error("configuration_maintenance_busy"); + await expect(createJobRow({ kind: "example-sync" })).rejects.toThrow( + "configuration_maintenance_busy", + ); + expect(insertCalls).toEqual([]); + }); + it("fails when no job row is returned", async () => { const { createJobRow } = await import("../src/jobs/persistence.js"); await expect(createJobRow({ kind: "example-sync" })).rejects.toThrow( diff --git a/services/data-manager/__tests__/notable-places/schedule.test.ts b/services/data-manager/__tests__/notable-places/schedule.test.ts index 607cff0cf..4bacf9bd0 100644 --- a/services/data-manager/__tests__/notable-places/schedule.test.ts +++ b/services/data-manager/__tests__/notable-places/schedule.test.ts @@ -58,7 +58,9 @@ describe("notable-places schedule", () => { operationLock: lock, logger, cronExpression: "0 2 1 * *", - build, + runOwned: async () => { + await build(); + }, }); await schedule.buildIfMissing(); schedule.stop(); @@ -73,7 +75,9 @@ describe("notable-places schedule", () => { operationLock: lock, logger, cronExpression: "0 2 1 * *", - build, + runOwned: async () => { + await build(); + }, }); await schedule.buildIfMissing(); schedule.stop(); @@ -88,7 +92,9 @@ describe("notable-places schedule", () => { operationLock: lock, logger, cronExpression: "off", - build, + runOwned: async () => { + await build(); + }, }); await schedule.buildIfMissing(); schedule.stop(); @@ -106,7 +112,9 @@ describe("notable-places schedule", () => { operationLock: lock, logger: { info: vi.fn(), warn }, cronExpression: "0 2 1 * *", - build, + runOwned: async () => { + await build(); + }, }); await expect(schedule.buildIfMissing()).resolves.toBeUndefined(); schedule.stop(); diff --git a/services/data-manager/__tests__/offline-package-postgres-accounting.test.ts b/services/data-manager/__tests__/offline-package-postgres-accounting.test.ts index 6d5d88a93..135329a83 100644 --- a/services/data-manager/__tests__/offline-package-postgres-accounting.test.ts +++ b/services/data-manager/__tests__/offline-package-postgres-accounting.test.ts @@ -11,11 +11,13 @@ import { OfflinePackageGenerator } from "../src/offline-packages/generator.js"; import { PostgresOfflinePackageAccountingStore } from "../src/offline-packages/postgres-accounting.js"; import { OfflinePackageStorage } from "../src/offline-packages/storage.js"; import type { OfflinePackageJobRecord } from "../src/offline-packages/types.js"; +import { installDisposableDataAdmission } from "./helpers/data-admission-fixture.js"; const principal = "a".repeat(64); const integration = describe.runIf(process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"); let container: StartedPostgreSqlContainer; let sql: Sql; +let closeAdmission: (() => void) | undefined; const artifacts = new Map(); const artifactOptions = { readManifest: async (id: string) => artifacts.get(id) }; @@ -134,6 +136,7 @@ integration("PostgreSQL offline-package accounting", () => { "utf8", ).replaceAll("--> statement-breakpoint", ""); await sql.unsafe(migration); + closeAdmission = await installDisposableDataAdmission(sql); }, 120_000); afterEach(async () => { @@ -145,6 +148,7 @@ integration("PostgreSQL offline-package accounting", () => { }); afterAll(async () => { + closeAdmission?.(); await sql?.end(); await container?.stop(); }); diff --git a/services/data-manager/__tests__/ops-client-ownership.test.ts b/services/data-manager/__tests__/ops-client-ownership.test.ts new file mode 100644 index 000000000..90577cc04 --- /dev/null +++ b/services/data-manager/__tests__/ops-client-ownership.test.ts @@ -0,0 +1,53 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { OpsClient } from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { resetOpsClient, runOpsOperation } from "../src/ops-client.js"; + +it("refuses an unowned synchronous proposal before the actual client POST", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-proposal-owner-")), + tokenFile = join(root, "token"); + writeFileSync(tokenFile, "a".repeat(43), { mode: 0o600 }); + let posts = 0; + const operation = { + kind: "transitousLock.propose" as const, + ref: `main@${"a".repeat(40)}`, + submodules: { "transitland-atlas": "b".repeat(40) }, + lockedBy: "public-fixture", + }; + resetOpsClient( + new OpsClient({ + baseUrl: "http://127.0.0.1:4300", + tokenFile, + fetch: async (_url, options) => { + if (options?.method === "GET") + return Response.json({ + version: 1, + requestId: new Headers(options.headers).get("x-ops-request-id"), + ok: true, + result: { versions: [1, 2] }, + }); + posts++; + const request = JSON.parse(String(options?.body)); + return Response.json({ + version: 1, + requestId: request.requestId, + ok: true, + result: { + execution: "sync", + kind: operation.kind, + value: { ref: operation.ref, proposed: true }, + }, + }); + }, + }), + ); + try { + await expect(runOpsOperation(operation)).rejects.toThrow(/retained.*owner|retained.*mutation/i); + expect(posts).toBe(0); + } finally { + resetOpsClient(); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/services/data-manager/__tests__/overture/overture-cron.test.ts b/services/data-manager/__tests__/overture/overture-cron.test.ts index f36962ab1..0cfc9797c 100644 --- a/services/data-manager/__tests__/overture/overture-cron.test.ts +++ b/services/data-manager/__tests__/overture/overture-cron.test.ts @@ -2,386 +2,177 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { setupCron } from "../../src/cron.js"; -import { - createSingleFlightController, - type SingleFlightController, -} from "../../src/jobs/transitous/single-flight.js"; +import { type CronSetupOptions, setupCron } from "../../src/cron.js"; +import { createSingleFlightController } from "../../src/jobs/transitous/single-flight.js"; +import { dataReceipt, receiverFixture } from "../helpers/execution-receiver.js"; -function buildStubDb() { - let counter = 0; - const stub = { - select() { - return { - from(_table: unknown) { - return { - where(_predicate: unknown) { - return { - limit(_n: number) { - return Promise.resolve([]); - }, - }; - }, - }; - }, - }; - }, - insert(_table: unknown) { - return { - values(_v: unknown) { - const id = `job-${++counter}`; - return { - returning() { - return Promise.resolve([{ id }]); - }, - }; - }, - }; - }, - update() { - return { - set() { - return { - where() { - return Promise.resolve(); - }, - }; - }, - }; - }, - }; - return stub; -} - -function makeController(): SingleFlightController { - return createSingleFlightController({ db: buildStubDb() as never }); -} - -describe("Overture cron gating", () => { +describe("Overture cron durable ownership", () => { let dataDir: string; const started: ReturnType[] = []; - - /** - * `setupCron`, but the handles are also stopped in `afterEach`. croner keys - * its jobs by name globally, and a test that throws or times out before - * reaching its own `handles.stop()` leaves that name registered — so every - * later `setupCron` in this file dies with "name already taken", turning one - * failure into a cascade of unrelated ones. - */ - function startCron(options: Parameters[0]): ReturnType { - const handles = setupCron(options); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + function start(extra: Partial = {}) { + // This collaborator is never used to start data work: the receiver owns it. + const singleFlight = createSingleFlightController({ db: {} as never }); + const handles = setupCron({ + dataDir, + countries: [], + store: {} as never, + singleFlight, + logger, + syncCronExpression: "disabled", + feedProxyReloadCronExpression: "disabled", + stalenessCheckCronExpression: "disabled", + trafficExtractCronExpression: "disabled", + trafficLiveCronExpression: "disabled", + trafficPredictedCronExpression: "disabled", + overtureCronExpression: "disabled", + overtureConflationRetryCronExpression: "disabled", + executionReceiver: receiverFixture(), + discoverOvertureRelease: async () => "2026-07-22.0", + getInstalledOvertureRelease: async () => "2026-06-17.0", + runStalenessCheck: async () => {}, + githubIssueSink: null, + ...extra, + }); started.push(handles); return handles; } - beforeEach(() => { - dataDir = mkdtempSync(join(tmpdir(), "openmapx-overture-cron-")); + dataDir = mkdtempSync(join(tmpdir(), "omx-overture-owned-cron-")); + vi.clearAllMocks(); }); - afterEach(() => { - for (const handles of started.splice(0)) handles.stop(); + for (const h of started.splice(0)) h.stop(); rmSync(dataDir, { recursive: true, force: true }); + vi.unstubAllEnvs(); }); - - it("overtureCron is null when overtureEnabled is false (default)", () => { - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureCronExpression: "0 5 1 * *", - }); - expect(handles.overtureCron).toBeNull(); - handles.stop(); + it("leaves both schedules off when disabled", () => { + const h = start({ overtureEnabled: false }); + expect(h.overtureCron).toBeNull(); + expect(h.overtureConflationRetryCron).toBeNull(); }); - - it("overtureCron is null when overtureCronExpression is a disable sentinel", () => { - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "disabled", - }); - expect(handles.overtureCron).toBeNull(); - handles.stop(); + it("honors schedule disable sentinels", () => { + const h = start({ overtureEnabled: true }); + expect(h.overtureCron).toBeNull(); }); - - it("overtureCron is registered when overtureEnabled=true and valid cron expression", () => { - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "0 5 1 * *", - }); - expect(handles.overtureCron).not.toBeNull(); - handles.stop(); + it("registers the enabled release schedule", () => { + expect( + start({ overtureEnabled: true, overtureCronExpression: "0 5 1 * *" }).overtureCron, + ).not.toBeNull(); }); - - it("runOvertureNow is always present as a callable function", () => { - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - }); - expect(typeof handles.runOvertureNow).toBe("function"); - handles.stop(); + it("registers the enabled independent retry", () => { + expect( + start({ overtureEnabled: true, overtureConflationRetryCronExpression: "*/15 * * * *" }) + .overtureConflationRetryCron, + ).not.toBeNull(); }); - - it("runOvertureNow with overtureEnabled=true contains sync failures", async () => { - let overtureCalled = false; - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "0 5 1 * *", - runStalenessCheck: async () => {}, - runPipeline: async () => { - overtureCalled = true; - return { finalStatus: "done", results: [] } as never; - }, - // Every outbound seam is stubbed so the failure under test is the one we - // inject. Left to their defaults these reach the live Overture STAC - // catalogue and then the real regional pull, which is neither a fast nor - // a deterministic way to observe error containment. - discoverOvertureRelease: async () => "2026-07-22.0", - getInstalledOvertureRelease: async () => null, - syncOvertureRelease: async () => { - throw new Error("overture sync failed"); - }, - }); - expect(typeof handles.runOvertureNow).toBe("function"); - handles.stop(); - // The cron handler swallows sync errors, so this resolves rather than rejecting. - await expect(handles.runOvertureNow()).resolves.toBeUndefined(); - void overtureCalled; + it("uses the environment opt-in", () => { + vi.stubEnv("OVERTURE_ENABLED", "true"); + expect(start({ overtureCronExpression: "0 5 1 * *" }).overtureCron).not.toBeNull(); }); - - it("discovers and imports a newer release before updating feed state", async () => { - const calls: string[] = []; - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "disabled", - discoverOvertureRelease: async () => "2026-07-22.0", - getInstalledOvertureRelease: async () => "2026-06-17.0", - syncOvertureRelease: async ({ release }) => { - calls.push(`sync:${release}`); - return { - release: release ?? "2026-07-22.0", - path: "/tmp/region.parquet", - conflation: "waiting_for_osm", - linked: 0, - }; - }, - writeOvertureFeedState: async (_region, release) => { - calls.push(`state:${release}`); - }, - runStalenessCheck: async () => { - calls.push("staleness"); - }, + it("admits the actual typed sync once with the cron actor and observes its original identity", async () => { + const admitted = dataReceipt({ status: "running", publication: "unknown" }), + admitLocal = vi.fn(async () => admitted), + inspect = vi.fn(async () => dataReceipt()), + staleness = vi.fn(async () => {}), + cancel = vi.fn(); + const h = start({ + executionReceiver: receiverFixture({ admitLocal, inspect, requestCancel: cancel }), + runStalenessCheck: staleness, }); - - await handles.runOvertureNow(); - expect(calls).toEqual(["sync:2026-07-22.0", "state:2026-07-22.0", "staleness"]); - handles.stop(); + await h.runOvertureNow(); + expect(admitLocal).toHaveBeenCalledExactlyOnceWith( + { kind: "overture-sync", region: "europe/germany/berlin" }, + { triggeredBy: "cron" }, + ); + expect(inspect).toHaveBeenCalledExactlyOnceWith(admitted.id); + expect(staleness).toHaveBeenCalledOnce(); + expect(cancel).not.toHaveBeenCalled(); }); - - it("does not re-import when the latest release is installed but retries links", async () => { - const syncOvertureRelease = vi.fn(); - const writeOvertureFeedState = vi.fn(); - const rebuildOvertureLinks = vi.fn(async () => ({ - status: "already_completed" as const, - linked: 17, - })); - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "disabled", - discoverOvertureRelease: async () => "2026-07-22.0", - getInstalledOvertureRelease: async () => "2026-07-22.0", - syncOvertureRelease, - rebuildOvertureLinks, - writeOvertureFeedState, - // An `already_completed` rebuild goes on to finalize the release files. - // Left to its default that is the real implementation, walking the actual - // release directories — unrelated to what this test asserts, and slow - // enough to exhaust the timeout. - finalizeOvertureReleaseFiles: async () => ({ retained: ["2026-07-22.0"], removed: [] }), + it("keeps partial Places publication visible while the independent link stage is incomplete", async () => { + const staleness = vi.fn(async () => {}); + const h = start({ + executionReceiver: receiverFixture({ + admitLocal: async () => + dataReceipt({ + status: "partial", + result: { + kind: "overture-sync", + region: "europe/germany/berlin", + release: "2026-07-22.0", + metrics: { processed: 0 }, + }, + }), + }), + runStalenessCheck: staleness, }); - - await handles.runOvertureNow(); - expect(syncOvertureRelease).not.toHaveBeenCalled(); - expect(rebuildOvertureLinks).toHaveBeenCalledWith( - expect.objectContaining({ release: "2026-07-22.0" }), + await h.runOvertureNow(); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining("link rebuild remains incomplete"), + expect.objectContaining({ jobId: expect.any(String) }), ); - expect(writeOvertureFeedState).not.toHaveBeenCalled(); - handles.stop(); + expect(staleness).toHaveBeenCalledOnce(); }); - - it("retries conflation without release discovery or Places import", async () => { - const discoverOvertureRelease = vi.fn(async () => { - throw new Error("STAC unavailable"); - }); - const syncOvertureRelease = vi.fn(); - const finalizeOvertureReleaseFiles = vi.fn(async () => ({ - retained: ["2026-07-22.0"], - removed: [], - })); - const rebuildOvertureLinks = vi.fn(async () => ({ - status: "completed" as const, - linked: 23, - emitted: 32, - extracted: 30, - candidates: 25, - components: 20, - phaseDurationsMs: {}, - })); - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "disabled", - overtureConflationRetryCronExpression: "disabled", - discoverOvertureRelease, + it("does not re-import an installed release, and delegates link retry to its own durable worker", async () => { + const admitLocal = vi.fn(async () => dataReceipt({ kind: "overture-conflate" })); + const h = start({ getInstalledOvertureRelease: async () => "2026-07-22.0", - syncOvertureRelease, - rebuildOvertureLinks, - finalizeOvertureReleaseFiles, + executionReceiver: receiverFixture({ admitLocal }), }); - - await handles.runOvertureConflationRetryNow(); - expect(rebuildOvertureLinks).toHaveBeenCalledWith( - expect.objectContaining({ release: "2026-07-22.0" }), + await h.runOvertureNow(); + expect(admitLocal).toHaveBeenCalledExactlyOnceWith( + { kind: "overture-conflate", region: "europe/germany/berlin" }, + { triggeredBy: "cron" }, ); - expect(discoverOvertureRelease).not.toHaveBeenCalled(); - expect(syncOvertureRelease).not.toHaveBeenCalled(); - expect(finalizeOvertureReleaseFiles).toHaveBeenCalledWith( - expect.objectContaining({ activeRelease: "2026-07-22.0" }), + }); + it("does not downgrade an installed newer release", async () => { + const admitLocal = vi.fn(async () => dataReceipt({ kind: "overture-conflate" })); + const h = start({ + getInstalledOvertureRelease: async () => "2026-08-19.0", + executionReceiver: receiverFixture({ admitLocal }), + }); + await h.runOvertureNow(); + expect(admitLocal).toHaveBeenCalledExactlyOnceWith( + { kind: "overture-conflate", region: "europe/germany/berlin" }, + { triggeredBy: "cron" }, ); - handles.stop(); }); - - it("marks Places imported even when the link rebuild reports failure", async () => { - const calls: string[] = []; - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "disabled", - overtureConflationRetryCronExpression: "disabled", - discoverOvertureRelease: async () => "2026-07-22.0", - getInstalledOvertureRelease: async () => "2026-06-17.0", - syncOvertureRelease: async () => ({ - release: "2026-07-22.0", - path: "/data/release.parquet", - conflation: "failed", - linked: 0, - conflationError: "temporary failure", + it("retries links without catalogue discovery or a Places import", async () => { + const discover = vi.fn(async () => { + throw Error("STAC unavailable"); }), - writeOvertureFeedState: async () => { - calls.push("state"); - }, - runStalenessCheck: async () => { - calls.push("staleness"); - }, + admitLocal = vi.fn(async () => dataReceipt({ kind: "overture-conflate" })); + const h = start({ + discoverOvertureRelease: discover, + executionReceiver: receiverFixture({ admitLocal }), }); - - await handles.runOvertureNow(); - expect(calls).toEqual(["state", "staleness"]); - handles.stop(); + await h.runOvertureConflationRetryNow(); + expect(discover).not.toHaveBeenCalled(); + expect(admitLocal).toHaveBeenCalledExactlyOnceWith( + { kind: "overture-conflate", region: "europe/germany/berlin" }, + { triggeredBy: "cron" }, + ); }); - - it("does not mark a release imported when the sync fails", async () => { - const writeOvertureFeedState = vi.fn(); - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureEnabled: true, - overtureCronExpression: "disabled", - discoverOvertureRelease: async () => "2026-07-22.0", - getInstalledOvertureRelease: async () => "2026-06-17.0", - syncOvertureRelease: async () => { - throw new Error("import failed"); - }, - writeOvertureFeedState, - }); - - await handles.runOvertureNow(); - expect(writeOvertureFeedState).not.toHaveBeenCalled(); - handles.stop(); + it("skips link retry without an installed release", async () => { + const admitLocal = vi.fn(); + await start({ + getInstalledOvertureRelease: async () => null, + executionReceiver: receiverFixture({ admitLocal }), + }).runOvertureConflationRetryNow(); + expect(admitLocal).not.toHaveBeenCalled(); }); - - it("overtureCron is registered when OVERTURE_ENABLED env var is 'true'", () => { - const prev = process.env.OVERTURE_ENABLED; - try { - process.env.OVERTURE_ENABLED = "true"; - const handles = startCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - overtureCronExpression: "0 5 1 * *", + it("contains failed or unavailable admission without publication or a success check", async () => { + const staleness = vi.fn(async () => {}), + admitLocal = vi.fn(async () => { + throw Error("configuration_maintenance_busy"); }); - expect(handles.overtureCron).not.toBeNull(); - handles.stop(); - } finally { - if (prev === undefined) { - delete process.env.OVERTURE_ENABLED; - } else { - process.env.OVERTURE_ENABLED = prev; - } - } + const h = start({ + executionReceiver: receiverFixture({ admitLocal }), + runStalenessCheck: staleness, + }); + await expect(h.runOvertureNow()).resolves.toBeUndefined(); + expect(logger.error).toHaveBeenCalled(); + expect(staleness).not.toHaveBeenCalled(); + await expect(start({ executionReceiver: undefined }).runOvertureNow()).resolves.toBeUndefined(); }); }); diff --git a/services/data-manager/__tests__/reconcile.test.ts b/services/data-manager/__tests__/reconcile.test.ts index e52c44d95..4769fb817 100644 --- a/services/data-manager/__tests__/reconcile.test.ts +++ b/services/data-manager/__tests__/reconcile.test.ts @@ -14,15 +14,15 @@ import { */ function buildFakeDb(rows: Array<{ id: string }>): { handle: JobReconcileWriter; - captured: { set?: { status: string; finishedAt: Date }; whereCalled: boolean }; + captured: { set?: { status: string; finishedAt: Date | null }; whereCalled: boolean }; } { - const captured: { set?: { status: string; finishedAt: Date }; whereCalled: boolean } = { + const captured: { set?: { status: string; finishedAt: Date | null }; whereCalled: boolean } = { whereCalled: false, }; const handle = { update(_table: unknown) { return { - set(values: { status: string; finishedAt: Date }) { + set(values: { status: string; finishedAt: Date | null }) { captured.set = values; return { where(_predicate: unknown) { @@ -42,7 +42,7 @@ function buildFakeDb(rows: Array<{ id: string }>): { } describe("reconcileOrphanedJobs", () => { - it("stamps running jobs as interrupted with a finish time and returns their ids", async () => { + it("marks orphaned legacy jobs interrupted without inventing a containment time", async () => { const now = new Date("2026-06-27T10:00:00.000Z"); const { handle, captured } = buildFakeDb([{ id: "job-a" }, { id: "job-b" }]); @@ -50,7 +50,7 @@ describe("reconcileOrphanedJobs", () => { expect(ids).toEqual(["job-a", "job-b"]); expect(captured.whereCalled).toBe(true); - expect(captured.set).toEqual({ status: INTERRUPTED_STATUS, finishedAt: now }); + expect(captured.set).toEqual({ status: INTERRUPTED_STATUS, finishedAt: null }); expect(INTERRUPTED_STATUS).toBe("interrupted"); }); diff --git a/services/data-manager/__tests__/transitous/motis-import.test.ts b/services/data-manager/__tests__/transitous/motis-import.test.ts index 6f180f099..771606687 100644 --- a/services/data-manager/__tests__/transitous/motis-import.test.ts +++ b/services/data-manager/__tests__/transitous/motis-import.test.ts @@ -1,3 +1,10 @@ +vi.mock("../../src/jobs/transitous/owned-ops.js", async () => { + const original = await vi.importActual( + "../../src/jobs/transitous/owned-ops.js", + ); + return { ...original, ...(await import("../helpers/owned-transit-unit.js")) }; +}); + import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; diff --git a/services/data-manager/__tests__/transitous/pipeline.test.ts b/services/data-manager/__tests__/transitous/pipeline.test.ts index af0ec3420..5343c82a4 100644 --- a/services/data-manager/__tests__/transitous/pipeline.test.ts +++ b/services/data-manager/__tests__/transitous/pipeline.test.ts @@ -59,6 +59,55 @@ const ORDERED_STAGES: StageName[] = [ ]; describe("runTransitousPipeline orchestrator", () => { + it("stops at a failed owner checkpoint before the next stage can mutate", async () => { + tmp = mkdtempSync(join(tmpdir(), "openmapx-pipeline-owner-")); + mkdirSync(join(tmp, ".transitous-catalog", ".git"), { recursive: true }); + mkdirSync(join(tmp, ".transitous-catalog", "feeds"), { recursive: true }); + writeFileSync( + join(tmp, ".transitous-catalog", "feeds", "de.json"), + JSON.stringify({ sources: [{ name: "fixture" }] }), + ); + const stages: StageResult[] = []; + const ctx = buildJobContext({ + dataDir: tmp, + store: new StateStore(tmp), + countries: ["de"], + runner: async () => {}, + runScript: async () => {}, + onStageComplete: async (result) => { + stages.push(result); + throw Error("durable execution owner checkpoint lost"); + }, + }); + await expect(runTransitousPipeline(ctx, { stopAt: "filter" })).rejects.toThrow( + "durable execution owner checkpoint lost", + ); + expect(stages.map((result) => result.stage)).toEqual(["prepare"]); + }); + it("never reports ok when cancellation follows a successful stage", async () => { + tmp = mkdtempSync(join(tmpdir(), "openmapx-pipeline-cancel-")); + const catalogDir = join(tmp, ".transitous-catalog"); + mkdirSync(join(catalogDir, ".git"), { recursive: true }); + const controller = new AbortController(); + const persisted: StageResult[] = []; + const ctx = buildJobContext({ + dataDir: tmp, + store: new StateStore(tmp), + countries: ["de"], + runner: async () => {}, + abortSignal: controller.signal, + onStageComplete: async (result) => { + persisted.push(result); + controller.abort(new Error("operator canceled transit execution")); + }, + }); + + await expect(runTransitousPipeline(ctx)).rejects.toThrow(/cancel|abort/i); + expect(persisted.map(({ stage, status }) => ({ stage, status }))).toEqual([ + { stage: "prepare", status: "ok" }, + ]); + }); + it("invokes all stages in order against an in-memory persistence hook", async () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-pipeline-orchestrator-")); const dataDir = tmp; diff --git a/services/data-manager/__tests__/transitous/promote.test.ts b/services/data-manager/__tests__/transitous/promote.test.ts index 699ffd797..82fc0d7b1 100644 --- a/services/data-manager/__tests__/transitous/promote.test.ts +++ b/services/data-manager/__tests__/transitous/promote.test.ts @@ -1,3 +1,10 @@ +vi.mock("../../src/jobs/transitous/owned-ops.js", async () => { + const original = await vi.importActual( + "../../src/jobs/transitous/owned-ops.js", + ); + return { ...original, ...(await import("../helpers/owned-transit-unit.js")) }; +}); + import { chmodSync, closeSync, @@ -434,117 +441,26 @@ describe("promote stage", () => { }, ); - it("bounds nonterminal old-primary recovery after job cancellation with real status polling", async () => { - const { fx, ctx } = readyFixture(layout); - const actual = - await vi.importActual("../../src/ops-client.js"); - const tokenFile = join(fx.dataDir, "ops-test-token"); - writeFileSync(tokenFile, "a".repeat(43)); - const unchanged = directorySnapshot(fx.dataDir); - vi.stubEnv("OPS_AGENT_URL", "http://ops-agent:4300"); - vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); - actual.resetOpsClient(); - const cancelled = new AbortController(); - ctx.abortSignal = cancelled.signal; - const cleanup = new AbortController(); - const nativeTimeout = AbortSignal.timeout.bind(AbortSignal); - const timeout = vi - .spyOn(AbortSignal, "timeout") - .mockImplementation((ms) => nativeTimeout(ms > 120_000 ? 100 : ms)); - let polls = 0; - let deletes = 0; - let admitted = false; - let operationKey = ""; - globalThis.fetch = vi.fn(async (input: unknown, init?: RequestInit) => { - const url = String(input); - if (!url.startsWith("http://ops-agent:4300/")) return jsonResponse(successfulBody(url)); - init?.signal?.throwIfAborted(); - if (init?.method === "DELETE") deletes++; - const operationId = "job1_pendingRecovery000"; - let requestId: string | null; - let result: unknown; - if (init?.method === "POST") { - const request = JSON.parse(String(init.body)); - requestId = request.requestId; - operationKey = request.operationKey; - admitted = true; - result = { - execution: "async", - operationId, - operationKey, - kind: "motis.primary.restart", - state: "running", - }; - } else { - polls++; - requestId = new Headers(init?.headers).get("x-ops-request-id"); - result = { - version: 1, - operationId, - operationKey, - kind: "motis.primary.restart", - resourceId: "motis", - state: "termination_pending", - submittedAt: "2026-05-01T00:00:00.000Z", - updatedAt: "2026-05-01T00:00:01.000Z", - }; - } - return jsonResponse({ version: 1, requestId, ok: true, result }); - }) as typeof fetch; - vi.mocked(runOpsOperation).mockImplementation(async (operation, options) => { + it("does not start old-primary recovery after job cancellation", async () => { + const { fx, ctx } = readyFixture(layout), + before = directorySnapshot(fx.dataDir), + controller = new AbortController(); + ctx.abortSignal = controller.signal; + vi.mocked(runOpsOperation).mockImplementation(async (operation) => { opsCalls.push(operation); if (operation.kind === "motis.staging.stop") { - cancelled.abort(); - throw new Error("staging stop timed out"); - } - if (operation.kind === "motis.primary.restart") { - return actual.runOpsOperation(operation, { - ...options, - signal: options?.signal ?? cleanup.signal, - pollIntervalMs: 10, - }); + controller.abort(); + throw Error("staging stop timed out"); } return { changed: true, state: "stopped" }; }); - const pending = promoteRun(ctx); - let timer: ReturnType | undefined; - try { - // The shortened 100 ms recovery deadline is what must end promotion; - // this watchdog only turns a regression into a failure instead of a - // hang, so it leaves room for a saturated runner. - const result = await Promise.race([ - pending, - new Promise((resolve) => { - timer = setTimeout(() => resolve(undefined), 10_000); - }), - ]); - expect(cancelled.signal.aborted).toBe(true); - expect(admitted).toBe(true); - expect(polls).toBeGreaterThan(1); - expect( - result, - "nonterminal recovery must not keep promotion pending indefinitely", - ).toBeDefined(); - expect(result?.status).toBe("error"); - expect(result?.message).toContain("staging stop timed out"); - expect(result?.message).toMatch(/recovery completion unknown.*deadline/i); - expect(timeout).toHaveBeenCalledWith(125_000); - expect(deletes).toBe(0); - expect(directorySnapshot(fx.dataDir)).toEqual(unchanged); - expect(opsCalls.map(({ kind }) => kind)).toEqual([ - "motis.primary.stop", - "motis.staging.stop", - "motis.primary.restart", - ]); - expect(recordPromotedSource).not.toHaveBeenCalled(); - } finally { - if (timer) clearTimeout(timer); - cleanup.abort(); - await pending; - timeout.mockRestore(); - actual.resetOpsClient(); - vi.unstubAllEnvs(); - } + await expect(promoteRun(ctx)).rejects.toThrow(/abort/i); + expect(directorySnapshot(fx.dataDir)).toEqual(before); + expect(opsCalls.map((call) => call.kind)).toEqual([ + "motis.primary.stop", + "motis.staging.stop", + ]); + expect(recordPromotedSource).not.toHaveBeenCalled(); }); it("refuses promotion when the data target changes despite confirmed stops", async () => { @@ -863,8 +779,9 @@ describe("promote stage", () => { opsBehaviour.failPromote = false; }); expect(result.status).toBe("error"); - // initial promote + rollback promote - expect(opsCalls.filter((call) => call.kind === "motis.primary.promote")).toHaveLength(2); + // The retained recovery phase restarts the original primary, never replays promotion. + expect(opsCalls.filter((call) => call.kind === "motis.primary.promote")).toHaveLength(1); + expect(opsCalls.filter((call) => call.kind === "motis.primary.restart")).toHaveLength(1); // After successful rollback the previous data is back in `currentDir`, // and staging holds the (now-stale) new data. expect(existsSync(fx.currentDir)).toBe(true); @@ -902,7 +819,8 @@ describe("promote stage", () => { expect(result.status).toBe("error"); expect(result.message).toMatch(/slot B restart failed.*rollback ok/); - expect(promoteCalls).toBe(2); + expect(promoteCalls).toBe(1); + expect(opsCalls.filter((call) => call.kind === "motis.primary.restart")).toHaveLength(1); expect(aliasSlot(ctx.slotLayout, "live")).toBe("A"); expect(aliasSlot(ctx.slotLayout, "staging")).toBe("B"); expect(ctx.slotLayout.record.activeSlot).toBe("A"); diff --git a/services/data-manager/__tests__/transitous/proxy-transaction.test.ts b/services/data-manager/__tests__/transitous/proxy-transaction.test.ts index 82b99d190..2ab4d87fd 100644 --- a/services/data-manager/__tests__/transitous/proxy-transaction.test.ts +++ b/services/data-manager/__tests__/transitous/proxy-transaction.test.ts @@ -1,3 +1,10 @@ +vi.mock("../../src/jobs/transitous/owned-ops.js", async () => { + const original = await vi.importActual( + "../../src/jobs/transitous/owned-ops.js", + ); + return { ...original, ...(await import("../helpers/owned-transit-unit.js")) }; +}); + import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 6b085041d..e729212c2 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -76,6 +76,9427 @@ } }, "paths": { + "/executions": { + "post": { + "operationId": "startDataExecution", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "input": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "restart": { + "type": "boolean" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + } + }, + "required": [ + "kind", + "region" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + } + ] + }, + "requestedDeadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "operationKey", + "input" + ], + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "startDataExecution", + "x-openmapx-auth": "service", + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/executions/generation-hardlinks": { + "post": { + "operationId": "startGenerationHardlinksExecution", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "requestedDeadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "input": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "planDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "coordinatorMode": { + "type": "string", + "enum": [ + "ensure-running", + "require-running" + ] + } + }, + "required": [ + "kind", + "revisionId", + "planDigest", + "coordinatorMode" + ], + "additionalProperties": false + }, + "plan": { + "type": "object", + "properties": { + "plan": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "target": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "consumerService": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "dataType": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "instance": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "targetFilename": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "source", + "target", + "consumerService", + "dataType" + ], + "additionalProperties": false + } + }, + "prune": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "plan", + "prune" + ], + "additionalProperties": false + } + }, + "required": [ + "operationKey", + "requestedDeadlineAt", + "input", + "plan" + ], + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "startGenerationHardlinksExecution", + "x-openmapx-auth": "service", + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/executions/configuration-maintenance/prepare": { + "post": { + "operationId": "prepareDataConfigurationFence", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maintenanceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "candidateId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "ownerPrepare": { + "type": "object", + "properties": { + "executor": { + "type": "string", + "enum": [ + "ops-agent", + "data-manager", + "host-cli" + ] + }, + "principal": { + "type": "string", + "enum": [ + "api", + "data-manager", + "host-cli" + ] + }, + "executionId": { + "type": "string" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "operationFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "executor", + "principal", + "executionId", + "operationKey", + "operationFingerprint", + "fingerprint", + "deadlineAt" + ], + "additionalProperties": false + }, + "planDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "hardlinkPlanDigest": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "serviceIds": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + } + }, + "coordinatorMode": { + "type": "string", + "enum": [ + "ensure-running", + "require-running" + ] + } + }, + "required": [ + "maintenanceId", + "candidateId", + "ownerPrepare", + "planDigest", + "revisionId", + "hardlinkPlanDigest", + "serviceIds", + "coordinatorMode" + ], + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "prepareDataConfigurationFence", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maintenanceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "candidateId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "ownerPrepare": { + "type": "object", + "properties": { + "executor": { + "type": "string", + "enum": [ + "ops-agent", + "data-manager", + "host-cli" + ] + }, + "principal": { + "type": "string", + "enum": [ + "api", + "data-manager", + "host-cli" + ] + }, + "executionId": { + "type": "string" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "operationFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "executor", + "principal", + "executionId", + "operationKey", + "operationFingerprint", + "fingerprint", + "deadlineAt" + ], + "additionalProperties": false + }, + "planDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "hardlinkPlanDigest": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "fenceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "state": { + "type": "string", + "enum": [ + "prepared", + "settled", + "recovery_required" + ] + } + }, + "required": [ + "maintenanceId", + "candidateId", + "ownerPrepare", + "planDigest", + "revisionId", + "hardlinkPlanDigest", + "fenceId", + "sequence", + "state" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/executions/configuration-maintenance/{maintenanceId}": { + "get": { + "operationId": "inspectDataConfigurationFence", + "tags": [ + "data-manager" + ], + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "maintenanceId", + "required": true + } + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "inspectDataConfigurationFence", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maintenanceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "candidateId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "ownerPrepare": { + "type": "object", + "properties": { + "executor": { + "type": "string", + "enum": [ + "ops-agent", + "data-manager", + "host-cli" + ] + }, + "principal": { + "type": "string", + "enum": [ + "api", + "data-manager", + "host-cli" + ] + }, + "executionId": { + "type": "string" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "operationFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "executor", + "principal", + "executionId", + "operationKey", + "operationFingerprint", + "fingerprint", + "deadlineAt" + ], + "additionalProperties": false + }, + "planDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "hardlinkPlanDigest": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "fenceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "state": { + "type": "string", + "enum": [ + "prepared", + "settled", + "recovery_required" + ] + } + }, + "required": [ + "maintenanceId", + "candidateId", + "ownerPrepare", + "planDigest", + "revisionId", + "hardlinkPlanDigest", + "fenceId", + "sequence", + "state" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/executions/configuration-maintenance/{maintenanceId}/settle": { + "post": { + "operationId": "settleDataConfigurationFence", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maintenanceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "candidateId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "planDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "decisionId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedSequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outcome": { + "type": "string", + "enum": [ + "unused", + "applied", + "contained" + ] + }, + "ownerReceiptDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "maintenanceId", + "candidateId", + "planDigest", + "decisionId", + "expectedSequence", + "outcome", + "ownerReceiptDigest" + ], + "additionalProperties": false + } + } + } + }, + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "maintenanceId", + "required": true + } + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "settleDataConfigurationFence", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maintenanceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "candidateId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "ownerPrepare": { + "type": "object", + "properties": { + "executor": { + "type": "string", + "enum": [ + "ops-agent", + "data-manager", + "host-cli" + ] + }, + "principal": { + "type": "string", + "enum": [ + "api", + "data-manager", + "host-cli" + ] + }, + "executionId": { + "type": "string" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "operationFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "executor", + "principal", + "executionId", + "operationKey", + "operationFingerprint", + "fingerprint", + "deadlineAt" + ], + "additionalProperties": false + }, + "planDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "hardlinkPlanDigest": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "fenceId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "state": { + "type": "string", + "enum": [ + "prepared", + "settled", + "recovery_required" + ] + } + }, + "required": [ + "maintenanceId", + "candidateId", + "ownerPrepare", + "planDigest", + "revisionId", + "hardlinkPlanDigest", + "fenceId", + "sequence", + "state" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/executions/by-key/{operationKey}": { + "get": { + "operationId": "lookupDataExecution", + "tags": [ + "data-manager" + ], + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "in": "path", + "name": "operationKey", + "required": true + } + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "lookupDataExecution", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/executions/{id}": { + "get": { + "operationId": "getDataExecution", + "tags": [ + "data-manager" + ], + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "id", + "required": true + } + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "getDataExecution", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/executions/{id}/cancel": { + "post": { + "operationId": "cancelDataExecution", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "operationKey", + "fingerprint" + ], + "additionalProperties": false + } + } + } + }, + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "id", + "required": true + } + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "cancelDataExecution", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, "/ambient-places/status": { "get": { "operationId": "getAmbientPlacesStatus", @@ -842,17 +10263,288 @@ } } }, - "429": { + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/ambient-places/build": { + "post": { + "operationId": "buildAmbientPlaces", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "bounds": { + "type": "array", + "items": false, + "minItems": 4, + "maxItems": 4, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "coverage": { + "type": "string", + "enum": [ + "germany", + "planet" + ] + } + }, + "required": [ + "name", + "bounds" + ], + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "buildAmbientPlaces", + "x-openmapx-auth": "service", + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "accepted": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "accepted" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -911,7 +10603,7 @@ } } }, - "500": { + "409": { "description": "Response", "content": { "application/json": { @@ -970,99 +10662,8 @@ } } } - } - } - } - }, - "/ambient-places/build": { - "post": { - "operationId": "buildAmbientPlaces", - "tags": [ - "data-manager" - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "maxLength": 80 - }, - "bounds": { - "type": "array", - "items": false, - "minItems": 4, - "maxItems": 4, - "prefixItems": [ - { - "type": "number" - }, - { - "type": "number" - }, - { - "type": "number" - }, - { - "type": "number" - } - ] - }, - "coverage": { - "type": "string", - "enum": [ - "germany", - "planet" - ] - } - }, - "required": [ - "name", - "bounds" - ], - "additionalProperties": false - } - } - } - }, - "security": [ - { - "serviceBearer": [] }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "buildAmbientPlaces", - "x-openmapx-auth": "service", - "responses": { - "202": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "accepted": { - "type": "boolean", - "enum": [ - true - ] - } - }, - "required": [ - "accepted" - ], - "additionalProperties": false - } - } - } - }, - "400": { + "413": { "description": "Response", "content": { "application/json": { @@ -1122,7 +10723,7 @@ } } }, - "401": { + "415": { "description": "Response", "content": { "application/json": { @@ -1182,8 +10783,17 @@ } } }, - "409": { + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -1242,7 +10852,7 @@ } } }, - "413": { + "500": { "description": "Response", "content": { "application/json": { @@ -1302,7 +10912,7 @@ } } }, - "415": { + "503": { "description": "Response", "content": { "application/json": { @@ -1361,18 +10971,71 @@ } } } + } + } + } + }, + "/ambient-places/resume": { + "post": { + "operationId": "resumeAmbientPlaces", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "generation": { + "type": "string", + "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" + } + }, + "required": [ + "generation" + ], + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] }, - "429": { + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "resumeAmbientPlaces", + "x-openmapx-auth": "service", + "responses": { + "202": { "description": "Response", - "headers": { - "retry-after": { + "content": { + "application/json": { "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" + "type": "object", + "properties": { + "accepted": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "accepted" + ], + "additionalProperties": false + } } - }, + } + }, + "400": { + "description": "Response", "content": { "application/json": { "schema": { @@ -1431,7 +11094,7 @@ } } }, - "500": { + "401": { "description": "Response", "content": { "application/json": { @@ -1490,70 +11153,8 @@ } } } - } - } - } - }, - "/ambient-places/resume": { - "post": { - "operationId": "resumeAmbientPlaces", - "tags": [ - "data-manager" - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "generation": { - "type": "string", - "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" - } - }, - "required": [ - "generation" - ], - "additionalProperties": false - } - } - } - }, - "security": [ - { - "serviceBearer": [] }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "resumeAmbientPlaces", - "x-openmapx-auth": "service", - "responses": { - "202": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "accepted": { - "type": "boolean", - "enum": [ - true - ] - } - }, - "required": [ - "accepted" - ], - "additionalProperties": false - } - } - } - }, - "400": { + "409": { "description": "Response", "content": { "application/json": { @@ -1613,7 +11214,7 @@ } } }, - "401": { + "413": { "description": "Response", "content": { "application/json": { @@ -1673,7 +11274,7 @@ } } }, - "409": { + "415": { "description": "Response", "content": { "application/json": { @@ -1733,68 +11334,17 @@ } } }, - "413": { + "429": { "description": "Response", - "content": { - "application/json": { + "headers": { + "retry-after": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" } - } - }, - "415": { - "description": "Response", + }, "content": { "application/json": { "schema": { @@ -1853,17 +11403,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -1922,7 +11463,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -2472,6 +12013,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -2714,68 +12315,137 @@ } } }, - "413": { + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { "description": "Response", - "content": { - "application/json": { + "headers": { + "retry-after": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" } - } - }, - "415": { - "description": "Response", + }, "content": { "application/json": { "schema": { @@ -2834,17 +12504,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -2903,7 +12564,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -3433,6 +13094,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -10775,6 +20496,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -10835,8 +20616,77 @@ } } }, - "415": { + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -10895,17 +20745,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -10964,7 +20805,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -11460,6 +21301,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -11708,6 +21609,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -11983,6 +21944,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -12043,8 +22064,77 @@ } } }, - "415": { + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -12103,17 +22193,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -12172,7 +22253,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -12630,6 +22711,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -12878,6 +23019,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -13429,6 +23630,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -13703,6 +23964,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -14754,39 +25075,557 @@ }, "url": { "type": "string" - }, - "sizeBytes": { - "type": "number", - "minimum": 0 - }, - "downloadedAt": { - "type": "string", - "minLength": 1 - }, - "sha256": { + }, + "sizeBytes": { + "type": "number", + "minimum": 0 + }, + "downloadedAt": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string" + }, + "md5": { + "type": "string" + }, + "path": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "type", + "id", + "sizeBytes", + "downloadedAt", + "path" + ], + "additionalProperties": false + } + } + }, + "required": [ + "datasets" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/transit/profile": { + "get": { + "operationId": "getTransitProfile", + "tags": [ + "data-manager" + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "getTransitProfile", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "policy": { + "type": "object", + "properties": { + "profile": { + "type": "string", + "enum": [ + "regional-assisted", + "regional-sovereign", + "planet" + ] + }, + "countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "feedAllowList": { + "type": "array", + "items": { "type": "string" - }, - "md5": { + } + }, + "acquisition": { + "type": "string", + "enum": [ + "mirror", + "build" + ] + }, + "artifactOrigins": { + "type": "array", + "items": { "type": "string" - }, - "path": { - "type": "string", - "minLength": 1 } }, - "required": [ - "type", - "id", - "sizeBytes", - "downloadedAt", - "path" - ], - "additionalProperties": false - } + "originDownloadsRequired": { + "type": "boolean" + }, + "hostedRuntimeFallbackAllowed": { + "type": "boolean" + }, + "osmInput": { + "type": "string" + }, + "gbfsSelection": { + "type": "string", + "enum": [ + "explicit-countries", + "explicit-feeds", + "allow-listed-batches" + ] + }, + "maxFeedCount": { + "type": "number", + "minimum": 0 + }, + "retentionGenerations": { + "type": "number", + "minimum": 0 + }, + "resourceEnvelope": { + "type": "object", + "properties": { + "minimumMemoryGb": { + "type": "number", + "minimum": 0 + }, + "minimumCpu": { + "type": "number", + "minimum": 0 + }, + "minimumDiskGb": { + "type": "number", + "minimum": 0 + }, + "operatorCapacityRequired": { + "type": "boolean" + } + }, + "required": [ + "minimumMemoryGb", + "minimumCpu", + "minimumDiskGb", + "operatorCapacityRequired" + ], + "additionalProperties": false + }, + "updateCadenceHours": { + "type": "number", + "minimum": 0 + }, + "experimental": { + "type": "boolean" + } + }, + "required": [ + "profile", + "countries", + "feedAllowList", + "acquisition", + "artifactOrigins", + "originDownloadsRequired", + "hostedRuntimeFallbackAllowed", + "gbfsSelection", + "maxFeedCount", + "retentionGenerations", + "resourceEnvelope", + "updateCadenceHours", + "experimental" + ], + "additionalProperties": false } }, "required": [ - "datasets" + "policy" ], "additionalProperties": false } @@ -15165,12 +26004,67 @@ } } }, - "/transit/profile": { + "/transit/sources": { "get": { - "operationId": "getTransitProfile", + "operationId": "getTransitSources", "tags": [ "data-manager" ], + "parameters": [ + { + "schema": { + "type": "string" + }, + "in": "query", + "name": "search", + "required": false + }, + { + "schema": { + "type": "string", + "enum": [ + "active", + "add-pending", + "update-pending", + "removal-pending", + "disabled", + "failed", + "stale" + ] + }, + "in": "query", + "name": "lifecycle", + "required": false + }, + { + "schema": { + "type": "string", + "enum": [ + "catalog", + "operator" + ] + }, + "in": "query", + "name": "origin", + "required": false + }, + { + "schema": { + "type": "number" + }, + "in": "query", + "name": "limit", + "required": false + }, + { + "schema": { + "type": "number" + }, + "in": "query", + "name": "offset", + "required": false + } + ], "security": [ { "serviceBearer": [] @@ -15179,7 +26073,7 @@ "serviceToken": [] } ], - "x-openmapx-contract-id": "getTransitProfile", + "x-openmapx-contract-id": "getTransitSources", "x-openmapx-auth": "service", "responses": { "200": { @@ -15189,129 +26083,267 @@ "schema": { "type": "object", "properties": { - "policy": { - "type": "object", - "properties": { - "profile": { - "type": "string", - "enum": [ - "regional-assisted", - "regional-sovereign", - "planet" - ] - }, - "countries": { - "type": "array", - "items": { + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "string" - } - }, - "feedAllowList": { - "type": "array", - "items": { + }, + "region": { + "type": "string" + }, + "name": { + "type": "string" + }, + "format": { + "type": "string", + "enum": [ + "gtfs", + "netex" + ] + }, + "origin": { + "type": "string", + "enum": [ + "catalog", + "operator" + ] + }, + "originUrl": { + "type": "string" + }, + "license": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DataManager_getTransitSources_response_200___schema0" + } + }, + "requested": { + "type": "boolean" + }, + "active": { + "type": "boolean" + }, + "activeEpoch": { + "type": "string" + }, + "artifact": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "sizeBytes": { + "type": "number", + "minimum": 0 + }, + "retrievedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "path", + "sha256", + "sizeBytes", + "retrievedAt" + ], + "additionalProperties": false + }, + "lastFetchedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lastImportedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "validationStatus": { + "type": "string" + }, + "validationMessage": { "type": "string" + }, + "lifecycle": { + "type": "string", + "enum": [ + "active", + "add-pending", + "update-pending", + "removal-pending", + "disabled", + "failed", + "stale" + ] } }, - "acquisition": { - "type": "string", + "required": [ + "id", + "region", + "name", + "format", + "origin", + "license", + "requested", + "active", + "lifecycle" + ], + "additionalProperties": false + } + }, + "total": { + "type": "number", + "minimum": 0 + }, + "limit": { + "type": "number", + "minimum": 0 + }, + "offset": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "sources", + "total", + "limit", + "offset" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", "enum": [ - "mirror", - "build" + false ] }, - "artifactOrigins": { - "type": "array", - "items": { - "type": "string" - } + "message": { + "type": "string" }, - "originDownloadsRequired": { - "type": "boolean" + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, - "hostedRuntimeFallbackAllowed": { - "type": "boolean" + "error": { + "type": "string" }, - "osmInput": { + "message": { "type": "string" }, - "gbfsSelection": { - "type": "string", + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", "enum": [ - "explicit-countries", - "explicit-feeds", - "allow-listed-batches" + false ] }, - "maxFeedCount": { - "type": "number", - "minimum": 0 + "message": { + "type": "string" }, - "retentionGenerations": { - "type": "number", - "minimum": 0 + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, - "resourceEnvelope": { - "type": "object", - "properties": { - "minimumMemoryGb": { - "type": "number", - "minimum": 0 - }, - "minimumCpu": { - "type": "number", - "minimum": 0 - }, - "minimumDiskGb": { - "type": "number", - "minimum": 0 - }, - "operatorCapacityRequired": { - "type": "boolean" - } - }, - "required": [ - "minimumMemoryGb", - "minimumCpu", - "minimumDiskGb", - "operatorCapacityRequired" - ], - "additionalProperties": false + "error": { + "type": "string" }, - "updateCadenceHours": { - "type": "number", - "minimum": 0 + "message": { + "type": "string" }, - "experimental": { - "type": "boolean" + "code": { + "type": "string" } }, "required": [ - "profile", - "countries", - "feedAllowList", - "acquisition", - "artifactOrigins", - "originDownloadsRequired", - "hostedRuntimeFallbackAllowed", - "gbfsSelection", - "maxFeedCount", - "retentionGenerations", - "resourceEnvelope", - "updateCadenceHours", - "experimental" + "statusCode", + "error", + "message" ], "additionalProperties": false } - }, - "required": [ - "policy" - ], - "additionalProperties": false + ] } } } }, - "400": { + "404": { "description": "Response", "content": { "application/json": { @@ -15371,7 +26403,7 @@ } } }, - "401": { + "409": { "description": "Response", "content": { "application/json": { @@ -15681,69 +26713,76 @@ } } } - } - }, - "/transit/sources": { - "get": { - "operationId": "getTransitSources", + }, + "post": { + "operationId": "addTransitSource", "tags": [ "data-manager" ], - "parameters": [ - { - "schema": { - "type": "string" - }, - "in": "query", - "name": "search", - "required": false - }, - { - "schema": { - "type": "string", - "enum": [ - "active", - "add-pending", - "update-pending", - "removal-pending", - "disabled", - "failed", - "stale" - ] - }, - "in": "query", - "name": "lifecycle", - "required": false - }, - { - "schema": { - "type": "string", - "enum": [ - "catalog", - "operator" - ] - }, - "in": "query", - "name": "origin", - "required": false - }, - { - "schema": { - "type": "number" - }, - "in": "query", - "name": "limit", - "required": false - }, - { - "schema": { - "type": "number" - }, - "in": "query", - "name": "offset", - "required": false + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "triggeredBy": { + "type": "string", + "maxLength": 200 + }, + "region": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri" + }, + "license": { + "type": "object", + "properties": { + "spdxIdentifier": { + "type": "string" + }, + "url": { + "type": "string" + }, + "attribution": { + "type": "string", + "minLength": 1 + }, + "publisher": { + "type": "string" + }, + "publisherUrl": { + "type": "string" + } + }, + "required": [ + "attribution" + ], + "additionalProperties": false + } + }, + "required": [ + "region", + "name", + "url", + "license" + ], + "additionalProperties": false + } + } } - ], + }, "security": [ { "serviceBearer": [] @@ -15752,150 +26791,33 @@ "serviceToken": [] } ], - "x-openmapx-contract-id": "getTransitSources", + "x-openmapx-contract-id": "addTransitSource", "x-openmapx-auth": "service", "responses": { - "200": { + "202": { "description": "Response", "content": { "application/json": { "schema": { "type": "object", "properties": { - "sources": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "region": { - "type": "string" - }, - "name": { - "type": "string" - }, - "format": { - "type": "string", - "enum": [ - "gtfs", - "netex" - ] - }, - "origin": { - "type": "string", - "enum": [ - "catalog", - "operator" - ] - }, - "originUrl": { - "type": "string" - }, - "license": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/DataManager_getTransitSources_response_200___schema0" - } - }, - "requested": { - "type": "boolean" - }, - "active": { - "type": "boolean" - }, - "activeEpoch": { - "type": "string" - }, - "artifact": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "sha256": { - "type": "string" - }, - "sizeBytes": { - "type": "number", - "minimum": 0 - }, - "retrievedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - } - }, - "required": [ - "path", - "sha256", - "sizeBytes", - "retrievedAt" - ], - "additionalProperties": false - }, - "lastFetchedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "lastImportedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "validationStatus": { - "type": "string" - }, - "validationMessage": { - "type": "string" - }, - "lifecycle": { - "type": "string", - "enum": [ - "active", - "add-pending", - "update-pending", - "removal-pending", - "disabled", - "failed", - "stale" - ] - } - }, - "required": [ - "id", - "region", - "name", - "format", - "origin", - "license", - "requested", - "active", - "lifecycle" - ], - "additionalProperties": false - } - }, - "total": { - "type": "number", - "minimum": 0 + "jobId": { + "type": "string" }, - "limit": { - "type": "number", - "minimum": 0 + "sourceId": { + "type": "string" }, - "offset": { - "type": "number", - "minimum": 0 + "status": { + "type": "string", + "enum": [ + "started" + ] } }, "required": [ - "sources", - "total", - "limit", - "offset" + "jobId", + "sourceId", + "status" ], "additionalProperties": false } @@ -16022,7 +26944,221 @@ } } }, - "404": { + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] + }, + "existingJobId": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "ok", + "reason", + "existingJobId" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { "description": "Response", "content": { "application/json": { @@ -16082,8 +27218,17 @@ } } }, - "409": { + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -16142,7 +27287,7 @@ } } }, - "413": { + "500": { "description": "Response", "content": { "application/json": { @@ -16202,7 +27347,7 @@ } } }, - "415": { + "503": { "description": "Response", "content": { "application/json": { @@ -16261,77 +27406,85 @@ } } } + } + } + } + }, + "/transit/catalog": { + "get": { + "operationId": "getTransitCatalog", + "tags": [ + "data-manager" + ], + "security": [ + { + "serviceBearer": [] }, - "429": { + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "getTransitCatalog", + "x-openmapx-auth": "service", + "responses": { + "200": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" + "type": "object", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "region": { + "type": "string" + }, + "name": { + "type": "string" + }, + "format": { + "type": "string", + "enum": [ + "gtfs", + "netex" + ] + }, + "originUrl": { + "type": "string" + }, + "license": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DataManager_getTransitCatalog_response_200___schema0" + } + } }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false + "required": [ + "id", + "region", + "name", + "format", + "license" + ], + "additionalProperties": false + } } - ] + }, + "required": [ + "sources" + ], + "additionalProperties": false } } } }, - "500": { + "400": { "description": "Response", "content": { "application/json": { @@ -16390,120 +27543,8 @@ } } } - } - } - }, - "post": { - "operationId": "addTransitSource", - "tags": [ - "data-manager" - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "idempotencyKey": { - "type": "string", - "maxLength": 200 - }, - "triggeredBy": { - "type": "string", - "maxLength": 200 - }, - "region": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "url": { - "type": "string", - "format": "uri" - }, - "license": { - "type": "object", - "properties": { - "spdxIdentifier": { - "type": "string" - }, - "url": { - "type": "string" - }, - "attribution": { - "type": "string", - "minLength": 1 - }, - "publisher": { - "type": "string" - }, - "publisherUrl": { - "type": "string" - } - }, - "required": [ - "attribution" - ], - "additionalProperties": false - } - }, - "required": [ - "region", - "name", - "url", - "license" - ], - "additionalProperties": false - } - } - } - }, - "security": [ - { - "serviceBearer": [] }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "addTransitSource", - "x-openmapx-auth": "service", - "responses": { - "202": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "jobId": { - "type": "string" - }, - "sourceId": { - "type": "string" - }, - "status": { - "type": "string", - "enum": [ - "started" - ] - } - }, - "required": [ - "jobId", - "sourceId", - "status" - ], - "additionalProperties": false - } - } - } - }, - "400": { + "401": { "description": "Response", "content": { "application/json": { @@ -16563,7 +27604,7 @@ } } }, - "401": { + "404": { "description": "Response", "content": { "application/json": { @@ -16623,7 +27664,7 @@ } } }, - "404": { + "409": { "description": "Response", "content": { "application/json": { @@ -16683,100 +27724,6 @@ } } }, - "409": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - }, - { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "reason": { - "type": "string", - "enum": [ - "in-flight", - "duplicate-idempotency-key" - ] - }, - "existingJobId": { - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "ok", - "reason", - "existingJobId" - ], - "additionalProperties": false - } - ] - } - } - } - }, "413": { "description": "Response", "content": { @@ -17029,12 +27976,44 @@ } } }, - "/transit/catalog": { - "get": { - "operationId": "getTransitCatalog", + "/transit/sources/{sourceId}": { + "delete": { + "operationId": "removeTransitSource", "tags": [ "data-manager" ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "triggeredBy": { + "type": "string", + "maxLength": 200 + } + }, + "additionalProperties": false + } + } + } + }, + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "sourceId", + "required": true + } + ], "security": [ { "serviceBearer": [] @@ -17043,60 +28022,33 @@ "serviceToken": [] } ], - "x-openmapx-contract-id": "getTransitCatalog", + "x-openmapx-contract-id": "removeTransitSource", "x-openmapx-auth": "service", "responses": { - "200": { + "202": { "description": "Response", "content": { "application/json": { "schema": { "type": "object", "properties": { - "sources": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "region": { - "type": "string" - }, - "name": { - "type": "string" - }, - "format": { - "type": "string", - "enum": [ - "gtfs", - "netex" - ] - }, - "originUrl": { - "type": "string" - }, - "license": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/DataManager_getTransitCatalog_response_200___schema0" - } - } - }, - "required": [ - "id", - "region", - "name", - "format", - "license" - ], - "additionalProperties": false - } + "jobId": { + "type": "string" + }, + "sourceId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "started" + ] } }, "required": [ - "sources" + "jobId", + "sourceId", + "status" ], "additionalProperties": false } @@ -17289,52 +28241,86 @@ "application/json": { "schema": { "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, { "type": "object", "properties": { - "error": { - "type": "string" - }, "ok": { "type": "boolean", "enum": [ false ] }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] }, - "code": { - "type": "string" + "existingJobId": { + "type": [ + "string", + "null" + ] } }, "required": [ - "statusCode", - "error", - "message" + "ok", + "reason", + "existingJobId" ], "additionalProperties": false } @@ -17591,13 +28577,73 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } }, - "/transit/sources/{sourceId}": { - "delete": { - "operationId": "removeTransitSource", + "/transit/sources/{sourceId}/enable": { + "post": { + "operationId": "enableTransitSource", "tags": [ "data-manager" ], @@ -17641,7 +28687,7 @@ "serviceToken": [] } ], - "x-openmapx-contract-id": "removeTransitSource", + "x-openmapx-contract-id": "enableTransitSource", "x-openmapx-auth": "service", "responses": { "202": { @@ -18061,226 +29107,24 @@ "error", "message" ], - "additionalProperties": false - } - ] - } - } - } - }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "500": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - } - } - } - }, - "/transit/sources/{sourceId}/enable": { - "post": { - "operationId": "enableTransitSource", - "tags": [ - "data-manager" - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "idempotencyKey": { - "type": "string", - "maxLength": 200 - }, - "triggeredBy": { - "type": "string", - "maxLength": 200 - } - }, - "additionalProperties": false - } - } - } - }, - "parameters": [ - { - "schema": { - "type": "string", - "minLength": 1 - }, - "in": "path", - "name": "sourceId", - "required": true - } - ], - "security": [ - { - "serviceBearer": [] - }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "enableTransitSource", - "x-openmapx-auth": "service", - "responses": { - "202": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "jobId": { - "type": "string" - }, - "sourceId": { - "type": "string" - }, - "status": { - "type": "string", - "enum": [ - "started" - ] + "additionalProperties": false } - }, - "required": [ - "jobId", - "sourceId", - "status" - ], - "additionalProperties": false + ] } } } }, - "400": { + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -18339,7 +29183,7 @@ } } }, - "401": { + "500": { "description": "Response", "content": { "application/json": { @@ -18399,7 +29243,7 @@ } } }, - "404": { + "503": { "description": "Response", "content": { "application/json": { @@ -18458,93 +29302,376 @@ } } } + } + } + } + }, + "/transit/preflight": { + "post": { + "operationId": "preflightTransit", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "feedCount": { + "type": "number", + "minimum": 0 + }, + "measuredCompressedBytes": { + "type": "number", + "minimum": 0 + }, + "osmBytes": { + "type": "number", + "minimum": 0 + }, + "osmAvailable": { + "type": "boolean" + }, + "freeDiskBytes": { + "type": "number", + "minimum": 0 + }, + "freeInodes": { + "type": "number", + "minimum": 0 + }, + "slotMemoryGb": { + "type": "number", + "minimum": 0 + }, + "slotCpu": { + "type": "number", + "minimum": 0 + }, + "fileDescriptorLimit": { + "type": "number", + "minimum": 0 + }, + "buildTimeoutHours": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] }, - "409": { + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "preflightTransit", + "x-openmapx-auth": "service", + "responses": { + "200": { "description": "Response", "content": { "application/json": { "schema": { - "anyOf": [ - { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "policy": { + "type": "object", + "properties": { + "profile": { + "type": "string", + "enum": [ + "regional-assisted", + "regional-sovereign", + "planet" + ] }, - { + "countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "feedAllowList": { + "type": "array", + "items": { + "type": "string" + } + }, + "acquisition": { + "type": "string", + "enum": [ + "mirror", + "build" + ] + }, + "artifactOrigins": { + "type": "array", + "items": { + "type": "string" + } + }, + "originDownloadsRequired": { + "type": "boolean" + }, + "hostedRuntimeFallbackAllowed": { + "type": "boolean" + }, + "osmInput": { + "type": "string" + }, + "gbfsSelection": { + "type": "string", + "enum": [ + "explicit-countries", + "explicit-feeds", + "allow-listed-batches" + ] + }, + "maxFeedCount": { + "type": "number", + "minimum": 0 + }, + "retentionGenerations": { + "type": "number", + "minimum": 0 + }, + "resourceEnvelope": { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "minimumMemoryGb": { + "type": "number", + "minimum": 0 }, - "error": { - "type": "string" + "minimumCpu": { + "type": "number", + "minimum": 0 }, - "message": { - "type": "string" + "minimumDiskGb": { + "type": "number", + "minimum": 0 }, - "code": { - "type": "string" + "operatorCapacityRequired": { + "type": "boolean" } }, "required": [ - "statusCode", - "error", - "message" + "minimumMemoryGb", + "minimumCpu", + "minimumDiskGb", + "operatorCapacityRequired" ], "additionalProperties": false + }, + "updateCadenceHours": { + "type": "number", + "minimum": 0 + }, + "experimental": { + "type": "boolean" } - ] + }, + "required": [ + "profile", + "countries", + "feedAllowList", + "acquisition", + "artifactOrigins", + "originDownloadsRequired", + "hostedRuntimeFallbackAllowed", + "gbfsSelection", + "maxFeedCount", + "retentionGenerations", + "resourceEnvelope", + "updateCadenceHours", + "experimental" + ], + "additionalProperties": false + }, + "estimate": { + "type": "object", + "properties": { + "feedCount": { + "type": "number", + "minimum": 0 + }, + "measuredCompressedBytes": { + "type": "number", + "minimum": 0 + }, + "compressedBytes": { + "type": "number", + "minimum": 0 + }, + "expandedGtfsBytes": { + "type": "number", + "minimum": 0 + }, + "importedMotisBytes": { + "type": "number", + "minimum": 0 + }, + "osmBytes": { + "type": "number", + "minimum": 0 + }, + "proxyCacheBytes": { + "type": "number", + "minimum": 0 + }, + "requiredDiskBytes": { + "type": "number", + "minimum": 0 + }, + "basis": { + "type": "string", + "enum": [ + "measured-and-conservative", + "conservative-defaults" + ] + } + }, + "required": [ + "feedCount", + "measuredCompressedBytes", + "compressedBytes", + "expandedGtfsBytes", + "importedMotisBytes", + "osmBytes", + "proxyCacheBytes", + "requiredDiskBytes", + "basis" + ], + "additionalProperties": false + }, + "capacity": { + "type": "object", + "properties": { + "freeDiskBytes": { + "type": "number", + "minimum": 0 + }, + "freeInodes": { + "type": "number", + "minimum": 0 + }, + "slotMemoryGb": { + "type": "number", + "minimum": 0 + }, + "slotCpu": { + "type": "number", + "minimum": 0 + }, + "fileDescriptorLimit": { + "type": "number", + "minimum": 0 + }, + "buildTimeoutHours": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "freeDiskBytes", + "slotMemoryGb", + "slotCpu", + "fileDescriptorLimit", + "buildTimeoutHours" + ], + "additionalProperties": false + }, + "blockers": { + "type": "array", + "items": { + "type": "string" + } }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "ok", + "policy", + "estimate", + "capacity", + "blockers", + "warnings" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ { "type": "object", "properties": { + "error": { + "type": "string" + }, "ok": { "type": "boolean", "enum": [ false ] }, - "reason": { - "type": "string", - "enum": [ - "in-flight", - "duplicate-idempotency-key" - ] + "message": { + "type": "string" }, - "existingJobId": { - "type": [ - "string", - "null" - ] + "jobId": { + "type": "string" } }, "required": [ - "ok", - "reason", - "existingJobId" + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" ], "additionalProperties": false } @@ -18553,7 +29680,7 @@ } } }, - "413": { + "401": { "description": "Response", "content": { "application/json": { @@ -18613,7 +29740,7 @@ } } }, - "415": { + "409": { "description": "Response", "content": { "application/json": { @@ -18673,17 +29800,8 @@ } } }, - "429": { + "413": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -18742,7 +29860,7 @@ } } }, - "500": { + "415": { "description": "Response", "content": { "application/json": { @@ -18801,80 +29919,8 @@ } } } - } - } - } - }, - "/transit/preflight": { - "post": { - "operationId": "preflightTransit", - "tags": [ - "data-manager" - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "feedCount": { - "type": "number", - "minimum": 0 - }, - "measuredCompressedBytes": { - "type": "number", - "minimum": 0 - }, - "osmBytes": { - "type": "number", - "minimum": 0 - }, - "osmAvailable": { - "type": "boolean" - }, - "freeDiskBytes": { - "type": "number", - "minimum": 0 - }, - "freeInodes": { - "type": "number", - "minimum": 0 - }, - "slotMemoryGb": { - "type": "number", - "minimum": 0 - }, - "slotCpu": { - "type": "number", - "minimum": 0 - }, - "fileDescriptorLimit": { - "type": "number", - "minimum": 0 - }, - "buildTimeoutHours": { - "type": "number", - "minimum": 0 - } - }, - "additionalProperties": false - } - } - } - }, - "security": [ - { - "serviceBearer": [] }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "preflightTransit", - "x-openmapx-auth": "service", - "responses": { - "200": { + "422": { "description": "Response", "content": { "application/json": { @@ -18884,7 +29930,7 @@ "ok": { "type": "boolean", "enum": [ - true + false ] }, "policy": { @@ -19119,8 +30165,17 @@ } } }, - "400": { + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -19179,7 +30234,7 @@ } } }, - "401": { + "500": { "description": "Response", "content": { "application/json": { @@ -19239,7 +30294,7 @@ } } }, - "413": { + "503": { "description": "Response", "content": { "application/json": { @@ -19298,8 +30353,55 @@ } } } + } + } + } + }, + "/datasets/reload": { + "post": { + "operationId": "reloadDatasets", + "tags": [ + "data-manager" + ], + "security": [ + { + "serviceBearer": [] }, - "415": { + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "reloadDatasets", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "datasets": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "ok", + "datasets" + ], + "additionalProperties": false + } + } + } + }, + "400": { "description": "Response", "content": { "application/json": { @@ -19359,262 +30461,128 @@ } } }, - "422": { + "401": { "description": "Response", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "policy": { + "anyOf": [ + { "type": "object", "properties": { - "profile": { - "type": "string", - "enum": [ - "regional-assisted", - "regional-sovereign", - "planet" - ] - }, - "countries": { - "type": "array", - "items": { - "type": "string" - } - }, - "feedAllowList": { - "type": "array", - "items": { - "type": "string" - } - }, - "acquisition": { - "type": "string", - "enum": [ - "mirror", - "build" - ] - }, - "artifactOrigins": { - "type": "array", - "items": { - "type": "string" - } - }, - "originDownloadsRequired": { - "type": "boolean" - }, - "hostedRuntimeFallbackAllowed": { - "type": "boolean" - }, - "osmInput": { + "error": { "type": "string" }, - "gbfsSelection": { - "type": "string", + "ok": { + "type": "boolean", "enum": [ - "explicit-countries", - "explicit-feeds", - "allow-listed-batches" + false ] }, - "maxFeedCount": { - "type": "number", - "minimum": 0 - }, - "retentionGenerations": { - "type": "number", - "minimum": 0 - }, - "resourceEnvelope": { - "type": "object", - "properties": { - "minimumMemoryGb": { - "type": "number", - "minimum": 0 - }, - "minimumCpu": { - "type": "number", - "minimum": 0 - }, - "minimumDiskGb": { - "type": "number", - "minimum": 0 - }, - "operatorCapacityRequired": { - "type": "boolean" - } - }, - "required": [ - "minimumMemoryGb", - "minimumCpu", - "minimumDiskGb", - "operatorCapacityRequired" - ], - "additionalProperties": false - }, - "updateCadenceHours": { - "type": "number", - "minimum": 0 + "message": { + "type": "string" }, - "experimental": { - "type": "boolean" + "jobId": { + "type": "string" } }, "required": [ - "profile", - "countries", - "feedAllowList", - "acquisition", - "artifactOrigins", - "originDownloadsRequired", - "hostedRuntimeFallbackAllowed", - "gbfsSelection", - "maxFeedCount", - "retentionGenerations", - "resourceEnvelope", - "updateCadenceHours", - "experimental" + "error" ], "additionalProperties": false }, - "estimate": { + { "type": "object", "properties": { - "feedCount": { - "type": "number", - "minimum": 0 - }, - "measuredCompressedBytes": { - "type": "number", - "minimum": 0 - }, - "compressedBytes": { - "type": "number", - "minimum": 0 - }, - "expandedGtfsBytes": { - "type": "number", - "minimum": 0 - }, - "importedMotisBytes": { - "type": "number", - "minimum": 0 + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, - "osmBytes": { - "type": "number", - "minimum": 0 + "error": { + "type": "string" }, - "proxyCacheBytes": { - "type": "number", - "minimum": 0 + "message": { + "type": "string" }, - "requiredDiskBytes": { - "type": "number", - "minimum": 0 + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" }, - "basis": { - "type": "string", + "ok": { + "type": "boolean", "enum": [ - "measured-and-conservative", - "conservative-defaults" + false ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" } }, "required": [ - "feedCount", - "measuredCompressedBytes", - "compressedBytes", - "expandedGtfsBytes", - "importedMotisBytes", - "osmBytes", - "proxyCacheBytes", - "requiredDiskBytes", - "basis" + "error" ], "additionalProperties": false }, - "capacity": { + { "type": "object", "properties": { - "freeDiskBytes": { - "type": "number", - "minimum": 0 - }, - "freeInodes": { - "type": "number", - "minimum": 0 - }, - "slotMemoryGb": { - "type": "number", - "minimum": 0 + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, - "slotCpu": { - "type": "number", - "minimum": 0 + "error": { + "type": "string" }, - "fileDescriptorLimit": { - "type": "number", - "minimum": 0 + "message": { + "type": "string" }, - "buildTimeoutHours": { - "type": "number", - "minimum": 0 + "code": { + "type": "string" } }, "required": [ - "freeDiskBytes", - "slotMemoryGb", - "slotCpu", - "fileDescriptorLimit", - "buildTimeoutHours" + "statusCode", + "error", + "message" ], "additionalProperties": false - }, - "blockers": { - "type": "array", - "items": { - "type": "string" - } - }, - "warnings": { - "type": "array", - "items": { - "type": "string" - } } - }, - "required": [ - "ok", - "policy", - "estimate", - "capacity", - "blockers", - "warnings" - ], - "additionalProperties": false + ] } } } }, - "429": { + "413": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -19673,7 +30641,7 @@ } } }, - "500": { + "415": { "description": "Response", "content": { "application/json": { @@ -19732,116 +30700,18 @@ } } } - } - } - } - }, - "/datasets/reload": { - "post": { - "operationId": "reloadDatasets", - "tags": [ - "data-manager" - ], - "security": [ - { - "serviceBearer": [] - }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "reloadDatasets", - "x-openmapx-auth": "service", - "responses": { - "200": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] - }, - "datasets": { - "type": "number", - "minimum": 0 - } - }, - "required": [ - "ok", - "datasets" - ], - "additionalProperties": false - } - } - } }, - "400": { + "429": { "description": "Response", - "content": { - "application/json": { + "headers": { + "retry-after": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" } - } - }, - "401": { - "description": "Response", + }, "content": { "application/json": { "schema": { @@ -19900,7 +30770,7 @@ } } }, - "413": { + "500": { "description": "Response", "content": { "application/json": { @@ -19960,7 +30830,7 @@ } } }, - "415": { + "503": { "description": "Response", "content": { "application/json": { @@ -20019,77 +30889,50 @@ } } } + } + } + } + }, + "/traffic/predicted/bake": { + "post": { + "operationId": "bakePredictedTraffic", + "tags": [ + "data-manager" + ], + "security": [ + { + "serviceBearer": [] }, - "429": { + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "bakePredictedTraffic", + "x-openmapx-auth": "service", + "responses": { + "202": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false + "type": "object", + "properties": { + "accepted": { + "type": "boolean", + "enum": [ + true + ] } - ] + }, + "required": [ + "accepted" + ], + "additionalProperties": false } } } }, - "500": { + "400": { "description": "Response", "content": { "application/json": { @@ -20148,50 +30991,8 @@ } } } - } - } - } - }, - "/traffic/predicted/bake": { - "post": { - "operationId": "bakePredictedTraffic", - "tags": [ - "data-manager" - ], - "security": [ - { - "serviceBearer": [] }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "bakePredictedTraffic", - "x-openmapx-auth": "service", - "responses": { - "202": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "accepted": { - "type": "boolean", - "enum": [ - true - ] - } - }, - "required": [ - "accepted" - ], - "additionalProperties": false - } - } - } - }, - "400": { + "401": { "description": "Response", "content": { "application/json": { @@ -20251,7 +31052,7 @@ } } }, - "401": { + "409": { "description": "Response", "content": { "application/json": { @@ -20311,7 +31112,7 @@ } } }, - "409": { + "413": { "description": "Response", "content": { "application/json": { @@ -20371,7 +31172,7 @@ } } }, - "413": { + "415": { "description": "Response", "content": { "application/json": { @@ -20431,8 +31232,17 @@ } } }, - "415": { + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -20491,17 +31301,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -20560,7 +31361,7 @@ } } }, - "500": { + "501": { "description": "Response", "content": { "application/json": { @@ -20620,7 +31421,7 @@ } } }, - "501": { + "503": { "description": "Response", "content": { "application/json": { @@ -21627,6 +32428,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -21875,6 +32736,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -22744,6 +33665,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -22804,8 +33785,77 @@ } } }, - "415": { + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, "content": { "application/json": { "schema": { @@ -22864,17 +33914,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -22933,7 +33974,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -23830,6 +34871,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -24078,6 +35179,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -24241,6 +35402,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -24489,6 +35710,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -24728,6 +36009,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -24976,6 +36317,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -25188,7 +36589,187 @@ } } }, - "401": { + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { "description": "Response", "content": { "application/json": { @@ -25248,68 +36829,17 @@ } } }, - "413": { + "429": { "description": "Response", - "content": { - "application/json": { + "headers": { + "retry-after": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" } - } - }, - "415": { - "description": "Response", + }, "content": { "application/json": { "schema": { @@ -25368,17 +36898,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -25437,7 +36958,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -25757,68 +37278,197 @@ } } }, - "413": { + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { "description": "Response", - "content": { - "application/json": { + "headers": { + "retry-after": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" } - } - }, - "415": { - "description": "Response", + }, "content": { "application/json": { "schema": { @@ -25877,17 +37527,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -25946,7 +37587,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -27009,7 +38650,187 @@ } } }, - "401": { + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { "description": "Response", "content": { "application/json": { @@ -27069,68 +38890,17 @@ } } }, - "413": { + "429": { "description": "Response", - "content": { - "application/json": { + "headers": { + "retry-after": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" } - } - }, - "415": { - "description": "Response", + }, "content": { "application/json": { "schema": { @@ -27189,17 +38959,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -27258,7 +39019,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -27574,68 +39335,197 @@ } } }, - "413": { + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { "description": "Response", - "content": { - "application/json": { + "headers": { + "retry-after": { "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" } - } - }, - "415": { - "description": "Response", + }, "content": { "application/json": { "schema": { @@ -27694,17 +39584,8 @@ } } }, - "429": { + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -27763,7 +39644,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -28079,6 +39960,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -28327,6 +40268,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -28771,6 +40772,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -29019,6 +41080,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -29227,34 +41348,91 @@ "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "reason": { - "type": "string", - "enum": [ - "in-flight", - "duplicate-idempotency-key" + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } ] }, - "existingJobId": { - "type": [ - "string", - "null" - ] + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] + }, + "existingJobId": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "ok", + "reason", + "existingJobId" + ], + "additionalProperties": false } - }, - "required": [ - "ok", - "reason", - "existingJobId" - ], - "additionalProperties": false + ] } } } @@ -29379,43 +41557,103 @@ } } }, - "422": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "reason": { - "type": "string" - } - }, - "required": [ - "ok", - "reason" - ], - "additionalProperties": false - } - } - } - }, - "429": { + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "reason": { + "type": "string" + } + }, + "required": [ + "ok", + "reason" + ], + "additionalProperties": false + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, "content": { "application/json": { "schema": { @@ -29474,7 +41712,7 @@ } } }, - "500": { + "503": { "description": "Response", "content": { "application/json": { @@ -29703,6 +41941,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -30641,6 +42939,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } @@ -30900,6 +43258,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -31181,6 +43599,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } } } diff --git a/services/data-manager/src/api.ts b/services/data-manager/src/api.ts index 9b726ecbe..d003d5140 100644 --- a/services/data-manager/src/api.ts +++ b/services/data-manager/src/api.ts @@ -1,55 +1,39 @@ import { createReadStream } from "node:fs"; import { join } from "node:path"; +import { setTimeout as observerDelay } from "node:timers/promises"; import { type OfflinePackageRequest, parseOfflinePackageRequest, type TrafficApplicationSnapshot, } from "@openmapx/core"; +import { + type DataExecutionInput, + type DataExecutionStatus, + isDataExecutionTerminal, +} from "@openmapx/core/data-manager-contracts"; import { feedState } from "@openmapx/db-schema"; -import { parseTransitSource } from "@openmapx/transitous-core"; import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; import type postgres from "postgres"; import { type CoverageApiOptions, registerCoverageEvidenceRoute } from "./coverage/api.js"; import type { CoverageSnapshotStore } from "./coverage/snapshot-store.js"; import { db, sql } from "./db/index.js"; import { registerAmbientPlacesApi } from "./jobs/ambient-places/api.js"; -import { convertPbfToBz2ForRegion } from "./jobs/convert-overpass.js"; -import { downloadFonts } from "./jobs/download-fonts.js"; -import { downloadOsm } from "./jobs/download-osm.js"; -import { applyHardlinkPlan } from "./jobs/link.js"; +import { type DataExecutionReceiver, registerDataExecutionApi } from "./jobs/execution/api.js"; import { - type BuildNotablePlacesOptions, - buildNotablePlaces, - type NotablePlacesBuildResult, -} from "./jobs/notable-places/build.js"; -import { notablePlacesSettings } from "./jobs/notable-places/schedule.js"; + startLegacyDataMutation, + withLegacyDataMutation, +} from "./jobs/execution/legacy-mutation.js"; import { createNotablePlacesRuntimeState, getNotablePlacesStatus, type NotablePlacesRuntimeState, } from "./jobs/notable-places/state.js"; -import { extractOsmPois } from "./jobs/overture/extract-osm-pois.js"; -import { ingestOverture } from "./jobs/overture/ingest.js"; -import { withOvertureOperationLock } from "./jobs/overture/operation-lock.js"; -import { assertValidRegion, pullOverture } from "./jobs/overture/pull.js"; -import { getOvertureConflationState, rebuildOvertureLinks } from "./jobs/overture/rebuild-links.js"; -import { syncOvertureRegion } from "./jobs/overture/sync.js"; -import { - type BuildOsmSearchIndexOptions, - buildOsmSearchIndex, - type SearchIndexBuildResult, -} from "./jobs/search-index/build.js"; +import { getOvertureConflationState } from "./jobs/overture/rebuild-links.js"; import { createNotablePlacesOperationLock, - createSearchIndexOperationLock, type SearchIndexOperationLock, } from "./jobs/search-index/operation-lock.js"; -import { - createSearchIndexRuntimeState, - fingerprintDataset, - getSearchIndexStatus, - updateCurrentSearchIndexFingerprint, -} from "./jobs/search-index/state.js"; +import { createSearchIndexRuntimeState, getSearchIndexStatus } from "./jobs/search-index/state.js"; import type { BakePredictedResult } from "./jobs/traffic/bake-predicted.js"; import { CatalogBumpError, @@ -57,7 +41,6 @@ import { lockFromCandidate, resolveCatalogBumpCandidate, } from "./jobs/transitous/catalog-bump.js"; -import { buildJobContext, runTransitousPipeline } from "./jobs/transitous/index.js"; import { resolveTransitOverlayPath } from "./jobs/transitous/internal.js"; import { type MotisOperationsPolicy, @@ -69,11 +52,10 @@ import { OperatorFeedRelayCapabilityError, type OperatorFeedRelayStore, } from "./jobs/transitous/operator-feed-relay.js"; -import { finalizeJobRow, makePersistingOnStageComplete } from "./jobs/transitous/persistence.js"; +import { finalizeJobRow } from "./jobs/transitous/persistence.js"; import { runMotisPreflight } from "./jobs/transitous/preflight.js"; import { getSingleFlightController } from "./jobs/transitous/runtime.js"; import type { SingleFlightController } from "./jobs/transitous/single-flight.js"; -import { asJobLogger, jobChildLogger } from "./logger.js"; import { OFFLINE_PACKAGE_PRINCIPAL_PATTERN, OfflinePackagePrincipalQuotaError, @@ -102,6 +84,7 @@ import { } from "./transit-sources.js"; export interface ApiOptions { + executionReceiver?: DataExecutionReceiver; dataDir?: string; /** * Single-flight controller. Defaults to the process-wide singleton so the @@ -109,8 +92,6 @@ export interface ApiOptions { */ singleFlight?: SingleFlightController; operationsPolicy?: MotisOperationsPolicy; - /** Test seam invoked after reservation; production launches the real pipeline. */ - launchTransitSync?: (args: { jobId: string; countries: string[]; trigger: string }) => void; /** * Runs the predicted-traffic bake. Wired in `service.ts` only when * `OPENCONDITIONS_URL` is configured, so the route can answer 501 rather than @@ -123,8 +104,7 @@ export interface ApiOptions { offlinePackages?: OfflinePackageGenerator; /** Search-index database/test seam. */ searchIndexSql?: postgres.Sql; - /** Test seam for the country-scale OSM search-index build. */ - buildSearchIndex?: (opts: BuildOsmSearchIndexOptions) => Promise; + searchIndexRuntimeState?: ReturnType; /** * Build state and lock shared with the scheduled refresh, so a build started * here and one started by the schedule never overlap. Created here when the @@ -133,7 +113,6 @@ export interface ApiOptions { notablePlaces?: { runtimeState: NotablePlacesRuntimeState; operationLock: SearchIndexOperationLock; - build?: (opts: BuildNotablePlacesOptions) => Promise; }; /** Process startup readiness; omitted by isolated route tests, which are ready immediately. */ readiness?: () => DataManagerReadinessSnapshot; @@ -418,13 +397,13 @@ function registerOfflinePackageRoutes( } export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { + registerDataExecutionApi(app, opts.executionReceiver); const dataDir = opts.dataDir ?? process.env.DATA_DIR ?? "/data"; const singleFlight = opts.singleFlight ?? getSingleFlightController(); const store = new StateStore(dataDir); const searchIndexSql = opts.searchIndexSql ?? sql; registerAmbientPlacesApi(app, searchIndexSql); - const searchIndexRuntimeState = createSearchIndexRuntimeState(); - const searchIndexOperationLock = createSearchIndexOperationLock(searchIndexSql); + const searchIndexRuntimeState = opts.searchIndexRuntimeState ?? createSearchIndexRuntimeState(); const notablePlaces = opts.notablePlaces ?? { runtimeState: createNotablePlacesRuntimeState(), operationLock: createNotablePlacesOperationLock(searchIndexSql), @@ -450,41 +429,62 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { if (opts.offlinePackages) registerOfflinePackageRoutes(app, opts.offlinePackages); - function launchReservedTransitSync(jobId: string, countries: string[], trigger: string): void { - if (opts.launchTransitSync) { - opts.launchTransitSync({ jobId, countries, trigger }); - return; - } - const jobLog = asJobLogger(jobChildLogger({ job: "transitous-sync", jobId, trigger })); - const persistingHook = makePersistingOnStageComplete(jobId, jobLog); - void (async () => { - try { - const ctx = buildJobContext({ - dataDir, - store, - countries, - source: parseTransitSource(), - operationsPolicy: { ...operationsPolicy, countries }, - jobId, - logger: jobLog, - onStageComplete: persistingHook, - feedsOverlayPath: overlayPath, - operatorFeedRelay, - }); - const result = await runTransitousPipeline(ctx); - await finalizeJobRow(jobId, result.finalStatus); - app.log.info({ jobId, finalStatus: result.finalStatus }, "transitous-api: sync finished"); - } catch (err) { - app.log.error({ jobId, err }, "transitous-api: sync threw"); - try { - await finalizeJobRow(jobId, "error"); - } catch { - // The row remains visible as running if Postgres itself is unavailable. - } - } finally { - singleFlight.markSyncFinished(); + async function admitLegacyData(input: DataExecutionInput) { + if (!opts.executionReceiver) + throw Object.assign(Error("Durable data receiver is unavailable"), { statusCode: 503 }); + return opts.executionReceiver.admitLocal(input); + } + async function observeLegacyData( + status: DataExecutionStatus, + reply: FastifyReply, + operationId?: import("@openmapx/core/data-manager-contracts").DataManagerOperationKey, + ) { + const receiver = opts.executionReceiver; + if (!receiver) + throw Object.assign(Error("Durable data receiver is unavailable"), { statusCode: 503 }); + const observer = new AbortController(), + closed = () => observer.abort(); + reply.raw.once("close", closed); + reply.request.raw.once("aborted", closed); + const stream = operationId ? openNdjsonStream(reply, operationId) : undefined; + const unsubscribe = receiver.subscribe(status.id, (event) => stream?.writeLine(event)); + try { + let current = status; + while (!isDataExecutionTerminal(current) || current.publication === "unknown") { + if (observer.signal.aborted) return undefined; + const remaining = status.deadlineAt - Date.now(); + if (remaining <= 0) + throw Error( + `Data execution ${status.id} observer deadline elapsed; inspect its retained status`, + ); + await observerDelay(Math.min(200, remaining), undefined, { signal: observer.signal }); + const found = await receiver.inspect(status.id); + if ( + !found || + found.operationKey !== status.operationKey || + found.fingerprint !== status.fingerprint || + found.admittedAt !== status.admittedAt || + found.deadlineAt !== status.deadlineAt + ) + throw Error(`Execution ${status.id} requires retained status observation`); + current = found; + } + if (current.status !== "ok" && current.status !== "partial") + throw Error(`Execution ${status.id} ${current.status}`); + if (stream) stream.writeLine(receiver.legacyResult(status.id) ?? { event: "done", ok: true }); + return current; + } catch (error) { + if (!observer.signal.aborted) { + if (stream) stream.writeLine({ event: "error", message: (error as Error).message }); + else throw error; } - })(); + return undefined; + } finally { + unsubscribe(); + reply.raw.removeListener("close", closed); + reply.request.raw.removeListener("aborted", closed); + stream?.end(); + } } async function reserveSourceMutation(options: { @@ -496,11 +496,17 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }): Promise< { ok: true; jobId: string } | { ok: false; status: number; body: Record } > { + if (!opts.executionReceiver) + return { + ok: false, + status: 503, + body: { ok: false, error: "Durable data receiver is unavailable" }, + }; const start = await singleFlight.tryStartSync({ trigger: "api", triggeredBy: options.triggeredBy, idempotencyKey: options.idempotencyKey, - kind: "transitous-sync", + kind: `transit-source-${options.action}`, metadata: { source: "source-mutation", action: options.action, @@ -532,8 +538,18 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { body: { ok: false, error: (error as Error).message, jobId: start.jobId }, }; } - launchReservedTransitSync(start.jobId, operationsPolicy.countries, `source-${options.action}`); - return { ok: true, jobId: start.jobId }; + singleFlight.markSyncFinished(start.jobId); + try { + const admitted = await opts.executionReceiver.admitLocal( + { kind: "transit-sync", countries: operationsPolicy.countries }, + { sourceJobId: start.jobId }, + ); + await finalizeJobRow(start.jobId, "ok"); + return { ok: true, jobId: admitted.id }; + } catch (error) { + await finalizeJobRow(start.jobId, "error"); + throw error; + } } registerDataManagerRoute(app, "getLive", async () => ({ ok: true })); @@ -734,8 +750,11 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }); registerDataManagerRoute(app, "reloadDatasets", async () => { - const result = store.reload(); - return { ok: true, ...result }; + return withLegacyDataMutation( + "data-reload", + async () => ({ ok: true, ...store.reload() }), + searchIndexSql, + ); }); // Single-flight: the bake shells out to a slow tile-rewriting tool and @@ -752,10 +771,20 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { return { error: "a predicted bake is already running" }; } bakeInFlight = true; - void opts - .bakePredicted() + let owned: Awaited>; + try { + owned = await startLegacyDataMutation( + "traffic-predicted", + opts.bakePredicted, + searchIndexSql, + ); + } catch (error) { + bakeInFlight = false; + throw error; + } + void owned.completion .then((result) => { - app.log.info({ ...result }, "traffic-predicted: manual bake complete"); + app.log.info({ result }, "traffic-predicted: manual bake complete"); }) .catch((err: unknown) => { app.log.error({ err: (err as Error).message }, "traffic-predicted: manual bake failed"); @@ -778,53 +807,8 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }); registerDataManagerRoute(app, "downloadOsm", async (req, reply) => { - const { region } = req.body ?? {}; - if (!region || typeof region !== "string") throw new Error("region required"); - // Same Geofabrik path shape the Overture routes enforce — the region becomes - // both a download URL segment and a local filename. - assertValidRegion(region); - - // Stream NDJSON progress events back to the client. Hijacking the reply - // lets us write line-by-line; Fastify otherwise buffers the full body. - const stream = openNdjsonStream(reply, "downloadOsm"); - - try { - const result = await downloadOsm({ - region, - dataDir, - store, - onProgress: (bytes, totalBytes) => - stream.writeLine({ event: "progress", bytes, totalBytes }), - // Network transfer and checksum verification stay outside the lock; - // only the final atomic rename can race Overture fingerprinting. - withPublishLock: (publish) => - withOvertureOperationLock(async () => { - publish(); - }), - }); - const dataset = store - .getAll() - .find((entry) => entry.type === "osm-pbf" && (entry.region ?? entry.id) === region); - if (dataset) { - try { - await updateCurrentSearchIndexFingerprint( - searchIndexSql, - region, - await fingerprintDataset(dataset), - ); - } catch (error) { - req.log.warn( - { err: error }, - "download-osm: could not update current search-index fingerprint", - ); - } - } - stream.writeLine({ event: "done", ok: true, ...result }); - } catch (err) { - stream.writeLine({ event: "error", message: (err as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ kind: "download-osm", region: req.body.region }); + await observeLegacyData(admitted, reply, "downloadOsm"); }); registerDataManagerRoute(app, "getSearchIndexStatus", async (_req, reply) => { @@ -841,29 +825,8 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }); registerDataManagerRoute(app, "buildSearchIndex", async (req, reply) => { - const { region } = req.body ?? {}; - if (!region || typeof region !== "string") { - return reply.code(400).send({ ok: false, error: "region required" }); - } - assertValidRegion(region); - const stream = openNdjsonStream(reply, "buildSearchIndex"); - try { - const build = opts.buildSearchIndex ?? buildOsmSearchIndex; - const result = await build({ - region, - dataDir, - store, - sql: searchIndexSql, - runtimeState: searchIndexRuntimeState, - operationLock: searchIndexOperationLock, - onProgress: (progress) => stream.writeLine({ event: "progress", ...progress }), - }); - stream.writeLine({ event: "done", ok: true, ...result }); - } catch (error) { - stream.writeLine({ event: "error", message: (error as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ kind: "search-index-build", region: req.body.region }); + await observeLegacyData(admitted, reply, "buildSearchIndex"); }); registerDataManagerRoute(app, "getNotablePlacesStatus", async (_req, reply) => { @@ -878,75 +841,33 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }); registerDataManagerRoute(app, "buildNotablePlaces", async (_req, reply) => { - const stream = openNdjsonStream(reply, "buildNotablePlaces"); - try { - const build = notablePlaces.build ?? buildNotablePlaces; - const result = await build({ - sql: searchIndexSql, - runtimeState: notablePlaces.runtimeState, - operationLock: notablePlaces.operationLock, - ...notablePlacesSettings(), - onProgress: (progress) => stream.writeLine({ event: "progress", ...progress }), - }); - stream.writeLine({ event: "done", ok: true, ...result }); - } catch (error) { - stream.writeLine({ event: "error", message: (error as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ kind: "notable-places-build" }); + await observeLegacyData(admitted, reply, "buildNotablePlaces"); }); - registerDataManagerRoute(app, "downloadFonts", async () => { - await downloadFonts({ dataDir, store }); - return { ok: true }; + registerDataManagerRoute(app, "downloadFonts", async (_req, reply) => { + const admitted = await admitLegacyData({ kind: "download-fonts" }); + await observeLegacyData(admitted, reply); + if (!reply.raw.destroyed) return { ok: true }; + return undefined; }); - registerDataManagerRoute(app, "link", async (req) => { - const { plan, prune } = req.body; - const result = await applyHardlinkPlan(plan, { rootDir: dataDir, prune }); - return { ok: true, ...result }; - }); + registerDataManagerRoute(app, "link", async (_req, reply) => + reply.code(503).send({ error: "A signed generation-hardlinks execution is required" }), + ); registerDataManagerRoute(app, "convertOverpass", async (req, reply) => { - const { region } = req.body ?? {}; - - const stream = openNdjsonStream(reply, "convertOverpass"); - try { - const result = await convertPbfToBz2ForRegion({ - region, - dataDir, - store, - onProgress: (bytes, totalBytes) => - stream.writeLine({ event: "progress", bytes, totalBytes }), - }); - stream.writeLine({ event: "done", ok: true, ...result }); - } catch (err) { - stream.writeLine({ event: "error", message: (err as Error).message }); - } finally { - stream.end(); - } - return undefined; + const admitted = await admitLegacyData({ + kind: "convert-overpass", + region: + req.body?.region ?? store.getAll().find((entry) => entry.type === "osm-pbf")?.region ?? "", + }); + await observeLegacyData(admitted, reply, "convertOverpass"); }); registerDataManagerRoute(app, "pullOverture", async (req, reply) => { - const { region } = req.body; - if (!region) throw new Error("region required"); - assertValidRegion(region); - - const stream = openNdjsonStream(reply, "pullOverture"); - - try { - const result = await pullOverture({ - region, - dataDir, - onProgress: (msg) => stream.writeLine({ event: "progress", message: msg }), - }); - stream.writeLine({ event: "done", ok: true, path: result }); - } catch (err) { - stream.writeLine({ event: "error", message: (err as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ kind: "overture-pull", region: req.body.region }); + await observeLegacyData(admitted, reply, "pullOverture"); }); registerDataManagerRoute(app, "getOvertureStatus", async (_req, reply) => { @@ -965,177 +886,54 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }); registerDataManagerRoute(app, "syncOverture", async (req, reply) => { - const { region } = req.body; - if (!region) throw new Error("region required"); - assertValidRegion(region); - - const stream = openNdjsonStream(reply, "syncOverture"); - - try { - const result = await syncOvertureRegion({ - region, - dataDir, - onProgress: (message) => stream.writeLine({ event: "progress", message }), - }); - stream.writeLine({ event: "done", ok: true, ...result }); - } catch (err) { - stream.writeLine({ event: "error", message: (err as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ kind: "overture-sync", region: req.body.region }); + await observeLegacyData(admitted, reply, "syncOverture"); }); registerDataManagerRoute(app, "ingestOverture", async (req, reply) => { - const { region } = req.body; - if (!region) throw new Error("region required"); - assertValidRegion(region); - - const stream = openNdjsonStream(reply, "ingestOverture"); - - try { - await withOvertureOperationLock(() => - ingestOverture({ - region, - dataDir, - onProgress: (msg) => stream.writeLine({ event: "progress", message: msg }), - }), - ); - stream.writeLine({ event: "done", ok: true }); - } catch (err) { - stream.writeLine({ event: "error", message: (err as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ kind: "overture-ingest", region: req.body.region }); + await observeLegacyData(admitted, reply, "ingestOverture"); }); registerDataManagerRoute(app, "extractOverture", async (req, reply) => { - const { region } = req.body; - if (!region) throw new Error("region required"); - assertValidRegion(region); - - const stream = openNdjsonStream(reply, "extractOverture"); - - try { - await withOvertureOperationLock(() => - extractOsmPois({ - region, - dataDir, - onProgress: (msg) => stream.writeLine({ event: "progress", message: msg }), - }), - ); - stream.writeLine({ event: "done", ok: true }); - } catch (err) { - stream.writeLine({ event: "error", message: (err as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ kind: "overture-extract", region: req.body.region }); + await observeLegacyData(admitted, reply, "extractOverture"); }); registerDataManagerRoute(app, "conflateOverture", async (req, reply) => { - const { region, restart } = req.body; - if (!region) throw new Error("region required"); - assertValidRegion(region); - - const ollamaUrl = process.env.OLLAMA_URL || "http://local-ai:11434"; - - const stream = openNdjsonStream(reply, "conflateOverture"); - - try { - const result = await rebuildOvertureLinks({ - region, - dataDir, - force: restart === true, - ollamaUrl, - useEmbeddings: false, - onProgress: (msg) => stream.writeLine({ event: "progress", message: msg }), - }); - stream.writeLine({ - event: "done", - ok: result.status !== "failed" && result.status !== "waiting_for_osm", - message: - result.status === "failed" - ? result.error - : result.status === "waiting_for_osm" - ? `OSM PBF not found at ${result.pbfPath}` - : result.status === "already_running" - ? "Another worker still owns the active conflation lease" - : undefined, - ...result, - }); - } catch (err) { - stream.writeLine({ event: "error", message: (err as Error).message }); - } finally { - stream.end(); - } + const admitted = await admitLegacyData({ + kind: "overture-conflate", + region: req.body.region, + restart: req.body.restart, + }); + await observeLegacyData(admitted, reply, "conflateOverture"); }); // POST /transit/sync — fire-and-forget Transitous pipeline trigger. Honours // the single-flight lock + idempotency key. apps/api proxies user-facing // sync requests here. registerDataManagerRoute(app, "syncTransit", async (req, reply) => { - const body = req.body ?? {}; - // Default to the deployment's configured countries (same as the cron) when - // the caller doesn't specify any — an empty list means "every country", - // which would kick off a global multi-GB fetch by accident. - const countries = body.countries ?? operationsPolicy.countries; - const outsideScope = countries.filter( - (country) => !operationsPolicy.countries.includes(country.toLowerCase()), - ); - if (operationsPolicy.profile !== "planet" && outsideScope.length > 0) { - return reply.code(422).send({ - ok: false, - reason: `countries outside configured operations profile: ${outsideScope.join(", ")}`, - }); - } - const start = await singleFlight.tryStartSync({ - trigger: "api", - triggeredBy: body.triggeredBy ?? "api", - idempotencyKey: body.idempotencyKey, - kind: "transitous-sync", - metadata: { - source: "api", - countries, - }, + const admitted = await admitLegacyData({ + kind: "transit-sync", + countries: req.body?.countries ?? operationsPolicy.countries, }); - - if (!start.ok) { - // 409 Conflict captures both "in-flight" and "duplicate-idempotency-key". - // The caller distinguishes via the `reason` payload. - return reply.code(409).send({ - ok: false, - reason: start.reason, - existingJobId: start.existingJobId, - }); - } - - const jobId = start.jobId; - launchReservedTransitSync(jobId, countries, "api"); - - reply.code(202); - return { ok: true, jobId, status: "started" }; + return reply.code(202).send({ ok: true, jobId: admitted.id, status: "started" }); }); // POST /transit/restart-motis — bounce the primary MOTIS container. Used // when a config change requires a full restart rather than the partial // reloads the pipeline already performs. registerDataManagerRoute(app, "restartMotis", async (_req, reply) => { - try { - // Data-manager holds no Docker socket: the restart is a typed operation - // the agent performs against its own fixed container. - await runOpsOperation({ kind: "motis.primary.restart" }); - return { ok: true, status: "restart-initiated" }; - } catch (err) { - // The most common failure mode is that the operations agent is - // unreachable or has not been configured. Surface a 503 so the operator - // sees an actionable error. - app.log.warn({ err }, "transitous-api: motis restart operation failed"); - reply.code(503); - return { + if (!opts.executionReceiver) + return reply.code(503).send({ ok: false, error: "ops-agent-unavailable", - message: (err as Error).message, - }; - } + message: "Durable data receiver is unavailable", + }); + const admitted = await opts.executionReceiver.admitMaintenance("restart-primary"); + await observeLegacyData(admitted, reply); + if (!reply.raw.destroyed) return { ok: true, status: "restart-initiated" }; + return undefined; }); // POST /transit/bump — propose a new pin set. It never activates the lock; @@ -1177,13 +975,15 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { ); // The proposal file lives under the repository's `infra/docker/`, which only // the operations agent may write. - await runOpsOperation({ - kind: "transitousLock.propose", - ref: lock.ref, - submodules: lock.submodules, - lockedBy: lock.lockedBy, - ...(lock.comment ? { comment: lock.comment } : {}), - }); + await withLegacyDataMutation("transit-proposal", () => + runOpsOperation({ + kind: "transitousLock.propose", + ref: lock.ref, + submodules: lock.submodules, + lockedBy: lock.lockedBy, + ...(lock.comment ? { comment: lock.comment } : {}), + }), + ); return { ok: true, @@ -1197,23 +997,9 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }; }); - registerDataManagerRoute(app, "approveTransitBump", async (req, reply) => { - const { proposed } = await runOpsOperation({ kind: "transitousLock.inspect" }); - if (!proposed) return reply.code(404).send({ ok: false, error: "no-proposal" }); - if (req.body?.approveRef !== proposed.ref) { - return reply.code(422).send({ - ok: false, - error: "typed-confirmation-mismatch", - expected: proposed.ref, - }); - } - // The agent re-matches the ref under its own read, so an approval cannot - // land on a proposal that changed after this check. - const approved = await runOpsOperation({ - kind: "transitousLock.approve", - ref: proposed.ref, - approvedBy: req.body?.approvedBy?.trim() || "api-approval", - }); - return { ok: true, activated: true, ref: approved.ref, lockedAt: approved.lockedAt }; + registerDataManagerRoute(app, "approveTransitBump", async (_req, reply) => { + return reply + .code(503) + .send({ error: "Catalog approval requires paired validation and durable activation" }); }); } diff --git a/services/data-manager/src/auth.ts b/services/data-manager/src/auth.ts index adb8858e2..dd16fa0c9 100644 --- a/services/data-manager/src/auth.ts +++ b/services/data-manager/src/auth.ts @@ -75,7 +75,13 @@ export function resolveAuthToken(app: FastifyInstance): string { * before body parsing. Health endpoints remain open so container/k8s probes * and cross-service discovery keep working. */ -export function registerAuth(app: FastifyInstance, expectedToken: string): void { +export function registerAuth( + app: FastifyInstance, + expectedToken: string, + options: { + executionReceiver?: () => import("./jobs/execution/api.js").DataExecutionReceiver | undefined; + } = {}, +): void { // Register the maintained limiter in this root scope, while invoking it only // for failed credentials. Successful long-running jobs and probes have no // authentication-attempt budget. The root hook runs before body parsing. @@ -103,7 +109,19 @@ export function registerAuth(app: FastifyInstance, expectedToken: string): void // Task02.3 owns durable receiver admission. Never silently ignore an internal authority header. try { const headers = controlHeaders(req.raw.rawHeaders); - if (headers.delegation !== undefined || headers.parent !== undefined) + const path = req.url.split("?")[0] ?? ""; + const admission = + req.method === "POST" && + ([ + "/executions", + "/executions/generation-hardlinks", + "/executions/configuration-maintenance/prepare", + ].includes(path) || + /^\/executions\/configuration-maintenance\/[a-f0-9-]{36}\/settle$/.test(path)); + if ( + (headers.delegation !== undefined || headers.parent !== undefined) && + (!admission || !options.executionReceiver?.()) + ) return reply .code(503) .send({ error: "Managed data-manager execution requires the durable receiver upgrade" }); diff --git a/services/data-manager/src/cron.ts b/services/data-manager/src/cron.ts index b663e736e..8d5929aed 100644 --- a/services/data-manager/src/cron.ts +++ b/services/data-manager/src/cron.ts @@ -1,27 +1,29 @@ import { createHash, randomUUID } from "node:crypto"; import { existsSync, statSync } from "node:fs"; import { dirname, join } from "node:path"; +import { setTimeout as durableDelay } from "node:timers/promises"; import { readBoundedResponseText } from "@openmapx/core"; +import { + type DataExecutionStatus, + isDataExecutionTerminal, +} from "@openmapx/core/data-manager-contracts"; import { envInt, envString } from "@openmapx/core/server-env"; -import { feedState } from "@openmapx/db-schema"; import { FEED_PROXY_CONFIG_FILENAME, FEED_PROXY_CONFIG_SUBDIR, } from "@openmapx/motis-feed-proxy-config"; -import { parseTransitSource } from "@openmapx/transitous-core"; import { Cron } from "croner"; -import { and, eq } from "drizzle-orm"; import type { FastifyBaseLogger } from "fastify"; +import { z } from "zod/v4"; import { db } from "./db/index.js"; +import type { DataExecutionReceiver } from "./jobs/execution/api.js"; +import { + independentLegacyDataMutation, + withLegacyDataMutation, +} from "./jobs/execution/legacy-mutation.js"; import { createGithubIssueSink, type GithubIssueSink } from "./jobs/github-issue-sink.js"; import { discoverLatestOvertureRelease } from "./jobs/overture/pull.js"; -import { rebuildOvertureLinks } from "./jobs/overture/rebuild-links.js"; -import { - compareOvertureReleases, - finalizeOvertureReleaseFiles, - overtureReleaseRetentionFromEnv, -} from "./jobs/overture/retention.js"; -import { syncOvertureRegion } from "./jobs/overture/sync.js"; +import { compareOvertureReleases } from "./jobs/overture/retention.js"; import { type BakePredictedDeps, type BakePredictedResult, @@ -77,25 +79,17 @@ import { } from "./jobs/traffic/write-live.js"; import { type CatalogBumpCandidate, - candidateMatchesLock, lockFromCandidate, resolveCatalogBumpCandidate, } from "./jobs/transitous/catalog-bump.js"; -import { - buildJobContext, - type RunPipelineResult, - runTransitousPipeline, -} from "./jobs/transitous/index.js"; import { fetchWithTimeout } from "./jobs/transitous/motis-probe.js"; import type { MotisOperationsPolicy } from "./jobs/transitous/operations-profile.js"; -import { finalizeJobRow, makePersistingOnStageComplete } from "./jobs/transitous/persistence.js"; import type { SingleFlightController } from "./jobs/transitous/single-flight.js"; import { detectStaleFeeds, emitFeedAlerts, emitPipelineFailureAlert, } from "./jobs/transitous/staleness-alerts.js"; -import { asJobLogger, jobChildLogger } from "./logger.js"; import { runOpsOperation } from "./ops-client.js"; import type { StateStore } from "./state.js"; import { pickCronExpression, pickOptInCronExpression } from "./utils/cron-expression.js"; @@ -155,8 +149,6 @@ const TRAFFIC_LIVE_FETCH_TIMEOUT_MS = 10_000; */ const TRAFFIC_PREDICTED_CRON_DEFAULT = "0 6 * * 1"; -const AUTO_BUMP_PROPOSAL_COMMENT = - "Candidate Transitous pin under auto-bump canary validation (services/data-manager)."; export interface CronLogger { info: (msg: string, extra?: Record) => void; warn: (msg: string, extra?: Record) => void; @@ -164,22 +156,13 @@ export interface CronLogger { } export interface CronSetupOptions { + executionReceiver?: DataExecutionReceiver; dataDir: string; countries: string[]; operationsPolicy?: MotisOperationsPolicy; store: StateStore; singleFlight: SingleFlightController; logger: FastifyBaseLogger | CronLogger; - /** - * Test seam: invoked instead of the real Transitous pipeline. Production - * callers omit this and get `runTransitousPipeline`. - */ - runPipeline?: (jobId: string) => Promise; - /** - * Test seam: invoked instead of the typed agent operation so the - * unit suite never shells out to docker. - */ - reloadFeedProxy?: (candidateId: string) => Promise; /** Override default schedule strings (e.g. for tests). */ syncCronExpression?: string; feedProxyReloadCronExpression?: string; @@ -208,14 +191,6 @@ export interface CronSetupOptions { discoverOvertureRelease?: () => Promise; /** Test seam: read the release currently installed in Postgres. */ getInstalledOvertureRelease?: () => Promise; - /** Test seam: run the Overture update for a resolved release. */ - syncOvertureRelease?: typeof syncOvertureRegion; - /** Test seam: rebuild links for the installed release without re-importing Places. */ - rebuildOvertureLinks?: typeof rebuildOvertureLinks; - /** Test seam: persist successful Overture release state. */ - writeOvertureFeedState?: typeof writeFeedState; - /** Test seam: finalize successful release-file retention after any completion path. */ - finalizeOvertureReleaseFiles?: typeof finalizeOvertureReleaseFiles; /** Override the traffic-extract guard cron schedule (e.g. for tests). */ trafficExtractCronExpression?: string; /** @@ -344,13 +319,6 @@ export interface CronSetupOptions { catalogDir: string; branch: string; }) => Promise; - /** - * Test seam: run the canary + promote pipeline against the proposed lock. - * Resolves to the pipeline result; rejects when a critical stage (incl. the - * canary) fails. Production callers omit this and get a real pipeline run - * with `useProposedLock: true`. - */ - runBumpPipeline?: (jobId: string) => Promise; } export interface CronHandles { @@ -429,46 +397,6 @@ function asCronLogger(logger: FastifyBaseLogger | CronLogger): CronLogger { return logger as CronLogger; } -async function writeFeedState(region: string, hash: string, log: CronLogger): Promise { - try { - const feedRegion = `overture-places-${region}`; - const feedName = "overture-places"; - const existing = await db - .select({ id: feedState.id }) - .from(feedState) - .where(and(eq(feedState.region, feedRegion), eq(feedState.name, feedName))) - .limit(1); - if (existing[0]) { - await db - .update(feedState) - .set({ - hash, - lastFetchedAt: new Date(), - lastImportedAt: new Date(), - validationStatus: "ok", - validationMessage: `Imported Overture release ${hash}`, - consecutiveFailures: 0, - status: "active", - }) - .where(eq(feedState.id, existing[0].id)); - } else { - await db.insert(feedState).values({ - region: feedRegion, - name: feedName, - hash, - lastFetchedAt: new Date(), - lastImportedAt: new Date(), - validationStatus: "ok", - validationMessage: `Imported Overture release ${hash}`, - consecutiveFailures: 0, - status: "active", - }); - } - } catch (err) { - log.warn("overture-cron: feed_state write failed", { err: (err as Error).message }); - } -} - export async function getInstalledOvertureRelease(): Promise { try { const rows = await db.execute<{ release: string }>( @@ -488,6 +416,15 @@ export async function getInstalledOvertureRelease(): Promise { * are owned by the data-manager process so that they share its inflight lock * (one sync at a time, no matter the trigger). */ +const transitFailureSchema = z.object({ + results: z.array( + z.object({ + stage: z.string(), + status: z.string(), + error: z.object({ message: z.string() }).optional(), + }), + ), +}); export function setupCron(options: CronSetupOptions): CronHandles { const log = asCronLogger(options.logger); const syncExpr = pickCronExpression( @@ -525,103 +462,69 @@ export function setupCron(options: CronSetupOptions): CronHandles { FEED_PROXY_CONFIG_FILENAME, ); - const runSync = async (): Promise => { - const start = await options.singleFlight.tryStartSync({ - trigger: "cron", - triggeredBy: "data-manager-cron", - kind: "transitous-sync", - metadata: { source: "cron" }, - }); - if (!start.ok) { - // Don't escalate: cron firing while a manual sync is in-flight is fine. - log.warn("transitous-cron: skipped scheduled run", { reason: start.reason }); - return; + const receiver = () => { + if (!options.executionReceiver) throw Error("Durable data execution receiver is unavailable"); + return options.executionReceiver; + }; + const observeOwned = async (status: DataExecutionStatus) => { + let current = status; + while (!isDataExecutionTerminal(current) || current.publication === "unknown") { + const remaining = status.deadlineAt - Date.now(); + if (remaining <= 0) + throw Error(`Data execution ${status.id} observation pending at its original deadline`); + await durableDelay(Math.min(250, remaining)); + const found = await receiver().inspect(status.id); + if ( + !found || + found.operationKey !== status.operationKey || + found.fingerprint !== status.fingerprint || + found.admittedAt !== status.admittedAt || + found.deadlineAt !== status.deadlineAt + ) + throw Error(`Data execution ${status.id} requires retained reconciliation`); + current = found; } - log.info("transitous-cron: starting scheduled sync", { jobId: start.jobId }); - let finalStatus: RunPipelineResult["finalStatus"] = "error"; - let pipelineResult: RunPipelineResult | undefined; - let threwReason: string | undefined; + return current; + }; + const requireSuccessful = async (status: DataExecutionStatus) => { + const result = await observeOwned(status); + if (result.status !== "ok" && result.status !== "partial") + throw Error(`Data execution ${result.id} ${result.status}`); + return result; + }; + const runSync = async (): Promise => { try { - let result: RunPipelineResult; - if (options.runPipeline) { - result = await options.runPipeline(start.jobId); - } else { - const jobLog = asJobLogger( - jobChildLogger({ job: "transitous-sync", jobId: start.jobId, trigger: "cron" }), - ); - const ctx = buildJobContext({ - dataDir: options.dataDir, - store: options.store, - countries: options.countries, - source: parseTransitSource(), - operationsPolicy: options.operationsPolicy, - jobId: start.jobId, - logger: jobLog, - onStageComplete: makePersistingOnStageComplete(start.jobId, jobLog), - }); - result = await runTransitousPipeline(ctx); - } - pipelineResult = result; - finalStatus = result.finalStatus; - log.info("transitous-cron: sync completed", { - jobId: start.jobId, - finalStatus, - stageCount: result.results.length, - }); - } catch (err) { - threwReason = (err as Error).message; - log.error("transitous-cron: sync threw", { - jobId: start.jobId, - err: threwReason, - }); - finalStatus = "error"; - } finally { - try { - await finalizeJobRow(start.jobId, finalStatus); - } catch (err) { - log.warn("transitous-cron: finalizeJobRow failed", { - jobId: start.jobId, - err: (err as Error).message, - }); - } - options.singleFlight.markSyncFinished(); - } - - // A hard failure means the canary rejected the candidate (or a critical - // stage errored) and nothing promoted — the live dataset silently ages. - // Per-feed staleness can't catch this in mirror mode (feed_state is written - // before the canary), so surface it as its own pipeline-level alert. - if (finalStatus === "error") { - const failed = pipelineResult?.results.find((stage) => stage.status === "error"); - try { + const admitted = await receiver().admitLocal( + { kind: "transit-sync", countries: options.countries }, + { triggeredBy: "cron" }, + ); + log.info("transitous-cron: queued durable sync", { jobId: admitted.id }); + const result = await observeOwned(admitted); + log.info("transitous-cron: sync settled", { jobId: result.id, status: result.status }); + if (result.status !== "ok") { + const trusted = receiver().legacyResult(result.id), + failure = transitFailureSchema.safeParse(trusted), + failed = failure.success + ? failure.data.results.find((stage) => stage.status === "error") + : undefined; await emitPipelineFailureAlert({ alert: { trigger: "cron", - jobId: start.jobId, + jobId: result.id, failedStage: failed?.stage, - reason: failed?.error?.message ?? failed?.message ?? threwReason ?? "sync failed", + reason: + failed?.error?.message ?? + result.diagnostics.tail.at(-1)?.message ?? + `Transit execution ${result.status}`, }, log, githubIssue: githubIssue ?? undefined, }); - } catch (err) { - log.warn("transitous-cron: pipeline failure alert failed", { - jobId: start.jobId, - err: (err as Error).message, - }); } - } - - // Run the staleness check immediately after every sync so an - // out-of-band failure that would otherwise wait for the daily 04:00 cron - // surfaces at the same wallclock minute as the sync that caused it. - try { - await runStalenessCheck(); } catch (err) { - log.warn("transitous-cron: post-sync staleness check failed", { - err: (err as Error).message, - }); + log.error("transitous-cron: sync remains failed or pending", { err: (err as Error).message }); } + await runStalenessCheck(); }; const defaultStalenessCheck = async (): Promise => { @@ -642,24 +545,6 @@ export function setupCron(options: CronSetupOptions): CronHandles { const catalogDir = join(options.dataDir, ".transitous-catalog"); const resolveBumpCandidate = options.resolveBumpCandidate ?? ((o) => resolveCatalogBumpCandidate(o)); - const defaultRunBumpPipeline = async (jobId: string): Promise => { - const jobLog = asJobLogger( - jobChildLogger({ job: "transitous-auto-bump", jobId, trigger: "cron" }), - ); - const ctx = buildJobContext({ - dataDir: options.dataDir, - store: options.store, - countries: options.countries, - source: parseTransitSource(), - operationsPolicy: options.operationsPolicy, - jobId, - logger: jobLog, - useProposedLock: true, - onStageComplete: makePersistingOnStageComplete(jobId, jobLog), - }); - return runTransitousPipeline(ctx); - }; - const runBumpPipeline = options.runBumpPipeline ?? defaultRunBumpPipeline; /** * Auto-bump the pinned Transitous catalog behind the pipeline's own canary. @@ -669,110 +554,29 @@ export function setupCron(options: CronSetupOptions): CronHandles { * untouched, the proposal is retained for review, and a failure alert fires. */ const runAutoBump = async (): Promise => { - const start = await options.singleFlight.tryStartSync({ - trigger: "cron", - triggeredBy: "data-manager-auto-bump", - kind: "transitous-auto-bump", - metadata: { source: "auto-bump" }, - }); - if (!start.ok) { - log.warn("transitous-auto-bump: skipped, a sync is in-flight", { reason: start.reason }); - return; - } - - let finalStatus: RunPipelineResult["finalStatus"] = "ok"; - let pipelineResult: RunPipelineResult | undefined; - let threwReason: string | undefined; - let ranPipeline = false; - try { - const candidate = await resolveBumpCandidate({ catalogDir, branch: "main" }); - const { active } = await runOpsOperation({ kind: "transitousLock.inspect" }); - if (candidateMatchesLock(candidate, active)) { - log.info("transitous-auto-bump: already at upstream tip; nothing to do", { - ref: candidate.ref, - }); - return; - } - log.info("transitous-auto-bump: proposing candidate pin", { - ref: candidate.ref, - previousRef: active?.ref ?? null, + await withLegacyDataMutation("transit-proposal", async () => { + const candidate = await resolveBumpCandidate({ + catalogDir, + branch: process.env.TRANSITOUS_AUTO_BUMP_BRANCH ?? "master", }); - const proposal = lockFromCandidate(candidate, "auto-bump", AUTO_BUMP_PROPOSAL_COMMENT); - // Agent-owned write: the auto-bump proposes, it never activates. + const { active } = await runOpsOperation({ kind: "transitousLock.inspect" }); + if (active?.ref === candidate.ref) return; + const lock = lockFromCandidate( + candidate, + "data-manager-auto-bump", + "Proposed by the scheduled catalog check; validation-bound activation is required", + ); await runOpsOperation({ kind: "transitousLock.propose", - ref: proposal.ref, - submodules: proposal.submodules, - lockedBy: proposal.lockedBy, - comment: AUTO_BUMP_PROPOSAL_COMMENT, + ref: lock.ref, + submodules: lock.submodules, + lockedBy: lock.lockedBy, + ...(lock.comment ? { comment: lock.comment } : {}), }); - - ranPipeline = true; - pipelineResult = await runBumpPipeline(start.jobId); - finalStatus = pipelineResult.finalStatus; - - if (finalStatus === "ok") { - // Only a passing canary activates, and the agent re-matches the exact - // ref it is activating. - await runOpsOperation({ - kind: "transitousLock.approve", - ref: candidate.ref, - approvedBy: "auto-bump", - }); - log.info("transitous-auto-bump: canary passed; activated new catalog pin", { - ref: candidate.ref, - }); - } else { - log.warn("transitous-auto-bump: canary did not fully pass; keeping current pin", { - ref: candidate.ref, - finalStatus, - }); - } - } catch (err) { - // A canary rejection surfaces here (the pipeline rethrows on a critical - // stage). Keep the active pin and retain the proposal for review. - threwReason = (err as Error).message; - finalStatus = "error"; - log.error("transitous-auto-bump: failed", { err: threwReason }); - } finally { - try { - await finalizeJobRow(start.jobId, finalStatus); - } catch (err) { - log.warn("transitous-auto-bump: finalizeJobRow failed", { - jobId: start.jobId, - err: (err as Error).message, - }); - } - options.singleFlight.markSyncFinished(); - } - - if (ranPipeline && finalStatus !== "ok") { - const failed = pipelineResult?.results.find((stage) => stage.status === "error"); - try { - await emitPipelineFailureAlert({ - alert: { - trigger: "auto-bump", - jobId: start.jobId, - failedStage: failed?.stage, - reason: - failed?.error?.message ?? failed?.message ?? threwReason ?? "auto-bump canary failed", - }, - log, - githubIssue: githubIssue ?? undefined, - }); - } catch (err) { - log.warn("transitous-auto-bump: failure alert failed", { - jobId: start.jobId, - err: (err as Error).message, - }); - } - } - }; - - const defaultReload = async (candidateId: string): Promise => { - // The agent validates the configuration before reloading and owns the - // container name; data-manager only names the candidate. - await runOpsOperation({ kind: "feedProxy.validateAndReload", candidateId }); + log.info( + "transitous-auto-bump: proposal retained; validation-bound activation is unavailable", + ); + }); }; const runFeedProxyReload = async (): Promise => { @@ -794,9 +598,9 @@ export function setupCron(options: CronSetupOptions): CronHandles { return; } try { - const reload = options.reloadFeedProxy ?? defaultReload; - // The config file's mtime identifies the candidate being activated. - await reload(`feedproxy-${Math.trunc(mtimeMs)}`); + await requireSuccessful( + await receiver().admitMaintenance("reload-feed-proxy", { triggeredBy: "cron" }), + ); lastFeedProxyReloadAt = mtimeMs; log.info("transitous-cron: feed-proxy reloaded", { mtimeMs }); } catch (err) { @@ -868,97 +672,41 @@ export function setupCron(options: CronSetupOptions): CronHandles { : null; const runOvertureConflationRetry = async (): Promise => { - const region = - options.overtureRegion ?? (process.env.OPENMAPX_REGION || "europe/germany/berlin"); - const readInstalledRelease = options.getInstalledOvertureRelease ?? getInstalledOvertureRelease; - const rebuildLinks = options.rebuildOvertureLinks ?? rebuildOvertureLinks; - const finalizeReleaseFiles = - options.finalizeOvertureReleaseFiles ?? finalizeOvertureReleaseFiles; + const region = options.overtureRegion ?? process.env.OPENMAPX_REGION ?? "europe/germany/berlin"; try { - const release = await readInstalledRelease(); - if (!release) { - log.info("overture-conflation: no installed Places release; nothing to rebuild", { - region, - }); - return; - } - const result = await rebuildLinks({ - region, - release, - dataDir: options.dataDir, - onProgress: (msg) => log.info(msg), - }); - if (result.status === "failed") { - log.error("overture-conflation: rebuild failed and remains retryable", { - region, - release, - err: result.error, - }); - } else { - if (result.status === "completed" || result.status === "already_completed") { - await finalizeReleaseFiles({ - dataDir: options.dataDir, - activeRelease: release, - retain: overtureReleaseRetentionFromEnv(process.env.OVERTURE_RELEASE_RETENTION), - onProgress: (msg) => log.info(msg), - }); - } - log.info("overture-conflation: retry check complete", { - region, - release, - status: result.status, - linked: result.linked, - }); - } + if (!(await (options.getInstalledOvertureRelease ?? getInstalledOvertureRelease)())) return; + await requireSuccessful( + await receiver().admitLocal({ kind: "overture-conflate", region }, { triggeredBy: "cron" }), + ); } catch (err) { - log.error("overture-conflation: retry check failed", { err: (err as Error).message }); + log.error("overture-conflation: durable retry failed or remains pending", { + err: (err as Error).message, + }); } }; const runOvertureSync = async (): Promise => { - const region = - options.overtureRegion ?? (process.env.OPENMAPX_REGION || "europe/germany/berlin"); - const discoverRelease = options.discoverOvertureRelease ?? discoverLatestOvertureRelease; - const readInstalledRelease = options.getInstalledOvertureRelease ?? getInstalledOvertureRelease; - const syncRelease = options.syncOvertureRelease ?? syncOvertureRegion; - const persistFeedState = options.writeOvertureFeedState ?? writeFeedState; + const region = options.overtureRegion ?? process.env.OPENMAPX_REGION ?? "europe/germany/berlin"; try { - const release = await discoverRelease(); - const installedRelease = await readInstalledRelease(); - if (installedRelease === release) { - log.info("overture-cron: latest Places release already installed; checking links", { - region, - release, - }); + const latest = await (options.discoverOvertureRelease ?? discoverLatestOvertureRelease)(), + installed = await (options.getInstalledOvertureRelease ?? getInstalledOvertureRelease)(); + if (installed && compareOvertureReleases(latest, installed) <= 0) { await runOvertureConflationRetry(); return; } - if (installedRelease && compareOvertureReleases(installedRelease, release) > 0) { - log.warn("overture-cron: installed release is newer than upstream catalog", { - region, - installedRelease, - upstreamRelease: release, + const result = await requireSuccessful( + await receiver().admitLocal({ kind: "overture-sync", region }, { triggeredBy: "cron" }), + ); + if (result.status === "partial") + log.warn("overture-cron: Places imported; link rebuild remains incomplete", { + jobId: result.id, + diagnostics: result.diagnostics, }); - return; - } - - const result = await syncRelease({ - region, - release, - dataDir: options.dataDir, - onProgress: (msg) => log.info(msg), - }); - await persistFeedState(region, release, log); await runStalenessCheck(); - if (result.conflation === "failed") { - log.error("overture-cron: Places imported; link rebuild will retry independently", { - region, - release, - err: result.conflationError, - }); - } } catch (err) { - log.error("overture-cron: sync failed", { err: (err as Error).message }); + log.error("overture-cron: durable sync failed or remains pending", { + err: (err as Error).message, + }); } }; @@ -1009,7 +757,9 @@ export function setupCron(options: CronSetupOptions): CronHandles { } try { const coveredWayIds = await options.getCoveredWayIds(); - const result = await refreshWaysToEdges(coveredWayIds, { logger: log }); + const result = await withLegacyDataMutation("traffic-maintenance", () => + refreshWaysToEdges(coveredWayIds, { logger: log }), + ); log.info("ways-to-edges: refresh complete", { wayCount: result.wayCount, edgeCount: result.edgeCount, @@ -1022,7 +772,9 @@ export function setupCron(options: CronSetupOptions): CronHandles { const runTrafficExtractStartup = async (): Promise => { let graphRebuilt = false; try { - const result = await ensureExtract({ logger: log }); + const result = await withLegacyDataMutation("traffic-maintenance", () => + ensureExtract({ logger: log }), + ); graphRebuilt = result.built; log.info("traffic-extract: startup check complete", { built: result.built }); } catch (err) { @@ -1052,7 +804,9 @@ export function setupCron(options: CronSetupOptions): CronHandles { // it. `force` skips the redundant presence check: we already know the // file exists, just that it's outdated. log.info("traffic-extract: graph tiles newer than traffic.tar, rebuilding"); - const result = await ensureExtract({ logger: log, force: true }); + const result = await withLegacyDataMutation("traffic-maintenance", () => + ensureExtract({ logger: log, force: true }), + ); log.info("traffic-extract: guard rebuild complete", { built: result.built }); } else { log.info("traffic-extract: guard check found no rebuild needed"); @@ -1073,7 +827,10 @@ export function setupCron(options: CronSetupOptions): CronHandles { let graphMaintenanceCheck: Promise | null = null; const scheduleGraphMaintenanceCheck = (): void => { if (graphMaintenanceCheck) return; - graphMaintenanceCheck = runTrafficExtractGuard().finally(() => { + graphMaintenanceCheck = independentLegacyDataMutation( + "traffic-maintenance", + runTrafficExtractGuard, + ).finally(() => { graphMaintenanceCheck = null; }); }; @@ -1262,7 +1019,7 @@ export function setupCron(options: CronSetupOptions): CronHandles { } trafficLiveRunning = true; try { - await runTrafficLiveCycle(); + await withLegacyDataMutation("traffic-live", runTrafficLiveCycle); } finally { trafficLiveRunning = false; } @@ -1409,9 +1166,11 @@ export function setupCron(options: CronSetupOptions): CronHandles { }); // Fire-and-forget: a `valhalla_ways_to_edges` run is far slower than a // 2-minute cycle, and this cycle's write must not wait on it. - void runWaysToEdgesRefresh().catch((err) => { - log.warn("traffic-live: way→edge refresh failed", { err: (err as Error).message }); - }); + void independentLegacyDataMutation("traffic-maintenance", runWaysToEdgesRefresh).catch( + (err) => { + log.warn("traffic-live: way→edge refresh failed", { err: (err as Error).message }); + }, + ); } // Live speeds alone hold the full lease. Written road-event overrides // also hold it to the policy, the conditions set and each contributing @@ -1608,7 +1367,9 @@ export function setupCron(options: CronSetupOptions): CronHandles { return; } try { - const result = await bake({ openConditionsUrl, openConditionsToken, logger: log }); + const result = await withLegacyDataMutation("traffic-predicted", () => + bake({ openConditionsUrl, openConditionsToken, logger: log }), + ); log.info("traffic-predicted: cycle complete", { ...result }); } catch (err) { log.error("traffic-predicted: cycle failed", { err: (err as Error).message }); diff --git a/services/data-manager/src/db/index.ts b/services/data-manager/src/db/index.ts index bac7d9d4c..74a48d518 100644 --- a/services/data-manager/src/db/index.ts +++ b/services/data-manager/src/db/index.ts @@ -9,6 +9,7 @@ import { } from "@openmapx/db-schema"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; +import { withDataSqlCancellation } from "../jobs/execution/worker-scope.js"; const connectionString = process.env.DATABASE_URL ?? "postgresql://postgres:postgres@postgis:5432/openmapx"; @@ -17,10 +18,11 @@ assertProductionDatabaseUrlSecret(connectionString, process.env.NODE_ENV); // `postgres-js` is happy with a small connection pool here — the data-manager // is single-tenant and writes are infrequent (one INSERT per stage). -export const sql = postgres(connectionString, { +export const rawSql = postgres(connectionString, { max: 4, connection: { application_name: "openmapx-data-manager" }, }); +export const sql = withDataSqlCancellation(rawSql); export const db = drizzle(sql, { schema: { jobs, diff --git a/services/data-manager/src/jobs/ambient-places/api.ts b/services/data-manager/src/jobs/ambient-places/api.ts index bfdc98024..a27994653 100644 --- a/services/data-manager/src/jobs/ambient-places/api.ts +++ b/services/data-manager/src/jobs/ambient-places/api.ts @@ -8,6 +8,7 @@ import { readAmbientManifest } from "@openmapx/core/ambient-places-server"; import type { FastifyInstance } from "fastify"; import type postgres from "postgres"; import { registerDataManagerRoute } from "../../route-contract.js"; +import { withLegacyDataMutation } from "../execution/legacy-mutation.js"; import { buildAmbientPlaces, discardPlanetPlaces, @@ -106,16 +107,21 @@ export function registerAmbientPlacesApi(app: FastifyInstance, sql: postgres.Sql accept = resolve; reject = rejectPromise; }); - void job( - () => { - claimed = true; - accept(); - }, - { - onProgress: (value) => { - if (building === attempt) progress = value; - }, - }, + void withLegacyDataMutation( + "ambient-publication", + () => + job( + () => { + claimed = true; + accept(); + }, + { + onProgress: (value) => { + if (building === attempt) progress = value; + }, + }, + ), + sql, ) .catch((error) => { if (!claimed) { @@ -164,7 +170,11 @@ export function registerAmbientPlacesApi(app: FastifyInstance, sql: postgres.Sql ); if (building) return reply.code(409).send({ error: "An ambient build is already running" }); try { - await discardPlanetPlaces(sql, generation); + await withLegacyDataMutation( + "ambient-publication", + () => discardPlanetPlaces(sql, generation), + sql, + ); return { ok: true }; } catch (error) { return reply.code(409).send({ error: (error as Error).message }); @@ -174,7 +184,11 @@ export function registerAmbientPlacesApi(app: FastifyInstance, sql: postgres.Sql } registerDataManagerRoute(app, "setAmbientPlacesEnabled", async (request, reply) => { try { - await setAmbientEnabled(sql, request.body.enabled); + await withLegacyDataMutation( + "ambient-publication", + () => setAmbientEnabled(sql, request.body.enabled), + sql, + ); return { ok: true }; } catch (error) { return reply.code(409).send({ error: (error as Error).message }); @@ -182,7 +196,7 @@ export function registerAmbientPlacesApi(app: FastifyInstance, sql: postgres.Sql }); registerDataManagerRoute(app, "rollbackAmbientPlaces", async (_request, reply) => { try { - await rollbackAmbientPlaces(sql); + await withLegacyDataMutation("ambient-publication", () => rollbackAmbientPlaces(sql), sql); return { ok: true }; } catch (error) { return reply.code(409).send({ error: (error as Error).message }); diff --git a/services/data-manager/src/jobs/atomic-download.ts b/services/data-manager/src/jobs/atomic-download.ts index 639eddc98..eb2f84333 100644 --- a/services/data-manager/src/jobs/atomic-download.ts +++ b/services/data-manager/src/jobs/atomic-download.ts @@ -1,7 +1,10 @@ import { existsSync, renameSync, statSync, unlinkSync } from "node:fs"; import { execa } from "execa"; +import { dataWorkerSignal, publishDataChange } from "./execution/worker-scope.js"; export interface CurlAtomicOptions { + publication?: "data-output"; + signal?: AbortSignal; /** * Invoked roughly every second with the current byte count of the in-flight * temp file and (when available from a HEAD request) the total expected @@ -36,6 +39,8 @@ export async function curlAtomicWithResult( targetPath: string, opts: CurlAtomicOptions = {}, ): Promise { + opts = { ...opts, signal: dataWorkerSignal(opts.signal) }; + opts.signal?.throwIfAborted(); const rand = Math.random().toString(36).slice(2, 10); const tmpPath = `${targetPath}.tmp.${rand}`; @@ -45,13 +50,17 @@ export async function curlAtomicWithResult( let totalBytes: number | undefined; if (opts.onProgress) { try { - const head = await execa("curl", ["-sIL", url], { timeout: 15_000 }); + const head = await execa("curl", ["-sIL", url], { + timeout: 15_000, + cancelSignal: opts.signal, + }); const match = head.stdout.match(/^\s*content-length:\s*(\d+)/im); if (match?.[1]) totalBytes = Number(match[1]); } catch { // ignore — totalBytes stays undefined } } + opts.signal?.throwIfAborted(); // Skip the download entirely if the upstream resource hasn't changed since // we last fetched it. `curl -z ` adds an `If-Modified-Since` header @@ -64,7 +73,11 @@ export async function curlAtomicWithResult( } args.push(url); - const child = execa("curl", args, { stdio: "inherit" }); + const child = execa("curl", args, { + stdio: "inherit", + cancelSignal: opts.signal, + forceKillAfterDelay: 5000, + }); const poll = opts.onProgress ? setInterval(() => { try { @@ -78,6 +91,7 @@ export async function curlAtomicWithResult( try { await child; + opts.signal?.throwIfAborted(); if (poll) clearInterval(poll); // 304 Not Modified: curl writes nothing to the tmp path. Leave the // existing target in place and return without renaming. @@ -104,20 +118,26 @@ export async function curlAtomicWithResult( } } await opts.beforePublish?.(tmpPath); + opts.signal?.throwIfAborted(); let published = false; const publish = (): void => { + opts.signal?.throwIfAborted(); if (published) throw new Error("Atomic download publish callback invoked more than once"); renameSync(tmpPath, targetPath); published = true; }; - if (opts.withPublishLock) { - await opts.withPublishLock(publish); - if (!published) { - throw new Error("Atomic download publish lock resolved without publishing the temp file"); + const commit = async () => { + if (opts.withPublishLock) { + await opts.withPublishLock(publish); + if (!published) { + throw new Error("Atomic download publish lock resolved without publishing the temp file"); + } + } else { + publish(); } - } else { - publish(); - } + }; + if (opts.publication === "data-output") await publishDataChange(commit); + else await commit(); return { published: true }; } catch (err) { if (poll) clearInterval(poll); diff --git a/services/data-manager/src/jobs/convert-overpass.ts b/services/data-manager/src/jobs/convert-overpass.ts index a48764bd4..225de1c27 100644 --- a/services/data-manager/src/jobs/convert-overpass.ts +++ b/services/data-manager/src/jobs/convert-overpass.ts @@ -3,8 +3,10 @@ import { availableParallelism } from "node:os"; import { dirname, join } from "node:path"; import { execa } from "execa"; import type { StateStore } from "../state.js"; +import { publishDataChange } from "./execution/worker-scope.js"; interface ConvertOverpassOptions { + signal?: AbortSignal; /** State-store-resolved input .osm.pbf path. */ sourcePbf: string; /** Canonical output .osm.bz2 path under `dataDir`. */ @@ -14,6 +16,7 @@ interface ConvertOverpassOptions { } async function convertPbfToBz2(opts: ConvertOverpassOptions): Promise { + opts.signal?.throwIfAborted(); mkdirSync(dirname(opts.targetBz2), { recursive: true }); // Overpass wants bz2-compressed OSM XML (`planet.osm.bz2`). libosmium's bz2 @@ -24,12 +27,20 @@ async function convertPbfToBz2(opts: ConvertOverpassOptions): Promise { // `data.osm.bz2` behind. const threads = Math.max(2, availableParallelism()); const temporary = `${opts.targetBz2}.tmp`; + const pipeFailure = new AbortController(); + const signal = opts.signal + ? AbortSignal.any([opts.signal, pipeFailure.signal]) + : pipeFailure.signal; const osmium = execa("osmium", ["cat", opts.sourcePbf, "-f", "osm", "-o", "-"], { stderr: "inherit", + cancelSignal: signal, + forceKillAfterDelay: 5000, }); const compressed = osmium.pipe("lbzip2", ["-n", String(threads), "-c"], { stdout: { file: temporary }, stderr: "inherit", + cancelSignal: signal, + forceKillAfterDelay: 5000, }); const poll = opts.onProgress @@ -43,8 +54,19 @@ async function convertPbfToBz2(opts: ConvertOverpassOptions): Promise { : null; try { - await compressed; - renameSync(temporary, opts.targetBz2); + const contain = async (child: PromiseLike) => { + try { + return await child; + } catch (error) { + pipeFailure.abort(error); + throw error; + } + }; + const outcomes = await Promise.allSettled([contain(osmium), contain(compressed)]); + const failed = outcomes.find((result) => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; + opts.signal?.throwIfAborted(); + await publishDataChange(() => renameSync(temporary, opts.targetBz2)); } catch (error) { rmSync(temporary, { force: true }); throw error; @@ -61,6 +83,7 @@ async function convertPbfToBz2(opts: ConvertOverpassOptions): Promise { } export interface ConvertOverpassForRegionOptions { + signal?: AbortSignal; /** * OSM region id (e.g. `europe/germany`, or `planet`). When absent, the * single most-recently-downloaded `osm-pbf` entry in the state store is @@ -127,6 +150,7 @@ export async function convertPbfToBz2ForRegion( // makes a misleading progress target. Let the CLI fall back to its // unknown-total renderer (bytes + rate, no bar, no ETA). await convertPbfToBz2({ + signal: opts.signal, sourcePbf: picked.path, targetBz2, onProgress: opts.onProgress ? (bytes) => opts.onProgress?.(bytes) : undefined, diff --git a/services/data-manager/src/jobs/download-fonts.ts b/services/data-manager/src/jobs/download-fonts.ts index b53ccd594..50c83e606 100644 --- a/services/data-manager/src/jobs/download-fonts.ts +++ b/services/data-manager/src/jobs/download-fonts.ts @@ -11,6 +11,7 @@ import { join } from "node:path"; import { execa } from "execa"; import type { StateStore } from "../state.js"; import { curlAtomic } from "./atomic-download.js"; +import { assertDataPublication, publishDataChange } from "./execution/worker-scope.js"; export const DEFAULT_OPENMAPTILES_FONTS_URL = "https://github.com/openmaptiles/fonts/releases/download/v2.0/v2.0.zip"; @@ -48,12 +49,14 @@ function containsGlyphPbf(root: string): boolean { } export interface DownloadFontsOptions { + signal?: AbortSignal; dataDir: string; store: StateStore; } /** Download and atomically replace the glyph tree used online and by offline packages. */ export async function downloadFonts(opts: DownloadFontsOptions): Promise { + opts.signal?.throwIfAborted(); mkdirSync(opts.dataDir, { recursive: true }); const url = resolveFontAssetUrl(); const fontsDir = join(opts.dataDir, "tile-fonts"); @@ -63,21 +66,29 @@ export async function downloadFonts(opts: DownloadFontsOptions): Promise { const backup = join(stagingRoot, "previous"); let previousMoved = false; let installedNew = false; + let retainRecovery = false; try { - await curlAtomic(url, fontsZip); - const archive = await execa("unzip", ["-Z1", fontsZip]); + await curlAtomic(url, fontsZip, { signal: opts.signal }); + const archive = await execa("unzip", ["-Z1", fontsZip], { cancelSignal: opts.signal }); validateFontArchiveEntries(archive.stdout.split("\n").filter(Boolean)); - await execa("unzip", ["-qo", fontsZip, "-d", stagingFonts], { stdio: "inherit" }); + await execa("unzip", ["-qo", fontsZip, "-d", stagingFonts], { + stdio: "inherit", + cancelSignal: opts.signal, + forceKillAfterDelay: 5000, + }); if (!containsGlyphPbf(stagingFonts)) throw new Error("font archive contains no glyph PBF files"); const sizeBytes = statSync(fontsZip).size; - if (existsSync(fontsDir)) { - renameSync(fontsDir, backup); - previousMoved = true; - } - renameSync(stagingFonts, fontsDir); - installedNew = true; + opts.signal?.throwIfAborted(); + await publishDataChange(() => { + if (existsSync(fontsDir)) { + renameSync(fontsDir, backup); + previousMoved = true; + } + renameSync(stagingFonts, fontsDir); + installedNew = true; + }); opts.store.upsert({ type: "tile-fonts", id: "openmaptiles-v2", @@ -91,10 +102,21 @@ export async function downloadFonts(opts: DownloadFontsOptions): Promise { installedNew = false; if (removeBackup) rmSync(backup, { recursive: true, force: true }); } catch (error) { - if (installedNew && existsSync(fontsDir)) rmSync(fontsDir, { recursive: true, force: true }); - if (previousMoved && existsSync(backup)) renameSync(backup, fontsDir); + // A completed replacement survives later observer/store failures. A lost + // ownership/publication checkpoint never authorizes another rename. + if (installedNew) retainRecovery = true; + else if (previousMoved && existsSync(backup)) { + try { + opts.signal?.throwIfAborted(); + assertDataPublication(); + renameSync(backup, fontsDir); + previousMoved = false; + } catch { + retainRecovery = true; + } + } throw error; } finally { - rmSync(stagingRoot, { recursive: true, force: true }); + if (!retainRecovery) rmSync(stagingRoot, { recursive: true, force: true }); } } diff --git a/services/data-manager/src/jobs/download-osm.ts b/services/data-manager/src/jobs/download-osm.ts index 9631bdfbe..6f91e2ef4 100644 --- a/services/data-manager/src/jobs/download-osm.ts +++ b/services/data-manager/src/jobs/download-osm.ts @@ -51,29 +51,34 @@ export function resolveOsmMd5Url(osmUrl: string): string { return `${osmUrl}.md5`; } -function computeMd5(filePath: string): Promise { +function computeMd5(filePath: string, signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { const hash = createHash("md5"); - const stream = createReadStream(filePath); + const stream = createReadStream(filePath, { signal }); stream.on("data", (chunk) => hash.update(chunk)); stream.on("end", () => resolve(hash.digest("hex"))); stream.on("error", reject); }); } -async function fetchExpectedMd5(url: string): Promise { +async function fetchExpectedMd5(url: string, signal?: AbortSignal): Promise { try { - const { stdout } = await execa("curl", ["-fsSL", url], { timeout: 30_000 }); + const { stdout } = await execa("curl", ["-fsSL", url], { + timeout: 30_000, + cancelSignal: signal, + }); // ` ` — take the first whitespace-delimited token. const token = stdout.trim().split(/\s+/)[0]; if (token && /^[0-9a-f]{32}$/i.test(token)) return token.toLowerCase(); } catch { + signal?.throwIfAborted(); // Sidecar not published or curl failed; treat as "no verification available". } return null; } export interface DownloadOsmOptions { + signal?: AbortSignal; region: string; dataDir: string; store: StateStore; @@ -102,6 +107,7 @@ export interface DownloadOsmResult { } export async function downloadOsm(opts: DownloadOsmOptions): Promise { + opts.signal?.throwIfAborted(); const url = resolveOsmUrl(opts.region); const fileName = osmPbfName(opts.region); const targetDir = join(opts.dataDir, "osm"); @@ -110,14 +116,18 @@ export async function downloadOsm(opts: DownloadOsmOptions): Promise { if (expected) { - const actual = await computeMd5(tempPath); + const actual = await computeMd5(tempPath, opts.signal); if (actual !== expected) { throw new Error( `OSM PBF checksum mismatch for ${opts.region}: expected ${expected}, got ${actual}. The download is corrupt — re-run to retry.`, @@ -131,7 +141,7 @@ export async function downloadOsm(opts: DownloadOsmOptions): Promise { + if (!maintenance) + throw Object.assign(Error("Data admission prerequisites are unavailable"), { statusCode: 503 }); + const lock = await tx`SELECT pg_try_advisory_xact_lock(${0x4f4d58},${2}) AS held`; + if (lock[0]?.held !== true) + throw Object.assign(Error("configuration_maintenance_busy"), { statusCode: 409 }); + await maintenance.assertAdmission(tx); +} diff --git a/services/data-manager/src/jobs/execution/api.test.ts b/services/data-manager/src/jobs/execution/api.test.ts new file mode 100644 index 000000000..8180babf0 --- /dev/null +++ b/services/data-manager/src/jobs/execution/api.test.ts @@ -0,0 +1,51 @@ +import Fastify from "fastify"; +import { afterEach, expect, it } from "vitest"; +import { registerApi } from "../../api.js"; +import { registerAuth } from "../../auth.js"; + +const apps: ReturnType[] = []; +afterEach(async () => { + await Promise.all(apps.splice(0).map((app) => app.close())); +}); +it("keeps observation and mutation routes closed without a durable receiver", async () => { + const app = Fastify(); + apps.push(app); + registerAuth(app, "synthetic-execution-token"); + registerApi(app, { dataDir: "/tmp/openmapx-nonexistent-execution-fixture" }); + const headers = { authorization: "Bearer synthetic-execution-token" }; + const id = "00000000-0000-4000-8000-000000000001"; + for (const request of [ + { + method: "POST" as const, + url: "/executions", + payload: { operationKey: "opk1_abcdefghijklmnop", input: { kind: "download-fonts" } }, + }, + { method: "GET" as const, url: `/executions/${id}` }, + { method: "GET" as const, url: "/executions/by-key/opk1_abcdefghijklmnop" }, + { + method: "POST" as const, + url: `/executions/${id}/cancel`, + payload: { operationKey: "opk1_abcdefghijklmnop", fingerprint: "a".repeat(64) }, + }, + { method: "GET" as const, url: `/executions/configuration-maintenance/${id}` }, + ]) { + const result = await app.inject({ ...request, headers }); + expect(result.statusCode, result.body).toBe(503); + expect((await app.inject(request)).statusCode).toBe(401); + } + const privateInput = await app.inject({ + method: "POST", + url: "/executions", + headers, + payload: { + operationKey: "opk1_abcdefghijklmnop", + input: { + kind: "generation-hardlinks", + revisionId: `cfg1_${"a".repeat(43)}`, + planDigest: "a".repeat(64), + coordinatorMode: "require-running", + }, + }, + }); + expect(privateInput.statusCode).toBe(400); +}); diff --git a/services/data-manager/src/jobs/execution/api.ts b/services/data-manager/src/jobs/execution/api.ts new file mode 100644 index 000000000..1578e5f0b --- /dev/null +++ b/services/data-manager/src/jobs/execution/api.ts @@ -0,0 +1,109 @@ +import type { + DataConfigurationFencePrepare, + DataConfigurationFenceReceipt, + DataConfigurationFenceSettle, + DataExecutionIdentity, + DataExecutionInput, + DataExecutionStatus, + startGenerationHardlinksExecutionSchema, +} from "@openmapx/core/data-manager-contracts"; +import type { FastifyInstance, FastifyReply } from "fastify"; +import type z from "zod/v4"; +import { registerDataManagerRoute } from "../../route-contract.js"; + +/** The receiver performs signed admission/adoption, separately from bearer + * authentication. A missing receiver never authorizes a compatibility worker. */ +export interface DataExecutionReceiver { + admitLocal( + input: DataExecutionInput, + options?: { sourceJobId?: string; triggeredBy?: "cron" | "bootstrap" }, + ): Promise; + admitMaintenance( + action: "restart-primary" | "reload-feed-proxy", + options?: { triggeredBy: "cron" }, + ): Promise; + subscribe(id: string, onEvent: (event: Record) => void): () => void; + legacyResult(id: string): Record | undefined; + admit( + input: DataExecutionInput, + options: { operationKey: string; requestedDeadlineAt?: number; rawHeaders: readonly string[] }, + ): Promise; + admitHardlinks( + request: z.infer, + rawHeaders: readonly string[], + ): Promise; + inspect(id: string): Promise; + lookup(operationKey: string): Promise; + requestCancel(identity: DataExecutionIdentity): Promise; + prepareFence( + input: DataConfigurationFencePrepare, + rawHeaders: readonly string[], + ): Promise; + inspectFence( + maintenanceId: string, + rawHeaders: readonly string[], + ): Promise; + settleFence( + input: DataConfigurationFenceSettle, + rawHeaders: readonly string[], + ): Promise; +} +function unavailable(reply: FastifyReply) { + return reply.code(503).send({ error: "Durable data execution receiver is unavailable" }); +} +export function registerDataExecutionApi( + app: FastifyInstance, + receiver?: DataExecutionReceiver, +): void { + registerDataManagerRoute(app, "startDataExecution", async (request, reply) => { + if (!receiver) return unavailable(reply); + const { input, ...admission } = request.body; + return reply + .code(202) + .send(await receiver.admit(input, { ...admission, rawHeaders: request.raw.rawHeaders })); + }); + registerDataManagerRoute(app, "startGenerationHardlinksExecution", async (request, reply) => { + if (!receiver) return unavailable(reply); + return reply + .code(202) + .send(await receiver.admitHardlinks(request.body, request.raw.rawHeaders)); + }); + registerDataManagerRoute(app, "prepareDataConfigurationFence", async (request, reply) => { + if (!receiver) return unavailable(reply); + return receiver.prepareFence(request.body, request.raw.rawHeaders); + }); + registerDataManagerRoute(app, "inspectDataConfigurationFence", async (request, reply) => { + if (!receiver) return unavailable(reply); + const receipt = await receiver.inspectFence( + request.params.maintenanceId, + request.raw.rawHeaders, + ); + return ( + receipt ?? reply.code(404).send({ error: "Configuration maintenance identity is unknown" }) + ); + }); + registerDataManagerRoute(app, "settleDataConfigurationFence", async (request, reply) => { + if (!receiver) return unavailable(reply); + if (request.params.maintenanceId !== request.body.maintenanceId) + return reply.code(409).send({ error: "Configuration maintenance identity conflict" }); + return receiver.settleFence(request.body, request.raw.rawHeaders); + }); + registerDataManagerRoute(app, "lookupDataExecution", async (request, reply) => { + if (!receiver) return unavailable(reply); + return ( + (await receiver.lookup(request.params.operationKey)) ?? + reply.code(404).send({ error: "Execution identity is unknown" }) + ); + }); + registerDataManagerRoute(app, "getDataExecution", async (request, reply) => { + if (!receiver) return unavailable(reply); + return ( + (await receiver.inspect(request.params.id)) ?? + reply.code(404).send({ error: "Execution identity is unknown" }) + ); + }); + registerDataManagerRoute(app, "cancelDataExecution", async (request, reply) => { + if (!receiver) return unavailable(reply); + return receiver.requestCancel({ id: request.params.id, ...request.body }); + }); +} diff --git a/services/data-manager/src/jobs/execution/configuration-maintenance.test.ts b/services/data-manager/src/jobs/execution/configuration-maintenance.test.ts new file mode 100644 index 000000000..da55a3f2c --- /dev/null +++ b/services/data-manager/src/jobs/execution/configuration-maintenance.test.ts @@ -0,0 +1,686 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + createControlRootOperationKey, + type ExecutionRef, + executionFingerprint, +} from "@openmapx/core/ops"; +import { + feedState, + jobStages, + jobs, + offlinePackageArtifactReferences, + offlinePackageJobOwners, + offlinePackageJobs, +} from "@openmapx/db-schema"; +import Fastify from "fastify"; +import postgres from "postgres"; +import { describe, expect, it } from "vitest"; +import { registerApi } from "../../api.js"; +import { registerAuth } from "../../auth.js"; +import { setupCron } from "../../cron.js"; +import { PostgresOfflinePackageAccountingStore } from "../../offline-packages/postgres-accounting.js"; +import type { OfflinePackageJobRecord } from "../../offline-packages/types.js"; +import { StateStore } from "../../state.js"; +import { createJobRow } from "../persistence.js"; +import { resolveOperationsProfile } from "../transitous/operations-profile.js"; +import { createSingleFlightController } from "../transitous/single-flight.js"; +import { installDataMutationAdmission } from "./admission.js"; +import { createDataConfigurationMaintenance } from "./configuration-maintenance.js"; +import { createDataExecutionDispatcher } from "./dispatch.js"; +import { startLegacyDataMutation } from "./legacy-mutation.js"; +import { createDataExecutionReceiver } from "./receiver.js"; +import { createDataExecutionRuntime } from "./runtime.js"; +import { createPostgresDataExecutionStore } from "./store.js"; + +describe.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)("configuration fence real PostgreSQL and retained grants", () => { + it("prepares a terminal logical guard, blocks concurrent admissions, and restores it before new mutation", async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }), + root = mkdtempSync(join(tmpdir(), "omx-data-fence-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const services = ["data-manager", "motis"], + agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: services, + }), + dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: services, + }); + const now = Date.now(), + deadlineAt = now + 60000, + rootBody = { kind: "configurationMaintenance.prepare", maintenanceId: randomUUID() }, + operationFingerprint = createHash("sha256").update(canonicalJson(rootBody)).digest("hex"); + const owner: ExecutionRef = { + executor: "ops-agent", + principal: "api", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: rootBody.kind, operationFingerprint, deadlineAt }), + }; + const input = { + maintenanceId: rootBody.maintenanceId, + candidateId: randomUUID(), + ownerPrepare: owner, + planDigest: "1".repeat(64), + serviceIds: services, + revisionId: `cfg1_${"a".repeat(43)}`, + hardlinkPlanDigest: null, + coordinatorMode: "require-running" as const, + }; + const body = { kind: "configuration-maintenance.prepare", ...input }, + bodyHash = createHash("sha256").update(canonicalJson(body)).digest("hex"); + const child: ExecutionRef = { + executor: "data-manager", + principal: "api", + executionId: randomUUID(), + operationKey: control.controlChildOperationKey(owner.operationKey, "configuration-fence", 0), + operationFingerprint: bodyHash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: body.kind, + operationFingerprint: bodyHash, + deadlineAt, + parent: control.retainedControlParent(owner), + }), + }; + const plan = { + phase: "configuration-fence" as const, + ordinal: 0, + kind: body.kind, + execution: child, + serviceIds: services, + recipient: "data-manager" as const, + }; + const authority = control.retainAdmittedControlExecution(agent, { + execution: owner, + kind: rootBody.kind, + operation: rootBody, + admittedAt: now, + requestedDeadlineAt: deadlineAt, + serviceIds: services, + plan: [plan], + }); + const lease = await control.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: services, + owner: authority, + }); + const assertion = await control.delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: dm.publicKey, + }); + try { + const fence = createDataConfigurationMaintenance(sql, dm); + await fence.startup(); + const receipt = await fence.prepare(input, ["x-openmapx-control-delegation", assertion]); + expect(receipt).toMatchObject({ fenceId: child.executionId, state: "prepared", sequence: 0 }); + const rows = + await sql`SELECT status,finished_at,metadata FROM data_manager.jobs WHERE id=${child.executionId}`; + expect(rows[0]?.status).toBe("ok"); + expect(rows[0]?.finished_at).toBeInstanceOf(Date); + const store = createPostgresDataExecutionStore(sql), + deadline = Date.now() + 30000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + await expect( + store.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ), + ).rejects.toThrow("configuration_maintenance_busy"); + installDataMutationAdmission(fence); + let dataEffects = 0; + const actualDb = drizzle(sql, { + schema: { + jobs, + jobStages, + feedState, + offlinePackageJobs, + offlinePackageJobOwners, + offlinePackageArtifactReferences, + }, + }), + singleFlight = createSingleFlightController({ db: actualDb, admissionSql: sql }), + journal = control.createDataExecutionControlJournal(dm, sql), + ownedStore = createPostgresDataExecutionStore(sql, { + controlJournal: journal, + configurationFence: fence, + }), + runtime = createDataExecutionRuntime({ + store: ownedStore, + dispatch: async () => { + dataEffects++; + throw Error("A prepared guard must reject before dispatch"); + }, + }), + operationsPolicy = resolveOperationsProfile({ countries: ["de"] }), + dispatcher = createDataExecutionDispatcher({ + dataDir: join(root, "data"), + state: new StateStore(join(root, "data")), + sql, + singleFlight, + operationsPolicy, + runHardlinks: async () => { + dataEffects++; + throw Error("Closed generation continuation"); + }, + }), + receiver = createDataExecutionReceiver({ + runtime, + installation: dm, + configurationFence: fence, + dispatcher, + operationsPolicy, + }), + app = Fastify(); + registerAuth(app, "guard-fixture-token", { executionReceiver: () => receiver }); + registerApi(app, { + executionReceiver: receiver, + dataDir: join(root, "data"), + singleFlight, + operationsPolicy, + searchIndexSql: sql, + bakePredicted: async () => { + dataEffects++; + throw Error("Guard must precede bake"); + }, + }); + const beforeMirror = readFileSync( + join( + root, + "control-private/data-manager/configuration-maintenance", + `${input.maintenanceId}.json`, + ), + ), + beforeRows = + await sql`SELECT id,metadata,status,finished_at FROM data_manager.jobs ORDER BY id`; + try { + for (const [url, payload] of [ + ["/download/fonts", {}], + ["/download/osm", { region: "europe/germany/berlin" }], + ["/search-index/build", { region: "europe/germany/berlin" }], + ["/notable-places/build", {}], + ["/transit/sync", { countries: ["de"] }], + ["/transit/restart-motis", {}], + ["/traffic/predicted/bake", {}], + ["/datasets/reload", {}], + ] as const) { + const response = await app.inject({ + method: "POST", + url, + payload, + headers: { authorization: "Bearer guard-fixture-token" }, + }); + expect(response.statusCode, url + response.body).toBe(409); + } + await expect( + singleFlight.tryStartSync({ trigger: "cron", triggeredBy: "fixture" }), + ).rejects.toThrow("configuration_maintenance_busy"); + await expect(createJobRow({ kind: "traffic-predicted" }, sql)).rejects.toThrow( + "configuration_maintenance_busy", + ); + await expect( + startLegacyDataMutation( + "traffic-predicted", + async () => { + dataEffects++; + }, + sql, + ), + ).rejects.toThrow("configuration_maintenance_busy"); + const offline = new PostgresOfflinePackageAccountingStore(sql), + candidate: OfflinePackageJobRecord = { + jobId: randomUUID(), + request: { + request: { + bbox: { west: 1, south: 1, east: 2, north: 2 }, + minZoom: 1, + maxZoom: 2, + provider: "openmapx", + }, + effective: { bbox: { west: 1, south: 1, east: 2, north: 2 }, minZoom: 1, maxZoom: 2 }, + source: { + datasetId: "openmapx", + datasetVersion: "fixture", + sourceMaxZoom: 2, + sourceBounds: { west: 0, south: 0, east: 3, north: 3 }, + tileSchema: "openmaptiles", + glyphsVersion: "fixture", + packageAlgorithmVersion: "pmtiles-area-v1", + attribution: ["Public fixture"], + }, + requestKey: "guard-fixture", + }, + status: "preparing", + packageId: `omp2-${"a".repeat(64)}`, + createdAtMs: Date.now(), + updatedAtMs: Date.now(), + }; + await expect( + offline.admit("a".repeat(64), candidate, { + readManifest: async () => { + dataEffects++; + return undefined; + }, + }), + ).rejects.toThrow("configuration_maintenance_busy"); + const cron = setupCron({ + dataDir: join(root, "data"), + countries: ["de"], + store: new StateStore(join(root, "data")), + singleFlight, + executionReceiver: receiver, + operationsPolicy, + logger: { info: () => {}, warn: () => {}, error: () => {} }, + syncCronExpression: "disabled", + feedProxyReloadCronExpression: "disabled", + stalenessCheckCronExpression: "disabled", + trafficExtractCronExpression: "disabled", + trafficLiveCronExpression: "disabled", + trafficPredictedCronExpression: "disabled", + overtureCronExpression: "disabled", + overtureConflationRetryCronExpression: "disabled", + runStalenessCheck: async () => {}, + }); + try { + await cron.runSyncNow(); + } finally { + cron.stop(); + } + expect(dataEffects).toBe(0); + expect( + await sql`SELECT id,metadata,status,finished_at FROM data_manager.jobs ORDER BY id`, + ).toEqual(beforeRows); + expect( + readFileSync( + join( + root, + "control-private/data-manager/configuration-maintenance", + `${input.maintenanceId}.json`, + ), + ), + ).toEqual(beforeMirror); + } finally { + await app.close(); + await runtime.close(); + journal.close(); + } + const restarted = createDataConfigurationMaintenance(sql, dm); + await restarted.startup(); + await expect(restarted.withAdmission(async () => undefined)).rejects.toThrow( + "configuration_maintenance_busy", + ); + await expect( + fence.prepare({ ...input, candidateId: randomUUID() }, [ + "x-openmapx-control-delegation", + assertion, + ]), + ).rejects.toThrow(); + expect(await fence.prepare(input, ["x-openmapx-control-delegation", assertion])).toEqual( + receipt, + ); + expect(control.inspectArtifactTargetLeaseChildSettlement(lease, child)?.publication).toBe( + "committed", + ); + await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); + } finally { + await sql`DELETE FROM data_manager.jobs WHERE id=${child.executionId}`; + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(dm); + control.closeControlInstallation(agent); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, 60000); +}); + +import { existsSync, readFileSync } from "node:fs"; +import { drizzle } from "drizzle-orm/postgres-js"; + +async function ownedFenceFixture() { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + drizzle(sql); + const root = mkdtempSync(join(tmpdir(), "omx-fence-race-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const services = ["data-manager", "motis"], + agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: [...services, "foreign-consumer"].sort(), + }), + dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: [...services, "foreign-consumer"].sort(), + }), + now = Date.now(), + deadlineAt = now + 60000, + operation = { kind: "configurationMaintenance.prepare", maintenanceId: randomUUID() }, + bodyHash = createHash("sha256").update(canonicalJson(operation)).digest("hex"), + owner: ExecutionRef = { + executor: "ops-agent", + principal: "api", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint: bodyHash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: bodyHash, + deadlineAt, + }), + }; + const hardlinkPlanDigest = createHash("sha256") + .update(canonicalJson({ plan: [], prune: true })) + .digest("hex"), + input = { + maintenanceId: operation.maintenanceId, + candidateId: randomUUID(), + ownerPrepare: owner, + planDigest: "a".repeat(64), + serviceIds: services, + revisionId: `cfg1_${"a".repeat(43)}`, + hardlinkPlanDigest, + coordinatorMode: "require-running" as const, + }, + prepare = { kind: "configuration-maintenance.prepare", ...input }, + fingerprint = createHash("sha256").update(canonicalJson(prepare)).digest("hex"), + child: ExecutionRef = { + executor: "data-manager", + principal: "api", + executionId: randomUUID(), + operationKey: control.controlChildOperationKey(owner.operationKey, "configuration-fence", 0), + operationFingerprint: fingerprint, + deadlineAt, + fingerprint: executionFingerprint({ + kind: prepare.kind, + operationFingerprint: fingerprint, + deadlineAt, + parent: control.retainedControlParent(owner), + }), + }, + slot = { + phase: "configuration-fence" as const, + ordinal: 0, + kind: prepare.kind, + execution: child, + serviceIds: services, + recipient: "data-manager" as const, + }; + const authority = control.retainAdmittedControlExecution(agent, { + execution: owner, + kind: operation.kind, + operation, + admittedAt: now, + requestedDeadlineAt: deadlineAt, + serviceIds: services, + plan: [slot], + }), + lease = await control.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: services, + owner: authority, + }), + assertion = await control.delegateArtifactTargetLease(lease, { + ...slot, + childPublicKey: dm.publicKey, + }); + return { + sql, + root, + agent, + dm, + owner, + child, + authority, + lease, + assertion, + input, + async close() { + await sql`DELETE FROM data_manager.jobs WHERE id=${child.executionId}`; + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agent); + control.closeControlInstallation(dm); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + }, + }; +} +it.skipIf(!process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL)( + "retains a half-written signed guard before insert and blocks a racing real admission on another connection", + async () => { + const f = await ownedFenceFixture(); + let release = () => {}, + pause = () => {}; + const held = new Promise((resolve) => { + release = resolve; + }), + ready = new Promise((resolve) => { + pause = resolve; + }); + const fence = control.createDataConfigurationFenceStore(f.dm, { + unsafe: (query, parameters) => f.sql.unsafe(query, parameters), + async transaction(run) { + const result = await f.sql.begin(async (tx) => ({ + value: await run({ + unsafe: async (query, parameters) => { + if (query.startsWith("INSERT INTO data_manager.jobs")) { + pause(); + await held; + throw Error("Synthetic process crash before guard insert"); + } + return tx.unsafe(query, parameters); + }, + }), + })); + return result.value; + }, + }); + await fence.startup(); + const attempt = fence.prepare(f.input, ["x-openmapx-control-delegation", f.assertion]); + const rejected = expect(attempt).rejects.toThrow("Synthetic process crash"); + try { + await ready; + const deadline = Date.now() + 30000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + store = createPostgresDataExecutionStore(f.sql); + await expect( + store.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ), + ).rejects.toThrow("configuration_maintenance_busy"); + expect(await store.lookup(key)).toBeNull(); + expect(await f.sql`SELECT id FROM data_manager.jobs WHERE id=${f.child.executionId}`).toEqual( + [], + ); + release(); + await rejected; + const restarted = createDataConfigurationMaintenance(f.sql, f.dm); + await expect(restarted.startup()).rejects.toThrow(/Half-written/); + await expect(restarted.withAdmission(async () => undefined)).rejects.toThrow( + "configuration_maintenance_busy", + ); + expect( + existsSync( + join( + f.root, + "control-private/data-manager/configuration-maintenance", + `${f.input.maintenanceId}.json`, + ), + ), + ).toBe(true); + expect(control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.child)).toBeNull(); + await expect(control.releaseArtifactTargetLease(f.lease)).rejects.toThrow(); + } finally { + release(); + await attempt.catch(() => {}); + await f.close(); + } + }, + 60000, +); +it.skipIf(!process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL)( + "rejects signed foreign generation owners and changed revision/hash/coordinator without changing the prepared guard", + async () => { + const f = await ownedFenceFixture(), + fence = createDataConfigurationMaintenance(f.sql, f.dm), + host = control.openControlInstallation({ + infraDir: f.root, + role: "host-cli", + serviceIds: ["foreign-consumer"], + }); + let foreignLease: control.ArtifactTargetLease | undefined; + try { + await fence.startup(); + await fence.prepare(f.input, ["x-openmapx-control-delegation", f.assertion]); + const mirror = join( + f.root, + "control-private/data-manager/configuration-maintenance", + `${f.input.maintenanceId}.json`, + ), + original = readFileSync(mirror), + before = + await f.sql`SELECT metadata,ownership_version FROM data_manager.jobs WHERE id=${f.child.executionId}`; + const now = Date.now(), + deadline = now + 30000, + operation = { kind: "data.link" as const }, + opHash = createHash("sha256").update(canonicalJson(operation)).digest("hex"), + owner: ExecutionRef = { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + operationFingerprint: opHash, + deadlineAt: deadline, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: opHash, + deadlineAt: deadline, + }), + }, + plan = { plan: [], prune: true as const }; + const requests = [ + { + revisionId: f.input.revisionId, + planDigest: f.input.hardlinkPlanDigest, + coordinatorMode: "require-running" as const, + }, + { + revisionId: `cfg1_${"b".repeat(43)}`, + planDigest: f.input.hardlinkPlanDigest, + coordinatorMode: "require-running" as const, + }, + { + revisionId: f.input.revisionId, + planDigest: "b".repeat(64), + coordinatorMode: "require-running" as const, + }, + { + revisionId: f.input.revisionId, + planDigest: f.input.hardlinkPlanDigest, + coordinatorMode: "ensure-running" as const, + }, + ].map((body, ordinal) => { + const input = { kind: "generation-hardlinks" as const, ...body }, + hash = createHash("sha256").update(canonicalJson(input)).digest("hex"), + execution: ExecutionRef = { + executor: "data-manager", + principal: "host-cli", + executionId: randomUUID(), + operationKey: control.controlChildOperationKey( + owner.operationKey, + "hardlinks", + ordinal, + ), + operationFingerprint: hash, + deadlineAt: deadline, + fingerprint: executionFingerprint({ + kind: input.kind, + operationFingerprint: hash, + deadlineAt: deadline, + parent: control.retainedControlParent(owner), + }), + }; + return { + input, + plan, + slot: { + phase: "hardlinks" as const, + ordinal, + kind: input.kind, + recipient: "data-manager" as const, + execution, + serviceIds: ["foreign-consumer"], + }, + }; + }); + const authority = control.retainAdmittedControlExecution(host, { + execution: owner, + kind: operation.kind, + operation, + admittedAt: now, + requestedDeadlineAt: deadline, + serviceIds: ["foreign-consumer"], + plan: requests.map((request) => request.slot), + }); + foreignLease = await control.acquireArtifactTargetLease({ + dataRoot: join(f.root, "data"), + serviceIds: ["foreign-consumer"], + owner: authority, + }); + for (const request of requests) { + const assertion = await control.delegateArtifactTargetLease(foreignLease, { + ...request.slot, + childPublicKey: f.dm.publicKey, + }); + await expect( + fence.assertGenerationAdmission( + { + operationKey: request.slot.execution.operationKey, + requestedDeadlineAt: deadline, + input: request.input, + plan, + }, + ["x-openmapx-control-delegation", assertion], + ), + ).rejects.toMatchObject({ statusCode: 403 }); + expect(await f.sql`SELECT id FROM data_manager.jobs WHERE execution_version=2`).toEqual([]); + } + expect(readFileSync(mirror)).toEqual(original); + expect( + await f.sql`SELECT metadata,ownership_version FROM data_manager.jobs WHERE id=${f.child.executionId}`, + ).toEqual(before); + await expect( + fence.withAdmission(async () => { + throw Error("No data effects may run"); + }), + ).rejects.toThrow("configuration_maintenance_busy"); + } finally { + if (foreignLease) control.closeArtifactTargetLeaseHandle(foreignLease); + control.closeControlInstallation(host); + await f.close(); + } + }, + 60000, +); diff --git a/services/data-manager/src/jobs/execution/configuration-maintenance.ts b/services/data-manager/src/jobs/execution/configuration-maintenance.ts new file mode 100644 index 000000000..4b3cca84b --- /dev/null +++ b/services/data-manager/src/jobs/execution/configuration-maintenance.ts @@ -0,0 +1,22 @@ +import { + type ControlInstallation, + createDataConfigurationFenceStore, +} from "@openmapx/core/build-artifacts"; +import type postgres from "postgres"; +/** The adapter keeps every guard check and mutation on the exact transaction's + * connection. It exposes no HTTP lock or caller-provided ownership boolean. */ +export function createDataConfigurationMaintenance( + sql: postgres.Sql, + installation: ControlInstallation, +) { + return createDataConfigurationFenceStore(installation, { + unsafe: (query, parameters) => sql.unsafe(query, parameters), + async transaction(run) { + const result = await sql.begin(async (tx) => ({ + value: await run({ unsafe: (query, parameters) => tx.unsafe(query, parameters) }), + })); + return result.value; + }, + }); +} +export type DataConfigurationMaintenance = ReturnType; diff --git a/services/data-manager/src/jobs/execution/diagnostics.test.ts b/services/data-manager/src/jobs/execution/diagnostics.test.ts new file mode 100644 index 000000000..5c92d6b08 --- /dev/null +++ b/services/data-manager/src/jobs/execution/diagnostics.test.ts @@ -0,0 +1,16 @@ +import { expect, it } from "vitest"; +import { dataExecutionDiagnostic } from "./diagnostics"; + +it("scrubs credentials and host paths before bounding complete UTF-8 characters", () => { + const value = dataExecutionDiagnostic( + "\u001b[31mAuthorization: Bearer synthetic-secret /Users/operator/private/data C:\\private\\artifact " + + "🗺".repeat(2000), + ); + expect(value.message).not.toContain("synthetic-secret"); + expect(value.message).not.toContain("/Users/"); + expect(value.message).not.toContain("C:\\private"); + expect(value.message).not.toContain("\u001b"); + expect(value.message).not.toContain("�"); + expect(Buffer.byteLength(value.message)).toBeLessThanOrEqual(4096); + expect(value.truncated).toBe(true); +}); diff --git a/services/data-manager/src/jobs/execution/diagnostics.ts b/services/data-manager/src/jobs/execution/diagnostics.ts new file mode 100644 index 000000000..82bf9e9ba --- /dev/null +++ b/services/data-manager/src/jobs/execution/diagnostics.ts @@ -0,0 +1,19 @@ +import { stripVTControlCharacters } from "node:util"; +import { scrubSecrets } from "@openmapx/core/scrub-secrets"; +/** Operator-safe status text. Rich worker artifacts stay in their trusted store. */ +export function dataExecutionDiagnostic(value: string): { message: string; truncated: boolean } { + const safe = scrubSecrets(stripVTControlCharacters(value)) + // biome-ignore lint/suspicious/noControlCharactersInRegex: remove C0/C1 diagnostic controls. + .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/g, "") + .replace(/(^|[\s("'=])\/(?:[^\s"'<>]+)/g, "$1[host-path]") + .replace(/\b[A-Za-z]:\\[^\s"'<>]+/g, "[host-path]"); + let message = "", + bytes = 0; + for (const character of safe) { + const width = Buffer.byteLength(character); + if (bytes + width > 4096) break; + message += character; + bytes += width; + } + return { message, truncated: message.length < safe.length }; +} diff --git a/services/data-manager/src/jobs/execution/dispatch.ts b/services/data-manager/src/jobs/execution/dispatch.ts new file mode 100644 index 000000000..12bec1c27 --- /dev/null +++ b/services/data-manager/src/jobs/execution/dispatch.ts @@ -0,0 +1,349 @@ +import { + DataExecutionPartialResult, + retainOwnedDataTransitCandidate, +} from "@openmapx/core/build-artifacts"; +import type { + DataExecutionInput, + DataExecutionResult, +} from "@openmapx/core/data-manager-contracts"; +import type postgres from "postgres"; +import { asJobLogger, jobChildLogger } from "../../logger.js"; +import type { StateStore } from "../../state.js"; +import { convertPbfToBz2ForRegion } from "../convert-overpass.js"; +import { downloadFonts } from "../download-fonts.js"; +import { downloadOsm } from "../download-osm.js"; +import { buildNotablePlaces } from "../notable-places/build.js"; +import { notablePlacesSettings } from "../notable-places/schedule.js"; +import { createNotablePlacesRuntimeState } from "../notable-places/state.js"; +import { extractOsmPois } from "../overture/extract-osm-pois.js"; +import { writeOvertureFeedState } from "../overture/feed-state.js"; +import { ingestOverture } from "../overture/ingest.js"; +import { withOvertureOperationLock } from "../overture/operation-lock.js"; +import { assertValidRegion, pullOverture } from "../overture/pull.js"; +import { + getOvertureConflationState, + rebuildOvertureLinksUnlocked, +} from "../overture/rebuild-links.js"; +import { + finalizeOvertureReleaseFiles, + overtureReleaseRetentionFromEnv, +} from "../overture/retention.js"; +import { syncOvertureRegion } from "../overture/sync.js"; +import { buildOsmSearchIndex } from "../search-index/build.js"; +import { + createNotablePlacesOperationLock, + createSearchIndexOperationLock, +} from "../search-index/operation-lock.js"; +import { + createSearchIndexRuntimeState, + fingerprintDataset, + updateCurrentSearchIndexFingerprint, +} from "../search-index/state.js"; +import type { MotisOperationsPolicy } from "../transitous/operations-profile.js"; +import { getOperatorFeedRelayStore } from "../transitous/operator-feed-relay.js"; +import { makePersistingOnStageComplete } from "../transitous/persistence.js"; +import { buildJobContext, runTransitousPipeline } from "../transitous/pipeline.js"; +import type { SingleFlightController } from "../transitous/single-flight.js"; +import { dataExecutionDiagnostic } from "./diagnostics.js"; +import { runOwnedDataOpsOperation } from "./ops-children.js"; +import type { DataExecutionContext } from "./runtime.js"; +import { + DataPublicationUnknown, + publishDataChange, + retainDataPublication, +} from "./worker-scope.js"; + +type Event = Record; +export function createDataExecutionDispatcher(deps: { + dataDir: string; + state: StateStore; + sql: postgres.Sql; + singleFlight: SingleFlightController; + operationsPolicy: MotisOperationsPolicy; + runHardlinks: (context: DataExecutionContext) => Promise; + notablePlaces?: { + runtimeState: ReturnType; + operationLock: ReturnType; + }; +}) { + const listeners = new Map void>>(), + done = new Map(); + const searchState = createSearchIndexRuntimeState(), + searchLock = createSearchIndexOperationLock(deps.sql), + notableState = deps.notablePlaces?.runtimeState ?? createNotablePlacesRuntimeState(), + notableLock = deps.notablePlaces?.operationLock ?? createNotablePlacesOperationLock(deps.sql); + function emit(id: string, event: Event) { + for (const fn of listeners.get(id) ?? []) fn(event); + } + function finish(id: string, result: Event) { + done.set(id, result); + while (done.size > 1024) done.delete(done.keys().next().value!); + } + async function dispatch( + input: DataExecutionInput, + context: DataExecutionContext, + ): Promise { + if ("region" in input) assertValidRegion(input.region); + const common = { + dataDir: deps.dataDir, + signal: context.signal, + onProgress: (message: string) => + emit(context.execution.executionId, { event: "progress", message }), + }; + switch (input.kind) { + case "download-osm": { + const result = await downloadOsm({ + ...common, + region: input.region, + store: deps.state, + onProgress: (bytes, totalBytes) => + emit(context.execution.executionId, { event: "progress", bytes, totalBytes }), + withPublishLock: (publish) => + withOvertureOperationLock(async () => { + context.assertForward(); + publish(); + }), + }); + const dataset = deps.state + .getAll() + .find((entry) => entry.type === "osm-pbf" && (entry.region ?? entry.id) === input.region); + if (dataset) + try { + await updateCurrentSearchIndexFingerprint( + deps.sql, + input.region, + await fingerprintDataset(dataset), + ); + } catch (error) { + if (error instanceof DataPublicationUnknown) throw error; + jobChildLogger({ job: "download-osm", jobId: context.execution.executionId }).warn( + "download-osm: could not update current search-index fingerprint", + ); + } + await retainDataPublication(); + finish(context.execution.executionId, { event: "done", ok: true, ...result }); + return { + kind: input.kind, + region: input.region, + sizeBytes: result.sizeBytes, + ...(result.md5 ? { md5: result.md5 } : {}), + }; + } + case "download-fonts": + await downloadFonts({ ...common, store: deps.state }); + await retainDataPublication(); + finish(context.execution.executionId, { event: "done", ok: true }); + return { kind: input.kind, completed: true }; + case "convert-overpass": { + const result = await convertPbfToBz2ForRegion({ + ...common, + region: input.region, + store: deps.state, + onProgress: (bytes, totalBytes) => + emit(context.execution.executionId, { event: "progress", bytes, totalBytes }), + }); + await retainDataPublication(); + finish(context.execution.executionId, { event: "done", ok: true, ...result }); + return { kind: input.kind, region: input.region, sizeBytes: result.sizeBytes }; + } + case "transit-sync": { + if ( + deps.operationsPolicy.profile !== "planet" && + input.countries.some((country) => !deps.operationsPolicy.countries.includes(country)) + ) + throw Error("Countries outside configured operations profile"); + if (!deps.singleFlight.reserveOwned(context.execution.executionId)) + throw Error("Transit reservation is busy"); + const logger = asJobLogger( + jobChildLogger({ job: "transit-sync", jobId: context.execution.executionId }), + ); + try { + const ctx = await publishDataChange(() => + buildJobContext({ + dataDir: deps.dataDir, + store: deps.state, + countries: input.countries, + source: deps.operationsPolicy.acquisition, + operationsPolicy: { ...deps.operationsPolicy, countries: input.countries }, + jobId: context.execution.executionId, + logger, + abortSignal: context.signal, + executionContext: context, + operatorFeedRelay: getOperatorFeedRelayStore(), + onStageComplete: async (stage) => { + await makePersistingOnStageComplete(context.execution.executionId, logger)(stage); + const diagnostic = dataExecutionDiagnostic(stage.message ?? stage.stage); + await context.checkpoint({ + diagnostics: { + tail: [{ stream: "stdout", message: diagnostic.message }], + truncated: diagnostic.truncated, + }, + }); + }, + }), + ); + const result = await runTransitousPipeline(ctx); + finish(context.execution.executionId, { + event: "done", + ok: result.finalStatus !== "error", + ...result, + }); + const sourceRevision = ctx.state.transitousLock?.ref ?? "unavailable"; + if (sourceRevision === "unavailable") + throw Error("Transit source revision was not retained"); + return { + kind: input.kind, + finalStatus: result.finalStatus === "skipped" ? "error" : result.finalStatus, + sourceRevision, + countries: input.countries, + }; + } finally { + deps.singleFlight.markSyncFinished(context.execution.executionId); + } + } + case "overture-pull": { + const path = await pullOverture({ ...common, region: input.region }); + finish(context.execution.executionId, { event: "done", ok: true, path }); + return { kind: input.kind, region: input.region, metrics: {} }; + } + case "overture-ingest": + await withOvertureOperationLock(() => ingestOverture({ ...common, region: input.region })); + finish(context.execution.executionId, { event: "done", ok: true }); + return { kind: input.kind, region: input.region, metrics: {} }; + case "overture-extract": + await withOvertureOperationLock(() => extractOsmPois({ ...common, region: input.region })); + finish(context.execution.executionId, { event: "done", ok: true }); + return { kind: input.kind, region: input.region, metrics: {} }; + case "overture-conflate": { + const result = await withOvertureOperationLock(async () => { + const installed = await getOvertureConflationState(); + if (!installed) throw Error("No installed Overture release"); + const result = await rebuildOvertureLinksUnlocked({ + ...common, + region: input.region, + release: installed.release, + force: input.restart === true, + ollamaUrl: process.env.OLLAMA_URL ?? "http://local-ai:11434", + useEmbeddings: false, + }); + if (result.status === "completed" || result.status === "already_completed") + await finalizeOvertureReleaseFiles({ + dataDir: deps.dataDir, + activeRelease: installed.release, + retain: overtureReleaseRetentionFromEnv(process.env.OVERTURE_RELEASE_RETENTION), + onProgress: common.onProgress, + }); + return result; + }); + finish(context.execution.executionId, { + event: "done", + ok: result.status !== "failed" && result.status !== "waiting_for_osm", + ...result, + }); + if ( + result.status === "failed" || + result.status === "waiting_for_osm" || + result.status === "already_running" + ) + throw Error("Overture conflation did not complete"); + return { kind: input.kind, region: input.region, metrics: { processed: result.linked } }; + } + case "overture-sync": { + const result = await syncOvertureRegion({ ...common, region: input.region }); + await writeOvertureFeedState( + input.region, + result.release, + asJobLogger( + jobChildLogger({ job: "overture-sync", jobId: context.execution.executionId }), + ), + ); + const incomplete = ["failed", "waiting_for_osm", "already_running"].includes( + result.conflation, + ); + finish(context.execution.executionId, { event: "done", ok: !incomplete, ...result }); + const safe = { + kind: input.kind, + region: input.region, + release: result.release, + metrics: { processed: result.linked }, + }; + if (incomplete) { + const diagnostic = dataExecutionDiagnostic( + result.conflationError ?? `Conflation ${result.conflation}`, + ); + await context.checkpoint({ + diagnostics: { + tail: [{ stream: "stderr", message: diagnostic.message }], + truncated: diagnostic.truncated, + }, + }); + throw new DataExecutionPartialResult(safe); + } + return safe; + } + case "search-index-build": { + const result = await buildOsmSearchIndex({ + ...common, + region: input.region, + store: deps.state, + sql: deps.sql, + runtimeState: searchState, + operationLock: searchLock, + onProgress: (progress) => + emit(context.execution.executionId, { event: "progress", ...progress }), + }); + finish(context.execution.executionId, { event: "done", ok: true, ...result }); + return { + kind: input.kind, + region: input.region, + metrics: { processed: result.placeCount }, + }; + } + case "notable-places-build": { + const result = await buildNotablePlaces({ + ...notablePlacesSettings(), + sql: deps.sql, + signal: context.signal, + runtimeState: notableState, + operationLock: notableLock, + onProgress: (progress) => + emit(context.execution.executionId, { event: "progress", ...progress }), + }); + finish(context.execution.executionId, { event: "done", ok: true, ...result }); + return { kind: input.kind, metrics: { processed: result.placeCount } }; + } + case "generation-hardlinks": + return deps.runHardlinks(context); + case "transit-maintenance": { + const slot = (await context.record()).reservations[0]; + if (!slot) throw Error("Maintenance child reservation missing"); + if (input.action === "reload-feed-proxy") + retainOwnedDataTransitCandidate(context.authority, slot); + await runOwnedDataOpsOperation(slot.operation, { + context, + phase: slot.phase, + ordinal: slot.ordinal, + serviceIds: slot.serviceIds, + }); + return { kind: input.kind, action: input.action, changed: true }; + } + } + } + return { + dispatch, + searchIndexRuntimeState: searchState, + legacyResult: (id: string) => done.get(id), + subscribe(id: string, fn: (event: Event) => void) { + let group = listeners.get(id); + if (!group) { + group = new Set(); + listeners.set(id, group); + } + group.add(fn); + return () => { + group.delete(fn); + if (!group.size) listeners.delete(id); + }; + }, + }; +} diff --git a/services/data-manager/src/jobs/execution/fixed-cli.test.ts b/services/data-manager/src/jobs/execution/fixed-cli.test.ts new file mode 100644 index 000000000..12a0ee46c --- /dev/null +++ b/services/data-manager/src/jobs/execution/fixed-cli.test.ts @@ -0,0 +1,251 @@ +import { execFileSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + createControlRootOperationKey, + normalizeExecutionAdmission, + OpsClient, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { build } from "esbuild"; +import Fastify from "fastify"; +import postgres from "postgres"; +import { expect, it } from "vitest"; +import { + createAdministrativeRuntime, + createDefaultFixedCli, +} from "../../../../../apps/ops-agent/src/administrative-runtime"; +import { openOpsJobJournal } from "../../../../../apps/ops-agent/src/journal"; +import { createUnavailableRuntime } from "../../../../../apps/ops-agent/src/runtime"; +import { buildOpsAgentServer } from "../../../../../apps/ops-agent/src/server"; +import { registerApi } from "../../api"; +import { registerAuth } from "../../auth"; +import { StateStore } from "../../state"; +import { resolveOperationsProfile } from "../transitous/operations-profile"; +import { getSingleFlightController } from "../transitous/runtime"; +import { createDataConfigurationMaintenance } from "./configuration-maintenance"; +import { createDataExecutionDispatcher } from "./dispatch"; +import { createDataExecutionReceiver } from "./receiver"; +import { createDataExecutionRuntime } from "./runtime"; +import { createPostgresDataExecutionStore } from "./store"; + +it.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)( + "runs the actual fixed single data command through durable authenticated DM admission after a lost response", + async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const root = mkdtempSync(join(tmpdir(), "omx-fixed-data-cli-")), + infra = join(root, "infra/docker"), + data = join(infra, "data"); + mkdirSync(data, { recursive: true, mode: 0o700 }); + await control.provisionControlInstallation(infra); + const envKeys = [ + "OPENMAPX_ROOT_DIR", + "DATA_MANAGER_URL", + "DATA_MANAGER_AUTH_TOKEN", + "OPENMAPTILES_FONTS_URL", + ] as const, + prior = Object.fromEntries(envKeys.map((key) => [key, process.env[key]])); + process.env.OPENMAPX_ROOT_DIR = root; + const assets = join(root, "font-source/Test Font"); + mkdirSync(assets, { recursive: true }); + writeFileSync(join(assets, "0-255.pbf"), "synthetic-public-glyph"); + execFileSync("zip", ["-qr", join(root, "fonts.zip"), "Test Font"], { + cwd: join(root, "font-source"), + }); + const archive = readFileSync(join(root, "fonts.zip")); + let downloads = 0; + const fontServer = createServer((_req, res) => { + downloads++; + res.writeHead(200, { "content-type": "application/zip", "content-length": archive.length }); + res.end(archive); + }); + await new Promise((resolve) => fontServer.listen(0, "127.0.0.1", resolve)); + const fontAddress = fontServer.address(); + if (!fontAddress || typeof fontAddress === "string") + throw Error("Synthetic font endpoint missing"); + process.env.OPENMAPTILES_FONTS_URL = `http://127.0.0.1:${fontAddress.port}/fonts.zip`; + await build({ + entryPoints: [resolve("packages/cli/__tests__/fixtures/single-data-cli-process.ts")], + outfile: join(root, "cli.mjs"), + bundle: true, + platform: "node", + format: "esm", + target: "node24", + tsconfig: resolve("packages/cli/tsconfig.json"), + banner: { + js: `import {createRequire} from 'node:module';const require=createRequire(${JSON.stringify(resolve("packages/cli/package.json"))});`, + }, + }); + const sql = postgres(url, { max: 4 }), + dm = control.openControlInstallation({ + infraDir: infra, + role: "data-manager", + serviceIds: ["data-manager"], + }), + agentInstallation = control.openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["data-manager"], + }), + journal = control.createDataExecutionControlJournal(dm, sql), + fence = createDataConfigurationMaintenance(sql, dm); + await fence.startup(); + const store = createPostgresDataExecutionStore(sql, { + controlJournal: journal, + configurationFence: fence, + }), + singleFlight = getSingleFlightController(), + operationsPolicy = resolveOperationsProfile({ countries: ["de"] }), + dispatcher = createDataExecutionDispatcher({ + dataDir: data, + state: new StateStore(data), + sql, + singleFlight, + operationsPolicy, + runHardlinks: async () => { + throw Error("No generation grant"); + }, + }), + runtime = createDataExecutionRuntime({ store, dispatch: dispatcher.dispatch }), + receiver = createDataExecutionReceiver({ + runtime, + installation: dm, + configurationFence: fence, + dispatcher, + operationsPolicy, + }); + const app = Fastify(); + let admissions = 0; + app.addHook("onSend", async (request, reply, payload) => { + if (request.method === "POST" && request.url === "/executions") { + admissions++; + if (admissions === 1) reply.raw.destroy(); + } + return payload; + }); + registerAuth(app, "synthetic-dm-token", { executionReceiver: () => receiver }); + registerApi(app, { + dataDir: data, + singleFlight, + operationsPolicy, + executionReceiver: receiver, + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("DM address missing"); + process.env.DATA_MANAGER_URL = `http://127.0.0.1:${address.port}`; + process.env.DATA_MANAGER_AUTH_TOKEN = "synthetic-dm-token"; + const agentJournal = await openOpsJobJournal(join(data, "ops-agent/jobs-v1.json")), + agentRuntime = createAdministrativeRuntime(createUnavailableRuntime(), { + rootDir: root, + runFixedCli: createDefaultFixedCli(root, join(root, "agent.mjs")), + }), + token = "a".repeat(43), + tokenFile = join(root, "api-token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + const agent = buildOpsAgentServer({ + runtime: agentRuntime, + journal: agentJournal, + controlInstallation: agentInstallation, + tokens: { api: token, "data-manager": "d".repeat(43) }, + resourcePolicy: { allowGlobal: () => true, allowService: () => true }, + }); + await agent.listen({ host: "127.0.0.1", port: 0 }); + const agentAddress = agent.server.address(); + if (!agentAddress || typeof agentAddress === "string") throw Error("Agent address missing"); + const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agentAddress.port}`, tokenFile }); + let id: string | undefined; + try { + const operation = { kind: "data.downloadFonts" as const }, + admittedAt = Date.now(), + deadline = admittedAt + 60000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + normalized = normalizeExecutionAdmission({ + kind: operation.kind, + operationKey: key, + operationFingerprint: opsOperationFingerprint(operation), + admittedAt, + requestedDeadlineAt: deadline, + }); + const admitted = await client.execute(operation, { + operationKey: key, + execution: { requestedDeadlineAt: deadline }, + }); + expect(admitted.execution).toBe("async"); + if (admitted.execution !== "async") throw Error("Async root required"); + id = admitted.operationId; + try { + await expect + .poll(async () => await client.status(operation.kind, id!), { + timeout: 25000, + interval: 200, + }) + .toMatchObject({ state: "succeeded", fingerprint: normalized.fingerprint }); + } catch (error) { + console.info( + "Synthetic fixed-command diagnostics", + await client.events(id!), + await client.status(operation.kind, id!), + ); + throw error; + } + expect(downloads).toBe(1); + expect(admissions).toBe(1); + expect(readFileSync(join(data, "tile-fonts/Test Font/0-255.pbf"), "utf8")).toBe( + "synthetic-public-glyph", + ); + const rows = (await store.listUnsettled()).filter((row) => + existsSync( + join(infra, "control-private/data-manager/executions", `${row.status.operationKey}.json`), + ), + ); + expect(rows).toHaveLength(1); + expect(rows[0]?.status.status).toBe("ok"); + expect(rows[0]?.parent?.issuer).toBe("host-cli"); + expect(rows[0]?.execution.principal).toBe("host-cli"); + control.verifyDataControlTerminalWitness( + agentInstallation, + rows[0]!.execution, + rows[0]!.status, + ); + expect(existsSync(join(data, ".admin-target-control/locks/data-manager"))).toBe(false); + } finally { + await runtime.close(); + agent.server.closeAllConnections(); + await agent.close(); + app.server.closeAllConnections(); + await app.close(); + fontServer.closeAllConnections(); + await new Promise((resolve) => fontServer.close(() => resolve())); + for (const row of await store.listUnsettled()) + if ( + existsSync( + join( + infra, + "control-private/data-manager/executions", + `${row.status.operationKey}.json`, + ), + ) + ) + await sql`DELETE FROM data_manager.jobs WHERE id=${row.status.id}`; + journal.close(); + control.closeControlInstallation(dm); + control.closeControlInstallation(agentInstallation); + await sql.end(); + for (const key of envKeys) { + if (prior[key] === undefined) delete process.env[key]; + else process.env[key] = prior[key]; + } + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); diff --git a/services/data-manager/src/jobs/execution/hardlinks.test.ts b/services/data-manager/src/jobs/execution/hardlinks.test.ts new file mode 100644 index 000000000..ef99bc530 --- /dev/null +++ b/services/data-manager/src/jobs/execution/hardlinks.test.ts @@ -0,0 +1,310 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + createDataManagerTransport, + lookupDataExecution, + observeDataExecution, + startGenerationHardlinksExecution, + unwrapDataManagerResponse, +} from "@openmapx/core/data-manager-sdk"; +import { + canonicalJson, + createControlRootOperationKey, + type ExecutionRef, + executionFingerprint, + OpsClient, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import Fastify from "fastify"; +import postgres from "postgres"; +import { expect, it } from "vitest"; +import { registerApi } from "../../api"; +import { registerAuth } from "../../auth"; +import { resetOpsClient } from "../../ops-client"; +import { StateStore } from "../../state"; +import { resolveOperationsProfile } from "../transitous/operations-profile"; +import { getSingleFlightController } from "../transitous/runtime"; +import { createDataConfigurationMaintenance } from "./configuration-maintenance"; +import { createDataExecutionDispatcher } from "./dispatch"; +import { applyOwnedGenerationHardlinks } from "./hardlinks"; +import { createDataExecutionReceiver } from "./receiver"; +import { createDataExecutionRuntime } from "./runtime"; +import { createPostgresDataExecutionStore } from "./store"; + +it + .skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", + ) + .each(["ordinary", "restart"] as const)( + "observes an ordinary signed generation leaf after response loss without repeat pruning or container lifecycle (%s)", + async (mode) => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }), + root = mkdtempSync(join(tmpdir(), "omx-generation-leaf-")), + data = join(root, "data"); + mkdirSync(join(data, "src"), { recursive: true }); + writeFileSync(join(data, "src/fixture"), "public-artifact"); + await control.provisionControlInstallation(root); + const services = ["consumer", "data-manager"], + host = control.openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: services, + }), + dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: services, + }), + journal = control.createDataExecutionControlJournal(dm, sql), + fence = createDataConfigurationMaintenance(sql, dm); + await fence.startup(); + const now = Date.now(), + deadline = now + 60000, + operation = { kind: "data.link" as const }, + owner: ExecutionRef = { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + operationFingerprint: opsOperationFingerprint(operation), + deadlineAt: deadline, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: opsOperationFingerprint(operation), + deadlineAt: deadline, + }), + }; + const plan = { + plan: [ + { source: "src", target: "consumer", consumerService: "consumer", dataType: "fixture" }, + ], + prune: true as const, + }, + planDigest = createHash("sha256").update(canonicalJson(plan)).digest("hex"), + input = { + kind: "generation-hardlinks" as const, + revisionId: `cfg1_${"a".repeat(43)}`, + planDigest, + coordinatorMode: "require-running" as const, + }, + bodyHash = createHash("sha256").update(canonicalJson(input)).digest("hex"), + child: ExecutionRef = { + executor: "data-manager", + principal: "host-cli", + executionId: randomUUID(), + operationKey: control.controlChildOperationKey(owner.operationKey, "hardlinks", 0), + operationFingerprint: bodyHash, + deadlineAt: deadline, + fingerprint: executionFingerprint({ + kind: input.kind, + operationFingerprint: bodyHash, + deadlineAt: deadline, + parent: control.retainedControlParent(owner), + }), + }, + slot = { + phase: "hardlinks" as const, + ordinal: 0, + kind: input.kind, + recipient: "data-manager" as const, + execution: child, + serviceIds: services, + }; + const authority = control.retainAdmittedControlExecution(host, { + execution: owner, + kind: operation.kind, + operation, + admittedAt: now, + requestedDeadlineAt: deadline, + serviceIds: services, + plan: [slot], + }), + lease = await control.acquireArtifactTargetLease({ + dataRoot: data, + serviceIds: services, + owner: authority, + }), + assertion = await control.delegateArtifactTargetLease(lease, { + ...slot, + childPublicKey: dm.publicKey, + }); + let lifecycle = 0, + applies = 0; + const tokenFile = join(root, "token"); + writeFileSync(tokenFile, "a".repeat(43), { mode: 0o600 }); + resetOpsClient( + new OpsClient({ + baseUrl: "http://127.0.0.1:1", + tokenFile, + fetch: async () => { + lifecycle++; + throw Error("Generation leaf cannot call host lifecycle"); + }, + }), + ); + const store = createPostgresDataExecutionStore(sql, { + controlJournal: journal, + configurationFence: fence, + }), + singleFlight = getSingleFlightController(), + operationsPolicy = resolveOperationsProfile({ countries: ["de"] }), + dispatcher = createDataExecutionDispatcher({ + dataDir: data, + state: new StateStore(data), + sql, + singleFlight, + operationsPolicy, + runHardlinks: async (context) => { + applies++; + return applyOwnedGenerationHardlinks(context); + }, + }), + runtime = createDataExecutionRuntime({ store, dispatch: dispatcher.dispatch }), + receiver = createDataExecutionReceiver({ + runtime, + installation: dm, + configurationFence: fence, + dispatcher, + operationsPolicy, + }), + app = Fastify(); + const prepareTerminal = journal.prepareTerminal.bind(journal); + let failTerminal = mode === "restart"; + journal.prepareTerminal = async (...args) => { + if (failTerminal) { + failTerminal = false; + throw Error("Synthetic crash before borrowed terminal preparation"); + } + return prepareTerminal(...args); + }; + let restarted: ReturnType | undefined; + let posts = 0; + app.addHook("onSend", async (request, reply, payload) => { + if (request.url === "/executions/generation-hardlinks" && request.method === "POST") { + posts++; + reply.raw.destroy(); + } + return payload; + }); + registerAuth(app, "synthetic-gen-token", { executionReceiver: () => receiver }); + registerApi(app, { + dataDir: data, + singleFlight, + operationsPolicy, + executionReceiver: receiver, + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("Generation endpoint missing"); + const transport = createDataManagerTransport({ + baseUrl: `http://127.0.0.1:${address.port}`, + authToken: "synthetic-gen-token", + }); + try { + await expect(receiver.admitLocal({ kind: "transit-sync", countries: [] })).rejects.toThrow( + "Countries outside configured operations profile", + ); + await expect( + startGenerationHardlinksExecution( + { operationKey: child.operationKey, requestedDeadlineAt: deadline, input, plan }, + { transport, headers: { "x-openmapx-control-delegation": assertion } }, + ), + ).rejects.toThrow(); + const admitted = unwrapDataManagerResponse( + await lookupDataExecution(child.operationKey, { transport }), + ); + expect(admitted.id).toBe(child.executionId); + if (mode === "restart") { + await expect.poll(() => failTerminal, { timeout: 15000 }).toBe(false); + await expect + .poll(async () => (await store.lookup(child.operationKey))?.status.publication, { + timeout: 15000, + }) + .toBe("unknown"); + await runtime.close(); + const recovered = await journal.restoreCompletion(child); + expect(recovered.lease.mode).toBe("borrowed"); + expect(() => control.assertArtifactTargetLeaseForward(recovered.lease)).toThrow(); + control.closeArtifactTargetLeaseHandle(recovered.lease); + restarted = createDataExecutionRuntime({ + store, + dispatch: async () => { + throw Error("Borrowed restart must not repeat hardlink/prune"); + }, + }); + await restarted.reconcile(); + await expect + .poll(async () => (await store.lookup(child.operationKey))?.status.status, { + timeout: 15000, + }) + .toBe("ok"); + expect(existsSync(join(data, ".admin-target-control/locks/data-manager"))).toBe(true); + } + const terminal = await observeDataExecution( + { id: admitted.id, operationKey: child.operationKey, fingerprint: child.fingerprint }, + { transport, observerDeadlineAt: deadline }, + ); + expect(terminal).toMatchObject({ + status: "ok", + result: { kind: "generation-hardlinks", hardlinks: { linked: 1, via: "data-manager" } }, + }); + await expect + .poll(() => control.inspectArtifactTargetLeaseChildSettlement(lease, child)?.publication, { + timeout: 10000, + }) + .toBe("committed"); + const inode = statSync(join(data, "consumer/fixture")).ino; + unlinkSync(join(data, "src/fixture")); + const observed = unwrapDataManagerResponse( + await lookupDataExecution(child.operationKey, { transport }), + ); + expect(observed).toEqual(terminal); + expect(statSync(join(data, "consumer/fixture")).ino).toBe(inode); + expect(readFileSync(join(data, "consumer/fixture"), "utf8")).toBe("public-artifact"); + expect(applies).toBe(1); + expect(posts).toBe(1); + expect(lifecycle).toBe(0); + expect((await store.inspect(child.executionId))?.children).toEqual([]); + control.retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "ops-result", result: { completed: true } }, + }); + await control.settleArtifactTargetLeaseChild( + lease, + control.verifyControlEffectSettlement(host, owner), + ); + await control.releaseArtifactTargetLease(lease); + } finally { + await runtime.close(); + await restarted?.close(); + app.server.closeAllConnections(); + await app.close(); + const own = await store.lookup(child.operationKey); + if (own) await sql`DELETE FROM data_manager.jobs WHERE id=${own.status.id}`; + resetOpsClient(); + control.closeArtifactTargetLeaseHandle(lease); + journal.close(); + control.closeControlInstallation(host); + control.closeControlInstallation(dm); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); diff --git a/services/data-manager/src/jobs/execution/hardlinks.ts b/services/data-manager/src/jobs/execution/hardlinks.ts new file mode 100644 index 000000000..feff3ffa8 --- /dev/null +++ b/services/data-manager/src/jobs/execution/hardlinks.ts @@ -0,0 +1,30 @@ +import type { DataExecutionResult } from "@openmapx/core/data-manager-contracts"; +import { applyHardlinkPlan } from "@openmapx/hardlinks"; +import type { DataExecutionContext } from "./runtime.js"; +import { publishDataChange } from "./worker-scope.js"; +/** Generation leaf: the only I/O is the confined, root-fixed plan. It never + * creates/restarts a container or obtains a host configuration writer. */ +export async function applyOwnedGenerationHardlinks( + context: DataExecutionContext, +): Promise { + const journal = context.store.controlJournal; + if (!journal) throw Error("Durable generation journal is unavailable"); + const record = await context.record(); + if (record.input.kind !== "generation-hardlinks") throw Error("Generation leaf required"); + const plan = await journal.readHardlinksPlan(context.execution); + context.assertForward(); + const applied = await publishDataChange(() => + applyHardlinkPlan(plan.plan, { rootDir: context.dataRoot, prune: true }), + ); + return { + kind: "generation-hardlinks", + hardlinks: { + applied: true, + entries: plan.plan.length, + linked: applied.linked, + skipped: applied.skipped, + pruned: applied.pruned, + via: "data-manager", + }, + }; +} diff --git a/services/data-manager/src/jobs/execution/index.ts b/services/data-manager/src/jobs/execution/index.ts new file mode 100644 index 000000000..f74b30403 --- /dev/null +++ b/services/data-manager/src/jobs/execution/index.ts @@ -0,0 +1,23 @@ +export { type DataExecutionReceiver, registerDataExecutionApi } from "./api.js"; +export { + createDataConfigurationMaintenance, + type DataConfigurationMaintenance, +} from "./configuration-maintenance.js"; +export { + cancelAndSettleOwnedDataOpsChildren, + type DataEffectKind, + type DataOpsChildAttempt, + type DataOpsChildPhase, + lookupOwnedDataOpsChildren, + runOwnedDataOpsOperation, +} from "./ops-children.js"; +export { + createDataExecutionRuntime, + type DataExecutionContext, + type DataExecutionRuntime, +} from "./runtime.js"; +export { + createPostgresDataExecutionStore, + type DataExecutionRecord, + type DataExecutionStore, +} from "./store.js"; diff --git a/services/data-manager/src/jobs/execution/legacy-mutation.ts b/services/data-manager/src/jobs/execution/legacy-mutation.ts new file mode 100644 index 000000000..8d717eed8 --- /dev/null +++ b/services/data-manager/src/jobs/execution/legacy-mutation.ts @@ -0,0 +1,93 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type postgres from "postgres"; +import { rawSql } from "../../db/index.js"; +import { AmbientPublicationBusyError } from "../ambient-places/schema.js"; +import { createJobRow } from "../persistence.js"; + +const current = new AsyncLocalStorage<{ + id: string; + version: number; + operations: Record[]; + database: postgres.Sql; +}>(); +export function legacyDataMutation() { + return current.getStore(); +} +export async function checkpointLegacyDataOperation( + operation: Record, +): Promise { + const context = current.getStore(); + if (!context) throw Error("Retained data mutation admission required"); + const index = context.operations.findIndex( + (entry) => entry.operationKey === operation.operationKey, + ), + next = [...context.operations]; + if (index < 0) next.push(operation); + else next[index] = operation; + if (next.length > 32 || Buffer.byteLength(JSON.stringify(next)) > 128 * 1024) + throw Error("Retained legacy operation capacity exceeded"); + const rows = + await context.database`UPDATE data_manager.jobs SET metadata=jsonb_set(coalesce(metadata,'{}'::jsonb),'{retainedOps}',${JSON.stringify(next)}::text::jsonb),ownership_version=ownership_version+1 WHERE id=${context.id} AND ownership_version=${context.version} AND finished_at IS NULL RETURNING ownership_version`; + if (rows.length !== 1) throw Error("Legacy data operation owner CAS lost"); + context.version = Number(rows[0]!.ownership_version); + context.operations = next; +} +export type LegacyDataMutationKind = + | "traffic-live" + | "traffic-maintenance" + | "traffic-predicted" + | "transit-proposal" + | "data-reload" + | "ambient-publication"; +/** Admission is awaited before returning202; completion retains its own actual + * SQL owner and every nested host-operation checkpoint until containment. */ +export async function startLegacyDataMutation( + kind: LegacyDataMutationKind, + run: () => Promise, + database: postgres.Sql = rawSql, +): Promise<{ id: string; completion: Promise }> { + const id = await createJobRow( + { kind, triggeredBy: "data-manager", metadata: { version: 1, retainedOps: [] } }, + database, + ); + const context = { id, version: 0, operations: [] as Record[], database }; + const finish = async (status: string) => { + const rows = + await database`UPDATE data_manager.jobs SET status=${status},finished_at=${new Date().toISOString()}::text::timestamptz WHERE id=${id} AND ownership_version=${context.version} AND execution_version IS NULL AND kind <> 'configuration-maintenance' AND finished_at IS NULL RETURNING id`; + if (rows.length !== 1) throw Error("Legacy writer terminal CAS lost"); + }; + const completion = current.run(context, async () => { + try { + const value = await run(); + await finish("ok"); + return value; + } catch (error) { + if (kind === "ambient-publication" && error instanceof AmbientPublicationBusyError) { + await finish("error"); + throw error; + } + await database`UPDATE data_manager.jobs SET status='interrupted',finished_at=NULL,metadata=jsonb_set(coalesce(metadata,'{}'::jsonb),'{recoveryRequired}','true'::jsonb) WHERE id=${id} AND finished_at IS NULL`; + throw error; + } + }); + void completion.catch(() => {}); + return { id, completion }; +} +export async function withLegacyDataMutation( + kind: LegacyDataMutationKind, + run: () => Promise, + database: postgres.Sql = rawSql, +): Promise { + if (current.getStore()) return run(); + const owned = await startLegacyDataMutation(kind, run, database); + return owned.completion; +} + +/** A background maintenance operation admits its own durable row; it must not + * inherit a parent cycle that can finish while the operation is still alive. */ +export function independentLegacyDataMutation( + kind: Parameters[0], + run: () => Promise, +): Promise { + return current.exit(() => withLegacyDataMutation(kind, run)); +} diff --git a/services/data-manager/src/jobs/execution/ops-children.test.ts b/services/data-manager/src/jobs/execution/ops-children.test.ts new file mode 100644 index 000000000..8102e1fab --- /dev/null +++ b/services/data-manager/src/jobs/execution/ops-children.test.ts @@ -0,0 +1,1124 @@ +import { spawn } from "node:child_process"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import * as control from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + createControlRootOperationKey, + type ExecutionRef, + executionFingerprint, + OpsClient, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { applyHardlinkPlan } from "@openmapx/hardlinks"; +import { build } from "esbuild"; +import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { transitChildCandidateId } from "./ops-children.js"; + +let bundle: string, bundleRoot: string; +beforeAll(async () => { + bundleRoot = mkdtempSync(join(tmpdir(), "omx-data-agent-bundle-")); + bundle = join(bundleRoot, "agent.mjs"); + await build({ + entryPoints: [resolve("apps/ops-agent/src/test-fixtures/data-owned-http-process.ts")], + bundle: true, + platform: "node", + format: "esm", + target: "node24", + outfile: bundle, + banner: { + js: `import {createRequire} from "node:module"; const require=createRequire(${JSON.stringify(resolve("apps/ops-agent/package.json"))});`, + }, + }); +}); +afterAll(() => { + if (bundleRoot) rmSync(bundleRoot, { recursive: true, force: true }); +}); +it.each(["promotion", "reload"] as const)( + "keeps actual separate-process agent %s fenced across lost admission and cancellation responses", + async (kind) => { + const root = mkdtempSync(join(tmpdir(), "omx-data-http-cancel-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis", "motis-feed-proxy"], + }); + const agentVerifier = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis", "motis-feed-proxy"], + }); + const executionId = randomUUID(), + deadlineAt = Date.now() + 60000, + operationKey = createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }); + const operation = + kind === "promotion" + ? { kind: "motis.primary.promote" as const, preparedRunId: executionId } + : { + kind: "feedProxy.validateAndReload" as const, + candidateId: transitChildCandidateId({ operationKey }, "proxy-stage", 0), + }; + const phase = kind === "promotion" ? ("promote-primary" as const) : ("proxy-stage" as const); + const serviceIds = kind === "promotion" ? ["motis"] : ["motis-feed-proxy"]; + const input = { kind: "transit-sync", countries: ["de"] }; + const operationFingerprint = createHash("sha256").update(canonicalJson(input)).digest("hex"); + const owner: ExecutionRef = { + executor: "data-manager", + principal: "data-manager", + executionId, + operationKey, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: input.kind, operationFingerprint, deadlineAt }), + }; + const child: ExecutionRef = { + executor: "ops-agent", + principal: "data-manager", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: control.controlChildOperationKey(owner.operationKey, phase, 0), + operationFingerprint: opsOperationFingerprint(operation), + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: opsOperationFingerprint(operation), + deadlineAt, + parent: control.retainedControlParent(owner), + }), + }; + const plan = { + phase, + ordinal: 0, + kind: operation.kind, + execution: child, + serviceIds, + recipient: "ops-agent" as const, + }; + const authority = control.retainAdmittedControlExecution(dm, { + execution: owner, + kind: input.kind, + operation: input, + admittedAt: deadlineAt - 60000, + requestedDeadlineAt: deadlineAt, + serviceIds, + plan: [plan], + }); + const lease = await control.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds, + owner: authority, + }); + if (kind === "promotion") { + writePreparedFixture(join(root, "data/motis/staging"), executionId); + control.retainOwnedDataTransitCandidate(authority, { phase, ordinal: 0 }); + renameSync(join(root, "data/motis/staging"), join(root, "data/motis/live")); + } else { + mkdirSync(join(root, "data/motis-feed-proxy/conf"), { recursive: true }); + writeFileSync( + join(root, "data/motis-feed-proxy/conf/default.conf"), + "server { listen 80; }\n", + ); + writeFileSync(join(root, "data/motis-feed-proxy/feed-proxy-vars.json"), "{}\n"); + linkProxyFixture(join(root, "data")); + control.retainOwnedDataTransitCandidate(authority, { phase, ordinal: 0 }); + } + const assertion = await control.delegateArtifactTargetLease(lease, { + ...plan, + childPublicKey: agentVerifier.publicKey, + }); + const token = "b".repeat(43), + tokenFile = join(root, "token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + const process = spawn(globalThis.process.execPath, [bundle], { + stdio: ["pipe", "pipe", "pipe"], + }); + let stderr = ""; + process.stderr.on("data", (data) => { + stderr += String(data); + }); + const lines = createInterface({ input: process.stdout }); + const iterator = lines[Symbol.asyncIterator](); + const next = async () => { + const value = await Promise.race([ + iterator.next(), + new Promise((_, reject) => { + const timer = setTimeout( + () => reject(Error(`Agent process did not respond: ${stderr}`)), + 8000, + ); + timer.unref(); + }), + ]); + if (value.done) throw Error(`Agent process exited: ${stderr}`); + return JSON.parse(value.value) as Record; + }; + const send = (value: unknown) => process.stdin.write(`${JSON.stringify(value)}\n`); + const exited = new Promise((done) => process.once("close", () => done())); + send({ root, token }); + try { + const ready = await next(); + expect(ready.type).toBe("ready"); + let admissions = 0, + cancels = 0; + const client = new OpsClient({ + baseUrl: `http://127.0.0.1:${ready.port}`, + tokenFile, + fetch: async (url, options) => { + const response = await fetch(url, options); + if (options?.method === "POST") { + admissions++; + await response.body?.cancel(); + throw Error("synthetic lost admission response"); + } + if (options?.method === "DELETE") { + cancels++; + await response.body?.cancel(); + throw Error("synthetic lost cancel response"); + } + return response; + }, + }); + await expect( + client.execute(operation, { + operationKey: child.operationKey, + execution: { requestedDeadlineAt: deadlineAt }, + controlAssertion: assertion, + }), + ).rejects.toThrow(); + expect((await next()).type).toBe("effect"); + const adopted = await client.lookup(operation, child.operationKey, { + fingerprint: child.fingerprint, + execution: { requestedDeadlineAt: deadlineAt }, + }); + expect(adopted.operationId).toBe(child.executionId); + const cancellation = client + .cancel(child.executionId, { fingerprint: child.fingerprint }) + .then( + () => null, + (error) => error, + ); + expect((await next()).type).toBe("aborted"); + send({ type: "probe" }); + const probe = await next(); + expect(probe).toMatchObject({ + type: "probe", + forward: false, + effects: 1, + state: "termination_pending", + terminalAt: null, + operationId: child.executionId, + }); + expect((await client.status(operation.kind, child.executionId)).state).toBe( + "termination_pending", + ); + await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); + send({ type: "release" }); + expect(await cancellation).toBeInstanceOf(Error); + await expect + .poll(async () => (await client.status(operation.kind, child.executionId)).state) + .toBe("succeeded"); + expect(admissions).toBe(1); + expect(cancels).toBe(1); + expect( + (await client.lookup(operation, child.operationKey, { fingerprint: child.fingerprint })) + .operationId, + ).toBe(child.executionId); + send({ type: "close" }); + process.stdin.end(); + await exited; + } finally { + process.kill("SIGKILL"); + await exited; + lines.close(); + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agentVerifier); + control.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + } + }, +); + +import { existsSync } from "node:fs"; +import { + cancelDataExecution, + createDataManagerTransport, + getDataExecution, + unwrapDataManagerResponse, +} from "@openmapx/core/data-manager-sdk"; +import Fastify from "fastify"; +import postgres from "postgres"; +import { registerApi } from "../../api.js"; +import { registerAuth } from "../../auth.js"; +import { resetOpsClient } from "../../ops-client.js"; +import { StateStore } from "../../state.js"; +import { resolveOperationsProfile } from "../transitous/operations-profile.js"; +import { getSingleFlightController } from "../transitous/runtime.js"; +import { createDataConfigurationMaintenance } from "./configuration-maintenance.js"; +import { createDataExecutionDispatcher } from "./dispatch.js"; +import { createDataExecutionReceiver } from "./receiver.js"; +import { createDataExecutionRuntime } from "./runtime.js"; +import { createPostgresDataExecutionStore } from "./store.js"; + +it.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)( + "assembles durable DM SQL/runtime/dispatcher/HTTP cancellation through the separate production agent", + async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + + const sql = postgres(url, { max: 4 }), + root = mkdtempSync(join(tmpdir(), "omx-durable-dm-agent-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], + }); + const journal = control.createDataExecutionControlJournal(dm, sql), + fence = createDataConfigurationMaintenance(sql, dm); + await fence.startup(); + const store = createPostgresDataExecutionStore(sql, { + controlJournal: journal, + configurationFence: fence, + }), + singleFlight = getSingleFlightController(), + operationsPolicy = resolveOperationsProfile({ countries: ["de"] }); + const dispatcher = createDataExecutionDispatcher({ + dataDir: join(root, "data"), + state: new StateStore(join(root, "data")), + sql, + singleFlight, + operationsPolicy, + runHardlinks: async () => { + throw Error("This fixture has no generation grant"); + }, + }); + const runtime = createDataExecutionRuntime({ store, dispatch: dispatcher.dispatch }), + receiver = createDataExecutionReceiver({ + runtime, + installation: dm, + configurationFence: fence, + dispatcher, + operationsPolicy, + }); + const token = "c".repeat(43), + tokenFile = join(root, "agent-token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + const agent = spawn(process.execPath, [bundle], { stdio: ["pipe", "pipe", "pipe"] }), + lines = createInterface({ input: agent.stdout }), + messages = lines[Symbol.asyncIterator](); + let stderr = ""; + agent.stderr.on("data", (chunk) => (stderr += String(chunk))); + const next = async () => { + const result = await Promise.race([ + messages.next(), + new Promise((_, reject) => { + const timer = setTimeout(() => reject(Error(`Agent IPC timeout: ${stderr}`)), 15000); + timer.unref(); + }), + ]); + if (result.done) throw Error(`Agent exited: ${stderr}`); + return JSON.parse(result.value) as Record; + }; + const send = (value: unknown) => agent.stdin.write(`${JSON.stringify(value)}\n`), + exited = new Promise((resolve) => agent.once("close", () => resolve())); + send({ root, token }); + const ready = await next(); + let posts = 0, + cancels = 0; + resetOpsClient( + new OpsClient({ + baseUrl: `http://127.0.0.1:${ready.port}`, + tokenFile, + fetch: async (url, options) => { + const response = await fetch(url, options); + if (options?.method === "POST") { + posts++; + await response.body?.cancel(); + throw Error("Synthetic lost actual agent admission response"); + } + if (options?.method === "DELETE") { + cancels++; + await response.body?.cancel(); + throw Error("Synthetic lost actual agent cancellation response"); + } + return response; + }, + }), + ); + const app = Fastify(); + registerAuth(app, "synthetic-dm-token", { executionReceiver: () => receiver }); + registerApi(app, { + dataDir: join(root, "data"), + singleFlight, + operationsPolicy, + executionReceiver: receiver, + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("DM HTTP address missing"); + const baseUrl = `http://127.0.0.1:${address.port}`, + transport = createDataManagerTransport({ baseUrl, authToken: "synthetic-dm-token" }); + let id: string | undefined; + + try { + const observer = new AbortController(); + const observation = fetch(`${baseUrl}/transit/restart-motis`, { + method: "POST", + headers: { authorization: "Bearer synthetic-dm-token" }, + signal: observer.signal, + }).catch((error) => error); + expect( + ( + await Promise.race([ + next(), + observation.then(async (value) => { + if (value instanceof Response) + throw Error(`Early DM HTTP ${value.status}: ${await value.text()}`); + throw value; + }), + ]) + ).type, + ).toBe("effect"); + observer.abort(); + await observation; + + const owned = (await store.listUnsettled()).find( + (record) => record.input.kind === "transit-maintenance", + ); + if (!owned) throw Error("Missing actual retained DM root"); + id = owned.status.id; + const before = unwrapDataManagerResponse(await getDataExecution(id, { transport })); + expect(before.status).toBe("running"); + expect(owned.finishedAt).toBeNull(); + const losingTransport = createDataManagerTransport({ + baseUrl, + authToken: "synthetic-dm-token", + fetch: async (url, options) => { + const response = await fetch(url, options); + if (options?.method === "POST") { + await response.body?.cancel(); + throw Error("Synthetic lost actual DM cancellation response"); + } + return response; + }, + }); + await expect( + cancelDataExecution( + id, + { operationKey: before.operationKey, fingerprint: before.fingerprint }, + { transport: losingTransport }, + ), + ).rejects.toThrow(); + + expect((await next()).type).toBe("aborted"); + const pending = await store.inspect(id); + expect(pending?.status.status).toBe("cancel_pending"); + expect(pending?.finishedAt).toBeNull(); + expect(existsSync(join(root, "data/.admin-target-control/locks/motis"))).toBe(true); + send({ type: "probe" }); + expect(await next()).toMatchObject({ + forward: false, + effects: 1, + state: "termination_pending", + terminalAt: null, + }); + send({ type: "release" }); + await expect + .poll(async () => (await runtime.inspect(id!))?.status, { timeout: 15000 }) + .toBe("ok"); + expect((await runtime.inspect(id))?.result).toEqual({ + kind: "transit-maintenance", + action: "restart-primary", + changed: true, + }); + await expect + .poll(() => existsSync(join(root, "data/.admin-target-control/locks/motis")), { + timeout: 10000, + }) + .toBe(false); + expect(posts).toBe(1); + expect(cancels).toBe(1); + expect((await store.inspect(id))?.children).toMatchObject([ + { state: "terminal", publication: "committed", dispatchState: "claimed" }, + ]); + } finally { + send({ type: "release" }); + await runtime.close(); + app.server.closeAllConnections(); + await app.close(); + send({ type: "close" }); + agent.stdin.end(); + await exited; + resetOpsClient(); + const fixtureRows = await store.listUnsettled(); + const ownIds = new Set([ + ...(id ? [id] : []), + ...fixtureRows + .filter((row) => + existsSync( + join( + root, + "control-private/data-manager/executions", + `${row.status.operationKey}.json`, + ), + ), + ) + .map((row) => row.status.id), + ]); + for (const ownId of ownIds) await sql`DELETE FROM data_manager.jobs WHERE id=${ownId}`; + journal.close(); + control.closeControlInstallation(dm); + await sql.end(); + lines.close(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); + +import { readFileSync, renameSync, statSync } from "node:fs"; +import { startDataExecution } from "@openmapx/core/data-manager-sdk"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { createCandidateManifest } from "../transitous/candidate.js"; +import { retainTransitPhaseCandidate, runTransitOwnedPhase } from "../transitous/owned-ops.js"; +import { buildJobContext } from "../transitous/pipeline.js"; +import { run as stageProxy } from "../transitous/proxy-transaction.js"; +import { publishDataChange } from "./worker-scope.js"; + +it + .skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", + ) + .each([ + "promotion", + "reload", + "validation-failure", + "restore-validation-failure", + "cancel-before-restore", + "expire-before-restore", + "restore-reservation-crash", + "restore-files-crash", + ] as const)( + "runs an hours-late owned %s through the real DM receiver/store/runtime and separate assembled agent", + async (kind) => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + drizzle(sql); + const root = mkdtempSync(join(tmpdir(), "omx-late-durable-child-")), + data = join(root, "data"); + mkdirSync(data, { mode: 0o700 }); + await control.provisionControlInstallation(root); + const services = ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], + dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: services, + }), + journal = control.createDataExecutionControlJournal(dm, sql), + fence = createDataConfigurationMaintenance(sql, dm); + await fence.startup(); + const originalAdmittedAt = Date.now() - 4 * 3600000, + deadline = originalAdmittedAt + 24 * 3600000; + let storeNow = originalAdmittedAt; + const store = createPostgresDataExecutionStore(sql, { + controlJournal: journal, + configurationFence: fence, + now: () => storeNow, + }); + const phase = kind === "promotion" ? ("promote-primary" as const) : ("proxy-stage" as const), + state = new StateStore(data), + runtime = createDataExecutionRuntime({ + store, + dispatch: async (_input, context) => { + const job = buildJobContext({ + dataDir: data, + store: state, + countries: ["de"], + jobId: context.execution.executionId, + executionContext: context, + abortSignal: context.signal, + }); + let finalStatus: "ok" | "error" = "ok"; + if (kind === "promotion") { + mkdirSync(join(data, "motis/staging"), { recursive: true }); + writePreparedFixture(join(data, "motis/staging"), context.execution.executionId); + await retainTransitPhaseCandidate(job, phase); + await publishDataChange(() => + renameSync(join(data, "motis/staging"), join(data, "motis/live")), + ); + } else { + mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); + writeFileSync( + join(data, "motis-feed-proxy/conf/default.conf"), + "server { listen 80; }\n", + ); + writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "{}\n"); + linkProxyFixture(data); + writePreparedFixture(job.motisStagingDataDir, context.execution.executionId); + const staged = await stageProxy(job); + if (staged.status !== "ok") finalStatus = "error"; + } + if (kind === "promotion") await runTransitOwnedPhase(job, phase); + context.signal.throwIfAborted(); + return { + kind: "transit-sync", + finalStatus, + sourceRevision: "public-fixture", + countries: ["de"], + }; + }, + }), + singleFlight = getSingleFlightController(), + operationsPolicy = resolveOperationsProfile({ countries: ["de"] }), + dispatcher = createDataExecutionDispatcher({ + dataDir: data, + state, + sql, + singleFlight, + operationsPolicy, + runHardlinks: async () => { + throw Error("No generation grant"); + }, + }), + receiver = createDataExecutionReceiver({ + runtime, + installation: dm, + configurationFence: fence, + dispatcher, + operationsPolicy, + }); + let restoreFault: ReturnType | undefined, + recoveryFaultReached = false; + const retainRecovery = journal.retainProxyRecoveryIntent.bind(journal); + journal.retainProxyRecoveryIntent = async (authority, backupDigest) => { + await retainRecovery(authority, backupDigest); + if (kind === "cancel-before-restore") + await runtime.requestCancel({ + id: authority.execution.executionId, + operationKey: authority.execution.operationKey, + fingerprint: authority.execution.fingerprint, + }); + if (kind === "expire-before-restore") + restoreFault = vi.spyOn(Date, "now").mockReturnValue(deadline + 1); + recoveryFaultReached = true; + if (kind === "restore-reservation-crash") + throw Error("Synthetic crash after confirmed restore reservation"); + }; + const checkpointChild = store.checkpointChild.bind(store); + store.checkpointChild = async (id, version, child) => { + const saved = await checkpointChild(id, version, child); + if ( + saved && + child.phase === "proxy-restore" && + child.state === "bound" && + kind === "restore-files-crash" + ) + throw Error( + "Synthetic response loss after files and confirmed binding, before validation dispatch", + ); + return saved; + }; + const token = "c".repeat(43), + tokenFile = join(root, "token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + const agent = spawn(process.execPath, [bundle], { stdio: ["pipe", "pipe", "pipe"] }), + lines = createInterface({ input: agent.stdout }), + messages = lines[Symbol.asyncIterator](); + let stderr = ""; + agent.stderr.on("data", (chunk) => (stderr += String(chunk))); + const next = async () => { + const result = await Promise.race([ + messages.next(), + new Promise((_, reject) => { + const timeout = setTimeout( + () => reject(Error(`Agent fixture IPC timeout: ${stderr}`)), + 15000, + ); + timeout.unref(); + }), + ]); + if (result.done) throw Error(`Agent exited: ${stderr}`); + return JSON.parse(result.value) as Record; + }, + send = (value: unknown) => agent.stdin.write(`${JSON.stringify(value)}\n`), + exited = new Promise((resolve) => agent.once("close", () => resolve())); + send({ + root, + token, + validationFailure: + kind === "restore-validation-failure" ? "twice" : !["promotion", "reload"].includes(kind), + }); + const ready = await next(); + let posts = 0, + cancels = 0; + resetOpsClient( + new OpsClient({ + baseUrl: `http://127.0.0.1:${ready.port}`, + tokenFile, + fetch: async (url, options) => { + const response = await fetch(url, options); + if (options?.method === "POST") { + posts++; + await response.body?.cancel(); + throw Error("Lost real child admission response"); + } + if (options?.method === "DELETE") { + cancels++; + await response.body?.cancel(); + throw Error("Lost real child cancel response"); + } + return response; + }, + }), + ); + const app = Fastify(); + registerAuth(app, "late-fixture-token", { executionReceiver: () => receiver }); + registerApi(app, { + dataDir: data, + singleFlight, + operationsPolicy, + executionReceiver: receiver, + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("DM endpoint missing"); + const transport = createDataManagerTransport({ + baseUrl: `http://127.0.0.1:${address.port}`, + authToken: "late-fixture-token", + }); + let id: string | undefined; + let closingRuntime: Promise | undefined; + const closeRuntime = () => (closingRuntime ??= runtime.close()); + try { + const key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + admitted = unwrapDataManagerResponse( + await startDataExecution( + { + operationKey: key, + requestedDeadlineAt: deadline, + input: { kind: "transit-sync", countries: ["de"] }, + }, + { transport }, + ), + ); + id = admitted.id; + storeNow = Date.now(); + expect(admitted.admittedAt).toBe(originalAdmittedAt); + if ( + !["promotion", "reload", "validation-failure", "restore-validation-failure"].includes(kind) + ) { + expect((await next()).type).toBe("validation-failed"); + send({ type: "release" }); + await expect + .poll( + async () => + (await store.inspect(id!))?.children.some((child) => child.phase === "proxy-restore"), + { timeout: 15000 }, + ) + .toBe(true); + await expect.poll(() => recoveryFaultReached, { timeout: 15000 }).toBe(true); + if (kind === "restore-files-crash") + await expect + .poll( + async () => + (await store.inspect(id!))?.children.find( + (child) => child.phase === "proxy-restore", + )?.state, + { timeout: 15000 }, + ) + .toBe("bound"); + const lock = join(data, ".admin-target-control/locks/motis-feed-proxy"), + inode = statSync(lock).ino; + await expect( + journal.assertOwnedProxyRecoverySettled((await store.inspect(id))!.execution), + ).rejects.toThrow(); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + expect(posts).toBe(1); + restoreFault?.mockRestore(); + restoreFault = undefined; + await closeRuntime(); + const restarted = createDataExecutionRuntime({ + store, + dispatch: async () => { + throw Error("Restart cannot repeat an unfinished restore"); + }, + }); + await restarted.reconcile(); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + expect((await store.inspect(id))?.status.publication).toBe("unknown"); + expect(statSync(lock).ino).toBe(inode); + expect(posts).toBe(1); + await restarted.close(); + return; + } + if (kind === "validation-failure" || kind === "restore-validation-failure") { + expect((await next()).type).toBe("validation-failed"); + send({ type: "release" }); + if (kind === "restore-validation-failure") { + expect((await next()).type).toBe("validation-failed"); + await expect + .poll( + async () => + (await store.inspect(id!))?.children.filter((child) => child.state === "terminal") + .length, + { timeout: 15000 }, + ) + .toBe(2); + const lock = join(data, ".admin-target-control/locks/motis-feed-proxy"), + inode = statSync(lock).ino; + await expect + .poll( + () => + existsSync( + join(root, "control-private/data-manager/observations", `worker-${key}.json`), + ), + { timeout: 15000 }, + ) + .toBe(true); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + expect((await receiver.inspect(id))?.publication).toBe("unknown"); + expect((await store.inspect(id))?.children).toMatchObject([ + { + phase: "proxy-stage", + state: "terminal", + publication: "not_started", + settlement: { outcome: "contained" }, + }, + { + phase: "proxy-restore", + state: "terminal", + publication: "not_started", + settlement: { outcome: "contained" }, + }, + ]); + await closeRuntime(); + // An older/inconsistent terminal SQL row is not release authority. + await sql`UPDATE data_manager.jobs SET status='error',finished_at=${new Date().toISOString()}::text::timestamptz,execution_state=jsonb_set(execution_state,'{publication}','"committed"'::jsonb) WHERE id=${id}`; + const restarted = createDataExecutionRuntime({ + store, + dispatch: async () => { + throw Error("Restart may not dispatch a restore"); + }, + }); + await restarted.reconcile(); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + expect((await receiver.inspect(id))?.publication).toBe("unknown"); + expect(statSync(lock).ino).toBe(inode); + await expect( + journal.assertOwnedProxyRecoverySettled((await store.inspect(id))!.execution), + ).rejects.toThrow(); + const clock = vi.spyOn(Date, "now").mockReturnValue(deadline + 72 * 3600000 + 1); + try { + expect((await control.reconcileControlRetention(dm, { maxFamilies: 1 })).retired).toBe( + 0, + ); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + expect(statSync(lock).ino).toBe(inode); + } finally { + clock.mockRestore(); + } + await restarted.close(); + expect(posts).toBe(2); + expect(cancels).toBe(0); + return; + } + await expect + .poll(async () => (await receiver.inspect(id!))?.status, { timeout: 15000 }) + .toBe("error"); + const failed = await store.inspect(id); + expect(failed?.children).toMatchObject([ + { + phase: "proxy-stage", + state: "terminal", + publication: "not_started", + settlement: { outcome: "contained" }, + }, + { phase: "proxy-restore", state: "terminal", publication: "committed" }, + ]); + expect(posts).toBe(2); + expect(cancels).toBe(0); + return; + } + expect((await next()).type).toBe("effect"); + const record = await store.inspect(id); + if (!record) throw Error("Original data root missing"); + const attempted = record.children[0]; + if (!attempted || attempted.state === "reserved") throw Error("Actual bound child missing"); + expect(attempted.execution.deadlineAt).toBeGreaterThan(Date.now()); + expect(attempted.execution.deadlineAt - Date.now()).toBeLessThanOrEqual( + kind === "promotion" ? 600000 : 120000, + ); + expect(record.status.deadlineAt).toBe(deadline); + const lock = join( + data, + ".admin-target-control/locks", + kind === "promotion" ? "motis" : "motis-feed-proxy", + ), + inode = statSync(lock).ino, + rootReceipt = join(root, "control-private/data-manager/executions", `${key}.json`), + original = readFileSync(rootReceipt); + await receiver.requestCancel({ id, operationKey: key, fingerprint: admitted.fingerprint }); + expect((await next()).type).toBe("aborted"); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + expect(statSync(lock).ino).toBe(inode); + expect((await receiver.inspect(id))?.status).toBe("cancel_pending"); + send({ type: "probe" }); + expect(await next()).toMatchObject({ + forward: false, + effects: 1, + state: "termination_pending", + terminalAt: null, + }); + send({ type: "release" }); + await expect + .poll(async () => (await receiver.inspect(id!))?.status, { timeout: 15000 }) + .toBe("canceled"); + expect((await receiver.inspect(id))?.publication).toBe("committed"); + await expect.poll(() => existsSync(lock), { timeout: 10000 }).toBe(false); + expect(readFileSync(rootReceipt)).toEqual(original); + expect(posts).toBe(1); + expect(cancels).toBe(1); + expect((await store.inspect(id))?.children).toMatchObject([ + { phase, state: "terminal", publication: "committed" }, + ]); + } finally { + restoreFault?.mockRestore(); + send({ type: "release" }); + await closeRuntime(); + app.server.closeAllConnections(); + await app.close(); + send({ type: "close" }); + agent.stdin.end(); + await exited; + resetOpsClient(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(dm); + await sql.end(); + lines.close(); + rmSync(root, { recursive: true, force: true }); + } + }, + 120000, +); + +function writePreparedFixture(staging: string, executionId: string) { + mkdirSync(staging, { recursive: true }); + mkdirSync(join(staging, ".openmapx-feed-proxy/conf"), { recursive: true }); + for (const [name, body] of [ + ["config.yml", "timetable:\n datasets:\n public: test.gtfs.zip\n"], + ["license.json", "{}"], + [".openmapx-feed-proxy/conf/default.conf", "server { listen 80; }"], + [".openmapx-feed-proxy/feed-proxy-vars.json", "{}"], + [ + "transit-source-manifest.json", + JSON.stringify({ version: 1, generatedAt: new Date().toISOString(), sources: [] }), + ], + ["test.gtfs.zip", "public-synthetic-dataset"], + ]) + writeFileSync(join(staging, name!), body!); + createCandidateManifest(staging, executionId, new Date().toISOString()); +} +function linkProxyFixture(data: string) { + applyHardlinkPlan( + [ + { + source: "motis-feed-proxy/conf", + target: "motis-feed-proxy/motis-feed-proxy-config", + consumerService: "motis-feed-proxy", + dataType: "motis-feed-proxy-config", + }, + ], + { rootDir: data }, + ); +} + +it.each(["before-proof", "before-terminal", "after-terminal"] as const)( + "restarts the actual proxy agent after %s without inventing proof or replaying validation", + async (crash) => { + const root = mkdtempSync(join(tmpdir(), "omx-proxy-proof-crash-")), + data = join(root, "data"); + mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); + writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "server { listen 80; }\n"); + writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "{}\n"); + linkProxyFixture(data); + await control.provisionControlInstallation(root); + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["motis-feed-proxy"], + }); + const agentVerifier = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis-feed-proxy"], + }); + const deadlineAt = Date.now() + 120000, + operationKey = createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }); + const input = { kind: "transit-sync", countries: ["de"] }, + operationFingerprint = createHash("sha256").update(canonicalJson(input)).digest("hex"); + const parent: ExecutionRef = { + executor: "data-manager", + principal: "data-manager", + executionId: randomUUID(), + operationKey, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: input.kind, operationFingerprint, deadlineAt }), + }; + const operation = { + kind: "feedProxy.validateAndReload" as const, + candidateId: transitChildCandidateId(parent, "proxy-stage", 0), + }; + const reservation: control.ControlChildReservation = { + version: 1, + phase: "proxy-stage", + ordinal: 0, + kind: operation.kind, + recipient: "ops-agent", + serviceIds: ["motis-feed-proxy"], + operationFingerprint: opsOperationFingerprint(operation), + execution: { + executor: "ops-agent", + principal: "data-manager", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: control.controlChildOperationKey(operationKey, "proxy-stage", 0), + }, + }; + const authority = control.retainAdmittedControlExecution(dm, { + execution: parent, + kind: input.kind, + operation: input, + admittedAt: Date.now(), + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis-feed-proxy"], + reservations: [reservation], + }); + const lease = await control.acquireArtifactTargetLease({ + dataRoot: data, + serviceIds: ["motis-feed-proxy"], + owner: authority, + }); + control.retainOwnedDataTransitCandidate(authority, reservation); + const bound = await control.bindControlChildPhase(authority, reservation), + issued = await control.issueControlChildPhase(authority, { + ...reservation, + lease, + childPublicKey: agentVerifier.publicKey, + }); + const token = "e".repeat(43), + tokenFile = join(root, "token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + async function start(fault?: typeof crash) { + const child = spawn(process.execPath, [bundle], { stdio: ["pipe", "pipe", "pipe"] }), + lines = createInterface({ input: child.stdout }), + iterator = lines[Symbol.asyncIterator](); + let stderr = ""; + child.stderr.on("data", (data) => { + stderr += String(data); + }); + const exited = new Promise((resolve) => child.once("close", resolve)); + child.stdin.write( + JSON.stringify({ + root, + token, + validationFailure: true, + ...(fault ? { crash: fault } : {}), + }) + "\n", + ); + const first = await iterator.next(); + if (first.done) throw Error("Agent startup failed: " + stderr); + const ready = JSON.parse(first.value) as { type: string; port: number }; + expect(ready.type).toBe("ready"); + return { + child, + lines, + exited, + client: new OpsClient({ baseUrl: `http://127.0.0.1:${ready.port}`, tokenFile }), + async close() { + if (child.exitCode === null) { + child.stdin.write('{"type":"close"}\n'); + child.stdin.end(); + } + await exited; + lines.close(); + }, + }; + } + let first: Awaited> | undefined, + restarted: Awaited> | undefined; + try { + first = await start(crash); + control.consumeControlChildDispatch(issued.dispatchTicket); + await first.client + .execute(operation, { + operationKey: bound.execution.operationKey, + execution: { requestedDeadlineAt: bound.execution.deadlineAt }, + controlAssertion: issued.assertion, + }) + .catch(() => undefined); + expect(await first.exited).toBe(86); + const terminalPath = join( + root, + "control-private/ops-agent/observations", + `terminal-${bound.execution.operationKey}.json`, + ); + const originalTerminal = existsSync(terminalPath) + ? JSON.parse(readFileSync(terminalPath, "utf8"))?.payload + : undefined; + restarted = await start(); + const status = await restarted.client.lookup(operation, bound.execution.operationKey, { + fingerprint: bound.execution.fingerprint, + }); + expect(status.operationId).toBe(bound.execution.executionId); + expect(status.deadlineAt).toBe(bound.execution.deadlineAt); + if (crash === "after-terminal") { + expect(status.state).toBe("failed"); + expect(status.errorClass).toBe("runtime"); + expect(control.verifyFeedProxyCanonicalTerminal(authority, bound.execution).state).toBe( + "failed", + ); + expect( + control.inspectArtifactTargetLeaseChildSettlement(lease, bound.execution), + ).toMatchObject({ outcome: "contained", publication: "not_started" }); + const repaired = JSON.parse(readFileSync(terminalPath, "utf8")).payload; + expect(repaired.terminalDigest).toBe(originalTerminal.terminalDigest); + expect(repaired.terminalAt).toBe(originalTerminal.terminalAt); + } else { + expect(status.errorClass).toBe("recovery_required"); + expect(() => + control.verifyFeedProxyCanonicalTerminal(authority, bound.execution), + ).toThrow(); + } + const jobs = JSON.parse(readFileSync(join(data, "ops-agent/jobs-v1.json"), "utf8")); + expect(JSON.stringify(jobs)).toContain(bound.execution.operationKey); + await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); + expect(existsSync(join(data, ".admin-target-control/locks/motis-feed-proxy"))).toBe(true); + // No POST is sent to the restarted process; original admission remains. + } finally { + await first?.close(); + await restarted?.close(); + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agentVerifier); + control.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); diff --git a/services/data-manager/src/jobs/execution/ops-children.ts b/services/data-manager/src/jobs/execution/ops-children.ts new file mode 100644 index 000000000..6a889b88b --- /dev/null +++ b/services/data-manager/src/jobs/execution/ops-children.ts @@ -0,0 +1,645 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + type ControlChildReservation, + controlChildOperationKey, +} from "@openmapx/core/build-artifacts"; +import type { DataExecutionInput } from "@openmapx/core/data-manager-contracts"; +import { + canonicalJson, + type ExecutionRef, + executionRefSchema, + type OpsOperation, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import z from "zod/v4"; +export const DATA_EFFECT_KINDS = [ + "motis.primary.stop", + "motis.staging.stop", + "motis.staging.restart", + "motis.primary.promote", + "motis.primary.restart", + "feedProxy.validateAndReload", + "transitousLock.approve", +] as const; +export type DataEffectKind = (typeof DATA_EFFECT_KINDS)[number]; +export const dataOpsChildPhaseSchema = z.enum([ + "stop-primary", + "stop-staging", + "restart-staging", + "promote-primary", + "recover-primary", + "proxy-stage", + "proxy-commit", + "proxy-restore", + "paired-pin-commit", +]); +export type DataOpsChildPhase = z.infer; +export const DATA_PHASE_KINDS: Record = { + "stop-primary": "motis.primary.stop", + "stop-staging": "motis.staging.stop", + "restart-staging": "motis.staging.restart", + "promote-primary": "motis.primary.promote", + "recover-primary": "motis.primary.restart", + "proxy-stage": "feedProxy.validateAndReload", + "proxy-commit": "feedProxy.validateAndReload", + "proxy-restore": "feedProxy.validateAndReload", + "paired-pin-commit": "transitousLock.approve", +}; +const digest = z.string().regex(/^[a-f0-9]{64}$/), + key = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + receiverId = z.string().regex(/^job1_[A-Za-z0-9_-]{16,64}$/); + +import { opsOperationSchema } from "@openmapx/core/ops"; + +const operationSchema = opsOperationSchema.transform((operation) => { + switch (operation.kind) { + case "motis.primary.stop": + case "motis.staging.stop": + case "motis.staging.restart": + case "motis.primary.promote": + case "motis.primary.restart": + case "feedProxy.validateAndReload": + case "transitousLock.approve": + return operation; + default: + throw Error("Unsupported data-manager host effect"); + } +}); +const common = { + proxyRecovery: z.strictObject({ version: z.literal(1), backupDigest: digest }).optional(), + parentExecution: executionRefSchema, + phase: dataOpsChildPhaseSchema, + ordinal: z.number().int().min(0).max(31), + operation: operationSchema, + operationFingerprint: digest, + operationKey: key, + proposedId: receiverId, + serviceIds: z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine((ids) => ids.every((id, i) => i === 0 || (ids[i - 1] ?? id) < id)), + cancellationRequestedAt: z.iso.datetime({ offset: true }).nullable(), +}; +const bound = { + ...common, + execution: executionRefSchema, + requestedDeadlineAt: z.number().int().nonnegative().safe(), + bindingDigest: digest, + candidateDigest: digest.nullable(), +}; +const attempted = { + ...bound, + delegationId: z.uuid(), + grantDigest: digest, + leaseId: z.uuid(), + dispatchState: z.enum(["pending", "claimed"]), +}; +export const dataOpsChildReservedSchema = z.strictObject({ + ...common, + state: z.literal("reserved"), + publication: z.literal("not_started"), +}); +export const dataOpsChildAttemptSchema = z + .discriminatedUnion("state", [ + dataOpsChildReservedSchema, + z.strictObject({ ...bound, state: z.literal("bound"), publication: z.literal("not_started") }), + z.strictObject({ + ...attempted, + state: z.literal("attempted"), + publication: z.enum(["not_started", "unknown"]), + }), + z.strictObject({ + ...attempted, + state: z.literal("observing"), + acceptedId: receiverId, + publication: z.enum(["not_started", "committed", "unknown"]), + }), + z.strictObject({ + ...attempted, + state: z.literal("terminal"), + acceptedId: receiverId, + publication: z.enum(["not_started", "committed"]), + settlement: z.strictObject({ + outcome: z.enum(["contained", "committed", "failed_without_effect"]), + terminalReceiptDigest: digest, + }), + }), + z.strictObject({ + ...attempted, + state: z.literal("unknown"), + acceptedId: receiverId.optional(), + publication: z.literal("unknown"), + }), + ]) + .superRefine((child, ctx) => { + if (child.proxyRecovery && child.phase !== "proxy-restore") + ctx.addIssue({ + code: "custom", + message: "Only the original restore may retain backup provenance", + }); + if ( + DATA_PHASE_KINDS[child.phase] !== child.operation.kind || + opsOperationFingerprint(child.operation) !== child.operationFingerprint || + child.operationKey !== + controlChildOperationKey(child.parentExecution.operationKey, child.phase, child.ordinal) + ) + ctx.addIssue({ code: "custom", message: "Child body or phase identity mismatch" }); + if ( + child.state !== "reserved" && + (child.execution.operationKey !== child.operationKey || + child.execution.executionId !== child.proposedId || + child.execution.operationFingerprint !== child.operationFingerprint || + child.execution.executor !== "ops-agent" || + child.execution.principal !== "data-manager" || + child.execution.deadlineAt !== child.requestedDeadlineAt) + ) + ctx.addIssue({ code: "custom", message: "Bound child identity mismatch" }); + if ( + "acceptedId" in child && + child.acceptedId !== undefined && + child.acceptedId !== child.proposedId + ) + ctx.addIssue({ code: "custom", message: "Foreign accepted child" }); + if (new TextEncoder().encode(JSON.stringify(child)).length > 8192) + ctx.addIssue({ code: "custom", message: "Child receipt capacity exceeded" }); + }); +export type DataOpsChildAttempt = z.infer; +export type DataOpsChildReserved = z.infer; +export function transitChildCandidateId( + execution: Pick, + phase: DataOpsChildPhase, + ordinal: number, +): string { + return `ctrl-${createHash("sha256").update(`openmapx-transit-candidate-v1\0${execution.operationKey}\0${phase}\0${ordinal}`).digest("hex").slice(0, 32)}`; +} +export function dataChildControlReservation( + child: DataOpsChildReserved | DataOpsChildAttempt, +): ControlChildReservation { + return { + version: 1, + phase: child.phase, + ordinal: child.ordinal, + kind: child.operation.kind, + recipient: "ops-agent", + serviceIds: child.serviceIds, + operationFingerprint: child.operationFingerprint, + execution: { + executor: "ops-agent", + principal: "data-manager", + executionId: child.proposedId, + operationKey: child.operationKey, + }, + }; +} +/** Full typed bodies and IDs are persisted at root admission, before build writes. */ +export function reserveTransitChildOperations( + execution: ExecutionRef, + input: DataExecutionInput, +): DataOpsChildReserved[] { + const entries: Array<{ phase: DataOpsChildPhase; operation: OpsOperation; targets: string[] }> = + []; + if (input.kind === "transit-sync") { + entries.push( + { phase: "stop-primary", operation: { kind: "motis.primary.stop" }, targets: ["motis"] }, + { + phase: "stop-staging", + operation: { kind: "motis.staging.stop" }, + targets: ["motis-staging"], + }, + { + phase: "restart-staging", + operation: { kind: "motis.staging.restart" }, + targets: ["motis-staging"], + }, + { + phase: "promote-primary", + operation: { kind: "motis.primary.promote", preparedRunId: execution.executionId }, + targets: ["motis"], + }, + { + phase: "recover-primary", + operation: { kind: "motis.primary.restart" }, + targets: ["motis"], + }, + ); + for (const phase of ["proxy-stage", "proxy-commit", "proxy-restore"] as const) { + const ordinal = entries.length; + entries.push({ + phase, + operation: { + kind: "feedProxy.validateAndReload", + candidateId: transitChildCandidateId(execution, phase, ordinal), + }, + targets: ["motis-feed-proxy"], + }); + } + } else if (input.kind === "transit-maintenance") { + if (input.action === "restart-primary") + entries.push({ + phase: "recover-primary", + operation: { kind: "motis.primary.restart" }, + targets: ["motis"], + }); + else { + const expected = transitChildCandidateId(execution, "proxy-commit", 0); + if (input.candidateId !== expected) + throw Error("Feed proxy candidate must be reserved by this root"); + entries.push({ + phase: "proxy-commit", + operation: { kind: "feedProxy.validateAndReload", candidateId: expected }, + targets: ["motis-feed-proxy"], + }); + } + } + return entries.map(({ phase, operation, targets }, ordinal) => + dataOpsChildReservedSchema.parse({ + parentExecution: execution, + phase, + ordinal, + operation, + operationFingerprint: opsOperationFingerprint(operation), + operationKey: controlChildOperationKey(execution.operationKey, phase, ordinal), + proposedId: `job1_${randomUUID().replaceAll("-", "")}`, + serviceIds: targets, + cancellationRequestedAt: null, + state: "reserved", + publication: "not_started", + }), + ); +} +export function sameDataChildBody( + a: DataOpsChildReserved | DataOpsChildAttempt, + b: DataOpsChildReserved | DataOpsChildAttempt, +): boolean { + const immutable = (c: DataOpsChildReserved | DataOpsChildAttempt) => ({ + parentExecution: c.parentExecution, + phase: c.phase, + ordinal: c.ordinal, + operation: c.operation, + operationFingerprint: c.operationFingerprint, + operationKey: c.operationKey, + proposedId: c.proposedId, + serviceIds: c.serviceIds, + }); + return canonicalJson(immutable(a)) === canonicalJson(immutable(b)); +} + +import { setTimeout as delay } from "node:timers/promises"; +import { + assertOwnedDataTransitCandidate, + bindControlChildPhase, + consumeControlChildDispatch, + controlRolePublicKey, + inspectArtifactTargetLeaseChildSettlement, + issueControlChildPhase, + lookupControlChildPhaseAttempt, + lookupControlChildPhaseBinding, + verifyFeedProxyCanonicalTerminal, +} from "@openmapx/core/build-artifacts"; +import { type OpsJobStatus, type OpsResultFor, parseOpsResult } from "@openmapx/core/ops"; +import { opsClient } from "../../ops-client.js"; +import type { DataExecutionContext } from "./runtime.js"; +import type { FreshDataChildReservation } from "./store.js"; +export class DataOpsObservationPending extends Error { + constructor(readonly operationKey: string) { + super("Owned data child requires observation and settlement"); + } +} +export class DataOpsChildFailed extends Error {} +export async function lookupOwnedDataOpsChildren( + context: DataExecutionContext, +): Promise { + return (await context.record()).children; +} +function sameStatus(child: DataOpsChildAttempt, status: OpsJobStatus) { + if ( + child.state === "reserved" || + status.version !== 2 || + status.operationId !== child.proposedId || + status.operationKey !== child.operationKey || + status.kind !== child.operation.kind || + status.fingerprint !== child.execution.fingerprint || + status.deadlineAt !== child.execution.deadlineAt + ) + throw new DataOpsObservationPending(child.operationKey); +} +async function checkpointCurrent( + context: DataExecutionContext, + child: T, +) { + await context.checkpointChild(child); + return child; +} +async function observeChild( + context: DataExecutionContext, + attempt: DataOpsChildAttempt, + signal: AbortSignal, + cancel = false, +): Promise { + if (attempt.state === "reserved" || attempt.state === "bound") + throw new DataOpsObservationPending(attempt.operationKey); + let child = attempt; + if (cancel && child.state !== "terminal") { + child = await checkpointCurrent(context, { + ...child, + cancellationRequestedAt: child.cancellationRequestedAt ?? new Date().toISOString(), + }); + try { + await opsClient().cancel(child.proposedId, { + fingerprint: child.execution.fingerprint, + signal: AbortSignal.any([signal, AbortSignal.timeout(15000)]), + }); + } catch { + /* exact durable intent permits cancel retry, never POST replay */ + } + } + while (true) { + signal.throwIfAborted(); + let status: OpsJobStatus; + try { + status = await opsClient().lookup(child.operation, child.operationKey, { + fingerprint: child.execution.fingerprint, + execution: { requestedDeadlineAt: child.requestedDeadlineAt }, + signal, + }); + } catch { + await context.checkpoint({ publication: "unknown" }); + await delay(100, undefined, { signal }); + continue; + } + sameStatus(child, status); + const saved: DataOpsChildAttempt = { + ...child, + state: "observing", + acceptedId: status.operationId, + publication: child.publication, + }; + if (child.state !== "terminal") child = await checkpointCurrent(context, saved); + if (["succeeded", "failed", "timed_out"].includes(status.state)) { + let evidence: ReturnType; + try { + evidence = inspectArtifactTargetLeaseChildSettlement( + context.targetAuthority, + child.execution, + ); + } catch { + await delay(100, undefined, { signal }); + continue; + } + if (evidence && evidence.outcome !== "unknown" && evidence.publication !== "unknown") { + if (child.operation.kind === "feedProxy.validateAndReload") { + try { + verifyFeedProxyCanonicalTerminal(context.authority, child.execution); + } catch { + await delay(100, undefined, { signal }); + continue; + } + } + child = await checkpointCurrent(context, { + ...child, + state: "terminal", + acceptedId: status.operationId, + publication: evidence.publication, + settlement: { + outcome: evidence.outcome, + terminalReceiptDigest: evidence.terminalReceiptDigest, + }, + }); + if (evidence.publication === "committed") + await context.checkpoint({ publication: "committed" }); + return status; + } + } + await delay(100, undefined, { signal }); + } +} +/** Body-first confirmed CAS -> signed one-time binding -> grant/ref -> confirmed + * claimed CAS -> consume fresh nominal ticket -> exactly one mutation POST. */ +type OwnedDataOpsOptions = { + context: DataExecutionContext; + phase: DataOpsChildPhase; + ordinal: number; + serviceIds: readonly string[]; + reservedContinuation?: FreshDataChildReservation; +}; +/** Reserve the original restore before touching either live file. Only the + * confirmed live CAS continuation may later dispatch this exact same phase. */ +export async function reserveOwnedProxyRecovery( + context: DataExecutionContext, + backupDigest: string, +): Promise { + context.assertForward(); + const record = await context.record(), + slot = record.reservations.find((child) => child.phase === "proxy-restore"); + if (!slot) throw Error("Original proxy restore reservation missing"); + const reserved = await context.write(async (current) => { + const saved = await context.store.reserveChild(current.status.id, current.version, { + ...slot, + proxyRecovery: { version: 1, backupDigest }, + }); + if (!saved?.continuation) throw new DataOpsObservationPending(slot.operationKey); + return { record: saved.record, value: saved.continuation }; + }); + if (!context.store.controlJournal) throw Error("Original proxy recovery journal missing"); + await context.store.controlJournal.retainProxyRecoveryIntent(context.authority, backupDigest); + context.assertForward(); + return reserved; +} +export async function runOwnedDataOpsOperation( + value: Extract, + options: OwnedDataOpsOptions, +): Promise> { + const operation = operationSchema.parse(value); + const { context } = options, + record = await context.record(), + slot = record.reservations.find( + (child) => child.phase === options.phase && child.ordinal === options.ordinal, + ); + if ( + !slot || + slot.operationFingerprint !== opsOperationFingerprint(operation) || + canonicalJson(slot.operation) !== canonicalJson(operation) || + canonicalJson(slot.serviceIds) !== canonicalJson(options.serviceIds) + ) + throw Error("Operation differs from original data phase reservation"); + if (operation.kind === "transitousLock.approve") + throw Error("Paired pin approval requires the validation-bound activation runtime"); + let child: DataOpsChildAttempt | undefined = record.children.find( + (value) => value.operationKey === slot.operationKey, + ); + if (!child || (child.state === "reserved" && options.reservedContinuation)) { + context.assertForward(); + const candidateDigest = assertOwnedDataTransitCandidate(context.authority, slot); + if (!child && slot.phase === "proxy-restore") + throw Error("Restore must reserve before filesystem writes"); + const reserved = options.reservedContinuation + ? { record, continuation: options.reservedContinuation } + : await context.write(async (current) => { + const saved = await context.store.reserveChild(current.status.id, current.version, slot); + if (!saved) throw new DataOpsObservationPending(slot.operationKey); + return { record: saved.record, value: saved }; + }); + child = reserved.record.children.find((value) => value.operationKey === slot.operationKey); + if (reserved.continuation) { + const reservedBody = child?.state === "reserved" ? child : slot; + const exact = await bindControlChildPhase(context.authority, slot), + binding = lookupControlChildPhaseBinding(context.authority, slot); + if (!binding) throw new DataOpsObservationPending(slot.operationKey); + const bound: DataOpsChildAttempt = { + ...reservedBody, + state: "bound", + execution: exact.execution, + requestedDeadlineAt: binding.requestedDeadlineAt, + bindingDigest: binding.bindingDigest, + candidateDigest, + }; + await context.checkpointChild(bound); + child = bound; + context.assertForward(); + if (assertOwnedDataTransitCandidate(context.authority, slot) !== candidateDigest) + throw Error("Owned candidate changed before grant"); + const issued = await issueControlChildPhase(context.authority, { + phase: slot.phase, + ordinal: slot.ordinal, + lease: context.targetAuthority, + childPublicKey: controlRolePublicKey(context.installation, "ops-agent"), + }); + const reference = lookupControlChildPhaseAttempt(context.authority, slot); + if (!reference?.leaseId) throw new DataOpsObservationPending(slot.operationKey); + const attempted: DataOpsChildAttempt = { + ...bound, + state: "attempted", + delegationId: reference.delegationId, + grantDigest: reference.grantDigest, + leaseId: reference.leaseId, + dispatchState: "claimed", + publication: "unknown", + }; + if (attempted.state !== "attempted") throw Error("Invalid owned dispatch state"); + await context.write(async (current) => { + const saved = await context.store.claimChildDispatch( + current.status.id, + current.version, + attempted, + reserved.continuation!, + ); + if (!saved) throw new DataOpsObservationPending(slot.operationKey); + return { record: saved, value: undefined }; + }); + child = attempted; + await context.checkpoint({ publication: "unknown" }); + consumeControlChildDispatch(issued.dispatchTicket); + try { + const admitted = await opsClient().execute(operation, { + operationKey: child.operationKey, + execution: { requestedDeadlineAt: child.execution.deadlineAt }, + controlAssertion: issued.assertion, + signal: context.signal, + }); + if (admitted.execution !== "async" || admitted.operationId !== child.proposedId) + throw new DataOpsObservationPending(child.operationKey); + child = await checkpointCurrent(context, { + ...child, + state: "observing", + acceptedId: admitted.operationId, + }); + } catch { + /* first POST is conservatively attempted; only exact lookup follows */ + } + } + } + if (!child) throw new DataOpsObservationPending(slot.operationKey); + try { + let status: OpsJobStatus; + try { + status = await observeChild(context, child, context.signal); + } catch (error) { + if (!context.signal.aborted) throw error; + // A canceled progress/worker signal cannot own descendant settlement. + // Repeat bounded reads/cancel intent on the original identity; these + // live read budgets grant no new forward work or execution time. + for (;;) { + const current = (await context.record()).children.find( + (entry) => entry.operationKey === slot.operationKey, + ); + if (!current) throw error; + try { + status = await observeChild(context, current, AbortSignal.timeout(30000), true); + break; + } catch (pending) { + if (pending instanceof DataOpsChildFailed) throw pending; + await delay(250); + } + } + } + if (status.state !== "succeeded") + throw new DataOpsChildFailed("Owned host child failed with contained effect evidence"); + // Runtime parsing validates the exact admitted kind; this bridges the + // generic Extract result type and constructs no authority. + return parseOpsResult(operation.kind, status.result) as OpsResultFor; + } catch (error) { + if (error instanceof DataOpsChildFailed) throw error; + throw new DataOpsObservationPending(slot.operationKey); + } +} +export async function cancelAndSettleOwnedDataOpsChildren( + context: DataExecutionContext, +): Promise<"settled" | "pending"> { + const record = await context.record(), + signal = AbortSignal.timeout(30000); + let pending = false; + for (const slot of record.reservations) { + let child = record.children.find((value) => value.operationKey === slot.operationKey); + let reference: ReturnType; + try { + reference = lookupControlChildPhaseAttempt(context.authority, slot); + } catch { + pending = true; + continue; + } + if (!child && !reference) continue; + if (child?.state === "terminal") continue; + try { + if (reference && (!child || child.state === "reserved" || child.state === "bound")) { + // Recover only the original binding/grant. No issuance or dispatch ticket. + const binding = lookupControlChildPhaseBinding(context.authority, slot); + if (!binding || !reference.leaseId) throw new DataOpsObservationPending(slot.operationKey); + if (!child) throw new DataOpsObservationPending(slot.operationKey); + const bound = { + ...slot, + state: "bound" as const, + execution: binding.child.execution, + requestedDeadlineAt: binding.requestedDeadlineAt, + bindingDigest: binding.bindingDigest, + candidateDigest: + child.state === "bound" + ? child.candidateDigest + : assertOwnedDataTransitCandidate(context.authority, slot), + }; + if (child.state === "reserved") await context.checkpointChild(bound); + child = { + ...bound, + state: "attempted", + delegationId: reference.delegationId, + grantDigest: reference.grantDigest, + leaseId: reference.leaseId, + dispatchState: "pending", + publication: "unknown", + }; + await context.checkpointChild(child); + } + if (!child || child.state === "reserved" || child.state === "bound") + throw new DataOpsObservationPending(slot.operationKey); + await observeChild( + context, + child, + signal, + record.status.cancellationRequestedAt !== null || context.signal.aborted, + ); + } catch { + pending = true; + } + } + return pending ? "pending" : "settled"; +} diff --git a/services/data-manager/src/jobs/execution/receiver.ts b/services/data-manager/src/jobs/execution/receiver.ts new file mode 100644 index 000000000..5ca16ce37 --- /dev/null +++ b/services/data-manager/src/jobs/execution/receiver.ts @@ -0,0 +1,117 @@ +import { randomBytes } from "node:crypto"; +import { + type ControlInstallation, + controlHeaders, + inspectDataExecutionAdmission, +} from "@openmapx/core/build-artifacts"; +import { + normalizeDataExecutionInput, + startGenerationHardlinksExecutionSchema, +} from "@openmapx/core/data-manager-contracts"; +import { createControlRootOperationKey, opsExecutionBudget } from "@openmapx/core/ops"; +import type { MotisOperationsPolicy } from "../transitous/operations-profile.js"; +import type { DataExecutionReceiver } from "./api.js"; +import type { DataConfigurationMaintenance } from "./configuration-maintenance.js"; +import type { createDataExecutionDispatcher } from "./dispatch.js"; +import { transitChildCandidateId } from "./ops-children.js"; +import type { DataExecutionRuntime } from "./runtime.js"; +import { dataExecutionInputFingerprint } from "./store.js"; +export function createDataExecutionReceiver(deps: { + runtime: DataExecutionRuntime; + installation: ControlInstallation; + configurationFence: DataConfigurationMaintenance; + dispatcher: ReturnType; + operationsPolicy: MotisOperationsPolicy; +}): DataExecutionReceiver { + const { runtime, installation, configurationFence } = deps; + const validate = (value: Parameters[0]) => { + const input = normalizeDataExecutionInput(value); + if ( + input.kind === "transit-sync" && + deps.operationsPolicy.profile !== "planet" && + (input.countries.length === 0 || + input.countries.some((country) => !deps.operationsPolicy.countries.includes(country))) + ) + throw Object.assign(Error("Countries outside configured operations profile"), { + statusCode: 422, + }); + return input; + }; + return { + async admit(value, options) { + const input = validate(value); + if (input.kind === "generation-hardlinks" || input.kind === "transit-maintenance") + throw Object.assign(Error("Service-internal data input"), { statusCode: 403 }); + const verified = inspectDataExecutionAdmission(installation, { + ...options, + input, + operationFingerprint: dataExecutionInputFingerprint(input), + }); + return runtime.admit(input, { + operationKey: options.operationKey, + requestedDeadlineAt: options.requestedDeadlineAt, + ...(verified + ? { + parent: verified.parent, + id: verified.execution.executionId, + principal: verified.execution.principal, + adopt: verified.adopt, + } + : {}), + }); + }, + async admitHardlinks(value, rawHeaders) { + const request = startGenerationHardlinksExecutionSchema.parse(value), + verified = inspectDataExecutionAdmission(installation, { + ...request, + rawHeaders, + operationFingerprint: dataExecutionInputFingerprint(request.input), + }); + if (!verified || !controlHeaders(rawHeaders).delegation) + throw Object.assign(Error("Signed borrowed generation leaf required"), { statusCode: 403 }); + await configurationFence.assertGenerationAdmission(request, rawHeaders); + return runtime.admit(request.input, { + operationKey: request.operationKey, + requestedDeadlineAt: request.requestedDeadlineAt, + parent: verified.parent, + id: verified.execution.executionId, + principal: verified.execution.principal, + adopt: verified.adopt, + hardlinkPlan: request.plan, + }); + }, + async admitLocal(value, source) { + const input = validate(value), + requestedDeadlineAt = Date.now() + opsExecutionBudget(input.kind).processMs, + operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }); + return runtime.admit(input, { operationKey, requestedDeadlineAt, ...source }); + }, + async admitMaintenance(action, options) { + const requestedDeadlineAt = Date.now() + 120000, + operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }); + const input = + action === "restart-primary" + ? { kind: "transit-maintenance" as const, action } + : { + kind: "transit-maintenance" as const, + action, + candidateId: transitChildCandidateId({ operationKey }, "proxy-commit", 0), + }; + return runtime.admit(input, { operationKey, requestedDeadlineAt, ...options }); + }, + inspect: runtime.inspect, + lookup: runtime.lookup, + requestCancel: runtime.requestCancel, + prepareFence: (input, headers) => configurationFence.prepare(input, headers), + inspectFence: (id, headers) => configurationFence.inspect(id, headers), + settleFence: (input, headers) => configurationFence.settle(input, headers), + subscribe: deps.dispatcher.subscribe, + legacyResult: deps.dispatcher.legacyResult, + }; +} diff --git a/services/data-manager/src/jobs/execution/runner-cancellation.test.ts b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts new file mode 100644 index 000000000..b69c5fea8 --- /dev/null +++ b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts @@ -0,0 +1,130 @@ +import { randomBytes } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; +import postgres from "postgres"; +import { expect, it } from "vitest"; +import { buildTransitousRunnerServer } from "../../../../../apps/transitous-runner/src/server"; +import { createTransitousScriptRunner } from "../transitous/script-runner"; +import { createDataExecutionRuntime } from "./runtime"; +import { createPostgresDataExecutionStore } from "./store"; + +it.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)( + "cancels the actual retained remote Python process group after the cancellation response is lost", + async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }), + root = mkdtempSync(join(tmpdir(), "omx-data-runner-cancel-")), + catalog = join(root, "catalog"), + ready = join(catalog, "ready"), + released = join(catalog, "release"); + mkdirSync(join(root, "data"), { mode: 0o700 }); + mkdirSync(join(catalog, "src"), { recursive: true }); + await control.provisionControlInstallation(root); + writeFileSync( + join(catalog, "src/garbage-collect.py"), + "import os,time\nfrom pathlib import Path\nPath('ready').write_text(str(os.getpid()))\nwhile not Path('release').exists(): time.sleep(0.05)\n", + ); + const key = Buffer.alloc(32, 7), + runner = buildTransitousRunnerServer({ + catalogDir: catalog, + stagingDir: root, + capabilityKey: key, + }); + let posts = 0, + cancels = 0; + runner.addHook("onSend", async (request, reply, payload) => { + if (request.url === "/v1/run") posts++; + if (request.url === "/v1/run/cancel") { + cancels++; + if (cancels === 1) reply.raw.destroy(); + } + return payload; + }); + await runner.listen({ host: "127.0.0.1", port: 0 }); + const address = runner.server.address(); + if (!address || typeof address === "string") throw Error("Runner endpoint missing"); + const run = createTransitousScriptRunner({ + baseUrl: `http://127.0.0.1:${address.port}`, + capabilityKey: key, + }); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }), + runtime = createDataExecutionRuntime({ + store, + dispatch: async (_input, context) => { + await run({ script: "garbage-collect" }); + context.signal.throwIfAborted(); + return { + kind: "transit-sync", + finalStatus: "ok", + sourceRevision: "public-fixture", + countries: ["de"], + }; + }, + }); + let id: string | undefined, operationKey: string | undefined; + try { + const deadline = Date.now() + 60000; + operationKey = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + const admitted = await runtime.admit( + { kind: "transit-sync", countries: ["de"] }, + { operationKey, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await expect.poll(() => existsSync(ready), { timeout: 10000 }).toBe(true); + const pid = Number(readFileSync(ready, "utf8")); + const pending = await runtime.requestCancel({ + id, + operationKey, + fingerprint: admitted.fingerprint, + }); + expect(pending.status).toBe("cancel_pending"); + await expect + .poll( + () => { + try { + process.kill(-pid, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } + }, + { timeout: 2000 }, + ) + .toBe(false); + await expect + .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) + .toBe("canceled"); + expect((await store.inspect(id))?.status.publication).toBe("not_started"); + expect((await store.inspect(id))?.finishedAt).not.toBeNull(); + expect(posts).toBe(1); + expect(cancels).toBeGreaterThanOrEqual(1); + expect(existsSync(join(root, "data/.admin-target-control/locks/motis"))).toBe(false); + } finally { + writeFileSync(released, "release"); + await runtime.close(); + runner.server.closeAllConnections(); + await runner.close(); + if (!id && operationKey) id = (await store.lookup(operationKey))?.status.id; + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); diff --git a/services/data-manager/src/jobs/execution/runtime.test.ts b/services/data-manager/src/jobs/execution/runtime.test.ts new file mode 100644 index 000000000..9bee92232 --- /dev/null +++ b/services/data-manager/src/jobs/execution/runtime.test.ts @@ -0,0 +1,618 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, existsSync as requireExists, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { describe, expect, it, vi } from "vitest"; +import { StateStore } from "../../state.js"; +import { preserveTransitPending } from "../transitous/owned-ops.js"; +import { buildJobContext } from "../transitous/pipeline.js"; +import { createDataExecutionRuntime } from "./runtime.js"; +import { createPostgresDataExecutionStore } from "./store.js"; +import { publishDataChange, withDataSqlCancellation } from "./worker-scope.js"; + +describe.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)("real durable data runtime containment", () => { + it.each(["committed", "canceled"] as const)( + "retains cancel-pending and no final time until actual worker settles (%s)", + async (outcome) => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + drizzle(sql); + const root = mkdtempSync(join(tmpdir(), "omx-runtime-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + let release = () => {}, + started = () => {}, + aborted = () => {}; + const held = new Promise((resolve) => { + release = resolve; + }), + ready = new Promise((resolve) => { + started = resolve; + }), + canceled = new Promise((resolve) => { + aborted = resolve; + }); + let effects = 0; + const runtime = createDataExecutionRuntime({ + store, + dispatch: async (_input, context) => { + effects++; + started(); + context.signal.addEventListener("abort", aborted, { once: true }); + await held; + if (outcome === "canceled") context.signal.throwIfAborted(); + return { kind: "download-fonts", completed: true }; + }, + }); + let id: string | undefined; + let key: string | undefined; + try { + const deadline = Date.now() + 60000; + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + const admitted = await runtime.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await ready; + const running = await store.inspect(id); + await expect( + store.checkpoint(id, running!.version, { + status: "ok", + publication: "committed", + result: { kind: "download-fonts", completed: true }, + }), + ).rejects.toThrow("Actual contained worker terminal commitment required"); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + const duplicate = await runtime.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ); + expect(duplicate.id).toBe(id); + expect(effects).toBe(1); + const pending = await runtime.requestCancel({ + id, + operationKey: key, + fingerprint: admitted.fingerprint, + }); + await canceled; + expect(pending.status).toBe("cancel_pending"); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + release(); + await expect + .poll(async () => (await runtime.inspect(id!))?.status, { timeout: 15000 }) + .toBe(outcome === "committed" ? "ok" : "canceled"); + const terminal = await store.inspect(id); + expect(terminal?.finishedAt).not.toBeNull(); + expect(terminal?.status.publication).toBe( + outcome === "committed" ? "committed" : "not_started", + ); + expect( + ( + await runtime.requestCancel({ + id, + operationKey: key, + fingerprint: admitted.fingerprint, + }) + ).status, + ).toBe(terminal?.status.status); + const proof = await journal.repairTerminal(terminal!.execution); + expect(proof.execution).toEqual(terminal!.execution); + await journal.publishTerminal(terminal!.execution); + await expect( + runtime.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline + 1 }, + ), + ).rejects.toThrow(/conflict/i); + } finally { + release(); + await runtime.close(); + if (!id && key) id = (await store.lookup(key))?.status.id; + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + ); + it("repairs a worker-completion/publication crash from the original signed completion without redispatch", async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }), + root = mkdtempSync(join(tmpdir(), "omx-data-completion-gap-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + let installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }); + let journal = control.createDataExecutionControlJournal(installation, sql); + let store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + let dispatches = 0; + const broken = new Proxy(store, { + get(target, key, receiver) { + return key === "finalize" + ? async () => { + throw Error("Synthetic crash before terminal preparation"); + } + : Reflect.get(target, key, receiver); + }, + }); + let runtime = createDataExecutionRuntime({ + store: broken, + dispatch: async () => { + dispatches++; + return { kind: "download-fonts", completed: true }; + }, + }), + id: string | undefined, + key: string | undefined; + try { + const deadline = Date.now() + 60000; + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + const admitted = await runtime.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await expect + .poll(async () => (await store.inspect(id!))?.status.publication, { timeout: 15000 }) + .toBe("unknown"); + await runtime.close(); + journal.close(); + control.closeControlInstallation(installation); + installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }); + journal = control.createDataExecutionControlJournal(installation, sql); + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + runtime = createDataExecutionRuntime({ + store, + dispatch: async () => { + dispatches++; + throw Error("Old execution must never be replayed"); + }, + }); + await runtime.reconcile(); + expect((await store.inspect(id))?.status).toMatchObject({ + status: "ok", + publication: "committed", + result: { kind: "download-fonts", completed: true }, + }); + expect((await store.inspect(id))?.finishedAt).not.toBeNull(); + expect(dispatches).toBe(1); + expect( + (await import("node:fs")).existsSync( + join(root, "data/.admin-target-control/locks/data-manager"), + ), + ).toBe(false); + const recovered = await store.inspect(id); + expect( + control.verifyDataControlTerminalWitness(installation, recovered!.execution).status, + ).toEqual(recovered!.status); + await runtime.reconcile(); + expect(dispatches).toBe(1); + } finally { + await runtime.close(); + if (!id && key) id = (await store.lookup(key))?.status.id; + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, 60000); + it.each(["before-commit", "during-commit"] as const)( + "cancels the actual owned PostgreSQL transaction (%s)", + async (mode) => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 5 }), + workerSql = withDataSqlCancellation(sql), + root = mkdtempSync(join(tmpdir(), "omx-data-transaction-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + const suffix = randomUUID().replaceAll("-", ""), + parent = `task023_parent_${suffix}`, + child = `task023_child_${suffix}`; + await sql.unsafe(`CREATE TABLE data_manager.${parent}(id int PRIMARY KEY)`); + await sql.unsafe( + `CREATE TABLE data_manager.${child}(id int PRIMARY KEY,parent int REFERENCES data_manager.${parent}(id) DEFERRABLE INITIALLY DEFERRED)`, + ); + await sql.unsafe(`INSERT INTO data_manager.${parent} VALUES(1)`); + let ready = () => {}, + unblock = () => {}, + locked = () => {}, + unlock = () => {}, + workerPid = 0; + const workerReady = new Promise((resolve) => { + ready = resolve; + }), + held = new Promise((resolve) => { + unblock = resolve; + }), + lockerReady = new Promise((resolve) => { + locked = resolve; + }), + lockerHeld = new Promise((resolve) => { + unlock = resolve; + }); + const locker = + mode === "during-commit" + ? sql + .begin(async (tx) => { + await tx.unsafe(`DELETE FROM data_manager.${parent} WHERE id=1`); + locked(); + await lockerHeld; + throw Error("Synthetic locker rollback"); + }) + .catch(() => undefined) + : Promise.resolve(); + if (mode === "during-commit") await lockerReady; + const runtime = createDataExecutionRuntime({ + store, + dispatch: async () => { + await publishDataChange(() => + workerSql.begin(async (tx) => { + const pid = await tx`SELECT pg_backend_pid() AS pid`; + workerPid = Number(pid[0]?.pid); + await tx.unsafe(`INSERT INTO data_manager.${child} VALUES(1,1)`); + ready(); + if (mode === "before-commit") await held; + }), + ); + return { kind: "download-fonts", completed: true }; + }, + }); + let id: string | undefined, key: string | undefined; + try { + const deadline = Date.now() + 60000; + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + const admitted = await runtime.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await workerReady; + if (mode === "during-commit") + await expect + .poll( + async () => { + const rows = + await sql`SELECT query,wait_event_type FROM pg_stat_activity WHERE pid=${workerPid}`; + return rows[0]?.query === "COMMIT" && rows[0]?.wait_event_type === "Lock"; + }, + { timeout: 10000 }, + ) + .toBe(true); + const pending = await runtime.requestCancel({ + id, + operationKey: key, + fingerprint: admitted.fingerprint, + }); + expect(pending.status).toBe("cancel_pending"); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + if (mode === "before-commit") { + unblock(); + await expect + .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) + .toBe("canceled"); + expect((await store.inspect(id))?.status.publication).toBe("not_started"); + } else { + await expect + .poll( + async () => { + const rows = await sql`SELECT state FROM pg_stat_activity WHERE pid=${workerPid}`; + return rows[0]?.state; + }, + { timeout: 15000 }, + ) + .toBe("idle"); + expect((await store.inspect(id))?.status).toMatchObject({ + status: "cancel_pending", + publication: "unknown", + }); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + } + expect( + (await sql.unsafe(`SELECT count(*)::int AS count FROM data_manager.${child}`))[0]?.count, + ).toBe(0); + } finally { + unblock(); + unlock(); + await locker; + await runtime.close(); + if (!id && key) id = (await store.lookup(key))?.status.id; + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + await sql.unsafe(`DROP TABLE data_manager.${child}`); + await sql.unsafe(`DROP TABLE data_manager.${parent}`); + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, + ); + + it("retains partial publication and its original completion across restart without replay", async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + drizzle(sql); + const root = mkdtempSync(join(tmpdir(), "omx-data-partial-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + let installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }), + dispatches = 0; + const broken = new Proxy(store, { + get(target, key, receiver) { + return key === "finalize" + ? async () => { + throw Error("Synthetic crash before partial terminal publication"); + } + : Reflect.get(target, key, receiver); + }, + }); + let runtime = createDataExecutionRuntime({ + store: broken, + dispatch: async () => { + dispatches++; + throw new control.DataExecutionPartialResult({ + kind: "overture-sync", + region: "europe/germany/berlin", + release: "2026-07-22.0", + metrics: { processed: 0 }, + }); + }, + }), + id: string | undefined; + try { + const deadline = Date.now() + 60000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + admitted = await runtime.admit( + { kind: "overture-sync", region: "europe/germany/berlin" }, + { operationKey: key, requestedDeadlineAt: deadline, triggeredBy: "cron" }, + ); + id = admitted.id; + await expect.poll(async () => (await store.inspect(id!))?.status.publication).toBe("unknown"); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + await runtime.close(); + journal.close(); + control.closeControlInstallation(installation); + installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }); + journal = control.createDataExecutionControlJournal(installation, sql); + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + runtime = createDataExecutionRuntime({ + store, + dispatch: async () => { + dispatches++; + throw Error("No replay"); + }, + }); + await runtime.reconcile(); + const retained = await store.inspect(id); + expect(retained?.status).toMatchObject({ + status: "partial", + publication: "committed", + result: { kind: "overture-sync", release: "2026-07-22.0" }, + }); + expect(retained?.triggeredBy).toBe("cron"); + expect(retained?.finishedAt).not.toBeNull(); + expect( + control.verifyDataControlTerminalWitness(installation, retained!.execution).status, + ).toEqual(retained!.status); + expect(dispatches).toBe(1); + } finally { + await runtime.close(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, 60000); + it("never compensates an uncertain local transit publication", async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + drizzle(sql); + const root = mkdtempSync(join(tmpdir(), "omx-transit-publication-")), + data = join(root, "data"); + mkdirSync(data, { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + let forwards = 0; + const runtime = createDataExecutionRuntime({ + store, + dispatch: async (_input, context) => { + const job = buildJobContext({ + dataDir: data, + store: new StateStore(data), + jobId: context.execution.executionId, + countries: ["de"], + executionContext: context, + abortSignal: context.signal, + }); + try { + await publishDataChange(async () => { + const { writeFileSync } = await import("node:fs"); + writeFileSync(join(data, "synthetic-live-publication"), "committed prefix"); + throw Error("Synthetic lost local publication response"); + }); + } catch (error) { + preserveTransitPending(job, error); + forwards++; + throw error; + } + return { + kind: "transit-sync", + finalStatus: "ok", + sourceRevision: "public-fixture", + countries: ["de"], + }; + }, + }); + let id: string | undefined; + try { + const deadline = Date.now() + 60000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + admitted = await runtime.admit( + { kind: "transit-sync", countries: ["de"] }, + { operationKey: key, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await expect + .poll( + async () => { + const { existsSync } = await import("node:fs"); + return existsSync( + join(root, "control-private/data-manager/observations", `worker-${key}.json`), + ); + }, + { timeout: 15000 }, + ) + .toBe(true); + expect(forwards).toBe(0); + expect((await store.inspect(id))?.finishedAt).toBeNull(); + expect((await store.inspect(id))?.status.publication).toBe("unknown"); + expect( + (await import("node:fs")).existsSync(join(data, ".admin-target-control/locks/motis")), + ).toBe(true); + } finally { + await runtime.close(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, 60000); + it("retires the indexed unbound phase cells with the original completed family after its expiry floor", async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + drizzle(sql); + const root = mkdtempSync(join(tmpdir(), "omx-phase-retirement-")), + data = join(root, "data"); + mkdirSync(data, { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { + controlJournal: journal, + now: () => Date.now(), + }), + runtime = createDataExecutionRuntime({ + store, + dispatch: async () => ({ + kind: "transit-sync", + finalStatus: "ok", + sourceRevision: "public-fixture", + countries: ["de"], + }), + }); + let id: string | undefined, clock: ReturnType | undefined; + try { + const deadline = Date.now() + 60000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + admitted = await runtime.admit( + { kind: "transit-sync", countries: ["de"] }, + { operationKey: key, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await expect + .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) + .toBe("ok"); + await expect + .poll( + async () => + !(await import("node:fs")).existsSync( + join(data, ".admin-target-control/locks/data-manager"), + ), + { timeout: 15000 }, + ) + .toBe(true); + await runtime.close(); + const record = await store.inspect(id); + expect(record?.reservations).toHaveLength(8); + const cells = record!.reservations.map((slot) => + join( + root, + "control-private/data-manager/observations", + `phase-${key}-${slot.ordinal}.json`, + ), + ); + expect(cells.every((path) => requireExists(path))).toBe(true); + clock = vi.spyOn(Date, "now").mockReturnValue(deadline + 72 * 3600000 + 1); + let retired = 0; + for (let n = 0; n < 64 && retired === 0; n++) + retired += (await control.reconcileControlRetention(installation, { maxFamilies: 1 })) + .retired; + expect(retired).toBe(1); + expect(await store.inspect(id)).toBeNull(); + expect(cells.every((path) => !requireExists(path))).toBe(true); + await expect( + store.admit( + { kind: "transit-sync", countries: ["de"] }, + { operationKey: key, requestedDeadlineAt: deadline }, + ), + ).rejects.toThrow(/fresh dated|expired/i); + } finally { + clock?.mockRestore(); + await runtime.close(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, 60000); +}); diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts new file mode 100644 index 000000000..b4b20a786 --- /dev/null +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -0,0 +1,542 @@ +import { setTimeout as settleDelay } from "node:timers/promises"; +import { + acquireArtifactTargetLease, + assertArtifactTargetLeaseForward, + assertControlExecutionForward, + type BorrowedArtifactTargetLease, + bindBorrowedControlExecution, + ControlExecutionFencedError, + type ControlInstallation, + cancelArtifactTargetLease, + cancelControlExecution, + closeArtifactTargetLeaseHandle, + releaseArtifactTargetLease, + restoreAdmittedControlExecution, + restoreArtifactTargetLeaseForSettlement, + restoreGenerationHardlinksLeaseForSettlement, + retainAdmittedControlExecution, + settleArtifactTargetLeaseChild, + type TargetLeaseAuthority, + type VerifiedExecutionAuthority, + verifyControlEffectSettlement, +} from "@openmapx/core/build-artifacts"; +import { + type DataExecutionIdentity, + type DataExecutionInput, + type DataExecutionResult, + type DataExecutionStatus, + dataExecutionResultSchema, + isDataExecutionTerminal, +} from "@openmapx/core/data-manager-contracts"; +import type { ExecutionRef, VerifiedParentExecution } from "@openmapx/core/ops"; +import { + cancelAndSettleOwnedDataOpsChildren, + type DataOpsChildAttempt, + dataChildControlReservation, +} from "./ops-children.js"; +import type { DataExecutionPatch, DataExecutionRecord, DataExecutionStore } from "./store.js"; +import { withDataExecutionScope } from "./worker-scope.js"; +export type DataExecutionProgressPatch = Partial< + Pick +>; +export class DataExecutionOwnershipLost extends Error {} +export class DataExecutionRuntimeUnavailable extends Error { + readonly statusCode = 503; +} +export interface DataExecutionContext { + readonly execution: ExecutionRef; + readonly executionVersion: number; + readonly signal: AbortSignal; + readonly targetAuthority: TargetLeaseAuthority; + readonly authority: VerifiedExecutionAuthority; + readonly installation: ControlInstallation; + readonly store: DataExecutionStore; + readonly dataRoot: string; + record(): Promise; + checkpoint(patch: DataExecutionProgressPatch): Promise; + checkpointChild(attempt: DataOpsChildAttempt): Promise; + write( + run: (record: DataExecutionRecord) => Promise<{ record: DataExecutionRecord; value: T }>, + ): Promise; + assertForward(): void; +} +export interface DataExecutionRuntime { + admit( + input: DataExecutionInput, + options: { + operationKey: string; + requestedDeadlineAt?: number; + parent?: VerifiedParentExecution; + id?: string; + principal?: ExecutionRef["principal"]; + hardlinkPlan?: unknown; + sourceJobId?: string; + triggeredBy?: "cron" | "bootstrap"; + adopt?: (execution: ExecutionRef) => Promise; + }, + ): Promise; + inspect(id: string): Promise; + lookup(key: string): Promise; + requestCancel(identity: DataExecutionIdentity): Promise; + reconcile(): Promise; + close(): Promise; +} +export function dataExecutionTargetIds(input: DataExecutionInput): string[] { + return input.kind === "transit-sync" + ? ["data-manager", "motis", "motis-feed-proxy", "motis-staging"] + : input.kind === "transit-maintenance" + ? input.action === "restart-primary" + ? ["data-manager", "motis"] + : ["data-manager", "motis-feed-proxy"] + : ["data-manager"]; +} +export function createDataExecutionRuntime(deps: { + store: DataExecutionStore; + dispatch(input: DataExecutionInput, context: DataExecutionContext): Promise; + now?: () => number; +}): DataExecutionRuntime { + const { store } = deps, + journal = store.controlJournal, + now = deps.now ?? Date.now; + type LiveEntry = { + record: DataExecutionRecord; + context: DataExecutionContext; + controller: AbortController; + task: Promise; + tail: Promise; + }; + const active = new Map(); + let closed = false; + function requireJournal() { + if (!journal) + throw new DataExecutionRuntimeUnavailable("Durable control prerequisites are unavailable"); + return journal; + } + async function start( + record: DataExecutionRecord, + adopt?: (execution: ExecutionRef) => Promise, + parent?: VerifiedParentExecution, + hardlinkPlan?: unknown, + ) { + const owner = requireJournal(), + targets = dataExecutionTargetIds(record.input); + const borrowed = await adopt?.(record.execution); + const authority = retainAdmittedControlExecution(owner.installation, { + execution: record.execution, + kind: record.input.kind, + operation: record.input, + admittedAt: record.status.admittedAt, + ...(record.requestedDeadlineAt !== null + ? { requestedDeadlineAt: record.requestedDeadlineAt } + : {}), + ...(parent ? { parent } : {}), + serviceIds: borrowed?.serviceIds ?? targets, + reservations: record.reservations.map(dataChildControlReservation), + }); + if (borrowed) bindBorrowedControlExecution(borrowed, authority); + if (record.input.kind === "generation-hardlinks") { + if (hardlinkPlan === undefined) throw Error("Signed durable generation plan required"); + await owner.retainHardlinksPlan(authority, hardlinkPlan); + } + await owner.begin(authority); + const lease = + borrowed ?? + (await acquireArtifactTargetLease({ + dataRoot: owner.dataRoot, + serviceIds: targets, + owner: authority, + })); + const controller = new AbortController(); + let entry: LiveEntry; + const context: DataExecutionContext = { + execution: record.execution, + get executionVersion() { + return entry.record.version; + }, + signal: controller.signal, + targetAuthority: lease, + authority, + installation: owner.installation, + store, + dataRoot: owner.dataRoot, + async record() { + const latest = await store.inspect(record.status.id); + if (!latest) throw new DataExecutionOwnershipLost("Original execution disappeared"); + return latest; + }, + async write(run) { + const work = entry.tail.then(async () => { + const result = await run(entry.record); + entry.record = result.record; + return result.value; + }); + entry.tail = work.catch(() => {}); + return work; + }, + async checkpoint(patch) { + await context.write(async (current) => { + const saved = await store.checkpoint(current.status.id, current.version, patch); + if (!saved) throw new DataExecutionOwnershipLost("Data execution CAS lost"); + return { record: saved, value: undefined }; + }); + }, + async checkpointChild(child) { + await context.write(async (current) => { + const saved = await store.checkpointChild(current.status.id, current.version, child); + if (!saved) throw new DataExecutionOwnershipLost("Data child CAS lost"); + return { record: saved, value: undefined }; + }); + }, + assertForward() { + controller.signal.throwIfAborted(); + assertControlExecutionForward(authority); + assertArtifactTargetLeaseForward(lease); + }, + }; + entry = { record, controller, context, task: Promise.resolve(), tail: Promise.resolve() }; + active.set(record.status.id, entry); + const timer = setTimeout( + () => { + void cancel({ + id: record.status.id, + operationKey: record.status.operationKey, + fingerprint: record.status.fingerprint, + }).catch(() => {}); + }, + Math.max(0, record.status.deadlineAt - now()), + ); + timer.unref(); + const ancestry = setInterval(() => { + if (controller.signal.aborted) return; + try { + assertControlExecutionForward(authority); + } catch (error) { + if (error instanceof ControlExecutionFencedError || now() >= record.status.deadlineAt) + void cancel({ + id: record.status.id, + operationKey: record.status.operationKey, + fingerprint: record.status.fingerprint, + }).catch(() => {}); + } + }, 250); + ancestry.unref(); + entry.task = (async () => { + try { + await context.checkpoint({}); + await context.write(async (current) => { + const saved = await store.checkpoint(current.status.id, current.version, { + status: "running", + }); + if (!saved) throw new DataExecutionOwnershipLost("Data start CAS lost"); + return { record: saved, value: undefined }; + }); + const contained = await owner.contain( + authority, + Promise.resolve() + .then(() => { + context.assertForward(); + return withDataExecutionScope(context, () => deps.dispatch(record.input, context)); + }) + .then( + async (result) => { + await settleChildren(); + return result; + }, + async (error) => { + await settleChildren(); + throw error; + }, + ), + lease, + ); + async function settleChildren() { + for (;;) { + if ((await cancelAndSettleOwnedDataOpsChildren(context)) === "settled") return; + await context.checkpoint({ publication: "unknown" }); + await settleDelay(1000); + } + } + await owner.assertOwnedProxyRecoverySettled(record.execution); + await context.write(async (current) => { + const result = + contained.result === undefined + ? undefined + : dataExecutionResultSchema.parse(contained.result); + if (result && result.kind !== record.input.kind) + throw Error("Dispatcher result kind mismatch"); + const canceled = current.status.cancellationRequestedAt !== null; + const status = result + ? result.kind === "transit-sync" + ? result.finalStatus + : contained.partial + ? "partial" + : "ok" + : canceled + ? "canceled" + : "error"; + const patch: DataExecutionPatch = { + status, + publication: result ? "committed" : current.status.publication, + ...(result ? { result } : {}), + progress: result ? 1 : current.status.progress, + }; + if (patch.publication === "unknown") return { record: current, value: undefined }; + const saved = await store.finalize( + current.status.id, + current.version, + patch, + contained.completion, + ); + if (!saved) throw new DataExecutionOwnershipLost("Data terminal CAS lost"); + return { record: saved, value: undefined }; + }); + if (isDataExecutionTerminal(entry.record.status)) { + await owner.assertOwnedProxyRecoverySettled(record.execution); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(owner.installation, record.execution), + ); + if (lease.mode === "owner") { + await owner.assertOwnedProxyRecoverySettled(record.execution); + await releaseArtifactTargetLease(lease); + await owner.publishContainedTerminal(record.execution); + } + } + } catch { + // An I/O/CAS/containment gap is not a terminal receipt or permission to + // release. Keep original identities and reconcile by observation only. + try { + await context.checkpoint({ publication: "unknown" }); + } catch {} + } finally { + clearTimeout(timer); + clearInterval(ancestry); + closeArtifactTargetLeaseHandle(lease); + active.delete(record.status.id); + } + })(); + } + async function cancel(identity: DataExecutionIdentity) { + const entry = active.get(identity.id); + const apply = async () => { + const saved = await store.requestCancel(identity); + if (!saved) { + const e = new Error("Unknown data execution"); + Object.assign(e, { statusCode: 404 }); + throw e; + } + if (!isDataExecutionTerminal(saved.status)) { + const owner = requireJournal(), + authority = + entry?.context.authority ?? + restoreAdmittedControlExecution(owner.installation, saved.execution); + cancelControlExecution(authority); + if (entry) { + const until = now() + 15000; + for (;;) { + try { + await cancelArtifactTargetLease(entry.context.targetAuthority); + break; + } catch (error) { + if (now() >= until) throw error; + await settleDelay(50); + } + } + } + if (entry) + entry.controller.abort( + new DOMException("Execution cancellation requested", "AbortError"), + ); + } + return saved; + }; + if (entry) + return entry.context.write(async () => { + const saved = await apply(); + return { record: saved, value: saved.status }; + }); + return (await apply()).status; + } + return { + async admit(input, options) { + if (closed) throw new DataExecutionRuntimeUnavailable("Data execution runtime is closed"); + requireJournal(); + const { created, record } = await store.admit(input, options); + if (created) + try { + await start(record, options.adopt, options.parent, options.hardlinkPlan); + } catch (cause) { + await store.checkpoint(record.status.id, record.version, { publication: "unknown" }); + throw new DataExecutionRuntimeUnavailable("Data admission requires ownership recovery", { + cause, + }); + } + return record.status; + }, + inspect: async (id) => (await store.inspect(id))?.status ?? null, + lookup: async (key) => (await store.lookup(key))?.status ?? null, + requestCancel: cancel, + async reconcile() { + const owner = requireJournal(); + for (const record of await store.listUnsettled()) { + if (active.has(record.status.id)) continue; + if (isDataExecutionTerminal(record.status)) { + try { + await owner.assertOwnedProxyRecoverySettled(record.execution); + await owner.repairTerminal(record.execution); + await owner.publishTerminal(record.execution); + try { + await owner.publishContainedTerminal(record.execution); + continue; + } catch {} + const authority = restoreAdmittedControlExecution(owner.installation, record.execution), + lease = + record.input.kind === "generation-hardlinks" + ? restoreGenerationHardlinksLeaseForSettlement(authority) + : restoreArtifactTargetLeaseForSettlement(authority); + try { + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(owner.installation, record.execution), + ); + if (lease.mode === "owner") { + await releaseArtifactTargetLease(lease); + await owner.publishContainedTerminal(record.execution); + } + } finally { + closeArtifactTargetLeaseHandle(lease); + } + } catch { + /* Original absent proof or released lease grants no new authority. */ + await owner.fenceUnsettledProxyRecovery(record.execution).catch(() => {}); + } + continue; + } + let recovered: Awaited> | undefined; + try { + recovered = await owner.restoreCompletion(record.execution); + cancelControlExecution(recovered.authority); + await cancelArtifactTargetLease(recovered.lease); + let current = record, + tail: Promise = Promise.resolve(); + const context: DataExecutionContext = { + execution: record.execution, + get executionVersion() { + return current.version; + }, + signal: AbortSignal.abort( + new DOMException("Original worker is contained", "AbortError"), + ), + targetAuthority: recovered.lease, + authority: recovered.authority, + installation: owner.installation, + store, + dataRoot: owner.dataRoot, + record: async () => { + const loaded = await store.inspect(record.status.id); + if (!loaded) throw new DataExecutionOwnershipLost("Original execution disappeared"); + return loaded; + }, + async write(run) { + const work = tail.then(async () => { + const saved = await run(current); + current = saved.record; + return saved.value; + }); + tail = work.catch(() => {}); + return work; + }, + async checkpoint(patch) { + await context.write(async (value) => { + const saved = await store.checkpoint(value.status.id, value.version, patch); + if (!saved) throw new DataExecutionOwnershipLost("Recovery CAS lost"); + return { record: saved, value: undefined }; + }); + }, + async checkpointChild(child) { + await context.write(async (value) => { + const saved = await store.checkpointChild(value.status.id, value.version, child); + if (!saved) throw new DataExecutionOwnershipLost("Recovery child CAS lost"); + return { record: saved, value: undefined }; + }); + }, + assertForward() { + throw new DataExecutionOwnershipLost("Recovery permits observation/settlement only"); + }, + }; + if ( + (await cancelAndSettleOwnedDataOpsChildren(context)) !== "settled" || + recovered.publication === "unknown" + ) + throw Error("Original containment/publication remains unknown"); + const result = recovered.result, + status = result + ? result.kind === "transit-sync" + ? result.finalStatus + : recovered.partial + ? "partial" + : "ok" + : current.status.cancellationRequestedAt + ? "canceled" + : "error"; + const saved = await store.finalize( + current.status.id, + current.version, + { + status, + publication: recovered.publication, + ...(result ? { result, progress: 1 } : {}), + }, + recovered.completion, + ); + if (!saved) throw new DataExecutionOwnershipLost("Recovery terminal CAS lost"); + await owner.assertOwnedProxyRecoverySettled(record.execution); + await settleArtifactTargetLeaseChild( + recovered.lease, + verifyControlEffectSettlement(owner.installation, record.execution), + ); + if (recovered.lease.mode === "owner") { + await releaseArtifactTargetLease(recovered.lease); + await owner.publishContainedTerminal(record.execution); + } + } catch { + try { + const latest = await store.inspect(record.status.id); + if (latest && !isDataExecutionTerminal(latest.status)) { + try { + cancelControlExecution( + restoreAdmittedControlExecution(owner.installation, record.execution), + ); + } catch {} + await store.checkpoint(latest.status.id, latest.version, { + status: latest.status.cancellationRequestedAt ? "cancel_pending" : "interrupted", + publication: "unknown", + }); + } + } catch {} + } finally { + if (recovered) closeArtifactTargetLeaseHandle(recovered.lease); + } + } + }, + async close() { + closed = true; + await Promise.allSettled( + [...active.values()].map((entry) => + cancel({ + id: entry.record.status.id, + operationKey: entry.record.status.operationKey, + fingerprint: entry.record.status.fingerprint, + }), + ), + ); + await Promise.race([ + Promise.allSettled([...active.values()].map((entry) => entry.task)), + new Promise((resolve) => { + const t = setTimeout(resolve, 30000); + t.unref(); + }), + ]); + }, + }; +} diff --git a/services/data-manager/src/jobs/execution/store-postgres.test.ts b/services/data-manager/src/jobs/execution/store-postgres.test.ts new file mode 100644 index 000000000..89c51c682 --- /dev/null +++ b/services/data-manager/src/jobs/execution/store-postgres.test.ts @@ -0,0 +1,271 @@ +import { randomBytes } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; +import postgres from "postgres"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { dataChildControlReservation } from "./ops-children.js"; +import { createPostgresDataExecutionStore, DATA_ADMISSION_LOCK } from "./store.js"; + +describe.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)("durable execution PostgreSQL admission and CAS", () => { + let sql: ReturnType; + const ids: string[] = []; + beforeAll(() => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Explicit owned synthetic database required"); + sql = postgres(url, { max: 4, connection: { statement_timeout: 5000 } }); + }); + afterAll(async () => { + if (sql) { + if (ids.length) await sql`DELETE FROM data_manager.jobs WHERE id=ANY(${ids}::uuid[])`; + await sql.end(); + } + }); + it("retains one normalized admission under concurrent connections and rejects a renewed deadline", async () => { + const one = createPostgresDataExecutionStore(sql), + two = createPostgresDataExecutionStore(sql); + const requestedDeadlineAt = Date.now() + 60000; + const operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }); + const input = { kind: "download-fonts" as const }; + const first = await one.admit(input, { operationKey, requestedDeadlineAt }); + ids.push(first.record.status.id); + const results = await Promise.all([ + one.admit(input, { operationKey, requestedDeadlineAt }), + two.admit(input, { operationKey, requestedDeadlineAt }), + ]); + expect(results.map((result) => result.record.status.id)).toEqual([ + first.record.status.id, + first.record.status.id, + ]); + expect(results.every((result) => !result.created)).toBe(true); + await expect( + two.admit(input, { operationKey, requestedDeadlineAt: requestedDeadlineAt + 1000 }), + ).rejects.toThrow(/conflict/i); + await expect( + two.admit({ kind: "notable-places-build" }, { operationKey, requestedDeadlineAt }), + ).rejects.toThrow(/conflict/i); + }); + it("allows one owner-version writer and retains cancel intent before finalization", async () => { + const store = createPostgresDataExecutionStore(sql); + const requestedDeadlineAt = Date.now() + 60000; + const operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }); + const admitted = await store.admit( + { kind: "download-fonts" }, + { operationKey, requestedDeadlineAt }, + ); + const id = admitted.record.status.id; + ids.push(id); + const results = await Promise.all([ + store.checkpoint(id, 0, { status: "running", progress: 0.1 }), + store.checkpoint(id, 0, { status: "running", progress: 0.2 }), + ]); + expect(results.filter(Boolean)).toHaveLength(1); + const canceled = await store.requestCancel({ + id, + operationKey, + fingerprint: admitted.record.status.fingerprint, + }); + expect(canceled?.status.status).toBe("cancel_pending"); + expect(canceled?.status.cancellationRequestedAt).not.toBeNull(); + expect( + await store.checkpoint(id, 1, { + status: "ok", + publication: "committed", + result: { kind: "download-fonts", completed: true }, + }), + ).toBeNull(); + const rows = await sql`SELECT finished_at FROM data_manager.jobs WHERE id=${id}`; + expect(rows[0]?.finished_at).toBeNull(); + }); + it("refuses an admission on another connection while the configuration transaction owns the nonblocking guard", async () => { + let acquired = () => {}, + release = () => {}; + const ready = new Promise((resolve) => { + acquired = resolve; + }); + const released = new Promise((resolve) => { + release = resolve; + }); + const guard = sql.begin(async (tx) => { + await tx`SELECT pg_advisory_xact_lock(${DATA_ADMISSION_LOCK[0]},${DATA_ADMISSION_LOCK[1]})`; + acquired(); + await released; + }); + try { + await ready; + const store = createPostgresDataExecutionStore(sql), + requestedDeadlineAt = Date.now() + 60000; + const operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }); + await expect( + store.admit({ kind: "download-fonts" }, { operationKey, requestedDeadlineAt }), + ).rejects.toThrow("configuration_maintenance_busy"); + expect(await store.lookup(operationKey)).toBeNull(); + } finally { + release(); + await guard; + } + }); + it.each(["claimed", "stale"] as const)( + "uses real body/binding/grant evidence for a single child CAS continuation (%s)", + async (mode) => { + const store = createPostgresDataExecutionStore(sql), + requestedDeadlineAt = Date.now() + 60000; + const operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }); + const { record } = await store.admit( + { kind: "transit-sync", countries: ["de"] }, + { operationKey, requestedDeadlineAt }, + ); + ids.push(record.status.id); + const child = record.reservations[0]; + if (!child) throw Error("Missing original fixed child body"); + const root = mkdtempSync(join(tmpdir(), "omx-data-child-cas-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const services = ["motis", "motis-feed-proxy", "motis-staging"]; + const dm = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: services, + }); + const agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: services, + }); + const authority = control.retainAdmittedControlExecution(dm, { + execution: record.execution, + kind: record.input.kind, + operation: record.input, + admittedAt: record.status.admittedAt, + requestedDeadlineAt, + serviceIds: services, + reservations: record.reservations.map(dataChildControlReservation), + }); + const lease = await control.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: services, + owner: authority, + }); + try { + const races = await Promise.all([ + store.reserveChild(record.status.id, 0, child), + store.reserveChild(record.status.id, 0, child), + ]); + expect(races.filter((result) => result?.continuation)).toHaveLength(1); + const winner = races.find((result) => result?.continuation); + if (!winner?.continuation) throw Error("Missing confirmed reserved continuation"); + expect( + (await store.reserveChild(record.status.id, winner.record.version, child))?.continuation, + ).toBeNull(); + await expect( + store.checkpointChild(record.status.id, winner.record.version, { + ...child, + operationFingerprint: "b".repeat(64), + }), + ).rejects.toThrow(/child|identity/i); + await expect( + store.checkpoint(record.status.id, winner.record.version, { + status: "ok", + publication: "committed", + }), + ).rejects.toThrow(/unsettled/i); + const exact = await control.bindControlChildPhase(authority, child); + const binding = control.lookupControlChildPhaseBinding(authority, child); + if (!binding) throw Error("Missing original binding"); + const bound = { + ...child, + state: "bound" as const, + execution: exact.execution, + requestedDeadlineAt: binding.requestedDeadlineAt, + bindingDigest: binding.bindingDigest, + candidateDigest: null, + }; + const saved = await store.checkpointChild(record.status.id, winner.record.version, bound); + if (!saved) throw Error("Lost bound checkpoint"); + const issued = await control.issueControlChildPhase(authority, { + phase: child.phase, + ordinal: child.ordinal, + lease, + childPublicKey: agent.publicKey, + }); + const reference = control.lookupControlChildPhaseAttempt(authority, child); + if (!reference?.leaseId) throw Error("Missing original grant reference"); + const attempted = { + ...bound, + state: "attempted" as const, + delegationId: reference.delegationId, + grantDigest: reference.grantDigest, + leaseId: reference.leaseId, + dispatchState: "claimed" as const, + }; + await expect( + store.claimChildDispatch(record.status.id, saved.version, attempted, { + ...winner.continuation, + }), + ).rejects.toThrow(/continuation/i); + if (mode === "stale") { + const moved = await store.checkpoint(record.status.id, saved.version, { progress: 0.25 }); + if (!moved) throw Error("Lost intervening CAS"); + expect( + await store.claimChildDispatch( + record.status.id, + saved.version, + attempted, + winner.continuation, + ), + ).toBeNull(); + await expect( + store.claimChildDispatch( + record.status.id, + moved.version, + attempted, + winner.continuation, + ), + ).rejects.toThrow(/continuation/i); + expect((await store.inspect(record.status.id))?.children[0]?.state).toBe("bound"); + } else { + const claimed = await store.claimChildDispatch( + record.status.id, + saved.version, + attempted, + winner.continuation, + ); + expect(claimed?.children[0]).toEqual(attempted); + control.consumeControlChildDispatch(issued.dispatchTicket); + expect(() => control.consumeControlChildDispatch(issued.dispatchTicket)).toThrow(); + await expect( + store.claimChildDispatch( + record.status.id, + claimed?.version ?? 0, + attempted, + winner.continuation, + ), + ).rejects.toThrow(/continuation/i); + } + await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); + } finally { + control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(agent); + control.closeControlInstallation(dm); + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/services/data-manager/src/jobs/execution/store.test.ts b/services/data-manager/src/jobs/execution/store.test.ts new file mode 100644 index 000000000..d323bf8e0 --- /dev/null +++ b/services/data-manager/src/jobs/execution/store.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from "vitest"; +import { dataExecutionInputFingerprint, validateDataExecutionPatch } from "./store.js"; + +it("uses normalized countries for execution identity", () => { + expect( + dataExecutionInputFingerprint({ kind: "transit-sync", countries: ["DE", "ch", "de"] }), + ).toBe(dataExecutionInputFingerprint({ kind: "transit-sync", countries: ["ch", "de"] })); + expect(dataExecutionInputFingerprint({ kind: "download-osm", region: "de" })).not.toBe( + dataExecutionInputFingerprint({ kind: "download-osm", region: "ch" }), + ); +}); +it("refuses identity fields at the checkpoint boundary", () => { + expect(() => validateDataExecutionPatch({ deadlineAt: 123 })).toThrow(); + expect(() => validateDataExecutionPatch({ progress: 2 })).toThrow(); + expect(() => + validateDataExecutionPatch({ + diagnostics: { tail: [{ stream: "stderr", message: "x".repeat(4097) }], truncated: false }, + }), + ).toThrow(); + expect(validateDataExecutionPatch({ progress: 0.5, publication: "not_started" })).toEqual({ + progress: 0.5, + publication: "not_started", + }); +}); +it("retains the explicitly unfiltered planet transit scope in normalized identity", () => { + expect(() => + dataExecutionInputFingerprint({ kind: "transit-sync", countries: [] }), + ).not.toThrow(); +}); diff --git a/services/data-manager/src/jobs/execution/store.ts b/services/data-manager/src/jobs/execution/store.ts new file mode 100644 index 000000000..89c024940 --- /dev/null +++ b/services/data-manager/src/jobs/execution/store.ts @@ -0,0 +1,645 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + type ContainedDataExecution, + type DataExecutionControlJournal, + retainedControlParent, + retainedDataDatabaseTimestamp, +} from "@openmapx/core/build-artifacts"; +import { + type DataExecutionIdentity, + type DataExecutionInput, + type DataExecutionStatus, + dataExecutionStatusSchema, + isDataExecutionTerminal, + normalizeDataExecutionInput, +} from "@openmapx/core/data-manager-contracts"; +import { + canonicalJson, + type ExecutionRef, + executionFingerprint, + executionRefSchema, + normalizeExecutionAdmission, + parseControlRootOperationKey, + type RetainedParentExecution, + retainedParentExecutionSchema, + type VerifiedParentExecution, +} from "@openmapx/core/ops"; +import type postgres from "postgres"; +import z from "zod/v4"; +import { + type DataOpsChildAttempt, + type DataOpsChildReserved, + dataOpsChildAttemptSchema, + dataOpsChildReservedSchema, + reserveTransitChildOperations, + sameDataChildBody, +} from "./ops-children.js"; + +const reservationBrand = Symbol("data-child-reservation"); +export type FreshDataChildReservation = Readonly<{ [reservationBrand]: true }>; +const reservationContinuations = new WeakMap< + FreshDataChildReservation, + { store: symbol; id: string; key: string; used: boolean } +>(); + +import type { DataConfigurationMaintenance } from "./configuration-maintenance.js"; +export const DATA_ADMISSION_LOCK = [0x4f4d58, 2] as const; +export class DataExecutionConflict extends Error { + readonly statusCode = 409; +} +export class DataConfigurationMaintenanceBusy extends Error { + readonly statusCode = 409; + constructor() { + super("configuration_maintenance_busy"); + } +} +export interface DataExecutionRecord { + triggeredBy: string | null; + execution: ExecutionRef; + status: DataExecutionStatus; + input: DataExecutionInput; + operationFingerprint: string; + requestedDeadlineAt: number | null; + parent: RetainedParentExecution | null; + version: number; + children: DataOpsChildAttempt[]; + reservations: DataOpsChildReserved[]; + finishedAt: number | null; +} +export interface DataExecutionAdmissionOptions { + operationKey: string; + requestedDeadlineAt?: number; + parent?: VerifiedParentExecution; + id?: string; + principal?: ExecutionRef["principal"]; + sourceJobId?: string; + triggeredBy?: "cron" | "bootstrap"; +} +export type DataExecutionPatch = Partial< + Pick +>; +const fields = dataExecutionStatusSchema.shape; +const patchSchema = z + .strictObject({ + status: fields.status.optional(), + progress: fields.progress.optional(), + diagnostics: fields.diagnostics.optional(), + publication: fields.publication.optional(), + result: fields.result, + }) + .partial(); +export function validateDataExecutionPatch(value: unknown): DataExecutionPatch { + return patchSchema.parse(value); +} +export function dataExecutionInputFingerprint(value: unknown): string { + return createHash("sha256") + .update(canonicalJson(normalizeDataExecutionInput(value))) + .digest("hex"); +} +export interface DataExecutionStore { + readonly controlJournal?: DataExecutionControlJournal; + admit( + input: DataExecutionInput, + options: DataExecutionAdmissionOptions, + ): Promise<{ created: boolean; record: DataExecutionRecord }>; + inspect(id: string): Promise; + lookup(operationKey: string): Promise; + checkpoint( + id: string, + expectedVersion: number, + patch: DataExecutionPatch, + ): Promise; + checkpointChild( + id: string, + expectedVersion: number, + attempt: DataOpsChildAttempt, + ): Promise; + reserveChild( + id: string, + expectedVersion: number, + child: DataOpsChildReserved, + ): Promise<{ + record: DataExecutionRecord; + continuation: FreshDataChildReservation | null; + } | null>; + claimChildDispatch( + id: string, + expectedVersion: number, + child: Extract, + continuation: FreshDataChildReservation, + ): Promise; + requestCancel(identity: DataExecutionIdentity): Promise; + listUnsettled(): Promise; + finalize( + id: string, + expectedVersion: number, + patch: DataExecutionPatch, + completion: ContainedDataExecution, + ): Promise; +} +interface Row extends Record { + triggered_by: string | null; + id: string; + kind: string; + status: string; + operation_key: string; + operation_fingerprint: string; + execution_fingerprint: string; + admitted_at: string | number; + deadline_at: string | number; + requested_deadline_at: string | number | null; + execution_parent: unknown; + execution_input: unknown; + execution_state: unknown; + execution_children: unknown; + ownership_version: number; + finished_at: Date | string | null; + metadata: unknown; +} +function safeEpoch(value: string | number): number { + const result = Number(value); + if (!Number.isSafeInteger(result) || result < 0) throw Error("Invalid stored execution time"); + return result; +} +function decode(row: Row): DataExecutionRecord { + const input = normalizeDataExecutionInput(row.execution_input); + const mutable = z + .strictObject({ + cancellationRequestedAt: z.iso.datetime({ offset: true }).nullable(), + progress: z.number().nullable(), + diagnostics: z.unknown(), + publication: z.enum(["not_started", "committed", "unknown"]), + result: z.unknown().optional(), + }) + .parse(row.execution_state); + const status = dataExecutionStatusSchema.parse({ + id: row.id, + kind: input.kind, + status: row.status, + operationKey: row.operation_key, + fingerprint: row.execution_fingerprint, + admittedAt: safeEpoch(row.admitted_at), + deadlineAt: safeEpoch(row.deadline_at), + ...mutable, + }); + if (dataExecutionInputFingerprint(input) !== row.operation_fingerprint || row.kind !== input.kind) + throw Error("Stored execution body mismatch"); + const { + dataExecution: { execution, childReservations }, + } = z + .object({ + dataExecution: z.strictObject({ + version: z.literal(1), + execution: executionRefSchema, + childReservations: z.array(dataOpsChildReservedSchema).max(32).optional(), + }), + }) + .parse(row.metadata); + if ( + execution.executor !== "data-manager" || + execution.executionId !== status.id || + execution.operationKey !== status.operationKey || + execution.operationFingerprint !== row.operation_fingerprint || + execution.fingerprint !== status.fingerprint || + execution.deadlineAt !== status.deadlineAt + ) + throw Error("Stored execution authority identity mismatch"); + const parent = + row.execution_parent === null + ? null + : retainedParentExecutionSchema.parse(row.execution_parent); + if ( + execution.fingerprint !== + executionFingerprint({ + kind: input.kind, + operationFingerprint: execution.operationFingerprint, + deadlineAt: execution.deadlineAt, + ...(parent ? { parent } : {}), + }) + ) + throw Error("Stored execution fingerprint mismatch"); + return { + triggeredBy: z + .string() + .nullable() + .parse(row.triggered_by ?? null), + execution, + status, + input, + operationFingerprint: row.operation_fingerprint, + requestedDeadlineAt: + row.requested_deadline_at === null ? null : safeEpoch(row.requested_deadline_at), + parent, + version: z.number().int().nonnegative().safe().parse(row.ownership_version), + children: validateChildren(row.execution_children), + reservations: (childReservations ?? []).map((value) => { + const parsed = dataOpsChildAttemptSchema.parse(value); + if (parsed.state !== "reserved") throw Error("Invalid reserved body"); + return parsed; + }), + finishedAt: + row.finished_at === null ? null : retainedDataDatabaseTimestamp(row.finished_at).getTime(), + }; +} +function validateChildren(value: unknown): DataOpsChildAttempt[] { + const children = z.array(dataOpsChildAttemptSchema).max(32).parse(value); + if (Buffer.byteLength(JSON.stringify(children)) > 128 * 1024) + throw Error("Data child capacity exceeded"); + const keys = new Set(children.map((child) => child.operationKey)); + if (keys.size !== children.length) throw Error("Duplicate child identity"); + return children; +} +/** The check and job/reservation insertion must share this connection and + * transaction. A contending admission never waits while retaining targets. */ +export async function withDataAdmissionTransaction( + sql: postgres.Sql, + run: (tx: postgres.TransactionSql) => Promise, +): Promise { + const result = await sql.begin(async (tx) => { + const lock = + await tx`SELECT pg_try_advisory_xact_lock(${DATA_ADMISSION_LOCK[0]},${DATA_ADMISSION_LOCK[1]}) AS held`; + if (lock[0]?.held !== true) throw new DataConfigurationMaintenanceBusy(); + const guard = + await tx`SELECT id FROM data_manager.jobs WHERE kind='configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled' LIMIT 1`; + if (guard.length) throw new DataConfigurationMaintenanceBusy(); + return { value: await run(tx) }; + }); + return result.value; +} +export function createPostgresDataExecutionStore( + sql: postgres.Sql, + options: { + now?: () => number; + controlJournal?: DataExecutionControlJournal; + configurationFence?: DataConfigurationMaintenance; + } = {}, +): DataExecutionStore { + const now = options.now ?? Date.now; + const storeIdentity = Symbol("data-store"); + const find = async ( + tx: postgres.Sql | postgres.TransactionSql, + key: "id" | "operation_key", + value: string, + lock = false, + ) => { + const rows = await tx.unsafe( + `SELECT * FROM data_manager.jobs WHERE execution_version=2 AND ${key}=$1${lock ? " FOR UPDATE" : ""}`, + [value], + ); + return rows[0] ? decode(rows[0]) : null; + }; + const update = async ( + tx: postgres.TransactionSql, + record: DataExecutionRecord, + status: DataExecutionStatus, + children: DataOpsChildAttempt[], + terminalAt?: number, + ) => { + const { cancellationRequestedAt, progress, diagnostics, publication, result } = status; + const finishedAt = isDataExecutionTerminal(status) + ? (record.finishedAt ?? terminalAt ?? now()) + : null; + const rows = await tx< + Row[] + >`UPDATE data_manager.jobs SET status=${status.status},ownership_version=ownership_version+1, + execution_state=${canonicalJson({ cancellationRequestedAt, progress, diagnostics, publication, ...(result ? { result } : {}) })}::text::jsonb, + execution_children=${canonicalJson(children)}::text::jsonb,finished_at=${finishedAt === null ? null : new Date(finishedAt).toISOString()}::text::timestamptz + WHERE id=${status.id} AND ownership_version=${record.version} RETURNING *`; + return rows[0] ? decode(rows[0]) : null; + }; + function validateOwnedChild(record: DataExecutionRecord, child: DataOpsChildAttempt) { + const template = record.reservations.find((slot) => slot.operationKey === child.operationKey); + if ( + !template || + !sameDataChildBody(template, child) || + canonicalJson(child.parentExecution) !== canonicalJson(record.execution) + ) + throw new DataExecutionConflict("Child is absent from original owned reservations"); + if ( + child.state !== "reserved" && + (child.execution.deadlineAt > record.status.deadlineAt || + child.execution.fingerprint !== + executionFingerprint({ + kind: child.operation.kind, + operationFingerprint: child.operationFingerprint, + deadlineAt: child.requestedDeadlineAt, + parent: retainedControlParent(record.execution), + })) + ) + throw new DataExecutionConflict("Invalid retained child execution identity"); + } + async function writeChild( + tx: postgres.TransactionSql, + record: DataExecutionRecord, + child: DataOpsChildAttempt, + claim: boolean, + ) { + validateOwnedChild(record, child); + const at = record.children.findIndex((entry) => entry.operationKey === child.operationKey), + previous = record.children[at]; + if (!previous) + throw new DataExecutionConflict("A fresh reserved CAS is required before binding"); + if (canonicalJson(previous) === canonicalJson(child)) return record; + if (isDataExecutionTerminal(record.status)) + throw new DataExecutionConflict("Terminal execution cannot change children"); + const transitions: Record< + DataOpsChildAttempt["state"], + readonly DataOpsChildAttempt["state"][] + > = { + reserved: ["reserved", "bound"], + bound: ["bound", "attempted"], + attempted: ["attempted", "observing", "terminal", "unknown"], + observing: ["observing", "terminal", "unknown"], + unknown: ["unknown", "observing", "terminal"], + terminal: ["terminal"], + }; + if ( + !transitions[previous.state].includes(child.state) || + !sameDataChildBody(previous, child) || + canonicalJson(previous.proxyRecovery ?? null) !== + canonicalJson(child.proxyRecovery ?? null) || + previous.state === "terminal" || + (previous.cancellationRequestedAt && + previous.cancellationRequestedAt !== child.cancellationRequestedAt) + ) + throw new DataExecutionConflict("Retained child identity/state conflict"); + if (previous.state !== "reserved") { + if ( + child.state === "reserved" || + canonicalJson(previous.execution) !== canonicalJson(child.execution) || + previous.requestedDeadlineAt !== child.requestedDeadlineAt || + previous.bindingDigest !== child.bindingDigest || + previous.candidateDigest !== child.candidateDigest + ) + throw new DataExecutionConflict("Bound child identity is immutable"); + } + if ("delegationId" in previous) { + if ( + !("delegationId" in child) || + previous.delegationId !== child.delegationId || + previous.grantDigest !== child.grantDigest || + previous.leaseId !== child.leaseId || + (previous.dispatchState === "claimed" && child.dispatchState !== "claimed") + ) + throw new DataExecutionConflict("Attempted child grant is immutable"); + } + if ( + "acceptedId" in previous && + previous.acceptedId !== undefined && + (!("acceptedId" in child) || previous.acceptedId !== child.acceptedId) + ) + throw new DataExecutionConflict("Accepted child identity is immutable"); + const claiming = + "dispatchState" in child && + child.dispatchState === "claimed" && + (!("dispatchState" in previous) || previous.dispatchState !== "claimed"); + if (claiming && !claim) + throw new DataExecutionConflict("A live confirmed reservation is required to claim dispatch"); + const children = [...record.children]; + children[at] = child; + return update(tx, record, record.status, validateChildren(children)); + } + return { + controlJournal: options.controlJournal, + async admit(value, admission) { + const input = normalizeDataExecutionInput(value), + operationFingerprint = dataExecutionInputFingerprint(input); + // Known duplicates are observations and remain available while fenced or expired. + const existing = await find(sql, "operation_key", admission.operationKey); + const match = (record: DataExecutionRecord) => { + const normalized = normalizeExecutionAdmission({ + kind: input.kind, + operationKey: admission.operationKey, + operationFingerprint, + admittedAt: record.status.admittedAt, + requestedDeadlineAt: admission.requestedDeadlineAt, + parent: admission.parent, + }); + if ( + record.operationFingerprint !== operationFingerprint || + record.status.fingerprint !== normalized.fingerprint || + canonicalJson(record.parent) !== canonicalJson(admission.parent ?? null) || + record.execution.principal !== (admission.principal ?? "data-manager") || + (admission.id !== undefined && admission.id !== record.status.id) || + (admission.triggeredBy !== undefined && record.triggeredBy !== admission.triggeredBy) + ) + throw new DataExecutionConflict("Data execution identity conflict"); + return { created: false, record }; + }; + if (existing) return match(existing); + if (!admission.parent) { + const root = parseControlRootOperationKey(admission.operationKey), + wall = now(); + if ( + root.notAfter !== admission.requestedDeadlineAt || + root.notAfter <= wall || + root.notAfter - wall > 7 * 24 * 3600000 + ) + throw new DataExecutionConflict("A fresh dated data execution root is required"); + } + return withDataAdmissionTransaction(sql, async (tx) => { + await options.configurationFence?.assertAdmission(tx); + const retained = await find(tx, "operation_key", admission.operationKey); + if (retained) return match(retained); + if (options.controlJournal) { + const active = + await tx`SELECT id FROM data_manager.jobs WHERE execution_version=2 AND finished_at IS NULL LIMIT 1`; + if (active.length) throw new DataConfigurationMaintenanceBusy(); + } + if (admission.sourceJobId) { + const source = + await tx`SELECT id FROM data_manager.jobs WHERE id=${admission.sourceJobId} AND kind LIKE 'transit-source-%' AND status='running' AND finished_at IS NULL FOR UPDATE`; + if (source.length !== 1) + throw new DataExecutionConflict("Original source reservation is not current"); + } + const normalized = normalizeExecutionAdmission({ + kind: input.kind, + operationKey: admission.operationKey, + operationFingerprint, + admittedAt: now(), + requestedDeadlineAt: admission.requestedDeadlineAt, + parent: admission.parent, + }); + const status = dataExecutionStatusSchema.parse({ + id: admission.id ?? randomUUID(), + operationKey: admission.operationKey, + ...normalized, + kind: input.kind, + status: "queued", + cancellationRequestedAt: null, + progress: null, + diagnostics: { tail: [], truncated: false }, + publication: "not_started", + }); + const execution = executionRefSchema.parse({ + executor: "data-manager", + principal: admission.principal ?? "data-manager", + executionId: status.id, + operationKey: status.operationKey, + operationFingerprint, + fingerprint: status.fingerprint, + deadlineAt: status.deadlineAt, + }); + const childReservations = reserveTransitChildOperations(execution, input); + const rows = await tx< + Row[] + >`INSERT INTO data_manager.jobs(id,kind,status,triggered_by,started_at,execution_version,operation_key,operation_fingerprint, + execution_fingerprint,admitted_at,deadline_at,requested_deadline_at,execution_parent,execution_input,execution_state,metadata) + VALUES(${status.id},${input.kind},'queued',${admission.triggeredBy ?? null},${new Date(status.admittedAt).toISOString()}::text::timestamptz,2,${status.operationKey},${operationFingerprint}, + ${status.fingerprint},${status.admittedAt},${status.deadlineAt},${admission.requestedDeadlineAt ?? null}, + ${admission.parent ? canonicalJson(admission.parent) : null}::text::jsonb,${canonicalJson(input)}::text::jsonb,${canonicalJson({ cancellationRequestedAt: null, progress: null, diagnostics: status.diagnostics, publication: status.publication })}::text::jsonb,${canonicalJson({ dataExecution: { version: 1, execution, childReservations } })}::text::jsonb) RETURNING *`; + if (!rows[0]) throw Error("Execution admission did not return its durable row"); + if (admission.sourceJobId) + await tx`UPDATE data_manager.jobs SET metadata=jsonb_set(coalesce(metadata,'{}'::jsonb),'{durableExecution}',${canonicalJson({ id: status.id, operationKey: status.operationKey, fingerprint: status.fingerprint })}::text::jsonb),ownership_version=ownership_version+1 WHERE id=${admission.sourceJobId}`; + return { created: true, record: decode(rows[0]) }; + }); + }, + inspect: (id) => find(sql, "id", id), + lookup: (key) => find(sql, "operation_key", key), + async checkpoint(id, version, value) { + const patch = validateDataExecutionPatch(value); + return sql.begin(async (tx) => { + const record = await find(tx, "id", id, true); + if (!record || record.version !== version) return null; + const status = dataExecutionStatusSchema.parse({ ...record.status, ...patch }); + if ( + options.controlJournal && + !isDataExecutionTerminal(record.status) && + isDataExecutionTerminal(status) + ) + throw new DataExecutionConflict("Actual contained worker terminal commitment required"); + if ( + isDataExecutionTerminal(status) && + record.children.some((child) => child.state !== "terminal") + ) + throw new DataExecutionConflict("Unsettled children retain the execution fence"); + if ( + isDataExecutionTerminal(record.status) && + canonicalJson(status) !== canonicalJson(record.status) + ) + throw new DataExecutionConflict("Terminal execution cannot change"); + if (record.status.cancellationRequestedAt && ["queued", "running"].includes(status.status)) + throw new DataExecutionConflict("Cancellation is irreversible"); + if (record.status.publication === "committed" && status.publication === "not_started") + throw new DataExecutionConflict("Committed publication cannot change"); + if (canonicalJson(status) === canonicalJson(record.status)) return record; + return update(tx, record, status, record.children); + }); + }, + async checkpointChild(id, version, value) { + const child = dataOpsChildAttemptSchema.parse(value); + return sql.begin(async (tx) => { + const record = await find(tx, "id", id, true); + if (!record || record.version !== version) return null; + return writeChild(tx, record, child, false); + }); + }, + async reserveChild(id, version, value) { + const child = dataOpsChildAttemptSchema.parse(value); + if (child.state !== "reserved") throw new DataExecutionConflict("Reserved body required"); + const saved = await sql.begin(async (tx) => { + const record = await find(tx, "id", id, true); + if (!record) return null; + const existing = record.children.find((entry) => entry.operationKey === child.operationKey); + if (existing) { + if ( + !sameDataChildBody(existing, child) || + canonicalJson(existing.proxyRecovery ?? null) !== + canonicalJson(child.proxyRecovery ?? null) + ) + throw new DataExecutionConflict("Reserved child identity conflict"); + return { record, created: false }; + } + if (record.version !== version) return null; + if (record.status.cancellationRequestedAt || isDataExecutionTerminal(record.status)) + throw new DataExecutionConflict("Canceled execution cannot reserve a phase"); + validateOwnedChild(record, child); + const updated = await update( + tx, + record, + record.status, + validateChildren([...record.children, child]), + ); + return updated ? { record: updated, created: true } : null; + }); + if (!saved) return null; + if (!saved.created) return { record: saved.record, continuation: null }; + const continuation = Object.freeze({ [reservationBrand]: true as const }); + reservationContinuations.set(continuation, { + store: storeIdentity, + id, + key: child.operationKey, + used: false, + }); + return { record: saved.record, continuation }; + }, + async claimChildDispatch(id, version, value, continuation) { + const permission = reservationContinuations.get(continuation); + if ( + !permission || + permission.used || + permission.store !== storeIdentity || + permission.id !== id || + permission.key !== value.operationKey + ) + throw new DataExecutionConflict("Missing live reservation continuation"); + permission.used = true; + const child = dataOpsChildAttemptSchema.parse(value); + if (child.state !== "attempted" || child.dispatchState !== "claimed") + throw new DataExecutionConflict("Dispatch claim must retain its real attempted grant"); + return sql.begin(async (tx) => { + const record = await find(tx, "id", id, true); + if (!record || record.version !== version) return null; + if (record.status.cancellationRequestedAt || isDataExecutionTerminal(record.status)) + throw new DataExecutionConflict("Cancellation prevents a fresh dispatch claim"); + return writeChild(tx, record, child, true); + }); + }, + async requestCancel(identity) { + return sql.begin(async (tx) => { + const record = await find(tx, "id", identity.id, true); + if (!record) return null; + if ( + record.status.operationKey !== identity.operationKey || + record.status.fingerprint !== identity.fingerprint + ) + throw new DataExecutionConflict("Cancellation identity conflict"); + if (isDataExecutionTerminal(record.status) || record.status.cancellationRequestedAt) + return record; + return update( + tx, + record, + { + ...record.status, + status: "cancel_pending", + cancellationRequestedAt: new Date(now()).toISOString(), + }, + record.children, + ); + }); + }, + async listUnsettled() { + const rows = await sql< + Row[] + >`SELECT * FROM data_manager.jobs WHERE execution_version=2 ORDER BY admitted_at,id`; + return rows.map(decode); + }, + async finalize(id, version, patch, completion) { + if (!options.controlJournal) throw Error("Durable data journal is unavailable"); + const record = await find(sql, "id", id); + if (!record || record.version !== version) return null; + const status = dataExecutionStatusSchema.parse({ + ...record.status, + ...validateDataExecutionPatch(patch), + }); + const terminalAt = await options.controlJournal.prepareTerminal(completion, status, now()); + const result = await sql.begin(async (tx) => { + const current = await find(tx, "id", id, true); + if (!current || current.version !== version) return null; + return update(tx, current, status, current.children, terminalAt); + }); + if (result) await options.controlJournal.publishTerminal(result.execution); + return result; + }, + }; +} diff --git a/services/data-manager/src/jobs/execution/worker-scope.ts b/services/data-manager/src/jobs/execution/worker-scope.ts new file mode 100644 index 000000000..aab0fb3d0 --- /dev/null +++ b/services/data-manager/src/jobs/execution/worker-scope.ts @@ -0,0 +1,180 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type postgres from "postgres"; +import type { DataExecutionContext } from "./runtime.js"; + +const workers = new AsyncLocalStorage(); +export function dataExecutionWorkerContext(): DataExecutionContext | undefined { + return workers.getStore(); +} +const settlements = new AsyncLocalStorage(); +export function withDataSqlSettlement(run: () => Promise): Promise { + return settlements.run(AbortSignal.timeout(30000), run); +} +export function withDataExecutionScope( + context: DataExecutionContext, + run: () => Promise, +): Promise { + return workers.run(context, run); +} +export function dataWorkerSignal(signal?: AbortSignal): AbortSignal | undefined { + const owned = settlements.getStore() ?? workers.getStore()?.signal; + return signal && owned && signal !== owned ? AbortSignal.any([signal, owned]) : (signal ?? owned); +} +export function dataWorkerDeadline(): number | undefined { + return workers.getStore()?.execution.deadlineAt; +} +export function assertDataPublication(): void { + workers.getStore()?.assertForward(); +} +export async function retainDataPublication(): Promise { + await workers.getStore()?.checkpoint({ publication: "committed" }); +} +export class DataPublicationUnknown extends Error { + constructor(original: unknown) { + super(original instanceof Error ? original.message : "Data publication remains unknown", { + cause: original, + }); + } +} +class DataSqlRolledBack extends Error { + constructor(readonly original: unknown) { + super("Owned database transaction rolled back", { cause: original }); + } +} +/** The marker precedes the actual live write. A lost commit/rename response + * keeps publication unknown; a verified PostgreSQL rollback restores the prior + * publication receipt. Only the static worker consumers call this local scope. */ +export async function publishDataChange(run: () => Promise | T): Promise { + const context = workers.getStore(); + if (!context) return run(); + context.assertForward(); + const previous = (await context.record()).status.publication; + await context.checkpoint({ publication: "unknown" }); + context.assertForward(); + try { + const value = await run(); + await context.checkpoint({ publication: "committed" }); + return value; + } catch (error) { + if (error instanceof DataSqlRolledBack) { + await context.checkpoint({ publication: previous }); + throw error.original; + } + throw new DataPublicationUnknown(error); + } +} +/** Awaited PostgreSQL requests are canceled on the actual connection. Awaiting + * their rejection/rollback contains the request; observer disconnection never + * enters this scope. Durable ownership rows use the independent raw pool. */ +function ownedQuery(query: T): T { + if ( + !query || + typeof query !== "object" || + !("cancel" in query) || + typeof query.cancel !== "function" + ) + return query; + const cancel = query.cancel.bind(query); + return new Proxy(query, { + get(target, property, receiver) { + const value = Reflect.get(target, property, receiver); + if (property !== "then" || typeof value !== "function") + return typeof value === "function" ? value.bind(target) : value; + return ( + resolved?: ((value: unknown) => unknown) | null, + rejected?: ((reason: unknown) => unknown) | null, + ) => { + const signal = dataWorkerSignal(); + if (signal?.aborted) return Promise.reject(signal.reason).then(resolved, rejected); + const abort = () => { + void Promise.resolve(cancel()).catch(() => {}); + }; + signal?.addEventListener("abort", abort, { once: true }); + const pending = Reflect.apply(value, target, []); + return Promise.resolve(pending) + .finally(() => signal?.removeEventListener("abort", abort)) + .then(resolved, rejected); + }; + }, + }); +} +export function withDataSqlCancellation( + sql: T, +): T { + return new Proxy(sql, { + apply(target, thisArg, args) { + return ownedQuery(Reflect.apply(target, thisArg, args)); + }, + get(target, property, receiver) { + const value = Reflect.get(target, property, receiver); + if (typeof value !== "function") return value; + if (property === "begin") + return (...args: unknown[]) => { + if (!workers.getStore()) return Reflect.apply(value, target, args); + const callback = args.find((arg) => typeof arg === "function"); + if (typeof callback !== "function") throw Error("Data transaction callback required"); + const reserve = Reflect.get(target, "reserve"); + if (typeof reserve !== "function") + throw Error("Owned transaction requires its own connection"); + return (async () => { + const connection = await Reflect.apply(reserve, target, []), + options = typeof args[0] === "string" ? args[0].replace(/[^a-z ]/gi, "") : ""; + let commitAttempted = false, + releasable = false; + let originalBackend: unknown; + try { + originalBackend = await ownedQuery( + connection.unsafe( + "SELECT pg_backend_pid() AS pid,backend_start::text FROM pg_stat_activity WHERE pid=pg_backend_pid()", + ), + ); + await ownedQuery(connection.unsafe(`BEGIN ${options}`)); + const pending = Reflect.apply(callback, undefined, [ + withDataSqlCancellation(connection), + ]); + const result = await (Array.isArray(pending) ? Promise.all(pending) : pending); + // Publication authority was checked before BEGIN. The physical + // target remains owned; the actual connection is cancelable through + // COMMIT, without acquiring filesystem writers under DB locks. + dataWorkerSignal()?.throwIfAborted(); + const execution = workers.getStore()?.execution; + if (execution && Date.now() >= execution.deadlineAt) + throw new DOMException("Execution deadline elapsed", "AbortError"); + commitAttempted = true; + await ownedQuery(connection.unsafe("COMMIT")); + releasable = true; + return result; + } catch (error) { + try { + const after = await withDataSqlSettlement(async () => { + await ownedQuery(connection.unsafe("ROLLBACK")); + return ownedQuery( + connection.unsafe( + "SELECT pg_backend_pid() AS pid,backend_start::text FROM pg_stat_activity WHERE pid=pg_backend_pid()", + ), + ); + }); + if (JSON.stringify(after) === JSON.stringify(originalBackend)) { + releasable = true; + if (!commitAttempted) throw new DataSqlRolledBack(error); + } + } catch (rollback) { + if (rollback instanceof DataSqlRolledBack) throw rollback; + } + await workers.getStore()?.checkpoint({ publication: "unknown" }); + throw error; + } finally { + // An uncertain transaction never returns its connection for reuse. + if (releasable) connection.release(); + } + })(); + }; + if (property === "reserve") + return (...args: unknown[]) => + Promise.resolve(Reflect.apply(value, target, args)).then((connection) => + withDataSqlCancellation(connection), + ); + return (...args: unknown[]) => ownedQuery(Reflect.apply(value, target, args)); + }, + }); +} diff --git a/services/data-manager/src/jobs/notable-places/build.ts b/services/data-manager/src/jobs/notable-places/build.ts index 99bb9b7e8..8157ab0d6 100644 --- a/services/data-manager/src/jobs/notable-places/build.ts +++ b/services/data-manager/src/jobs/notable-places/build.ts @@ -7,6 +7,12 @@ import type { } from "@openmapx/core/data-manager-contracts"; import type postgres from "postgres"; import { scrubSecrets } from "../../utils/scrub-secrets.js"; +import { + assertDataPublication, + dataWorkerSignal, + publishDataChange, + retainDataPublication, +} from "../execution/worker-scope.js"; import { createNotablePlacesOperationLock, type SearchIndexOperationLock, @@ -185,7 +191,11 @@ export async function buildNotablePlaces( const endpoint = opts.endpoint ?? DEFAULT_NOTABLE_PLACES_SPARQL_URL; const minSitelinks = opts.minSitelinks ?? DEFAULT_NOTABLE_MIN_SITELINKS; const lock = opts.operationLock ?? createNotablePlacesOperationLock(opts.sql); - const fetchOptions = { endpoint, signal: opts.signal, fetchImpl: opts.fetchImpl }; + const fetchOptions = { + endpoint, + signal: dataWorkerSignal(opts.signal), + fetchImpl: opts.fetchImpl, + }; return lock.run(async () => { // Bound as text: the client is shared with drizzle, which replaces its // date serializers. @@ -319,23 +329,27 @@ export async function buildNotablePlaces( const live = await opts.sql.unsafe<{ exists: boolean }[]>( `SELECT to_regnamespace('notable_places') IS NOT NULL AS exists`, ); - await opts.sql.begin(async (transaction) => { - await transaction.unsafe( - `UPDATE notable_places__staging.index_state + assertDataPublication(); + await publishDataChange(() => + opts.sql.begin(async (transaction) => { + await transaction.unsafe( + `UPDATE notable_places__staging.index_state SET status='ready', place_count=$1, name_count=$2, published_at=NOW(), updated_at=NOW(), last_error=NULL WHERE singleton=1`, - [placeCount, nameCount], - ); - await transaction.unsafe(`DROP SCHEMA IF EXISTS notable_places__previous CASCADE`); - if (live[0]?.exists) { - await transaction.unsafe( - `ALTER SCHEMA notable_places RENAME TO notable_places__previous`, + [placeCount, nameCount], ); - } - await transaction.unsafe(`ALTER SCHEMA notable_places__staging RENAME TO notable_places`); - await transaction.unsafe(`DROP SCHEMA IF EXISTS notable_places__previous CASCADE`); - }); + await transaction.unsafe(`DROP SCHEMA IF EXISTS notable_places__previous CASCADE`); + if (live[0]?.exists) { + await transaction.unsafe( + `ALTER SCHEMA notable_places RENAME TO notable_places__previous`, + ); + } + await transaction.unsafe(`ALTER SCHEMA notable_places__staging RENAME TO notable_places`); + await transaction.unsafe(`DROP SCHEMA IF EXISTS notable_places__previous CASCADE`); + }), + ); + await retainDataPublication(); progress("complete", `Published ${placeCount} places and ${nameCount} names`); return { epoch, source: endpoint, minSitelinks, placeCount, nameCount }; } catch (error) { diff --git a/services/data-manager/src/jobs/notable-places/schedule.ts b/services/data-manager/src/jobs/notable-places/schedule.ts index 4165fc681..73f96c9e4 100644 --- a/services/data-manager/src/jobs/notable-places/schedule.ts +++ b/services/data-manager/src/jobs/notable-places/schedule.ts @@ -2,7 +2,6 @@ import { Cron } from "croner"; import type postgres from "postgres"; import { pickCronExpression } from "../../utils/cron-expression.js"; import type { SearchIndexOperationLock } from "../search-index/operation-lock.js"; -import { buildNotablePlaces } from "./build.js"; import { DEFAULT_NOTABLE_MIN_SITELINKS } from "./sparql.js"; import { hasPublishedNotablePlaces, type NotablePlacesRuntimeState } from "./state.js"; @@ -35,7 +34,7 @@ export interface NotablePlacesScheduleOptions { logger: NotablePlacesLogger; /** Overrides `NOTABLE_PLACES_REFRESH_CRON`; "off" disables refreshes and the first build. */ cronExpression?: string; - build?: typeof buildNotablePlaces; + runOwned: () => Promise; } export interface NotablePlacesSchedule { @@ -56,21 +55,12 @@ export function scheduleNotablePlaces(opts: NotablePlacesScheduleOptions): Notab "NOTABLE_PLACES_REFRESH_CRON", NOTABLE_PLACES_REFRESH_CRON_DEFAULT, ); - const build = opts.build ?? buildNotablePlaces; const run = async (reason: string): Promise => { if (opts.operationLock.inFlight) return; opts.logger.info({ reason }, "notable-places: build starting"); try { - const result = await build({ - sql: opts.sql, - runtimeState: opts.runtimeState, - operationLock: opts.operationLock, - ...notablePlacesSettings(), - }); - opts.logger.info( - { places: result.placeCount, names: result.nameCount, epoch: result.epoch }, - "notable-places: published", - ); + if (!opts.runOwned) throw Error("Durable notable-places admission is unavailable"); + await opts.runOwned(); } catch (error) { opts.logger.warn({ err: (error as Error).message }, "notable-places: build failed"); } diff --git a/services/data-manager/src/jobs/overture/conflate.ts b/services/data-manager/src/jobs/overture/conflate.ts index 2d211e462..3ef56eea3 100644 --- a/services/data-manager/src/jobs/overture/conflate.ts +++ b/services/data-manager/src/jobs/overture/conflate.ts @@ -1,3 +1,5 @@ +import { assertDataPublication, publishDataChange } from "../execution/worker-scope.js"; + export { buildConflationComponents } from "./components.js"; import { @@ -641,17 +643,20 @@ export async function publishOvertureLinks( ): Promise<{ linked: number }> { assertValidOvertureSchema(schema); - await sql.begin(async (tx) => { - await tx.unsafe(`DELETE FROM "${schema}".poi_conflation_link`); - await tx.unsafe( - `INSERT INTO "${schema}".poi_conflation_link + assertDataPublication(); + await publishDataChange(() => + sql.begin(async (tx) => { + await tx.unsafe(`DELETE FROM "${schema}".poi_conflation_link`); + await tx.unsafe( + `INSERT INTO "${schema}".poi_conflation_link (osm_type, osm_id, gers_id, source_confidence, match_confidence, distance_m, method, evidence, release) SELECT osm_type, osm_id, gers_id, source_confidence, match_confidence, distance_m, method, evidence, release FROM "${schema}".poi_conflation_link_next`, - ); - }); + ); + }), + ); const [{ linked }] = await sql.unsafe<{ linked: string }[]>( `SELECT COUNT(*)::TEXT AS linked FROM "${schema}".poi_conflation_link`, diff --git a/services/data-manager/src/jobs/overture/duckdb.ts b/services/data-manager/src/jobs/overture/duckdb.ts index b3b9580ad..ad181fd4e 100644 --- a/services/data-manager/src/jobs/overture/duckdb.ts +++ b/services/data-manager/src/jobs/overture/duckdb.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { envString } from "@openmapx/core/server-env"; import { type Options as ExecaOptions, type Result as ExecaResult, execa } from "execa"; +import { dataWorkerSignal } from "../execution/worker-scope.js"; /** * Replaces the password in any embedded Postgres connection string with `***`. @@ -33,7 +34,7 @@ export async function runDuckDb(args: string[], options?: ExecaOptions): Promise return (await execa( "duckdb", ["-bail", "-cmd", duckDbResourceSql(process.env, scratch), ...args], - options ?? {}, + { ...options, cancelSignal: dataWorkerSignal(options?.cancelSignal) }, )) as ExecaResult; } catch (err) { const e = err as { message?: string; stderr?: unknown; stdout?: unknown; exitCode?: number }; diff --git a/services/data-manager/src/jobs/overture/extract-osm-pois.ts b/services/data-manager/src/jobs/overture/extract-osm-pois.ts index caefde399..6c3c911d5 100644 --- a/services/data-manager/src/jobs/overture/extract-osm-pois.ts +++ b/services/data-manager/src/jobs/overture/extract-osm-pois.ts @@ -6,6 +6,12 @@ import { execa } from "execa"; import { latLngToCell } from "h3-js"; import { sql } from "../../db/index.js"; import { osmPbfName } from "../download-osm.js"; +import { + assertDataPublication, + dataWorkerSignal, + publishDataChange, + retainDataPublication, +} from "../execution/worker-scope.js"; import { withOsmiumLocationIndex } from "../osm-resources.js"; import { assertValidRegion } from "./pull.js"; import { assertValidOvertureSchema } from "./schema.js"; @@ -195,7 +201,7 @@ export async function extractOsmPois( await execa( "osmium", ["tags-filter", pbfPath, ...uniqueFilters.map((f) => `nwr/${f}`), "-o", filteredPbf, "-O"], - { stdio: "inherit" }, + { stdio: "inherit", cancelSignal: dataWorkerSignal() }, ); const stagingTable = "osm_pois__staging"; opts.onProgress?.("Preparing a fresh OSM POI staging table..."); @@ -224,7 +230,7 @@ export async function extractOsmPois( ], // Execa buffers piped output by default. Germany-scale GeoJSON exceeds // that buffer, so consume stdout exclusively through the line iterator. - OSMIUM_EXPORT_STREAM_OPTIONS, + { ...OSMIUM_EXPORT_STREAM_OPTIONS, cancelSignal: dataWorkerSignal() }, ); opts.onProgress?.("Streaming osmium GeoJSON output into PostGIS..."); const { createInterface } = await import("node:readline"); @@ -283,22 +289,26 @@ export async function extractOsmPois( `Emitted ${emitted} OSM geometries and retained ${extracted} unique POIs; ` + "publishing the staged snapshot...", ); - await sql.begin(async (tx) => { - await tx.unsafe(`DROP TABLE IF EXISTS "${schema}".osm_pois__previous`); - await tx.unsafe(`ALTER TABLE "${schema}".osm_pois RENAME TO osm_pois__previous`); - await tx.unsafe(`ALTER TABLE "${schema}"."${stagingTable}" RENAME TO osm_pois`); - await tx.unsafe(`DROP TABLE "${schema}".osm_pois__previous`); - await tx.unsafe(`CREATE INDEX idx_osm_pois_category ON "${schema}".osm_pois (category)`); - await tx.unsafe( - `CREATE INDEX idx_osm_pois_h3 + assertDataPublication(); + await publishDataChange(() => + sql.begin(async (tx) => { + await tx.unsafe(`DROP TABLE IF EXISTS "${schema}".osm_pois__previous`); + await tx.unsafe(`ALTER TABLE "${schema}".osm_pois RENAME TO osm_pois__previous`); + await tx.unsafe(`ALTER TABLE "${schema}"."${stagingTable}" RENAME TO osm_pois`); + await tx.unsafe(`DROP TABLE "${schema}".osm_pois__previous`); + await tx.unsafe(`CREATE INDEX idx_osm_pois_category ON "${schema}".osm_pois (category)`); + await tx.unsafe( + `CREATE INDEX idx_osm_pois_h3 ON "${schema}".osm_pois (h3_r8, osm_type, osm_id)`, - ); - await tx.unsafe( - `CREATE INDEX idx_osm_pois_geom + ); + await tx.unsafe( + `CREATE INDEX idx_osm_pois_geom ON "${schema}".osm_pois USING GIST (ST_Point(lng, lat))`, - ); - await tx.unsafe(`ANALYZE "${schema}".osm_pois`); - }); + ); + await tx.unsafe(`ANALYZE "${schema}".osm_pois`); + }), + ); + await retainDataPublication(); opts.onProgress?.( `Published ${extracted} unique OSM POIs to ${schema}.osm_pois ` + `(${emitted} geometries emitted).`, diff --git a/services/data-manager/src/jobs/overture/feed-state.ts b/services/data-manager/src/jobs/overture/feed-state.ts new file mode 100644 index 000000000..f633a1edd --- /dev/null +++ b/services/data-manager/src/jobs/overture/feed-state.ts @@ -0,0 +1,48 @@ +import { feedState } from "@openmapx/db-schema"; +import { and, eq } from "drizzle-orm"; +import { db } from "../../db/index.js"; +import { assertDataPublication } from "../execution/worker-scope.js"; +export async function writeOvertureFeedState( + region: string, + hash: string, + log: { warn: (message: string, extra?: Record) => void }, +): Promise { + try { + assertDataPublication(); + const feedRegion = `overture-places-${region}`; + const feedName = "overture-places"; + const existing = await db + .select({ id: feedState.id }) + .from(feedState) + .where(and(eq(feedState.region, feedRegion), eq(feedState.name, feedName))) + .limit(1); + if (existing[0]) { + await db + .update(feedState) + .set({ + hash, + lastFetchedAt: new Date(), + lastImportedAt: new Date(), + validationStatus: "ok", + validationMessage: `Imported Overture release ${hash}`, + consecutiveFailures: 0, + status: "active", + }) + .where(eq(feedState.id, existing[0].id)); + } else { + await db.insert(feedState).values({ + region: feedRegion, + name: feedName, + hash, + lastFetchedAt: new Date(), + lastImportedAt: new Date(), + validationStatus: "ok", + validationMessage: `Imported Overture release ${hash}`, + consecutiveFailures: 0, + status: "active", + }); + } + } catch (err) { + log.warn("overture-cron: feed_state write failed", { err: (err as Error).message }); + } +} diff --git a/services/data-manager/src/jobs/overture/ingest.ts b/services/data-manager/src/jobs/overture/ingest.ts index d2b21d745..88d0f81e0 100644 --- a/services/data-manager/src/jobs/overture/ingest.ts +++ b/services/data-manager/src/jobs/overture/ingest.ts @@ -3,6 +3,11 @@ import { join } from "node:path"; import { assertSupportedOvertureContributors } from "@openmapx/core/utils/overtureSource"; import { latLngToCell } from "h3-js"; import { sql } from "../../db/index.js"; +import { + assertDataPublication, + publishDataChange, + retainDataPublication, +} from "../execution/worker-scope.js"; import { assertOverturePostgresCapacity, estimateOvertureIngestBytes } from "./capacity.js"; import { duckDbSqlLiteral, runDuckDbScript } from "./duckdb.js"; import { validateOvertureQuality } from "./eval/quality-gate.js"; @@ -98,14 +103,16 @@ export async function activateOvertureStagingSchema( `SELECT to_regclass($1) IS NOT NULL AS exists`, [`${schema}.osm_pois`], ); - await sql.begin(async (tx) => { - if (existingOsmSnapshot?.exists) { - // `rebuildOvertureLinks` verifies this fingerprint against the current - // on-disk PBF before reuse and restarts extraction if it changed. - await tx.unsafe(`DROP TABLE "${stagingSchema}".osm_pois CASCADE`); - await tx.unsafe(`ALTER TABLE "${schema}".osm_pois SET SCHEMA "${stagingSchema}"`); - await tx.unsafe( - `UPDATE "${stagingSchema}".conflation_state AS target + assertDataPublication(); + await publishDataChange(() => + sql.begin(async (tx) => { + if (existingOsmSnapshot?.exists) { + // `rebuildOvertureLinks` verifies this fingerprint against the current + // on-disk PBF before reuse and restarts extraction if it changed. + await tx.unsafe(`DROP TABLE "${stagingSchema}".osm_pois CASCADE`); + await tx.unsafe(`ALTER TABLE "${schema}".osm_pois SET SCHEMA "${stagingSchema}"`); + await tx.unsafe( + `UPDATE "${stagingSchema}".conflation_state AS target SET phase = 'score', source_fingerprint = source.source_fingerprint, emitted_count = source.emitted_count, extracted_count = source.extracted_count, processed_count = 0, candidate_count = 0, @@ -115,17 +122,19 @@ export async function activateOvertureStagingSchema( AND source.region = target.region AND source.source_fingerprint IS NOT NULL AND source.extracted_count IS NOT NULL`, - ); - } - await tx.unsafe(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`); - await tx.unsafe(`ALTER SCHEMA "${stagingSchema}" RENAME TO "${schema}"`); - await tx.unsafe( - `UPDATE "${schema}".conflation_state + ); + } + await tx.unsafe(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`); + await tx.unsafe(`ALTER SCHEMA "${stagingSchema}" RENAME TO "${schema}"`); + await tx.unsafe( + `UPDATE "${schema}".conflation_state SET places_published_at = $1 WHERE singleton = 1`, - [placesPublishedAt], - ); - }); + [placesPublishedAt], + ); + }), + ); + await retainDataPublication(); } export async function ingestOverture(opts: IngestOvertureOptions): Promise { diff --git a/services/data-manager/src/jobs/overture/operation-lock.ts b/services/data-manager/src/jobs/overture/operation-lock.ts index 056b4f767..69dbc55dc 100644 --- a/services/data-manager/src/jobs/overture/operation-lock.ts +++ b/services/data-manager/src/jobs/overture/operation-lock.ts @@ -1,4 +1,5 @@ import { sql } from "../../db/index.js"; +import { withDataSqlSettlement } from "../execution/worker-scope.js"; // Stable, application-owned two-key advisory lock. Every operation that can // replace the Overture schema or its OSM snapshot uses the same lock. @@ -15,8 +16,10 @@ export async function withOvertureOperationLock(operation: () => Promise): return await operation(); } finally { try { - await connection.unsafe( - `SELECT pg_advisory_unlock(${OVERTURE_LOCK_NAMESPACE}, ${OVERTURE_LOCK_KEY})`, + await withDataSqlSettlement(() => + connection.unsafe( + `SELECT pg_advisory_unlock(${OVERTURE_LOCK_NAMESPACE}, ${OVERTURE_LOCK_KEY})`, + ), ); } finally { connection.release(); diff --git a/services/data-manager/src/jobs/overture/pull.ts b/services/data-manager/src/jobs/overture/pull.ts index c2f15e1e6..5726b516e 100644 --- a/services/data-manager/src/jobs/overture/pull.ts +++ b/services/data-manager/src/jobs/overture/pull.ts @@ -2,6 +2,7 @@ import { mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node: import { basename, join } from "node:path"; import { execa } from "execa"; import { resolveOsmPolyUrl } from "../download-osm.js"; +import { dataWorkerSignal, publishDataChange } from "../execution/worker-scope.js"; import { captureOvertureCatalogue } from "./attribution-capture.js"; import { assertOvertureDiskCapacity, estimateOverturePullBytes, freeBytesAt } from "./capacity.js"; import { runDuckDb } from "./duckdb.js"; @@ -106,7 +107,10 @@ export async function fetchRegionBbox(region: string): Promise { const safePolyUrl = polyUrl.toString(); let stdout: string; try { - ({ stdout } = await execa("curl", ["-fsSL", safePolyUrl], { timeout: 30_000 })); + ({ stdout } = await execa("curl", ["-fsSL", safePolyUrl], { + timeout: 30_000, + cancelSignal: dataWorkerSignal(), + })); } catch { throw new Error( `Could not fetch Geofabrik boundary for region "${region}" (${safePolyUrl}). ` + @@ -262,10 +266,11 @@ export async function pullOverture(opts: PullOvertureOptions): Promise { flag: "wx", mode: 0o600, }); - renameSync(partialPath, outPath); - // Publish the contract last: ingest never observes a new contract pointing - // at a partial parquet snapshot. - renameSync(partialContractPath, contractPath); + await publishDataChange(() => { + renameSync(partialPath, outPath); + // Publish metadata last; interruption of this pair remains unknown. + renameSync(partialContractPath, contractPath); + }); opts.onProgress?.( `Validated ${contract.rowCount} places and ${contract.contributors.length} contributor(s).`, ); diff --git a/services/data-manager/src/jobs/overture/retention.ts b/services/data-manager/src/jobs/overture/retention.ts index d85427fc3..932f85e82 100644 --- a/services/data-manager/src/jobs/overture/retention.ts +++ b/services/data-manager/src/jobs/overture/retention.ts @@ -1,6 +1,7 @@ import { existsSync, readdirSync, rmSync } from "node:fs"; import { join, resolve } from "node:path"; import { sql } from "../../db/index.js"; +import { assertDataPublication, publishDataChange } from "../execution/worker-scope.js"; import { assertValidOvertureSchema } from "./schema.js"; import { assertValidOvertureRelease } from "./stac.js"; @@ -78,6 +79,7 @@ export function pruneOvertureReleases( if (target !== join(root, release)) { throw new Error(`Refusing to prune Overture release outside ${root}: ${target}`); } + assertDataPublication(); rmSync(target, { recursive: true, force: true }); removed.push(release); opts.onProgress?.(`Pruned superseded Overture snapshot ${release}.`); @@ -109,7 +111,8 @@ export async function finalizeOvertureReleaseFiles( } if (state.release_files_pruned_at !== null) return null; - const result = pruneOvertureReleases(opts); + const result = await publishDataChange(() => pruneOvertureReleases(opts)); + assertDataPublication(); await sql.unsafe( `UPDATE "${schema}".conflation_state SET release_files_pruned_at = COALESCE(release_files_pruned_at, NOW()), diff --git a/services/data-manager/src/jobs/persistence.ts b/services/data-manager/src/jobs/persistence.ts index f0ff192d8..3ea92c4b4 100644 --- a/services/data-manager/src/jobs/persistence.ts +++ b/services/data-manager/src/jobs/persistence.ts @@ -1,7 +1,10 @@ import { jobStages, jobs } from "@openmapx/db-schema"; -import { eq } from "drizzle-orm"; -import { db } from "../db/index.js"; +import { and, eq, isNull, ne } from "drizzle-orm"; +import type postgres from "postgres"; +import z from "zod/v4"; +import { db, rawSql } from "../db/index.js"; import { scrubDiagnosticValue, scrubSecrets } from "../utils/scrub-secrets.js"; +import { assertDataMutationAdmission } from "./execution/admission.js"; export interface CreateJobRowOptions { kind: string; @@ -24,23 +27,34 @@ export interface PersistenceLogger { warn(message: string): unknown; } -export async function createJobRow(options: CreateJobRowOptions): Promise { - const inserted = await db - .insert(jobs) - .values({ - kind: options.kind, - status: "running", - triggeredBy: options.triggeredBy ?? null, - metadata: options.metadata ?? null, - }) - .returning({ id: jobs.id }); +export async function createJobRow( + options: CreateJobRowOptions, + database: postgres.Sql = rawSql, +): Promise { + const { inserted } = await database.begin(async (tx) => { + await assertDataMutationAdmission(tx); + const inserted = await tx< + { id: string }[] + >`INSERT INTO data_manager.jobs(kind,status,triggered_by,metadata) + VALUES(${options.kind},'running',${options.triggeredBy ?? null},${options.metadata ? JSON.stringify(z.json().parse(options.metadata)) : null}::text::jsonb) RETURNING id`; + return { inserted }; + }); const row = inserted[0]; if (!row) throw new Error("Failed to create data_manager.jobs row"); return row.id; } export async function finalizeJobRow(jobId: string, status: string): Promise { - await db.update(jobs).set({ status, finishedAt: new Date() }).where(eq(jobs.id, jobId)); + await db + .update(jobs) + .set({ status, finishedAt: new Date() }) + .where( + and( + eq(jobs.id, jobId), + isNull(jobs.executionVersion), + ne(jobs.kind, "configuration-maintenance"), + ), + ); } export function makePersistingOnStageComplete( diff --git a/services/data-manager/src/jobs/reconcile.ts b/services/data-manager/src/jobs/reconcile.ts index 44bd01be8..d791e715a 100644 --- a/services/data-manager/src/jobs/reconcile.ts +++ b/services/data-manager/src/jobs/reconcile.ts @@ -1,5 +1,5 @@ import { jobs } from "@openmapx/db-schema"; -import { eq } from "drizzle-orm"; +import { and, eq, isNull, ne } from "drizzle-orm"; import { db as defaultDb } from "../db/index.js"; /** Terminal status stamped on jobs orphaned in `running` by a process death. */ @@ -13,7 +13,7 @@ export const INTERRUPTED_STATUS = "interrupted"; */ export type JobReconcileWriter = { update: (table: typeof jobs) => { - set: (values: { status: string; finishedAt: Date }) => { + set: (values: { status: string; finishedAt: Date | null }) => { where: (predicate: unknown) => { returning: (columns: { id: typeof jobs.id }) => PromiseLike>; }; @@ -28,31 +28,27 @@ export interface ReconcileOrphanedJobsOptions { now?: () => Date; } -/** - * Mark every job still in `running` as `interrupted`, returning the ids it - * reconciled. - * - * The data-manager is a single instance whose "a sync is running" signal lives - * only in process memory (the single-flight `inflight` flag). So any `running` - * row found at startup is a zombie left by a previous process that died mid-run - * — a restart / redeploy / OOM during a multi-hour Transitous sync (graceful - * shutdown only waits 30s before forcing exit). Without this sweep those rows - * stay `running` forever: the admin "Sync in progress" banner shows a phantom - * job indefinitely and `lastSyncStatus` reads null. - * - * MUST run only at startup, before any new job is created. At that point nothing - * is genuinely running, so a blanket sweep across every job kind sharing the - * table is safe. - */ +/** Startup marks legacy workers interrupted without asserting process/child + * containment. New versioned executions and logical guards have their own + * retained reconciler; unfinished legacy evidence continues to block writes. */ export async function reconcileOrphanedJobs( opts: ReconcileOrphanedJobsOptions = {}, ): Promise { - const handle: JobReconcileWriter = opts.db ?? (defaultDb as unknown as JobReconcileWriter); - const finishedAt = opts.now ? opts.now() : new Date(); - const updated = await handle - .update(jobs) - .set({ status: INTERRUPTED_STATUS, finishedAt }) - .where(eq(jobs.status, "running")) - .returning({ id: jobs.id }); + const predicate = and( + eq(jobs.status, "running"), + isNull(jobs.executionVersion), + ne(jobs.kind, "configuration-maintenance"), + ); + const updated = opts.db + ? await opts.db + .update(jobs) + .set({ status: INTERRUPTED_STATUS, finishedAt: null }) + .where(predicate) + .returning({ id: jobs.id }) + : await defaultDb + .update(jobs) + .set({ status: INTERRUPTED_STATUS, finishedAt: null }) + .where(predicate) + .returning({ id: jobs.id }); return updated.map((row) => row.id); } diff --git a/services/data-manager/src/jobs/search-index/build.ts b/services/data-manager/src/jobs/search-index/build.ts index 5683eb66e..ef44dc36c 100644 --- a/services/data-manager/src/jobs/search-index/build.ts +++ b/services/data-manager/src/jobs/search-index/build.ts @@ -7,6 +7,11 @@ import type { import type postgres from "postgres"; import type { StateStore } from "../../state.js"; import { scrubSecrets } from "../../utils/scrub-secrets.js"; +import { + assertDataPublication, + publishDataChange, + retainDataPublication, +} from "../execution/worker-scope.js"; import { osmFileIdentity } from "../osm-resources.js"; import { extractSearchPlaces, type SearchPlaceRecord } from "./extract.js"; import { createSearchIndexOperationLock, type SearchIndexOperationLock } from "./operation-lock.js"; @@ -208,19 +213,23 @@ export async function buildOsmSearchIndex( const live = await opts.sql.unsafe<{ exists: boolean }[]>( `SELECT to_regnamespace('osm_search') IS NOT NULL AS exists`, ); - await opts.sql.begin(async (transaction) => { - await transaction.unsafe( - `UPDATE osm_search__staging.index_state + assertDataPublication(); + await publishDataChange(() => + opts.sql.begin(async (transaction) => { + await transaction.unsafe( + `UPDATE osm_search__staging.index_state SET status='ready', place_count=$1, term_count=$2, published_at=NOW(), updated_at=NOW(), last_error=NULL WHERE singleton=1`, - [placeCount, termCount], - ); - await transaction.unsafe(`DROP SCHEMA IF EXISTS osm_search__previous CASCADE`); - if (live[0]?.exists) - await transaction.unsafe(`ALTER SCHEMA osm_search RENAME TO osm_search__previous`); - await transaction.unsafe(`ALTER SCHEMA osm_search__staging RENAME TO osm_search`); - await transaction.unsafe(`DROP SCHEMA IF EXISTS osm_search__previous CASCADE`); - }); + [placeCount, termCount], + ); + await transaction.unsafe(`DROP SCHEMA IF EXISTS osm_search__previous CASCADE`); + if (live[0]?.exists) + await transaction.unsafe(`ALTER SCHEMA osm_search RENAME TO osm_search__previous`); + await transaction.unsafe(`ALTER SCHEMA osm_search__staging RENAME TO osm_search`); + await transaction.unsafe(`DROP SCHEMA IF EXISTS osm_search__previous CASCADE`); + }), + ); + await retainDataPublication(); progress("complete", `Published ${placeCount} places and ${termCount} terms`); return { region: opts.region, epoch, sourceFingerprint, placeCount, termCount }; } catch (error) { diff --git a/services/data-manager/src/jobs/search-index/extract.ts b/services/data-manager/src/jobs/search-index/extract.ts index 9a8a97d14..1f0cf739d 100644 --- a/services/data-manager/src/jobs/search-index/extract.ts +++ b/services/data-manager/src/jobs/search-index/extract.ts @@ -2,6 +2,7 @@ import { mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { createInterface } from "node:readline"; import { execa } from "execa"; +import { dataWorkerSignal } from "../execution/worker-scope.js"; import { withOsmiumLocationIndex } from "../osm-resources.js"; export { withOsmiumLocationIndex } from "../osm-resources.js"; @@ -135,6 +136,7 @@ export async function extractSearchPlaces( opts.onProgress?.(`Filtering named OSM features from ${pbfPath}`); await execa("osmium", ["tags-filter", pbfPath, "nwr/name", "-o", filteredPbf, "-O"], { stdio: "inherit", + cancelSignal: dataWorkerSignal(), }); const process = execa( "osmium", @@ -148,6 +150,7 @@ export async function extractSearchPlaces( filteredPbf, ], { + cancelSignal: dataWorkerSignal(), stdout: "pipe", stderr: "inherit", buffer: false, diff --git a/services/data-manager/src/jobs/search-index/operation-lock.ts b/services/data-manager/src/jobs/search-index/operation-lock.ts index d72638f88..bcb09a562 100644 --- a/services/data-manager/src/jobs/search-index/operation-lock.ts +++ b/services/data-manager/src/jobs/search-index/operation-lock.ts @@ -1,4 +1,5 @@ import type postgres from "postgres"; +import { withDataSqlSettlement } from "../execution/worker-scope.js"; export const SEARCH_INDEX_LOCK_NAMESPACE = 1_330_466_120; export const SEARCH_INDEX_LOCK_KEY = 2; @@ -35,8 +36,10 @@ function createAdvisoryOperationLock( } finally { if (connection) { try { - await connection.unsafe( - `SELECT pg_advisory_unlock(${SEARCH_INDEX_LOCK_NAMESPACE}, ${key})`, + await withDataSqlSettlement(() => + connection!.unsafe( + `SELECT pg_advisory_unlock(${SEARCH_INDEX_LOCK_NAMESPACE}, ${key})`, + ), ); } finally { connection.release(); diff --git a/services/data-manager/src/jobs/search-index/state.ts b/services/data-manager/src/jobs/search-index/state.ts index bbe5b0f8b..0b170e31e 100644 --- a/services/data-manager/src/jobs/search-index/state.ts +++ b/services/data-manager/src/jobs/search-index/state.ts @@ -3,6 +3,7 @@ import { createReadStream } from "node:fs"; import type { SearchIndexStatus as CanonicalSearchIndexStatus } from "@openmapx/core/data-manager-contracts"; import type postgres from "postgres"; import type { DatasetMetadata, StateStore } from "../../state.js"; +import { dataWorkerSignal, publishDataChange } from "../execution/worker-scope.js"; export interface SearchIndexRuntimeState { building: boolean; @@ -20,7 +21,7 @@ export async function fingerprintDataset(dataset: DatasetMetadata): Promise((resolve, reject) => { const hash = createHash("sha256"); - const stream = createReadStream(dataset.path); + const stream = createReadStream(dataset.path, { signal: dataWorkerSignal() }); stream.on("data", (chunk) => hash.update(chunk)); stream.on("end", () => resolve(`sha256:${hash.digest("hex")}`)); stream.on("error", reject); @@ -45,11 +46,13 @@ export async function updateCurrentSearchIndexFingerprint( `SELECT to_regclass('osm_search.index_state') IS NOT NULL AS exists`, ); if (!rows[0]?.exists) return; - await sql.unsafe( - `UPDATE osm_search.index_state + await publishDataChange(() => + sql.unsafe( + `UPDATE osm_search.index_state SET current_fingerprint = $2, updated_at = NOW() WHERE singleton = 1 AND region = $1`, - [region, fingerprint], + [region, fingerprint], + ), ); } diff --git a/services/data-manager/src/jobs/transitous/functional-probes.ts b/services/data-manager/src/jobs/transitous/functional-probes.ts index dc52e890e..d1bfa30b4 100644 --- a/services/data-manager/src/jobs/transitous/functional-probes.ts +++ b/services/data-manager/src/jobs/transitous/functional-probes.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; +import { setTimeout as containedProbeDelay } from "node:timers/promises"; import type { HealthResponse, InitialResponse, @@ -10,6 +11,7 @@ import type { import { MOTIS_VERSION } from "@openmapx/transitous-core"; import type { TransitousLock } from "../../transitous-lock.js"; import { atomicWriteJsonSync } from "../../utils/atomic-write.js"; +import { dataWorkerDeadline, dataWorkerSignal } from "../execution/worker-scope.js"; import { CAPABILITY_SNAPSHOT_FILENAME, type MotisCandidateManifest } from "./candidate.js"; import { elevationPlanUrl, @@ -257,7 +259,7 @@ async function executeOneToManyCanary( maxPostTransitTime: 900, maxDirectTime: 900, }), - signal: controller.signal, + signal: dataWorkerSignal(controller.signal), }); } finally { clearTimeout(timer); @@ -298,8 +300,6 @@ export interface RunFunctionalProbesOptions { rentalsPollIntervalMs?: number; } -const realSleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); - /** * Run one probe, retrying while it fails for up to `warmupMs` (bounded by attempt * count so it terminates deterministically even under an instant/no-op sleep). @@ -334,7 +334,13 @@ export async function runFunctionalProbes( deadline: number, options: RunFunctionalProbesOptions = {}, ): Promise { - const sleep = options.sleep ?? realSleep; + deadline = Math.min(deadline, dataWorkerDeadline() ?? Infinity); + const sleep = + options.sleep ?? + (async (ms: number) => { + dataWorkerSignal()?.throwIfAborted(); + await containedProbeDelay(ms, undefined, { signal: dataWorkerSignal() }); + }); const rentalsWarmupMs = options.rentalsWarmupMs ?? parseIntEnv("MOTIS_RENTALS_WARMUP_MS", DEFAULT_RENTALS_WARMUP_MS); const rentalsPollIntervalMs = diff --git a/services/data-manager/src/jobs/transitous/internal.ts b/services/data-manager/src/jobs/transitous/internal.ts index 21e9bc0c1..6297c2b34 100644 --- a/services/data-manager/src/jobs/transitous/internal.ts +++ b/services/data-manager/src/jobs/transitous/internal.ts @@ -29,6 +29,7 @@ import { type TransitousFeedSource, } from "@openmapx/transitous-core"; import { execa } from "execa"; +import { dataWorkerSignal } from "../execution/worker-scope.js"; import type { TransitousScriptRunner } from "./script-runner.js"; import type { FeedDownloadFailure, FeedFileEntry, JobLogger } from "./types.js"; @@ -93,7 +94,11 @@ export async function defaultRunner( args: string[], opts: { cwd?: string; stdio?: "inherit" | "pipe" }, ): Promise { - await execa(command, args, { cwd: opts.cwd, stdio: opts.stdio ?? "pipe" }); + await execa(command, args, { + cwd: opts.cwd, + stdio: opts.stdio ?? "pipe", + cancelSignal: dataWorkerSignal(), + }); } export function normaliseCountries(countries: string[]): string[] { diff --git a/services/data-manager/src/jobs/transitous/motis-import.ts b/services/data-manager/src/jobs/transitous/motis-import.ts index d3d2e13f0..f5731078b 100644 --- a/services/data-manager/src/jobs/transitous/motis-import.ts +++ b/services/data-manager/src/jobs/transitous/motis-import.ts @@ -1,9 +1,9 @@ import { existsSync, writeFileSync } from "node:fs"; import { join } from "node:path"; -import { runOpsOperation } from "../../ops-client.js"; import { verifyCandidateManifest } from "./candidate.js"; import { IMPORT_MARKER_FILE } from "./internal.js"; import { STAGING_CONTAINER as STAGING_CONTAINER_NAME } from "./motis-containers.js"; +import { assertTransitForward, preserveTransitPending, runTransitOwnedPhase } from "./owned-ops.js"; import type { StageFn, StageResult } from "./types.js"; /** @@ -75,8 +75,9 @@ export const run: StageFn = async (ctx) => { // here anyway. try { // Restarting a container is host authority; the agent owns the name. - await runOpsOperation({ kind: "motis.staging.restart" }); + await runTransitOwnedPhase(ctx, "restart-staging"); } catch (error) { + preserveTransitPending(ctx, error); const err = error as Error; return { stage: "motis-import", @@ -97,6 +98,7 @@ export const run: StageFn = async (ctx) => { // Best-effort: if the write fails (read-only mount, race with restart) we // still return ok and let promote fall through to its MOTIS-file sentinel. try { + assertTransitForward(ctx); writeFileSync( join(ctx.motisStagingDataDir, IMPORT_MARKER_FILE), `${JSON.stringify( @@ -130,6 +132,7 @@ export const run: StageFn = async (ctx) => { }, } satisfies StageResult; } catch (error) { + preserveTransitPending(ctx, error); const err = error as Error; return { stage: "motis-import", diff --git a/services/data-manager/src/jobs/transitous/motis-probe.ts b/services/data-manager/src/jobs/transitous/motis-probe.ts index 5e9c4c088..71f22c5f7 100644 --- a/services/data-manager/src/jobs/transitous/motis-probe.ts +++ b/services/data-manager/src/jobs/transitous/motis-probe.ts @@ -1,3 +1,5 @@ +import { setTimeout as probeDelay } from "node:timers/promises"; +import { dataWorkerDeadline, dataWorkerSignal } from "../execution/worker-scope.js"; /** * Shared HTTP-probe helpers for the MOTIS staging/primary smoke checks. Both * motis-health and promote used to carry byte-identical copies of these; the @@ -71,7 +73,7 @@ function rawHttpGet( clearTimeout(timer); fn(); }; - const req = client.get(url, { headers }, (res) => { + const req = client.get(url, { headers, signal: dataWorkerSignal() }, (res) => { const chunks: Buffer[] = []; res.on("data", (chunk) => chunks.push(chunk as Buffer)); res.on("end", () => @@ -116,6 +118,8 @@ export async function fetchWithTimeout( timeoutMs: number, options: { retries?: number; headers?: Record } = {}, ): Promise { + dataWorkerSignal()?.throwIfAborted(); + timeoutMs = Math.max(1, Math.min(timeoutMs, (dataWorkerDeadline() ?? Infinity) - Date.now())); const retries = options.retries ?? 2; let lastError: unknown; for (let attempt = 0; attempt <= retries; attempt++) { @@ -124,7 +128,7 @@ export async function fetchWithTimeout( } catch (error) { lastError = error; if (attempt === retries || !isTransientFetchError(error)) throw error; - await new Promise((resolve) => setTimeout(resolve, 200)); + await probeDelay(200, undefined, { signal: dataWorkerSignal() }); } } throw lastError; @@ -158,13 +162,6 @@ export async function probe( } } -function delay(ms: number): Promise { - return new Promise((resolve) => { - const t = setTimeout(resolve, ms); - if (typeof t.unref === "function") t.unref(); - }); -} - /** * Poll `url` until it answers 200 + JSON (returns null) or `deadline` passes. * @@ -181,17 +178,19 @@ export async function pollUntilHealthy( deadline: number, opts: { name?: string; intervalMs?: number; probeTimeoutMs?: number } = {}, ): Promise { + deadline = Math.min(deadline, dataWorkerDeadline() ?? Infinity); const name = opts.name ?? "health"; const interval = opts.intervalMs ?? 1_000; const probeTimeout = opts.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; let last: ProbeFailure | null = null; while (Date.now() < deadline) { + dataWorkerSignal()?.throwIfAborted(); const remaining = deadline - Date.now(); last = await probe(name, url, Math.min(probeTimeout, remaining)); if (!last) return null; const sleep = Math.min(interval, deadline - Date.now()); if (sleep <= 0) break; - await delay(sleep); + await probeDelay(sleep, undefined, { signal: dataWorkerSignal() }); } return last ?? { probe: name, reason: "did not become healthy within budget" }; } diff --git a/services/data-manager/src/jobs/transitous/owned-ops.ts b/services/data-manager/src/jobs/transitous/owned-ops.ts new file mode 100644 index 000000000..30f6ea5f6 --- /dev/null +++ b/services/data-manager/src/jobs/transitous/owned-ops.ts @@ -0,0 +1,67 @@ +import { retainOwnedDataTransitCandidate } from "@openmapx/core/build-artifacts"; +import { + type DataOpsChildPhase, + DataOpsObservationPending, + reserveOwnedProxyRecovery, + runOwnedDataOpsOperation, +} from "../execution/ops-children.js"; +import type { FreshDataChildReservation } from "../execution/store.js"; +import { DataPublicationUnknown, publishDataChange } from "../execution/worker-scope.js"; +import type { JobContext } from "./types.js"; +export async function transitOwnedPhase(ctx: JobContext, phase: DataOpsChildPhase) { + const context = ctx.executionContext; + if (!context) throw Error("Durable transit ownership is required"); + const record = await context.record(), + child = record.reservations.find((slot) => slot.phase === phase); + if (!child) throw Error("Transit phase is not in the original reservation"); + return { context, child }; +} +export async function retainTransitPhaseCandidate( + ctx: JobContext, + phase: DataOpsChildPhase, +): Promise { + const { context, child } = await transitOwnedPhase(ctx, phase); + retainOwnedDataTransitCandidate(context.authority, child); +} +export async function runTransitOwnedPhase( + ctx: JobContext, + phase: DataOpsChildPhase, + reservedContinuation?: FreshDataChildReservation, +) { + const { context, child } = await transitOwnedPhase(ctx, phase); + return runOwnedDataOpsOperation(child.operation, { + context, + phase, + ordinal: child.ordinal, + serviceIds: child.serviceIds, + ...(reservedContinuation ? { reservedContinuation } : {}), + }); +} +export async function reserveTransitProxyRecovery( + ctx: JobContext, + backupDigest: string, +): Promise { + if (!ctx.executionContext) throw Error("Original proxy recovery owner missing"); + return reserveOwnedProxyRecovery(ctx.executionContext, backupDigest); +} +export function assertTransitForward(ctx: JobContext): void { + ctx.abortSignal.throwIfAborted(); + if (!ctx.executionContext) throw Error("Durable transit ownership is required"); + ctx.executionContext.assertForward(); +} +export function preserveTransitPending(ctx: JobContext, error: unknown): void { + if (error instanceof DataOpsObservationPending || error instanceof DataPublicationUnknown) + throw error; + ctx.abortSignal.throwIfAborted(); +} + +export async function publishTransitChange( + ctx: JobContext, + run: () => T | Promise, +): Promise { + assertTransitForward(ctx); + return publishDataChange(() => { + assertTransitForward(ctx); + return run(); + }); +} diff --git a/services/data-manager/src/jobs/transitous/pipeline.ts b/services/data-manager/src/jobs/transitous/pipeline.ts index 8df0186ac..96f6fb399 100644 --- a/services/data-manager/src/jobs/transitous/pipeline.ts +++ b/services/data-manager/src/jobs/transitous/pipeline.ts @@ -144,17 +144,11 @@ export async function runTransitousPipeline( if (!entry) continue; if (ctx.abortSignal.aborted) { ctx.logger.warn(`transitous-pipeline: aborted before ${entry.name}`); - break; + ctx.abortSignal.throwIfAborted(); } const result = await entry.run(ctx); results.push(result); - try { - await ctx.onStageComplete(result); - } catch (err) { - ctx.logger.warn( - `transitous-pipeline: onStageComplete threw for ${entry.name}: ${(err as Error).message}`, - ); - } + await ctx.onStageComplete(result); if (result.status === "error" && entry.criticality === "critical") { hardStopError = new Error( result.error?.message ?? result.message ?? `${entry.name} failed`, @@ -196,6 +190,7 @@ export async function runTransitousPipeline( } if (hardStopError) throw hardStopError; + ctx.abortSignal.throwIfAborted(); return { jobId: ctx.jobId, @@ -214,6 +209,7 @@ function aggregateFinalStatus(results: StageResult[]): StageStatus { } export interface BuildJobContextOptions { + executionContext?: import("../execution/runtime.js").DataExecutionContext; dataDir: string; store: StateStore; countries?: string[]; @@ -269,6 +265,7 @@ export function buildJobContext(opts: BuildJobContextOptions): JobContext { const slotLayout = process.env.MOTIS_TWO_SLOT === "true" ? ensureMotisSlotLayout(opts.dataDir) : undefined; return { + executionContext: opts.executionContext, jobId: opts.jobId ?? randomUUID(), dataDir: opts.dataDir, catalogDir, diff --git a/services/data-manager/src/jobs/transitous/promote.ts b/services/data-manager/src/jobs/transitous/promote.ts index 2b0726aaf..3dc84c082 100644 --- a/services/data-manager/src/jobs/transitous/promote.ts +++ b/services/data-manager/src/jobs/transitous/promote.ts @@ -10,8 +10,7 @@ import { statSync, } from "node:fs"; import { join } from "node:path"; -import { OPS_KIND_POLICIES, type OpsResultFor, parseOpsResult } from "@openmapx/core/ops"; -import { runOpsOperation } from "../../ops-client.js"; +import { type OpsResultFor, parseOpsResult } from "@openmapx/core/ops"; import { CANDIDATE_MANIFEST_FILENAME, type MotisCandidateManifest, @@ -24,6 +23,13 @@ import { IMPORT_MARKER_FILE } from "./internal.js"; import { PRIMARY_CONTAINER, STAGING_CONTAINER } from "./motis-containers.js"; import { healthUrl } from "./motis-endpoints.js"; import { parseIntEnv, pollUntilHealthy } from "./motis-probe.js"; +import { + assertTransitForward, + preserveTransitPending, + publishTransitChange, + retainTransitPhaseCandidate, + runTransitOwnedPhase, +} from "./owned-ops.js"; import { commitProxyTransaction } from "./proxy-transaction.js"; import { aliasSlot, @@ -38,7 +44,7 @@ const PRIMARY_URL = process.env.MOTIS_URL ?? "http://localhost:8081"; const STAGING_URL = process.env.MOTIS_STAGING_URL ?? "http://localhost:8082"; const SMOKE_BUDGET_MS = 30_000; -const RECOVERY_BUDGET_MS = OPS_KIND_POLICIES["motis.primary.restart"].timeoutMs + 5_000; + // How long to wait for the primary to come back healthy after the swap+restart. // The restarted container re-loads the promoted dataset before it binds its // server — for a real region (e.g. Germany) that's several minutes, and a @@ -173,7 +179,10 @@ async function stopContainer( ? "motis.staging.stop" : null; if (!kind) throw new Error("Unapproved MOTIS stop target"); - const result = await runOpsOperation({ kind }, { signal: ctx.abortSignal }); + const result = await runTransitOwnedPhase( + ctx, + container === PRIMARY_CONTAINER ? "stop-primary" : "stop-staging", + ); try { return parseOpsResult(kind, result); } catch { @@ -199,7 +208,7 @@ function dataIdentity(path: string): string | null { } } -async function stopBeforeSwap(ctx: JobContext, prepare?: () => void): Promise { +async function stopBeforeSwap(ctx: JobContext, prepare?: () => Promise): Promise { const live = dataIdentity(ctx.motisDataDir); const staging = dataIdentity(ctx.motisStagingDataDir); let primary: OpsResultFor<"motis.primary.stop"> | undefined; @@ -212,11 +221,12 @@ async function stopBeforeSwap(ctx: JobContext, prepare?: () => void): Promise void): Promise void): Promise { +async function restartPrimary(ctx: JobContext, recovery = false): Promise { try { // The data swap already happened here; recreating the primary against the // new A/B target is host authority and belongs to the agent, which owns the // compose file, the container name, and the `--force-recreate --no-deps` // semantics that keep the post-swap health probe from racing a cascade. // The slot now aliased live is the run being activated. - const preparedRunId = ctx.slotLayout - ? (aliasSlot(ctx.slotLayout, "live") ?? "motis-primary") - : "motis-primary"; - await runOpsOperation({ kind: "motis.primary.promote", preparedRunId }); + await runTransitOwnedPhase(ctx, recovery ? "recover-primary" : "promote-primary"); return null; } catch (error) { + preserveTransitPending(ctx, error); return (error as Error).message; } } @@ -269,6 +272,7 @@ async function restartPrimary(ctx: JobContext): Promise { * data/motis/live`. */ function tryRollback( + ctx: JobContext, currentDir: string, stagingDir: string, previousDir: string, @@ -277,16 +281,19 @@ function tryRollback( if (!existsSync(previousDir)) { return { ok: false, reason: `previous dir ${previousDir} missing; cannot rollback` }; } + assertTransitForward(ctx); if (existsSync(stagingDir)) { // Staging was recreated empty after the swap; remove the empty shell // so we can move current → staging cleanly. rmSync(stagingDir, { recursive: true, force: true }); } + assertTransitForward(ctx); if (existsSync(currentDir)) renameSync(currentDir, stagingDir); + assertTransitForward(ctx); renameSync(previousDir, currentDir); return { ok: true }; } catch (error) { - return { ok: false, reason: (error as Error).message }; + throw error; } } @@ -305,15 +312,18 @@ async function promoteTwoSlot( throw new Error("MOTIS slot aliases do not identify distinct active/inactive slots"); } const rollback = async (): Promise => { - flipMotisSlotAliases(layout, previous); - return restartPrimary(ctx); + assertTransitForward(ctx); + await publishTransitChange(ctx, () => flipMotisSlotAliases(layout, previous)); + return restartPrimary(ctx, true); }; await stopBeforeSwap(ctx); - flipMotisSlotAliases(layout, candidate); + assertTransitForward(ctx); + await publishTransitChange(ctx, () => flipMotisSlotAliases(layout, candidate)); const restartStart = Date.now(); const restartError = await restartPrimary(ctx); if (restartError) { + assertTransitForward(ctx); const rollbackError = await rollback(); return { stage: "promote", @@ -334,6 +344,7 @@ async function promoteTwoSlot( ? null : await runFunctionalProbes(PRIMARY_URL, manifest, Date.now() + SMOKE_BUDGET_MS); if (healthError || !primaryReport?.ok) { + assertTransitForward(ctx); const rollbackError = await rollback(); return { stage: "promote", @@ -352,6 +363,8 @@ async function promoteTwoSlot( try { await commitProxyTransaction(ctx); } catch (error) { + preserveTransitPending(ctx, error); + assertTransitForward(ctx); const rollbackError = await rollback(); return { stage: "promote", @@ -367,13 +380,15 @@ async function promoteTwoSlot( }, }; } - const record = commitMotisSlotActivation(layout, { - activeSlot: candidate as MotisSlot, - datasetEpoch: manifest.epoch, - manifestHash: manifest.artifacts.config.sha256, - imageDigest: process.env.MOTIS_IMAGE_DIGEST, - activatedAt: ctx.now(), - }); + const record = await publishTransitChange(ctx, () => + commitMotisSlotActivation(layout, { + activeSlot: candidate as MotisSlot, + datasetEpoch: manifest.epoch, + manifestHash: manifest.artifacts.config.sha256, + imageDigest: process.env.MOTIS_IMAGE_DIGEST, + activatedAt: ctx.now(), + }), + ); return { stage: "promote", status: "ok", @@ -442,6 +457,7 @@ export const run: StageFn = async (ctx) => { } const manifest = verifyCandidateManifest(stagingDir); + await retainTransitPhaseCandidate(ctx, "promote-primary"); assertCandidateEpochIsNew(ctx, currentDir, manifest.epoch); const sourceManifest = readTransitSourceManifest( join(stagingDir, TRANSIT_SOURCE_MANIFEST_FILENAME), @@ -471,8 +487,9 @@ export const run: StageFn = async (ctx) => { if (result.status === "ok") { for (const source of sourceManifest.sources) { try { - await recordPromotedSource(source); + await publishTransitChange(ctx, () => recordPromotedSource(source)); } catch (error) { + preserveTransitPending(ctx, error); ctx.logger.warn( `transitous-promote: feed_state import timestamp failed for ${source.sourceId}: ${(error as Error).message}`, ); @@ -482,10 +499,12 @@ export const run: StageFn = async (ctx) => { return result; } - await stopBeforeSwap(ctx, () => { + await stopBeforeSwap(ctx, async () => { // Retain rollback data until both containers are confirmed stopped. if (existsSync(previousDir)) { - rmSync(previousDir, { recursive: true, force: true }); + await publishTransitChange(ctx, () => + rmSync(previousDir, { recursive: true, force: true }), + ); } }); @@ -495,13 +514,15 @@ export const run: StageFn = async (ctx) => { let firstRenameDone = false; try { if (existsSync(currentDir)) { - renameSync(currentDir, previousDir); + assertTransitForward(ctx); + await publishTransitChange(ctx, () => renameSync(currentDir, previousDir)); firstRenameDone = true; } } catch (error) { + preserveTransitPending(ctx, error); // We stopped the primary above; bring it back before returning so a // rename failure doesn't leave it down on the (unchanged) old data. - await restartPrimary(ctx); + await restartPrimary(ctx, true); return { stage: "promote", status: "error", @@ -526,18 +547,20 @@ export const run: StageFn = async (ctx) => { // disappeared between the pre-flight check and the swap. } try { - renameSync(stagingDir, currentDir); + assertTransitForward(ctx); + await publishTransitChange(ctx, () => renameSync(stagingDir, currentDir)); } catch (error) { + preserveTransitPending(ctx, error); // Roll back the first rename if it happened. if (firstRenameDone) { try { - renameSync(previousDir, currentDir); + await publishTransitChange(ctx, () => renameSync(previousDir, currentDir)); } catch { // Swallow — the outer error is more useful. } } // Restore the primary we stopped above on the (rolled-back) old data. - await restartPrimary(ctx); + await restartPrimary(ctx, true); return { stage: "promote", status: "error", @@ -549,6 +572,7 @@ export const run: StageFn = async (ctx) => { } satisfies StageResult; } + assertTransitForward(ctx); // Recreate an empty staging dir for the next cycle. try { mkdirSync(stagingDir, { recursive: true, mode: stagingMode }); @@ -565,14 +589,17 @@ export const run: StageFn = async (ctx) => { if (restartErr) { // Rollback rename and try to restart again so we at least restore the // previous live state. - const rb = tryRollback(currentDir, stagingDir, previousDir); - const secondRestartErr = await restartPrimary(ctx); + assertTransitForward(ctx); + const rb = await publishTransitChange(ctx, () => + tryRollback(ctx, currentDir, stagingDir, previousDir), + ); + const secondRestartErr = await restartPrimary(ctx, true); const restartDurationMs = Date.now() - restartStart; // Worst case: rollback failed AND restart still fails. Mark the broken // state on disk so the operator can inspect/recover manually. if (!rb.ok && secondRestartErr && existsSync(previousDir)) { try { - renameSync(previousDir, `${previousDir}-broken`); + await publishTransitChange(ctx, () => renameSync(previousDir, `${previousDir}-broken`)); } catch { // Best effort — surface the original error. } @@ -604,8 +631,11 @@ export const run: StageFn = async (ctx) => { const restartDurationMs = Date.now() - restartStart; if (healthErr) { // Roll back rename and restart again — same flow as restart failure. - const rb = tryRollback(currentDir, stagingDir, previousDir); - const secondRestartErr = await restartPrimary(ctx); + assertTransitForward(ctx); + const rb = await publishTransitChange(ctx, () => + tryRollback(ctx, currentDir, stagingDir, previousDir), + ); + const secondRestartErr = await restartPrimary(ctx, true); const rollbackMsg = rb.ok ? "ok" : `failed: ${(rb as { reason: string }).reason}`; const restartMsg = secondRestartErr ? `; second restart failed: ${secondRestartErr}` : ""; return { @@ -632,8 +662,11 @@ export const run: StageFn = async (ctx) => { Date.now() + SMOKE_BUDGET_MS, ); if (!primaryReport.ok) { - const rb = tryRollback(currentDir, stagingDir, previousDir); - const secondRestartErr = await restartPrimary(ctx); + assertTransitForward(ctx); + const rb = await publishTransitChange(ctx, () => + tryRollback(ctx, currentDir, stagingDir, previousDir), + ); + const secondRestartErr = await restartPrimary(ctx, true); const rollbackMsg = rb.ok ? "ok" : `failed: ${(rb as { reason: string }).reason}`; const restartMsg = secondRestartErr ? `; rollback restart failed: ${secondRestartErr}` : ""; return { @@ -657,8 +690,12 @@ export const run: StageFn = async (ctx) => { try { await commitProxyTransaction(ctx); } catch (error) { - const rb = tryRollback(currentDir, stagingDir, previousDir); - await restartPrimary(ctx); + preserveTransitPending(ctx, error); + assertTransitForward(ctx); + const rb = await publishTransitChange(ctx, () => + tryRollback(ctx, currentDir, stagingDir, previousDir), + ); + await restartPrimary(ctx, true); return { stage: "promote", status: "error", @@ -680,8 +717,9 @@ export const run: StageFn = async (ctx) => { for (const source of sourceManifest.sources) { try { - await recordPromotedSource(source); + await publishTransitChange(ctx, () => recordPromotedSource(source)); } catch (error) { + preserveTransitPending(ctx, error); ctx.logger.warn( `transitous-promote: feed_state import timestamp failed for ${source.sourceId}: ${(error as Error).message}`, ); @@ -705,6 +743,7 @@ export const run: StageFn = async (ctx) => { }, } satisfies StageResult; } catch (error) { + preserveTransitPending(ctx, error); const err = error as Error; return { stage: "promote", diff --git a/services/data-manager/src/jobs/transitous/proxy-transaction.ts b/services/data-manager/src/jobs/transitous/proxy-transaction.ts index fed45bae7..5ecaeb0d1 100644 --- a/services/data-manager/src/jobs/transitous/proxy-transaction.ts +++ b/services/data-manager/src/jobs/transitous/proxy-transaction.ts @@ -1,18 +1,28 @@ -import { existsSync, readFileSync, rmSync, statSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, rmSync } from "node:fs"; import { join } from "node:path"; +import { canonicalJson } from "@openmapx/core/ops"; import { appliedHardlinkEntries, applyHardlinkPlan } from "@openmapx/hardlinks"; import { type FeedProxyVars, normalizeFeedProxyVars, renderFeedProxyNginxConfig, } from "@openmapx/motis-feed-proxy-config"; -import { runOpsOperation } from "../../ops-client.js"; import { atomicWriteFileSync } from "../../utils/atomic-write.js"; +import type { FreshDataChildReservation } from "../execution/store.js"; import { CANDIDATE_MANIFEST_FILENAME, CANDIDATE_PROXY_DIRNAME, verifyCandidateManifest, } from "./candidate.js"; +import { + assertTransitForward, + preserveTransitPending, + publishTransitChange, + reserveTransitProxyRecovery, + retainTransitPhaseCandidate, + runTransitOwnedPhase, +} from "./owned-ops.js"; import type { JobContext, ProxyTransactionState, StageFn, StageResult } from "./types.js"; export const ACTIVE_PROXY_ROOT_DIRNAME = "motis-feed-proxy"; @@ -96,28 +106,34 @@ function relinkFeedProxyConfig(ctx: JobContext): void { applyHardlinkPlan(entries, { rootDir: ctx.dataDir, prune: false }); } -async function validateAndReload(ctx: JobContext): Promise { - // Propagate the just-written config into the container's mounted copy before - // asking nginx to validate/reload it. - relinkFeedProxyConfig(ctx); - // The agent validates the configuration and only then reloads, so a bad - // candidate can never take the proxy down. - await runOpsOperation({ - kind: "feedProxy.validateAndReload", - // The mtime of the config just written identifies the candidate being - // activated; the agent needs only that opaque id. - candidateId: `feedproxy-${Math.trunc(statSync(join(ctx.dataDir, ACTIVE_PROXY_ROOT_DIRNAME, CONFIG_RELATIVE_PATH)).mtimeMs)}`, - }); +async function validateAndReload( + ctx: JobContext, + phase: "proxy-stage" | "proxy-commit" | "proxy-restore", + reservedContinuation?: FreshDataChildReservation, +): Promise { + assertTransitForward(ctx); + await publishTransitChange(ctx, () => relinkFeedProxyConfig(ctx)); + await retainTransitPhaseCandidate(ctx, phase); + await runTransitOwnedPhase(ctx, phase, reservedContinuation); } async function restoreAndReload(ctx: JobContext, state: ProxyTransactionState): Promise { - restore(state.activeConfigPath, state.previousConfig); - restore(state.activeVarsPath, state.previousVars); - try { - await validateAndReload(ctx); - } finally { - state.phase = "rolled-back"; - } + assertTransitForward(ctx); + const continuation = await reserveTransitProxyRecovery( + ctx, + createHash("sha256") + .update( + canonicalJson({ previousConfig: state.previousConfig, previousVars: state.previousVars }), + ) + .digest("hex"), + ); + await publishTransitChange(ctx, () => { + restore(state.activeConfigPath, state.previousConfig); + assertTransitForward(ctx); + restore(state.activeVarsPath, state.previousVars); + }); + await validateAndReload(ctx, "proxy-restore", continuation); + state.phase = "rolled-back"; } export async function rollbackProxyTransaction(ctx: JobContext): Promise { @@ -131,11 +147,16 @@ export async function commitProxyTransaction(ctx: JobContext): Promise { const state = ctx.state.proxyTransaction; if (state?.phase !== "staged") return; try { - writeProxyFile(state.activeConfigPath, state.candidateConfig); - writeProxyFile(state.activeVarsPath, state.candidateVars); - await validateAndReload(ctx); + assertTransitForward(ctx); + await publishTransitChange(ctx, () => { + writeProxyFile(state.activeConfigPath, state.candidateConfig); + assertTransitForward(ctx); + writeProxyFile(state.activeVarsPath, state.candidateVars); + }); + await validateAndReload(ctx, "proxy-commit"); state.phase = "committed"; } catch (error) { + preserveTransitPending(ctx, error); await restoreAndReload(ctx, state); throw new Error( `feed-proxy commit failed and previous routing was restored: ${(error as Error).message}`, @@ -193,10 +214,15 @@ export const run: StageFn = async (ctx) => { }; ctx.state.proxyTransaction = state; try { - writeProxyFile(activeConfigPath, unionConfig); - writeProxyFile(activeVarsPath, unionVars); - await validateAndReload(ctx); + assertTransitForward(ctx); + await publishTransitChange(ctx, () => { + writeProxyFile(activeConfigPath, unionConfig); + assertTransitForward(ctx); + writeProxyFile(activeVarsPath, unionVars); + }); + await validateAndReload(ctx, "proxy-stage"); } catch (error) { + preserveTransitPending(ctx, error); await restoreAndReload(ctx, state); throw new Error( `feed-proxy activation failed and previous routing was restored: ${(error as Error).message}`, @@ -209,6 +235,7 @@ export const run: StageFn = async (ctx) => { activeEntries: Object.keys(union).length, }); } catch (error) { + preserveTransitPending(ctx, error); return finish("error", (error as Error).message, { rollback: true }); } }; diff --git a/services/data-manager/src/jobs/transitous/runtime.ts b/services/data-manager/src/jobs/transitous/runtime.ts index 797232ea0..ebdcfc0f4 100644 --- a/services/data-manager/src/jobs/transitous/runtime.ts +++ b/services/data-manager/src/jobs/transitous/runtime.ts @@ -1,4 +1,4 @@ -import { db } from "../../db/index.js"; +import { db, rawSql } from "../../db/index.js"; import { createSingleFlightController, type SingleFlightController } from "./single-flight.js"; /** @@ -14,7 +14,7 @@ let cached: SingleFlightController | null = null; export function getSingleFlightController(): SingleFlightController { if (!cached) { - cached = createSingleFlightController({ db }); + cached = createSingleFlightController({ db, admissionSql: rawSql }); } return cached; } diff --git a/services/data-manager/src/jobs/transitous/script-runner.ts b/services/data-manager/src/jobs/transitous/script-runner.ts index afdc634bc..91fc1f40c 100644 --- a/services/data-manager/src/jobs/transitous/script-runner.ts +++ b/services/data-manager/src/jobs/transitous/script-runner.ts @@ -1,4 +1,5 @@ import { readFileSync } from "node:fs"; +import { setTimeout as runnerDelay } from "node:timers/promises"; import { mintTransitousCapability, parseCapabilityKey, @@ -6,8 +7,10 @@ import { TRANSITOUS_RUNNER_TIMEOUT_MS, type TransitousRunnerScript, transitousRunnerResultSchema, + transitousRunnerStatusSchema, } from "@openmapx/core/transitous-runner"; import { scrubSecrets } from "../../utils/scrub-secrets.js"; +import { dataExecutionWorkerContext, dataWorkerSignal } from "../execution/worker-scope.js"; /** * Client for the private Transitous runner. @@ -56,45 +59,112 @@ export function createTransitousScriptRunner( // The capability is minted at dispatch, not at queue time: it authorizes // this one run and expires within minutes. const capability = mintTransitousCapability(options.capabilityKey, { now: now(), run }); - const controller = new AbortController(); - // Slightly beyond the runner's own ceiling so its bounded result, not a - // client abort, is what the pipeline sees for a script that overruns. - const timer = setTimeout(() => controller.abort(), TRANSITOUS_RUNNER_TIMEOUT_MS + 30_000); - - let response: Response; - try { - response = await fetchImpl(`${options.baseUrl.replace(/\/+$/, "")}/v1/run`, { + const controller = new AbortController(), + workerSignal = dataWorkerSignal(), + context = dataExecutionWorkerContext(), + request = { version: TRANSITOUS_RUNNER_PROTOCOL_VERSION, capability, run }; + const journal = context?.store.controlJournal; + if (context && !journal) throw Error("Retained runner owner is unavailable"); + journal?.retainRunnerAttempt(context!.authority, request); + const timer = setTimeout(() => controller.abort(), TRANSITOUS_RUNNER_TIMEOUT_MS + 30000), + signal = workerSignal + ? AbortSignal.any([workerSignal, controller.signal]) + : controller.signal; + const endpoint = options.baseUrl.replace(/\/+$/, ""); + async function body(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) throw Error("Runner response missing"); + const chunks: Uint8Array[] = []; + let length = 0; + try { + for (;;) { + const next = await reader.read(); + if (next.done) break; + length += next.value.byteLength; + if (length > 8 * 1024 * 1024) { + await reader.cancel(); + throw Error("Runner response capacity exceeded"); + } + chunks.push(next.value); + } + return JSON.parse(Buffer.concat(chunks, length).toString("utf8")); + } finally { + reader.releaseLock(); + } + } + const control = async ( + action: "inspect" | "cancel" | "acknowledge", + settlement: AbortSignal, + ) => { + const response = await fetchImpl(`${endpoint}/v1/run/${action}`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ - version: TRANSITOUS_RUNNER_PROTOCOL_VERSION, - capability, - run, - }), - signal: controller.signal, + body: JSON.stringify(request), + signal: settlement, }); + if (!response.ok) throw Error("Original runner outcome is unknown"); + return body(response); + }; + let result: unknown, + remoteContained = false; + try { + try { + const response = await fetchImpl(`${endpoint}/v1/run`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + signal, + }); + if (!response.ok) throw Error(`Runner HTTP ${response.status}`); + result = await body(response); + } catch (error) { + if (!context && !signal.aborted) throw error; // Standalone client tests have no durable platform owner. + const settlement = AbortSignal.timeout(30000); + for (;;) { + settlement.throwIfAborted(); + if (signal.aborted) + try { + await control("cancel", settlement); + } catch { + /* Retry cancellation only, never execution. */ + } + const state = transitousRunnerStatusSchema.parse(await control("inspect", settlement)); + if (state.state === "terminal") { + result = state.result; + break; + } + await runnerDelay(100, undefined, { signal: settlement }); + } + } + const checked = transitousRunnerResultSchema.safeParse(result); + if (!checked.success) + throw new TransitousScriptError( + `Transitous runner returned an unexpected response for ${describe(run)}`, + ); + const parsed = checked.data; + journal?.completeRunnerAttempt(context!.authority, request, parsed); + remoteContained = true; + if (context) + try { + await control("acknowledge", AbortSignal.timeout(5000)); + } catch { + /* Original contained result is already retained locally. */ + } + if (signal.aborted) signal.throwIfAborted(); + result = parsed; } catch (error) { - // The request body carries a live capability; only the reason is safe to - // repeat, never the request itself. + if (context && !remoteContained) await context.checkpoint({ publication: "unknown" }); throw new TransitousScriptError( - `Transitous runner is unreachable for ${describe(run)}: ${scrubSecrets((error as Error).message)}`, + `Transitous runner outcome is unresolved for ${describe(run)}: ${scrubSecrets((error as Error).message)}`, ); } finally { clearTimeout(timer); } - - if (!response.ok) { - throw new TransitousScriptError( - `Transitous runner refused ${describe(run)} (HTTP ${response.status})`, - ); - } - - const parsed = transitousRunnerResultSchema.safeParse(await response.json()); - if (!parsed.success) { + const parsed = transitousRunnerResultSchema.safeParse(result); + if (!parsed.success) throw new TransitousScriptError( `Transitous runner returned an unexpected response for ${describe(run)}`, ); - } if (!parsed.data.ok) { // The output is the script's own stderr, which downstream failure // attribution parses. It is scrubbed first: a feed URL in it can carry an @@ -125,5 +195,10 @@ export function createDefaultTransitousScriptRunner(): TransitousScriptRunner { // runner, which is the correct outcome for a missing secret. } } - return createTransitousScriptRunner({ baseUrl, capabilityKey }); + const runner = createTransitousScriptRunner({ baseUrl, capabilityKey }); + return async (run) => { + if (!dataExecutionWorkerContext()) + throw new TransitousScriptError("Durable transit worker ownership is required"); + return runner(run); + }; } diff --git a/services/data-manager/src/jobs/transitous/single-flight.ts b/services/data-manager/src/jobs/transitous/single-flight.ts index 1ea16ad8b..d920dbdab 100644 --- a/services/data-manager/src/jobs/transitous/single-flight.ts +++ b/services/data-manager/src/jobs/transitous/single-flight.ts @@ -1,6 +1,10 @@ import { jobs } from "@openmapx/db-schema"; import { and, desc, eq, gt } from "drizzle-orm"; +import type postgres from "postgres"; +import z from "zod/v4"; import type { db as defaultDb } from "../../db/index.js"; +import { assertDataMutationAdmission } from "../execution/admission.js"; +import { withDataAdmissionTransaction } from "../execution/store.js"; /** * Single-flight controller for the Transitous sync pipeline. Prevents two @@ -38,13 +42,15 @@ export interface SingleFlightDeps { db: typeof defaultDb; /** `Date.now()` indirection so tests can advance the clock deterministically. */ now?: () => number; + admissionSql?: postgres.Sql; } const IDEMPOTENCY_WINDOW_MS = 24 * 60 * 60 * 1000; export interface SingleFlightController { tryStartSync(opts: TryStartSyncOptions): Promise; - markSyncFinished(): void; + reserveOwned(jobId: string): boolean; + markSyncFinished(jobId?: string): void; getInflight(): { jobId: string; startedAt: Date } | null; } @@ -70,42 +76,72 @@ export function createSingleFlightController(deps: SingleFlightDeps): SingleFlig starting = true; try { - if (opts.idempotencyKey) { - const cutoff = new Date(now() - IDEMPOTENCY_WINDOW_MS); - const existing = await db - .select({ id: jobs.id }) - .from(jobs) - .where(and(eq(jobs.idempotencyKey, opts.idempotencyKey), gt(jobs.startedAt, cutoff))) - .orderBy(desc(jobs.startedAt)) - .limit(1); - const row = existing[0]; - if (row) { - return { ok: false, reason: "duplicate-idempotency-key", existingJobId: row.id }; + const reserve = async ( + db: Pick, + ): Promise => { + if (opts.idempotencyKey) { + const cutoff = new Date(now() - IDEMPOTENCY_WINDOW_MS); + const existing = await db + .select({ id: jobs.id }) + .from(jobs) + .where(and(eq(jobs.idempotencyKey, opts.idempotencyKey), gt(jobs.startedAt, cutoff))) + .orderBy(desc(jobs.startedAt)) + .limit(1); + const row = existing[0]; + if (row) { + return { ok: false, reason: "duplicate-idempotency-key", existingJobId: row.id }; + } } - } - - const inserted = await db - .insert(jobs) - .values({ - kind: opts.kind ?? "transitous-sync", - status: "running", - triggeredBy: `${opts.trigger}:${opts.triggeredBy}`, - idempotencyKey: opts.idempotencyKey ?? null, - metadata: opts.metadata ?? null, - }) - .returning({ id: jobs.id }); - - const row = inserted[0]; - if (!row) throw new Error("Failed to create data_manager.jobs row"); - - inflight = { jobId: row.id, startedAt: new Date(now()) }; - return { ok: true, jobId: row.id }; + + const inserted = await db + .insert(jobs) + .values({ + kind: opts.kind ?? "transitous-sync", + status: "running", + triggeredBy: `${opts.trigger}:${opts.triggeredBy}`, + idempotencyKey: opts.idempotencyKey ?? null, + metadata: opts.metadata ?? null, + }) + .returning({ id: jobs.id }); + + const row = inserted[0]; + if (!row) throw new Error("Failed to create data_manager.jobs row"); + + inflight = { jobId: row.id, startedAt: new Date(now()) }; + return { ok: true, jobId: row.id }; + }; + if (!deps.admissionSql) return await reserve(db); + return await withDataAdmissionTransaction( + deps.admissionSql, + async (tx): Promise => { + await assertDataMutationAdmission(tx); + if (opts.idempotencyKey) { + const existing = await tx< + { id: string }[] + >`SELECT id FROM data_manager.jobs WHERE idempotency_key=${opts.idempotencyKey} AND started_at>${new Date(now() - IDEMPOTENCY_WINDOW_MS).toISOString()}::text::timestamptz ORDER BY started_at DESC LIMIT 1`; + if (existing[0]) + return { + ok: false, + reason: "duplicate-idempotency-key", + existingJobId: existing[0].id, + }; + } + const inserted = await tx< + { id: string }[] + >`INSERT INTO data_manager.jobs(kind,status,triggered_by,idempotency_key,metadata) VALUES(${opts.kind ?? "transitous-sync"},'running',${`${opts.trigger}:${opts.triggeredBy}`},${opts.idempotencyKey ?? null},${opts.metadata ? JSON.stringify(z.json().parse(opts.metadata)) : null}::text::jsonb) RETURNING id`; + const row = inserted[0]; + if (!row) throw Error("Transit reservation did not persist"); + inflight = { jobId: row.id, startedAt: new Date(now()) }; + return { ok: true, jobId: row.id }; + }, + ); } finally { starting = false; } } - function markSyncFinished(): void { + function markSyncFinished(jobId?: string): void { + if (jobId !== undefined && inflight?.jobId !== jobId) return; inflight = null; } @@ -113,5 +149,14 @@ export function createSingleFlightController(deps: SingleFlightDeps): SingleFlig return inflight ? { ...inflight } : null; } - return { tryStartSync, markSyncFinished, getInflight }; + return { + tryStartSync, + markSyncFinished, + getInflight, + reserveOwned(jobId) { + if (inflight || starting) return false; + inflight = { jobId, startedAt: new Date(now()) }; + return true; + }, + }; } diff --git a/services/data-manager/src/jobs/transitous/types.ts b/services/data-manager/src/jobs/transitous/types.ts index 9fc30d825..fe46ec43b 100644 --- a/services/data-manager/src/jobs/transitous/types.ts +++ b/services/data-manager/src/jobs/transitous/types.ts @@ -138,6 +138,7 @@ export interface ProxyTransactionState { } export interface JobContext { + executionContext?: import("../execution/runtime.js").DataExecutionContext; /** ULID/UUID */ jobId: string; /** Absolute path, typically `infra/docker/data`. */ diff --git a/services/data-manager/src/offline-packages/postgres-accounting.ts b/services/data-manager/src/offline-packages/postgres-accounting.ts index 2e78c8982..2b1304f1b 100644 --- a/services/data-manager/src/offline-packages/postgres-accounting.ts +++ b/services/data-manager/src/offline-packages/postgres-accounting.ts @@ -1,5 +1,6 @@ import type { CanonicalOfflinePackageRequest, OfflineMapPackageManifest } from "@openmapx/core"; import type postgres from "postgres"; +import { assertDataMutationAdmission } from "../jobs/execution/admission.js"; import { assertOfflineArtifactIdentity, assertOfflinePackagePrincipal, @@ -223,6 +224,7 @@ export class PostgresOfflinePackageAccountingStore implements OfflinePackageAcco ): Promise { assertOfflinePackagePrincipal(principal); return await this.sql.begin(async (tx) => { + await assertDataMutationAdmission(tx); await this.lockGlobal(tx); await this.lockPrincipal(tx, principal); await this.lockRequest(tx, candidate.request.requestKey); diff --git a/services/data-manager/src/ops-client.ts b/services/data-manager/src/ops-client.ts index 29498ea7f..b797bfb96 100644 --- a/services/data-manager/src/ops-client.ts +++ b/services/data-manager/src/ops-client.ts @@ -1,11 +1,22 @@ -import { createHash, randomBytes } from "node:crypto"; +import { randomBytes } from "node:crypto"; import { isAbsolute } from "node:path"; +import { setTimeout as waitForOpsStatus } from "node:timers/promises"; import { + createControlRootOperationKey, + normalizeExecutionAdmission, + OPS_KIND_POLICIES, OpsClient, type OpsOperation, type OpsOperationKind, type OpsResultFor, + opsOperationFingerprint, + parseControlRootOperationKey, + parseOpsResult, } from "@openmapx/core/ops"; +import { + checkpointLegacyDataOperation, + legacyDataMutation, +} from "./jobs/execution/legacy-mutation.js"; /** * Data-manager's client for the private operations agent. @@ -36,23 +47,8 @@ export function opsClient(): OpsClient { } /** Test seam: drop the memoized client so a new environment is picked up. */ -export function resetOpsClient(): void { - cached = undefined; -} - -/** - * A deterministic operation key derived from a durable identity, so a retry - * after a lost response reuses the same agent-side job instead of starting a - * second one. - */ -export function durableOpsKey(namespace: string, durableIdentity: string): string { - const digest = createHash("sha256") - .update("openmapx-ops-key-v1\0") - .update(namespace) - .update("\0") - .update(durableIdentity) - .digest("base64url"); - return `opk1_${digest.slice(0, 32)}`; +export function resetOpsClient(client?: OpsClient): void { + cached = client; } /** @@ -64,22 +60,128 @@ export async function runOpsOperation( options: { signal?: AbortSignal; operationKey?: string; pollIntervalMs?: number } = {}, ): Promise> { const client = opsClient(); - const operationKey = - options.operationKey ?? `opk1_${randomBytes(18).toString("base64url").slice(0, 32)}`; - const admission = await client.execute(operation, { operationKey, signal: options.signal }); - if (admission.execution === "sync") return admission.value as OpsResultFor; - - const pollIntervalMs = options.pollIntervalMs ?? 1_000; - for (;;) { - options.signal?.throwIfAborted(); - const status = await client.status(operation.kind, admission.operationId, { + if (OPS_KIND_POLICIES[operation.kind].execution === "sync") { + if (operation.kind === "transitousLock.propose") { + if (!legacyDataMutation()) + throw Error("A retained data mutation owner is required for a proposal"); + const admittedAt = Date.now(), + requestedDeadlineAt = options.operationKey + ? parseControlRootOperationKey(options.operationKey).notAfter + : admittedAt + OPS_KIND_POLICIES[operation.kind].timeoutMs, + operationKey = + options.operationKey ?? + createControlRootOperationKey({ notAfter: requestedDeadlineAt, nonce: randomBytes(16) }), + operationFingerprint = opsOperationFingerprint(operation), + normalized = normalizeExecutionAdmission({ + kind: operation.kind, + operationKey, + operationFingerprint, + admittedAt, + requestedDeadlineAt, + }), + retained = { + operationKey, + operation, + operationFingerprint, + ...normalized, + state: "attempted", + }; + await checkpointLegacyDataOperation(retained); + const accepted = await client.execute(operation, { + operationKey, + execution: { requestedDeadlineAt }, + signal: options.signal, + }); + if (accepted.execution !== "sync") throw Error("Unexpected asynchronous proposal"); + const result = parseOpsResult(operation.kind, accepted.value); + await checkpointLegacyDataOperation({ ...retained, state: "terminal", result }); + return result as OpsResultFor; + } + const admitted = await client.execute(operation, { + operationKey: options.operationKey ?? `opk1_${randomBytes(18).toString("base64url")}`, + signal: options.signal, + }); + if (admitted.execution !== "sync") throw Error("Unexpected async host effect"); + return parseOpsResult(operation.kind, admitted.value) as OpsResultFor; + } + if (!legacyDataMutation()) + throw Error("A retained data mutation must own asynchronous host effects"); + const admittedAt = Date.now(), + requestedDeadlineAt = admittedAt + OPS_KIND_POLICIES[operation.kind].timeoutMs, + operationKey = + options.operationKey ?? + createControlRootOperationKey({ notAfter: requestedDeadlineAt, nonce: randomBytes(16) }); + const normalized = normalizeExecutionAdmission({ + kind: operation.kind, + operationKey, + operationFingerprint: opsOperationFingerprint(operation), + admittedAt, + requestedDeadlineAt, + }); + let retained = { + operationKey, + operation, + operationFingerprint: opsOperationFingerprint(operation), + fingerprint: normalized.fingerprint, + admittedAt, + deadlineAt: normalized.deadlineAt, + state: "attempted", + operationId: undefined as string | undefined, + cancellationRequestedAt: null as string | null, + }; + await checkpointLegacyDataOperation(retained); + try { + const admitted = await client.execute(operation, { + operationKey, + execution: { requestedDeadlineAt }, signal: options.signal, }); - if (status.state === "succeeded") return status.result as OpsResultFor; - if (status.state === "failed" || status.state === "timed_out") { - // The agent's public error class only; no host detail crosses back. - throw new Error(`Operation ${operation.kind} did not succeed (${status.errorClass})`); + if (admitted.execution !== "async") throw Error("Unexpected synchronous host effect"); + retained = { ...retained, operationId: admitted.operationId, state: "observing" }; + await checkpointLegacyDataOperation(retained); + } catch { + /* One POST only. Resolve ambiguous admission through the exact retained key. */ + } + const settlement = AbortSignal.timeout(30000); + for (;;) { + if (options.signal?.aborted && retained.cancellationRequestedAt === null) { + retained = { ...retained, cancellationRequestedAt: new Date().toISOString() }; + await checkpointLegacyDataOperation(retained); + } + const signal = options.signal?.aborted ? settlement : options.signal; + const status = await client.lookup(operation, operationKey, { + fingerprint: normalized.fingerprint, + execution: { requestedDeadlineAt }, + signal, + }); + if ( + status.version !== 2 || + status.operationKey !== operationKey || + status.kind !== operation.kind || + status.fingerprint !== normalized.fingerprint || + status.deadlineAt !== normalized.deadlineAt || + (retained.operationId && status.operationId !== retained.operationId) + ) + throw Error("Retained host effect identity conflict"); + if (!retained.operationId) { + retained = { ...retained, operationId: status.operationId, state: "observing" }; + await checkpointLegacyDataOperation(retained); + } + if (["succeeded", "failed", "timed_out"].includes(status.state)) { + await checkpointLegacyDataOperation({ ...retained, state: "terminal", status }); + if (status.state === "succeeded") + return parseOpsResult(operation.kind, status.result) as OpsResultFor; + throw Error(`Operation ${operation.kind} did not succeed (${status.errorClass})`); } - await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + if (options.signal?.aborted) + try { + await client.cancel(status.operationId, { + fingerprint: normalized.fingerprint, + signal: settlement, + }); + } catch { + /* Lost cancel response is observation-only. */ + } + await waitForOpsStatus(options.pollIntervalMs ?? 1000, undefined, { signal }); } } diff --git a/services/data-manager/src/service.ts b/services/data-manager/src/service.ts index d0f2fe71f..b0eb6d61b 100644 --- a/services/data-manager/src/service.ts +++ b/services/data-manager/src/service.ts @@ -1,12 +1,25 @@ import { accessSync, constants } from "node:fs"; import { join } from "node:path"; -import { loadControlStartup } from "@openmapx/core/build-artifacts"; +import { setTimeout as waitForDataStatus } from "node:timers/promises"; +import { + createDataExecutionControlJournal, + loadControlStartup, +} from "@openmapx/core/build-artifacts"; +import { isDataExecutionTerminal } from "@openmapx/core/data-manager-contracts"; import { createFatalProcessHandler } from "@openmapx/core/server"; import Fastify from "fastify"; import { registerApi } from "./api.js"; import { registerAuth, resolveAuthToken } from "./auth.js"; import { awaitInflightSync, type CronHandles, setupCron } from "./cron.js"; -import { sql } from "./db/index.js"; +import { rawSql, sql } from "./db/index.js"; +import { installDataMutationAdmission } from "./jobs/execution/admission.js"; +import type { DataExecutionReceiver } from "./jobs/execution/api.js"; +import { createDataConfigurationMaintenance } from "./jobs/execution/configuration-maintenance.js"; +import { createDataExecutionDispatcher } from "./jobs/execution/dispatch.js"; +import { applyOwnedGenerationHardlinks } from "./jobs/execution/hardlinks.js"; +import { createDataExecutionReceiver } from "./jobs/execution/receiver.js"; +import { createDataExecutionRuntime } from "./jobs/execution/runtime.js"; +import { createPostgresDataExecutionStore } from "./jobs/execution/store.js"; import { type NotablePlacesSchedule, scheduleNotablePlaces, @@ -28,12 +41,14 @@ import { initializeRequiredSubsystems } from "./startup.js"; import { StateStore } from "./state.js"; const app = Fastify({ loggerInstance: rootLogger }); -registerAuth(app, resolveAuthToken(app)); +let executionReceiver: DataExecutionReceiver | undefined; +registerAuth(app, resolveAuthToken(app), { executionReceiver: () => executionReceiver }); const readiness = new DataManagerReadiness(); const controlStartup = loadControlStartup({ infraDir: "/data", role: "data-manager", container: true, + serviceIds: ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], }); if (controlStartup.reason) app.log.warn(controlStartup.reason); @@ -65,7 +80,43 @@ const notablePlaces = { operationLock: createNotablePlacesOperationLock(sql), }; +const dataState = new StateStore(dataDir); +const executionJournal = controlStartup.installation + ? createDataExecutionControlJournal(controlStartup.installation, rawSql) + : undefined; +const configurationFence = controlStartup.installation + ? createDataConfigurationMaintenance(rawSql, controlStartup.installation) + : undefined; +if (configurationFence) installDataMutationAdmission(configurationFence); +const executionStore = createPostgresDataExecutionStore(rawSql, { + controlJournal: executionJournal, + configurationFence, +}); +const dispatcher = createDataExecutionDispatcher({ + dataDir, + state: dataState, + sql, + singleFlight, + operationsPolicy, + runHardlinks: applyOwnedGenerationHardlinks, + notablePlaces, +}); +const executionRuntime = createDataExecutionRuntime({ + store: executionStore, + dispatch: dispatcher.dispatch, +}); +if (controlStartup.installation && configurationFence) + executionReceiver = createDataExecutionReceiver({ + runtime: executionRuntime, + installation: controlStartup.installation, + configurationFence, + dispatcher, + operationsPolicy, + }); + registerApi(app, { + executionReceiver, + searchIndexRuntimeState: dispatcher.searchIndexRuntimeState, dataDir, offlinePackages, singleFlight, @@ -133,7 +184,16 @@ let cronHandles: CronHandles | null = null; let notablePlacesSchedule: NotablePlacesSchedule | null = null; async function start(): Promise { - const store = new StateStore(dataDir); + const store = dataState; + try { + await configurationFence?.startup(); + await executionRuntime.reconcile(); + } catch (err) { + app.log.error( + { err }, + "Data mutation admission remains closed pending retained ownership recovery", + ); + } const countries = operationsPolicy.countries; const initialized = await initializeRequiredSubsystems({ readiness, @@ -145,6 +205,7 @@ async function start(): Promise { setupCronSchedulers: () => setupCron({ dataDir, + executionReceiver, countries, operationsPolicy, store, @@ -184,7 +245,29 @@ async function start(): Promise { // These maintenance operations already contain/log their own failures and // do not affect whether the registered HTTP and scheduled work is safe. void cronHandles.runTrafficExtractStartupNow(); - notablePlacesSchedule = scheduleNotablePlaces({ sql, logger: app.log, ...notablePlaces }); + notablePlacesSchedule = scheduleNotablePlaces({ + sql, + logger: app.log, + ...notablePlaces, + runOwned: async () => { + if (!executionReceiver) throw Error("Durable data receiver is unavailable"); + const admitted = await executionReceiver.admitLocal( + { kind: "notable-places-build" }, + { triggeredBy: "bootstrap" }, + ); + for (;;) { + const status = await executionReceiver.inspect(admitted.id); + if (!status) throw Error("Notable build observation remains unknown"); + if (isDataExecutionTerminal(status)) { + if (status.status !== "ok") throw Error("Notable build did not complete"); + return; + } + if (Date.now() >= admitted.deadlineAt) + throw Error("Notable build observation remains pending"); + await waitForDataStatus(250); + } + }, + }); void notablePlacesSchedule.buildIfMissing(); if ((process.env.OVERTURE_ENABLED || "").trim().toLowerCase() === "true") { void cronHandles.runOvertureConflationRetryNow(); @@ -211,6 +294,7 @@ async function shutdown(signal: NodeJS.Signals): Promise { cronHandles?.stop(); notablePlacesSchedule?.stop(); await offlinePackages.close(); + await executionRuntime.close(); await cronHandles?.closeTrafficWriter(); const result = await awaitInflightSync(singleFlight, 30_000); if (result === "timeout") { From d8b96d7e65dc409e01aad887525afb9b11242a66 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Oct 2026 23:37:33 +0100 Subject: [PATCH 020/135] fix(data): preserve retries and cancel orphaned children --- .../test-fixtures/data-owned-http-process.ts | 13 +- .../build-artifacts/data-execution-journal.ts | 24 + scripts/database-test-groups.json | 1 + services/data-manager/__tests__/api.test.ts | 14 +- services/data-manager/__tests__/cron.test.ts | 3 + services/data-manager/src/api.ts | 104 ++-- .../data-manager/src/jobs/execution/api.ts | 2 +- .../legacy-admission-postgres.test.ts | 538 ++++++++++++++++++ .../src/jobs/execution/legacy-mutation.ts | 61 +- .../execution/runner-cancellation.test.ts | 310 +++++++++- .../src/jobs/execution/runtime.test.ts | 160 ++++++ .../src/jobs/execution/runtime.ts | 175 ++++-- .../data-manager/src/jobs/execution/store.ts | 57 +- .../test-fixtures/dm-recovery-process.ts | 106 ++++ .../src/jobs/execution/worker-scope.ts | 56 +- .../src/jobs/transitous/catalog-bump.ts | 8 + .../src/jobs/transitous/script-runner.ts | 99 +++- services/data-manager/src/service.ts | 2 + 18 files changed, 1580 insertions(+), 153 deletions(-) create mode 100644 services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts create mode 100644 services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts diff --git a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts index 67d258e60..3315f3e13 100644 --- a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts +++ b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts @@ -19,11 +19,12 @@ const lines = createInterface({ input: process.stdin }); const messages = lines[Symbol.asyncIterator](); const initial = await messages.next(); if (initial.done) throw Error("Missing fixed fixture startup"); -const { root, token, validationFailure, crash } = JSON.parse(initial.value) as { +const { root, token, validationFailure, crash, loseCancelResponse } = JSON.parse(initial.value) as { root: string; token: string; validationFailure?: boolean | "twice"; crash?: "before-proof" | "before-terminal" | "after-terminal"; + loseCancelResponse?: boolean; }; const installation = openControlInstallation({ infraDir: root, @@ -122,6 +123,16 @@ const app = buildOpsAgentServer({ ), }), }); +let cancels = 0; +if (loseCancelResponse) + app.addHook("onSend", async (request, reply, payload) => { + if (request.method === "DELETE") { + cancels++; + send({ type: "cancel-response", cancels }); + if (cancels === 1) reply.raw.destroy(); + } + return payload; + }); await app.listen({ host: "127.0.0.1", port: 0 }); const address = app.server.address(); if (!address || typeof address === "string") throw Error("Missing actual agent address"); diff --git a/packages/core/src/build-artifacts/data-execution-journal.ts b/packages/core/src/build-artifacts/data-execution-journal.ts index 8c8480c08..9a36a0e0e 100644 --- a/packages/core/src/build-artifacts/data-execution-journal.ts +++ b/packages/core/src/build-artifacts/data-execution-journal.ts @@ -556,6 +556,30 @@ export function createDataExecutionControlJournal( s.origin, ); }, + pendingRunnerAttempts(authority: VerifiedExecutionAuthority) { + const state = executionAuthorityState(authority); + if (state.installation !== local || state.kind !== "transit-sync" || !state.origin) + throw Error("Original transit worker required"); + try { + const saved = verifyControl( + installation.publicKey, + "openmapx-data-runner-journal-v1", + readPrivateRecord( + local, + "observations", + `runners-${state.execution.operationKey}.json`, + 65536, + ), + runnerJournalSchema, + ); + if (controlDigest(saved.execution) !== controlDigest(state.execution)) + throw Error("Original runner owner changed"); + return saved.attempts; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return []; + } + }, retainRunnerAttempt(authority: VerifiedExecutionAuthority, request: unknown): void { const state = executionAuthorityState(authority), body = transitousRunnerRequestSchema.parse(request); diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index de6224b47..2e0a769b7 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -27,6 +27,7 @@ "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", "services/data-manager/src/jobs/execution/store-postgres.test.ts", + "services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts", "services/data-manager/src/jobs/execution/runtime.test.ts", "services/data-manager/src/jobs/execution/fixed-cli.test.ts", "services/data-manager/src/jobs/execution/runner-cancellation.test.ts", diff --git a/services/data-manager/__tests__/api.test.ts b/services/data-manager/__tests__/api.test.ts index e79b3e773..52c4c6d12 100644 --- a/services/data-manager/__tests__/api.test.ts +++ b/services/data-manager/__tests__/api.test.ts @@ -1,4 +1,7 @@ vi.mock("../src/jobs/execution/legacy-mutation.js", () => ({ + beginLegacyCatalogPreflight: async () => {}, + rejectLegacyMissingCatalog: async () => {}, + markLegacyCatalogFetchAttempt: async () => {}, startLegacyDataMutation: async (_kind: string, run: () => Promise) => ({ id: "unit-writer", completion: run(), @@ -517,12 +520,19 @@ describe("data-manager API", () => { const res = await app.inject({ method: "POST", url: "/transit/sync", - payload: { countries: ["DE"] }, + payload: { + countries: ["DE"], + idempotencyKey: "legacy-test-key", + triggeredBy: "api:synthetic-user", + }, }); expect(res.statusCode).toBe(202); expect(res.json()).toEqual({ ok: true, jobId: dataReceipt().id, status: "started" }); - expect(admitLocal).toHaveBeenCalledWith({ kind: "transit-sync", countries: ["DE"] }); + expect(admitLocal).toHaveBeenCalledWith( + { kind: "transit-sync", countries: ["DE"] }, + { legacyTransitKey: "legacy-test-key", triggeredBy: "api:synthetic-user" }, + ); }); it.each(["../etc", "Europe/Germany"])( diff --git a/services/data-manager/__tests__/cron.test.ts b/services/data-manager/__tests__/cron.test.ts index 87fad5208..94a2a569c 100644 --- a/services/data-manager/__tests__/cron.test.ts +++ b/services/data-manager/__tests__/cron.test.ts @@ -76,6 +76,9 @@ import { } from "../src/jobs/transitous/single-flight.js"; vi.mock("../src/jobs/execution/legacy-mutation.js", () => ({ + beginLegacyCatalogPreflight: async () => {}, + rejectLegacyMissingCatalog: async () => {}, + markLegacyCatalogFetchAttempt: async () => {}, withLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), independentLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), })); diff --git a/services/data-manager/src/api.ts b/services/data-manager/src/api.ts index d003d5140..8050f4eeb 100644 --- a/services/data-manager/src/api.ts +++ b/services/data-manager/src/api.ts @@ -23,6 +23,7 @@ import { startLegacyDataMutation, withLegacyDataMutation, } from "./jobs/execution/legacy-mutation.js"; +import { LegacyTransitDuplicate } from "./jobs/execution/store.js"; import { createNotablePlacesRuntimeState, getNotablePlacesStatus, @@ -429,10 +430,13 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { if (opts.offlinePackages) registerOfflinePackageRoutes(app, opts.offlinePackages); - async function admitLegacyData(input: DataExecutionInput) { + async function admitLegacyData( + input: DataExecutionInput, + source?: { triggeredBy?: string; legacyTransitKey?: string }, + ) { if (!opts.executionReceiver) throw Object.assign(Error("Durable data receiver is unavailable"), { statusCode: 503 }); - return opts.executionReceiver.admitLocal(input); + return opts.executionReceiver.admitLocal(input, source); } async function observeLegacyData( status: DataExecutionStatus, @@ -913,11 +917,24 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { // the single-flight lock + idempotency key. apps/api proxies user-facing // sync requests here. registerDataManagerRoute(app, "syncTransit", async (req, reply) => { - const admitted = await admitLegacyData({ - kind: "transit-sync", - countries: req.body?.countries ?? operationsPolicy.countries, - }); - return reply.code(202).send({ ok: true, jobId: admitted.id, status: "started" }); + try { + const admitted = await admitLegacyData( + { + kind: "transit-sync", + countries: req.body?.countries ?? operationsPolicy.countries, + }, + { triggeredBy: req.body?.triggeredBy, legacyTransitKey: req.body?.idempotencyKey }, + ); + return reply.code(202).send({ ok: true, jobId: admitted.id, status: "started" }); + } catch (error) { + if (error instanceof LegacyTransitDuplicate) + return reply.code(409).send({ + ok: false, + reason: "duplicate-idempotency-key", + existingJobId: error.existingJobId, + }); + throw error; + } }); // POST /transit/restart-motis — bounce the primary MOTIS container. Used @@ -945,9 +962,40 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { const catalogDir = join(dataDir, ".transitous-catalog"); - let candidate: Awaited>; try { - candidate = await resolveCatalogBumpCandidate({ catalogDir, branch }); + return await withLegacyDataMutation("transit-proposal", async () => { + const candidate = await resolveCatalogBumpCandidate({ catalogDir, branch }); + const { active: existing } = await runOpsOperation({ kind: "transitousLock.inspect" }); + if (candidateMatchesLock(candidate, existing) && !force) + return { + ok: true, + unchanged: true, + ref: candidate.ref, + previousRef: existing?.ref ?? null, + }; + const lock = lockFromCandidate( + candidate, + lockedBy, + "Pinned commit of public-transport/transitous consumed by services/data-manager. Bumped via POST /transit/bump.", + ); + await runOpsOperation({ + kind: "transitousLock.propose", + ref: lock.ref, + submodules: lock.submodules, + lockedBy: lock.lockedBy, + ...(lock.comment ? { comment: lock.comment } : {}), + }); + return { + ok: true, + unchanged: false, + proposed: true, + ref: lock.ref, + previousRef: existing?.ref ?? null, + submoduleSha: candidate.transitlandAtlasSha, + lockedAt: lock.lockedAt, + lockedBy: lock.lockedBy, + }; + }); } catch (err) { if (err instanceof CatalogBumpError) { const status = @@ -957,44 +1005,6 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { } throw err; } - - const { active: existing } = await runOpsOperation({ kind: "transitousLock.inspect" }); - if (candidateMatchesLock(candidate, existing) && !force) { - return { - ok: true, - unchanged: true, - ref: candidate.ref, - previousRef: existing?.ref ?? null, - }; - } - - const lock = lockFromCandidate( - candidate, - lockedBy, - "Pinned commit of public-transport/transitous consumed by services/data-manager. Bumped via POST /transit/bump.", - ); - // The proposal file lives under the repository's `infra/docker/`, which only - // the operations agent may write. - await withLegacyDataMutation("transit-proposal", () => - runOpsOperation({ - kind: "transitousLock.propose", - ref: lock.ref, - submodules: lock.submodules, - lockedBy: lock.lockedBy, - ...(lock.comment ? { comment: lock.comment } : {}), - }), - ); - - return { - ok: true, - unchanged: false, - proposed: true, - ref: lock.ref, - previousRef: existing?.ref ?? null, - submoduleSha: candidate.transitlandAtlasSha, - lockedAt: lock.lockedAt, - lockedBy: lock.lockedBy, - }; }); registerDataManagerRoute(app, "approveTransitBump", async (_req, reply) => { diff --git a/services/data-manager/src/jobs/execution/api.ts b/services/data-manager/src/jobs/execution/api.ts index 1578e5f0b..6f3704f13 100644 --- a/services/data-manager/src/jobs/execution/api.ts +++ b/services/data-manager/src/jobs/execution/api.ts @@ -16,7 +16,7 @@ import { registerDataManagerRoute } from "../../route-contract.js"; export interface DataExecutionReceiver { admitLocal( input: DataExecutionInput, - options?: { sourceJobId?: string; triggeredBy?: "cron" | "bootstrap" }, + options?: { sourceJobId?: string; triggeredBy?: string; legacyTransitKey?: string }, ): Promise; admitMaintenance( action: "restart-primary" | "reload-feed-proxy", diff --git a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts new file mode 100644 index 000000000..69eecf8d9 --- /dev/null +++ b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts @@ -0,0 +1,538 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { OpsClient } from "@openmapx/core/ops"; +import { afterAll, beforeAll, expect, it } from "vitest"; +import { setupCron } from "../../cron"; +import { db, rawSql } from "../../db"; +import { resetOpsClient } from "../../ops-client"; +import { StateStore } from "../../state"; +import { createSingleFlightController } from "../transitous/single-flight"; +import { installDataMutationAdmission } from "./admission"; +import { createDataConfigurationMaintenance } from "./configuration-maintenance"; +import { reconcileLegacyCatalogPreflightRejections } from "./legacy-mutation"; + +const enabled = + Boolean(process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL) || + process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"; +let root: string, + installation: control.ControlInstallation, + fence: ReturnType; +beforeAll(async () => { + if (!enabled) return; + if ( + !process.env.DATABASE_URL || + process.env.DATABASE_URL !== process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL + ) + throw Error("Explicit production adapter must use the verified owned synthetic DB"); + root = mkdtempSync(join(tmpdir(), "omx-fix1-legacy-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-staging", "motis-feed-proxy"], + }); + fence = createDataConfigurationMaintenance(rawSql, installation); + await fence.startup(); + installDataMutationAdmission(fence); +}); +afterAll(async () => { + if (!enabled) return; + control.closeControlInstallation(installation); + await rawSql.end(); + rmSync(root, { recursive: true, force: true }); +}); +it.skipIf(!enabled)( + "I4 records actual missing-catalog preflight as terminal error and permits the first subsequent admission", + async () => { + const before = new Set( + (await rawSql`SELECT id FROM data_manager.jobs`).map((row) => String(row.id)), + ); + const cron = setupCron({ + dataDir: join(root, "data"), + countries: ["de"], + store: new StateStore(join(root, "data")), + singleFlight: createSingleFlightController({ db, admissionSql: rawSql }), + logger: { info: () => {}, warn: () => {}, error: () => {} }, + syncCronExpression: "disabled", + feedProxyReloadCronExpression: "disabled", + stalenessCheckCronExpression: "disabled", + autoBumpCronExpression: "disabled", + trafficExtractCronExpression: "disabled", + trafficLiveCronExpression: "disabled", + trafficPredictedCronExpression: "disabled", + overtureCronExpression: "disabled", + overtureConflationRetryCronExpression: "disabled", + }); + const owned: string[] = []; + try { + await expect(cron.runAutoBumpNow()).rejects.toMatchObject({ code: "catalog-not-cloned" }); + const rows = ( + await rawSql`SELECT id,status,finished_at,metadata FROM data_manager.jobs WHERE kind='transit-proposal'` + ).filter((row) => !before.has(String(row.id))); + owned.push(...rows.map((row) => String(row.id))); + expect(rows).toHaveLength(1); + expect(rows[0]?.status).toBe("error"); + expect(control.retainedDataDatabaseTimestamp(rows[0]?.finished_at).getTime()).toBeGreaterThan( + 0, + ); + // Crash after the fixed no-effect fact, before terminal SQL publication. + await rawSql`UPDATE data_manager.jobs SET status='interrupted',finished_at=NULL WHERE id=${rows[0]!.id}`; + await reconcileLegacyCatalogPreflightRejections(rawSql); + const [repaired] = + await rawSql`SELECT status,finished_at FROM data_manager.jobs WHERE id=${rows[0]!.id}`; + expect(repaired?.status).toBe("error"); + expect(repaired?.finished_at).not.toBeNull(); + await expect(fence.withAdmission(async () => "first sync may admit")).resolves.toBe( + "first sync may admit", + ); + const subsequent = await legacyFixture(); + try { + const sync = await subsequent.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { countries: ["de"], triggeredBy: "bootstrap" }, + }); + expect(sync.statusCode, sync.body).toBe(202); + await expect + .poll(async () => (await subsequent.store.inspect(sync.json().jobId))?.status.status, { + timeout: 15000, + }) + .toBe("ok"); + } finally { + await subsequent.close(); + } + } finally { + cron.stop(); + if (!owned.length) + owned.push( + ...(await rawSql`SELECT id FROM data_manager.jobs`) + .map((row) => String(row.id)) + .filter((id) => !before.has(id)), + ); + for (const id of owned) await rawSql`DELETE FROM data_manager.jobs WHERE id=${id}`; + } + }, + 30000, +); + +function proposalCron(overrides: Partial[0]> = {}) { + return setupCron({ + dataDir: join(root, "data"), + countries: ["de"], + store: new StateStore(join(root, "data")), + singleFlight: createSingleFlightController({ db, admissionSql: rawSql }), + logger: { info: () => {}, warn: () => {}, error: () => {} }, + syncCronExpression: "disabled", + feedProxyReloadCronExpression: "disabled", + stalenessCheckCronExpression: "disabled", + autoBumpCronExpression: "disabled", + trafficExtractCronExpression: "disabled", + trafficLiveCronExpression: "disabled", + trafficPredictedCronExpression: "disabled", + overtureCronExpression: "disabled", + overtureConflationRetryCronExpression: "disabled", + ...overrides, + }); +} +it.skipIf(!enabled).each(["git", "proposal"] as const)( + "I4 retains the original %s attempt as unknown and blocks admission after recovery", + async (phase) => { + const before = new Set( + (await rawSql`SELECT id FROM data_manager.jobs`).map((row) => String(row.id)), + ); + const catalog = join(root, "data/.transitous-catalog"); + let posts = 0; + if (phase === "git") + mkdirSync(join(catalog, ".git"), { recursive: true }); // Actual git fails locally; no upstream. + else { + const tokenFile = join(root, "proposal-token"); + writeFileSync(tokenFile, "p".repeat(43), { mode: 0o600 }); + resetOpsClient( + new OpsClient({ + baseUrl: "http://127.0.0.1:4300", + tokenFile, + fetch: async (_url, options) => { + if (options?.method === "GET") + return Response.json({ + version: 1, + requestId: new Headers(options.headers).get("x-ops-request-id"), + ok: true, + result: { versions: [1, 2] }, + }); + const request = JSON.parse(String(options?.body)); + if (request.operation.kind === "transitousLock.inspect") + return Response.json({ + version: 1, + requestId: request.requestId, + ok: true, + result: { + execution: "sync", + kind: request.operation.kind, + value: { active: null, proposed: null }, + }, + }); + posts++; + throw Error("Synthetic proposal response lost after attempted POST"); + }, + }), + ); + } + const cron = proposalCron( + phase === "proposal" + ? { + resolveBumpCandidate: async () => ({ + branch: "main", + ref: "main@" + "a".repeat(40), + transitousSha: "a".repeat(40), + transitlandAtlasSha: "b".repeat(40), + }), + } + : {}, + ); + try { + await expect(cron.runAutoBumpNow()).rejects.toThrow(); + const rows = ( + await rawSql`SELECT * FROM data_manager.jobs WHERE kind='transit-proposal'` + ).filter((row) => !before.has(String(row.id))); + expect(rows).toHaveLength(1); + const row = rows[0]!; + expect(row.status).toBe("interrupted"); + expect(row.finished_at).toBeNull(); + if (phase === "git") expect(row.metadata.catalogPreflight.state).toBe("git-fetch-attempted"); + else { + expect(posts).toBe(1); + expect(row.metadata.retainedOps).toHaveLength(1); + expect(row.metadata.retainedOps[0].state).toBe("attempted"); + expect(row.metadata.retainedOps[0].operationKey).toMatch(/^opk1_/); + expect(row.metadata.retainedOps[0].fingerprint).toMatch(/^[a-f0-9]{64}$/); + } + await reconcileLegacyCatalogPreflightRejections(rawSql); + const [after] = await rawSql`SELECT * FROM data_manager.jobs WHERE id=${row.id}`; + expect(after?.metadata).toEqual(row.metadata); + expect(after?.finished_at).toBeNull(); + await expect(fence.withAdmission(async () => "must not admit")).rejects.toMatchObject({ + statusCode: 409, + }); + } finally { + cron.stop(); + resetOpsClient(); + rmSync(catalog, { recursive: true, force: true }); + const ids = (await rawSql`SELECT id FROM data_manager.jobs`) + .map((row) => String(row.id)) + .filter((id) => !before.has(id)); + for (const id of ids) await rawSql`DELETE FROM data_manager.jobs WHERE id=${id}`; + } + }, + 30000, +); + +import { randomUUID } from "node:crypto"; +import Fastify from "fastify"; +import { registerApi } from "../../api"; +import { registerAuth } from "../../auth"; +import { resolveOperationsProfile } from "../transitous/operations-profile"; +import { createDataExecutionDispatcher } from "./dispatch"; +import { createDataExecutionReceiver } from "./receiver"; +import { createDataExecutionRuntime } from "./runtime"; +import { createPostgresDataExecutionStore } from "./store"; + +async function legacyFixture(held = false) { + const journal = control.createDataExecutionControlJournal(installation, rawSql); + const store = createPostgresDataExecutionStore(rawSql, { + controlJournal: journal, + configurationFence: fence, + }); + let dispatches = 0, + release = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + const runtime = createDataExecutionRuntime({ + store, + dispatch: async (input) => { + dispatches++; + if (held) await gate; + if (input.kind !== "transit-sync") throw Error("This admission fixture selects transit only"); + return { + kind: input.kind, + finalStatus: "ok", + sourceRevision: "synthetic-public", + countries: input.countries, + }; + }, + }); + const singleFlight = createSingleFlightController({ db, admissionSql: rawSql }), + policy = resolveOperationsProfile({ countries: ["de", "ch"] }); + const dispatcher = createDataExecutionDispatcher({ + dataDir: join(root, "data"), + state: new StateStore(join(root, "data")), + sql: rawSql, + singleFlight, + operationsPolicy: policy, + runHardlinks: async () => { + throw Error("No hardlink operation"); + }, + }); + const receiver = createDataExecutionReceiver({ + runtime, + installation, + configurationFence: fence, + dispatcher, + operationsPolicy: policy, + }); + const app = Fastify(); + registerAuth(app, "fix1-synthetic-token", { executionReceiver: () => receiver }); + registerApi(app, { + dataDir: join(root, "data"), + singleFlight, + operationsPolicy: policy, + executionReceiver: receiver, + searchIndexSql: rawSql, + }); + const before = new Set( + (await rawSql`SELECT id FROM data_manager.jobs`).map((row) => String(row.id)), + ); + return { + app, + runtime, + receiver, + store, + release, + dispatches: () => dispatches, + async close() { + release(); + await runtime.close(); + app.server.closeAllConnections(); + await app.close(); + journal.close(); + const created = (await rawSql`SELECT id FROM data_manager.jobs`) + .map((row) => String(row.id)) + .filter((id) => !before.has(id)); + for (const id of created) await rawSql`DELETE FROM data_manager.jobs WHERE id=${id}`; + }, + }; +} +const headers = { authorization: "Bearer fix1-synthetic-token" }; +it.skipIf(!enabled)( + "I1 preserves the documented duplicate response for an original pre-upgrade job", + async () => { + const f = await legacyFixture(), + key = "fix1-v1-" + randomUUID(), + id = randomUUID(); + try { + await rawSql`INSERT INTO data_manager.jobs(id,kind,status,finished_at,idempotency_key) VALUES(${id},'transitous-sync','ok',now(),${key})`; + const response = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { countries: ["de"], idempotencyKey: key }, + }); + expect(response.statusCode).toBe(409); + expect(response.json()).toEqual({ + ok: false, + reason: "duplicate-idempotency-key", + existingJobId: id, + }); + expect(f.dispatches()).toBe(0); + } finally { + await f.close(); + } + }, +); +it.skipIf(!enabled)( + "I1 observes the original completed transit identity and actor after keyed retries", + async () => { + const f = await legacyFixture(); + try { + const body = { + countries: ["de"], + idempotencyKey: "fix1-completed-" + randomUUID(), + triggeredBy: "api:user:synthetic-admin", + }; + const first = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: body, + }); + expect(first.statusCode, first.body).toBe(202); + const id = first.json().jobId as string; + await expect + .poll(async () => (await f.store.inspect(id))?.status.status, { timeout: 15000 }) + .toBe("ok"); + const original = await f.store.inspect(id); + const retry = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: body, + }); + expect(retry.statusCode, retry.body).toBe(202); + expect(retry.json().jobId).toBe(id); + expect(f.dispatches()).toBe(1); + const changedActor = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { ...body, triggeredBy: "other:actor" }, + }); + expect(changedActor.statusCode).toBe(409); + expect((await f.store.inspect(id))?.execution).toEqual(original?.execution); + expect((await f.store.inspect(id))?.triggeredBy).toBe(body.triggeredBy); + const changed = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { ...body, countries: ["ch"] }, + }); + expect(changed.statusCode).toBe(409); + expect(f.dispatches()).toBe(1); + } finally { + await f.close(); + } + }, + 45000, +); +it.skipIf(!enabled)( + "I1 serializes simultaneous first admission, keeps no-key calls distinct and expires only completed legacy aliases after 24 hours", + async () => { + const f = await legacyFixture(true); + try { + const body = { + countries: ["de"], + idempotencyKey: "fix1-race-" + randomUUID(), + triggeredBy: "synthetic:actor", + }; + const responses = await Promise.all( + [1, 2].map(() => + f.app.inject({ method: "POST", url: "/transit/sync", headers, payload: body }), + ), + ); + expect(responses.map((r) => r.statusCode)).toEqual([202, 202]); + const id = responses[0]!.json().jobId as string; + expect(responses[1]!.json().jobId).toBe(id); + f.release(); + await expect + .poll(async () => (await f.store.inspect(id))?.status.status, { timeout: 15000 }) + .toBe("ok"); + await rawSql`UPDATE data_manager.jobs SET started_at=now()-interval '23 hours 59 minutes' WHERE id=${id}`; + expect( + ( + await f.app.inject({ method: "POST", url: "/transit/sync", headers, payload: body }) + ).json().jobId, + ).toBe(id); + await rawSql`UPDATE data_manager.jobs SET started_at=now()-interval '24 hours 1 minute' WHERE id=${id}`; + const afterWindow = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: body, + }); + expect(afterWindow.statusCode, afterWindow.body).toBe(202); + expect(afterWindow.json().jobId).not.toBe(id); + await expect + .poll(async () => (await f.store.inspect(afterWindow.json().jobId))?.status.status, { + timeout: 15000, + }) + .toBe("ok"); + const noKeyIds: string[] = []; + for (let i = 0; i < 2; i++) { + const response = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { countries: ["de"], triggeredBy: "synthetic:no-key" }, + }); + expect(response.statusCode, response.body).toBe(202); + noKeyIds.push(response.json().jobId); + await expect + .poll(async () => (await f.store.inspect(response.json().jobId))?.status.status, { + timeout: 15000, + }) + .toBe("ok"); + } + expect(new Set(noKeyIds).size).toBe(2); + expect(f.dispatches()).toBe(4); + } finally { + await f.close(); + } + }, + 45000, +); +it.skipIf(!enabled)( + "I1 resolves a lost actual 202 and concurrent keyed retries without another dispatch", + async () => { + const f = await legacyFixture(true); + let drop = true; + f.app.addHook("onSend", async (request, reply, payload) => { + if ( + request.method === "POST" && + request.url === "/transit/sync" && + reply.statusCode === 202 && + drop + ) { + drop = false; + reply.raw.destroy(); + } + return payload; + }); + await f.app.listen({ host: "127.0.0.1", port: 0 }); + const address = f.app.server.address(); + if (!address || typeof address === "string") throw Error("Actual receiver endpoint missing"); + const base = `http://127.0.0.1:${address.port}`, + body = { + countries: ["de"], + idempotencyKey: "fix1-lost-" + randomUUID(), + triggeredBy: "manual:admin:synthetic", + }; + const send = () => + fetch(base + "/transit/sync", { + method: "POST", + headers: { ...headers, "content-type": "application/json" }, + body: JSON.stringify(body), + }); + try { + await expect(send()).rejects.toThrow(); + await expect.poll(f.dispatches, { timeout: 15000 }).toBe(1); + const original = (await f.store.listUnsettled()).find( + (row) => row.status.kind === "transit-sync", + ); + if (!original) throw Error("Original retained admission missing"); + const retries = await Promise.all([send(), send()]); + const replies = await Promise.all( + retries.map(async (response) => ({ code: response.status, body: await response.json() })), + ); + expect(replies).toEqual( + replies.map(() => ({ + code: 202, + body: { ok: true, jobId: original.status.id, status: "started" }, + })), + ); + expect(f.dispatches()).toBe(1); + const changedKey = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { ...body, idempotencyKey: body.idempotencyKey + "-other" }, + }); + expect(changedKey.statusCode).toBe(409); + expect((await f.store.inspect(original.status.id))?.execution).toEqual(original.execution); + f.release(); + await expect + .poll(async () => (await f.store.inspect(original.status.id))?.status.status, { + timeout: 15000, + }) + .toBe("ok"); + const afterCompletion = await send(); + expect((await afterCompletion.json()).jobId).toBe(original.status.id); + expect(f.dispatches()).toBe(1); + } finally { + await f.close(); + } + }, + 45000, +); diff --git a/services/data-manager/src/jobs/execution/legacy-mutation.ts b/services/data-manager/src/jobs/execution/legacy-mutation.ts index 8d717eed8..4def27361 100644 --- a/services/data-manager/src/jobs/execution/legacy-mutation.ts +++ b/services/data-manager/src/jobs/execution/legacy-mutation.ts @@ -1,4 +1,6 @@ import { AsyncLocalStorage } from "node:async_hooks"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; import type postgres from "postgres"; import { rawSql } from "../../db/index.js"; import { AmbientPublicationBusyError } from "../ambient-places/schema.js"; @@ -9,6 +11,8 @@ const current = new AsyncLocalStorage<{ version: number; operations: Record[]; database: postgres.Sql; + kind: LegacyDataMutationKind; + catalogPhase?: "checking" | "missing" | "git-fetch-attempted"; }>(); export function legacyDataMutation() { return current.getStore(); @@ -18,6 +22,8 @@ export async function checkpointLegacyDataOperation( ): Promise { const context = current.getStore(); if (!context) throw Error("Retained data mutation admission required"); + if (context.catalogPhase === "missing") + throw Error("Rejected catalog preflight cannot attempt effects"); const index = context.operations.findIndex( (entry) => entry.operationKey === operation.operationKey, ), @@ -39,6 +45,46 @@ export type LegacyDataMutationKind = | "transit-proposal" | "data-reload" | "ambient-publication"; +async function catalogPhase(next: "checking" | "missing" | "git-fetch-attempted") { + const context = current.getStore(); + if (!context) return; + if ( + context.kind !== "transit-proposal" || + context.operations.length || + (next === "checking" ? context.catalogPhase !== undefined : context.catalogPhase !== "checking") + ) + throw Error("Original unattempted catalog preflight required"); + const fact = { + version: 1, + state: next, + ...(next === "missing" ? { reason: "catalog-not-cloned", effects: "not_started" } : {}), + }; + const rows = + await context.database`UPDATE data_manager.jobs SET metadata=jsonb_set(metadata,'{catalogPreflight}',${JSON.stringify(fact)}::text::jsonb),ownership_version=ownership_version+1 WHERE id=${context.id} AND ownership_version=${context.version} AND kind='transit-proposal' AND finished_at IS NULL AND metadata->'retainedOps'='[]'::jsonb RETURNING ownership_version`; + if (rows.length !== 1) throw Error("Catalog phase owner CAS lost"); + context.version = Number(rows[0]!.ownership_version); + context.catalogPhase = next; +} +export function beginLegacyCatalogPreflight(): Promise { + return catalogPhase("checking"); +} +export async function rejectLegacyMissingCatalog(catalogDir: string): Promise { + if (existsSync(join(catalogDir, ".git"))) throw Error("Catalog preflight changed"); + await catalogPhase("missing"); +} +export function markLegacyCatalogFetchAttempt(): Promise { + return catalogPhase("git-fetch-attempted"); +} +/** The fixed missing-catalog producer recorded no-command/no-proposal before + * throwing. Only that exact unattempted fact may repair startup finality. */ +export async function reconcileLegacyCatalogPreflightRejections( + database: postgres.Sql = rawSql, +): Promise { + await database`UPDATE data_manager.jobs SET status='error',finished_at=coalesce(finished_at,now()),ownership_version=ownership_version+1 + WHERE execution_version IS NULL AND kind='transit-proposal' AND status IN ('running','interrupted') AND finished_at IS NULL + AND metadata->'retainedOps'='[]'::jsonb + AND metadata->'catalogPreflight'=${JSON.stringify({ version: 1, state: "missing", reason: "catalog-not-cloned", effects: "not_started" })}::text::jsonb`; +} /** Admission is awaited before returning202; completion retains its own actual * SQL owner and every nested host-operation checkpoint until containment. */ export async function startLegacyDataMutation( @@ -50,7 +96,13 @@ export async function startLegacyDataMutation( { kind, triggeredBy: "data-manager", metadata: { version: 1, retainedOps: [] } }, database, ); - const context = { id, version: 0, operations: [] as Record[], database }; + const context: NonNullable> = { + id, + version: 0, + operations: [], + database, + kind, + }; const finish = async (status: string) => { const rows = await database`UPDATE data_manager.jobs SET status=${status},finished_at=${new Date().toISOString()}::text::timestamptz WHERE id=${id} AND ownership_version=${context.version} AND execution_version IS NULL AND kind <> 'configuration-maintenance' AND finished_at IS NULL RETURNING id`; @@ -62,7 +114,12 @@ export async function startLegacyDataMutation( await finish("ok"); return value; } catch (error) { - if (kind === "ambient-publication" && error instanceof AmbientPublicationBusyError) { + if ( + (kind === "ambient-publication" && error instanceof AmbientPublicationBusyError) || + (kind === "transit-proposal" && + context.catalogPhase === "missing" && + context.operations.length === 0) + ) { await finish("error"); throw error; } diff --git a/services/data-manager/src/jobs/execution/runner-cancellation.test.ts b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts index b69c5fea8..3ad05acb7 100644 --- a/services/data-manager/src/jobs/execution/runner-cancellation.test.ts +++ b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts @@ -1,16 +1,318 @@ -import { randomBytes } from "node:crypto"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { randomBytes, randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; +import { createInterface } from "node:readline"; import * as control from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { build } from "esbuild"; import postgres from "postgres"; -import { expect, it } from "vitest"; +import { afterAll, beforeAll, expect, it } from "vitest"; import { buildTransitousRunnerServer } from "../../../../../apps/transitous-runner/src/server"; import { createTransitousScriptRunner } from "../transitous/script-runner"; import { createDataExecutionRuntime } from "./runtime"; import { createPostgresDataExecutionStore } from "./store"; +let recoveryBundleRoot: string, dmBundle: string, agentBundle: string; +beforeAll(async () => { + recoveryBundleRoot = mkdtempSync(join(tmpdir(), "omx-dm-crash-bundle-")); + dmBundle = join(recoveryBundleRoot, "dm.mjs"); + agentBundle = join(recoveryBundleRoot, "agent.mjs"); + for (const [entry, outfile, manifest] of [ + [ + "services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts", + dmBundle, + "services/data-manager/package.json", + ], + [ + "apps/ops-agent/src/test-fixtures/data-owned-http-process.ts", + agentBundle, + "apps/ops-agent/package.json", + ], + ]) + await build({ + entryPoints: [resolve(entry!)], + outfile, + bundle: true, + platform: "node", + format: "esm", + target: "node24", + banner: { + js: `import {createRequire as fixtureCreateRequire} from "node:module"; const require=fixtureCreateRequire(${JSON.stringify(resolve(manifest!))});`, + }, + }); +}); +afterAll(() => { + if (recoveryBundleRoot) rmSync(recoveryBundleRoot, { recursive: true, force: true }); +}); + +function processPeer(file: string, initial: unknown, env: NodeJS.ProcessEnv = process.env) { + const child: ChildProcessWithoutNullStreams = spawn(process.execPath, [file], { + env, + stdio: ["pipe", "pipe", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (data) => { + stderr = (stderr + String(data)).slice(-6000); + }); + const lines = createInterface({ input: child.stdout }), + iterator = lines[Symbol.asyncIterator](); + const exited = new Promise((resolve) => child.once("close", () => resolve())); + const send = (value: unknown) => child.stdin.write(JSON.stringify(value) + "\n"); + async function next(type: string): Promise> { + for (;;) { + const line = await Promise.race([ + iterator.next(), + new Promise((_, reject) => { + const timer = setTimeout(() => reject(Error(`Child ${type} timeout: ${stderr}`)), 15000); + timer.unref(); + }), + ]); + if (line.done) throw Error(`Child exited before ${type}: ${stderr}`); + const value = JSON.parse(line.value) as Record; + if (value.type === type) return value; + } + } + send(initial); + return { + child, + send, + next, + async kill() { + child.kill("SIGKILL"); + await exited; + lines.close(); + }, + }; +} + +it + .skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", + ) + .each(["agent", "python"] as const)( + "I2 cancels the original live %s effect after a separate-process DM crash without worker completion", + async (mode) => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Explicit owned DB required"); + const sql = postgres(url, { max: 4 }), + root = mkdtempSync(join(tmpdir(), "omx-dm-active-crash-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const token = "b".repeat(43), + tokenFile = join(root, "token"), + keyFile = join(root, "runner-key"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + writeFileSync(keyFile, "7".repeat(32), { mode: 0o600 }); + let agent: ReturnType | undefined, + dm: ReturnType | undefined; + let runner: ReturnType | undefined; + let id: string | undefined, + posts = 0, + cancels = 0, + inspections = 0; + let originalRunnerRequest: unknown; + const runnerControls: unknown[] = []; + const catalog = join(root, "catalog"), + ready = join(catalog, "ready"); + let endpoint = ""; + try { + if (mode === "agent") { + agent = processPeer(agentBundle, { root, token, loseCancelResponse: true }); + const peer = await agent.next("ready"); + endpoint = `http://127.0.0.1:${peer.port}`; + } else { + mkdirSync(join(catalog, "src"), { recursive: true }); + writeFileSync( + join(catalog, "src/garbage-collect.py"), + "import os,time\nfrom pathlib import Path\nPath('ready').write_text(str(os.getpid()))\nwhile not Path('release').exists(): time.sleep(0.05)\n", + ); + runner = buildTransitousRunnerServer({ + catalogDir: catalog, + stagingDir: root, + capabilityKey: Buffer.from("7".repeat(32)), + }); + runner.addHook("onRequest", async (request) => { + if (request.url === "/v1/run") posts++; + if (request.url === "/v1/run/inspect") inspections++; + }); + runner.addHook("preHandler", async (request) => { + if (request.url === "/v1/run") originalRunnerRequest = request.body; + else runnerControls.push(request.body); + }); + runner.addHook("onSend", async (request, reply, payload) => { + if (request.url === "/v1/run/cancel") { + cancels++; + if (cancels === 1) reply.raw.destroy(); + } + return payload; + }); + await runner.listen({ host: "127.0.0.1", port: 0 }); + const address = runner.server.address(); + if (!address || typeof address === "string") throw Error("Runner address missing"); + endpoint = `http://127.0.0.1:${address.port}`; + } + const env = { + ...process.env, + DATABASE_URL: url, + OPS_AGENT_URL: endpoint, + OPS_AGENT_TOKEN_FILE: tokenFile, + TRANSITOUS_RUNNER_URL: endpoint, + TRANSITOUS_RUNNER_CAPABILITY_KEY_FILE: keyFile, + }; + const start = async (replacement: boolean) => { + dm = processPeer(dmBundle, { root, token, mode, replacement }, env); + const state = await dm.next("ready"); + return `http://127.0.0.1:${state.port}`; + }; + let base = await start(false); + const headers = { authorization: `Bearer ${token}`, "content-type": "application/json" }; + const input = { + countries: ["de"], + idempotencyKey: "fix1-crash-" + randomUUID(), + triggeredBy: "synthetic:operator", + }; + if (mode === "agent") { + const controller = new AbortController(); + const observation = fetch(base + "/transit/restart-motis", { + method: "POST", + headers: { authorization: headers.authorization }, + signal: controller.signal, + }).catch((error) => error); + id = String((await dm!.next("dispatch")).id); + await agent!.next("effect"); + // The actual maintenance observer may disconnect while its child lives. + controller.abort(); + await observation; + agent!.send({ type: "probe" }); + expect((await agent!.next("probe")).state).toBe("running"); + } else { + const admitted = await fetch(base + "/transit/sync", { + method: "POST", + headers, + body: JSON.stringify(input), + }); + expect(admitted.status).toBe(202); + id = (await admitted.json()).jobId; + await expect.poll(() => existsSync(ready), { timeout: 15000 }).toBe(true); + } + if (!id) throw Error("Original admission missing"); + const [original] = await sql`SELECT * FROM data_manager.jobs WHERE id=${id}`; + if (!original) throw Error("Original row missing"); + const operationKey = String(original.operation_key), + lock = join(root, "data/.admin-target-control/locks/motis"); + const inode = statSync(lock).ino; + await dm!.kill(); + dm = undefined; + // SIGKILL occurred while dispatcher awaited the independent child. + const observations = join(root, "control-private/data-manager/observations"); + expect(existsSync(join(observations, `worker-${operationKey}.json`))).toBe(false); + base = await start(true); + if (mode === "python") { + const retry = await fetch(base + "/transit/sync", { + method: "POST", + headers, + body: JSON.stringify(input), + }); + expect(retry.status).toBe(202); + expect((await retry.json()).jobId).toBe(id); + } + const observer = await fetch(base + `/executions/${id}`, { headers }); + expect(observer.status).toBe(200); + await observer.body?.cancel(); + if (mode === "python") { + expect(cancels).toBe(0); + process.kill(-Number(readFileSync(ready, "utf8")), 0); + } else { + agent!.send({ type: "probe" }); + expect((await agent!.next("probe")).state).toBe("running"); + } + const canceled = await fetch(base + `/executions/${id}/cancel`, { + method: "POST", + headers, + body: JSON.stringify({ operationKey, fingerprint: original.execution_fingerprint }), + }); + expect(canceled.status, await canceled.text()).toBe(200); + if (mode === "agent") { + await agent!.next("aborted"); + expect((await agent!.next("cancel-response")).cancels).toBeGreaterThan(0); + } else { + await expect.poll(() => cancels, { timeout: 8000 }).toBeGreaterThan(0); + const pid = Number(readFileSync(ready, "utf8")); + await expect + .poll( + () => { + try { + process.kill(-pid, 0); + return true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === "ESRCH") return false; + throw e; + } + }, + { timeout: 8000 }, + ) + .toBe(false); + await expect.poll(() => inspections, { timeout: 8000 }).toBeGreaterThan(0); + expect(posts).toBe(1); + expect(runnerControls.length).toBeGreaterThan(0); + for (const request of runnerControls) expect(request).toEqual(originalRunnerRequest); + } + const [current] = await sql`SELECT * FROM data_manager.jobs WHERE id=${id}`; + expect(current?.finished_at).toBeNull(); + expect(current?.status).toBe("cancel_pending"); + expect(current?.execution_state.publication).toBe("unknown"); + for (const key of [ + "id", + "operation_key", + "execution_fingerprint", + "operation_fingerprint", + "deadline_at", + "execution_input", + "triggered_by", + ]) + expect(current?.[key]).toEqual(original[key]); + expect(statSync(lock).ino).toBe(inode); + expect(existsSync(join(observations, `worker-${operationKey}.json`))).toBe(false); + dm!.send({ type: "probe" }); + expect((await dm!.next("probe")).dispatches).toBe(0); + if (agent) { + agent.send({ type: "probe" }); + const probe = await agent.next("probe"); + expect(probe.effects).toBe(1); + expect(probe.forward).toBe(false); + expect(probe.operationId).toBe( + original.execution_children.find( + (child: { phase: string }) => child.phase === "recover-primary", + )?.proposedId, + ); + } + } finally { + if (dm) await dm.kill(); + if (agent) await agent.kill(); + if (runner) { + writeFileSync(join(catalog, "release"), "release"); + runner.server.closeAllConnections(); + await runner.close(); + } + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); + it.skipIf( !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", diff --git a/services/data-manager/src/jobs/execution/runtime.test.ts b/services/data-manager/src/jobs/execution/runtime.test.ts index 9bee92232..48931441d 100644 --- a/services/data-manager/src/jobs/execution/runtime.test.ts +++ b/services/data-manager/src/jobs/execution/runtime.test.ts @@ -616,3 +616,163 @@ describe.skipIf( } }, 60000); }); + +describe.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)("locked production Drizzle and postgres query cancellation", () => { + it.each([ + "mapped-held", + "mapped-aborted", + "values-then", + "values-catch", + "values-finally", + "raw-finally", + ] as const)( + "contains the actual mapped/fluent query (%s)", + async (mode) => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 8 }), + workerSql = withDataSqlCancellation(sql); + const { sql: expression } = await import("drizzle-orm"); + const { pgSchema, integer } = await import("drizzle-orm/pg-core"); + const db = drizzle(workerSql), + suffix = randomUUID().replaceAll("-", ""), + tableName = `fix1_query_${suffix}`, + fn = `fix1_hit_${suffix}`; + const table = pgSchema("data_manager").table(tableName, { + id: integer("id"), + hits: integer("hits"), + }); + await sql.unsafe( + `CREATE TABLE data_manager.${tableName}(id int PRIMARY KEY,hits int NOT NULL)`, + ); + await sql.unsafe(`INSERT INTO data_manager.${tableName} VALUES(1,0)`); + await sql.unsafe( + `CREATE FUNCTION data_manager.${fn}() RETURNS integer LANGUAGE plpgsql AS $$ BEGIN UPDATE data_manager.${tableName} SET hits=hits+1 WHERE id=1; RETURN 1; END $$`, + ); + const root = mkdtempSync(join(tmpdir(), "omx-fix1-mapped-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }); + const journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + let workerReady = () => {}, + letQueryStart = () => {}, + locked = () => {}, + unlock = () => {}; + const ready = new Promise((resolve) => { + workerReady = resolve; + }), + start = new Promise((resolve) => { + letQueryStart = resolve; + }); + const lockReady = new Promise((resolve) => { + locked = resolve; + }), + lockHeld = new Promise((resolve) => { + unlock = resolve; + }); + const locker = + mode === "mapped-held" + ? sql.begin(async (tx) => { + await tx.unsafe(`LOCK TABLE data_manager.${tableName} IN ACCESS EXCLUSIVE MODE`); + locked(); + await lockHeld; + }) + : Promise.resolve(); + if (mode === "mapped-held") await lockReady; + let querySettled = false, + queryRejected = false, + fluentIdentity = true, + finalized = 0; + const runtime = createDataExecutionRuntime({ + store, + dispatch: async (_input, context) => { + workerReady(); + await start; + try { + if (mode.startsWith("mapped")) { + await db + .select({ seen: expression`data_manager.${expression.identifier(fn)}()` }) + .from(table); + } else { + const query = workerSql.unsafe(`SELECT data_manager.${fn}() AS seen`); + const fluent = mode === "raw-finally" ? query.raw() : query.values(); + fluentIdentity = Object.is(fluent, query); + if (mode === "values-catch") + await fluent.catch((error) => { + throw error; + }); + else if (mode.endsWith("finally")) + await fluent.finally(() => { + finalized++; + }); + else await fluent.then((value) => value); + } + } catch (error) { + queryRejected = true; + throw error; + } finally { + querySettled = true; + } + context.signal.throwIfAborted(); + return { kind: "download-fonts", completed: true }; + }, + }); + let id: string | undefined; + try { + const deadline = Date.now() + 60000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + const admitted = await runtime.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await ready; + if (mode === "mapped-held") { + letQueryStart(); + await expect + .poll( + async () => + ( + await sql`SELECT count(*)::int AS count FROM pg_stat_activity WHERE query LIKE ${"%" + tableName + "%"} AND wait_event_type='Lock'` + )[0]?.count, + { timeout: 10000 }, + ) + .toBe(1); + } + await runtime.requestCancel({ id, operationKey: key, fingerprint: admitted.fingerprint }); + letQueryStart(); + await expect.poll(() => querySettled, { timeout: 5000 }).toBe(true); + expect(queryRejected).toBe(true); + expect(fluentIdentity).toBe(true); + if (mode.endsWith("finally")) expect(finalized).toBe(1); + unlock(); + await locker; + expect((await sql.unsafe(`SELECT hits FROM data_manager.${tableName}`))[0]?.hits).toBe(0); + await expect + .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) + .toBe("canceled"); + } finally { + letQueryStart(); + unlock(); + await locker; + await runtime.close(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + await sql.unsafe(`DROP FUNCTION data_manager.${fn}()`); + await sql.unsafe(`DROP TABLE data_manager.${tableName}`); + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, + ); +}); diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index b4b20a786..ea30fcf6a 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -29,6 +29,7 @@ import { isDataExecutionTerminal, } from "@openmapx/core/data-manager-contracts"; import type { ExecutionRef, VerifiedParentExecution } from "@openmapx/core/ops"; +import { cancelAndSettleRetainedTransitousScripts } from "../transitous/script-runner.js"; import { cancelAndSettleOwnedDataOpsChildren, type DataOpsChildAttempt, @@ -71,7 +72,8 @@ export interface DataExecutionRuntime { principal?: ExecutionRef["principal"]; hardlinkPlan?: unknown; sourceJobId?: string; - triggeredBy?: "cron" | "bootstrap"; + triggeredBy?: string; + legacyTransitKey?: string; adopt?: (execution: ExecutionRef) => Promise; }, ): Promise; @@ -106,12 +108,114 @@ export function createDataExecutionRuntime(deps: { tail: Promise; }; const active = new Map(); + const recovering = new Map>(); let closed = false; function requireJournal() { if (!journal) throw new DataExecutionRuntimeUnavailable("Durable control prerequisites are unavailable"); return journal; } + function observationContext(record: DataExecutionRecord): DataExecutionContext { + const owner = requireJournal(), + authority = restoreAdmittedControlExecution(owner.installation, record.execution), + lease = + record.input.kind === "generation-hardlinks" + ? restoreGenerationHardlinksLeaseForSettlement(authority) + : restoreArtifactTargetLeaseForSettlement(authority); + let tail: Promise = Promise.resolve(), + version = record.version; + const context: DataExecutionContext = { + execution: record.execution, + get executionVersion() { + return version; + }, + signal: AbortSignal.abort( + new DOMException("Recovery permits cancellation/observation only", "AbortError"), + ), + targetAuthority: lease, + authority, + installation: owner.installation, + store, + dataRoot: owner.dataRoot, + async record() { + const value = await store.inspect(record.status.id); + if (!value) throw new DataExecutionOwnershipLost("Original execution disappeared"); + return value; + }, + async write(run) { + const work = tail.then(async () => { + const current = await context.record(), + saved = await run(current); + version = saved.record.version; + return saved.value; + }); + tail = work.catch(() => {}); + return work; + }, + async checkpoint(patch) { + await context.write(async (current) => { + const saved = await store.checkpoint(current.status.id, current.version, patch); + if (!saved) throw new DataExecutionOwnershipLost("Recovery CAS lost"); + return { record: saved, value: undefined }; + }); + }, + async checkpointChild(child) { + await context.write(async (current) => { + const saved = await store.checkpointChild(current.status.id, current.version, child); + if (!saved) throw new DataExecutionOwnershipLost("Recovery child CAS lost"); + return { record: saved, value: undefined }; + }); + }, + assertForward() { + throw new DataExecutionOwnershipLost("Recovery permits observation/settlement only"); + }, + }; + return context; + } + function cancelRecovered(record: DataExecutionRecord): void { + if (recovering.has(record.status.id)) return; + const task = Promise.resolve() + .then(async () => { + let context: DataExecutionContext | undefined; + try { + context = observationContext(record); + // A stale or contended forward writer is not containment evidence. + // It must not prevent authenticated cancellation of an already live child. + const owned = context; + const cancelOwnership = async () => { + const signal = AbortSignal.timeout(15000); + for (;;) { + let executionCanceled = false, + leaseCanceled = false; + try { + cancelControlExecution(owned.authority); + executionCanceled = true; + } catch {} + try { + await cancelArtifactTargetLease(owned.targetAuthority); + leaseCanceled = true; + } catch {} + if (executionCanceled && leaseCanceled) return; + // Retry only cancellation; never steal a busy writer or renew time. + await settleDelay(50, undefined, { signal }); + } + }; + await Promise.allSettled([ + cancelOwnership(), + cancelAndSettleOwnedDataOpsChildren(context), + cancelAndSettleRetainedTransitousScripts(context), + ]); + // Remote containment cannot manufacture the dead local worker's proof. + await context.checkpoint({ publication: "unknown" }); + } catch { + /* Original identity/lease gaps retain the durable cancellation intent. */ + } finally { + if (context) closeArtifactTargetLeaseHandle(context.targetAuthority); + } + }) + .finally(() => recovering.delete(record.status.id)); + recovering.set(record.status.id, task); + } async function start( record: DataExecutionRecord, adopt?: (execution: ExecutionRef) => Promise, @@ -330,7 +434,8 @@ export function createDataExecutionRuntime(deps: { authority = entry?.context.authority ?? restoreAdmittedControlExecution(owner.installation, saved.execution); - cancelControlExecution(authority); + if (entry) cancelControlExecution(authority); + else cancelRecovered(saved); if (entry) { const until = now() + 15000; for (;;) { @@ -379,7 +484,7 @@ export function createDataExecutionRuntime(deps: { async reconcile() { const owner = requireJournal(); for (const record of await store.listUnsettled()) { - if (active.has(record.status.id)) continue; + if (active.has(record.status.id) || recovering.has(record.status.id)) continue; if (isDataExecutionTerminal(record.status)) { try { await owner.assertOwnedProxyRecoverySettled(record.execution); @@ -412,64 +517,23 @@ export function createDataExecutionRuntime(deps: { } continue; } - let recovered: Awaited> | undefined; + let recovered: Awaited> | undefined, + observation: DataExecutionContext | undefined; try { + // Recover authentic observation authority without assuming that the + // killed local producer ever reached worker completion publication. + observation = observationContext(record); recovered = await owner.restoreCompletion(record.execution); cancelControlExecution(recovered.authority); await cancelArtifactTargetLease(recovered.lease); - let current = record, - tail: Promise = Promise.resolve(); - const context: DataExecutionContext = { - execution: record.execution, - get executionVersion() { - return current.version; - }, - signal: AbortSignal.abort( - new DOMException("Original worker is contained", "AbortError"), - ), - targetAuthority: recovered.lease, - authority: recovered.authority, - installation: owner.installation, - store, - dataRoot: owner.dataRoot, - record: async () => { - const loaded = await store.inspect(record.status.id); - if (!loaded) throw new DataExecutionOwnershipLost("Original execution disappeared"); - return loaded; - }, - async write(run) { - const work = tail.then(async () => { - const saved = await run(current); - current = saved.record; - return saved.value; - }); - tail = work.catch(() => {}); - return work; - }, - async checkpoint(patch) { - await context.write(async (value) => { - const saved = await store.checkpoint(value.status.id, value.version, patch); - if (!saved) throw new DataExecutionOwnershipLost("Recovery CAS lost"); - return { record: saved, value: undefined }; - }); - }, - async checkpointChild(child) { - await context.write(async (value) => { - const saved = await store.checkpointChild(value.status.id, value.version, child); - if (!saved) throw new DataExecutionOwnershipLost("Recovery child CAS lost"); - return { record: saved, value: undefined }; - }); - }, - assertForward() { - throw new DataExecutionOwnershipLost("Recovery permits observation/settlement only"); - }, - }; + const context = observation; if ( (await cancelAndSettleOwnedDataOpsChildren(context)) !== "settled" || recovered.publication === "unknown" ) throw Error("Original containment/publication remains unknown"); - const result = recovered.result, + const current = await context.record(), + result = recovered.result, status = result ? result.kind === "transit-sync" ? result.finalStatus @@ -512,10 +576,13 @@ export function createDataExecutionRuntime(deps: { status: latest.status.cancellationRequestedAt ? "cancel_pending" : "interrupted", publication: "unknown", }); + if (latest.status.cancellationRequestedAt || now() >= latest.status.deadlineAt) + cancelRecovered(latest); } } catch {} } finally { if (recovered) closeArtifactTargetLeaseHandle(recovered.lease); + if (observation) closeArtifactTargetLeaseHandle(observation.targetAuthority); } } }, @@ -531,7 +598,9 @@ export function createDataExecutionRuntime(deps: { ), ); await Promise.race([ - Promise.allSettled([...active.values()].map((entry) => entry.task)), + Promise.allSettled( + [...active.values()].map((entry) => entry.task).concat([...recovering.values()]), + ), new Promise((resolve) => { const t = setTimeout(resolve, 30000); t.unref(); diff --git a/services/data-manager/src/jobs/execution/store.ts b/services/data-manager/src/jobs/execution/store.ts index 89c024940..1cd41f103 100644 --- a/services/data-manager/src/jobs/execution/store.ts +++ b/services/data-manager/src/jobs/execution/store.ts @@ -47,6 +47,12 @@ export const DATA_ADMISSION_LOCK = [0x4f4d58, 2] as const; export class DataExecutionConflict extends Error { readonly statusCode = 409; } +export class LegacyTransitDuplicate extends Error { + readonly statusCode = 409; + constructor(readonly existingJobId: string) { + super("duplicate-idempotency-key"); + } +} export class DataConfigurationMaintenanceBusy extends Error { readonly statusCode = 409; constructor() { @@ -73,7 +79,8 @@ export interface DataExecutionAdmissionOptions { id?: string; principal?: ExecutionRef["principal"]; sourceJobId?: string; - triggeredBy?: "cron" | "bootstrap"; + triggeredBy?: string; + legacyTransitKey?: string; } export type DataExecutionPatch = Partial< Pick @@ -276,6 +283,13 @@ export function createPostgresDataExecutionStore( ): DataExecutionStore { const now = options.now ?? Date.now; const storeIdentity = Symbol("data-store"); + const findLegacyTransit = async (tx: postgres.Sql | postgres.TransactionSql, key: string) => { + const rows = await tx.unsafe( + "SELECT * FROM data_manager.jobs WHERE idempotency_key=$1 AND (started_at >= $2::text::timestamptz OR finished_at IS NULL) ORDER BY started_at DESC,id LIMIT 1", + [key, new Date(now() - 24 * 3600000).toISOString()], + ); + return rows[0]; + }; const find = async ( tx: postgres.Sql | postgres.TransactionSql, key: "id" | "operation_key", @@ -404,6 +418,28 @@ export function createPostgresDataExecutionStore( async admit(value, admission) { const input = normalizeDataExecutionInput(value), operationFingerprint = dataExecutionInputFingerprint(input); + const legacyKey = admission.legacyTransitKey + ? z.string().max(200).parse(admission.legacyTransitKey) + : undefined; + if (legacyKey && (input.kind !== "transit-sync" || admission.parent)) + throw new DataExecutionConflict( + "Legacy transit key is bound only to local transit admission", + ); + const matchLegacy = (row: Row) => { + if (row.execution_version !== 2) throw new LegacyTransitDuplicate(row.id); + const record = decode(row); + if ( + record.input.kind !== "transit-sync" || + record.operationFingerprint !== operationFingerprint || + record.triggeredBy !== (admission.triggeredBy ?? null) + ) + throw new DataExecutionConflict("Legacy transit key body or actor conflict"); + return { created: false, record }; + }; + if (legacyKey) { + const original = await findLegacyTransit(sql, legacyKey); + if (original) return matchLegacy(original); + } // Known duplicates are observations and remain available while fenced or expired. const existing = await find(sql, "operation_key", admission.operationKey); const match = (record: DataExecutionRecord) => { @@ -438,6 +474,10 @@ export function createPostgresDataExecutionStore( throw new DataExecutionConflict("A fresh dated data execution root is required"); } return withDataAdmissionTransaction(sql, async (tx) => { + if (legacyKey) { + const original = await findLegacyTransit(tx, legacyKey); + if (original) return matchLegacy(original); + } await options.configurationFence?.assertAdmission(tx); const retained = await find(tx, "operation_key", admission.operationKey); if (retained) return match(retained); @@ -483,15 +523,26 @@ export function createPostgresDataExecutionStore( const childReservations = reserveTransitChildOperations(execution, input); const rows = await tx< Row[] - >`INSERT INTO data_manager.jobs(id,kind,status,triggered_by,started_at,execution_version,operation_key,operation_fingerprint, + >`INSERT INTO data_manager.jobs(id,kind,status,triggered_by,idempotency_key,started_at,execution_version,operation_key,operation_fingerprint, execution_fingerprint,admitted_at,deadline_at,requested_deadline_at,execution_parent,execution_input,execution_state,metadata) - VALUES(${status.id},${input.kind},'queued',${admission.triggeredBy ?? null},${new Date(status.admittedAt).toISOString()}::text::timestamptz,2,${status.operationKey},${operationFingerprint}, + VALUES(${status.id},${input.kind},'queued',${admission.triggeredBy ?? null},${legacyKey ?? null},${new Date(status.admittedAt).toISOString()}::text::timestamptz,2,${status.operationKey},${operationFingerprint}, ${status.fingerprint},${status.admittedAt},${status.deadlineAt},${admission.requestedDeadlineAt ?? null}, ${admission.parent ? canonicalJson(admission.parent) : null}::text::jsonb,${canonicalJson(input)}::text::jsonb,${canonicalJson({ cancellationRequestedAt: null, progress: null, diagnostics: status.diagnostics, publication: status.publication })}::text::jsonb,${canonicalJson({ dataExecution: { version: 1, execution, childReservations } })}::text::jsonb) RETURNING *`; if (!rows[0]) throw Error("Execution admission did not return its durable row"); if (admission.sourceJobId) await tx`UPDATE data_manager.jobs SET metadata=jsonb_set(coalesce(metadata,'{}'::jsonb),'{durableExecution}',${canonicalJson({ id: status.id, operationKey: status.operationKey, fingerprint: status.fingerprint })}::text::jsonb),ownership_version=ownership_version+1 WHERE id=${admission.sourceJobId}`; return { created: true, record: decode(rows[0]) }; + }).catch(async (error) => { + if (legacyKey && error instanceof DataConfigurationMaintenanceBusy) { + // The competing admission may still be committing its mapping. Only + // read the winner; never repeat a transaction or start another root. + for (let n = 0; n < 10; n++) { + const original = await findLegacyTransit(sql, legacyKey); + if (original) return matchLegacy(original); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + throw error; }); }, inspect: (id) => find(sql, "id", id), diff --git a/services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts b/services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts new file mode 100644 index 000000000..7859fe60f --- /dev/null +++ b/services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts @@ -0,0 +1,106 @@ +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import * as control from "@openmapx/core/build-artifacts"; +import Fastify from "fastify"; +import { registerApi } from "../../../api.js"; +import { registerAuth } from "../../../auth.js"; +import { db, rawSql } from "../../../db/index.js"; +import { StateStore } from "../../../state.js"; +import { resolveOperationsProfile } from "../../transitous/operations-profile.js"; +import { createDefaultTransitousScriptRunner } from "../../transitous/script-runner.js"; +import { createSingleFlightController } from "../../transitous/single-flight.js"; +import { installDataMutationAdmission } from "../admission.js"; +import { createDataConfigurationMaintenance } from "../configuration-maintenance.js"; +import { createDataExecutionDispatcher } from "../dispatch.js"; +import { createDataExecutionReceiver } from "../receiver.js"; +import { createDataExecutionRuntime } from "../runtime.js"; +import { createPostgresDataExecutionStore } from "../store.js"; + +// Host mode uses the production maintenance dispatcher. Python mode isolates +// the actual owned script producer, so SIGKILL lands during its remote process. +// Admission, PostgreSQL, retained authority and HTTP cancellation are real. +const lines = createInterface({ input: process.stdin }); +const messages = lines[Symbol.asyncIterator](); +const initial = await messages.next(); +if (initial.done) throw Error("Missing fixture startup"); +const { root, token, mode, replacement } = JSON.parse(initial.value) as { + root: string; + token: string; + mode: "agent" | "python"; + replacement: boolean; +}; +const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-staging", "motis-feed-proxy"], +}); +const fence = createDataConfigurationMaintenance(rawSql, installation); +await fence.startup(); +installDataMutationAdmission(fence); +const journal = control.createDataExecutionControlJournal(installation, rawSql); +const store = createPostgresDataExecutionStore(rawSql, { + controlJournal: journal, + configurationFence: fence, +}); +const singleFlight = createSingleFlightController({ db, admissionSql: rawSql }); +const policy = resolveOperationsProfile({ countries: ["de"] }); +const dispatcher = createDataExecutionDispatcher({ + dataDir: join(root, "data"), + state: new StateStore(join(root, "data")), + sql: rawSql, + singleFlight, + operationsPolicy: policy, + runHardlinks: async () => { + throw Error("No hardlinks in this fixture"); + }, +}); +const runner = createDefaultTransitousScriptRunner(); +let dispatches = 0; +const send = (value: unknown) => process.stdout.write(`${JSON.stringify(value)}\n`); +const runtime = createDataExecutionRuntime({ + store, + dispatch: async (input, context) => { + dispatches++; + send({ type: "dispatch", dispatches, id: context.execution.executionId }); + if (replacement) throw Error("Recovery must never redispatch"); + if (mode === "agent") return dispatcher.dispatch(input, context); + if (input.kind !== "transit-sync") throw Error("Fixture selects transit only"); + await runner({ script: "garbage-collect" }); + context.signal.throwIfAborted(); + return { + kind: input.kind, + finalStatus: "ok", + countries: input.countries, + sourceRevision: "public-fixture", + }; + }, +}); +const receiver = createDataExecutionReceiver({ + runtime, + installation, + configurationFence: fence, + dispatcher, + operationsPolicy: policy, +}); +await runtime.reconcile(); +const app = Fastify(); +registerAuth(app, token, { executionReceiver: () => receiver }); +registerApi(app, { + dataDir: join(root, "data"), + singleFlight, + operationsPolicy: policy, + executionReceiver: receiver, + searchIndexSql: rawSql, +}); +await app.listen({ host: "127.0.0.1", port: 0 }); +const address = app.server.address(); +if (!address || typeof address === "string") throw Error("Missing DM endpoint"); +send({ type: "ready", port: address.port }); +for await (const line of { [Symbol.asyncIterator]: () => messages }) { + const command = JSON.parse(line) as { type: string }; + if (command.type === "probe") send({ type: "probe", dispatches }); + if (command.type === "reconcile") { + await runtime.reconcile(); + send({ type: "reconciled" }); + } +} diff --git a/services/data-manager/src/jobs/execution/worker-scope.ts b/services/data-manager/src/jobs/execution/worker-scope.ts index aab0fb3d0..9e49a3aca 100644 --- a/services/data-manager/src/jobs/execution/worker-scope.ts +++ b/services/data-manager/src/jobs/execution/worker-scope.ts @@ -75,28 +75,50 @@ function ownedQuery(query: T): T { ) return query; const cancel = query.cancel.bind(query); - return new Proxy(query, { + let consumed: Promise | undefined; + const consume = () => { + if (consumed) return consumed; + const signal = dataWorkerSignal(); + if (signal?.aborted) { + consumed = Promise.reject(signal.reason); + return consumed; + } + const abort = () => { + void Promise.resolve(cancel()).catch(() => {}); + }; + signal?.addEventListener("abort", abort, { once: true }); + try { + const then = Reflect.get(query, "then"); + if (typeof then !== "function") throw Error("PostgreSQL query is not thenable"); + consumed = Promise.resolve(Reflect.apply(then, query, [])).finally(() => + signal?.removeEventListener("abort", abort), + ); + } catch (error) { + signal?.removeEventListener("abort", abort); + consumed = Promise.reject(error); + } + return consumed; + }; + const wrapped = new Proxy(query, { get(target, property, receiver) { const value = Reflect.get(target, property, receiver); - if (property !== "then" || typeof value !== "function") - return typeof value === "function" ? value.bind(target) : value; - return ( - resolved?: ((value: unknown) => unknown) | null, - rejected?: ((reason: unknown) => unknown) | null, - ) => { - const signal = dataWorkerSignal(); - if (signal?.aborted) return Promise.reject(signal.reason).then(resolved, rejected); - const abort = () => { - void Promise.resolve(cancel()).catch(() => {}); - }; - signal?.addEventListener("abort", abort, { once: true }); - const pending = Reflect.apply(value, target, []); - return Promise.resolve(pending) - .finally(() => signal?.removeEventListener("abort", abort)) - .then(resolved, rejected); + if (property === "then") + return ( + resolved?: ((value: unknown) => unknown) | null, + rejected?: ((reason: unknown) => unknown) | null, + ) => consume().then(resolved, rejected); + if (property === "catch") + return (rejected?: ((reason: unknown) => unknown) | null) => consume().catch(rejected); + if (property === "finally") + return (settled?: (() => void) | null) => consume().finally(settled); + if (typeof value !== "function") return value; + return (...args: unknown[]) => { + const result = Reflect.apply(value, target, args); + return result === target ? wrapped : result; }; }, }); + return wrapped; } export function withDataSqlCancellation( sql: T, diff --git a/services/data-manager/src/jobs/transitous/catalog-bump.ts b/services/data-manager/src/jobs/transitous/catalog-bump.ts index c49050906..ad037e7a2 100644 --- a/services/data-manager/src/jobs/transitous/catalog-bump.ts +++ b/services/data-manager/src/jobs/transitous/catalog-bump.ts @@ -2,6 +2,11 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; import { execa } from "execa"; import { parseRefShaPair, type TransitousLock } from "../../transitous-lock.js"; +import { + beginLegacyCatalogPreflight, + markLegacyCatalogFetchAttempt, + rejectLegacyMissingCatalog, +} from "../execution/legacy-mutation.js"; /** * Minimal command-runner seam so the resolver can be unit-tested without @@ -54,8 +59,10 @@ export async function resolveCatalogBumpCandidate( const branch = opts.branch?.trim() || "main"; const runner = opts.runner ?? defaultRunner; const { catalogDir } = opts; + await beginLegacyCatalogPreflight(); if (!existsSync(join(catalogDir, ".git"))) { + await rejectLegacyMissingCatalog(catalogDir); throw new CatalogBumpError( "catalog-not-cloned", `Transitous catalog not found at ${catalogDir}; run a sync first to clone it.`, @@ -63,6 +70,7 @@ export async function resolveCatalogBumpCandidate( } try { + await markLegacyCatalogFetchAttempt(); await runner("git", ["-C", catalogDir, "fetch", "origin", branch]); } catch (err) { throw new CatalogBumpError("git-fetch-failed", (err as Error).message); diff --git a/services/data-manager/src/jobs/transitous/script-runner.ts b/services/data-manager/src/jobs/transitous/script-runner.ts index 91fc1f40c..62f3225ff 100644 --- a/services/data-manager/src/jobs/transitous/script-runner.ts +++ b/services/data-manager/src/jobs/transitous/script-runner.ts @@ -10,8 +10,82 @@ import { transitousRunnerStatusSchema, } from "@openmapx/core/transitous-runner"; import { scrubSecrets } from "../../utils/scrub-secrets.js"; +import type { DataExecutionContext } from "../execution/runtime.js"; import { dataExecutionWorkerContext, dataWorkerSignal } from "../execution/worker-scope.js"; +async function runnerResponseBody(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) throw Error("Runner response missing"); + const chunks: Uint8Array[] = []; + let length = 0; + try { + for (;;) { + const next = await reader.read(); + if (next.done) break; + length += next.value.byteLength; + if (length > 8 * 1024 * 1024) { + await reader.cancel(); + throw Error("Runner response capacity exceeded"); + } + chunks.push(next.value); + } + return JSON.parse(Buffer.concat(chunks, length).toString("utf8")); + } finally { + reader.releaseLock(); + } +} + +/** Recovery uses only the signed original pending requests. The capability + * authorizes cancel/inspect after expiry, never another execution POST. */ +export async function cancelAndSettleRetainedTransitousScripts( + context: DataExecutionContext, +): Promise<"settled" | "pending"> { + const journal = context.store.controlJournal; + if (!journal) throw Error("Original runner journal required"); + const attempts = journal.pendingRunnerAttempts(context.authority); + if (!attempts.length) return "settled"; + const endpoint = process.env.TRANSITOUS_RUNNER_URL?.replace(/\/+$/, ""); + if (!endpoint) return "pending"; + const signal = AbortSignal.timeout(30000); + let pending = false; + for (const request of attempts) { + const control = async (action: "cancel" | "inspect" | "acknowledge") => { + const response = await fetch(`${endpoint}/v1/run/${action}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + signal, + }); + if (!response.ok) throw Error("Original runner outcome is unknown"); + return runnerResponseBody(response); + }; + try { + for (;;) { + signal.throwIfAborted(); + try { + await control("cancel"); + } catch { + /* Lost cancel response permits only exact observation. */ + } + const state = transitousRunnerStatusSchema.parse(await control("inspect")); + if (state.state === "terminal") { + journal.completeRunnerAttempt(context.authority, request, state.result); + try { + await control("acknowledge"); + } catch { + /* Original result is retained. */ + } + break; + } + await runnerDelay(100, undefined, { signal }); + } + } catch { + pending = true; + } + } + return pending ? "pending" : "settled"; +} + /** * Client for the private Transitous runner. * @@ -71,27 +145,6 @@ export function createTransitousScriptRunner( ? AbortSignal.any([workerSignal, controller.signal]) : controller.signal; const endpoint = options.baseUrl.replace(/\/+$/, ""); - async function body(response: Response): Promise { - const reader = response.body?.getReader(); - if (!reader) throw Error("Runner response missing"); - const chunks: Uint8Array[] = []; - let length = 0; - try { - for (;;) { - const next = await reader.read(); - if (next.done) break; - length += next.value.byteLength; - if (length > 8 * 1024 * 1024) { - await reader.cancel(); - throw Error("Runner response capacity exceeded"); - } - chunks.push(next.value); - } - return JSON.parse(Buffer.concat(chunks, length).toString("utf8")); - } finally { - reader.releaseLock(); - } - } const control = async ( action: "inspect" | "cancel" | "acknowledge", settlement: AbortSignal, @@ -103,7 +156,7 @@ export function createTransitousScriptRunner( signal: settlement, }); if (!response.ok) throw Error("Original runner outcome is unknown"); - return body(response); + return runnerResponseBody(response); }; let result: unknown, remoteContained = false; @@ -116,7 +169,7 @@ export function createTransitousScriptRunner( signal, }); if (!response.ok) throw Error(`Runner HTTP ${response.status}`); - result = await body(response); + result = await runnerResponseBody(response); } catch (error) { if (!context && !signal.aborted) throw error; // Standalone client tests have no durable platform owner. const settlement = AbortSignal.timeout(30000); diff --git a/services/data-manager/src/service.ts b/services/data-manager/src/service.ts index b0eb6d61b..1bbc5e8a1 100644 --- a/services/data-manager/src/service.ts +++ b/services/data-manager/src/service.ts @@ -17,6 +17,7 @@ import type { DataExecutionReceiver } from "./jobs/execution/api.js"; import { createDataConfigurationMaintenance } from "./jobs/execution/configuration-maintenance.js"; import { createDataExecutionDispatcher } from "./jobs/execution/dispatch.js"; import { applyOwnedGenerationHardlinks } from "./jobs/execution/hardlinks.js"; +import { reconcileLegacyCatalogPreflightRejections } from "./jobs/execution/legacy-mutation.js"; import { createDataExecutionReceiver } from "./jobs/execution/receiver.js"; import { createDataExecutionRuntime } from "./jobs/execution/runtime.js"; import { createPostgresDataExecutionStore } from "./jobs/execution/store.js"; @@ -186,6 +187,7 @@ let notablePlacesSchedule: NotablePlacesSchedule | null = null; async function start(): Promise { const store = dataState; try { + await reconcileLegacyCatalogPreflightRejections(); await configurationFence?.startup(); await executionRuntime.reconcile(); } catch (err) { From 981ffb19a024e5f0c5edc5c620c0cf90f1f94f7c Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 00:17:43 +0100 Subject: [PATCH 021/135] fix(data): cancel every runner attempt and contain eager SQL --- .../execution/runner-cancellation.test.ts | 208 ++++++++++++++- .../src/jobs/execution/runtime.test.ts | 247 +++++++++++++++++- .../test-fixtures/dm-recovery-process.ts | 27 +- .../src/jobs/execution/worker-scope.ts | 22 +- .../src/jobs/transitous/script-runner.ts | 70 ++--- 5 files changed, 534 insertions(+), 40 deletions(-) diff --git a/services/data-manager/src/jobs/execution/runner-cancellation.test.ts b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts index 3ad05acb7..f2bb628a8 100644 --- a/services/data-manager/src/jobs/execution/runner-cancellation.test.ts +++ b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts @@ -1,5 +1,5 @@ import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; -import { randomBytes, randomUUID } from "node:crypto"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; import { existsSync, mkdirSync, @@ -14,6 +14,7 @@ import { join, resolve } from "node:path"; import { createInterface } from "node:readline"; import * as control from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { transitousRunnerRequestSchema } from "@openmapx/core/transitous-runner"; import { build } from "esbuild"; import postgres from "postgres"; import { afterAll, beforeAll, expect, it } from "vitest"; @@ -95,6 +96,211 @@ function processPeer(file: string, initial: unknown, env: NodeJS.ProcessEnv = pr }; } +it.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)( + "R2-I1 cancels live runner B after DM replacement while authentic attempt A exhausts its control budget", + async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Explicit owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }), + root = mkdtempSync(join(tmpdir(), "omx-runner-union-crash-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const catalog = join(root, "catalog"), + ready = join(catalog, "ready"), + keyFile = join(root, "runner-key"), + token = "b".repeat(43); + mkdirSync(join(catalog, "src"), { recursive: true }); + writeFileSync(keyFile, "7".repeat(32), { mode: 0o600 }); + writeFileSync( + join(catalog, "src/generate-attribution.py"), + "print('public synthetic attempt A')\n", + ); + writeFileSync( + join(catalog, "src/garbage-collect.py"), + "import os,time\nfrom pathlib import Path\nPath('ready').write_text(str(os.getpid()))\nwhile not Path('release').exists(): time.sleep(0.05)\n", + ); + const runner = buildTransitousRunnerServer({ + catalogDir: catalog, + stagingDir: root, + capabilityKey: Buffer.from("7".repeat(32)), + }); + let releaseA = () => {}, + aControls = 0, + bCancels = 0, + bInspections = 0, + bAcks = 0, + posts = 0; + const heldA = new Promise((resolve) => { + releaseA = resolve; + }); + const originalDigests = new Map(), + controlledDigests: Array<[string, string]> = []; + const digest = (value: unknown) => + createHash("sha256").update(JSON.stringify(value)).digest("hex"); + runner.addHook("preHandler", async (request, reply) => { + const body = transitousRunnerRequestSchema.parse(request.body), + name = body.run.script; + if (request.url === "/v1/run") { + posts++; + originalDigests.set(name, digest(body)); + return; + } + controlledDigests.push([name, digest(body)]); + if (name === "generate-attribution") { + if (request.url === "/v1/run/cancel") { + aControls++; + await heldA; + } else if (request.url === "/v1/run/inspect") + return reply.code(503).send({ error: "synthetic unknown A" }); + } else { + if (request.url === "/v1/run/cancel") bCancels++; + if (request.url === "/v1/run/inspect") bInspections++; + if (request.url === "/v1/run/acknowledge") bAcks++; + } + }); + runner.addHook("onSend", async (request, reply, payload) => { + const body = transitousRunnerRequestSchema.parse(request.body); + if (request.url === "/v1/run" && body.run.script === "generate-attribution") + reply.raw.destroy(); + if ( + request.url === "/v1/run/cancel" && + body.run.script === "garbage-collect" && + bCancels === 1 + ) + reply.raw.destroy(); + return payload; + }); + let dm: ReturnType | undefined, id: string | undefined; + try { + await runner.listen({ host: "127.0.0.1", port: 0 }); + const address = runner.server.address(); + if (!address || typeof address === "string") throw Error("Runner endpoint missing"); + const env = { + ...process.env, + DATABASE_URL: url, + TRANSITOUS_RUNNER_URL: `http://127.0.0.1:${address.port}`, + TRANSITOUS_RUNNER_CAPABILITY_KEY_FILE: keyFile, + }; + const start = async (replacement: boolean) => { + dm = processPeer(dmBundle, { root, token, mode: "python-union", replacement }, env); + return `http://127.0.0.1:${(await dm.next("ready")).port}`; + }; + let base = await start(false); + const headers = { authorization: `Bearer ${token}`, "content-type": "application/json" }, + input = { + countries: ["de"], + idempotencyKey: "fix2-union-" + randomUUID(), + triggeredBy: "synthetic:operator", + }; + const admitted = await fetch(base + "/transit/sync", { + method: "POST", + headers, + body: JSON.stringify(input), + }); + expect(admitted.status).toBe(202); + id = (await admitted.json()).jobId; + if (!id) throw Error("Original admission missing"); + await expect.poll(() => existsSync(ready), { timeout: 15000 }).toBe(true); + dm!.send({ type: "probe", id }); + expect((await dm!.next("probe")).pendingRunners).toBe(2); + expect(posts).toBe(2); + const [original] = await sql`SELECT * FROM data_manager.jobs WHERE id=${id}`; + if (!original) throw Error("Original row missing"); + const lock = join(root, "data/.admin-target-control/locks/motis"), + inode = statSync(lock).ino; + const proof = join( + root, + `control-private/data-manager/observations/worker-${original.operation_key}.json`, + ); + await dm!.kill(); + dm = undefined; + expect(existsSync(proof)).toBe(false); + base = await start(true); + const retry = await fetch(base + "/transit/sync", { + method: "POST", + headers, + body: JSON.stringify(input), + }); + expect(retry.status).toBe(202); + expect((await retry.json()).jobId).toBe(id); + const observer = await fetch(base + `/executions/${id}`, { headers }); + await observer.body?.cancel(); + expect(bCancels).toBe(0); + const canceledAt = Date.now(); + const response = await fetch(base + `/executions/${id}/cancel`, { + method: "POST", + headers, + body: JSON.stringify({ + operationKey: original.operation_key, + fingerprint: original.execution_fingerprint, + }), + }); + expect(response.status, await response.text()).toBe(200); + // A's authenticated cancellation transport is held indefinitely. B must + // receive its own cancel immediately, before A's control/settlement budget. + await expect.poll(() => aControls, { timeout: 5000 }).toBeGreaterThan(0); + await expect.poll(() => bCancels, { timeout: 5000 }).toBeGreaterThan(0); + const pid = Number(readFileSync(ready, "utf8")); + await expect + .poll( + () => { + try { + process.kill(-pid, 0); + return true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === "ESRCH") return false; + throw e; + } + }, + { timeout: 5000 }, + ) + .toBe(false); + await expect.poll(() => bAcks, { timeout: 5000 }).toBe(1); + expect(bInspections).toBeGreaterThan(0); + // Keep A unknown past the original full observation budget. No sibling + // result or exhausted read budget can discharge the dead local worker. + await new Promise((resolve) => + setTimeout(resolve, Math.max(0, canceledAt + 31000 - Date.now())), + ); + dm!.send({ type: "probe", id }); + const probe = await dm!.next("probe"); + expect(probe).toMatchObject({ dispatches: 0, pendingRunners: 1 }); + const [current] = await sql`SELECT * FROM data_manager.jobs WHERE id=${id}`; + expect(current?.status).toBe("cancel_pending"); + expect(current?.finished_at).toBeNull(); + expect(current?.execution_state.publication).toBe("unknown"); + for (const key of [ + "id", + "operation_key", + "operation_fingerprint", + "execution_fingerprint", + "execution_input", + "deadline_at", + "triggered_by", + ]) + expect(current?.[key]).toEqual(original[key]); + expect(statSync(lock).ino).toBe(inode); + expect(existsSync(proof)).toBe(false); + expect(posts).toBe(2); + for (const [name, controlDigest] of controlledDigests) + expect(controlDigest).toBe(originalDigests.get(name)); + } finally { + releaseA(); + if (dm) await dm.kill(); + writeFileSync(join(catalog, "release"), "release"); + runner.server.closeAllConnections(); + await runner.close(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); + it .skipIf( !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && diff --git a/services/data-manager/src/jobs/execution/runtime.test.ts b/services/data-manager/src/jobs/execution/runtime.test.ts index 48931441d..bfc13baaa 100644 --- a/services/data-manager/src/jobs/execution/runtime.test.ts +++ b/services/data-manager/src/jobs/execution/runtime.test.ts @@ -1,5 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { mkdirSync, mkdtempSync, existsSync as requireExists, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, existsSync as requireExists, rmSync, statSync } from "node:fs"; +import { connect, createServer } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; @@ -14,6 +15,62 @@ import { createDataExecutionRuntime } from "./runtime.js"; import { createPostgresDataExecutionStore } from "./store.js"; import { publishDataChange, withDataSqlCancellation } from "./worker-scope.js"; +/** Relay real PostgreSQL traffic; hold only the separate CancelRequest packet. + * This makes the active-query/worker-finality ordering observable without + * replacing the driver, query promises, backend or SQL cancellation. */ +async function heldPostgresCancellation(url: string) { + const target = new URL(url), + host = target.hostname, + port = Number(target.port), + held: Array<() => void> = []; + let released = false, + packets = 0; + const server = createServer((client) => { + const upstream = connect({ host, port }); + upstream.on("error", () => client.destroy()); + client.on("error", () => upstream.destroy()); + upstream.pipe(client); + let first = Buffer.alloc(0); + const initial = (chunk: Buffer) => { + first = Buffer.concat([first, chunk]); + if (first.length < 8) return; + client.removeListener("data", initial); + client.pause(); + const forward = () => { + upstream.write(first); + client.pipe(upstream); + client.resume(); + }; + if (first.readInt32BE(0) === 16 && first.readInt32BE(4) === 80877102) { + packets++; + if (!released) { + held.push(forward); + return; + } + } + forward(); + }; + client.on("data", initial); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw Error("PG relay address missing"); + target.hostname = "127.0.0.1"; + target.port = String(address.port); + return { + url: target.toString(), + packets: () => packets, + release() { + released = true; + for (const send of held.splice(0)) send(); + }, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} + describe.skipIf( !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", @@ -621,6 +678,194 @@ describe.skipIf( !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", )("locked production Drizzle and postgres query cancellation", () => { + it.each([ + ["execute", "already-aborted"], + ["forEach", "already-aborted"], + ["execute", "consume-after-abort"], + ["forEach", "consume-after-abort"], + ] as const)( + "R2-I2 awaits actual eager SQL containment (%s, %s)", + async (method, mode) => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Explicit owned PostgreSQL required"); + const sql = postgres(url, { max: 5 }), + proxy = await heldPostgresCancellation(url), + workerRaw = postgres(proxy.url, { max: 1 }); + const db = drizzle(withDataSqlCancellation(workerRaw)), + suffix = randomUUID().replaceAll("-", ""), + table = `fix2_eager_${suffix}`, + prior = `fix2_prior_${suffix}`, + fn = `fix2_hit_${suffix}`; + await sql.unsafe(`CREATE TABLE data_manager.${table}(hits int NOT NULL)`); + await sql.unsafe(`INSERT INTO data_manager.${table} VALUES(0)`); + await sql.unsafe(`CREATE TABLE data_manager.${prior}(value int NOT NULL)`); + await sql.unsafe( + `CREATE FUNCTION data_manager.${fn}() RETURNS int LANGUAGE plpgsql AS $$ BEGIN UPDATE data_manager.${table} SET hits=hits+1; RETURN 1; END $$`, + ); + const root = mkdtempSync(join(tmpdir(), "omx-eager-sql-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + }); + const journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + let start = () => {}, + consume = () => {}, + ready = () => {}, + unlock = () => {}, + locked = () => {}; + const startGate = new Promise((resolve) => { + start = resolve; + }), + consumeGate = new Promise((resolve) => { + consume = resolve; + }), + readyGate = new Promise((resolve) => { + ready = resolve; + }); + const lockGate = new Promise((resolve) => { + unlock = resolve; + }), + lockReady = new Promise((resolve) => { + locked = resolve; + }); + const locker = + mode === "consume-after-abort" + ? sql.begin(async (tx) => { + await tx.unsafe(`LOCK TABLE data_manager.${table} IN ACCESS EXCLUSIVE MODE`); + locked(); + await lockGate; + }) + : Promise.resolve(); + if (mode === "consume-after-abort") await lockReady; + let querySettled = false, + queryRejected = false, + callbacks = 0, + fluentIdentity = false; + const active = async () => + Number( + ( + await sql`SELECT count(*)::int AS count FROM pg_stat_activity WHERE query LIKE ${"%" + fn + "%"} AND state='active' AND wait_event_type='Lock'` + )[0]?.count, + ); + const runtime = createDataExecutionRuntime({ + store, + dispatch: async (_input, context) => { + await publishDataChange(async () => { + await db.$client.unsafe(`INSERT INTO data_manager.${prior} VALUES(1)`); + }); + ready(); + await startGate; + try { + const query = db.$client.unsafe(`SELECT data_manager.${fn}()`); + const eager = + method === "execute" + ? query.execute() + : query.forEach(() => { + callbacks++; + }); + fluentIdentity = Object.is(eager, query); + if (mode === "consume-after-abort") await consumeGate; + await eager; + } catch (error) { + queryRejected = true; + throw error; + } finally { + querySettled = true; + } + context.signal.throwIfAborted(); + return { kind: "download-fonts", completed: true }; + }, + }); + let id: string | undefined; + try { + const deadline = Date.now() + 60000, + operationKey = createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }); + const admitted = await runtime.admit( + { kind: "download-fonts" }, + { operationKey, requestedDeadlineAt: deadline }, + ); + id = admitted.id; + await readyGate; + const lock = join(root, "data/.admin-target-control/locks/data-manager"), + inode = statSync(lock).ino; + if (mode === "consume-after-abort") { + start(); + await expect.poll(active, { timeout: 5000 }).toBe(1); + } + await runtime.requestCancel({ id, operationKey, fingerprint: admitted.fingerprint }); + start(); + consume(); + if (mode === "consume-after-abort") { + await expect + .poll( + async () => { + const row = await store.inspect(id!); + return { + active: await active(), + querySettled, + cancelPackets: proxy.packets(), + status: row?.status.status, + finished: row?.finishedAt !== null, + fence: requireExists(lock), + }; + }, + { timeout: 5000 }, + ) + .toEqual({ + active: 1, + querySettled: false, + cancelPackets: 1, + status: "cancel_pending", + finished: false, + fence: true, + }); + expect(statSync(lock).ino).toBe(inode); + } + proxy.release(); + await expect.poll(() => querySettled, { timeout: 5000 }).toBe(true); + expect(queryRejected).toBe(true); + expect(fluentIdentity).toBe(true); + unlock(); + await locker; + await expect + .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) + .toBe("canceled"); + // A direct query behind the eager request on its one real connection is + // a completion barrier, so an unobserved late write cannot evade the hit check. + await workerRaw`SELECT 1`; + expect((await sql.unsafe(`SELECT hits FROM data_manager.${table}`))[0]?.hits).toBe(0); + expect((await sql.unsafe(`SELECT value FROM data_manager.${prior}`))[0]?.value).toBe(1); + expect(callbacks).toBe(0); + expect(requireExists(lock)).toBe(false); + } finally { + proxy.release(); + start(); + consume(); + unlock(); + await locker; + await workerRaw`SELECT 1`; + await runtime.close(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + await workerRaw.end(); + await proxy.close(); + journal.close(); + control.closeControlInstallation(installation); + await sql.unsafe(`DROP FUNCTION data_manager.${fn}()`); + await sql.unsafe(`DROP TABLE data_manager.${table}`); + await sql.unsafe(`DROP TABLE data_manager.${prior}`); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, + ); it.each([ "mapped-held", "mapped-aborted", diff --git a/services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts b/services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts index 7859fe60f..1c32be445 100644 --- a/services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts +++ b/services/data-manager/src/jobs/execution/test-fixtures/dm-recovery-process.ts @@ -26,7 +26,7 @@ if (initial.done) throw Error("Missing fixture startup"); const { root, token, mode, replacement } = JSON.parse(initial.value) as { root: string; token: string; - mode: "agent" | "python"; + mode: "agent" | "python" | "python-union"; replacement: boolean; }; const installation = control.openControlInstallation({ @@ -65,6 +65,18 @@ const runtime = createDataExecutionRuntime({ if (replacement) throw Error("Recovery must never redispatch"); if (mode === "agent") return dispatcher.dispatch(input, context); if (input.kind !== "transit-sync") throw Error("Fixture selects transit only"); + if (mode === "python-union") { + // The real feed loop proceeds after an unresolved script failure. Keep + // that authentic first request retained before starting the second run. + let rejected = false; + try { + await runner({ script: "generate-attribution" }); + } catch { + rejected = true; + } + if (!rejected) throw Error("The first script must have an unresolved response"); + send({ type: "first-runner-pending" }); + } await runner({ script: "garbage-collect" }); context.signal.throwIfAborted(); return { @@ -97,8 +109,17 @@ const address = app.server.address(); if (!address || typeof address === "string") throw Error("Missing DM endpoint"); send({ type: "ready", port: address.port }); for await (const line of { [Symbol.asyncIterator]: () => messages }) { - const command = JSON.parse(line) as { type: string }; - if (command.type === "probe") send({ type: "probe", dispatches }); + const command = JSON.parse(line) as { type: string; id?: string }; + if (command.type === "probe") { + const record = command.id ? await store.inspect(command.id) : undefined; + const pendingRunners = + record?.input.kind === "transit-sync" + ? journal.pendingRunnerAttempts( + control.restoreAdmittedControlExecution(installation, record.execution), + ).length + : undefined; + send({ type: "probe", dispatches, pendingRunners }); + } if (command.type === "reconcile") { await runtime.reconcile(); send({ type: "reconciled" }); diff --git a/services/data-manager/src/jobs/execution/worker-scope.ts b/services/data-manager/src/jobs/execution/worker-scope.ts index 9e49a3aca..82f7f8087 100644 --- a/services/data-manager/src/jobs/execution/worker-scope.ts +++ b/services/data-manager/src/jobs/execution/worker-scope.ts @@ -75,12 +75,16 @@ function ownedQuery(query: T): T { ) return query; const cancel = query.cancel.bind(query); - let consumed: Promise | undefined; - const consume = () => { + let consumed: Promise | undefined, + started = false; + const consume = (eager?: () => void) => { if (consumed) return consumed; const signal = dataWorkerSignal(); if (signal?.aborted) { consumed = Promise.reject(signal.reason); + // Eager callers may attach their consumer later. Keep the original + // rejection observable then, without an unhandled-rejection gap now. + void consumed.catch(() => {}); return consumed; } const abort = () => { @@ -88,6 +92,8 @@ function ownedQuery(query: T): T { }; signal?.addEventListener("abort", abort, { once: true }); try { + started = true; + eager?.(); const then = Reflect.get(query, "then"); if (typeof then !== "function") throw Error("PostgreSQL query is not thenable"); consumed = Promise.resolve(Reflect.apply(then, query, [])).finally(() => @@ -97,6 +103,7 @@ function ownedQuery(query: T): T { signal?.removeEventListener("abort", abort); consumed = Promise.reject(error); } + void consumed.catch(() => {}); return consumed; }; const wrapped = new Proxy(query, { @@ -112,6 +119,17 @@ function ownedQuery(query: T): T { if (property === "finally") return (settled?: (() => void) | null) => consume().finally(settled); if (typeof value !== "function") return value; + if (property === "execute" || property === "forEach") + return (...args: unknown[]) => { + if (!consumed) + void consume(() => { + Reflect.apply(value, target, args); + }); + else if (started) Reflect.apply(value, target, args); + // A started query always retains its real settlement promise, even + // when its eventual consumer arrives after cancellation. + return wrapped; + }; return (...args: unknown[]) => { const result = Reflect.apply(value, target, args); return result === target ? wrapped : result; diff --git a/services/data-manager/src/jobs/transitous/script-runner.ts b/services/data-manager/src/jobs/transitous/script-runner.ts index 62f3225ff..00fedcbc9 100644 --- a/services/data-manager/src/jobs/transitous/script-runner.ts +++ b/services/data-manager/src/jobs/transitous/script-runner.ts @@ -46,44 +46,48 @@ export async function cancelAndSettleRetainedTransitousScripts( if (!attempts.length) return "settled"; const endpoint = process.env.TRANSITOUS_RUNNER_URL?.replace(/\/+$/, ""); if (!endpoint) return "pending"; - const signal = AbortSignal.timeout(30000); - let pending = false; - for (const request of attempts) { - const control = async (action: "cancel" | "inspect" | "acknowledge") => { - const response = await fetch(`${endpoint}/v1/run/${action}`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(request), - signal, - }); - if (!response.ok) throw Error("Original runner outcome is unknown"); - return runnerResponseBody(response); - }; - try { - for (;;) { - signal.throwIfAborted(); - try { - await control("cancel"); - } catch { - /* Lost cancel response permits only exact observation. */ - } - const state = transitousRunnerStatusSchema.parse(await control("inspect")); - if (state.state === "terminal") { - journal.completeRunnerAttempt(context.authority, request, state.result); + // Start every original cancellation independently. An unknown sibling or a + // hung control must not spend another attempt's delivery/observation budget. + const pending = await Promise.all( + attempts.map(async (request) => { + const settlement = AbortSignal.timeout(30000); + const control = async (action: "cancel" | "inspect" | "acknowledge") => { + const response = await fetch(`${endpoint}/v1/run/${action}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + signal: AbortSignal.any([settlement, AbortSignal.timeout(5000)]), + }); + if (!response.ok) throw Error("Original runner outcome is unknown"); + return runnerResponseBody(response); + }; + try { + for (;;) { + settlement.throwIfAborted(); try { - await control("acknowledge"); + await control("cancel"); } catch { - /* Original result is retained. */ + /* Lost cancel response permits only exact observation. */ + } + const state = transitousRunnerStatusSchema.parse(await control("inspect")); + if (state.state === "terminal") { + journal.completeRunnerAttempt(context.authority, request, state.result); + try { + await control("acknowledge"); + } catch { + /* Original result is retained. */ + } + break; } - break; + await runnerDelay(100, undefined, { signal: settlement }); } - await runnerDelay(100, undefined, { signal }); + return false; + } catch { + return true; } - } catch { - pending = true; - } - } - return pending ? "pending" : "settled"; + }), + ); + return pending.some(Boolean) ? "pending" : "settled"; } /** From 842d00d657203b2322e11f049fbb551b7039959c Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 01:24:02 +0100 Subject: [PATCH 022/135] feat(admin): authorize owned service cleanup and logs --- .../__tests__/admin-services-sdk.test.ts | 4 +- .../routes/__tests__/admin-services.test.ts | 15 + apps/api/src/routes/admin-services.ts | 6 - .../__tests__/admin-ops-authority.test.ts | 20 +- apps/api/src/services/admin-ops.ts | 28 +- .../src/__fixtures__/owned-service-host.ts | 262 ++++++++++++++ .../src/community-repositories.test.ts | 28 +- apps/ops-agent/src/community-repositories.ts | 33 +- apps/ops-agent/src/docker-runtime.test.ts | 68 +++- apps/ops-agent/src/docker-runtime.ts | 63 +++- apps/ops-agent/src/index.ts | 51 +-- apps/ops-agent/src/owned-services.test.ts | 109 ++++++ apps/ops-agent/src/owned-services.ts | 261 ++++++++++++++ apps/ops-agent/src/policy.test.ts | 91 +++-- apps/ops-agent/src/policy.ts | 121 +++++-- apps/ops-agent/src/production-runtime.test.ts | 335 ++++++++++++++++++ apps/ops-agent/src/production-runtime.ts | 190 ++++++++++ apps/ops-agent/src/server.ts | 1 + .../src/service-action-policy.test.ts | 53 +++ apps/ops-agent/src/service-action-policy.ts | 51 +++ .../src/trusted-config-ownership.test.ts | 31 +- .../src/trusted-config-runtime.test.ts | 17 +- apps/ops-agent/src/trusted-config-runtime.ts | 20 ++ .../api/contracts/service-admin/services.ts | 1 + packages/core/src/ops/contract.test.ts | 1 + packages/core/src/ops/contract.ts | 21 +- packages/core/src/ops/service-results.ts | 40 +++ .../src/services/configuration-generation.ts | 179 +++++++++- packages/core/src/services/deployment.ts | 11 +- packages/core/src/services/index.ts | 1 + scripts/ops-authority-inventory.json | 7 + 31 files changed, 1967 insertions(+), 152 deletions(-) create mode 100644 apps/ops-agent/src/__fixtures__/owned-service-host.ts create mode 100644 apps/ops-agent/src/owned-services.test.ts create mode 100644 apps/ops-agent/src/owned-services.ts create mode 100644 apps/ops-agent/src/production-runtime.test.ts create mode 100644 apps/ops-agent/src/production-runtime.ts create mode 100644 apps/ops-agent/src/service-action-policy.test.ts create mode 100644 apps/ops-agent/src/service-action-policy.ts diff --git a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts index 67af69ba9..cdaa9d35d 100644 --- a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts @@ -1067,7 +1067,7 @@ it("keeps log pre-hijack JSON statuses/media, required single headers and bounde ...options, onTextChunk: () => {}, }), - ).toMatchObject({ status: 409, data: { error: expect.any(String) } }); + ).toMatchObject({ status: 200 }); registry.mockImplementationOnce(() => { throw Error("unavailable"); }); @@ -1077,7 +1077,7 @@ it("keeps log pre-hijack JSON statuses/media, required single headers and bounde onTextChunk: () => {}, }), ).toMatchObject({ status: 503, data: { error: "Service registry not available" } }); - expect(execute).not.toHaveBeenCalled(); + expect(execute).toHaveBeenCalledTimes(1); for (const [lines, tail] of [ ["", 1], ["-2", 1], diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index 4a07881fd..a04534291 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -381,6 +381,21 @@ describe("GET /admin/services/:id", () => { }); describe("GET /admin/services/:id/logs", () => { + it("passes disabled registered service logs to the agent ownership decision", async () => { + mockRegistryGet.mockReturnValueOnce({ ...MOCK_COMMUNITY_RUNNING, enabled: false }); + const response = await app.inject({ + method: "GET", + url: "/admin/services/community-weather/logs?lines=20", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, + }); + expect(response.statusCode).toBe(200); + expect(mockDockerComposeLogs).toHaveBeenCalledWith( + "community-weather", + expect.anything(), + expect.objectContaining({ tail: 20 }), + ); + }); + it("streams community logs through the same typed operation as built-ins", async () => { mockRegistryGet.mockReturnValueOnce(MOCK_COMMUNITY_RUNNING); diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index 0e209932e..d3e8277d2 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -669,12 +669,6 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { reply.status(404); return reply.send({ error: `Service "${(req.params as { id: string }).id}" not found` }); } - if (!service.enabled) { - reply.status(409); - return reply.send({ - error: `Service "${(req.params as { id: string }).id}" is disabled`, - }); - } try { parseDirectOpsIdempotency(req.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]); } catch (error) { diff --git a/apps/api/src/services/__tests__/admin-ops-authority.test.ts b/apps/api/src/services/__tests__/admin-ops-authority.test.ts index db211ddff..0926a5761 100644 --- a/apps/api/src/services/__tests__/admin-ops-authority.test.ts +++ b/apps/api/src/services/__tests__/admin-ops-authority.test.ts @@ -45,13 +45,13 @@ function context(): JobContext { describe("admin service authority split", () => { it("keeps enabled built-ins on typed ops-agent lifecycle", async () => { - registryGet.mockReturnValue({ manifest: { id: "redis" }, enabled: true, isBuiltIn: true }); - await serviceStop("redis", context()); + registryGet.mockReturnValue({ manifest: { id: "motis" }, enabled: true, isBuiltIn: true }); + await serviceStop("motis", context()); expect(executeAdminJobOperation).toHaveBeenCalledWith( expect.objectContaining({ jobId: "job-123" }), - { kind: "service.stop", serviceId: "redis" }, + { kind: "service.stop", serviceId: "motis" }, "admin-job.service.stop", - { durableIdentity: "redis" }, + { durableIdentity: "motis" }, ); expect(dockerComposeAction).not.toHaveBeenCalled(); }); @@ -76,13 +76,21 @@ describe("admin service authority split", () => { expect(dockerComposeAction).not.toHaveBeenCalled(); }); - it("denies disabled services before either authority boundary", async () => { + it("passes disabled known-service Stop to agent ownership policy but still denies Restart", async () => { registryGet.mockReturnValue({ manifest: { id: "community-weather" }, enabled: false, isBuiltIn: false, }); - await expect(serviceStop("community-weather", context())).rejects.toThrow("is disabled"); + await serviceStop("community-weather", context()); + expect(executeAdminJobOperation).toHaveBeenCalledWith( + expect.anything(), + { kind: "service.stop", serviceId: "community-weather" }, + "admin-job.service.stop", + { durableIdentity: "community-weather" }, + ); + executeAdminJobOperation.mockClear(); + await expect(serviceRestart("community-weather", context())).rejects.toThrow("is disabled"); expect(dockerComposeAction).not.toHaveBeenCalled(); expect(executeAdminJobOperation).not.toHaveBeenCalled(); }); diff --git a/apps/api/src/services/admin-ops.ts b/apps/api/src/services/admin-ops.ts index 6c57f60aa..4ddc14b8f 100644 --- a/apps/api/src/services/admin-ops.ts +++ b/apps/api/src/services/admin-ops.ts @@ -7,6 +7,8 @@ import { link, unwrapDataManagerResponse, } from "@openmapx/core/data-manager-sdk"; +import type { HardlinkApplySummary } from "@openmapx/core/ops"; + import { repoPaths } from "@openmapx/core/server"; import { type DockerComposeAction, @@ -21,18 +23,11 @@ import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-c import { getServiceRegistry } from "./service-registry"; import { applyTrustedConfiguration } from "./trusted-config-operations"; +export type { HardlinkApplySummary } from "@openmapx/core/ops"; + const DATA_MANAGER_STARTUP_TIMEOUT_MS = 60_000; const DATA_MANAGER_POLL_INTERVAL_MS = 2_000; -export interface HardlinkApplySummary { - applied: boolean; - linked: number; - skipped: number; - pruned: number; - entries: number; - via?: string; -} - // Path resolution for an API running on the host (`pnpm dev`), which can read // the deployment directly. The app-api container has no host checkout: its // `/app/infra/docker` holds no configuration, and the deployment reaches it @@ -211,7 +206,7 @@ export async function applyGenerationHardlinks(opts: { } await opts.log?.(`Applying hardlink plan (${plan.length} entries, prune enabled)...`); - const { transport, url } = await waitForDataManagerTransport(signal); + const { transport } = await waitForDataManagerTransport(signal); signal?.throwIfAborted(); const checkpoint = async (phase: "attempted" | "complete", result?: HardlinkApplySummary) => { if (!opts.ctx) return; @@ -236,7 +231,12 @@ export async function applyGenerationHardlinks(opts: { const result = unwrapDataManagerResponse( await link({ plan, prune: true }, { transport, signal }), ); - const summary = { applied: true, entries: plan.length, via: url, ...result }; + const summary: HardlinkApplySummary = { + applied: true, + entries: plan.length, + via: "data-manager", + ...result, + }; await checkpoint("complete", summary); return summary; } catch (error) { @@ -249,7 +249,7 @@ export async function applyGenerationHardlinks(opts: { // Service control (job handlers) — registry-authorized + routed through the // docker-compose helper that targets the generated compose file. -function assertKnownService(service: string) { +function assertKnownService(service: string, requireEnabled = true) { let svc: ReturnType["get"]>; try { svc = getServiceRegistry().get(service); @@ -259,7 +259,7 @@ function assertKnownService(service: string) { if (!svc) { throw new Error(`Unknown service "${service}" — not present in the service registry`); } - if (!svc.enabled) { + if (requireEnabled && !svc.enabled) { throw new Error(`Service "${service}" is disabled in the registry`); } return svc; @@ -270,7 +270,7 @@ async function serviceComposeAction( action: Exclude, ctx: JobContext, ): Promise<{ exitCode: number }> { - assertKnownService(service); + assertKnownService(service, action !== "stop"); // Community and built-in services now take the same typed path: the agent // authorizes the id against its own trusted selection, so the API no longer // needs a direct-Docker adapter for community lifecycle. diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts new file mode 100644 index 000000000..714dcec53 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -0,0 +1,262 @@ +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + opsOperationFingerprint, + sealTrustedConfigurationSnapshot, + type OpsOperation, +} from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { CommunityRepositoryStore } from "../community-repositories"; +import type { DockerRuntimeOptions } from "../docker-runtime"; +import { createProductionRegistryResourceClaimer } from "../policy"; +import { + createProductionOpsRuntime, + productionOwnedServices, + serviceActionAuthority, +} from "../production-runtime"; +import { createTrustedConfigurationAuthorityLoader } from "../trusted-configuration-authority"; +import { createFileTrustedOpsDataSource } from "../trusted-config-source"; +import { dispatchOpsOperation } from "../runtime"; +import { testExecution } from "./execution"; + +export async function ownedServiceHost() { + const root = mkdtempSync(join(tmpdir(), "openmapx-owned-")); + chmodSync(root, 0o755); + const infraDir = join(root, "infra/docker"); + mkdirSync(infraDir, { recursive: true }); + mkdirSync(join(root, "services")); + mkdirSync(join(root, "custom_integrations")); + const loaded: services.LoadedService[] = [ + "motis", + "ops-agent", + "app-api", + "traefik", + "postgis", + "redis", + ].map((id) => ({ + manifest: { + id, + name: id, + version: "1.0.0", + quality: "built-in", + container: { image: `example/${id}`, tag: "1" }, + }, + directory: join(root, "services", id), + enabled: true, + isBuiltIn: true, + })); + const loadAuthority = createTrustedConfigurationAuthorityLoader({ + rootDir: root, + builtInServices: loaded, + }); + const initial = await services.commitConfigurationGeneration({ + infraDir, + services: loaded, + input: { + domain: "example.test", + selectedRoots: ["motis"], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + ownedServiceGenerations: [], + }); + services.writeServiceSelectionFile(infraDir, ["motis"]); + const composePath = services.currentConfigurationFile(infraDir, services.GENERATED_COMPOSE_FILE); + const firstCompose = realpathSync(composePath); + const stack = { + infraDir, + composePath, + composeReleasePath: join(infraDir, "docker-compose.release.yml"), + }; + const imageId = `sha256:${"b".repeat(64)}`; + const row = { + Id: "a".repeat(64), + Image: imageId, + State: "running", + ConfigImage: "example/motis:1", + Labels: { + "com.docker.compose.project": services.STACK_PROJECT, + "com.docker.compose.service": "motis", + "com.docker.compose.project.working_dir": dirname(firstCompose), + "com.docker.compose.project.config_files": firstCompose, + }, + }; + let rows = [row]; + let rawListing: string | undefined; + let malformedInspect = false; + const calls: string[][] = []; + const execFile: NonNullable = async (_file, args) => { + calls.push([...args]); + let stdout = ""; + if (args[0] === "container" && args[1] === "ls") + stdout = rawListing ?? rows.map((r) => r.Id).join("\n"); + else if (args[0] === "container" && args[1] === "inspect") + stdout = malformedInspect ? "{" : rows.map((r) => JSON.stringify(r)).join("\n"); + else if (args[0] === "compose" && args.includes("config")) + stdout = JSON.stringify({ + services: Object.fromEntries( + loaded.map((s) => [ + s.manifest.id, + { image: services.serviceContainerImageReference(s.manifest.container) }, + ]), + ), + }); + else if (args[0] === "image" && args[1] === "inspect") stdout = imageId; + else if (args[0] === "stop") row.State = "exited"; + else if (args[0] === "logs") stdout = "retained log\n"; + return { stdout, stderr: "" }; + }; + const repositories = new CommunityRepositoryStore({ + rootDir: root, + uid: process.geteuid?.() ?? 0, + loadAuthority, + clone: async ({ targetDir }) => { + mkdirSync(targetDir, { recursive: true, mode: 0o755 }); + writeFileSync( + join(targetDir, "service.json"), + JSON.stringify({ + id: "community-live", + name: "Community", + version: "1.0.0", + quality: "community", + container: { + image: "example/community-live", + tag: "1", + digest: `sha256:${"b".repeat(64)}`, + }, + }), + ); + return { directory: targetDir, commit: "1".repeat(40) }; + }, + }); + await repositories.recover(); + const runtime = await createProductionOpsRuntime({ + rootDir: root, + stack, + loadAuthority, + communityRepositories: repositories, + docker: { + execFile, + fileExists: existsSync, + followLogs: async (_file, args, options) => { + calls.push([...args]); + options.onLine("stdout", "retained follow"); + return { lines: 1, truncated: false }; + }, + }, + }); + const authority = await loadAuthority(); + const queue = join(root, "trusted-queue"); + mkdirSync(queue, { mode: 0o700 }); + mkdirSync(join(queue, ".claimed"), { mode: 0o700 }); + const token = Buffer.alloc(32, 17).toString("base64url"); + const trustedData = createFileTrustedOpsDataSource({ + directory: queue, + token, + allowedUids: [statSync(queue).uid], + expectedGid: statSync(queue).gid, + }); + const claimer = createProductionRegistryResourceClaimer({ + services: authority.services.map((s) => ({ + serviceId: s.manifest.id, + enabled: s.enabled, + isBuiltIn: s.isBuiltIn, + })), + integrationIds: [], + loadAuthority: async () => { + const a = await loadAuthority(); + return { + revisionId: a.revisionId, + services: a.services.map((s) => ({ + serviceId: s.manifest.id, + enabled: s.enabled, + isBuiltIn: s.isBuiltIn, + })), + integrationIds: [...a.integrationSchemas.keys()], + }; + }, + enabledServiceIds: () => services.readAppliedServiceIds(infraDir, loaded) ?? new Set(), + trustedData, + ownedServices: productionOwnedServices(runtime), + serviceMetadata: (ss) => ss.map((s) => ({ ...s, buildable: false })), + }); + const context = async (operation: OpsOperation) => { + const signal = new AbortController().signal; + const claim = await claimer.claim(operation, opsOperationFingerprint(operation), signal, { + role: "api", + operationKey: testExecution().operationKey, + }); + if (!claim) throw new Error("Admission denied"); + return { execution: testExecution(), signal, emitLog() {}, claim }; + }; + let counter = 0; + const select = async (selectedRoots: string[]) => { + const sealed = sealTrustedConfigurationSnapshot({ + role: "api", + token, + nonce: `nonce_ownership_${String(++counter).padStart(12, "0")}`, + operationKey: testExecution().operationKey, + operationForRevision: (revisionId) => ({ + kind: "serviceSelection.apply", + revisionId, + expectedCurrentRevisionId: services.currentConfigurationGeneration(infraDir), + }), + payload: { + domain: "example.test", + selectedRoots, + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + }); + writeFileSync(join(queue, `${sealed.revisionId}.json`), sealed.bytes, { mode: 0o600 }); + const ctx = await context(sealed.operation); + await ctx.claim.admission?.commit(); + try { + return await dispatchOpsOperation(runtime, sealed.operation, ctx); + } finally { + await ctx.claim.admission?.release(); + } + }; + const deselect = () => select([]); + return { + root, + token, + infraDir, + loaded, + initial, + firstCompose, + stack, + runtime, + repositories, + claimer, + row, + calls, + execFile, + loadAuthority, + context, + deselect, + select, + owned: productionOwnedServices(runtime), + actionAuthority: async () => serviceActionAuthority(await loadAuthority(), infraDir), + setRows(value: typeof rows) { + rows = value; + }, + setRawListing(value: string) { + rawListing = value; + }, + setMalformedInspect() { + malformedInspect = true; + }, + }; +} diff --git a/apps/ops-agent/src/community-repositories.test.ts b/apps/ops-agent/src/community-repositories.test.ts index 0cc6fd8f8..a1188641e 100644 --- a/apps/ops-agent/src/community-repositories.test.ts +++ b/apps/ops-agent/src/community-repositories.test.ts @@ -348,11 +348,15 @@ describe("community repository store", () => { }); expect((await store.list()).repositories[0]).toMatchObject({ commit: FIRST, pending: null }); - await store.remove({ - kind: "serviceRepository.remove", - repository: REPOSITORY, - extensionId: "live-bundle", - }); + await store.remove( + { + kind: "serviceRepository.remove", + repository: REPOSITORY, + extensionId: "live-bundle", + }, + signal(), + { inspectOwnedServices: async () => [] }, + ); const fresh = await publish(store, SECOND); await store.settle({ kind: "serviceRepository.settle", @@ -405,11 +409,15 @@ describe("community repository store", () => { }), ).rejects.toThrow("The repository belongs to another extension"); await expect( - store.remove({ - kind: "serviceRepository.remove", - repository: REPOSITORY, - extensionId: "live-bundle", - }), + store.remove( + { + kind: "serviceRepository.remove", + repository: REPOSITORY, + extensionId: "live-bundle", + }, + signal(), + { inspectOwnedServices: async () => [] }, + ), ).resolves.toEqual({ repository: REPOSITORY, removed: true }); }); diff --git a/apps/ops-agent/src/community-repositories.ts b/apps/ops-agent/src/community-repositories.ts index b47cef3fd..107ce60ec 100644 --- a/apps/ops-agent/src/community-repositories.ts +++ b/apps/ops-agent/src/community-repositories.ts @@ -21,6 +21,7 @@ import { type GitShallowCloneOptions, gitShallowCloneSnapshot, } from "@openmapx/core/server"; +import type { OwnedServiceResolver } from "./owned-services"; import { OpsAgentBusyError, type OpsRuntime } from "./runtime"; import { TrustedAuthorityFilesystem } from "./trusted-authority-filesystem"; import type { TrustedConfigurationAuthoritySnapshot } from "./trusted-config-runtime"; @@ -598,6 +599,8 @@ export class CommunityRepositoryStore { remove( operation: Extract, + signal: AbortSignal = AbortSignal.timeout(15_000), + owned?: OwnedServiceResolver, ): Promise> { return this.serialize(async () => { const { repository, extensionId } = operation; @@ -606,6 +609,32 @@ export class CommunityRepositoryStore { this.assertOwner(repository, record, extensionId); const checkout = this.checkout(repository); const existed = existsSync(checkout) || record !== null; + if (existed) { + const authority = await this.options.loadAuthority(); + const ids = new Set( + authority.services + .filter( + (service) => + !service.isBuiltIn && + (service.directory === checkout || + service.directory.startsWith(`${checkout}${sep}`)), + ) + .map((service) => service.manifest.id), + ); + if (ids.size > 0) { + if (!owned) throw new Error("Owned service cleanup inspection required before removal"); + const inventory = owned.inspectInventory + ? await owned.inspectInventory(signal) + : { services: await owned.inspectOwnedServices(signal), unverifiedServiceIds: [] }; + if (inventory.unverifiedServiceIds.some((id) => ids.has(id))) + throw new Error( + "Owned container provenance requires host cleanup before repository removal", + ); + if (inventory.services.some((container) => ids.has(container.serviceId))) + throw new Error("Owned containers require cleanup before repository removal"); + } + } + signal.throwIfAborted(); rmSync(checkout, { recursive: true, force: true }); if (existsSync(this.community)) syncDirectory(this.community); if (record) this.deleteRecord(repository); @@ -661,12 +690,14 @@ export class CommunityRepositoryStore { export function installCommunityRepositoryRuntime( runtime: OpsRuntime, store: CommunityRepositoryStore, + owned?: OwnedServiceResolver, ): void { runtime["serviceRepository.inspect"] = (operation, context) => store.inspect(operation, context.signal); runtime["serviceRepository.publish"] = (operation, context) => store.publish(operation, context.signal); runtime["serviceRepository.settle"] = (operation) => store.settle(operation); - runtime["serviceRepository.remove"] = (operation) => store.remove(operation); + runtime["serviceRepository.remove"] = (operation, context) => + store.remove(operation, context.signal, owned); runtime["communityServices.inspect"] = () => store.list(); } diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index e9812cb4c..689761d6f 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -21,7 +21,21 @@ import { } from "./docker-runtime"; import { dispatchOpsOperation } from "./runtime"; -const context = () => ({ +const ownedSnapshot = (serviceId = "redis") => ({ + serviceId, + containerId: "a".repeat(64), + state: "running" as const, + provenance: { + project: "docker", + composeWorkingDirectory: "/trusted", + composeConfigFiles: ["/trusted/compose.yml"], + imageId: `sha256:${"b".repeat(64)}`, + }, +}); +const ownedServices = { + inspectOwnedServices: async () => ["redis", "motis", "postgis"].map(ownedSnapshot), +}; +const context = (serviceId = "redis") => ({ execution: testExecution(), signal: new AbortController().signal, emitLog: () => undefined, @@ -29,7 +43,10 @@ const context = () => ({ fingerprint: "f".repeat(64), operation: { kind: "docker.status" } as const, source: "registry" as const, - capability: { revisionId: "registry-v1", values: {} }, + capability: { + revisionId: "registry-v1", + values: { ownedService: JSON.stringify(ownedSnapshot(serviceId)) }, + }, }, }); @@ -804,7 +821,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done it.each([ { kind: "stack.start" }, { kind: "service.start", serviceId: "postgis" }, - { kind: "service.stop", serviceId: "postgis" }, + { kind: "service.stop", serviceId: "motis" }, { kind: "service.restart", serviceId: "postgis" }, { kind: "service.recreate", serviceId: "postgis" }, { kind: "service.recreateIsolated", serviceId: "postgis" }, @@ -817,13 +834,20 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async (_file, _args, options) => { expect(options.timeout).toBe(600_000); expect(options.signal.aborted).toBe(false); return { stdout: "", stderr: "" }; }, }); - await expect(dispatchOpsOperation(runtime, operation, context())).resolves.toEqual({ + await expect( + dispatchOpsOperation( + runtime, + operation, + context("serviceId" in operation ? operation.serviceId : "redis"), + ), + ).resolves.toEqual({ changed: true, }); }); @@ -843,12 +867,17 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async (_file, _args, options) => { expect(options.timeout).toBe(timeout); return { stdout: "", stderr: "" }; }, }); - await dispatchOpsOperation(runtime, operation, context()); + await dispatchOpsOperation( + runtime, + operation, + context("serviceId" in operation ? operation.serviceId : "redis"), + ); }, ); @@ -863,6 +892,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: (_file, _args, options) => new Promise((resolve, reject) => { const deadline = setTimeout(() => { @@ -951,6 +981,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async (_file, args) => { calls.push([...args]); return { @@ -993,6 +1024,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile, }); @@ -1133,6 +1165,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async (_file, args) => { calls.push([...args]); return { @@ -1180,6 +1213,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: execFile as never, }); @@ -1195,6 +1229,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, trafficDataRoot: dataRoot, execFile: async (_file, args) => { calls.push([...args]); @@ -1280,6 +1315,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, trafficDataRoot: dataRoot, execFile: async (_file, args) => { calls.push([...args]); @@ -1329,6 +1365,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, trafficDataRoot: dataRoot, execFile: async (_file, args) => { if (args[0] === "inspect" && args.includes("{{.State.Running}}")) { @@ -1409,6 +1446,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, trafficDataRoot: dataRoot, execFile: async (_file, args) => ({ stdout: @@ -1485,6 +1523,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async () => ({ stdout: "", stderr: "" }), }); await expect( @@ -1513,6 +1552,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async (_file, args) => { calls.push([...args]); if (args.includes("ps")) { @@ -1577,16 +1617,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done it("follows one fixed service with a bounded duration and typed event emitter", async () => { const emitted: Array<{ stream: "stdout" | "stderr"; message: string }> = []; const followLogs = vi.fn(async (_file, args, options) => { - expect(args).toEqual([ - "compose", - "-f", - "/trusted/compose.yml", - "logs", - "-f", - "--no-color", - "--tail=20", - "redis", - ]); + expect(args).toEqual(["logs", "--follow", "--tail=20", "a".repeat(64)]); expect(options.timeout).toBe(3_000); options.onLine("stdout", "ready"); options.onLine("stderr", "bounded warning"); @@ -1599,6 +1630,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async () => ({ stdout: "", stderr: "" }), followLogs, }); @@ -1632,6 +1664,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: async () => ({ stdout: `${"😀".repeat(1_500)}\nsecond\nthird\n`, stderr: "" }), }); const result = await dispatchOpsOperation( @@ -1768,6 +1801,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, + ownedServices, execFile: (_file, _args, options) => runContainedProcess(process.execPath, ["-e", script, ready, marker], { ...options, @@ -1776,8 +1810,8 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done }); const operation = dispatchOpsOperation( runtime, - { kind: "service.stop", serviceId: "postgis" }, - { ...context(), signal: controller.signal }, + { kind: "service.stop", serviceId: "motis" }, + { ...context("motis"), signal: controller.signal }, ); const rejected = expect(operation).rejects.toThrow("Contained process failed"); try { diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index aea3a8cf5..f8d756563 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -23,6 +23,7 @@ import { createFeedProxyValidationInvocation, parseFeedProxyValidationCompletion, } from "./feed-proxy-validation"; +import type { OwnedServiceResolver, OwnedServiceSnapshot } from "./owned-services"; import { createBoundedProcessOutput, safeProcessDiagnostic } from "./process-output"; import { createUnavailableRuntime, @@ -30,6 +31,7 @@ import { OpsExecutionFailure, type OpsRuntime, } from "./runtime"; +import { decideServiceAction } from "./service-action-policy"; const DEFAULT_TIMEOUT_MS = 120_000; const LIFECYCLE_TIMEOUT_MS = 10 * 60_000; @@ -72,6 +74,7 @@ export interface DockerRuntimeOptions { stack: services.StackPaths; fileExists: (path: string) => boolean; execFile?: ExecFile; + ownedServices?: OwnedServiceResolver; followLogs?: FollowLogs; /** * Owner of the shared `/data` mount. Artifacts a container writes as root @@ -477,6 +480,40 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { if (after === "running") throw new Error("MOTIS container remains running after stop"); return { changed: true, state: after }; }; + const ownedTarget = async ( + serviceId: string, + context: OpsExecutionContext, + action: "stop" | "logs", + ): Promise => { + const encoded = context.claim.capability.values.ownedService; + if (!options.ownedServices || typeof encoded !== "string") + throw new Error("Owned service claim required"); + const claimed = JSON.parse(encoded) as OwnedServiceSnapshot; + const current = (await options.ownedServices.inspectOwnedServices(context.signal)).find( + (s) => s.serviceId === serviceId, + ); + if ( + !current || + claimed.serviceId !== serviceId || + claimed.containerId !== current.containerId || + JSON.stringify(claimed.provenance) !== JSON.stringify(current.provenance) + ) + throw new Error("Owned service claim changed"); + if ( + !decideServiceAction( + { + revisionId: "owned", + services: [{ serviceId, isBuiltIn: true, buildable: false }], + enabledServiceIds: new Set(), + }, + current, + action, + ).available + ) + throw new Error("Service action denied"); + context.signal.throwIfAborted(); + return current; + }; const runtime = createUnavailableRuntime(); const followLogs: FollowLogs = options.followLogs ?? runContainedLogProcess; @@ -492,8 +529,10 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { changed([...composePrefix(), "up", "-d"], context, LIFECYCLE_TIMEOUT_MS); runtime["service.start"] = (operation, context) => changed([...composePrefix(), "up", "-d", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); - runtime["service.stop"] = (operation, context) => - changed([...composePrefix(), "stop", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); + runtime["service.stop"] = async (operation, context) => { + const target = await ownedTarget(operation.serviceId, context, "stop"); + return changed(["stop", target.containerId], context, LIFECYCLE_TIMEOUT_MS); + }; runtime["service.restart"] = (operation, context) => changed([...composePrefix(), "restart", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.recreate"] = (operation, context) => @@ -513,11 +552,12 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { runtime["service.remove"] = (operation, context) => changed([...composePrefix(), "rm", "-sf", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.logs"] = async (operation, context) => { + const target = await ownedTarget(operation.serviceId, context, "logs"); const result = await run( - [...composePrefix(), "logs", "--no-color", `--tail=${operation.tail}`, operation.serviceId], + ["logs", `--tail=${operation.tail}`, target.containerId], context.signal, ); - const rawLines = result.stdout.split(/\r?\n/).filter(Boolean); + const rawLines = `${result.stdout}\n${result.stderr}`.split(/\r?\n/).filter(Boolean); const selected = rawLines.slice(0, operation.tail); const lines = selected.map((line) => truncateUtf8(line)); return { @@ -526,17 +566,11 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { rawLines.length > operation.tail || lines.some((line, index) => line !== selected[index]), }; }; - runtime["service.logs.follow"] = (operation, context) => - followLogs( + runtime["service.logs.follow"] = async (operation, context) => { + const target = await ownedTarget(operation.serviceId, context, "logs"); + return followLogs( "docker", - [ - ...composePrefix(), - "logs", - "-f", - "--no-color", - `--tail=${operation.tail}`, - operation.serviceId, - ], + ["logs", "--follow", `--tail=${operation.tail}`, target.containerId], { signal: context.signal, timeout: operation.maxDurationSeconds * 1_000, @@ -544,6 +578,7 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { onLine: context.emitLog, }, ); + }; runtime["dawarich.provisioning.inspect"] = async (_operation, context) => { const statusResult = await run([...composePrefix(), "ps", "--format", "json"], context.signal); const status = parseStackStatus(statusResult.stdout); diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 171c16626..9366389d4 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -1,13 +1,10 @@ import { constants, existsSync } from "node:fs"; import { open } from "node:fs/promises"; -import { join } from "node:path"; import { loadControlStartup } from "@openmapx/core/build-artifacts"; import { readOpsTokenFile } from "@openmapx/core/ops"; import { repoPaths, services } from "@openmapx/core/server"; import { - createAdministrativeRuntime, createDefaultFixedCli, - createDefaultReleaseEffects, inspectBackupAuthority, inspectDataTypeAuthority, inspectReleaseAuthority, @@ -16,17 +13,13 @@ import { } from "./administrative-runtime"; import { isTerminalHandoff, readAgentHandoff } from "./agent-handoff"; import { startOpsAgentServer } from "./bootstrap"; -import { - CommunityRepositoryStore, - installCommunityRepositoryRuntime, -} from "./community-repositories"; +import { CommunityRepositoryStore } from "./community-repositories"; import { loadOpsAgentConfig } from "./config"; import { createDataTransitTrustedOpsDataSource } from "./data-transit-resource-source"; import { createDockerDawarichCollector, createDockerDawarichRuntimeInspector, } from "./dawarich-subject-export"; -import { createDockerRuntime } from "./docker-runtime"; import { openOpsJobJournal } from "./journal"; import { createProductionRegistryResourceClaimer } from "./policy"; import type { PrivacyBackupExtractionRouteOptions } from "./privacy-backup-extraction"; @@ -35,14 +28,17 @@ import { janitorPrivacyBackupScratch, probePrivacyBackupRuntime, } from "./privacy-backup-runtime"; +import { + createProductionOpsRuntime, + productionOwnedServices, + productionReleaseEffects, +} from "./production-runtime"; import { afterValidatedServiceAuthority, resolveBootstrapEnabledServiceIds, } from "./startup-authority"; -import { createTransitousLockRuntime } from "./transitous-lock-runtime"; import { initializeTrustedConfigurationRuntime, - installTrustedConfigurationRuntime, readTrustedEnabledServiceIds, } from "./trusted-config-runtime"; import { @@ -154,18 +150,6 @@ async function main(): Promise { builtInServices, }); const initialAuthority = await authorityLoader(); - const runtime = createDockerRuntime({ - stack: paths, - fileExists: existsSync, - trafficDataRoot: join(paths.infraDir, "data"), - }); - installTrustedConfigurationRuntime(runtime, { - services: initialAuthority.services, - integrationSchemas: initialAuthority.integrationSchemas, - loadAuthority: authorityLoader, - infraDir: paths.infraDir, - }); - installCommunityRepositoryRuntime(runtime, communityRepositories); // The managed timeline endpoint is registered only when the trusted // service selection explicitly enables `dawarich-app`. Its request path, // container name and collector script remain fixed; every call re-checks @@ -184,20 +168,14 @@ async function main(): Promise { ? createDockerDawarichCollector({ inspect: dawarichInspector }) : undefined; const administrativeCli = createDefaultFixedCli(config.rootDir); - const releaseEffects = createDefaultReleaseEffects(config.rootDir, administrativeCli); - createTransitousLockRuntime(runtime, { rootDir: config.rootDir }); - createAdministrativeRuntime(runtime, { + const runtime = await createProductionOpsRuntime({ rootDir: config.rootDir, + stack: paths, + loadAuthority: authorityLoader, + communityRepositories, runFixedCli: administrativeCli, - releaseEffects, - loadBuildAuthority: async () => - (await authorityLoader()).services.map((service) => ({ - serviceId: service.manifest.id, - enabled: service.enabled, - isBuiltIn: service.isBuiltIn, - ...(service.manifest.buildCommand ? { buildCommand: service.manifest.buildCommand } : {}), - })), }); + const releaseEffects = productionReleaseEffects(runtime); const retentionDays = Number(process.env.BACKUP_RETENTION_DAYS?.trim() || "30"); if (!Number.isSafeInteger(retentionDays) || retentionDays <= 0 || retentionDays > 36_500) { throw new Error("Invalid BACKUP_RETENTION_DAYS"); @@ -228,6 +206,13 @@ async function main(): Promise { services: initialPolicyAuthority.services, integrationIds: initialPolicyAuthority.integrationIds, loadAuthority: policyAuthority, + ownedServices: productionOwnedServices(runtime), + serviceMetadata: (authorityServices) => + authorityServices.map((service) => ({ + serviceId: service.serviceId, + isBuiltIn: service.isBuiltIn, + buildable: !!service.selectionAuthority.manifest.buildCommand, + })), enabledServiceIds: (authorityServices) => { const trusted = readTrustedEnabledServiceIds( paths.infraDir, diff --git a/apps/ops-agent/src/owned-services.test.ts b/apps/ops-agent/src/owned-services.test.ts new file mode 100644 index 000000000..de6b197f6 --- /dev/null +++ b/apps/ops-agent/src/owned-services.test.ts @@ -0,0 +1,109 @@ +import { rmSync } from "node:fs"; +import { afterEach, describe, expect, it } from "vitest"; +import { ownedServiceHost } from "./__fixtures__/owned-service-host"; +import { createDockerRuntime } from "./docker-runtime"; +import { dispatchOpsOperation } from "./runtime"; + +const roots: string[] = []; +async function fixture() { + const host = await ownedServiceHost(); + roots.push(host.root); + return host; +} +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +const signal = () => new AbortController().signal; + +describe("owned service cleanup", () => { + it("deselected owned service retains stop and logs but cannot start", async () => { + const host = await fixture(); + await host.deselect(); + const runtime = createDockerRuntime({ + stack: host.stack, + fileExists: () => false, + execFile: host.execFile, + ownedServices: host.owned, + }); + const stop = { kind: "service.stop", serviceId: "motis" } as const; + await dispatchOpsOperation(runtime, stop, await host.context(stop)); + expect(host.calls).toContainEqual(["stop", host.row.Id]); + expect(host.row.State).toBe("exited"); + const logs = { kind: "service.logs", serviceId: "motis", tail: 20 } as const; + expect((await dispatchOpsOperation(runtime, logs, await host.context(logs))).lines).toEqual([ + "retained log", + ]); + expect(host.calls).toContainEqual(["logs", "--tail=20", host.row.Id]); + for (const kind of [ + "service.start", + "service.restart", + "service.recreate", + "service.recreateIsolated", + "service.pull", + "service.update", + "service.build", + ] as const) + await expect( + host.claimer.claim({ kind, serviceId: "motis" }, "a".repeat(64), signal()), + ).resolves.toBeNull(); + }); + + it.each([ + ["unknown registry ID", "com.docker.compose.service", "unknown"], + ["same name in another project", "com.docker.compose.project", "other"], + ["forged service label", "com.docker.compose.service", "app-api"], + ["foreign working directory", "com.docker.compose.project.working_dir", "/tmp"], + ["foreign configuration file", "com.docker.compose.project.config_files", "/etc/passwd"], + ])("denies %s", async (_name, label, value) => { + const host = await fixture(); + (host.row.Labels as Record)[label] = value; + await expect(host.owned.inspectOwnedServices(signal())).resolves.toEqual([]); + await expect( + host.claimer.claim({ kind: "service.stop", serviceId: "motis" }, "a".repeat(64), signal()), + ).resolves.toBeNull(); + expect(host.calls.some((args) => args[0] === "stop")).toBe(false); + }); + + it("rejects image mismatch and ambiguous duplicate claims", async () => { + const host = await fixture(); + host.row.Image = `sha256:${"c".repeat(64)}`; + expect(await host.owned.inspectOwnedServices(signal())).toEqual([]); + host.row.Image = `sha256:${"b".repeat(64)}`; + host.setRows([host.row, { ...host.row, Id: "c".repeat(64) }]); + await expect(host.owned.inspectOwnedServices(signal())).rejects.toThrow( + "inventory unavailable", + ); + }); + + it.each(["oversized", "malformed IDs", "blank record", "malformed inspection"])( + "fails %s acquisition rather than truncating", + async (mode) => { + const host = await fixture(); + if (mode === "oversized") + host.setRawListing( + Array.from({ length: 257 }, (_, i) => i.toString(16).padStart(64, "0")).join("\n"), + ); + else if (mode === "malformed IDs") host.setRawListing("short-id"); + else if (mode === "blank record") host.setRawListing(`${host.row.Id}\n\n${"c".repeat(64)}`); + else host.setMalformedInspect(); + await expect(host.owned.inspectOwnedServices(signal())).rejects.toThrow( + "inventory unavailable", + ); + expect(host.calls.some((args) => args[0] === "stop")).toBe(false); + }, + ); + + it.each(["ID", "label", "image"])( + "rejects a replaced %s at the effect boundary", + async (mode) => { + const host = await fixture(); + const operation = { kind: "service.stop", serviceId: "motis" } as const; + const context = await host.context(operation); + if (mode === "ID") host.row.Id = "c".repeat(64); + if (mode === "label") host.row.Labels["com.docker.compose.project"] = "other"; + if (mode === "image") host.row.Image = `sha256:${"c".repeat(64)}`; + await expect(dispatchOpsOperation(host.runtime, operation, context)).rejects.toThrow(); + expect(host.calls.some((args) => args[0] === "stop")).toBe(false); + }, + ); +}); diff --git a/apps/ops-agent/src/owned-services.ts b/apps/ops-agent/src/owned-services.ts new file mode 100644 index 000000000..c982b9b53 --- /dev/null +++ b/apps/ops-agent/src/owned-services.ts @@ -0,0 +1,261 @@ +import { lstatSync, realpathSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { services } from "@openmapx/core/server"; +import type { DockerRuntimeOptions } from "./docker-runtime"; +import type { TrustedConfigurationAuthoritySnapshot } from "./trusted-config-runtime"; + +export interface OwnedServiceSnapshot { + serviceId: string; + containerId: string; + state: "running" | "stopped" | "created" | "restarting" | "paused" | "unknown"; + provenance: { + project: string; + composeWorkingDirectory: string; + composeConfigFiles: string[]; + imageId: string; + }; +} +/** Docker-host metadata provenance; it does not authenticate a hostile Docker administrator. */ +export interface OwnedServiceInventory { + services: OwnedServiceSnapshot[]; + /** Retention-only observations, never authorization claims. */ + observedContainerIds: string[]; + /** Registered candidates whose provenance could not be verified; denial only. */ + unverifiedServiceIds: string[]; +} +export interface OwnedServiceResolver { + inspectOwnedServices(signal: AbortSignal): Promise; + inspectInventory?(signal: AbortSignal): Promise; +} +const FAILED = "Owned service inventory unavailable"; +const ID = /^[a-f0-9]{64}$/; +const IMAGE = /^sha256:[a-f0-9]{64}$/; +const FORMAT = + '{"Id":{{json .Id}},"Image":{{json .Image}},"State":{{json .State.Status}},"ConfigImage":{{json .Config.Image}},"Labels":{{json .Config.Labels}}}'; + +export function createOwnedServiceResolver(options: { + stack: services.StackPaths; + execFile: NonNullable; + loadAuthority(): Promise; +}): OwnedServiceResolver { + const inspectInventory = async (signal: AbortSignal): Promise => { + signal = AbortSignal.any([signal, AbortSignal.timeout(15_000)]); + const run = async (args: string[]) => { + signal.throwIfAborted(); + const result = await options.execFile("docker", args, { + signal, + timeout: 15_000, + maxBuffer: 1024 * 1024, + }); + signal.throwIfAborted(); + if (Buffer.byteLength(result.stdout) > 1024 * 1024) throw new Error(FAILED); + return result.stdout; + }; + try { + const authority = await options.loadAuthority(); + if ( + authority.services.length > 256 || + new Set(authority.services.map((s) => s.manifest.id)).size !== authority.services.length + ) + throw new Error(FAILED); + const registry = new Map(authority.services.map((s) => [s.manifest.id, s])); + const listing = ( + await run([ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + `label=com.docker.compose.project=${services.STACK_PROJECT}`, + "--format", + "{{.ID}}", + ]) + ).trim(); + const ids = listing ? listing.split(/\r?\n/) : []; + if (ids.length > 256 || ids.some((id) => !ID.test(id)) || new Set(ids).size !== ids.length) + throw new Error(FAILED); + if (!ids.length) return { services: [], observedContainerIds: [], unverifiedServiceIds: [] }; + const rows = (await run(["container", "inspect", "--format", FORMAT, ...ids])) + .trim() + .split(/\r?\n/); + if (rows.length !== ids.length) throw new Error(FAILED); + const infra = realpathSync(options.stack.infraDir); + const configuredInfra = resolve(options.stack.infraDir); + const canonical = (path: string): string => { + if (![infra, configuredInfra].some((root) => path === root || path.startsWith(`${root}/`))) + throw new Error(FAILED); + if (!path.startsWith("/") || path !== resolve(path) || /[\0\r\n]/.test(path)) + throw new Error(FAILED); + return realpathSync(path); + }; + const found: OwnedServiceSnapshot[] = []; + const candidates = new Set(); + const images = new Map(); + const configurations = new Map>(); + for (let index = 0; index < rows.length; index++) { + const row = JSON.parse(rows[index] ?? "") as Record; + if ( + row.Id !== ids[index] || + typeof row.Image !== "string" || + !IMAGE.test(row.Image) || + typeof row.State !== "string" || + typeof row.ConfigImage !== "string" || + !row.Labels || + typeof row.Labels !== "object" || + Array.isArray(row.Labels) + ) + throw new Error(FAILED); + const labels = row.Labels as Record; + const serviceId = labels["com.docker.compose.service"]; + if ( + labels["com.docker.compose.project"] !== services.STACK_PROJECT || + typeof serviceId !== "string" || + !registry.has(serviceId) + ) + continue; + candidates.add(serviceId); + const working = labels["com.docker.compose.project.working_dir"]; + const files = labels["com.docker.compose.project.config_files"]; + if (typeof working !== "string" || typeof files !== "string" || files.length > 8192) + continue; + let configFiles: string[], workingDirectory: string; + try { + configFiles = files.split(",").map(canonical); + workingDirectory = canonical(working); + } catch { + continue; + } + const first = configFiles[0]; + const generation = first?.slice(`${infra}/${services.CONFIGURATION_GENERATIONS}/`.length); + const generationFile = + first?.startsWith(`${infra}/${services.CONFIGURATION_GENERATIONS}/`) && + /^cfg1_[A-Za-z0-9_-]{43}\/docker-compose.generated.yml$/.test(generation ?? ""); + if ( + !first || + (!generationFile && first !== join(infra, services.GENERATED_COMPOSE_FILE)) || + ![infra, dirname(first)].includes(workingDirectory) || + configFiles.length > 2 || + new Set(configFiles).size !== configFiles.length || + (configFiles[1] !== undefined && + (configFiles[1] !== join(infra, "docker-compose.release.yml") || + configFiles[1] !== canonical(options.stack.composeReleasePath))) + ) + continue; + try { + if (generationFile) + services.configurationGenerationComposeFile( + infra, + dirname(first).split("/").at(-1) ?? "", + ); + for (const file of configFiles) { + const stat = lstatSync(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.uid !== (process.geteuid?.() ?? 0) || + (stat.mode & 0o022) !== 0 || + stat.nlink !== 1 + ) + throw new Error(FAILED); + } + } catch { + continue; + } + const key = JSON.stringify(configFiles); + let configuration = configurations.get(key); + if (!configuration) { + const config = JSON.parse( + await run([ + "compose", + "--project-name", + services.STACK_PROJECT, + ...configFiles.flatMap((file) => ["-f", file]), + ...realpathEnv(infra), + "config", + "--format", + "json", + ]), + ) as { services?: Record }; + if ( + !config.services || + typeof config.services !== "object" || + Array.isArray(config.services) || + Object.keys(config.services).length > 256 || + Object.values(config.services).some( + (entry) => + !entry || + typeof entry !== "object" || + Array.isArray(entry) || + (entry.image !== undefined && typeof entry.image !== "string"), + ) + ) + throw new Error(FAILED); + configuration = config.services; + configurations.set(key, configuration); + } + // Deselection removes Compose membership, but the registered manifest + // remains authority for its effective image. Release overlays still win + // when they provide an explicit image for that service. + const registered = registry.get(serviceId); + const reference = + configuration[serviceId]?.image ?? + (registered + ? services.serviceContainerImageReference(registered.manifest.container) + : undefined); + if ( + !reference || + row.ConfigImage !== reference || + reference.length > 512 || + /[\0\r\n]/.test(reference) + ) + continue; + let imageId = images.get(reference); + if (!imageId) { + imageId = (await run(["image", "inspect", "--format", "{{.Id}}", reference])).trim(); + if (!IMAGE.test(imageId)) throw new Error(FAILED); + images.set(reference, imageId); + } + if (row.Image !== imageId) continue; + if (found.some((entry) => entry.serviceId === serviceId)) throw new Error(FAILED); + found.push({ + serviceId, + containerId: row.Id as string, + state: + row.State === "exited" || row.State === "dead" + ? "stopped" + : ["running", "stopped", "created", "restarting", "paused"].includes(row.State) + ? (row.State as OwnedServiceSnapshot["state"]) + : "unknown", + provenance: { + project: services.STACK_PROJECT, + composeWorkingDirectory: workingDirectory, + composeConfigFiles: configFiles, + imageId, + }, + }); + } + return { + services: found, + observedContainerIds: ids, + unverifiedServiceIds: [...candidates].filter( + (id) => !found.some((service) => service.serviceId === id), + ), + }; + } catch { + throw new Error(FAILED); + } + }; + return { + inspectInventory, + inspectOwnedServices: async (signal) => (await inspectInventory(signal)).services, + }; +} + +function realpathEnv(infra: string): string[] { + try { + return ["--env-file", realpathSync(join(infra, ".env"))]; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } +} diff --git a/apps/ops-agent/src/policy.test.ts b/apps/ops-agent/src/policy.test.ts index da3bfc6b4..2b3969127 100644 --- a/apps/ops-agent/src/policy.test.ts +++ b/apps/ops-agent/src/policy.test.ts @@ -8,6 +8,60 @@ import { } from "./policy"; describe("operation-specific authority policy", () => { + it("deselected owned service retains stop and logs but cannot start", async () => { + const claimer = createRegistryResourceClaimer({ + serviceIds: ["worker", "ops-agent", "app-api", "traefik"], + enabledServiceIds: () => new Set(), + integrationIds: [], + trustedData: denyAllTrustedOpsData, + ownedServices: { + inspectOwnedServices: async () => + ["worker", "ops-agent"].map((serviceId) => ({ + serviceId, + containerId: "a".repeat(64), + state: "running", + provenance: { + project: "docker", + composeWorkingDirectory: "/trusted", + composeConfigFiles: ["/trusted/docker-compose.generated.yml"], + imageId: `sha256:${"b".repeat(64)}`, + }, + })), + }, + }); + const signal = new AbortController().signal; + for (const kind of ["service.stop", "service.logs", "service.logs.follow"] as const) { + const operation = { kind, serviceId: "worker", tail: 20, maxDurationSeconds: 1 }; + await expect(claimer.claim(operation, "a".repeat(64), signal)).resolves.not.toBeNull(); + } + for (const kind of [ + "service.start", + "service.restart", + "service.recreate", + "service.recreateIsolated", + "service.pull", + "service.update", + "service.build", + ] as const) { + await expect( + claimer.claim({ kind, serviceId: "worker" }, "b".repeat(64), signal), + ).resolves.toBeNull(); + } + await expect( + claimer.claim( + { kind: "service.logs", serviceId: "ops-agent", tail: 20 }, + "c".repeat(64), + signal, + ), + ).resolves.not.toBeNull(); + await expect( + claimer.claim({ kind: "service.stop", serviceId: "ops-agent" }, "d".repeat(64), signal), + ).resolves.toBeNull(); + await expect( + claimer.claim({ kind: "service.stop", serviceId: "unknown" }, "e".repeat(64), signal), + ).resolves.toBeNull(); + }); + it("uses agent-owned backup authority without consulting caller-published trusted data", async () => { const trustedClaims: string[] = []; const backupClaims: string[] = []; @@ -173,10 +227,10 @@ describe("operation-specific authority policy", () => { }); it("refreshes custom service and integration authority without restarting the agent", async () => { - let enabled = new Set(["redis"]); + let enabled = new Set(["worker"]); let authority = { revisionId: "authority-initial", - services: [{ serviceId: "redis", enabled: true, isBuiltIn: true }], + services: [{ serviceId: "worker", enabled: true, isBuiltIn: true }], integrationIds: [] as string[], }; const claimer = createProductionRegistryResourceClaimer({ @@ -197,12 +251,12 @@ describe("operation-specific authority policy", () => { authority = { revisionId: "authority-installed", services: [ - { serviceId: "redis", enabled: true, isBuiltIn: true }, + { serviceId: "worker", enabled: true, isBuiltIn: true }, { serviceId: "community-live", enabled: true, isBuiltIn: false }, ], integrationIds: ["integration-live"], }; - enabled = new Set(["redis", "community-live"]); + enabled = new Set(["worker", "community-live"]); await expect( claimer.claim({ kind: "service.start", serviceId: "community-live" }, "b".repeat(64), signal), ).resolves.not.toBeNull(); @@ -223,10 +277,10 @@ describe("operation-specific authority policy", () => { authority = { revisionId: "authority-removed", - services: [{ serviceId: "redis", enabled: true, isBuiltIn: true }], + services: [{ serviceId: "worker", enabled: true, isBuiltIn: true }], integrationIds: [], }; - enabled = new Set(["redis"]); + enabled = new Set(["worker"]); await expect( claimer.claim({ kind: "service.start", serviceId: "community-live" }, "d".repeat(64), signal), ).resolves.toBeNull(); @@ -247,11 +301,11 @@ describe("operation-specific authority policy", () => { it("rolls back the exact pre-admission snapshot when refreshed authority rejects its binding", async () => { let rollbacks = 0; const claimer = createProductionRegistryResourceClaimer({ - services: [{ serviceId: "redis", enabled: true, isBuiltIn: true }], + services: [{ serviceId: "worker", enabled: true, isBuiltIn: true }], integrationIds: [], loadAuthority: async () => ({ revisionId: "authority-current", - services: [{ serviceId: "redis", enabled: true, isBuiltIn: true }], + services: [{ serviceId: "worker", enabled: true, isBuiltIn: true }], integrationIds: [], }), trustedData: { @@ -286,10 +340,10 @@ describe("operation-specific authority policy", () => { }); it("tracks committed generation enable/disable state for lifecycle authority across restart", async () => { - let enabled = new Set(["redis", "app-api", "traefik", "ops-agent"]); + let enabled = new Set(["worker", "app-api", "traefik", "ops-agent"]); const claimer = createProductionRegistryResourceClaimer({ services: [ - { serviceId: "redis", enabled: true, isBuiltIn: true }, + { serviceId: "worker", enabled: true, isBuiltIn: true }, { serviceId: "motis", enabled: false, isBuiltIn: true }, { serviceId: "community-enabled", enabled: true, isBuiltIn: false }, { serviceId: "community-disabled", enabled: false, isBuiltIn: false }, @@ -302,10 +356,7 @@ describe("operation-specific authority policy", () => { enabledServiceIds: () => enabled, }); const signal = new AbortController().signal; - const operations: OpsOperation[] = [ - { kind: "service.restart", serviceId: "redis" }, - { kind: "service.logs", serviceId: "redis", tail: 20 }, - ]; + const operations: OpsOperation[] = [{ kind: "service.start", serviceId: "worker" }]; for (const operation of operations) { await expect(claimer.claim(operation, "a".repeat(64), signal)).resolves.not.toBeNull(); for (const serviceId of [ @@ -325,12 +376,12 @@ describe("operation-specific authority policy", () => { claimer.claim({ kind: "service.start", serviceId: "motis" }, "c".repeat(64), signal), ).resolves.not.toBeNull(); await expect( - claimer.claim({ kind: "service.start", serviceId: "redis" }, "d".repeat(64), signal), + claimer.claim({ kind: "service.start", serviceId: "worker" }, "d".repeat(64), signal), ).resolves.toBeNull(); // A reconstructed claimer reads the same recovered generation-owned set. const restarted = createProductionRegistryResourceClaimer({ services: [ - { serviceId: "redis", enabled: true, isBuiltIn: true }, + { serviceId: "worker", enabled: true, isBuiltIn: true }, { serviceId: "motis", enabled: false, isBuiltIn: true }, ], integrationIds: [], @@ -344,12 +395,12 @@ describe("operation-specific authority policy", () => { it("excludes the agent, edge proxy, and self-replaced API from generic lifecycle", async () => { const policy = createRegistryResourcePolicy({ - serviceIds: ["redis", "ops-agent", "traefik", "app-api"], + serviceIds: ["worker", "ops-agent", "traefik", "app-api"], integrationIds: [], trustedData: denyAllTrustedOpsData, }); await expect( - authorizeOpsResources({ kind: "service.restart", serviceId: "redis" }, policy), + authorizeOpsResources({ kind: "service.start", serviceId: "worker" }, policy), ).resolves.toBe(true); for (const serviceId of ["ops-agent", "traefik", "app-api"]) { await expect( @@ -401,7 +452,7 @@ describe("operation-specific authority policy", () => { it("reapplies registry and never-manage constraints after a trusted complete-operation claim", async () => { let sourceCalls = 0; const claimer = createRegistryResourceClaimer({ - serviceIds: ["redis", "motis"], + serviceIds: ["worker", "motis"], integrationIds: ["routing-motis"], trustedData: { claim: async () => { @@ -421,7 +472,7 @@ describe("operation-specific authority policy", () => { { kind: "backup.restore", backupId: "backup_1", - serviceIds: ["redis", "traefik"], + serviceIds: ["worker", "traefik"], }, { expectedCurrentRevisionId: null, diff --git a/apps/ops-agent/src/policy.ts b/apps/ops-agent/src/policy.ts index 194c0e4b6..ce202b295 100644 --- a/apps/ops-agent/src/policy.ts +++ b/apps/ops-agent/src/policy.ts @@ -1,10 +1,13 @@ +import type { ServiceAction } from "@openmapx/core/ops"; import { authorizeOpsResources, type OpsOperation, type OpsOperationKind, type OpsResourcePolicy, } from "@openmapx/core/ops"; +import type { OwnedServiceResolver } from "./owned-services"; import type { OpsTrustedClaim } from "./runtime"; +import { decideServiceAction } from "./service-action-policy"; export interface OpsClaimOwner { role: "api" | "data-manager"; @@ -40,23 +43,36 @@ export const denyAllTrustedOpsData: TrustedOpsDataSource = { claim: async () => null, }; -const genericLifecycleKinds = new Set([ - "service.start", - "service.stop", - "service.restart", - "service.recreate", - "service.recreateIsolated", - "service.pull", - "service.remove", - "service.update", - "service.build", - "service.logs", - "service.logs.follow", -]); -const neverGenericLifecycle = new Set(["ops-agent", "traefik", "app-api"]); +export function serviceActionForOperation(kind: OpsOperationKind): ServiceAction | null { + switch (kind) { + case "service.start": + return "start"; + case "service.stop": + return "stop"; + case "service.restart": + return "restart"; + case "service.recreate": + case "service.recreateIsolated": + case "service.remove": + return "apply"; + case "service.pull": + return "pull"; + case "service.update": + return "update"; + case "service.build": + return "build"; + case "service.logs": + case "service.logs.follow": + return "logs"; + default: + return null; + } +} export function createRegistryResourcePolicy(options: { serviceIds: Iterable; + ownedServices?: OwnedServiceResolver; + serviceMetadata?: readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; enabledServiceIds?: () => ReadonlySet; integrationIds: Iterable; trustedData: TrustedOpsDataSource; @@ -75,11 +91,30 @@ export function createRegistryResourcePolicy(options: { const integrations = new Set(options.integrationIds); return { allowGlobal: () => true, - allowService: (kind, serviceId) => - services.has(serviceId) && - (!genericLifecycleKinds.has(kind) || - (options.enabledServiceIds?.().has(serviceId) ?? true)) && - !(genericLifecycleKinds.has(kind) && neverGenericLifecycle.has(serviceId)), + allowService: async (kind, serviceId) => { + if (!services.has(serviceId)) return false; + const action = serviceActionForOperation(kind); + if (!action) return true; + const snapshot = options.ownedServices + ? (await options.ownedServices.inspectOwnedServices(AbortSignal.timeout(15_000))).find( + (s) => s.serviceId === serviceId, + ) + : undefined; + const service = options.serviceMetadata?.find((s) => s.serviceId === serviceId) ?? { + serviceId, + isBuiltIn: true, + buildable: true, + }; + return decideServiceAction( + { + revisionId: "registry-v1", + services: [service], + enabledServiceIds: options.enabledServiceIds?.() ?? services, + }, + snapshot, + action, + ).available; + }, allowBackup: options.resourceAuthority?.allowBackup ?? (() => false), allowPreparedRun: () => false, allowCandidate: () => false, @@ -211,6 +246,8 @@ export function createPolicyResourceClaimer(policy: OpsResourcePolicy): OpsResou export function createRegistryResourceClaimer(options: { serviceIds: Iterable; + ownedServices?: OwnedServiceResolver; + serviceMetadata?: readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; enabledServiceIds?: () => ReadonlySet; integrationIds: Iterable; trustedData: TrustedOpsDataSource; @@ -226,11 +263,47 @@ export function createRegistryResourceClaimer(options: { | "allowUpdateJobId" >; }): OpsResourceClaimer { - const policy = createRegistryResourcePolicy(options); + const registeredServiceIds = new Set(options.serviceIds); + const policy = createRegistryResourcePolicy({ ...options, serviceIds: registeredServiceIds }); return { claim: async (operation, fingerprint, signal, owner) => { if (signal.aborted) throw signal.reason; if (!requiresTrustedSource(operation)) { + const action = serviceActionForOperation(operation.kind); + if ("serviceId" in operation && action && options.ownedServices) { + const snapshot = (await options.ownedServices.inspectOwnedServices(signal)).find( + (s) => s.serviceId === operation.serviceId, + ); + const service = options.serviceMetadata?.find( + (s) => s.serviceId === operation.serviceId, + ) ?? { serviceId: operation.serviceId, isBuiltIn: true, buildable: true }; + if ( + !registeredServiceIds.has(operation.serviceId) || + !decideServiceAction( + { + revisionId: options.registryRevision ?? "registry-v1", + services: [service], + enabledServiceIds: options.enabledServiceIds?.() ?? registeredServiceIds, + }, + snapshot, + action, + ).available + ) + return null; + if ( + !(await authorizeOpsResources(operation, { + ...policy, + allowService: (_kind, id) => id === operation.serviceId, + })) + ) + return null; + signal.throwIfAborted(); + // The admission is bound to the immutable ID and full verified provenance. + return immutableClaim(operation, fingerprint, "registry", { + revisionId: options.registryRevision ?? "registry-v1", + values: snapshot ? { ownedService: JSON.stringify(snapshot) } : {}, + }); + } return (await authorizeOpsResources(operation, policy)) ? immutableClaim( operation, @@ -285,6 +358,10 @@ export function createProductionRegistryResourceClaimer< Service extends { serviceId: string; enabled: boolean; isBuiltIn: boolean }, >(options: { services: Iterable; + ownedServices?: OwnedServiceResolver; + serviceMetadata?: ( + services: readonly Service[], + ) => readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; integrationIds: Iterable; trustedData: TrustedOpsDataSource; resourceAuthority?: Pick< @@ -312,6 +389,8 @@ export function createProductionRegistryResourceClaimer< if (!options.loadAuthority) { return createRegistryResourceClaimer({ serviceIds, + ownedServices: options.ownedServices, + serviceMetadata: options.serviceMetadata?.(builtIns), enabledServiceIds: () => options.enabledServiceIds?.(builtIns) ?? initialEnabled, integrationIds: options.integrationIds, trustedData: options.trustedData, @@ -362,6 +441,8 @@ export function createProductionRegistryResourceClaimer< }; return createRegistryResourceClaimer({ serviceIds: authority.services.map((service) => service.serviceId), + ownedServices: options.ownedServices, + serviceMetadata: options.serviceMetadata?.(authority.services), enabledServiceIds: () => options.enabledServiceIds?.(authority.services) ?? initialEnabled, integrationIds: authority.integrationIds, trustedData, diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts new file mode 100644 index 000000000..7dfb73ac4 --- /dev/null +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -0,0 +1,335 @@ +import { randomBytes } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey, type OpsOperation } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { afterEach, describe, expect, it } from "vitest"; +import { ownedServiceHost } from "./__fixtures__/owned-service-host"; +import { openOpsJobJournal } from "./journal"; +import { dispatchOpsOperation } from "./runtime"; +import { buildOpsAgentServer } from "./server"; +import { initializeTrustedConfigurationRuntime } from "./trusted-config-runtime"; + +const roots: string[] = []; +async function fixture() { + const host = await ownedServiceHost(); + roots.push(host.root); + return host; +} +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("assembled production service authority", () => { + it("executes Stop and post-stop Logs for a deselected owned container through the installed adapters", async () => { + const host = await fixture(); + await host.deselect(); + expect(host.calls.some((args) => args[0] === "stop")).toBe(false); + const inspect = { kind: "services.inspect" } as const; + const before = await dispatchOpsOperation(host.runtime, inspect, await host.context(inspect)); + const service = before.services.find((s) => s.serviceId === "motis"); + expect(service?.actions.stop.available).toBe(true); + expect(service?.actions.logs.available).toBe(true); + for (const action of ["start", "restart", "update", "build", "pull", "apply"] as const) + expect(service?.actions[action]).toMatchObject({ available: false, reason: "disabled" }); + const stop = { kind: "service.stop", serviceId: "motis" } as const; + await dispatchOpsOperation(host.runtime, stop, await host.context(stop)); + const logs = { kind: "service.logs", serviceId: "motis", tail: 20 } as const; + expect(await dispatchOpsOperation(host.runtime, logs, await host.context(logs))).toEqual({ + lines: ["retained log"], + truncated: false, + }); + expect(host.calls).toContainEqual(["stop", host.row.Id]); + expect(host.calls).toContainEqual(["logs", "--tail=20", host.row.Id]); + const follow = { + kind: "service.logs.follow", + serviceId: "motis", + tail: 20, + maxDurationSeconds: 1, + } as const; + await expect( + dispatchOpsOperation(host.runtime, follow, await host.context(follow)), + ).resolves.toEqual({ lines: 1, truncated: false }); + expect(host.calls).toContainEqual(["logs", "--follow", "--tail=20", host.row.Id]); + expect(host.calls.some((args) => args[0] === "compose" && args.includes("stop"))).toBe(false); + expect(existsSync(host.firstCompose)).toBe(true); + expect((await host.loadAuthority()).services.some((s) => s.manifest.id === "motis")).toBe(true); + }); + + it("authorizes deselected Stop and post-stop Logs through authenticated HTTP with original control authority", async () => { + const host = await fixture(); + await host.deselect(); + mkdirSync(join(host.infraDir, "data"), { mode: 0o700 }); + await provisionControlInstallation(host.infraDir); + const installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + const headers = { authorization: `Bearer ${host.token}`, "x-ops-protocol-version": "2" }; + const requestedDeadlineAt = Date.now() + 60_000; + let count = 0; + const submit = (operation: OpsOperation) => + app.inject({ + method: "POST", + url: "/v1/operations", + headers, + payload: { + version: 2, + requestId: `ops1_cleanupHttp${String(++count).padStart(12, "0")}`, + operationKey: createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20_000).toISOString(), + execution: { requestedDeadlineAt }, + operation, + }, + }); + try { + const admitted = await submit({ kind: "service.stop", serviceId: "motis" }); + expect(admitted.statusCode, admitted.body).toBe(202); + const operationId = admitted.json().result.operationId; + let status: { state: string; result?: unknown } = { state: "pending" }; + const until = Date.now() + 3_000; + do { + status = ( + await app.inject({ method: "GET", url: `/v1/operations/${operationId}`, headers }) + ).json().result; + if (status.state !== "succeeded") await new Promise((resolve) => setTimeout(resolve, 10)); + } while (status.state !== "succeeded" && Date.now() < until); + expect(status).toMatchObject({ state: "succeeded", result: { changed: true } }); + expect(journal.records()).toHaveLength(1); + const logs = await submit({ kind: "service.logs", serviceId: "motis", tail: 20 }); + expect(logs.statusCode, logs.body).toBe(200); + expect(logs.json().result.value.lines).toEqual(["retained log"]); + expect(host.calls).toContainEqual(["stop", host.row.Id]); + expect(host.calls).toContainEqual(["logs", "--tail=20", host.row.Id]); + } finally { + await app.close(); + closeControlInstallation(installation); + } + }); + + it("preserves legacy retained generations at startup before the first owned inventory acquisition", async () => { + const host = await fixture(); + for (let i = 0; i < 15; i++) + await services.commitConfigurationGeneration({ + infraDir: host.infraDir, + services: host.loaded, + input: { + domain: "example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + }); + // Deployments predating owned retention have complete generations but no new metadata file. + rmSync(join(host.infraDir, ".owned-service-generations.json")); + await initializeTrustedConfigurationRuntime(host.infraDir); + expect(existsSync(host.firstCompose)).toBe(true); + const logs = { kind: "service.logs", serviceId: "motis", tail: 20 } as const; + await expect( + dispatchOpsOperation(host.runtime, logs, await host.context(logs)), + ).resolves.toMatchObject({ lines: ["retained log"] }); + }); + + it("retains the original generation for a live legacy current-alias container and releases it only after cleanup", async () => { + const host = await fixture(); + host.row.Labels["com.docker.compose.project.config_files"] = host.stack.composePath; + host.row.Labels["com.docker.compose.project.working_dir"] = host.stack.composePath.slice( + 0, + host.stack.composePath.lastIndexOf("/"), + ); + expect(await host.deselect()).toMatchObject({ outcome: "committed" }); + for (let i = 0; i < 18; i++) + expect(await host.select([])).toMatchObject({ outcome: "committed" }); + expect(existsSync(host.firstCompose)).toBe(true); + const logs = { kind: "service.logs", serviceId: "motis", tail: 20 } as const; + await expect( + dispatchOpsOperation(host.runtime, logs, await host.context(logs)), + ).resolves.toMatchObject({ lines: ["retained log"] }); + host.row.Image = `sha256:${"c".repeat(64)}`; + expect(await host.select([])).toMatchObject({ outcome: "committed" }); + expect(existsSync(host.firstCompose)).toBe(true); + host.setRows([]); + expect(await host.select([])).toMatchObject({ outcome: "committed" }); + expect(existsSync(host.firstCompose)).toBe(false); + await expect(host.context(logs)).rejects.toThrow("Admission denied"); + }); + + it("rechecks deselection after admission before invoking a lifecycle effect", async () => { + const host = await fixture(); + const operation = { kind: "service.start", serviceId: "motis" } as const; + const context = await host.context(operation); + await host.deselect(); + await expect(dispatchOpsOperation(host.runtime, operation, context)).rejects.toThrow( + "Service action denied", + ); + expect(host.calls.some((args) => args.includes("up"))).toBe(false); + }); + + it("rechecks the admitted immutable ID before an enabled Restart", async () => { + const host = await fixture(); + const operation = { kind: "service.restart", serviceId: "motis" } as const; + const ctx = await host.context(operation); + host.row.Id = "c".repeat(64); + await expect(dispatchOpsOperation(host.runtime, operation, ctx)).rejects.toThrow( + "Owned service claim changed", + ); + expect(host.calls.some((args) => args.includes("restart"))).toBe(false); + }); + + it("keeps core workflows unavailable and dependency lifecycle host-only while permitting verified core logs", async () => { + const host = await fixture(); + host.row.Labels["com.docker.compose.service"] = "ops-agent"; + host.row.ConfigImage = "example/ops-agent:1"; + const inspect = { kind: "services.inspect" } as const; + const result = await dispatchOpsOperation(host.runtime, inspect, await host.context(inspect)); + const agent = result.services.find((s) => s.serviceId === "ops-agent"); + expect(agent?.actions.logs.available).toBe(true); + expect(agent?.actions.stop).toMatchObject({ available: false, reason: "control_plane" }); + for (const id of ["app-api", "traefik"]) + expect(result.services.find((s) => s.serviceId === id)?.actions.restart).toMatchObject({ + available: false, + execution: "workflow", + reason: "unavailable", + workflow: "core-restart", + }); + for (const id of ["redis", "postgis"]) + expect(result.services.find((s) => s.serviceId === id)?.actions.stop).toMatchObject({ + available: false, + execution: "host", + reason: "host_only", + }); + const logs = { kind: "service.logs", serviceId: "ops-agent", tail: 20 } as const; + await expect( + dispatchOpsOperation(host.runtime, logs, await host.context(logs)), + ).resolves.toMatchObject({ lines: ["retained log"] }); + }); + + it("refuses installed community removal until its stopped owned container is removed", async () => { + const host = await fixture(); + const published = await host.repositories.publish( + { + kind: "serviceRepository.publish", + url: "https://github.com/example/services", + commit: "1".repeat(40), + extensionId: "test-extension", + recovery: { + selectionBefore: ["motis"], + touchedServiceIds: ["community-live"], + previouslyEnabledServiceIds: [], + }, + }, + new AbortController().signal, + ); + await host.repositories.settle({ + kind: "serviceRepository.settle", + repository: published.repository, + transactionId: published.transactionId, + outcome: "commit", + }); + host.row.Labels["com.docker.compose.service"] = "community-live"; + host.row.ConfigImage = `example/community-live:1@sha256:${"b".repeat(64)}`; + host.row.State = "exited"; + const remove = { + kind: "serviceRepository.remove", + repository: published.repository, + extensionId: "test-extension", + } as const; + await expect( + dispatchOpsOperation(host.runtime, remove, await host.context(remove)), + ).rejects.toThrow("require cleanup"); + expect( + (await host.loadAuthority()).services.some((s) => s.manifest.id === "community-live"), + ).toBe(true); + host.row.Image = `sha256:${"c".repeat(64)}`; + await expect( + dispatchOpsOperation(host.runtime, remove, await host.context(remove)), + ).rejects.toThrow("provenance requires host cleanup"); + host.setMalformedInspect(); + await expect( + dispatchOpsOperation(host.runtime, remove, await host.context(remove)), + ).rejects.toThrow("inventory unavailable"); + host.setRows([]); + await expect( + dispatchOpsOperation(host.runtime, remove, await host.context(remove)), + ).resolves.toMatchObject({ removed: true }); + expect( + (await host.loadAuthority()).services.some((s) => s.manifest.id === "community-live"), + ).toBe(false); + }); + + it("uses API-role synchronous inspection and rejects unauthenticated/data-manager requests without Docker I/O", async () => { + const host = await fixture(); + const api = Buffer.alloc(32, 31).toString("base64url"), + dm = Buffer.alloc(32, 32).toString("base64url"); + const app = buildOpsAgentServer({ + tokens: { api, "data-manager": dm }, + runtime: host.runtime, + resourceClaimer: host.claimer, + }); + const payload = { + version: 2, + requestId: "ops1_inspectionRole0000", + operationKey: "opk1_inspectionRole0000", + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20_000).toISOString(), + operation: { kind: "services.inspect" }, + }; + const before = host.calls.length; + try { + for (const token of [undefined, dm]) { + const res = await app.inject({ + method: "POST", + url: "/v1/operations", + payload, + headers: { + "x-ops-protocol-version": "2", + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + }); + expect(res.statusCode).toBe(token ? 403 : 401); + } + expect(host.calls).toHaveLength(before); + const result = await app.inject({ + method: "POST", + url: "/v1/operations", + payload, + headers: { "x-ops-protocol-version": "2", authorization: `Bearer ${api}` }, + }); + expect(result.statusCode).toBe(200); + expect(result.json().result.value.services).toHaveLength(host.loaded.length); + } finally { + await app.close(); + } + }); + + it("keeps the production DM source, startup ordering and all installer calls in the actual entry point", () => { + const entry = readFileSync(new URL("./index.ts", import.meta.url), "utf8"); + expect(entry).toContain("createProductionOpsRuntime({"); + expect(entry).toContain( + "createDataTransitTrustedOpsDataSource(\n controlStartup.installation,", + ); + expect(entry).toContain("createFileTrustedOpsDataSource({"); + expect(entry.indexOf("await communityRepositories.recover()")).toBeLessThan( + entry.indexOf("const controlStartup ="), + ); + }); +}); diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts new file mode 100644 index 000000000..39b171ee6 --- /dev/null +++ b/apps/ops-agent/src/production-runtime.ts @@ -0,0 +1,190 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { type OpsOperation, SERVICE_ACTIONS, type ServiceActionDecision } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { + type AdministrativeReleaseEffects, + createAdministrativeRuntime, + createDefaultFixedCli, + createDefaultReleaseEffects, + type FixedCliRunner, +} from "./administrative-runtime"; +import { + type CommunityRepositoryStore, + installCommunityRepositoryRuntime, +} from "./community-repositories"; +import { + createDockerRuntime, + type DockerRuntimeOptions, + runContainedProcess, +} from "./docker-runtime"; +import { createOwnedServiceResolver, type OwnedServiceResolver } from "./owned-services"; +import { serviceActionForOperation } from "./policy"; +import type { OpsExecutionContext, OpsRuntime } from "./runtime"; +import { decideServiceAction, type ServiceAuthority } from "./service-action-policy"; +import { resolveBootstrapEnabledServiceIds } from "./startup-authority"; +import { createTransitousLockRuntime } from "./transitous-lock-runtime"; +import { + installTrustedConfigurationRuntime, + readTrustedEnabledServiceIds, + type TrustedConfigurationAuthoritySnapshot, +} from "./trusted-config-runtime"; + +export interface ProductionOpsRuntimeOptions { + rootDir: string; + stack: services.StackPaths; + loadAuthority(): Promise; + communityRepositories: CommunityRepositoryStore; + docker?: Pick; + runFixedCli?: FixedCliRunner; +} +const resolvers = new WeakMap(); +const releaseAdapters = new WeakMap(); +export function productionReleaseEffects(runtime: OpsRuntime): AdministrativeReleaseEffects { + const effects = releaseAdapters.get(runtime); + if (!effects) throw new Error("Production release adapter missing"); + return effects; +} +export function productionOwnedServices(runtime: OpsRuntime): OwnedServiceResolver { + const resolver = resolvers.get(runtime); + if (!resolver) throw new Error("Production ownership resolver missing"); + return resolver; +} + +export function serviceActionAuthority( + authority: TrustedConfigurationAuthoritySnapshot, + infraDir: string, +): ServiceAuthority { + return { + revisionId: authority.revisionId, + enabledServiceIds: + readTrustedEnabledServiceIds(infraDir, authority.services) ?? + resolveBootstrapEnabledServiceIds( + authority.services, + services.readDesiredSelection(infraDir), + ), + services: authority.services.map((service) => ({ + serviceId: service.manifest.id, + isBuiltIn: service.isBuiltIn, + buildable: !!service.manifest.buildCommand, + })), + }; +} + +/** The production installer order; test seams are external I/O, never lifecycle handlers. */ +export async function createProductionOpsRuntime( + options: ProductionOpsRuntimeOptions, +): Promise { + const initial = await options.loadAuthority(); + const owned = createOwnedServiceResolver({ + stack: options.stack, + execFile: options.docker?.execFile ?? runContainedProcess, + loadAuthority: options.loadAuthority, + }); + const runtime = createDockerRuntime({ + stack: options.stack, + fileExists: options.docker?.fileExists ?? existsSync, + execFile: options.docker?.execFile, + followLogs: options.docker?.followLogs, + trafficDataRoot: join(options.stack.infraDir, "data"), + ownedServices: owned, + }); + installTrustedConfigurationRuntime(runtime, { + services: initial.services, + integrationSchemas: initial.integrationSchemas, + loadAuthority: options.loadAuthority, + infraDir: options.stack.infraDir, + ownedServices: owned, + }); + installCommunityRepositoryRuntime(runtime, options.communityRepositories, owned); + const runFixedCli = options.runFixedCli ?? createDefaultFixedCli(options.rootDir); + const releaseEffects = createDefaultReleaseEffects(options.rootDir, runFixedCli); + createTransitousLockRuntime(runtime, { rootDir: options.rootDir }); + createAdministrativeRuntime(runtime, { + rootDir: options.rootDir, + runFixedCli, + releaseEffects, + loadBuildAuthority: async () => { + const authority = await options.loadAuthority(); + const configured = serviceActionAuthority(authority, options.stack.infraDir); + return authority.services.map((service) => ({ + serviceId: service.manifest.id, + enabled: configured.enabledServiceIds.has(service.manifest.id), + isBuiltIn: service.isBuiltIn, + ...(service.manifest.buildCommand ? { buildCommand: service.manifest.buildCommand } : {}), + })); + }, + }); + runtime["services.inspect"] = async (_operation, context) => { + const authority = serviceActionAuthority(await options.loadAuthority(), options.stack.infraDir); + const snapshots = await owned.inspectOwnedServices(context.signal); + return { + services: authority.services.map((service) => { + const snapshot = snapshots.find((entry) => entry.serviceId === service.serviceId); + const scoped = { ...authority, services: [service] }; + const actions = Object.fromEntries( + SERVICE_ACTIONS.map((action) => [action, decideServiceAction(scoped, snapshot, action)]), + ) as Record<(typeof SERVICE_ACTIONS)[number], ServiceActionDecision>; + return { serviceId: service.serviceId, state: snapshot?.state ?? "stopped", actions }; + }), + }; + }; + // Admission is not the final authorization boundary. Refresh selection and provenance at execution. + for (const kind of [ + "service.start", + "service.stop", + "service.restart", + "service.recreate", + "service.recreateIsolated", + "service.pull", + "service.remove", + "service.update", + "service.build", + "service.logs", + "service.logs.follow", + ] as const) { + const action = serviceActionForOperation(kind); + if (!action) throw new Error("Service action mapping missing"); + const handler = runtime[kind] as ( + operation: OpsOperation, + context: OpsExecutionContext, + ) => Promise; + const guarded = async ( + operation: Extract, + context: OpsExecutionContext, + ) => { + context.signal.throwIfAborted(); + const authority = serviceActionAuthority( + await options.loadAuthority(), + options.stack.infraDir, + ); + const snapshot = (await owned.inspectOwnedServices(context.signal)).find( + (s) => s.serviceId === operation.serviceId, + ); + const encoded = context.claim.capability.values.ownedService; + if (typeof encoded === "string") { + const admitted = JSON.parse(encoded) as { containerId?: string; provenance?: unknown }; + if ( + !snapshot || + admitted.containerId !== snapshot.containerId || + JSON.stringify(admitted.provenance) !== JSON.stringify(snapshot.provenance) + ) + throw new Error("Owned service claim changed"); + } + const service = authority.services.find((s) => s.serviceId === operation.serviceId); + const decision = decideServiceAction( + { ...authority, services: service ? [service] : [] }, + snapshot, + action, + ); + if (!decision.available || decision.execution !== "direct") + throw new Error("Service action denied"); + return handler(operation, context); + }; + // The finite loop preserves each handler's operation/result correspondence. + Object.assign(runtime, { [kind]: guarded }); + } + resolvers.set(runtime, owned); + releaseAdapters.set(runtime, releaseEffects); + return runtime; +} diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 7a2545a33..ec3c63e7a 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -128,6 +128,7 @@ export const OPS_AGENT_RETENTION_LIMITS = { const RELEASE_RECOVERY_READ_KINDS = new Set([ "docker.status", "stack.status", + "services.inspect", "configuration.inspect", "configuration.hardlinks.inspect", "service.logs", diff --git a/apps/ops-agent/src/service-action-policy.test.ts b/apps/ops-agent/src/service-action-policy.test.ts new file mode 100644 index 000000000..c66c70d96 --- /dev/null +++ b/apps/ops-agent/src/service-action-policy.test.ts @@ -0,0 +1,53 @@ +import { SERVICE_ACTIONS, serviceActionDecisionSchema } from "@openmapx/core/ops"; +import { describe, expect, it } from "vitest"; +import type { OwnedServiceSnapshot } from "./owned-services"; +import { decideServiceAction, type ServiceAuthority } from "./service-action-policy"; + +const authority: ServiceAuthority = { + revisionId: "test", + enabledServiceIds: new Set(["motis"]), + services: [{ serviceId: "motis", isBuiltIn: true, buildable: true }], +}; +const snapshot: OwnedServiceSnapshot = { + serviceId: "motis", + containerId: "a".repeat(64), + state: "running", + provenance: { + project: "docker", + composeWorkingDirectory: "/trusted", + composeConfigFiles: ["/trusted/docker-compose.generated.yml"], + imageId: `sha256:${"b".repeat(64)}`, + }, +}; +describe("shared service action decisions", () => { + it("keeps every available direct decision consistent with its portable schema", () => { + for (const action of SERVICE_ACTIONS) + expect( + serviceActionDecisionSchema.safeParse(decideServiceAction(authority, snapshot, action)) + .success, + ).toBe(true); + expect(decideServiceAction(authority, snapshot, "update")).toMatchObject({ + available: false, + reason: "unavailable", + }); + }); + it("retains logs after stop, requires proven ownership for cleanup and refuses unknown registry IDs", () => { + expect( + decideServiceAction(authority, { ...snapshot, state: "stopped" }, "logs").available, + ).toBe(true); + expect(decideServiceAction(authority, { ...snapshot, state: "stopped" }, "stop").reason).toBe( + "not_running", + ); + expect(decideServiceAction(authority, undefined, "stop").reason).toBe("not_owned"); + expect( + decideServiceAction(authority, { ...snapshot, serviceId: "unknown" }, "logs").reason, + ).toBe("not_owned"); + expect( + decideServiceAction( + { ...authority, services: [{ serviceId: "motis", isBuiltIn: true, buildable: false }] }, + snapshot, + "build", + ).reason, + ).toBe("not_buildable"); + }); +}); diff --git a/apps/ops-agent/src/service-action-policy.ts b/apps/ops-agent/src/service-action-policy.ts new file mode 100644 index 000000000..bd8462e27 --- /dev/null +++ b/apps/ops-agent/src/service-action-policy.ts @@ -0,0 +1,51 @@ +import type { ServiceAction, ServiceActionDecision } from "@openmapx/core/ops"; +import type { OwnedServiceSnapshot } from "./owned-services"; + +export interface ServiceAuthority { + revisionId: string; + enabledServiceIds: ReadonlySet; + services: readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; +} + +const controlPlane = new Set(["ops-agent", "app-api", "traefik"]); +const dependencies = new Set(["postgis", "redis"]); + +/** With no container, callers scope services to the single target being decided. */ +export function decideServiceAction( + authority: ServiceAuthority, + snapshot: OwnedServiceSnapshot | undefined, + action: ServiceAction, +): ServiceActionDecision { + const service = snapshot + ? authority.services.find((entry) => entry.serviceId === snapshot.serviceId) + : authority.services.length === 1 + ? authority.services[0] + : undefined; + const denied = ( + reason: NonNullable, + execution: ServiceActionDecision["execution"] = "direct", + workflow: ServiceActionDecision["workflow"] = null, + ): ServiceActionDecision => ({ available: false, execution, reason, workflow }); + if (!service) return denied("not_owned"); + if (action === "logs") + return snapshot + ? { available: true, execution: "direct", reason: null, workflow: null } + : denied("not_owned"); + if (controlPlane.has(service.serviceId)) { + if (action === "restart" && service.serviceId !== "ops-agent") + return denied("unavailable", "workflow", "core-restart"); + return denied("control_plane"); + } + if (dependencies.has(service.serviceId)) return denied("host_only", "host"); + if (action !== "stop" && !authority.enabledServiceIds.has(service.serviceId)) + return denied("disabled"); + // These workflows belong to 01.2/01.4. A route or a declared type is not an installer. + if (action === "update") return denied("unavailable"); + if (action === "build" && (!service.isBuiltIn || !service.buildable)) + return denied("not_buildable"); + if (action === "stop" || action === "restart") { + if (!snapshot) return denied("not_owned"); + if (!["running", "restarting", "paused"].includes(snapshot.state)) return denied("not_running"); + } + return { available: true, execution: "direct", reason: null, workflow: null }; +} diff --git a/apps/ops-agent/src/trusted-config-ownership.test.ts b/apps/ops-agent/src/trusted-config-ownership.test.ts index 34577fa75..600cbec06 100644 --- a/apps/ops-agent/src/trusted-config-ownership.test.ts +++ b/apps/ops-agent/src/trusted-config-ownership.test.ts @@ -1,8 +1,15 @@ +import { randomBytes } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { + createControlRootOperationKey, type OpsOperation, opsOperationFingerprint, opsOperationSchema, @@ -360,6 +367,18 @@ describe("authenticated conditional configuration admission", () => { async (fault) => { const f = fixture(); const token = Buffer.alloc(32, 7).toString("base64url"); + mkdirSync(join(f.infraDir, "data"), { mode: 0o700 }); + await provisionControlInstallation(f.infraDir); + const installation = openControlInstallation({ + infraDir: f.infraDir, + role: "ops-agent", + serviceIds: ["alpha"], + }); + const requestedDeadlineAt = Date.now() + 60_000; + const operationKey = createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }); const queue = join(f.root, "queue"); mkdirSync(queue, { mode: 0o700 }); mkdirSync(join(queue, ".claimed"), { mode: 0o700 }); @@ -367,7 +386,7 @@ describe("authenticated conditional configuration admission", () => { role: "api", token, nonce: "nonce_conditional1234567", - operationKey: "opk1_conditionalApply123", + operationKey, operationForRevision: (revisionId) => ({ kind: "stack.render", revisionId, @@ -462,6 +481,10 @@ describe("authenticated conditional configuration admission", () => { const app = buildOpsAgentServer({ tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, resourceClaimer: claimer, + controlInstallation: installation, + // This deliberately holds one authenticated claim across the other real + // control-store/configuration commits; its fixture budget must cover that I/O. + claimTimeoutMs: 15_000, journal: { records: () => journalRecords, replace: async (records) => { @@ -473,13 +496,14 @@ describe("authenticated conditional configuration admission", () => { runtime, }); let counter = 0; - const submit = (operation: OpsOperation, key = "opk1_conditionalApply123") => + const submit = (operation: OpsOperation, key = operationKey) => app.inject({ method: "POST", url: "/v1/operations", headers: { authorization: `Bearer ${token}` }, payload: { - version: 1, + version: 2, + execution: { requestedDeadlineAt }, requestId: `ops1_conditional${String(++counter).padStart(12, "0")}`, operationKey: key, issuedAt: new Date().toISOString(), @@ -597,6 +621,7 @@ describe("authenticated conditional configuration admission", () => { returnCleanup.resolve(); finish(); await app.close(); + closeControlInstallation(installation); } }, ); diff --git a/apps/ops-agent/src/trusted-config-runtime.test.ts b/apps/ops-agent/src/trusted-config-runtime.test.ts index 7aab0acbc..059708b32 100644 --- a/apps/ops-agent/src/trusted-config-runtime.test.ts +++ b/apps/ops-agent/src/trusted-config-runtime.test.ts @@ -917,7 +917,7 @@ describe("ops-agent trusted configuration runtime", () => { expect(existsSync(join(paths.current, ".generated-secrets"))).toBe(false); }); - it("prunes validated inactive generations so legitimate revisions cannot brick apply", async () => { + it("retains unobserved generations and prunes after complete cleanup acquisition", async () => { const paths = fixture(); const generations = join(paths.infraDir, ".trusted-config-generations"); mkdirSync(generations, { recursive: true, mode: 0o700 }); @@ -929,6 +929,21 @@ describe("ops-agent trusted configuration runtime", () => { writeFileSync(join(directory, "service-selection.json"), "{}", { mode: 0o600 }); } await initializeTrustedConfigurationRuntime(paths.infraDir); + expect(readdirSync(generations)).toHaveLength(17); + await services.commitConfigurationGeneration({ + infraDir: paths.infraDir, + services: [service("alpha")], + input: { + domain: "example.test", + selectedRoots: ["alpha"], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + ownedServiceGenerations: [], + observedServiceContainerIds: [], + }); + await initializeTrustedConfigurationRuntime(paths.infraDir); expect(readdirSync(generations)).toHaveLength(15); }); diff --git a/apps/ops-agent/src/trusted-config-runtime.ts b/apps/ops-agent/src/trusted-config-runtime.ts index 55a2d39d4..92df24c24 100644 --- a/apps/ops-agent/src/trusted-config-runtime.ts +++ b/apps/ops-agent/src/trusted-config-runtime.ts @@ -1,4 +1,5 @@ import { readFileSync } from "node:fs"; +import { basename, dirname } from "node:path"; import { type ConfigurationApplyResult, type OpsOperation, @@ -6,6 +7,7 @@ import { type TrustedConfigurationPayload, } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; +import type { OwnedServiceResolver } from "./owned-services"; import type { OpsRuntime, OpsTrustedClaim } from "./runtime"; const FAILED = "Trusted configuration apply failed"; @@ -21,6 +23,7 @@ export interface TrustedConfigurationRuntimeOptions { integrationSchemas: ReadonlyMap>; loadAuthority?: () => Promise; infraDir: string; + ownedServices?: OwnedServiceResolver; /** The process environment, read for the operator's selection override. */ env?: NodeJS.ProcessEnv; beforeCommit?: () => Promise; @@ -86,6 +89,21 @@ async function commitGeneration( } const services = authority?.services ?? options.services; const env = options.env ?? process.env; + // Capture cleanup retention before selection effects under the same writer. + const inventory = options.ownedServices?.inspectInventory + ? await options.ownedServices.inspectInventory(signal) + : options.ownedServices + ? { + services: await options.ownedServices.inspectOwnedServices(signal), + observedContainerIds: undefined, + } + : undefined; + const ownedServiceGenerations = inventory?.services.map((snapshot) => ({ + containerId: snapshot.containerId, + revisionIds: snapshot.provenance.composeConfigFiles + .map((file) => basename(dirname(file))) + .filter((id) => /^cfg1_[A-Za-z0-9_-]{43}$/.test(id)), + })); let selectedRoots: string[]; let allowMissingSelected = false; if (operation.kind === "serviceSelection.apply") { @@ -112,6 +130,8 @@ async function commitGeneration( } const committed = await writer.commit({ services, + ownedServiceGenerations, + observedServiceContainerIds: inventory?.observedContainerIds, integrationSchemas: authority?.integrationSchemas ?? options.integrationSchemas, input: { ...payload, selectedRoots }, allowMissingSelected, diff --git a/packages/core/src/api/contracts/service-admin/services.ts b/packages/core/src/api/contracts/service-admin/services.ts index f01fa9a06..55c26783d 100644 --- a/packages/core/src/api/contracts/service-admin/services.ts +++ b/packages/core/src/api/contracts/service-admin/services.ts @@ -53,6 +53,7 @@ export const adminServiceStatusSchema = z.enum([ "created", "paused", "not-running", + "unknown", ]); export const adminServiceSummarySchema = z.strictObject({ id: serviceManifestSchema.shape.id, diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index b411eeb74..8b66c92cd 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -40,6 +40,7 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { }, "docker.status": { kind: "docker.status" }, "stack.status": { kind: "stack.status" }, + "services.inspect": { kind: "services.inspect" }, "configuration.inspect": { kind: "configuration.inspect" }, "configuration.hardlinks.inspect": { kind: "configuration.hardlinks.inspect", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 83cb9b459..4150fc4e8 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -4,9 +4,10 @@ import { pairedPinApprovalResultSchema, } from "./catalog-pin-approval"; import { opsFailureResultSchema, processDiagnosticsSchema } from "./job-evidence"; +import { serviceActionsSchema } from "./service-results"; export const OPS_PROTOCOL_VERSION = 2 as const; -export const OPS_MAX_LOG_TAIL = 2_000; +export const OPS_MAX_LOG_TAIL = 1_000; export const OPS_MAX_EVENT_MESSAGE_BYTES = 4_096; export const OPS_MAX_EVENT_BATCH = 100; export const OPS_MAX_FOLLOW_LOG_EVENTS = 2_000; @@ -21,6 +22,7 @@ export const OPS_OPERATION_KINDS = [ "controlRetention.acknowledge", "docker.status", "stack.status", + "services.inspect", "stack.render", "configuration.inspect", "configuration.hardlinks.inspect", @@ -138,6 +140,7 @@ export const OPS_KIND_POLICIES = { "controlRetention.acknowledge": sync("api", 15_000), "docker.status": sync("api", 5_000), "stack.status": sync("api", 15_000, 256 * 1024), + "services.inspect": sync("api", 15_000, 512 * 1024), "configuration.inspect": sync("api", 15_000), "configuration.hardlinks.inspect": sync("api", 15_000, 1024 * 1024), "stack.render": asyncEffect("api", 120_000), @@ -312,6 +315,7 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ }), kindOnly("docker.status"), kindOnly("stack.status"), + kindOnly("services.inspect"), kindOnly("configuration.inspect"), z.strictObject({ kind: z.literal("stack.render"), @@ -625,7 +629,9 @@ export async function authorizeOpsResources( ): Promise { if ("serviceId" in operation) { if ( - NEVER_MANAGE.has(operation.serviceId) || + (NEVER_MANAGE.has(operation.serviceId) && + operation.kind !== "service.logs" && + operation.kind !== "service.logs.follow") || !(await allowed(policy.allowService?.(operation.kind, operation.serviceId))) ) return false; @@ -890,6 +896,17 @@ export const OPS_RESULT_SCHEMAS = { reachable: z.boolean(), version: z.string().max(128).optional(), }), + "services.inspect": z.strictObject({ + services: z + .array( + z.strictObject({ + serviceId: serviceIdSchema, + state: serviceRuntimeState, + actions: serviceActionsSchema, + }), + ) + .max(256), + }), "stack.status": z.strictObject({ services: z .array( diff --git a/packages/core/src/ops/service-results.ts b/packages/core/src/ops/service-results.ts index 3721b5d4b..7910bfdc2 100644 --- a/packages/core/src/ops/service-results.ts +++ b/packages/core/src/ops/service-results.ts @@ -67,3 +67,43 @@ export const serviceLifecycleFailureEvidenceSchema = z.strictObject({ .optional(), }); export type ServiceLifecycleFailureEvidence = z.infer; + +export const SERVICE_ACTIONS = [ + "start", + "stop", + "restart", + "update", + "build", + "logs", + "pull", + "apply", +] as const; +export type ServiceAction = (typeof SERVICE_ACTIONS)[number]; +export const serviceActionDecisionSchema = z + .strictObject({ + available: z.boolean(), + execution: z.enum(["direct", "workflow", "host"]), + reason: z + .enum([ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable", + ]) + .nullable(), + workflow: z.enum(["core-restart", "system-update"]).nullable(), + }) + .refine((decision) => + decision.available + ? decision.reason === null && + (decision.execution === "direct" + ? decision.workflow === null + : decision.execution === "workflow" && decision.workflow !== null) + : decision.reason !== null && + (decision.execution !== "host" || decision.reason === "host_only"), + ); +export type ServiceActionDecision = z.infer; +export const serviceActionsSchema = z.record(z.enum(SERVICE_ACTIONS), serviceActionDecisionSchema); diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index 921afa904..5cfc4e51a 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -76,6 +76,10 @@ export interface CommitConfigurationOptions { input: ConfigurationInput; /** Tolerate selected services that are not installed (the default selection in a reduced checkout). */ allowMissingSelected?: boolean; + /** Verified Docker generations that still contain owned containers; absent means retain conservatively. */ + ownedServiceGenerations?: readonly { containerId: string; revisionIds: readonly string[] }[]; + /** A complete bounded acquisition; omission cannot prove a previous container disappeared. */ + observedServiceContainerIds?: readonly string[]; /** The authenticated candidate identity; omitted, use a fresh activation identity (no ABA). */ revisionId?: string; /** Applied identity to compare under writer ownership; null means no pointer. */ @@ -643,6 +647,170 @@ export async function initializeConfigurationGenerations(infraDir: string): Prom ); } +const RETAINED_GENERATIONS = ".owned-service-generations.json"; +interface GenerationRetention { + conservative: string[]; + containers: Record; +} +function readGenerationRetention(infraDir: string): GenerationRetention { + const path = join(infraDir, RETAINED_GENERATIONS); + try { + const metadata = lstatSync(path, { bigint: true }); + const { contents } = readGenerationFile(path, metadata, 64 * 1024, [0o600n]); + const raw = JSON.parse(contents.toString("utf8")) as GenerationRetention; + if ( + !raw || + Object.keys(raw).sort().join(",") !== "conservative,containers" || + !Array.isArray(raw.conservative) || + !raw.containers || + typeof raw.containers !== "object" || + Array.isArray(raw.containers) + ) + throw new Error(UNSAFE); + const values = Object.values(raw.containers); + if ( + Object.keys(raw.containers).length > 256 || + Object.keys(raw.containers).some((id) => !/^[a-f0-9]{64}$/.test(id)) || + values.some((ids) => !Array.isArray(ids) || ids.length > 256) + ) + throw new Error(UNSAFE); + const revisions = [...raw.conservative, ...values.flat()]; + if ( + revisions.length > 512 || + revisions.some((id) => typeof id !== "string" || !REVISION.test(id)) || + new Set(revisions).size > 256 + ) + throw new Error(UNSAFE); + return raw; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + // An upgraded deployment has no ownership observations yet. Startup and + // host-only renders must not prune its old containers' generation files. + const generations = join(infraDir, CONFIGURATION_GENERATIONS); + let entries: string[]; + try { + const stat = lstatSync(generations); + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.uid !== (process.geteuid?.() ?? 0) || + (stat.mode & 0o777) !== 0o700 + ) + throw new Error(UNSAFE); + entries = readdirSync(generations).filter((name) => REVISION.test(name)); + } catch (missing) { + if ((missing as NodeJS.ErrnoException).code === "ENOENT") + return { conservative: [], containers: {} }; + throw missing; + } + if (entries.length > 256) throw new Error(UNSAFE); + for (const revision of entries) configurationGenerationComposeFile(infraDir, revision); + return { conservative: entries, containers: {} }; + } + throw error; + } +} +function retainedConfigurationGenerations(infraDir: string): Set { + const retained = readGenerationRetention(infraDir); + return new Set([...retained.conservative, ...Object.values(retained.containers).flat()]); +} +async function retainConfigurationGenerations( + infraDir: string, + owned?: CommitConfigurationOptions["ownedServiceGenerations"], + observed?: readonly string[], +): Promise { + const previous = readGenerationRetention(infraDir); + let retained = previous; + if (owned) { + if ( + owned.length > 256 || + new Set(owned.map((entry) => entry.containerId)).size !== owned.length + ) + throw new Error(UNSAFE); + if ( + observed && + (observed.length > 256 || + observed.some((id) => !/^[a-f0-9]{64}$/.test(id)) || + new Set(observed).size !== observed.length) + ) + throw new Error(UNSAFE); + // A missing verified claim is not evidence of cleanup. Retain previously + // owned IDs still observed by Docker even when their image/provenance no longer verifies. + const containers: Record = Object.fromEntries( + Object.entries(previous.containers).filter(([id]) => !observed || observed.includes(id)), + ); + for (const entry of owned) { + if ( + !/^[a-f0-9]{64}$/.test(entry.containerId) || + entry.revisionIds.some((id) => !REVISION.test(id)) + ) + throw new Error(UNSAFE); + // Labels on a live immutable Docker ID do not change. Remember the first + // resolved generation when older containers named the movable current alias. + containers[entry.containerId] = previous.containers[entry.containerId] ?? [ + ...new Set(entry.revisionIds), + ]; + } + // Unobserved legacy/CLI generations have no per-container attribution. + // Only a complete empty inventory proves that conservative group is unused. + retained = { conservative: observed?.length === 0 ? [] : previous.conservative, containers }; + } else { + const active = currentConfigurationGeneration(infraDir); + if (active) + retained = { ...previous, conservative: [...new Set([...previous.conservative, active])] }; + } + if (new Set([...retained.conservative, ...Object.values(retained.containers).flat()]).size > 256) + throw new Error(UNSAFE); + const encoded = `${JSON.stringify(retained)}\n`; + if ( + [...retained.conservative, ...Object.values(retained.containers).flat()].length > 512 || + Buffer.byteLength(encoded) > 64 * 1024 + ) + throw new Error(UNSAFE); + const path = join(infraDir, RETAINED_GENERATIONS); + // The generation writer is exclusive. A fixed staging name bounds crash leftovers. + const temporary = `${path}.tmp`; + try { + const stat = lstatSync(temporary); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + stat.uid !== (process.geteuid?.() ?? 0) || + (stat.mode & 0o777) !== 0o600 || + stat.size > 64 * 1024 + ) + throw new Error(UNSAFE); + rmSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + try { + await writeDurable(temporary, encoded, 0o600); + renameSync(temporary, path); + await syncDirectory(infraDir); + } finally { + rmSync(temporary, { force: true }); + } +} + +/** Validate a retained generation with the same filesystem rules as the active pointer. */ +export function configurationGenerationComposeFile(infraDir: string, revisionId: string): string { + if (!REVISION.test(revisionId)) throw new Error(UNSAFE); + const generations = join(infraDir, CONFIGURATION_GENERATIONS); + const root = lstatSync(generations); + if ( + !root.isDirectory() || + root.isSymbolicLink() || + root.uid !== (process.geteuid?.() ?? 0) || + (root.mode & 0o777) !== 0o700 + ) + throw new Error(UNSAFE); + const path = join(generations, revisionId); + safeGenerationTree(path); + return join(path, GENERATED_COMPOSE_FILE); +} + async function initializeOwnedConfigurationGenerations(infraDir: string): Promise { const generations = join(infraDir, CONFIGURATION_GENERATIONS); try { @@ -665,7 +833,8 @@ async function initializeOwnedConfigurationGenerations(infraDir: string): Promis } const active = currentConfigurationGeneration(infraDir); const entries = readdirSync(generations, { withFileTypes: true }); - if (entries.length > MAX_GENERATIONS + 8) throw new Error(`${generations} has too many entries`); + if (entries.length > MAX_GENERATIONS + 256 + 8) + throw new Error(`${generations} has too many entries`); const temporaryPaths: string[] = []; const complete: Array<{ name: string; path: string; mtimeMs: number }> = []; for (const entry of entries) { @@ -698,8 +867,9 @@ async function initializeOwnedConfigurationGenerations(infraDir: string): Promis if (pointerTemporaries.length > 0) await syncDirectory(infraDir); // Keep space for the next generation and prune only after every existing // entry has passed validation. + const retained = retainedConfigurationGenerations(infraDir); const removable = complete - .filter((entry) => entry.name !== active) + .filter((entry) => entry.name !== active && !retained.has(entry.name)) .sort((a, b) => a.mtimeMs - b.mtimeMs || a.name.localeCompare(b.name)); let retainedCount = complete.length; while (retainedCount > MAX_GENERATIONS - 1) { @@ -807,6 +977,11 @@ async function commitOwnedConfigurationGeneration( options.signal?.throwIfAborted(); const { resolved, context, rendered } = renderConfiguration(options); const generations = join(infraDir, CONFIGURATION_GENERATIONS); + await retainConfigurationGenerations( + infraDir, + options.ownedServiceGenerations, + options.observedServiceContainerIds, + ); await initializeOwnedConfigurationGenerations(infraDir); const nonce = randomUUID().replaceAll("-", ""); const placeholder = options.revisionId ?? `cfg1_${"0".repeat(43)}`; diff --git a/packages/core/src/services/deployment.ts b/packages/core/src/services/deployment.ts index 66eb8cd28..ec42b5b3b 100644 --- a/packages/core/src/services/deployment.ts +++ b/packages/core/src/services/deployment.ts @@ -3,6 +3,7 @@ import { mkdirSync, readdirSync, readFileSync, + realpathSync, renameSync, writeFileSync, } from "node:fs"; @@ -54,11 +55,19 @@ export function stackComposeArgs( fileExists: (path: string) => boolean = existsSync, ): string[] { const envFile = join(paths.infraDir, ".env"); + // Compose records this path in container provenance. Pin the generation, + // rather than the movable current symlink, so deselection retains cleanup authority. + let composePath = paths.composePath; + try { + composePath = realpathSync(composePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } return [ "compose", ...(fileExists(envFile) ? ["--env-file", envFile] : []), "-f", - paths.composePath, + composePath, ...(fileExists(paths.composeReleasePath) ? ["-f", paths.composeReleasePath] : []), ]; } diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 80bd9b3d3..132d4be01 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -32,6 +32,7 @@ export { type ConfigurationGenerationWriter, type ConfigurationInput, commitConfigurationGeneration, + configurationGenerationComposeFile, currentConfigurationGeneration, initializeConfigurationGenerations, inspectAppliedConfiguration, diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 724518a51..b2b271bff 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -77,6 +77,7 @@ "serviceSelection.apply", "serviceSelection.inspect", "services.buildAll", + "services.inspect", "stack.render", "stack.start", "stack.status", @@ -145,6 +146,12 @@ ] }, "businessEffects": [ + { + "id": "agent.services-inventory", + "callSiteIds": [], + "semanticSources": ["apps/ops-agent/src/production-runtime.ts#createProductionOpsRuntime"], + "typedKinds": ["services.inspect"] + }, { "id": "api.control-retention.acknowledge", "callSiteIds": [], From a8767fd8e395677677106969969f7b11ec766aef Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 01:31:47 +0100 Subject: [PATCH 023/135] chore(api): regenerate service status contracts --- apps/api/openapi.json | 9 ++++++--- .../checkAdminServices200StatusesItemState.ts | 1 + .../admin/service-models/getAdminService200Status.ts | 1 + .../listAdminServices200ServicesItemStatus.ts | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 93e9efbe4..bf032004f 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -34333,7 +34333,8 @@ "restarting", "created", "paused", - "not-running" + "not-running", + "unknown" ] } }, @@ -37379,7 +37380,8 @@ "restarting", "created", "paused", - "not-running" + "not-running", + "unknown" ] } }, @@ -40726,7 +40728,8 @@ "restarting", "created", "paused", - "not-running" + "not-running", + "unknown" ] } }, diff --git a/packages/core/src/api/generated/admin/service-models/checkAdminServices200StatusesItemState.ts b/packages/core/src/api/generated/admin/service-models/checkAdminServices200StatusesItemState.ts index b57973e9a..5e7381d9b 100644 --- a/packages/core/src/api/generated/admin/service-models/checkAdminServices200StatusesItemState.ts +++ b/packages/core/src/api/generated/admin/service-models/checkAdminServices200StatusesItemState.ts @@ -11,4 +11,5 @@ export const CheckAdminServices200StatusesItemState = { created: "created", paused: "paused", "not-running": "not-running", + unknown: "unknown", } as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200Status.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200Status.ts index 634f324bd..7c3a8fa1e 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminService200Status.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200Status.ts @@ -11,4 +11,5 @@ export const GetAdminService200Status = { created: "created", paused: "paused", "not-running": "not-running", + unknown: "unknown", } as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemStatus.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemStatus.ts index 0b6c34646..df74a014f 100644 --- a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemStatus.ts +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemStatus.ts @@ -11,4 +11,5 @@ export const ListAdminServices200ServicesItemStatus = { created: "created", paused: "paused", "not-running": "not-running", + unknown: "unknown", } as const; From b2796fd932dd740b1f0a43b42efa531b57de47c8 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 02:40:04 +0100 Subject: [PATCH 024/135] fix(admin): preserve Compose bases and fence service removal --- .../owned-compose-path-runtime.ts | 113 +++++++ .../src/__fixtures__/owned-service-host.ts | 41 ++- apps/ops-agent/src/community-repositories.ts | 25 +- apps/ops-agent/src/docker-runtime.ts | 19 +- apps/ops-agent/src/lifecycle-exclusion.ts | 202 ++++++++++++ apps/ops-agent/src/owned-services.test.ts | 69 ++++ apps/ops-agent/src/owned-services.ts | 48 +-- apps/ops-agent/src/production-runtime.test.ts | 300 +++++++++++++++++- apps/ops-agent/src/production-runtime.ts | 51 ++- apps/ops-agent/src/runtime.ts | 2 + .../__fixtures__/compose-path-host.ts | 213 +++++++++++++ .../configuration-generation.test.ts | 75 ++++- .../src/services/configuration-generation.ts | 64 ++++ packages/core/src/services/deployment.ts | 40 ++- packages/core/src/services/index.ts | 1 + 15 files changed, 1221 insertions(+), 42 deletions(-) create mode 100644 apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts create mode 100644 apps/ops-agent/src/lifecycle-exclusion.ts create mode 100644 packages/core/src/services/__fixtures__/compose-path-host.ts diff --git a/apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts b/apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts new file mode 100644 index 000000000..b321018d9 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts @@ -0,0 +1,113 @@ +import { existsSync, realpathSync } from "node:fs"; +import { dirname } from "node:path"; +import { opsOperationFingerprint, type OpsOperation } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { composePathHost } from "../../../../packages/core/src/services/__fixtures__/compose-path-host"; +import { CommunityRepositoryStore } from "../community-repositories"; +import type { DockerRuntimeOptions } from "../docker-runtime"; +import { createProductionRegistryResourceClaimer, denyAllTrustedOpsData } from "../policy"; +import { createProductionOpsRuntime, productionOwnedServices } from "../production-runtime"; +import { createTrustedConfigurationAuthorityLoader } from "../trusted-configuration-authority"; +import { testExecution } from "./execution"; + +export async function ownedComposePathRuntime() { + const host = await composePathHost(); + const loadAuthority = createTrustedConfigurationAuthorityLoader({ + rootDir: host.root, + builtInServices: host.loaded, + }); + const source = host.config(host.currentArgs()); + if (!source.model?.services.alpha) throw Error("Real Compose fixture acquisition failed"); + const row = { + Id: "d".repeat(64), + Image: `sha256:${"e".repeat(64)}`, + State: "running", + ConfigImage: source.model.services.alpha.image, + Labels: { + "com.docker.compose.project": services.STACK_PROJECT, + "com.docker.compose.service": "alpha", + "com.docker.compose.project.working_dir": dirname(host.firstCompose), + "com.docker.compose.project.config_files": [ + host.firstCompose, + realpathSync(host.stack.composeReleasePath), + ].join(","), + }, + }; + const calls: string[][] = []; + const pathChecks: Record[] = []; + const execFile: NonNullable = async (_file, args) => { + calls.push([...args]); + let stdout = ""; + if (args[0] === "container" && args[1] === "ls") stdout = row.Id; + else if (args[0] === "container" && args[1] === "inspect") stdout = JSON.stringify(row); + else if (args[0] === "image" && args[1] === "inspect") stdout = row.Image; + else if (args[0] === "compose" && args.includes("config")) { + const result = host.runConfigArgv(args); + if (!result.model) throw Error("Real ownership Compose acquisition failed"); + pathChecks.push(host.pathChecks(result.model, host.firstCompose)); + stdout = JSON.stringify(result.model); + } else if (args[0] === "stop") row.State = "exited"; + else if (args[0] === "logs") stdout = "public fixture log\n"; + return { stdout, stderr: "" }; + }; + const repositories = new CommunityRepositoryStore({ + rootDir: host.root, + uid: process.geteuid?.() ?? 0, + loadAuthority, + }); + await repositories.recover(); + const runtime = await createProductionOpsRuntime({ + rootDir: host.root, + stack: host.stack, + loadAuthority, + communityRepositories: repositories, + docker: { execFile, fileExists: existsSync }, + }); + const snapshot = () => + loadAuthority().then((authority) => ({ + revisionId: authority.revisionId, + services: authority.services.map((service) => ({ + serviceId: service.manifest.id, + enabled: service.enabled, + isBuiltIn: service.isBuiltIn, + selectionAuthority: service, + })), + integrationIds: [...authority.integrationSchemas.keys()], + })); + const initial = await snapshot(); + const claimer = createProductionRegistryResourceClaimer({ + ...initial, + loadAuthority: snapshot, + trustedData: denyAllTrustedOpsData, + enabledServiceIds: (current) => + services.readAppliedServiceIds( + host.infraDir, + current.map((service) => service.selectionAuthority), + ) ?? new Set(), + serviceMetadata: (current) => + current.map((service) => ({ + serviceId: service.serviceId, + isBuiltIn: service.isBuiltIn, + buildable: !!service.selectionAuthority.manifest.buildCommand, + })), + ownedServices: productionOwnedServices(runtime), + }); + const context = async (operation: OpsOperation) => { + const signal = new AbortController().signal, + claim = await claimer.claim(operation, opsOperationFingerprint(operation), signal, { + role: "api", + operationKey: testExecution().operationKey, + }); + if (!claim) throw Error("Admission denied"); + return { signal, claim, execution: testExecution(), emitLog() {} }; + }; + return { + host, + runtime, + owned: productionOwnedServices(runtime), + row, + calls, + pathChecks, + context, + }; +} diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index 714dcec53..878c1d04c 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -95,12 +95,19 @@ export async function ownedServiceHost() { let rawListing: string | undefined; let malformedInspect = false; const calls: string[][] = []; - const execFile: NonNullable = async (_file, args) => { + let creatingProcess: + | ((args: readonly string[], signal: AbortSignal) => Promise) + | undefined; + let inventoryProcess: (() => Promise) | undefined; + const execFile: NonNullable = async (_file, args, options) => { calls.push([...args]); let stdout = ""; - if (args[0] === "container" && args[1] === "ls") + if (args[0] === "container" && args[1] === "ls") { stdout = rawListing ?? rows.map((r) => r.Id).join("\n"); - else if (args[0] === "container" && args[1] === "inspect") + const pending = inventoryProcess; + inventoryProcess = undefined; + await pending?.(); + } else if (args[0] === "container" && args[1] === "inspect") stdout = malformedInspect ? "{" : rows.map((r) => JSON.stringify(r)).join("\n"); else if (args[0] === "compose" && args.includes("config")) stdout = JSON.stringify({ @@ -112,8 +119,12 @@ export async function ownedServiceHost() { ), }); else if (args[0] === "image" && args[1] === "inspect") stdout = imageId; - else if (args[0] === "stop") row.State = "exited"; - else if (args[0] === "logs") stdout = "retained log\n"; + else if (args[0] === "compose" && args.includes("up")) + await creatingProcess?.(args, options.signal); + else if (args[0] === "stop") { + const stopped = rows.find((candidate) => candidate.Id === args[1]); + if (stopped) stopped.State = "exited"; + } else if (args[0] === "logs") stdout = "retained log\n"; return { stdout, stderr: "" }; }; const repositories = new CommunityRepositoryStore({ @@ -171,6 +182,7 @@ export async function ownedServiceHost() { serviceId: s.manifest.id, enabled: s.enabled, isBuiltIn: s.isBuiltIn, + selectionAuthority: s, })), integrationIds: [], loadAuthority: async () => { @@ -181,17 +193,24 @@ export async function ownedServiceHost() { serviceId: s.manifest.id, enabled: s.enabled, isBuiltIn: s.isBuiltIn, + selectionAuthority: s, })), integrationIds: [...a.integrationSchemas.keys()], }; }, - enabledServiceIds: () => services.readAppliedServiceIds(infraDir, loaded) ?? new Set(), + enabledServiceIds: (authorityServices) => + services.readAppliedServiceIds( + infraDir, + authorityServices.map((service) => service.selectionAuthority), + ) ?? new Set(), trustedData, ownedServices: productionOwnedServices(runtime), serviceMetadata: (ss) => ss.map((s) => ({ ...s, buildable: false })), }); - const context = async (operation: OpsOperation) => { - const signal = new AbortController().signal; + const context = async ( + operation: OpsOperation, + signal: AbortSignal = new AbortController().signal, + ) => { const claim = await claimer.claim(operation, opsOperationFingerprint(operation), signal, { role: "api", operationKey: testExecution().operationKey, @@ -249,6 +268,12 @@ export async function ownedServiceHost() { select, owned: productionOwnedServices(runtime), actionAuthority: async () => serviceActionAuthority(await loadAuthority(), infraDir), + setCreatingProcess(value: typeof creatingProcess) { + creatingProcess = value; + }, + setInventoryProcess(value: typeof inventoryProcess) { + inventoryProcess = value; + }, setRows(value: typeof rows) { rows = value; }, diff --git a/apps/ops-agent/src/community-repositories.ts b/apps/ops-agent/src/community-repositories.ts index 107ce60ec..95e5a42d5 100644 --- a/apps/ops-agent/src/community-repositories.ts +++ b/apps/ops-agent/src/community-repositories.ts @@ -21,6 +21,7 @@ import { type GitShallowCloneOptions, gitShallowCloneSnapshot, } from "@openmapx/core/server"; +import type { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; import type { OwnedServiceResolver } from "./owned-services"; import { OpsAgentBusyError, type OpsRuntime } from "./runtime"; import { TrustedAuthorityFilesystem } from "./trusted-authority-filesystem"; @@ -601,6 +602,7 @@ export class CommunityRepositoryStore { operation: Extract, signal: AbortSignal = AbortSignal.timeout(15_000), owned?: OwnedServiceResolver, + expectedServiceIds?: readonly string[], ): Promise> { return this.serialize(async () => { const { repository, extensionId } = operation; @@ -621,6 +623,11 @@ export class CommunityRepositoryStore { ) .map((service) => service.manifest.id), ); + if ( + expectedServiceIds && + JSON.stringify([...ids].sort()) !== JSON.stringify([...expectedServiceIds].sort()) + ) + throw new Error("Repository service authority changed while waiting"); if (ids.size > 0) { if (!owned) throw new Error("Owned service cleanup inspection required before removal"); const inventory = owned.inspectInventory @@ -690,14 +697,26 @@ export class CommunityRepositoryStore { export function installCommunityRepositoryRuntime( runtime: OpsRuntime, store: CommunityRepositoryStore, - owned?: OwnedServiceResolver, + owned: OwnedServiceResolver, + lifecycle: ServiceLifecycleExclusion, ): void { runtime["serviceRepository.inspect"] = (operation, context) => store.inspect(operation, context.signal); runtime["serviceRepository.publish"] = (operation, context) => store.publish(operation, context.signal); runtime["serviceRepository.settle"] = (operation) => store.settle(operation); - runtime["serviceRepository.remove"] = (operation, context) => - store.remove(operation, context.signal, owned); + runtime["serviceRepository.remove"] = async (operation, context) => { + const authority = await store.list(); + const ids = authority.services + .filter((service) => service.repository === operation.repository) + .map((service) => { + const id = service.manifest.id; + if (typeof id !== "string") throw new Error("Repository service authority unavailable"); + return id; + }); + return lifecycle.run(ids, context.signal, () => + store.remove(operation, context.signal, owned, ids), + ); + }; runtime["communityServices.inspect"] = () => store.list(); } diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index f8d756563..438d4c245 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -33,6 +33,12 @@ import { } from "./runtime"; import { decideServiceAction } from "./service-action-policy"; +const containedProcessFailures = new WeakSet(); +/** Only actual contained-process adapters mint this observation; it is not request authority. */ +export function isContainedProcessFailure(error: unknown): boolean { + return typeof error === "object" && error !== null && containedProcessFailures.has(error); +} + const DEFAULT_TIMEOUT_MS = 120_000; const LIFECYCLE_TIMEOUT_MS = 10 * 60_000; const PULL_TIMEOUT_MS = 10 * 60_000; @@ -162,8 +168,11 @@ export async function runContainedStreamingProcess( exit.containmentFailed || exit.code !== 0 || exit.signal !== null - ) - throw new OpsExecutionFailure(timedOut ? "timeout" : "runtime", { diagnostics }); + ) { + const error = new OpsExecutionFailure(timedOut ? "timeout" : "runtime", { diagnostics }); + if (!exit.residualDescendants && !exit.containmentFailed) containedProcessFailures.add(error); + throw error; + } return diagnostics; } finally { clearTimeout(timer); @@ -268,7 +277,10 @@ export function runContainedProcess( exit.code !== 0 || exit.signal !== null ) { - reject(new Error("Contained process failed")); + const error = new Error("Contained process failed"); + if (!exit.residualDescendants && !exit.containmentFailed) + containedProcessFailures.add(error); + reject(error); return; } resolve({ @@ -434,6 +446,7 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { ) => { const remaining = context.execution.deadlineAt - Date.now(); if (remaining <= 0 || context.signal.aborted) throw new OpsExecutionFailure("timeout", {}); + context.beginServiceCreation?.(); if (options.execFile) await run(args, context.signal, Math.min(timeout, remaining)); else await runContainedStreamingProcess("docker", args, { diff --git a/apps/ops-agent/src/lifecycle-exclusion.ts b/apps/ops-agent/src/lifecycle-exclusion.ts new file mode 100644 index 000000000..211ff4c4e --- /dev/null +++ b/apps/ops-agent/src/lifecycle-exclusion.ts @@ -0,0 +1,202 @@ +import { + closeSync, + constants, + fstatSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { isContainedProcessFailure } from "./docker-runtime"; + +interface Waiter { + ids: string[]; + signal: AbortSignal; + grant(release: () => void): void; + abort(): void; +} +const FAILED = "Service lifecycle settlement is unresolved; host reconciliation required"; +const ID = /^[a-z0-9][a-z0-9-]{0,63}$/; + +/** Auxiliary serialization/denial, never a substitute for existing root/target authority. + * One instance is shared by the actual production lifecycle and repository adapters. + * Pending effects are durable denials across restart; no TTL or PID can clear them. */ +export class ServiceLifecycleExclusion { + private active = new Set(); + private waiting: Waiter[] = []; + private pending: Set; + private directory: string; + private file: string; + constructor(infraDir: string) { + this.directory = join(infraDir, ".service-lifecycle"); + try { + mkdirSync(this.directory, { mode: 0o700 }); + const parent = openSync(infraDir, constants.O_RDONLY | constants.O_DIRECTORY); + try { + fsyncSync(parent); + } finally { + closeSync(parent); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + const dir = lstatSync(this.directory); + if ( + !dir.isDirectory() || + dir.isSymbolicLink() || + dir.uid !== (process.geteuid?.() ?? 0) || + (dir.mode & 0o777) !== 0o700 + ) + throw Error(FAILED); + this.file = join(this.directory, "pending.json"); + let fd: number | undefined; + try { + fd = openSync(this.file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const stat = fstatSync(fd); + if ( + !stat.isFile() || + stat.uid !== dir.uid || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o600 || + stat.size > 64 * 1024 + ) + throw Error(FAILED); + const ids: unknown = JSON.parse(readFileSync(fd, "utf8")); + if ( + !Array.isArray(ids) || + ids.length > 256 || + ids.some((id) => typeof id !== "string" || !ID.test(id)) || + new Set(ids).size !== ids.length + ) + throw Error(FAILED); + this.pending = new Set(ids); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + this.pending = new Set(); + } finally { + if (fd !== undefined) closeSync(fd); + } + } + private persist(next: Set): void { + if (next.size > 256) throw Error(FAILED); + const temporary = `${this.file}.tmp`; + try { + const stat = lstatSync(temporary); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.uid !== (process.geteuid?.() ?? 0) || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o600 || + stat.size > 64 * 1024 + ) + throw Error(FAILED); + rmSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const fd = openSync( + temporary, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ); + try { + writeFileSync(fd, JSON.stringify([...next].sort())); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(temporary, this.file); + const parent = openSync(this.directory, constants.O_RDONLY | constants.O_DIRECTORY); + try { + fsyncSync(parent); + } finally { + closeSync(parent); + } + this.pending = next; + } + private drain(): void { + const earlier = new Set(); + for (const waiter of [...this.waiting]) { + if (waiter.ids.some((id) => this.active.has(id) || earlier.has(id))) { + for (const id of waiter.ids) earlier.add(id); + continue; + } + this.waiting.splice(this.waiting.indexOf(waiter), 1); + waiter.signal.removeEventListener("abort", waiter.abort); + for (const id of waiter.ids) this.active.add(id); + waiter.grant(() => { + for (const id of waiter.ids) this.active.delete(id); + this.drain(); + }); + } + } + private async acquire(ids: string[], signal: AbortSignal): Promise<() => void> { + signal.throwIfAborted(); + return new Promise((grant, reject) => { + const waiter: Waiter = { + ids, + signal, + grant, + abort: () => { + const index = this.waiting.indexOf(waiter); + if (index < 0) return; + this.waiting.splice(index, 1); + reject(signal.reason); + this.drain(); + }, + }; + this.waiting.push(waiter); + signal.addEventListener("abort", waiter.abort, { once: true }); + if (signal.aborted) waiter.abort(); + else this.drain(); + }); + } + async run( + serviceIds: readonly string[], + signal: AbortSignal, + work: () => Promise, + ): Promise { + if (serviceIds.length > 256 || serviceIds.some((id) => !ID.test(id))) throw Error(FAILED); + const ids = [...new Set(serviceIds)].sort(); + const release = await this.acquire(ids, signal); + try { + signal.throwIfAborted(); + if (ids.some((id) => this.pending.has(id))) throw Error(FAILED); + return await work(); + } finally { + release(); + } + } + async creating( + serviceIds: readonly string[], + signal: AbortSignal, + work: (begin: () => void) => Promise, + ): Promise { + return this.run(serviceIds, signal, async () => { + let begun = false; + const begin = () => { + signal.throwIfAborted(); + if (!begun) { + this.persist(new Set([...this.pending, ...serviceIds])); + begun = true; + } + }; + const clear = () => + this.persist(new Set([...this.pending].filter((id) => !serviceIds.includes(id)))); + try { + const result = await work(begin); + if (begun) clear(); + return result; + } catch (error) { + if (begun && isContainedProcessFailure(error)) clear(); + throw error; + } + }); + } +} diff --git a/apps/ops-agent/src/owned-services.test.ts b/apps/ops-agent/src/owned-services.test.ts index de6b197f6..38ea2bc69 100644 --- a/apps/ops-agent/src/owned-services.test.ts +++ b/apps/ops-agent/src/owned-services.test.ts @@ -1,5 +1,8 @@ import { rmSync } from "node:fs"; +import { basename, dirname } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { installedCompose } from "../../../packages/core/src/services/__fixtures__/compose-path-host"; +import { ownedComposePathRuntime } from "./__fixtures__/owned-compose-path-runtime"; import { ownedServiceHost } from "./__fixtures__/owned-service-host"; import { createDockerRuntime } from "./docker-runtime"; import { dispatchOpsOperation } from "./runtime"; @@ -107,3 +110,69 @@ describe("owned service cleanup", () => { }, ); }); + +it.runIf(installedCompose)( + "verifies owned retained provenance using actual Compose path resolution", + async () => { + const fixture = await ownedComposePathRuntime(); + roots.push(fixture.host.root); + await fixture.host.commit("advanced"); + const found = await fixture.owned.inspectOwnedServices(new AbortController().signal); + expect(found.map((service) => service.containerId)).toEqual([fixture.row.Id]); + expect(fixture.pathChecks.length).toBeGreaterThan(0); + expect(fixture.pathChecks.every((checks) => Object.values(checks).every(Boolean))).toBe(true); + expect( + fixture.calls + .filter((args) => args[0] === "compose") + .every((args) => args.includes("--project-directory")), + ).toBe(true); + }, +); + +it.runIf(installedCompose)( + "pins a known legacy current label to its retained generation during ownership config", + async () => { + const fixture = await ownedComposePathRuntime(); + roots.push(fixture.host.root); + fixture.row.Labels["com.docker.compose.project.config_files"] = [ + fixture.host.stack.composePath, + fixture.host.stack.composeReleasePath, + ].join(","); + fixture.row.Labels["com.docker.compose.project.working_dir"] = dirname( + fixture.host.stack.composePath, + ); + const first = await fixture.owned.inspectOwnedServices(new AbortController().signal); + await fixture.host.commit( + "advanced", + first.map((snapshot) => ({ + containerId: snapshot.containerId, + revisionIds: snapshot.provenance.composeConfigFiles + .map((file) => basename(dirname(file))) + .filter((id) => /^cfg1_/.test(id)), + })), + ); + const retained = await fixture.owned.inspectOwnedServices(new AbortController().signal); + expect(retained[0]?.provenance.composeConfigFiles[0]).toBe(fixture.host.firstCompose); + expect(fixture.pathChecks.every((checks) => Object.values(checks).every(Boolean))).toBe(true); + }, +); + +it.runIf(installedCompose)( + "denies an ambiguous legacy current label without reading newer generation secrets", + async () => { + const fixture = await ownedComposePathRuntime(); + roots.push(fixture.host.root); + fixture.row.Labels["com.docker.compose.project.config_files"] = [ + fixture.host.stack.composePath, + fixture.host.stack.composeReleasePath, + ].join(","); + fixture.row.Labels["com.docker.compose.project.working_dir"] = dirname( + fixture.host.stack.composePath, + ); + await fixture.host.commit("advanced-without-container-observation"); + const inventory = await fixture.owned.inspectInventory?.(new AbortController().signal); + expect(inventory?.services).toEqual([]); + expect(inventory?.unverifiedServiceIds).toEqual(["alpha"]); + expect(fixture.calls.some((args) => args.includes("config"))).toBe(false); + }, +); diff --git a/apps/ops-agent/src/owned-services.ts b/apps/ops-agent/src/owned-services.ts index c982b9b53..3e8e6a13f 100644 --- a/apps/ops-agent/src/owned-services.ts +++ b/apps/ops-agent/src/owned-services.ts @@ -1,4 +1,4 @@ -import { lstatSync, realpathSync } from "node:fs"; +import { existsSync, lstatSync, realpathSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { services } from "@openmapx/core/server"; import type { DockerRuntimeOptions } from "./docker-runtime"; @@ -122,6 +122,21 @@ export function createOwnedServiceResolver(options: { try { configFiles = files.split(",").map(canonical); workingDirectory = canonical(working); + const legacyCurrentFiles = [infra, configuredInfra].map((root) => + join(root, services.CURRENT_CONFIGURATION, services.GENERATED_COMPOSE_FILE), + ); + if (legacyCurrentFiles.includes(files.split(",")[0] ?? "")) { + const retained = services.retainedContainerComposeFile(infra, row.Id as string); + if (retained) { + configFiles[0] = retained; + if ( + [infra, configuredInfra].some( + (root) => working === join(root, services.CURRENT_CONFIGURATION), + ) + ) + workingDirectory = dirname(retained); + } + } } catch { continue; } @@ -164,18 +179,16 @@ export function createOwnedServiceResolver(options: { const key = JSON.stringify(configFiles); let configuration = configurations.get(key); if (!configuration) { - const config = JSON.parse( - await run([ - "compose", - "--project-name", - services.STACK_PROJECT, - ...configFiles.flatMap((file) => ["-f", file]), - ...realpathEnv(infra), - "config", - "--format", - "json", - ]), - ) as { services?: Record }; + const releasePath = join(infra, "docker-compose.release.yml"); + const args = services.stackComposeArgs( + { infraDir: infra, composePath: first, composeReleasePath: releasePath }, + (path) => (path === releasePath ? configFiles.includes(releasePath) : existsSync(path)), + ); + if (!args.includes("--project-name")) + args.splice(1, 0, "--project-name", services.STACK_PROJECT); + const config = JSON.parse(await run([...args, "config", "--format", "json"])) as { + services?: Record; + }; if ( !config.services || typeof config.services !== "object" || @@ -250,12 +263,3 @@ export function createOwnedServiceResolver(options: { inspectOwnedServices: async (signal) => (await inspectInventory(signal)).services, }; } - -function realpathEnv(infra: string): string[] { - try { - return ["--env-file", realpathSync(join(infra, ".env"))]; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; - throw error; - } -} diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index 7dfb73ac4..578556e8f 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -1,6 +1,6 @@ import { randomBytes } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; -import { join } from "node:path"; +import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import { closeControlInstallation, openControlInstallation, @@ -9,8 +9,12 @@ import { import { createControlRootOperationKey, type OpsOperation } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterEach, describe, expect, it } from "vitest"; +import { installedCompose } from "../../../packages/core/src/services/__fixtures__/compose-path-host"; +import { ownedComposePathRuntime } from "./__fixtures__/owned-compose-path-runtime"; import { ownedServiceHost } from "./__fixtures__/owned-service-host"; +import { runContainedProcess } from "./docker-runtime"; import { openOpsJobJournal } from "./journal"; +import { createProductionOpsRuntime } from "./production-runtime"; import { dispatchOpsOperation } from "./runtime"; import { buildOpsAgentServer } from "./server"; import { initializeTrustedConfigurationRuntime } from "./trusted-config-runtime"; @@ -333,3 +337,295 @@ describe("assembled production service authority", () => { ); }); }); + +async function installedCommunity() { + const host = await fixture(); + host.setRows([]); + const publication = await host.repositories.publish( + { + kind: "serviceRepository.publish", + url: "https://github.com/example/services", + commit: "1".repeat(40), + extensionId: "test-extension", + recovery: { + selectionBefore: ["motis"], + touchedServiceIds: ["community-live"], + previouslyEnabledServiceIds: [], + }, + }, + new AbortController().signal, + ); + await host.repositories.settle({ + kind: "serviceRepository.settle", + repository: publication.repository, + transactionId: publication.transactionId, + outcome: "commit", + }); + expect(await host.select(["community-live"])).toMatchObject({ outcome: "committed" }); + const remove = { + kind: "serviceRepository.remove", + repository: publication.repository, + extensionId: "test-extension", + } as const; + const start = { kind: "service.start", serviceId: "community-live" } as const; + const checkout = join(host.root, "services/.community", publication.repository); + const record = join( + host.root, + "services/.community", + `.repository-${publication.repository}.json`, + ); + const create = () => { + const file = realpathSync(host.stack.composePath); + host.setRows([ + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: `example/community-live:1@sha256:${"b".repeat(64)}`, + Labels: { + ...host.row.Labels, + "com.docker.compose.service": "community-live", + "com.docker.compose.project.working_dir": dirname(file), + "com.docker.compose.project.config_files": file, + }, + }, + ]); + }; + return { host, start, remove, checkout, record, create }; +} +const drain = () => new Promise((resolve) => setImmediate(resolve)); +describe("assembled lifecycle/removal exclusion", () => { + it("fences selected dependencies created by another service's Compose Start", async () => { + const { host, remove, checkout, create } = await installedCommunity(); + await host.select(["motis", "community-live"]); + writeFileSync( + host.stack.composeReleasePath, + "services:\n motis:\n depends_on:\n community-live:\n condition: service_started\n", + ); + const start = { kind: "service.start", serviceId: "motis" } as const; + const entered = Promise.withResolvers(), + finish = Promise.withResolvers(); + host.setCreatingProcess(async () => { + entered.resolve(); + await finish.promise; + create(); + }); + const starting = dispatchOpsOperation(host.runtime, start, await host.context(start)); + await entered.promise; + const removing = dispatchOpsOperation(host.runtime, remove, await host.context(remove)).then( + (value) => ({ value }), + (error) => ({ error }), + ); + try { + await drain(); + expect(existsSync(checkout), "Compose can also create the selected dependency").toBe(true); + finish.resolve(); + await starting; + expect("error" in (await removing)).toBe(true); + } finally { + finish.resolve(); + await starting.catch(() => undefined); + await removing; + } + }); + + it("retains authority throughout a community Start process and renewed removal inventory", async () => { + const { host, start, remove, checkout, record, create } = await installedCommunity(); + const entered = Promise.withResolvers(), + finish = Promise.withResolvers(); + host.setCreatingProcess(async () => { + entered.resolve(); + await finish.promise; + create(); + }); + const starting = dispatchOpsOperation(host.runtime, start, await host.context(start)); + await entered.promise; + let removed = false; + const removing = dispatchOpsOperation(host.runtime, remove, await host.context(remove)) + .then( + (value) => ({ value }), + (error) => ({ error }), + ) + .finally(() => { + removed = true; + }); + try { + await drain(); + expect( + existsSync(checkout), + "checkout cannot disappear while Start's real process is pending", + ).toBe(true); + expect(existsSync(record)).toBe(true); + expect(removed).toBe(false); + finish.resolve(); + await starting; + const outcome = await removing; + expect( + "error" in outcome && + outcome.error instanceof Error && + outcome.error.message.includes("require cleanup"), + ).toBe(true); + const stop = { kind: "service.stop", serviceId: "community-live" } as const; + await dispatchOpsOperation(host.runtime, stop, await host.context(stop)); + const logs = { kind: "service.logs", serviceId: "community-live", tail: 20 } as const; + await expect( + dispatchOpsOperation(host.runtime, logs, await host.context(logs)), + ).resolves.toMatchObject({ lines: ["retained log"] }); + } finally { + finish.resolve(); + await starting.catch(() => undefined); + await removing; + } + }); + + it("revalidates a previously admitted Start after removal wins the same boundary", async () => { + const { host, start, remove, checkout, create } = await installedCommunity(); + const startContext = await host.context(start), + removeContext = await host.context(remove); + const stackStart = { kind: "stack.start" } as const; + const stackContext = await host.context(stackStart); + const entered = Promise.withResolvers(), + finish = Promise.withResolvers(); + host.setInventoryProcess(async () => { + entered.resolve(); + await finish.promise; + }); + let starts = 0; + host.setCreatingProcess(async () => { + starts++; + create(); + }); + const removing = dispatchOpsOperation(host.runtime, remove, removeContext); + await entered.promise; + const starting = dispatchOpsOperation(host.runtime, start, startContext).then( + (value) => ({ value }), + (error) => ({ error }), + ); + try { + await drain(); + expect(starts).toBe(0); + finish.resolve(); + await expect(removing).resolves.toMatchObject({ removed: true }); + expect(existsSync(checkout)).toBe(false); + const outcome = await starting; + expect("error" in outcome && outcome.error instanceof Error).toBe(true); + await expect(dispatchOpsOperation(host.runtime, stackStart, stackContext)).rejects.toThrow(); + expect(starts).toBe(0); + } finally { + finish.resolve(); + await removing.catch(() => undefined); + await starting; + } + }); +}); + +it.runIf(installedCompose)( + "executes assembled retained-generation cleanup after real Compose path qualification", + async () => { + const fixture = await ownedComposePathRuntime(); + roots.push(fixture.host.root); + await fixture.host.commit("advanced"); + const stop = { kind: "service.stop", serviceId: "alpha" } as const; + await dispatchOpsOperation(fixture.runtime, stop, await fixture.context(stop)); + const logs = { kind: "service.logs", serviceId: "alpha", tail: 20 } as const; + await expect( + dispatchOpsOperation(fixture.runtime, logs, await fixture.context(logs)), + ).resolves.toMatchObject({ lines: ["public fixture log"] }); + expect(fixture.calls).toContainEqual(["stop", fixture.row.Id]); + expect(fixture.calls).toContainEqual(["logs", "--tail=20", fixture.row.Id]); + expect(fixture.pathChecks.every((checks) => Object.values(checks).every(Boolean))).toBe(true); + }, +); + +describe("settlement and cancellation at the installed lifecycle boundary", () => { + it("cancels a waiting removal without stealing the still-executing Start guard", async () => { + const { host, start, remove, checkout, create } = await installedCommunity(); + const entered = Promise.withResolvers(), + finish = Promise.withResolvers(); + host.setCreatingProcess(async () => { + entered.resolve(); + await finish.promise; + create(); + }); + const starting = dispatchOpsOperation(host.runtime, start, await host.context(start)); + await entered.promise; + const controller = new AbortController(); + const removeContext = await host.context(remove, controller.signal); + const removing = dispatchOpsOperation(host.runtime, remove, removeContext).then( + (value) => ({ value }), + (error) => ({ error }), + ); + await drain(); + controller.abort(new Error("synthetic waiting cancellation")); + try { + const result = await removing; + expect("error" in result).toBe(true); + expect(existsSync(checkout)).toBe(true); + expect(host.row.State).toBe("running"); + } finally { + finish.resolve(); + await starting; + } + await expect( + dispatchOpsOperation(host.runtime, remove, await host.context(remove)), + ).rejects.toThrow("require cleanup"); + }); + + it("retains an unknown aborted process fence across production runtime reconstruction", async () => { + const { host, start, remove, checkout } = await installedCommunity(); + const entered = Promise.withResolvers(), + finish = Promise.withResolvers(); + host.setCreatingProcess(async () => { + entered.resolve(); + await finish.promise; + throw Error("unknown external process settlement"); + }); + const controller = new AbortController(), + ctx = await host.context(start, controller.signal); + const starting = dispatchOpsOperation(host.runtime, start, ctx).then( + (value) => ({ value }), + (error) => ({ error }), + ); + await entered.promise; + controller.abort(); + finish.resolve(); + expect("error" in (await starting)).toBe(true); + const restarted = await createProductionOpsRuntime({ + rootDir: host.root, + stack: host.stack, + loadAuthority: host.loadAuthority, + communityRepositories: host.repositories, + docker: { execFile: host.execFile, fileExists: existsSync }, + }); + await expect( + dispatchOpsOperation(restarted, remove, await host.context(remove)), + ).rejects.toThrow("settlement is unresolved"); + expect(existsSync(checkout)).toBe(true); + }); + + it("releases a known-contained failing process and orders multi-service Start without deadlock", async () => { + const { host, start, remove, checkout } = await installedCommunity(); + let first = true; + host.setCreatingProcess(async (_args, signal) => { + if (first) { + first = false; + await runContainedProcess(process.execPath, ["-e", "process.exit(7)"], { + signal, + timeout: 5_000, + maxBuffer: 1024, + }); + } + }); + await expect( + dispatchOpsOperation(host.runtime, start, await host.context(start)), + ).rejects.toThrow("Contained process failed"); + const global = { kind: "stack.start" } as const; + await expect( + dispatchOpsOperation(host.runtime, global, await host.context(global)), + ).resolves.toMatchObject({ changed: true }); + await expect( + dispatchOpsOperation(host.runtime, remove, await host.context(remove)), + ).resolves.toMatchObject({ removed: true }); + expect(existsSync(checkout)).toBe(false); + }); +}); diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 39b171ee6..11c227f67 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -18,6 +18,7 @@ import { type DockerRuntimeOptions, runContainedProcess, } from "./docker-runtime"; +import { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; import { createOwnedServiceResolver, type OwnedServiceResolver } from "./owned-services"; import { serviceActionForOperation } from "./policy"; import type { OpsExecutionContext, OpsRuntime } from "./runtime"; @@ -76,6 +77,7 @@ export async function createProductionOpsRuntime( options: ProductionOpsRuntimeOptions, ): Promise { const initial = await options.loadAuthority(); + const lifecycle = new ServiceLifecycleExclusion(options.stack.infraDir); const owned = createOwnedServiceResolver({ stack: options.stack, execFile: options.docker?.execFile ?? runContainedProcess, @@ -96,7 +98,7 @@ export async function createProductionOpsRuntime( infraDir: options.stack.infraDir, ownedServices: owned, }); - installCommunityRepositoryRuntime(runtime, options.communityRepositories, owned); + installCommunityRepositoryRuntime(runtime, options.communityRepositories, owned, lifecycle); const runFixedCli = options.runFixedCli ?? createDefaultFixedCli(options.rootDir); const releaseEffects = createDefaultReleaseEffects(options.rootDir, runFixedCli); createTransitousLockRuntime(runtime, { rootDir: options.rootDir }); @@ -149,7 +151,7 @@ export async function createProductionOpsRuntime( operation: OpsOperation, context: OpsExecutionContext, ) => Promise; - const guarded = async ( + const execute = async ( operation: Extract, context: OpsExecutionContext, ) => { @@ -181,9 +183,54 @@ export async function createProductionOpsRuntime( throw new Error("Service action denied"); return handler(operation, context); }; + const guarded = async ( + operation: Extract, + context: OpsExecutionContext, + ) => { + if ( + ![ + "service.start", + "service.restart", + "service.recreate", + "service.recreateIsolated", + ].includes(kind) + ) + return execute(operation, context); + // Compose up may also create dependencies, including release-overlay + // dependencies. Conservatively exclude every registered service for those + // commands; this local boundary does not acquire or replace root authority. + const ids = + kind === "service.start" || kind === "service.recreate" + ? (await options.loadAuthority()).services.map((service) => service.manifest.id).sort() + : [operation.serviceId]; + return lifecycle.creating(ids, context.signal, async (begin) => { + if (kind === "service.start" || kind === "service.recreate") { + const current = (await options.loadAuthority()).services + .map((service) => service.manifest.id) + .sort(); + if (JSON.stringify(current) !== JSON.stringify(ids)) + throw new Error("Service authority changed while waiting"); + } + return execute(operation, { ...context, beginServiceCreation: begin }); + }); + }; // The finite loop preserves each handler's operation/result correspondence. Object.assign(runtime, { [kind]: guarded }); } + const stackStart = runtime["stack.start"]; + runtime["stack.start"] = async (operation, context) => { + const ids = (await options.loadAuthority()).services + .map((service) => service.manifest.id) + .sort(); + return lifecycle.creating(ids, context.signal, async (begin) => { + const authority = await options.loadAuthority(); + const current = authority.services.map((service) => service.manifest.id).sort(); + if (JSON.stringify(current) !== JSON.stringify(ids)) + throw new Error("Service authority changed while waiting"); + serviceActionAuthority(authority, options.stack.infraDir); + return stackStart(operation, { ...context, beginServiceCreation: begin }); + }); + }; resolvers.set(runtime, owned); releaseAdapters.set(runtime, releaseEffects); return runtime; diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index b5588d3ee..fc3ae846c 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -59,6 +59,8 @@ export interface OpsExecutionContext { completionDigest: string; }): Promise; markFeedProxyReloadAttempted?(): Promise; + /** Installed only by the shared production exclusion; records denial before external creation. */ + beginServiceCreation?(): void; signal: AbortSignal; emitLog(stream: "stdout" | "stderr", message: string): void; claim: OpsTrustedClaim; diff --git a/packages/core/src/services/__fixtures__/compose-path-host.ts b/packages/core/src/services/__fixtures__/compose-path-host.ts new file mode 100644 index 000000000..d95593978 --- /dev/null +++ b/packages/core/src/services/__fixtures__/compose-path-host.ts @@ -0,0 +1,213 @@ +import { spawnSync } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + statSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + commitConfigurationGeneration, + type CommitConfigurationOptions, +} from "../configuration-generation"; +import { GENERATED_COMPOSE_FILE, currentConfigurationFile, stackComposeArgs } from "../deployment"; +import type { LoadedService } from "../types"; + +const lookup = spawnSync("/usr/bin/which", ["docker-compose"], { encoding: "utf8" }); +export const installedCompose = lookup.status === 0 ? lookup.stdout.trim() : undefined; +export interface PathConfig { + name: string; + services: Record< + string, + { + image: string; + environment?: Record; + env_file?: { path: string }[]; + volumes?: { target: string; source: string }[]; + } + >; + secrets: Record; +} +export async function composePathHost() { + const root = realpathSync(mkdtempSync(join(tmpdir(), "openmapx-compose-paths-"))); + const infraDir = join(root, "infra/docker"); + mkdirSync(infraDir, { recursive: true }); + const loaded: LoadedService[] = ["app-api", "alpha"].map((id) => { + const directory = join(root, "services", id); + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "local-public.txt"), "public service fixture\n"); + return { + manifest: { + id, + name: id, + version: "1.0.0", + quality: "built-in", + container: { image: `example/${id}`, tag: "1", envFile: [".env"] }, + bindMounts: [ + { source: "./local-public.txt", target: "/service-public", readOnly: true }, + { source: "@infra:public-data.txt", target: "/infra-public", readOnly: true }, + ], + configSchema: { + type: "object", + properties: { PRIVATE_SETTING: { type: "string", "x-openmapx-secret": true } }, + }, + }, + directory, + enabled: true, + isBuiltIn: true, + }; + }); + writeFileSync(join(infraDir, ".env"), "PUBLIC_FIXTURE_SETTING=synthetic-public\n", { + mode: 0o600, + }); + writeFileSync(join(infraDir, "public-data.txt"), "public infrastructure fixture\n"); + writeFileSync(join(infraDir, "overlay-public.txt"), "public overlay fixture\n"); + const stack = { + infraDir, + composePath: currentConfigurationFile(infraDir, GENERATED_COMPOSE_FILE), + composeReleasePath: join(infraDir, "docker-compose.release.yml"), + }; + writeFileSync( + stack.composeReleasePath, + "services:\n alpha:\n image: example/alpha:release\n volumes:\n - ../overlay-public.txt:/overlay-public:ro\n", + ); + const dockerConfig = join(root, "docker-config"); + mkdirSync(join(dockerConfig, "cli-plugins"), { recursive: true }); + if (installedCompose) + symlinkSync(installedCompose, join(dockerConfig, "cli-plugins/docker-compose")); + const env = { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: root, DOCKER_CONFIG: dockerConfig }; + const commit = ( + version: string, + ownership?: CommitConfigurationOptions["ownedServiceGenerations"], + ) => + commitConfigurationGeneration({ + infraDir, + services: loaded, + ownedServiceGenerations: ownership, + observedServiceContainerIds: ownership?.map((entry) => entry.containerId), + input: { + domain: "fixture.example.test", + selectedRoots: ["app-api", "alpha"], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: loaded.map((service) => ({ + serviceId: service.manifest.id, + values: { PRIVATE_SETTING: `synthetic-public-fixture-${version}` }, + })), + }, + }); + const first = await commit("first"); + const firstCompose = realpathSync(stack.composePath); + const secretFile = (compose: string, id = "alpha") => + join(dirname(compose), ".generated-secrets", id, "PRIVATE_SETTING"); + const hash = (file: string) => createHash("sha256").update(readFileSync(file)).digest("hex"); + const sameFile = (a: string, b: string) => { + const x = statSync(a), + y = statSync(b); + return x.dev === y.dev && x.ino === y.ino; + }; + const record = (entry: unknown) => { + const output = process.env.OMX_TASK011_FIX1_EVIDENCE; + if (!output) return; + mkdirSync(output, { recursive: true }); + writeFileSync(join(output, `compose-${randomUUID()}.json`), JSON.stringify(entry, null, 2)); + }; + const runConfigArgv = (argv: readonly string[]) => { + const before = new Date().toISOString(); + const result = spawnSync("docker", argv, { + env, + encoding: "utf8", + timeout: 15_000, + maxBuffer: 1024 * 1024, + }); + record({ + argv: ["docker", ...argv], + startUTC: before, + endUTC: new Date().toISOString(), + exit: result.status, + errorCode: result.error && "code" in result.error ? result.error.code : null, + stderr: result.stderr, + stdoutSHA256: createHash("sha256") + .update(result.stdout ?? "") + .digest("hex"), + stdoutOmitted: "Contains fixture configuration; path/inode checks are recorded separately", + environmentKeys: Object.keys(env), + }); + return { + status: result.status, + model: result.status === 0 ? (JSON.parse(result.stdout) as PathConfig) : undefined, + }; + }; + const config = (args: readonly string[], noEnvResolution = false) => + runConfigArgv([ + ...args, + "config", + "--format", + "json", + ...(noEnvResolution ? ["--no-env-resolution"] : []), + ]); + const pathChecks = (model: PathConfig, compose: string) => { + const checks: Record = { + releaseImage: model.services.alpha?.image === "example/alpha:release", + }; + for (const service of loaded) { + const id = service.manifest.id, + entry = model.services[id]; + checks[`${id}.infraBind`] = sameFile( + entry?.volumes?.find((v) => v.target === "/infra-public")?.source ?? "", + join(infraDir, "public-data.txt"), + ); + checks[`${id}.serviceBind`] = sameFile( + entry?.volumes?.find((v) => v.target === "/service-public")?.source ?? "", + join(service.directory, "local-public.txt"), + ); + const secret = Object.values(model.secrets).find((s) => + s.file.endsWith(`/${id}/PRIVATE_SETTING`), + ); + checks[`${id}.secret`] = sameFile(secret?.file ?? "", secretFile(compose, id)); + if (entry?.env_file) + checks[`${id}.envFile`] = sameFile(entry.env_file[0]?.path ?? "", join(infraDir, ".env")); + else + checks[`${id}.envFileRead`] = + entry?.environment?.PUBLIC_FIXTURE_SETTING === "synthetic-public"; + } + checks.overlayBind = sameFile( + model.services.alpha?.volumes?.find((v) => v.target === "/overlay-public")?.source ?? "", + join(infraDir, "overlay-public.txt"), + ); + record({ + pathChecks: checks, + generationFile: compose, + generationSHA256: hash(compose), + secretInodes: loaded.map((s) => { + const stat = statSync(secretFile(compose, s.manifest.id)); + return { serviceId: s.manifest.id, dev: stat.dev, ino: stat.ino }; + }), + }); + return checks; + }; + return { + root, + infraDir, + stack, + loaded, + first, + firstCompose, + env, + config, + runConfigArgv, + pathChecks, + commit, + secretFile, + hash, + sameFile, + currentArgs: () => stackComposeArgs(stack, existsSync), + retainedArgs: () => stackComposeArgs({ ...stack, composePath: firstCompose }, existsSync), + }; +} diff --git a/packages/core/src/services/__tests__/configuration-generation.test.ts b/packages/core/src/services/__tests__/configuration-generation.test.ts index 04bbb2246..80c9a7dd7 100644 --- a/packages/core/src/services/__tests__/configuration-generation.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation.test.ts @@ -1,16 +1,28 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { composePathHost, installedCompose } from "../__fixtures__/compose-path-host"; import { type ConfigurationInput, commitConfigurationGeneration, + initializeConfigurationGenerations, readAppliedConfiguration, readAppliedServiceIds, renderConfiguration, withConfigurationGenerationWriter, } from "../configuration-generation"; import { + configurationGenerationBasePath, readDesiredSelection, readServiceSelectionFile, stackComposeArgs, @@ -230,6 +242,67 @@ describe("desired selection", () => { }); describe("stack compose arguments", () => { + it("adds fixed bases to existing generations without modifying their bytes and rejects forged bases", async () => { + const host = await composePathHost(); + roots.push(host.root); + const composeHash = host.hash(host.firstCompose), + secretHash = host.hash(host.secretFile(host.firstCompose)); + const base = configurationGenerationBasePath(host.infraDir, host.first.revisionId); + rmSync(base); + expect(() => host.currentArgs()).toThrow(); + await initializeConfigurationGenerations(host.infraDir); + expect(realpathSync(base)).toBe(realpathSync(join(host.infraDir, ".trusted-config-current"))); + expect(host.currentArgs()).toContain(base); + expect(host.hash(host.firstCompose)).toBe(composeHash); + expect(host.hash(host.secretFile(host.firstCompose))).toBe(secretHash); + rmSync(base); + symlinkSync(".trusted-config-current", base); + expect(() => host.currentArgs()).toThrow("Unsafe generation Compose base"); + await expect(initializeConfigurationGenerations(host.infraDir)).rejects.toThrow( + "Unsafe generation Compose base", + ); + expect(host.hash(host.firstCompose)).toBe(composeHash); + expect(host.hash(host.secretFile(host.firstCompose))).toBe(secretHash); + }); + + it.runIf(installedCompose)( + "resolves real rendered current and retained paths without redirecting generation secrets", + async () => { + const host = await composePathHost(); + roots.push(host.root); + const originalBytes = host.hash(host.firstCompose), + originalSecret = host.hash(host.secretFile(host.firstCompose)); + const oldArgs = host.currentArgs(); + for (const noEnv of [false, true]) { + const result = host.config(oldArgs, noEnv); + expect(result.status, "real Compose must validate production-rendered env_file paths").toBe( + 0, + ); + if (!result.model) throw Error("Real Compose fixture acquisition failed"); + expect(Object.values(host.pathChecks(result.model, host.firstCompose)).every(Boolean)).toBe( + true, + ); + } + await host.commit("second"); + const nextCompose = realpathSync(host.stack.composePath); + expect(host.sameFile(host.secretFile(host.firstCompose), host.secretFile(nextCompose))).toBe( + false, + ); + for (const [args, file] of [ + [oldArgs, host.firstCompose], + [host.retainedArgs(), host.firstCompose], + [host.currentArgs(), nextCompose], + ] as const) { + const result = host.config(args); + expect(result.status).toBe(0); + if (!result.model) throw Error("Real Compose fixture acquisition failed"); + expect(Object.values(host.pathChecks(result.model, file)).every(Boolean)).toBe(true); + } + expect(host.hash(host.firstCompose)).toBe(originalBytes); + expect(host.hash(host.secretFile(host.firstCompose))).toBe(originalSecret); + }, + ); + it("addresses the stack as one project with the deployment environment", () => { const paths = { infraDir: "/stack", diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index 5cfc4e51a..7c15fc319 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -43,10 +43,13 @@ import { } from "./configuration-ownership"; import { validateTrustedConfigurationValues } from "./configuration-schema"; import { + CONFIGURATION_GENERATION_BASE_PREFIX, CONFIGURATION_GENERATIONS, CURRENT_CONFIGURATION, + configurationGenerationBasePath, GENERATED_COMPOSE_FILE, GENERATED_HARDLINK_PLAN_FILE, + readConfigurationGenerationBase, SERVICE_SELECTION_FILE, } from "./deployment"; import { buildAppApiServiceEnv, expandServiceSelection } from "./selection"; @@ -714,6 +717,29 @@ function retainedConfigurationGenerations(infraDir: string): Set { const retained = readGenerationRetention(infraDir); return new Set([...retained.conservative, ...Object.values(retained.containers).flat()]); } + +/** + * Recover only the first recorded generation of a legacy container whose Docker + * labels named the movable current alias. This is path selection, never an + * ownership claim: callers must still verify registry, labels and image identity. + */ +export function retainedContainerComposeFile(infraDir: string, containerId: string): string | null { + if (!/^[a-f0-9]{64}$/.test(containerId)) throw new Error(UNSAFE); + const retained = readGenerationRetention(infraDir); + const revisions = retained.containers[containerId]; + if (!revisions) { + const current = currentConfigurationGeneration(infraDir); + if ( + [...retained.conservative, ...Object.values(retained.containers).flat()].some( + (id) => id !== current, + ) + ) + throw new Error(UNSAFE); + return null; + } + if (revisions.length !== 1) throw new Error(UNSAFE); + return configurationGenerationComposeFile(infraDir, revisions[0] ?? ""); +} async function retainConfigurationGenerations( infraDir: string, owned?: CommitConfigurationOptions["ownedServiceGenerations"], @@ -811,6 +837,21 @@ export function configurationGenerationComposeFile(infraDir: string, revisionId: return join(path, GENERATED_COMPOSE_FILE); } +function ensureConfigurationGenerationBase(infraDir: string, revisionId: string): boolean { + try { + readConfigurationGenerationBase(infraDir, revisionId); + return false; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + symlinkSync( + `${CONFIGURATION_GENERATIONS}/${revisionId}`, + configurationGenerationBasePath(infraDir, revisionId), + ); + readConfigurationGenerationBase(infraDir, revisionId); + return true; +} + async function initializeOwnedConfigurationGenerations(infraDir: string): Promise { const generations = join(infraDir, CONFIGURATION_GENERATIONS); try { @@ -850,8 +891,17 @@ async function initializeOwnedConfigurationGenerations(infraDir: string): Promis complete.push({ name: entry.name, path, mtimeMs: lstatSync(path).mtimeMs }); } const pointerTemporaries: string[] = []; + const generationBases: { revisionId: string; path: string }[] = []; const pointerName = new RegExp(`^\\.\\${CURRENT_CONFIGURATION}\\.[a-f0-9]{32}\\.tmp$`); for (const entry of readdirSync(infraDir, { withFileTypes: true })) { + if (entry.name.startsWith(CONFIGURATION_GENERATION_BASE_PREFIX)) { + const revisionId = entry.name.slice(CONFIGURATION_GENERATION_BASE_PREFIX.length); + generationBases.push({ + revisionId, + path: readConfigurationGenerationBase(infraDir, revisionId), + }); + continue; + } if (!pointerName.test(entry.name)) continue; const path = join(infraDir, entry.name); const stats = lstatSync(path); @@ -879,6 +929,17 @@ async function initializeOwnedConfigurationGenerations(infraDir: string): Promis retainedCount -= 1; } if (complete.some((entry) => !existsSync(entry.path))) await syncDirectory(generations); + let basesChanged = false; + for (const base of generationBases) { + if (!existsSync(join(generations, base.revisionId))) { + rmSync(base.path); + basesChanged = true; + } + } + for (const entry of complete) + if (existsSync(entry.path)) + basesChanged = ensureConfigurationGenerationBase(infraDir, entry.name) || basesChanged; + if (basesChanged) await syncDirectory(infraDir); if (active && !existsSync(join(generations, active))) { throw new Error(`The applied generation ${active} is missing`); } @@ -936,6 +997,8 @@ export function renderConfiguration(options: RenderConfigurationOptions) { ); const context: RenderContext = { domain: input.domain, + // Runtime supplies a fixed generation alias at this same directory depth. + // Keep these bytes compatible with existing current/retained generations. composeOutDir: join(infraDir, CURRENT_CONFIGURATION), infraDir, allServices: resolved.allServices, @@ -1050,6 +1113,7 @@ async function commitOwnedConfigurationGeneration( await options.afterGenerationRename?.(); } options.signal?.throwIfAborted(); + if (ensureConfigurationGenerationBase(infraDir, revisionId)) await syncDirectory(infraDir); const currentRevisionId = currentConfigurationGeneration(infraDir); if ( options.expectedCurrentRevisionId !== undefined && diff --git a/packages/core/src/services/deployment.ts b/packages/core/src/services/deployment.ts index ec42b5b3b..76fa7c0eb 100644 --- a/packages/core/src/services/deployment.ts +++ b/packages/core/src/services/deployment.ts @@ -1,13 +1,15 @@ import { existsSync, + lstatSync, mkdirSync, readdirSync, readFileSync, + readlinkSync, realpathSync, renameSync, writeFileSync, } from "node:fs"; -import { join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { DEFAULT_SELECTED_SERVICE_IDS, normalizeServiceIds, @@ -27,6 +29,26 @@ export const CURRENT_CONFIGURATION = ".trusted-config-current"; export const CONFIGURATION_GENERATIONS = ".trusted-config-generations"; export const GENERATED_COMPOSE_FILE = "docker-compose.generated.yml"; export const GENERATED_HARDLINK_PLAN_FILE = "docker-compose.generated.hardlinks.json"; +export const CONFIGURATION_GENERATION_BASE_PREFIX = ".trusted-config-base-"; + +export function configurationGenerationBasePath(infraDir: string, revisionId: string): string { + if (!/^cfg1_[A-Za-z0-9_-]{43}$/.test(revisionId)) + throw new Error("Unsafe generation Compose base"); + return join(infraDir, `${CONFIGURATION_GENERATION_BASE_PREFIX}${revisionId}`); +} +/** A fixed same-depth alias preserves legacy relative paths and pins its own secrets. */ +export function readConfigurationGenerationBase(infraDir: string, revisionId: string): string { + const base = configurationGenerationBasePath(infraDir, revisionId), + stat = lstatSync(base); + if ( + !stat.isSymbolicLink() || + stat.uid !== (process.geteuid?.() ?? 0) || + stat.nlink !== 1 || + readlinkSync(base) !== `${CONFIGURATION_GENERATIONS}/${revisionId}` + ) + throw new Error("Unsafe generation Compose base"); + return base; +} /** * The Compose project of the stack, written into the generated compose file: @@ -58,13 +80,29 @@ export function stackComposeArgs( // Compose records this path in container provenance. Pin the generation, // rather than the movable current symlink, so deselection retains cleanup authority. let composePath = paths.composePath; + let projectArgs: string[] = []; + let existing = false; try { composePath = realpathSync(composePath); + existing = true; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (existing) { + const infra = realpathSync(paths.infraDir); + if ( + dirname(dirname(composePath)) === join(infra, CONFIGURATION_GENERATIONS) && + basename(composePath) === GENERATED_COMPOSE_FILE + ) { + const base = readConfigurationGenerationBase(infra, basename(dirname(composePath))); + if (realpathSync(base) !== dirname(composePath)) + throw new Error("Unsafe generation Compose base"); + projectArgs = ["--project-name", STACK_PROJECT, "--project-directory", base]; + } + } return [ "compose", + ...projectArgs, ...(fileExists(envFile) ? ["--env-file", envFile] : []), "-f", composePath, diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 132d4be01..3a93cad5e 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -40,6 +40,7 @@ export { readAppliedConfiguration, readAppliedServiceIds, renderConfiguration, + retainedContainerComposeFile, withConfigurationGenerationWriter, } from "./configuration-generation"; export { From 2d06d10e0df98656d7e55d19d706295b393ad40c Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 05:17:20 +0100 Subject: [PATCH 025/135] docs(admin): specify fixed-reference fence settlement controls --- .../2026-10-23-admin-instance-management.md | 4 + ...-10-26-admin-fence-settlement-reference.md | 71 ++++ .../2026-10-23-admin-control-protocols.md | 12 +- ...admin-fence-settlement-reference-design.md | 388 ++++++++++++++++++ 4 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-26-admin-fence-settlement-reference.md create mode 100644 docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index 2911ec689..b6761689d 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -15,6 +15,10 @@ and its binding supplement `docs/superpowers/specs/2026-10-23-admin-control-prot All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Delegated work uses only Codex6.1-Sol or ClaudeOpus, preferablyCodex; Astra is prohibited by the latest user instruction. Local rendered verification, no screenshots attached to PR. +## Approved Task01.2 fixed-reference settlement correction + +Task01.2 consumes the [approved settlement design](../specs/2026-10-26-admin-fence-settlement-reference-design.md) and [implementation steps](2026-10-26-admin-fence-settlement-reference.md), independently READY at R5. Whole root-fixed settle bodies reference preallocated authenticated final proof; the logical-only control is distinct from ordinary forward execution. All revisions apply, including truthful nonterminal rollback, authenticated original reservation, cross-role refusal/UNUSED proof and bounded two-cell representation. No new task/migration/root/general late-body/raised global capacity. Existing50-task order and actual product/runtime/CI acceptance remain unchanged. + ## Approved Task06.1 retention correction Task06.1 consumes the [approved retention amendment](../specs/2026-10-25-admin-control-retention-amendment.md) and its [implementation plan](2026-10-25-admin-control-retention-amendment.md). Its fresh root key/origin, bounded renewable family GC, API receipt delivery and atomic stable-step reservation supersede conflicting earlier provisional interfaces;50-task order/scope remains unchanged. Follow-on02.3/02.4/01.2 use its actual exports. Earlier source clearance for02.2 does not qualify the new projectionv3/ack changes: those are reviewed and tested in this06.1 correction window. Agent-first then currentAPI/CLI producer cutover preserves legacy observations while rejecting fresh unsupported authority roots before effects. diff --git a/docs/superpowers/plans/2026-10-26-admin-fence-settlement-reference.md b/docs/superpowers/plans/2026-10-26-admin-fence-settlement-reference.md new file mode 100644 index 000000000..d0db1bfeb --- /dev/null +++ b/docs/superpowers/plans/2026-10-26-admin-fence-settlement-reference.md @@ -0,0 +1,71 @@ +# Fixed-reference fence settlement implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans. This is an amendment INSIDE Task01.2, not a new program task. + +**Status:** Approved after independent R5 READY; implementation and all actual proof gates remain Task01.2 work. + +**Goal:** Settle the original DM guard from authenticated actual final evidence without mutable reserved bodies or a new root. +**Architecture:** Root-fixed reference body, preallocated signed actual-owner commitment, one fixed settlement-only recovery control. Generic forward reservations unchanged. +**Tech Stack:** TypeScript/Zod, existing indexed control records/Ed25519/nominal authority, PostgreSQL guard/mirror and generated service-auth client. +**Spec:** docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md + +## Global Constraints + +- Wire version2 fixed reference; signed commitment version2≤16384bytes, signed control binding outerversion2/bindingVersion3; no generic late fields. +- Original root/origin/fingerprint/forwarddeadline unchanged; control≤60000ms/request≤15000ms, one exact attempted identity, no renewal/reacquire. +- Existing per-node128KiB/64intents/family128executions256intents/2MiBindex caps; preallocate before effects. +- Applied0042 immutable; legacy settled metadata readable, fresh legacy mutation closed; DM bearer+anchored agent proof required. + +## Review Focus + +- A forged reference or role must never supply outcome authority. +- Recovery after original expiry permits only exact guard settlement, never forward effects. +- Lost responses/crashes/concurrency preserve one immutable proof/bind/attempt. +- Unknown child/runtime/DB evidence cannot clear physical or logical fences. +- All reserved witness/control bytes/indices/ACKs participate in existing capacity/retirement. + +### Task A: Fixed contract and reserved proof producer + +**Files:** core services/data-manager/contracts/configuration-maintenance.ts; ops/configuration-maintenance.ts; build-artifacts/configuration-fence-settlement.ts (new focused module), control-authority.ts, child-phase.ts, target-lease.ts/retention integration; agent configuration-maintenance-{store,runtime}.ts. Preserve existing source under sole Task01.2 writer. +**Interfaces:** exact version2 DataConfigurationFenceSettle; publishOwnedConfigurationFenceSettlement(authority,maintenanceId); prepareConfigurationFenceSettlementControl(originalAuthority,maintenanceId); delegateConfigurationFenceSettlementControl(control). Nominal state/construction as spec, no caller proof fields. + +- [ ] RED genuine old body/late-outcome contradiction and fixed-ref fingerprint invariance; forged/missing/unknown proof and conflicting producer cases. +- [ ] Implement bounded preallocation/producer/private→shared one-way CAS from actual assigned owner finalizer; no generic mutation of reservations. +- [ ] Implement Revision2 exact OwnerIdentity/PreparedGuard/DecisionEvidence/ChildEvidence/GenerationEvidence/FollowUpEvidence/RuntimeAttestation/FinalEvidence/OwnerFinal schemas and domain digest preimages. Fixed inspector/store transition is sole nominal producer; prepared root/signed summary negatives and restored genuine final positives. +- [ ] GREEN focused core control/phase/target/family tests and agent maintenance-store/runtime tests. Record exact argv/output/causal source. + +### Task B: Control adoption, original-deadline exception and compatibility + +**Files:** core fixed settlement module, peer-admission/control-authority/target-lease/child-phase as exact closed integration needs; data-configuration-fence.ts; DM execution/configuration-maintenance.ts; canonical contracts and typed client, existing execution/guard tests. + +- [ ] RED original-root-expired valid settlement, fresh forward rejection, control-expired/no renewal, lost admission/SQL/mirror response and legacy-read/new-mutation distinction. +- [ ] Implement exact signed receiver-readable envelope/assertion, fresh process-local one-use ticket/outbox, nominal peer/adoption handle (NO physical lease borrower/forward execution), immutable60s control; ordinary gates reject variant. Specialized original-family slots reserve/bind execution/intent digests without late allocation, with both role capacity certificates before prepare readiness. +- [ ] For the known rollback/no-settlement row implement Revision3 ContainedNotSettledObservation only: nonterminal recovery_required, unchanged prepared guard/sequence/time, no ACK/release/retry or fabricated terminal. +- [ ] Implement authenticated PREPARE-time ReservationProjection in the preallocated pending control cell; bind both role capacities/envelope to projectionDigest/reservationDigest, wrong role/family/address/ordinal/body/subset rejected BEFORE any guard/capacity mutation. +- [ ] Implement EVERY Revision2 receiver recovery-table row with atomic SQL ControlTerminalTuple and original DB-session identity. Only conclusive committed publication may repair after expiry; no logical SQL replay, unchanged terminal digest/time. Keep ambiguous/live-session and legacy partial cases fenced. +- [ ] Implement Revision2 inventory/terminal/control-child/ACK/release/SQL-retirement verification and capacity ceilings. All root release/completion/GC use Revision3 disjoint prepared-and-settled OR authenticated never-prepared closure predicate; API delivery remains distinct from physical release and preserves no-deadlock ordering. +- [ ] GREEN actual owned-PG guard tests plus fixed peer/process tests, unchanged generic-forward regressions, capacity/ACK/retention negatives. + +### Task C: Assembled proof and canonical checkpoint + +**Files:** agent configuration-maintenance.integration.test.ts; API configuration-maintenance-postgres.test.ts; core/DM focused owned process fixtures already assigned01.2; exact new suites registered through existing required DB gating. + +- [ ] Implement Revision4 fixed DM refusal projection and minimum preallocated compound proof channel, strict UNUSED intent/CAS/confirmation producer and agent/family verifier, closed NoGuardEvidence preimages and acyclic publication order. NeverIssued has actual frozen publication proof; ReceiverRefused uses only genuine original DM canonical/effect source, no sibling-private reads. No opaque digest/caller absence shortcuts. +- [ ] Implement/test Revision3 NoGuardClosure, NeverIssued versus actual ReceiverRefused sources, issuance freeze race and UNUSED capacity proofs. Known local/DM capacity or busy refusal closes original root with zero settlement POST/UPDATE; ambiguous/actually prepared cases never select this branch. Retain counts until normal retirement, no fake peer/control execution. +- [ ] Exercise Revision4 lost-refusal response/restart and crashes between private canonical→shared refusal→DM intent/CAS/confirmation→agent freeze/own intent/CAS/confirmation→closure. Verify source-only repair, no circular digest, original target release and ordinary all-role family retirement; missing/ambiguous minimum channel or source stays unknown. +- [ ] Apply Revision5 exact two-cell representation: full signed NoGuardBundle in ALREADY reserved16384-byte commitment cell, original signed projection+compact links in8192-byte control cell. Verify both immutable cells/source preimages before closure; committed and unused variants mutually exclusive. No cap increase or late record/index allocation. +- [ ] Build real signed/validated capacity-null AND busy-after-reservation packets; assert complete framing/padding/mandatory-list counts fit all individual+aggregate bounds before effects. Compare static7240/2964 example, test exact overbudget refusal and bundle-before-link crash/tamper/cross-root denial/release+ordinaryretirement. Preserve actual api principal, never broaden principal union. +- [ ] Pin zero late execution/intent/index allocation, one-use dispatch denial on duplicate/restart, all receiver recovery-table states, private/shared/DB repair after expiry, historical duplicate after physical release, capacity boundary and delayed DM/API ACK/GC. +- [ ] Run actual API→agent→DM producer chain for applied/unused/contained, expiry/cancel, crash/lost-response/conflicting-writer/unknown child schedules. External Docker I/O alone may be controlled and named; no fake actual-owner witness. +- [ ] At coherent real registration ask controller standard DM OpenAPI→SDK; inspect generated diff and run freshness. No hand edits or migration. +- [ ] Final affected types, authority/inventory, scoped style, exact final-source covering receipts; append Task01.2 report with amendment dispositions/limits. Normal commit as part of Task01.2; controller fresh product review covers actual implementation. + +Self-review performed against all spec sections and five risk classes. NEW independent design readiness review required before product edits; user already authorized autonomous execution/no questions. + +Revision2 self-review: each R1 I1–I4 has strict type/producer/gate/recovery/capacity/ACK contract above and a task acceptance step; generic forward helpers remain unchanged, logical control has no physical borrower, original records/digests preserved. Independent R2 must verify these corrections before product authorization. + +Revision3 self-review: R2-I3 chooses no new failure-terminal variant; R2-I4 has complete authenticated reservation preimage BEFORE receiver capacity; R2-N1 provides disjoint truthful root closure. I1/I2 prior contracts remain, no new product authority granted until independent R3 readiness. + +Revision4 self-review: remaining R2-N1 now has exact public proof/UNUSED tuple, fixed local producers and agent/family consumers, preallocated channel/budgets and acyclic crash recovery. I1/I2/I3/I4 remain cleared except correction interactions. Independent scoped R4 readiness required. + +Revision5 self-review: R4size blocker fixed by using already reserved16KiB+8KiB cells with exact linked preimages; purestaticcompletepacket7240/2964bytes, no cryptographic/runtimeclaim. Independent R5 readiness before product authorization. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 390dbf507..a955cb8bf 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -8,6 +8,8 @@ The approved [late-child binding correction](2026-10-25-admin-late-child-binding The approved [fixed proxy validation-failure contract](2026-10-25-admin-proxy-validation-failure-design.md) and [Task02.3 steps](../plans/2026-10-25-admin-proxy-validation-failure.md) add only a private actual-producer proof for conclusively completed nginx validation before reload. The failed child stays failed/contained; a durable parent recovery obligation prevents live/restart release without exact original successful restore evidence. Other failure/transport uncertainty retains existing fences; no public operation, graph edge, signing key or migration changes. +The approved [fixed-reference fence settlement amendment](2026-10-26-admin-fence-settlement-reference-design.md) and [Task01.2 steps](../plans/2026-10-26-admin-fence-settlement-reference.md) supersede only the DM settlement wire/proof/control/preallocation/refusal/UNUSED/recovery/ACK/retirement details below. The complete settlement operation stays root-fixed; actual final facts live in authenticated reserved proof cells. This is a purpose-bound logical settlement control with no ordinary forward/physical lease capability. Apply revisions1–5 together (latest representation governs), including disjoint proved-never-prepared closure and exact two-cell proof bounds; no new root, generic late body, migration or raised global cap. Product acceptance remains required. + ## 1. Source constraints and ownership - `configuration-ownership.ts:acquireConfigurationOwnership` already uses exclusive mkdir, a 5-second wait, inode checks and no stealing after a crash. `configuration-generation.ts:withConfigurationGenerationWriter` owns the current pointer and commit evidence. Extend this discipline; PID age and TTL are not ownership. @@ -426,16 +428,16 @@ type DataConfigurationFenceReceipt = { state: "prepared" | "settled" | "recovery_required"; }; type DataConfigurationFenceSettle = { + version: 2; maintenanceId: string; candidateId: string; planDigest: string; - decisionId: string; - expectedSequence: number; - outcome: "unused" | "applied" | "contained"; - ownerReceiptDigest: string; + settlementId: string; }; ``` +The version2 settle body is the complete original root-fixed reference. Its settlementId derivation, receiver-readable reservation projection, signed final/NoGuard proof, late control correlation/ticket, logical-only adopter, crash repair and both-role reserved capacity are EXACTLY the approved fixed-reference amendment above. decisionId/expectedSequence/outcome/ownerReceiptDigest are verified commitment fields, never fresh wire authority. The old inline settle body is historical observation-only. Ordinary PREPARE still uses its original borrowed leaf; SETTLE must not use ordinary forward binding/adoption or mutate the generic child budget. + `planDigest` here identifies the entire frozen maintenance plan; `hardlinkPlanDigest` is the distinct existing§4 digest of canonical generation-link bytes, or null for no link work. Never compare that leaf's `DataExecutionInput.planDigest` to the whole maintenance digest. The signed root/grant supplies the maintenance identity, and the guard matches the leaf's revision/hash to its retained hardlinkPlanDigest. Register `prepareDataConfigurationFence` POST `/executions/configuration-maintenance/prepare`, `inspectDataConfigurationFence` GET `/executions/configuration-maintenance/:maintenanceId`, and `settleDataConfigurationFence` POST `/executions/configuration-maintenance/:maintenanceId/settle`, ahead of dynamic execution IDs. Prepare returns200 only after durable preparation, not202 worker admission; request/receipt≤32KiB,≤256 IDs,15s request limit. The typed client signatures are `prepareDataConfigurationFence(input:DataConfigurationFencePrepare,options:{transport:DataManagerTransport;controlAssertion:string;signal?:AbortSignal}):Promise`, `inspectDataConfigurationFence(maintenanceId:string,options:{transport:DataManagerTransport;witness:string;signal?:AbortSignal}):Promise` and `settleDataConfigurationFence(input:DataConfigurationFenceSettle,options:{transport:DataManagerTransport;controlAssertion:string;signal?:AbortSignal}):Promise`; adapters unwrap canonical response unions and throw bounded missing/busy/unavailable errors. Internal headers are set only by these service adapters; no browser parameter can supply them. Exact retries observe the retained record; changed owner/candidate/plan conflicts409. Lost requests use inspect only. These are **service-only**, never browser/proxy DataExecutionInput variants. Require existing DM bearer plus pinned ops-agent signature:§3.4 compact assertion references an anchored grant phase `configuration-fence`/`configuration-fence-settle`, exact body/owner/root/plan/target subset and one-use ID. Inspect verifies a purpose-bound ops-agent signed execution witness naming the same maintenance/candidate/root; the shared bearer alone cannot impersonate this role.02.3 defines the strict inspect witness domain `openmapx-configuration-fence-inspect-v1`, fields `{installationId,maintenanceId,candidateId,ownerCertificateDigest,issuedAt,expiresAt}`,≤4096 encoded bytes, lifetime≤15s; repeated reads grant no mutation. No arbitrary operation, path or callback is accepted. The DM preparation adopter settles its short borrowed node with committed **logical guard** evidence before returning prepared, retaining no live target borrower; future hardlinks adopt their own exact authorized grant under the original agent root. @@ -450,7 +452,7 @@ Persist a signed mirror at the existing DM-private recovery root `configuration- Known transaction rollback/no save permits only an explicit retained `abandon` decision for this exact unused reservation, under the same job/writer CAS. Agent verifies no continue decision, apply claim, effect attempt or child publication exists, then settles unused guard/root. A failed COMMIT response, lost prepare/decision POST, API death or unavailable checkpoint is **unknown**, keeps private candidate/target/DM/API-writer fences and is lookup-only. No TTL steal, automatic abort, optimistic release, proof replay or compensation. After API restart, reclaim the job under02 ownership CAS and inspect the original candidate/checkpoint on a transaction that locks the job row (thereby waiting out an earlier in-flight save transaction); exact committed checkpoint determines continue, exact known rolled-back/pre-save state plus no attempted decision/effect permits recording abandon. Absence/unavailable/ambiguous evidence alone never authorizes it. A retained decision marked not_attempted may be delivered once; any attempted decision/effect is only inspected. New desired state is never used to reconstruct the old decision. Expired unconsumed seals cannot be renewed; report saved/pending and retain or settle only from proven containment. -**Settlement/recreation.** `configurationMaintenance.settle` is a verification request, not caller-supplied success authority. Agent reads the exact plan's generation commit, hardlink, recreate and helper receipts, checks all attempts/descendants terminal or contained, immutable root ownership and exact desired runtime configuration/resource/image/mount/readiness evidence (private keyed comparisons for secret mounts). For applied, appliedRevisionId must be this candidate and all required follow-up complete. A known failed follow-up can settle as contained only with no live/unknown effect and truthful saved/committed/failed evidence; it never reports applied. Before release, compute evidenceDigest from the exact retained non-secret child/commit identities, write `settling` with that digest, send the signed finite DM settle leaf carrying that exact owner receipt, and reconcile its DB/mirror settlement. Agent attests the verified private owner receipt in the signed settle grant's exact body (`decisionId`, `outcome`, `ownerReceiptDigest`); DM verifies this pinned-role attestation, its anchored root/settling phase and owned decision binding plus no active authorized link child. It never tries to read the agent-private directory or trusts an unsigned digest. DM commits settled under the same admission guard and fsyncs its mirror/tombstone; admissions reopen only when both agree and no unresolved owner evidence remains. Agent releases root only after exact DM settled receipt plus every child agree. Response loss between guard settlement and release retains the root until lookup proves settlement; fresh target-bound work still conflicts. Unknown generation/helper/DM settlement retains all fences.08's helper uses this same maintenance identity/guard/frozen subset and terminal-before-drain rule; it creates neither a second reservation nor a second root. +**Settlement/recreation.** `configurationMaintenance.settle` is a verification request, not caller-supplied success authority. Agent reads the exact plan's generation commit, hardlink, recreate and helper receipts, checks all attempts/descendants terminal or contained, immutable root ownership and exact desired runtime configuration/resource/image/mount/readiness evidence (private keyed comparisons for secret mounts). For applied, appliedRevisionId must be this candidate and all required follow-up complete. A known failed follow-up can settle as contained only with no live/unknown effect and truthful saved/committed/failed evidence; it never reports applied. Before release, compute evidenceDigest from the exact retained non-secret child/commit identities, write `settling` with that digest, send the signed finite DM settle leaf carrying that exact owner receipt, and reconcile its DB/mirror settlement. Agent publishes the exact final owner commitment and root-fixed version2 reference control from the approved settlement amendment; DM derives `decisionId`, `expectedSequence`, `outcome` and `ownerReceiptDigest` only from that verified pinned-role commitment, checking original root/guard/decision plus no active authorized link child. Its specialized settlement-only handle, one-use attempt, conclusive SQL publication repair, ACK and disjoint NeverPrepared closure follow that amendment; they cannot be inferred from an unsigned digest or ordinary forward grant. It never tries to read the agent-private directory or trusts an unsigned digest. DM commits settled under the same admission guard and fsyncs its mirror/tombstone; admissions reopen only when both agree and no unresolved owner evidence remains. Agent releases root only after exact DM settled receipt plus every child agree. Response loss between guard settlement and release retains the root until lookup proves settlement; fresh target-bound work still conflicts. Unknown generation/helper/DM settlement retains all fences.08's helper uses this same maintenance identity/guard/frozen subset and terminal-before-drain rule; it creates neither a second reservation nor a second root. Required acceptance is future work:01.2 real API/agent/DM preparation/DB transaction/recreation probes;02.3 real DB admission races/startup guards;05.6 direct feed and04.6 mixed-import activation races paused **after final verify and before DB commit**, asserting one exclusive valid commit or zero desired/proof/invalidation writes. Drop prepare/decision/settle POST responses; kill API before and after save; recreate DM and exercise startup cron/CLI/data denial before readiness. Assert one retained root, one decision, no nested acquisition/self-conflict, no proof double consumption, no repeated generation/pin transition, no release while any outcome is unknown, and precise applied versus contained/failed classification.08.3 adds protected helper/DB outage coverage using the same guard; no product execution is claimed here. diff --git a/docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md b/docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md new file mode 100644 index 000000000..97b8e0d8d --- /dev/null +++ b/docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md @@ -0,0 +1,388 @@ +# Fixed-reference configuration-fence settlement amendment + +Status: APPROVED Revision5 after independent R5 READY (0 Critical/Important/Minor). Design readiness authorizes the assigned Task01.2 implementation; cryptographic/runtime/product qualification remains mandatory. Apply revisions together, with later revisions governing conflicting earlier representation details. Owner: Task01.2. No new program task, migration, key, principal, physical root, phase, target expansion or generic late-body mechanism. The user authorized autonomous design decisions and waived questions/approval gates. Independent R5 readiness has now cleared this design gate. + +## Problem and chosen approach + +Accepted ControlChildReservation fixes the entire operation fingerprint at root admission; its late-child binder changes identity/time only. Protocol5.4 instead puts actual final outcome, guard sequence and verified owner receipt digest into the future DM settlement body. Those facts do not exist during ownership-only prepare. Normal forward child normalization also cannot admit a new settlement leaf after the original forward deadline, although5.4 explicitly permits containment/settlement controls then. + +Considered: late mutable settlement body/field recipe would broaden the root-fixed-body invariant; predicting or enumerating outcome/digest is impossible; an unauthenticated reference would not prove settlement. Chosen: keep the COMPLETE wire operation root-fixed and reference one preallocated authenticated settlement commitment. The actual fixed owner producer publishes that commitment once after verifying final facts. Add only a purpose-bound recovery-control binder/adopter for the existing configuration-fence-settle edge, never generic deadline relaxation or another forward grant. All other reservations and old signed records remain exact. + +## Root-fixed wire contract and reference + +Replace fresh internal DataConfigurationFenceSettle with strict version2: + +```ts +type DataConfigurationFenceSettle = { + version: 2; + maintenanceId: string; + candidateId: string; + planDigest: string; + settlementId: string; +}; +``` + +IDs remain existing UUID/digest validators; settlementId is64 lowercase hex: +SHA256("openmapx-configuration-fence-settlement-v2\0" + installationId + "\0" + ownerPrepare.operationKey + "\0" + maintenanceId + "\0" + candidateId + "\0" + planDigest). +Root admission fixes this whole `{kind:"configuration-maintenance.settle",...body}` operation, original derived phase/ordinal/key, proposed DM execution UUID, recipient and exact sorted subset as today. No late field, null fingerprint or body mutation. Derive the reference independently at issuer and receiver from the original prepared binding; caller reference alone grants nothing. DataConfigurationFencePrepare/Receipt do not gain secret or mutable fields. API outer ConfigurationMaintenanceSettle remains the verification request defined by5.4; its evidenceDigest is never DM mutation authority. + +decisionId, expectedSequence, outcome and ownerReceiptDigest become fields of the signed final owner commitment, not caller-selectable DM body fields. Their validation and exact semantics remain required. No preallocated decision ID/schema extension is necessary merely to freeze this request. + +## Fixed authenticated producer and bounded commitment + +At root admission, before readiness or any effects, reserve one issuer-signed version2 uncommitted cell. Use the EXISTING indexed shared execution namespace `.admin-target-control/executions/ops-agent/.configuration-settlement.json`, plus the existing role-private owner record; no sibling-private read by DM. Domain `openmapx-configuration-fence-settlement-v2`, maximum16384 bytes signed/padded envelope. Account actual worst-case cell/index capacity with all other phase/control records before root effects, within existing128KiB per-node aggregate/64 intents/128 family executions/256 intents/2MiB family-index limits. Refuse prospective overflow; no cap increase, lifetime scan or completion-time index allocation. + +Cell binds installation, ORIGINAL origin and admitted prepare execution/fingerprint/admission digest, root certificate digest, settlementId, exact reference-body fingerprint, maintenance/candidate/plan and state uncommitted|committed. A committed value adds decisionId/value and its authenticated DB-decision commitment; original prepared DM receipt/fenceId/sequence; final owner receipt digest/evidenceDigest; outcome unused|applied|contained; exact original child/commit receipt digests; frozen desired runtime verification commitment and committedAt. Full fixed-owner record remains within5.4's128KiB and wire receipt64KiB caps; this small shared cell exposes bounded non-secret evidence only, no paths, seal, secret hash, proof nonce or config values. + +The ONLY producer is configuration-maintenance-runtime's actual finalizer from5.4, after its real owner/store/plan/child/commit/mount/image/readiness checks. Its core publisher `publishOwnedConfigurationFenceSettlement(authority, maintenanceId)` reads the fixed signed private owner record and authentic original child/decision evidence; it accepts NO caller outcome, digest, success flag or callback. It verifies owner/role/kind, same certificate/origin/binding/plan, prepared guard, decision and all attempted descendants. Actual runtime comparisons are made by the fixed production inspector before owner-record transition; the source path must install no public or injected replacement proof producer. Missing/unknown/mismatched facts refuse publication. + +Unused requires exact known rollback/abandon, no continue/apply/effect/child publication. Applied requires this candidate's committed generation plus ALL frozen follow-up/runtime evidence. Contained requires conclusive containment of every attempted effect with truthful saved/committed/failure state. Unknown can never publish a usable commitment. Persist the exact owner settling record first, then CAS the preallocated shared cell uncommitted→committed once under original ownership. Same byte-equivalent publication observes; conflicting second outcome/sequence/digest refuses. A crash between these writes repairs ONLY from that original authentic private commitment; no reconstruction from current desired state and no rerun. Once committed, later current config/clock cannot rewrite historical proof. + +## Purpose-bound settlement control, including expired roots + +Expose server-only `prepareConfigurationFenceSettlementControl(originalAuthority, maintenanceId)` returning a NOMINAL VerifiedConfigurationFenceSettlementControl. It accepts only the original restored ops-agent configurationMaintenance.prepare authority and fixed ID; it loads/validates the above committed cell, fixed private owner and still-owned root/guard. No caller body/time/mode argument. An unavailable/corrupt/missing original record or already-retired/released family cannot produce it. This type has no forward/delegation-general capability. + +`delegateConfigurationFenceSettlementControl(control)` performs one fixed bind/publication/admission outbox under original non-stealable lease/root/private-writer order. Reuse existing derived key/UUID/full reference-body fingerprint and fixed phase configuration-fence-settle/kind configuration-maintenance.settle/recipient data-manager/subset. Its write-once signed binding has outer `version:2`, inner `bindingVersion:3`, domain `openmapx-configuration-settlement-control-v3`, explicitly recording original parentForwardDeadlineAt plus controlStartedAt and controlDeadlineAt=controlStartedAt+60000. Root and origin bytes/deadline/fingerprint NEVER change. The separate child/control ExecutionRef uses that60s CONTROL deadline and original parent identity in executionFingerprint. This is the sole admitted exception to child-forward-deadline/origin clamping; it creates no new origin/family and authorizes no forward effect. Regular bindControlChildPhase, generic delegate/adopt and forward verifiers cannot create or use this control variant, even if given its reference; reject all other kinds/phases/roles/targets. + +The control variant is valid after original cancellation/expiry ONLY for consuming the preexisting committed settlement proof and settling the same logical guard. It never changes data/images/config/runtime or creates/delegates a descendant; root physical ownership remains original. Normal origin/cert inode and retained-family proof checks still apply; a forward retirement floor cannot resurrect an erased family. Use an explicit purpose-bound recovery decoder rather than weakening validateOriginRecord/verifyDeadlineEdge for every child. Generic admission cannot accept it as an ordinary forward parent/peer. Existing authenticated recovery semantics are extended only for this fixed leaf; the original root may be terminal-prepared or recovery-required, but all facts needed for the chosen settlement must be conclusive from original records, not public terminal status. + +Bind lookup precedes reading time; same original binding always returns its original key/ID/body/deadline. Preallocate binding/outbox space before root readiness. Persist original reference/commitment digest and one-use grant/admission intent BEFORE POST. A fresh missing binding after original corruption is unknown, never a new attempt. Each transport request≤15s under the stored60s attempt. If that control expires before verified completion, preserve the guard/root and inspect only; no rebinding, fresh key/UUID, time renewal or second effect attempt. An exact already-settled duplicate may return its historical receipt without a new mutation/adoption. This bounds the one settlement attempt without renewing any forward budget. + +DM's ONLY fixed adopter verifies pinned ops-agent signature, original root/certificate/family/held physical identity, fixed reference/reservation, committed owner cell and exact control binding/peer/body/execution fingerprint. It derives decision/expectedSequence/outcome/ownerReceiptDigest from that commitment, checks the stored guard and no active authorized hardlink child, then runs existing same-connection admission-guard SQL/mirror settlement. It cannot read agent-private state or trust the shared bearer/reference alone. Generic DM data admission rejects this control peer. Retain actual awaited DB/mirror publication/child settlement; deadline/abort/response loss is not containment. Existing DM→agent acknowledgement/shared settlement/indexed retention must cover this control, and parent root release still requires exact DM settled receipt plus every child. + +## Revision2 — authoritative concrete contracts (R1 I1–I4) + +This section refines the preceding architecture and governs any ambiguity there. The control is a **logical-guard settlement control**, not an ordinary forward execution or physical lease borrower. It retains an immutable ExecutionRef for correlation/terminal delivery, but generic execution/target-lease constructors never authorize it. The existing physical root and prepared guard retain ownership; fixed release/retirement predicates require the control proofs below. The previous normal physical adoption/tree implication is superseded: no future-deadline node is inserted into the ordinary physical lease tree and no TargetLeaseAuthority is returned. + +### I1: exact final proof and producer + +All hashes below use SHA256(domain + "\0" + canonicalJson(value)), excluding signatures/padding; hashes are64lowercasehex. Define `OwnerIdentity = {installationId,origin,ownerPrepare:ExecutionRef,admissionDigest,rootCertificateDigest,maintenanceId,candidateId,planDigest,settlementId}`. Define `PreparedGuard = {fenceId,preparedReceipt:DataConfigurationFenceReceipt,preparedReceiptDigest,preparedCanonicalDigest,preparedTerminalAt}`. The receipt remains ORIGINAL prepared state/sequence; `preparedReceiptDigest` hashes its exact canonical bytes with domain `openmapx-configuration-prepared-receipt-v2`. All original signed preparation/DB-mirror/canonical evidence must match. + +`DecisionEvidence = {decisionId,decision:"continue"|"abandon",checkpointVersion,checkpointDigest,apiParentAttestationDigest}` binds the original API job/candidate/plan and exact persisted row-version receipts, using5.4's original attestation verifier; no session/proof/value data. `ChildEvidence = {planIndex:0..31,execution:ExecutionRef,terminalDigest,terminalAt,effectSettlementDigest,publication:"not_started"|"committed"}` is the complete unique sorted set of attempted frozen steps/descendants. Reject unknown/unaccounted attempts; each tuple is checked against actual role-signed canonical/effect receipts and original graph, including nested authorized hardlinks. Private owner record may retain full identities; shared projection replaces each execution with its canonical digest, retaining planIndex and terminal/effect tuple. No public status or cached summary supplies a tuple. + +`GenerationEvidence = {state:"not_committed"|"committed",revisionId:string|null,commitReceiptDigest:string|null}` binds the original candidate and real generation-writer receipt; committed requires both exact revision/digest; not_committed requires nulls and actual noncommit evidence. `FollowUpEvidence = {state:"not_started"|"complete"|"contained",receiptDigests:string[]}` is sorted/unique and covers the frozen required follow-up; no unknown. `RuntimeAttestation = {verificationId:UUID,mode:"unused"|"applied"|"contained",requiredCheckSetDigest,checkedAt}`. requiredCheckSetDigest hashes ONLY the frozen non-secret list of service/step IDs and required check names/revisions/image expectations, never actual secrets, secret mounts' keyed comparisons or secret-derived values. Mode applied asserts ALL required config/image/mount/readiness checks passed; contained attests actual known runtime disposition after conclusively contained failure; unused requires no target effects. Full detailed observations and keyed secret-mount comparisons stay in the private inspection witness and never in shared hashes/projection. + +`FinalEvidence = {identity:OwnerIdentity,guard:PreparedGuard,decision:DecisionEvidence,children:ChildEvidence[],generation:GenerationEvidence,followUp:FollowUpEvidence,runtime:RuntimeAttestation,outcome:"unused"|"applied"|"contained"}`. `evidenceDigest` hashes its NON-SECRET projection (child executions replaced by their digests, no private inspection detail) under `openmapx-configuration-final-evidence-v2`. `OwnerFinal = {version:2,identity,ownerSequence,settlingAt,expectedDmSequence:guard.preparedReceipt.sequence,evidenceDigest,outcome,privateInspectionId}`. `ownerReceiptDigest` hashes OwnerFinal under `openmapx-configuration-owner-final-v2`. privateInspectionId is a random correlation UUID, not a digest of secret material. ownerSequence is the private owner's CAS sequence, unrelated to expectedDmSequence; neither may substitute for the other. Shared committed cell contains exact OwnerFinal/FinalEvidence NON-SECRET projection and their digests plus `phase:"committed"`; commitmentDigest hashes that payload under `openmapx-configuration-settlement-commitment-v2`. ACK/binding/control times/padding are outside these digest preimages. + +The fixed `inspectFrozenMaintenanceRuntime(originalOwner)` in `apps/ops-agent/src/configuration-maintenance-runtime.ts` invokes the production Docker/mount/readiness inspector (external process I/O is the only test seam), derives the required matrix from the frozen plan and creates module-private nominal `MaintenanceRuntimeInspection`. No injected inspector/result predicate, wire field or parsed summary creates it. Its actual signed private inspection witness domain is `openmapx-configuration-runtime-inspection-v2`, fixed owner-derived inspection subrecord≤64KiB INSIDE the already reserved private owner record (no separate file/index allocation), binding identity/inspectionId/check set and actual observations. Missing/unknown actual checks refuse the appropriate eligible mode. Shared projection contains only the RuntimeAttestation above. + +`finalizeOwnedMaintenanceState` verifies actual decision/generation/all-child evidence plus this nominal inspection, then the store's dedicated `commitFinal(expectedOwnerSequence, finalEvidence, inspection)` CAS writes signed immutable OwnerFinal and exact evidence under the existing bounded owner record. Ordinary checkpoint cannot enter final/settling-eligible state or supply the inspection. Recovery `restoreOwnedMaintenanceFinal` verifies the exact signed original final record, inspection witness, canonical child/decision/commit receipts and digest preimages without substituting current desired state; it returns nominal `VerifiedMaintenanceFinal`. A prepared execution, root recovery authority, signed summary or arbitrary signature is insufficient. `publishOwnedConfigurationFenceSettlement(authority,maintenanceId)` loads/restores this nominal final proof and CAS-publishes the exact committed cell; callers supply no outcome/digest/boolean. + +DM `verifyConfigurationFenceSettlementCommitment(originalGuard,controlEnvelope)` returns module-private nominal `VerifiedDmFenceSettlementCommitment` only after pinned-role/identity/body/certificate/final-proof/digest consistency and outcome implications. It obtains late decision/sequence/outcome/ownerReceiptDigest only from that verified committed projection. Its guard effect runner accepts only this nominal result and the adoption below, never ordinary parsed JSON. + +### I2: complete wire, guarded issuance and fresh dispatch + +One strict signed `ConfigurationFenceControlEnvelope` is the cross-role companion: `{version:2,bindingVersion:3,purpose:"configuration-fence-settlement",installationId,origin,ownerPrepare,admissionDigest,rootCertificateDigest,settlementId,reservationDigest,commitmentDigest,execution:ExecutionRef,operation:FixedReferenceSettleOperation,serviceIds,controlStartedAt,controlDeadlineAt,originalParentForwardDeadlineAt,dispatchId:UUID}`. No extension map. serviceIds equals the exact original prepared guard subset (no wider than root), operation equals the root-fixed entire reference body, execution is the original preallocated DM address plus derived full fingerprint and stored control deadline. Signed domain `openmapx-configuration-fence-control-v3`. Outer origin payload stays version2; do not pass origin-bearing version3 to generic validators. Original root origin/deadline remains byte-identical even when control deadline is later; only this fixed decoder accepts that explicit recovery distinction. + +Issuer commits envelope to its preallocated private observation and indexed shared `.admin-target-control/executions/ops-agent/.configuration-settlement-control.json`; DM reads that exact root-derived shared path and checks signature/digest, never an arbitrary referenced path. Internal assertion contains only fixed settlementId/envelope digest and pinned-role signature bound to body/execution, max4096encoded bytes, domain `openmapx-configuration-fence-control-assertion-v2`; shared envelope≤8192. It conveys no general parent/peer/lease assertion. Receiver requires existing bearer AND this assertion. + +`prepareConfigurationFenceSettlementControl` returns reconstructible nominal control for observation/verification. `delegateConfigurationFenceSettlementControl(control)` returns `{envelope,assertion,dispatch:"observe-only"}` OR `{envelope,assertion,dispatch:"fresh",ticket:ConfigurationFenceSettlementDispatchTicket}`. Ticket is process-local WeakMap branded, one-use and created ONLY by the winning unbound→bound+prepared publication, after both roles' capacity certificates and current original physical/family ownership are verified under lease→root ancestry→local publication writers. The subsequent private/outbox attempted CAS precedes transport. `dispatchConfigurationFenceSettlementControl(ticket,transport)` consumes it, revalidates same original commitment/control/root identities while marking attempted, then sends exactly ONE POST to the fixed settle endpoint. It never returns a reusable ticket; transport is fixed typed DM transport, not an arbitrary callback. Lookup/duplicate publication/restart/lost publication acknowledgement/attempted CAS loser always returns observe-only. No helper reconstructs a ticket from an envelope, file, UUID or live root. Lost POST invokes inspect only; exact already-settled observation is not another mutation. + +`verifyConfigurationFenceSettlementControl(installation,rawHeaders,originalGuard)` returns nominal `VerifiedFenceSettlementPeer`; `adoptConfigurationFenceSettlementControl(peer)` returns `ConfigurationFenceSettlementAdoption` with only `{inspect,runOriginalGuardTransition,repairCommittedPublication,close}` fixed methods, not VerifiedControlPeer/VerifiedParentExecution/VerifiedExecutionAuthority/TargetLeaseAuthority. It validates original root certificate/inode/family still held for FIRST use, committed witness and original guard before binding the reserved receiver slot. It is authorized after cancellation, expired forward deadline or terminal-prepared/recovery-required state ONLY for that original guard and reference. Missing/retiring/erased family or changed physical ownership denies first use, including a family older than the forward clock floor. Extant unresolved reserved families are read via their exact signed reserved-member recovery path; this cannot create/revive a family or bypass ordinary forward floors. + +Receiver admission has its own process-local one-use effect ticket: only the winning reserved→bound+attempted inode-CAS may construct a fresh adoption handle with runOriginalGuardTransition. Concurrent duplicate, restart or ambiguous adoption returns an observe-only handle exposing inspect/repairCommittedPublication/close, with no mutation method. The fresh run method consumes its ticket once; copied parsed peers/envelopes never recover it. The guard-transition method performs all actual effect-start/DB-deadline/publication checks below. Generic phase binder/ticket, retainAdmittedControlExecution, beginControlEffect, peer-admission, target-lease delegate/adopt/tree checks and forward clock/normalization remain unchanged: this specialized type never enters those constructors. Generic functions explicitly reject the specialized envelope/assertion. Shared **control-child** proof below is not a physical lease-tree child and permits no descendant. Fixed maintenance-root release/retirement verifies this mandatory logical-control obligation in addition to ordinary children; no generic release can bypass it. This avoids inserting a deadline-extended physical borrower or granting forward capability while still authenticating the one logical settlement. + +### I3: exact receiver journal, deadline and repair table + +Receiver uses the original guard SQL row/mirror, plus preallocated control admission/effect/terminal cells. It durably records verified envelope/commitment and attempted marker before the guarded SQL transaction. Under the SAME nonblocking admission guard and `SELECT ... FOR UPDATE` on original guard row, recheck peer/commitment/original root for first transition and check current time` on that reserved connection (under PostgreSQL63-byte limit). The uncertain connection is never returned/reused to replay SQL. + +The one UPDATE atomically stores `metadata.settlementControl` containing exact envelope/commitment/admission digests, outcome, original expected sequence, original prepared receipt digest and a `ControlTerminalTuple`. Tuple is `{execution,settlementId,commitmentDigest,fenceId,result:DataConfigurationFenceReceipt(state:"settled",sequence:expectedDmSequence+1),publication:"committed",terminalAt,terminalDigest}`. terminalAt is captured once by the actual SQL producer inside this transaction before UPDATE; terminalDigest hashes the tuple excluding terminalDigest under `openmapx-dm-configuration-settlement-terminal-v2`. The setting statement is executed once. The settling mirror contains the same proposed tuple before SQL COMMIT. Response loss is not a new UPDATE. Original prepared row's canonical receipt/time remain separately unchanged. + +| Durable state after crash | Authority/proof and permitted repair | Result/fence | +| ----------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Reserved/unattempted adoption, no effect-start/SQL | Exact preallocated slot/envelope; no recovered ticket or automatic POST | Inspect pending; keep root/guard; only original still-live fresh ticket may dispatch before deadline | +| Effect-start/SQL outcome unknown, recorded DB session still live or unverifiable | Neither timeout nor a prepared row proves noncommit; read-only observation only | Recovery-required, no terminal/release, no second logical transition | +| Prepared row/no control tuple after locking, original exact DB session conclusively absent or explicit actual rollback proved | Same guard/row lock plus pid+backend_start+nonce identity proves original connection cannot later execute; fixed repair may restore only the exact original prepared mirror and persist a signed NONTERMINAL contained-not-settled recovery observation as specified in Revision3 | Guard remains prepared, parent not released; no retry/renewed logical UPDATE; supervised recovery required for that exhausted control | +| Settled SQL with EXACT stored ControlTerminalTuple, unfinished mirror | Row-lock/admission-guard read of that actual atomic tuple plus verified original envelope/commitment/admission; repair copies exact tuple to mirror/private effect/terminal/shared control-child | Original committed publication repair ONLY, permitted after control expiry, no logical UPDATE or new adoption/execution | +| Equal settled DB/mirror, missing effect/shared child/ACK | Same original tuple/proofs; complete missing preallocated cells with identical bytes/time, then ACK protocol below | Historical committed outcome; root held until verified delivery/ACK, no effect replay | +| Fully settled historical duplicate after expiry/cancel/root release | Exact retained envelope/commitment/tuple/ACK chain; first-use held-lock check is NOT reapplied to historical observation | Return original receipt/time; no adoption, SQL transition or new grant | +| Legacy settled inline-body metadata | Strict legacy signed receipt/canonical preparation/settlement decoder, old bytes/time unchanged | Historical inspect/retirement only; no new commitment or effect | +| Legacy partial/uncertain or mismatched new halves/tuple | No manufactured new final proof | Admission closed/recovery-required; exact host recovery, no automatic settlement | + +Fixed `recoverCommittedConfigurationFenceSettlement` is the ONLY publication-repair implementation, shared by startup/inspect/duplicate. It never calls ordinary `withLeaf`/beginControlEffect or a logical mutation; it acquires role-private non-stealable writer plus DB advisory guard/row lock, waits out an earlier actual transaction before examining its tuple and compares signed original proof identities. Absence of a tuple is not noncommit proof without exact connection containment above. Repair cannot change terminalAt/digest/outcome. Missing/mismatched expected cells are recovery-required unless an exact original atomic SQL tuple plus verified admission proves the precise missing publication; no parsed public status fills it. DB unavailable remains closed. A new inspect may repair conclusive committed publication after expiry, but can never start the original SQL transition. + +### I4: preallocated cross-role capacity and terminal/ACK lifecycle + +At agent root admission reserve the sender control slot and records below. During the original DM fence PREPARE, BEFORE its logical reservation commits or returns prepared, DM reserves its control capacity on the same original family and proposed settlement address. It publishes signed shared `ConfigurationSettlementCapacityReceipt {version:2,installationId,origin,ownerPrepareDigest,settlementId,proposedExecutionAddress,reservedExecutionOrdinal,reservedIntentOrdinal,recordDescriptorsDigest,maxBytes}` under `openmapx-configuration-settlement-capacity-v2`. Agent cannot finish root readiness/prepare prepared result or allow configuration effects until this exact DM certificate is verified. Capacity failure precedes guard preparation/config effects; rollback/busy only releases unused reservation capacity through exact no-guard/no-effect proof, never age. + +Introduce a CLOSED family metadata entry `configurationSettlementSlot` keyed by settlementId/proposed address, not the late execution digest. It reserves one receiving DM execution member and one receiving intent, plus one sender intent in the existing agent parent. The sender parent already has execution membership; no late sender execution is allocated. Count these against128executions/256intents/64pernode from reservation onward. Each role's stable ordinal/index entries and maximum record bytes are reserved at preparation, even while executionDigest is null. Binding CAS fills `{executionDigest,controlBindingDigest}` ONCE from the exact signed control envelope; it neither allocates an ordinal/ticket nor renames/removes the reserved entry. Normal execution-digest lookup may resolve this bounded exact slot only when its binding matches and the fixed control verifier succeeds; unbound slots confer no authority. No generic arbitrary reservation/alias API. Ordinary registration/ticket/index allocation is not called at late binding, adoption, effect, terminal or ACK. + +All descriptors are root/role/settlement derived, no caller paths. Reserve the following maximum signed bytes before readiness; the sum of ACTUAL WORST-CASE encodings of the complete descriptor set must be≤65536 for this control (both roles combined), in addition to EACH existing node/lease/family/index cap. Individual maxima are ceilings, not a promise that all maxima fit. Reject the whole prospective plan if either role/index cannot reserve. Preallocated fixed-size cells are updated by inode-CAS; record sizes may not grow beyond reserved bytes. + +| Role / stable record suffix | Domain/version and maximum bytes | Lifecycle | +| ----------------------------------------------------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------- | +| Agent shared `.configuration-settlement.json` | openmapx-configuration-fence-settlement-v2/version2,16384 | uncommitted→one immutable final projection | +| Agent private `.configuration-settlement-binding.json` | openmapx-configuration-settlement-control-v3/outerversion2,8192 | unbound→exact control binding | +| Agent shared `.configuration-settlement-control.json` | openmapx-configuration-fence-control-v3/outerversion2,8192 | pending→exact receiver-readable envelope | +| Agent private `.configuration-settlement-dispatch.json` | dispatch-v2/version2,4096 | unattempted→attempted immutable outbox, fresh ticket ephemeral only | +| Agent private/shared `.configuration-settlement-ack.json` | ack-v2/version2,4096each | empty→copied exact DM terminal and matching agent ACK | +| DM shared `.configuration-settlement-capacity.json` | capacity-v2/version2,4096 | reserved capacity certificate before prepared | +| DM private `.configuration-settlement-admission.json` | admission-v2/version2,8192 | reserved→bound exact envelope/commitment/slot; specialized, no forward authority | +| DM private `.configuration-settlement-effect.json` | effect-v2/version2,4096 | not_started→attempted SQL/session identity→conclusive original tuple or unknown | +| DM private/shared `.configuration-settlement-terminal.json` | terminal-v2/version2,8192each | empty→immutable ControlTerminalTuple and signed committed settlement | +| DM private `.configuration-settlement-ack.json` | ack-v2/version2,4096 | empty→verified sender ACK | + +Other table domain names expand exactly: dispatch-v2=`openmapx-configuration-settlement-dispatch-v2`, ack-v2=`openmapx-configuration-settlement-ack-v2`, capacity-v2=`openmapx-configuration-settlement-capacity-v2`, admission-v2=`openmapx-configuration-settlement-admission-v2`, effect-v2=`openmapx-configuration-settlement-effect-v2`, terminal-v2=`openmapx-dm-configuration-settlement-terminal-v2`. The shared terminal signs the exact ControlTerminalTuple plus original specialized admission digest; its domain is the same named terminal-v2. All have outer version2/original origin; purpose-specific verification explicitly validates original origin structure/signature and recovery membership without entering a generic forward-clock gate. + +These fixed suffixes use existing role-private observations and indexed shared execution namespaces/family GC. Exact descriptors include scope/namespace/role/name/cap and are signed in the role's capacity receipt. Existing owner store≤128KiB, guard mirror≤64KiB, SQL metadata≤64KiB, lease ledger≤128KiB, admission receipt≤64KiB, phase cells≤8KiB/128KiBaggregate and family2MiB index are checked independently. The logical control adds zero physical lease-tree borrower nodes; its sender intent/required release obligation is accounted in the existing owner metadata. DM SQL `metadata.settlementControl` is bounded as part of existing64KiB guard metadata, no new row/table/migration. Private inspection witness is charged to owner record capacity and existing index BEFORE configuration effects, not allocated at finalization. + +DM publishes its actual committed `ControlTerminalTuple` with pinned DM signature and effect-settlement digest into both reserved terminal cells. The shared half is the **control-child proof**, explicitly not a normal physical lease child. Agent `observeConfigurationFenceSettlementControl` verifies signature, original specialized admission/capacity slot/envelope/commitment and exact result/time; copies tuple to its reserved private ACK cell and fsyncs it before publishing signed shared `ConfigurationSettlementAck {version:2,installationId,origin,settlementId,controlExecutionDigest,terminalDigest,terminalAt,commitmentDigest}`. DM copies/verifies that ACK into its own preallocated slot; loss merely delays cleanup. No ACK can change terminal evidence. + +Fixed `assertConfigurationMaintenanceRootClosure` gates EVERY maintenance-root physical release, owner completion and family retirement using the disjoint prepared-and-settled OR proved-never-prepared branches defined in Revision3. The prepared branch calls `assertConfigurationMaintenanceGuardSettled`, requiring original prepared guard plus matching DM committed terminal/control-child, sender durable copy/ACK and all ordinary children known/contained. Physical root release need not wait for API delivery once all these conclusive facts exist; the original API job's canonical projection/receiptDelivery ACK still gates API-parent completion and family cleanup. This avoids an ACK/result deadlock; no API ACK is forged. Agent outer settle result uses actual original DM tuple; API receives/persists/ACKs through existing job machinery, without exposing private plan/proofs. A contained-not-settled recovery observation is nonterminal and cannot satisfy guard-settled or root closure. + +Retirement's specialized admission verifier validates exact slot/envelope/commitment/DM terminal/ACK chain instead of claiming a normal forward admission/lease. Original guard SQL retirement adapter keeps original preparation canonical receipt/time and also requires the later control tuple, mirror equality, shared ACK and original owner/API family delivery obligations.72h starts at max(original preparation terminalAt, control terminalAt), without rewriting either. DM role ready-to-retire follows exact peer/API delivery, then existing copy-before-ACK/all-ready commit/all-clean/all-release family transaction; delayed/lost ACK, unknown control, active guard, absent counterpart, retiring family or busy writer denies cleanup. Only indexed preallocated records are reclaimed; no new collector. A root never reaching guard preparation may mark these reserved slots UNUSED only from exact no-row/no-mirror/no-effect refusal proof, preserving accounting until normal family retirement; no synthetic execution digest/terminal is manufactured for an unbound slot. + +Revision2 acceptance adds: a genuine prepared root/signed summary cannot create final proof; actual fixed finalizer and restoration can. Test all receiver table rows, exact DB-session containment versus live session, private/shared repair after expiry with unchanged tuple and zero UPDATE, historical duplicate after root release, reserved capacity at EVERY receiver stage and delayed API/DM ACK. Specialized envelope is refused by every ordinary peer/forward/adopt interface. Inspect may complete only conclusive original committed publications. Tests must demonstrate zero late capacity allocations and no grant/ticket reconstructed by duplicate/restart. + +## Revision3 — remaining recovery/reservation/closure contracts (R2) + +This section governs conflicts with earlier wording. I1's exact eligible final proof and I2's one-use control/ticket boundaries remain unchanged. Three corrections follow. + +### R2-I3: known rollback is NONTERMINAL recovery, never a settlement result + +Remove the earlier promise to publish a contained-no-settlement failure terminal. ControlTerminalTuple remains the ONE committed-settlement variant; no second terminal schema/domain is introduced. For a conclusively rolled-back/nonexecuting original SQL attempt, `recoverCommittedConfigurationFenceSettlement` may restore ONLY the byte-equivalent original prepared guard mirror and write into the already reserved private effect cell: + +```ts +type ContainedNotSettledObservation = { + version: 2; + state: "contained_not_settled"; + settlementId: string; + controlExecutionDigest: string; + controlEnvelopeDigest: string; + commitmentDigest: string; + preparedReceiptDigest: string; + expectedDmSequence: number; + attemptedAt: number; + observationAt: number; + sqlSession: { pid: number; backendStart: string; sessionNonce: string }; + containment: "explicit_rollback" | "original_session_absent"; +}; +``` + +Digest/ID/time bounds and original session proof checks are those already specified; backendStart is the exact recorded PostgreSQL timestamp, nonce is original UUID. Domain is existing `openmapx-configuration-settlement-effect-v2`. The observation is a signed, write-once refinement of that actual attempted effect; changed session/digest/outcome conflicts. It contains NO terminalAt, terminalDigest, success/settled result or incremented guard sequence. Keep DB guard and canonical preparation receipt/sequence/time unchanged; restored prepared mirror equals its original. Inspection projects `state:"recovery_required"` with the original sequence and a bounded recovery reason, never a fabricated terminal receipt. This is an observational projection; it does not rewrite the canonical prepared result. + +The one attempted control remains exhausted and the actual logical guard/owner fenced. No fresh ticket, SQL transition, retry, renewal, ACK, root release or family-ready/retirement is permitted from this observation. Operator recovery is required. Future duplicate inspections may return the same nonterminal observation; they cannot convert it to ControlTerminalTuple. A live/unverifiable session remains the earlier unknown branch and cannot produce even this observation. This implements the table without lying about a settled guard or adding an unbudgeted terminal variant. + +### R2-I4: authenticated PREPARE-time reservation projection + +The existing preallocated shared `.configuration-settlement-control.json` is now an exact disjoint cell: `{version:2,projection:SignedReservationProjection,binding:null}` at root admission; it may later become `{version:2,projection:,binding:SignedControlEnvelope}`. A separately defined UNUSED closure below is its only non-binding terminal state. No completion-time namespace/file/index is added. Include the whole pending and bound variants' worst-case signed bytes within its existing8192-byte descriptor before readiness. + +`ReservationProjection` is strict: + +```ts +type ReservationProjection = { + version: 2; + purpose: "configuration-fence-settlement-reservation"; + installationId: string; + origin: ControlExecutionOrigin; + ownerPrepare: ExecutionRef; + rootAdmissionDigest: string; + rootCertificateDigest: string; + maintenanceId: string; + candidateId: string; + planDigest: string; + settlementId: string; + reservation: { + version: 1; + phase: "configuration-fence-settle"; + ordinal: number; + kind: "configuration-maintenance.settle"; + recipient: "data-manager"; + serviceIds: string[]; + execution: { + executor: "data-manager"; + principal: "api"; + executionId: string; + operationKey: string; + }; + operationFingerprint: string; + }; + operation: FixedReferenceSettleOperation; + reservationDigest: string; + issuerMembership: { + role: "ops-agent"; + familyId: string; + rootMemberOrdinal: number; + rootExecutionDigest: string; + rootAdmissionDigest: string; + }; +}; +``` + +Existing validators apply (UUID proposed executionId, ordinal0..63, exact sorted bounded subset, digest64hex). reservationDigest hashes the canonical reservation under `openmapx-configuration-settlement-reservation-tuple-v2`; projectionDigest hashes the projection under `openmapx-configuration-settlement-reservation-projection-v2`. Signed projection domain `openmapx-configuration-settlement-reservation-v2`; the cell's signed envelope keeps existing control-v3 domain/outerversion2. The source publisher `publishConfigurationSettlementReservationProjection(originalAuthority)` accepts ONLY nominal original authority, reads the ACTUAL original admitted private reservation/operation and matching root certificate/family member under the original publication guards, and writes this exact projection before any DM prepare grant publication. It cannot accept a caller proposed UUID, descriptor or membership tuple. Same publication returns identical bytes; any conflict refuses. + +DM prepare reads this exact path derived from its authenticated PREPARE grant's parent ownerPrepare.operationKey. Before any capacity CAS, mirror/guard write or preparation success it verifies pinned ops-agent signature, installation/origin/owner/admission/certificate against the actual PREPARE grant and retained root witness, and original retained family membership via the issuer's signed source-verified membership attestation plus existing retained shared root certificate/witness. It does not read agent-private state. It checks phase/kind/recipient/ordinal/subset/address against the finite edge and original parent; recomputes child operationKey from original root/phase/ordinal, full operation fingerprint and derived settlementId; and rejects changed root/family/address/body/ordinal even when otherwise parseable. The proposed UUID comes ONLY from this authenticated original projection; DM never chooses or re-derives a random one. A digest without its preimage cannot reserve capacity. + +Capacity receipt now additionally binds `projectionDigest` and `reservationDigest` (and its existing proposed address/descriptors). Receiver reserved slot and guard metadata pin both digests; later signed control envelope MUST include projectionDigest and match that same complete projection and capacity certificate. The pending projection is retained verbatim when binding is published, so historical/recovery readers do not lose the PREPARE-time source. No later deadline/control fingerprint can replace its proposed identity. Invalid/missing projection returns refusal before guard preparation and zero capacity/SQL/mirror mutations. New acceptance covers wrong role/family/proposed address/ordinal/body/subset and projection substitution after capacity; final binding fills only previously reserved execution/binding digests and time, with zero late capacity allocation. + +### R2-N1: authenticated disjoint root closure + +`assertConfigurationMaintenanceRootClosure(originalOwner)` is the sole root release/completion/family-ready predicate and accepts exactly ONE of: + +1. **PreparedAndSettled**: an original guard DID commit. Require the full original prepared guard, eligible final owner commitment, committed DM ControlTerminalTuple/control-child and sender copy/ACK plus all actual ordinary descendant/own effect containment. The existing guard-settled predicate verifies this branch. A contained-not-settled observation cannot pass it. +2. **NeverPrepared**: original guard NEVER committed. Require exact `NoGuardClosure` below, all original own/child effect containment and UNUSED capacity-slot proofs. No prepared receipt/control execution/settlement terminal/ACK is fabricated or required. + +NoGuardClosure is a different strict signed proof, NOT OwnerFinal/FinalEvidence with a null guard. Its payload is `{version:2,identity:OwnerIdentity,ownerSequence,closedAt,noGuardEvidence,originalContainmentDigests:string[],unusedSlotDigests:string[],closureDigest}`. closureDigest hashes this exact non-secret tuple excluding itself under `openmapx-configuration-no-guard-closure-v2`; signature uses that same purpose domain. It contains no runtime-success/settlement claim. Its sole fixed producer first freezes original root issuance under the lease→root ancestry→private writer boundary, CASing the owner to `closing_without_guard` so no concurrent/future grant/POST/target effect can be issued; it then verifies one of these two exact evidence sources: + +- **NeverIssued**: original preparing outbox/grant/admission publication cells prove no DM prepare grant was ever published and no prepare POST/claim/effect attempt exists; no pending publication/outbox is ambiguous. This is read from authentic immutable original issuance state, not a caller boolean, timeout or absence inferred from PID. Since DM requires that grant before capacity/guard effects, the source proves no authorized receiver effect was possible. Planned but unjoined DM participant slots are UNUSED, not a fabricated peer admission. +- **ReceiverRefused**: the actual original DM prepare leaf has its pinned-role signed canonical refusal/effect proof (`failed_without_effect`,`not_started`) tied to exact owner/candidate/plan, with actual guarded checks showing no row and no mirror for THIS maintenance identity and no live original preparation SQL session. A capacity/busy refusal may provide it; another owner's guard is untouched. Any reserved receiver future-control slots are signed UNUSED by their actual DM owner and tied back to the original projection/capacity receipt. This uses the existing02.3 fixed no-guard refusal verifier; do not infer it from a public error/404 or manufacture an earlier canonical time. + +A crash after a prepare grant/POST/SQL attempt but before conclusive authenticated refusal, a live/unverified DB session, uncertain commit, any original mirror/guard or unknown child denies NeverPrepared. It stays fenced until original evidence/recovery proves PreparedAndSettled or a genuine original refusal; no automatic new NoGuard proof is synthesized from row absence. If guard actually committed before response loss, only PreparedAndSettled can close it. Known live rollback may use ReceiverRefused ONLY after the fixed original producer's real no-row/no-mirror/connection containment and canonical refusal publication; otherwise uncertainty remains. + +Publish NoGuardClosure into the existing reserved shared control cell's disjoint `{version:2,projection:,unused:SignedNoGuardClosure,binding:null}` variant,≤8192bytes; private owner keeps exact copy. Fixed capacity slots transition reserved→UNUSED once with this proof, without executionDigest/terminal. Already bound/attempted control, prepared guard or conflicting proof rejects this branch. Current signed private owner + shared proof gate physical root release only after every original actual child/own effect is conclusive; root completes with truthful not_committed/refused failure evidence, not applied or settled-guard success. Zero settlement POST/SQL UPDATE is allowed on this branch. + +Family verifier accepts this exact disjoint UNUSED proof in place of a nonexistent control terminal, including planned-but-never-joined participants ONLY with authentic NeverIssued proof and no corresponding admission/record publication. Reserved counts/indices are retained until ordinary all-role/peer/API-delivery retirement at≥72h after the actual original refusal/closure terminal time; no early capacity reuse, new collector or fake late execution. If DM reserved capacity, its signed UNUSED confirmation is required before family retirement; if never joined, NeverIssued source proves why no DM record could exist. Original normal refusal/containment canonical times remain immutable. Physical release must still occur before the existing released-lease retirement check; the NeverPrepared predicate now provides a real legal path to it. + +Required closure schedules: acquire root→local capacity refusal before grant, acquire root→actual busy/capacity DM refusal with no own guard, crash before/after grant and before/after row/mirror commit. Prove known no-guard cases release original targets with zero settlement POST/UPDATE and no fake execution/terminal, while every ambiguous/actually prepared case remains fenced or follows genuine settlement. Test race between NeverIssued closure and first grant publication: one wins under the same writer and the other is forbidden. + +## Revision4 — cross-role refusal and UNUSED construction (remaining R2-N1) + +This defines the missing `noGuardEvidence` preimages/publication/ordering. Other cleared contracts remain unchanged. All receipts below are non-secret, pinned-role signed, outerversion2/original origin; max lengths use the existing validators. No agent reads a DM-private file and no null inspection is proof. + +### Fixed DM proof channel and local source + +The existing shared DM capacity cell `.configuration-settlement-capacity.json` is a strict compound variant `{version:2,capacity:SignedCapacityReceipt|null,refusal:SignedDmNoGuardRefusal|null,unused:{intent:SignedUnusedCapacityIntent,confirmation:SignedUnusedCapacityConfirmation|null}|null}`. Its ceiling is8192bytes (replaces the earlier4096 capacity-only ceiling); the actual full worst-case signed compound bytes are reserved BEFORE use and still count toward the unchanged combined65536/node/family limits. Capacity certificate itself≤4096, refusal projection≤4096, unused intent≤2048, confirmation≤1024; the compound must fit8192, so ceilings are NOT summed as guaranteed space. No new late file/index/collector. The normal original DM PREPARE admission reserves this MINIMUM refusal channel before attempting the future-control execution/intent/record reservation. If even this base channel cannot be reserved, do not claim a conclusive receiver refusal: return unavailable/unknown, preserve issuer fence; a missing channel cannot be fabricated at closure. Thus a known future-control capacity refusal can be durable, but arbitrary resource exhaustion is not automatically proof. + +`publishDataConfigurationFenceRefusalProjection(originalPrepareAuthority)` first calls the actual DM-local `verifyRetainedDataFenceRefusal` on THAT installation. It reads original private admission, canonical refusal/time and signed failed_without_effect/not_started effect, plus actual no-row/no-mirror/connection containment established by the original guarded producer. It accepts no caller digest/reason/absence flag. Extend that fixed local producer/verifier only with a closed capacity-refusal reason for failure of the future-control reservation before guard writes; old busy proofs retain raw bytes. Capacity reason requires the allocator's actual known rolled-back/unused reservation transaction, not a caught arbitrary error. Unknown allocator/SQL/publication or a guard/mirror for this identity refuses proof. The actual initial canonical terminal must exist and remain byte-identical; this is source-verified projection of it, not a new terminal. + +Strict `DmNoGuardRefusal = {version:2,installationId,origin,ownerPrepare:ExecutionRef,rootAdmissionDigest,rootCertificateDigest,maintenanceId,candidateId,planDigest,settlementId,projectionDigest,reservationDigest,prepareExecution:ExecutionRef,prepareAdmissionDigest,refusalReceipt:{kind:"data-configuration-fence-refused",maintenanceId,reason:"busy"|"capacity"},effectReceiptDigest,effectSettlementDigest,canonicalTerminalDigest,canonicalTerminalAt,capacityReceiptDigest:string|null}`. Signed domain `openmapx-dm-configuration-no-guard-refusal-v2`. Projection digest is SHA256(domain+"\0"+canonicalJson(payload)), no signature/padding/ACK. The producer verifies `effectReceiptDigest = controlDigest(refusalReceipt)` and the actual canonical refusal digest `controlDigest({version:1,refused:refusalReceipt})`, retaining its original canonical time. If the closed capacity extension uses a new private version, its exact same digest preimage is fixed here; old busy bytes are never rewritten. + +`verifyPublishedDataConfigurationFenceRefusal(originalOwner,sharedCell)` in the agent verifies pinned DM signature, original prepare grant/root/admission/reference/projection/capacity identities and the actual shared preparation-child settlement. That child must have the exact prepareExecution, failed_without_effect/not_started and receipt digest matching the exposed refusal preimage. The signed projection attests the canonical private digest/time returned by the local verifier; the agent does NOT call that private verifier itself or infer time from status. It returns nominal `VerifiedPublishedDmNoGuardRefusal`. Family verification uses the same public projection/child linkage; the DM participant additionally checks its own original private canonical bytes at retirement. + +Publication is a fixed repairable projection: original refusal canonical/effect first, shared refusal projection next, before response/closure. A crash after the private canonical record may republish ONLY the byte-identical projection from that exact local source; a crash before a conclusive source remains unknown. The agent can read the root-derived shared cell after lost HTTP response; no new endpoint/callback or reliance on inspect-null is needed. + +### Exact UNUSED intent/confirmation, with no circular closure dependency + +`UnusedCapacityIntent = {version:2,installationId,origin,role:"ops-agent"|"data-manager",ownerPrepareDigest,prepareExecutionDigest:string|null,settlementId,projectionDigest,capacityReceiptDigest:string|null,sourceProofDigest,executionOrdinal:number|null,intentOrdinal:number|null,recordDescriptorsDigest,allocatedDescriptorDigests:string[],createdAt}`. Array is sorted unique,≤12, and exactly enumerates actually reserved fixed descriptors, not caller additions. Ordinals are the authentic original reserved ordinals or null when allocation never occurred; any bound/attempted control execution denies UNUSED. sourceProofDigest names the actual DmNoGuardRefusal for DM; agent source is the appropriate signed issuer freeze proof (NeverIssued OR IssuedPreparationRefused below), plus for ReceiverRefused the verified DM refusal/UNUSED confirmation digests, domain-hashed as a strict tuple. An issued prepare can NEVER use a proof asserting no grant was issued. Signature/digest domain `openmapx-configuration-unused-capacity-intent-v2`; size≤2048. + +`UnusedCapacityConfirmation = {version:2,installationId,origin,role,ownerPrepareDigest,settlementId,projectionDigest,intentDigest,slotStateDigest,confirmedAt}`; domain `openmapx-configuration-unused-capacity-confirmation-v2`,≤1024. slotStateDigest hashes the exact safe final metadata projection `{executionOrdinal,intentOrdinal,recordDescriptorsDigest,allocatedDescriptorDigests,state:"UNUSED",intentDigest}`. NO closureDigest appears in either payload. No dummy execution digest/terminal is created. A confirmation is valid only from the role that owns those reserved slots. + +DM's fixed `markRefusedConfigurationSettlementCapacityUnused(originalPrepareAuthority)` uses its actual local refusal verifier and reserved family metadata under the same non-stealable metadata/publication writer. It (1) persists signed UnusedCapacityIntent into the already reserved compound shared cell, (2) CASes the exact listed reserved/unbound slots to UNUSED with intentDigest, retaining counts/records, (3) rereads that exact metadata state, (4) signs/publishes confirmation in the same cell. It never derives a new proof from an agent closure. No late ticket/index allocation. Exact duplicate intent/CAS/confirmation observes original bytes/times; any conflicting or bound slot denies. A crash after intent or CAS repairs only the missing confirmation from the ORIGINAL intent and exact metadata. Ambiguous allocation state remains unknown. Partial allocations may be listed only when the existing family reservation transaction proves the full exact allocated set and no pending allocation; otherwise no UNUSED confirmation. + +Agent `verifyPublishedUnusedConfigurationCapacity(originalOwner,cell)` requires nominal verified DM refusal and validates the DM intent, confirmation, capacity/projection linkage and exact source/descriptor/ordinal digest preimages; returns nominal `VerifiedDmUnusedCapacity`. It never reads DM-private family state: the pinned DM confirmation is the fixed source attestation of that checked CAS. If no future slots were allocated, the signed intent/confirmation has null ordinals and the actual empty descriptor set; if DM reserved capacity before refusing, its actual certificate/digests must match. Missing confirmation prohibits agent closure and retirement. + +Agent's own UNUSED intent/confirmation uses the same schema/domains, its actual frozen issuance/family metadata and private reserved binding/dispatch cells; it does NOT depend on NoGuardClosure. Its signed values are embedded in the already reserved shared control cell when publishing the closure, within the existing8192 ceiling and combined budget. Agent fixed producer freezes ownerSequence/closedAt and source proof IDs in its private owner closure-intent BEFORE its UNUSED CAS; these original fields are reused after crashes, not reminted. It can repair missing own confirmation/closure only from that original intent and actual matching CAS/proof state. Future grant/ticket issuance stays blocked once closing_without_guard wins. + +### Closed noGuardEvidence union and cross-role order + +Replace opaque noGuardEvidence with exactly: + +```ts +type NoGuardEvidence = + | { + kind: "never_issued"; + issuerFreezeProofDigest: string; + prepareReservationDigest: string; + publicationStatesDigest: string; + issuerSequence: number; + frozenAt: number; + } + | { + kind: "receiver_refused"; + prepareExecutionDigest: string; + prepareAdmissionDigest: string; + refusalProjectionDigest: string; + effectSettlementDigest: string; + canonicalTerminalDigest: string; + canonicalTerminalAt: number; + dmUnusedIntentDigest: string; + dmUnusedConfirmationDigest: string; + capacityReceiptDigest: string | null; + }; +``` + +NeverIssued's `FrozenIssuerProof` is the actual root-writer source projection `{version:2,identity:OwnerIdentity,issuerSequence,frozenAt,prepareReservationDigest,publicationStates:[{phase,ordinal,operationKey,grantState:"unissued",intentState:"not_attempted"}]}`; complete original relevant grant/outbox entries only, signed `openmapx-configuration-never-issued-v2`,≤4096. publicationStatesDigest hashes that exact preimage under that domain. The fixed issuer producer reads source cells under issuance freeze; it cannot accept a caller list. A pending/issued grant, unknown publication or attempt denies. The shared NoGuardClosure embeds this signed source for family verification. Genuinely unjoined DM roles require no invented DM signature; NeverIssued proves no valid prepare/capacity action could have joined them. If any DM source/admission/capacity publication exists or issuance is ambiguous, this variant is rejected. + +ReceiverRefused uses a DIFFERENT strict `IssuedPreparationRefusedFreezeProof = {version:2,kind:"issued_preparation_refused",identity:OwnerIdentity,issuerSequence,frozenAt,prepareGrantDigest,prepareExecutionDigest,refusalProjectionDigest,remainingPublicationStates:[{phase,ordinal,operationKey,grantState:"unissued",intentState:"not_attempted"}],candidateClaimState:"not_consumed"}`. Its actual original prepare IS issued/attempted; all other target-mutating/future-control publications must be conclusively unissued/unattempted. The same fixed root writer reads the complete frozen plan/outbox/claim state and refuses missing/unknown/other-issued entries before signing, domain `openmapx-configuration-issued-prepare-refused-v2`,≤4096. ReceiverRefused.issuerFreezeProofDigest hashes this exact proof payload under that domain. NeverIssued instead keeps its existing unissued FrozenIssuerProof; the forms are disjoint and cannot substitute. + +ReceiverRefused's digests name the actual shared DM compound packet above. `unusedSlotDigests` in NoGuardClosure becomes a sorted unique list of `{role,intentDigest,confirmationDigest}` (≤2 roles), not opaque free hashes. Its closureDigest preimage/signature otherwise remain Revision3's exact tuple, now including these strict union/confirmation tuples. The final shared control cell's UNUSED variant is `{version:2,projection:,unused:{closure:SignedNoGuardClosure,issuerUnused:{intent,confirmation},issuerFreezeProof:SignedFrozenIssuerProof|SignedIssuedPreparationRefusedFreezeProof},binding:null}`. ReceiverRefused embeds its actual issued-preparation-refused freeze proof and refers to the immutable fixed DM compound cell via original prepare key; no arbitrary path. NeverIssued embeds its different unissued source proof and no DM role entry in unusedSlotDigests because that role genuinely never joined. Worst-case complete bytes for each variant are reserved before effects; overbudget plan refuses before root readiness. + +Acyclic ReceiverRefused sequence: + +1. Original DM prepare admission reserves base proof channel; authenticate projection and reserve future capacity if possible. +2. Actual guarded capacity/busy refusal proves no own guard row/mirror/effect; writes original private canonical/effect and shared normal child containment. +3. DM publishes source-verified refusal projection, then UNUSED intent→actual reserved-slot CAS→confirmation. All are independent of any future agent closure. +4. Agent freezes original issuance; verifies shared child/refusal/DM UNUSED packet; persists its closure-intent with exact ownerSequence/closedAt/source digests. +5. Agent own UNUSED intent→CAS→confirmation, then builds/publishes NoGuardClosure hashing both roles' existing confirmations. Revalidate all sources under original writer; no mutation/dispatch can restart. +6. Root release verifies the whole chain, original own/child containment and exact physical identity. Family retirement later requires original participant canonical containment, both exact UNUSED confirmations, peer/API delivery and72h/all-role transaction. Counts remain reserved until that cleanup. No settlement POST/UPDATE/control execution/terminal is fabricated. + +Crash repair follows those arrows only. Lost refusal response reads the shared packet; after private refusal/before projection DM may source-repair projection; after intent/CAS it may confirm exact original state; after agent freeze/own CAS it may finish the exact closure. A missing/ambiguous required stage remains fenced, never skipped by reading null/absence. No participant can rebind UNUSED to an execution or recover dispatch permission. Ordinary prepared guards must use the genuine committed settlement branch. + +NeverIssued follows issuer freeze→authenticated no-publication source→own UNUSED intent/CAS/confirmation→closure→physical release→ordinary retirement, with no DM call or signature. All actually joined participants still require their own original canonical/UNUSED proofs. Plan acceptance must run lost-refusal/restart and each publication/CAS crash boundary, plus proof substitution, duplicate confirmation, conflicting source and genuinely unjoined peer cases; assert original release and full ordinary family retirement without sibling-private reads or a made-up terminal. + +## Revision5 — fit proof across the TWO existing preallocated cells (R4) + +This replaces ALL earlier instructions embedding the entire UNUSED proof packet in the8192-byte shared control cell. It changes no proof, digest, signature, role/CAS/closure requirement or capacity ceiling. Both cells below were ALREADY reserved before effects; no late file/index or larger record is introduced. + +1. The existing **16384-byte shared commitment cell** `.configuration-settlement.json` has a disjoint UNUSED variant carrying the full original signed proof bundle: + +```ts +type NoGuardBundle = { + version: 2; + phase: "unused"; + settlementId: string; + projectionDigest: string; + closure: SignedNoGuardClosure; + issuerFreezeProof: SignedFrozenIssuerProof | SignedIssuedPreparationRefusedFreezeProof; + issuerUnused: { + intent: SignedUnusedCapacityIntent; + confirmation: SignedUnusedCapacityConfirmation; + }; +}; +``` + +Outer signature domain remains `openmapx-configuration-fence-settlement-v2`. `bundleDigest` hashes the exact canonical NoGuardBundle INCLUDING its original nested signed records but excluding its own outer signature/padding, domain `openmapx-configuration-no-guard-bundle-v2`. Nested proof signatures/bytes and all original digest preimages remain unchanged. This is NOT a committed settlement value: DM settlement verifier accepts ONLY phase committed and MUST reject phase unused. The producer cannot replace a previously committed cell or change a prior UNUSED bundle. + +2. The existing **8192-byte shared control cell** retains the complete ORIGINAL signed ReservationProjection exactly once, plus a compact immutable link: + +```ts +type UnusedControlCell = { + version: 2; + projection: SignedReservationProjection; + binding: null; + unused: { + bundleDigest: string; + closureDigest: string; + issuerFreezeProofDigest: string; + issuerUnusedIntentDigest: string; + issuerUnusedConfirmationDigest: string; + }; +}; +``` + +It contains NO embedded closure/freeze/UNUSED source payloads. Outer control-v3 signature/version remains as before. Agent/family NoGuard verifier reads BOTH fixed indexed root-derived cells, verifies their pinned-role signatures and exact projection/settlement/owner identities, recomputes bundleDigest and ALL compact links, then performs every original source/role/DM compound packet/child/UNUSED/closure check. References without their complete original authenticated preimages deny; no sibling-private read or arbitrary path. Missing/torn/different bundle or link cannot release, retire or authorize mutation. + +Acyclic order is unchanged through issuer own UNUSED CAS/confirmation. Next publish full NoGuardBundle into the original uncommitted commitment cell, then publish the compact link variant into the original control cell. Root physical release/completion requires BOTH exact values and all original proofs. Crash between these publications permits only filling the exact missing link from the original immutable bundle/private closure intent; no replacement bundle/proof/time/CAS, no new grant/POST/SQL. Shared commitment may transition uncommitted→committed OR uncommitted→unused, never between committed and unused. Owner issuance freeze and original role writers make competing eligibility mutually exclusive; ambiguous half-publication remains fenced. Original aggregate budgets reserve worst-case success AND UNUSED variants across both cells from initial readiness, including signature/container/padding bytes. + +The ReceiverRefused branch of NoGuardEvidence includes mandatory issuerFreezeProofDigest as already described in Revision4 prose; the exact strict schema must include it. Proposed DM ExecutionRef.principal is EXACTLY original ownerPrepare.principal=`api` for this API-only root, while executor/recipient are data-manager and the signer/issuer role is ops-agent. Never add ops-agent to the unchanged principal union or confuse transport issuer with principal. The ReservationProjection type above is corrected to api accordingly. + +### Static viable size accounting (not runtime or cryptographic proof) + +The pure canonical-JSON width fixture is retained in the controller workspace as `settlement-reference-static-size.py/.json`. It imports NO product code and uses synthetic86-character signature placeholders, not valid authority or runtime evidence. It uses the ACTUAL45-character timed root key, derived37-character child keys, normal api principal, data-manager target, UUIDs/64-hex digests/13-digit times. It includes THREE mandatory remaining publication entries (commit/recreate/settle with unique ordinals1/2/3), SIX allocated sender descriptor digests, TWO original containment references and BOTH role UNUSED references, plus the required issuerFreezeProofDigest. This is a viable MINIMUM schema-width ReceiverRefused/capacity-null case, not a guarantee all maximum plans fit. + +| Complete signed value | Canonical UTF-8 bytes | Existing reserved ceiling | +| --------------------------------------------------------------- | --------------------: | ---------------------------------------------------------------: | +| Signed ReservationProjection (included only in control cell) | 2346 | part of8192 | +| Signed closure | 2675 | part of bundle | +| Signed issued-preparation-refused freeze | 2014 | part of bundle | +| Signed issuer UNUSED intent | 1386 | part of bundle | +| Signed issuer UNUSED confirmation | 779 | part of bundle | +| COMPLETE signed NoGuardBundle incl wrapper/links | 7240 | 16384 | +| COMPLETE signed UnusedControlCell incl projection/links/wrapper | 2964 | 8192 | +| Both complete signed cells | 10204 | existing two ceilings total24576; also overall65536 packet check | + +Unlike R4's8295-byte SUBTOTAL forced into one8192 cell, each COMPLETE new packet fits its own unchanged cell with room for the explicitly included mandatory fields. Prospective validation MUST still compute every actual plan's complete signed/padded worst-case variants and all other role/terminal/ACK records against the independent65536/node/family limits before effects. A larger plan that exceeds either individual or aggregate limit is refused then, never split late or truncated. The DM compound channel ceiling remains8192; its variants and every other record remain within the already assigned accounting. + +Required product acceptance builds REAL signed/validated encoded packets for this capacity-null case AND busy-after-full-reservation, including full canonical source/confirmation inputs and all actual framing/padding. Assert individual+aggregate fit BEFORE guard/config effects, both-cell cross-verification through release/ordinary family retirement, crash between bundle/link, tampered/missing/cross-root bundle/link denial, success/UNUSED mutual exclusion and exact overbudget rejection. The controller's static fixture is sizing evidence only; it replaces no actual signature/producer/DB/retention proof. + +## Compatibility and generation + +Keep old root/grant/reservation raw bytes and comparison semantics unchanged. Old settled guard metadata/tombstones with the previous inline settle body remain readable, inspectable and retireable through strict legacy decoder; do not rewrite them. Fresh legacy inline settlement mutation is closed with explicit observation-only conflict; no missing commitment fallback. Preserve supported historical inspection/cancellation without fake migration. New service-auth request schema and generated SDK use version2 fixed reference. No changes to applied0042 or snapshot; metadata schema supports both exact historical and new retained forms. Root runs standard DM OpenAPI then SDK generation only at coherent actual-registration checkpoint; ops/main API generation only if actual contracts require it. + +## Required acceptance and self-review + +Causal test: genuine ownership-only root admission cannot freeze old late outcome body; new fixed reference remains byte-identical before/after actual effects and real commitment production. Actual API→agent→DM owned PostgreSQL prepare/decision/effects/settle, no synthetic signed success or permissive verifier. Both applied and known-unused/contained; missing/forged/cross-root/candidate/plan/decision/sequence/body/ref/witness denied. Required hardlink positive/no-decision tests remain. + +Pause/crash around owner→shared commitment, bind, grant/outbox, receiver adoption, SQL/mirror commit and response/ack/root release. Restart only repairs exact original records, no effect replay. Race two publishers/binders/receivers, conflicting outcome refused. Expire/cancel ORIGINAL root before settlement and prove one bounded control can settle verified work while zero new forward work; expire CONTROL and prove unknown/fenced/no renewed attempt. Busy writers never stolen; hardlink/child unknown denies proof/release; root certificate/inode unchanged; families/retention/accounting cover every reserved/final cell/control including ACK and delayed GC. + +Self-review: fixes body contradiction without weakening generic reservations; exact control exception needed by already-approved5.4 expiry semantics is confined to one kind; budgets/compatibility/no-migration preserved. Product implementation remains blocked on independent review of this amendment only; independent Task01.2 milestones continue. From f7265f9262bc8b508c5417494e5c8a022de0272f Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 05:52:51 +0100 Subject: [PATCH 026/135] docs(admin): allocate current batch import to strict candidate producer --- .../2026-10-23-admin-01-service-control.md | 2 + ...-23-admin-04-integrations-and-migration.md | 2 + .../2026-10-23-admin-instance-management.md | 4 ++ .../2026-10-23-admin-control-protocols.md | 4 +- ...dmin-ordinary-integration-import-design.md | 47 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-26-admin-ordinary-integration-import-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index ac013ae2a..d10235136 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -85,6 +85,8 @@ Task2's separate-process probes race CLI Start, single/bulk API Start, Update/bu ### Task 2: Wire update and share generation preparation — S01, S02 +The [approved current ordinary-import ruling](../specs/2026-10-26-admin-ordinary-integration-import-design.md) is part of this task: move only `configuration-migration/legacy-integration-import.ts` forward from04.5, using the existing nominal migration participant/purpose and one full-candidate transaction. Apply its exact replacement/enabled/secret preservation, strict producer and causal acceptance requirements; no future auth/binding/credential/feed constructor is opened. The current registered import must work through retained jobs, not an uninstalled-participant refusal. No task/migration count changes. + **Files:** Create `apps/api/src/services/{service-preparation,prospective-configuration,administrative-configuration-writer,configuration-maintenance}.ts`, agent `apps/ops-agent/src/configuration-maintenance-{runtime,store}.ts`, core `packages/core/src/ops/configuration-maintenance.ts`, API `__tests__/{configuration-maintenance-postgres,prospective-configuration-postgres}.test.ts` and agent `configuration-maintenance.integration.test.ts` plus focused module tests, `packages/cli/src/lib/service-lifecycle.ts`, `packages/cli/__tests__/cli-service-lifecycle.test.ts`, `packages/core/src/services/service-lifecycle-receipt.ts` and its test. Modify API `services/{admin-ops,admin-job-handlers,service-config-resolver,service-secrets,secrets,trusted-config-operations,trusted-config-publisher}.ts`, `integration-config.ts`, `server.ts`; agent `administrative-runtime.ts`, `production-runtime.ts`, `trusted-config-runtime.ts`, `trusted-config-source.ts`; CLI `commands/{services,compose}.ts`. Controller integrates core ops contracts/exports, finite candidate/maintenance types, source/preview/maintenance operations, safe failure projection and existing publisher/journal parsers in this exclusive window. Tests include existing trusted-config publisher/source, agent production runtime/writer concurrency, API admin-job-handlers/admin-ops-hardlinks and CLI update tests. No new migration; existing admin_job metadata holds the durable non-secret candidate fence, with bounded private filesystem candidate receipts. **Interfaces:** Implement protocol§5.4's exact `ConfigurationMaintenanceBinding/Prepare/Identity/Decision/Settle/Step/Receipt`, API `prepare/inspect/decide/settleConfigurationMaintenance` and typed `configurationMaintenance.prepare/inspect/decide/settle`. Consume02.3's `DataConfigurationFencePrepare/Receipt/Settle` and generated service-auth prepare/inspect/settle adapters;06.1 owns authority verification. Implement protocol§5's writer, strict service/integration readers, `ProspectiveConfigurationOverlay`, opaque `ValidatedConfigurationCandidate`, `ProspectiveConfigurationReceipt`, and source inspection; implement protocol§6's exact lifecycle helper/effects/receipt signatures. Consume02's `ExecutionRef`/void `FixedCliRunner` and06.1's nominal lease authority. Host-local optional expected revision is captured once under writer; agent wire requires it. `prepare` always renders the full selection. `ServiceLifecycleEffects.withOwnership` supplies the actual `ConfigurationGenerationWriter` and `TargetLeaseAuthority` to the inner helpers; no recursive lock acquisition. Export portable `CoordinatorMode`/`ServicePreparationResult` for04/06/08/09 and `ServiceLifecycleReceipt` for agent/CLI/API evidence. Existing `HardlinkApplySummary` moves to core service-results with API compatibility re-export. diff --git a/docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md b/docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md index f16aa1643..a76125ca4 100644 --- a/docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md +++ b/docs/superpowers/plans/2026-10-23-admin-04-integrations-and-migration.md @@ -153,6 +153,8 @@ The binding01.2 `persistAndApplyProspectiveConfiguration` signature remains unch ## Task 5: Configuration migration inventory, preview and apply (I05) +Allocation correction:01.2 owns the existing ordinary-only batch import adapter per the [approved ruling](../specs/2026-10-26-admin-ordinary-integration-import-design.md). It remains in this plan's owning `configuration-migration` module with the existing nominal migration participant. This task reuses that path and installs a SEPARATE staged migration constructor for pending-auth drafts/selected-binding CAS; a legacy intent never supplies that authorization. All other Task5 scope and migration0049 remain here. + **Files:** Create `apps/api/src/services/configuration-migration/inventory.ts`, `preview.ts`, `apply.ts`, `migration.test.ts`, `migration-postgres.test.ts`, `apps/api/src/routes/admin-configuration-migration.ts`, `packages/core/src/api/contracts/integration-admin/configuration-migration.ts`, `apps/web/src/components/admin/settings/ConfigurationMigrationPanel.tsx`, `ConfigurationMigrationPanel.test.tsx`; modify `apps/api/src/routes/admin-settings.ts`, `apps/web/src/components/admin/settings/SystemSettings.tsx`, `apps/api/src/services/integration-config-import.ts`, `docs/docs/administration/integrations-administration.md` via controller's shared-file queue. **Interfaces:** diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index b6761689d..63a01f199 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -15,6 +15,10 @@ and its binding supplement `docs/superpowers/specs/2026-10-23-admin-control-prot All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Delegated work uses only Codex6.1-Sol or ClaudeOpus, preferablyCodex; Astra is prohibited by the latest user instruction. Local rendered verification, no screenshots attached to PR. +## Approved Task01.2 existing ordinary-import allocation + +The [approved ordinary-import ruling](../specs/2026-10-26-admin-ordinary-integration-import-design.md), independently READY at R1, moves only the existing nonsecret integration-config batch import adapter from04.5 into01.2. It retains the closed migration participant/purpose, whole-candidate preflight and one writer transaction; future auth/binding/vault/feed workflows stay with their assigned tasks. No new task or migration; exact current-route and runtime acceptance remain required. + ## Approved Task01.2 fixed-reference settlement correction Task01.2 consumes the [approved settlement design](../specs/2026-10-26-admin-fence-settlement-reference-design.md) and [implementation steps](2026-10-26-admin-fence-settlement-reference.md), independently READY at R5. Whole root-fixed settle bodies reference preallocated authenticated final proof; the logical-only control is distinct from ordinary forward execution. All revisions apply, including truthful nonterminal rollback, authenticated original reservation, cross-role refusal/UNUSED proof and bounded two-cell representation. No new task/migration/root/general late-body/raised global capacity. Existing50-task order and actual product/runtime/CI acceptance remain unchanged. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index a955cb8bf..7360547ed 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -250,7 +250,7 @@ withConfigurationCommitParticipant(writer:AdministrativeConfigurationWriter, `authorize` is read-only before prepare; `verify` is read-only and binds actor/proof/source/preview/pin/map revisions to the candidate before persistence and again inside the write transaction. Only after all prospective budgets/schema/ownership checks pass, `persistAndApplyProspectiveConfiguration` invokes `commit` once inside its own transaction, alongside generic overlay and job checkpoint writes. Exclude only the exact managed feed key from generic persistence when its participant claims it, so05.6 writes it once. Commit failure rolls back ordinary+managed rows, credential revision/invalidation and consumed03 proofs; no target-mutating continuation is authorized. The already-prepared ownership-only reservation, when required by§5.4, can be abandoned only with exact known-unused/rollback evidence. Persist only participant kind/actor/operation+preview IDs/completed identity in the private job metadata, never callback/proof nonce/session/secret. Restart after commit observes original candidate/identity and never repeats proof consumption. Preview participant cannot enter persistAndApply, consume proof, save targets, enqueue or publish; discard its inert candidate by bounded TTL/queue cleanup. -04.1 owns binding participant expected-prior CAS/receipt;04.5 owns pending auth drafts and binding CAS;04.6 owns decrypt-stage/import intent and03 `check/checkMany` before prepare versus `consume/consumeMany` in transaction, with optional05 `PreparedFeedCredentialPatch`.05.6 owns `persistPreparedFeedCredentialPatch` on this **same** writer/transaction.01.2 lands the closed context/dispatcher and rejects uninstalled kinds until their owner task wires its adapter; it does not invent early mock proof/pin policy or expose those workflows prematurely. Acceptance includes two real connections with inverse caller lock schedules, nested credential/feed apply on one context, changed pin/proof after verify, failure after proof consumption (all rows/proofs roll back), and preview persist rejection. This is the finite extension seam for existing administrative invariants, not a general transaction callback service. +The [approved ordinary-import allocation](2026-10-26-admin-ordinary-integration-import-design.md) moves only the existing ordinary integration-config batch import constructor into01.2 under the owning built-in configuration-migration module and existing nominal participant/purpose. Its closed legacy-integration-config intent cannot authorize auth, binding, vault, service, selection, platform or managed-feed writes. All other uninstalled participant subcases still reject.04.1 owns binding participant expected-prior CAS/receipt;04.5 owns pending auth drafts and binding CAS;04.6 owns decrypt-stage/import intent and03 `check/checkMany` before prepare versus `consume/consumeMany` in transaction, with optional05 `PreparedFeedCredentialPatch`.05.6 owns `persistPreparedFeedCredentialPatch` on this **same** writer/transaction.01.2 lands the closed context/dispatcher and rejects uninstalled kinds until their owner task wires its adapter; it does not invent early mock proof/pin policy or expose those workflows prematurely. Acceptance includes two real connections with inverse caller lock schedules, nested credential/feed apply on one context, changed pin/proof after verify, failure after proof consumption (all rows/proofs roll back), and preview persist rejection. This is the finite extension seam for existing administrative invariants, not a general transaction callback service. Final proof naming is exclusively C03's `check` (single, exact consume argument shape) and `checkMany` (exactly the finite two-proof consumeMany argument shape), both read-only on the supplied transaction context. Direct transit uses check/consume; credential-import plus transit uses checkMany/consumeMany. Do not introduce validateProofs, a variable-size proof batch or a separate validator implementation. Root reconciles B05 to these names and the common prepared-patch `overlayChange` name; this producer accepts only the actual finite overlay variant. @@ -265,7 +265,7 @@ Final proof naming is exclusively C03's `check` (single, exact consume argument `BoundedConfigValue` follows `trustedConfigurationPayloadSchema`'s bounded JSON; schema-declared ordinary keys only, no raw env map. Disallow duplicate/contradictory changes to one field and limit each set/unset collection to128 keys; IDs use existing schema validators. `PlatformConfigurationPatch` has only optional `domain`, `acmeEmail`, `releaseSelector`, `defaultRegion`, `backupRetentionDays`, each `PlatformField = {source:"host"}|{source:"database",value:T}`; exact value validators are08.1's hostname/email/release-selection/allowed-region/1–36500 bounds.01.2 declares the closed patch type and rejects platform purpose until08.1 installs its typed resolver; it preserves existing platform state in every unrelated candidate once installed. Managed `data-manager/TRANSITOUS_API_KEYS` is accepted only with purpose transit-credentials and the validated05.6 authorization context, or04.6's adapter proving both credential-import and catalog-bound transit.credentials; generic secret purpose rejects. The scope consumer supplies its purpose-specific proof before calling prepare; no public caller can opt into a purpose to bypass proof. -Choose the typed apply operation deterministically: render/migration/platform→stack.render; selection→serviceSelection.apply (one selection change); service-config→serviceConfig.apply (one service ID); service-secret/transit-credentials→vault.apply (one service ID, managed map fixed data-manager); integration-config/integration-secret→integrationConfig.apply (one integration ID). Multi-target imports use migration and its finite participant. Every kind binds expectedCurrentRevisionId, expectedSourceRevision, coordinatorMode and the sealed revision in its body fingerprint. Integration vault changes remain in the API's private overlay/runtime reload context, not in the trusted envelope's existing projected integrationConfigs; do not leak integration secrets into host config by flattening effective resolution. +Choose the typed apply operation deterministically: render/migration/platform→stack.render; selection→serviceSelection.apply (one selection change); service-config→serviceConfig.apply (one service ID); service-secret/transit-credentials→vault.apply (one service ID, managed map fixed data-manager); integration-config/integration-secret→integrationConfig.apply (one integration ID). Multi-target imports use migration and its finite participant; the current ordinary integration-config import uses only the approved01.2 legacy subcase, and04.5 owns the separate staged migration/auth/binding constructor. Every kind binds expectedCurrentRevisionId, expectedSourceRevision, coordinatorMode and the sealed revision in its body fingerprint. Integration vault changes remain in the API's private overlay/runtime reload context, not in the trusted envelope's existing projected integrationConfigs; do not leak integration secrets into host config by flattening effective resolution. `ValidatedConfigurationCandidate` is an opaque frozen server-private object with readonly internal metadata `{candidateId:UUID,operationKey,expectedCurrentRevisionId,revisionId,sourceFingerprint,sealedBytes,affectedServiceIds}`. `sealedBytes` is a **count**, not byte content. It privately retains merged row values, source versions/ownership, payload, exact sealed envelope bytes, nonce/issue/expiry and typed operation. `ProspectiveConfigurationReceipt` is `{candidateId,revisionId,saved:"unchanged"|"committed"|"conflict"|"unknown",application:TrustedConfigurationApplyOutcome}`. Public routes project only approved metadata; never serialize candidate, source fingerprint, secret content or a public digest of a secret. Source fingerprint is HMAC-SHA256 over complete observed source/revision data using a domain-separated subkey derived from the API's existing32-byte ops token (`HMAC(token,"openmapx-prospective-source-v1")`); keep this server-private. This token is already required for sealing and avoids requiring a vault key when the instance has no stored vault secrets. Agent host sourceRevision uses its own role key to sign the private witness; public projection contains only an opaque random witness ID, never a public hash of host values. No new user-managed secret service. diff --git a/docs/superpowers/specs/2026-10-26-admin-ordinary-integration-import-design.md b/docs/superpowers/specs/2026-10-26-admin-ordinary-integration-import-design.md new file mode 100644 index 000000000..ce25978e4 --- /dev/null +++ b/docs/superpowers/specs/2026-10-26-admin-ordinary-integration-import-design.md @@ -0,0 +1,47 @@ +# Task01.2 current ordinary integration-config import ruling + +Status: APPROVED allocation ruling after independent R1 READY (0 Critical / 0 Important). This is an allocation correction inside the existing 50 tasks, not a new task or a generic authorization mechanism. Implement inside existing Task01.2 of [Service Control](../plans/2026-10-23-admin-01-service-control.md); product/runtime acceptance and independent Task01.2 review remain required. + +## Problem, decision and alternatives + +The current `importAdminIntegrationConfigs` route accepts a batch of installed integration configuration entries. `prepareIntegrationImport` validates known nonsecret fields and the existing boolean `enabled` field; the old `persistIntegrationImport` replaces target config rows before a full trusted snapshot is sealed. Protocol5.1 requires multi-target imports to use purpose `migration` and its nominal `configuration-migration` participant, while allocating that constructor to future04.5. Task01.2 simultaneously requires all current configuration producers to use preflighted retained candidates. These statements cannot all hold without an explicit subcase allocation. + +Ruling: move ONLY the adapter for this existing ordinary integration-config import forward from04.5 into01.2. Put it in the owning built-in module `apps/api/src/services/configuration-migration/legacy-integration-import.ts`. It constructs the existing `configuration-migration` participant for its closed **legacy-integration-config** intent. Use existing purpose `migration` and operation `stack.render`; do not add a participant kind, purpose, generic registry or ordinary multi-target bypass. The staged migration/auth/binding adapter remains04.5 and uninstalled; credential-import, transit-credentials and integration-binding adapters remain their assigned later tasks. + +Alternative defer the existing import until04.5 contradicts01.2's current-producer cutover. Alternative treat arbitrary multi-target changes as ordinary configuration would bypass the finite participant contract. Installing all04.5 now would prematurely pull in auth drafts, bindings, UI and migration receipts. The selected narrow adapter preserves current functionality with no additional privilege. + +## Closed intent and authority + +The module exports a fixed constructor/preparer for the existing route's body plus the live `AdministrativeConfigurationWriter`. It obtains integration schemas/registry and prior state through that writer's strict captured read context. It calls the actual existing `prepareIntegrationImport` validation; it does not accept caller-provided schemas, authorization callbacks, participant objects, purpose strings, proof handles or an asserted validated flag. The resulting intent is nominal, frozen and bound to this writer instance, actor, operation, exact normalized entries, lock-set, observed row identities/versions, registry/schema observation and exact overlay. Reusing it under another writer or operation, changing its batch, or forging a structural object rejects. Internal brand issuance must be fixed to this actual built-in producer; no exported arbitrary callback/registration surface. A private optional subcase discriminator is not wire authority. + +Only installed, declared integration IDs and `integration-config` overlay changes are permitted. Entries contain schema-writable NONSECRET ordinary keys and the current special boolean `enabled`; this is the existing integration enable flag, not a capability binding or selected-service change. Retain existing `prepareIntegrationImport` behavior for supplied schema-secret fields: omit them without returning their values. They never enter the candidate intent, typed envelope, audit or job metadata. No integration/service vault mutation, managed feed map, service configuration, selection, platform value, auth setting, binding or arbitrary SQL mutation is admitted through this subcase. No new reauthentication policy is invented for this existing admin-only nonsecret operation. + +Keep the route's nonempty/unique-ID/schema bounds and the whole prospective overlay limits, using the smaller effective bound. Batch import REPLACES each target's declared ordinary DB fields: set supplied ordinary values, unset omitted existing declared ordinary keys, preserve the previous boolean enabled when omitted (apply explicit true/false), and preserve vault entries and any schema-secret legacy DB values. No secret is converted into an ordinary unset. Unknown/undeclared legacy rows or incompatible fields receive the existing strict sanitized correction diagnostic; do not silently discard them to make the candidate fit. The legacy secret-preservation rule deliberately prevents a nonsecret import from deleting credentials. Unrelated integrations and all unchanged state remain included and unchanged. An empty effective patch is a no-op with truthful metadata, not a fabricated application. + +All proposed effective set/delete changes obey default < database < vault < config.json < env and01.2's host-source witness rules. Validate host/file ownership before persistence; compare any source observations again before commit. A pure unchanged entry does not constitute adoption or authorize overwriting a host source. + +## Participant and atomic cutover + +The fixed participant is mode apply, kind configuration-migration, managedServiceSecretKeys empty. `authorize` validates its live writer/actor/job/lock set and closed intent before candidate preparation. `verify` binds the exact retained candidate/overlay and original row/schema/source observations before persistence and again inside the candidate transaction. `commit` validates that same immutable import identity on the supplied writer transaction; it performs NO auth/binding/proof/vault/managed-feed writes. Generic overlay persistence is the SOLE writer of the ordinary config rows. A fixed read-only verification/no-additional-writes commit is legitimate here; it is not a fake reauthentication or proof consumer. Unsupported future migration intents still reject at construction, not via caller-selected empty proof requirements. + +Flow: current authenticated route -> retained stable API job -> global writer -> sorted complete batch integration locks -> fixed built-in import intent/participant -> prepare full candidate -> exact seal/size/source/private queue preflight -> one candidate DB transaction containing ordinary overlays + job checkpoint -> original retained typed apply -> awaited integration reload/evidence. No `persistIntegrationImport` before candidate preflight, no second global-pool transaction, no per-entry applies, no reconstruction/reseal from later rows. Generic participant `verify` and transaction behavior remain protocol5.1. Ordinary import alone does not claim a managed-map maintenance exception; if the frozen follow-up genuinely recreates DM, existing5.4 preparation/decision/settlement is mandatory. + +Retain original candidate/revision, stable operation key and unknown/apply evidence under the current job protocol. HTTP/browser callers must consume that retained job/result through the standard coherent contract cutover, not receive a permanent upgrade-client or unavailable-participant response. Count imported entries only after desired commit; application/reload failed or unknown must not report complete success. Keep audit event integration.config.import with nonsecret IDs/count/outcomes. Do not reuse legacy in-memory `saved.owns/restore` closures after restart. Desired committed rows remain visible with failed/unknown application until explicit authorized reconciliation; any existing conditional compensation may remain only if it proves exact ownership and is expressible within the original frozen candidate/effect authority. No blind rollback, fresh root or automatic mutation retry is introduced. + +04.5 extends the owning migration module with a SEPARATE fixed constructor for validated pending-auth drafts and selected-binding CAS. It must reuse the shared candidate/participant/writer path and this ordinary-import adapter where applicable; it may not reinterpret a legacy intent as proof of auth/binding authorization. Existing03 proof and05 managed-feed contracts remain unchanged. + +## Required causal acceptance inside01.2 + +Use actual route registration, parser, fixed participant, candidate and persister with the verified owned PostgreSQL fixture; external Docker I/O may be controlled only at the existing named boundary. Preserve all failed attempts/RED evidence and distinguish saved receipts from fresh runs. + +1. Existing two-integration import produces one full candidate/one retained stack.render admission and matching reload, preserving unrelated rows, vault/legacy secrets and omitted enabled; explicit enabled false applies. Generated-client-visible result remains truthful. +2. Whole seal oversize, one malformed/unreadable unchanged row, host/file override conflict, second-row validation failure or private candidate capacity failure: ZERO changed desired rows, proof consumption, effect dispatch and target-mutating publication. +3. Race source/row version after preparation; reverse caller ID orders on two real connections; verify fixed lock order and conflict/no lost unrelated writes. Failure after first ordinary row inside transaction rolls back the whole batch and job checkpoint. +4. Forged/stale/foreign/mutated intent or participant; migration overlay requesting service, vault, binding, selection, auth, platform or managed key; arbitrary multi-target ordinary-purpose construction: reject before any effect. Future migration/proof adapters remain unavailable. +5. Application POST response loss/API restart: observe original operation/candidate only, no second POST, reseal, import persistence or compensation. Reload failure/unknown retains truthful desired-versus-applied evidence; no complete-success DTO or leaked secret/source fingerprint. + +Affected exact tests: extend `apps/api/src/services/__tests__/prospective-configuration-postgres.test.ts` and create `apps/api/src/services/configuration-migration/legacy-integration-import.test.ts`; update actual registered route/generated-client coverage in the existing integration-admin test family. Run focused node suites with the project's existing owned-DB opt-in/registration, affected API/core types and authority checks; request controller standard generation only at coherent source/route/consumer checkpoint. No migration or hand-edit of generated files. + +## Self-review + +Coverage: this fixes only the current producer allocation; purpose/kind counts and50-task order unchanged. Exact replacement semantics, secret preservation, enabled ownership, closed issuer, same-transaction rows/checkpoint, source drift, unknown outcomes and later adapter separation are explicit. It neither fabricates missing auth proofs nor bypasses managed credentials. Cost if wrong: confined adapter/API compatibility corrections, not broader host authority. Runtime/product completion still requires the independent01.2 product review and program gates. From 2131330e3fcb12627959d42cc8635028bd257748 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 07:39:36 +0100 Subject: [PATCH 027/135] docs(admin): order coordinator readiness before maintenance ownership --- .../2026-10-23-admin-01-service-control.md | 2 + .../2026-10-23-admin-instance-management.md | 4 ++ .../2026-10-23-admin-control-protocols.md | 4 +- ...in-maintenance-coordinator-order-design.md | 42 +++++++++++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index d10235136..ed94268c7 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -85,6 +85,8 @@ Task2's separate-process probes race CLI Start, single/bulk API Start, Update/bu ### Task 2: Wire update and share generation preparation — S01, S02 +The [approved maintenance coordinator ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) is part of this task: maintenance-bound nonempty ensure-running performs the existing ordinary helper Start only after whole-candidate preflight and BEFORE maintenance root/guard acquisition, conditioned on the observed old/current revision. Original helper settlement/release/readiness and source/TTL rechecks precede unchanged ownership-only prepare; no new frozen Start edge or post-acquisition fallback. Retain its exact loss/restart/cancel/initially-stopped/require-running causal acceptance. + The [approved current ordinary-import ruling](../specs/2026-10-26-admin-ordinary-integration-import-design.md) is part of this task: move only `configuration-migration/legacy-integration-import.ts` forward from04.5, using the existing nominal migration participant/purpose and one full-candidate transaction. Apply its exact replacement/enabled/secret preservation, strict producer and causal acceptance requirements; no future auth/binding/credential/feed constructor is opened. The current registered import must work through retained jobs, not an uninstalled-participant refusal. No task/migration count changes. **Files:** Create `apps/api/src/services/{service-preparation,prospective-configuration,administrative-configuration-writer,configuration-maintenance}.ts`, agent `apps/ops-agent/src/configuration-maintenance-{runtime,store}.ts`, core `packages/core/src/ops/configuration-maintenance.ts`, API `__tests__/{configuration-maintenance-postgres,prospective-configuration-postgres}.test.ts` and agent `configuration-maintenance.integration.test.ts` plus focused module tests, `packages/cli/src/lib/service-lifecycle.ts`, `packages/cli/__tests__/cli-service-lifecycle.test.ts`, `packages/core/src/services/service-lifecycle-receipt.ts` and its test. Modify API `services/{admin-ops,admin-job-handlers,service-config-resolver,service-secrets,secrets,trusted-config-operations,trusted-config-publisher}.ts`, `integration-config.ts`, `server.ts`; agent `administrative-runtime.ts`, `production-runtime.ts`, `trusted-config-runtime.ts`, `trusted-config-source.ts`; CLI `commands/{services,compose}.ts`. Controller integrates core ops contracts/exports, finite candidate/maintenance types, source/preview/maintenance operations, safe failure projection and existing publisher/journal parsers in this exclusive window. Tests include existing trusted-config publisher/source, agent production runtime/writer concurrency, API admin-job-handlers/admin-ops-hardlinks and CLI update tests. No new migration; existing admin_job metadata holds the durable non-secret candidate fence, with bounded private filesystem candidate receipts. diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index 63a01f199..f86a6a3af 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -15,6 +15,10 @@ and its binding supplement `docs/superpowers/specs/2026-10-23-admin-control-prot All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Delegated work uses only Codex6.1-Sol or ClaudeOpus, preferablyCodex; Astra is prohibited by the latest user instruction. Local rendered verification, no screenshots attached to PR. +## Approved Task01.2 maintenance coordinator ordering + +The [approved coordinator prerequisite ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) resolves the held-root/helper-Start conflict without extending the graph. Maintenance-bound nonempty ensure-running uses the existing ordinary old-revision Start after full candidate preflight, conclusively settles/releases it, then acquires the original maintenance root/guard and rechecks. Require-running/empty links retain zero-lifecycle guarantees; no late fallback or new deadline. Existing Task01.2/runtime acceptance and50-task order remain binding. + ## Approved Task01.2 existing ordinary-import allocation The [approved ordinary-import ruling](../specs/2026-10-26-admin-ordinary-integration-import-design.md), independently READY at R1, moves only the existing nonsecret integration-config batch import adapter from04.5 into01.2. It retains the closed migration participant/purpose, whole-candidate preflight and one writer transaction; future auth/binding/vault/feed workflows stay with their assigned tasks. No new task or migration; exact current-route and runtime acceptance remain required. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 7360547ed..c09ee732d 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -401,7 +401,7 @@ API module exports `prepareConfigurationMaintenance(candidate:ValidatedConfigura **Durable root and finite continuation.** Agent stores a signed≤128KiB record at its existing private recovery `configuration-maintenance/.json`, with immutable binding, API attestation digest, root certificate, frozen plan and per-phase attempts/results. Its≤32 effect children plus fence controls stay within§3's64 intents/128KiB ledger bounds; refuse capacity before effects. Private candidate bytes remain in the existing reserved128-entry/16MiB trusted queue budget, pinned until settled, with no duplicate accounting. Keep terminal identities72h; unresolved records never expire/evict. A crash after root acquisition/before DM fence admission or after fence admission/before prepared result keeps `preparing/recovery_required` and uses exact lookup, never another root/prepare POST. -The frozen effect plan contains one original typed `stack.render|serviceSelection.apply|serviceConfig.apply|integrationConfig.apply|vault.apply`, then at most one `configuration.hardlinks.apply`, exact ordinary `service.recreate|service.recreateIsolated` children, and at most one `configurationHandoff.prepare`. Compute planDigest from canonical version1 binding, target closure and safe normalized step intents **excluding** their planDigest fields and derived fingerprints/grant IDs, then fill those fields and compute each operation fingerprint; no self-referential hash. Derive stable keys from§3.5 finite phases and sorted ordinal; each record fixes kind/body hash/deadline/target subset and prerequisites. The generation body includes maintenanceId/candidateId before sealing; follow-ups include maintenanceId, desired revision and planDigest. 01.2 owns the closed protected step intent with the exact08.3 target enum; it refuses platform/protected continuation until08.3 installs its fixed adapter.08.3 adds the typed operation schema and then materializes the frozen target/image/preview intent before save; no absent-module forward import, generic adapter registry or change of closure. Each API request still carries its purpose-bound parent HMAC; the agent matches it to the frozen step and issues/adopts the one-use signed grant **locally from its retained owner**, supplying a borrowed `OpsExecutionContext.artifactLease`. No API-made lease/assertion and no later root acquire. The fixed verifier recognizes a terminal prepare journal only with its same retained maintenance record and terminal authorized decision, then only for the next frozen unattempted step; terminal status alone cannot issue a grant. Internal grant/admission intent is persisted before any effect, using§3.4's normal one-use pending-adoption sequence. Unknown child admission is lookup-only. Hardlinks may delegate only to the generation-hardlinks leaf at depth2. The helper may be depth2 below terminal handoff preparation, with no descendant. Generation commit obtains the host writer under this authority, rechecks source/current revision and commits the exact retained seal; it releases the host writer before a child needing that writer. Runtime data/images cannot change during prepare/decide. Continue decision is terminally journaled **before** any child can start; no live maintenance or decide operation waits for its children and blocks helper drain. API job coordinates the frozen children; the external helper/agent recovery can settle already-authorized work after API replacement by inspecting receipts, never resubmitting attempted children. +The frozen effect plan contains one original typed `stack.render|serviceSelection.apply|serviceConfig.apply|integrationConfig.apply|vault.apply`, then at most one `configuration.hardlinks.apply`, exact ordinary `service.recreate|service.recreateIsolated` children, and at most one `configurationHandoff.prepare`. The maintenance-bound coordinator Start prerequisite, when required by§6, completes and releases its ordinary original child ownership BEFORE this maintenance root is acquired; it is not an extra frozen effect or authority edge. Compute planDigest from canonical version1 binding, target closure and safe normalized step intents **excluding** their planDigest fields and derived fingerprints/grant IDs, then fill those fields and compute each operation fingerprint; no self-referential hash. Derive stable keys from§3.5 finite phases and sorted ordinal; each record fixes kind/body hash/deadline/target subset and prerequisites. The generation body includes maintenanceId/candidateId before sealing; follow-ups include maintenanceId, desired revision and planDigest. 01.2 owns the closed protected step intent with the exact08.3 target enum; it refuses platform/protected continuation until08.3 installs its fixed adapter.08.3 adds the typed operation schema and then materializes the frozen target/image/preview intent before save; no absent-module forward import, generic adapter registry or change of closure. Each API request still carries its purpose-bound parent HMAC; the agent matches it to the frozen step and issues/adopts the one-use signed grant **locally from its retained owner**, supplying a borrowed `OpsExecutionContext.artifactLease`. No API-made lease/assertion and no later root acquire. The fixed verifier recognizes a terminal prepare journal only with its same retained maintenance record and terminal authorized decision, then only for the next frozen unattempted step; terminal status alone cannot issue a grant. Internal grant/admission intent is persisted before any effect, using§3.4's normal one-use pending-adoption sequence. Unknown child admission is lookup-only. Hardlinks may delegate only to the generation-hardlinks leaf at depth2. The helper may be depth2 below terminal handoff preparation, with no descendant. Generation commit obtains the host writer under this authority, rechecks source/current revision and commits the exact retained seal; it releases the host writer before a child needing that writer. Runtime data/images cannot change during prepare/decide. Continue decision is terminally journaled **before** any child can start; no live maintenance or decide operation waits for its children and blocks helper drain. API job coordinates the frozen children; the external helper/agent recovery can settle already-authorized work after API replacement by inspecting receipts, never resubmitting attempted children. **Fixed DM logical guard —02.3 produces before01.2.** Add `packages/core/src/services/data-manager/contracts/configuration-maintenance.ts` (portable types/canonical service-auth operations), `packages/core/src/services/data-manager/configuration-maintenance.ts` (typed client), and `services/data-manager/src/jobs/execution/configuration-maintenance.ts` plus tests. Define: @@ -460,7 +460,7 @@ Required acceptance is future work:01.2 real API/agent/DM preparation/DB transac Export portable `CoordinatorMode = "ensure-running" | "require-running"` from `ops/service-results.ts`. Make mode mandatory on preparation/hardlink typed operations, execution fingerprints and checkpoints. `prepareServiceGeneration(ctx:JobContext,scope:string,options:{coordinatorMode:CoordinatorMode}): Promise`; `ServicePreparationResult = {revisionId:string,hardlinks:HardlinkApplySummary,coordinatorMode:CoordinatorMode}`. Produce `publishServiceGeneration(...)` inside01.2 as the candidate publication stage and `applyGenerationHardlinks({revisionId,operationIdentity,coordinatorMode,signal?}): Promise` as the conditioned follow-up. The public wrapper retains all commit/follow-up evidence from `TrustedConfigurationApplyOutcome`; unknown cannot return an ordinary completed result. -`ensure-running` is Start/Update's policy: derive the full hardlink plan, obtain required coordinator readiness with an exactly revision-checked, durably owned `service.start`, then run `configuration.hardlinks.apply` under target→writer with the signed leaf delegation. Never call Start from inside the link leaf or hold a host writer while awaiting another agent operation that requires it. CLI uses its existing local Docker adapter under its owned writer where possible, not a recursive agent request. +`ensure-running` is Start/Update's policy: derive the full hardlink plan, obtain required coordinator readiness with an exactly revision-checked, durably owned `service.start`, then run `configuration.hardlinks.apply` under target→writer with the signed leaf delegation. For maintenance-bound candidates, the [approved coordinator ordering ruling](2026-10-26-admin-maintenance-coordinator-order-design.md) requires this helper after whole-candidate/queue/preview preflight but BEFORE ownership-only maintenance/root/DM-guard acquisition, using the nonnull observed OLD current revision. Observe its original settled result, conclusive release and actual readiness, then recheck full source/TTL/current/readiness under the unchanged maintenance protocol; no service.start is added to its frozen graph. After maintenance acquisition no independent Start/fallback is permitted. Nonmaintenance helper ordering and candidate-revision condition remain unchanged. Never call Start from inside the link leaf or hold a host writer while awaiting another agent operation that requires it. CLI uses its existing local Docker adapter under its owned writer where possible, not a recursive agent request. `require-running` is Compose Render: before any generation publication, validate the **prospective** confined hardlink plan. Empty plan needs no data-manager and produces empty applied summary. Nonempty plan requires selected/verified-running data-manager plus readiness; stopped/unavailable is a prerequisite refusal before commit. Recheck under target/writer immediately before linking; coordinator exit after preflight/commit returns committed/follow-up-failed (or unknown if unobservable), with zero Start/restart/recreate/pull effects across the whole call graph. Link failure never activates ensure-running as fallback.01.2 adds API-role sync15s `configuration.hardlinks.preview` input `{candidateId,revisionId,expectedCurrentRevisionId,expectedSourceRevision,coordinatorMode}`. Agent reads only the fixed private candidate receipt, uses existing `inspectTrustedConfigurationSnapshot` HMAC verification plus embedded original typed apply-operation/body-fingerprint validation, then renders a plan in memory without committing/consuming the sealed apply claim. Output is `{entries:number,requiredServiceIds:string[],coordinatorState:"not_required"|"running"|"stopped"|"disabled"|"unavailable",previewDigest:string}` with≤256 entries/IDs and≤32KiB result. Browser cannot submit a plan. Candidate budget validation and private prepared publication precede this read-only preview; DB save/generation publication follow it. Bind preview digest+mode to the apply execution/receipt, revalidate under host target/writer before commit, and verify candidates do not accidentally authorize a different typed kind.02.2 registers `compose.render` for checkpoint/lookup-only recovery;09.6 consumes this mode. diff --git a/docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md b/docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md new file mode 100644 index 000000000..e9a028e23 --- /dev/null +++ b/docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md @@ -0,0 +1,42 @@ +# Maintenance coordinator readiness ordering ruling + +Status: APPROVED after the independent R1 written report found READY (0 Critical / 0 Important). The provider failed final delivery after writing its complete report; this accepts the report artifact as design readiness only. Allocation remains Task01.2; no new program task, graph edge, principal, typed operation, migration or capacity increase. Implement inside existing [Task01.2](../plans/2026-10-23-admin-01-service-control.md); product/runtime acceptance remains required. + +## Conflict and selected disposition + +Protocol5.4 freezes generation -> hardlinks -> recreate/handoff and holds the full target closure from ownership-only preparation through settlement. Protocol6 requires ensure-running's nonempty hardlink plan to obtain readiness via a separate revision-checked settled `service.start` before the link owner's configuration writer. Starting that helper as an independent root after maintenance preparation conflicts with the retained target owner; running it during ownership-only prepare violates the no-effects contract. Adding a service.start effect to maintenance also fails to solve the initially stopped coordinator: its HTTP guard cannot prepare until it is reachable. + +Ruling: for MAINTENANCE-BOUND candidates only, ensure-running coordinator readiness is an explicit **preparation prerequisite**, executed BEFORE ownership-only maintenance/root/DM-guard admission. Use the already supported ordinary API-job `service.start` for the fixed serviceId `data-manager`, exactly conditioned on the candidate's observed current/seed configuration revision, not its future candidate revision. Fully settle and observe readiness, then enter the unchanged ownership-only maintenance protocol. This narrows the ordering rule in6, not the finite effect graph in5.4. Do not add service.start to ConfigurationMaintenanceEffect, childPhaseKinds maintenance-recreate or any target delegation allowlist. + +Alternatives: adding a maintenance-start edge introduces unnecessary graph/authority surface and still needs an initially stopped-coordinator prerequisite. Performing Start within ownership-only preparation violates5.4. Recreate substitution, a new root while the original is held, or requiring CLI for every stopped-coordinator maintenance operation violate the intended behavior. The selected ordering reuses existing actual producers and current-generation ownership. + +## Exact prerequisite sequence and bounds + +1. Under the same live nominal API job/global writer/sorted integration ownership, strictly acquire all current state and build the complete prospective candidate. Validate actual whole seal UTF-8/schema/source/ownership budgets and private queue capacity and retain its exact bytes/typed identity BEFORE helper admission. Derive the actual prospective hardlink plan via the existing non-consuming fixed candidate preview. No desired-row/proof mutation, apply publication, generation commit or maintenance prepare occurs before this preflight succeeds. +2. If mode require-running, perform NO coordinator lifecycle operation anywhere in this sequence. A nonempty plan requires enabled verified-running and reachable-ready data-manager before commit and rechecks under ownership, otherwise refuse with existing prerequisite outcome. An empty plan requires no coordinator Start. If a later follow-up independently recreates DM, its existing5.4 guard still requires reachable DM; empty links do not waive that guard or bootstrap first host installation. +3. For maintenance-bound ensure-running with a nonempty plan, use the existing fixed helper child identity under the original retained API job and its original deadline. Persist the exact service.start body/key/fingerprint/execution/admission attempt BEFORE POST using the existing stable child protocol. Body is `serviceId:data-manager, expectedConfigurationRevisionId:candidate.expectedCurrentRevisionId`. That observed revision MUST be nonnull/current cfg1 and data-manager must be enabled/registered. Missing current rendered generation is the existing initial-host/bootstrap prerequisite, not permission to render an unvalidated seed. The ordinary Start runtime must not render, reseal or create a new configuration generation implicitly. +4. This ordinary helper obtains/releases only its existing properly scoped target ownership and uses the existing production lifecycle exclusion, configuration-writer and release-store ordering. Respect actual Compose dependency closure, resource policy and protection rules exactly as ordinary Start; do not add generic privileged dependency authority, raw Docker access or a user-selected helper target. No maintenance target lease or DM prepared guard may yet exist. It never calls Start from the hardlink leaf or while holding a host configuration writer. Holding the API global+integration ownership is normal; host work never calls back to acquire it. +5. Wait for the helper's actual authenticated settlement/result and actual ready coordinator observation. Its target lease must be conclusively released before maintenance root acquisition. This result is an attempted lifecycle prerequisite, not configuration apply. Retain its original child identity/result in the API job checkpoint so response loss/restart does lookup-only observation and NEVER issues a second Start under a new identity. If attempted admission/containment/release/readiness is unknown, do not prepare maintenance, mutate desired rows, infer a missing lookup as success or free its fence. A conclusive failure is a truthful failed prerequisite. Cancellation/expiry blocks any new helper/maintenance admission and still observes already attempted work; no new deadline. +6. Recheck the full candidate source/row/schema/current-revision witness, private candidate TTL and actual coordinator readiness after the prerequisite and again during normal5.4 ownership-only preparation. A different generation or host source conflicts before desired save; do not reseal the old identity. Root acquisition is now the ORIGINAL maintenance root for this candidate; it is not concurrent with a still-live helper root. DM prepares its durable guard under that root and rechecks its normal busy/admission invariants. The ordinary Start does not assert the guard or exempt unrelated active work. A data job winning the gap makes preparation busy with zero desired writes; do not cancel it or steal ownership. +7. Continue existing DB decision/authorized frozen generation/hardlinks/recreate/handoff/settlement under that SAME maintenance root. There is no additional coordinator Start child in the frozen plan. Immediately before the hardlink writer and inside it, check original candidate/current revision/mode, same root and actual coordinator readiness, as existing6 requires. If the coordinator stops between prerequisite/prepare/commit/link, fail truthfully (or retain unknown) and use existing original-root containment/settlement; NEVER issue a fallback independent Start, recursive Start from the link owner, new root or recreate substitution. A committed generation plus failed links stays committed/follow-up-failed or unknown, never applied. + +A successful prerequisite may leave data-manager running on the OLD current generation if later preparation/save fails. This is the explicit ensure-running effect, recorded in its settled child receipt; do not automatically stop it or claim no lifecycle effects for ensure-running. Require-running and empty-plan render paths retain their zero-lifecycle guarantees. Existing frozen recreate applies the candidate's DM configuration when required; the preliminary Start does not claim to apply new desired settings. DM startup admission fencing before cron/workers and durable guard survival across its later recreation remain mandatory. + +## Scope/compatibility + +Nonmaintenance ensure-running keeps its existing candidate-publication then helper-Start/hardlinks behavior and exact candidate revision condition. Only maintenance-bound callers route their nonempty helper prerequisite earlier, and must suppress the old unconditional helper call later. Keep portable CoordinatorMode and body/operation fingerprints unchanged. Current API services/config/credentials/selection/import/Compose callers consume the same existing retained job/candidate machinery; no public boolean, path, generic callback, fake ready flag or new maintenance purpose grants this behavior. Condition is derived privately from the actual validated candidate plus its fixed follow-up plan, never browser authority. + +The preflight/Start may consume the existing5-minute candidate TTL. Expiry before maintenance/save fails or abandons only the exact unattempted candidate under existing rules; it never reseals a previously attempted identity or repeats Start automatically. No Stage/04/05/08 ownership or Full Update02.4 graph change is implied. Existing32 effect/64 intent/queue caps remain and count the actual API-job helper identity; no extra intent is hidden or added to the frozen maintenance record. + +## Causal acceptance in01.2 + +- Preserve actual-maintenance-current-producer-red-01 as baseline, then use the actual API job/strict candidate/real PostgreSQL, production agent runtime and real DM SQL/runtime receiver. Control only named external Docker I/O. A nonempty ensure-running maintenance candidate with initially stopped Docker coordinator must show exact old-revision helper Start and settled release BEFORE maintenance acquisition/fence, followed by original-root generation+authorized hardlinks, one start and no nested acquire. Do not substitute a permissive decision/resource verifier or treat an in-process fixture with no real DM admission as full proof. +- Already-running coordinator, explicit current-revision/source race and stopped protected/disabled/prerequisite dependency cases use the actual policy/closure. A changed seed/failed readiness, failed seal/queue/schema or busy DM guard yields no desired/proof writes; no unsupported privileged dependency action. Record the preliminary Start if it legitimately occurred before a later busy/conflict. +- Kill/drop helper POST/result before receipt checkpoint and restart the API: exact original lookup/adoption/settlement, at most one Docker Start; no maintenance root while helper effect/release unknown. Cancel/expire both before and after settled Start; no new phase/key/deadline, safe existing containment. +- Stop coordinator after successful prepare/decision or after generation commit: no late Start/recreate fallback; truthful saved/commit/follow-up outcome, existing root/guard settlement rules. Require-running nonempty stopped/running and empty-plan paths show zero lifecycle end-to-end. Preserve the mandatory matching/no-decision negative and real authorized middle-owner positive proofs; no new graph phase appears. + +Extend existing01.2 owned-PG/current-producer/require-running tests and actual production lifecycle containment fixtures. RED/GREEN commands and exact failed attempts/source hashes remain the implementer's responsibility. Standard generation only upon a coherent checkpoint; no applied0042 edit. Product acceptance and fresh independent Task01.2 review remain required. + +## Self-review + +This ordering resolves both retained-root conflict and initially unreachable coordinator without adding authority. It states seed versus candidate revisions, whole-preflight-before-lifecycle, release-before-prepare, truthful preliminary side effect, TTL/race/cancellation/unknown behavior and exact no-fallback boundaries. The maintenance root/frozen plan and settled logical control amendment remain unchanged. Cost if wrong is confined ordering/receipt/test rework; no generic graph permission is introduced. From 672b1933b79e8b366cf82d006a64cf5a78aa6933 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 26 Oct 2026 15:44:26 +0100 Subject: [PATCH 028/135] docs(admin): define outer settlement request evidence checksum --- .../2026-10-23-admin-01-service-control.md | 2 + .../2026-10-23-admin-instance-management.md | 4 ++ .../2026-10-23-admin-control-protocols.md | 2 +- ...admin-fence-settlement-reference-design.md | 2 +- ...0-26-admin-settle-request-digest-design.md | 65 +++++++++++++++++++ 5 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-26-admin-settle-request-digest-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index ed94268c7..c73ac8d49 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -85,6 +85,8 @@ Task2's separate-process probes race CLI Start, single/bulk API Start, Update/bu ### Task 2: Wire update and share generation preparation — S01, S02 +The [approved outer settle request digest](../specs/2026-10-26-admin-settle-request-digest-design.md) is part of this task: implement the exact portable observation-checksum helper, API original-attempt checkpoint, actual owner/sequence comparison before first settle control CAS, and duplicate/restart retention. Keep actual final-v2 runtime evidence distinct and agent-produced; execute the specified mismatch/stale/unknown/race/recovery/positive causal tests without adding a receipt field or endpoint. + The [approved maintenance coordinator ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) is part of this task: maintenance-bound nonempty ensure-running performs the existing ordinary helper Start only after whole-candidate preflight and BEFORE maintenance root/guard acquisition, conditioned on the observed old/current revision. Original helper settlement/release/readiness and source/TTL rechecks precede unchanged ownership-only prepare; no new frozen Start edge or post-acquisition fallback. Retain its exact loss/restart/cancel/initially-stopped/require-running causal acceptance. The [approved current ordinary-import ruling](../specs/2026-10-26-admin-ordinary-integration-import-design.md) is part of this task: move only `configuration-migration/legacy-integration-import.ts` forward from04.5, using the existing nominal migration participant/purpose and one full-candidate transaction. Apply its exact replacement/enabled/secret preservation, strict producer and causal acceptance requirements; no future auth/binding/credential/feed constructor is opened. The current registered import must work through retained jobs, not an uninstalled-participant refusal. No task/migration count changes. diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index f86a6a3af..c33abc5c5 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -15,6 +15,10 @@ and its binding supplement `docs/superpowers/specs/2026-10-23-admin-control-prot All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Delegated work uses only Codex6.1-Sol or ClaudeOpus, preferablyCodex; Astra is prohibited by the latest user instruction. Local rendered verification, no screenshots attached to PR. +## Approved Task01.2 outer settle request digest + +The [approved digest derivation](../specs/2026-10-26-admin-settle-request-digest-design.md), independently READY at R1, separates the existing outer request observation checksum from the agent-produced final-v2 evidence digest. Its exact portable receipt projection, first-admission owner/sequence CAS and retained winner/restart semantics add no schema, endpoint or authority. Existing Task01.2 product/runtime acceptance and50-task order remain unchanged. + ## Approved Task01.2 maintenance coordinator ordering The [approved coordinator prerequisite ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) resolves the held-root/helper-Start conflict without extending the graph. Maintenance-bound nonempty ensure-running uses the existing ordinary old-revision Start after full candidate preflight, conclusively settles/releases it, then acquires the original maintenance root/guard and rechecks. Require-running/empty links retain zero-lifecycle guarantees; no late fallback or new deadline. Existing Task01.2/runtime acceptance and50-task order remain binding. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index c09ee732d..0a5c07fcf 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -452,7 +452,7 @@ Persist a signed mirror at the existing DM-private recovery root `configuration- Known transaction rollback/no save permits only an explicit retained `abandon` decision for this exact unused reservation, under the same job/writer CAS. Agent verifies no continue decision, apply claim, effect attempt or child publication exists, then settles unused guard/root. A failed COMMIT response, lost prepare/decision POST, API death or unavailable checkpoint is **unknown**, keeps private candidate/target/DM/API-writer fences and is lookup-only. No TTL steal, automatic abort, optimistic release, proof replay or compensation. After API restart, reclaim the job under02 ownership CAS and inspect the original candidate/checkpoint on a transaction that locks the job row (thereby waiting out an earlier in-flight save transaction); exact committed checkpoint determines continue, exact known rolled-back/pre-save state plus no attempted decision/effect permits recording abandon. Absence/unavailable/ambiguous evidence alone never authorizes it. A retained decision marked not_attempted may be delivered once; any attempted decision/effect is only inspected. New desired state is never used to reconstruct the old decision. Expired unconsumed seals cannot be renewed; report saved/pending and retain or settle only from proven containment. -**Settlement/recreation.** `configurationMaintenance.settle` is a verification request, not caller-supplied success authority. Agent reads the exact plan's generation commit, hardlink, recreate and helper receipts, checks all attempts/descendants terminal or contained, immutable root ownership and exact desired runtime configuration/resource/image/mount/readiness evidence (private keyed comparisons for secret mounts). For applied, appliedRevisionId must be this candidate and all required follow-up complete. A known failed follow-up can settle as contained only with no live/unknown effect and truthful saved/committed/failed evidence; it never reports applied. Before release, compute evidenceDigest from the exact retained non-secret child/commit identities, write `settling` with that digest, send the signed finite DM settle leaf carrying that exact owner receipt, and reconcile its DB/mirror settlement. Agent publishes the exact final owner commitment and root-fixed version2 reference control from the approved settlement amendment; DM derives `decisionId`, `expectedSequence`, `outcome` and `ownerReceiptDigest` only from that verified pinned-role commitment, checking original root/guard/decision plus no active authorized link child. Its specialized settlement-only handle, one-use attempt, conclusive SQL publication repair, ACK and disjoint NeverPrepared closure follow that amendment; they cannot be inferred from an unsigned digest or ordinary forward grant. It never tries to read the agent-private directory or trusts an unsigned digest. DM commits settled under the same admission guard and fsyncs its mirror/tombstone; admissions reopen only when both agree and no unresolved owner evidence remains. Agent releases root only after exact DM settled receipt plus every child agree. Response loss between guard settlement and release retains the root until lookup proves settlement; fresh target-bound work still conflicts. Unknown generation/helper/DM settlement retains all fences.08's helper uses this same maintenance identity/guard/frozen subset and terminal-before-drain rule; it creates neither a second reservation nor a second root. +**Settlement/recreation.** `configurationMaintenance.settle` is a verification request, not caller-supplied success authority. Its mandatory outer `evidenceDigest` is the API observation checksum defined by the [approved outer request digest derivation](2026-10-26-admin-settle-request-digest-design.md), using the exact existing portable receipt projection and `openmapx-configuration-settle-request-evidence-v1` domain. Compare it and expectedSequence against the actual original owner under CAS BEFORE first control admission, retain that checked request for duplicate/restart, and never substitute it for agent-produced final-v2 evidence. No receipt/schema extension or private runtime-attestation read is introduced. Agent reads the exact plan's generation commit, hardlink, recreate and helper receipts, checks all attempts/descendants terminal or contained, immutable root ownership and exact desired runtime configuration/resource/image/mount/readiness evidence (private keyed comparisons for secret mounts). For applied, appliedRevisionId must be this candidate and all required follow-up complete. A known failed follow-up can settle as contained only with no live/unknown effect and truthful saved/committed/failed evidence; it never reports applied. Before release, compute evidenceDigest from the exact retained non-secret child/commit identities, write `settling` with that digest, send the signed finite DM settle leaf carrying that exact owner receipt, and reconcile its DB/mirror settlement. Agent publishes the exact final owner commitment and root-fixed version2 reference control from the approved settlement amendment; DM derives `decisionId`, `expectedSequence`, `outcome` and `ownerReceiptDigest` only from that verified pinned-role commitment, checking original root/guard/decision plus no active authorized link child. Its specialized settlement-only handle, one-use attempt, conclusive SQL publication repair, ACK and disjoint NeverPrepared closure follow that amendment; they cannot be inferred from an unsigned digest or ordinary forward grant. It never tries to read the agent-private directory or trusts an unsigned digest. DM commits settled under the same admission guard and fsyncs its mirror/tombstone; admissions reopen only when both agree and no unresolved owner evidence remains. Agent releases root only after exact DM settled receipt plus every child agree. Response loss between guard settlement and release retains the root until lookup proves settlement; fresh target-bound work still conflicts. Unknown generation/helper/DM settlement retains all fences.08's helper uses this same maintenance identity/guard/frozen subset and terminal-before-drain rule; it creates neither a second reservation nor a second root. Required acceptance is future work:01.2 real API/agent/DM preparation/DB transaction/recreation probes;02.3 real DB admission races/startup guards;05.6 direct feed and04.6 mixed-import activation races paused **after final verify and before DB commit**, asserting one exclusive valid commit or zero desired/proof/invalidation writes. Drop prepare/decision/settle POST responses; kill API before and after save; recreate DM and exercise startup cron/CLI/data denial before readiness. Assert one retained root, one decision, no nested acquisition/self-conflict, no proof double consumption, no repeated generation/pin transition, no release while any outcome is unknown, and precise applied versus contained/failed classification.08.3 adds protected helper/DB outage coverage using the same guard; no product execution is claimed here. diff --git a/docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md b/docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md index 97b8e0d8d..a81a2bdd1 100644 --- a/docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md +++ b/docs/superpowers/specs/2026-10-26-admin-fence-settlement-reference-design.md @@ -24,7 +24,7 @@ type DataConfigurationFenceSettle = { IDs remain existing UUID/digest validators; settlementId is64 lowercase hex: SHA256("openmapx-configuration-fence-settlement-v2\0" + installationId + "\0" + ownerPrepare.operationKey + "\0" + maintenanceId + "\0" + candidateId + "\0" + planDigest). -Root admission fixes this whole `{kind:"configuration-maintenance.settle",...body}` operation, original derived phase/ordinal/key, proposed DM execution UUID, recipient and exact sorted subset as today. No late field, null fingerprint or body mutation. Derive the reference independently at issuer and receiver from the original prepared binding; caller reference alone grants nothing. DataConfigurationFencePrepare/Receipt do not gain secret or mutable fields. API outer ConfigurationMaintenanceSettle remains the verification request defined by5.4; its evidenceDigest is never DM mutation authority. +Root admission fixes this whole `{kind:"configuration-maintenance.settle",...body}` operation, original derived phase/ordinal/key, proposed DM execution UUID, recipient and exact sorted subset as today. No late field, null fingerprint or body mutation. Derive the reference independently at issuer and receiver from the original prepared binding; caller reference alone grants nothing. DataConfigurationFencePrepare/Receipt do not gain secret or mutable fields. API outer ConfigurationMaintenanceSettle remains the verification request defined by5.4; its evidenceDigest is never DM mutation authority. The [approved outer digest derivation](2026-10-26-admin-settle-request-digest-design.md) defines that mandatory request field as an exact API-observable receipt checksum with a separate domain/preimage and actual owner/sequence comparison before first control CAS; accepted duplicate/restart uses the retained checked request. It is explicitly distinct from the actual FinalEvidence digest below; private RuntimeAttestation fields stay agent-owned and unexposed. decisionId, expectedSequence, outcome and ownerReceiptDigest become fields of the signed final owner commitment, not caller-selectable DM body fields. Their validation and exact semantics remain required. No preallocated decision ID/schema extension is necessary merely to freeze this request. diff --git a/docs/superpowers/specs/2026-10-26-admin-settle-request-digest-design.md b/docs/superpowers/specs/2026-10-26-admin-settle-request-digest-design.md new file mode 100644 index 000000000..e5e068e75 --- /dev/null +++ b/docs/superpowers/specs/2026-10-26-admin-settle-request-digest-design.md @@ -0,0 +1,65 @@ +# Outer maintenance settle request digest derivation + +Status: APPROVED after independent R1 READY (0 Critical / 0 Important), design readiness only. Implement inside existing [Task01.2](../plans/2026-10-23-admin-01-service-control.md). This clarifies its outer request only; actual source/runtime tests and independent product review remain required. No new task, endpoint, wire field, participant, authority role, root, migration or raised bound. + +## Decision and two distinct digests + +The mandatory `ConfigurationMaintenanceSettle.evidenceDigest` is an **API observation checksum**, named requestEvidenceDigest in implementation-local variables. It is NOT R5's agent-owned `FinalEvidence.evidenceDigest`. Their preimages and domains are distinct and neither may substitute for the other. Final-v2 still includes the actual inspector-generated RuntimeAttestation verificationId/checkedAt and all actual commit/child/runtime evidence; only that actual agent source constructs it, signs OwnerFinal and publishes the root-fixed DM settlement commitment. The API neither invents nor receives private runtime attestations. + +Keep the outer schema unchanged. Supply one portable pure helper from `packages/core/src/ops/configuration-maintenance.ts`, re-exported by ops index: `configurationMaintenanceSettleRequestDigest(receipt: ConfigurationMaintenanceReceipt): string`. Parse the existing strict bounded receipt schema first, construct EXACT projection below, then return lowercase hex SHA-256 of UTF-8 `"openmapx-configuration-settle-request-evidence-v1\0" + canonicalJson(projection)`. canonicalJson is the existing ops canonicalizer. Do not hash the whole receipt, secret state, caller request body or raw JSON property order. This helper calculates a checksum; it returns no nominal verification/authority handle. + +Projection, using the parsed existing receipt `r`: + +```ts +{ + version: 1, + binding: r.binding, + prepare: r.prepare, + apiJobId: r.apiJobId, + parentOperationKey: r.parentOperationKey, + parentFingerprint: r.parentFingerprint, + parentDeadlineAt: r.parentDeadlineAt, + planDigest: r.planDigest, + serviceIds: r.serviceIds, + steps: r.steps.map(s => ({ + phase: s.phase, ordinal: s.ordinal, + operationKey: s.operationKey, operationFingerprint: s.operationFingerprint, + serviceIds: s.serviceIds, prerequisites: s.prerequisites + })), + sequence: r.sequence, + decision: r.decision, + dmFence: r.dmFence, + generation: r.generation, + appliedRevisionId: r.appliedRevisionId, + followUp: r.followUp +} +``` + +`binding`, `prepare`, `decision` and `dmFence` are their EXACT already-defined portable nested objects, including existing nulls; no inferred fields/defaults or secret/config values. Preserve schema-canonical array order; no resorting changes. Operation bodies are excluded; their original frozen fingerprints/planDigest bind them. Only `preflight` (fresh inspect observation) and `state` (derived phase label) are omitted from the receipt fields; neither is settlement authority and neither omission permits actual outcomes to be skipped. Existing64KiB receipt/128KiB owner/32steps bounds remain; checksum projection creates no journal capacity allocation or public digest of secret values. + +Alternative expose an immutable final digest via a new inspection/receipt field requires an unnecessary wire/producer lifecycle extension. Alternative use a random/caller digest or silently ignore the field defeats a mandatory condition. The selected checksum is fully derivable from existing authenticated observation and is checked by the receiver, while final evidence remains private and actual. + +## API producer, actual agent comparison and replay + +API obtains the actual existing `configurationMaintenance.inspect` receipt via the authenticated typed client. Verify its binding/prepare/parent/candidate/plan/decision against the retained original job/candidate/decision as already required. Select the receipt at the intended final verification request, after observing original frozen child attempts; no public/caller receipt is authority. Build the unchanged outer body with its exact identity, `decisionId:r.decision.decisionId`, `expectedSequence:r.sequence`, and the helper digest. Missing decision cannot be filled with a fabricated UUID. Persist the WHOLE exact body/key/fingerprint/admission attempt under existing job CAS before its one POST/HMAC. Keep the observed checksum projection or its digest in the existing bounded private control checkpoint as needed for exact recovery; never store private runtime details. + +On FIRST outer settle control admission, the agent restores the original retained owner and original API parent from its actual fixed stores. Under that owner's existing metadata writer/CAS, BEFORE mutating its settle-control slot, sequence, receipt phase or final inspection, require exact request identity/decision, `expectedSequence === owner.receipt.sequence`, and constant-time equality of request.evidenceDigest with `configurationMaintenanceSettleRequestDigest(owner.receipt)`. This is a read-condition comparison on actual retained state, never verification of a caller-supplied receipt. A mismatch/stale sequence is existing conflict and performs zero final/control/DM publication, guard update or target effect. Existing unknown/fenced owner/parent conditions remain closed. + +Persist the accepted exact outer operation fingerprint/body AND the original checked sequence/request checksum in the already allocated private settle-control slot atomically with its first admitted intent. No later inspection time, changing current pointer, preflight refresh or final-v2 value replaces this accepted request observation. If helper publication, sequence advancement or control state races that first CAS, exactly one owner version wins; the loser performs no mutation/admission. No automatic changed-body retry or rekeying is introduced. Existing explicit reconciliation for a conclusively refused request remains governed by original control/reservation constraints; it cannot acquire a new root or renew a deadline. + +After accepted request retention, perform the existing ACTUAL frozen child/commit/containment/runtime inspection and final-v2 producer. A matching checksum—even for a prepared/unresolved receipt—does NOT establish final eligibility, no effects, success or containment. All existing R5 decision, original root/guard, final evidence, immutable terminal tuple, SQL/mirror, shared proof/ACK and disjoint closure gates remain mandatory. The final owner/cell's evidenceDigest is computed from FinalEvidence under `openmapx-configuration-final-evidence-v2`, never copied from this outer field. The DM reference body remains root-fixed v2; it does not gain/request/use the API checksum. + +A duplicate operation/key/changed body follows existing immutable operation conflict. A duplicate/restart of the ACCEPTED same body observes the retained original checked request/slot and repairs only allowed already-committed publication; do NOT compare its old expectedSequence/checksum against a later owner receipt and reject the original winner, do not reinspect to manufacture a new runtime verificationId/time and do not replay DM mutation. Any continuation after an accepted request uses only original retained control/recovery capabilities and lifetimes. Changed owner sequence is not permission to issue a fresh outer intent, overwrite the accepted slot, reseal a candidate or forge final evidence. Missing/uncertain accepted checkpoint stays unknown; no structural boolean bypass. + +## Required causal acceptance in Task01.2 + +1. Exact portable helper vectors: API-observed receipt and actual retained owner derive identical checksum; reordered JSON object keys do not change it. Mutate each included identity/plan/step/sequence/decision/guard/generation/followUp field: checksum changes. Change preflight/state alone: checksum unchanged. Malformed/oversize receipt rejects; no operation body/secret values enter the preimage. +2. Actual API -> production agent with owned PostgreSQL and real final inspector: legal observed request passes initial compare; bad checksum, foreign candidate/decision and stale sequence reject BEFORE settle-control CAS/effects. A correct checksum on prepared/live/unknown child evidence still refuses final settlement and retains original guard/root. A caller-supplied final-v2 digest cannot stand in for the request checksum. +3. Competing first settle requests/read-owner sequence advance under the actual owner writer: one exact original intent wins, no partial slot/changed-body acceptance. API checkpoints before POST; loss/restart with later owner phase/sequence observes exact accepted body/checksum and original final evidence, no second DM mutation/runtime attestation or new root/time. +4. Existing final-v2 positive/failed containment and SQL/mirror/ACK/crash/expiry proofs remain mandatory. Demonstrate non-equal request and final digests computed by their actual distinct producers; root-fixed DM control verifies only actual agent final commitment. No generator needed for this pure helper unless actual existing route/schema definitions otherwise change at a separately coherent checkpoint. + +Extend `packages/core/src/ops/configuration-maintenance.test.ts` and existing actual01.2 owner/control/API/PG fixtures; use current package scripts and named external-I/O substitutions only. Full task independent product review remains required. No changes to generic late-child binding, current applied0042 or unknown-result guarantees. + +## Self-review + +All preimage fields already exist in closed portable receipt; private runtime observations remain unexposed. Expected sequence/read checksum is enforced before first control mutation; duplicate winner/restart is explicitly separate from first admission. A valid checksum is never a producer proof, and final-v2/DM control/NoGuard authority is unchanged. Cost if wrong: confined comparison/projection/API checkpoint tests, no authority expansion or unbounded new storage. From d9fa028d834905bf2906511065a29064f19ec875 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 27 Oct 2026 08:09:42 +0100 Subject: [PATCH 029/135] docs(admin): define guarded coordinator apply and update workflows --- .../2026-10-23-admin-01-service-control.md | 4 +- ...026-10-23-admin-09-maintenance-controls.md | 2 + .../2026-10-23-admin-instance-management.md | 4 ++ .../2026-10-23-admin-control-protocols.md | 4 +- ...in-maintenance-coordinator-order-design.md | 4 ++ ...0-27-admin-coordinator-lifecycle-design.md | 66 +++++++++++++++++++ 6 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index c73ac8d49..ce10ea067 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -46,7 +46,7 @@ ## Cross-plan interfaces -- `ServiceAction = "start" | "stop" | "restart" | "update" | "build" | "logs" | "pull" | "apply"`; `ServiceActionDecision = { available: boolean; execution: "direct" | "workflow" | "host"; reason: "disabled" | "not_owned" | "not_running" | "control_plane" | "host_only" | "unavailable" | "not_buildable" | null; workflow: "core-restart" | "system-update" | null }`. A denied decision has a non-null reason; a direct available decision has null reason/workflow. An available workflow decision has null reason and a non-null workflow; `execution: "host"` is unavailable with `reason: "host_only"`. API/edge Restart selects `core-restart`; available application Update selects `system-update` only when existing checked-release eligibility permits it. Protected dependency lifecycle stays host-only. Never advertise a workflow solely because its route exists. +- `ServiceAction = "start" | "stop" | "restart" | "update" | "build" | "logs" | "pull" | "apply"`; `ServiceActionDecision = { available: boolean; execution: "direct" | "workflow" | "host"; reason: "disabled" | "not_owned" | "not_running" | "control_plane" | "host_only" | "unavailable" | "not_buildable" | null; workflow: "core-restart" | "system-update" | "coordinator-apply" | "coordinator-update" | null }`. A denied decision has a non-null reason; a direct available decision has null reason/workflow. An available workflow decision has null reason and a non-null workflow; `execution: "host"` is unavailable with `reason: "host_only"`. API/edge Restart selects `core-restart`; available application Update selects `system-update` only when existing checked-release eligibility permits it. Protected dependency lifecycle stays host-only. Never advertise a workflow solely because its route exists. - New typed `services.inspect` is API-role synchronous, 15 seconds, maximum 256 services: `{ services: Array<{ serviceId: string; state: existingStackState; actions: Record }> }`. It returns only trusted registered services; acquisition failure throws. API adds `"unknown"` to its service status enum and returns `acquisition: { state: "available" | "unavailable"; reason: "ops_unavailable" | null }` plus `summary.unknown`. - Move/re-export `HardlinkApplySummary` from `apps/api/src/services/admin-ops.ts` to controller-owned `packages/core/src/ops/service-results.ts` so the CLI and API import the same portable type; API preserves its current re-export for callers. - `prepareServiceGeneration(ctx: JobContext, scope: string, options: { coordinatorMode: CoordinatorMode }): Promise` uses the exact strict candidate and publication/follow-up split in protocol§5–6. Portable `CoordinatorMode = "ensure-running" | "require-running"`; `ServicePreparationResult = { revisionId: string; hardlinks: HardlinkApplySummary; coordinatorMode: CoordinatorMode }`. Start/Update selects ensure-running;09.6 Compose Render selects require-running. Mode, original revisions, sealed candidate identity and both preparation child identities are retained under `servicePreparation[scope]` before effects. Replays observe those identities; they never reconstruct a candidate or silently overwrite newer configuration. @@ -85,6 +85,8 @@ Task2's separate-process probes race CLI Start, single/bulk API Start, Update/bu ### Task 2: Wire update and share generation preparation — S01, S02 +The [approved DM lifecycle disposition](../specs/2026-10-27-admin-coordinator-lifecycle-design.md) is mandatory current-producer scope: implement its exact nominal API coordinator Apply/Update adapter, empty-overlay guarded continuation, conditioned service.pull/submission/actual source witness, helper-before-pull order, DM seed/default-CLI exception, policy bypass closure, mixed bulk revision chain and specified native/failure/image/TTL acceptance. No fake vault change or frozen graph expansion. New workflow enum/pull contracts require a NEW coherent generation checkpoint after actual routes/consumers work; prior freshness is not qualification of these future edits. Ordinary non-DM default CLI remains covered. + The [approved outer settle request digest](../specs/2026-10-26-admin-settle-request-digest-design.md) is part of this task: implement the exact portable observation-checksum helper, API original-attempt checkpoint, actual owner/sequence comparison before first settle control CAS, and duplicate/restart retention. Keep actual final-v2 runtime evidence distinct and agent-produced; execute the specified mismatch/stale/unknown/race/recovery/positive causal tests without adding a receipt field or endpoint. The [approved maintenance coordinator ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) is part of this task: maintenance-bound nonempty ensure-running performs the existing ordinary helper Start only after whole-candidate preflight and BEFORE maintenance root/guard acquisition, conditioned on the observed old/current revision. Original helper settlement/release/readiness and source/TTL rechecks precede unchanged ownership-only prepare; no new frozen Start edge or post-acquisition fallback. Retain its exact loss/restart/cancel/initially-stopped/require-running causal acceptance. diff --git a/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md b/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md index d923cf45f..23a6c68f8 100644 --- a/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md +++ b/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md @@ -72,6 +72,8 @@ lines and256KiB, including a bounded incomplete line and truncation indicator. ## Task M3: Pull-only staging and release pin clearing +Consume the [approved conditioned pull contract](../specs/2026-10-27-admin-coordinator-lifecycle-design.md) installed by01.2: fresh service.pull requires exact expectedConfigurationRevisionId, optional candidateId has only its fixed verified purpose, and successful image/source metadata comes from the actual producer. Preserve original normal submission/admission/lookup and legacy observation rules; do not reintroduce service-ID-only fresh pulls, caller image/path/source assertions or cached-image continuation after failure. This task still owns pull-only UI/staging and release pin clearing; no task/migration reallocation. + **Files:** Create focused `apps/api/src/routes/admin-release-controls.ts`, `apps/api/src/services/admin-release-controls.ts` and tests. Modify service-admin contracts, core ops contract/policy, `apps/ops-agent/src/administrative-runtime.ts`, diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index c33abc5c5..8969e14dd 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -15,6 +15,10 @@ and its binding supplement `docs/superpowers/specs/2026-10-23-admin-control-prot All Global Constraints and Product and security decisions in the specification are binding. User authorization includes autonomous decisions and execution through one PR with green CI; no additional approval or planning pause is required. Delegated work uses only Codex6.1-Sol or ClaudeOpus, preferablyCodex; Astra is prohibited by the latest user instruction. Local rendered verification, no screenshots attached to PR. +## Approved Task01.2 coordinator Apply/Update workflows + +The [approved coordinator lifecycle disposition](../specs/2026-10-27-admin-coordinator-lifecycle-design.md), full R1 plus scoped R2 READY, closes the existing non-credential DM guard gap using fixed API-owned guarded render/link/recreate and a conditioned original pull prerequisite. Exact submission is frozen before sealing; actual normal ExecutionRef/result resolves only into its owner witness. Readiness precedes pull, no unguarded DM seed/default CLI, no new frozen graph/authority. Existing50-task scope/order and actual full runtime/source/CI qualification remain required. + ## Approved Task01.2 outer settle request digest The [approved digest derivation](../specs/2026-10-26-admin-settle-request-digest-design.md), independently READY at R1, separates the existing outer request observation checksum from the agent-produced final-v2 evidence digest. Its exact portable receipt projection, first-admission owner/sequence CAS and retained winner/restart semantics add no schema, endpoint or authority. Existing Task01.2 product/runtime acceptance and50-task order remain unchanged. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 0a5c07fcf..6157da3bc 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -289,7 +289,7 @@ If legacy unchanged state is already over a budget, allow an explicit size-reduc ### 5.4 Remote configuration/feed maintenance (R2-02) -**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. +**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. The [approved coordinator Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) closes the current non-credential DM follow-up gap: fixed API-owned render/empty-overlay intents, mandatory guard, conditioned normal pull submission/source witness before Update maintenance, and original frozen recreate. Its explicit seed/default-CLI/readiness exceptions supersede only those earlier generic clauses; the maintenance graph and DMv2/control/SQL/ACK authority stay unchanged. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. **Preflight before ownership admission.** Under the API global→sorted integration writer, first perform non-consuming `check/checkMany`, strict full prospective schema/source/size validation, exact seal, queue-capacity reservation and durable private candidate receipt. No remote ownership admission is allowed if any of these fails. Read-only host source/hardlink/health inspection is permitted during preflight. The inert candidate may be staged privately for those reads; it is not a consumable apply claim. Derive all eventual targets/effects before prepare: changed service closures, full prospective hardlink producer/consumer closure, DM if needed, MOTIS/motis-staging/motis-feed-proxy closure for feed credentials, and the protected helper subset. Include dependencies required for recreation, never just caller IDs. Candidate retains the exact safe follow-up intent, current image/container expectations and plan preview. Agent independently derives and freezes the closure from trusted manifests, candidate and source/preview witnesses. Reject unavailable prerequisites, >256 sorted targets, >32 effect children or changed plan before prepared; no partial plan or late target expansion. `require-running` never starts DM; its feed flow may explicitly recreate an already-running DM for the new mount, whereas `render-only` permits no lifecycle at all. These distinct actions are frozen before save. Before sealing, allocate candidateId/maintenanceId and the prepare key once using the existing durable API key helper with namespace `configuration-maintenance.prepare` and durable identity `apiJobId + ":" + candidateId`. Derive applyOperationKey from that prepare key with§3.5 phase `configuration-commit`, ordinal0, so the sealed apply identity already equals the frozen child key. Preallocate all safe effect inputs; no post-prepare reseal is required. @@ -479,7 +479,7 @@ runServiceUpdate(request: { }, effects: ServiceLifecycleEffects): Promise<{changed:true;revisionId:string}>; ``` -Optional expected revision is **host-local only**: absence captures it once under writer ownership before effects. API agent calls require the explicit cfg1 revision. `ServiceLifecycleEffects` passes nominal `VerifiedLifecycleOwnership` containing the existing `ConfigurationGenerationWriter` and `TargetLeaseAuthority` to its fixed prepare adapter, as fully typed below. Inner `renderComposeWithWriter` never reacquires its own writer. Update always uses ensure-running and pins→successful pull→full preparation/hardlinks→recreate, with seed configuration publication separately disclosed before the CLI phase. +Optional expected revision is **host-local only**: absence captures it once under writer ownership before effects. API agent calls require the explicit cfg1 revision. `ServiceLifecycleEffects` passes nominal `VerifiedLifecycleOwnership` containing the existing `ConfigurationGenerationWriter` and `TargetLeaseAuthority` to its fixed prepare adapter, as fully typed below. Inner `renderComposeWithWriter` never reacquires its own writer. Update always uses ensure-running and pins→successful pull→full preparation/hardlinks→recreate, with seed configuration publication separately disclosed before the CLI phase. The approved coordinator disposition is the explicit API data-manager exception: no unguarded standalone seed or generic fixed-CLI Update; original full candidate/preflight and any old non-recreate readiness helper precede a conditioned original pull, then guarded generation/links/recreate. The immutable candidate fixes a submission reference, never a predicted normal ExecutionRef/effective clock. An already-ready DM is observed without a preliminary up; explicit coordinator guard readiness may be required even when links are empty. Generic non-DM/default host CLI contract remains as stated. The exact effect signature is `ServiceLifecycleEffects = {withOwnership(request:LifecycleOwnershipRequest,run:(ownership:VerifiedLifecycleOwnership)=>Promise):Promise;validatePins(ids:readonly string[]):Promise;pull(ids:readonly string[],signal?:AbortSignal):Promise;prepare(options:{ownership:VerifiedLifecycleOwnership;coordinatorMode:CoordinatorMode;signal?:AbortSignal}):Promise;recreate(ids:readonly string[],noDeps:boolean,signal?:AbortSignal):Promise}`. `LifecycleOwnershipRequest = {execution:ExecutionRef;serviceIds:readonly string[];expectedConfigurationRevisionId?:string;inheritedLease?:BorrowedArtifactTargetLease;signal?:AbortSignal}`. `VerifiedLifecycleOwnership` is nominal and contains readonly `{writer:ConfigurationGenerationWriter,lease:TargetLeaseAuthority}` plus private receipt ownership; produced only inside withOwnership and invalid after its callback. Its writer commits return the exact revision used to checkpoint this operation's receipt, not an independently read pointer. diff --git a/docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md b/docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md index e9a028e23..5e0be9273 100644 --- a/docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md +++ b/docs/superpowers/specs/2026-10-26-admin-maintenance-coordinator-order-design.md @@ -2,6 +2,10 @@ Status: APPROVED after the independent R1 written report found READY (0 Critical / 0 Important). The provider failed final delivery after writing its complete report; this accepts the report artifact as design readiness only. Allocation remains Task01.2; no new program task, graph edge, principal, typed operation, migration or capacity increase. Implement inside existing [Task01.2](../plans/2026-10-23-admin-01-service-control.md); product/runtime acceptance remains required. +## Approved coordinator lifecycle specialization + +The [approved DM Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) specializes the prerequisite sequence for its fixed API lifecycle intents: observe an already-ready coordinator without an unnecessary Start; if needed, the old-revision non-recreate helper settles BEFORE the Update pull and before maintenance ownership. Its logical guard may require readiness even for empty hardlink plans; empty-plan render-only/require-running still has zero lifecycle. Pull uses an immutable pre-sealed submission, actual normal admission/source resolution, original clocks and no reseal. This overrides conflicting generic ordering only for that closed specialization; no Start/pull is added to the frozen graph. + ## Conflict and selected disposition Protocol5.4 freezes generation -> hardlinks -> recreate/handoff and holds the full target closure from ownership-only preparation through settlement. Protocol6 requires ensure-running's nonempty hardlink plan to obtain readiness via a separate revision-checked settled `service.start` before the link owner's configuration writer. Starting that helper as an independent root after maintenance preparation conflicts with the retained target owner; running it during ownership-only prepare violates the no-effects contract. Adding a service.start effect to maintenance also fails to solve the initially stopped coordinator: its HTTP guard cannot prepare until it is reachable. diff --git a/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md new file mode 100644 index 000000000..37d808f57 --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md @@ -0,0 +1,66 @@ +# Guarded data-manager Apply and Update + +Status: APPROVED Revision2 after full R1 plus scoped R2 READY (0 Critical / 0 Important). Implement inside existing [Task01.2](../plans/2026-10-23-admin-01-service-control.md); actual product/runtime and independent source qualification remain required. Existing Task01.2 scope correction; no new task/migration/principal, maintenance effect kind, child phase, descendant edge or generic authority mechanism. + +## Decision and alternatives + +Every API-managed configuration follow-up that can recreate data-manager requires5.4. Current serviceApply publishes ordinary generation/links then independent recreate; current Update seeds then runs the ordinary fixed CLI. Neither supplies this guard for DM. The current credential-only candidate continuation is insufficient. Do NOT fake a vault change, execute Update inside a held maintenance root, add pull/Update to the frozen graph, or advertise DM Update as an ordinary direct operation. + +Install two fixed API-owned coordinator workflows for serviceId=data-manager: Apply and Update. Both reuse the ALREADY IMPLEMENTED maintenance graph stack.render -> optional hardlinks -> exact service.recreate/recreateIsolated, actual decision/DM guard/final/SQL/ACK/root-release path. Update performs a separately retained existing service.pull prerequisite BEFORE maintenance root acquisition. Its successful order remains pin validation -> pull -> final generation/full hardlinks -> recreate. DM is the explicit API exception to generic fixed-CLI Update/standalone seed publication; ordinary non-DM Update still uses the actual existing default fixed CLI and its lifecycle receipt. Host-local CLI behavior is outside this API dispatch correction and remains its existing contract. + +Alternative put the fixed CLI under the held maintenance root conflicts with its generation/ownership graph; adding a pull edge changes ownership-only preparation. Simply disabling DM Update permanently misses authorized parity. The selected typed pull prelude and existing guarded continuation introduce no new delegation edge. + +## Closed producer and candidate + +Add the fixed API producer module apps/api/src/services/coordinator-lifecycle.ts. It alone constructs a nominal CoordinatorLifecycleIntent under the live AdministrativeConfigurationWriter/JobContext. Constructor validates actual retained job type/payload/actor/key and the exact data-manager target (including its original ordinal in a bulk request); it cannot accept an arbitrary target, callback, participant, result or authorization flag. Intent variants are apply and update(noDeps:boolean, originalPullReference:the closed submission defined below). The private MAC-bound candidate records apiFollowUp coordinator-apply|coordinator-update, exact fixed target/mode/noDeps and original pull reference when applicable. This is NOT a new ConfigurationCommitParticipant kind or public/browser purpose. Existing credential service-apply remains vault-only and does not authorize this new variant. + +Use purpose render, changes:[], ensure-running and typed stack.render. Strictly acquire/validate/seal the WHOLE unchanged desired state; never inject a dummy secret/config row. affected targets include DM from the fixed intent, not fabricated overlay changes. maintenanceId is MANDATORY even for an empty hardlink plan because the explicit continuation recreates DM. Its one desired-write transaction may write only the existing job/continue checkpoint and exact unchanged row-version receipts; no desired row/encryption/resave is invented. The actual agent derives the same finite intent from the original private candidate/API parent and freezes its exact recreate/noDeps operation, target closure, image expectations and all future budgets BEFORE ownership-only prepare. Source conditions and actual authorized decision remain mandatory; a string apiFollowUp is not proof by itself. + +DM Apply is this retained candidate through full guard/decision/generation/links/recreate/settlement. It returns its OWN exact applied revision/result. Callers MUST NOT subsequently call ordinary applyGenerationHardlinks/service.recreate or publish another generation. Unsupported protected targets still use08.3's closed boundary, never this adapter. + +## Typed image pull condition and original source + +Extend the EXISTING service.pull operation for fresh API mutations with mandatory expectedConfigurationRevisionId:cfg1 and optional candidateId:UUID. Coordinator Update MUST supply its exact prepared candidateId; ordinary pull staging may omit it and then uses only the actual immutable current generation image source. Keep its existing API principal, original15-minute budget, dated operation key, exact parent, normal control admission/family and no descendant edges. It already uses normal control authority; it does NOT acquire a data-target maintenance lease or issue a borrower. No new pull kind is introduced. + +Return bounded actual ServicePullResult={changed:true,revisionId:cfg1,imageReference:string,imageId:string,releaseSelectionDigest:string|null}. imageReference uses the existing trusted image-reference validator/bound (maximum2048 UTF-8 bytes); imageId is the actual Docker sha256 image identity; releaseSelectionDigest is the domain-separated hash of NONSECRET selected release identity/pins, null only for a legitimately non-release-managed image. Managed missing/invalid pins refuse BEFORE pull. Neither credentials nor raw Docker output/paths are returned. + +The actual production pull takes the existing configuration writer -> release-store lock (after its normal control admission), compares the exact current revision and runs the SAME release-pin eligibility rules as CLI Update using a read-only fixed validator. For coordinator Update it non-consumingly reads/verifies the exact private MAC candidate, original parent/job and frozen pull reference, readonly-renders its prospective image reference WITHOUT generation/queue-claim/publication, and pulls ONLY that derived reference via the contained Docker image-pull adapter. No caller image/path and no mutable release-override rewrite are accepted. Ordinary staging without candidateId reads the immutable current-generation image and requires matching pins. Factor existing validation as needed; do not call an ensure helper that writes a current-generation/release override as a shortcut. It awaits the real contained process and inspects the exact local image identity under that same lock. The existing private signed host-source witness is extended to capture the finite release selection/image-pin source at source inspection; public configuration.sources.inspect shape stays unchanged. Before pull and maintenance admission, actual readonly source must match the original candidate expectedSourceRevision witness; no later current release is substituted. This capture counts against existing witness128entry/1MiB/5min/pinning bounds. Source/revision/pins drift, failed pull, failed identity read or unknown process is failed/unknown; no cached-image continuation. No config generation, container Start/recreate, API row write or maintenance admission occurs in this handler. Existing resource policy and exclusion against conflicting configuration/release operations remain intact. Historical pull records/results missing the new condition/evidence are STRICTLY observation-only; preserve byte-identical historical observation without admitting a fresh legacy mutation or upgrading it into a coordinator pull proof. + +Actual normal admitted pull execution/body/parent, terminal canonical result, effect settlement and immutable role signatures are the ONLY source for VerifiedCoordinatorPull, constructed by a fixed agent source reader (not an API-supplied result/digest). It verifies exact original API parent/job, data-manager, expected seed revision, successful contained pull and source image/pins. A reference to a public success or changed:true is insufficient. Before prepare, the actual fixed source reader creates one immutable compact CoordinatorPullWitness from those verified normal sources, stored INSIDE the same existing private maintenance owner record (no new file/store/key/collector). Strict witness fields are {version:1,installationId,maintenanceId,candidateId,maintenancePrepare:ExecutionRef,pull:ExecutionRef,submissionDigest,operationFingerprint,apiParentDigest,sourceRevision,revisionId,imageReference,imageId,releaseSelectionDigest,terminalReceiptDigest,effectReceiptDigest,verifiedAt}; digests/UUIDs/epochs use existing validators and the signed encoding is capped at8192bytes. submissionDigest is SHA256 of UTF-8 "openmapx-coordinator-pull-submission-v1\0" + existing canonicalJson(originalPullReference); it binds the exact immutable submission, not a caller assertion. Sign with the existing ops-agent role key under openmapx-coordinator-pull-observation-v1 only inside this actual source reader, never from caller fields/callbacks. It attests actual successful immutable pull observation, not target-effect/final settlement authority. Persist it exactly once with the frozen original owner; duplicate/recovery reuses original bytes/time. Its digests bind genuine normal admission/terminal/effect sources checked before signing; no logs or secret/canonical payload copies inflate it. The owner signature/index and source binding preserve this exact admission-time observation if the old ordinary pull family later legitimately retires; final target image/runtime checks remain actual. Future owner/inspection capacity preflight includes its full worst-case encoding. The actual pull image identity must match the frozen desired DM image and the final actual runtime container; do not substitute whatever current tag resolves to later. + +## Exact Update ordering, clocks and prerequisite safety + +1. Bind the original current seed cfg1 and selected-release source under the same API global writer. Allocate candidateId FIRST, then preallocate/checkpoint originalPullReference as a **submission**, NOT an ExecutionRef: strict `{version:1,requestVersion:2,expectedExecutor:"ops-agent",expectedPrincipal:"api",stepAddress,operation,operationKey,operationFingerprint,requestedDeadlineAt,attempt:ApiJobChildAttempt}`. operation is the exact conditioned service.pull body including that candidateId and seed revision. attempt is the ACTUAL existing prepareApiJobChildAttempt result (version,apiJobId,apiOwnerVersion,parentOperationKey,parentFingerprint,parentDeadlineAt,childOperationKey,childOperationFingerprint,requestedDeadlineAt,attemptId), persisted under original job CAS before sealing; duplicate fields must agree. No executionId, admittedAt, effective deadline or execution fingerprint is predicted. Original managed-step allocation picks requested deadline within original API job and15-minute policy; delayed admission/deployment cap may reduce effective time, never renew it. The immutable MAC candidate includes this exact reference BEFORE helper/pull POST. On reclaim, only the current nominal job owner may deliver the retained originally unattempted submission with its original attempt/parent binding, never regenerate attemptId/owner version/requested time merely because a new worker owns the row. Attempted is lookup-only. + + After normal pull admission, ONLY the fixed actual source reader resolves the unique normal admitted record whose authenticated API parent/attempt, role, operation key/body fingerprint, requested deadline and original candidate reference exactly match the submission. It obtains the agent-assigned ExecutionRef from that actual signed admission/canonical source, including actual effective deadline/fingerprint calculated by normal admission from its ORIGINAL retained admittedAt/deployment budget/parent. No pre-POST execution-ID allocation, assumed equality of requested/effective clocks, changed normal admission or late candidate update is permitted. Missing/foreign/ambiguous/mismatched source stays lookup-only unknown. The resolved ExecutionRef/result appears ONLY in the already specified signed owner CoordinatorPullWitness/frozen plan after successful normal pull; original candidate/reference bytes/MAC remain unchanged. + +2. Prepare/stage the full exact candidate/seal/queue/preview and all future owner/inspection/control budgets BEFORE lifecycle/pull admission, as5.3 requires. This fixes the only final candidate; do not rebuild/reseal after pull. The maintenance root is still only preallocated, not admitted/held. +3. Obtain coordinator readiness FIRST, before this Update pulls a new image into its tag. If actual owned DM is running+ready, observe it and do NOT issue Compose up merely to ensure it: that could recreate it from a newer cached image before the guard. If stopped/missing, use at most the existing original old-revision prerequisite Start from the coordinator-order ruling, settle/release/observe readiness BEFORE pull. Its fixed actual runtime must not recreate an existing DM container outside a guard: a DM Start uses the fixed non-recreate start behavior (Compose --no-recreate or exact owned container start), not force/apply. A running container needing configuration recreation requires guarded Apply, not an implicit Start. No new boolean grants authority; the restriction is derived from the fixed DM target and actual original prerequisite source. For a DM lifecycle intent, readiness is required for the logical guard EVEN when links are empty. Empty-plan RENDER-ONLY/require-running paths retain zero lifecycle. +4. Persist the original pull attempt before its one POST; await exact successful actual source/normal effect containment. Unknown/lost admission is lookup-only. There is no maintenance root while helper/pull is unsettled. Do not Start again AFTER pull: the next operation is ownership-only prepare on the already-ready old coordinator. Source/readiness drift, cancellation, expiry or pull failure prevents prepare/save/generation/recreate. A data job winning the gap makes guard preparation busy, not an excuse to cancel it. +5. Recheck exact seed/source/pins, candidate TTL, original pull and actual coordinator readiness under normal prepare. Freeze expected imageId from the actual referenced pull source before acquiring/preparing guard. Continue the existing one root/DM guard/DB decision/frozen effects/final/ACK/release path. A stopped coordinator after prepare/commit does NOT permit another ordinary Start or root; use existing truthful failed/unknown handling. noDeps false/true chooses the original exact recreate/recreateIsolated semantics. Protected dependency effects cannot be smuggled into these operations; source preflight must prove their unchanged safe runtime prerequisites or refuse before forward effects. + +Candidate TTL remains5minutes. Helper/pull consumes it; an expired final candidate fails before prepare/save, retains truthful successful image staging and uses exact nonissuance cleanup. No automatic reseal, renewed pull/control budget or effect retry is allowed. An explicit NEW user Update may benefit from cached successful downloads but must still execute/verify its own pin-checked pull; this is never fallback after a failed pull. + +Native recovery joins exact original helper/pull attempts and candidate MAC/source/row/release/image proofs before any unattempted next phase. Attempted work is lookup-only. Already accepted maintenance keeps its original root/frozen plan. Missing/ambiguous source stays unknown/fenced; no phase reconstruction from caller action or current pointer. + +## Dispatch, policy and bulk + +API serviceApply(DM), single Update(DM) and bulk Update(DM ordinal) use this fixed workflow. DM-only Update does not publish an ordinary unguarded seed or launch fixedCLI service.update. In mixed bulk, inspect the current working revision first; the existing shared ordinary seed may be published once for the first ordinary non-DM target, preserving original request order. DM consumes the exact preceding working revision and returns its own applied revision; every next target conditions on that returned revision. Unknown stops the batch. No extra DM generation/recreate, reordered targets or hidden second continuation. Successful workflow returns {changed:true,revisionId:itsOwnAppliedRevision}. Report real maintenance application/failure metadata; never manufacture a fixed-CLI lifecycle receipt or claim this job published a seed that was merely observed. Pull failure has no new committed generation; after final generation, preserve the real committed/follow-up evidence. + +Expose workflow metadata coordinator-apply and coordinator-update through the existing ServiceActionDecision workflow enum and current action endpoints, execution=workflow. Advertise only the actual installed fixed API/agent workflow. Generic direct service.update(DM) must refuse before CLI/effects; actual maintenance-bound recreate(DM) requires its original verified frozen child/decision. Generic recreate/stack/start paths MUST NOT implicitly recreate DM without that source: direct DM Start is fixed non-recreate, and any other current action that would recreate DM must enter its already-assigned guarded producer or refuse before effects with truthful prerequisite capability. There are currently no built-in service dependsOn references to DM (controller inventory); still test a changed/foreign dependency so it cannot bypass this rule. No permanent DM Apply/Update refusal is an acceptable completed implementation.09/08 retain their assigned full-stack/protected producer integration; no generic bypass is added here. + +The direct-action policy cannot treat an available workflow as permission for the generic Docker/CLI operation. The actual retained maintenance child verifier remains the sole exception for frozen recreate, never a caller flag/UUID. Source and runtime must both enforce this. Keep ordinary non-DM defaultCLI Update and host-protected boundaries unchanged. + +## Implementation and causal acceptance in01.2 + +- Portable changes: core ops service.pull conditioned input/result; coordinator workflow enum; private prepared-candidate continuation variants; fixed API module and agent pull/source adapters. Reuse existing normal admission/effect/private owner/family/receipt machinery, not a second authority system. Freeze exact SUBMISSION reference and resolve its actual normal-admission ExecutionRef/result only before the original maintenance plan is admitted; do not mutate the candidate with those later facts. Include copied pull proof/future private metadata in real preflight budgets. +- Update actual API single/bulk/Apply and current credential callers only where they share this fixed continuation, original-parent recovery, result/checkpoint suppression and capability paths. Host CLI/default non-DM runner remains actual. No hand-edit generated files; changed canonical MAIN workflow enums require a NEW coherent source/route/consumer checkpoint before controller generation. DM v2 wire/reference,32effects/64intents/128KiB owner/64KiB receipt/768KiB candidate/queue caps unchanged. +- Actual local-admin API -> production agent -> real owned-PG DM: DM Apply and Update each prove full mandatory guard/decision/generation/middle/sole leaf/recreate/R5/SQL/mirror/bothACK/root release, with ONE recreate; Update also proves original pin-checked pull BEFORE maintenance and exact new runtime image. The unguarded original current paths must first fail causally. Empty links STILL guard a DM recreate. +- Already-ready coordinator/no preliminary Start; initially stopped prerequisite Start BEFORE pull; stopped-after-pull/after-commit; expired candidate; failed pins/pull and malicious changed release/image sources: no unsupported generation/recreate, no cache fallback/reseal/second root. Require-running/render-only zero lifecycle remains. Direct service.update/recreate bypass requests and correct-but-foreign pull reference fail before effects; actual legitimate frozen recreate remains positive. +- Actual normal admission assigns the pull executionId and may clamp its deadline under a stricter real deployment policy: assert immutable submission/candidate/MAC, correct unique lookup and actual resolved reference in the signed witness, not a predicted ID/deadline/fingerprint. After native reclaim BEFORE pull, deliver only original retained unattempted body/attempt; foreign parent/attempt, ambiguous source and same-key changed body reject, attempted lookup never issues another POST. +- Native API death before/after helper, during/after pull and before/after candidate/DB/remote commit: exact original normal pull/candidate/child identities, no reauthorization/resave/reseal/effect replay, truthful staged versus saved/applied/unknown. Cancellation after pull, competing release/current config and unknown pull containment block later phases/next writer appropriately. +- Mixed bulk ordinary+DM+ordinary with noDeps both modes proves exact revision chain/order, shared ordinary seed once, no extra DM seed/recreate, partial/unknown outcome. Keep actual ordinary default fixedCLI pins/pull/prepare/recreate/crash/lost-receipt regression, actual callback-free source predicates, full types/canonical/host authority/style/hooks and fresh final-source covering. + +## Self-review + +This closes the actual non-credential DM guard gap without a frozen Update/pull edge. Revision2 explicitly separates pre-sealed submission identity from agent-assigned normal ExecutionRef/effective clock; the owner witness alone receives resolved facts. It explicitly handles the otherwise unsafe helper-after-pull order, old/current versus final image, empty-link guard, fixed original pull result provenance, candidate TTL, ordinary seed exception, direct-operation bypass, historical records and copied-source retention/budgets. Scope remains one Task01.2 and its existing review; real product/runtime proof and independent source review remain required. From e5737e4c993f007728df3e79565205c343ae4f43 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 27 Oct 2026 10:36:05 +0100 Subject: [PATCH 030/135] docs(admin): bind pre-pull coordinator ownership evidence --- .../2026-10-23-admin-01-service-control.md | 2 + .../2026-10-23-admin-instance-management.md | 2 + .../2026-10-23-admin-control-protocols.md | 2 +- ...0-27-admin-coordinator-lifecycle-design.md | 4 +- ...in-coordinator-prepull-ownership-design.md | 80 +++++++++++++++++++ 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index ce10ea067..7b5e5bcb6 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -87,6 +87,8 @@ Task2's separate-process probes race CLI Start, single/bulk API Start, Update/bu The [approved DM lifecycle disposition](../specs/2026-10-27-admin-coordinator-lifecycle-design.md) is mandatory current-producer scope: implement its exact nominal API coordinator Apply/Update adapter, empty-overlay guarded continuation, conditioned service.pull/submission/actual source witness, helper-before-pull order, DM seed/default-CLI exception, policy bypass closure, mixed bulk revision chain and specified native/failure/image/TTL acceptance. No fake vault change or frozen graph expansion. New workflow enum/pull contracts require a NEW coherent generation checkpoint after actual routes/consumers work; prior freshness is not qualification of these future edits. Ordinary non-DM default CLI remains covered. +The [approved pre-pull ownership addendum](../specs/2026-10-27-admin-coordinator-prepull-ownership-design.md), full R1 plus scoped R2/R3 READY, is part of Task01.2: implement its exact literal-resolver opaque capture, raw-claim attachment/consumption BEFORE server clone, dual raw/policy revision join in the first signed normal admission,3072/65536/8192byte caps, version2 copied owner proof, private exact-OLD inspection and normalized tag-alias refusal. Prove actual B-to-E Update through full guard/recreate/settlement, general resolver unchanged, forged/foreign/oversize/alias/recovery/retirement negatives and genuine production-server positive. The sealed submission and original authority graph stay unchanged. + The [approved outer settle request digest](../specs/2026-10-26-admin-settle-request-digest-design.md) is part of this task: implement the exact portable observation-checksum helper, API original-attempt checkpoint, actual owner/sequence comparison before first settle control CAS, and duplicate/restart retention. Keep actual final-v2 runtime evidence distinct and agent-produced; execute the specified mismatch/stale/unknown/race/recovery/positive causal tests without adding a receipt field or endpoint. The [approved maintenance coordinator ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) is part of this task: maintenance-bound nonempty ensure-running performs the existing ordinary helper Start only after whole-candidate preflight and BEFORE maintenance root/guard acquisition, conditioned on the observed old/current revision. Original helper settlement/release/readiness and source/TTL rechecks precede unchanged ownership-only prepare; no new frozen Start edge or post-acquisition fallback. Retain its exact loss/restart/cancel/initially-stopped/require-running causal acceptance. diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index 8969e14dd..1eae1d968 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -19,6 +19,8 @@ All Global Constraints and Product and security decisions in the specification a The [approved coordinator lifecycle disposition](../specs/2026-10-27-admin-coordinator-lifecycle-design.md), full R1 plus scoped R2 READY, closes the existing non-credential DM guard gap using fixed API-owned guarded render/link/recreate and a conditioned original pull prerequisite. Exact submission is frozen before sealing; actual normal ExecutionRef/result resolves only into its owner witness. Readiness precedes pull, no unguarded DM seed/default CLI, no new frozen graph/authority. Existing50-task scope/order and actual full runtime/source/CI qualification remain required. +The [approved pre-pull ownership addendum](../specs/2026-10-27-admin-coordinator-prepull-ownership-design.md), full R1 plus scoped R2/R3 READY, closes the same-tag OLD-runtime source gap. Exact actual resolver capture, first signed admission and version2 copied owner witness preserve ordinary ownership while verifying OLD B and applied NEW E separately. Full raw authority and composite policy revisions remain distinct. It stays inside Task01.2;50-task scope/order and actual source/runtime/CI gates remain unchanged. + ## Approved Task01.2 outer settle request digest The [approved digest derivation](../specs/2026-10-26-admin-settle-request-digest-design.md), independently READY at R1, separates the existing outer request observation checksum from the agent-produced final-v2 evidence digest. Its exact portable receipt projection, first-admission owner/sequence CAS and retained winner/restart semantics add no schema, endpoint or authority. Existing Task01.2 product/runtime acceptance and50-task order remain unchanged. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 6157da3bc..4566bb1fe 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -289,7 +289,7 @@ If legacy unchanged state is already over a budget, allow an explicit size-reduc ### 5.4 Remote configuration/feed maintenance (R2-02) -**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. The [approved coordinator Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) closes the current non-credential DM follow-up gap: fixed API-owned render/empty-overlay intents, mandatory guard, conditioned normal pull submission/source witness before Update maintenance, and original frozen recreate. Its explicit seed/default-CLI/readiness exceptions supersede only those earlier generic clauses; the maintenance graph and DMv2/control/SQL/ACK authority stay unchanged. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. +**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. The [approved coordinator Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) closes the current non-credential DM follow-up gap: fixed API-owned render/empty-overlay intents, mandatory guard, conditioned normal pull submission/source witness before Update maintenance, and original frozen recreate. Its binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md) retains actual OLD container provenance in the first normal pull admission through the literal resolver/claim channel, separates full raw and composite policy revisions, and copies it into the version2 owner witness for exact OLD-only inspection while applied final verifies NEW; ordinary ownership remains unchanged. Its explicit seed/default-CLI/readiness exceptions supersede only those earlier generic clauses; the maintenance graph and DMv2/control/SQL/ACK authority stay unchanged. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. **Preflight before ownership admission.** Under the API global→sorted integration writer, first perform non-consuming `check/checkMany`, strict full prospective schema/source/size validation, exact seal, queue-capacity reservation and durable private candidate receipt. No remote ownership admission is allowed if any of these fails. Read-only host source/hardlink/health inspection is permitted during preflight. The inert candidate may be staged privately for those reads; it is not a consumable apply claim. Derive all eventual targets/effects before prepare: changed service closures, full prospective hardlink producer/consumer closure, DM if needed, MOTIS/motis-staging/motis-feed-proxy closure for feed credentials, and the protected helper subset. Include dependencies required for recreation, never just caller IDs. Candidate retains the exact safe follow-up intent, current image/container expectations and plan preview. Agent independently derives and freezes the closure from trusted manifests, candidate and source/preview witnesses. Reject unavailable prerequisites, >256 sorted targets, >32 effect children or changed plan before prepared; no partial plan or late target expansion. `require-running` never starts DM; its feed flow may explicitly recreate an already-running DM for the new mount, whereas `render-only` permits no lifecycle at all. These distinct actions are frozen before save. Before sealing, allocate candidateId/maintenanceId and the prepare key once using the existing durable API key helper with namespace `configuration-maintenance.prepare` and durable identity `apiJobId + ":" + candidateId`. Derive applyOperationKey from that prepare key with§3.5 phase `configuration-commit`, ordinal0, so the sealed apply identity already equals the frozen child key. Preallocate all safe effect inputs; no post-prepare reseal is required. diff --git a/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md index 37d808f57..2a8df09f8 100644 --- a/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md +++ b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md @@ -1,6 +1,6 @@ # Guarded data-manager Apply and Update -Status: APPROVED Revision2 after full R1 plus scoped R2 READY (0 Critical / 0 Important). Implement inside existing [Task01.2](../plans/2026-10-23-admin-01-service-control.md); actual product/runtime and independent source qualification remain required. Existing Task01.2 scope correction; no new task/migration/principal, maintenance effect kind, child phase, descendant edge or generic authority mechanism. +Status: APPROVED Revision2 after full R1 plus scoped R2 READY (0 Critical / 0 Important), with the binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md). Implement inside existing [Task01.2](../plans/2026-10-23-admin-01-service-control.md); actual product/runtime and independent source qualification remain required. Existing Task01.2 scope correction; no new task/migration/principal, maintenance effect kind, child phase, descendant edge or generic authority mechanism. ## Decision and alternatives @@ -26,7 +26,7 @@ Return bounded actual ServicePullResult={changed:true,revisionId:cfg1,imageRefer The actual production pull takes the existing configuration writer -> release-store lock (after its normal control admission), compares the exact current revision and runs the SAME release-pin eligibility rules as CLI Update using a read-only fixed validator. For coordinator Update it non-consumingly reads/verifies the exact private MAC candidate, original parent/job and frozen pull reference, readonly-renders its prospective image reference WITHOUT generation/queue-claim/publication, and pulls ONLY that derived reference via the contained Docker image-pull adapter. No caller image/path and no mutable release-override rewrite are accepted. Ordinary staging without candidateId reads the immutable current-generation image and requires matching pins. Factor existing validation as needed; do not call an ensure helper that writes a current-generation/release override as a shortcut. It awaits the real contained process and inspects the exact local image identity under that same lock. The existing private signed host-source witness is extended to capture the finite release selection/image-pin source at source inspection; public configuration.sources.inspect shape stays unchanged. Before pull and maintenance admission, actual readonly source must match the original candidate expectedSourceRevision witness; no later current release is substituted. This capture counts against existing witness128entry/1MiB/5min/pinning bounds. Source/revision/pins drift, failed pull, failed identity read or unknown process is failed/unknown; no cached-image continuation. No config generation, container Start/recreate, API row write or maintenance admission occurs in this handler. Existing resource policy and exclusion against conflicting configuration/release operations remain intact. Historical pull records/results missing the new condition/evidence are STRICTLY observation-only; preserve byte-identical historical observation without admitting a fresh legacy mutation or upgrading it into a coordinator pull proof. -Actual normal admitted pull execution/body/parent, terminal canonical result, effect settlement and immutable role signatures are the ONLY source for VerifiedCoordinatorPull, constructed by a fixed agent source reader (not an API-supplied result/digest). It verifies exact original API parent/job, data-manager, expected seed revision, successful contained pull and source image/pins. A reference to a public success or changed:true is insufficient. Before prepare, the actual fixed source reader creates one immutable compact CoordinatorPullWitness from those verified normal sources, stored INSIDE the same existing private maintenance owner record (no new file/store/key/collector). Strict witness fields are {version:1,installationId,maintenanceId,candidateId,maintenancePrepare:ExecutionRef,pull:ExecutionRef,submissionDigest,operationFingerprint,apiParentDigest,sourceRevision,revisionId,imageReference,imageId,releaseSelectionDigest,terminalReceiptDigest,effectReceiptDigest,verifiedAt}; digests/UUIDs/epochs use existing validators and the signed encoding is capped at8192bytes. submissionDigest is SHA256 of UTF-8 "openmapx-coordinator-pull-submission-v1\0" + existing canonicalJson(originalPullReference); it binds the exact immutable submission, not a caller assertion. Sign with the existing ops-agent role key under openmapx-coordinator-pull-observation-v1 only inside this actual source reader, never from caller fields/callbacks. It attests actual successful immutable pull observation, not target-effect/final settlement authority. Persist it exactly once with the frozen original owner; duplicate/recovery reuses original bytes/time. Its digests bind genuine normal admission/terminal/effect sources checked before signing; no logs or secret/canonical payload copies inflate it. The owner signature/index and source binding preserve this exact admission-time observation if the old ordinary pull family later legitimately retires; final target image/runtime checks remain actual. Future owner/inspection capacity preflight includes its full worst-case encoding. The actual pull image identity must match the frozen desired DM image and the final actual runtime container; do not substitute whatever current tag resolves to later. +Actual normal admitted pull execution/body/parent, terminal canonical result, effect settlement and immutable role signatures are the ONLY source for VerifiedCoordinatorPull, constructed by a fixed agent source reader, never public result/digest data. The binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md) supersedes this paragraph's original version1 copied witness: the actual literal resolver captures OLD provenance through a private nominal channel, the fixed server consumes it BEFORE structural cloning and signs it in the SAME first normal admission; its full configuration authority revision and composite policy revision remain distinct. After actual successful containment, version2 CoordinatorPullWitness copies that baseline and exact original admission digest inside the existing owner. Its strict signed8192byte ceiling,3072byte baseline/65536byte admission limits, pre-effect confirmation, actual same-container OLD verification, Docker-normalized sibling-reference refusal, original submission binding and source-only recovery/retirement rules are mandatory. No late candidate mutation, new root/store/key/graph or general resolver relaxation is permitted. Original candidate/source/image/pins/parent/terminal/effect checks and actual final NEW image verification remain required; copied observations are not final authority. ## Exact Update ordering, clocks and prerequisite safety diff --git a/docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md b/docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md new file mode 100644 index 000000000..31647ba31 --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md @@ -0,0 +1,80 @@ +# Coordinator Update: retain the actual pre-pull ownership source + +Status: APPROVED Revision3 after full R1 and scoped R2/R3 READY (0 Critical / 0 Important). Design readiness only; implementation, runtime and independent full source qualification remain required. Addendum to the approved coordinator lifecycle Revision2, inside existing Task01.2. No new root, graph edge, operation, principal, API purpose, key, store, collector or migration. + +## Defect and selected approach + +Task01.1's ordinary owned-services resolver checks the actual container image ID against the current local image ID of its trusted Compose reference. Pulling the same tag changes that reference from B to E while the original running container remains B. R2 correctly requires this OLD container to remain running before guard preparation. It cannot then acquire originalContainers through the unchanged ordinary resolver. Source inspection proves this contradiction independently of the Update observer timeout in293; that timeout alone does not diagnose its runtime cause. + +Preserve the ordinary resolver and every existing provenance predicate. Retain the actual OLD data-manager ownership observation in the EXISTING immutable signed normal service.pull admission, from the actual production registry claim, before the pull. Revalidate it through ordinary ownership immediately before Docker pull under the existing configuration-writer/release locks. Only the exact successful normal pull source can carry that original observation into the existing private maintenance owner witness. A private candidate/owner-scoped inspector can use that authenticated original image ID for the SAME original container while applying every other ordinary provenance predicate. It does not bless arbitrary old images, containers or tags. + +Rejected alternatives: relaxing ordinary image equality changes Task01.1 for unrelated requests; repulling the old tag or using a cached-image fixture changes the real Update; writing OLD runtime facts into the already sealed candidate breaks its immutable submission. A separate ownership service/store is unnecessary because the normal admission and owner already retain authenticated observations. + +## Actual producer and immutable normal admission + +Only fresh service.pull(data-manager,candidateId) from the actual production registry claimer may produce a CoordinatorPullBaseline. IMPORTANT R1 correction: today's immutableClaim and OpsTrustedClaim are STRUCTURAL, not nominal; the JSON ownedService capability cannot authenticate this source. Add the following fixed, ephemeral capture channel in apps/ops-agent/src/owned-services.ts; it confers read-only provenance, not operation permission. + +- That module privately registers a capture closure for each resolver instance created by its literal createOwnedServiceResolver implementation in a WeakMap. There is NO exported resolver registration, arbitrary callback installer or structural fallback. The closure shares the actual bounded ordinary inventory/file/reference inspection and actual loadAuthority, and obtains the single running OLD DM snapshot, trusted Config.Image/reference and FD source digest. A generic injected resolver, including one returning an identical snapshot, is absent from this WeakMap and cannot capture. +- Internal export captureCoordinatorPullOwnership(resolver, operation, operationFingerprint, revisions:{configurationAuthorityRevision,policyRevision}, signal) accepts only the exact fresh candidate-bound DM pull; verifies its literal resolver loader's FULL raw configurationAuthorityRevision equals the expected full raw value, the computed body fingerprint and conditioned current revision, invokes that registered literal closure and returns opaque CoordinatorPullOwnershipCapture. A second private WeakMap brands this empty frozen token and retains its immutable observation {operationFingerprint,candidateId,configurationAuthorityRevision,policyRevision,observedAt,configImageReference,composeSourcesDigest,container}. Parsing/copying/type assertions cannot create the token. No caller supplies the observation. The existing production factory still supplies only named external Docker I/O substitutions in tests. +- The fixed actual registry claimer calls this only on its conditioned candidate-bound DM pull branch, after its ordinary action/resource checks. It constructs its existing immutable structural claim, then calls attachCoordinatorPullOwnershipCapture(rawClaim,capture). This internal function verifies the branded capture, exact claim object, source=registry, complete operation/body/fingerprint/candidateId and exact policyRevision===rawClaim.capability.revisionId before placing rawClaim -> capture in a third private WeakMap. It can bind a capture to only one claim. No JSON capability or new structural OpsTrustedClaim property carries the authority. +- The fixed server calls consumeCoordinatorPullOwnershipCapture(rawClaim,expected) BEFORE its existing immutableSnapshot clone and before the first retainAdmittedControlExecution call. expected names the parsed exact operation fingerprint/candidateId and exact raw claim policyRevision. The function requires raw object identity in that private map, rechecks every captured match and consumes/removes the association once; it returns the frozen literal observation including BOTH revision domains. A copied/spread/deserialized claim, policy-only claimer, substituted structural claim or generic resolver fails authorization at this actual server boundary. Ordinary claims retain their current structural behavior; only this candidate-bound DM pull requires the channel. +- The server holds the consumed observation in its local admission variable across the structural clone. Under the existing original admission lock, before signing, it non-consumingly verifies the actual MAC candidate and its exact original submission/API parent/attempt/expected current+source against this claim/observation. It joins sourceRevision and submissionDigest from that verified original candidate and passes the completed baseline in the SAME first normal signed admission. Low-level retention validates strict body/role/candidate/parent/metadata agreement and byte caps; it is not an exposed structural producer endpoint. An existing by-key winner is returned without capturing/signing/dispatching a replacement; a losing concurrent capture is discarded, never used for a second effect. Subsequent runtime confirmation reads the original signed admission, not the consumed ephemeral token. + +These exact internal interfaces are new implementation duties, not claims that today's code already authenticates a nominal registry observation. No persistent capture registry or new cleanup family is added; WeakMaps disappear on process exit and actual durable recovery uses only the signed original admission. Ordinary staging without candidateId has no baseline and gains no exception. + +### Two revision domains: fixed production join + +R2 correction: the literal owned resolver loads the full TrustedConfigurationAuthoritySnapshot.revisionId (currently authority1_), whereas production policyAuthority returns the distinct composite `${authority.revisionId.slice(0,48)}-${resources.revisionId}`. They MUST NOT compare equal and a truncated prefix is never authentication of the full raw source. + +In apps/ops-agent/src/index.ts policyAuthority's SAME existing load of authority+resources, keep revisionId and its current composite semantics unchanged and add configurationAuthorityRevision:authority.revisionId in the returned read-only snapshot. Extend createProductionRegistryResourceClaimer.loadAuthority's result shape to carry this extra raw field; it can remain optional for ordinary existing callers, but missing means candidate-bound DM pull cannot capture. In that function's same per-claim snapshot, pass revisionId unchanged as the inner registryRevision/policyRevision AND the full configurationAuthorityRevision as a separate field to createRegistryResourceClaimer's fixed capture branch. Do not recompute from a prefix, take a second unrelated raw policy load, accept a caller value or overload capability.values.authorityRevision. Existing trusted-data policy authority binding remains exactly its original composite domain. + +The literal resolver capture compares its own freshly observed full raw loader revision against that supplied full raw snapshot field. The captured policyRevision is bound only to the exact claim.capability.revisionId from the same policy snapshot. The server, before first signing, calls existing verifyConfigurationSourceWitness with candidate.expectedSourceRevision, exact expected current revision and captured configurationAuthorityRevision; it independently checks policyRevision against the actual claim before cloning. Source witness validation authenticates the full raw source, not the composite. Preserve BOTH in the baseline. Pre-effect and maintenance source checks use the full configurationAuthorityRevision with the actual raw loader/source witness, while the original claim/admission checks use policyRevision. Existing policy revalidation/resource semantics stay unchanged. Test a genuine composite DIFFERENT from raw succeeding, raw-only drift with an unchanged/truncation-colliding prefix refusing, policy mismatch refusing, and missing raw field refusing before capture/admission/pull. + +Extend ONLY this normal signed admission with optional coordinatorPullBaseline, present on every newly admitted candidate-bound DM pull. Its strict payload is: + +`{version:1,candidateId,sourceRevision,configurationRevisionId,configurationAuthorityRevision,policyRevision,submissionDigest,observedAt,configImageReference,composeSourcesDigest,container:OwnedServiceSnapshot}`. + +Reuse the existing UUID/cfg1/bounded revision/digest/image-reference/epoch and OwnedServiceSnapshot validators. The formerly ambiguous registryRevision baseline field is REMOVED in this unimplemented addendum, replaced by the two explicit fields above; do not retain an alias that might compare their domains. container.serviceId is exactly data-manager, state is running, containerId is the actual immutable 64hex ID and provenance.imageId is the OLD sha256 ID. Compose working/config paths are actual canonical resolver observations, never request paths. sourceRevision is the candidate's existing signed host-source witness, configurationRevisionId is the exact conditioned current seed, submissionDigest is R2's immutable submission digest. composeSourcesDigest is SHA256("openmapx-coordinator-pull-compose-baseline-v1\\0" + canonicalJson of the ordered actual files' tuples {path,dev,ino,uid,mode,nlink,size,mtimeNs,ctimeNs,sha256Bytes}); integer metadata uses canonical decimal strings where required by existing FD helpers. Only the at-most-two canonical trusted files from that exact resolver are read, each bounded to1MiB, with NOFOLLOW FD ownership/mode/single-link and before/after identity+change checks. No content or tuples are retained or exposed. Recompute this same digest before pull and each OLD verification, so an identical path with changed bytes/inode cannot inherit provenance. The enclosing normal admission supplies actual assigned execution, API parent/attempt, body, origin, installation, admittedAt and requested/effective clocks. The baseline is immutable signed metadata, not a change to the operation body/fingerprint/plan or an additional grant. + +The fixed server branch consumes the newly specified actual capture before the structural clone and validates it against the authenticated private candidate/submission before passing this metadata into the original retainAdmittedControlExecution call. It is signed as part of that SAME first admission. Do not create an admission and mutate it later. Duplicate by-key delivery uses the original signed admission/baseline; a changed duplicate conflicts and no second capture/pull runs. Historical admissions lacking baseline remain readable observations but cannot become a fresh candidate-bound mutation or a coordinator proof. + +Bound canonical UTF-8 encoded baseline to3072bytes and the original signed normal admission to its existing65536byte cell. Existing candidate preflight reserves the full8192byte copied owner-witness allowance. Actual baseline/admission size is checked before admission/effect/pull; oversized provenance refuses before external work, with the ordinary inert candidate cleanup rules. Do not truncate paths or enlarge cells. Production claim source acquisition may perform bounded read-only Docker/filesystem inspection only. + +## Pre-effect confirmation and successful copied proof + +After original admission and under the existing configuration writer -> release lock, the actual pull handler verifies exact candidate MAC/source/current revision/pins/submission/admission and readiness as R2 already requires. It re-runs the UNCHANGED ordinary resolver before image pull and compares the single DM snapshot and trusted image reference byte-for-byte to the retained baseline (observedAt is the original capture time, never refreshed). If ID/image/state/provenance/config/release/registry changes, refuse before Docker pull. The actual registry/Compose source check and readiness remain required; no signed snapshot makes stale reality current. + +Reject before pull when another registered service's trusted effective current OR prospective image reference has the same normalized Docker reference identity as the pulled reference. Exact raw-string comparison is insufficient. Add fixed pure coordinatorPullReferenceIdentity(reference):string for this refusal ONLY: validate the existing bounded schema, parse repository/optional tag/digest, and use the Docker reference normalization rules: absent registry -> docker.io; index.docker.io -> docker.io; a first path component containing '.' or ':' or exactly localhost is a registry, otherwise it is a namespace on docker.io; on docker.io a single repository component gains library/. A reference without digest uses its exact explicit tag, or latest when omitted. A sha256-digested reference uses normalized repository@digest and ignores any accompanying tag, matching ParseDockerRef. Preserve tag case and explicit registry port; different tags currently sharing an image ID do NOT compare equal. Refuse unsupported/ambiguous/malformed spellings before pull, never guess or fall back to raw strings. This does not rewrite the actual pulled reference, relax the existing image schema or change ordinary ownership comparison. + +The bounded inventory derives all registered services' references from the same actual trusted Compose/manifest/release sources used for source validation, never caller aliases. Apply normalization to BOTH sides. Examples that MUST compare equal: example/coordinator:main and docker.io/example/coordinator:main; ubuntu and index.docker.io/library/ubuntu:latest; docker.io/ubuntu and ubuntu:latest. main versus other remains distinct even if both currently resolve to B. Immutable digest identity remains distinct from a mutable tag, and two identical immutable references may conservatively be refused as shared without granting a sibling exception. These rules follow the primary [Docker reference normalization source](https://github.com/distribution/reference/blob/main/normalize.go); future unsupported forms fail closed. Alias-pair tests must demonstrate refusal BEFORE actual pull. This narrowly prevents shrinking the ordinary sibling inventory; no sibling-service image-equality exception is introduced. + +Await the actual contained pull, successful actual NEW image inspection and normal immutable canonical terminal/effect settlement. Failure, interruption, lost/unknown containment or missing OLD baseline never authorizes maintenance. Existing lookup-only/no-fallback rules remain. The success effect/result schema need not expose OLD container/path data publicly. + +Amend the private CoordinatorPullWitness to version2 and domain openmapx-coordinator-pull-observation-v2. Preserve every R2 field, add `baseline:CoordinatorPullBaseline` and `baselineAdmissionDigest` (SHA256 of canonicalJson of the exact signed original normal admission packet, using existing canonical JSON). The fixed source reader verifies the actual normal admission/terminal/effect, the exact baseline/submission/candidate/parent/source relation and copies the baseline; it cannot accept these from API/result fields. Its existing signature attests this verified admission-time observation. It is still not target authority or final evidence. Strict signed encoding remains8192bytes; future budgets reserve this full maximum before helper/pull, and the real producer refuses oversize rather than claiming success. Old version1 witnesses are observation-only, never upgraded by rereading current cache. + +Persist the version2 witness exactly once INSIDE the existing signed/indexed maintenance owner. Original normal pull sources must be present for the initial copy; absence cannot be filled from current Docker state. After that original copy, the authenticated owner/witness retains the exact observation if the ordinary pull family legitimately retires. Recovery reuses original bytes/time/baseline/admission digest. No extra file, record family or retention floor is introduced. + +## Narrow original-container inspection + +Introduce an opaque VerifiedCoordinatorPullBaseline, minted ONLY by a fixed actual source reader after the above normal source verification, or by the fixed reader of the exact already signed original maintenance owner and its version2 witness. Use a private WeakMap/brand with runtime validation, as existing nominal proofs do; a parsed shape/type assertion cannot mint it. It is bound to installation, candidate, exact maintenance prepare execution, original source revision and actual original pull, and grants read-only verification only. It is not a lease, grant, mutation permission or alternate action policy. + +The production owned-services module owns the private original-container inspection implementation. It consumes that nominal proof, never an expected-image argument or public boolean. Share its literal ordinary filesystem/registry/Compose/container verification code so all current Task01.1 checks remain: registered service/project, canonical confined Compose paths and retained legacy alias, exact file ownership/mode/single-link identity, trusted effective Config.Image reference, no duplicate IDs/services, bounded inventory and actual Docker container metadata. ONLY for data-manager's exact retained original containerId, exact baseline paths/project/reference and exact OLD container Image, compare image identity against the authenticated baseline OLD sha256 instead of the now repointed tag. All other rows use ordinary current-cache equality. A foreign/replaced container, changed labels/files/source/reference/image or missing original fails; it is never synthesized into an inventory. + +Before maintenance admission, require actual OLD DM snapshot to equal the retained baseline, still running+ready. Merge it only with genuine ordinary observations for other services and freeze it as owner.originalContainers; do not replace originalContainers with the NEW desired image. Derive frozen final image expectation E from the actual successful pull exactly as R2 requires. + +Use this same fixed baseline verification wherever this original owner must observe its OLD runtime: preparation rechecks/recovery, UNUSED and precise-contained inspection, or before the exact authorized frozen DM recreate. The actual registry claimer may use it for that recreate ONLY after the existing nominal owned-maintenance-step/decision verifier proves this exact original owner/step. It cannot turn generic service.update/recreate/start or an available workflow into permission. After actual recreation, applied inspection uses the UNCHANGED ordinary ownership path and actual NEW E plus full original config/hash/mount/secret/readiness checks. Never let the baseline OLD B satisfy an applied NEW-E final. A missing/replaced OLD source before recreation remains fenced; independently created NEW container is not silently adopted. + +Audit current API source/recovery/readiness and registry callers so none falls back to ordinary post-pull ownership to reconstruct OLD facts or mints a second root/helper. Pure host-source/pin checks remain read-only and must match the original witness; this addendum does not disable them. Public status/navigation ownership remains ordinary, including its temporary unverified result while the tag changed; only this fixed workflow can consume the retained source. + +## Implementation order and causal checks + +1. Keep RED290 and293 original mixed result. Add a portable failing source test for same tag B->E with exact unchanged OLD container, signed baseline absent; test existing ordinary resolver still excludes B after cache E. No same-ID cached-image positive. +2. Implement the fixed literal-resolver opaque capture -> actual raw-claim identity attachment -> server consumption BEFORE structural clone -> initial signed admission baseline, pre-effect ordinary equality and3072/65536byte refusal. Test the ACTUAL server with a structural copy/deserialized claim carrying identical JSON ownedService, generic resolver, policy-only claimer, absent/foreign/reused capture, changed baseline, same-key changed snapshot and oversized actual paths before pull. All deny with zero admission/pull effects; a genuine literal factory capture with distinct full raw and composite policy revisions must pass through this SAME production server path. Missing raw source, full-raw drift even with same first48 characters and policy mismatch refuse. Exact original admission time/body/submission remain unchanged. +3. Implement strict version2 copied witness/nominal reader and private old-container inspector without generic resolver input. Existing8192budget and maximum source sizes get real encoding tests. All Task01.1 ordinary ownership/creation-removal tests stay intact. Test raw-identical AND normalized-alias sibling references refuse before pull, default registry/namespace/implicit tag/legacy index aliases, and distinct mutable tags currently sharing B do not trigger a false equality. Unsupported normalization fails before effects. +4. Actual local-admin API -> production agent/real owned-PG DM: real old B remains running/owned before guard after actual named Docker pull changes cache to distinct E; full R2 Update guard/decision/render/hardlinks/recreate/R5/SQL/ACK/root release, exactly one pull/recreate, immutable candidate. Actual applied runtime image E and OLD originalContainers B must both be asserted. Apply293 remains positive. +5. Changed container ID, labels/path/file bytes/reference, stopped/unready old runtime, foreign candidate/parent/pull, source/pin drift, failed pull, missing/tampered baseline/witness and same-tag substitution refuse with no unsupported generation/recreate. Known UNUSED and precise-contained branches must verify actual OLD B after pull; an unchanged OLD B cannot pass applied-E final. Actual frozen recreate positive and generic direct bypass negatives use the production registry claimer. +6. Native API death after successful pull/before maintenance and during original owner recovery: original admission/baseline/witness/candidate/old ID/image bytes preserved; one POST, no helper after pull/reseal/reacquire. Original owner's source still verifies after legitimate pull-family retirement; corrupt/missing owner proof denies. Future capacity refusal precedes helper/pull. Re-run relevant ordinary ownership, coordinator lifecycle, retirement/recovery/types/authority/style and final actual source checks, then request only coherent canonical generation when needed. + +## Controller self-review + +This is a correction to R2's absent OLD provenance source, not a weaker general ownership policy. Revision2 explicitly acknowledges existing structural claims and installs the missing literal-resolver capture/raw-claim identity/server-before-clone consumption channel. It signs the observation in the existing first normal admission, rechecks it before pull, copies only after true successful containment and verifies actual same-container reality at every use. Docker-normalized identities close qualified/unqualified/default-tag alias shrinkage without comparing unrelated equal current image IDs. It separates immutable candidate submission, OLD runtime provenance and NEW desired image. Exact byte caps, failed copy/recovery, normal family retirement and applied-versus-contained inspection remain explicit. Cost if wrong is confined producer/source/witness and inspector rework; no new forward or recovery authority is permitted. Product/runtime acceptance and independent source review remain required even after design READY. From 1c53034f8aa682a6252e618448a1b4b9ac576e6f Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 27 Oct 2026 13:49:01 +0100 Subject: [PATCH 031/135] docs(admin): preserve ownership for fresh staged retries --- .../2026-10-23-admin-01-service-control.md | 4 +- .../2026-10-23-admin-instance-management.md | 2 + .../2026-10-23-admin-control-protocols.md | 2 +- ...0-27-admin-coordinator-lifecycle-design.md | 2 +- ...in-coordinator-prepull-ownership-design.md | 4 +- ...min-staged-coordinator-ownership-design.md | 75 +++++++++++++++++++ 6 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index 7b5e5bcb6..bdc7ffd16 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -77,7 +77,7 @@ Task2's separate-process probes race CLI Start, single/bulk API Start, Update/bu - [ ] **RED test:** Add `deselected owned service retains stop and logs but cannot start` to `policy.test.ts` and `owned-services.test.ts`. Assert available stop/logs, denied start/recreate/pull/update/build; post-stop logs remain readable while the owned container exists. Include unknown registry ID, same name in another project, forged service label, replaced ID and community removal cases. - [ ] **RED command:** `pnpm exec vitest run --project node apps/ops-agent/src/policy.test.ts apps/ops-agent/src/owned-services.test.ts`. At RED the new ownership test calls existing `createDockerRuntime`/`dispatchOpsOperation`; do not import the uncreated extractor. The existing policy must fail the available-stop/log assertions for disabled/core fixtures, and the existing Compose adapter must fail the immutable-ID argv assertion. After extraction switch the same assertions to the shared resolver. -- [ ] **Implement ownership:** Enumerate all containers for the fixed `services.STACK_PROJECT`, including stopped/orphaned containers; do not depend on current Compose membership. Verify `com.docker.compose.project`, `com.docker.compose.service`, `com.docker.compose.project.working_dir` and `com.docker.compose.project.config_files` against the fixed stack project, trusted registry and canonical infrastructure/current-or-retained generation paths; verify container image against the trusted effective image reference. Paths remain internal. Unverifiable/ambiguous provenance is denied; matching a caller's name or one label is insufficient. Capture immutable container IDs in the claim; re-inspect labels/identity at the effect boundary. Stop/log use fixed `docker stop`/`docker logs` argv on those IDs after verification, never a request-supplied name/path. A host principal with Docker authority is outside this web authorization boundary; do not promise that arbitrary hostile Docker metadata is cryptographic ownership. Oversized/malformed inventory fails acquisition instead of returning a truncated authoritative list. +- [ ] **Implement ownership:** Enumerate all containers for the fixed `services.STACK_PROJECT`, including stopped/orphaned containers; do not depend on current Compose membership. Verify `com.docker.compose.project`, `com.docker.compose.service`, `com.docker.compose.project.working_dir` and `com.docker.compose.project.config_files` against the fixed stack project, trusted registry and canonical infrastructure/current-or-retained generation paths; verify container image against the trusted effective image reference. The binding [staged coordinator ownership source](../specs/2026-10-27-admin-staged-coordinator-ownership-design.md) is the only added DM-specific authenticated alternative after successful staging: actual signed normal provenance plus unchanged current container/config, with fresh action permission separate; no-proof and unrelated services retain ordinary cache-image equality. Paths remain internal. Unverifiable/ambiguous provenance is denied; matching a caller's name or one label is insufficient. Capture immutable container IDs in the claim; re-inspect labels/identity at the effect boundary. Stop/log use fixed `docker stop`/`docker logs` argv on those IDs after verification, never a request-supplied name/path. A host principal with Docker authority is outside this web authorization boundary; do not promise that arbitrary hostile Docker metadata is cryptographic ownership. Oversized/malformed inventory fails acquisition instead of returning a truncated authoritative list. - [ ] **Implement authority:** Keep current configured authority for start/build/recreate/pull/update. On deselection retain only stop/log access for a still-registered, proven owned container; no implicit automatic stop. Protect `ops-agent` from stop/restart/recreate/update, while permitting verified read-only logs. Tasks 4/6 consume the same policy decisions; routes are not the final authorization boundary. Remove `admin-ops.ts:assertKnownService`'s unconditional enabled check from Stop and log routes: retain known-ID checks, then consult the shared agent decision. Selection commits preserve registered disabled manifests and retained generations until owned cleanup; installed community removal must refuse while verified containers still need cleanup, rather than discarding their authority. - [ ] **Implement assembly:** Move existing runtime composition into `production-runtime.ts`, call it from `index.ts`, and expose `services.inspect` there. Preserve trusted-config authority refresh, community recovery and all privacy/transit installers. `policy.ts` consumes the same resolver for admission; revalidation remains mandatory during execution. - [ ] **GREEN command:** Run the RED command plus `pnpm exec vitest run --project node apps/ops-agent/src/production-runtime.test.ts apps/ops-agent/src/trusted-config-ownership.test.ts apps/ops-agent/src/startup-authority.test.ts`. @@ -89,6 +89,8 @@ The [approved DM lifecycle disposition](../specs/2026-10-27-admin-coordinator-li The [approved pre-pull ownership addendum](../specs/2026-10-27-admin-coordinator-prepull-ownership-design.md), full R1 plus scoped R2/R3 READY, is part of Task01.2: implement its exact literal-resolver opaque capture, raw-claim attachment/consumption BEFORE server clone, dual raw/policy revision join in the first signed normal admission,3072/65536/8192byte caps, version2 copied owner proof, private exact-OLD inspection and normalized tag-alias refusal. Prove actual B-to-E Update through full guard/recreate/settlement, general resolver unchanged, forged/foreign/oversize/alias/recovery/retirement negatives and genuine production-server positive. The sealed submission and original authority graph stay unchanged. +The [approved staged ownership addendum](../specs/2026-10-27-admin-staged-coordinator-ownership-design.md), full R1 plus scoped R2 READY, is part of Task01.2. Install the fixed actual historical source/capture for fresh retry/Apply/allowed non-recreate Start; separately bind Apply-owner provenance across only its own generation, freeze actual desired DM image for installed guarded follow-ups and reserve the exact allowance before effects. Implement last-source liveness/safe supersession plus authenticated GC phase context, bounded preflight/manifest/commit revalidation and original committed cursor/journal cleanup. Preserve existing limits/no new store/collector/root, and execute the specified native/expiry/foreign/GC crash/capacity/guarded positive acceptance. + The [approved outer settle request digest](../specs/2026-10-26-admin-settle-request-digest-design.md) is part of this task: implement the exact portable observation-checksum helper, API original-attempt checkpoint, actual owner/sequence comparison before first settle control CAS, and duplicate/restart retention. Keep actual final-v2 runtime evidence distinct and agent-produced; execute the specified mismatch/stale/unknown/race/recovery/positive causal tests without adding a receipt field or endpoint. The [approved maintenance coordinator ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) is part of this task: maintenance-bound nonempty ensure-running performs the existing ordinary helper Start only after whole-candidate preflight and BEFORE maintenance root/guard acquisition, conditioned on the observed old/current revision. Original helper settlement/release/readiness and source/TTL rechecks precede unchanged ownership-only prepare; no new frozen Start edge or post-acquisition fallback. Retain its exact loss/restart/cancel/initially-stopped/require-running causal acceptance. diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index 1eae1d968..88bd10678 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -21,6 +21,8 @@ The [approved coordinator lifecycle disposition](../specs/2026-10-27-admin-coord The [approved pre-pull ownership addendum](../specs/2026-10-27-admin-coordinator-prepull-ownership-design.md), full R1 plus scoped R2/R3 READY, closes the same-tag OLD-runtime source gap. Exact actual resolver capture, first signed admission and version2 copied owner witness preserve ordinary ownership while verifying OLD B and applied NEW E separately. Full raw authority and composite policy revisions remain distinct. It stays inside Task01.2;50-task scope/order and actual source/runtime/CI gates remain unchanged. +The [approved staged ownership addendum](../specs/2026-10-27-admin-staged-coordinator-ownership-design.md), full R1 plus scoped R2 READY, closes fresh retry/Apply after successful image staging and expired candidate cleanup. DM-only authenticated historical provenance is separated from fresh current authority; owner-bound Apply proof and phase-aware last-source retention use existing bounded records and GC. It is an explicit narrow source-boundary amendment inside Task01.2;50-task scope/order and actual full source/runtime/CI qualification remain required. + ## Approved Task01.2 outer settle request digest The [approved digest derivation](../specs/2026-10-26-admin-settle-request-digest-design.md), independently READY at R1, separates the existing outer request observation checksum from the agent-produced final-v2 evidence digest. Its exact portable receipt projection, first-admission owner/sequence CAS and retained winner/restart semantics add no schema, endpoint or authority. Existing Task01.2 product/runtime acceptance and50-task order remain unchanged. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 4566bb1fe..134d86079 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -289,7 +289,7 @@ If legacy unchanged state is already over a budget, allow an explicit size-reduc ### 5.4 Remote configuration/feed maintenance (R2-02) -**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. The [approved coordinator Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) closes the current non-credential DM follow-up gap: fixed API-owned render/empty-overlay intents, mandatory guard, conditioned normal pull submission/source witness before Update maintenance, and original frozen recreate. Its binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md) retains actual OLD container provenance in the first normal pull admission through the literal resolver/claim channel, separates full raw and composite policy revisions, and copies it into the version2 owner witness for exact OLD-only inspection while applied final verifies NEW; ordinary ownership remains unchanged. Its explicit seed/default-CLI/readiness exceptions supersede only those earlier generic clauses; the maintenance graph and DMv2/control/SQL/ACK authority stay unchanged. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. +**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. The [approved coordinator Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) closes the current non-credential DM follow-up gap: fixed API-owned render/empty-overlay intents, mandatory guard, conditioned normal pull submission/source witness before Update maintenance, and original frozen recreate. Its binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md) retains actual OLD container provenance in the first normal pull admission through the literal resolver/claim channel, separates full raw and composite policy revisions, and copies it into the version2 owner witness for exact OLD-only inspection while applied final verifies NEW; ordinary ownership remains unchanged. The [staged ownership addendum](2026-10-27-admin-staged-coordinator-ownership-design.md) adds the closed DM-only authenticated historical source for a fresh request after successful staging/expired MAC, separately bound Apply-owner provenance and existing-family last-source/phase-aware cleanup dependencies; fresh current policy/own pull/root/decision remain mandatory and expired authority cannot forward. Its explicit seed/default-CLI/readiness exceptions supersede only those earlier generic clauses; the maintenance graph and DMv2/control/SQL/ACK authority stay unchanged. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. **Preflight before ownership admission.** Under the API global→sorted integration writer, first perform non-consuming `check/checkMany`, strict full prospective schema/source/size validation, exact seal, queue-capacity reservation and durable private candidate receipt. No remote ownership admission is allowed if any of these fails. Read-only host source/hardlink/health inspection is permitted during preflight. The inert candidate may be staged privately for those reads; it is not a consumable apply claim. Derive all eventual targets/effects before prepare: changed service closures, full prospective hardlink producer/consumer closure, DM if needed, MOTIS/motis-staging/motis-feed-proxy closure for feed credentials, and the protected helper subset. Include dependencies required for recreation, never just caller IDs. Candidate retains the exact safe follow-up intent, current image/container expectations and plan preview. Agent independently derives and freezes the closure from trusted manifests, candidate and source/preview witnesses. Reject unavailable prerequisites, >256 sorted targets, >32 effect children or changed plan before prepared; no partial plan or late target expansion. `require-running` never starts DM; its feed flow may explicitly recreate an already-running DM for the new mount, whereas `render-only` permits no lifecycle at all. These distinct actions are frozen before save. Before sealing, allocate candidateId/maintenanceId and the prepare key once using the existing durable API key helper with namespace `configuration-maintenance.prepare` and durable identity `apiJobId + ":" + candidateId`. Derive applyOperationKey from that prepare key with§3.5 phase `configuration-commit`, ordinal0, so the sealed apply identity already equals the frozen child key. Preallocate all safe effect inputs; no post-prepare reseal is required. diff --git a/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md index 2a8df09f8..bdd734504 100644 --- a/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md +++ b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md @@ -39,7 +39,7 @@ Actual normal admitted pull execution/body/parent, terminal canonical result, ef 4. Persist the original pull attempt before its one POST; await exact successful actual source/normal effect containment. Unknown/lost admission is lookup-only. There is no maintenance root while helper/pull is unsettled. Do not Start again AFTER pull: the next operation is ownership-only prepare on the already-ready old coordinator. Source/readiness drift, cancellation, expiry or pull failure prevents prepare/save/generation/recreate. A data job winning the gap makes guard preparation busy, not an excuse to cancel it. 5. Recheck exact seed/source/pins, candidate TTL, original pull and actual coordinator readiness under normal prepare. Freeze expected imageId from the actual referenced pull source before acquiring/preparing guard. Continue the existing one root/DM guard/DB decision/frozen effects/final/ACK/release path. A stopped coordinator after prepare/commit does NOT permit another ordinary Start or root; use existing truthful failed/unknown handling. noDeps false/true chooses the original exact recreate/recreateIsolated semantics. Protected dependency effects cannot be smuggled into these operations; source preflight must prove their unchanged safe runtime prerequisites or refuse before forward effects. -Candidate TTL remains5minutes. Helper/pull consumes it; an expired final candidate fails before prepare/save, retains truthful successful image staging and uses exact nonissuance cleanup. No automatic reseal, renewed pull/control budget or effect retry is allowed. An explicit NEW user Update may benefit from cached successful downloads but must still execute/verify its own pin-checked pull; this is never fallback after a failed pull. +Candidate TTL remains5minutes. Helper/pull consumes it; an expired final candidate fails before prepare/save, retains truthful successful image staging and uses exact nonissuance cleanup. No automatic reseal, renewed pull/control budget or effect retry is allowed. An explicit NEW user Update may benefit from cached successful downloads but must still execute/verify its own pin-checked pull; this is never fallback after a failed pull. The binding [staged ownership addendum](2026-10-27-admin-staged-coordinator-ownership-design.md) makes that fresh-request promise implementable after the old MAC expires: authenticate the same OLD DM from successful retained normal source observations, keep fresh authority separate, carry Apply provenance through only its own generation and preserve the last usable source through phase-aware existing GC. No expired MAC/grant/body/clock is renewed. Native recovery joins exact original helper/pull attempts and candidate MAC/source/row/release/image proofs before any unattempted next phase. Attempted work is lookup-only. Already accepted maintenance keeps its original root/frozen plan. Missing/ambiguous source stays unknown/fenced; no phase reconstruction from caller action or current pointer. diff --git a/docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md b/docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md index 31647ba31..fe5f7a6b0 100644 --- a/docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md +++ b/docs/superpowers/specs/2026-10-27-admin-coordinator-prepull-ownership-design.md @@ -2,6 +2,8 @@ Status: APPROVED Revision3 after full R1 and scoped R2/R3 READY (0 Critical / 0 Important). Design readiness only; implementation, runtime and independent full source qualification remain required. Addendum to the approved coordinator lifecycle Revision2, inside existing Task01.2. No new root, graph edge, operation, principal, API purpose, key, store, collector or migration. +The binding [staged ownership addendum](2026-10-27-admin-staged-coordinator-ownership-design.md) explicitly amends this revision's original-candidate-only and ordinary-current-cache-only source boundary for fresh requests after successful staging. Its DM-only authenticated historical source, fresh current-policy/candidate authority, own-generation Apply proof and last-source/phase-aware GC contract supersede those clauses; all other caps/source/graph boundaries remain. + ## Defect and selected approach Task01.1's ordinary owned-services resolver checks the actual container image ID against the current local image ID of its trusted Compose reference. Pulling the same tag changes that reference from B to E while the original running container remains B. R2 correctly requires this OLD container to remain running before guard preparation. It cannot then acquire originalContainers through the unchanged ordinary resolver. Source inspection proves this contradiction independently of the Update observer timeout in293; that timeout alone does not diagnose its runtime cause. @@ -64,7 +66,7 @@ Before maintenance admission, require actual OLD DM snapshot to equal the retain Use this same fixed baseline verification wherever this original owner must observe its OLD runtime: preparation rechecks/recovery, UNUSED and precise-contained inspection, or before the exact authorized frozen DM recreate. The actual registry claimer may use it for that recreate ONLY after the existing nominal owned-maintenance-step/decision verifier proves this exact original owner/step. It cannot turn generic service.update/recreate/start or an available workflow into permission. After actual recreation, applied inspection uses the UNCHANGED ordinary ownership path and actual NEW E plus full original config/hash/mount/secret/readiness checks. Never let the baseline OLD B satisfy an applied NEW-E final. A missing/replaced OLD source before recreation remains fenced; independently created NEW container is not silently adopted. -Audit current API source/recovery/readiness and registry callers so none falls back to ordinary post-pull ownership to reconstruct OLD facts or mints a second root/helper. Pure host-source/pin checks remain read-only and must match the original witness; this addendum does not disable them. Public status/navigation ownership remains ordinary, including its temporary unverified result while the tag changed; only this fixed workflow can consume the retained source. +Audit current API source/recovery/readiness and registry callers so none falls back to ordinary post-pull ownership to reconstruct OLD facts or mints a second root/helper. Pure host-source/pin checks remain read-only and must match the original witness; this addendum does not disable them. Without the installed authenticated staged source, ordinary ownership still denies an unproved OLD image after tag drift. With the binding staged ownership addendum, the literal resolver may verify the same DM identity from actual successful signed observations for fresh current-policy requests; this is read-only provenance, never expired forward authority. ## Implementation order and causal checks diff --git a/docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md b/docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md new file mode 100644 index 000000000..ff9b52848 --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md @@ -0,0 +1,75 @@ +# Fresh requests after successful image staging + +Status: APPROVED Revision2 after full R1 plus scoped R2 READY (0 Critical / 0 Important), both findings addressed. Design readiness only; actual implementation, source, runtime and CI qualification remain required. Finite Task01.2 correction to lifecycle Revision2 TTL/new-request rule and pre-pull ownership Revision3. No new operation, forward edge, principal, key, durable store, journal, collector or migration. + +## Decision and source boundary + +333/335 expose a genuine gap: successful pull changes cache B->E, then the sealed candidate expires and is correctly deleted without maintenance. The OLD B container/config remain. A fresh explicit Update is promised its own checked pull, but R3's ordinary capture cannot own B against cache E; Apply similarly lacks the original candidate/owner. Never reuse/renew/reseal the expired candidate or its parent/grants. This is a historical read-only provenance problem, not authority continuation. + +Install one authenticated alternative source in the LITERAL production owned-services resolver, for data-manager ONLY. Its unchanged ordinary cache-image equality remains the default. If that cannot establish DM provenance, a fixed reader of existing actual successful staged-pull records may establish the SAME OLD container's provenance. All other services and all no-proof cases retain Task01.1's exact ordinary checks. This is an explicit narrow amendment to R3's original-candidate-only restriction and Task01.1's current-cache-only source, not a claim that those texts already allow it. Ownership source is separate from action permission: current enabled/protected policy, normal fresh admission, non-recreate Start and mandatory DM maintenance recreate/decision gates remain unchanged. + +The authenticated source also permits the currently stopped SAME OLD DM container to be observed as stopped; it does not report readiness. Existing permitted fixed --no-recreate DM Start can then use this proven identity and its OWN fresh policy/admission. No helper is issued after a new pull or while a maintenance root is held. Existing logs/stop permissions are neither expanded nor inferred from ownership alone. Guarded Apply still needs actual running/readiness and its own candidate/root/decision/final; NEW Update still requires its OWN successful checked pull even if downloads are cached. Missing/unknown prior containment never licenses fresh effects. + +Reject alternatives: a permanent failed-retry/host-only repair contract violates authorized remote instance management for this normal TTL outcome; accepting arbitrary old tag images weakens ownership; retaining expired MAC as live authority violates TTL; another permanent ownership store is unnecessary because signed normal admissions already contain actual bounded baseline observations. + +## Fixed production installation and source verification + +Create apps/ops-agent/src/coordinator-staged-ownership.ts for the fixed source reader/retirement dependency, sharing pure records with core/build-artifacts/coordinator-pull-records.ts. Wire it at the SAME actual server startup point that attaches the runtime's configuration-maintenance journal, using the actual controlInstallation, actual bounded OpsJobJournal and productionOwnedServices(runtime). If runtime is not a real factory or resolver is not a literal createOwnedServiceResolver instance, installation refuses. No exported arbitrary callback registration or caller result/digest provider. + +The owned-services module retains a private resolver->fixed-source-state WeakMap. A fixed installCoordinatorStagedOwnershipSource(runtime,installation,journal) may bind that state ONCE after verifying actual literal resolver identity, ops-agent installation role and matching fixed infra/data roots. The journal is candidate INDEX DATA ONLY, never authority: scan at most its existing256 records, require complete bounded valid records, and authenticate every source from the existing indexed role-private normal admission/canonical/effect/receipt machinery. Do not enumerate arbitrary paths or read API/DM-private stores. Generic resolvers/structural claims cannot mint the source. + +An eligible record is a current strict candidate-bound service.pull(data-manager) with the SAME installation, a real signed first normal admission containing R3's3072byte baseline, actual succeeded immutable canonical terminal, committed contained ops-result settlement matching that terminal, and confirmed actual API receipt delivery for that exact canonical digest. All operation/body/execution/API-parent/attempt/origin/requested/effective fingerprint signatures must verify against their ORIGINAL values. No live-forward/deadline assertion is used to read historical evidence. A bare journal success, changed:true, obsolete changed-only result, unverified data, copied public receipt, failed/unknown/unfinished pull or missing ACK is ineligible. No original candidate file/TTL/row is required for this observation; expired/deleted candidate cannot issue anything. + +The fixed core reader returns opaque VerifiedStagedCoordinatorOwnership via private runtime branding, with original signed baseline, source ExecutionRef/origin, normal admittedAt, canonical terminal digest, effect settlement digest and API-copy digest. Only actual source verification can mint it. The fixed literal resolver consumes it read-only, never caller image/ID/path/flag. Existing signed normal admission is an authenticated bounded observation; it does not become a new lease or grant. + +## Current reality and fresh capture + +For each eligible candidate source, re-inspect actual Docker inventory through the SAME literal ownership implementation. Registered DM/project, exact original immutable container ID and OLD image ID, Config.Image reference, canonical retained generation/legacy aliases, file owner/mode/single-link, exact composeSourcesDigest/FD change checks, duplicate detection and inventory bounds all remain mandatory. Current state comes from actual Docker (running/stopped/etc), not the old baseline's captured running state. The historical baseline authenticates OLD identity/configuration, not current liveness. No missing or replaced row is synthesized. Unrelated source/pin/config/container changes and ambiguous conflicting provenance refuse. + +Compare actual current cfg1 and effective trusted DM reference/files to the retained OLD provenance. That cfg1 must be the baseline configurationRevisionId; source file digests must still match. Re-evaluate current registry membership/trusted Compose source and current policy. The old FULL raw configuration authority/policy revisions remain historical facts; they are NOT required to equal a newly observed global revision merely because unrelated registry/resources changed. The new candidate/source witness and live capture bind the full CURRENT raw+composite domains afresh under R3. A changed effective DM configuration/paths/reference fails rather than transplanting old proof to another runtime. Current cache equality to E is not provenance authority: the original signed OLD observation plus unchanged actual OLD row is. New Update must still independently validate pins and complete its own pull; Apply retains its approved trusted cached-image semantics and actual frozen/final image checks. + +In a fresh candidate-bound pull, the literal resolver's ordinary capture may consume this authenticated staged source instead of cache equality for DM. It then produces a NEW actual current observation (same verified OLD runtime), bound through the existing opaque raw-claim channel to that NEW candidate/body/current source witness/API parent/normal admission. Keep R3 baseline schema/version/3072 cap and copied witness/version2/8192 cap unchanged: the new role-signed baseline attests an actual fixed authenticated observation; it is not a recursive copy of old candidate authority. No ancestral packet chain is embedded. Original source must be present when this fresh observation is made. Each new normal admission, pull and API copy must complete under its own clocks/policy; a new admission alone does not supersede an older eligible source for retention. + +Use the same installed fixed source in all production ordinary ownership reads, claim-time capture, effect-time identity refresh and readiness preflight so a fresh Apply/Update/allowed non-recreate Start does not disagree about this proven DM identity. Generic recreate/Update bypasses remain denied by their existing workflow/decision policy. R3's original-owner private inspector remains valid and stricter for its original candidate; applied NEW final remains ordinary verified E, never OLD B. The ordinary resolver WITHOUT this authenticated installed source must still reject B when cache is E; unrelated rows never receive an exception. + +### Apply's own generation: fixed owner copy, not historical cfg1 bypass + +R1-I1 correction: the fresh resolver above still requires historical baseline cfg1 while the request has not committed its own generation. Apply has NO own pull witness, so it must freeze this verified provenance at preparation before that pointer changes. Add private optional stagedCoordinatorProvenance to the EXISTING signed/indexed maintenance owner. Its fixed actual producer alone creates a signed8192byte-bounded packet under domain openmapx-maintenance-staged-coordinator-observation-v1 with payload `{version:1,installationId,ownerPrepare:ExecutionRef,candidateId,planDigest,source:{execution:ExecutionRef,admittedAt,baseline:CoordinatorPullBaseline,admissionDigest,canonicalDigest,effectDigest,apiCopyDigest},capturedAt}`. Source baseline/candidate/parent facts remain HISTORICAL and unmodified; new ownerPrepare/candidate/plan are separate bindings. No relabeling of the prior CoordinatorPullWitness as this Apply's own pull proof. Source normal admission/canonical/effect/API ACK must be present and verified for this first copy. + +The opaque original-owner reader authenticates this exact signed copy against the actual owner's signature/identity/plan and original current revision. It permits ONLY baseline.configurationRevisionId before commit, and ONLY this owner's own committed generation revision afterward, verified from the actual signed generation-child canonical/effect/physical source (same original preparation/decision). A public receipt boolean or caller current pointer cannot select the exception. It continues rechecking the SAME actual B container/reference/OLD Compose FD digest and source paths. Use that owner-bound reader for Apply's frozen recreate admission/effect and original-runtime rechecks/recovery/UNUSED/contained inspections after generation. Foreign owner/plan/generation refuses. Applied final still requires actual frozen E/config/mount/readiness; B cannot satisfy it. + +Update with its OWN R3 coordinatorPull already has a self-contained original-owner baseline; use that, not an extra staged copy. The two owner proof variants are mutually exclusive. Any currently installed guarded follow-up that recreates DM and has no own pull, including credential service-apply sharing this ownership source, requires the staged copy when needed and must freeze the trusted actual desired DM image using the same coordinatorImage.action=apply expectation. Otherwise it would inherit the same post-staging OLD-vs-NEW image defect. This changes image evidence, not purpose/participant or recreate authorization. Before ANY helper/pull/effect, prospective budgeting reserves one full8192byte provenance allowance for EVERY owner with a DM recreate step, regardless of Apply/Update/credential origin. Existing owner131072, final inspection65536 and receipt65536 limits stay fixed; actual copy also validates8192 and refuses rather than truncating. No duplicate16KiB copy, extra pull or live use of expired source authority. + +## Existing-family retention dependency, no extra store + +The retained normal-pull family is the durable source. Its existing72h minimum is unchanged, but the LAST usable successful staged-source family cannot retire while the SAME OLD container still needs it. Add the fixed dependency to the existing ops-agent retentionJournal.prepare hook before retirement commitment, using the same literal production source. IMPORTANT R1-I2 correction: that hook currently runs on EVERY step, including after its own source records were deleted. It must become phase-aware as specified below, not repeatedly demand deleted admission/effect bytes during already committed cleanup. This is an additional liveness veto on existing GC, not an alternative collector, synthetic control node, new minimum floor or eviction. + +The fixed liveness check uses literal cache-only Docker/FD inspection INTERNALLY, without the new fallback, avoiding recursion. If successful bounded observation proves the original container absent, or ordinary cache-only ownership now proves the actual current DM container/config without this source, this dependency is no longer needed. A known stopped OLD B still needs provenance. Docker/source read failure, ambiguous replacement, corrupt/missing required proof or unknown state vetoes GC rather than implying absence. Existing creation/removal exclusion and file/record guards remain mandatory. + +An older source may otherwise retire only when a DIFFERENT already eligible successful/contained/API-ACKed source family authenticates the SAME current OLD container/config/FD facts and has a strictly GREATER fixed total rank (normal signed admittedAt, then operationKey lexicographically). This is an ordering of authenticated observations for GC only, never a time/forward permission. Candidate-less/unfinished/failed/unknown new pulls are not successors. Both families must be read under existing serialized role-retention/journal guards; successor must be active/not retiring at the checked boundary. Strictly increasing rank prevents mutual supersession/cycles; the maximum eligible source remains pinned while needed. A newly role-signed baseline already attests the exact original observation verified by the fixed producer, so after its successful canonical/effect/API copy it is a self-contained replacement without retaining every ancestor. Wall-clock rollback remains governed by existing fail-closed retention clock; no clock renewal or ordering based on caller time. + +Normal journal/family caps remain256/unchanged byte limits; pinned sources are not evicted to admit new work. Repeated successful-expired requests can consume their ordinary72h capacity before GC; report existing capacity refusal truthfully. On actual guarded replacement/restored ordinary ownership, existing all-role retirement can reclaim source families after their original floors. Missing/corrupt source never creates an expired candidate or synthetic ACK. Restart reconstructs the fixed reader and GC dependency from actual existing sources, not an in-memory liveness promise. + +### Authenticated retirement phases and cleanup resume + +Extend the existing core-owned prepare callback to `prepare(executions, context?:ControlRetentionPreparationContext):Promise`. Existing unrelated callbacks can ignore the optional second argument. The context is an opaque runtime WeakMap-backed token created ONLY by the existing collector after verifying family identity/progress high-water and applicable signed proof/manifest/decision. It is not a structural phase flag. Fixed accessors expose mode preparing|before-commit|committed-cleanup, exact installation/origin/family/generation/execution-digest set, progress sequence, manifest digest when present and a single-use check nonce. A forged/copied context refuses. No caller can create cleanup authority. + +The actual fixed staged-source adapter implements prepareCoordinatorStagedRetention(context,runtime,installation,journal), consumes the literal source verifier/liveness check and supplies a purpose-bound signed dependency packet, NEVER an injected boolean/result callback. A strict4096byte packet under existing ops-agent role/domain openmapx-staged-coordinator-retirement-v1 records `{version:1,installationId,originDigest,familyId,generation,execution,terminalDigest,sourceAdmissionDigest,oldContainerDigest,rank,contextMode,checkNonce,manifestDigest,checkedAt,condition,successor}`. condition is actual not-needed or higher-ranked-successor; successor is null or exact authenticated eligible source execution/admission/canonical/effect/API-copy digests+rank. No full Docker content, paths, baseline or ancestral packet chain. The core recorder validates the nominal context, current sequence/nonce, exact signed packet identity/caps and source family membership; signature minting remains inside the literal actual producer, not a generic endpoint accepting a caller condition. + +At preparing, require complete source bytes and actual fixed safety predicate. Bind the signed preflight packet into optional stagedCoordinatorDependency in the existing role-signed retirement Candidate; bind its digest in the existing Manifest as stagedCoordinatorDependencyDigest (null/absent for unrelated families). Private proof.json retains that exact signed Candidate through cleanup. At before-commit, immediately before existing writeDecision(phase=committed), call the fixed hook AGAIN with a fresh core context nonce and exact manifest. Re-evaluate actual need or a currently valid strictly higher-ranked successor under the existing serialized journal/retirement guards. A different safe higher-ranked successor is permitted, but arbitrary missing evidence is not. Persist this fresh signed4096byte check packet in existing Progress.stagedCoordinatorRevalidation before commitment, then include its digest in the original signed committed Decision. This check adds no execution/node or new record file. No async forward effect or new admission is dispatched by it. Candidate/manifest/decision/progress original byte ceilings and reserved collector headroom remain; account for BOTH4096 packets in pre-admission future budget (one candidate proof, one progress), and refuse source-family admission before pull if that reserved space cannot fit. Manifest/Decision add only bounded digests, not packet copies. + +The collector creates committed-cleanup context ONLY after verifying the exact original signed committed Decision/manifest/private Candidate packet and persisted signed revalidation packet/digest, family/generation/execution set and existing deletion cursor. Those retained proofs remain available after admission/effect deletion. The fixed adapter then permits ONLY continuation of that same committed deletion/journal cleanup, without rereading the source records this transaction already deleted or re-deciding dependency liveness. It cannot use committed-cleanup to approve any active/prepared family, another manifest or new source. Missing/corrupt ACTIVE/PRECOMMIT evidence vetoes; missing/corrupt committed cleanup proofs also veto. Historical already committed retirements without this new field may complete only via their original existing authenticated cleanup contract and supply no staged-ownership source; active baseline families cannot use a historical absence as exemption. + +Place phase classification/verification BEFORE the current unconditional prepare hook; preserve existing generic prepare and SQL-guard callbacks' safety. Keep the original lock ordering: serialized existing collector/journal admission guards, no host target/config writer acquisition from GC and no resolver fallback recursion. Actual source admission/copy holds its existing guards; no new cross-role lock or private-store read. Once committed, core resumes its existing inode-CAS deletion cursor and exact journal retire, then normal slot release. Required fault case: after a permitted F->G supersession and first F private deletion, restart collector; authenticate original commitment, finish F journal+slot reclamation with G retained, no missing-active-source waiver or new packet/execution. + +## Implementation and causal acceptance + +1. Preserve333/335 successful staging/expiry cleanup and NEW explicit retry failure. Actual NEW API Update uses distinct job/key/parent/candidate/MAC, its own checked pull, one own guard/recreate/R5SQLACKrelease; original expired packet remains absent and original canonical/ACK bytes unchanged. Guarded NEW Apply after staging must cross its OWN generation commit/recreate/restart through the bound owner copy, with foreign generation/owner negatives (no invented pull or expired authority). Include currently installed credential DM guarded follow-up image expectations/owner budget regression. +2. Pure/actual source tests: no-source cache-only denial remains; actual signed eligible baseline+canonical/effect/API ACK proves SAME B while cache E; generic resolver/structural/foreign-installation/source/currentcfg/ID/image/reference/FD mismatch and failed/unknown/missing ACK records refuse. Fresh raw/composite joins use current identities, old observations remain historical. Actual stopped OLD source can use one permitted non-recreate Start before NEW pull; actual readiness remains required. +3. Native process restart AFTER old expiry cleanup and BEFORE a fresh user request reconstructs the source, not old MAC. Repeated explicit successful-staging/TTL expiry followed by new own successful Update proves no authority replay, no recursive packet growth or candidate reseal. All captures/baselines/witnesses keep existing byte limits. +4. Actual owned all-role GC beyond72h refuses to remove the LAST source while B/cacheE remains, including stopped B and unavailable Docker. Two eligible sources cannot mutually retire; only lower-ranked source retires after successful ACKed successor. A merely admitted/failed/unknown source cannot unpin its predecessor. Actual later guarded E replacement/ordinary ownership permits normal retirement and original slots/journal cleanup. Crash AFTER first F private deletion/BEFORE journal retire must resume original authenticated committed cleanup with G still protected; active missing/corrupt proof and forged phase/context/digest refuse. Fault/restart before manifest/commit and changed successor must revalidate without losing last proof. Actual maximum candidate/manifest/decision/progress packet bounds and future reservation refusal tested before effects. No new collector/process/store or ordinary settlement evidence fabricated. +5. Run relevant ordinary ownership/capture/copied-claim/guarded Apply/Update/native/TTL/bulk/retirement/capacity regressions, types/authority/style/full final-source gates. Request root generation only if coherent public source actually changes; this design intends no public wire/migration. Full Task01.2 source review must assess this deliberately narrowed ownership-source addition and source-lifetime dependency. + +## Self-review + +The policy still requires authenticated actual ownership and current action permission. The new source is limited to DM and signed past observations of the SAME still-present container/config, rechecked now; arbitrary old cached images remain denied. Historical signatures establish source facts without renewing their authority. New requests create only their own fresh work. Revision2 carries Apply provenance across ONLY its own authenticated generation using a separately bound owner copy/frozen desired image, and reserves the same bounded allowance for installed guarded DM follow-ups. GC distinguishes authenticated precommit liveness decisions from committed cleanup whose original source bytes may already be gone, with bounded signed predicate/revalidation packets in EXISTING files and exact manifest/decision binding. Source installation, historical/current revisions, no-proof denial, acyclic ACK-backed supersession, native/restart/GC/capacity acceptance remain required. Cost if wrong is confined source/consumer/retention work; no new forward graph or expired intent continuation. Design clearance is not runtime/source/CI completion. From 415a86f5286786cad9b1bcbcdd2c5f50c4cd1e2c Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 27 Oct 2026 16:38:26 +0100 Subject: [PATCH 032/135] docs(admin): guard staged coordinator configuration transitions --- .../2026-10-23-admin-01-service-control.md | 2 + ...026-10-23-admin-09-maintenance-controls.md | 2 + .../2026-10-23-admin-instance-management.md | 2 + .../2026-10-23-admin-control-protocols.md | 2 +- ...0-27-admin-coordinator-lifecycle-design.md | 2 + ...n-staged-configuration-interlock-design.md | 61 +++++++++++++++++++ ...min-staged-coordinator-ownership-design.md | 2 + 7 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index bdc7ffd16..95cfe4d72 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -91,6 +91,8 @@ The [approved pre-pull ownership addendum](../specs/2026-10-27-admin-coordinator The [approved staged ownership addendum](../specs/2026-10-27-admin-staged-coordinator-ownership-design.md), full R1 plus scoped R2 READY, is part of Task01.2. Install the fixed actual historical source/capture for fresh retry/Apply/allowed non-recreate Start; separately bind Apply-owner provenance across only its own generation, freeze actual desired DM image for installed guarded follow-ups and reserve the exact allowance before effects. Implement last-source liveness/safe supersession plus authenticated GC phase context, bounded preflight/manifest/commit revalidation and original committed cursor/journal cleanup. Preserve existing limits/no new store/collector/root, and execute the specified native/expiry/foreign/GC crash/capacity/guarded positive acceptance. +The [approved staged configuration interlock](../specs/2026-10-27-admin-staged-configuration-interlock-design.md), independently READY at R1, is part of Task01.2: exact source predicate at first/repeated preview BEFORE desired save, host writer BEFORE side effects, actual nominal DM generation exemption, ordinary fixed-CLI whole-root retention and reciprocal nonblocking DM-pull lease check. Actual preflight eligibility uses the MAC/fixed planned continuation, not a future decision; publication still requires owner/child/decision. Prove existing not-started settlement/own-root release, both races, stopped helper, zero-write refusal, explicit remote DM repair then NEW retry, bulk-prefix refusal/no reordering and native replay. Bare DM pull remains truthfully unavailable until existing Plan09M3 installs its real purpose-bound producer; no recommendation is dropped. + The [approved outer settle request digest](../specs/2026-10-26-admin-settle-request-digest-design.md) is part of this task: implement the exact portable observation-checksum helper, API original-attempt checkpoint, actual owner/sequence comparison before first settle control CAS, and duplicate/restart retention. Keep actual final-v2 runtime evidence distinct and agent-produced; execute the specified mismatch/stale/unknown/race/recovery/positive causal tests without adding a receipt field or endpoint. The [approved maintenance coordinator ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) is part of this task: maintenance-bound nonempty ensure-running performs the existing ordinary helper Start only after whole-candidate preflight and BEFORE maintenance root/guard acquisition, conditioned on the observed old/current revision. Original helper settlement/release/readiness and source/TTL rechecks precede unchanged ownership-only prepare; no new frozen Start edge or post-acquisition fallback. Retain its exact loss/restart/cancel/initially-stopped/require-running causal acceptance. diff --git a/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md b/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md index 23a6c68f8..dc631a51e 100644 --- a/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md +++ b/docs/superpowers/plans/2026-10-23-admin-09-maintenance-controls.md @@ -72,6 +72,8 @@ lines and256KiB, including a bounded incomplete line and truncation indicator. ## Task M3: Pull-only staging and release pin clearing +The [staged configuration interlock](../specs/2026-10-27-admin-staged-configuration-interlock-design.md) makes current bare DM pull unavailable until THIS task installs an actual purpose-bound pull-only producer. Complete that scope: share the existing API administrative writer and fixed authenticated OLD-baseline source/capture, normal submission/containment/API copy and source-lifetime discipline; do not fake coordinator-Update private intent or bypass the reciprocal target-lease check. Pull-only still performs no generation/start/recreate. Release source mutations must honor the staged prerequisite before writes, with explicit guarded DM repair first. Restore truthful installed capabilities only after the real workflow exists and add actual pure-stage/failed/unknown/expiry/fresh-retry/current-source coverage. No extra task/migration or silent permanent refusal. + Consume the [approved conditioned pull contract](../specs/2026-10-27-admin-coordinator-lifecycle-design.md) installed by01.2: fresh service.pull requires exact expectedConfigurationRevisionId, optional candidateId has only its fixed verified purpose, and successful image/source metadata comes from the actual producer. Preserve original normal submission/admission/lookup and legacy observation rules; do not reintroduce service-ID-only fresh pulls, caller image/path/source assertions or cached-image continuation after failure. This task still owns pull-only UI/staging and release pin clearing; no task/migration reallocation. **Files:** Create focused `apps/api/src/routes/admin-release-controls.ts`, diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index 88bd10678..42107256f 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -23,6 +23,8 @@ The [approved pre-pull ownership addendum](../specs/2026-10-27-admin-coordinator The [approved staged ownership addendum](../specs/2026-10-27-admin-staged-coordinator-ownership-design.md), full R1 plus scoped R2 READY, closes fresh retry/Apply after successful image staging and expired candidate cleanup. DM-only authenticated historical provenance is separated from fresh current authority; owner-bound Apply proof and phase-aware last-source retention use existing bounded records and GC. It is an explicit narrow source-boundary amendment inside Task01.2;50-task scope/order and actual full source/runtime/CI qualification remain required. +The [approved staged configuration interlock](../specs/2026-10-27-admin-staged-configuration-interlock-design.md), R1 READY, selects explicit pre-write refusal over unrelated-generation lineage carry. Pending staged DM is resolved through guarded Apply/Update before ordinary publication, with actual pre-save/host-writer checks and existing managed-root serialization. Plan09M3 still MUST finish real DM pull-only staging; temporary unavailability is not completed coverage. Existing50-task order and full actual/source/CI qualification remain unchanged. + ## Approved Task01.2 outer settle request digest The [approved digest derivation](../specs/2026-10-26-admin-settle-request-digest-design.md), independently READY at R1, separates the existing outer request observation checksum from the agent-produced final-v2 evidence digest. Its exact portable receipt projection, first-admission owner/sequence CAS and retained winner/restart semantics add no schema, endpoint or authority. Existing Task01.2 product/runtime acceptance and50-task order remain unchanged. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 134d86079..12ab84663 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -289,7 +289,7 @@ If legacy unchanged state is already over a budget, allow an explicit size-reduc ### 5.4 Remote configuration/feed maintenance (R2-02) -**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. The [approved coordinator Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) closes the current non-credential DM follow-up gap: fixed API-owned render/empty-overlay intents, mandatory guard, conditioned normal pull submission/source witness before Update maintenance, and original frozen recreate. Its binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md) retains actual OLD container provenance in the first normal pull admission through the literal resolver/claim channel, separates full raw and composite policy revisions, and copies it into the version2 owner witness for exact OLD-only inspection while applied final verifies NEW; ordinary ownership remains unchanged. The [staged ownership addendum](2026-10-27-admin-staged-coordinator-ownership-design.md) adds the closed DM-only authenticated historical source for a fresh request after successful staging/expired MAC, separately bound Apply-owner provenance and existing-family last-source/phase-aware cleanup dependencies; fresh current policy/own pull/root/decision remain mandatory and expired authority cannot forward. Its explicit seed/default-CLI/readiness exceptions supersede only those earlier generic clauses; the maintenance graph and DMv2/control/SQL/ACK authority stay unchanged. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. +**Scope and owner.** This is one purpose-bound configuration protocol, not a remote lock API. The [approved coordinator Apply/Update disposition](2026-10-27-admin-coordinator-lifecycle-design.md) closes the current non-credential DM follow-up gap: fixed API-owned render/empty-overlay intents, mandatory guard, conditioned normal pull submission/source witness before Update maintenance, and original frozen recreate. Its binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md) retains actual OLD container provenance in the first normal pull admission through the literal resolver/claim channel, separates full raw and composite policy revisions, and copies it into the version2 owner witness for exact OLD-only inspection while applied final verifies NEW; ordinary ownership remains unchanged. The [staged ownership addendum](2026-10-27-admin-staged-coordinator-ownership-design.md) adds the closed DM-only authenticated historical source for a fresh request after successful staging/expired MAC, separately bound Apply-owner provenance and existing-family last-source/phase-aware cleanup dependencies; fresh current policy/own pull/root/decision remain mandatory and expired authority cannot forward. The [staged configuration interlock](2026-10-27-admin-staged-configuration-interlock-design.md) adds the explicit pre-save/pre-publication prerequisite for unrelated configuration while that source is needed; actual guarded DM repair is reachable without weakening provenance, and managed CLI/pull serialization plus proven not-started settlement preserve race/cleanup safety. Its explicit seed/default-CLI/readiness exceptions supersede only those earlier generic clauses; the maintenance graph and DMv2/control/SQL/ACK authority stay unchanged. API has only the trusted-config subtree; it neither constructs `VerifiedExecutionAuthority`/`TargetLeaseAuthority` nor gains a shared-data/private-key mount. The agent owns one durable root from preparation through settlement. Required consumers are direct feed edits, imports containing the managed map, any configuration follow-up recreating DM, and08.3 protected configuration handoff; other candidates keep their existing apply path. A candidate requiring this boundary cannot omit it.01.2 owns API `services/configuration-maintenance.ts`, agent `configuration-maintenance-runtime.ts`/`configuration-maintenance-store.ts`, portable `ops/configuration-maintenance.ts`, registration/policy and candidate/lifecycle adapters.02.3 owns the earlier DM foundation below, not an undefined future01 import.06.1 supplies only the signed authority/finite edges.05.6/04.6/08.3 install their existing participants/consumers;50 tasks and master order stay unchanged. **Preflight before ownership admission.** Under the API global→sorted integration writer, first perform non-consuming `check/checkMany`, strict full prospective schema/source/size validation, exact seal, queue-capacity reservation and durable private candidate receipt. No remote ownership admission is allowed if any of these fails. Read-only host source/hardlink/health inspection is permitted during preflight. The inert candidate may be staged privately for those reads; it is not a consumable apply claim. Derive all eventual targets/effects before prepare: changed service closures, full prospective hardlink producer/consumer closure, DM if needed, MOTIS/motis-staging/motis-feed-proxy closure for feed credentials, and the protected helper subset. Include dependencies required for recreation, never just caller IDs. Candidate retains the exact safe follow-up intent, current image/container expectations and plan preview. Agent independently derives and freezes the closure from trusted manifests, candidate and source/preview witnesses. Reject unavailable prerequisites, >256 sorted targets, >32 effect children or changed plan before prepared; no partial plan or late target expansion. `require-running` never starts DM; its feed flow may explicitly recreate an already-running DM for the new mount, whereas `render-only` permits no lifecycle at all. These distinct actions are frozen before save. Before sealing, allocate candidateId/maintenanceId and the prepare key once using the existing durable API key helper with namespace `configuration-maintenance.prepare` and durable identity `apiJobId + ":" + candidateId`. Derive applyOperationKey from that prepare key with§3.5 phase `configuration-commit`, ordinal0, so the sealed apply identity already equals the frozen child key. Preallocate all safe effect inputs; no post-prepare reseal is required. diff --git a/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md index bdd734504..566deb7c6 100644 --- a/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md +++ b/docs/superpowers/specs/2026-10-27-admin-coordinator-lifecycle-design.md @@ -28,6 +28,8 @@ The actual production pull takes the existing configuration writer -> release-st Actual normal admitted pull execution/body/parent, terminal canonical result, effect settlement and immutable role signatures are the ONLY source for VerifiedCoordinatorPull, constructed by a fixed agent source reader, never public result/digest data. The binding [pre-pull ownership addendum](2026-10-27-admin-coordinator-prepull-ownership-design.md) supersedes this paragraph's original version1 copied witness: the actual literal resolver captures OLD provenance through a private nominal channel, the fixed server consumes it BEFORE structural cloning and signs it in the SAME first normal admission; its full configuration authority revision and composite policy revision remain distinct. After actual successful containment, version2 CoordinatorPullWitness copies that baseline and exact original admission digest inside the existing owner. Its strict signed8192byte ceiling,3072byte baseline/65536byte admission limits, pre-effect confirmation, actual same-container OLD verification, Docker-normalized sibling-reference refusal, original submission binding and source-only recovery/retirement rules are mandatory. No late candidate mutation, new root/store/key/graph or general resolver relaxation is permitted. Original candidate/source/image/pins/parent/terminal/effect checks and actual final NEW image verification remain required; copied observations are not final authority. +The binding [staged configuration interlock](2026-10-27-admin-staged-configuration-interlock-design.md) requires explicit guarded DM repair before unrelated publication while staged provenance is needed. Keep original request order, strict source/current-revision equality and fresh authority; no unrelated generation becomes a DM owner. Nonpublishing old-revision --no-recreate helper Start remains available, and actual publication still needs its original nominal owner/decision. + ## Exact Update ordering, clocks and prerequisite safety 1. Bind the original current seed cfg1 and selected-release source under the same API global writer. Allocate candidateId FIRST, then preallocate/checkpoint originalPullReference as a **submission**, NOT an ExecutionRef: strict `{version:1,requestVersion:2,expectedExecutor:"ops-agent",expectedPrincipal:"api",stepAddress,operation,operationKey,operationFingerprint,requestedDeadlineAt,attempt:ApiJobChildAttempt}`. operation is the exact conditioned service.pull body including that candidateId and seed revision. attempt is the ACTUAL existing prepareApiJobChildAttempt result (version,apiJobId,apiOwnerVersion,parentOperationKey,parentFingerprint,parentDeadlineAt,childOperationKey,childOperationFingerprint,requestedDeadlineAt,attemptId), persisted under original job CAS before sealing; duplicate fields must agree. No executionId, admittedAt, effective deadline or execution fingerprint is predicted. Original managed-step allocation picks requested deadline within original API job and15-minute policy; delayed admission/deployment cap may reduce effective time, never renew it. The immutable MAC candidate includes this exact reference BEFORE helper/pull POST. On reclaim, only the current nominal job owner may deliver the retained originally unattempted submission with its original attempt/parent binding, never regenerate attemptId/owner version/requested time merely because a new worker owns the row. Attempted is lookup-only. diff --git a/docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md b/docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md new file mode 100644 index 000000000..7e7cb6902 --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md @@ -0,0 +1,61 @@ +# Resolve staged DM before unrelated configuration publication + +Status: APPROVED Revision1 after independent R1 READY (0 Critical / 0 Important). Design readiness only; actual source, runtime and CI qualification remain required. Same Task01.2, addressing actual371. No generation-lineage carry, new root/graph/operation/key/store/collector/migration or expired authority. + +## Decision and operator behavior + +371 proves ordinary motis configuration successfully advances global cfg1 after successful DM staging/expiry cleanup, without recreating DM. The historical DM source correctly refuses the unrelated cfg1 afterward. Do not relax current-revision equality, relabel that generation as the DM owner's, synthesize a copy, re-seal the old candidate or implicitly recreate DM. + +Choose the explicitly permitted PRE-WRITE REFUSAL alternative. While actual DM ownership requires the authenticated staged-source alternative, ordinary managed configuration publication must first resolve it by an EXPLICIT guarded DM Apply/Update. Fixed guidance: "Apply or update Data Manager before changing instance configuration." Users can do that entirely through the existing UI/API, then retry the original change. This is a temporary prerequisite, not a permanent feature refusal. No implicit repair, target reordering or extra pull/recreate is added. + +The blocked set is ALL currently managed desired/configuration mutations that publish global generation or mutate desired selection/release image sources: service/integration config, credentials/import/selection/ordinary render/apply, ordinary shared bulk seed/default-CLI preparation, Compose Up generation and release-source writes. Read-only inspection/YAML preview/status/logs and nonconfiguration users/privacy/data operations stay their normal policy. Conservatively block even a purported no-op mutation; do not rely on a caller revision or inferred unchanged render. A genuine authenticated existing maintenance owner with a frozen DM recreation is the ONLY publication exception, below. Guarded feed-map/render-only owners without a DM recreate must first resolve staged DM as well. + +The actual fixed OLD-revision data-manager --no-recreate helper Start DOES NOT publish a generation and is not blocked by this gate. It still requires current proven identity/policy/original expected revision and settles/releases before the new pull; it cannot render/force-recreate under that name. Gate derivation must distinguish actual publishing preparation from a nonpublishing lifecycle leaf, not indiscriminately reject every service.start and deadlock the stopped-DM remedy. + +This deliberately specializes prior statements that all unrelated ordinary configuration can proceed immediately. It does NOT change the staged source's cfg1/FD/identity verification. Existing unprovable already-orphaned/foreign generation state remains unknown and cannot be silently repaired from this rule; the managed system must prevent creating it. Host-local unmediated Docker/config writes remain the existing host-authority/recovery boundary, never a claim of web atomicity against a host administrator. + +## Fixed source predicate and exemption + +Implement the predicate in the existing apps/ops-agent/src/coordinator-staged-ownership.ts/owned-services literal source owner. It uses the installed real runtime/role/journal and actual cache-only inventory plus indexed signature/digest/inode-verified staged sources. No public flag/result/callback can assert dependency absence. States are no dependency, required, unknown as private data only: normal cache-only ownership or conclusive absent original container makes dependency unnecessary; same OLD proven solely by staged source (running OR stopped) means required; failed/ambiguous acquisition or unproved image/config drift is unknown. Required and unknown refuse ordinary publication. A corrupt unneeded historical family need not override conclusive actual ordinary ownership; missing required proof never becomes false. + +Add fixed assertStagedCoordinatorConfigurationWrite(runtime, operation, context, signal) and its read-only preview integration. The mutation is derived from the ACTUAL typed operation/MAC candidate/producer, not caller booleans. When required, allow publication ONLY if context authenticates the exact original nominal maintenance generation child/decision, its signed owner/plan has a frozen DM recreate step and actual desired image expectation, and this child's candidate/source/expected-current/new generation matches that owner. Reuse inspectOwnedMaintenanceGenerationRuntime and existing owner-step verifiers; mere maintenanceId/apiFollowUp/DM target is insufficient. Fresh DM Apply/Update and installed credential-DM guarded follow-up use their original contracts; no fake vault change. Actual same-owner OLD-before/own-generation-after checks remain mandatory. + +At this read-only preflight, authenticate the actual private MAC candidate and fixed producer whose continuation plans a DM recreation; do NOT require a future continue decision or generation child before desired persistence. This establishes only eligibility for preparation, never publication authority. At actual publication the exact nominal original owner/child/decision and frozen DM recreate/image checks above remain mandatory. Read-only candidate preview may evaluate the predicate and return its normal safe conflict/prerequisite error. It does not itself authorize later writing. The real writer must recheck at its own effect boundary. No new wire operation or browser-provided dependency/override is introduced. + +## API before desired save and agent before publication + +All current API configuration producers continue to hold the SAME existing withAdministrativeConfigurationWriter around prepare -> persistence -> apply. trusted-configuration-producer.ts currently does so, and coordinator intent requires that live writer. At FIRST actual configuration.hardlinks.preview and EVERY verifyCurrentPreview before persistOverlay, enforce the fixed predicate. A required/unknown ordinary candidate is refused before ANY desired row, encryption/reload, selection/generation/release write or consumable claim. It may have an inert bounded candidate/job; existing exact unissued cleanup must close/remove that only, preserve original F source, and allow the next explicit guarded DM writer. No cleanup based merely on a public failed status. + +Reuse existing HTTP409 conflict/prerequisite response shape, queued failed/noncommitted outcome and sanitized guidance. If a current route lacks that existing shape in its declared responses, add canonical declaration/actual route mapping and request serialized MAIN generation; do not hand-edit clients. Immediate mutation tests must assert real client-visible refusal/zero desired writes. Queued jobs may be accepted then truthfully fail prerequisite with zero forward effects; acceptance is not success. + +Recheck in actual trusted-config-runtime commitGeneration under its existing configuration writer BEFORE desired-selection side effect, materialization or pointer publication. Check all current typed config writers, not just stack.render. Actual permitted DM owner uses its nominal exception. This host check is a defense for stale/in-flight/private requests, not a substitute for API pre-save validation. Release selection/override mutation must call the same fixed predicate BEFORE its writes under existing configuration writer -> release lock. Identity/auth/privacy/database-only administration that does not publish/mutate these host sources is not in this gate. + +## Managed fixed CLI and staging serialization + +Ordinary default-CLI Update/Start preparation can publish the global generation. Its real normal agent root already holds the full registered target closure (server.ts targets for service.start/recreate/update and acquireArtifactTargetLease), including DM. Before beginning the control effect or launching that fixed CLI, validate this gate under the configuration writer while retaining that original target root. Release only the configuration writer before calling CLI, so CLI can acquire it normally; keep the original target lease until actual normal settlement. No recursive writer acquisition or sibling-private read from fixed CLI is needed. The existing CLI receipt/body/seed/pull->prepare->recreate semantics remain unchanged when the prerequisite is satisfied. Do not substitute another root or manufacture a CLI receipt when denied. + +Add the reciprocal fixed, read-only normal-DM-staging check assertCoordinatorPullLeaseCompatibility(authority): it derives the exact original service.pull(data-manager) from VerifiedExecutionAuthority and inspects the existing bounded signed physical target index. If ANY unreleased/unknown lease covers DM, refuse before Docker pull. A completely settled/released old prerequisite helper is fine. This check runs under the existing pull configuration writer/release lock but acquires NO target slot/lease; target-index reading is strictly nonblocking/fail-busy, so no target-after-writer wait or lock inversion. Unknown/expired-but-unsettled physical owner is not treated as released. This can conservatively refuse staging while DM data work is active; a job winning after pull still makes later maintenance busy as before. + +The two checks close the gap: if an ordinary generation root already exists, DM pull refuses; if it is admitted while pull is running, its writer-side check sees the new staged/unknown dependency and refuses before publication; if root-side check completes first, retained full target ownership prevents another DM pull until CLI settles. Pure API desired writes are serialized with candidate-bound staging by the existing API global writer. All currently managed staging producers must obey it. A CURRENT candidate-less DM pull has no purpose-bound baseline/producer and must refuse before admission/effects rather than orphaning ownership; generic non-DM staging stays its contract. Existing Plan09 Task M3 remains responsible for a real purpose-bound DM pull-only producer sharing this writer/source discipline—never fake coordinator-Update metadata or silently omit that promised coverage. This is a temporary internal boundary until that already-planned coverage lands, not deletion of pull-only scope. + +Current capability metadata must reflect that bare DM pull is unavailable with prerequisite guidance until M3 installs its actual staging workflow; no installed-looking generic permission may bypass this check. Do not invent a new workflow enum for an uninstalled producer. Existing Update/Apply workflows remain usable to resolve the prerequisite. + +Known prerequisite denial after a normal root is retained but BEFORE beginControlEffect must settle ONLY via existing actual receipt kind not-started: its original effect-started record is absent, actual root/reservation/child issuance never began, and existing core verifies failed_without_effect/not_started. Freeze/verify original nonissuance through the existing source helpers, durably retain canonical failure and release the SAME physical root only after exact settlement/no-child proof. No boolean or public status can assert this, and a begun/unknown effect remains fenced. Wire the finite fixed pre-effect source into server transition handling; never classify a failure after CLI dispatch as not-started. Add actual root-release/next-writer and race checks. This uses existing not-started settlement, not a new failure authority. + +## Bulk and recovery + +Preserve request order. A mixed batch with an ordinary generation-changing prefix before its DM item, while a dependency is already known, fails the whole unstarted batch prerequisite BEFORE that prefix or any pull/root. It does not silently move DM earlier. A DM-first batch may explicitly resolve the source then execute its later ordinary targets under the existing exact revision chain. Existing nonpending mixed-order behavior remains unchanged. Actual next-user DM repair then a NEW original-order batch must succeed. + +Known unissued blocked candidates use current inode/MAC/job-CAS nonissuance cleanup. Retrying the same failed job/key returns its original failure, not newly authorized work; a NEW explicit request after successful DM repair may proceed. On native restart, stale attempted/unknown writers stay lookup-only and desired/save facts remain truthful; never turn the new gate into a cleanup override. Existing source retention pin and GC phase proofs remain unchanged. Existing older pending configuration rows already saved under prior code retain truthful desired/applied/unknown handling; no claimed retroactive zero-write guarantee or fabricated rollback. + +## Causal acceptance and implementation order + +1. Keep371 actual successful ordinary save/currentAdvanced/nextDMfailure RED. SAME setup: canceled ACKed DM staging/expired cleanup, ordinary motis config now gets real409/noncommitted guidance BEFORE write; original rows/xmins/currentcfg/F bytes unchanged, no reload/render/recreate. Explicit guarded DM Apply or Update then the NEW motis config succeeds; no host intervention/implicit DM recreation. +2. Actual current ordinary config/secret/integration/import/selection/Compose/release/default-CLI generation paths exercise the shared gate, exact inert cleanup and next DM repair; no raw paths/secrets in guidance. Read-only status/preview/nonconfiguration actions remain their existing contract. Actual nominal DM generation/credential positive and foreign owner/UUID/decision/target negatives. +3. Actual managed root with DM retained blocks candidate-bound DM pull before Docker, with nonblocking busy/unknown lease behavior; a root admitted during held pull refuses publication before effects. Actual pre-effect not-started settlement releases only original root/no children/next writer. Begun or issued-child variants stay fenced. Test both race orders using actual production assembly and core target ledger, no injected grant/containment boolean. +4. Pending-stage [ordinary,DM] bulk refuses BEFORE prefix; [DM,ordinary] resolves then succeeds; after separate explicit repair the former order succeeds with exact ordinary seed/revision chain. Existing three-target/noDeps, unknown prefixes and original nonpending fixed-CLI crash/receipt tests remain. +5. Native API death before/after blocked candidate cleanup, stable failed replay then new repair/write; preserve normal source/current-generation/FD drift rejection and all last-source/GC/native/capacity guards. Run affected types/authority/style/current actual fixtures, final source covering, normal hooks/full Task01.2 review. Public declaration changes only through coherent root generation checkpoints. + +## Self-review + +This selects the user-permitted pre-write refusal instead of another generation-lineage protocol. It retains strict cfg1/source ownership, gives a working remote prerequisite, checks before desired persistence and actual host effects, and serializes managed CLI/staging through existing full target ownership and nonblocking reciprocal check. New current requests keep their own authority; no expired candidate or unrelated generation becomes a DM owner. Existing not-started source handles only proven pre-effect refusal; race/cleanup/native proof remains required. Plan09 M3 scope stays explicit. Cost if wrong is confined shared-gate/consumer/serialization work, not a new recovery authority or source carry. Design readiness is not implementation/runtime/CI qualification. diff --git a/docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md b/docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md index ff9b52848..d3c995b96 100644 --- a/docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md +++ b/docs/superpowers/specs/2026-10-27-admin-staged-coordinator-ownership-design.md @@ -2,6 +2,8 @@ Status: APPROVED Revision2 after full R1 plus scoped R2 READY (0 Critical / 0 Important), both findings addressed. Design readiness only; actual implementation, source, runtime and CI qualification remain required. Finite Task01.2 correction to lifecycle Revision2 TTL/new-request rule and pre-pull ownership Revision3. No new operation, forward edge, principal, key, durable store, journal, collector or migration. +The binding [configuration interlock](2026-10-27-admin-staged-configuration-interlock-design.md) prevents an unrelated ordinary generation from orphaning this strict source: while staged DM provenance is needed, resolve it through explicit guarded DM Apply/Update before managed configuration publication. Its pre-save/host-writer gates, source-preserving refusal/normal not-started cleanup and managed staging serialization add no generation-lineage carry or expired authority. + ## Decision and source boundary 333/335 expose a genuine gap: successful pull changes cache B->E, then the sealed candidate expires and is correctly deleted without maintenance. The OLD B container/config remain. A fresh explicit Update is promised its own checked pull, but R3's ordinary capture cannot own B against cache E; Apply similarly lacks the original candidate/owner. Never reuse/renew/reseal the expired candidate or its parent/grants. This is a historical read-only provenance problem, not authority continuation. From 07ae01322d91088b237124c7290487a20e02c7e3 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 27 Oct 2026 19:58:20 +0100 Subject: [PATCH 033/135] docs: define guarded release handoff allocation --- .../2026-10-23-admin-01-service-control.md | 4 +- ...6-10-23-admin-08-platform-configuration.md | 2 + .../2026-10-23-admin-instance-management.md | 4 ++ .../2026-10-23-admin-control-protocols.md | 2 + ...admin-release-handoff-allocation-design.md | 41 +++++++++++++++++++ ...n-staged-configuration-interlock-design.md | 4 ++ 6 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/specs/2026-10-27-admin-release-handoff-allocation-design.md diff --git a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md index 95cfe4d72..9db60d0c9 100644 --- a/docs/superpowers/plans/2026-10-23-admin-01-service-control.md +++ b/docs/superpowers/plans/2026-10-23-admin-01-service-control.md @@ -93,6 +93,8 @@ The [approved staged ownership addendum](../specs/2026-10-27-admin-staged-coordi The [approved staged configuration interlock](../specs/2026-10-27-admin-staged-configuration-interlock-design.md), independently READY at R1, is part of Task01.2: exact source predicate at first/repeated preview BEFORE desired save, host writer BEFORE side effects, actual nominal DM generation exemption, ordinary fixed-CLI whole-root retention and reciprocal nonblocking DM-pull lease check. Actual preflight eligibility uses the MAC/fixed planned continuation, not a future decision; publication still requires owner/child/decision. Prove existing not-started settlement/own-root release, both races, stopped helper, zero-write refusal, explicit remote DM repair then NEW retry, bulk-prefix refusal/no reordering and native replay. Bare DM pull remains truthfully unavailable until existing Plan09M3 installs its real purpose-bound producer; no recommendation is dropped. +The [approved release/handoff allocation](../specs/2026-10-27-admin-release-handoff-allocation-design.md), independently READY at R1, assigns01.2 actual release pre-write source enforcement plus temporary fail-closed legacy apply/update/helper/startup recovery. Missing guarded authority cannot write overlay/transaction/handoff, replay CLI or falsely publish rollback; existing status and original recovery evidence stay available.08.3 restores real coordinated release/helper functionality in this same final PR.01.4 MUST respect this boundary; no new root/graph/key or parent-only precheck permission. Preserve419 causal zero-write acceptance, actual production-binding refusal and read-only startup tests; future08.3 real helper/restart/race functionality is an unwaived final gate. + The [approved outer settle request digest](../specs/2026-10-26-admin-settle-request-digest-design.md) is part of this task: implement the exact portable observation-checksum helper, API original-attempt checkpoint, actual owner/sequence comparison before first settle control CAS, and duplicate/restart retention. Keep actual final-v2 runtime evidence distinct and agent-produced; execute the specified mismatch/stale/unknown/race/recovery/positive causal tests without adding a receipt field or endpoint. The [approved maintenance coordinator ordering](../specs/2026-10-26-admin-maintenance-coordinator-order-design.md) is part of this task: maintenance-bound nonempty ensure-running performs the existing ordinary helper Start only after whole-candidate preflight and BEFORE maintenance root/guard acquisition, conditioned on the observed old/current revision. Original helper settlement/release/readiness and source/TTL rechecks precede unchanged ownership-only prepare; no new frozen Start edge or post-acquisition fallback. Retain its exact loss/restart/cancel/initially-stopped/require-running causal acceptance. @@ -142,7 +144,7 @@ The [approved current ordinary-import ruling](../specs/2026-10-26-admin-ordinary - [ ] **RED test:** In real assembly, `core API restart survives loss of API observer`: accept dedicated job, checkpoint identity, drop observer before Docker effect, restart API runner and recover the same agent result without a second restart. Repeat for traefik and retained failure. Generic `service.stop/restart/update` on core IDs remains denied; core logs succeed. - [ ] **RED command:** `pnpm exec vitest run --project node apps/ops-agent/src/core-service-runtime.test.ts apps/api/src/services/__tests__/core-service-maintenance.test.ts apps/api/src/services/__tests__/job-runner-initialize.test.ts`. - [ ] **Implement core workflow:** Agent executes a fixed restart against verified container IDs, independent of the API/browser request lifecycle. Check current generation and current selection. Never offer web Stop for API/edge/agent: these remove the recovery channel. Protect all platform dependencies that would destroy the coordinating API's DB/queue availability (notably postgis/redis) from generic restart/stop/update too; return host guidance unless an existing dedicated checked workflow covers them. Task 1's policy uses a fixed trusted core-policy table, not just the current three-ID denylist. A service may still expose read-only logs while all lifecycle decisions are host-only. Never use this operation for image replacement or pin changes. -- [ ] **Reuse replacement:** API Update routes to existing `system.update` release validation/migrations/`appApi.replace` and pre-restart checkpoint. Agent Update routes to existing external `agent.update` helper/handoff only, with busy/checked-release guards. Keep agent ordinary start/restart/apply/remove disabled. Edge configuration apply belongs to Plan 08's guarded generation workflow, not generic lifecycle. +- [ ] **Reuse guarded replacement:** API Update retains checked-release validation/migrations/`appApi.replace` and the pre-restart checkpoint; Agent Update uses only the installed guarded external handoff. Apply the approved release/handoff allocation: while08.3's guarded release/helper installer is absent, report truthful unavailable capability and preserve inspection/recovery evidence; do not invoke the unsupported legacy mutation path or advertise it as runnable.08.3 MUST restore supported checked-release application/replacement in this same final PR, including actual helper process/restart/race proof. Keep agent ordinary start/restart/apply/remove disabled. Edge configuration apply belongs to Plan08's guarded generation workflow, not generic lifecycle. - [ ] **GREEN command:** Run RED plus `pnpm exec vitest run --project node apps/ops-agent/src/agent-update.test.ts apps/ops-agent/src/agent-handoff.test.ts apps/api/src/services/__tests__/system-maintenance.test.ts apps/ops-agent/src/production-runtime.test.ts`. - [ ] **Review/commit:** Confirm every available workflow capability has a real route/handler; the dedicated job is in restart-safe observation recovery. Commit `feat(admin): add durable core service restart controls`; retain host instructions for whole-stack shutdown and volume destruction. diff --git a/docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md b/docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md index d22da277e..959ff97b8 100644 --- a/docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md +++ b/docs/superpowers/plans/2026-10-23-admin-08-platform-configuration.md @@ -101,6 +101,8 @@ these reserved fields to escape their existing sandbox/resource authority. ## Task C3: Typed platform preview/apply and recovery +**Binding release/helper ownership:** Consume the [approved release/handoff allocation](../specs/2026-10-27-admin-release-handoff-allocation-design.md), R1 READY.01.2 supplies the actual staged-source pre-write predicate and temporary unsupported boundary for legacy release application, agent replacement and helper/startup recovery;01.4 respects it. THIS task MUST replace that boundary with actual coordinated supported release/helper functionality in the SAME final PR. Cover production release adapter transaction/overlay/state, detached helper write/restore/reconcile and startup recovery; every effect requires original owned target -> actual configuration writer -> release lock and live source revalidation, never a parent checked boolean or writer spanning process replacement/fixed CLI. Existing same-image configuration and new-image release are distinct modes: the same-image frozen graph below does not implicitly authorize a new-image mode. Obtain a reviewed finite amendment if the installed accepted graph cannot express that release mode. No new root/key/graph by allocation, no permanent feature removal. Add actual old/new-image helper process death/restart, delayed write versus concurrent staged DM, lost response, failed restore/unknown ownership, original evidence preservation and lock-order acceptance before re-enabling capabilities. Final full PR/CI cannot be complete while release parity remains unsupported. + **Files:** Create `packages/core/src/api/contracts/service-admin/platform.ts`, `apps/api/src/routes/admin-platform.ts`, `apps/api/src/services/platform-configuration.ts`, `apps/ops-agent/src/platform-configuration-runtime.ts` and focused tests. Modify diff --git a/docs/superpowers/plans/2026-10-23-admin-instance-management.md b/docs/superpowers/plans/2026-10-23-admin-instance-management.md index 42107256f..35cd44ba9 100644 --- a/docs/superpowers/plans/2026-10-23-admin-instance-management.md +++ b/docs/superpowers/plans/2026-10-23-admin-instance-management.md @@ -25,6 +25,10 @@ The [approved staged ownership addendum](../specs/2026-10-27-admin-staged-coordi The [approved staged configuration interlock](../specs/2026-10-27-admin-staged-configuration-interlock-design.md), R1 READY, selects explicit pre-write refusal over unrelated-generation lineage carry. Pending staged DM is resolved through guarded Apply/Update before ordinary publication, with actual pre-save/host-writer checks and existing managed-root serialization. Plan09M3 still MUST finish real DM pull-only staging; temporary unavailability is not completed coverage. Existing50-task order and full actual/source/CI qualification remain unchanged. +## Approved release-source and legacy handoff allocation + +The [approved release/handoff allocation](../specs/2026-10-27-admin-release-handoff-allocation-design.md), independently READY at R1, closes the current419 allocation gap without adding authority.01.2 implements the real pre-write source predicate and temporary fail-closed legacy release/helper/recovery boundary;01.4 shows truthful unavailable capability.08.3 MUST restore supported guarded release application/replacement/recovery through the actual helper in this SAME final PR. Existing inspection/evidence remains reachable, no false rollback/record deletion, no release-lock across writer-taking CLI, and no unreviewed new-image graph expansion. Full02.4 data update and09 staging remain their existing tasks. All50 tasks/order and final runtime/CI gates remain unchanged; temporary refusal is not completed release parity. + ## Approved Task01.2 outer settle request digest The [approved digest derivation](../specs/2026-10-26-admin-settle-request-digest-design.md), independently READY at R1, separates the existing outer request observation checksum from the agent-produced final-v2 evidence digest. Its exact portable receipt projection, first-admission owner/sequence CAS and retained winner/restart semantics add no schema, endpoint or authority. Existing Task01.2 product/runtime acceptance and50-task order remain unchanged. diff --git a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md index 12ab84663..d51271707 100644 --- a/docs/superpowers/specs/2026-10-23-admin-control-protocols.md +++ b/docs/superpowers/specs/2026-10-23-admin-control-protocols.md @@ -10,6 +10,8 @@ The approved [fixed proxy validation-failure contract](2026-10-25-admin-proxy-va The approved [fixed-reference fence settlement amendment](2026-10-26-admin-fence-settlement-reference-design.md) and [Task01.2 steps](../plans/2026-10-26-admin-fence-settlement-reference.md) supersede only the DM settlement wire/proof/control/preallocation/refusal/UNUSED/recovery/ACK/retirement details below. The complete settlement operation stays root-fixed; actual final facts live in authenticated reserved proof cells. This is a purpose-bound logical settlement control with no ordinary forward/physical lease capability. Apply revisions1–5 together (latest representation governs), including disjoint proved-never-prepared closure and exact two-cell proof bounds; no new root, generic late body, migration or raised global cap. Product acceptance remains required. +The approved [release/handoff allocation](2026-10-27-admin-release-handoff-allocation-design.md) assigns01.2 the real staged-source pre-write predicate and temporary fail-closed legacy release/helper/recovery capability boundary.08.3 owns installing coordinated supported same/new-image release, helper source/ownership and restart/rollback in this same final PR;01.4 cannot bypass the boundary by reusing the unsupported helper. No new root/key/graph or parent-only checked flag is added. Preserve original evidence and read-only recovery inspection; never hold release lock across a fixed child that acquires configuration writer. The allocation is design readiness, not source/runtime acceptance or a permanent feature deletion. + ## 1. Source constraints and ownership - `configuration-ownership.ts:acquireConfigurationOwnership` already uses exclusive mkdir, a 5-second wait, inode checks and no stealing after a crash. `configuration-generation.ts:withConfigurationGenerationWriter` owns the current pointer and commit evidence. Extend this discipline; PID age and TTL are not ownership. diff --git a/docs/superpowers/specs/2026-10-27-admin-release-handoff-allocation-design.md b/docs/superpowers/specs/2026-10-27-admin-release-handoff-allocation-design.md new file mode 100644 index 000000000..e3eb2cf2d --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-admin-release-handoff-allocation-design.md @@ -0,0 +1,41 @@ +# Release source interlock and legacy handoff allocation + +Status: APPROVED Revision1 after fresh independent R1 READY (0 Critical / 0 Important / 0 Minor). Controller self-review and source-backed design review are complete; actual implementation, product review and runtime/CI gates remain required. This narrows task allocation within the same50 tasks/one PR. It does not drop release management, create a root/graph/key/store/operation, or qualify source/runtime/CI. + +## Actual problem and decision + +419 is a real actual-API/owned-PG staged-source + productionReleaseEffects.apply causal RED: legacy apply writes transaction/overlay before the staged-source prerequisite. The installed literal agent can check the source under configuration-writer -> release-store order. The detached legacy helper cannot: createAgentHandoffBindings exposes synchronous writeOverlay/restoreOverlay and only a release lock; agent-handoff-main has checkout/socket/job journal but no installed literal resolver/private authenticated normal-pull source. updateAgent has no target lease. Parent-only precheck does not protect a later helper write after the parent exits or a concurrent staging operation. initialize holds the release lock across completeForward -> fixed CLI acquiring configuration writer; wrapping that scope in a writer deadlocks. + +Decision: Task01.2 owns the concrete shared release PRE-WRITE predicate and a TEMPORARY fail-closed capability boundary for all legacy release publication/replacement/recovery paths that lack the already-planned guarded workflow. Task08.3 owns replacing that boundary with actual protected release/helper coordination, including restart/rollback/source revalidation. Task01.4 must display/propagate the honest unsupported prerequisite while the guarded installer is absent; it may not revive the legacy helper as a shortcut. Task02.4's separate full data.update graph is unchanged. Task09 release-selection/pull-only features consume the installed source discipline and cannot bypass it. No final PR/whole-branch completion until08.3 has restored working managed release application/replacement with the actual required tests. + +Alternatives rejected: parent checked boolean has a race; exposing helper sibling-private files/signing material violates role/source boundaries; writer around whole initialize/fixed CLI deadlocks; moving the entire protected graph into01.2 repeats already-assigned08.3 and widens today's authority. Keeping unsafe legacy effects enabled with a ledger note is insufficient. The temporary capability boundary has no lasting feature removal and is removed only by the real reviewed installer, not a boolean override. + +## Task01.2 concrete implementation contract + +1. In the actual production release adapter, evaluate the SAME fixed staged-coordinator predicate under existing configuration writer BEFORE fresh transaction, current selection/state, release overlay, consumed-handoff, launch-intent or target effects. Use writer -> release-store order. Staged-required/unknown returns the existing sanitized prerequisite guidance, zero writes/CLI/helper/Docker mutation; real source observations remain nominal. Context-absent helper may NOT invent an absence result. Keep ordinary immutable manifest cache/inspection handling separate from current selection/overlay publication; this ruling grants no new DM image-staging producer. +2. A successful precheck is NOT authority for a later write. Current legacy apply/updateAgent/recovery have no retained root covering the writer gap and no accepted protected continuation. Until08.3 installs the real workflow, they must refuse the mutating branch BEFORE any transaction/overlay/handoff/launch/forward publication, even when no staged dependency is currently observed. Use an existing sanitized unavailable/unsupported error path, not new operation/schema/workflow flag and not the misleading DM-repair guidance for this missing-capability case. Production capability/route consumers must not present a runnable legacy replacement. Read-only inspections remain available with existing truthful unavailable/unknown/recovery information. A check-only valid source must not be cached as permission. +3. Production createAgentHandoffBindings writeOverlay/restoreOverlay must NOT retain a reachable legacy mutation bypass. With no installed guarded helper authority those functions refuse without writes; the existing helper's ordinary runDocker/recreate must not occur after a refused write. Do not retrofit source authority from rootDir/socket/journal/public handoffId or construct an ephemeral root. Existing generic Docker/helper machinery may stay as an internal implementation building block for08.3; production bindings are the boundary. Tests must reach these actual bindings, not replace them with a throwing fake. +4. Existing nonterminal legacy transaction/handoff recovery is read/inspect-only while the guarded recovery installer is absent. No initialize/completeForward/restorePrevious/reconcileAgentHandoff startup write, CLI replay, restoreOverlay or new release selection is allowed by a precheck alone. Keep exact prior evidence and report recovery-required/unknown through existing shapes; do not erase transaction/handoff, rewrite original canonical bytes, call it rolled_back/nothing_changed, or clear a fence merely because capability is missing. Agent startup and read-only status must remain reachable. Do not acquire configuration writer under release lock or keep either writer across a fixed CLI that reacquires it. Historical earlier-image helpers outside this binary are not retroactively made safe; an observed live/unknown handoff retains existing mutation/drain denial. +5. Do not alter ordinary per-service coordinator Apply/Update, nonpublishing stopped-DM helper Start, current credential/import/selection source gates, or full source signature/index/inode checks. No synthetic receipt, fake secret overlay, new signing key, target acquisition, fallback, TTL renewal or callback-selected authorization. Unsupported is a truthful temporary capability result, not success/containment authority. + +## Task08.3 binding ownership and final deliverable + +08.3 must install the protected helper's real source/ownership, actual fixed generation/decision/maintenance context and original delegation, then re-enable supported release mutation via that actual installer. Same-image configuration and new-image release are distinct validated modes, not reuse of updateAgent's image-only no-op. If the existing accepted finite helper graph cannot express a new-image release mode,08.3 must obtain a reviewed finite amendment before coding that mode; this allocation grants no new edge now. No arbitrary overlay bytes/path/caller predicate/captured checked flag may authorize helper writes. + +Each helper publication/restore and source transition must occur under target -> configuration writer -> release-store with real source revalidation at the actual effect boundary; no configuration writer crosses process replacement. Release locks cannot surround a fixed child that needs writer -> release. Retained publication attempts/revisions/old and new image/source identities, exact rollback ownership, API/agent death, unknown effects, startup/reconcile and actual last-source retention are part of that task. Restore of an overlay is not proof that target configuration/image is restored. Use the same original maintenance/target authority and exact current runtime evidence; no second root, sibling-private reads, new operation-by-allocation, or missing-source success. Preserve supported image release validation and full platform completion through the eventual real API/UI path. + +Before the one PR can be complete, real production API -> agent -> detached helper with original process exit/replacement must prove supported release application, healthy same/new image verification, lost response/restart, competing staged DM during the delayed helper write, failed restore/unknown retained ownership, and no writer/release lock inversion. Final unfiltered CI must pass; no permanent unsupported control, silent scope reduction or claimed full release parity at01.2. + +## Required focused causal acceptance in01.2 + +- Preserve419 exact old behavior RED. SAME genuine staged F/production release adapter now refuses the prerequisite with zero overlay bytes/inodes, transaction/current pointers, launch/helper/CLI/desired writes and unchanged F. No fake release adapter or synthetic lifecycle receipt. +- No-stage production adapter refuses absent guarded-release capability before ALL listed effects; read-only inspection/status remains usable. Existing completed evidence is still observable and idempotent. Actual legacy production helper write/restore bindings refuse, zero Docker recreation, and no false rollback publication. +- Seed genuine legacy transaction/handoff through existing current producers/actual original artifact writers, restart actual production initialization and prove no forward/rollback effects or record deletion/clock replacement; error/status accurately retains recovery uncertainty. Do not fabricate signed authority to make a positive unavailable workflow. +- Verify real lock schedule/finite completion/no recursive writer or release->writer path introduced. No full protected handoff release positive is claimed here; those cases are explicit08.3 gates. +- Affected actual current service/credential/selection producers stay passing; types, source-authority inventory, style, generated freshness when needed, final source checks/normal hooks and full independent01.2 review remain required. No generator/public-wire change is implied by this ruling. + +## Plan integration and self-review + +This allocation governs interlock normative, protocol/master and01/08 plans, and01.4's replacement reuse note. Preserve50 tasks/exact order and existing future02.4/09 coverage. Add an explicit unwaived08.3 final release-source gate and record intermediate capability refusal. Task01.2 completion may establish foundation + actual fail-closed boundary, never full release implementation. Cost is a small temporary availability boundary and later assigned installer, rather than an unreviewed second helper authority. + +Self-review: actual419 causality, parent/helper gap, every production bypass/startup/reconcile/restore path, truthful capability/status, no deadlock, no false rollback/receipt, exact future owner/final PR acceptance and no recommendation loss are explicit. User opted out of questions and authorized autonomous allocation/implementation. This document is not a product source approval or runtime/CI receipt. diff --git a/docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md b/docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md index 7e7cb6902..a0a5f3a73 100644 --- a/docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md +++ b/docs/superpowers/specs/2026-10-27-admin-staged-configuration-interlock-design.md @@ -14,6 +14,10 @@ The actual fixed OLD-revision data-manager --no-recreate helper Start DOES NOT p This deliberately specializes prior statements that all unrelated ordinary configuration can proceed immediately. It does NOT change the staged source's cfg1/FD/identity verification. Existing unprovable already-orphaned/foreign generation state remains unknown and cannot be silently repaired from this rule; the managed system must prevent creating it. Host-local unmediated Docker/config writes remain the existing host-authority/recovery boundary, never a claim of web atomicity against a host administrator. +## Release and detached helper allocation + +The [approved release/handoff allocation](2026-10-27-admin-release-handoff-allocation-design.md), independently READY at R1, specializes implementation ownership of the release-source requirement. Task01.2 installs the actual pre-write predicate and a temporary fail-closed boundary for unsupported legacy release application, replacement and startup/helper recovery. A parent precheck alone cannot authorize a later helper write. Actual production helper bindings and reconciliation may not mutate or falsely terminalize existing evidence while their guarded authority is absent; read-only inspection and recovery status remain available. Task01.4 must respect the missing capability. Task08.3 MUST install real coordinated supported release/helper application and recovery in this SAME PR, obtaining a reviewed finite new-image-mode amendment if required. No final parity/completion claim with a permanently unavailable release workflow; no new root, graph, key or writer held across fixed CLI is authorized here. Existing02.4 full data update and09 pull-only obligations remain unchanged. + ## Fixed source predicate and exemption Implement the predicate in the existing apps/ops-agent/src/coordinator-staged-ownership.ts/owned-services literal source owner. It uses the installed real runtime/role/journal and actual cache-only inventory plus indexed signature/digest/inode-verified staged sources. No public flag/result/callback can assert dependency absence. States are no dependency, required, unknown as private data only: normal cache-only ownership or conclusive absent original container makes dependency unnecessary; same OLD proven solely by staged source (running OR stopped) means required; failed/ambiguous acquisition or unproved image/config drift is unknown. Required and unknown refuse ordinary publication. A corrupt unneeded historical family need not override conclusive actual ordinary ownership; missing required proof never becomes false. From 8dbdc5bf58be3ead0ef4438924ecbb87ff2da7e3 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 08:18:47 +0100 Subject: [PATCH 034/135] feat(admin): enforce prepared configuration and service lifecycle --- apps/api/openapi.json | 1713 +++- apps/api/src/integration-config.ts | 112 +- apps/api/src/integration-host.ts | 48 +- .../__tests__/admin-integrations-sdk.test.ts | 347 +- .../__tests__/admin-services-sdk.test.ts | 158 +- apps/api/src/routes/admin-compose.ts | 14 +- .../src/routes/admin-integration-contract.ts | 5 +- .../routes/admin-integration-credentials.ts | 46 +- apps/api/src/routes/admin-integrations.ts | 105 +- apps/api/src/routes/admin-service-contract.ts | 5 +- apps/api/src/routes/admin-services.ts | 141 +- apps/api/src/server.ts | 4 +- .../__tests__/admin-job-handlers.test.ts | 57 + .../services/__tests__/admin-job-ops.test.ts | 20 +- .../__tests__/admin-ops-hardlinks.test.ts | 349 +- ...configuration-maintenance-postgres.test.ts | 8436 +++++++++++++++++ .../coordinator-lifecycle-postgres.test.ts | 3859 ++++++++ .../fixtures/native-configuration-api.ts | 111 + .../fixtures/native-coordinator-api.ts | 141 + .../fixtures/native-integration-import-api.ts | 91 + .../fixtures/postgres-commit-relay.ts | 182 + .../src/services/__tests__/ops-client.test.ts | 41 +- ...prospective-configuration-postgres.test.ts | 220 + .../prospective-configuration.test.ts | 104 + .../__tests__/service-preparation.test.ts | 116 + apps/api/src/services/admin-job-checkpoint.ts | 168 +- apps/api/src/services/admin-job-failure.ts | 18 + apps/api/src/services/admin-job-handlers.ts | 53 +- apps/api/src/services/admin-job-ops.ts | 393 +- apps/api/src/services/admin-job-parent.ts | 18 +- apps/api/src/services/admin-job-projection.ts | 337 + apps/api/src/services/admin-ops.ts | 266 +- .../administrative-configuration-context.ts | 66 + ...ministrative-configuration-writer-owner.ts | 369 + .../administrative-configuration-writer.ts | 2 + .../configuration-commit-participant.ts | 35 + .../src/services/configuration-hardlinks.ts | 58 + apps/api/src/services/configuration-input.ts | 81 + .../configuration-maintenance-decision.ts | 119 + ...onfiguration-maintenance-settle-records.ts | 24 + .../configuration-maintenance-step-records.ts | 205 + .../configuration-maintenance-step.ts | 177 + .../src/services/configuration-maintenance.ts | 9 + .../legacy-integration-import-intent.ts | 227 + .../legacy-integration-import.test.ts | 804 ++ .../legacy-integration-import.ts | 215 + .../api/src/services/coordinator-lifecycle.ts | 361 + apps/api/src/services/job-runner.ts | 452 +- apps/api/src/services/ops-client.ts | 10 +- .../src/services/prospective-configuration.ts | 5415 +++++++++++ apps/api/src/services/secrets.ts | 23 + .../src/services/service-config-resolver.ts | 62 + apps/api/src/services/service-preparation.ts | 228 + apps/api/src/services/service-secrets.ts | 29 +- apps/api/src/services/service-update-batch.ts | 243 + .../src/services/trusted-config-operations.ts | 373 +- .../src/services/trusted-config-outcome.ts | 42 +- .../trusted-config-publication-store.ts | 667 ++ .../services/trusted-config-publisher.test.ts | 34 + .../src/services/trusted-config-publisher.ts | 471 +- .../trusted-configuration-producer.ts | 337 + .../test/configuration-postgres-fixture.ts | 56 + apps/api/src/utils/docker-compose.ts | 30 +- .../src/__fixtures__/configuration-source.ts | 27 + .../src/__fixtures__/configuration-writer.mjs | 15 +- .../src/__fixtures__/owned-data-manager.ts | 152 + .../src/__fixtures__/owned-service-host.ts | 97 +- apps/ops-agent/src/administrative-runtime.ts | 77 +- apps/ops-agent/src/agent-handoff-main.ts | 3 + apps/ops-agent/src/bootstrap.test.ts | 7 +- .../src/configuration-candidate-source.ts | 104 + .../src/configuration-hardlinks-runtime.ts | 90 + ...iguration-maintenance-docker-inspection.ts | 269 + .../src/configuration-maintenance-runtime.ts | 20 + .../src/configuration-maintenance-store.ts | 5216 ++++++++++ .../configuration-writer-concurrency.test.ts | 4 + .../src/configured-resource-authority.ts | 57 + .../src/coordinator-staged-ownership.ts | 80 + apps/ops-agent/src/docker-runtime.test.ts | 174 +- apps/ops-agent/src/docker-runtime.ts | 127 +- apps/ops-agent/src/index.ts | 16 +- apps/ops-agent/src/journal.test.ts | 90 +- apps/ops-agent/src/journal.ts | 126 +- apps/ops-agent/src/owned-services.test.ts | 6 +- apps/ops-agent/src/owned-services.ts | 651 +- apps/ops-agent/src/policy.test.ts | 93 +- apps/ops-agent/src/policy.ts | 117 +- apps/ops-agent/src/production-runtime.test.ts | 471 +- apps/ops-agent/src/production-runtime.ts | 158 +- apps/ops-agent/src/runtime.ts | 1 + apps/ops-agent/src/server-foundation.test.ts | 19 +- apps/ops-agent/src/server-review-2.test.ts | 52 +- apps/ops-agent/src/server.ts | 1001 +- .../src/service-action-policy.test.ts | 5 +- apps/ops-agent/src/service-action-policy.ts | 25 +- .../src/service-lifecycle-runtime.ts | 145 + apps/ops-agent/src/service-pull-runtime.ts | 438 + apps/ops-agent/src/startup-authority.test.ts | 30 +- .../staged-coordinator-request-process.mjs | 220 + .../staged-coordinator-retention-process.mjs | 177 + .../terminal-retention-process.ts | 31 +- .../src/trusted-config-ownership.test.ts | 79 +- .../src/trusted-config-runtime.test.ts | 68 +- apps/ops-agent/src/trusted-config-runtime.ts | 319 +- .../src/trusted-config-source.test.ts | 4 + apps/ops-agent/src/trusted-config-source.ts | 41 +- apps/web/src/hooks/useServices.test.tsx | 14 + apps/web/src/hooks/useServices.ts | 37 +- .../src/lib/admin/integrationAdmin.test.tsx | 37 + apps/web/src/lib/admin/integrationAdmin.ts | 41 +- .../src/lib/admin/integrationErrors.test.ts | 42 + apps/web/src/lib/admin/integrationErrors.ts | 15 +- .../cli/__tests__/cli-control-context.test.ts | 55 +- .../__tests__/cli-service-lifecycle.test.ts | 319 + .../cli/__tests__/cli-services-update.test.ts | 130 +- packages/cli/src/commands/compose.ts | 233 +- packages/cli/src/commands/services.ts | 161 +- packages/cli/src/index.ts | 20 +- packages/cli/src/lib/control-context.ts | 4 + packages/cli/src/lib/release.ts | 51 +- packages/cli/src/lib/service-lifecycle.ts | 481 + .../api/contracts/integration-admin/common.ts | 3 + .../contracts/integration-admin/operations.ts | 27 +- .../src/api/contracts/service-admin/index.ts | 30 +- .../api/contracts/service-admin/services.ts | 10 +- .../admin/integration-admin.generated.ts | 59 +- .../deleteAdminIntegrationCredential409.ts | 20 + ...deleteAdminIntegrationCredentialHeaders.ts | 9 + .../disableAdminIntegration409.ts | 20 + .../disableAdminIntegrationHeaders.ts | 9 + .../enableAdminIntegration409.ts | 20 + .../enableAdminIntegrationHeaders.ts | 9 + .../importAdminIntegrationConfigs409.ts | 107 +- .../importAdminIntegrationConfigs409Apply.ts | 34 - ...tAdminIntegrationConfigs409Compensation.ts | 34 - ...mportAdminIntegrationConfigs409Database.ts | 12 - .../importAdminIntegrationConfigs409Error.ts | 9 - .../importAdminIntegrationConfigs409Phase.ts | 12 - .../importAdminIntegrationConfigs409Reason.ts | 10 - ...mportAdminIntegrationConfigs409Recovery.ts | 10 - ...importAdminIntegrationConfigs409Runtime.ts | 11 - .../importAdminIntegrationConfigsHeaders.ts | 9 + .../admin/integration-models/index.ts | 19 +- .../patchAdminIntegrationConfig409.ts | 20 + .../patchAdminIntegrationConfigHeaders.ts | 9 + .../setAdminIntegrationCredential409.ts | 20 + .../setAdminIntegrationCredentialHeaders.ts | 9 + .../admin/service-admin.generated.ts | 37 +- .../deleteAdminServiceCredential409.ts | 26 +- .../deleteAdminServiceCredential409Code.ts | 9 - .../deleteAdminServiceCredentialHeaders.ts | 9 + .../generated/admin/service-models/index.ts | 9 +- .../runAdminServiceActionBodyAction.ts | 1 + .../runAdminServicesBulkActionBody.ts | 12 +- .../setAdminServiceCredential409.ts | 26 +- .../setAdminServiceCredential409Code.ts | 9 - .../setAdminServiceCredentialHeaders.ts | 9 + .../setAdminServiceSelection409.ts | 20 + .../setAdminServiceSelectionHeaders.ts | 9 + .../service-models/startAdminCompose409.ts | 20 + .../updateAdminServiceConfig409.ts | 20 + .../updateAdminServiceConfigHeaders.ts | 9 + .../core/src/build-artifacts/child-phase.ts | 59 +- .../configuration-fence-settlement-budget.ts | 546 ++ .../configuration-fence-settlement-cells.ts | 8 + ...figuration-fence-settlement-descriptors.ts | 81 + .../configuration-fence-settlement-records.ts | 669 ++ ...onfiguration-fence-settlement-retention.ts | 2 + .../configuration-fence-settlement.test.ts | 46 + .../configuration-fence-settlement.ts | 726 ++ .../configuration-maintenance-budget.ts | 433 + .../configuration-maintenance-closure.ts | 2 + ...onfiguration-maintenance-docker-records.ts | 53 + ...onfiguration-maintenance-final-children.ts | 2 + ...configuration-maintenance-final-records.ts | 429 + ...nfiguration-maintenance-forward-records.ts | 43 + ...configuration-maintenance-owner-records.ts | 526 + .../build-artifacts/configuration-no-guard.ts | 462 + .../configuration-pre-effect.test.ts | 259 + .../configuration-source-witness.test.ts | 219 + .../configuration-source-witness.ts | 317 + .../src/build-artifacts/control-authority.ts | 1992 +++- .../build-artifacts/control-child-records.ts | 138 + .../src/build-artifacts/control-records.ts | 8 +- .../build-artifacts/control-retention.test.ts | 9 + .../coordinator-pull-records.ts | 130 + .../data-configuration-fence-records.ts | 41 + ...ata-configuration-fence-witness-records.ts | 30 + .../data-configuration-fence-witness.ts | 107 + .../data-configuration-fence.ts | 2044 +++- .../data-execution-candidate.ts | 427 +- .../build-artifacts/data-execution-journal.ts | 7 +- .../build-artifacts/feed-proxy-validation.ts | 429 +- .../fixed-configuration-maintenance.test.ts | 143 + .../fixed-configuration-maintenance.ts | 380 + .../fixed-service-lifecycle.ts | 172 + .../generation-hardlinks-owner.ts | 260 + .../generation-hardlinks-records.ts | 56 + packages/core/src/build-artifacts/index.ts | 91 +- .../src/build-artifacts/peer-process.test.ts | 65 +- .../src/build-artifacts/retention-family.ts | 3625 ++++++- .../src/build-artifacts/target-lease.test.ts | 67 +- .../core/src/build-artifacts/target-lease.ts | 1932 +++- packages/core/src/ops/api-job-attempt.ts | 16 + packages/core/src/ops/canonical-json.ts | 12 + packages/core/src/ops/client.test.ts | 60 +- packages/core/src/ops/client.ts | 2 + ...guration-maintenance-control-validation.ts | 46 + .../src/ops/configuration-maintenance.test.ts | 220 + .../core/src/ops/configuration-maintenance.ts | 196 + .../ops/configuration-operation-records.ts | 97 + .../ops/configuration-preparation-records.ts | 154 + .../core/src/ops/configuration-preparation.ts | 52 + packages/core/src/ops/contract.test.ts | 100 +- packages/core/src/ops/contract.ts | 190 +- .../core/src/ops/control-retention-receipt.ts | 6 +- .../src/ops/coordinator-lifecycle-records.ts | 59 + .../core/src/ops/execution-identity.test.ts | 6 + packages/core/src/ops/execution-identity.ts | 12 +- .../core/src/ops/feed-credential-map.test.ts | 41 + packages/core/src/ops/feed-credential-map.ts | 75 + packages/core/src/ops/fingerprint.ts | 13 +- .../core/src/ops/foundation-review.test.ts | 27 +- packages/core/src/ops/index.ts | 7 + .../core/src/ops/owned-service-records.ts | 13 + .../core/src/ops/service-operation-records.ts | 31 + packages/core/src/ops/service-pull.test.ts | 69 + packages/core/src/ops/service-results.ts | 9 +- packages/core/src/ops/trusted-config.test.ts | 76 + packages/core/src/ops/trusted-config.ts | 11 +- .../src/services/configuration-generation.ts | 116 +- .../coordinator-pull-reference.test.ts | 26 + .../services/coordinator-pull-reference.ts | 38 + .../src/services/creation-dependencies.ts | 73 + .../contracts/configuration-maintenance.ts | 13 +- .../data-manager/generated/models/index.ts | 2 +- .../settleDataConfigurationFenceBody.ts | 13 +- ...settleDataConfigurationFenceBodyOutcome.ts | 11 - ...settleDataConfigurationFenceBodyVersion.ts | 9 + packages/core/src/services/index.ts | 13 + .../src/services/release-image-selection.ts | 102 + .../service-lifecycle-receipt.test.ts | 278 + .../src/services/service-lifecycle-receipt.ts | 590 ++ packages/extension-contracts/src/service.ts | 20 + scripts/database-test-groups.json | 4 + scripts/ops-authority-inventory.json | 42 +- services/data-manager/openapi.json | 32 +- .../configuration-maintenance.test.ts | 57 + .../execution/configuration-maintenance.ts | 24 + .../src/jobs/execution/receiver.ts | 7 +- .../src/jobs/execution/runtime.ts | 11 +- .../data-manager/src/jobs/execution/store.ts | 26 +- services/ops-agent/service.json | 2 + 253 files changed, 58265 insertions(+), 4089 deletions(-) create mode 100644 apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts create mode 100644 apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts create mode 100644 apps/api/src/services/__tests__/fixtures/native-configuration-api.ts create mode 100644 apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts create mode 100644 apps/api/src/services/__tests__/fixtures/native-integration-import-api.ts create mode 100644 apps/api/src/services/__tests__/fixtures/postgres-commit-relay.ts create mode 100644 apps/api/src/services/__tests__/prospective-configuration-postgres.test.ts create mode 100644 apps/api/src/services/__tests__/prospective-configuration.test.ts create mode 100644 apps/api/src/services/__tests__/service-preparation.test.ts create mode 100644 apps/api/src/services/admin-job-projection.ts create mode 100644 apps/api/src/services/administrative-configuration-context.ts create mode 100644 apps/api/src/services/administrative-configuration-writer-owner.ts create mode 100644 apps/api/src/services/administrative-configuration-writer.ts create mode 100644 apps/api/src/services/configuration-commit-participant.ts create mode 100644 apps/api/src/services/configuration-hardlinks.ts create mode 100644 apps/api/src/services/configuration-input.ts create mode 100644 apps/api/src/services/configuration-maintenance-decision.ts create mode 100644 apps/api/src/services/configuration-maintenance-settle-records.ts create mode 100644 apps/api/src/services/configuration-maintenance-step-records.ts create mode 100644 apps/api/src/services/configuration-maintenance-step.ts create mode 100644 apps/api/src/services/configuration-maintenance.ts create mode 100644 apps/api/src/services/configuration-migration/legacy-integration-import-intent.ts create mode 100644 apps/api/src/services/configuration-migration/legacy-integration-import.test.ts create mode 100644 apps/api/src/services/configuration-migration/legacy-integration-import.ts create mode 100644 apps/api/src/services/coordinator-lifecycle.ts create mode 100644 apps/api/src/services/prospective-configuration.ts create mode 100644 apps/api/src/services/service-preparation.ts create mode 100644 apps/api/src/services/service-update-batch.ts create mode 100644 apps/api/src/services/trusted-config-publication-store.ts create mode 100644 apps/api/src/services/trusted-configuration-producer.ts create mode 100644 apps/api/src/test/configuration-postgres-fixture.ts create mode 100644 apps/ops-agent/src/__fixtures__/configuration-source.ts create mode 100644 apps/ops-agent/src/__fixtures__/owned-data-manager.ts create mode 100644 apps/ops-agent/src/configuration-candidate-source.ts create mode 100644 apps/ops-agent/src/configuration-hardlinks-runtime.ts create mode 100644 apps/ops-agent/src/configuration-maintenance-docker-inspection.ts create mode 100644 apps/ops-agent/src/configuration-maintenance-runtime.ts create mode 100644 apps/ops-agent/src/configuration-maintenance-store.ts create mode 100644 apps/ops-agent/src/configured-resource-authority.ts create mode 100644 apps/ops-agent/src/coordinator-staged-ownership.ts create mode 100644 apps/ops-agent/src/service-lifecycle-runtime.ts create mode 100644 apps/ops-agent/src/service-pull-runtime.ts create mode 100644 apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs create mode 100644 apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs create mode 100644 apps/web/src/lib/admin/integrationErrors.test.ts create mode 100644 packages/cli/__tests__/cli-service-lifecycle.test.ts create mode 100644 packages/cli/src/lib/service-lifecycle.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialHeaders.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/disableAdminIntegrationHeaders.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/enableAdminIntegrationHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Apply.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Compensation.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Database.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Error.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Phase.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Reason.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Recovery.ts delete mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Runtime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigsHeaders.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfigHeaders.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409Code.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredentialHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409Code.ts create mode 100644 packages/core/src/api/generated/admin/service-models/setAdminServiceCredentialHeaders.ts create mode 100644 packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/setAdminServiceSelectionHeaders.ts create mode 100644 packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/updateAdminServiceConfigHeaders.ts create mode 100644 packages/core/src/build-artifacts/configuration-fence-settlement-budget.ts create mode 100644 packages/core/src/build-artifacts/configuration-fence-settlement-cells.ts create mode 100644 packages/core/src/build-artifacts/configuration-fence-settlement-descriptors.ts create mode 100644 packages/core/src/build-artifacts/configuration-fence-settlement-records.ts create mode 100644 packages/core/src/build-artifacts/configuration-fence-settlement-retention.ts create mode 100644 packages/core/src/build-artifacts/configuration-fence-settlement.test.ts create mode 100644 packages/core/src/build-artifacts/configuration-fence-settlement.ts create mode 100644 packages/core/src/build-artifacts/configuration-maintenance-budget.ts create mode 100644 packages/core/src/build-artifacts/configuration-maintenance-closure.ts create mode 100644 packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts create mode 100644 packages/core/src/build-artifacts/configuration-maintenance-final-children.ts create mode 100644 packages/core/src/build-artifacts/configuration-maintenance-final-records.ts create mode 100644 packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts create mode 100644 packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts create mode 100644 packages/core/src/build-artifacts/configuration-no-guard.ts create mode 100644 packages/core/src/build-artifacts/configuration-pre-effect.test.ts create mode 100644 packages/core/src/build-artifacts/configuration-source-witness.test.ts create mode 100644 packages/core/src/build-artifacts/configuration-source-witness.ts create mode 100644 packages/core/src/build-artifacts/control-child-records.ts create mode 100644 packages/core/src/build-artifacts/coordinator-pull-records.ts create mode 100644 packages/core/src/build-artifacts/data-configuration-fence-records.ts create mode 100644 packages/core/src/build-artifacts/data-configuration-fence-witness-records.ts create mode 100644 packages/core/src/build-artifacts/data-configuration-fence-witness.ts create mode 100644 packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts create mode 100644 packages/core/src/build-artifacts/fixed-configuration-maintenance.ts create mode 100644 packages/core/src/build-artifacts/fixed-service-lifecycle.ts create mode 100644 packages/core/src/build-artifacts/generation-hardlinks-owner.ts create mode 100644 packages/core/src/build-artifacts/generation-hardlinks-records.ts create mode 100644 packages/core/src/ops/api-job-attempt.ts create mode 100644 packages/core/src/ops/canonical-json.ts create mode 100644 packages/core/src/ops/configuration-maintenance-control-validation.ts create mode 100644 packages/core/src/ops/configuration-maintenance.test.ts create mode 100644 packages/core/src/ops/configuration-maintenance.ts create mode 100644 packages/core/src/ops/configuration-operation-records.ts create mode 100644 packages/core/src/ops/configuration-preparation-records.ts create mode 100644 packages/core/src/ops/configuration-preparation.ts create mode 100644 packages/core/src/ops/coordinator-lifecycle-records.ts create mode 100644 packages/core/src/ops/feed-credential-map.test.ts create mode 100644 packages/core/src/ops/feed-credential-map.ts create mode 100644 packages/core/src/ops/owned-service-records.ts create mode 100644 packages/core/src/ops/service-operation-records.ts create mode 100644 packages/core/src/ops/service-pull.test.ts create mode 100644 packages/core/src/services/coordinator-pull-reference.test.ts create mode 100644 packages/core/src/services/coordinator-pull-reference.ts create mode 100644 packages/core/src/services/creation-dependencies.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts create mode 100644 packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyVersion.ts create mode 100644 packages/core/src/services/release-image-selection.ts create mode 100644 packages/core/src/services/service-lifecycle-receipt.test.ts create mode 100644 packages/core/src/services/service-lifecycle-receipt.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index bf032004f..81dbae41e 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -6994,6 +6994,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -9067,6 +9127,15 @@ "in": "path", "name": "key", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -9367,6 +9436,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -9686,6 +9815,15 @@ "in": "path", "name": "key", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -9979,7 +10117,7 @@ } } }, - "413": { + "409": { "description": "Response", "content": { "application/json": { @@ -10039,7 +10177,7 @@ } } }, - "415": { + "413": { "description": "Response", "content": { "application/json": { @@ -10099,40 +10237,7 @@ } } }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^\\d+$" - } - } - }, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "retryAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "error", - "retryAfter" - ], - "additionalProperties": false - } - } - } - }, - "500": { + "415": { "description": "Response", "content": { "application/json": { @@ -10191,137 +10296,230 @@ } } } - } - }, - "tags": [ - "integration-admin" - ], - "x-openmapx-auth": "admin" - } - }, - "/api/admin/dawarich": { - "get": { - "operationId": "getAdminDawarich", - "responses": { - "200": { + }, + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, "content": { "application/json": { "schema": { "type": "object", "properties": { - "installed": { - "type": "boolean" - }, - "selected": { - "type": "boolean" - }, - "running": { - "type": "boolean" - }, - "healthy": { - "type": "boolean" - }, - "publicOrigin": { - "type": [ - "string", - "null" - ] - }, - "oauthClient": { - "type": "object", - "properties": { - "present": { - "type": "boolean" - }, - "clientId": { - "type": [ - "string", - "null" - ] - }, - "redirectUriMatches": { - "type": "boolean" - }, - "settingsMatch": { - "type": "boolean" - }, - "recoveryRequired": { - "type": "boolean" - } - }, - "required": [ - "present", - "clientId", - "redirectUriMatches", - "settingsMatch", - "recoveryRequired" - ], - "additionalProperties": false - }, - "secrets": { - "type": "object", - "properties": { - "databasePassword": { - "type": "string", - "enum": [ - "missing", - "consistent", - "conflict" - ] - }, - "secretKeyBase": { - "type": "string", - "enum": [ - "missing", - "consistent", - "conflict" - ] - }, - "oidcClientSecret": { - "type": "string", - "enum": [ - "missing", - "consistent", - "conflict" - ] - } - }, - "required": [ - "databasePassword", - "secretKeyBase", - "oidcClientSecret" - ], - "additionalProperties": false - }, - "configReady": { - "type": "boolean" - }, - "readyToStart": { - "type": "boolean" + "error": { + "type": "string" }, - "needsApply": { - "type": "boolean" + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "required": [ - "installed", - "selected", - "running", - "healthy", - "publicOrigin", - "oauthClient", - "secrets", - "configReady", - "readyToStart", - "needsApply" + "error", + "retryAfter" ], "additionalProperties": false } } } }, - "400": { + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "integration-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/dawarich": { + "get": { + "operationId": "getAdminDawarich", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "installed": { + "type": "boolean" + }, + "selected": { + "type": "boolean" + }, + "running": { + "type": "boolean" + }, + "healthy": { + "type": "boolean" + }, + "publicOrigin": { + "type": [ + "string", + "null" + ] + }, + "oauthClient": { + "type": "object", + "properties": { + "present": { + "type": "boolean" + }, + "clientId": { + "type": [ + "string", + "null" + ] + }, + "redirectUriMatches": { + "type": "boolean" + }, + "settingsMatch": { + "type": "boolean" + }, + "recoveryRequired": { + "type": "boolean" + } + }, + "required": [ + "present", + "clientId", + "redirectUriMatches", + "settingsMatch", + "recoveryRequired" + ], + "additionalProperties": false + }, + "secrets": { + "type": "object", + "properties": { + "databasePassword": { + "type": "string", + "enum": [ + "missing", + "consistent", + "conflict" + ] + }, + "secretKeyBase": { + "type": "string", + "enum": [ + "missing", + "consistent", + "conflict" + ] + }, + "oidcClientSecret": { + "type": "string", + "enum": [ + "missing", + "consistent", + "conflict" + ] + } + }, + "required": [ + "databasePassword", + "secretKeyBase", + "oidcClientSecret" + ], + "additionalProperties": false + }, + "configReady": { + "type": "boolean" + }, + "readyToStart": { + "type": "boolean" + }, + "needsApply": { + "type": "boolean" + } + }, + "required": [ + "installed", + "selected", + "running", + "healthy", + "publicOrigin", + "oauthClient", + "secrets", + "configReady", + "readyToStart", + "needsApply" + ], + "additionalProperties": false + } + } + } + }, + "400": { "description": "Response", "content": { "application/json": { @@ -20420,6 +20618,17 @@ "/api/admin/integrations/import": { "post": { "operationId": "importAdminIntegrationConfigs", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], "requestBody": { "required": true, "content": { @@ -20736,333 +20945,390 @@ "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "error": { - "type": "string", - "enum": [ - "Integration import failed" - ] - }, - "phase": { - "type": "string", - "enum": [ - "persistence", - "apply", - "reload", - "ownership" - ] - }, - "recovery": { - "type": "string", - "enum": [ - "restored", - "reconciliation_required" - ] - }, - "database": { - "type": "string", - "enum": [ - "unknown", - "retained", - "conflict", - "restored" - ] - }, - "runtime": { - "type": "string", - "enum": [ - "unchanged", - "unknown", - "restored" - ] - }, - "apply": { - "anyOf": [ - { - "type": "object", - "properties": { - "outcome": { - "type": "string", - "enum": [ - "committed" - ] - }, - "revisionId": { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - "enabledServiceIds": { - "maxItems": 256, - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9][a-z0-9-]*$" - } - }, - "followUp": { - "type": "string", - "enum": [ - "complete", - "failed" - ] - }, - "cleanupFailed": { - "type": "boolean" - }, - "publisherCleanupFailed": { - "type": "boolean" - }, - "registryUpdateFailed": { - "type": "boolean" - } - }, - "required": [ - "outcome", - "revisionId", - "enabledServiceIds", - "followUp", - "cleanupFailed", - "publisherCleanupFailed", - "registryUpdateFailed" - ], - "additionalProperties": false + "anyOf": [ + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] }, - { - "type": "object", - "properties": { - "outcome": { - "type": "string", - "enum": [ - "not_committed" - ] - }, - "reason": { - "type": "string", - "enum": [ - "conflict", - "busy", - "failed" - ] - }, - "currentRevisionId": { - "anyOf": [ - { + "error": { + "type": "string", + "enum": [ + "Integration import failed" + ] + }, + "phase": { + "type": "string", + "enum": [ + "persistence", + "apply", + "reload", + "ownership" + ] + }, + "recovery": { + "type": "string", + "enum": [ + "restored", + "reconciliation_required" + ] + }, + "database": { + "type": "string", + "enum": [ + "unknown", + "retained", + "conflict", + "restored" + ] + }, + "runtime": { + "type": "string", + "enum": [ + "unchanged", + "unknown", + "restored" + ] + }, + "apply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { "type": "string", "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" }, - { - "type": "null" + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" } - ] - }, - "cleanupFailed": { - "type": "boolean" + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false }, - "publisherCleanupFailed": { - "type": "boolean" + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false }, - "registryUpdateFailed": { - "type": "boolean" + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false } - }, - "required": [ - "outcome", - "reason", - "cleanupFailed", - "publisherCleanupFailed", - "registryUpdateFailed" - ], - "additionalProperties": false + ] }, - { - "type": "object", - "properties": { - "outcome": { - "type": "string", - "enum": [ - "unknown" - ] - }, - "reason": { - "type": "string", - "enum": [ - "transport_or_result" - ] + "compensation": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false }, - "publisherCleanupFailed": { - "type": "boolean" + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false }, - "registryUpdateFailed": { - "type": "boolean" + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false } - }, - "required": [ - "outcome", - "reason", - "publisherCleanupFailed", - "registryUpdateFailed" - ], - "additionalProperties": false + ] + }, + "reason": { + "type": "string", + "enum": [ + "submitted_snapshot_not_owned", + "previous_applied_not_proven" + ] } - ] + }, + "required": [ + "ok", + "error", + "phase", + "recovery", + "database" + ], + "additionalProperties": false }, - "compensation": { + { "anyOf": [ { "type": "object", "properties": { - "outcome": { - "type": "string", - "enum": [ - "committed" - ] - }, - "revisionId": { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - "enabledServiceIds": { - "maxItems": 256, - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9][a-z0-9-]*$" - } - }, - "followUp": { - "type": "string", - "enum": [ - "complete", - "failed" - ] - }, - "cleanupFailed": { - "type": "boolean" - }, - "publisherCleanupFailed": { - "type": "boolean" - }, - "registryUpdateFailed": { - "type": "boolean" - } - }, - "required": [ - "outcome", - "revisionId", - "enabledServiceIds", - "followUp", - "cleanupFailed", - "publisherCleanupFailed", - "registryUpdateFailed" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "outcome": { - "type": "string", - "enum": [ - "not_committed" - ] + "error": { + "type": "string" }, - "reason": { - "type": "string", + "ok": { + "type": "boolean", "enum": [ - "conflict", - "busy", - "failed" - ] - }, - "currentRevisionId": { - "anyOf": [ - { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - { - "type": "null" - } + false ] }, - "cleanupFailed": { - "type": "boolean" - }, - "publisherCleanupFailed": { - "type": "boolean" + "message": { + "type": "string" }, - "registryUpdateFailed": { - "type": "boolean" + "jobId": { + "type": "string" } }, "required": [ - "outcome", - "reason", - "cleanupFailed", - "publisherCleanupFailed", - "registryUpdateFailed" + "error" ], "additionalProperties": false }, { "type": "object", "properties": { - "outcome": { - "type": "string", - "enum": [ - "unknown" - ] + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, - "reason": { - "type": "string", - "enum": [ - "transport_or_result" - ] + "error": { + "type": "string" }, - "publisherCleanupFailed": { - "type": "boolean" + "message": { + "type": "string" }, - "registryUpdateFailed": { - "type": "boolean" + "code": { + "type": "string" } }, "required": [ - "outcome", - "reason", - "publisherCleanupFailed", - "registryUpdateFailed" + "statusCode", + "error", + "message" ], "additionalProperties": false } ] - }, - "reason": { - "type": "string", - "enum": [ - "submitted_snapshot_not_owned", - "previous_applied_not_proven" - ] } - }, - "required": [ - "ok", - "error", - "phase", - "recovery", - "database" - ], - "additionalProperties": false + ] } } } @@ -23625,6 +23891,15 @@ "in": "path", "name": "id", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -23950,6 +24225,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -24182,6 +24517,15 @@ "in": "path", "name": "id", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -24475,6 +24819,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -24707,6 +25111,15 @@ "in": "path", "name": "id", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -25000,6 +25413,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -36855,7 +37328,33 @@ "enum": [ "start", "stop", - "restart", + "restart" + ] + }, + "serviceIds": { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "continueOnError": { + "type": "boolean" + } + }, + "required": [ + "action", + "serviceIds" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ "update" ] }, @@ -36867,6 +37366,9 @@ "minLength": 1 } }, + "noDeps": { + "type": "boolean" + }, "continueOnError": { "type": "boolean" } @@ -39221,6 +39723,17 @@ }, "put": { "operationId": "setAdminServiceSelection", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], "requestBody": { "required": true, "content": { @@ -39399,7 +39912,67 @@ } } }, - "401": { + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { "description": "Response", "content": { "application/json": { @@ -39459,7 +40032,7 @@ } } }, - "403": { + "409": { "description": "Response", "content": { "application/json": { @@ -41291,7 +41864,8 @@ "enum": [ "start", "stop", - "restart" + "restart", + "apply" ] } }, @@ -42404,6 +42978,15 @@ "in": "path", "name": "id", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -42715,6 +43298,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -43545,6 +44188,15 @@ "in": "path", "name": "key", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -43865,23 +44517,80 @@ "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "code": { - "type": "string", - "enum": [ - "DAWARICH_CREDENTIAL_MANAGED" - ] + "anyOf": [ + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "DAWARICH_CREDENTIAL_MANAGED" + ] + }, + "error": { + "type": "string" + } + }, + "required": [ + "code", + "error" + ], + "additionalProperties": false }, - "error": { - "type": "string" + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] } - }, - "required": [ - "code", - "error" - ], - "additionalProperties": false + ] } } } @@ -44125,6 +44834,15 @@ "in": "path", "name": "key", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -44438,23 +45156,80 @@ "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "code": { - "type": "string", - "enum": [ - "DAWARICH_CREDENTIAL_MANAGED" - ] + "anyOf": [ + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "DAWARICH_CREDENTIAL_MANAGED" + ] + }, + "error": { + "type": "string" + } + }, + "required": [ + "code", + "error" + ], + "additionalProperties": false }, - "error": { - "type": "string" + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] } - }, - "required": [ - "code", - "error" - ], - "additionalProperties": false + ] } } } diff --git a/apps/api/src/integration-config.ts b/apps/api/src/integration-config.ts index 77891c0fd..e914d101e 100644 --- a/apps/api/src/integration-config.ts +++ b/apps/api/src/integration-config.ts @@ -1,10 +1,17 @@ -import { existsSync, readFileSync } from "node:fs"; +import { constants, existsSync, readFileSync } from "node:fs"; +import { lstat, open } from "node:fs/promises"; import { join } from "node:path"; +import { services as coreServices } from "@openmapx/core/server"; import { type IntegrationManifest, integrationEnvVarName } from "@openmapx/integration-framework"; import { eq } from "drizzle-orm"; import { db } from "./db"; import { integrationConfig } from "./db/schema"; -import { resolveVaultSecrets } from "./services/secrets"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, + recordAdministrativeConfigurationFile, +} from "./services/administrative-configuration-context"; +import { resolveIntegrationVaultSecretsStrict, resolveVaultSecrets } from "./services/secrets"; export type ConfigSource = "default" | "database" | "vault" | "config.json" | "env"; @@ -73,10 +80,10 @@ export async function resolveConfigWithSources( ): Promise> { const result: Record = {}; const schema = manifest.configSchema as Record | undefined; - const knownKeys = new Set(); + const knownKeys = new Set(["enabled"]); // Canonical key → JSON-schema declared type. Used to coerce env-string values // to the type the integration expects (env vars are always strings). - const keyTypes = new Map(); + const keyTypes = new Map([["enabled", "boolean"]]); if (schema) { const props = (schema.properties ?? schema) as Record< @@ -164,6 +171,103 @@ export async function resolveConfig( return config; } +/** Strict administrative cascade. ENOENT alone means an absent config file. */ +export async function resolveIntegrationConfigWithSourcesStrict( + manifest: IntegrationManifest, + directory: string, + context: AdministrativeConfigurationReadContext, +): Promise> { + try { + assertAdministrativeConfigurationRead(context); + if (context.integrationReloadSnapshot) { + const retained = context.integrationReloadSnapshot[manifest.id]; + if (!retained) throw Error("Captured integration configuration unavailable"); + return structuredClone(retained); + } + const properties: Record = { + ...((manifest.configSchema?.properties ?? manifest.configSchema ?? {}) as Record< + string, + { default?: unknown; type?: string } + >), + enabled: { type: "boolean" }, + }; + const result: Record = {}; + for (const [key, definition] of Object.entries(properties)) { + if (definition && typeof definition === "object" && definition.default !== undefined) + result[key] = { value: definition.default, source: "default" }; + } + const rows = context.integrationRows.filter((row) => row.integrationId === manifest.id); + if (rows.length > 1) throw new Error("duplicate config"); + const overlay = (values: unknown, source: ConfigSource) => { + if (!values || typeof values !== "object" || Array.isArray(values)) + throw new Error("malformed config"); + for (const [key, value] of Object.entries(values)) { + if (!Object.hasOwn(properties, key)) throw new Error("undeclared config"); + if (key === "enabled" && typeof value !== "boolean") throw new Error("malformed enabled"); + result[key] = { value, source }; + } + }; + overlay(rows[0]?.config ?? {}, "database"); + overlay(await resolveIntegrationVaultSecretsStrict(manifest.id, context), "vault"); + const path = join(directory, "config.json"); + let handle: Awaited> | undefined; + try { + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const stat = await handle.stat(); + if (!stat.isFile() || stat.nlink !== 1 || stat.size > 524288) + throw new Error("unsafe config"); + const bytes = await handle.readFile(); + const after = await handle.stat(); + const current = await lstat(path); + if ( + stat.dev !== current.dev || + stat.ino !== current.ino || + stat.size !== after.size || + stat.mtimeMs !== after.mtimeMs || + stat.ctimeMs !== after.ctimeMs + ) + throw new Error("changed config"); + recordAdministrativeConfigurationFile(context, path, { + dev: stat.dev, + ino: stat.ino, + size: stat.size, + mtime: stat.mtimeMs, + ctime: stat.ctimeMs, + }); + overlay(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), "config.json"); + } catch (error) { + if (handle !== undefined || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + recordAdministrativeConfigurationFile(context, path, null); + } finally { + await handle?.close(); + } + for (const [key, definition] of Object.entries(properties)) { + const value = context.env[integrationEnvVarName(manifest.id, key)]; + if (isEnvValuePresent(value)) + result[key] = { value: coerceEnvValue(value, definition?.type), source: "env" }; + } + for (const [key, entry] of Object.entries(result)) { + // This is acquisition/type validation. Writability/secret ownership is + // checked separately against the original definition by the overlay. + const valueSchema = Object.fromEntries( + Object.entries(properties[key] ?? {}).filter( + ([annotation]) => annotation !== "readOnly" && annotation !== "x-openmapx-secret", + ), + ); + if ( + !coreServices.validateTrustedConfigurationValues( + { [key]: entry.value }, + { type: "object", properties: { [key]: valueSchema } }, + ) + ) + throw Error("Malformed integration configuration value"); + } + return result; + } catch { + throw new Error("Configuration acquisition unavailable"); + } +} + /** * Emits advisory warnings for required/enum config keys that violate the * manifest's `configSchema`. Never blocks load. Shared by cold start and reload. diff --git a/apps/api/src/integration-host.ts b/apps/api/src/integration-host.ts index 19e6e9598..175812f6e 100644 --- a/apps/api/src/integration-host.ts +++ b/apps/api/src/integration-host.ts @@ -41,6 +41,7 @@ import { type ConfigValueWithSource, resolveConfig, resolveConfigWithSources, + resolveIntegrationConfigWithSourcesStrict, warnInvalidConfig, } from "./integration-config"; import { @@ -52,6 +53,10 @@ import { rollbackIntegrationRouteStaging, } from "./integration-routes"; import { redis } from "./redis"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./services/administrative-configuration-context"; import { AttributionIndex, defaultMotisLicenseFile, @@ -1146,7 +1151,19 @@ let pendingReload: Promise | null = null; * files is guaranteed a discover+setup pass starting at or after its call, * while N concurrent callers cause at most one queued rebuild. */ -export async function reloadIntegrations(): Promise { +export async function reloadIntegrations( + context?: AdministrativeConfigurationReadContext, +): Promise { + if (context) { + assertAdministrativeConfigurationRead(context); + // Each committed candidate receives its own exact pass. Coalescing it with + // another caller's snapshot could publish a different desired generation. + while (activeReload) await activeReload.catch(() => undefined); + activeReload = doReloadIntegrations(context).finally(() => { + activeReload = null; + }); + return activeReload; + } if (!activeReload) { activeReload = doReloadIntegrations().finally(() => { activeReload = null; @@ -1170,8 +1187,11 @@ export async function reloadIntegrations(): Promise { * previous generation. A failed rebuild is discarded without disturbing the * integrations that are currently serving requests. */ -async function doReloadIntegrations(): Promise { +async function doReloadIntegrations( + context?: AdministrativeConfigurationReadContext, +): Promise { if (!_fastify) throw new Error("Integration host not initialized"); + if (context) assertAdministrativeConfigurationRead(context); const previousCount = integrations.size; const previousIntegrations = Array.from(integrations.values()); @@ -1187,12 +1207,26 @@ async function doReloadIntegrations(): Promise { // Re-discover and re-setup (topological sort by dependencies, same as cold start). const discovered = await discoverManifests(_integrationDirs); const sorted = topologicalSort(discovered); + if ( + context && + (sorted.length !== context.integrations.length || + sorted.some((item) => { + const captured = context.integrations.find((i) => i.id === item.manifest.id); + return ( + !captured || + captured.directory !== item.directory || + JSON.stringify(captured.manifest) !== JSON.stringify(item.manifest) + ); + })) + ) + throw Error("Captured integration registry changed"); // Reload capability bindings and service registry for requires: resolution. let reloadBindings = new Map>(); try { reloadBindings = await loadAllBindingsByIntegration(); } catch { + if (context) throw Error("Captured integration bindings unavailable"); _fastify.log.debug("Capability bindings unavailable during reload"); } @@ -1201,7 +1235,9 @@ async function doReloadIntegrations(): Promise { try { reloadRegistry = getServiceRegistry(); reloadServices = reloadRegistry.list(); + if (context) reloadServices = [...context.services]; } catch { + if (context) throw Error("Captured service registry unavailable"); _fastify.log.debug("Service registry unavailable during reload"); } @@ -1240,7 +1276,13 @@ async function doReloadIntegrations(): Promise { } } - const config = await resolveConfig(manifest, directory); + const config = context + ? Object.fromEntries( + Object.entries( + await resolveIntegrationConfigWithSourcesStrict(manifest, directory, context), + ).map(([key, entry]) => [key, entry.value]), + ) + : await resolveConfig(manifest, directory); const log = createLogger(id, _fastify); const http = createHttpClient(log); const cache = createCacheClient(id); diff --git a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts index 4870112ed..c2267a7b0 100644 --- a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts @@ -1,6 +1,5 @@ import { createApiClient } from "@openmapx/core"; import { integrationAdminOperations } from "@openmapx/core/api-contracts"; -import { configurationProjectionFingerprint } from "@openmapx/core/ops"; import { licensingFromDeclaration } from "@openmapx/mobility-core/licensing"; import Fastify from "fastify"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -18,10 +17,12 @@ const { requireAdmin: mockRequireAdmin, session: fakeSession } = installAdminRou // Job runner const mockJobRunnerEnqueue = vi.fn().mockResolvedValue("job-main-123"); const mockJobRunnerCancel = vi.fn().mockResolvedValue(true); +const mockImmediateImport = vi.fn(); vi.mock("../../services/job-runner.js", () => ({ jobRunner: { enqueue: (...args: unknown[]) => mockJobRunnerEnqueue(...args), cancel: (...args: unknown[]) => mockJobRunnerCancel(...args), + importImmediateIntegrationConfiguration: (...args: unknown[]) => mockImmediateImport(...args), }, })); @@ -229,7 +230,18 @@ beforeEach(() => { }, }); mockCompensate.mockReset().mockRejectedValue(Error("Must not compensate")); - mockApply.mockReset().mockResolvedValue({ revisionId: "revision_1" }); + mockApply.mockReset().mockResolvedValue({ + revisionId: "revision_1", + integrationReload: { message: "Integrations reloaded", reloaded: 1, enabled: 1 }, + }); + mockImmediateImport.mockReset().mockImplementation(async ({ body }) => ({ + ok: true, + imported: body.integrations.length, + skipped: [], + message: "Integrations reloaded", + reloaded: 1, + enabled: 1, + })); mockRequireAdmin.mockReset().mockResolvedValue(fakeSession); mockGetIntegration.mockReset().mockReturnValue(MOCK_INTEGRATION); mockGetAllIntegrations.mockReset().mockReturnValue([MOCK_INTEGRATION]); @@ -306,13 +318,32 @@ describe("actual integration producers through generated SDK", () => { () => sdk.importAdminIntegrationConfigs( { integrations: [{ integrationId: id, config: { label: "new" } }] }, + { "idempotency-key": crypto.randomUUID() }, options, ), () => sdk.runAdminIntegrationHealthChecks(undefined, options), () => sdk.reloadAdminIntegrations(undefined, options), - () => sdk.patchAdminIntegrationConfig(id, { label: "new" }, options), - () => sdk.enableAdminIntegration(id, undefined, options), - () => sdk.disableAdminIntegration(id, undefined, options), + () => + sdk.patchAdminIntegrationConfig( + id, + { label: "new" }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), + () => + sdk.enableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + options, + ), + () => + sdk.disableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + options, + ), () => sdk.getAdminIntegrationHealth(id, options), () => sdk.getAdminIntegrationHealthHistory(id, undefined, options), () => sdk.reloadAdminIntegration(id, undefined, options), @@ -337,9 +368,17 @@ describe("actual integration producers through generated SDK", () => { id, "apiKey", { value: "synthetic-credential" }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), + () => + sdk.deleteAdminIntegrationCredential( + id, + "apiKey", + { "idempotency-key": crypto.randomUUID() }, + undefined, options, ), - () => sdk.deleteAdminIntegrationCredential(id, "apiKey", undefined, options), () => sdk.getAdminOverview(options), ]; for (const call of cases) @@ -361,65 +400,61 @@ describe("actual integration producers through generated SDK", () => { } }); it.each(["patch", "enable", "disable"] as const)( - "merges overlapping ordinary %s with a completed import at the locked row", + "forwards ordinary %s as a finite candidate overlay without route-local writes", async (kind) => { - const sdk = await import("@openmapx/core/api-admin-sdk"); - const { app, client } = await wire(); + const sdk = await import("@openmapx/core/api-admin-sdk"), + { app, client } = await wire(); const id = MOCK_INTEGRATION.id; - const initialEnabled = kind === "disable"; - configDb.reset([ - { - id: "row-a", - integrationId: id, - config: { label: "old", nested: { count: 1 }, enabled: initialEnabled }, - version: "0", - }, - ]); - const reached = Promise.withResolvers(); - const resume = Promise.withResolvers(); - // Before the row lock: the old handler has already read/merged stale data. - // A corrected handler carries only intent into this same transaction boundary. - configDb.beforeTransaction = async () => { - reached.resolve(); - await resume.promise; - }; - const ordinary = - kind === "patch" - ? sdk.patchAdminIntegrationConfig(id, { nested: { count: 9 } }, { client }) - : kind === "enable" - ? sdk.enableAdminIntegration(id, undefined, { client }) - : sdk.disableAdminIntegration(id, undefined, { client }); try { - await reached.promise; - expect( - await sdk.importAdminIntegrationConfigs( - { - integrations: [ + const response = + kind === "patch" + ? await sdk.patchAdminIntegrationConfig( + id, + { nested: { count: 9 } }, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ) + : kind === "enable" + ? await sdk.enableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ) + : await sdk.disableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ); + expect(response).toMatchObject({ status: 200, data: { ok: true } }); + expect(mockApply).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + kind: "integrationConfig.apply", + integrationId: id, + overlay: { + purpose: "integration-config", + coordinatorMode: "ensure-running", + changes: [ { + kind: "integration-config", integrationId: id, - config: { label: "imported", nested: { count: 2 }, enabled: initialEnabled }, + set: kind === "patch" ? { nested: { count: 9 } } : { enabled: kind === "enable" }, + unset: [], }, ], }, - { client }, - ), - ).toMatchObject({ status: 200, data: { ok: true, imported: 1 } }); - resume.resolve(); - expect(await ordinary).toMatchObject({ status: 200, data: { ok: true } }); - expect(configDb.rows[0].config).toEqual({ - label: "imported", - nested: { count: kind === "patch" ? 9 : 2 }, - enabled: kind === "patch" ? initialEnabled : kind === "enable", - }); + }), + ); + expect(configDb.writes).toBe(0); + expect(mockReloadIntegrations).not.toHaveBeenCalled(); } finally { - resume.resolve(); - await ordinary; await app.close(); } }, ); it.each(["patch", "enable", "disable"] as const)( - "does not let an older ordinary %s rollback overwrite a completed import", + "does not perform a route-local inverse after an older ordinary %s producer fails", async (kind) => { const sdk = await import("@openmapx/core/api-admin-sdk"); const { app, client } = await wire(); @@ -447,21 +482,38 @@ describe("actual integration producers through generated SDK", () => { }); const ordinary = kind === "patch" - ? sdk.patchAdminIntegrationConfig(id, { nested: { count: 9 } }, { client }) + ? sdk.patchAdminIntegrationConfig( + id, + { nested: { count: 9 } }, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ) : kind === "enable" - ? sdk.enableAdminIntegration(id, undefined, { client }) - : sdk.disableAdminIntegration(id, undefined, { client }); + ? sdk.enableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ) + : sdk.disableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ); try { await reached.promise; expect( await sdk.importAdminIntegrationConfigs( { integrations: [{ integrationId: id, config: { label: "imported", enabled: true } }] }, + { "idempotency-key": crypto.randomUUID() }, { client }, ), ).toMatchObject({ status: 200 }); resume.resolve(); expect(await ordinary).toMatchObject({ status: 500 }); - expect(configDb.rows[0].config).toEqual({ label: "imported", enabled: true }); + expect(configDb.rows[0].config).toEqual({ label: "old", enabled: kind === "disable" }); + expect(configDb.writes).toBe(0); } finally { resume.resolve(); await ordinary; @@ -550,10 +602,21 @@ describe("actual integration producers through generated SDK", () => { expect(limited.headers.get("retry-after")).toMatch(/^\d+$/); mockIsSecretsConfigured.mockReturnValueOnce(false); expect( - await sdk.setAdminIntegrationCredential(id, "apiKey", { value: "synthetic" }, { client }), + await sdk.setAdminIntegrationCredential( + id, + "apiKey", + { value: "synthetic" }, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ), ).toMatchObject({ status: 503, data: { hint: expect.any(String) } }); - mockReloadIntegrations.mockRejectedValueOnce(Error("synthetic-secret")); - const failed = await sdk.enableAdminIntegration(id, undefined, { client }); + mockApply.mockRejectedValueOnce(Error("synthetic-secret")); + const failed = await sdk.enableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ); expect(failed.status).toBe(500); expect(JSON.stringify(failed)).not.toContain("synthetic-secret"); const malformed = await app.inject({ @@ -611,6 +674,7 @@ describe("actual integration producers through generated SDK", () => { }, ], }, + { "idempotency-key": crypto.randomUUID() }, { client }, ); expect(response.status).toBe(400); @@ -635,71 +699,31 @@ describe("actual integration producers through generated SDK", () => { await app.close(); } }); - describe.each(["incomplete", "reload"] as const)("unproven prior state after %s", (failure) => { - it.each(["conflict", "unknown", "retained"] as const)( - "carries %s through actual import, generated SDK and browser formatter", - async (database) => { - const sdk = await import("@openmapx/core/api-admin-sdk"); - const { integrationErrorMessage } = await import( - "../../../../web/src/lib/admin/integrationErrors" - ); - const { app, client } = await wire(); - const id = MOCK_INTEGRATION.id; - configDb.reset([ - { id: "row-alpha", integrationId: id, config: { label: "saved-before" }, version: "0" }, - ]); - const settleFailure = () => { - if (database === "conflict") { - configDb.rows[0].config = { label: "competing-edit" }; - configDb.rows[0].version = "competing-version"; - } - if (database === "unknown") - mockDbSelect.mockImplementationOnce(() => { - throw Error("synthetic ownership failure"); - }); - }; - const outcome = { - outcome: "committed" as const, - revisionId: `cfg1_${"b".repeat(43)}`, - enabledServiceIds: [], - followUp: failure === "incomplete" ? ("failed" as const) : ("complete" as const), - cleanupFailed: false, - publisherCleanupFailed: false, - registryUpdateFailed: false, - }; - mockApplyWithEvidence.mockImplementationOnce(async () => { - if (failure === "incomplete") settleFailure(); - return { - payload: { - domain: "fixture.invalid", - serviceConfigs: [], - serviceSecrets: [], - integrationConfigs: [{ integrationId: id, values: { label: "imported" } }], - }, - previousApplied: { - revisionId: `cfg1_${"a".repeat(43)}`, - integrationConfigDigests: [ - { - integrationId: id, - digest: configurationProjectionFingerprint({ label: "live-before" }), - }, - ], - }, - outcome, - }; - }); - if (failure === "reload") - mockReloadIntegrations.mockImplementationOnce(async () => { - settleFailure(); - throw Error("synthetic reload failure"); - }); - try { - const response = await sdk.importAdminIntegrationConfigs( - { integrations: [{ integrationId: id, config: { label: "imported" } }] }, - { client }, + describe.each(["incomplete", "reload"] as const)( + "retained failure projection after %s", + (failure) => { + it.each(["conflict", "unknown", "retained"] as const)( + "carries retained %s evidence through the registered route, generated SDK and browser formatter", + async (database) => { + const sdk = await import("@openmapx/core/api-admin-sdk"); + const { integrationErrorMessage } = await import( + "../../../../web/src/lib/admin/integrationErrors" ); - expect(response.status).toBe(409); - expect(response.data).toEqual({ + const { app, client } = await wire(); + const id = MOCK_INTEGRATION.id; + configDb.reset([ + { id: "row-alpha", integrationId: id, config: { label: "saved-before" }, version: "0" }, + ]); + const outcome = { + outcome: "committed" as const, + revisionId: `cfg1_${"b".repeat(43)}`, + enabledServiceIds: [], + followUp: failure === "incomplete" ? ("failed" as const) : ("complete" as const), + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + mockImmediateImport.mockResolvedValueOnce({ ok: false, error: "Integration import failed", recovery: "reconciliation_required", @@ -709,35 +733,52 @@ describe("actual integration producers through generated SDK", () => { runtime: failure === "incomplete" ? "unchanged" : "unknown", apply: outcome, }); - let displayed = ""; try { - sdk.unwrapAdminResponse(response); - } catch (error) { - displayed = integrationErrorMessage(error, "Import failed"); - } - if (database === "retained") - expect(displayed).toContain("The imported configuration is saved."); - else { - expect(displayed).toContain( - database === "conflict" - ? "newer edits were preserved" - : "The saved configuration is uncertain.", + const response = await sdk.importAdminIntegrationConfigs( + { integrations: [{ integrationId: id, config: { label: "imported" } }] }, + { "idempotency-key": crypto.randomUUID() }, + { client }, ); - expect(displayed).not.toContain("The imported configuration is saved."); + expect(response.status).toBe(409); + expect(response.data).toEqual({ + ok: false, + error: "Integration import failed", + recovery: "reconciliation_required", + reason: "previous_applied_not_proven", + database, + phase: failure === "incomplete" ? "apply" : "reload", + runtime: failure === "incomplete" ? "unchanged" : "unknown", + apply: outcome, + }); + let displayed = ""; + try { + sdk.unwrapAdminResponse(response); + } catch (error) { + displayed = integrationErrorMessage(error, "Import failed"); + } + if (database === "retained") + expect(displayed).toContain("The imported configuration is saved."); + else { + expect(displayed).toContain( + database === "conflict" + ? "newer edits were preserved" + : "The saved configuration is uncertain.", + ); + expect(displayed).not.toContain("The imported configuration is saved."); + } + expect(configDb.rows[0].config).toEqual({ label: "saved-before" }); + expect(configDb.writes).toBe(0); + expect(mockImmediateImport).toHaveBeenCalledTimes(1); + expect(mockCompensate).not.toHaveBeenCalled(); + expect(mockApplyWithEvidence).not.toHaveBeenCalled(); + expect(mockReloadIntegrations).not.toHaveBeenCalled(); + } finally { + await app.close(); } - expect(configDb.rows[0].config).toEqual({ - label: database === "conflict" ? "competing-edit" : "imported", - }); - expect(configDb.writes).toBe(1); - expect(mockCompensate).not.toHaveBeenCalled(); - expect(mockApplyWithEvidence).toHaveBeenCalledTimes(1); - expect(mockReloadIntegrations).toHaveBeenCalledTimes(failure === "reload" ? 1 : 0); - } finally { - await app.close(); - } - }, - ); - }); + }, + ); + }, + ); it("validates the whole import before writes and carries explicit reconciliation evidence over generated transport", async () => { const sdk = await import("@openmapx/core/api-admin-sdk"); const { app, client } = await wire(); @@ -768,9 +809,17 @@ describe("actual integration producers through generated SDK", () => { ).toBe(400); } expect(configDb.writes).toBe(0); - configDb.failWrite = true; + mockImmediateImport.mockResolvedValueOnce({ + ok: false, + error: "Integration import failed", + phase: "persistence", + recovery: "reconciliation_required", + database: "unknown", + runtime: "unchanged", + }); const failed = await sdk.importAdminIntegrationConfigs( { integrations: [{ integrationId: "geocoding-maptiler", config: {} }] }, + { "idempotency-key": crypto.randomUUID() }, { client }, ); expect(failed).toMatchObject({ diff --git a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts index cdaa9d35d..eeac40a39 100644 --- a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts @@ -69,10 +69,12 @@ const registry = vi.fn(() => ({ list: () => [service], get: getService })); vi.mock("../../services/service-registry", () => ({ getServiceRegistry: () => registry() })); const enqueue = vi.fn().mockResolvedValue("job-accepted"); const cancelJob = vi.fn(); +const composeStart = vi.fn(); vi.mock("../../services/job-runner", () => ({ jobRunner: { enqueue: (...args: unknown[]) => enqueue(...args), cancel: (...args: unknown[]) => cancelJob(...args), + startImmediateCompose: (...args: unknown[]) => composeStart(...args), }, })); const apply = vi.fn().mockResolvedValue({ revisionId: "revision_1" }); @@ -279,12 +281,21 @@ vi.mock("../../services/ops-client", async (actual) => ({ })); const sdk = await import("@openmapx/core/api-admin-sdk"); +const executeDefault = execute.getMockImplementation()!; const { adminServicesRoute } = await import("../admin-services"); const { adminOperationsRoute } = await import("../admin-operations"); const { adminDawarichRoute } = await import("../admin-dawarich"); const { registerAdminComposeRoutes } = await import("../admin-compose"); beforeEach(() => { vi.clearAllMocks(); + apply + .mockReset() + .mockResolvedValue({ revisionId: `cfg1_${"a".repeat(43)}`, jobId: "job-accepted" }); + execute.mockReset().mockImplementation(executeDefault); + composeStart + .mockReset() + .mockResolvedValue({ ok: true, stdout: "", hardlinks: serviceHardlinksFixture }); + dockerAvailable.mockReset().mockResolvedValue(true); internalFailures.length = 0; restore.mockReset().mockResolvedValue(true); persist.mockReset().mockImplementation(async (...args: unknown[]) => { @@ -344,7 +355,7 @@ async function wire(max = 1000) { describe("actual service producers through final exported SDK", () => { it.each([null, { count: 1 }])( - "preserves a newer request while an older noncommit fails (prior %j)", + "forwards overlapping intents without route-local desired writes or inverse compensation (prior %j)", async (prior) => { storedConfig = prior; let version = 0; @@ -377,15 +388,40 @@ describe("actual service producers through final exported SDK", () => { }); }); const { options, close } = await wire(); - const older = sdk.updateAdminServiceConfig("motis", { config: { count: 3 } }, options); + const older = sdk.updateAdminServiceConfig( + "motis", + { config: { count: 3 } }, + { "idempotency-key": crypto.randomUUID() }, + options, + ); try { await vi.waitFor(() => expect(apply).toHaveBeenCalledTimes(1), { timeout: 2000 }); expect( - (await sdk.updateAdminServiceConfig("motis", { config: { count: 4 } }, options)).status, + ( + await sdk.updateAdminServiceConfig( + "motis", + { config: { count: 4 } }, + { "idempotency-key": crypto.randomUUID() }, + options, + ) + ).status, ).toBe(200); resume.resolve(); expect((await older).status).toBe(500); - expect(storedConfig).toEqual({ count: 4 }); + expect(storedConfig).toEqual(prior); + expect(persist).not.toHaveBeenCalled(); + expect(restore).not.toHaveBeenCalled(); + expect(apply).toHaveBeenLastCalledWith( + expect.objectContaining({ + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-config", serviceId: "motis", set: { count: 4 }, unset: [] }, + ], + }, + }), + ); } finally { resume.resolve(); await older; @@ -420,12 +456,13 @@ describe("actual service producers through final exported SDK", () => { const response = await sdk.updateAdminServiceConfig( "motis", { config: { count: 3 } }, + { "idempotency-key": crypto.randomUUID() }, options, ); expect(response.status).toBe(500); expect(response.data).toEqual({ error: "Service administration failed" }); - expect(persist).toHaveBeenCalledWith("motis", { count: 3 }); - expect(restore).toHaveBeenCalledTimes(outcome === "not_committed" ? 1 : 0); + expect(persist).not.toHaveBeenCalled(); + expect(restore).not.toHaveBeenCalled(); expect(internalFailures).toEqual([failure]); expect(writeAuditLog).not.toHaveBeenCalled(); } finally { @@ -434,7 +471,7 @@ describe("actual service producers through final exported SDK", () => { }, ); - it("retains original apply failure internally when the inverse fails without leaking either error", async () => { + it("retains the original producer failure internally without a route-local inverse", async () => { const failure = new TrustedConfigurationApplyError({ outcome: "not_committed", reason: "failed", @@ -451,13 +488,13 @@ describe("actual service producers through final exported SDK", () => { const response = await sdk.updateAdminServiceConfig( "motis", { config: { count: 3 } }, + { "idempotency-key": crypto.randomUUID() }, options, ); expect(response.status).toBe(500); expect(response.data).toEqual({ error: "Service administration failed" }); - expect(internalFailures[0]).toBeInstanceOf(AggregateError); - expect((internalFailures[0] as AggregateError).cause).toBe(failure); - expect((internalFailures[0] as AggregateError).errors).toEqual([failure, inverseFailure]); + expect(internalFailures[0]).toBe(failure); + expect(restore).not.toHaveBeenCalled(); } finally { await close(); } @@ -545,10 +582,35 @@ describe("actual service producers through final exported SDK", () => { () => sdk.listAdminServices(options), () => sdk.getAdminDeployment(options), () => sdk.getAdminServiceSelection(options), - () => sdk.setAdminServiceSelection({ selectedRoots: [] }, options), - () => sdk.updateAdminServiceConfig("motis", { config: { count: 3 } }, options), - () => sdk.setAdminServiceCredential("motis", "API_KEY", { value: " synthetic " }, options), - () => sdk.deleteAdminServiceCredential("motis", "API_KEY", undefined, options), + () => + sdk.setAdminServiceSelection( + { selectedRoots: [] }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), + () => + sdk.updateAdminServiceConfig( + "motis", + { config: { count: 3 } }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), + () => + sdk.setAdminServiceCredential( + "motis", + "API_KEY", + { value: " synthetic " }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), + () => + sdk.deleteAdminServiceCredential( + "motis", + "API_KEY", + { "idempotency-key": crypto.randomUUID() }, + undefined, + options, + ), () => sdk.runAdminServiceAction("motis", { action: "restart" }, options), () => sdk.runAdminServicesBulkAction({ action: "build", all: true }, options), () => sdk.createAdminServiceBackup(undefined, options), @@ -594,8 +656,27 @@ describe("actual service producers through final exported SDK", () => { ), ).toMatchObject({ status: 200, data: undefined }); expect(chunks.join("")).toBe("hello €\n"); - expect(setSecret).toHaveBeenCalledWith("motis", "API_KEY", " synthetic ", session.user.id); - expect(enqueue).toHaveBeenCalledTimes(8); + expect(setSecret).not.toHaveBeenCalled(); + expect(apply).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "vault.apply", + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "motis", + key: "API_KEY", + action: "set", + value: " synthetic ", + }, + ], + }, + }), + ); + expect(enqueue).toHaveBeenCalledTimes(6); expect(cancelJob).not.toHaveBeenCalled(); expect(opsCancel).not.toHaveBeenCalled(); } finally { @@ -758,16 +839,26 @@ it("rejects malformed mutation envelopes and members before persistence or host } }); -it("retains structured domain failures and manual-render credential results through SDK validation", async () => { +it("retains structured domain failures and retained credential jobs through SDK validation", async () => { const { options, close } = await wire(); try { expect( - await sdk.updateAdminServiceConfig("motis", { config: { count: 0 } }, options), + await sdk.updateAdminServiceConfig( + "motis", + { config: { count: 0 } }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), ).toMatchObject({ status: 400, data: { errors: [expect.any(String)] } }); expect(merge).not.toHaveBeenCalled(); expect(apply).not.toHaveBeenCalled(); expect( - await sdk.updateAdminServiceConfig("motis", { config: { locked: "changed" } }, options), + await sdk.updateAdminServiceConfig( + "motis", + { config: { locked: "changed" } }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), ).toMatchObject({ status: 400, data: { errors: [expect.any(String)] } }); expect(merge).not.toHaveBeenCalled(); expect( @@ -780,15 +871,21 @@ it("retains structured domain failures and manual-render credential results thro issues: expect.arrayContaining([{ path: expect.any(String), message: expect.any(String) }]), }, }); - dockerAvailable.mockResolvedValueOnce(false); expect( - await sdk.setAdminServiceCredential("motis", "API_KEY", { value: "synthetic" }, options), - ).toMatchObject({ status: 200, data: { ok: true, needsRender: true } }); + await sdk.setAdminServiceCredential( + "motis", + "API_KEY", + { value: "synthetic" }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), + ).toMatchObject({ status: 200, data: { ok: true, jobId: "job-accepted" } }); expect( await sdk.setAdminServiceCredential( "dawarich-app", "OIDC_CLIENT_SECRET", { value: "synthetic" }, + { "idempotency-key": crypto.randomUUID() }, options, ), ).toMatchObject({ @@ -1110,7 +1207,11 @@ it("keeps log pre-hijack JSON statuses/media, required single headers and bounde it("returns complete Compose HTTP200 failure and does not retry the host action", async () => { const { options, close } = await wire(); try { - execute.mockRejectedValueOnce(Error("synthetic-host-failure")); + composeStart.mockResolvedValueOnce({ + ok: false, + stdout: "", + hardlinks: serviceHardlinksFixture, + }); expect( await sdk.startAdminCompose( { "idempotency-key": "compose-synthetic-intent-002" }, @@ -1121,7 +1222,7 @@ it("returns complete Compose HTTP200 failure and does not retry the host action" status: 200, data: { ok: false, stdout: "", hardlinks: serviceHardlinksFixture }, }); - expect(execute).toHaveBeenCalledTimes(1); + expect(composeStart).toHaveBeenCalledTimes(1); expect(opsCancel).not.toHaveBeenCalled(); } finally { await close(); @@ -1165,7 +1266,13 @@ it("retains env-owned selection failures and explicit trimmed/deduplicated inten validateSelection.mockRejectedValueOnce( Error("OPENMAPX_SERVICES is set; unset it before editing selection"), ); - expect(await sdk.setAdminServiceSelection({ selectedRoots: [] }, options)).toMatchObject({ + expect( + await sdk.setAdminServiceSelection( + { selectedRoots: [] }, + { "idempotency-key": crypto.randomUUID() }, + options, + ), + ).toMatchObject({ status: 400, data: { error: expect.stringContaining("OPENMAPX_SERVICES") }, }); @@ -1173,6 +1280,7 @@ it("retains env-owned selection failures and explicit trimmed/deduplicated inten const response = await app.inject({ method: "PUT", url: "/api/admin/services/selection", + headers: { "idempotency-key": crypto.randomUUID() }, payload: { selectedRoots: [" motis ", "motis"] }, }); expect(response.statusCode).toBe(200); diff --git a/apps/api/src/routes/admin-compose.ts b/apps/api/src/routes/admin-compose.ts index addd0d15e..7136616dd 100644 --- a/apps/api/src/routes/admin-compose.ts +++ b/apps/api/src/routes/admin-compose.ts @@ -1,15 +1,15 @@ import { repoPaths, services } from "@openmapx/core/server"; import type { FastifyInstance } from "fastify"; -import { applyGenerationHardlinks, renderAndPersistCompose } from "../services/admin-ops"; import { readDesiredSelection } from "../services/desired-selection"; import { createDirectAdminOpsKey, DIRECT_OPS_IDEMPOTENCY_HEADER, parseDirectOpsIdempotency, } from "../services/direct-ops-idempotency"; +import { jobRunner } from "../services/job-runner"; import { getServiceRegistry } from "../services/service-registry"; import { buildConfigurationInput, integrationSchemas } from "../services/trusted-config-operations"; -import { dockerComposeAction, STACK_STOP_GUIDANCE } from "../utils/docker-compose"; +import { STACK_STOP_GUIDANCE } from "../utils/docker-compose"; import { getAdminSession, requireAdmin } from "../utils/require-admin"; import { declareRouteAuth } from "../utils/route-auth"; import { registerServiceAdminRoute } from "./admin-service-contract"; @@ -54,15 +54,7 @@ export async function registerAdminComposeRoutes( req.headers[DIRECT_OPS_IDEMPOTENCY_HEADER], ); const operationKey = createDirectAdminOpsKey(adminSession.user.id, idempotencyValue); - const { revisionId } = await renderAndPersistCompose({ operationKey }); - const hardlinks = await applyGenerationHardlinks({ - revisionId, - operationIdentity: operationKey, - }); - const r = await dockerComposeAction("", "start", { - operationKey, - }); - return { ok: r.exitCode === 0, stdout: r.stdout, hardlinks }; + return jobRunner.startImmediateCompose({ actorId: adminSession.user.id, operationKey }); }, { beforeValidation: async (request, reply) => { diff --git a/apps/api/src/routes/admin-integration-contract.ts b/apps/api/src/routes/admin-integration-contract.ts index 49cbc905d..1b302f410 100644 --- a/apps/api/src/routes/admin-integration-contract.ts +++ b/apps/api/src/routes/admin-integration-contract.ts @@ -1,4 +1,5 @@ import { integrationAdminOperations } from "@openmapx/core/api-contracts"; +import { ConfigurationPrerequisiteFailure } from "../services/admin-job-failure"; import { createAdminRouteRegistrar } from "./admin-contract"; const register = createAdminRouteRegistrar(integrationAdminOperations, { @@ -15,7 +16,9 @@ export const registerIntegrationAdminRoute: typeof register = (app, operation, h async (request, reply) => { try { return await handler(request, reply); - } catch { + } catch (error) { + if (error instanceof ConfigurationPrerequisiteFailure) + return reply.code(409).send({ error: error.message }); return reply.code(500).send({ error: "Integration administration failed" }); } }, diff --git a/apps/api/src/routes/admin-integration-credentials.ts b/apps/api/src/routes/admin-integration-credentials.ts index e92672ecc..e15c0224e 100644 --- a/apps/api/src/routes/admin-integration-credentials.ts +++ b/apps/api/src/routes/admin-integration-credentials.ts @@ -1,11 +1,17 @@ import type { FastifyInstance } from "fastify"; -import { getAllIntegrations, getIntegration, reloadIntegrations } from "../integration-host"; +import { getAllIntegrations, getIntegration } from "../integration-host"; +import { + createDirectAdminOpsKey, + DIRECT_OPS_IDEMPOTENCY_HEADER, + parseDirectOpsIdempotency, +} from "../services/direct-ops-idempotency"; import { computeCredentialStatus, getIntegrationDisplayName, } from "../services/integration-credential-status"; import { executeIntegrationHealthCheck } from "../services/integration-health"; -import { deleteSecret, isSecretsConfigured, listSecrets, setSecret } from "../services/secrets"; +import { isSecretsConfigured, listSecrets } from "../services/secrets"; +import { applyTrustedConfiguration } from "../services/trusted-config-operations"; import { writeAuditLog } from "../utils/audit-log"; import { getAdminSession } from "../utils/require-admin"; import { getSecretFields } from "../utils/validate-config-body"; @@ -79,7 +85,20 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr }); } - await setSecret(integrationId, key, value, adminSession.user.id); + const applied = await applyTrustedConfiguration({ + kind: "integrationConfig.apply", + integrationId, + actorId: adminSession.user.id, + operationKey: createDirectAdminOpsKey( + adminSession.user.id, + parseDirectOpsIdempotency(request.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), + ), + overlay: { + purpose: "integration-secret", + coordinatorMode: "ensure-running", + changes: [{ kind: "integration-secret", integrationId, key, action: "set", value }], + }, + }); await writeAuditLog({ actorId: adminSession.user.id, @@ -94,7 +113,8 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr // integration captures its resolved config (incl. vault secrets) once at // `setup(ctx)` load time, so without this a freshly-set key never reaches // the provider until the next restart. Mirrors the config-PATCH endpoint. - const reloadResult = await reloadIntegrations(); + const reloadResult = applied.integrationReload; + if (!reloadResult) throw Error("Captured integration reload unavailable"); return { ok: true, ...reloadResult }; }); @@ -106,7 +126,20 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr const adminSession = getAdminSession(request); - await deleteSecret(integrationId, key); + const applied = await applyTrustedConfiguration({ + kind: "integrationConfig.apply", + integrationId, + actorId: adminSession.user.id, + operationKey: createDirectAdminOpsKey( + adminSession.user.id, + parseDirectOpsIdempotency(request.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), + ), + overlay: { + purpose: "integration-secret", + coordinatorMode: "ensure-running", + changes: [{ kind: "integration-secret", integrationId, key, action: "delete" }], + }, + }); await writeAuditLog({ actorId: adminSession.user.id, @@ -119,7 +152,8 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr // Reload so the running integration drops the removed secret (see the // set-secret handler above). - const reloadResult = await reloadIntegrations(); + const reloadResult = applied.integrationReload; + if (!reloadResult) throw Error("Captured integration reload unavailable"); return { ok: true, ...reloadResult }; }); diff --git a/apps/api/src/routes/admin-integrations.ts b/apps/api/src/routes/admin-integrations.ts index a769c18b6..3da8ffce0 100644 --- a/apps/api/src/routes/admin-integrations.ts +++ b/apps/api/src/routes/admin-integrations.ts @@ -3,23 +3,18 @@ import { integrationEnvVarName } from "@openmapx/integration-framework"; import type { FastifyInstance } from "fastify"; import { db } from "../db"; import { integrationConfig } from "../db/schema"; +import { getAllIntegrations, getIntegration, resolveConfigWithSources } from "../integration-host"; import { - getAllIntegrations, - getIntegration, - reloadIntegrations, - resolveConfigWithSources, -} from "../integration-host"; + createDirectAdminOpsKey, + DIRECT_OPS_IDEMPOTENCY_HEADER, + parseDirectOpsIdempotency, +} from "../services/direct-ops-idempotency"; import { getTimeline } from "../services/health-history"; import { exportIntegrationConfigs, IntegrationImportValidationError, prepareIntegrationImport, - runIntegrationImport, } from "../services/integration-config-import"; -import { - persistIntegrationConfigPatch, - persistIntegrationImport, -} from "../services/integration-config-store"; import { computeCredentialStatus, getIntegrationDisplayName, @@ -30,15 +25,11 @@ import { getCachedHealthStatus, } from "../services/integration-health"; import { jobRunner } from "../services/job-runner"; -import { createDurableOpsKey } from "../services/ops-client"; import { isSecretsConfigured } from "../services/secrets"; import { applyTrustedConfiguration, - applyTrustedConfigurationSnapshot, - applyTrustedConfigurationWithEvidence, integrationSchemas, } from "../services/trusted-config-operations"; -import { isProvenConfigurationNonCommit } from "../services/trusted-config-outcome"; import { writeAuditLog } from "../utils/audit-log"; import { maskSecretConfigValues } from "../utils/mask-config.js"; import { healthCheckSweepLimit } from "../utils/rate-limit"; @@ -50,27 +41,28 @@ async function persistAndApplyIntegrationConfig(options: { integrationId: string; config: Record; actorId: string; - requestId: string; -}): Promise { - const saved = await persistIntegrationConfigPatch({ + operationKey: string; +}): Promise { + const applied = await applyTrustedConfiguration({ + kind: "integrationConfig.apply", integrationId: options.integrationId, - config: adminIntegrationJsonObjectSchema.parse(options.config), + actorId: options.actorId, + operationKey: options.operationKey, + overlay: { + purpose: "integration-config", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "integration-config", + integrationId: options.integrationId, + set: adminIntegrationJsonObjectSchema.parse(options.config), + unset: [], + }, + ], + }, }); - try { - await applyTrustedConfiguration({ - kind: "integrationConfig.apply", - integrationId: options.integrationId, - operationKey: createDurableOpsKey( - "admin-route.integration-config.apply", - `${options.actorId}\0${options.requestId}`, - ), - }); - } catch (error) { - if (!isProvenConfigurationNonCommit(error)) throw error; - // The same conditional row ownership protects imports from an older edit's recovery. - await saved.restore(); - throw error; - } + if (!applied.integrationReload) throw Error("Captured integration reload unavailable"); + return applied.integrationReload; } export async function adminIntegrationsRoute(app: FastifyInstance): Promise { @@ -262,11 +254,14 @@ export async function adminIntegrationsRoute(app: FastifyInstance): Promise applyTrustedConfigurationWithEvidence({ kind: "stack.render", operationKey }), - compensate: (payload, expectedCurrentRevisionId) => - applyTrustedConfigurationSnapshot({ - payload, - expectedCurrentRevisionId, - operationKey: createDurableOpsKey( - "admin-route.integration-import.recovery", - operationKey, - ), - }), - reload: reloadIntegrations, + const result = await jobRunner.importImmediateIntegrationConfiguration({ + body: { integrations: entries }, + actorId: adminSession.user.id, + operationKey, }); if (!result.ok) return reply.status(409).send(result); await writeAuditLog({ diff --git a/apps/api/src/routes/admin-service-contract.ts b/apps/api/src/routes/admin-service-contract.ts index 717fa08a9..88d06bd25 100644 --- a/apps/api/src/routes/admin-service-contract.ts +++ b/apps/api/src/routes/admin-service-contract.ts @@ -1,4 +1,5 @@ import { serviceAdminOperations } from "@openmapx/core/api-contracts"; +import { ConfigurationPrerequisiteFailure } from "../services/admin-job-failure"; import { createAdminRouteRegistrar } from "./admin-contract"; const register = createAdminRouteRegistrar(serviceAdminOperations, { @@ -15,7 +16,9 @@ export const registerServiceAdminRoute: typeof register = (app, operation, handl async (request, reply) => { try { return await handler(request, reply); - } catch { + } catch (error) { + if (error instanceof ConfigurationPrerequisiteFailure) + return reply.code(409).send({ error: error.message }); return reply.code(500).send({ error: "Service administration failed" }); } }, diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index d3e8277d2..ad17b961c 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -1,5 +1,5 @@ import { existsSync } from "node:fs"; -import type { OpsResultFor } from "@openmapx/core/ops"; +import { type OpsResultFor, trustedConfigurationPayloadSchema } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; import { isValidSecretKey } from "@openmapx/core/services/secret-key"; import { @@ -13,7 +13,6 @@ import { getServiceSelectionSummary, validateServiceSelectionForWrite, } from "../services/admin-cli"; -import { isDockerAvailable } from "../services/admin-ops"; import { DAWARICH_CREDENTIAL_MANAGED_BY, DAWARICH_CREDENTIAL_MANAGED_CODE, @@ -29,15 +28,9 @@ import { jobRunner } from "../services/job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client"; import { isSecretsConfigured } from "../services/secrets"; import { resolveServiceConfigWithSources } from "../services/service-config-resolver"; -import { persistServiceConfigPatch } from "../services/service-config-writer"; import { getServiceRegistry } from "../services/service-registry"; -import { - deleteServiceSecret, - listServiceSecrets, - setServiceSecret, -} from "../services/service-secrets"; +import { listServiceSecrets } from "../services/service-secrets"; import { applyTrustedConfiguration } from "../services/trusted-config-operations"; -import { isProvenConfigurationNonCommit } from "../services/trusted-config-outcome"; import { writeAuditLog } from "../utils/audit-log"; import { dockerComposeLogs, dockerComposePs } from "../utils/docker-compose"; import { maskSecretConfigValues } from "../utils/mask-config.js"; @@ -175,9 +168,9 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { async (req, reply) => { const { id } = req.params; const { action } = req.body; - if (!["start", "stop", "restart"].includes(action)) { + if (!["start", "stop", "restart", "apply"].includes(action)) { reply.status(400); - return { error: "Invalid action — must be start | stop | restart" }; + return { error: "Invalid action — must be start | stop | restart | apply" }; } // Verify the id is a known service before enqueueing — prevents the job // runner from spawning a `docker compose start ` call (or worse, @@ -246,13 +239,14 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { return { error: (err as Error).message }; } - const operationKey = createDurableOpsKey( - "admin-route.selection.apply", - `${getAdminSession(req).user.id}\0${String(req.id)}`, + const operationKey = createDirectAdminOpsKey( + getAdminSession(req).user.id, + parseDirectOpsIdempotency(req.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), ); const applied = await applyTrustedConfiguration({ kind: "serviceSelection.apply", selectedRoots: validated.normalized, + actorId: getAdminSession(req).user.id, operationKey, }); const summary = await getServiceSelectionSummary(registry, validated.normalized); @@ -292,7 +286,9 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { const allowedKeys = action === "build" ? new Set(["action", "serviceIds", "all", "region", "continueOnError"]) - : new Set(["action", "serviceIds", "continueOnError"]); + : action === "update" + ? new Set(["action", "serviceIds", "continueOnError", "noDeps"]) + : new Set(["action", "serviceIds", "continueOnError"]); if (!hasOnlyBodyKeys(req.body, allowedKeys)) { reply.status(400); return { error: "Request contains unsupported fields" }; @@ -322,6 +318,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { all: ("all" in req.body ? req.body.all : undefined) === true, region, continueOnError: req.body?.continueOnError, + ...("noDeps" in req.body ? { noDeps: req.body.noDeps } : {}), }, adminSession.user.id, ); @@ -392,31 +389,29 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { return { errors }; } const adminSession = getAdminSession(req); - const saved = await persistServiceConfigPatch(req.params.id, config); - try { - await applyTrustedConfiguration({ - kind: "serviceConfig.apply", - serviceId: req.params.id, - operationKey: createDurableOpsKey( - "admin-route.service-config.apply", - `${adminSession.user.id}\0${String(req.id)}`, - ), - }); - } catch (error) { - if (!isProvenConfigurationNonCommit(error)) throw error; - try { - // False means a later writer owns the row. Either way the apply failed; - // do not promise restoration or turn this into a successful response. - await saved.restore(); - } catch (restoreError) { - throw new AggregateError( - [error, restoreError], - "Service configuration apply and restoration failed", - { cause: error }, - ); - } - throw error; - } + await applyTrustedConfiguration({ + kind: "serviceConfig.apply", + serviceId: req.params.id, + actorId: adminSession.user.id, + operationKey: createDirectAdminOpsKey( + adminSession.user.id, + parseDirectOpsIdempotency(req.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), + ), + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-config", + serviceId: req.params.id, + set: trustedConfigurationPayloadSchema.shape.serviceConfigs.element.shape.values.parse( + config, + ), + unset: [], + }, + ], + }, + }); await writeAuditLog({ actorId: adminSession.user.id, targetId: req.params.id, @@ -441,21 +436,6 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { // replaces the service container (services bake env/secrets at create time — no live // reload). Requires Docker host-control; otherwise the caller gets // `needsRender` and must render + apply on the host. - async function applyServiceSecretChange( - serviceId: string, - userId: string, - ): Promise<{ ok: true; jobId?: string; needsRender?: boolean }> { - if (await isDockerAvailable()) { - const jobId = await jobRunner.enqueue( - "service.apply", - { service: serviceId, configurationKind: "vault" }, - userId, - ); - return { ok: true, jobId }; - } - return { ok: true, needsRender: true }; - } - // GET /admin/services/:id/credentials — declared secret fields + per-field // status (vault | missing), setup guides, and who set each + when. registerServiceAdminRoute(app, "getAdminServiceCredentials", async (req, reply) => { @@ -526,7 +506,29 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { return { error: "value must be a non-empty string" }; } const adminSession = getAdminSession(req); - await setServiceSecret(svc.manifest.id, req.params.key, value, adminSession.user.id); + const applied = await applyTrustedConfiguration({ + kind: "vault.apply", + serviceId: svc.manifest.id, + actorId: adminSession.user.id, + followUp: "service-apply", + operationKey: createDirectAdminOpsKey( + adminSession.user.id, + parseDirectOpsIdempotency(req.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), + ), + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: svc.manifest.id, + key: req.params.key, + action: "set", + value, + }, + ], + }, + }); await writeAuditLog({ actorId: adminSession.user.id, targetId: svc.manifest.id, @@ -535,7 +537,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { details: { key: req.params.key }, request: req, }); - return applyServiceSecretChange(svc.manifest.id, adminSession.user.id); + return { ok: true as const, jobId: applied.jobId }; }, { beforeValidation: async (req, reply) => { @@ -590,7 +592,28 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { } } const adminSession = getAdminSession(req); - await deleteServiceSecret(svc.manifest.id, req.params.key); + const applied = await applyTrustedConfiguration({ + kind: "vault.apply", + serviceId: svc.manifest.id, + actorId: adminSession.user.id, + followUp: "service-apply", + operationKey: createDirectAdminOpsKey( + adminSession.user.id, + parseDirectOpsIdempotency(req.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), + ), + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: svc.manifest.id, + key: req.params.key, + action: "delete", + }, + ], + }, + }); await writeAuditLog({ actorId: adminSession.user.id, targetId: svc.manifest.id, @@ -599,7 +622,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { details: { key: req.params.key }, request: req, }); - return applyServiceSecretChange(svc.manifest.id, adminSession.user.id); + return { ok: true as const, jobId: applied.jobId }; }, { beforeValidation: async (req, reply) => { diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 66f72aa50..cf58aa1f0 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -688,8 +688,8 @@ jobRunner.register("service.restart", async (ctx) => { jobRunner.register("service.apply", async (ctx) => { const service = ctx.payload.service as string; - await serviceApply(service, ctx); - return { service, action: "apply" }; + const result = await serviceApply(service, ctx); + return { service, action: "apply", ...result }; }); jobRunner.register("data.operation", handleDataOperationJob); diff --git a/apps/api/src/services/__tests__/admin-job-handlers.test.ts b/apps/api/src/services/__tests__/admin-job-handlers.test.ts index 87e572ce5..af81848b2 100644 --- a/apps/api/src/services/__tests__/admin-job-handlers.test.ts +++ b/apps/api/src/services/__tests__/admin-job-handlers.test.ts @@ -2,6 +2,11 @@ import Fastify from "fastify"; import { describe, expect, it, vi } from "vitest"; const runCliMock = vi.hoisted(() => vi.fn().mockResolvedValue(undefined)); +const preparedRevision = `cfg1_${"p".repeat(43)}`; +const prepareGenerationMock = vi.hoisted(() => vi.fn()); +vi.mock("../service-preparation", () => ({ + prepareServiceGeneration: (...args: unknown[]) => prepareGenerationMock(...args), +})); const runAdminOperationMock = vi.hoisted(() => vi .fn() @@ -344,6 +349,58 @@ describe("admin job handlers", () => { }); describe("bulk service operations", () => { + it("prepares one full generation before bulk Start and carries it to deterministic target children", async () => { + prepareGenerationMock.mockReset().mockResolvedValue({ + revisionId: preparedRevision, + coordinatorMode: "ensure-running", + hardlinks: { applied: true, entries: 0, linked: 0, skipped: 0, pruned: 0 }, + }); + runAdminOperationMock.mockClear(); + const ctx = { + jobId: "bulk-start-fixture", + payload: { action: "start", serviceIds: ["valhalla", "osrm"] }, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn(), + }; + await handleServiceBulkJob(ctx); + expect(prepareGenerationMock).toHaveBeenCalledTimes(1); + expect(prepareGenerationMock).toHaveBeenCalledWith(ctx, "bulk-start", { + coordinatorMode: "ensure-running", + }); + expect(runAdminOperationMock.mock.calls.map((call) => call[1])).toEqual([ + { + kind: "service.start", + serviceId: "valhalla", + expectedConfigurationRevisionId: preparedRevision, + }, + { + kind: "service.start", + serviceId: "osrm", + expectedConfigurationRevisionId: preparedRevision, + }, + ]); + expect(runAdminOperationMock.mock.calls.map((call) => call[3])).toEqual([ + { durableIdentity: "valhalla" }, + { durableIdentity: "osrm" }, + ]); + }); + it("starts no bulk target when full preparation or hardlinks fail", async () => { + prepareGenerationMock.mockReset().mockRejectedValue(new Error("synthetic hardlink failure")); + runAdminOperationMock.mockClear(); + await expect( + handleServiceBulkJob({ + jobId: "bulk-start-refusal", + payload: { action: "start", serviceIds: ["valhalla", "osrm"] }, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn(), + }), + ).rejects.toThrow("synthetic hardlink failure"); + expect(runAdminOperationMock).not.toHaveBeenCalled(); + }); it("does not continue a batch after unknown admission even with continueOnError", async () => { const { OpsObservationPending } = await import("../ops-client"); const pending = new OpsObservationPending( diff --git a/apps/api/src/services/__tests__/admin-job-ops.test.ts b/apps/api/src/services/__tests__/admin-job-ops.test.ts index b0745d720..43277b554 100644 --- a/apps/api/src/services/__tests__/admin-job-ops.test.ts +++ b/apps/api/src/services/__tests__/admin-job-ops.test.ts @@ -7,6 +7,16 @@ import { } from "../admin-job-ops"; import type { JobContext } from "../job-runner"; +const previousAuthSecret = vi.hoisted(() => { + const previous = process.env.BETTER_AUTH_SECRET; + process.env.BETTER_AUTH_SECRET ??= "task012-unit-only-synthetic-auth-secret"; + return { previous }; +}); +afterAll(() => { + if (previousAuthSecret.previous === undefined) delete process.env.BETTER_AUTH_SECRET; + else process.env.BETTER_AUTH_SECRET = previousAuthSecret.previous; +}); + let fixtureRoot: string, previousToken: string | undefined; beforeAll(async () => { const { mkdtemp, writeFile } = await import("node:fs/promises"), @@ -572,8 +582,10 @@ it("retains a service workflow parent budget beyond its short render child", asy ctx, { kind: "stack.render", - revisionId: "generation", + revisionId: `cfg1_${"a".repeat(43)}`, expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", }, "opk1_renderAttempt12345", ); @@ -588,7 +600,11 @@ it("retains a service workflow parent budget beyond its short render child", asy now.mockReturnValue(1_800_000_000_000 + opsExecutionBudget("stack.render").processMs); const start = prepareApiJobChildAttempt( ctx, - { kind: "service.start", serviceId: "tileserver" }, + { + kind: "service.start", + serviceId: "tileserver", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "opk1_serviceAttempt1234", ); expect(start?.parentDeadlineAt).toBe(render?.parentDeadlineAt); diff --git a/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts b/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts index eefc6e3cd..e85364357 100644 --- a/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts +++ b/apps/api/src/services/__tests__/admin-ops-hardlinks.test.ts @@ -1,8 +1,18 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { randomUUID } from "node:crypto"; +import type { CoordinatorMode } from "@openmapx/core/ops"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { JobObservationPending } from "../admin-job-failure"; +import type { JobContext } from "../job-runner"; -// The module reaches `integration-routes` and therefore Better Auth, which -// asserts on this at import time. -process.env.BETTER_AUTH_SECRET ||= "admin-ops-hardlinks-test-stub-secret"; +const previousAuthSecret = vi.hoisted(() => { + const previous = process.env.BETTER_AUTH_SECRET; + process.env.BETTER_AUTH_SECRET ??= "task012-unit-only-synthetic-auth-secret"; + return { previous }; +}); +afterAll(() => { + if (previousAuthSecret.previous === undefined) delete process.env.BETTER_AUTH_SECRET; + else process.env.BETTER_AUTH_SECRET = previousAuthSecret.previous; +}); const REVISION = `cfg1_${"r".repeat(43)}`; const plan = [ @@ -13,161 +23,216 @@ const plan = [ dataType: "osm-pbf", }, ]; -const executeAndWait = vi.fn(); -const dockerComposeAction = vi.fn(); -const link = vi.fn(); -const getStatus = vi.fn(); -const broker = vi.fn(); -const registryGet = vi.fn(); - -vi.mock("../ops-client.js", () => ({ +const summary = { + applied: true, + entries: 1, + linked: 1, + skipped: 0, + pruned: 0, + via: "data-manager", +}; +// These are typed transport/publication boundary tests. The separate owned-PG +// suites qualify the actual agent, role authority and durable DM leaf. +const inspect = vi.fn(), + broker = vi.fn(), + publish = vi.fn(); +vi.mock("../ops-client", async (original) => ({ + ...(await original()), createApiOpsClient: () => ({}), - createDurableOpsKey: (scope: string, identity: string) => `${scope}:${identity}`, - executeAndWait: (...args: unknown[]) => executeAndWait(...args), -})); -vi.mock("../../utils/docker-compose.js", () => ({ - dockerComposeAction: (...args: unknown[]) => dockerComposeAction(...args), - dockerStatus: vi.fn(), + executeAndWait: (...args: unknown[]) => inspect(...args), })); -vi.mock("@openmapx/core/data-manager-sdk", () => ({ - createDataManagerTransport: () => ({}), - getStatus: (...args: unknown[]) => getStatus(...args), - link: (...args: unknown[]) => link(...args), - unwrapDataManagerResponse: (response: unknown) => response, -})); -vi.mock("../service-registry.js", () => ({ - getServiceRegistry: () => ({ get: registryGet }), -})); - vi.mock("../admin-job-ops", () => ({ executeAdminJobOperation: (...args: unknown[]) => broker(...args), })); - vi.mock("../trusted-config-operations", () => ({ - applyTrustedConfiguration: async () => ({ revisionId: REVISION }), + applyTrustedConfiguration: (...args: unknown[]) => publish(...args), })); - -const { applyGenerationHardlinks, serviceStart } = await import("../admin-ops.js"); - +vi.mock("../trusted-configuration-producer", () => ({ + applyTrustedConfiguration: (...args: unknown[]) => publish(...args), +})); +vi.mock("../service-registry", () => ({ + getServiceRegistry: () => ({ get: () => ({ enabled: true }) }), +})); +const { applyGenerationHardlinks, serviceStart } = await import("../admin-ops"); +function context(): JobContext { + const ctx: JobContext = { + jobId: randomUUID(), + execution: { owner: randomUUID(), version: 1 }, + payload: {}, + signal: new AbortController().signal, + log: async () => {}, + setProgress: async () => {}, + checkpointResult: {}, + checkpoint: async (value) => { + ctx.checkpointResult = structuredClone(value); + }, + }; + return ctx; +} beforeEach(() => { - vi.clearAllMocks(); - executeAndWait.mockResolvedValue({ revisionId: REVISION, plan }); - dockerComposeAction.mockResolvedValue({ exitCode: 0, stdout: "", stderr: "" }); - link.mockResolvedValue({ linked: 1, skipped: 0, pruned: 0 }); - registryGet.mockReturnValue({ enabled: true }); - getStatus.mockResolvedValue({ data: {}, status: 200 }); - broker.mockResolvedValue({ changed: true }); + inspect.mockReset().mockResolvedValue({ revisionId: REVISION, plan }); + publish.mockReset().mockResolvedValue({ revisionId: REVISION }); + broker + .mockReset() + .mockImplementation(async (_ctx, operation) => + operation.kind === "configuration.hardlinks.apply" ? summary : { changed: true }, + ); }); - -describe("applyGenerationHardlinks", () => { - it("has data-manager apply the plan of the generation just applied", async () => { - const summary = await applyGenerationHardlinks({ +describe("retained generation hardlinks", () => { + it("settles a separate revision-checked coordinator Start before the typed middle owner", async () => { + const ctx = context(); + expect( + await applyGenerationHardlinks({ + ctx, + revisionId: REVISION, + operationIdentity: "job:valhalla", + coordinatorMode: "ensure-running", + }), + ).toEqual(summary); + expect(inspect.mock.calls[0]![1]).toEqual({ + kind: "configuration.hardlinks.inspect", revisionId: REVISION, - operationIdentity: "job-1:valhalla", }); - - // The plan comes from the operations agent, for exactly this generation. - expect(executeAndWait).toHaveBeenCalledWith( - expect.anything(), - { kind: "configuration.hardlinks.inspect", revisionId: REVISION }, - "admin-job.hardlinks.inspect:job-1:valhalla", - expect.anything(), - ); - // Data-manager, whose mounts confine it to the data root, applies it. - expect(dockerComposeAction).toHaveBeenCalledWith("data-manager", "start", expect.anything()); - expect(link).toHaveBeenCalledWith({ plan, prune: true }, expect.anything()); - expect(summary).toMatchObject({ applied: true, entries: 1, linked: 1 }); + expect(broker.mock.calls.map((call) => call[1])).toEqual([ + { + kind: "service.start", + serviceId: "data-manager", + expectedConfigurationRevisionId: REVISION, + }, + { + kind: "configuration.hardlinks.apply", + revisionId: REVISION, + coordinatorMode: "ensure-running", + }, + ]); }); - - it("does nothing more for an empty plan", async () => { - executeAndWait.mockResolvedValue({ revisionId: REVISION, plan: [] }); + it("require-running delegates only the exact conditioned hardlink operation", async () => { + const ctx = context(); + await applyGenerationHardlinks({ + ctx, + revisionId: REVISION, + operationIdentity: "render", + coordinatorMode: "require-running", + }); + expect(broker.mock.calls.map((call) => call[1])).toEqual([ + { + kind: "configuration.hardlinks.apply", + revisionId: REVISION, + coordinatorMode: "require-running", + }, + ]); + }); + it.each(["ensure-running", "require-running"] as CoordinatorMode[])( + "needs no coordinator for an empty plan in %s", + async (coordinatorMode) => { + inspect.mockResolvedValue({ revisionId: REVISION, plan: [] }); + expect( + await applyGenerationHardlinks({ + ctx: context(), + revisionId: REVISION, + operationIdentity: "empty", + coordinatorMode, + }), + ).toEqual({ applied: true, entries: 0, linked: 0, skipped: 0, pruned: 0, via: "none" }); + expect(broker).not.toHaveBeenCalled(); + }, + ); + it("does not admit hardlinks after coordinator Start fails", async () => { + broker.mockRejectedValueOnce(Error("Synthetic coordinator failure")); await expect( - applyGenerationHardlinks({ revisionId: REVISION, operationIdentity: "job-2" }), - ).resolves.toEqual({ applied: true, linked: 0, skipped: 0, pruned: 0, entries: 0 }); - expect(dockerComposeAction).not.toHaveBeenCalled(); - expect(link).not.toHaveBeenCalled(); + applyGenerationHardlinks({ + ctx: context(), + revisionId: REVISION, + operationIdentity: "failed", + coordinatorMode: "ensure-running", + }), + ).rejects.toThrow(); + expect(broker).toHaveBeenCalledTimes(1); }); - - it("refuses a non-empty plan while data-manager is disabled", async () => { - registryGet.mockReturnValue({ enabled: false }); + it("keeps an unknown durable leaf pending", async () => { + broker.mockRejectedValueOnce(new JobObservationPending("recovery_required")); await expect( - applyGenerationHardlinks({ revisionId: REVISION, operationIdentity: "job-3" }), - ).rejects.toThrow('service "data-manager" is disabled'); - expect(link).not.toHaveBeenCalled(); + applyGenerationHardlinks({ + ctx: context(), + revisionId: REVISION, + operationIdentity: "unknown", + coordinatorMode: "require-running", + }), + ).rejects.toBeInstanceOf(JobObservationPending); + expect(broker).toHaveBeenCalledTimes(1); }); -}); - -it("does not link or start a later lifecycle effect after canceled readiness resolves", async () => { - const ready = Promise.withResolvers(); - getStatus.mockReturnValue(ready.promise); - const abort = new AbortController(); - const ctx = { - jobId: "fix1-service", - payload: {}, - signal: abort.signal, - log: vi.fn(), - setProgress: vi.fn(), - checkpoint: vi.fn(), - checkpointResult: {}, - }; - const work = serviceStart("valhalla", ctx); - const outcome = work.then( - () => "fulfilled", - (error) => error, - ); - await vi.waitFor(() => expect(getStatus).toHaveBeenCalled()); - abort.abort(); - ready.resolve({ data: {}, status: 200 }); - expect(await outcome).toBeInstanceOf(Error); - expect(link).not.toHaveBeenCalled(); - expect(broker.mock.calls.map((call) => call[1])).toEqual([ - { kind: "service.start", serviceId: "data-manager" }, - ]); -}); - -it("fences the link attempt against current ownership after readiness", async () => { - const { JobOwnershipLost } = await import("../admin-job-state"); - const ctx = { - jobId: "fix1-owner", - payload: {}, - signal: new AbortController().signal, - log: vi.fn(), - setProgress: vi.fn(), - checkpoint: vi.fn().mockRejectedValue(new JobOwnershipLost()), - checkpointResult: {}, - }; - await expect( - applyGenerationHardlinks({ - revisionId: REVISION, - operationIdentity: "fix1-owner:valhalla", + it("rejects a call without a retained job before host inspection", async () => { + await expect( + applyGenerationHardlinks({ + revisionId: REVISION, + operationIdentity: "foreign", + coordinatorMode: "ensure-running", + }), + ).rejects.toThrow(); + expect(inspect).not.toHaveBeenCalled(); + }); + it("observes an old unknown link attempt without another coordinator or leaf call", async () => { + const ctx = context(); + ctx.checkpointResult = { + _hardlinkAttempts: { retained: { revisionId: REVISION, phase: "attempted" } }, + }; + const options = { ctx, - }), - ).rejects.toBeInstanceOf(JobOwnershipLost); - expect(link).not.toHaveBeenCalled(); -}); - -it("retains an unknown link attempt and refuses any coordinator/link replay", async () => { - const { JobObservationPending } = await import("../admin-job-failure"); - const ctx: import("../job-runner").JobContext = { - jobId: "fix1-link", - payload: {}, - signal: new AbortController().signal, - log: vi.fn(), - setProgress: vi.fn(), - checkpoint: async (value) => { - ctx.checkpointResult = structuredClone(value); - }, - checkpointResult: {}, - }; - link.mockImplementation(async () => { - expect(ctx.checkpointResult).toMatchObject({ - _hardlinkAttempts: { "fix1-link:valhalla": { revisionId: REVISION, phase: "attempted" } }, + revisionId: REVISION, + operationIdentity: "retained", + coordinatorMode: "ensure-running" as const, + }; + await expect(applyGenerationHardlinks(options)).rejects.toBeInstanceOf(JobObservationPending); + await expect(applyGenerationHardlinks(options)).rejects.toBeInstanceOf(JobObservationPending); + expect(inspect).not.toHaveBeenCalled(); + expect(broker).not.toHaveBeenCalled(); + }); + it("returns only the exact completed legacy link receipt", async () => { + const ctx = context(); + ctx.checkpointResult = { + _hardlinkAttempts: { retained: { revisionId: REVISION, phase: "complete", result: summary } }, + }; + expect( + await applyGenerationHardlinks({ + ctx, + revisionId: REVISION, + operationIdentity: "retained", + coordinatorMode: "require-running", + }), + ).toEqual(summary); + expect(inspect).not.toHaveBeenCalled(); + }); + it("single Start uses shared full preparation and targets only its final revision", async () => { + const ctx = context(); + await serviceStart("valhalla", ctx); + expect(publish).toHaveBeenCalledTimes(1); + expect(publish.mock.calls[0]![0]).toMatchObject({ + kind: "stack.render", + coordinatorMode: "ensure-running", + }); + expect(broker.mock.calls.map((call) => call[1])).toEqual([ + { + kind: "service.start", + serviceId: "data-manager", + expectedConfigurationRevisionId: REVISION, + }, + { + kind: "configuration.hardlinks.apply", + revisionId: REVISION, + coordinatorMode: "ensure-running", + }, + { kind: "service.start", serviceId: "valhalla", expectedConfigurationRevisionId: REVISION }, + ]); + const published = broker.mock.calls.length; + await (await import("../service-preparation")).prepareServiceGeneration(ctx, "start:valhalla", { + coordinatorMode: "ensure-running", }); - throw new Error("response lost after remote acceptance"); + expect(publish).toHaveBeenCalledTimes(1); + expect(broker).toHaveBeenCalledTimes(published); + }); + it("single Start does not reach its target after preparation fails", async () => { + broker.mockRejectedValueOnce(Error("Synthetic helper failure")); + await expect(serviceStart("valhalla", context())).rejects.toThrow(); + expect(broker).toHaveBeenCalledTimes(1); }); - const options = { revisionId: REVISION, operationIdentity: "fix1-link:valhalla", ctx }; - await expect(applyGenerationHardlinks(options)).rejects.toBeInstanceOf(JobObservationPending); - await expect(applyGenerationHardlinks(options)).rejects.toBeInstanceOf(JobObservationPending); - expect(link).toHaveBeenCalledTimes(1); - expect(broker).toHaveBeenCalledTimes(1); }); diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts new file mode 100644 index 000000000..2a03bfbff --- /dev/null +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -0,0 +1,8436 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import * as nodeFs from "node:fs"; +import { + existsSync, + linkSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + watch, + writeFileSync, +} from "node:fs"; +import * as nodeFsPromises from "node:fs/promises"; +import { createServer } from "node:http"; +import { createRequire } from "node:module"; +import { basename, join } from "node:path"; +import { setTimeout as nativeDelay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { + closeControlInstallation, + lookupControlChildPhaseAttempt, + lookupControlChildPhaseBinding, + openControlInstallation, + reconcileControlRetention, + restoreAdmittedControlExecution, + verifyDataControlTerminalWitness, +} from "@openmapx/core/build-artifacts"; +import { canonicalJson } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { eq } from "drizzle-orm"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; +import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; +import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; +import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; +import * as schema from "../../db/schema"; +import { createConfigurationPostgresFixture } from "../../test/configuration-postgres-fixture"; +import type { JobContext } from "../job-runner"; +import { postgresCommitRelay } from "./fixtures/postgres-commit-relay"; + +type Fixture = Awaited>; +const state = vi.hoisted(() => ({ + fixture: null as unknown as Fixture, + services: [] as services.LoadedService[], +})); +// Named external filesystem availability/publication I/O only. Authority, +// private readers, signatures, capacity CAS, SQL and routes stay installed. +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, statfsSync: vi.fn(actual.statfsSync), renameSync: vi.fn(actual.renameSync) }; +}); +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, opendir: vi.fn(actual.opendir), unlink: vi.fn(actual.unlink) }; +}); +vi.mock("../../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); +vi.mock("../../integration-host", () => ({ getAllIntegrations: () => [] })); +vi.mock("../service-registry", () => ({ + getServiceRegistry: () => ({ + list: () => state.services, + get: (id: string) => state.services.find((service) => service.manifest.id === id), + applyEnabledIds: (ids: Set) => { + for (const service of state.services) service.enabled = ids.has(service.manifest.id); + }, + }), +})); +const writer = await import("../administrative-configuration-writer"); +const prospective = await import("../prospective-configuration"); +const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); +const { encrypt } = await import("../secrets"); + +describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "actual API candidate / production agent / PostgreSQL preparation", + () => { + let fixture: Fixture; + let host: Awaited>; + let app: ReturnType; + let installation: ReturnType; + let dm: Awaited> | undefined; + let actualPosts: Map; + beforeEach(async (context) => { + fixture = await createConfigurationPostgresFixture(); + state.fixture = fixture; + const schemas = Object.fromEntries( + ["motis", "ops-agent", "app-api", "traefik", "postgis", "redis", "data-manager"].map( + (id) => [ + id, + { + type: "object", + properties: { + WORKERS: { type: "integer", default: 2 }, + TOKEN: { type: "string", "x-openmapx-secret": true }, + }, + }, + ], + ), + ); + if (context.task.name.includes("future private mount inspection")) + Object.assign( + schemas["data-manager"]!.properties, + Object.fromEntries( + Array.from({ length: 80 }, (_, index) => [ + `P${String(index).padStart(3, "0")}_${"K".repeat(85)}`, + { type: "string", "x-openmapx-secret": true }, + ]), + ), + ); + host = await ownedServiceHost({ + schemas, + actualComposeConfig: + context.task.name.includes("applied final") || + context.task.name.includes("contained final") || + context.task.name.includes("one ownership-only maintenance decision") || + context.task.name.includes("killed native API"), + withHardlinks: context.task.name.includes("durable nonempty"), + competeAfterDefaultCli: context.task.name.includes("competing generation=true"), + ...(context.task.name.includes("actual default CLI") + ? { + defaultCliEntry: fileURLToPath( + new URL("../../../../ops-agent/src/index.ts", import.meta.url), + ), + } + : {}), + }); + state.services = host.loaded; + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + actualPosts = new Map(); + app.addHook("onSend", async (request, _reply, payload) => { + const kind = (request.body as { operation?: { kind?: string } } | undefined)?.operation + ?.kind; + if (request.method === "POST" && request.url === "/v1/operations" && kind) + actualPosts.set(kind, (actualPosts.get(kind) ?? 0) + 1); + return payload; + }); + const baseUrl = await app.listen({ host: "127.0.0.1", port: 0 }); + const tokenFile = join(host.root, "api-token"); + writeFileSync(tokenFile, host.token, { mode: 0o600 }); + for (const [key, value] of Object.entries({ + OPS_AGENT_URL: baseUrl, + OPS_AGENT_TOKEN_FILE: tokenFile, + OPS_TRUSTED_CONFIG_DIR: host.queue, + OPS_TRUSTED_CONFIG_UID: String(statSync(host.queue).uid), + OPS_TRUSTED_CONFIG_GID: String(statSync(host.queue).gid), + OPENMAPX_SECRETS_KEY: "71".repeat(32), + })) + vi.stubEnv(key, value); + }); + afterEach(async () => { + await dm?.close(); + dm = undefined; + await app?.close(); + if (installation) closeControlInstallation(installation); + await fixture?.stop(); + if (host) rmSync(host.root, { recursive: true, force: true }); + vi.unstubAllEnvs(); + state.services = []; + }); + async function job(): Promise { + const jobId = randomUUID(), + execution = { owner: randomUUID(), version: 1 }; + await fixture.db.insert(schema.adminJob).values({ + id: jobId, + type: "service.apply", + status: "running", + result: { _execution: execution }, + }); + const ctx: JobContext = { + jobId, + jobType: "service.apply", + execution, + payload: {}, + signal: new AbortController().signal, + checkpointResult: { _execution: execution }, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (result, progress, admissionKey, stepAddress) => { + if ((result.opsProjection as { version?: number } | undefined)?.version === 3) + ctx.checkpointResult = await checkpointOwnedAdminJob( + jobId, + execution, + result, + progress, + admissionKey, + stepAddress, + ctx.checkpointResult ?? {}, + ); + else { + await fixture.db + .update(schema.adminJob) + .set({ result }) + .where(eq(schema.adminJob.id, jobId)); + ctx.checkpointResult = result; + } + }, + }; + return ctx; + } + function owned(ctx: JobContext, run: () => Promise) { + return writer.withAdministrativeConfigurationJob(ctx, () => + writer.withAdministrativeConfigurationWriter( + { + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + integrationIds: [], + }, + run, + ), + ); + } + it.each([ + "saved", + "applied", + "missing", + "corrupt", + "hardlink", + "credential-saved", + "credential-recreated", + ])( + "reclaims a killed native ordinary configuration without resealing or saving again (mode=%s)", + async (mode) => { + const key = `opk1_native_ordinary_${randomUUID().replaceAll("-", "")}`, + database = new URL(process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL!); + database.searchParams.set("search_path", fixture.schema); + const credential = mode.startsWith("credential-"), + afterRecreate = mode === "credential-recreated", + applyKind = credential ? "vault.apply" : "serviceConfig.apply", + readVersions = () => + credential + ? fixture.sql`SELECT xmin::text AS version FROM service_secret WHERE service_id='motis'` + : fixture.sql`SELECT xmin::text AS version FROM service_config WHERE service_id='motis'`; + const relay = await postgresCommitRelay(database, async () => { + const rows = + await fixture.sql`SELECT result FROM admin_job WHERE payload->>'operationKey'=${key} LIMIT 2`; + if ( + rows.length !== 1 || + rows[0]!.result?._configurationCandidate?.phase !== + (afterRecreate ? "follow_up_pending" : "saved") || + rows[0]!.result?._configurationCandidate?.maintenanceId !== null + ) + return false; + return ( + (mode !== "applied" && !afterRecreate) || + Object.values(rows[0]!.result?.opsProjection?.operations ?? {}).some((value) => { + const operation = value as { + kind?: string; + terminal?: boolean; + receiptDelivery?: { state?: string }; + }; + return ( + operation.kind === (afterRecreate ? "service.recreate" : applyKind) && + operation.terminal === true && + operation.receiptDelivery?.state === "pending" + ); + }) + ); + }); + const loader = createRequire(new URL("../../../package.json", import.meta.url)).resolve( + "tsx/esm", + ), + script = fileURLToPath( + new URL("./fixtures/native-configuration-api.ts", import.meta.url), + ), + children: ReturnType[] = []; + const start = () => { + const args = ["--import", loader, script], + child = spawn(process.execPath, args, { + cwd: host.root, + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + DATABASE_URL: relay.databaseUrl, + OPENMAPX_ROOT_DIR: host.root, + NODE_ENV: "test", + BETTER_AUTH_SECRET: "synthetic-task012-native-api-auth-fixture", + }, + }); + children.push(child); + child.stdout!.on("data", () => {}); + child.stderr!.on("data", () => {}); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => { + console.info( + JSON.stringify({ + proof: "native-ordinary-api-exit", + argv: [process.execPath, ...args], + code, + signal, + }), + ); + resolve({ code, signal }); + }); + }, + ); + child.stdin!.end( + JSON.stringify({ + operationKey: key, + action: credential ? "ordinary-service-secret" : "ordinary-service-config", + }), + ); + return { child, exited }; + }; + let readyObserver: ReturnType | undefined; + const first = start(); + try { + await Promise.race([ + relay.reached, + first.exited.then(() => { + throw Error("Native ordinary API exited before saved checkpoint"); + }), + ]); + const [original] = await fixture.db.select().from(schema.adminJob), + originalResult = original!.result as Record, + candidate = originalResult._configurationCandidate as Record, + path = join(host.queue, ".candidates", `${candidate.sealedOperationKey}.json`), + bytes = readFileSync(path), + record = JSON.parse(bytes.toString()).record, + parent = { + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }, + versions = await readVersions(); + const refused = ["missing", "corrupt", "hardlink"].includes(mode); + expect(versions).toHaveLength(1); + expect(candidate.phase).toBe(afterRecreate ? "follow_up_pending" : "saved"); + expect(actualPosts.get(applyKind) ?? 0).toBe(mode === "applied" || afterRecreate ? 1 : 0); + first.child.kill("SIGKILL"); + expect((await first.exited).signal).toBe("SIGKILL"); + relay.release(); + if (mode === "missing") rmSync(path); + if (mode === "corrupt") { + const value = JSON.parse(bytes.toString()); + value.mac = value.mac === "A".repeat(43) ? "B".repeat(43) : "A".repeat(43); + writeFileSync(path, JSON.stringify(value), { mode: 0o600 }); + } + if (mode === "hardlink") linkSync(path, join(host.root, "candidate-alias")); + const [retained] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)), + lease = Date.parse( + (retained!.result as Record)._observationLeaseUntil as string, + ); + expect(Number.isFinite(lease)).toBe(true); + while (Date.now() <= lease + 100) + await nativeDelay(Math.min(1000, lease + 101 - Date.now())); + if (!refused) expect(readFileSync(path).equals(bytes)).toBe(true); + let readyPublications = 0; + if (mode === "applied") { + expect(existsSync(join(host.queue, `${candidate.revisionId}.json`))).toBe(false); + readyObserver = watch(host.queue, (_event, name) => { + if ( + name?.toString() === `${candidate.revisionId}.json` && + existsSync(join(host.queue, name.toString())) + ) + readyPublications++; + }); + } + expect((await start().exited).code).toBe(refused ? 2 : 0); + const [finished] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)), + result = finished!.result as Record; + expect(finished!.status).toBe(refused ? "running" : "success"); + expect((result._execution as { version: number }).version).toBe(2); + if (refused) { + expect(canonicalJson(result._configurationCandidate) === canonicalJson(candidate)).toBe( + true, + ); + expect(result.observation).toMatchObject({ + state: "unknown", + reason: "recovery_required", + }); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(actualPosts.get(applyKind) ?? 0).toBe(0); + expect(await readVersions()).toEqual(versions); + const next = await job(); + await expect(owned(next, async () => undefined)).rejects.toThrow( + "Configuration writer unavailable", + ); + return; + } + expect(result._parentExecution).toEqual(parent); + expect(result._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + phase: "settled", + }); + expect(await readVersions()).toEqual(versions); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(candidate.revisionId); + expect(actualPosts.get(applyKind)).toBe(1); + expect(existsSync(path)).toBe(false); + if (credential) { + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + expect(actualPosts.get("service.recreate")).toBe(1); + } else + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect((await start().exited).code).toBe(0); + const [duplicate] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + expect(duplicate!.result).toEqual(result); + expect(actualPosts.get(applyKind)).toBe(1); + expect(readyPublications).toBe(0); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + console.info( + JSON.stringify({ + proof: "native-ordinary-candidate-reclaimed", + originalJobId: original!.id, + ownerVersion: 2, + originalCandidateId: candidate.candidateId, + applyPosts: actualPosts.get(applyKind), + desiredRowVersionUnchanged: true, + nextWriterAvailable: true, + }), + ); + } finally { + readyObserver?.close(); + relay.release(); + for (const child of children) + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await relay.close(); + } + }, + 180000, + ); + it.each([ + "before-save", + "during-save", + "after-save", + "after-settle", + "after-ack-loss", + "before-root-start", + "before-root-start-source", + "before-root-start-daemon", + "before-root-start-cancel-daemon", + "before-save-cancel-daemon", + "before-save-refused", + "after-retirement-daemon", + "before-retirement-daemon", + "known-rollback-daemon", + "unused-settled-daemon", + "contained-retirement-daemon", + "before-root-start-unattempted-cancel-daemon", + "before-root-start-unattempted-daemon", + ])( + "reclaims the same killed native API through its durable nonempty original preparation (mode=%s)", + async (mode) => { + const preRootHelper = mode.startsWith("before-root-start"), + beforeSave = mode.startsWith("before-save"), + refusedPreparation = mode === "before-save-refused", + contained = mode === "contained-retirement-daemon", + retired = mode === "after-retirement-daemon" || contained, + retirement = mode.includes("retirement"), + unusedAfterSettle = mode === "unused-settled-daemon", + knownRollback = mode === "known-rollback-daemon" || unusedAfterSettle, + unattemptedHelper = mode.includes("before-root-start-unattempted"), + unused = beforeSave || knownRollback; + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const initialCoordinator = { + ...host.row, + Id: "c".repeat(64), + State: preRootHelper ? "exited" : "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + if (contained) { + const args = services.stackComposeArgs(host.stack), + options = { + signal: new AbortController().signal, + timeout: 15000, + maxBuffer: 1024 * 1024, + }, + config = JSON.parse( + (await host.execFile("docker", [...args, "config", "--format", "json"], options)) + .stdout, + ), + hash = ( + await host.execFile("docker", [...args, "config", "--hash", "data-manager"], options) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!, + spec = config.services["data-manager"]; + Object.assign(initialCoordinator.Labels, { "com.docker.compose.config-hash": hash }); + host.setRuntimeInspections({ + [initialCoordinator.Id]: { + Id: initialCoordinator.Id, + Image: initialCoordinator.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: initialCoordinator.ConfigImage, + Labels: initialCoordinator.Labels, + Env: Object.entries(spec.environment ?? {}).map( + ([key, value]) => `${key}=${value}`, + ), + }, + Mounts: (spec.volumes ?? []).map( + (value: { source: string; target: string; type: string; read_only?: boolean }) => ({ + Type: value.type, + Source: value.source, + Destination: value.target, + RW: !value.read_only, + }), + ), + }, + }); + } + host.setRows([host.row, initialCoordinator]); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-native-api-fixture"); + const prepareObserved = Promise.withResolvers(), + releaseReply = Promise.withResolvers(), + actualUrl = process.env.OPS_AGENT_URL!, + key = `opk1_native_api_${randomUUID().replaceAll("-", "")}`; + const busyId = refusedPreparation ? randomUUID() : undefined; + let busyBefore: unknown; + let holdPreparedReply = + beforeSave || + mode === "after-settle" || + mode === "after-ack-loss" || + preRootHelper || + unusedAfterSettle, + preparePosts = 0, + settlePosts = 0, + originalPrepareKey: string | undefined, + ackInterrupted = false; + const { AnchoredDirectory } = await import( + "../../../../../packages/core/src/build-artifacts/filesystem" + ), + writeCell = AnchoredDirectory.prototype.write, + ackFault = + mode === "after-ack-loss" + ? vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(function ( + this: InstanceType, + name, + bytes, + ...args + ) { + if ( + !ackInterrupted && + originalPrepareKey && + name === `${originalPrepareKey}.configuration-settlement-ack.json` && + Buffer.byteLength(bytes) === 1024 && + typeof JSON.parse(String(bytes))?.payload?.terminalDigest === "string" + ) { + ackInterrupted = true; + throw Error("Synthetic original native sender ACK filesystem interruption"); + } + return writeCell.call(this, name, bytes, ...args); + }) + : undefined; + const relay = createServer(async (request, response) => { + try { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = Buffer.concat(chunks), + headers = Object.fromEntries( + Object.entries(request.headers).filter( + (entry): entry is [string, string] => + typeof entry[1] === "string" && + !["host", "content-length", "connection"].includes(entry[0]), + ), + ), + forwarded = await fetch(actualUrl + request.url, { + method: request.method, + headers, + ...(body.length ? { body } : {}), + }), + bytes = await forwarded.text(); + if ( + request.method === "POST" && + body.length && + JSON.parse(body.toString()).operation?.kind === "configurationMaintenance.prepare" + ) { + preparePosts++; + originalPrepareKey = JSON.parse(body.toString()).operationKey; + } + if ( + request.method === "POST" && + body.length && + JSON.parse(body.toString()).operation?.kind === "configurationMaintenance.settle" + ) + settlePosts++; + if ( + unattemptedHelper && + holdPreparedReply && + request.method === "POST" && + body.length && + JSON.parse(body.toString()).operation?.kind === "configuration.hardlinks.preview" + ) { + const [original] = + await fixture.sql`SELECT result FROM admin_job WHERE payload->>'operationKey'=${key}`; + if ( + original?.result?._configurationCandidate?.phase === "prepared" && + !Object.values(original.result?.opsProjection?.operations ?? {}).some( + (value) => (value as { kind?: string }).kind === "service.start", + ) + ) { + prepareObserved.resolve(); + await releaseReply.promise; + } + } + const parsed = JSON.parse(bytes); + const wantedObservation = refusedPreparation + ? parsed.result?.kind === "configurationMaintenance.prepare" && + parsed.result?.state === "failed" + : preRootHelper + ? parsed.result?.kind === "service.start" && parsed.result?.state === "succeeded" + : mode === "after-ack-loss" + ? parsed.result?.kind === "configurationMaintenance.settle" && + parsed.result?.state === "failed" + : parsed.result?.kind === + (beforeSave + ? "configurationMaintenance.prepare" + : "configurationMaintenance.settle") && + parsed.result?.state === "succeeded" && + parsed.result?.result?.state === (beforeSave ? "prepared" : "settled"); + if (holdPreparedReply && request.method === "GET" && wantedObservation) { + prepareObserved.resolve(); + await releaseReply.promise; + } + if (!response.destroyed) { + response.writeHead(forwarded.status, { "content-type": "application/json" }); + response.end(bytes); + } + } catch { + if (!response.destroyed) { + response.writeHead(503); + response.end("{}"); + } + } + }); + await new Promise((resolve) => relay.listen(0, "127.0.0.1", resolve)); + const address = relay.address(); + if (!address || typeof address === "string") throw Error("Native API relay unavailable"); + const database = new URL(process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL!); + database.searchParams.set("search_path", fixture.schema); + let originalBackendPid: number | undefined, racedBackendPid: number | undefined; + let rollbackRaceRows: { id: string; version: string }[] | undefined; + const sqlRelay = + mode === "after-save" || mode === "during-save" || retirement || knownRollback + ? await postgresCommitRelay( + database, + async (observation) => { + const rows = + await fixture.sql`SELECT result FROM admin_job WHERE payload->>'operationKey'=${key} LIMIT 2`; + if (rows.length !== 1) return false; + if (knownRollback) { + if (observation.command === "BEFORE_SERVICE_SECRET_INSERT") { + expect(rows[0]!.result?._configurationCandidate?.phase).toBe( + "save_attempted", + ); + expect(rows[0]!.result?._configurationMaintenance?.phase).toBe("prepared"); + racedBackendPid = observation.backendPid; + await fixture.db.insert(schema.serviceSecret).values({ + id: randomUUID(), + serviceId: "data-manager", + key: "TOKEN", + ...encrypt("synthetic-independent-race"), + }); + rollbackRaceRows = await fixture.sql< + { id: string; version: string }[] + >`SELECT id,xmin::text AS version FROM service_secret WHERE service_id='data-manager' AND key='TOKEN'`; + return false; + } + return ( + !unusedAfterSettle && + observation.backendPid === racedBackendPid && + rows[0]!.result?._configurationCandidate?.phase === "rolled_back" && + rows[0]!.result?._configurationMaintenanceRollback?.disposition === + "known_rolled_back" + ); + } + if (retirement) { + const c = rows[0]!.result?._configurationCandidate; + return ( + c?.phase === "settled" && + (retired + ? !existsSync( + join(host.queue, ".candidates", `${c.sealedOperationKey}.json`), + ) + : c.retirement?.version === 1 && + existsSync( + join(host.queue, ".candidates", `${c.sealedOperationKey}.json`), + )) + ); + } + if (mode === "during-save") { + if (rows[0]!.result?._configurationCandidate?.phase !== "save_attempted") + return false; + const [actual] = + await fixture.sql`SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=${observation.backendPid} + AND state='idle in transaction' AND query LIKE 'insert into "service_secret"%') AS original_insert_pending`; + if (actual?.original_insert_pending !== true) return false; + originalBackendPid = observation.backendPid; + return true; + } + return ( + rows[0]!.result?._configurationCandidate?.phase === "saved" && + rows[0]!.result?._configurationMaintenanceDecision?.attempt === "not_attempted" + ); + }, + { + ...(knownRollback ? { observeFrontend: "service-secret-insert" as const } : {}), + holdCommand: retirement + ? retired + ? "API_TERMINAL_UPDATE" + : "UPDATE" + : mode === "during-save" + ? "INSERT" + : "COMMIT", + }, + ) + : undefined; + let nativeRevision: string | undefined; + host.setCreatingProcess(async (creatingArgs) => { + if (preRootHelper && !creatingArgs.includes("--force-recreate")) { + expect(creatingArgs.at(-1)).toBe("data-manager"); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(dm!.preparePosts).toBe(0); + initialCoordinator.State = "running"; + host.setRows([host.row, initialCoordinator]); + return; + } + if (!nativeRevision) { + const [saved] = + await fixture.sql`SELECT result FROM admin_job WHERE payload->>'operationKey'=${key}`; + nativeRevision = saved?.result?._configurationCandidate?.revisionId; + } + if (!nativeRevision) throw Error("Original native candidate revision unavailable"); + const args = services.stackComposeArgs(host.stack), + options = { + signal: new AbortController().signal, + timeout: 15000, + maxBuffer: 1024 * 1024, + }, + config = JSON.parse( + (await host.execFile("docker", [...args, "config", "--format", "json"], options)) + .stdout, + ), + hash = ( + await host.execFile("docker", [...args, "config", "--hash", "data-manager"], options) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!, + compose = services.configurationGenerationComposeFile(host.infraDir, nativeRevision), + service = config.services["data-manager"], + row = { + ...host.row, + Id: "d".repeat(64), + ConfigImage: "example/data-manager:1", + Labels: { + ...host.row.Labels, + "com.docker.compose.service": "data-manager", + "com.docker.compose.project.working_dir": join( + host.infraDir, + ".trusted-config-generations", + nativeRevision, + ), + "com.docker.compose.project.config_files": compose, + "com.docker.compose.config-hash": hash, + }, + }; + host.setRows([host.row, row]); + host.setRuntimeInspections({ + [row.Id]: { + Id: row.Id, + Image: row.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: row.ConfigImage, + Labels: row.Labels, + Env: Object.entries(service.environment ?? {}).map( + ([key, value]) => `${key}=${value}`, + ), + }, + Mounts: [ + ...(service.volumes ?? []).map( + (value: { + source: string; + target: string; + type: string; + read_only?: boolean; + }) => ({ + Type: value.type, + Source: value.source, + Destination: value.target, + RW: !value.read_only, + }), + ), + ...(service.secrets ?? []).map((value: { source: string; target: string }) => ({ + Type: "bind", + Source: config.secrets[value.source].file, + Destination: `/run/secrets/${value.target}`, + RW: false, + })), + ], + }, + }); + }); + const apiRequire = createRequire(new URL("../../../package.json", import.meta.url)), + loader = apiRequire.resolve("tsx/esm"), + script = fileURLToPath( + new URL("./fixtures/native-configuration-api.ts", import.meta.url), + ), + children: ReturnType[] = []; + const start = (action?: "reconcile") => { + const args = ["--import", loader, script], + child = spawn(process.execPath, args, { + cwd: host.root, + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + DATABASE_URL: sqlRelay?.databaseUrl ?? database.toString(), + OPENMAPX_ROOT_DIR: host.root, + OPS_AGENT_URL: `http://127.0.0.1:${address.port}`, + NODE_ENV: "test", + BETTER_AUTH_SECRET: "synthetic-task012-native-api-auth-fixture", + }, + }); + children.push(child); + let output = ""; + child.stdout!.on("data", (chunk) => { + if (output.length < 8192) output += chunk.toString(); + }); + child.stderr!.on("data", () => {}); + const exited = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + output: string; + }>((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => { + console.info( + JSON.stringify({ + proof: "native-api-producer-exit", + argv: [process.execPath, ...args], + code, + signal, + }), + ); + resolve({ code, signal, output }); + }); + }); + child.stdin!.end( + JSON.stringify({ + operationKey: key, + ...(action + ? { action } + : preRootHelper + ? { action: "maintenance-ensure-running" } + : {}), + }), + ); + return { child, exited }; + }; + if (busyId) { + await fixture.sql.begin(async (tx) => { + const [lock] = await tx`SELECT pg_try_advisory_xact_lock(${0x4f4d58},${2}) AS acquired`; + expect(lock!.acquired).toBe(true); + await tx`INSERT INTO data_manager.jobs(id,kind,status,started_at) VALUES(${busyId}::uuid,'download-fonts','running',now())`; + }); + busyBefore = + await fixture.sql`SELECT *,xmin::text AS row_version FROM data_manager.jobs WHERE id=${busyId}::uuid`; + } + const nativeRename = (await vi.importActual("node:fs")) + .renameSync; + let containedRenameFaults = 0; + const containedFault = contained + ? vi.spyOn(nodeFs, "renameSync").mockImplementation((source, target) => { + if ( + basename(String(target)).startsWith("cfg1_") && + basename(String(source)).endsWith(".tmp") + ) { + containedRenameFaults++; + throw Error("External native generation publication refused"); + } + return nativeRename(source, target); + }) + : undefined; + const first = start(); + try { + await Promise.race([ + unusedAfterSettle + ? prepareObserved.promise + : (sqlRelay?.reached ?? prepareObserved.promise), + first.exited.then(async (exit) => { + const [row] = await fixture.db.select().from(schema.adminJob), + result = row?.result as Record | undefined; + const projection = (await import("../admin-job-projection")).readProjection(result); + console.info( + JSON.stringify({ + proof: "native-before-interruption-diagnostic", + mode, + exitCode: exit.code, + fixtureOutput: exit.output.trim(), + status: row?.status, + candidatePhase: (result?._configurationCandidate as { phase?: string })?.phase, + maintenancePhase: (result?._configurationMaintenance as { phase?: string }) + ?.phase, + operations: Object.values(projection.operations).map((value) => ({ + kind: value.kind, + phase: value.phase, + errorClass: value.errorClass, + })), + prepares: preparePosts, + settlements: settlePosts, + }), + ); + throw Error("Native API exited before original checkpoint observation"); + }), + ]); + const [original] = await fixture.db.select().from(schema.adminJob); + const originalResult = original!.result as Record, + parent = originalResult._parentExecution, + candidate = originalResult._configurationCandidate as Record, + remote = (originalResult._configurationMaintenance ?? { + maintenanceId: candidate.maintenanceId, + }) as Record; + const originalOwner = + mode === "after-settle" || mode === "after-ack-loss" || retirement || unusedAfterSettle + ? ( + await import("../../../../ops-agent/src/configuration-maintenance-store") + ).readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution( + installation, + (remote.receipt as import("@openmapx/core/ops").ConfigurationMaintenanceReceipt) + .prepare, + ), + ) + : null; + const candidatePath = join( + host.queue, + ".candidates", + `${candidate.sealedOperationKey}.json`, + ), + candidateBytes = retired ? undefined : readFileSync(candidatePath), + rootLock = join(host.infraDir, "data/.admin-target-control/locks/data-manager"), + rootInode = existsSync(rootLock) ? statSync(rootLock).ino : null; + expect(rootInode === null).toBe( + mode === "after-settle" || + preRootHelper || + refusedPreparation || + retirement || + unusedAfterSettle, + ); + const refusedOwnerPath = join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${candidate.maintenanceId}.json`, + ), + refusedOwnerBytes = refusedPreparation ? readFileSync(refusedOwnerPath) : undefined; + const helper = preRootHelper + ? JSON.parse(candidateBytes!.toString()).record.coordinatorPrerequisite + : null, + helperAdmission = + helper && !unattemptedHelper + ? readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${helper.root.operationKey}.json`, + ), + ) + : null; + if (helper) { + expect(preparePosts).toBe(0); + expect(dm!.preparePosts).toBe(0); + expect(actualPosts.get("service.start") ?? 0).toBe(unattemptedHelper ? 0 : 1); + if (unattemptedHelper) + expect( + existsSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${helper.root.operationKey}.json`, + ), + ), + ).toBe(false); + } + nativeRevision = String(candidate.revisionId); + expect(candidate.phase).toBe( + beforeSave || preRootHelper + ? "prepared" + : mode === "during-save" + ? "save_attempted" + : retirement + ? "settled" + : knownRollback + ? "rolled_back" + : "saved", + ); + expect(original!.status).toBe("running"); + const savedRows = + await fixture.sql`SELECT xmin::text AS version FROM service_secret WHERE service_id='data-manager' AND key='TOKEN'`; + expect(savedRows).toHaveLength( + beforeSave || preRootHelper || mode === "during-save" ? 0 : 1, + ); + first.child.kill("SIGKILL"); + expect((await first.exited).signal).toBe("SIGKILL"); + if (mode.endsWith("-cancel-daemon")) { + await (await import("../job-runner")).jobRunner.cancel(original!.id); + const [canceled] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + expect(canceled!.status).toBe("cancel_pending"); + expect((canceled!.result as Record)._cancelRequestedAt).toBeTypeOf( + "string", + ); + } + if (mode === "after-ack-loss") expect(ackInterrupted).toBe(true); + ackFault?.mockRestore(); + holdPreparedReply = false; + releaseReply.resolve(); + sqlRelay?.release(); + const [retained] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + const lease = Date.parse( + (retained!.result as Record)._observationLeaseUntil as string, + ); + expect(Number.isFinite(lease)).toBe(true); + while (Date.now() <= lease + 100) + await nativeDelay(Math.min(1000, lease + 101 - Date.now())); + expect( + retired + ? !existsSync(candidatePath) + : readFileSync(candidatePath).equals(candidateBytes!), + ).toBe(true); + expect(existsSync(rootLock) ? statSync(rootLock).ino : null).toBe(rootInode); + if (mode === "before-root-start-source") + await fixture.db + .insert(schema.serviceConfig) + .values({ id: randomUUID(), serviceId: "motis", config: { WORKERS: 7 } }); + const resumed = await start(mode.endsWith("-daemon") ? "reconcile" : undefined).exited; + expect(resumed.code).toBe( + contained || + unused || + mode === "during-save" || + mode === "before-root-start-source" || + mode === "before-root-start-cancel-daemon" || + (unattemptedHelper && mode.endsWith("-cancel-daemon")) + ? 2 + : 0, + ); + const [finished] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)), + result = finished!.result as Record; + if (refusedPreparation) { + expect(finished!.status).toBe("failed"); + expect(result._execution).toMatchObject({ version: 2 }); + expect(result._parentExecution).toEqual(parent); + expect(result._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + expect(result.configuration).toMatchObject({ + outcome: "not_committed", + reason: "busy", + }); + expect(result._configurationMaintenance).toMatchObject({ + phase: "settled", + receipt: { state: "settled", dmFence: null, decision: null }, + }); + expect(result._configurationMaintenanceDecision).toBeUndefined(); + expect(settlePosts).toBe(0); + expect(dm!.settlePosts).toBe(0); + expect(preparePosts).toBe(1); + expect(dm!.preparePosts).toBe(1); + expect( + await fixture.sql`SELECT *,xmin::text AS row_version FROM data_manager.jobs WHERE id=${busyId!}::uuid`, + ).toEqual(busyBefore); + expect(readFileSync(refusedOwnerPath).equals(refusedOwnerBytes!)).toBe(true); + const core = await import("@openmapx/core/build-artifacts"), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records" + ), + owner = core.verifyControl( + installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + JSON.parse(refusedOwnerBytes!.toString()), + records.configurationMaintenanceOwnerRecordSchema, + ), + authority = restoreAdmittedControlExecution(installation, owner.plan.prepare); + core.assertConfigurationMaintenanceRootClosure(authority); + core.verifyRetainedArtifactTargetRootRelease(authority); + expect(existsSync(candidatePath)).toBe(false); + expect(existsSync(rootLock)).toBe(false); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect((await start().exited).code).toBe(2); + const [duplicate] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + expect(duplicate!.result).toEqual(finished!.result); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + console.info( + JSON.stringify({ + proof: "native-api-original-noguard-refusal-reclaimed", + originalJobId: original!.id, + prepares: 1, + dmSettlements: 0, + noGuardOwnerUnchanged: true, + winningSqlRowUnchanged: true, + desiredSecretRows: 0, + originalCandidateDiscarded: true, + nextWriterAvailable: true, + }), + ); + return; + } + if (mode === "before-save-cancel-daemon") { + expect(resumed.output).toContain('"state":"observed","status":"canceled"'); + expect(finished!.status).toBe("canceled"); + expect(result._execution).toMatchObject({ version: 2 }); + expect(result._parentExecution).toEqual(parent); + expect(result._cancelRequestedAt).toBeTypeOf("string"); + expect(result._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + expect(result.configuration).toMatchObject({ outcome: "not_committed" }); + expect(result._configurationMaintenance).toMatchObject({ + maintenanceId: remote.maintenanceId, + receipt: { + state: "settled", + decision: { decision: "abandon" }, + dmFence: { state: "settled" }, + generation: "not_started", + followUp: "not_started", + }, + }); + expect(result._configurationMaintenanceDecision).toMatchObject({ + operation: { decision: "abandon" }, + attempt: "attempted", + }); + expect(preparePosts).toBe(1); + expect(settlePosts).toBe(1); + expect(dm!.preparePosts).toBe(1); + expect(dm!.settlePosts).toBe(1); + expect(actualPosts.get("service.start") ?? 0).toBe(0); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect(existsSync(candidatePath)).toBe(false); + expect(existsSync(rootLock)).toBe(false); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + const final = ( + result._configurationMaintenance as { + receipt: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + } + ).receipt, + core = await import("@openmapx/core/build-artifacts"); + expect(final.prepare.operationKey).toBe(candidate.prepareOperationKey); + core.verifyRetainedArtifactTargetRootRelease( + restoreAdmittedControlExecution(installation, final.prepare), + ); + expect((await start().exited).code).toBe(2); + const [duplicate] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + expect(duplicate!.result).toEqual(finished!.result); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + console.info( + JSON.stringify({ + proof: "native-prepared-api-canceled-unused", + originalJobId: original!.id, + originalRootOnly: true, + prepares: 1, + dmSettlements: 1, + desiredSecretRows: 0, + originalCandidateDiscarded: true, + nextWriterAvailable: true, + }), + ); + return; + } + if ( + mode === "before-root-start-cancel-daemon" || + (unattemptedHelper && mode.endsWith("-cancel-daemon")) + ) { + expect(resumed.output).toContain('"state":"observed","status":"canceled"'); + expect(finished!.status).toBe("canceled"); + expect(result._execution).toMatchObject({ version: 2 }); + expect(result._parentExecution).toEqual(parent); + expect(result._cancelRequestedAt).toBeTypeOf("string"); + expect(result._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + expect(result.configuration).toMatchObject({ outcome: "not_committed" }); + expect(result._configurationMaintenance).toBeUndefined(); + expect(result._configurationMaintenanceDecision).toBeUndefined(); + expect(preparePosts).toBe(0); + expect(settlePosts).toBe(0); + expect(dm!.preparePosts).toBe(0); + expect(dm!.settlePosts).toBe(0); + expect(actualPosts.get("service.start") ?? 0).toBe(unattemptedHelper ? 0 : 1); + expect(host.calls.filter((args) => args.includes("up"))).toHaveLength( + unattemptedHelper ? 0 : 1, + ); + if (!unattemptedHelper) + expect( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${helper.root.operationKey}.json`, + ), + ).equals(helperAdmission!), + ).toBe(true); + expect(existsSync(candidatePath)).toBe(false); + expect(existsSync(rootLock)).toBe(false); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect((await start().exited).code).toBe(2); + const [duplicate] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + expect(duplicate!.result).toEqual(finished!.result); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + console.info( + JSON.stringify({ + proof: "native-pre-root-helper-canceled", + originalJobId: original!.id, + sameHelperAdmission: unattemptedHelper ? null : true, + starts: unattemptedHelper ? 0 : 1, + maintenancePosts: 0, + desiredSecretRows: 0, + originalCandidateDiscarded: true, + nextWriterAvailable: true, + }), + ); + return; + } + if (mode === "before-root-start-source") { + expect(finished!.status).toBe("failed"); + expect(result._execution).toMatchObject({ version: 2 }); + expect(result._parentExecution).toEqual(parent); + expect(result._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + expect(result.configuration).toMatchObject({ + outcome: "not_committed", + reason: "conflict", + }); + expect(result._configurationMaintenance).toBeUndefined(); + expect(result._configurationMaintenanceDecision).toBeUndefined(); + expect(preparePosts).toBe(0); + expect(settlePosts).toBe(0); + expect(dm!.preparePosts).toBe(0); + expect(dm!.settlePosts).toBe(0); + expect(actualPosts.get("service.start")).toBe(1); + expect(host.calls.filter((args) => args.includes("up"))).toHaveLength(1); + expect( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${helper.root.operationKey}.json`, + ), + ).equals(helperAdmission!), + ).toBe(true); + expect(existsSync(candidatePath)).toBe(false); + expect(existsSync(rootLock)).toBe(false); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect((await start().exited).code).toBe(2); + const [duplicate] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + expect(duplicate!.result).toEqual(finished!.result); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + console.info( + JSON.stringify({ + proof: "native-pre-root-helper-source-conflict", + originalJobId: original!.id, + sameHelperAdmission: true, + starts: 1, + maintenancePosts: 0, + desiredSecretRows: 0, + originalCandidateDiscarded: true, + nextWriterAvailable: true, + }), + ); + return; + } + if (mode === "during-save") { + expect(originalBackendPid).toBeTypeOf("number"); + expect( + await fixture.sql`SELECT pid FROM pg_stat_activity WHERE pid=${originalBackendPid!}`, + ).toHaveLength(0); + expect(finished!.status).toBe("running"); + expect(result._execution).toMatchObject({ version: 2 }); + expect(canonicalJson(result._configurationCandidate) === canonicalJson(candidate)).toBe( + true, + ); + expect(result.observation).toMatchObject({ + state: "unknown", + reason: "recovery_required", + }); + expect(result._configurationMaintenanceDecision).toBeUndefined(); + expect(result._configurationMaintenanceRollback).toBeUndefined(); + expect(result._configurationMaintenance).toMatchObject({ + receipt: { state: "prepared", decision: null, generation: "not_started" }, + }); + expect(readFileSync(candidatePath).equals(candidateBytes!)).toBe(true); + expect(statSync(rootLock).ino).toBe(rootInode); + expect(preparePosts).toBe(1); + expect(settlePosts).toBe(0); + expect(dm!.settlePosts).toBe(0); + expect( + await fixture.sql`SELECT xmin::text AS version FROM service_secret WHERE service_id='data-manager' AND key='TOKEN'`, + ).toEqual(savedRows); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + const next = await job(); + await expect(owned(next, async () => undefined)).rejects.toThrow( + "Configuration writer unavailable", + ); + console.info( + JSON.stringify({ + proof: "native-api-attempted-save-remains-unknown", + originalJobId: original!.id, + candidateId: candidate.candidateId, + realOriginalTransactionTerminated: true, + desiredRows: 0, + abandonOrSettlePosts: 0, + nextWriterRefused: true, + }), + ); + return; + } + expect(finished!.status).toBe(unused || contained ? "failed" : "success"); + expect((result._execution as { version: number }).version).toBe(2); + expect(result._parentExecution).toEqual(parent); + expect(result._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + phase: unused ? "discarded" : "settled", + }); + expect(result._configurationMaintenance).toMatchObject({ + maintenanceId: remote.maintenanceId, + receipt: { + state: "settled", + decision: { decision: unused ? "abandon" : "continue" }, + dmFence: { state: "settled" }, + }, + }); + if (originalOwner) { + const finalOwner = ( + await import("../../../../ops-agent/src/configuration-maintenance-store") + ).readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, originalOwner.plan.prepare), + ), + canonical = (await import("@openmapx/core/ops")).canonicalJson; + expect(canonical(finalOwner.final) === canonical(originalOwner.final)).toBe(true); + expect(result._configurationMaintenanceSettle).toEqual( + originalResult._configurationMaintenanceSettle, + ); + expect(finalOwner.rootCertificateDigest).toBe(originalOwner.rootCertificateDigest); + } + if (knownRollback) { + expect(result._configurationMaintenanceRollback).toEqual( + originalResult._configurationMaintenanceRollback, + ); + expect( + await fixture.sql`SELECT id,xmin::text AS version FROM service_secret WHERE service_id='data-manager' AND key='TOKEN'`, + ).toEqual(rollbackRaceRows); + expect(result.configuration).toMatchObject({ outcome: "not_committed" }); + } + expect(preparePosts).toBe(1); + expect(await fixture.db.select().from(schema.adminJob)).toHaveLength(1); + if (preRootHelper) { + const retainedRecord = JSON.parse(candidateBytes!.toString()).record + .privateMaintenancePreparation.serviceSecrets[0], + [actualSaved] = await fixture.db.select().from(schema.serviceSecret); + expect( + actualSaved?.id === retainedRecord.id && + actualSaved?.ciphertext === retainedRecord.ciphertext && + actualSaved?.iv === retainedRecord.iv && + actualSaved?.tag === retainedRecord.tag, + ).toBe(true); + } else + expect( + await fixture.sql`SELECT xmin::text AS version FROM service_secret WHERE service_id='data-manager' AND key='TOKEN'`, + ).toEqual(savedRows); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + unused || contained ? host.initial.revisionId : nativeRevision, + ); + expect( + existsSync(join(host.queue, ".candidates", `${candidate.sealedOperationKey}.json`)), + ).toBe(false); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect(host.calls.filter((args) => args.includes("up"))).toHaveLength( + unused || contained ? 0 : preRootHelper ? 2 : 1, + ); + if (helper) { + expect(actualPosts.get("service.start")).toBe(1); + if (helperAdmission) + expect( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${helper.root.operationKey}.json`, + ), + ).equals(helperAdmission!), + ).toBe(true); + else { + const sourceControl = await import( + "../../../../../packages/core/src/build-artifacts/control-authority" + ); + expect(() => + sourceControl.verifyMaintenanceCoordinatorPrerequisite( + installation, + JSON.parse(candidateBytes!.toString()).record, + ), + ).not.toThrow(); + } + } + expect(host.calls.some((args) => args.includes("pull") || args.includes("restart"))).toBe( + false, + ); + if (contained) { + expect(containedRenameFaults).toBe(1); + expect(result.configuration).toMatchObject({ + outcome: "not_committed", + reason: "failed", + }); + } + expect((await start().exited).code).toBe(unused || contained ? 2 : 0); + const [duplicate] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, original!.id)); + expect(duplicate!.result).toEqual(finished!.result); + expect(preparePosts).toBe(1); + expect(settlePosts).toBe(1); + expect(dm!.settlePosts).toBe(1); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + console.info( + JSON.stringify({ + proof: "native-api-original-candidate-reclaimed", + mode, + originalJobId: original!.id, + ownerVersion: 2, + originalCandidateId: candidate.candidateId, + preparedPosts: preparePosts, + realObserverLeaseElapsed: true, + desiredRows: preRootHelper ? 1 : savedRows.length, + ...(knownRollback + ? { + actualAcknowledgedRollback: true, + independentRaceRowUnchanged: true, + candidateDesiredWrites: 0, + } + : {}), + sameCandidateUntilSettlement: true, + nextWriterAvailable: true, + }), + ); + } finally { + containedFault?.mockRestore(); + ackFault?.mockRestore(); + holdPreparedReply = false; + releaseReply.resolve(); + sqlRelay?.release(); + for (const child of children) + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + relay.closeAllConnections(); + await new Promise((resolve) => relay.close(() => resolve())); + await sqlRelay?.close(); + if (busyId) + await fixture.sql`DELETE FROM data_manager.jobs WHERE id=${busyId}::uuid AND kind='download-fonts'`; + } + }, + 240000, + ); + it("stages an inert exact candidate before a real desired-row transaction and actual agent generation commit", async () => { + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-config", serviceId: "motis", set: { WORKERS: 4 }, unset: [] }, + ], + }, + }); + expect(await fixture.db.select().from(schema.serviceConfig)).toHaveLength(0); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(1); + expect(readdirSync(host.queue).filter((name) => /^cfg1_/.test(name))).toHaveLength(0); + expect(host.calls.some((args) => args.includes("up") || args.includes("pull"))).toBe(false); + const receipt = await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + }); + expect(receipt).toMatchObject({ + saved: "committed", + revisionId: candidate.revisionId, + application: { + outcome: "committed", + revisionId: candidate.revisionId, + followUp: "complete", + }, + }); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(candidate.revisionId); + expect((await fixture.db.select().from(schema.serviceConfig))[0]?.config).toEqual({ + WORKERS: 4, + }); + const [saved] = + await fixture.sql`SELECT id,xmin::text AS version FROM service_config WHERE service_id='motis'`; + const savedCandidate = ctx.checkpointResult?._configurationCandidate as + | { rowVersions?: unknown } + | undefined; + expect(savedCandidate?.rowVersions).toEqual([ + { + kind: "service-config", + ownerId: "motis", + rowId: saved!.id, + rowVersion: saved!.version, + previousRowVersion: null, + state: "present", + }, + ]); + }); + }); + it("keeps a captured strict registry generation when the live service registry changes during a read", async () => { + const { resolveServiceConfigWithSourcesStrict } = await import("../service-config-resolver"); + await writer.withAdministrativeConfigurationRead({}, async (read) => { + const service = state.services.find((service) => service.manifest.id === "motis")!; + const properties = service.manifest.configSchema!.properties as Record< + string, + { default?: unknown } + >; + properties.WORKERS!.default = 91; + const captured = read.services.find((service) => service.manifest.id === "motis")!.manifest; + const resolved = await resolveServiceConfigWithSourcesStrict( + { id: captured.id, configSchema: captured.configSchema }, + read, + ); + expect(resolved.WORKERS?.value).toBe(2); + }); + }); + it("refuses consumable publication before the exact desired-row transaction commits", async () => { + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-config", serviceId: "motis", set: { WORKERS: 4 }, unset: [] }, + ], + }, + }); + const { publishPreparedTrustedConfigurationSnapshot } = await import( + "../trusted-config-publisher" + ); + // Diagnostic state is not commit authority, even for an internal caller. + prospective.validatedConfigurationCandidateState(candidate).persisted = true; + await expect(publishPreparedTrustedConfigurationSnapshot(candidate)).rejects.toThrow(); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual([]); + expect(readdirSync(host.queue).filter((name) => name.startsWith("cfg1_"))).toEqual([]); + }); + }); + it("refuses an oversized retained full envelope before a desired-row write or consumable publication", async () => { + for (const serviceId of ["motis", "ops-agent", "app-api", "traefik", "postgis"]) + await fixture.db + .insert(schema.serviceSecret) + .values({ id: randomUUID(), serviceId, key: "TOKEN", ...encrypt("ü".repeat(60000)) }); + const before = await fixture.db.select().from(schema.serviceConfig); + const ctx = await job(); + await expect( + owned(ctx, () => + prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-config", serviceId: "motis", set: { WORKERS: 4 }, unset: [] }, + ], + }, + }), + ), + ).rejects.toThrow(); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual(before); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + expect(readdirSync(host.queue).filter((name) => /^cfg1_/.test(name))).toHaveLength(0); + expect(host.calls.some((args) => args.includes("up") || args.includes("pull"))).toBe(false); + }); + it("refuses host-source drift after preview before any desired-row persistence", async () => { + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-config", serviceId: "motis", set: { WORKERS: 8 }, unset: [] }, + ], + }, + }); + writeFileSync(join(host.infraDir, ".env"), "MOTIS_SLOT_MEMORY=8g\n", { mode: 0o600 }); + const result = await prospective + .persistAndApplyProspectiveConfiguration(candidate, { coordinatorMode: "ensure-running" }) + .catch((error) => error); + expect(await fixture.db.select().from(schema.serviceConfig)).toHaveLength(0); + expect(result).toBeInstanceOf(Error); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + }); + }); + it("accepts a complete size-reducing correction of oversized unchanged legacy state", async () => { + for (const serviceId of ["motis", "ops-agent", "app-api", "traefik", "postgis"]) + await fixture.db + .insert(schema.serviceSecret) + .values({ id: randomUUID(), serviceId, key: "TOKEN", ...encrypt("ü".repeat(60000)) }); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-secret", serviceId: "postgis", key: "TOKEN", action: "delete" }, + ], + }, + }); + expect(candidate.sealedBytes).toBeLessThanOrEqual(524288); + const receipt = await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + }); + expect(receipt).toMatchObject({ + saved: "committed", + application: { outcome: "committed" }, + }); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(4); + }); + }); + it("runs the current immediate producer through a retained nonsecret API job and actual agent", async () => { + const { applyTrustedConfiguration } = await import("../trusted-config-operations"); + const { createDurableOpsKey } = await import("../ops-client"); + const key = createDurableOpsKey("task012.immediate", randomUUID()); + const result = await applyTrustedConfiguration({ + kind: "serviceConfig.apply", + serviceId: "motis", + actorId: "loopback-admin", + operationKey: key, + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [{ kind: "service-config", serviceId: "motis", set: { WORKERS: 7 }, unset: [] }], + }, + }); + expect(result.revisionId).toBe(services.currentConfigurationGeneration(host.infraDir)); + const [retained] = await fixture.db.select().from(schema.adminJob); + expect(retained).toMatchObject({ + type: "configuration.apply", + status: "success", + payload: { operationKey: key }, + }); + expect(JSON.stringify(retained?.payload)).not.toContain("WORKERS"); + expect( + (retained?.result as { _configurationCandidate?: { phase?: string } }) + ?._configurationCandidate?.phase, + ).toBe("settled"); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + }); + it("replays the retained immediate candidate result without publishing or reusing a changed intent", async () => { + const { applyTrustedConfiguration } = await import("../trusted-config-operations"); + const { createDurableOpsKey } = await import("../ops-client"); + const options = { + kind: "serviceConfig.apply" as const, + serviceId: "motis", + actorId: "loopback-admin", + operationKey: createDurableOpsKey("task012.immediate-replay", randomUUID()), + overlay: { + purpose: "service-config" as const, + coordinatorMode: "ensure-running" as const, + changes: [ + { kind: "service-config" as const, serviceId: "motis", set: { WORKERS: 7 }, unset: [] }, + ], + }, + }; + const result = await applyTrustedConfiguration(options), + calls = host.calls.length; + expect(await applyTrustedConfiguration(options)).toEqual(result); + expect(host.calls).toHaveLength(calls); + expect(await fixture.db.select().from(schema.adminJob)).toHaveLength(1); + await expect( + applyTrustedConfiguration({ + ...options, + overlay: { + ...options.overlay, + changes: [{ ...options.overlay.changes[0]!, set: { WORKERS: 9 } }], + }, + }), + ).rejects.toThrow("conflicts"); + const { jobRunner } = await import("../job-runner"); + await expect( + jobRunner.startImmediateCompose({ + actorId: options.actorId, + operationKey: options.operationKey, + }), + ).rejects.toThrow("conflicts"); + expect((await fixture.db.select().from(schema.serviceConfig))[0]?.config).toEqual({ + WORKERS: 7, + }); + }); + it("publishes one credential seed and recreates from it in the same retained job without another render", async () => { + const { applyTrustedConfiguration } = await import("../trusted-config-operations"), + { createDurableOpsKey } = await import("../ops-client"); + const before = readdirSync(join(host.infraDir, ".trusted-config-generations")).length; + host.setCreatingProcess(async () => { + const connection = await fixture.sql.reserve(); + let held = false; + try { + const [row] = await connection`SELECT pg_try_advisory_lock(${0x4f4d58},1) AS held`; + held = row!.held === true; + expect(held).toBe(false); + const [checkpoint] = + await connection`SELECT type, result->'_configurationCandidate'->>'phase' AS phase, + result->'_completion' AS completion FROM admin_job`; + expect(checkpoint!.phase).toBe("follow_up_pending"); + expect(checkpoint!.type).toBe("service.apply"); + expect(checkpoint!.completion).toMatchObject({ + result: { serviceCredential: { serviceId: "motis", followUp: "complete" } }, + }); + } finally { + if (held) await connection`SELECT pg_advisory_unlock(${0x4f4d58},1)`; + connection.release(); + } + }); + const options = { + kind: "vault.apply" as const, + serviceId: "motis", + actorId: "loopback-admin", + followUp: "service-apply" as const, + operationKey: createDurableOpsKey("task012.credential-apply", randomUUID()), + overlay: { + purpose: "service-secret" as const, + coordinatorMode: "ensure-running" as const, + changes: [ + { + kind: "service-secret" as const, + serviceId: "motis", + key: "TOKEN", + action: "set" as const, + value: "synthetic-private-credential", + }, + ], + }, + }; + const applied = await applyTrustedConfiguration(options); + expect(applied.revisionId).toBe(services.currentConfigurationGeneration(host.infraDir)); + expect(readdirSync(join(host.infraDir, ".trusted-config-generations"))).toHaveLength( + before + 1, + ); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + const [job] = await fixture.db.select().from(schema.adminJob); + expect((job?.result as Record)?.serviceCredential).toMatchObject({ + serviceId: "motis", + revisionId: applied.revisionId, + followUp: "complete", + }); + expect(await applyTrustedConfiguration(options)).toEqual(applied); + expect(await fixture.db.select().from(schema.adminJob)).toHaveLength(1); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + }); + it.each([false, true])( + "retains current Compose preparation and exactly one stack effect with host failure=%s", + async (fail) => { + const { jobRunner } = await import("../job-runner"); + const { createDurableOpsKey } = await import("../ops-client"); + if (fail) + host.setCreatingProcess(async () => { + throw Error("Synthetic external Docker failure"); + }); + const options = { + actorId: "loopback-admin", + operationKey: createDurableOpsKey("task012.compose", randomUUID()), + }; + const result = await jobRunner.startImmediateCompose(options); + expect(result).toMatchObject({ + ok: !fail, + stdout: "", + hardlinks: { via: "none", entries: 0 }, + }); + expect(await jobRunner.startImmediateCompose(options)).toEqual(result); + expect(host.calls.filter((args) => args.includes("up"))).toHaveLength(1); + const [retained] = await fixture.db.select().from(schema.adminJob); + expect(retained).toMatchObject({ + type: "compose.start", + status: fail ? "failed" : "success", + }); + expect( + ( + (retained?.result as Record)?._servicePreparations as Record< + string, + { revisionId: string } + > + )?.["compose-start"]?.revisionId, + ).toBe(services.currentConfigurationGeneration(host.infraDir)); + }, + ); + it("settles an actual asynchronous empty hardlink owner without a coordinator or Docker lifecycle", async () => { + const ctx = await job(); + const { executeAdminJobOperation } = await import("../admin-job-ops"); + const result = await executeAdminJobOperation( + ctx, + { + kind: "configuration.hardlinks.apply", + revisionId: host.initial.revisionId, + coordinatorMode: "require-running", + }, + "task012.empty-hardlinks", + { durableIdentity: host.initial.revisionId }, + ); + expect(result).toEqual({ + applied: true, + entries: 0, + linked: 0, + skipped: 0, + pruned: 0, + via: "none", + }); + expect(host.calls.some((args) => args.includes("up") || args.includes("restart"))).toBe( + false, + ); + }); + it.each(["missing-generation", "disabled-coordinator"] as const)( + "refuses a durable nonempty unavailable maintenance prerequisite before staging a writer fence (mode=%s)", + async (mode) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + if (mode === "disabled-coordinator") await host.select(["motis"]); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "synthetic-prerequisite-osm"); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "exited", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const pointer = join(host.infraDir, ".trusted-config-current"), + hidden = join(host.root, "fixture-hidden-current"); + // External availability fault on this synthetic host's actual pointer; + // immutable generation/config bytes are never rewritten. + if (mode === "missing-generation") nodeFs.renameSync(pointer, hidden); + try { + const expectedCurrentRevisionId = services.currentConfigurationGeneration(host.infraDir); + expect(expectedCurrentRevisionId === null).toBe(mode === "missing-generation"); + const key = `opk1_prerequisite_${randomUUID().replaceAll("-", "")}`; + await expect( + (await import("../trusted-config-operations")).applyTrustedConfiguration({ + kind: "vault.apply", + serviceId: "data-manager", + actorId: "loopback-admin", + operationKey: key, + expectedCurrentRevisionId, + followUp: "service-apply", + coordinatorMode: "ensure-running", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-prerequisite-secret", + }, + ], + }, + }), + ).rejects.toThrow(); + const [original] = await fixture.db.select().from(schema.adminJob); + expect(original?.status).toBe("failed"); + expect( + (original?.result as Record)?._configurationCandidate === undefined, + ).toBe(true); + const candidates = join(host.queue, ".candidates"); + expect(existsSync(candidates) ? readdirSync(candidates) : []).toEqual([]); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(actualPosts.get("service.start") ?? 0).toBe(0); + expect(actualPosts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(dm.preparePosts).toBe(0); + expect(dm.settlePosts).toBe(0); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + expectedCurrentRevisionId, + ); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + } finally { + if (existsSync(hidden) && !existsSync(pointer)) nodeFs.renameSync(hidden, pointer); + } + }, + 60000, + ); + it("refuses an actual durable nonempty candidate whose future private mount inspection exceeds capacity before any root or DM guard", async () => { + for (let index = 0; index < 80; index++) { + const key = `P${String(index).padStart(3, "0")}_${"K".repeat(85)}`; + await fixture.db.insert(schema.serviceSecret).values({ + id: randomUUID(), + serviceId: "data-manager", + key, + ...encrypt("synthetic-retained-secret"), + }); + } + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-prospective-capacity-credential", + }, + ], + }, + }); + expect(candidate.sealedBytes).toBeLessThan(524288); + const maintenance = await import("../configuration-maintenance"); + await expect(maintenance.prepareConfigurationMaintenance(candidate)).rejects.toBeInstanceOf( + Error, + ); + expect(dm!.preparePosts).toBe(0); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(80); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + }); + }, 60000); + it("uses one ownership-only maintenance decision before a durable nonempty coordinator credential continuation", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const dmRow = { + ...host.row, + Id: "c".repeat(64), + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, dmRow]); + // Named external Docker I/O: the actual native Compose config/hash and + // committed mount files drive the replacement container observation. + host.setCreatingProcess(async (creatingArgs) => { + if (!creatingArgs.includes("--force-recreate")) return; + expect(creatingArgs.at(-1)).toBe("data-manager"); + expect(dm!.preparePosts).toBe(1); + const revision = services.currentConfigurationGeneration(host.infraDir); + if (!revision) throw Error("Actual committed generation unavailable"); + const args = services.stackComposeArgs(host.stack), + options = { + signal: new AbortController().signal, + timeout: 15000, + maxBuffer: 1024 * 1024, + }, + config = JSON.parse( + (await host.execFile("docker", [...args, "config", "--format", "json"], options)) + .stdout, + ), + hash = ( + await host.execFile("docker", [...args, "config", "--hash", "data-manager"], options) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!, + row = { + ...dmRow, + Id: "d".repeat(64), + Labels: { + ...dmRow.Labels, + "com.docker.compose.project.working_dir": join( + host.infraDir, + ".trusted-config-generations", + revision, + ), + "com.docker.compose.project.config_files": + services.configurationGenerationComposeFile(host.infraDir, revision), + "com.docker.compose.config-hash": hash, + }, + }, + service = config.services["data-manager"]; + host.setRows([host.row, row]); + host.setRuntimeInspections({ + [row.Id]: { + Id: row.Id, + Image: row.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: row.ConfigImage, + Labels: row.Labels, + Env: Object.entries(service.environment ?? {}).map( + ([key, value]) => `${key}=${value}`, + ), + }, + Mounts: [ + ...(service.volumes ?? []).map( + (value: { source: string; target: string; type: string; read_only?: boolean }) => ({ + Type: value.type, + Source: value.source, + Destination: value.target, + RW: !value.read_only, + }), + ), + ...(service.secrets ?? []).map((value: { source: string; target: string }) => ({ + Type: "bind", + Source: config.secrets[value.source].file, + Destination: `/run/secrets/${value.target}`, + RW: false, + })), + ], + }, + }); + }); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-osm-fixture"); + const { applyTrustedConfiguration } = await import("../trusted-config-operations"), + { createDurableOpsKey } = await import("../ops-client"); + const options = { + kind: "vault.apply" as const, + serviceId: "data-manager", + actorId: "loopback-admin", + followUp: "service-apply" as const, + operationKey: createDurableOpsKey("task012.maintenance-credential", randomUUID()), + overlay: { + purpose: "service-secret" as const, + coordinatorMode: "ensure-running" as const, + changes: [ + { + kind: "service-secret" as const, + serviceId: "data-manager", + key: "TOKEN", + action: "set" as const, + value: "synthetic-private-coordinator-token", + }, + ], + }, + }; + const applied = await applyTrustedConfiguration(options); + expect(applied.revisionId).toBe(services.currentConfigurationGeneration(host.infraDir)); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(1); + const [retained] = await fixture.db.select().from(schema.adminJob); + expect(retained?.status).toBe("success"); + const checkpoint = (retained?.result as Record) + ._configurationMaintenance as { + phase: string; + decision: { decision: string }; + }; + expect(checkpoint.phase).toBe("settled"); + expect(checkpoint.decision.decision).toBe("continue"); + expect(await applyTrustedConfiguration(options)).toEqual(applied); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + expect(actualPosts.get("configurationMaintenance.prepare")).toBe(1); + expect(actualPosts.get("configurationMaintenance.decide")).toBe(1); + expect(actualPosts.get("configurationMaintenance.settle")).toBe(1); + }, 60000); + it("preflights an inert durable nonempty coordinator candidate with original maintenance and apply identities", async () => { + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-coordinator-credential", + }, + ], + }, + }); + const { record, sealed } = prospective.validatedConfigurationCandidateState(candidate); + const { controlChildOperationKey } = await import("@openmapx/core/build-artifacts"); + expect(record.maintenanceId).toEqual(expect.any(String)); + expect(record.prepareOperationKey).toEqual(expect.stringMatching(/^opk1_t1_/)); + expect(record.operationKey).toBe( + controlChildOperationKey(record.prepareOperationKey!, "configuration-commit", 0), + ); + expect(sealed.operation).toMatchObject({ + maintenanceId: record.maintenanceId, + candidateId: candidate.candidateId, + kind: "vault.apply", + serviceId: "data-manager", + }); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(host.calls.some((args) => args.includes("up") || args.includes("pull"))).toBe(false); + }); + }); + it("settles the old-revision prerequisite before durable nonempty maintenance admission with an initially stopped coordinator", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const coordinator = { + ...host.row, + Id: "c".repeat(64), + State: "exited", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, coordinator]); + let started = 0; + host.setCreatingProcess(async (args) => { + expect(args.at(-1)).toBe("data-manager"); + expect(args).not.toContain("--force-recreate"); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE kind='configuration-maintenance'`, + ).toEqual([]); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(1); + coordinator.State = "running"; + host.setRows([host.row, coordinator]); + started++; + }); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-coordinator-credential", + }, + ], + }, + }); + expect(started).toBe(0); + const { verifyMaintenanceCoordinatorPrerequisite } = await import( + "@openmapx/core/build-artifacts" + ); + const { record } = prospective.validatedConfigurationCandidateState(candidate); + expect(() => verifyMaintenanceCoordinatorPrerequisite(installation, record)).toThrow(); + // Stage only the real ordinary prerequisite, before the independently + // coordinated maintenance admission and desired transaction. + await prospective.stageProspectiveConfigurationMaintenance(candidate); + expect(started).toBe(1); + const projection = ctx.checkpointResult!.opsProjection as { + operations: Record< + string, + { operation: unknown; phase: string; receiptDelivery?: { phase?: string } } + >; + }; + const starts = Object.values(projection.operations).filter( + (operation) => (operation.operation as { kind: string }).kind === "service.start", + ); + expect(starts).toHaveLength(1); + expect(starts[0]!.operation).toEqual({ + kind: "service.start", + serviceId: "data-manager", + expectedConfigurationRevisionId: host.initial.revisionId, + }); + expect(starts[0]!.phase).toBe("terminal"); + expect(starts[0]!.receiptDelivery).toMatchObject({ state: "confirmed" }); + const source = verifyMaintenanceCoordinatorPrerequisite(installation, record); + expect(source.execution.operationKey).toBe( + record.coordinatorPrerequisite!.root.operationKey, + ); + expect(() => + verifyMaintenanceCoordinatorPrerequisite(installation, { + ...record, + coordinatorPrerequisite: { + ...record.coordinatorPrerequisite!, + operation: { + ...record.coordinatorPrerequisite!.operation, + expectedConfigurationRevisionId: candidate.revisionId, + }, + }, + }), + ).toThrow(); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE kind='configuration-maintenance'`, + ).toEqual([]); + }); + }, 60000); + it("discards an actual durable nonempty unissued candidate after its settled helper and original parent deadline expire", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const coordinator = { + ...host.row, + Id: "c".repeat(64), + State: "exited", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, coordinator]); + let starts = 0; + host.setCreatingProcess(async (args) => { + expect(args.at(-1)).toBe("data-manager"); + expect(args).not.toContain("--force-recreate"); + coordinator.State = "running"; + host.setRows([host.row, coordinator]); + starts++; + }); + const ctx = await job(), + { prepareApiJobRoot } = await import("../admin-job-parent"), + initial = prepareApiJobRoot( + ctx, + "configurationMaintenance.prepare", + "synthetic-pre-root-short-initial-budget", + ); + await ctx.checkpoint({ + ...ctx.checkpointResult, + _parentExecution: { ...initial.parent, deadlineAt: Date.now() + 60000 }, + }); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-helper-parent-expiry", + }, + ], + }, + }), + state = prospective.validatedConfigurationCandidateState(candidate), + parent = structuredClone(ctx.checkpointResult!._parentExecution); + await prospective.stageProspectiveConfigurationMaintenance(candidate); + expect(starts).toBe(1); + expect(dm!.preparePosts).toBe(0); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + const helperPath = join( + host.infraDir, + "control-private/ops-agent/executions", + `${state.record.coordinatorPrerequisite!.root.operationKey}.json`, + ), + admission = readFileSync(helperPath); + while (Date.now() <= state.record.parentDeadlineAt) + await nativeDelay(Math.min(1000, state.record.parentDeadlineAt - Date.now() + 10)); + expect(JSON.parse(state.sealed.bytes.toString()).expiresAt > new Date().toISOString()).toBe( + true, + ); + await expect( + prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + }), + ).rejects.toThrow("Configuration candidate expired"); + expect(ctx.checkpointResult!._parentExecution).toEqual(parent); + expect(ctx.checkpointResult!._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + expect(readFileSync(helperPath).equals(admission)).toBe(true); + expect(starts).toBe(1); + expect(dm!.preparePosts).toBe(0); + expect(dm!.settlePosts).toBe(0); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + }); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + }, 120000); + it("retains an actual durable nonempty unissued candidate after a helper observer failure even when its next lookup succeeds", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const coordinator = { + ...host.row, + Id: "c".repeat(64), + State: "exited", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, coordinator]); + let starts = 0; + host.setCreatingProcess(async (args) => { + expect(args.at(-1)).toBe("data-manager"); + expect(args).not.toContain("--force-recreate"); + coordinator.State = "running"; + host.setRows([host.row, coordinator]); + starts++; + }); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-helper-observer-loss", + }, + ], + }, + }), + state = prospective.validatedConfigurationCandidateState(candidate), + path = join(host.queue, ".candidates", `${state.record.operationKey}.json`), + bytes = readFileSync(path), + actualFetch = globalThis.fetch; + let losses = 0, + failure: unknown; + const transport = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, options) => { + const response = await actualFetch(input, options); + if ( + (options?.method ?? "GET") === "GET" && + new URL(String(input)).pathname.startsWith("/v1/operations/") && + !String(input).includes("/events") && + losses < 3 + ) { + const body = await response.clone().json(); + if (body.result?.kind === "service.start" && body.result?.state === "succeeded") { + losses++; + throw Error("Synthetic original helper terminal HTTP observation loss"); + } + } + return response; + }); + try { + await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + }); + } catch (error) { + failure = error; + } finally { + transport.mockRestore(); + } + expect(losses).toBe(3); + expect(failure).toBeInstanceOf((await import("../ops-client")).OpsObservationPending); + expect(ctx.checkpointResult!._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "prepared", + }); + expect(readFileSync(path).equals(bytes)).toBe(true); + expect(dm!.preparePosts).toBe(0); + expect(starts).toBe(1); + await prospective.stageProspectiveConfigurationMaintenance(candidate); + expect(starts).toBe(1); + expect(dm!.preparePosts).toBe(0); + expect(readFileSync(path).equals(bytes)).toBe(true); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + }); + }, 60000); + it("refuses and discards an actual durable nonempty pre-root candidate after a settled helper source race", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const coordinator = { + ...host.row, + Id: "c".repeat(64), + State: "exited", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, coordinator]); + let starts = 0; + host.setCreatingProcess(async (args) => { + expect(args).not.toContain("--force-recreate"); + expect(args.at(-1)).toBe("data-manager"); + coordinator.State = "running"; + host.setRows([host.row, coordinator]); + starts++; + // An independent actual connection races the complete original source + // after the genuine old-revision Start, before maintenance admission. + await fixture.db + .insert(schema.serviceConfig) + .values({ id: randomUUID(), serviceId: "motis", config: { WORKERS: 7 } }); + }); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-pre-root-race", + }, + ], + }, + }); + await expect( + prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + }), + ).rejects.toThrow("Configuration source conflict"); + expect(starts).toBe(1); + expect(dm!.preparePosts).toBe(0); + expect(dm!.settlePosts).toBe(0); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect(coordinator.State).toBe("running"); + expect(ctx.checkpointResult!._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + }); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + }, 60000); + it.each([false, true])( + "publishes original reserved settlement cells through an actual durable nonempty maintenance route without any desired save with lost prepare response=%s", + async (lost) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + losePrepareResponse: lost, + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const coordinator = { + ...host.row, + Id: "c".repeat(64), + State: lost ? "exited" : "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, coordinator]); + host.setCreatingProcess(async (args) => { + expect(args.at(-1)).toBe("data-manager"); + expect(args).not.toContain("--force-recreate"); + expect(dm!.preparePosts).toBe(0); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + coordinator.State = "running"; + host.setRows([host.row, coordinator]); + }); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-coordinator-credential", + }, + ], + }, + }); + const { record } = prospective.validatedConfigurationCandidateState(candidate), + { prepareConfigurationMaintenance } = await import("../configuration-maintenance"); + // Corrupt only external HTTP copies of the ACTUAL issued packet; + // the original still reaches the actual receiver unchanged. + const actualFetch = globalThis.fetch; + let rejectedControls = 0; + const transport = !lost + ? vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + if ( + String(input) === `${dm!.baseUrl}/executions/configuration-maintenance/prepare` && + init?.method === "POST" + ) { + const headers = new Headers(init.headers), + assertion = headers.get("x-openmapx-control-delegation"); + if (!assertion) throw Error("Actual issued preparation control required"); + const original = JSON.parse(Buffer.from(assertion, "base64url").toString("utf8")); + for (const changed of [ + { ...original, signature: "a".repeat(86) }, + { ...original, installationId: randomUUID() }, + ]) { + const foreign = new Headers(headers); + foreign.set( + "x-openmapx-control-delegation", + Buffer.from(JSON.stringify(changed)).toString("base64url"), + ); + const response = await actualFetch(input, { ...init, headers: foreign }); + expect(response.status).toBe(403); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE kind='configuration-maintenance' AND metadata->'configurationMaintenance'->'receipt'->>'maintenanceId'=${record.maintenanceId!}`, + ).toEqual([]); + expect( + existsSync( + join( + host.infraDir, + "control-private/data-manager/configuration-maintenance", + `${record.maintenanceId}.json`, + ), + ), + ).toBe(false); + await response.arrayBuffer(); + rejectedControls++; + } + } + return actualFetch(input, init); + }) + : undefined; + let prepared: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + try { + prepared = await prepareConfigurationMaintenance(candidate).catch(async (error) => { + throw Error(`${String(error)}; DM=${JSON.stringify(await dm!.diagnostics())}`); + }); + } finally { + transport?.mockRestore(); + } + expect(rejectedControls).toBe(lost ? 0 : 2); + const control = join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${record.prepareOperationKey}.configuration-settlement-control.json`, + ); + expect(existsSync(control)).toBe(true); + expect(prepared).toMatchObject({ + state: "prepared", + preflight: "current", + decision: null, + generation: "not_started", + followUp: "not_started", + }); + const sourceControl = await import( + "../../../../../packages/core/src/build-artifacts/control-authority" + ), + originalAuthority = restoreAdmittedControlExecution(installation, prepared.prepare); + const rootControl = await import("@openmapx/core/build-artifacts"); + expect(() => + rootControl.verifyRetainedArtifactTargetRootRelease(originalAuthority), + ).toThrow(); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + expect(() => + sourceControl.verifyMaintenanceCoordinatorPrerequisite(installation, record), + ).not.toThrow(); + expect( + host.calls.filter((args) => args.includes("up") && args.includes("data-manager")), + ).toHaveLength(1); + expect(dm!.preparePosts).toBe(lost ? 1 : 3); + expect(() => + sourceControl.withControlDelegationPublication( + originalAuthority, + "configuration-commit", + () => undefined, + ), + ).toThrow("Original authorized configuration decision required"); + expect(statSync(control).size).toBe(8192); + const { verifyControl, controlDigest } = await import("@openmapx/core/build-artifacts"), + { + configurationSettlementControlCellSchema, + configurationSettlementReservationProjectionSchema, + configurationSettlementCommitmentCellSchema, + configurationSettlementDomains, + } = await import( + "../../../../../packages/core/src/build-artifacts/configuration-fence-settlement-records" + ); + const original = readFileSync(control, "utf8"), + pending = verifyControl( + installation.publicKey, + configurationSettlementDomains.control, + JSON.parse(original), + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + installation.publicKey, + configurationSettlementDomains.signedProjection, + pending.projection, + configurationSettlementReservationProjectionSchema, + ); + expect(projection).toMatchObject({ + maintenanceId: record.maintenanceId, + candidateId: record.candidateId, + ownerPrepare: { operationKey: record.prepareOperationKey }, + operation: { + version: 2, + maintenanceId: record.maintenanceId, + candidateId: record.candidateId, + }, + }); + expect(pending.binding).toBeNull(); + const commitmentPath = join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${record.prepareOperationKey}.configuration-settlement.json`, + ), + commitment = verifyControl( + installation.publicKey, + configurationSettlementDomains.reference, + JSON.parse(readFileSync(commitmentPath, "utf8")), + configurationSettlementCommitmentCellSchema.shape.payload, + ); + expect(commitment).toMatchObject({ + phase: "uncommitted", + identity: { settlementId: projection.settlementId }, + }); + expect(statSync(commitmentPath).size).toBe(16384); + const location = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/retention-locations", + `${controlDigest(projection.origin)}.json`, + ), + "utf8", + ), + ), + progress = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/retention-families", + location.name, + "progress.json", + ), + "utf8", + ), + ); + expect(progress.configurationSettlementSlot).toMatchObject({ + role: "ops-agent", + state: "reserved", + executionDigest: null, + controlBindingDigest: null, + settlementId: projection.settlementId, + proposedExecutionAddress: projection.reservation.execution, + }); + expect(progress.configurationSettlementSlot.allocatedDescriptorDigests).toHaveLength(6); + const beforeCalls = host.calls.filter((args) => args.includes("up")).length; + expect(await prepareConfigurationMaintenance(candidate)).toEqual(prepared); + expect(readFileSync(control, "utf8")).toBe(original); + expect(host.calls.filter((args) => args.includes("up"))).toHaveLength(beforeCalls); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + const guard = + await fixture.sql`SELECT id,status,metadata FROM data_manager.jobs WHERE kind='configuration-maintenance'`; + expect(guard).toHaveLength(1); + expect(guard[0]!.metadata.configurationMaintenance).toMatchObject({ + guardState: "prepared", + receipt: { state: "prepared", sequence: 0 }, + settlementReservation: { version: 2, settlementId: projection.settlementId }, + }); + expect(dm!.preparePosts).toBe(lost ? 1 : 3); + const core = await import("@openmapx/core/build-artifacts"), + authority = core.restoreAdmittedControlExecution(installation, prepared.prepare), + lease = core.restoreArtifactTargetLeaseForSettlement(authority); + try { + await expect( + core.settleArtifactTargetLeaseChild( + lease, + core.verifyControlEffectSettlement(installation, prepared.prepare), + ), + ).rejects.toThrow("Conclusive original maintenance"); + await expect( + core.reconcileArtifactTargetLease({ + dataRoot: join(host.infraDir, "data"), + leaseId: lease.leaseId, + authority: core.verifyControlRecoveryAuthority(authority), + }), + ).rejects.toThrow("Conclusive original maintenance"); + } finally { + core.closeArtifactTargetLeaseHandle(lease); + } + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + }); + }, + 60000, + ); + it.each(["source", "guard"] as const)( + "refreshes actual durable nonempty maintenance inspection from the original API job with drift=%s", + async (drift) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-coordinator-credential", + }, + ], + }, + }); + await prospective.stageProspectiveConfigurationMaintenance(candidate); + const maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + identity = { + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + }, + beforeLifecycle = host.calls.filter( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ).length; + expect(await maintenance.inspectConfigurationMaintenance(identity)).toMatchObject({ + state: "prepared", + preflight: "current", + sequence: prepared.sequence, + }); + await expect(async () => + prospective.discardNeverPreparedProspectiveConfiguration(candidate), + ).rejects.toThrow("Conclusive original NoGuard closure required"); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(1); + const { createApiOpsClient, createDurableOpsKey } = await import("../ops-client"), + { prepareApiJobChildAttempt, issueApiJobChildAssertion } = await import( + "../admin-job-parent" + ), + inspect = { kind: "configurationMaintenance.inspect" as const, ...identity }, + client = createApiOpsClient(); + await expect(client.execute(inspect)).rejects.toMatchObject({ + errorClass: "authorization", + }); + const other = await job(), + key = createDurableOpsKey("task012.wrong-maintenance-inspector", other.jobId), + attempt = prepareApiJobChildAttempt(other, inspect, key)!; + await other.checkpoint({ ...other.checkpointResult, _inspectionAttempt: attempt }); + await expect( + client.execute(inspect, { + operationKey: key, + execution: { requestedDeadlineAt: attempt.requestedDeadlineAt }, + parentAssertion: await issueApiJobChildAssertion(attempt), + }), + ).rejects.toMatchObject({ errorClass: "authorization" }); + if (drift === "source") { + const envPath = join(host.infraDir, ".env"), + original = existsSync(envPath) ? readFileSync(envPath) : undefined; + try { + writeFileSync( + envPath, + Buffer.concat([ + original ?? Buffer.alloc(0), + Buffer.from("\nTASK012_INSPECTION_SOURCE_DRIFT=changed\n"), + ]), + { mode: 0o600 }, + ); + expect(await maintenance.inspectConfigurationMaintenance(identity)).toMatchObject({ + state: "prepared", + preflight: "drifted", + sequence: prepared.sequence, + }); + } finally { + if (original) writeFileSync(envPath, original); + else rmSync(envPath); + } + } else { + await fixture.sql`DELETE FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid AND kind='configuration-maintenance'`; + expect(await maintenance.inspectConfigurationMaintenance(identity)).toMatchObject({ + state: "prepared", + preflight: "unavailable", + sequence: prepared.sequence, + }); + } + expect(dm!.preparePosts).toBe(1); + expect( + host.calls.filter( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ), + ).toHaveLength(beforeLifecycle); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + }); + }, + 60000, + ); + it("inspects actual durable nonempty maintenance on the same owned desired-write transaction connection", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-transaction-inspection", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + identity = { + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + }; + let pending: ReturnType | undefined, + timer: ReturnType | undefined; + try { + await writer + .withConfigurationWriteTransaction( + writer.currentAdministrativeConfigurationWriter(), + async (tx) => { + pending = maintenance.inspectConfigurationMaintenance(identity); + const receipt = await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(Error("Inspection blocked by its own owned job row lock")), + 5000, + ); + }), + ]); + expect(receipt).toMatchObject({ + state: "prepared", + preflight: "current", + sequence: prepared.sequence, + }); + const [row] = await tx + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, ctx.jobId)); + expect(row?.result).toMatchObject({ + _configurationMaintenance: { inspection: { state: "complete", receipt } }, + }); + }, + ) + .catch((error) => { + if (error instanceof writer.ConfigurationWriteRolledBack) throw error.failure; + throw error; + }); + } finally { + if (timer) clearTimeout(timer); + // A RED observer timeout releases the actual SQL row lock before + // awaiting that original read; no unobserved background task remains. + await pending?.catch(() => undefined); + } + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(dm!.preparePosts).toBe(1); + }); + }, 60000); + it.each([false, true])( + "commits a durable nonempty maintenance desired-row and continue decision in one actual transaction with rollback=%s", + async (rollback) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-committed-maintenance-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + lifecycle = host.calls.filter( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ).length; + if (rollback) { + // An actual owned-PG constraint fails the desired write; no authority, + // transaction runner, verifier or proof result is substituted. + await fixture.sql`ALTER TABLE service_secret ADD CONSTRAINT fixture_reject_desired_token CHECK (key <> 'TOKEN')`; + await expect( + prospective.commitProspectiveConfigurationMaintenance(candidate), + ).rejects.toBeInstanceOf(writer.ConfigurationWriteRolledBack); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, ctx.jobId)); + expect(row?.result).not.toHaveProperty("_configurationMaintenanceDecision"); + expect(row?.result).toMatchObject({ + _configurationCandidate: { phase: "rolled_back" }, + _configurationMaintenanceRollback: { + disposition: "known_rolled_back", + candidateId: candidate.candidateId, + maintenanceId: prepared.binding.maintenanceId, + }, + }); + await expect( + ctx.checkpoint({ + ...ctx.checkpointResult, + _configurationMaintenanceRollback: { + ...(ctx.checkpointResult!._configurationMaintenanceRollback as Record< + string, + unknown + >), + rollbackId: randomUUID(), + }, + }), + ).rejects.toThrow("Known maintenance rollback changed outside its fixed producer"); + } else { + const decision = await prospective.commitProspectiveConfigurationMaintenance(candidate), + [saved] = + await fixture.sql`SELECT id,xmin::text AS version FROM service_secret WHERE service_id='data-manager' AND key='TOKEN'`, + [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, ctx.jobId)); + expect(saved).toBeDefined(); + expect(decision).toMatchObject({ + decision: "continue", + expectedSequence: prepared.sequence, + prepareOperationKey: prepared.prepare.operationKey, + planDigest: prepared.planDigest, + }); + const versions = [ + { + kind: "service-secret", + ownerId: "data-manager", + key: "TOKEN", + rowId: saved!.id, + rowVersion: saved!.version, + previousRowVersion: null, + state: "present", + }, + ]; + expect(row?.result).toMatchObject({ + _configurationCandidate: { phase: "saved", rowVersions: versions }, + _configurationMaintenanceDecision: { + version: 1, + operation: decision, + rowVersions: versions, + attempt: "not_attempted", + }, + }); + expect(JSON.stringify(row?.result)).not.toContain( + "synthetic-committed-maintenance-credential", + ); + expect(await prospective.commitProspectiveConfigurationMaintenance(candidate)).toEqual( + decision, + ); + const authorized = await maintenance.decideConfigurationMaintenance(decision); + expect(authorized).toMatchObject({ + state: "authorized", + decision: { decision: "continue", decisionId: decision.decisionId }, + generation: "not_started", + followUp: "not_started", + dmFence: prepared.dmFence, + }); + const { configurationMaintenanceSettleRequestDigest } = await import( + "@openmapx/core/ops" + ), + store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + authority = restoreAdmittedControlExecution(installation, prepared.prepare); + expect(configurationMaintenanceSettleRequestDigest(authorized)).toBe( + configurationMaintenanceSettleRequestDigest( + store.readOwnedConfigurationMaintenance(authority).receipt, + ), + ); + } + expect(dm!.preparePosts).toBe(1); + expect( + host.calls.filter( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ), + ).toHaveLength(lifecycle); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(readdirSync(host.queue).filter((name) => name.startsWith("cfg1_"))).toEqual([]); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + }); + }, + 60000, + ); + it("closes actual durable nonempty known rolled-back current persistence as unused and releases the next writer", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-rollback-unused-credential", + }, + ], + }, + }), + current = await import("../trusted-config-operations"); + await fixture.sql`ALTER TABLE service_secret ADD CONSTRAINT fixture_reject_desired_token CHECK (key <> 'TOKEN')`; + const result = await current.applyTrustedConfigurationWithEvidence({ + ctx, + actorId: ctx.jobId, + kind: "vault.apply", + serviceId: "data-manager", + operationKey: candidate.operationKey, + candidate, + coordinatorMode: "require-running", + followUp: "service-apply", + expectedCurrentRevisionId: host.initial.revisionId, + }); + expect(result.outcome).toMatchObject({ outcome: "not_committed", reason: "conflict" }); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(ctx.checkpointResult).toMatchObject({ + _configurationCandidate: { phase: "discarded" }, + _configurationMaintenanceDecision: { + operation: { decision: "abandon" }, + rowVersions: [], + }, + _configurationMaintenance: { + phase: "settled", + receipt: { + state: "settled", + generation: "not_started", + followUp: "not_started", + dmFence: { state: "settled" }, + }, + }, + }); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect(JSON.stringify(ctx.checkpointResult)).not.toContain( + "synthetic-rollback-unused-credential", + ); + }); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + }, 120000); + it("commits the actual durable nonempty frozen generation child under its original maintenance owner", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-frozen-generation-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await prospective.commitProspectiveConfigurationMaintenance(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const calls = host.calls.length, + core = await import("@openmapx/core/build-artifacts"), + authority = restoreAdmittedControlExecution(installation, prepared.prepare), + lease = core.restoreArtifactTargetLeaseForSettlement(authority), + leaseDirectory = join(host.infraDir, "data/.admin-target-control/leases", lease.leaseId), + ownerPath = join(leaseDirectory, "owner.json"), + ownerBytes = readFileSync(ownerPath), + ownerInode = statSync(leaseDirectory).ino, + originalAdmissionPath = join( + host.infraDir, + "control-private/ops-agent/executions", + `${prepared.prepare.operationKey}.json`, + ), + originalAdmission = readFileSync(originalAdmissionPath), + familiesPath = join(host.infraDir, "control-private/ops-agent/retention-locations"), + families = readdirSync(familiesPath).sort(); + core.closeArtifactTargetLeaseHandle(lease); + const nativeFetch = globalThis.fetch, + admissions: { status: number; errorClass?: string }[] = [], + transportProbe = vi.spyOn(globalThis, "fetch").mockImplementation(async (...args) => { + const response = await nativeFetch(...args); + if (String(args[0]).endsWith("/v1/operations") && args[1]?.method === "POST") { + const frame = (await response.clone().json()) as { error?: { class?: string } }; + admissions.push({ status: response.status, errorClass: frame.error?.class }); + } + return response; + }); + let result: unknown; + try { + result = await maintenance.executeConfigurationMaintenanceStep(candidate, 0); + } catch (error) { + const store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + actual = store.readOwnedConfigurationMaintenance(authority), + slot = actual.plan.reservations[0]!, + bound = lookupControlChildPhaseBinding(authority, slot), + attempt = lookupControlChildPhaseAttempt(authority, slot), + journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + record = journal + .records() + .find((value) => value.operationKey === prepared.steps[0]!.operationKey); + throw Error( + `${(error as Error).name}; actual admission frames=${JSON.stringify(admissions)}; phase=${JSON.stringify( + { + bound: Boolean(bound), + issued: Boolean(attempt), + actorSlots: actual.apiControls.steps.length, + actorGrant: Boolean(actual.apiControls.steps[0]?.grant), + journalState: record?.state, + adoption: record?.controlAdoption?.phase, + state: actual.receipt.state, + sequence: actual.sequence, + generationCommitted: + services.currentConfigurationGeneration(host.infraDir) === candidate.revisionId, + }, + )}`, + ); + } finally { + transportProbe.mockRestore(); + } + expect(result).toMatchObject({ outcome: "committed", revisionId: candidate.revisionId }); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(candidate.revisionId); + expect( + host.calls + .slice(calls) + .some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect(dm!.preparePosts).toBe(1); + expect(JSON.parse(readFileSync(ownerPath, "utf8")).payload.certificate).toEqual( + JSON.parse(ownerBytes.toString()).payload.certificate, + ); + expect(JSON.parse(readFileSync(ownerPath, "utf8")).payload.origin).toEqual( + JSON.parse(ownerBytes.toString()).payload.origin, + ); + expect(statSync(leaseDirectory).ino).toBe(ownerInode); + expect(readFileSync(originalAdmissionPath)).toEqual(originalAdmission); + expect(readdirSync(familiesPath).sort()).toEqual(families); + const step = prepared.steps[0]!, + projected = (await import("../admin-job-ops")).readProjection(ctx.checkpointResult) + .operations[step.operationKey]; + expect(projected).toMatchObject({ + operation: step.operation, + operationKey: step.operationKey, + terminal: true, + result: { outcome: "committed", revisionId: candidate.revisionId }, + receiptDelivery: { state: "confirmed" }, + }); + expect(await maintenance.executeConfigurationMaintenanceStep(candidate, 0)).toEqual(result); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + }); + }, 60000); + it.each([false, true])( + "applies the actual durable nonempty maintenance hardlink middle owner and its sole DM leaf (recreate=%s)", + async (recreate) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-osm-fixture"); + const ctx = await job(), + observation = new AbortController(); + ctx.signal = observation.signal; + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-maintenance-hardlink-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await prospective.commitProspectiveConfigurationMaintenance(candidate); + await maintenance.decideConfigurationMaintenance(decision); + await maintenance.executeConfigurationMaintenanceStep(candidate, 0); + const step = prepared.steps[1]!; + expect(step).toMatchObject({ + phase: "maintenance-hardlinks", + ordinal: 1, + prerequisites: [0], + operation: { kind: "configuration.hardlinks.apply" }, + }); + // Bound this test's observer only. If the real protocol deliberately + // retains an unknown, cancel/await that SAME attempted observation before + // fixture teardown instead of abandoning a timed-out API/SQL scope. + const observer = setTimeout(() => { + const journal = JSON.parse( + readFileSync(join(host.infraDir, "data/ops-agent/jobs-v1.json"), "utf8"), + ); + console.info( + JSON.stringify({ + proof: "actual-maintenance-middle-owner-observation", + dmPosts: dm!.posts, + dmResponses: dm!.hardlinkResponseStatuses, + jobs: journal.jobs + .filter( + (value: { operation: { maintenanceId?: string } }) => + value.operation.maintenanceId === prepared.binding.maintenanceId, + ) + .map( + (value: { + operation: { kind: string }; + state: string; + errorClass?: string; + controlAdoption?: { phase: string }; + }) => ({ + kind: value.operation.kind, + state: value.state, + errorClass: value.errorClass, + adoption: value.controlAdoption?.phase, + }), + ), + }), + ); + observation.abort(); + }, 45000); + let result: unknown; + try { + result = await maintenance.executeConfigurationMaintenanceStep(candidate, 1); + } finally { + clearTimeout(observer); + } + expect(result).toMatchObject({ + applied: true, + entries: 1, + linked: 1, + via: "data-manager", + }); + expect(dm!.posts).toBe(1); + expect(dm!.preparePosts).toBe(1); + const plan = JSON.parse( + readFileSync( + services.currentConfigurationFile( + host.infraDir, + services.GENERATED_HARDLINK_PLAN_FILE, + ), + "utf8", + ), + ), + target = join(host.infraDir, plan[0].target, "fixture.pbf"), + projected = (await import("../admin-job-ops")).readProjection(ctx.checkpointResult) + .operations[step.operationKey]; + expect(statSync(target).ino).toBe(statSync(join(source, "fixture.pbf")).ino); + expect(projected).toMatchObject({ + operation: step.operation, + terminal: true, + receiptDelivery: { state: "confirmed" }, + }); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/motis"))).toBe( + true, + ); + if (recreate) { + expect(prepared.steps[2]).toMatchObject({ + phase: "maintenance-recreate", + prerequisites: [1], + operation: { kind: "service.recreate", serviceId: "data-manager" }, + }); + const timer = setTimeout(() => observation.abort(), 45000); + try { + expect(await maintenance.executeConfigurationMaintenanceStep(candidate, 2)).toEqual({ + changed: true, + }); + } finally { + clearTimeout(timer); + } + const after = await maintenance.inspectConfigurationMaintenance(prepared.binding); + expect(after).toMatchObject({ + generation: "committed", + followUp: "complete", + state: "applying", + }); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + } + }); + }, + 120000, + ); + it.each(["contained", "retirement", "pointer-ack-loss"])( + "settles the actual durable nonempty contained final after a known native generation noncommit through the current persister (mode=%s)", + async (mode) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const args = services.stackComposeArgs(host.stack), + signal = new AbortController().signal, + options = { signal, timeout: 15000, maxBuffer: 1024 * 1024 }, + config = JSON.parse( + (await host.execFile("docker", [...args, "config", "--format", "json"], options)) + .stdout, + ), + hash = ( + await host.execFile("docker", [...args, "config", "--hash", "data-manager"], options) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!, + row = { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { + ...host.row.Labels, + "com.docker.compose.service": "data-manager", + "com.docker.compose.config-hash": hash, + }, + }, + spec = config.services["data-manager"]; + host.setRows([host.row, row]); + host.setRuntimeInspections({ + [row.Id]: { + Id: row.Id, + Image: row.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: row.ConfigImage, + Labels: row.Labels, + Env: Object.entries(spec.environment ?? {}).map(([key, value]) => `${key}=${value}`), + }, + Mounts: (spec.volumes ?? []).map( + (value: { source: string; target: string; type: string; read_only?: boolean }) => ({ + Type: value.type, + Source: value.source, + Destination: value.target, + RW: !value.read_only, + }), + ), + }, + }); + const ctx = await job(), + observer = new AbortController(); + ctx.signal = observer.signal; + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-contained-candidate-credential", + }, + ], + }, + }), + nativeRename = (await vi.importActual("node:fs")).renameSync; + let faults = 0; + const fault = vi.spyOn(nodeFs, "renameSync").mockImplementation((source, target) => { + if ( + (mode === "pointer-ack-loss" + ? basename(String(target)) === ".trusted-config-current" + : basename(String(target)) === candidate.revisionId) && + basename(String(source)).endsWith(".tmp") + ) { + if (mode === "pointer-ack-loss") nativeRename(source, target); + faults++; + throw Error("Injected actual generation rename refusal before pointer commit"); + } + return nativeRename(source, target); + }), + timeout = setTimeout(() => observer.abort(), 90000); + let result: Awaited< + ReturnType + >; + if (mode === "pointer-ack-loss") { + try { + await expect( + prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "require-running", + signal: ctx.signal, + }), + ).rejects.toBeInstanceOf(Error); + } finally { + clearTimeout(timeout); + fault.mockRestore(); + } + expect(faults).toBe(1); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + candidate.revisionId, + ); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(1); + const retained = ( + ctx.checkpointResult!._configurationMaintenance as { + receipt: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + } + ).receipt, + store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + owner = store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, retained.prepare), + ); + expect(owner.final).toBeNull(); + expect(owner.inspection).toBeNull(); + expect(owner.receipt.dmFence!.state).toBe("prepared"); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + await expect(dm!.receiver.admitLocal({ kind: "download-fonts" })).rejects.toThrow( + "configuration_maintenance_busy", + ); + expect( + host.calls.some( + (value) => + value.includes("up") || value.includes("pull") || value.includes("restart"), + ), + ).toBe(false); + return; + } + try { + result = await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "require-running", + signal: ctx.signal, + }); + } finally { + clearTimeout(timeout); + fault.mockRestore(); + } + expect(faults).toBe(1); + expect(result).toMatchObject({ + saved: "committed", + application: { outcome: "not_committed", reason: "failed", cleanupFailed: false }, + }); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(1); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + const receipt = ( + ctx.checkpointResult!._configurationMaintenance as { + receipt: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + } + ).receipt, + store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + owner = store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, receipt.prepare), + ); + expect(owner.final).toMatchObject({ + ownerFinal: { payload: { outcome: "contained" } }, + evidence: { + outcome: "contained", + generation: { state: "not_committed" }, + followUp: { state: "contained" }, + }, + }); + expect(owner.receipt).toMatchObject({ + state: "settled", + generation: "not_committed", + followUp: "contained", + dmFence: { state: "settled" }, + }); + expect(owner.apiControls.steps).toHaveLength(1); + expect(owner.apiControls.steps[0]!.receiptDelivery).not.toBeNull(); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect( + existsSync( + join(host.queue, ".candidates", `${owner.plan.operation.applyOperationKey}.json`), + ), + ).toBe(false); + expect( + host.calls.some( + (value) => + value.includes("up") || value.includes("pull") || value.includes("restart"), + ), + ).toBe(false); + if (mode === "retirement") { + const core = await import("@openmapx/core/build-artifacts"), + location = (role: string) => + join( + host.infraDir, + "control-private", + role, + "retention-locations", + `${core.controlDigest(owner.origin)}.json`, + ), + clock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 3600000); + try { + for ( + let round = 0; + round < 16 && + (existsSync(location("ops-agent")) || existsSync(location("data-manager"))); + round++ + ) { + await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm!.reconcileRetention(); + } + expect( + existsSync(location("ops-agent")) || existsSync(location("data-manager")), + ).toBe(false); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE id=${owner.guardSource!.preparedReceipt.fenceId}::uuid`, + ).toEqual([]); + await app.close(); + closeControlInstallation(installation); + const journal = await openOpsJobJournal( + join(host.infraDir, "data/ops-agent/jobs-v1.json"), + ); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + app = buildOpsAgentServer({ + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + vi.stubEnv("OPS_AGENT_URL", await app.listen({ host: "127.0.0.1", port: 0 })); + expect((await app.inject({ method: "GET", url: "/health" })).statusCode).toBe(200); + expect( + journal + .records() + .some( + (value) => + (value.operation.kind === "configurationMaintenance.decide" || + value.operation.kind === "configurationMaintenance.settle") && + value.operation.prepareOperationKey === owner.plan.prepare.operationKey, + ), + ).toBe(false); + await dm!.close(); + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + expect( + ( + await fetch(`${dm.baseUrl}/status`, { + headers: { authorization: `Bearer ${dm.token}` }, + }) + ).status, + ).toBe(200); + } finally { + clock.mockRestore(); + } + } + }); + const next = await job(); + if (mode === "pointer-ack-loss") + await expect(owned(next, async () => undefined)).rejects.toThrow( + "Configuration writer unavailable", + ); + else + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + }, + 120000, + ); + it.each([ + "component", + "outer-api", + "persister", + "retirement", + "sql-commit-ack-loss", + "sql-update-rollback", + "sql-session-absent", + "sql-session-live", + "publication-sender-ack", + "publication-dm-ack", + "publication-mirror", + "publication-root-release", + "publication-root-owner-ack", + "publication-root-index", + "sql-control-expired", + ])( + "constructs the actual durable nonempty applied final from production runtime configuration, mounts, image and readiness (mode=%s)", + async (mode) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + loseSettlementCommitAcknowledgement: mode === "sql-commit-ack-loss", + ...(mode === "sql-update-rollback" + ? { settlementSqlFault: "update-ack-loss" as const } + : mode === "sql-session-absent" || mode === "sql-session-live" + ? { settlementSqlFault: "terminate-before-commit" as const } + : {}), + pauseSettlementBeforeCommit: mode === "sql-session-live", + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const dmRow = { + ...host.row, + Id: "c".repeat(64), + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, dmRow]); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-osm-fixture"); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-applied-final-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"); + let installedDocker: Parameters[0] = {}; + host.setCreatingProcess(async () => { + const args = services.stackComposeArgs(host.stack), + signal = new AbortController().signal, + options = { signal, timeout: 15000, maxBuffer: 1024 * 1024 }, + config = JSON.parse( + (await host.execFile("docker", [...args, "config", "--format", "json"], options)) + .stdout, + ), + hash = ( + await host.execFile( + "docker", + [...args, "config", "--hash", "data-manager"], + options, + ) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!, + compose = services.configurationGenerationComposeFile( + host.infraDir, + candidate.revisionId, + ), + row = { + ...dmRow, + Id: "d".repeat(64), + Labels: { + ...dmRow.Labels, + "com.docker.compose.project.working_dir": join( + host.infraDir, + ".trusted-config-generations", + candidate.revisionId, + ), + "com.docker.compose.project.config_files": compose, + "com.docker.compose.config-hash": hash, + }, + }, + service = config.services["data-manager"]; + host.setRows([host.row, row]); + installedDocker = { + [row.Id]: { + Id: row.Id, + Image: row.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: row.ConfigImage, + Labels: row.Labels, + Env: Object.entries(service.environment ?? {}).map( + ([key, value]) => `${key}=${value}`, + ), + }, + Mounts: [ + ...(service.volumes ?? []).map( + (value: { + source: string; + target: string; + type: string; + read_only?: boolean; + }) => ({ + Type: value.type, + Source: value.source, + Destination: value.target, + RW: !value.read_only, + }), + ), + ...(service.secrets ?? []).map((value: { source: string; target: string }) => ({ + Type: "bind", + Source: config.secrets[value.source].file, + Destination: `/run/secrets/${value.target}`, + RW: false, + })), + ], + }, + }; + host.setRuntimeInspections(installedDocker); + }); + let prepared: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + let decisionId: string; + if ( + mode === "persister" || + mode === "retirement" || + mode.startsWith("sql-") || + mode.startsWith("publication-") + ) { + const { AnchoredDirectory } = await import( + "../../../../../packages/core/src/build-artifacts/filesystem" + ), + writeCell = AnchoredDirectory.prototype.write, + expectedAckOwner = + prospective.validatedConfigurationCandidateState(candidate).record + .prepareOperationKey, + expectedOwnerName = `${prospective.validatedConfigurationCandidateState(candidate).record.maintenanceId}.json`; + let interrupted = false, + heldRootInode: number | undefined; + const nativeRename = (await vi.importActual("node:fs")) + .renameSync; + const publicationFault = + mode === "publication-root-index" + ? vi.spyOn(nodeFs, "renameSync").mockImplementation((source, target) => { + if (!interrupted && basename(String(target)) === expectedOwnerName) { + const payload = JSON.parse(readFileSync(source, "utf8")).payload; + if ( + payload?.receipt?.state === "settled" && + payload.apiControls?.settle?.result === null + ) { + interrupted = true; + expect( + existsSync( + join(host.infraDir, "data/.admin-target-control/locks/data-manager"), + ), + ).toBe(false); + throw Error( + "Synthetic actual released-owner rename failure after family preindex", + ); + } + } + return nativeRename(source, target); + }) + : mode.startsWith("publication-") + ? vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(function ( + this: InstanceType, + name, + bytes, + ...args + ) { + const payload = JSON.parse(String(bytes))?.payload, + ack = + ((mode === "publication-sender-ack" && + name === `${expectedAckOwner}.configuration-settlement-ack.json`) || + (mode === "publication-dm-ack" && + name !== `${expectedAckOwner}.configuration-settlement-ack.json` && + name.endsWith(".configuration-settlement-ack.json"))) && + Buffer.byteLength(bytes) === 1024 && + typeof payload?.terminalDigest === "string", + mirror = + mode === "publication-mirror" && + name === expectedOwnerName && + payload?.metadata?.guardState === "settled" && + payload.metadata.settlementControl?.tuple?.publication === "committed", + releasedOwner = + ["publication-root-release", "publication-root-owner-ack"].includes( + mode, + ) && + name === expectedOwnerName && + payload?.receipt?.state === "settled" && + payload.apiControls?.settle?.result === null; + if ( + (!interrupted || mode === "publication-mirror") && + (ack || mirror || releasedOwner) + ) { + interrupted = true; + const rootLock = join( + host.infraDir, + "data/.admin-target-control/locks/data-manager", + ); + if (ack || mirror) heldRootInode = statSync(rootLock).ino; + else expect(existsSync(rootLock)).toBe(false); + if (mode === "publication-root-owner-ack") + writeCell.call(this, name, bytes, ...args); + throw Error( + "Synthetic exact settlement publication filesystem interruption", + ); + } + return writeCell.call(this, name, bytes, ...args); + }) + : undefined; + const expiredDelivery = Promise.withResolvers(); + let originalExpiredControl: + | { + key: string; + deadlineAt: number; + binding: Buffer; + dispatch: Buffer; + admission: Buffer; + effect: Buffer; + rootInode: number; + } + | undefined; + const fetchActual = globalThis.fetch, + delayControl = + mode === "sql-control-expired" + ? vi.spyOn(globalThis, "fetch").mockImplementation(async (input, options) => { + if ( + options?.method === "POST" && + String(input).startsWith( + `${dm!.baseUrl}/executions/configuration-maintenance/`, + ) && + String(input).endsWith("/settle") + ) { + if (originalExpiredControl) + throw Error("A second settlement POST is forbidden"); + const binding = readFileSync( + join( + host.infraDir, + "control-private/ops-agent/observations", + `${expectedAckOwner}.configuration-settlement-binding.json`, + ), + ), + { installationState } = await import( + "../../../../../packages/core/src/build-artifacts/control-authority" + ), + { verifyControl } = await import( + "../../../../../packages/core/src/build-artifacts/control-records" + ), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-fence-settlement-records" + ), + envelope = verifyControl( + installationState(installation).installation.publicKey, + records.configurationSettlementDomains.binding, + JSON.parse(binding.toString()), + records.configurationSettlementControlEnvelopeSchema, + ), + key = envelope.execution.operationKey; + originalExpiredControl = { + key, + deadlineAt: envelope.controlDeadlineAt, + binding, + dispatch: readFileSync( + join( + host.infraDir, + "control-private/ops-agent/observations", + `${expectedAckOwner}.configuration-settlement-dispatch.json`, + ), + ), + admission: readFileSync( + join( + host.infraDir, + "control-private/data-manager/observations", + `${key}.configuration-settlement-admission.json`, + ), + ), + effect: readFileSync( + join( + host.infraDir, + "control-private/data-manager/observations", + `${key}.configuration-settlement-effect.json`, + ), + ), + rootInode: statSync( + join(host.infraDir, "data/.admin-target-control/locks/data-manager"), + ).ino, + }; + expect(envelope.controlDeadlineAt - envelope.controlStartedAt).toBe(60000); + expect(Date.now()).toBeLessThan(envelope.controlDeadlineAt); + // External delayed original packet, not a retry/new assertion. The + // transport's own15s timeout still ends the original waiter; a + // network can deliver its already-sent bytes after cancellation. + await nativeDelay( + Math.max(1, envelope.controlDeadlineAt - Date.now() + 50), + ); + try { + const response = await fetchActual(input, { + ...options, + signal: undefined, + }); + expiredDelivery.resolve(response.status); + return response; + } catch (error) { + expiredDelivery.reject(error); + throw error; + } + } + return fetchActual(input, options); + }) + : undefined; + const current = await import("../trusted-config-operations"), + application = current.applyTrustedConfigurationWithEvidence({ + ctx, + actorId: ctx.jobId, + kind: "vault.apply", + serviceId: "data-manager", + operationKey: candidate.operationKey, + candidate, + coordinatorMode: "require-running", + followUp: "service-apply", + expectedCurrentRevisionId: host.initial.revisionId, + }); + void application.catch(() => {}); + if (mode === "sql-control-expired") { + let lateStatus: number; + try { + await expect(application).rejects.toBeInstanceOf( + (await import("../ops-client")).OpsObservationPending, + ); + lateStatus = await expiredDelivery.promise; + } finally { + delayControl!.mockRestore(); + } + const original = originalExpiredControl!; + expect(original).toBeDefined(); + expect(Date.now()).toBeGreaterThan(original.deadlineAt); + expect([403, 409]).toContain(lateStatus!); + expect(dm!.settlePosts).toBe(1); + const store = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ), + preparedReceipt = ( + ctx.checkpointResult!._configurationMaintenance as { + receipt: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + } + ).receipt, + authority = restoreAdmittedControlExecution(installation, preparedReceipt.prepare), + before = store.readOwnedConfigurationMaintenance(authority), + [sqlBefore] = + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${before.guardSource!.preparedReceipt.fenceId}::uuid`; + expect(sqlBefore!.metadata.configurationMaintenance).toMatchObject({ + guardState: "prepared", + receipt: { state: "prepared", sequence: 0 }, + }); + expect( + sqlBefore!.metadata.configurationMaintenance.settlementControl, + ).toBeUndefined(); + expect(before.final).not.toBeNull(); + expect(before.apiControls.settle!.result).toBeNull(); + await maintenance.inspectConfigurationMaintenance(preparedReceipt.binding); + const after = store.readOwnedConfigurationMaintenance(authority); + expect(canonicalJson(after.final) === canonicalJson(before.final)).toBe(true); + expect(after.apiControls.settle!.result).toBeNull(); + expect( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/observations", + `${expectedAckOwner}.configuration-settlement-binding.json`, + ), + ).equals(original.binding), + ).toBe(true); + expect( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/observations", + `${expectedAckOwner}.configuration-settlement-dispatch.json`, + ), + ).equals(original.dispatch), + ).toBe(true); + expect( + readFileSync( + join( + host.infraDir, + "control-private/data-manager/observations", + `${original.key}.configuration-settlement-admission.json`, + ), + ).equals(original.admission), + ).toBe(true); + expect( + readFileSync( + join( + host.infraDir, + "control-private/data-manager/observations", + `${original.key}.configuration-settlement-effect.json`, + ), + ).equals(original.effect), + ).toBe(true); + expect( + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${before.guardSource!.preparedReceipt.fenceId}::uuid`, + ).toEqual([sqlBefore]); + expect( + statSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")).ino, + ).toBe(original.rootInode); + expect(dm!.settlePosts).toBe(1); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength( + 1, + ); + console.info( + JSON.stringify({ + proof: "actual-original-settlement-control-expired", + lateOriginalPacketStatus: lateStatus!, + realDeadlineElapsed: true, + originalControlUnchanged: true, + sqlUnchanged: true, + guardAndRootRetained: true, + secondPost: false, + }), + ); + return; + } + if (mode.startsWith("publication-")) { + let publicationFailure: unknown; + try { + await application; + } catch (error) { + publicationFailure = error; + } finally { + publicationFault!.mockRestore(); + } + expect(publicationFailure).toBeInstanceOf( + (await import("../ops-client")).OpsObservationPending, + ); + expect(interrupted).toBe(true); + const store = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ), + core = await import("@openmapx/core/build-artifacts"), + ownerRecords = await import( + "../../../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records" + ), + original = ( + ctx.checkpointResult!._configurationMaintenance as { + receipt: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + } + ).receipt, + authority = restoreAdmittedControlExecution(installation, original.prepare), + // Supporting comparison only: the physical old owner is signed, + // but its family index pins the next exact release preimage. This + // snapshot supplies no production repair/settlement authority. + before = + mode === "publication-root-index" + ? core.verifyControl( + installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + expectedOwnerName, + ), + "utf8", + ), + ), + ownerRecords.configurationMaintenanceOwnerRecordSchema, + ) + : store.readOwnedConfigurationMaintenance(authority), + [sqlBefore] = + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${before.guardSource!.preparedReceipt.fenceId}::uuid`; + expect(before.final).not.toBeNull(); + expect(before.apiControls.settle!.result).toBeNull(); + expect(sqlBefore!.metadata.configurationMaintenance).toMatchObject({ + guardState: "settled", + receipt: { state: "prepared" }, + settlementControl: { + tuple: { result: { state: "settled" }, publication: "committed" }, + }, + }); + const releasedBefore = [ + "publication-sender-ack", + "publication-dm-ack", + "publication-mirror", + ].includes(mode) + ? null + : core.verifyRetainedArtifactTargetRootRelease(authority); + if (!releasedBefore) + expect( + statSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")) + .ino, + ).toBe(heldRootInode); + else + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect(dm!.settlePosts).toBe(1); + if (mode === "publication-root-index") { + expect(() => store.readOwnedConfigurationMaintenance(authority)).toThrow(); + await app.close(); + closeControlInstallation(installation); + const journal = await openOpsJobJournal( + join(host.infraDir, "data/ops-agent/jobs-v1.json"), + ); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + app = buildOpsAgentServer({ + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + vi.stubEnv("OPS_AGENT_URL", await app.listen({ host: "127.0.0.1", port: 0 })); + } + // Actual inspection can repair ONLY this already committed immutable + // final/SQL publication chain; no new attestation or mutation POST. + expect( + await maintenance.inspectConfigurationMaintenance(original.binding), + ).toMatchObject({ state: "settled", dmFence: { state: "settled" } }); + const observedAuthority = restoreAdmittedControlExecution( + installation, + original.prepare, + ), + final = store.readOwnedConfigurationMaintenance(observedAuthority); + expect(canonicalJson(final.final) === canonicalJson(before.final)).toBe(true); + expect(canonicalJson(final.inspection) === canonicalJson(before.inspection)).toBe( + true, + ); + expect(final.apiControls.settle!.operation).toEqual( + before.apiControls.settle!.operation, + ); + if (releasedBefore) + expect( + core.verifyRetainedArtifactTargetRootRelease(observedAuthority).releasedAt, + ).toBe(releasedBefore.releasedAt); + expect( + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${before.guardSource!.preparedReceipt.fenceId}::uuid`, + ).toEqual([sqlBefore]); + expect( + ( + await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "require-running", + }) + ).application, + ).toMatchObject({ outcome: "committed", followUp: "complete" }); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect( + existsSync( + join( + host.queue, + ".candidates", + `${before.plan.operation.applyOperationKey}.json`, + ), + ), + ).toBe(false); + expect(dm!.settlePosts).toBe(1); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength( + 1, + ); + return; + } + if ( + mode === "sql-update-rollback" || + mode === "sql-session-absent" || + mode === "sql-session-live" + ) { + if (mode === "sql-session-live") { + // Observe an actual still-live reserved transaction, not a fake + // prepared row or public status. The external edge is paused only + // after its real UPDATE and before its COMMIT transport call. + await dm!.settlementCommitEntered; + try { + const store = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ), + original = ( + ctx.checkpointResult!._configurationMaintenance as { + receipt: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + } + ).receipt, + owner = store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, original.prepare), + ), + key = owner.plan.reservations.find( + (value) => value.phase === "configuration-fence-settle", + )!.execution.operationKey, + effectPath = join( + host.infraDir, + "control-private/data-manager/observations", + `${key}.configuration-settlement-effect.json`, + ), + bytes = readFileSync(effectPath), + effect = JSON.parse(bytes.toString()).payload; + expect(effect).toMatchObject({ + phase: "attempted", + terminal: null, + observation: null, + }); + const sessions = + await fixture.sql`SELECT backend_start::text AS backend_start,application_name FROM pg_stat_activity WHERE pid=${effect.sqlSession.pid}::int`; + expect(sessions).toHaveLength(1); + expect(sessions[0]).toMatchObject({ + backend_start: effect.sqlSession.backendStart, + application_name: `omx-config-settle:${effect.sqlSession.sessionNonce}`, + }); + const observed = await dm!.inspectWithOriginalWitness(); + expect(observed).toMatchObject({ + status: 409, + body: { message: "Invalid data-manager response" }, + }); + expect(readFileSync(effectPath)).toEqual(bytes); + expect(owner.apiControls.settle!.result).toBeNull(); + expect( + existsSync( + join(host.infraDir, "data/.admin-target-control/locks/data-manager"), + ), + ).toBe(true); + const replacement = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + try { + expect(replacement.startupRecoveryRequired).toBe(true); + expect( + ( + await fetch(`${replacement.baseUrl}/status`, { + headers: { authorization: `Bearer ${replacement.token}` }, + }) + ).status, + ).toBe(200); + await expect(replacement.assertDataAdmission()).rejects.toThrow( + "configuration_maintenance_busy", + ); + expect(readFileSync(effectPath)).toEqual(bytes); + } finally { + await replacement.close(); + } + } finally { + dm!.releaseSettlementCommit(); + await application.catch(() => {}); + } + } + await expect(application).rejects.toBeInstanceOf( + (await import("../ops-client")).OpsObservationPending, + ); + const original = ( + ctx.checkpointResult!._configurationMaintenance as { + receipt: import("@openmapx/core/ops").ConfigurationMaintenanceReceipt; + } + ).receipt; + const store = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ), + owner = store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, original.prepare), + ), + stateModule = await import( + "../../../../../packages/core/src/build-artifacts/control-authority" + ), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-fence-settlement-records" + ), + crypto = await import( + "../../../../../packages/core/src/build-artifacts/control-records" + ), + key = owner.plan.reservations.find( + (value) => value.phase === "configuration-fence-settle", + )!.execution.operationKey, + effectPath = join( + host.infraDir, + "control-private/data-manager/observations", + `${key}.configuration-settlement-effect.json`, + ), + bytes = readFileSync(effectPath), + effect = crypto.verifyControl( + stateModule.installationState(installation).publicKeys["data-manager"], + records.configurationSettlementDomains.effect, + JSON.parse(bytes.toString()), + records.configurationSettlementEffectSchema, + ); + expect(effect).toMatchObject({ + phase: "contained_not_settled", + terminal: null, + observation: { + state: "contained_not_settled", + containment: + mode === "sql-update-rollback" + ? "explicit_rollback" + : "original_session_absent", + }, + }); + expect(effect.observation!.sqlSession).toEqual(effect.sqlSession); + expect(effect.observation).not.toHaveProperty("terminalAt"); + expect(effect.observation).not.toHaveProperty("terminalDigest"); + expect(owner.apiControls.settle!.result).toBeNull(); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + expect(JSON.stringify(dm!.fenceInspections.at(-1)?.body)).toContain( + '"state":"recovery_required"', + ); + const [saved] = + await fixture.sql`SELECT metadata->'configurationMaintenance' AS value FROM data_manager.jobs WHERE id=${owner.guardSource!.preparedReceipt.fenceId}::uuid`; + expect(saved!.value).toMatchObject({ + guardState: "prepared", + receipt: { sequence: 0, state: "prepared" }, + }); + expect(saved!.value.settlementControl).toBeUndefined(); + expect(dm!.settlementSqlIo).toEqual({ commits: 0, updates: 1 }); + expect(dm!.settlementContainmentIo).toEqual({ + rollbacks: mode === "sql-update-rollback" ? 1 : 0, + terminated: mode !== "sql-update-rollback", + }); + expect(dm!.settlePosts).toBe(1); + const old = dm!; + await old.stopForRestart(); + try { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + expect( + ( + await fetch(`${dm.baseUrl}/status`, { + headers: { authorization: `Bearer ${dm.token}` }, + }) + ).status, + ).toBe(200); + expect(readFileSync(effectPath)).toEqual(bytes); + const [after] = + await fixture.sql`SELECT metadata->'configurationMaintenance' AS value FROM data_manager.jobs WHERE id=${owner.guardSource!.preparedReceipt.fenceId}::uuid`; + expect(after!.value).toEqual(saved!.value); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + } finally { + await old.close(); + } + return; + } + const result = await application; + expect(result.outcome).toMatchObject({ + outcome: "committed", + revisionId: candidate.revisionId, + followUp: "complete", + }); + const ops = await import("@openmapx/core/ops"); + prepared = ops.configurationMaintenanceReceiptSchema.parse( + (ctx.checkpointResult!._configurationMaintenance as Record).receipt, + ); + decisionId = prepared.decision!.decisionId; + expect( + (ctx.checkpointResult!._configurationCandidate as Record).phase, + ).toBe("settled"); + expect( + existsSync( + join(host.queue, ".candidates", `${prepared.binding.applyOperationKey}.json`), + ), + ).toBe(false); + } else { + prepared = await maintenance.prepareConfigurationMaintenance(candidate); + const decision = await prospective.commitProspectiveConfigurationMaintenance(candidate); + decisionId = decision.decisionId; + await maintenance.decideConfigurationMaintenance(decision); + await maintenance.executeConfigurationMaintenanceStep(candidate, 0); + await maintenance.executeConfigurationMaintenanceStep(candidate, 1); + await maintenance.executeConfigurationMaintenanceStep(candidate, 2); + } + const authority = restoreAdmittedControlExecution(installation, prepared.prepare), + runtime = await import("../../../../ops-agent/src/configuration-maintenance-runtime"), + store = await import("../../../../ops-agent/src/configuration-maintenance-store"); + if (mode === "outer-api") { + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding), + ops = await import("@openmapx/core/ops"); + const result = await maintenance.settleConfigurationMaintenance({ + kind: "configurationMaintenance.settle", + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId, + expectedSequence: observed.sequence, + evidenceDigest: ops.configurationMaintenanceSettleRequestDigest(observed), + }); + expect(result).toMatchObject({ + state: "settled", + generation: "committed", + appliedRevisionId: candidate.revisionId, + followUp: "complete", + dmFence: { state: "settled" }, + }); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + } else if (mode === "component") { + const shapes = await import( + "../../../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records" + ), + id = "d".repeat(64), + original = shapes.maintenanceDockerContainerSchema.parse(installedDocker[id]), + path = join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${prepared.binding.maintenanceId}.json`, + ), + before = readFileSync(path); + for (const mismatch of [ + "hash", + "environment", + "image", + "mount", + "secret", + "readiness", + ] as const) { + const changed = structuredClone(original), + secret = changed.Mounts.find((value) => + value.Destination.startsWith("/run/secrets/"), + )!; + if (mismatch === "hash") + changed.Config.Labels["com.docker.compose.config-hash"] = "0".repeat(64); + if (mismatch === "environment") + changed.Config.Env = changed.Config.Env.map((value) => + value.startsWith("TOKEN_FILE=") ? "TOKEN_FILE=/unexpected" : value, + ); + if (mismatch === "image") changed.Image = `sha256:${"0".repeat(64)}`; + if (mismatch === "mount") secret.Source = host.firstCompose; + if (mismatch === "secret") + host.setRuntimeSecretBytes({ + [`${id}:${secret.Destination}`]: + Buffer.from("stale-runtime-secret").toString("base64"), + }); + if (mismatch === "readiness") changed.State.Health = { Status: "unhealthy" }; + host.setRuntimeInspections({ [id]: changed }); + let refused = false; + try { + await runtime.inspectFrozenMaintenanceRuntime( + host.runtime, + authority, + new AbortController().signal, + ); + } catch { + refused = true; + } finally { + host.setRuntimeInspections(installedDocker); + host.setRuntimeSecretBytes({}); + } + console.info( + JSON.stringify({ proof: "actual-runtime-mismatch-refusal", mismatch, refused }), + ); + expect(refused, `Actual runtime mismatch accepted: ${mismatch}`).toBe(true); + expect(readFileSync(path)).toEqual(before); + } + const inspection = await runtime.inspectFrozenMaintenanceRuntime( + host.runtime, + authority, + new AbortController().signal, + ); + expect(store.finalizeOwnedMaintenanceState(authority, inspection)).toBeDefined(); + } + const owner = store.readOwnedConfigurationMaintenance(authority); + if (mode === "sql-commit-ack-loss") { + expect(dm!.settlementSqlIo).toEqual({ commits: 1, updates: 1 }); + expect(owner.receipt).toMatchObject({ + state: "settled", + dmFence: { state: "settled" }, + }); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + const [saved] = + await fixture.sql`SELECT metadata->'configurationMaintenance' AS value FROM data_manager.jobs WHERE id=${owner.guardSource!.preparedReceipt.fenceId}::uuid`; + expect(saved!.value).toMatchObject({ + guardState: "settled", + settlementControl: { expectedDmSequence: 0, tuple: { publication: "committed" } }, + }); + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding); + expect(observed.dmFence).toEqual(owner.receipt.dmFence); + expect(dm!.settlementSqlIo).toEqual({ commits: 1, updates: 1 }); + const [after] = + await fixture.sql`SELECT metadata->'configurationMaintenance' AS value FROM data_manager.jobs WHERE id=${owner.guardSource!.preparedReceipt.fenceId}::uuid`; + expect(after!.value).toEqual(saved!.value); + } + expect(owner.final).toMatchObject({ + ownerFinal: { payload: { outcome: "applied" } }, + evidence: { + outcome: "applied", + generation: { state: "committed", revisionId: candidate.revisionId }, + followUp: { state: "complete" }, + runtime: { mode: "applied" }, + }, + }); + expect(owner.final!.evidence.children).toHaveLength(4); + expect(owner.inspection!.payload.requiredChecks.map((check) => check.check)).toEqual( + expect.arrayContaining(["configuration", "image", "mount", "readiness"]), + ); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + if (mode === "retirement") { + const core = await import("@openmapx/core/build-artifacts"), + retentionPath = (role: string) => + join( + host.infraDir, + "control-private", + role, + "retention-locations", + `${core.controlDigest(owner.origin)}.json`, + ), + clock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 3600000); + try { + for ( + let round = 0; + round < 16 && + (existsSync(retentionPath("ops-agent")) || + existsSync(retentionPath("data-manager"))); + round++ + ) { + await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm!.reconcileRetention(); + } + expect(existsSync(retentionPath("ops-agent"))).toBe(false); + expect(existsSync(retentionPath("data-manager"))).toBe(false); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`, + ).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + candidate.revisionId, + ); + } finally { + clock.mockRestore(); + } + } + }); + if (mode === "sql-control-expired") { + const next = await job(); + await expect(owned(next, async () => undefined)).rejects.toThrow( + "Configuration writer unavailable", + ); + } else if (mode.startsWith("publication-")) { + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + } + }, + 180000, + ); + it("rejects a real durable nonempty middle-owner grant with only the original pending decision witness", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(), + observation = new AbortController(); + ctx.signal = observation.signal; + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-no-decision-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + witnessPath = join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${prepared.prepare.operationKey}.configuration-maintenance.json`, + ), + pending = readFileSync(witnessPath), + decision = await prospective.commitProspectiveConfigurationMaintenance(candidate); + expect(JSON.parse(pending.toString()).payload).toMatchObject({ + decision: null, + generation: null, + }); + await maintenance.decideConfigurationMaintenance(decision); + await maintenance.executeConfigurationMaintenanceStep(candidate, 0); + const nativeFetch = globalThis.fetch, + core = await import("@openmapx/core/build-artifacts"); + let denied = 0, + validGrant = false, + leafKey = ""; + const transport = vi.spyOn(globalThis, "fetch").mockImplementation(async (...args) => { + if ( + !String(args[0]).endsWith("/executions/generation-hardlinks") || + args[1]?.method !== "POST" + ) + return nativeFetch(...args); + const headers = new Headers(args[1].headers), + grant = core.inspectArtifactDelegation({ + installation, + dataRoot: join(host.infraDir, "data"), + assertion: headers.get("x-openmapx-control-delegation")!, + }); + leafKey = grant.execution.operationKey; + validGrant = + grant.kind === "generation-hardlinks" && + grant.parent.operationKey === prepared.steps[1]!.operationKey && + grant.owner.operationKey === prepared.prepare.operationKey; + const current = readFileSync(witnessPath); + // Named native shared-publication rollback only; both packets are the + // actual producer's signatures. No private source/grant is fabricated. + writeFileSync(witnessPath, pending); + try { + const response = await nativeFetch(...args); + denied = response.status; + observation.abort(); + return response; + } finally { + writeFileSync(witnessPath, current); + } + }); + try { + await expect( + maintenance.executeConfigurationMaintenanceStep(candidate, 1), + ).rejects.toMatchObject({ name: "OpsObservationPending" }); + } finally { + transport.mockRestore(); + } + expect(validGrant).toBe(true); + expect(denied).toBe(409); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE kind='generation-hardlinks' AND operation_key=${leafKey}`, + ).toHaveLength(0); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + }); + }, 120000); + it.each(["expired", "host-source"])( + "refuses actual durable nonempty committed continue after %s drift without admitting any effect", + async (reason) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + try { + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-committed-drift-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await prospective.commitProspectiveConfigurationMaintenance(candidate), + beforeCalls = host.calls.length; + if (reason === "expired") { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 6 * 60000); + } else + writeFileSync(join(host.infraDir, ".env"), "MOTIS_SLOT_MEMORY=8g\n", { mode: 0o600 }); + await expect(maintenance.decideConfigurationMaintenance(decision)).rejects.toThrow(); + const store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + actual = store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, prepared.prepare), + ); + expect(actual.apiControls.decide).toBeNull(); + expect(actual.receipt).toMatchObject({ + state: "prepared", + decision: null, + dmFence: prepared.dmFence, + generation: "not_started", + followUp: "not_started", + }); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(1); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + host.calls + .slice(beforeCalls) + .some( + (args) => + args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect(dm!.preparePosts).toBe(1); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + }); + } finally { + vi.useRealTimers(); + } + }, + 60000, + ); + it.each(["checksum", "sequence", "decision", "candidate"])( + "refuses actual durable nonempty outer settle %s before its first owner CAS", + async (mismatch) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-settle-checksum-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding), + ops = await import("@openmapx/core/ops"), + path = join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${prepared.binding.maintenanceId}.json`, + ), + before = readFileSync(path), + operation = { + kind: "configurationMaintenance.settle" as const, + maintenanceId: prepared.binding.maintenanceId, + candidateId: mismatch === "candidate" ? randomUUID() : candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: mismatch === "decision" ? randomUUID() : decision.decisionId, + expectedSequence: observed.sequence + (mismatch === "sequence" ? 1 : 0), + evidenceDigest: + mismatch === "checksum" + ? "0".repeat(64) + : ops.configurationMaintenanceSettleRequestDigest(observed), + }; + const nativeFetch = globalThis.fetch, + frames: { status: number; errorClass?: string }[] = [], + transport = vi.spyOn(globalThis, "fetch").mockImplementation(async (...args) => { + const response = await nativeFetch(...args); + if (String(args[0]).endsWith("/v1/operations") && args[1]?.method === "POST") { + const frame = (await response.clone().json()) as { error?: { class?: string } }; + frames.push({ status: response.status, errorClass: frame.error?.class }); + } + return response; + }); + try { + // Faulty API producer negative: use the ACTUAL API issuer and durable + // checkpoint, not a handwritten HMAC/grant or a weakened API adapter. + const { prepareApiJobRoot, prepareApiJobChildAttempt, issueApiJobChildAssertion } = + await import("../admin-job-parent"), + { createDurableOpsKey, createApiOpsClient, executeAndWait } = await import( + "../ops-client" + ), + root = prepareApiJobRoot( + ctx, + operation.kind, + createDurableOpsKey("task012.settle-negative", ctx.jobId), + ), + attempt = prepareApiJobChildAttempt(ctx, operation, root.operationKey, root)!; + await ctx.checkpoint({ + ...ctx.checkpointResult, + _settleNegativeProbe: { operation, attempt, root }, + }); + await expect( + executeAndWait(createApiOpsClient(), operation, root.operationKey, { + execution: { requestedDeadlineAt: root.requestedDeadlineAt }, + parentAssertion: await issueApiJobChildAssertion(attempt), + }), + ).rejects.toBeInstanceOf(Error); + } finally { + transport.mockRestore(); + } + expect(frames).toEqual([ + mismatch === "candidate" + ? { status: 403, errorClass: "authorization" } + : { status: 409, errorClass: "conflict" }, + ]); + expect(readFileSync(path)).toEqual(before); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + expect(dm!.preparePosts).toBe(1); + }); + }, + 60000, + ); + it("serializes competing actual durable nonempty first settle observations into one original winner", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-settle-race-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding), + { configurationMaintenanceSettleRequestDigest } = await import("@openmapx/core/ops"), + request = { + kind: "configurationMaintenance.settle" as const, + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: decision.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: configurationMaintenanceSettleRequestDigest(observed), + }, + fetchActual = globalThis.fetch, + frames: number[] = [], + transport = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, options) => { + if ( + options?.method === "POST" && + typeof options.body === "string" && + JSON.parse(options.body).operation?.kind === "configurationMaintenance.settle" + ) { + // Duplicate exact real checkpointed request bytes/HMAC at the named + // transport edge. Both reach the receiver; no alternate key/root. + const responses = await Promise.all([ + fetchActual(input, options), + fetchActual(input, options), + ]); + frames.push(...responses.map((response) => response.status)); + await responses[1]!.arrayBuffer(); + return responses[0]!; + } + return fetchActual(input, options); + }); + try { + try { + expect(await maintenance.settleConfigurationMaintenance(request)).toMatchObject({ + state: "settled", + }); + } catch (error) { + expect(error).toBeInstanceOf( + (await import("../admin-job-failure")).JobObservationPending, + ); + } + } finally { + transport.mockRestore(); + } + expect(frames).toHaveLength(2); + expect(frames).toContain(202); + expect(frames.every((status) => status === 202 || status === 409)).toBe(true); + // If the caller saw the losing conflict, its ORIGINAL attempted address + // is now lookup-only. This is reconciliation, never a second POST. + expect(await maintenance.settleConfigurationMaintenance(request)).toMatchObject({ + state: "settled", + }); + expect(frames).toHaveLength(2); + const store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + owner = store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, prepared.prepare), + ); + expect(owner.apiControls.settle).toMatchObject({ + operation: request, + checkedSequence: observed.sequence, + requestEvidenceDigest: request.evidenceDigest, + result: { state: "settled", generation: "not_started", followUp: "not_started" }, + }); + expect(owner.final).not.toBeNull(); + expect(owner.final!.evidenceDigest).not.toBe(request.evidenceDigest); + expect(owner.plan.prepare).toEqual(prepared.prepare); + expect(dm!.settlePosts).toBe(1); + const { createApiOpsClient } = await import("../ops-client"), + retained = ctx.checkpointResult!._configurationMaintenanceSettle as { + delivery: { root: { operationKey: string; requestedDeadlineAt: number } }; + }, + lookedUp = await createApiOpsClient().lookup( + request, + retained.delivery.root.operationKey, + { execution: { requestedDeadlineAt: retained.delivery.root.requestedDeadlineAt } }, + ); + expect(lookedUp).toMatchObject({ + state: "succeeded", + operationId: owner.apiControls.settle!.execution.executionId, + result: { state: "settled" }, + }); + expect( + canonicalJson( + store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, prepared.prepare), + ).final, + ) === canonicalJson(owner.final), + ).toBe(true); + expect(dm!.settlePosts).toBe(1); + console.info( + JSON.stringify({ + proof: "outer-settle-original-first-cas-winner", + postStatuses: frames, + dmSettlementPosts: dm!.settlePosts, + checkedSequence: owner.apiControls.settle!.checkedSequence, + originalFinalUnchanged: true, + sameAddressLookupOnly: true, + }), + ); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + }); + }, 60000); + it("retains actual durable nonempty accepted but ineligible settle intent as unknown across agent restart with read-only health", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-ineligible-final-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await prospective.commitProspectiveConfigurationMaintenance(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding), + ops = await import("@openmapx/core/ops"), + request = { + kind: "configurationMaintenance.settle" as const, + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: decision.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: ops.configurationMaintenanceSettleRequestDigest(observed), + }, + [beforeSql] = + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`; + await expect(maintenance.settleConfigurationMaintenance(request)).rejects.toBeInstanceOf( + Error, + ); + const store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + authority = restoreAdmittedControlExecution(installation, prepared.prepare), + actual = store.readOwnedConfigurationMaintenance(authority), + accepted = actual.apiControls.settle!; + expect(accepted).toMatchObject({ + operation: request, + checkedSequence: observed.sequence, + requestEvidenceDigest: request.evidenceDigest, + result: null, + canonicalDigest: null, + }); + expect(actual.final).toBeNull(); + expect(actual.inspection).toBeNull(); + expect(actual.receipt).toMatchObject({ + state: "authorized", + generation: "not_started", + followUp: "not_started", + dmFence: { state: "prepared" }, + }); + const path = join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${prepared.binding.maintenanceId}.json`, + ), + original = readFileSync(path); + await app.close(); + closeControlInstallation(installation); + const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + vi.stubEnv("OPS_AGENT_URL", await app.listen({ host: "127.0.0.1", port: 0 })); + expect((await app.inject({ method: "GET", url: "/health" })).statusCode).toBe(200); + const lookup = await app.inject({ + method: "GET", + url: `/v1/operation-keys/${accepted.execution.operationKey}`, + headers: { + authorization: `Bearer ${host.token}`, + "x-ops-request-id": randomUUID(), + "x-ops-protocol-version": "2", + "x-ops-body-fingerprint": accepted.execution.operationFingerprint, + "x-ops-operation-fingerprint": accepted.execution.fingerprint, + }, + }); + expect(lookup.statusCode).toBe(200); + expect(lookup.json().result).toMatchObject({ + state: "failed", + errorClass: "recovery_required", + operationId: accepted.execution.executionId, + operationKey: accepted.execution.operationKey, + admittedAt: accepted.admittedAt, + deadlineAt: accepted.execution.deadlineAt, + }); + expect(readFileSync(path)).toEqual(original); + expect( + ( + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid` + )[0], + ).toEqual(beforeSql); + await expect(dm!.receiver.admitLocal({ kind: "download-fonts" })).rejects.toThrow( + "configuration_maintenance_busy", + ); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + }); + const next = await job(); + await expect(owned(next, async () => undefined)).rejects.toThrow( + "Configuration writer unavailable", + ); + }, 120000); + it.each(["expired", "canceled"])( + "settles actual durable nonempty known-unused original ownership after the forward root is %s", + async (condition) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + if (condition === "expired") { + const { prepareApiJobRoot } = await import("../admin-job-parent"), + initial = prepareApiJobRoot( + ctx, + "configurationMaintenance.prepare", + "synthetic-initial-short-job-budget", + ); + // Select a short INITIAL synthetic job lifetime, before any candidate or + // root admission; all later HMACs/deadlines retain this original source. + await ctx.checkpoint({ + ...ctx.checkpointResult, + _parentExecution: { ...initial.parent, deadlineAt: Date.now() + 60000 }, + }); + } + try { + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-late-unused-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + core = await import("@openmapx/core/build-artifacts"), + authority = restoreAdmittedControlExecution(installation, prepared.prepare), + admissionPath = join( + host.infraDir, + "control-private/ops-agent/executions", + `${prepared.prepare.operationKey}.json`, + ), + originalAdmission = readFileSync(admissionPath); + if (condition === "expired") + await new Promise((resolve) => + setTimeout(resolve, Math.max(1, prepared.prepare.deadlineAt - Date.now() + 5)), + ); + else { + core.cancelControlExecution(authority); + const lease = core.restoreArtifactTargetLeaseForSettlement(authority); + try { + await core.cancelArtifactTargetLease(lease); + } finally { + core.closeArtifactTargetLeaseHandle(lease); + } + } + const decision = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding), + ops = await import("@openmapx/core/ops"), + request = { + kind: "configurationMaintenance.settle" as const, + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: decision.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: ops.configurationMaintenanceSettleRequestDigest(observed), + }, + result = await maintenance.settleConfigurationMaintenance(request), + retained = ctx.checkpointResult!._configurationMaintenanceSettle as { + delivery: { root: { operationKey: string; requestedDeadlineAt: number } }; + }, + { createApiOpsClient } = await import("../ops-client"), + lookup = await createApiOpsClient().lookup( + request, + retained.delivery.root.operationKey, + { execution: { requestedDeadlineAt: retained.delivery.root.requestedDeadlineAt } }, + ); + expect(lookup).toMatchObject({ + state: "succeeded", + result: { state: "settled", prepare: prepared.prepare }, + }); + await expect( + createApiOpsClient().lookup(request, retained.delivery.root.operationKey, { + execution: { + requestedDeadlineAt: retained.delivery.root.requestedDeadlineAt + 1, + }, + }), + ).rejects.toMatchObject({ errorClass: "conflict" }); + expect(result).toMatchObject({ + state: "settled", + parentDeadlineAt: prepared.parentDeadlineAt, + prepare: prepared.prepare, + generation: "not_started", + followUp: "not_started", + dmFence: { state: "settled" }, + }); + expect(readFileSync(admissionPath)).toEqual(originalAdmission); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect(() => core.assertControlExecutionForward(authority)).toThrow(); + }); + } finally { + vi.useRealTimers(); + } + }, + 120000, + ); + it("repairs actual durable nonempty settle completion publication after root release without another inspection or SQL transition", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-completion-publication-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding), + ops = await import("@openmapx/core/ops"), + request = { + kind: "configurationMaintenance.settle" as const, + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: decision.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: ops.configurationMaintenanceSettleRequestDigest(observed), + }, + nativeRename = (await vi.importActual("node:fs")).renameSync; + let faults = 0; + const originalSignal = ctx.signal, + observer = new AbortController(); + ctx.signal = observer.signal; + const fault = vi.spyOn(nodeFs, "renameSync").mockImplementation((source, target) => { + if ( + basename(String(target)) === `${prepared.binding.maintenanceId}.json` && + JSON.parse(readFileSync(source, "utf8")).payload?.apiControls?.settle?.result + ) { + faults++; + observer.abort(); + throw Error("Injected native original completion publication failure"); + } + return nativeRename(source, target); + }); + try { + await expect(maintenance.settleConfigurationMaintenance(request)).rejects.toBeInstanceOf( + Error, + ); + } finally { + fault.mockRestore(); + ctx.signal = originalSignal; + } + expect(faults).toBeGreaterThan(0); + const store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + core = await import("@openmapx/core/build-artifacts"), + authority = restoreAdmittedControlExecution(installation, prepared.prepare), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records" + ), + owner = core.verifyControl( + installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${prepared.binding.maintenanceId}.json`, + ), + "utf8", + ), + ), + records.configurationMaintenanceOwnerRecordSchema, + ), + released = core.verifyRetainedArtifactTargetRootRelease(authority), + originalFinal = JSON.stringify(owner.final), + originalInspection = JSON.stringify(owner.inspection), + [originalSql] = + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`; + expect(owner.receipt.state).toBe("settled"); + expect(owner.apiControls.settle!.result).toBeNull(); + await app.close(); + closeControlInstallation(installation); + const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + vi.stubEnv("OPS_AGENT_URL", await app.listen({ host: "127.0.0.1", port: 0 })); + const result = await maintenance.settleConfigurationMaintenance(request), + recovered = store.readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, prepared.prepare), + ); + expect(result).toEqual(owner.receipt); + expect(recovered.apiControls.settle).toMatchObject({ + operation: request, + checkedSequence: observed.sequence, + completedAt: released.releasedAt, + requestEvidenceDigest: request.evidenceDigest, + result: { state: "settled" }, + }); + expect(recovered.apiControls.settle!.receiptDelivery).not.toBeNull(); + expect(JSON.stringify(recovered.final)).toBe(originalFinal); + expect(JSON.stringify(recovered.inspection)).toBe(originalInspection); + expect( + ( + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid` + )[0], + ).toEqual(originalSql); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + }); + }, 60000); + it.each(["journal", "retirement"])( + "settles actual durable nonempty unused maintenance through the original API control and retains its canonical delivery (%s)", + async (mode) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-api-unused-final-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const observed = await maintenance.inspectConfigurationMaintenance(prepared.binding), + ops = await import("@openmapx/core/ops"), + request = { + kind: "configurationMaintenance.settle" as const, + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: decision.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: ops.configurationMaintenanceSettleRequestDigest(observed), + }, + result = await maintenance.settleConfigurationMaintenance(request), + authority = restoreAdmittedControlExecution(installation, prepared.prepare), + store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + owner = store.readOwnedConfigurationMaintenance(authority); + expect(result).toMatchObject({ + state: "settled", + dmFence: { state: "settled" }, + generation: "not_started", + followUp: "not_started", + }); + expect(owner.apiControls.settle).toMatchObject({ + operation: request, + checkedSequence: observed.sequence, + requestEvidenceDigest: request.evidenceDigest, + result: { state: "settled" }, + }); + expect(owner.apiControls.settle!.receiptDelivery).not.toBeNull(); + expect(owner.final!.evidenceDigest).not.toBe(request.evidenceDigest); + const final = JSON.stringify(owner.final), + inspection = JSON.stringify(owner.inspection), + [beforeSql] = + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`; + expect(await maintenance.settleConfigurationMaintenance(request)).toEqual(result); + expect(JSON.stringify(store.readOwnedConfigurationMaintenance(authority).final)).toBe( + final, + ); + expect( + JSON.stringify(store.readOwnedConfigurationMaintenance(authority).inspection), + ).toBe(inspection); + expect( + ( + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid` + )[0], + ).toEqual(beforeSql); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + await prospective.discardUnusedPreparedProspectiveConfiguration(candidate); + expect( + existsSync( + join(host.queue, ".candidates", `${prepared.binding.applyOperationKey}.json`), + ), + ).toBe(false); + expect( + (ctx.checkpointResult!._configurationCandidate as Record).phase, + ).toBe("discarded"); + await app.close(); + closeControlInstallation(installation); + const journal = await openOpsJobJournal( + join(host.infraDir, "data/ops-agent/jobs-v1.json"), + ), + original = journal + .records() + .find((entry) => entry.operation.kind === "configurationMaintenance.settle")!; + expect(original).toBeDefined(); + await journal.replace( + journal.records().filter((entry) => entry.operationKey !== original.operationKey), + ); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + vi.stubEnv("OPS_AGENT_URL", await app.listen({ host: "127.0.0.1", port: 0 })); + const lookup = await app.inject({ + method: "GET", + url: `/v1/operation-keys/${original.operationKey}`, + headers: { + authorization: `Bearer ${host.token}`, + "x-ops-request-id": randomUUID(), + "x-ops-protocol-version": "2", + "x-ops-body-fingerprint": original.operationFingerprint!, + "x-ops-operation-fingerprint": original.fingerprint, + }, + }); + expect(lookup.statusCode).toBe(200); + expect(lookup.json().result).toMatchObject({ + state: "succeeded", + operationId: original.operationId, + result, + admittedAt: original.admittedAt, + deadlineAt: original.deadlineAt, + updatedAt: original.updatedAt, + }); + expect( + journal.records().find((entry) => entry.operationKey === original.operationKey), + ).toEqual(original); + expect(await maintenance.settleConfigurationMaintenance(request)).toEqual(result); + if (mode === "retirement") { + const core = await import("@openmapx/core/build-artifacts"), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-fence-settlement-records" + ), + control = JSON.parse( + readFileSync( + join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${prepared.prepare.operationKey}.configuration-settlement-control.json`, + ), + "utf8", + ), + ), + envelope = core.verifyControl( + installation.publicKey, + records.configurationSettlementDomains.control, + control.payload.binding, + records.configurationSettlementControlEnvelopeSchema, + ), + sqlControl = beforeSql!.metadata.configurationMaintenance.settlementControl, + retentionPath = (role: string) => + join( + host.infraDir, + "control-private", + role, + "retention-locations", + `${core.controlDigest(owner.origin)}.json`, + ), + clock = vi + .spyOn(Date, "now") + .mockReturnValue(sqlControl.tuple.terminalAt + 72 * 3600000 - 1); + try { + await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm!.reconcileRetention(); + expect(existsSync(retentionPath("ops-agent"))).toBe(true); + expect(existsSync(retentionPath("data-manager"))).toBe(true); + const ackPath = join( + host.infraDir, + "control-private/data-manager/observations", + `${envelope.execution.operationKey}.configuration-settlement-ack.json`, + ), + originalAck = readFileSync(ackPath), + corrupt = Buffer.from(originalAck); + corrupt[12] ^= 1; + clock.mockReturnValue(sqlControl.tuple.terminalAt + 96 * 3600000); + try { + writeFileSync(ackPath, corrupt); + await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm!.reconcileRetention(); + expect(existsSync(retentionPath("data-manager"))).toBe(true); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`, + ).toHaveLength(1); + } finally { + writeFileSync(ackPath, originalAck); + } + for ( + let round = 0; + round < 16 && + (existsSync(retentionPath("ops-agent")) || + existsSync(retentionPath("data-manager"))); + round++ + ) { + await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm!.reconcileRetention(); + } + const diagnostics = ["ops-agent", "data-manager"].map((role) => { + if (!existsSync(retentionPath(role))) return { role, retired: true }; + const location = JSON.parse(readFileSync(retentionPath(role), "utf8")); + const progress = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private", + role, + "retention-families", + location.name, + "progress.json", + ), + "utf8", + ), + ); + return { + role, + phase: progress.phase, + lastError: progress.lastError, + slot: progress.configurationSettlementSlot?.state, + executions: Object.values<{ + known: boolean; + requiresJournal: boolean; + journalContained: boolean; + canonicalDigest?: string; + terminalDigest?: string; + execution: { operationKey: string }; + }>(progress.executions).map((value) => ({ + key: value.execution.operationKey, + known: value.known, + requiresJournal: value.requiresJournal, + journalContained: value.journalContained, + canonical: value.canonicalDigest, + terminal: value.terminalDigest, + })), + }; + }); + expect(existsSync(retentionPath("ops-agent")), JSON.stringify(diagnostics)).toBe( + false, + ); + expect(existsSync(retentionPath("data-manager")), JSON.stringify(diagnostics)).toBe( + false, + ); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`, + ).toEqual([]); + } finally { + clock.mockRestore(); + } + } + }); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + }, + 60000, + ); + it.each([ + [false, false], + [true, false], + [true, true], + ])( + "constructs actual durable nonempty unused runtime evidence and commits one immutable original owner final (dispatch=%s mirror-loss=%s)", + async (dispatch, mirrorLoss) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-unused-final-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + decision = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + await maintenance.decideConfigurationMaintenance(decision); + const authority = restoreAdmittedControlExecution(installation, prepared.prepare), + runtime = await import("../../../../ops-agent/src/configuration-maintenance-runtime"), + store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + before = store.readOwnedConfigurationMaintenance(authority), + inspection = await runtime.inspectFrozenMaintenanceRuntime( + host.runtime, + authority, + new AbortController().signal, + ), + final = store.finalizeOwnedMaintenanceState(authority, inspection), + after = store.readOwnedConfigurationMaintenance(authority); + expect(after.final).toMatchObject({ + ownerFinal: { payload: { outcome: "unused", ownerSequence: before.sequence + 1 } }, + evidence: { + outcome: "unused", + children: [], + generation: { state: "not_committed" }, + runtime: { mode: "unused" }, + }, + }); + expect(after.receipt.state).toBe("settling"); + expect(after.receipt.dmFence?.state).toBe("prepared"); + expect(after.inspection!.payload.observations[0]!.detail).toBe( + JSON.stringify({ currentRevisionId: host.initial.revisionId }), + ); + expect(store.restoreOwnedMaintenanceFinal(authority)).toEqual(final); + store.publishOwnedConfigurationFenceSettlement(authority, prepared.binding.maintenanceId); + const cellPath = join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${prepared.prepare.operationKey}.configuration-settlement.json`, + ), + bytes = readFileSync(cellPath), + packet = JSON.parse(bytes.toString()), + core = await import("@openmapx/core/build-artifacts"), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-fence-settlement-records" + ); + expect(bytes.length).toBe(16384); + expect(Buffer.byteLength(JSON.stringify(packet))).toBeLessThanOrEqual(16384); + const committed = core.verifyControl( + installation.publicKey, + records.configurationSettlementDomains.reference, + packet, + records.configurationCommittedSettlementSchema, + ); + expect(committed).toMatchObject({ + phase: "committed", + final: { outcome: "unused" }, + evidence: { outcome: "unused" }, + }); + expect(committed.evidenceDigest).toBe(after.final!.evidenceDigest); + store.publishOwnedConfigurationFenceSettlement(authority, prepared.binding.maintenanceId); + expect(readFileSync(cellPath)).toEqual(bytes); + const control = store.prepareConfigurationFenceSettlementControl( + authority, + prepared.binding.maintenanceId, + ), + delegated = store.delegateConfigurationFenceSettlementControl(control), + repeated = store.delegateConfigurationFenceSettlementControl(control); + expect(delegated).toMatchObject({ + dispatch: "fresh", + envelope: { + payload: { + bindingVersion: 3, + originalParentForwardDeadlineAt: prepared.prepare.deadlineAt, + }, + }, + }); + expect( + delegated.envelope.payload.controlDeadlineAt - + delegated.envelope.payload.controlStartedAt, + ).toBe(60000); + expect(repeated).toMatchObject({ + dispatch: "observe-only", + envelope: delegated.envelope, + assertion: delegated.assertion, + }); + const privateBinding = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/observations", + `${prepared.prepare.operationKey}.configuration-settlement-binding.json`, + ), + "utf8", + ), + ); + expect( + core.verifyControl( + installation.publicKey, + records.configurationSettlementDomains.binding, + privateBinding, + records.configurationSettlementControlEnvelopeSchema, + ), + ).toEqual(delegated.envelope.payload); + expect(() => + core.inspectArtifactDelegation({ + dataRoot: join(host.infraDir, "data"), + installation, + assertion: delegated.assertion, + }), + ).toThrow(); + if (dispatch) { + if (delegated.dispatch !== "fresh") throw Error("Original fresh ticket required"); + const { configurationDataManagerTransport } = await import( + "../../../../ops-agent/src/configuration-hardlinks-runtime" + ), + [beforeSql] = + await fixture.sql`SELECT ownership_version FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`, + nativeRename = (await vi.importActual("node:fs")) + .renameSync; + let faulted = false; + const fault = mirrorLoss + ? vi.spyOn(nodeFs, "renameSync").mockImplementation((source, target) => { + if ( + !faulted && + basename(String(target)) === `${prepared.binding.maintenanceId}.json` + ) { + const value = JSON.parse(readFileSync(source, "utf8")).payload; + if ( + value?.metadata?.guardState === "settled" && + value.metadata.settlementControl?.settlementId === + delegated.envelope.payload.settlementId + ) { + faulted = true; + throw Error( + "Injected native mirror publication failure after actual SQL COMMIT", + ); + } + } + return nativeRename(source, target); + }) + : undefined; + let settled: unknown; + try { + if (mirrorLoss) + await expect( + store.dispatchConfigurationFenceSettlementControl( + delegated.ticket, + configurationDataManagerTransport(), + ), + ).rejects.toBeInstanceOf(Error); + else + settled = await store.dispatchConfigurationFenceSettlementControl( + delegated.ticket, + configurationDataManagerTransport(), + ); + } finally { + fault?.mockRestore(); + } + const [sqlRow] = + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`, + sqlMetadata = sqlRow!.metadata.configurationMaintenance, + tuple = sqlMetadata.settlementControl.tuple, + controlKey = delegated.envelope.payload.execution.operationKey, + privateDirectory = join(host.infraDir, "control-private/data-manager/observations"), + terminalPath = join( + privateDirectory, + `${controlKey}.configuration-settlement-terminal.json`, + ), + mirrorPath = join( + host.infraDir, + "control-private/data-manager/configuration-maintenance", + `${prepared.binding.maintenanceId}.json`, + ); + expect(sqlRow!.ownership_version).toBe(beforeSql!.ownership_version + 1); + expect(sqlMetadata.receipt).toEqual(prepared.dmFence); + expect(tuple).toMatchObject({ + execution: delegated.envelope.payload.execution, + result: { + state: "settled", + fenceId: prepared.dmFence!.fenceId, + sequence: prepared.dmFence!.sequence + 1, + }, + }); + const session = sqlMetadata.settlementControl.sqlSession, + [backend] = + await fixture.sql`SELECT backend_start::text AS backend_start,application_name FROM pg_stat_activity WHERE pid=${session.pid}`; + expect(backend).toMatchObject({ + backend_start: session.backendStart, + application_name: `omx-config-settle:${session.sessionNonce}`, + }); + expect(session.sessionNonce).toBe(delegated.envelope.payload.execution.executionId); + if (mirrorLoss) { + expect(faulted).toBe(true); + expect(JSON.parse(readFileSync(mirrorPath, "utf8")).payload.metadata.guardState).toBe( + "settling", + ); + expect(JSON.parse(readFileSync(terminalPath, "utf8"))).toBeNull(); + await expect(dm!.receiver.admitLocal({ kind: "download-fonts" })).rejects.toThrow( + "configuration_maintenance_busy", + ); + // ACTUAL API -> agent-produced inspection witness -> DM GET. No + // second settlement POST/ticket, UPDATE or runtime attestation. + const nativeFetch = globalThis.fetch, + frames: { path: string; status: number }[] = [], + observer = vi.spyOn(globalThis, "fetch").mockImplementation(async (...args) => { + const response = await nativeFetch(...args); + frames.push({ path: new URL(String(args[0])).pathname, status: response.status }); + return response; + }); + let inspected: unknown; + try { + inspected = await maintenance.inspectConfigurationMaintenance(prepared.binding); + } finally { + observer.mockRestore(); + } + expect( + JSON.parse(readFileSync(terminalPath, "utf8")), + JSON.stringify({ frames, inspected }), + ).not.toBeNull(); + } + const terminal = JSON.parse(readFileSync(terminalPath, "utf8")), + mirror = JSON.parse(readFileSync(mirrorPath, "utf8")).payload, + admission = JSON.parse( + readFileSync( + join(privateDirectory, `${controlKey}.configuration-settlement-admission.json`), + "utf8", + ), + ), + effect = JSON.parse( + readFileSync( + join(privateDirectory, `${controlKey}.configuration-settlement-effect.json`), + "utf8", + ), + ); + expect(terminal.payload.tuple).toEqual(tuple); + expect(effect.payload).toMatchObject({ + phase: "committed", + terminal: tuple, + sqlSession: session, + }); + expect(terminal.payload).toHaveProperty("effectSource"); + const originalEffect = { + payload: { ...terminal.payload.effectSource.fields, terminal: tuple }, + signature: terminal.payload.effectSource.signature, + }; + expect( + core.verifyControl( + core.controlRolePublicKey(installation, "data-manager"), + records.configurationSettlementDomains.effect, + originalEffect, + records.configurationSettlementEffectSchema, + ), + ).toEqual(effect.payload); + expect(terminal.payload.effectSettlementDigest).toBe( + records.settlementRecordDigest( + records.configurationSettlementDomains.effect, + effect.payload, + ), + ); + expect(mirror.metadata).toEqual(sqlMetadata); + expect(admission.payload.controlEnvelope).toEqual(delegated.envelope); + expect(terminal.payload).toHaveProperty("admissionSource"); + expect(terminal.payload.admissionSource).toEqual({ + boundAt: admission.payload.boundAt, + signature: admission.signature, + }); + if (!mirrorLoss) expect(settled).toEqual(tuple.result); + expect(store.observeConfigurationFenceSettlementControl(control)).toEqual(tuple.result); + const copied = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/observations", + `${prepared.prepare.operationKey}.configuration-settlement-ack.json`, + ), + "utf8", + ), + ), + sharedAck = JSON.parse( + readFileSync( + join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${prepared.prepare.operationKey}.configuration-settlement-ack.json`, + ), + "utf8", + ), + ); + expect(copied.terminal).toEqual(terminal); + expect(copied.ack).toEqual(sharedAck); + expect(sharedAck.payload).toMatchObject({ + terminalDigest: tuple.terminalDigest, + terminalAt: tuple.terminalAt, + }); + expect(store.observeConfigurationFenceSettlementControl(control)).toEqual(tuple.result); + const dmAckPath = join( + privateDirectory, + `${controlKey}.configuration-settlement-ack.json`, + ); + expect(JSON.parse(readFileSync(dmAckPath, "utf8"))).toBeNull(); + await maintenance.inspectConfigurationMaintenance(prepared.binding); + expect(JSON.parse(readFileSync(dmAckPath, "utf8"))).toEqual(sharedAck); + core.assertConfigurationMaintenanceRootClosure(authority); + if (!mirrorLoss) { + const lease = core.restoreArtifactTargetLeaseForSettlement(authority), + leasePath = join(host.infraDir, "data/.admin-target-control/leases", lease.leaseId), + directoryInode = statSync(leasePath).ino, + certificate = JSON.parse(readFileSync(join(leasePath, "owner.json"), "utf8")) + .payload.certificate, + admissionPath = join( + host.infraDir, + "control-private/ops-agent/executions", + `${prepared.prepare.operationKey}.json`, + ), + originalAdmission = readFileSync(admissionPath); + core.closeArtifactTargetLeaseHandle(lease); + expect(await store.releaseOwnedConfigurationMaintenanceRoot(authority)).toMatchObject( + { state: "settled", dmFence: { state: "settled" } }, + ); + core.verifyRetainedArtifactTargetRootRelease(authority); + expect(readFileSync(admissionPath)).toEqual(originalAdmission); + expect(statSync(leasePath).ino).toBe(directoryInode); + expect( + JSON.parse(readFileSync(join(leasePath, "owner.json"), "utf8")).payload.certificate, + ).toEqual(certificate); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/motis")), + ).toBe(false); + } + const [afterSql] = + await fixture.sql`SELECT metadata,ownership_version,xmin::text AS xmin FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`; + expect(afterSql).toEqual(sqlRow); + console.info( + JSON.stringify({ + proof: "actual-unused-logical-settlement", + mirrorLoss, + session, + terminalAt: tuple.terminalAt, + commitmentBytes: Buffer.byteLength(JSON.stringify(packet)), + boundControlBytes: Buffer.byteLength( + readFileSync( + join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${prepared.prepare.operationKey}.configuration-settlement-control.json`, + ), + ), + ), + }), + ); + await expect( + store.dispatchConfigurationFenceSettlementControl( + delegated.ticket, + configurationDataManagerTransport(), + ), + ).rejects.toThrow("Original unconsumed settlement dispatch ticket required"); + } + expect(() => store.finalizeOwnedMaintenanceState(authority, {} as never)).toThrow( + "Actual maintenance runtime inspection required", + ); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(!(dispatch && !mirrorLoss)); + }); + }, + 60000, + ); + it.each([false, true])( + "retains one actual durable nonempty unsaved maintenance abandon decision and rejects an attempted save=%s", + async (saveAttempted) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-abandon-credential", + }, + ], + }, + }), + maintenance = await import("../configuration-maintenance"), + prepared = await maintenance.prepareConfigurationMaintenance(candidate), + lifecycle = host.calls.filter( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ).length; + if (saveAttempted) { + // Actual durable DB state disagrees with the stale in-memory checkpoint. + await fixture.db + .update(schema.adminJob) + .set({ + result: { + ...ctx.checkpointResult, + _configurationCandidate: { + ...(ctx.checkpointResult!._configurationCandidate as Record), + phase: "save_attempted", + }, + }, + }) + .where(eq(schema.adminJob.id, ctx.jobId)); + await expect( + maintenance.prepareConfigurationMaintenanceAbandon(candidate), + ).rejects.toThrow("Original known-unsaved maintenance checkpoint required"); + expect( + ( + await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, ctx.jobId)) + )[0]?.result, + ).not.toHaveProperty("_configurationMaintenanceDecision"); + } else { + const decision = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + expect(decision).toMatchObject({ + kind: "configurationMaintenance.decide", + maintenanceId: prepared.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: prepared.prepare.operationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decision: "abandon", + expectedSequence: prepared.sequence, + checkpointVersion: ctx.execution!.version, + }); + expect(await maintenance.prepareConfigurationMaintenanceAbandon(candidate)).toEqual( + decision, + ); + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, ctx.jobId)); + expect(row?.result).toMatchObject({ + _configurationMaintenanceDecision: { + version: 1, + operation: decision, + rowVersions: [], + attempt: "not_attempted", + }, + }); + expect(JSON.stringify(row?.result)).not.toContain("synthetic-abandon-credential"); + const executionsPath = join(host.infraDir, "control-private/ops-agent/executions"), + familiesPath = join(host.infraDir, "control-private/ops-agent/retention-locations"), + executionsBefore = readdirSync(executionsPath).sort(), + familiesBefore = readdirSync(familiesPath).sort(), + rootBytes = readFileSync( + join(executionsPath, `${prepared.prepare.operationKey}.json`), + ), + [guardBefore] = + await fixture.sql`SELECT xmin::text AS version FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`; + const core = await import("@openmapx/core/build-artifacts"), + originalAuthority = restoreAdmittedControlExecution(installation, prepared.prepare), + lease = core.restoreArtifactTargetLeaseForSettlement(originalAuthority), + ownerPath = join( + host.infraDir, + "data/.admin-target-control/leases", + lease.leaseId, + "owner.json", + ), + ownerBytes = readFileSync(ownerPath), + ownerInode = statSync(ownerPath).ino; + core.closeArtifactTargetLeaseHandle(lease); + await expect( + maintenance.decideConfigurationMaintenance({ + ...decision, + checkpointDigest: "f".repeat(64), + }), + ).rejects.toThrow("Original known-unsaved maintenance checkpoint required"); + const authorized = await maintenance.decideConfigurationMaintenance(decision); + expect(authorized).toMatchObject({ + state: "authorized", + sequence: prepared.sequence + 1, + decision: { + decisionId: decision.decisionId, + decision: "abandon", + checkpointDigest: decision.checkpointDigest, + }, + dmFence: prepared.dmFence, + generation: "not_started", + followUp: "not_started", + }); + const store = await import("../../../../ops-agent/src/configuration-maintenance-store"), + authority = restoreAdmittedControlExecution(installation, prepared.prepare), + original = JSON.stringify( + store.readOwnedConfigurationMaintenance(authority).apiControls.decide, + ), + projection = (await import("../admin-job-ops")).readProjection(ctx.checkpointResult), + [control] = Object.values(projection.operations).filter( + (operation) => operation.kind === "configurationMaintenance.decide", + ); + expect(control).toMatchObject({ + operation: decision, + terminal: true, + receiptDelivery: { state: "confirmed" }, + }); + expect( + store.readOwnedConfigurationMaintenance(authority).apiControls.decide, + ).toMatchObject({ + execution: { operationKey: control!.operationKey }, + canonicalDigest: control!.receiptDelivery!.terminalDigest, + receiptDelivery: { terminalCopyId: control!.receiptDelivery!.terminalCopyId }, + }); + const actualControl = + store.readOwnedConfigurationMaintenance(authority).apiControls.decide!, + { verifyApiJobParentExecution } = await import("@openmapx/core/ops"), + actualParent = verifyApiJobParentExecution( + actualControl.parentAssertion, + host.token, + { + operationKey: actualControl.execution.operationKey, + operationFingerprint: actualControl.execution.operationFingerprint, + requestedDeadlineAt: actualControl.requestedDeadlineAt, + }, + ), + handle = store.admitConfigurationMaintenanceControlOwner( + installation, + decision, + actualParent, + { + operationKey: actualControl.execution.operationKey, + requestedDeadlineAt: actualControl.requestedDeadlineAt, + }, + ), + exposed = store.configurationMaintenanceControlOwnerState( + handle, + decision, + actualControl.execution.operationKey, + ), + foreignKey = "opk1_foreignControlScope0001"; + expect(Reflect.set(exposed, "requestKey", foreignKey)).toBe(false); + expect(() => + store.configurationMaintenanceControlOwnerState(handle, decision, foreignKey), + ).toThrow(); + expect(await maintenance.decideConfigurationMaintenance(decision)).toEqual(authorized); + expect(await maintenance.prepareConfigurationMaintenance(candidate)).toEqual(prepared); + expect(ctx.checkpointResult?._configurationMaintenance).toMatchObject({ + phase: "authorized", + decision, + receipt: authorized, + }); + expect( + JSON.stringify(store.readOwnedConfigurationMaintenance(authority).apiControls.decide), + ).toBe(original); + expect(readdirSync(executionsPath).sort()).toEqual(executionsBefore); + expect(readdirSync(familiesPath).sort()).toEqual(familiesBefore); + expect( + readFileSync(join(executionsPath, `${prepared.prepare.operationKey}.json`)), + ).toEqual(rootBytes); + expect(readFileSync(ownerPath)).toEqual(ownerBytes); + expect(statSync(ownerPath).ino).toBe(ownerInode); + const [guardAfter] = + await fixture.sql`SELECT xmin::text AS version FROM data_manager.jobs WHERE id=${prepared.dmFence!.fenceId}::uuid`; + expect(guardAfter?.version).toBe(guardBefore?.version); + // Lose only the secondary jobs journal entry after the actual private + // original-owner CAS and authenticated API copy/ACK. Reopen the real + // assembly: lookup must recover that exact immutable source, not admit + // another request/root or infer success from the API's cached result. + await app.close(); + closeControlInstallation(installation); + const journalPath = join(host.infraDir, "data/ops-agent/jobs-v1.json"), + journal = await openOpsJobJournal(journalPath), + originalEntry = journal + .records() + .find((entry) => entry.operationKey === control!.operationKey)!; + expect(originalEntry).toBeDefined(); + await journal.replace( + journal.records().filter((entry) => entry.operationKey !== control!.operationKey), + ); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + app = buildOpsAgentServer({ + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + vi.stubEnv("OPS_AGENT_URL", await app.listen({ host: "127.0.0.1", port: 0 })); + const lookup = await app.inject({ + method: "GET", + url: `/v1/operation-keys/${control!.operationKey}`, + headers: { + authorization: `Bearer ${host.token}`, + "x-ops-request-id": randomUUID(), + "x-ops-protocol-version": "2", + "x-ops-body-fingerprint": originalEntry.operationFingerprint!, + "x-ops-operation-fingerprint": originalEntry.fingerprint, + }, + }); + expect(lookup.statusCode).toBe(200); + expect(lookup.json().result).toMatchObject({ + state: "succeeded", + operationId: originalEntry.operationId, + operationKey: originalEntry.operationKey, + fingerprint: originalEntry.fingerprint, + result: authorized, + admittedAt: originalEntry.admittedAt, + deadlineAt: originalEntry.deadlineAt, + updatedAt: originalEntry.updatedAt, + }); + expect( + journal.records().find((entry) => entry.operationKey === control!.operationKey), + ).toEqual(originalEntry); + await app.close(); + const privateOwner = join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${prepared.binding.maintenanceId}.json`, + ), + hiddenOwner = join(host.root, "temporarily-unavailable-maintenance-owner.json"); + nodeFs.renameSync(privateOwner, hiddenOwner); + try { + app = buildOpsAgentServer({ + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + const failure = await app.ready().then( + () => null, + (error: Error) => error, + ); + expect(failure?.message).toBe( + "Original maintenance control private source unavailable", + ); + } finally { + nodeFs.renameSync(hiddenOwner, privateOwner); + } + expect(readFileSync(ownerPath)).toEqual(ownerBytes); + expect(statSync(ownerPath).ino).toBe(ownerInode); + expect(readdirSync(executionsPath).sort()).toEqual(executionsBefore); + expect(readdirSync(familiesPath).sort()).toEqual(familiesBefore); + } + expect(dm!.preparePosts).toBe(1); + expect( + host.calls.filter( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ), + ).toHaveLength(lifecycle); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + }); + }, + 60000, + ); + it.each([ + { issued: false, capacityNull: false }, + { issued: true, capacityNull: false }, + { issued: true, capacityNull: true }, + ])( + "closes an original durable nonempty never-issued or receiver-refused owner automatically with issued=$issued capacity-null=$capacityNull", + async ({ issued, capacityNull }) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const coordinator = { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, coordinator]); + const ctx = await job(), + busyId = randomUUID(); + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-coordinator-credential", + }, + ], + }, + }); + await prospective.stageProspectiveConfigurationMaintenance(candidate); + const { record } = prospective.validatedConfigurationCandidateState(candidate), + ownerPath = join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${record.maintenanceId}.json`, + ); + let afterAdmission = false; + const loseCoordinator = async () => { + if (existsSync(ownerPath)) { + afterAdmission = true; + host.setRows([host.row, { ...coordinator, State: "exited" }]); + } else host.setInventoryProcess(loseCoordinator); + }; + if (issued) + await fixture.sql`INSERT INTO data_manager.jobs(id,kind,status,started_at) VALUES(${busyId}::uuid,'download-fonts','running',now())`; + else host.setInventoryProcess(loseCoordinator); + const { prepareConfigurationMaintenance } = await import("../configuration-maintenance"); + const nativeStatfs = (await vi.importActual("node:fs")) + .statfsSync, + privateIdentity = statSync(join(host.infraDir, "control-private/data-manager")), + pressure = capacityNull + ? vi.spyOn(nodeFs, "statfsSync").mockImplementation((( + path: nodeFs.PathLike, + options?: { bigint?: boolean }, + ) => { + const identity = statSync(path), + actual = nativeStatfs(path, options as { bigint: true }); + return identity.dev === privateIdentity.dev && + identity.ino === privateIdentity.ino + ? { + ...actual, + bavail: typeof actual.bavail === "bigint" ? 4n : 4, + bsize: typeof actual.bsize === "bigint" ? 4096n : 4096, + } + : actual; + }) as typeof nodeFs.statfsSync) + : null; + try { + await expect(prepareConfigurationMaintenance(candidate)).rejects.toBeInstanceOf(Error); + } finally { + pressure?.mockRestore(); + } + expect(issued || afterAdmission).toBe(true); + const core = await import("@openmapx/core/build-artifacts"), + owner = JSON.parse(readFileSync(ownerPath, "utf8")).payload, + authority = core.restoreAdmittedControlExecution(installation, owner.plan.prepare), + ownerStore = await import("../../../../ops-agent/src/configuration-maintenance-store"); + expect(owner.issuance).toBe("closed_without_guard"); + await ownerStore.closeOwnedConfigurationMaintenanceWithoutGuard(host.runtime, authority); + const completed = ownerStore.readOwnedConfigurationMaintenance(authority); + if (issued) { + expect(completed.noGuard?.noGuardEvidence.kind).toBe("receiver_refused"); + if (capacityNull) + expect(completed.noGuard?.noGuardEvidence).toMatchObject({ + capacityReceiptDigest: null, + }); + } else + expect(completed.noGuard).toMatchObject({ + noGuardEvidence: { kind: "never_issued" }, + issuerFreezeProof: { + payload: { + publicationStates: owner.plan.reservations.map( + (value: { + phase: string; + ordinal: number; + execution: { operationKey: string }; + }) => ({ + phase: value.phase, + ordinal: value.ordinal, + operationKey: value.execution.operationKey, + grantState: "unissued", + intentState: "not_attempted", + }), + ), + }, + }, + }); + core.assertConfigurationMaintenanceRootClosure(authority); + core.verifyRetainedArtifactTargetRootRelease(authority); + // Repeat only the original attempted producer: it must observe the same + // failed root and deliver its durable canonical receipt, never POST again. + await expect(prepareConfigurationMaintenance(candidate)).rejects.toBeInstanceOf(Error); + expect( + ( + ctx.checkpointResult?.opsProjection as { + operations: Record; + } + ).operations[record.prepareOperationKey!]?.receiptDelivery?.state, + ).toBe("confirmed"); + const location = join( + host.infraDir, + "control-private/ops-agent/retention-locations", + `${core.controlDigest(owner.origin)}.json`, + ), + retirementClock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 3600000); + try { + expect(existsSync(location)).toBe(true); + const bundlePath = join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${owner.plan.prepare.operationKey}.configuration-settlement.json`, + ), + original = readFileSync(bundlePath), + corrupt = Buffer.from(original); + corrupt[12] = corrupt[12] === 120 ? 121 : 120; + try { + writeFileSync(bundlePath, corrupt); + await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm!.reconcileRetention(); + expect(existsSync(location)).toBe(true); + } finally { + writeFileSync(bundlePath, original); + } + for (let round = 0; round < 12 && existsSync(location); round++) { + await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm!.reconcileRetention(); + } + const diagnostics = ["ops-agent", "data-manager"].map((role) => { + const path = join( + host.infraDir, + "control-private", + role, + "retention-locations", + `${core.controlDigest(owner.origin)}.json`, + ); + if (!existsSync(path)) return { role, retired: true }; + const retained = JSON.parse(readFileSync(path, "utf8")), + progress = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private", + role, + "retention-families", + retained.name, + "progress.json", + ), + "utf8", + ), + ); + const shared = join( + host.infraDir, + "data/.admin-target-control/retention-families", + core.controlDigest(owner.origin), + ), + candidatePath = join(shared, `candidate-${role}.json`), + rawCandidate = existsSync(candidatePath) + ? JSON.parse(readFileSync(candidatePath, "utf8")) + : null, + ackPath = join(shared, "api-ack.json"), + apiAck = existsSync(ackPath) ? JSON.parse(readFileSync(ackPath, "utf8")) : null; + return { + role, + phase: progress.phase, + lastError: progress.lastError, + terminalAt: progress.terminalAt, + slotState: progress.configurationSettlementSlot?.state, + candidatePublished: Boolean(rawCandidate), + apiAck: Boolean(apiAck), + participants: progress.participants, + executions: Object.values<{ + execution?: { executor?: string }; + known?: boolean; + requiresJournal?: boolean; + journalContained?: boolean; + canonicalDigest?: string; + }>(progress.executions).map((fact) => ({ + kind: fact.execution?.executor, + known: fact.known, + requiresJournal: fact.requiresJournal, + journalContained: fact.journalContained, + canonical: Boolean(fact.canonicalDigest), + })), + }; + }); + expect(existsSync(location), JSON.stringify(diagnostics)).toBe(false); + } finally { + retirementClock.mockRestore(); + } + expect(dm!.preparePosts).toBe(issued ? 1 : 0); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE kind='configuration-maintenance'`, + ).toEqual([]); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + }).finally(async () => { + await fixture.sql`DELETE FROM data_manager.jobs WHERE id=${busyId}::uuid`; + }); + }, + 60000, + ); + it("cleans the current persister's actual durable nonempty NoGuard busy refusal after the old helper settles", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const coordinator = { + ...host.row, + Id: "c".repeat(64), + State: "exited", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, coordinator]); + let starts = 0; + host.setCreatingProcess(async (args) => { + expect(args.at(-1)).toBe("data-manager"); + expect(args).not.toContain("--force-recreate"); + coordinator.State = "running"; + host.setRows([host.row, coordinator]); + starts++; + }); + const ctx = await job(), + busyId = randomUUID(); + try { + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-current-noguard-busy", + }, + ], + }, + }), + { record } = prospective.validatedConfigurationCandidateState(candidate); + await prospective.stageProspectiveConfigurationMaintenance(candidate); + expect(starts).toBe(1); + expect(dm!.preparePosts).toBe(0); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(false); + // A competing synthetic live row wins the ACTUAL common SQL admission + // guard in the gap, after helper release. No worker/result is faked. + await fixture.sql.begin(async (tx) => { + const [lock] = await tx`SELECT pg_try_advisory_xact_lock(${0x4f4d58},${2}) AS acquired`; + expect(lock!.acquired).toBe(true); + await tx`INSERT INTO data_manager.jobs(id,kind,status,started_at) VALUES(${busyId}::uuid,'download-fonts','running',now())`; + }); + const before = + await fixture.sql`SELECT *,xmin::text AS row_version FROM data_manager.jobs WHERE id=${busyId}::uuid`; + let failure: unknown; + try { + await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf((await import("../ops-client")).ApiOpsError); + expect(failure).toMatchObject({ errorClass: "busy" }); + const core = await import("@openmapx/core/build-artifacts"), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records" + ), + owner = core.verifyControl( + installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${record.maintenanceId}.json`, + ), + "utf8", + ), + ), + records.configurationMaintenanceOwnerRecordSchema, + ), + authority = restoreAdmittedControlExecution(installation, owner.plan.prepare); + expect(owner).toMatchObject({ + issuance: "closed_without_guard", + receipt: { state: "settled", dmFence: null, decision: null, generation: "not_started" }, + }); + core.assertConfigurationMaintenanceRootClosure(authority); + core.verifyRetainedArtifactTargetRootRelease(authority); + expect( + await fixture.sql`SELECT *,xmin::text AS row_version FROM data_manager.jobs WHERE id=${busyId}::uuid`, + ).toEqual(before); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(dm!.preparePosts).toBe(1); + expect(dm!.settlePosts).toBe(0); + expect(starts).toBe(1); + expect( + host.calls.some( + (args) => + args.includes("--force-recreate") || + args.includes("pull") || + args.includes("restart"), + ), + ).toBe(false); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(ctx.checkpointResult!._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + expect(ctx.checkpointResult!.configuration).toMatchObject({ + outcome: "not_committed", + reason: "busy", + }); + expect(existsSync(join(host.queue, ".candidates", `${record.operationKey}.json`))).toBe( + false, + ); + }); + const next = await job(); + await owned(next, async () => + expect(writer.currentAdministrativeConfigurationWriter()).toBeDefined(), + ); + } finally { + await fixture.sql`DELETE FROM data_manager.jobs WHERE id=${busyId}::uuid AND kind='download-fonts'`; + } + }, 120000); + it.each([false, true])( + "publishes actual durable nonempty receiver busy refusal after full reservation with capacity-null=%s without changing the winning data job", + async (capacityNull) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const ctx = await job(), + busyId = randomUUID(); + try { + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "ensure-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-coordinator-credential", + }, + ], + }, + }); + await prospective.stageProspectiveConfigurationMaintenance(candidate); + await fixture.sql`INSERT INTO data_manager.jobs(id,kind,status,started_at) VALUES(${busyId}::uuid,'download-fonts','running',now())`; + const before = + await fixture.sql`SELECT * FROM data_manager.jobs WHERE id=${busyId}::uuid`; + const { prepareConfigurationMaintenance } = await import( + "../configuration-maintenance" + ); + const nativeStatfs = (await vi.importActual("node:fs")) + .statfsSync, + privateIdentity = statSync(join(host.infraDir, "control-private/data-manager")), + pressure = capacityNull + ? vi.spyOn(nodeFs, "statfsSync").mockImplementation((( + path: nodeFs.PathLike, + options?: { bigint?: boolean }, + ) => { + const identity = statSync(path), + actual = nativeStatfs(path, options as { bigint: true }); + return identity.dev === privateIdentity.dev && + identity.ino === privateIdentity.ino + ? { + ...actual, + bavail: typeof actual.bavail === "bigint" ? 4n : 4, + bsize: typeof actual.bsize === "bigint" ? 4096n : 4096, + } + : actual; + }) as typeof nodeFs.statfsSync) + : null; + const nativeOpenDir = ( + await vi.importActual("node:fs/promises") + ).opendir, + unavailable = vi.spyOn(nodeFsPromises, "opendir").mockImplementation((async ( + path, + options, + ) => { + if (String(path) === host.queue) + throw Error( + "Injected original queue temporarily unavailable for automatic refusal closure", + ); + return nativeOpenDir(path, options); + }) as typeof nodeFsPromises.opendir); + try { + await expect(prepareConfigurationMaintenance(candidate)).rejects.toBeInstanceOf( + Error, + ); + } finally { + pressure?.mockRestore(); + unavailable.mockRestore(); + } + expect( + await fixture.sql`SELECT * FROM data_manager.jobs WHERE id=${busyId}::uuid`, + ).toEqual(before); + const { record } = prospective.validatedConfigurationCandidateState(candidate), + owner = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${record.maintenanceId}.json`, + ), + "utf8", + ), + ).payload, + prepare = owner.plan.reservations.find( + (value: { phase: string }) => value.phase === "configuration-fence", + ), + path = join( + host.infraDir, + "data/.admin-target-control/executions/data-manager", + `${prepare.execution.operationKey}.configuration-settlement-capacity.json`, + ), + channel = JSON.parse(readFileSync(path, "utf8")); + expect(statSync(path).size).toBe(8192); + if (capacityNull) expect(channel.capacity).toBeNull(); + else expect(channel.capacity).not.toBeNull(); + expect(channel.refusal).toMatchObject({ + payload: { + refusalReceipt: { + kind: "data-configuration-fence-refused", + reason: capacityNull ? "capacity" : "busy", + }, + maintenanceId: record.maintenanceId, + prepareExecution: { operationKey: prepare.execution.operationKey }, + }, + }); + const core = await import("@openmapx/core/build-artifacts"), + records = await import( + "../../../../../packages/core/src/build-artifacts/configuration-fence-settlement-records" + ), + publicKey = core.controlRolePublicKey(installation, "data-manager"), + refusal = core.verifyControl( + publicKey, + records.configurationSettlementDomains.refusal, + channel.refusal, + records.configurationDmNoGuardRefusalSchema, + ), + capacity = + channel.capacity && + core.verifyControl( + publicKey, + records.configurationSettlementDomains.capacity, + channel.capacity, + records.configurationSettlementCapacitySchema, + ), + authority = core.restoreAdmittedControlExecution(installation, owner.plan.prepare), + lease = core.restoreArtifactTargetLeaseForSettlement(authority); + try { + const child = core.lookupControlChildPhaseBinding(authority, prepare)!.child, + settlement = core.inspectArtifactTargetLeaseChildSettlement(lease, child.execution); + expect(settlement).toMatchObject({ + outcome: "failed_without_effect", + publication: "not_started", + }); + expect(refusal.effectReceiptDigest).toBe(core.controlDigest(refusal.refusalReceipt)); + expect(refusal.effectSettlementDigest).toBe( + core.controlDigest({ ...settlement, receipt: refusal.refusalReceipt }), + ); + expect(refusal.canonicalTerminalDigest).toBe( + core.controlDigest({ version: 1, refused: refusal.refusalReceipt }), + ); + expect(refusal.capacityReceiptDigest).toBe( + capacity + ? records.settlementRecordDigest( + records.configurationSettlementDomains.capacity, + capacity, + ) + : null, + ); + expect(() => + core.verifyControl( + publicKey, + records.configurationSettlementDomains.refusal, + { ...channel.refusal, payload: { ...refusal, candidateId: randomUUID() } }, + records.configurationDmNoGuardRefusalSchema, + ), + ).toThrow(); + } finally { + core.closeArtifactTargetLeaseHandle(lease); + } + expect(channel.unused).toMatchObject({ + intent: { payload: { role: "data-manager", settlementId: refusal.settlementId } }, + confirmation: { + payload: { role: "data-manager", settlementId: refusal.settlementId }, + }, + }); + const intent = core.verifyControl( + publicKey, + records.configurationSettlementDomains.unusedIntent, + channel.unused.intent, + records.configurationUnusedCapacityIntentSchema, + ), + confirmation = core.verifyControl( + publicKey, + records.configurationSettlementDomains.unusedConfirmation, + channel.unused.confirmation, + records.configurationUnusedCapacityConfirmationSchema, + ), + location = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/data-manager/retention-locations", + `${core.controlDigest(refusal.origin)}.json`, + ), + "utf8", + ), + ), + progress = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/data-manager/retention-families", + location.name, + "progress.json", + ), + "utf8", + ), + ), + slot = progress.configurationSettlementSlot; + expect(slot).toMatchObject({ + state: "UNUSED", + executionDigest: null, + controlBindingDigest: null, + executionOrdinal: capacity?.reservedExecutionOrdinal ?? null, + intentOrdinal: capacity?.reservedIntentOrdinal ?? null, + unusedIntentDigest: records.settlementRecordDigest( + records.configurationSettlementDomains.unusedIntent, + intent, + ), + }); + expect(Object.keys(progress.executions)).toHaveLength(1); + expect(intent.allocatedDescriptorDigests).toHaveLength(capacityNull ? 1 : 6); + expect(intent.sourceProofDigest).toBe( + records.settlementRecordDigest( + records.configurationSettlementDomains.refusal, + refusal, + ), + ); + expect(confirmation.intentDigest).toBe(slot.unusedIntentDigest); + expect(confirmation.confirmedAt).toBe(slot.unusedConfirmedAt); + expect(confirmation.slotStateDigest).toBe( + core.controlDigest({ + executionOrdinal: slot.executionOrdinal, + intentOrdinal: slot.intentOrdinal, + recordDescriptorsDigest: slot.recordDescriptorsDigest, + allocatedDescriptorDigests: slot.allocatedDescriptorDigests, + state: "UNUSED", + intentDigest: slot.unusedIntentDigest, + }), + ); + if (slot.executionOrdinal !== null) { + const ticket = JSON.parse( + readFileSync( + join( + host.infraDir, + "data/.admin-target-control/retention-families", + core.controlDigest(refusal.origin), + "executions", + `slot-${slot.executionOrdinal}.json`, + ), + "utf8", + ), + ); + expect(ticket.value).toMatchObject({ + state: "UNUSED", + executionDigest: null, + controlBindingDigest: null, + unusedIntentDigest: slot.unusedIntentDigest, + }); + } else + expect(slot).toMatchObject({ intentOrdinal: null, refusalConstraint: "filesystem" }); + const verifiedRefusal = core.verifyPublishedDataConfigurationFenceRefusal(authority), + verifiedUnused = core.verifyPublishedUnusedConfigurationCapacity( + authority, + verifiedRefusal, + ); + expect(core.publishedDmUnusedCapacityState(verifiedUnused, authority)).toMatchObject({ + intentDigest: confirmation.intentDigest, + }); + expect(() => + core.publishedDmNoGuardRefusalState({} as typeof verifiedRefusal, authority), + ).toThrow(); + expect(() => + core.publishedDmUnusedCapacityState({} as typeof verifiedUnused, authority), + ).toThrow(); + const observation = await import( + "../../../../ops-agent/src/configuration-maintenance-runtime" + ); + let scoped: + | Parameters[0] + | undefined; + await observation.withUnconsumedMaintenanceCandidate( + host.runtime, + authority, + (witness) => { + scoped = witness; + expect( + observation.unconsumedMaintenanceCandidateState(witness, authority) + .candidateRecordDigest, + ).toBe(owner.plan.candidateRecordDigest); + }, + ); + expect(() => + observation.unconsumedMaintenanceCandidateState(scoped!, authority), + ).toThrow(); + const claimed = join(host.queue, ".claimed", `${candidate.revisionId}.json`); + writeFileSync( + claimed, + prospective.validatedConfigurationCandidateState(candidate).sealed.bytes, + { mode: 0o600 }, + ); + try { + await expect( + observation.withUnconsumedMaintenanceCandidate(host.runtime, authority, () => { + throw Error("Must not publish from a claimed candidate"); + }), + ).rejects.toThrow("Candidate publication/claim requires retained observation"); + } finally { + rmSync(claimed); + } + const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 3600000); + try { + const source = { + directory: host.queue, + token: host.token, + ownerUid: statSync(host.queue).uid, + ownerGid: statSync(host.queue).gid, + }, + readers = await import("../../../../ops-agent/src/configuration-candidate-source"); + await expect( + readers.readConfigurationCandidate(source, candidate.candidateId), + ).rejects.toThrow(); + expect( + (await readers.readRetainedConfigurationCandidate(source, candidate.candidateId)) + .snapshot.revisionId, + ).toBe(candidate.revisionId); + await observation.withUnconsumedMaintenanceCandidate( + host.runtime, + authority, + (witness) => { + expect( + observation.unconsumedMaintenanceCandidateState(witness, authority) + .candidateRecordDigest, + ).toBe(owner.plan.candidateRecordDigest); + }, + ); + } finally { + clock.mockRestore(); + } + const ownerStore = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ), + bundlePath = join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${owner.plan.prepare.operationKey}.configuration-settlement.json`, + ); + let originalBundle: Buffer | undefined; + if (!capacityNull) { + const nativeRename = (await vi.importActual("node:fs")) + .renameSync; + let faulted = false; + const fault = vi.spyOn(nodeFs, "renameSync").mockImplementation((source, target) => { + nativeRename(source, target); + if ( + !faulted && + String(target).endsWith( + `${owner.plan.prepare.operationKey}.configuration-settlement.json`, + ) + ) { + faulted = true; + throw Error( + "Injected filesystem failure after bundle publication before compact link", + ); + } + }); + try { + await expect( + ownerStore.closeOwnedConfigurationMaintenanceWithoutGuard( + host.runtime, + authority, + ), + ).rejects.toThrow("Injected filesystem failure"); + } finally { + fault.mockRestore(); + } + expect(faulted).toBe(true); + originalBundle = readFileSync(bundlePath); + expect(() => core.assertConfigurationMaintenanceRootClosure(authority)).toThrow(); + await expect( + core.bindControlChildPhase(authority, owner.plan.reservations[0]), + ).rejects.toThrow(); + } + const recoveryClock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 3600000), + reopened = core.openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((value) => value.manifest.id), + }); + try { + await ownerStore.closeOwnedConfigurationMaintenanceWithoutGuard( + host.runtime, + core.restoreAdmittedControlExecution(reopened, owner.plan.prepare), + ); + } finally { + core.closeControlInstallation(reopened); + recoveryClock.mockRestore(); + } + if (originalBundle) expect(readFileSync(bundlePath)).toEqual(originalBundle); + const controlPath = join( + host.infraDir, + "data/.admin-target-control/executions/ops-agent", + `${owner.plan.prepare.operationKey}.configuration-settlement-control.json`, + ), + closedBundle = JSON.parse(readFileSync(bundlePath, "utf8")), + closedControl = JSON.parse(readFileSync(controlPath, "utf8")); + expect(closedBundle.payload).toMatchObject({ + phase: "unused", + closure: { payload: { noGuardEvidence: { kind: "receiver_refused" } } }, + }); + expect(closedControl.payload).toMatchObject({ + binding: null, + unused: { + bundleDigest: records.settlementRecordDigest( + records.configurationSettlementDomains.bundle, + closedBundle.payload, + ), + }, + }); + expect(() => core.assertConfigurationMaintenanceRootClosure(authority)).not.toThrow(); + const closedBytes = [readFileSync(bundlePath), readFileSync(controlPath)]; + for (const [index, target] of [bundlePath, controlPath].entries()) { + const original = closedBytes[index]!, + tampered = Buffer.from(original); + tampered[12] = tampered[12] === 120 ? 121 : 120; + try { + writeFileSync(target, tampered); + expect(() => core.assertConfigurationMaintenanceRootClosure(authority)).toThrow(); + } finally { + writeFileSync(target, original); + } + const missing = `${target}.temporarily-missing`; + try { + nodeFs.renameSync(target, missing); + expect(() => core.assertConfigurationMaintenanceRootClosure(authority)).toThrow(); + } finally { + nodeFs.renameSync(missing, target); + } + } + await ownerStore.closeOwnedConfigurationMaintenanceWithoutGuard( + host.runtime, + authority, + ); + expect([readFileSync(bundlePath), readFileSync(controlPath)]).toEqual(closedBytes); + await expect( + core.bindControlChildPhase(authority, owner.plan.reservations[0]), + ).rejects.toThrow(); + const release = core.restoreArtifactTargetLeaseForSettlement(authority); + try { + await core.settleArtifactTargetLeaseChild( + release, + core.verifyControlEffectSettlement(installation, owner.plan.prepare), + ); + await core.releaseArtifactTargetLease(release); + core.verifyRetainedArtifactTargetRootRelease(authority); + } finally { + core.closeArtifactTargetLeaseHandle(release); + } + const journalPath = join(host.infraDir, "data/ops-agent/jobs-v1.json"), + originalJournal = readFileSync(journalPath), + maintenance = await import("../configuration-maintenance"); + const inspectionAt = Date.now() + 3600000; + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(inspectionAt); + try { + expect( + await maintenance.inspectConfigurationMaintenance({ + maintenanceId: record.maintenanceId!, + candidateId: candidate.candidateId, + }), + ).toMatchObject({ state: "settled", dmFence: null, generation: "not_started" }); + const nativeFs = + await vi.importActual("node:fs/promises"), + candidatePath = join(host.queue, ".candidates", `${record.operationKey}.json`); + let unlinked = false; + vi.mocked(nodeFsPromises.unlink).mockImplementation(async (path) => { + await nativeFs.unlink(path); + if (String(path) === candidatePath && !unlinked) { + unlinked = true; + throw Error("Synthetic external unlink acknowledgement lost"); + } + }); + try { + await expect( + prospective.discardNeverPreparedProspectiveConfiguration(candidate), + ).rejects.toThrow("Synthetic external unlink acknowledgement lost"); + } finally { + vi.mocked(nodeFsPromises.unlink).mockImplementation(nativeFs.unlink); + } + expect(unlinked).toBe(true); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + expect(ctx.checkpointResult?._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarding_without_guard", + }); + await prospective.discardNeverPreparedProspectiveConfiguration(candidate); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + expect(ctx.checkpointResult?._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "discarded", + }); + } finally { + vi.useRealTimers(); + } + const delivered = ( + ctx.checkpointResult?.opsProjection as { + operations: Record< + string, + { errorClass?: string; receiptDelivery?: { state: string } } + >; + } + ).operations[record.prepareOperationKey!]; + expect(delivered).toMatchObject({ + errorClass: "recovery_required", + receiptDelivery: { state: "confirmed" }, + }); + expect(readFileSync(journalPath)).toEqual(originalJournal); + process.stdout.write( + `${JSON.stringify({ + proof: "actual-signed-sender-noguard-two-cells-released", + capacityNull, + bundleBytes: Buffer.byteLength(JSON.stringify(closedBundle)), + bundleReserved: statSync(bundlePath).size, + controlBytes: Buffer.byteLength(JSON.stringify(closedControl)), + controlReserved: statSync(controlPath).size, + })}\n`, + ); + process.stdout.write( + `${JSON.stringify({ proof: "actual-dm-busy-capacity-unused", encodedBytes: Buffer.byteLength(JSON.stringify(channel)), reservedBytes: statSync(path).size })}\n`, + ); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + await fixture.sql`SELECT id FROM data_manager.jobs WHERE kind='configuration-maintenance'`, + ).toEqual([]); + expect(dm!.preparePosts).toBe(1); + }); + const next = await job(); + await owned(next, async () => { + expect( + ( + await fixture.db + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(eq(schema.adminJob.id, ctx.jobId)) + )[0]?.result, + ).toMatchObject({ _configurationCandidate: { phase: "discarded" } }); + }); + } finally { + await fixture.sql`DELETE FROM data_manager.jobs WHERE id=${busyId}::uuid`; + } + }, + 60000, + ); + it.each([false, true])( + "prepares durable nonempty render-only hardlinks with a running coordinator=%s and zero lifecycle", + async (running) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: running ? "running" : "exited", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-osm-fixture"); + const ctx = await job(), + { prepareServiceGeneration } = await import("../service-preparation"); + if (running) { + const result = await prepareServiceGeneration(ctx, "render-only", { + coordinatorMode: "require-running", + }); + expect(result).toMatchObject({ + coordinatorMode: "require-running", + hardlinks: { applied: true, entries: 1, linked: 1 }, + }); + } else { + await expect( + prepareServiceGeneration(ctx, "render-only", { coordinatorMode: "require-running" }), + ).rejects.toThrow(); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual([]); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + } + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ), + ).toBe(false); + }, + ); + it.each([false, true])( + "applies a durable nonempty generation leaf through API, actual agent and DM, with lost response=%s", + async (loseResponse) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + loseHardlinkResponse: loseResponse, + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-osm-fixture"); + const plan = JSON.parse( + readFileSync( + services.currentConfigurationFile(host.infraDir, services.GENERATED_HARDLINK_PLAN_FILE), + "utf8", + ), + ); + expect(plan).toHaveLength(1); + const ctx = await job(), + { executeAdminJobOperation } = await import("../admin-job-ops"); + const result = await executeAdminJobOperation( + ctx, + { + kind: "configuration.hardlinks.apply", + revisionId: host.initial.revisionId, + coordinatorMode: "require-running", + }, + "task012.nonempty-hardlinks", + { durableIdentity: host.initial.revisionId }, + ).catch(async (error) => { + const operation = Object.values( + ( + ctx.checkpointResult?.opsProjection as { + operations: Record; + } + ).operations, + )[0]!; + const execution = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${operation.operationKey}.json`, + ), + "utf8", + ), + ).payload.execution; + const authority = restoreAdmittedControlExecution(installation, execution), + bound = lookupControlChildPhaseBinding(authority, { phase: "hardlinks", ordinal: 0 }); + let witness: unknown = "no-binding"; + if (bound) { + const child = await dm!.store.lookup(bound.child.execution.operationKey); + try { + verifyDataControlTerminalWitness(installation, bound.child.execution, child?.status); + witness = "verified"; + } catch (failure) { + witness = String(failure); + } + } + const attempt = lookupControlChildPhaseAttempt(authority, { + phase: "hardlinks", + ordinal: 0, + }); + const files = attempt?.leaseId + ? readdirSync(join(host.infraDir, "data/.admin-target-control/leases", attempt.leaseId)) + : []; + throw Error( + `${String(error)}; witness=${String(witness)}; lease files=${JSON.stringify(files)}; DM replies: ${JSON.stringify(await dm?.diagnostics())}`, + ); + }); + expect(result).toMatchObject({ applied: true, entries: 1, linked: 1, via: "data-manager" }); + expect(dm.posts).toBe(1); + const destination = join(host.infraDir, plan[0].target, "fixture.pbf"); + expect(statSync(destination).ino).toBe(statSync(join(source, "fixture.pbf")).ino); + expect(host.calls.some((args) => args.includes("up") || args.includes("restart"))).toBe( + false, + ); + expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/motis"))).toBe( + false, + ); + }, + 60000, + ); + it.each(["ensure-running", "require-running"] as const)( + "retains committed desired state when the durable nonempty coordinator stops after generation commit (mode=%s)", + async (coordinatorMode) => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const dmRow = { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, dmRow]); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-stopped-coordinator"); + const ctx = await job(); + let stopped = false, + stoppedCallIndex = 0; + await owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode, + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-postcommit-coordinator-stop", + }, + ], + }, + }); + const stopAfterCommit = async () => { + if (services.currentConfigurationGeneration(host.infraDir) === candidate.revisionId) { + stopped = true; + stoppedCallIndex = host.calls.length; + dmRow.State = "exited"; + host.setRows([host.row, dmRow]); + } else host.setInventoryProcess(stopAfterCommit); + }; + host.setInventoryProcess(stopAfterCommit); + let failure: unknown; + try { + await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode, + }); + } catch (error) { + failure = error; + } + expect(failure).toBeDefined(); + expect(stopped).toBe(true); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(candidate.revisionId); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(1); + expect(ctx.checkpointResult!._configurationMaintenanceDecision).toMatchObject({ + operation: { decision: "continue" }, + attempt: "attempted", + }); + expect(ctx.checkpointResult!._configurationCandidate).toMatchObject({ + candidateId: candidate.candidateId, + phase: "saved", + }); + expect( + host.calls + .slice(stoppedCallIndex) + .some( + (args) => args.includes("up") || args.includes("restart") || args.includes("pull"), + ), + ).toBe(false); + expect(actualPosts.get("service.start") ?? 0).toBe( + coordinatorMode === "ensure-running" ? 1 : 0, + ); + expect(dm!.settlePosts).toBe(0); + expect( + existsSync( + join( + host.queue, + ".candidates", + `${prospective.validatedConfigurationCandidateState(candidate).record.operationKey}.json`, + ), + ), + ).toBe(true); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + }); + const next = await job(); + await expect(owned(next, async () => undefined)).rejects.toThrow( + "Configuration writer unavailable", + ); + console.info( + JSON.stringify({ + proof: "postcommit-coordinator-stop-no-fallback", + coordinatorMode, + desiredRows: 1, + postcommitStarts: 0, + rootFenced: true, + nextWriterRefused: true, + }), + ); + }, + 120000, + ); + it.each(["pull", "recreate", "cancel", "crash-pull", "crash-recreate", "missing-receipt"])( + "retains the actual default CLI Update failure and borrowed-child containment (mode=%s)", + async (mode) => { + const bin = join(host.root, "fixture-bin"), + trace = join(host.root, "docker-argv.jsonl"), + marker = join(host.root, "pull-started"), + receiptDirectory = join(host.infraDir, "data/ops-agent/lifecycle"), + recreated = ["recreate", "crash-recreate", "missing-receipt"].includes(mode), + crash = + "fs.writeFileSync(" + + JSON.stringify(join(host.root, "cli-killed")) + + ',String(process.ppid));process.kill(process.ppid,"SIGKILL");process.exit(0);'; + mkdirSync(bin); + writeFileSync( + join(bin, "docker"), + `#!${process.execPath}\nconst fs=require("node:fs"),args=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(trace)},JSON.stringify(args)+"\\n"); +if(args.includes("pull")){fs.writeFileSync(${JSON.stringify(marker)},String(process.pid));${mode === "cancel" ? "setInterval(()=>{},1000);" : mode === "pull" ? "process.exit(7);" : mode === "crash-pull" ? crash : ""}} +if(args.includes("--force-recreate")){${mode === "recreate" ? "process.exit(9);" : mode === "crash-recreate" ? crash : mode === "missing-receipt" ? `const directory=${JSON.stringify(receiptDirectory)},names=fs.readdirSync(directory).filter(n=>/^opk1_[A-Za-z0-9_-]+\\.json$/.test(n));if(names.length!==1)throw Error("Expected sole owned lifecycle receipt");fs.unlinkSync(require("node:path").join(directory,names[0]));` : ""}}\n`, + { mode: 0o700 }, + ); + for (const [key, value] of Object.entries({ + PATH: `${bin}:${process.env.PATH}`, + OPENMAPX_ROOT_DIR: host.root, + OPENMAPX_RELEASE: "local", + DOMAIN: "example.test", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + DATA_MANAGER_AUTH_TOKEN: "d".repeat(32), + OPENMAPX_ENABLED_SERVICES: "", + })) + vi.stubEnv(key, value); + const ctx = await job(), + { executeAdminJobOperation, readProjection } = await import("../admin-job-ops"), + { createApiOpsClient } = await import("../ops-client"); + let failure: unknown; + ctx.signal = AbortSignal.timeout(45000); + const running = executeAdminJobOperation( + ctx, + { + kind: "service.update", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + "task012.default-cli-failure", + { durableIdentity: "motis" }, + ).catch((error) => { + failure = error; + }); + if (mode === "cancel") { + const until = Date.now() + 30000; + while (!existsSync(marker)) { + if (Date.now() > until) throw Error("Actual fixed CLI pull not reached"); + await nativeDelay(20); + } + const entry = Object.values(readProjection(ctx.checkpointResult).operations)[0]; + expect(entry?.operationId).toBeTypeOf("string"); + await createApiOpsClient().cancel(entry!.operationId!, { + signal: AbortSignal.timeout(15000), + }); + } + await running; + expect(failure).toBeDefined(); + const [entry] = Object.values(readProjection(ctx.checkpointResult).operations), + core = await import("@openmapx/core/build-artifacts"), + root = restoreAdmittedControlExecution(installation, { + executor: "ops-agent", + principal: "api", + executionId: entry!.operationId!, + operationKey: entry!.operationKey, + operationFingerprint: (await import("@openmapx/core/ops")).opsOperationFingerprint( + entry!.operation!, + ), + fingerprint: entry!.fingerprint!, + deadlineAt: entry!.deadlineAt!, + }), + child = lookupControlChildPhaseBinding(root, { phase: "lifecycle", ordinal: 0 }); + expect(child).toBeDefined(); + const receiptRead = services.readServiceLifecycleReceipt({ + dataRoot: join(host.infraDir, "data"), + execution: child!.child.execution, + }); + const calls = readFileSync(trace, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + expect(calls.filter((args) => args.includes("pull"))).toHaveLength(1); + expect(calls.filter((args) => args.includes("up"))).toHaveLength(recreated ? 1 : 0); + expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/motis"))).toBe( + true, + ); + expect(() => core.verifyRetainedArtifactTargetRootRelease(root)).toThrow(); + if (mode === "missing-receipt") { + await expect(receiptRead).rejects.toThrow(); + expect(readdirSync(receiptDirectory)).toEqual([]); + expect(failure).toMatchObject({ name: "OpsObservationPending" }); + const observed = await createApiOpsClient().status( + "service.update", + entry!.operationId!, + { + signal: AbortSignal.timeout(15000), + }, + ); + expect(["running", "termination_pending", "failed"]).toContain(observed.state); + expect(observed.result).toBeUndefined(); + expect(JSON.stringify(observed.diagnostics ?? {})).not.toContain(host.root); + expect(JSON.stringify(observed.diagnostics ?? {})).not.toContain("/.vol/"); + expect(services.currentConfigurationGeneration(host.infraDir)).not.toBe( + host.initial.revisionId, + ); + console.info( + JSON.stringify({ + proof: "actual-default-cli-missing-receipt", + authenticatedResult: "unknown", + noPointerSubstitution: true, + rootFenced: true, + }), + ); + return; + } + const receipt = await receiptRead; + expect( + receipt.phase === "failed" || + receipt.phase === "unknown" || + (["cancel", "crash-pull"].includes(mode) && receipt.phase === "pins_validated") || + (mode === "crash-recreate" && receipt.phase === "hardlinks_applied"), + ).toBe(true); + expect(receipt.seedRevisionId).toBe(host.initial.revisionId); + expect(receipt.commit).toBe(recreated ? "final_committed" : "seed_committed"); + expect(receipt.followUp).not.toBe("complete"); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + recreated ? receipt.finalRevisionId : host.initial.revisionId, + ); + if (mode === "cancel") { + const pid = Number(readFileSync(marker, "utf8")); + await expect + .poll(() => { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } + }) + .toBe(false); + } + if (mode.startsWith("crash-")) { + const pid = Number(readFileSync(join(host.root, "cli-killed"), "utf8")); + expect(() => process.kill(pid, 0)).toThrow(); + expect(receipt.finalRevisionId === null).toBe(mode === "crash-pull"); + } + console.info( + JSON.stringify({ + proof: "actual-default-cli-failure-contained", + mode, + commit: receipt.commit, + phase: receipt.phase, + rootFenced: true, + }), + ); + }, + 90000, + ); + it.each([false, true])( + "returns the actual default CLI Update's signed final revision with competing generation=%s", + async (competing) => { + const { chmodSync } = await import("node:fs"); + const bin = join(host.root, "fixture-bin"), + trace = join(host.root, "docker-argv.jsonl"); + mkdirSync(bin); + // The only child replacement is external Docker I/O. The default runner, + // installed tsx, real CLI entry, manifests, writer, pins, lease and receipt run. + const docker = join(bin, "docker"); + writeFileSync( + docker, + `#!${process.execPath}\nrequire("node:fs").appendFileSync(${JSON.stringify(trace)},JSON.stringify(process.argv.slice(2))+"\\n");\n`, + { mode: 0o700 }, + ); + chmodSync(docker, 0o700); + for (const [key, value] of Object.entries({ + PATH: `${bin}:${process.env.PATH}`, + OPENMAPX_ROOT_DIR: host.root, + OPENMAPX_RELEASE: "local", + DOMAIN: "example.test", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + DATA_MANAGER_AUTH_TOKEN: "d".repeat(32), + OPENMAPX_ENABLED_SERVICES: "", + })) + vi.stubEnv(key, value); + const ctx = await job(), + { executeAdminJobOperation } = await import("../admin-job-ops"); + const result = await executeAdminJobOperation( + ctx, + { + kind: "service.update", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + "task012.default-cli-update", + { durableIdentity: "motis" }, + ); + expect(result.changed).toBe(true); + if (competing) { + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.competingRevisionId, + ); + expect(result.revisionId).not.toBe(host.competingRevisionId); + const { updatePreparedService } = await import("../service-preparation"); + await expect( + updatePreparedService(ctx, "motis", result.revisionId), + ).rejects.toMatchObject({ + result: { serviceLifecycle: { failure: { reason: "revision_conflict" } } }, + }); + } else + expect(result.revisionId).toBe(services.currentConfigurationGeneration(host.infraDir)); + expect(result.revisionId).not.toBe(host.initial.revisionId); + const calls = readFileSync(trace, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + expect( + calls.map((args) => args.slice(args.indexOf("--project-directory") + 2)), + ).toHaveLength(2); + expect(calls[0]?.slice(-2)).toEqual(["pull", "motis"]); + expect(calls[1]?.slice(-4)).toEqual(["up", "-d", "--force-recreate", "motis"]); + if (!competing) { + const directory = join(host.infraDir, "data/ops-agent/lifecycle"), + [name] = readdirSync(directory); + expect(name).toBeDefined(); + const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 3600000); + try { + let retired = 0; + for (let i = 0; i < 12 && existsSync(join(directory, name!)); i++) + retired += (await reconcileControlRetention(installation, { maxFamilies: 1 })) + .retired; + expect(retired).toBeGreaterThan(0); + expect(existsSync(join(directory, name!))).toBe(false); + } finally { + clock.mockRestore(); + } + } + }, + 60000, + ); + it("returns the actual default CLI Update's signed final revision through durable nonempty links", async () => { + dm = await ownedDataManager({ + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + State: "running", + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }, + ]); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "actual-default-cli-osm"); + const bin = join(host.root, "fixture-bin"), + trace = join(host.root, "docker-argv.jsonl"); + mkdirSync(bin); + // Only external Docker I/O is replaced. The actual default runner launches + // the installed CLI entry, real shared writer, signed receipt and DM leaf. + writeFileSync( + join(bin, "docker"), + `#!${process.execPath}\nrequire("node:fs").appendFileSync(${JSON.stringify(trace)},JSON.stringify(process.argv.slice(2))+"\\n");\n`, + { mode: 0o700 }, + ); + for (const [key, value] of Object.entries({ + PATH: `${bin}:${process.env.PATH}`, + OPENMAPX_ROOT_DIR: host.root, + OPENMAPX_RELEASE: "local", + DOMAIN: "example.test", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + DATA_MANAGER_AUTH_TOKEN: dm.token, + OPENMAPX_ENABLED_SERVICES: "", + })) + vi.stubEnv(key, value); + const ctx = await job(), + { executeAdminJobOperation, readProjection } = await import("../admin-job-ops"), + result = await executeAdminJobOperation( + ctx, + { + kind: "service.update", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + "task012.default-cli-nonempty-update", + { durableIdentity: "motis" }, + ); + expect(result).toMatchObject({ + changed: true, + revisionId: services.currentConfigurationGeneration(host.infraDir), + }); + expect(result.revisionId).not.toBe(host.initial.revisionId); + const calls = readFileSync(trace, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + expect(calls).toHaveLength(2); + expect(calls[0]?.slice(-2)).toEqual(["pull", "motis"]); + expect(calls[1]?.slice(-4)).toEqual(["up", "-d", "--force-recreate", "motis"]); + expect(dm.posts).toBe(1); + const [entry] = Object.values(readProjection(ctx.checkpointResult).operations), + root = restoreAdmittedControlExecution(installation, { + executor: "ops-agent", + principal: "api", + executionId: entry!.operationId!, + operationKey: entry!.operationKey, + operationFingerprint: (await import("@openmapx/core/ops")).opsOperationFingerprint( + entry!.operation!, + ), + fingerprint: entry!.fingerprint!, + deadlineAt: entry!.deadlineAt!, + }), + child = lookupControlChildPhaseBinding(root, { phase: "lifecycle", ordinal: 0 }); + expect(child).toBeDefined(); + const receipt = await services.readServiceLifecycleReceipt({ + dataRoot: join(host.infraDir, "data"), + execution: child!.child.execution, + }); + expect(receipt).toMatchObject({ + phase: "complete", + finalRevisionId: result.revisionId, + followUp: "complete", + commit: "final_committed", + hardlinks: { entries: 1, linked: 1, via: "data-manager" }, + }); + const plan = JSON.parse( + readFileSync( + services.currentConfigurationFile(host.infraDir, services.GENERATED_HARDLINK_PLAN_FILE), + "utf8", + ), + ) as { target: string }[], + destination = join(host.infraDir, plan[0]!.target, "fixture.pbf"); + expect(statSync(destination).ino).toBe(statSync(join(source, "fixture.pbf")).ino); + expect( + JSON.parse( + readFileSync( + services.currentConfigurationFile(host.infraDir, services.SERVICE_SELECTION_FILE), + "utf8", + ), + ).selected, + ).toEqual(["data-manager", "motis"]); + expect(existsSync(join(host.infraDir, ".trusted-config-writer"))).toBe(false); + for (const id of ["data-manager", "motis"]) + expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks", id))).toBe(false); + }, 60000); + }, +); diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts new file mode 100644 index 000000000..60ec9f65f --- /dev/null +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -0,0 +1,3859 @@ +import { execFileSync, spawn } from "node:child_process"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import filesystem, { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createRequire, syncBuiltinESMExports } from "node:module"; +import { dirname, join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { + acquireArtifactTargetLease, + assertControlExecutionForward, + assertCoordinatorPullLeaseCompatibility, + closeControlInstallation, + controlDigest, + controlRetentionPreparationState, + lookupControlChildPhase, + openControlInstallation, + ownedCoordinatorPullBaseline, + reconcileControlRetention, + recordStagedCoordinatorRetentionCheck, + releaseArtifactTargetLease, + restoreAdmittedControlExecution, + retainAdmittedControlExecution, + retainControlEffectReceipt, + settleArtifactTargetLeaseChild, + verifyControl, + verifyControlEffectSettlement, + verifyRetainedArtifactTargetRootRelease, +} from "@openmapx/core/build-artifacts"; +import { + type CoordinatorPullSubmission, + createControlRootOperationKey, + executionFingerprint, + inspectPreparedConfigurationRecord, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { eq } from "drizzle-orm"; +import Fastify, { type FastifyRequest } from "fastify"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + stagedCoordinatorRetirementDomain, + stagedCoordinatorRetirementPayloadSchema, +} from "../../../../../packages/core/src/build-artifacts/coordinator-pull-records"; +import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; +import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; +import { configurationPolicyRevision } from "../../../../ops-agent/src/configured-resource-authority"; +import { runContainedProcess } from "../../../../ops-agent/src/docker-runtime"; +import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; +import { + consumeCoordinatorPullOwnershipCapture, + inspectOriginalCoordinatorServices, +} from "../../../../ops-agent/src/owned-services"; +import { + createPolicyResourceClaimer, + createProductionRegistryResourceClaimer, + denyAllTrustedOpsData, +} from "../../../../ops-agent/src/policy"; +import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; +import * as schema from "../../db/schema"; +import { createConfigurationPostgresFixture } from "../../test/configuration-postgres-fixture"; + +const coordinatorReferenceCases = [ + { + name: "qualified sibling coordinator image alias", + target: { image: "example/data-manager", tag: "1" }, + sibling: { image: "docker.io/example/data-manager", tag: "1" }, + }, + { + name: "legacy index sibling coordinator image alias", + target: { image: "ubuntu", tag: "latest" }, + sibling: { image: "index.docker.io/library/ubuntu", tag: "latest" }, + }, + { + name: "default library sibling coordinator image alias", + target: { image: "docker.io/ubuntu", tag: "latest" }, + sibling: { image: "ubuntu", tag: "latest" }, + }, + { + name: "distinct sibling coordinator tags", + target: { image: "example/data-manager", tag: "1" }, + sibling: { image: "example/data-manager", tag: "other" }, + }, +]; +type Fixture = Awaited>; +const state = vi.hoisted(() => ({ fixture: null as unknown as Fixture })); +// Only the database connection boundary is replaced; every query/transaction +// executes on the verified owned PostgreSQL instance and current schema copies. +vi.mock("../../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); + +describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "actual coordinator lifecycle API / production agent / PostgreSQL", + () => { + let fixture: Fixture, host: Awaited>; + let dm: Awaited>; + let agent: ReturnType, api: ReturnType; + let installation: ReturnType; + let refusedDecisionWrites = 0, + issuedKnownRollbacks = 0; + let nativeTrace: string; + let pulledImageId: string | null; + let originalPull: CoordinatorPullSubmission | undefined; + let afterImageStaging: (() => void) | undefined; + let afterPullTerminal: (() => void) | undefined; + let beforeLifecycleProcess: ((args: readonly string[]) => Promise) | undefined; + let decisionReplyGate: { reached: () => void; release: Promise } | undefined; + let pullAdmissionReplyGate: { reached: () => void; release: Promise } | undefined; + let pullTerminalReplyGate: { reached: () => void; release: Promise } | undefined; + let claimCase: + | "genuine" + | "generic resolver" + | "policy-only" + | "missing raw revision" + | "raw suffix drift" + | "consumed capture" = "genuine"; + let copyPullClaim = false, + observePullReply: ((status: number) => void) | undefined; + const posts = new Map(); + const arrivals = new Map(); + + beforeEach(async (test) => { + fixture = await createConfigurationPostgresFixture(); + state.fixture = fixture; + refusedDecisionWrites = 0; + issuedKnownRollbacks = 0; + if (test.task.name.includes("unused OLD runtime")) + state.fixture = { + ...fixture, + sql: new Proxy(fixture.sql, { + get(target, name, receiver) { + if (name !== "reserve") return Reflect.get(target, name, receiver); + return async () => { + const reserved = await target.reserve(); + return new Proxy(reserved, { + get(connection, member, owned) { + if (member !== "unsafe") return Reflect.get(connection, member, owned); + return ( + query: string, + parameters?: Parameters[1], + ) => { + const continuation = parameters?.some((value) => { + if (typeof value !== "string" || !value.startsWith("{")) return false; + try { + const v = JSON.parse(value); + return ( + v._configurationCandidate?.phase === "saved" && + v._configurationMaintenanceDecision?.operation?.decision === "continue" + ); + } catch { + return false; + } + }); + if ( + refusedDecisionWrites === 0 && + /^update "admin_job"/.test(query) && + continuation + ) { + refusedDecisionWrites++; + throw Error("Named SQL continuation write refused before delivery"); + } + if (refusedDecisionWrites && query === "ROLLBACK") issuedKnownRollbacks++; + return connection.unsafe(query, parameters); + }; + }, + }); + }; + }, + }), + }; + pulledImageId = null; + originalPull = undefined; + afterImageStaging = undefined; + afterPullTerminal = undefined; + beforeLifecycleProcess = undefined; + decisionReplyGate = undefined; + pullTerminalReplyGate = undefined; + pullAdmissionReplyGate = undefined; + copyPullClaim = false; + claimCase = "genuine"; + observePullReply = undefined; + vi.stubEnv("BETTER_AUTH_SECRET", "task012-coordinator-synthetic-auth-fixture"); + vi.stubEnv("BETTER_AUTH_URL", "http://localhost:3001"); + vi.stubEnv("OPENMAPX_RELEASE", "local"); + vi.stubEnv("OPENMAPX_ENABLED_SERVICES", ""); + const references = coordinatorReferenceCases.find((value) => + test.task.name.includes(value.name), + ); + host = await ownedServiceHost({ + withCoordinator: true, + ...(test.task.name.includes("three-target") ? { extraServiceIds: ["valhalla"] } : {}), + ...(test.task.name.includes("ordinary unrelated configuration") + ? { + schemas: { + motis: { type: "object", properties: { MODE: { type: "string", default: "old" } } }, + }, + } + : {}), + ...(test.task.name.includes("all current configuration producers") + ? { + schemas: { + motis: { + type: "object", + properties: { + MODE: { type: "string", default: "old" }, + TOKEN: { type: "string", "x-openmapx-secret": true }, + }, + }, + }, + } + : {}), + ...(test.task.name.includes("installed credential follow-up") + ? { + schemas: { + "data-manager": { + type: "object", + properties: { TOKEN: { type: "string", "x-openmapx-secret": true } }, + }, + }, + } + : {}), + ...(test.task.name.includes("full future owner budget") + ? { oversizedMaintenancePlan: true } + : {}), + ...(test.task.name.includes("3072-byte baseline") ? { nestedRootSegments: 9 } : {}), + ...(references + ? { imageReferences: { motis: references.sibling, "data-manager": references.target } } + : {}), + actualComposeConfig: true, + defaultCliEntry: fileURLToPath( + new URL("../../../../ops-agent/src/index.ts", import.meta.url), + ), + }); + if (test.task.name.includes("all current configuration producers")) { + for (const id of ["alpha", "beta"]) { + const directory = join(host.root, "integrations", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id, + version: "1.0.0", + author: "Task01.2 fixture", + license: "MIT", + domains: ["knowledge"], + quality: "built-in", + configSchema: { + properties: { + VALUE: { type: "string", default: "old" }, + TOKEN: { type: "string", "x-openmapx-secret": true }, + }, + }, + }), + ); + } + } + dm = await ownedDataManager({ + observeSettlementSql: true, + sql: fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + if (test.task.name.includes("3072-byte baseline")) { + const images = Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `example/${name}@sha256:${"b".repeat(64)}`, + ]), + ); + writeFileSync( + host.stack.composeReleasePath, + JSON.stringify({ + "x-openmapx-release": { release: "task012-baseline-budget", images }, + services: { + "app-api": { + image: images.api, + environment: { + [services.PRIVACY_BACKUP_COLLECTOR_IMAGE_ENV]: images["privacy-backup"], + }, + }, + "app-web": { image: images.web }, + "ops-agent": { + image: images["ops-agent"], + environment: { + [services.PRIVACY_BACKUP_COLLECTOR_IMAGE_ENV]: images["privacy-backup"], + }, + }, + "data-manager": { + image: images["data-manager"], + environment: { [services.TRANSITOUS_TOOLS_IMAGE_ENV]: images["transitous-tools"] }, + }, + "transitous-runner": { image: images["transitous-runner"] }, + }, + }), + { mode: 0o600 }, + ); + host.row.Labels["com.docker.compose.project.config_files"] = [ + host.firstCompose, + host.stack.composeReleasePath, + ].join(","); + } + const old = { + ...host.row, + Id: "c".repeat(64), + ConfigImage: + services.readServiceReleaseImageSelection(host.infraDir, "data-manager").imageReference ?? + services.serviceContainerImageReference( + host.loaded.find((s) => s.manifest.id === "data-manager")!.manifest.container, + ), + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + if (test.task.name.includes("stopped coordinator before pull")) { + old.State = "exited"; + await dm.pauseHttp(); + } + host.setRows([host.row, old]); + host.setImagePullProcess(async (reference) => { + const candidates = readdirSync(join(host.queue, ".candidates")); + expect(candidates).toHaveLength(1); + const candidate = inspectPreparedConfigurationRecord( + readFileSync(join(host.queue, ".candidates", candidates[0]!)), + host.token, + ).record, + fixed = candidate.privateCoordinatorLifecycle; + if (fixed?.action !== "update") throw Error("Actual original sealed Update source missing"); + originalPull = fixed.pull; + if (test.task.name.includes("stopped coordinator before pull")) { + const helper = candidate.coordinatorPrerequisite!; + const actual = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${helper.root.operationKey}.json`, + ), + "utf8", + ), + ).payload; + console.info( + "actual coordinator helper clamp", + JSON.stringify({ + requestedMatchesOriginal: + actual.requestedDeadlineAt === helper.root.requestedDeadlineAt, + actualDeadlineBeforeRequested: + actual.execution.deadlineAt < actual.requestedDeadlineAt, + oldRevisionMatches: + actual.operation.expectedConfigurationRevisionId === host.initial.revisionId, + }), + ); + } + expect(reference).toBe( + services.serviceContainerImageReference( + host.loaded.find((s) => s.manifest.id === "data-manager")!.manifest.container, + ), + ); + pulledImageId = `sha256:${"e".repeat(64)}`; + host.setImageIdentity(reference, pulledImageId); + afterImageStaging?.(); + }); + let recreatedContainers = 0; + host.setCreatingProcess(async (args) => { + await beforeLifecycleProcess?.(args); + if (!args.includes("--force-recreate")) { + if (args.includes("--no-recreate") && args.at(-1) === "data-manager") { + old.State = "running"; + await dm.resumeHttp(); + } + return; + } + const revision = services.currentConfigurationGeneration(host.infraDir); + if (!revision) throw Error("Actual committed coordinator generation unavailable"); + const prefix = services.stackComposeArgs(host.stack), + io = { signal: new AbortController().signal, timeout: 15000, maxBuffer: 1024 * 1024 }, + config = JSON.parse( + (await host.execFile("docker", [...prefix, "config", "--format", "json"], io)).stdout, + ), + hash = ( + await host.execFile("docker", [...prefix, "config", "--hash", "data-manager"], io) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!, + row = { + ...old, + Image: pulledImageId ?? old.Image, + Id: (BigInt(`0x${"d".repeat(64)}`) + BigInt(recreatedContainers++)) + .toString(16) + .padStart(64, "0"), + Labels: { + ...old.Labels, + "com.docker.compose.project.working_dir": dirname( + services.configurationGenerationComposeFile(host.infraDir, revision), + ), + "com.docker.compose.project.config_files": + services.configurationGenerationComposeFile(host.infraDir, revision), + "com.docker.compose.config-hash": hash, + }, + }, + service = config.services["data-manager"]; + host.setRows([host.row, row]); + host.setRuntimeInspections({ + [row.Id]: { + Id: row.Id, + Image: row.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: row.ConfigImage, + Labels: row.Labels, + Env: Object.entries(service.environment ?? {}).map(([k, v]) => `${k}=${v}`), + }, + Mounts: [ + ...(service.volumes ?? []).map( + (v: { type: string; source: string; target: string; read_only?: boolean }) => ({ + Type: v.type, + Source: v.source, + Destination: v.target, + RW: !v.read_only, + }), + ), + ...(service.secrets ?? []).map((v: { source: string; target: string }) => ({ + Type: "bind", + Source: config.secrets[v.source].file, + Destination: `/run/secrets/${v.target}`, + RW: false, + })), + ], + }, + }); + }); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + agent = buildOpsAgentServer({ + operationTimeoutMs: 180000, + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: { + claim: async (...args) => { + if ( + args[0].kind === "service.pull" && + claimCase !== "genuine" && + claimCase !== "consumed capture" + ) { + if (claimCase === "policy-only") + return createPolicyResourceClaimer({ allowService: () => true }).claim(...args); + const source = await host.loadAuthority(), + applied = await host.actionAuthority(); + const registered = source.services.map((s) => ({ + serviceId: s.manifest.id, + enabled: s.enabled, + isBuiltIn: s.isBuiltIn, + })); + return createProductionRegistryResourceClaimer({ + services: registered, + integrationIds: [], + trustedData: denyAllTrustedOpsData, + ownedServices: + claimCase === "generic resolver" + ? { inspectOwnedServices: (signal) => host.owned.inspectOwnedServices(signal) } + : host.owned, + enabledServiceIds: () => applied.enabledServiceIds, + serviceMetadata: (ss) => ss.map((value) => ({ ...value, buildable: false })), + loadAuthority: async () => ({ + revisionId: configurationPolicyRevision(source.revisionId), + services: registered, + integrationIds: [], + ...(claimCase === "missing raw revision" + ? {} + : { + configurationAuthorityRevision: + claimCase === "raw suffix drift" + ? `${source.revisionId.slice(0, -1)}${source.revisionId.endsWith("A") ? "B" : "A"}` + : source.revisionId, + }), + }), + }).claim(...args); + } + const raw = await host.claimer.claim(...args); + if ( + raw && + args[0].kind === "service.pull" && + args[0].candidateId && + claimCase === "consumed capture" + ) + consumeCoordinatorPullOwnershipCapture(raw, { + operationFingerprint: args[1], + candidateId: args[0].candidateId, + policyRevision: raw.capability.revisionId, + }); + return raw && copyPullClaim && args[0].kind === "service.pull" + ? { + ...raw, + operation: structuredClone(raw.operation), + capability: structuredClone(raw.capability), + } + : raw; + }, + }, + journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + controlInstallation: installation, + }); + posts.clear(); + arrivals.clear(); + agent.addHook("preHandler", async (request) => { + const kind = (request.body as { operation?: { kind?: string } } | undefined)?.operation + ?.kind; + if (request.method === "POST" && kind) arrivals.set(kind, (arrivals.get(kind) ?? 0) + 1); + }); + agent.addHook("onSend", async (request, _reply, payload) => { + const kind = (request.body as { operation?: { kind?: string } } | undefined)?.operation + ?.kind; + if (request.method === "POST" && request.url === "/v1/operations" && kind) + posts.set(kind, (posts.get(kind) ?? 0) + 1); + if (kind === "service.pull") { + observePullReply?.(_reply.statusCode); + if (pullAdmissionReplyGate) { + pullAdmissionReplyGate.reached(); + await pullAdmissionReplyGate.release; + } + } + if ( + request.method === "GET" && + (pullTerminalReplyGate || afterPullTerminal) && + typeof payload === "string" + ) { + const observed = JSON.parse(payload) as { result?: { kind?: string; state?: string } }; + if (observed.result?.kind === "service.pull" && observed.result.state === "succeeded") { + const observed = afterPullTerminal; + afterPullTerminal = undefined; + observed?.(); + const gate = pullTerminalReplyGate; + if (gate) { + gate.reached(); + await gate.release; + } + } + } + if (kind === "configurationMaintenance.decide" && decisionReplyGate) { + const gate = decisionReplyGate; + gate.reached(); + await gate.release; + } + return payload; + }); + const tokenFile = join(host.root, "api-token"); + writeFileSync(tokenFile, host.token, { mode: 0o600 }); + const realDocker = execFileSync("which", ["docker"], { encoding: "utf8" }).trim(), + bin = join(host.root, "fixture-bin"); + nativeTrace = join(host.root, "native-docker-argv.jsonl"); + mkdirSync(bin); + // Real default CLI/process/containment; only Docker's external effects + // are synthetic. Native Compose config/hash still executes unchanged. + writeFileSync( + join(bin, "docker"), + `#!${process.execPath}\nconst fs=require("node:fs"),cp=require("node:child_process"),a=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(nativeTrace)},JSON.stringify(a)+"\\n");if(a[0]==="compose"&&a.includes("config")){const r=cp.spawnSync(${JSON.stringify(realDocker)},a,{encoding:"utf8"});process.stdout.write(r.stdout||"");process.stderr.write(r.stderr||"");process.exit(r.status??1);}\n`, + { mode: 0o700 }, + ); + for (const [key, value] of Object.entries({ + OPS_AGENT_URL: await agent.listen({ host: "127.0.0.1", port: 0 }), + OPS_AGENT_TOKEN_FILE: tokenFile, + OPS_TRUSTED_CONFIG_DIR: host.queue, + OPS_TRUSTED_CONFIG_UID: String(statSync(host.queue).uid), + OPS_TRUSTED_CONFIG_GID: String(statSync(host.queue).gid), + OPENMAPX_ROOT_DIR: host.root, + OPENMAPX_SECRETS_KEY: "71".repeat(32), + OPENMAPX_LOCAL_ADMIN_TOKEN: "task012-coordinator-local-admin", + OPENMAPX_DISABLE_LOCALHOST_AUTH: "0", + DATA_MANAGER_URL: dm.baseUrl, + DATA_MANAGER_AUTH_TOKEN: dm.token, + DOMAIN: "example.test", + POSTGRES_PASSWORD: "p".repeat(24), + PATH: `${bin}:${process.env.PATH}`, + })) + vi.stubEnv(key, value); + const { initServiceRegistry } = await import("../service-registry"); + await initServiceRegistry({ strict: true }); + const { jobRunner } = await import("../job-runner"), + { serviceApply } = await import("../admin-ops"), + { handleServiceBulkJob } = await import("../admin-job-handlers"); + // Exact current server registration wrapper, with the actual production + // serviceApply and JobRunner; no agent handler/authority is substituted. + jobRunner.register("service.apply", async (ctx) => { + const result = await serviceApply(ctx.payload.service as string, ctx); + return { service: ctx.payload.service, action: "apply", ...result }; + }); + jobRunner.register("service.bulk", async (ctx) => { + try { + return await handleServiceBulkJob(ctx); + } catch (error) { + if (test.task.name.includes("actual bulk Update")) + console.info( + "actual bulk boundary error", + JSON.stringify({ + name: error instanceof Error ? error.name : "unknown", + message: error instanceof Error ? error.message : "unknown", + stack: error instanceof Error ? error.stack?.split("\n").slice(1, 5) : [], + }), + ); + throw error; + } + }); + api = Fastify({ logger: false }); + api.setErrorHandler((await import("../../server-wiring")).uniformErrorHandler); + const { initIntegrations } = await import("../../integration-host"); + await initIntegrations( + api, + test.task.name.includes("all current configuration producers") + ? [{ directory: join(host.root, "integrations"), isBuiltIn: true }] + : [], + ); + const { requireAdmin } = await import("../../utils/require-admin"); + api.addHook("preHandler", async (request: FastifyRequest) => { + request.adminSession = await requireAdmin(request); + }); + const { adminServicesRoute } = await import("../../routes/admin-services"); + await api.register(adminServicesRoute, { prefix: "/api" }); + if (test.task.name.includes("all current configuration producers")) { + const { adminIntegrationsRoute } = await import("../../routes/admin-integrations"), + { registerAdminComposeRoutes } = await import("../../routes/admin-compose"); + await api.register(adminIntegrationsRoute, { prefix: "/api" }); + const { adminIntegrationCredentialsRoute } = await import( + "../../routes/admin-integration-credentials" + ); + await api.register(adminIntegrationCredentialsRoute, { prefix: "/api" }); + await api.register(registerAdminComposeRoutes, { prefix: "/api" }); + } + await api.ready(); + }, 60000); + afterEach(async () => { + await api?.close(); + // Observe the actual runner's scheduled pump/active work before closing + // its owned SQL transport. This changes no scheduler or handler result. + const { jobRunner } = await import("../job-runner"), + runner = jobRunner as unknown as { active: Map; pumping: boolean }; + const until = Date.now() + 5000; + let idleTurns = 0; + while (idleTurns < 2) { + await delay(0); + idleTurns = runner.active.size === 0 && !runner.pumping ? idleTurns + 1 : 0; + if (Date.now() > until) throw Error("Actual fixture runner did not quiesce"); + } + await agent?.close(); + await dm?.close(); + if (installation) closeControlInstallation(installation); + await fixture?.stop(); + if (host) rmSync(host.cleanupRoot, { recursive: true, force: true }); + vi.unstubAllEnvs(); + }); + async function finished(id: string) { + const until = performance.now() + 150000; + let diagnostic = false; + for (;;) { + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + if (row && ["success", "failed", "canceled"].includes(row.status)) return row; + if (!diagnostic && performance.now() >= until - 105000) { + diagnostic = true; + console.info( + "actual host-root later source", + JSON.stringify({ + status: row?.status, + error: row?.error, + posts: [...posts], + dm: await dm.diagnostics(), + projection: Object.values( + ( + row?.result as { + opsProjection?: { + operations?: Record< + string, + { kind?: string; phase?: string; errorClass?: string } + >; + }; + } + )?.opsProjection?.operations ?? {}, + ).map((value) => ({ + kind: value.kind, + phase: value.phase, + errorClass: value.errorClass, + })), + candidate: (row?.result as { _configurationCandidate?: { phase?: string } }) + ?._configurationCandidate?.phase, + maintenance: (row?.result as { _configurationMaintenance?: { phase?: string } }) + ?._configurationMaintenance?.phase, + }), + ); + } + if (performance.now() >= until) throw Error("Actual coordinator API job did not settle"); + await delay(25); + } + } + async function canceledSuccessfulCoordinatorStage( + beforeAcknowledgement?: () => Promise, + serviceIds = ["data-manager"], + ) { + const reached = Promise.withResolvers(), + release = Promise.withResolvers(); + pullTerminalReplyGate = { reached: () => reached.resolve(), release: release.promise }; + try { + const first = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds }, + }), + id = first.json().jobId; + await reached.promise; + await beforeAcknowledgement?.(); + const { jobRunner } = await import("../job-runner"), + cancellation = jobRunner.cancel(id); + const until = performance.now() + 10000; + for (;;) { + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + if (row?.status === "cancel_pending" || row?.status === "canceled") break; + if (performance.now() > until) throw Error("Actual cancellation marker unavailable"); + await delay(10); + } + pullTerminalReplyGate = undefined; + release.resolve(); + await cancellation; + // The actual API server schedules reconcile; this route fixture owns + // that schedule explicitly and waits for the original real claim lease. + const cleanupUntil = performance.now() + 90000; + for (;;) { + await jobRunner.reconcile({ limit: 1 }); + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + if (row?.status === "canceled") break; + if (performance.now() > cleanupUntil) + throw Error("Canceled staged candidate did not reconcile"); + await delay(500); + } + expect((await finished(id)).status).toBe("canceled"); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + const [retained] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + const pull = Object.values( + ( + retained!.result as { + opsProjection: { operations: Record }; + } + ).opsProjection.operations, + ).find((value) => value.kind === "service.pull")!; + return { + id, + pullPath: join( + host.infraDir, + "control-private/ops-agent/executions", + `${pull.operationKey}.json`, + ), + }; + } finally { + pullTerminalReplyGate = undefined; + release.resolve(); + } + } + async function restartNativeStagedAgent( + options: { stopAfterFirstTargetRelease?: "held" | "unwound" } = {}, + ) { + await agent.close(); + const bridge = Fastify({ logger: false, bodyLimit: 64 * 1024 }); + bridge.post("/docker", async (request, reply) => { + const args = (request.body as { args: unknown }).args; + if (!Array.isArray(args) || args.some((arg) => typeof arg !== "string")) + throw Error("Invalid named Docker fixture arguments"); + try { + return await host.execFile("docker", args, { + signal: new AbortController().signal, + timeout: 180000, + maxBuffer: 4 * 1024 * 1024, + }); + } catch { + return reply.code(502).send({ error: "Named Docker IO failed" }); + } + }); + const dockerBridge = await bridge.listen({ host: "127.0.0.1", port: 0 }); + const argv = [ + "--import", + createRequire(import.meta.url).resolve("tsx/esm"), + fileURLToPath( + new URL( + "../../../../ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs", + import.meta.url, + ), + ), + ]; + console.info( + "actual native fresh-request agent argv", + JSON.stringify([process.execPath, ...argv]), + ); + const child = spawn(process.execPath, argv, { + cwd: process.cwd(), + env: process.env, + stdio: ["pipe", "pipe", "pipe"], + }), + ready = Promise.withResolvers<{ url: string; ownedDM: boolean }>(), + fault = Promise.withResolvers(); + let observedFault = false, + killedForFault = false, + closed = false; + let output = "", + stderr = ""; + child.stdout.on("data", (chunk) => { + output += chunk.toString(); + const lines = output.split("\n"); + output = lines.pop()!; + for (const line of lines) { + const event = JSON.parse(line); + if (event.event === "ready") ready.resolve(event); + if (event.event === "source-writer-busy") + console.info( + "actual native source writer busy", + JSON.stringify({ sameProcess: event.sameProcess, frames: event.frames }), + ); + if (event.event === "pre-effect-release-unlinked") { + observedFault = true; + fault.resolve(); + } + if (event.event === "operation") { + posts.set(event.kind, (posts.get(event.kind) ?? 0) + 1); + if (event.status >= 400) + console.info( + "actual native operation rejection", + JSON.stringify({ + kind: event.kind, + status: event.status, + errorClass: event.errorClass, + }), + ); + } + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + if (stderr.length > 1024 * 1024) child.kill("SIGKILL"); + }); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve) => + child.once("exit", (code, signal) => { + ready.reject( + Error(`Native fresh-request agent exited ${code}: ${stderr.slice(-2500)}`), + ); + resolve({ code, signal }); + }), + ); + child.stdin.end( + JSON.stringify({ + root: host.root, + infraDir: host.infraDir, + stack: host.stack, + queue: host.queue, + token: host.token, + serviceIds: host.loaded.map((service) => service.manifest.id), + dockerBridge: `${dockerBridge}/docker`, + ...(options.stopAfterFirstTargetRelease + ? { stopAfterFirstTargetRelease: options.stopAfterFirstTargetRelease } + : {}), + }), + ); + try { + const observed = await ready.promise; + expect(observed.ownedDM).toBe(true); + vi.stubEnv("OPS_AGENT_URL", observed.url); + } catch (error) { + child.kill("SIGKILL"); + await exited; + await bridge.close(); + throw error; + } + const killAfterFault = async () => { + await fault.promise; + expect(observedFault).toBe(true); + expect(child.kill("SIGKILL")).toBe(true); + killedForFault = true; + const result = await exited; + expect(result).toEqual({ code: null, signal: "SIGKILL" }); + }; + const stop = async () => { + if (closed) return; + if (observedFault && !killedForFault) await killAfterFault(); + if (!killedForFault) child.kill("SIGTERM"); + const result = await exited; + await bridge.close(); + closed = true; + expect(result).toEqual( + killedForFault ? { code: null, signal: "SIGKILL" } : { code: 0, signal: null }, + ); + }; + return Object.assign(stop, { fault: fault.promise, killAfterFault }); + } + it("requires the actual API ACK before a successful staged pull becomes ownership evidence", async () => { + await canceledSuccessfulCoordinatorStage(async () => { + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(false); + }); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(true); + }, 180000); + it("reconstructs staged ownership in a NEW native production agent before a fresh user Update", async () => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + original = readFileSync(pullPath), + stop = await restartNativeStagedAgent(); + try { + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + row = await finished(next.json().jobId); + expect(row.status, row.error ?? "No public error").toBe("success"); + expect(posts.get("service.pull")).toBe(2); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(dm.settlementResponseStatuses).toEqual([200]); + expect(readFileSync(pullPath).equals(original)).toBe(true); + } finally { + await stop(); + } + }, 240000); + it("uses fresh authority after repeated native expiry staging without recursive provenance", async () => { + const originals: Array<{ path: string; bytes: Buffer }> = []; + try { + for (let n = 0; n < 2; n++) { + afterPullTerminal = () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 6 * 60000); + }; + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + row = await finished(reply.json().jobId), + projection = ( + row.result as { + opsProjection: { + operations: Record< + string, + { + kind: string; + operationKey: string; + receiptDelivery?: { terminalReceipt: { state: string } }; + } + >; + }; + } + ).opsProjection, + pull = Object.values(projection.operations).find((op) => op.kind === "service.pull")!; + expect(row.status, row.error ?? "No public error").toBe("failed"); + console.info( + "actual repeated expiry original", + JSON.stringify({ + ordinal: n, + error: row.error, + operations: Object.values(projection.operations).map((op) => ({ + kind: op.kind, + receipt: !!op.receiptDelivery, + })), + }), + ); + expect(pull.receiptDelivery?.terminalReceipt.state).toBe("succeeded"); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + const path = join( + host.infraDir, + "control-private/ops-agent/executions", + `${pull.operationKey}.json`, + ); + originals.push({ path, bytes: readFileSync(path) }); + // Each request keeps its original clocks and signed bytes. Let the + // real wall/SQL clock catch up before the NEXT fresh request. + await delay(6 * 60000 + 1000); + vi.useRealTimers(); + } + vi.useRealTimers(); + const stop = await restartNativeStagedAgent(); + try { + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + row = await finished(reply.json().jobId); + console.info( + "actual repeated expiry native result", + JSON.stringify({ + state: row.status, + error: row.error, + configuration: (row.result as Record)?.configuration, + coordinatorLifecycle: (row.result as Record)?.coordinatorLifecycle, + settlementStatuses: dm.settlementResponseStatuses, + posts: [...posts], + }), + ); + expect(row.status, row.error ?? "No public error").toBe("success"); + expect(posts.get("service.pull")).toBe(3); + expect(dm.settlementResponseStatuses).toEqual([200]); + for (const original of originals) + expect(readFileSync(original.path).equals(original.bytes)).toBe(true); + } finally { + await stop(); + } + } finally { + vi.useRealTimers(); + } + }, 1200000); + it("carries an authenticated staged source through a fresh Apply's OWN generation after canceled staging", async () => { + await canceledSuccessfulCoordinatorStage(); + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/data-manager/action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "apply" }, + }); + const row = await finished(next.json().jobId); + expect(row.status, row.error ?? "No public error").toBe("success"); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(dm.settlementResponseStatuses).toEqual([200]); + }, 300000); + it("guards all current configuration producers before any desired write while staged DM is required", async () => { + await fixture.db + .insert(schema.serviceConfig) + .values({ id: randomUUID(), serviceId: "motis", config: { MODE: "old" } }); + await fixture.db.insert(schema.integrationConfig).values([ + { id: randomUUID(), integrationId: "alpha", config: { VALUE: "old", enabled: true } }, + { id: randomUUID(), integrationId: "beta", config: { VALUE: "old", enabled: true } }, + ]); + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + original = readFileSync(pullPath), + before = services.currentConfigurationGeneration(host.infraDir), + rows = async () => + Promise.all([ + fixture.sql`SELECT id,xmin::text AS version FROM service_config ORDER BY id`, + fixture.sql`SELECT id,xmin::text AS version FROM integration_config ORDER BY id`, + fixture.sql`SELECT id,xmin::text AS version FROM service_secret ORDER BY id`, + fixture.sql`SELECT id,xmin::text AS version FROM integration_secret ORDER BY id`, + ]), + unchanged = await rows(), + selection = services.readDesiredSelection(host.infraDir), + sdk = await import("@openmapx/core/api-admin-sdk"), + { createApiClient } = await import("@openmapx/core"), + baseUrl = await api.listen({ host: "127.0.0.1", port: 0 }), + options = { + client: createApiClient({ baseUrl }), + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + }, + mutations: readonly [ + string, + (headers: { "idempotency-key": string }) => Promise<{ status: number; data: unknown }>, + ][] = [ + [ + "service config", + (headers) => + sdk.updateAdminServiceConfig("motis", { config: { MODE: "new" } }, headers, options), + ], + [ + "service credential set", + (headers) => + sdk.setAdminServiceCredential( + "motis", + "TOKEN", + { value: "synthetic-new-secret" }, + headers, + options, + ), + ], + [ + "service credential delete", + (headers) => + sdk.deleteAdminServiceCredential("motis", "TOKEN", headers, undefined, options), + ], + [ + "selection", + (headers) => + sdk.setAdminServiceSelection( + { selectedRoots: ["data-manager", "motis"] }, + headers, + options, + ), + ], + [ + "integration config", + (headers) => + sdk.patchAdminIntegrationConfig("alpha", { VALUE: "new" }, headers, options), + ], + [ + "integration credential set", + (headers) => + sdk.setAdminIntegrationCredential( + "alpha", + "TOKEN", + { value: "synthetic-new-secret" }, + headers, + options, + ), + ], + [ + "integration credential delete", + (headers) => + sdk.deleteAdminIntegrationCredential("alpha", "TOKEN", headers, undefined, options), + ], + [ + "integration enable", + (headers) => sdk.enableAdminIntegration("alpha", headers, undefined, options), + ], + [ + "integration disable", + (headers) => sdk.disableAdminIntegration("alpha", headers, undefined, options), + ], + [ + "ordinary batch import", + (headers) => + sdk.importAdminIntegrationConfigs( + { + integrations: [ + { integrationId: "alpha", config: { VALUE: "batch-a" } }, + { integrationId: "beta", config: { VALUE: "batch-b", enabled: false } }, + ], + }, + headers, + options, + ), + ], + [ + "unchanged ordinary import", + (headers) => + sdk.importAdminIntegrationConfigs( + { + integrations: [ + { integrationId: "alpha", config: { VALUE: "old", enabled: true } }, + ], + }, + headers, + options, + ), + ], + ["Compose Up", (headers) => sdk.startAdminCompose(headers, undefined, options)], + ]; + let index = 0; + for (const [name, run] of mutations) { + const response = await run({ "idempotency-key": `task012-staged-current-${index++}` }); + expect(response.status, `${name}: ${JSON.stringify(response.data)}`).toBe(409); + expect(JSON.stringify(response.data)).toContain( + "Apply or update Data Manager before changing instance configuration.", + ); + expect(await rows()).toEqual(unchanged); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); + expect(services.readDesiredSelection(host.infraDir)).toEqual(selection); + expect(readFileSync(pullPath)).toEqual(original); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + } + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + for (const kind of [ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + "stack.start", + ]) + expect(posts.get(kind) ?? 0).toBe(0); + const preview = await api.inject({ + method: "GET", + url: "/api/admin/compose/preview", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + }); + expect(preview.statusCode, preview.body).toBe(200); + const repair = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(repair.json().jobId)).status).toBe("success"); + // Original failed requests remain failed after repair; only a NEW key + // can evaluate new source state. No old import proof/intent is replayed. + for (const index of [9, 10]) { + const [name, run] = mutations[index]!; + const replay = await run({ "idempotency-key": `task012-staged-current-${index}` }); + expect(replay.status, name).toBe(409); + expect(JSON.stringify(replay.data)).toContain( + "Apply or update Data Manager before changing instance configuration.", + ); + } + const resumed = await sdk.importAdminIntegrationConfigs( + { integrations: [{ integrationId: "alpha", config: { VALUE: "old", enabled: true } }] }, + { "idempotency-key": "task012-staged-current-new-unchanged" }, + options, + ); + expect(resumed.status, JSON.stringify(resumed.data)).toBe(200); + expect(resumed.data).toMatchObject({ ok: true, imported: 0 }); + }, 300000); + it("checks the literal staged prerequisite before current release-source writes", async () => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + originalPull = readFileSync(pullPath); + const { productionReleaseEffects } = await import( + "../../../../ops-agent/src/production-runtime" + ), + directory = join(host.infraDir, ".ops-agent-releases"), + manifest = services.parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: "interlock-release", + images: Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${"f".repeat(64)}`, + ]), + ), + privacyReleaseValidation: { + version: 1, + sourceBuildFingerprint: "f".repeat(64), + validatedAt: "2026-09-12T12:00:00.000Z", + checks: { + translationsConsistent: true, + openApiConsistent: true, + policyConsistent: true, + }, + }, + }), + ); + writeFileSync( + join(directory, "interlock-release.json"), + services.canonicalReleaseManifest(manifest), + { mode: 0o600 }, + ); + const effects = productionReleaseEffects(host.runtime), + context = { signal: new AbortController().signal, emitLog() {} }, + before = services.currentConfigurationGeneration(host.infraDir), + originalFiles = readdirSync(directory).sort(), + outcome = await effects.apply("interlock-release", ["app-api"], context).then( + () => null, + (error) => error, + ); + expect(existsSync(host.stack.composeReleasePath)).toBe(false); + expect(existsSync(join(directory, "transaction.json"))).toBe(false); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); + expect(readdirSync(directory).sort()).toEqual(originalFiles); + expect(readFileSync(pullPath)).toEqual(originalPull); + expect(outcome).toMatchObject({ errorClass: "configuration_prerequisite" }); + await expect(effects.resolve(context)).rejects.toMatchObject({ + errorClass: "configuration_prerequisite", + }); + const repair = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(repair.json().jobId)).status).toBe("success"); + const { OpsNotWiredError } = await import("../../../../ops-agent/src/runtime"); + await expect(effects.apply("interlock-release", ["app-api"], context)).rejects.toBeInstanceOf( + OpsNotWiredError, + ); + expect(existsSync(host.stack.composeReleasePath)).toBe(false); + expect(existsSync(join(directory, "transaction.json"))).toBe(false); + // Local release selection keeps its existing refusal after the real DM + // prerequisite is resolved; no release/app update permission is added. + await expect(effects.resolve(context)).rejects.toBeInstanceOf( + services.ReleaseChannelDisabledError, + ); + }, 240000); + it("carries staged ownership and desired E through the installed credential follow-up's OWN generation", async () => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + original = readFileSync(pullPath); + const changed = await api.inject({ + method: "PUT", + url: "/api/admin/services/data-manager/credentials/TOKEN", + headers: { + "x-openmapx-local-admin": "task012-coordinator-local-admin", + "idempotency-key": "task012-staged-credential", + }, + payload: { value: "synthetic-private-credential" }, + }); + expect(changed.statusCode, changed.body).toBe(200); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(dm.settlementResponseStatuses).toEqual([200]); + expect(readFileSync(pullPath).equals(original)).toBe(true); + const jobs = await fixture.db.select().from(schema.adminJob), + ownerJob = jobs.find( + (job) => (job.result as Record)?._configurationMaintenance, + ), + checkpoint = ownerJob!.result as { + _configurationMaintenance: { receipt: { binding: { maintenanceId: string } } }; + }, + owner = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${checkpoint._configurationMaintenance.receipt.binding.maintenanceId}.json`, + ), + "utf8", + ), + ).payload; + expect(owner.stagedCoordinatorProvenance.payload.ownerPrepare).toEqual(owner.plan.prepare); + expect(owner.coordinatorPull ?? null).toBeNull(); + expect(owner.plan.coordinatorImage).toMatchObject({ + action: "apply", + imageId: pulledImageId, + }); + expect( + Buffer.byteLength(JSON.stringify(owner.stagedCoordinatorProvenance)), + ).toBeLessThanOrEqual(8192); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).find( + (s) => s.serviceId === "data-manager", + )?.provenance.imageId, + ).toBe(pulledImageId); + expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(1); + }, 180000); + it("pins the LAST authenticated staged normal family beyond 72h while OLD B still needs it", async () => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(); + expect(existsSync(pullPath)).toBe(true); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(true); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 96 * 3600000); + try { + for (let step = 0; step < 12; step++) + await reconcileControlRetention(installation, { maxFamilies: 64 }); + expect(existsSync(pullPath)).toBe(true); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(true); + } finally { + vi.useRealTimers(); + } + }, 180000); + it("retires the authenticated staged family after actual guarded E replacement restores ordinary ownership", async () => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(); + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/data-manager/action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "apply" }, + }); + expect((await finished(next.json().jobId)).status).toBe("success"); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 96 * 3600000); + try { + let retired = 0; + for (let step = 0; step < 12; step++) + retired += (await reconcileControlRetention(installation, { maxFamilies: 64 })).retired; + expect(existsSync(pullPath)).toBe(false); + expect(retired).toBeGreaterThan(0); + const jobs = JSON.parse( + readFileSync(join(host.infraDir, "data/ops-agent/jobs-v1.json"), "utf8"), + ).jobs as { operationKey: string }[]; + expect(jobs.some((job) => pullPath.endsWith(`/${job.operationKey}.json`))).toBe(false); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(true); + } finally { + vi.useRealTimers(); + } + }, 180000); + it("refuses candidate DM staging before Docker while an actual ordinary full-target root remains held", async () => { + const reached = Promise.withResolvers(), + release = Promise.withResolvers(); + beforeLifecycleProcess = async (args) => { + if (args.includes("up") && args.at(-1) === "motis" && !args.includes("--force-recreate")) { + reached.resolve(); + await release.promise; + } + }; + const started = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "start", serviceIds: ["motis"] }, + }), + startId = started.json().jobId; + try { + await reached.promise; + const [live] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, startId)), + root = Object.values( + ( + live!.result as { + opsProjection: { + operations: Record; + }; + } + ).opsProjection.operations, + ).find((operation) => operation.kind === "service.start")!, + lease = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/execution-leases", + `${root.operationKey}.json`, + ), + "utf8", + ), + ), + owner = JSON.parse( + readFileSync( + join(host.infraDir, "data/.admin-target-control/leases", lease.leaseId, "owner.json"), + "utf8", + ), + ); + expect(owner.payload.certificate.payload.serviceIds).toContain("data-manager"); + const request = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + stopped = await finished(request.json().jobId); + expect(stopped.status).toBe("failed"); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + } finally { + beforeLifecycleProcess = undefined; + release.resolve(); + expect((await finished(startId)).status).toBe("success"); + } + const repaired = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(repaired.json().jobId)).status).toBe("success"); + }, 240000); + it("observes its genuine pull admission without target metadata writes and rejects replacement of its indexed inode", async () => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + bytes = readFileSync(pullPath), + actual = JSON.parse(bytes.toString("utf8")).payload, + authority = restoreAdmittedControlExecution(installation, actual.execution), + snapshot = () => + Object.fromEntries( + ["data/.admin-target-control", "control-private/ops-agent/observations"].flatMap( + (base) => { + const directory = join(host.infraDir, base); + return readdirSync(directory, { recursive: true }) + .map(String) + .sort() + .map((name) => { + const path = join(directory, name), + stat = statSync(path); + return [ + `${base}/${name}`, + `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(path)).digest("hex") : "directory"}`, + ]; + }); + }, + ), + ); + expect(existsSync(join(host.infraDir, "data/.admin-target-control/admission-index-v1"))).toBe( + false, + ); + const before = snapshot(), + writes = vi.spyOn(filesystem, "mkdirSync"); + syncBuiltinESMExports(); + try { + assertCoordinatorPullLeaseCompatibility(authority); + expect(writes).not.toHaveBeenCalled(); + } finally { + writes.mockRestore(); + syncBuiltinESMExports(); + } + expect(snapshot()).toEqual(before); + renameSync(pullPath, `${pullPath}.original-indexed-inode`); + writeFileSync(pullPath, bytes, { mode: 0o600 }); + expect(() => assertCoordinatorPullLeaseCompatibility(authority)).toThrow(); + }, 180000); + it("refuses actual candidate pull before Docker against a separately admitted physical host-command root", async () => { + // Component competitor uses the real normal host issuer and physical + // target producer. The candidate/pull side is the actual public API, + // production registry/server/journal and owned PostgreSQL throughout. + const hostInstallation = openControlInstallation({ + infraDir: host.infraDir, + role: "host-cli", + serviceIds: host.loaded.map((service) => service.manifest.id), + }), + admittedAt = Date.now(), + deadlineAt = admittedAt + 15000, + operation = { + kind: "service.start" as const, + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + hash = opsOperationFingerprint(operation), + execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint: hash, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: hash, + deadlineAt, + }), + }, + authority = retainAdmittedControlExecution(hostInstallation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: host.loaded.map((service) => service.manifest.id).sort(), + }), + lease = await acquireArtifactTargetLease({ + dataRoot: join(host.infraDir, "data"), + owner: authority, + serviceIds: host.loaded.map((service) => service.manifest.id).sort(), + }); + expect(lease.serviceIds).toContain("data-manager"); + let pullObserver: ReturnType | undefined; + const snapshot = () => + Object.fromEntries( + [ + "data/.admin-target-control/locks", + "data/.admin-target-control/leases", + "data/.admin-target-control/admission-index-v1", + "control-private/ops-agent/observations", + ].flatMap((base) => { + const directory = join(host.infraDir, base); + return readdirSync(directory, { recursive: true }) + .map(String) + .sort() + .map((name) => { + const path = join(directory, name), + stat = statSync(path); + return [ + `${base}/${name}`, + `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(path)).digest("hex") : "directory"}`, + ]; + }); + }), + ); + // Make a forbidden Docker dispatch conclusively observable without + // progressing to the separately held maintenance target. This controls + // only the external process edge; the actual admission remains intact. + host.setImagePullProcess(async (_reference, signal) => { + await runContainedProcess(process.execPath, ["-e", "process.exit(17)"], { + signal, + timeout: 15000, + maxBuffer: 1024, + }); + }); + try { + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + row = await finished(next.json().jobId); + console.info( + "actual held-host-root/pull", + JSON.stringify({ + state: row.status, + error: row.error, + posts: [...posts], + imagePulls: host.calls.filter((args) => args[0] === "image" && args[1] === "pull") + .length, + }), + ); + expect(row.status).toBe("failed"); + expect(posts.get("service.pull")).toBe(1); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + const directory = join(host.infraDir, "control-private/ops-agent/executions"), + actualPulls = readdirSync(directory) + .filter((name) => name.endsWith(".json")) + .map((name) => JSON.parse(readFileSync(join(directory, name), "utf8"))) + .filter((raw) => raw.payload?.operation?.kind === "service.pull"); + expect(actualPulls).toHaveLength(1); + const observer = restoreAdmittedControlExecution( + installation, + actualPulls[0].payload.execution, + ); + pullObserver = observer; + const journal = join( + host.infraDir, + "data/.admin-target-control/leases", + lease.leaseId, + ".journal-writer", + ); + mkdirSync(journal, { mode: 0o700 }); + try { + const before = snapshot(), + started = performance.now(); + expect(() => assertCoordinatorPullLeaseCompatibility(observer)).toThrow(); + expect(performance.now() - started).toBeLessThan(5000); + expect(snapshot()).toEqual(before); + } finally { + rmdirSync(journal); + } + await delay(Math.max(0, deadlineAt - Date.now() + 25)); + expect(Date.now()).toBeGreaterThan(deadlineAt); + const expired = snapshot(); + expect(() => assertCoordinatorPullLeaseCompatibility(observer)).toThrow(); + expect(snapshot()).toEqual(expired); + } finally { + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(hostInstallation, execution), + ); + await releaseArtifactTargetLease(lease); + closeControlInstallation(hostInstallation); + } + if (!pullObserver) throw Error("Actual original normal pull source required"); + const releasedObserver = pullObserver; + const released = snapshot(); + assertCoordinatorPullLeaseCompatibility(pullObserver); + expect(snapshot()).toEqual(released); + const slot = join( + host.infraDir, + "data/.admin-target-control/admission-index-v1/active/slot-255", + ); + mkdirSync(slot, { mode: 0o700 }); + writeFileSync(join(slot, "generation.json"), JSON.stringify({ generation: randomUUID() }), { + mode: 0o600, + }); + try { + const partial = snapshot(); + expect(() => assertCoordinatorPullLeaseCompatibility(releasedObserver)).toThrow(); + expect(snapshot()).toEqual(partial); + } finally { + rmSync(slot, { recursive: true }); + } + // A second independent component root holds only a certified unrelated + // target. Its body/clock/certificate are issued by the real normal host + // producer; this is kernel observation, not a native CLI-effect claim. + const unrelatedInstallation = openControlInstallation({ + infraDir: host.infraDir, + role: "host-cli", + serviceIds: host.loaded.map((service) => service.manifest.id), + }), + unrelatedAt = Date.now(), + unrelatedDeadline = unrelatedAt + 120000, + unrelatedOperation = { + kind: "service.start" as const, + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + unrelatedHash = opsOperationFingerprint(unrelatedOperation), + unrelatedExecution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ + notAfter: unrelatedDeadline, + nonce: randomBytes(16), + }), + operationFingerprint: unrelatedHash, + deadlineAt: unrelatedDeadline, + fingerprint: executionFingerprint({ + kind: unrelatedOperation.kind, + operationFingerprint: unrelatedHash, + deadlineAt: unrelatedDeadline, + }), + }, + unrelatedAuthority = retainAdmittedControlExecution(unrelatedInstallation, { + execution: unrelatedExecution, + operation: unrelatedOperation, + kind: unrelatedOperation.kind, + admittedAt: unrelatedAt, + requestedDeadlineAt: unrelatedDeadline, + serviceIds: ["motis"], + }), + unrelatedLease = await acquireArtifactTargetLease({ + dataRoot: join(host.infraDir, "data"), + owner: unrelatedAuthority, + serviceIds: ["motis"], + }); + try { + const before = snapshot(); + assertCoordinatorPullLeaseCompatibility(releasedObserver); + expect(snapshot()).toEqual(before); + } finally { + retainControlEffectReceipt(unrelatedAuthority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await settleArtifactTargetLeaseChild( + unrelatedLease, + verifyControlEffectSettlement(unrelatedInstallation, unrelatedExecution), + ); + await releaseArtifactTargetLease(unrelatedLease); + closeControlInstallation(unrelatedInstallation); + } + console.info( + "actual target-index observation", + JSON.stringify({ + genuineNormalBaseline: true, + readonlyHeldBusy: true, + actualExpiredStillBusy: true, + actualReleasedAccepted: true, + partialSlotRefused: true, + unrelatedSignedTargetAccepted: true, + }), + ); + host.setImagePullProcess(async (reference) => { + pulledImageId = `sha256:${"e".repeat(64)}`; + host.setImageIdentity(reference, pulledImageId); + }); + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(next.json().jobId)).status).toBe("success"); + }, 240000); + it.each(["capability", "admission"])( + "keeps bare coordinator pull unavailable at its actual %s boundary", + async (boundary) => { + const { createApiOpsClient } = await import("../ops-client"), + client = createApiOpsClient(); + if (boundary === "capability") { + const status = await client.execute({ kind: "services.inspect" }); + if (status.execution !== "sync") throw Error("Actual service capabilities required"); + expect( + status.value.services.find((service) => service.serviceId === "data-manager")?.actions + .pull, + ).toMatchObject({ + available: false, + execution: "direct", + reason: "unavailable", + workflow: null, + }); + return; + } + host.setImagePullProcess(async (_reference, signal) => { + await runContainedProcess(process.execPath, ["-e", "process.exit(17)"], { + signal, + timeout: 15000, + maxBuffer: 1024, + }); + }); + const deadlineAt = Date.now() + 180000, + operationKey = createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }); + let failure: unknown, + admitted = false; + try { + const result = await client.execute( + { + kind: "service.pull", + serviceId: "data-manager", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + { operationKey, execution: { requestedDeadlineAt: deadlineAt } }, + ); + if (result.execution === "async") { + admitted = true; + const until = performance.now() + 30000; + while ( + !["failed", "succeeded", "timed_out"].includes( + (await client.status("service.pull", result.operationId)).state, + ) + ) { + if (performance.now() > until) throw Error("Actual bare pull did not settle"); + await delay(25); + } + } + } catch (error) { + failure = error; + } + console.info( + "actual bare DM pull", + JSON.stringify({ + admitted, + imagePulls: host.calls.filter((args) => args[0] === "image" && args[1] === "pull") + .length, + }), + ); + expect(failure).toMatchObject({ errorClass: "configuration_prerequisite" }); + expect(admitted).toBe(false); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect( + existsSync( + join(host.infraDir, "control-private/ops-agent/executions", `${operationKey}.json`), + ), + ).toBe(false); + }, + 90000, + ); + it("settles only the original unstarted normal CLI Update root when staged configuration is required", async () => { + await canceledSuccessfulCoordinatorStage(); + const { createApiOpsClient } = await import("../ops-client"), + client = createApiOpsClient(), + before = services.currentConfigurationGeneration(host.infraDir), + deadlineAt = Date.now() + 180000, + operationKey = createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + submitted = await client.execute( + { kind: "service.update", serviceId: "motis", expectedConfigurationRevisionId: before! }, + { operationKey, execution: { requestedDeadlineAt: deadlineAt } }, + ); + if (submitted.execution !== "async") throw Error("Actual normal CLI admission required"); + const until = performance.now() + 90000; + let terminal = await client.status("service.update", submitted.operationId); + while (!["succeeded", "failed", "timed_out"].includes(terminal.state)) { + if (performance.now() > until) throw Error("Actual managed CLI did not settle"); + await delay(25); + terminal = await client.status("service.update", submitted.operationId); + } + console.info( + "actual staged CLI pre-effect", + JSON.stringify({ + state: terminal.state, + errorClass: terminal.errorClass, + nativeCalls: existsSync(nativeTrace) + ? readFileSync(nativeTrace, "utf8").trim().split("\n").filter(Boolean).length + : 0, + currentUnchanged: services.currentConfigurationGeneration(host.infraDir) === before, + }), + ); + expect(terminal.state).toBe("failed"); + expect(terminal.errorClass).toBe("configuration_prerequisite"); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); + const native = existsSync(nativeTrace) + ? readFileSync(nativeTrace, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as string[]) + : []; + expect( + native.filter((args) => args.includes("up") || (args[0] === "image" && args[1] === "pull")), + ).toHaveLength(0); + const privateRoot = join(host.infraDir, "control-private/ops-agent"), + admitted = JSON.parse( + readFileSync(join(privateRoot, "executions", `${operationKey}.json`), "utf8"), + ).payload, + effect = JSON.parse( + readFileSync(join(privateRoot, "effects", `${operationKey}.json`), "utf8"), + ).payload; + expect( + JSON.parse( + readFileSync(join(privateRoot, "effect-started", `${operationKey}.json`), "utf8"), + ), + ).toBeNull(); + expect(effect.receipt.kind).toBe("not-started"); + expect(effect.outcome).toBe("failed_without_effect"); + const originalRoot = restoreAdmittedControlExecution(installation, admitted.execution); + expect(lookupControlChildPhase(originalRoot, { phase: "lifecycle", ordinal: 0 })).toBeNull(); + verifyRetainedArtifactTargetRootRelease(originalRoot); + const repaired = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(repaired.json().jobId)).status).toBe("success"); + }, 240000); + it("refuses a normal ordinary root admitted DURING actual image pull before publication or CLI dispatch", async () => { + const reached = Promise.withResolvers(), + release = Promise.withResolvers(), + terminalReached = Promise.withResolvers(), + terminalRelease = Promise.withResolvers(); + pullTerminalReplyGate = { + reached: () => terminalReached.resolve(), + release: terminalRelease.promise, + }; + host.setImagePullProcess(async (reference) => { + pulledImageId = `sha256:${"e".repeat(64)}`; + host.setImageIdentity(reference, pulledImageId); + reached.resolve(); + await release.promise; + }); + const { createApiOpsClient } = await import("../ops-client"), + client = createApiOpsClient(); + let id: string | undefined, + completed = false; + try { + const response = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const jobId: unknown = response.json().jobId; + if (typeof jobId !== "string") throw Error("Actual coordinator job required"); + id = jobId; + await reached.promise; + const deadlineAt = Date.now() + 120000, + operationKey = createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + admitted = await client.execute( + { + kind: "service.update", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + { operationKey, execution: { requestedDeadlineAt: deadlineAt } }, + ); + if (admitted.execution !== "async") + throw Error("Actual normal managed CLI admission required"); + const source = JSON.parse( + readFileSync( + join(host.infraDir, "control-private/ops-agent/executions", `${operationKey}.json`), + "utf8", + ), + ).payload; + expect( + JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/effect-started", + `${operationKey}.json`, + ), + "utf8", + ), + ), + ).toBeNull(); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + release.resolve(); + await terminalReached.promise; + let state = await client.status("service.update", admitted.operationId); + const until = performance.now() + 45000; + while (!["failed", "succeeded", "timed_out"].includes(state.state)) { + if (performance.now() > until) throw Error("Actual racing CLI did not settle"); + await delay(25); + state = await client.status("service.update", admitted.operationId); + } + expect(state).toMatchObject({ state: "failed", errorClass: "configuration_prerequisite" }); + const authority = restoreAdmittedControlExecution(installation, source.execution); + expect(lookupControlChildPhase(authority, { phase: "lifecycle", ordinal: 0 })).toBeNull(); + verifyRetainedArtifactTargetRootRelease(authority); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + terminalRelease.resolve(); + expect((await finished(jobId)).status).toBe("success"); + completed = true; + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(dm.settlementResponseStatuses).toEqual([200]); + } finally { + release.resolve(); + terminalRelease.resolve(); + pullTerminalReplyGate = undefined; + if (id && !completed) await finished(id); + } + }, 240000); + it.each([false, true])( + "preserves pending staged bulk request order with coordinatorFirst=%s", + async (coordinatorFirst) => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + original = readFileSync(pullPath), + before = services.currentConfigurationGeneration(host.infraDir), + serviceIds = coordinatorFirst ? ["data-manager", "motis"] : ["motis", "data-manager"], + response = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds }, + }), + originalId = response.json().jobId, + row = await finished(originalId); + console.info( + "actual pending staged order", + JSON.stringify({ + serviceIds, + state: row.status, + error: row.error, + posts: [...posts], + completed: (row.result as { completedServiceIds?: string[] })?.completedServiceIds, + }), + ); + if (coordinatorFirst) { + expect(row.status, row.error ?? "No public error").toBe("success"); + expect((row.result as { completedServiceIds: string[] }).completedServiceIds).toEqual( + serviceIds, + ); + expect(posts.get("service.pull")).toBe(2); + expect(posts.get("service.update")).toBe(1); + } else { + expect(row.status).toBe("failed"); + expect(row.error).toBe( + "Apply or update Data Manager before changing instance configuration.", + ); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("service.update") ?? 0).toBe(0); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + expect(readFileSync(pullPath)).toEqual(original); + const repair = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(repair.json().jobId)).status).toBe("success"); + const retry = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds }, + }), + retried = await finished(retry.json().jobId); + expect(retried.status, retried.error ?? "No public error").toBe("success"); + expect((retried.result as { completedServiceIds: string[] }).completedServiceIds).toEqual( + serviceIds, + ); + expect((await finished(originalId)).status).toBe("failed"); + expect((await finished(originalId)).error).toBe(row.error); + } + expect(services.readAppliedServiceIds(host.infraDir, host.loaded)).toEqual( + new Set(["data-manager", "motis"]), + ); + }, + 420000, + ); + it.each(["held", "unwound"] as const)( + "observes native pre-effect first-unlink SIGKILL with journal guard %s", + async (guard) => { + await canceledSuccessfulCoordinatorStage(); + const first = await restartNativeStagedAgent({ stopAfterFirstTargetRelease: guard }), + { createApiOpsClient } = await import("../ops-client"), + client = createApiOpsClient(), + deadlineAt = Date.now() + 120000, + operation = { + kind: "service.update" as const, + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + operationKey = createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + request = client + .execute(operation, { operationKey, execution: { requestedDeadlineAt: deadlineAt } }) + .catch(() => undefined); + try { + await Promise.race([ + first.fault, + delay(45000).then(() => { + throw Error("Actual native release fault not reached"); + }), + ]); + const privateRoot = join(host.infraDir, "control-private/ops-agent"), + admissionPath = join(privateRoot, "executions", `${operationKey}.json`), + admissionBytes = readFileSync(admissionPath), + source = JSON.parse(admissionBytes.toString("utf8")).payload, + effect = JSON.parse( + readFileSync(join(privateRoot, "effects", `${operationKey}.json`), "utf8"), + ).payload, + terminal = JSON.parse( + readFileSync( + join(privateRoot, "observations", `terminal-${operationKey}.json`), + "utf8", + ), + ).payload; + expect(effect).toMatchObject({ + identity: source.execution, + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/app-api"))).toBe( + false, + ); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + const leaseRef = JSON.parse( + readFileSync(join(privateRoot, "execution-leases", `${operationKey}.json`), "utf8"), + ), + journalGuard = join( + host.infraDir, + "data/.admin-target-control/leases", + leaseRef.leaseId, + ".journal-writer", + ); + expect(existsSync(journalGuard)).toBe(guard === "held"); + console.info( + "actual native pre-effect kill", + JSON.stringify({ + terminalPhase: terminal.phase, + originalDeadline: source.execution.deadlineAt, + firstTargetAbsent: true, + dmStillHeld: true, + }), + ); + await first.killAfterFault(); + await request; + await first(); + const next = await restartNativeStagedAgent(); + try { + const observed = await createApiOpsClient() + .lookup(operation, operationKey, { + execution: { requestedDeadlineAt: deadlineAt }, + fingerprint: source.execution.fingerprint, + }) + .catch((error: unknown) => { + if ( + guard === "held" && + error instanceof Error && + "errorClass" in error && + error.errorClass === "recovery_required" + ) + return undefined; + throw error; + }); + if (guard === "held") { + if (observed) expect(observed.errorClass).toBe("recovery_required"); + expect(existsSync(journalGuard)).toBe(true); + expect( + existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), + ).toBe(true); + expect(readFileSync(admissionPath)).toEqual(admissionBytes); + return; + } + expect(observed).toBeDefined(); + if (!observed) throw Error("Missing original pre-effect failure"); + console.info( + "actual native pre-effect restored", + JSON.stringify({ + state: observed.state, + errorClass: observed.errorClass, + originalDeadlinePreserved: observed.deadlineAt === source.execution.deadlineAt, + }), + ); + expect(observed).toMatchObject({ + state: "failed", + errorClass: "configuration_prerequisite", + operationId: source.execution.executionId, + deadlineAt: source.execution.deadlineAt, + fingerprint: source.execution.fingerprint, + }); + expect(observed.updatedAt).toBe(new Date(terminal.terminalAt).toISOString()); + expect(readFileSync(admissionPath)).toEqual(admissionBytes); + verifyRetainedArtifactTargetRootRelease( + restoreAdmittedControlExecution(installation, source.execution), + ); + const repaired = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(repaired.json().jobId)).status).toBe("success"); + } finally { + await next(); + } + } finally { + await first(); + await request; + } + }, + 240000, + ); + it("refuses an ordinary unrelated configuration write until explicit guarded DM repair and a NEW request", async () => { + await fixture.db + .insert(schema.serviceConfig) + .values({ id: crypto.randomUUID(), serviceId: "motis", config: { MODE: "old" } }); + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + original = readFileSync(pullPath), + before = services.currentConfigurationGeneration(host.infraDir), + rows = await fixture.sql`SELECT id,config,xmin::text AS version FROM service_config`, + request = { + method: "POST", + url: "/api/admin/services/motis/config" as const, + headers: { + "x-openmapx-local-admin": "task012-coordinator-local-admin", + "idempotency-key": "task012-unrelated-config-after-staging", + }, + payload: { config: { MODE: "new" } }, + } as const; + const saved = await api.inject(request); + expect(saved.statusCode, saved.body).toBe(409); + expect(saved.json()).toEqual({ + error: "Apply or update Data Manager before changing instance configuration.", + }); + expect(await fixture.sql`SELECT id,config,xmin::text AS version FROM service_config`).toEqual( + rows, + ); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); + expect(readFileSync(pullPath).equals(original)).toBe(true); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + expect(posts.get("serviceConfig.apply") ?? 0).toBe(0); + expect(posts.get("stack.render") ?? 0).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + row = await finished(next.json().jobId); + expect(row.status, row.error ?? "No public error").toBe("success"); + expect(posts.get("service.pull")).toBe(2); + expect((await api.inject(request)).statusCode).toBe(409); + const retried = await api.inject({ + ...request, + headers: { ...request.headers, "idempotency-key": "task012-unrelated-config-after-repair" }, + }); + expect(retried.statusCode, retried.body).toBe(200); + expect((await fixture.db.select().from(schema.serviceConfig))[0]?.config).toEqual({ + MODE: "new", + }); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + }, 180000); + it.each([ + "container-id", + "image-id", + "reference", + "old-compose-inode", + "current-generation", + "ambiguous-duplicate", + ] as const)( + "refuses staged provenance after actual %s drift", + async (kind) => { + await canceledSuccessfulCoordinatorStage(); + if (kind === "old-compose-inode") { + const file = services.configurationGenerationComposeFile( + host.infraDir, + host.initial.revisionId, + ), + bytes = readFileSync(file); + renameSync(file, `${file}.fixture-original`); + writeFileSync(file, bytes, { mode: 0o600, flag: "wx" }); + } else if (kind === "current-generation") { + const owned = await host.owned.inspectOwnedServices(new AbortController().signal); + await services.commitConfigurationGeneration({ + infraDir: host.infraDir, + services: host.loaded, + input: { + domain: "competing.example.test", + selectedRoots: ["data-manager", "motis"], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + ownedServiceGenerations: owned.map((service) => ({ + containerId: service.containerId, + revisionIds: service.provenance.composeConfigFiles + .map((file) => dirname(file).split("/").at(-1)!) + .filter((id) => /^cfg1_[A-Za-z0-9_-]{43}$/.test(id)), + })), + }); + } else { + const rows = ( + await host.execFile("docker", ["container", "inspect", "--format", "owned-snapshot"], { + signal: new AbortController().signal, + timeout: 15000, + maxBuffer: 1024 * 1024, + }) + ).stdout + .trim() + .split("\n") + .map((row) => JSON.parse(row)); + const dm = rows.find( + (row) => row.Labels["com.docker.compose.service"] === "data-manager", + ); + if (kind === "container-id") dm.Id = "f".repeat(64); + if (kind === "image-id") dm.Image = `sha256:${"f".repeat(64)}`; + if (kind === "reference") dm.ConfigImage = "example/unrelated:1"; + if (kind === "ambiguous-duplicate") { + const extra = structuredClone(dm); + extra.Id = "f".repeat(64); + extra.Image = pulledImageId; + dm.Labels["com.docker.compose.project.config_files"] = + "/tmp/untrusted-staged-compose.yml"; + rows.push(extra); + } + host.setRows(rows); + } + let observed: { serviceId: string }[] | undefined; + try { + observed = await host.owned.inspectOwnedServices(new AbortController().signal); + } catch (error) { + expect((error as Error).message).toBe("Owned service inventory unavailable"); + } + expect(observed?.some((service) => service.serviceId === "data-manager") ?? false).toBe( + false, + ); + }, + 120000, + ); + it("uses staged OLD stopped provenance for one NEW non-recreate helper BEFORE its own pull", async () => { + await canceledSuccessfulCoordinatorStage(); + host.setServiceState("data-manager", "exited"); + await dm.pauseHttp(); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).find( + (service) => service.serviceId === "data-manager", + )?.state, + ).toBe("stopped"); + let beforePull: { starts: number; helperUps: number } | undefined; + afterImageStaging = () => { + beforePull = { + starts: posts.get("service.start") ?? 0, + helperUps: host.calls.filter( + (args) => args.includes("--no-recreate") && args.at(-1) === "data-manager", + ).length, + }; + }; + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect((await finished(next.json().jobId)).status).toBe("success"); + expect(beforePull).toEqual({ starts: 1, helperUps: 1 }); + expect(posts.get("service.start")).toBe(1); + expect(posts.get("service.pull")).toBe(2); + expect(dm.settlementResponseStatuses).toEqual([200]); + }, 180000); + it("retains the LAST source for a currently stopped OLD container without reporting historical running state", async () => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(); + host.setServiceState("data-manager", "exited"); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 96 * 3600000); + try { + for (let pass = 0; pass < 4; pass++) + await reconcileControlRetention(installation, { maxFamilies: 64 }); + expect(existsSync(pullPath)).toBe(true); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).find( + (service) => service.serviceId === "data-manager", + )?.state, + ).toBe("stopped"); + } finally { + vi.useRealTimers(); + } + }, 120000); + it("vetoes lower-ranked source GC while actual Docker observation is unavailable, then rechecks its real successor", async () => { + const first = await canceledSuccessfulCoordinatorStage(), + second = await canceledSuccessfulCoordinatorStage(); + let failures = 0; + const unavailable = async () => { + failures++; + host.setInventoryProcess(unavailable); + throw Error("Named Docker inventory unavailable"); + }; + host.setInventoryProcess(unavailable); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 96 * 3600000); + try { + for (let pass = 0; pass < 4; pass++) + await reconcileControlRetention(installation, { maxFamilies: 64 }); + expect(failures).toBeGreaterThan(0); + expect(existsSync(first.pullPath)).toBe(true); + expect(existsSync(second.pullPath)).toBe(true); + host.setInventoryProcess(undefined); + for (let pass = 0; pass < 8; pass++) + await reconcileControlRetention(installation, { maxFamilies: 64 }); + expect(existsSync(first.pullPath)).toBe(false); + expect(existsSync(second.pullPath)).toBe(true); + } finally { + host.setInventoryProcess(undefined); + vi.useRealTimers(); + } + }, 180000); + it.each(["admission", "canonical", "effect"] as const)( + "refuses a byte-identical %s source copied onto a different private indexed inode", + async (kind) => { + const { pullPath } = await canceledSuccessfulCoordinatorStage(), + key = pullPath.split("/").at(-1)!, + path = + kind === "admission" + ? pullPath + : join( + host.infraDir, + "control-private/ops-agent", + kind === "canonical" ? "observations" : "effects", + kind === "canonical" ? `terminal-${key}` : key, + ), + original = readFileSync(path), + backup = `${path}.fixture-original`; + renameSync(path, backup); + writeFileSync(path, original, { mode: 0o600, flag: "wx" }); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(false); + }, + 120000, + ); + it("retires ONLY the lower-ranked authenticated F family while the actual G source remains pinned", async () => { + const first = await canceledSuccessfulCoordinatorStage(), + second = await canceledSuccessfulCoordinatorStage(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 96 * 3600000); + try { + for (let step = 0; step < 12; step++) + await reconcileControlRetention(installation, { maxFamilies: 64 }); + expect(existsSync(first.pullPath)).toBe(false); + expect(existsSync(second.pullPath)).toBe(true); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(true); + } finally { + vi.useRealTimers(); + } + }, 180000); + it("authenticates signed F-to-G retirement and NATIVE restart after first actual private unlink", async () => { + const first = await canceledSuccessfulCoordinatorStage(), + second = await canceledSuccessfulCoordinatorStage(); + const raw = JSON.parse(readFileSync(first.pullPath, "utf8")), + originHash = controlDigest(raw.payload.origin), + privateRoot = join(host.infraDir, "control-private/ops-agent"), + location = JSON.parse( + readFileSync(join(privateRoot, "retention-locations", `${originHash}.json`), "utf8"), + ), + slot = join(privateRoot, "retention-families", location.name), + io = { signal: new AbortController().signal, timeout: 15000, maxBuffer: 1024 * 1024 }; + const rows = ( + await host.execFile("docker", ["container", "inspect", "--format", "owned-snapshot"], io) + ).stdout + .trim() + .split("\n") + .map((row) => JSON.parse(row)); + const images: Record = {}; + for (const row of rows) + images[row.ConfigImage] = ( + await host.execFile( + "docker", + ["image", "inspect", "--format", "{{.Id}}", row.ConfigImage], + io, + ) + ).stdout.trim(); + const directory = process.env.OPENMAPX_TEST_EVIDENCE_DIR ?? host.cleanupRoot; + const packetsDirectory = join(directory, "staged-retirement-signed-packets"); + mkdirSync(packetsDirectory, { recursive: true }); + const evidenceFile = join(packetsDirectory, `${raw.payload.execution.operationKey}.json`); + await agent.close(); + const base = { + root: host.root, + infraDir: host.infraDir, + stack: host.stack, + queue: host.queue, + token: host.token, + serviceIds: host.loaded.map((service) => service.manifest.id), + now: Date.now() + 96 * 3600000, + rows, + images, + firstPullPath: first.pullPath, + secondPullPath: second.pullPath, + firstSlot: slot, + evidenceFile, + }; + const start = (mode: "fault" | "resume") => { + const argv = [ + "--import", + createRequire(import.meta.url).resolve("tsx/esm"), + fileURLToPath( + new URL( + "../../../../ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs", + import.meta.url, + ), + ), + ]; + console.info("actual native staged GC argv", JSON.stringify([process.execPath, ...argv])); + const child = spawn(process.execPath, argv, { + cwd: process.cwd(), + env: process.env, + stdio: ["pipe", "pipe", "pipe"], + }); + let output = "", + stderr = ""; + const boundary = Promise.withResolvers>(); + child.stdout.on("data", (chunk) => { + output += chunk.toString(); + for (const line of output.split("\n")) { + try { + const value = JSON.parse(line); + if (value.boundary) boundary.resolve(value); + } catch {} + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + if (stderr.length > 1024 * 1024) child.kill("SIGKILL"); + }); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve) => + child.once("exit", (code, signal) => { + if (!output.includes('"boundary"')) + boundary.reject(Error(`Native staged GC exited ${code}: ${stderr.slice(-2500)}`)); + resolve({ code, signal }); + }), + ); + child.stdin.end(JSON.stringify({ ...base, mode })); + return { child, boundary: boundary.promise, exited }; + }; + // An active source with a valid successor still cannot retire from + // missing/corrupt original signed bytes. Restore only the exact fixture + // bytes after proving refusal; production supplies no corruption bypass. + const originalAdmission = readFileSync(first.pullPath), + brokenAdmission = JSON.parse(originalAdmission.toString()); + brokenAdmission.signature = + (brokenAdmission.signature[0] === "A" ? "B" : "A") + brokenAdmission.signature.slice(1); + writeFileSync(first.pullPath, JSON.stringify(brokenAdmission)); + try { + const blocked = start("resume"), + observed = await blocked.boundary; + expect(await blocked.exited).toEqual({ code: 0, signal: null }); + expect(observed).toMatchObject({ + Fpresent: true, + Gpresent: true, + FslotPresent: true, + ordinaryOwnedDM: true, + }); + expect(observed.journalKeys as string[]).toContain(raw.payload.execution.operationKey); + } finally { + writeFileSync(first.pullPath, originalAdmission); + } + const crashed = start("fault"); + try { + const actual = await crashed.boundary; + expect(actual).toMatchObject({ + boundary: "actual-first-F-private-unlink", + faulted: true, + Fpresent: false, + Gpresent: true, + FslotPresent: true, + }); + expect(actual.journalKeys as string[]).toContain(raw.payload.execution.operationKey); + crashed.child.kill("SIGKILL"); + expect((await crashed.exited).signal).toBe("SIGKILL"); + const packets = JSON.parse(readFileSync(evidenceFile, "utf8")), + before = packets.candidate.payload.stagedCoordinatorDependency, + after = packets.progress.stagedCoordinatorRevalidation; + expect(packets.publicKey).toBe(installation.publicKey); + verifyControl( + packets.publicKey, + stagedCoordinatorRetirementDomain, + before, + stagedCoordinatorRetirementPayloadSchema, + ); + verifyControl( + packets.publicKey, + stagedCoordinatorRetirementDomain, + after, + stagedCoordinatorRetirementPayloadSchema, + ); + expect(before.payload.condition).toBe("higher-ranked-successor"); + expect(after.payload.condition).toBe("higher-ranked-successor"); + expect(Buffer.byteLength(JSON.stringify(before))).toBeLessThanOrEqual(4096); + expect(Buffer.byteLength(JSON.stringify(after))).toBeLessThanOrEqual(4096); + expect(packets.manifest.payload.stagedCoordinatorDependencyDigest).toBe( + controlDigest(before), + ); + expect(packets.decision.payload.stagedCoordinatorRevalidationDigest).toBe( + controlDigest(after), + ); + expect(after.payload.manifestDigest).toBe(controlDigest(packets.manifest.payload)); + expect(after.payload.checkNonce).not.toBe(before.payload.checkNonce); + expect(() => + controlRetentionPreparationState({ + ...after.payload, + mode: "committed-cleanup", + } as never), + ).toThrow(); + expect(() => + recordStagedCoordinatorRetentionCheck({ ...after.payload } as never, after), + ).toThrow(); + const proofPath = join(slot, "proof.json"), + originalProof = readFileSync(proofPath), + brokenProof = JSON.parse(originalProof.toString()); + brokenProof.signature = + (brokenProof.signature[0] === "A" ? "B" : "A") + brokenProof.signature.slice(1); + writeFileSync(proofPath, JSON.stringify(brokenProof)); + try { + const blocked = start("resume"), + observed = await blocked.boundary; + expect(await blocked.exited).toEqual({ code: 0, signal: null }); + expect(observed).toMatchObject({ + Fpresent: false, + Gpresent: true, + FslotPresent: true, + ordinaryOwnedDM: true, + }); + expect(observed.journalKeys as string[]).toContain(raw.payload.execution.operationKey); + } finally { + writeFileSync(proofPath, originalProof); + } + const replacement = start("resume"), + observed = await replacement.boundary; + expect(await replacement.exited).toEqual({ code: 0, signal: null }); + expect(observed).toMatchObject({ + Fpresent: false, + Gpresent: true, + FslotPresent: false, + ordinaryOwnedDM: true, + }); + expect(observed.journalKeys as string[]).not.toContain(raw.payload.execution.operationKey); + } finally { + crashed.child.kill("SIGKILL"); + await crashed.exited; + } + }, 180000); + it("guards the actual existing coordinator Apply entry even with empty links", async () => { + const response = await api.inject({ + method: "POST", + url: "/api/admin/services/data-manager/action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "apply" }, + }); + expect(response.statusCode).toBe(200); + const row = await finished(response.json().jobId); + expect( + row.status, + `${row.error ?? "no job error"}; posted kinds ${JSON.stringify([...posts])}`, + ).toBe("success"); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(1); + expect(dm.preparePosts).toBe(1); + expect(posts.get("configurationMaintenance.decide")).toBe(1); + expect(posts.get("configurationMaintenance.settle")).toBe(1); + expect(host.calls.filter((a) => a.includes("--force-recreate"))).toHaveLength(1); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual([]); + expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); + expect(posts.get("service.start") ?? 0).toBe(0); + }, 180000); + it("guards current local-admin coordinator Update without an ordinary seed or fixed CLI", async () => { + const response = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect(response.statusCode).toBe(200); + const row = await finished(response.json().jobId); + expect( + row.status, + `${row.error ?? "no job error"}; posted kinds ${JSON.stringify([...posts])}`, + ).toBe("success"); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(1); + expect(dm.preparePosts).toBe(1); + expect(posts.get("service.update") ?? 0).toBe(0); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("stack.render")).toBe(1); + expect(posts.get("service.start") ?? 0).toBe(0); + const checkpoint = row.result as { + _configurationMaintenance: { receipt: { binding: { maintenanceId: string } } }; + }, + owner = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private", + "ops-agent", + "configuration-maintenance", + `${checkpoint._configurationMaintenance.receipt.binding.maintenanceId}.json`, + ), + "utf8", + ), + ).payload; + expect(owner.coordinatorPull.payload.imageId).toBe(pulledImageId); + expect(owner.coordinatorPull.payload.version).toBe(2); + expect(owner.receipt.state).toBe("settled"); + expect(owner.final.ownerFinal.payload.outcome).toBe("applied"); + expect(owner.apiControls.settle.result.state).toBe("settled"); + expect(owner.apiControls.settle.receiptDelivery).not.toBeNull(); + expect(dm.settlePosts).toBe(1); + expect(dm.settlementSqlIo).toEqual({ commits: 1, updates: 1 }); + expect(host.calls.filter((a) => a.includes("--force-recreate"))).toHaveLength(1); + const actual = (await host.owned.inspectOwnedServices(new AbortController().signal)).find( + (s) => s.serviceId === "data-manager", + ); + expect(actual?.provenance.imageId).toBe(pulledImageId); + const baseline = owner.coordinatorPull.payload.baseline; + expect(baseline.policyRevision).not.toBe(baseline.configurationAuthorityRevision); + expect(baseline.container).toEqual( + owner.originalContainers.find((s: { serviceId: string }) => s.serviceId === "data-manager"), + ); + expect(Buffer.byteLength(JSON.stringify(baseline))).toBeLessThanOrEqual(3072); + expect(Buffer.byteLength(JSON.stringify(owner.coordinatorPull))).toBeLessThanOrEqual(8192); + expect( + owner.originalContainers.find((s: { serviceId: string }) => s.serviceId === "data-manager") + .provenance.imageId, + ).not.toBe(pulledImageId); + expect(owner.plan.coordinatorImage.imageId).toBe(pulledImageId); + expect(originalPull).toBeDefined(); + expect(owner.coordinatorPull.payload.pull.operationKey).toBe(originalPull!.operationKey); + expect(owner.coordinatorPull.payload.pull.deadlineAt).toBeLessThan( + originalPull!.requestedDeadlineAt, + ); + expect(Object.hasOwn(originalPull!, "executionId")).toBe(false); + const native = readFileSync(nativeTrace, "utf8") + .trim() + .split("\n") + .map((s) => JSON.parse(s) as string[]); + expect(native.filter((a) => a.includes("--force-recreate"))).toHaveLength(0); + }, 180000); + it.each([ + { action: "apply", point: "decision" }, + { action: "update", point: "decision" }, + { action: "update", point: "pull" }, + { action: "update", point: "family retirement" }, + { action: "apply", point: "candidate unlink" }, + { action: "update", point: "candidate unlink" }, + { action: "update", point: "mixed pull" }, + { action: "update", point: "canceled pull" }, + { action: "update", point: "candidate write" }, + { action: "update", point: "orphan candidate" }, + ] as const)( + "reclaims native coordinator $action after original $point without another admission or save", + async ({ action, point }) => { + const pullBoundary = + point === "pull" || point === "mixed pull" || point === "canceled pull"; + let reached!: () => void, release!: () => void; + const atDecision = new Promise((r) => { + reached = r; + }), + reply = new Promise((r) => { + release = r; + }); + if ( + !pullBoundary && + point !== "candidate unlink" && + point !== "candidate write" && + point !== "orphan candidate" + ) + decisionReplyGate = { reached, release: reply }; + else if (pullBoundary) pullTerminalReplyGate = { reached, release: reply }; + const database = new URL(process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL!); + database.searchParams.set("search_path", fixture.schema); + const loader = createRequire(new URL("../../../package.json", import.meta.url)).resolve( + "tsx/esm", + ), + script = fileURLToPath(new URL("./fixtures/native-coordinator-api.ts", import.meta.url)), + children: ReturnType[] = []; + const start = (input: Record) => { + const args = ["--import", loader, script], + child = spawn(process.execPath, args, { + cwd: host.root, + env: { ...process.env, DATABASE_URL: database.toString(), NODE_ENV: "test" }, + stdio: ["pipe", "pipe", "pipe"], + }); + children.push(child); + let queued!: (id: string) => void, + output = ""; + const jobId = new Promise((r) => { + queued = r; + }); + child.stdout!.on("data", (chunk) => { + output += chunk.toString(); + const lines = output.split("\n"); + output = lines.pop()!; + for (const line of lines) { + const event = JSON.parse(line); + if (event.event === "queued") queued(event.jobId); + } + }); + child.stderr!.on("data", () => {}); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => { + console.info( + JSON.stringify({ + proof: "native-coordinator-api-exit", + argv: [process.execPath, ...args], + code, + signal, + }), + ); + resolve({ code, signal }); + }); + }, + ); + child.stdin!.end(JSON.stringify(input)); + return { child, jobId, exited }; + }; + const first = start({ + action, + ...(point === "mixed pull" ? { serviceIds: ["motis", "data-manager"] } : {}), + ...(point === "candidate unlink" + ? { filesystemFault: "after-retirement-unlink" } + : point === "candidate write" + ? { filesystemFault: "before-candidate-write" } + : point === "orphan candidate" + ? { filesystemFault: "after-candidate-sync" } + : {}), + }); + try { + const id = await first.jobId; + if (point === "candidate write" || point === "orphan candidate") { + const marker = join( + host.root, + point === "candidate write" + ? "coordinator-candidate-write-paused" + : "coordinator-candidate-sync-paused", + ), + until = Date.now() + 45000; + while (!existsSync(marker)) { + if (Date.now() > until) throw Error("Actual candidate write boundary missing"); + await delay(10); + } + const [original] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)), + before = original!.result as Record; + expect(before._configurationCandidate).toBeUndefined(); + expect( + Object.values( + (before.opsProjection as { operations: Record }) + .operations, + ).map((value) => value.phase), + ).toEqual(["prepared"]); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength( + point === "orphan candidate" ? 1 : 0, + ); + first.child.kill("SIGKILL"); + expect((await first.exited).signal).toBe("SIGKILL"); + const lease = Date.parse(before._observationLeaseUntil as string); + while (Date.now() <= lease + 100) await delay(Math.min(1000, lease + 101 - Date.now())); + const replacement = start({ action: "reconcile", jobId: id }); + await replacement.exited; + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + expect(row!.status).toBe("failed"); + expect( + Object.values( + ( + row!.result as { + opsProjection: { operations: Record }; + } + ).opsProjection.operations, + ).map((value) => value.phase), + ).toEqual(["unissued"]); + expect(posts.get("service.pull") ?? 0).toBe(0); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + const failed = JSON.stringify(row!.result); + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/data-manager/action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "apply" }, + }); + expect((await finished(next.json().jobId)).status).toBe("success"); + const [unchanged] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + expect(JSON.stringify(unchanged!.result)).toBe(failed); + return; + } + let retainedBytes: Buffer | undefined; + const unlinkBoundary = async () => { + const marker = join(host.root, "coordinator-retirement-unlinked"), + until = Date.now() + 150000; + while (!existsSync(marker)) { + const [observed] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)), + candidate = ( + observed?.result as + | { _configurationCandidate?: { sealedOperationKey?: string } } + | undefined + )?._configurationCandidate; + if (candidate?.sealedOperationKey) { + const path = join( + host.queue, + ".candidates", + `${candidate.sealedOperationKey}.json`, + ); + if (existsSync(path)) retainedBytes = readFileSync(path); + } + if (Date.now() > until) throw Error("Actual retirement unlink boundary missing"); + await delay(10); + } + }; + await Promise.race([ + point === "candidate unlink" ? unlinkBoundary() : atDecision, + first.exited.then(() => { + throw Error("Native API exited before original source boundary"); + }), + ]); + const [before] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)), + prior = before!.result as Record, + candidate = prior._configurationCandidate as Record, + path = join(host.queue, ".candidates", `${candidate.sealedOperationKey}.json`), + bytes = point === "candidate unlink" ? retainedBytes! : readFileSync(path), + decision = JSON.stringify(prior._configurationMaintenanceDecision), + inspected = inspectPreparedConfigurationRecord(bytes, host.token).record, + originalNormal = + inspected.privateCoordinatorLifecycle?.action === "update" + ? readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${inspected.privateCoordinatorLifecycle.pull.operationKey}.json`, + ), + ) + : undefined; + expect(candidate.phase).toBe( + point === "candidate unlink" ? "settled" : !pullBoundary ? "saved" : "prepared", + ); + if (point === "candidate unlink") { + expect(bytes).toBeDefined(); + expect(existsSync(path)).toBe(false); + expect(candidate.retirement).toMatchObject({ version: 1, integrationIds: [] }); + expect(posts.get("configurationMaintenance.settle")).toBe(1); + } + expect(await fixture.db.select().from(schema.serviceConfig)).toHaveLength(0); + expect(posts.get("configurationMaintenance.decide") ?? 0).toBe(!pullBoundary ? 1 : 0); + if (pullBoundary) expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + first.child.kill("SIGKILL"); + expect((await first.exited).signal).toBe("SIGKILL"); + decisionReplyGate = undefined; + pullTerminalReplyGate = undefined; + release(); + if (point === "canceled pull") { + const { jobRunner } = await import("../job-runner"); + await jobRunner.cancel(id); + } + if (point === "family retirement") { + if (!originalNormal || inspected.privateCoordinatorLifecycle?.action !== "update") + throw Error("Actual original pull source required"); + const normalPath = join( + host.infraDir, + "control-private/ops-agent/executions", + `${inspected.privateCoordinatorLifecycle.pull.operationKey}.json`, + ); + const maintenanceId = ( + prior._configurationMaintenance as { receipt: { binding: { maintenanceId: string } } } + ).receipt.binding.maintenanceId; + const ownerPath = join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${maintenanceId}.json`, + ), + originalOwner = readFileSync(ownerPath), + owner = JSON.parse(originalOwner.toString()).payload, + authority = restoreAdmittedControlExecution(installation, owner.plan.prepare), + copied = JSON.stringify(owner.coordinatorPull); + expect( + Buffer.byteLength(JSON.stringify(JSON.parse(originalNormal.toString()))), + ).toBeLessThanOrEqual(65536); + expect( + owner.coordinatorPull.payload.baselineAdmissionDigest === + controlDigest(JSON.parse(originalNormal.toString())), + ).toBe(true); + const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now() + 96 * 3600000); + try { + let retired = 0; + for (let round = 0; round < 24 && existsSync(normalPath); round++) + retired += (await reconcileControlRetention(installation, { maxFamilies: 64 })) + .retired; + expect(retired).toBeGreaterThan(0); + expect(existsSync(normalPath)).toBe(false); + expect(existsSync(ownerPath)).toBe(true); + const proof = ownedCoordinatorPullBaseline(authority), + old = await inspectOriginalCoordinatorServices( + host.owned, + proof, + new AbortController().signal, + ); + expect( + old.find((value) => value.serviceId === "data-manager")?.provenance.imageId, + ).toBe(`sha256:${"b".repeat(64)}`); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (value) => value.serviceId === "data-manager", + ), + ).toBe(false); + expect( + JSON.stringify(JSON.parse(readFileSync(ownerPath, "utf8")).payload.coordinatorPull), + ).toBe(copied); + expect(() => assertControlExecutionForward(authority)).toThrow(); + const corrupt = Buffer.from(originalOwner); + corrupt[12] = corrupt[12] === 120 ? 121 : 120; + try { + writeFileSync(ownerPath, corrupt); + expect(() => ownedCoordinatorPullBaseline(authority)).toThrow(); + } finally { + writeFileSync(ownerPath, originalOwner); + } + expect(() => ownedCoordinatorPullBaseline(authority)).not.toThrow(); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(posts.get("service.pull")).toBe(1); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength( + 0, + ); + } finally { + clock.mockRestore(); + } + return; + } + const [retained] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)), + lease = Date.parse( + (retained!.result as Record)._observationLeaseUntil as string, + ); + expect(Number.isFinite(lease)).toBe(true); + while (Date.now() <= lease + 100) await delay(Math.min(1000, lease + 101 - Date.now())); + if (point !== "candidate unlink") expect(readFileSync(path).equals(bytes)).toBe(true); + else expect(existsSync(path)).toBe(false); + const replacement = start({ action: "reconcile", jobId: id }); + const exit = await replacement.exited; + if (point === "canceled pull") { + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + expect(row!.status).toBe("canceled"); + expect(existsSync(path)).toBe(false); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + return; + } + expect(exit.code).toBe(0); + const row = await finished(id), + result = row.result as Record; + expect(row.status).toBe("success"); + expect((result._execution as { version: number }).version).toBe(2); + if (point === "decision") + expect(JSON.stringify(result._configurationMaintenanceDecision)).toBe(decision); + if (originalNormal && inspected.privateCoordinatorLifecycle?.action === "update") { + expect( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/executions", + `${inspected.privateCoordinatorLifecycle.pull.operationKey}.json`, + ), + ).equals(originalNormal), + ).toBe(true); + expect(posts.get("service.pull")).toBe(1); + } + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(posts.get("configurationMaintenance.decide")).toBe(1); + expect(posts.get("configurationMaintenance.settle")).toBe(1); + expect(host.calls.filter((a) => a.includes("--force-recreate"))).toHaveLength(1); + expect(await fixture.db.select().from(schema.serviceConfig)).toHaveLength(0); + } finally { + release(); + decisionReplyGate = undefined; + pullTerminalReplyGate = undefined; + for (const child of children) + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + } + }, + 240000, + ); + it("denies a structural copy of the real production pull claim before signed admission or image staging", async () => { + copyPullClaim = true; + const observed = new Promise((resolve) => { + observePullReply = resolve; + }); + const response = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect(response.statusCode).toBe(200); + const observedStatus = await observed; + const id = (response.json() as { jobId: string }).jobId; + await finished(id); + expect(observedStatus).toBe(403); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + }, 60000); + it.each(coordinatorReferenceCases.slice(0, 3).map((value) => value.name))( + "refuses a %s before actual image staging", + async () => { + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect(reply.statusCode).toBe(200); + const row = await finished(reply.json().jobId); + expect(row.status).toBe("failed"); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + }, + 180000, + ); + it("allows distinct sibling coordinator tags currently sharing the OLD image", async () => { + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const row = await finished(reply.json().jobId); + expect(row.status).toBe("success"); + expect(posts.get("service.pull")).toBe(1); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + const snapshots = await host.owned.inspectOwnedServices(new AbortController().signal); + expect(snapshots.find((s) => s.serviceId === "motis")?.provenance.imageId).toBe( + `sha256:${"b".repeat(64)}`, + ); + expect(snapshots.find((s) => s.serviceId === "data-manager")?.provenance.imageId).toBe( + pulledImageId, + ); + expect(dm.settlementSqlIo).toEqual({ updates: 1, commits: 1 }); + }, 180000); + it.each([ + "generic resolver", + "policy-only", + "missing raw revision", + "raw suffix drift", + "consumed capture", + ] as const)( + "rejects %s at the actual candidate pull server before first signed admission", + async (mode) => { + claimCase = mode; + const observed = new Promise((resolve) => { + observePullReply = resolve; + }); + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + expect(reply.statusCode).toBe(200); + const status = await observed; + await finished(reply.json().jobId); + expect(status).toBe(403); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + const executions = join(host.infraDir, "control-private/ops-agent/executions"); + expect( + (existsSync(executions) ? readdirSync(executions) : []).filter( + (name) => + JSON.parse(readFileSync(join(executions, name), "utf8")).payload?.kind === + "service.pull", + ), + ).toHaveLength(0); + }, + 60000, + ); + it("rejects actual canonical Compose provenance exceeding the 3072-byte baseline before first admission", async () => { + const owned = await host.owned.inspectOwnedServices(new AbortController().signal); + const dmOwned = owned.find((value) => value.serviceId === "data-manager"); + expect(dmOwned?.state).toBe("running"); + expect(dmOwned?.provenance.composeConfigFiles).toHaveLength(2); + const observed = new Promise((resolve) => { + observePullReply = resolve; + }); + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const status = await observed; + await finished(reply.json().jobId); + expect(status).toBe(403); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + const executions = join(host.infraDir, "control-private/ops-agent/executions"); + expect( + (existsSync(executions) ? readdirSync(executions) : []).filter( + (name) => + JSON.parse(readFileSync(join(executions, name), "utf8")).payload?.kind === + "service.pull", + ), + ).toHaveLength(0); + }, 60000); + it("starts an initially stopped coordinator before pull using only its clamped original old-revision helper", async () => { + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + await expect(fetch(dm.baseUrl + "/health")).rejects.toThrow(); + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const row = await finished(reply.json().jobId); + expect(row.status).toBe("success"); + expect(posts.get("service.start")).toBe(1); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + const start = host.calls.findIndex( + (args) => args.includes("--no-recreate") && args.at(-1) === "data-manager", + ), + pull = host.calls.findIndex((args) => args[0] === "image" && args[1] === "pull"), + recreate = host.calls.findIndex((args) => args.includes("--force-recreate")); + expect(start).toBeGreaterThanOrEqual(0); + expect(pull).toBeGreaterThan(start); + expect(recreate).toBeGreaterThan(pull); + expect(host.calls.filter((args) => args.includes("--no-recreate"))).toHaveLength(1); + expect(dm.settlementSqlIo).toEqual({ updates: 1, commits: 1 }); + }, 180000); + it("closes a prepared pull locally after whole-candidate coordinator refusal without inventing an admission", async () => { + services.writeServiceSelectionFile(host.infraDir, ["motis"]); + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const id = reply.json().jobId, + until = Date.now() + 45000; + let row: typeof schema.adminJob.$inferSelect | undefined; + for (;;) { + [row] = await fixture.db.select().from(schema.adminJob).where(eq(schema.adminJob.id, id)); + const result = row?.result as { observation?: { state?: string } } | undefined; + if ( + row && + (["failed", "success", "canceled"].includes(row.status) || + result?.observation?.state === "unknown") + ) + break; + if (Date.now() > until) throw Error("Actual refused candidate did not become observable"); + await delay(10); + } + expect(row!.status).toBe("failed"); + const result = row!.result as { + opsProjection: { + operations: Record< + string, + { phase: string; terminal: boolean; operationId?: string; receiptDelivery?: unknown } + >; + }; + }; + const entries = Object.values(result.opsProjection.operations); + expect(entries).toHaveLength(1); + expect(entries[0]!.phase).toBe("unissued"); + expect(entries[0]!.terminal).toBe(true); + expect(entries[0]!.operationId).toBeUndefined(); + expect(entries[0]!.receiptDelivery).toBeUndefined(); + expect(posts.get("service.pull") ?? 0).toBe(0); + expect(posts.get("service.start") ?? 0).toBe(0); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + expect(await fixture.db.select().from(schema.serviceConfig)).toHaveLength(0); + const original = JSON.stringify(row!.result); + services.writeServiceSelectionFile(host.infraDir, ["data-manager", "motis"]); + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/data-manager/action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "apply" }, + }); + expect(next.statusCode).toBe(200); + expect((await finished(next.json().jobId)).status).toBe("success"); + const [unchanged] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + expect(JSON.stringify(unchanged!.result)).toBe(original); + expect(posts.get("service.pull") ?? 0).toBe(0); + }, 180000); + it("settles the authenticated unused OLD runtime after a real acknowledged API SQL rollback following image staging", async () => { + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const row = await finished(reply.json().jobId); + expect(refusedDecisionWrites).toBe(1); + expect(issuedKnownRollbacks).toBeGreaterThan(0); + expect(row.status).toBe("failed"); + const result = row.result as { + _configurationMaintenance: { receipt: { binding: { maintenanceId: string } } }; + _configurationMaintenanceRollback: { disposition: string }; + }; + expect(result._configurationMaintenanceRollback.disposition).toBe("known_rolled_back"); + const owner = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/configuration-maintenance", + `${result._configurationMaintenance.receipt.binding.maintenanceId}.json`, + ), + "utf8", + ), + ).payload; + expect(owner.receipt.state).toBe("settled"); + expect(owner.final.ownerFinal.payload.outcome).toBe("unused"); + expect(owner.receipt.decision.decision).toBe("abandon"); + expect(owner.receipt.generation).toBe("not_started"); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("stack.render") ?? 0).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(posts.get("configurationMaintenance.settle")).toBe(1); + expect(dm.settlementSqlIo).toEqual({ updates: 1, commits: 1 }); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + const original = await inspectOriginalCoordinatorServices( + host.owned, + ownedCoordinatorPullBaseline( + restoreAdmittedControlExecution(installation, owner.plan.prepare), + ), + new AbortController().signal, + ); + expect(original.find((s) => s.serviceId === "data-manager")?.provenance.imageId).toBe( + `sha256:${"b".repeat(64)}`, + ); + expect( + (await host.owned.inspectOwnedServices(new AbortController().signal)).some( + (s) => s.serviceId === "data-manager", + ), + ).toBe(false); + expect(await fixture.db.select().from(schema.serviceConfig)).toHaveLength(0); + expect(row.error ?? "").not.toContain('update "admin_job"'); + expect(row.error ?? "").not.toContain(host.root); + }, 180000); + it("refuses a valid full future owner budget before image staging or lifecycle", async () => { + expect(host.loaded.find((s) => s.manifest.id === "motis")?.manifest.consumes).toHaveLength( + 256, + ); + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const id = reply.json().jobId, + until = Date.now() + 45000; + let row: typeof schema.adminJob.$inferSelect | undefined; + for (;;) { + [row] = await fixture.db.select().from(schema.adminJob).where(eq(schema.adminJob.id, id)); + const result = row?.result as { observation?: { state?: string } } | undefined; + if ( + row && + (["failed", "success", "canceled"].includes(row.status) || + result?.observation?.state === "unknown") + ) + break; + if (Date.now() > until) throw Error("Actual owner capacity refusal not observable"); + await delay(10); + } + const observed = row!.result as { + observation?: { state?: string; reason?: string }; + opsProjection?: { + operations?: Record< + string, + { kind: string; phase: string; terminal: boolean; errorClass?: string } + >; + }; + _configurationCandidate?: { phase?: string }; + }; + console.info( + "actual prelude budget diagnostic", + JSON.stringify({ + jobState: row!.status, + observation: observed.observation, + candidatePhase: observed._configurationCandidate?.phase, + posts: [...posts], + operations: Object.values(observed.opsProjection?.operations ?? {}).map((value) => ({ + kind: value.kind, + phase: value.phase, + terminal: value.terminal, + errorClass: value.errorClass, + })), + }), + ); + expect(posts.get("service.pull") ?? 0).toBe(0); + expect(host.calls.filter((args) => args[0] === "image" && args[1] === "pull")).toHaveLength( + 0, + ); + expect(row!.status).toBe("failed"); + expect(posts.get("service.start") ?? 0).toBe(0); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + expect(dm.preparePosts).toBe(0); + expect(await fixture.db.select().from(schema.serviceConfig)).toHaveLength(0); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + }, 60000); + it("preserves actual first-delivery CAS against stale prepared and forged unissued generic checkpoints", async () => { + const reached = Promise.withResolvers(), + release = Promise.withResolvers(); + pullAdmissionReplyGate = { reached: () => reached.resolve(), release: release.promise }; + try { + const response = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + id = response.json().jobId; + await reached.promise; + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)), + original = row!.result as Record, + stale = structuredClone(original); + type Entry = { + phase: string; + terminal: boolean; + operation: { candidateId: string }; + nonissuance?: unknown; + }; + const entries = (stale.opsProjection as { operations: Record }).operations, + key = Object.keys(entries)[0]!, + entry = entries[key]!; + expect(entry.phase).toBe("admitting"); + entry.phase = "prepared"; + const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"), + execution = original._execution as { owner: string; version: number }; + const merged = await checkpointOwnedAdminJob( + id, + execution, + stale, + undefined, + undefined, + undefined, + original, + ); + expect( + (merged.opsProjection as { operations: Record }).operations[key]!.phase, + ).toBe("admitting"); + const forged = structuredClone(original), + fake = (forged.opsProjection as { operations: Record }).operations[key]!; + fake.phase = "unissued"; + fake.terminal = true; + fake.nonissuance = { + version: 1, + candidateId: fake.operation.candidateId, + checkedOwnerVersion: execution.version, + checkedAt: Date.now(), + }; + await expect( + checkpointOwnedAdminJob(id, execution, forged, undefined, undefined, undefined, original), + ).rejects.toThrow(); + const [unchanged] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + expect(JSON.stringify(unchanged!.result)).toBe(JSON.stringify(merged)); + pullAdmissionReplyGate = undefined; + release.resolve(); + expect((await finished(id)).status).toBe("success"); + expect(posts.get("service.pull")).toBe(1); + } finally { + pullAdmissionReplyGate = undefined; + release.resolve(); + } + }, 180000); + it.each([ + { name: "original cleanup", retry: false }, + { name: "fresh explicit retry", retry: true }, + ])( + "expires the original candidate TTL after actual image staging without acquiring maintenance or leaving a writer fence: $name", + async ({ retry }) => { + afterPullTerminal = () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 6 * 60000); + }; + try { + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }); + const id = reply.json().jobId, + until = performance.now() + 45000; + let row: typeof schema.adminJob.$inferSelect | undefined; + for (;;) { + [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + const observed = row?.result as { observation?: { state?: string } } | undefined; + if ( + row && + (["success", "failed", "canceled"].includes(row.status) || + observed?.observation?.state === "unknown") + ) + break; + if (performance.now() > until) throw Error("Expired candidate did not settle"); + await delay(10); + } + const operations = Object.values( + ( + row!.result as { + opsProjection: { + operations: Record< + string, + { + kind: string; + phase: string; + terminal: boolean; + errorClass?: string; + receiptDelivery?: { terminalReceipt: { state: string } }; + } + >; + }; + } + ).opsProjection.operations, + ); + console.info( + "actual TTL boundary", + JSON.stringify({ + state: row!.status, + operations: operations.map((op) => ({ + kind: op.kind, + phase: op.phase, + terminal: op.terminal, + errorClass: op.errorClass, + canonical: op.receiptDelivery?.terminalReceipt.state, + })), + }), + ); + expect(posts.get("service.pull")).toBe(1); + expect( + operations.find((op) => op.kind === "service.pull")?.receiptDelivery?.terminalReceipt + .state, + ).toBe("succeeded"); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + expect(posts.get("service.start") ?? 0).toBe(0); + expect(row!.status).toBe("failed"); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + if (retry) { + // Preserve monotonic process/retention time AND actual PG clock evidence. + // No SQL clock or signed settlement preimage is substituted. + await delay(6 * 60000 + 1000); + vi.useRealTimers(); + const stopNative = await restartNativeStagedAgent(); + try { + const next = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["data-manager"] }, + }), + nextRow = await finished(next.json().jobId); + console.info( + "actual fresh staged retry", + JSON.stringify({ + state: nextRow.status, + error: nextRow.error, + configuration: (nextRow.result as Record)?.configuration, + pullPosts: posts.get("service.pull") ?? 0, + preparePosts: posts.get("configurationMaintenance.prepare") ?? 0, + ordinaryOwnedDM: ( + await host.owned.inspectOwnedServices(new AbortController().signal) + ).some((s) => s.serviceId === "data-manager"), + }), + ); + expect(nextRow.status).toBe("success"); + expect(posts.get("service.pull")).toBe(2); + } finally { + await stopNative(); + } + } + } finally { + vi.useRealTimers(); + } + }, + 600000, + ); + it.each([false, true])( + "preserves the three-target requested revision chain and full selection with noDeps=%s", + async (noDeps) => { + const serviceIds = ["motis", "data-manager", "valhalla"], + reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds, noDeps }, + }), + row = await finished(reply.json().jobId); + expect(row.status, row.error ?? "No public error").toBe("success"); + expect((row.result as { completedServiceIds: string[] }).completedServiceIds).toEqual( + serviceIds, + ); + expect(posts.get("service.update")).toBe(2); + expect(posts.get("service.pull")).toBe(1); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(dm.settlementResponseStatuses).toEqual([200]); + const native = readFileSync(nativeTrace, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]), + recreates = native.filter((args) => args.includes("--force-recreate")); + expect(recreates.map((args) => args.at(-1))).toEqual(["motis", "valhalla"]); + expect(recreates.every((args) => args.includes("--no-deps") === noDeps)).toBe(true); + expect(services.readAppliedServiceIds(host.infraDir, host.loaded)).toEqual( + new Set(["data-manager", "motis", "valhalla"]), + ); + }, + 300000, + ); + it("closes a contained three-target pull failure after the acknowledged ordinary prefix without losing that prefix", async () => { + host.setImagePullProcess(async (_reference, signal) => { + await runContainedProcess(process.execPath, ["-e", "process.exit(17)"], { + signal, + timeout: 15000, + maxBuffer: 1024, + }); + }); + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds: ["motis", "data-manager", "valhalla"] }, + }), + id = reply.json().jobId, + until = performance.now() + 180000; + let row: typeof schema.adminJob.$inferSelect | undefined; + for (;;) { + [row] = await fixture.db.select().from(schema.adminJob).where(eq(schema.adminJob.id, id)); + if ( + row && + (["success", "failed", "canceled"].includes(row.status) || + (row.result as { observation?: { state: string } })?.observation?.state === "unknown") + ) + break; + if (performance.now() > until) + throw Error("Actual mixed failure did not become observable"); + await delay(25); + } + console.info( + "actual three-target contained prefix", + JSON.stringify({ + state: row!.status, + error: row!.error, + completed: (row!.result as Record)?.completedServiceIds, + observation: (row!.result as Record)?.observation, + posts: [...posts], + }), + ); + expect(row!.status).toBe("failed"); + expect((row!.result as { completedServiceIds: string[] }).completedServiceIds).toEqual([ + "motis", + ]); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(0); + expect(posts.get("service.update")).toBe(1); + expect(posts.get("configurationMaintenance.prepare") ?? 0).toBe(0); + }, 240000); + it("closes three-target cancellation after an acknowledged ordinary prefix and successful coordinator staging", async () => { + const { id } = await canceledSuccessfulCoordinatorStage(undefined, [ + "motis", + "data-manager", + "valhalla", + ]), + row = await finished(id); + expect((row.result as { completedServiceIds: string[] }).completedServiceIds).toEqual([ + "motis", + ]); + expect(posts.get("service.update")).toBe(1); + }, 240000); + it.each([ + ["data-manager", "motis"], + ["motis", "data-manager"], + ])( + "carries each actual bulk Update's own revision in requested order %j", + async (first, second) => { + const serviceIds = [first, second]; + const reply = await api.inject({ + method: "POST", + url: "/api/admin/services/bulk-action", + headers: { "x-openmapx-local-admin": "task012-coordinator-local-admin" }, + payload: { action: "update", serviceIds }, + }); + expect(reply.statusCode).toBe(200); + const id = reply.json().jobId, + until = Date.now() + 180000; + let diagnostic = false; + let row: typeof schema.adminJob.$inferSelect | undefined; + for (;;) { + [row] = await fixture.db.select().from(schema.adminJob).where(eq(schema.adminJob.id, id)); + const value = row?.result as + | { + observation?: { state?: string; reason?: string }; + opsProjection?: { + operations?: Record< + string, + { + kind: string; + phase: string; + terminal: boolean; + errorClass?: string; + failureResult?: { + serviceLifecycle?: { phase?: string; failure?: { reason?: string } }; + }; + } + >; + }; + _configurationCandidate?: { phase?: string }; + _configurationMaintenance?: { phase?: string }; + } + | undefined; + if ( + !diagnostic && + (Date.now() >= until - 160000 || value?.observation?.state === "unknown") + ) { + diagnostic = true; + const { readProjection } = await import("../admin-job-projection"); + try { + readProjection(value); + console.info("actual bulk projection valid"); + } catch (e) { + console.info( + "actual bulk projection rejected", + e instanceof Error ? e.name : "unknown", + ); + } + console.info( + "actual mixed Update diagnostic", + JSON.stringify({ + order: serviceIds, + jobState: row?.status, + observation: value?.observation, + candidatePhase: value?._configurationCandidate?.phase, + maintenancePhase: value?._configurationMaintenance?.phase, + arrivals: [...arrivals], + posts: [...posts], + operations: Object.values(value?.opsProjection?.operations ?? {}).map((op) => ({ + kind: op.kind, + phase: op.phase, + terminal: op.terminal, + errorClass: op.errorClass, + lifecyclePhase: op.failureResult?.serviceLifecycle?.phase, + lifecycleFailure: op.failureResult?.serviceLifecycle?.failure?.reason, + })), + }), + ); + } + if ( + row && + (["failed", "success", "canceled"].includes(row.status) || + value?.observation?.state === "unknown") + ) + break; + if (Date.now() > until) throw Error("Actual mixed Update did not settle"); + await delay(20); + } + expect(row!.status).toBe("success"); + const result = row!.result as { completedServiceIds: string[] }; + expect(result.completedServiceIds).toEqual(serviceIds); + expect(posts.get("configurationMaintenance.prepare")).toBe(1); + expect(posts.get("configurationMaintenance.decide")).toBe(1); + expect(posts.get("configurationMaintenance.settle")).toBe(1); + expect(posts.get("service.update")).toBe(1); + expect(posts.get("service.pull")).toBe(1); + const native = readFileSync(nativeTrace, "utf8") + .trim() + .split("\n") + .map((value) => JSON.parse(value) as string[]); + expect( + native.filter((args) => args.includes("pull") && args.at(-1) === "motis"), + ).toHaveLength(1); + expect( + native.filter((args) => args.includes("--force-recreate") && args.at(-1) === "motis"), + ).toHaveLength(1); + expect( + native.some( + (args) => args.includes("--force-recreate") && args.at(-1) === "data-manager", + ), + ).toBe(false); + }, + 240000, + ); + }, +); diff --git a/apps/api/src/services/__tests__/fixtures/native-configuration-api.ts b/apps/api/src/services/__tests__/fixtures/native-configuration-api.ts new file mode 100644 index 000000000..491eb8023 --- /dev/null +++ b/apps/api/src/services/__tests__/fixtures/native-configuration-api.ts @@ -0,0 +1,111 @@ +/** Native API producer process. No service/DB/authority replacement. The parent + * test controls only HTTP transport and kills this process at an actual reply. */ +import { sql } from "../../../db"; +import { initServiceRegistry } from "../../service-registry"; +import { applyTrustedConfiguration } from "../../trusted-config-operations"; + +let text = ""; +for await (const chunk of process.stdin) text += chunk; +const input = JSON.parse(text) as { + operationKey: string; + action?: + | "maintenance" + | "ordinary-service-config" + | "ordinary-service-secret" + | "maintenance-ensure-running" + | "reconcile"; +}; +if (!/^opk1_[A-Za-z0-9_-]{16,64}$/.test(input.operationKey)) throw Error("Invalid fixture key"); +if ( + input.action !== undefined && + ![ + "maintenance", + "ordinary-service-config", + "ordinary-service-secret", + "maintenance-ensure-running", + "reconcile", + ].includes(input.action) +) + throw Error("Invalid fixture action"); +let exit = 0, + phase = "registry"; +try { + await initServiceRegistry({ strict: true }); + if (input.action === "reconcile") { + phase = "reconcile"; + await (await import("../../job-runner")).jobRunner.reconcile({ limit: 1 }); + const rows = + await sql`SELECT status FROM admin_job WHERE payload->>'operationKey'=${input.operationKey} LIMIT 2`; + if (rows.length !== 1 || rows[0]!.status !== "success") exit = 2; + process.stdout.write( + JSON.stringify({ event: "result", state: "observed", status: rows[0]?.status ?? "missing" }) + + "\n", + ); + // Let the runner's already-scheduled empty-queue pump enter its SQL call; + // sql.end then joins that owned call before closing this fixture process. + await new Promise((resolve) => setTimeout(resolve, 0)); + } else { + phase = "apply"; + const credentialService = input.action === "ordinary-service-secret" ? "motis" : "data-manager", + coordinatorMode = + input.action === "ordinary-service-secret" || input.action === "maintenance-ensure-running" + ? ("ensure-running" as const) + : ("require-running" as const); + const value = await applyTrustedConfiguration( + input.action === "ordinary-service-config" + ? { + kind: "serviceConfig.apply", + serviceId: "motis", + actorId: "loopback-admin", + operationKey: input.operationKey, + coordinatorMode: "ensure-running", + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-config", serviceId: "motis", set: { WORKERS: 7 }, unset: [] }, + ], + }, + } + : { + kind: "vault.apply", + serviceId: credentialService, + actorId: "loopback-admin", + operationKey: input.operationKey, + coordinatorMode, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode, + changes: [ + { + kind: "service-secret", + serviceId: credentialService, + key: "TOKEN", + action: "set", + value: "synthetic-native-api-credential", + }, + ], + }, + }, + ); + process.stdout.write( + JSON.stringify({ event: "result", state: "applied", revisionId: value.revisionId }) + "\n", + ); + } +} catch (error) { + // Driver/HTTP errors can include private material. Only the fixed class is + // returned to the parent; neither stderr nor error messages are evidence. + process.stdout.write( + JSON.stringify({ + event: "result", + state: "refused", + phase, + errorClass: error instanceof Error ? error.name : "unknown", + }) + "\n", + ); + exit = 2; +} finally { + await sql.end({ timeout: 2 }); +} +process.exit(exit); diff --git a/apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts b/apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts new file mode 100644 index 000000000..22ea47d60 --- /dev/null +++ b/apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts @@ -0,0 +1,141 @@ +/** Actual queued API producer/reconciler. Only its owning test controls HTTP + * reply delivery and kills this native process; no DB/service/authority mocks. */ +import { constants, writeFileSync } from "node:fs"; +import filesystem from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import { join } from "node:path"; +import { sql } from "../../../db"; +import { handleServiceBulkJob } from "../../admin-job-handlers"; +import { serviceApply } from "../../admin-ops"; +import { jobRunner } from "../../job-runner"; +import { initServiceRegistry } from "../../service-registry"; + +let body = ""; +for await (const chunk of process.stdin) body += chunk; +const input = JSON.parse(body) as { + action: "apply" | "update" | "reconcile"; + jobId?: string; + serviceIds?: string[]; + noDeps?: boolean; + filesystemFault?: "after-retirement-unlink" | "before-candidate-write" | "after-candidate-sync"; +}; +if ( + input.filesystemFault !== undefined && + input.filesystemFault !== "after-retirement-unlink" && + input.filesystemFault !== "before-candidate-write" && + input.filesystemFault !== "after-candidate-sync" +) + throw Error("Invalid fixture edge"); +if (input.filesystemFault === "after-retirement-unlink") { + const unlink = filesystem.unlink, + directory = join(process.env.OPS_TRUSTED_CONFIG_DIR!, ".candidates") + "/"; + filesystem.unlink = async (path) => { + await unlink(path); + if (String(path).startsWith(directory)) { + writeFileSync( + join(process.env.OPENMAPX_ROOT_DIR!, "coordinator-retirement-unlinked"), + "unlinked", + ); + await new Promise(() => {}); + } + }; + syncBuiltinESMExports(); +} +if (input.filesystemFault === "before-candidate-write") { + const open = filesystem.open, + directory = join(process.env.OPS_TRUSTED_CONFIG_DIR!, ".candidates") + "/"; + filesystem.open = (async (path, flags, ...rest) => { + if ( + String(path).startsWith(directory) && + typeof flags === "number" && + (flags & constants.O_EXCL) !== 0 + ) { + writeFileSync( + join(process.env.OPENMAPX_ROOT_DIR!, "coordinator-candidate-write-paused"), + "paused", + ); + await new Promise(() => {}); + } + return open(path, flags, ...rest); + }) as typeof filesystem.open; + syncBuiltinESMExports(); +} +if (input.filesystemFault === "after-candidate-sync") { + const open = filesystem.open, + directory = join(process.env.OPS_TRUSTED_CONFIG_DIR!, ".candidates") + "/"; + filesystem.open = (async (path, flags, ...rest) => { + const handle = await open(path, flags, ...rest); + if ( + String(path).startsWith(directory) && + typeof flags === "number" && + (flags & constants.O_EXCL) !== 0 + ) { + const sync = handle.sync.bind(handle); + handle.sync = async () => { + await sync(); + writeFileSync( + join(process.env.OPENMAPX_ROOT_DIR!, "coordinator-candidate-sync-paused"), + "paused", + ); + await new Promise(() => {}); + }; + } + return handle; + }) as typeof filesystem.open; + syncBuiltinESMExports(); +} +let exit = 2; +try { + await initServiceRegistry({ strict: true }); + jobRunner.register("service.apply", async (ctx) => ({ + service: ctx.payload.service, + action: "apply", + ...(await serviceApply(ctx.payload.service as string, ctx)), + })); + jobRunner.register("service.bulk", handleServiceBulkJob); + let id: string; + if (input.action === "reconcile") { + if (typeof input.jobId !== "string" || !/^[a-f0-9-]{36}$/.test(input.jobId)) + throw Error("Invalid fixture job"); + id = input.jobId; + await jobRunner.reconcile({ limit: 1 }); + } else { + if (input.action !== "apply" && input.action !== "update") + throw Error("Invalid fixture action"); + id = await jobRunner.enqueue( + input.action === "apply" ? "service.apply" : "service.bulk", + input.action === "apply" + ? { service: "data-manager" } + : { + action: "update", + serviceIds: input.serviceIds ?? ["data-manager"], + ...(input.noDeps ? { noDeps: true } : {}), + }, + ); + process.stdout.write(JSON.stringify({ event: "queued", jobId: id }) + "\n"); + const until = Date.now() + 240000; + for (;;) { + const [row] = await sql`SELECT status FROM admin_job WHERE id=${id}`; + if (row && ["success", "failed", "canceled"].includes(row.status)) break; + if (Date.now() >= until) throw Error("Fixture observation expired"); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + const [row] = await sql`SELECT status FROM admin_job WHERE id=${id}`; + process.stdout.write( + JSON.stringify({ event: "result", status: row?.status ?? "missing" }) + "\n", + ); + exit = row?.status === "success" ? 0 : 2; + await new Promise((resolve) => setTimeout(resolve, 0)); +} catch (error) { + process.stdout.write( + JSON.stringify({ + event: "result", + state: "refused", + errorClass: error instanceof Error ? error.name : "unknown", + }) + "\n", + ); +} finally { + await sql.end({ timeout: 2 }); +} +process.exit(exit); diff --git a/apps/api/src/services/__tests__/fixtures/native-integration-import-api.ts b/apps/api/src/services/__tests__/fixtures/native-integration-import-api.ts new file mode 100644 index 000000000..958868507 --- /dev/null +++ b/apps/api/src/services/__tests__/fixtures/native-integration-import-api.ts @@ -0,0 +1,91 @@ +/** Actual local-admin HTTP route and generated client in a native API process. + * Only fixed public synthetic inputs enter this fixture; no DB/host handlers. */ + +import { writeFileSync } from "node:fs"; +import filesystem from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import { join } from "node:path"; +import { createApiClient } from "@openmapx/core"; +import { importAdminIntegrationConfigs } from "@openmapx/core/api-admin-sdk"; +import Fastify from "fastify"; +import { sql } from "../../../db"; +import { getIntegration, initIntegrations, shutdownIntegrations } from "../../../integration-host"; +import { adminIntegrationsRoute } from "../../../routes/admin-integrations"; +import { requireAdmin } from "../../../utils/require-admin"; +import { initServiceRegistry } from "../../service-registry"; + +let text = ""; +for await (const chunk of process.stdin) text += chunk; +const input = JSON.parse(text) as { + idempotencyKey: string; + filesystemFault?: "pause-retirement-unlink"; +}; +if (!/^ordinary-native-[a-f0-9-]{36}$/.test(input.idempotencyKey)) + throw Error("Invalid fixture key"); +if (input.filesystemFault !== undefined && input.filesystemFault !== "pause-retirement-unlink") + throw Error("Invalid filesystem fixture edge"); +if (input.filesystemFault) { + const unlink = filesystem.unlink, + directory = join(process.env.OPS_TRUSTED_CONFIG_DIR!, ".candidates") + "/"; + filesystem.unlink = async (path) => { + if (String(path).startsWith(directory)) { + writeFileSync(join(process.env.OPENMAPX_ROOT_DIR!, "retirement-unlink-paused"), "paused"); + await new Promise(() => {}); + } + return unlink(path); + }; + syncBuiltinESMExports(); +} +const api = Fastify({ logger: false }); +let exit = 0, + phase = "registry"; +try { + await initServiceRegistry({ strict: true }); + await initIntegrations(api, [ + { directory: join(process.env.OPENMAPX_ROOT_DIR!, "integrations"), isBuiltIn: true }, + ]); + api.addHook("preHandler", async (request) => { + request.adminSession = await requireAdmin(request); + }); + await api.register(adminIntegrationsRoute, { prefix: "/api" }); + const baseUrl = await api.listen({ host: "127.0.0.1", port: 0 }); + phase = "import"; + const result = await importAdminIntegrationConfigs( + { + integrations: [ + { integrationId: "alpha", config: { VALUE: "native-new", TOKEN: "synthetic-strip-this" } }, + { integrationId: "beta", config: { VALUE: "native-new-b", enabled: false } }, + ], + }, + { "idempotency-key": input.idempotencyKey }, + { + client: createApiClient({ baseUrl }), + headers: { "x-openmapx-local-admin": process.env.OPENMAPX_LOCAL_ADMIN_TOKEN! }, + }, + ); + if (!result.data || !("ok" in result.data) || result.data.ok !== true) exit = 2; + process.stdout.write( + JSON.stringify({ + event: "result", + state: exit === 0 ? "imported" : "refused", + status: result.status, + capturedOrdinaryValueMatches: getIntegration("alpha")?.config.VALUE === "native-new", + betaDisabled: getIntegration("beta")?.enabled === false, + }) + "\n", + ); +} catch (error) { + process.stdout.write( + JSON.stringify({ + event: "result", + state: "refused", + phase, + errorClass: error instanceof Error ? error.name : "unknown", + }) + "\n", + ); + exit = 2; +} finally { + await shutdownIntegrations(); + await api.close(); + await sql.end({ timeout: 2 }); +} +process.exit(exit); diff --git a/apps/api/src/services/__tests__/fixtures/postgres-commit-relay.ts b/apps/api/src/services/__tests__/fixtures/postgres-commit-relay.ts new file mode 100644 index 000000000..f3a1a20bb --- /dev/null +++ b/apps/api/src/services/__tests__/fixtures/postgres-commit-relay.ts @@ -0,0 +1,182 @@ +/** External PostgreSQL transport edge for native API death probes. Only the + * real backend's selected COMMIT or INSERT acknowledgement is held. SQL bytes, execution and + * durable tuples are never replaced; credentials/packets are never logged. */ +import { connect, createServer, type Socket } from "node:net"; + +export async function postgresCommitRelay( + database: URL, + shouldHold: (observation: { backendPid: number; command: string }) => Promise, + options: { + holdCommand?: "COMMIT" | "INSERT" | "UPDATE" | "API_TERMINAL_UPDATE"; + observeFrontend?: "service-secret-insert"; + } = {}, +) { + if (database.hostname !== "127.0.0.1" || !/^\d+$/.test(database.port)) + throw Error("Explicit owned local PostgreSQL required"); + const reached = Promise.withResolvers(), + sockets = new Set(), + releases = new Set<() => void>(); + let retained = false, + released = false; + const server = createServer((client) => { + const backend = connect({ host: database.hostname, port: Number(database.port) }); + sockets.add(client); + sockets.add(backend); + let pending = Buffer.alloc(0), + suspended = false, + held: Buffer[] = [], + backendPid: number | undefined; + const flush = () => { + if (!client.destroyed && held.length) client.write(Buffer.concat(held)); + held = []; + suspended = false; + if (!backend.destroyed) backend.resume(); + }; + releases.add(flush); + let frontend = Buffer.alloc(0), + startup = true, + frontendSuspended = false, + observedSecretInsert = false; + const forwardFrontend = () => { + if (!backend.destroyed && frontend.length) backend.write(frontend); + frontend = Buffer.alloc(0); + frontendSuspended = false; + client.resume(); + }; + releases.add(forwardFrontend); + client.on("data", (bytes) => { + if (options.holdCommand !== "API_TERMINAL_UPDATE" && !options.observeFrontend) { + if (!backend.destroyed) backend.write(bytes); + return; + } + frontend = Buffer.concat([frontend, Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes)]); + if (frontendSuspended) return; + while (frontend.length >= (startup ? 4 : 5)) { + const size = frontend.readInt32BE(startup ? 0 : 1) + (startup ? 0 : 1); + if (size < 5 || size > 16 * 1024 * 1024) { + client.destroy(); + backend.destroy(); + return; + } + if (frontend.length < size) return; + const packet = frontend.subarray(0, size); + const secretInsert = + options.observeFrontend === "service-secret-insert" && + !observedSecretInsert && + packet.includes(Buffer.from('insert into "service_secret"')); + if ( + !startup && + !retained && + !released && + backendPid !== undefined && + (packet[0] === 0x50 || packet[0] === 0x51) && + (secretInsert || + (options.holdCommand === "API_TERMINAL_UPDATE" && + packet.includes(Buffer.from('update "admin_job" set "status"')))) + ) { + if (secretInsert) observedSecretInsert = true; + frontendSuspended = true; + client.pause(); + void shouldHold({ + backendPid, + command: secretInsert ? "BEFORE_SERVICE_SECRET_INSERT" : "API_TERMINAL_UPDATE", + }) + .then((yes) => { + if (!secretInsert && yes && !retained && !released) { + retained = true; + reached.resolve(); + } else forwardFrontend(); + }) + .catch(() => { + client.destroy(); + backend.destroy(); + }); + return; + } + startup = false; + frontend = frontend.subarray(size); + if (!backend.destroyed) backend.write(packet); + } + }); + client.on("close", () => { + sockets.delete(client); + releases.delete(flush); + releases.delete(forwardFrontend); + backend.destroy(); + }); + client.on("error", () => backend.destroy()); + backend.on("error", () => client.destroy()); + backend.on("close", () => { + sockets.delete(backend); + client.destroy(); + }); + backend.on("data", (chunk) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + if (suspended) { + held.push(bytes); + return; + } + pending = Buffer.concat([pending, bytes]); + while (pending.length >= 5) { + const size = pending.readInt32BE(1) + 1; + if (size < 5 || size > 16 * 1024 * 1024) { + client.destroy(); + backend.destroy(); + return; + } + if (pending.length < size) return; + const packet = pending.subarray(0, size); + pending = pending.subarray(size); + if (packet[0] === 0x4b && size === 13) backendPid = packet.readInt32BE(5); + const command = packet[0] === 0x43 ? packet.subarray(5, -1).toString() : null, + selected = + options.holdCommand === "API_TERMINAL_UPDATE" + ? false + : options.holdCommand === "UPDATE" + ? command?.startsWith("UPDATE ") + : options.holdCommand === "INSERT" + ? command?.startsWith("INSERT ") + : command === "COMMIT"; + if (!retained && !released && selected && command && backendPid !== undefined) { + suspended = true; + backend.pause(); + held = [packet, pending]; + pending = Buffer.alloc(0); + void shouldHold({ backendPid, command }) + .then((yes) => { + if (yes && !retained && !released) { + retained = true; + reached.resolve(); + } else flush(); + }) + .catch(() => { + client.destroy(); + backend.destroy(); + }); + return; + } + if (!client.destroyed) client.write(packet); + } + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw Error("Owned PostgreSQL relay unavailable"); + const url = new URL(database); + url.hostname = "127.0.0.1"; + url.port = String(address.port); + url.searchParams.set("sslmode", "disable"); + return { + databaseUrl: url.toString(), + reached: reached.promise, + release() { + released = true; + for (const flush of releases) flush(); + }, + async close() { + released = true; + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} diff --git a/apps/api/src/services/__tests__/ops-client.test.ts b/apps/api/src/services/__tests__/ops-client.test.ts index 0bec9f180..7f59d6395 100644 --- a/apps/api/src/services/__tests__/ops-client.test.ts +++ b/apps/api/src/services/__tests__/ops-client.test.ts @@ -18,6 +18,15 @@ import { waitForOpsResult, } from "../ops-client.js"; +// Typed transport fixture only; not an ownership or image-source proof. +const pullResult = { + changed: true as const, + revisionId: `cfg1_${"a".repeat(43)}`, + imageReference: "example/redis:1", + imageId: `sha256:${"b".repeat(64)}`, + releaseSelectionDigest: null, +}; + function fakeClient(overrides: Partial> = {}): ApiOpsClient { return { execute: vi.fn(), @@ -182,7 +191,7 @@ describe("bounded async ops helpers", () => { state: "succeeded", submittedAt: "2026-08-23T10:00:00.000Z", updatedAt: "2026-08-23T10:00:01.000Z", - result: { changed: true }, + result: pullResult, }); const lookup = vi.fn().mockResolvedValue({ version: 1, @@ -197,19 +206,31 @@ describe("bounded async ops helpers", () => { await expect( executeAndWait( fakeClient({ execute, status, lookup }), - { kind: "service.pull", serviceId: "redis" }, + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, operationKey, { parentAssertion: "retained-test-parent" }, ), - ).resolves.toEqual({ changed: true }); + ).resolves.toEqual(pullResult); expect(execute).toHaveBeenCalledTimes(1); expect(execute).toHaveBeenNthCalledWith( 1, - { kind: "service.pull", serviceId: "redis" }, + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, expect.objectContaining({ operationKey }), ); expect(lookup).toHaveBeenCalledWith( - { kind: "service.pull", serviceId: "redis" }, + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, operationKey, expect.anything(), ); @@ -232,7 +253,7 @@ describe("bounded async ops helpers", () => { state: "succeeded" as const, submittedAt: "2026-08-23T10:00:00.000Z", updatedAt: "2026-08-23T10:00:01.000Z", - result: { changed: true }, + result: pullResult, }; const transient = vi .fn() @@ -242,7 +263,7 @@ describe("bounded async ops helpers", () => { waitForOpsResult(fakeClient({ status: transient }), admission.kind, admission, { sleep: async () => undefined, }), - ).resolves.toEqual({ changed: true }); + ).resolves.toEqual(pullResult); expect(transient).toHaveBeenCalledTimes(2); const denied = vi.fn().mockRejectedValue(new ApiOpsError("authorization")); @@ -563,6 +584,8 @@ describe("bounded async ops helpers", () => { describe("configuration outcome evidence", () => { const operation = { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render" as const, revisionId: `cfg1_${"a".repeat(43)}`, }; @@ -660,7 +683,7 @@ describe("independent observation deadlines", () => { submittedAt: new Date(0).toISOString(), updatedAt: new Date(now).toISOString(), state: ++polls === 1 ? "running" : "succeeded", - ...(polls === 1 ? {} : { result: { changed: true } }), + ...(polls === 1 ? {} : { result: { completed: true } }), }), }); await expect( @@ -670,7 +693,7 @@ describe("independent observation deadlines", () => { now = 31 * 60_000; }, }), - ).resolves.toEqual({ changed: true }); + ).resolves.toEqual({ completed: true }); expect(client.cancel).not.toHaveBeenCalled(); expect(client.execute).not.toHaveBeenCalled(); }); diff --git a/apps/api/src/services/__tests__/prospective-configuration-postgres.test.ts b/apps/api/src/services/__tests__/prospective-configuration-postgres.test.ts new file mode 100644 index 000000000..aaee1ead8 --- /dev/null +++ b/apps/api/src/services/__tests__/prospective-configuration-postgres.test.ts @@ -0,0 +1,220 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import * as schema from "../../db/schema"; +import { createConfigurationPostgresFixture } from "../../test/configuration-postgres-fixture"; +import type { JobContext } from "../job-runner"; + +type Fixture = Awaited>; +const connection = vi.hoisted(() => ({ fixture: null as unknown as Fixture })); +vi.mock("../../db", () => ({ + get db() { + return connection.fixture.db; + }, + get sql() { + return connection.fixture.sql; + }, +})); +vi.mock("../../integration-host", () => ({ getAllIntegrations: () => [] })); +vi.mock("../service-registry", () => ({ getServiceRegistry: () => ({ list: () => [] }) })); +const subject = await import("../administrative-configuration-writer"); + +describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "prospective configuration dedicated PostgreSQL writer", + () => { + let fixture: Fixture; + beforeAll(async () => { + fixture = await createConfigurationPostgresFixture(); + connection.fixture = fixture; + }); + afterAll(async () => { + await fixture?.stop(); + }); + async function job(): Promise { + const jobId = randomUUID(), + execution = { owner: randomUUID(), version: 1 }; + await fixture.db.insert(schema.adminJob).values({ + id: jobId, + type: "compose.render", + status: "running", + result: { _execution: execution }, + }); + const ctx: JobContext = { + jobId, + jobType: "compose.render", + execution, + payload: {}, + signal: new AbortController().signal, + checkpointResult: { _execution: execution }, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (result) => { + await fixture.db + .update(schema.adminJob) + .set({ result }) + .where(eq(schema.adminJob.id, jobId)); + ctx.checkpointResult = result; + }, + }; + return ctx; + } + function writer( + ctx: JobContext, + ids: string[], + run: Parameters>[1], + ) { + return subject.withAdministrativeConfigurationJob(ctx, () => + subject.withAdministrativeConfigurationWriter( + { + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + integrationIds: ids, + }, + run, + ), + ); + } + + it("serializes two real connections with inverse caller integration order and reuses nested ownership", async () => { + const first = await job(), + second = await job(); + const entered = Promise.withResolvers(), + release = Promise.withResolvers(); + const order: string[] = [], + pids: number[] = []; + const a = writer(first, ["zeta", "alpha"], async (owned) => { + expect(owned.integrationIds).toEqual(["alpha", "zeta"]); + const state = subject.administrativeConfigurationState(owned); + pids.push( + (await state.connection.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`))[0]! + .pid, + ); + await subject.withAdministrativeConfigurationWriter( + { actorId: first.jobId, operationKey: owned.operationKey, integrationIds: ["alpha"] }, + async (nested) => { + expect(nested).toBe(owned); + }, + ); + order.push("first"); + entered.resolve(); + await release.promise; + }); + void a.catch(entered.reject); + await entered.promise; + const b = writer(second, ["alpha", "zeta"], async (owned) => { + const state = subject.administrativeConfigurationState(owned); + pids.push( + (await state.connection.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`))[0]! + .pid, + ); + order.push("second"); + }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(order).toEqual(["first"]); + release.resolve(); + await Promise.all([a, b]); + expect(order).toEqual(["first", "second"]); + expect(new Set(pids).size).toBe(2); + }); + + it("rolls back desired writes on the same dedicated connection and proves rollback before classifying it", async () => { + const ctx = await job(); + await writer(ctx, [], async (owned) => { + const state = subject.administrativeConfigurationState(owned); + const [owner] = await state.connection.execute<{ pid: number }>( + sql`select pg_backend_pid() as pid`, + ); + await expect( + subject.withConfigurationWriteTransaction(owned, async (tx) => { + const [actual] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + expect(actual!.pid).toBe(owner!.pid); + await tx + .insert(schema.serviceConfig) + .values({ id: randomUUID(), serviceId: "rollback-fixture", config: { WORKERS: 4 } }); + throw Error("synthetic failure after desired write"); + }), + ).rejects.toBeInstanceOf(subject.ConfigurationWriteRolledBack); + }); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual([]); + }); + + it("rejects a nested transaction before it can commit the outer writer's desired row", async () => { + const ctx = await job(); + let nestedEntered = false; + await writer(ctx, [], async (owned) => { + await expect( + subject.withConfigurationWriteTransaction(owned, async (tx) => { + await tx.insert(schema.serviceConfig).values({ + id: randomUUID(), + serviceId: "nested-rollback-fixture", + config: { WORKERS: 7 }, + }); + await expect( + subject.withConfigurationWriteTransaction(owned, async () => { + nestedEntered = true; + }), + ).rejects.toThrow("Configuration transaction already active"); + throw Error("synthetic outer rollback after refused nested transaction"); + }), + ).rejects.toBeInstanceOf(subject.ConfigurationWriteRolledBack); + }); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual([]); + expect(nestedEntered).toBe(false); + }); + + it("refuses expansion, conflicting job identity, foreign participant and expired writer", async () => { + const ctx = await job(); + let expired: Parameters[0] | undefined; + await writer(ctx, ["alpha"], async (owned) => { + expired = owned; + await expect( + subject.withAdministrativeConfigurationWriter( + { actorId: ctx.jobId, operationKey: owned.operationKey, integrationIds: ["beta"] }, + async () => {}, + ), + ).rejects.toThrow(); + await expect( + subject.withAdministrativeConfigurationWriter( + { actorId: randomUUID(), operationKey: owned.operationKey, integrationIds: [] }, + async () => {}, + ), + ).rejects.toThrow(); + await expect( + subject.withConfigurationCommitParticipant( + owned, + { + kind: "configuration-migration", + mode: "preview", + actorId: ctx.jobId, + operationKey: owned.operationKey, + } as never, + async () => {}, + ), + ).rejects.toThrow(); + }); + expect(() => subject.administrativeConfigurationState(expired!)).toThrow(); + }); + + it("retains a durable unresolved candidate fence after the physical session lock releases", async () => { + const first = await job(), + second = await job(); + await writer(first, [], async (owned) => { + await first.checkpoint({ + ...first.checkpointResult, + _configurationCandidate: { operationKey: owned.operationKey, phase: "save_attempted" }, + }); + }); + await expect(writer(second, [], async () => {})).rejects.toThrow( + "Configuration writer unavailable", + ); + await first.checkpoint({ + ...first.checkpointResult, + _configurationCandidate: { + operationKey: `opk1_${first.jobId.replaceAll("-", "")}`, + phase: "settled", + }, + }); + await expect(writer(second, [], async () => "allowed")).resolves.toBe("allowed"); + }); + }, +); diff --git a/apps/api/src/services/__tests__/prospective-configuration.test.ts b/apps/api/src/services/__tests__/prospective-configuration.test.ts new file mode 100644 index 000000000..ab8a6f70f --- /dev/null +++ b/apps/api/src/services/__tests__/prospective-configuration.test.ts @@ -0,0 +1,104 @@ +import { afterEach, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ failService: false, vaultRows: [] as unknown[] })); +vi.mock("../../db", () => ({ + db: { + transaction: async (run: (tx: unknown) => Promise) => { + const { db } = await import("../../db"); + return run(db); + }, + select: () => ({ + from: (table: { name: string }) => { + const rows = () => { + if (table.name === "service_config" && state.failService) + throw new Error("synthetic unavailable service query"); + return table.name === "service_secret" ? state.vaultRows : []; + }; + return { + // biome-ignore lint/suspicious/noThenProperty: This unit fixture models Drizzle lazy awaitable query acquisition. + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + Promise.resolve().then(rows).then(resolve, reject), + where: () => ({ + // biome-ignore lint/suspicious/noThenProperty: The filtered query retains the same lazy acquisition/error behavior. + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + Promise.resolve().then(rows).then(resolve, reject), + limit: rows, + }), + }; + }, + }), + }, +})); +vi.mock("../../db/schema", () => ({ + serviceConfig: { name: "service_config", serviceId: "service_id" }, + integrationConfig: { name: "integration_config" }, + serviceSecret: { name: "service_secret", serviceId: "service_id" }, + integrationSecret: { name: "integration_secret", integrationId: "integration_id" }, +})); +vi.mock("../../integration-host", () => ({ getAllIntegrations: () => [] })); +vi.mock("../service-registry", () => ({ + getServiceRegistry: () => ({ + list: () => + ["alpha", "beta", "data-manager"].map((id) => ({ + manifest: { + id, + configSchema: { properties: { WORKERS: { type: "integer", default: 2 } } }, + container: { image: `fixture/${id}`, tag: "1" }, + }, + })), + }), +})); + +const { buildConfigurationInput } = await import("../trusted-config-operations"); +const { encrypt } = await import("../secrets"); +const { withAdministrativeConfigurationRead } = await import( + "../administrative-configuration-writer" +); +const { resolveAllServiceConfigsStrict } = await import("../service-config-resolver"); +afterEach(() => { + state.failService = false; + state.vaultRows = []; + vi.unstubAllEnvs(); +}); + +it("authoritative full preparation rejects a failed service read while other sources succeed", async () => { + state.failService = true; + await expect(buildConfigurationInput()).rejects.toThrow("Configuration acquisition unavailable"); +}); + +it("authoritative full preparation cannot treat existing vault rows without a key as absent", async () => { + vi.stubEnv("OPENMAPX_SECRETS_KEY", ""); + state.vaultRows = [{ key: "TOKEN", ciphertext: "fixture", iv: "fixture", tag: "fixture" }]; + await expect(buildConfigurationInput()).rejects.toThrow("Configuration acquisition unavailable"); +}); + +it("counts retained orphan feed credentials in UTF-8 before publishing a full snapshot", async () => { + vi.stubEnv("OPENMAPX_SECRETS_KEY", "51".repeat(32)); + const value = JSON.stringify( + Object.fromEntries(Array.from({ length: 8 }, (_, n) => [`orphan-${n}`, 'ü\\"'.repeat(1500)])), + ); + expect(value.length).toBeLessThan(65536); + expect(Buffer.byteLength(value)).toBeGreaterThan(65536); + state.vaultRows = [{ serviceId: "data-manager", key: "TRANSITOUS_API_KEYS", ...encrypt(value) }]; + await expect(buildConfigurationInput()).rejects.toThrow(); +}); + +it("rejects an expired administrative read context", async () => { + let saved: Parameters[1] | undefined; + await withAdministrativeConfigurationRead({}, async (context) => { + saved = context; + }); + await expect(resolveAllServiceConfigsStrict([], saved!)).rejects.toThrow( + "Configuration acquisition unavailable", + ); +}); + +it("rejects a foreign still-live read context inside another coherent read", async () => { + await withAdministrativeConfigurationRead({}, async (first) => { + await withAdministrativeConfigurationRead({}, async () => { + await expect(resolveAllServiceConfigsStrict([], first)).rejects.toThrow( + "Configuration acquisition unavailable", + ); + }); + }); +}); diff --git a/apps/api/src/services/__tests__/service-preparation.test.ts b/apps/api/src/services/__tests__/service-preparation.test.ts new file mode 100644 index 000000000..2acb59a28 --- /dev/null +++ b/apps/api/src/services/__tests__/service-preparation.test.ts @@ -0,0 +1,116 @@ +import { afterAll, beforeEach, expect, it, vi } from "vitest"; +import type { JobContext } from "../job-runner"; + +const previousAuthSecret = vi.hoisted(() => { + const previous = process.env.BETTER_AUTH_SECRET; + process.env.BETTER_AUTH_SECRET ??= "task012-unit-only-synthetic-auth-secret"; + return { previous }; +}); +afterAll(() => { + if (previousAuthSecret.previous === undefined) delete process.env.BETTER_AUTH_SECRET; + else process.env.BETTER_AUTH_SECRET = previousAuthSecret.previous; +}); + +// Transport-level consumer tests. Real producer/ownership qualification lives +// in configuration-maintenance-postgres.test.ts with the production agent. +const transport = vi.hoisted(() => ({ publish: vi.fn(), execute: vi.fn(), inspect: vi.fn() })); +vi.mock("../trusted-configuration-producer", () => ({ + applyTrustedConfiguration: transport.publish, +})); +vi.mock("../admin-job-ops", async (actual) => ({ + ...(await actual()), + executeAdminJobOperation: transport.execute, +})); +vi.mock("../ops-client", async (actual) => ({ + ...(await actual()), + createApiOpsClient: () => ({}), + executeAndWait: transport.inspect, +})); + +import { prepareServiceGeneration, updatePreparedService } from "../service-preparation"; + +const revision = `cfg1_${"a".repeat(43)}`, + final = `cfg1_${"b".repeat(43)}`; +function context(): JobContext { + const ctx: JobContext = { + jobId: "11111111-1111-4111-8111-111111111111", + payload: {}, + signal: new AbortController().signal, + checkpointResult: {}, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (value) => { + ctx.checkpointResult = value; + }, + }; + return ctx; +} +beforeEach(() => { + transport.publish.mockReset().mockResolvedValue({ revisionId: revision }); + transport.inspect.mockReset().mockResolvedValue({ plan: [] }); + transport.execute.mockReset(); +}); +it("reuses the completed full-generation preparation for the same bulk scope", async () => { + const ctx = context(), + result = await prepareServiceGeneration(ctx, "bulk-start", { + coordinatorMode: "ensure-running", + }); + expect(result).toEqual({ + revisionId: revision, + coordinatorMode: "ensure-running", + hardlinks: { applied: true, entries: 0, linked: 0, skipped: 0, pruned: 0, via: "none" }, + }); + transport.publish.mockRejectedValue(Error("A replay must not publish")); + transport.inspect.mockRejectedValue(Error("A replay must not inspect another generation")); + expect( + await prepareServiceGeneration(ctx, "bulk-start", { coordinatorMode: "ensure-running" }), + ).toEqual(result); +}); +it("refuses a changed coordinator mode on a retained preparation before new transport", async () => { + const ctx = context(); + await prepareServiceGeneration(ctx, "render", { coordinatorMode: "require-running" }); + transport.publish.mockClear(); + transport.inspect.mockClear(); + await expect( + prepareServiceGeneration(ctx, "render", { coordinatorMode: "ensure-running" }), + ).rejects.toThrow(); + expect(transport.publish).not.toHaveBeenCalled(); + expect(transport.inspect).not.toHaveBeenCalled(); +}); +it("does not publish a completed preparation when the durable hardlink owner is unknown", async () => { + const ctx = context(); + transport.inspect.mockResolvedValue({ plan: [{ consumerService: "motis" }] }); + transport.execute.mockRejectedValue(Error("Unknown retained hardlink owner")); + await expect( + prepareServiceGeneration(ctx, "render", { coordinatorMode: "require-running" }), + ).rejects.toThrow("Unknown retained"); + expect(ctx.checkpointResult?._servicePreparations).toBeUndefined(); + expect(transport.execute).toHaveBeenCalledExactlyOnceWith( + ctx, + { + kind: "configuration.hardlinks.apply", + revisionId: revision, + coordinatorMode: "require-running", + }, + "admin-job.hardlinks.apply", + { durableIdentity: `${ctx.jobId}:render` }, + ); +}); +it("carries the Update target's own final revision and refuses a competing next generation", async () => { + const ctx = context(); + transport.inspect + .mockResolvedValueOnce({ revisionId: revision }) + .mockResolvedValueOnce({ revisionId: `cfg1_${"c".repeat(43)}` }); + transport.execute.mockResolvedValue({ changed: true, revisionId: final }); + expect(await updatePreparedService(ctx, "alpha", revision)).toEqual({ + changed: true, + revisionId: final, + }); + await expect(updatePreparedService(ctx, "beta", final)).rejects.toThrow("Configuration changed"); + expect(transport.execute).toHaveBeenCalledExactlyOnceWith( + ctx, + { kind: "service.update", serviceId: "alpha", expectedConfigurationRevisionId: revision }, + "admin-job.service.update", + { durableIdentity: "alpha" }, + ); +}); diff --git a/apps/api/src/services/admin-job-checkpoint.ts b/apps/api/src/services/admin-job-checkpoint.ts index c608cc0ef..04392b22a 100644 --- a/apps/api/src/services/admin-job-checkpoint.ts +++ b/apps/api/src/services/admin-job-checkpoint.ts @@ -1,7 +1,11 @@ -import { canonicalJson } from "@openmapx/core/ops"; +import { + canonicalJson, + configurationMaintenanceReceiptSchema, + parseControlRootOperationKey, +} from "@openmapx/core/ops"; import { db } from "../db"; import { adminJob } from "../db/schema"; -import { readProjection } from "./admin-job-ops"; +import { readProjection } from "./admin-job-projection"; import { ADMIN_JOB_LOG_LIMITS, type JobExecution, @@ -10,6 +14,7 @@ import { JobStepCapacityExceeded, jobOwnership, } from "./admin-job-state"; +import { assertConfigurationMaintenanceProjectedStep } from "./configuration-maintenance-step-records"; const equal = (a: unknown, b: unknown) => canonicalJson(a ?? null) === canonicalJson(b ?? null); /** Called only for v3. The database row lock owns uniqueness, capacity and merges. */ @@ -34,6 +39,8 @@ export async function checkpointOwnedAdminJob( next = readProjection(candidate); if (next.version !== 3) throw Error("Invalid managed checkpoint version"); let mergedProjection = old; + let maintenanceDecisionAttempt: Record | undefined; + let maintenanceSettleAttempt: Record | undefined; if (admissionKey || stepAddress) { if (!admissionKey || !stepAddress || !execution) throw Error("Incomplete stable step reservation"); @@ -45,15 +52,72 @@ export async function checkpointOwnedAdminJob( !operation?.attempt || operation.attempt.apiJobId !== id || operation.attempt.apiOwnerVersion !== execution.version || - operation.phase !== "admitting" || + !["admitting", "prepared"].includes(operation.phase) || operation.terminal ) throw Error("Invalid stable step reservation"); + if ( + operation.phase === "prepared" && + (row.type !== "service.bulk" || + (row.payload as { action?: unknown })?.action !== "update" || + operation.operation?.kind !== "service.pull" || + operation.operation.serviceId !== "data-manager" || + !operation.operation.candidateId) + ) + throw Error("Only the fixed coordinator pull can be preallocated"); if (old.steps[stepAddress] || old.operations[stepAddress] || old.operations[admissionKey]) return { conflict: true as const, result: current }; if (Object.keys(old.steps).length >= 64 || Object.keys(old.operations).length >= 64) return { capacity: true as const, result: current }; - if (row.status !== "running" || current._cancelRequestedAt) throw new JobOwnershipLost(); + if (row.status !== "running") throw new JobOwnershipLost(); + if (current._cancelRequestedAt !== undefined) { + // Cancellation never authorizes a forward phase. Only the original + // no-effect Abandon/Settle controls may be checkpointed against the + // actual already-delivered preparation in THIS owned job row. + const body = operation.operation; + if ( + !body || + (body.kind !== "configurationMaintenance.settle" && + !(body.kind === "configurationMaintenance.decide" && body.decision === "abandon")) || + typeof current._cancelRequestedAt !== "string" || + !Number.isFinite(Date.parse(current._cancelRequestedAt)) + ) + throw new JobOwnershipLost(); + const original = old.operations[body.prepareOperationKey], + delivery = original?.receiptDelivery, + retained = current._configurationCandidate as Record | undefined, + maintenance = current._configurationMaintenance as Record | undefined; + if ( + original?.kind !== "configurationMaintenance.prepare" || + delivery?.state !== "confirmed" || + delivery.terminalReceipt.state !== "succeeded" + ) + throw new JobOwnershipLost(); + const prepared = configurationMaintenanceReceiptSchema.parse( + delivery.terminalReceipt.result, + ); + if ( + prepared.apiJobId !== id || + prepared.state !== "prepared" || + !prepared.dmFence || + prepared.binding.candidateId !== body.candidateId || + prepared.binding.maintenanceId !== body.maintenanceId || + prepared.prepare.operationKey !== body.prepareOperationKey || + prepared.prepare.operationFingerprint !== body.prepareFingerprint || + prepared.planDigest !== body.planDigest || + retained?.candidateId !== body.candidateId || + retained.maintenanceId !== body.maintenanceId || + retained.prepareOperationKey !== body.prepareOperationKey || + maintenance?.candidateId !== body.candidateId || + maintenance.maintenanceId !== body.maintenanceId || + maintenance.prepareOperationKey !== body.prepareOperationKey || + maintenance.prepareOperationFingerprint !== body.prepareFingerprint + ) + throw new JobOwnershipLost(); + // The exact retained decision/settle body/address/attempt is checked + // below as usual. The actual agent independently verifies its owner, + // unused/final eligibility and original-root control; no new grant. + } if (current._parentExecution && !equal(current._parentExecution, candidate._parentExecution)) return { conflict: true as const, result: current }; const parent = candidate._parentExecution as @@ -66,6 +130,64 @@ export async function checkpointOwnedAdminJob( parent.deadlineAt !== operation.attempt.parentDeadlineAt ) throw Error("Incomplete atomic parent reservation"); + if (operation.operation?.kind === "configurationMaintenance.decide") { + const decision = current._configurationMaintenanceDecision as + | Record + | undefined, + delivery = decision?.delivery as + | { + stepAddress?: unknown; + root?: { operationKey?: unknown; requestedDeadlineAt?: unknown; parent?: unknown }; + } + | undefined; + if ( + !decision || + decision.attempt !== "not_attempted" || + !equal(decision.operation, operation.operation) || + delivery?.stepAddress !== stepAddress || + delivery.root?.operationKey !== admissionKey || + delivery.root.requestedDeadlineAt !== operation.attempt.requestedDeadlineAt || + !equal(delivery.root.parent, parent) + ) + throw Error("Actual retained maintenance decision admission required"); + maintenanceDecisionAttempt = { ...decision, attempt: "attempted" }; + } + if (operation.operation?.kind === "configurationMaintenance.settle") { + const settlement = current._configurationMaintenanceSettle as + | Record + | undefined, + delivery = settlement?.delivery as + | { + stepAddress?: unknown; + root?: { operationKey?: unknown; requestedDeadlineAt?: unknown; parent?: unknown }; + } + | undefined; + if ( + !settlement || + settlement.attempt !== "not_attempted" || + !equal(settlement.operation, operation.operation) || + delivery?.stepAddress !== stepAddress || + delivery.root?.operationKey !== admissionKey || + delivery.root.requestedDeadlineAt !== operation.attempt.requestedDeadlineAt || + !equal(delivery.root.parent, parent) + ) + throw Error("Actual retained maintenance settlement admission required"); + maintenanceSettleAttempt = { ...settlement, attempt: "attempted" }; + } + try { + if ( + parseControlRootOperationKey(admissionKey).notAfter !== + operation.attempt.requestedDeadlineAt + ) + throw Error("Original root changed"); + } catch { + assertConfigurationMaintenanceProjectedStep( + current, + admissionKey, + operation.operation, + operation.attempt, + ); + } mergedProjection = { ...old, version: 3, @@ -80,6 +202,18 @@ export async function checkpointOwnedAdminJob( for (const [key, entry] of Object.entries(next.operations)) { const previous = old.operations[key]; if (!previous) throw Error("New operation requires stable address reservation"); + // A stale checkpoint cannot resurrect an unattempted slot after the + // fixed delivery/nonissuance CAS has won. Keep that actual winner. + if (previous.phase !== "prepared" && entry.phase === "prepared") { + operations[key] = previous; + continue; + } + if (entry.phase === "unissued" && previous.phase !== "unissued") + throw Error("Local nonissuance requires its fixed source CAS"); + if (previous.phase === "unissued" && !equal(previous, entry)) + throw Error("Immutable local nonissuance changed"); + if (previous.phase === "prepared" && entry.phase !== "prepared") + throw Error("Original coordinator delivery requires its fixed source CAS"); for (const field of ["kind", "operationKey", "operation", "attempt"] as const) if (!equal(previous[field], entry[field])) throw Error("Immutable managed operation changed"); @@ -150,9 +284,35 @@ export async function checkpointOwnedAdminJob( : Object.fromEntries( Object.entries(patch).filter(([key, value]) => !equal(value, baseline[key])), ); + if ( + changed._configurationMaintenanceDecision !== undefined && + !equal(current._configurationMaintenanceDecision, changed._configurationMaintenanceDecision) + ) + throw Error("Immutable configuration decision changed outside its owned transaction"); + if ( + changed._configurationMaintenanceSteps !== undefined && + !equal(current._configurationMaintenanceSteps, changed._configurationMaintenanceSteps) + ) + throw Error("Frozen maintenance steps changed outside their owned transaction"); + if ( + changed._configurationMaintenanceSettle !== undefined && + !equal(current._configurationMaintenanceSettle, changed._configurationMaintenanceSettle) + ) + throw Error("Immutable maintenance settlement changed outside its owned transaction"); + if ( + changed._configurationMaintenanceRollback !== undefined && + !equal(current._configurationMaintenanceRollback, changed._configurationMaintenanceRollback) + ) + throw Error("Known maintenance rollback changed outside its fixed producer"); const merged = { ...current, ...changed, + ...(maintenanceDecisionAttempt + ? { _configurationMaintenanceDecision: maintenanceDecisionAttempt } + : {}), + ...(maintenanceSettleAttempt + ? { _configurationMaintenanceSettle: maintenanceSettleAttempt } + : {}), _opsLogEvents: current._opsLogEvents ?? old.eventTotal, _opsLogBytes: current._opsLogBytes ?? old.byteTotal, ...(admissionKey diff --git a/apps/api/src/services/admin-job-failure.ts b/apps/api/src/services/admin-job-failure.ts index 2654f7c1f..1af83f97f 100644 --- a/apps/api/src/services/admin-job-failure.ts +++ b/apps/api/src/services/admin-job-failure.ts @@ -9,6 +9,24 @@ export class AdminJobFailure extends Error { } } +/** A fixed observed prerequisite, never publication or cleanup authority. */ +export class ConfigurationPrerequisiteFailure extends AdminJobFailure { + readonly statusCode = 409; + constructor() { + super("Apply or update Data Manager before changing instance configuration.", { + errorCode: "configuration_prerequisite", + configuration: { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }, + }); + this.name = "ConfigurationPrerequisiteFailure"; + } +} + /** Internal admission bodies, ownership and proofs never cross the activity boundary. */ export function publicAdminJobResult(result: T): T { if (!result || typeof result !== "object" || Array.isArray(result)) return result; diff --git a/apps/api/src/services/admin-job-handlers.ts b/apps/api/src/services/admin-job-handlers.ts index e551eb65b..0b8fe0d0f 100644 --- a/apps/api/src/services/admin-job-handlers.ts +++ b/apps/api/src/services/admin-job-handlers.ts @@ -6,6 +6,12 @@ import { getAdminOperation, parseAdminOperationInput } from "./admin-operation-c import { assertKnownServiceIds, assertRegion } from "./admin-operation-input"; import type { JobContext } from "./job-runner"; import { OpsObservationPending } from "./ops-client"; +import { + applyCoordinatorService, + prepareServiceGeneration, + publishServiceGeneration, + updatePreparedService, +} from "./service-preparation"; type BackupOperation = "create" | "restore" | "delete"; type BulkServiceAction = "start" | "stop" | "restart" | "update" | "build"; @@ -120,6 +126,7 @@ export async function handleServiceBulkJob(ctx: JobContext): Promise; - eventTotal: number; - byteTotal: number; - truncated: boolean; - operations: Record; -} export interface ExecuteAdminJobOperationOptions { client?: ApiOpsClient; @@ -115,195 +80,6 @@ export interface ExecuteAdminJobOperationOptions { preparedRoot?: ApiPreparedRoot; } -function boundedInteger(value: unknown, maximum: number): value is number { - return Number.isInteger(value) && (value as number) >= 0 && (value as number) <= maximum; -} - -function emptyProjection(): OpsProjection { - return { version: 3, steps: {}, eventTotal: 0, byteTotal: 0, truncated: false, operations: {} }; -} - -export function readProjection(result: unknown): OpsProjection { - if (!result || typeof result !== "object" || Array.isArray(result)) return emptyProjection(); - const candidate = (result as Record).opsProjection; - if (candidate === undefined) return emptyProjection(); - if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) { - throw new JobObservationPending("invalid_identity"); - } - const value = candidate as Record; - if ( - (value.version !== 1 && value.version !== 2 && value.version !== 3) || - !boundedInteger(value.eventTotal, ADMIN_EVENT_LIMIT) || - !boundedInteger(value.byteTotal, ADMIN_EVENT_BYTE_LIMIT) || - typeof value.truncated !== "boolean" || - !value.operations || - typeof value.operations !== "object" || - Array.isArray(value.operations) - ) { - throw new JobObservationPending("invalid_identity"); - } - const entries = Object.entries(value.operations as Record); - if (entries.length > MAX_PROJECTED_OPERATIONS) - throw new JobObservationPending("invalid_identity"); - const operations: Record = {}; - for (const [key, raw] of entries) { - if (!OPERATION_KEY.test(key) || !raw || typeof raw !== "object" || Array.isArray(raw)) { - throw new JobObservationPending("invalid_identity"); - } - const operation = raw as Record; - if ( - operation.operationKey !== key || - (operation.operationId !== undefined && - (typeof operation.operationId !== "string" || !OPERATION_ID.test(operation.operationId))) || - (value.version === 1 && operation.operationId === undefined) || - typeof operation.kind !== "string" || - !boundedInteger(operation.cursor, Number.MAX_SAFE_INTEGER) || - !boundedInteger(operation.events, ADMIN_EVENT_LIMIT) || - !boundedInteger(operation.bytes, ADMIN_EVENT_BYTE_LIMIT) || - typeof operation.truncated !== "boolean" || - typeof operation.terminal !== "boolean" - ) { - throw new JobObservationPending("invalid_identity"); - } - if (!Object.hasOwn(OPS_KIND_POLICIES, operation.kind as string)) - throw new JobObservationPending("invalid_identity"); - if ( - value.version !== 1 && - !["admitting", "observing", "terminal", ...(value.version === 3 ? ["refused"] : [])].includes( - operation.phase as string, - ) - ) - throw new JobObservationPending("invalid_identity"); - const checked = - operation.operation === undefined - ? undefined - : opsOperationSchema.safeParse(operation.operation); - if (checked && !checked.success) throw new JobObservationPending("invalid_identity"); - const parsed = checked?.data; - if ( - operation.fingerprint !== undefined && - (typeof operation.fingerprint !== "string" || !/^[a-f0-9]{64}$/.test(operation.fingerprint)) - ) - throw new JobObservationPending("invalid_identity"); - if ( - operation.deadlineAt !== undefined && - !boundedInteger(operation.deadlineAt, Number.MAX_SAFE_INTEGER) - ) - throw new JobObservationPending("invalid_identity"); - if ( - operation.failureResult !== undefined && - !opsFailureResultSchema.safeParse(operation.failureResult).success - ) - throw new JobObservationPending("invalid_identity"); - if ( - operation.diagnostics !== undefined && - !processDiagnosticsSchema.safeParse(operation.diagnostics).success - ) - throw new JobObservationPending("invalid_identity"); - if (operation.attempt !== undefined) { - const attempt = apiJobChildAttemptSchema.safeParse(operation.attempt); - if ( - !attempt.success || - !parsed || - attempt.data.childOperationKey !== key || - attempt.data.childOperationFingerprint !== opsOperationFingerprint(parsed) - ) - throw new JobObservationPending("invalid_identity"); - } - if ( - operation.phase === "refused" && - (!operation.terminal || - !ADMISSION_REFUSALS.has(operation.errorClass as string) || - operation.operationId !== undefined || - operation.result !== undefined || - operation.receiptDelivery !== undefined || - operation.fingerprint !== undefined || - operation.deadlineAt !== undefined) - ) - throw new JobObservationPending("invalid_identity"); - if (operation.receiptDelivery !== undefined) { - const d = operation.receiptDelivery as ProjectedOperation["receiptDelivery"]; - try { - if ( - value.version !== 3 || - !operation.attempt || - operation.terminal !== true || - operation.phase !== "terminal" || - !d || - !["pending", "confirmed"].includes(d.state) || - !apiRetentionProofSchema.shape.terminalCopyId.safeParse(d.terminalCopyId).success || - !Number.isSafeInteger(d.checkpointOwnerVersion) || - d.checkpointOwnerVersion < 1 || - controlTerminalReceiptDigest(d.terminalReceipt) !== d.terminalDigest || - d.terminalReceipt.operationId !== operation.operationId || - d.terminalReceipt.kind !== operation.kind || - d.terminalReceipt.operationKey !== key || - canonicalJson(d.terminalReceipt.result ?? null) !== - canonicalJson(operation.result ?? null) || - canonicalJson(d.terminalReceipt.failureResult ?? null) !== - canonicalJson(operation.failureResult ?? null) || - (d.terminalReceipt.errorClass ?? null) !== (operation.errorClass ?? null) || - d.terminalReceipt.fingerprint !== operation.fingerprint || - d.terminalReceipt.deadlineAt !== operation.deadlineAt - ) - throw Error(); - } catch { - throw new JobObservationPending("recovery_required"); - } - } - if (parsed && parsed.kind !== operation.kind) - throw new JobObservationPending("invalid_identity"); - operations[key] = { - ...operation, - ...(parsed ? { operation: parsed } : {}), - phase: - value.version === 1 ? (operation.terminal ? "terminal" : "observing") : operation.phase, - } as ProjectedOperation; - } - const steps: OpsProjection["steps"] = {}; - if (value.version === 3) { - if ( - !value.steps || - typeof value.steps !== "object" || - Array.isArray(value.steps) || - Object.keys(value.steps).length > 64 - ) - throw new JobObservationPending("invalid_identity"); - for (const [address, raw] of Object.entries(value.steps)) { - const step = raw as { operationKey: string; requestedDeadlineAt: number }; - if ( - !OPERATION_KEY.test(address) || - !step || - Object.keys(step).length !== 2 || - !operations[step.operationKey] || - parseControlRootOperationKey(step.operationKey).notAfter !== step.requestedDeadlineAt || - operations[step.operationKey]?.attempt?.requestedDeadlineAt !== step.requestedDeadlineAt - ) - throw new JobObservationPending("invalid_identity"); - steps[address] = step; - } - } - for (const mapping of Object.values(steps)) { - const entry = operations[mapping.operationKey]; - if (!entry) throw new JobObservationPending("invalid_identity"); - if ( - entry.terminal && - entry.phase !== "refused" && - opsKindRequiresControlAuthority(entry.kind) && - !entry.receiptDelivery - ) - throw new JobObservationPending("recovery_required"); - } - return { - version: value.version as 1 | 2 | 3, - steps, - eventTotal: value.eventTotal, - byteTotal: value.byteTotal, - truncated: value.truncated, - operations, - }; -} - async function persistProjection( context: JobContext, projection: OpsProjection, @@ -331,6 +107,7 @@ export async function ensureAdminJobReceiptDelivery( ): Promise { if ( projected.phase === "refused" || + projected.phase === "unissued" || projection.version !== 3 || !projected.attempt || !Object.values(projection.steps).some((s) => s.operationKey === projected.operationKey) || @@ -381,10 +158,35 @@ export async function ensureAdminJobReceiptDelivery( fingerprint: fingerprint, terminalDigest: delivery.terminalDigest, }); + const ownerControl = + projected.operation?.kind === "configurationMaintenance.decide" || + projected.operation?.kind === "configurationMaintenance.settle" + ? projected.operation + : undefined; + if ( + ["configurationMaintenance.decide", "configurationMaintenance.settle"].includes( + projected.kind, + ) && + !ownerControl + ) + throw new JobObservationPending("recovery_required"); + const frozenOrigin = + projected.operation && + "maintenanceId" in projected.operation && + projected.operation.maintenanceId && + !ownerControl && + context.checkpointResult?._configurationMaintenanceSteps + ? configurationMaintenanceProjectedStepOrigin( + context.checkpointResult, + projected.operationKey, + projected.operation, + projected.attempt, + ) + : undefined; const response = await client.execute( { kind: "controlRetention.acknowledge", - originRootKey: projected.operationKey, + originRootKey: ownerControl?.prepareOperationKey ?? frozenOrigin ?? projected.operationKey, operationKey: projected.operationKey, fingerprint: fingerprint, terminalDigest: delivery.terminalDigest, @@ -429,6 +231,8 @@ export async function executeAdminJobOperation( let projection = readProjection(context.checkpointResult); const retainedKey = projection.steps[stepAddress]?.operationKey ?? stepAddress; let restored = projection.operations[retainedKey]; + if (restored?.phase === "prepared" || restored?.phase === "unissued") + throw new JobObservationPending("recovery_required"); if (restored && restored.kind !== operation.kind) throw new JobObservationPending("invalid_identity"); if (restored) @@ -447,8 +251,12 @@ export async function executeAdminJobOperation( if ( root.operationKey !== createControlRootOperationKey({ notAfter: root.requestedDeadlineAt, nonce: expectedNonce }) - ) - throw new JobObservationPending("invalid_identity"); + ) { + const { assertOwnedConfigurationMaintenanceStep } = await import( + "./configuration-maintenance-step" + ); + await assertOwnedConfigurationMaintenanceStep(context, exactOperation, root.operationKey, root); + } const operationKey = root.operationKey, attempt = prepareApiJobChildAttempt(context, exactOperation, operationKey, root); if (!attempt) throw Error("Fresh managed operation requires a durable job attempt"); @@ -505,6 +313,89 @@ export async function executeAdminJobOperation( >; } +/** Finite preallocation for the nominal coordinator Update only. No dispatch + * occurs; the original attempt is checkpointed before the candidate is sealed. */ +export async function reserveCoordinatorPullSubmission( + context: JobContext, + intent: CoordinatorLifecycleIntent, + expectedConfigurationRevisionId: string, + parent: ApiPreparedRoot["parent"], +): Promise { + const fixed = await assertCoordinatorPullIntent(intent, context), + stepAddress = createDurableOpsKey( + "configuration.coordinator-pull", + `${context.jobId}:${fixed.originalOrdinal}`, + ), + operation = { + kind: "service.pull" as const, + serviceId: "data-manager" as const, + expectedConfigurationRevisionId, + candidateId: fixed.candidateId, + }, + root = prepareApiJobRoot( + { ...context, checkpointResult: { ...context.checkpointResult, _parentExecution: parent } }, + "service.pull", + stepAddress, + ), + attempt = prepareApiJobChildAttempt(context, operation, root.operationKey, root); + if (!attempt) throw Error("Original coordinator job owner required"); + const submission = coordinatorPullSubmissionSchema.parse({ + version: 1, + requestVersion: 2, + expectedExecutor: "ops-agent", + expectedPrincipal: "api", + stepAddress, + operation, + operationKey: root.operationKey, + operationFingerprint: attempt.childOperationFingerprint, + requestedDeadlineAt: attempt.requestedDeadlineAt, + attempt, + }), + entry: ProjectedOperation = { + kind: "service.pull", + operationKey: root.operationKey, + operation, + attempt, + phase: "prepared", + cursor: 0, + events: 0, + bytes: 0, + truncated: false, + terminal: false, + }, + projection = readProjection(context.checkpointResult); + await persistProjection( + context, + { + ...projection, + version: 3, + steps: { + [stepAddress]: { + operationKey: root.operationKey, + requestedDeadlineAt: root.requestedDeadlineAt, + }, + }, + operations: { [root.operationKey]: entry }, + }, + root.operationKey, + stepAddress, + parent, + ); + const retained = readProjection(context.checkpointResult).operations[root.operationKey]; + if (!retained || canonicalJson(retained.attempt) !== canonicalJson(attempt)) + throw new JobObservationPending("recovery_required"); + return submission; +} +/** Only the fixed original MAC candidate/owned-row CAS may release a prepared + * normal pull for its first delivery, including after API owner reclaim. */ +export async function deliverCoordinatorPullSubmission(context: JobContext, candidateId: string) { + const original = await claimCoordinatorPullDelivery(context, candidateId), + projection = readProjection(context.checkpointResult), + entry = projection.operations[original.operationKey]; + if (!entry) throw new JobObservationPending("recovery_required"); + return observeOperation(context, projection, entry, createApiOpsClient(), {}, original.fresh); +} + export function prepareAdminJobOperationRoot( ctx: JobContext, kind: OpsOperationKind, @@ -749,6 +640,20 @@ async function observeOperation( return result; } catch (error) { if (error instanceof ApiOpsError) { + if ( + projected.kind === "configurationMaintenance.settle" && + terminalStatus?.errorClass === "recovery_required" + ) { + // The original accepted verification intent has NO private canonical + // completion. Preserve observation-only recovery; never copy/ACK it as + // a terminal outcome or admit another request/body/budget. + projected.outcome = "unknown"; + projected.phase = "observing"; + projected.terminal = false; + delete projected.errorClass; + delete projected.failureResult; + throw new JobObservationPending("recovery_required"); + } if (!terminalStatus) { if (!projected.operationId && fresh && ADMISSION_REFUSALS.has(error.errorClass)) { projected.phase = "refused"; @@ -967,7 +872,7 @@ export async function cancelAdminJobOperations( ? AbortSignal.any([options.signal, AbortSignal.timeout(2_000)]) : AbortSignal.timeout(2_000); for (const operation of pending) { - if (operation.phase === "refused") { + if (operation.phase === "refused" || operation.phase === "unissued") { outcomes.push("already_terminal"); continue; } diff --git a/apps/api/src/services/admin-job-parent.ts b/apps/api/src/services/admin-job-parent.ts index 91b12faa0..495f76053 100644 --- a/apps/api/src/services/admin-job-parent.ts +++ b/apps/api/src/services/admin-job-parent.ts @@ -1,5 +1,6 @@ import { createHash, createHmac, randomUUID } from "node:crypto"; import { + apiJobChildAttemptSchema, canonicalJson, createControlRootOperationKey, type OpsOperation, @@ -8,23 +9,12 @@ import { opsOperationFingerprint, readOpsTokenFile, } from "@openmapx/core/ops"; -import { z } from "zod"; import type { JobContext } from "./job-runner"; import { createDurableOpsKey } from "./ops-client"; -export const apiJobChildAttemptSchema = z.strictObject({ - version: z.literal(1), - apiJobId: z.uuid(), - apiOwnerVersion: z.number().int().positive().safe(), - parentOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), - parentFingerprint: z.string().regex(/^[a-f0-9]{64}$/), - parentDeadlineAt: z.number().int().nonnegative().safe(), - childOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), - childOperationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), - requestedDeadlineAt: z.number().int().nonnegative().safe(), - attemptId: z.uuid(), -}); -export type ApiJobChildAttempt = z.infer; +export { type ApiJobChildAttempt, apiJobChildAttemptSchema } from "@openmapx/core/ops"; + +import type { ApiJobChildAttempt } from "@openmapx/core/ops"; function parentJobBudget(ctx: JobContext, kind: OpsOperationKind): number { if (ctx.jobType === "system.update") return opsExecutionBudget("system.update").processMs; diff --git a/apps/api/src/services/admin-job-projection.ts b/apps/api/src/services/admin-job-projection.ts new file mode 100644 index 000000000..cac4eb445 --- /dev/null +++ b/apps/api/src/services/admin-job-projection.ts @@ -0,0 +1,337 @@ +/** Pure retained operation decoder. It never admits a child or opens a writer. */ +import { + apiRetentionProofSchema, + canonicalJson, + controlTerminalReceiptDigest, + OPS_KIND_POLICIES, + type OpsFailureResult, + type OpsJobStatus, + type OpsOperation, + type OpsOperationKind, + opsFailureResultSchema, + opsKindRequiresControlAuthority, + opsOperationFingerprint, + opsOperationSchema, + type ProcessDiagnostics, + parseControlRootOperationKey, + processDiagnosticsSchema, +} from "@openmapx/core/ops"; +import { JobObservationPending } from "./admin-job-failure"; +import { type ApiJobChildAttempt, apiJobChildAttemptSchema } from "./admin-job-parent"; +import { ADMIN_JOB_LOG_LIMITS } from "./admin-job-state"; +import { assertConfigurationMaintenanceProjectedStep } from "./configuration-maintenance-step-records"; + +const ADMIN_EVENT_LIMIT = ADMIN_JOB_LOG_LIMITS.events; +const ADMIN_EVENT_BYTE_LIMIT = ADMIN_JOB_LOG_LIMITS.bytes; +const MAX_PROJECTED_OPERATIONS = 64; +const ADMISSION_REFUSALS = new Set([ + "authentication", + "authorization", + "validation", + "expired", + "future", + "busy", + "not_wired", + "host_update_required", + "control_upgrade_required", + "recovery_required", +]); +const OPERATION_KEY = /^opk1_[A-Za-z0-9_-]{16,64}$/; +const OPERATION_ID = /^job1_[A-Za-z0-9_-]{16,64}$/; + +export interface ProjectedOperation { + kind: OpsOperationKind; + operationKey: string; + operationId?: string; + operation?: OpsOperation; + phase: "unissued" | "prepared" | "admitting" | "observing" | "terminal" | "refused"; + nonissuance?: { version: 1; candidateId: string; checkedOwnerVersion: number; checkedAt: number }; + attempt?: ApiJobChildAttempt; + fingerprint?: string; + deadlineAt?: number; + outcome?: "unknown"; + result?: unknown; + failureResult?: OpsFailureResult; + diagnostics?: ProcessDiagnostics; + errorClass?: string; + mirrorStopped?: boolean; + cursor: number; + events: number; + bytes: number; + truncated: boolean; + terminal: boolean; + receiptDelivery?: { + state: "pending" | "confirmed"; + terminalCopyId: string; + checkpointOwnerVersion: number; + terminalDigest: string; + terminalReceipt: OpsJobStatus; + }; +} + +export interface OpsProjection { + version: 1 | 2 | 3; + steps: Record; + eventTotal: number; + byteTotal: number; + truncated: boolean; + operations: Record; +} + +function boundedInteger(value: unknown, maximum: number): value is number { + return Number.isInteger(value) && (value as number) >= 0 && (value as number) <= maximum; +} + +function emptyProjection(): OpsProjection { + return { version: 3, steps: {}, eventTotal: 0, byteTotal: 0, truncated: false, operations: {} }; +} + +export function readProjection(result: unknown): OpsProjection { + if (!result || typeof result !== "object" || Array.isArray(result)) return emptyProjection(); + const candidate = (result as Record).opsProjection; + if (candidate === undefined) return emptyProjection(); + if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) { + throw new JobObservationPending("invalid_identity"); + } + const value = candidate as Record; + if ( + (value.version !== 1 && value.version !== 2 && value.version !== 3) || + !boundedInteger(value.eventTotal, ADMIN_EVENT_LIMIT) || + !boundedInteger(value.byteTotal, ADMIN_EVENT_BYTE_LIMIT) || + typeof value.truncated !== "boolean" || + !value.operations || + typeof value.operations !== "object" || + Array.isArray(value.operations) + ) { + throw new JobObservationPending("invalid_identity"); + } + const entries = Object.entries(value.operations as Record); + if (entries.length > MAX_PROJECTED_OPERATIONS) + throw new JobObservationPending("invalid_identity"); + const operations: Record = {}; + for (const [key, raw] of entries) { + if (!OPERATION_KEY.test(key) || !raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new JobObservationPending("invalid_identity"); + } + const operation = raw as Record; + if ( + operation.operationKey !== key || + (operation.operationId !== undefined && + (typeof operation.operationId !== "string" || !OPERATION_ID.test(operation.operationId))) || + (value.version === 1 && operation.operationId === undefined) || + typeof operation.kind !== "string" || + !boundedInteger(operation.cursor, Number.MAX_SAFE_INTEGER) || + !boundedInteger(operation.events, ADMIN_EVENT_LIMIT) || + !boundedInteger(operation.bytes, ADMIN_EVENT_BYTE_LIMIT) || + typeof operation.truncated !== "boolean" || + typeof operation.terminal !== "boolean" + ) { + throw new JobObservationPending("invalid_identity"); + } + if (!Object.hasOwn(OPS_KIND_POLICIES, operation.kind as string)) + throw new JobObservationPending("invalid_identity"); + if ( + value.version !== 1 && + ![ + "admitting", + "observing", + "terminal", + ...(value.version === 3 ? ["refused", "prepared", "unissued"] : []), + ].includes(operation.phase as string) + ) + throw new JobObservationPending("invalid_identity"); + const checked = + operation.operation === undefined + ? undefined + : opsOperationSchema.safeParse(operation.operation); + if (checked && !checked.success) throw new JobObservationPending("invalid_identity"); + const parsed = checked?.data; + if ( + operation.phase === "prepared" && + (parsed?.kind !== "service.pull" || + !parsed.candidateId || + parsed.serviceId !== "data-manager" || + !operation.attempt || + operation.terminal || + operation.operationId !== undefined || + operation.fingerprint !== undefined || + operation.deadlineAt !== undefined || + operation.result !== undefined || + operation.receiptDelivery !== undefined) + ) + throw new JobObservationPending("invalid_identity"); + if (operation.phase === "unissued") { + const closed = operation.nonissuance as ProjectedOperation["nonissuance"]; + if ( + parsed?.kind !== "service.pull" || + parsed.serviceId !== "data-manager" || + !parsed.candidateId || + !operation.attempt || + operation.terminal !== true || + operation.operationId !== undefined || + operation.fingerprint !== undefined || + operation.deadlineAt !== undefined || + operation.result !== undefined || + operation.failureResult !== undefined || + operation.diagnostics !== undefined || + operation.errorClass !== undefined || + operation.receiptDelivery !== undefined || + operation.outcome !== undefined || + operation.cursor !== 0 || + operation.events !== 0 || + operation.bytes !== 0 || + operation.truncated || + !closed || + Object.keys(closed).length !== 4 || + closed.version !== 1 || + closed.candidateId !== parsed.candidateId || + !Number.isSafeInteger(closed.checkedOwnerVersion) || + closed.checkedOwnerVersion < 1 || + !Number.isSafeInteger(closed.checkedAt) || + closed.checkedAt < 0 + ) + throw new JobObservationPending("invalid_identity"); + } else if (operation.nonissuance !== undefined) + throw new JobObservationPending("invalid_identity"); + if ( + operation.fingerprint !== undefined && + (typeof operation.fingerprint !== "string" || !/^[a-f0-9]{64}$/.test(operation.fingerprint)) + ) + throw new JobObservationPending("invalid_identity"); + if ( + operation.deadlineAt !== undefined && + !boundedInteger(operation.deadlineAt, Number.MAX_SAFE_INTEGER) + ) + throw new JobObservationPending("invalid_identity"); + if ( + operation.failureResult !== undefined && + !opsFailureResultSchema.safeParse(operation.failureResult).success + ) + throw new JobObservationPending("invalid_identity"); + if ( + operation.diagnostics !== undefined && + !processDiagnosticsSchema.safeParse(operation.diagnostics).success + ) + throw new JobObservationPending("invalid_identity"); + if (operation.attempt !== undefined) { + const attempt = apiJobChildAttemptSchema.safeParse(operation.attempt); + if ( + !attempt.success || + (operation.phase === "unissued" && + (operation.nonissuance as ProjectedOperation["nonissuance"])!.checkedOwnerVersion < + attempt.data.apiOwnerVersion) || + !parsed || + attempt.data.childOperationKey !== key || + attempt.data.childOperationFingerprint !== opsOperationFingerprint(parsed) + ) + throw new JobObservationPending("invalid_identity"); + } + if ( + operation.phase === "refused" && + (!operation.terminal || + !ADMISSION_REFUSALS.has(operation.errorClass as string) || + operation.operationId !== undefined || + operation.result !== undefined || + operation.receiptDelivery !== undefined || + operation.fingerprint !== undefined || + operation.deadlineAt !== undefined) + ) + throw new JobObservationPending("invalid_identity"); + if (operation.receiptDelivery !== undefined) { + const d = operation.receiptDelivery as ProjectedOperation["receiptDelivery"]; + try { + if ( + value.version !== 3 || + !operation.attempt || + operation.terminal !== true || + operation.phase !== "terminal" || + !d || + !["pending", "confirmed"].includes(d.state) || + !apiRetentionProofSchema.shape.terminalCopyId.safeParse(d.terminalCopyId).success || + !Number.isSafeInteger(d.checkpointOwnerVersion) || + d.checkpointOwnerVersion < 1 || + controlTerminalReceiptDigest(d.terminalReceipt) !== d.terminalDigest || + d.terminalReceipt.operationId !== operation.operationId || + d.terminalReceipt.kind !== operation.kind || + d.terminalReceipt.operationKey !== key || + canonicalJson(d.terminalReceipt.result ?? null) !== + canonicalJson(operation.result ?? null) || + canonicalJson(d.terminalReceipt.failureResult ?? null) !== + canonicalJson(operation.failureResult ?? null) || + (d.terminalReceipt.errorClass ?? null) !== (operation.errorClass ?? null) || + d.terminalReceipt.fingerprint !== operation.fingerprint || + d.terminalReceipt.deadlineAt !== operation.deadlineAt + ) + throw Error(); + } catch { + throw new JobObservationPending("recovery_required"); + } + } + if (parsed && parsed.kind !== operation.kind) + throw new JobObservationPending("invalid_identity"); + operations[key] = { + ...operation, + ...(parsed ? { operation: parsed } : {}), + phase: + value.version === 1 ? (operation.terminal ? "terminal" : "observing") : operation.phase, + } as ProjectedOperation; + } + const steps: OpsProjection["steps"] = {}; + if (value.version === 3) { + if ( + !value.steps || + typeof value.steps !== "object" || + Array.isArray(value.steps) || + Object.keys(value.steps).length > 64 + ) + throw new JobObservationPending("invalid_identity"); + for (const [address, raw] of Object.entries(value.steps)) { + const step = raw as { operationKey: string; requestedDeadlineAt: number }; + if ( + !OPERATION_KEY.test(address) || + !step || + Object.keys(step).length !== 2 || + !operations[step.operationKey] || + operations[step.operationKey]?.attempt?.requestedDeadlineAt !== step.requestedDeadlineAt + ) + throw new JobObservationPending("invalid_identity"); + try { + if (parseControlRootOperationKey(step.operationKey).notAfter !== step.requestedDeadlineAt) + throw Error("Original managed root deadline changed"); + } catch { + const operation = operations[step.operationKey]!; + try { + assertConfigurationMaintenanceProjectedStep( + result as Record, + step.operationKey, + operation.operation, + operation.attempt, + ); + } catch { + throw new JobObservationPending("invalid_identity"); + } + } + steps[address] = step; + } + } + for (const mapping of Object.values(steps)) { + const entry = operations[mapping.operationKey]; + if (!entry) throw new JobObservationPending("invalid_identity"); + if ( + entry.terminal && + entry.phase !== "refused" && + entry.phase !== "unissued" && + opsKindRequiresControlAuthority(entry.kind) && + !entry.receiptDelivery + ) + throw new JobObservationPending("recovery_required"); + } + return { + version: value.version as 1 | 2 | 3, + steps, + eventTotal: value.eventTotal, + byteTotal: value.byteTotal, + truncated: value.truncated, + operations, + }; +} diff --git a/apps/api/src/services/admin-ops.ts b/apps/api/src/services/admin-ops.ts index 4ddc14b8f..89d2a1fca 100644 --- a/apps/api/src/services/admin-ops.ts +++ b/apps/api/src/services/admin-ops.ts @@ -1,33 +1,16 @@ import { dirname, join } from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; -import { - createDataManagerTransport, - getStatus, - link, - unwrapDataManagerResponse, -} from "@openmapx/core/data-manager-sdk"; -import type { HardlinkApplySummary } from "@openmapx/core/ops"; - import { repoPaths } from "@openmapx/core/server"; -import { - type DockerComposeAction, - dockerComposeAction, - dockerStatus, -} from "../utils/docker-compose"; -import { JobObservationPending } from "./admin-job-failure"; +import { dockerStatus } from "../utils/docker-compose"; import { executeAdminJobOperation } from "./admin-job-ops"; -import { JobOwnershipLost } from "./admin-job-state"; import type { JobContext } from "./job-runner"; -import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { createDurableOpsKey } from "./ops-client"; +import { prepareServiceGeneration } from "./service-preparation"; import { getServiceRegistry } from "./service-registry"; import { applyTrustedConfiguration } from "./trusted-config-operations"; export type { HardlinkApplySummary } from "@openmapx/core/ops"; -const DATA_MANAGER_STARTUP_TIMEOUT_MS = 60_000; -const DATA_MANAGER_POLL_INTERVAL_MS = 2_000; - // Path resolution for an API running on the host (`pnpm dev`), which can read // the deployment directly. The app-api container has no host checkout: its // `/app/infra/docker` holds no configuration, and the deployment reaches it @@ -91,160 +74,7 @@ export async function renderAndPersistCompose(options: { return { revisionId }; } -function dataManagerEnabled(): boolean { - try { - const svc = getServiceRegistry().get("data-manager"); - return !!svc?.enabled; - } catch { - return false; - } -} - -function dataManagerUrlCandidates(): string[] { - const out: string[] = []; - if (process.env.DATA_MANAGER_URL?.trim()) out.push(process.env.DATA_MANAGER_URL.trim()); - // In docker-compose, app-api reaches data-manager over the service DNS name. - out.push("http://data-manager:4000"); - // Keep localhost fallback for non-container local API runs. - out.push("http://localhost:4000"); - return [...new Set(out)]; -} - -async function waitForDataManagerTransport(signal?: AbortSignal): Promise<{ - transport: ReturnType; - url: string; -}> { - const urls = dataManagerUrlCandidates(); - const deadline = Date.now() + DATA_MANAGER_STARTUP_TIMEOUT_MS; - let lastErr: unknown; - - while (Date.now() < deadline) { - for (const url of urls) { - signal?.throwIfAborted(); - const transport = createDataManagerTransport({ - baseUrl: url, - authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), - }); - try { - unwrapDataManagerResponse(await getStatus({ transport, signal })); - signal?.throwIfAborted(); - return { transport, url }; - } catch (err) { - signal?.throwIfAborted(); - lastErr = err; - } - } - await delay(DATA_MANAGER_POLL_INTERVAL_MS, undefined, { signal }); - } - - const detail = (lastErr as Error | undefined)?.message ?? String(lastErr ?? "unknown error"); - throw new Error( - `data-manager did not become reachable within ${DATA_MANAGER_STARTUP_TIMEOUT_MS / 1000}s (${urls.join(", ")}): ${detail}`, - ); -} - -/** - * Link consumer data directories to their producers as the configuration - * generation just applied describes. The plan lives in that generation on the - * host, which only the operations agent can read; data-manager applies it, - * because its mounts confine the links and prunes to the data root. - */ -export async function applyGenerationHardlinks(opts: { - revisionId: string; - operationIdentity: string; - log?: (msg: string) => Promise | void; - signal?: AbortSignal; - ctx?: JobContext; -}): Promise { - const signal = opts.ctx?.signal ?? opts.signal; - signal?.throwIfAborted(); - const attempts = opts.ctx?.checkpointResult?._hardlinkAttempts as - | Record - | undefined; - const prior = attempts?.[opts.operationIdentity]; - if (prior) { - if (prior.revisionId !== opts.revisionId || prior.phase !== "complete" || !prior.result) - throw new JobObservationPending("recovery_required"); - return prior.result; - } - if (opts.ctx?.observationOnly) throw new JobObservationPending("recovery_required"); - const { plan } = await executeAndWait( - createApiOpsClient(), - { kind: "configuration.hardlinks.inspect", revisionId: opts.revisionId }, - createDurableOpsKey("admin-job.hardlinks.inspect", opts.operationIdentity), - { signal }, - ); - if (plan.length === 0) { - return { applied: true, linked: 0, skipped: 0, pruned: 0, entries: 0 }; - } - if (!dataManagerEnabled()) { - throw new Error( - `Hardlink plan contains ${plan.length} entr${plan.length === 1 ? "y" : "ies"}, but service "data-manager" is disabled. Enable data-manager before starting services from Admin, or apply links via CLI first.`, - ); - } - - await opts.log?.("Ensuring data-manager is running for hardlink apply..."); - signal?.throwIfAborted(); - const coordinatorKey = createDurableOpsKey( - "admin-job.hardlinks.start", - `${opts.operationIdentity}:data-manager`, - ); - if (opts.ctx) { - await executeAdminJobOperation( - opts.ctx, - { kind: "service.start", serviceId: "data-manager" }, - "admin-job.hardlinks.start", - { operationKey: coordinatorKey }, - ); - } else { - const dmStart = await dockerComposeAction("data-manager", "start", { - operationKey: coordinatorKey, - signal, - }); - if (dmStart.exitCode !== 0) - throw new Error(`docker compose up data-manager exited with ${dmStart.exitCode}`); - } - - await opts.log?.(`Applying hardlink plan (${plan.length} entries, prune enabled)...`); - const { transport } = await waitForDataManagerTransport(signal); - signal?.throwIfAborted(); - const checkpoint = async (phase: "attempted" | "complete", result?: HardlinkApplySummary) => { - if (!opts.ctx) return; - await opts.ctx.checkpoint({ - ...opts.ctx.checkpointResult, - _hardlinkAttempts: { - ...(opts.ctx.checkpointResult?._hardlinkAttempts as Record), - [opts.operationIdentity]: { - revisionId: opts.revisionId, - coordinatorKey, - phase, - ...(result ? { result } : {}), - }, - }, - }); - }; - // This records only the current caller's exact attempt, not a receiver lease - // or containment proof. Unknown remote work is never submitted again. - await checkpoint("attempted"); - signal?.throwIfAborted(); - try { - const result = unwrapDataManagerResponse( - await link({ plan, prune: true }, { transport, signal }), - ); - const summary: HardlinkApplySummary = { - applied: true, - entries: plan.length, - via: "data-manager", - ...result, - }; - await checkpoint("complete", summary); - return summary; - } catch (error) { - if (error instanceof JobOwnershipLost) throw error; - if (opts.ctx) throw new JobObservationPending("recovery_required"); - throw error; - } -} +export { applyGenerationHardlinks } from "./service-preparation"; // Service control (job handlers) — registry-authorized + routed through the // docker-compose helper that targets the generated compose file. @@ -267,7 +97,7 @@ function assertKnownService(service: string, requireEnabled = true) { async function serviceComposeAction( service: string, - action: Exclude, + action: "stop" | "restart", ctx: JobContext, ): Promise<{ exitCode: number }> { assertKnownService(service, action !== "stop"); @@ -292,61 +122,53 @@ async function serviceComposeAction( export async function serviceStart(service: string, ctx: JobContext): Promise { assertKnownService(service); - await ctx.log(`Rendering compose for latest config...`); - const { revisionId } = await renderAndPersistCompose({ - ctx, - operationKey: createDurableOpsKey("admin-job.config.render", `${ctx.jobId}:${service}`), - signal: ctx.signal, + const prepared = await prepareServiceGeneration(ctx, `start:${service}`, { + coordinatorMode: "ensure-running", }); - const hardlinks = await applyGenerationHardlinks({ - ctx, - revisionId, - operationIdentity: `${ctx.jobId}:${service}`, - log: (m) => ctx.log(m), - signal: ctx.signal, + await ctx.checkpoint({ + ...ctx.checkpointResult, + _completion: { + operationKey: createDurableOpsKey("admin-job.service.start", `${ctx.jobId}:${service}`), + result: { service, action: "start" }, + }, }); - await ctx.log( - `Hardlinks applied (${hardlinks.linked} linked, ${hardlinks.skipped} already linked, ${hardlinks.pruned} pruned)`, + await executeAdminJobOperation( + ctx, + { + kind: "service.start", + serviceId: service, + expectedConfigurationRevisionId: prepared.revisionId, + }, + "admin-job.service.start", + { durableIdentity: service }, ); - await ctx.log(`Starting ${service}...`); - const r = await serviceComposeAction(service, "start", ctx); - if (r.exitCode !== 0) throw new Error(`docker compose up exited with ${r.exitCode}`); - await ctx.log(`${service} started.`); } -export async function serviceApply(service: string, ctx: JobContext): Promise { +export async function serviceApply(service: string, ctx: JobContext) { assertKnownService(service); - await ctx.log(`Rendering compose for latest config + secrets...`); - const configurationKind = ctx.payload.configurationKind; - const { revisionId } = - configurationKind === "vault" - ? await applyTrustedConfiguration({ - ctx, - kind: "vault.apply", - serviceId: service, - operationKey: createDurableOpsKey("admin-job.vault.apply", `${ctx.jobId}:${service}`), - signal: ctx.signal, - }) - : await renderAndPersistCompose({ - ctx, - operationKey: createDurableOpsKey("admin-job.config.apply", `${ctx.jobId}:${service}`), - signal: ctx.signal, - }); - const hardlinks = await applyGenerationHardlinks({ - ctx, - revisionId, - operationIdentity: `${ctx.jobId}:${service}`, - log: (m) => ctx.log(m), - signal: ctx.signal, + if (service === "data-manager") + return (await import("./service-preparation")).applyCoordinatorService(ctx, null); + const prepared = await prepareServiceGeneration(ctx, `apply:${service}`, { + coordinatorMode: "ensure-running", + }); + await ctx.checkpoint({ + ...ctx.checkpointResult, + _completion: { + operationKey: createDurableOpsKey("admin-job.service.recreate", `${ctx.jobId}:${service}`), + result: { service, action: "recreate" }, + }, }); - await ctx.log( - `Hardlinks applied (${hardlinks.linked} linked, ${hardlinks.skipped} already linked, ${hardlinks.pruned} pruned)`, + await executeAdminJobOperation( + ctx, + { + kind: "service.recreate", + serviceId: service, + expectedConfigurationRevisionId: prepared.revisionId, + }, + "admin-job.service.recreate", + { durableIdentity: service }, ); - await ctx.log(`Applying rendered configuration to ${service}...`); - const r = await serviceComposeAction(service, "recreate", ctx); - if (r.exitCode !== 0) - throw new Error(`docker compose up --force-recreate exited with ${r.exitCode}`); - await ctx.log(`${service} configuration applied.`); + return { changed: true as const, revisionId: prepared.revisionId }; } export async function serviceStop(service: string, ctx: JobContext): Promise { diff --git a/apps/api/src/services/administrative-configuration-context.ts b/apps/api/src/services/administrative-configuration-context.ts new file mode 100644 index 000000000..fb4cb96c8 --- /dev/null +++ b/apps/api/src/services/administrative-configuration-context.ts @@ -0,0 +1,66 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type * as schema from "../db/schema"; +import type { getAllIntegrations } from "../integration-host"; +import type { getServiceRegistry } from "./service-registry"; + +const readBrand: unique symbol = Symbol("administrative configuration read"); +export interface AdministrativeConfigurationReadContext { + readonly [readBrand]: true; + readonly services: readonly ReturnType["list"]>[number][]; + readonly integrations: readonly ReturnType[number][]; + readonly env: Readonly; + readonly serviceRows: readonly (typeof schema.serviceConfig.$inferSelect)[]; + readonly integrationRows: readonly (typeof schema.integrationConfig.$inferSelect)[]; + readonly serviceSecretRows: readonly (typeof schema.serviceSecret.$inferSelect)[]; + readonly integrationSecretRows: readonly (typeof schema.integrationSecret.$inferSelect)[]; + /** Only the committed-candidate reload scope supplies an already resolved snapshot. */ + readonly integrationReloadSnapshot?: Readonly< + Record>> + >; +} +const readers = new WeakSet(); +const scopes = new AsyncLocalStorage(); +const files = new WeakMap>(); +export function openAdministrativeConfigurationRead( + input: Omit, +): AdministrativeConfigurationReadContext { + const context = Object.freeze({ ...input, [readBrand]: true as const }); + readers.add(context); + files.set(context, new Map()); + return context; +} +export function closeAdministrativeConfigurationRead( + context: AdministrativeConfigurationReadContext, +): void { + readers.delete(context); +} +export function assertAdministrativeConfigurationRead( + context: AdministrativeConfigurationReadContext, +): void { + if (!readers.has(context) || scopes.getStore() !== context) + throw new Error("Configuration acquisition unavailable"); +} +export function recordAdministrativeConfigurationFile( + context: AdministrativeConfigurationReadContext, + path: string, + identity: unknown, +): void { + assertAdministrativeConfigurationRead(context); + const observed = files.get(context)!; + if (observed.has(path) && JSON.stringify(observed.get(path)) !== JSON.stringify(identity)) + throw new Error("Configuration acquisition unavailable"); + observed.set(path, identity); +} +export function administrativeConfigurationFileObservations( + context: AdministrativeConfigurationReadContext, +) { + assertAdministrativeConfigurationRead(context); + return [...files.get(context)!.entries()].sort(([a], [b]) => a.localeCompare(b)); +} +export function withAdministrativeConfigurationReadScope( + context: AdministrativeConfigurationReadContext, + run: () => Promise, +): Promise { + if (!readers.has(context)) throw new Error("Configuration acquisition unavailable"); + return scopes.run(context, run); +} diff --git a/apps/api/src/services/administrative-configuration-writer-owner.ts b/apps/api/src/services/administrative-configuration-writer-owner.ts new file mode 100644 index 000000000..406c71ce1 --- /dev/null +++ b/apps/api/src/services/administrative-configuration-writer-owner.ts @@ -0,0 +1,369 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { setTimeout as delay } from "node:timers/promises"; +import { getTableColumns, sql as query } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { db, sql } from "../db"; +import * as schema from "../db/schema"; +import { getAllIntegrations } from "../integration-host"; +import { JobOwnershipLost, jobOwnership } from "./admin-job-state"; +import type { JobContext } from "./job-runner"; +import type { ValidatedConfigurationCandidate } from "./prospective-configuration"; +import { getServiceRegistry } from "./service-registry"; + +const unavailable = () => new Error("Configuration acquisition unavailable"); +const rejected = () => new Error("Configuration writer unavailable"); + +import { + type AdministrativeConfigurationReadContext, + closeAdministrativeConfigurationRead, + openAdministrativeConfigurationRead, + withAdministrativeConfigurationReadScope, +} from "./administrative-configuration-context"; + +export { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./administrative-configuration-context"; + +const writerBrand: unique symbol = Symbol("administrative configuration writer"); +const participantBrand: unique symbol = Symbol("configuration participant"); +export type ConfigurationWriteTransaction = Pick< + typeof db, + "select" | "insert" | "update" | "delete" | "execute" +>; +export class ConfigurationWriteRolledBack extends Error { + constructor(readonly failure: unknown) { + super("Configuration transaction rolled back"); + } +} +export class ConfigurationWriteCommitUnknown extends Error { + constructor(readonly failure: unknown) { + super("Configuration transaction outcome unknown"); + } +} +export interface AdministrativeConfigurationWriter { + readonly [writerBrand]: true; + readonly actorId: string; + readonly operationKey: string; + readonly integrationIds: readonly string[]; +} +export interface ConfigurationCommitParticipant { + readonly [participantBrand]: true; + readonly kind: + | "integration-binding" + | "configuration-migration" + | "credential-import" + | "transit-credentials"; + readonly mode: "preview" | "apply"; + readonly actorId: string; + readonly operationKey: string; + readonly managedServiceSecretKeys: + | readonly [{ readonly serviceId: "data-manager"; readonly key: "TRANSITOUS_API_KEYS" }] + | readonly []; + authorize(writer: AdministrativeConfigurationWriter): Promise; + verify( + candidate: ValidatedConfigurationCandidate, + writer: AdministrativeConfigurationWriter, + ): Promise; + commit( + candidate: ValidatedConfigurationCandidate, + writer: AdministrativeConfigurationWriter, + tx: ConfigurationWriteTransaction, + ): Promise; +} +interface WriterState { + active: boolean; + writer: AdministrativeConfigurationWriter; + connection: typeof db; + job: JobContext; + participant?: ConfigurationCommitParticipant; + transactionContext?: AdministrativeConfigurationReadContext; + transactionActive: boolean; + transactionMode?: "read" | "write"; +} +const scope = new AsyncLocalStorage(); +const jobs = new AsyncLocalStorage(); + +export function currentAdministrativeConfigurationWriter(): AdministrativeConfigurationWriter { + const state = scope.getStore(); + if (!state?.active || state.job.signal.aborted) throw rejected(); + return state.writer; +} +export function administrativeConfigurationState(writer: AdministrativeConfigurationWriter) { + const state = scope.getStore(); + if (!state?.active || state.writer !== writer || state.job.signal.aborted) throw rejected(); + return state; +} +/** Internal job binding, independent of caller-selected overlay purpose. */ +export function withAdministrativeConfigurationJob( + job: JobContext, + run: () => Promise, +): Promise { + const current = jobs.getStore(); + if (current && current !== job) throw rejected(); + if (!job.execution) throw rejected(); + return jobs.run(job, run); +} + +async function capture( + tx: ConfigurationWriteTransaction, +): Promise { + try { + const immutable = (value: T): T => { + if (value && typeof value === "object") { + for (const child of Object.values(value)) immutable(child); + Object.freeze(value); + } + return value; + }; + // Registry reload/selection may replace or mutate live metadata while this + // transaction awaits I/O. Freeze independent JSON observations, not the + // serving integration's live providers or shutdown handlers. + const services = Object.freeze( + getServiceRegistry() + .list() + .map((service) => + Object.freeze({ ...service, manifest: immutable(structuredClone(service.manifest)) }), + ), + ); + const integrations = Object.freeze( + getAllIntegrations().map((integration) => + Object.freeze({ + ...integration, + manifest: immutable(structuredClone(integration.manifest)), + config: immutable(structuredClone(integration.config)), + }), + ), + ); + // Sequential reads belong to this transaction, including disabled and orphan rows. + const serviceRows = await tx + .select({ ...getTableColumns(schema.serviceConfig), _rowVersion: query`xmin::text` }) + .from(schema.serviceConfig); + const integrationRows = await tx + .select({ + ...getTableColumns(schema.integrationConfig), + _rowVersion: query`xmin::text`, + }) + .from(schema.integrationConfig); + const serviceSecretRows = await tx + .select({ ...getTableColumns(schema.serviceSecret), _rowVersion: query`xmin::text` }) + .from(schema.serviceSecret); + const integrationSecretRows = await tx + .select({ + ...getTableColumns(schema.integrationSecret), + _rowVersion: query`xmin::text`, + }) + .from(schema.integrationSecret); + const context = openAdministrativeConfigurationRead({ + services, + integrations, + env: Object.freeze({ ...process.env }), + serviceRows, + integrationRows, + serviceSecretRows, + integrationSecretRows, + }); + return context; + } catch { + throw unavailable(); + } +} +export async function withAdministrativeConfigurationRead( + options: { signal?: AbortSignal }, + run: (context: AdministrativeConfigurationReadContext) => Promise, +): Promise { + options.signal?.throwIfAborted(); + const state = scope.getStore(); + if (state?.transactionContext) + return withAdministrativeConfigurationReadScope(state.transactionContext, () => + run(state.transactionContext!), + ); + if (state) { + if (state.transactionActive) throw unavailable(); + state.transactionActive = true; + state.transactionMode = "read"; + const connection = state.connection; + let context: AdministrativeConfigurationReadContext | undefined; + let started = false; + try { + await connection.execute(query`BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY`); + started = true; + context = await capture(connection); + state.transactionContext = context; + const result = await withAdministrativeConfigurationReadScope(context, () => run(context!)); + await connection.execute(query`COMMIT`); + return result; + } catch (error) { + if (started) await connection.execute(query`ROLLBACK`); + throw error; + } finally { + state.transactionContext = undefined; + if (context) closeAdministrativeConfigurationRead(context); + state.transactionActive = false; + state.transactionMode = undefined; + } + } + return db.transaction( + async (tx) => { + const context = await capture(tx); + try { + options.signal?.throwIfAborted(); + return await withAdministrativeConfigurationReadScope(context, () => run(context)); + } finally { + closeAdministrativeConfigurationRead(context); + } + }, + { isolationLevel: "repeatable read", accessMode: "read only" }, + ); +} +export async function withConfigurationWriteTransaction( + writer: AdministrativeConfigurationWriter, + run: ( + tx: ConfigurationWriteTransaction, + context: AdministrativeConfigurationReadContext, + ) => Promise, +): Promise { + const state = administrativeConfigurationState(writer); + if (state.transactionActive) throw Error("Configuration transaction already active"); + state.transactionActive = true; + state.transactionMode = "write"; + const tx = state.connection; + let started = false, + commitAttempted = false; + try { + await tx.execute(query`BEGIN`); + started = true; + const [job] = await tx + .select() + .from(schema.adminJob) + .where(jobOwnership(state.job.jobId, state.job.execution)) + .for("update"); + if (!job) throw new JobOwnershipLost(); + const context = await capture(tx); + state.transactionContext = context; + let value: T; + try { + value = await withAdministrativeConfigurationReadScope(context, () => run(tx, context)); + } finally { + state.transactionContext = undefined; + closeAdministrativeConfigurationRead(context); + } + commitAttempted = true; + await tx.execute(query`COMMIT`); + return value; + } catch (error) { + if (commitAttempted) throw new ConfigurationWriteCommitUnknown(error); + if (started) { + try { + await tx.execute(query`ROLLBACK`); + } catch (rollback) { + throw new ConfigurationWriteCommitUnknown(rollback); + } + throw new ConfigurationWriteRolledBack(error); + } + throw new ConfigurationWriteCommitUnknown(error); + } finally { + state.transactionActive = false; + state.transactionMode = undefined; + } +} + +export async function withAdministrativeConfigurationWriter( + options: { + actorId: string; + operationKey: string; + integrationIds: readonly string[]; + signal?: AbortSignal; + }, + run: (writer: AdministrativeConfigurationWriter) => Promise, +): Promise { + options.signal?.throwIfAborted(); + const existing = scope.getStore(); + if (existing) { + if ( + !existing.active || + existing.writer.actorId !== options.actorId || + existing.writer.operationKey !== options.operationKey || + options.integrationIds.some((id) => !existing.writer.integrationIds.includes(id)) + ) + throw rejected(); + return run(existing.writer); + } + const job = jobs.getStore(); + if (!job?.execution || !options.actorId || !options.operationKey) throw rejected(); + const ids = [...new Set(options.integrationIds)].sort(); + if (ids.length > 256 || ids.some((id) => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id))) throw rejected(); + const reserved = await sql.reserve(); + const locks: number[] = []; + let global = false; + try { + // postgres-js reserves query execution but omits the parent's parser metadata. + // Drizzle needs that metadata; it must still execute through this reserved SQL. + const connection = drizzle(Object.assign(reserved, { options: sql.options }), { schema }); + const deadline = Date.now() + 5_000; + while (!global) { + options.signal?.throwIfAborted(); + const [row] = await reserved`select pg_try_advisory_lock(${0x4f4d58}, 1) as acquired`; + global = row?.acquired === true; + if (!global) { + if (Date.now() >= deadline) throw rejected(); + await delay(20, undefined, { signal: options.signal }); + } + } + // Existing integration namespace; acquire only after the global writer. + for (const id of ids) { + const [row] = await reserved`select hashtext(${id}) as key`; + const key = Number(row?.key); + while (true) { + const [lock] = await reserved`select pg_try_advisory_lock(${0x494e54}, ${key}) as acquired`; + if (lock?.acquired === true) break; + if (Date.now() >= deadline) throw rejected(); + await delay(20, undefined, { signal: options.signal }); + } + locks.push(key); + } + const unresolved = await connection + .select({ id: schema.adminJob.id, result: schema.adminJob.result }) + .from(schema.adminJob) + .where( + query`${schema.adminJob.result}->'_configurationCandidate' IS NOT NULL AND coalesce(${schema.adminJob.result}->'_configurationCandidate'->>'phase', 'unknown') NOT IN ('settled','discarded')`, + ); + if ( + unresolved.some( + (row) => + row.id !== job.jobId || + (row.result as { _configurationCandidate?: { operationKey?: string } }) + ?._configurationCandidate?.operationKey !== options.operationKey, + ) + ) + throw rejected(); + const [owned] = await connection + .select({ id: schema.adminJob.id }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)); + if (!owned) throw new JobOwnershipLost(); + const writer: AdministrativeConfigurationWriter = Object.freeze({ + [writerBrand]: true as const, + actorId: options.actorId, + operationKey: options.operationKey, + integrationIds: Object.freeze(ids), + }); + const state: WriterState = { active: true, writer, connection, job, transactionActive: false }; + try { + return await scope.run(state, () => run(writer)); + } finally { + state.active = false; + } + } finally { + try { + for (const key of locks.reverse()) + await reserved`select pg_advisory_unlock(${0x494e54}, ${key})`; + } finally { + try { + if (global) await reserved`select pg_advisory_unlock(${0x4f4d58}, 1)`; + } finally { + reserved.release(); + } + } + } +} diff --git a/apps/api/src/services/administrative-configuration-writer.ts b/apps/api/src/services/administrative-configuration-writer.ts new file mode 100644 index 000000000..60e9643b4 --- /dev/null +++ b/apps/api/src/services/administrative-configuration-writer.ts @@ -0,0 +1,2 @@ +export * from "./administrative-configuration-writer-owner"; +export { withConfigurationCommitParticipant } from "./configuration-commit-participant"; diff --git a/apps/api/src/services/configuration-commit-participant.ts b/apps/api/src/services/configuration-commit-participant.ts new file mode 100644 index 000000000..d841f4c17 --- /dev/null +++ b/apps/api/src/services/configuration-commit-participant.ts @@ -0,0 +1,35 @@ +/** Closed participant dispatch; writer ownership does not depend on handlers. */ +import { + type AdministrativeConfigurationWriter, + administrativeConfigurationState, + type ConfigurationCommitParticipant, +} from "./administrative-configuration-writer-owner"; + +const rejected = () => new Error("Configuration writer unavailable"); + +export async function withConfigurationCommitParticipant( + writer: AdministrativeConfigurationWriter, + participant: ConfigurationCommitParticipant, + run: () => Promise, +): Promise { + const state = administrativeConfigurationState(writer); + const { isLegacyIntegrationImportParticipant } = await import( + "./configuration-migration/legacy-integration-import-intent" + ); + // Exact installed built-in issuer, never a registration or caller callback. + if ( + !isLegacyIntegrationImportParticipant(participant, writer) || + participant.actorId !== writer.actorId || + participant.operationKey !== writer.operationKey || + (state.participant && state.participant !== participant) + ) + throw rejected(); + if (state.participant) return run(); + state.participant = participant; + try { + await participant.authorize(writer); + return await run(); + } finally { + state.participant = undefined; + } +} diff --git a/apps/api/src/services/configuration-hardlinks.ts b/apps/api/src/services/configuration-hardlinks.ts new file mode 100644 index 000000000..8e37ca20b --- /dev/null +++ b/apps/api/src/services/configuration-hardlinks.ts @@ -0,0 +1,58 @@ +import type { CoordinatorMode, HardlinkApplySummary } from "@openmapx/core/ops"; +import { JobObservationPending } from "./admin-job-failure"; +import { executeAdminJobOperation } from "./admin-job-ops"; +import type { JobContext } from "./job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; + +export async function applyGenerationHardlinks(options: { + revisionId: string; + operationIdentity: string; + coordinatorMode: CoordinatorMode; + signal?: AbortSignal; + ctx?: JobContext; + log?: (message: string) => Promise | void; +}): Promise { + const ctx = options.ctx; + if (!ctx) throw Error("A retained configuration job is required"); + const legacy = ( + ctx.checkpointResult?._hardlinkAttempts as + | Record + | undefined + )?.[options.operationIdentity]; + if (legacy) { + if (legacy.revisionId !== options.revisionId || legacy.phase !== "complete" || !legacy.result) + throw new JobObservationPending("recovery_required"); + return legacy.result; + } + const { plan } = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.hardlinks.inspect", revisionId: options.revisionId }, + createDurableOpsKey("admin-job.hardlinks.inspect", options.operationIdentity), + { signal: ctx.signal }, + ); + if (!plan.length) + return { applied: true, entries: 0, linked: 0, skipped: 0, pruned: 0, via: "none" }; + if (options.coordinatorMode === "ensure-running") { + await options.log?.("Ensuring data-manager is running for prepared hardlinks..."); + await executeAdminJobOperation( + ctx, + { + kind: "service.start", + serviceId: "data-manager", + expectedConfigurationRevisionId: options.revisionId, + }, + "admin-job.hardlinks.start", + { durableIdentity: options.operationIdentity }, + ); + } + return executeAdminJobOperation( + ctx, + { + kind: "configuration.hardlinks.apply", + revisionId: options.revisionId, + coordinatorMode: options.coordinatorMode, + }, + "admin-job.hardlinks.apply", + { durableIdentity: options.operationIdentity }, + ); +} diff --git a/apps/api/src/services/configuration-input.ts b/apps/api/src/services/configuration-input.ts new file mode 100644 index 000000000..3b35b8f4b --- /dev/null +++ b/apps/api/src/services/configuration-input.ts @@ -0,0 +1,81 @@ +/** Strict coherent input collector; no job dispatch or candidate mutation. */ +import { + type TrustedConfigurationPayload, + trustedConfigurationPayloadSchema, +} from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; +import { resolveIntegrationConfigWithSourcesStrict } from "../integration-config"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, + withAdministrativeConfigurationRead, +} from "./administrative-configuration-writer"; +import { projectIntegrationConfig } from "./integration-config-projection"; +import { resolveAllServiceConfigsStrict } from "./service-config-resolver"; +import { resolveServiceVaultSecretsStrict } from "./service-secrets"; + +/** + * Everything a render needs from the database, for every registered service: + * a generation keeps what was saved for services that are not enabled, so + * enabling one later, from here or the CLI, renders it with its settings. + */ +export async function buildConfigurationInput( + context?: AdministrativeConfigurationReadContext, +): Promise { + if (!context) return withAdministrativeConfigurationRead({}, buildConfigurationInput); + assertAdministrativeConfigurationRead(context); + const services = context.services; + const serviceConfigMap = await resolveAllServiceConfigsStrict( + services.map((service) => ({ + id: service.manifest.id, + configSchema: service.manifest.configSchema, + containerEnv: service.manifest.container.environment, + proxyHostConfigKey: service.manifest.exposure?.proxy?.host?.configKey, + })), + context, + ); + const serviceSecrets = await Promise.all( + services.map(async (service) => ({ + serviceId: service.manifest.id, + values: await resolveServiceVaultSecretsStrict(service.manifest.id, context), + })), + ); + const integrations = context.integrations; + const serviceIds = new Set(services.map((service) => service.manifest.id)); + const integrationIds = new Set(integrations.map((integration) => integration.id)); + if ( + context.serviceSecretRows.some((row) => !serviceIds.has(row.serviceId)) || + context.integrationRows.some((row) => !integrationIds.has(row.integrationId)) || + context.integrationSecretRows.some((row) => !integrationIds.has(row.integrationId)) + ) + throw new Error("Configuration acquisition unavailable"); + const integrationConfigs = await Promise.all( + integrations.map(async (integration) => { + const resolved = await resolveIntegrationConfigWithSourcesStrict( + integration.manifest, + integration.directory, + context, + ); + return { + integrationId: integration.id, + values: projectIntegrationConfig( + integration.manifest.configSchema as Record | undefined, + Object.fromEntries(Object.entries(resolved).map(([key, entry]) => [key, entry.value])), + ), + }; + }), + ); + return trustedConfigurationPayloadSchema.parse({ + domain: envString("DOMAIN", "localhost"), + serviceConfigs: services.map((service) => { + const envKeys = serviceConfigMap.envKeys.get(service.manifest.id); + return { + serviceId: service.manifest.id, + values: serviceConfigMap.values.get(service.manifest.id) ?? {}, + ...(envKeys ? { envKeys } : {}), + }; + }), + integrationConfigs, + serviceSecrets, + }); +} diff --git a/apps/api/src/services/configuration-maintenance-decision.ts b/apps/api/src/services/configuration-maintenance-decision.ts new file mode 100644 index 000000000..26eadbd42 --- /dev/null +++ b/apps/api/src/services/configuration-maintenance-decision.ts @@ -0,0 +1,119 @@ +/** Fixed non-secret DB decision data. Decoding/checksums confer no authority; + * producers and delivery read these only under the actual owned job row lock. */ +import { createHash } from "node:crypto"; +import { + type ConfigurationMaintenanceDecision, + canonicalJson, + configurationMaintenanceDecisionSchema, +} from "@openmapx/core/ops"; +import { eq, sql } from "drizzle-orm"; +import { z } from "zod"; +import * as schema from "../db/schema"; +import type { ConfigurationWriteTransaction } from "./administrative-configuration-writer"; + +const version = z.string().regex(/^\d{1,10}$/), + id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +export const configurationSavedRowSchema = z + .strictObject({ + kind: z.enum(["service-config", "integration-config", "service-secret", "integration-secret"]), + ownerId: id, + key: z.string().min(1).max(128).optional(), + rowId: z.string().min(1).max(128), + rowVersion: version.nullable(), + previousRowVersion: version.nullable(), + state: z.enum(["present", "absent"]), + }) + .refine( + (row) => + row.kind.endsWith("secret") === (row.key !== undefined) && + (row.state === "present" ? row.rowVersion !== null : row.rowVersion === null), + ); +export type ConfigurationSavedRow = z.infer; +/** Written ONLY by the fixed desired-transaction catch after the actual SAME + * connection's awaited ROLLBACK acknowledgement. No error/value is serialized. */ +export const configurationMaintenanceRollbackCheckpointSchema = z.strictObject({ + version: z.literal(1), + disposition: z.literal("known_rolled_back"), + rollbackId: z.uuid(), + candidateId: z.uuid(), + maintenanceId: z.uuid(), + prepareOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + prepareFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + planDigest: z.string().regex(/^[a-f0-9]{64}$/), + checkpointVersion: z.number().int().positive().safe(), +}); +const parent = z.strictObject({ + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), +}); +export const configurationMaintenanceDecisionCheckpointSchema = z + .strictObject({ + version: z.literal(1), + operation: configurationMaintenanceDecisionSchema, + rowVersions: z + .array(configurationSavedRowSchema) + .max(512) + .refine((rows) => Buffer.byteLength(canonicalJson(rows)) <= 65536), + attempt: z.enum(["not_attempted", "attempted"]), + delivery: z + .strictObject({ + stepAddress: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + root: z.strictObject({ + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + parent, + }), + }) + .optional(), + }) + .refine((value) => value.operation.decision !== "abandon" || value.rowVersions.length === 0); +export function configurationMaintenanceDbDecisionDigest( + operation: Omit, + rowVersions: readonly ConfigurationSavedRow[], +): string { + const { kind: _kind, expectedSequence: _sequence, ...tuple } = operation; + return createHash("sha256") + .update("openmapx-configuration-maintenance-db-decision-v1\0") + .update(canonicalJson({ ...tuple, rowVersions })) + .digest("hex"); +} +/** Actual row versions on the SAME physical writer transaction. No desired + * values or hashes of secret material enter this receipt or its checksum. */ +export async function assertConfigurationSavedRows( + tx: ConfigurationWriteTransaction, + input: readonly ConfigurationSavedRow[], +): Promise { + const rows = z.array(configurationSavedRowSchema).max(512).parse(input); + const seen = new Set(); + for (const row of rows) { + const key = `${row.kind}:${row.ownerId}:${row.key ?? ""}`; + if (seen.has(key)) throw Error("Configuration saved row ownership changed"); + seen.add(key); + const table = + row.kind === "service-config" + ? schema.serviceConfig + : row.kind === "integration-config" + ? schema.integrationConfig + : row.kind === "service-secret" + ? schema.serviceSecret + : schema.integrationSecret; + const ownerColumn = "serviceId" in table ? table.serviceId : table.integrationId, + where = + row.key !== undefined && "key" in table + ? sql`${ownerColumn} = ${row.ownerId} AND ${table.key} = ${row.key}` + : eq(ownerColumn, row.ownerId), + actual = await tx + .select({ id: table.id, version: sql`xmin::text` }) + .from(table) + .where(where); + if ( + row.state === "absent" + ? actual.length !== 0 + : actual.length !== 1 || + actual[0]!.id !== row.rowId || + actual[0]!.version !== row.rowVersion + ) + throw Error("Configuration saved row ownership changed"); + } +} diff --git a/apps/api/src/services/configuration-maintenance-settle-records.ts b/apps/api/src/services/configuration-maintenance-settle-records.ts new file mode 100644 index 000000000..2c6732d57 --- /dev/null +++ b/apps/api/src/services/configuration-maintenance-settle-records.ts @@ -0,0 +1,24 @@ +/** Non-secret original request checkpoint; parsing confers no authority. */ + +import { configurationMaintenanceSettleSchema } from "@openmapx/core/ops"; +import { z } from "zod"; + +const key = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + epoch = z.number().int().nonnegative().safe(); +export const configurationMaintenanceSettleCheckpointSchema = z.strictObject({ + version: z.literal(1), + operation: configurationMaintenanceSettleSchema, + attempt: z.enum(["not_attempted", "attempted"]), + delivery: z.strictObject({ + stepAddress: key, + root: z.strictObject({ + operationKey: key, + requestedDeadlineAt: epoch, + parent: z.strictObject({ + operationKey: key, + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: epoch, + }), + }), + }), +}); diff --git a/apps/api/src/services/configuration-maintenance-step-records.ts b/apps/api/src/services/configuration-maintenance-step-records.ts new file mode 100644 index 000000000..853739ef9 --- /dev/null +++ b/apps/api/src/services/configuration-maintenance-step-records.ts @@ -0,0 +1,205 @@ +/** Closed original-root API step preparation. Structural parsing is only data + * validation; fresh submission additionally reads the actual owned DB row. */ + +import { controlChildOperationKey } from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + configurationMaintenanceReceiptSchema, + type OpsOperation, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { z } from "zod"; +import { apiJobChildAttemptSchema } from "./admin-job-parent"; + +const key = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + epoch = z.number().int().nonnegative().safe(); +const parent = z.strictObject({ + operationKey: key, + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: epoch, +}); +export const descriptor = z.strictObject({ + ordinal: z.number().int().min(0).max(31), + operationKey: key, + requestedDeadlineAt: epoch, + parent, +}); +export const records = z + .record(z.string().regex(/^(?:[0-9]|[12][0-9]|3[01])$/), descriptor) + .refine((value) => Object.keys(value).length <= 32); +export function observedStep( + result: Record, + operationKey: string, + operation: OpsOperation, +) { + const remote = result._configurationMaintenance as Record | undefined, + receipt = configurationMaintenanceReceiptSchema.parse(remote?.receipt), + retained = records.parse(result._configurationMaintenanceSteps), + source = Object.values(retained).find((value) => value.operationKey === operationKey), + step = source && receipt.steps[source.ordinal], + candidate = result._configurationCandidate as Record | undefined; + if ( + !source || + !step || + String(source.ordinal) !== Object.keys(retained).find((index) => retained[index] === source) || + !receipt.decision || + receipt.decision.decision !== "continue" || + step.operationKey !== operationKey || + operationKey !== + controlChildOperationKey(receipt.prepare.operationKey, step.phase, step.ordinal) || + step.operationFingerprint !== opsOperationFingerprint(operation) || + canonicalJson(step.operation) !== canonicalJson(operation) || + source.requestedDeadlineAt !== receipt.prepare.deadlineAt || + canonicalJson(source.parent) !== + canonicalJson({ + operationKey: receipt.parentOperationKey, + fingerprint: receipt.parentFingerprint, + deadlineAt: receipt.parentDeadlineAt, + }) || + candidate?.candidateId !== receipt.binding.candidateId || + candidate.maintenanceId !== receipt.binding.maintenanceId || + candidate.prepareOperationKey !== receipt.prepare.operationKey || + candidate.revisionId !== receipt.binding.revisionId + ) + throw Error("Original frozen maintenance step checkpoint required"); + return { receipt, source, step }; +} +/** One completed DM ordinal in the existing batch. Observation only: fresh + * frozen-step constructors deliberately continue to call observedStep above. */ +export const completedCoordinatorMaintenanceSchema = z.strictObject({ + version: z.literal(1), + originalOrdinal: z.number().int().min(0).max(255), + candidate: z.strictObject({ + candidateId: z.uuid(), + maintenanceId: z.uuid(), + prepareOperationKey: key, + revisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), + operationKey: key, + phase: z.literal("settled"), + }), + receipt: configurationMaintenanceReceiptSchema, + steps: records, + settlementOperationKey: key, +}); +export function completedCoordinatorMaintenance(result: Record) { + const archived = completedCoordinatorMaintenanceSchema.parse( + result._completedCoordinatorMaintenance, + ), + receipt = archived.receipt, + operations = ( + result.opsProjection as + | { + operations?: Record< + string, + { + terminal?: boolean; + phase?: string; + operation?: OpsOperation; + result?: unknown; + receiptDelivery?: { + state?: string; + terminalReceipt?: { state?: string; result?: unknown }; + }; + } + >; + } + | undefined + )?.operations, + settle = operations?.[archived.settlementOperationKey]; + if ( + receipt.state !== "settled" || + receipt.dmFence?.state !== "settled" || + receipt.decision?.decision !== "continue" || + receipt.generation !== "committed" || + receipt.followUp !== "complete" || + receipt.appliedRevisionId !== archived.candidate.revisionId || + receipt.binding.candidateId !== archived.candidate.candidateId || + receipt.binding.maintenanceId !== archived.candidate.maintenanceId || + receipt.prepare.operationKey !== archived.candidate.prepareOperationKey || + !settle?.terminal || + settle.phase !== "terminal" || + settle.operation?.kind !== "configurationMaintenance.settle" || + settle.operation.candidateId !== receipt.binding.candidateId || + settle.operation.maintenanceId !== receipt.binding.maintenanceId || + settle.operation.prepareOperationKey !== receipt.prepare.operationKey || + settle.operation.prepareFingerprint !== receipt.prepare.operationFingerprint || + settle.operation.planDigest !== receipt.planDigest || + settle.operation.decisionId !== receipt.decision.decisionId || + settle.receiptDelivery?.state !== "confirmed" || + settle.receiptDelivery.terminalReceipt?.state !== "succeeded" || + canonicalJson(settle.receiptDelivery.terminalReceipt.result) !== canonicalJson(receipt) || + canonicalJson(settle.result) !== canonicalJson(receipt) + ) + throw Error("Original completed coordinator receipt required"); + for (const source of Object.values(archived.steps)) { + const step = receipt.steps[source.ordinal], + actual = operations?.[source.operationKey]; + if ( + !step || + !actual?.terminal || + actual.phase !== "terminal" || + actual.receiptDelivery?.state !== "confirmed" || + actual.receiptDelivery.terminalReceipt?.state !== "succeeded" || + canonicalJson(actual.operation) !== canonicalJson(step.operation) + ) + throw Error("Original completed coordinator step required"); + } + return archived; +} +function projectedStep( + result: Record, + operationKey: string, + operation: OpsOperation, +) { + const candidate = result._configurationCandidate as { candidateId?: unknown } | undefined, + remote = result._configurationMaintenance as + | { receipt?: { binding?: { candidateId?: unknown } } } + | undefined; + if ( + typeof candidate?.candidateId === "string" && + candidate.candidateId === remote?.receipt?.binding?.candidateId + ) + return observedStep(result, operationKey, operation); + const archived = completedCoordinatorMaintenance(result); + return observedStep( + { + ...result, + _configurationCandidate: archived.candidate, + _configurationMaintenance: { receipt: archived.receipt }, + _configurationMaintenanceSteps: archived.steps, + }, + operationKey, + operation, + ); +} +/** Parser consistency only. This never constructs a source/authority handle. */ +export function assertConfigurationMaintenanceProjectedStep( + result: Record, + operationKey: string, + operation: OpsOperation | undefined, + rawAttempt: unknown, +): void { + if (!operation) throw Error("Original frozen maintenance body required"); + const { source, receipt } = projectedStep(result, operationKey, operation), + attempt = apiJobChildAttemptSchema.parse(rawAttempt); + if ( + attempt.apiJobId !== receipt.apiJobId || + attempt.childOperationKey !== operationKey || + attempt.childOperationFingerprint !== opsOperationFingerprint(operation) || + attempt.requestedDeadlineAt !== source.requestedDeadlineAt || + attempt.parentOperationKey !== source.parent.operationKey || + attempt.parentFingerprint !== source.parent.fingerprint || + attempt.parentDeadlineAt !== source.parent.deadlineAt + ) + throw Error("Original API purpose-bound maintenance step required"); +} +/** Correlation address only, after the exact frozen projection has matched. */ +export function configurationMaintenanceProjectedStepOrigin( + result: Record, + operationKey: string, + operation: OpsOperation, + attempt: unknown, +): string { + assertConfigurationMaintenanceProjectedStep(result, operationKey, operation, attempt); + return projectedStep(result, operationKey, operation).receipt.prepare.operationKey; +} diff --git a/apps/api/src/services/configuration-maintenance-step.ts b/apps/api/src/services/configuration-maintenance-step.ts new file mode 100644 index 000000000..b0cd7a97f --- /dev/null +++ b/apps/api/src/services/configuration-maintenance-step.ts @@ -0,0 +1,177 @@ +/** Fixed actual writer/job step producer. Pure decoding is separate so ordinary + * managed-operation readers do not load configuration/auth runtime services. */ +import { + type ConfigurationMaintenanceStep, + canonicalJson, + configurationMaintenanceReceiptSchema, + type OpsOperation, +} from "@openmapx/core/ops"; +import * as schema from "../db/schema"; +import type { ApiPreparedRoot } from "./admin-job-parent"; +import { jobOwnership } from "./admin-job-state"; +import { + administrativeConfigurationState, + currentAdministrativeConfigurationWriter, + withConfigurationWriteTransaction, +} from "./administrative-configuration-writer"; +import { + assertConfigurationSavedRows, + configurationMaintenanceDecisionCheckpointSchema, +} from "./configuration-maintenance-decision"; +import { descriptor, observedStep, records } from "./configuration-maintenance-step-records"; +import type { JobContext } from "./job-runner"; +/** Called for a non-root managed key before its atomic admission checkpoint. + * Parsed metadata or a caller preparedRoot cannot bypass the real writer/job. */ +export async function assertOwnedConfigurationMaintenanceStep( + job: JobContext, + operation: OpsOperation, + operationKey: string, + root: ApiPreparedRoot, +): Promise { + const owner = administrativeConfigurationState(currentAdministrativeConfigurationWriter()); + if (owner.job !== job || !job.execution) throw Error("Actual maintenance job writer required"); + await withConfigurationWriteTransaction(owner.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined; + if (!result) throw Error("Actual maintenance job checkpoint required"); + const { source, receipt } = observedStep(result, operationKey, operation), + decision = configurationMaintenanceDecisionCheckpointSchema.parse( + result._configurationMaintenanceDecision, + ); + if ( + receipt.apiJobId !== job.jobId || + decision.operation.decision !== "continue" || + decision.operation.decisionId !== receipt.decision?.decisionId || + decision.attempt !== "attempted" || + root.operationKey !== source.operationKey || + root.requestedDeadlineAt !== source.requestedDeadlineAt || + canonicalJson(root.parent) !== canonicalJson(source.parent) + ) + throw Error("Actual authorized frozen maintenance step required"); + }); +} +export async function prepareOwnedConfigurationMaintenanceStep( + ordinal: number, + candidateId: string, + maintenanceId: string, +): Promise<{ step: ConfigurationMaintenanceStep; root: ApiPreparedRoot }> { + const owner = administrativeConfigurationState(currentAdministrativeConfigurationWriter()), + { job } = owner, + { readProjection } = await import("./admin-job-projection"); + let committed: Record | undefined; + const prepared = await withConfigurationWriteTransaction(owner.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined, + remote = result?._configurationMaintenance as Record | undefined, + receipt = configurationMaintenanceReceiptSchema.parse(remote?.receipt), + decision = configurationMaintenanceDecisionCheckpointSchema.parse( + result?._configurationMaintenanceDecision, + ), + step = receipt.steps[ordinal]; + if ( + !result || + receipt.binding.candidateId !== candidateId || + receipt.binding.maintenanceId !== maintenanceId || + !Number.isInteger(ordinal) || + !step || + receipt.apiJobId !== job.jobId || + !receipt.decision || + receipt.decision.decision !== "continue" || + decision.operation.decisionId !== receipt.decision.decisionId || + decision.operation.prepareOperationKey !== receipt.prepare.operationKey || + decision.operation.planDigest !== receipt.planDigest || + decision.attempt !== "attempted" || + !decision.delivery + ) + throw Error("Actual committed maintenance decision required"); + const projection = readProjection(result), + control = projection.operations[decision.delivery.root.operationKey], + delivered = control?.receiptDelivery; + if ( + !control?.terminal || + delivered?.state !== "confirmed" || + delivered.terminalReceipt.state !== "succeeded" || + configurationMaintenanceReceiptSchema.parse(delivered.terminalReceipt.result).decision + ?.decisionId !== decision.operation.decisionId + ) + throw Error("Actual delivered maintenance decision required"); + const previous = + result._configurationMaintenanceSteps === undefined + ? {} + : records.parse(result._configurationMaintenanceSteps), + existing = previous[String(ordinal)]; + if (existing) { + const observed = observedStep(result, step.operationKey, step.operation); + committed = result; + return { + step, + root: { + operationKey: observed.source.operationKey, + requestedDeadlineAt: observed.source.requestedDeadlineAt, + parent: observed.source.parent, + }, + }; + } + await assertConfigurationSavedRows(tx, decision.rowVersions); + for (const index of step.prerequisites) { + const prior = receipt.steps[index]!, + actual = projection.operations[prior.operationKey]; + if ( + !actual?.terminal || + actual.errorClass || + actual.kind !== prior.operation.kind || + canonicalJson(actual.operation) !== canonicalJson(prior.operation) || + actual.receiptDelivery?.state !== "confirmed" || + actual.receiptDelivery.terminalReceipt.state !== "succeeded" + ) + throw Error("Original frozen maintenance prerequisite is unresolved"); + if ( + prior.phase === "configuration-commit" && + (!actual.result || + typeof actual.result !== "object" || + (actual.result as { outcome?: string }).outcome !== "committed" || + (actual.result as { revisionId?: string }).revisionId !== receipt.binding.revisionId) + ) + throw Error("Original maintenance generation is not committed"); + } + const source = descriptor.parse({ + ordinal, + operationKey: step.operationKey, + requestedDeadlineAt: receipt.prepare.deadlineAt, + parent: { + operationKey: receipt.parentOperationKey, + fingerprint: receipt.parentFingerprint, + deadlineAt: receipt.parentDeadlineAt, + }, + }); + committed = { + ...result, + _configurationMaintenanceSteps: records.parse({ ...previous, [String(ordinal)]: source }), + }; + const saved = await tx + .update(schema.adminJob) + .set({ result: committed }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw Error("Original maintenance job ownership changed"); + return { + step, + root: { + operationKey: source.operationKey, + requestedDeadlineAt: source.requestedDeadlineAt, + parent: source.parent, + }, + }; + }); + if (!committed) throw Error("Original maintenance step checkpoint unavailable"); + job.checkpointResult = committed; + return prepared; +} diff --git a/apps/api/src/services/configuration-maintenance.ts b/apps/api/src/services/configuration-maintenance.ts new file mode 100644 index 000000000..65e5a6e30 --- /dev/null +++ b/apps/api/src/services/configuration-maintenance.ts @@ -0,0 +1,9 @@ +/** Existing typed maintenance surface, backed by the sole nominal candidate owner. */ +export { + decideConfigurationMaintenance, + executeConfigurationMaintenanceStep, + inspectConfigurationMaintenance, + prepareConfigurationMaintenance, + prepareConfigurationMaintenanceAbandon, + settleConfigurationMaintenance, +} from "./prospective-configuration"; diff --git a/apps/api/src/services/configuration-migration/legacy-integration-import-intent.ts b/apps/api/src/services/configuration-migration/legacy-integration-import-intent.ts new file mode 100644 index 000000000..2a4754309 --- /dev/null +++ b/apps/api/src/services/configuration-migration/legacy-integration-import-intent.ts @@ -0,0 +1,227 @@ +/** Sole closed legacy-import intent issuer. Depends on live writer ownership, + * strict reads and the existing parser, never the persister/job runner. */ +import { canonicalJson, type TrustedConfigurationPayload } from "@openmapx/core/ops"; +import { + type AdministrativeConfigurationWriter, + administrativeConfigurationState, + type ConfigurationCommitParticipant, + withAdministrativeConfigurationRead, +} from "../administrative-configuration-writer-owner"; +import { prepareIntegrationImport } from "../integration-config-import"; +import type { + ProspectiveConfigurationOverlay, + ValidatedConfigurationCandidate, +} from "../prospective-configuration"; + +const intentBrand: unique symbol = Symbol("legacy integration import"); +export interface LegacyIntegrationImportIntent { + readonly [intentBrand]: true; + readonly integrationIds: readonly string[]; + readonly changedIntegrationIds: readonly string[]; +} +interface IntentState { + writer: AdministrativeConfigurationWriter; + overlay: ProspectiveConfigurationOverlay; + rows: string; + registry: string; + participant: ConfigurationCommitParticipant; +} +const intents = new WeakMap(); +const issued = new WeakMap(); +const refuse = () => new Error("Ordinary integration import unavailable"); +const freeze = (value: T): T => { + if (value && typeof value === "object") { + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); + } + return value; +}; +function owned(intent: LegacyIntegrationImportIntent, writer: AdministrativeConfigurationWriter) { + const state = intents.get(intent); + if (!state || state.writer !== writer) throw refuse(); + administrativeConfigurationState(writer); + return state; +} +/** A verifier of this one built-in issuer; it cannot register a foreign implementation. */ +export function isLegacyIntegrationImportParticipant( + participant: ConfigurationCommitParticipant, + writer: AdministrativeConfigurationWriter, +): boolean { + return issued.get(participant)?.writer === writer; +} +/** Serialize ONLY provenance issued by this actual built-in constructor. The + * authenticated private candidate retains it before any desired transaction. */ +export function retainedLegacyIntegrationImportIntent(writer: AdministrativeConfigurationWriter) { + const owner = administrativeConfigurationState(writer), + participant = owner.participant, + state = participant && issued.get(participant); + if ( + !state || + state.writer !== writer || + state.participant !== participant || + participant.mode !== "apply" + ) + throw refuse(); + const integrationIds = [...writer.integrationIds], + changedIntegrationIds = state.overlay.changes.map((change) => { + if (change.kind !== "integration-config" || !integrationIds.includes(change.integrationId)) + throw refuse(); + return change.integrationId; + }); + if (!changedIntegrationIds.length) throw refuse(); + return freeze({ + version: 1 as const, + kind: "legacy-integration-config" as const, + integrationIds, + changedIntegrationIds, + }); +} + +/** Revalidate the actual route body under the complete sorted lock set and strict snapshot. */ +export async function prepareLegacyIntegrationImport( + body: unknown, + writer: AdministrativeConfigurationWriter, +): Promise { + administrativeConfigurationState(writer); + return withAdministrativeConfigurationRead({}, async (context) => { + const registry = canonicalJson( + context.integrations.map((i) => ({ id: i.id, directory: i.directory, manifest: i.manifest })), + ); + const schemas = new Map( + context.integrations.map((i) => [ + i.id, + i.manifest.configSchema as Record | undefined, + ]), + ); + const entries = prepareIntegrationImport(body, schemas).sort((a, b) => + a.integrationId.localeCompare(b.integrationId), + ); + const ids = entries.map((e) => e.integrationId); + if (canonicalJson(ids) !== canonicalJson(writer.integrationIds)) throw refuse(); + const changes: ProspectiveConfigurationOverlay["changes"] = []; + for (const entry of entries) { + const prior = context.integrationRows.filter( + (row) => row.integrationId === entry.integrationId, + ); + if (prior.length > 1) throw refuse(); + const old = prior[0]?.config ?? {}; + // This diagnostic checks the old ordinary keys too. An import cannot erase + // unknown/incompatible legacy fields merely to produce an acceptable seal. + prepareIntegrationImport( + { integrations: [{ integrationId: entry.integrationId, config: old }] }, + schemas, + ); + const properties = (schemas.get(entry.integrationId)?.properties ?? + schemas.get(entry.integrationId) ?? + {}) as Record>; + const set: Record = {}, + unset: string[] = []; + for (const [key, value] of Object.entries(entry.config)) + if (!Object.hasOwn(old, key) || canonicalJson(value) !== canonicalJson(old[key])) + set[key] = value; + for (const key of Object.keys(old)) { + if (key === "enabled" || properties[key]?.["x-openmapx-secret"] === true) continue; + if (!Object.hasOwn(entry.config, key)) unset.push(key); + } + if (Object.keys(set).length || unset.length) + changes.push({ + kind: "integration-config", + integrationId: entry.integrationId, + set: set as TrustedConfigurationPayload["integrationConfigs"][number]["values"], + unset: unset.sort(), + }); + } + const overlay = freeze({ + purpose: "migration" as const, + coordinatorMode: "ensure-running" as const, + changes, + }); + let state: IntentState; + const assert = async ( + actual: AdministrativeConfigurationWriter, + candidate?: ValidatedConfigurationCandidate, + ) => { + if ( + actual !== writer || + issued.get(state.participant) !== state || + actual.actorId !== writer.actorId || + actual.operationKey !== writer.operationKey + ) + throw refuse(); + administrativeConfigurationState(actual); + if (candidate && candidate.operationKey !== actual.operationKey) throw refuse(); + await withAdministrativeConfigurationRead({}, async (read) => { + if ( + canonicalJson(read.integrationRows) !== state.rows || + canonicalJson( + read.integrations.map((i) => ({ + id: i.id, + directory: i.directory, + manifest: i.manifest, + })), + ) !== registry + ) + throw refuse(); + }); + }; + const participant = Object.freeze({ + kind: "configuration-migration", + mode: "apply", + actorId: writer.actorId, + operationKey: writer.operationKey, + managedServiceSecretKeys: Object.freeze([]), + authorize: (actual: AdministrativeConfigurationWriter) => assert(actual), + verify: ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + ) => assert(actual, candidate), + commit: ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + ) => assert(actual, candidate), + }) as unknown as ConfigurationCommitParticipant; + state = { + writer, + overlay, + rows: canonicalJson(context.integrationRows), + registry, + participant, + }; + const intent = freeze({ + [intentBrand]: true as const, + integrationIds: ids, + changedIntegrationIds: changes.map((c) => (c as { integrationId: string }).integrationId), + }); + intents.set(intent, state); + issued.set(participant, state); + return intent; + }); +} + +/** Read-only participation from THIS issuer's original opaque intent. No + * participant registration, callback construction or caller-selected purpose. */ +export function legacyIntegrationImportParticipation( + intent: LegacyIntegrationImportIntent, + writer: AdministrativeConfigurationWriter, +) { + const state = owned(intent, writer); + return Object.freeze({ overlay: state.overlay, participant: state.participant }); +} +/** The nominal candidate dispatcher supplies its actual private overlay before + * preparation and every participant verification/commit. This comparison is + * read-only; raw overlay data cannot construct an intent or participant. */ +export function assertLegacyIntegrationImportOverlay( + writer: AdministrativeConfigurationWriter, + overlay: unknown, +): void { + const owner = administrativeConfigurationState(writer), + participant = owner.participant, + state = participant && issued.get(participant); + if ( + !state || + state.writer !== writer || + state.participant !== participant || + canonicalJson(state.overlay) !== canonicalJson(overlay) + ) + throw refuse(); +} diff --git a/apps/api/src/services/configuration-migration/legacy-integration-import.test.ts b/apps/api/src/services/configuration-migration/legacy-integration-import.test.ts new file mode 100644 index 000000000..7e86a5cb2 --- /dev/null +++ b/apps/api/src/services/configuration-migration/legacy-integration-import.test.ts @@ -0,0 +1,804 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { setTimeout as nativeDelay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import Fastify from "fastify"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; +import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; +import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; +import * as schema from "../../db/schema"; +import { createConfigurationPostgresFixture } from "../../test/configuration-postgres-fixture"; +import { postgresCommitRelay } from "../__tests__/fixtures/postgres-commit-relay"; + +type Fixture = Awaited>; +const state = vi.hoisted(() => ({ + fixture: null as unknown as Fixture, + loaded: [] as services.LoadedService[], +})); +vi.mock("../../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); +vi.mock("../service-registry", async (actual) => ({ + ...(await actual()), + getServiceRegistry: () => ({ + list: () => state.loaded, + get: (id: string) => state.loaded.find((s) => s.manifest.id === id), + applyEnabledIds: () => {}, + }), +})); + +describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "current ordinary import actual route / PostgreSQL / agent", + () => { + let fixture: Fixture, host: Awaited>; + let agent: ReturnType, api: ReturnType; + let installation: ReturnType; + let driftRegistryAfterApply = false; + let renderPosts = 0; + let driftRowAfterPreview = false; + const configSchema = { + properties: { + VALUE: { type: "string", default: "default" }, + OMIT: { type: "integer" }, + TOKEN: { type: "string", "x-openmapx-secret": true }, + ...Object.fromEntries( + Array.from({ length: 5 }, (_, n) => [`BLOCK${n}`, { type: "string" }]), + ), + }, + }; + beforeEach(async () => { + fixture = await createConfigurationPostgresFixture(); + state.fixture = fixture; + host = await ownedServiceHost(); + state.loaded = host.loaded; + const integrationRoot = join(host.root, "integrations"); + mkdirSync(integrationRoot); + for (const id of ["alpha", "beta", "unrelated"]) { + const directory = join(integrationRoot, id); + mkdirSync(directory); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id, + version: "1.0.0", + author: "Task01.2 fixture", + license: "MIT", + domains: ["knowledge"], + quality: "built-in", + configSchema, + }), + ); + } + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + agent = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + controlInstallation: installation, + }); + driftRegistryAfterApply = false; + renderPosts = 0; + driftRowAfterPreview = false; + agent.addHook("onSend", async (request, _reply, payload) => { + if ( + request.method === "POST" && + (request.body as { operation?: { kind?: string } })?.operation?.kind === "stack.render" + ) + renderPosts++; + if ( + driftRowAfterPreview && + (request.body as { operation?: { kind?: string } })?.operation?.kind === + "configuration.hardlinks.preview" + ) { + driftRowAfterPreview = false; + // A second real connection writes outside the advisory protocol. The + // retained snapshot/version checks must refuse both import overlays. + await fixture.sql`UPDATE integration_config SET config='{"VALUE":"external-race"}'::jsonb WHERE integration_id='beta'`; + } + if ( + driftRegistryAfterApply && + request.method === "GET" && + request.url.startsWith("/v1/operations/") && + typeof payload === "string" + ) { + const response = JSON.parse(payload); + if (response.result?.result?.outcome === "committed") { + driftRegistryAfterApply = false; + const path = join(integrationRoot, "alpha/manifest.json"); + writeFileSync( + path, + JSON.stringify({ ...JSON.parse(readFileSync(path, "utf8")), version: "1.0.1" }), + ); + } + } + return payload; + }); + const baseUrl = await agent.listen({ host: "127.0.0.1", port: 0 }), + tokenFile = join(host.root, "api-token"); + writeFileSync(tokenFile, host.token, { mode: 0o600 }); + for (const [key, value] of Object.entries({ + OPS_AGENT_URL: baseUrl, + OPS_AGENT_TOKEN_FILE: tokenFile, + OPS_TRUSTED_CONFIG_DIR: host.queue, + OPS_TRUSTED_CONFIG_UID: String(statSync(host.queue).uid), + OPS_TRUSTED_CONFIG_GID: String(statSync(host.queue).gid), + OPENMAPX_SECRETS_KEY: "71".repeat(32), + OPENMAPX_LOCAL_ADMIN_TOKEN: "task012-owned-local-admin", + OPENMAPX_DISABLE_LOCALHOST_AUTH: "0", + BETTER_AUTH_SECRET: "task012-owned-synthetic-auth-secret-only", + BETTER_AUTH_URL: "http://localhost:3001", + })) + vi.stubEnv(key, value); + api = Fastify({ logger: false }); + const { initIntegrations } = await import("../../integration-host"); + await initIntegrations(api, [{ directory: integrationRoot, isBuiltIn: true }]); + const { requireAdmin } = await import("../../utils/require-admin"); + api.addHook("preHandler", async (request: import("fastify").FastifyRequest) => { + request.adminSession = await requireAdmin(request); + }); + const { adminIntegrationsRoute } = await import("../../routes/admin-integrations"); + await api.register(adminIntegrationsRoute, { prefix: "/api" }); + await api.ready(); + }); + afterEach(async () => { + const { shutdownIntegrations } = await import("../../integration-host"); + await shutdownIntegrations(); + await api?.close(); + await agent?.close(); + if (installation) closeControlInstallation(installation); + await fixture?.stop(); + if (host) rmSync(host.root, { recursive: true, force: true }); + vi.unstubAllEnvs(); + state.loaded = []; + }); + function submit( + integrations: { integrationId: string; config: Record }[], + key = `ordinary-import-${randomUUID()}`, + ) { + return api.inject({ + method: "POST", + url: "/api/admin/integrations/import", + remoteAddress: "127.0.0.1", + headers: { "x-openmapx-local-admin": "task012-owned-local-admin", "idempotency-key": key }, + payload: { integrations }, + }); + } + it.each(["saved", "applied", "before-retirement", "after-retirement"])( + "recovers the original ordinary import through native API death and actual HTTP (mode=%s)", + async (mode) => { + const retirement = mode.includes("retirement"), + retired = mode === "after-retirement"; + const idempotencyKey = `ordinary-native-${randomUUID()}`, + database = new URL(process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL!); + database.searchParams.set("search_path", fixture.schema); + const relay = await postgresCommitRelay( + database, + async () => { + const rows = + await fixture.sql`SELECT result FROM admin_job WHERE type='integration.config.import' LIMIT 2`; + if (rows.length !== 1) return false; + const c = rows[0]!.result?._configurationCandidate; + if (mode === "before-retirement") return false; + if (retirement) + return ( + c?.phase === "settled" && + c.retirement?.version === 1 && + existsSync(join(host.queue, ".candidates", `${c.sealedOperationKey}.json`)) !== + retired + ); + if (c?.phase !== "saved") return false; + return ( + mode !== "applied" || + Object.values(rows[0]!.result?.opsProjection?.operations ?? {}).some((value) => { + const operation = value as { + kind?: string; + terminal?: boolean; + receiptDelivery?: { state?: string }; + }; + return ( + operation.kind === "stack.render" && + operation.terminal === true && + operation.receiptDelivery?.state === "pending" + ); + }) + ); + }, + { holdCommand: retirement ? (retired ? "API_TERMINAL_UPDATE" : "UPDATE") : "COMMIT" }, + ); + const loader = createRequire(new URL("../../../package.json", import.meta.url)).resolve( + "tsx/esm", + ), + script = fileURLToPath( + new URL("../__tests__/fixtures/native-integration-import-api.ts", import.meta.url), + ), + children: ReturnType[] = []; + const start = (replacement = false) => { + const args = ["--import", loader, script], + child = spawn(process.execPath, args, { + cwd: host.root, + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + DATABASE_URL: relay.databaseUrl, + OPENMAPX_ROOT_DIR: host.root, + NODE_ENV: "test", + REDIS_URL: "", + REDIS_PASSWORD_FILE: "", + ...(replacement + ? { INTEGRATION_ALPHA_VALUE: "synthetic-later-process-environment" } + : {}), + }, + }); + let output = ""; + children.push(child); + child.stdout!.on("data", (chunk) => { + output += chunk; + }); + child.stderr!.on("data", () => {}); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => { + console.info( + JSON.stringify({ + proof: "native-import-api-exit", + argv: [process.execPath, ...args], + code, + signal, + }), + ); + resolve({ code, signal }); + }); + }, + ); + child.stdin!.end( + JSON.stringify({ + idempotencyKey, + ...(mode === "before-retirement" && children.length === 1 + ? { filesystemFault: "pause-retirement-unlink" } + : {}), + }), + ); + return { + child, + exited, + result: () => + output + .split("\n") + .flatMap((line) => { + try { + const value = JSON.parse(line); + return value.event === "result" ? [value] : []; + } catch { + return []; + } + }) + .at(-1) as + | { capturedOrdinaryValueMatches?: boolean; betaDisabled?: boolean } + | undefined, + }; + }; + const first = start(); + try { + const beforeUnlink = async () => { + const until = Date.now() + 30000; + while (!existsSync(join(host.root, "retirement-unlink-paused"))) { + if (Date.now() > until) throw Error("Actual retirement unlink not reached"); + await nativeDelay(20); + } + }; + await Promise.race([ + mode === "before-retirement" ? beforeUnlink() : relay.reached, + first.exited.then(() => { + throw Error("Native import API exited before original saved checkpoint"); + }), + ]); + const [original] = await fixture.db.select().from(schema.adminJob), + checkpoint = original!.result as Record, + candidate = checkpoint._configurationCandidate as Record, + path = join(host.queue, ".candidates", `${candidate.sealedOperationKey}.json`), + bytes = retired ? undefined : readFileSync(path), + versions = + await fixture.sql`SELECT integration_id,xmin::text AS version FROM integration_config ORDER BY integration_id`; + expect(versions).toHaveLength(2); + expect(renderPosts).toBe(mode === "applied" || retirement ? 1 : 0); + first.child.kill("SIGKILL"); + expect((await first.exited).signal).toBe("SIGKILL"); + relay.release(); + const [retained] = await fixture.db.select().from(schema.adminJob), + lease = Date.parse( + (retained!.result as Record)._observationLeaseUntil as string, + ); + expect(Number.isFinite(lease)).toBe(true); + while (Date.now() <= lease + 100) + await nativeDelay(Math.min(1000, lease + 101 - Date.now())); + expect(retired ? !existsSync(path) : readFileSync(path).equals(bytes!)).toBe(true); + const replacement = start(!retirement); + expect((await replacement.exited).code).toBe(0); + expect(replacement.result()).toMatchObject({ + capturedOrdinaryValueMatches: true, + betaDisabled: true, + }); + const [finished] = await fixture.db.select().from(schema.adminJob), + result = finished!.result as Record; + expect(finished!.id).toBe(original!.id); + expect(finished!.status).toBe("success"); + expect(result._execution).toMatchObject({ version: 2 }); + expect(result.integrationImport).toMatchObject({ + ok: true, + imported: 2, + skipped: [], + reloaded: 3, + enabled: 2, + }); + expect(result.integrationReload).toMatchObject({ + state: "complete", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }); + expect( + await fixture.sql`SELECT integration_id,xmin::text AS version FROM integration_config ORDER BY integration_id`, + ).toEqual(versions); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(candidate.revisionId); + expect(renderPosts).toBe(1); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + expect(JSON.stringify(result)).not.toMatch(/synthetic-strip-this|native-new/); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + expect((await start().exited).code).toBe(0); + const [duplicate] = await fixture.db.select().from(schema.adminJob); + expect(duplicate!.result).toEqual(result); + expect(renderPosts).toBe(1); + console.info( + JSON.stringify({ + proof: "native-import-original-recovered", + originalJobId: original!.id, + candidateId: candidate.candidateId, + renderPosts, + rowVersionsUnchanged: true, + retainedBytesConsumed: true, + mode, + }), + ); + } finally { + relay.release(); + for (const child of children) + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await relay.close(); + } + }, + 180000, + ); + it("replaces two ordinary rows in one candidate, preserving enabled, legacy secrets, vault and unrelated state", async () => { + const { encrypt } = await import("../secrets"); + await fixture.db.insert(schema.integrationConfig).values([ + { + id: randomUUID(), + integrationId: "alpha", + config: { VALUE: "old", OMIT: 2, enabled: false, TOKEN: "legacy-secret" }, + }, + { id: randomUUID(), integrationId: "beta", config: { VALUE: "old-b", enabled: true } }, + { id: randomUUID(), integrationId: "unrelated", config: { VALUE: "retain" } }, + ]); + await fixture.db.insert(schema.integrationSecret).values({ + id: randomUUID(), + integrationId: "alpha", + key: "TOKEN", + ...encrypt("vault-secret"), + }); + const key = `ordinary-import-${randomUUID()}`, + body = [ + { integrationId: "alpha", config: { VALUE: "new", TOKEN: "supplied-must-be-stripped" } }, + { integrationId: "beta", config: { VALUE: "new-b", enabled: false } }, + ]; + const response = await submit(body, key); + expect(response.statusCode, response.body).toBe(200); + expect(response.json()).toMatchObject({ ok: true, imported: 2 }); + const rows = await fixture.db.select().from(schema.integrationConfig); + expect(rows.find((row) => row.integrationId === "alpha")?.config).toEqual({ + VALUE: "new", + TOKEN: "legacy-secret", + enabled: false, + }); + expect(rows.find((row) => row.integrationId === "beta")?.config).toEqual({ + VALUE: "new-b", + enabled: false, + }); + expect(rows.find((row) => row.integrationId === "unrelated")?.config).toEqual({ + VALUE: "retain", + }); + expect(await fixture.db.select().from(schema.integrationSecret)).toHaveLength(1); + const { getIntegration } = await import("../../integration-host"); + expect(getIntegration("alpha")).toMatchObject({ + enabled: false, + config: { VALUE: "new", TOKEN: "vault-secret" }, + }); + const jobs = await fixture.db.select().from(schema.adminJob); + expect(jobs).toHaveLength(1); + expect(jobs[0]).toMatchObject({ type: "integration.config.import", status: "success" }); + expect(JSON.stringify(jobs[0])).not.toMatch( + /supplied-must-be-stripped|legacy-secret|vault-secret/, + ); + expect((await submit(body, key)).json()).toEqual(response.json()); + expect(await fixture.db.select().from(schema.adminJob)).toHaveLength(1); + expect(host.calls.some((args) => args.includes("up") || args.includes("pull"))).toBe(false); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + }); + it("refuses whole-envelope oversize before ordinary rows, effects or consumable publication", async () => { + const config = Object.fromEntries( + Array.from({ length: 5 }, (_, n) => [`BLOCK${n}`, "ü".repeat(60000)]), + ); + await fixture.db + .insert(schema.integrationConfig) + .values({ id: randomUUID(), integrationId: "unrelated", config }); + const response = await submit([ + { integrationId: "alpha", config: { VALUE: "changed" } }, + { integrationId: "beta", config: { VALUE: "changed-b" } }, + ]); + expect(response.statusCode).toBe(409); + const retained = await fixture.db.select().from(schema.integrationConfig); + expect(retained.map((row) => row.integrationId)).toEqual(["unrelated"]); + expect(retained[0]?.config === null ? 0 : JSON.stringify(retained[0]?.config).length).toBe( + JSON.stringify(config).length, + ); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + expect(readdirSync(host.queue).filter((name) => name.startsWith("cfg1_"))).toEqual([]); + expect(host.calls.some((args) => args.includes("up") || args.includes("pull"))).toBe(false); + }); + it("refuses both overlays after a real row race at the captured preview boundary", async () => { + await fixture.db.insert(schema.integrationConfig).values([ + { id: randomUUID(), integrationId: "alpha", config: { VALUE: "old-a" } }, + { id: randomUUID(), integrationId: "beta", config: { VALUE: "old-b" } }, + ]); + driftRowAfterPreview = true; + const response = await submit([ + { integrationId: "beta", config: { VALUE: "new-b" } }, + { integrationId: "alpha", config: { VALUE: "new-a" } }, + ]); + expect(response.statusCode).toBe(409); + expect(driftRowAfterPreview).toBe(false); + const rows = await fixture.db.select().from(schema.integrationConfig); + expect(rows.find((row) => row.integrationId === "alpha")?.config).toEqual({ VALUE: "old-a" }); + expect(rows.find((row) => row.integrationId === "beta")?.config).toEqual({ + VALUE: "external-race", + }); + expect(renderPosts).toBe(0); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + expect(readdirSync(host.queue).filter((name) => name.startsWith("cfg1_"))).toEqual([]); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + }); + it("refuses the whole import at actual shared queue capacity before any desired write", async () => { + const { buildConfigurationInput } = await import("../trusted-config-operations"), + { publishTrustedConfigurationSnapshot } = await import("../trusted-config-publisher"), + { executeAndWait, createApiOpsClient, createDurableOpsKey } = await import("../ops-client"), + source = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.sources.inspect" }, + createDurableOpsKey("configuration.sources.inspect", randomUUID()), + ), + payload = await buildConfigurationInput(); + // Actual inert producer packets, never admitted as effects or signed grants. + for (let n = 0; n < 128; n++) + await publishTrustedConfigurationSnapshot({ + directory: host.queue, + token: host.token, + ownerUid: statSync(host.queue).uid, + ownerGid: statSync(host.queue).gid, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationForRevision: (revisionId) => ({ + kind: "stack.render", + revisionId, + expectedCurrentRevisionId: source.revisionId, + expectedSourceRevision: source.sourceRevision, + coordinatorMode: "ensure-running", + }), + payload, + }); + const before = readdirSync(host.queue) + .filter((name) => name.startsWith("cfg1_")) + .sort(); + expect(before).toHaveLength(128); + const response = await submit([ + { integrationId: "alpha", config: { VALUE: "must-not-save" } }, + { integrationId: "beta", config: { VALUE: "must-not-save-b" } }, + ]); + expect(response.statusCode).toBe(409); + expect(await fixture.db.select().from(schema.integrationConfig)).toEqual([]); + expect( + readdirSync(host.queue) + .filter((name) => name.startsWith("cfg1_")) + .sort(), + ).toEqual(before); + expect(readdirSync(join(host.queue, ".candidates"))).toEqual([]); + expect(renderPosts).toBe(0); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + expect( + host.calls.some( + (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), + ), + ).toBe(false); + }, 60000); + it.each(["host-source", "unknown-unchanged", "malformed-legacy-secret", "invalid-second-row"])( + "refuses %s before changing either import target", + async (reason) => { + if (reason === "host-source") vi.stubEnv("INTEGRATION_ALPHA_VALUE", "host-owned"); + if (reason === "unknown-unchanged") + await fixture.db.insert(schema.integrationConfig).values({ + id: randomUUID(), + integrationId: "unrelated", + config: { UNKNOWN: "must-not-drop" }, + }); + if (reason === "malformed-legacy-secret") + await fixture.db + .insert(schema.integrationConfig) + .values({ id: randomUUID(), integrationId: "unrelated", config: { TOKEN: 17 } }); + const response = await submit([ + { integrationId: "alpha", config: { VALUE: "new" } }, + { + integrationId: "beta", + config: { VALUE: reason === "invalid-second-row" ? 17 : "new-b" }, + }, + ]); + expect(response.statusCode).toBe(reason === "invalid-second-row" ? 400 : 409); + const rows = await fixture.db.select().from(schema.integrationConfig); + expect( + rows.some((row) => row.integrationId === "alpha" || row.integrationId === "beta"), + ).toBe(false); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(readdirSync(host.queue).filter((name) => name.startsWith("cfg1_"))).toEqual([]); + }, + ); + it("keeps the imported ordinary enabled flag through an actual integration host cold restart", async () => { + expect( + (await submit([{ integrationId: "alpha", config: { enabled: false } }])).statusCode, + ).toBe(200); + const { getIntegration, shutdownIntegrations, initIntegrations } = await import( + "../../integration-host" + ); + expect(getIntegration("alpha")?.enabled).toBe(false); + await shutdownIntegrations(); + await api.close(); + api = Fastify({ logger: false }); + await initIntegrations(api, [ + { directory: join(host.root, "integrations"), isBuiltIn: true }, + ]); + expect(getIntegration("alpha")?.enabled).toBe(false); + }); + it("retains an unresolved candidate fence after actual reload source drift and refuses a later writer", async () => { + driftRegistryAfterApply = true; + const response = await submit([ + { integrationId: "alpha", config: { VALUE: "committed-before-drift" } }, + ]); + expect(driftRegistryAfterApply).toBe(false); + expect(response.statusCode, response.body).toBe(409); + expect(response.json()).toMatchObject({ ok: false, phase: "reload", runtime: "unknown" }); + const [row] = await fixture.db.select().from(schema.integrationConfig); + expect(row?.config).toEqual({ VALUE: "committed-before-drift" }); + const [job] = await fixture.db.select().from(schema.adminJob); + expect((job?.result as Record)._configurationCandidate).toMatchObject({ + phase: "unknown", + }); + expect(readdirSync(join(host.queue, ".candidates"))).toHaveLength(1); + const second = await submit([{ integrationId: "beta", config: { VALUE: "must-not-save" } }]); + expect(second.statusCode).not.toBe(200); + expect( + (await fixture.db.select().from(schema.integrationConfig)).map((row) => row.integrationId), + ).toEqual(["alpha"]); + }); + it("rolls back the first ordinary row and saved checkpoint when PostgreSQL rejects the second row", async () => { + await fixture.sql.unsafe( + `ALTER TABLE "${fixture.schema}".integration_config ADD CHECK (integration_id <> 'beta' OR config->>'VALUE' <> 'deny-second-write')`, + ); + const response = await submit([ + { integrationId: "alpha", config: { VALUE: "new" } }, + { integrationId: "beta", config: { VALUE: "deny-second-write" } }, + ]); + expect(response.statusCode).toBe(409); + expect(await fixture.db.select().from(schema.integrationConfig)).toEqual([]); + const [job] = await fixture.db.select().from(schema.adminJob); + expect((job?.result as Record)?._configurationCandidate).toMatchObject({ + phase: "settled", + saved: "conflict", + }); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + }); + it("reports an unchanged batch without a candidate, apply admission or generation change", async () => { + await fixture.db.insert(schema.integrationConfig).values({ + id: randomUUID(), + integrationId: "alpha", + config: { VALUE: "same", enabled: false }, + }); + const response = await submit([ + { integrationId: "alpha", config: { VALUE: "same", TOKEN: "stripped" } }, + ]); + expect(response.statusCode, response.body).toBe(200); + expect(response.json()).toMatchObject({ + ok: true, + imported: 0, + skipped: ["alpha"], + reloaded: 0, + }); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + const [job] = await fixture.db.select().from(schema.adminJob); + expect((job?.result as Record)?._configurationCandidate).toBeUndefined(); + }); + it("returns only the declared import DTO through the generated SDK and actual loopback HTTP", async () => { + const { createApiClient } = await import("@openmapx/core"); + const sdk = await import("@openmapx/core/api-admin-sdk"); + const baseUrl = await api.listen({ host: "127.0.0.1", port: 0 }); + const client = createApiClient({ baseUrl }), + headers = { "idempotency-key": `ordinary-sdk-${randomUUID()}` }; + const body = { + integrations: [ + { + integrationId: "alpha", + config: { VALUE: "sdk-value", TOKEN: "stripped-before-retention" }, + }, + { integrationId: "beta", config: { VALUE: "sdk-b", enabled: false } }, + ], + }; + const options = { + client, + headers: { "x-openmapx-local-admin": "task012-owned-local-admin" }, + }; + const response = await sdk.importAdminIntegrationConfigs(body, headers, options); + expect(response).toMatchObject({ + status: 200, + data: { ok: true, imported: 2, reloaded: 3, enabled: 2 }, + }); + expect(Object.keys(response.data).sort()).toEqual([ + "enabled", + "imported", + "message", + "ok", + "reloaded", + "skipped", + ]); + expect(await sdk.importAdminIntegrationConfigs(body, headers, options)).toEqual(response); + const jobs = await fixture.db.select().from(schema.adminJob); + expect(jobs).toHaveLength(1); + expect(JSON.stringify(jobs)).not.toContain("stripped-before-retention"); + }); + it("issues only a writer-bound immutable legacy participant and validates its real candidate", async () => { + const writer = await import("../administrative-configuration-writer"); + const subject = await import("./legacy-integration-import"); + const prospective = await import("../prospective-configuration"); + const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); + const { encrypt } = await import("../secrets"); + await fixture.db.insert(schema.integrationConfig).values([ + { + id: randomUUID(), + integrationId: "alpha", + config: { VALUE: "old", OMIT: 2, enabled: false, TOKEN: "legacy-secret" }, + }, + { id: randomUUID(), integrationId: "beta", config: { VALUE: "old-b", enabled: true } }, + ]); + await fixture.db.insert(schema.integrationSecret).values({ + id: randomUUID(), + integrationId: "alpha", + key: "TOKEN", + ...encrypt("vault-secret"), + }); + const jobId = randomUUID(), + execution = { owner: randomUUID(), version: 1 }; + await fixture.db.insert(schema.adminJob).values({ + id: jobId, + type: "integration.config.import", + status: "running", + result: { _execution: execution }, + }); + const ctx: import("../job-runner").JobContext = { + jobId, + jobType: "integration.config.import", + execution, + payload: {}, + signal: new AbortController().signal, + checkpointResult: { _execution: execution }, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (result, progress, admissionKey, stepAddress) => { + ctx.checkpointResult = await checkpointOwnedAdminJob( + jobId, + execution, + result, + progress, + admissionKey, + stepAddress, + ctx.checkpointResult ?? {}, + ); + }, + }; + await writer.withAdministrativeConfigurationJob(ctx, () => + writer.withAdministrativeConfigurationWriter( + { + actorId: "loopback", + operationKey: `opk1_${jobId.replaceAll("-", "")}`, + integrationIds: ["beta", "alpha"], + }, + async (owned) => { + const intent = await subject.prepareLegacyIntegrationImport( + { + integrations: [ + { integrationId: "alpha", config: { VALUE: "new" } }, + { integrationId: "beta", config: { VALUE: "b", enabled: false } }, + ], + }, + owned, + ); + expect(Object.isFrozen(intent)).toBe(true); + expect(() => + subject.withLegacyIntegrationImport({ ...intent }, owned, async () => {}), + ).toThrow(); + await subject.withLegacyIntegrationImport(intent, owned, async (overlay) => { + expect(overlay.purpose).toBe("migration"); + const originalRows = await fixture.db + .select() + .from(schema.integrationConfig) + .orderBy(schema.integrationConfig.integrationId); + await expect( + prospective.prepareProspectiveConfiguration({ + actorId: "loopback", + operationKey: owned.operationKey, + expectedCurrentRevisionId: host.initial.revisionId, + overlay: { + ...overlay, + changes: [ + { + kind: "integration-config", + integrationId: "alpha", + set: { VALUE: "foreign-intent" }, + unset: [], + }, + ], + }, + }), + ).rejects.toThrow("Ordinary integration import unavailable"); + expect( + await fixture.db + .select() + .from(schema.integrationConfig) + .orderBy(schema.integrationConfig.integrationId), + ).toEqual(originalRows); + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: "loopback", + operationKey: owned.operationKey, + expectedCurrentRevisionId: host.initial.revisionId, + overlay, + }); + expect(candidate.sealedBytes).toBeLessThan(524288); + }); + }, + ), + ); + }); + }, +); diff --git a/apps/api/src/services/configuration-migration/legacy-integration-import.ts b/apps/api/src/services/configuration-migration/legacy-integration-import.ts new file mode 100644 index 000000000..9025d5cb6 --- /dev/null +++ b/apps/api/src/services/configuration-migration/legacy-integration-import.ts @@ -0,0 +1,215 @@ +/** Existing ordinary integration batch adapter; its sole issuer is the + * adjacent built-in intent module, with no future workflow registration. */ +import type { + AdminIntegrationConfigImported, + AdminIntegrationImportFailure, +} from "@openmapx/core/api-contracts"; +import { ConfigurationPrerequisiteFailure, JobObservationPending } from "../admin-job-failure"; +import { + type AdministrativeConfigurationWriter, + withAdministrativeConfigurationJob, + withAdministrativeConfigurationWriter, + withConfigurationCommitParticipant, +} from "../administrative-configuration-writer"; +import { completeConfigurationApply, prepareIntegrationImport } from "../integration-config-import"; +import type { JobContext } from "../job-runner"; +import { + createApiOpsClient, + createDurableOpsKey, + executeAndWait, + OpsObservationPending, +} from "../ops-client"; +import { + discardUnadmittedTrustedConfigurationCandidate, + type ProspectiveConfigurationOverlay, + persistAndApplyProspectiveConfiguration, + prepareProspectiveConfiguration, + reloadProspectiveIntegrations, + retireSettledProspectiveConfiguration, +} from "../prospective-configuration"; +import { + type LegacyIntegrationImportIntent, + legacyIntegrationImportParticipation, + prepareLegacyIntegrationImport, +} from "./legacy-integration-import-intent"; + +export { + isLegacyIntegrationImportParticipant, + type LegacyIntegrationImportIntent, + prepareLegacyIntegrationImport, + retainedLegacyIntegrationImportIntent, +} from "./legacy-integration-import-intent"; + +export function withLegacyIntegrationImport( + intent: LegacyIntegrationImportIntent, + writer: AdministrativeConfigurationWriter, + run: (overlay: ProspectiveConfigurationOverlay) => Promise, +): Promise { + const state = legacyIntegrationImportParticipation(intent, writer); + return withConfigurationCommitParticipant(writer, state.participant, () => run(state.overlay)); +} + +export async function runLegacyIntegrationImport( + ctx: JobContext, + options: { body: unknown; actorId: string; operationKey: string }, +): Promise { + // Validation here determines only the finite lock set. The constructor repeats + // it from its own strict registry/schema observation before issuing authority. + const { getAllIntegrations } = await import("../../integration-host"); + const entries = prepareIntegrationImport( + options.body, + new Map( + getAllIntegrations().map((i) => [ + i.id, + i.manifest.configSchema as Record | undefined, + ]), + ), + ); + return withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { + actorId: options.actorId, + operationKey: options.operationKey, + integrationIds: entries.map((e) => e.integrationId), + signal: ctx.signal, + }, + async (writer) => { + const failure = { + ok: false as const, + error: "Integration import failed" as const, + recovery: "reconciliation_required" as const, + }; + const progress = { phase: "persistence" as "persistence" | "apply" | "reload" }; + let receipt: + | Awaited> + | undefined; + try { + const intent = await prepareLegacyIntegrationImport({ integrations: entries }, writer); + if (!intent.changedIntegrationIds.length) { + // A no-op is still a managed write request. Use the ordinary + // strict render preflight ONLY for read-only eligibility; this + // candidate is never persisted/applied or a migration participant. + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("configuration.inspect", options.operationKey), + { signal: ctx.signal }, + ); + const inert = await prepareProspectiveConfiguration({ + actorId: options.actorId, + operationKey: options.operationKey, + expectedCurrentRevisionId: current.revisionId, + overlay: { purpose: "render", coordinatorMode: "ensure-running", changes: [] }, + signal: ctx.signal, + }); + await discardUnadmittedTrustedConfigurationCandidate(inert); + return { + ok: true as const, + imported: 0, + skipped: [...intent.integrationIds], + message: "No configuration changes", + reloaded: 0, + enabled: getAllIntegrations().filter((i) => i.enabled).length, + }; + } + return await withLegacyIntegrationImport(intent, writer, async (overlay) => { + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("configuration.inspect", options.operationKey), + { signal: ctx.signal }, + ); + const candidate = await prepareProspectiveConfiguration({ + actorId: options.actorId, + operationKey: options.operationKey, + expectedCurrentRevisionId: current.revisionId, + overlay, + signal: ctx.signal, + }); + progress.phase = "apply"; + receipt = await persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + signal: ctx.signal, + }); + if (!completeConfigurationApply(receipt.application)) + return { + ...failure, + phase: progress.phase, + database: + receipt.saved === "unknown" + ? ("unknown" as const) + : receipt.saved === "conflict" + ? ("conflict" as const) + : ("retained" as const), + runtime: "unchanged" as const, + apply: receipt.application, + }; + progress.phase = "reload"; + await ctx.checkpoint({ + ...ctx.checkpointResult, + integrationReload: { + state: "attempted", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }, + }); + const reloaded = await reloadProspectiveIntegrations(candidate); + await ctx.checkpoint({ + ...ctx.checkpointResult, + _configurationCandidate: { + ...(ctx.checkpointResult?._configurationCandidate as Record), + phase: "settled", + }, + integrationReload: { + state: "complete", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + result: reloaded, + }, + }); + await retireSettledProspectiveConfiguration(candidate); + return { + ok: true as const, + imported: intent.changedIntegrationIds.length, + skipped: intent.integrationIds.filter( + (id) => !intent.changedIntegrationIds.includes(id), + ), + ...reloaded, + }; + }); + } catch (error) { + if ( + error instanceof JobObservationPending || + error instanceof OpsObservationPending || + error instanceof ConfigurationPrerequisiteFailure + ) + throw error; + if (progress.phase === "reload") + await ctx.checkpoint({ + ...ctx.checkpointResult, + _configurationCandidate: { + ...(ctx.checkpointResult?._configurationCandidate as Record), + phase: "unknown", + }, + integrationReload: { + ...(ctx.checkpointResult?.integrationReload as Record), + state: "unknown", + }, + }); + return { + ...failure, + phase: progress.phase, + database: + progress.phase === "persistence" || + receipt?.saved === "committed" || + receipt?.saved === "unchanged" + ? "retained" + : "unknown", + runtime: progress.phase === "reload" ? "unknown" : "unchanged", + ...(receipt ? { apply: receipt.application } : {}), + }; + } + }, + ), + ); +} diff --git a/apps/api/src/services/coordinator-lifecycle.ts b/apps/api/src/services/coordinator-lifecycle.ts new file mode 100644 index 000000000..fa536b393 --- /dev/null +++ b/apps/api/src/services/coordinator-lifecycle.ts @@ -0,0 +1,361 @@ +/** Sole nominal issuer for the existing queued DM Apply / Update producers. + * It derives provenance from the retained owned job; no target, callback, + * participant, result, authority flag or prepared attempt can be supplied. */ +import { randomUUID } from "node:crypto"; +import { + canonicalJson, + coordinatorLifecycleMetadataSchema, + type PreparedConfigurationRecord, + readOpsTokenFile, +} from "@openmapx/core/ops"; +import { db } from "../db"; +import { adminJob } from "../db/schema"; +import { readProjection } from "./admin-job-projection"; +import { jobOwnership } from "./admin-job-state"; +import { + type AdministrativeConfigurationWriter, + administrativeConfigurationState, +} from "./administrative-configuration-writer-owner"; +import type { JobContext } from "./job-runner"; +import { createDurableOpsKey } from "./ops-client"; +import { + configurationPublisherFromEnvironment, + readPreparedTrustedConfigurationCandidate, +} from "./trusted-config-publication-store"; + +declare const coordinatorIntent: unique symbol; +export interface CoordinatorLifecycleIntent { + readonly [coordinatorIntent]: true; +} +type State = { + writer: AdministrativeConfigurationWriter; + job: JobContext; + provenance: string; + candidateId: string; + action: "apply" | "update"; + originalOrdinal: number | null; + noDeps: boolean; +}; +const intents = new WeakMap(); +const refuse = () => Error("Original coordinator lifecycle job required"); +export function coordinatorLifecycleOperationKey(jobId: string, ordinal: number | null) { + return createDurableOpsKey( + "configuration.coordinator-lifecycle", + `${jobId}:${ordinal ?? "apply"}`, + ); +} +function provenance(row: typeof adminJob.$inferSelect) { + return canonicalJson({ + id: row.id, + type: row.type, + payload: row.payload, + createdBy: row.createdBy, + }); +} +export async function prepareCoordinatorLifecycleIntent( + writer: AdministrativeConfigurationWriter, + originalOrdinal: number | null, +): Promise { + const { job, participant, transactionActive } = administrativeConfigurationState(writer); + if ( + !job.execution || + participant || + transactionActive || + job.observationOnly || + writer.actorId !== job.jobId || + writer.operationKey !== coordinatorLifecycleOperationKey(job.jobId, originalOrdinal) || + writer.integrationIds.length + ) + throw refuse(); + const [row] = await db.select().from(adminJob).where(jobOwnership(job.jobId, job.execution)); + if (!row || row.type !== job.jobType || canonicalJson(row.payload) !== canonicalJson(job.payload)) + throw refuse(); + const payload = row.payload as Record; + let action: "apply" | "update", + noDeps = false; + if (originalOrdinal === null) { + if ( + row.type !== "service.apply" || + payload.service !== "data-manager" || + Object.keys(payload).some((k) => k !== "service") + ) + throw refuse(); + action = "apply"; + } else { + if ( + !Number.isInteger(originalOrdinal) || + originalOrdinal < 0 || + originalOrdinal > 255 || + row.type !== "service.bulk" || + payload.action !== "update" || + !Array.isArray(payload.serviceIds) || + payload.serviceIds[originalOrdinal] !== "data-manager" || + payload.serviceIds.filter((id) => id === "data-manager").length !== 1 || + (payload.noDeps !== undefined && typeof payload.noDeps !== "boolean") || + Object.keys(payload).some( + (k) => !["action", "serviceIds", "all", "region", "continueOnError", "noDeps"].includes(k), + ) + ) + throw refuse(); + action = "update"; + noDeps = payload.noDeps === true; + } + const intent = Object.freeze({}) as CoordinatorLifecycleIntent; + intents.set(intent, { + writer, + job, + provenance: provenance(row), + candidateId: randomUUID(), + action, + originalOrdinal, + noDeps, + }); + return intent; +} +export async function assertCoordinatorLifecycleIntent( + intent: CoordinatorLifecycleIntent, + writer: AdministrativeConfigurationWriter, +) { + const state = intents.get(intent), + owner = administrativeConfigurationState(writer); + if ( + !state || + state.writer !== writer || + state.job !== owner.job || + owner.participant || + (owner.transactionActive && owner.transactionMode !== "read") + ) + throw refuse(); + const [row] = await db + .select() + .from(adminJob) + .where(jobOwnership(state.job.jobId, state.job.execution)); + if (!row || state.provenance !== provenance(row)) throw refuse(); + return state; +} +export async function assertCoordinatorPullIntent( + intent: CoordinatorLifecycleIntent, + context: JobContext, +) { + const state = intents.get(intent); + if (!state || state.job !== context || state.action !== "update") throw refuse(); + return assertCoordinatorLifecycleIntent(intent, state.writer); +} + +/** Historical MAC provenance recheck against the actual current owned row. + * This does not mint a fresh intent, candidate or attempt. */ +export function assertRetainedCoordinatorLifecycleProvenance( + row: typeof adminJob.$inferSelect, + record: PreparedConfigurationRecord, +) { + const fixed = coordinatorLifecycleMetadataSchema.parse(record.privateCoordinatorLifecycle), + payload = row.payload as Record; + if ( + row.id !== record.apiJobId || + record.actorId !== row.id || + record.apiFollowUp !== `coordinator-${fixed.action}` || + !record.maintenanceId || + !record.prepareOperationKey || + canonicalJson(record.affectedServiceIds) !== '["data-manager"]' || + canonicalJson(record.privateOverlay) !== + canonicalJson({ purpose: "render", coordinatorMode: "ensure-running", changes: [] }) || + (fixed.action === "apply" + ? row.type !== "service.apply" || + payload.service !== "data-manager" || + Object.keys(payload).some((k) => k !== "service") + : row.type !== "service.bulk" || + payload.action !== "update" || + !Array.isArray(payload.serviceIds) || + payload.serviceIds[fixed.originalOrdinal] !== "data-manager" || + payload.serviceIds.filter((id) => id === "data-manager").length !== 1 || + fixed.noDeps !== (payload.noDeps === true) || + fixed.pull.operation.candidateId !== record.candidateId || + fixed.pull.attempt.apiJobId !== row.id || + fixed.pull.attempt.parentOperationKey !== record.parentOperationKey || + fixed.pull.attempt.parentFingerprint !== record.parentFingerprint || + fixed.pull.attempt.parentDeadlineAt !== record.parentDeadlineAt) + ) + throw refuse(); + return fixed; +} + +/** The one first-delivery CAS. Generic managed checkpoints cannot change a + * prepared pull into an attempted one. Old-owner attempt bytes are preserved. */ +export async function claimCoordinatorPullDelivery(context: JobContext, candidateId: string) { + if (!context.execution || context.observationOnly) throw refuse(); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + let updated: Record | undefined; + const delivery = await db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(adminJob) + .where(jobOwnership(context.jobId, context.execution)) + .for("update"); + if (!row) throw refuse(); + const result = row.result as Record, + checkpoint = result?._configurationCandidate as Record; + if ( + !checkpoint || + checkpoint.candidateId !== candidateId || + typeof checkpoint.sealedOperationKey !== "string" || + checkpoint.operationKey !== + coordinatorLifecycleOperationKey( + context.jobId, + (row.payload as { serviceIds?: string[] })?.serviceIds?.indexOf("data-manager") ?? null, + ) + ) + throw refuse(); + const retained = await readPreparedTrustedConfigurationCandidate( + configurationPublisherFromEnvironment(token), + checkpoint.sealedOperationKey, + ), + record = retained.record, + fixed = assertRetainedCoordinatorLifecycleProvenance(row, record); + if ( + fixed.action !== "update" || + record.candidateId !== candidateId || + checkpoint.phase !== "prepared" || + result._configurationMaintenance !== undefined + ) + throw refuse(); + const pull = fixed.pull, + projection = readProjection(result), + entry = projection.operations[pull.operationKey]; + if ( + !entry || + projection.steps[pull.stepAddress]?.operationKey !== pull.operationKey || + canonicalJson(entry.operation) !== canonicalJson(pull.operation) || + canonicalJson(entry.attempt) !== canonicalJson(pull.attempt) + ) + throw refuse(); + if (entry.phase === "unissued") throw refuse(); + if (entry.phase !== "prepared") { + updated = result; + return { operationKey: pull.operationKey, fresh: false }; + } + if ( + result._cancelRequestedAt !== undefined || + Date.now() >= pull.requestedDeadlineAt || + Date.now() >= record.parentDeadlineAt || + retained.snapshot.expiresAtMs <= Date.now() + ) + throw refuse(); + projection.operations[pull.operationKey] = { ...entry, phase: "admitting" }; + updated = { ...result, opsProjection: projection }; + const saved = await tx + .update(adminJob) + .set({ result: updated }) + .where(jobOwnership(context.jobId, context.execution)) + .returning({ id: adminJob.id }); + if (saved.length !== 1) throw refuse(); + return { operationKey: pull.operationKey, fresh: true }; + }); + if (!updated) throw refuse(); + context.checkpointResult = updated; + return delivery; +} + +/** Local nonissuance is derived from the actual locked PREPARED slot. First + * delivery MUST atomically leave that phase before any HTTP POST; generic + * checkpoints cannot restore it. This is no agent refusal or terminal copy. */ +export async function closeUnissuedCoordinatorPull( + context: JobContext, + candidateId: string, + packetOperationKey?: string, +): Promise { + if (!context.execution || context.observationOnly) throw refuse(); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const updated = await db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(adminJob) + .where(jobOwnership(context.jobId, context.execution)) + .for("update"); + if (!row || row.type !== "service.bulk") throw refuse(); + const payload = row.payload as { action?: string; serviceIds?: string[] }, + ordinal = payload.serviceIds?.indexOf("data-manager") ?? -1; + if ( + payload.action !== "update" || + ordinal < 0 || + payload.serviceIds?.filter((id) => id === "data-manager").length !== 1 + ) + throw refuse(); + const result = (row.result ?? {}) as Record, + projection = readProjection(result), + stepAddress = createDurableOpsKey( + "configuration.coordinator-pull", + `${context.jobId}:${ordinal}`, + ), + key = projection.steps[stepAddress]?.operationKey, + entry = key ? projection.operations[key] : undefined; + if (!entry) return result; + if ( + entry.operation?.kind !== "service.pull" || + entry.operation.serviceId !== "data-manager" || + entry.operation.candidateId !== candidateId || + !entry.attempt || + entry.attempt.apiJobId !== row.id || + entry.attempt.apiOwnerVersion > context.execution!.version || + canonicalJson({ + operationKey: entry.attempt.parentOperationKey, + fingerprint: entry.attempt.parentFingerprint, + deadlineAt: entry.attempt.parentDeadlineAt, + }) !== canonicalJson(result._parentExecution) + ) + throw refuse(); + if (entry.phase !== "prepared") return result; + if ( + result._configurationMaintenance !== undefined || + result._configurationMaintenanceDecision !== undefined || + result._configurationMaintenanceRollback !== undefined + ) + throw refuse(); + try { + const checkpoint = result._configurationCandidate as + | { sealedOperationKey?: string } + | undefined; + const originalPacket = checkpoint?.sealedOperationKey ?? packetOperationKey; + if (!originalPacket) { + // This caller failed before constructing any private packet. A first + // delivery still required both that packet and the staged checkpoint. + } else { + const source = await readPreparedTrustedConfigurationCandidate( + configurationPublisherFromEnvironment(token), + originalPacket, + ), + fixed = assertRetainedCoordinatorLifecycleProvenance(row, source.record); + if ( + fixed.action !== "update" || + source.record.candidateId !== candidateId || + canonicalJson(fixed.pull.operation) !== canonicalJson(entry.operation) || + canonicalJson(fixed.pull.attempt) !== canonicalJson(entry.attempt) + ) + throw refuse(); + } + } catch (error) { + // Missing private bytes cannot have been delivered from a PREPARED slot: + // the fixed first-delivery CAS also requires this MAC candidate/checkpoint. + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + projection.operations[key!] = { + ...entry, + phase: "unissued", + terminal: true, + nonissuance: { + version: 1, + candidateId, + checkedOwnerVersion: context.execution!.version, + checkedAt: Date.now(), + }, + }; + const next = { ...result, opsProjection: projection }; + const saved = await tx + .update(adminJob) + .set({ result: next }) + .where(jobOwnership(context.jobId, context.execution)) + .returning({ id: adminJob.id }); + if (saved.length !== 1) throw refuse(); + return next; + }); + context.checkpointResult = updated; +} diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 0b20bdef7..159a5fd30 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -1,11 +1,22 @@ -import { randomUUID } from "node:crypto"; +import { createHmac, randomUUID } from "node:crypto"; +import { + adminIntegrationConfigImportedSchema, + adminIntegrationImportFailureSchema, +} from "@openmapx/core/api-contracts"; +import { canonicalJson, hardlinkApplySummarySchema, readOpsTokenFile } from "@openmapx/core/ops"; import { and, asc, eq, inArray, sql } from "drizzle-orm"; import { db } from "../db"; import { adminJob } from "../db/schema"; import { dbActorId } from "../utils/actor"; import { checkpointOwnedAdminJob } from "./admin-job-checkpoint"; -import { AdminJobFailure, JobObservationPending, publicAdminJobResult } from "./admin-job-failure"; +import { + AdminJobFailure, + ConfigurationPrerequisiteFailure, + JobObservationPending, + publicAdminJobResult, +} from "./admin-job-failure"; import { cancelAdminJobOperations, resumeAdminJobObservation } from "./admin-job-ops"; +import { readProjection } from "./admin-job-projection"; import { adminJobObservationDue, claimAdminJob, @@ -22,9 +33,14 @@ import { readJobExecution, renewAdminJobClaim, } from "./admin-job-state"; +import { coordinatorLifecycleOperationKey } from "./coordinator-lifecycle"; import { type JobStatusEvent, jobEventBus, TERMINAL_JOB_STATUSES } from "./job-events"; import { ApiOpsError, OpsObservationPending } from "./ops-client"; import { appApiRestartCheckpoint } from "./system-update-state"; +import { + completeTrustedConfigurationApply, + parseTrustedConfigurationApplyOutcome, +} from "./trusted-config-outcome"; type AppApiReplacementOutcome = "applied" | "rolled-back" | "failed"; interface AppApiRuntimeInfo { @@ -93,10 +109,64 @@ export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ "service.restart", "service.apply", "compose.render", + "configuration.apply", + "compose.start", + "integration.config.import", "system.diagnostics", "system.update", ] as const; +/** Routing only, never admission authority. The selected fixed producer must + * still authenticate the original private packet, owned row/nonce, source and + * every retained attempt before it can deliver an original continuation. */ +function hasRetainedImmediateConfiguration(job: { + id: string; + type: string; + payload: unknown; + result: unknown; +}): boolean { + const result = job.result as Record | null, + payload = job.payload as Record | null; + return ( + ["service.apply", "configuration.apply", "integration.config.import"].includes(job.type) && + result?._immediateJobId === job.id && + typeof result._immediateActorId === "string" && + typeof result._immediateIntentDigest === "string" && + /^[a-f0-9]{64}$/.test(result._immediateIntentDigest) && + typeof payload?.operationKey === "string" && + /^opk1_[A-Za-z0-9_-]{16,64}$/.test(payload.operationKey) && + !!result._configurationCandidate + ); +} +/** Routing only. The fixed producer reauthenticates the original MAC packet, + * actual owned row/payload and saved decision before any next phase. */ +function hasRetainedQueuedCoordinator(job: { + id: string; + type: string; + payload: unknown; + result: unknown; +}) { + const payload = job.payload as Record | null, + candidate = (job.result as { _configurationCandidate?: { operationKey?: unknown } } | null) + ?._configurationCandidate; + if (!payload) return false; + const batch = (job.result as Record | null)?._serviceUpdateBatch; + if (job.type === "service.bulk" && payload.action === "update" && batch) return true; // Routing only; the fixed handler validates the actual owned batch/ACK chain. + if (!candidate) return false; + const ordinal = + job.type === "service.apply" && + payload.service === "data-manager" && + Object.keys(payload).every((k) => k === "service") + ? null + : job.type === "service.bulk" && + payload.action === "update" && + Array.isArray(payload.serviceIds) + ? payload.serviceIds.indexOf("data-manager") + : -1; + return ( + ordinal !== -1 && candidate.operationKey === coordinatorLifecycleOperationKey(job.id, ordinal) + ); +} class JobPersistenceUnavailable extends Error {} class AdminJobRunner { @@ -127,6 +197,342 @@ class AdminJobRunner { return id; } + private async immediate( + options: { + type: "configuration.apply" | "service.apply" | "compose.start" | "integration.config.import"; + actorId?: string; + operationKey: string; + intent: unknown; + signal?: AbortSignal; + }, + run: ( + ctx: JobContext, + ) => Promise<{ value: T; result: Record; failed?: boolean }>, + restore: (result: Record) => T, + ): Promise { + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const intentDigest = createHmac("sha256", token) + .update("openmapx-immediate-configuration-intent-v1\0") + .update(canonicalJson(options.intent)) + .digest("hex"); + const actorId = options.actorId ?? "internal"; + const find = (connection: Pick) => + connection + .select() + .from(adminJob) + .where( + and( + sql`${adminJob.payload}->>'operationKey' = ${options.operationKey}`, + sql`${adminJob.result}->>'_immediateActorId' = ${actorId}`, + ), + ) + .limit(2); + const prior = await find(db); + const retained = prior.length + ? prior + : await db.transaction(async (tx) => { + const [lock] = await tx.execute<{ held: boolean }>( + sql`SELECT pg_try_advisory_xact_lock(${0x4f4d58},1) AS held`, + ); + if (lock?.held !== true) throw new JobObservationPending("recovery_required"); + const current = await find(tx); + if (current.length) return current; + const id = randomUUID(), + execution = { owner: randomUUID(), version: 1 }; + const [created] = await tx + .insert(adminJob) + .values({ + id, + type: options.type, + status: "running", + startedAt: new Date(), + payload: { operationKey: options.operationKey }, + createdBy: dbActorId(options.actorId), + result: { + _execution: execution, + _observationLeaseUntil: new Date(Date.now() + 60000).toISOString(), + _immediateJobId: id, + _immediateActorId: actorId, + _immediateIntentDigest: intentDigest, + }, + }) + .returning(); + if (!created) + throw new JobPersistenceUnavailable("Immediate administrative job unavailable"); + return [{ ...created, _fresh: true }]; + }); + if (retained.length !== 1) throw new JobObservationPending("recovery_required"); + let job = retained[0]!, + checkpoint = (job.result ?? {}) as Record, + recovering = false; + if (job.type !== options.type || checkpoint._immediateIntentDigest !== intentDigest) + throw new AdminJobFailure("Retained configuration intent conflicts", { + errorCode: "conflict", + }); + if (!(job as { _fresh?: boolean })._fresh) { + if (["success", "failed"].includes(job.status)) return restore(checkpoint); + if ( + !["running", "cancel_pending"].includes(job.status) || + !checkpoint._configurationCandidate || + !["service.apply", "configuration.apply", "integration.config.import"].includes( + options.type, + ) + ) + throw new JobObservationPending("recovery_required"); + const claimed = await claimAdminJob(job, {}, true); + if (!claimed) throw new JobObservationPending("recovery_required"); + job = claimed; + checkpoint = (claimed.result ?? {}) as Record; + recovering = true; + } + const execution = readJobExecution(checkpoint)!; + const controller = new AbortController(), + signal = options.signal + ? AbortSignal.any([controller.signal, options.signal]) + : controller.signal; + this.active.set(job.id, controller); + let value: T | undefined, failure: unknown; + const heartbeat = setInterval(() => { + void renewAdminJobClaim(job.id, execution).catch((error) => controller.abort(error)); + }, 10000); + heartbeat.unref(); + try { + await this.executeJob( + job.id, + options.type, + { operationKey: options.operationKey }, + checkpoint, + signal, + false, + async (ctx) => { + if (recovering) { + return this.resumeImmediateConfiguration(ctx); + } + try { + const result = await run(ctx); + value = result.value; + if (result.failed) + throw new AdminJobFailure("Administrative effect failed", result.result); + return result.result; + } catch (error) { + failure = error; + throw error; + } + }, + ); + const [settled] = await db.select().from(adminJob).where(eq(adminJob.id, job.id)); + if (!settled || !["success", "failed"].includes(settled.status)) + throw new JobObservationPending("recovery_required"); + if (failure && !(value !== undefined && failure instanceof AdminJobFailure)) throw failure; + return value ?? restore((settled.result ?? {}) as Record); + } finally { + clearInterval(heartbeat); + this.active.delete(job.id); + this.logSeq.delete(job.id); + } + } + + /** The same fixed original producer serves a repeated request and startup + * reconciliation. No registered callback, new intent or latest desired state + * is used; all authority is reconstructed by its nominal private reader. */ + private async resumeImmediateConfiguration(ctx: JobContext): Promise> { + if ( + !hasRetainedImmediateConfiguration({ + id: ctx.jobId, + type: ctx.jobType ?? "", + payload: ctx.payload, + result: ctx.checkpointResult, + }) + ) + throw new JobObservationPending("recovery_required"); + const actorId = ctx.checkpointResult!._immediateActorId as string, + operationKey = ctx.payload.operationKey as string, + { withAdministrativeConfigurationJob, withAdministrativeConfigurationWriter } = await import( + "./administrative-configuration-writer" + ), + { resumeRetainedConfigurationCandidate, retainedConfigurationIntegrationIds } = await import( + "./prospective-configuration" + ), + integrationIds = await retainedConfigurationIntegrationIds(ctx, actorId, operationKey), + result = await withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { actorId, operationKey, integrationIds, signal: ctx.signal }, + () => resumeRetainedConfigurationCandidate(), + ), + ); + const [canceled] = await db + .select() + .from(adminJob) + .where(jobOwnership(ctx.jobId, ctx.execution, ["canceled"])); + if (canceled) { + // The fixed producer already published its actual canceled/nonissuance + // completion under this nonce. Observe it; generic settlement must not + // overwrite that terminal source with success or failure. + this.announce(ctx.jobId, "canceled", { + result: publicAdminJobResult(canceled.result as Record | null), + finishedAt: canceled.finishedAt?.toISOString(), + }); + throw new JobOwnershipLost(); + } + if ( + !completeTrustedConfigurationApply( + parseTrustedConfigurationApplyOutcome(result.configuration), + ) + ) + throw new AdminJobFailure("Original configuration did not apply", result); + return result; + } + + /** Desired values remain transient or in the guarded private candidate, never job payload/result. */ + applyImmediateConfiguration( + options: import("./trusted-config-operations").ApplyTrustedConfigurationOptions, + ) { + return this.immediate( + { + type: options.followUp === "service-apply" ? "service.apply" : "configuration.apply", + actorId: options.actorId, + operationKey: options.operationKey, + signal: options.signal, + intent: { + kind: options.kind, + serviceId: options.serviceId, + integrationId: options.integrationId, + selectedRoots: options.selectedRoots, + overlay: options.overlay, + coordinatorMode: options.coordinatorMode, + snapshot: options.snapshot, + followUp: options.followUp, + ...(Object.hasOwn(options, "expectedCurrentRevisionId") + ? { expectedCurrentRevisionId: options.expectedCurrentRevisionId } + : {}), + }, + }, + async (ctx) => { + const { applyTrustedConfigurationWithEvidence } = await import( + "./trusted-configuration-producer" + ); + const value = await applyTrustedConfigurationWithEvidence({ + ...options, + ctx, + signal: ctx.signal, + }); + return { + value: { ...value, jobId: ctx.jobId }, + result: { configuration: value.outcome }, + failed: value.outcome.outcome !== "committed" || value.outcome.followUp !== "complete", + }; + }, + (result) => { + if (result.errorCode === "configuration_prerequisite") + throw new ConfigurationPrerequisiteFailure(); + let outcome = parseTrustedConfigurationApplyOutcome(result.configuration); + let integrationReload: + | import("./trusted-config-operations").CapturedIntegrationReloadResult + | undefined; + if (options.kind === "integrationConfig.apply" && outcome.outcome === "committed") { + const retained = result.integrationReload as + | { state?: string; revisionId?: string; result?: unknown } + | undefined; + if (retained?.state === "failed") outcome = { ...outcome, followUp: "failed" }; + else if (retained?.state === "complete" && retained.revisionId === outcome.revisionId) { + const parsed = adminIntegrationConfigImportedSchema + .pick({ message: true, reloaded: true, enabled: true }) + .parse(retained.result); + integrationReload = parsed; + } else throw new JobObservationPending("recovery_required"); + } + if (options.followUp && outcome.outcome === "committed") { + const retained = result.serviceCredential as + | { serviceId?: string; revisionId?: string; followUp?: string } + | undefined; + if ( + !retained || + retained.serviceId !== options.serviceId || + retained.revisionId !== outcome.revisionId || + !["complete", "failed"].includes(retained.followUp ?? "") + ) + throw new JobObservationPending("recovery_required"); + if (retained.followUp === "failed") outcome = { ...outcome, followUp: "failed" }; + } + return { + outcome, + ...(integrationReload ? { integrationReload } : {}), + ...(typeof result._immediateJobId === "string" ? { jobId: result._immediateJobId } : {}), + }; + }, + ); + } + + startImmediateCompose(options: { actorId: string; operationKey: string; signal?: AbortSignal }) { + return this.immediate( + { + ...options, + type: "compose.start", + intent: { action: "compose.start", coordinatorMode: "ensure-running" }, + }, + async (ctx) => { + const { prepareServiceGeneration } = await import("./service-preparation"); + const { executeAdminJobOperation } = await import("./admin-job-ops"); + const prepared = await prepareServiceGeneration(ctx, "compose-start", { + coordinatorMode: "ensure-running", + }); + let ok = true; + try { + await executeAdminJobOperation( + ctx, + { kind: "stack.start", expectedConfigurationRevisionId: prepared.revisionId }, + "admin-job.compose.start", + ); + } catch (error) { + if (error instanceof JobObservationPending || error instanceof OpsObservationPending) + throw error; + ok = false; + } + const value = { ok, stdout: "", hardlinks: prepared.hardlinks }; + return { value, result: { compose: value }, failed: !ok }; + }, + (result) => { + const value = result.compose as + | { ok?: boolean; stdout?: string; hardlinks?: unknown } + | undefined; + if (typeof value?.ok !== "boolean" || typeof value.stdout !== "string") + throw new JobObservationPending("recovery_required"); + return { + ok: value.ok, + stdout: value.stdout, + hardlinks: hardlinkApplySummarySchema.parse(value.hardlinks), + }; + }, + ); + } + + importImmediateIntegrationConfiguration(options: { + body: unknown; + actorId: string; + operationKey: string; + signal?: AbortSignal; + }) { + return this.immediate( + { ...options, type: "integration.config.import", intent: options.body }, + async (ctx) => { + const { runLegacyIntegrationImport } = await import( + "./configuration-migration/legacy-integration-import" + ); + const value = await runLegacyIntegrationImport(ctx, options); + return { value, result: { integrationImport: value }, failed: !value.ok }; + }, + (result) => { + // Only exact public, non-secret import evidence is retained/restored. + if (result.errorCode === "configuration_prerequisite") + throw new ConfigurationPrerequisiteFailure(); + const success = adminIntegrationConfigImportedSchema.safeParse(result.integrationImport); + return success.success + ? success.data + : adminIntegrationImportFailureSchema.parse(result.integrationImport); + }, + ); + } + /** Publish a status transition so live viewers see it without polling. */ private announce( jobId: string, @@ -367,7 +773,9 @@ class AdminJobRunner { observe = false, observationSignal?: AbortSignal, ): Promise { - const controller = new AbortController(); + const controller = new AbortController(), + retainedConfiguration = + observe && (hasRetainedImmediateConfiguration(job) || hasRetainedQueuedCoordinator(job)); this.active.set(job.id, controller); this.logSeq.set(job.id, 0); @@ -386,7 +794,7 @@ class AdminJobRunner { heartbeat.unref?.(); } }; - if (!observe) { + if (!observe || retainedConfiguration) { heartbeat = setTimeout(() => void renew(), 10_000); heartbeat.unref?.(); } @@ -398,7 +806,7 @@ class AdminJobRunner { observe ? AbortSignal.any([ controller.signal, - AbortSignal.timeout(3_000), + AbortSignal.timeout(retainedConfiguration ? 120_000 : 3_000), ...(observationSignal ? [observationSignal] : []), ]) : controller.signal, @@ -426,6 +834,7 @@ class AdminJobRunner { checkpointResult: Record | null, signal: AbortSignal, observe = false, + immediateHandler?: JobHandler, ): Promise { const retirement = new AbortController(); signal = AbortSignal.any([signal, retirement.signal]); @@ -434,7 +843,20 @@ class AdminJobRunner { retirement.abort(error); throw error; }; - const handler = observe ? resumeAdminJobObservation : this.handlers.get(type); + const retainedImmediate = + observe && + hasRetainedImmediateConfiguration({ id: jobId, type, payload, result: checkpointResult }), + retainedQueuedCoordinator = + observe && + hasRetainedQueuedCoordinator({ id: jobId, type, payload, result: checkpointResult }), + retainedConfiguration = retainedImmediate || retainedQueuedCoordinator, + handler = retainedImmediate + ? (ctx: JobContext) => this.resumeImmediateConfiguration(ctx) + : retainedQueuedCoordinator + ? this.handlers.get(type) + : observe + ? resumeAdminJobObservation + : (immediateHandler ?? this.handlers.get(type)); const execution = readJobExecution(checkpointResult); let checkpointTail = Promise.resolve(); let checkpointBaseline = structuredClone((checkpointResult ?? {}) as Record); @@ -442,7 +864,7 @@ class AdminJobRunner { jobId, jobType: type, execution, - observationOnly: observe, + observationOnly: observe && !retainedConfiguration, payload, signal, checkpointResult, @@ -688,8 +1110,20 @@ class AdminJobRunner { }, }); if (outcome === "pending") { - await pending("transport"); - return; + const original = current.result as Record | null; + if ( + !retainedConfiguration || + Object.keys(readProjection(original).operations).length !== 0 || + original?._dataExecutions !== undefined + ) { + await pending("transport"); + return; + } + // Routing only: before any child admission the generic canceller + // has no terminal to observe. The fixed producer must authenticate + // its private candidate and locked nonissuance state before cleanup + // or canceled publication. The cancel marker still blocks forward + // admission; neither an empty projection nor this status grants it. } if (outcome === "canceled") { const finishedAt = new Date(); diff --git a/apps/api/src/services/ops-client.ts b/apps/api/src/services/ops-client.ts index 713f0f13d..a23980e7f 100644 --- a/apps/api/src/services/ops-client.ts +++ b/apps/api/src/services/ops-client.ts @@ -23,7 +23,9 @@ import { opsKindRequiresControlAuthority, PRIVACY_BACKUP_TAR_MEDIA_TYPE, type PrivacyBackupSubjectExportRequest, + parseOpsResult, privacyBackupSubjectExportRequestSchema, + type TrustedConfigurationApplyOperation, } from "@openmapx/core/ops"; import { DAWARICH_TAR_MEDIA_TYPE, @@ -46,7 +48,9 @@ export class ApiOpsError extends Error { readonly failureResult?: OpsFailureResult, ) { super( - errorClass === "host_update_required" || errorClass === "control_upgrade_required" + errorClass === "host_update_required" || + errorClass === "control_upgrade_required" || + errorClass === "configuration_prerequisite" ? OPS_PUBLIC_ERROR_MESSAGES[errorClass] : "Operations request failed", ); @@ -393,7 +397,7 @@ export async function waitForOpsResult( ) pending("invalid_identity"); await options.onStatus?.(status); - if (status.state === "succeeded") return status.result; + if (status.state === "succeeded") return parseOpsResult(kind, status.result); if (status.state === "failed" || status.state === "timed_out") { if (status.errorClass === "recovery_required") pending("recovery_required"); throw new ApiOpsError(status.errorClass, status.failureResult); @@ -647,7 +651,7 @@ export async function submitOpsOperationWithRecovery /** A failed transport/status is never evidence that a configuration did not commit. */ export async function executeTrustedConfigurationAndWait( client: ApiOpsClient, - operation: Extract, + operation: TrustedConfigurationApplyOperation, operationKey: string, options: WaitForOpsResultOptions = {}, ): Promise { diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts new file mode 100644 index 000000000..4dac09640 --- /dev/null +++ b/apps/api/src/services/prospective-configuration.ts @@ -0,0 +1,5415 @@ +import { createHash, createHmac, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, open, unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { controlChildOperationKey } from "@openmapx/core/build-artifacts"; +import { + acquireTrustedConfigurationQueueLock, + type ConfigurationHardlinksPreview, + type ConfigurationMaintenanceDecision, + type ConfigurationMaintenanceIdentity, + type ConfigurationMaintenanceReceipt, + type ConfigurationMaintenanceSettle, + type ConfigurationSourcesInspection, + type CoordinatorMode, + canonicalJson, + configurationApplyResultSchema, + configurationHardlinksPreviewSchema, + configurationMaintenanceDecisionSchema, + configurationMaintenanceIdentitySchema, + configurationMaintenancePrepareSchema, + configurationMaintenanceReceiptSchema, + configurationMaintenanceSettleRequestDigest, + configurationMaintenanceSettleSchema, + coordinatorModeSchema, + createControlRootOperationKey, + encodePreparedConfigurationRecord, + inspectPreparedConfigurationRecord, + type OpsOperation, + opsOperationFingerprint, + opsOperationSchema, + type PreparedConfigurationRecord, + parseControlRootOperationKey, + readOpsTokenFile, + sealTrustedConfigurationSnapshot, + type TrustedConfigurationPayload, + trustedConfigurationPayloadSchema, +} from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; +import { and, eq, sql } from "drizzle-orm"; +import { z } from "zod"; +import * as schema from "../db/schema"; +import { resolveIntegrationConfigWithSourcesStrict } from "../integration-config"; +import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; +import { + deliverCoordinatorPullSubmission, + ensureAdminJobReceiptDelivery, + executeAdminJobOperation, + prepareAdminJobOperationRoot, + readProjection, + reserveCoordinatorPullSubmission, + resumeAdminJobObservation, +} from "./admin-job-ops"; +import { + type ApiJobChildAttempt, + issueApiJobChildAssertion, + prepareApiJobChildAttempt, + prepareApiJobRoot, +} from "./admin-job-parent"; +import { + hasPendingReceiptDelivery, + hasUnsettledManagedOperation, + jobOwnership, +} from "./admin-job-state"; +import { + administrativeConfigurationFileObservations, + closeAdministrativeConfigurationRead, + openAdministrativeConfigurationRead, + withAdministrativeConfigurationReadScope, +} from "./administrative-configuration-context"; +import { + type AdministrativeConfigurationReadContext, + type AdministrativeConfigurationWriter, + administrativeConfigurationState, + assertAdministrativeConfigurationRead, + ConfigurationWriteCommitUnknown, + ConfigurationWriteRolledBack, + type ConfigurationWriteTransaction, + currentAdministrativeConfigurationWriter, + withAdministrativeConfigurationRead, + withConfigurationWriteTransaction, +} from "./administrative-configuration-writer"; +import { + configurationMaintenanceDecisionCheckpointSchema as abandonCheckpointSchema, + assertConfigurationSavedRows, + type ConfigurationSavedRow, + configurationMaintenanceDbDecisionDigest, + configurationMaintenanceDecisionCheckpointSchema, + configurationMaintenanceRollbackCheckpointSchema, + configurationSavedRowSchema, +} from "./configuration-maintenance-decision"; +import { configurationMaintenanceSettleCheckpointSchema } from "./configuration-maintenance-settle-records"; +import { prepareOwnedConfigurationMaintenanceStep } from "./configuration-maintenance-step"; +import { + completedCoordinatorMaintenance, + completedCoordinatorMaintenanceSchema, +} from "./configuration-maintenance-step-records"; +import { assertLegacyIntegrationImportOverlay } from "./configuration-migration/legacy-integration-import-intent"; +import { + assertCoordinatorLifecycleIntent, + assertRetainedCoordinatorLifecycleProvenance, + type CoordinatorLifecycleIntent, + closeUnissuedCoordinatorPull, + coordinatorLifecycleOperationKey, +} from "./coordinator-lifecycle"; +import type { JobContext } from "./job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { encrypt } from "./secrets"; +import { resolveServiceConfigWithSourcesStrict } from "./service-config-resolver"; +import { getServiceRegistry } from "./service-registry"; +import { + completeTrustedConfigurationApply, + parseTrustedConfigurationApplyOutcome, + type TrustedConfigurationApplyOutcome, +} from "./trusted-config-outcome"; +import { + consumePublishedTrustedConfiguration, + findPreparedTrustedConfigurationCandidate, + publishSnapshot, + REJECTED, + readPreparedTrustedConfigurationCandidate, + retainPreparedTrustedConfigurationCandidate, + SNAPSHOT_MODE, + syncDirectory, + validateDirectory, + withPublicationLock, +} from "./trusted-config-publication-store"; + +export type BoundedConfigValue = + TrustedConfigurationPayload["serviceConfigs"][number]["values"][string]; +export type PlatformField = { source: "host" } | { source: "database"; value: T }; +export type PlatformConfigurationPatch = { + domain?: PlatformField; + acmeEmail?: PlatformField; + releaseSelector?: PlatformField; + defaultRegion?: PlatformField; + backupRetentionDays?: PlatformField; +}; +const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const key = z + .string() + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/) + .refine((key) => !key.includes("..") && !["__proto__", "prototype", "constructor"].includes(key)); +const config = trustedConfigurationPayloadSchema.shape.serviceConfigs.element.shape.values; +const ordinary = { set: config, unset: z.array(key).max(128) }; +const secretSet = { + key, + action: z.literal("set"), + value: z + .string() + .max(65536) + .refine((value) => value.isWellFormed()), +}; +const secretDelete = { key, action: z.literal("delete") }; +const overlaySchema = z.strictObject({ + purpose: z.enum([ + "render", + "selection", + "service-config", + "service-secret", + "integration-config", + "integration-secret", + "migration", + "platform", + "transit-credentials", + ]), + coordinatorMode: coordinatorModeSchema, + changes: z + .array( + z.union([ + z.strictObject({ kind: z.literal("service-config"), serviceId: id, ...ordinary }), + z.strictObject({ kind: z.literal("integration-config"), integrationId: id, ...ordinary }), + z.strictObject({ kind: z.literal("service-secret"), serviceId: id, ...secretSet }), + z.strictObject({ kind: z.literal("service-secret"), serviceId: id, ...secretDelete }), + z.strictObject({ kind: z.literal("integration-secret"), integrationId: id, ...secretSet }), + z.strictObject({ + kind: z.literal("integration-secret"), + integrationId: id, + ...secretDelete, + }), + z.strictObject({ kind: z.literal("selection"), selectedRoots: z.array(id).max(256) }), + z.strictObject({ + kind: z.literal("platform"), + patch: z.strictObject({ + domain: z.unknown().optional(), + acmeEmail: z.unknown().optional(), + releaseSelector: z.unknown().optional(), + defaultRegion: z.unknown().optional(), + backupRetentionDays: z.unknown().optional(), + }), + }), + ]), + ) + .max(512), +}); +type Overlay = z.infer; +export type ProspectiveConfigurationOverlay = Omit & { + changes: ( + | Exclude + | { kind: "platform"; patch: PlatformConfigurationPatch } + )[]; +}; +declare const candidateBrand: unique symbol; +export interface ValidatedConfigurationCandidate { + readonly [candidateBrand]: true; + readonly candidateId: string; + readonly operationKey: string; + readonly expectedCurrentRevisionId: string | null; + readonly revisionId: string; + readonly sourceFingerprint: string; + readonly sealedBytes: number; + readonly affectedServiceIds: readonly string[]; +} +export type ProspectiveConfigurationReceipt = { + candidateId: string; + revisionId: string; + saved: "unchanged" | "committed" | "conflict" | "unknown"; + application: TrustedConfigurationApplyOutcome; +}; +interface CandidateState { + writer: AdministrativeConfigurationWriter; + overlay: Overlay; + payload: TrustedConfigurationPayload; + sealed: ReturnType; + record: PreparedConfigurationRecord; + publisher: { directory: string; token: string; ownerUid: number; ownerGid: number }; + source: ConfigurationSourcesInspection | null; + preview?: ConfigurationHardlinksPreview; + rows: Pick< + AdministrativeConfigurationReadContext, + | "services" + | "integrations" + | "env" + | "serviceRows" + | "integrationRows" + | "serviceSecretRows" + | "integrationSecretRows" + > | null; + sourceRows: Pick< + AdministrativeConfigurationReadContext, + "serviceRows" | "integrationRows" | "serviceSecretRows" | "integrationSecretRows" + > | null; + persisted: boolean; + integrationReloadSnapshot: NonNullable< + AdministrativeConfigurationReadContext["integrationReloadSnapshot"] + > | null; +} +const candidates = new WeakMap(); +const recoveredCommittedCandidates = new WeakSet(); +const recoveredPreRootCandidates = new WeakSet(); +const rejectedUnissuedCandidates = new WeakSet(); +const discardedUnusedCandidates = new WeakMap< + ValidatedConfigurationCandidate, + ReturnType +>(); +/** A known comparison/refusal produced here selects an inert intent failure. + * External exceptions and observer aborts never mint this local evidence. */ +function rejectUnissuedCandidate( + candidate: ValidatedConfigurationCandidate, + message: string, +): never { + rejectedUnissuedCandidates.add(candidate); + throw Error(message); +} +const candidateStages = new WeakMap< + ValidatedConfigurationCandidate, + | "preflight" + | "prepared" + | "recovered_pre_save" + | "recovered_unissued" + | "save_attempted" + | "rolled_back" + | "committed" + | "never_prepared" + | "unissued" + | "unused" + | "discarded" + | "retired" +>(); +export function validatedConfigurationCandidateState( + candidate: ValidatedConfigurationCandidate, +): CandidateState { + const state = candidates.get(candidate); + if (!state || currentAdministrativeConfigurationWriter() !== state.writer) + throw Error("Configuration candidate unavailable"); + administrativeConfigurationState(state.writer); + return state; +} +/** Verify against the actual nominal candidate state, then the issuer's + * unchanged source/row checks. No caller metadata stands in for this join. */ +async function verifyConfigurationCommitParticipant( + candidate: ValidatedConfigurationCandidate, + writer: AdministrativeConfigurationWriter, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(writer); + if (state.writer !== writer) throw Error("Configuration candidate unavailable"); + if (state.record.privateCoordinatorLifecycle) { + const [actual] = await owner.connection + .select() + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)); + if (!actual) throw Error("Actual coordinator job unavailable"); + assertRetainedCoordinatorLifecycleProvenance(actual, state.record); + } + if (state.overlay.purpose === "migration") + assertLegacyIntegrationImportOverlay(writer, state.overlay); + await owner.participant?.verify(candidate, writer); +} +export function assertCommittedConfigurationCandidate( + candidate: ValidatedConfigurationCandidate, +): void { + validatedConfigurationCandidateState(candidate); + if (candidateStages.get(candidate) !== "committed") + throw Error("Committed configuration candidate required"); +} +export function assertKnownRolledBackConfigurationCandidate( + candidate: ValidatedConfigurationCandidate, +): void { + const state = validatedConfigurationCandidateState(candidate); + if (state.persisted || candidateStages.get(candidate) !== "rolled_back") + throw Error("Actual original rolled-back candidate required"); +} +export function assertUnadmittedConfigurationCandidate( + candidate: ValidatedConfigurationCandidate, +): void { + const state = validatedConfigurationCandidateState(candidate); + const stage = candidateStages.get(candidate); + if ( + !["preflight", "prepared", "rolled_back", "never_prepared", "unissued", "unused"].includes( + stage ?? "", + ) || + (state.record.maintenanceId && + !["preflight", "never_prepared", "unissued", "unused"].includes(stage ?? "")) + ) + throw Error("Known unadmitted configuration candidate required"); +} +/** Before any maintenance admission attempt, the original locked API row is + * the producer of nonissuance. A settled ordinary prerequisite is observed at + * its original address; it is neither undone nor permission to acquire a root. + * Missing/unknown helper delivery or any root/save attempt keeps the fence. */ +async function discardUnissuedMaintenanceCandidate( + candidate: ValidatedConfigurationCandidate, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer), + record = state.record; + if ( + !record.maintenanceId || + !record.prepareOperationKey || + !["prepared", "recovered_unissued"].includes(candidateStages.get(candidate) ?? "") || + state.persisted + ) + throw new JobObservationPending("recovery_required"); + const fixed = record.coordinatorPrerequisite, + projection = readProjection(owner.job.checkpointResult); + if (fixed && projection.operations[fixed.root.operationKey]) + await executeAdminJobOperation( + owner.job, + fixed.operation, + "configuration-maintenance.coordinator", + { operationKey: fixed.stepAddress, preparedRoot: fixed.root }, + ); + const originalPull = + record.privateCoordinatorLifecycle?.action === "update" + ? record.privateCoordinatorLifecycle.pull + : undefined; + if (originalPull) { + const entry = readProjection(owner.job.checkpointResult).operations[originalPull.operationKey]; + if (!entry) throw new JobObservationPending("recovery_required"); + if (entry.phase === "prepared") + await closeUnissuedCoordinatorPull(owner.job, candidate.candidateId, record.operationKey); + else if (entry.phase !== "unissued") { + const { observeRetainedAdminJobOperation } = await import("./admin-job-ops"), + { ApiOpsError } = await import("./ops-client"); + try { + await observeRetainedAdminJobOperation(owner.job, originalPull.operationKey); + } catch (error) { + if (!(error instanceof ApiOpsError)) throw error; + } + } + } + let checkpoint: Record | undefined; + await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [row] = await tx + .select({ + result: schema.adminJob.result, + payload: schema.adminJob.payload, + type: schema.adminJob.type, + }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + actual = readProjection(result), + helper = fixed && actual.operations[fixed.root.operationKey]; + if ( + !row || + row.type !== owner.job.jobType || + canonicalJson(row.payload) !== canonicalJson(owner.job.payload) + ) + throw new JobObservationPending("invalid_identity"); + const lifecycle = record.privateCoordinatorLifecycle, + prefix = + lifecycle?.action === "update" && result + ? await (await import("./service-update-batch")).completedServiceUpdatePrefix( + owner.job.jobId, + row.payload, + result, + lifecycle.originalOrdinal, + ) + : { operationKeys: new Set(), revisionId: candidate.expectedCurrentRevisionId }; + if (prefix.revisionId !== candidate.expectedCurrentRevisionId) + throw new JobObservationPending("invalid_identity"); + if ( + !result || + retained?.candidateId !== candidate.candidateId || + retained.operationKey !== candidate.operationKey || + retained.revisionId !== candidate.revisionId || + retained.sealedOperationKey !== record.operationKey || + retained.maintenanceId !== record.maintenanceId || + retained.prepareOperationKey !== record.prepareOperationKey || + retained.phase !== "prepared" || + retained.rowVersions !== undefined || + result._configurationMaintenance !== undefined || + result._configurationMaintenanceDecision !== undefined || + result._configurationMaintenanceRollback !== undefined || + actual.operations[record.prepareOperationKey!] || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) || + (helper && + (canonicalJson(helper.operation) !== canonicalJson(fixed!.operation) || + helper.receiptDelivery?.state !== "confirmed" || + helper.receiptDelivery.terminalReceipt.state !== "succeeded")) || + Object.values(actual.operations).some( + (entry) => + entry.operationKey !== fixed?.root.operationKey && + !prefix.operationKeys.has(entry.operationKey) && + entry.operationKey !== + (record.privateCoordinatorLifecycle?.action === "update" + ? record.privateCoordinatorLifecycle.pull.operationKey + : undefined), + ) + ) + throw new JobObservationPending("recovery_required"); + checkpoint = { + ...result, + _configurationCandidate: { ...retained, phase: "discarding_unissued" }, + }; + const saved = await tx + .update(schema.adminJob) + .set({ result: checkpoint }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + }); + if (!checkpoint) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = checkpoint; + candidateStages.set(candidate, "unissued"); + await discardUnadmittedTrustedConfigurationCandidate(candidate); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { + ...(checkpoint._configurationCandidate as Record), + phase: "discarded", + }, + }); + candidateStages.set(candidate, "discarded"); +} +/** Final API publication of this canceled, conclusively UNISSUED intent. + * The private nominal stage and locked original row/child delivery prove no + * maintenance root/save exists. A cancel marker alone supplies none of that. */ +async function completeUnissuedConfigurationCancellation( + candidate: ValidatedConfigurationCandidate, +): Promise> { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer), + record = state.record; + if ( + candidateStages.get(candidate) !== "discarded" || + state.persisted || + !record.maintenanceId || + !record.prepareOperationKey + ) + throw new JobObservationPending("recovery_required"); + let committed: Record | undefined; + await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [row] = await tx + .select({ + result: schema.adminJob.result, + payload: schema.adminJob.payload, + type: schema.adminJob.type, + }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + projection = readProjection(result), + fixed = record.coordinatorPrerequisite, + helper = fixed && projection.operations[fixed.root.operationKey]; + if ( + !row || + row.type !== owner.job.jobType || + canonicalJson(row.payload) !== canonicalJson(owner.job.payload) + ) + throw new JobObservationPending("invalid_identity"); + const lifecycle = record.privateCoordinatorLifecycle, + prefix = + lifecycle?.action === "update" && result + ? await (await import("./service-update-batch")).completedServiceUpdatePrefix( + owner.job.jobId, + row.payload, + result, + lifecycle.originalOrdinal, + ) + : { operationKeys: new Set(), revisionId: candidate.expectedCurrentRevisionId }; + if (prefix.revisionId !== candidate.expectedCurrentRevisionId) + throw new JobObservationPending("invalid_identity"); + if ( + !result || + typeof result._cancelRequestedAt !== "string" || + !Number.isFinite(Date.parse(result._cancelRequestedAt)) || + retained?.candidateId !== candidate.candidateId || + retained.operationKey !== candidate.operationKey || + retained.sealedOperationKey !== record.operationKey || + retained.revisionId !== candidate.revisionId || + retained.maintenanceId !== record.maintenanceId || + retained.prepareOperationKey !== record.prepareOperationKey || + retained.phase !== "discarded" || + retained.rowVersions !== undefined || + result._configurationMaintenance !== undefined || + result._configurationMaintenanceDecision !== undefined || + result._configurationMaintenanceRollback !== undefined || + projection.operations[record.prepareOperationKey!] || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) || + (helper && + (helper.receiptDelivery?.state !== "confirmed" || + helper.receiptDelivery.terminalReceipt.state !== "succeeded" || + canonicalJson(helper.operation) !== canonicalJson(fixed!.operation))) || + Object.values(projection.operations).some( + (entry) => + entry.operationKey !== fixed?.root.operationKey && + !prefix.operationKeys.has(entry.operationKey) && + entry.operationKey !== + (record.privateCoordinatorLifecycle?.action === "update" + ? record.privateCoordinatorLifecycle.pull.operationKey + : undefined), + ) + ) + throw new JobObservationPending("recovery_required"); + const configuration: TrustedConfigurationApplyOutcome = { + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + committed = { + ...result, + configuration, + _configurationOutcomes: { + ...(result._configurationOutcomes as Record), + [candidate.operationKey]: configuration, + }, + }; + delete committed.observation; + const saved = await tx + .update(schema.adminJob) + .set({ status: "canceled", finishedAt: new Date(), result: committed }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + }); + if (!committed) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = committed; + return { configuration: committed.configuration }; +} +/** Discard only this original inert candidate after the fixed host proves its + * NoGuard closure and delivers the original canonical failure. Inspection is + * performed here; a portable receipt/boolean supplied by a caller is useless. */ +export async function discardNeverPreparedProspectiveConfiguration( + candidate: ValidatedConfigurationCandidate, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer), + stage = candidateStages.get(candidate); + if ( + !state.record.maintenanceId || + !state.record.prepareOperationKey || + !["prepared", "recovered_pre_save", "never_prepared"].includes(stage ?? "") + ) + throw Error("Original inert maintenance candidate required"); + const { inspectConfigurationMaintenance } = await configurationMaintenanceProtocol, + closed = await inspectConfigurationMaintenance({ + maintenanceId: state.record.maintenanceId, + candidateId: candidate.candidateId, + }); + if ( + closed.state !== "settled" || + closed.dmFence !== null || + closed.decision !== null || + closed.generation !== "not_started" || + closed.appliedRevisionId !== null || + closed.followUp !== "not_started" + ) + throw Error("Conclusive original NoGuard closure required"); + let checkpoint: Record | undefined; + try { + await withConfigurationWriteTransaction(state.writer, async (tx) => { + // The same owned row lock waits out any earlier save/checkpoint transaction. + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"); + const result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + maintenance = result?._configurationMaintenance as Record | undefined, + inspection = maintenance?.inspection as Record | undefined, + projected = readProjection(result).operations[state.record.prepareOperationKey!]; + if ( + !result || + retained?.candidateId !== candidate.candidateId || + retained.operationKey !== candidate.operationKey || + retained.revisionId !== candidate.revisionId || + retained.sealedOperationKey !== state.record.operationKey || + retained.maintenanceId !== state.record.maintenanceId || + retained.prepareOperationKey !== state.record.prepareOperationKey || + !["prepared", "discarding_without_guard"].includes(String(retained.phase)) || + retained.rowVersions !== undefined || + maintenance?.decision !== undefined || + result._configurationDecision !== undefined || + inspection?.state !== "complete" || + canonicalJson(inspection.receipt) !== canonicalJson(closed) || + !projected?.terminal || + projected.receiptDelivery?.state !== "confirmed" || + projected.kind !== "configurationMaintenance.prepare" || + projected.operationId !== closed.prepare.executionId || + projected.fingerprint !== closed.prepare.fingerprint || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) + ) + throw Error("Original unsaved candidate closure checkpoint required"); + checkpoint = { + ...result, + _configurationCandidate: { ...retained, phase: "discarding_without_guard" }, + }; + const saved = await tx + .update(schema.adminJob) + .set({ result: checkpoint }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw Error("Original candidate cleanup ownership changed"); + }); + } catch (error) { + if (error instanceof ConfigurationWriteCommitUnknown) + throw new JobObservationPending("recovery_required"); + throw error; + } + if (!checkpoint) throw Error("Original candidate cleanup checkpoint unavailable"); + owner.job.checkpointResult = checkpoint; + candidateStages.set(candidate, "never_prepared"); + // Pending cleanup retains the global candidate fence. Only exact bytes under + // the real queue ownership can be removed; an I/O failure leaves it pending. + await discardUnadmittedTrustedConfigurationCandidate(candidate); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { + ...(checkpoint._configurationCandidate as Record), + phase: "discarded", + }, + }); + candidateStages.set(candidate, "discarded"); +} +/** A failed prepare is not itself NoGuard evidence. Only the actual fixed + * inspector/disjoint closure and original canonical failure above permit this + * inert cleanup and truthful failed configuration result. */ +async function completeNeverPreparedConfiguration( + candidate: ValidatedConfigurationCandidate, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer), + projected = + state.record.prepareOperationKey && + readProjection(owner.job.checkpointResult).operations[state.record.prepareOperationKey], + terminal = + projected && projected.receiptDelivery?.state === "confirmed" + ? projected.receiptDelivery.terminalReceipt + : undefined; + if ( + !projected || + projected.kind !== "configurationMaintenance.prepare" || + !projected.terminal || + !terminal || + !["failed", "timed_out"].includes(terminal.state) + ) + throw new JobObservationPending("recovery_required"); + try { + await discardNeverPreparedProspectiveConfiguration(candidate); + } catch { + throw new JobObservationPending("recovery_required"); + } + const retained = owner.job.checkpointResult?._configurationMaintenance as Record, + inspection = retained.inspection as { state?: string; receipt?: unknown }, + closed = configurationMaintenanceReceiptSchema.parse(inspection.receipt); + if ( + inspection.state !== "complete" || + closed.state !== "settled" || + closed.dmFence !== null || + closed.decision !== null || + closed.binding.candidateId !== candidate.candidateId || + closed.prepare.operationKey !== state.record.prepareOperationKey + ) + throw new JobObservationPending("recovery_required"); + const configuration: TrustedConfigurationApplyOutcome = { + outcome: "not_committed", + reason: terminal.errorClass === "busy" ? "busy" : "failed", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationMaintenance: { ...retained, phase: "settled", receipt: closed }, + configuration, + _configurationOutcomes: { + ...(owner.job.checkpointResult?._configurationOutcomes as Record), + [candidate.operationKey]: configuration, + }, + }); + return configuration; +} +/** A PREPARED reservation is discarded only through its actual unused final, + * both-role settled guard/root and original delivered API control sources. + * This is disjoint from NeverPrepared, and accepts no receipt/boolean input. */ +export async function discardUnusedPreparedProspectiveConfiguration( + candidate: ValidatedConfigurationCandidate, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer); + if ( + !state.record.maintenanceId || + !state.record.prepareOperationKey || + !["prepared", "recovered_pre_save", "rolled_back", "unused", "discarded"].includes( + candidateStages.get(candidate) ?? "", + ) + ) + throw Error("Original unsaved prepared candidate required"); + const maintenance = await configurationMaintenanceProtocol, + closed = await maintenance.inspectConfigurationMaintenance({ + maintenanceId: state.record.maintenanceId, + candidateId: candidate.candidateId, + }); + if ( + closed.state !== "settled" || + closed.dmFence?.state !== "settled" || + closed.decision?.decision !== "abandon" || + closed.generation !== "not_started" || + closed.appliedRevisionId !== null || + closed.followUp !== "not_started" || + closed.prepare.operationKey !== state.record.prepareOperationKey || + closed.binding.applyOperationKey !== state.record.operationKey + ) + throw Error("Actual original prepared-unused closure required"); + let committed: Record | undefined; + await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + remote = result?._configurationMaintenance as Record | undefined, + decision = configurationMaintenanceDecisionCheckpointSchema.parse( + result?._configurationMaintenanceDecision, + ), + settle = configurationMaintenanceSettleCheckpointSchema.parse( + result?._configurationMaintenanceSettle, + ), + projection = readProjection(result), + delivered = projection.operations[settle.delivery.root.operationKey]?.receiptDelivery, + canonical = + delivered?.terminalReceipt.state === "succeeded" + ? configurationMaintenanceReceiptSchema.parse(delivered.terminalReceipt.result) + : null; + if ( + !result || + retained?.candidateId !== candidate.candidateId || + retained.operationKey !== candidate.operationKey || + retained.revisionId !== candidate.revisionId || + retained.sealedOperationKey !== state.record.operationKey || + retained.maintenanceId !== state.record.maintenanceId || + retained.prepareOperationKey !== state.record.prepareOperationKey || + !["prepared", "rolled_back", "discarding_unused", "discarded"].includes( + String(retained.phase), + ) || + retained.rowVersions !== undefined || + remote?.phase !== "settled" || + decision.operation.decision !== "abandon" || + decision.operation.decisionId !== closed.decision?.decisionId || + decision.rowVersions.length || + decision.attempt !== "attempted" || + settle.attempt !== "attempted" || + settle.operation.candidateId !== candidate.candidateId || + settle.operation.maintenanceId !== state.record.maintenanceId || + settle.operation.decisionId !== decision.operation.decisionId || + delivered?.state !== "confirmed" || + !canonical || + canonicalJson(canonical) !== canonicalJson(remote.receipt) || + canonicalJson({ ...closed, preflight: canonical.preflight }) !== canonicalJson(canonical) || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) + ) + throw Error("Original delivered unused candidate checkpoint required"); + committed = { ...result, _configurationCandidate: { ...retained, phase: "discarding_unused" } }; + const saved = await tx + .update(schema.adminJob) + .set({ result: committed }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw Error("Original unused cleanup ownership changed"); + }); + if (!committed) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = committed; + candidateStages.set(candidate, "unused"); + await discardUnadmittedTrustedConfigurationCandidate(candidate); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { + ...(committed._configurationCandidate as Record), + phase: "discarded", + }, + }); + candidateStages.set(candidate, "discarded"); + freeze(closed); + discardedUnusedCandidates.set(candidate, closed); +} +/** Actual prepared-unused cancellation completion. Only the fixed discarded + * unused producer above mints this private source after full original + * final/SQL/mirror/ACK/release/API-copy verification and exact file cleanup. */ +async function completeUnusedConfigurationCancellation( + candidate: ValidatedConfigurationCandidate, +): Promise> { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer), + record = state.record, + closed = discardedUnusedCandidates.get(candidate); + if ( + !closed || + candidateStages.get(candidate) !== "discarded" || + state.persisted || + !record.maintenanceId || + !record.prepareOperationKey + ) + throw new JobObservationPending("recovery_required"); + let committed: Record | undefined; + await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + remote = result?._configurationMaintenance as Record | undefined, + decision = configurationMaintenanceDecisionCheckpointSchema.parse( + result?._configurationMaintenanceDecision, + ), + settle = configurationMaintenanceSettleCheckpointSchema.parse( + result?._configurationMaintenanceSettle, + ), + projection = readProjection(result), + delivered = projection.operations[settle.delivery.root.operationKey]?.receiptDelivery, + canonical = + delivered?.terminalReceipt.state === "succeeded" + ? configurationMaintenanceReceiptSchema.parse(delivered.terminalReceipt.result) + : null; + if ( + !result || + typeof result._cancelRequestedAt !== "string" || + !Number.isFinite(Date.parse(result._cancelRequestedAt)) || + retained?.candidateId !== candidate.candidateId || + retained.operationKey !== candidate.operationKey || + retained.revisionId !== candidate.revisionId || + retained.sealedOperationKey !== record.operationKey || + retained.maintenanceId !== record.maintenanceId || + retained.prepareOperationKey !== record.prepareOperationKey || + retained.phase !== "discarded" || + retained.rowVersions !== undefined || + remote?.phase !== "settled" || + decision.operation.decision !== "abandon" || + decision.operation.decisionId !== closed.decision?.decisionId || + decision.rowVersions.length || + decision.attempt !== "attempted" || + settle.attempt !== "attempted" || + settle.operation.candidateId !== candidate.candidateId || + settle.operation.maintenanceId !== record.maintenanceId || + settle.operation.decisionId !== decision.operation.decisionId || + delivered?.state !== "confirmed" || + !canonical || + canonicalJson(canonical) !== canonicalJson(remote.receipt) || + canonicalJson({ ...closed, preflight: canonical.preflight }) !== canonicalJson(canonical) || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) + ) + throw new JobObservationPending("recovery_required"); + const configuration: TrustedConfigurationApplyOutcome = { + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + committed = { + ...result, + configuration, + _configurationOutcomes: { + ...(result._configurationOutcomes as Record), + [candidate.operationKey]: configuration, + }, + }; + delete committed.observation; + const saved = await tx + .update(schema.adminJob) + .set({ status: "canceled", finishedAt: new Date(), result: committed }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + }); + if (!committed) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = committed; + return { configuration: committed.configuration }; +} +/** Cleanup authority is the original committed candidate plus its durable, + * fully delivered completion, never a caller-selected terminal flag. */ +export async function assertSettledConfigurationCandidate( + candidate: ValidatedConfigurationCandidate, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer); + assertCommittedConfigurationCandidate(candidate); + let contained = false; + if (state.record.maintenanceId) { + const maintenance = await configurationMaintenanceProtocol, + actual = await maintenance.inspectConfigurationMaintenance({ + maintenanceId: state.record.maintenanceId, + candidateId: candidate.candidateId, + }); + contained = + actual.generation === "not_committed" && + actual.appliedRevisionId === null && + actual.followUp === "contained"; + if ( + actual.state !== "settled" || + actual.dmFence?.state !== "settled" || + actual.decision?.decision !== "continue" || + (!contained && + (actual.generation !== "committed" || + actual.appliedRevisionId !== candidate.revisionId || + actual.followUp !== "complete")) || + actual.prepare.operationKey !== state.record.prepareOperationKey + ) + throw Error("Actual original maintenance closure required before cleanup"); + } + const [job] = await owner.connection + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)); + const result = job?.result as Record | undefined; + const receipt = result?._configurationCandidate as Record | undefined; + if ( + !result || + receipt?.candidateId !== candidate.candidateId || + receipt.revisionId !== candidate.revisionId || + receipt.operationKey !== candidate.operationKey || + receipt.sealedOperationKey !== state.record.operationKey || + receipt.phase !== "settled" || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) + ) + throw Error("Configuration candidate settlement unavailable"); + const outcomes = result._configurationOutcomes as Record | undefined; + const outcome = parseTrustedConfigurationApplyOutcome(outcomes?.[candidate.operationKey]); + if ( + contained + ? outcome.outcome !== "not_committed" || + outcome.cleanupFailed || + outcome.publisherCleanupFailed || + outcome.registryUpdateFailed + : !completeTrustedConfigurationApply(outcome) || + outcome.outcome !== "committed" || + outcome.revisionId !== candidate.revisionId + ) + throw Error("Configuration candidate application unavailable"); + if (state.record.maintenanceId) { + const remote = result._configurationMaintenance as Record | undefined, + settled = configurationMaintenanceReceiptSchema.parse(remote?.receipt), + projection = readProjection(result), + delivered = Object.values(projection.operations).find( + (value) => + value.operation?.kind === "configurationMaintenance.settle" && + value.operation.maintenanceId === state.record.maintenanceId && + value.operation.candidateId === candidate.candidateId, + ); + if ( + remote?.phase !== "settled" || + settled.state !== "settled" || + (contained + ? settled.generation !== "not_committed" || + settled.appliedRevisionId !== null || + settled.followUp !== "contained" + : settled.generation !== "committed" || + settled.appliedRevisionId !== candidate.revisionId || + settled.followUp !== "complete") || + delivered?.receiptDelivery?.state !== "confirmed" || + delivered.receiptDelivery.terminalReceipt.state !== "succeeded" || + canonicalJson(delivered.receiptDelivery.terminalReceipt.result) !== canonicalJson(settled) + ) + throw Error("Original delivered maintenance completion required"); + } + if (state.record.apiFollowUp === "service-apply") { + const followUp = result.serviceCredential as Record | undefined; + if ( + followUp?.followUp !== (contained ? "contained" : "complete") || + followUp.revisionId !== candidate.revisionId || + !state.record.affectedServiceIds.includes(String(followUp.serviceId)) + ) + throw Error("Configuration follow-up unavailable"); + } else if (state.record.apiFollowUp === "integration-reload") { + const reload = result.integrationReload as Record | undefined; + if ( + reload?.state !== "complete" || + reload.candidateId !== candidate.candidateId || + reload.revisionId !== candidate.revisionId + ) + throw Error("Configuration reload unavailable"); + } else if (state.record.privateCoordinatorLifecycle) { + const followUp = result.coordinatorLifecycle as Record | undefined, + fixed = state.record.privateCoordinatorLifecycle; + if ( + state.record.apiFollowUp !== `coordinator-${fixed.action}` || + followUp?.action !== fixed.action || + followUp.revisionId !== candidate.revisionId || + followUp.followUp !== (contained ? "contained" : "complete") + ) + throw Error("Original coordinator lifecycle completion required"); + } else if (state.record.apiFollowUp !== "none") + throw Error("Configuration continuation unavailable"); +} +export async function retireSettledProspectiveConfiguration( + candidate: ValidatedConfigurationCandidate, +): Promise { + await assertSettledConfigurationCandidate(candidate); + const state = validatedConfigurationCandidateState(candidate), + { job } = administrativeConfigurationState(state.writer); + const queuedCoordinator = + !!state.record.privateCoordinatorLifecycle && + ((job.jobType === "service.apply" && job.payload.service === "data-manager") || + (job.jobType === "service.bulk" && job.payload.action === "update")); + if ( + queuedCoordinator || + (job.checkpointResult?._immediateJobId === job.jobId && + ["service.apply", "configuration.apply", "integration.config.import"].includes( + job.jobType ?? "", + )) + ) { + const original = job.checkpointResult; + if (!original) throw new JobObservationPending("recovery_required"); + const provenance = state.record.privateIntegrationReload?.legacyImport, + reload = original.integrationReload as { result?: Record } | undefined, + result = { + configuration: original.configuration, + ...(original.serviceCredential ? { serviceCredential: original.serviceCredential } : {}), + ...(original.coordinatorLifecycle + ? { coordinatorLifecycle: original.coordinatorLifecycle } + : {}), + ...(original.integrationReload ? { integrationReload: original.integrationReload } : {}), + ...(provenance + ? { + integrationImport: { + ok: true, + imported: provenance.changedIntegrationIds.length, + skipped: provenance.integrationIds.filter( + (id) => !provenance.changedIntegrationIds.includes(id), + ), + ...reload?.result, + }, + } + : {}), + }, + operationKey = state.record.maintenanceId + ? configurationMaintenanceSettleCheckpointSchema.parse( + original._configurationMaintenanceSettle, + ).delivery.root.operationKey + : state.record.apiFollowUp === "service-apply" && "serviceId" in state.sealed.operation + ? createDurableOpsKey( + "admin-job.credential.recreate", + `${job.jobId}:${state.sealed.operation.serviceId}`, + ) + : state.record.operationKey; + let completedCoordinator: Record = {}; + if (queuedCoordinator && state.record.privateCoordinatorLifecycle?.action === "update") { + const settled = configurationMaintenanceSettleCheckpointSchema.parse( + original._configurationMaintenanceSettle, + ); + const source = original._configurationCandidate as Record; + const archived = completedCoordinatorMaintenanceSchema.parse({ + version: 1, + originalOrdinal: state.record.privateCoordinatorLifecycle.originalOrdinal, + candidate: { + candidateId: source.candidateId, + maintenanceId: source.maintenanceId, + prepareOperationKey: source.prepareOperationKey, + revisionId: source.revisionId, + operationKey: source.operationKey, + phase: source.phase, + }, + receipt: (original._configurationMaintenance as { receipt: unknown }).receipt, + steps: original._configurationMaintenanceSteps, + settlementOperationKey: settled.delivery.root.operationKey, + }); + completedCoordinator = { _completedCoordinatorMaintenance: archived }; + completedCoordinatorMaintenance({ ...original, ...completedCoordinator }); + } + // The original producer has verified every required follow-up and actual + // canonical receipt. Retain that completion BEFORE unlink, so death at + // either side of file retirement needs only observation and exact cleanup. + await job.checkpoint({ + ...original, + ...completedCoordinator, + _completion: { operationKey, result }, + _configurationCandidate: { + ...(original._configurationCandidate as Record), + retirement: { version: 1, integrationIds: [...state.writer.integrationIds] }, + }, + }); + } + try { + await retireSettledTrustedConfigurationCandidate(candidate); + } catch { + const state = validatedConfigurationCandidateState(candidate), + { job } = administrativeConfigurationState(state.writer); + if ((job.checkpointResult?._configurationCandidate as Record)?.retirement) + throw new JobObservationPending("recovery_required"); + const outcome = { + ...(job.checkpointResult?.configuration as Record), + publisherCleanupFailed: true, + }; + await job.checkpoint({ + ...job.checkpointResult, + _configurationCandidate: { + ...(job.checkpointResult?._configurationCandidate as Record), + phase: (job.checkpointResult?._configurationCandidate as Record) + ?.retirement + ? "settled" + : "unknown", + }, + configuration: outcome, + _configurationOutcomes: { + ...(job.checkpointResult?._configurationOutcomes as Record), + [candidate.operationKey]: outcome, + }, + }); + throw new JobObservationPending("recovery_required"); + } + candidateStages.set(candidate, "retired"); +} + +declare const retirementBrand: unique symbol; +export interface RetainedConfigurationRetirement { + readonly [retirementBrand]: true; +} +const retirements = new WeakMap< + RetainedConfigurationRetirement, + { + writer: AdministrativeConfigurationWriter; + publisher: ReturnType; + checkpoint: Record; + apiJobId: string; + actorId: string; + } +>(); +/** Only the actual locked completion verifier below creates these handles. */ +export function retainedConfigurationRetirementState(handle: RetainedConfigurationRetirement) { + const state = retirements.get(handle); + if (!state || currentAdministrativeConfigurationWriter() !== state.writer) + throw new JobObservationPending("recovery_required"); + administrativeConfigurationState(state.writer); + return state; +} +const retirementSchema = z.strictObject({ + version: z.literal(1), + integrationIds: z.array(id).max(256), +}); +async function resumeCompletedConfigurationRetirement(): Promise> { + const writer = currentAdministrativeConfigurationWriter(), + { job } = administrativeConfigurationState(writer); + let original: Record | undefined; + await withConfigurationWriteTransaction(writer, async (tx) => { + const [row] = await tx + .select() + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"); + original = row?.result as Record | undefined; + const retained = original?._configurationCandidate as Record | undefined, + retirement = retirementSchema.parse(retained?.retirement); + const queued = + row?.type === job.jobType && + canonicalJson(row?.payload) === canonicalJson(job.payload) && + writer.actorId === job.jobId && + ((row.type === "service.apply" && + job.payload.service === "data-manager" && + Object.keys(job.payload).length === 1 && + writer.operationKey === coordinatorLifecycleOperationKey(job.jobId, null)) || + (row.type === "service.bulk" && + job.payload.action === "update" && + completedCoordinatorMaintenance(original!).receipt.apiJobId === job.jobId && + writer.operationKey === + coordinatorLifecycleOperationKey( + job.jobId, + completedCoordinatorMaintenance(original!).originalOrdinal, + ))); + if ( + !original || + (!queued && + (original._immediateJobId !== job.jobId || + original._immediateActorId !== writer.actorId)) || + retained?.operationKey !== writer.operationKey || + retained.phase !== "settled" || + canonicalJson(retirement.integrationIds) !== canonicalJson(writer.integrationIds) || + hasPendingReceiptDelivery(original) || + hasUnsettledManagedOperation(original) + ) + throw new JobObservationPending("recovery_required"); + }); + if (!original) throw new JobObservationPending("recovery_required"); + job.checkpointResult = original; + const retained = original._configurationCandidate as Record, + completion = original._completion as + | { operationKey?: string; result?: Record } + | undefined; + if ( + !completion?.operationKey || + !completion.result || + canonicalJson(completion.result.configuration) !== + canonicalJson( + (original._configurationOutcomes as Record)?.[writer.operationKey], + ) + ) + throw new JobObservationPending("recovery_required"); + // This is the existing canonical observer, not replay of a producer/child. + // A missing packet or the retirement marker alone never proves completion. + await resumeAdminJobObservation(job); + if (retained.maintenanceId) { + const actual = await (await configurationMaintenanceProtocol).inspectConfigurationMaintenance({ + maintenanceId: String(retained.maintenanceId), + candidateId: String(retained.candidateId), + }), + remote = original._configurationMaintenance as + | { receipt?: unknown; phase?: string } + | undefined; + if ( + actual.state !== "settled" || + actual.dmFence?.state !== "settled" || + actual.decision?.decision !== "continue" || + remote?.phase !== "settled" || + canonicalJson({ + ...actual, + preflight: configurationMaintenanceReceiptSchema.parse(remote.receipt).preflight, + }) !== canonicalJson(remote.receipt) + ) + throw new JobObservationPending("recovery_required"); + } + const handle = Object.freeze({}) as RetainedConfigurationRetirement; + retirements.set(handle, { + writer, + publisher: publisherOptions(await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? "")), + checkpoint: retained, + apiJobId: job.jobId, + actorId: writer.actorId, + }); + try { + await completeRetainedConfigurationRetirement(handle); + } finally { + retirements.delete(handle); + } + await job.checkpoint({ ...job.checkpointResult, ...completion.result }); + return completion.result; +} +/** Reload from this exact committed candidate, while its captured read scope is live. */ +export async function withProspectiveConfigurationRead( + candidate: ValidatedConfigurationCandidate, + run: (context: AdministrativeConfigurationReadContext) => Promise, +): Promise { + const state = validatedConfigurationCandidateState(candidate); + assertCommittedConfigurationCandidate(candidate); + if (!state.rows || !state.integrationReloadSnapshot) + throw Error("Captured candidate reload state unavailable"); + const context = openAdministrativeConfigurationRead({ + ...state.rows, + services: state.rows.services, + integrations: state.rows.integrations, + env: state.rows.env, + integrationReloadSnapshot: state.integrationReloadSnapshot, + }); + try { + return await withAdministrativeConfigurationReadScope(context, () => run(context)); + } finally { + closeAdministrativeConfigurationRead(context); + } +} +/** The finite reload continuation. Recovery consumes the authenticated resolved + * values, with exact registry metadata revalidation; it never reconstructs the + * candidate from current desired rows or captures unrelated process secrets. */ +export async function reloadProspectiveIntegrations(candidate: ValidatedConfigurationCandidate) { + const state = validatedConfigurationCandidateState(candidate); + assertCommittedConfigurationCandidate(candidate); + const { reloadIntegrations } = await import("../integration-host"); + if (!recoveredCommittedCandidates.has(candidate)) + return withProspectiveConfigurationRead(candidate, (read) => reloadIntegrations(read)); + const retained = state.record.privateIntegrationReload; + if (state.record.apiFollowUp !== "integration-reload" || !retained) + throw new JobObservationPending("recovery_required"); + return withAdministrativeConfigurationRead({}, async (read) => { + if ( + canonicalJson(retained.services) !== canonicalJson(read.services) || + canonicalJson(retained.integrations) !== + canonicalJson( + read.integrations.map((i) => ({ + id: i.id, + directory: i.directory, + manifest: i.manifest, + isBuiltIn: i.isBuiltIn, + })), + ) || + canonicalJson(Object.keys(retained.values).sort()) !== + canonicalJson(read.integrations.map((i) => i.id).sort()) + ) + throw new JobObservationPending("recovery_required"); + const captured = openAdministrativeConfigurationRead({ + ...read, + integrationReloadSnapshot: retained.values, + }); + try { + return await withAdministrativeConfigurationReadScope(captured, () => + reloadIntegrations(captured), + ); + } finally { + closeAdministrativeConfigurationRead(captured); + } + }); +} +/** Fixed current non-DM credential continuation, shared by fresh and restored + * original candidates. Attempted child addresses remain lookup-only. */ +export async function completeProspectiveServiceCredential( + candidate: ValidatedConfigurationCandidate, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + { job: ctx } = administrativeConfigurationState(state.writer), + operation = state.sealed.operation; + assertCommittedConfigurationCandidate(candidate); + const application = parseTrustedConfigurationApplyOutcome(ctx.checkpointResult?.configuration); + if ( + state.record.maintenanceId || + state.record.apiFollowUp !== "service-apply" || + operation.kind !== "vault.apply" || + operation.serviceId === "data-manager" || + !completeTrustedConfigurationApply(application) || + application.outcome !== "committed" || + application.revisionId !== candidate.revisionId + ) + throw new JobObservationPending("recovery_required"); + const previous = ctx.checkpointResult?.serviceCredential as + | { serviceId?: string; revisionId?: string; followUp?: string } + | undefined; + if ( + previous && + (previous.serviceId !== operation.serviceId || + previous.revisionId !== candidate.revisionId || + !["not_started", "unknown", "complete"].includes(previous.followUp ?? "")) + ) + throw new JobObservationPending("recovery_required"); + if (!("serviceId" in operation)) throw Error("Credential continuation target missing"); + const serviceId = operation.serviceId, + revisionId = application.revisionId; + const { applyGenerationHardlinks } = await import("./configuration-hardlinks"); + if (!ctx.checkpointResult?.serviceCredential) + await ctx.checkpoint({ + ...ctx.checkpointResult, + serviceCredential: { serviceId, revisionId, followUp: "not_started" }, + }); + try { + const hardlinks = await applyGenerationHardlinks({ + ctx, + revisionId, + operationIdentity: `${ctx.jobId}:credential`, + coordinatorMode: "ensure-running", + signal: ctx.signal, + }); + const completion = { + configuration: application, + _configurationCandidate: { + ...(ctx.checkpointResult?._configurationCandidate as Record), + phase: "settled", + }, + serviceCredential: { serviceId, revisionId, followUp: "complete", hardlinks }, + }; + await ctx.checkpoint({ + ...ctx.checkpointResult, + _completion: { + operationKey: createDurableOpsKey( + "admin-job.credential.recreate", + `${ctx.jobId}:${serviceId}`, + ), + result: completion, + }, + }); + await executeAdminJobOperation( + ctx, + { kind: "service.recreate", serviceId, expectedConfigurationRevisionId: revisionId }, + "admin-job.credential.recreate", + { durableIdentity: serviceId }, + ); + await ctx.checkpoint({ ...ctx.checkpointResult, ...completion }); + await retireSettledProspectiveConfiguration(candidate); + } catch (error) { + const { ApiOpsError, OpsObservationPending } = await import("./ops-client"); + const known = error instanceof ApiOpsError || error instanceof AdminJobFailure; + await ctx.checkpoint({ + ...ctx.checkpointResult, + _configurationCandidate: { + ...(ctx.checkpointResult?._configurationCandidate as Record), + phase: known ? "settled" : "unknown", + }, + serviceCredential: { serviceId, revisionId, followUp: known ? "failed" : "unknown" }, + }); + if (error instanceof JobObservationPending || error instanceof OpsObservationPending) + throw error; + if (known) + throw new AdminJobFailure("Credential follow-up did not complete", { + configuration: { ...application, followUp: "failed" }, + serviceCredential: { serviceId, revisionId, followUp: "failed" }, + }); + throw new JobObservationPending("recovery_required"); + } +} +function retainedIntegrationIds(record: PreparedConfigurationRecord, overlay: Overlay): string[] { + if (overlay.purpose === "migration") { + const provenance = record.privateIntegrationReload?.legacyImport; + if ( + !provenance || + record.apiFollowUp !== "integration-reload" || + canonicalJson(provenance.integrationIds) !== + canonicalJson([...new Set(provenance.integrationIds)].sort()) || + canonicalJson(provenance.changedIntegrationIds) !== + canonicalJson( + overlay.changes.map((change) => { + if ( + change.kind !== "integration-config" || + !provenance.integrationIds.includes(change.integrationId) + ) + throw new JobObservationPending("recovery_required"); + return change.integrationId; + }), + ) || + new Set(provenance.changedIntegrationIds).size !== provenance.changedIntegrationIds.length + ) + throw new JobObservationPending("recovery_required"); + return provenance.integrationIds; + } + if (record.privateIntegrationReload?.legacyImport) + throw new JobObservationPending("recovery_required"); + return [ + ...new Set( + overlay.changes.flatMap((change) => + "integrationId" in change ? [change.integrationId] : [], + ), + ), + ].sort(); +} +/** Lock-set observation only. Full private packet, job owner, saved receipts and + * closed producer checks are repeated under the actual writer/row lock. */ +export async function retainedConfigurationIntegrationIds( + job: import("./job-runner").JobContext, + actorId: string, + operationKey: string, +): Promise { + try { + const retained = job.checkpointResult?._configurationCandidate as + | Record + | undefined; + if (retained?.retirement !== undefined) + return retirementSchema.parse(retained.retirement).integrationIds; + if (typeof retained?.sealedOperationKey !== "string") throw Error(); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""), + { record } = await readPreparedTrustedConfigurationCandidate( + publisherOptions(token), + retained.sealedOperationKey, + ); + if ( + record.apiJobId !== job.jobId || + record.actorId !== actorId || + retained.operationKey !== operationKey || + record.candidateId !== retained.candidateId + ) + throw Error(); + return retainedIntegrationIds(record, overlaySchema.parse(record.privateOverlay)); + } catch { + throw new JobObservationPending("recovery_required"); + } +} +function freeze(value: unknown): void { + if (!value || typeof value !== "object" || Object.isFrozen(value)) return; + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); +} +async function sourceFingerprint( + context: AdministrativeConfigurationReadContext, + token: string, +): Promise { + const integrations = []; + for (const integration of context.integrations) + integrations.push({ + id: integration.id, + config: await resolveIntegrationConfigWithSourcesStrict( + integration.manifest, + integration.directory, + context, + ), + }); + const subkey = createHmac("sha256", token).update("openmapx-prospective-source-v1").digest(); + return createHmac("sha256", subkey) + .update( + canonicalJson({ + services: context.services.map((service) => service.manifest), + integrations, + env: context.env, + files: administrativeConfigurationFileObservations(context), + serviceRows: context.serviceRows, + integrationRows: context.integrationRows, + serviceSecretRows: context.serviceSecretRows, + integrationSecretRows: context.integrationSecretRows, + }), + ) + .digest("hex"); +} +function publisherOptions(token: string) { + const number = (name: string) => { + const value = process.env[name]; + if (!value || !/^(0|[1-9][0-9]{0,9})$/.test(value) || !Number.isSafeInteger(Number(value))) + throw Error("Configuration candidate unavailable"); + return Number(value); + }; + const directory = process.env.OPS_TRUSTED_CONFIG_DIR; + if (!directory) throw Error("Configuration candidate unavailable"); + return { + directory, + token, + ownerUid: number("OPS_TRUSTED_CONFIG_UID"), + ownerGid: number("OPS_TRUSTED_CONFIG_GID"), + }; +} +/** Rehydrate the original pre-save or atomically committed candidate under its + * newly claimed job. The row lock waits out the former transaction; its old + * ownership CAS cannot save again. Current desired reads never rebuild the + * candidate. Only the exact saved decision/row receipts permit continuation; + * pre-save recovery cannot persist, preview or publish a configuration. */ +async function restoreRetainedMaintenanceCandidate(): Promise { + const writer = currentAdministrativeConfigurationWriter(), + owner = administrativeConfigurationState(writer), + { job } = owner; + if (!job.execution || job.execution.version < 2 || owner.participant) + throw new JobObservationPending("recovery_required"); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""), + publisher = publisherOptions(token); + let checkpoint: Record | undefined; + const candidate = await withConfigurationWriteTransaction(writer, async (tx, read) => { + const [row] = await tx + .select() + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined; + if ( + !result || + !retained || + !["prepared", "saved", "rolled_back"].includes(String(retained.phase)) || + retained.operationKey !== writer.operationKey || + typeof retained.sealedOperationKey !== "string" || + (retained.phase !== "rolled_back" && result._configurationMaintenanceRollback !== undefined) + ) + throw new JobObservationPending("recovery_required"); + const saved = retained.phase === "saved", + rolledBack = retained.phase === "rolled_back", + unusedDecision = + !saved && result._configurationMaintenanceDecision !== undefined + ? configurationMaintenanceDecisionCheckpointSchema.parse( + result._configurationMaintenanceDecision, + ) + : undefined, + unusedSettlement = + !saved && result._configurationMaintenanceSettle !== undefined + ? configurationMaintenanceSettleCheckpointSchema.parse( + result._configurationMaintenanceSettle, + ) + : undefined; + if ( + !saved && + (retained.rowVersions !== undefined || + (unusedDecision && + (unusedDecision.operation.decision !== "abandon" || + unusedDecision.rowVersions.length !== 0 || + unusedDecision.operation.checkpointVersion >= job.execution!.version)) || + (unusedSettlement && !unusedDecision)) + ) + throw new JobObservationPending("recovery_required"); + const inspected = await readPreparedTrustedConfigurationCandidate( + publisher, + retained.sealedOperationKey, + ), + record = inspected.record, + coordinator = + record.privateCoordinatorLifecycle && row + ? assertRetainedCoordinatorLifecycleProvenance(row, record) + : undefined, + operation = opsOperationSchema.parse(record.operation), + overlay = overlaySchema.parse(record.privateOverlay), + projection = readProjection(result), + preparation = record.prepareOperationKey + ? projection.operations[record.prepareOperationKey] + : undefined, + preRoot = + !saved && + !preparation && + result._configurationMaintenance === undefined && + (record.privateMaintenancePreparation !== undefined || + record.privateCoordinatorSource !== undefined); + const recoveryControls = [ + ...(unusedDecision?.delivery + ? [ + { + operation: unusedDecision.operation, + delivery: unusedDecision.delivery, + attempt: unusedDecision.attempt, + }, + ] + : []), + ...(unusedSettlement ? [unusedSettlement] : []), + ]; + const exactRecoveryControl = (entry: ReturnType["operations"][string]) => + recoveryControls.some((source) => { + const { operation, delivery } = source, + attempt = entry.attempt; + return ( + source.attempt === "attempted" && + entry.operationKey === delivery.root.operationKey && + canonicalJson(entry.operation) === canonicalJson(operation) && + attempt?.apiJobId === job.jobId && + attempt.apiOwnerVersion < job.execution!.version && + attempt.childOperationKey === delivery.root.operationKey && + attempt.childOperationFingerprint === opsOperationFingerprint(operation) && + attempt.requestedDeadlineAt === delivery.root.requestedDeadlineAt && + attempt.parentOperationKey === record.parentOperationKey && + attempt.parentFingerprint === record.parentFingerprint && + attempt.parentDeadlineAt === record.parentDeadlineAt && + delivery.stepAddress === + createDurableOpsKey( + operation.kind === "configurationMaintenance.decide" + ? "configuration-maintenance.decide" + : "configuration-maintenance.settle", + `${job.jobId}:${record.candidateId}`, + ) && + canonicalJson(delivery.root.parent) === + canonicalJson({ + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }) + ); + }); + for (const source of recoveryControls) { + const entry = projection.operations[source.delivery.root.operationKey]; + if ( + source.delivery.stepAddress !== + createDurableOpsKey( + source.operation.kind === "configurationMaintenance.decide" + ? "configuration-maintenance.decide" + : "configuration-maintenance.settle", + `${job.jobId}:${record.candidateId}`, + ) || + canonicalJson(source.delivery.root.parent) !== + canonicalJson({ + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }) || + (source.attempt === "attempted" + ? !entry || !exactRecoveryControl(entry) + : entry !== undefined) + ) + throw new JobObservationPending("recovery_required"); + } + // This finite current adapter has no integration reload/participant proof + // to reconstruct. Other recovery cases stay fenced until installed. + if ( + (operation.kind !== "vault.apply" && operation.kind !== "stack.render") || + (operation.kind === "vault.apply" + ? coordinator !== undefined || + operation.serviceId !== "data-manager" || + record.apiFollowUp !== "service-apply" || + overlay.purpose !== "service-secret" || + overlay.changes.some( + (change) => change.kind !== "service-secret" || change.serviceId !== "data-manager", + ) + : !coordinator || + overlay.purpose !== "render" || + overlay.changes.length !== 0 || + overlay.coordinatorMode !== "ensure-running") || + !record.maintenanceId || + !record.prepareOperationKey || + record.apiJobId !== job.jobId || + record.actorId !== writer.actorId || + record.candidateId !== retained.candidateId || + inspected.snapshot.revisionId !== retained.revisionId || + record.sourceFingerprint !== retained.sourceFingerprint || + record.maintenanceId !== retained.maintenanceId || + record.prepareOperationKey !== retained.prepareOperationKey || + operation.candidateId !== record.candidateId || + operation.maintenanceId !== record.maintenanceId || + operation.expectedSourceRevision !== record.expectedSourceRevision || + operation.coordinatorMode !== overlay.coordinatorMode || + inspected.snapshot.operationFingerprint !== opsOperationFingerprint(operation) || + (!preRoot && + (!preparation?.attempt || + preparation.kind !== "configurationMaintenance.prepare" || + preparation.attempt.apiJobId !== job.jobId || + preparation.attempt.apiOwnerVersion >= job.execution!.version)) || + canonicalJson(result._parentExecution) !== + canonicalJson({ + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }) || + (!saved && + Object.values(projection.operations).some( + (value) => + !exactRecoveryControl(value) && + (value.operation?.kind === "configurationMaintenance.decide" || + value.operation?.kind === "configurationMaintenance.settle" || + (value.operation && + "maintenanceId" in value.operation && + value.operation.maintenanceId === record.maintenanceId && + value.kind !== "configurationMaintenance.prepare")), + )) + ) + throw new JobObservationPending("recovery_required"); + if (rolledBack || unusedDecision) { + // Recover only original unused controls. A rolled-back candidate still + // requires the actual same-connection ROLLBACK acknowledgement. A retained + // ABANDON binds the original prepared receipt and exact control attempts; + // neither branch restores desired rows or a forward capability. + const remote = result._configurationMaintenance as Record | undefined, + delivery = preparation?.receiptDelivery, + prepared = configurationMaintenanceReceiptSchema.parse(delivery?.terminalReceipt.result), + observed = configurationMaintenanceReceiptSchema.parse(remote?.receipt); + if (rolledBack) { + const rollback = configurationMaintenanceRollbackCheckpointSchema.parse( + result._configurationMaintenanceRollback, + ); + if ( + rollback.candidateId !== record.candidateId || + rollback.maintenanceId !== record.maintenanceId || + rollback.prepareOperationKey !== record.prepareOperationKey || + rollback.prepareFingerprint !== prepared.prepare.operationFingerprint || + rollback.planDigest !== prepared.planDigest || + rollback.checkpointVersion >= job.execution!.version + ) + throw new JobObservationPending("recovery_required"); + } + if (unusedDecision) { + const { checkpointDigest, ...body } = unusedDecision.operation; + if ( + body.candidateId !== record.candidateId || + body.maintenanceId !== record.maintenanceId || + body.prepareOperationKey !== record.prepareOperationKey || + body.prepareFingerprint !== prepared.prepare.operationFingerprint || + body.planDigest !== prepared.planDigest || + body.expectedSequence !== prepared.sequence || + checkpointDigest !== configurationMaintenanceDbDecisionDigest(body, []) || + (observed.decision !== null && + canonicalJson(observed.decision) !== + canonicalJson({ + decisionId: body.decisionId, + decision: body.decision, + checkpointVersion: body.checkpointVersion, + checkpointDigest, + })) || + (unusedSettlement && + (unusedSettlement.operation.candidateId !== body.candidateId || + unusedSettlement.operation.maintenanceId !== body.maintenanceId || + unusedSettlement.operation.prepareOperationKey !== body.prepareOperationKey || + unusedSettlement.operation.prepareFingerprint !== body.prepareFingerprint || + unusedSettlement.operation.planDigest !== body.planDigest || + unusedSettlement.operation.decisionId !== body.decisionId)) + ) + throw new JobObservationPending("recovery_required"); + } + if ( + !remote || + !( + unusedDecision ? ["prepared", "authorized", "settling", "settled"] : ["prepared"] + ).includes(String(remote.phase)) || + remote.prepareOperationKey !== record.prepareOperationKey || + remote.prepareOperationFingerprint !== prepared.prepare.operationFingerprint || + prepared.state !== "prepared" || + prepared.decision !== null || + prepared.dmFence?.state !== "prepared" || + prepared.generation !== "not_started" || + prepared.followUp !== "not_started" || + prepared.appliedRevisionId !== null || + prepared.apiJobId !== job.jobId || + prepared.binding.candidateId !== record.candidateId || + prepared.binding.maintenanceId !== record.maintenanceId || + prepared.binding.applyOperationKey !== record.operationKey || + prepared.binding.applyOperationFingerprint !== opsOperationFingerprint(operation) || + prepared.binding.revisionId !== inspected.snapshot.revisionId || + prepared.prepare.operationKey !== record.prepareOperationKey || + delivery?.state !== "confirmed" || + delivery.terminalReceipt.state !== "succeeded" || + canonicalJson(observed.binding) !== canonicalJson(prepared.binding) || + canonicalJson(observed.prepare) !== canonicalJson(prepared.prepare) || + observed.planDigest !== prepared.planDigest || + observed.apiJobId !== prepared.apiJobId || + observed.parentOperationKey !== prepared.parentOperationKey || + observed.parentFingerprint !== prepared.parentFingerprint || + observed.parentDeadlineAt !== prepared.parentDeadlineAt || + canonicalJson(observed.steps) !== canonicalJson(prepared.steps) || + canonicalJson(observed.serviceIds) !== canonicalJson(prepared.serviceIds) || + !observed.dmFence || + canonicalJson({ + ...observed.dmFence, + sequence: prepared.dmFence?.sequence, + state: prepared.dmFence?.state, + }) !== canonicalJson(prepared.dmFence) || + observed.generation !== "not_started" || + observed.followUp !== "not_started" || + observed.appliedRevisionId !== null || + (!unusedDecision && observed.decision !== null) || + result._dataExecutions !== undefined || + Object.values(projection.operations).some( + (entry) => + !exactRecoveryControl(entry) && + ((entry.operationKey !== record.prepareOperationKey && + (entry.operationKey !== record.coordinatorPrerequisite?.root.operationKey || + canonicalJson(entry.operation) !== + canonicalJson(record.coordinatorPrerequisite?.operation))) || + !entry.terminal || + entry.receiptDelivery?.state !== "confirmed"), + ) + ) + throw new JobObservationPending("recovery_required"); + } + if (saved) { + const decision = configurationMaintenanceDecisionCheckpointSchema.parse( + result._configurationMaintenanceDecision, + ), + { checkpointDigest, ...body } = decision.operation, + remote = result._configurationMaintenance as Record | undefined, + receipt = configurationMaintenanceReceiptSchema.parse(remote?.receipt); + if ( + body.decision !== "continue" || + body.candidateId !== record.candidateId || + body.maintenanceId !== record.maintenanceId || + body.prepareOperationKey !== record.prepareOperationKey || + body.prepareFingerprint !== receipt.prepare.operationFingerprint || + body.planDigest !== receipt.planDigest || + body.checkpointVersion >= job.execution!.version || + canonicalJson(retained.rowVersions) !== canonicalJson(decision.rowVersions) || + checkpointDigest !== configurationMaintenanceDbDecisionDigest(body, decision.rowVersions) || + receipt.apiJobId !== job.jobId || + receipt.prepare.operationKey !== record.prepareOperationKey || + receipt.binding.candidateId !== record.candidateId || + receipt.binding.applyOperationKey !== record.operationKey || + receipt.binding.applyOperationFingerprint !== opsOperationFingerprint(operation) || + receipt.binding.revisionId !== inspected.snapshot.revisionId || + preparation?.receiptDelivery?.state !== "confirmed" + ) + throw new JobObservationPending("recovery_required"); + await assertConfigurationSavedRows(tx, decision.rowVersions); + if ( + coordinator && + (decision.rowVersions.some((row) => row.rowVersion !== row.previousRowVersion) || + (await sourceFingerprint(read, token)) !== record.sourceFingerprint) + ) + throw new JobObservationPending("recovery_required"); + } + // inspectPrepared... already authenticated these EXACT historical bytes. + // Parsing their payload is not opening an expired apply claim or resealing. + const payload = trustedConfigurationPayloadSchema.parse( + JSON.parse(inspected.sealed.toString("utf8")).payload, + ), + candidate = Object.freeze({ + candidateId: record.candidateId, + operationKey: writer.operationKey, + expectedCurrentRevisionId: operation.expectedCurrentRevisionId, + revisionId: inspected.snapshot.revisionId, + sourceFingerprint: record.sourceFingerprint, + sealedBytes: inspected.sealed.length, + affectedServiceIds: Object.freeze([...record.affectedServiceIds]), + }) as ValidatedConfigurationCandidate; + freeze(record); + freeze(overlay); + freeze(payload); + freeze(operation); + const state: CandidateState = { + writer, + record, + overlay, + payload, + publisher, + source: null, + rows: null, + sourceRows: null, + integrationReloadSnapshot: null, + persisted: saved, + sealed: { + revisionId: candidate.revisionId, + operation, + operationFingerprint: inspected.snapshot.operationFingerprint, + bytes: inspected.sealed, + }, + }; + if (preRoot && coordinator) { + const prefix = + coordinator.action === "update" + ? await (await import("./service-update-batch")).completedServiceUpdatePrefix( + job.jobId, + row!.payload, + result, + coordinator.originalOrdinal, + ) + : { operationKeys: new Set(), revisionId: operation.expectedCurrentRevisionId }; + if (prefix.revisionId !== operation.expectedCurrentRevisionId) + throw new JobObservationPending("invalid_identity"); + const original = record.privateCoordinatorSource, + fixed = record.coordinatorPrerequisite, + preview = configurationHardlinksPreviewSchema.parse(retained.preview), + pull = + coordinator.action === "update" + ? projection.operations[coordinator.pull.operationKey] + : undefined, + helper = fixed ? projection.operations[fixed.root.operationKey] : undefined; + const originalAttempt = ( + entry: NonNullable, + key: string, + fingerprint: string, + deadline: number, + ) => + entry.attempt?.apiJobId === job.jobId && + entry.attempt.apiOwnerVersion < job.execution!.version && + entry.attempt.childOperationKey === key && + entry.attempt.childOperationFingerprint === fingerprint && + entry.attempt.requestedDeadlineAt === deadline && + entry.attempt.parentOperationKey === record.parentOperationKey && + entry.attempt.parentFingerprint === record.parentFingerprint && + entry.attempt.parentDeadlineAt === record.parentDeadlineAt; + if ( + !original || + !fixed || + overlay.purpose !== "render" || + overlay.changes.length !== 0 || + overlay.coordinatorMode !== "ensure-running" || + !operation.expectedCurrentRevisionId || + preview.previewDigest !== retained.previewDigest || + original.sourceRevision !== record.expectedSourceRevision || + original.revisionId !== operation.expectedCurrentRevisionId || + fixed.operation.expectedConfigurationRevisionId !== operation.expectedCurrentRevisionId || + canonicalJson(fixed.root.parent) !== + canonicalJson({ + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }) || + (helper && + (helper.kind !== "service.start" || + canonicalJson(helper.operation) !== canonicalJson(fixed.operation) || + !originalAttempt( + helper, + fixed.root.operationKey, + opsOperationFingerprint(fixed.operation), + fixed.root.requestedDeadlineAt, + ))) || + (coordinator.action === "update" && + (!pull || + projection.steps[coordinator.pull.stepAddress]?.operationKey !== + coordinator.pull.operationKey || + canonicalJson(pull.operation) !== canonicalJson(coordinator.pull.operation) || + canonicalJson(pull.attempt) !== canonicalJson(coordinator.pull.attempt) || + !originalAttempt( + pull, + coordinator.pull.operationKey, + coordinator.pull.operationFingerprint, + coordinator.pull.requestedDeadlineAt, + ))) || + Object.values(projection.operations).some( + (entry) => + entry !== helper && entry !== pull && !prefix.operationKeys.has(entry.operationKey), + ) || + result._dataExecutions !== undefined + ) + throw new JobObservationPending("recovery_required"); + if ( + result._cancelRequestedAt !== undefined || + Date.now() >= record.parentDeadlineAt || + Date.now() >= parseControlRootOperationKey(record.prepareOperationKey!).notAfter || + JSON.parse(inspected.sealed.toString()).expiresAt <= new Date().toISOString() || + (await sourceFingerprint(read, token)) !== record.sourceFingerprint + ) { + candidates.set(candidate, state); + candidateStages.set(candidate, "recovered_unissued"); + checkpoint = result; + return candidate; + } + // The exact unchanged full source, private MAC candidate and old attempts + // survive the new owned-row claim. No merge/encryption/seal or clock is + // reconstructed; an attempted pull is observation-only below. + state.source = original; + state.preview = preview; + state.sourceRows = { + serviceRows: read.serviceRows, + integrationRows: read.integrationRows, + serviceSecretRows: read.serviceSecretRows, + integrationSecretRows: read.integrationSecretRows, + }; + state.rows = { + services: read.services, + integrations: read.integrations, + env: read.env, + serviceRows: read.serviceRows, + integrationRows: read.integrationRows, + serviceSecretRows: read.serviceSecretRows, + integrationSecretRows: read.integrationSecretRows, + }; + recoveredPreRootCandidates.add(candidate); + } else if (preRoot) { + const original = record.privateMaintenancePreparation!, + fixed = record.coordinatorPrerequisite, + helper = fixed && projection.operations[fixed.root.operationKey], + unattemptedHelper = + helper === undefined && + Object.keys(projection.operations).length === 0 && + result._dataExecutions === undefined, + preview = configurationHardlinksPreviewSchema.parse(retained.preview), + changes = overlay.changes.filter( + (change): change is Extract => + change.kind === "service-secret", + ); + if ( + !fixed || + overlay.coordinatorMode !== "ensure-running" || + !operation.expectedCurrentRevisionId || + !preview.entries || + preview.previewDigest !== retained.previewDigest || + original.source.sourceRevision !== record.expectedSourceRevision || + original.source.revisionId !== operation.expectedCurrentRevisionId || + fixed.operation.expectedConfigurationRevisionId !== operation.expectedCurrentRevisionId || + canonicalJson(fixed.root.parent) !== + canonicalJson({ + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }) || + (!unattemptedHelper && + (!helper?.attempt || + helper.kind !== "service.start" || + canonicalJson(helper.operation) !== canonicalJson(fixed.operation) || + helper.attempt.apiJobId !== job.jobId || + helper.attempt.apiOwnerVersion >= job.execution!.version || + helper.attempt.childOperationKey !== fixed.root.operationKey || + helper.attempt.childOperationFingerprint !== opsOperationFingerprint(fixed.operation) || + helper.attempt.requestedDeadlineAt !== fixed.root.requestedDeadlineAt)) || + canonicalJson(original.serviceSecrets.map((row) => row.key).sort()) !== + canonicalJson( + changes + .filter((change) => change.action === "set") + .map((change) => change.key) + .sort(), + ) + ) + throw new JobObservationPending("recovery_required"); + if ( + result._cancelRequestedAt !== undefined || + Date.now() >= record.parentDeadlineAt || + Date.now() >= parseControlRootOperationKey(record.prepareOperationKey!).notAfter || + JSON.parse(inspected.sealed.toString()).expiresAt <= new Date().toISOString() || + (await sourceFingerprint(read, token)) !== record.sourceFingerprint + ) { + // The original private bytes/locked prepared checkpoint prove which + // helper to OBSERVE, or its explicit nonissuance before this new claim. + // The cleanup rechecks that original locked no-attempt state. This + // recovery-only stage has no prepared rows/source and cannot persist, + // render, reseal or prepare a maintenance root. + candidates.set(candidate, state); + candidateStages.set(candidate, "recovered_unissued"); + checkpoint = result; + return candidate; + } + // These are the exact PREPARED row IDs/ciphertexts/timestamps. Current + // untouched rows are accepted only after their complete original source + // HMAC matches; nothing is merged, encrypted, rendered or sealed again. + // An unattempted helper may first use ONLY its already-fixed body/root/ + // deadline. The new owned row lock fenced the former writer, and every + // POST requires its durable child attempt; no missing lookup is success. + const changedKeys = new Set(changes.map((change) => change.key)), + preparedRows = original.serviceSecrets.map((row) => ({ + ...row, + createdAt: new Date(row.createdAt), + updatedAt: new Date(row.updatedAt), + })); + state.source = original.source; + state.preview = preview; + state.sourceRows = { + serviceRows: read.serviceRows, + integrationRows: read.integrationRows, + serviceSecretRows: read.serviceSecretRows, + integrationSecretRows: read.integrationSecretRows, + }; + state.rows = { + services: read.services, + integrations: read.integrations, + env: read.env, + serviceRows: read.serviceRows, + integrationRows: read.integrationRows, + integrationSecretRows: read.integrationSecretRows, + serviceSecretRows: [ + ...read.serviceSecretRows.filter( + (row) => row.serviceId !== "data-manager" || !changedKeys.has(row.key), + ), + ...preparedRows, + ], + }; + recoveredPreRootCandidates.add(candidate); + } + candidates.set(candidate, state); + candidateStages.set( + candidate, + saved + ? "committed" + : rolledBack + ? "rolled_back" + : preRoot + ? "prepared" + : "recovered_pre_save", + ); + if (saved) recoveredCommittedCandidates.add(candidate); + checkpoint = result; + return candidate; + }); + if (!checkpoint) throw new JobObservationPending("recovery_required"); + job.checkpointResult = checkpoint; + return candidate; +} +/** The existing ordinary generation producer, with no reload/credential or + * participant continuation. Its saved row receipts and exact original private + * packet are authoritative; current desired values never rebuild its seal. */ +async function restoreRetainedOrdinaryCandidate(): Promise { + const writer = currentAdministrativeConfigurationWriter(), + owner = administrativeConfigurationState(writer), + { job } = owner; + if (!job.execution || job.execution.version < 2 || owner.participant) + throw new JobObservationPending("recovery_required"); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""), + publisher = publisherOptions(token); + let checkpoint: Record | undefined; + const candidate = await withConfigurationWriteTransaction(writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined; + if ( + !result || + !["saved", "follow_up_pending"].includes(String(retained?.phase)) || + retained?.operationKey !== writer.operationKey || + retained.maintenanceId !== null || + retained.prepareOperationKey !== null || + typeof retained.sealedOperationKey !== "string" || + result._configurationMaintenance !== undefined || + result._configurationMaintenanceDecision !== undefined + ) + throw new JobObservationPending("recovery_required"); + const inspected = await readPreparedTrustedConfigurationCandidate( + publisher, + retained.sealedOperationKey, + ), + record = inspected.record, + operation = opsOperationSchema.parse(record.operation), + overlay = overlaySchema.parse(record.privateOverlay), + parent = { + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }, + projection = readProjection(result), + attempt = projection.operations[record.operationKey], + allowed = + (operation.kind === "stack.render" && + overlay.purpose === "render" && + overlay.changes.length === 0) || + (operation.kind === "serviceSelection.apply" && + overlay.purpose === "selection" && + overlay.changes.length === 1 && + overlay.changes[0]?.kind === "selection") || + (operation.kind === "serviceConfig.apply" && + overlay.purpose === "service-config" && + overlay.changes.length === 1 && + overlay.changes[0]?.kind === "service-config" && + overlay.changes[0].serviceId === operation.serviceId) || + (operation.kind === "integrationConfig.apply" && + ["integration-config", "integration-secret"].includes(overlay.purpose) && + overlay.changes.length === 1 && + "integrationId" in overlay.changes[0]! && + overlay.changes[0].integrationId === operation.integrationId && + overlay.changes[0].kind === overlay.purpose) || + (operation.kind === "stack.render" && + overlay.purpose === "migration" && + !!record.privateIntegrationReload?.legacyImport); + const reload = record.apiFollowUp === "integration-reload", + credential = record.apiFollowUp === "service-apply", + ordinaryCredential = + credential && + operation.kind === "vault.apply" && + operation.serviceId !== "data-manager" && + overlay.purpose === "service-secret" && + overlay.changes.length === 1 && + overlay.changes[0]?.kind === "service-secret" && + overlay.changes[0].serviceId === operation.serviceId && + job.jobType === "service.apply"; + if ( + !(allowed || ordinaryCredential) || + ![ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ].includes(operation.kind) || + !("expectedCurrentRevisionId" in operation) || + !("expectedSourceRevision" in operation) || + !("coordinatorMode" in operation) || + ("maintenanceId" in operation && operation.maintenanceId !== undefined) || + ("candidateId" in operation && operation.candidateId !== undefined) || + record.maintenanceId !== null || + record.prepareOperationKey !== null || + !( + record.apiFollowUp === "none" || + (reload && record.privateIntegrationReload) || + ordinaryCredential + ) || + record.apiJobId !== job.jobId || + record.actorId !== writer.actorId || + record.candidateId !== retained.candidateId || + record.sourceFingerprint !== retained.sourceFingerprint || + inspected.snapshot.revisionId !== retained.revisionId || + operation.expectedSourceRevision !== record.expectedSourceRevision || + operation.coordinatorMode !== overlay.coordinatorMode || + inspected.snapshot.operationFingerprint !== opsOperationFingerprint(operation) || + canonicalJson(writer.integrationIds) !== + canonicalJson(retainedIntegrationIds(record, overlay)) || + reload !== + (operation.kind === "integrationConfig.apply" || overlay.purpose === "migration") || + (overlay.purpose === "migration") !== (job.jobType === "integration.config.import") || + (result._parentExecution !== undefined && + canonicalJson(result._parentExecution) !== canonicalJson(parent)) || + (attempt && + (!attempt.attempt || + attempt.attempt.apiJobId !== job.jobId || + attempt.attempt.apiOwnerVersion >= job.execution!.version || + canonicalJson(attempt.operation) !== canonicalJson(operation))) + ) + throw new JobObservationPending("recovery_required"); + const rows = z.array(configurationSavedRowSchema).max(512).parse(retained.rowVersions); + await assertConfigurationSavedRows(tx, rows); + const payload = trustedConfigurationPayloadSchema.parse( + JSON.parse(inspected.sealed.toString("utf8")).payload, + ), + candidate = Object.freeze({ + candidateId: record.candidateId, + operationKey: writer.operationKey, + expectedCurrentRevisionId: operation.expectedCurrentRevisionId, + revisionId: inspected.snapshot.revisionId, + sourceFingerprint: record.sourceFingerprint, + sealedBytes: inspected.sealed.length, + affectedServiceIds: Object.freeze([...record.affectedServiceIds]), + }) as ValidatedConfigurationCandidate; + freeze(record); + freeze(operation); + freeze(overlay); + freeze(payload); + candidates.set(candidate, { + writer, + record, + overlay, + payload, + publisher, + source: null, + rows: null, + sourceRows: null, + integrationReloadSnapshot: null, + persisted: true, + sealed: { + revisionId: candidate.revisionId, + operation, + operationFingerprint: inspected.snapshot.operationFingerprint, + bytes: inspected.sealed, + }, + }); + candidateStages.set(candidate, "committed"); + recoveredCommittedCandidates.add(candidate); + checkpoint = result; + return candidate; + }); + if (!checkpoint) throw new JobObservationPending("recovery_required"); + job.checkpointResult = checkpoint; + return candidate; +} +/** Death before the candidate checkpoint cannot authorize reconstruction or + * another pull. Close only the original never-issued slot; an authenticated + * orphan packet, if present, is usable solely for exact inert-file cleanup. */ +export async function reconcileUnstagedCoordinatorPull(): Promise { + const writer = currentAdministrativeConfigurationWriter(), + { job } = administrativeConfigurationState(writer); + if ( + job.jobType !== "service.bulk" || + !job.execution || + job.execution.version < 2 || + job.checkpointResult?._configurationCandidate !== undefined + ) + return; + let original: Record | undefined, + entry: ReturnType["operations"][string] | undefined; + await withConfigurationWriteTransaction(writer, async (tx) => { + const [row] = await tx + .select() + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"); + if ( + !row || + row.type !== job.jobType || + canonicalJson(row.payload) !== canonicalJson(job.payload) + ) + throw new JobObservationPending("invalid_identity"); + original = (row.result ?? {}) as Record; + if (original._configurationCandidate !== undefined) + throw new JobObservationPending("recovery_required"); + const { parseServiceUpdateBatch, completedServiceUpdatePrefix } = await import( + "./service-update-batch" + ), + batch = parseServiceUpdateBatch(row.payload, original), + ordinal = batch.serviceIds.indexOf("data-manager"), + projection = readProjection(original), + address = createDurableOpsKey("configuration.coordinator-pull", `${job.jobId}:${ordinal}`); + entry = projection.operations[projection.steps[address]?.operationKey ?? address]; + if (!entry) return; + const prefix = await completedServiceUpdatePrefix(job.jobId, row.payload, original, ordinal); + if ( + !["prepared", "unissued"].includes(entry.phase) || + entry.operation?.kind !== "service.pull" || + entry.operation.serviceId !== "data-manager" || + !entry.operation.candidateId || + entry.operation.expectedConfigurationRevisionId !== prefix.revisionId || + entry.attempt?.apiJobId !== job.jobId || + entry.attempt.apiOwnerVersion >= job.execution!.version || + Object.values(projection.operations).some( + (value) => value !== entry && !prefix.operationKeys.has(value.operationKey), + ) || + original._configurationMaintenance !== undefined || + original._dataExecutions !== undefined + ) + throw new JobObservationPending("recovery_required"); + }); + if (!entry || !original) return; + job.checkpointResult = original; + const operation = entry.operation; + if (operation?.kind !== "service.pull" || !operation.candidateId) + throw new JobObservationPending("invalid_identity"); + const publisher = publisherOptions( + await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""), + ), + retained = await findPreparedTrustedConfigurationCandidate(publisher, operation.candidateId); + if (retained) { + const record = retained.record, + [row] = await administrativeConfigurationState(writer) + .connection.select() + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)); + if (!row) throw new JobObservationPending("recovery_required"); + const fixed = assertRetainedCoordinatorLifecycleProvenance(row, record); + if ( + fixed.action !== "update" || + record.candidateId !== operation.candidateId || + canonicalJson(fixed.pull.operation) !== canonicalJson(entry.operation) || + canonicalJson(fixed.pull.attempt) !== canonicalJson(entry.attempt) || + fixed.pull.operationKey !== entry.operationKey + ) + throw new JobObservationPending("invalid_identity"); + } + await closeUnissuedCoordinatorPull(job, operation.candidateId, retained?.record.operationKey); + if (retained) { + const record = retained.record, + typed = opsOperationSchema.parse(record.operation); + if (!("expectedCurrentRevisionId" in typed)) + throw new JobObservationPending("invalid_identity"); + const candidate = Object.freeze({ + candidateId: record.candidateId, + operationKey: writer.operationKey, + expectedCurrentRevisionId: typed.expectedCurrentRevisionId, + revisionId: retained.snapshot.revisionId, + sourceFingerprint: record.sourceFingerprint, + sealedBytes: retained.sealed.length, + affectedServiceIds: Object.freeze([...record.affectedServiceIds]), + }) as ValidatedConfigurationCandidate; + candidates.set(candidate, { + writer, + record, + overlay: overlaySchema.parse(record.privateOverlay), + payload: trustedConfigurationPayloadSchema.parse( + JSON.parse(retained.sealed.toString()).payload, + ), + publisher, + source: null, + rows: null, + sourceRows: null, + integrationReloadSnapshot: null, + persisted: false, + sealed: { + revisionId: candidate.revisionId, + operation: typed, + operationFingerprint: retained.snapshot.operationFingerprint, + bytes: retained.sealed, + }, + }); + candidateStages.set(candidate, "unissued"); + await discardUnadmittedTrustedConfigurationCandidate(candidate); + candidateStages.set(candidate, "discarded"); + } + throw new AdminJobFailure("Original configuration candidate was not issued", { + configuration: { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }, + }); +} +/** Fixed replacement-owner continuation. Every attempted operation is looked + * up at its original address. Only the exact locked pre-save state permits + * recording ABANDON; there is no fresh preparation, desired save or seal. */ +export async function resumeRetainedConfigurationCandidate(): Promise> { + const current = administrativeConfigurationState(currentAdministrativeConfigurationWriter()).job + .checkpointResult?._configurationCandidate as Record | undefined; + if (current?.retirement !== undefined) return resumeCompletedConfigurationRetirement(); + let candidate: ValidatedConfigurationCandidate; + try { + const { job } = administrativeConfigurationState(currentAdministrativeConfigurationWriter()), + retained = job.checkpointResult?._configurationCandidate as + | Record + | undefined; + candidate = await (retained?.maintenanceId === null + ? restoreRetainedOrdinaryCandidate() + : restoreRetainedMaintenanceCandidate()); + } catch { + throw new JobObservationPending("recovery_required"); + } + const state = validatedConfigurationCandidateState(candidate), + { job } = administrativeConfigurationState(state.writer), + maintenance = await configurationMaintenanceProtocol; + if (candidateStages.get(candidate) === "recovered_unissued") { + await discardUnissuedMaintenanceCandidate(candidate); + if (job.checkpointResult?._cancelRequestedAt !== undefined) + return completeUnissuedConfigurationCancellation(candidate); + const configuration: TrustedConfigurationApplyOutcome = { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + await job.checkpoint({ + ...job.checkpointResult, + configuration, + _configurationOutcomes: { + ...(job.checkpointResult?._configurationOutcomes as Record), + [candidate.operationKey]: configuration, + }, + }); + return { configuration }; + } + if (recoveredPreRootCandidates.has(candidate)) { + const receipt = await persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: state.overlay.coordinatorMode, + signal: job.signal, + }); + return { configuration: receipt.application }; + } + if (recoveredCommittedCandidates.has(candidate)) { + const receipt = state.record.maintenanceId + ? await persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: state.overlay.coordinatorMode, + signal: job.signal, + }) + : await applyCommittedOrdinaryCandidate(candidate, job.signal); + if ( + !state.record.maintenanceId && + state.record.apiFollowUp === "service-apply" && + completeTrustedConfigurationApply(receipt.application) + ) + await completeProspectiveServiceCredential(candidate); + if ( + state.record.apiFollowUp !== "integration-reload" || + !completeTrustedConfigurationApply(receipt.application) + ) + return { configuration: receipt.application }; + // An interrupted activation is not permission to invoke setup again. Only + // an unattempted reload may begin; a durable completion can be copied. + const original = job.checkpointResult?.integrationReload as Record | undefined; + if ( + original && + (original.state !== "complete" || + original.candidateId !== candidate.candidateId || + original.revisionId !== candidate.revisionId) + ) + throw new JobObservationPending("recovery_required"); + let reloaded: { message: string; reloaded: number; enabled: number }; + if (original) + reloaded = z + .strictObject({ + message: z.string(), + reloaded: z.number().int().nonnegative(), + enabled: z.number().int().nonnegative(), + }) + .parse(original.result); + else { + await job.checkpoint({ + ...job.checkpointResult, + integrationReload: { + state: "attempted", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }, + }); + try { + reloaded = await reloadProspectiveIntegrations(candidate); + } catch { + throw new JobObservationPending("recovery_required"); + } + await job.checkpoint({ + ...job.checkpointResult, + _configurationCandidate: { + ...(job.checkpointResult?._configurationCandidate as Record), + phase: "settled", + }, + integrationReload: { + state: "complete", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + result: reloaded, + }, + }); + } + await retireSettledProspectiveConfiguration(candidate); + const provenance = state.record.privateIntegrationReload?.legacyImport; + return { + configuration: receipt.application, + ...(provenance + ? { + integrationImport: { + ok: true, + imported: provenance.changedIntegrationIds.length, + skipped: provenance.integrationIds.filter( + (id) => !provenance.changedIntegrationIds.includes(id), + ), + ...reloaded, + }, + } + : {}), + }; + } + try { + await maintenance.prepareConfigurationMaintenance(candidate); + } catch (error) { + const preparation = + state.record.prepareOperationKey && + readProjection(job.checkpointResult).operations[state.record.prepareOperationKey]; + if ( + preparation && + preparation.terminal && + preparation.receiptDelivery?.state === "confirmed" && + ["failed", "timed_out"].includes(preparation.receiptDelivery.terminalReceipt.state) + ) + return { configuration: await completeNeverPreparedConfiguration(candidate) }; + throw error; + } + const retainedAbandon = job.checkpointResult?._configurationMaintenanceDecision; + const abandon = + retainedAbandon === undefined + ? await maintenance.prepareConfigurationMaintenanceAbandon(candidate) + : configurationMaintenanceDecisionCheckpointSchema.parse(retainedAbandon).operation; + if (abandon.decision !== "abandon") throw new JobObservationPending("recovery_required"); + await maintenance.decideConfigurationMaintenance(abandon); + const observed = await maintenance.inspectConfigurationMaintenance({ + maintenanceId: state.record.maintenanceId!, + candidateId: candidate.candidateId, + }); + const settlement = job.checkpointResult?._configurationMaintenanceSettle; + await maintenance.settleConfigurationMaintenance( + settlement === undefined + ? { + kind: "configurationMaintenance.settle", + maintenanceId: observed.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: observed.prepare.operationKey, + prepareFingerprint: observed.prepare.operationFingerprint, + planDigest: observed.planDigest, + decisionId: abandon.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: configurationMaintenanceSettleRequestDigest(observed), + } + : configurationMaintenanceSettleCheckpointSchema.parse(settlement).operation, + ); + await discardUnusedPreparedProspectiveConfiguration(candidate); + if (job.checkpointResult?._cancelRequestedAt !== undefined) + return completeUnusedConfigurationCancellation(candidate); + const configuration: TrustedConfigurationApplyOutcome = { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + await job.checkpoint({ + ...job.checkpointResult, + configuration, + _configurationOutcomes: { + ...(job.checkpointResult?._configurationOutcomes as Record), + [candidate.operationKey]: configuration, + }, + }); + return { configuration }; +} +function fieldDefinition(schema: Record | undefined, key: string) { + const fields = (schema?.properties ?? schema ?? {}) as Record>; + return Object.hasOwn(fields, key) ? fields[key] : undefined; +} +function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWriter): void { + const state = administrativeConfigurationState(writer); + if (overlay.purpose === "migration") assertLegacyIntegrationImportOverlay(writer, overlay); + if (overlay.purpose === "migration" && state.participant?.kind !== "configuration-migration") + throw Error("Configuration commit participant unavailable"); + if ( + overlay.purpose === "platform" || + overlay.changes.some((change) => change.kind === "platform") + ) + throw Error("Platform configuration adapter unavailable"); + const fields = new Set(); + for (const change of overlay.changes) { + if (change.kind === "platform") throw Error("Platform configuration adapter unavailable"); + const target = + "serviceId" in change + ? change.serviceId + : "integrationId" in change + ? change.integrationId + : "selection"; + if ("integrationId" in change && !writer.integrationIds.includes(change.integrationId)) + throw Error("Configuration integration lock unavailable"); + const keys = + "set" in change + ? [...Object.keys(change.set), ...change.unset] + : "key" in change + ? [change.key] + : ["selection"]; + for (const key of keys) { + const identity = `${change.kind.startsWith("integration") ? "integration" : "service"}:${target}:${key}`; + if (fields.has(identity)) throw Error("Conflicting configuration overlay"); + fields.add(identity); + } + if ( + change.kind === "service-secret" && + change.serviceId === "data-manager" && + change.key === "TRANSITOUS_API_KEYS" + ) { + if ( + overlay.purpose !== "transit-credentials" || + !state.participant?.managedServiceSecretKeys.some( + (item) => item.serviceId === "data-manager" && item.key === change.key, + ) + ) + throw Error("Managed credential participant unavailable"); + } + } + const serviceIds = new Set( + overlay.changes.flatMap((change) => ("serviceId" in change ? [change.serviceId] : [])), + ); + const integrationIds = new Set( + overlay.changes.flatMap((change) => ("integrationId" in change ? [change.integrationId] : [])), + ); + if ( + (overlay.purpose === "render" && overlay.changes.length) || + (overlay.purpose === "selection" && + (overlay.changes.length !== 1 || overlay.changes[0]?.kind !== "selection")) || + (["service-config", "service-secret", "transit-credentials"].includes(overlay.purpose) && + (serviceIds.size !== 1 || integrationIds.size > 0)) || + (["integration-config", "integration-secret"].includes(overlay.purpose) && + (integrationIds.size !== 1 || serviceIds.size > 0)) + ) + throw Error("Configuration purpose conflicts with overlay"); + const permitted = { + "service-config": "service-config", + "service-secret": "service-secret", + "integration-config": "integration-config", + "integration-secret": "integration-secret", + "transit-credentials": "service-secret", + selection: "selection", + } as const; + const kind = permitted[overlay.purpose as keyof typeof permitted]; + if (kind && overlay.changes.some((change) => change.kind !== kind)) + throw Error("Configuration purpose conflicts with overlay"); +} + +async function mergeOverlay( + context: AdministrativeConfigurationReadContext, + overlay: Overlay, + source: ConfigurationSourcesInspection, +) { + const rows = { + services: context.services, + integrations: context.integrations, + env: context.env, + serviceRows: structuredClone([...context.serviceRows]), + integrationRows: structuredClone([...context.integrationRows]), + serviceSecretRows: structuredClone([...context.serviceSecretRows]), + integrationSecretRows: structuredClone([...context.integrationSecretRows]), + }; + const now = new Date(); + for (const change of overlay.changes) { + if (change.kind === "platform" || change.kind === "selection") continue; + const service = + "serviceId" in change + ? context.services.find((service) => service.manifest.id === change.serviceId) + : undefined; + const integration = + "integrationId" in change + ? context.integrations.find((integration) => integration.id === change.integrationId) + : undefined; + const manifest = service?.manifest ?? integration?.manifest; + if (!manifest) throw Error("Unknown configuration target"); + const resolved = service + ? await resolveServiceConfigWithSourcesStrict( + { + id: service.manifest.id, + configSchema: service.manifest.configSchema, + containerEnv: service.manifest.container.environment, + }, + context, + ) + : await resolveIntegrationConfigWithSourcesStrict( + integration!.manifest, + integration!.directory, + context, + ); + const changedKeys = + "set" in change ? [...Object.keys(change.set), ...change.unset] : [change.key]; + for (const key of changedKeys) { + const definition = + integration && key === "enabled" + ? { type: "boolean" } + : fieldDefinition(manifest.configSchema, key); + if ( + !definition || + definition.readOnly === true || + "key" in change !== (definition["x-openmapx-secret"] === true) + ) + throw Error("Undeclared or immutable configuration field"); + const host = + service && + source.services + .find((item) => item.serviceId === service.manifest.id) + ?.fields.find((field) => field.key === key); + if (host?.source === "host" || ["env", "config.json"].includes(resolved[key]?.source ?? "")) + throw Error("Configuration field is host-owned"); + } + if (change.kind === "service-config") { + let row = rows.serviceRows.find((row) => row.serviceId === change.serviceId); + if (!row) { + row = { + id: randomUUID(), + serviceId: change.serviceId, + config: {}, + createdAt: now, + updatedAt: now, + }; + rows.serviceRows.push(row); + } + row.config = { ...row.config, ...change.set }; + for (const key of change.unset) delete row.config[key]; + } else if (change.kind === "integration-config") { + let row = rows.integrationRows.find((row) => row.integrationId === change.integrationId); + if (!row) { + row = { + id: randomUUID(), + integrationId: change.integrationId, + config: {}, + createdAt: now, + updatedAt: now, + }; + rows.integrationRows.push(row); + } + row.config = { ...row.config, ...change.set }; + for (const key of change.unset) delete row.config[key]; + } else if (change.kind === "service-secret") { + rows.serviceSecretRows = rows.serviceSecretRows.filter( + (row) => row.serviceId !== change.serviceId || row.key !== change.key, + ); + if (change.action === "set") + rows.serviceSecretRows.push({ + id: randomUUID(), + serviceId: change.serviceId, + key: change.key, + ...encrypt(change.value), + createdAt: now, + updatedAt: now, + updatedBy: null, + }); + } else { + rows.integrationSecretRows = rows.integrationSecretRows.filter( + (row) => row.integrationId !== change.integrationId || row.key !== change.key, + ); + if (change.action === "set") + rows.integrationSecretRows.push({ + id: randomUUID(), + integrationId: change.integrationId, + key: change.key, + ...encrypt(change.value), + createdAt: now, + updatedAt: now, + updatedBy: null, + }); + } + } + return rows; +} + +export async function prepareProspectiveConfiguration(input: { + actorId: string; + expectedCurrentRevisionId: string | null; + overlay: ProspectiveConfigurationOverlay; + operationKey: string; + signal?: AbortSignal; + followUp?: "service-apply"; + coordinatorIntent?: CoordinatorLifecycleIntent; +}): Promise { + const writer = currentAdministrativeConfigurationWriter(); + if (writer.actorId !== input.actorId || writer.operationKey !== input.operationKey) + throw Error("Configuration writer identity conflict"); + const owner = administrativeConfigurationState(writer); + if (owner.job.jobType === "service.bulk" && owner.job.payload.action === "update") { + const { advanceCompletedBulkCandidate } = await import("./service-update-batch"); + await advanceCompletedBulkCandidate(writer); + } + const coordinator = input.coordinatorIntent + ? await assertCoordinatorLifecycleIntent(input.coordinatorIntent, writer) + : null; + input.signal?.throwIfAborted(); + const overlay = overlaySchema.parse(input.overlay); + validateOverlay(overlay, writer); + if ( + coordinator && + (input.expectedCurrentRevisionId === null || + input.followUp || + canonicalJson(overlay) !== + canonicalJson({ purpose: "render", coordinatorMode: "ensure-running", changes: [] })) + ) + throw Error("Fixed unchanged coordinator render required"); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const publisher = publisherOptions(token); + const source = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.sources.inspect" }, + createDurableOpsKey("configuration.sources.inspect", input.operationKey), + { signal: input.signal }, + ); + if (source.revisionId !== input.expectedCurrentRevisionId) + throw Error("Configuration revision conflict"); + let cleanupOperationKey: string | undefined; + const result = await withAdministrativeConfigurationRead( + { signal: input.signal }, + async (context) => { + const fingerprint = await sourceFingerprint(context, token); + const merged = await mergeOverlay(context, overlay, source); + const read = openAdministrativeConfigurationRead(merged); + try { + const { buildConfigurationInput } = await import("./configuration-input"); + const selection = overlay.changes.find((change) => change.kind === "selection"); + const payload = trustedConfigurationPayloadSchema.parse({ + ...(await withAdministrativeConfigurationReadScope(read, () => + buildConfigurationInput(read), + )), + ...(selection?.kind === "selection" ? { selectedRoots: selection.selectedRoots } : {}), + }); + const integrationReloadSnapshot = await withAdministrativeConfigurationReadScope( + read, + async () => + Object.fromEntries( + await Promise.all( + read.integrations.map( + async (integration) => + [ + integration.id, + await resolveIntegrationConfigWithSourcesStrict( + integration.manifest, + integration.directory, + read, + ), + ] as const, + ), + ), + ), + ); + freeze(integrationReloadSnapshot); + if ((await sourceFingerprint(context, token)) !== fingerprint) + throw Error("Configuration source changed during preparation"); + for (const entry of payload.serviceConfigs) { + const manifest = context.services.find( + (service) => service.manifest.id === entry.serviceId, + )!.manifest; + if (!coreServices.validateTrustedConfigurationValues(entry.values, manifest.configSchema)) + throw Error("Configuration schema rejected"); + } + const candidateId = coordinator?.candidateId ?? randomUUID(); + const affectedServiceIds = coordinator + ? ["data-manager"] + : [ + ...new Set( + overlay.changes.flatMap((change) => + "serviceId" in change ? [change.serviceId] : [], + ), + ), + ].sort(); + const kind = + overlay.purpose === "selection" + ? "serviceSelection.apply" + : overlay.purpose === "service-config" + ? "serviceConfig.apply" + : overlay.purpose === "service-secret" || overlay.purpose === "transit-credentials" + ? "vault.apply" + : overlay.purpose.startsWith("integration-") + ? "integrationConfig.apply" + : "stack.render"; + if (input.followUp && (kind !== "vault.apply" || affectedServiceIds.length !== 1)) + throw Error("Invalid credential continuation"); + const maintenanceId = + coordinator || (input.followUp && affectedServiceIds[0] === "data-manager") + ? randomUUID() + : null; + const apiFollowUp = + (coordinator ? (`coordinator-${coordinator.action}` as const) : input.followUp) ?? + (kind === "integrationConfig.apply" || + (overlay.purpose === "migration" && owner.participant?.kind === "configuration-migration") + ? "integration-reload" + : "none"); + const root = maintenanceId + ? prepareAdminJobOperationRoot( + owner.job, + "configurationMaintenance.prepare", + createDurableOpsKey( + "configuration-maintenance.prepare", + `${owner.job.jobId}:${candidateId}`, + ), + ) + : prepareAdminJobOperationRoot(owner.job, kind, input.operationKey); + const coordinatorLifecycle = coordinator + ? coordinator.action === "apply" + ? { + version: 1 as const, + serviceId: "data-manager" as const, + action: "apply" as const, + originalOrdinal: null, + noDeps: false as const, + } + : { + version: 1 as const, + serviceId: "data-manager" as const, + action: "update" as const, + originalOrdinal: coordinator.originalOrdinal!, + noDeps: coordinator.noDeps, + pull: await reserveCoordinatorPullSubmission( + owner.job, + input.coordinatorIntent!, + input.expectedCurrentRevisionId!, + root.parent, + ), + } + : undefined; + const applyOperationKey = maintenanceId + ? controlChildOperationKey(root.operationKey, "configuration-commit", 0) + : root.operationKey; + cleanupOperationKey = applyOperationKey; + const prerequisiteAddress = createDurableOpsKey( + "configuration-maintenance.coordinator", + `${owner.job.jobId}:${candidateId}`, + ); + const coordinatorPrerequisite = + maintenanceId && + overlay.coordinatorMode === "ensure-running" && + input.expectedCurrentRevisionId !== null + ? { + stepAddress: prerequisiteAddress, + operation: { + kind: "service.start" as const, + serviceId: "data-manager" as const, + expectedConfigurationRevisionId: input.expectedCurrentRevisionId, + }, + root: prepareAdminJobOperationRoot( + { + ...owner.job, + checkpointResult: { + ...owner.job.checkpointResult, + _parentExecution: root.parent, + }, + }, + "service.start", + prerequisiteAddress, + ), + } + : null; + const operationForRevision = (revisionId: string): OpsOperation => + ({ + kind, + revisionId, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + expectedSourceRevision: source.sourceRevision, + coordinatorMode: overlay.coordinatorMode, + ...(maintenanceId ? { maintenanceId, candidateId } : {}), + ...(kind === "serviceConfig.apply" || kind === "vault.apply" + ? { serviceId: affectedServiceIds[0]! } + : {}), + ...(kind === "integrationConfig.apply" + ? { integrationId: (overlay.changes[0] as { integrationId: string }).integrationId } + : {}), + }) as OpsOperation; + const sealed = sealTrustedConfigurationSnapshot({ + role: "api", + operationKey: applyOperationKey, + operationForRevision, + payload, + token, + nonce: `nonce_${randomUUID().replaceAll("-", "")}`, + }); + const record: PreparedConfigurationRecord = { + version: 1, + candidateId, + operationKey: applyOperationKey, + operation: sealed.operation, + sourceFingerprint: fingerprint, + expectedSourceRevision: source.sourceRevision, + sealed: sealed.bytes.toString("base64"), + privateOverlay: overlay, + apiJobId: owner.job.jobId, + actorId: writer.actorId, + parentOperationKey: root.parent.operationKey, + parentFingerprint: root.parent.fingerprint, + parentDeadlineAt: root.parent.deadlineAt, + affectedServiceIds, + apiFollowUp, + ...(coordinatorLifecycle + ? { + privateCoordinatorLifecycle: coordinatorLifecycle, + privateCoordinatorSource: source, + } + : {}), + maintenanceId, + prepareOperationKey: maintenanceId ? root.operationKey : null, + coordinatorPrerequisite, + ...(maintenanceId && !owner.participant && overlay.purpose === "service-secret" + ? { + privateMaintenancePreparation: { + version: 1 as const, + source, + serviceSecrets: merged.serviceSecretRows + .filter( + (row) => + row.serviceId === "data-manager" && + overlay.changes.some( + (change) => + change.kind === "service-secret" && + change.serviceId === row.serviceId && + change.key === row.key && + change.action === "set", + ), + ) + .map((row) => ({ + id: row.id, + serviceId: "data-manager" as const, + key: row.key, + ciphertext: row.ciphertext, + iv: row.iv, + tag: row.tag, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + updatedBy: row.updatedBy, + })), + }, + } + : {}), + ...(apiFollowUp === "integration-reload" + ? { + privateIntegrationReload: { + version: 1 as const, + services: [...merged.services], + integrations: merged.integrations.map((i) => ({ + id: i.id, + directory: i.directory, + manifest: i.manifest, + isBuiltIn: i.isBuiltIn, + })), + values: integrationReloadSnapshot, + ...(overlay.purpose === "migration" + ? { + legacyImport: ( + await import("./configuration-migration/legacy-integration-import-intent") + ).retainedLegacyIntegrationImportIntent(writer), + } + : {}), + }, + } + : {}), + }; + const candidate = Object.freeze({ + candidateId, + operationKey: input.operationKey, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + revisionId: sealed.revisionId, + sourceFingerprint: fingerprint, + sealedBytes: sealed.bytes.length, + affectedServiceIds: Object.freeze(affectedServiceIds), + }) as ValidatedConfigurationCandidate; + freeze(overlay); + freeze(payload); + freeze(record); + candidates.set(candidate, { + writer, + overlay, + payload, + sealed, + record, + publisher, + source, + rows: merged, + sourceRows: { + serviceRows: context.serviceRows, + integrationRows: context.integrationRows, + serviceSecretRows: context.serviceSecretRows, + integrationSecretRows: context.integrationSecretRows, + }, + persisted: false, + integrationReloadSnapshot, + }); + candidateStages.set(candidate, "preflight"); + await verifyConfigurationCommitParticipant(candidate, writer); + assertRowReceiptBudget( + savedRowIntents(validatedConfigurationCandidateState(candidate)).map((row) => ({ + ...row, + rowVersion: row.state === "present" ? "4294967295" : null, + })), + ); + await retainPreparedTrustedConfigurationCandidate(publisher, record); + const state = validatedConfigurationCandidateState(candidate); + try { + state.preview = await executeAndWait( + createApiOpsClient(), + { + kind: "configuration.hardlinks.preview", + candidateId, + revisionId: candidate.revisionId, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + expectedSourceRevision: source.sourceRevision, + coordinatorMode: overlay.coordinatorMode, + }, + createDurableOpsKey("configuration.hardlinks.preview", input.operationKey), + { signal: input.signal }, + ); + if ( + overlay.coordinatorMode === "require-running" && + (state.preview.entries || coordinator) && + state.preview.coordinatorState !== "running" + ) + throw Error("Configuration coordinator prerequisite unavailable"); + if ( + maintenanceId && + overlay.coordinatorMode === "ensure-running" && + (state.preview.entries || coordinator) && + (input.expectedCurrentRevisionId === null || + !["running", "stopped"].includes(state.preview.coordinatorState)) + ) + throw Error("Configuration coordinator prerequisite unavailable"); + assertMaintenanceIntentCapacity(state, owner.job.checkpointResult); + } catch (error) { + if (coordinator?.action === "update") + await closeUnissuedCoordinatorPull(owner.job, candidateId, record.operationKey); + // Preview is read-only. This exact inert candidate has no apply/guard + // admission, so a conclusive local refusal may free its reserved queue slot. + await discardUnadmittedTrustedConfigurationCandidate(candidate); + candidateStages.set(candidate, "discarded"); + // This operation is read-only and the exact local slot/packet closure + // above proves no apply/maintenance/pull dispatch. A lost preview + // response is a known local preflight failure, not an unknown effect. + const { ApiOpsError } = await import("./ops-client"); + if (error instanceof ApiOpsError && error.errorClass === "configuration_prerequisite") { + const { ConfigurationPrerequisiteFailure } = await import("./admin-job-failure"); + throw new ConfigurationPrerequisiteFailure(); + } + throw new AdminJobFailure("Configuration candidate preflight failed", { + errorCode: "configuration_preflight_failed", + }); + } + candidateStages.set(candidate, "prepared"); + return candidate; + } finally { + closeAdministrativeConfigurationRead(read); + } + }, + ).catch(async (error) => { + if (coordinator?.action === "update") + await closeUnissuedCoordinatorPull(owner.job, coordinator.candidateId, cleanupOperationKey); + throw error; + }); + return result; +} + +/** Local count arithmetic only. No future decision body/root is allocated. + * All exact known middle-owner addresses plus the two original controls must + * fit the existing shared64 map before helper/pull; later CAS still rechecks. */ +function assertMaintenanceIntentCapacity(state: CandidateState, result: unknown): void { + const record = state.record; + if (!record.maintenanceId || !record.prepareOperationKey) return; + const projection = readProjection(result), + keys = new Set([record.prepareOperationKey]); + const recreate = + record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + ? record.affectedServiceIds + : []; + const count = 1 + (state.preview?.entries ? 1 : 0) + recreate.length; + if (count > 32) throw Error("Prospective maintenance effect capacity exceeded"); + keys.add(controlChildOperationKey(record.prepareOperationKey, "configuration-commit", 0)); + let ordinal = 1; + if (state.preview?.entries) + keys.add( + controlChildOperationKey(record.prepareOperationKey, "maintenance-hardlinks", ordinal++), + ); + for (const _service of recreate) + keys.add( + controlChildOperationKey(record.prepareOperationKey, "maintenance-recreate", ordinal++), + ); + if ( + record.coordinatorPrerequisite && + (state.preview?.entries || record.privateCoordinatorLifecycle) + ) + keys.add(record.coordinatorPrerequisite.root.operationKey); + const missing = [...keys].filter((key) => !projection.operations[key]).length; + if ( + Math.max(Object.keys(projection.operations).length, Object.keys(projection.steps).length) + + missing + + 2 > + 64 + ) + throw Error("Prospective maintenance intent capacity exceeded"); +} + +function savedRowIntents(state: CandidateState) { + const { rows, sourceRows } = state; + if (!rows || !sourceRows) throw Error("Original prospective desired state required"); + if (state.record.privateCoordinatorLifecycle) { + const rowVersion = (row: object) => { + const version = (row as { _rowVersion?: unknown })._rowVersion; + if (typeof version !== "string" || !/^\d{1,10}$/.test(version)) + throw Error("Configuration row version unavailable"); + return version; + }; + // The unchanged full desired snapshot is recorded truthfully, without a + // dummy write, resave, new ciphertext or invented xmin. + const ordinary = ( + kind: "service-config" | "integration-config", + values: typeof sourceRows.serviceRows | typeof sourceRows.integrationRows, + ) => + values.map((row) => ({ + kind, + ownerId: "serviceId" in row ? row.serviceId : row.integrationId, + rowId: row.id, + previousRowVersion: rowVersion(row), + key: undefined as string | undefined, + state: "present" as const, + })); + const secrets = ( + kind: "service-secret" | "integration-secret", + values: typeof sourceRows.serviceSecretRows | typeof sourceRows.integrationSecretRows, + ) => + values.map((row) => ({ + kind, + ownerId: "serviceId" in row ? row.serviceId : row.integrationId, + key: row.key, + rowId: row.id, + previousRowVersion: rowVersion(row), + state: "present" as const, + })); + const receipts = [ + ...ordinary("service-config", sourceRows.serviceRows), + ...ordinary("integration-config", sourceRows.integrationRows), + ...secrets("service-secret", sourceRows.serviceSecretRows), + ...secrets("integration-secret", sourceRows.integrationSecretRows), + ]; + if (receipts.length > 512 || receipts.some((row) => !/^\d{1,10}$/.test(row.previousRowVersion))) + throw Error("Configuration unchanged row receipt capacity unavailable"); + return receipts; + } + const seen = new Set(); + return state.overlay.changes.flatMap((change) => { + if (change.kind === "selection" || change.kind === "platform") return []; + const ownerId = "serviceId" in change ? change.serviceId : change.integrationId; + const secretKey = "key" in change ? change.key : undefined, + identity = `${change.kind}:${ownerId}:${secretKey ?? ""}`; + if (seen.has(identity)) return []; + seen.add(identity); + const source = + change.kind === "service-config" + ? sourceRows.serviceRows.find((row) => row.serviceId === ownerId) + : change.kind === "integration-config" + ? sourceRows.integrationRows.find((row) => row.integrationId === ownerId) + : change.kind === "service-secret" + ? sourceRows.serviceSecretRows.find( + (row) => row.serviceId === ownerId && row.key === secretKey, + ) + : sourceRows.integrationSecretRows.find( + (row) => row.integrationId === ownerId && row.key === secretKey, + ); + const desired = + change.kind === "service-config" + ? rows.serviceRows.find((row) => row.serviceId === ownerId) + : change.kind === "integration-config" + ? rows.integrationRows.find((row) => row.integrationId === ownerId) + : change.kind === "service-secret" + ? rows.serviceSecretRows.find( + (row) => row.serviceId === ownerId && row.key === secretKey, + ) + : rows.integrationSecretRows.find( + (row) => row.integrationId === ownerId && row.key === secretKey, + ); + if (!source && !desired) return []; + const rowId = source?.id ?? desired!.id, + previousRowVersion = source ? (source as { _rowVersion?: unknown })._rowVersion : null; + if ( + typeof rowId !== "string" || + rowId.length > 128 || + (previousRowVersion !== null && + (typeof previousRowVersion !== "string" || !/^\d{1,10}$/.test(previousRowVersion))) + ) + throw Error("Configuration row version unavailable"); + return [ + { + kind: change.kind, + ownerId, + ...(secretKey ? { key: secretKey } : {}), + rowId, + previousRowVersion: previousRowVersion as string | null, + state: desired ? ("present" as const) : ("absent" as const), + }, + ]; + }); +} +function assertRowReceiptBudget(rows: unknown) { + if (Buffer.byteLength(canonicalJson(rows)) > 65536) + throw Error("Configuration row receipt capacity unavailable"); +} +async function readSavedRowVersions( + tx: ConfigurationWriteTransaction, + state: CandidateState, +): Promise { + // Same transaction/physical connection as the writes. No values or secret + // hashes enter the checkpoint; versions belong to this actual SQL outcome. + const serviceRows = await tx + .select({ + id: schema.serviceConfig.id, + ownerId: schema.serviceConfig.serviceId, + rowVersion: sql`xmin::text`, + }) + .from(schema.serviceConfig); + const integrationRows = await tx + .select({ + id: schema.integrationConfig.id, + ownerId: schema.integrationConfig.integrationId, + rowVersion: sql`xmin::text`, + }) + .from(schema.integrationConfig); + const serviceSecrets = await tx + .select({ + id: schema.serviceSecret.id, + ownerId: schema.serviceSecret.serviceId, + key: schema.serviceSecret.key, + rowVersion: sql`xmin::text`, + }) + .from(schema.serviceSecret); + const integrationSecrets = await tx + .select({ + id: schema.integrationSecret.id, + ownerId: schema.integrationSecret.integrationId, + key: schema.integrationSecret.key, + rowVersion: sql`xmin::text`, + }) + .from(schema.integrationSecret); + const receipts = savedRowIntents(state).map((intent) => { + const rows = + intent.kind === "service-config" + ? serviceRows + : intent.kind === "integration-config" + ? integrationRows + : intent.kind === "service-secret" + ? serviceSecrets + : integrationSecrets; + const matches = rows.filter( + (row) => + row.ownerId === intent.ownerId && (!intent.key || ("key" in row && row.key === intent.key)), + ); + if (intent.state === "absent") { + if (matches.length) throw Error("Configuration deletion receipt conflict"); + return { ...intent, rowVersion: null }; + } + if ( + matches.length !== 1 || + matches[0]!.id !== intent.rowId || + !/^\d{1,10}$/.test(matches[0]!.rowVersion) + ) + throw Error("Configuration saved row receipt conflict"); + return { ...intent, rowVersion: matches[0]!.rowVersion }; + }); + assertRowReceiptBudget(receipts); + return receipts; +} + +async function persistOverlay(tx: ConfigurationWriteTransaction, state: CandidateState) { + if (!state.rows || !state.sourceRows) throw Error("Original prospective desired state required"); + const version = (row: { id: string } | undefined) => { + const value = (row as { _rowVersion?: unknown } | undefined)?._rowVersion; + if (row && (typeof value !== "string" || !/^\d+$/.test(value))) + throw Error("Configuration row version unavailable"); + return value as string; + }; + const one = (rows: unknown[]) => { + if (rows.length !== 1) throw Error("Configuration row ownership conflict"); + }; + const touched = new Set(); + for (const change of state.overlay.changes) { + if (change.kind === "service-config") { + const row = state.rows.serviceRows.find((row) => row.serviceId === change.serviceId)!; + if (touched.has(`s:${row.serviceId}`)) continue; + touched.add(`s:${row.serviceId}`); + const prior = state.sourceRows.serviceRows.find((item) => item.serviceId === row.serviceId); + if (prior) + one( + await tx + .update(schema.serviceConfig) + .set({ config: row.config, updatedAt: new Date() }) + .where(and(eq(schema.serviceConfig.id, prior.id), sql`xmin::text = ${version(prior)}`)) + .returning({ id: schema.serviceConfig.id }), + ); + else + one( + await tx + .insert(schema.serviceConfig) + .values({ id: row.id, serviceId: row.serviceId, config: row.config }) + .onConflictDoNothing() + .returning({ id: schema.serviceConfig.id }), + ); + } else if (change.kind === "integration-config") { + const row = state.rows.integrationRows.find( + (row) => row.integrationId === change.integrationId, + )!; + if (touched.has(`i:${row.integrationId}`)) continue; + touched.add(`i:${row.integrationId}`); + const prior = state.sourceRows.integrationRows.find( + (item) => item.integrationId === row.integrationId, + ); + if (prior) + one( + await tx + .update(schema.integrationConfig) + .set({ config: row.config, updatedAt: new Date() }) + .where( + and(eq(schema.integrationConfig.id, prior.id), sql`xmin::text = ${version(prior)}`), + ) + .returning({ id: schema.integrationConfig.id }), + ); + else + one( + await tx + .insert(schema.integrationConfig) + .values({ id: row.id, integrationId: row.integrationId, config: row.config }) + .onConflictDoNothing() + .returning({ id: schema.integrationConfig.id }), + ); + } else if (change.kind === "service-secret") { + const owner = administrativeConfigurationState(state.writer); + if ( + owner.participant?.managedServiceSecretKeys.some( + (entry) => entry.serviceId === change.serviceId && entry.key === change.key, + ) + ) + continue; + const prior = state.sourceRows.serviceSecretRows.find( + (row) => row.serviceId === change.serviceId && row.key === change.key, + ); + const where = + prior && and(eq(schema.serviceSecret.id, prior.id), sql`xmin::text = ${version(prior)}`); + if (change.action === "delete") { + if (prior) + one( + await tx + .delete(schema.serviceSecret) + .where(where) + .returning({ id: schema.serviceSecret.id }), + ); + } else { + const row = state.rows.serviceSecretRows.find( + (row) => row.serviceId === change.serviceId && row.key === change.key, + )!; + if (prior) + one( + await tx + .update(schema.serviceSecret) + .set({ ciphertext: row.ciphertext, iv: row.iv, tag: row.tag, updatedAt: new Date() }) + .where(where) + .returning({ id: schema.serviceSecret.id }), + ); + else + one( + await tx + .insert(schema.serviceSecret) + .values(row) + .onConflictDoNothing() + .returning({ id: schema.serviceSecret.id }), + ); + } + } else if (change.kind === "integration-secret") { + const prior = state.sourceRows.integrationSecretRows.find( + (row) => row.integrationId === change.integrationId && row.key === change.key, + ); + const where = + prior && + and(eq(schema.integrationSecret.id, prior.id), sql`xmin::text = ${version(prior)}`); + if (change.action === "delete") { + if (prior) + one( + await tx + .delete(schema.integrationSecret) + .where(where) + .returning({ id: schema.integrationSecret.id }), + ); + } else { + const row = state.rows.integrationSecretRows.find( + (row) => row.integrationId === change.integrationId && row.key === change.key, + )!; + if (prior) + one( + await tx + .update(schema.integrationSecret) + .set({ ciphertext: row.ciphertext, iv: row.iv, tag: row.tag, updatedAt: new Date() }) + .where(where) + .returning({ id: schema.integrationSecret.id }), + ); + else + one( + await tx + .insert(schema.integrationSecret) + .values(row) + .onConflictDoNothing() + .returning({ id: schema.integrationSecret.id }), + ); + } + } + } + return readSavedRowVersions(tx, state); +} + +async function verifyCurrentPreview( + candidate: ValidatedConfigurationCandidate, + state: CandidateState, + signal?: AbortSignal, +) { + if (!state.source) throw Error("Original prospective preview source required"); + const preview = await executeAndWait( + createApiOpsClient(), + { + kind: "configuration.hardlinks.preview", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + expectedCurrentRevisionId: candidate.expectedCurrentRevisionId, + expectedSourceRevision: state.source.sourceRevision, + coordinatorMode: state.overlay.coordinatorMode, + }, + createDurableOpsKey("configuration.hardlinks.preview", candidate.operationKey), + { signal }, + ); + if ( + preview.previewDigest !== state.preview?.previewDigest || + (state.overlay.coordinatorMode === "require-running" && + preview.entries && + preview.coordinatorState !== "running") + ) + rejectUnissuedCandidate(candidate, "Configuration preview changed"); + return preview; +} + +/** The ordinary prerequisite is deliberately outside the maintenance graph. + * Its exact old-revision body/address/deadline were sealed in the inert private + * candidate. Managed execution persists the admission attempt before POST and + * completes only after authenticated terminal receipt delivery and release. */ +async function prepareMaintenanceCoordinator( + candidate: ValidatedConfigurationCandidate, + state: CandidateState, + signal?: AbortSignal, +): Promise { + if ( + !state.record.maintenanceId || + (!state.preview?.entries && !state.record.privateCoordinatorLifecycle) || + state.overlay.coordinatorMode !== "ensure-running" + ) + return; + const owner = administrativeConfigurationState(state.writer), + fixed = state.record.coordinatorPrerequisite; + if ( + !fixed || + !candidate.expectedCurrentRevisionId || + fixed.operation.expectedConfigurationRevisionId !== candidate.expectedCurrentRevisionId || + fixed.root.parent.operationKey !== state.record.parentOperationKey || + fixed.root.parent.fingerprint !== state.record.parentFingerprint || + fixed.root.parent.deadlineAt !== state.record.parentDeadlineAt || + fixed.root.operationKey === state.record.prepareOperationKey + ) + throw Error("Configuration coordinator prerequisite unavailable"); + signal?.throwIfAborted(); + owner.job.signal.throwIfAborted(); + if ( + Date.now() >= fixed.root.requestedDeadlineAt || + JSON.parse(state.sealed.bytes.toString()).expiresAt <= new Date().toISOString() + ) + rejectUnissuedCandidate(candidate, "Configuration candidate expired"); + await withAdministrativeConfigurationRead({ signal }, async (read) => { + if ((await sourceFingerprint(read, state.publisher.token)) !== candidate.sourceFingerprint) + rejectUnissuedCandidate(candidate, "Configuration source conflict"); + }); + await verifyConfigurationCommitParticipant(candidate, state.writer); + // Explicit lifecycle candidates observe an already-ready OLD coordinator. + // In particular never Compose up after Update has staged a new image. + const beforePull = + state.record.privateCoordinatorLifecycle?.action === "update" + ? readProjection(owner.job.checkpointResult).operations[ + state.record.privateCoordinatorLifecycle.pull.operationKey + ] + : undefined; + if (state.record.privateCoordinatorLifecycle && beforePull && beforePull.phase !== "prepared") { + const helper = readProjection(owner.job.checkpointResult).operations[fixed.root.operationKey]; + if (helper) + await executeAdminJobOperation( + owner.job, + fixed.operation, + "configuration-maintenance.coordinator", + { operationKey: fixed.stepAddress, preparedRoot: fixed.root }, + ); + const ready = await verifyCurrentPreview(candidate, state, signal); + if (ready.coordinatorState !== "running") throw new JobObservationPending("recovery_required"); + return; + } + if (state.record.privateCoordinatorLifecycle && state.preview?.coordinatorState === "running") { + await verifyCurrentPreview(candidate, state, signal); + return; + } + if (beforePull && beforePull.phase !== "prepared") + throw new JobObservationPending("recovery_required"); + await executeAdminJobOperation( + owner.job, + fixed.operation, + "configuration-maintenance.coordinator", + { + operationKey: fixed.stepAddress, + preparedRoot: fixed.root, + }, + ); + signal?.throwIfAborted(); + owner.job.signal.throwIfAborted(); + if (JSON.parse(state.sealed.bytes.toString()).expiresAt <= new Date().toISOString()) + rejectUnissuedCandidate(candidate, "Configuration candidate expired"); + await withAdministrativeConfigurationRead({ signal }, async (read) => { + if ((await sourceFingerprint(read, state.publisher.token)) !== candidate.sourceFingerprint) + rejectUnissuedCandidate(candidate, "Configuration source conflict"); + }); + const ready = await verifyCurrentPreview(candidate, state, signal); + if (ready.coordinatorState !== "running") + rejectUnissuedCandidate(candidate, "Configuration coordinator prerequisite unavailable"); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCoordinatorPrerequisite: { + candidateId: candidate.candidateId, + operationKey: fixed.root.operationKey, + stepAddress: fixed.stepAddress, + expectedRevisionId: candidate.expectedCurrentRevisionId, + state: "settled_ready", + }, + }); +} + +function candidateCheckpoint(candidate: ValidatedConfigurationCandidate, state: CandidateState) { + return { + candidateId: candidate.candidateId, + operationKey: candidate.operationKey, + revisionId: candidate.revisionId, + sourceFingerprint: candidate.sourceFingerprint, + phase: "prepared", + previewDigest: state.preview?.previewDigest, + preview: state.preview, + coordinatorMode: state.overlay.coordinatorMode, + sealedOperationKey: state.record.operationKey, + maintenanceId: state.record.maintenanceId, + prepareOperationKey: state.record.prepareOperationKey, + }; +} +/** Shared ownership-only entry: the exact inert candidate and ordinary OLD + * revision coordinator prerequisite precede the first maintenance admission. + * This never persists desired rows or publishes a consumable apply claim. */ +export async function stageProspectiveConfigurationMaintenance( + candidate: ValidatedConfigurationCandidate, + signal?: AbortSignal, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer), + checkpoint = candidateCheckpoint(candidate, state), + retained = owner.job.checkpointResult?._configurationCandidate as + | Record + | undefined, + parent = { + operationKey: state.record.parentOperationKey, + fingerprint: state.record.parentFingerprint, + deadlineAt: state.record.parentDeadlineAt, + }; + if ( + candidateStages.get(candidate) !== "prepared" || + !state.record.maintenanceId || + owner.transactionActive || + owner.participant?.mode === "preview" || + (retained && + (retained.candidateId !== checkpoint.candidateId || + retained.operationKey !== checkpoint.operationKey || + retained.sealedOperationKey !== checkpoint.sealedOperationKey || + retained.phase !== "prepared")) || + (owner.job.checkpointResult?._parentExecution && + canonicalJson(owner.job.checkpointResult._parentExecution) !== canonicalJson(parent)) + ) + throw Error("Original inert maintenance candidate required"); + await verifyConfigurationCommitParticipant(candidate, state.writer); + if (!retained) + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: checkpoint, + _parentExecution: parent, + }); + if (JSON.parse(state.sealed.bytes.toString()).expiresAt <= new Date().toISOString()) + rejectUnissuedCandidate(candidate, "Configuration candidate expired"); + await verifyCurrentPreview(candidate, state, signal); + await prepareMaintenanceCoordinator(candidate, state, signal); + if (state.record.privateCoordinatorLifecycle?.action === "update") { + try { + await deliverCoordinatorPullSubmission(owner.job, candidate.candidateId); + } catch (error) { + const { ApiOpsError } = await import("./ops-client"), + pull = readProjection(owner.job.checkpointResult).operations[ + state.record.privateCoordinatorLifecycle.pull.operationKey + ]; + // Only this fixed delivery's confirmed original failure selects cleanup. + // The locked nonissuance producer still re-observes its actual ACK and + // excludes every maintenance/save attempt before removing the packet. + if ( + error instanceof ApiOpsError && + error.errorClass !== "recovery_required" && + pull?.phase === "terminal" && + pull.terminal && + pull.receiptDelivery?.state === "confirmed" && + ["failed", "timed_out"].includes(pull.receiptDelivery.terminalReceipt.state) + ) + rejectUnissuedCandidate(candidate, "Original coordinator pull failed before maintenance"); + throw error; + } + if (JSON.parse(state.sealed.bytes.toString()).expiresAt <= new Date().toISOString()) + rejectUnissuedCandidate(candidate, "Configuration candidate expired"); + await verifyCurrentPreview(candidate, state, signal); + } +} + +/** The fixed prepared candidate producer. Desired rows, participant commit, + * actual xmin receipts and ONE continue decision share the original writer + * connection/transaction. This entry does not publish an apply claim or issue + * a host grant; the persister coordinates those only after decision delivery. */ +export async function commitProspectiveConfigurationMaintenance( + candidate: ValidatedConfigurationCandidate, +): Promise { + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer), + record = state.record, + stage = candidateStages.get(candidate), + maintenance = await configurationMaintenanceProtocol; + if ( + !record.maintenanceId || + !record.prepareOperationKey || + !owner.job.execution || + owner.transactionActive || + owner.participant?.mode === "preview" || + !["prepared", "committed"].includes(stage ?? "") + ) + throw Error("Original maintenance candidate transaction required"); + if (stage === "committed") { + let observed: Record | undefined; + const operation = await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + source = configurationMaintenanceDecisionCheckpointSchema.parse( + result?._configurationMaintenanceDecision, + ), + { checkpointDigest, ...body } = source.operation; + if ( + !result || + retained?.candidateId !== candidate.candidateId || + retained.revisionId !== candidate.revisionId || + retained.sealedOperationKey !== record.operationKey || + retained.maintenanceId !== record.maintenanceId || + retained.phase !== "saved" || + body.decision !== "continue" || + body.candidateId !== candidate.candidateId || + body.prepareOperationKey !== record.prepareOperationKey || + canonicalJson(retained.rowVersions) !== canonicalJson(source.rowVersions) || + checkpointDigest !== configurationMaintenanceDbDecisionDigest(body, source.rowVersions) + ) + throw Error("Original committed maintenance decision required"); + await assertConfigurationSavedRows(tx, source.rowVersions); + observed = result; + return source.operation; + }); + if (!observed) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = observed; + return operation; + } + const prepared = await maintenance.prepareConfigurationMaintenance(candidate), + checkpoint = candidateCheckpoint(candidate, state); + if ( + prepared.state !== "prepared" || + prepared.decision !== null || + prepared.dmFence?.state !== "prepared" || + prepared.generation !== "not_started" || + prepared.followUp !== "not_started" || + prepared.preflight !== "current" + ) + throw Error("Original prepared maintenance reservation required"); + await verifyCurrentPreview(candidate, state, owner.job.signal); + await verifyConfigurationCommitParticipant(candidate, state.writer); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { ...checkpoint, phase: "save_attempted" }, + }); + candidateStages.set(candidate, "save_attempted"); + let committed: Record | undefined, + decision: ConfigurationMaintenanceDecision | undefined; + try { + await withConfigurationWriteTransaction(state.writer, async (tx, context) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + remote = result?._configurationMaintenance as Record | undefined, + preparation = readProjection(result).operations[record.prepareOperationKey!]; + if ( + !result || + retained?.candidateId !== candidate.candidateId || + retained.operationKey !== candidate.operationKey || + retained.sealedOperationKey !== record.operationKey || + retained.revisionId !== candidate.revisionId || + retained.phase !== "save_attempted" || + retained.rowVersions !== undefined || + remote?.phase !== "prepared" || + canonicalJson(remote.receipt) !== canonicalJson(prepared) || + result._configurationMaintenanceDecision !== undefined || + result._configurationMaintenanceRollback !== undefined || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) || + preparation?.receiptDelivery?.state !== "confirmed" || + preparation.receiptDelivery.terminalReceipt.state !== "succeeded" || + canonicalJson(preparation.receiptDelivery.terminalReceipt.result) !== + canonicalJson(prepared) + ) + throw Error("Original owned prepared candidate checkpoint required"); + if ((await sourceFingerprint(context, state.publisher.token)) !== candidate.sourceFingerprint) + throw Error("Configuration source conflict"); + if (JSON.parse(state.sealed.bytes.toString()).expiresAt <= new Date().toISOString()) + throw Error("Configuration candidate expired"); + await verifyConfigurationCommitParticipant(candidate, state.writer); + await verifyCurrentPreview(candidate, state, owner.job.signal); + const fresh = await maintenance.inspectConfigurationMaintenance({ + maintenanceId: record.maintenanceId!, + candidateId: candidate.candidateId, + }); + if ( + fresh.preflight !== "current" || + canonicalJson({ ...fresh, preflight: prepared.preflight }) !== canonicalJson(prepared) + ) + throw Error("Original prepared maintenance preflight changed"); + await owner.participant?.commit(candidate, state.writer, tx); + const rowVersions = await persistOverlay(tx, state), + body = { + kind: "configurationMaintenance.decide" as const, + maintenanceId: record.maintenanceId!, + candidateId: candidate.candidateId, + prepareOperationKey: record.prepareOperationKey!, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: randomUUID(), + decision: "continue" as const, + expectedSequence: prepared.sequence, + checkpointVersion: owner.job.execution!.version, + }; + decision = configurationMaintenanceDecisionSchema.parse({ + ...body, + checkpointDigest: configurationMaintenanceDbDecisionDigest(body, rowVersions), + }); + const source = configurationMaintenanceDecisionCheckpointSchema.parse({ + version: 1, + operation: decision, + rowVersions, + attempt: "not_attempted", + }), + [latest] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"); + if (!latest) throw Error("Configuration job ownership changed"); + committed = { + ...(latest.result as Record), + _configurationCandidate: { ...checkpoint, phase: "saved", rowVersions }, + _configurationMaintenanceDecision: source, + }; + const saved = await tx + .update(schema.adminJob) + .set({ result: committed }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw Error("Configuration job ownership changed"); + }); + } catch (error) { + if (error instanceof ConfigurationWriteRolledBack) { + // This catch consumes the actual transaction wrapper's conclusive native + // rollback, never a caller-supplied error, boolean or row-absence guess. + candidateStages.set(candidate, "rolled_back"); + let rollbackCheckpoint: Record | undefined; + await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + actual = row?.result as Record | undefined, + retained = actual?._configurationCandidate as Record | undefined, + remote = actual?._configurationMaintenance as Record | undefined; + if ( + !actual || + retained?.candidateId !== candidate.candidateId || + retained.revisionId !== candidate.revisionId || + retained.sealedOperationKey !== record.operationKey || + retained.maintenanceId !== record.maintenanceId || + retained.phase !== "save_attempted" || + retained.rowVersions !== undefined || + actual._configurationMaintenanceDecision !== undefined || + actual._configurationMaintenanceRollback !== undefined || + remote?.phase !== "prepared" || + canonicalJson(remote.receipt) !== canonicalJson(prepared) || + hasPendingReceiptDelivery(actual) || + hasUnsettledManagedOperation(actual) + ) + throw new JobObservationPending("recovery_required"); + const proof = configurationMaintenanceRollbackCheckpointSchema.parse({ + version: 1, + disposition: "known_rolled_back", + rollbackId: randomUUID(), + candidateId: candidate.candidateId, + maintenanceId: record.maintenanceId, + prepareOperationKey: record.prepareOperationKey, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + checkpointVersion: owner.job.execution!.version, + }); + rollbackCheckpoint = { + ...actual, + _configurationCandidate: { ...retained, phase: "rolled_back" }, + _configurationMaintenanceRollback: proof, + }; + const saved = await tx + .update(schema.adminJob) + .set({ result: rollbackCheckpoint }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + }); + if (!rollbackCheckpoint) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = rollbackCheckpoint; + } + if (error instanceof ConfigurationWriteCommitUnknown) + throw new JobObservationPending("recovery_required"); + throw error; + } + if (!committed || !decision) throw new JobObservationPending("recovery_required"); + state.persisted = true; + candidateStages.set(candidate, "committed"); + owner.job.checkpointResult = committed; + return decision; +} + +export async function persistAndApplyProspectiveConfiguration( + candidate: ValidatedConfigurationCandidate, + options: { coordinatorMode: CoordinatorMode; signal?: AbortSignal }, +): Promise { + const state = validatedConfigurationCandidateState(candidate); + const owner = administrativeConfigurationState(state.writer), + stage = candidateStages.get(candidate); + if ( + (stage !== "prepared" && stage !== "committed") || + options.coordinatorMode !== state.overlay.coordinatorMode || + owner.participant?.mode === "preview" + ) + throw Error("Configuration candidate persistence refused"); + if (["migration", "transit-credentials"].includes(state.overlay.purpose) && !owner.participant) + throw Error("Configuration commit participant unavailable"); + if (!state.record.maintenanceId && stage === "committed") + return applyCommittedOrdinaryCandidate(candidate, options.signal); + if (state.record.maintenanceId) { + const maintenance = await configurationMaintenanceProtocol; + let decision: ConfigurationMaintenanceDecision; + try { + decision = await commitProspectiveConfigurationMaintenance(candidate); + } catch (error) { + // A known original pre-root rejection has no remote reservation to + // abandon. Prove nonissuance under the same row lock before deleting its + // inert packet; unknown helper/root/save attempts stay pinned. + if ( + candidateStages.get(candidate) === "prepared" && + rejectedUnissuedCandidates.has(candidate) && + owner.job.checkpointResult?._configurationMaintenance === undefined + ) { + await discardUnissuedMaintenanceCandidate(candidate); + throw new AdminJobFailure("Configuration candidate preflight failed", { + configuration: { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }, + }); + } + const preparation = + state.record.prepareOperationKey && + readProjection(owner.job.checkpointResult).operations[state.record.prepareOperationKey]; + if ( + candidateStages.get(candidate) === "prepared" && + preparation && + preparation.terminal && + preparation.receiptDelivery?.state === "confirmed" && + ["failed", "timed_out"].includes(preparation.receiptDelivery.terminalReceipt.state) + ) { + await completeNeverPreparedConfiguration(candidate); + throw error; + } + if ( + !(error instanceof ConfigurationWriteRolledBack) || + candidateStages.get(candidate) !== "rolled_back" + ) + throw error; + const abandon = await maintenance.prepareConfigurationMaintenanceAbandon(candidate); + await maintenance.decideConfigurationMaintenance(abandon); + const observed = await maintenance.inspectConfigurationMaintenance({ + maintenanceId: state.record.maintenanceId, + candidateId: candidate.candidateId, + }); + await maintenance.settleConfigurationMaintenance({ + kind: "configurationMaintenance.settle", + maintenanceId: observed.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: observed.prepare.operationKey, + prepareFingerprint: observed.prepare.operationFingerprint, + planDigest: observed.planDigest, + decisionId: abandon.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: configurationMaintenanceSettleRequestDigest(observed), + }); + await discardUnusedPreparedProspectiveConfiguration(candidate); + const application: TrustedConfigurationApplyOutcome = { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + configuration: application, + _configurationOutcomes: { + ...(owner.job.checkpointResult?._configurationOutcomes as Record), + [candidate.operationKey]: application, + }, + }); + return { + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + saved: "conflict", + application, + }; + } + const authorized = await maintenance.decideConfigurationMaintenance(decision), + generation = configurationApplyResultSchema.parse( + await maintenance.executeConfigurationMaintenanceStep(candidate, 0), + ); + const contained = generation.outcome === "not_committed" && !generation.cleanupFailed; + if ( + !contained && + (generation.outcome !== "committed" || + generation.revisionId !== candidate.revisionId || + generation.followUp !== "complete" || + generation.cleanupFailed) + ) + throw new JobObservationPending("recovery_required"); + let registryUpdateFailed = false; + if (generation.outcome === "committed") + try { + getServiceRegistry().applyEnabledIds(new Set(generation.enabledServiceIds)); + } catch { + registryUpdateFailed = true; + } + if (!contained) + for (const step of authorized.steps.slice(1)) + await maintenance.executeConfigurationMaintenanceStep(candidate, step.ordinal); + const observed = await maintenance.inspectConfigurationMaintenance(authorized.binding); + if ( + !observed.decision || + observed.decision.decisionId !== decision.decisionId || + (contained + ? observed.generation !== "not_committed" || + observed.appliedRevisionId !== null || + observed.followUp !== "contained" + : observed.generation !== "committed" || + observed.appliedRevisionId !== candidate.revisionId || + observed.followUp !== "complete") + ) + throw new JobObservationPending("recovery_required"); + // Once admitted, the original request is immutable even when finalization + // advanced the observed owner sequence. Recovery delivers/copies that + // exact retained winner; it never derives a new checksum/body or key. + const originalSettlement = owner.job.checkpointResult?._configurationMaintenanceSettle, + settled = await maintenance.settleConfigurationMaintenance( + originalSettlement === undefined + ? { + kind: "configurationMaintenance.settle", + maintenanceId: observed.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: observed.prepare.operationKey, + prepareFingerprint: observed.prepare.operationFingerprint, + planDigest: observed.planDigest, + decisionId: observed.decision.decisionId, + expectedSequence: observed.sequence, + evidenceDigest: configurationMaintenanceSettleRequestDigest(observed), + } + : configurationMaintenanceSettleCheckpointSchema.parse(originalSettlement).operation, + ); + if ( + contained + ? settled.generation !== "not_committed" || + settled.appliedRevisionId !== null || + settled.followUp !== "contained" + : settled.generation !== "committed" || + settled.appliedRevisionId !== candidate.revisionId || + settled.followUp !== "complete" + ) + throw new JobObservationPending("recovery_required"); + const application: TrustedConfigurationApplyOutcome = { + ...generation, + publisherCleanupFailed: false, + registryUpdateFailed, + }, + checkpoint = owner.job.checkpointResult?._configurationCandidate as Record, + source = configurationMaintenanceDecisionCheckpointSchema.parse( + owner.job.checkpointResult?._configurationMaintenanceDecision, + ), + serviceId = "serviceId" in state.sealed.operation ? state.sealed.operation.serviceId : null; + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { + ...checkpoint, + phase: registryUpdateFailed ? "unknown" : "settled", + }, + configuration: application, + _configurationOutcomes: { + ...(owner.job.checkpointResult?._configurationOutcomes as Record), + [candidate.operationKey]: application, + }, + ...(state.record.apiFollowUp === "service-apply" && serviceId + ? { + serviceCredential: { + serviceId, + revisionId: candidate.revisionId, + followUp: contained ? "contained" : "complete", + }, + } + : {}), + ...(state.record.privateCoordinatorLifecycle + ? { + coordinatorLifecycle: { + serviceId: "data-manager", + action: state.record.privateCoordinatorLifecycle.action, + revisionId: candidate.revisionId, + followUp: contained ? "contained" : "complete", + }, + } + : {}), + }); + if (!registryUpdateFailed) await retireSettledProspectiveConfiguration(candidate); + return { + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + saved: state.record.privateCoordinatorLifecycle + ? "unchanged" + : source.rowVersions.length + ? "committed" + : "unchanged", + application, + }; + } + const checkpoint = candidateCheckpoint(candidate, state); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: checkpoint, + }); + try { + await verifyCurrentPreview(candidate, state, options.signal); + await verifyConfigurationCommitParticipant(candidate, state.writer); + } catch (error) { + await discardUnadmittedTrustedConfigurationCandidate(candidate); + candidateStages.set(candidate, "discarded"); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { ...checkpoint, phase: "discarded" }, + }); + throw error; + } + let rowVersions: ConfigurationSavedRow[] = []; + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { ...checkpoint, phase: "save_attempted" }, + }); + candidateStages.set(candidate, "save_attempted"); + try { + await withConfigurationWriteTransaction(state.writer, async (tx, context) => { + if ((await sourceFingerprint(context, state.publisher.token)) !== candidate.sourceFingerprint) + throw Error("Configuration source conflict"); + if (JSON.parse(state.sealed.bytes.toString()).expiresAt <= new Date().toISOString()) + throw Error("Configuration candidate expired"); + await verifyConfigurationCommitParticipant(candidate, state.writer); + await verifyCurrentPreview(candidate, state, options.signal); + await owner.participant?.commit(candidate, state.writer, tx); + rowVersions = await persistOverlay(tx, state); + const rows = await tx + .update(schema.adminJob) + .set({ + result: sql`coalesce(${schema.adminJob.result}, '{}'::jsonb) || ${JSON.stringify({ _configurationCandidate: { ...checkpoint, phase: "saved", rowVersions } })}::jsonb`, + }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (rows.length !== 1) throw Error("Configuration job ownership changed"); + }); + } catch (error) { + if (error instanceof ConfigurationWriteRolledBack) { + candidateStages.set(candidate, "rolled_back"); + await discardUnadmittedTrustedConfigurationCandidate(candidate); + candidateStages.set(candidate, "discarded"); + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { ...checkpoint, phase: "settled", saved: "conflict" }, + }); + return { + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + saved: "conflict", + application: { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }, + }; + } + if (error instanceof ConfigurationWriteCommitUnknown) + throw new JobObservationPending("recovery_required"); + throw error; + } + state.persisted = true; + candidateStages.set(candidate, "committed"); + owner.job.checkpointResult = { + ...owner.job.checkpointResult, + _configurationCandidate: { ...checkpoint, phase: "saved", rowVersions }, + }; + return applyCommittedOrdinaryCandidate(candidate, options.signal); +} +async function applyCommittedOrdinaryCandidate( + candidate: ValidatedConfigurationCandidate, + signal?: AbortSignal, +): Promise { + assertCommittedConfigurationCandidate(candidate); + const state = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(state.writer); + if (state.record.maintenanceId) throw Error("Ordinary candidate continuation required"); + let committed: Record | undefined; + const rowVersions = await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined; + if ( + !result || + !["saved", "follow_up_pending"].includes(String(retained?.phase)) || + retained?.candidateId !== candidate.candidateId || + retained.revisionId !== candidate.revisionId || + retained.operationKey !== candidate.operationKey || + retained.sealedOperationKey !== state.record.operationKey + ) + throw new JobObservationPending("recovery_required"); + const rows = z.array(configurationSavedRowSchema).max(512).parse(retained.rowVersions); + await assertConfigurationSavedRows(tx, rows); + committed = result; + return rows; + }); + if (!committed) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = committed; + signal?.throwIfAborted(); + const execute = () => + executeAdminJobOperation(owner.job, state.sealed.operation, "admin-job.configuration", { + operationKey: candidate.operationKey, + preparedRoot: { + operationKey: state.record.operationKey, + requestedDeadlineAt: parseControlRootOperationKey(state.record.operationKey).notAfter, + parent: { + operationKey: state.record.parentOperationKey, + fingerprint: state.record.parentFingerprint, + deadlineAt: state.record.parentDeadlineAt, + }, + }, + }); + const attempted = readProjection(committed).operations[state.record.operationKey]?.attempt; + const consumed = attempted + ? { submission: { status: "fulfilled" as const, value: await execute() }, cleanupFailed: false } + : await consumePublishedTrustedConfiguration( + await publishPreparedTrustedConfigurationSnapshot(candidate), + execute, + ); + if (consumed.submission.status === "rejected") throw consumed.submission.reason; + const result = consumed.submission.value as TrustedConfigurationApplyOutcome; + let registryUpdateFailed = false; + if (result.outcome === "committed") { + try { + getServiceRegistry().applyEnabledIds(new Set(result.enabledServiceIds)); + } catch { + registryUpdateFailed = true; + } + } + const application = { + ...result, + publisherCleanupFailed: consumed.cleanupFailed, + registryUpdateFailed, + }; + const phase = + application.outcome === "unknown" || + application.publisherCleanupFailed || + application.registryUpdateFailed || + application.cleanupFailed + ? "unknown" + : application.outcome === "committed" && + state.record.apiFollowUp !== "none" && + application.followUp === "complete" + ? "follow_up_pending" + : "settled"; + await owner.job.checkpoint({ + ...owner.job.checkpointResult, + _configurationCandidate: { + ...(committed._configurationCandidate as Record), + phase, + rowVersions, + }, + configuration: application, + _configurationOutcomes: { + ...(owner.job.checkpointResult?._configurationOutcomes as Record), + [candidate.operationKey]: application, + }, + }); + if (phase === "settled" && completeTrustedConfigurationApply(application)) + await retireSettledProspectiveConfiguration(candidate); + return { + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + saved: rowVersions.length ? "committed" : "unchanged", + application, + }; +} + +// Fixed nominal candidate publication and maintenance share this sole owner. +/** Consume the producer's exact seal, never reseal or rebuild after desired rows save. */ +export async function publishPreparedTrustedConfigurationSnapshot( + candidate: ValidatedConfigurationCandidate, +) { + assertCommittedConfigurationCandidate(candidate); + const state = validatedConfigurationCandidateState(candidate); + const inspected = inspectPreparedConfigurationRecord( + encodePreparedConfigurationRecord(state.record, state.publisher.token), + state.publisher.token, + ); + if (inspected.snapshot.expiresAtMs <= Date.now()) throw new Error(REJECTED); + return publishSnapshot( + { + ...state.publisher, + operationKey: state.record.operationKey, + operationForRevision: () => state.sealed.operation, + payload: state.payload, + }, + {}, + state.sealed, + ); +} + +/** Fixed producer cleanup: an inert, unadmitted candidate cannot have a ready/claimed alias. */ +export async function discardUnadmittedTrustedConfigurationCandidate( + candidate: ValidatedConfigurationCandidate, +): Promise { + assertUnadmittedConfigurationCandidate(candidate); + const state = validatedConfigurationCandidateState(candidate); + return removeExactRetainedCandidate(candidate, state); +} +export async function retireSettledTrustedConfigurationCandidate( + candidate: ValidatedConfigurationCandidate, +): Promise { + await assertSettledConfigurationCandidate(candidate); + return removeExactRetainedCandidate(candidate, validatedConfigurationCandidateState(candidate)); +} +/** The fixed current producer's actual canonical completion authorizes only + * retirement of its original inert packet, including an interrupted unlink. + * The nominal handle is scoped to its live writer; raw IDs/flags cannot enter. */ +export async function completeRetainedConfigurationRetirement( + handle: RetainedConfigurationRetirement, +): Promise { + const state = retainedConfigurationRetirementState(handle), + options = state.publisher, + expected = state.checkpoint; + if ( + typeof expected.sealedOperationKey !== "string" || + !/^opk1_[A-Za-z0-9_-]{16,64}$/.test(expected.sealedOperationKey) || + typeof expected.revisionId !== "string" || + !/^cfg1_[A-Za-z0-9_-]{43}$/.test(expected.revisionId) + ) + throw Error(REJECTED); + await withPublicationLock(async () => { + const queue = await acquireTrustedConfigurationQueueLock({ + ...options, + participant: "api", + operationKey: expected.sealedOperationKey as string, + }); + try { + for (const path of [ + join(options.directory, `${expected.revisionId}.json`), + join(options.directory, ".claimed", `${expected.revisionId}.json`), + ]) { + try { + await lstat(path); + throw Error(REJECTED); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + const directory = join(options.directory, ".candidates"), + path = join(directory, `${expected.sealedOperationKey}.json`); + await validateDirectory(directory, options.ownerUid, options.ownerGid); + let file: Awaited>; + try { + file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + queue.assertHeld(); + await syncDirectory(directory); + return; + } + try { + const before = await file.stat(), + current = await lstat(path); + if ( + !before.isFile() || + before.nlink !== 1 || + before.size > 768 * 1024 || + before.uid !== options.ownerUid || + before.gid !== options.ownerGid || + (before.mode & 0o777) !== SNAPSHOT_MODE || + before.dev !== current.dev || + before.ino !== current.ino + ) + throw Error(REJECTED); + const { record, snapshot } = inspectPreparedConfigurationRecord( + await file.readFile(), + options.token, + ); + if ( + record.apiJobId !== state.apiJobId || + record.actorId !== state.actorId || + record.candidateId !== expected.candidateId || + record.operationKey !== expected.sealedOperationKey || + snapshot.revisionId !== expected.revisionId || + record.sourceFingerprint !== expected.sourceFingerprint || + record.maintenanceId !== expected.maintenanceId || + record.prepareOperationKey !== expected.prepareOperationKey + ) + throw Error(REJECTED); + retainedConfigurationRetirementState(handle); + queue.assertHeld(); + const latest = await lstat(path); + if (latest.dev !== before.dev || latest.ino !== before.ino) throw Error(REJECTED); + await unlink(path); + await syncDirectory(directory); + } finally { + await file.close(); + } + } finally { + queue.release(); + } + }); +} +async function removeExactRetainedCandidate( + candidate: ValidatedConfigurationCandidate, + state: ReturnType, +): Promise { + await withPublicationLock(async () => { + const options = state.publisher; + const queue = await acquireTrustedConfigurationQueueLock({ + ...options, + participant: "api", + operationKey: state.record.operationKey, + }); + try { + for (const path of [ + join(options.directory, `${candidate.revisionId}.json`), + join(options.directory, ".claimed", `${candidate.revisionId}.json`), + ]) { + try { + await lstat(path); + throw new Error(REJECTED); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + const directory = join(options.directory, ".candidates"); + await validateDirectory(directory, options.ownerUid, options.ownerGid); + const path = join(directory, `${state.record.operationKey}.json`); + let handle: Awaited>; + try { + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + // Both callers have already proved this nominal candidate unadmitted + // or conclusively settled. Absence completes only its inert-file + // deletion, never establishes an effect/guard outcome. Flush a prior + // unlink whose acknowledgement or directory sync was interrupted. + queue.assertHeld(); + await syncDirectory(directory); + return; + } + try { + const before = await handle.stat(), + current = await lstat(path); + if ( + !before.isFile() || + before.nlink !== 1 || + before.uid !== options.ownerUid || + before.gid !== options.ownerGid || + (before.mode & 0o777) !== SNAPSHOT_MODE || + before.dev !== current.dev || + before.ino !== current.ino || + !(await handle.readFile()).equals( + encodePreparedConfigurationRecord(state.record, options.token), + ) + ) + throw new Error(REJECTED); + queue.assertHeld(); + const latest = await lstat(path); + if (latest.dev !== before.dev || latest.ino !== before.ino) throw new Error(REJECTED); + await unlink(path); + await syncDirectory(directory); + } finally { + await handle.close(); + } + } finally { + queue.release(); + } + }); +} + +/** Request correlation only: the receiver restores the SAME admitted root. + * The original API parent deadline stays exact; only this fixed no-effect + * metadata attempt receives its own bounded reconciliation clock. */ +function prepareMaintenanceControlRequest( + job: JobContext, + operation: ConfigurationMaintenanceDecision | ConfigurationMaintenanceSettle, + stepAddress: string, +) { + if (operation.kind === "configurationMaintenance.decide" && operation.decision === "continue") + return prepareApiJobRoot(job, operation.kind, stepAddress); + const parent = z + .strictObject({ + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + }) + .parse(job.checkpointResult?._parentExecution); + if (!job.execution) throw Error("Original owned maintenance control required"); + const requestedDeadlineAt = Date.now() + 60000, + nonce = createHash("sha256") + .update("openmapx-api-control-root-nonce-v1\0") + .update(canonicalJson({ jobId: job.jobId, stepAddress })) + .digest() + .subarray(0, 16); + return { + operationKey: createControlRootOperationKey({ notAfter: requestedDeadlineAt, nonce }), + requestedDeadlineAt, + parent, + }; +} +class MaintenanceUnsavedCheckpointConflict extends Error { + constructor() { + super("Original known-unsaved maintenance checkpoint required"); + } +} +function abandonCheckpointDigest( + operation: Omit, +) { + return configurationMaintenanceDbDecisionDigest(operation, []); +} + +/** Retain only a known pre-save abandonment. The actual owned row lock waits + * out an earlier transaction; a stale cache, timeout or missing desired row + * cannot create it. This records the decision and issues NO host request/grant. + * Delivery must consume the exact committed checkpoint, never this return + * value as caller-supplied authorization. */ +export async function prepareConfigurationMaintenanceAbandon( + candidate: ValidatedConfigurationCandidate, +): Promise { + const source = validatedConfigurationCandidateState(candidate), + owner = administrativeConfigurationState(source.writer), + record = source.record; + if (!record.maintenanceId || !record.prepareOperationKey || !owner.job.execution) + throw new MaintenanceUnsavedCheckpointConflict(); + const prepared = await inspectConfigurationMaintenance({ + maintenanceId: record.maintenanceId, + candidateId: candidate.candidateId, + }); + if ( + prepared.state !== "prepared" || + prepared.decision !== null || + prepared.dmFence?.state !== "prepared" || + prepared.generation !== "not_started" || + prepared.followUp !== "not_started" || + prepared.appliedRevisionId !== null || + prepared.binding.applyOperationKey !== record.operationKey || + prepared.binding.applyOperationFingerprint !== + opsOperationFingerprint(source.sealed.operation) || + prepared.binding.revisionId !== candidate.revisionId || + prepared.prepare.operationKey !== record.prepareOperationKey + ) + throw new MaintenanceUnsavedCheckpointConflict(); + let committed: Record | undefined, + operation: ConfigurationMaintenanceDecision | undefined; + try { + await withConfigurationWriteTransaction(source.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined, + maintenance = result?._configurationMaintenance as Record | undefined, + projected = readProjection(result).operations[record.prepareOperationKey!], + delivery = projected?.receiptDelivery; + if (retained?.phase === "rolled_back") { + assertKnownRolledBackConfigurationCandidate(candidate); + const rollback = configurationMaintenanceRollbackCheckpointSchema.parse( + result?._configurationMaintenanceRollback, + ); + if ( + rollback.candidateId !== candidate.candidateId || + rollback.maintenanceId !== record.maintenanceId || + rollback.prepareOperationKey !== record.prepareOperationKey || + rollback.prepareFingerprint !== prepared.prepare.operationFingerprint || + rollback.planDigest !== prepared.planDigest || + rollback.checkpointVersion > owner.job.execution!.version + ) + throw new MaintenanceUnsavedCheckpointConflict(); + } else if (result?._configurationMaintenanceRollback !== undefined) + throw new MaintenanceUnsavedCheckpointConflict(); + if ( + !result || + retained?.candidateId !== candidate.candidateId || + retained.operationKey !== candidate.operationKey || + retained.revisionId !== candidate.revisionId || + retained.sealedOperationKey !== record.operationKey || + retained.maintenanceId !== record.maintenanceId || + retained.prepareOperationKey !== record.prepareOperationKey || + !["prepared", "rolled_back"].includes(String(retained.phase)) || + retained.rowVersions !== undefined || + maintenance?.phase !== "prepared" || + maintenance.prepareOperationKey !== record.prepareOperationKey || + maintenance.prepareOperationFingerprint !== prepared.prepare.operationFingerprint || + !projected?.terminal || + projected.kind !== "configurationMaintenance.prepare" || + delivery?.state !== "confirmed" || + delivery.terminalReceipt.state !== "succeeded" || + canonicalJson(delivery.terminalReceipt.result) !== canonicalJson(maintenance.receipt) || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) || + canonicalJson(result._parentExecution) !== + canonicalJson({ + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }) + ) + throw new MaintenanceUnsavedCheckpointConflict(); + const previous = result._configurationMaintenanceDecision; + if (previous !== undefined) { + const existing = abandonCheckpointSchema.parse(previous), + { checkpointDigest, ...body } = existing.operation; + if ( + body.decision !== "abandon" || + body.candidateId !== candidate.candidateId || + body.maintenanceId !== record.maintenanceId || + body.prepareOperationKey !== record.prepareOperationKey || + body.prepareFingerprint !== prepared.prepare.operationFingerprint || + body.planDigest !== prepared.planDigest || + body.expectedSequence !== prepared.sequence || + checkpointDigest !== abandonCheckpointDigest(body) + ) + throw new MaintenanceUnsavedCheckpointConflict(); + committed = result; + operation = existing.operation; + return; + } + const body = { + kind: "configurationMaintenance.decide" as const, + maintenanceId: record.maintenanceId!, + candidateId: candidate.candidateId, + prepareOperationKey: record.prepareOperationKey!, + prepareFingerprint: prepared.prepare.operationFingerprint, + planDigest: prepared.planDigest, + decisionId: randomUUID(), + decision: "abandon" as const, + expectedSequence: prepared.sequence, + checkpointVersion: owner.job.execution!.version, + }; + operation = configurationMaintenanceDecisionSchema.parse({ + ...body, + checkpointDigest: abandonCheckpointDigest(body), + }); + const checkpoint = abandonCheckpointSchema.parse({ + version: 1, + operation, + rowVersions: [], + attempt: "not_attempted", + }); + committed = { ...result, _configurationMaintenanceDecision: checkpoint }; + const saved = await tx + .update(schema.adminJob) + .set({ result: committed }) + .where(jobOwnership(owner.job.jobId, owner.job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new MaintenanceUnsavedCheckpointConflict(); + }); + } catch (error) { + if (error instanceof ConfigurationWriteCommitUnknown) + throw new JobObservationPending("recovery_required"); + if ( + error instanceof ConfigurationWriteRolledBack && + error.failure instanceof MaintenanceUnsavedCheckpointConflict + ) + throw error.failure; + throw error; + } + if (!committed || !operation) throw new JobObservationPending("recovery_required"); + owner.job.checkpointResult = committed; + return operation; +} + +/** Deliver the actual committed checkpoint, not a caller decision/digest. Its + * one stable API step consumes the existing64-step capacity; the host binds the + * request to the ORIGINAL owner rather than admitting a new physical root. */ +export async function decideConfigurationMaintenance( + decision: ConfigurationMaintenanceDecision, +): Promise { + const expected = configurationMaintenanceDecisionSchema.parse(decision), + exact = configurationMaintenanceIdentitySchema.parse({ + maintenanceId: expected.maintenanceId, + candidateId: expected.candidateId, + prepareOperationKey: expected.prepareOperationKey, + prepareFingerprint: expected.prepareFingerprint, + planDigest: expected.planDigest, + }), + owner = administrativeConfigurationState(currentAdministrativeConfigurationWriter()), + { job } = owner; + let committed: Record | undefined, + retained: z.infer | undefined; + try { + await withConfigurationWriteTransaction(owner.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined; + if (!result) throw new MaintenanceUnsavedCheckpointConflict(); + const source = abandonCheckpointSchema.parse(result._configurationMaintenanceDecision), + operation = source.operation, + { checkpointDigest, ...body } = operation; + if ( + canonicalJson(operation) !== canonicalJson(expected) || + checkpointDigest !== configurationMaintenanceDbDecisionDigest(body, source.rowVersions) || + canonicalJson({ + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + prepareOperationKey: operation.prepareOperationKey, + prepareFingerprint: operation.prepareFingerprint, + planDigest: operation.planDigest, + }) !== canonicalJson(exact) + ) + throw new MaintenanceUnsavedCheckpointConflict(); + if (operation.decision === "continue") { + const candidate = result._configurationCandidate as Record | undefined; + if ( + candidate?.candidateId !== exact.candidateId || + candidate.maintenanceId !== exact.maintenanceId || + candidate.prepareOperationKey !== exact.prepareOperationKey || + !["saved", "applying", "follow_up_pending", "settling", "settled"].includes( + String(candidate.phase), + ) || + canonicalJson(candidate.rowVersions) !== canonicalJson(source.rowVersions) + ) + throw new MaintenanceUnsavedCheckpointConflict(); + await assertConfigurationSavedRows(tx, source.rowVersions); + } + const maintenance = result._configurationMaintenance as Record | undefined, + prepared = configurationMaintenanceReceiptSchema.parse(maintenance?.receipt); + if ( + prepared.prepare.operationKey !== exact.prepareOperationKey || + prepared.prepare.operationFingerprint !== exact.prepareFingerprint || + prepared.binding.maintenanceId !== exact.maintenanceId || + prepared.binding.candidateId !== exact.candidateId || + prepared.planDigest !== exact.planDigest + ) + throw new MaintenanceUnsavedCheckpointConflict(); + if (source.delivery) { + retained = source; + committed = result; + return; + } + if (source.attempt !== "not_attempted" || maintenance?.phase !== "prepared") + throw new MaintenanceUnsavedCheckpointConflict(); + const stepAddress = createDurableOpsKey( + "configuration-maintenance.decide", + `${job.jobId}:${exact.candidateId}`, + ), + root = prepareMaintenanceControlRequest( + { ...job, checkpointResult: result }, + source.operation, + stepAddress, + ); + retained = abandonCheckpointSchema.parse({ ...source, delivery: { stepAddress, root } }); + committed = { ...result, _configurationMaintenanceDecision: retained }; + const saved = await tx + .update(schema.adminJob) + .set({ result: committed }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new MaintenanceUnsavedCheckpointConflict(); + }); + } catch (error) { + if (error instanceof ConfigurationWriteCommitUnknown) + throw new JobObservationPending("recovery_required"); + if ( + error instanceof ConfigurationWriteRolledBack && + error.failure instanceof MaintenanceUnsavedCheckpointConflict + ) + throw error.failure; + throw error; + } + if (!committed || !retained?.delivery) throw new JobObservationPending("recovery_required"); + job.checkpointResult = committed; + const source = retained, + receipt = configurationMaintenanceReceiptSchema.parse( + await executeAdminJobOperation(job, source.operation, "configuration-maintenance.decide", { + operationKey: source.delivery!.stepAddress, + preparedRoot: source.delivery!.root, + }), + ); + if ( + receipt.binding.maintenanceId !== exact.maintenanceId || + receipt.binding.candidateId !== exact.candidateId || + receipt.prepare.operationKey !== exact.prepareOperationKey || + receipt.prepare.operationFingerprint !== exact.prepareFingerprint || + receipt.planDigest !== exact.planDigest || + receipt.decision?.decisionId !== source.operation.decisionId || + receipt.decision.decision !== source.operation.decision || + receipt.decision.checkpointVersion !== source.operation.checkpointVersion || + receipt.decision.checkpointDigest !== source.operation.checkpointDigest + ) + throw new JobObservationPending("invalid_identity"); + const latest = job.checkpointResult?._configurationMaintenance as Record; + if (!["settling", "settled"].includes(String(latest.phase))) + await job.checkpoint({ + ...job.checkpointResult, + _configurationMaintenance: { + ...latest, + phase: "authorized", + decision: source.operation, + receipt, + }, + }); + return receipt; +} + +/** API-only coordinator for ONE frozen phase. The body/address comes from the + * actual committed job/decision, never a browser plan or preparedRoot recipe. + * An attempted phase is lookup-only and never republishes the sealed claim. */ +export async function executeConfigurationMaintenanceStep( + candidate: ValidatedConfigurationCandidate, + ordinal: number, +): Promise { + assertCommittedConfigurationCandidate(candidate); + const state = validatedConfigurationCandidateState(candidate), + { job } = administrativeConfigurationState(state.writer); + if (!state.record.maintenanceId) throw Error("Original maintenance candidate required"); + const prepared = await prepareOwnedConfigurationMaintenanceStep( + ordinal, + candidate.candidateId, + state.record.maintenanceId, + ), + execute = () => + executeAdminJobOperation(job, prepared.step.operation, "configuration-maintenance.step", { + operationKey: prepared.step.operationKey, + preparedRoot: prepared.root, + }); + if ( + prepared.step.phase !== "configuration-commit" || + readProjection(job.checkpointResult).operations[prepared.step.operationKey] + ) + return execute(); + const published = await publishPreparedTrustedConfigurationSnapshot(candidate), + consumed = await consumePublishedTrustedConfiguration(published, execute); + if (consumed.submission.status === "rejected") throw consumed.submission.reason; + if (consumed.cleanupFailed) throw new JobObservationPending("recovery_required"); + return consumed.submission.value; +} + +/** Retain the exact API-observed request before HMAC/POST. An attempted winner + * observes its original body after later owner sequences, never rekeys. */ +export async function settleConfigurationMaintenance( + request: ConfigurationMaintenanceSettle, +): Promise { + const exact = configurationMaintenanceSettleSchema.parse(request), + owner = administrativeConfigurationState(currentAdministrativeConfigurationWriter()), + { job } = owner; + if (owner.transactionActive || !job.execution) + throw Error("Original settlement job writer required"); + const prior = job.checkpointResult?._configurationMaintenanceSettle; + let observed: ConfigurationMaintenanceReceipt | undefined; + if (prior === undefined) { + observed = await inspectConfigurationMaintenance(exact); + if ( + observed.decision?.decisionId !== exact.decisionId || + observed.sequence !== exact.expectedSequence || + observed.prepare.operationKey !== exact.prepareOperationKey || + observed.prepare.operationFingerprint !== exact.prepareFingerprint || + observed.planDigest !== exact.planDigest || + configurationMaintenanceSettleRequestDigest(observed) !== exact.evidenceDigest + ) + throw Error("Actual API settlement observation conflict"); + } + let committed: Record | undefined; + const retained = await withConfigurationWriteTransaction(owner.writer, async (tx) => { + const [row] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined, + maintenance = result?._configurationMaintenance as Record | undefined, + decision = abandonCheckpointSchema.parse(result?._configurationMaintenanceDecision); + if ( + !result || + maintenance?.candidateId !== exact.candidateId || + maintenance.maintenanceId !== exact.maintenanceId || + maintenance.prepareOperationKey !== exact.prepareOperationKey || + maintenance.prepareOperationFingerprint !== exact.prepareFingerprint || + decision.operation.decisionId !== exact.decisionId || + decision.operation.planDigest !== exact.planDigest || + decision.attempt !== "attempted" + ) + throw Error("Actual original settlement checkpoint required"); + if (result._configurationMaintenanceSettle !== undefined) { + const original = configurationMaintenanceSettleCheckpointSchema.parse( + result._configurationMaintenanceSettle, + ); + if (canonicalJson(original.operation) !== canonicalJson(exact)) + throw Error("Original accepted settlement request changed"); + committed = result; + return original; + } + if ( + !observed || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) || + !decision.delivery + ) + throw new JobObservationPending("recovery_required"); + const delivered = + readProjection(result).operations[decision.delivery.root.operationKey]?.receiptDelivery; + if ( + delivered?.state !== "confirmed" || + delivered.terminalReceipt.state !== "succeeded" || + configurationMaintenanceReceiptSchema.parse(delivered.terminalReceipt.result).decision + ?.decisionId !== exact.decisionId + ) + throw Error("Original delivered decision required"); + const stepAddress = createDurableOpsKey( + "configuration-maintenance.settle", + `${job.jobId}:${exact.candidateId}`, + ), + checkpoint = configurationMaintenanceSettleCheckpointSchema.parse({ + version: 1, + operation: exact, + attempt: "not_attempted", + delivery: { + stepAddress, + root: prepareMaintenanceControlRequest( + { ...job, checkpointResult: result }, + exact, + stepAddress, + ), + }, + }); + committed = { ...result, _configurationMaintenanceSettle: checkpoint }; + const saved = await tx + .update(schema.adminJob) + .set({ result: committed }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + return checkpoint; + }); + if (!committed) throw new JobObservationPending("recovery_required"); + job.checkpointResult = committed; + const receipt = configurationMaintenanceReceiptSchema.parse( + await executeAdminJobOperation(job, retained.operation, "configuration-maintenance.settle", { + operationKey: retained.delivery.stepAddress, + preparedRoot: retained.delivery.root, + }), + ); + if ( + receipt.binding.maintenanceId !== exact.maintenanceId || + receipt.binding.candidateId !== exact.candidateId || + receipt.prepare.operationKey !== exact.prepareOperationKey || + receipt.prepare.operationFingerprint !== exact.prepareFingerprint || + receipt.planDigest !== exact.planDigest || + receipt.decision?.decisionId !== exact.decisionId || + receipt.state !== "settled" || + receipt.dmFence?.state !== "settled" + ) + throw new JobObservationPending("invalid_identity"); + const latest = job.checkpointResult?._configurationMaintenance as Record; + await job.checkpoint({ + ...job.checkpointResult, + _configurationMaintenance: { ...latest, phase: "settled", receipt }, + }); + return receipt; +} + +export async function prepareConfigurationMaintenance( + candidate: ValidatedConfigurationCandidate, +): Promise { + const source = validatedConfigurationCandidateState(candidate), + { job } = administrativeConfigurationState(source.writer), + record = source.record, + apply = source.sealed.operation; + if ( + !record.maintenanceId || + !record.prepareOperationKey || + !("expectedSourceRevision" in apply) || + !("coordinatorMode" in apply) + ) + throw Error("Actual maintenance-bound candidate required"); + const operation = configurationMaintenancePrepareSchema.parse({ + kind: "configurationMaintenance.prepare", + maintenanceId: record.maintenanceId, + candidateId: record.candidateId, + applyOperationKey: record.operationKey, + applyOperationFingerprint: opsOperationFingerprint(apply), + expectedCurrentRevisionId: candidate.expectedCurrentRevisionId, + revisionId: candidate.revisionId, + expectedSourceRevision: record.expectedSourceRevision, + coordinatorMode: source.overlay.coordinatorMode, + followUp: + record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + ? "service-apply" + : "render-only", + }); + const checkpoint = { + maintenanceId: record.maintenanceId, + candidateId: record.candidateId, + prepareOperationKey: record.prepareOperationKey, + prepareOperationFingerprint: opsOperationFingerprint(operation), + operation, + phase: "preparing", + }; + const retained = job.checkpointResult?._configurationMaintenance as + | Record + | undefined; + if ( + retained && + (retained.prepareOperationKey !== checkpoint.prepareOperationKey || + retained.prepareOperationFingerprint !== checkpoint.prepareOperationFingerprint || + retained.candidateId !== checkpoint.candidateId) + ) + throw Error("Original maintenance attempt changed"); + // An attempted root is lookup-only. Its replay never reruns preparation, + // helper admission or candidate validation against a newer desired state. + if (!readProjection(job.checkpointResult).operations[record.prepareOperationKey]?.attempt) + await stageProspectiveConfigurationMaintenance(candidate, job.signal); + if (!retained) + await job.checkpoint({ ...job.checkpointResult, _configurationMaintenance: checkpoint }); + const receipt = configurationMaintenanceReceiptSchema.parse( + await executeAdminJobOperation(job, operation, "configuration-maintenance.prepare", { + operationKey: createDurableOpsKey( + "configuration-maintenance.prepare", + `${job.jobId}:${candidate.candidateId}`, + ), + preparedRoot: { + operationKey: record.prepareOperationKey, + requestedDeadlineAt: parseControlRootOperationKey(record.prepareOperationKey).notAfter, + parent: { + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + }, + }, + }), + ); + if ( + receipt.binding.maintenanceId !== record.maintenanceId || + receipt.binding.candidateId !== candidate.candidateId || + receipt.binding.applyOperationKey !== record.operationKey || + receipt.binding.applyOperationFingerprint !== operation.applyOperationFingerprint || + receipt.prepare.operationKey !== record.prepareOperationKey || + receipt.prepare.operationFingerprint !== checkpoint.prepareOperationFingerprint || + receipt.apiJobId !== job.jobId || + receipt.parentOperationKey !== record.parentOperationKey || + receipt.parentFingerprint !== record.parentFingerprint || + receipt.parentDeadlineAt !== record.parentDeadlineAt || + receipt.state !== "prepared" || + receipt.preflight !== "current" + ) + throw Error("Original prepared maintenance receipt required"); + const latest = job.checkpointResult?._configurationMaintenance as + | Record + | undefined; + if (!["authorized", "applying", "settling", "settled"].includes(String(latest?.phase))) + await job.checkpoint({ + ...job.checkpointResult, + _configurationMaintenance: { ...checkpoint, phase: "prepared", receipt }, + }); + return receipt; +} +type MaintenanceInspectionCheckpoint = + | { attempt: ApiJobChildAttempt; state: "attempted" } + | { attempt: ApiJobChildAttempt; state: "complete"; receipt: ConfigurationMaintenanceReceipt }; +/** An inspection inside the final desired-write transaction uses THAT owned + * connection. It never waits on its own row through the generic checkpoint + * pool, changes the caller's committed cache or acknowledges mutation evidence + * from an uncommitted transaction. The host request itself is read-only. */ +async function checkpointMaintenanceInspection( + retained: Record, + inspection: MaintenanceInspectionCheckpoint, + transaction: AdministrativeConfigurationReadContext | null, +): Promise { + const owner = administrativeConfigurationState(currentAdministrativeConfigurationWriter()), + { job } = owner; + if (transaction === null) { + if (owner.transactionActive) throw Error("Maintenance inspection ownership changed"); + await job.checkpoint({ + ...job.checkpointResult, + _configurationMaintenance: { ...retained, inspection }, + }); + return; + } + if ( + !owner.transactionActive || + owner.transactionMode !== "write" || + owner.transactionContext !== transaction + ) + throw Error("Actual original owned write inspection transaction required"); + assertAdministrativeConfigurationRead(owner.transactionContext); + const [row] = await owner.connection + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined, + actual = result?._configurationMaintenance as Record | undefined, + attempt = inspection.attempt; + if ( + !result || + !actual || + actual.maintenanceId !== retained.maintenanceId || + actual.candidateId !== retained.candidateId || + actual.prepareOperationKey !== retained.prepareOperationKey || + actual.prepareOperationFingerprint !== retained.prepareOperationFingerprint || + canonicalJson(actual.operation) !== canonicalJson(retained.operation) || + attempt.apiJobId !== job.jobId || + attempt.apiOwnerVersion !== job.execution?.version || + canonicalJson(result._parentExecution) !== + canonicalJson({ + operationKey: attempt.parentOperationKey, + fingerprint: attempt.parentFingerprint, + deadlineAt: attempt.parentDeadlineAt, + }) + ) + throw Error("Original owned maintenance inspection checkpoint required"); + const rows = await owner.connection + .update(schema.adminJob) + .set({ result: { ...result, _configurationMaintenance: { ...actual, inspection } } }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: schema.adminJob.id }); + if (rows.length !== 1) throw Error("Original maintenance inspection ownership changed"); +} +/** Each inspection is a bounded authenticated read of the ORIGINAL owner. It + * never reuses a cached sync result, renews the seal/root or authorizes effects. */ +export async function inspectConfigurationMaintenance( + identity: Pick, +): Promise { + const owner = administrativeConfigurationState(currentAdministrativeConfigurationWriter()), + { job } = owner, + transaction = owner.transactionActive ? owner.transactionContext : null, + retained = job.checkpointResult?._configurationMaintenance as + | Record + | undefined, + prepared = configurationMaintenancePrepareSchema.parse(retained?.operation); + if (transaction === undefined) + throw Error("Actual original maintenance inspection scope required"); + if ( + !job.execution || + prepared.maintenanceId !== identity.maintenanceId || + prepared.candidateId !== identity.candidateId || + typeof retained?.prepareOperationKey !== "string" || + retained.prepareOperationFingerprint !== opsOperationFingerprint(prepared) + ) + throw Error("Original retained maintenance inspection required"); + const parent = z + .strictObject({ + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + }) + .parse(job.checkpointResult?._parentExecution), + operation = { + kind: "configurationMaintenance.inspect" as const, + maintenanceId: prepared.maintenanceId, + candidateId: prepared.candidateId, + }, + key = createDurableOpsKey( + "configuration-maintenance.inspect", + `${job.jobId}:${retained.prepareOperationKey}:${prepared.candidateId}`, + ), + attempt = prepareApiJobChildAttempt(job, operation, key, { + operationKey: key, + parent, + requestedDeadlineAt: Date.now() + 15000, + }); + if (!attempt) throw Error("Original live API inspection owner required"); + await checkpointMaintenanceInspection(retained, { attempt, state: "attempted" }, transaction); + const response = await createApiOpsClient().execute(operation, { + operationKey: key, + signal: AbortSignal.any([job.signal, AbortSignal.timeout(15000)]), + execution: { requestedDeadlineAt: attempt.requestedDeadlineAt }, + parentAssertion: await issueApiJobChildAssertion(attempt), + }); + if (response.execution !== "sync") throw Error("Original maintenance inspection unavailable"); + const receipt = configurationMaintenanceReceiptSchema.parse(response.value), + { kind: _kind, ...binding } = prepared; + if ( + JSON.stringify(receipt.binding) !== JSON.stringify(binding) || + receipt.prepare.operationKey !== retained.prepareOperationKey || + receipt.prepare.operationFingerprint !== retained.prepareOperationFingerprint || + receipt.apiJobId !== job.jobId || + receipt.parentOperationKey !== parent.operationKey || + receipt.parentFingerprint !== parent.fingerprint || + receipt.parentDeadlineAt !== parent.deadlineAt + ) + throw Error("Original maintenance inspection identity changed"); + const latest = job.checkpointResult?._configurationMaintenance as Record; + await checkpointMaintenanceInspection( + latest, + { attempt, state: "complete", receipt }, + transaction, + ); + // Only the fixed host inspector's full, released NoGuard proof can resolve a + // historical unknown preparation. Deliver its ORIGINAL canonical failure; + // neither that failure's classification nor its immutable bytes are changed. + if ( + receipt.state === "settled" && + receipt.dmFence === null && + receipt.decision === null && + receipt.generation === "not_started" && + receipt.followUp === "not_started" + ) { + if ( + administrativeConfigurationState(currentAdministrativeConfigurationWriter()).transactionActive + ) + throw new JobObservationPending("receipt_delivery"); + const projection = readProjection(job.checkpointResult), + projected = projection.operations[receipt.prepare.operationKey]; + if ( + !projected?.attempt || + projected.kind !== "configurationMaintenance.prepare" || + !projected.operation || + opsOperationFingerprint(projected.operation) !== receipt.prepare.operationFingerprint || + projected.operationId !== receipt.prepare.executionId || + projected.fingerprint !== receipt.prepare.fingerprint || + projected.deadlineAt !== receipt.prepare.deadlineAt || + projected.attempt.apiJobId !== receipt.apiJobId || + projected.attempt.parentOperationKey !== receipt.parentOperationKey || + projected.attempt.parentFingerprint !== receipt.parentFingerprint || + projected.attempt.parentDeadlineAt !== receipt.parentDeadlineAt + ) + throw Error("Original closed maintenance preparation required"); + const client = createApiOpsClient(), + status = await client.status(projected.kind, projected.operationId, { + signal: AbortSignal.timeout(15000), + }); + if ( + (status.state !== "failed" && status.state !== "timed_out") || + status.kind !== projected.kind || + status.operationId !== projected.operationId || + status.operationKey !== projected.operationKey || + status.fingerprint !== projected.fingerprint || + status.deadlineAt !== projected.deadlineAt + ) + throw Error("Original closed maintenance canonical failure required"); + projected.phase = "terminal"; + projected.terminal = true; + projected.errorClass = status.errorClass; + projected.failureResult = status.failureResult; + delete projected.outcome; + await ensureAdminJobReceiptDelivery(job, projection, projected, client, status); + if (projected.receiptDelivery?.state !== "confirmed") + throw Error("Original closed maintenance receipt delivery unavailable"); + } + return receipt; +} + +const configurationMaintenanceProtocol = Object.freeze({ + prepareConfigurationMaintenanceAbandon, + decideConfigurationMaintenance, + executeConfigurationMaintenanceStep, + settleConfigurationMaintenance, + prepareConfigurationMaintenance, + inspectConfigurationMaintenance, +}); diff --git a/apps/api/src/services/secrets.ts b/apps/api/src/services/secrets.ts index 54e2e3d5b..2d9fb0e9b 100644 --- a/apps/api/src/services/secrets.ts +++ b/apps/api/src/services/secrets.ts @@ -2,6 +2,10 @@ import { createCipheriv, createDecipheriv, randomBytes, randomUUID } from "node: import { and, eq } from "drizzle-orm"; import { db } from "../db"; import { integrationSecret } from "../db/schema"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./administrative-configuration-context"; const ALGORITHM = "aes-256-gcm" as const; @@ -157,3 +161,22 @@ export async function listAllSecrets(): Promise< }) .from(integrationSecret); } + +export async function resolveIntegrationVaultSecretsStrict( + integrationId: string, + context: AdministrativeConfigurationReadContext, +): Promise> { + try { + assertAdministrativeConfigurationRead(context); + const result: Record = {}; + for (const row of context.integrationSecretRows.filter( + (row) => row.integrationId === integrationId, + )) { + if (Object.hasOwn(result, row.key)) throw new Error("duplicate secret"); + result[row.key] = decrypt(row.ciphertext, row.iv, row.tag); + } + return result; + } catch { + throw new Error("Configuration acquisition unavailable"); + } +} diff --git a/apps/api/src/services/service-config-resolver.ts b/apps/api/src/services/service-config-resolver.ts index 4cc4e9552..8ed9d9910 100644 --- a/apps/api/src/services/service-config-resolver.ts +++ b/apps/api/src/services/service-config-resolver.ts @@ -24,6 +24,10 @@ import { services as coreServices } from "@openmapx/core/server"; import { eq } from "drizzle-orm"; import { db } from "../db"; import { serviceConfig } from "../db/schema"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./administrative-configuration-context"; const { configSchemaKeys, resolveServiceConfigFromEnv } = coreServices; @@ -173,3 +177,61 @@ export async function resolveAllServiceConfigs( ), }; } + +/** Authoritative producer variant. Failed acquisition never becomes absence. */ +export async function resolveServiceConfigWithSourcesStrict( + manifest: ResolveServiceConfigInput, + context: AdministrativeConfigurationReadContext, +): Promise> { + assertAdministrativeConfigurationRead(context); + const rows = context.serviceRows.filter((row) => row.serviceId === manifest.id); + if (rows.length > 1) throw new Error("Configuration acquisition unavailable"); + const known = new Set(configSchemaKeys(manifest.configSchema).map((field) => field.key)); + const config = rows[0]?.config ?? {}; + if ( + !config || + typeof config !== "object" || + Array.isArray(config) || + Object.keys(config).some((key) => !known.has(key)) + ) + throw new Error("Configuration acquisition unavailable"); + const envLayer = resolveServiceConfigFromEnv( + { + id: manifest.id, + configSchema: manifest.configSchema, + container: manifest.containerEnv ? { environment: manifest.containerEnv } : undefined, + }, + context.env, + ); + const result: Record = {}; + for (const [key, entry] of Object.entries(envLayer)) + if (entry.source === "default") result[key] = entry; + for (const [key, value] of Object.entries(config)) result[key] = { value, source: "database" }; + for (const [key, entry] of Object.entries(envLayer)) + if (entry.source === "env") result[key] = entry; + return result; +} + +export async function resolveAllServiceConfigsStrict( + manifests: ResolveServiceConfigInput[], + context: AdministrativeConfigurationReadContext, +): Promise { + assertAdministrativeConfigurationRead(context); + const ids = new Set(manifests.map((manifest) => manifest.id)); + if (context.serviceRows.some((row) => !ids.has(row.serviceId))) + throw new Error("Configuration acquisition unavailable"); + const values = new Map>(); + const envKeys = new Map(); + for (const manifest of manifests) { + const resolved = await resolveServiceConfigWithSourcesStrict(manifest, context); + const flat: Record = {}; + const fromEnv: string[] = []; + for (const [key, entry] of Object.entries(resolved)) { + if (entry.source === "env" && key !== manifest.proxyHostConfigKey) fromEnv.push(key); + else if (entry.source !== "default") flat[key] = entry.value; + } + values.set(manifest.id, flat); + if (fromEnv.length) envKeys.set(manifest.id, fromEnv); + } + return { values, envKeys }; +} diff --git a/apps/api/src/services/service-preparation.ts b/apps/api/src/services/service-preparation.ts new file mode 100644 index 000000000..f68dcfc92 --- /dev/null +++ b/apps/api/src/services/service-preparation.ts @@ -0,0 +1,228 @@ +import { + type CoordinatorMode, + canonicalJson, + type ServicePreparationResult, +} from "@openmapx/core/ops"; +import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; +import { + executeAdminJobOperation, + hasRetainedAdminJobOperation, + observeRetainedAdminJobOperation, + retainedAdminJobOperation, +} from "./admin-job-ops"; +import { + currentAdministrativeConfigurationWriter, + withAdministrativeConfigurationJob, + withAdministrativeConfigurationWriter, +} from "./administrative-configuration-writer"; +import { applyGenerationHardlinks } from "./configuration-hardlinks"; +import { + coordinatorLifecycleOperationKey, + prepareCoordinatorLifecycleIntent, +} from "./coordinator-lifecycle"; +import type { JobContext } from "./job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { + persistAndApplyProspectiveConfiguration, + prepareProspectiveConfiguration, + reconcileUnstagedCoordinatorPull, + resumeRetainedConfigurationCandidate, +} from "./prospective-configuration"; +import { + completeTrustedConfigurationApply, + parseTrustedConfigurationApplyOutcome, +} from "./trusted-config-outcome"; +import { applyTrustedConfiguration } from "./trusted-configuration-producer"; + +/** Fixed current queued DM producer. A nominal intent, unchanged full candidate + * and the existing maintenance persister own the entire continuation. */ +export async function applyCoordinatorService( + ctx: JobContext, + originalOrdinal: number | null, + expectedCurrentRevisionId?: string, +) { + const operationKey = coordinatorLifecycleOperationKey(ctx.jobId, originalOrdinal); + if (ctx.checkpointResult?._completedCoordinatorMaintenance && originalOrdinal !== null) { + const { completedCoordinatorMaintenance } = await import( + "./configuration-maintenance-step-records" + ); + const archived = completedCoordinatorMaintenance(ctx.checkpointResult); + if ( + archived.originalOrdinal !== originalOrdinal || + archived.receipt.apiJobId !== ctx.jobId || + archived.candidate.operationKey !== operationKey + ) + throw new JobObservationPending("invalid_identity"); + const observed = await observeRetainedAdminJobOperation(ctx, archived.settlementOperationKey); + if (canonicalJson(observed) !== canonicalJson(archived.receipt)) + throw new JobObservationPending("invalid_identity"); + return { changed: true as const, revisionId: archived.candidate.revisionId }; + } + return withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { actorId: ctx.jobId, operationKey, integrationIds: [], signal: ctx.signal }, + async () => { + await reconcileUnstagedCoordinatorPull(); + const checkpoint = ctx.checkpointResult?._configurationCandidate as + | Record + | undefined; + let application: unknown; + if (checkpoint?.operationKey === operationKey) { + const result = await resumeRetainedConfigurationCandidate(); + application = result.configuration; + } else { + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("configuration.coordinator-current", operationKey), + { signal: ctx.signal }, + ); + if ( + !current.revisionId || + (expectedCurrentRevisionId !== undefined && + current.revisionId !== expectedCurrentRevisionId) + ) + throw new AdminJobFailure("Coordinator current configuration changed", {}); + const intent = await prepareCoordinatorLifecycleIntent( + currentAdministrativeConfigurationWriter(), + originalOrdinal, + ), + candidate = await prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey, + expectedCurrentRevisionId: current.revisionId, + overlay: { purpose: "render", coordinatorMode: "ensure-running", changes: [] }, + coordinatorIntent: intent, + signal: ctx.signal, + }); + application = ( + await persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + signal: ctx.signal, + }) + ).application; + } + const applied = parseTrustedConfigurationApplyOutcome(application); + if ( + applied.outcome !== "committed" || + applied.followUp !== "complete" || + !completeTrustedConfigurationApply(applied) + ) + throw new AdminJobFailure("Coordinator configuration did not complete", { + configuration: application, + }); + return { changed: true as const, revisionId: applied.revisionId }; + }, + ), + ); +} + +export type { CoordinatorMode, ServicePreparationResult } from "@openmapx/core/ops"; + +/** Bulk and single Update carry the exact prior target's own final revision. */ +export async function updatePreparedService( + ctx: JobContext, + serviceId: string, + revisionId: string, + noDeps = false, +) { + const address = createDurableOpsKey("admin-job.service.update", `${ctx.jobId}:${serviceId}`); + if (hasRetainedAdminJobOperation(ctx, address)) { + const original = retainedAdminJobOperation(ctx, address); + if ( + original?.kind !== "service.update" || + original.serviceId !== serviceId || + original.expectedConfigurationRevisionId !== revisionId || + (original.noDeps ?? false) !== noDeps + ) + throw new JobObservationPending("invalid_identity"); + return executeAdminJobOperation(ctx, original, "admin-job.service.update", { + durableIdentity: serviceId, + }); + } + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("admin-job.update.inspect", `${ctx.jobId}:${serviceId}`), + { signal: ctx.signal }, + ); + if (current.revisionId !== revisionId) + throw new AdminJobFailure("Configuration changed before the next Update target", { + serviceLifecycle: { + expectedRevisionId: revisionId, + seedRevisionId: revisionId, + finalRevisionId: null, + phase: "failed", + commit: "seed_committed", + followUp: "not_started", + failure: { reason: "revision_conflict", failedServiceIds: [serviceId] }, + }, + }); + return executeAdminJobOperation( + ctx, + { + kind: "service.update", + serviceId, + expectedConfigurationRevisionId: revisionId, + ...(noDeps ? { noDeps } : {}), + }, + "admin-job.service.update", + { durableIdentity: serviceId }, + ); +} + +/** Publication is a seed/commit receipt, not proof of hardlinks or service application. */ +export async function publishServiceGeneration( + ctx: JobContext, + scope: string, + options: { coordinatorMode: CoordinatorMode; expectedCurrentRevisionId?: string }, +): Promise<{ revisionId: string }> { + return applyTrustedConfiguration({ + ctx, + kind: "stack.render", + coordinatorMode: options.coordinatorMode, + ...(Object.hasOwn(options, "expectedCurrentRevisionId") + ? { expectedCurrentRevisionId: options.expectedCurrentRevisionId! } + : {}), + operationKey: createDurableOpsKey("admin-job.config.prepare", `${ctx.jobId}:${scope}`), + signal: ctx.signal, + }); +} + +export { applyGenerationHardlinks } from "./configuration-hardlinks"; + +export async function prepareServiceGeneration( + ctx: JobContext, + scope: string, + options: { coordinatorMode: CoordinatorMode }, +): Promise { + const previous = ( + ctx.checkpointResult?._servicePreparations as + | Record + | undefined + )?.[scope]; + if (previous) { + if (previous.coordinatorMode !== options.coordinatorMode) + throw new JobObservationPending("invalid_identity"); + return previous; + } + await ctx.log("Preparing the full selected service generation..."); + const { revisionId } = await publishServiceGeneration(ctx, scope, options); + const hardlinks = await applyGenerationHardlinks({ + ctx, + revisionId, + operationIdentity: `${ctx.jobId}:${scope}`, + coordinatorMode: options.coordinatorMode, + signal: ctx.signal, + log: (message) => ctx.log(message), + }); + const result = { revisionId, hardlinks, coordinatorMode: options.coordinatorMode }; + await ctx.checkpoint({ + ...ctx.checkpointResult, + _servicePreparations: { + ...(ctx.checkpointResult?._servicePreparations as Record), + [scope]: result, + }, + }); + return result; +} diff --git a/apps/api/src/services/service-secrets.ts b/apps/api/src/services/service-secrets.ts index 56fe1b407..10b49a6f9 100644 --- a/apps/api/src/services/service-secrets.ts +++ b/apps/api/src/services/service-secrets.ts @@ -1,7 +1,12 @@ import { randomUUID } from "node:crypto"; +import { canonicalFeedCredentialMap } from "@openmapx/core/ops"; import { and, eq } from "drizzle-orm"; import { db } from "../db"; import { serviceSecret } from "../db/schema"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./administrative-configuration-context"; import { decrypt, encrypt, isSecretsConfigured } from "./secrets"; // Vault storage for self-hosted *service* secrets (the container track). Mirrors @@ -117,12 +122,26 @@ export async function resolveServiceVaultSecrets( /** Fail-closed variant used to publish an authoritative ops-agent snapshot. */ export async function resolveServiceVaultSecretsStrict( serviceId: string, + context?: AdministrativeConfigurationReadContext, ): Promise> { - if (!isSecretsConfigured()) return {}; - const rows = await db.select().from(serviceSecret).where(eq(serviceSecret.serviceId, serviceId)); - const result: Record = {}; - for (const row of rows) result[row.key] = decrypt(row.ciphertext, row.iv, row.tag); - return result; + try { + if (context) assertAdministrativeConfigurationRead(context); + const rows = context + ? context.serviceSecretRows.filter((row) => row.serviceId === serviceId) + : await db.select().from(serviceSecret).where(eq(serviceSecret.serviceId, serviceId)); + const result: Record = {}; + for (const row of rows) { + if (Object.hasOwn(result, row.key)) throw new Error("duplicate secret"); + const value = decrypt(row.ciphertext, row.iv, row.tag); + result[row.key] = + serviceId === "data-manager" && row.key === "TRANSITOUS_API_KEYS" + ? canonicalFeedCredentialMap(value) + : value; + } + return result; + } catch { + throw new Error("Configuration acquisition unavailable"); + } } /** All service secrets across every service — used to render the secret files. */ diff --git a/apps/api/src/services/service-update-batch.ts b/apps/api/src/services/service-update-batch.ts new file mode 100644 index 000000000..653a42024 --- /dev/null +++ b/apps/api/src/services/service-update-batch.ts @@ -0,0 +1,243 @@ +/** Durable ordering for the existing queued Update producer. These records are + * correlation data; every completed stage still reads its original agent ACK. */ +import { canonicalJson } from "@openmapx/core/ops"; +import { z } from "zod"; +import { db } from "../db"; +import { adminJob } from "../db/schema"; +import { JobObservationPending } from "./admin-job-failure"; +import { jobOwnership } from "./admin-job-state"; +import type { JobContext } from "./job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; + +const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +export const serviceUpdateBatchSchema = z.strictObject({ + version: z.literal(1), + serviceIds: z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256), + noDeps: z.boolean(), + initialRevisionId: revision, +}); +export function parseServiceUpdateBatch(payload: unknown, result: unknown) { + const original = payload as { action?: unknown; serviceIds?: unknown; noDeps?: unknown }; + const batch = serviceUpdateBatchSchema.parse( + (result as Record | null)?._serviceUpdateBatch, + ); + if ( + original.action !== "update" || + canonicalJson(original.serviceIds) !== canonicalJson(batch.serviceIds) || + batch.noDeps !== (original.noDeps === true) || + new Set(batch.serviceIds).size !== batch.serviceIds.length + ) + throw new JobObservationPending("invalid_identity"); + return batch; +} +export async function prepareServiceUpdateBatch(ctx: JobContext) { + if (!ctx.execution || ctx.observationOnly || ctx.jobType !== "service.bulk") + throw new JobObservationPending("recovery_required"); + if (ctx.checkpointResult?._serviceUpdateBatch) + return parseServiceUpdateBatch(ctx.payload, ctx.checkpointResult); + const observed = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("admin-job.bulk-update-current", ctx.jobId), + { signal: ctx.signal }, + ); + const original = serviceUpdateBatchSchema.parse({ + version: 1, + serviceIds: ctx.payload.serviceIds, + noDeps: ctx.payload.noDeps === true, + initialRevisionId: observed.revisionId, + }); + let retained: Record | undefined; + const batch = await db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(adminJob) + .where(jobOwnership(ctx.jobId, ctx.execution)) + .for("update"); + if ( + !row || + row.type !== ctx.jobType || + canonicalJson(row.payload) !== canonicalJson(ctx.payload) + ) + throw new JobObservationPending("invalid_identity"); + const result = (row.result ?? {}) as Record; + if (result._serviceUpdateBatch) { + retained = result; + return parseServiceUpdateBatch(row.payload, result); + } + // An older partially attempted batch has no trustworthy original revision. + // Never infer that input from the current pointer on recovery. + if (result.opsProjection || result._configurationCandidate) + throw new JobObservationPending("recovery_required"); + retained = { ...result, _serviceUpdateBatch: original }; + const rows = await tx + .update(adminJob) + .set({ result: retained }) + .where(jobOwnership(ctx.jobId, ctx.execution)) + .returning({ id: adminJob.id }); + if (rows.length !== 1) throw new JobObservationPending("recovery_required"); + return original; + }); + ctx.checkpointResult = retained; + return batch; +} + +/** Hand off only a completed configuration stage in THIS fixed batch. No + * operation/attempt/ACK is removed, and the single DM graph stays observable. */ +export async function advanceCompletedBulkCandidate( + writer: import("./administrative-configuration-writer-owner").AdministrativeConfigurationWriter, +) { + const { administrativeConfigurationState, withConfigurationWriteTransaction } = await import( + "./administrative-configuration-writer-owner" + ); + const { completedCoordinatorMaintenance } = await import( + "./configuration-maintenance-step-records" + ); + const { hasPendingReceiptDelivery, hasUnsettledManagedOperation } = await import( + "./admin-job-state" + ); + const { completeTrustedConfigurationApply, parseTrustedConfigurationApplyOutcome } = await import( + "./trusted-config-outcome" + ); + const { coordinatorLifecycleOperationKey } = await import("./coordinator-lifecycle"); + const { job } = administrativeConfigurationState(writer); + let committed: Record | undefined; + await withConfigurationWriteTransaction(writer, async (tx) => { + const [row] = await tx + .select() + .from(adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"); + if ( + !row || + row.type !== "service.bulk" || + canonicalJson(row.payload) !== canonicalJson(job.payload) + ) + throw new JobObservationPending("invalid_identity"); + const result = (row.result ?? {}) as Record, + batch = parseServiceUpdateBatch(row.payload, result), + previous = result._configurationCandidate as Record | undefined; + if (!previous || previous.operationKey === writer.operationKey) return; + const ordinal = batch.serviceIds.indexOf("data-manager"), + seedKey = createDurableOpsKey("admin-job.config.prepare", `${job.jobId}:bulk-update`), + coordinatorKey = coordinatorLifecycleOperationKey(job.jobId, ordinal); + if ( + ordinal < 0 || + !( + (previous.operationKey === seedKey && writer.operationKey === coordinatorKey) || + (previous.operationKey === coordinatorKey && writer.operationKey === seedKey) + ) || + previous.phase !== "settled" || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) + ) + throw new JobObservationPending("recovery_required"); + const outcome = parseTrustedConfigurationApplyOutcome( + (result._configurationOutcomes as Record)?.[String(previous.operationKey)], + ); + if ( + !completeTrustedConfigurationApply(outcome) || + outcome.outcome !== "committed" || + outcome.revisionId !== previous.revisionId + ) + throw new JobObservationPending("recovery_required"); + if (previous.maintenanceId) { + const archived = completedCoordinatorMaintenance(result); + if ( + archived.receipt.apiJobId !== job.jobId || + archived.originalOrdinal !== ordinal || + archived.candidate.candidateId !== previous.candidateId + ) + throw new JobObservationPending("recovery_required"); + } else if (result._configurationMaintenance) + throw new JobObservationPending("recovery_required"); + committed = { ...result }; + for (const field of [ + "_configurationCandidate", + "_configurationMaintenance", + "_configurationMaintenanceDecision", + "_configurationMaintenanceSteps", + "_configurationMaintenanceSettle", + "_configurationMaintenanceRollback", + "_configurationCoordinatorPrerequisite", + "_completion", + ]) + delete committed[field]; + const saved = await tx + .update(adminJob) + .set({ result: committed }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + }); + if (committed) job.checkpointResult = committed; +} + +/** Validate the exact successful prefix before the unique DM ordinal. No + * completed flag/current pointer can replace these original body/result/ACKs. */ +export async function completedServiceUpdatePrefix( + jobId: string, + payload: unknown, + result: Record, + ordinal: number, +) { + const batch = parseServiceUpdateBatch(payload, result), + { readProjection } = await import("./admin-job-projection"), + projection = readProjection(result), + { configurationApplyResultSchema, parseOpsResult } = await import("@openmapx/core/ops"), + keys = new Set(); + if ( + batch.serviceIds[ordinal] !== "data-manager" || + batch.serviceIds.indexOf("data-manager") !== ordinal + ) + throw new JobObservationPending("invalid_identity"); + let working = batch.initialRevisionId, + seed = false; + const original = (address: string) => { + const entry = projection.operations[projection.steps[address]?.operationKey ?? address]; + if ( + !entry?.terminal || + entry.phase !== "terminal" || + entry.errorClass || + entry.attempt?.apiJobId !== jobId || + entry.receiptDelivery?.state !== "confirmed" || + entry.receiptDelivery.terminalReceipt.state !== "succeeded" + ) + throw new JobObservationPending("recovery_required"); + keys.add(entry.operationKey); + return entry; + }; + for (const serviceId of batch.serviceIds.slice(0, ordinal)) { + if (!seed) { + const address = createDurableOpsKey("admin-job.config.prepare", `${jobId}:bulk-update`), + entry = original(address), + outcome = configurationApplyResultSchema.parse(entry.result); + if ( + entry.operation?.kind !== "stack.render" || + entry.operation.expectedCurrentRevisionId !== working || + outcome.outcome !== "committed" || + outcome.followUp !== "complete" || + outcome.cleanupFailed || + (result._configurationOutcomes as Record)?.[address] === undefined + ) + throw new JobObservationPending("invalid_identity"); + working = outcome.revisionId; + seed = true; + } + const entry = original( + createDurableOpsKey("admin-job.service.update", `${jobId}:${serviceId}`), + ); + if ( + entry.operation?.kind !== "service.update" || + entry.operation.serviceId !== serviceId || + entry.operation.expectedConfigurationRevisionId !== working || + (entry.operation.noDeps ?? false) !== batch.noDeps + ) + throw new JobObservationPending("invalid_identity"); + working = parseOpsResult("service.update", entry.result).revisionId; + } + return { operationKeys: keys, revisionId: working }; +} diff --git a/apps/api/src/services/trusted-config-operations.ts b/apps/api/src/services/trusted-config-operations.ts index 7803c9b17..6d2d4544b 100644 --- a/apps/api/src/services/trusted-config-operations.ts +++ b/apps/api/src/services/trusted-config-operations.ts @@ -1,364 +1,21 @@ +/** Existing public producer entry. Optional-context calls become retained + * immediate jobs; owned handlers call the same strict private implementation. */ import { - configurationApplyResultSchema, - configurationGenerationRevisionSchema, - type OpsOperation, - type OpsResultFor, - readOpsTokenFile, - type TrustedConfigurationPayload, - trustedConfigurationPayloadSchema, -} from "@openmapx/core/ops"; -import { envString } from "@openmapx/core/server-env"; -import { db } from "../db"; -import { integrationConfig } from "../db/schema"; -import { getAllIntegrations } from "../integration-host"; -import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; -import { - executeAdminJobOperation, - hasRetainedAdminJobOperation, - observeRetainedAdminJobOperation, - prepareAdminJobOperationRoot, -} from "./admin-job-ops"; -import { projectIntegrationConfig } from "./integration-config-projection"; -import type { JobContext } from "./job-runner"; -import { - createApiOpsClient, - createDurableOpsKey, - executeAndWait, - executeTrustedConfigurationAndWait, -} from "./ops-client"; -import { resolveAllServiceConfigs } from "./service-config-resolver"; -import { getServiceRegistry } from "./service-registry"; -import { resolveServiceVaultSecretsStrict } from "./service-secrets"; -import { - TrustedConfigurationApplyError, - type TrustedConfigurationApplyOutcome, - TrustedConfigurationUpgradeRequired, -} from "./trusted-config-outcome"; -import { - consumePublishedTrustedConfiguration, - publishTrustedConfigurationSnapshot, -} from "./trusted-config-publisher"; - -type ConfigurationKind = - | "stack.render" - | "serviceSelection.apply" - | "serviceConfig.apply" - | "integrationConfig.apply" - | "vault.apply"; - -export interface ApplyTrustedConfigurationOptions { - kind: ConfigurationKind; - operationKey: string; - serviceId?: string; - integrationId?: string; - selectedRoots?: string[]; - signal?: AbortSignal; - ctx?: JobContext; -} - -function exactInteger(env: NodeJS.ProcessEnv, name: string): number { - const value = env[name]; - if (!value || !/^(?:0|[1-9][0-9]{0,9})$/.test(value)) - throw new Error("Trusted configuration unavailable"); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) throw new Error("Trusted configuration unavailable"); - return parsed; -} - -/** - * Everything a render needs from the database, for every registered service: - * a generation keeps what was saved for services that are not enabled, so - * enabling one later, from here or the CLI, renders it with its settings. - */ -export async function buildConfigurationInput(): Promise { - const services = getServiceRegistry().list(); - const serviceConfigMap = await resolveAllServiceConfigs( - services.map((service) => ({ - id: service.manifest.id, - configSchema: service.manifest.configSchema, - containerEnv: service.manifest.container.environment, - proxyHostConfigKey: service.manifest.exposure?.proxy?.host?.configKey, - })), - ); - const serviceSecrets = await Promise.all( - services.map(async (service) => ({ - serviceId: service.manifest.id, - values: await resolveServiceVaultSecretsStrict(service.manifest.id), - })), - ); - const integrations = getAllIntegrations(); - const configured = await db.select().from(integrationConfig); - const configuredById = new Map( - configured.map((entry) => [entry.integrationId, entry.config as Record]), - ); - return trustedConfigurationPayloadSchema.parse({ - domain: envString("DOMAIN", "localhost"), - serviceConfigs: services.map((service) => { - const envKeys = serviceConfigMap.envKeys.get(service.manifest.id); - return { - serviceId: service.manifest.id, - values: serviceConfigMap.values.get(service.manifest.id) ?? {}, - ...(envKeys ? { envKeys } : {}), - }; - }), - integrationConfigs: integrations.map((integration) => ({ - integrationId: integration.id, - values: projectIntegrationConfig( - integration.manifest.configSchema as Record | undefined, - configuredById.get(integration.id) ?? {}, - ), - })), - serviceSecrets, - }); -} - -/** - * Hand the database's configuration to the ops-agent, which renders it with - * the operator's selection (or `selectedRoots`, which become it) and makes it - * the applied generation. The registry follows what was applied. - */ -export async function applyTrustedConfiguration( - options: ApplyTrustedConfigurationOptions, -): Promise<{ revisionId: string; enabledServiceIds: string[] }> { - const { outcome } = await applyTrustedConfigurationWithEvidence(options); - if (options.ctx && outcome.outcome === "unknown") throw new JobObservationPending("transport"); - if ( - outcome.outcome !== "committed" || - outcome.followUp !== "complete" || - outcome.cleanupFailed || - outcome.publisherCleanupFailed || - outcome.registryUpdateFailed - ) - if (options.ctx) - throw new AdminJobFailure("Configuration did not complete", { configuration: outcome }); - else throw new TrustedConfigurationApplyError(outcome); - return { revisionId: outcome.revisionId, enabledServiceIds: outcome.enabledServiceIds }; -} + type ApplyTrustedConfigurationOptions, + applyTrustedConfigurationWithEvidence as applyOwned, + completeTrustedConfigurationApplication, +} from "./trusted-configuration-producer"; -/** Internal evidence for scoped compensation. Payload may contain secrets; never serialize it. */ +export * from "./trusted-configuration-producer"; export async function applyTrustedConfigurationWithEvidence( options: ApplyTrustedConfigurationOptions, -): Promise<{ - payload?: TrustedConfigurationPayload; - previousApplied?: OpsResultFor<"configuration.inspect">; - outcome: TrustedConfigurationApplyOutcome; -}> { - if (options.ctx && hasRetainedAdminJobOperation(options.ctx, options.operationKey)) { - const outcome = configurationApplyResultSchema.parse( - await observeRetainedAdminJobOperation(options.ctx, options.operationKey), - ); - const retained = ( - options.ctx.checkpointResult?._configurationOutcomes as - | Record - | undefined - )?.[options.operationKey]; - if (!retained) { - await options.ctx.checkpoint({ ...options.ctx.checkpointResult, configuration: outcome }); - throw new JobObservationPending("recovery_required"); - } - return { outcome: retained }; - } - if (!options.ctx) throw new TrustedConfigurationUpgradeRequired(); - const preparedRoot = prepareAdminJobOperationRoot( - options.ctx, - options.kind, - options.operationKey, - ); - let payload: TrustedConfigurationPayload; - let previousApplied: OpsResultFor<"configuration.inspect">; - try { - if ((options.kind === "serviceSelection.apply") !== (options.selectedRoots !== undefined)) { - throw new Error("Trusted configuration unavailable"); - } - previousApplied = await executeAndWait( - createApiOpsClient(), - { kind: "configuration.inspect" }, - createDurableOpsKey("configuration.inspect", options.operationKey), - { signal: options.signal }, - ); - payload = trustedConfigurationPayloadSchema.parse({ - ...(await buildConfigurationInput()), - ...(options.selectedRoots ? { selectedRoots: options.selectedRoots } : {}), - }); - } catch { - return { - outcome: { - outcome: "not_committed", - reason: "failed", - cleanupFailed: false, - publisherCleanupFailed: false, - registryUpdateFailed: false, - }, - }; - } - return { - payload, - previousApplied, - outcome: await applySnapshot(options, payload, previousApplied.revisionId, preparedRoot), - }; -} - -/** Full-snapshot compensation; compare the applied generation, NOT the manifest authority. */ -export async function applyTrustedConfigurationSnapshot(options: { - payload: TrustedConfigurationPayload; - expectedCurrentRevisionId: string | null; - operationKey: string; - signal?: AbortSignal; - ctx?: JobContext; -}): Promise { - if ( - !configurationGenerationRevisionSchema.nullable().safeParse(options.expectedCurrentRevisionId) - .success - ) { - return { - outcome: "not_committed", - reason: "failed", - cleanupFailed: false, - publisherCleanupFailed: false, - registryUpdateFailed: false, - }; - } - return applySnapshot( - { - kind: "stack.render", - operationKey: options.operationKey, - signal: options.signal, - ctx: options.ctx, - }, - options.payload, - options.expectedCurrentRevisionId, - ); -} - -async function applySnapshot( - options: ApplyTrustedConfigurationOptions, - input: TrustedConfigurationPayload, - expectedCurrentRevisionId: string | null, - rootCandidate?: import("./admin-job-parent").ApiPreparedRoot, -): Promise { - if (!options.ctx) throw new TrustedConfigurationUpgradeRequired(); - if (hasRetainedAdminJobOperation(options.ctx, options.operationKey)) { - await observeRetainedAdminJobOperation(options.ctx, options.operationKey); - const outcome = ( - options.ctx.checkpointResult?._configurationOutcomes as - | Record - | undefined - )?.[options.operationKey]; - if (!outcome) throw new JobObservationPending("recovery_required"); - return outcome; - } - const preparedRoot = - rootCandidate ?? prepareAdminJobOperationRoot(options.ctx, options.kind, options.operationKey); - let submitted = false; - try { - const payload = trustedConfigurationPayloadSchema.parse(input); - const directory = process.env.OPS_TRUSTED_CONFIG_DIR; - const tokenFile = process.env.OPS_AGENT_TOKEN_FILE; - if (!directory || !tokenFile) throw new Error("Trusted configuration unavailable"); - const token = await readOpsTokenFile(tokenFile); - const operationForRevision = (revisionId: string): OpsOperation => { - switch (options.kind) { - case "stack.render": - return { - kind: options.kind, - revisionId, - expectedCurrentRevisionId, - }; - case "serviceSelection.apply": - return { kind: options.kind, revisionId, expectedCurrentRevisionId }; - case "serviceConfig.apply": - case "vault.apply": - if (!options.serviceId) throw new Error("Trusted configuration unavailable"); - return { - kind: options.kind, - serviceId: options.serviceId, - revisionId, - expectedCurrentRevisionId, - }; - case "integrationConfig.apply": - if (!options.integrationId) throw new Error("Trusted configuration unavailable"); - return { - kind: options.kind, - integrationId: options.integrationId, - revisionId, - expectedCurrentRevisionId, - }; - } - }; - const sealed = await publishTrustedConfigurationSnapshot({ - directory, - token, - ownerUid: exactInteger(process.env, "OPS_TRUSTED_CONFIG_UID"), - ownerGid: exactInteger(process.env, "OPS_TRUSTED_CONFIG_GID"), - operationKey: preparedRoot.operationKey, - operationForRevision, - payload, - }); - const consumed = await consumePublishedTrustedConfiguration(sealed, () => { - const client = createApiOpsClient(); - submitted = true; - if (options.ctx) - return executeAdminJobOperation(options.ctx, sealed.operation, "admin-job.configuration", { - operationKey: options.operationKey, - preparedRoot, - }).then((result) => configurationApplyResultSchema.parse(result)); - return executeTrustedConfigurationAndWait( - client, - sealed.operation as Extract, - options.operationKey, - { signal: options.signal }, - ); - }); - if (options.ctx && consumed.submission.status === "rejected") throw consumed.submission.reason; - const result = - consumed.submission.status === "fulfilled" - ? consumed.submission.value - : submitted - ? { outcome: "unknown" as const, reason: "transport_or_result" as const } - : { outcome: "not_committed" as const, reason: "failed" as const, cleanupFailed: false }; - let registryUpdateFailed = false; - if (result.outcome === "committed") { - try { - getServiceRegistry().applyEnabledIds(new Set(result.enabledServiceIds)); - } catch { - registryUpdateFailed = true; - } - } - const outcome = { - ...result, - publisherCleanupFailed: consumed.cleanupFailed, - registryUpdateFailed, - }; - if (options.ctx) - await options.ctx.checkpoint({ - ...options.ctx.checkpointResult, - configuration: outcome, - _configurationOutcomes: { - ...((options.ctx.checkpointResult?._configurationOutcomes as Record) ?? - {}), - [options.operationKey]: outcome, - }, - }); - return outcome; - } catch (error) { - if (options.ctx && submitted) throw error; - // Publication/preparation failed before execute, or an unexpected error after submission. - return { - ...(submitted - ? { outcome: "unknown" as const, reason: "transport_or_result" as const } - : { outcome: "not_committed" as const, reason: "failed" as const, cleanupFailed: false }), - publisherCleanupFailed: false, - registryUpdateFailed: false, - }; - } -} - -/** The config schema of every loaded integration, by id. */ -export function integrationSchemas(): Map> { - return new Map( - getAllIntegrations().map((integration) => [ - integration.id, - (integration.manifest.configSchema ?? {}) as Record, - ]), +): ReturnType { + if (options.ctx) return applyOwned(options); + return (await import("./job-runner")).jobRunner.applyImmediateConfiguration(options); +} +export async function applyTrustedConfiguration(options: ApplyTrustedConfigurationOptions) { + return completeTrustedConfigurationApplication( + options, + await applyTrustedConfigurationWithEvidence(options), ); } diff --git a/apps/api/src/services/trusted-config-outcome.ts b/apps/api/src/services/trusted-config-outcome.ts index 018fa066c..873e3afe8 100644 --- a/apps/api/src/services/trusted-config-outcome.ts +++ b/apps/api/src/services/trusted-config-outcome.ts @@ -1,9 +1,49 @@ -import type { ConfigurationApplyResult } from "@openmapx/core/ops"; +import { type ConfigurationApplyResult, configurationApplyResultSchema } from "@openmapx/core/ops"; export type TrustedConfigurationApplyOutcome = ( | ConfigurationApplyResult | { outcome: "unknown"; reason: "transport_or_result" } ) & { publisherCleanupFailed: boolean; registryUpdateFailed: boolean }; +export function completeTrustedConfigurationApply( + outcome: TrustedConfigurationApplyOutcome, +): boolean { + return ( + outcome.outcome === "committed" && + outcome.followUp === "complete" && + !outcome.cleanupFailed && + !outcome.publisherCleanupFailed && + !outcome.registryUpdateFailed + ); +} + +/** Decode the exact retained API outcome, including the API-owned cleanup evidence. */ +export function parseTrustedConfigurationApplyOutcome( + value: unknown, +): TrustedConfigurationApplyOutcome { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("Configuration outcome unavailable"); + const { publisherCleanupFailed, registryUpdateFailed, ...application } = value as Record< + string, + unknown + >; + if (typeof publisherCleanupFailed !== "boolean" || typeof registryUpdateFailed !== "boolean") + throw new Error("Configuration outcome unavailable"); + if (application.outcome === "unknown") { + if (application.reason !== "transport_or_result" || Object.keys(application).length !== 2) + throw new Error("Configuration outcome unavailable"); + return { + outcome: "unknown", + reason: "transport_or_result", + publisherCleanupFailed, + registryUpdateFailed, + }; + } + return { + ...configurationApplyResultSchema.parse(application), + publisherCleanupFailed, + registryUpdateFailed, + }; +} export class TrustedConfigurationApplyError extends Error { constructor(readonly outcome: TrustedConfigurationApplyOutcome) { diff --git a/apps/api/src/services/trusted-config-publication-store.ts b/apps/api/src/services/trusted-config-publication-store.ts new file mode 100644 index 000000000..007edffe7 --- /dev/null +++ b/apps/api/src/services/trusted-config-publication-store.ts @@ -0,0 +1,667 @@ +import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, readdir, unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { + acquireTrustedConfigurationQueueLock, + encodePreparedConfigurationRecord, + inspectPreparedConfigurationRecord, + inspectTrustedConfigurationSnapshot, + OPS_TRUSTED_CONFIG_MAX_BYTES, + OPS_TRUSTED_CONFIG_QUEUE_LOCK_NAME, + type PreparedConfigurationRecord, + type SealTrustedConfigurationSnapshotOptions, + sealTrustedConfigurationSnapshot, + type TrustedConfigurationPayload, + trustedConfigurationQueueFits, +} from "@openmapx/core/ops"; + +const DIRECTORY_MODE = 0o700; +const SNAPSHOT_MODE = 0o600; +const REJECTED = "Trusted configuration snapshot publish failed"; +const MAX_SCAN_ENTRIES = 1_024; +const MAX_SCAN_BYTES = 512 * 1024 * 1024; +/** Fixed deployment source, never a caller-selected candidate path/owner. */ +export function configurationPublisherFromEnvironment(token: string) { + const number = (name: string) => { + const value = process.env[name]; + if (!value || !/^(0|[1-9][0-9]{0,9})$/.test(value) || !Number.isSafeInteger(Number(value))) + throw Error("Configuration candidate unavailable"); + return Number(value); + }; + const directory = process.env.OPS_TRUSTED_CONFIG_DIR; + if (!directory) throw Error("Configuration candidate unavailable"); + return { + directory, + token, + ownerUid: number("OPS_TRUSTED_CONFIG_UID"), + ownerGid: number("OPS_TRUSTED_CONFIG_GID"), + }; +} +const SNAPSHOT_NAME = /^cfg1_[A-Za-z0-9_-]{43}\.json$/; +const TEMPORARY_NAME = /^\.cfg1_[A-Za-z0-9_-]{43}\.nonce_[A-Za-z0-9_-]{16,96}\.tmp$/; +const ABORT_NAME = /^\.cfg1_[A-Za-z0-9_-]{43}\.[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.abort$/; +let publicationLock: Promise = Promise.resolve(); + +export interface SnapshotPublishDependencies { + beforeRename(): Promise; + beforeAbortRename(): Promise; + afterAbortLink(): Promise; + afterAbortLinkSync(): Promise; + afterAbortSourceUnlink(): Promise; + afterAbortSourceSync(): Promise; + afterAbortTombstoneUnlink(): Promise; + afterAbortTombstoneSync(): Promise; +} + +export interface PublishTrustedConfigurationSnapshotOptions { + directory: string; + token: string; + ownerUid: number; + ownerGid: number; + operationKey: string; + operationForRevision: SealTrustedConfigurationSnapshotOptions["operationForRevision"]; + payload: TrustedConfigurationPayload; + now?: () => number; + nonce?: () => string; +} + +async function syncDirectory(path: string): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_DIRECTORY); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function validateDirectory(path: string, ownerUid: number, ownerGid: number): Promise { + const stats = await lstat(path); + if ( + !stats.isDirectory() || + stats.isSymbolicLink() || + stats.gid !== ownerGid || + stats.uid !== ownerUid || + (stats.mode & 0o777) !== (DIRECTORY_MODE & 0o777) + ) { + throw new Error(REJECTED); + } +} + +async function withPublicationLock(run: () => Promise): Promise { + const previous = publicationLock; + let release!: () => void; + publicationLock = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await run(); + } finally { + release(); + } +} + +async function assertReadyCapacity( + directory: string, + ownerUid: number, + ownerGid: number, + token: string, + incomingBytes: number, + incomingRevision?: string, +): Promise { + let entries = 0; + let bytes = 0; + let scannedEntries = 0; + let scannedBytes = 0; + const revisions = new Set(); + const retainedRevisions = new Set(); + const inspectRetainedSnapshot = async ( + path: string, + expectedRevision: string, + ): Promise => { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await handle.stat(); + const current = await lstat(path); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.dev !== current.dev || + stats.ino !== current.ino || + stats.uid !== ownerUid || + stats.gid !== ownerGid || + (stats.mode & 0o777) !== SNAPSHOT_MODE || + stats.size > OPS_TRUSTED_CONFIG_MAX_BYTES + ) { + throw new Error(REJECTED); + } + const bytes = await handle.readFile(); + const inspected = inspectTrustedConfigurationSnapshot(bytes, { token }); + if (inspected.revisionId !== expectedRevision) throw new Error(REJECTED); + return stats.size; + } finally { + await handle.close(); + } + }; + + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (entry.name === OPS_TRUSTED_CONFIG_QUEUE_LOCK_NAME) continue; + if (entry.name === ".candidates") continue; + if (entry.name === ".claimed") { + await validateDirectory(join(directory, entry.name), ownerUid, ownerGid); + for (const claimedEntry of await readdir(join(directory, entry.name), { + withFileTypes: true, + })) { + if (!claimedEntry.isFile() || !SNAPSHOT_NAME.test(claimedEntry.name)) { + throw new Error(REJECTED); + } + const revisionId = claimedEntry.name.slice(0, -".json".length); + if (revisions.has(revisionId)) throw new Error(REJECTED); + revisions.add(revisionId); + const size = await inspectRetainedSnapshot( + join(directory, entry.name, claimedEntry.name), + revisionId, + ); + scannedEntries += 1; + scannedBytes += size; + entries += 1; + bytes += size; + retainedRevisions.add(revisionId); + } + continue; + } + if ( + !entry.isFile() || + (!SNAPSHOT_NAME.test(entry.name) && + !TEMPORARY_NAME.test(entry.name) && + !ABORT_NAME.test(entry.name)) + ) { + throw new Error(REJECTED); + } + const stats = await lstat(join(directory, entry.name)); + if ( + !stats.isFile() || + stats.isSymbolicLink() || + stats.nlink !== 1 || + stats.uid !== ownerUid || + stats.gid !== ownerGid || + (stats.mode & 0o777) !== SNAPSHOT_MODE || + stats.size > OPS_TRUSTED_CONFIG_MAX_BYTES + ) { + throw new Error(REJECTED); + } + scannedEntries += 1; + scannedBytes += stats.size; + if (scannedEntries > MAX_SCAN_ENTRIES || scannedBytes > MAX_SCAN_BYTES) { + throw new Error(REJECTED); + } + if (SNAPSHOT_NAME.test(entry.name)) { + const revisionId = entry.name.slice(0, -".json".length); + if (revisions.has(revisionId)) throw new Error(REJECTED); + revisions.add(revisionId); + const retainedSize = await inspectRetainedSnapshot(join(directory, entry.name), revisionId); + entries += 1; + bytes += retainedSize; + retainedRevisions.add(revisionId); + } else if (ABORT_NAME.test(entry.name)) { + const revisionId = entry.name.slice(1, 49); + if (revisions.has(revisionId)) throw new Error(REJECTED); + revisions.add(revisionId); + await inspectRetainedSnapshot(join(directory, entry.name), revisionId); + } + } + try { + const candidates = join(directory, ".candidates"); + await validateDirectory(candidates, ownerUid, ownerGid); + const names = await readdir(candidates, { withFileTypes: true }); + if (names.length > 128) throw new Error(REJECTED); + const candidateRevisions = new Set(); + for (const name of names) { + if (!name.isFile() || !/^opk1_[A-Za-z0-9_-]{16,64}\.json$/.test(name.name)) + throw new Error(REJECTED); + const handle = await open( + join(candidates, name.name), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stats = await handle.stat(), + current = await lstat(join(candidates, name.name)); + if ( + !stats.isFile() || + stats.nlink !== 1 || + stats.dev !== current.dev || + stats.ino !== current.ino || + stats.uid !== ownerUid || + stats.gid !== ownerGid || + (stats.mode & 0o777) !== SNAPSHOT_MODE || + stats.size > 768 * 1024 + ) + throw new Error(REJECTED); + const inspected = inspectPreparedConfigurationRecord(await handle.readFile(), token); + if ( + `${inspected.record.operationKey}.json` !== name.name || + candidateRevisions.has(inspected.snapshot.revisionId) + ) + throw new Error(REJECTED); + candidateRevisions.add(inspected.snapshot.revisionId); + scannedEntries++; + scannedBytes += stats.size; + if (retainedRevisions.has(inspected.snapshot.revisionId)) + bytes += stats.size - inspected.sealed.length; + else { + entries++; + bytes += stats.size; + retainedRevisions.add(inspected.snapshot.revisionId); + } + } finally { + await handle.close(); + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if ( + scannedEntries > MAX_SCAN_ENTRIES || + scannedBytes > MAX_SCAN_BYTES || + !trustedConfigurationQueueFits({ + retainedEntries: entries, + retainedBytes: bytes, + reservedEntries: incomingRevision && retainedRevisions.has(incomingRevision) ? 0 : 1, + reservedBytes: + incomingRevision && retainedRevisions.has(incomingRevision) ? 0 : incomingBytes, + }) + ) { + throw new Error(REJECTED); + } +} + +function contentDigest(bytes: Uint8Array): Buffer { + return createHash("sha256") + .update("openmapx-trusted-config-publisher-candidate-v1\0") + .update(bytes) + .digest(); +} + +export async function publishTrustedConfigurationSnapshot( + options: PublishTrustedConfigurationSnapshotOptions, + dependencies: Partial = {}, +) { + return publishSnapshot(options, dependencies); +} +async function publishSnapshot( + options: PublishTrustedConfigurationSnapshotOptions, + dependencies: Partial, + prepared?: ReturnType, +) { + return withPublicationLock(async () => { + let temporary: string | undefined; + let queueLock: Awaited> | undefined; + try { + await validateDirectory(options.directory, options.ownerUid, options.ownerGid); + const nowMs = options.now?.(); + queueLock = await acquireTrustedConfigurationQueueLock({ + directory: options.directory, + token: options.token, + ownerUid: options.ownerUid, + ownerGid: options.ownerGid, + participant: "api", + operationKey: options.operationKey, + }); + const nonce = options.nonce?.() ?? `nonce_${randomUUID().replaceAll("-", "")}`; + const sealed = + prepared ?? + sealTrustedConfigurationSnapshot({ + role: "api", + operationKey: options.operationKey, + operationForRevision: options.operationForRevision, + payload: options.payload, + token: options.token, + issuedAtMs: nowMs, + nonce, + }); + await assertReadyCapacity( + options.directory, + options.ownerUid, + options.ownerGid, + options.token, + sealed.bytes.byteLength, + sealed.revisionId, + ); + queueLock.assertHeld(); + const finalPath = join(options.directory, `${sealed.revisionId}.json`); + temporary = join(options.directory, `.${sealed.revisionId}.${nonce}.tmp`); + const handle = await open( + temporary, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, + SNAPSHOT_MODE, + ); + try { + await handle.writeFile(sealed.bytes); + await handle.chmod(SNAPSHOT_MODE); + await handle.chown(options.ownerUid, options.ownerGid); + await handle.sync(); + } finally { + await handle.close(); + } + await dependencies.beforeRename?.(); + queueLock.assertHeld(); + await link(temporary, finalPath); + await unlink(temporary); + temporary = undefined; + await syncDirectory(options.directory); + const publishedStats = await lstat(finalPath); + const expectedDigest = contentDigest(sealed.bytes); + const abortPath = join(options.directory, `.${sealed.revisionId}.${nonce}.abort`); + let aborted = false; + let abortInFlight: Promise | undefined; + const validateCandidate = async ( + path: string, + expectedLinks: number, + ): Promise<{ dev: number; ino: number }> => { + const candidateHandle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stats = await candidateHandle.stat(); + const current = await lstat(path); + const bytes = await candidateHandle.readFile(); + const actualDigest = contentDigest(bytes); + if ( + !stats.isFile() || + stats.nlink !== expectedLinks || + stats.dev !== current.dev || + stats.ino !== current.ino || + stats.dev !== publishedStats.dev || + stats.ino !== publishedStats.ino || + stats.uid !== options.ownerUid || + stats.gid !== options.ownerGid || + (stats.mode & 0o777) !== SNAPSHOT_MODE || + stats.size !== sealed.bytes.byteLength || + actualDigest.byteLength !== expectedDigest.byteLength || + !timingSafeEqual(actualDigest, expectedDigest) + ) { + throw new Error(REJECTED); + } + return { dev: stats.dev, ino: stats.ino }; + } finally { + await candidateHandle.close(); + } + }; + const pathExists = async (path: string): Promise => { + try { + await lstat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } + }; + const assertMissing = async (path: string): Promise => { + try { + await lstat(path); + throw new Error(REJECTED); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + }; + const runAbort = async (): Promise => { + await dependencies.beforeAbortRename?.(); + const finalExists = await pathExists(finalPath); + const tombstoneExists = await pathExists(abortPath); + if (!finalExists && !tombstoneExists) { + await syncDirectory(options.directory); + aborted = true; + return; + } + let final: { dev: number; ino: number } | undefined; + let tombstone: { dev: number; ino: number } | undefined; + if (finalExists && tombstoneExists) { + final = await validateCandidate(finalPath, 2); + tombstone = await validateCandidate(abortPath, 2); + if (final.dev !== tombstone.dev || final.ino !== tombstone.ino) { + throw new Error(REJECTED); + } + } else if (finalExists) { + final = await validateCandidate(finalPath, 1); + try { + await link(finalPath, abortPath); + } catch { + throw new Error(REJECTED); + } + await dependencies.afterAbortLink?.(); + final = await validateCandidate(finalPath, 2); + tombstone = await validateCandidate(abortPath, 2); + if (final.dev !== tombstone.dev || final.ino !== tombstone.ino) { + throw new Error(REJECTED); + } + await syncDirectory(options.directory); + await dependencies.afterAbortLinkSync?.(); + } else { + tombstone = await validateCandidate(abortPath, 1); + } + if (final) { + const currentFinal = await validateCandidate(finalPath, 2); + const currentTombstone = await validateCandidate(abortPath, 2); + if ( + currentFinal.dev !== currentTombstone.dev || + currentFinal.ino !== currentTombstone.ino + ) { + throw new Error(REJECTED); + } + await unlink(finalPath); + await dependencies.afterAbortSourceUnlink?.(); + await syncDirectory(options.directory); + await dependencies.afterAbortSourceSync?.(); + } + await assertMissing(finalPath); + await validateCandidate(abortPath, 1); + await unlink(abortPath); + await dependencies.afterAbortTombstoneUnlink?.(); + await syncDirectory(options.directory); + await dependencies.afterAbortTombstoneSync?.(); + aborted = true; + }; + const abort = async (): Promise => { + if (aborted) return; + if (abortInFlight) return abortInFlight; + const attempt = runAbort().catch(() => { + throw new Error(REJECTED); + }); + abortInFlight = attempt; + try { + await attempt; + } finally { + if (!aborted) abortInFlight = undefined; + } + }; + return { ...sealed, mode: SNAPSHOT_MODE, abort }; + } catch { + if (temporary) await unlink(temporary).catch(() => undefined); + throw new Error(REJECTED); + } finally { + try { + queueLock?.release(); + } catch { + // biome-ignore lint/correctness/noUnsafeFinally: lock interference must fail closed + throw new Error(REJECTED); + } + } + }); +} + +/** Inert durable reservation. No ready snapshot/apply claim exists at this stage. */ +export async function retainPreparedTrustedConfigurationCandidate( + options: Pick< + PublishTrustedConfigurationSnapshotOptions, + "directory" | "token" | "ownerUid" | "ownerGid" + >, + record: PreparedConfigurationRecord, +): Promise { + const bytes = encodePreparedConfigurationRecord(record, options.token); + inspectPreparedConfigurationRecord(bytes, options.token); + await withPublicationLock(async () => { + await validateDirectory(options.directory, options.ownerUid, options.ownerGid); + const queue = await acquireTrustedConfigurationQueueLock({ + ...options, + participant: "api", + operationKey: record.operationKey, + }); + try { + const directory = join(options.directory, ".candidates"); + await mkdir(directory, { mode: DIRECTORY_MODE }).catch((error) => { + if (error.code !== "EEXIST") throw error; + }); + await validateDirectory(directory, options.ownerUid, options.ownerGid); + await assertReadyCapacity( + options.directory, + options.ownerUid, + options.ownerGid, + options.token, + bytes.length, + ); + queue.assertHeld(); + const path = join(directory, `${record.operationKey}.json`); + const handle = await open( + path, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, + SNAPSHOT_MODE, + ); + try { + await handle.writeFile(bytes); + await handle.chmod(SNAPSHOT_MODE); + await handle.chown(options.ownerUid, options.ownerGid); + await handle.sync(); + } finally { + await handle.close(); + } + await syncDirectory(directory); + await syncDirectory(options.directory); + } finally { + queue.release(); + } + }); +} + +/** Historical read of the ORIGINAL private candidate. Authentication never + * renews its seal or creates a consumable claim. The recovering owned job must + * separately prove its exact checkpoint before using any returned metadata. */ +export async function readPreparedTrustedConfigurationCandidate( + options: Pick< + PublishTrustedConfigurationSnapshotOptions, + "directory" | "token" | "ownerUid" | "ownerGid" + >, + operationKey: string, +) { + if (!/^opk1_[A-Za-z0-9_-]{16,64}$/.test(operationKey)) throw Error(REJECTED); + return withPublicationLock(async () => { + await validateDirectory(options.directory, options.ownerUid, options.ownerGid); + const queue = await acquireTrustedConfigurationQueueLock({ + ...options, + participant: "api", + operationKey, + }); + try { + const directory = join(options.directory, ".candidates"); + await validateDirectory(directory, options.ownerUid, options.ownerGid); + const path = join(directory, `${operationKey}.json`), + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await handle.stat(), + first = await lstat(path); + if ( + !before.isFile() || + before.nlink !== 1 || + before.uid !== options.ownerUid || + before.gid !== options.ownerGid || + (before.mode & 0o777) !== SNAPSHOT_MODE || + before.size > 768 * 1024 || + before.dev !== first.dev || + before.ino !== first.ino + ) + throw Error(REJECTED); + const bytes = await handle.readFile(), + inspected = inspectPreparedConfigurationRecord(bytes, options.token), + after = await handle.stat(), + last = await lstat(path); + if ( + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + before.dev !== last.dev || + before.ino !== last.ino || + inspected.record.operationKey !== operationKey + ) + throw Error(REJECTED); + queue.assertHeld(); + return inspected; + } finally { + await handle.close(); + } + } finally { + queue.release(); + } + }); +} + +export async function consumePublishedTrustedConfiguration( + published: { abort(): Promise }, + submit: () => Promise, +): Promise<{ submission: PromiseSettledResult; cleanupFailed: boolean }> { + let submission: PromiseSettledResult; + try { + submission = { status: "fulfilled", value: await submit() }; + } catch (reason) { + submission = { status: "rejected", reason }; + } + let cleanupFailed = false; + try { + await published.abort(); + } catch { + cleanupFailed = true; + } + return { submission, cleanupFailed }; +} + +// Private I/O primitives: no candidate, admission or participant constructors. +export { + publishSnapshot, + REJECTED, + SNAPSHOT_MODE, + syncDirectory, + validateDirectory, + withPublicationLock, +}; + +/** Bounded historical lookup in the assigned API-private namespace. It never + * opens a consumable seal or supplies authority to continue the candidate. */ +export async function findPreparedTrustedConfigurationCandidate( + options: Pick< + PublishTrustedConfigurationSnapshotOptions, + "directory" | "token" | "ownerUid" | "ownerGid" + >, + candidateId: string, +) { + if (!/^[a-f0-9-]{36}$/.test(candidateId)) throw Error(REJECTED); + const directory = join(options.directory, ".candidates"); + try { + await validateDirectory(directory, options.ownerUid, options.ownerGid); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + const entries = await readdir(directory, { withFileTypes: true }); + if (entries.length > 128) throw Error(REJECTED); + let match: Awaited> | undefined; + let bytes = 0; + for (const entry of entries) { + if (!entry.isFile() || !/^opk1_[A-Za-z0-9_-]{16,64}\.json$/.test(entry.name)) + throw Error(REJECTED); + const value = await readPreparedTrustedConfigurationCandidate(options, entry.name.slice(0, -5)); + bytes += encodePreparedConfigurationRecord(value.record, options.token).length; + if (bytes > 16 * 1024 * 1024) throw Error(REJECTED); + if (value.record.candidateId === candidateId) { + if (match) throw Error(REJECTED); + match = value; + } + } + return match; +} diff --git a/apps/api/src/services/trusted-config-publisher.test.ts b/apps/api/src/services/trusted-config-publisher.test.ts index efef171fa..0d26dd2a0 100644 --- a/apps/api/src/services/trusted-config-publisher.test.ts +++ b/apps/api/src/services/trusted-config-publisher.test.ts @@ -60,6 +60,8 @@ function seedReadySnapshot( operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -86,6 +88,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -104,11 +108,15 @@ describe("trusted configuration snapshot publisher", () => { expect(published.mode).toBe(0o600); expect(published.operation).toEqual({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId: published.revisionId, }); expect(Object.keys(published.operation).sort()).toEqual([ + "coordinatorMode", "expectedCurrentRevisionId", + "expectedSourceRevision", "kind", "revisionId", ]); @@ -131,6 +139,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -159,6 +169,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -190,6 +202,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -222,6 +236,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -256,6 +272,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -315,6 +333,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -371,6 +391,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -419,6 +441,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -460,6 +484,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -498,6 +524,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -541,6 +569,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -579,6 +609,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -612,6 +644,8 @@ describe("trusted configuration snapshot publisher", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), diff --git a/apps/api/src/services/trusted-config-publisher.ts b/apps/api/src/services/trusted-config-publisher.ts index 4ef1802ab..5b81b6430 100644 --- a/apps/api/src/services/trusted-config-publisher.ts +++ b/apps/api/src/services/trusted-config-publisher.ts @@ -1,422 +1,51 @@ -import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; -import { constants } from "node:fs"; -import { link, lstat, open, readdir, unlink } from "node:fs/promises"; -import { join } from "node:path"; -import { - acquireTrustedConfigurationQueueLock, - inspectTrustedConfigurationSnapshot, - OPS_TRUSTED_CONFIG_MAX_BYTES, - OPS_TRUSTED_CONFIG_QUEUE_LOCK_NAME, - type SealTrustedConfigurationSnapshotOptions, - sealTrustedConfigurationSnapshot, - type TrustedConfigurationPayload, - trustedConfigurationQueueFits, -} from "@openmapx/core/ops"; - -const DIRECTORY_MODE = 0o700; -const SNAPSHOT_MODE = 0o600; -const REJECTED = "Trusted configuration snapshot publish failed"; -const MAX_SCAN_ENTRIES = 1_024; -const MAX_SCAN_BYTES = 512 * 1024 * 1024; -const SNAPSHOT_NAME = /^cfg1_[A-Za-z0-9_-]{43}\.json$/; -const TEMPORARY_NAME = /^\.cfg1_[A-Za-z0-9_-]{43}\.nonce_[A-Za-z0-9_-]{16,96}\.tmp$/; -const ABORT_NAME = /^\.cfg1_[A-Za-z0-9_-]{43}\.[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.abort$/; -let publicationLock: Promise = Promise.resolve(); - -export interface SnapshotPublishDependencies { - beforeRename(): Promise; - beforeAbortRename(): Promise; - afterAbortLink(): Promise; - afterAbortLinkSync(): Promise; - afterAbortSourceUnlink(): Promise; - afterAbortSourceSync(): Promise; - afterAbortTombstoneUnlink(): Promise; - afterAbortTombstoneSync(): Promise; -} - -export interface PublishTrustedConfigurationSnapshotOptions { - directory: string; - token: string; - ownerUid: number; - ownerGid: number; - operationKey: string; - operationForRevision: SealTrustedConfigurationSnapshotOptions["operationForRevision"]; - payload: TrustedConfigurationPayload; - now?: () => number; - nonce?: () => string; -} - -async function syncDirectory(path: string): Promise { - const handle = await open(path, constants.O_RDONLY | constants.O_DIRECTORY); - try { - await handle.sync(); - } finally { - await handle.close(); - } -} - -async function validateDirectory(path: string, ownerUid: number, ownerGid: number): Promise { - const stats = await lstat(path); - if ( - !stats.isDirectory() || - stats.isSymbolicLink() || - stats.gid !== ownerGid || - stats.uid !== ownerUid || - (stats.mode & 0o777) !== (DIRECTORY_MODE & 0o777) - ) { - throw new Error(REJECTED); - } -} - -async function withPublicationLock(run: () => Promise): Promise { - const previous = publicationLock; - let release!: () => void; - publicationLock = new Promise((resolve) => { - release = resolve; - }); - await previous; - try { - return await run(); - } finally { - release(); - } -} - -async function assertReadyCapacity( - directory: string, - ownerUid: number, - ownerGid: number, - token: string, - incomingBytes: number, -): Promise { - let entries = 0; - let bytes = 0; - let scannedEntries = 0; - let scannedBytes = 0; - const revisions = new Set(); - const inspectRetainedSnapshot = async ( - path: string, - expectedRevision: string, - ): Promise => { - const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const stats = await handle.stat(); - const current = await lstat(path); - if ( - !stats.isFile() || - stats.nlink !== 1 || - stats.dev !== current.dev || - stats.ino !== current.ino || - stats.uid !== ownerUid || - stats.gid !== ownerGid || - (stats.mode & 0o777) !== SNAPSHOT_MODE || - stats.size > OPS_TRUSTED_CONFIG_MAX_BYTES - ) { - throw new Error(REJECTED); - } - const bytes = await handle.readFile(); - const inspected = inspectTrustedConfigurationSnapshot(bytes, { token }); - if (inspected.revisionId !== expectedRevision) throw new Error(REJECTED); - return stats.size; - } finally { - await handle.close(); - } - }; - - for (const entry of await readdir(directory, { withFileTypes: true })) { - if (entry.name === OPS_TRUSTED_CONFIG_QUEUE_LOCK_NAME) continue; - if (entry.name === ".claimed") { - await validateDirectory(join(directory, entry.name), ownerUid, ownerGid); - for (const claimedEntry of await readdir(join(directory, entry.name), { - withFileTypes: true, - })) { - if (!claimedEntry.isFile() || !SNAPSHOT_NAME.test(claimedEntry.name)) { - throw new Error(REJECTED); - } - const revisionId = claimedEntry.name.slice(0, -".json".length); - if (revisions.has(revisionId)) throw new Error(REJECTED); - revisions.add(revisionId); - const size = await inspectRetainedSnapshot( - join(directory, entry.name, claimedEntry.name), - revisionId, - ); - scannedEntries += 1; - scannedBytes += size; - entries += 1; - bytes += size; - } - continue; - } - if ( - !entry.isFile() || - (!SNAPSHOT_NAME.test(entry.name) && - !TEMPORARY_NAME.test(entry.name) && - !ABORT_NAME.test(entry.name)) - ) { - throw new Error(REJECTED); - } - const stats = await lstat(join(directory, entry.name)); - if ( - !stats.isFile() || - stats.isSymbolicLink() || - stats.nlink !== 1 || - stats.uid !== ownerUid || - stats.gid !== ownerGid || - (stats.mode & 0o777) !== SNAPSHOT_MODE || - stats.size > OPS_TRUSTED_CONFIG_MAX_BYTES - ) { - throw new Error(REJECTED); - } - scannedEntries += 1; - scannedBytes += stats.size; - if (scannedEntries > MAX_SCAN_ENTRIES || scannedBytes > MAX_SCAN_BYTES) { - throw new Error(REJECTED); - } - if (SNAPSHOT_NAME.test(entry.name)) { - const revisionId = entry.name.slice(0, -".json".length); - if (revisions.has(revisionId)) throw new Error(REJECTED); - revisions.add(revisionId); - const retainedSize = await inspectRetainedSnapshot(join(directory, entry.name), revisionId); - entries += 1; - bytes += retainedSize; - } else if (ABORT_NAME.test(entry.name)) { - const revisionId = entry.name.slice(1, 49); - if (revisions.has(revisionId)) throw new Error(REJECTED); - revisions.add(revisionId); - await inspectRetainedSnapshot(join(directory, entry.name), revisionId); - } - } - if ( - scannedEntries > MAX_SCAN_ENTRIES || - scannedBytes > MAX_SCAN_BYTES || - !trustedConfigurationQueueFits({ - retainedEntries: entries, - retainedBytes: bytes, - reservedEntries: 1, - reservedBytes: incomingBytes, - }) - ) { - throw new Error(REJECTED); - } -} - -function contentDigest(bytes: Uint8Array): Buffer { - return createHash("sha256") - .update("openmapx-trusted-config-publisher-candidate-v1\0") - .update(bytes) - .digest(); -} - -export async function publishTrustedConfigurationSnapshot( - options: PublishTrustedConfigurationSnapshotOptions, - dependencies: Partial = {}, -) { - return withPublicationLock(async () => { - let temporary: string | undefined; - let queueLock: Awaited> | undefined; - try { - await validateDirectory(options.directory, options.ownerUid, options.ownerGid); - const nowMs = options.now?.(); - queueLock = await acquireTrustedConfigurationQueueLock({ - directory: options.directory, - token: options.token, - ownerUid: options.ownerUid, - ownerGid: options.ownerGid, - participant: "api", - operationKey: options.operationKey, - }); - const nonce = options.nonce?.() ?? `nonce_${randomUUID().replaceAll("-", "")}`; - const sealed = sealTrustedConfigurationSnapshot({ - role: "api", - operationKey: options.operationKey, - operationForRevision: options.operationForRevision, - payload: options.payload, - token: options.token, - issuedAtMs: nowMs, - nonce, - }); - await assertReadyCapacity( - options.directory, - options.ownerUid, - options.ownerGid, - options.token, - sealed.bytes.byteLength, - ); - queueLock.assertHeld(); - const finalPath = join(options.directory, `${sealed.revisionId}.json`); - temporary = join(options.directory, `.${sealed.revisionId}.${nonce}.tmp`); - const handle = await open( - temporary, - constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, - SNAPSHOT_MODE, - ); - try { - await handle.writeFile(sealed.bytes); - await handle.chmod(SNAPSHOT_MODE); - await handle.chown(options.ownerUid, options.ownerGid); - await handle.sync(); - } finally { - await handle.close(); - } - await dependencies.beforeRename?.(); - queueLock.assertHeld(); - await link(temporary, finalPath); - await unlink(temporary); - temporary = undefined; - await syncDirectory(options.directory); - const publishedStats = await lstat(finalPath); - const expectedDigest = contentDigest(sealed.bytes); - const abortPath = join(options.directory, `.${sealed.revisionId}.${nonce}.abort`); - let aborted = false; - let abortInFlight: Promise | undefined; - const validateCandidate = async ( - path: string, - expectedLinks: number, - ): Promise<{ dev: number; ino: number }> => { - const candidateHandle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const stats = await candidateHandle.stat(); - const current = await lstat(path); - const bytes = await candidateHandle.readFile(); - const actualDigest = contentDigest(bytes); - if ( - !stats.isFile() || - stats.nlink !== expectedLinks || - stats.dev !== current.dev || - stats.ino !== current.ino || - stats.dev !== publishedStats.dev || - stats.ino !== publishedStats.ino || - stats.uid !== options.ownerUid || - stats.gid !== options.ownerGid || - (stats.mode & 0o777) !== SNAPSHOT_MODE || - stats.size !== sealed.bytes.byteLength || - actualDigest.byteLength !== expectedDigest.byteLength || - !timingSafeEqual(actualDigest, expectedDigest) - ) { - throw new Error(REJECTED); - } - return { dev: stats.dev, ino: stats.ino }; - } finally { - await candidateHandle.close(); - } - }; - const pathExists = async (path: string): Promise => { - try { - await lstat(path); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; - throw error; - } - }; - const assertMissing = async (path: string): Promise => { - try { - await lstat(path); - throw new Error(REJECTED); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - }; - const runAbort = async (): Promise => { - await dependencies.beforeAbortRename?.(); - const finalExists = await pathExists(finalPath); - const tombstoneExists = await pathExists(abortPath); - if (!finalExists && !tombstoneExists) { - await syncDirectory(options.directory); - aborted = true; - return; - } - let final: { dev: number; ino: number } | undefined; - let tombstone: { dev: number; ino: number } | undefined; - if (finalExists && tombstoneExists) { - final = await validateCandidate(finalPath, 2); - tombstone = await validateCandidate(abortPath, 2); - if (final.dev !== tombstone.dev || final.ino !== tombstone.ino) { - throw new Error(REJECTED); - } - } else if (finalExists) { - final = await validateCandidate(finalPath, 1); - try { - await link(finalPath, abortPath); - } catch { - throw new Error(REJECTED); - } - await dependencies.afterAbortLink?.(); - final = await validateCandidate(finalPath, 2); - tombstone = await validateCandidate(abortPath, 2); - if (final.dev !== tombstone.dev || final.ino !== tombstone.ino) { - throw new Error(REJECTED); - } - await syncDirectory(options.directory); - await dependencies.afterAbortLinkSync?.(); - } else { - tombstone = await validateCandidate(abortPath, 1); - } - if (final) { - const currentFinal = await validateCandidate(finalPath, 2); - const currentTombstone = await validateCandidate(abortPath, 2); - if ( - currentFinal.dev !== currentTombstone.dev || - currentFinal.ino !== currentTombstone.ino - ) { - throw new Error(REJECTED); - } - await unlink(finalPath); - await dependencies.afterAbortSourceUnlink?.(); - await syncDirectory(options.directory); - await dependencies.afterAbortSourceSync?.(); - } - await assertMissing(finalPath); - await validateCandidate(abortPath, 1); - await unlink(abortPath); - await dependencies.afterAbortTombstoneUnlink?.(); - await syncDirectory(options.directory); - await dependencies.afterAbortTombstoneSync?.(); - aborted = true; - }; - const abort = async (): Promise => { - if (aborted) return; - if (abortInFlight) return abortInFlight; - const attempt = runAbort().catch(() => { - throw new Error(REJECTED); - }); - abortInFlight = attempt; - try { - await attempt; - } finally { - if (!aborted) abortInFlight = undefined; - } - }; - return { ...sealed, mode: SNAPSHOT_MODE, abort }; - } catch { - if (temporary) await unlink(temporary).catch(() => undefined); - throw new Error(REJECTED); - } finally { - try { - queueLock?.release(); - } catch { - // biome-ignore lint/correctness/noUnsafeFinally: lock interference must fail closed - throw new Error(REJECTED); - } - } - }); -} - -export async function consumePublishedTrustedConfiguration( - published: { abort(): Promise }, - submit: () => Promise, -): Promise<{ submission: PromiseSettledResult; cleanupFailed: boolean }> { - let submission: PromiseSettledResult; - try { - submission = { status: "fulfilled", value: await submit() }; - } catch (reason) { - submission = { status: "rejected", reason }; - } - let cleanupFailed = false; - try { - await published.abort(); - } catch { - cleanupFailed = true; - } - return { submission, cleanupFailed }; +/** Stable publisher entry points. Generic queue I/O has one mutex/store; + * nominal candidate operations defer to the sole actual private owner. */ +import type { + RetainedConfigurationRetirement, + ValidatedConfigurationCandidate, +} from "./prospective-configuration"; + +export { + consumePublishedTrustedConfiguration, + type PublishTrustedConfigurationSnapshotOptions, + publishTrustedConfigurationSnapshot, + readPreparedTrustedConfigurationCandidate, + retainPreparedTrustedConfigurationCandidate, + type SnapshotPublishDependencies, +} from "./trusted-config-publication-store"; +export async function publishPreparedTrustedConfigurationSnapshot( + value: ValidatedConfigurationCandidate, +): ReturnType< + typeof import("./prospective-configuration").publishPreparedTrustedConfigurationSnapshot +> { + return (await import("./prospective-configuration")).publishPreparedTrustedConfigurationSnapshot( + value, + ); +} +export async function discardUnadmittedTrustedConfigurationCandidate( + value: ValidatedConfigurationCandidate, +): ReturnType< + typeof import("./prospective-configuration").discardUnadmittedTrustedConfigurationCandidate +> { + return ( + await import("./prospective-configuration") + ).discardUnadmittedTrustedConfigurationCandidate(value); +} +export async function retireSettledTrustedConfigurationCandidate( + value: ValidatedConfigurationCandidate, +): ReturnType< + typeof import("./prospective-configuration").retireSettledTrustedConfigurationCandidate +> { + return (await import("./prospective-configuration")).retireSettledTrustedConfigurationCandidate( + value, + ); +} +export async function completeRetainedConfigurationRetirement( + value: RetainedConfigurationRetirement, +): ReturnType< + typeof import("./prospective-configuration").completeRetainedConfigurationRetirement +> { + return (await import("./prospective-configuration")).completeRetainedConfigurationRetirement( + value, + ); } diff --git a/apps/api/src/services/trusted-configuration-producer.ts b/apps/api/src/services/trusted-configuration-producer.ts new file mode 100644 index 000000000..419a64b83 --- /dev/null +++ b/apps/api/src/services/trusted-configuration-producer.ts @@ -0,0 +1,337 @@ +import { + type CoordinatorMode, + canonicalJson, + configurationApplyResultSchema, + configurationGenerationRevisionSchema, + type OpsResultFor, + type TrustedConfigurationPayload, + trustedConfigurationPayloadSchema, +} from "@openmapx/core/ops"; +import { getAllIntegrations } from "../integration-host"; +import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; +import { hasRetainedAdminJobOperation, observeRetainedAdminJobOperation } from "./admin-job-ops"; +import { + withAdministrativeConfigurationJob, + withAdministrativeConfigurationWriter, +} from "./administrative-configuration-writer"; +import type { JobContext } from "./job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { + type ProspectiveConfigurationOverlay, + persistAndApplyProspectiveConfiguration, + prepareProspectiveConfiguration, + retireSettledProspectiveConfiguration, + type ValidatedConfigurationCandidate, +} from "./prospective-configuration"; +import { + completeTrustedConfigurationApply, + TrustedConfigurationApplyError, + type TrustedConfigurationApplyOutcome, + TrustedConfigurationUpgradeRequired, +} from "./trusted-config-outcome"; + +type ConfigurationKind = + | "stack.render" + | "serviceSelection.apply" + | "serviceConfig.apply" + | "integrationConfig.apply" + | "vault.apply"; +export type CapturedIntegrationReloadResult = { + message: string; + reloaded: number; + enabled: number; +}; + +export interface ApplyTrustedConfigurationOptions { + kind: ConfigurationKind; + operationKey: string; + serviceId?: string; + integrationId?: string; + selectedRoots?: string[]; + signal?: AbortSignal; + ctx?: JobContext; + actorId?: string; + coordinatorMode?: CoordinatorMode; + /** Current service-credential producer's fixed continuation; never a render-only fallback. */ + followUp?: "service-apply"; + expectedCurrentRevisionId?: string | null; + overlay?: ProspectiveConfigurationOverlay; + candidate?: ValidatedConfigurationCandidate; + /** Internal compensation must still equal the strictly acquired desired snapshot. */ + snapshot?: TrustedConfigurationPayload; +} + +export { buildConfigurationInput } from "./configuration-input"; + +/** + * Hand the database's configuration to the ops-agent, which renders it with + * the operator's selection (or `selectedRoots`, which become it) and makes it + * the applied generation. The registry follows what was applied. + */ +export async function applyTrustedConfiguration( + options: ApplyTrustedConfigurationOptions, +): Promise<{ + revisionId: string; + enabledServiceIds: string[]; + jobId?: string; + integrationReload?: CapturedIntegrationReloadResult; +}> { + return completeTrustedConfigurationApplication( + options, + await applyTrustedConfigurationWithEvidence(options), + ); +} + +/** Internal evidence for scoped compensation. Payload may contain secrets; never serialize it. */ +export async function applyTrustedConfigurationWithEvidence( + options: ApplyTrustedConfigurationOptions, +): Promise<{ + payload?: TrustedConfigurationPayload; + previousApplied?: OpsResultFor<"configuration.inspect">; + outcome: TrustedConfigurationApplyOutcome; + jobId?: string; + integrationReload?: CapturedIntegrationReloadResult; +}> { + if (options.ctx && hasRetainedAdminJobOperation(options.ctx, options.operationKey)) { + const outcome = configurationApplyResultSchema.parse( + await observeRetainedAdminJobOperation(options.ctx, options.operationKey), + ); + const retained = ( + options.ctx.checkpointResult?._configurationOutcomes as + | Record + | undefined + )?.[options.operationKey]; + if (!retained) { + await options.ctx.checkpoint({ ...options.ctx.checkpointResult, configuration: outcome }); + throw new JobObservationPending("recovery_required"); + } + return { outcome: retained }; + } + if (!options.ctx) throw new TrustedConfigurationUpgradeRequired(); + const ctx = options.ctx; + const coordinatorMode = options.coordinatorMode ?? "ensure-running"; + const actorId = options.actorId ?? ctx.jobId; + const overlay = + options.overlay ?? + (options.kind === "serviceSelection.apply" && options.selectedRoots + ? { + purpose: "selection" as const, + coordinatorMode, + changes: [{ kind: "selection" as const, selectedRoots: options.selectedRoots }], + } + : options.kind === "stack.render" && options.selectedRoots === undefined + ? { purpose: "render" as const, coordinatorMode, changes: [] } + : undefined); + if (!overlay && !options.candidate) + return { + outcome: { + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }, + }; + return withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { + actorId, + operationKey: options.operationKey, + integrationIds: + overlay?.changes.flatMap((change) => + "integrationId" in change ? [change.integrationId] : [], + ) ?? [], + signal: options.signal ?? ctx.signal, + }, + async () => { + const previousApplied = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("configuration.inspect", options.operationKey), + { signal: options.signal ?? ctx.signal }, + ); + const expectedCurrentRevisionId = Object.hasOwn(options, "expectedCurrentRevisionId") + ? options.expectedCurrentRevisionId! + : previousApplied.revisionId; + const candidate = + options.candidate ?? + (await prepareProspectiveConfiguration({ + actorId, + operationKey: options.operationKey, + expectedCurrentRevisionId, + overlay: overlay!, + signal: options.signal ?? ctx.signal, + followUp: options.followUp, + })); + const { validatedConfigurationCandidateState } = await import( + "./prospective-configuration" + ); + const candidateState = validatedConfigurationCandidateState(candidate), + payload = candidateState.payload; + const operation = candidateState.sealed.operation; + if ( + operation.kind !== options.kind || + ("serviceId" in operation && + options.serviceId && + options.serviceId !== operation.serviceId) || + ("integrationId" in operation && + options.integrationId && + options.integrationId !== operation.integrationId) || + (options.selectedRoots && + canonicalJson(options.selectedRoots) !== canonicalJson(payload.selectedRoots)) + ) + throw Error("Configuration candidate intent conflict"); + if (options.followUp && candidateState.record.apiFollowUp !== options.followUp) + throw Error("Configuration continuation intent conflict"); + if (options.snapshot && canonicalJson(options.snapshot) !== canonicalJson(payload)) + throw new Error("Strict compensation requires current owned desired rows"); + const receipt = await persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode, + signal: options.signal ?? ctx.signal, + }); + if ( + !candidateState.record.maintenanceId && + candidateState.record.apiFollowUp === "service-apply" && + completeTrustedConfigurationApply(receipt.application) + ) { + const { completeProspectiveServiceCredential } = await import( + "./prospective-configuration" + ); + await completeProspectiveServiceCredential(candidate); + } + if ( + candidateState.record.apiFollowUp === "integration-reload" && + completeTrustedConfigurationApply(receipt.application) + ) { + await ctx.checkpoint({ + ...ctx.checkpointResult, + integrationReload: { + state: "attempted", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }, + }); + try { + const { reloadProspectiveIntegrations } = await import("./prospective-configuration"); + const integrationReload = await reloadProspectiveIntegrations(candidate); + await ctx.checkpoint({ + ...ctx.checkpointResult, + _configurationCandidate: { + ...(ctx.checkpointResult?._configurationCandidate as Record), + phase: "settled", + }, + integrationReload: { + state: "complete", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + result: integrationReload, + }, + }); + await retireSettledProspectiveConfiguration(candidate); + return { payload, previousApplied, outcome: receipt.application, integrationReload }; + } catch { + await ctx.checkpoint({ + ...ctx.checkpointResult, + _configurationCandidate: { + ...(ctx.checkpointResult?._configurationCandidate as Record), + phase: "unknown", + }, + integrationReload: { + state: "failed", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }, + }); + throw new AdminJobFailure("Integration reload did not complete", { + configuration: receipt.application, + integrationReload: { state: "failed" }, + }); + } + } + return { payload, previousApplied, outcome: receipt.application }; + }, + ), + ); +} + +/** Full-snapshot compensation; compare the applied generation, NOT the manifest authority. */ +export async function applyTrustedConfigurationSnapshot(options: { + payload: TrustedConfigurationPayload; + expectedCurrentRevisionId: string | null; + operationKey: string; + signal?: AbortSignal; + ctx?: JobContext; +}): Promise { + if ( + !configurationGenerationRevisionSchema.nullable().safeParse(options.expectedCurrentRevisionId) + .success + ) { + return { + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + } + return applySnapshot( + { + kind: "stack.render", + operationKey: options.operationKey, + signal: options.signal, + ctx: options.ctx, + }, + options.payload, + options.expectedCurrentRevisionId, + ); +} + +async function applySnapshot( + options: ApplyTrustedConfigurationOptions, + input: TrustedConfigurationPayload, + expectedCurrentRevisionId: string | null, +): Promise { + if (!options.ctx) throw new TrustedConfigurationUpgradeRequired(); + return ( + await applyTrustedConfigurationWithEvidence({ + ...options, + expectedCurrentRevisionId, + snapshot: trustedConfigurationPayloadSchema.parse(input), + }) + ).outcome; +} + +/** The config schema of every loaded integration, by id. */ +export function integrationSchemas(): Map> { + return new Map( + getAllIntegrations().map((integration) => [ + integration.id, + (integration.manifest.configSchema ?? {}) as Record, + ]), + ); +} + +/** Safe result projection only; this function constructs no candidate/job/grant. */ +export function completeTrustedConfigurationApplication( + options: ApplyTrustedConfigurationOptions, + evidence: Awaited>, +) { + const { outcome, jobId, integrationReload } = evidence; + if (options.ctx && outcome.outcome === "unknown") throw new JobObservationPending("transport"); + if ( + outcome.outcome !== "committed" || + outcome.followUp !== "complete" || + outcome.cleanupFailed || + outcome.publisherCleanupFailed || + outcome.registryUpdateFailed + ) + if (options.ctx) + throw new AdminJobFailure("Configuration did not complete", { configuration: outcome }); + else throw new TrustedConfigurationApplyError(outcome); + return { + revisionId: outcome.revisionId, + enabledServiceIds: outcome.enabledServiceIds, + ...(jobId ? { jobId } : {}), + ...(integrationReload ? { integrationReload } : {}), + }; +} diff --git a/apps/api/src/test/configuration-postgres-fixture.ts b/apps/api/src/test/configuration-postgres-fixture.ts new file mode 100644 index 000000000..6fae6ac0b --- /dev/null +++ b/apps/api/src/test/configuration-postgres-fixture.ts @@ -0,0 +1,56 @@ +import { randomUUID } from "node:crypto"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import * as schema from "../db/schema"; + +/** An explicitly supplied owned/migrated synthetic database. No migrations run. + * Only empty fixture copies of the current configuration/job tables are created and removed. */ +export async function createConfigurationPostgresFixture() { + const url = process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL; + if (!url) throw Error("Explicit owned synthetic PostgreSQL connection required"); + const name = `task012_${randomUUID().replaceAll("-", "")}`; + const control = postgres(url, { max: 1, onnotice: () => {} }); + let sql: ReturnType | undefined; + try { + await control.unsafe(`CREATE SCHEMA "${name}"`); + for (const table of [ + "admin_job", + "admin_job_log", + "admin_audit_log", + "capability_binding", + "service_config", + "integration_config", + "service_secret", + "integration_secret", + ]) + await control.unsafe( + `CREATE TABLE "${name}"."${table}" (LIKE public."${table}" INCLUDING ALL)`, + ); + sql = postgres(url, { + max: 6, + connection: { search_path: name, application_name: "openmapx-task012-test" }, + }); + const ownedSql = sql; + return { + sql: ownedSql, + db: drizzle(ownedSql, { schema }), + schema: name, + async stop() { + await ownedSql.end({ timeout: 2 }); + try { + await control.unsafe(`DROP SCHEMA "${name}" CASCADE`); + } finally { + await control.end({ timeout: 2 }); + } + }, + }; + } catch (error) { + await sql?.end({ timeout: 2 }); + try { + await control.unsafe(`DROP SCHEMA IF EXISTS "${name}" CASCADE`); + } finally { + await control.end({ timeout: 2 }); + } + throw error; + } +} diff --git a/apps/api/src/utils/docker-compose.ts b/apps/api/src/utils/docker-compose.ts index f6365f8ff..ba2064e80 100644 --- a/apps/api/src/utils/docker-compose.ts +++ b/apps/api/src/utils/docker-compose.ts @@ -22,6 +22,7 @@ interface OpsDependencies { interface AsyncOpsDependencies extends OpsDependencies { operationKey?: string; signal?: AbortSignal; + expectedConfigurationRevisionId?: string; } let sharedClient: ApiOpsClient | undefined; @@ -86,23 +87,38 @@ export type DockerComposeAction = export const STACK_STOP_GUIDANCE = "Stack shutdown is unavailable from the web API because it would stop the operations agent. Run the documented host shutdown command instead."; -function typedAction(serviceId: string, action: DockerComposeAction): OpsOperation | null { - if (!serviceId) return action === "start" ? { kind: "stack.start" } : null; +function typedAction( + serviceId: string, + action: DockerComposeAction, + expectedConfigurationRevisionId?: string, +): OpsOperation | null { + if (!serviceId) + return action === "start" && expectedConfigurationRevisionId + ? { kind: "stack.start", expectedConfigurationRevisionId } + : null; switch (action) { case "start": - return { kind: "service.start", serviceId }; + return expectedConfigurationRevisionId + ? { kind: "service.start", serviceId, expectedConfigurationRevisionId } + : null; case "stop": return { kind: "service.stop", serviceId }; case "restart": return { kind: "service.restart", serviceId }; case "recreate": - return { kind: "service.recreate", serviceId }; + return expectedConfigurationRevisionId + ? { kind: "service.recreate", serviceId, expectedConfigurationRevisionId } + : null; case "recreate-isolated": - return { kind: "service.recreateIsolated", serviceId }; + return expectedConfigurationRevisionId + ? { kind: "service.recreateIsolated", serviceId, expectedConfigurationRevisionId } + : null; case "remove": return { kind: "service.remove", serviceId }; case "pull": - return { kind: "service.pull", serviceId }; + return expectedConfigurationRevisionId + ? { kind: "service.pull", serviceId, expectedConfigurationRevisionId } + : null; } } @@ -114,7 +130,7 @@ export async function dockerComposeAction( if (!serviceId && action === "stop") { return { exitCode: 1, stdout: "", stderr: STACK_STOP_GUIDANCE }; } - const operation = typedAction(serviceId, action); + const operation = typedAction(serviceId, action, options.expectedConfigurationRevisionId); if (!operation || !options.operationKey) { return { exitCode: 1, diff --git a/apps/ops-agent/src/__fixtures__/configuration-source.ts b/apps/ops-agent/src/__fixtures__/configuration-source.ts new file mode 100644 index 000000000..206279d30 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/configuration-source.ts @@ -0,0 +1,27 @@ +/** Real read-only source witness for standalone runtime fixtures. Never a + * caller-generated UUID or a replacement mutation/inspection handler. */ +import { provisionControlInstallation } from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import { opsOperationFingerprint } from "@openmapx/core/ops"; +import { dispatchOpsOperation, type OpsRuntime } from "../runtime"; +import { testExecution } from "./execution"; +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +const sources = new WeakMap }>(); +export async function configurationSource(runtime: OpsRuntime, infraDir: string) { + const revision = services.currentConfigurationGeneration(infraDir); + let retained = sources.get(runtime); + if (!retained || retained.revision !== revision) { + const value = (async () => { + mkdirSync(join(infraDir, "data"), { recursive: true, mode: 0o700 }); + await provisionControlInstallation(infraDir); + const operation = { kind: "configuration.sources.inspect" } as const; + const source = await dispatchOpsOperation(runtime, operation, { execution: testExecution(), signal: new AbortController().signal, + emitLog() {}, claim: { operation, fingerprint: opsOperationFingerprint(operation), source: "registry", + capability: { revisionId: "registry-v1", values: {} } } }); + return source.sourceRevision; + })(); + retained = { revision, value }; sources.set(runtime, retained); + } + return { expectedSourceRevision: await retained.value, coordinatorMode: "require-running" as const }; +} diff --git a/apps/ops-agent/src/__fixtures__/configuration-writer.mjs b/apps/ops-agent/src/__fixtures__/configuration-writer.mjs index 8847d27eb..d23c09a8a 100644 --- a/apps/ops-agent/src/__fixtures__/configuration-writer.mjs +++ b/apps/ops-agent/src/__fixtures__/configuration-writer.mjs @@ -1,9 +1,12 @@ import { once } from "node:events"; +import { mkdirSync } from "node:fs"; import { join } from "node:path"; import { opsOperationFingerprint } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; +import { closeControlInstallation, observeConfigurationSources, openControlInstallation, provisionControlInstallation } from "@openmapx/core/build-artifacts"; import { renderComposeForRepo } from "../../../../packages/cli/src/commands/compose"; import { createUnavailableRuntime, dispatchOpsOperation } from "../runtime"; +import { testExecution } from "./execution"; import { installTrustedConfigurationRuntime } from "../trusted-config-runtime"; const [root, mode, letter, expected, hold] = process.argv.slice(2); @@ -58,9 +61,17 @@ try { integrationConfigs: [], serviceSecrets: [], }; + mkdirSync(join(infraDir, "data"), { recursive: true }); + await provisionControlInstallation(infraDir); + const installation = openControlInstallation({ infraDir, role: "ops-agent", serviceIds: ["alpha"] }); + let source; + try { source = observeConfigurationSources({ installation, infraDir, services: [service], authorityRevision: "registry-v1", env: {} }); } + finally { closeControlInstallation(installation); } const operation = { kind: "stack.render", revisionId: `cfg1_${letter.repeat(43)}`, + expectedSourceRevision: source.sourceRevision, + coordinatorMode: "ensure-running", // This synthetic writer captures its initial baseline before entering the // held activation schedule. Explicit values model stale compensation. expectedCurrentRevisionId: expected @@ -71,6 +82,8 @@ try { }; if (hold === "captured") await barrier(); result = await dispatchOpsOperation(runtime, operation, { + // Adapter identity only; this native fixture proves writer exclusion, not admission. + execution: testExecution(), signal: new AbortController().signal, emitLog() {}, claim: { @@ -79,7 +92,7 @@ try { source: "trusted-data", capability: { revisionId: operation.revisionId, - values: {}, + values: { authorityRevision: "registry-v1" }, trustedConfiguration: payload, }, }, diff --git a/apps/ops-agent/src/__fixtures__/owned-data-manager.ts b/apps/ops-agent/src/__fixtures__/owned-data-manager.ts new file mode 100644 index 000000000..51dbe28a7 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/owned-data-manager.ts @@ -0,0 +1,152 @@ +import { join } from "node:path"; +import { closeControlInstallation, createDataExecutionControlJournal, openControlInstallation, reconcileControlRetention } from "@openmapx/core/build-artifacts"; +import Fastify from "fastify"; +import { registerApi } from "../../../../services/data-manager/src/api"; +import { registerAuth } from "../../../../services/data-manager/src/auth"; +import { StateStore } from "../../../../services/data-manager/src/state"; +import { resolveOperationsProfile } from "../../../../services/data-manager/src/jobs/transitous/operations-profile"; +import { getSingleFlightController } from "../../../../services/data-manager/src/jobs/transitous/runtime"; +import { createDataConfigurationMaintenance } from "../../../../services/data-manager/src/jobs/execution/configuration-maintenance"; +import { createDataExecutionDispatcher } from "../../../../services/data-manager/src/jobs/execution/dispatch"; +import { applyOwnedGenerationHardlinks } from "../../../../services/data-manager/src/jobs/execution/hardlinks"; +import { createDataExecutionReceiver } from "../../../../services/data-manager/src/jobs/execution/receiver"; +import { createDataExecutionRuntime } from "../../../../services/data-manager/src/jobs/execution/runtime"; +import { createPostgresDataExecutionStore } from "../../../../services/data-manager/src/jobs/execution/store"; + +/** Actual HTTP receiver, SQL admission guard, dispatcher, filesystem leaf and journal. */ +export async function ownedDataManager(options: { + sql: Parameters[0]; infraDir: string; serviceIds: string[]; loseHardlinkResponse?: boolean; losePrepareResponse?: boolean; + observeSettlementSql?: boolean; + loseSettlementCommitAcknowledgement?: boolean; + settlementSqlFault?: "update-ack-loss" | "terminate-before-commit"; + pauseSettlementBeforeCommit?: boolean; +}) { + let settlementCommits = 0, settlementUpdates = 0, settlementRollbacks = 0, terminatedSettlementSession = false; + const commitEntered = Promise.withResolvers(), continueCommit = Promise.withResolvers(); + const uncertainConnections = new Set>>(); + // Named external SQL transport fault. The installed production adapter and + // original reserved PostgreSQL connection still perform the actual UPDATE + // and COMMIT; only the COMMIT acknowledgement is lost. + const fenceSql = options.observeSettlementSql || options.loseSettlementCommitAcknowledgement || options.settlementSqlFault ? new Proxy(options.sql, { + get(target, name) { + if (name !== "reserve") return Reflect.get(target, name); + return async () => { + const connection = await target.reserve(); + return new Proxy(connection, { get(owned, member) { + if (member !== "unsafe") return Reflect.get(owned, member); + return async (query: string, parameters?: Parameters[1]) => { + if (query.startsWith("UPDATE data_manager.jobs SET metadata=jsonb_set")) settlementUpdates++; + if (query === "COMMIT" && options.pauseSettlementBeforeCommit) { commitEntered.resolve(); await continueCommit.promise; } + if (query === "COMMIT" && options.settlementSqlFault === "terminate-before-commit") { + const [session] = await owned.unsafe("SELECT pg_backend_pid() AS pid"); + const [killed] = await options.sql`SELECT pg_terminate_backend(${session!.pid}) AS terminated`; + if (killed!.terminated !== true) throw Error("Owned settlement backend not terminated"); + terminatedSettlementSession = true; + } + const result = await owned.unsafe(query, parameters); + if (query === "ROLLBACK") settlementRollbacks++; + if (query.startsWith("UPDATE data_manager.jobs SET metadata=jsonb_set") && options.settlementSqlFault === "update-ack-loss") + throw Error("External settlement UPDATE acknowledgement lost"); + if (query === "COMMIT") settlementCommits++; + if (query === "COMMIT" && options.loseSettlementCommitAcknowledgement) { + uncertainConnections.add(owned); + throw Error("External settlement COMMIT acknowledgement lost"); + } + return result; + }; + } }); + }; + }, + }) : options.sql; + const installation = openControlInstallation({ infraDir: options.infraDir, role: "data-manager", serviceIds: options.serviceIds }); + const dataDir = join(options.infraDir, "data"), journal = createDataExecutionControlJournal(installation, options.sql); + const fence = createDataConfigurationMaintenance(fenceSql, installation); + let startupRecoveryRequired = false; + try { await fence.startup(); } + catch { + // Match services/data-manager/src/service.ts:start: its real boundary + // retains CLOSED data admission but still starts read/control HTTP after + // startup recovery fails. This is not a successful guard/readiness proof. + startupRecoveryRequired = true; + } + const store = createPostgresDataExecutionStore(options.sql, { controlJournal: journal, configurationFence: fence }); + const singleFlight = getSingleFlightController(), operationsPolicy = resolveOperationsProfile({ countries: ["de"] }); + const dispatcher = createDataExecutionDispatcher({ dataDir, state: new StateStore(dataDir), sql: options.sql, singleFlight, operationsPolicy, runHardlinks: applyOwnedGenerationHardlinks }); + const runtime = createDataExecutionRuntime({ store, dispatch: dispatcher.dispatch }); + const receiver = createDataExecutionReceiver({ runtime, installation, configurationFence: fence, dispatcher, operationsPolicy }); + const app = Fastify(); let posts = 0, preparePosts = 0, settlePosts = 0; + const maintenanceIds = new Set(); + const maintenanceReplies: { status: number; error: unknown }[] = []; + const settlementResponseStatuses:number[]=[]; + const fenceInspectionReplies: { status: number; body: unknown }[] = []; + let originalInspection: { url: string; headers: Record } | undefined; + const keys = new Set(), replies: { status: number; body: unknown }[] = []; + app.addHook("onSend", async (request, reply, payload) => { + if (request.method === "POST" && /^\/executions\/configuration-maintenance\/[a-f0-9-]{36}\/settle$/.test(request.url)) {settlePosts++;settlementResponseStatuses.push(reply.statusCode);} + if (request.method === "GET" && request.url.startsWith("/executions/configuration-maintenance/")) { + fenceInspectionReplies.push({ status: reply.statusCode, body: typeof payload === "string" ? JSON.parse(payload) : null }); + originalInspection = { url: request.url, headers: Object.fromEntries(Object.entries(request.headers).filter((entry): entry is [string,string] => typeof entry[1] === "string")) }; + } + if (request.method === "POST" && request.url === "/executions/configuration-maintenance/prepare") { + const id = (request.body as { maintenanceId?: string } | undefined)?.maintenanceId; + if (id) maintenanceIds.add(id); + preparePosts++; + maintenanceReplies.push({ status: reply.statusCode, error: typeof payload === "string" ? JSON.parse(payload).error : null }); + if (options.losePrepareResponse) reply.raw.destroy(); + } + if (request.method === "POST" && request.url === "/executions/generation-hardlinks") { + const key = (request.body as { operationKey?: string } | undefined)?.operationKey; + if (key) keys.add(key); + replies.push({ status: reply.statusCode, body: typeof payload === "string" ? JSON.parse(payload) : null }); + posts++; if (options.loseHardlinkResponse) reply.raw.destroy(); + } + return payload; + }); + const token = "synthetic-task012-data-manager-token"; + registerAuth(app, token, { executionReceiver: () => receiver }); + registerApi(app, { dataDir, singleFlight, operationsPolicy, executionReceiver: receiver }); + const baseUrl = await app.listen({ host: "127.0.0.1", port: 0 }); + const resumeHttp=async()=>{ + if(app.server.listening)return; + const address=new URL(baseUrl); + await new Promise((resolve,reject)=>{ + app.server.once("error",reject); + app.server.listen(Number(address.port),address.hostname,()=>{app.server.removeListener("error",reject);resolve();}); + }); + }; + const pauseHttp=async()=>{ + app.server.closeAllConnections(); + await new Promise((resolve,reject)=>app.server.close(error=>error?reject(error):resolve())); + }; + let stopped = false; + const stopForRestart = async () => { + if (stopped) return; + await runtime.close(); app.server.closeAllConnections(); await app.close(); + for (const connection of uncertainConnections) connection.release(); + uncertainConnections.clear(); journal.close(); closeControlInstallation(installation); stopped = true; + }; + return { baseUrl, token, store, receiver, pauseHttp, resumeHttp, get posts() { return posts; }, get preparePosts() { return preparePosts; }, get settlePosts() { return settlePosts; }, + get settlementSqlIo() { return { commits: settlementCommits, updates: settlementUpdates }; }, + get settlementContainmentIo() { return { rollbacks: settlementRollbacks, terminated: terminatedSettlementSession }; }, + get fenceInspections() { return structuredClone(fenceInspectionReplies); }, stopForRestart, + get startupRecoveryRequired() { return startupRecoveryRequired; }, + assertDataAdmission: () => options.sql.begin(async tx => { await fence.assertAdmission({ unsafe: (query, parameters) => tx.unsafe(query, parameters) }); }), + settlementCommitEntered: commitEntered.promise, releaseSettlementCommit: () => continueCommit.resolve(), + async inspectWithOriginalWitness() { + if (!originalInspection) throw Error("Original agent inspection request unavailable"); + const response = await fetch(baseUrl + originalInspection.url, { headers: originalInspection.headers }); + return { status: response.status, body: await response.json() }; + }, + get settlementResponseStatuses(){return [...settlementResponseStatuses];}, + get hardlinkResponseStatuses() { return replies.map(reply => reply.status); }, + reconcileRetention: () => reconcileControlRetention(installation, { maxFamilies: 64 }), + async diagnostics() { return { maintenanceReplies, replies: replies.map(reply => ({ status: reply.status, error: (reply.body as { error?: unknown } | null)?.error })), + executions: await Promise.all([...keys].map(async key => { const value = await store.lookup(key); return value && { status: value.status.status, publication: value.status.publication, diagnostics: value.status.diagnostics }; })) }; }, + async close() { + await stopForRestart(); + // Exact fixture-owned execution IDs only; no broad job cleanup. + for (const key of keys) await options.sql`DELETE FROM data_manager.jobs WHERE operation_key=${key}`; + for (const id of maintenanceIds) await options.sql`DELETE FROM data_manager.jobs WHERE kind='configuration-maintenance' AND metadata->'configurationMaintenance'->'receipt'->>'maintenanceId'=${id}`; + }, + }; +} diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index 878c1d04c..bdbd070b0 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -1,9 +1,11 @@ +import {configurationPolicyRevision} from "../configured-resource-authority"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, realpathSync, + readFileSync, statSync, writeFileSync, } from "node:fs"; @@ -15,6 +17,7 @@ import { type OpsOperation, } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; +import { provisionControlInstallation } from "@openmapx/core/build-artifacts"; import { CommunityRepositoryStore } from "../community-repositories"; import type { DockerRuntimeOptions } from "../docker-runtime"; import { createProductionRegistryResourceClaimer } from "../policy"; @@ -27,14 +30,20 @@ import { createTrustedConfigurationAuthorityLoader } from "../trusted-configurat import { createFileTrustedOpsDataSource } from "../trusted-config-source"; import { dispatchOpsOperation } from "../runtime"; import { testExecution } from "./execution"; +import { createDefaultFixedCli } from "../administrative-runtime"; +import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost() { - const root = mkdtempSync(join(tmpdir(), "openmapx-owned-")); +export async function ownedServiceHost(options: { extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { + const withCoordinator = options.withHardlinks || options.withCoordinator; + const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); + const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); + mkdirSync(root,{recursive:true}); chmodSync(root, 0o755); const infraDir = join(root, "infra/docker"); mkdirSync(infraDir, { recursive: true }); mkdirSync(join(root, "services")); mkdirSync(join(root, "custom_integrations")); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); const loaded: services.LoadedService[] = [ "motis", "ops-agent", @@ -42,18 +51,34 @@ export async function ownedServiceHost() { "traefik", "postgis", "redis", + ...(withCoordinator ? ["data-manager"] : []), + ...(options.extraServiceIds ?? []), ].map((id) => ({ manifest: { id, name: id, version: "1.0.0", quality: "built-in", - container: { image: `example/${id}`, tag: "1" }, + container: {...(options.imageReferences?.[id] ?? { image: `example/${id}`, tag: "1" }), + ...(options.coordinatorDependency && id==="motis"?{dependsOn:[{service:"data-manager"}]}:{})}, + ...(options.schemas?.[id] ? { configSchema: options.schemas[id] } : {}), + ...(options.oversizedMaintenancePlan && id==="motis" ? { + produces:Array.from({length:256},(_,index)=>({type:`capacity-${index}`,sourceDir:`data/${Array.from({length:10},(_,part)=>`${part}${"p".repeat(79)}`).join("/")}/${index}`})), + consumes:Array.from({length:256},(_,index)=>({type:`capacity-${index}`,mountAt:`/input/${index}`,readOnly:true,required:true})), + }:{}), + ...(options.withHardlinks && id === "motis" ? { + produces: [{ type: "osm-pbf", sourceDir: "data/source-osm" }], + consumes: [{ type: "osm-pbf", mountAt: "/input", readOnly: true, required: true }], + } : {}), }, directory: join(root, "services", id), enabled: true, isBuiltIn: true, })); + for (const service of loaded) { + mkdirSync(service.directory, { recursive: true }); + writeFileSync(join(service.directory, "service.json"), JSON.stringify(service.manifest)); + } const loadAuthority = createTrustedConfigurationAuthorityLoader({ rootDir: root, builtInServices: loaded, @@ -63,14 +88,14 @@ export async function ownedServiceHost() { services: loaded, input: { domain: "example.test", - selectedRoots: ["motis"], + selectedRoots: [...(withCoordinator ? ["data-manager", "motis"] : ["motis"]),...(options.extraServiceIds??[])], serviceConfigs: [], integrationConfigs: [], serviceSecrets: [], }, ownedServiceGenerations: [], }); - services.writeServiceSelectionFile(infraDir, ["motis"]); + services.writeServiceSelectionFile(infraDir, [...(withCoordinator ? ["data-manager", "motis"] : ["motis"]),...(options.extraServiceIds??[])]); const composePath = services.currentConfigurationFile(infraDir, services.GENERATED_COMPOSE_FILE); const firstCompose = realpathSync(composePath); const stack = { @@ -83,7 +108,7 @@ export async function ownedServiceHost() { Id: "a".repeat(64), Image: imageId, State: "running", - ConfigImage: "example/motis:1", + ConfigImage: services.serviceContainerImageReference(loaded.find(s=>s.manifest.id==="motis")!.manifest.container), Labels: { "com.docker.compose.project": services.STACK_PROJECT, "com.docker.compose.service": "motis", @@ -94,11 +119,16 @@ export async function ownedServiceHost() { let rows = [row]; let rawListing: string | undefined; let malformedInspect = false; + let runtimeInspections: Record = {}; + let runtimeSecretBytes: Record = {}; const calls: string[][] = []; + const actualComposeConfig = options.actualComposeConfig; let creatingProcess: | ((args: readonly string[], signal: AbortSignal) => Promise) | undefined; let inventoryProcess: (() => Promise) | undefined; + let imagePullProcess: ((reference:string,signal:AbortSignal)=>Promise) | undefined; + const imageIdentities = new Map(); const execFile: NonNullable = async (_file, args, options) => { calls.push([...args]); let stdout = ""; @@ -108,9 +138,17 @@ export async function ownedServiceHost() { inventoryProcess = undefined; await pending?.(); } else if (args[0] === "container" && args[1] === "inspect") - stdout = malformedInspect ? "{" : rows.map((r) => JSON.stringify(r)).join("\n"); + stdout = malformedInspect ? "{" : args[3] === "{{json .}}" + ? args.slice(4).map(id => JSON.stringify(runtimeInspections[id] ?? null)).join("\n") : rows.map((r) => JSON.stringify(r)).join("\n"); + else if (args[0] === "container" && args[1] === "exec" && args[3] === "base64") { + const record = runtimeInspections[args[2]!] as { Mounts?: { Source?: string; Destination: string }[] } | undefined, + source = record?.Mounts?.find(value => value.Destination === args[4])?.Source; + if (!source) throw Error("Synthetic Docker secret mount unavailable"); + stdout = runtimeSecretBytes[`${args[2]}:${args[4]}`] ?? readFileSync(source).toString("base64"); + } else if (args[0] === "compose" && args.includes("ps")) + stdout = JSON.stringify(rows.map(row => ({ Service: row.Labels["com.docker.compose.service"], State: row.State, Health: "" }))); else if (args[0] === "compose" && args.includes("config")) - stdout = JSON.stringify({ + stdout = actualComposeConfig ? (await runContainedProcess("docker", args, options)).stdout : JSON.stringify({ services: Object.fromEntries( loaded.map((s) => [ s.manifest.id, @@ -118,7 +156,8 @@ export async function ownedServiceHost() { ]), ), }); - else if (args[0] === "image" && args[1] === "inspect") stdout = imageId; + else if (args[0] === "image" && args[1] === "inspect") stdout = imageIdentities.get(args.at(-1)!) ?? imageId; + else if (args[0] === "image" && args[1] === "pull") await imagePullProcess?.(args[2]!,options.signal); else if (args[0] === "compose" && args.includes("up")) await creatingProcess?.(args, options.signal); else if (args[0] === "stop") { @@ -151,11 +190,25 @@ export async function ownedServiceHost() { }, }); await repositories.recover(); + const queue = join(root, "trusted-queue"); + mkdirSync(queue, { mode: 0o700 }); + mkdirSync(join(queue, ".claimed"), { mode: 0o700 }); + mkdirSync(join(infraDir, "data"), { mode: 0o700, recursive: true }); + await provisionControlInstallation(infraDir); + const token = Buffer.alloc(32, 17).toString("base64url"); + let competingRevisionId: string | undefined; + const defaultCli = options.defaultCliEntry ? createDefaultFixedCli(root, options.defaultCliEntry) : undefined; const runtime = await createProductionOpsRuntime({ rootDir: root, stack, loadAuthority, communityRepositories: repositories, + ...(defaultCli ? { runFixedCli: async (...args: Parameters) => { + await defaultCli(...args); + if (options.competeAfterDefaultCli) competingRevisionId = (await services.commitConfigurationGeneration({ infraDir, services: loaded, + input: { ...services.readAppliedConfiguration(infraDir), domain: "example.test", selectedRoots: withCoordinator ? ["data-manager", "motis"] : ["motis"] } })).revisionId; + } } : {}), + candidateSource: { directory: queue, token, ownerUid: statSync(queue).uid, ownerGid: statSync(queue).gid }, docker: { execFile, fileExists: existsSync, @@ -167,10 +220,6 @@ export async function ownedServiceHost() { }, }); const authority = await loadAuthority(); - const queue = join(root, "trusted-queue"); - mkdirSync(queue, { mode: 0o700 }); - mkdirSync(join(queue, ".claimed"), { mode: 0o700 }); - const token = Buffer.alloc(32, 17).toString("base64url"); const trustedData = createFileTrustedOpsDataSource({ directory: queue, token, @@ -188,7 +237,8 @@ export async function ownedServiceHost() { loadAuthority: async () => { const a = await loadAuthority(); return { - revisionId: a.revisionId, + revisionId: configurationPolicyRevision(a.revisionId), + configurationAuthorityRevision: a.revisionId, services: a.services.map((s) => ({ serviceId: s.manifest.id, enabled: s.enabled, @@ -205,7 +255,8 @@ export async function ownedServiceHost() { ) ?? new Set(), trustedData, ownedServices: productionOwnedServices(runtime), - serviceMetadata: (ss) => ss.map((s) => ({ ...s, buildable: false })), + serviceMetadata: (ss) => ss.map((s) => ({ ...s, buildable: false, + creationDependencies:services.serviceCreationDependencyIds(ss.map(value=>value.selectionAuthority),[s.serviceId]) })), }); const context = async ( operation: OpsOperation, @@ -220,6 +271,8 @@ export async function ownedServiceHost() { }; let counter = 0; const select = async (selectedRoots: string[]) => { + const inspection = { kind: "configuration.sources.inspect" } as const; + const source = await dispatchOpsOperation(runtime, inspection, await context(inspection)); const sealed = sealTrustedConfigurationSnapshot({ role: "api", token, @@ -227,6 +280,8 @@ export async function ownedServiceHost() { operationKey: testExecution().operationKey, operationForRevision: (revisionId) => ({ kind: "serviceSelection.apply", + expectedSourceRevision: source.sourceRevision, + coordinatorMode: "ensure-running", revisionId, expectedCurrentRevisionId: services.currentConfigurationGeneration(infraDir), }), @@ -250,10 +305,13 @@ export async function ownedServiceHost() { const deselect = () => select([]); return { root, + cleanupRoot, + queue, token, infraDir, loaded, initial, + get competingRevisionId() { return competingRevisionId; }, firstCompose, stack, runtime, @@ -274,9 +332,18 @@ export async function ownedServiceHost() { setInventoryProcess(value: typeof inventoryProcess) { inventoryProcess = value; }, + setImagePullProcess(value:typeof imagePullProcess) {imagePullProcess=value;}, + setImageIdentity(reference:string,value:string) {imageIdentities.set(reference,value);}, setRows(value: typeof rows) { rows = value; }, + setServiceState(serviceId:string,state:string) { + const service=rows.find(value=>value.Labels["com.docker.compose.service"]===serviceId); + if(!service)throw Error("Named Docker fixture service unavailable"); + service.State=state; + }, + setRuntimeInspections(value: typeof runtimeInspections) { runtimeInspections = value; }, + setRuntimeSecretBytes(value: typeof runtimeSecretBytes) { runtimeSecretBytes = value; }, setRawListing(value: string) { rawListing = value; }, diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 594c29afe..b59ef05a5 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -1,3 +1,10 @@ +import { loadConfiguredResourceAuthority } from "./configured-resource-authority"; + +export { + type ConfiguredResourceAuthority, + loadConfiguredResourceAuthority, +} from "./configured-resource-authority"; + import { createHash, randomUUID } from "node:crypto"; import { accessSync, @@ -66,6 +73,7 @@ import { type OpsExecutionContext, OpsExecutionFailure, OpsHostUpdateRequiredError, + OpsNotWiredError, type OpsRuntime, } from "./runtime"; @@ -232,53 +240,6 @@ export function inspectRegionAuthority(regionId: string): boolean { return loadConfiguredResourceAuthority().regions.has(regionId); } -const COUNTRY_ID = /^[A-Z]{2,3}$/; - -export interface ConfiguredResourceAuthority { - revisionId: string; - regions: ReadonlySet; - countries: ReadonlySet; -} - -function exactConfiguredIds( - raw: string | undefined, - pattern: RegExp, - maximumLength: number, -): string[] { - const values = (raw ?? "") - .split(",") - .map((value) => value.trim()) - .filter(Boolean); - if ( - values.length > 64 || - values.some((value) => value.length > maximumLength || !pattern.test(value)) || - new Set(values).size !== values.length - ) { - throw new Error("Resource authority configuration rejected"); - } - return values.sort(); -} - -export function loadConfiguredResourceAuthority( - env: NodeJS.ProcessEnv = process.env, -): ConfiguredResourceAuthority { - const regions = exactConfiguredIds( - env.OPENMAPX_ALLOWED_REGIONS ?? "europe/germany", - REGION_ID, - 128, - ); - const countries = exactConfiguredIds(env.OPENMAPX_ALLOWED_COUNTRIES ?? "DE", COUNTRY_ID, 3); - const digest = createHash("sha256") - .update("openmapx-resource-authority-v1\0") - .update(JSON.stringify({ regions, countries })) - .digest("hex"); - return { - revisionId: `resources-${digest}`, - regions: new Set(regions), - countries: new Set(countries), - }; -} - export function inspectDataTypeAuthority( services: readonly DataAuthorityService[], dataTypeId: string, @@ -843,7 +804,14 @@ export function createAgentHandoffBindings(rootDir: string) { }; return { stackArgs: bindings.stackArgs, - restoreOverlay: bindings.restoreOverlay, + // Task08.3 installs the guarded helper. A parent observation cannot + // authorize this detached process or its rollback after the parent exits. + assertAvailable: (): never => { + throw new OpsNotWiredError(); + }, + restoreOverlay: (_previousOverlay: string | null): never => { + throw new OpsNotWiredError(); + }, withStoreLock: bindings.withStoreLock, releasePins: (releaseId: string, digest: string) => { const manifest = loadStored(releaseId, digest); @@ -852,11 +820,9 @@ export function createAgentHandoffBindings(rootDir: string) { helperImage: manifest.images["privacy-backup"], }; }, - writeOverlay: (releaseId: string, digest: string) => - coreServices.writeReleaseComposeArtifacts( - loadStored(releaseId, digest), - bindings.repositoryPaths.composeReleasePath, - ), + writeOverlay: (_releaseId: string, _digest: string): never => { + throw new OpsNotWiredError(); + }, }; } @@ -1341,7 +1307,6 @@ export function createDefaultReleaseEffects( initialize, validateCandidate, resolve: async (context) => { - await initialize(); manifestNames(); const selection = coreServices.releaseSelection(); if (selection.kind === "local") throw new coreServices.ReleaseChannelDisabledError(); @@ -1414,7 +1379,6 @@ export function createDefaultReleaseEffects( } }, pull: async (releaseId, context) => { - await initialize(); const manifest = loadManifest(releaseId); const names = [ "api", @@ -1438,11 +1402,9 @@ export function createDefaultReleaseEffects( } }, inspect: async () => { - await initialize(); return inspectRelease(); }, inspectSystem: async (context) => { - await initialize(); const release = await inspectRelease(); const manifest = release.availableReleaseId ? loadManifest(release.availableReleaseId) @@ -1680,7 +1642,6 @@ export function createDefaultReleaseEffects( }; }, runtimeInspect: async () => { - await initialize(); const state = readState(); return { ...(state.releaseId ? { releaseId: state.releaseId } : {}), diff --git a/apps/ops-agent/src/agent-handoff-main.ts b/apps/ops-agent/src/agent-handoff-main.ts index f9f789a00..fac4b30c5 100644 --- a/apps/ops-agent/src/agent-handoff-main.ts +++ b/apps/ops-agent/src/agent-handoff-main.ts @@ -20,6 +20,9 @@ async function main(): Promise { } const journal = join(rootDir, "infra", "docker", "data", "ops-agent", "jobs-v1.json"); const bindings = createAgentHandoffBindings(rootDir); + // Refuse before the legacy helper can advance a phase, drain, recreate or + // falsely report rollback. Task08.3 owns the real guarded installer. + bindings.assertAvailable(); const phase = await runAgentHandoff({ rootDir, handoffId, diff --git a/apps/ops-agent/src/bootstrap.test.ts b/apps/ops-agent/src/bootstrap.test.ts index 259d7e704..1ac3ee921 100644 --- a/apps/ops-agent/src/bootstrap.test.ts +++ b/apps/ops-agent/src/bootstrap.test.ts @@ -67,12 +67,17 @@ it.each(["legacy", "expired"])( const token = Buffer.alloc(32, 1).toString("base64url"); const retained = { role: "api" as const, - operation: { kind: "service.pull" as const, serviceId: "redis" }, + operation: { + kind: "service.pull" as const, + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, operationId: "job1_0123456789abcdef", operationKey: "opk1_0123456789abcdef", fingerprint: (await import("@openmapx/core/ops")).opsOperationFingerprint({ kind: "service.pull", serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, }), resourceId: "redis", state: "succeeded" as const, diff --git a/apps/ops-agent/src/configuration-candidate-source.ts b/apps/ops-agent/src/configuration-candidate-source.ts new file mode 100644 index 000000000..cbbc3653e --- /dev/null +++ b/apps/ops-agent/src/configuration-candidate-source.ts @@ -0,0 +1,104 @@ +import { constants } from "node:fs"; +import { lstat, open, readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { + inspectPreparedConfigurationRecord, + openTrustedConfigurationSnapshot, + opsOperationFingerprint, + opsOperationSchema, +} from "@openmapx/core/ops"; + +export interface ConfigurationCandidateSource { + directory: string; + token: string; + ownerUid: number; + ownerGid: number; +} +/** Reads only the fixed bounded candidate namespace, never a request path. */ +export async function readRetainedConfigurationCandidate( + source: ConfigurationCandidateSource, + candidateId: string, +) { + const directory = join(source.directory, ".candidates"); + const dir = await lstat(directory); + if ( + !dir.isDirectory() || + dir.isSymbolicLink() || + dir.uid !== source.ownerUid || + dir.gid !== source.ownerGid || + (dir.mode & 0o777) !== 0o700 + ) + throw Error("Configuration candidate unavailable"); + const entries = await readdir(directory, { withFileTypes: true }); + if (entries.length > 128) throw Error("Configuration candidate unavailable"); + let match: ReturnType | undefined; + for (const entry of entries) { + if (!entry.isFile() || !/^opk1_[A-Za-z0-9_-]{16,64}\.json$/.test(entry.name)) + throw Error("Configuration candidate unavailable"); + const path = join(directory, entry.name); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await handle.stat(), + current = await lstat(path); + if ( + !before.isFile() || + before.nlink !== 1 || + before.uid !== source.ownerUid || + before.gid !== source.ownerGid || + (before.mode & 0o777) !== 0o600 || + before.size > 768 * 1024 || + before.dev !== current.dev || + before.ino !== current.ino + ) + throw Error("Configuration candidate unavailable"); + const inspected = inspectPreparedConfigurationRecord(await handle.readFile(), source.token); + const after = await handle.stat(), + latest = await lstat(path); + if ( + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + before.dev !== latest.dev || + before.ino !== latest.ino || + `${inspected.record.operationKey}.json` !== entry.name + ) + throw Error("Configuration candidate unavailable"); + if (inspected.record.candidateId === candidateId) { + if (match) throw Error("Configuration candidate identity conflict"); + match = inspected; + } + } finally { + await handle.close(); + } + } + if (!match) throw Error("Configuration candidate missing"); + const operation = opsOperationSchema.parse(match.record.operation); + if ( + ![ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ].includes(operation.kind) + ) + throw Error("Configuration candidate typed kind rejected"); + // Authenticated historical metadata only. It cannot open a payload, consume + // a claim or renew the original seal; NoGuard recovery may observe expiry. + return { ...match, operation }; +} +export async function readConfigurationCandidate( + source: ConfigurationCandidateSource, + candidateId: string, +) { + const match = await readRetainedConfigurationCandidate(source, candidateId), + operation = match.operation; + const { payload } = openTrustedConfigurationSnapshot(match.sealed, { + role: "api", + token: source.token, + operationKey: match.record.operationKey, + operation, + fingerprint: opsOperationFingerprint(operation), + }); + return { ...match, operation, payload }; +} diff --git a/apps/ops-agent/src/configuration-hardlinks-runtime.ts b/apps/ops-agent/src/configuration-hardlinks-runtime.ts new file mode 100644 index 000000000..ae3924404 --- /dev/null +++ b/apps/ops-agent/src/configuration-hardlinks-runtime.ts @@ -0,0 +1,90 @@ +import { + executeOwnedGenerationHardlinks, + type GenerationHardlinksOwnerPlan, + planGenerationHardlinksOwner, + readGenerationHardlinksOwnerPlan, +} from "@openmapx/core/build-artifacts"; +import { + createDataManagerTransport, + DataExecutionObservationPending, + getStatus, + unwrapDataManagerResponse, +} from "@openmapx/core/data-manager-sdk"; +import type { ExecutionRef, OpsOperation } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { OpsExecutionFailure, type OpsRuntime } from "./runtime"; + +const sources = new WeakMap(); +/** The production factory's fixed source; callers cannot inject a handler or a plan. */ +export function prepareConfigurationHardlinksAdmission( + runtime: OpsRuntime, + parent: ExecutionRef, + operation: Extract, + serviceIds: readonly string[], +): GenerationHardlinksOwnerPlan { + const source = sources.get(runtime); + if (!source) throw Error("Production hardlink source unavailable"); + return planGenerationHardlinksOwner({ + ...source, + parent, + revisionId: operation.revisionId, + coordinatorMode: operation.coordinatorMode, + serviceIds, + }); +} +export function configurationDataManagerTransport() { + return createDataManagerTransport({ + baseUrl: process.env.DATA_MANAGER_URL?.trim() || "http://data-manager:4000", + authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), + requestTimeoutMs: 15000, + maxJsonBytes: 65536, + }); +} +export async function assertConfigurationCoordinatorReady(signal?: AbortSignal) { + const status = unwrapDataManagerResponse( + await getStatus({ transport: configurationDataManagerTransport(), signal }), + ); + if (!status.ok || status.status !== "ready") throw Error("Configuration coordinator unavailable"); +} +export function installConfigurationHardlinksRuntime( + runtime: OpsRuntime, + options: { infraDir: string }, +): void { + sources.set(runtime, options); + runtime["configuration.hardlinks.apply"] = async (operation, context) => { + const authority = context.controlAuthority, + lease = context.artifactLease ?? context.ownedTargetLease; + if (!authority || !lease) throw Error("Original hardlink root authority required"); + const plan = readGenerationHardlinksOwnerPlan(authority); + if ( + plan.revisionId !== operation.revisionId || + plan.coordinatorMode !== operation.coordinatorMode + ) + throw Error("Hardlink owner binding conflict"); + // A stopped coordinator never causes a lifecycle fallback inside this owner. + if (plan.plan.plan.length) await assertConfigurationCoordinatorReady(context.signal); + try { + return await services.withConfigurationGenerationWriter( + options.infraDir, + async () => { + if (plan.plan.plan.length) await assertConfigurationCoordinatorReady(context.signal); + return executeOwnedGenerationHardlinks({ + authority, + lease, + ...options, + transport: configurationDataManagerTransport(), + signal: context.signal, + }); + }, + context.signal, + ); + } catch (error) { + let cause = error; + while (cause instanceof services.ConfigurationGenerationError && cause.cause instanceof Error) + cause = cause.cause; + if (cause instanceof DataExecutionObservationPending) + throw new OpsExecutionFailure("recovery_required", {}); + throw error; + } + }; +} diff --git a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts new file mode 100644 index 000000000..b0e93766c --- /dev/null +++ b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts @@ -0,0 +1,269 @@ +/** Fixed production Docker inspector. Only execFile is an external-I/O seam; + * no injected result/inspector can qualify the original maintenance final. */ +import { createHmac, type KeyObject, timingSafeEqual } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, open, realpath } from "node:fs/promises"; +import { canonicalJson } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { + maintenanceDockerComposeSchema as compose, + maintenanceDockerContainerSchema as container, +} from "../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records"; +import type { ConfigurationMaintenanceOwnerRecord } from "./configuration-maintenance-store"; +import type { DockerRuntimeOptions } from "./docker-runtime"; +import type { OwnedServiceSnapshot } from "./owned-services"; + +export async function inspectMaintenanceDockerRuntime(options: { + stack: services.StackPaths; + execFile: NonNullable; + owner: ConfigurationMaintenanceOwnerRecord; + containers: readonly OwnedServiceSnapshot[]; + privateKey: KeyObject; + signal: AbortSignal; +}) { + const { owner, containers, signal } = options, + checks = [], + details = [], + inspectedRevision = + owner.receipt.generation === "not_committed" + ? owner.plan.operation.expectedCurrentRevisionId + : owner.plan.operation.revisionId, + ids = [ + ...new Set( + owner.plan.steps + .filter((value) => value.phase === "maintenance-recreate") + .map((value) => ("serviceId" in value.operation ? value.operation.serviceId : "")), + ), + ].sort(); + const run = async (args: string[]) => { + signal.throwIfAborted(); + const response = await options.execFile("docker", args, { + signal: AbortSignal.any([signal, AbortSignal.timeout(15000)]), + timeout: 15000, + maxBuffer: 1024 * 1024, + }); + if (Buffer.byteLength(response.stdout) > 1024 * 1024) + throw Error("Maintenance Docker observation capacity"); + signal.throwIfAborted(); + return response.stdout; + }, + keyed = (value: string | Buffer) => + createHmac("sha256", options.privateKey.export({ type: "pkcs8", format: "der" })) + .update("openmapx-private-maintenance-runtime-v1\0") + .update(value) + .digest("hex"), + lock = await services.acquireReleaseStoreLock(options.stack.infraDir, {}, { failIfBusy: true }); + try { + if ( + !inspectedRevision || + services.currentConfigurationGeneration(options.stack.infraDir) !== inspectedRevision + ) + throw Error("Original inspected generation changed"); + const args = services.stackComposeArgs(options.stack), + expected = compose.parse(JSON.parse(await run([...args, "config", "--format", "json"]))); + for (const id of ids) { + const current = containers.filter((value) => value.serviceId === id), + original = owner.originalContainers.filter((value) => value.serviceId === id), + spec = expected.services[id]; + if (!spec || current.length !== 1 || original.length !== 1 || current[0]!.state !== "running") + throw Error("Original applied container expectation unavailable"); + const frozenImage = + id === "data-manager" && owner.receipt.generation === "committed" + ? owner.plan.coordinatorImage + : undefined, + expectedImageId = frozenImage?.imageId ?? original[0]!.provenance.imageId; + if (frozenImage && spec.image !== frozenImage.imageReference) + throw Error("Frozen coordinator image reference changed"); + const owned = current[0]!, + observed = container.parse( + JSON.parse( + ( + await run(["container", "inspect", "--format", "{{json .}}", owned.containerId]) + ).trim(), + ), + ), + hashLine = (await run([...args, "config", "--hash", id])).trim().split(/\s+/), + hash = hashLine.at(-1); + if ( + hashLine.length !== 2 || + hashLine[0] !== id || + !hash || + !/^[a-f0-9]{64}$/.test(hash) || + observed.Id !== owned.containerId || + observed.Config.Labels["com.docker.compose.config-hash"] !== hash || + observed.Config.Image !== spec.image + ) + throw Error("Original applied runtime configuration differs"); + const actualEnv = Object.fromEntries( + observed.Config.Env.map((value) => { + const separator = value.indexOf("="); + if (separator < 1) throw Error("Malformed runtime environment"); + return [value.slice(0, separator), value.slice(separator + 1)]; + }), + ), + expectedEnv = Object.fromEntries( + Object.entries(spec.environment).map(([key, value]) => [ + key, + value === null ? "" : String(value), + ]), + ); + if (Object.keys(expectedEnv).some((key) => actualEnv[key] !== expectedEnv[key])) + throw Error("Original applied runtime environment differs"); + checks.push({ + serviceId: id, + ordinal: null, + check: "configuration", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + details.push({ + containerId: owned.containerId, + expectedConfigHash: hash, + actualConfigHash: observed.Config.Labels["com.docker.compose.config-hash"], + expectedEnvironmentDigest: keyed(canonicalJson(expectedEnv)), + actualEnvironmentDigest: keyed( + canonicalJson( + Object.fromEntries(Object.keys(expectedEnv).map((key) => [key, actualEnv[key]])), + ), + ), + }); + if (observed.Image !== expectedImageId || observed.Image !== owned.provenance.imageId) + throw Error("Original applied image differs"); + checks.push({ + serviceId: id, + ordinal: null, + check: "image", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + details.push({ + containerId: owned.containerId, + imageId: observed.Image, + expectedImageId, + }); + const mounts = []; + for (const volume of spec.volumes) { + const matches = observed.Mounts.filter((value) => value.Destination === volume.target), + actual = matches[0]; + if ( + matches.length !== 1 || + !actual || + actual.Type !== volume.type || + actual.RW !== !volume.read_only + ) + throw Error("Original applied mount differs"); + if ( + volume.type === "bind" + ? !volume.source || + !actual.Source || + (await realpath(volume.source)) !== (await realpath(actual.Source)) + : volume.type === "volume" && + (!volume.source || + actual.Name !== (expected.volumes[volume.source]?.name ?? volume.source)) + ) + throw Error("Original applied mount source differs"); + mounts.push({ + targetDigest: keyed(volume.target), + type: actual.Type, + sourceDigest: volume.type === "bind" ? keyed(await realpath(volume.source!)) : null, + nameDigest: actual.Name ? keyed(actual.Name) : null, + readOnly: !actual.RW, + }); + } + for (const secret of spec.secrets) { + const path = expected.secrets[secret.source]?.file, + target = `/run/secrets/${secret.target ?? secret.source}`, + matches = observed.Mounts.filter((value) => value.Destination === target), + actual = matches[0]; + if ( + !path || + matches.length !== 1 || + !actual || + actual.Type !== "bind" || + actual.RW || + !actual.Source || + (await realpath(path)) !== (await realpath(actual.Source)) + ) + throw Error("Original applied secret mount differs"); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await file.stat(), + named = await lstat(path); + if ( + !before.isFile() || + before.nlink !== 1 || + before.size > 65536 * 4 || + before.dev !== named.dev || + before.ino !== named.ino + ) + throw Error("Original secret source unavailable"); + const expectedBytes = await file.readFile(), + encoded = (await run(["container", "exec", observed.Id, "base64", target])).replace( + /[\r\n]/g, + "", + ), + actualBytes = Buffer.from(encoded, "base64"), + expectedDigest = keyed(expectedBytes), + actualDigest = keyed(actualBytes), + after = await file.stat(), + latest = await lstat(path); + if ( + before.dev !== latest.dev || + before.ino !== latest.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + encoded !== actualBytes.toString("base64") || + !timingSafeEqual(Buffer.from(expectedDigest, "hex"), Buffer.from(actualDigest, "hex")) + ) + throw Error("Original runtime secret content differs"); + mounts.push({ + targetDigest: keyed(target), + type: actual.Type, + sourceDigest: keyed(await realpath(path)), + readOnly: true, + expectedDigest, + actualDigest, + }); + } finally { + await file.close(); + } + } + checks.push({ + serviceId: id, + ordinal: null, + check: "mount", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + details.push({ containerId: owned.containerId, mounts }); + if ( + observed.State.Status !== "running" || + !observed.State.Running || + (observed.State.Health && observed.State.Health.Status !== "healthy") || + (spec.healthcheck && + !spec.healthcheck.disable && + observed.State.Health?.Status !== "healthy") + ) + throw Error("Original applied container not ready"); + checks.push({ + serviceId: id, + ordinal: null, + check: "readiness", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + details.push({ + containerId: owned.containerId, + state: observed.State.Status, + running: observed.State.Running, + health: observed.State.Health?.Status ?? null, + }); + } + if (services.currentConfigurationGeneration(options.stack.infraDir) !== inspectedRevision) + throw Error("Original inspected generation changed"); + return { checks, details }; + } finally { + lock.release(); + } +} diff --git a/apps/ops-agent/src/configuration-maintenance-runtime.ts b/apps/ops-agent/src/configuration-maintenance-runtime.ts new file mode 100644 index 000000000..4310f47e5 --- /dev/null +++ b/apps/ops-agent/src/configuration-maintenance-runtime.ts @@ -0,0 +1,20 @@ +/** Stable production maintenance runtime interface. Actual nominal producers + * live beside the private owner they alone can initialize/settle. */ +export { + type ConfigurationMaintenanceAdmission, + configurationMaintenanceAdmissionState, + consumeMaintenanceContinueObservation, + inspectFrozenMaintenanceRuntime, + type MaintenanceContinueObservation, + type MaintenanceRuntimeInspection, + maintenanceRuntimeInspectionState, + prepareConfigurationMaintenanceAdmission, + prepareMaintenanceContinueObservation, + registerConfigurationMaintenanceSource, + type UnconsumedMaintenanceCandidate, + unconsumedMaintenanceCandidateState, + verifyFrozenMaintenanceGenerationSource, + verifyFrozenMaintenanceHardlinksSource, + verifyFrozenMaintenanceRecreateSource, + withUnconsumedMaintenanceCandidate, +} from "./configuration-maintenance-store"; diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts new file mode 100644 index 000000000..84fdf168d --- /dev/null +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -0,0 +1,5216 @@ +import { lstat, opendir } from "node:fs/promises"; +import { join } from "node:path"; +import { + observeCoordinatorPullForMaintenance, + observedCoordinatorPullBaseline, + ownedCoordinatorPullBaseline, + ownedStagedCoordinatorBaseline, + type VerifiedCoordinatorPullBaseline, + verifiedStagedCoordinatorOwnershipState, + verifyConfigurationReleaseSourceWitness, +} from "@openmapx/core/build-artifacts"; +import { coordinatorPullSubmissionDigest, parseOpsJobStatus } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { assertProspectiveConfigurationSettlementPacketBudget } from "../../../packages/core/src/build-artifacts/configuration-fence-settlement-budget"; +import { assertProspectiveConfigurationMaintenanceBudget } from "../../../packages/core/src/build-artifacts/configuration-maintenance-budget"; +import { + type CoordinatorPullWitness, + coordinatorPullObservationDomain, + coordinatorPullWitnessPayloadSchema, + type StagedCoordinatorObservation, + stagedCoordinatorObservationDomain, + stagedCoordinatorObservationPayloadSchema, + stagedCoordinatorObservationSchema, +} from "../../../packages/core/src/build-artifacts/coordinator-pull-records"; +import { readOwnedGenerationHardlinkPlan } from "../../../packages/core/src/build-artifacts/generation-hardlinks-owner"; +import { + type ConfigurationCandidateSource, + readConfigurationCandidate, + readRetainedConfigurationCandidate, +} from "./configuration-candidate-source"; +import { + assertConfigurationCoordinatorReady, + configurationDataManagerTransport, +} from "./configuration-hardlinks-runtime"; +import { inspectMaintenanceDockerRuntime } from "./configuration-maintenance-docker-inspection"; +import type { DockerRuntimeOptions } from "./docker-runtime"; +import type { OpsJobJournal } from "./journal"; +import { + inspectLiteralStagedConfigurationDependency, + inspectOriginalCoordinatorServices, + inspectStagedCoordinatorMaintenanceSource, + inspectStagedCoordinatorOwnerServices, + type OwnedServiceResolver, + type OwnedServiceSnapshot, +} from "./owned-services"; +import { coordinatorCandidateMetadata, renderCoordinatorCandidate } from "./service-pull-runtime"; +import type { TrustedConfigurationAuthoritySnapshot } from "./trusted-config-runtime"; +/** The fixed agent owner store. The only initializer consumes an actual + * production admission; no structural plan or caller-selected final state is + * an owner-store capability. */ + +import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; +import { + type ArtifactTargetLease, + adoptArtifactTargetLease, + assertConfigurationMaintenanceRootClosure, + assertFixedConfigurationMaintenancePlan, + bindControlChildPhase, + buildFixedConfigurationMaintenancePlan, + type ControlInstallation, + closeArtifactTargetLeaseHandle, + configurationMaintenanceTargetOwner, + consumeControlChildDispatch, + controlDigest, + controlRolePublicKey, + executionAuthorityState, + type FixedConfigurationMaintenancePlan, + inspectArtifactDelegation, + inspectArtifactTargetLeaseChildSettlement, + issueControlChildPhase, + lookupControlChildPhaseBinding, + preparedDataConfigurationFenceState, + publishedConfigurationSettlementCapacityState, + publishedDmNoGuardRefusalState, + publishedDmUnusedCapacityState, + readPrivateRecord, + reconcileArtifactTargetLease, + restoreAdmittedControlExecution, + restoreArtifactTargetLeaseForSettlement, + restoreConfigurationMaintenancePhaseIssuer, + retainControlEffectReceipt, + signControl, + type VerifiedExecutionAuthority, + type VerifiedPreparedDataConfigurationFence, + type VerifiedPublishedConfigurationSettlementCapacity, + verifiedParentForArtifactDelegation, + verifyConfigurationSourceWitness, + verifyControl, + verifyControlEffectSettlement, + verifyControlRecoveryAuthority, + verifyFamilyOwnedRecord, + verifyMaintenanceCoordinatorPrerequisite, + verifyPublishedConfigurationSettlementCapacity, + verifyPublishedDataConfigurationFencePrepared, + verifyPublishedDataConfigurationFenceRefusal, + verifyPublishedUnusedConfigurationCapacity, + verifyRetainedArtifactTargetRootRelease, + verifyRetainedLocalControlPeer, + withConfigurationMaintenanceDecisionPublication, + withConfigurationMaintenanceNoGuardPublication, + withConfigurationMaintenanceOwnerInitialization, + withConfigurationMaintenanceOwnerObservation, + withConfigurationMaintenancePhasePublication, + withConfigurationMaintenanceSettlementPublication, + writePrivateRecord, +} from "@openmapx/core/build-artifacts"; +import { + type DataManagerTransport, + inspectDataConfigurationFence, + prepareDataConfigurationFence, + settleDataConfigurationFence, +} from "@openmapx/core/data-manager-sdk"; +import { + acquireTrustedConfigurationQueueLock, + type ConfigurationMaintenanceDecision, + type ConfigurationMaintenancePrepare, + type ConfigurationMaintenanceReceipt, + type ConfigurationMaintenanceSettle, + canonicalJson, + configurationMaintenancePrepareSchema, + configurationMaintenanceSettleRequestDigest, + controlTerminalReceiptDigest, + type ExecutionRef, + executionFingerprint, + executionRefSchema, + type OpsJobStatus, + type OpsOperation, + opsOperationFingerprint, + opsResourceId, + type RetainedParentExecution, + type TrustedConfigurationQueueLock, + type VerifiedParentExecution, + verifiedApiJobParentBinding, + verifyApiRetentionProof, +} from "@openmapx/core/ops"; +import { + inspectControlChildPublication, + lookupControlChildPhaseAttempt, +} from "../../../packages/core/src/build-artifacts/child-phase"; +import { + assertNoUnissuedConfigurationReceiverPublication, + configurationSettlementRecordDescriptors, + readConfigurationSettlementCell, + writeConfigurationSettlementCell, +} from "../../../packages/core/src/build-artifacts/configuration-fence-settlement-cells"; +import { + configurationCommittedSettlementSchema, + configurationDmCapacityCellSchema, + configurationFinalEvidenceProjectionSchema, + configurationFinalEvidenceSchema, + configurationIssuedPrepareRefusedProofSchema, + configurationIssuerUnusedSourceDigest, + configurationNeverIssuedProofSchema, + configurationNoGuardBundleSchema, + configurationNoGuardClosureSchema, + configurationNoGuardEvidenceSchema, + configurationNoGuardIntentSchema, + configurationOwnerFinalSchema, + configurationSettlementAckSchema, + configurationSettlementAdmissionSchema, + configurationSettlementAssertionSchema, + configurationSettlementCapacitySchema, + configurationSettlementCommitmentCellSchema, + configurationSettlementControlCellSchema, + configurationSettlementControlEnvelopeSchema, + configurationSettlementDispatchSchema, + configurationSettlementDomains, + configurationSettlementEffectSchema, + configurationSettlementOwnerIdentitySchema, + configurationSettlementPublishedTerminalSchema, + configurationSettlementReservationProjectionSchema, + configurationSettlementSenderAckCellSchema, + configurationUnusedCapacityConfirmationSchema, + configurationUnusedCapacityIntentSchema, + configurationSettlementDomains as domains, + settlementRecordDigest, +} from "../../../packages/core/src/build-artifacts/configuration-fence-settlement-records"; +import { readOriginalMaintenanceFinalChildren } from "../../../packages/core/src/build-artifacts/configuration-maintenance-final-children"; +import { + assertAppliedConfigurationMaintenanceFinalState, + assertContainedConfigurationMaintenanceFinalState, + assertUnusedConfigurationMaintenanceFinalState, + configurationMaintenanceFinalRecordSchema, + configurationMaintenanceRequiredChecksSchema, + configurationMaintenanceRuntimeInspectionSchema, +} from "../../../packages/core/src/build-artifacts/configuration-maintenance-final-records"; +import { + assertConfigurationMaintenanceDecisionSource, + type ConfigurationMaintenanceOwnerRecord, + configurationMaintenanceControlSnapshotSchema, + configurationMaintenanceDecisionCommitmentDomain, + configurationMaintenanceDecisionCommitmentPayload, + configurationMaintenanceDecisionCommitmentPayloadSchema, + configurationMaintenanceDecisionSourceSchema, + configurationMaintenanceReleasedOwner, + configurationMaintenanceSettleCompletion, + configurationMaintenanceSettleControlSourceSchema, + configurationMaintenanceStepControlSourceSchema, + configurationMaintenanceOwnerRecordSchema as ownerSchema, +} from "../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records"; +import { + configurationMaintenanceChildState, + installationState, + publishConfigurationMaintenanceForwardWitness, + verifyConfigurationMaintenanceChildTerminal, + withPrivateEffectWriter, +} from "../../../packages/core/src/build-artifacts/control-authority"; +import { CONTROL_RETENTION_LIMITS } from "../../../packages/core/src/build-artifacts/control-retention"; +import { AnchoredDirectory } from "../../../packages/core/src/build-artifacts/filesystem"; +import { + bindOwnedConfigurationSettlementFamilySlot, + commitFrozenConfigurationSettlementSlotUnused, + readConfigurationSettlementFamilySlot, + repairIndexedConfigurationMaintenanceSettleCompletion, + verifyPublishedBoundDmConfigurationSettlementSlots, + verifyPublishedDmConfigurationSettlementSlots, +} from "../../../packages/core/src/build-artifacts/retention-family"; +import { observeConfigurationMaintenanceRuntimeTree } from "../../../packages/core/src/build-artifacts/target-lease"; + +import type { PersistedOpsJob } from "./journal"; +import { OpsExecutionFailure, type OpsRuntime } from "./runtime"; + +export type { ConfigurationMaintenanceOwnerRecord }; + +const domain = "openmapx-configuration-maintenance-owner-v2"; +function ownerName(authority: VerifiedExecutionAuthority): string { + const state = executionAuthorityState(authority); + const admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: unknown } }; + return `${configurationMaintenancePrepareSchema.parse(admitted.payload?.operation).maintenanceId}.json`; +} +function assertOwner( + authority: VerifiedExecutionAuthority, + owner: ConfigurationMaintenanceOwnerRecord, +): void { + const state = executionAuthorityState(authority), + parent = owner.plan.apiParent, + admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: unknown; parent?: unknown } }, + { kind: _kind, ...binding } = owner.plan.operation; + assertFixedConfigurationMaintenancePlan(owner.plan, owner.installationId); + assertConfigurationMaintenanceDecisionSource(owner); + if (owner.apiControls.decide) + verifyControl( + state.installation.installation.publicKey, + configurationMaintenanceDecisionCommitmentDomain, + owner.apiControls.decide.commitment, + configurationMaintenanceDecisionCommitmentPayloadSchema, + ); + for (const source of owner.apiControls.steps) { + const step = owner.plan.steps[source.ordinal], + reservation = owner.plan.reservations.find((value) => value.ordinal === source.ordinal), + binding = reservation && lookupControlChildPhaseBinding(authority, reservation), + publication = + reservation && inspectControlChildPublication(authority, reservation).publication; + if ( + !step || + !binding || + canonicalJson(binding.child.execution) !== canonicalJson(source.execution) || + (source.grant && + (!publication || + publication.state !== "grant_prepared" || + publication.leaseId !== source.grant.leaseId || + publication.delegationId !== source.grant.delegationId || + publication.grantDigest !== source.grant.grantDigest)) + ) + throw Error("Original frozen maintenance phase source changed"); + if (source.grant) { + // Private integrity only. A signed/indexed original phase cell binds + // these exact original outbox bytes. Physical tree verification belongs + // to the caller's lease scope/adoption; reopening its writer here nests + // the same lease journal while that scope is held. + const outbox = readPrivateRecord( + state.installation, + "attempts", + `${source.grant.delegationId}.json`, + 65536, + ) as { grant?: unknown }; + if (controlDigest(outbox.grant) !== source.grant.grantDigest) + throw Error("Original frozen phase outbox changed"); + } + } + if ( + state.installation.fixedChildId || + state.installation.installation.role !== "ops-agent" || + state.kind !== "configurationMaintenance.prepare" || + !state.origin || + owner.installationId !== state.installation.installation.installationId || + controlDigest(owner.origin) !== controlDigest(state.origin) || + owner.admissionDigest !== state.receiptDigest || + canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) || + controlDigest(admitted) !== state.receiptDigest || + canonicalJson(owner.plan.operation) !== canonicalJson(admitted.payload?.operation) || + canonicalJson(parent) !== canonicalJson(admitted.payload?.parent) || + canonicalJson(owner.receipt.binding) !== canonicalJson(binding) || + canonicalJson(owner.plan.reservations) !== canonicalJson(state.reservations) || + canonicalJson(owner.receipt.prepare) !== canonicalJson(state.execution) || + owner.receipt.planDigest !== owner.plan.planDigest || + owner.receipt.binding.maintenanceId !== owner.plan.operation.maintenanceId || + owner.receipt.binding.candidateId !== owner.plan.operation.candidateId || + parent.issuer !== "api" || + owner.receipt.apiJobId !== parent.apiJobId || + owner.receipt.parentOperationKey !== parent.operationKey || + owner.receipt.parentFingerprint !== parent.fingerprint || + owner.receipt.parentDeadlineAt !== parent.deadlineAt || + canonicalJson(owner.receipt.steps) !== canonicalJson(owner.plan.steps) || + canonicalJson(owner.receipt.serviceIds) !== canonicalJson(owner.plan.serviceIds) || + owner.apiParentAttestationDigest !== parent.assertionDigest || + owner.sequence !== owner.receipt.sequence + ) + throw Error("Original configuration maintenance owner identity changed"); + const expectedImage = owner.plan.coordinatorImage; + if (owner.stagedCoordinatorProvenance) { + const copy = verifyControl( + state.installation.installation.publicKey, + stagedCoordinatorObservationDomain, + owner.stagedCoordinatorProvenance, + stagedCoordinatorObservationPayloadSchema, + ); + if ( + owner.coordinatorPull || + expectedImage?.action !== "apply" || + copy.installationId !== owner.installationId || + canonicalJson(copy.ownerPrepare) !== canonicalJson(owner.plan.prepare) || + copy.candidateId !== binding.candidateId || + copy.planDigest !== owner.plan.planDigest || + copy.source.baseline.configurationRevisionId !== binding.expectedCurrentRevisionId || + canonicalJson(owner.originalContainers.find((c) => c.serviceId === "data-manager")) !== + canonicalJson(copy.source.baseline.container) + ) + throw Error("Original staged owner observation changed"); + } + if (expectedImage?.action === "update") { + if (!owner.coordinatorPull) throw Error("Original copied coordinator pull required"); + const pull = verifyControl( + state.installation.installation.publicKey, + coordinatorPullObservationDomain, + owner.coordinatorPull, + coordinatorPullWitnessPayloadSchema, + ); + if ( + pull.installationId !== owner.installationId || + pull.maintenanceId !== binding.maintenanceId || + pull.candidateId !== binding.candidateId || + canonicalJson(pull.maintenancePrepare) !== canonicalJson(owner.plan.prepare) || + pull.sourceRevision !== binding.expectedSourceRevision || + pull.revisionId !== binding.expectedCurrentRevisionId || + pull.submissionDigest !== expectedImage.pullSubmissionDigest || + pull.imageId !== expectedImage.imageId || + pull.imageReference !== expectedImage.imageReference || + pull.releaseSelectionDigest !== expectedImage.releaseSelectionDigest + ) + throw Error("Original coordinator pull/image expectation changed"); + } else if (owner.coordinatorPull) throw Error("Unexpected coordinator pull observation"); +} +function read(authority: VerifiedExecutionAuthority): ConfigurationMaintenanceOwnerRecord { + const state = executionAuthorityState(authority), + name = ownerName(authority), + raw = readPrivateRecord(state.installation, "configuration-maintenance", name); + const owner = verifyControl(state.installation.installation.publicKey, domain, raw, ownerSchema); + assertOwner(authority, owner); + verifyFamilyOwnedRecord( + state.installation, + owner.origin, + { scope: "private", namespace: "configuration-maintenance", name, container: "" }, + canonicalJson(raw), + ); + return owner; +} +function write( + authority: VerifiedExecutionAuthority, + value: ConfigurationMaintenanceOwnerRecord, + exclusive = false, +): void { + const state = executionAuthorityState(authority), + owner = ownerSchema.parse(value); + assertOwner(authority, owner); + const signed = signControl(state.installation.privateKey, domain, owner); + if (Buffer.byteLength(canonicalJson(signed)) > 128 * 1024) + throw Error("Configuration maintenance owner capacity exceeded"); + writePrivateRecord( + state.installation, + "configuration-maintenance", + ownerName(authority), + signed, + exclusive, + owner.origin, + ); +} +export function retainOwnedConfigurationMaintenance( + authority: VerifiedExecutionAuthority, + lease: ArtifactTargetLease, + admission: ConfigurationMaintenanceAdmission, +): void { + const state = executionAuthorityState(authority), + source = configurationMaintenanceAdmissionState(admission), + root = configurationMaintenanceTargetOwner(lease), + plan = source.plan; + if ( + !state.origin || + canonicalJson(root.execution) !== canonicalJson(plan.prepare) || + canonicalJson(root.serviceIds) !== canonicalJson(plan.serviceIds) || + plan.apiParent.issuer !== "api" + ) + throw Error("Actual admitted maintenance closure required"); + const { kind: _kind, ...binding } = plan.operation; + const owner = ownerSchema.parse({ + version: 2, + installationId: state.installation.installation.installationId, + origin: state.origin, + admissionDigest: state.receiptDigest, + rootCertificateDigest: root.certificateDigest, + apiParentAttestationDigest: plan.apiParent.assertionDigest, + plan, + originalContainers: source.originalContainers, + ...(source.coordinatorPull ? { coordinatorPull: source.coordinatorPull } : {}), + ...(source.stagedCoordinatorProvenance + ? { stagedCoordinatorProvenance: source.stagedCoordinatorProvenance } + : {}), + sequence: 0, + guardSource: null, + capacity: null, + apiControls: { decide: null, settle: null, steps: [] }, + final: null, + inspection: null, + issuance: "open", + noGuard: null, + receipt: { + version: 1, + binding, + prepare: plan.prepare, + apiJobId: plan.apiParent.apiJobId, + parentOperationKey: plan.apiParent.operationKey, + parentFingerprint: plan.apiParent.fingerprint, + parentDeadlineAt: plan.apiParent.deadlineAt, + planDigest: plan.planDigest, + serviceIds: plan.serviceIds, + steps: plan.steps, + preflight: "current", + sequence: 0, + state: "preparing", + decision: null, + dmFence: null, + generation: "not_started", + appliedRevisionId: null, + followUp: "not_started", + }, + }); + withConfigurationMaintenanceOwnerInitialization(lease, authority, () => + write(authority, owner, true), + ); + publishConfigurationMaintenanceForwardWitness(authority); +} +export function readOwnedConfigurationMaintenance( + authority: VerifiedExecutionAuthority, +): ConfigurationMaintenanceOwnerRecord { + return read(authority); +} +declare const controlOwnerBrand: unique symbol; +export type ConfigurationMaintenanceControlOwner = Readonly<{ [controlOwnerBrand]: true }>; +const controlOwners = new WeakMap< + ConfigurationMaintenanceControlOwner, + Readonly<{ + authority: VerifiedExecutionAuthority; + requestKey: string; + operationFingerprint: string; + parent: VerifiedParentExecution; + requestedDeadlineAt: number; + }> +>(); +export class ConfigurationMaintenanceControlUnauthorized extends Error {} +export class ConfigurationMaintenanceControlConflict extends Error {} +/** Fixed original-owner lookup consumes the actual live API HMAC binding. + * UUIDs, parsed parents, a body hash or public prepared status grant nothing. */ +export function admitConfigurationMaintenanceControlOwner( + installation: ControlInstallation, + operation: Extract< + OpsOperation, + { + kind: + | "configurationMaintenance.inspect" + | "configurationMaintenance.decide" + | "configurationMaintenance.settle"; + } + >, + parent: VerifiedParentExecution | undefined, + request: { operationKey: string; requestedDeadlineAt?: number }, +): ConfigurationMaintenanceControlOwner { + const actor = verifiedApiJobParentBinding(parent), + fingerprint = opsOperationFingerprint(operation), + local = installationState(installation); + if ( + installation.role !== "ops-agent" || + local.fixedChildId || + actor.childOperationKey !== request.operationKey || + actor.childOperationFingerprint !== fingerprint || + actor.requestedDeadlineAt !== request.requestedDeadlineAt + ) + throw new ConfigurationMaintenanceControlUnauthorized("Original API control body required"); + const raw = readPrivateRecord( + local, + "configuration-maintenance", + `${operation.maintenanceId}.json`, + ), + observed = verifyControl(installation.publicKey, domain, raw, ownerSchema), + authority = restoreAdmittedControlExecution(installation, observed.plan.prepare), + owner = read(authority), + original = owner.plan.apiParent; + if ( + owner.plan.operation.maintenanceId !== operation.maintenanceId || + owner.plan.operation.candidateId !== operation.candidateId || + actor.apiJobId !== original.apiJobId || + actor.parentOperationKey !== original.operationKey || + actor.parentFingerprint !== original.fingerprint || + actor.parentDeadlineAt !== original.deadlineAt + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Original API maintenance actor required", + ); + if ( + operation.kind !== "configurationMaintenance.inspect" && + (operation.prepareOperationKey !== owner.plan.prepare.operationKey || + operation.prepareFingerprint !== owner.plan.prepare.operationFingerprint || + operation.planDigest !== owner.plan.planDigest) + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Original prepared decision binding required", + ); + const value = Object.freeze({}) as ConfigurationMaintenanceControlOwner; + controlOwners.set( + value, + Object.freeze({ + authority, + requestKey: request.operationKey, + operationFingerprint: fingerprint, + parent: parent!, + requestedDeadlineAt: actor.requestedDeadlineAt, + }), + ); + return value; +} +export function configurationMaintenanceControlOwnerState( + value: ConfigurationMaintenanceControlOwner | undefined, + operation: OpsOperation, + requestKey: string, +) { + const source = value && controlOwners.get(value); + if ( + !source || + source.requestKey !== requestKey || + source.operationFingerprint !== opsOperationFingerprint(operation) + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual original maintenance control source required", + ); + executionAuthorityState(source.authority); + return source; +} +/** Fixed original-owner local generation admission. Actual API HMAC provenance, + * candidate/guard preflight and retained one-use phase are all mandatory. The + * ordinary transport peer resolver is unchanged; no API self-peer is accepted. */ +export async function prepareOwnedConfigurationMaintenanceStepAdmission( + runtime: OpsRuntime, + installation: ControlInstallation, + operation: OpsOperation, + parent: VerifiedParentExecution | undefined, + request: { + operationKey: string; + requestedDeadlineAt?: number; + parentAssertion: string; + signal: AbortSignal; + }, +) { + const actor = verifiedApiJobParentBinding(parent), + local = installationState(installation), + fingerprint = opsOperationFingerprint(operation); + if ( + installation.role !== "ops-agent" || + local.fixedChildId || + !parent || + parent.issuer !== "api" || + !("maintenanceId" in operation) || + !operation.maintenanceId || + actor.childOperationKey !== request.operationKey || + actor.childOperationFingerprint !== fingerprint || + actor.requestedDeadlineAt !== request.requestedDeadlineAt || + createHash("sha256").update(request.parentAssertion).digest("hex") !== parent.assertionDigest + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual API maintenance phase request required", + ); + const observed = verifyControl( + installation.publicKey, + domain, + readPrivateRecord(local, "configuration-maintenance", `${operation.maintenanceId}.json`), + ownerSchema, + ), + authority = restoreAdmittedControlExecution(installation, observed.plan.prepare), + owner = read(authority), + original = owner.plan.apiParent, + step = owner.plan.steps.find( + (value) => + value.operationKey === request.operationKey && value.operationFingerprint === fingerprint, + ); + if ( + !step || + !["configuration-commit", "maintenance-hardlinks", "maintenance-recreate"].includes( + step.phase, + ) || + canonicalJson(step.operation) !== canonicalJson(operation) || + actor.apiJobId !== original.apiJobId || + actor.parentOperationKey !== original.operationKey || + actor.parentFingerprint !== original.fingerprint || + actor.parentDeadlineAt !== original.deadlineAt || + request.requestedDeadlineAt !== owner.plan.prepare.deadlineAt || + owner.receipt.decision?.decision !== "continue" || + !owner.apiControls.decide?.receiptDelivery + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Original authorized frozen generation phase required", + ); + const reservation = owner.plan.reservations.find( + (value) => value.ordinal === step.ordinal && value.phase === step.phase, + )!; + let source = owner.apiControls.steps.find((value) => value.ordinal === step.ordinal), + freshDispatch: Awaited>["dispatchTicket"] | undefined; + if (!source) { + if (step.phase === "configuration-commit") + await verifyFrozenMaintenanceGenerationSource(runtime, authority, request.signal); + else if (step.phase === "maintenance-hardlinks") + await verifyFrozenMaintenanceHardlinksSource(runtime, authority, request.signal); + else if (step.phase === "maintenance-recreate") + await verifyFrozenMaintenanceRecreateSource(runtime, authority, step.ordinal, request.signal); + else + throw new ConfigurationMaintenanceControlUnauthorized( + "Uninstalled frozen maintenance continuation", + ); + publishConfigurationMaintenanceForwardWitness(authority); + const lease = restoreConfigurationMaintenancePhaseIssuer(authority, step.phase); + try { + if (lookupControlChildPhaseBinding(authority, reservation)) + throw new ConfigurationMaintenanceControlConflict( + "Attempted frozen phase requires original observation", + ); + const bound = await bindControlChildPhase(authority, reservation); + source = configurationMaintenanceStepControlSourceSchema.parse({ + version: 1, + ordinal: step.ordinal, + execution: bound.execution, + apiParent: parent, + parentAssertion: request.parentAssertion, + requestedDeadlineAt: request.requestedDeadlineAt, + receivedAt: Date.now(), + grant: null, + canonical: null, + receiptDelivery: null, + }); + const retained = source; + withConfigurationMaintenancePhasePublication(lease, authority, step.phase, () => { + const current = read(authority); + if (canonicalJson(current) !== canonicalJson(owner)) + throw new ConfigurationMaintenanceControlConflict("Original frozen phase owner raced"); + const sequence = current.sequence + 1; + write(authority, { + ...current, + sequence, + receipt: { ...current.receipt, sequence, state: "applying" }, + apiControls: { + ...current.apiControls, + steps: [...current.apiControls.steps, retained].sort((a, b) => a.ordinal - b.ordinal), + }, + }); + }); + const issued = await issueControlChildPhase(authority, { + phase: step.phase, + ordinal: step.ordinal, + lease, + childPublicKey: controlRolePublicKey(installation, "ops-agent"), + }); + freshDispatch = issued.dispatchTicket; + const attempted = lookupControlChildPhaseAttempt(authority, reservation); + if ( + !attempted || + attempted.channel !== "delegation" || + attempted.assertion !== issued.assertion + ) + throw new ConfigurationMaintenanceControlConflict("Original local phase grant required"); + source = configurationMaintenanceStepControlSourceSchema.parse({ + ...retained, + grant: { + leaseId: attempted.leaseId, + delegationId: attempted.delegationId, + grantDigest: attempted.grantDigest, + }, + }); + const granted = source; + withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const current = read(authority), + previous = current.apiControls.steps.find((value) => value.ordinal === step.ordinal); + if (canonicalJson(previous) !== canonicalJson(retained)) + throw new ConfigurationMaintenanceControlConflict("Original local phase intent raced"); + write(authority, { + ...current, + apiControls: { + ...current.apiControls, + steps: current.apiControls.steps.map((value) => + value.ordinal === step.ordinal ? granted : value, + ), + }, + }); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } + } else if ( + canonicalJson(source.apiParent) !== canonicalJson(parent) || + source.parentAssertion !== request.parentAssertion + ) + throw new ConfigurationMaintenanceControlConflict("Original local phase request changed"); + const binding = lookupControlChildPhaseBinding(authority, reservation), + attempted = lookupControlChildPhaseAttempt(authority, reservation); + if (!source.grant || !binding || !attempted || attempted.channel !== "delegation") + throw new ConfigurationMaintenanceControlConflict( + "Original attempted local phase observation required", + ); + const inspected = inspectArtifactDelegation({ + dataRoot: local.dataRoot, + installation, + assertion: attempted.assertion, + }), + roleParent = verifiedParentForArtifactDelegation( + installation, + attempted.assertion, + binding.child.execution, + ), + assertion = attempted.assertion; + if ( + canonicalJson(inspected.execution) !== canonicalJson(source.execution) || + inspected.kind !== operation.kind || + inspected.leaseId !== source.grant.leaseId || + inspected.delegationId !== source.grant.delegationId + ) + throw new ConfigurationMaintenanceControlConflict("Original frozen local admission changed"); + const admission = Object.freeze({ + ...inspected, + parent: roleParent, + boundAt: binding.boundAt, + reservations: + step.phase === "maintenance-hardlinks" && owner.plan.nestedHardlinks + ? [owner.plan.nestedHardlinks] + : [], + async adopt(execution: ExecutionRef) { + if (!freshDispatch || canonicalJson(execution) !== canonicalJson(binding.child.execution)) + throw new ConfigurationMaintenanceControlConflict("Attempted frozen phase is lookup-only"); + consumeControlChildDispatch(freshDispatch); + const peer = verifyRetainedLocalControlPeer(installation, { + assertion, + execution, + kind: operation.kind, + }); + return adoptArtifactTargetLease({ + dataRoot: local.dataRoot, + assertion, + peer, + execution, + serviceIds: inspected.serviceIds, + }); + }, + }); + if (step.phase === "maintenance-recreate") + frozenRecreateAdmissions.set(admission, { + operation: canonicalJson(operation), + operationKey: request.operationKey, + runtime, + authority, + ordinal: step.ordinal, + }); + return admission; +} +const frozenRecreateAdmissions = new WeakMap< + object, + { + operation: string; + operationKey: string; + runtime: OpsRuntime; + authority: VerifiedExecutionAuthority; + ordinal: number; + } +>(); +export async function inspectOwnedMaintenanceRecreateAdmission( + value: unknown, + operation: OpsOperation, + operationKey: string, + signal: AbortSignal, +): Promise { + if (!isOwnedMaintenanceRecreateAdmission(value, operation, operationKey)) + throw Error("Actual frozen recreate admission required"); + const fixed = frozenRecreateAdmissions.get(value as object)!; + await verifyFrozenMaintenanceRecreateSource( + fixed.runtime, + fixed.authority, + fixed.ordinal, + signal, + ); + const source = sources.get(fixed.runtime); + if (!source) throw Error("Actual original source required"); + return inspectOriginalOwnerServices(source, fixed.authority, signal); +} +export function isOwnedMaintenanceRecreateAdmission( + value: unknown, + operation: OpsOperation, + operationKey: string, +) { + const source = + value && typeof value === "object" ? frozenRecreateAdmissions.get(value) : undefined; + return ( + !!source && + source.operation === canonicalJson(operation) && + source.operationKey === operationKey + ); +} +/** Runtime recheck from the actual borrowed child and original signed owner, + * never workflow availability, a maintenance UUID or an admission boolean. */ +export async function assertOwnedMaintenanceRecreateRuntime( + runtime: OpsRuntime, + operation: OpsOperation, + context: import("./runtime").OpsExecutionContext, +) { + if ( + (operation.kind !== "service.recreate" && operation.kind !== "service.recreateIsolated") || + !operation.maintenanceId || + !context.controlAuthority || + !context.artifactLease + ) + throw Error("Actual original borrowed maintenance recreate required"); + const child = executionAuthorityState(context.controlAuthority), + observed = verifyControl( + child.installation.installation.publicKey, + domain, + readPrivateRecord( + child.installation, + "configuration-maintenance", + `${operation.maintenanceId}.json`, + ), + ownerSchema, + ), + root = restoreAdmittedControlExecution(child.installation.installation, observed.plan.prepare), + owner = readOwnedConfigurationMaintenance(root), + step = owner.plan.steps.find( + (value) => + value.operationKey === child.execution.operationKey && + value.operationFingerprint === child.execution.operationFingerprint, + ); + if ( + !step || + !child.origin || + canonicalJson(child.origin) !== canonicalJson(owner.origin) || + canonicalJson(step.operation) !== canonicalJson(operation) + ) + throw Error("Frozen borrowed recreate changed"); + await verifyFrozenMaintenanceRecreateSource(runtime, root, step.ordinal, context.signal); + const source = sources.get(runtime); + if (!source) throw Error("Actual original source required"); + return inspectOriginalOwnerServices(source, root, context.signal); +} +/** Retention capture for the exact already-authorized borrowed generation child. + * No caller snapshot or maintenance UUID alone can select OLD ownership. */ +export async function inspectOwnedMaintenanceGenerationRuntime( + runtime: OpsRuntime, + context: import("./runtime").OpsExecutionContext, + operation: Extract, + signal: AbortSignal, +) { + if (!context.controlAuthority || !context.artifactLease || !operation.maintenanceId) + throw Error("Actual borrowed generation required"); + const child = executionAuthorityState(context.controlAuthority), + observed = verifyControl( + child.installation.installation.publicKey, + domain, + readPrivateRecord( + child.installation, + "configuration-maintenance", + `${operation.maintenanceId}.json`, + ), + ownerSchema, + ), + root = restoreAdmittedControlExecution(child.installation.installation, observed.plan.prepare), + owner = readOwnedConfigurationMaintenance(root), + step = owner.plan.steps.find((value) => value.operationKey === child.execution.operationKey); + if ( + !step || + step.phase !== "configuration-commit" || + step.operationFingerprint !== child.execution.operationFingerprint || + canonicalJson(step.operation) !== canonicalJson(operation) || + !child.origin || + canonicalJson(child.origin) !== canonicalJson(owner.origin) || + owner.receipt.decision?.decision !== "continue" || + !owner.apiControls.decide?.receiptDelivery || + owner.receipt.generation !== "not_started" || + owner.issuance !== "open" + ) + throw Error("Original authorized generation source changed"); + if (!owner.coordinatorPull && !owner.stagedCoordinatorProvenance) return undefined; + // The resolver is the same literal production instance registered for this owner runtime. + const source = sources.get(runtime); + if (!source) throw Error("Original production generation source unavailable"); + const services = await inspectOriginalOwnerServices(source, root, signal), + ordinary = await source.ownedServices.inspectInventory?.(signal); + if (!ordinary || ordinary.unverifiedServiceIds.some((id) => id !== "data-manager")) + throw Error("Original generation inventory unavailable"); + return { + services, + observedContainerIds: ordinary.observedContainerIds, + unverifiedServiceIds: [], + }; +} +/** Read-only eligibility from the actual MAC packet and closed existing + * producer. It grants no generation child, decision or publication authority. */ +export async function assertPlannedCoordinatorConfigurationWrite( + runtime: OpsRuntime, + operation: Extract, +) { + const source = sources.get(runtime); + if (!source?.candidateSource) throw Error("Actual planned coordinator source required"); + const candidate = await readConfigurationCandidate(source.candidateSource, operation.candidateId), + original = candidate.operation, + record = candidate.record; + if ( + !["stack.render", "vault.apply"].includes(original.kind) || + !("revisionId" in original) || + !("expectedCurrentRevisionId" in original) || + !("expectedSourceRevision" in original) || + !("coordinatorMode" in original) || + original.revisionId !== operation.revisionId || + original.expectedCurrentRevisionId !== operation.expectedCurrentRevisionId || + original.expectedSourceRevision !== operation.expectedSourceRevision || + original.coordinatorMode !== operation.coordinatorMode || + record.candidateId !== operation.candidateId || + !record.maintenanceId || + !record.prepareOperationKey || + canonicalJson(record.affectedServiceIds) !== '["data-manager"]' + ) + throw Error("Planned coordinator identity changed"); + if (record.privateCoordinatorLifecycle) { + renderCoordinatorCandidate(candidate, await source.loadAuthority(), source.infraDir); + return; + } + const overlay = record.privateOverlay as { purpose?: unknown; changes?: unknown } | null; + if ( + original.kind !== "vault.apply" || + original.serviceId !== "data-manager" || + record.apiFollowUp !== "service-apply" || + !overlay || + typeof overlay !== "object" || + Array.isArray(overlay) || + overlay.purpose !== "service-secret" || + !Array.isArray(overlay.changes) || + !overlay.changes.length || + !overlay.changes.every( + (change) => + change && + typeof change === "object" && + !Array.isArray(change) && + change.kind === "service-secret" && + change.serviceId === "data-manager", + ) + ) + throw Error("Fixed planned DM recreation producer required"); +} +/** Actual publication is stricter than the read-only planned eligibility. */ +export async function assertOwnedCoordinatorConfigurationWrite( + runtime: OpsRuntime, + context: import("./runtime").OpsExecutionContext, + operation: Extract, + signal: AbortSignal, +) { + await inspectOwnedMaintenanceGenerationRuntime(runtime, context, operation, signal); + if (!context.controlAuthority || !operation.maintenanceId) + throw Error("Nominal coordinator generation required"); + const child = executionAuthorityState(context.controlAuthority), + observed = verifyControl( + child.installation.installation.publicKey, + domain, + readPrivateRecord( + child.installation, + "configuration-maintenance", + `${operation.maintenanceId}.json`, + ), + ownerSchema, + ), + root = restoreAdmittedControlExecution(child.installation.installation, observed.plan.prepare), + owner = readOwnedConfigurationMaintenance(root); + if ( + !owner.plan.coordinatorImage || + !owner.plan.steps.some( + (step) => + step.phase === "maintenance-recreate" && + "serviceId" in step.operation && + step.operation.serviceId === "data-manager", + ) + ) + throw Error("Original frozen DM recreation required"); +} +/** The outer digest is a mandatory observation checksum ONLY. Compare against + * the ACTUAL owner under its writer BEFORE the first control intent/sequence + * mutation. Accepted duplicates use the retained original body/observation. */ +export function retainConfigurationMaintenanceSettleControl( + value: ConfigurationMaintenanceControlOwner, + operation: ConfigurationMaintenanceSettle, + execution: ExecutionRef, + request: { + admittedAt: number; + requestedDeadlineAt: number; + parentAssertion: string; + requestVersion: 1 | 2; + deploymentBudgetMs: number; + }, +): void { + const source = configurationMaintenanceControlOwnerState( + value, + operation, + execution.operationKey, + ), + authority = source.authority, + parent = source.parent; + if ( + parent.issuer !== "api" || + execution.executor !== "ops-agent" || + execution.principal !== "api" || + execution.operationFingerprint !== source.operationFingerprint || + execution.fingerprint !== + executionFingerprint({ + kind: operation.kind, + operationFingerprint: source.operationFingerprint, + deadlineAt: execution.deadlineAt, + parent, + }) || + request.requestedDeadlineAt !== source.requestedDeadlineAt || + execution.deadlineAt <= request.admittedAt || + !Number.isSafeInteger(request.deploymentBudgetMs) || + request.deploymentBudgetMs < 1 || + request.deploymentBudgetMs > 60000 || + execution.deadlineAt !== + Math.min(request.requestedDeadlineAt, request.admittedAt + request.deploymentBudgetMs) || + createHash("sha256").update(request.parentAssertion).digest("hex") !== parent.assertionDigest + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual original settlement request required", + ); + const lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const owner = read(authority), + previous = owner.apiControls.settle; + if (previous) { + if ( + canonicalJson(previous.operation) !== canonicalJson(operation) || + canonicalJson(previous.execution) !== canonicalJson(execution) || + previous.admittedAt !== request.admittedAt || + previous.requestedDeadlineAt !== request.requestedDeadlineAt || + previous.parentAssertion !== request.parentAssertion || + canonicalJson(previous.apiParent) !== canonicalJson(parent) || + previous.requestVersion !== request.requestVersion || + previous.deploymentBudgetMs !== request.deploymentBudgetMs + ) + throw new ConfigurationMaintenanceControlConflict( + "Original accepted settlement request changed", + ); + return; + } + const acceptedAt = Date.now(); + if ( + owner.final || + owner.inspection || + owner.noGuard || + owner.issuance !== "open" || + !owner.guardSource || + !owner.capacity || + !owner.apiControls.decide?.receiptDelivery || + owner.receipt.decision?.decisionId !== operation.decisionId || + owner.sequence !== operation.expectedSequence || + acceptedAt >= execution.deadlineAt || + !timingSafeEqual( + Buffer.from(operation.evidenceDigest, "hex"), + Buffer.from(configurationMaintenanceSettleRequestDigest(owner.receipt), "hex"), + ) + ) + throw new ConfigurationMaintenanceControlConflict( + "Original settlement request observation conflict", + ); + const retained = configurationMaintenanceSettleControlSourceSchema.parse({ + version: 1, + operation, + execution, + apiParent: parent, + parentAssertion: request.parentAssertion, + admittedAt: request.admittedAt, + requestedDeadlineAt: request.requestedDeadlineAt, + acceptedAt, + requestVersion: request.requestVersion, + deploymentBudgetMs: request.deploymentBudgetMs, + checkedSequence: owner.sequence, + requestEvidenceDigest: operation.evidenceDigest, + completedAt: null, + result: null, + canonicalDigest: null, + receiptDelivery: null, + }); + if (Buffer.byteLength(canonicalJson(retained)) > 8192) + throw new ConfigurationMaintenanceControlConflict( + "Original settlement control capacity exceeded", + ); + const sequence = owner.sequence + 1; + write(authority, { + ...owner, + sequence, + receipt: { ...owner.receipt, sequence }, + apiControls: { ...owner.apiControls, settle: retained }, + }); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} +type DecisionSource = NonNullable; +function decisionReceipt( + owner: ConfigurationMaintenanceOwnerRecord, + source: Pick, +): ConfigurationMaintenanceReceipt { + return { ...owner.receipt, ...source.result }; +} +function decisionStatus( + owner: ConfigurationMaintenanceOwnerRecord, + source: Pick, +): OpsJobStatus { + return { + version: 2, + admittedAt: source.admittedAt, + deadlineAt: source.execution.deadlineAt, + fingerprint: source.execution.fingerprint, + operationId: source.execution.executionId, + operationKey: source.execution.operationKey, + kind: "configurationMaintenance.decide", + resourceId: opsResourceId(source.operation), + state: "succeeded", + submittedAt: new Date(source.admittedAt).toISOString(), + updatedAt: new Date(source.acceptedAt).toISOString(), + result: decisionReceipt(owner, source), + }; +} +/** Actual API HMAC admission -> original held owner -> ONE metadata CAS. The + * request execution is correlation only; it receives no origin, target lease, + * grant, ordinary child or forward authority. Abandon cannot issue effects. */ +export function retainConfigurationMaintenanceDecisionControl( + value: ConfigurationMaintenanceControlOwner, + operation: ConfigurationMaintenanceDecision, + execution: ExecutionRef, + request: { + admittedAt: number; + requestedDeadlineAt: number; + parentAssertion: string; + requestVersion: 1 | 2; + deploymentBudgetMs: number; + }, + continueObservation?: MaintenanceContinueObservation, +): ConfigurationMaintenanceReceipt { + const source = configurationMaintenanceControlOwnerState( + value, + operation, + execution.operationKey, + ), + authority = source.authority, + parent = source.parent; + if ( + parent.issuer !== "api" || + execution.executor !== "ops-agent" || + execution.principal !== "api" || + execution.operationFingerprint !== source.operationFingerprint || + execution.fingerprint !== + executionFingerprint({ + kind: operation.kind, + operationFingerprint: source.operationFingerprint, + deadlineAt: execution.deadlineAt, + parent, + }) || + request.requestedDeadlineAt !== source.requestedDeadlineAt || + execution.deadlineAt <= request.admittedAt || + !Number.isSafeInteger(request.deploymentBudgetMs) || + request.deploymentBudgetMs < 1 || + request.deploymentBudgetMs > 60000 || + execution.deadlineAt !== + Math.min( + request.requestedDeadlineAt, + operation.decision === "continue" ? parent.deadlineAt : Infinity, + request.admittedAt + request.deploymentBudgetMs, + ) || + createHash("sha256").update(request.parentAssertion).digest("hex") !== parent.assertionDigest + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual fixed API decision request required", + ); + const lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + const publish = () => { + const owner = read(authority), + previous = owner.apiControls.decide; + if (previous) { + if ( + canonicalJson(previous.operation) !== canonicalJson(operation) || + canonicalJson(previous.execution) !== canonicalJson(execution) || + previous.admittedAt !== request.admittedAt || + previous.requestedDeadlineAt !== request.requestedDeadlineAt || + previous.requestVersion !== request.requestVersion || + previous.deploymentBudgetMs !== request.deploymentBudgetMs || + previous.parentAssertion !== request.parentAssertion || + canonicalJson(previous.apiParent) !== canonicalJson(parent) || + previous.canonicalDigest !== controlTerminalReceiptDigest(decisionStatus(owner, previous)) + ) + throw new ConfigurationMaintenanceControlConflict("Original decision control changed"); + return decisionReceipt(owner, previous); + } + const acceptedAt = Date.now(); + if ( + acceptedAt >= execution.deadlineAt || + owner.issuance !== "open" || + owner.noGuard || + !owner.guardSource || + !owner.capacity || + owner.receipt.state !== "prepared" || + owner.receipt.decision !== null || + owner.receipt.dmFence?.state !== "prepared" || + owner.receipt.generation !== "not_started" || + owner.receipt.followUp !== "not_started" || + owner.sequence !== operation.expectedSequence + ) + throw new ConfigurationMaintenanceControlConflict( + "Original unused prepared decision source required", + ); + if (operation.decision === "continue") + consumeMaintenanceContinueObservation(continueObservation, authority); + const next = owner.sequence + 1, + receipt: ConfigurationMaintenanceReceipt = { + ...owner.receipt, + sequence: next, + state: "authorized", + preflight: operation.decision === "continue" ? "current" : "unavailable", + decision: { + decisionId: operation.decisionId, + decision: operation.decision, + checkpointVersion: operation.checkpointVersion, + checkpointDigest: operation.checkpointDigest, + }, + }, + pending = { + version: 1 as const, + operation, + execution, + apiParent: parent, + parentAssertion: request.parentAssertion, + admittedAt: request.admittedAt, + requestedDeadlineAt: request.requestedDeadlineAt, + acceptedAt, + result: configurationMaintenanceControlSnapshotSchema.parse({ + preflight: receipt.preflight, + sequence: receipt.sequence, + state: receipt.state, + decision: receipt.decision, + dmFence: receipt.dmFence, + generation: receipt.generation, + appliedRevisionId: receipt.appliedRevisionId, + followUp: receipt.followUp, + }), + requestVersion: request.requestVersion, + deploymentBudgetMs: request.deploymentBudgetMs, + receiptDelivery: null, + }, + canonicalDigest = controlTerminalReceiptDigest(decisionStatus(owner, pending)), + commitment = signControl( + executionAuthorityState(authority).installation.privateKey, + configurationMaintenanceDecisionCommitmentDomain, + configurationMaintenanceDecisionCommitmentPayload(owner, { ...pending, canonicalDigest }), + ), + retained = configurationMaintenanceDecisionSourceSchema.parse({ + ...pending, + canonicalDigest, + commitment, + }); + if (Buffer.byteLength(canonicalJson(retained)) > 8192) + throw new ConfigurationMaintenanceControlConflict( + "Original decision control capacity exceeded", + ); + write(authority, { + ...owner, + sequence: next, + receipt, + apiControls: { ...owner.apiControls, decide: retained }, + }); + return receipt; + }; + return operation.decision === "continue" + ? withConfigurationMaintenanceDecisionPublication(lease, authority, publish) + : withConfigurationMaintenanceOwnerObservation(lease, authority, publish); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} +export function inspectConfigurationMaintenanceDecisionControl( + value: ConfigurationMaintenanceControlOwner, + operation: ConfigurationMaintenanceDecision, + execution: ExecutionRef, +) { + const source = configurationMaintenanceControlOwnerState( + value, + operation, + execution.operationKey, + ), + owner = read(source.authority), + retained = owner.apiControls.decide; + if ( + !retained || + canonicalJson(retained.operation) !== canonicalJson(operation) || + canonicalJson(retained.execution) !== canonicalJson(execution) || + retained.canonicalDigest !== controlTerminalReceiptDigest(decisionStatus(owner, retained)) + ) + throw new ConfigurationMaintenanceControlConflict( + "Original accepted decision control required", + ); + return { + receipt: decisionReceipt(owner, retained), + status: decisionStatus(owner, retained), + terminalAt: retained.acceptedAt, + }; +} +type SettleSource = NonNullable; +function settleStatus( + owner: ConfigurationMaintenanceOwnerRecord, + source: SettleSource, +): OpsJobStatus { + if (!source.result || source.completedAt === null) + return { + version: 2, + admittedAt: source.admittedAt, + deadlineAt: source.execution.deadlineAt, + fingerprint: source.execution.fingerprint, + operationId: source.execution.executionId, + operationKey: source.execution.operationKey, + kind: "configurationMaintenance.settle", + resourceId: opsResourceId(source.operation), + state: "failed", + errorClass: "recovery_required", + submittedAt: new Date(source.admittedAt).toISOString(), + updatedAt: new Date(source.acceptedAt).toISOString(), + }; + return { + version: 2, + admittedAt: source.admittedAt, + deadlineAt: source.execution.deadlineAt, + fingerprint: source.execution.fingerprint, + operationId: source.execution.executionId, + operationKey: source.execution.operationKey, + kind: "configurationMaintenance.settle", + resourceId: opsResourceId(source.operation), + state: "succeeded", + submittedAt: new Date(source.admittedAt).toISOString(), + updatedAt: new Date(source.completedAt).toISOString(), + result: { ...owner.receipt, ...source.result }, + }; +} +/** Publication follows the original immutable final, actual DM/ACK closure and + * retained physical release. Its time is that release's original source time. */ +export function completeConfigurationMaintenanceSettleControl( + value: ConfigurationMaintenanceControlOwner, + operation: ConfigurationMaintenanceSettle, + execution: ExecutionRef, +): ConfigurationMaintenanceReceipt { + const source = configurationMaintenanceControlOwnerState( + value, + operation, + execution.operationKey, + ), + owner = read(source.authority), + retained = owner.apiControls.settle; + if ( + !retained || + canonicalJson(retained.operation) !== canonicalJson(operation) || + canonicalJson(retained.execution) !== canonicalJson(execution) + ) + throw Error("Original settle completion binding changed"); + return completeOwnedConfigurationMaintenanceSettleControl(source.authority); +} +/** Publication-only recovery has no live API-parent/dispatch capability. */ +function completeOwnedConfigurationMaintenanceSettleControl( + authority: VerifiedExecutionAuthority, +): ConfigurationMaintenanceReceipt { + const released = verifyRetainedArtifactTargetRootRelease(authority); + restoreOwnedMaintenanceFinal(authority); + assertConfigurationMaintenanceRootClosure(authority); + return withPrivateEffectWriter(executionAuthorityState(authority), () => { + const owner = read(authority), + retained = owner.apiControls.settle; + if (!retained) throw Error("Original accepted settle intent required"); + const completed = configurationMaintenanceSettleCompletion(owner, released.releasedAt); + if (retained.result) { + if (retained.canonicalDigest !== controlTerminalReceiptDigest(settleStatus(owner, retained))) + throw Error("Original settle completion changed"); + return { ...owner.receipt, ...retained.result }; + } + write(authority, completed); + return { ...owner.receipt, ...completed.apiControls.settle!.result }; + }); +} +/** Observation-only completion of the ORIGINAL accepted control. The callee + * requires its actual full closure and retained physical release timestamp. */ +export function completeObservedConfigurationMaintenanceSettlement( + authority: VerifiedExecutionAuthority, +): ConfigurationMaintenanceReceipt { + return completeOwnedConfigurationMaintenanceSettleControl(authority); +} +export function inspectConfigurationMaintenanceSettleControl( + value: ConfigurationMaintenanceControlOwner, + operation: ConfigurationMaintenanceSettle, + execution: ExecutionRef, +) { + const source = configurationMaintenanceControlOwnerState( + value, + operation, + execution.operationKey, + ), + owner = read(source.authority), + retained = owner.apiControls.settle; + if ( + !retained?.result || + retained.completedAt === null || + canonicalJson(retained.operation) !== canonicalJson(operation) || + canonicalJson(retained.execution) !== canonicalJson(execution) || + retained.canonicalDigest !== controlTerminalReceiptDigest(settleStatus(owner, retained)) + ) + throw Error("Original settle canonical source required"); + restoreOwnedMaintenanceFinal(source.authority); + assertConfigurationMaintenanceRootClosure(source.authority); + if (verifyRetainedArtifactTargetRootRelease(source.authority).releasedAt !== retained.completedAt) + throw Error("Original settle completion time changed"); + return { + receipt: { ...owner.receipt, ...retained.result }, + status: settleStatus(owner, retained), + terminalAt: retained.completedAt!, + }; +} +/** An accepted intent without actual final completion is UNKNOWN. This fixed + * projection carries its original admission/acceptance times and no canonical + * completion or delivery authority. It never inspects again or runs effects. */ +export function inspectConfigurationMaintenanceSettleRecovery( + value: ConfigurationMaintenanceControlOwner, + operation: ConfigurationMaintenanceSettle, + execution: ExecutionRef, +): OpsJobStatus { + const source = configurationMaintenanceControlOwnerState( + value, + operation, + execution.operationKey, + ), + owner = read(source.authority), + retained = owner.apiControls.settle; + if ( + !retained || + retained.result || + retained.canonicalDigest || + retained.completedAt !== null || + retained.receiptDelivery || + canonicalJson(retained.operation) !== canonicalJson(operation) || + canonicalJson(retained.execution) !== canonicalJson(execution) + ) + throw Error("Original unresolved settlement intent required"); + return settleStatus(owner, retained); +} +/** Fixed runtime publication AFTER the child's actual private canonical and + * physical settlement. A supplied status is only that private tuple's exact + * preimage; it cannot authorize success, containment or another phase. */ +export function recordOwnedConfigurationMaintenanceStepTerminal( + installation: ControlInstallation, + operation: OpsOperation, + execution: ExecutionRef, + status: OpsJobStatus, + terminalAt: number, +): void { + if (!("maintenanceId" in operation) || !operation.maintenanceId) + throw Error("Original maintenance step terminal required"); + const local = installationState(installation), + child = restoreAdmittedControlExecution(installation, execution), + childState = executionAuthorityState(child), + admission = readPrivateRecord(local, "executions", `${execution.operationKey}.json`, 65536) as { + payload?: { operation?: unknown }; + }, + observed = verifyControl( + installation.publicKey, + domain, + readPrivateRecord(local, "configuration-maintenance", `${operation.maintenanceId}.json`), + ownerSchema, + ), + authority = restoreAdmittedControlExecution(installation, observed.plan.prepare), + owner = read(authority), + source = owner.apiControls.steps.find( + (value) => value.execution.operationKey === execution.operationKey, + ); + if ( + !source || + canonicalJson(admission.payload?.operation) !== canonicalJson(operation) || + canonicalJson(source.execution) !== canonicalJson(execution) || + childState.kind !== operation.kind + ) + throw Error("Original phase terminal admission changed"); + const verified = verifyConfigurationMaintenanceChildTerminal( + authority, + source.ordinal, + status, + terminalAt, + ), + proof = configurationMaintenanceChildState(verified, authority), + lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + const physical = inspectArtifactTargetLeaseChildSettlement(lease, execution); + if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) + throw Error("Original phase physical settlement unavailable"); + withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const current = read(authority), + actual = current.apiControls.steps.find((value) => value.ordinal === source.ordinal), + canonical = { + status: proof.status, + terminalDigest: proof.terminalDigest, + terminalAt: proof.terminalAt, + effectSettlementDigest: proof.effectSettlementDigest, + }; + if (!actual || canonicalJson(actual.execution) !== canonicalJson(execution)) + throw Error("Original phase terminal owner changed"); + if (actual.canonical) { + if (canonicalJson(actual.canonical) !== canonicalJson(canonical)) + throw Error("Original phase canonical tuple changed"); + return; + } + const steps = current.apiControls.steps.map((value) => + value.ordinal === source.ordinal ? { ...value, canonical } : value, + ), + sequence = current.sequence + 1, + complete = current.plan.steps.every((step) => + steps.some((value) => value.ordinal === step.ordinal && value.canonical !== null), + ), + generation = current.plan.steps[source.ordinal]!.phase === "configuration-commit", + notCommitted = + generation && + (proof.status.result as { outcome?: string } | undefined)?.outcome === "not_committed"; + write(authority, { + ...current, + sequence, + apiControls: { ...current.apiControls, steps }, + receipt: { + ...current.receipt, + sequence, + state: ["authorized", "applying"].includes(current.receipt.state) + ? "applying" + : current.receipt.state, + ...(generation + ? { + generation: notCommitted ? "not_committed" : "committed", + appliedRevisionId: notCommitted ? null : current.plan.operation.revisionId, + } + : {}), + ...(notCommitted ? { followUp: "contained" } : complete ? { followUp: "complete" } : {}), + }, + }); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} +/** API durable-copy ACK belongs to the original request HMAC, while the + * effect's execution parent is the original ops root. Neither is substituted. */ +export function acknowledgeConfigurationMaintenanceStep( + installation: ControlInstallation, + operation: Extract, + token: string, +) { + const local = installationState(installation), + rootRaw = readPrivateRecord(local, "executions", `${operation.originRootKey}.json`, 65536) as { + payload?: { execution?: ExecutionRef }; + }, + authority = restoreAdmittedControlExecution( + installation, + executionRefSchema.parse(rootRaw.payload?.execution), + ), + owner = read(authority), + source = owner.apiControls.steps.find( + (value) => value.execution.operationKey === operation.operationKey, + ), + proof = verifyApiRetentionProof(token, operation.proof); + if ( + !source?.canonical || + !source.grant || + source.apiParent.issuer !== "api" || + operation.originRootKey !== owner.plan.prepare.operationKey || + operation.fingerprint !== source.execution.fingerprint || + operation.terminalDigest !== source.canonical.terminalDigest || + proof.apiJobId !== source.apiParent.apiJobId || + proof.originalApiOwnerVersion !== source.apiParent.apiOwnerVersion || + proof.attemptId !== source.apiParent.attemptId || + proof.operationKey !== source.execution.operationKey || + proof.fingerprint !== source.execution.fingerprint || + proof.terminalDigest !== source.canonical.terminalDigest || + proof.checkpointOwnerVersion < proof.originalApiOwnerVersion + ) + throw Error("Actual original phase canonical copy required"); + const verified = verifyConfigurationMaintenanceChildTerminal( + authority, + source.ordinal, + source.canonical.status, + source.canonical.terminalAt, + ), + actual = configurationMaintenanceChildState(verified, authority), + lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + const physical = inspectArtifactTargetLeaseChildSettlement(lease, source.execution); + if (!physical || canonicalJson(physical) !== canonicalJson(actual.settlement)) + throw Error("Original copied phase settlement unavailable"); + return withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const current = read(authority), + retained = current.apiControls.steps.find((value) => value.ordinal === source.ordinal), + copy = { + terminalCopyId: proof.terminalCopyId, + checkpointOwnerVersion: proof.checkpointOwnerVersion, + proofDigest: controlDigest(proof), + }; + if (!retained || canonicalJson(retained.canonical) !== canonicalJson(source.canonical)) + throw Error("Original copied phase source changed"); + if (retained.receiptDelivery) { + if (canonicalJson(retained.receiptDelivery) !== canonicalJson(copy)) + throw Error("Original phase copy changed"); + } else + write(authority, { + ...current, + apiControls: { + ...current.apiControls, + steps: current.apiControls.steps.map((value) => + value.ordinal === source.ordinal ? { ...value, receiptDelivery: copy } : value, + ), + }, + }); + return { acknowledged: true as const }; + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} +/** Recover only actual indexed, pinned-role private owner sources. This fixed + * namespace has at most one owner per retained root family. No public job, + * current desired state, elapsed clock or missing lookup supplies a result. */ +export function restoreConfigurationMaintenanceControlJournal( + installation: ControlInstallation, +): PersistedOpsJob[] { + const local = installationState(installation); + if (installation.role !== "ops-agent" || local.fixedChildId) + throw new ConfigurationMaintenanceControlUnauthorized("Original host owner required"); + let directory: ReturnType; + try { + directory = local.privateRoot.directory("configuration-maintenance"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + let names: string[]; + try { + names = directory.listBounded(CONTROL_RETENTION_LIMITS.familiesPerRole + 5); + } finally { + directory.close(); + } + const records: PersistedOpsJob[] = [], + keys = new Set(), + ids = new Set(); + for (const name of names.sort()) { + if ( + [ + ".retention-count", + ".retention-highwater", + ".retention-transaction", + ".retention-writer", + ].includes(name) + ) + continue; + if ( + !/^[a-f0-9]{8}-[a-f0-9]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}\.json$/.test(name) + ) + throw new ConfigurationMaintenanceControlConflict("Unexpected original owner record"); + const observed = verifyControl( + installation.publicKey, + domain, + readPrivateRecord(local, "configuration-maintenance", name), + ownerSchema, + ), + authority = restoreAdmittedControlExecution(installation, observed.plan.prepare); + let owner: ConfigurationMaintenanceOwnerRecord; + try { + owner = read(authority); + } catch { + // A failed original rename may leave the already-indexed completion + // preimage ahead of its old physical owner. Only that exact preimage is + // repairable; all other mismatches remain unavailable. + repairIndexedConfigurationMaintenanceSettleCompletion(authority); + owner = read(authority); + } + if (owner.apiControls.settle?.result === null && owner.receipt.state === "settled") { + completeOwnedConfigurationMaintenanceSettleControl(authority); + owner = read(authority); + } + if (name !== `${owner.plan.operation.maintenanceId}.json`) + throw new ConfigurationMaintenanceControlConflict("Original owner address changed"); + for (const source of [owner.apiControls.decide, owner.apiControls.settle]) { + if (!source) continue; + const status = + source.operation.kind === "configurationMaintenance.decide" + ? decisionStatus(owner, source as DecisionSource) + : settleStatus(owner, source as SettleSource); + const unresolved = + source.operation.kind === "configurationMaintenance.settle" && source.result === null; + if (!unresolved && source.canonicalDigest !== controlTerminalReceiptDigest(status)) + throw Error("Original control canonical source changed"); + if (source.operation.kind === "configurationMaintenance.settle" && !unresolved) { + restoreOwnedMaintenanceFinal(authority); + assertConfigurationMaintenanceRootClosure(authority); + verifyRetainedArtifactTargetRootRelease(authority); + } + if (keys.has(source.execution.operationKey) || ids.has(source.execution.executionId)) + throw new ConfigurationMaintenanceControlConflict("Duplicate original control identity"); + keys.add(source.execution.operationKey); + ids.add(source.execution.executionId); + records.push({ + executionVersion: 2, + requestVersion: source.requestVersion, + admittedAt: source.admittedAt, + deadlineAt: source.execution.deadlineAt, + operationFingerprint: source.execution.operationFingerprint, + requestedDeadlineAt: source.requestedDeadlineAt, + deploymentBudgetMs: source.deploymentBudgetMs, + parent: source.apiParent, + role: "api", + operation: source.operation, + operationId: source.execution.executionId, + operationKey: source.execution.operationKey, + fingerprint: source.execution.fingerprint, + resourceId: opsResourceId(source.operation), + state: status.state, + submittedAt: status.submittedAt, + updatedAt: status.updatedAt, + ...(unresolved + ? { errorClass: "recovery_required" as const } + : { result: status.result, terminalAt: status.updatedAt }), + }); + } + } + return records; +} +/** Copy-before-ACK for the original owner's metadata control, not an invented + * ordinary family execution. The API proof must name the actual accepted + * request/parent and its exact immutable private canonical source. */ +export function acknowledgeConfigurationMaintenanceControl( + installation: ControlInstallation, + operation: Extract, + token: string, +) { + const local = installationState(installation); + if ( + installation.role !== "ops-agent" || + local.fixedChildId || + operation.originRootKey === operation.operationKey + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Original maintenance control receipt required", + ); + const admitted = readPrivateRecord( + local, + "executions", + `${operation.originRootKey}.json`, + 65536, + ) as { payload?: { operation?: unknown; execution?: ExecutionRef } }, + prepare = configurationMaintenancePrepareSchema.parse(admitted.payload?.operation), + authority = restoreAdmittedControlExecution( + installation, + executionRefSchema.parse(admitted.payload?.execution), + ), + proof = verifyApiRetentionProof(token, operation.proof); + const publish = () => { + const owner = read(authority), + slot = + owner.apiControls.settle?.execution.operationKey === operation.operationKey + ? "settle" + : "decide", + source = owner.apiControls[slot]; + if ( + !source || + source.apiParent.issuer !== "api" || + owner.plan.operation.maintenanceId !== prepare.maintenanceId || + owner.plan.prepare.operationKey !== operation.originRootKey || + source.execution.operationKey !== operation.operationKey || + source.execution.fingerprint !== operation.fingerprint || + source.canonicalDigest !== operation.terminalDigest || + source.canonicalDigest !== + controlTerminalReceiptDigest( + source.operation.kind === "configurationMaintenance.decide" + ? decisionStatus(owner, source as DecisionSource) + : settleStatus(owner, source as SettleSource), + ) || + createHash("sha256").update(source.parentAssertion).digest("hex") !== + source.apiParent.assertionDigest || + proof.apiJobId !== source.apiParent.apiJobId || + proof.originalApiOwnerVersion !== source.apiParent.apiOwnerVersion || + proof.attemptId !== source.apiParent.attemptId || + proof.operationKey !== source.execution.operationKey || + proof.fingerprint !== source.execution.fingerprint || + proof.terminalDigest !== source.canonicalDigest || + proof.checkpointOwnerVersion < proof.originalApiOwnerVersion + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual durable maintenance receipt copy required", + ); + const copy = { + terminalCopyId: proof.terminalCopyId, + checkpointOwnerVersion: proof.checkpointOwnerVersion, + proofDigest: controlDigest(proof), + }; + if (source.receiptDelivery) { + if (canonicalJson(source.receiptDelivery) !== canonicalJson(copy)) + throw new ConfigurationMaintenanceControlConflict( + "Original maintenance receipt copy changed", + ); + } else + write(authority, { + ...owner, + apiControls: { ...owner.apiControls, [slot]: { ...source, receiptDelivery: copy } }, + }); + return { acknowledged: true as const }; + }; + let lease: ArtifactTargetLease | undefined; + try { + lease = restoreArtifactTargetLeaseForSettlement(authority); + return withConfigurationMaintenanceOwnerObservation(lease, authority, publish); + } catch { + // Only the actual immutable released-root proof permits late delivery. + // No physical handle or forward authority is recreated by this ACK. + verifyRetainedArtifactTargetRootRelease(authority); + return withPrivateEffectWriter(executionAuthorityState(authority), publish); + } finally { + if (lease) closeArtifactTargetLeaseHandle(lease); + } +} +/** This is only a bounded synchronous observation of an authenticated original + * owner. It produces no forward authority, grant, lease or retained execution; + * the generic child/parent normalization and original root deadline stay exact. */ +export function normalizeConfigurationMaintenanceInspection( + value: ConfigurationMaintenanceControlOwner, + operation: Extract, + requestKey: string, + admittedAt: number, + deploymentBudgetMs: number, +) { + const source = configurationMaintenanceControlOwnerState(value, operation, requestKey); + if ( + !Number.isSafeInteger(admittedAt) || + admittedAt < 0 || + !Number.isSafeInteger(deploymentBudgetMs) || + deploymentBudgetMs <= 0 + ) + throw Error("Bounded original maintenance inspection required"); + const deadlineAt = Math.min( + source.requestedDeadlineAt, + admittedAt + Math.min(15000, deploymentBudgetMs), + ); + if (!Number.isSafeInteger(deadlineAt) || deadlineAt <= admittedAt) + throw Error("Elapsed original maintenance inspection"); + return { + admittedAt, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: source.operationFingerprint, + deadlineAt, + parent: source.parent, + }), + }; +} +/** Only this live original-owner source can select the finite cancellation / + * expiry control clock. It grants no forward child, lease or renewed root. */ +export function normalizeConfigurationMaintenanceRecoveryControl( + value: ConfigurationMaintenanceControlOwner, + operation: ConfigurationMaintenanceDecision | ConfigurationMaintenanceSettle, + requestKey: string, + admittedAt: number, + deploymentBudgetMs: number, +) { + const source = configurationMaintenanceControlOwnerState(value, operation, requestKey); + verifyControlRecoveryAuthority(source.authority); + if ( + (operation.kind === "configurationMaintenance.decide" && operation.decision !== "abandon") || + !Number.isSafeInteger(admittedAt) || + admittedAt < 0 || + !Number.isSafeInteger(deploymentBudgetMs) || + deploymentBudgetMs < 1 || + deploymentBudgetMs > 60000 + ) + throw Error("Original no-effect maintenance recovery control required"); + const deadlineAt = Math.min(source.requestedDeadlineAt, admittedAt + deploymentBudgetMs); + if (!Number.isSafeInteger(deadlineAt) || deadlineAt <= admittedAt) + throw Error("Elapsed original maintenance control attempt"); + return { + admittedAt, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: source.operationFingerprint, + deadlineAt, + parent: source.parent, + }), + }; +} +/** Only the fixed cross-role verifier can create the source handle. A parsed + * prepared response or a public terminal status cannot make this transition. */ +export function recordOwnedConfigurationMaintenancePrepared( + authority: VerifiedExecutionAuthority, + lease: ArtifactTargetLease, + expectedSequence: number, + verified: VerifiedPreparedDataConfigurationFence, + verifiedCapacity: VerifiedPublishedConfigurationSettlementCapacity, +): void { + const source = preparedDataConfigurationFenceState(verified), + capacity = publishedConfigurationSettlementCapacityState(verifiedCapacity, authority); + withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const owner = read(authority), + receipt = source.preparedReceipt, + plan = owner.plan; + if ( + owner.sequence !== expectedSequence || + owner.issuance !== "open" || + owner.noGuard || + owner.receipt.state !== "preparing" || + owner.guardSource || + owner.capacity || + source.rootCertificateDigest !== owner.rootCertificateDigest || + canonicalJson(source.ownerPrepare) !== canonicalJson(plan.prepare) || + receipt.maintenanceId !== plan.operation.maintenanceId || + receipt.candidateId !== plan.operation.candidateId || + receipt.planDigest !== plan.planDigest || + receipt.revisionId !== plan.operation.revisionId || + receipt.hardlinkPlanDigest !== plan.dmPrepare.hardlinkPlanDigest || + capacity.signed.payload.settlementId !== plan.dmSettle.settlementId + ) + throw Error("Original prepared maintenance guard conflict"); + const next = owner.sequence + 1; + write(authority, { + ...owner, + sequence: next, + guardSource: source, + capacity: capacity.signed, + receipt: { ...owner.receipt, sequence: next, state: "prepared", dmFence: receipt }, + }); + }); +} + +declare const maintenanceFinalBrand: unique symbol; +export type VerifiedMaintenanceFinal = Readonly<{ [maintenanceFinalBrand]: true }>; +const maintenanceFinals = new WeakMap< + VerifiedMaintenanceFinal, + { + authority: VerifiedExecutionAuthority; + record: NonNullable; + } +>(); +export function ownedMaintenanceFinalState( + value: VerifiedMaintenanceFinal, + authority: VerifiedExecutionAuthority, +) { + const source = maintenanceFinals.get(value); + if (!source || source.authority !== authority) + throw Error("Actual original maintenance final required"); + return source; +} +/** Dedicated original-owner final CAS. No parsed evidence, requested outcome + * or signature from a caller creates inspection/final authority. */ +export function finalizeOwnedMaintenanceState( + authority: VerifiedExecutionAuthority, + inspection: MaintenanceRuntimeInspection, +): VerifiedMaintenanceFinal { + const actual = maintenanceRuntimeInspectionState(inspection, authority), + applied = + actual.signed.payload.runtime.mode !== "unused" + ? readOriginalMaintenanceFinalChildren( + authority, + observeConfigurationMaintenanceRuntimeTree(authority), + ) + : null, + lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const owner = read(authority), + payload = verifyControl( + executionAuthorityState(authority).installation.installation.publicKey, + domains.inspection, + actual.signed, + configurationMaintenanceRuntimeInspectionSchema, + ); + if (owner.final) { + if (canonicalJson(owner.inspection) !== canonicalJson(actual.signed)) + throw Error("Immutable original maintenance final changed"); + return; + } + if ( + owner.sequence !== payload.ownerSequence || + controlDigest(owner) !== payload.ownerSourceDigest || + owner.inspection || + !owner.guardSource || + !owner.apiControls.decide?.receiptDelivery || + owner.noGuard || + owner.issuance !== "open" || + !(payload.runtime.mode === "unused" + ? owner.receipt.decision?.decision === "abandon" && + owner.receipt.generation === "not_started" && + owner.receipt.followUp === "not_started" && + owner.apiControls.steps.length === 0 + : (payload.runtime.mode === "applied" && + applied?.complete && + applied.ownerDigest === controlDigest(owner) && + applied.generation && + owner.receipt.decision?.decision === "continue" && + owner.receipt.generation === "committed" && + owner.receipt.followUp === "complete") || + (payload.runtime.mode === "contained" && + applied?.generationResult?.outcome === "not_committed" && + !applied.generationResult.cleanupFailed && + applied.children.length === 1 && + owner.apiControls.steps.length === 1 && + applied.ownerDigest === controlDigest(owner) && + owner.receipt.decision?.decision === "continue" && + owner.receipt.generation === "not_committed" && + owner.receipt.appliedRevisionId === null && + owner.receipt.followUp === "contained")) + ) + throw Error("Actual immutable final source raced"); + for (const step of payload.runtime.mode === "unused" ? owner.plan.steps : []) { + const publication = inspectControlChildPublication(authority, step); + if (publication.binding !== null || publication.publication.state !== "unissued") + throw Error("Unused final has attempted frozen work"); + } + const state = executionAuthorityState(authority), + decision = owner.apiControls.decide, + prepared = owner.guardSource, + evidence = configurationFinalEvidenceSchema.parse({ + identity: payload.identity, + guard: { + fenceId: prepared.preparedReceipt.fenceId, + preparedReceipt: prepared.preparedReceipt, + preparedReceiptDigest: prepared.preparedReceiptDigest, + preparedCanonicalDigest: prepared.preparedCanonicalDigest, + preparedTerminalAt: prepared.preparedTerminalAt, + }, + decision: { + ...owner.receipt.decision, + apiParentAttestationDigest: decision.apiParent.assertionDigest, + }, + children: applied?.children ?? [], + generation: + applied && payload.runtime.mode === "applied" + ? { + state: "committed", + revisionId: owner.plan.operation.revisionId, + commitReceiptDigest: applied.generation!.terminalDigest, + } + : { state: "not_committed", revisionId: null, commitReceiptDigest: null }, + followUp: applied + ? { + state: payload.runtime.mode === "contained" ? "contained" : "complete", + receiptDigests: applied.followUpReceiptDigests, + } + : { state: "not_started", receiptDigests: [] }, + runtime: payload.runtime, + outcome: payload.runtime.mode, + }), + evidenceProjection = configurationFinalEvidenceProjectionSchema.parse({ + ...evidence, + children: evidence.children.map(({ execution, ...child }) => ({ + ...child, + executionDigest: controlDigest(execution), + })), + }), + evidenceDigest = settlementRecordDigest(domains.finalEvidence, evidenceProjection), + sequence = owner.sequence + 1, + finalPayload = configurationOwnerFinalSchema.parse({ + version: 2, + identity: payload.identity, + ownerSequence: sequence, + settlingAt: Date.now(), + expectedDmSequence: prepared.preparedReceipt.sequence, + evidenceDigest, + outcome: payload.runtime.mode, + privateInspectionId: payload.inspectionId, + }), + final = configurationMaintenanceFinalRecordSchema.parse({ + ownerFinal: signControl(state.installation.privateKey, domains.ownerFinal, finalPayload), + evidence, + evidenceProjection, + evidenceDigest, + ownerReceiptDigest: settlementRecordDigest(domains.ownerFinal, finalPayload), + }); + write(authority, { + ...owner, + sequence, + final, + inspection: actual.signed, + receipt: { ...owner.receipt, sequence, state: "settling" }, + }); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } + return restoreOwnedMaintenanceFinal(authority); +} +/** Restore the original actual signed private inspection/final and all digest + * preimages. This never samples current desired state or invents a new runtime + * verification ID/time. Missing/torn/different originals stay fenced. */ +export function restoreOwnedMaintenanceFinal( + authority: VerifiedExecutionAuthority, +): VerifiedMaintenanceFinal { + const owner = read(authority), + final = owner.final, + state = executionAuthorityState(authority); + if ( + !final || + !owner.inspection || + !owner.guardSource || + !owner.apiControls.decide?.receiptDelivery || + owner.noGuard || + owner.issuance !== "open" + ) + throw Error("Actual immutable maintenance final required"); + if ( + final.ownerFinal.payload.outcome === "applied" || + final.ownerFinal.payload.outcome === "contained" + ) { + const contained = final.ownerFinal.payload.outcome === "contained"; + if (contained) assertContainedConfigurationMaintenanceFinalState(owner); + else assertAppliedConfigurationMaintenanceFinalState(owner); + const actual = readOriginalMaintenanceFinalChildren( + authority, + observeConfigurationMaintenanceRuntimeTree(authority), + ); + if ( + (!contained && !actual.complete) || + (contained && + (actual.generationResult?.outcome !== "not_committed" || + actual.generationResult.cleanupFailed || + actual.children.length !== 1)) || + canonicalJson(actual.children) !== canonicalJson(final.evidence.children) || + final.evidence.generation.commitReceiptDigest !== + (contained ? null : actual.generation?.terminalDigest) || + canonicalJson(final.evidence.followUp.receiptDigests) !== + canonicalJson(actual.followUpReceiptDigests) + ) + throw Error("Original applied final child sources changed"); + } else assertUnusedConfigurationMaintenanceFinalState(owner); + const inspection = verifyControl( + state.installation.installation.publicKey, + domains.inspection, + owner.inspection, + configurationMaintenanceRuntimeInspectionSchema, + ), + actual = verifyControl( + state.installation.installation.publicKey, + domains.ownerFinal, + final.ownerFinal, + configurationOwnerFinalSchema, + ), + evidence = final.evidence, + { children, ...base } = evidence, + projection = configurationFinalEvidenceProjectionSchema.parse({ + ...base, + children: children.map(({ execution, ...child }) => ({ + ...child, + executionDigest: controlDigest(execution), + })), + }); + if ( + actual.outcome !== evidence.outcome || + inspection.runtime.mode !== actual.outcome || + actual.ownerSequence > owner.sequence || + actual.ownerSequence !== inspection.ownerSequence + 1 || + actual.privateInspectionId !== inspection.inspectionId || + canonicalJson(actual.identity) !== canonicalJson(inspection.identity) || + canonicalJson(actual.identity) !== canonicalJson(evidence.identity) || + actual.identity.admissionDigest !== state.receiptDigest || + canonicalJson(actual.identity.ownerPrepare) !== canonicalJson(state.execution) || + actual.identity.maintenanceId !== owner.plan.operation.maintenanceId || + actual.identity.candidateId !== owner.plan.operation.candidateId || + actual.identity.planDigest !== owner.plan.planDigest || + actual.identity.rootCertificateDigest !== owner.rootCertificateDigest || + canonicalJson(evidence.runtime) !== canonicalJson(inspection.runtime) || + inspection.runtime.requiredCheckSetDigest !== + settlementRecordDigest(domains.inspection, inspection.requiredChecks) || + inspection.observations.length !== inspection.requiredChecks.length || + !inspection.observations.every((value, index) => value.checkIndex === index && value.passed) || + canonicalJson(projection) !== canonicalJson(final.evidenceProjection) || + settlementRecordDigest(domains.finalEvidence, projection) !== final.evidenceDigest || + final.evidenceDigest !== actual.evidenceDigest || + settlementRecordDigest(domains.ownerFinal, actual) !== final.ownerReceiptDigest || + canonicalJson(evidence.guard.preparedReceipt) !== + canonicalJson(owner.guardSource.preparedReceipt) || + evidence.guard.preparedReceiptDigest !== owner.guardSource.preparedReceiptDigest || + evidence.guard.preparedCanonicalDigest !== owner.guardSource.preparedCanonicalDigest || + evidence.guard.preparedTerminalAt !== owner.guardSource.preparedTerminalAt || + actual.expectedDmSequence !== owner.guardSource.preparedReceipt.sequence || + evidence.decision.decisionId !== owner.receipt.decision?.decisionId || + evidence.decision.decision !== owner.receipt.decision?.decision || + evidence.decision.checkpointVersion !== owner.receipt.decision.checkpointVersion || + evidence.decision.checkpointDigest !== owner.receipt.decision.checkpointDigest || + evidence.decision.apiParentAttestationDigest !== + owner.apiControls.decide.apiParent.assertionDigest || + (actual.outcome === "unused" && (children.length > 0 || owner.apiControls.steps.length > 0)) + ) + throw Error("Original maintenance final proof changed"); + for (const step of actual.outcome === "unused" ? owner.plan.steps : []) { + const publication = inspectControlChildPublication(authority, step); + if (publication.binding !== null || publication.publication.state !== "unissued") + throw Error("Final unused source has attempted frozen work"); + } + const value = Object.freeze({}) as VerifiedMaintenanceFinal; + const freezeProof = (input: T): T => { + if (input && typeof input === "object") { + for (const child of Object.values(input)) freezeProof(child); + Object.freeze(input); + } + return input; + }; + maintenanceFinals.set(value, Object.freeze({ authority, record: freezeProof(final) })); + return value; +} + +/** Publishes ONLY this original owner's restored actual private final. It fills + * the existing PREPARE-reserved cell; no namespace/capacity is added late. */ +export function publishOwnedConfigurationFenceSettlement( + authority: VerifiedExecutionAuthority, + maintenanceId: string, +): void { + const proof = restoreOwnedMaintenanceFinal(authority), + source = ownedMaintenanceFinalState(proof, authority), + state = executionAuthorityState(authority), + owner = read(authority), + prepare = owner.plan.reservations.find((value) => value.phase === "configuration-fence"); + if ( + owner.plan.operation.maintenanceId !== maintenanceId || + !prepare || + !owner.capacity || + !state.origin + ) + throw Error("Original prepared final publisher required"); + const projectionDescriptor = { + scope: "shared" as const, + namespace: "execution" as const, + role: "ops-agent" as const, + name: `${state.execution.operationKey}.configuration-settlement-control.json`, + cap: 8192, + }, + raw = readConfigurationSettlementCell( + state.installation, + { origin: state.origin }, + projectionDescriptor, + ), + control = verifyControl( + state.installation.installation.publicKey, + domains.control, + raw, + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + state.installation.installation.publicKey, + domains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + descriptor = configurationSettlementRecordDescriptors( + projection, + prepare.execution.operationKey, + ).sender[0]!, + final = source.record, + identity = final.ownerFinal.payload.identity; + if ( + projection.maintenanceId !== maintenanceId || + canonicalJson(projection.ownerPrepare) !== canonicalJson(state.execution) || + projection.rootAdmissionDigest !== state.receiptDigest || + projection.rootCertificateDigest !== owner.rootCertificateDigest || + projection.settlementId !== identity.settlementId || + projection.planDigest !== owner.plan.planDigest || + canonicalJson(projection.reservation) !== + canonicalJson( + owner.plan.reservations.find((value) => value.phase === "configuration-fence-settle"), + ) + ) + throw Error("Original final reservation projection changed"); + const committed = configurationCommittedSettlementSchema.parse({ + version: 2, + phase: "committed", + identity, + referenceOperationFingerprint: projection.reservation.operationFingerprint, + final: final.ownerFinal.payload, + evidence: final.evidenceProjection, + evidenceDigest: final.evidenceDigest, + ownerReceiptDigest: final.ownerReceiptDigest, + committedAt: final.ownerFinal.payload.settlingAt, + }), + signed = signControl(state.installation.privateKey, domains.reference, committed), + lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const current = read(authority); + if ( + canonicalJson(current.final) !== canonicalJson(final) || + current.noGuard || + current.issuance !== "open" + ) + throw Error("Original immutable final publication raced"); + const prior = readConfigurationSettlementCell(state.installation, projection, descriptor), + payload = verifyControl( + state.installation.installation.publicKey, + domains.reference, + prior, + configurationSettlementCommitmentCellSchema.shape.payload, + ); + if (payload.phase === "committed") { + if (canonicalJson(prior) !== canonicalJson(signed)) + throw Error("Committed original final changed"); + return; + } + if ( + payload.phase !== "uncommitted" || + canonicalJson(payload.identity) !== canonicalJson(identity) || + payload.referenceOperationFingerprint !== projection.reservation.operationFingerprint + ) + throw Error("Original pending final cell changed"); + writeConfigurationSettlementCell(state.installation, projection, descriptor, signed); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} + +declare const settlementControlBrand: unique symbol, settlementDispatchBrand: unique symbol; +export type VerifiedConfigurationFenceSettlementControl = Readonly<{ + [settlementControlBrand]: true; +}>; +export type ConfigurationFenceSettlementDispatchTicket = Readonly<{ + [settlementDispatchBrand]: true; +}>; +type LogicalControlSource = { + authority: VerifiedExecutionAuthority; + maintenanceId: string; + final: NonNullable; + projection: ReturnType; + signedProjection: ReturnType< + typeof signControl> + >; + commitmentDigest: string; + prepareKey: string; +}; +const settlementControls = new WeakMap< + VerifiedConfigurationFenceSettlementControl, + LogicalControlSource +>(); +const settlementDispatches = new WeakMap< + ConfigurationFenceSettlementDispatchTicket, + { + control: VerifiedConfigurationFenceSettlementControl; + envelope: ReturnType< + typeof signControl> + >; + assertion: string; + } +>(); +function freezeOwnedSource(value: T): T { + if (value && typeof value === "object" && !Object.isFrozen(value)) { + for (const child of Object.values(value)) freezeOwnedSource(child); + Object.freeze(value); + } + return value; +} +/** Reconstructible observation capability. The caller supplies ONLY the actual + * original authority/ID. Full final source and existing committed cell must + * agree; this function never chooses a time/address or creates a ticket. */ +export function prepareConfigurationFenceSettlementControl( + authority: VerifiedExecutionAuthority, + maintenanceId: string, +): VerifiedConfigurationFenceSettlementControl { + const actual = ownedMaintenanceFinalState(restoreOwnedMaintenanceFinal(authority), authority), + owner = read(authority), + state = executionAuthorityState(authority), + prepare = owner.plan.reservations.find((value) => value.phase === "configuration-fence"); + if ( + !prepare || + !state.origin || + !owner.guardSource || + !owner.capacity || + owner.plan.operation.maintenanceId !== maintenanceId + ) + throw Error("Original prepared final control source required"); + const descriptor = { + scope: "shared" as const, + namespace: "execution" as const, + role: "ops-agent" as const, + name: `${state.execution.operationKey}.configuration-settlement-control.json`, + cap: 8192, + }, + control = verifyControl( + state.installation.installation.publicKey, + domains.control, + readConfigurationSettlementCell(state.installation, { origin: state.origin }, descriptor), + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + state.installation.installation.publicKey, + domains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + committedDescriptor = configurationSettlementRecordDescriptors( + projection, + prepare.execution.operationKey, + ).sender[0]!, + committed = verifyControl( + state.installation.installation.publicKey, + domains.reference, + readConfigurationSettlementCell(state.installation, projection, committedDescriptor), + configurationCommittedSettlementSchema, + ); + if ( + "unused" in control || + projection.maintenanceId !== maintenanceId || + projection.rootAdmissionDigest !== state.receiptDigest || + canonicalJson(projection.ownerPrepare) !== canonicalJson(state.execution) || + projection.rootCertificateDigest !== owner.rootCertificateDigest || + canonicalJson(projection.reservation) !== + canonicalJson( + owner.plan.reservations.find((value) => value.phase === "configuration-fence-settle"), + ) || + canonicalJson(committed.identity) !== + canonicalJson(actual.record.ownerFinal.payload.identity) || + canonicalJson(committed.final) !== canonicalJson(actual.record.ownerFinal.payload) || + canonicalJson(committed.evidence) !== canonicalJson(actual.record.evidenceProjection) || + committed.evidenceDigest !== actual.record.evidenceDigest || + committed.ownerReceiptDigest !== actual.record.ownerReceiptDigest || + committed.referenceOperationFingerprint !== projection.reservation.operationFingerprint || + committed.committedAt !== committed.final.settlingAt + ) + throw Error("Actual original committed settlement source required"); + const value = Object.freeze({}) as VerifiedConfigurationFenceSettlementControl; + settlementControls.set( + value, + freezeOwnedSource({ + authority, + maintenanceId, + final: actual.record, + projection, + signedProjection: control.projection, + commitmentDigest: settlementRecordDigest(domains.commitment, committed), + prepareKey: prepare.execution.operationKey, + }), + ); + return value; +} +/** Restore only an ACTUALLY attempted original logical control. This reader + * cannot bind/issue a ticket, renew its deadline or publish a first attempt. */ +export function restoreAttemptedConfigurationFenceSettlementControl( + authority: VerifiedExecutionAuthority, + maintenanceId: string, +): VerifiedConfigurationFenceSettlementControl { + const value = prepareConfigurationFenceSettlementControl(authority, maintenanceId), + source = actualSettlementControl(value), + state = executionAuthorityState(authority), + descriptors = configurationSettlementRecordDescriptors( + source.projection, + source.prepareKey, + ).sender, + binding = verifyControl( + state.installation.installation.publicKey, + domains.binding, + readConfigurationSettlementCell(state.installation, source.projection, descriptors[1]!), + configurationSettlementControlEnvelopeSchema, + ), + dispatch = verifyControl( + state.installation.installation.publicKey, + domains.dispatch, + readConfigurationSettlementCell(state.installation, source.projection, descriptors[3]!), + configurationSettlementDispatchSchema, + ), + slot = readConfigurationSettlementFamilySlot(state.installation, source.projection); + if ( + dispatch.phase !== "attempted" || + dispatch.attemptedAt === null || + dispatch.attemptedAt < binding.controlStartedAt || + dispatch.attemptedAt >= binding.controlDeadlineAt || + dispatch.dispatchId !== binding.dispatchId || + dispatch.controlEnvelopeDigest !== settlementRecordDigest(domains.binding, binding) || + dispatch.controlExecutionDigest !== controlDigest(binding.execution) || + dispatch.commitmentDigest !== source.commitmentDigest || + binding.commitmentDigest !== source.commitmentDigest || + slot.state !== "bound" || + slot.controlBindingDigest !== dispatch.controlEnvelopeDigest || + slot.executionDigest !== dispatch.controlExecutionDigest + ) + throw Error("Actual original attempted settlement observation required"); + return value; +} +function assertCurrentSettlementFinal(source: LogicalControlSource) { + const owner = read(source.authority); + if ( + canonicalJson(owner.final) !== canonicalJson(source.final) || + !owner.inspection || + owner.noGuard || + owner.issuance !== "open" + ) + throw Error("Original immutable settlement final source changed"); + return owner; +} +function actualSettlementControl( + value: VerifiedConfigurationFenceSettlementControl, +): LogicalControlSource { + const source = settlementControls.get(value); + if (!source) throw Error("Actual fixed configuration settlement control required"); + return source; +} +/** One original binding/publication winner alone gets a process-local ticket. + * A recovered binding or partial publication is observe-only, forever using + * the original time/key/UUID/body. No ordinary phase/execution is admitted. */ +export function delegateConfigurationFenceSettlementControl( + value: VerifiedConfigurationFenceSettlementControl, +) { + const source = actualSettlementControl(value), + state = executionAuthorityState(source.authority), + projection = source.projection, + descriptors = configurationSettlementRecordDescriptors(projection, source.prepareKey).sender; + const current = actualSettlementControl( + prepareConfigurationFenceSettlementControl(source.authority, source.maintenanceId), + ); + return withConfigurationMaintenanceSettlementPublication(source.authority, () => { + assertCurrentSettlementFinal(source); + if ( + current.commitmentDigest !== source.commitmentDigest || + canonicalJson(current.projection) !== canonicalJson(projection) + ) + throw Error("Original settlement control source changed"); + const prior = readConfigurationSettlementCell(state.installation, projection, descriptors[1]!), + outbox = readConfigurationSettlementCell(state.installation, projection, descriptors[3]!); + let envelope: ReturnType< + typeof signControl> + >, + fresh = false; + if (prior !== null) + envelope = signControl( + state.installation.privateKey, + domains.control, + verifyControl( + state.installation.installation.publicKey, + domains.binding, + prior, + configurationSettlementControlEnvelopeSchema, + ), + ); + else { + const slot = readConfigurationSettlementFamilySlot(state.installation, projection), + owner = read(source.authority); + if (slot.state !== "reserved" || outbox !== null || !owner.capacity) + throw Error("Original unbound settlement capacity required"); + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(state.installation.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + dm = own(executions.directory("data-manager")), + bytes = dm.read(`${source.prepareKey}.configuration-settlement-capacity.json`, 8192), + compound = configurationDmCapacityCellSchema.parse(JSON.parse(bytes.toString("utf8"))); + if ( + bytes.length !== 8192 || + !compound.capacity || + compound.refusal || + compound.unused || + canonicalJson(compound.capacity) !== canonicalJson(owner.capacity) + ) + throw Error("Actual original DM settlement capacity required"); + const capacity = verifyControl( + state.installation.publicKeys["data-manager"], + domains.capacity, + compound.capacity, + configurationSettlementCapacitySchema, + ); + if ( + capacity.settlementId !== projection.settlementId || + capacity.projectionDigest !== settlementRecordDigest(domains.projection, projection) || + capacity.reservationDigest !== projection.reservationDigest || + canonicalJson(capacity.proposedExecutionAddress) !== + canonicalJson(projection.reservation.execution) + ) + throw Error("Actual original DM capacity binding conflict"); + verifyPublishedDmConfigurationSettlementSlots(state.installation, projection, capacity); + } finally { + for (const directory of opened.reverse()) directory.close(); + } + const controlStartedAt = Date.now(), + controlDeadlineAt = controlStartedAt + 60000, + operationFingerprint = projection.reservation.operationFingerprint, + execution = { + ...projection.reservation.execution, + operationFingerprint, + deadlineAt: controlDeadlineAt, + fingerprint: executionFingerprint({ + kind: projection.operation.kind, + operationFingerprint, + deadlineAt: controlDeadlineAt, + parent: { + issuer: "ops-agent", + operationKey: state.execution.operationKey, + fingerprint: state.execution.fingerprint, + deadlineAt: state.execution.deadlineAt, + execution: state.execution, + }, + }), + }, + payload = configurationSettlementControlEnvelopeSchema.parse({ + version: 2, + bindingVersion: 3, + purpose: "configuration-fence-settlement", + installationId: projection.installationId, + origin: projection.origin, + ownerPrepare: projection.ownerPrepare, + admissionDigest: projection.rootAdmissionDigest, + rootCertificateDigest: projection.rootCertificateDigest, + settlementId: projection.settlementId, + projectionDigest: settlementRecordDigest(domains.projection, projection), + reservationDigest: projection.reservationDigest, + commitmentDigest: source.commitmentDigest, + execution, + operation: projection.operation, + serviceIds: projection.reservation.serviceIds, + controlStartedAt, + controlDeadlineAt, + originalParentForwardDeadlineAt: state.execution.deadlineAt, + dispatchId: randomUUID(), + }); + envelope = signControl(state.installation.privateKey, domains.control, payload); + writeConfigurationSettlementCell( + state.installation, + projection, + descriptors[1]!, + signControl(state.installation.privateKey, domains.binding, payload), + ); + fresh = true; + } + const binding = envelope.payload, + { + operationFingerprint: _operationFingerprint, + fingerprint: _fingerprint, + deadlineAt: _deadlineAt, + ...address + } = binding.execution; + if ( + binding.commitmentDigest !== source.commitmentDigest || + binding.projectionDigest !== settlementRecordDigest(domains.projection, projection) || + binding.reservationDigest !== projection.reservationDigest || + canonicalJson(binding.ownerPrepare) !== canonicalJson(state.execution) || + binding.admissionDigest !== state.receiptDigest || + binding.rootCertificateDigest !== projection.rootCertificateDigest || + canonicalJson(binding.operation) !== canonicalJson(projection.operation) || + canonicalJson(address) !== canonicalJson(projection.reservation.execution) || + canonicalJson(binding.serviceIds) !== canonicalJson(projection.reservation.serviceIds) + ) + throw Error("Original logical settlement binding changed"); + bindOwnedConfigurationSettlementFamilySlot(state.installation, projection); + const prepared = signControl( + state.installation.privateKey, + domains.dispatch, + configurationSettlementDispatchSchema.parse({ + version: 2, + origin: projection.origin, + settlementId: projection.settlementId, + phase: "prepared", + dispatchId: binding.dispatchId, + controlEnvelopeDigest: settlementRecordDigest(domains.binding, binding), + controlExecutionDigest: controlDigest(binding.execution), + commitmentDigest: source.commitmentDigest, + attemptedAt: null, + }), + ); + if (outbox === null) + writeConfigurationSettlementCell(state.installation, projection, descriptors[3]!, prepared); + else { + const original = verifyControl( + state.installation.installation.publicKey, + domains.dispatch, + outbox, + configurationSettlementDispatchSchema, + ); + if ( + original.dispatchId !== binding.dispatchId || + original.controlEnvelopeDigest !== prepared.payload.controlEnvelopeDigest || + original.controlExecutionDigest !== prepared.payload.controlExecutionDigest || + original.commitmentDigest !== source.commitmentDigest + ) + throw Error("Original settlement outbox changed"); + fresh = false; + } + const controlCell = verifyControl( + state.installation.installation.publicKey, + domains.control, + readConfigurationSettlementCell(state.installation, projection, descriptors[2]!), + configurationSettlementControlCellSchema.shape.payload, + ); + if ( + "unused" in controlCell || + canonicalJson(controlCell.projection) !== canonicalJson(source.signedProjection) || + (controlCell.binding && canonicalJson(controlCell.binding) !== canonicalJson(envelope)) + ) + throw Error("Original settlement shared binding changed"); + if (!controlCell.binding) + writeConfigurationSettlementCell( + state.installation, + projection, + descriptors[2]!, + signControl(state.installation.privateKey, domains.control, { + version: 2 as const, + projection: source.signedProjection, + binding: envelope, + }), + ); + const assertion = Buffer.from( + canonicalJson( + signControl( + state.installation.privateKey, + domains.assertion, + configurationSettlementAssertionSchema.parse({ + version: 2, + purpose: "configuration-fence-settlement", + settlementId: projection.settlementId, + controlEnvelopeDigest: settlementRecordDigest(domains.binding, binding), + }), + ), + ), + ).toString("base64url"); + if (assertion.length > 4096) throw Error("Fixed settlement assertion capacity"); + freezeOwnedSource(envelope); + if (!fresh) return Object.freeze({ envelope, assertion, dispatch: "observe-only" as const }); + const ticket = Object.freeze({}) as ConfigurationFenceSettlementDispatchTicket; + settlementDispatches.set(ticket, Object.freeze({ control: value, envelope, assertion })); + return Object.freeze({ envelope, assertion, dispatch: "fresh" as const, ticket }); + }); +} + +/** Consumes the winning process-local ticket once, records attempted before + * external I/O, then sends ONE fixed typed POST. Lost response/cancellation is + * observed through the retained guard; this function cannot renew/retry it. */ +export async function dispatchConfigurationFenceSettlementControl( + ticket: ConfigurationFenceSettlementDispatchTicket, + transport: DataManagerTransport, +) { + const dispatch = settlementDispatches.get(ticket); + if (!dispatch) throw Error("Original unconsumed settlement dispatch ticket required"); + settlementDispatches.delete(ticket); + const source = actualSettlementControl(dispatch.control), + state = executionAuthorityState(source.authority), + projection = source.projection, + descriptors = configurationSettlementRecordDescriptors(projection, source.prepareKey).sender; + const actual = actualSettlementControl( + prepareConfigurationFenceSettlementControl(source.authority, source.maintenanceId), + ); + withConfigurationMaintenanceSettlementPublication(source.authority, () => { + assertCurrentSettlementFinal(source); + const binding = readConfigurationSettlementCell( + state.installation, + projection, + descriptors[1]!, + ), + original = verifyControl( + state.installation.installation.publicKey, + domains.dispatch, + readConfigurationSettlementCell(state.installation, projection, descriptors[3]!), + configurationSettlementDispatchSchema, + ), + slot = readConfigurationSettlementFamilySlot(state.installation, projection), + attemptedAt = Date.now(); + if ( + actual.commitmentDigest !== source.commitmentDigest || + canonicalJson( + verifyControl( + state.installation.installation.publicKey, + domains.binding, + binding, + configurationSettlementControlEnvelopeSchema, + ), + ) !== canonicalJson(dispatch.envelope.payload) || + slot.state !== "bound" || + slot.executionDigest !== controlDigest(dispatch.envelope.payload.execution) || + slot.controlBindingDigest !== + settlementRecordDigest(domains.binding, dispatch.envelope.payload) || + original.phase !== "prepared" || + original.dispatchId !== dispatch.envelope.payload.dispatchId || + original.controlEnvelopeDigest !== slot.controlBindingDigest || + original.controlExecutionDigest !== slot.executionDigest || + original.commitmentDigest !== source.commitmentDigest || + attemptedAt < dispatch.envelope.payload.controlStartedAt || + attemptedAt >= dispatch.envelope.payload.controlDeadlineAt + ) + throw Error("Original settlement dispatch is fenced"); + writeConfigurationSettlementCell( + state.installation, + projection, + descriptors[3]!, + signControl(state.installation.privateKey, domains.dispatch, { + ...original, + phase: "attempted" as const, + attemptedAt, + }), + ); + }); + const remaining = dispatch.envelope.payload.controlDeadlineAt - Date.now(); + if (remaining <= 0) throw Error("Original settlement control expired before POST"); + const { kind: _kind, ...operation } = dispatch.envelope.payload.operation; + return settleDataConfigurationFence(operation, { + transport, + controlAssertion: dispatch.assertion, + signal: AbortSignal.timeout(Math.min(15000, remaining)), + }); +} + +/** Verifies the complete original DM terminal, admission and effect preimages + * using ONLY shared DM records plus this role's original retained source. The + * durable copy is fsynced before shared ACK publication; there is no DM-private + * read, new request/inspection time, SQL effect, ordinary grant or outcome arg. */ +export function observeConfigurationFenceSettlementControl( + value: VerifiedConfigurationFenceSettlementControl, +) { + const source = actualSettlementControl(value), + state = executionAuthorityState(source.authority), + projection = source.projection, + descriptors = configurationSettlementRecordDescriptors(projection, source.prepareKey).sender; + const currentSource = actualSettlementControl( + prepareConfigurationFenceSettlementControl(source.authority, source.maintenanceId), + ); + return withConfigurationMaintenanceSettlementPublication(source.authority, () => { + const owner = assertCurrentSettlementFinal(source), + control = verifyControl( + state.installation.installation.publicKey, + domains.control, + readConfigurationSettlementCell(state.installation, projection, descriptors[2]!), + configurationSettlementControlCellSchema.shape.payload, + ); + if ( + !control.binding || + "unused" in control || + !owner.capacity || + !owner.guardSource || + currentSource.commitmentDigest !== source.commitmentDigest + ) + throw Error("Original published settlement source required"); + const envelope = verifyControl( + state.installation.installation.publicKey, + domains.control, + control.binding, + configurationSettlementControlEnvelopeSchema, + ), + binding = verifyControl( + state.installation.installation.publicKey, + domains.binding, + readConfigurationSettlementCell(state.installation, projection, descriptors[1]!), + configurationSettlementControlEnvelopeSchema, + ), + capacity = verifyControl( + state.installation.publicKeys["data-manager"], + domains.capacity, + owner.capacity, + configurationSettlementCapacitySchema, + ); + if ( + canonicalJson(binding) !== canonicalJson(envelope) || + envelope.commitmentDigest !== source.commitmentDigest || + envelope.settlementId !== projection.settlementId || + canonicalJson(control.projection) !== canonicalJson(source.signedProjection) + ) + throw Error("Original control binding changed before observation"); + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + let signedTerminal: ReturnType< + typeof signControl> + >; + try { + const root = own(new AnchoredDirectory(state.installation.dataRoot)), + shared = own(root.directory(".admin-target-control")), + executions = own(shared.directory("executions")), + dm = own(executions.directory("data-manager")), + bytes = dm.read( + `${envelope.execution.operationKey}.configuration-settlement-terminal.json`, + 4096, + ), + raw = JSON.parse(bytes.toString("utf8")), + payload = verifyControl( + state.installation.publicKeys["data-manager"], + domains.terminal, + raw, + configurationSettlementPublishedTerminalSchema, + ); + if (bytes.length !== 4096) throw Error("Original DM terminal capacity cell changed"); + signedTerminal = { payload, signature: (raw as { signature: string }).signature }; + } finally { + for (const directory of opened.reverse()) directory.close(); + } + const terminal = signedTerminal.payload, + tuple = terminal.tuple, + guard = owner.guardSource, + admissionPayload = configurationSettlementAdmissionSchema.parse({ + version: 2, + purpose: "configuration-fence-settlement", + installationId: owner.installationId, + origin: owner.origin, + settlementId: projection.settlementId, + projectionDigest: envelope.projectionDigest, + reservationDigest: envelope.reservationDigest, + prepareExecutionDigest: controlDigest(guard.prepareExecution), + prepareAdmissionDigest: guard.prepareAdmissionDigest, + preparedReceiptDigest: guard.preparedReceiptDigest, + capacityReceiptDigest: settlementRecordDigest(domains.capacity, capacity), + rootCertificateDigest: owner.rootCertificateDigest, + controlEnvelope: control.binding, + commitmentDigest: source.commitmentDigest, + executionOrdinal: capacity.reservedExecutionOrdinal, + intentOrdinal: capacity.reservedIntentOrdinal, + sqlSessionNonce: envelope.execution.executionId, + boundAt: terminal.admissionSource.boundAt, + }), + admission = verifyControl( + state.installation.publicKeys["data-manager"], + domains.admission, + { payload: admissionPayload, signature: terminal.admissionSource.signature }, + configurationSettlementAdmissionSchema, + ), + effect = verifyControl( + state.installation.publicKeys["data-manager"], + domains.effect, + { + payload: { ...terminal.effectSource.fields, terminal: tuple }, + signature: terminal.effectSource.signature, + }, + configurationSettlementEffectSchema, + ), + { terminalDigest, ...tuplePreimage } = tuple; + if ( + canonicalJson(terminal.origin) !== canonicalJson(owner.origin) || + canonicalJson(tuple.execution) !== canonicalJson(envelope.execution) || + tuple.settlementId !== projection.settlementId || + tuple.commitmentDigest !== source.commitmentDigest || + tuple.fenceId !== guard.preparedReceipt.fenceId || + tuple.terminalAt < envelope.controlStartedAt || + tuple.terminalAt >= envelope.controlDeadlineAt || + terminalDigest !== settlementRecordDigest(domains.terminal, tuplePreimage) || + canonicalJson(tuple.result) !== + canonicalJson({ + ...guard.preparedReceipt, + state: "settled", + sequence: guard.preparedReceipt.sequence + 1, + }) || + terminal.admissionDigest !== settlementRecordDigest(domains.admission, admission) || + admission.boundAt < envelope.controlStartedAt || + admission.boundAt >= envelope.controlDeadlineAt || + terminal.effectSettlementDigest !== settlementRecordDigest(domains.effect, effect) || + effect.phase !== "committed" || + canonicalJson(effect.origin) !== canonicalJson(owner.origin) || + effect.settlementId !== projection.settlementId || + effect.controlExecutionDigest !== controlDigest(envelope.execution) || + effect.controlEnvelopeDigest !== settlementRecordDigest(domains.binding, envelope) || + effect.admissionDigest !== terminal.admissionDigest || + effect.commitmentDigest !== source.commitmentDigest || + effect.preparedReceiptDigest !== guard.preparedReceiptDigest || + effect.expectedDmSequence !== guard.preparedReceipt.sequence || + effect.sqlSession.sessionNonce !== envelope.execution.executionId || + effect.attemptedAt < admission.boundAt || + effect.attemptedAt >= envelope.controlDeadlineAt + ) + throw Error("Actual original DM settlement proof changed"); + verifyPublishedBoundDmConfigurationSettlementSlots( + state.installation, + projection, + owner.capacity, + ); + const ack = signControl( + state.installation.privateKey, + domains.ack, + configurationSettlementAckSchema.parse({ + version: 2, + installationId: owner.installationId, + origin: owner.origin, + settlementId: projection.settlementId, + controlExecutionDigest: controlDigest(envelope.execution), + terminalDigest: tuple.terminalDigest, + terminalAt: tuple.terminalAt, + commitmentDigest: source.commitmentDigest, + }), + ), + copied = readConfigurationSettlementCell(state.installation, projection, descriptors[4]!); + if (copied === null) + writeConfigurationSettlementCell( + state.installation, + projection, + descriptors[4]!, + configurationSettlementSenderAckCellSchema.parse({ + version: 2, + terminal: signedTerminal, + ack: null, + }), + ); + else { + const original = configurationSettlementSenderAckCellSchema.parse(copied); + if ( + canonicalJson(original.terminal) !== canonicalJson(signedTerminal) || + (original.ack && canonicalJson(original.ack) !== canonicalJson(ack)) + ) + throw Error("Original sender durable terminal copy changed"); + } + writeConfigurationSettlementCell( + state.installation, + projection, + descriptors[4]!, + configurationSettlementSenderAckCellSchema.parse({ + version: 2, + terminal: signedTerminal, + ack, + }), + ); + const sharedAck = readConfigurationSettlementCell( + state.installation, + projection, + descriptors[5]!, + ); + if (sharedAck === null) + writeConfigurationSettlementCell(state.installation, projection, descriptors[5]!, ack); + else if (canonicalJson(sharedAck) !== canonicalJson(ack)) + throw Error("Original shared settlement ACK changed"); + if (owner.receipt.dmFence?.state === "settled") { + if (canonicalJson(owner.receipt.dmFence) !== canonicalJson(tuple.result)) + throw Error("Original settled owner receipt changed"); + } else { + if (canonicalJson(owner.receipt.dmFence) !== canonicalJson(guard.preparedReceipt)) + throw Error("Original prepared owner receipt changed"); + const sequence = owner.sequence + 1; + write(source.authority, { + ...owner, + sequence, + receipt: { ...owner.receipt, sequence, dmFence: tuple.result }, + }); + } + return tuple.result; + }); +} + +/** Uses the existing original-root recovery sequence, never a new owner or a + * caller-selected settlement. Every physical settle/release path rechecks the + * same full disjoint closure predicate. Completion metadata follows the actual + * retained release proof; a lost metadata write can only finish that proof. */ +export async function releaseOwnedConfigurationMaintenanceRoot( + authority: VerifiedExecutionAuthority, +): Promise { + const state = executionAuthorityState(authority), + original = read(authority); + restoreOwnedMaintenanceFinal(authority); + assertConfigurationMaintenanceRootClosure(authority); + if (original.receipt.state === "settled") { + verifyRetainedArtifactTargetRootRelease(authority); + return original.receipt; + } + // Physical release precedes this owner's metadata publication. A lost owner + // write must observe that retained release, rather than reopen a lease whose + // active target entry was already removed. Failure still takes only the + // original settlement-only reconciliation path, which verifies its sources. + let physicallyReleased = false; + try { + verifyRetainedArtifactTargetRootRelease(authority); + physicallyReleased = true; + } catch { + /* A held/partial original release is reconciled below. */ + } + if (!physicallyReleased) { + const lease = restoreArtifactTargetLeaseForSettlement(authority), + leaseId = lease.leaseId; + closeArtifactTargetLeaseHandle(lease); + const released = await reconcileArtifactTargetLease({ + dataRoot: state.installation.dataRoot, + leaseId, + authority: verifyControlRecoveryAuthority(authority), + }); + if (released !== "released") throw Error("Original prepared maintenance root remains fenced"); + } + verifyRetainedArtifactTargetRootRelease(authority); + assertConfigurationMaintenanceRootClosure(authority); + return withPrivateEffectWriter(state, () => { + const current = read(authority); + if ( + canonicalJson(current.final) !== canonicalJson(original.final) || + !current.guardSource || + current.noGuard || + current.receipt.dmFence?.state !== "settled" || + !["settling", "settled"].includes(current.receipt.state) + ) + throw Error("Original released maintenance completion source changed"); + if (current.receipt.state === "settled") return current.receipt; + const completed = configurationMaintenanceReleasedOwner(current); + write(authority, completed); + return completed.receipt; + }); +} + +/** Fixed NoGuard producer. All late fields come from original retained source; + * it accepts neither a caller proof tuple nor a requested successful outcome. */ +export async function closeOwnedConfigurationMaintenanceWithoutGuard( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, +): Promise { + const state = executionAuthorityState(authority); + if (read(authority).issuance === "closed_without_guard") { + try { + verifyRetainedArtifactTargetRootRelease(authority); + assertConfigurationMaintenanceRootClosure(authority); + return; + } catch { + /* A held/partial release still uses the original fixed repair path. */ + } + } + await withUnconsumedMaintenanceCandidate(runtime, authority, (witness) => { + const owner = read(authority), + observation = unconsumedMaintenanceCandidateState(witness, authority), + prepare = owner.plan.reservations.find((value) => value.phase === "configuration-fence"); + if ( + !prepare || + owner.guardSource || + owner.receipt.dmFence || + owner.receipt.decision?.decision === "continue" || + owner.receipt.generation !== "not_started" || + observation.ownerPayloadDigest !== controlDigest(owner) + ) + throw Error("Actual original never-applied candidate required"); + const prepareState = inspectControlChildPublication(authority, prepare), + issued = prepareState.publication.state !== "unissued", + verifiedRefusal = issued ? verifyPublishedDataConfigurationFenceRefusal(authority) : null, + verifiedUnused = verifiedRefusal + ? verifyPublishedUnusedConfigurationCapacity(authority, verifiedRefusal) + : null, + refusal = verifiedRefusal && publishedDmNoGuardRefusalState(verifiedRefusal, authority), + dmUnused = verifiedUnused && publishedDmUnusedCapacityState(verifiedUnused, authority); + // Reservation/projection is the same pre-effect value even on recovery. + const projectionDescriptor = { + scope: "shared" as const, + namespace: "execution" as const, + role: "ops-agent" as const, + name: `${state.execution.operationKey}.configuration-settlement-control.json`, + cap: 8192, + }; + // Only the family-indexed original cell is read; the provisional identity + // carries no authority and is used solely by that index verification. + const rawProjection = readConfigurationSettlementCell( + state.installation, + { origin: state.origin! }, + projectionDescriptor, + ), + control = verifyControl( + state.installation.installation.publicKey, + domains.control, + rawProjection, + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + state.installation.installation.publicKey, + domains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + projectionDigest = settlementRecordDigest(domains.projection, projection), + descriptors = configurationSettlementRecordDescriptors( + projection, + prepare.execution.operationKey, + ).sender, + identity = configurationSettlementOwnerIdentitySchema.parse({ + installationId: owner.installationId, + origin: owner.origin, + ownerPrepare: state.execution, + admissionDigest: state.receiptDigest, + rootCertificateDigest: owner.rootCertificateDigest, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + planDigest: owner.plan.planDigest, + settlementId: owner.plan.dmSettle.settlementId, + }); + if ( + control.binding || + projection.rootAdmissionDigest !== state.receiptDigest || + projection.rootCertificateDigest !== owner.rootCertificateDigest || + projection.settlementId !== identity.settlementId || + canonicalJson(projection.ownerPrepare) !== canonicalJson(state.execution) || + canonicalJson(projection.reservation) !== + canonicalJson( + owner.plan.reservations.find((value) => value.phase === "configuration-fence-settle"), + ) || + (refusal && + (refusal.projectionDigest !== projectionDigest || + refusal.signedRefusal.payload.maintenanceId !== identity.maintenanceId)) + ) + throw Error("Original frozen reservation projection changed"); + withConfigurationMaintenanceNoGuardPublication(authority, () => { + const current = read(authority); + if (canonicalJson(current) !== canonicalJson(owner)) + throw Error("Original issuer changed during candidate observation"); + const publications = owner.plan.reservations + .filter((value) => !issued || value !== prepare) + .map((value) => { + const source = inspectControlChildPublication(authority, value); + if ( + source.publication.state !== "unissued" || + lookupControlChildPhaseAttempt(authority, value) + ) + throw Error("Issued or ambiguous remaining maintenance phase"); + return { + phase: value.phase, + ordinal: value.ordinal, + operationKey: value.execution.operationKey, + grantState: "unissued" as const, + intentState: "not_attempted" as const, + }; + }); + if (!issued) + assertNoUnissuedConfigurationReceiverPublication( + state.installation, + prepare.execution.operationKey, + ); + if (owner.noGuard) { + if ( + owner.issuance === "open" || + owner.sequence !== owner.noGuard.ownerSequence || + canonicalJson(publications) !== + canonicalJson( + "remainingPublicationStates" in owner.noGuard.issuerFreezeProof.payload + ? owner.noGuard.issuerFreezeProof.payload.remainingPublicationStates + : owner.noGuard.issuerFreezeProof.payload.publicationStates, + ) + ) + throw Error("Original issuance freeze conflict"); + return; + } + if (owner.issuance !== "open") throw Error("Original issuer cannot reopen"); + const ownerSequence = owner.sequence + 1, + frozenAt = Date.now(), + freezeDomain = issued ? domains.issuedRefused : domains.neverIssued, + proof = issued + ? configurationIssuedPrepareRefusedProofSchema.parse({ + version: 2, + kind: "issued_preparation_refused", + identity, + issuerSequence: ownerSequence, + frozenAt, + prepareGrantDigest: + prepareState.publication.state === "grant_prepared" + ? prepareState.publication.grantDigest + : "", + prepareExecutionDigest: controlDigest( + refusal!.signedRefusal.payload.prepareExecution, + ), + refusalProjectionDigest: settlementRecordDigest( + domains.refusal, + refusal!.signedRefusal.payload, + ), + remainingPublicationStates: publications, + candidateClaimState: "not_consumed", + }) + : configurationNeverIssuedProofSchema.parse({ + version: 2, + identity, + issuerSequence: ownerSequence, + frozenAt, + prepareReservationDigest: controlDigest(prepare), + publicationStates: publications, + }), + issuerFreezeProof = signControl(state.installation.privateKey, freezeDomain, proof), + issuerFreezeProofDigest = settlementRecordDigest(freezeDomain, proof), + noGuardEvidence = configurationNoGuardEvidenceSchema.parse( + issued + ? { + kind: "receiver_refused", + issuerFreezeProofDigest, + prepareExecutionDigest: controlDigest( + refusal!.signedRefusal.payload.prepareExecution, + ), + prepareAdmissionDigest: refusal!.signedRefusal.payload.prepareAdmissionDigest, + refusalProjectionDigest: settlementRecordDigest( + domains.refusal, + refusal!.signedRefusal.payload, + ), + effectSettlementDigest: refusal!.signedRefusal.payload.effectSettlementDigest, + canonicalTerminalDigest: refusal!.signedRefusal.payload.canonicalTerminalDigest, + canonicalTerminalAt: refusal!.signedRefusal.payload.canonicalTerminalAt, + dmUnusedIntentDigest: dmUnused!.intentDigest, + dmUnusedConfirmationDigest: dmUnused!.confirmationDigest, + capacityReceiptDigest: refusal!.signedRefusal.payload.capacityReceiptDigest, + } + : { + kind: "never_issued", + issuerFreezeProofDigest, + prepareReservationDigest: controlDigest(prepare), + publicationStatesDigest: settlementRecordDigest(domains.neverIssued, proof), + issuerSequence: ownerSequence, + frozenAt, + }, + ), + slot = readConfigurationSettlementFamilySlot(state.installation, projection), + intent = configurationUnusedCapacityIntentSchema.parse({ + version: 2, + installationId: owner.installationId, + origin: owner.origin, + role: "ops-agent", + ownerPrepareDigest: controlDigest(state.execution), + prepareExecutionDigest: issued + ? noGuardEvidence.kind === "receiver_refused" + ? noGuardEvidence.prepareExecutionDigest + : null + : null, + settlementId: identity.settlementId, + projectionDigest, + capacityReceiptDigest: issued + ? refusal!.signedRefusal.payload.capacityReceiptDigest + : null, + sourceProofDigest: configurationIssuerUnusedSourceDigest(noGuardEvidence), + executionOrdinal: slot.executionOrdinal, + intentOrdinal: slot.intentOrdinal, + recordDescriptorsDigest: slot.recordDescriptorsDigest, + allocatedDescriptorDigests: slot.allocatedDescriptorDigests, + createdAt: frozenAt, + }); + if (Buffer.byteLength(canonicalJson(issuerFreezeProof)) > 4096) + throw Error("Original issuer freeze exceeds preallocated proof budget"); + write(authority, { + ...owner, + sequence: ownerSequence, + issuance: "closing_without_guard", + receipt: { ...owner.receipt, sequence: ownerSequence, state: "recovery_required" }, + noGuard: configurationNoGuardIntentSchema.parse({ + ownerSequence, + closedAt: frozenAt, + noGuardEvidence, + issuerFreezeProof, + issuerUnused: { + intent: signControl(state.installation.privateKey, domains.unusedIntent, intent), + confirmation: null, + }, + bundle: null, + }), + }); + }); + const slot = commitFrozenConfigurationSettlementSlotUnused(state.installation, projection); + withConfigurationMaintenanceNoGuardPublication(authority, () => { + const current = read(authority), + intent = current.noGuard; + if ( + !intent || + current.issuance === "open" || + slot.unusedIntentDigest !== + settlementRecordDigest(domains.unusedIntent, intent.issuerUnused.intent.payload) + ) + throw Error("Original frozen issuer CAS required"); + if (!intent.issuerUnused.confirmation) { + const confirmation = configurationUnusedCapacityConfirmationSchema.parse({ + version: 2, + installationId: current.installationId, + origin: current.origin, + role: "ops-agent", + ownerPrepareDigest: controlDigest(state.execution), + settlementId: identity.settlementId, + projectionDigest, + intentDigest: slot.unusedIntentDigest, + slotStateDigest: controlDigest({ + executionOrdinal: slot.executionOrdinal, + intentOrdinal: slot.intentOrdinal, + recordDescriptorsDigest: slot.recordDescriptorsDigest, + allocatedDescriptorDigests: slot.allocatedDescriptorDigests, + state: "UNUSED", + intentDigest: slot.unusedIntentDigest, + }), + confirmedAt: slot.unusedConfirmedAt, + }); + write(authority, { + ...current, + noGuard: { + ...intent, + issuerUnused: { + ...intent.issuerUnused, + confirmation: signControl( + state.installation.privateKey, + domains.unusedConfirmation, + confirmation, + ), + }, + }, + }); + } + }); + const frozen = read(authority).noGuard!; + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { + kind: "configuration-no-guard", + issuerFreezeProofDigest: frozen.noGuardEvidence.issuerFreezeProofDigest, + issuerUnusedConfirmationDigest: settlementRecordDigest( + domains.unusedConfirmation, + frozen.issuerUnused.confirmation!.payload, + ), + }, + }); + verifyControlEffectSettlement(state.installation.installation, state.execution); + const effect = readPrivateRecord( + state.installation, + "effects", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload: unknown }; + withConfigurationMaintenanceNoGuardPublication(authority, () => { + const current = read(authority), + intent = current.noGuard!; + if (!intent.issuerUnused.confirmation) throw Error("Original issuer confirmation required"); + const { issuerUnused } = intent, + confirmation = intent.issuerUnused.confirmation!, + ownIntentDigest = settlementRecordDigest(domains.unusedIntent, issuerUnused.intent.payload), + ownConfirmationDigest = settlementRecordDigest( + domains.unusedConfirmation, + confirmation.payload, + ), + containment = [ + controlDigest(effect.payload), + ...(refusal ? [refusal.signedRefusal.payload.effectSettlementDigest] : []), + ].sort(), + closureSource = { + version: 2 as const, + identity, + ownerSequence: intent.ownerSequence, + closedAt: intent.closedAt, + noGuardEvidence: intent.noGuardEvidence, + originalContainmentDigests: containment, + unusedSlotDigests: [ + ...(dmUnused + ? [ + { + role: "data-manager" as const, + intentDigest: dmUnused.intentDigest, + confirmationDigest: dmUnused.confirmationDigest, + }, + ] + : []), + { + role: "ops-agent" as const, + intentDigest: ownIntentDigest, + confirmationDigest: ownConfirmationDigest, + }, + ], + }, + closure = configurationNoGuardClosureSchema.parse({ + ...closureSource, + closureDigest: settlementRecordDigest(domains.closure, closureSource), + }), + bundle = configurationNoGuardBundleSchema.parse({ + version: 2, + phase: "unused", + settlementId: identity.settlementId, + projectionDigest, + closure: signControl(state.installation.privateKey, domains.closure, closure), + issuerFreezeProof: intent.issuerFreezeProof, + issuerUnused: { intent: issuerUnused.intent, confirmation }, + }), + signedBundle = + intent.bundle ?? signControl(state.installation.privateKey, domains.reference, bundle); + if (canonicalJson(signedBundle.payload) !== canonicalJson(bundle)) + throw Error("Original immutable NoGuard bundle conflict"); + const prior = verifyControl( + state.installation.installation.publicKey, + domains.reference, + readConfigurationSettlementCell(state.installation, projection, descriptors[0]!), + configurationSettlementCommitmentCellSchema.shape.payload, + ); + if (prior.phase !== "uncommitted" && canonicalJson(prior) !== canonicalJson(bundle)) + throw Error("Committed settlement cannot become UNUSED"); + const linked = signControl(state.installation.privateKey, domains.control, { + version: 2 as const, + projection: control.projection, + binding: null, + unused: { + bundleDigest: settlementRecordDigest(domains.bundle, bundle), + closureDigest: closure.closureDigest, + issuerFreezeProofDigest: intent.noGuardEvidence.issuerFreezeProofDigest, + issuerUnusedIntentDigest: ownIntentDigest, + issuerUnusedConfirmationDigest: ownConfirmationDigest, + }, + }); + if ( + Buffer.byteLength(canonicalJson(signedBundle)) > descriptors[0]!.cap || + Buffer.byteLength(canonicalJson(linked)) > descriptors[2]!.cap + ) + throw Error("Complete signed NoGuard packet exceeds original cells"); + if (!intent.bundle) + write(authority, { ...current, noGuard: { ...intent, bundle: signedBundle } }); + if (prior.phase === "uncommitted") + writeConfigurationSettlementCell( + state.installation, + projection, + descriptors[0]!, + signedBundle, + ); + const currentControl = readConfigurationSettlementCell( + state.installation, + projection, + descriptors[2]!, + ); + if (canonicalJson(currentControl) !== canonicalJson(linked)) { + if (canonicalJson(currentControl) !== canonicalJson(rawProjection)) + throw Error("Original NoGuard link conflict"); + writeConfigurationSettlementCell(state.installation, projection, descriptors[2]!, linked); + } + const complete = read(authority); + if (complete.issuance !== "closed_without_guard") + write(authority, { + ...complete, + issuance: "closed_without_guard", + receipt: { ...complete.receipt, state: "settled" }, + }); + }); + }); +} + +// The actual admission/inspection producer and its private consumer share ONE nominal owner. +interface Source { + infraDir: string; + candidateSource?: ConfigurationCandidateSource; + stack: services.StackPaths; + execFile: NonNullable; + loadAuthority(): Promise; + ownedServices: OwnedServiceResolver; +} +/** Repair only the already attempted original immutable SQL/proof/ACK chain. + * This path has no dispatch ticket and never inspects runtime or POSTs again. */ +async function observeCommittedMaintenanceSettlement( + authority: VerifiedExecutionAuthority, + signal: AbortSignal, +) { + // A physical release already proves the full terminal/copy/ACK chain. Do not + // re-enter its now-closed settlement publication scope after a lost owner + // metadata write; complete only that original retained release. + let released = false; + try { + verifyRetainedArtifactTargetRootRelease(authority); + released = true; + } catch { + /* Held/partial roots still need the original observation chain. */ + } + if (released) { + await releaseOwnedConfigurationMaintenanceRoot(authority); + return; + } + const owner = readOwnedConfigurationMaintenance(authority), + state = executionAuthorityState(authority), + settlement = restoreAttemptedConfigurationFenceSettlementControl( + authority, + owner.plan.operation.maintenanceId, + ), + issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: issuedAt + 15000, + }, + ), + options = { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal, + }; + await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, options); + observeConfigurationFenceSettlementControl(settlement); + await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, options); + await releaseOwnedConfigurationMaintenanceRoot(authority); +} +const sources = new WeakMap(); +/** The normal root remains held while this writer-side preflight runs. Release + * the writer before invoking the fixed CLI, which acquires its own writer. */ +export async function assertManagedCliConfigurationPreflight( + runtime: OpsRuntime, + operation: OpsOperation, + context: import("./runtime").OpsExecutionContext, +): Promise { + if ( + !( + operation.kind === "service.update" || + operation.kind === "stack.start" || + (operation.kind === "service.start" && operation.serviceId !== "data-manager") + ) + ) + return; + const source = sources.get(runtime), + lease = context.ownedTargetLease; + if (!source || !context.controlAuthority || !lease || context.artifactLease) + throw Error("Original full managed configuration target owner required"); + const targets = (await source.loadAuthority()).services + .map((service) => service.manifest.id) + .sort(); + if (canonicalJson(targets) !== canonicalJson(lease.serviceIds)) + throw Error("Managed configuration target closure changed"); + try { + await services.withConfigurationGenerationWriter( + source.infraDir, + () => assertStagedCoordinatorConfigurationWrite(runtime, operation, context, context.signal), + context.signal, + ); + } catch (error) { + if ( + error instanceof services.ConfigurationGenerationError && + error.result.outcome === "not_committed" && + !error.result.cleanupFailed && + error.cause instanceof OpsExecutionFailure && + error.cause.errorClass === "configuration_prerequisite" + ) + throw error.cause; + throw error; + } +} +/** Read only from this runtime's existing fixed source and literal resolver. + * Planned eligibility never substitutes for actual nominal publication. */ +export async function assertStagedCoordinatorReleaseWrite( + runtime: OpsRuntime, + signal: AbortSignal, +): Promise { + const source = sources.get(runtime); + if ( + !source || + (await inspectLiteralStagedConfigurationDependency(source.ownedServices, signal)) !== "none" + ) + throw new OpsExecutionFailure("configuration_prerequisite", {}); +} + +export async function assertStagedCoordinatorConfigurationWrite( + runtime: OpsRuntime, + operation: OpsOperation, + context: import("./runtime").OpsExecutionContext, + signal: AbortSignal, +): Promise { + const source = sources.get(runtime); + if (!source) throw new OpsExecutionFailure("configuration_prerequisite", {}); + const dependency = await inspectLiteralStagedConfigurationDependency( + source.ownedServices, + signal, + ); + if (dependency === "none") return; + if (dependency === "required") { + try { + if (operation.kind === "configuration.hardlinks.preview") { + await assertPlannedCoordinatorConfigurationWrite(runtime, operation); + return; + } + if ( + (operation.kind === "stack.render" || operation.kind === "vault.apply") && + operation.maintenanceId + ) { + await assertOwnedCoordinatorConfigurationWrite(runtime, context, operation, signal); + return; + } + } catch { + /* Changed or foreign producer keeps the prerequisite closed. */ + } + } + throw new OpsExecutionFailure("configuration_prerequisite", {}); +} +const journals = new WeakMap(); +/** Bound to the server's actual durable journal. Records alone cannot qualify + * a pull: the fixed reader authenticates canonical/effect/private admissions. */ +export function registerConfigurationMaintenanceJournal( + runtime: OpsRuntime, + journal: OpsJobJournal, +) { + if (journals.has(runtime)) throw Error("Maintenance journal source already bound"); + journals.set(runtime, journal); + return () => { + if (journals.get(runtime) === journal) journals.delete(runtime); + }; +} +declare const admissionBrand: unique symbol; +export type ConfigurationMaintenanceAdmission = Readonly<{ [admissionBrand]: true }>; +interface AdmissionState { + plan: FixedConfigurationMaintenancePlan; + originalContainers: readonly OwnedServiceSnapshot[]; + coordinatorPull?: CoordinatorPullWitness; + stagedCoordinatorProvenance?: StagedCoordinatorObservation; +} +const admissions = new WeakMap(); +declare const continueObservationBrand: unique symbol; +export type MaintenanceContinueObservation = Readonly<{ [continueObservationBrand]: true }>; +const continueObservations = new WeakMap< + MaintenanceContinueObservation, + { + authority: VerifiedExecutionAuthority; + ownerDigest: string; + observedAt: number; + sealExpiresAt: number; + } +>(); +/** Only the fixed production sources create this short-lived observation. + * It never authorizes an effect or survives as a parsed summary. */ +export async function prepareMaintenanceContinueObservation( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + signal: AbortSignal, +): Promise { + const source = sources.get(runtime), + owner = readOwnedConfigurationMaintenance(authority), + state = executionAuthorityState(authority); + if ( + !source?.candidateSource || + owner.receipt.state !== "prepared" || + owner.receipt.decision || + owner.noGuard || + !owner.guardSource || + owner.issuance !== "open" + ) + throw Error("Original prepared maintenance continue source required"); + const fresh = configurationMaintenanceAdmissionState( + await prepareConfigurationMaintenanceAdmission(runtime, { + installation: state.installation.installation, + execution: owner.plan.prepare, + operation: owner.plan.operation, + apiParent: owner.plan.apiParent, + signal, + }), + ); + if ( + canonicalJson(fresh.plan) !== canonicalJson(owner.plan) || + canonicalJson(fresh.originalContainers) !== canonicalJson(owner.originalContainers) + ) + throw Error("Original maintenance source drifted before decision"); + const candidate = await readConfigurationCandidate( + source.candidateSource, + owner.plan.operation.candidateId, + ), + issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), + }, + ), + actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal, + }); + if ( + canonicalJson(actual) !== canonicalJson(owner.guardSource.preparedReceipt) || + controlDigest(candidate.record) !== owner.plan.candidateRecordDigest || + canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) + ) + throw Error("Original prepared maintenance guard/candidate changed"); + signal.throwIfAborted(); + const value = Object.freeze({}) as MaintenanceContinueObservation; + continueObservations.set(value, { + authority, + ownerDigest: controlDigest(owner), + observedAt: Date.now(), + sealExpiresAt: candidate.snapshot.expiresAtMs, + }); + return value; +} +export function consumeMaintenanceContinueObservation( + value: MaintenanceContinueObservation | undefined, + authority: VerifiedExecutionAuthority, +): void { + const observed = value && continueObservations.get(value), + now = Date.now(); + if ( + !observed || + observed.authority !== authority || + !Number.isSafeInteger(observed.sealExpiresAt) || + now < observed.observedAt || + now - observed.observedAt > 15000 || + now >= observed.sealExpiresAt || + observed.ownerDigest !== controlDigest(readOwnedConfigurationMaintenance(authority)) + ) + throw Error("Actual current maintenance continue observation required"); + continueObservations.delete(value!); +} +/** Fixed first-generation source check; no callback or caller receipt can + * replace the actual candidate/current/host/container/ready/DM observations. */ +export async function verifyFrozenMaintenanceGenerationSource( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + signal: AbortSignal, +): Promise { + const source = sources.get(runtime), + owner = readOwnedConfigurationMaintenance(authority), + state = executionAuthorityState(authority); + if ( + !source?.candidateSource || + owner.receipt.state !== "authorized" || + owner.receipt.decision?.decision !== "continue" || + !owner.apiControls.decide?.receiptDelivery || + !owner.guardSource || + owner.noGuard || + owner.issuance !== "open" || + owner.receipt.generation !== "not_started" + ) + throw Error("Original frozen maintenance generation source required"); + const fresh = configurationMaintenanceAdmissionState( + await prepareConfigurationMaintenanceAdmission(runtime, { + installation: state.installation.installation, + execution: owner.plan.prepare, + operation: owner.plan.operation, + apiParent: owner.plan.apiParent, + signal, + }), + ); + if ( + canonicalJson(fresh.plan) !== canonicalJson(owner.plan) || + canonicalJson(fresh.originalContainers) !== canonicalJson(owner.originalContainers) + ) + throw Error("Frozen maintenance generation source drifted"); + const issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), + }, + ), + actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal, + }); + if ( + canonicalJson(actual) !== canonicalJson(owner.guardSource.preparedReceipt) || + canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) + ) + throw Error("Frozen maintenance generation guard/source changed"); + signal.throwIfAborted(); +} +/** Follow-up consumes only this owner's actual committed generation and + * terminal first child. It never rereads desired rows, renders or starts DM. */ +export async function verifyFrozenMaintenanceHardlinksSource( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + signal: AbortSignal, +): Promise { + const source = sources.get(runtime), + owner = readOwnedConfigurationMaintenance(authority), + state = executionAuthorityState(authority), + generation = owner.apiControls.steps.find((value) => value.ordinal === 0); + if ( + !source || + owner.receipt.state !== "applying" || + owner.receipt.decision?.decision !== "continue" || + owner.noGuard || + owner.issuance !== "open" || + owner.receipt.generation !== "committed" || + owner.receipt.appliedRevisionId !== owner.plan.operation.revisionId || + !generation?.canonical || + !generation.receiptDelivery || + !owner.guardSource || + !owner.plan.nestedHardlinks + ) + throw Error("Original committed maintenance generation prerequisite required"); + const verified = verifyConfigurationMaintenanceChildTerminal( + authority, + 0, + generation.canonical.status, + generation.canonical.terminalAt, + ), + proof = configurationMaintenanceChildState(verified, authority), + lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + const physical = inspectArtifactTargetLeaseChildSettlement(lease, generation.execution); + if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) + throw Error("Original generation child settlement required"); + } finally { + closeArtifactTargetLeaseHandle(lease); + } + if ( + services.currentConfigurationGeneration(source.infraDir) !== owner.plan.operation.revisionId || + canonicalJson( + readOwnedGenerationHardlinkPlan(source.infraDir, owner.plan.operation.revisionId), + ) !== canonicalJson(owner.plan.hardlinkPlan) + ) + throw Error("Original committed hardlink generation changed"); + const containers = await inspectOriginalOwnerServices(source, authority, signal); + if (containers.find((value) => value.serviceId === "data-manager")?.state !== "running") + throw Error("Original maintenance coordinator stopped"); + await assertConfigurationCoordinatorReady(signal); + const issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), + }, + ), + actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal, + }); + if ( + canonicalJson(actual) !== canonicalJson(owner.guardSource.preparedReceipt) || + canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) || + services.currentConfigurationGeneration(source.infraDir) !== owner.plan.operation.revisionId + ) + throw Error("Original maintenance follow-up source changed"); + signal.throwIfAborted(); +} +/** Ordinary frozen recreation only. Every earlier required child has its + * original private canonical/effect, physical settlement and API copy ACK; + * the committed generation and existing DM guard remain exact. */ +export async function verifyFrozenMaintenanceRecreateSource( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + ordinal: number, + signal: AbortSignal, +): Promise { + const source = sources.get(runtime), + owner = readOwnedConfigurationMaintenance(authority), + state = executionAuthorityState(authority), + step = owner.plan.steps[ordinal]; + if ( + !source || + !step || + step.phase !== "maintenance-recreate" || + (step.operation.kind !== "service.recreate" && + step.operation.kind !== "service.recreateIsolated") || + owner.receipt.state !== "applying" || + owner.receipt.decision?.decision !== "continue" || + !owner.apiControls.decide?.receiptDelivery || + owner.noGuard || + owner.issuance !== "open" || + !owner.guardSource || + owner.receipt.generation !== "committed" || + owner.receipt.appliedRevisionId !== owner.plan.operation.revisionId || + services.currentConfigurationGeneration(source.infraDir) !== owner.plan.operation.revisionId + ) + throw Error("Original frozen recreation generation required"); + const lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + for (const index of step.prerequisites) { + const child = owner.apiControls.steps.find((value) => value.ordinal === index); + if (!child?.canonical || !child.receiptDelivery) + throw Error("Original recreation prerequisite copy required"); + const proof = configurationMaintenanceChildState( + verifyConfigurationMaintenanceChildTerminal( + authority, + index, + child.canonical.status, + child.canonical.terminalAt, + ), + authority, + ), + physical = inspectArtifactTargetLeaseChildSettlement(lease, child.execution); + if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) + throw Error("Original recreation prerequisite settlement required"); + } + } finally { + closeArtifactTargetLeaseHandle(lease); + } + const currentAuthority = await source.loadAuthority(); + if (currentAuthority.revisionId !== owner.plan.sourceAuthorityRevision) + throw Error("Frozen recreation authority changed"); + if (owner.plan.hardlinkPlan.plan.length || owner.plan.coordinatorImage) { + const containers = await inspectOriginalOwnerServices(source, authority, signal); + if (containers.find((value) => value.serviceId === "data-manager")?.state !== "running") + throw Error("Original recreation coordinator stopped"); + await assertConfigurationCoordinatorReady(signal); + } + const issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), + }, + ), + actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal, + }); + if ( + canonicalJson(actual) !== canonicalJson(owner.guardSource.preparedReceipt) || + canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) || + services.currentConfigurationGeneration(source.infraDir) !== owner.plan.operation.revisionId + ) + throw Error("Original recreation guard/generation changed"); + signal.throwIfAborted(); +} +declare const runtimeInspectionBrand: unique symbol; +export type MaintenanceRuntimeInspection = Readonly<{ [runtimeInspectionBrand]: true }>; +const runtimeInspections = new WeakMap< + MaintenanceRuntimeInspection, + { + authority: VerifiedExecutionAuthority; + signed: ReturnType< + typeof signControl> + >; + } +>(); +export function maintenanceRuntimeInspectionState( + value: MaintenanceRuntimeInspection, + authority: VerifiedExecutionAuthority, +) { + const source = runtimeInspections.get(value); + if (!source || source.authority !== authority) + throw Error("Actual maintenance runtime inspection required"); + return source; +} +/** Fixed actual unused inspector. A decision/receipt alone proves nothing: + * observe the real held tree, all original phase publications, retained + * authenticated unconsumed candidate/queue, Docker inventory and DM SQL/mirror. + * Applied/contained modes remain closed until their complete check matrix is + * installed; they cannot borrow this no-effect producer. */ +export async function inspectFrozenMaintenanceRuntime( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + signal: AbortSignal, +): Promise { + const source = sources.get(runtime), + owner = readOwnedConfigurationMaintenance(authority), + state = executionAuthorityState(authority); + if (owner.receipt.decision?.decision === "continue") { + const contained = + owner.receipt.generation === "not_committed" && + owner.receipt.followUp === "contained" && + owner.receipt.appliedRevisionId === null; + if ( + !source?.candidateSource || + owner.final || + owner.inspection || + owner.noGuard || + owner.issuance !== "open" || + !owner.guardSource || + owner.receipt.state !== "applying" || + !owner.apiControls.decide?.receiptDelivery || + (!contained && + (owner.receipt.generation !== "committed" || + owner.receipt.appliedRevisionId !== owner.plan.operation.revisionId || + owner.receipt.followUp !== "complete")) + ) + throw Error("Conclusive applied maintenance runtime source required"); + const children = readOriginalMaintenanceFinalChildren( + authority, + observeConfigurationMaintenanceRuntimeTree(authority), + ); + if ( + (!contained && !children.complete) || + children.ownerDigest !== controlDigest(owner) || + !children.generation || + (contained && + (children.generationResult?.outcome !== "not_committed" || + children.generationResult.cleanupFailed || + children.children.length !== 1 || + owner.apiControls.steps.length !== 1)) + ) + throw Error("Complete original runtime child sources required"); + const candidate = await readRetainedConfigurationCandidate( + source.candidateSource, + owner.plan.operation.candidateId, + ); + if ( + controlDigest(candidate.record) !== owner.plan.candidateRecordDigest || + candidate.snapshot.revisionId !== owner.plan.operation.revisionId + ) + throw Error("Original committed candidate changed"); + const inspectContainers = () => + contained + ? inspectOriginalOwnerServices(source, authority, signal) + : source.ownedServices.inspectOwnedServices(signal); + const containers = await inspectContainers(), + recreateIds = owner.plan.steps + .filter((step) => step.phase === "maintenance-recreate") + .map((step) => ("serviceId" in step.operation ? step.operation.serviceId : "")), + unchanged = containers.filter((value) => !recreateIds.includes(value.serviceId)); + if ( + canonicalJson(unchanged) !== + canonicalJson( + owner.originalContainers.filter((value) => !recreateIds.includes(value.serviceId)), + ) + ) + throw Error("Unchanged original owned runtime differs"); + if (contained && canonicalJson(containers) !== canonicalJson(owner.originalContainers)) + throw Error("Original noncommitted runtime changed"); + const docker = await inspectMaintenanceDockerRuntime({ + stack: source.stack, + execFile: source.execFile, + owner, + containers, + privateKey: state.installation.privateKey, + signal, + }); + if (owner.plan.hardlinkPlan.plan.length) await assertConfigurationCoordinatorReady(signal); + const issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: issuedAt + 15000, + }, + ), + guard = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal, + }); + if ( + canonicalJson(guard) !== canonicalJson(owner.guardSource.preparedReceipt) || + canonicalJson(containers) !== canonicalJson(await inspectContainers()) || + canonicalJson(children) !== + canonicalJson( + readOriginalMaintenanceFinalChildren( + authority, + observeConfigurationMaintenanceRuntimeTree(authority), + ), + ) || + canonicalJson(owner) !== canonicalJson(readOwnedConfigurationMaintenance(authority)) + ) + throw Error("Original applied runtime source changed"); + signal.throwIfAborted(); + const identity = configurationSettlementOwnerIdentitySchema.parse({ + installationId: owner.installationId, + origin: owner.origin, + ownerPrepare: owner.plan.prepare, + admissionDigest: owner.admissionDigest, + rootCertificateDigest: owner.rootCertificateDigest, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + planDigest: owner.plan.planDigest, + settlementId: owner.plan.dmSettle.settlementId, + }), + requiredChecks = configurationMaintenanceRequiredChecksSchema.parse([ + ...["generation", "owner-containers", "dm-guard"].map((check) => ({ + serviceId: null, + ordinal: null, + check, + expectedRevisionId: contained + ? owner.plan.operation.expectedCurrentRevisionId + : owner.plan.operation.revisionId, + expectedImageId: null, + })), + ...owner.plan.steps.map((step) => ({ + serviceId: null, + ordinal: step.ordinal, + check: "frozen-attempts", + expectedRevisionId: owner.plan.operation.revisionId, + expectedImageId: null, + })), + ...docker.checks, + ]), + details = [ + { currentRevisionId: services.currentConfigurationGeneration(source.infraDir) }, + { containers }, + { guard }, + ...owner.plan.steps.map((step) => ({ + children: children.children.filter((value) => value.planIndex === step.ordinal), + })), + ...docker.details, + ], + inspectionId = randomUUID(), + signed = signControl( + state.installation.privateKey, + configurationSettlementDomains.inspection, + configurationMaintenanceRuntimeInspectionSchema.parse({ + version: 2, + identity, + inspectionId, + ownerSequence: owner.sequence, + ownerSourceDigest: controlDigest(owner), + requiredChecks, + observations: details.map((detail, checkIndex) => ({ + checkIndex, + passed: true, + detail: canonicalJson(detail), + })), + runtime: { + verificationId: inspectionId, + mode: contained ? "contained" : "applied", + requiredCheckSetDigest: settlementRecordDigest( + configurationSettlementDomains.inspection, + requiredChecks, + ), + checkedAt: Date.now(), + }, + }), + ); + if (Buffer.byteLength(canonicalJson(signed)) > 65536) + throw Error("Private maintenance inspection capacity"); + const freeze = (input: T): T => { + if (input && typeof input === "object") { + for (const child of Object.values(input)) freeze(child); + Object.freeze(input); + } + return input; + }, + value = Object.freeze({}) as MaintenanceRuntimeInspection; + runtimeInspections.set(value, Object.freeze({ authority, signed: freeze(signed) })); + return value; + } + if ( + !source?.candidateSource || + owner.final || + owner.inspection || + owner.noGuard || + owner.issuance !== "open" || + !owner.guardSource || + owner.receipt.state !== "authorized" || + owner.receipt.decision?.decision !== "abandon" || + !owner.apiControls.decide?.receiptDelivery || + owner.receipt.generation !== "not_started" || + owner.receipt.followUp !== "not_started" || + owner.apiControls.steps.length + ) + throw Error("Conclusive unused maintenance runtime source required"); + const identity = configurationSettlementOwnerIdentitySchema.parse({ + installationId: owner.installationId, + origin: owner.origin, + ownerPrepare: owner.plan.prepare, + admissionDigest: owner.admissionDigest, + rootCertificateDigest: owner.rootCertificateDigest, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + planDigest: owner.plan.planDigest, + settlementId: owner.plan.dmSettle.settlementId, + }), + queue = source.candidateSource; + let lease: ReturnType | undefined, + lock: TrustedConfigurationQueueLock | undefined; + try { + lock = await acquireTrustedConfigurationQueueLock({ + ...queue, + participant: "ops-agent", + operationKey: state.execution.operationKey, + }); + lease = restoreArtifactTargetLeaseForSettlement(authority); + const phaseObservations = []; + for (const step of owner.plan.steps) { + const publication = inspectControlChildPublication(authority, step); + if (publication.binding !== null || publication.publication.state !== "unissued") + throw Error("Unused runtime has attempted frozen work"); + phaseObservations.push({ + phase: step.phase, + ordinal: step.ordinal, + operationKey: step.operationKey, + binding: publication.binding, + publication: publication.publication.state, + }); + } + const prepared = verifyPublishedDataConfigurationFencePrepared( + lease, + owner.guardSource.prepareExecution, + ), + physical = inspectArtifactTargetLeaseChildSettlement( + lease, + owner.guardSource.prepareExecution, + ); + if (!prepared || physical?.outcome !== "committed" || physical.publication !== "committed") + throw Error("Original prepared guard settlement unavailable"); + const candidate = await readRetainedConfigurationCandidate( + queue, + owner.plan.operation.candidateId, + ); + if ( + controlDigest(candidate.record) !== owner.plan.candidateRecordDigest || + candidate.record.prepareOperationKey !== state.execution.operationKey || + candidate.snapshot.revisionId !== owner.plan.operation.revisionId + ) + throw Error("Original unused candidate changed"); + const directories = []; + for (const [name, path] of [ + ["queue", queue.directory], + ["claimed", join(queue.directory, ".claimed")], + ] as const) { + lock.assertHeld(); + const before = await lstat(path); + if ( + !before.isDirectory() || + before.isSymbolicLink() || + before.uid !== queue.ownerUid || + before.gid !== queue.ownerGid || + (before.mode & 0o777) !== 0o700 + ) + throw Error("Original unused queue unavailable"); + const directory = await opendir(path); + let count = 0; + // Dir's async iterator closes its descriptor on completion or an abrupt exit. + for await (const entry of directory) + if (++count > 1024 || entry.name.includes(candidate.snapshot.revisionId)) + throw Error("Unused candidate publication/claim uncertain"); + const after = await lstat(path); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.uid !== after.uid || + before.gid !== after.gid || + before.mode !== after.mode + ) + throw Error("Original unused queue changed"); + directories.push({ + name, + dev: before.dev, + ino: before.ino, + uid: before.uid, + gid: before.gid, + mode: before.mode & 0o777, + entries: count, + matchingCandidateEntries: 0, + }); + } + const current = services.currentConfigurationGeneration(source.infraDir), + containers = await inspectOriginalOwnerServices(source, authority, signal); + if ( + current !== owner.plan.operation.expectedCurrentRevisionId || + canonicalJson(containers) !== canonicalJson(owner.originalContainers) + ) + throw Error("Unused runtime differs from original owned generation/containers"); + const issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: issuedAt + 15000, + }, + ), + guard = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal, + }); + if ( + canonicalJson(guard) !== canonicalJson(owner.guardSource.preparedReceipt) || + canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) + ) + throw Error("Original unused guard/owner changed"); + signal.throwIfAborted(); + lock.assertHeld(); + const requiredChecks = configurationMaintenanceRequiredChecksSchema.parse([ + ...["generation", "owner-containers", "candidate-not-consumed", "dm-guard"].map( + (check) => ({ + serviceId: null, + ordinal: null, + check, + expectedRevisionId: owner.plan.operation.expectedCurrentRevisionId, + expectedImageId: null, + }), + ), + ...owner.plan.steps.map((step) => ({ + serviceId: null, + ordinal: step.ordinal, + check: "frozen-attempts", + expectedRevisionId: owner.plan.operation.revisionId, + expectedImageId: null, + })), + ]), + details = [ + { currentRevisionId: current }, + { containers }, + { + candidateId: candidate.record.candidateId, + revisionId: candidate.snapshot.revisionId, + recordDigest: controlDigest(candidate.record), + directories, + }, + { guard }, + ...phaseObservations, + ], + inspectionId = randomUUID(), + checkedAt = Date.now(), + payload = configurationMaintenanceRuntimeInspectionSchema.parse({ + version: 2, + identity, + inspectionId, + ownerSequence: owner.sequence, + ownerSourceDigest: controlDigest(owner), + requiredChecks, + observations: details.map((detail, checkIndex) => ({ + checkIndex, + passed: true, + detail: canonicalJson(detail), + })), + runtime: { + verificationId: inspectionId, + mode: "unused", + requiredCheckSetDigest: settlementRecordDigest( + configurationSettlementDomains.inspection, + requiredChecks, + ), + checkedAt, + }, + }), + signed = signControl( + state.installation.privateKey, + configurationSettlementDomains.inspection, + payload, + ); + if (Buffer.byteLength(canonicalJson(signed)) > 65536) + throw Error("Private maintenance inspection capacity"); + const value = Object.freeze({}) as MaintenanceRuntimeInspection; + const freezeProof = (input: T): T => { + if (input && typeof input === "object") { + for (const child of Object.values(input)) freezeProof(child); + Object.freeze(input); + } + return input; + }; + runtimeInspections.set(value, Object.freeze({ authority, signed: freezeProof(signed) })); + return value; + } finally { + lock?.release(); + if (lease) closeArtifactTargetLeaseHandle(lease); + } +} +declare const unconsumedBrand: unique symbol; +export type UnconsumedMaintenanceCandidate = Readonly<{ [unconsumedBrand]: true }>; +const unconsumed = new WeakMap< + UnconsumedMaintenanceCandidate, + { + authority: VerifiedExecutionAuthority; + lock: TrustedConfigurationQueueLock; + ownerPayloadDigest: string; + candidateRecordDigest: string; + } +>(); +export function unconsumedMaintenanceCandidateState( + value: UnconsumedMaintenanceCandidate, + authority: VerifiedExecutionAuthority, +) { + const state = unconsumed.get(value); + if (!state || state.authority !== authority) + throw Error("Live fixed unconsumed candidate observation required"); + executionAuthorityState(authority); + state.lock.assertHeld(); + return state; +} +/** The callback is fixed internal synchronous publication work. Its proof is + * invalid outside the existing real queue ownership, including after expiry. + * Historical seal authentication here never opens a forward/apply payload. */ +export async function withUnconsumedMaintenanceCandidate( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + publish: (witness: UnconsumedMaintenanceCandidate) => void, +): Promise { + const source = sources.get(runtime), + owner = readOwnedConfigurationMaintenance(authority), + state = executionAuthorityState(authority); + if ( + !source?.candidateSource || + state.kind !== "configurationMaintenance.prepare" || + owner.guardSource || + owner.receipt.dmFence || + owner.receipt.decision?.decision === "continue" || + owner.receipt.generation !== "not_started" + ) + throw Error("Original never-applied candidate owner required"); + const queue = source.candidateSource, + lock = await acquireTrustedConfigurationQueueLock({ + ...queue, + participant: "ops-agent", + operationKey: state.execution.operationKey, + }); + let witness: UnconsumedMaintenanceCandidate | undefined; + try { + const candidate = await readRetainedConfigurationCandidate( + queue, + owner.plan.operation.candidateId, + ), + candidateRecordDigest = controlDigest(candidate.record); + if ( + candidateRecordDigest !== owner.plan.candidateRecordDigest || + candidate.record.prepareOperationKey !== state.execution.operationKey || + candidate.record.maintenanceId !== owner.plan.operation.maintenanceId || + candidate.record.operationKey !== owner.plan.operation.applyOperationKey || + candidate.snapshot.revisionId !== owner.plan.operation.revisionId || + candidate.snapshot.operationFingerprint !== owner.plan.operation.applyOperationFingerprint || + candidate.record.apiJobId !== owner.receipt.apiJobId || + candidate.record.parentOperationKey !== owner.receipt.parentOperationKey || + candidate.record.parentFingerprint !== owner.receipt.parentFingerprint || + candidate.record.parentDeadlineAt !== owner.receipt.parentDeadlineAt + ) + throw Error("Original retained maintenance candidate changed"); + for (const path of [queue.directory, join(queue.directory, ".claimed")]) { + lock.assertHeld(); + const before = await lstat(path); + if ( + !before.isDirectory() || + before.isSymbolicLink() || + before.uid !== queue.ownerUid || + before.gid !== queue.ownerGid || + (before.mode & 0o777) !== 0o700 + ) + throw Error("Original candidate queue unavailable"); + const dir = await opendir(path); + let entries = 0; + // Dir's async iterator owns close on success and on thrown refusal. + for await (const entry of dir) { + if (++entries > 1024 || entry.name.includes(candidate.snapshot.revisionId)) + throw Error("Candidate publication/claim requires retained observation"); + } + const after = await lstat(path); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.uid !== after.uid || + before.gid !== after.gid || + before.mode !== after.mode + ) + throw Error("Original candidate queue changed during observation"); + } + lock.assertHeld(); + if (canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner)) + throw Error("Original owner changed during candidate observation"); + witness = Object.freeze({}) as UnconsumedMaintenanceCandidate; + unconsumed.set( + witness, + Object.freeze({ + authority, + lock, + ownerPayloadDigest: controlDigest(owner), + candidateRecordDigest, + }), + ); + publish(witness); + lock.assertHeld(); + } finally { + if (witness) unconsumed.delete(witness); + lock.release(); + } +} +/** Only the actual production factory supplies this source; no handler/result + * predicate or inspector can be registered by a request. */ +export function registerConfigurationMaintenanceSource(runtime: OpsRuntime, source: Source): void { + if (sources.has(runtime)) throw Error("Configuration maintenance source already installed"); + sources.set(runtime, source); + runtime["configurationMaintenance.decide"] = async (operation, context) => { + // Acceptance is the fixed original-owner metadata CAS during authenticated + // admission. This worker only observes its exact immutable result. + return inspectConfigurationMaintenanceDecisionControl( + context.configurationMaintenanceControl!, + operation, + context.execution, + ).receipt; + }; + runtime["configurationMaintenance.settle"] = async (operation, context) => { + const control = configurationMaintenanceControlOwnerState( + context.configurationMaintenanceControl, + operation, + context.execution.operationKey, + ), + authority = control.authority, + owner = readOwnedConfigurationMaintenance(authority); + if (owner.apiControls.settle?.result) + return inspectConfigurationMaintenanceSettleControl( + context.configurationMaintenanceControl!, + operation, + context.execution, + ).receipt; + if (!owner.final) { + const inspection = await inspectFrozenMaintenanceRuntime(runtime, authority, context.signal); + finalizeOwnedMaintenanceState(authority, inspection); + } else restoreOwnedMaintenanceFinal(authority); + publishOwnedConfigurationFenceSettlement(authority, operation.maintenanceId); + const settlement = prepareConfigurationFenceSettlementControl( + authority, + operation.maintenanceId, + ), + delegated = delegateConfigurationFenceSettlementControl(settlement); + if (delegated.dispatch === "fresh") { + try { + await dispatchConfigurationFenceSettlementControl( + delegated.ticket, + configurationDataManagerTransport(), + ); + } catch { + /* Lookup of that original immutable tuple only; never retry POST. */ + } + } + const state = executionAuthorityState(authority), + issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: issuedAt + 15000, + }, + ); + // A lost COMMIT acknowledgement may precede DM's shared terminal write. + // Its authenticated GET repairs publication only from the original atomic + // SQL tuple before the sender tries to copy that terminal. No POST/replay. + await inspectDataConfigurationFence(operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal: context.signal, + }); + observeConfigurationFenceSettlementControl(settlement); + // This second read lets DM retain the sender's now-durable copy ACK. + await inspectDataConfigurationFence(operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal: context.signal, + }); + await releaseOwnedConfigurationMaintenanceRoot(authority); + return completeConfigurationMaintenanceSettleControl( + context.configurationMaintenanceControl!, + operation, + context.execution, + ); + }; + runtime["configurationMaintenance.inspect"] = async (operation, context) => { + const control = configurationMaintenanceControlOwnerState( + context.configurationMaintenanceControl, + operation, + context.execution.operationKey, + ), + authority = control.authority, + state = executionAuthorityState(authority); + let owner = readOwnedConfigurationMaintenance(authority); + context.signal.throwIfAborted(); + if (owner.final && owner.apiControls.settle?.result === null) { + try { + if (owner.receipt.state !== "settled") + await observeCommittedMaintenanceSettlement(authority, context.signal); + completeObservedConfigurationMaintenanceSettlement(authority); + } catch { + /* Missing/unknown source remains fenced; no first-use ticket or retry. */ + } + owner = readOwnedConfigurationMaintenance(authority); + } + if (owner.issuance === "closed_without_guard") { + assertConfigurationMaintenanceRootClosure(authority); + verifyRetainedArtifactTargetRootRelease(authority); + } + if (owner.receipt.state === "settled" && owner.guardSource) { + restoreOwnedMaintenanceFinal(authority); + assertConfigurationMaintenanceRootClosure(authority); + verifyRetainedArtifactTargetRootRelease(authority); + } + let preflight: "current" | "drifted" | "unavailable" = "unavailable"; + try { + const fresh = configurationMaintenanceAdmissionState( + await prepareConfigurationMaintenanceAdmission(runtime, { + installation: state.installation.installation, + execution: owner.plan.prepare, + operation: owner.plan.operation, + apiParent: owner.plan.apiParent, + signal: context.signal, + }), + ); + preflight = + canonicalJson(fresh.plan) === canonicalJson(owner.plan) && + canonicalJson(fresh.originalContainers) === canonicalJson(owner.originalContainers) + ? "current" + : "drifted"; + if (owner.receipt.dmFence) { + if (!owner.guardSource) throw Error("Original prepared guard source unavailable"); + const issuedAt = Date.now(), + witness = signControl( + state.installation.privateKey, + "openmapx-configuration-fence-inspect-v1", + { + installationId: owner.installationId, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + ownerCertificateDigest: owner.rootCertificateDigest, + issuedAt, + expiresAt: Math.min(context.execution.deadlineAt, issuedAt + 15000), + }, + ), + actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + witness: Buffer.from(canonicalJson(witness)).toString("base64url"), + signal: context.signal, + }); + if (canonicalJson(actual) !== canonicalJson(owner.receipt.dmFence)) + throw Error("Original prepared guard source changed"); + } + } catch (error) { + preflight = "unavailable"; + if ( + error instanceof Error && + /Host sources changed|preflight drifted|Settled ready coordinator/.test(error.message) + ) + preflight = "drifted"; + } + context.signal.throwIfAborted(); + const current = readOwnedConfigurationMaintenance(authority); + if (canonicalJson(current) !== canonicalJson(owner)) preflight = "unavailable"; + return { ...current.receipt, preflight }; + }; + runtime["configurationMaintenance.prepare"] = async (operation, context) => { + const authority = context.controlAuthority, + lease = context.ownedTargetLease; + if (!authority || !lease) throw Error("Actual original maintenance owner required"); + const state = executionAuthorityState(authority), + owner = readOwnedConfigurationMaintenance(authority), + plan = owner.plan; + try { + if ( + owner.receipt.state !== "preparing" || + canonicalJson(operation) !== canonicalJson(plan.operation) + ) + throw Error("Original preparing maintenance identity required"); + // Repeat source/TTL/current/ready/registry checks after acquiring the full + // original target closure, before any grant/DM logical reservation. + const current = await prepareConfigurationMaintenanceAdmission(runtime, { + installation: state.installation.installation, + execution: state.execution, + operation, + apiParent: plan.apiParent, + signal: context.signal, + }); + if ( + canonicalJson(configurationMaintenanceAdmissionState(current).plan) !== canonicalJson(plan) + ) + throw Error("Frozen maintenance source changed under ownership"); + const reserved = plan.reservations.find((value) => value.phase === "configuration-fence"); + if (!reserved) throw Error("Original DM preparation reservation required"); + const previous = lookupControlChildPhaseBinding(authority, reserved), + child = await bindControlChildPhase(authority, reserved); + if (!previous) { + const issued = await issueControlChildPhase(authority, { + phase: reserved.phase, + ordinal: reserved.ordinal, + lease, + childPublicKey: controlRolePublicKey(state.installation.installation, "data-manager"), + }); + consumeControlChildDispatch(issued.dispatchTicket); + try { + await prepareDataConfigurationFence(plan.dmPrepare, { + transport: configurationDataManagerTransport(), + controlAssertion: issued.assertion, + signal: context.signal, + }); + } catch { + /* Attempt already retained. Only exact original source observation follows. */ + } + } + try { + const prepared = verifyPublishedDataConfigurationFencePrepared(lease, child.execution), + capacity = verifyPublishedConfigurationSettlementCapacity( + authority, + operation.maintenanceId, + ); + recordOwnedConfigurationMaintenancePrepared( + authority, + lease, + owner.sequence, + prepared, + capacity, + ); + } catch { + throw new OpsExecutionFailure("recovery_required", {}); + } + return readOwnedConfigurationMaintenance(authority).receipt; + } catch (error) { + try { + await closeOwnedConfigurationMaintenanceWithoutGuard(runtime, authority); + } catch { + /* Ambiguous source/guard/session/publication retains this root. */ + } + throw error; + } + }; +} +export function configurationMaintenanceAdmissionState( + value: ConfigurationMaintenanceAdmission, +): AdmissionState { + const state = admissions.get(value); + if (!state) throw Error("Actual configuration maintenance admission required"); + return state; +} +export async function prepareConfigurationMaintenanceAdmission( + runtime: OpsRuntime, + input: { + installation: ControlInstallation; + execution: ExecutionRef; + operation: ConfigurationMaintenancePrepare; + apiParent: RetainedParentExecution; + signal: AbortSignal; + }, +): Promise { + const source = sources.get(runtime); + if ( + !source?.candidateSource || + input.installation.role !== "ops-agent" || + input.apiParent.issuer !== "api" + ) + throw Error("Original API configuration maintenance source required"); + input.signal.throwIfAborted(); + const candidate = await readConfigurationCandidate( + source.candidateSource, + input.operation.candidateId, + ), + original = candidate.operation; + if ( + candidate.record.apiJobId !== input.apiParent.apiJobId || + candidate.record.parentOperationKey !== input.apiParent.operationKey || + candidate.record.parentFingerprint !== input.apiParent.fingerprint || + candidate.record.parentDeadlineAt !== input.apiParent.deadlineAt || + candidate.record.maintenanceId !== input.operation.maintenanceId || + candidate.record.prepareOperationKey !== input.execution.operationKey || + candidate.record.operationKey !== input.operation.applyOperationKey || + opsOperationFingerprint(original) !== input.operation.applyOperationFingerprint || + !("revisionId" in original) || + !("expectedCurrentRevisionId" in original) || + !("expectedSourceRevision" in original) || + !("coordinatorMode" in original) || + original.revisionId !== input.operation.revisionId || + original.expectedCurrentRevisionId !== input.operation.expectedCurrentRevisionId || + original.expectedSourceRevision !== input.operation.expectedSourceRevision || + original.coordinatorMode !== input.operation.coordinatorMode || + ![ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ].includes(original.kind) + ) + throw Error("Original candidate/parent maintenance binding conflict"); + const recreateServiceIds = + candidate.record.apiFollowUp === "service-apply" || candidate.record.privateCoordinatorLifecycle + ? candidate.record.affectedServiceIds + : []; + if ( + input.operation.followUp === "platform-apply" || + (recreateServiceIds.length ? "service-apply" : "render-only") !== input.operation.followUp + ) + throw Error("Configuration maintenance continuation unavailable"); + const authority = await source.loadAuthority(); + const coordinator = candidate.record.privateCoordinatorLifecycle + ? renderCoordinatorCandidate(candidate, authority, source.infraDir) + : undefined; + if (coordinator) + verifyConfigurationReleaseSourceWitness({ + installation: input.installation, + infraDir: source.infraDir, + sourceRevision: input.operation.expectedSourceRevision, + expectedCurrentRevisionId: input.operation.expectedCurrentRevisionId, + authorityRevision: authority.revisionId, + }); + verifyConfigurationSourceWitness({ + installation: input.installation, + infraDir: source.infraDir, + sourceRevision: input.operation.expectedSourceRevision, + expectedCurrentRevisionId: input.operation.expectedCurrentRevisionId, + authorityRevision: authority.revisionId, + }); + const desired = services.readDesiredSelection(source.infraDir); + const rendered = services.renderConfiguration({ + infraDir: source.infraDir, + services: authority.services, + integrationSchemas: authority.integrationSchemas, + input: { + ...candidate.payload, + selectedRoots: candidate.payload.selectedRoots ?? desired.roots, + }, + allowMissingSelected: !candidate.payload.selectedRoots && desired.source === "default", + }); + const prospective = new Set(rendered.resolved.enabled.map((service) => service.manifest.id)); + if (!prospective.has("data-manager") || recreateServiceIds.some((id) => !prospective.has(id))) + throw Error("Configuration maintenance coordinator/target prerequisite unavailable"); + const current = services.readAppliedServiceIds(source.infraDir, authority.services); + if (!current) throw Error("Current configuration generation required for maintenance"); + const serviceIds = [ + ...new Set([ + ...current, + ...prospective, + ...candidate.record.affectedServiceIds, + "data-manager", + ]), + ].sort(); + const plan = rendered.rendered.hardlinkPlan; + if (plan.length && input.operation.coordinatorMode === "ensure-running" && !coordinator) { + if ( + !input.operation.expectedCurrentRevisionId || + candidate.record.coordinatorPrerequisite?.operation.expectedConfigurationRevisionId !== + input.operation.expectedCurrentRevisionId + ) + throw Error("Original settled old-revision coordinator prerequisite required"); + verifyMaintenanceCoordinatorPrerequisite(input.installation, candidate.record); + } + let coordinatorPull: CoordinatorPullWitness | undefined, + coordinatorBaseline: VerifiedCoordinatorPullBaseline | undefined, + coordinatorImage: FixedConfigurationMaintenancePlan["coordinatorImage"]; + if (coordinator) { + const fixed = coordinatorCandidateMetadata(candidate.record); + // A used helper remains an exact settled original. An already-ready old + // coordinator needs no Compose up and is observed again above. + const helper = candidate.record.coordinatorPrerequisite; + if (helper) { + let present = false; + try { + readPrivateRecord( + installationState(input.installation), + "executions", + `${helper.root.operationKey}.json`, + 65536, + ); + present = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (present) verifyMaintenanceCoordinatorPrerequisite(input.installation, candidate.record); + } + if (fixed.action === "update") { + let retained: ConfigurationMaintenanceOwnerRecord | undefined; + try { + const originalOwner = restoreAdmittedControlExecution(input.installation, input.execution); + retained = readOwnedConfigurationMaintenance(originalOwner); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (retained) { + coordinatorPull = retained.coordinatorPull; + coordinatorBaseline = ownedCoordinatorPullBaseline( + restoreAdmittedControlExecution(input.installation, input.execution), + ); + } else { + const matches = journals + .get(runtime) + ?.records() + .filter((job) => job.role === "api" && job.operationKey === fixed.pull.operationKey); + if (!matches || matches.length !== 1) + throw Error("Unique original coordinator pull source unavailable"); + const job = matches[0]!; + if ( + job.executionVersion !== 2 || + job.operation.kind !== "service.pull" || + job.state !== "succeeded" || + job.requestVersion !== 2 || + canonicalJson(job.operation) !== canonicalJson(fixed.pull.operation) + ) + throw Error("Actual successful current coordinator pull required"); + const status = parseOpsJobStatus({ + version: 2, + admittedAt: job.admittedAt, + deadlineAt: job.deadlineAt, + fingerprint: job.fingerprint, + operationId: job.operationId, + operationKey: job.operationKey, + kind: job.operation.kind, + resourceId: job.resourceId, + state: job.state, + submittedAt: job.submittedAt, + updatedAt: job.updatedAt, + result: job.result, + ...(job.terminationRequestedAt + ? { terminationRequestedAt: job.terminationRequestedAt } + : {}), + }); + coordinatorPull = observeCoordinatorPullForMaintenance( + input.installation, + candidate.record, + input.execution, + status, + ); + coordinatorBaseline = observedCoordinatorPullBaseline(coordinatorPull); + } + const proof = + coordinatorPull && + verifyControl( + input.installation.publicKey, + coordinatorPullObservationDomain, + coordinatorPull, + coordinatorPullWitnessPayloadSchema, + ); + if ( + !proof || + proof.imageReference !== coordinator.imageReference || + proof.releaseSelectionDigest !== coordinator.releaseSelectionDigest || + proof.submissionDigest !== coordinatorPullSubmissionDigest(fixed.pull) + ) + throw Error("Original pulled coordinator image source changed"); + coordinatorImage = { + action: "update", + noDeps: fixed.noDeps, + imageReference: proof.imageReference, + imageId: proof.imageId, + releaseSelectionDigest: proof.releaseSelectionDigest, + pullSubmissionDigest: proof.submissionDigest, + }; + } else { + const observed = await source.execFile( + "docker", + ["image", "inspect", "--format", "{{.Id}}", coordinator.imageReference], + { signal: input.signal, timeout: 15000, maxBuffer: 4096 }, + ); + coordinatorImage = { + action: "apply", + noDeps: false, + imageReference: coordinator.imageReference, + imageId: observed.stdout.trim(), + releaseSelectionDigest: coordinator.releaseSelectionDigest, + pullSubmissionDigest: null, + }; + } + } + if (!coordinator && recreateServiceIds.includes("data-manager")) { + verifyConfigurationReleaseSourceWitness({ + installation: input.installation, + infraDir: source.infraDir, + sourceRevision: input.operation.expectedSourceRevision, + expectedCurrentRevisionId: input.operation.expectedCurrentRevisionId, + authorityRevision: authority.revisionId, + }); + const selected = services.readServiceReleaseImageSelection(source.infraDir, "data-manager"), + imageReference = + selected.imageReference ?? + services.configurationImageReference(rendered.rendered.composeYaml, "data-manager"); + const actual = await source.execFile( + "docker", + ["image", "inspect", "--format", "{{.Id}}", imageReference], + { signal: input.signal, timeout: 15000, maxBuffer: 4096 }, + ); + coordinatorImage = { + action: "apply", + noDeps: false, + imageReference, + imageId: actual.stdout.trim(), + releaseSelectionDigest: selected.releaseSelectionDigest, + pullSubmissionDigest: null, + }; + } + const staged = + !coordinatorBaseline && recreateServiceIds.includes("data-manager") + ? await inspectStagedCoordinatorMaintenanceSource(source.ownedServices, input.signal) + : undefined; + const originalContainers = coordinatorBaseline + ? await inspectOriginalCoordinatorServices( + source.ownedServices, + coordinatorBaseline, + input.signal, + ) + : (staged?.services ?? (await source.ownedServices.inspectOwnedServices(input.signal))); + if ( + originalContainers.find((service) => service.serviceId === "data-manager")?.state !== "running" + ) + throw Error("Settled ready coordinator prerequisite required before maintenance"); + await assertConfigurationCoordinatorReady(input.signal); + const previewDigest = createHash("sha256") + .update("openmapx-configuration-hardlinks-preview-v1\0") + .update( + canonicalJson({ + candidateId: input.operation.candidateId, + revisionId: input.operation.revisionId, + expectedSourceRevision: input.operation.expectedSourceRevision, + coordinatorMode: input.operation.coordinatorMode, + plan, + }), + ) + .digest("hex"); + const frozen = buildFixedConfigurationMaintenancePlan({ + installationId: input.installation.installationId, + prepare: input.execution, + operation: input.operation, + apiParent: input.apiParent, + apply: original as Parameters[0]["apply"], + candidateRecordDigest: createHash("sha256") + .update(canonicalJson(candidate.record)) + .digest("hex"), + sourceAuthorityRevision: authority.revisionId, + previewDigest, + hardlinkPlan: { plan, prune: true }, + serviceIds, + recreateServiceIds, + ...(coordinatorImage ? { coordinatorImage } : {}), + }); + assertProspectiveConfigurationMaintenanceBudget({ + installationId: input.installation.installationId, + plan: frozen, + originalContainers: structuredClone(originalContainers), + rendered, + }); + assertProspectiveConfigurationSettlementPacketBudget(frozen, input.installation.installationId); + if ( + (await source.loadAuthority()).revisionId !== authority.revisionId || + services.currentConfigurationGeneration(source.infraDir) !== + input.operation.expectedCurrentRevisionId + ) + throw Error("Configuration maintenance preflight drifted"); + const stagedSource = staged?.proof && verifiedStagedCoordinatorOwnershipState(staged.proof); + if ( + stagedSource && + (coordinatorPull || + stagedSource.installationId !== input.installation.installationId || + stagedSource.source.baseline.configurationRevisionId !== + input.operation.expectedCurrentRevisionId) + ) + throw Error("Original staged maintenance source changed"); + const stagedCoordinatorProvenance = stagedSource + ? stagedCoordinatorObservationSchema.parse( + signControl( + installationState(input.installation).privateKey, + stagedCoordinatorObservationDomain, + { + version: 1, + installationId: input.installation.installationId, + ownerPrepare: input.execution, + candidateId: input.operation.candidateId, + planDigest: frozen.planDigest, + source: stagedSource.source, + capturedAt: Date.now(), + }, + ), + ) + : undefined; + const value = Object.freeze({}) as ConfigurationMaintenanceAdmission; + const freeze = (value: T): T => { + if (value && typeof value === "object") { + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); + } + return value; + }; + admissions.set( + value, + freeze({ + plan: frozen, + originalContainers: structuredClone(originalContainers), + ...(coordinatorPull ? { coordinatorPull } : {}), + ...(stagedCoordinatorProvenance ? { stagedCoordinatorProvenance } : {}), + }), + ); + return value; +} + +async function inspectOriginalOwnerServices( + source: NonNullable>, + authority: VerifiedExecutionAuthority, + signal: AbortSignal, +) { + const owner = readOwnedConfigurationMaintenance(authority); + return owner.plan.coordinatorImage?.action === "update" + ? inspectOriginalCoordinatorServices( + source.ownedServices, + ownedCoordinatorPullBaseline(authority), + signal, + ) + : owner.stagedCoordinatorProvenance + ? inspectStagedCoordinatorOwnerServices( + source.ownedServices, + ownedStagedCoordinatorBaseline(authority), + signal, + ) + : source.ownedServices.inspectOwnedServices(signal); +} diff --git a/apps/ops-agent/src/configuration-writer-concurrency.test.ts b/apps/ops-agent/src/configuration-writer-concurrency.test.ts index 41d032a73..fac3a4771 100644 --- a/apps/ops-agent/src/configuration-writer-concurrency.test.ts +++ b/apps/ops-agent/src/configuration-writer-concurrency.test.ts @@ -157,6 +157,10 @@ describe("overlapping configuration writers", () => { await coreServices.commitConfigurationGeneration({ infraDir: infra, services: registry.list(), + // This pruning fixture has no containers. Supply the complete empty + // inventory; omitted evidence correctly preserves conservative bases. + ownedServiceGenerations: [], + observedServiceContainerIds: [], input: { domain: "fixture.example.test", selectedRoots: ["alpha"], diff --git a/apps/ops-agent/src/configured-resource-authority.ts b/apps/ops-agent/src/configured-resource-authority.ts new file mode 100644 index 000000000..1a6d2cc1e --- /dev/null +++ b/apps/ops-agent/src/configured-resource-authority.ts @@ -0,0 +1,57 @@ +/** Same actual resource snapshot used by production registry policy. */ +import { createHash } from "node:crypto"; + +const REGION_ID = /^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$/; +const COUNTRY_ID = /^[A-Z]{2,3}$/; + +export interface ConfiguredResourceAuthority { + revisionId: string; + regions: ReadonlySet; + countries: ReadonlySet; +} + +function exactConfiguredIds( + raw: string | undefined, + pattern: RegExp, + maximumLength: number, +): string[] { + const values = (raw ?? "") + .split(",") + .map((value) => value.trim()) + .filter(Boolean); + if ( + values.length > 64 || + values.some((value) => value.length > maximumLength || !pattern.test(value)) || + new Set(values).size !== values.length + ) { + throw new Error("Resource authority configuration rejected"); + } + return values.sort(); +} + +export function loadConfiguredResourceAuthority( + env: NodeJS.ProcessEnv = process.env, +): ConfiguredResourceAuthority { + const regions = exactConfiguredIds( + env.OPENMAPX_ALLOWED_REGIONS ?? "europe/germany", + REGION_ID, + 128, + ); + const countries = exactConfiguredIds(env.OPENMAPX_ALLOWED_COUNTRIES ?? "DE", COUNTRY_ID, 3); + const digest = createHash("sha256") + .update("openmapx-resource-authority-v1\0") + .update(JSON.stringify({ regions, countries })) + .digest("hex"); + return { + revisionId: `resources-${digest}`, + regions: new Set(regions), + countries: new Set(countries), + }; +} + +export function configurationPolicyRevision( + configurationAuthorityRevision: string, + env: NodeJS.ProcessEnv = process.env, +): string { + return `${configurationAuthorityRevision.slice(0, 48)}-${loadConfiguredResourceAuthority(env).revisionId}`; +} diff --git a/apps/ops-agent/src/coordinator-staged-ownership.ts b/apps/ops-agent/src/coordinator-staged-ownership.ts new file mode 100644 index 000000000..eeba21cc5 --- /dev/null +++ b/apps/ops-agent/src/coordinator-staged-ownership.ts @@ -0,0 +1,80 @@ +/** Fixed production installation; journal records are locators, never authority. */ +import { + type ControlInstallation, + type ControlRetentionPreparationContext, + controlRetentionPreparationState, + recordStagedCoordinatorRetentionCheck, +} from "@openmapx/core/build-artifacts"; +import { installationState } from "../../../packages/core/src/build-artifacts/control-authority"; +import { signControl } from "../../../packages/core/src/build-artifacts/control-records"; +import { + stagedCoordinatorRetirementDomain, + stagedCoordinatorRetirementPacketSchema, +} from "../../../packages/core/src/build-artifacts/coordinator-pull-records"; +import type { OpsJobJournal } from "./journal"; +import { + bindCoordinatorStagedOwnershipSource, + readLiteralCoordinatorStagedRetention, +} from "./owned-services"; + +export { assertStagedCoordinatorConfigurationWrite } from "./configuration-maintenance-store"; + +import { productionOwnedServices } from "./production-runtime"; +import type { OpsRuntime } from "./runtime"; + +export function installCoordinatorStagedOwnershipSource( + runtime: OpsRuntime, + installation: ControlInstallation, + journal: OpsJobJournal, +): void { + bindCoordinatorStagedOwnershipSource(productionOwnedServices(runtime), installation, journal); +} + +/** Called only by the existing serialized role collector/journal hook. */ +export async function prepareCoordinatorStagedRetention( + context: ControlRetentionPreparationContext, + runtime: OpsRuntime, + installation: ControlInstallation, + journal: OpsJobJournal, +): Promise { + const state = controlRetentionPreparationState(context), + resolver = productionOwnedServices(runtime), + actual = await readLiteralCoordinatorStagedRetention(resolver, installation, journal, context); + if (!actual) return; + const source = actual.source, + successor = actual.successor; + const packet = stagedCoordinatorRetirementPacketSchema.parse( + signControl(installationState(installation).privateKey, stagedCoordinatorRetirementDomain, { + version: 1, + installationId: installation.installationId, + originDigest: state.originDigest, + familyId: state.familyId, + generation: state.generation, + execution: source.execution, + terminalDigest: source.canonicalDigest, + sourceAdmissionDigest: source.admissionDigest, + oldContainerDigest: actual.oldContainerDigest, + rank: { admittedAt: source.admittedAt, operationKey: source.execution.operationKey }, + contextMode: state.mode, + checkNonce: state.checkNonce, + manifestDigest: state.manifestDigest, + checkedAt: Date.now(), + condition: actual.condition, + successor: successor + ? { + execution: successor.execution, + admissionDigest: successor.admissionDigest, + canonicalDigest: successor.canonicalDigest, + effectDigest: successor.effectDigest, + apiCopyDigest: successor.apiCopyDigest, + rank: { + admittedAt: successor.admittedAt, + operationKey: successor.execution.operationKey, + }, + } + : null, + }), + ); + // No checksum or public success boolean substitutes for this actual producer. + recordStagedCoordinatorRetentionCheck(context, packet); +} diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index 689761d6f..be5c6f6e8 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -11,7 +11,8 @@ import { import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; import type { OpsOperation } from "@openmapx/core/ops"; -import { describe, expect, it, vi } from "vitest"; +import { services } from "@openmapx/core/server"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { testExecution } from "./__fixtures__/execution"; import { createDockerRuntime, @@ -21,6 +22,48 @@ import { } from "./docker-runtime"; import { dispatchOpsOperation } from "./runtime"; +const preparedRoots: string[] = []; +afterEach(() => { + for (const root of preparedRoots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +async function preparedStack() { + const root = mkdtempSync(join(tmpdir(), "omx-docker-prepared-")), + infraDir = join(root, "infra/docker"); + preparedRoots.push(root); + mkdirSync(infraDir, { recursive: true }); + const loaded: services.LoadedService[] = ["motis", "postgis", "redis"].map((id) => ({ + manifest: { + id, + name: id, + version: "1.0.0", + quality: "built-in", + container: { image: `example/${id}`, tag: "1" }, + }, + directory: join(root, "services", id), + enabled: true, + isBuiltIn: true, + })); + const generation = await services.commitConfigurationGeneration({ + infraDir, + services: loaded, + input: { + domain: "fixture.invalid", + selectedRoots: loaded.map((s) => s.manifest.id), + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + }); + return { + revisionId: generation.revisionId, + stack: { + infraDir, + composePath: services.currentConfigurationFile(infraDir, services.GENERATED_COMPOSE_FILE), + composeReleasePath: join(infraDir, "docker-compose.release.yml"), + }, + }; +} + const ownedSnapshot = (serviceId = "redis") => ({ serviceId, containerId: "a".repeat(64), @@ -819,23 +862,45 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done ); it.each([ - { kind: "stack.start" }, - { kind: "service.start", serviceId: "postgis" }, + { kind: "stack.start", expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}` }, + { + kind: "service.start", + serviceId: "postgis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, { kind: "service.stop", serviceId: "motis" }, { kind: "service.restart", serviceId: "postgis" }, - { kind: "service.recreate", serviceId: "postgis" }, - { kind: "service.recreateIsolated", serviceId: "postgis" }, + { + kind: "service.recreate", + serviceId: "postgis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, + { + kind: "service.recreateIsolated", + serviceId: "postgis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, { kind: "service.remove", serviceId: "postgis" }, ] satisfies OpsOperation[])("allows the built-in shutdown grace for $kind", async (operation) => { + const prepared = await preparedStack(); + const bound: OpsOperation = + operation.kind === "stack.start" + ? { kind: operation.kind, expectedConfigurationRevisionId: prepared.revisionId } + : operation.kind === "service.start" || + operation.kind === "service.recreate" || + operation.kind === "service.recreateIsolated" + ? { + kind: operation.kind, + serviceId: operation.serviceId, + expectedConfigurationRevisionId: prepared.revisionId, + } + : operation; const runtime = createDockerRuntime({ - stack: { - infraDir: "/trusted", - composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", - }, + stack: prepared.stack, fileExists: () => false, ownedServices, - execFile: async (_file, _args, options) => { + execFile: async (file, args, options) => { + if (args.includes("config")) return runContainedProcess(file, args, options); expect(options.timeout).toBe(600_000); expect(options.signal.aborted).toBe(false); return { stdout: "", stderr: "" }; @@ -844,7 +909,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done await expect( dispatchOpsOperation( runtime, - operation, + bound, context("serviceId" in operation ? operation.serviceId : "redis"), ), ).resolves.toEqual({ @@ -856,7 +921,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done [{ kind: "docker.status" }, 5_000], [{ kind: "stack.status" }, 120_000], [{ kind: "service.logs", serviceId: "postgis", tail: 10 }, 120_000], - [{ kind: "service.pull", serviceId: "postgis" }, 600_000], + [{ kind: "service.restart", serviceId: "postgis" }, 600_000], ] satisfies [OpsOperation, number][])( "preserves the execution budget for %j", async (operation, timeout) => { @@ -882,19 +947,19 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done ); it("terminates aggregate recreation work exactly at the whole-command deadline", async () => { + const prepared = await preparedStack(), + entered = Promise.withResolvers(); vi.useFakeTimers(); const completed: number[] = []; let terminated = false; const runtime = createDockerRuntime({ - stack: { - infraDir: "/trusted", - composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", - }, + stack: prepared.stack, fileExists: () => false, ownedServices, - execFile: (_file, _args, options) => - new Promise((resolve, reject) => { + execFile: async (file, args, options) => { + if (args.includes("config")) return runContainedProcess(file, args, options); + return new Promise((resolve, reject) => { + entered.resolve(); const deadline = setTimeout(() => { terminated = true; for (const timer of steps) clearTimeout(timer); @@ -909,15 +974,21 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done } }, delay), ); - }), + }); + }, }); try { const result = dispatchOpsOperation( runtime, - { kind: "service.recreate", serviceId: "postgis" }, + { + kind: "service.recreate", + serviceId: "postgis", + expectedConfigurationRevisionId: prepared.revisionId, + }, context(), ); const rejected = expect(result).rejects.toThrow("Contained process failed"); + await entered.promise; await vi.advanceTimersByTimeAsync(599_999); expect(completed).toEqual([300_000]); expect(terminated).toBe(false); @@ -932,43 +1003,46 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done }); it("maps lifecycle effects to fixed Compose argv without accepting caller argv or paths", async () => { + const prepared = await preparedStack(); + writeFileSync(prepared.stack.composeReleasePath, "services: {}\n"); + writeFileSync(join(prepared.stack.infraDir, ".env"), ""); const calls: Array<{ file: string; args: readonly string[] }> = []; const runtime = createDockerRuntime({ - stack: { - infraDir: "/stack", - composePath: "/stack/.trusted-config-current/docker-compose.generated.yml", - composeReleasePath: "/stack/docker-compose.release.yml", - }, + stack: prepared.stack, fileExists: () => true, - execFile: async (file, args) => { + execFile: async (file, args, options) => { calls.push({ file, args }); + if (args.includes("config")) return runContainedProcess(file, args, options); return { stdout: "ok\n", stderr: "" }; }, }); await dispatchOpsOperation( runtime, - { kind: "service.recreate", serviceId: "motis" }, + { + kind: "service.recreate", + serviceId: "motis", + expectedConfigurationRevisionId: prepared.revisionId, + }, context(), ); await dispatchOpsOperation( runtime, - { kind: "service.recreateIsolated", serviceId: "motis" }, + { + kind: "service.recreateIsolated", + serviceId: "motis", + expectedConfigurationRevisionId: prepared.revisionId, + }, context(), ); - const stack = [ - "compose", - "--env-file", - "/stack/.env", - "-f", - "/stack/.trusted-config-current/docker-compose.generated.yml", - "-f", - "/stack/docker-compose.release.yml", - ]; - expect(calls).toEqual([ + const stack = services.stackComposeArgs(prepared.stack, () => true); + expect(calls.filter((call) => call.args.includes("up"))).toEqual([ { file: "docker", args: [...stack, "up", "-d", "--force-recreate", "motis"] }, { file: "docker", args: [...stack, "up", "-d", "--force-recreate", "--no-deps", "motis"] }, ]); + expect(calls.filter((call) => call.args.includes("config"))).toEqual([ + { file: "docker", args: [...stack, "config", "--format", "json"] }, + ]); }); it("uses fixed container identities for MOTIS effects", async () => { @@ -1529,7 +1603,13 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done await expect( dispatchOpsOperation( runtime, - { expectedCurrentRevisionId: null, kind: "stack.render", revisionId: "revision_1" }, + { + expectedCurrentRevisionId: null, + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running", + kind: "stack.render", + revisionId: `cfg1_${"a".repeat(43)}`, + }, context(), ), ).rejects.toMatchObject({ @@ -1539,7 +1619,15 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done dispatchOpsOperation(runtime, { kind: "stack.stop" }, context()), ).rejects.toMatchObject({ name: "OpsNotWiredError" }); await expect( - dispatchOpsOperation(runtime, { kind: "service.update", serviceId: "app-api" }, context()), + dispatchOpsOperation( + runtime, + { + kind: "service.update", + serviceId: "app-api", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, + context(), + ), ).rejects.toMatchObject({ name: "OpsNotWiredError" }); }); @@ -1963,7 +2051,7 @@ it("streams the actual default fixed Docker human output without the JSON collec }); const completion = dispatchOpsOperation( runtime, - { kind: "service.pull", serviceId: "redis" }, + { kind: "service.restart", serviceId: "redis" }, { ...context(), emitLog: (_s, line) => { diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 438d4c245..17015e55f 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -41,7 +41,6 @@ export function isContainedProcessFailure(error: unknown): boolean { const DEFAULT_TIMEOUT_MS = 120_000; const LIFECYCLE_TIMEOUT_MS = 10 * 60_000; -const PULL_TIMEOUT_MS = 10 * 60_000; const MAX_OUTPUT_BYTES = 1024 * 1024; const KILL_GRACE_MS = 2_000; @@ -422,7 +421,14 @@ function parseStackStatus(stdout: string): OpsResultFor<"stack.status"> { const state = ["running", "stopped", "created", "restarting", "paused"].includes(rawState) ? (rawState as "running" | "stopped" | "created" | "restarting" | "paused") : "unknown"; - return { serviceId: String(item.Service ?? item.Name ?? "unknown"), state }; + const rawHealth = + item.Health === undefined || item.Health === "" + ? "none" + : String(item.Health).toLowerCase(); + const health = ["healthy", "unhealthy", "starting", "none"].includes(rawHealth) + ? (rawHealth as "healthy" | "unhealthy" | "starting" | "none") + : ("unknown" as const); + return { serviceId: String(item.Service ?? item.Name ?? "unknown"), state, health }; }), }; } catch { @@ -538,10 +544,94 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { const result = await run([...composePrefix(), "ps", "--format", "json"], context.signal); return parseStackStatus(result.stdout); }; - runtime["stack.start"] = (_operation, context) => - changed([...composePrefix(), "up", "-d"], context, LIFECYCLE_TIMEOUT_MS); + const preparedCreation = async ( + operation: { expectedConfigurationRevisionId: string; serviceId?: string }, + context: OpsExecutionContext, + args: () => string[], + ) => { + if (context.artifactLease) assertArtifactTargetLeaseForward(context.artifactLease); + if (context.controlAuthority) assertControlExecutionForward(context.controlAuthority); + try { + return await services.withConfigurationGenerationWriter( + options.stack.infraDir, + async () => { + const currentRevisionId = services.currentConfigurationGeneration(options.stack.infraDir); + if (operation.expectedConfigurationRevisionId !== currentRevisionId) + throw new OpsExecutionFailure("conflict", { + serviceLifecycle: { + expectedRevisionId: operation.expectedConfigurationRevisionId, + seedRevisionId: operation.expectedConfigurationRevisionId, + finalRevisionId: null, + phase: "failed", + commit: "seed_committed", + followUp: "not_started", + failure: { + reason: "revision_conflict", + failedServiceIds: operation.serviceId ? [operation.serviceId] : [], + }, + }, + }); + const store = join(options.stack.infraDir, ".ops-agent-releases"); + await mkdir(store, { mode: 0o700, recursive: true }); + const release = await services.acquireReleaseStoreLock(store); + try { + const command = args(); + // Direct DM Start is fixed --no-recreate; guarded DM recreation is + // checked by the actual frozen-step verifier before this adapter. + // All implicit coordinator creation stays outside this generic path. + if (operation.serviceId !== "data-manager" && !command.includes("--no-deps")) { + const configuration = JSON.parse( + ( + await run( + [...composePrefix(), "config", "--format", "json"], + context.signal, + 15000, + ) + ).stdout, + ), + targets = operation.serviceId + ? [operation.serviceId] + : Object.keys(configuration.services ?? {}); + if ( + services + .composeCreationDependencyIds(configuration, targets) + .includes("data-manager") + ) + throw Error("Coordinator maintenance required before service creation"); + } + return await changed(command, context, LIFECYCLE_TIMEOUT_MS); + } finally { + release.release(); + } + }, + context.signal, + ); + } catch (error) { + if ( + error instanceof services.ConfigurationGenerationError && + error.cause instanceof OpsExecutionFailure + ) + throw error.cause; + if ( + error instanceof services.ConfigurationGenerationError && + error.result.outcome === "not_committed" && + !error.result.cleanupFailed && + isContainedProcessFailure(error.cause) + ) + throw error.cause; + throw error; + } + }; + runtime["stack.start"] = (operation, context) => + preparedCreation(operation, context, () => [...composePrefix(), "up", "-d"]); runtime["service.start"] = (operation, context) => - changed([...composePrefix(), "up", "-d", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); + preparedCreation(operation, context, () => [ + ...composePrefix(), + "up", + "-d", + ...(operation.serviceId === "data-manager" ? ["--no-recreate"] : []), + operation.serviceId, + ]); runtime["service.stop"] = async (operation, context) => { const target = await ownedTarget(operation.serviceId, context, "stop"); return changed(["stop", target.containerId], context, LIFECYCLE_TIMEOUT_MS); @@ -549,19 +639,22 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { runtime["service.restart"] = (operation, context) => changed([...composePrefix(), "restart", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.recreate"] = (operation, context) => - changed( - [...composePrefix(), "up", "-d", "--force-recreate", operation.serviceId], - context, - LIFECYCLE_TIMEOUT_MS, - ); + preparedCreation(operation, context, () => [ + ...composePrefix(), + "up", + "-d", + "--force-recreate", + operation.serviceId, + ]); runtime["service.recreateIsolated"] = (operation, context) => - changed( - [...composePrefix(), "up", "-d", "--force-recreate", "--no-deps", operation.serviceId], - context, - LIFECYCLE_TIMEOUT_MS, - ); - runtime["service.pull"] = (operation, context) => - changed([...composePrefix(), "pull", operation.serviceId], context, PULL_TIMEOUT_MS); + preparedCreation(operation, context, () => [ + ...composePrefix(), + "up", + "-d", + "--force-recreate", + "--no-deps", + operation.serviceId, + ]); runtime["service.remove"] = (operation, context) => changed([...composePrefix(), "rm", "-sf", operation.serviceId], context, LIFECYCLE_TIMEOUT_MS); runtime["service.logs"] = async (operation, context) => { diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 9366389d4..9b969c882 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -174,6 +174,12 @@ async function main(): Promise { loadAuthority: authorityLoader, communityRepositories, runFixedCli: administrativeCli, + candidateSource: { + directory: config.trustedConfigDirectory, + token: apiToken, + ownerUid: uid, + ownerGid: gid, + }, }); const releaseEffects = productionReleaseEffects(runtime); const retentionDays = Number(process.env.BACKUP_RETENTION_DAYS?.trim() || "30"); @@ -189,6 +195,7 @@ async function main(): Promise { const resources = loadConfiguredResourceAuthority(); return { revisionId: `${authority.revisionId.slice(0, 48)}-${resources.revisionId}`, + configurationAuthorityRevision: authority.revisionId, services: authority.services.map((service) => ({ serviceId: service.manifest.id, enabled: service.enabled, @@ -212,6 +219,10 @@ async function main(): Promise { serviceId: service.serviceId, isBuiltIn: service.isBuiltIn, buildable: !!service.selectionAuthority.manifest.buildCommand, + creationDependencies: services.serviceCreationDependencyIds( + authorityServices.map((value) => value.selectionAuthority), + [service.serviceId], + ), })), enabledServiceIds: (authorityServices) => { const trusted = readTrustedEnabledServiceIds( @@ -254,8 +265,9 @@ async function main(): Promise { const handoff = readAgentHandoff(config.rootDir); return handoff !== null && !isTerminalHandoff(handoff.phase); } catch { - // An unreadable record fails release operations, not every operation. - return false; + // Unknown legacy replacement retains the existing mutation/drain + // boundary; inspection and health remain reachable. + return true; } }, audit: (event) => process.stdout.write(`${JSON.stringify(event)}\n`), diff --git a/apps/ops-agent/src/journal.test.ts b/apps/ops-agent/src/journal.test.ts index 388e07e00..a971d2ecd 100644 --- a/apps/ops-agent/src/journal.test.ts +++ b/apps/ops-agent/src/journal.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { chmodSync, existsSync, @@ -11,7 +12,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { opsOperationFingerprint } from "@openmapx/core/ops"; +import { canonicalJson, opsOperationFingerprint } from "@openmapx/core/ops"; import { afterEach, describe, expect, it } from "vitest"; import { OPS_JOB_JOURNAL_MAX_BYTES, @@ -19,6 +20,7 @@ import { openOpsJobJournal, type PersistedOpsJob, } from "./journal"; +import { buildOpsAgentServer } from "./server"; const roots: string[] = []; afterEach(() => { @@ -32,7 +34,11 @@ function root(): string { } function job(overrides: Partial = {}): PersistedOpsJob { - const operation = overrides.operation ?? { kind: "service.pull", serviceId: "redis" }; + const operation = overrides.operation ?? { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }; return { role: "api", operation, @@ -48,6 +54,73 @@ function job(overrides: Partial = {}): PersistedOpsJob { } describe("durable ops job journal", () => { + it("keeps an original historical pull journal as byte-identical observation without dispatchable work", async () => { + const path = join(root(), "journal", "jobs-v1.json"); + await openOpsJobJournal(path); + const operation = { kind: "service.pull", serviceId: "redis" }, + record = { + role: "api", + operation, + operationId: "job1_legacyPullObservation0", + operationKey: "opk1_legacyPullObservation0", + fingerprint: createHash("sha256").update(canonicalJson(operation)).digest("hex"), + resourceId: "redis", + state: "succeeded", + submittedAt: "2026-08-23T18:00:00.000Z", + updatedAt: "2026-08-23T18:00:01.000Z", + terminalAt: "2026-08-23T18:00:01.000Z", + result: { changed: true }, + }, + encoded = JSON.stringify(record); + writeFileSync(path, JSON.stringify({ version: 2, jobs: [record] }), { mode: 0o600 }); + const journal = await openOpsJobJournal(path); + expect(journal.records()).toHaveLength(0); + expect(JSON.stringify(journal.historicalPullRecords?.()[0])).toBe(encoded); + const token = Buffer.alloc(32, 3).toString("base64url"), + app = buildOpsAgentServer({ + tokens: { api: token, "data-manager": Buffer.alloc(32, 4).toString("base64url") }, + journal, + }); + try { + const response = await app.inject({ + method: "GET", + url: `/v1/operations/${record.operationId}`, + headers: { authorization: `Bearer ${token}` }, + }); + expect(response.statusCode).toBe(200); + expect(response.json().result).toEqual({ + version: 1, + operationId: record.operationId, + operationKey: record.operationKey, + kind: "service.pull", + resourceId: "redis", + state: record.state, + submittedAt: record.submittedAt, + updatedAt: record.updatedAt, + result: record.result, + }); + const now = Date.now(), + rejected = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${token}` }, + payload: { + version: 1, + requestId: "ops1_legacyPullRejected00", + operationKey: record.operationKey, + issuedAt: new Date(now).toISOString(), + expiresAt: new Date(now + 20000).toISOString(), + operation, + }, + }); + expect(rejected.statusCode).toBe(400); + expect(journal.records()).toHaveLength(0); + } finally { + await app.close(); + } + await journal.replace([]); + expect(JSON.stringify(JSON.parse(readFileSync(path, "utf8")).jobs[0])).toBe(encoded); + }); it("durably probes a missing journal before open succeeds", async () => { const path = join(root(), "journal", "jobs-v1.json"); await openOpsJobJournal(path); @@ -143,14 +216,23 @@ describe("durable ops job journal", () => { await restored.replace([ job({ state: "succeeded", - result: { changed: true }, + result: { + changed: true, + revisionId: `cfg1_${"a".repeat(43)}`, + imageReference: "redis:8", + imageId: `sha256:${"b".repeat(64)}`, + releaseSelectionDigest: null, + }, updatedAt: "2026-08-23T18:03:00.000Z", terminalAt: "2026-08-23T18:03:00.000Z", }), ]); const terminal = await openOpsJobJournal(path); expect(terminal.records()).toEqual([ - expect.objectContaining({ state: "succeeded", result: { changed: true } }), + expect.objectContaining({ + state: "succeeded", + result: expect.objectContaining({ changed: true, imageReference: "redis:8" }), + }), ]); }); diff --git a/apps/ops-agent/src/journal.ts b/apps/ops-agent/src/journal.ts index d8fc3c06a..3d8c7cfce 100644 --- a/apps/ops-agent/src/journal.ts +++ b/apps/ops-agent/src/journal.ts @@ -1,9 +1,13 @@ -import { randomBytes } from "node:crypto"; -import { constants } from "node:fs"; +import { createHash, randomBytes } from "node:crypto"; +import { constants, realpathSync } from "node:fs"; import { mkdir, open, opendir, rename, unlink } from "node:fs/promises"; -import { basename, dirname, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { type ControlAdoption, controlAdoptionSchema } from "@openmapx/core/build-artifacts"; import { + canonicalJson, + type HistoricalServicePullOperation, + historicalServicePullOperationSchema, + historicalServicePullResultSchema, OPS_KIND_POLICIES, OPS_PUBLIC_ERROR_MESSAGES, type OpsErrorClass, @@ -19,6 +23,7 @@ import { parseBoundedOpsResult, processDiagnosticsSchema, type RetainedExecutionAdmission, + validateRetainedConfigurationMaintenanceRecoveryControl, validateRetainedExecutionAdmission, } from "@openmapx/core/ops"; @@ -71,8 +76,35 @@ export interface PersistedOpsJob { export interface OpsJobJournal { records(): readonly PersistedOpsJob[]; + /** Historical pulls are an immutable observation view, never worker input. */ + historicalPullRecords?(): readonly HistoricalPullJob[]; replace(records: readonly PersistedOpsJob[]): Promise; } +const productionJournalIndexes = new WeakMap< + OpsJobJournal, + { + path: string; + read(): readonly PersistedOpsJob[]; + } +>(); +/** A bounded locator view of this actual journal, never source authority. */ +export function readProductionCoordinatorJournalIndex(journal: OpsJobJournal, dataRoot: string) { + const owned = productionJournalIndexes.get(journal); + if (!owned || owned.path !== join(realpathSync(dataRoot), "ops-agent", "jobs-v1.json")) + throw Error("Actual fixed ops-agent journal required"); + const records = owned.read(); + if (records.length > OPS_JOB_JOURNAL_MAX_ENTRIES) throw Error("Ops journal index limit"); + return records; +} +export interface HistoricalPullJob extends Omit { + operation: HistoricalServicePullOperation; +} +type RetainedOpsJob = PersistedOpsJob | HistoricalPullJob; +function historical(job: RetainedOpsJob): job is HistoricalPullJob { + return ( + job.operation.kind === "service.pull" && !("expectedConfigurationRevisionId" in job.operation) + ); +} export interface OpenOpsJobJournalOptions { now?: () => Date; @@ -93,7 +125,7 @@ function iso(value: unknown): value is string { ); } -function parseJob(value: unknown): PersistedOpsJob { +function parseJob(value: unknown): RetainedOpsJob { if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid"); const raw = value as Record; if ( @@ -146,19 +178,33 @@ function parseJob(value: unknown): PersistedOpsJob { controlAdoptionSchema.parse(raw.controlAdoption); if (raw.executionVersion !== 2) throw Error("invalid control adoption"); } - const operation = opsOperationSchema.parse(raw.operation); + const legacy = historicalServicePullOperationSchema.safeParse(raw.operation), + operation = legacy.success ? legacy.data : opsOperationSchema.parse(raw.operation), + bodyFingerprint = legacy.success + ? createHash("sha256").update(canonicalJson(operation)).digest("hex") + : opsOperationFingerprint(opsOperationSchema.parse(operation)); if ( - opsOperationFingerprint(operation) !== - (raw.executionVersion === 2 ? raw.operationFingerprint : raw.fingerprint) || - opsResourceId(operation) !== raw.resourceId + bodyFingerprint !== (raw.executionVersion === 2 ? raw.operationFingerprint : raw.fingerprint) || + (legacy.success + ? legacy.data.serviceId + : opsResourceId(opsOperationSchema.parse(operation))) !== raw.resourceId ) { throw new Error("invalid"); } if (raw.executionVersion === 2) { - validateRetainedExecutionAdmission( - operation.kind, - raw as unknown as RetainedExecutionAdmission, - ); + if ( + operation.kind === "configurationMaintenance.settle" || + (operation.kind === "configurationMaintenance.decide" && operation.decision === "abandon") + ) + validateRetainedConfigurationMaintenanceRecoveryControl( + operation, + raw as unknown as RetainedExecutionAdmission, + ); + else + validateRetainedExecutionAdmission( + operation.kind, + raw as unknown as RetainedExecutionAdmission, + ); if (raw.requestVersion !== undefined && raw.requestVersion !== 1 && raw.requestVersion !== 2) throw new Error("invalid request version"); if (raw.admittedAt !== Date.parse(raw.submittedAt as string)) throw new Error("invalid"); @@ -188,9 +234,23 @@ function parseJob(value: unknown): PersistedOpsJob { if (state === "succeeded") { if (raw.result === undefined || raw.errorClass !== undefined || raw.terminalAt === undefined) throw new Error("invalid"); - parseBoundedOpsResult(operation.kind, raw.result); + if (legacy.success) historicalServicePullResultSchema.parse(raw.result); + else parseBoundedOpsResult(operation.kind, raw.result); } else if (state === "failed" || state === "timed_out") { - if (raw.result !== undefined || raw.errorClass === undefined || raw.terminalAt === undefined) + // This finite metadata control can retain an accepted intent without any + // conclusive completion. Startup rejoins it to the authenticated private + // owner before lookup; parsing this projection grants no completion/ACK. + const unresolvedSettle = + raw.executionVersion === 2 && + operation.kind === "configurationMaintenance.settle" && + state === "failed" && + raw.errorClass === "recovery_required" && + raw.failureResult === undefined; + if ( + raw.result !== undefined || + raw.errorClass === undefined || + (raw.terminalAt === undefined && !unresolvedSettle) + ) throw new Error("invalid"); } else if ( raw.result !== undefined || @@ -199,10 +259,10 @@ function parseJob(value: unknown): PersistedOpsJob { ) { throw new Error("invalid"); } - return raw as unknown as PersistedOpsJob; + return raw as unknown as RetainedOpsJob; } -function parseFile(bytes: Uint8Array, maxEntries: number): PersistedOpsJob[] { +function parseFile(bytes: Uint8Array, maxEntries: number): RetainedOpsJob[] { const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); const decoded = JSON.parse(text) as unknown; if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid"); @@ -292,7 +352,7 @@ async function safeRead(path: string, maxBytes: number): Promise { +export async function readOpsJobJournalSnapshot(path: string): Promise { if (basename(path) !== "jobs-v1.json") throw new Error("Ops job journal is unavailable"); const bytes = await safeRead(path, OPS_JOB_JOURNAL_MAX_BYTES); if (!bytes) return []; @@ -324,7 +384,7 @@ export async function openOpsJobJournal( throw new Error("Invalid ops job journal limits"); } const now = options.now ?? (() => new Date()); - let current: PersistedOpsJob[] = []; + let current: RetainedOpsJob[] = []; try { await prepareJournalDirectory(path); const bytes = await safeRead(path, maxBytes); @@ -335,8 +395,11 @@ export async function openOpsJobJournal( let chain = Promise.resolve(); const writeSnapshot = async (records: readonly PersistedOpsJob[]): Promise => { - if (records.length > maxEntries) throw new Error("Ops job journal capacity exceeded"); - const validated = records.map(parseJob); + const observed = current.filter(historical), + fresh = records.map(parseJob); + if (fresh.some(historical)) throw Error("Historical pull cannot enter the worker journal"); + const validated = [...observed, ...fresh]; + if (validated.length > maxEntries) throw new Error("Ops job journal capacity exceeded"); for (const record of validated) { const retained = current.find( (prior) => @@ -392,7 +455,11 @@ export async function openOpsJobJournal( }; const journal: OpsJobJournal = { - records: () => current.map((entry) => structuredClone(entry)), + records: () => + current + .filter((entry): entry is PersistedOpsJob => !historical(entry)) + .map((entry) => structuredClone(entry)), + historicalPullRecords: () => current.filter(historical).map((entry) => structuredClone(entry)), replace: (records) => { const snapshot = records.map((entry) => structuredClone(entry)); const next = chain.then(() => writeSnapshot(snapshot)); @@ -401,14 +468,14 @@ export async function openOpsJobJournal( }, }; - const interrupted = current.some((entry) => - ["queued", "running", "termination_pending"].includes(entry.state), - ); + const interrupted = journal + .records() + .some((entry) => ["queued", "running", "termination_pending"].includes(entry.state)); try { if (interrupted) { const timestamp = now().toISOString(); await journal.replace( - current.map((entry) => + journal.records().map((entry) => ["queued", "running", "termination_pending"].includes(entry.state) ? { ...entry, @@ -427,10 +494,17 @@ export async function openOpsJobJournal( ), ); } else { - await journal.replace(current); + await journal.replace(journal.records()); } } catch { throw new Error("Ops job journal is unavailable"); } + productionJournalIndexes.set(journal, { + path: join(realpathSync(dirname(path)), basename(resolve(path))), + read: () => + current + .filter((entry): entry is PersistedOpsJob => !historical(entry)) + .map((entry) => structuredClone(entry)), + }); return journal; } diff --git a/apps/ops-agent/src/owned-services.test.ts b/apps/ops-agent/src/owned-services.test.ts index 38ea2bc69..b90327a74 100644 --- a/apps/ops-agent/src/owned-services.test.ts +++ b/apps/ops-agent/src/owned-services.test.ts @@ -47,7 +47,11 @@ describe("owned service cleanup", () => { "service.build", ] as const) await expect( - host.claimer.claim({ kind, serviceId: "motis" }, "a".repeat(64), signal()), + host.claimer.claim( + { kind, serviceId: "motis", expectedConfigurationRevisionId: host.initial.revisionId }, + "a".repeat(64), + signal(), + ), ).resolves.toBeNull(); }); diff --git a/apps/ops-agent/src/owned-services.ts b/apps/ops-agent/src/owned-services.ts index 3e8e6a13f..4e56d7be2 100644 --- a/apps/ops-agent/src/owned-services.ts +++ b/apps/ops-agent/src/owned-services.ts @@ -1,7 +1,44 @@ -import { existsSync, lstatSync, realpathSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + existsSync, + fstatSync, + lstatSync, + openSync, + readSync, + realpathSync, +} from "node:fs"; import { dirname, join, resolve } from "node:path"; +import { + type ControlInstallation, + type ControlRetentionPreparationContext, + controlDigest, + controlInstallationDataRoot, + controlRetentionPreparationState, + readVerifiedStagedCoordinatorOwnership, + type VerifiedCoordinatorPullBaseline, + type VerifiedStagedCoordinatorOwnerBaseline, + type VerifiedStagedCoordinatorOwnership, + verifiedCoordinatorPullBaselineState, + verifiedStagedCoordinatorOwnerBaselineState, + verifiedStagedCoordinatorOwnershipState, +} from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + type ExecutionRef, + type OpsOperation, + opsOperationFingerprint, + parseOpsJobStatus, +} from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import type { DockerRuntimeOptions } from "./docker-runtime"; +import { + type OpsJobJournal, + type PersistedOpsJob, + readProductionCoordinatorJournalIndex, +} from "./journal"; +import type { OpsTrustedClaim } from "./runtime"; import type { TrustedConfigurationAuthoritySnapshot } from "./trusted-config-runtime"; export interface OwnedServiceSnapshot { @@ -27,6 +64,390 @@ export interface OwnedServiceResolver { inspectOwnedServices(signal: AbortSignal): Promise; inspectInventory?(signal: AbortSignal): Promise; } +declare const captureBrand: unique symbol; +export type CoordinatorPullOwnershipCapture = Readonly<{ [captureBrand]: true }>; +export interface CoordinatorPullOwnershipObservation { + operationFingerprint: string; + candidateId: string; + configurationAuthorityRevision: string; + policyRevision: string; + observedAt: number; + configImageReference: string; + composeSourcesDigest: string; + container: OwnedServiceSnapshot; +} +const literalCaptures = new WeakMap< + OwnedServiceResolver, + ( + operation: Extract, + fingerprint: string, + revisions: { configurationAuthorityRevision: string; policyRevision: string }, + signal: AbortSignal, + ) => Promise +>(); +const captures = new WeakMap< + CoordinatorPullOwnershipCapture, + CoordinatorPullOwnershipObservation +>(); +const attached = new WeakSet(); +const claimCaptures = new WeakMap(); +const originalInspectors = new WeakMap< + OwnedServiceResolver, + (proof: VerifiedCoordinatorPullBaseline, signal: AbortSignal) => Promise +>(); +const stagedMaintenanceInspectors = new WeakMap< + OwnedServiceResolver, + ( + signal: AbortSignal, + ) => Promise<{ services: OwnedServiceSnapshot[]; proof?: VerifiedStagedCoordinatorOwnership }> +>(); +const stagedOwnerInspectors = new WeakMap< + OwnedServiceResolver, + ( + proof: VerifiedStagedCoordinatorOwnerBaseline, + signal: AbortSignal, + ) => Promise +>(); +export function inspectStagedCoordinatorMaintenanceSource( + resolver: OwnedServiceResolver, + signal: AbortSignal, +) { + const inspect = stagedMaintenanceInspectors.get(resolver); + if (!inspect) throw Error(FAILED); + return inspect(signal); +} +export function inspectStagedCoordinatorOwnerServices( + resolver: OwnedServiceResolver, + proof: VerifiedStagedCoordinatorOwnerBaseline, + signal: AbortSignal, +) { + const inspect = stagedOwnerInspectors.get(resolver); + if (!inspect) throw Error(FAILED); + return inspect(proof, signal); +} +const literalRoots = new WeakMap(); +const stagedSources = new WeakMap< + OwnedServiceResolver, + { installation: ControlInstallation; journal: OpsJobJournal } +>(); +/** Only fixed startup can bind a real journal and installation to a literal resolver. */ +export function bindCoordinatorStagedOwnershipSource( + resolver: OwnedServiceResolver, + installation: ControlInstallation, + journal: OpsJobJournal, +): void { + const infra = literalRoots.get(resolver); + if ( + !infra || + installation.role !== "ops-agent" || + stagedSources.has(resolver) || + realpathSync(controlInstallationDataRoot(installation)) !== join(infra, "data") + ) + throw Error(FAILED); + readProductionCoordinatorJournalIndex(journal, controlInstallationDataRoot(installation)); + stagedSources.set(resolver, { installation, journal }); +} +function stagedProofForJob( + installation: ControlInstallation, + job: PersistedOpsJob, + context?: ControlRetentionPreparationContext, +) { + const status = parseOpsJobStatus({ + version: 2, + admittedAt: job.admittedAt, + deadlineAt: job.deadlineAt, + fingerprint: job.fingerprint, + operationId: job.operationId, + operationKey: job.operationKey, + kind: job.operation.kind, + resourceId: job.resourceId, + state: job.state, + submittedAt: job.submittedAt, + updatedAt: job.updatedAt, + result: job.result, + ...(job.terminationRequestedAt ? { terminationRequestedAt: job.terminationRequestedAt } : {}), + }); + const execution: ExecutionRef = { + executor: "ops-agent", + principal: "api", + executionId: job.operationId, + operationKey: job.operationKey, + operationFingerprint: job.operationFingerprint!, + fingerprint: job.fingerprint, + deadlineAt: job.deadlineAt!, + }; + return readVerifiedStagedCoordinatorOwnership(installation, execution, status, context); +} +const stagedRetentionInspectors = new WeakMap< + OwnedServiceResolver, + ( + proof: VerifiedStagedCoordinatorOwnership, + signal: AbortSignal, + ) => Promise<"needed" | "not-needed"> +>(); +const stagedConfigurationInspectors = new WeakMap< + OwnedServiceResolver, + (signal: AbortSignal) => Promise<"none" | "required" | "unknown"> +>(); +/** Actual literal Docker/cache/source observation. No structural resolver or + * caller-supplied absence flag can qualify a managed configuration write. */ +export async function inspectLiteralStagedConfigurationDependency( + resolver: OwnedServiceResolver, + signal: AbortSignal, +) { + const inspect = stagedConfigurationInspectors.get(resolver); + if (!inspect) throw Error(FAILED); + return inspect(signal); +} +const oldStagedDigest = ( + baseline: import("@openmapx/core/build-artifacts").CoordinatorPullBaseline, +) => + controlDigest({ + container: baseline.container, + configImageReference: baseline.configImageReference, + composeSourcesDigest: baseline.composeSourcesDigest, + configurationRevisionId: baseline.configurationRevisionId, + }); +export async function readLiteralCoordinatorStagedRetention( + resolver: OwnedServiceResolver, + installation: ControlInstallation, + journal: OpsJobJournal, + context: ControlRetentionPreparationContext, +) { + const bound = stagedSources.get(resolver), + state = controlRetentionPreparationState(context); + if ( + !bound || + bound.installation !== installation || + bound.journal !== journal || + state.installationId !== installation.installationId || + state.role !== "ops-agent" + ) + throw Error(FAILED); + if (state.mode === "committed-cleanup") return undefined; + const records = readProductionCoordinatorJournalIndex( + journal, + controlInstallationDataRoot(installation), + ); + let result: + | { + source: ReturnType["source"]; + condition: "not-needed" | "higher-ranked-successor"; + successor: ReturnType["source"] | null; + oldContainerDigest: string; + } + | undefined; + for (const fact of state.executions) { + const job = records.find( + (job) => + job.operationId === fact.execution.executionId && + job.operationKey === fact.execution.operationKey, + ); + if (!job) { + if (fact.requiresJournal) throw Error("Actual retiring journal member required"); + continue; + } + if ( + job.operation.kind !== "service.pull" || + job.operation.serviceId !== "data-manager" || + !job.operation.candidateId || + job.executionVersion !== 2 || + job.requestVersion !== 2 || + job.state !== "succeeded" + ) + continue; + if (result) throw Error("Ambiguous staged retirement source family"); + const proof = stagedProofForJob(installation, job, context), + source = verifiedStagedCoordinatorOwnershipState(proof).source; + const inspect = stagedRetentionInspectors.get(resolver); + if (!inspect) throw Error(FAILED); + const need = await inspect(proof, AbortSignal.timeout(15000)), + oldContainerDigest = oldStagedDigest(source.baseline); + if (need === "not-needed") { + result = { source, condition: "not-needed", successor: null, oldContainerDigest }; + continue; + } + const successor = stagedProofs(resolver) + .map((value) => verifiedStagedCoordinatorOwnershipState(value).source) + .find( + (value) => + value.execution.operationKey !== source.execution.operationKey && + (value.admittedAt > source.admittedAt || + (value.admittedAt === source.admittedAt && + value.execution.operationKey > source.execution.operationKey)) && + oldStagedDigest(value.baseline) === oldContainerDigest, + ); + if (!successor) throw Error("Last staged coordinator source remains required"); + result = { source, condition: "higher-ranked-successor", successor, oldContainerDigest }; + } + return result; +} +function stagedProofs(resolver: OwnedServiceResolver): VerifiedStagedCoordinatorOwnership[] { + const source = stagedSources.get(resolver); + if (!source) return []; + const records = readProductionCoordinatorJournalIndex( + source.journal, + controlInstallationDataRoot(source.installation), + ); + const proofs: VerifiedStagedCoordinatorOwnership[] = []; + for (const job of records) { + if ( + job.role !== "api" || + job.operation.kind !== "service.pull" || + job.operation.serviceId !== "data-manager" || + !job.operation.candidateId || + job.executionVersion !== 2 || + job.requestVersion !== 2 || + job.state !== "succeeded" + ) + continue; + try { + proofs.push(stagedProofForJob(source.installation, job)); + } catch { + /* An ineligible journal locator cannot authenticate provenance. */ + } + } + return proofs.sort((a, b) => { + const x = verifiedStagedCoordinatorOwnershipState(a).source, + y = verifiedStagedCoordinatorOwnershipState(b).source; + return ( + y.admittedAt - x.admittedAt || + (x.execution.operationKey < y.execution.operationKey + ? 1 + : x.execution.operationKey > y.execution.operationKey + ? -1 + : 0) + ); + }); +} +export async function captureCoordinatorPullOwnership( + resolver: OwnedServiceResolver, + operation: OpsOperation, + fingerprint: string, + revisions: { configurationAuthorityRevision: string; policyRevision: string }, + signal: AbortSignal, +): Promise { + const capture = literalCaptures.get(resolver); + if ( + !capture || + operation.kind !== "service.pull" || + operation.serviceId !== "data-manager" || + !operation.candidateId || + opsOperationFingerprint(operation) !== fingerprint || + !revisions.configurationAuthorityRevision || + !revisions.policyRevision + ) + throw Error(FAILED); + const observation = await capture(operation, fingerprint, revisions, signal); + const token = Object.freeze({}) as CoordinatorPullOwnershipCapture; + captures.set(token, observation); + return token; +} +function matchingClaim(claim: OpsTrustedClaim, value: CoordinatorPullOwnershipObservation) { + return ( + claim.source === "registry" && + claim.operation.kind === "service.pull" && + claim.operation.serviceId === "data-manager" && + claim.operation.candidateId === value.candidateId && + claim.fingerprint === value.operationFingerprint && + opsOperationFingerprint(claim.operation as OpsOperation) === value.operationFingerprint && + claim.capability.revisionId === value.policyRevision && + claim.capability.values.ownedService === JSON.stringify(value.container) + ); +} +export function attachCoordinatorPullOwnershipCapture( + claim: OpsTrustedClaim, + capture: CoordinatorPullOwnershipCapture, +): void { + const value = captures.get(capture); + if (!value || attached.has(capture) || claimCaptures.has(claim) || !matchingClaim(claim, value)) + throw Error(FAILED); + attached.add(capture); + claimCaptures.set(claim, capture); +} +export function consumeCoordinatorPullOwnershipCapture( + claim: OpsTrustedClaim, + expected: { operationFingerprint: string; candidateId: string; policyRevision: string }, +): CoordinatorPullOwnershipObservation { + const token = claimCaptures.get(claim); + claimCaptures.delete(claim); + const value = token && captures.get(token); + if (token) captures.delete(token); + if ( + !value || + !matchingClaim(claim, value) || + value.operationFingerprint !== expected.operationFingerprint || + value.candidateId !== expected.candidateId || + value.policyRevision !== expected.policyRevision + ) + throw Error(FAILED); + return value; +} +/** No caller-supplied image or snapshot enters the private literal inspector. */ +export async function inspectOriginalCoordinatorServices( + resolver: OwnedServiceResolver, + proof: VerifiedCoordinatorPullBaseline, + signal: AbortSignal, +): Promise { + const inspect = originalInspectors.get(resolver); + if (!inspect) throw Error(FAILED); + return inspect(proof, signal); +} +/** Actual ordered, bounded FD bytes and identity, never retained contents. */ +function composeSourcesDigest(files: readonly string[]): string { + if (files.length < 1 || files.length > 2) throw Error(FAILED); + const tuples = files.map((path) => { + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = fstatSync(fd, { bigint: true }), + current = lstatSync(path, { bigint: true }); + const same = (a: typeof before, b: typeof before) => + ["dev", "ino", "uid", "mode", "nlink", "size", "mtimeNs", "ctimeNs"].every( + (key) => a[key as keyof typeof a] === b[key as keyof typeof b], + ); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== BigInt(process.geteuid?.() ?? 0) || + (before.mode & 0o022n) !== 0n || + before.size > 1048576n || + !same(before, current) + ) + throw Error(FAILED); + const bytes = Buffer.alloc(Number(before.size) + 1); + let used = 0; + while (used < bytes.length) { + const count = readSync(fd, bytes, used, bytes.length - used, null); + if (!count) break; + used += count; + } + if ( + BigInt(used) !== before.size || + !same(before, fstatSync(fd, { bigint: true })) || + !same(before, lstatSync(path, { bigint: true })) + ) + throw Error(FAILED); + return { + path, + dev: String(before.dev), + ino: String(before.ino), + uid: String(before.uid), + mode: String(before.mode), + nlink: String(before.nlink), + size: String(before.size), + mtimeNs: String(before.mtimeNs), + ctimeNs: String(before.ctimeNs), + sha256Bytes: createHash("sha256").update(bytes.subarray(0, used)).digest("hex"), + }; + } finally { + closeSync(fd); + } + }); + return createHash("sha256") + .update("openmapx-coordinator-pull-compose-baseline-v1\0") + .update(canonicalJson(tuples)) + .digest("hex"); +} const FAILED = "Owned service inventory unavailable"; const ID = /^[a-f0-9]{64}$/; const IMAGE = /^sha256:[a-f0-9]{64}$/; @@ -38,7 +459,20 @@ export function createOwnedServiceResolver(options: { execFile: NonNullable; loadAuthority(): Promise; }): OwnedServiceResolver { - const inspectInventory = async (signal: AbortSignal): Promise => { + const inspectInventory = async ( + signal: AbortSignal, + original?: { + baseline: import("@openmapx/core/build-artifacts").CoordinatorPullBaseline; + expectedCurrentRevisionId: string; + currentAuthorityRevision: string; + }, + captured?: { + revision?: string; + observations: Map; + }, + stagedObservation?: { proof?: VerifiedStagedCoordinatorOwnership }, + cacheOnly = false, + ): Promise => { signal = AbortSignal.any([signal, AbortSignal.timeout(15_000)]); const run = async (args: string[]) => { signal.throwIfAborted(); @@ -53,6 +487,14 @@ export function createOwnedServiceResolver(options: { }; try { const authority = await options.loadAuthority(); + if (captured) captured.revision = authority.revisionId; + if ( + original && + (original.currentAuthorityRevision !== authority.revisionId || + services.currentConfigurationGeneration(options.stack.infraDir) !== + original.expectedCurrentRevisionId) + ) + throw Error(FAILED); if ( authority.services.length > 256 || new Set(authority.services.map((s) => s.manifest.id)).size !== authority.services.length @@ -74,7 +516,13 @@ export function createOwnedServiceResolver(options: { const ids = listing ? listing.split(/\r?\n/) : []; if (ids.length > 256 || ids.some((id) => !ID.test(id)) || new Set(ids).size !== ids.length) throw new Error(FAILED); - if (!ids.length) return { services: [], observedContainerIds: [], unverifiedServiceIds: [] }; + if (!ids.length) + return { + services: [], + observedContainerIds: [], + unverifiedServiceIds: [], + coordinatorObserved: 0, + }; const rows = (await run(["container", "inspect", "--format", FORMAT, ...ids])) .trim() .split(/\r?\n/); @@ -92,6 +540,7 @@ export function createOwnedServiceResolver(options: { const candidates = new Set(); const images = new Map(); const configurations = new Map>(); + let coordinatorObserved = 0; for (let index = 0; index < rows.length; index++) { const row = JSON.parse(rows[index] ?? "") as Record; if ( @@ -107,12 +556,25 @@ export function createOwnedServiceResolver(options: { throw new Error(FAILED); const labels = row.Labels as Record; const serviceId = labels["com.docker.compose.service"]; + if ( + labels["com.docker.compose.project"] === services.STACK_PROJECT && + serviceId === "data-manager" + ) + coordinatorObserved++; if ( labels["com.docker.compose.project"] !== services.STACK_PROJECT || typeof serviceId !== "string" || !registry.has(serviceId) ) continue; + // The installed staged source must never choose between registered + // DM rows, including a row whose paths or image cannot be verified. + if ( + serviceId === "data-manager" && + stagedSources.has(resolver) && + candidates.has(serviceId) + ) + throw Error(FAILED); candidates.add(serviceId); const working = labels["com.docker.compose.project.working_dir"]; const files = labels["com.docker.compose.project.config_files"]; @@ -176,6 +638,10 @@ export function createOwnedServiceResolver(options: { } catch { continue; } + const digestBefore = + captured || original || (serviceId === "data-manager" && stagedSources.has(resolver)) + ? composeSourcesDigest(configFiles) + : undefined; const key = JSON.stringify(configFiles); let configuration = configurations.get(key); if (!configuration) { @@ -228,7 +694,43 @@ export function createOwnedServiceResolver(options: { if (!IMAGE.test(imageId)) throw new Error(FAILED); images.set(reference, imageId); } + if (digestBefore && composeSourcesDigest(configFiles) !== digestBefore) throw Error(FAILED); + if (original && serviceId === "data-manager") { + const baseline = original.baseline; + if ( + row.Id !== baseline.container.containerId || + reference !== baseline.configImageReference || + digestBefore !== baseline.composeSourcesDigest || + row.Image !== baseline.container.provenance.imageId || + workingDirectory !== baseline.container.provenance.composeWorkingDirectory || + canonicalJson(configFiles) !== + canonicalJson(baseline.container.provenance.composeConfigFiles) + ) + throw Error(FAILED); + imageId = baseline.container.provenance.imageId; + } + if (row.Image !== imageId && serviceId === "data-manager" && !original && !cacheOnly) { + const current = services.currentConfigurationGeneration(options.stack.infraDir); + for (const proof of stagedProofs(resolver)) { + const baseline = verifiedStagedCoordinatorOwnershipState(proof).source.baseline; + if ( + current === baseline.configurationRevisionId && + row.Id === baseline.container.containerId && + reference === baseline.configImageReference && + digestBefore === baseline.composeSourcesDigest && + row.Image === baseline.container.provenance.imageId && + workingDirectory === baseline.container.provenance.composeWorkingDirectory && + canonicalJson(configFiles) === + canonicalJson(baseline.container.provenance.composeConfigFiles) + ) { + imageId = baseline.container.provenance.imageId; + if (stagedObservation) stagedObservation.proof = proof; + break; + } + } + } if (row.Image !== imageId) continue; + captured?.observations.set(serviceId, { reference, digest: digestBefore! }); if (found.some((entry) => entry.serviceId === serviceId)) throw new Error(FAILED); found.push({ serviceId, @@ -253,13 +755,152 @@ export function createOwnedServiceResolver(options: { unverifiedServiceIds: [...candidates].filter( (id) => !found.some((service) => service.serviceId === id), ), + coordinatorObserved, }; } catch { throw new Error(FAILED); } }; - return { - inspectInventory, + const resolver: OwnedServiceResolver = { + inspectInventory: async (signal) => { + const { coordinatorObserved: _privateCount, ...inventory } = await inspectInventory(signal); + return inventory; + }, inspectOwnedServices: async (signal) => (await inspectInventory(signal)).services, }; + literalRoots.set(resolver, realpathSync(options.stack.infraDir)); + literalCaptures.set(resolver, async (operation, fingerprint, revisions, signal) => { + const captured = { + revision: undefined as string | undefined, + observations: new Map(), + }; + const inventory = await inspectInventory(signal, undefined, captured), + container = inventory.services.find((s) => s.serviceId === "data-manager"), + observation = captured.observations.get("data-manager"); + if ( + !container || + container.state !== "running" || + !observation || + captured.revision !== revisions.configurationAuthorityRevision || + (await options.loadAuthority()).revisionId !== revisions.configurationAuthorityRevision || + services.currentConfigurationGeneration(options.stack.infraDir) !== + operation.expectedConfigurationRevisionId + ) + throw Error(FAILED); + Object.freeze(container.provenance.composeConfigFiles); + Object.freeze(container.provenance); + Object.freeze(container); + return Object.freeze({ + operationFingerprint: fingerprint, + candidateId: operation.candidateId!, + ...revisions, + observedAt: Date.now(), + configImageReference: observation.reference, + composeSourcesDigest: observation.digest, + container, + }); + }); + originalInspectors.set(resolver, async (proof, signal) => { + const original = verifiedCoordinatorPullBaselineState(proof), + result = await inspectInventory(signal, { + ...original, + currentAuthorityRevision: original.baseline.configurationAuthorityRevision, + }); + if ( + canonicalJson(result.services.find((s) => s.serviceId === "data-manager")) !== + canonicalJson(original.baseline.container) + ) + throw Error(FAILED); + return result.services; + }); + stagedMaintenanceInspectors.set(resolver, async (signal) => { + const observation: { proof?: VerifiedStagedCoordinatorOwnership } = {}; + const result = await inspectInventory(signal, undefined, undefined, observation); + return { + services: result.services, + ...(observation.proof ? { proof: observation.proof } : {}), + }; + }); + stagedConfigurationInspectors.set(resolver, async (signal) => { + try { + const ordinary = await inspectInventory(signal, undefined, undefined, undefined, true); + if (ordinary.coordinatorObserved > 1) return "unknown"; + if (ordinary.services.some((service) => service.serviceId === "data-manager")) return "none"; + if (ordinary.coordinatorObserved === 0) return "none"; + const observed: { proof?: VerifiedStagedCoordinatorOwnership } = {}; + const actual = await inspectInventory(signal, undefined, undefined, observed); + return observed.proof && + actual.services.some((service) => service.serviceId === "data-manager") + ? "required" + : "unknown"; + } catch { + return "unknown"; + } + }); + stagedOwnerInspectors.set(resolver, async (proof, signal) => { + const original = verifiedStagedCoordinatorOwnerBaselineState(proof); + services.configurationGenerationComposeFile( + options.stack.infraDir, + original.expectedCurrentRevisionId, + ); + const result = await inspectInventory(signal, original); + const actual = result.services.find((s) => s.serviceId === "data-manager"), + old = original.baseline.container; + if (!actual || canonicalJson({ ...actual, state: old.state }) !== canonicalJson(old)) + throw Error(FAILED); + return result.services; + }); + stagedRetentionInspectors.set(resolver, async (proof, signal) => { + const source = verifiedStagedCoordinatorOwnershipState(proof), + baseline = source.source.baseline; + const ordinary = await inspectInventory(signal, undefined, undefined, undefined, true), + dm = ordinary.services.find((s) => s.serviceId === "data-manager"); + if (dm) return "not-needed"; + if ( + !ordinary.observedContainerIds.includes(baseline.container.containerId) && + !ordinary.unverifiedServiceIds.includes("data-manager") + ) + return "not-needed"; + const current = services.currentConfigurationGeneration(options.stack.infraDir), + authority = await options.loadAuthority(); + if (!current) throw Error(FAILED); + const actual = await inspectInventory( + signal, + { + baseline, + expectedCurrentRevisionId: current, + currentAuthorityRevision: authority.revisionId, + }, + undefined, + undefined, + true, + ); + const old = actual.services.find((s) => s.serviceId === "data-manager"); + if ( + !old || + canonicalJson({ ...old, state: baseline.container.state }) !== + canonicalJson(baseline.container) + ) + throw Error(FAILED); + return "needed"; + }); + return resolver; +} + +export async function confirmCoordinatorPullOwnership( + resolver: OwnedServiceResolver, + operation: OpsOperation, + fingerprint: string, + baseline: import("@openmapx/core/build-artifacts").CoordinatorPullBaseline, + signal: AbortSignal, +): Promise { + const capture = literalCaptures.get(resolver); + if (!capture || operation.kind !== "service.pull") throw Error(FAILED); + const value = await capture(operation, fingerprint, baseline, signal); + if ( + canonicalJson(value.container) !== canonicalJson(baseline.container) || + value.configImageReference !== baseline.configImageReference || + value.composeSourcesDigest !== baseline.composeSourcesDigest + ) + throw Error(FAILED); } diff --git a/apps/ops-agent/src/policy.test.ts b/apps/ops-agent/src/policy.test.ts index 2b3969127..8d4927559 100644 --- a/apps/ops-agent/src/policy.test.ts +++ b/apps/ops-agent/src/policy.test.ts @@ -44,7 +44,11 @@ describe("operation-specific authority policy", () => { "service.build", ] as const) { await expect( - claimer.claim({ kind, serviceId: "worker" }, "b".repeat(64), signal), + claimer.claim( + { kind, serviceId: "worker", expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}` }, + "b".repeat(64), + signal, + ), ).resolves.toBeNull(); } await expect( @@ -245,7 +249,15 @@ describe("operation-specific authority policy", () => { }); const signal = new AbortController().signal; await expect( - claimer.claim({ kind: "service.start", serviceId: "community-live" }, "a".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "community-live", + }, + "a".repeat(64), + signal, + ), ).resolves.toBeNull(); authority = { @@ -258,10 +270,20 @@ describe("operation-specific authority policy", () => { }; enabled = new Set(["worker", "community-live"]); await expect( - claimer.claim({ kind: "service.start", serviceId: "community-live" }, "b".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "community-live", + }, + "b".repeat(64), + signal, + ), ).resolves.not.toBeNull(); const configClaim = await claimer.claim( { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "integrationConfig.apply", integrationId: "integration-live", @@ -282,11 +304,21 @@ describe("operation-specific authority policy", () => { }; enabled = new Set(["worker"]); await expect( - claimer.claim({ kind: "service.start", serviceId: "community-live" }, "d".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "community-live", + }, + "d".repeat(64), + signal, + ), ).resolves.toBeNull(); await expect( claimer.claim( { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "integrationConfig.apply", integrationId: "integration-live", @@ -327,6 +359,8 @@ describe("operation-specific authority policy", () => { await expect( claimer.claim( { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "stack.render", revisionId: `cfg1_${"r".repeat(43)}`, @@ -356,7 +390,13 @@ describe("operation-specific authority policy", () => { enabledServiceIds: () => enabled, }); const signal = new AbortController().signal; - const operations: OpsOperation[] = [{ kind: "service.start", serviceId: "worker" }]; + const operations: OpsOperation[] = [ + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "worker", + }, + ]; for (const operation of operations) { await expect(claimer.claim(operation, "a".repeat(64), signal)).resolves.not.toBeNull(); for (const serviceId of [ @@ -373,10 +413,26 @@ describe("operation-specific authority policy", () => { } enabled = new Set(["motis"]); await expect( - claimer.claim({ kind: "service.start", serviceId: "motis" }, "c".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "motis", + }, + "c".repeat(64), + signal, + ), ).resolves.not.toBeNull(); await expect( - claimer.claim({ kind: "service.start", serviceId: "worker" }, "d".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "worker", + }, + "d".repeat(64), + signal, + ), ).resolves.toBeNull(); // A reconstructed claimer reads the same recovered generation-owned set. const restarted = createProductionRegistryResourceClaimer({ @@ -389,7 +445,15 @@ describe("operation-specific authority policy", () => { enabledServiceIds: () => enabled, }); await expect( - restarted.claim({ kind: "service.start", serviceId: "motis" }, "e".repeat(64), signal), + restarted.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "motis", + }, + "e".repeat(64), + signal, + ), ).resolves.not.toBeNull(); }); @@ -400,7 +464,14 @@ describe("operation-specific authority policy", () => { trustedData: denyAllTrustedOpsData, }); await expect( - authorizeOpsResources({ kind: "service.start", serviceId: "worker" }, policy), + authorizeOpsResources( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "worker", + }, + policy, + ), ).resolves.toBe(true); for (const serviceId of ["ops-agent", "traefik", "app-api"]) { await expect( @@ -464,6 +535,8 @@ describe("operation-specific authority policy", () => { const signal = new AbortController().signal; const forbidden: OpsOperation[] = [ { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "serviceConfig.apply", serviceId: "ops-agent", @@ -475,6 +548,8 @@ describe("operation-specific authority policy", () => { serviceIds: ["worker", "traefik"], }, { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "integrationConfig.apply", integrationId: "unregistered-integration", diff --git a/apps/ops-agent/src/policy.ts b/apps/ops-agent/src/policy.ts index ce202b295..4b4e4317d 100644 --- a/apps/ops-agent/src/policy.ts +++ b/apps/ops-agent/src/policy.ts @@ -5,13 +5,22 @@ import { type OpsOperationKind, type OpsResourcePolicy, } from "@openmapx/core/ops"; -import type { OwnedServiceResolver } from "./owned-services"; +import { + inspectOwnedMaintenanceRecreateAdmission, + isOwnedMaintenanceRecreateAdmission, +} from "./configuration-maintenance-store"; +import { + attachCoordinatorPullOwnershipCapture, + captureCoordinatorPullOwnership, + type OwnedServiceResolver, +} from "./owned-services"; import type { OpsTrustedClaim } from "./runtime"; import { decideServiceAction } from "./service-action-policy"; export interface OpsClaimOwner { role: "api" | "data-manager"; operationKey: string; + maintenanceStep?: unknown; } export interface OpsResourceClaimer { @@ -72,7 +81,12 @@ export function serviceActionForOperation(kind: OpsOperationKind): ServiceAction export function createRegistryResourcePolicy(options: { serviceIds: Iterable; ownedServices?: OwnedServiceResolver; - serviceMetadata?: readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; + serviceMetadata?: readonly { + serviceId: string; + isBuiltIn: boolean; + buildable: boolean; + creationDependencies?: readonly string[]; + }[]; enabledServiceIds?: () => ReadonlySet; integrationIds: Iterable; trustedData: TrustedOpsDataSource; @@ -247,11 +261,17 @@ export function createPolicyResourceClaimer(policy: OpsResourcePolicy): OpsResou export function createRegistryResourceClaimer(options: { serviceIds: Iterable; ownedServices?: OwnedServiceResolver; - serviceMetadata?: readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; + serviceMetadata?: readonly { + serviceId: string; + isBuiltIn: boolean; + buildable: boolean; + creationDependencies?: readonly string[]; + }[]; enabledServiceIds?: () => ReadonlySet; integrationIds: Iterable; trustedData: TrustedOpsDataSource; registryRevision?: string; + configurationAuthorityRevision?: string; resourceAuthority?: Pick< OpsResourcePolicy, | "allowBackup" @@ -269,25 +289,87 @@ export function createRegistryResourceClaimer(options: { claim: async (operation, fingerprint, signal, owner) => { if (signal.aborted) throw signal.reason; if (!requiresTrustedSource(operation)) { + if ( + operation.kind === "stack.start" && + (options.enabledServiceIds?.() ?? registeredServiceIds).has("data-manager") + ) + return null; const action = serviceActionForOperation(operation.kind); if ("serviceId" in operation && action && options.ownedServices) { - const snapshot = (await options.ownedServices.inspectOwnedServices(signal)).find( - (s) => s.serviceId === operation.serviceId, - ); + const snapshots = + operation.serviceId === "data-manager" && + isOwnedMaintenanceRecreateAdmission( + owner?.maintenanceStep, + operation, + owner?.operationKey ?? "", + ) + ? await inspectOwnedMaintenanceRecreateAdmission( + owner?.maintenanceStep, + operation, + owner?.operationKey ?? "", + signal, + ) + : await options.ownedServices.inspectOwnedServices(signal); + const snapshot = snapshots.find((s) => s.serviceId === operation.serviceId); const service = options.serviceMetadata?.find( (s) => s.serviceId === operation.serviceId, ) ?? { serviceId: operation.serviceId, isBuiltIn: true, buildable: true }; + const decision = decideServiceAction( + { + revisionId: options.registryRevision ?? "registry-v1", + services: [service], + enabledServiceIds: options.enabledServiceIds?.() ?? registeredServiceIds, + }, + snapshot, + action, + ); + let coordinatorCapture: + | Awaited> + | undefined; if ( - !registeredServiceIds.has(operation.serviceId) || - !decideServiceAction( + operation.kind === "service.pull" && + operation.serviceId === "data-manager" && + operation.candidateId + ) { + if (!options.configurationAuthorityRevision || !options.registryRevision) return null; + try { + coordinatorCapture = await captureCoordinatorPullOwnership( + options.ownedServices, + operation, + fingerprint, + { + configurationAuthorityRevision: options.configurationAuthorityRevision, + policyRevision: options.registryRevision, + }, + signal, + ); + } catch { + return null; + } + } + // Only the literal opaque capture admits the existing internal Update + // submission to its mandatory first-server MAC/parent join. Bare pull + // remains unavailable and cannot obtain this private claim association. + const checkedCoordinatorUpdate = + coordinatorCapture !== undefined && + decideServiceAction( { revisionId: options.registryRevision ?? "registry-v1", services: [service], enabledServiceIds: options.enabledServiceIds?.() ?? registeredServiceIds, }, snapshot, - action, - ).available + "update", + ).workflow === "coordinator-update"; + if ( + !registeredServiceIds.has(operation.serviceId) || + (!decision.available && !checkedCoordinatorUpdate) || + (decision.execution !== "direct" && + !isOwnedMaintenanceRecreateAdmission( + owner?.maintenanceStep, + operation, + owner?.operationKey ?? "", + )) ) return null; if ( @@ -299,10 +381,12 @@ export function createRegistryResourceClaimer(options: { return null; signal.throwIfAborted(); // The admission is bound to the immutable ID and full verified provenance. - return immutableClaim(operation, fingerprint, "registry", { + const claim = immutableClaim(operation, fingerprint, "registry", { revisionId: options.registryRevision ?? "registry-v1", values: snapshot ? { ownedService: JSON.stringify(snapshot) } : {}, }); + if (coordinatorCapture) attachCoordinatorPullOwnershipCapture(claim, coordinatorCapture); + return claim; } return (await authorizeOpsResources(operation, policy)) ? immutableClaim( @@ -359,9 +443,12 @@ export function createProductionRegistryResourceClaimer< >(options: { services: Iterable; ownedServices?: OwnedServiceResolver; - serviceMetadata?: ( - services: readonly Service[], - ) => readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; + serviceMetadata?: (services: readonly Service[]) => readonly { + serviceId: string; + isBuiltIn: boolean; + buildable: boolean; + creationDependencies?: readonly string[]; + }[]; integrationIds: Iterable; trustedData: TrustedOpsDataSource; resourceAuthority?: Pick< @@ -377,6 +464,7 @@ export function createProductionRegistryResourceClaimer< enabledServiceIds?: (services: readonly Service[]) => ReadonlySet; loadAuthority?: () => Promise<{ revisionId: string; + configurationAuthorityRevision?: string; services: readonly Service[]; integrationIds: readonly string[]; }>; @@ -447,6 +535,7 @@ export function createProductionRegistryResourceClaimer< integrationIds: authority.integrationIds, trustedData, registryRevision: authority.revisionId, + configurationAuthorityRevision: authority.configurationAuthorityRevision, resourceAuthority: options.resourceAuthority, }).claim(operation, fingerprint, signal, owner); }, diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index 578556e8f..fa1b1f73b 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -1,6 +1,17 @@ +import { execFileSync, spawnSync } from "node:child_process"; import { randomBytes } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, + existsSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { closeControlInstallation, openControlInstallation, @@ -8,28 +19,444 @@ import { } from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey, type OpsOperation } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { parse as parseYaml } from "yaml"; import { installedCompose } from "../../../packages/core/src/services/__fixtures__/compose-path-host"; import { ownedComposePathRuntime } from "./__fixtures__/owned-compose-path-runtime"; import { ownedServiceHost } from "./__fixtures__/owned-service-host"; +import { createAgentHandoffBindings, createDefaultReleaseEffects } from "./administrative-runtime"; +import { agentHandoffPath, readAgentHandoff, writeAgentHandoff } from "./agent-handoff"; +import { startOpsAgentServer } from "./bootstrap"; import { runContainedProcess } from "./docker-runtime"; import { openOpsJobJournal } from "./journal"; -import { createProductionOpsRuntime } from "./production-runtime"; -import { dispatchOpsOperation } from "./runtime"; +import { createProductionOpsRuntime, productionReleaseEffects } from "./production-runtime"; +import { dispatchOpsOperation, OpsNotWiredError } from "./runtime"; import { buildOpsAgentServer } from "./server"; import { initializeTrustedConfigurationRuntime } from "./trusted-config-runtime"; const roots: string[] = []; -async function fixture() { - const host = await ownedServiceHost(); +async function fixture(options: Parameters[0] = {}) { + const host = await ownedServiceHost(options); roots.push(host.root); return host; } afterEach(() => { + vi.unstubAllEnvs(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); describe("assembled production service authority", () => { + it("refuses unsupported legacy release mutation without staging and keeps inspection available", async () => { + const host = await fixture(); + const effects = productionReleaseEffects(host.runtime); + const io = { signal: new AbortController().signal, emitLog() {} }; + const before = services.currentConfigurationGeneration(host.infraDir); + host.calls.length = 0; + await expect(effects.apply("release-unavailable", ["app-api"], io)).rejects.toBeInstanceOf( + OpsNotWiredError, + ); + await expect( + effects.updateAgent?.( + { + releaseId: "release-unavailable", + expectedDigest: "a".repeat(64), + handoffId: "handoff-unavailable", + otherActiveOperations: 0, + }, + io, + ), + ).rejects.toBeInstanceOf(OpsNotWiredError); + await expect( + dispatchOpsOperation( + host.runtime, + { + kind: "system.update", + releaseId: "release-unavailable", + expectedDigest: "a".repeat(64), + createBackup: true, + backupId: "unavailable-backup", + }, + await host.context({ kind: "configuration.inspect" }), + ), + ).rejects.toBeInstanceOf(OpsNotWiredError); + expect( + host.calls.every( + (args) => + (args[0] === "container" && ["ls", "inspect"].includes(args[1] ?? "")) || + (args[0] === "image" && args[1] === "inspect") || + (args[0] === "compose" && args.includes("config")), + ), + ).toBe(true); + expect(existsSync(join(host.infraDir, ".ops-agent-releases", "transaction.json"))).toBe(false); + expect(existsSync(agentHandoffPath(host.root))).toBe(false); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); + await expect(effects.inspect()).resolves.toBeDefined(); + await expect(effects.runtimeInspect?.()).resolves.toEqual({}); + await expect(effects.inspectSystem(io)).resolves.toMatchObject({ maintenanceReady: false }); + }); + + it("production helper entry and write bindings preserve original handoff evidence", async () => { + const host = await fixture(); + const record = { + version: 1 as const, + handoffId: "handoff-unavailable", + releaseId: "release-unavailable", + digest: "a".repeat(64), + phase: "launched" as const, + previousOverlay: "previous\n", + createdAt: "2026-10-20T00:00:00.000Z", + updatedAt: "2026-10-20T00:00:00.000Z", + }; + mkdirSync(dirname(agentHandoffPath(host.root)), { recursive: true, mode: 0o700 }); + writeAgentHandoff(host.root, record); + const path = agentHandoffPath(host.root), + bytes = readFileSync(path); + const bindings = createAgentHandoffBindings(host.root); + expect(() => bindings.writeOverlay(record.releaseId, record.digest)).toThrow(OpsNotWiredError); + expect(() => bindings.restoreOverlay(record.previousOverlay)).toThrow(OpsNotWiredError); + expect(() => bindings.assertAvailable()).toThrow(OpsNotWiredError); + const entry = spawnSync( + process.execPath, + [ + "--import", + createRequire(import.meta.url).resolve("tsx/esm"), + fileURLToPath(new URL("./agent-handoff-main.ts", import.meta.url)), + ], + { + env: { + ...process.env, + OPENMAPX_ROOT_DIR: host.root, + OPS_AGENT_HANDOFF_ID: record.handoffId, + }, + encoding: "utf8", + timeout: 15000, + }, + ); + expect(entry.status, entry.stderr).toBe(1); + expect(entry.stderr).toContain("Typed adapter is not wired"); + expect(readFileSync(path)).toEqual(bytes); + expect(readAgentHandoff(host.root)).toEqual(record); + expect(existsSync(host.stack.composeReleasePath)).toBe(false); + }); + + it("production startup preserves actual interrupted legacy transaction and handoff while reads remain available", async () => { + const host = await fixture(); + const directory = join(host.infraDir, ".ops-agent-releases"); + const manifest = services.parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: "release-legacy", + images: Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`, + ]), + ), + privacyReleaseValidation: { + version: 1, + sourceBuildFingerprint: "a".repeat(64), + validatedAt: "2026-09-05T12:00:00.000Z", + checks: { translationsConsistent: true, openApiConsistent: true, policyConsistent: true }, + }, + }), + ); + writeFileSync( + join(directory, "release-legacy.json"), + services.canonicalReleaseManifest(manifest), + { mode: 0o600 }, + ); + const legacy = createDefaultReleaseEffects( + host.root, + async () => { + throw Error("No CLI expected before prepared"); + }, + { + afterReleasePhase(phase) { + if (phase === "prepared") throw Error("actual legacy prepared interruption"); + }, + }, + ); + await expect( + legacy.apply( + manifest.release, + ["app-api"], + await host.context({ kind: "configuration.inspect" }), + ), + ).rejects.toThrow("actual legacy prepared interruption"); + mkdirSync(dirname(agentHandoffPath(host.root)), { recursive: true, mode: 0o700 }); + writeAgentHandoff(host.root, { + version: 1, + handoffId: "handoff-legacy", + releaseId: manifest.release, + digest: "a".repeat(64), + phase: "overlay_written", + previousOverlay: null, + createdAt: "2026-10-20T00:00:00.000Z", + updatedAt: "2026-10-20T00:00:00.000Z", + }); + const transaction = join(directory, "transaction.json"), + handoff = agentHandoffPath(host.root); + const before = [readFileSync(transaction), readFileSync(handoff)]; + const restarted = await createProductionOpsRuntime({ + rootDir: host.root, + stack: host.stack, + loadAuthority: host.loadAuthority, + communityRepositories: host.repositories, + docker: { execFile: host.execFile, fileExists: existsSync }, + }); + const effects = productionReleaseEffects(restarted); + host.calls.length = 0; + await expect(effects.initialize?.()).rejects.toMatchObject({ errorClass: "recovery_required" }); + await expect(effects.inspect()).resolves.toMatchObject({ + availableReleaseId: manifest.release, + }); + await expect(effects.runtimeInspect?.()).resolves.toEqual({}); + await expect(effects.inspectAgentHandoff?.("handoff-legacy")).resolves.toMatchObject({ + phase: "overlay_written", + }); + expect([readFileSync(transaction), readFileSync(handoff)]).toEqual(before); + expect(host.calls).toEqual([]); + const app = await startOpsAgentServer( + { + tokens: { + api: Buffer.alloc(32, 1).toString("base64url"), + "data-manager": Buffer.alloc(32, 2).toString("base64url"), + }, + runtime: restarted, + resourcePolicy: { allowGlobal: () => true, allowService: () => true }, + }, + effects, + { host: "127.0.0.1", port: 0 }, + ); + try { + expect((await app.inject({ method: "GET", url: "/health" })).json()).toMatchObject({ + ok: true, + releaseRecovery: { ready: false, reason: "recovery_required" }, + }); + expect([readFileSync(transaction), readFileSync(handoff)]).toEqual(before); + } finally { + await app.close(); + } + }); + it("production service.update pulls then prepares and recreates", async () => { + const host = await fixture({ + defaultCliEntry: fileURLToPath(new URL("./index.ts", import.meta.url)), + actualComposeConfig: true, + }); + const bin = join(host.root, "fake-bin"), + trace = join(host.root, "docker-trace.jsonl"); + mkdirSync(bin); + const docker = join(bin, "docker"), + realDocker = execFileSync("which", ["docker"], { encoding: "utf8" }).trim(); + writeFileSync( + docker, + `#!${process.execPath}\nconst fs=require("node:fs"),cp=require("node:child_process"),a=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(trace)},JSON.stringify(a)+"\\n");if(a[0]==="compose"&&a.includes("config")){const r=cp.spawnSync(${JSON.stringify(realDocker)},a,{encoding:"utf8"});process.stdout.write(r.stdout||"");process.stderr.write(r.stderr||"");process.exit(r.status??1);}\n`, + { mode: 0o700 }, + ); + chmodSync(docker, 0o700); + for (const [key, value] of Object.entries({ + PATH: `${bin}:${process.env.PATH}`, + OPENMAPX_ROOT_DIR: host.root, + OPENMAPX_RELEASE: "local", + DOMAIN: "example.test", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + DATA_MANAGER_AUTH_TOKEN: "d".repeat(32), + OPENMAPX_ENABLED_SERVICES: "", + })) + vi.stubEnv(key, value); + const installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + const operation = { + kind: "service.update", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + } as const; + const requestedDeadlineAt = Date.now() + 90000, + headers = { authorization: `Bearer ${host.token}`, "x-ops-protocol-version": "2" }; + try { + const submitted = await app.inject({ + method: "POST", + url: "/v1/operations", + headers, + payload: { + version: 2, + requestId: "ops1_productionUpdate0000001", + operationKey: createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20000).toISOString(), + execution: { requestedDeadlineAt }, + operation, + }, + }); + expect(submitted.statusCode, submitted.body).toBe(202); + const id = submitted.json().result.operationId, + until = Date.now() + 45000; + let status: { + state: string; + result?: { changed: boolean; revisionId: string }; + failureResult?: unknown; + } = { state: "pending" }; + do { + status = (await app.inject({ method: "GET", url: `/v1/operations/${id}`, headers })).json() + .result; + if (!["succeeded", "failed", "timed_out"].includes(status.state)) + await new Promise((resolve) => setTimeout(resolve, 20)); + } while (!["succeeded", "failed", "timed_out"].includes(status.state) && Date.now() < until); + expect(status).toMatchObject({ + state: "succeeded", + result: { changed: true, revisionId: expect.stringMatching(/^cfg1_/) }, + }); + expect(status.result?.revisionId).toBe( + services.currentConfigurationGeneration(host.infraDir), + ); + expect(status.result?.revisionId).not.toBe(host.initial.revisionId); + const calls = readFileSync(trace, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + const effects = calls.filter( + (args) => args.includes("pull") || args.includes("--force-recreate"), + ); + expect(effects).toHaveLength(2); + expect(effects[0]!.slice(-2)).toEqual(["pull", "motis"]); + expect(effects[1]!.slice(-4)).toEqual(["up", "-d", "--force-recreate", "motis"]); + expect(calls.some((args) => args.includes("config"))).toBe(true); + expect( + JSON.parse( + readFileSync( + services.currentConfigurationFile(host.infraDir, services.SERVICE_SELECTION_FILE), + "utf8", + ), + ).selected, + ).toContain("motis"); + } finally { + await app.close(); + closeControlInstallation(installation); + } + }, 60000); + it("refuses a declared indirect coordinator recreation at actual production registry admission", async () => { + const host = await fixture({ + withCoordinator: true, + coordinatorDependency: true, + actualComposeConfig: true, + }), + operation = { + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + } as const; + const claim = await host.claimer.claim( + operation, + (await import("@openmapx/core/ops")).opsOperationFingerprint(operation), + new AbortController().signal, + { + role: "api", + operationKey: (await import("./__fixtures__/execution")).testExecution().operationKey, + }, + ); + expect(claim).toBeNull(); + const inspect = { kind: "services.inspect" } as const, + status = await dispatchOpsOperation(host.runtime, inspect, await host.context(inspect)), + motis = status.services.find((service) => service.serviceId === "motis"); + for (const action of ["start", "apply", "update"] as const) + expect(motis?.actions[action]).toMatchObject({ available: false, reason: "unavailable" }); + expect(host.calls.some((args) => args.includes("up") || args.includes("pull"))).toBe(false); + }, 60000); + it("refuses a foreign native Compose dependency before an actual normal Start can recreate the coordinator", async () => { + const host = await fixture({ withCoordinator: true, actualComposeConfig: true }), + compose = parseYaml(readFileSync(host.firstCompose, "utf8")); + compose.services.motis.depends_on = { + "data-manager": { condition: "service_started", required: true }, + }; + writeFileSync(host.firstCompose, JSON.stringify(compose)); + mkdirSync(join(host.infraDir, "data/ops-agent"), { recursive: true, mode: 0o700 }); + const installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((service) => service.manifest.id), + }), + app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + controlInstallation: installation, + }), + deadline = Date.now() + 60000, + headers = { authorization: `Bearer ${host.token}`, "x-ops-protocol-version": "2" }; + try { + const submitted = await app.inject({ + method: "POST", + url: "/v1/operations", + headers, + payload: { + version: 2, + requestId: "ops1_indirectCoordinator0001", + operationKey: createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20000).toISOString(), + execution: { requestedDeadlineAt: deadline }, + operation: { + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + }, + }); + expect(submitted.statusCode).toBe(202); + let state = "queued"; + const until = Date.now() + 45000; + while (!["succeeded", "failed", "timed_out"].includes(state)) { + state = ( + await app.inject({ + method: "GET", + url: `/v1/operations/${submitted.json().result.operationId}`, + headers, + }) + ).json().result.state; + if (Date.now() > until) throw Error("Actual indirect Start did not settle"); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(state).toBe("failed"); + expect(host.calls.some((args) => args.includes("up"))).toBe(false); + } finally { + await app.close(); + closeControlInstallation(installation); + } + }, 60000); + it("refuses a prepared Start after a competing generation before any up effect", async () => { + const host = await fixture(); + const operation = { + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + } as const; + const context = await host.context(operation); + expect(await host.select(["motis"])).toMatchObject({ outcome: "committed" }); + host.calls.length = 0; + await expect(dispatchOpsOperation(host.runtime, operation, context)).rejects.toMatchObject({ + errorClass: "conflict", + }); + expect(host.calls.some((args) => args.includes("up"))).toBe(false); + }); it("executes Stop and post-stop Logs for a deselected owned container through the installed adapters", async () => { const host = await fixture(); await host.deselect(); @@ -68,7 +495,7 @@ describe("assembled production service authority", () => { it("authorizes deselected Stop and post-stop Logs through authenticated HTTP with original control authority", async () => { const host = await fixture(); await host.deselect(); - mkdirSync(join(host.infraDir, "data"), { mode: 0o700 }); + mkdirSync(join(host.infraDir, "data"), { mode: 0o700, recursive: true }); await provisionControlInstallation(host.infraDir); const installation = openControlInstallation({ infraDir: host.infraDir, @@ -179,7 +606,11 @@ describe("assembled production service authority", () => { it("rechecks deselection after admission before invoking a lifecycle effect", async () => { const host = await fixture(); - const operation = { kind: "service.start", serviceId: "motis" } as const; + const operation = { + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, + } as const; const context = await host.context(operation); await host.deselect(); await expect(dispatchOpsOperation(host.runtime, operation, context)).rejects.toThrow( @@ -339,7 +770,7 @@ describe("assembled production service authority", () => { }); async function installedCommunity() { - const host = await fixture(); + const host = await fixture({ actualComposeConfig: true }); host.setRows([]); const publication = await host.repositories.publish( { @@ -367,7 +798,11 @@ async function installedCommunity() { repository: publication.repository, extensionId: "test-extension", } as const; - const start = { kind: "service.start", serviceId: "community-live" } as const; + const start = { + kind: "service.start", + serviceId: "community-live", + expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, + } as const; const checkout = join(host.root, "services/.community", publication.repository); const record = join( host.root, @@ -402,7 +837,11 @@ describe("assembled lifecycle/removal exclusion", () => { host.stack.composeReleasePath, "services:\n motis:\n depends_on:\n community-live:\n condition: service_started\n", ); - const start = { kind: "service.start", serviceId: "motis" } as const; + const start = { + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, + } as const; const entered = Promise.withResolvers(), finish = Promise.withResolvers(); host.setCreatingProcess(async () => { @@ -482,7 +921,10 @@ describe("assembled lifecycle/removal exclusion", () => { const { host, start, remove, checkout, create } = await installedCommunity(); const startContext = await host.context(start), removeContext = await host.context(remove); - const stackStart = { kind: "stack.start" } as const; + const stackStart = { + kind: "stack.start", + expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, + } as const; const stackContext = await host.context(stackStart); const entered = Promise.withResolvers(), finish = Promise.withResolvers(); @@ -619,7 +1061,10 @@ describe("settlement and cancellation at the installed lifecycle boundary", () = await expect( dispatchOpsOperation(host.runtime, start, await host.context(start)), ).rejects.toThrow("Contained process failed"); - const global = { kind: "stack.start" } as const; + const global = { + kind: "stack.start", + expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, + } as const; await expect( dispatchOpsOperation(host.runtime, global, await host.context(global)), ).resolves.toMatchObject({ changed: true }); diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 11c227f67..732cf5bce 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -1,4 +1,4 @@ -import { existsSync } from "node:fs"; +import { existsSync, lstatSync } from "node:fs"; import { join } from "node:path"; import { type OpsOperation, SERVICE_ACTIONS, type ServiceActionDecision } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; @@ -9,10 +9,18 @@ import { createDefaultReleaseEffects, type FixedCliRunner, } from "./administrative-runtime"; +import { isTerminalHandoff, readAgentHandoff } from "./agent-handoff"; import { type CommunityRepositoryStore, installCommunityRepositoryRuntime, } from "./community-repositories"; +import type { ConfigurationCandidateSource } from "./configuration-candidate-source"; +import { installConfigurationHardlinksRuntime } from "./configuration-hardlinks-runtime"; +import { registerConfigurationMaintenanceSource } from "./configuration-maintenance-runtime"; +import { + assertOwnedMaintenanceRecreateRuntime, + assertStagedCoordinatorReleaseWrite, +} from "./configuration-maintenance-store"; import { createDockerRuntime, type DockerRuntimeOptions, @@ -21,8 +29,18 @@ import { import { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; import { createOwnedServiceResolver, type OwnedServiceResolver } from "./owned-services"; import { serviceActionForOperation } from "./policy"; -import type { OpsExecutionContext, OpsRuntime } from "./runtime"; +import { + type OpsExecutionContext, + OpsExecutionFailure, + OpsNotWiredError, + type OpsRuntime, +} from "./runtime"; import { decideServiceAction, type ServiceAuthority } from "./service-action-policy"; +import { installServiceLifecycleRuntime } from "./service-lifecycle-runtime"; +import { + assertAdmittedCoordinatorPullRuntime, + installServicePullRuntime, +} from "./service-pull-runtime"; import { resolveBootstrapEnabledServiceIds } from "./startup-authority"; import { createTransitousLockRuntime } from "./transitous-lock-runtime"; import { @@ -38,9 +56,80 @@ export interface ProductionOpsRuntimeOptions { communityRepositories: CommunityRepositoryStore; docker?: Pick; runFixedCli?: FixedCliRunner; + candidateSource?: ConfigurationCandidateSource; } const resolvers = new WeakMap(); const releaseAdapters = new WeakMap(); + +/** The legacy engine remains an internal building block for08.3. Production + * never runs its mutation/recovery branches without that guarded installer. */ +function installReleaseCapabilityBoundary( + runtime: OpsRuntime, + rootDir: string, + infraDir: string, + legacy: AdministrativeReleaseEffects, +): AdministrativeReleaseEffects { + const directory = join(infraDir, ".ops-agent-releases"); + const withSource = async (signal: AbortSignal, action: () => Promise): Promise => { + try { + return await services.withConfigurationGenerationWriter( + infraDir, + async () => { + const lock = await services.acquireReleaseStoreLock(directory); + try { + await assertStagedCoordinatorReleaseWrite(runtime, signal); + } finally { + lock.release(); + } + return action(); + }, + signal, + ); + } catch (error) { + if ( + error instanceof services.ConfigurationGenerationError && + (error.cause instanceof OpsExecutionFailure || + error.cause instanceof services.ReleaseChannelDisabledError) + ) + throw error.cause; + throw error; + } + }; + const unavailable = async (signal: AbortSignal): Promise => { + // The successful observation grants no continuation authority. No legacy + // transaction, helper launch, backup or image pull follows this check. + await withSource(signal, async () => undefined); + throw new OpsNotWiredError(); + }; + return { + ...legacy, + initialize: async () => { + // Inspection bypasses the legacy cached initializer, which may replay + // fixed CLI under the release lock or mutate handoff evidence. + await legacy.inspect(); + let transaction = false; + try { + lstatSync(join(directory, "transaction.json")); + transaction = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const handoff = readAgentHandoff(rootDir); + if (transaction || (handoff && !isTerminalHandoff(handoff.phase))) + throw new OpsExecutionFailure("recovery_required", {}); + }, + resolve: (context) => withSource(context.signal, () => legacy.resolve(context)), + inspectSystem: async (context) => ({ + ...(await legacy.inspectSystem(context)), + // Docker/Compose readiness alone cannot advertise the absent guarded + // release installer as runnable maintenance. + maintenanceReady: false, + }), + pull: (_releaseId, context) => unavailable(context.signal), + apply: (_releaseId, _serviceIds, context) => unavailable(context.signal), + updateAgent: (_request, context) => unavailable(context.signal), + }; +} export function productionReleaseEffects(runtime: OpsRuntime): AdministrativeReleaseEffects { const effects = releaseAdapters.get(runtime); if (!effects) throw new Error("Production release adapter missing"); @@ -68,6 +157,9 @@ export function serviceActionAuthority( serviceId: service.manifest.id, isBuiltIn: service.isBuiltIn, buildable: !!service.manifest.buildCommand, + creationDependencies: services.serviceCreationDependencyIds(authority.services, [ + service.manifest.id, + ]), })), }; } @@ -97,10 +189,37 @@ export async function createProductionOpsRuntime( loadAuthority: options.loadAuthority, infraDir: options.stack.infraDir, ownedServices: owned, + candidateSource: options.candidateSource, }); installCommunityRepositoryRuntime(runtime, options.communityRepositories, owned, lifecycle); + installConfigurationHardlinksRuntime(runtime, { infraDir: options.stack.infraDir }); + registerConfigurationMaintenanceSource(runtime, { + infraDir: options.stack.infraDir, + candidateSource: options.candidateSource, + stack: options.stack, + execFile: options.docker?.execFile ?? runContainedProcess, + loadAuthority: options.loadAuthority, + ownedServices: owned, + }); const runFixedCli = options.runFixedCli ?? createDefaultFixedCli(options.rootDir); - const releaseEffects = createDefaultReleaseEffects(options.rootDir, runFixedCli); + installServiceLifecycleRuntime(runtime, { + rootDir: options.rootDir, + infraDir: options.stack.infraDir, + runFixedCli, + }); + installServicePullRuntime(runtime, { + infraDir: options.stack.infraDir, + candidateSource: options.candidateSource, + loadAuthority: options.loadAuthority, + ownedServices: owned, + execFile: options.docker?.execFile, + }); + const releaseEffects = installReleaseCapabilityBoundary( + runtime, + options.rootDir, + options.stack.infraDir, + createDefaultReleaseEffects(options.rootDir, runFixedCli), + ); createTransitousLockRuntime(runtime, { rootDir: options.rootDir }); createAdministrativeRuntime(runtime, { rootDir: options.rootDir, @@ -117,6 +236,12 @@ export async function createProductionOpsRuntime( })); }, }); + // system.update's legacy handler performs backup/pull before apply. Refuse + // that entire branch through the same actual source boundary before effects. + runtime["system.update"] = async (_operation, context) => { + await releaseEffects.apply(_operation.releaseId, ["app-api"], context); + throw new OpsNotWiredError(); + }; runtime["services.inspect"] = async (_operation, context) => { const authority = serviceActionAuthority(await options.loadAuthority(), options.stack.infraDir); const snapshots = await owned.inspectOwnedServices(context.signal); @@ -160,7 +285,13 @@ export async function createProductionOpsRuntime( await options.loadAuthority(), options.stack.infraDir, ); - const snapshot = (await owned.inspectOwnedServices(context.signal)).find( + const frozen = + operation.serviceId === "data-manager" && + (operation.kind === "service.recreate" || operation.kind === "service.recreateIsolated") && + operation.maintenanceId + ? await assertOwnedMaintenanceRecreateRuntime(runtime, operation, context) + : undefined; + const snapshot = (frozen ?? (await owned.inspectOwnedServices(context.signal))).find( (s) => s.serviceId === operation.serviceId, ); const encoded = context.claim.capability.values.ownedService; @@ -179,8 +310,18 @@ export async function createProductionOpsRuntime( snapshot, action, ); - if (!decision.available || decision.execution !== "direct") - throw new Error("Service action denied"); + if ( + decision.available && + decision.execution === "workflow" && + operation.serviceId === "data-manager" && + (operation.kind === "service.recreate" || operation.kind === "service.recreateIsolated") + ) + await assertOwnedMaintenanceRecreateRuntime(runtime, operation, context); + else if (!decision.available || decision.execution !== "direct") { + if (operation.kind === "service.pull" && operation.serviceId === "data-manager") + await assertAdmittedCoordinatorPullRuntime(runtime, operation, context); + else throw new Error("Service action denied"); + } return handler(operation, context); }; const guarded = async ( @@ -193,6 +334,7 @@ export async function createProductionOpsRuntime( "service.restart", "service.recreate", "service.recreateIsolated", + "service.update", ].includes(kind) ) return execute(operation, context); @@ -200,11 +342,11 @@ export async function createProductionOpsRuntime( // dependencies. Conservatively exclude every registered service for those // commands; this local boundary does not acquire or replace root authority. const ids = - kind === "service.start" || kind === "service.recreate" + kind === "service.start" || kind === "service.recreate" || kind === "service.update" ? (await options.loadAuthority()).services.map((service) => service.manifest.id).sort() : [operation.serviceId]; return lifecycle.creating(ids, context.signal, async (begin) => { - if (kind === "service.start" || kind === "service.recreate") { + if (kind === "service.start" || kind === "service.recreate" || kind === "service.update") { const current = (await options.loadAuthority()).services .map((service) => service.manifest.id) .sort(); diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index fc3ae846c..d41ebf175 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -49,6 +49,7 @@ export class OpsAgentBusyError extends Error { } export interface OpsExecutionContext { + readonly configurationMaintenanceControl?: import("./configuration-maintenance-store").ConfigurationMaintenanceControlOwner; readonly execution: ExecutionRef; readonly artifactLease?: BorrowedArtifactTargetLease; readonly ownedTargetLease?: ArtifactTargetLease; diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index 0d5175b44..3e9126ffc 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -73,7 +73,11 @@ describe("ops-agent async foundation", () => { dispatch, }); const operationKey = "opk1_stableTransportRetry"; - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; const admitted = await app.inject({ method: "POST", url: "/v1/operations", @@ -98,7 +102,11 @@ describe("ops-agent async foundation", () => { url: "/v1/operations", headers: auth(), payload: request( - { kind: "service.pull", serviceId: "motis" }, + { + kind: "service.pull", + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "ops1_conflictRequest000", operationKey, ), @@ -157,6 +165,7 @@ describe("ops-agent async foundation", () => { "x-ops-operation-fingerprint": opsOperationFingerprint({ kind: "service.pull", serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, }), }, }); @@ -668,7 +677,11 @@ describe("ops-agent async foundation", () => { dispatch: async () => ({ changed: true }), }); const firstPayload = request( - { kind: "service.pull", serviceId: "redis" }, + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "ops1_retainedFirst0000", "opk1_retainedFirst0000", ); diff --git a/apps/ops-agent/src/server-review-2.test.ts b/apps/ops-agent/src/server-review-2.test.ts index 5b3262b63..33fc8eca1 100644 --- a/apps/ops-agent/src/server-review-2.test.ts +++ b/apps/ops-agent/src/server-review-2.test.ts @@ -67,6 +67,8 @@ describe("reviewed durable admission boundary", () => { let finish!: () => void; let rollbacks = 0; const operation = { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "stack.render" as const, revisionId: `cfg1_${"t".repeat(43)}`, @@ -118,6 +120,8 @@ describe("reviewed durable admission boundary", () => { let retained = true; let committed = false; const operation = { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "stack.render" as const, revisionId: `cfg1_${"r".repeat(43)}`, @@ -188,7 +192,11 @@ describe("reviewed durable admission boundary", () => { ); it("retains recovery-required tombstones beyond TTL and capacity to prevent silent rerun", async () => { - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; const tombstone = persisted(operation, { operationId: "job1_recoveryTombstone0", operationKey: "opk1_recoveryTombstone0", @@ -313,7 +321,11 @@ describe("reviewed durable admission boundary", () => { now: () => new Date(baseTime), dispatch, }); - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; const first = await app.inject({ method: "POST", url: "/v1/operations", @@ -349,7 +361,11 @@ describe("reviewed durable admission boundary", () => { now: () => new Date(baseTime), dispatch: async () => ({ changed: true }), }); - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; const admitted = await app.inject({ method: "POST", url: "/v1/operations", @@ -427,7 +443,11 @@ describe("reviewed durable admission boundary", () => { url: "/v1/operations", headers: auth(), payload: request( - { kind: "service.pull", serviceId: "redis" }, + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "ops1_interleaveFirst00", "opk1_interleaveFirst00", ), @@ -471,7 +491,11 @@ describe("reviewed durable admission boundary", () => { now: () => new Date(baseTime), dispatch, }); - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; const [first, second] = await Promise.all([ app.inject({ method: "POST", @@ -532,7 +556,11 @@ describe("reviewed durable admission boundary", () => { "failed", "timed_out", ] as const; - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; const records = states.map((state, index) => persisted(operation, { operationId: `job1_retainedState${index}0000`, @@ -577,7 +605,11 @@ describe("reviewed durable admission boundary", () => { roots.push(root); const path = join(root, "journal", "jobs-v1.json"); const before = await openOpsJobJournal(path); - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; await before.replace([persisted(operation)]); const journal = await openOpsJobJournal(path, { now: () => new Date(baseTime + 1_000), @@ -629,7 +661,11 @@ describe("reviewed durable admission boundary", () => { it("fails closed after bounded collisions with a restored operation ID", async () => { const bytes = Buffer.alloc(18, 7); const collisionId = `job1_${bytes.toString("base64url")}`; - const operation = { kind: "service.pull", serviceId: "redis" } as const; + const operation = { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + } as const; const existing = persisted(operation, { operationId: collisionId, state: "failed", diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index ec3c63e7a..8b42590c5 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -5,6 +5,7 @@ import { assertArtifactTargetLeaseForward, assertFreshControlRoot, beginControlEffect, + bindBorrowedControlExecution, type ControlInstallation, cancelArtifactTargetLease, cancelControlExecution, @@ -13,12 +14,15 @@ import { completeControlJournalExecution, controlInstallationDataRoot, controlRegisteredServiceIds, + inspectConfigurationPreEffectRefusal, inspectControlAdmission, inspectFeedProxyValidationFailure, isFixedSingleDataCommand, markFeedProxyReloadAttempted, planFixedSingleDataCommand, + publishConfigurationSettlementReservationProjection, controlHeaders as readControlHeaders, + reconcileConfigurationPreEffectRoot, reconcileControlRetention, registerControlRetentionJournal, releaseArtifactTargetLease, @@ -26,24 +30,33 @@ import { requireControlJournalRetention, reserveFeedProxyValidation, restoreAdmittedControlExecution, + restoreConfigurationPreEffectTerminal, retainAdmittedControlExecution, + retainConfigurationPreEffectRefusal, retainControlEffectReceipt, + retainControlPreparedTerminal, retainControlTerminalCommitment, retainControlTerminalStatus, retainedControlAdoptionAssertion, + retainFixedServiceLifecyclePlan, retainFixedSingleDataPlan, + retainGenerationHardlinksOwnerPlan, settleArtifactTargetLeaseChild, type TargetLeaseAuthority, type VerifiedExecutionAuthority, verifyControlEffectSettlement, + verifyRetainedArtifactTargetRootRelease, verifyRetainedArtifactTargetSettlement, } from "@openmapx/core/build-artifacts"; import { type ConfigurationApplyResult, + canonicalJson, configurationApplyResultSchema, + configurationMaintenanceReceiptSchema, controlTerminalReceiptDigest, type ExecutionRef, executionFingerprint, + isTrustedConfigurationApplyOperation, normalizeExecutionAdmission, OPS_KIND_POLICIES, OPS_MAX_CLOCK_SKEW_MS, @@ -63,6 +76,7 @@ import { type OpsOperationKind, type OpsResourcePolicy, type OpsRole, + opsExecutionBudget, opsFailureResultSchema, opsKindRequiresControlAuthority, opsOperationFingerprint, @@ -78,17 +92,55 @@ import { verifyApiJobParentExecution, } from "@openmapx/core/ops"; import Fastify, { type FastifyInstance } from "fastify"; +import { inspectConfigurationGenerationNonCommit } from "../../../packages/core/src/build-artifacts/control-authority"; +import { prepareConfigurationHardlinksAdmission } from "./configuration-hardlinks-runtime"; +import { + configurationMaintenanceAdmissionState, + prepareConfigurationMaintenanceAdmission, + prepareMaintenanceContinueObservation, +} from "./configuration-maintenance-runtime"; +import { + acknowledgeConfigurationMaintenanceControl, + acknowledgeConfigurationMaintenanceStep, + admitConfigurationMaintenanceControlOwner, + assertManagedCliConfigurationPreflight, + ConfigurationMaintenanceControlConflict, + ConfigurationMaintenanceControlUnauthorized, + configurationMaintenanceControlOwnerState, + inspectConfigurationMaintenanceDecisionControl, + inspectConfigurationMaintenanceSettleControl, + inspectConfigurationMaintenanceSettleRecovery, + normalizeConfigurationMaintenanceInspection, + normalizeConfigurationMaintenanceRecoveryControl, + prepareOwnedConfigurationMaintenanceStepAdmission, + readOwnedConfigurationMaintenance, + recordOwnedConfigurationMaintenanceStepTerminal, + registerConfigurationMaintenanceJournal, + restoreConfigurationMaintenanceControlJournal, + retainConfigurationMaintenanceDecisionControl, + retainConfigurationMaintenanceSettleControl, + retainOwnedConfigurationMaintenance, +} from "./configuration-maintenance-store"; +import { + installCoordinatorStagedOwnershipSource, + prepareCoordinatorStagedRetention, +} from "./coordinator-staged-ownership"; import { type DawarichSubjectExportRouteOptions, registerDawarichSubjectExportRoute, } from "./dawarich-subject-export"; -import type { OpsJobJournal, PersistedOpsJob } from "./journal"; +import type { HistoricalPullJob, OpsJobJournal, PersistedOpsJob } from "./journal"; +import { + type CoordinatorPullOwnershipObservation, + consumeCoordinatorPullOwnershipCapture, +} from "./owned-services"; import { createPolicyResourceClaimer, type OpsResourceClaimer } from "./policy"; import { type PrivacyBackupExtractionRouteOptions, registerPrivacyBackupExtractionRoute, } from "./privacy-backup-extraction"; import { safeProcessDiagnostic } from "./process-output"; +import { productionOwnedServices } from "./production-runtime"; import { createUnavailableRuntime, dispatchOpsOperation, @@ -100,6 +152,8 @@ import { type OpsRuntime, type OpsTrustedClaim, } from "./runtime"; +import { prepareServiceUpdateAdmission } from "./service-lifecycle-runtime"; +import { prepareCoordinatorPullBaseline } from "./service-pull-runtime"; import { type VerifiedTerminalClaimCleanup, @@ -325,7 +379,15 @@ function classify(error: unknown): { errorClass: OpsErrorClass; statusCode: numb return { errorClass: error.errorClass, statusCode: error.statusCode }; } if (error instanceof OpsExecutionFailure) - return { errorClass: error.errorClass, statusCode: error.errorClass === "timeout" ? 504 : 500 }; + return { + errorClass: error.errorClass, + statusCode: + error.errorClass === "timeout" + ? 504 + : error.errorClass === "configuration_prerequisite" + ? 409 + : 500, + }; if (error instanceof OpsHostUpdateRequiredError) return { errorClass: "host_update_required", statusCode: 409 }; if (error instanceof OpsAgentBusyError) return { errorClass: "busy", statusCode: 409 }; @@ -371,6 +433,40 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const journal: OpsJobJournal = options.journal ?? ({ records: () => [], replace: async () => undefined } satisfies OpsJobJournal); + const unregisterMaintenanceJournal = registerConfigurationMaintenanceJournal(runtime, journal); + app.addHook("onClose", async () => unregisterMaintenanceJournal()); + let literalProductionRuntime = false; + try { + productionOwnedServices(runtime); + literalProductionRuntime = true; + } catch { + /* Generic runtimes retain cache-only ownership. */ + } + if (literalProductionRuntime && options.controlInstallation && options.journal) + installCoordinatorStagedOwnershipSource(runtime, options.controlInstallation, journal); + const historicalPulls = journal.historicalPullRecords?.() ?? []; + const historicalStatus = (job: HistoricalPullJob, version: 1 | 2): OpsJobStatus => ({ + version: version === 2 && job.executionVersion === 2 ? 2 : 1, + ...(version === 2 && job.executionVersion === 2 + ? { + admittedAt: job.admittedAt, + deadlineAt: job.deadlineAt, + fingerprint: job.fingerprint, + ...(job.diagnostics ? { diagnostics: job.diagnostics } : {}), + ...(job.failureResult ? { failureResult: job.failureResult } : {}), + } + : {}), + operationId: job.operationId, + operationKey: job.operationKey, + kind: "service.pull", + resourceId: job.resourceId, + state: job.state, + submittedAt: job.submittedAt, + updatedAt: job.updatedAt, + ...(job.result === undefined ? {} : { result: job.result }), + ...(job.errorClass ? { errorClass: job.errorClass } : {}), + ...(job.terminationRequestedAt ? { terminationRequestedAt: job.terminationRequestedAt } : {}), + }); const resourceClaimer = options.resourceClaimer ?? createPolicyResourceClaimer(options.resourcePolicy ?? {}); const jobsById = new Map(); @@ -400,7 +496,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }; const restoredRecords = journal.records(); - if (restoredRecords.length > maxJobEntries) { + if (restoredRecords.length + historicalPulls.length > maxJobEntries) { throw new Error("Ops job journal exceeds the configured server capacity"); } for (const persisted of restoredRecords) { @@ -440,10 +536,15 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const controlLeases = new Map(); const controlAuthorities = new Map(); + const maintenanceControls = new Map< + string, + import("./configuration-maintenance-store").ConfigurationMaintenanceControlOwner + >(); app.addHook("onClose", async () => { for (const lease of controlLeases.values()) closeArtifactTargetLeaseHandle(lease); controlLeases.clear(); controlAuthorities.clear(); + maintenanceControls.clear(); }); const persisted = (job: JobRecord): PersistedOpsJob => ({ ...(job.controlAdoption ? { controlAdoption: job.controlAdoption } : {}), @@ -573,15 +674,83 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ): Promise => { await withAdmissionLock(async () => { if (state === "termination_pending" && job.terminalAt !== undefined) return; - const updatedAt = now().toISOString(); + let updatedAt = now().toISOString(); + const maintenanceControl = maintenanceControls.get(job.operationId); + if ( + maintenanceControl && + job.operation.kind === "configurationMaintenance.decide" && + ["succeeded", "failed", "timed_out"].includes(state) + ) { + const accepted = inspectConfigurationMaintenanceDecisionControl( + maintenanceControl, + job.operation, + executionFor(job), + ); + state = "succeeded"; + updatedAt = accepted.status.updatedAt; + updates = { + result: accepted.receipt, + terminalAt: accepted.terminalAt, + errorClass: undefined, + failureResult: undefined, + terminationRequestedAt: undefined, + }; + } + if ( + maintenanceControl && + job.operation.kind === "configurationMaintenance.settle" && + ["succeeded", "failed", "timed_out"].includes(state) + ) { + try { + const accepted = inspectConfigurationMaintenanceSettleControl( + maintenanceControl, + job.operation, + executionFor(job), + ); + state = "succeeded"; + updatedAt = accepted.status.updatedAt; + updates = { + result: accepted.receipt, + terminalAt: accepted.terminalAt, + errorClass: undefined, + failureResult: undefined, + terminationRequestedAt: undefined, + }; + } catch (error) { + if (state === "succeeded") throw error; + const unresolved = inspectConfigurationMaintenanceSettleRecovery( + maintenanceControl, + job.operation, + executionFor(job), + ); + state = "failed"; + updatedAt = unresolved.updatedAt; + updates = { + result: undefined, + terminalAt: undefined, + errorClass: "recovery_required", + failureResult: undefined, + terminationRequestedAt: undefined, + }; + } + } const lease = controlLeases.get(job.operationId); const authority = controlAuthorities.get(job.operationId); + let releaseUnstartedRoot = false; if (authority && state === "termination_pending") cancelControlExecution(authority); if (lease && state === "termination_pending") await cancelArtifactTargetLease(lease); if (authority && ["succeeded", "failed", "timed_out"].includes(state)) { if (!options.controlInstallation) throw Error("Missing control settlement installation"); // The fixed runtime's validated result is own-effect evidence. Failure alone is not containment. const committed = state === "succeeded" && updates.result !== undefined; + const preparedMaintenance = + committed && job.operation.kind === "configurationMaintenance.prepare"; + if ( + preparedMaintenance && + canonicalJson(readOwnedConfigurationMaintenance(authority).receipt) !== + canonicalJson(updates.result) + ) + throw Error("Actual original prepared owner receipt required"); let validationFailure: string | null = null; if (state === "failed" && job.operation.kind === "feedProxy.validateAndReload") { try { @@ -590,21 +759,67 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif /* Missing/corrupt producer evidence stays unknown. */ } } - retainControlEffectReceipt(authority, { - outcome: committed ? "committed" : validationFailure ? "contained" : "unknown", - publication: committed ? "committed" : validationFailure ? "not_started" : "unknown", - receipt: committed - ? { kind: "ops-result", result: updates.result } - : validationFailure - ? { kind: "feed-proxy-validation-failure", commitmentDigest: validationFailure } - : { kind: "unresolved" }, - }); - if (lease) { + let originalGenerationNonCommit = false; + let unstartedConfiguration = false; + if (state === "failed") { + try { + inspectConfigurationPreEffectRefusal(authority); + unstartedConfiguration = true; + } catch { + /* Only the actual fixed pre-effect source qualifies. */ + } + } + if ( + state === "succeeded" && + isTrustedConfigurationApplyOperation(job.operation) && + updates.result && + (updates.result as { outcome?: string }).outcome === "not_committed" + ) { + const actual = inspectConfigurationGenerationNonCommit( + options.controlInstallation, + authority.execution, + ); + if (canonicalJson(actual.result) !== canonicalJson(updates.result)) + throw Error("Original writer noncommit result changed"); + originalGenerationNonCommit = true; + } + if (!originalGenerationNonCommit && !unstartedConfiguration) + retainControlEffectReceipt(authority, { + outcome: committed ? "committed" : validationFailure ? "contained" : "unknown", + publication: committed ? "committed" : validationFailure ? "not_started" : "unknown", + receipt: committed + ? { kind: "ops-result", result: updates.result } + : validationFailure + ? { kind: "feed-proxy-validation-failure", commitmentDigest: validationFailure } + : { kind: "unresolved" }, + }); + if (lease && job.operation.kind === "configurationMaintenance.prepare" && !committed) { + try { + // Only the complete disjoint source gate permits a failed prepare + // to complete/release. Prepared or ambiguous guards stay fenced. + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(options.controlInstallation, authority.execution), + ); + if (lease.mode === "owner") { + await releaseArtifactTargetLease(lease); + verifyRetainedArtifactTargetRootRelease(authority); + // Classify only this freshly proved, fully released no-guard + // refusal before the immutable terminal is first committed. + // An unknown/partial release never takes this branch. + if (state === "failed" && updates.errorClass === "recovery_required") + updates = { ...updates, errorClass: "busy" }; + } + } catch { + /* Original retained owner/recovery path remains mandatory. */ + } + } else if (lease && job.operation.kind !== "configurationMaintenance.prepare") { await settleArtifactTargetLeaseChild( lease, verifyControlEffectSettlement(options.controlInstallation, authority.execution), ); if (lease.mode === "owner" && committed) await releaseArtifactTargetLease(lease); + if (lease.mode === "owner" && unstartedConfiguration) releaseUnstartedRoot = true; } } const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; @@ -617,9 +832,19 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ); updates = { ...updates, terminalAt: candidate.terminalAt }; } + if (releaseUnstartedRoot && lease?.mode === "owner" && authority) { + await releaseArtifactTargetLease(lease); + verifyRetainedArtifactTargetRootRelease(authority); + } await journal.replace( [...jobsById.values()].map((entry) => persisted(entry === job ? candidate : entry)), ); + if ( + authority && + state === "succeeded" && + job.operation.kind === "configurationMaintenance.prepare" + ) + retainControlPreparedTerminal(authority); Object.assign(job, updates); job.updatedAt = updatedAt; job.state = state; @@ -663,6 +888,24 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif // Compatibility records never resume a forward execution or gain a new budget. deadlineAt: job.deadlineAt ?? Date.parse(job.submittedAt), }); + /** The secondary UNKNOWN projection may be repaired only from the actual + * original owner's completed canonical source, never a runtime retry. */ + const refreshOriginalMaintenanceControl = async (job: JobRecord): Promise => { + if ( + job.operation.kind !== "configurationMaintenance.settle" || + job.errorClass !== "recovery_required" || + job.terminalAt !== undefined + ) + return; + const control = maintenanceControls.get(job.operationId); + if (!control) return; + try { + inspectConfigurationMaintenanceSettleControl(control, job.operation, executionFor(job)); + } catch { + return; + } + await transition(job, "succeeded"); + }; function borrowedLeaseFor(id: string) { const lease = controlLeases.get(id); return lease?.mode === "borrowed" ? lease : undefined; @@ -682,6 +925,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif artifactLease: borrowedLeaseFor(job.operationId), ownedTargetLease: ownedLeaseFor(job.operationId), controlAuthority: controlAuthorities.get(job.operationId), + ...(maintenanceControls.has(job.operationId) + ? { configurationMaintenanceControl: maintenanceControls.get(job.operationId) } + : {}), ...(job.operation.kind === "feedProxy.validateAndReload" ? { async commitFeedProxyValidationFailure(input: { @@ -798,6 +1044,23 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const unregisterRetention = options.controlInstallation ? registerControlRetentionJournal(options.controlInstallation, { + ...(literalProductionRuntime && options.journal + ? { + prepare: async ( + _executions: readonly { execution: ExecutionRef; digest?: string }[], + context?: import("@openmapx/core/build-artifacts").ControlRetentionPreparationContext, + ) => + withAdmissionLock(async () => { + if (!context) throw Error("Actual collector preparation required"); + await prepareCoordinatorStagedRetention( + context, + runtime, + options.controlInstallation!, + journal, + ); + }), + } + : {}), contained: (execution, digest) => { const job = jobsById.get(execution.executionId); return ( @@ -823,6 +1086,51 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw Error("Agent terminal journal changed during retirement"); remove.add(job.operationId); } + // These two metadata controls deliberately have no ordinary + // execution/family node. Retire their secondary projections only + // with the original root's committed all-role retirement. Private + // owner cleanup has already happened at this point; startup must + // continue to reject an orphan outside this cleanup capability. + for (const job of jobsById.values()) { + const body = job.operation; + if ( + body.kind !== "configurationMaintenance.decide" && + body.kind !== "configurationMaintenance.settle" + ) + continue; + const original = executions.find( + ({ execution }) => execution.operationKey === body.prepareOperationKey, + ); + if (!original) continue; + const receipt = configurationMaintenanceReceiptSchema.parse(job.result), + parent = job.parent; + if ( + activeControllers.has(job.operationId) || + job.state !== "succeeded" || + job.terminalAt === undefined || + job.role !== "api" || + job.executionVersion !== 2 || + parent?.issuer !== "api" || + canonicalJson(receipt.prepare) !== canonicalJson(original.execution) || + original.execution.operationFingerprint !== body.prepareFingerprint || + receipt.binding.maintenanceId !== body.maintenanceId || + receipt.binding.candidateId !== body.candidateId || + receipt.planDigest !== body.planDigest || + receipt.decision?.decisionId !== body.decisionId || + receipt.apiJobId !== parent.apiJobId || + receipt.parentOperationKey !== parent.operationKey || + receipt.parentFingerprint !== parent.fingerprint || + receipt.parentDeadlineAt !== parent.deadlineAt || + (body.kind === "configurationMaintenance.decide" && + (receipt.decision.decision !== body.decision || + receipt.decision.checkpointVersion !== body.checkpointVersion || + receipt.decision.checkpointDigest !== body.checkpointDigest)) || + (body.kind === "configurationMaintenance.settle" && + (receipt.state !== "settled" || receipt.dmFence?.state !== "settled")) + ) + throw Error("Agent maintenance control journal changed during retirement"); + remove.add(job.operationId); + } await journal.replace( [...jobsById.values()].filter((job) => !remove.has(job.operationId)).map(persisted), ); @@ -870,7 +1178,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ); }, timeoutMs); const settlement = Promise.resolve() - .then(() => { + .then(async () => { if ((job.deadlineAt ?? 0) <= now().getTime()) { timedOut = true; throw new OpsExecutionFailure("timeout", {}); @@ -879,12 +1187,37 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (authority) { const lease = controlLeases.get(job.operationId); if (lease) assertArtifactTargetLeaseForward(lease); + if (literalProductionRuntime) { + try { + await assertManagedCliConfigurationPreflight( + runtime, + job.operation, + contextFor(job, controller.signal, claim), + ); + } catch (error) { + if ( + error instanceof OpsExecutionFailure && + error.errorClass === "configuration_prerequisite" + ) + retainConfigurationPreEffectRefusal(authority); + throw error; + } + } if (job.operation.kind === "feedProxy.validateAndReload") { const digest = claim.capability.values.candidateDigest; if (typeof digest !== "string") throw Error("Original candidate claim missing"); reserveFeedProxyValidation(authority, job.operation.candidateId, digest); } - beginControlEffect(authority); + // The fixed DM pull runtime performs its source/target-index checks + // under writer -> release lock, then begins immediately before Docker. + if ( + !( + job.operation.kind === "service.pull" && + job.operation.serviceId === "data-manager" && + job.operation.candidateId + ) + ) + beginControlEffect(authority); } return dispatch(job.operation, runtime, contextFor(job, controller.signal, claim)); }) @@ -892,7 +1225,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif async (rawResult) => { clearTimeout(timer); try { - if ("revisionId" in job.operation) { + if (isTrustedConfigurationApplyOperation(job.operation)) { // Persist commit evidence before releasing confidential claim data. // Public status remains running until cleanup evidence is settled. const result = configurationApplyResultSchema.parse( @@ -1062,6 +1395,33 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ); if (terminalDurable && claimReleased && authority) completeControlJournalExecution(authority); + if ( + terminalDurable && + claimReleased && + authority && + options.controlInstallation && + job.terminalAt !== undefined && + job.state === "succeeded" && + (isTrustedConfigurationApplyOperation(job.operation) || + job.operation.kind === "configuration.hardlinks.apply" || + job.operation.kind === "service.recreate" || + job.operation.kind === "service.recreateIsolated") && + job.operation.maintenanceId + ) { + try { + recordOwnedConfigurationMaintenanceStepTerminal( + options.controlInstallation, + job.operation, + executionFor(job), + statusFor(job, 2), + job.terminalAt, + ); + } catch { + stickyRecoveryRequired.add( + job.operationId, + ); /* Exact ACK publication may repair this committed source; no target replay. */ + } + } activeControllers.delete(job.operationId); active -= 1; } @@ -1206,6 +1566,80 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif // retained own-effect proof; absent containment evidence keeps its fence. app.addHook("onReady", async () => { let changed = false; + if (options.controlInstallation) { + const accepted = restoreConfigurationMaintenanceControlJournal(options.controlInstallation), + next = new Map(jobsById); + const acceptedKeys = new Set(accepted.map((source) => source.operationKey)); + for (const job of jobsById.values()) + if ( + (job.operation.kind === "configurationMaintenance.decide" || + job.operation.kind === "configurationMaintenance.settle") && + !acceptedKeys.has(job.operationKey) + ) + throw Error("Original maintenance control private source unavailable"); + for (const source of accepted) { + const prior = jobsByKey.get(`api:${source.operationKey}`), + byId = next.get(source.operationId); + if (byId && byId !== prior) throw Error("Original maintenance control execution conflict"); + if (prior) { + const identity = (record: PersistedOpsJob) => { + const { + state: _state, + result: _result, + updatedAt: _updated, + terminalAt: _terminal, + errorClass: _error, + failureResult: _failure, + terminationRequestedAt: _cancel, + diagnostics: _diagnostics, + ...binding + } = record; + return binding; + }; + // Only the authenticated original owner can supply this unresolved + // projection. The secondary journal's interruption timestamp is not a + // canonical completion and cannot prevent read-only recovery. + const unresolvedSettle = + source.operation.kind === "configurationMaintenance.settle" && + source.errorClass === "recovery_required" && + source.terminalAt === undefined; + const priorUnresolvedSettle = + source.operation.kind === "configurationMaintenance.settle" && + prior.state === "failed" && + prior.errorClass === "recovery_required" && + prior.result === undefined; + if ( + canonicalJson(identity(persisted(prior))) !== canonicalJson(identity(source)) || + (!unresolvedSettle && + !priorUnresolvedSettle && + prior.terminalAt !== undefined && + canonicalJson(persisted(prior)) !== canonicalJson(source)) + ) + throw Error("Original maintenance control canonical conflict"); + if (canonicalJson(persisted(prior)) === canonicalJson(source)) continue; + } else if (next.size >= maxJobEntries) + throw Error("Original maintenance control recovery capacity exceeded"); + const restored: JobRecord = { + ...source, + terminalAt: source.terminalAt === undefined ? undefined : Date.parse(source.terminalAt), + cursor: prior?.cursor ?? 0, + events: prior?.events ?? [], + eventBytes: prior?.eventBytes ?? 0, + eventsTruncated: true, + }; + next.set(restored.operationId, restored); + changed = true; + } + // Publish the exact canonical secondary projection before making it + // visible to HTTP lookup. Failure leaves the original private source. + if (changed) { + await journal.replace([...next.values()].map(persisted)); + for (const [id, restored] of next) { + jobsById.set(id, restored); + jobsByKey.set(`${restored.role}:${restored.operationKey}`, restored); + } + } + } for (const job of jobsById.values()) { if (job.controlAdoption) { try { @@ -1255,6 +1689,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } if ( job.executionVersion === 2 && + job.operation.kind !== "configurationMaintenance.decide" && + job.operation.kind !== "configurationMaintenance.settle" && opsKindRequiresControlAuthority(job.operation.kind) && ["succeeded", "failed", "timed_out"].includes(job.state) ) { @@ -1262,7 +1698,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (!options.controlInstallation || job.terminalAt === undefined) throw Error("Original terminal authority unavailable"); if ( - "revisionId" in job.operation && + isTrustedConfigurationApplyOperation(job.operation) && !verifiesTerminalClaimCleanup( options.terminalClaimCleanup, job.operation, @@ -1271,6 +1707,51 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ) throw Error("Terminal claim cleanup is not proved"); const execution = executionFor(job); + if ( + literalProductionRuntime && + job.state === "failed" && + ["configuration_prerequisite", "busy", "recovery_required"].includes( + job.errorClass ?? "", + ) && + (job.operation.kind === "service.update" || + job.operation.kind === "stack.start" || + (job.operation.kind === "service.start" && + job.operation.serviceId !== "data-manager") || + (job.operation.kind === "service.pull" && + job.operation.serviceId === "data-manager" && + job.operation.candidateId)) + ) { + const recovered = restoreConfigurationPreEffectTerminal( + options.controlInstallation, + execution, + statusFor(job, 2), + ), + authority = repairControlJournalTerminal( + options.controlInstallation, + execution, + recovered.status, + recovered.terminalAt, + ), + restored: JobRecord = { + ...job, + state: "failed", + result: undefined, + errorClass: recovered.status.errorClass, + failureResult: recovered.status.failureResult, + terminationRequestedAt: recovered.status.terminationRequestedAt, + terminalAt: recovered.terminalAt, + updatedAt: recovered.status.updatedAt, + }; + // Retain the exact original body before finishing its physical + // release. A failed/unknown write cannot create a fresh attempt. + await journal.replace( + [...jobsById.values()].map((value) => persisted(value === job ? restored : value)), + ); + await reconcileConfigurationPreEffectRoot(authority); + Object.assign(job, restored); + controlAuthorities.set(job.operationId, authority); + continue; + } if (job.controlAdoption) verifyRetainedArtifactTargetSettlement( options.controlInstallation, @@ -1312,6 +1793,21 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const timing = validateTiming(parsed.issuedAt, parsed.expiresAt, nowMs); replay.accept(parsed.requestId, timing.expiresAtMs, nowMs); const policy = OPS_KIND_POLICIES[operation.kind]; + // Read-only historical records can neither reserve a fresh normal key + // nor turn an unresolved old effect into permission for another writer. + if ( + historicalPulls.some((job) => job.role === role && job.operationKey === parsed.operationKey) + ) + throw new RequestGuardError("conflict", 409); + if ( + opsKindRequiresControlAuthority(operation.kind) && + historicalPulls.some( + (job) => + ["queued", "running", "termination_pending"].includes(job.state) || + job.errorClass === "recovery_required", + ) + ) + throw new RequestGuardError("recovery_required", 409); const operationFingerprint = opsOperationFingerprint(operation); if (operation.kind === "controlRetention.acknowledge") { if (role !== "api" || !options.controlInstallation) @@ -1323,11 +1819,40 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif controlTerminalReceiptDigest(statusFor(existing, 2)) !== operation.terminalDigest ) throw Error("Current terminal digest conflict"); - const value = acknowledgeControlRetention( - options.controlInstallation, - operation, - options.tokens.api, - ); + const phase = + existing && + (isTrustedConfigurationApplyOperation(existing.operation) || + existing.operation.kind === "configuration.hardlinks.apply" || + existing.operation.kind === "service.recreate" || + existing.operation.kind === "service.recreateIsolated") && + existing.operation.maintenanceId; + if (phase && existing.terminalAt !== undefined) + recordOwnedConfigurationMaintenanceStepTerminal( + options.controlInstallation, + existing.operation, + executionFor(existing), + statusFor(existing, 2), + existing.terminalAt, + ); + const value = phase + ? acknowledgeConfigurationMaintenanceStep( + options.controlInstallation, + operation, + options.tokens.api, + ) + : operation.originRootKey !== operation.operationKey && + (existing?.operation.kind === "configurationMaintenance.decide" || + existing?.operation.kind === "configurationMaintenance.settle") + ? acknowledgeConfigurationMaintenanceControl( + options.controlInstallation, + operation, + options.tokens.api, + ) + : acknowledgeControlRetention( + options.controlInstallation, + operation, + options.tokens.api, + ); return reply .code(200) .send(opsSuccess(parsed.requestId, { execution: "sync", kind: operation.kind, value })); @@ -1335,8 +1860,23 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("recovery_required", 409); } } + if ( + operation.kind === "service.pull" && + operation.serviceId === "data-manager" && + !operation.candidateId + ) + throw new OpsExecutionFailure("configuration_prerequisite", {}); let parent: VerifiedParentExecution | undefined; - let controlAdmission: ReturnType | undefined; + let configurationMaintenanceControl: + | import("./configuration-maintenance-store").ConfigurationMaintenanceControlOwner + | undefined; + let maintenanceStepAdmission: + | Awaited> + | undefined; + let controlAdmission: + | ReturnType + | Awaited> + | undefined; const parentHeader = request.headers["x-openmapx-parent-execution"]; const delegationHeader = request.headers["x-openmapx-control-delegation"]; // Approval is exclusively a borrowed activation leaf. Installing its @@ -1386,6 +1926,69 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("authorization", 403); } } + if ( + operation.kind === "configurationMaintenance.inspect" || + operation.kind === "configurationMaintenance.decide" || + operation.kind === "configurationMaintenance.settle" + ) { + if (!options.controlInstallation || !parent) + throw new RequestGuardError("authorization", 403); + try { + configurationMaintenanceControl = admitConfigurationMaintenanceControlOwner( + options.controlInstallation, + operation, + parent, + { + operationKey: parsed.operationKey, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + }, + ); + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + throw new RequestGuardError( + (error as NodeJS.ErrnoException).code === "ENOENT" ? "not_found" : "recovery_required", + (error as NodeJS.ErrnoException).code === "ENOENT" ? 404 : 409, + ); + } + } + if ( + (isTrustedConfigurationApplyOperation(operation) || + operation.kind === "configuration.hardlinks.apply" || + operation.kind === "service.recreate" || + operation.kind === "service.recreateIsolated") && + operation.maintenanceId + ) { + if ( + !options.controlInstallation || + role !== "api" || + !parent || + typeof parentHeader !== "string" + ) + throw new RequestGuardError("authorization", 403); + try { + maintenanceStepAdmission = await prepareOwnedConfigurationMaintenanceStepAdmission( + runtime, + options.controlInstallation, + operation, + parent, + { + operationKey: parsed.operationKey, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + parentAssertion: parentHeader, + signal: AbortSignal.timeout(15000), + }, + ); + controlAdmission = maintenanceStepAdmission; + parent = maintenanceStepAdmission.parent; + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + if (error instanceof ConfigurationMaintenanceControlConflict) + throw new RequestGuardError("recovery_required", 409); + throw error; + } + } const normalize = ( originalAdmittedAt: number, deploymentBudgetMs = Math.min( @@ -1394,6 +1997,36 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ), ) => { try { + if (maintenanceStepAdmission) + return { + admittedAt: maintenanceStepAdmission.boundAt, + deadlineAt: maintenanceStepAdmission.execution.deadlineAt, + fingerprint: maintenanceStepAdmission.execution.fingerprint, + }; + if ( + operation.kind === "configurationMaintenance.inspect" && + configurationMaintenanceControl + ) + return normalizeConfigurationMaintenanceInspection( + configurationMaintenanceControl, + operation, + parsed.operationKey, + originalAdmittedAt, + deploymentBudgetMs, + ); + if ( + configurationMaintenanceControl && + (operation.kind === "configurationMaintenance.settle" || + (operation.kind === "configurationMaintenance.decide" && + operation.decision === "abandon")) + ) + return normalizeConfigurationMaintenanceRecoveryControl( + configurationMaintenanceControl, + operation, + parsed.operationKey, + originalAdmittedAt, + deploymentBudgetMs, + ); return normalizeExecutionAdmission({ kind, operationKey: parsed.operationKey, @@ -1436,7 +2069,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("recovery_required", 409); } } - if (jobsById.size >= maxJobEntries) { + if (jobsById.size + historicalPulls.length >= maxJobEntries) { throw new RequestGuardError("busy", 429); } // A resumed agent update re-attaches above; nothing new may start. @@ -1446,7 +2079,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } if (options.releaseRecovery?.ready === false && !RELEASE_RECOVERY_READ_KINDS.has(kind)) throw new RequestGuardError("recovery_required", 409); - if (opsKindRequiresControlAuthority(operation.kind) && !controlAdmission) { + if ( + opsKindRequiresControlAuthority(operation.kind) && + !controlAdmission && + !configurationMaintenanceControl + ) { if (!options.controlInstallation) throw new RequestGuardError("recovery_required", 409); try { const key = parseControlRootOperationKey(parsed.operationKey); @@ -1472,6 +2109,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif resourceClaimer.claim(operation, fingerprint, claimController.signal, { role, operationKey: parsed.operationKey, + ...(maintenanceStepAdmission ? { maintenanceStep: maintenanceStepAdmission } : {}), }), ); let claimTimer: ReturnType | undefined; @@ -1508,6 +2146,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw error; } let claimAdmission: OpsTrustedClaim["admission"]; + let coordinatorCapture: CoordinatorPullOwnershipObservation | undefined; try { if ( !claim || @@ -1529,6 +2168,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ) { throw new OpsContractError("authorization", "Operation resource is not permitted"); } + if (operation.kind === "service.pull" && operation.candidateId) + coordinatorCapture = consumeCoordinatorPullOwnershipCapture(claim, { + operationFingerprint: fingerprint, + candidateId: operation.candidateId, + policyRevision: claim.capability.revisionId, + }); claimAdmission = claim.admission; claim = immutableSnapshot({ fingerprint: claim.fingerprint, @@ -1552,7 +2197,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } return { type: "retained", job: retained } as const; } - if (jobsById.size >= maxJobEntries) throw new RequestGuardError("busy", 429); + if (jobsById.size + historicalPulls.length >= maxJobEntries) + throw new RequestGuardError("busy", 429); if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); const timestamp = new Date(normalized.admittedAt).toISOString(); let operationId: string | undefined = controlAdmission?.execution.executionId; @@ -1590,10 +2236,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif operationFingerprint, requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, parent, - deploymentBudgetMs: Math.min( - policy.timeoutMs, - options.operationTimeoutMs ?? policy.timeoutMs, - ), + deploymentBudgetMs: maintenanceStepAdmission + ? opsExecutionBudget(operation.kind).processMs + : Math.min(policy.timeoutMs, options.operationTimeoutMs ?? policy.timeoutMs), resourceId, state: "queued", submittedAt: timestamp, @@ -1603,17 +2248,151 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif eventBytes: 0, eventsTruncated: false, }; + if ( + configurationMaintenanceControl && + operation.kind === "configurationMaintenance.decide" + ) { + try { + if ( + job.admittedAt === undefined || + job.requestedDeadlineAt === undefined || + typeof parentHeader !== "string" + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual decision request source required", + ); + const continueObservation = + operation.decision === "continue" + ? await prepareMaintenanceContinueObservation( + runtime, + configurationMaintenanceControlOwnerState( + configurationMaintenanceControl, + operation, + job.operationKey, + ).authority, + AbortSignal.any([ + claimController.signal, + AbortSignal.timeout(Math.max(1, job.deadlineAt! - Date.now())), + ]), + ) + : undefined; + retainConfigurationMaintenanceDecisionControl( + configurationMaintenanceControl, + operation, + executionFor(job), + { + admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, + parentAssertion: parentHeader, + requestVersion: job.requestVersion!, + deploymentBudgetMs: job.deploymentBudgetMs!, + }, + continueObservation, + ); + maintenanceControls.set(job.operationId, configurationMaintenanceControl); + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + if (error instanceof ConfigurationMaintenanceControlConflict) + throw new RequestGuardError("conflict", 409); + throw error; + } + } + if ( + configurationMaintenanceControl && + operation.kind === "configurationMaintenance.settle" + ) { + try { + if ( + job.admittedAt === undefined || + job.requestedDeadlineAt === undefined || + typeof parentHeader !== "string" + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual settlement request source required", + ); + retainConfigurationMaintenanceSettleControl( + configurationMaintenanceControl, + operation, + executionFor(job), + { + admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, + parentAssertion: parentHeader, + requestVersion: job.requestVersion!, + deploymentBudgetMs: job.deploymentBudgetMs!, + }, + ); + maintenanceControls.set(job.operationId, configurationMaintenanceControl); + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + if (error instanceof ConfigurationMaintenanceControlConflict) + throw new RequestGuardError("conflict", 409); + throw error; + } + } + let coordinatorBaseline: + | Awaited> + | undefined; + if (job.operation.kind === "service.pull" && job.operation.candidateId) { + if (!options.controlInstallation || !coordinatorCapture) + throw new RequestGuardError("authorization", 403); + try { + coordinatorBaseline = await prepareCoordinatorPullBaseline( + runtime, + options.controlInstallation, + { + operation: job.operation, + execution: executionFor(job), + parent: job.parent, + requestedDeadlineAt: job.requestedDeadlineAt, + }, + coordinatorCapture, + ); + } catch { + throw new RequestGuardError("authorization", 403); + } + } await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); jobsById.set(job.operationId, job); jobsByKey.set(ownerKey, job); - if (!controlAdmission && opsKindRequiresControlAuthority(operation.kind)) { + if ( + !controlAdmission && + !configurationMaintenanceControl && + opsKindRequiresControlAuthority(operation.kind) + ) { try { if (!options.controlInstallation || job.admittedAt === undefined) throw Error("Control trust unavailable"); + const maintenanceAdmission = + job.operation.kind === "configurationMaintenance.prepare" + ? await prepareConfigurationMaintenanceAdmission(runtime, { + installation: options.controlInstallation, + execution: executionFor(job), + operation: job.operation, + apiParent: + job.parent ?? + (() => { + throw Error("Original retained API maintenance parent required"); + })(), + signal: claimController.signal, + }) + : undefined; + const maintenancePlan = + maintenanceAdmission && + configurationMaintenanceAdmissionState(maintenanceAdmission).plan; const targets = - "serviceId" in operation + maintenancePlan?.serviceIds ?? + ("serviceId" in operation && + ![ + "service.start", + "service.recreate", + "service.recreateIsolated", + "service.update", + ].includes(operation.kind) ? [operation.serviceId] - : controlRegisteredServiceIds(options.controlInstallation); + : controlRegisteredServiceIds(options.controlInstallation)); const fixedDataPlan = isFixedSingleDataCommand(job.operation.kind) ? planFixedSingleDataCommand( executionFor(job), @@ -1622,6 +2401,24 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif process.env.OPENMAPX_REGION, ) : undefined; + const hardlinksPlan = + job.operation.kind === "configuration.hardlinks.apply" + ? prepareConfigurationHardlinksAdmission( + runtime, + executionFor(job), + job.operation, + targets, + ) + : undefined; + const lifecyclePlan = + job.operation.kind === "service.update" + ? prepareServiceUpdateAdmission( + runtime, + executionFor(job), + job.operation, + targets, + ) + : undefined; const authority = retainAdmittedControlExecution(options.controlInstallation, { execution: executionFor(job), operation: job.operation, @@ -1631,21 +2428,55 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif parent: job.parent, serviceIds: targets, ...(fixedDataPlan ? { reservations: [fixedDataPlan.rootReservation] } : {}), + ...(hardlinksPlan + ? { reservations: hardlinksPlan.reservation ? [hardlinksPlan.reservation] : [] } + : {}), + ...(lifecyclePlan ? { reservations: [lifecyclePlan.rootReservation] } : {}), + ...(maintenancePlan ? { reservations: maintenancePlan.reservations } : {}), + ...(coordinatorBaseline ? { coordinatorPullBaseline: coordinatorBaseline } : {}), }); if (fixedDataPlan) retainFixedSingleDataPlan(authority, fixedDataPlan); + if (hardlinksPlan) retainGenerationHardlinksOwnerPlan(authority, hardlinksPlan); + if (lifecyclePlan) retainFixedServiceLifecyclePlan(authority, lifecyclePlan); requireControlJournalRetention(authority); controlAuthorities.set(job.operationId, authority); if ( - ["valhalla.traffic.maintain", "valhalla.traffic.refreshWaysToEdges"].includes( - job.operation.kind, - ) + [ + "configurationMaintenance.prepare", + "stack.start", + "service.start", + "service.recreate", + "service.recreateIsolated", + "service.update", + "configuration.hardlinks.apply", + "valhalla.traffic.maintain", + "valhalla.traffic.refreshWaysToEdges", + ].includes(job.operation.kind) ) { const lease = await acquireArtifactTargetLease({ dataRoot: controlInstallationDataRoot(options.controlInstallation), - serviceIds: ["valhalla"], + serviceIds: + maintenancePlan || + hardlinksPlan || + lifecyclePlan || + [ + "stack.start", + "service.start", + "service.recreate", + "service.recreateIsolated", + ].includes(job.operation.kind) + ? targets + : ["valhalla"], owner: authority, }); controlLeases.set(job.operationId, lease); + if (maintenanceAdmission) { + retainOwnedConfigurationMaintenance(authority, lease, maintenanceAdmission); + publishConfigurationSettlementReservationProjection( + authority, + maintenancePlan!.operation.maintenanceId, + ); + } } } catch (error) { forceRecoveryRequired(job); @@ -1667,9 +2498,27 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif kind: job.operation.kind, operation: job.operation, admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, parent: job.parent, serviceIds: lease.serviceIds, + ...(maintenanceStepAdmission + ? { reservations: maintenanceStepAdmission.reservations } + : {}), }); + if (maintenanceStepAdmission) bindBorrowedControlExecution(lease, authority); + if ( + maintenanceStepAdmission && + job.operation.kind === "configuration.hardlinks.apply" + ) + retainGenerationHardlinksOwnerPlan( + authority, + prepareConfigurationHardlinksAdmission( + runtime, + executionFor(job), + job.operation, + lease.serviceIds, + ), + ); requireControlJournalRetention(authority); controlLeases.set(job.operationId, lease); controlAuthorities.set(job.operationId, authority); @@ -1737,6 +2586,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); return dispatch(operation, runtime, { signal: controller.signal, + ...(configurationMaintenanceControl ? { configurationMaintenanceControl } : {}), execution: Object.freeze({ executor: "ops-agent", principal: role, @@ -1813,7 +2663,26 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } await pruneJobs(now().getTime()); const job = jobsByKey.get(`${role}:${operationKey}`); - if (!job) return reply.code(404).send(redactedOpsError(requestId, "not_found")); + if (!job) { + const legacy = historicalPulls.find( + (value) => value.role === role && value.operationKey === operationKey, + ); + if (!legacy) return reply.code(404).send(redactedOpsError(requestId, "not_found")); + const fingerprint = request.headers["x-ops-operation-fingerprint"], + body = request.headers["x-ops-body-fingerprint"], + requested = request.headers["x-ops-requested-deadline"]; + if ( + (body !== undefined && body !== (legacy.operationFingerprint ?? legacy.fingerprint)) || + ((body === undefined || fingerprint !== undefined) && + fingerprint !== legacy.fingerprint && + fingerprint !== legacy.operationFingerprint) || + (requested !== undefined && requested !== String(legacy.requestedDeadlineAt)) + ) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); + return reply + .code(200) + .send(opsSuccess(requestId, historicalStatus(legacy, projectionVersion(request)))); + } const fingerprint = request.headers["x-ops-operation-fingerprint"]; const bodyFingerprint = request.headers["x-ops-body-fingerprint"]; // Explicit compatibility: a v1 lookup used the body hash in the sole header. @@ -1835,11 +2704,39 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif requestedDeadlineAt < 0 ) return reply.code(400).send(redactedOpsError(requestId, "validation")); - const deadlineAt = Math.min( + let deadlineAt = Math.min( requestedDeadlineAt, job.parent?.deadlineAt ?? Infinity, (job.admittedAt ?? 0) + (job.deploymentBudgetMs ?? 0), ); + const originalControl = maintenanceControls.get(job.operationId); + if ( + originalControl && + (job.operation.kind === "configurationMaintenance.settle" || + (job.operation.kind === "configurationMaintenance.decide" && + job.operation.decision === "abandon")) + ) { + // These accepted metadata controls retained their own bounded clock + // under the original expired/canceled root. Lookup must authenticate + // that SAME intent, without applying the forward-parent clamp again. + if ( + requestedDeadlineAt !== job.requestedDeadlineAt || + job.admittedAt === undefined || + job.deploymentBudgetMs === undefined + ) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); + try { + deadlineAt = normalizeConfigurationMaintenanceRecoveryControl( + originalControl, + job.operation, + job.operationKey, + job.admittedAt, + job.deploymentBudgetMs, + ).deadlineAt; + } catch { + return reply.code(409).send(redactedOpsError(requestId, "recovery_required")); + } + } if ( executionFingerprint({ kind: job.operation.kind, @@ -1856,6 +2753,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (fingerprint !== job.fingerprint && fingerprint !== job.operationFingerprint) return reply.code(409).send(redactedOpsError(requestId, "conflict")); } + await refreshOriginalMaintenanceControl(job); return reply.code(200).send(opsSuccess(requestId, statusFor(job, projectionVersion(request)))); }); @@ -1870,8 +2768,16 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif await pruneJobs(now().getTime()); const job = jobsById.get(operationId); if (!job || job.role !== role) { + const legacy = historicalPulls.find( + (value) => value.operationId === operationId && value.role === role, + ); + if (legacy) + return reply + .code(200) + .send(opsSuccess(requestId, historicalStatus(legacy, projectionVersion(request)))); return reply.code(404).send(redactedOpsError(requestId, "not_found")); } + await refreshOriginalMaintenanceControl(job); return reply.code(200).send(opsSuccess(requestId, statusFor(job, projectionVersion(request)))); }); @@ -1886,6 +2792,21 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif await pruneJobs(now().getTime()); const job = jobsById.get(operationId); if (!job || job.role !== role) { + const legacy = historicalPulls.find( + (value) => value.operationId === operationId && value.role === role, + ); + if (legacy) { + if ( + legacy.executionVersion === 2 && + request.headers["x-ops-operation-fingerprint"] !== legacy.fingerprint + ) + return reply.code(409).send(redactedOpsError(requestId, "conflict")); + return legacy.terminalAt === undefined + ? reply.code(409).send(redactedOpsError(requestId, "recovery_required")) + : reply + .code(200) + .send(opsSuccess(requestId, historicalStatus(legacy, projectionVersion(request)))); + } return reply.code(404).send(redactedOpsError(requestId, "not_found")); } if ( diff --git a/apps/ops-agent/src/service-action-policy.test.ts b/apps/ops-agent/src/service-action-policy.test.ts index c66c70d96..cade72fdd 100644 --- a/apps/ops-agent/src/service-action-policy.test.ts +++ b/apps/ops-agent/src/service-action-policy.test.ts @@ -27,8 +27,9 @@ describe("shared service action decisions", () => { .success, ).toBe(true); expect(decideServiceAction(authority, snapshot, "update")).toMatchObject({ - available: false, - reason: "unavailable", + available: true, + execution: "direct", + reason: null, }); }); it("retains logs after stop, requires proven ownership for cleanup and refuses unknown registry IDs", () => { diff --git a/apps/ops-agent/src/service-action-policy.ts b/apps/ops-agent/src/service-action-policy.ts index bd8462e27..de791afe1 100644 --- a/apps/ops-agent/src/service-action-policy.ts +++ b/apps/ops-agent/src/service-action-policy.ts @@ -4,7 +4,12 @@ import type { OwnedServiceSnapshot } from "./owned-services"; export interface ServiceAuthority { revisionId: string; enabledServiceIds: ReadonlySet; - services: readonly { serviceId: string; isBuiltIn: boolean; buildable: boolean }[]; + services: readonly { + serviceId: string; + isBuiltIn: boolean; + buildable: boolean; + creationDependencies?: readonly string[]; + }[]; } const controlPlane = new Set(["ops-agent", "app-api", "traefik"]); @@ -39,8 +44,22 @@ export function decideServiceAction( if (dependencies.has(service.serviceId)) return denied("host_only", "host"); if (action !== "stop" && !authority.enabledServiceIds.has(service.serviceId)) return denied("disabled"); - // These workflows belong to 01.2/01.4. A route or a declared type is not an installer. - if (action === "update") return denied("unavailable"); + // Pull-only DM staging needs Plan09 M3's purpose-bound producer. The installed + // coordinator Update's private checked submission is a separate fixed path. + if (service.serviceId === "data-manager" && action === "pull") return denied("unavailable"); + if (service.serviceId === "data-manager" && (action === "apply" || action === "update")) + return { + available: true, + execution: "workflow", + reason: null, + workflow: action === "apply" ? "coordinator-apply" : "coordinator-update", + }; + if ( + ["start", "apply", "update"].includes(action) && + service.serviceId !== "data-manager" && + service.creationDependencies?.includes("data-manager") + ) + return denied("unavailable"); if (action === "build" && (!service.isBuiltIn || !service.buildable)) return denied("not_buildable"); if (action === "stop" || action === "restart") { diff --git a/apps/ops-agent/src/service-lifecycle-runtime.ts b/apps/ops-agent/src/service-lifecycle-runtime.ts new file mode 100644 index 000000000..59bea71f8 --- /dev/null +++ b/apps/ops-agent/src/service-lifecycle-runtime.ts @@ -0,0 +1,145 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { + bindControlChildPhase, + closeControlInstallation, + consumeControlChildDispatch, + type FixedServiceLifecyclePlan, + fixedCliArguments, + inspectArtifactTargetLeaseChildSettlement, + issueControlChildPhase, + openFixedChildInstallation, + planFixedServiceUpdate, + prepareFixedControlChild, + readFixedServiceLifecyclePlan, + restoreAdmittedControlExecution, + retainFixedCliSpawnIntent, + type TargetLeaseAuthority, + verifyControlEffectSettlement, +} from "@openmapx/core/build-artifacts"; +import type { ExecutionRef, OpsOperation } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import type { FixedCliRunner } from "./administrative-runtime"; +import { OpsExecutionFailure, OpsHostUpdateRequiredError, type OpsRuntime } from "./runtime"; + +const sources = new WeakMap(); +export function prepareServiceUpdateAdmission( + runtime: OpsRuntime, + parent: ExecutionRef, + operation: Extract, + serviceIds: readonly string[], +): FixedServiceLifecyclePlan { + const source = sources.get(runtime); + if (!source) throw Error("Production lifecycle source unavailable"); + return planFixedServiceUpdate({ ...source, parent, operation, serviceIds }); +} +export function installServiceLifecycleRuntime( + runtime: OpsRuntime, + options: { rootDir: string; infraDir: string; runFixedCli: FixedCliRunner }, +): void { + sources.set(runtime, options); + runtime["service.update"] = async (operation, context) => { + const authority = context.controlAuthority, + lease: TargetLeaseAuthority | undefined = context.artifactLease ?? context.ownedTargetLease; + if (!authority || !lease) throw new OpsHostUpdateRequiredError(); + const plan = readFixedServiceLifecyclePlan(authority); + const child = await bindControlChildPhase(authority, { phase: "lifecycle", ordinal: 0 }); + const publicKey = prepareFixedControlChild(authority, plan.delegationId), + args = fixedCliArguments(operation); + retainFixedCliSpawnIntent(authority, { + delegationId: plan.delegationId, + execution: child.execution, + operation, + args, + reservations: plan.cliReservations, + }); + const issued = await issueControlChildPhase(authority, { + phase: "lifecycle", + ordinal: 0, + childPublicKey: publicKey, + lease, + }); + await services.withServiceLifecycleReceiptAuthority( + authority, + () => + services.openServiceLifecycleReceipt({ + dataRoot: join(options.infraDir, "data"), + execution: child.execution, + serviceIds: [operation.serviceId], + action: "update", + noDeps: operation.noDeps ?? false, + coordinatorMode: "ensure-running", + expectedRevisionId: operation.expectedConfigurationRevisionId, + seedRevisionId: operation.expectedConfigurationRevisionId, + }), + { execution: child.execution, publicKey }, + ); + consumeControlChildDispatch(issued.dispatchTicket); + let failure: unknown; + try { + context.beginServiceCreation?.(); + await options.runFixedCli(args, { + ...context, + execution: child.execution, + operationKind: "service.update", + controlAssertion: issued.assertion, + }); + } catch (error) { + failure = error; + } + const installation = openFixedChildInstallation({ + infraDir: options.infraDir, + assertion: issued.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const childAuthority = restoreAdmittedControlExecution(installation, child.execution); + const receipt = await services.withServiceLifecycleReceiptAuthority(childAuthority, () => + services.readServiceLifecycleReceipt({ + dataRoot: join(options.infraDir, "data"), + execution: child.execution, + }), + ); + const { + version: _version, + execution: _execution, + serviceIds: _ids, + action: _action, + noDeps: _noDeps, + coordinatorMode: _mode, + hardlinks: _hardlinks, + sequence: _sequence, + ...evidence + } = receipt; + if ( + failure || + receipt.phase !== "complete" || + receipt.finalRevisionId !== plan.finalRevisionId + ) + throw new OpsExecutionFailure( + receipt.phase === "failed" ? "runtime" : "recovery_required", + { serviceLifecycle: evidence }, + ); + verifyControlEffectSettlement(installation, child.execution); + const settled = inspectArtifactTargetLeaseChildSettlement(lease, child.execution); + if (settled?.outcome !== "committed" || settled.publication !== "committed") + throw new OpsExecutionFailure("recovery_required", { serviceLifecycle: evidence }); + return { changed: true, revisionId: receipt.finalRevisionId }; + } catch (error) { + if (error instanceof OpsExecutionFailure) throw error; + throw new OpsExecutionFailure("recovery_required", { + serviceLifecycle: { + expectedRevisionId: operation.expectedConfigurationRevisionId, + seedRevisionId: operation.expectedConfigurationRevisionId, + finalRevisionId: null, + phase: "unknown", + commit: "seed_committed", + followUp: "unknown", + failure: { reason: "receipt_io", failedServiceIds: [operation.serviceId] }, + }, + }); + } finally { + closeControlInstallation(installation); + } + }; +} diff --git a/apps/ops-agent/src/service-pull-runtime.ts b/apps/ops-agent/src/service-pull-runtime.ts new file mode 100644 index 000000000..9eb96b60d --- /dev/null +++ b/apps/ops-agent/src/service-pull-runtime.ts @@ -0,0 +1,438 @@ +/** Existing normal pull: only fixed trusted image sources, with no generation + * publication, container lifecycle, mutable overlay repair or target lease. */ +import { readFileSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { + assertControlExecutionForward, + assertCoordinatorPullLeaseCompatibility, + beginControlEffect, + type ControlInstallation, + coordinatorPullBaselineSchema, + executionAuthorityState, + retainConfigurationPreEffectRefusal, + verifyConfigurationReleaseSourceWitness, + verifyConfigurationSourceWitness, + verifyCoordinatorPullAdmissionSource, +} from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + coordinatorLifecycleMetadataSchema, + coordinatorPullSubmissionDigest, + type ExecutionRef, + type OpsOperation, + type PreparedConfigurationRecord, + type RetainedParentExecution, + servicePullResultSchema, +} from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { + type ConfigurationCandidateSource, + readConfigurationCandidate, +} from "./configuration-candidate-source"; +import { assertConfigurationCoordinatorReady } from "./configuration-hardlinks-runtime"; +import { + type DockerRuntimeOptions, + runContainedProcess, + runContainedStreamingProcess, +} from "./docker-runtime"; +import { + type CoordinatorPullOwnershipObservation, + confirmCoordinatorPullOwnership, + type OwnedServiceResolver, +} from "./owned-services"; +import { safeProcessDiagnostic } from "./process-output"; +import type { OpsExecutionContext, OpsRuntime } from "./runtime"; +import { OpsExecutionFailure } from "./runtime"; +import type { TrustedConfigurationAuthoritySnapshot } from "./trusted-config-runtime"; + +export function coordinatorCandidateMetadata(record: PreparedConfigurationRecord) { + const fixed = coordinatorLifecycleMetadataSchema.parse(record.privateCoordinatorLifecycle); + if ( + record.apiFollowUp !== `coordinator-${fixed.action}` || + !record.maintenanceId || + !record.prepareOperationKey || + canonicalJson(record.affectedServiceIds) !== '["data-manager"]' + ) + throw Error("Fixed coordinator candidate provenance required"); + return fixed; +} +export function renderCoordinatorCandidate( + candidate: Awaited>, + authority: TrustedConfigurationAuthoritySnapshot, + infraDir: string, +) { + const fixed = coordinatorCandidateMetadata(candidate.record), + operation = candidate.operation; + if ( + operation.kind !== "stack.render" || + !operation.maintenanceId || + operation.candidateId !== candidate.record.candidateId || + operation.coordinatorMode !== "ensure-running" || + canonicalJson(candidate.record.privateOverlay) !== + canonicalJson({ purpose: "render", coordinatorMode: "ensure-running", changes: [] }) + ) + throw Error("Fixed unchanged coordinator render required"); + const desired = services.readDesiredSelection(infraDir), + rendered = services.renderConfiguration({ + infraDir, + services: authority.services, + integrationSchemas: authority.integrationSchemas, + input: { + ...candidate.payload, + selectedRoots: candidate.payload.selectedRoots ?? desired.roots, + }, + allowMissingSelected: !candidate.payload.selectedRoots && desired.source === "default", + }), + selected = services.readServiceReleaseImageSelection(infraDir, "data-manager"), + imageReference = + selected.imageReference ?? + services.configurationImageReference(rendered.rendered.composeYaml, "data-manager"); + return { + fixed, + rendered, + imageReference, + releaseSelectionDigest: selected.releaseSelectionDigest, + }; +} +interface ServicePullSource { + infraDir: string; + candidateSource?: ConfigurationCandidateSource; + loadAuthority(): Promise; + ownedServices: OwnedServiceResolver; + execFile?: DockerRuntimeOptions["execFile"]; +} +const sources = new WeakMap(); +/** The generic action wrapper cannot infer this internal workflow from a UUID. + * Require its actual MAC candidate and original signed normal admission. */ +export async function assertAdmittedCoordinatorPullRuntime( + runtime: OpsRuntime, + operation: Extract, + context: OpsExecutionContext, +): Promise { + const source = sources.get(runtime); + if ( + !source?.candidateSource || + !context.controlAuthority || + operation.serviceId !== "data-manager" || + !operation.candidateId + ) + throw Error("Actual internal coordinator Update pull required"); + const candidate = await readConfigurationCandidate(source.candidateSource, operation.candidateId), + state = executionAuthorityState(context.controlAuthority), + actual = verifyCoordinatorPullAdmissionSource( + state.installation.installation, + candidate.record, + ); + if ( + coordinatorCandidateMetadata(candidate.record).action !== "update" || + canonicalJson(actual.operation) !== canonicalJson(operation) || + canonicalJson(actual.execution) !== canonicalJson(context.execution) || + canonicalJson(state.execution) !== canonicalJson(context.execution) + ) + throw Error("Original internal coordinator Update pull changed"); +} +/** Actual MAC source join, under the first original server admission lock. */ +export async function prepareCoordinatorPullBaseline( + runtime: OpsRuntime, + installation: ControlInstallation, + input: { + operation: Extract; + execution: ExecutionRef; + parent?: RetainedParentExecution; + requestedDeadlineAt?: number; + }, + observation: CoordinatorPullOwnershipObservation, +) { + const source = sources.get(runtime), + parent = input.parent; + if ( + !source?.candidateSource || + !input.operation.candidateId || + !parent || + parent.issuer !== "api" + ) + throw Error("Original coordinator source required"); + const candidate = await readConfigurationCandidate( + source.candidateSource, + input.operation.candidateId, + ), + registry = await source.loadAuthority(), + rendered = renderCoordinatorCandidate(candidate, registry, source.infraDir), + fixed = rendered.fixed; + if (fixed.action !== "update") throw Error("Original update submission required"); + const submission = fixed.pull, + attempt = submission.attempt; + if ( + canonicalJson(submission.operation) !== canonicalJson(input.operation) || + submission.operationKey !== input.execution.operationKey || + submission.operationFingerprint !== input.execution.operationFingerprint || + submission.requestedDeadlineAt !== input.requestedDeadlineAt || + observation.operationFingerprint !== submission.operationFingerprint || + observation.candidateId !== candidate.record.candidateId || + registry.revisionId !== observation.configurationAuthorityRevision || + candidate.record.apiJobId !== parent.apiJobId || + parent.apiJobId !== attempt.apiJobId || + parent.apiOwnerVersion !== attempt.apiOwnerVersion || + parent.attemptId !== attempt.attemptId || + parent.operationKey !== attempt.parentOperationKey || + parent.fingerprint !== attempt.parentFingerprint || + parent.deadlineAt !== attempt.parentDeadlineAt || + candidate.record.parentOperationKey !== parent.operationKey || + candidate.record.parentFingerprint !== parent.fingerprint || + candidate.record.parentDeadlineAt !== parent.deadlineAt || + services.currentConfigurationGeneration(source.infraDir) !== + input.operation.expectedConfigurationRevisionId || + rendered.imageReference !== observation.configImageReference + ) + throw Error("Original coordinator claim/source changed"); + verifyConfigurationSourceWitness({ + installation, + infraDir: source.infraDir, + sourceRevision: candidate.record.expectedSourceRevision, + expectedCurrentRevisionId: input.operation.expectedConfigurationRevisionId, + authorityRevision: observation.configurationAuthorityRevision, + }); + return coordinatorPullBaselineSchema.parse({ + version: 1, + candidateId: candidate.record.candidateId, + sourceRevision: candidate.record.expectedSourceRevision, + configurationRevisionId: input.operation.expectedConfigurationRevisionId, + configurationAuthorityRevision: observation.configurationAuthorityRevision, + policyRevision: observation.policyRevision, + submissionDigest: coordinatorPullSubmissionDigest(submission), + observedAt: observation.observedAt, + configImageReference: observation.configImageReference, + composeSourcesDigest: observation.composeSourcesDigest, + container: observation.container, + }); +} +export function installServicePullRuntime(runtime: OpsRuntime, options: ServicePullSource) { + sources.set(runtime, options); + runtime["service.pull"] = async (operation, context) => { + if (operation.serviceId === "data-manager" && !operation.candidateId) + throw new OpsExecutionFailure("configuration_prerequisite", {}); + try { + return await services.withConfigurationGenerationWriter( + options.infraDir, + async () => { + const authority = context.controlAuthority; + if (!authority || !context.execution || context.artifactLease || context.ownedTargetLease) + throw Error("Normal pull admission without a target lease required"); + assertControlExecutionForward(authority); + const admitted = executionAuthorityState(authority); + if ( + services.currentConfigurationGeneration(options.infraDir) !== + operation.expectedConfigurationRevisionId + ) + throw Error("Pull configuration revision changed"); + const directory = join(options.infraDir, ".ops-agent-releases"); + await mkdir(directory, { mode: 0o700, recursive: true }); + const lock = await services.acquireReleaseStoreLock(directory); + try { + const registry = await options.loadAuthority(); + let imageReference: string, releaseSelectionDigest: string | null; + if (operation.candidateId) { + if (!options.candidateSource) throw Error("Original pull candidate unavailable"); + const candidate = await readConfigurationCandidate( + options.candidateSource, + operation.candidateId, + ), + source = renderCoordinatorCandidate(candidate, registry, options.infraDir), + fixed = source.fixed, + actual = verifyCoordinatorPullAdmissionSource( + admitted.installation.installation, + candidate.record, + ), + parent = actual.parent; + if ( + fixed.action !== "update" || + !parent || + parent.issuer !== "api" || + canonicalJson(fixed.pull.operation) !== canonicalJson(operation) || + fixed.pull.operationKey !== context.execution.operationKey || + fixed.pull.operationFingerprint !== context.execution.operationFingerprint || + candidate.record.apiJobId !== parent.apiJobId || + fixed.pull.attempt.apiJobId !== parent.apiJobId || + fixed.pull.attempt.apiOwnerVersion !== parent.apiOwnerVersion || + fixed.pull.attempt.attemptId !== parent.attemptId || + fixed.pull.attempt.parentOperationKey !== parent.operationKey || + fixed.pull.attempt.parentFingerprint !== parent.fingerprint || + fixed.pull.attempt.parentDeadlineAt !== parent.deadlineAt || + fixed.pull.requestedDeadlineAt !== actual.requestedDeadlineAt + ) + throw Error("Original coordinator pull submission changed"); + verifyConfigurationReleaseSourceWitness({ + installation: admitted.installation.installation, + infraDir: options.infraDir, + sourceRevision: candidate.record.expectedSourceRevision, + expectedCurrentRevisionId: operation.expectedConfigurationRevisionId, + authorityRevision: registry.revisionId, + }); + ({ imageReference, releaseSelectionDigest } = source); + if ( + actual.coordinatorPullBaseline.configurationAuthorityRevision !== + registry.revisionId || + actual.coordinatorPullBaseline.policyRevision !== + context.claim.capability.revisionId || + actual.coordinatorPullBaseline.configImageReference !== imageReference + ) + throw Error("Original pull baseline source changed"); + await confirmCoordinatorPullOwnership( + options.ownedServices, + operation, + context.execution.operationFingerprint, + actual.coordinatorPullBaseline, + context.signal, + ); + if ( + (await options.ownedServices.inspectOwnedServices(context.signal)).find( + (s) => s.serviceId === "data-manager", + )?.state !== "running" + ) + throw Error("Ready old coordinator required before image pull"); + await assertConfigurationCoordinatorReady(context.signal); + // Same actual current native Compose/release sources plus the full + // readonly prospective render/registered manifest sources. Aliases + // are a refusal condition; no sibling OLD-image exception exists. + const args = services.stackComposeArgs({ + infraDir: options.infraDir, + composePath: services.configurationGenerationComposeFile( + options.infraDir, + operation.expectedConfigurationRevisionId, + ), + composeReleasePath: join(options.infraDir, "docker-compose.release.yml"), + }); + const config = JSON.parse( + ( + await (options.execFile ?? runContainedProcess)( + "docker", + [...args, "config", "--format", "json"], + { signal: context.signal, timeout: 15000, maxBuffer: 1024 * 1024 }, + ) + ).stdout, + ) as { services?: Record }; + if ( + !config.services || + typeof config.services !== "object" || + Array.isArray(config.services) || + Object.keys(config.services).length > 256 + ) + throw Error("Current coordinator image inventory unavailable"); + const identity = services.coordinatorPullReferenceIdentity(imageReference), + enabled = new Set( + source.rendered.resolved.enabled.map((value) => value.manifest.id), + ); + for (const service of registry.services) { + if (service.manifest.id === "data-manager") continue; + const selected = services.readServiceReleaseImageSelection( + options.infraDir, + service.manifest.id, + ), + prospective = + selected.imageReference ?? + (enabled.has(service.manifest.id) + ? services.configurationImageReference( + source.rendered.rendered.composeYaml, + service.manifest.id, + ) + : services.serviceContainerImageReference(service.manifest.container)); + const current = + config.services[service.manifest.id]?.image ?? + selected.imageReference ?? + services.serviceContainerImageReference(service.manifest.container); + if ( + [current, prospective].some( + (value) => services.coordinatorPullReferenceIdentity(value) === identity, + ) + ) + throw Error("Coordinator image reference shared by another registered service"); + } + } else { + const selected = services.readServiceReleaseImageSelection( + options.infraDir, + operation.serviceId, + ); + imageReference = + selected.imageReference ?? + services.configurationImageReference( + readFileSync( + services.configurationGenerationComposeFile( + options.infraDir, + operation.expectedConfigurationRevisionId, + ), + "utf8", + ), + operation.serviceId, + ); + releaseSelectionDigest = selected.releaseSelectionDigest; + } + const remaining = () => { + const value = context.execution.deadlineAt - Date.now(); + if (value <= 0 || context.signal.aborted) + throw new OpsExecutionFailure("timeout", {}); + return value; + }; + if (operation.serviceId === "data-manager") { + try { + assertCoordinatorPullLeaseCompatibility(authority); + } catch { + retainConfigurationPreEffectRefusal(authority); + throw new OpsExecutionFailure("busy", {}); + } + beginControlEffect(authority); + } + const pullArgs = ["image", "pull", imageReference]; + if (options.execFile) + await options.execFile("docker", pullArgs, { + signal: context.signal, + timeout: Math.min(15 * 60000, remaining()), + maxBuffer: 1024 * 1024, + }); + else + await runContainedStreamingProcess("docker", pullArgs, { + signal: context.signal, + timeout: Math.min(15 * 60000, remaining()), + onLine: context.emitLog, + redact: safeProcessDiagnostic, + }); + const imageId = ( + await (options.execFile ?? runContainedProcess)( + "docker", + ["image", "inspect", "--format", "{{.Id}}", imageReference], + { signal: context.signal, timeout: Math.min(15000, remaining()), maxBuffer: 4096 }, + ) + ).stdout.trim(); + context.signal.throwIfAborted(); + assertControlExecutionForward(authority); + if ( + (await options.loadAuthority()).revisionId !== registry.revisionId || + services.currentConfigurationGeneration(options.infraDir) !== + operation.expectedConfigurationRevisionId + ) + throw Error("Pull source changed"); + return servicePullResultSchema.parse({ + changed: true, + revisionId: operation.expectedConfigurationRevisionId, + imageReference, + imageId, + releaseSelectionDigest, + }); + } finally { + lock.release(); + } + }, + context.signal, + ); + } catch (error) { + if ( + error instanceof services.ConfigurationGenerationError && + error.result.outcome === "not_committed" && + !error.result.cleanupFailed && + error.cause instanceof OpsExecutionFailure + ) + throw error.cause; + throw error; + } + }; +} diff --git a/apps/ops-agent/src/startup-authority.test.ts b/apps/ops-agent/src/startup-authority.test.ts index 158daad79..859f47ec8 100644 --- a/apps/ops-agent/src/startup-authority.test.ts +++ b/apps/ops-agent/src/startup-authority.test.ts @@ -58,15 +58,39 @@ describe("ops-agent startup authority gate", () => { }); const signal = new AbortController().signal; await expect( - claimer.claim({ kind: "service.start", serviceId: "dawarich-app" }, "a".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "dawarich-app", + }, + "a".repeat(64), + signal, + ), ).resolves.toBeNull(); trusted = new Set(["dawarich-app"]); await expect( - claimer.claim({ kind: "service.start", serviceId: "dawarich-app" }, "b".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "dawarich-app", + }, + "b".repeat(64), + signal, + ), ).resolves.not.toBeNull(); await expect( - claimer.claim({ kind: "service.start", serviceId: "redis" }, "c".repeat(64), signal), + claimer.claim( + { + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + kind: "service.start", + serviceId: "redis", + }, + "c".repeat(64), + signal, + ), ).resolves.toBeNull(); }); diff --git a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs new file mode 100644 index 000000000..4da060e1a --- /dev/null +++ b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs @@ -0,0 +1,220 @@ +/** Native production assembly. Only Docker's external process IO crosses the + * owning fixture's loopback bridge; authority, journal and HTTP handlers are real. */ +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { dirname, join } from "node:path"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import { CommunityRepositoryStore } from "../community-repositories.ts"; +import { configurationPolicyRevision } from "../configured-resource-authority.ts"; +import { openOpsJobJournal } from "../journal.ts"; +import { createProductionRegistryResourceClaimer } from "../policy.ts"; +import { createProductionOpsRuntime, productionOwnedServices } from "../production-runtime.ts"; +import { buildOpsAgentServer } from "../server.ts"; +import { createFileTrustedOpsDataSource } from "../trusted-config-source.ts"; +import { createTrustedConfigurationAuthorityLoader } from "../trusted-configuration-authority.ts"; + +let raw = ""; +for await (const chunk of process.stdin) raw += chunk; +const input = JSON.parse(raw); +// Pass-through observation of the named filesystem edge. Every original mkdir +// and rmdir still executes; this records whether an actual busy guard is ours. +const mkdir = fs.mkdirSync, + rmdir = fs.rmdirSync, + heldSourceWriters = new Set(); +let busySourceObservations = 0; +const writerIdentity = (path) => { + const parent = fs.statSync(dirname(String(path))); + return `${parent.dev}:${parent.ino}`; +}; +fs.mkdirSync = (path, ...options) => { + const sourceWriter = String(path).endsWith("/.family-writer"), + key = sourceWriter ? writerIdentity(path) : undefined; + try { + const result = mkdir(path, ...options); + if (sourceWriter) heldSourceWriters.add(key); + return result; + } catch (error) { + if (sourceWriter && error.code === "EEXIST" && busySourceObservations++ < 12) + fs.writeSync( + 1, + `${JSON.stringify({ event: "source-writer-busy", sameProcess: heldSourceWriters.has(key), frames: new Error().stack.split("\n").slice(1, 13) })}\n`, + ); + throw error; + } +}; +fs.rmdirSync = (path, ...options) => { + const sourceWriter = String(path).endsWith("/.family-writer"), + key = sourceWriter ? writerIdentity(path) : undefined; + const result = rmdir(path, ...options); + if (sourceWriter) heldSourceWriters.delete(key); + return result; +}; +syncBuiltinESMExports(); +if (["held", "unwound"].includes(input.stopAfterFirstTargetRelease)) { + const remove = fs.rmdirSync, + locks = join(input.infraDir, "data/.admin-target-control/locks"); + let stopped = false, + acknowledgementLost = false; + fs.rmdirSync = (path, ...options) => { + let target = false; + if (String(path).endsWith("/app-api")) { + try { + const actual = fs.statSync(dirname(String(path))), + expected = fs.statSync(locks); + target = actual.dev === expected.dev && actual.ino === expected.ino; + } catch { + /* Only this original physical lock is the named fault edge. */ + } + } + remove(path, ...options); + if (target && !stopped && input.stopAfterFirstTargetRelease === "unwound") { + acknowledgementLost = true; + throw Error("Named target release acknowledgement loss"); + } + if ( + !stopped && + ((target && input.stopAfterFirstTargetRelease === "held") || + (acknowledgementLost && String(path).endsWith("/.journal-writer"))) + ) { + stopped = true; + fs.writeSync( + 1, + `${JSON.stringify({ event: "pre-effect-release-unlinked", guard: input.stopAfterFirstTargetRelease })}\n`, + ); + process.kill(process.pid, "SIGSTOP"); + } + }; + syncBuiltinESMExports(); +} +if ( + !Array.isArray(input.serviceIds) || + input.serviceIds.length > 256 || + input.serviceIds.some((id) => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) +) + throw Error("Invalid fixed fixture registry"); +const builtInServices = input.serviceIds.map((id) => ({ + manifest: JSON.parse(fs.readFileSync(join(input.root, "services", id, "service.json"), "utf8")), + directory: join(input.root, "services", id), + enabled: true, + isBuiltIn: true, +})); +const loadAuthority = createTrustedConfigurationAuthorityLoader({ + rootDir: input.root, + builtInServices, +}); +const repositories = new CommunityRepositoryStore({ rootDir: input.root }); +await repositories.recover(); +const runtime = await createProductionOpsRuntime({ + rootDir: input.root, + stack: input.stack, + loadAuthority, + communityRepositories: repositories, + candidateSource: { + directory: input.queue, + token: input.token, + ownerUid: fs.statSync(input.queue).uid, + ownerGid: fs.statSync(input.queue).gid, + }, + docker: { + fileExists: fs.existsSync, + execFile: async (file, args, options) => { + if (file !== "docker") throw Error("Only named Docker IO crosses fixture bridge"); + const response = await fetch(input.dockerBridge, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ args }), + signal: options.signal, + }); + if (!response.ok) throw Error("Named Docker IO failed"); + return response.json(); + }, + }, +}); +const policyAuthority = async () => { + const a = await loadAuthority(); + return { + revisionId: configurationPolicyRevision(a.revisionId), + configurationAuthorityRevision: a.revisionId, + services: a.services.map((s) => ({ + serviceId: s.manifest.id, + enabled: s.enabled, + isBuiltIn: s.isBuiltIn, + selectionAuthority: s, + })), + integrationIds: [...a.integrationSchemas.keys()], + }; +}; +const authority = await policyAuthority(); +const installation = openControlInstallation({ + infraDir: input.infraDir, + role: "ops-agent", + serviceIds: input.serviceIds, +}); +const journal = await openOpsJobJournal(join(input.infraDir, "data/ops-agent/jobs-v1.json")); +const app = buildOpsAgentServer({ + runtime, + journal, + controlInstallation: installation, + operationTimeoutMs: 180000, + tokens: { api: input.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + resourceClaimer: createProductionRegistryResourceClaimer({ + services: authority.services, + integrationIds: authority.integrationIds, + loadAuthority: policyAuthority, + trustedData: createFileTrustedOpsDataSource({ + directory: input.queue, + token: input.token, + allowedUids: [fs.statSync(input.queue).uid], + expectedGid: fs.statSync(input.queue).gid, + }), + ownedServices: productionOwnedServices(runtime), + enabledServiceIds: (ss) => + services.readAppliedServiceIds( + input.infraDir, + ss.map((s) => s.selectionAuthority), + ) ?? new Set(), + serviceMetadata: (ss) => + ss.map((s) => ({ + ...s, + buildable: !!s.selectionAuthority.manifest.buildCommand, + creationDependencies: services.serviceCreationDependencyIds( + ss.map((v) => v.selectionAuthority), + [s.serviceId], + ), + })), + }), +}); +app.addHook("onSend", async (request, reply, payload) => { + const kind = request.body?.operation?.kind; + if (request.method === "POST" && request.url === "/v1/operations" && kind) + process.stdout.write( + JSON.stringify({ + event: "operation", + kind, + status: reply.statusCode, + ...(reply.statusCode >= 400 ? { errorClass: JSON.parse(payload).error?.class } : {}), + }) + "\n", + ); + return payload; +}); +const url = await app.listen({ host: "127.0.0.1", port: 0 }); +const owned = await productionOwnedServices(runtime).inspectOwnedServices( + new AbortController().signal, +); +process.stdout.write( + JSON.stringify({ + event: "ready", + url, + ownedDM: owned.some((s) => s.serviceId === "data-manager"), + }) + "\n", +); +process.once("SIGTERM", () => { + void app + .close() + .then(() => { + closeControlInstallation(installation); + process.exit(0); + }) + .catch(() => process.exit(2)); +}); diff --git a/apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs b/apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs new file mode 100644 index 000000000..78bcbec30 --- /dev/null +++ b/apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs @@ -0,0 +1,177 @@ +/** Native production assembly; only named Docker inventory/cache and the + * filesystem acknowledgement fault are controlled by the owning test. */ +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { basename, dirname, join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +let rawInput = ""; +for await (const chunk of process.stdin) rawInput += chunk; +const input = JSON.parse(rawInput); +Date.now = () => input.now; +let faulted = false, + packetBundle; +const originalUnlink = fs.unlinkSync; +if (input.mode === "fault") { + fs.unlinkSync = (path) => { + const same = + typeof path === "string" && + basename(path) === basename(input.firstPullPath) && + fs.statSync(dirname(path), { bigint: true }).dev === + fs.statSync(dirname(input.firstPullPath), { bigint: true }).dev && + fs.statSync(dirname(path), { bigint: true }).ino === + fs.statSync(dirname(input.firstPullPath), { bigint: true }).ino; + if (same && !faulted) { + packetBundle = { + publicKey: JSON.parse( + fs.readFileSync(join(input.infraDir, "secrets/control-verifiers.json"), "utf8"), + ).publicKeys["ops-agent"], + candidate: JSON.parse(fs.readFileSync(join(input.firstSlot, "proof.json"), "utf8")), + progress: JSON.parse(fs.readFileSync(join(input.firstSlot, "progress.json"), "utf8")), + manifest: JSON.parse(fs.readFileSync(join(input.firstSlot, "manifest.json"), "utf8")), + decision: JSON.parse(fs.readFileSync(join(input.firstSlot, "commit.json"), "utf8")), + }; + originalUnlink(path); + faulted = true; + throw Error("Named external filesystem acknowledgement lost after actual first F unlink"); + } + return originalUnlink(path); + }; + syncBuiltinESMExports(); +} +// Dynamic imports intentionally follow installation of the native FS edge. +const core = await import("@openmapx/core/build-artifacts"); +const { services } = await import("@openmapx/core/server"); +const { CommunityRepositoryStore } = await import("../community-repositories.ts"); +const { createProductionOpsRuntime, productionOwnedServices } = await import( + "../production-runtime.ts" +); +const { createTrustedConfigurationAuthorityLoader } = await import( + "../trusted-configuration-authority.ts" +); +const { openOpsJobJournal } = await import("../journal.ts"); +const { buildOpsAgentServer } = await import("../server.ts"); +const { runContainedProcess } = await import("../docker-runtime.ts"); +if ( + !Array.isArray(input.serviceIds) || + input.serviceIds.length > 256 || + input.serviceIds.some((id) => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) +) + throw Error("Invalid fixed fixture registry"); +const builtInServices = input.serviceIds.map((id) => ({ + manifest: JSON.parse(fs.readFileSync(join(input.root, "services", id, "service.json"), "utf8")), + directory: join(input.root, "services", id), + enabled: true, + isBuiltIn: true, +})); +const loadAuthority = createTrustedConfigurationAuthorityLoader({ + rootDir: input.root, + builtInServices, +}); +const repositories = new CommunityRepositoryStore({ rootDir: input.root }); +await repositories.recover(); +const runtime = await createProductionOpsRuntime({ + rootDir: input.root, + stack: input.stack, + loadAuthority, + communityRepositories: repositories, + candidateSource: { + directory: input.queue, + token: input.token, + ownerUid: fs.statSync(input.queue).uid, + ownerGid: fs.statSync(input.queue).gid, + }, + docker: { + fileExists: fs.existsSync, + execFile: async (file, args, options) => { + if (file !== "docker") throw Error("Only fixed Docker IO expected"); + if (args[0] === "container" && args[1] === "ls") + return { stdout: input.rows.map((row) => row.Id).join("\n") }; + if (args[0] === "container" && args[1] === "inspect") + return { stdout: input.rows.map((row) => JSON.stringify(row)).join("\n") }; + if (args[0] === "image" && args[1] === "inspect") { + const image = input.images[args.at(-1)]; + if (!image) throw Error("Unknown named Docker cache observation"); + return { stdout: image }; + } + if (args[0] === "compose" && args.includes("config")) + return runContainedProcess(file, args, options); + throw Error("GC must not issue a Docker mutation"); + }, + }, +}); +const installation = core.openControlInstallation({ + infraDir: input.infraDir, + role: "ops-agent", + serviceIds: input.serviceIds, +}); +const journal = await openOpsJobJournal(join(input.infraDir, "data/ops-agent/jobs-v1.json")); +const app = buildOpsAgentServer({ + runtime, + journal, + controlInstallation: installation, + tokens: { api: input.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, +}); +await app.ready(); +let retired = 0, + pending = 0; +for (let pass = 0; pass < 12; pass++) { + if (input.mode === "fault" && faulted) break; + const result = await core.reconcileControlRetention(installation, { maxFamilies: 64 }); + retired += result.retired; + pending = result.pending; + if (input.mode === "fault" && faulted) break; + await delay(0); +} +if (input.mode === "fault") { + const until = performance.now() + 10000; + while ( + fs.existsSync(join(input.infraDir, "control-private/ops-agent/.retention-reconcile-writer")) + ) { + if (performance.now() > until) + throw Error("Original collector did not release its guard after IO loss"); + await delay(10); + } + if (!faulted || !packetBundle) { + const p = fs.existsSync(join(input.firstSlot, "progress.json")) + ? JSON.parse(fs.readFileSync(join(input.firstSlot, "progress.json"), "utf8")) + : null; + throw Error( + `Actual first F unlink fault not reached: ${JSON.stringify({ retired, pending, Fpresent: fs.existsSync(input.firstPullPath), Gpresent: fs.existsSync(input.secondPullPath), phase: p?.phase, lastError: p?.lastError ?? null })}`, + ); + } + if (input.evidenceFile) + fs.writeFileSync(input.evidenceFile, JSON.stringify(packetBundle, null, 2), { + flag: "wx", + mode: 0o600, + }); + process.stdout.write( + JSON.stringify({ + boundary: "actual-first-F-private-unlink", + faulted, + Fpresent: fs.existsSync(input.firstPullPath), + Gpresent: fs.existsSync(input.secondPullPath), + FslotPresent: fs.existsSync(input.firstSlot), + journalKeys: journal.records().map((job) => job.operationKey), + }) + "\n", + ); + process.kill(process.pid, "SIGSTOP"); + throw Error("Stopped native fixture resumed unexpectedly"); +} +const owned = await productionOwnedServices(runtime).inspectOwnedServices( + new AbortController().signal, +); +process.stdout.write( + JSON.stringify({ + boundary: "native-retirement-observation", + retired, + pending, + Fpresent: fs.existsSync(input.firstPullPath), + Gpresent: fs.existsSync(input.secondPullPath), + FslotPresent: fs.existsSync(input.firstSlot), + journalKeys: journal.records().map((job) => job.operationKey), + ordinaryOwnedDM: owned.some((service) => service.serviceId === "data-manager"), + }) + "\n", +); +await app.close(); +core.closeControlInstallation(installation); diff --git a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts index 68e307ba4..6789aef29 100644 --- a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts +++ b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts @@ -19,9 +19,12 @@ import { sealTrustedConfigurationSnapshot, } from "@openmapx/core/ops"; import { AnchoredDirectory } from "../../../../packages/core/src/build-artifacts/filesystem"; +import { configurationSource } from "../__fixtures__/configuration-source"; import { openOpsJobJournal, type PersistedOpsJob } from "../journal"; import { createRegistryResourceClaimer } from "../policy"; +import { createUnavailableRuntime } from "../runtime"; import { buildOpsAgentServer } from "../server"; +import { installTrustedConfigurationRuntime } from "../trusted-config-runtime"; import { createFileTrustedOpsDataSource, initializeTrustedSnapshotDirectory, @@ -45,6 +48,27 @@ let operation: OpsOperation = input.borrowed : { kind: "backup.delete", backupId: "daily" }; let resourceClaimer: ReturnType | undefined; if (input.outcome === "configuration") { + const sourceRuntime = createUnavailableRuntime(); + installTrustedConfigurationRuntime(sourceRuntime, { + infraDir: input.root, + env: {}, + integrationSchemas: new Map(), + services: [ + { + manifest: { + id: "motis", + name: "motis", + version: "1.0.0", + quality: "built-in", + container: { image: "example/motis", tag: "1" }, + }, + directory: join(input.root, "services/motis"), + enabled: true, + isBuiltIn: true, + }, + ], + }); + const source = await configurationSource(sourceRuntime, input.root); const directory = join(input.root, "trusted-config"); mkdirSync(directory, { mode: 0o700 }); const policy = { @@ -60,6 +84,7 @@ if (input.outcome === "configuration") { issuedAtMs: Date.now(), nonce: "nonce_0123456789abcdef", operationForRevision: (revisionId) => ({ + ...source, kind: "stack.render", revisionId, expectedCurrentRevisionId: null, @@ -72,11 +97,7 @@ if (input.outcome === "configuration") { serviceSecrets: [], }, }); - operation = { - kind: "stack.render", - revisionId: snapshot.revisionId, - expectedCurrentRevisionId: null, - }; + operation = snapshot.operation; writeFileSync(join(directory, `${snapshot.revisionId}.json`), snapshot.bytes, { mode: 0o600 }); resourceClaimer = createRegistryResourceClaimer({ serviceIds: ["motis"], diff --git a/apps/ops-agent/src/trusted-config-ownership.test.ts b/apps/ops-agent/src/trusted-config-ownership.test.ts index 600cbec06..e30fd7ee1 100644 --- a/apps/ops-agent/src/trusted-config-ownership.test.ts +++ b/apps/ops-agent/src/trusted-config-ownership.test.ts @@ -1,4 +1,4 @@ -import { randomBytes } from "node:crypto"; +import { randomUUID } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,14 +9,20 @@ import { provisionControlInstallation, } from "@openmapx/core/build-artifacts"; import { - createControlRootOperationKey, type OpsOperation, opsOperationFingerprint, opsOperationSchema, sealTrustedConfigurationSnapshot, } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + issueApiJobChildAssertion, + prepareApiJobChildAttempt, + prepareApiJobRoot, +} from "../../api/src/services/admin-job-parent"; +import type { JobContext } from "../../api/src/services/job-runner"; +import { configurationSource } from "./__fixtures__/configuration-source"; import { testExecution } from "./__fixtures__/execution"; import type { PersistedOpsJob } from "./journal"; import { createRegistryResourceClaimer } from "./policy"; @@ -30,6 +36,7 @@ import { createFileTrustedOpsDataSource } from "./trusted-config-source"; const roots: string[] = []; afterEach(() => { + vi.unstubAllEnvs(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); async function bounded(promise: Promise): Promise { @@ -81,6 +88,7 @@ function fixture() { ...hooks, }); const operation = { + ...(await configurationSource(runtime, infraDir)), expectedCurrentRevisionId: coreServices.currentConfigurationGeneration(infraDir), kind: "stack.render", revisionId: revision(letter), @@ -147,6 +155,7 @@ describe("trusted configuration ownership and outcomes", () => { env: {}, }); const operation = { + ...(await configurationSource(runtime, f.infraDir)), kind, revisionId: revision("c"), expectedCurrentRevisionId: capturedRevision, @@ -313,6 +322,7 @@ describe("trusted configuration ownership and outcomes", () => { env: {}, }); const claimed = { + ...(await configurationSource(runtime, f.infraDir)), kind: "stack.render" as const, revisionId: revision("a"), expectedCurrentRevisionId: null, @@ -344,6 +354,8 @@ describe("trusted configuration ownership and outcomes", () => { }); it("binds strict expected-current metadata into operation identity", () => { const base = { + expectedSourceRevision: "00000000-0000-4000-8000-000000000000", + coordinatorMode: "require-running" as const, expectedCurrentRevisionId: null, kind: "stack.render" as const, revisionId: revision("a"), @@ -374,11 +386,29 @@ describe("authenticated conditional configuration admission", () => { role: "ops-agent", serviceIds: ["alpha"], }); - const requestedDeadlineAt = Date.now() + 60_000; - const operationKey = createControlRootOperationKey({ - notAfter: requestedDeadlineAt, - nonce: randomBytes(16), + // Portable API-producer fixture. This is not a PostgreSQL ownership proof; + // the current PG suite covers the actual durable job/checkpoint boundary. + const parentContext: JobContext = { + jobId: randomUUID(), + execution: { owner: randomUUID(), version: 1 }, + payload: {}, + signal: new AbortController().signal, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (value) => { + parentContext.checkpointResult = value; + }, + }; + const prepared = prepareApiJobRoot(parentContext, "stack.render", "ownership.render"), + { operationKey, requestedDeadlineAt } = prepared; + const sourceRuntime = createUnavailableRuntime(); + installTrustedConfigurationRuntime(sourceRuntime, { + infraDir: f.infraDir, + services: f.services, + integrationSchemas: new Map(), + env: {}, }); + const source = await configurationSource(sourceRuntime, f.infraDir); const queue = join(f.root, "queue"); mkdirSync(queue, { mode: 0o700 }); mkdirSync(join(queue, ".claimed"), { mode: 0o700 }); @@ -388,6 +418,7 @@ describe("authenticated conditional configuration admission", () => { nonce: "nonce_conditional1234567", operationKey, operationForRevision: (revisionId) => ({ + ...source, kind: "stack.render", revisionId, expectedCurrentRevisionId: null, @@ -399,6 +430,17 @@ describe("authenticated conditional configuration admission", () => { serviceSecrets: [], }, }); + const tokenPath = join(f.root, "api-token"); + writeFileSync(tokenPath, token, { mode: 0o600 }); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenPath); + const attempt = prepareApiJobChildAttempt( + parentContext, + sealed.operation, + operationKey, + prepared, + )!; + await parentContext.checkpoint({ _parentExecution: prepared.parent, attempt }); + const parentAssertion = await issueApiJobChildAssertion(attempt); writeFileSync(join(queue, `${sealed.revisionId}.json`), sealed.bytes, { mode: 0o600 }); const fileSource = createFileTrustedOpsDataSource({ directory: queue, @@ -496,11 +538,11 @@ describe("authenticated conditional configuration admission", () => { runtime, }); let counter = 0; - const submit = (operation: OpsOperation, key = operationKey) => + const submit = (operation: OpsOperation, key = operationKey, assertion = parentAssertion) => app.inject({ method: "POST", url: "/v1/operations", - headers: { authorization: `Bearer ${token}` }, + headers: { authorization: `Bearer ${token}`, "x-openmapx-parent-execution": assertion }, payload: { version: 2, execution: { requestedDeadlineAt }, @@ -611,10 +653,25 @@ describe("authenticated conditional configuration admission", () => { expect(replay.json().result).toMatchObject({ operationId: id, state: "succeeded" }); expect(executions).toBe(1); expect(f.label()).toBe("b"); - const changed = await submit({ + const changedBody = { ...sealed.operation, expectedCurrentRevisionId: `cfg1_${"b".repeat(43)}`, - } as OpsOperation); + } as OpsOperation; + const changedAttempt = prepareApiJobChildAttempt( + parentContext, + changedBody, + operationKey, + prepared, + )!; + await parentContext.checkpoint({ + _parentExecution: prepared.parent, + attempt: changedAttempt, + }); + const changed = await submit( + changedBody, + operationKey, + await issueApiJobChildAssertion(changedAttempt), + ); expect(changed.statusCode).toBe(409); } finally { returnClaim.resolve(); diff --git a/apps/ops-agent/src/trusted-config-runtime.test.ts b/apps/ops-agent/src/trusted-config-runtime.test.ts index 059708b32..a02155feb 100644 --- a/apps/ops-agent/src/trusted-config-runtime.test.ts +++ b/apps/ops-agent/src/trusted-config-runtime.test.ts @@ -20,7 +20,9 @@ import { } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { configurationSource } from "./__fixtures__/configuration-source"; import { testExecution } from "./__fixtures__/execution"; +import { configurationPolicyRevision } from "./configured-resource-authority"; import { createUnavailableRuntime, dispatchOpsOperation, type OpsTrustedClaim } from "./runtime"; import { initializeTrustedConfigurationRuntime, @@ -142,12 +144,13 @@ describe("ops-agent trusted configuration runtime", () => { }); const revisionId = "cfg1_0123456789abcdef0123456789abcdef0123456789a"; const secretValue = Buffer.from([9, 8, 7, 6]).toString("base64url"); + select(paths.infraDir, ["beta"]); const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "serviceSelection.apply" as const, revisionId, }; - select(paths.infraDir, ["beta"]); const trusted = claim( operation, payload({ @@ -200,6 +203,7 @@ describe("ops-agent trusted configuration runtime", () => { }); const revisionId = "cfg1_0123456789abcdef0123456789abcdef0123456789a"; const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render" as const, revisionId, @@ -281,6 +285,7 @@ describe("ops-agent trusted configuration runtime", () => { dispatchOpsOperation( runtime, { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render", revisionId, @@ -291,6 +296,7 @@ describe("ops-agent trusted configuration runtime", () => { emitLog: vi.fn(), claim: claim( { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render", revisionId, @@ -315,6 +321,7 @@ describe("ops-agent trusted configuration runtime", () => { }); const revisionId = "cfg1_0123456789abcdef0123456789abcdef0123456789a"; const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render" as const, revisionId, @@ -350,12 +357,13 @@ describe("ops-agent trusted configuration runtime", () => { infraDir: paths.infraDir, }); const revisionId = "cfg1_0123456789abcdef0123456789abcdef0123456789a"; + select(paths.infraDir, ["app-api", "alpha"]); const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render" as const, revisionId, }; - select(paths.infraDir, ["app-api", "alpha"]); await dispatchOpsOperation(runtime, operation, { execution: testExecution(), signal: new AbortController().signal, @@ -425,6 +433,7 @@ describe("ops-agent trusted configuration runtime", () => { }); const revisionId = `cfg1_${"v".repeat(43)}`; const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "integrationConfig.apply" as const, integrationId: "search-nlp", @@ -465,6 +474,7 @@ describe("ops-agent trusted configuration runtime", () => { }); const revisionId = `cfg1_${"w".repeat(43)}`; const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "serviceConfig.apply" as const, serviceId: "dawarich-app", @@ -508,6 +518,7 @@ describe("ops-agent trusted configuration runtime", () => { }); const revisionId = `cfg1_${"x".repeat(43)}`; const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "integrationConfig.apply" as const, integrationId: "custom", @@ -555,6 +566,7 @@ describe("ops-agent trusted configuration runtime", () => { }; const selectionRevision = `cfg1_${"i".repeat(43)}`; const selection = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "serviceSelection.apply" as const, revisionId: selectionRevision, @@ -571,7 +583,7 @@ describe("ops-agent trusted configuration runtime", () => { serviceConfigs: [{ serviceId: "community-live", values: {} }], integrationConfigs: [{ integrationId: "integration-live", values: { region: "eu" } }], }), - { authorityRevision: authority.revisionId }, + { authorityRevision: configurationPolicyRevision(authority.revisionId) }, ), }), ).resolves.toMatchObject({ @@ -590,6 +602,7 @@ describe("ops-agent trusted configuration runtime", () => { }; const configRevision = `cfg1_${"j".repeat(43)}`; const config = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "integrationConfig.apply" as const, integrationId: "integration-live", @@ -606,7 +619,7 @@ describe("ops-agent trusted configuration runtime", () => { serviceConfigs: [{ serviceId: "community-live", values: {} }], integrationConfigs: [{ integrationId: "integration-live", values: { region: "eu" } }], }), - { authorityRevision: authority.revisionId }, + { authorityRevision: configurationPolicyRevision(authority.revisionId) }, ), }), ).resolves.toMatchObject({ outcome: "not_committed", reason: "failed" }); @@ -618,6 +631,7 @@ describe("ops-agent trusted configuration runtime", () => { }; const removedRevision = `cfg1_${"k".repeat(43)}`; const removed = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "serviceSelection.apply" as const, revisionId: removedRevision, @@ -634,7 +648,7 @@ describe("ops-agent trusted configuration runtime", () => { serviceConfigs: [{ serviceId: "community-live", values: {} }], integrationConfigs: [], }), - { authorityRevision: authority.revisionId }, + { authorityRevision: configurationPolicyRevision(authority.revisionId) }, ), }), ).resolves.toMatchObject({ outcome: "not_committed", reason: "failed" }); @@ -657,6 +671,7 @@ describe("ops-agent trusted configuration runtime", () => { dispatchOpsOperation( runtime, { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render", revisionId, @@ -667,6 +682,7 @@ describe("ops-agent trusted configuration runtime", () => { emitLog: vi.fn(), claim: claim( { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render", revisionId, @@ -698,6 +714,7 @@ describe("ops-agent trusted configuration runtime", () => { dispatchOpsOperation( runtime, { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render", revisionId, @@ -708,6 +725,7 @@ describe("ops-agent trusted configuration runtime", () => { emitLog: vi.fn(), claim: claim( { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "stack.render", revisionId, @@ -725,11 +743,7 @@ describe("ops-agent trusted configuration runtime", () => { it("resumes an identical orphan generation after a crash before the pointer swap", async () => { const paths = fixture(); const revisionId = "cfg1_0123456789abcdef0123456789abcdef0123456789a"; - const operation = { - expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), - kind: "stack.render" as const, - revisionId, - }; + const crashing = createUnavailableRuntime(); installTrustedConfigurationRuntime(crashing, { services: [service("alpha")], @@ -739,6 +753,12 @@ describe("ops-agent trusted configuration runtime", () => { throw new Error("simulated crash"); }, }); + const operation = { + ...(await configurationSource(crashing, paths.infraDir)), + expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), + kind: "stack.render" as const, + revisionId, + }; await expect( dispatchOpsOperation(crashing, operation, { execution: testExecution(), @@ -784,17 +804,19 @@ describe("ops-agent trusted configuration runtime", () => { it("enters recovery-only state when the current pointer disappears after activation", async () => { const paths = fixture(); const revisionId = `cfg1_${"p".repeat(43)}`; - const operation = { - expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), - kind: "stack.render" as const, - revisionId, - }; + const runtime = createUnavailableRuntime(); installTrustedConfigurationRuntime(runtime, { services: [service("alpha")], integrationSchemas: new Map(), infraDir: paths.infraDir, }); + const operation = { + ...(await configurationSource(runtime, paths.infraDir)), + expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), + kind: "stack.render" as const, + revisionId, + }; await dispatchOpsOperation(runtime, operation, { execution: testExecution(), signal: new AbortController().signal, @@ -822,11 +844,7 @@ describe("ops-agent trusted configuration runtime", () => { it("rejects a direct replay after a post-pointer crash without reapplying", async () => { const paths = fixture(); const revisionId = "cfg1_0123456789abcdef0123456789abcdef0123456789a"; - const operation = { - expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), - kind: "stack.render" as const, - revisionId, - }; + const crashing = createUnavailableRuntime(); installTrustedConfigurationRuntime(crashing, { services: [service("alpha")], @@ -836,6 +854,12 @@ describe("ops-agent trusted configuration runtime", () => { throw new Error("simulated crash"); }, }); + const operation = { + ...(await configurationSource(crashing, paths.infraDir)), + expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), + kind: "stack.render" as const, + revisionId, + }; await expect( dispatchOpsOperation(crashing, operation, { execution: testExecution(), @@ -874,6 +898,7 @@ describe("ops-agent trusted configuration runtime", () => { }); const firstRevision = `cfg1_${"s".repeat(43)}`; const first = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "vault.apply" as const, serviceId: "alpha", @@ -897,6 +922,7 @@ describe("ops-agent trusted configuration runtime", () => { const secondRevision = `cfg1_${"t".repeat(43)}`; const second = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "vault.apply" as const, serviceId: "alpha", @@ -958,6 +984,7 @@ describe("ops-agent trusted configuration runtime", () => { env: {}, }); const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "serviceConfig.apply" as const, serviceId: "alpha", @@ -994,6 +1021,7 @@ describe("ops-agent trusted configuration runtime", () => { env: { OPENMAPX_ENABLED_SERVICES: "alpha" }, }); const operation = { + ...(await configurationSource(runtime, paths.infraDir)), expectedCurrentRevisionId: services.currentConfigurationGeneration(paths.infraDir), kind: "serviceSelection.apply" as const, revisionId: `cfg1_${"e".repeat(43)}`, diff --git a/apps/ops-agent/src/trusted-config-runtime.ts b/apps/ops-agent/src/trusted-config-runtime.ts index 92df24c24..63021b7d9 100644 --- a/apps/ops-agent/src/trusted-config-runtime.ts +++ b/apps/ops-agent/src/trusted-config-runtime.ts @@ -1,21 +1,55 @@ -import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, realpathSync } from "node:fs"; import { basename, dirname } from "node:path"; +import { + closeControlInstallation, + observeConfigurationSources, + openControlInstallation, + verifyConfigurationSourceWitness, +} from "@openmapx/core/build-artifacts"; import { type ConfigurationApplyResult, + type ConfigurationHardlinksPreview, + canonicalJson, + coordinatorPullSubmissionDigest, type OpsOperation, opsOperationFingerprint, type TrustedConfigurationPayload, } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; +import { assertProspectiveConfigurationMaintenancePreludeBudget } from "../../../packages/core/src/build-artifacts/configuration-maintenance-budget"; +import { + executionAuthorityState, + retainConfigurationGenerationNonCommit, +} from "../../../packages/core/src/build-artifacts/control-authority"; +import { configurationGenerationNonCommitForError } from "../../../packages/core/src/services/configuration-generation"; +import { + type ConfigurationCandidateSource, + readConfigurationCandidate, +} from "./configuration-candidate-source"; +import { assertConfigurationCoordinatorReady } from "./configuration-hardlinks-runtime"; +import { + assertStagedCoordinatorConfigurationWrite, + inspectOwnedMaintenanceGenerationRuntime, +} from "./configuration-maintenance-store"; +import { configurationPolicyRevision } from "./configured-resource-authority"; import type { OwnedServiceResolver } from "./owned-services"; -import type { OpsRuntime, OpsTrustedClaim } from "./runtime"; +import type { OpsExecutionContext, OpsRuntime, OpsTrustedClaim } from "./runtime"; +import { renderCoordinatorCandidate } from "./service-pull-runtime"; const FAILED = "Trusted configuration apply failed"; const PLAN_FAILED = "Configuration hardlink plan unavailable"; const PLAN_STALE = "Another configuration was applied before its hardlinks"; -type TrustedConfigurationOperation = Exclude< - Extract, - { kind: "configuration.hardlinks.inspect" } +type TrustedConfigurationOperation = Extract< + OpsOperation, + { + kind: + | "stack.render" + | "serviceSelection.apply" + | "serviceConfig.apply" + | "integrationConfig.apply" + | "vault.apply"; + } >; export interface TrustedConfigurationRuntimeOptions { @@ -24,6 +58,7 @@ export interface TrustedConfigurationRuntimeOptions { loadAuthority?: () => Promise; infraDir: string; ownedServices?: OwnedServiceResolver; + candidateSource?: ConfigurationCandidateSource; /** The process environment, read for the operator's selection override. */ env?: NodeJS.ProcessEnv; beforeCommit?: () => Promise; @@ -58,11 +93,13 @@ export async function initializeTrustedConfigurationRuntime(infraDir: string): P } async function commitGeneration( + runtime: OpsRuntime, options: TrustedConfigurationRuntimeOptions, operation: TrustedConfigurationOperation, payload: TrustedConfigurationPayload, claim: OpsTrustedClaim, signal: AbortSignal, + executionContext?: OpsExecutionContext, ): Promise { try { return await coreServices.withConfigurationGenerationWriter( @@ -79,25 +116,64 @@ async function commitGeneration( currentRevisionId, cleanupFailed: false, }); + if (!executionContext) throw Error("Actual configuration publication context required"); + await assertStagedCoordinatorConfigurationWrite( + runtime, + operation, + executionContext, + signal, + ); const authority = await options.loadAuthority?.(); if ( authority && (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(authority.revisionId) || - claim.capability.values.authorityRevision !== authority.revisionId) + claim.capability.values.authorityRevision !== + configurationPolicyRevision(authority.revisionId, options.env)) ) { throw new Error(FAILED); } const services = authority?.services ?? options.services; + const installation = openControlInstallation({ + infraDir: options.infraDir, + role: "ops-agent", + serviceIds: services.map((service) => service.manifest.id), + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + verifyConfigurationSourceWitness({ + installation, + infraDir: options.infraDir, + sourceRevision: operation.expectedSourceRevision, + expectedCurrentRevisionId: operation.expectedCurrentRevisionId, + authorityRevision: authority?.revisionId ?? "registry-v1", + env: options.env, + }); + } finally { + closeControlInstallation(installation); + } const env = options.env ?? process.env; // Capture cleanup retention before selection effects under the same writer. - const inventory = options.ownedServices?.inspectInventory - ? await options.ownedServices.inspectInventory(signal) - : options.ownedServices - ? { - services: await options.ownedServices.inspectOwnedServices(signal), - observedContainerIds: undefined, - } + const frozen = + (operation.kind === "stack.render" || operation.kind === "vault.apply") && + operation.maintenanceId && + executionContext + ? await inspectOwnedMaintenanceGenerationRuntime( + runtime, + executionContext, + operation, + signal, + ) : undefined; + const inventory = + frozen ?? + (options.ownedServices?.inspectInventory + ? await options.ownedServices.inspectInventory(signal) + : options.ownedServices + ? { + services: await options.ownedServices.inspectOwnedServices(signal), + observedContainerIds: undefined, + } + : undefined); const ownedServiceGenerations = inventory?.services.map((snapshot) => ({ containerId: snapshot.containerId, revisionIds: snapshot.provenance.composeConfigFiles @@ -137,6 +213,11 @@ async function commitGeneration( allowMissingSelected, revisionId: operation.revisionId, expectedCurrentRevisionId: operation.expectedCurrentRevisionId, + operationKey: executionContext?.execution.operationKey, + installationId: executionContext?.controlAuthority + ? executionAuthorityState(executionContext.controlAuthority).installation.installation + .installationId + : undefined, beforeCommit: options.beforeCommit, afterGenerationRename: options.afterGenerationRename, afterCommit: options.afterCommit, @@ -152,6 +233,9 @@ async function commitGeneration( signal, ); } catch (error) { + const original = configurationGenerationNonCommitForError(error); + if (original && executionContext?.controlAuthority) + retainConfigurationGenerationNonCommit(executionContext.controlAuthority, original); if (error instanceof coreServices.ConfigurationGenerationError) return error.result; return { outcome: "not_committed", reason: "failed", cleanupFailed: false }; } @@ -178,6 +262,181 @@ export function installTrustedConfigurationRuntime( runtime: OpsRuntime, options: TrustedConfigurationRuntimeOptions, ): void { + runtime["configuration.sources.inspect"] = async () => { + for (let attempt = 0; attempt < 3; attempt++) { + const authority = await options.loadAuthority?.(); + const services = authority?.services ?? options.services; + const installation = openControlInstallation({ + infraDir: options.infraDir, + role: "ops-agent", + serviceIds: services.map((service) => service.manifest.id), + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const observed = observeConfigurationSources({ + installation, + infraDir: options.infraDir, + services, + authorityRevision: authority?.revisionId ?? "registry-v1", + env: options.env, + }); + if ((await options.loadAuthority?.())?.revisionId !== authority?.revisionId) continue; + return observed; + } finally { + closeControlInstallation(installation); + } + } + throw Error("Host sources unavailable"); + }; + runtime["configuration.hardlinks.preview"] = async (operation, context) => { + if (!options.candidateSource) throw Error("Configuration candidate source unavailable"); + const candidate = await readConfigurationCandidate( + options.candidateSource, + operation.candidateId, + ); + const original = candidate.operation; + if ( + !("expectedCurrentRevisionId" in original) || + !("coordinatorMode" in original) || + original.revisionId !== operation.revisionId || + original.expectedCurrentRevisionId !== operation.expectedCurrentRevisionId || + original.expectedSourceRevision !== operation.expectedSourceRevision || + original.coordinatorMode !== operation.coordinatorMode || + candidate.record.expectedSourceRevision !== operation.expectedSourceRevision + ) + throw Error("Configuration preview identity conflict"); + await assertStagedCoordinatorConfigurationWrite(runtime, operation, context, context.signal); + const authority = await options.loadAuthority?.(); + const services = authority?.services ?? options.services; + const installation = openControlInstallation({ + infraDir: options.infraDir, + role: "ops-agent", + serviceIds: services.map((service) => service.manifest.id), + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + verifyConfigurationSourceWitness({ + installation, + infraDir: options.infraDir, + sourceRevision: operation.expectedSourceRevision, + expectedCurrentRevisionId: operation.expectedCurrentRevisionId, + authorityRevision: authority?.revisionId ?? "registry-v1", + env: options.env, + }); + } finally { + closeControlInstallation(installation); + } + const desired = coreServices.readDesiredSelection(options.infraDir, options.env ?? process.env); + const rendered = coreServices.renderConfiguration({ + infraDir: options.infraDir, + services, + integrationSchemas: authority?.integrationSchemas ?? options.integrationSchemas, + input: { + ...candidate.payload, + selectedRoots: candidate.payload.selectedRoots ?? desired.roots, + }, + allowMissingSelected: !candidate.payload.selectedRoots && desired.source === "default", + }); + const plan = rendered.rendered.hardlinkPlan; + const coordinator = candidate.record.privateCoordinatorLifecycle + ? renderCoordinatorCandidate( + candidate, + authority ?? { + revisionId: "registry-v1", + services, + integrationSchemas: options.integrationSchemas, + }, + options.infraDir, + ) + : undefined; + const genericDmRecreate = + candidate.record.apiFollowUp === "service-apply" && + candidate.record.affectedServiceIds.includes("data-manager"); + const selectedDmImage = + !coordinator && genericDmRecreate + ? coreServices.readServiceReleaseImageSelection(options.infraDir, "data-manager") + : undefined; + const prospectiveCoordinatorImage = coordinator + ? { + action: coordinator.fixed.action, + noDeps: coordinator.fixed.noDeps, + imageReference: coordinator.imageReference, + releaseSelectionDigest: coordinator.releaseSelectionDigest, + pullSubmissionDigest: + coordinator.fixed.action === "update" + ? coordinatorPullSubmissionDigest(coordinator.fixed.pull) + : null, + } + : selectedDmImage + ? { + action: "apply" as const, + noDeps: false, + imageReference: + selectedDmImage.imageReference ?? + coreServices.configurationImageReference( + rendered.rendered.composeYaml, + "data-manager", + ), + releaseSelectionDigest: selectedDmImage.releaseSelectionDigest, + pullSubmissionDigest: null, + } + : undefined; + if (candidate.record.maintenanceId) { + if (!options.ownedServices) throw Error("Actual maintenance ownership inventory unavailable"); + assertProspectiveConfigurationMaintenancePreludeBudget({ + installationId: installation.installationId, + record: candidate.record, + sourceAuthorityRevision: authority?.revisionId ?? "registry-v1", + infraDir: realpathSync(options.infraDir), + currentServiceIds: [ + ...(coreServices.readAppliedServiceIds(options.infraDir, services) ?? []), + ], + originalContainers: await options.ownedServices.inspectOwnedServices(context.signal), + rendered, + ...(prospectiveCoordinatorImage ? { coordinatorImage: prospectiveCoordinatorImage } : {}), + }); + } + if (plan.length > 256) throw Error("Configuration preview budget exceeded"); + let coordinatorState: ConfigurationHardlinksPreview["coordinatorState"] = "not_required"; + if (plan.length || candidate.record.privateCoordinatorLifecycle) { + const dm = rendered.resolved.enabled.find( + (service) => service.manifest.id === "data-manager", + ); + if (!dm) coordinatorState = "disabled"; + else { + try { + const status = await runtime["stack.status"]({ kind: "stack.status" }, context); + const dm = status.services.find((service) => service.serviceId === "data-manager"); + coordinatorState = + dm?.state === "running" && (dm.health === "healthy" || dm.health === "none") + ? "running" + : "stopped"; + if (coordinatorState === "running") + await assertConfigurationCoordinatorReady(context.signal); + } catch { + coordinatorState = "unavailable"; + } + } + } + const requiredServiceIds = [...new Set(plan.map((entry) => entry.consumerService))].sort(); + return { + entries: plan.length, + requiredServiceIds, + coordinatorState, + previewDigest: createHash("sha256") + .update("openmapx-configuration-hardlinks-preview-v1\0") + .update( + canonicalJson({ + candidateId: operation.candidateId, + revisionId: operation.revisionId, + expectedSourceRevision: operation.expectedSourceRevision, + coordinatorMode: operation.coordinatorMode, + plan, + }), + ) + .digest("hex"), + }; + }; runtime["configuration.inspect"] = (_operation, context) => coreServices.withConfigurationGenerationWriter( options.infraDir, @@ -186,11 +445,13 @@ export function installTrustedConfigurationRuntime( ); runtime["stack.render"] = (operation, context) => commitGeneration( + runtime, options, operation, trustedPayload(operation, context), context.claim, context.signal, + context, ); // The plan lives in the applied generation, which only this service can // read. Data-manager applies it: its mounts confine links and prunes to the @@ -221,17 +482,27 @@ export function installTrustedConfigurationRuntime( ); runtime["serviceSelection.apply"] = (operation, context) => commitGeneration( + runtime, options, operation, trustedPayload(operation, context), context.claim, context.signal, + context, ); runtime["serviceConfig.apply"] = async (operation, context) => { const payload = trustedPayload(operation, context); if (!payload.serviceConfigs.some((entry) => entry.serviceId === operation.serviceId)) throw new Error(FAILED); - return commitGeneration(options, operation, payload, context.claim, context.signal); + return commitGeneration( + runtime, + options, + operation, + payload, + context.claim, + context.signal, + context, + ); }; runtime["integrationConfig.apply"] = async (operation, context) => { const payload = trustedPayload(operation, context); @@ -240,13 +511,29 @@ export function installTrustedConfigurationRuntime( ) { throw new Error(FAILED); } - return commitGeneration(options, operation, payload, context.claim, context.signal); + return commitGeneration( + runtime, + options, + operation, + payload, + context.claim, + context.signal, + context, + ); }; runtime["vault.apply"] = async (operation, context) => { const payload = trustedPayload(operation, context); if (!payload.serviceSecrets.some((entry) => entry.serviceId === operation.serviceId)) throw new Error(FAILED); - return commitGeneration(options, operation, payload, context.claim, context.signal); + return commitGeneration( + runtime, + options, + operation, + payload, + context.claim, + context.signal, + context, + ); }; runtime["serviceSelection.inspect"] = async () => { try { diff --git a/apps/ops-agent/src/trusted-config-source.test.ts b/apps/ops-agent/src/trusted-config-source.test.ts index 1d8269717..c756f9a08 100644 --- a/apps/ops-agent/src/trusted-config-source.test.ts +++ b/apps/ops-agent/src/trusted-config-source.test.ts @@ -60,6 +60,8 @@ function sealed(issuedAtMs = now) { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), @@ -76,6 +78,8 @@ function distinctSealed(index: number) { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId, }), diff --git a/apps/ops-agent/src/trusted-config-source.ts b/apps/ops-agent/src/trusted-config-source.ts index fa1e80f65..883b81c6e 100644 --- a/apps/ops-agent/src/trusted-config-source.ts +++ b/apps/ops-agent/src/trusted-config-source.ts @@ -17,7 +17,9 @@ import { join } from "node:path"; import type { OpsJobState, OpsOperation } from "@openmapx/core/ops"; import { acquireTrustedConfigurationQueueLock, + inspectPreparedConfigurationRecord, inspectTrustedConfigurationSnapshot, + isTrustedConfigurationApplyOperation, OPS_TRUSTED_CONFIG_MAX_BYTES, OPS_TRUSTED_CONFIG_QUEUE_LOCK_NAME, openTrustedConfigurationSnapshot, @@ -108,7 +110,7 @@ function journalRevisionIds(policy: TrustedSnapshotDirectoryPolicy): { for (const record of records) { const isTerminal = ["succeeded", "failed", "timed_out"].includes(record.state); if (isTerminal !== (record.terminalAt !== undefined)) throw new Error(DIRECTORY_REJECTED); - if (!("revisionId" in record.operation)) continue; + if (!isTrustedConfigurationApplyOperation(record.operation)) continue; if (!/^cfg1_[A-Za-z0-9_-]{43}$/.test(record.operation.revisionId)) { throw new Error(DIRECTORY_REJECTED); } @@ -184,6 +186,7 @@ function validateQueueFile( path: string, policy: TrustedSnapshotDirectoryPolicy, expectedLinks = 1, + maxBytes = OPS_TRUSTED_CONFIG_MAX_BYTES, ): QueueFileIdentity { const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); try { @@ -197,7 +200,7 @@ function validateQueueFile( !policy.allowedUids.includes(stats.uid) || stats.gid !== policy.expectedGid || (stats.mode & 0o777) !== SNAPSHOT_MODE || - stats.size > OPS_TRUSTED_CONFIG_MAX_BYTES + stats.size > maxBytes ) { throw new Error(DIRECTORY_REJECTED); } @@ -224,6 +227,7 @@ function validateQueue( ): QueueCleanupPlan[] { const cleanup = new Map(); const revisions = new Set(); + const retainedRevisions = new Set(); const nowMs = policy.nowMs ?? Date.now(); let entries = 0; let bytes = 0; @@ -293,6 +297,7 @@ function validateQueue( } for (const name of directoryNames) { if (name === ".claimed") continue; + if (name === ".candidates") continue; if (name === OPS_TRUSTED_CONFIG_QUEUE_LOCK_NAME) continue; if (!SNAPSHOT_NAME.test(name) && !TEMPORARY_NAME.test(name) && !ABORT_NAME.test(name)) { throw new Error(DIRECTORY_REJECTED); @@ -341,6 +346,7 @@ function validateQueue( else { retainedEntries += 1; retainedBytes += metadata.size; + retainedRevisions.add(revisionId); } } } @@ -366,8 +372,37 @@ function validateQueue( else { retainedEntries += 1; retainedBytes += metadata.size; + retainedRevisions.add(revisionId); } } + if (directoryNames.includes(".candidates")) { + const directoryPath = join(directory, ".candidates"); + const candidateDirectoryIdentity = captureQueueDirectory(directoryPath, policy); + const names = readdirSync(directoryPath); + if (names.length > 128) throw new Error(DIRECTORY_REJECTED); + const candidateRevisions = new Set(); + for (const name of names) { + if (!/^opk1_[A-Za-z0-9_-]{16,64}\.json$/.test(name)) throw new Error(DIRECTORY_REJECTED); + const metadata = validateQueueFile(join(directoryPath, name), policy, 1, 768 * 1024); + const inspected = inspectPreparedConfigurationRecord(metadata.bytes, policy.token); + if ( + `${inspected.record.operationKey}.json` !== name || + candidateRevisions.has(inspected.snapshot.revisionId) + ) + throw new Error(DIRECTORY_REJECTED); + candidateRevisions.add(inspected.snapshot.revisionId); + entries++; + bytes += metadata.size; + if (retainedRevisions.has(inspected.snapshot.revisionId)) + retainedBytes += metadata.size - inspected.sealed.length; + else { + retainedEntries++; + retainedBytes += metadata.size; + retainedRevisions.add(inspected.snapshot.revisionId); + } + } + captureQueueDirectory(directoryPath, policy, candidateDirectoryIdentity); + } if ( entries > MAX_SCAN_ENTRIES || bytes > MAX_SCAN_BYTES || @@ -549,7 +584,7 @@ async function syncDirectory(path: string): Promise { } function revisionFor(operation: Parameters[0]): string | null { - if (!("revisionId" in operation) || typeof operation.revisionId !== "string") return null; + if (!isTrustedConfigurationApplyOperation(operation)) return null; return /^cfg1_[A-Za-z0-9_-]{43}$/.test(operation.revisionId) ? operation.revisionId : null; } diff --git a/apps/web/src/hooks/useServices.test.tsx b/apps/web/src/hooks/useServices.test.tsx index d897cef0b..4a5d0f15e 100644 --- a/apps/web/src/hooks/useServices.test.tsx +++ b/apps/web/src/hooks/useServices.test.tsx @@ -96,6 +96,20 @@ describe("service feature hooks use the generated SDK", () => { ); }); expect(fetch).toHaveBeenCalledTimes(1); + if (["config", "set", "delete", "selection"].includes(kind)) { + const init = fetch.mock.calls[0]![1] as RequestInit; + expect(new Headers(init.headers).get("idempotency-key")).toMatch( + /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/, + ); + if (kind !== "delete") + expect(JSON.parse(String(init.body))).toEqual( + kind === "config" + ? { config: { count: 2 } } + : kind === "set" + ? { value: "synthetic-private" } + : { selectedRoots: [] }, + ); + } if (kind === "set" || kind === "delete") expect(fetch.mock.calls[0][0]).toContain("TOKEN%20%2Fkey"); }, diff --git a/apps/web/src/hooks/useServices.ts b/apps/web/src/hooks/useServices.ts index 00c74d801..d292d310d 100644 --- a/apps/web/src/hooks/useServices.ts +++ b/apps/web/src/hooks/useServices.ts @@ -62,7 +62,14 @@ export function useServiceConfigSave(id: string) { (client, config: Record) => { const body = sdk.adminServiceConfigUpdateSchema.safeParse({ config }); if (!body.success) throw new Error("Invalid service configuration"); - return serviceOrThrow(sdk.updateAdminServiceConfig(id, body.data, { client })); + return serviceOrThrow( + sdk.updateAdminServiceConfig( + id, + body.data, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ), + ); }, ); } @@ -77,12 +84,28 @@ export function useSetServiceCredential(id: string) { return useServiceWrite( { kind: "credentials", serviceId: id }, (client, { key, value }: { key: string; value: string }) => - serviceOrThrow(sdk.setAdminServiceCredential(id, key, { value }, { client })), + serviceOrThrow( + sdk.setAdminServiceCredential( + id, + key, + { value }, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ), + ), ); } export function useDeleteServiceCredential(id: string) { return useServiceWrite({ kind: "credentials", serviceId: id }, (client, key: string) => - serviceOrThrow(sdk.deleteAdminServiceCredential(id, key, undefined, { client })), + serviceOrThrow( + sdk.deleteAdminServiceCredential( + id, + key, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ), + ), ); } export function useServiceHealthCheck() { @@ -92,7 +115,13 @@ export function useServiceHealthCheck() { } export function useServiceSelectionSave() { return useServiceWrite({ kind: "catalog" }, (client, selectedRoots: string[]) => - serviceOrThrow(sdk.setAdminServiceSelection({ selectedRoots }, { client })), + serviceOrThrow( + sdk.setAdminServiceSelection( + { selectedRoots }, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ), + ), ); } export function useServicesBulkAction() { diff --git a/apps/web/src/lib/admin/integrationAdmin.test.tsx b/apps/web/src/lib/admin/integrationAdmin.test.tsx index affe8ffb9..57129deb7 100644 --- a/apps/web/src/lib/admin/integrationAdmin.test.tsx +++ b/apps/web/src/lib/admin/integrationAdmin.test.tsx @@ -56,6 +56,43 @@ afterEach(() => { }); describe("integration query boundaries and reload observation", () => { + it.each(["patch", "set", "delete", "enable", "disable", "import"] as const)( + "sends the generated %s body and retained header in their actual positions", + async (kind) => { + const requests = fakeFetch(() => + Response.json({ error: "Request rejected" }, { status: 400 }), + ); + const qc = client(), + { result } = renderHookWithQuery(useIntegrationAdmin, { queryClient: qc }); + await act(async () => { + const run = { + patch: () => result.current.patchConfig("demo", { endpoint: "synthetic" }), + set: () => result.current.setCredential("demo", "TOKEN", "synthetic-private"), + delete: () => result.current.deleteCredential("demo", "TOKEN"), + enable: () => result.current.toggle("demo", true), + disable: () => result.current.toggle("demo", false), + import: () => + result.current.importConfigs({ + integrations: [{ integrationId: "demo", config: { endpoint: "synthetic" } }], + }), + }; + await expect(run[kind]()).rejects.toThrow(); + }); + expect(requests).toHaveLength(1); + const init = requests[0]!.init; + expect(new Headers(init.headers).get("idempotency-key")).toMatch( + /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/, + ); + if (["patch", "set", "import"].includes(kind)) + expect(JSON.parse(String(init.body))).toEqual( + kind === "patch" + ? { endpoint: "synthetic" } + : kind === "set" + ? { value: "synthetic-private" } + : { integrations: [{ integrationId: "demo", config: { endpoint: "synthetic" } }] }, + ); + }, + ); it("keeps full raw manifest and canonical config values without response casts", async () => { const manifest = { id: "demo", diff --git a/apps/web/src/lib/admin/integrationAdmin.ts b/apps/web/src/lib/admin/integrationAdmin.ts index 948e55e66..8c025b3ea 100644 --- a/apps/web/src/lib/admin/integrationAdmin.ts +++ b/apps/web/src/lib/admin/integrationAdmin.ts @@ -123,8 +123,18 @@ export function useIntegrationAdmin() { toggle: (id: string, enabled: boolean) => integrationOrThrow( enabled - ? sdk.enableAdminIntegration(id, undefined, { client }) - : sdk.disableAdminIntegration(id, undefined, { client }), + ? sdk.enableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ) + : sdk.disableAdminIntegration( + id, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ), ), healthSweep: () => integrationOrThrow(sdk.runAdminIntegrationHealthChecks(undefined, { client })), @@ -133,13 +143,30 @@ export function useIntegrationAdmin() { sdk.patchAdminIntegrationConfig( id, sdk.adminIntegrationConfigPatchSchema.parse(updates), + { "idempotency-key": crypto.randomUUID() }, { client }, ), ), setCredential: (id: string, key: string, value: string) => - integrationOrThrow(sdk.setAdminIntegrationCredential(id, key, { value }, { client })), + integrationOrThrow( + sdk.setAdminIntegrationCredential( + id, + key, + { value }, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ), + ), deleteCredential: (id: string, key: string) => - integrationOrThrow(sdk.deleteAdminIntegrationCredential(id, key, undefined, { client })), + integrationOrThrow( + sdk.deleteAdminIntegrationCredential( + id, + key, + { "idempotency-key": crypto.randomUUID() }, + undefined, + { client }, + ), + ), setBinding: (id: string, capability: string, serviceId: string | null) => integrationOrThrow( serviceId === null @@ -154,7 +181,11 @@ export function useIntegrationAdmin() { ); try { return await integrationOrThrow( - sdk.importAdminIntegrationConfigs(parsed.data, { client }), + sdk.importAdminIntegrationConfigs( + parsed.data, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ), ); } finally { void invalidateIntegrationAdmin(qc, apiUrl); diff --git a/apps/web/src/lib/admin/integrationErrors.test.ts b/apps/web/src/lib/admin/integrationErrors.test.ts new file mode 100644 index 000000000..5f3578a42 --- /dev/null +++ b/apps/web/src/lib/admin/integrationErrors.test.ts @@ -0,0 +1,42 @@ +import { ApiClientError } from "@openmapx/core"; +import { expect, it } from "vitest"; +import { integrationErrorMessage } from "./integrationErrors"; + +it("describes a preflight refusal without claiming the imported rows were saved", () => { + const message = integrationErrorMessage( + new ApiClientError( + 409, + { + ok: false, + error: "Integration import failed", + phase: "persistence", + recovery: "reconciliation_required", + database: "retained", + runtime: "unchanged", + }, + null, + ), + "Import failed", + ); + expect(message).toContain("The saved configuration is unchanged."); + expect(message).not.toContain("The imported configuration is saved."); +}); +it("retains the desired-versus-runtime distinction after a committed import fails reload", () => { + const message = integrationErrorMessage( + new ApiClientError( + 409, + { + ok: false, + error: "Integration import failed", + phase: "reload", + recovery: "reconciliation_required", + database: "retained", + runtime: "unknown", + }, + null, + ), + "Import failed", + ); + expect(message).toContain("The imported configuration is saved."); + expect(message).toContain("The running configuration is uncertain."); +}); diff --git a/apps/web/src/lib/admin/integrationErrors.ts b/apps/web/src/lib/admin/integrationErrors.ts index a3369a898..9cfac64e6 100644 --- a/apps/web/src/lib/admin/integrationErrors.ts +++ b/apps/web/src/lib/admin/integrationErrors.ts @@ -5,12 +5,15 @@ import type { AdminIntegrationImportFailure } from "@openmapx/core/api-contracts function importFailureMessage(failure: AdminIntegrationImportFailure): string { if (failure.recovery === "restored") return "Import failed. Previous saved and applied configuration was restored."; - const saved = { - retained: "The imported configuration is saved.", - restored: "The previous saved configuration was restored.", - conflict: "Saved configuration changed during the import; newer edits were preserved.", - unknown: "The saved configuration is uncertain.", - }[failure.database]; + const saved = + failure.phase === "persistence" && failure.database === "retained" + ? "The saved configuration is unchanged." + : { + retained: "The imported configuration is saved.", + restored: "The previous saved configuration was restored.", + conflict: "Saved configuration changed during the import; newer edits were preserved.", + unknown: "The saved configuration is uncertain.", + }[failure.database]; const applied = failure.runtime === "restored" ? "The previous running configuration was restored." diff --git a/packages/cli/__tests__/cli-control-context.test.ts b/packages/cli/__tests__/cli-control-context.test.ts index 210891e13..97d5b5c04 100644 --- a/packages/cli/__tests__/cli-control-context.test.ts +++ b/packages/cli/__tests__/cli-control-context.test.ts @@ -8,8 +8,13 @@ import { executionFingerprint, opsOperationFingerprint, } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; import { expect, it, vi } from "vitest"; -import { completeCliCommand, withCliControlContext } from "../src/lib/control-context"; +import { + cliControlContext, + completeCliCommand, + withCliControlContext, +} from "../src/lib/control-context"; it("settles successful explicit CLI exits once through the real control context", async () => { const repo = mkdtempSync(join(tmpdir(), "omx-fixed-cli-entry-")); @@ -21,7 +26,11 @@ it("settles successful explicit CLI exits once through the real control context" role: "ops-agent", serviceIds: ["motis"], }); - const operation = { kind: "service.start" as const, serviceId: "motis" }; + const operation = { + kind: "service.start" as const, + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }; const operationFingerprint = opsOperationFingerprint(operation); const deadlineAt = Date.now() + 60000; const execution = { @@ -77,6 +86,21 @@ it("settles successful explicit CLI exits once through the real control context" }); const args = c.fixedCliArguments(operation); c.retainFixedCliSpawnIntent(authority, { delegationId, execution: child, operation, args }); + const receiptOptions = { + dataRoot: join(infra, "data"), + execution: child, + serviceIds: ["motis"], + action: "start" as const, + noDeps: false, + coordinatorMode: "ensure-running" as const, + expectedRevisionId: operation.expectedConfigurationRevisionId, + seedRevisionId: operation.expectedConfigurationRevisionId, + }; + await services.withServiceLifecycleReceiptAuthority( + authority, + () => services.openServiceLifecycleReceipt(receiptOptions), + { execution: child, publicKey }, + ); vi.stubEnv("OPENMAPX_ROOT_DIR", repo); vi.stubEnv("OPENMAPX_CONTROL_EXECUTION", JSON.stringify(child)); vi.stubEnv("OPENMAPX_CONTROL_DELEGATION", assertion); @@ -84,10 +108,37 @@ it("settles successful explicit CLI exits once through the real control context" let calls = 0; try { await withCliControlContext(args, async () => { + const context = cliControlContext()!; + const store = await services.withServiceLifecycleReceiptAuthority(context.authority, () => + services.openServiceLifecycleReceipt(receiptOptions), + ); + await services.withConfigurationGenerationWriter(infra, (writer) => + services.withVerifiedLifecycleOwnership( + { writer, lease: context.lease!, authority: context.authority }, + async (ownership) => { + const receipt = await store.read(); + await store.checkpoint( + receipt.sequence, + { ...receipt, sequence: receipt.sequence + 1, phase: "pins_validated" }, + ownership, + ); + }, + ), + ); calls++; completeCliCommand(0); }); expect(calls).toBe(1); + const childInstallation = c.openFixedChildInstallation({ infraDir: infra, assertion }); + try { + const childAuthority = c.restoreAdmittedControlExecution(childInstallation, child); + const receipt = await services.withServiceLifecycleReceiptAuthority(childAuthority, () => + services.readServiceLifecycleReceipt(receiptOptions), + ); + expect(receipt.phase).toBe("pins_validated"); + } finally { + c.closeControlInstallation(childInstallation); + } await expect( withCliControlContext(args, async () => { calls++; diff --git a/packages/cli/__tests__/cli-service-lifecycle.test.ts b/packages/cli/__tests__/cli-service-lifecycle.test.ts new file mode 100644 index 000000000..fd7a9204e --- /dev/null +++ b/packages/cli/__tests__/cli-service-lifecycle.test.ts @@ -0,0 +1,319 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; +import { Command } from "commander"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { renderComposeForRepo } from "../src/commands/compose"; +import { registerServicesCommands } from "../src/commands/services"; +import { dockerComposeStream } from "../src/lib/docker"; +import { selectRelease } from "../src/lib/release"; + +// These exercise fsynced cross-process authority/retention stores; concurrent +// package checks can exceed the generic 15s unit budget on this filesystem. +vi.setConfig({ testTimeout: 60000 }); + +// Named external Docker process edge; real manifests, renderer, files and command entry remain. +vi.mock("../src/lib/docker", async (original) => ({ + ...(await original()), + dockerComposeStream: vi.fn(), +})); +let root: string; +let seed: string; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "openmapx-owned-lifecycle-")); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(join(root, "infra/docker"), { recursive: true }); + for (const id of ["alpha", "beta"]) { + const directory = join(root, "services", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "service.json"), + JSON.stringify({ + id, + name: id, + version: "1.0.0", + quality: "built-in", + container: { image: `fixture/${id}`, tag: "1" }, + }), + ); + } + for (const [key, value] of Object.entries({ + OPENMAPX_ROOT_DIR: root, + OPENMAPX_RELEASE: "local", + DOMAIN: "example.test", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + DATA_MANAGER_AUTH_TOKEN: "d".repeat(32), + })) + vi.stubEnv(key, value); + vi.stubEnv("OPENMAPX_ENABLED_SERVICES", ""); + services.writeServiceSelectionFile(join(root, "infra/docker"), ["alpha", "beta"]); + seed = (await renderComposeForRepo({ rootDir: root, domain: "example.test" })).revisionId; + vi.mocked(dockerComposeStream).mockReset().mockResolvedValue(0); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process, "exit").mockImplementation((code) => { + throw Error(`exit:${code}`); + }); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + if (root) rmSync(root, { recursive: true, force: true }); +}); +function command(action: "start" | "update", args: string[] = []) { + const program = new Command(); + registerServicesCommands(program); + return program + .parseAsync(["services", action, "alpha", ...args], { from: "user" }) + .catch((error) => { + if (String(error).includes("exit:0")) throw error; + throw Error( + `${String(error)}; command diagnostics: ${JSON.stringify(vi.mocked(console.error).mock.calls)}`, + ); + }); +} +async function retainedReceipt() { + const dataRoot = join(root, "infra/docker/data"), + directory = join(dataRoot, "ops-agent/lifecycle"); + const names = readdirSync(directory); + expect(names).toHaveLength(1); + const envelope = JSON.parse(readFileSync(join(directory, names[0]!), "utf8")); + return services.readServiceLifecycleReceipt({ + dataRoot, + execution: envelope.seed.payload.execution, + }); +} +describe("actual CLI service lifecycle preparation", () => { + it.each(["data-manager", "alpha"])( + "refuses ordinary CLI Update of %s when it would recreate the coordinator", + async (target) => { + const directory = join(root, "services/data-manager"); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "service.json"), + JSON.stringify({ + id: "data-manager", + name: "data-manager", + version: "1.0.0", + quality: "built-in", + container: { image: "fixture/data-manager", tag: "1" }, + }), + ); + if (target === "alpha") { + const path = join(root, "services/alpha/service.json"), + manifest = JSON.parse(readFileSync(path, "utf8")); + manifest.container.dependsOn = [{ service: "data-manager" }]; + writeFileSync(path, JSON.stringify(manifest)); + } + services.writeServiceSelectionFile(join(root, "infra/docker"), [ + "alpha", + "beta", + "data-manager", + ]); + seed = (await renderComposeForRepo({ rootDir: root, domain: "example.test" })).revisionId; + const program = new Command(); + registerServicesCommands(program); + await expect( + program.parseAsync(["services", "update", target], { from: "user" }), + ).rejects.toThrow("exit:1"); + expect(vi.mocked(dockerComposeStream).mock.calls).toHaveLength(0); + expect(services.currentConfigurationGeneration(join(root, "infra/docker"))).toBe(seed); + }, + ); + it("holds writer and physical target ownership through target-only Start after full-selection preparation", async () => { + vi.mocked(dockerComposeStream).mockImplementation(async (args) => { + expect(args).toEqual(["up", "-d", "alpha"]); + expect(existsSync(join(root, "infra/docker/.trusted-config-writer"))).toBe(true); + expect(existsSync(join(root, "infra/docker/data/.admin-target-control/locks/alpha"))).toBe( + true, + ); + expect( + JSON.parse( + readFileSync( + services.currentConfigurationFile( + join(root, "infra/docker"), + services.SERVICE_SELECTION_FILE, + ), + "utf8", + ), + ).selected, + ).toEqual(["alpha", "beta"]); + return 0; + }); + await expect(command("start")).rejects.toThrow("exit:0"); + expect(dockerComposeStream).toHaveBeenCalledTimes(1); + expect(existsSync(join(root, "infra/docker/.trusted-config-writer"))).toBe(false); + }); + it("pulls under ownership before committing a full final generation and recreating only the target", async () => { + const calls: string[][] = []; + vi.mocked(dockerComposeStream).mockImplementation(async (args) => { + calls.push([...args]); + expect(existsSync(join(root, "infra/docker/.trusted-config-writer"))).toBe(true); + if (args[0] === "pull") + expect(services.currentConfigurationGeneration(join(root, "infra/docker"))).toBe(seed); + else { + expect(services.currentConfigurationGeneration(join(root, "infra/docker"))).not.toBe(seed); + expect( + JSON.parse( + readFileSync( + services.currentConfigurationFile( + join(root, "infra/docker"), + services.SERVICE_SELECTION_FILE, + ), + "utf8", + ), + ).selected, + ).toEqual(["alpha", "beta"]); + } + return 0; + }); + await expect(command("update")).rejects.toThrow("exit:0"); + expect(calls).toEqual([ + ["pull", "alpha"], + ["up", "-d", "--force-recreate", "alpha"], + ]); + }); + it("applies no-deps only to recreation while retaining the full selected generation", async () => { + await expect(command("update", ["--no-deps"])).rejects.toThrow("exit:0"); + expect(vi.mocked(dockerComposeStream).mock.calls.map((call) => call[0])).toEqual([ + ["pull", "alpha"], + ["up", "-d", "--force-recreate", "--no-deps", "alpha"], + ]); + const receipt = await retainedReceipt(); + expect(receipt).toMatchObject({ + phase: "complete", + noDeps: true, + followUp: "complete", + commit: "final_committed", + }); + expect( + JSON.parse( + readFileSync( + services.currentConfigurationFile( + join(root, "infra/docker"), + services.SERVICE_SELECTION_FILE, + ), + "utf8", + ), + ).selected, + ).toEqual(["alpha", "beta"]); + }); + it("retains failed pull evidence without committing a final generation or recreating", async () => { + vi.mocked(dockerComposeStream).mockResolvedValue(7); + await expect(command("update")).rejects.toThrow("exit:7"); + expect(vi.mocked(dockerComposeStream).mock.calls.map((call) => call[0])).toEqual([ + ["pull", "alpha"], + ]); + expect(services.currentConfigurationGeneration(join(root, "infra/docker"))).toBe(seed); + expect(await retainedReceipt()).toMatchObject({ + phase: "failed", + expectedRevisionId: seed, + finalRevisionId: null, + commit: "not_committed", + followUp: "not_started", + failure: { reason: "pull", failedServiceIds: ["alpha"] }, + }); + }); + it("excludes actual release reselection throughout Update pull and recreation", async () => { + const manifest = services.parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: "synthetic-next", + images: Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`, + ]), + ), + privacyReleaseValidation: { + version: 1, + sourceBuildFingerprint: "2".repeat(64), + validatedAt: "2026-09-05T12:00:00.000Z", + checks: { translationsConsistent: true, openApiConsistent: true, policyConsistent: true }, + }, + }), + "ghcr.io/openmapx", + ); + const path = join(root, "infra/docker/docker-compose.release.yml"); + vi.mocked(dockerComposeStream).mockImplementation(async () => { + await expect(selectRelease({ path, resolve: async () => manifest })).rejects.toThrow( + "Release store is busy", + ); + expect(existsSync(path)).toBe(false); + return 0; + }); + await expect(command("update")).rejects.toThrow("exit:0"); + expect(await retainedReceipt()).toMatchObject({ + phase: "complete", + commit: "final_committed", + followUp: "complete", + }); + }); + it("retains the original fence and signed last phase when canceled after pull before generation commit", async () => { + vi.mocked(dockerComposeStream).mockImplementation(async (args) => { + expect(args).toEqual(["pull", "alpha"]); + const control = await import("@openmapx/core/build-artifacts"), + dataRoot = join(root, "infra/docker/data"), + directory = join(dataRoot, "ops-agent/lifecycle"), + envelope = JSON.parse(readFileSync(join(directory, readdirSync(directory)[0]!), "utf8")), + installation = control.openControlInstallation({ + infraDir: join(root, "infra/docker"), + role: "host-cli", + serviceIds: ["alpha", "beta"], + }); + try { + control.cancelControlExecution( + control.restoreAdmittedControlExecution(installation, envelope.seed.payload.execution), + ); + } finally { + control.closeControlInstallation(installation); + } + return 0; + }); + await expect(command("update")).rejects.toBeInstanceOf(Error); + expect(vi.mocked(dockerComposeStream).mock.calls.map((call) => call[0])).toEqual([ + ["pull", "alpha"], + ]); + expect(services.currentConfigurationGeneration(join(root, "infra/docker"))).toBe(seed); + expect(await retainedReceipt()).toMatchObject({ + phase: "failed", + finalRevisionId: null, + commit: "not_committed", + followUp: "not_started", + }); + expect(existsSync(join(root, "infra/docker/data/.admin-target-control/locks/alpha"))).toBe( + true, + ); + expect(existsSync(join(root, "infra/docker/.trusted-config-writer"))).toBe(false); + }); + it("retains its signed final revision and target fence when recreation fails after commit", async () => { + vi.mocked(dockerComposeStream).mockImplementation(async (args) => (args[0] === "pull" ? 0 : 9)); + await expect(command("update")).rejects.toThrow("exit:9"); + const receipt = await retainedReceipt(); + expect(receipt).toMatchObject({ + phase: "failed", + expectedRevisionId: seed, + commit: "final_committed", + followUp: "unknown", + failure: { reason: "recreate", failedServiceIds: ["alpha"] }, + }); + expect(receipt.finalRevisionId).toBe( + services.currentConfigurationGeneration(join(root, "infra/docker")), + ); + expect(receipt.finalRevisionId).not.toBe(seed); + expect(existsSync(join(root, "infra/docker/data/.admin-target-control/locks/alpha"))).toBe( + true, + ); + expect(existsSync(join(root, "infra/docker/.trusted-config-writer"))).toBe(false); + }); +}); diff --git a/packages/cli/__tests__/cli-services-update.test.ts b/packages/cli/__tests__/cli-services-update.test.ts index f0ec95934..b78f80bc8 100644 --- a/packages/cli/__tests__/cli-services-update.test.ts +++ b/packages/cli/__tests__/cli-services-update.test.ts @@ -1,24 +1,58 @@ +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { renderComposeForRepo } from "../src/commands/compose"; import { registerServicesCommands } from "../src/commands/services"; import { dockerComposeStream } from "../src/lib/docker"; import { ensureReleaseOverlay } from "../src/lib/release"; -// Preparation writes deployment files and hardlinks; isolate those effects and -// Docker while exercising the actual update command's release/error branches. -vi.mock("../src/commands/compose", () => ({ - renderComposeForRepo: vi.fn(async () => ({ selectionWarnings: [] })), -})); -vi.mock("../src/lib/hardlinks", () => ({ - applyGeneratedHardlinks: vi.fn(async () => ({ linked: 0, skipped: 0, pruned: 0 })), +vi.setConfig({ testTimeout: 60000 }); + +// Named external Docker/registry edges. Commands, files, renderer, locks and receipts are real. +vi.mock("../src/lib/docker", async (original) => ({ + ...(await original()), + dockerComposeStream: vi.fn(), })); -vi.mock("../src/lib/docker", () => ({ dockerComposeStream: vi.fn() })); vi.mock("../src/lib/release", async (importOriginal) => ({ ...(await importOriginal()), ensureReleaseOverlay: vi.fn(), })); -beforeEach(() => { +let root: string, seed: string; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-cli-update-")); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(join(root, "infra/docker"), { recursive: true }); + const ids = ["ops-agent", "transitous-runner", "data-manager", "app-web", "app-api", "valhalla"]; + for (const id of ids) { + const directory = join(root, "services", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "service.json"), + JSON.stringify({ + id, + name: id, + version: "1.0.0", + quality: "built-in", + container: { image: `fixture/${id}`, tag: "1" }, + }), + ); + } + for (const [key, value] of Object.entries({ + OPENMAPX_ROOT_DIR: root, + OPENMAPX_RELEASE: "local", + DOMAIN: "example.test", + OPENMAPX_ENABLED_SERVICES: "", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + DATA_MANAGER_AUTH_TOKEN: "d".repeat(32), + })) + vi.stubEnv(key, value); + services.writeServiceSelectionFile(join(root, "infra/docker"), ids); + seed = (await renderComposeForRepo({ rootDir: root, domain: "example.test" })).revisionId; vi.mocked(dockerComposeStream).mockReset().mockResolvedValue(0); vi.mocked(ensureReleaseOverlay).mockReset(); vi.spyOn(console, "log").mockImplementation(() => {}); @@ -28,7 +62,11 @@ beforeEach(() => { throw new Error(`exit:${code}`); }); }); -afterEach(() => vi.restoreAllMocks()); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); +}); function update(...ids: string[]) { const program = new Command(); @@ -62,36 +100,22 @@ describe("services update pull failures", () => { ]); }); - it("passes --no-deps only to recreation so an admin update cannot restart dependencies", async () => { + it("passes --no-deps only to ordinary target recreation", async () => { vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "present", path: "release.yml" }); - await expect( - update("--no-deps", "data-manager", "app-web", "transitous-runner", "app-api"), - ).rejects.toThrow("exit:0"); + await expect(update("--no-deps", "app-web", "transitous-runner", "app-api")).rejects.toThrow( + "exit:0", + ); expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([ - [["pull", "data-manager", "app-web", "transitous-runner", "app-api"]], - [ - [ - "up", - "-d", - "--force-recreate", - "--no-deps", - "data-manager", - "app-web", - "transitous-runner", - "app-api", - ], - ], + [["pull", "app-api", "app-web", "transitous-runner"]], + [["up", "-d", "--force-recreate", "--no-deps", "app-api", "app-web", "transitous-runner"]], ]); }); - it("allows an explicitly unpinned local image after a failed pull", async () => { + it("refuses an explicitly unpinned local image after a failed pull", async () => { vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "disabled" }); vi.mocked(dockerComposeStream).mockResolvedValueOnce(7); - await expect(update("ops-agent")).rejects.toThrow("exit:0"); - expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([ - [["pull", "ops-agent"]], - [["up", "-d", "--force-recreate", "ops-agent"]], - ]); + await expect(update("ops-agent")).rejects.toThrow("exit:7"); + expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([[["pull", "ops-agent"]]]); }); it("does not pull or recreate when release resolution fails", async () => { @@ -103,13 +127,41 @@ describe("services update pull failures", () => { expect(dockerComposeStream).not.toHaveBeenCalled(); }); - it("preserves locally built third-party service update behavior", async () => { + it("refuses recreation of a locally built service after a failed pull", async () => { vi.mocked(dockerComposeStream).mockResolvedValueOnce(7); - await expect(update("valhalla")).rejects.toThrow("exit:0"); + await expect(update("valhalla")).rejects.toThrow("exit:7"); expect(ensureReleaseOverlay).not.toHaveBeenCalled(); - expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([ - [["pull", "valhalla"]], - [["up", "-d", "--force-recreate", "valhalla"]], - ]); + expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([[["pull", "valhalla"]]]); + }); + it("validates pins and pulls before full generation preparation, hardlinks and recreation", async () => { + const events: string[] = []; + vi.mocked(ensureReleaseOverlay).mockImplementation(async () => { + events.push("pins"); + return { status: "present", path: "release.yml" }; + }); + vi.mocked(dockerComposeStream).mockImplementation(async (args) => { + const current = services.currentConfigurationGeneration(join(root, "infra/docker")); + if (args[0] === "pull") { + expect(current).toBe(seed); + events.push("pull"); + } else { + expect(current).not.toBe(seed); + const directory = join(root, "infra/docker/data/ops-agent/lifecycle"); + const raw = JSON.parse(readFileSync(join(directory, readdirSync(directory)[0]!), "utf8")); + const receipt = await services.readServiceLifecycleReceipt({ + dataRoot: join(root, "infra/docker/data"), + execution: raw.current.payload.receipt.execution, + }); + expect(receipt).toMatchObject({ + phase: "hardlinks_applied", + finalRevisionId: current, + hardlinks: { applied: true, entries: 0 }, + }); + events.push("recreate"); + } + return 0; + }); + await expect(update("ops-agent")).rejects.toThrow("exit:0"); + expect(events).toEqual(["pins", "pull", "recreate"]); }); }); diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index f9b4c3a38..d868dc19f 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -51,6 +51,8 @@ export interface RenderRepoOptions { rootDir?: string; domain: string; services?: string[]; + expectedCurrentRevisionId?: string | null; + revisionId?: string; redisAuthHooks?: RedisAuthReconciliationHooks; } @@ -199,119 +201,128 @@ export async function renderComposeForRepo(opts: RenderRepoOptions): Promise { - assertCliExecutionAvailable(); - ensurePlatformPrivateDirectory(join(paths.infraDir, "data", "ops-agent", "trusted-config")); - const erasureDirectory = join(paths.infraDir, "data", "erasure"); - ensurePlatformPrivateDirectory(erasureDirectory); - ensurePlatformPrivateDirectory(join(paths.infraDir, "data", "privacy-extraction")); - ensurePlatformPrivateDirectory(join(paths.infraDir, "backups")); - const redisPasswordPath = join(paths.infraDir, "secrets", "redis-password"); - const redisAclPath = join(paths.infraDir, "secrets", "redis-acl.conf"); - const opsAgentApiTokenPath = join(paths.infraDir, "secrets", "ops-agent-api-token"); - const opsAgentDataManagerTokenPath = join( - paths.infraDir, - "secrets", - "ops-agent-data-manager-token", - ); - const offlinePackagePrincipalKeyPath = join( - paths.infraDir, - "secrets", - "offline-package-principal-key", - ); - const erasureJournalKeyPath = join(paths.infraDir, "secrets", "erasure-journal-key"); - const subjectExportsMasterKeyPath = join( - paths.infraDir, - "secrets", - "subject-exports-master-key", - ); - const privacyBackupCapabilityKeyPath = join( - paths.infraDir, - "secrets", - "privacy-backup-capability-key", - ); - // Shared only between data-manager and the private Transitous runner: it - // signs the single-use capability tokens that authorize one upstream run. - const transitousRunnerCapabilityPath = join( - paths.infraDir, - "secrets", - "transitous-runner-capability", - ); - ensurePlatformSecretFile(redisPasswordPath); - ensurePlatformSecretFile(offlinePackagePrincipalKeyPath); - const erasureJournalKey = Buffer.from( - ensurePlatformSecretFile(erasureJournalKeyPath), - "base64url", - ); - initializeErasureJournal(join(erasureDirectory, "journal.jsonl"), erasureJournalKey); - ensurePlatformExportsKeyRingFile(subjectExportsMasterKeyPath); - // app-api and ops-agent refuse this key unless only their uid can read it. - ensurePlatformOwnerOnlySecretFile(privacyBackupCapabilityKeyPath); - ensurePlatformSecretFile(transitousRunnerCapabilityPath); - const opsAgentApiToken = ensurePlatformSecretFile(opsAgentApiTokenPath); - const opsAgentDataManagerToken = ensurePlatformSecretFile(opsAgentDataManagerTokenPath); - if (opsAgentApiToken === opsAgentDataManagerToken) { - throw new Error("Ops-agent API and data-manager tokens must be distinct"); - } - reconcileRedisAuthFiles(redisPasswordPath, redisAclPath, opts.redisAuthHooks); - const registry = new ServiceRegistry({ rootDir: paths.root }); - await registry.load(); - const applied = applyServiceSelection(registry, { - rootDir: paths.root, - explicitIds: opts.services, - }); - const services = registry.list(); - const known = new Set(services.map((service) => service.manifest.id)); - const integrationSchemas = loadIntegrationSchemas(paths.root); - // The database is the admin panel's: what it saved arrives through the - // generation it last applied. The env layer is this host's. - const saved = readAppliedConfiguration(paths.infraDir); - const savedConfigs = new Map(saved.serviceConfigs.map((entry) => [entry.serviceId, entry])); - const serviceConfigs = services.flatMap((service) => { - const { id } = service.manifest; - const proxyHostKey = service.manifest.exposure?.proxy?.host?.configKey; - const fromEnv = Object.entries( - resolveServiceConfigFromEnv(service.manifest, process.env), - ).filter(([, entry]) => entry.source === "env"); - // An env value is a reference in the YAML, except the proxy host, which - // the routes need as a value. - const envKeys = fromEnv.map(([key]) => key).filter((key) => key !== proxyHostKey); - const values: coreServices.ConfigurationInput["serviceConfigs"][number]["values"] = - Object.fromEntries( - Object.entries(savedConfigs.get(id)?.values ?? {}).filter( - ([key]) => !envKeys.includes(key), - ), - ); - for (const [key, entry] of fromEnv) - if (key === proxyHostKey) values[key] = String(entry.value); - if (Object.keys(values).length === 0 && envKeys.length === 0) return []; - return [{ serviceId: id, values, ...(envKeys.length > 0 ? { envKeys } : {}) }]; - }); - assertCliExecutionAvailable(); - const committed = await writer.commit({ - services, - integrationSchemas, - input: { - domain: opts.domain, - selectedRoots: applied.requestedIds, - serviceConfigs, - integrationConfigs: saved.integrationConfigs.filter((entry) => - integrationSchemas.has(entry.integrationId), + return withConfigurationGenerationWriter(paths.infraDir, (writer) => + renderComposeWithWriter(opts, writer), + ); +} + +/** Inner lifecycle adapter; the caller already owns the one generation writer. */ +export async function renderComposeWithWriter( + opts: RenderRepoOptions, + writer: coreServices.ConfigurationGenerationWriter, +): Promise { + coreServices.assertConfigurationGenerationWriter(writer); + const paths = repoPaths(opts.rootDir); + assertCliExecutionAvailable(); + ensurePlatformPrivateDirectory(join(paths.infraDir, "data", "ops-agent", "trusted-config")); + const erasureDirectory = join(paths.infraDir, "data", "erasure"); + ensurePlatformPrivateDirectory(erasureDirectory); + ensurePlatformPrivateDirectory(join(paths.infraDir, "data", "privacy-extraction")); + ensurePlatformPrivateDirectory(join(paths.infraDir, "backups")); + const redisPasswordPath = join(paths.infraDir, "secrets", "redis-password"); + const redisAclPath = join(paths.infraDir, "secrets", "redis-acl.conf"); + const opsAgentApiTokenPath = join(paths.infraDir, "secrets", "ops-agent-api-token"); + const opsAgentDataManagerTokenPath = join( + paths.infraDir, + "secrets", + "ops-agent-data-manager-token", + ); + const offlinePackagePrincipalKeyPath = join( + paths.infraDir, + "secrets", + "offline-package-principal-key", + ); + const erasureJournalKeyPath = join(paths.infraDir, "secrets", "erasure-journal-key"); + const subjectExportsMasterKeyPath = join(paths.infraDir, "secrets", "subject-exports-master-key"); + const privacyBackupCapabilityKeyPath = join( + paths.infraDir, + "secrets", + "privacy-backup-capability-key", + ); + // Shared only between data-manager and the private Transitous runner: it + // signs the single-use capability tokens that authorize one upstream run. + const transitousRunnerCapabilityPath = join( + paths.infraDir, + "secrets", + "transitous-runner-capability", + ); + ensurePlatformSecretFile(redisPasswordPath); + ensurePlatformSecretFile(offlinePackagePrincipalKeyPath); + const erasureJournalKey = Buffer.from( + ensurePlatformSecretFile(erasureJournalKeyPath), + "base64url", + ); + initializeErasureJournal(join(erasureDirectory, "journal.jsonl"), erasureJournalKey); + ensurePlatformExportsKeyRingFile(subjectExportsMasterKeyPath); + // app-api and ops-agent refuse this key unless only their uid can read it. + ensurePlatformOwnerOnlySecretFile(privacyBackupCapabilityKeyPath); + ensurePlatformSecretFile(transitousRunnerCapabilityPath); + const opsAgentApiToken = ensurePlatformSecretFile(opsAgentApiTokenPath); + const opsAgentDataManagerToken = ensurePlatformSecretFile(opsAgentDataManagerTokenPath); + if (opsAgentApiToken === opsAgentDataManagerToken) { + throw new Error("Ops-agent API and data-manager tokens must be distinct"); + } + reconcileRedisAuthFiles(redisPasswordPath, redisAclPath, opts.redisAuthHooks); + const registry = new ServiceRegistry({ rootDir: paths.root }); + await registry.load(); + const applied = applyServiceSelection(registry, { + rootDir: paths.root, + explicitIds: opts.services, + }); + const services = registry.list(); + const known = new Set(services.map((service) => service.manifest.id)); + const integrationSchemas = loadIntegrationSchemas(paths.root); + // The database is the admin panel's: what it saved arrives through the + // generation it last applied. The env layer is this host's. + const saved = readAppliedConfiguration(paths.infraDir); + const savedConfigs = new Map(saved.serviceConfigs.map((entry) => [entry.serviceId, entry])); + const serviceConfigs = services.flatMap((service) => { + const { id } = service.manifest; + const proxyHostKey = service.manifest.exposure?.proxy?.host?.configKey; + const fromEnv = Object.entries( + resolveServiceConfigFromEnv(service.manifest, process.env), + ).filter(([, entry]) => entry.source === "env"); + // An env value is a reference in the YAML, except the proxy host, which + // the routes need as a value. + const envKeys = fromEnv.map(([key]) => key).filter((key) => key !== proxyHostKey); + const values: coreServices.ConfigurationInput["serviceConfigs"][number]["values"] = + Object.fromEntries( + Object.entries(savedConfigs.get(id)?.values ?? {}).filter( + ([key]) => !envKeys.includes(key), ), - serviceSecrets: saved.serviceSecrets.filter((entry) => known.has(entry.serviceId)), - }, - allowMissingSelected: applied.source === "default", - }); - return { - servicesRendered: committed.enabledServiceIds.length, - composePath: paths.composePath, - revisionId: committed.revisionId, - requestedServiceIds: applied.requestedIds, - enabledServiceIds: committed.enabledServiceIds, - selectionWarnings: applied.selection.warnings, - renderWarnings: committed.warnings, - }; + ); + for (const [key, entry] of fromEnv) if (key === proxyHostKey) values[key] = String(entry.value); + if (Object.keys(values).length === 0 && envKeys.length === 0) return []; + return [{ serviceId: id, values, ...(envKeys.length > 0 ? { envKeys } : {}) }]; + }); + assertCliExecutionAvailable(); + const committed = await writer.commit({ + services, + integrationSchemas, + input: { + domain: opts.domain, + selectedRoots: applied.requestedIds, + serviceConfigs, + integrationConfigs: saved.integrationConfigs.filter((entry) => + integrationSchemas.has(entry.integrationId), + ), + serviceSecrets: saved.serviceSecrets.filter((entry) => known.has(entry.serviceId)), + }, + allowMissingSelected: applied.source === "default", + ...(Object.hasOwn(opts, "expectedCurrentRevisionId") + ? { expectedCurrentRevisionId: opts.expectedCurrentRevisionId } + : {}), + ...(opts.revisionId ? { revisionId: opts.revisionId } : {}), }); + return { + servicesRendered: committed.enabledServiceIds.length, + composePath: paths.composePath, + revisionId: committed.revisionId, + requestedServiceIds: applied.requestedIds, + enabledServiceIds: committed.enabledServiceIds, + selectionWarnings: applied.selection.warnings, + renderWarnings: committed.warnings, + }; } export function registerComposeCommands(program: Command): void { diff --git a/packages/cli/src/commands/services.ts b/packages/cli/src/commands/services.ts index 31aad23fc..eadc13633 100644 --- a/packages/cli/src/commands/services.ts +++ b/packages/cli/src/commands/services.ts @@ -1,19 +1,14 @@ import { services as coreServices } from "@openmapx/core/server"; import type { Command } from "commander"; import kleur from "kleur"; -import { completeCliCommand } from "../lib/control-context"; +import { cliControlContext, completeCliCommand } from "../lib/control-context"; import { dockerComposeStream } from "../lib/docker"; import { applyGeneratedHardlinks } from "../lib/hardlinks"; import { log, table } from "../lib/output"; import { repoPaths } from "../lib/paths"; import { expandPresets, UnknownPresetError } from "../lib/presets"; -import { - ensureReleaseOverlay, - releaseStatusLines, - touchesReleasePinnedServices, - unpinnedReleaseWarning, -} from "../lib/release"; import { buildServices } from "../lib/service-builds"; +import { runCliServiceLifecycle, ServiceLifecycleCommandError } from "../lib/service-lifecycle"; import { applyServiceSelection, disableSelectedServices, @@ -376,46 +371,37 @@ export function registerServicesCommands(program: Command): void { "--preset ", "Comma/space-separated preset names (app, routing, transit, pelias, nominatim, photon, overpass, tiles, martin, proxy, dev)", ) - .action(async (ids: string[], options: { preset?: string }) => { - const allIds = mergeIdsWithPresets(ids, options.preset); - if (allIds.length === 0) { - log.err("No services selected. Pass or --preset."); - completeCliCommand(1); - } - try { - // Don't pass `services: allIds` — that would narrow the persisted - // compose YAML to just the start target list and drop every other - // enabled service. Render the full enabled selection (env / - // service-selection.json) and then `up -d` only the requested - // subset. Same rationale as the `update` branch below. - const rendered = await renderComposeForRepo({ - domain: process.env.DOMAIN ?? "localhost", - }); - for (const warning of rendered.selectionWarnings) log.warn(warning); - const linked = await applyGeneratedHardlinks({ prune: true, requirePlan: true }); - log.ok( - `Applied hardlinks: ${linked.linked} linked, ${linked.skipped} already linked, ${linked.pruned} stale file${linked.pruned === 1 ? "" : "s"} pruned`, - ); - } catch (err) { - log.err(`prepare/start failed: ${(err as Error).message}`); - completeCliCommand(1); - } - if (touchesReleasePinnedServices(allIds)) { - const overlay = await ensureReleaseOverlay(); - if (overlay.status === "resolved") { - log.ok(`Atomic release selection ${overlay.release} → ${overlay.path}`); - } else if (overlay.status === "present") { - for (const line of releaseStatusLines(overlay.path)) log.dim(line); - } else if (overlay.status === "unpinned") { - log.err(unpinnedReleaseWarning(overlay.reason)); + .option( + "--expected-configuration-revision ", + "Require this applied configuration generation", + ) + .action( + async ( + ids: string[], + options: { preset?: string; expectedConfigurationRevision?: string }, + ) => { + const allIds = mergeIdsWithPresets(ids, options.preset); + if (allIds.length === 0) { + log.err("No services selected. Pass or --preset."); completeCliCommand(1); - } else if (overlay.status === "disabled") { - log.dim("Release pinning disabled (OPENMAPX_RELEASE=local); using manifest image tags."); } - } - const code = await dockerComposeStream(["up", "-d", ...allIds]); - completeCliCommand(code); - }); + try { + await runCliServiceLifecycle({ + action: "start", + serviceIds: allIds, + expectedConfigurationRevisionId: options.expectedConfigurationRevision, + }); + } catch (error) { + log.err( + cliControlContext() + ? "Service Start failed; inspect the original operation status." + : `prepare/start failed: ${(error as Error).message}`, + ); + completeCliCommand(error instanceof ServiceLifecycleCommandError ? error.exitCode : 1); + } + completeCliCommand(0); + }, + ); services .command("stop [ids...]") @@ -454,66 +440,39 @@ export function registerServicesCommands(program: Command): void { ) .option("--preset ", "Comma/space-separated preset names") .option("--no-deps", "Replace only selected services without starting their dependencies") - .action(async (ids: string[], options: { preset?: string; deps?: boolean }) => { - const allIds = mergeIdsWithPresets(ids, options.preset); - if (allIds.length === 0) { - log.err("No services selected. Pass or --preset."); - completeCliCommand(1); - } - try { - // Render the operator's selection (env / service-selection.json), NOT - // one scoped to allIds: that would apply a generation holding only - // the update targets and drop every other enabled service. - const rendered = await renderComposeForRepo({ - domain: process.env.DOMAIN ?? "localhost", - }); - for (const warning of rendered.selectionWarnings) log.warn(warning); - const linked = await applyGeneratedHardlinks({ prune: true, requirePlan: true }); - log.ok( - `Applied hardlinks: ${linked.linked} linked, ${linked.skipped} already linked, ${linked.pruned} stale file${linked.pruned === 1 ? "" : "s"} pruned`, - ); - } catch (err) { - log.err(`prepare/update failed: ${(err as Error).message}`); - completeCliCommand(1); - } - - let releasePinned = false; - if (touchesReleasePinnedServices(allIds)) { - // Retain existing release pins, or resolve a complete release once. - const overlay = await ensureReleaseOverlay(); - releasePinned = overlay.status === "present" || overlay.status === "resolved"; - if (overlay.status === "resolved") { - log.ok(`Atomic release selection ${overlay.release} → ${overlay.path}`); - } else if (overlay.status === "present") { - for (const line of releaseStatusLines(overlay.path)) log.dim(line); - } else if (overlay.status === "unpinned") { - log.err(unpinnedReleaseWarning(overlay.reason)); + .option( + "--expected-configuration-revision ", + "Require this applied configuration generation", + ) + .action( + async ( + ids: string[], + options: { preset?: string; deps?: boolean; expectedConfigurationRevision?: string }, + ) => { + const allIds = mergeIdsWithPresets(ids, options.preset); + if (allIds.length === 0) { + log.err("No services selected. Pass or --preset."); completeCliCommand(1); - } else if (overlay.status === "disabled") { - log.warn( - "Release pinning is disabled (OPENMAPX_RELEASE=local); this update uses the manifest image tags instead of a digest-pinned release.", - ); } - } - const pullCode = await dockerComposeStream(["pull", ...allIds]); - if (pullCode !== 0) { - if (releasePinned) { - log.err("Could not pull the pinned release images; refusing to recreate services."); - completeCliCommand(pullCode); + try { + await runCliServiceLifecycle({ + action: "update", + serviceIds: allIds, + noDeps: options.deps === false, + expectedConfigurationRevisionId: options.expectedConfigurationRevision, + }); + } catch (error) { + log.err( + cliControlContext() + ? "Service Update failed; inspect the original operation status." + : `prepare/update failed: ${(error as Error).message}`, + ); + completeCliCommand(error instanceof ServiceLifecycleCommandError ? error.exitCode : 1); } - log.warn("Some images could not be pulled (service may be locally built). Continuing."); - } - - const code = await dockerComposeStream([ - "up", - "-d", - "--force-recreate", - ...(options.deps === false ? ["--no-deps"] : []), - ...allIds, - ]); - completeCliCommand(code); - }); + completeCliCommand(0); + }, + ); services .command("status [id]") diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 3c9dc0e36..960ed41c6 100755 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -11,6 +11,7 @@ import { registerServicesCommands } from "./commands/services"; import { registerTransitousCommands } from "./commands/transitous-bump"; import { registerTransportApisPinCommands } from "./commands/transport-apis-pin"; import { registerUsersCommands } from "./commands/users"; +import { fixedCliCommandExitCode } from "./lib/control-context"; import { assertCliExecutionAvailable, withCliExecutionDeadline } from "./lib/execution-deadline"; import { loadInfraEnv } from "./lib/infra-env"; @@ -39,8 +40,17 @@ registerCacheCommands(program); registerTransportApisPinCommands(program); registerTransitousCommands(program); -await withCliExecutionDeadline(process.argv.slice(2), async () => { - loadInfraEnv(); - program.hook("preAction", () => assertCliExecutionAvailable()); - await program.parseAsync(process.argv); -}); +try { + await withCliExecutionDeadline(process.argv.slice(2), async () => { + loadInfraEnv(); + program.hook("preAction", () => assertCliExecutionAvailable()); + await program.parseAsync(process.argv); + }); +} catch (error) { + if (!process.env.OPENMAPX_CONTROL_DELEGATION && !process.env.OPENMAPX_PARENT_EXECUTION) + throw error; + // The control context has already retained its actual disposition and closed + // local handles. Avoid forwarding exception paths/stacks to public job logs. + process.stderr.write("Fixed CLI command failed; inspect the original operation status.\n"); + process.exitCode = fixedCliCommandExitCode(error) ?? 1; +} diff --git a/packages/cli/src/lib/control-context.ts b/packages/cli/src/lib/control-context.ts index efb6cdaab..d1b5566bf 100644 --- a/packages/cli/src/lib/control-context.ts +++ b/packages/cli/src/lib/control-context.ts @@ -34,6 +34,10 @@ class FixedCliCommandExit extends Error { super(`Fixed CLI command exited with code ${code}`); } } +/** Exit classification only; it conveys no containment or execution proof. */ +export function fixedCliCommandExitCode(error: unknown): number | undefined { + return error instanceof FixedCliCommandExit ? error.code : undefined; +} /** Managed commands unwind through durable child settlement before the process exits. */ export function completeCliCommand(code = 0): never { if (contexts.getStore()) throw new FixedCliCommandExit(code); diff --git a/packages/cli/src/lib/release.ts b/packages/cli/src/lib/release.ts index 95ca4964b..304562956 100644 --- a/packages/cli/src/lib/release.ts +++ b/packages/cli/src/lib/release.ts @@ -128,20 +128,7 @@ export async function ensureReleaseOverlay( ): Promise { const path = opts.path ?? repoPaths().composeReleasePath; if (existsSync(path)) { - const selected = readSelection(path); - const channel = releaseChannel(); - const prefix = - channel.kind === "enabled" ? channel.imagePrefix : selected?.images.api?.split("/api@")[0]; - if ( - !selected?.release || - !prefix || - !coreServices.RELEASE_IMAGE_NAMES.every((name) => - selected.images[name]?.startsWith(`${prefix}/${name}@`), - ) - ) - throw new Error( - "Existing release overlay is malformed or belongs to another configured repository; explicitly reselect or clear it", - ); + coreServices.validateExistingReleaseSelection(readFileSync(path, "utf8")); return { status: "present", path }; } // An operator running local images opts out once instead of paying a @@ -200,22 +187,32 @@ export async function selectRelease( } = {}, ): Promise<{ path: string; release: string }> { const path = opts.path ?? repoPaths().composeReleasePath; - const previous = readSelection(path); - const manifest = await (opts.resolve ?? resolveReleaseManifest)(); - const report = opts.report ?? (() => undefined); - report( - `Previous release: ${previous ? (previous.release ?? "unknown (legacy or modified overlay)") : "none"}`, - ); - report(`Candidate release: ${manifest.release}`); - for (const name of coreServices.RELEASE_IMAGE_NAMES) { - const before = previous?.images[name]?.split("@")[1]; - const after = manifest.images[name].split("@")[1]; + const store = join(dirname(path), ".ops-agent-releases"); + mkdirSync(store, { mode: 0o700, recursive: true }); + const directory = lstatSync(store); + if (!directory.isDirectory() || directory.isSymbolicLink()) + throw new Error("Release selection directory is unsafe"); + const lock = await coreServices.acquireReleaseStoreLock(store, {}, { failIfBusy: true }); + try { + const previous = readSelection(path); + const manifest = await (opts.resolve ?? resolveReleaseManifest)(); + const report = opts.report ?? (() => undefined); report( - `${name}: ${before ? (before === after ? "reused" : "changed") : "previous digest unknown"} (${before ?? "unknown"} → ${after})`, + `Previous release: ${previous ? (previous.release ?? "unknown (legacy or modified overlay)") : "none"}`, ); + report(`Candidate release: ${manifest.release}`); + for (const name of coreServices.RELEASE_IMAGE_NAMES) { + const before = previous?.images[name]?.split("@")[1]; + const after = manifest.images[name].split("@")[1]; + report( + `${name}: ${before ? (before === after ? "reused" : "changed") : "previous digest unknown"} (${before ?? "unknown"} → ${after})`, + ); + } + writeReleaseOverlay(manifest, path); + return { path, release: manifest.release }; + } finally { + lock.release(); } - writeReleaseOverlay(manifest, path); - return { path, release: manifest.release }; } /** Clear only local Compose selection; preserve evidence still mounted by running services. */ diff --git a/packages/cli/src/lib/service-lifecycle.ts b/packages/cli/src/lib/service-lifecycle.ts new file mode 100644 index 000000000..c296299ca --- /dev/null +++ b/packages/cli/src/lib/service-lifecycle.ts @@ -0,0 +1,481 @@ +import type { + BorrowedArtifactTargetLease, + TargetLeaseAuthority, +} from "@openmapx/core/build-artifacts"; +import { assertControlExecutionForward } from "@openmapx/core/build-artifacts"; +import type { CoordinatorMode, ExecutionRef, ServicePreparationResult } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; + +export type LifecycleOwnershipRequest = { + execution: ExecutionRef; + serviceIds: readonly string[]; + expectedConfigurationRevisionId?: string; + inheritedLease?: BorrowedArtifactTargetLease; + signal?: AbortSignal; +}; +export type ServiceLifecycleEffects = { + withOwnership( + request: LifecycleOwnershipRequest, + run: (ownership: services.VerifiedLifecycleOwnership) => Promise, + ): Promise; + validatePins(ids: readonly string[]): Promise; + pull(ids: readonly string[], signal?: AbortSignal): Promise; + prepare(options: { + ownership: services.VerifiedLifecycleOwnership; + coordinatorMode: CoordinatorMode; + signal?: AbortSignal; + }): Promise; + recreate(ids: readonly string[], noDeps: boolean, signal?: AbortSignal): Promise; +}; +function forward(ownership: services.VerifiedLifecycleOwnership, signal?: AbortSignal) { + signal?.throwIfAborted(); + assertControlExecutionForward(services.assertVerifiedLifecycleOwnership(ownership)); +} +/** The supplied adapter owns the one writer and target lease; inner preparation never reacquires. */ +export function prepareServiceLifecycle( + request: LifecycleOwnershipRequest & { coordinatorMode: CoordinatorMode }, + effects: ServiceLifecycleEffects, +): Promise { + return effects.withOwnership(request, async (ownership) => { + forward(ownership, request.signal); + return effects.prepare({ + ownership, + coordinatorMode: request.coordinatorMode, + signal: request.signal, + }); + }); +} +export function runServiceUpdate( + request: LifecycleOwnershipRequest & { noDeps?: boolean }, + effects: ServiceLifecycleEffects, +): Promise<{ changed: true; revisionId: string }> { + return effects.withOwnership(request, async (ownership) => { + forward(ownership, request.signal); + await effects.validatePins(request.serviceIds); + forward(ownership, request.signal); + await effects.pull(request.serviceIds, request.signal); + forward(ownership, request.signal); + const prepared = await prepareServiceLifecycle( + { ...request, coordinatorMode: "ensure-running" }, + effects, + ); + forward(ownership, request.signal); + await effects.recreate(request.serviceIds, request.noDeps ?? false, request.signal); + return { changed: true, revisionId: prepared.revisionId }; + }); +} + +// The concrete CLI adapter is kept here so ordinary and delegated commands enter +// the same nominal writer/target/receipt scope. +export class ServiceLifecycleCommandError extends Error { + constructor( + readonly exitCode: number, + message: string, + ) { + super(message); + } +} +export async function runCliServiceLifecycle(options: { + action: "start" | "update"; + serviceIds: readonly string[]; + noDeps?: boolean; + expectedConfigurationRevisionId?: string; +}): Promise { + if (options.action === "update" && options.serviceIds.includes("data-manager")) + throw new ServiceLifecycleCommandError( + 1, + "Coordinator Update requires its guarded administrative workflow", + ); + const { randomBytes, randomUUID } = await import("node:crypto"); + const { existsSync, mkdirSync } = await import("node:fs"); + const { join } = await import("node:path"); + const control = await import("@openmapx/core/build-artifacts"); + const ops = await import("@openmapx/core/ops"); + const { repoPaths } = await import("./paths"); + const { cliControlContext } = await import("./control-context"); + const { assertCliDeploymentSecrets } = await import("./deployment-secret-policy"); + const { renderComposeWithWriter } = await import("../commands/compose"); + const { dockerComposeStream, dockerCompose } = await import("./docker"); + const { readGeneratedHardlinkPlan } = await import("./hardlinks"); + const release = await import("./release"); + const paths = repoPaths(); + assertCliDeploymentSecrets(); + mkdirSync(join(paths.infraDir, "data"), { recursive: true }); + if (!existsSync("/run/secrets/openmapx-control-verifiers")) + await control.provisionControlInstallation(paths.infraDir); + const ids = [...new Set(options.serviceIds)].sort(); + const registry = new services.ServiceRegistry({ rootDir: paths.root }); + await registry.load(); + const targets = registry + .list() + .map((service) => service.manifest.id) + .sort(); + if (!ids.length || ids.some((id) => !targets.includes(id))) + throw Error("Unknown lifecycle target"); + if ( + !ids.includes("data-manager") && + services.serviceCreationDependencyIds(registry.list(), ids).includes("data-manager") + ) + throw new ServiceLifecycleCommandError( + 1, + "Coordinator maintenance required before service creation", + ); + const assertNativeCreation = async () => { + if (ids.includes("data-manager") || (options.action === "update" && options.noDeps)) return; + const rendered = await dockerCompose(["config", "--format", "json"]); + if (rendered.exitCode !== 0) + throw new ServiceLifecycleCommandError(1, "Service creation dependencies unavailable"); + let dependencies: string[]; + try { + dependencies = services.composeCreationDependencyIds(JSON.parse(rendered.stdout), ids); + } catch { + throw new ServiceLifecycleCommandError(1, "Service creation dependencies unavailable"); + } + if (dependencies.includes("data-manager")) + throw new ServiceLifecycleCommandError( + 1, + "Coordinator maintenance required before service creation", + ); + }; + const inherited = cliControlContext(); + const seed = inherited + ? await services.withServiceLifecycleReceiptAuthority(inherited.authority, () => + services.readServiceLifecycleReceipt({ + dataRoot: join(paths.infraDir, "data"), + execution: inherited.authority.execution, + }), + ) + : undefined; + const expected = + options.expectedConfigurationRevisionId ?? + seed?.expectedRevisionId ?? + (await services.withConfigurationGenerationWriter(paths.infraDir, async () => + services.currentConfigurationGeneration(paths.infraDir), + )); + if (!expected) throw Error("An initial configuration generation is required"); + // The existing generation supplies the exact finite hardlink intent. A nonempty + // plan must use its retained durable leaf; it can never fall back to /link/local. + const plan = readGeneratedHardlinkPlan().plan; + const operation = + options.action === "update" + ? { + kind: "service.update" as const, + serviceId: ids[0]!, + expectedConfigurationRevisionId: expected, + ...(options.noDeps ? { noDeps: true } : {}), + } + : { + kind: "service.start" as const, + serviceId: ids[0]!, + expectedConfigurationRevisionId: expected, + }; + const installation = + inherited?.installation ?? + control.openControlInstallation({ + infraDir: paths.infraDir, + role: "host-cli", + serviceIds: targets, + }); + const admittedAt = Date.now(), + deadlineAt = admittedAt + ops.opsExecutionBudget(operation.kind).processMs; + const operationFingerprint = ops.opsOperationFingerprint(operation); + const execution = inherited?.authority.execution ?? { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: ops.createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: ops.executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }; + const finalRevisionId = inherited + ? control.ownedServiceLifecycleFinalRevisionId(inherited.authority) + : control.serviceLifecycleFinalRevisionId(execution.operationKey); + const frozenPlan = { plan, prune: true as const }; + const reservation = control.lifecycleHardlinkReservation({ + parent: execution, + revisionId: finalRevisionId, + plan: frozenPlan, + serviceIds: targets, + }); + const authority = + inherited?.authority ?? + control.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: targets, + reservations: reservation ? [reservation] : [], + }); + control.retainGenerationHardlinksOwnerPlan(authority, { + version: 1, + parent: execution, + revisionId: finalRevisionId, + coordinatorMode: "ensure-running", + plan: frozenPlan, + reservation, + }); + if (inherited && !inherited.lease) + throw Error("Delegated lifecycle requires its original borrowed lease"); + let ownership: services.VerifiedLifecycleOwnership | undefined; + let store: services.ServiceLifecycleReceiptStore | undefined; + let receipt: services.ServiceLifecycleReceipt | undefined; + let lease: TargetLeaseAuthority | undefined; + const checkpoint = async (patch: Partial) => { + if (!store || !receipt || !ownership) throw Error("Lifecycle receipt owner unavailable"); + const next = { ...receipt, ...patch, sequence: receipt.sequence + 1 }; + await store.checkpoint(receipt.sequence, next, ownership); + receipt = next; + }; + const request: LifecycleOwnershipRequest = { + execution, + serviceIds: ids, + expectedConfigurationRevisionId: expected, + ...(inherited?.lease ? { inheritedLease: inherited.lease } : {}), + }; + const effects: ServiceLifecycleEffects = { + async withOwnership(inner, run) { + if (ownership) { + forward(ownership, inner.signal); + if ( + ops.canonicalJson(inner.execution) !== ops.canonicalJson(execution) || + inner.serviceIds.some((id) => !ids.includes(id)) + ) + throw Error("Lifecycle ownership cannot expand"); + return run(ownership); + } + lease = + inherited?.lease ?? + (await control.acquireArtifactTargetLease({ + dataRoot: join(paths.infraDir, "data"), + owner: authority, + serviceIds: targets, + signal: inner.signal, + })); + return services.withConfigurationGenerationWriter( + paths.infraDir, + (writer) => + services.withVerifiedLifecycleOwnership( + { writer, lease: lease!, authority }, + async (verified) => { + ownership = verified; + const current = services.currentConfigurationGeneration(paths.infraDir); + if (current !== expected) throw Error("Lifecycle configuration revision conflict"); + const directory = join(paths.infraDir, ".ops-agent-releases"); + mkdirSync(directory, { mode: 0o700, recursive: true }); + const lock = await services.acquireReleaseStoreLock( + directory, + {}, + { failIfBusy: true }, + ); + try { + store = await services.openServiceLifecycleReceipt({ + dataRoot: join(paths.infraDir, "data"), + execution, + serviceIds: ids, + action: options.action, + noDeps: options.noDeps ?? false, + coordinatorMode: "ensure-running", + expectedRevisionId: expected, + seedRevisionId: seed?.seedRevisionId ?? null, + }); + receipt = await store.read(); + if (receipt.phase !== "prepared") + throw Error("Retained lifecycle can only be observed"); + try { + const result = await run(verified); + await checkpoint({ phase: "complete", followUp: "complete" }); + return result; + } catch (error) { + const committed = services.configurationGenerationWriterCommit(writer); + try { + await checkpoint({ + phase: committed && !receipt!.finalRevisionId ? "unknown" : "failed", + ...(committed + ? { commit: "final_committed", finalRevisionId: committed.revisionId } + : {}), + followUp: committed + ? "unknown" + : receipt!.finalRevisionId + ? "failed" + : "not_started", + failure: { + reason: + receipt!.phase === "prepared" + ? "pins" + : receipt!.phase === "pins_validated" + ? "pull" + : receipt!.phase === "pulled" + ? "receipt_io" + : receipt!.phase === "generation_committed" + ? "hardlinks" + : "recreate", + failedServiceIds: ids, + }, + }); + } catch { + /* An unflushed receipt remains unknown; no competing pointer substitutes for it. */ + } + throw error; + } + } finally { + lock.release(); + ownership = undefined; + } + }, + ), + inner.signal, + ); + }, + async validatePins(selected) { + if (release.touchesReleasePinnedServices(selected)) { + const overlay = await release.ensureReleaseOverlay(); + if (overlay.status === "unpinned") + throw new ServiceLifecycleCommandError(1, "Release selection unavailable"); + } + await assertNativeCreation(); + await checkpoint({ phase: "pins_validated" }); + }, + async pull(selected) { + const code = await dockerComposeStream(["pull", ...selected]); + if (code !== 0) + throw new ServiceLifecycleCommandError( + code, + "Image pull failed; refusing to recreate services", + ); + await checkpoint({ phase: "pulled" }); + }, + async prepare({ ownership: owned, coordinatorMode }) { + const rendered = await renderComposeWithWriter( + { + domain: process.env.DOMAIN ?? "localhost", + expectedCurrentRevisionId: expected, + revisionId: finalRevisionId, + }, + owned.writer, + ); + await checkpoint({ + phase: "generation_committed", + finalRevisionId: rendered.revisionId, + commit: "final_committed", + }); + if (ops.canonicalJson(readGeneratedHardlinkPlan().plan) !== ops.canonicalJson(plan)) + throw Error("Hardlink intent changed"); + const sdk = await import("@openmapx/core/data-manager-sdk"), + { dataManagerUrl } = await import("./data-manager-url"); + const transport = sdk.createDataManagerTransport({ + baseUrl: dataManagerUrl(), + authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), + requestTimeoutMs: 15000, + maxJsonBytes: 65536, + }); + if (plan.length) { + const ready = async () => { + try { + return sdk.unwrapDataManagerResponse(await sdk.getStatus({ transport })).ok; + } catch { + return false; + } + }; + if (!(await ready())) { + if (coordinatorMode !== "ensure-running") + throw Error("Configuration coordinator unavailable"); + const code = await dockerComposeStream(["up", "-d", "--no-recreate", "data-manager"]); + if (code !== 0) + throw new ServiceLifecycleCommandError(code, "Configuration coordinator Start failed"); + const until = Math.min(Date.now() + 30000, execution.deadlineAt); + while (!(await ready())) { + if (Date.now() >= until) throw Error("Configuration coordinator unavailable"); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + } + const hardlinks = await control.executeOwnedGenerationHardlinks({ + authority, + lease: owned.lease, + infraDir: paths.infraDir, + transport, + }); + await checkpoint({ phase: "hardlinks_applied", hardlinks }); + return { revisionId: rendered.revisionId, coordinatorMode, hardlinks }; + }, + async recreate(selected, noDeps) { + if (services.currentConfigurationGeneration(paths.infraDir) !== receipt?.finalRevisionId) + throw Error("Lifecycle generation changed before recreation"); + await assertNativeCreation(); + const code = await dockerComposeStream([ + "up", + "-d", + "--force-recreate", + ...(noDeps ? ["--no-deps"] : []), + ...selected, + ]); + if (code !== 0) throw new ServiceLifecycleCommandError(code, "Service recreation failed"); + await checkpoint({ phase: "recreated", followUp: "complete" }); + }, + }; + try { + await effects.withOwnership(request, async () => { + if (options.action === "update") + await runServiceUpdate({ ...request, noDeps: options.noDeps }, effects); + else { + await effects.validatePins(ids); + await prepareServiceLifecycle({ ...request, coordinatorMode: "ensure-running" }, effects); + if (services.currentConfigurationGeneration(paths.infraDir) !== receipt?.finalRevisionId) + throw Error("Lifecycle generation changed before Start"); + await assertNativeCreation(); + const code = await dockerComposeStream([ + "up", + "-d", + ...(ids.includes("data-manager") ? ["--no-recreate"] : []), + ...ids, + ]); + if (code !== 0) throw new ServiceLifecycleCommandError(code, "Service Start failed"); + await checkpoint({ phase: "recreated", followUp: "complete" }); + } + }); + if (!inherited && lease?.mode === "owner") { + const verified = await services.withServiceLifecycleReceiptAuthority(authority, () => + services.readServiceLifecycleReceipt({ dataRoot: join(paths.infraDir, "data"), execution }), + ); + if (verified.phase !== "complete" || !verified.finalRevisionId) + throw Error("Exact completed lifecycle receipt required"); + control.retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { + kind: "ops-result", + result: + options.action === "update" + ? { changed: true, revisionId: verified.finalRevisionId } + : { changed: true }, + }, + }); + await control.settleArtifactTargetLeaseChild( + lease, + control.verifyControlEffectSettlement(installation, execution), + ); + await control.releaseArtifactTargetLease(lease); + lease = undefined; + } + } catch (error) { + let cause = error; + while (cause instanceof services.ConfigurationGenerationError && cause.cause instanceof Error) + cause = cause.cause; + throw cause; + } finally { + if (!inherited) { + if (lease) control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(installation); + } + } +} diff --git a/packages/core/src/api/contracts/integration-admin/common.ts b/packages/core/src/api/contracts/integration-admin/common.ts index cb09e2c0b..e0c432649 100644 --- a/packages/core/src/api/contracts/integration-admin/common.ts +++ b/packages/core/src/api/contracts/integration-admin/common.ts @@ -152,6 +152,9 @@ export const adminIntegrationBindingsSchema = z.strictObject({ }); export const adminIntegrationBindingSetSchema = z.strictObject({ serviceId: z.string().min(1) }); export const adminIntegrationEmptyMutationSchema = z.strictObject({}).optional(); +export const adminIntegrationIdempotencyHeadersSchema = z.object({ + "idempotency-key": z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/), +}); export const adminIntegrationValidationErrorSchema = z.strictObject({ errors: z.array(z.string()), }); diff --git a/packages/core/src/api/contracts/integration-admin/operations.ts b/packages/core/src/api/contracts/integration-admin/operations.ts index 51921a9d8..462ee1364 100644 --- a/packages/core/src/api/contracts/integration-admin/operations.ts +++ b/packages/core/src/api/contracts/integration-admin/operations.ts @@ -13,6 +13,7 @@ import { adminIntegrationDetailSchema, adminIntegrationEmptyMutationSchema, adminIntegrationEnvVarsSchema, + adminIntegrationIdempotencyHeadersSchema, adminIntegrationImportFailureSchema, adminIntegrationParamsSchema, adminIntegrationRefParamsSchema, @@ -44,7 +45,11 @@ const mutation = { params: adminIntegrationParamsSchema, body: adminIntegrationEmptyMutationSchema, }; -const reloaded = { ...missing, 200: adminJson(adminIntegrationReloadResultSchema) }; +const reloaded = { + ...missing, + 200: adminJson(adminIntegrationReloadResultSchema), + 409: adminJson(httpErrorSchema), +}; const ok = adminJson(z.strictObject({ ok: z.literal(true) })); /** Complete integration/credential/overview family. Authentication metadata never enforces access. */ @@ -78,12 +83,15 @@ export const integrationAdminOperations = { method: "POST", path: "/api/admin/integrations/import", auth: "admin", - request: { body: adminIntegrationConfigImportSchema }, + request: { + body: adminIntegrationConfigImportSchema, + headers: adminIntegrationIdempotencyHeadersSchema, + }, responses: { ...common, 400: configErrors, 200: adminJson(adminIntegrationConfigImportedSchema), - 409: adminJson(adminIntegrationImportFailureSchema), + 409: adminJson(z.union([adminIntegrationImportFailureSchema, httpErrorSchema])), }, }, runAdminIntegrationHealthChecks: { @@ -104,25 +112,30 @@ export const integrationAdminOperations = { method: "PATCH", path: "/api/admin/integrations/:id/config", auth: "admin", - request: { params: adminIntegrationParamsSchema, body: adminIntegrationConfigPatchSchema }, + request: { + params: adminIntegrationParamsSchema, + body: adminIntegrationConfigPatchSchema, + headers: adminIntegrationIdempotencyHeadersSchema, + }, responses: { ...missing, 400: configErrors, 200: adminJson(adminIntegrationConfigUpdatedSchema), + 409: adminJson(httpErrorSchema), }, }, enableAdminIntegration: { method: "POST", path: "/api/admin/integrations/:id/enable", auth: "admin", - request: mutation, + request: { ...mutation, headers: adminIntegrationIdempotencyHeadersSchema }, responses: reloaded, }, disableAdminIntegration: { method: "POST", path: "/api/admin/integrations/:id/disable", auth: "admin", - request: mutation, + request: { ...mutation, headers: adminIntegrationIdempotencyHeadersSchema }, responses: reloaded, }, getAdminIntegrationHealth: { @@ -204,6 +217,7 @@ export const integrationAdminOperations = { path: "/api/admin/credentials/:integrationId/:key", auth: "admin", request: { + headers: adminIntegrationIdempotencyHeadersSchema, params: adminIntegrationCredentialParamsSchema, body: adminIntegrationCredentialSetSchema, }, @@ -217,6 +231,7 @@ export const integrationAdminOperations = { path: "/api/admin/credentials/:integrationId/:key", auth: "admin", request: { + headers: adminIntegrationIdempotencyHeadersSchema, params: adminIntegrationCredentialParamsSchema, body: adminIntegrationEmptyMutationSchema, }, diff --git a/packages/core/src/api/contracts/service-admin/index.ts b/packages/core/src/api/contracts/service-admin/index.ts index cd707f978..73ce7f622 100644 --- a/packages/core/src/api/contracts/service-admin/index.ts +++ b/packages/core/src/api/contracts/service-admin/index.ts @@ -105,10 +105,15 @@ export const serviceAdminOperations = { method: "POST", path: "/api/admin/services/:id/config", auth: "admin", - request: { params: adminServiceParamsSchema, body: adminServiceConfigUpdateSchema }, + request: { + params: adminServiceParamsSchema, + body: adminServiceConfigUpdateSchema, + headers: adminServiceIdempotencyHeadersSchema, + }, responses: { ...adminCommonResponses, 200: adminJson(adminServiceOkSchema), + 409: adminJson(httpErrorSchema), 404: adminJson(httpErrorSchema), 400: adminJson(z.union([httpErrorSchema, adminServiceValidationErrorSchema])), }, @@ -128,24 +133,32 @@ export const serviceAdminOperations = { method: "PUT", path: "/api/admin/services/:id/credentials/:key", auth: "admin", - request: { params: adminServiceCredentialParamsSchema, body: adminServiceCredentialSetSchema }, + request: { + params: adminServiceCredentialParamsSchema, + body: adminServiceCredentialSetSchema, + headers: adminServiceIdempotencyHeadersSchema, + }, responses: { ...adminCommonResponses, 200: adminJson(adminServiceCredentialChangedSchema), 404: adminJson(httpErrorSchema), - 409: adminJson(adminServiceCredentialManagedSchema), + 409: adminJson(z.union([adminServiceCredentialManagedSchema, httpErrorSchema])), }, }, deleteAdminServiceCredential: { method: "DELETE", path: "/api/admin/services/:id/credentials/:key", auth: "admin", - request: { params: adminServiceCredentialParamsSchema, body: adminServiceEmptyMutationSchema }, + request: { + params: adminServiceCredentialParamsSchema, + body: adminServiceEmptyMutationSchema, + headers: adminServiceIdempotencyHeadersSchema, + }, responses: { ...adminCommonResponses, 200: adminJson(adminServiceCredentialChangedSchema), 404: adminJson(httpErrorSchema), - 409: adminJson(adminServiceCredentialManagedSchema), + 409: adminJson(z.union([adminServiceCredentialManagedSchema, httpErrorSchema])), }, }, streamAdminServiceLogs: { @@ -211,10 +224,14 @@ export const serviceAdminOperations = { method: "PUT", path: "/api/admin/services/selection", auth: "admin", - request: { body: adminServiceSelectionSetSchema }, + request: { + body: adminServiceSelectionSetSchema, + headers: adminServiceIdempotencyHeadersSchema, + }, responses: { ...adminCommonResponses, 200: adminJson(adminServiceSelectionUpdatedSchema), + 409: adminJson(httpErrorSchema), 503: adminJson(httpErrorSchema), }, }, @@ -281,6 +298,7 @@ export const serviceAdminOperations = { responses: { ...adminCommonResponses, 200: adminJson(adminServiceComposeStartedSchema), + 409: adminJson(httpErrorSchema), 400: adminJson(z.union([httpErrorSchema, adminServiceFailureSchema])), }, }, diff --git a/packages/core/src/api/contracts/service-admin/services.ts b/packages/core/src/api/contracts/service-admin/services.ts index 55c26783d..4e86877e5 100644 --- a/packages/core/src/api/contracts/service-admin/services.ts +++ b/packages/core/src/api/contracts/service-admin/services.ts @@ -126,7 +126,7 @@ export const adminServiceCredentialManagedSchema = z.strictObject({ error: z.string(), }); export const adminServiceActionSchema = z.strictObject({ - action: z.enum(["start", "stop", "restart"]), + action: z.enum(["start", "stop", "restart", "apply"]), }); const build = z.strictObject({ action: z.literal("build"), @@ -137,10 +137,16 @@ const build = z.strictObject({ }); export const adminServicesBulkActionSchema = z.union([ z.strictObject({ - action: z.enum(["start", "stop", "restart", "update"]), + action: z.enum(["start", "stop", "restart"]), serviceIds: adminServiceIdListSchema.min(1), continueOnError: z.boolean().optional(), }), + z.strictObject({ + action: z.literal("update"), + serviceIds: adminServiceIdListSchema.min(1), + noDeps: z.boolean().optional(), + continueOnError: z.boolean().optional(), + }), build.refine( (value) => value.all === true || Boolean(value.serviceIds?.length), "Build requires serviceIds or all: true", diff --git a/packages/core/src/api/generated/admin/integration-admin.generated.ts b/packages/core/src/api/generated/admin/integration-admin.generated.ts index b99e9bd49..c656d0b3e 100644 --- a/packages/core/src/api/generated/admin/integration-admin.generated.ts +++ b/packages/core/src/api/generated/admin/integration-admin.generated.ts @@ -15,31 +15,37 @@ import type { DeleteAdminIntegrationCredential401, DeleteAdminIntegrationCredential403, DeleteAdminIntegrationCredential404, + DeleteAdminIntegrationCredential409, DeleteAdminIntegrationCredential413, DeleteAdminIntegrationCredential415, DeleteAdminIntegrationCredential429, DeleteAdminIntegrationCredential500, DeleteAdminIntegrationCredentialBody, + DeleteAdminIntegrationCredentialHeaders, DisableAdminIntegration200, DisableAdminIntegration400, DisableAdminIntegration401, DisableAdminIntegration403, DisableAdminIntegration404, + DisableAdminIntegration409, DisableAdminIntegration413, DisableAdminIntegration415, DisableAdminIntegration429, DisableAdminIntegration500, DisableAdminIntegrationBody, + DisableAdminIntegrationHeaders, EnableAdminIntegration200, EnableAdminIntegration400, EnableAdminIntegration401, EnableAdminIntegration403, EnableAdminIntegration404, + EnableAdminIntegration409, EnableAdminIntegration413, EnableAdminIntegration415, EnableAdminIntegration429, EnableAdminIntegration500, EnableAdminIntegrationBody, + EnableAdminIntegrationHeaders, ExportAdminIntegrationConfigs200, ExportAdminIntegrationConfigs400, ExportAdminIntegrationConfigs401, @@ -103,6 +109,7 @@ import type { ImportAdminIntegrationConfigs429, ImportAdminIntegrationConfigs500, ImportAdminIntegrationConfigsBody, + ImportAdminIntegrationConfigsHeaders, ListAdminCredentials200, ListAdminCredentials400, ListAdminCredentials401, @@ -140,11 +147,13 @@ import type { PatchAdminIntegrationConfig401, PatchAdminIntegrationConfig403, PatchAdminIntegrationConfig404, + PatchAdminIntegrationConfig409, PatchAdminIntegrationConfig413, PatchAdminIntegrationConfig415, PatchAdminIntegrationConfig429, PatchAdminIntegrationConfig500, PatchAdminIntegrationConfigBody, + PatchAdminIntegrationConfigHeaders, ReloadAdminIntegration200, ReloadAdminIntegration400, ReloadAdminIntegration401, @@ -188,12 +197,14 @@ import type { SetAdminIntegrationCredential401, SetAdminIntegrationCredential403, SetAdminIntegrationCredential404, + SetAdminIntegrationCredential409, SetAdminIntegrationCredential413, SetAdminIntegrationCredential415, SetAdminIntegrationCredential429, SetAdminIntegrationCredential500, SetAdminIntegrationCredential503, SetAdminIntegrationCredentialBody, + SetAdminIntegrationCredentialHeaders, TestAdminIntegrationCredentials200, TestAdminIntegrationCredentials400, TestAdminIntegrationCredentials401, @@ -514,6 +525,11 @@ export type setAdminIntegrationCredentialResponse404 = { status: 404; }; +export type setAdminIntegrationCredentialResponse409 = { + data: SetAdminIntegrationCredential409; + status: 409; +}; + export type setAdminIntegrationCredentialResponse413 = { data: SetAdminIntegrationCredential413; status: 413; @@ -548,6 +564,7 @@ export type setAdminIntegrationCredentialResponseError = ( | setAdminIntegrationCredentialResponse401 | setAdminIntegrationCredentialResponse403 | setAdminIntegrationCredentialResponse404 + | setAdminIntegrationCredentialResponse409 | setAdminIntegrationCredentialResponse413 | setAdminIntegrationCredentialResponse415 | setAdminIntegrationCredentialResponse429 @@ -569,6 +586,7 @@ export const setAdminIntegrationCredential = async ( integrationId: string, key: string, setAdminIntegrationCredentialBody: SetAdminIntegrationCredentialBody, + headers: SetAdminIntegrationCredentialHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -595,7 +613,7 @@ export const setAdminIntegrationCredential = async ( { ...options, method: "PUT", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(setAdminIntegrationCredentialBody), }, ); @@ -626,6 +644,11 @@ export type deleteAdminIntegrationCredentialResponse404 = { status: 404; }; +export type deleteAdminIntegrationCredentialResponse409 = { + data: DeleteAdminIntegrationCredential409; + status: 409; +}; + export type deleteAdminIntegrationCredentialResponse413 = { data: DeleteAdminIntegrationCredential413; status: 413; @@ -655,6 +678,7 @@ export type deleteAdminIntegrationCredentialResponseError = ( | deleteAdminIntegrationCredentialResponse401 | deleteAdminIntegrationCredentialResponse403 | deleteAdminIntegrationCredentialResponse404 + | deleteAdminIntegrationCredentialResponse409 | deleteAdminIntegrationCredentialResponse413 | deleteAdminIntegrationCredentialResponse415 | deleteAdminIntegrationCredentialResponse429 @@ -674,6 +698,7 @@ export const getDeleteAdminIntegrationCredentialUrl = (integrationId: string, ke export const deleteAdminIntegrationCredential = async ( integrationId: string, key: string, + headers: DeleteAdminIntegrationCredentialHeaders, deleteAdminIntegrationCredentialBody?: DeleteAdminIntegrationCredentialBody, options?: Parameters[1], ): Promise => { @@ -701,7 +726,7 @@ export const deleteAdminIntegrationCredential = async ( { ...options, method: "DELETE", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(deleteAdminIntegrationCredentialBody), }, ); @@ -1103,6 +1128,7 @@ export const getImportAdminIntegrationConfigsUrl = () => { export const importAdminIntegrationConfigs = async ( importAdminIntegrationConfigsBody: ImportAdminIntegrationConfigsBody, + headers: ImportAdminIntegrationConfigsHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -1129,7 +1155,7 @@ export const importAdminIntegrationConfigs = async ( { ...options, method: "POST", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(importAdminIntegrationConfigsBody), }, ); @@ -1336,6 +1362,11 @@ export type patchAdminIntegrationConfigResponse404 = { status: 404; }; +export type patchAdminIntegrationConfigResponse409 = { + data: PatchAdminIntegrationConfig409; + status: 409; +}; + export type patchAdminIntegrationConfigResponse413 = { data: PatchAdminIntegrationConfig413; status: 413; @@ -1364,6 +1395,7 @@ export type patchAdminIntegrationConfigResponseError = ( | patchAdminIntegrationConfigResponse401 | patchAdminIntegrationConfigResponse403 | patchAdminIntegrationConfigResponse404 + | patchAdminIntegrationConfigResponse409 | patchAdminIntegrationConfigResponse413 | patchAdminIntegrationConfigResponse415 | patchAdminIntegrationConfigResponse429 @@ -1383,6 +1415,7 @@ export const getPatchAdminIntegrationConfigUrl = (id: string) => { export const patchAdminIntegrationConfig = async ( id: string, patchAdminIntegrationConfigBody: PatchAdminIntegrationConfigBody, + headers: PatchAdminIntegrationConfigHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -1409,7 +1442,7 @@ export const patchAdminIntegrationConfig = async ( { ...options, method: "PATCH", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(patchAdminIntegrationConfigBody), }, ); @@ -1440,6 +1473,11 @@ export type disableAdminIntegrationResponse404 = { status: 404; }; +export type disableAdminIntegrationResponse409 = { + data: DisableAdminIntegration409; + status: 409; +}; + export type disableAdminIntegrationResponse413 = { data: DisableAdminIntegration413; status: 413; @@ -1468,6 +1506,7 @@ export type disableAdminIntegrationResponseError = ( | disableAdminIntegrationResponse401 | disableAdminIntegrationResponse403 | disableAdminIntegrationResponse404 + | disableAdminIntegrationResponse409 | disableAdminIntegrationResponse413 | disableAdminIntegrationResponse415 | disableAdminIntegrationResponse429 @@ -1486,6 +1525,7 @@ export const getDisableAdminIntegrationUrl = (id: string) => { export const disableAdminIntegration = async ( id: string, + headers: DisableAdminIntegrationHeaders, disableAdminIntegrationBody?: DisableAdminIntegrationBody, options?: Parameters[1], ): Promise => { @@ -1513,7 +1553,7 @@ export const disableAdminIntegration = async ( { ...options, method: "POST", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(disableAdminIntegrationBody), }, ); @@ -1544,6 +1584,11 @@ export type enableAdminIntegrationResponse404 = { status: 404; }; +export type enableAdminIntegrationResponse409 = { + data: EnableAdminIntegration409; + status: 409; +}; + export type enableAdminIntegrationResponse413 = { data: EnableAdminIntegration413; status: 413; @@ -1572,6 +1617,7 @@ export type enableAdminIntegrationResponseError = ( | enableAdminIntegrationResponse401 | enableAdminIntegrationResponse403 | enableAdminIntegrationResponse404 + | enableAdminIntegrationResponse409 | enableAdminIntegrationResponse413 | enableAdminIntegrationResponse415 | enableAdminIntegrationResponse429 @@ -1590,6 +1636,7 @@ export const getEnableAdminIntegrationUrl = (id: string) => { export const enableAdminIntegration = async ( id: string, + headers: EnableAdminIntegrationHeaders, enableAdminIntegrationBody?: EnableAdminIntegrationBody, options?: Parameters[1], ): Promise => { @@ -1617,7 +1664,7 @@ export const enableAdminIntegration = async ( { ...options, method: "POST", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(enableAdminIntegrationBody), }, ); diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts new file mode 100644 index 000000000..d3c7393b5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationCredential409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialHeaders.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialHeaders.ts new file mode 100644 index 000000000..218ad1fc4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationCredentialHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts new file mode 100644 index 000000000..cf1036277 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DisableAdminIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/disableAdminIntegrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegrationHeaders.ts new file mode 100644 index 000000000..47daa1d31 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegrationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DisableAdminIntegrationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts new file mode 100644 index 000000000..95900167d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type EnableAdminIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/enableAdminIntegrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegrationHeaders.ts new file mode 100644 index 000000000..129d4e043 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegrationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type EnableAdminIntegrationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts index d12fdccb9..fccd3b6f3 100644 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts +++ b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts @@ -1,22 +1,91 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. -import type { ImportAdminIntegrationConfigs409Apply } from "./importAdminIntegrationConfigs409Apply"; -import type { ImportAdminIntegrationConfigs409Compensation } from "./importAdminIntegrationConfigs409Compensation"; -import type { ImportAdminIntegrationConfigs409Database } from "./importAdminIntegrationConfigs409Database"; -import type { ImportAdminIntegrationConfigs409Error } from "./importAdminIntegrationConfigs409Error"; -import type { ImportAdminIntegrationConfigs409Phase } from "./importAdminIntegrationConfigs409Phase"; -import type { ImportAdminIntegrationConfigs409Reason } from "./importAdminIntegrationConfigs409Reason"; -import type { ImportAdminIntegrationConfigs409Recovery } from "./importAdminIntegrationConfigs409Recovery"; -import type { ImportAdminIntegrationConfigs409Runtime } from "./importAdminIntegrationConfigs409Runtime"; -export type ImportAdminIntegrationConfigs409 = { - ok: false; - error: ImportAdminIntegrationConfigs409Error; - phase: ImportAdminIntegrationConfigs409Phase; - recovery: ImportAdminIntegrationConfigs409Recovery; - database: ImportAdminIntegrationConfigs409Database; - runtime?: ImportAdminIntegrationConfigs409Runtime; - apply?: ImportAdminIntegrationConfigs409Apply; - compensation?: ImportAdminIntegrationConfigs409Compensation; - reason?: ImportAdminIntegrationConfigs409Reason; -}; +export type ImportAdminIntegrationConfigs409 = + | { + ok: false; + error: "Integration import failed"; + phase: "persistence" | "apply" | "reload" | "ownership"; + recovery: "restored" | "reconciliation_required"; + database: "unknown" | "retained" | "conflict" | "restored"; + runtime?: "unchanged" | "unknown" | "restored"; + apply?: + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; + compensation?: + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; + reason?: "submitted_snapshot_not_owned" | "previous_applied_not_proven"; + } + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Apply.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Apply.ts deleted file mode 100644 index 366b272b6..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Apply.ts +++ /dev/null @@ -1,34 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Apply = - | { - outcome: "committed"; - /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ - revisionId: string; - /** - * @maxItems 256 - * @items.minLength 1 - * @items.maxLength 64 - * @items.pattern ^[a-z0-9][a-z0-9-]*$ - */ - enabledServiceIds: string[]; - followUp: "complete" | "failed"; - cleanupFailed: boolean; - publisherCleanupFailed: boolean; - registryUpdateFailed: boolean; - } - | { - outcome: "not_committed"; - reason: "conflict" | "busy" | "failed"; - currentRevisionId?: string | null; - cleanupFailed: boolean; - publisherCleanupFailed: boolean; - registryUpdateFailed: boolean; - } - | { - outcome: "unknown"; - reason: "transport_or_result"; - publisherCleanupFailed: boolean; - registryUpdateFailed: boolean; - }; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Compensation.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Compensation.ts deleted file mode 100644 index 1e72224cb..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Compensation.ts +++ /dev/null @@ -1,34 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Compensation = - | { - outcome: "committed"; - /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ - revisionId: string; - /** - * @maxItems 256 - * @items.minLength 1 - * @items.maxLength 64 - * @items.pattern ^[a-z0-9][a-z0-9-]*$ - */ - enabledServiceIds: string[]; - followUp: "complete" | "failed"; - cleanupFailed: boolean; - publisherCleanupFailed: boolean; - registryUpdateFailed: boolean; - } - | { - outcome: "not_committed"; - reason: "conflict" | "busy" | "failed"; - currentRevisionId?: string | null; - cleanupFailed: boolean; - publisherCleanupFailed: boolean; - registryUpdateFailed: boolean; - } - | { - outcome: "unknown"; - reason: "transport_or_result"; - publisherCleanupFailed: boolean; - registryUpdateFailed: boolean; - }; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Database.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Database.ts deleted file mode 100644 index 24e1411c5..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Database.ts +++ /dev/null @@ -1,12 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Database = - (typeof ImportAdminIntegrationConfigs409Database)[keyof typeof ImportAdminIntegrationConfigs409Database]; - -export const ImportAdminIntegrationConfigs409Database = { - unknown: "unknown", - retained: "retained", - conflict: "conflict", - restored: "restored", -} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Error.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Error.ts deleted file mode 100644 index c68bdfdc6..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Error.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Error = - (typeof ImportAdminIntegrationConfigs409Error)[keyof typeof ImportAdminIntegrationConfigs409Error]; - -export const ImportAdminIntegrationConfigs409Error = { - Integration_import_failed: "Integration import failed", -} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Phase.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Phase.ts deleted file mode 100644 index e98241105..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Phase.ts +++ /dev/null @@ -1,12 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Phase = - (typeof ImportAdminIntegrationConfigs409Phase)[keyof typeof ImportAdminIntegrationConfigs409Phase]; - -export const ImportAdminIntegrationConfigs409Phase = { - persistence: "persistence", - apply: "apply", - reload: "reload", - ownership: "ownership", -} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Reason.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Reason.ts deleted file mode 100644 index 32bd5100a..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Reason.ts +++ /dev/null @@ -1,10 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Reason = - (typeof ImportAdminIntegrationConfigs409Reason)[keyof typeof ImportAdminIntegrationConfigs409Reason]; - -export const ImportAdminIntegrationConfigs409Reason = { - submitted_snapshot_not_owned: "submitted_snapshot_not_owned", - previous_applied_not_proven: "previous_applied_not_proven", -} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Recovery.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Recovery.ts deleted file mode 100644 index f095621e6..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Recovery.ts +++ /dev/null @@ -1,10 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Recovery = - (typeof ImportAdminIntegrationConfigs409Recovery)[keyof typeof ImportAdminIntegrationConfigs409Recovery]; - -export const ImportAdminIntegrationConfigs409Recovery = { - restored: "restored", - reconciliation_required: "reconciliation_required", -} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Runtime.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Runtime.ts deleted file mode 100644 index 1b29c927f..000000000 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409Runtime.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type ImportAdminIntegrationConfigs409Runtime = - (typeof ImportAdminIntegrationConfigs409Runtime)[keyof typeof ImportAdminIntegrationConfigs409Runtime]; - -export const ImportAdminIntegrationConfigs409Runtime = { - unchanged: "unchanged", - unknown: "unknown", - restored: "restored", -} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigsHeaders.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigsHeaders.ts new file mode 100644 index 000000000..e1f376aa8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigsHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ImportAdminIntegrationConfigsHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/index.ts b/packages/core/src/api/generated/admin/integration-models/index.ts index 3d75ce4c1..5df2d4dfd 100644 --- a/packages/core/src/api/generated/admin/integration-models/index.ts +++ b/packages/core/src/api/generated/admin/integration-models/index.ts @@ -21,31 +21,37 @@ export * from "./deleteAdminIntegrationCredential400"; export * from "./deleteAdminIntegrationCredential401"; export * from "./deleteAdminIntegrationCredential403"; export * from "./deleteAdminIntegrationCredential404"; +export * from "./deleteAdminIntegrationCredential409"; export * from "./deleteAdminIntegrationCredential413"; export * from "./deleteAdminIntegrationCredential415"; export * from "./deleteAdminIntegrationCredential429"; export * from "./deleteAdminIntegrationCredential500"; export * from "./deleteAdminIntegrationCredentialBody"; +export * from "./deleteAdminIntegrationCredentialHeaders"; export * from "./disableAdminIntegration200"; export * from "./disableAdminIntegration400"; export * from "./disableAdminIntegration401"; export * from "./disableAdminIntegration403"; export * from "./disableAdminIntegration404"; +export * from "./disableAdminIntegration409"; export * from "./disableAdminIntegration413"; export * from "./disableAdminIntegration415"; export * from "./disableAdminIntegration429"; export * from "./disableAdminIntegration500"; export * from "./disableAdminIntegrationBody"; +export * from "./disableAdminIntegrationHeaders"; export * from "./enableAdminIntegration200"; export * from "./enableAdminIntegration400"; export * from "./enableAdminIntegration401"; export * from "./enableAdminIntegration403"; export * from "./enableAdminIntegration404"; +export * from "./enableAdminIntegration409"; export * from "./enableAdminIntegration413"; export * from "./enableAdminIntegration415"; export * from "./enableAdminIntegration429"; export * from "./enableAdminIntegration500"; export * from "./enableAdminIntegrationBody"; +export * from "./enableAdminIntegrationHeaders"; export * from "./exportAdminIntegrationConfigs200"; export * from "./exportAdminIntegrationConfigs200IntegrationsItem"; export * from "./exportAdminIntegrationConfigs200IntegrationsItemConfig"; @@ -170,14 +176,6 @@ export * from "./importAdminIntegrationConfigs400"; export * from "./importAdminIntegrationConfigs401"; export * from "./importAdminIntegrationConfigs403"; export * from "./importAdminIntegrationConfigs409"; -export * from "./importAdminIntegrationConfigs409Apply"; -export * from "./importAdminIntegrationConfigs409Compensation"; -export * from "./importAdminIntegrationConfigs409Database"; -export * from "./importAdminIntegrationConfigs409Error"; -export * from "./importAdminIntegrationConfigs409Phase"; -export * from "./importAdminIntegrationConfigs409Reason"; -export * from "./importAdminIntegrationConfigs409Recovery"; -export * from "./importAdminIntegrationConfigs409Runtime"; export * from "./importAdminIntegrationConfigs413"; export * from "./importAdminIntegrationConfigs415"; export * from "./importAdminIntegrationConfigs429"; @@ -185,6 +183,7 @@ export * from "./importAdminIntegrationConfigs500"; export * from "./importAdminIntegrationConfigsBody"; export * from "./importAdminIntegrationConfigsBodyIntegrationsItem"; export * from "./importAdminIntegrationConfigsBodyIntegrationsItemConfig"; +export * from "./importAdminIntegrationConfigsHeaders"; export * from "./listAdminCredentials200"; export * from "./listAdminCredentials200CredentialsItem"; export * from "./listAdminCredentials400"; @@ -231,11 +230,13 @@ export * from "./patchAdminIntegrationConfig400"; export * from "./patchAdminIntegrationConfig401"; export * from "./patchAdminIntegrationConfig403"; export * from "./patchAdminIntegrationConfig404"; +export * from "./patchAdminIntegrationConfig409"; export * from "./patchAdminIntegrationConfig413"; export * from "./patchAdminIntegrationConfig415"; export * from "./patchAdminIntegrationConfig429"; export * from "./patchAdminIntegrationConfig500"; export * from "./patchAdminIntegrationConfigBody"; +export * from "./patchAdminIntegrationConfigHeaders"; export * from "./reloadAdminIntegration200"; export * from "./reloadAdminIntegration400"; export * from "./reloadAdminIntegration401"; @@ -281,12 +282,14 @@ export * from "./setAdminIntegrationCredential400"; export * from "./setAdminIntegrationCredential401"; export * from "./setAdminIntegrationCredential403"; export * from "./setAdminIntegrationCredential404"; +export * from "./setAdminIntegrationCredential409"; export * from "./setAdminIntegrationCredential413"; export * from "./setAdminIntegrationCredential415"; export * from "./setAdminIntegrationCredential429"; export * from "./setAdminIntegrationCredential500"; export * from "./setAdminIntegrationCredential503"; export * from "./setAdminIntegrationCredentialBody"; +export * from "./setAdminIntegrationCredentialHeaders"; export * from "./testAdminIntegrationCredentials200"; export * from "./testAdminIntegrationCredentials200Status"; export * from "./testAdminIntegrationCredentials400"; diff --git a/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts new file mode 100644 index 000000000..bce415243 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PatchAdminIntegrationConfig409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfigHeaders.ts b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfigHeaders.ts new file mode 100644 index 000000000..724e22310 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfigHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PatchAdminIntegrationConfigHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts new file mode 100644 index 000000000..71bd170f9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationCredential409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialHeaders.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialHeaders.ts new file mode 100644 index 000000000..8577cf1a2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationCredentialHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/service-admin.generated.ts b/packages/core/src/api/generated/admin/service-admin.generated.ts index 500a781d9..62db3aaa4 100644 --- a/packages/core/src/api/generated/admin/service-admin.generated.ts +++ b/packages/core/src/api/generated/admin/service-admin.generated.ts @@ -73,6 +73,7 @@ import type { DeleteAdminServiceCredential429, DeleteAdminServiceCredential500, DeleteAdminServiceCredentialBody, + DeleteAdminServiceCredentialHeaders, GetAdminComposePreview400, GetAdminComposePreview401, GetAdminComposePreview403, @@ -272,20 +273,24 @@ import type { SetAdminServiceCredential429, SetAdminServiceCredential500, SetAdminServiceCredentialBody, + SetAdminServiceCredentialHeaders, SetAdminServiceSelection200, SetAdminServiceSelection400, SetAdminServiceSelection401, SetAdminServiceSelection403, + SetAdminServiceSelection409, SetAdminServiceSelection413, SetAdminServiceSelection415, SetAdminServiceSelection429, SetAdminServiceSelection500, SetAdminServiceSelection503, SetAdminServiceSelectionBody, + SetAdminServiceSelectionHeaders, StartAdminCompose200, StartAdminCompose400, StartAdminCompose401, StartAdminCompose403, + StartAdminCompose409, StartAdminCompose413, StartAdminCompose415, StartAdminCompose429, @@ -318,11 +323,13 @@ import type { UpdateAdminServiceConfig401, UpdateAdminServiceConfig403, UpdateAdminServiceConfig404, + UpdateAdminServiceConfig409, UpdateAdminServiceConfig413, UpdateAdminServiceConfig415, UpdateAdminServiceConfig429, UpdateAdminServiceConfig500, UpdateAdminServiceConfigBody, + UpdateAdminServiceConfigHeaders, } from "./service-models"; export type stopAdminComposeResponse400 = { data: StopAdminCompose400; @@ -514,6 +521,11 @@ export type startAdminComposeResponse403 = { status: 403; }; +export type startAdminComposeResponse409 = { + data: StartAdminCompose409; + status: 409; +}; + export type startAdminComposeResponse413 = { data: StartAdminCompose413; status: 413; @@ -541,6 +553,7 @@ export type startAdminComposeResponseError = ( | startAdminComposeResponse400 | startAdminComposeResponse401 | startAdminComposeResponse403 + | startAdminComposeResponse409 | startAdminComposeResponse413 | startAdminComposeResponse415 | startAdminComposeResponse429 @@ -2081,6 +2094,11 @@ export type setAdminServiceSelectionResponse403 = { status: 403; }; +export type setAdminServiceSelectionResponse409 = { + data: SetAdminServiceSelection409; + status: 409; +}; + export type setAdminServiceSelectionResponse413 = { data: SetAdminServiceSelection413; status: 413; @@ -2113,6 +2131,7 @@ export type setAdminServiceSelectionResponseError = ( | setAdminServiceSelectionResponse400 | setAdminServiceSelectionResponse401 | setAdminServiceSelectionResponse403 + | setAdminServiceSelectionResponse409 | setAdminServiceSelectionResponse413 | setAdminServiceSelectionResponse415 | setAdminServiceSelectionResponse429 @@ -2132,6 +2151,7 @@ export const getSetAdminServiceSelectionUrl = () => { export const setAdminServiceSelection = async ( setAdminServiceSelectionBody: SetAdminServiceSelectionBody, + headers: SetAdminServiceSelectionHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -2158,7 +2178,7 @@ export const setAdminServiceSelection = async ( { ...options, method: "PUT", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(setAdminServiceSelectionBody), }, ); @@ -2464,6 +2484,11 @@ export type updateAdminServiceConfigResponse404 = { status: 404; }; +export type updateAdminServiceConfigResponse409 = { + data: UpdateAdminServiceConfig409; + status: 409; +}; + export type updateAdminServiceConfigResponse413 = { data: UpdateAdminServiceConfig413; status: 413; @@ -2492,6 +2517,7 @@ export type updateAdminServiceConfigResponseError = ( | updateAdminServiceConfigResponse401 | updateAdminServiceConfigResponse403 | updateAdminServiceConfigResponse404 + | updateAdminServiceConfigResponse409 | updateAdminServiceConfigResponse413 | updateAdminServiceConfigResponse415 | updateAdminServiceConfigResponse429 @@ -2511,6 +2537,7 @@ export const getUpdateAdminServiceConfigUrl = (id: string) => { export const updateAdminServiceConfig = async ( id: string, updateAdminServiceConfigBody: UpdateAdminServiceConfigBody, + headers: UpdateAdminServiceConfigHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -2537,7 +2564,7 @@ export const updateAdminServiceConfig = async ( { ...options, method: "POST", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(updateAdminServiceConfigBody), }, ); @@ -2704,6 +2731,7 @@ export const setAdminServiceCredential = async ( id: string, key: string, setAdminServiceCredentialBody: SetAdminServiceCredentialBody, + headers: SetAdminServiceCredentialHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -2730,7 +2758,7 @@ export const setAdminServiceCredential = async ( { ...options, method: "PUT", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(setAdminServiceCredentialBody), }, ); @@ -2815,6 +2843,7 @@ export const getDeleteAdminServiceCredentialUrl = (id: string, key: string) => { export const deleteAdminServiceCredential = async ( id: string, key: string, + headers: DeleteAdminServiceCredentialHeaders, deleteAdminServiceCredentialBody?: DeleteAdminServiceCredentialBody, options?: Parameters[1], ): Promise => { @@ -2842,7 +2871,7 @@ export const deleteAdminServiceCredential = async ( { ...options, method: "DELETE", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(deleteAdminServiceCredentialBody), }, ); diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts index f9e3f4961..3ef2f0d84 100644 --- a/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts @@ -1,8 +1,24 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. -import type { DeleteAdminServiceCredential409Code } from "./deleteAdminServiceCredential409Code"; -export type DeleteAdminServiceCredential409 = { - code: DeleteAdminServiceCredential409Code; - error: string; -}; +export type DeleteAdminServiceCredential409 = + | { + code: "DAWARICH_CREDENTIAL_MANAGED"; + error: string; + } + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409Code.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409Code.ts deleted file mode 100644 index 107c73467..000000000 --- a/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409Code.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type DeleteAdminServiceCredential409Code = - (typeof DeleteAdminServiceCredential409Code)[keyof typeof DeleteAdminServiceCredential409Code]; - -export const DeleteAdminServiceCredential409Code = { - DAWARICH_CREDENTIAL_MANAGED: "DAWARICH_CREDENTIAL_MANAGED", -} as const; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredentialHeaders.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredentialHeaders.ts new file mode 100644 index 000000000..7ecc599df --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredentialHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminServiceCredentialHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index c8c67768c..20c8173aa 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -44,12 +44,12 @@ export * from "./deleteAdminServiceCredential401"; export * from "./deleteAdminServiceCredential403"; export * from "./deleteAdminServiceCredential404"; export * from "./deleteAdminServiceCredential409"; -export * from "./deleteAdminServiceCredential409Code"; export * from "./deleteAdminServiceCredential413"; export * from "./deleteAdminServiceCredential415"; export * from "./deleteAdminServiceCredential429"; export * from "./deleteAdminServiceCredential500"; export * from "./deleteAdminServiceCredentialBody"; +export * from "./deleteAdminServiceCredentialHeaders"; export * from "./getAdminComposePreview400"; export * from "./getAdminComposePreview401"; export * from "./getAdminComposePreview403"; @@ -367,29 +367,32 @@ export * from "./setAdminServiceCredential401"; export * from "./setAdminServiceCredential403"; export * from "./setAdminServiceCredential404"; export * from "./setAdminServiceCredential409"; -export * from "./setAdminServiceCredential409Code"; export * from "./setAdminServiceCredential413"; export * from "./setAdminServiceCredential415"; export * from "./setAdminServiceCredential429"; export * from "./setAdminServiceCredential500"; export * from "./setAdminServiceCredentialBody"; +export * from "./setAdminServiceCredentialHeaders"; export * from "./setAdminServiceSelection200"; export * from "./setAdminServiceSelection200Summary"; export * from "./setAdminServiceSelection200SummarySource"; export * from "./setAdminServiceSelection400"; export * from "./setAdminServiceSelection401"; export * from "./setAdminServiceSelection403"; +export * from "./setAdminServiceSelection409"; export * from "./setAdminServiceSelection413"; export * from "./setAdminServiceSelection415"; export * from "./setAdminServiceSelection429"; export * from "./setAdminServiceSelection500"; export * from "./setAdminServiceSelection503"; export * from "./setAdminServiceSelectionBody"; +export * from "./setAdminServiceSelectionHeaders"; export * from "./startAdminCompose200"; export * from "./startAdminCompose200Hardlinks"; export * from "./startAdminCompose400"; export * from "./startAdminCompose401"; export * from "./startAdminCompose403"; +export * from "./startAdminCompose409"; export * from "./startAdminCompose413"; export * from "./startAdminCompose415"; export * from "./startAdminCompose429"; @@ -422,9 +425,11 @@ export * from "./updateAdminServiceConfig400"; export * from "./updateAdminServiceConfig401"; export * from "./updateAdminServiceConfig403"; export * from "./updateAdminServiceConfig404"; +export * from "./updateAdminServiceConfig409"; export * from "./updateAdminServiceConfig413"; export * from "./updateAdminServiceConfig415"; export * from "./updateAdminServiceConfig429"; export * from "./updateAdminServiceConfig500"; export * from "./updateAdminServiceConfigBody"; export * from "./updateAdminServiceConfigBodyConfig"; +export * from "./updateAdminServiceConfigHeaders"; diff --git a/packages/core/src/api/generated/admin/service-models/runAdminServiceActionBodyAction.ts b/packages/core/src/api/generated/admin/service-models/runAdminServiceActionBodyAction.ts index 5e9ef64b6..f89d6ab24 100644 --- a/packages/core/src/api/generated/admin/service-models/runAdminServiceActionBodyAction.ts +++ b/packages/core/src/api/generated/admin/service-models/runAdminServiceActionBodyAction.ts @@ -8,4 +8,5 @@ export const RunAdminServiceActionBodyAction = { start: "start", stop: "stop", restart: "restart", + apply: "apply", } as const; diff --git a/packages/core/src/api/generated/admin/service-models/runAdminServicesBulkActionBody.ts b/packages/core/src/api/generated/admin/service-models/runAdminServicesBulkActionBody.ts index 5cb4e0da9..08b2d8902 100644 --- a/packages/core/src/api/generated/admin/service-models/runAdminServicesBulkActionBody.ts +++ b/packages/core/src/api/generated/admin/service-models/runAdminServicesBulkActionBody.ts @@ -3,7 +3,7 @@ export type RunAdminServicesBulkActionBody = | { - action: "start" | "stop" | "restart" | "update"; + action: "start" | "stop" | "restart"; /** * @minItems 1 * @items.minLength 1 @@ -11,6 +11,16 @@ export type RunAdminServicesBulkActionBody = serviceIds: string[]; continueOnError?: boolean; } + | { + action: "update"; + /** + * @minItems 1 + * @items.minLength 1 + */ + serviceIds: string[]; + noDeps?: boolean; + continueOnError?: boolean; + } | { action: "build"; /** @items.minLength 1 */ diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts index 170dc2ff1..58860e71d 100644 --- a/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts +++ b/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts @@ -1,8 +1,24 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. -import type { SetAdminServiceCredential409Code } from "./setAdminServiceCredential409Code"; -export type SetAdminServiceCredential409 = { - code: SetAdminServiceCredential409Code; - error: string; -}; +export type SetAdminServiceCredential409 = + | { + code: "DAWARICH_CREDENTIAL_MANAGED"; + error: string; + } + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409Code.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409Code.ts deleted file mode 100644 index fc8deb711..000000000 --- a/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409Code.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type SetAdminServiceCredential409Code = - (typeof SetAdminServiceCredential409Code)[keyof typeof SetAdminServiceCredential409Code]; - -export const SetAdminServiceCredential409Code = { - DAWARICH_CREDENTIAL_MANAGED: "DAWARICH_CREDENTIAL_MANAGED", -} as const; diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceCredentialHeaders.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceCredentialHeaders.ts new file mode 100644 index 000000000..271b201af --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/setAdminServiceCredentialHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminServiceCredentialHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts new file mode 100644 index 000000000..9379a8a6b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminServiceSelection409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceSelectionHeaders.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelectionHeaders.ts new file mode 100644 index 000000000..71bec8602 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelectionHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminServiceSelectionHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts b/packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts new file mode 100644 index 000000000..86900a70c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type StartAdminCompose409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts new file mode 100644 index 000000000..53a8ce991 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UpdateAdminServiceConfig409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfigHeaders.ts b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfigHeaders.ts new file mode 100644 index 000000000..445fb8b7b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfigHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UpdateAdminServiceConfigHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/build-artifacts/child-phase.ts b/packages/core/src/build-artifacts/child-phase.ts index 4c0158832..3f2e04c11 100644 --- a/packages/core/src/build-artifacts/child-phase.ts +++ b/packages/core/src/build-artifacts/child-phase.ts @@ -1,5 +1,19 @@ +import { + type ControlChildPhase, + type ControlChildReservation, + type ControlRecipient, + childPhaseKinds, + controlChildOperationKey, +} from "./control-child-records"; + +export { + type ControlChildReservation, + controlChildReservationSchema, +} from "./control-child-records"; + /** Server-only fixed phase identities. An observation never mints a dispatch ticket. */ import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; import { type ExecutionRef, executionFingerprint, @@ -7,17 +21,12 @@ import { normalizeExecutionAdmission, opsExecutionBudget, } from "../ops/execution-identity"; -import { canonicalJson } from "../ops/fingerprint"; // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { assertControlDelegationForward, assertControlPublicationScope, attestParentExecution, - type ControlChildPhase, type ControlPlannedChild, - type ControlRecipient, - childPhaseKinds, - controlChildOperationKey, executionAuthorityState, readPrivateRecord, retainedControlParent, @@ -37,7 +46,9 @@ import { validateOriginRecord, verifyControl, } from "./control-records"; +// biome-ignore-start lint/suspicious/noImportCycles: Reserved-child publication verifies original indexed bytes; family recovery consults the same admitted-child guards. Both calls are deferred until initialization completes. import { verifyFamilyOwnedRecord } from "./retention-family"; +// biome-ignore-end lint/suspicious/noImportCycles: Exact deferred original-owner/index guard join. // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { delegateArtifactTargetLease, @@ -47,44 +58,6 @@ import { } from "./target-lease"; // biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. -export type ControlChildReservation = { - version: 1; - phase: ControlChildPhase; - ordinal: number; - kind: string; - recipient: ControlRecipient; - serviceIds: readonly string[]; - execution: Pick; - operationFingerprint: string; -}; -const address = z - .strictObject({ - executor: executionRefSchema.shape.executor, - principal: executionRefSchema.shape.principal, - executionId: z.string(), - operationKey: executionRefSchema.shape.operationKey, - }) - .refine((v) => - v.executor === "ops-agent" - ? /^job1_[A-Za-z0-9_-]{16,64}$/.test(v.executionId) - : z.uuid().safeParse(v.executionId).success, - ); -export const controlChildReservationSchema: z.ZodType = z.lazy(() => - z.strictObject({ - version: z.literal(1), - phase: z.enum(Object.keys(childPhaseKinds) as [ControlChildPhase, ...ControlChildPhase[]]), - ordinal: z.number().int().min(0).max(63), - kind: z.string().min(1).max(128), - recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), - serviceIds: z - .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) - .min(1) - .max(256) - .refine((a) => a.every((v, i) => i === 0 || (a[i - 1] ?? v) < v)), - execution: address, - operationFingerprint: digest, - }), -); export function validateControlChildReservations(input: { role: ControlRecipient | "host-cli"; kind: string; diff --git a/packages/core/src/build-artifacts/configuration-fence-settlement-budget.ts b/packages/core/src/build-artifacts/configuration-fence-settlement-budget.ts new file mode 100644 index 000000000..e146161d6 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-fence-settlement-budget.ts @@ -0,0 +1,546 @@ +/** Prospective encoding bounds, not settlement authority. Empty signatures in + * these LOCAL width expressions are invalid proof by construction. Their exact + * Ed25519 wire width is added arithmetically; no hypothetical signed packet is + * published or returned. Actual source producers/verifiers remain mandatory. */ +import type { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { parseControlRootOperationKey } from "../ops/control-root-key"; +import { + type ConfigurationSettlementProjection, + configurationSettlementRecordDescriptors, +} from "./configuration-fence-settlement-descriptors"; +import { + configurationFinalEvidenceProjectionSchema, + configurationOwnerFinalSchema, + configurationPreparedGuardSchema, + configurationSettlementCapacitySchema, + configurationSettlementDomains, + configurationSettlementOwnerIdentitySchema, + configurationSettlementReservationProjectionSchema, + configurationSettlementTerminalTupleSchema, + settlementRecordDigest, + type signedConfigurationReservationProjectionSchema, +} from "./configuration-fence-settlement-records"; +import { controlDigest } from "./control-records"; +import type { FixedConfigurationMaintenancePlan } from "./fixed-configuration-maintenance"; + +const uuid = "ffffffff-ffff-4fff-bfff-ffffffffffff", + hash = "f".repeat(64), + at = Number.MAX_SAFE_INTEGER; +const hashes = (count: number) => + Array.from({ length: count }, (_, index) => controlDigest({ index })).sort(); +const wrap = (payload: T) => ({ payload, signature: "" }); +function width(value: unknown): number { + let missingSignatures = 0; + const count = (item: unknown): void => { + if (!item || typeof item !== "object") return; + if (!Array.isArray(item) && "signature" in item && item.signature === "") missingSignatures++; + for (const child of Object.values(item)) count(child); + }; + count(value); + return Buffer.byteLength(canonicalJson(value)) + missingSignatures * 86; +} +export function assertConfigurationSettlementPacketBudget( + plan: FixedConfigurationMaintenancePlan, + projection: ConfigurationSettlementProjection, + signedProjection: z.infer, +): Readonly> { + const prepare = plan.reservations.find((value) => value.phase === "configuration-fence"); + if ( + !prepare || + canonicalJson(projection.ownerPrepare) !== canonicalJson(plan.prepare) || + projection.planDigest !== plan.planDigest || + canonicalJson(signedProjection.payload) !== canonicalJson(projection) || + signedProjection.signature.length !== 86 + ) + throw Error("Original fixed settlement encoding source required"); + return packetWidths(plan, projection, signedProjection); +} +/** Before root acquisition there is no certificate/family/signature to trust. + * Size their fixed-width fields arithmetically; return ONLY byte counts. */ +export function assertProspectiveConfigurationSettlementPacketBudget( + plan: FixedConfigurationMaintenancePlan, + installationId: string, +) { + const reservation = plan.reservations.find( + (value) => value.phase === "configuration-fence-settle", + ); + if (!reservation) throw Error("Original planned settlement reservation required"); + const projection = configurationSettlementReservationProjectionSchema.parse({ + version: 2, + purpose: "configuration-fence-settlement-reservation", + installationId, + origin: { + version: 1, + issuer: "ops-agent", + rootKey: plan.prepare.operationKey, + notAfter: parseControlRootOperationKey(plan.prepare.operationKey).notAfter, + }, + ownerPrepare: plan.prepare, + rootAdmissionDigest: hash, + rootCertificateDigest: hash, + maintenanceId: plan.operation.maintenanceId, + candidateId: plan.operation.candidateId, + planDigest: plan.planDigest, + settlementId: plan.dmSettle.settlementId, + reservation, + operation: { kind: "configuration-maintenance.settle", ...plan.dmSettle }, + reservationDigest: settlementRecordDigest( + configurationSettlementDomains.reservationTuple, + reservation, + ), + issuerMembership: { + role: "ops-agent", + familyId: uuid, + rootMemberOrdinal: 255, + rootExecutionDigest: hash, + rootAdmissionDigest: hash, + }, + }); + return packetWidths(plan, projection, wrap(projection)); +} +function packetWidths( + plan: FixedConfigurationMaintenancePlan, + projection: ConfigurationSettlementProjection, + signedProjection: { payload: ConfigurationSettlementProjection; signature: string }, +): Readonly> { + const prepare = plan.reservations.find((value) => value.phase === "configuration-fence"); + if (!prepare) throw Error("Original planned preparation required"); + const descriptors = configurationSettlementRecordDescriptors( + projection, + prepare.execution.operationKey, + ), + all = [...descriptors.sender, ...descriptors.receiver], + identity = configurationSettlementOwnerIdentitySchema.parse({ + installationId: projection.installationId, + origin: projection.origin, + ownerPrepare: plan.prepare, + admissionDigest: projection.rootAdmissionDigest, + rootCertificateDigest: projection.rootCertificateDigest, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + settlementId: projection.settlementId, + }), + originalReceipt = { + fenceId: prepare.execution.executionId, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + ownerPrepare: plan.prepare, + planDigest: plan.planDigest, + revisionId: plan.operation.revisionId, + hardlinkPlanDigest: plan.dmPrepare.hardlinkPlanDigest, + sequence: 0, + state: "prepared" as const, + }, + guard = configurationPreparedGuardSchema.parse({ + fenceId: prepare.execution.executionId, + preparedReceipt: originalReceipt, + preparedReceiptDigest: hash, + preparedCanonicalDigest: hash, + preparedTerminalAt: at, + }), + runtime = { + verificationId: uuid, + mode: "contained" as const, + requiredCheckSetDigest: hash, + checkedAt: at, + }, + attempts = [ + ...plan.steps.map((step) => ({ + index: step.ordinal, + reservation: plan.reservations.find( + (value) => value.phase === step.phase && value.ordinal === step.ordinal, + )!, + })), + ...(plan.nestedHardlinks + ? [ + { + index: plan.steps.find((step) => step.phase === "maintenance-hardlinks")!.ordinal, + reservation: plan.nestedHardlinks, + }, + ] + : []), + ], + evidence = configurationFinalEvidenceProjectionSchema.parse({ + identity, + guard, + decision: { + decisionId: uuid, + decision: "continue", + checkpointVersion: at, + checkpointDigest: hash, + apiParentAttestationDigest: hash, + }, + generation: { + state: "committed", + revisionId: plan.operation.revisionId, + commitReceiptDigest: hash, + }, + followUp: { state: "contained", receiptDigests: hashes(attempts.length) }, + runtime, + outcome: "contained", + children: attempts.map((attempt) => ({ + planIndex: attempt.index, + executionDigest: hash, + terminalDigest: hash, + terminalAt: at, + effectSettlementDigest: hash, + publication: "committed", + })), + }), + final = configurationOwnerFinalSchema.parse({ + version: 2, + identity, + ownerSequence: at, + settlingAt: at, + expectedDmSequence: 0, + evidenceDigest: hash, + outcome: "contained", + privateInspectionId: uuid, + }), + committed = { + version: 2, + phase: "committed", + identity, + referenceOperationFingerprint: projection.reservation.operationFingerprint, + final, + evidence, + evidenceDigest: hash, + ownerReceiptDigest: hash, + committedAt: at, + }, + execution = { + ...projection.reservation.execution, + deadlineAt: at, + operationFingerprint: projection.reservation.operationFingerprint, + fingerprint: hash, + }, + envelope = { + version: 2, + bindingVersion: 3, + purpose: "configuration-fence-settlement", + installationId: projection.installationId, + origin: projection.origin, + ownerPrepare: plan.prepare, + admissionDigest: projection.rootAdmissionDigest, + rootCertificateDigest: projection.rootCertificateDigest, + settlementId: projection.settlementId, + projectionDigest: hash, + reservationDigest: projection.reservationDigest, + commitmentDigest: hash, + execution, + operation: projection.operation, + serviceIds: projection.reservation.serviceIds, + controlStartedAt: at - 60000, + controlDeadlineAt: at, + originalParentForwardDeadlineAt: plan.prepare.deadlineAt, + dispatchId: uuid, + }, + terminal = configurationSettlementTerminalTupleSchema.parse({ + execution, + settlementId: projection.settlementId, + commitmentDigest: hash, + fenceId: prepare.execution.executionId, + result: { ...originalReceipt, state: "settled", sequence: 1 }, + publication: "committed", + terminalAt: at, + terminalDigest: hash, + }), + publishedTerminalBase = { + version: 2, + origin: projection.origin, + admissionDigest: hash, + tuple: terminal, + }, + ack = { + version: 2, + installationId: projection.installationId, + origin: projection.origin, + settlementId: projection.settlementId, + controlExecutionDigest: hash, + terminalDigest: hash, + terminalAt: at, + commitmentDigest: hash, + }, + capacity = configurationSettlementCapacitySchema.parse({ + version: 2, + installationId: projection.installationId, + origin: projection.origin, + ownerPrepareDigest: hash, + settlementId: projection.settlementId, + proposedExecutionAddress: projection.reservation.execution, + reservedExecutionOrdinal: 127, + reservedIntentOrdinal: 255, + recordDescriptorsDigest: hash, + maxBytes: descriptors.receiver.reduce((sum, descriptor) => sum + descriptor.cap, 0), + projectionDigest: hash, + reservationDigest: hash, + }); + const unused = (role: "ops-agent" | "data-manager") => { + const local = role === "ops-agent" ? descriptors.sender : descriptors.receiver; + return { + intent: wrap({ + version: 2, + installationId: projection.installationId, + origin: projection.origin, + role, + ownerPrepareDigest: hash, + prepareExecutionDigest: hash, + settlementId: projection.settlementId, + projectionDigest: hash, + capacityReceiptDigest: hash, + sourceProofDigest: hash, + executionOrdinal: 127, + intentOrdinal: 255, + recordDescriptorsDigest: hash, + allocatedDescriptorDigests: local.map((value) => controlDigest(value)).sort(), + createdAt: at, + }), + confirmation: wrap({ + version: 2, + installationId: projection.installationId, + origin: projection.origin, + role, + ownerPrepareDigest: hash, + settlementId: projection.settlementId, + projectionDigest: hash, + intentDigest: hash, + slotStateDigest: hash, + confirmedAt: at, + }), + }; + }; + const publications = plan.reservations.map((value) => ({ + phase: value.phase, + ordinal: value.ordinal, + operationKey: value.execution.operationKey, + grantState: "unissued", + intentState: "not_attempted", + })), + noGuard = { + kind: "receiver_refused", + issuerFreezeProofDigest: hash, + prepareExecutionDigest: hash, + prepareAdmissionDigest: hash, + refusalProjectionDigest: hash, + effectSettlementDigest: hash, + canonicalTerminalDigest: hash, + canonicalTerminalAt: at, + dmUnusedIntentDigest: hash, + dmUnusedConfirmationDigest: hash, + capacityReceiptDigest: hash, + }, + closure = { + version: 2, + identity, + ownerSequence: at, + closedAt: at, + noGuardEvidence: noGuard, + originalContainmentDigests: hashes(2), + unusedSlotDigests: ["data-manager", "ops-agent"].map((role) => ({ + role, + intentDigest: hash, + confirmationDigest: hash, + })), + closureDigest: hash, + }, + refusedFreeze = { + version: 2, + kind: "issued_preparation_refused", + identity, + issuerSequence: at, + frozenAt: at, + prepareGrantDigest: hash, + prepareExecutionDigest: hash, + refusalProjectionDigest: hash, + remainingPublicationStates: publications.filter( + (value) => value.phase !== "configuration-fence", + ), + candidateClaimState: "not_consumed", + }, + neverIssuedFreeze = { + version: 2, + identity, + issuerSequence: at, + frozenAt: at, + prepareReservationDigest: hash, + publicationStates: publications, + }, + bundle = { + version: 2, + phase: "unused", + settlementId: projection.settlementId, + projectionDigest: hash, + closure: wrap(closure), + issuerFreezeProof: wrap(refusedFreeze), + issuerUnused: unused("ops-agent"), + }, + neverIssuedBundle = { + ...bundle, + closure: wrap({ + ...closure, + noGuardEvidence: { + kind: "never_issued", + issuerFreezeProofDigest: hash, + prepareReservationDigest: hash, + publicationStatesDigest: hash, + issuerSequence: at, + frozenAt: at, + }, + }), + issuerFreezeProof: wrap(neverIssuedFreeze), + }, + linked = { + version: 2, + projection: signedProjection, + binding: null, + unused: { + bundleDigest: hash, + closureDigest: hash, + issuerFreezeProofDigest: hash, + issuerUnusedIntentDigest: hash, + issuerUnusedConfirmationDigest: hash, + }, + }, + dmRefusal = { + version: 2, + installationId: projection.installationId, + origin: projection.origin, + ownerPrepare: plan.prepare, + rootAdmissionDigest: projection.rootAdmissionDigest, + rootCertificateDigest: projection.rootCertificateDigest, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + settlementId: projection.settlementId, + projectionDigest: hash, + reservationDigest: projection.reservationDigest, + prepareExecution: { + ...prepare.execution, + operationFingerprint: prepare.operationFingerprint, + deadlineAt: plan.prepare.deadlineAt, + fingerprint: hash, + }, + prepareAdmissionDigest: hash, + refusalReceipt: { + kind: "data-configuration-fence-refused", + maintenanceId: projection.maintenanceId, + reason: "capacity", + }, + effectReceiptDigest: hash, + effectSettlementDigest: hash, + canonicalTerminalDigest: hash, + canonicalTerminalAt: at, + capacityReceiptDigest: hash, + }, + sqlSession = { pid: 2147483647, backendStart: "x".repeat(64), sessionNonce: uuid }, + effect = { + version: 2, + origin: projection.origin, + settlementId: projection.settlementId, + controlExecutionDigest: hash, + controlEnvelopeDigest: hash, + admissionDigest: hash, + commitmentDigest: hash, + preparedReceiptDigest: hash, + expectedDmSequence: 0, + attemptedAt: at, + sqlSession, + phase: "committed", + terminal, + observation: null, + }, + contained = { + ...effect, + phase: "contained_not_settled", + terminal: null, + observation: { + version: 2, + state: "contained_not_settled", + settlementId: projection.settlementId, + controlExecutionDigest: hash, + controlEnvelopeDigest: hash, + commitmentDigest: hash, + preparedReceiptDigest: hash, + expectedDmSequence: 0, + attemptedAt: at, + observationAt: at, + sqlSession, + containment: "original_session_absent", + }, + }; + const { terminal: _effectTerminal, ...effectFields } = effect, + publishedTerminal = { + ...publishedTerminalBase, + effectSettlementDigest: hash, + effectSource: { fields: effectFields, signature: "" }, + admissionSource: { boundAt: at, signature: "" }, + }; + const variants: unknown[][] = [ + [wrap(committed), wrap(bundle), wrap(neverIssuedBundle)], + [wrap(envelope)], + [wrap({ version: 2, projection: signedProjection, binding: wrap(envelope) }), wrap(linked)], + [ + wrap({ + version: 2, + origin: projection.origin, + settlementId: projection.settlementId, + phase: "attempted", + dispatchId: uuid, + controlEnvelopeDigest: hash, + controlExecutionDigest: hash, + commitmentDigest: hash, + attemptedAt: at, + }), + ], + [{ version: 2, terminal: wrap(publishedTerminal), ack: wrap(ack) }], + [wrap(ack)], + [ + { + version: 2, + capacity: wrap(capacity), + refusal: wrap(dmRefusal), + unused: unused("data-manager"), + }, + ], + [ + wrap({ + version: 2, + purpose: "configuration-fence-settlement", + installationId: projection.installationId, + origin: projection.origin, + settlementId: projection.settlementId, + projectionDigest: hash, + reservationDigest: hash, + prepareExecutionDigest: hash, + prepareAdmissionDigest: hash, + preparedReceiptDigest: hash, + capacityReceiptDigest: hash, + rootCertificateDigest: projection.rootCertificateDigest, + controlEnvelope: wrap(envelope), + commitmentDigest: hash, + executionOrdinal: 127, + intentOrdinal: 255, + sqlSessionNonce: uuid, + boundAt: at, + }), + ], + [wrap(effect), wrap(contained)], + [wrap(publishedTerminal)], + [wrap(publishedTerminal)], + [wrap(ack)], + ]; + const result: Record = {}; + for (const [index, descriptor] of all.entries()) { + const maximum = Math.max(...variants[index]!.map(width)); + if (maximum > descriptor.cap) + throw Error( + `Prospective settlement packet exceeds ${descriptor.role}/${descriptor.scope}/${descriptor.name}`, + ); + result[`${descriptor.role}/${descriptor.scope}/${descriptor.name}`] = maximum; + } + if (all.reduce((sum, descriptor) => sum + descriptor.cap, 0) > 65536) + throw Error("Whole padded settlement packet exceeds 65536 bytes"); + return Object.freeze(result); +} diff --git a/packages/core/src/build-artifacts/configuration-fence-settlement-cells.ts b/packages/core/src/build-artifacts/configuration-fence-settlement-cells.ts new file mode 100644 index 000000000..acb01bdf4 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-fence-settlement-cells.ts @@ -0,0 +1,8 @@ +/** Compatibility surface for fixed descriptors and the sole guarded family index. */ +export * from "./configuration-fence-settlement-descriptors"; +export { + assertNoUnissuedConfigurationReceiverPublication, + configurationSettlementDescriptorDigest, + readConfigurationSettlementCell, + writeConfigurationSettlementCell, +} from "./retention-family"; diff --git a/packages/core/src/build-artifacts/configuration-fence-settlement-descriptors.ts b/packages/core/src/build-artifacts/configuration-fence-settlement-descriptors.ts new file mode 100644 index 000000000..b0cc2c8df --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-fence-settlement-descriptors.ts @@ -0,0 +1,81 @@ +/** Pure fixed cell descriptors/bounds; no allocation or index mutation. */ +import type { z } from "zod"; +import type { ExecutionRef } from "../ops/execution-ref"; +import type { configurationSettlementReservationProjectionSchema } from "./configuration-fence-settlement-records"; + +export type ConfigurationSettlementProjection = z.infer< + typeof configurationSettlementReservationProjectionSchema +>; +export type ConfigurationSettlementRecordDescriptor = Readonly<{ + scope: "private" | "shared"; + namespace: "observations" | "execution"; + role: "ops-agent" | "data-manager"; + name: string; + cap: number; +}>; +export interface ConfigurationSettlementFamilySlot { + version: 2; + role: "ops-agent" | "data-manager"; + settlementId: string; + projectionDigest: string; + reservationDigest: string; + ownerPrepareDigest: string; + proposedExecutionAddress: Pick< + ExecutionRef, + "executor" | "principal" | "executionId" | "operationKey" + >; + executionOrdinal: number | null; + intentOrdinal: number | null; + executionTicketId: string; + intentTicketId: string; + recordDescriptorsDigest: string; + descriptors: ConfigurationSettlementRecordDescriptor[]; + state: "refusal_channel" | "capacity_refused" | "allocating" | "reserved" | "bound" | "UNUSED"; + reservedMetadataBytes: number; + refusalConstraint: "members" | "tickets" | "metadata" | "records" | "filesystem" | null; + executionDigest: string | null; + controlBindingDigest: string | null; + allocatedDescriptorDigests: string[]; + unusedIntentDigest: string | null; + unusedConfirmedAt: number | null; +} +export function configurationSettlementRecordDescriptors( + projection: Pick, + prepareKey: string, +) { + const rootKey = projection.ownerPrepare.operationKey, + controlKey = projection.reservation.execution.operationKey; + const descriptor = ( + scope: "private" | "shared", + role: "ops-agent" | "data-manager", + key: string, + suffix: string, + cap: number, + ): ConfigurationSettlementRecordDescriptor => + Object.freeze({ + scope, + namespace: scope === "private" ? "observations" : "execution", + role, + name: `${key}.configuration-settlement${suffix}.json`, + cap, + }); + const sender = [ + descriptor("shared", "ops-agent", rootKey, "", 16384), + descriptor("private", "ops-agent", rootKey, "-binding", 4096), + descriptor("shared", "ops-agent", rootKey, "-control", 8192), + descriptor("private", "ops-agent", rootKey, "-dispatch", 2048), + descriptor("private", "ops-agent", rootKey, "-ack", 4096), + descriptor("shared", "ops-agent", rootKey, "-ack", 1024), + ]; + const receiver = [ + descriptor("shared", "data-manager", prepareKey, "-capacity", 8192), + descriptor("private", "data-manager", controlKey, "-admission", 4096), + descriptor("private", "data-manager", controlKey, "-effect", 4096), + descriptor("private", "data-manager", controlKey, "-terminal", 4096), + descriptor("shared", "data-manager", controlKey, "-terminal", 4096), + descriptor("private", "data-manager", controlKey, "-ack", 1024), + ]; + if ([...sender, ...receiver].reduce((sum, record) => sum + record.cap, 0) > 65536) + throw Error("Complete settlement packet capacity exceeded"); + return Object.freeze({ sender: Object.freeze(sender), receiver: Object.freeze(receiver) }); +} diff --git a/packages/core/src/build-artifacts/configuration-fence-settlement-records.ts b/packages/core/src/build-artifacts/configuration-fence-settlement-records.ts new file mode 100644 index 000000000..cd79cdb28 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-fence-settlement-records.ts @@ -0,0 +1,669 @@ +/** Strict server-private records for the approved fixed-reference protocol. + * Parsing any of these records creates data, never execution/inspection authority. */ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { executionRefSchema } from "../ops/execution-ref"; +import { + dataConfigurationFenceReceiptSchema, + dataConfigurationFenceSettleSchema, +} from "../services/data-manager/contracts/configuration-maintenance"; +import { controlChildReservationSchema } from "./control-child-records"; +import { controlOriginSchema } from "./control-records"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/), + epoch = z.number().int().nonnegative().safe(); +const id = z.uuid(), + ordinal = z.number().int().min(0).max(255), + service = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const uniqueSorted = >(schema: T, max: number) => + z + .array(schema) + .max(max) + .refine((items) => items.every((item, i) => i === 0 || items[i - 1]! < item)); +const digests = uniqueSorted(digest, 64), + serviceIds = uniqueSorted(service, 256).min(1); +const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +export const configurationSettlementDomains = Object.freeze({ + reference: "openmapx-configuration-fence-settlement-v2", + preparedReceipt: "openmapx-configuration-prepared-receipt-v2", + finalEvidence: "openmapx-configuration-final-evidence-v2", + ownerFinal: "openmapx-configuration-owner-final-v2", + commitment: "openmapx-configuration-settlement-commitment-v2", + inspection: "openmapx-configuration-runtime-inspection-v2", + reservationTuple: "openmapx-configuration-settlement-reservation-tuple-v2", + projection: "openmapx-configuration-settlement-reservation-projection-v2", + signedProjection: "openmapx-configuration-settlement-reservation-v2", + control: "openmapx-configuration-fence-control-v3", + binding: "openmapx-configuration-settlement-control-v3", + assertion: "openmapx-configuration-fence-control-assertion-v2", + capacity: "openmapx-configuration-settlement-capacity-v2", + admission: "openmapx-configuration-settlement-admission-v2", + effect: "openmapx-configuration-settlement-effect-v2", + dispatch: "openmapx-configuration-settlement-dispatch-v2", + terminal: "openmapx-dm-configuration-settlement-terminal-v2", + ack: "openmapx-configuration-settlement-ack-v2", + refusal: "openmapx-dm-configuration-no-guard-refusal-v2", + unusedIntent: "openmapx-configuration-unused-capacity-intent-v2", + unusedConfirmation: "openmapx-configuration-unused-capacity-confirmation-v2", + neverIssued: "openmapx-configuration-never-issued-v2", + issuedRefused: "openmapx-configuration-issued-prepare-refused-v2", + closure: "openmapx-configuration-no-guard-closure-v2", + bundle: "openmapx-configuration-no-guard-bundle-v2", +}); +export const settlementRecordDigest = ( + domain: (typeof configurationSettlementDomains)[keyof typeof configurationSettlementDomains], + value: unknown, +) => + createHash("sha256") + .update(`${domain}\0${canonicalJson(value)}`) + .digest("hex"); +const signed = (payload: T) => + z.strictObject({ payload, signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }); +export const configurationSettlementOwnerIdentitySchema = z + .strictObject({ + installationId: id, + origin: controlOriginSchema, + ownerPrepare: executionRefSchema, + admissionDigest: digest, + rootCertificateDigest: digest, + maintenanceId: id, + candidateId: id, + planDigest: digest, + settlementId: digest, + }) + .refine( + (value) => + value.origin.issuer === "ops-agent" && + value.ownerPrepare.executor === "ops-agent" && + value.ownerPrepare.principal === "api" && + value.ownerPrepare.deadlineAt <= value.origin.notAfter, + ); +export type ConfigurationSettlementOwnerIdentity = z.infer< + typeof configurationSettlementOwnerIdentitySchema +>; +export function deriveConfigurationFenceSettlementId( + input: Pick< + ConfigurationSettlementOwnerIdentity, + "installationId" | "ownerPrepare" | "maintenanceId" | "candidateId" | "planDigest" + >, +): string { + id.parse(input.installationId); + executionRefSchema.parse(input.ownerPrepare); + id.parse(input.maintenanceId); + id.parse(input.candidateId); + digest.parse(input.planDigest); + return createHash("sha256") + .update( + `${configurationSettlementDomains.reference}\0${input.installationId}\0${input.ownerPrepare.operationKey}\0${input.maintenanceId}\0${input.candidateId}\0${input.planDigest}`, + ) + .digest("hex"); +} +export const configurationPreparedGuardSchema = z.strictObject({ + fenceId: id, + preparedReceipt: dataConfigurationFenceReceiptSchema.refine( + (receipt) => receipt.state === "prepared", + ), + preparedReceiptDigest: digest, + preparedCanonicalDigest: digest, + preparedTerminalAt: epoch, +}); +export const configurationDecisionEvidenceSchema = z.strictObject({ + decisionId: id, + decision: z.enum(["continue", "abandon"]), + checkpointVersion: epoch, + checkpointDigest: digest, + apiParentAttestationDigest: digest, +}); +export const configurationChildEvidenceSchema = z.strictObject({ + planIndex: z.number().int().min(0).max(31), + execution: executionRefSchema, + terminalDigest: digest, + terminalAt: epoch, + effectSettlementDigest: digest, + publication: z.enum(["not_started", "committed"]), +}); +const childProjection = configurationChildEvidenceSchema + .omit({ execution: true }) + .extend({ executionDigest: digest }); +const generation = z + .strictObject({ + state: z.enum(["not_committed", "committed"]), + revisionId: revision.nullable(), + commitReceiptDigest: digest.nullable(), + }) + .refine((value) => + value.state === "committed" + ? value.revisionId !== null && value.commitReceiptDigest !== null + : value.revisionId === null && value.commitReceiptDigest === null, + ); +const followUp = z.strictObject({ + state: z.enum(["not_started", "complete", "contained"]), + receiptDigests: digests, +}); +const runtime = z.strictObject({ + verificationId: id, + mode: z.enum(["unused", "applied", "contained"]), + requiredCheckSetDigest: digest, + checkedAt: epoch, +}); +const finalFields = { + identity: configurationSettlementOwnerIdentitySchema, + guard: configurationPreparedGuardSchema, + decision: configurationDecisionEvidenceSchema, + generation, + followUp, + runtime, + outcome: z.enum(["unused", "applied", "contained"]), +}; +function eligible(value: z.infer>) { + return ( + value.runtime.mode === value.outcome && + (value.outcome === "unused" + ? value.decision.decision === "abandon" && + value.generation.state === "not_committed" && + value.followUp.state === "not_started" + : value.decision.decision === "continue" && + (value.outcome !== "applied" || + (value.generation.state === "committed" && + value.generation.revisionId === value.guard.preparedReceipt.revisionId && + value.followUp.state === "complete"))) + ); +} +export const configurationFinalEvidenceSchema = z + .strictObject({ ...finalFields, children: z.array(configurationChildEvidenceSchema).max(64) }) + .refine(eligible); +export const configurationFinalEvidenceProjectionSchema = z + .strictObject({ ...finalFields, children: z.array(childProjection).max(64) }) + .refine(eligible); +export const configurationOwnerFinalSchema = z.strictObject({ + version: z.literal(2), + identity: configurationSettlementOwnerIdentitySchema, + ownerSequence: epoch, + settlingAt: epoch, + expectedDmSequence: epoch, + evidenceDigest: digest, + outcome: z.enum(["unused", "applied", "contained"]), + privateInspectionId: id, +}); +export const configurationCommittedSettlementSchema = z.strictObject({ + version: z.literal(2), + phase: z.literal("committed"), + identity: configurationSettlementOwnerIdentitySchema, + referenceOperationFingerprint: digest, + final: configurationOwnerFinalSchema, + evidence: configurationFinalEvidenceProjectionSchema, + evidenceDigest: digest, + ownerReceiptDigest: digest, + committedAt: epoch, +}); +export const fixedConfigurationFenceSettleOperationSchema = + dataConfigurationFenceSettleSchema.extend({ + kind: z.literal("configuration-maintenance.settle"), + }); +export const configurationSettlementReservationProjectionSchema = z.strictObject({ + version: z.literal(2), + purpose: z.literal("configuration-fence-settlement-reservation"), + installationId: id, + origin: controlOriginSchema, + ownerPrepare: executionRefSchema, + rootAdmissionDigest: digest, + rootCertificateDigest: digest, + maintenanceId: id, + candidateId: id, + planDigest: digest, + settlementId: digest, + reservation: z + .lazy(() => controlChildReservationSchema) + .refine( + (value) => + value.phase === "configuration-fence-settle" && + value.kind === "configuration-maintenance.settle" && + value.recipient === "data-manager" && + value.execution.executor === "data-manager" && + value.execution.principal === "api", + ), + operation: fixedConfigurationFenceSettleOperationSchema, + reservationDigest: digest, + issuerMembership: z.strictObject({ + role: z.literal("ops-agent"), + familyId: id, + rootMemberOrdinal: ordinal, + rootExecutionDigest: digest, + rootAdmissionDigest: digest, + }), +}); +export const signedConfigurationReservationProjectionSchema = signed( + configurationSettlementReservationProjectionSchema, +); +const address = z + .strictObject({ + executor: executionRefSchema.shape.executor, + principal: executionRefSchema.shape.principal, + executionId: executionRefSchema.shape.executionId, + operationKey: executionRefSchema.shape.operationKey, + }) + .refine( + (value) => + value.executor === "data-manager" && + value.principal === "api" && + id.safeParse(value.executionId).success, + ); +const descriptor = z.strictObject({ + scope: z.enum(["private", "shared"]), + namespace: z.enum(["observations", "execution"]), + role: z.enum(["ops-agent", "data-manager"]), + name: z + .string() + .min(1) + .max(255) + .regex(/^[A-Za-z0-9_.-]+$/), + cap: z.number().int().positive().max(16384), +}); +export const configurationSettlementCapacitySchema = z.strictObject({ + version: z.literal(2), + installationId: id, + origin: controlOriginSchema, + ownerPrepareDigest: digest, + settlementId: digest, + proposedExecutionAddress: address, + reservedExecutionOrdinal: ordinal, + reservedIntentOrdinal: ordinal, + recordDescriptorsDigest: digest, + maxBytes: z.number().int().positive().max(65536), + projectionDigest: digest, + reservationDigest: digest, +}); +export const signedConfigurationSettlementCapacitySchema = signed( + configurationSettlementCapacitySchema, +); +export const configurationSettlementControlEnvelopeSchema = z + .strictObject({ + version: z.literal(2), + bindingVersion: z.literal(3), + purpose: z.literal("configuration-fence-settlement"), + installationId: id, + origin: controlOriginSchema, + ownerPrepare: executionRefSchema, + admissionDigest: digest, + rootCertificateDigest: digest, + settlementId: digest, + projectionDigest: digest, + reservationDigest: digest, + commitmentDigest: digest, + execution: executionRefSchema, + operation: fixedConfigurationFenceSettleOperationSchema, + serviceIds, + controlStartedAt: epoch, + controlDeadlineAt: epoch, + originalParentForwardDeadlineAt: epoch, + dispatchId: id, + }) + .refine( + (value) => + value.controlDeadlineAt === value.controlStartedAt + 60000 && + value.execution.deadlineAt === value.controlDeadlineAt && + value.originalParentForwardDeadlineAt === value.ownerPrepare.deadlineAt && + value.execution.executor === "data-manager" && + value.execution.principal === value.ownerPrepare.principal && + value.ownerPrepare.principal === "api", + ); +export const configurationSettlementSqlSessionSchema = z.strictObject({ + pid: z.number().int().min(1).max(2147483647), + backendStart: z.string().min(1).max(64), + sessionNonce: id, +}); +/** A disjoint purpose assertion, never an ordinary parent/delegation token. + * Its envelope digest binds the whole fixed body/execution/installation. */ +export const configurationSettlementAssertionSchema = z.strictObject({ + version: z.literal(2), + purpose: z.literal("configuration-fence-settlement"), + settlementId: digest, + controlEnvelopeDigest: digest, +}); +export const configurationSettlementTerminalTupleSchema = z.strictObject({ + execution: executionRefSchema, + settlementId: digest, + commitmentDigest: digest, + fenceId: id, + result: dataConfigurationFenceReceiptSchema.refine((receipt) => receipt.state === "settled"), + publication: z.literal("committed"), + terminalAt: epoch, + terminalDigest: digest, +}); +export const configurationSettlementSqlControlSchema = z.strictObject({ + version: z.literal(2), + settlementId: digest, + controlEnvelopeDigest: digest, + commitmentDigest: digest, + admissionDigest: digest, + outcome: z.enum(["unused", "applied", "contained"]), + expectedDmSequence: epoch, + preparedReceiptDigest: digest, + attemptedAt: epoch, + sqlSession: configurationSettlementSqlSessionSchema, + tuple: configurationSettlementTerminalTupleSchema, +}); +export const configurationSettlementContainedObservationSchema = z.strictObject({ + version: z.literal(2), + state: z.literal("contained_not_settled"), + settlementId: digest, + controlExecutionDigest: digest, + controlEnvelopeDigest: digest, + commitmentDigest: digest, + preparedReceiptDigest: digest, + expectedDmSequence: epoch, + attemptedAt: epoch, + observationAt: epoch, + sqlSession: configurationSettlementSqlSessionSchema, + containment: z.enum(["explicit_rollback", "original_session_absent"]), +}); +export const configurationSettlementAckSchema = z.strictObject({ + version: z.literal(2), + installationId: id, + origin: controlOriginSchema, + settlementId: digest, + controlExecutionDigest: digest, + terminalDigest: digest, + terminalAt: epoch, + commitmentDigest: digest, +}); +export const configurationSettlementDispatchSchema = z.strictObject({ + version: z.literal(2), + origin: controlOriginSchema, + settlementId: digest, + phase: z.enum(["prepared", "attempted"]), + dispatchId: id, + controlEnvelopeDigest: digest, + controlExecutionDigest: digest, + commitmentDigest: digest, + attemptedAt: epoch.nullable(), +}); +export const configurationSettlementAdmissionSchema = z.strictObject({ + version: z.literal(2), + purpose: z.literal("configuration-fence-settlement"), + installationId: id, + origin: controlOriginSchema, + settlementId: digest, + projectionDigest: digest, + reservationDigest: digest, + prepareExecutionDigest: digest, + prepareAdmissionDigest: digest, + preparedReceiptDigest: digest, + capacityReceiptDigest: digest, + rootCertificateDigest: digest, + controlEnvelope: signed(configurationSettlementControlEnvelopeSchema), + commitmentDigest: digest, + executionOrdinal: ordinal, + intentOrdinal: ordinal, + sqlSessionNonce: id, + boundAt: epoch, +}); +export const configurationSettlementEffectSchema = z + .strictObject({ + version: z.literal(2), + origin: controlOriginSchema, + settlementId: digest, + controlExecutionDigest: digest, + controlEnvelopeDigest: digest, + admissionDigest: digest, + commitmentDigest: digest, + preparedReceiptDigest: digest, + expectedDmSequence: epoch, + attemptedAt: epoch, + sqlSession: configurationSettlementSqlSessionSchema, + phase: z.enum(["attempted", "committed", "unknown", "contained_not_settled"]), + terminal: configurationSettlementTerminalTupleSchema.nullable(), + observation: configurationSettlementContainedObservationSchema.nullable(), + }) + .refine((value) => + value.phase === "committed" + ? value.terminal !== null && value.observation === null + : value.phase === "contained_not_settled" + ? value.terminal === null && value.observation !== null + : value.terminal === null && value.observation === null, + ); +const { terminal: _effectTerminal, ...effectSourceFields } = + configurationSettlementEffectSchema.shape; +/** Preserve the original FULL signed effect preimage without encoding the + * same tuple twice: reconstruct { ...fields, terminal: tuple } and verify its + * original signature/domain. This is data, not a general effect capability. */ +export const configurationSettlementEffectSourceSchema = z.strictObject({ + fields: z.strictObject(effectSourceFields), + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), +}); +export const configurationSettlementPublishedTerminalSchema = z.strictObject({ + version: z.literal(2), + origin: controlOriginSchema, + admissionDigest: digest, + tuple: configurationSettlementTerminalTupleSchema, + effectSettlementDigest: digest, + effectSource: configurationSettlementEffectSourceSchema, + // All other original admission fields are fixed by the authenticated + // projection/capacity/envelope/preparation already retained by both roles. + admissionSource: z.strictObject({ + boundAt: epoch, + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), + }), +}); +export const configurationSettlementSenderAckCellSchema = z.strictObject({ + version: z.literal(2), + terminal: signed(configurationSettlementPublishedTerminalSchema), + ack: signed(configurationSettlementAckSchema).nullable(), +}); +export const configurationDmNoGuardRefusalSchema = z.strictObject({ + version: z.literal(2), + installationId: id, + origin: controlOriginSchema, + ownerPrepare: executionRefSchema, + rootAdmissionDigest: digest, + rootCertificateDigest: digest, + maintenanceId: id, + candidateId: id, + planDigest: digest, + settlementId: digest, + projectionDigest: digest, + reservationDigest: digest, + prepareExecution: executionRefSchema, + prepareAdmissionDigest: digest, + refusalReceipt: z.strictObject({ + kind: z.literal("data-configuration-fence-refused"), + maintenanceId: id, + reason: z.enum(["busy", "capacity"]), + }), + effectReceiptDigest: digest, + effectSettlementDigest: digest, + canonicalTerminalDigest: digest, + canonicalTerminalAt: epoch, + capacityReceiptDigest: digest.nullable(), +}); +export const configurationUnusedCapacityIntentSchema = z.strictObject({ + version: z.literal(2), + installationId: id, + origin: controlOriginSchema, + role: z.enum(["ops-agent", "data-manager"]), + ownerPrepareDigest: digest, + prepareExecutionDigest: digest.nullable(), + settlementId: digest, + projectionDigest: digest, + capacityReceiptDigest: digest.nullable(), + sourceProofDigest: digest, + executionOrdinal: ordinal.nullable(), + intentOrdinal: ordinal.nullable(), + recordDescriptorsDigest: digest, + allocatedDescriptorDigests: uniqueSorted(digest, 12), + createdAt: epoch, +}); +export const configurationUnusedCapacityConfirmationSchema = z.strictObject({ + version: z.literal(2), + installationId: id, + origin: controlOriginSchema, + role: z.enum(["ops-agent", "data-manager"]), + ownerPrepareDigest: digest, + settlementId: digest, + projectionDigest: digest, + intentDigest: digest, + slotStateDigest: digest, + confirmedAt: epoch, +}); +const publication = z.strictObject({ + phase: z.string().min(1).max(64), + ordinal: z.number().int().min(0).max(63), + operationKey: executionRefSchema.shape.operationKey, + grantState: z.literal("unissued"), + intentState: z.literal("not_attempted"), +}); +export const configurationNeverIssuedProofSchema = z.strictObject({ + version: z.literal(2), + identity: configurationSettlementOwnerIdentitySchema, + issuerSequence: epoch, + frozenAt: epoch, + prepareReservationDigest: digest, + publicationStates: z.array(publication).max(64), +}); +export const configurationIssuedPrepareRefusedProofSchema = z.strictObject({ + version: z.literal(2), + kind: z.literal("issued_preparation_refused"), + identity: configurationSettlementOwnerIdentitySchema, + issuerSequence: epoch, + frozenAt: epoch, + prepareGrantDigest: digest, + prepareExecutionDigest: digest, + refusalProjectionDigest: digest, + remainingPublicationStates: z.array(publication).max(64), + candidateClaimState: z.literal("not_consumed"), +}); +export const configurationNoGuardEvidenceSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("never_issued"), + issuerFreezeProofDigest: digest, + prepareReservationDigest: digest, + publicationStatesDigest: digest, + issuerSequence: epoch, + frozenAt: epoch, + }), + z.strictObject({ + kind: z.literal("receiver_refused"), + issuerFreezeProofDigest: digest, + prepareExecutionDigest: digest, + prepareAdmissionDigest: digest, + refusalProjectionDigest: digest, + effectSettlementDigest: digest, + canonicalTerminalDigest: digest, + canonicalTerminalAt: epoch, + dmUnusedIntentDigest: digest, + dmUnusedConfirmationDigest: digest, + capacityReceiptDigest: digest.nullable(), + }), +]); +export const configurationNoGuardClosureSchema = z.strictObject({ + version: z.literal(2), + identity: configurationSettlementOwnerIdentitySchema, + ownerSequence: epoch, + closedAt: epoch, + noGuardEvidence: configurationNoGuardEvidenceSchema, + originalContainmentDigests: digests, + unusedSlotDigests: z + .array( + z.strictObject({ + role: z.enum(["ops-agent", "data-manager"]), + intentDigest: digest, + confirmationDigest: digest, + }), + ) + .max(2) + .refine((items) => items.every((item, i) => i === 0 || items[i - 1]!.role < item.role)), + closureDigest: digest, +}); +const issuerUnused = z.strictObject({ + intent: signed(configurationUnusedCapacityIntentSchema), + confirmation: signed(configurationUnusedCapacityConfirmationSchema), +}); +export const configurationNoGuardBundleSchema = z.strictObject({ + version: z.literal(2), + phase: z.literal("unused"), + settlementId: digest, + projectionDigest: digest, + closure: signed(configurationNoGuardClosureSchema), + issuerFreezeProof: z.union([ + signed(configurationNeverIssuedProofSchema), + signed(configurationIssuedPrepareRefusedProofSchema), + ]), + issuerUnused, +}); +export const configurationNoGuardIntentSchema = z.strictObject({ + ownerSequence: epoch, + closedAt: epoch, + noGuardEvidence: configurationNoGuardEvidenceSchema, + issuerFreezeProof: z.union([ + signed(configurationNeverIssuedProofSchema), + signed(configurationIssuedPrepareRefusedProofSchema), + ]), + issuerUnused: z.strictObject({ + intent: signed(configurationUnusedCapacityIntentSchema), + confirmation: signed(configurationUnusedCapacityConfirmationSchema).nullable(), + }), + bundle: signed(configurationNoGuardBundleSchema).nullable(), +}); +/** Fixed acyclic issuer source tuple, independent of a later closure. */ +export function configurationIssuerUnusedSourceDigest( + evidence: z.infer, +): string { + return settlementRecordDigest( + configurationSettlementDomains.unusedIntent, + evidence.kind === "never_issued" + ? { issuerFreezeProofDigest: evidence.issuerFreezeProofDigest } + : { + issuerFreezeProofDigest: evidence.issuerFreezeProofDigest, + refusalProjectionDigest: evidence.refusalProjectionDigest, + dmUnusedConfirmationDigest: evidence.dmUnusedConfirmationDigest, + }, + ); +} +const pending = z.strictObject({ + version: z.literal(2), + phase: z.literal("uncommitted"), + identity: configurationSettlementOwnerIdentitySchema, + referenceOperationFingerprint: digest, +}); +export const configurationSettlementCommitmentCellSchema = signed( + z.union([pending, configurationCommittedSettlementSchema, configurationNoGuardBundleSchema]), +); +export const configurationSettlementControlCellSchema = signed( + z.union([ + z.strictObject({ + version: z.literal(2), + projection: signedConfigurationReservationProjectionSchema, + binding: z.literal(null), + }), + z.strictObject({ + version: z.literal(2), + projection: signedConfigurationReservationProjectionSchema, + binding: signed(configurationSettlementControlEnvelopeSchema), + }), + z.strictObject({ + version: z.literal(2), + projection: signedConfigurationReservationProjectionSchema, + binding: z.literal(null), + unused: z.strictObject({ + bundleDigest: digest, + closureDigest: digest, + issuerFreezeProofDigest: digest, + issuerUnusedIntentDigest: digest, + issuerUnusedConfirmationDigest: digest, + }), + }), + ]), +); +export const configurationDmCapacityCellSchema = z.strictObject({ + version: z.literal(2), + capacity: signedConfigurationSettlementCapacitySchema.nullable(), + refusal: signed(configurationDmNoGuardRefusalSchema).nullable(), + unused: z + .strictObject({ + intent: signed(configurationUnusedCapacityIntentSchema), + confirmation: signed(configurationUnusedCapacityConfirmationSchema).nullable(), + }) + .nullable(), +}); +export const configurationSettlementDescriptorSchema = descriptor; diff --git a/packages/core/src/build-artifacts/configuration-fence-settlement-retention.ts b/packages/core/src/build-artifacts/configuration-fence-settlement-retention.ts new file mode 100644 index 000000000..6b6b41e1d --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-fence-settlement-retention.ts @@ -0,0 +1,2 @@ +/** Retirement verification stays with the actual role-local index owner. */ +export { readPreparedConfigurationSettlementRetirement } from "./retention-family"; diff --git a/packages/core/src/build-artifacts/configuration-fence-settlement.test.ts b/packages/core/src/build-artifacts/configuration-fence-settlement.test.ts new file mode 100644 index 000000000..6972bc17f --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-fence-settlement.test.ts @@ -0,0 +1,46 @@ +import { expect, it } from "vitest"; +import { dataConfigurationFenceSettleSchema } from "../services/data-manager/contracts/configuration-maintenance"; +import { configurationNoGuardEvidenceSchema } from "./configuration-fence-settlement-records"; + +const binding = { + maintenanceId: "11111111-1111-4111-8111-111111111111", + candidateId: "22222222-2222-4222-8222-222222222222", + planDigest: "a".repeat(64), +}; +it("accepts the complete root-fixed version2 reference without late owner outcome fields", () => { + const body = { version: 2, ...binding, settlementId: "b".repeat(64) }; + expect(dataConfigurationFenceSettleSchema.parse(body)).toEqual(body); +}); +it("rejects fresh legacy inline outcome authority at the current service-auth mutation schema", () => { + expect( + dataConfigurationFenceSettleSchema.safeParse({ + ...binding, + decisionId: "33333333-3333-4333-8333-333333333333", + expectedSequence: 0, + outcome: "applied", + ownerReceiptDigest: "c".repeat(64), + }).success, + ).toBe(false); +}); +it("requires the actual issuer freeze proof reference in receiver-refused NoGuard data", () => { + // Decoder coverage only: these digests do not constitute signed authority. + const refusal = { + kind: "receiver_refused", + prepareExecutionDigest: "a".repeat(64), + prepareAdmissionDigest: "b".repeat(64), + refusalProjectionDigest: "c".repeat(64), + effectSettlementDigest: "d".repeat(64), + canonicalTerminalDigest: "e".repeat(64), + canonicalTerminalAt: 1, + dmUnusedIntentDigest: "f".repeat(64), + dmUnusedConfirmationDigest: "0".repeat(64), + capacityReceiptDigest: null, + }; + expect(configurationNoGuardEvidenceSchema.safeParse(refusal).success).toBe(false); + expect( + configurationNoGuardEvidenceSchema.safeParse({ + ...refusal, + issuerFreezeProofDigest: "1".repeat(64), + }).success, + ).toBe(true); +}); diff --git a/packages/core/src/build-artifacts/configuration-fence-settlement.ts b/packages/core/src/build-artifacts/configuration-fence-settlement.ts new file mode 100644 index 000000000..a51da84e6 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-fence-settlement.ts @@ -0,0 +1,726 @@ +/** Closed source construction for fixed-reference settlement. This stage + * reads the actual original owner/root/family; it neither publishes capacity + * nor produces final/dispatch authority. */ + +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { + type DataConfigurationFencePrepare, + dataConfigurationFencePrepareSchema, +} from "../services/data-manager/contracts/configuration-maintenance"; +import { inspectControlChildPublication } from "./child-phase"; +import { assertConfigurationSettlementPacketBudget } from "./configuration-fence-settlement-budget"; +import { + configurationSettlementRecordDescriptors, + readConfigurationSettlementCell, + writeConfigurationSettlementCell, +} from "./configuration-fence-settlement-cells"; +import { + configurationDmCapacityCellSchema, + configurationSettlementCapacitySchema, + configurationSettlementControlCellSchema, + configurationSettlementDomains, + configurationSettlementOwnerIdentitySchema, + configurationSettlementReservationProjectionSchema, + deriveConfigurationFenceSettlementId, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import { configurationMaintenanceOwnerRecordSchema } from "./configuration-maintenance-owner-records"; +import { + assertControlExecutionForward, + type ControlInstallation, + controlChildOperationKey, + controlPeerState, + executionAuthorityState, + installationState, + readControlExecutionWitness, + readPrivateRecord, + type VerifiedControlPeer, + type VerifiedExecutionAuthority, + withPrivateEffectWriter, +} from "./control-authority"; +import { controlDigest, signControl, uuid, verifyControl } from "./control-records"; +import { AnchoredDirectory } from "./filesystem"; +import { + assertFixedConfigurationMaintenancePlan, + type FixedConfigurationMaintenancePlan, +} from "./fixed-configuration-maintenance"; +import { + completeConfigurationSettlementFamilyReservation, + readConfigurationSettlementFamilySlot, + reserveConfigurationSettlementFamilySlot, + retainedConfigurationMaintenanceFamilyMember, + verifyFamilyOwnedRecord, + verifyPublishedConfigurationMaintenanceFamilyMember, + verifyPublishedDmConfigurationSettlementSlots, +} from "./retention-family"; +import { + closeArtifactTargetLeaseHandle, + configurationMaintenanceTargetOwner, + inspectArtifactDelegation, + restoreArtifactTargetLeaseForSettlement, + withConfigurationMaintenanceReservationPublication, +} from "./target-lease"; + +declare const reservationBrand: unique symbol; +export type OwnedConfigurationSettlementReservation = Readonly<{ [reservationBrand]: true }>; +const reservations = new WeakMap< + OwnedConfigurationSettlementReservation, + { + authority: VerifiedExecutionAuthority; + projection: ReturnType; + signedProjection: ReturnType< + typeof signControl< + ReturnType + > + >; + identity: ReturnType; + projectionDigest: string; + ownerRecordDigest: string; + plan: FixedConfigurationMaintenancePlan; + } +>(); +function freeze(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); + } + return value; +} +export function ownedConfigurationSettlementReservationState( + value: OwnedConfigurationSettlementReservation, + authority: VerifiedExecutionAuthority, +) { + const state = reservations.get(value); + if (!state || state.authority !== authority) + throw Error("Actual original settlement reservation required"); + executionAuthorityState(authority); + return state; +} +export function prepareOwnedConfigurationSettlementReservation( + authority: VerifiedExecutionAuthority, + maintenanceId: string, +): OwnedConfigurationSettlementReservation { + const state = executionAuthorityState(authority); + if ( + state.kind !== "configurationMaintenance.prepare" || + state.installation.fixedChildId || + state.installation.installation.role !== "ops-agent" || + !state.origin + ) + throw Error("Original maintenance owner required"); + assertControlExecutionForward(authority); + const name = `${uuid.parse(maintenanceId)}.json`; + const raw = readPrivateRecord(state.installation, "configuration-maintenance", name); + const owner = verifyControl( + state.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "configuration-maintenance", name, container: "" }, + canonicalJson(raw), + ); + assertFixedConfigurationMaintenancePlan( + owner.plan, + state.installation.installation.installationId, + ); + if ( + owner.plan.operation.maintenanceId !== maintenanceId || + owner.admissionDigest !== state.receiptDigest || + canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) || + canonicalJson(owner.origin) !== canonicalJson(state.origin) || + canonicalJson(owner.plan.reservations) !== canonicalJson(state.reservations) || + owner.receipt.state !== "preparing" || + owner.guardSource || + opsOperationFingerprint(owner.plan.operation) !== state.execution.operationFingerprint + ) + throw Error("Original pre-effect settlement reservation source required"); + const lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + const root = configurationMaintenanceTargetOwner(lease), + plan = owner.plan; + if ( + owner.rootCertificateDigest !== root.certificateDigest || + canonicalJson(root.serviceIds) !== canonicalJson(plan.serviceIds) + ) + throw Error("Original maintenance root certificate changed"); + const reservation = state.reservations.find( + (value) => value.phase === "configuration-fence-settle", + ); + const operation = { kind: "configuration-maintenance.settle" as const, ...plan.dmSettle }; + if ( + !reservation || + reservation.operationFingerprint !== controlDigest(operation) || + plan.dmSettle.settlementId !== + deriveConfigurationFenceSettlementId({ + installationId: owner.installationId, + ownerPrepare: state.execution, + maintenanceId, + candidateId: plan.operation.candidateId, + planDigest: plan.planDigest, + }) + ) + throw Error("Root-fixed settlement reference mismatch"); + const identity = configurationSettlementOwnerIdentitySchema.parse({ + installationId: owner.installationId, + origin: state.origin, + ownerPrepare: state.execution, + admissionDigest: state.receiptDigest, + rootCertificateDigest: root.certificateDigest, + maintenanceId, + candidateId: plan.operation.candidateId, + planDigest: plan.planDigest, + settlementId: plan.dmSettle.settlementId, + }); + const issuerMembership = retainedConfigurationMaintenanceFamilyMember( + state.installation, + state.origin, + state.execution, + ); + if (issuerMembership.rootAdmissionDigest !== state.receiptDigest) + throw Error("Original issuer admission changed"); + const projection = configurationSettlementReservationProjectionSchema.parse({ + version: 2, + purpose: "configuration-fence-settlement-reservation", + installationId: owner.installationId, + origin: state.origin, + ownerPrepare: state.execution, + rootAdmissionDigest: state.receiptDigest, + rootCertificateDigest: root.certificateDigest, + maintenanceId, + candidateId: plan.operation.candidateId, + planDigest: plan.planDigest, + settlementId: plan.dmSettle.settlementId, + reservation, + operation, + reservationDigest: settlementRecordDigest( + configurationSettlementDomains.reservationTuple, + reservation, + ), + issuerMembership, + }); + const signedProjection = signControl( + state.installation.privateKey, + configurationSettlementDomains.signedProjection, + projection, + ); + const value = Object.freeze({}) as OwnedConfigurationSettlementReservation; + reservations.set( + value, + Object.freeze({ + authority, + projection: freeze(projection), + signedProjection: freeze(signedProjection), + identity: freeze(identity), + projectionDigest: settlementRecordDigest( + configurationSettlementDomains.projection, + projection, + ), + ownerRecordDigest: controlDigest(raw), + plan: freeze(plan), + }), + ); + return value; + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} +/** Source publisher has no caller address, path, digest or outcome arguments. + * All fixed sender cells and its intent are retained before exposing the exact + * pending projection. Partial reservation stays indexed and blocks retirement. */ +export function publishConfigurationSettlementReservationProjection( + authority: VerifiedExecutionAuthority, + maintenanceId: string, +): void { + const prepared = prepareOwnedConfigurationSettlementReservation(authority, maintenanceId), + source = ownedConfigurationSettlementReservationState(prepared, authority), + owner = executionAuthorityState(authority), + projection = source.projection; + const prepare = owner.reservations.find((value) => value.phase === "configuration-fence"); + if (!prepare || !owner.origin) throw Error("Original PREPARE reservation required"); + const descriptors = configurationSettlementRecordDescriptors( + projection, + prepare.execution.operationKey, + ).sender; + withConfigurationMaintenanceReservationPublication(authority, () => { + const raw = readPrivateRecord( + owner.installation, + "configuration-maintenance", + `${uuid.parse(maintenanceId)}.json`, + ); + if ( + controlDigest(raw) !== source.ownerRecordDigest || + inspectControlChildPublication(authority, prepare).publication.state !== "unissued" + ) + throw Error("Maintenance owner/issuance changed before settlement reservation"); + assertConfigurationSettlementPacketBudget(source.plan, projection, source.signedProjection); + const slot = reserveConfigurationSettlementFamilySlot( + owner.installation, + projection, + source.projectionDigest, + undefined, + descriptors, + ); + for (const descriptor of descriptors) { + const value = descriptor.name.endsWith(".configuration-settlement-control.json") + ? signControl(owner.installation.privateKey, configurationSettlementDomains.control, { + version: 2 as const, + projection: source.signedProjection, + binding: null, + }) + : descriptor.name.endsWith(".configuration-settlement.json") + ? signControl(owner.installation.privateKey, configurationSettlementDomains.reference, { + version: 2 as const, + phase: "uncommitted" as const, + identity: source.identity, + referenceOperationFingerprint: projection.reservation.operationFingerprint, + }) + : null; + try { + const existing = readConfigurationSettlementCell( + owner.installation, + projection, + descriptor, + ); + if (canonicalJson(existing) !== canonicalJson(value)) + throw Error("Original pending settlement cell changed"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || slot.state !== "allocating") + throw error; + writeConfigurationSettlementCell(owner.installation, projection, descriptor, value, true); + } + } + completeConfigurationSettlementFamilyReservation(owner.installation, projection); + }); +} +declare const publishedReservationBrand: unique symbol; +export type VerifiedPublishedConfigurationSettlementReservation = Readonly<{ + [publishedReservationBrand]: true; +}>; +const publishedReservations = new WeakMap< + VerifiedPublishedConfigurationSettlementReservation, + { + installation: ControlInstallation; + prepareExecution: ReturnType["execution"]; + peer: VerifiedControlPeer; + projection: ReturnType; + projectionDigest: string; + publication: "pending" | "bound" | "unused"; + } +>(); +export function publishedConfigurationSettlementReservationState( + value: VerifiedPublishedConfigurationSettlementReservation, + installation: ControlInstallation, +) { + const state = publishedReservations.get(value); + if (!state || state.installation !== installation) + throw Error("Verified original published settlement reservation required"); + installationState(installation); + return state; +} +/** Pinned original PREPARE peer, full body, original reservation preimage and + * shared issuer membership must all agree BEFORE a receiver reserves slots. */ +export function verifyPublishedConfigurationSettlementReservation( + installation: ControlInstallation, + peer: VerifiedControlPeer, + value: DataConfigurationFencePrepare, +): VerifiedPublishedConfigurationSettlementReservation { + const local = installationState(installation), + incoming = controlPeerState(peer), + input = dataConfigurationFencePrepareSchema.parse(value); + if ( + installation.role !== "data-manager" || + local.fixedChildId || + incoming.installation !== local || + incoming.role !== "ops-agent" || + incoming.kind !== "configuration-maintenance.prepare" || + incoming.recipient !== "data-manager" + ) + throw Error("Original DM preparation peer required"); + const grant = inspectArtifactDelegation({ + dataRoot: local.dataRoot, + assertion: incoming.assertion, + installation, + }); + if ( + grant.kind !== incoming.kind || + grant.phase !== "configuration-fence" || + grant.recipient !== "data-manager" || + canonicalJson(grant.execution) !== canonicalJson(incoming.execution) || + canonicalJson(grant.parent) !== canonicalJson(input.ownerPrepare) || + grant.execution.operationFingerprint !== + controlDigest({ kind: "configuration-maintenance.prepare", ...input }) || + !grant.origin + ) + throw Error("Original fixed PREPARE reservation mismatch"); + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(local.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + role = own(executions.directory("ops-agent")); + const cell = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.control, + JSON.parse( + role + .read(`${grant.parent.operationKey}.configuration-settlement-control.json`, 8192) + .toString("utf8"), + ), + configurationSettlementControlCellSchema.shape.payload, + ); + const projection = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.signedProjection, + cell.projection, + configurationSettlementReservationProjectionSchema, + ); + const witness = readControlExecutionWitness(local, grant.parent, local.publicKeys["ops-agent"]); + const reservation = projection.reservation; + if ( + projection.installationId !== installation.installationId || + canonicalJson(projection.origin) !== canonicalJson(grant.origin) || + canonicalJson(projection.ownerPrepare) !== canonicalJson(grant.parent) || + witness.kind !== "configurationMaintenance.prepare" || + canonicalJson(witness.origin) !== canonicalJson(grant.origin) || + projection.rootAdmissionDigest !== witness.authorityDigest || + projection.issuerMembership.rootAdmissionDigest !== witness.authorityDigest || + projection.rootCertificateDigest !== grant.ownerCertificateDigest || + projection.maintenanceId !== input.maintenanceId || + projection.candidateId !== input.candidateId || + projection.planDigest !== input.planDigest || + canonicalJson(reservation.serviceIds) !== canonicalJson(grant.serviceIds) || + reservation.execution.operationKey !== + controlChildOperationKey( + grant.parent.operationKey, + "configuration-fence-settle", + reservation.ordinal, + ) || + reservation.operationFingerprint !== controlDigest(projection.operation) || + projection.reservationDigest !== + settlementRecordDigest(configurationSettlementDomains.reservationTuple, reservation) || + projection.operation.maintenanceId !== input.maintenanceId || + projection.operation.candidateId !== input.candidateId || + projection.operation.planDigest !== input.planDigest || + projection.operation.settlementId !== projection.settlementId || + projection.settlementId !== + deriveConfigurationFenceSettlementId({ + installationId: installation.installationId, + ownerPrepare: grant.parent, + maintenanceId: input.maintenanceId, + candidateId: input.candidateId, + planDigest: input.planDigest, + }) + ) + throw Error("Published original settlement projection changed"); + verifyPublishedConfigurationMaintenanceFamilyMember( + local, + grant.origin, + grant.parent, + projection.issuerMembership, + ); + const handle = Object.freeze({}) as VerifiedPublishedConfigurationSettlementReservation; + publishedReservations.set( + handle, + Object.freeze({ + installation, + prepareExecution: incoming.execution, + peer, + projection: freeze(projection), + projectionDigest: settlementRecordDigest( + configurationSettlementDomains.projection, + projection, + ), + publication: cell.binding ? "bound" : "unused" in cell ? "unused" : "pending", + }), + ); + return handle; + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +declare const capacityRefusalBrand: unique symbol; +export type RefusedConfigurationSettlementCapacity = Readonly<{ [capacityRefusalBrand]: true }>; +const capacityRefusals = new WeakMap< + RefusedConfigurationSettlementCapacity, + { + authority: VerifiedExecutionAuthority; + source: ReturnType; + } +>(); +export function refusedConfigurationSettlementCapacityState( + value: RefusedConfigurationSettlementCapacity, + authority: VerifiedExecutionAuthority, +) { + const refused = capacityRefusals.get(value), + owner = executionAuthorityState(authority); + if (!refused || refused.authority !== authority) + throw Error("Actual original allocator capacity refusal required"); + const slot = readConfigurationSettlementFamilySlot(owner.installation, refused.source.projection); + if ( + slot.state !== "capacity_refused" || + slot.executionOrdinal !== null || + slot.intentOrdinal !== null || + slot.executionDigest !== null || + slot.controlBindingDigest !== null || + !slot.refusalConstraint || + slot.allocatedDescriptorDigests.length !== 1 + ) + throw Error("Conclusive original unallocated capacity refusal required"); + return Object.freeze({ + projection: refused.source.projection, + projectionDigest: refused.source.projectionDigest, + slot: freeze(slot), + }); +} +/** Actual receiver reservation. Only the original normal PREPARE adopter and + * the fixed pinned published-source verifier can reach this constructor. It + * allocates no physical borrower for the later logical control. */ +export function reserveReceivedConfigurationSettlementCapacity( + authority: VerifiedExecutionAuthority, + verified: VerifiedPublishedConfigurationSettlementReservation, +) { + const state = executionAuthorityState(authority), + source = publishedConfigurationSettlementReservationState( + verified, + state.installation.installation, + ), + installation = state.installation.installation; + if ( + installation.role !== "data-manager" || + state.installation.fixedChildId || + state.kind !== "configuration-maintenance.prepare" || + canonicalJson(state.execution) !== canonicalJson(source.prepareExecution) || + !state.origin || + canonicalJson(state.origin) !== canonicalJson(source.projection.origin) || + source.publication !== "pending" + ) + throw Error("Actual original DM PREPARE adopter required for capacity"); + assertControlExecutionForward(authority); + const raw = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: unknown } }; + if (!raw.payload?.operation) throw Error("Original DM PREPARE body required"); + const { kind, ...input } = raw.payload.operation as { + kind?: unknown; + } & DataConfigurationFencePrepare; + if (kind !== "configuration-maintenance.prepare") + throw Error("Original DM PREPARE body required"); + const current = publishedConfigurationSettlementReservationState( + verifyPublishedConfigurationSettlementReservation(installation, source.peer, input), + installation, + ); + if (current.projectionDigest !== source.projectionDigest || current.publication !== "pending") + throw Error("Original pending settlement source changed"); + const projection = source.projection, + descriptors = configurationSettlementRecordDescriptors( + projection, + state.execution.operationKey, + ).receiver; + return withPrivateEffectWriter(state, () => { + // The fixed compound refusal channel is indexed and fully charged FIRST. + // Failure to obtain even this channel stays unknown, never a capacity proof. + const base = reserveConfigurationSettlementFamilySlot( + state.installation, + projection, + source.projectionDigest, + state.execution, + descriptors, + "refusal-channel", + ), + channel = descriptors[0]!; + try { + const old = configurationDmCapacityCellSchema.parse( + readConfigurationSettlementCell(state.installation, projection, channel), + ); + if (old.refusal || old.unused) throw Error("Original DM settlement reservation closed"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || base.state !== "refusal_channel") + throw error; + writeConfigurationSettlementCell( + state.installation, + projection, + channel, + { version: 2, capacity: null, refusal: null, unused: null }, + true, + ); + } + const slot = reserveConfigurationSettlementFamilySlot( + state.installation, + projection, + source.projectionDigest, + state.execution, + descriptors, + ); + if (slot.state === "capacity_refused") { + const refused = Object.freeze({}) as RefusedConfigurationSettlementCapacity; + capacityRefusals.set(refused, { authority, source }); + refusedConfigurationSettlementCapacityState(refused, authority); + return refused; + } + for (const descriptor of descriptors) { + const value = descriptor.name.endsWith("-capacity.json") + ? configurationDmCapacityCellSchema.parse({ + version: 2, + capacity: null, + refusal: null, + unused: null, + }) + : null; + try { + const existing = readConfigurationSettlementCell( + state.installation, + projection, + descriptor, + ); + if (descriptor.name.endsWith("-capacity.json")) { + const old = configurationDmCapacityCellSchema.parse(existing); + if (old.refusal || old.unused) throw Error("Original DM settlement reservation closed"); + } else if (existing !== null) throw Error("Original DM settlement control already bound"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || slot.state !== "allocating") + throw error; + writeConfigurationSettlementCell(state.installation, projection, descriptor, value, true); + } + } + const reserved = completeConfigurationSettlementFamilyReservation( + state.installation, + projection, + ), + capacity = configurationSettlementCapacitySchema.parse({ + version: 2, + installationId: installation.installationId, + origin: projection.origin, + ownerPrepareDigest: controlDigest(projection.ownerPrepare), + settlementId: projection.settlementId, + proposedExecutionAddress: projection.reservation.execution, + reservedExecutionOrdinal: reserved.executionOrdinal, + reservedIntentOrdinal: reserved.intentOrdinal, + recordDescriptorsDigest: reserved.recordDescriptorsDigest, + maxBytes: descriptors.reduce((sum, descriptor) => sum + descriptor.cap, 0), + projectionDigest: source.projectionDigest, + reservationDigest: projection.reservationDigest, + }), + signed = signControl( + state.installation.privateKey, + configurationSettlementDomains.capacity, + capacity, + ), + descriptor = descriptors[0]!, + old = configurationDmCapacityCellSchema.parse( + readConfigurationSettlementCell(state.installation, projection, descriptor), + ); + if (old.capacity && canonicalJson(old.capacity) !== canonicalJson(signed)) + throw Error("Original DM capacity certificate changed"); + writeConfigurationSettlementCell(state.installation, projection, descriptor, { + ...old, + capacity: signed, + }); + return freeze({ + capacity, + capacityDigest: settlementRecordDigest(configurationSettlementDomains.capacity, capacity), + }); + }); +} +declare const dmCapacityBrand: unique symbol; +export type VerifiedPublishedConfigurationSettlementCapacity = Readonly<{ + [dmCapacityBrand]: true; +}>; +const dmCapacities = new WeakMap< + VerifiedPublishedConfigurationSettlementCapacity, + { + authority: VerifiedExecutionAuthority; + projectionDigest: string; + capacityDigest: string; + signed: NonNullable["capacity"]>; + } +>(); +export function publishedConfigurationSettlementCapacityState( + value: VerifiedPublishedConfigurationSettlementCapacity, + authority: VerifiedExecutionAuthority, +) { + const state = dmCapacities.get(value); + if (!state || state.authority !== authority) + throw Error("Actual original published DM capacity required"); + executionAuthorityState(authority); + return state; +} +/** Only pinned DM source bytes plus the original shared slot preimages make a + * preparation-capacity proof. Receipt JSON or a claimed ready flag cannot. */ +export function verifyPublishedConfigurationSettlementCapacity( + authority: VerifiedExecutionAuthority, + maintenanceId: string, +): VerifiedPublishedConfigurationSettlementCapacity { + const source = ownedConfigurationSettlementReservationState( + prepareOwnedConfigurationSettlementReservation(authority, maintenanceId), + authority, + ), + state = executionAuthorityState(authority), + projection = source.projection, + prepare = state.reservations.find((value) => value.phase === "configuration-fence"); + if (!prepare) throw Error("Original DM preparation reservation missing"); + const descriptors = configurationSettlementRecordDescriptors( + projection, + prepare.execution.operationKey, + ).receiver, + opened: AnchoredDirectory[] = [], + own = (dir: AnchoredDirectory) => { + opened.push(dir); + return dir; + }; + try { + const root = own(new AnchoredDirectory(state.installation.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + dm = own(executions.directory("data-manager")), + bytes = dm.read(descriptors[0]!.name, descriptors[0]!.cap); + if (bytes.length !== descriptors[0]!.cap) + throw Error("Original DM compound capacity cell width changed"); + const cell = configurationDmCapacityCellSchema.parse(JSON.parse(bytes.toString("utf8"))); + if (!cell.capacity || cell.refusal || cell.unused) + throw Error("Conclusive live DM capacity certificate required"); + const capacity = verifyControl( + state.installation.publicKeys["data-manager"], + configurationSettlementDomains.capacity, + cell.capacity, + configurationSettlementCapacitySchema, + ); + if ( + capacity.installationId !== projection.installationId || + canonicalJson(capacity.origin) !== canonicalJson(projection.origin) || + capacity.ownerPrepareDigest !== controlDigest(projection.ownerPrepare) || + capacity.settlementId !== projection.settlementId || + capacity.projectionDigest !== source.projectionDigest || + capacity.reservationDigest !== projection.reservationDigest || + canonicalJson(capacity.proposedExecutionAddress) !== + canonicalJson(projection.reservation.execution) || + capacity.recordDescriptorsDigest !== controlDigest(descriptors) || + capacity.maxBytes !== descriptors.reduce((sum, descriptor) => sum + descriptor.cap, 0) + ) + throw Error("Original DM capacity binding changed"); + verifyPublishedDmConfigurationSettlementSlots(state.installation, projection, capacity); + const proof = Object.freeze({}) as VerifiedPublishedConfigurationSettlementCapacity; + dmCapacities.set( + proof, + Object.freeze({ + authority, + projectionDigest: source.projectionDigest, + capacityDigest: settlementRecordDigest(configurationSettlementDomains.capacity, capacity), + signed: freeze(cell.capacity), + }), + ); + return proof; + } finally { + for (const dir of opened.reverse()) dir.close(); + } +} diff --git a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts new file mode 100644 index 000000000..857c2c401 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts @@ -0,0 +1,433 @@ +/** Prospective encoding arithmetic only. These local maximum-width data + * expressions never become a signed witness, authority handle or publication. */ +import { dirname, join } from "node:path"; +import { canonicalJson } from "../ops/canonical-json"; +import { configurationMaintenancePrepareSchema } from "../ops/configuration-maintenance"; +import type { PreparedConfigurationRecord } from "../ops/configuration-preparation-records"; +import type { OpsOperation } from "../ops/contract"; +import { parseControlRootOperationKey } from "../ops/control-root-key"; +import { executionFingerprint, type RetainedParentExecution } from "../ops/execution-identity"; +import { opsOperationFingerprint } from "../ops/fingerprint"; +import { renderServiceSnippet } from "../services/compose-renderer"; +import type { renderConfiguration } from "../services/configuration-generation"; +import { + CONFIGURATION_GENERATIONS, + GENERATED_COMPOSE_FILE, + STACK_PROJECT, +} from "../services/deployment"; +import { assertProspectiveConfigurationSettlementPacketBudget } from "./configuration-fence-settlement-budget"; +import type { ConfigurationMaintenanceOwnerRecord } from "./configuration-maintenance-owner-records"; +import { + buildFixedConfigurationMaintenancePlan, + type FixedConfigurationMaintenancePlan, +} from "./fixed-configuration-maintenance"; + +const hash = "f".repeat(64), + uuid = "ffffffff-ffff-4fff-bfff-ffffffffffff", + at = Number.MAX_SAFE_INTEGER; +const bytes = (value: unknown) => Buffer.byteLength(canonicalJson(value)); +export function assertProspectiveConfigurationMaintenanceBudget(input: { + installationId: string; + plan: FixedConfigurationMaintenancePlan; + originalContainers: ConfigurationMaintenanceOwnerRecord["originalContainers"]; + rendered: ReturnType; +}) { + const { plan, originalContainers, rendered } = input, + { kind: _kind, ...binding } = plan.operation, + identity = { + installationId: input.installationId, + origin: { + version: 1, + rootKey: plan.prepare.operationKey, + issuer: "ops-agent", + notAfter: parseControlRootOperationKey(plan.prepare.operationKey).notAfter, + }, + ownerPrepare: plan.prepare, + admissionDigest: hash, + rootCertificateDigest: hash, + maintenanceId: binding.maintenanceId, + candidateId: binding.candidateId, + planDigest: plan.planDigest, + settlementId: plan.dmSettle.settlementId, + }, + guard = { ...plan.dmPrepare, fenceId: uuid, sequence: at, state: "recovery_required" }, + receipt = { + version: 1, + binding, + prepare: plan.prepare, + apiJobId: plan.apiParent.issuer === "api" ? plan.apiParent.apiJobId : uuid, + parentOperationKey: plan.apiParent.operationKey, + parentFingerprint: plan.apiParent.fingerprint, + parentDeadlineAt: plan.apiParent.deadlineAt, + planDigest: plan.planDigest, + serviceIds: plan.serviceIds, + steps: plan.steps, + preflight: "unavailable", + sequence: at, + state: "recovery_required", + decision: { + decisionId: uuid, + decision: "continue", + checkpointVersion: at, + checkpointDigest: hash, + }, + dmFence: guard, + generation: "not_committed", + appliedRevisionId: plan.operation.revisionId, + followUp: "not_started", + }, + reservations = [ + ...plan.reservations.filter( + (value) => !["configuration-fence", "configuration-fence-settle"].includes(value.phase), + ), + ...(plan.nestedHardlinks ? [plan.nestedHardlinks] : []), + ], + children = reservations.map((value) => ({ + planIndex: value.ordinal > 31 ? 31 : value.ordinal, + execution: { + ...value.execution, + operationFingerprint: hash, + fingerprint: hash, + deadlineAt: at, + }, + terminalDigest: hash, + terminalAt: at, + effectSettlementDigest: hash, + publication: "not_started", + })), + recreates = new Set( + plan.steps.flatMap((value) => + value.operation.kind === "service.recreate" || + value.operation.kind === "service.recreateIsolated" + ? [value.operation.serviceId] + : [], + ), + ), + futureBase = join( + rendered.context.infraDir!, + ".trusted-config-generations", + plan.operation.revisionId, + ), + longest = (a: string, b: string) => (bytes(a) > bytes(b) ? a : b), + futureContainers = originalContainers.map((value) => + recreates.has(value.serviceId) + ? { + ...value, + containerId: hash, + state: "restarting", + provenance: { + ...value.provenance, + composeWorkingDirectory: longest( + value.provenance.composeWorkingDirectory, + futureBase, + ), + composeConfigFiles: [0, 1].map((index) => + longest( + value.provenance.composeConfigFiles[index] ?? "", + join(futureBase, "docker-compose.generated.yml"), + ), + ), + }, + } + : value, + ), + checks = [ + "generation", + "owner-containers", + "dm-guard", + ...plan.steps.map(() => "frozen-attempts"), + ].map((check) => ({ + serviceId: null as string | null, + ordinal: null as number | null, + check, + expectedRevisionId: plan.operation.revisionId, + expectedImageId: null as string | null, + })), + details: unknown[] = [ + { currentRevisionId: plan.operation.revisionId }, + { containers: futureContainers }, + { guard }, + ...plan.steps.map((step) => ({ + children: children.filter((value) => value.planIndex === step.ordinal), + })), + ]; + for (const id of recreates) { + const service = rendered.resolved.enabled.find((value) => value.manifest.id === id); + if (!service) throw Error("Prospective maintenance runtime target unavailable"); + const snippet = renderServiceSnippet(service, rendered.context), + mountCount = (snippet.volumes?.length ?? 0) + (snippet.secrets?.length ?? 0); + for (const check of ["configuration", "image", "mount", "readiness"]) + checks.push({ + serviceId: id, + ordinal: null, + check, + expectedRevisionId: plan.operation.revisionId, + expectedImageId: `sha256:${hash}`, + }); + details.push( + { + containerId: hash, + expectedConfigHash: hash, + actualConfigHash: hash, + expectedEnvironmentDigest: hash, + actualEnvironmentDigest: hash, + }, + { containerId: hash, imageId: `sha256:${hash}`, expectedImageId: `sha256:${hash}` }, + { + containerId: hash, + mounts: Array.from({ length: mountCount }, () => ({ + targetDigest: hash, + type: "volume", + sourceDigest: hash, + nameDigest: hash, + readOnly: false, + expectedDigest: hash, + actualDigest: hash, + })), + }, + { containerId: hash, state: "running", running: false, health: "healthy" }, + ); + } + // UNUSED has no Docker checks, but adds exact candidate/queue descriptor facts. + // Budget both alternatives, including their independently escaped detail JSON. + const unusedDetails = [ + { currentRevisionId: binding.expectedCurrentRevisionId }, + { containers: originalContainers }, + { + candidateId: binding.candidateId, + revisionId: binding.revisionId, + recordDigest: hash, + directories: ["queue", "claimed"].map((name) => ({ + name, + dev: at, + ino: at, + uid: at, + gid: at, + mode: 448, + entries: 1024, + matchingCandidateEntries: 0, + })), + }, + { guard }, + ...plan.steps.map((step) => ({ + phase: step.phase, + ordinal: step.ordinal, + operationKey: step.operationKey, + binding: null, + publication: "unissued", + })), + ]; + if ([...details, ...unusedDetails].some((value) => bytes(value) > 16384)) + throw Error("Prospective private maintenance observation capacity exceeded"); + const inspectionWidth = (observations: unknown[]) => + bytes({ + payload: { + version: 2, + identity, + inspectionId: uuid, + ownerSequence: at, + ownerSourceDigest: hash, + requiredChecks: checks, + observations: observations.map((detail, checkIndex) => ({ + checkIndex, + passed: true, + detail: canonicalJson(detail), + })), + runtime: { + verificationId: uuid, + mode: "contained", + requiredCheckSetDigest: hash, + checkedAt: at, + }, + }, + signature: "", + }) + 86, + inspectionBytes = Math.max(inspectionWidth(details), inspectionWidth(unusedDetails)), + // The complete final contains both full private children and their compact + // shared projection; all other final/guard/decision fields have fixed widths. + finalBytes = + 8192 + + bytes(children) + + bytes( + children.map(({ execution: _execution, ...value }) => ({ + ...value, + executionDigest: hash, + })), + ), + apiParentBytes = bytes(plan.apiParent) + 64, + assertionBytes = 4096, + stepBytes = plan.steps.reduce((sum, step) => { + const resultBytes = + step.phase === "configuration-commit" + ? bytes({ + outcome: "committed", + revisionId: binding.revisionId, + enabledServiceIds: plan.serviceIds, + followUp: "complete", + cleanupFailed: false, + }) + : 512; + // Exact fixed fields: execution+API parent/assertion, grant, canonical + // status/result/digests/time and durable API receipt delivery. + return sum + bytes(step.operation) + 2048 + apiParentBytes + assertionBytes + resultBytes; + }, 0), + ownerBaseBytes = + bytes({ plan, originalContainers, receipt, identity }) + + 2048 + + (plan.steps.some( + (step) => + (step.operation.kind === "service.recreate" || + step.operation.kind === "service.recreateIsolated") && + step.operation.serviceId === "data-manager", + ) + ? 8192 + : 0), + ownerBytes = ownerBaseBytes + inspectionBytes + finalBytes + 2 * 8192 + stepBytes + 8192, + noGuardOwnerBytes = ownerBaseBytes + 32768, + receiptBytes = bytes(receipt), + phaseBytes = (plan.reservations.length + (plan.nestedHardlinks ? 1 : 0)) * 8192; + if ( + inspectionBytes > 65536 || + Math.max(ownerBytes, noGuardOwnerBytes) > 131072 || + receiptBytes > 65536 || + phaseBytes > 131072 + ) + throw Error("Prospective maintenance owner/inspection/receipt/phase capacity exceeded"); + return Object.freeze({ + ownerBytes: Math.max(ownerBytes, noGuardOwnerBytes), + inspectionBytes, + receiptBytes, + phaseBytes, + }); +} + +/** Pre-effect arithmetic for the SAME sealed candidate, not an admission or a + * proposed execution. Maximum-width future IDs/image/parent data stay local; + * no signature, nominal token, filesystem record or permission is produced. */ +export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { + installationId: string; + record: PreparedConfigurationRecord; + sourceAuthorityRevision: string; + infraDir: string; + currentServiceIds: readonly string[]; + originalContainers: ConfigurationMaintenanceOwnerRecord["originalContainers"]; + rendered: ReturnType; + coordinatorImage?: Omit< + NonNullable, + "imageId" + >; +}): void { + const { record, rendered } = input; + if (!record.maintenanceId || !record.prepareOperationKey) return; + const apply = record.operation as Parameters< + typeof buildFixedConfigurationMaintenancePlan + >[0]["apply"]; + const recreateServiceIds = + record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + ? record.affectedServiceIds + : []; + const operation = configurationMaintenancePrepareSchema.parse({ + kind: "configurationMaintenance.prepare", + maintenanceId: record.maintenanceId, + candidateId: record.candidateId, + applyOperationKey: record.operationKey, + applyOperationFingerprint: opsOperationFingerprint(apply as OpsOperation), + expectedCurrentRevisionId: apply.expectedCurrentRevisionId, + revisionId: apply.revisionId, + expectedSourceRevision: record.expectedSourceRevision, + coordinatorMode: apply.coordinatorMode, + followUp: recreateServiceIds.length ? "service-apply" : "render-only", + }); + const parent: RetainedParentExecution = { + issuer: "api", + operationKey: record.parentOperationKey, + fingerprint: record.parentFingerprint, + deadlineAt: record.parentDeadlineAt, + apiJobId: record.apiJobId, + apiOwnerVersion: at, + attemptId: uuid, + assertionDigest: hash, + }; + const deadlineAt = Math.min( + parseControlRootOperationKey(record.prepareOperationKey).notAfter, + parent.deadlineAt, + ), + operationFingerprint = opsOperationFingerprint(operation); + const prepare = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${"f".repeat(64)}`, + operationKey: record.prepareOperationKey, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + parent, + }), + }; + const serviceIds = [ + ...new Set([ + ...input.currentServiceIds, + ...rendered.resolved.enabled.map((service) => service.manifest.id), + ...record.affectedServiceIds, + "data-manager", + ]), + ].sort(); + const plan = buildFixedConfigurationMaintenancePlan({ + installationId: input.installationId, + prepare, + operation, + apiParent: parent, + apply, + candidateRecordDigest: hash, + sourceAuthorityRevision: input.sourceAuthorityRevision, + previewDigest: hash, + hardlinkPlan: { plan: rendered.rendered.hardlinkPlan, prune: true }, + serviceIds, + recreateServiceIds, + ...(input.coordinatorImage + ? { coordinatorImage: { ...input.coordinatorImage, imageId: `sha256:${hash}` } } + : {}), + }); + const currentCompose = join( + input.infraDir, + CONFIGURATION_GENERATIONS, + apply.expectedCurrentRevisionId ?? apply.revisionId, + GENERATED_COMPOSE_FILE, + ), + release = join(input.infraDir, "docker-compose.release.yml"); + const longest = (a: string, b: string) => (bytes(a) > bytes(b) ? a : b); + const observed = new Map(input.originalContainers.map((value) => [value.serviceId, value])); + // Any selected service may become an owned current-generation container in + // the pre-root gap. Reserve its full fixed provenance width before the helper + // or pull; actual admission independently rechecks reality and the cap. + const containers = [...new Set([...observed.keys(), ...serviceIds])].sort().map((serviceId) => { + const actual = observed.get(serviceId); + return { + serviceId, + containerId: hash, + state: "restarting" as const, + provenance: { + project: STACK_PROJECT, + composeWorkingDirectory: longest( + actual?.provenance.composeWorkingDirectory ?? "", + dirname(currentCompose), + ), + composeConfigFiles: [ + longest(actual?.provenance.composeConfigFiles[0] ?? "", currentCompose), + longest(actual?.provenance.composeConfigFiles[1] ?? "", release), + ], + imageId: `sha256:${hash}`, + }, + }; + }); + assertProspectiveConfigurationMaintenanceBudget({ + installationId: input.installationId, + plan, + originalContainers: containers, + rendered, + }); + assertProspectiveConfigurationSettlementPacketBudget(plan, input.installationId); +} diff --git a/packages/core/src/build-artifacts/configuration-maintenance-closure.ts b/packages/core/src/build-artifacts/configuration-maintenance-closure.ts new file mode 100644 index 000000000..ff35a6d79 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-maintenance-closure.ts @@ -0,0 +1,2 @@ +/** Fixed closure stays inside the actual physical tree owner; this path remains compatible. */ +export { assertConfigurationMaintenanceRootClosure } from "./target-lease"; diff --git a/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts new file mode 100644 index 000000000..5b181073d --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts @@ -0,0 +1,53 @@ +/** Fixed private Docker observation data; parsing grants no authority. */ +import { z } from "zod"; + +const string = z.string().max(65536), + mount = z.object({ + Type: z.enum(["bind", "volume", "tmpfs"]), + Source: string.optional(), + Name: string.optional(), + Destination: string, + RW: z.boolean(), + }); +export const maintenanceDockerContainerSchema = z.object({ + Id: z.string().regex(/^[a-f0-9]{64}$/), + Image: z.string().regex(/^sha256:[a-f0-9]{64}$/), + State: z.object({ + Status: z.string(), + Running: z.boolean(), + Health: z.object({ Status: z.string() }).optional(), + }), + Config: z.object({ + Image: string, + Labels: z.record(z.string(), string), + Env: z.array(string).max(1024), + }), + Mounts: z.array(mount).max(512), +}); +const service = z.object({ + image: string, + environment: z + .record(z.string(), z.union([string, z.number(), z.boolean(), z.null()])) + .default({}), + volumes: z + .array( + z.object({ + type: z.enum(["bind", "volume", "tmpfs"]), + source: string.optional(), + target: string, + read_only: z.boolean().optional(), + }), + ) + .max(256) + .default([]), + secrets: z + .array(z.object({ source: string, target: string.optional() })) + .max(128) + .default([]), + healthcheck: z.object({ disable: z.boolean().optional() }).optional(), +}); +export const maintenanceDockerComposeSchema = z.object({ + services: z.record(z.string(), service), + secrets: z.record(z.string(), z.object({ file: string.optional() })).default({}), + volumes: z.record(z.string(), z.object({ name: string.optional() })).default({}), +}); diff --git a/packages/core/src/build-artifacts/configuration-maintenance-final-children.ts b/packages/core/src/build-artifacts/configuration-maintenance-final-children.ts new file mode 100644 index 000000000..7a9df3d30 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-maintenance-final-children.ts @@ -0,0 +1,2 @@ +/** Original children are joined against the same nominal physical tree owner. */ +export { readOriginalMaintenanceFinalChildren } from "./target-lease"; diff --git a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts new file mode 100644 index 000000000..519b0e4a8 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts @@ -0,0 +1,429 @@ +/** Private fixed final producer records. Parsing is never final authority. */ +import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { + configurationFinalEvidenceProjectionSchema, + configurationFinalEvidenceSchema, + configurationOwnerFinalSchema, + configurationSettlementDomains, + configurationSettlementOwnerIdentitySchema, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import type { ConfigurationMaintenanceOwnerRecord } from "./configuration-maintenance-owner-records"; +import { controlDigest } from "./control-records"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/), + epoch = z.number().int().nonnegative().safe(); +const signed = (payload: T) => + z.strictObject({ payload, signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }); +export const configurationMaintenanceRequiredChecksSchema = z + .array( + z.strictObject({ + serviceId: z + .string() + .regex(/^[a-z0-9][a-z0-9-]{0,63}$/) + .nullable(), + ordinal: z.number().int().min(0).max(31).nullable(), + check: z.enum([ + "generation", + "owner-containers", + "frozen-attempts", + "candidate-not-consumed", + "dm-guard", + "configuration", + "image", + "mount", + "readiness", + ]), + expectedRevisionId: z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .nullable(), + expectedImageId: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .nullable(), + }), + ) + .max(1024); +export const configurationMaintenanceRuntimeInspectionSchema = z.strictObject({ + version: z.literal(2), + identity: z.lazy(() => configurationSettlementOwnerIdentitySchema), + inspectionId: z.uuid(), + ownerSequence: epoch, + ownerSourceDigest: digest, + requiredChecks: configurationMaintenanceRequiredChecksSchema, + observations: z + .array( + z.strictObject({ + checkIndex: z.number().int().min(0).max(1023), + passed: z.literal(true), + detail: z.string().max(16384), + }), + ) + .max(1024), + runtime: z.lazy(() => configurationFinalEvidenceSchema.shape.runtime), +}); +export const signedConfigurationMaintenanceRuntimeInspectionSchema = signed( + configurationMaintenanceRuntimeInspectionSchema, +); +export const configurationMaintenanceFinalRecordSchema = z.strictObject({ + ownerFinal: signed(z.lazy(() => configurationOwnerFinalSchema)), + evidence: z.lazy(() => configurationFinalEvidenceSchema), + evidenceProjection: z.lazy(() => configurationFinalEvidenceProjectionSchema), + evidenceDigest: digest, + ownerReceiptDigest: digest, +}); +const unusedCandidateObservationSchema = z.strictObject({ + candidateId: z.uuid(), + revisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), + recordDigest: digest, + directories: z + .array( + z.strictObject({ + name: z.enum(["queue", "claimed"]), + dev: epoch, + ino: epoch, + uid: epoch, + gid: epoch, + mode: z.literal(0o700), + entries: z.number().int().min(0).max(1024), + matchingCandidateEntries: z.literal(0), + }), + ) + .length(2), +}); +/** Consistency only. The caller must authenticate/index the ACTUAL owner, + * inspection and final first. No parsed state creates final authority. */ +export function assertUnusedConfigurationMaintenanceFinalState( + owner: ConfigurationMaintenanceOwnerRecord, +): void { + const final = owner.final, + signedInspection = owner.inspection, + decision = owner.apiControls.decide; + if ( + !final || + !signedInspection || + !decision?.receiptDelivery || + !owner.guardSource || + owner.noGuard || + owner.issuance !== "open" + ) + throw Error("Actual unused original final source required"); + const inspection = signedInspection.payload, + original = final.ownerFinal.payload, + evidence = final.evidence, + same = (a: unknown, b: unknown) => canonicalJson(a) === canonicalJson(b), + requiredChecks = configurationMaintenanceRequiredChecksSchema.parse([ + ...["generation", "owner-containers", "candidate-not-consumed", "dm-guard"].map((check) => ({ + serviceId: null, + ordinal: null, + check, + expectedRevisionId: owner.plan.operation.expectedCurrentRevisionId, + expectedImageId: null, + })), + ...owner.plan.steps.map((step) => ({ + serviceId: null, + ordinal: step.ordinal, + check: "frozen-attempts", + expectedRevisionId: owner.plan.operation.revisionId, + expectedImageId: null, + })), + ]), + settle = owner.apiControls.settle, + beforeInspection = { + ...owner, + sequence: inspection.ownerSequence, + final: null, + inspection: null, + apiControls: { + ...owner.apiControls, + settle: settle + ? { + ...settle, + completedAt: null, + result: null, + canonicalDigest: null, + receiptDelivery: null, + } + : null, + }, + receipt: { + ...owner.receipt, + sequence: inspection.ownerSequence, + state: "authorized", + dmFence: owner.guardSource.preparedReceipt, + }, + }; + const details = inspection.observations.map((value) => { + const parsed: unknown = JSON.parse(value.detail); + if (canonicalJson(parsed) !== value.detail) + throw Error("Original unused observation encoding changed"); + return parsed; + }), + candidate = unusedCandidateObservationSchema.parse(details[2]); + if ( + !same(details[0], { currentRevisionId: owner.plan.operation.expectedCurrentRevisionId }) || + !same(details[1], { containers: owner.originalContainers }) || + !same(details[3], { guard: owner.guardSource.preparedReceipt }) || + candidate.candidateId !== owner.plan.operation.candidateId || + candidate.revisionId !== owner.plan.operation.revisionId || + candidate.recordDigest !== owner.plan.candidateRecordDigest || + candidate.directories[0]!.name !== "queue" || + candidate.directories[1]!.name !== "claimed" || + candidate.directories[0]!.uid !== candidate.directories[1]!.uid || + candidate.directories[0]!.gid !== candidate.directories[1]!.gid || + !owner.plan.steps.every((step, index) => + same(details[index + 4], { + phase: step.phase, + ordinal: step.ordinal, + operationKey: step.operationKey, + binding: null, + publication: "unissued", + }), + ) + ) + throw Error("Actual original unused runtime observations changed"); + if ( + original.outcome !== "unused" || + evidence.outcome !== "unused" || + inspection.runtime.mode !== "unused" || + owner.receipt.generation !== "not_started" || + owner.receipt.followUp !== "not_started" || + owner.apiControls.steps.length || + inspection.ownerSourceDigest !== controlDigest(beforeInspection) || + original.ownerSequence !== inspection.ownerSequence + 1 || + original.ownerSequence > owner.sequence || + original.privateInspectionId !== inspection.inspectionId || + !same(inspection.requiredChecks, requiredChecks) || + inspection.observations.length !== requiredChecks.length || + !inspection.observations.every((value, index) => value.checkIndex === index && value.passed) || + inspection.runtime.requiredCheckSetDigest !== + settlementRecordDigest(configurationSettlementDomains.inspection, requiredChecks) || + !same(inspection.identity, original.identity) || + !same(evidence.identity, original.identity) || + !same(evidence.runtime, inspection.runtime) || + original.identity.installationId !== owner.installationId || + !same(original.identity.origin, owner.origin) || + !same(original.identity.ownerPrepare, owner.plan.prepare) || + original.identity.admissionDigest !== owner.admissionDigest || + original.identity.rootCertificateDigest !== owner.rootCertificateDigest || + original.identity.maintenanceId !== owner.plan.operation.maintenanceId || + original.identity.candidateId !== owner.plan.operation.candidateId || + original.identity.planDigest !== owner.plan.planDigest || + original.identity.settlementId !== owner.plan.dmSettle.settlementId || + evidence.children.length || + evidence.generation.state !== "not_committed" || + evidence.followUp.state !== "not_started" || + evidence.followUp.receiptDigests.length || + evidence.decision.decision !== "abandon" || + evidence.decision.decisionId !== owner.receipt.decision?.decisionId || + evidence.decision.checkpointVersion !== decision.operation.checkpointVersion || + evidence.decision.checkpointDigest !== decision.operation.checkpointDigest || + evidence.decision.apiParentAttestationDigest !== decision.apiParent.assertionDigest || + !same(evidence.guard.preparedReceipt, owner.guardSource.preparedReceipt) || + evidence.guard.fenceId !== owner.guardSource.preparedReceipt.fenceId || + evidence.guard.preparedReceiptDigest !== owner.guardSource.preparedReceiptDigest || + evidence.guard.preparedCanonicalDigest !== owner.guardSource.preparedCanonicalDigest || + evidence.guard.preparedTerminalAt !== owner.guardSource.preparedTerminalAt || + original.expectedDmSequence !== owner.guardSource.preparedReceipt.sequence || + !same(final.evidenceProjection, evidence) || + final.evidenceDigest !== + settlementRecordDigest( + configurationSettlementDomains.finalEvidence, + final.evidenceProjection, + ) || + original.evidenceDigest !== final.evidenceDigest || + final.ownerReceiptDigest !== + settlementRecordDigest(configurationSettlementDomains.ownerFinal, original) + ) + throw Error("Actual original unused final evidence changed"); +} +/** Consistency only; actual canonical/effect/tree sources are checked by the + * fixed producer/restorer and the mandatory root closure predicate. */ +export function assertAppliedConfigurationMaintenanceFinalState( + owner: ConfigurationMaintenanceOwnerRecord, +): void { + assertExecutedConfigurationMaintenanceFinalState(owner, "applied"); +} +export function assertContainedConfigurationMaintenanceFinalState( + owner: ConfigurationMaintenanceOwnerRecord, +): void { + assertExecutedConfigurationMaintenanceFinalState(owner, "contained"); +} +function assertExecutedConfigurationMaintenanceFinalState( + owner: ConfigurationMaintenanceOwnerRecord, + mode: "applied" | "contained", +): void { + const final = owner.final, + signedInspection = owner.inspection, + decision = owner.apiControls.decide; + if ( + !final || + !signedInspection || + !decision?.receiptDelivery || + !owner.guardSource || + owner.noGuard || + owner.issuance !== "open" + ) + throw Error("Actual applied final sources required"); + const inspection = signedInspection.payload, + original = final.ownerFinal.payload, + evidence = final.evidence, + same = (a: unknown, b: unknown) => canonicalJson(a) === canonicalJson(b), + contained = mode === "contained", + inspectedRevision = contained + ? owner.plan.operation.expectedCurrentRevisionId + : owner.plan.operation.revisionId, + ids = [ + ...new Set( + owner.plan.steps + .filter((value) => value.phase === "maintenance-recreate") + .map((value) => ("serviceId" in value.operation ? value.operation.serviceId : "")), + ), + ].sort(), + requiredChecks = configurationMaintenanceRequiredChecksSchema.parse([ + ...["generation", "owner-containers", "dm-guard"].map((check) => ({ + serviceId: null, + ordinal: null, + check, + expectedRevisionId: inspectedRevision, + expectedImageId: null, + })), + ...owner.plan.steps.map((step) => ({ + serviceId: null, + ordinal: step.ordinal, + check: "frozen-attempts", + expectedRevisionId: owner.plan.operation.revisionId, + expectedImageId: null, + })), + ...ids.flatMap((serviceId) => + ["configuration", "image", "mount", "readiness"].map((check) => ({ + serviceId, + ordinal: null, + check, + expectedRevisionId: inspectedRevision, + expectedImageId: + !contained && serviceId === "data-manager" && owner.plan.coordinatorImage + ? owner.plan.coordinatorImage.imageId + : owner.originalContainers.find((value) => value.serviceId === serviceId)?.provenance + .imageId, + })), + ), + ]), + settle = owner.apiControls.settle, + before = { + ...owner, + sequence: inspection.ownerSequence, + final: null, + inspection: null, + apiControls: { + ...owner.apiControls, + settle: settle + ? { + ...settle, + completedAt: null, + result: null, + canonicalDigest: null, + receiptDelivery: null, + } + : null, + }, + receipt: { + ...owner.receipt, + sequence: inspection.ownerSequence, + state: "applying", + dmFence: owner.guardSource.preparedReceipt, + }, + }, + details = inspection.observations.map((value) => { + const parsed: unknown = JSON.parse(value.detail); + if (canonicalJson(parsed) !== value.detail) + throw Error("Applied observation encoding changed"); + return parsed; + }), + { children, ...base } = evidence, + projection = { + ...base, + children: children.map(({ execution, ...child }) => ({ + ...child, + executionDigest: controlDigest(execution), + })), + }; + const generation = owner.apiControls.steps.find((value) => value.ordinal === 0)?.canonical?.status + .result as { outcome?: string; cleanupFailed?: boolean } | undefined; + if ( + original.outcome !== mode || + evidence.outcome !== mode || + inspection.runtime.mode !== mode || + (contained + ? !inspectedRevision || + owner.receipt.generation !== "not_committed" || + owner.receipt.appliedRevisionId !== null || + owner.receipt.followUp !== "contained" || + owner.apiControls.steps.length !== 1 || + generation?.outcome !== "not_committed" || + generation.cleanupFailed || + children.length !== 1 || + !same(details[1], { containers: owner.originalContainers }) + : owner.receipt.generation !== "committed" || + owner.receipt.appliedRevisionId !== owner.plan.operation.revisionId || + owner.receipt.followUp !== "complete" || + owner.apiControls.steps.length !== owner.plan.steps.length) || + owner.apiControls.steps.some((value) => !value.canonical || !value.receiptDelivery) || + inspection.ownerSourceDigest !== controlDigest(before) || + original.ownerSequence !== inspection.ownerSequence + 1 || + original.ownerSequence > owner.sequence || + original.privateInspectionId !== inspection.inspectionId || + !same(inspection.requiredChecks, requiredChecks) || + inspection.observations.length !== requiredChecks.length || + !inspection.observations.every((value, index) => value.checkIndex === index && value.passed) || + !same(details[0], { currentRevisionId: inspectedRevision }) || + !same(details[2], { guard: owner.guardSource.preparedReceipt }) || + !owner.plan.steps.every((step, index) => + same(details[index + 3], { + children: evidence.children.filter((value) => value.planIndex === step.ordinal), + }), + ) || + inspection.runtime.requiredCheckSetDigest !== + settlementRecordDigest(configurationSettlementDomains.inspection, requiredChecks) || + !same(inspection.identity, original.identity) || + !same(evidence.identity, original.identity) || + !same(evidence.runtime, inspection.runtime) || + original.identity.installationId !== owner.installationId || + !same(original.identity.origin, owner.origin) || + !same(original.identity.ownerPrepare, owner.plan.prepare) || + original.identity.admissionDigest !== owner.admissionDigest || + original.identity.rootCertificateDigest !== owner.rootCertificateDigest || + original.identity.maintenanceId !== owner.plan.operation.maintenanceId || + original.identity.candidateId !== owner.plan.operation.candidateId || + original.identity.planDigest !== owner.plan.planDigest || + original.identity.settlementId !== owner.plan.dmSettle.settlementId || + evidence.decision.decision !== "continue" || + evidence.decision.decisionId !== owner.receipt.decision?.decisionId || + evidence.decision.checkpointVersion !== decision.operation.checkpointVersion || + evidence.decision.checkpointDigest !== decision.operation.checkpointDigest || + evidence.decision.apiParentAttestationDigest !== decision.apiParent.assertionDigest || + !same(evidence.guard.preparedReceipt, owner.guardSource.preparedReceipt) || + evidence.guard.preparedReceiptDigest !== owner.guardSource.preparedReceiptDigest || + evidence.guard.preparedCanonicalDigest !== owner.guardSource.preparedCanonicalDigest || + evidence.guard.preparedTerminalAt !== owner.guardSource.preparedTerminalAt || + original.expectedDmSequence !== owner.guardSource.preparedReceipt.sequence || + (contained + ? evidence.generation.state !== "not_committed" || + evidence.generation.revisionId !== null || + evidence.generation.commitReceiptDigest !== null || + evidence.followUp.state !== "contained" || + evidence.followUp.receiptDigests.length !== 0 + : evidence.generation.state !== "committed" || + evidence.generation.revisionId !== owner.plan.operation.revisionId || + evidence.followUp.state !== "complete") || + !same(final.evidenceProjection, projection) || + final.evidenceDigest !== + settlementRecordDigest(configurationSettlementDomains.finalEvidence, projection) || + original.evidenceDigest !== final.evidenceDigest || + final.ownerReceiptDigest !== + settlementRecordDigest(configurationSettlementDomains.ownerFinal, original) + ) + throw Error("Actual original applied final evidence changed"); +} diff --git a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts new file mode 100644 index 000000000..d68b25b7c --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts @@ -0,0 +1,43 @@ +/** Shared non-secret original-source copies. Decoding grants no authority. */ +import { z } from "zod"; +import { configurationMaintenanceReceiptSchema } from "../ops/configuration-maintenance"; +import { type OpsJobStatus, parseOpsJobStatus } from "../ops/contract"; +import { executionRefSchema } from "../ops/execution-ref"; +import { configurationMaintenanceDecisionCommitmentSchema } from "./configuration-maintenance-owner-records"; + +export const configurationMaintenanceForwardDomain = + "openmapx-configuration-maintenance-forward-v1"; +const signed = z.strictObject({ + payload: z.unknown(), + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), +}); +const digest = z.string().regex(/^[a-f0-9]{64}$/); +export const configurationMaintenanceForwardGraphSchema = z.strictObject({ + version: z.literal(1), + binding: z.lazy(() => configurationMaintenanceReceiptSchema.shape.binding), + prepare: executionRefSchema, + apiJobId: z.uuid(), + parentOperationKey: z.string(), + parentFingerprint: digest, + parentDeadlineAt: z.number().int().positive().safe(), + planDigest: digest, + serviceIds: z.lazy(() => configurationMaintenanceReceiptSchema.shape.serviceIds), + steps: z.lazy(() => configurationMaintenanceReceiptSchema.shape.steps), +}); +export const configurationMaintenanceForwardSchema = z.strictObject({ + version: z.literal(1), + graph: configurationMaintenanceForwardGraphSchema, + rootAdmission: signed, + decision: z.lazy(() => configurationMaintenanceDecisionCommitmentSchema).nullable(), + generation: z + .strictObject({ + admission: signed, + terminal: signed, + effect: signed, + status: z + .unknown() + .transform((value) => parseOpsJobStatus(value) as OpsJobStatus) + .refine((value) => value.diagnostics === undefined), + }) + .nullable(), +}); diff --git a/packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts new file mode 100644 index 000000000..4bf5aea1b --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts @@ -0,0 +1,526 @@ +/** Private owner data schemas. Decoding does not create owner, inspection, + * decision or settlement authority. */ + +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { + configurationMaintenanceDecisionSchema, + configurationMaintenanceReceiptSchema, + configurationMaintenanceSettleSchema, +} from "../ops/configuration-maintenance"; +import { type OpsJobStatus, opsResourceId, parseOpsJobStatus } from "../ops/contract"; +import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; +import { + executionFingerprint, + executionRefSchema, + retainedParentExecutionSchema, +} from "../ops/execution-identity"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { ownedServiceSnapshotSchema } from "../ops/owned-service-records"; +import { + configurationNoGuardIntentSchema, + configurationSettlementOwnerIdentitySchema, + signedConfigurationSettlementCapacitySchema, +} from "./configuration-fence-settlement-records"; +import { + configurationMaintenanceFinalRecordSchema, + signedConfigurationMaintenanceRuntimeInspectionSchema, +} from "./configuration-maintenance-final-records"; +import { controlOriginSchema } from "./control-records"; +import { + coordinatorPullWitnessSchema, + stagedCoordinatorObservationSchema, +} from "./coordinator-pull-records"; +import { dataConfigurationFencePreparedWitnessSchema } from "./data-configuration-fence-witness-records"; +import { + assertFixedConfigurationMaintenancePlan, + fixedConfigurationMaintenancePlanSchema, +} from "./fixed-configuration-maintenance"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/), + sequence = z.number().int().nonnegative().safe(); +export const configurationMaintenanceControlSnapshotSchema = z.strictObject({ + preflight: z.lazy(() => configurationMaintenanceReceiptSchema.shape.preflight), + sequence: z.lazy(() => configurationMaintenanceReceiptSchema.shape.sequence), + state: z.lazy(() => configurationMaintenanceReceiptSchema.shape.state), + decision: z.lazy(() => configurationMaintenanceReceiptSchema.shape.decision), + dmFence: z.lazy(() => configurationMaintenanceReceiptSchema.shape.dmFence), + generation: z.lazy(() => configurationMaintenanceReceiptSchema.shape.generation), + appliedRevisionId: z.lazy(() => configurationMaintenanceReceiptSchema.shape.appliedRevisionId), + followUp: z.lazy(() => configurationMaintenanceReceiptSchema.shape.followUp), +}); +export const configurationMaintenanceDecisionCommitmentDomain = + "openmapx-configuration-maintenance-decision-v1"; +/** Produced in the actual original-owner authenticated decision CAS. This + * non-secret attestation is copied verbatim by the fixed shared publisher. */ +export const configurationMaintenanceDecisionCommitmentPayloadSchema = z.strictObject({ + version: z.literal(1), + identity: z.lazy(() => configurationSettlementOwnerIdentitySchema), + operation: z.lazy(() => configurationMaintenanceDecisionSchema), + execution: executionRefSchema, + apiParent: z + .lazy(() => retainedParentExecutionSchema) + .refine((parent) => parent.issuer === "api"), + admittedAt: sequence, + requestedDeadlineAt: sequence, + acceptedAt: sequence, + requestVersion: z.union([z.literal(1), z.literal(2)]), + deploymentBudgetMs: z.number().int().min(1).max(60000), + result: configurationMaintenanceControlSnapshotSchema, + canonicalDigest: digest, +}); +export const configurationMaintenanceDecisionCommitmentSchema = z.strictObject({ + payload: configurationMaintenanceDecisionCommitmentPayloadSchema, + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), +}); +/** The original owner's finite metadata control. This ExecutionRef is ONLY + * journal/receipt correlation; it is never admitted as a forward execution, + * target owner/borrower or a separate origin/family. */ +export const configurationMaintenanceDecisionSourceSchema = z.strictObject({ + version: z.literal(1), + operation: z.lazy(() => configurationMaintenanceDecisionSchema), + execution: executionRefSchema, + apiParent: z + .lazy(() => retainedParentExecutionSchema) + .refine((parent) => parent.issuer === "api"), + parentAssertion: z.string().min(1).max(4096), + admittedAt: sequence, + requestedDeadlineAt: sequence, + acceptedAt: sequence, + requestVersion: z.union([z.literal(1), z.literal(2)]), + deploymentBudgetMs: z.number().int().min(1).max(60000), + result: configurationMaintenanceControlSnapshotSchema, + canonicalDigest: digest, + commitment: configurationMaintenanceDecisionCommitmentSchema, + receiptDelivery: z + .strictObject({ + terminalCopyId: z.uuid(), + checkpointOwnerVersion: sequence, + proofDigest: digest, + }) + .nullable(), +}); +export const configurationMaintenanceStepControlSourceSchema = z.strictObject({ + version: z.literal(1), + ordinal: z.number().int().min(0).max(31), + execution: executionRefSchema, + apiParent: z + .lazy(() => retainedParentExecutionSchema) + .refine((parent) => parent.issuer === "api"), + parentAssertion: z.string().min(1).max(4096), + requestedDeadlineAt: sequence, + receivedAt: sequence, + grant: z + .strictObject({ leaseId: z.uuid(), delegationId: z.uuid(), grantDigest: digest }) + .nullable(), + canonical: z + .strictObject({ + status: z + .unknown() + .transform((value) => parseOpsJobStatus(value) as OpsJobStatus) + .refine((value) => value.diagnostics === undefined), + terminalDigest: digest, + terminalAt: sequence, + effectSettlementDigest: digest, + }) + .nullable(), + receiptDelivery: z + .strictObject({ + terminalCopyId: z.uuid(), + checkpointOwnerVersion: sequence, + proofDigest: digest, + }) + .nullable(), +}); +export const configurationMaintenanceSettleControlSourceSchema = z.strictObject({ + version: z.literal(1), + operation: z.lazy(() => configurationMaintenanceSettleSchema), + execution: executionRefSchema, + apiParent: z + .lazy(() => retainedParentExecutionSchema) + .refine((parent) => parent.issuer === "api"), + parentAssertion: z.string().min(1).max(4096), + admittedAt: sequence, + requestedDeadlineAt: sequence, + acceptedAt: sequence, + requestVersion: z.union([z.literal(1), z.literal(2)]), + deploymentBudgetMs: z.number().int().min(1).max(60000), + checkedSequence: sequence, + requestEvidenceDigest: digest, + completedAt: sequence.nullable(), + result: configurationMaintenanceControlSnapshotSchema.nullable(), + canonicalDigest: digest.nullable(), + receiptDelivery: z + .strictObject({ + terminalCopyId: z.uuid(), + checkpointOwnerVersion: sequence, + proofDigest: digest, + }) + .nullable(), +}); +export const configurationMaintenanceOwnerRecordSchema = z.strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + admissionDigest: digest, + rootCertificateDigest: digest, + apiParentAttestationDigest: digest, + plan: z.lazy(() => fixedConfigurationMaintenancePlanSchema), + originalContainers: z.array(ownedServiceSnapshotSchema).max(256), + coordinatorPull: coordinatorPullWitnessSchema.optional(), + stagedCoordinatorProvenance: stagedCoordinatorObservationSchema.optional(), + sequence, + receipt: z.lazy(() => configurationMaintenanceReceiptSchema), + guardSource: z.lazy(() => dataConfigurationFencePreparedWitnessSchema).nullable(), + capacity: z.lazy(() => signedConfigurationSettlementCapacitySchema).nullable(), + apiControls: z.strictObject({ + decide: configurationMaintenanceDecisionSourceSchema.nullable(), + settle: configurationMaintenanceSettleControlSourceSchema.nullable(), + steps: z + .array(configurationMaintenanceStepControlSourceSchema) + .max(32) + .refine((values) => + values.every((value, index) => index === 0 || values[index - 1]!.ordinal < value.ordinal), + ), + }), + // Only the actual fixed final/inspection producers may install these states. + final: z.lazy(() => configurationMaintenanceFinalRecordSchema).nullable(), + inspection: z.lazy(() => signedConfigurationMaintenanceRuntimeInspectionSchema).nullable(), + issuance: z.enum(["open", "closing_without_guard", "closed_without_guard"]), + noGuard: z.lazy(() => configurationNoGuardIntentSchema).nullable(), +}); +export type ConfigurationMaintenanceOwnerRecord = z.infer< + typeof configurationMaintenanceOwnerRecordSchema +>; +/** Pure release-publication preimage, not proof of release. Callers MUST + * independently verify the retained original physical release/full closure. + * There is no new timestamp, outcome, inspection or control identity here. */ +export function configurationMaintenanceReleasedOwner( + owner: ConfigurationMaintenanceOwnerRecord, +): ConfigurationMaintenanceOwnerRecord { + const retained = owner.apiControls.settle; + if ( + owner.receipt.state !== "settling" || + owner.receipt.dmFence?.state !== "settled" || + !owner.guardSource || + owner.noGuard || + owner.issuance !== "open" || + !owner.final || + !owner.inspection || + (retained && + (retained.result !== null || + retained.completedAt !== null || + retained.canonicalDigest !== null)) + ) + throw Error("Original released owner transition required"); + const sequence = owner.sequence + 1; + return configurationMaintenanceOwnerRecordSchema.parse({ + ...owner, + sequence, + receipt: { ...owner.receipt, sequence, state: "settled" }, + }); +} +/** Pure deterministic completion preimage. Callers must independently verify + * the original immutable final, full closure and retained physical release. */ +export function configurationMaintenanceSettleCompletion( + owner: ConfigurationMaintenanceOwnerRecord, + releasedAt: number, +): ConfigurationMaintenanceOwnerRecord { + const retained = owner.apiControls.settle; + if ( + !retained || + owner.receipt.state !== "settled" || + !Number.isSafeInteger(releasedAt) || + releasedAt < retained.acceptedAt + ) + throw Error("Original released settle completion required"); + const { preflight, sequence, state, decision, dmFence, generation, appliedRevisionId, followUp } = + owner.receipt, + result = configurationMaintenanceControlSnapshotSchema.parse({ + preflight, + sequence, + state, + decision, + dmFence, + generation, + appliedRevisionId, + followUp, + }), + status: OpsJobStatus = { + version: 2, + admittedAt: retained.admittedAt, + deadlineAt: retained.execution.deadlineAt, + fingerprint: retained.execution.fingerprint, + operationId: retained.execution.executionId, + operationKey: retained.execution.operationKey, + kind: "configurationMaintenance.settle", + resourceId: opsResourceId(retained.operation), + state: "succeeded", + submittedAt: new Date(retained.admittedAt).toISOString(), + updatedAt: new Date(releasedAt).toISOString(), + result: { ...owner.receipt, ...result }, + }, + completed = configurationMaintenanceSettleControlSourceSchema.parse({ + ...retained, + completedAt: releasedAt, + result, + canonicalDigest: controlTerminalReceiptDigest(status), + }); + if ( + retained.result && + (canonicalJson(retained.result) !== canonicalJson(completed.result) || + retained.completedAt !== completed.completedAt || + retained.canonicalDigest !== completed.canonicalDigest) + ) + throw Error("Original settle completion preimage changed"); + return { ...owner, apiControls: { ...owner.apiControls, settle: completed } }; +} +export function configurationMaintenanceDecisionCommitmentPayload( + owner: ConfigurationMaintenanceOwnerRecord, + source: Pick< + z.infer, + | "operation" + | "execution" + | "apiParent" + | "admittedAt" + | "requestedDeadlineAt" + | "acceptedAt" + | "requestVersion" + | "deploymentBudgetMs" + | "result" + | "canonicalDigest" + >, +) { + return configurationMaintenanceDecisionCommitmentPayloadSchema.parse({ + version: 1, + identity: { + installationId: owner.installationId, + origin: owner.origin, + ownerPrepare: owner.plan.prepare, + admissionDigest: owner.admissionDigest, + rootCertificateDigest: owner.rootCertificateDigest, + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + planDigest: owner.plan.planDigest, + settlementId: owner.plan.dmSettle.settlementId, + }, + operation: source.operation, + execution: source.execution, + apiParent: source.apiParent, + admittedAt: source.admittedAt, + requestedDeadlineAt: source.requestedDeadlineAt, + acceptedAt: source.acceptedAt, + requestVersion: source.requestVersion, + deploymentBudgetMs: source.deploymentBudgetMs, + result: source.result, + canonicalDigest: source.canonicalDigest, + }); +} +/** Consistency only. Callers MUST first authenticate the actual indexed private + * owner with its pinned role key; parsing or calling this creates no handle. */ +export function assertConfigurationMaintenanceDecisionSource( + owner: ConfigurationMaintenanceOwnerRecord, +): void { + assertFixedConfigurationMaintenancePlan(owner.plan, owner.installationId); + const source = owner.apiControls.decide; + if (!source) { + if ( + owner.receipt.decision !== null || + owner.apiControls.steps.length || + owner.apiControls.settle + ) + throw Error("Original authenticated decision source required"); + return; + } + const body = source.operation, + parent = source.apiParent, + original = owner.plan.apiParent, + result = source.result; + if ( + canonicalJson(source.commitment.payload) !== + canonicalJson(configurationMaintenanceDecisionCommitmentPayload(owner, source)) + ) + throw Error("Original signed decision commitment preimage changed"); + if ( + parent.issuer !== "api" || + original.issuer !== "api" || + !owner.guardSource || + owner.noGuard || + body.maintenanceId !== owner.plan.operation.maintenanceId || + body.candidateId !== owner.plan.operation.candidateId || + body.prepareOperationKey !== owner.plan.prepare.operationKey || + body.prepareFingerprint !== owner.plan.prepare.operationFingerprint || + body.planDigest !== owner.plan.planDigest || + body.expectedSequence + 1 !== result.sequence || + owner.sequence < result.sequence || + body.checkpointVersion < 1 || + body.checkpointVersion > parent.apiOwnerVersion || + parent.apiJobId !== original.apiJobId || + parent.operationKey !== original.operationKey || + parent.fingerprint !== original.fingerprint || + parent.deadlineAt !== original.deadlineAt || + createHash("sha256").update(source.parentAssertion).digest("hex") !== parent.assertionDigest || + source.execution.executor !== "ops-agent" || + source.execution.principal !== "api" || + source.execution.operationFingerprint !== opsOperationFingerprint(body) || + source.execution.fingerprint !== + executionFingerprint({ + kind: body.kind, + operationFingerprint: source.execution.operationFingerprint, + deadlineAt: source.execution.deadlineAt, + parent, + }) || + source.execution.deadlineAt <= source.admittedAt || + source.execution.deadlineAt !== + Math.min( + source.requestedDeadlineAt, + body.decision === "continue" ? parent.deadlineAt : Infinity, + source.admittedAt + source.deploymentBudgetMs, + ) || + source.acceptedAt < source.admittedAt || + source.acceptedAt >= source.execution.deadlineAt || + result.state !== "authorized" || + result.preflight !== (body.decision === "abandon" ? "unavailable" : "current") || + result.generation !== "not_started" || + result.appliedRevisionId !== null || + result.followUp !== "not_started" || + canonicalJson(result.dmFence) !== canonicalJson(owner.guardSource.preparedReceipt) || + canonicalJson(result.decision) !== + canonicalJson({ + decisionId: body.decisionId, + decision: body.decision, + checkpointVersion: body.checkpointVersion, + checkpointDigest: body.checkpointDigest, + }) || + canonicalJson(owner.receipt.decision) !== canonicalJson(result.decision) || + (source.receiptDelivery && + source.receiptDelivery.checkpointOwnerVersion < parent.apiOwnerVersion) + ) + throw Error("Original authenticated decision source changed"); + const receipt = configurationMaintenanceReceiptSchema.parse({ ...owner.receipt, ...result }); + if ( + source.canonicalDigest !== + controlTerminalReceiptDigest({ + version: 2, + admittedAt: source.admittedAt, + deadlineAt: source.execution.deadlineAt, + fingerprint: source.execution.fingerprint, + operationId: source.execution.executionId, + operationKey: source.execution.operationKey, + kind: body.kind, + resourceId: opsResourceId(body), + state: "succeeded", + submittedAt: new Date(source.admittedAt).toISOString(), + updatedAt: new Date(source.acceptedAt).toISOString(), + result: receipt, + }) + ) + throw Error("Original decision canonical source changed"); + for (const control of owner.apiControls.steps) { + const step = owner.plan.steps[control.ordinal], + actor = control.apiParent, + original = owner.plan.apiParent, + parent = { + issuer: "ops-agent" as const, + operationKey: owner.plan.prepare.operationKey, + fingerprint: owner.plan.prepare.fingerprint, + deadlineAt: owner.plan.prepare.deadlineAt, + execution: owner.plan.prepare, + }; + if ( + !step || + source.operation.decision !== "continue" || + !source.receiptDelivery || + actor.issuer !== "api" || + original.issuer !== "api" || + actor.apiJobId !== original.apiJobId || + actor.operationKey !== original.operationKey || + actor.fingerprint !== original.fingerprint || + actor.deadlineAt !== original.deadlineAt || + createHash("sha256").update(control.parentAssertion).digest("hex") !== + actor.assertionDigest || + control.requestedDeadlineAt !== owner.plan.prepare.deadlineAt || + control.execution.executor !== "ops-agent" || + control.execution.principal !== "api" || + control.execution.operationKey !== step.operationKey || + control.execution.operationFingerprint !== step.operationFingerprint || + control.execution.deadlineAt > owner.plan.prepare.deadlineAt || + control.execution.deadlineAt <= control.receivedAt || + control.execution.fingerprint !== + executionFingerprint({ + kind: step.operation.kind, + operationFingerprint: step.operationFingerprint, + deadlineAt: control.execution.deadlineAt, + parent, + }) || + (control.receiptDelivery && + (!control.grant || + !control.canonical || + control.receiptDelivery.checkpointOwnerVersion < actor.apiOwnerVersion)) + ) + throw Error("Original purpose-bound frozen maintenance step changed"); + if (control.canonical) { + const canonical = control.canonical, + status = canonical.status; + if ( + !control.grant || + status.state !== "succeeded" || + status.kind !== step.operation.kind || + status.operationId !== control.execution.executionId || + status.operationKey !== control.execution.operationKey || + status.fingerprint !== control.execution.fingerprint || + status.deadlineAt !== control.execution.deadlineAt || + canonical.terminalDigest !== controlTerminalReceiptDigest(status) + ) + throw Error("Original maintenance child canonical source changed"); + } + } + const settle = owner.apiControls.settle; + if (settle) { + const body = settle.operation, + parent = settle.apiParent; + if ( + parent.issuer !== "api" || + body.maintenanceId !== owner.plan.operation.maintenanceId || + body.candidateId !== owner.plan.operation.candidateId || + body.prepareOperationKey !== owner.plan.prepare.operationKey || + body.prepareFingerprint !== owner.plan.prepare.operationFingerprint || + body.planDigest !== owner.plan.planDigest || + body.decisionId !== owner.receipt.decision?.decisionId || + body.expectedSequence !== settle.checkedSequence || + body.evidenceDigest !== settle.requestEvidenceDigest || + owner.sequence <= settle.checkedSequence || + parent.apiJobId !== original.apiJobId || + parent.operationKey !== original.operationKey || + parent.fingerprint !== original.fingerprint || + parent.deadlineAt !== original.deadlineAt || + createHash("sha256").update(settle.parentAssertion).digest("hex") !== + parent.assertionDigest || + settle.execution.executor !== "ops-agent" || + settle.execution.principal !== "api" || + settle.execution.operationFingerprint !== opsOperationFingerprint(body) || + settle.execution.fingerprint !== + executionFingerprint({ + kind: body.kind, + operationFingerprint: settle.execution.operationFingerprint, + deadlineAt: settle.execution.deadlineAt, + parent, + }) || + settle.execution.deadlineAt !== + Math.min(settle.requestedDeadlineAt, settle.admittedAt + settle.deploymentBudgetMs) || + settle.acceptedAt < settle.admittedAt || + settle.acceptedAt >= settle.execution.deadlineAt || + (settle.result === null && + (settle.completedAt !== null || + settle.canonicalDigest !== null || + settle.receiptDelivery !== null)) || + (settle.result !== null && + (settle.completedAt === null || + settle.canonicalDigest === null || + settle.completedAt < settle.acceptedAt || + settle.result.state !== "settled" || + settle.result.dmFence?.state !== "settled")) + ) + throw Error("Original accepted settlement request source changed"); + } +} diff --git a/packages/core/src/build-artifacts/configuration-no-guard.ts b/packages/core/src/build-artifacts/configuration-no-guard.ts new file mode 100644 index 000000000..dbacd90a5 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-no-guard.ts @@ -0,0 +1,462 @@ +/** Cross-role NoGuard source observations. These handles verify original + * publications; they grant no forward operation or physical lease. */ + +import { configurationMaintenancePrepareSchema } from "../ops/configuration-maintenance"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { lookupControlChildPhaseAttempt, lookupControlChildPhaseBinding } from "./child-phase"; +import { + configurationSettlementRecordDescriptors, + readConfigurationSettlementCell, +} from "./configuration-fence-settlement-cells"; +import { + configurationDmCapacityCellSchema, + configurationDmNoGuardRefusalSchema, + configurationSettlementCapacitySchema, + configurationSettlementControlCellSchema, + configurationSettlementDomains, + configurationSettlementReservationProjectionSchema, + configurationUnusedCapacityConfirmationSchema, + configurationUnusedCapacityIntentSchema, + deriveConfigurationFenceSettlementId, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import { configurationMaintenanceOwnerRecordSchema } from "./configuration-maintenance-owner-records"; +import { + executionAuthorityState, + readControlExecutionWitness, + readPrivateRecord, + type VerifiedExecutionAuthority, +} from "./control-authority"; +import { controlDigest, verifyControl } from "./control-records"; +import { AnchoredDirectory } from "./filesystem"; +import { assertFixedConfigurationMaintenancePlan } from "./fixed-configuration-maintenance"; +import { + retainedConfigurationMaintenanceFamilyMember, + verifyFamilyOwnedRecord, + verifyPublishedUnusedDmConfigurationSettlementSlots, +} from "./retention-family"; +import { + closeArtifactTargetLeaseHandle, + configurationMaintenanceTargetOwner, + inspectArtifactTargetLeaseChildSettlement, + restoreArtifactTargetLeaseForSettlement, +} from "./target-lease"; + +function freeze(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); + } + return value; +} +function withObservedOwner( + authority: VerifiedExecutionAuthority, + run: (source: { + state: ReturnType; + owner: ReturnType; + ownerRecordDigest: string; + projection: ReturnType; + projectionDigest: string; + prepare: NonNullable>; + lease: ReturnType; + }) => T, +): T { + const state = executionAuthorityState(authority); + if ( + state.kind !== "configurationMaintenance.prepare" || + state.installation.installation.role !== "ops-agent" || + state.installation.fixedChildId || + !state.origin + ) + throw Error("Original maintenance NoGuard observer required"); + const admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: unknown; parent?: unknown } }, + operation = configurationMaintenancePrepareSchema.parse(admitted.payload?.operation), + name = `${operation.maintenanceId}.json`, + raw = readPrivateRecord(state.installation, "configuration-maintenance", name), + owner = verifyControl( + state.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "configuration-maintenance", name, container: "" }, + canonicalJson(raw), + ); + assertFixedConfigurationMaintenancePlan( + owner.plan, + state.installation.installation.installationId, + ); + if ( + owner.admissionDigest !== state.receiptDigest || + canonicalJson(owner.origin) !== canonicalJson(state.origin) || + canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) || + canonicalJson(owner.plan.operation) !== canonicalJson(operation) || + canonicalJson(owner.plan.apiParent) !== canonicalJson(admitted.payload?.parent) || + canonicalJson(owner.plan.reservations) !== canonicalJson(state.reservations) || + opsOperationFingerprint(operation) !== state.execution.operationFingerprint || + owner.guardSource || + owner.receipt.dmFence || + owner.receipt.decision?.decision === "continue" || + owner.receipt.generation !== "not_started" + ) + throw Error("Original never-prepared owner source required"); + const lease = restoreArtifactTargetLeaseForSettlement(authority), + opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = configurationMaintenanceTargetOwner(lease); + if ( + root.certificateDigest !== owner.rootCertificateDigest || + canonicalJson(root.serviceIds) !== canonicalJson(owner.plan.serviceIds) + ) + throw Error("Original NoGuard physical root changed"); + const data = own(new AnchoredDirectory(state.installation.dataRoot)), + control = own(data.directory(".admin-target-control")), + executions = own(control.directory("executions")), + role = own(executions.directory("ops-agent")), + cell = verifyControl( + state.installation.installation.publicKey, + configurationSettlementDomains.control, + JSON.parse( + role + .read(`${state.execution.operationKey}.configuration-settlement-control.json`, 8192) + .toString("utf8"), + ), + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + state.installation.installation.publicKey, + configurationSettlementDomains.signedProjection, + cell.projection, + configurationSettlementReservationProjectionSchema, + ), + dmPrepare = owner.plan.reservations.find((value) => value.phase === "configuration-fence"), + settle = owner.plan.reservations.find( + (value) => value.phase === "configuration-fence-settle", + ); + if ( + !dmPrepare || + !settle || + cell.binding || + projection.rootAdmissionDigest !== state.receiptDigest || + projection.rootCertificateDigest !== root.certificateDigest || + canonicalJson(projection.origin) !== canonicalJson(state.origin) || + canonicalJson(projection.ownerPrepare) !== canonicalJson(state.execution) || + projection.maintenanceId !== operation.maintenanceId || + projection.candidateId !== operation.candidateId || + projection.planDigest !== owner.plan.planDigest || + canonicalJson(projection.reservation) !== canonicalJson(settle) || + canonicalJson(projection.operation) !== + canonicalJson({ kind: "configuration-maintenance.settle", ...owner.plan.dmSettle }) || + projection.settlementId !== + deriveConfigurationFenceSettlementId({ + installationId: owner.installationId, + ownerPrepare: state.execution, + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + planDigest: owner.plan.planDigest, + }) || + canonicalJson(projection.issuerMembership) !== + canonicalJson( + retainedConfigurationMaintenanceFamilyMember( + state.installation, + state.origin, + state.execution, + ), + ) + ) + throw Error("Original NoGuard reservation preimage changed"); + const descriptors = configurationSettlementRecordDescriptors( + projection, + dmPrepare.execution.operationKey, + ); + readConfigurationSettlementCell(state.installation, projection, descriptors.sender[2]!); + const prepare = lookupControlChildPhaseBinding(authority, dmPrepare), + attempted = lookupControlChildPhaseAttempt(authority, dmPrepare); + if ( + !prepare || + !attempted || + attempted.channel !== "delegation" || + canonicalJson(prepare.child.execution) !== canonicalJson(attempted.execution) + ) + throw Error("Original issued normal DM PREPARE required"); + return run({ + state, + owner, + ownerRecordDigest: controlDigest(raw), + projection, + projectionDigest: settlementRecordDigest( + configurationSettlementDomains.projection, + projection, + ), + prepare, + lease, + }); + } finally { + for (const directory of opened.reverse()) directory.close(); + closeArtifactTargetLeaseHandle(lease); + } +} +function dmCell(source: Parameters[1]>[0]) { + const descriptors = configurationSettlementRecordDescriptors( + source.projection, + source.prepare.child.execution.operationKey, + ).receiver, + opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(source.state.installation.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + role = own(executions.directory("data-manager")), + bytes = role.read(descriptors[0]!.name, 8192); + if (bytes.length !== 8192) throw Error("Original DM compound NoGuard channel width changed"); + return { + descriptors, + cell: configurationDmCapacityCellSchema.parse(JSON.parse(bytes.toString("utf8"))), + }; + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +declare const refusalBrand: unique symbol; +export type VerifiedPublishedDmNoGuardRefusal = Readonly<{ [refusalBrand]: true }>; +const refusals = new WeakMap< + VerifiedPublishedDmNoGuardRefusal, + { + authority: VerifiedExecutionAuthority; + ownerRecordDigest: string; + projectionDigest: string; + projection: ReturnType; + signedRefusal: NonNullable< + ReturnType["refusal"] + >; + signedCapacity: ReturnType["capacity"]; + } +>(); +export function publishedDmNoGuardRefusalState( + value: VerifiedPublishedDmNoGuardRefusal, + authority: VerifiedExecutionAuthority, +) { + const source = refusals.get(value); + if (!source || source.authority !== authority) + throw Error("Actual original DM refusal source required"); + executionAuthorityState(authority); + return source; +} +export function verifyPublishedDataConfigurationFenceRefusal( + authority: VerifiedExecutionAuthority, +): VerifiedPublishedDmNoGuardRefusal { + return withObservedOwner(authority, (source) => { + const { cell, descriptors } = dmCell(source), + { projection, state } = source, + execution = source.prepare.child.execution; + if (!cell.refusal) throw Error("Original signed DM refusal publication required"); + const refusal = verifyControl( + state.installation.publicKeys["data-manager"], + configurationSettlementDomains.refusal, + cell.refusal, + configurationDmNoGuardRefusalSchema, + ), + witness = readControlExecutionWitness( + state.installation, + execution, + state.installation.publicKeys["data-manager"], + ), + child = inspectArtifactTargetLeaseChildSettlement(source.lease, execution); + if ( + refusal.installationId !== projection.installationId || + canonicalJson(refusal.origin) !== canonicalJson(projection.origin) || + canonicalJson(refusal.ownerPrepare) !== canonicalJson(projection.ownerPrepare) || + refusal.rootAdmissionDigest !== projection.rootAdmissionDigest || + refusal.rootCertificateDigest !== projection.rootCertificateDigest || + refusal.maintenanceId !== projection.maintenanceId || + refusal.candidateId !== projection.candidateId || + refusal.planDigest !== projection.planDigest || + refusal.settlementId !== projection.settlementId || + refusal.projectionDigest !== source.projectionDigest || + refusal.reservationDigest !== projection.reservationDigest || + canonicalJson(refusal.prepareExecution) !== canonicalJson(execution) || + witness.kind !== "configuration-maintenance.prepare" || + witness.authorityDigest !== refusal.prepareAdmissionDigest || + canonicalJson(witness.origin) !== canonicalJson(projection.origin) || + !child || + child.outcome !== "failed_without_effect" || + child.publication !== "not_started" || + child.terminalReceiptDigest !== refusal.effectReceiptDigest || + refusal.effectReceiptDigest !== controlDigest(refusal.refusalReceipt) || + refusal.effectSettlementDigest !== + controlDigest({ ...child, receipt: refusal.refusalReceipt }) || + refusal.canonicalTerminalDigest !== + controlDigest({ version: 1, refused: refusal.refusalReceipt }) + ) + throw Error("Original DM canonical refusal/child preimages changed"); + if (cell.capacity) { + const capacity = verifyControl( + state.installation.publicKeys["data-manager"], + configurationSettlementDomains.capacity, + cell.capacity, + configurationSettlementCapacitySchema, + ); + if ( + capacity.installationId !== projection.installationId || + canonicalJson(capacity.origin) !== canonicalJson(projection.origin) || + capacity.ownerPrepareDigest !== controlDigest(projection.ownerPrepare) || + capacity.settlementId !== projection.settlementId || + capacity.projectionDigest !== source.projectionDigest || + capacity.reservationDigest !== projection.reservationDigest || + canonicalJson(capacity.proposedExecutionAddress) !== + canonicalJson(projection.reservation.execution) || + capacity.recordDescriptorsDigest !== controlDigest(descriptors) || + capacity.maxBytes !== descriptors.reduce((sum, descriptor) => sum + descriptor.cap, 0) || + refusal.capacityReceiptDigest !== + settlementRecordDigest(configurationSettlementDomains.capacity, capacity) + ) + throw Error("Original DM refusal capacity preimage changed"); + } else if ( + refusal.capacityReceiptDigest !== null || + refusal.refusalReceipt.reason !== "capacity" + ) + throw Error("Original capacity-null refusal required"); + const value = Object.freeze({}) as VerifiedPublishedDmNoGuardRefusal; + refusals.set( + value, + Object.freeze({ + authority, + ownerRecordDigest: source.ownerRecordDigest, + projectionDigest: source.projectionDigest, + projection: freeze(projection), + signedRefusal: freeze(cell.refusal), + signedCapacity: cell.capacity && freeze(cell.capacity), + }), + ); + return value; + }); +} +declare const unusedBrand: unique symbol; +export type VerifiedDmUnusedCapacity = Readonly<{ [unusedBrand]: true }>; +const unusedProofs = new WeakMap< + VerifiedDmUnusedCapacity, + { + authority: VerifiedExecutionAuthority; + refusal: VerifiedPublishedDmNoGuardRefusal; + signedUnused: NonNullable["unused"]>; + intentDigest: string; + confirmationDigest: string; + } +>(); +export function publishedDmUnusedCapacityState( + value: VerifiedDmUnusedCapacity, + authority: VerifiedExecutionAuthority, +) { + const source = unusedProofs.get(value); + if (!source || source.authority !== authority) + throw Error("Actual original DM UNUSED source required"); + publishedDmNoGuardRefusalState(source.refusal, authority); + return source; +} +export function verifyPublishedUnusedConfigurationCapacity( + authority: VerifiedExecutionAuthority, + verifiedRefusal: VerifiedPublishedDmNoGuardRefusal, +): VerifiedDmUnusedCapacity { + const original = publishedDmNoGuardRefusalState(verifiedRefusal, authority); + return withObservedOwner(authority, (source) => { + const { cell, descriptors } = dmCell(source), + projection = source.projection, + key = source.state.installation.publicKeys["data-manager"]; + if ( + !cell.unused?.confirmation || + canonicalJson(cell.refusal) !== canonicalJson(original.signedRefusal) || + canonicalJson(cell.capacity) !== canonicalJson(original.signedCapacity) + ) + throw Error("Original confirmed DM UNUSED packet required"); + const intent = verifyControl( + key, + configurationSettlementDomains.unusedIntent, + cell.unused.intent, + configurationUnusedCapacityIntentSchema, + ), + confirmation = verifyControl( + key, + configurationSettlementDomains.unusedConfirmation, + cell.unused.confirmation, + configurationUnusedCapacityConfirmationSchema, + ), + capacity = cell.capacity?.payload, + expectedDescriptors = capacity ? descriptors : [descriptors[0]!], + intentDigest = settlementRecordDigest(configurationSettlementDomains.unusedIntent, intent), + expected = { + executionOrdinal: capacity?.reservedExecutionOrdinal ?? null, + intentOrdinal: capacity?.reservedIntentOrdinal ?? null, + recordDescriptorsDigest: controlDigest(descriptors), + allocatedDescriptorDigests: expectedDescriptors.map((value) => controlDigest(value)).sort(), + state: "UNUSED", + intentDigest, + }; + if ( + intent.installationId !== projection.installationId || + canonicalJson(intent.origin) !== canonicalJson(projection.origin) || + intent.role !== "data-manager" || + intent.ownerPrepareDigest !== controlDigest(projection.ownerPrepare) || + intent.prepareExecutionDigest !== controlDigest(source.prepare.child.execution) || + intent.settlementId !== projection.settlementId || + intent.projectionDigest !== source.projectionDigest || + intent.capacityReceiptDigest !== original.signedRefusal.payload.capacityReceiptDigest || + intent.sourceProofDigest !== + settlementRecordDigest( + configurationSettlementDomains.refusal, + original.signedRefusal.payload, + ) || + intent.executionOrdinal !== expected.executionOrdinal || + intent.intentOrdinal !== expected.intentOrdinal || + intent.recordDescriptorsDigest !== expected.recordDescriptorsDigest || + canonicalJson(intent.allocatedDescriptorDigests) !== + canonicalJson(expected.allocatedDescriptorDigests) || + confirmation.installationId !== projection.installationId || + canonicalJson(confirmation.origin) !== canonicalJson(projection.origin) || + confirmation.role !== "data-manager" || + confirmation.ownerPrepareDigest !== intent.ownerPrepareDigest || + confirmation.settlementId !== projection.settlementId || + confirmation.projectionDigest !== source.projectionDigest || + confirmation.intentDigest !== intentDigest || + confirmation.slotStateDigest !== controlDigest(expected) || + confirmation.confirmedAt < intent.createdAt + ) + throw Error("Original DM UNUSED source preimages changed"); + verifyPublishedUnusedDmConfigurationSettlementSlots(source.state.installation, projection, { + executionOrdinal: intent.executionOrdinal, + intentOrdinal: intent.intentOrdinal, + projectionDigest: intent.projectionDigest, + intentDigest, + }); + const value = Object.freeze({}) as VerifiedDmUnusedCapacity; + unusedProofs.set( + value, + Object.freeze({ + authority, + refusal: verifiedRefusal, + signedUnused: freeze(cell.unused), + intentDigest, + confirmationDigest: settlementRecordDigest( + configurationSettlementDomains.unusedConfirmation, + confirmation, + ), + }), + ); + return value; + }); +} diff --git a/packages/core/src/build-artifacts/configuration-pre-effect.test.ts b/packages/core/src/build-artifacts/configuration-pre-effect.test.ts new file mode 100644 index 000000000..1584736f4 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-pre-effect.test.ts @@ -0,0 +1,259 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, it } from "vitest"; +import type { OpsOperation } from "../ops/contract"; +import { createControlRootOperationKey } from "../ops/control-root-key"; +import { executionFingerprint } from "../ops/execution-identity"; +import { opsOperationFingerprint } from "../ops/fingerprint"; +import { + bindControlChildPhase, + type ControlChildReservation, + issueControlChildPhase, +} from "./child-phase"; +import { + beginControlEffect, + closeControlInstallation, + controlChildOperationKey, + inspectConfigurationPreEffectRefusal, + openControlInstallation, + prepareFixedControlChild, + provisionControlInstallation, + requireControlJournalRetention, + restoreConfigurationPreEffectTerminal, + retainAdmittedControlExecution, + retainConfigurationPreEffectRefusal, + retainControlTerminalCommitment, + verifyControlEffectSettlement, +} from "./control-authority"; +import { + acquireArtifactTargetLease, + releaseArtifactTargetLease, + settleArtifactTargetLeaseChild, + verifyRetainedArtifactTargetRootRelease, +} from "./target-lease"; + +let root: string, installation: ReturnType; +const targets = ["data-manager", "motis"], + revision = `cfg1_${"a".repeat(43)}`; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-configuration-pre-effect-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: targets, + }); +}); +afterEach(() => { + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); +}); +function admit(operation: OpsOperation, reserve = false, serviceIds = targets) { + const admittedAt = Date.now(), + deadlineAt = admittedAt + 120000, + operationFingerprint = opsOperationFingerprint(operation), + operationKey = createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomBytes(18).toString("base64url")}`, + operationKey, + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }, + reservations: ControlChildReservation[] = reserve + ? [ + { + version: 1, + phase: "lifecycle", + ordinal: 0, + kind: "service.update", + recipient: "fixed-cli", + serviceIds, + operationFingerprint, + execution: { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: controlChildOperationKey(operationKey, "lifecycle", 0), + }, + }, + ] + : []; + return retainAdmittedControlExecution(installation, { + operation, + kind: operation.kind, + execution, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds, + reservations, + }); +} +function start() { + return admit({ + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: revision, + }); +} +it("settles the actual unissued root and consumes only its indexed not-started effect", async () => { + const owner = start(), + lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: targets, + owner, + }); + retainConfigurationPreEffectRefusal(owner); + inspectConfigurationPreEffectRefusal(owner); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, owner.execution), + ); + await releaseArtifactTargetLease(lease); + verifyRetainedArtifactTargetRootRelease(owner); + expect(() => beginControlEffect(owner)).toThrow(); +}); +it.each(["begun", "bound", "issued"])( + "keeps an actual %s root fenced rather than claiming not-started", + async (boundary) => { + const owner = + boundary === "begun" + ? start() + : admit( + { + kind: "service.update", + serviceId: "motis", + expectedConfigurationRevisionId: revision, + }, + true, + ), + lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: targets, + owner, + }); + if (boundary === "begun") beginControlEffect(owner); + else { + await bindControlChildPhase(owner, { phase: "lifecycle", ordinal: 0 }); + if (boundary === "issued") + await issueControlChildPhase(owner, { + phase: "lifecycle", + ordinal: 0, + lease, + childPublicKey: prepareFixedControlChild(owner, randomUUID()), + }); + } + expect(() => retainConfigurationPreEffectRefusal(owner)).toThrow(); + expect(() => inspectConfigurationPreEffectRefusal(owner)).toThrow(); + expect(() => verifyRetainedArtifactTargetRootRelease(owner)).toThrow(); + expect(existsSync(join(root, "data/.admin-target-control/locks/data-manager"))).toBe(true); + }, +); +it("rejects a byte-identical not-started effect copied to a replacement inode", () => { + const owner = start(); + retainConfigurationPreEffectRefusal(owner); + const path = join( + root, + "control-private/ops-agent/effects", + `${owner.execution.operationKey}.json`, + ), + bytes = readFileSync(path); + renameSync(path, `${path}.original`); + writeFileSync(path, bytes, { mode: 0o600 }); + expect(() => inspectConfigurationPreEffectRefusal(owner)).toThrow(); +}); +it.each(["retain", "inspect"])( + "rejects a replaced indexed admission before pre-effect %s", + (phase) => { + const owner = start(); + if (phase === "inspect") retainConfigurationPreEffectRefusal(owner); + const path = join( + root, + "control-private/ops-agent/executions", + `${owner.execution.operationKey}.json`, + ), + bytes = readFileSync(path); + renameSync(path, `${path}.original`); + writeFileSync(path, bytes, { mode: 0o600 }); + expect(() => + phase === "retain" + ? retainConfigurationPreEffectRefusal(owner) + : inspectConfigurationPreEffectRefusal(owner), + ).toThrow(); + }, +); + +it.each(["exact", "different error", "missing", "copied commitment"] as const)( + "restores original pre-effect whole tuple only from %s commitment", + async (mode) => { + const owner = start(); + requireControlJournalRetention(owner); + retainConfigurationPreEffectRefusal(owner); + const admission = JSON.parse( + readFileSync( + join( + root, + "control-private/ops-agent/executions", + `${owner.execution.operationKey}.json`, + ), + "utf8", + ), + ).payload, + terminalAt = Date.now(), + status = { + version: 2 as const, + admittedAt: admission.admittedAt, + deadlineAt: owner.execution.deadlineAt, + fingerprint: owner.execution.fingerprint, + operationId: owner.execution.executionId, + operationKey: owner.execution.operationKey, + kind: "service.start" as const, + resourceId: "motis", + state: "failed" as const, + submittedAt: new Date(admission.admittedAt).toISOString(), + updatedAt: new Date(terminalAt).toISOString(), + errorClass: + mode === "different error" + ? ("runtime" as const) + : ("configuration_prerequisite" as const), + failureResult: {}, + }; + if (mode !== "missing") retainControlTerminalCommitment(owner, status, terminalAt); + if (mode === "copied commitment") { + const path = join( + root, + "control-private/ops-agent/observations", + `terminal-${owner.execution.operationKey}.json`, + ), + bytes = readFileSync(path); + renameSync(path, `${path}.original`); + writeFileSync(path, bytes, { mode: 0o600 }); + } + const observed = { + ...status, + errorClass: "recovery_required" as const, + updatedAt: new Date(terminalAt + 1000).toISOString(), + }; + if (mode === "exact") + expect( + restoreConfigurationPreEffectTerminal(installation, owner.execution, observed), + ).toEqual({ status, terminalAt }); + else + expect(() => + restoreConfigurationPreEffectTerminal(installation, owner.execution, observed), + ).toThrow(); + }, +); diff --git a/packages/core/src/build-artifacts/configuration-source-witness.test.ts b/packages/core/src/build-artifacts/configuration-source-witness.test.ts new file mode 100644 index 000000000..04e916d71 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-source-witness.test.ts @@ -0,0 +1,219 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, it } from "vitest"; +import { writeServiceSelectionFile } from "../services/deployment"; +import { + PRIVACY_BACKUP_COLLECTOR_IMAGE_ENV, + parseReleaseComposeSelection, + TRANSITOUS_TOOLS_IMAGE_ENV, +} from "../services/release-manifest"; +import type { LoadedService } from "../services/types"; +import { + observeConfigurationSources, + verifyConfigurationSourceWitness, +} from "./configuration-source-witness"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "./control-authority"; + +let root: string; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-source-witness-")); + mkdirSync(join(root, "data")); + await provisionControlInstallation(root); +}); +afterEach(() => rmSync(root, { recursive: true, force: true })); +it("retains the original private release selection instead of accepting a changed image pin", () => { + const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["data-manager"], + }); + try { + const pins = Object.fromEntries( + [ + "api", + "web", + "data-manager", + "ops-agent", + "transitous-runner", + "privacy-backup", + "transitous-tools", + ].map((name) => [name, `ghcr.io/openmapx/${name}@sha256:${"b".repeat(64)}`]), + ); + const document = { + services: { + "app-api": { + image: pins.api, + environment: { [PRIVACY_BACKUP_COLLECTOR_IMAGE_ENV]: pins["privacy-backup"] }, + }, + "app-web": { image: pins.web }, + "data-manager": { + image: pins["data-manager"], + environment: { [TRANSITOUS_TOOLS_IMAGE_ENV]: pins["transitous-tools"] }, + }, + "ops-agent": { + image: pins["ops-agent"], + environment: { [PRIVACY_BACKUP_COLLECTOR_IMAGE_ENV]: pins["privacy-backup"] }, + }, + "transitous-runner": { image: pins["transitous-runner"] }, + }, + "x-openmapx-release": { release: "public-release-fixture", images: pins }, + }; + const path = join(root, "docker-compose.release.yml"); + writeFileSync(path, JSON.stringify(document), { mode: 0o600 }); + expect(parseReleaseComposeSelection(JSON.stringify(document)).release).toBe( + "public-release-fixture", + ); + const options = { + installation, + infraDir: root, + services: [] as LoadedService[], + authorityRevision: "v1", + env: {}, + }, + observed = observeConfigurationSources(options), + verify = () => + verifyConfigurationSourceWitness({ + ...options, + sourceRevision: observed.sourceRevision, + expectedCurrentRevisionId: null, + }); + expect(Object.keys(observed).sort()).toEqual(["revisionId", "services", "sourceRevision"]); + verify(); + pins["data-manager"] = `ghcr.io/openmapx/data-manager@sha256:${"c".repeat(64)}`; + document.services["data-manager"].image = pins["data-manager"]; + writeFileSync(path, JSON.stringify(document), { mode: 0o600 }); + expect(() => verify()).toThrow("Host sources changed"); + } finally { + closeControlInstallation(installation); + } +}); +it("keeps host values private while detecting a changed persisted full selection", () => { + const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + try { + writeServiceSelectionFile(root, ["motis"]); + const options = { + installation, + infraDir: root, + services: [] as LoadedService[], + authorityRevision: "v1", + env: {}, + }; + const observed = observeConfigurationSources(options); + verifyConfigurationSourceWitness({ + ...options, + sourceRevision: observed.sourceRevision, + expectedCurrentRevisionId: null, + }); + writeServiceSelectionFile(root, []); + expect(() => + verifyConfigurationSourceWitness({ + ...options, + sourceRevision: observed.sourceRevision, + expectedCurrentRevisionId: null, + }), + ).toThrow("Host sources changed"); + } finally { + closeControlInstallation(installation); + } +}); +it("reports namespaced fields and the actual supported manifest memory token without values", () => { + const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + try { + writeFileSync(join(root, ".env"), "MOTIS_SLOT_MEMORY=8g\nSERVICE_MOTIS_WORKERS=77\n", { + mode: 0o600, + }); + const services: LoadedService[] = [ + { + directory: join(root, "motis"), + isBuiltIn: true, + enabled: true, + manifest: { + id: "motis", + name: "MOTIS", + version: "1.0.0", + quality: "built-in", + configSchema: { properties: { WORKERS: { type: "integer" } } }, + container: { image: "fixture/motis", tag: "1", memory: "${MOTIS_SLOT_MEMORY:-4g}" }, + }, + }, + ]; + const observed = observeConfigurationSources({ + installation, + infraDir: root, + services, + authorityRevision: "v1", + env: {}, + }); + expect(observed.services[0]?.fields).toContainEqual({ + key: "WORKERS", + source: "host", + envVar: "SERVICE_MOTIS_WORKERS", + }); + expect(observed.services[0]?.fields).toContainEqual({ + key: "OPENMAPX_MEMORY_MIB", + source: "host", + envVar: "MOTIS_SLOT_MEMORY", + }); + expect(JSON.stringify(observed)).not.toContain("8g"); + // Match the serialized value, not coincidental bytes inside its random source UUID. + expect(JSON.stringify(observed)).not.toContain('"77"'); + expect(JSON.stringify(observed)).not.toContain('"value"'); + } finally { + closeControlInstallation(installation); + } +}); +it("includes declared flat-schema host fields in the same private source witness", () => { + const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + try { + writeFileSync(join(root, ".env"), "SERVICE_MOTIS_WORKERS=77\n", { mode: 0o600 }); + const services: LoadedService[] = [ + { + directory: join(root, "motis"), + isBuiltIn: true, + enabled: true, + manifest: { + id: "motis", + name: "MOTIS", + version: "1.0.0", + quality: "built-in", + configSchema: { WORKERS: { type: "integer" } }, + container: { image: "fixture/motis", tag: "1" }, + }, + }, + ]; + const observed = observeConfigurationSources({ + installation, + infraDir: root, + services, + authorityRevision: "v1", + env: {}, + }); + expect(observed.services[0]?.fields).toContainEqual({ + key: "WORKERS", + source: "host", + envVar: "SERVICE_MOTIS_WORKERS", + }); + // Match the serialized value, not coincidental bytes inside its random source UUID. + expect(JSON.stringify(observed)).not.toContain('"77"'); + expect(JSON.stringify(observed)).not.toContain('"value"'); + } finally { + closeControlInstallation(installation); + } +}); diff --git a/packages/core/src/build-artifacts/configuration-source-witness.ts b/packages/core/src/build-artifacts/configuration-source-witness.ts new file mode 100644 index 000000000..5ec684afb --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-source-witness.ts @@ -0,0 +1,317 @@ +import { createHmac, randomUUID } from "node:crypto"; +import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseEnv } from "node:util"; +import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { + type ConfigurationSourcesInspection, + configurationSourcesInspectionSchema, +} from "../ops/configuration-preparation"; +import { serviceConfigEnvPrefix } from "../services/config-resolver"; +import { currentConfigurationGeneration } from "../services/configuration-generation"; +import { SERVICE_SELECTION_FILE } from "../services/deployment"; +import type { LoadedService } from "../services/types"; +import { + type ControlInstallation, + installationState, + readPrivateRecord, + writePrivateRecord, +} from "./control-authority"; +import { signControl, verifyControl } from "./control-records"; + +const domain = "openmapx-configuration-source-witness-v1"; +const witnessSchema = z.strictObject({ + version: z.union([z.literal(1), z.literal(2)]), + inspection: configurationSourcesInspectionSchema, + authorityRevision: z.string().max(128), + observation: z.string().regex(/^[a-f0-9]{64}$/), + expiresAt: z.number().int().safe(), + pins: z.array(z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/)).max(128), +}); +const name = (id: string) => `configuration-source-${z.uuid().parse(id)}.json`; +function hostEnv(infraDir: string, env: NodeJS.ProcessEnv) { + const path = join(infraDir, ".env"); + let file: NodeJS.ProcessEnv = {}; + let fd: number | undefined; + let identity: unknown = null; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1 || before.size > 128 * 1024) + throw Error("Host sources unavailable"); + const bytes = readFileSync(fd); + const after = fstatSync(fd), + current = lstatSync(path); + if ( + before.dev !== current.dev || + before.ino !== current.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw Error("Host sources unavailable"); + file = parseEnv(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + identity = { + dev: before.dev, + ino: before.ino, + mtime: before.mtimeMs, + ctime: before.ctimeMs, + size: before.size, + }; + } catch (error) { + if (fd !== undefined || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } finally { + if (fd !== undefined) closeSync(fd); + } + let selection: { identity: unknown; bytes: string } | null = null; + let selectionFd: number | undefined; + try { + const path = join(infraDir, SERVICE_SELECTION_FILE); + selectionFd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(selectionFd); + if (!before.isFile() || before.nlink !== 1 || before.size > 65536) + throw Error("Host sources unavailable"); + const bytes = readFileSync(selectionFd), + after = fstatSync(selectionFd), + current = lstatSync(path); + if ( + before.dev !== current.dev || + before.ino !== current.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw Error("Host sources unavailable"); + selection = { + identity: { + dev: before.dev, + ino: before.ino, + mtime: before.mtimeMs, + ctime: before.ctimeMs, + size: before.size, + }, + bytes: bytes.toString("base64"), + }; + } catch (error) { + if (selectionFd !== undefined || (error as NodeJS.ErrnoException).code !== "ENOENT") + throw error; + } finally { + if (selectionFd !== undefined) closeSync(selectionFd); + } + return { env: { ...file, ...env }, identity, selection }; +} + +/** Private, bounded source capture. Raw overlay bytes never enter the public inspection. */ +function releaseSource(infraDir: string) { + const path = join(infraDir, "docker-compose.release.yml"); + let fd: number | undefined; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1 || before.size > 128 * 1024) + throw Error("Host sources unavailable"); + const bytes = readFileSync(fd), + after = fstatSync(fd), + current = lstatSync(path); + if ( + before.dev !== current.dev || + before.ino !== current.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + current.size !== after.size || + current.mtimeMs !== after.mtimeMs || + current.ctimeMs !== after.ctimeMs + ) + throw Error("Host sources unavailable"); + return { + identity: { + dev: before.dev, + ino: before.ino, + mtime: before.mtimeMs, + ctime: before.ctimeMs, + size: before.size, + }, + bytes: bytes.toString("base64"), + }; + } catch (error) { + if (fd !== undefined || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return null; + } finally { + if (fd !== undefined) closeSync(fd); + } +} +function currentSources(infraDir: string, env: NodeJS.ProcessEnv) { + return { ...hostEnv(infraDir, env), release: releaseSource(infraDir) }; +} + +/** The renderer's supported resource interpolation is a finite mapping. */ +export function configurationHostFields( + services: readonly LoadedService[], + env: NodeJS.ProcessEnv, +) { + const memory: Record = { + brouter: "BROUTER_MEMORY", + "local-ai": "LOCAL_AI_MEMORY", + motis: "MOTIS_SLOT_MEMORY", + "motis-staging": "MOTIS_SLOT_MEMORY", + postgis: "POSTGIS_MEMORY", + }; + return services.map(({ manifest }) => { + const schema = (manifest.configSchema?.properties ?? manifest.configSchema ?? {}) as Record< + string, + unknown + >; + const declared = Object.entries(schema) + .filter( + ([key, definition]) => + key !== "type" && + key !== "properties" && + definition !== null && + typeof definition === "object" && + !Array.isArray(definition), + ) + .map(([key]) => key); + const keys = new Set([...declared, "OPENMAPX_MEMORY_MIB", "OPENMAPX_CPU_LIMIT"]); + const fields = [...keys].sort().map((key) => { + const names = [`${serviceConfigEnvPrefix(manifest.id)}${key.toUpperCase()}`]; + const variable = memory[manifest.id]; + if ( + key === "OPENMAPX_MEMORY_MIB" && + variable && + manifest.container.memory?.startsWith(`\${${variable}`) + ) + names.push(variable); + const envVar = names.find((name) => env[name] !== undefined && env[name] !== ""); + return { + key, + source: envVar ? ("host" as const) : ("none" as const), + ...(envVar ? { envVar } : {}), + }; + }); + return { serviceId: manifest.id, fields }; + }); +} +export function observeConfigurationSources(options: { + installation: ControlInstallation; + infraDir: string; + services: readonly LoadedService[]; + authorityRevision: string; + env?: NodeJS.ProcessEnv; +}): ConfigurationSourcesInspection { + const state = installationState(options.installation); + if (options.installation.role !== "ops-agent") throw Error("Host sources unavailable"); + for (let attempt = 0; attempt < 3; attempt++) { + const revisionId = currentConfigurationGeneration(options.infraDir); + const observed = currentSources(options.infraDir, options.env ?? process.env); + const recheck = currentSources(options.infraDir, options.env ?? process.env); + if ( + revisionId !== currentConfigurationGeneration(options.infraDir) || + canonicalJson(observed) !== canonicalJson(recheck) + ) + continue; + const inspection = configurationSourcesInspectionSchema.parse({ + revisionId, + sourceRevision: randomUUID(), + services: configurationHostFields(options.services, observed.env), + }); + const key = state.privateKey.export({ type: "pkcs8", format: "der" }); + const observation = createHmac("sha256", key) + .update(`${domain}\0`) + .update(canonicalJson(observed)) + .digest("hex"); + const directory = state.privateRoot.directory("observations", true, true); + try { + let count = 0, + bytes = 0; + for (const entry of directory + .listBounded(4096) + .filter((entry) => /^configuration-source-/.test(entry))) { + const retained = verifyControl( + options.installation.publicKey, + domain, + JSON.parse(directory.read(entry, 128 * 1024).toString("utf8")), + witnessSchema, + ); + if (retained.expiresAt <= Date.now() && !retained.pins.length) { + directory.removeFile(entry); + continue; + } + count++; + bytes += directory.read(entry, 128 * 1024).length; + } + const signed = signControl( + state.privateKey, + domain, + witnessSchema.parse({ + version: 2, + inspection, + authorityRevision: options.authorityRevision, + observation, + expiresAt: Date.now() + 300000, + pins: [], + }), + ); + if (count >= 128 || bytes + Buffer.byteLength(canonicalJson(signed)) > 1024 * 1024) + throw Error("Host source witness capacity unavailable"); + writePrivateRecord(state, "observations", name(inspection.sourceRevision), signed, true); + } finally { + directory.close(); + } + return inspection; + } + throw Error("Host sources unavailable"); +} +export function verifyConfigurationSourceWitness(options: { + installation: ControlInstallation; + infraDir: string; + sourceRevision: string; + expectedCurrentRevisionId: string | null; + authorityRevision: string; + env?: NodeJS.ProcessEnv; +}): void { + const state = installationState(options.installation); + const retained = verifyControl( + options.installation.publicKey, + domain, + readPrivateRecord(state, "observations", name(options.sourceRevision)), + witnessSchema, + ); + const observed = + retained.version === 2 + ? currentSources(options.infraDir, options.env ?? process.env) + : hostEnv(options.infraDir, options.env ?? process.env); + const key = state.privateKey.export({ type: "pkcs8", format: "der" }); + const digest = createHmac("sha256", key) + .update(`${domain}\0`) + .update(canonicalJson(observed)) + .digest("hex"); + if ( + retained.expiresAt <= Date.now() || + retained.authorityRevision !== options.authorityRevision || + retained.inspection.sourceRevision !== options.sourceRevision || + retained.inspection.revisionId !== options.expectedCurrentRevisionId || + options.expectedCurrentRevisionId !== currentConfigurationGeneration(options.infraDir) || + digest !== retained.observation + ) + throw Error("Host sources changed"); +} +/** New coordinator work requires the release-bound observation. Historical v1 + * witnesses remain readable for their original ordinary work only. */ +export function verifyConfigurationReleaseSourceWitness( + options: Parameters[0], +): void { + const retained = verifyControl( + options.installation.publicKey, + domain, + readPrivateRecord( + installationState(options.installation), + "observations", + name(options.sourceRevision), + ), + witnessSchema, + ); + if (retained.version !== 2) throw Error("Release-bound host source required"); + verifyConfigurationSourceWitness(options); +} diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 8d571bcec..e43d0d788 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1,3 +1,17 @@ +import { + type ControlChildPhase, + type ControlRecipient, + childPhaseKinds, + controlChildOperationKey, +} from "./control-child-records"; + +export { + type ControlChildPhase, + type ControlRecipient, + childPhaseKinds, + controlChildOperationKey, +} from "./control-child-records"; + import { type ControlInstallation, controlDigest, @@ -12,8 +26,16 @@ import { validateOriginRecord, verifyControl, } from "./control-records"; -// biome-ignore lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. -import { verifyRetainedArtifactTargetRootRelease } from "./target-lease"; +// biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { + closeArtifactTargetLeaseHandle, + inspectArtifactTargetLeaseChildSettlement, + restoreArtifactTargetLeaseForSettlement, + verifyRetainedArtifactTargetRootRelease, + verifyRetainedGenerationHardlinksSettlement, +} from "./target-lease"; + +// biome-ignore-end lint/suspicious/noImportCycles: Same exact deferred guard import. export { type ControlInstallation, @@ -31,7 +53,6 @@ export { } from "./control-records"; import { - createHash, createPrivateKey, createPublicKey, generateKeyPairSync, @@ -41,10 +62,12 @@ import { import { existsSync, fsyncSync, mkdirSync, realpathSync } from "node:fs"; import { join, relative } from "node:path"; import { z } from "zod"; +import type { PreparedConfigurationRecord } from "../ops/configuration-preparation"; import { type OpsJobStatus, type OpsOperationKind, opsOperationSchema, + opsResourceId, parseBoundedOpsResult, parseOpsJobStatus, } from "../ops/contract"; @@ -53,6 +76,7 @@ import { verifyApiRetentionProof, } from "../ops/control-retention-receipt"; import { type ControlExecutionOrigin, parseControlRootOperationKey } from "../ops/control-root-key"; +import { coordinatorPullSubmissionDigest } from "../ops/coordinator-lifecycle-records"; import { type ControlIssuer, type ExecutionRef, @@ -60,11 +84,20 @@ import { executionRefSchema, opsExecutionMaximumMs, type RetainedParentExecution, + retainedParentExecutionSchema, } from "../ops/execution-identity"; import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { servicePullResultSchema } from "../ops/service-operation-records"; import { verifiedParentOrigins, verifiedParents } from "../ops/verified-parent-internal"; +import { + type OriginalConfigurationGenerationNonCommit, + originalConfigurationGenerationNonCommitSource, +} from "../services/configuration-generation"; import { dataConfigurationFenceReceiptSchema } from "../services/data-manager/contracts/configuration-maintenance"; -import { dataExecutionStatusSchema } from "../services/data-manager/contracts/executions"; +import { + dataExecutionStatusSchema, + startGenerationHardlinksExecutionSchema, +} from "../services/data-manager/contracts/executions"; // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { type ControlChildReservation, @@ -76,21 +109,50 @@ import { resolveControlChildPlan, validateControlChildReservations, } from "./child-phase"; +import { + configurationNoGuardIntentSchema, + configurationSettlementDomains, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import { + configurationMaintenanceForwardDomain, + configurationMaintenanceForwardGraphSchema, + configurationMaintenanceForwardSchema, +} from "./configuration-maintenance-forward-records"; +import { + assertConfigurationMaintenanceDecisionSource, + configurationMaintenanceDecisionCommitmentDomain, + configurationMaintenanceDecisionCommitmentPayloadSchema, + configurationMaintenanceOwnerRecordSchema, +} from "./configuration-maintenance-owner-records"; // biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. import { CONTROL_RETENTION_LIMITS, writeRetainedRecord } from "./control-retention"; +import { + type ControlRetentionPreparationContext, + type CoordinatorPullBaseline, + coordinatorPullBaselineSchema, + coordinatorPullObservationDomain, + coordinatorPullWitnessPayloadSchema, + coordinatorPullWitnessSchema, + stagedCoordinatorObservationDomain, + stagedCoordinatorObservationPayloadSchema, + stagedCoordinatorSourceSchema, +} from "./coordinator-pull-records"; // biome-ignore-start lint/suspicious/noImportCycles: Fixed proof verification uses this module's original nominal authority and effect writer after initialization. import { inspectFeedProxyValidationFailure, publishFeedProxyCanonicalTerminal, recordFeedProxyReloadAttempted, -} from "./feed-proxy-validation"; +} from "./data-execution-candidate"; // biome-ignore-end lint/suspicious/noImportCycles: Fixed live proof dependency described above. import { AnchoredDirectory, identity } from "./filesystem"; +import { buildFixedConfigurationMaintenancePlan } from "./fixed-configuration-maintenance"; import { assertOriginClockForward, bootstrapRetentionClock, readRetentionClock, } from "./retention-clock"; +// biome-ignore-start lint/suspicious/noImportCycles: Authority records and terminal repair share this sole per-origin index. Runtime calls must consume the same nominal owner, never a second authority map or callback. import { acknowledgeRetentionDependency, assertFamilyCanonicalCompatibility, @@ -101,40 +163,15 @@ import { hasRetentionEvidence, markFamilyTerminal, publishFamilyCanonicalTerminal, + readActiveStagedCoordinatorApiCopy, reconcileRetentionFamilies, registerFamilyExecution, requireFamilyJournal, retentionApiDependency, + verifyFamilyOwnedRecord, } from "./retention-family"; +// biome-ignore-end lint/suspicious/noImportCycles: Exact deferred original-owner/index guard join. export const CONTROL_ROLES = ["ops-agent", "data-manager", "host-cli"] as const; -export type ControlRecipient = "fixed-cli" | "data-manager" | "ops-agent" | "configuration-helper"; -export type ControlChildPhase = - | "lifecycle" - | "build" - | "data-link" - | "data-clean" - | "data-workflow" - | "hardlinks" - | "stop-primary" - | "stop-staging" - | "restart-staging" - | "promote-primary" - | "recover-primary" - | "proxy-stage" - | "proxy-commit" - | "proxy-restore" - | "paired-pin-commit" - | "configuration-fence" - | "configuration-fence-settle" - | "configuration-commit" - | "maintenance-hardlinks" - | "maintenance-recreate" - | "maintenance-handoff" - | "osm" - | "transit" - | "fonts" - | "data-step" - | "configuration-handoff"; declare const authorityBrand: unique symbol, peerBrand: unique symbol, settlementBrand: unique symbol, @@ -384,72 +421,6 @@ export function closeControlInstallation(installation: ControlInstallation) { s.privateRoot.close(); installations.delete(installation); } -export const childPhaseKinds: Readonly> = { - lifecycle: ["service.start", "system.update"], - build: ["service.build", "services.buildAll"], - "data-link": ["data.link", "configuration.hardlinks.apply"], - "data-clean": ["data.clean"], - "data-workflow": [ - "data.update", - "data.downloadOsm", - "data.downloadFonts", - "data.convertOverpass", - "data.overtureSync", - "data.overtureConflate", - "data.searchIndexBuild", - "data.notablePlacesBuild", - ], - hardlinks: ["generation-hardlinks"], - "stop-primary": ["motis.primary.stop"], - "stop-staging": ["motis.staging.stop"], - "restart-staging": ["motis.staging.restart"], - "promote-primary": ["motis.primary.promote"], - "recover-primary": ["motis.primary.restart"], - "proxy-stage": ["feedProxy.validateAndReload"], - "proxy-commit": ["feedProxy.validateAndReload"], - "proxy-restore": ["feedProxy.validateAndReload"], - "paired-pin-commit": ["transitousLock.approve"], - "configuration-fence": ["configuration-maintenance.prepare"], - "configuration-fence-settle": ["configuration-maintenance.settle"], - "configuration-commit": [ - "stack.render", - "serviceSelection.apply", - "serviceConfig.apply", - "integrationConfig.apply", - "vault.apply", - ], - "maintenance-hardlinks": ["configuration.hardlinks.apply"], - "maintenance-recreate": ["service.recreate", "service.recreateIsolated"], - "maintenance-handoff": ["configurationHandoff.prepare"], - osm: ["download-osm"], - transit: ["transit-sync"], - fonts: ["download-fonts"], - "data-step": [ - "download-osm", - "download-fonts", - "convert-overpass", - "overture-sync", - "overture-conflate", - "search-index-build", - "notable-places-build", - ], - "configuration-handoff": ["configuration-helper"], -}; -export function controlChildOperationKey( - parentKey: string, - phase: ControlChildPhase, - ordinal: number, -): string { - if ( - !/^opk1_[A-Za-z0-9_-]{16,64}$/.test(parentKey) || - !Number.isSafeInteger(ordinal) || - ordinal < 0 || - ordinal > 63 || - !Object.hasOwn(childPhaseKinds, phase) - ) - throw Error("Invalid fixed phase key"); - return `opk1_${createHash("sha256").update(`openmapx-control-child-v1\0${parentKey}\0${phase}\0${ordinal}`).digest("base64url").slice(0, 32)}`; -} export interface ControlPlannedChild { phase: ControlChildPhase; ordinal: number; @@ -885,7 +856,15 @@ export function repairControlJournalTerminal( proof.publication === "not_started" && proof.receipt.kind === "feed-proxy-validation-failure" && inspectFeedProxyValidationFailure(authority, proof.receipt.commitmentDigest) !== null; - if (!succeeded && !failedWithoutEffect && !validationFailed) + const generationNotCommitted = + terminal.state === "succeeded" && + proof.outcome === "contained" && + proof.publication === "not_started" && + proof.receipt.kind === "configuration-generation-not-committed" && + canonicalJson(proof.receipt.result) === canonicalJson(terminal.result) && + canonicalJson(inspectConfigurationGenerationNonCommit(installation, execution)) === + canonicalJson(proof.receipt); + if (!succeeded && !failedWithoutEffect && !validationFailed && !generationNotCommitted) throw Error("Terminal worker containment is not proved"); const committed = verifyAndSealTerminalCommitment(s, terminal, terminalAt); if (s.kind === "feedProxy.validateAndReload") @@ -1026,6 +1005,7 @@ export function retainAdmittedControlExecution( serviceIds: readonly string[]; plan?: readonly ControlPlannedChild[]; reservations?: readonly ControlChildReservation[]; + coordinatorPullBaseline?: CoordinatorPullBaseline; }, ): VerifiedExecutionAuthority { const state = installationState(installation); @@ -1085,6 +1065,24 @@ export function retainAdmittedControlExecution( }) ) throw Error("Immutable execution identity mismatch"); + const coordinatorPull = + parsed.success && parsed.data.kind === "service.pull" && parsed.data.candidateId + ? parsed.data + : undefined; + if (coordinatorPull || input.coordinatorPullBaseline) { + const baseline = coordinatorPullBaselineSchema.parse(input.coordinatorPullBaseline); + if ( + !coordinatorPull || + coordinatorPull.serviceId !== "data-manager" || + installation.role !== "ops-agent" || + state.fixedChildId || + input.parent?.issuer !== "api" || + input.parent.apiJobId === undefined || + baseline.candidateId !== coordinatorPull.candidateId || + baseline.configurationRevisionId !== coordinatorPull.expectedConfigurationRevisionId + ) + throw Error("Original coordinator pull baseline required"); + } const plan = z .array(plannedChildSchema) .max(64) @@ -1212,7 +1210,14 @@ export function retainAdmittedControlExecution( }); if (Buffer.byteLength(canonicalJson(receipt)) > 65536) throw Error("Execution authority receipt capacity"); - registerFamilyExecution(state, origin, execution, controlDigest(receipt), input.parent); + registerFamilyExecution( + state, + origin, + execution, + controlDigest(receipt), + input.parent, + !!input.coordinatorPullBaseline, + ); const name = `${execution.operationKey}.json`; let newlyAdmitted = false; try { @@ -1502,10 +1507,30 @@ const settlementSchema = z.strictObject({ terminalReceiptDigest: digest, }); const effectObservationSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("configuration-generation-not-committed"), + revisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), + expectedCurrentRevisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), + result: z.strictObject({ + outcome: z.literal("not_committed"), + reason: z.enum(["conflict", "busy", "failed"]), + currentRevisionId: z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .nullable() + .optional(), + cleanupFailed: z.literal(false), + }), + }), + z.strictObject({ + kind: z.literal("configuration-no-guard"), + issuerFreezeProofDigest: digest, + issuerUnusedConfirmationDigest: digest, + }), z.strictObject({ kind: z.literal("data-configuration-fence-refused"), maintenanceId: uuid, - reason: z.literal("busy"), + reason: z.enum(["busy", "capacity"]), }), z.strictObject({ kind: z.literal("data-configuration-fence"), @@ -1529,6 +1554,123 @@ const effectObservationSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("not-started") }), z.strictObject({ kind: z.literal("unresolved") }), ]); +/** Fixed writer producer only. A DTO, caller error or parsed result cannot + * substitute for the actual completed native writer observation. */ +export function retainConfigurationGenerationNonCommit( + authority: VerifiedExecutionAuthority, + observation: OriginalConfigurationGenerationNonCommit, +): void { + const state = executionAuthorityState(authority), + actual = originalConfigurationGenerationNonCommitSource(observation), + admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: unknown } }, + operation = opsOperationSchema.parse(admitted.payload.operation); + if ( + state.installation.fixedChildId || + state.installation.installation.role !== "ops-agent" || + ![ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ].includes(state.kind) || + !("revisionId" in operation) || + !("expectedCurrentRevisionId" in operation) || + state.execution.operationKey !== actual.operationKey || + operation.revisionId !== actual.revisionId || + operation.expectedCurrentRevisionId !== actual.expectedCurrentRevisionId || + actual.installationId !== state.installation.installation.installationId || + controlDigest(admitted) !== state.receiptDigest + ) + throw Error("Original writer/execution noncommit binding required"); + withPrivateEffectWriter(state, () => { + const receipt = effectObservationSchema.parse({ + kind: "configuration-generation-not-committed", + revisionId: actual.revisionId, + expectedCurrentRevisionId: actual.expectedCurrentRevisionId, + result: actual.result, + }), + payload = { + identity: state.execution, + outcome: "contained" as const, + publication: "not_started" as const, + terminalReceiptDigest: controlDigest(receipt), + receipt, + }, + name = `${state.execution.operationKey}.json`; + const prior = readPrivateRecord(state.installation, "effects", name, 65536); + if (prior !== null) { + const retained = verifyControl( + state.installation.installation.publicKey, + "openmapx-control-settlement-v1", + prior, + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if (canonicalJson(retained) !== canonicalJson(payload)) + throw Error("Original writer noncommit source changed"); + return; + } + settleExecutionLifecycle(state); + writePrivateRecord( + state.installation, + "effects", + name, + signControl(state.installation.privateKey, "openmapx-control-settlement-v1", payload), + false, + state.origin, + ); + if (state.origin) + markFamilyTerminal( + state.installation, + state.origin, + state.execution, + true, + controlDigest(payload), + ); + }); +} +/** Actual private source, not a caller outcome flag. */ +export function inspectConfigurationGenerationNonCommit( + installation: ControlInstallation, + execution: ExecutionRef, +) { + const local = installationState(installation), + authority = restoreAdmittedControlExecution(installation, execution), + state = executionAuthorityState(authority), + raw = readPrivateRecord(local, "effects", `${execution.operationKey}.json`, 65536), + effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + raw, + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if ( + effect.receipt.kind !== "configuration-generation-not-committed" || + effect.outcome !== "contained" || + effect.publication !== "not_started" || + !state.origin || + canonicalJson(effect.identity) !== canonicalJson(execution) || + effect.terminalReceiptDigest !== controlDigest(effect.receipt) + ) + throw Error("Actual original generation noncommit required"); + verifyFamilyOwnedRecord( + local, + state.origin, + { + scope: "private", + namespace: "effects", + container: "", + name: `${execution.operationKey}.json`, + }, + canonicalJson(raw), + ); + return effect.receipt; +} export function withPrivateEffectWriter(s: AuthorityState, write: () => T): T { const guards = s.installation.privateRoot.directory("effect-writers", true, true); try { @@ -1573,6 +1715,166 @@ export function beginControlEffect(authority: VerifiedExecutionAuthority): void }), ); } +function configurationPreEffectState(authority: VerifiedExecutionAuthority) { + const state = executionAuthorityState(authority), + raw = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ), + operation = opsOperationSchema.parse( + (raw as { payload: { operation: unknown } }).payload.operation, + ); + if ( + !state.origin || + state.installation.fixedChildId || + state.installation.installation.role !== "ops-agent" || + controlDigest(raw) !== state.receiptDigest || + opsOperationFingerprint(operation) !== state.execution.operationFingerprint || + !( + operation.kind === "service.update" || + operation.kind === "stack.start" || + (operation.kind === "service.start" && operation.serviceId !== "data-manager") || + (operation.kind === "service.pull" && + operation.serviceId === "data-manager" && + operation.candidateId) + ) + ) + throw Error("Fixed managed configuration pre-effect source required"); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { + scope: "private", + namespace: "executions", + container: "", + name: `${state.execution.operationKey}.json`, + }, + canonicalJson(raw), + ); + return state; +} +/** Existing not-started evidence, only after the original lifecycle is frozen + * and every exact/reserved child remains unissued and unbound. */ +export function retainConfigurationPreEffectRefusal(authority: VerifiedExecutionAuthority): void { + const state = configurationPreEffectState(authority); + if ( + cancelControlExecution(authority).length || + state.reservations.some((slot) => { + const retained = inspectControlChildPublication(authority, slot); + return retained.binding !== null || retained.publication.state !== "unissued"; + }) + ) + throw Error("Issued or bound managed configuration must remain fenced"); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); +} +/** The transition/restart observer consumes the actual indexed signed effect; + * an error class or caller status never qualifies nonissuance. */ +export function inspectConfigurationPreEffectRefusal(authority: VerifiedExecutionAuthority): void { + const state = configurationPreEffectState(authority), + name = `${state.execution.operationKey}.json`, + raw = readPrivateRecord(state.installation, "effects", name, 65536), + effect = verifyControl( + state.installation.installation.publicKey, + "openmapx-control-settlement-v1", + raw, + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if ( + !state.origin || + effect.receipt.kind !== "not-started" || + effect.outcome !== "failed_without_effect" || + effect.publication !== "not_started" || + controlDigest(effect.identity) !== controlDigest(state.execution) || + effect.terminalReceiptDigest !== controlDigest(effect.receipt) + ) + throw Error("Actual unstarted managed configuration receipt required"); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "effects", container: "", name }, + canonicalJson(raw), + ); + if ( + cancelControlExecution(authority).length || + state.reservations.some((slot) => { + const retained = inspectControlChildPublication(authority, slot); + return retained.binding !== null || retained.publication.state !== "unissued"; + }) + ) + throw Error("Unstarted managed configuration child source changed"); +} +/** Recover only the exact failure ALREADY fixed by the original signed + * terminal commitment. The interrupted journal is a locator, never authority; + * neither a new error/time nor an uncommitted failure can pass this comparison. */ +export function restoreConfigurationPreEffectTerminal( + installation: ControlInstallation, + execution: ExecutionRef, + observed: OpsJobStatus, +): { status: OpsJobStatus; terminalAt: number } { + const authority = restoreAdmittedControlExecution(installation, execution); + inspectConfigurationPreEffectRefusal(authority); + const state = configurationPreEffectState(authority), + proof = readTerminalCommitment(state), + proofName = terminalCommitmentName(state.execution), + rawProof = readPrivateRecord(state.installation, "observations", proofName, 4096); + if ( + !state.origin || + controlDigest((rawProof as { payload: unknown }).payload) !== controlDigest(proof) + ) + throw Error("Original pre-effect terminal changed"); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "observations", container: "", name: proofName }, + canonicalJson(rawProof), + ); + const rawAdmission = readPrivateRecord( + state.installation, + "executions", + `${execution.operationKey}.json`, + 65536, + ); + if (controlDigest(rawAdmission) !== state.receiptDigest) + throw Error("Original pre-effect admission changed"); + const admitted = z + .object({ admittedAt: z.number().int().nonnegative().safe(), operation: opsOperationSchema }) + .passthrough() + .parse((rawAdmission as { payload: unknown }).payload); + if ("revisionId" in admitted.operation || proof.terminalAt < admitted.admittedAt) + throw Error("Immutable original pre-effect tuple required"); + // Startup may have added an expiry cancellation projection. Only the + // original signed whole-tuple digest decides whether it was present. + for (const terminationRequestedAt of new Set([undefined, observed.terminationRequestedAt])) { + const status = parseOpsJobStatus({ + version: 2, + admittedAt: admitted.admittedAt, + deadlineAt: execution.deadlineAt, + fingerprint: execution.fingerprint, + operationId: execution.executionId, + operationKey: execution.operationKey, + kind: admitted.operation.kind, + resourceId: opsResourceId(admitted.operation), + state: "failed", + submittedAt: new Date(admitted.admittedAt).toISOString(), + updatedAt: new Date(proof.terminalAt).toISOString(), + errorClass: + admitted.operation.kind === "service.pull" ? "busy" : "configuration_prerequisite", + failureResult: {}, + ...(terminationRequestedAt === undefined ? {} : { terminationRequestedAt }), + }); + if (controlTerminalReceiptDigest(status) === proof.terminalDigest) { + terminalIdentity(state, status); + return { status, terminalAt: proof.terminalAt }; + } + } + throw Error("Original pre-effect terminal unavailable"); +} /** Fixed executor adapters supply their parsed result or exact fixed-command exit; arbitrary text is never proof. */ export function retainControlEffectReceipt( authority: VerifiedExecutionAuthority, @@ -1644,6 +1946,67 @@ export function retainControlEffectReceipt( } outcome = "committed"; publication = "committed"; + } else if (receipt.kind === "configuration-no-guard") { + if ( + s.kind !== "configurationMaintenance.prepare" || + s.installation.fixedChildId || + !s.origin || + s.installation.installation.role !== "ops-agent" + ) + throw Error("Original frozen maintenance effect source required"); + const admitted = readPrivateRecord( + s.installation, + "executions", + `${s.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + name = `${uuid.parse(admitted.payload?.operation?.maintenanceId)}.json`, + raw = readPrivateRecord(s.installation, "configuration-maintenance", name), + owner = verifyControl( + s.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + z + .object({ + admissionDigest: digest, + issuance: z.enum(["closing_without_guard", "closed_without_guard"]), + noGuard: configurationNoGuardIntentSchema, + guardSource: z.null(), + final: z.null(), + inspection: z.null(), + receipt: z + .object({ dmFence: z.null(), generation: z.literal("not_started") }) + .passthrough(), + }) + .passthrough(), + ); + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace: "configuration-maintenance", name, container: "" }, + canonicalJson(raw), + ); + const confirmation = owner.noGuard.issuerUnused.confirmation; + if ( + owner.admissionDigest !== s.receiptDigest || + !confirmation || + receipt.issuerFreezeProofDigest !== owner.noGuard.noGuardEvidence.issuerFreezeProofDigest || + receipt.issuerUnusedConfirmationDigest !== + settlementRecordDigest( + configurationSettlementDomains.unusedConfirmation, + confirmation.payload, + ) + ) + throw Error("Actual frozen issuer UNUSED confirmation required"); + verifyControl( + s.installation.installation.publicKey, + configurationSettlementDomains.unusedConfirmation, + confirmation, + configurationNoGuardIntentSchema.shape.issuerUnused.shape.confirmation.unwrap().shape + .payload, + ); + outcome = "failed_without_effect"; + publication = "not_started"; } else if (receipt.kind === "not-started") { try { if ( @@ -1735,6 +2098,1229 @@ export function verifyControlEffectSettlement( settlements.set(verified, { state: s, evidence: publicEvidence }); return verified; } +/** Fixed read-only source check for maintenance's pre-root ordinary DM Start. + * It reads this role's actual original admission, canonical terminal, effect + * result and released target tree. Candidate metadata alone proves nothing. */ +export function verifyMaintenanceCoordinatorPrerequisite( + installation: ControlInstallation, + record: PreparedConfigurationRecord, +) { + const local = installationState(installation), + fixed = record.coordinatorPrerequisite; + if ( + installation.role !== "ops-agent" || + local.fixedChildId || + !record.maintenanceId || + !fixed || + fixed.operation.kind !== "service.start" || + fixed.operation.serviceId !== "data-manager" || + fixed.root.operationKey === record.prepareOperationKey || + fixed.root.parent.operationKey !== record.parentOperationKey || + fixed.root.parent.fingerprint !== record.parentFingerprint || + fixed.root.parent.deadlineAt !== record.parentDeadlineAt + ) + throw Error("Original maintenance coordinator prerequisite required"); + const raw = readPrivateRecord(local, "executions", `${fixed.root.operationKey}.json`, 65536); + const admitted = verifyControl( + installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z + .object({ + execution: executionRefSchema, + kind: z.literal("service.start"), + operation: opsOperationSchema, + admittedAt: z.number().int().nonnegative().safe(), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + parent: retainedParentExecutionSchema.refine((parent) => parent.issuer === "api"), + }) + .passthrough(), + ); + if ( + admitted.execution.operationKey !== fixed.root.operationKey || + admitted.requestedDeadlineAt !== fixed.root.requestedDeadlineAt || + admitted.execution.deadlineAt > admitted.requestedDeadlineAt || + admitted.execution.deadlineAt <= admitted.admittedAt || + admitted.execution.operationFingerprint !== opsOperationFingerprint(fixed.operation) || + admitted.execution.fingerprint !== + executionFingerprint({ + kind: "service.start", + operationFingerprint: opsOperationFingerprint(fixed.operation), + deadlineAt: admitted.execution.deadlineAt, + parent: admitted.parent, + }) || + canonicalJson(admitted.operation) !== canonicalJson(fixed.operation) || + admitted.parent.apiJobId !== record.apiJobId || + admitted.parent.operationKey !== record.parentOperationKey || + admitted.parent.fingerprint !== record.parentFingerprint || + admitted.parent.deadlineAt !== record.parentDeadlineAt + ) + throw Error("Original coordinator admission changed"); + const authority = restoreAdmittedControlExecution(installation, admitted.execution), + owner = executionAuthorityState(authority), + canonical = readTerminalCommitment(owner), + settled = verifyControlEffectSettlement(installation, admitted.execution), + effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord(local, "effects", `${fixed.root.operationKey}.json`, 65536), + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if ( + !owner.origin || + canonical.phase !== "canonical" || + effect.outcome !== "committed" || + effect.publication !== "committed" || + effect.receipt.kind !== "ops-result" || + (parseBoundedOpsResult("service.start", effect.receipt.result) as { changed: boolean }) + .changed !== true + ) + throw Error("Conclusive successful coordinator prerequisite required"); + assertFamilyCanonicalCompatibility( + local, + owner.origin, + admitted.execution, + canonical.terminalDigest, + canonical.terminalAt, + ); + verifyRetainedArtifactTargetRootRelease(authority); + return Object.freeze({ + execution: Object.freeze(admitted.execution), + terminalDigest: canonical.terminalDigest, + terminalAt: canonical.terminalAt, + effectSettlementDigest: controlDigest(childSettlementState(settled).evidence), + }); +} + +declare const stagedCoordinatorBrand: unique symbol; +export type VerifiedStagedCoordinatorOwnership = Readonly<{ [stagedCoordinatorBrand]: true }>; +const stagedCoordinatorProofs = new WeakMap< + VerifiedStagedCoordinatorOwnership, + { + installation: ControlInstallation; + origin: ControlExecutionOrigin; + source: z.infer; + status: OpsJobStatus; + } +>(); +export function verifiedStagedCoordinatorOwnershipState(proof: VerifiedStagedCoordinatorOwnership) { + const value = stagedCoordinatorProofs.get(proof); + if (!value) throw Error("Actual staged coordinator ownership required"); + return { + installationId: value.installation.installationId, + origin: structuredClone(value.origin), + source: structuredClone(value.source), + }; +} +/** Journal tuples locate source bytes only. This verifies the actual first + * signed admission, immutable canonical/effect settlement and role-signed API + * copy without requiring or renewing an expired API candidate or its grants. */ +export function readVerifiedStagedCoordinatorOwnership( + installation: ControlInstallation, + execution: ExecutionRef, + input: OpsJobStatus, + context?: ControlRetentionPreparationContext, +): VerifiedStagedCoordinatorOwnership { + const local = installationState(installation); + if (installation.role !== "ops-agent" || local.fixedChildId) + throw Error("Production ops-agent staged source required"); + const raw = readPrivateRecord(local, "executions", `${execution.operationKey}.json`, 65536), + admitted = verifyControl( + installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z + .object({ + version: z.literal(2), + installationId: uuid, + origin: controlOriginSchema, + kind: z.literal("service.pull"), + execution: executionRefSchema, + operation: opsOperationSchema, + admittedAt: z.number().int().nonnegative().safe(), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + parent: retainedParentExecutionSchema, + coordinatorPullBaseline: coordinatorPullBaselineSchema, + }) + .passthrough(), + ), + baseline = admitted.coordinatorPullBaseline, + operation = admitted.operation, + parent = admitted.parent; + if ( + admitted.installationId !== installation.installationId || + canonicalJson(admitted.execution) !== canonicalJson(execution) || + execution.executor !== "ops-agent" || + execution.principal !== "api" || + operation.kind !== "service.pull" || + operation.serviceId !== "data-manager" || + !operation.candidateId || + operation.candidateId !== baseline.candidateId || + operation.expectedConfigurationRevisionId !== baseline.configurationRevisionId || + execution.operationFingerprint !== opsOperationFingerprint(operation) || + parent.issuer !== "api" || + execution.deadlineAt > admitted.requestedDeadlineAt || + execution.deadlineAt <= admitted.admittedAt || + execution.fingerprint !== + executionFingerprint({ + kind: "service.pull", + operationFingerprint: execution.operationFingerprint, + deadlineAt: execution.deadlineAt, + parent, + }) + ) + throw Error("Original staged coordinator admission changed"); + const authority = restoreAdmittedControlExecution(installation, execution), + state = executionAuthorityState(authority), + canonical = readTerminalCommitment(state), + terminal = terminalIdentity(state, input), + settlement = verifyControlEffectSettlement(installation, execution), + effectRecord = readPrivateRecord(local, "effects", `${execution.operationKey}.json`, 65536), + effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + effectRecord, + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if ( + !state.origin || + canonicalJson(state.origin) !== canonicalJson(admitted.origin) || + canonical.phase !== "canonical" || + terminal.terminal.state !== "succeeded" || + canonical.terminalDigest !== terminal.digest || + effect.outcome !== "committed" || + effect.publication !== "committed" || + effect.receipt.kind !== "ops-result" || + canonicalJson(terminal.terminal.result) !== canonicalJson(effect.receipt.result) + ) + throw Error("Actual successful contained staged coordinator source required"); + const result = servicePullResultSchema.parse(effect.receipt.result); + if ( + result.revisionId !== baseline.configurationRevisionId || + result.imageReference !== baseline.configImageReference + ) + throw Error("Original staged coordinator result changed"); + const canonicalRecord = readPrivateRecord( + local, + "observations", + terminalCommitmentName(execution), + 4096, + ); + for (const record of [ + { namespace: "executions", name: `${execution.operationKey}.json`, raw }, + { namespace: "effects", name: `${execution.operationKey}.json`, raw: effectRecord }, + { namespace: "observations", name: terminalCommitmentName(execution), raw: canonicalRecord }, + ]) + verifyFamilyOwnedRecord( + local, + state.origin, + { scope: "private", namespace: record.namespace, container: "", name: record.name }, + canonicalJson(record.raw), + ); + assertFamilyCanonicalCompatibility( + local, + state.origin, + execution, + canonical.terminalDigest, + canonical.terminalAt, + ); + const copy = readActiveStagedCoordinatorApiCopy( + local, + state.origin, + execution, + canonical.terminalDigest, + parent, + context, + ), + source = stagedCoordinatorSourceSchema.parse({ + execution, + admittedAt: admitted.admittedAt, + baseline, + admissionDigest: controlDigest(raw), + canonicalDigest: canonical.terminalDigest, + effectDigest: controlDigest(childSettlementState(settlement).evidence), + apiCopyDigest: copy.apiCopyDigest, + }), + proof = Object.freeze({}) as VerifiedStagedCoordinatorOwnership; + stagedCoordinatorProofs.set(proof, { + installation, + origin: state.origin, + source: structuredClone(source), + status: structuredClone(input), + }); + return proof; +} +declare const stagedOwnerBaselineBrand: unique symbol; +export type VerifiedStagedCoordinatorOwnerBaseline = Readonly<{ [stagedOwnerBaselineBrand]: true }>; +const stagedOwnerBaselines = new WeakMap< + VerifiedStagedCoordinatorOwnerBaseline, + { + baseline: CoordinatorPullBaseline; + expectedCurrentRevisionId: string; + currentAuthorityRevision: string; + } +>(); +export function verifiedStagedCoordinatorOwnerBaselineState( + proof: VerifiedStagedCoordinatorOwnerBaseline, +) { + const value = stagedOwnerBaselines.get(proof); + if (!value) throw Error("Actual staged original-owner provenance required"); + return structuredClone(value); +} +/** A separately bound historical observation, never this owner's pull or a + * caller-selected current generation. All postcommit exceptions verify the + * original actual canonical/effect/physical child under the same decision. */ +export function ownedStagedCoordinatorBaseline( + authority: VerifiedExecutionAuthority, +): VerifiedStagedCoordinatorOwnerBaseline { + const state = executionAuthorityState(authority); + if ( + state.kind !== "configurationMaintenance.prepare" || + !state.origin || + state.installation.fixedChildId || + state.installation.installation.role !== "ops-agent" + ) + throw Error("Original staged maintenance owner required"); + const admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }; + const name = `${uuid.parse(admitted.payload?.operation?.maintenanceId)}.json`, + raw = readPrivateRecord(state.installation, "configuration-maintenance", name); + const owner = verifyControl( + state.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name }, + canonicalJson(raw), + ); + const observation = verifyControl( + state.installation.installation.publicKey, + stagedCoordinatorObservationDomain, + owner.stagedCoordinatorProvenance, + stagedCoordinatorObservationPayloadSchema, + ), + baseline = observation.source.baseline; + if ( + owner.coordinatorPull || + owner.plan.coordinatorImage?.action !== "apply" || + observation.installationId !== state.installation.installation.installationId || + owner.admissionDigest !== state.receiptDigest || + controlDigest(admitted) !== state.receiptDigest || + canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) || + canonicalJson(owner.origin) !== canonicalJson(state.origin) || + canonicalJson(observation.ownerPrepare) !== canonicalJson(state.execution) || + observation.candidateId !== owner.plan.operation.candidateId || + observation.planDigest !== owner.plan.planDigest || + baseline.configurationRevisionId !== owner.plan.operation.expectedCurrentRevisionId || + canonicalJson(owner.originalContainers.find((c) => c.serviceId === "data-manager")) !== + canonicalJson(baseline.container) + ) + throw Error("Original staged owner/source binding changed"); + let expectedCurrentRevisionId = baseline.configurationRevisionId; + if (owner.receipt.generation === "committed") { + const generation = owner.apiControls.steps.find((value) => value.ordinal === 0); + if ( + !generation?.canonical || + !generation.receiptDelivery || + owner.receipt.appliedRevisionId !== owner.plan.operation.revisionId + ) + throw Error("Original staged owner generation required"); + const child = verifyConfigurationMaintenanceChildTerminal( + authority, + 0, + generation.canonical.status, + generation.canonical.terminalAt, + ), + verified = configurationMaintenanceChildState(child, authority), + lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + const physical = inspectArtifactTargetLeaseChildSettlement(lease, generation.execution); + if (!physical || canonicalJson(physical) !== canonicalJson(verified.settlement)) + throw Error("Actual staged generation settlement required"); + } finally { + closeArtifactTargetLeaseHandle(lease); + } + expectedCurrentRevisionId = owner.plan.operation.revisionId; + } else if ( + owner.receipt.generation !== "not_started" && + owner.receipt.generation !== "not_committed" + ) + throw Error("Unknown staged owner generation"); + const proof = Object.freeze({}) as VerifiedStagedCoordinatorOwnerBaseline; + stagedOwnerBaselines.set(proof, { + baseline: structuredClone(baseline), + expectedCurrentRevisionId, + currentAuthorityRevision: owner.plan.sourceAuthorityRevision, + }); + return proof; +} +declare const coordinatorBaselineBrand: unique symbol; +export type VerifiedCoordinatorPullBaseline = Readonly<{ [coordinatorBaselineBrand]: true }>; +const baselineProofs = new WeakMap< + VerifiedCoordinatorPullBaseline, + Readonly< + z.infer & { expectedCurrentRevisionId: string } + > +>(); +const observedBaselineProofs = new WeakMap(); +export function verifiedCoordinatorPullBaselineState(proof: VerifiedCoordinatorPullBaseline) { + const value = baselineProofs.get(proof); + if (!value) throw Error("Verified original coordinator baseline required"); + // Return a copy so a consumer cannot mutate the authenticated private observation. + return structuredClone(value); +} +export function observedCoordinatorPullBaseline(witness: object): VerifiedCoordinatorPullBaseline { + const proof = observedBaselineProofs.get(witness); + if (!proof) throw Error("Actual original pull observation required"); + return proof; +} +/** Recovery derives only from the exact indexed signed owner, after the initial + * successful source copy. It does not depend on a retired normal pull family. */ +export function ownedCoordinatorPullBaseline( + authority: VerifiedExecutionAuthority, +): VerifiedCoordinatorPullBaseline { + const state = executionAuthorityState(authority); + if ( + state.kind !== "configurationMaintenance.prepare" || + state.installation.fixedChildId || + state.installation.installation.role !== "ops-agent" || + !state.origin + ) + throw Error("Original maintenance owner required"); + const admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }; + const name = `${uuid.parse(admitted.payload?.operation?.maintenanceId)}.json`, + raw = readPrivateRecord(state.installation, "configuration-maintenance", name); + const owner = verifyControl( + state.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name }, + canonicalJson(raw), + ); + const witness = verifyControl( + state.installation.installation.publicKey, + coordinatorPullObservationDomain, + owner.coordinatorPull, + coordinatorPullWitnessPayloadSchema, + ); + if ( + owner.admissionDigest !== state.receiptDigest || + controlDigest(admitted) !== state.receiptDigest || + canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) || + canonicalJson(owner.origin) !== canonicalJson(state.origin) || + canonicalJson(witness.maintenancePrepare) !== canonicalJson(state.execution) || + witness.installationId !== state.installation.installation.installationId || + witness.candidateId !== owner.plan.operation.candidateId || + witness.maintenanceId !== owner.plan.operation.maintenanceId || + witness.baseline.candidateId !== witness.candidateId || + witness.baseline.sourceRevision !== witness.sourceRevision || + witness.baseline.configurationAuthorityRevision !== owner.plan.sourceAuthorityRevision || + witness.baseline.submissionDigest !== witness.submissionDigest || + canonicalJson(owner.originalContainers.find((c) => c.serviceId === "data-manager")) !== + canonicalJson(witness.baseline.container) || + owner.plan.coordinatorImage?.imageId !== witness.imageId + ) + throw Error("Original coordinator owner source changed"); + const proof = Object.freeze({}) as VerifiedCoordinatorPullBaseline; + baselineProofs.set( + proof, + Object.freeze({ + ...structuredClone(witness), + expectedCurrentRevisionId: + owner.receipt.generation === "committed" + ? owner.plan.operation.revisionId + : witness.baseline.configurationRevisionId, + }), + ); + return proof; +} +/** Fixed source of this one pre-sealed normal pull. Its real signed admission + * owns the assigned execution ID and effective deadline; neither is predicted. */ +export function verifyCoordinatorPullAdmissionSource( + installation: ControlInstallation, + record: PreparedConfigurationRecord, +) { + const state = installationState(installation), + fixed = record.privateCoordinatorLifecycle; + if ( + installation.role !== "ops-agent" || + state.fixedChildId || + fixed?.action !== "update" || + record.apiFollowUp !== "coordinator-update" || + !record.maintenanceId || + fixed.pull.operation.candidateId !== record.candidateId + ) + throw Error("Original coordinator pull source required"); + const submission = fixed.pull, + raw = readPrivateRecord(state, "executions", `${submission.operationKey}.json`, 65536), + admitted = verifyControl( + installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z + .object({ + version: z.literal(2), + installationId: uuid, + origin: controlOriginSchema, + kind: z.literal("service.pull"), + execution: executionRefSchema, + operation: opsOperationSchema, + admittedAt: z.number().int().nonnegative().safe(), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + parent: retainedParentExecutionSchema, + coordinatorPullBaseline: coordinatorPullBaselineSchema, + }) + .passthrough(), + ), + parent = admitted.parent, + attempt = submission.attempt, + execution = admitted.execution; + if ( + admitted.installationId !== installation.installationId || + admitted.coordinatorPullBaseline.candidateId !== record.candidateId || + admitted.coordinatorPullBaseline.sourceRevision !== record.expectedSourceRevision || + admitted.coordinatorPullBaseline.configurationRevisionId !== + submission.operation.expectedConfigurationRevisionId || + admitted.coordinatorPullBaseline.submissionDigest !== + coordinatorPullSubmissionDigest(submission) || + parent.issuer !== "api" || + execution.executor !== "ops-agent" || + execution.principal !== "api" || + execution.operationKey !== submission.operationKey || + execution.operationFingerprint !== submission.operationFingerprint || + canonicalJson(admitted.operation) !== canonicalJson(submission.operation) || + admitted.requestedDeadlineAt !== submission.requestedDeadlineAt || + execution.deadlineAt > admitted.requestedDeadlineAt || + execution.deadlineAt <= admitted.admittedAt || + execution.fingerprint !== + executionFingerprint({ + kind: "service.pull", + operationFingerprint: submission.operationFingerprint, + deadlineAt: execution.deadlineAt, + parent, + }) || + parent.apiJobId !== record.apiJobId || + parent.apiJobId !== attempt.apiJobId || + parent.apiOwnerVersion !== attempt.apiOwnerVersion || + parent.attemptId !== attempt.attemptId || + parent.operationKey !== record.parentOperationKey || + parent.operationKey !== attempt.parentOperationKey || + parent.fingerprint !== record.parentFingerprint || + parent.fingerprint !== attempt.parentFingerprint || + parent.deadlineAt !== record.parentDeadlineAt || + parent.deadlineAt !== attempt.parentDeadlineAt + ) + throw Error("Original coordinator normal admission changed"); + restoreAdmittedControlExecution(installation, execution); + return admitted; +} + +/** Only a real immutable canonical terminal + contained effect can mint this + * copied observation. The structural public status alone supplies no proof. */ +export function observeCoordinatorPullForMaintenance( + installation: ControlInstallation, + record: PreparedConfigurationRecord, + maintenancePrepare: ExecutionRef, + status: OpsJobStatus, +) { + const admitted = verifyCoordinatorPullAdmissionSource(installation, record), + authority = restoreAdmittedControlExecution(installation, admitted.execution), + state = executionAuthorityState(authority), + canonical = readTerminalCommitment(state), + terminal = terminalIdentity(state, status), + settlement = verifyControlEffectSettlement(installation, admitted.execution), + effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord( + state.installation, + "effects", + `${admitted.execution.operationKey}.json`, + 65536, + ), + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + if ( + !state.origin || + canonical.phase !== "canonical" || + terminal.terminal.state !== "succeeded" || + canonical.terminalDigest !== terminal.digest || + effect.outcome !== "committed" || + effect.publication !== "committed" || + effect.receipt.kind !== "ops-result" || + canonicalJson(terminal.terminal.result) !== canonicalJson(effect.receipt.result) + ) + throw Error("Actual successful contained coordinator pull required"); + assertFamilyCanonicalCompatibility( + state.installation, + state.origin, + admitted.execution, + canonical.terminalDigest, + canonical.terminalAt, + ); + const { changed: _changed, ...result } = servicePullResultSchema.parse(effect.receipt.result), + fixed = record.privateCoordinatorLifecycle!; + if ( + fixed.action !== "update" || + result.revisionId !== fixed.pull.operation.expectedConfigurationRevisionId + ) + throw Error("Original coordinator pull revision changed"); + const witness = coordinatorPullWitnessSchema.parse( + signControl( + state.installation.privateKey, + coordinatorPullObservationDomain, + coordinatorPullWitnessPayloadSchema.parse({ + version: 2, + installationId: installation.installationId, + maintenanceId: record.maintenanceId, + candidateId: record.candidateId, + maintenancePrepare, + pull: admitted.execution, + submissionDigest: coordinatorPullSubmissionDigest(fixed.pull), + operationFingerprint: admitted.execution.operationFingerprint, + apiParentDigest: controlDigest(admitted.parent), + sourceRevision: record.expectedSourceRevision, + ...result, + terminalReceiptDigest: terminal.digest, + effectReceiptDigest: controlDigest(childSettlementState(settlement).evidence), + baseline: admitted.coordinatorPullBaseline, + baselineAdmissionDigest: controlDigest( + readPrivateRecord( + state.installation, + "executions", + `${admitted.execution.operationKey}.json`, + 65536, + ), + ), + verifiedAt: Date.now(), + }), + ), + ); + const proof = Object.freeze({}) as VerifiedCoordinatorPullBaseline; + baselineProofs.set( + proof, + Object.freeze({ + ...structuredClone(witness.payload), + expectedCurrentRevisionId: admitted.coordinatorPullBaseline.configurationRevisionId, + }), + ); + observedBaselineProofs.set(witness, proof); + return witness; +} +/** Copies actual retained source packets into the original indexed family. + * The initial empty projection is allocated before the DM guard. No caller + * supplies a decision, generation result, signature or authority digest. */ +export function publishConfigurationMaintenanceForwardWitness( + authority: VerifiedExecutionAuthority, +): void { + const root = executionAuthorityState(authority); + if ( + root.kind !== "configurationMaintenance.prepare" || + !root.origin || + root.installation.installation.role !== "ops-agent" || + root.installation.fixedChildId + ) + throw Error("Original maintenance forward publisher required"); + const raw = readPrivateRecord( + root.installation, + "executions", + `${root.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + name = `${uuid.parse(raw.payload?.operation?.maintenanceId)}.json`, + ownerRaw = readPrivateRecord(root.installation, "configuration-maintenance", name), + owner = verifyControl( + root.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + ownerRaw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + root.installation, + root.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name }, + canonicalJson(ownerRaw), + ); + assertConfigurationMaintenanceDecisionSource(owner); + if ( + owner.admissionDigest !== root.receiptDigest || + controlDigest(raw) !== root.receiptDigest || + canonicalJson(owner.plan.prepare) !== canonicalJson(root.execution) || + canonicalJson(owner.origin) !== canonicalJson(root.origin) + ) + throw Error("Original maintenance forward source changed"); + const receipt = owner.receipt, + graph = configurationMaintenanceForwardGraphSchema.parse({ + version: 1, + binding: receipt.binding, + prepare: receipt.prepare, + apiJobId: receipt.apiJobId, + parentOperationKey: receipt.parentOperationKey, + parentFingerprint: receipt.parentFingerprint, + parentDeadlineAt: receipt.parentDeadlineAt, + planDigest: receipt.planDigest, + serviceIds: receipt.serviceIds, + steps: receipt.steps, + }); + const decision = owner.apiControls.decide?.receiptDelivery + ? owner.apiControls.decide.commitment + : null; + if (decision) + verifyControl( + root.installation.installation.publicKey, + configurationMaintenanceDecisionCommitmentDomain, + decision, + configurationMaintenanceDecisionCommitmentPayloadSchema, + ); + const source = owner.apiControls.steps.find((value) => value.ordinal === 0); + let generation = null; + if (source?.canonical && source.receiptDelivery && decision) { + verifyConfigurationMaintenanceChildTerminal( + authority, + 0, + source.canonical.status, + source.canonical.terminalAt, + ); + generation = { + admission: readPrivateRecord( + root.installation, + "executions", + `${source.execution.operationKey}.json`, + 65536, + ), + terminal: readPrivateRecord( + root.installation, + "observations", + `terminal-${source.execution.operationKey}.json`, + 8192, + ), + effect: readPrivateRecord( + root.installation, + "effects", + `${source.execution.operationKey}.json`, + 65536, + ), + status: source.canonical.status, + }; + } + const packet = signControl( + root.installation.privateKey, + configurationMaintenanceForwardDomain, + configurationMaintenanceForwardSchema.parse({ + version: 1, + graph, + rootAdmission: raw, + decision, + generation, + }), + ), + bytes = canonicalJson(packet); + if (Buffer.byteLength(bytes) > 65536) + throw Error("Configuration maintenance forward witness capacity"); + const directory = executionWitnessDirectory(root.installation, "ops-agent", true); + try { + writeExecutionShared( + root.installation, + root.execution, + directory.role, + `${root.execution.operationKey}.configuration-maintenance.json`, + bytes, + ); + } finally { + directory.close(); + } +} +/** DM reads ONLY shared original signed packets and its pinned role keys. + * This does not inspect the sibling agent's private directory. The actual + * verified transport parent and anchored depth-two grant must match the + * exact original root graph, continue decision and committed generation. */ +export function verifyConfigurationMaintenanceGenerationWitness( + installation: ControlInstallation, + value: unknown, + parent: import("../ops/execution-identity").VerifiedParentExecution, + grant: ReturnType, + expected: { + maintenanceId: string; + candidateId: string; + ownerPrepare: ExecutionRef; + revisionId: string; + hardlinkPlanDigest: string; + coordinatorMode: "ensure-running" | "require-running"; + serviceIds: readonly string[]; + rootCertificateDigest: string; + }, +) { + const local = installationState(installation), + request = startGenerationHardlinksExecutionSchema.parse(value); + if ( + installation.role !== "data-manager" || + local.fixedChildId || + !verifiedParents.has(parent) || + parent.issuer !== "ops-agent" || + !parent.execution || + canonicalJson(parent.execution) !== canonicalJson(grant.parent) || + canonicalJson(grant.owner) !== canonicalJson(expected.ownerPrepare) || + grant.ownerCertificateDigest !== expected.rootCertificateDigest + ) + throw Error("Actual original maintenance middle parent required"); + const directory = executionWitnessDirectory(local, "ops-agent"); + try { + const packet = verifyControl( + local.publicKeys["ops-agent"], + configurationMaintenanceForwardDomain, + JSON.parse( + directory.role + .read(`${expected.ownerPrepare.operationKey}.configuration-maintenance.json`, 65536) + .toString("utf8"), + ), + configurationMaintenanceForwardSchema, + ), + graph = packet.graph; + const admissionSchema = z + .object({ + version: z.literal(2), + installationId: uuid, + origin: controlOriginSchema, + execution: executionRefSchema, + kind: z.string(), + operation: opsOperationSchema, + parent: retainedParentExecutionSchema, + admittedAt: z.number().int().nonnegative().safe(), + requestedDeadlineAt: z.number().int().positive().safe(), + serviceIds: targetSchema, + plan: z.array(z.unknown()), + reservations: z.array(z.lazy(() => controlChildReservationSchema)).optional(), + }) + .passthrough(); + const root = verifyControl( + local.publicKeys["ops-agent"], + "openmapx-control-execution-v2", + packet.rootAdmission, + admissionSchema, + ), + original = readControlExecutionWitness( + local, + expected.ownerPrepare, + local.publicKeys["ops-agent"], + ), + operation = root.operation, + first = graph.steps[0]; + if ( + !packet.decision || + !packet.generation || + operation.kind !== "configurationMaintenance.prepare" || + !first || + root.installationId !== installation.installationId || + root.kind !== operation.kind || + root.parent.issuer !== "api" || + original.authorityDigest !== controlDigest(packet.rootAdmission) || + canonicalJson(root.origin) !== canonicalJson(original.origin) || + canonicalJson(root.execution) !== canonicalJson(expected.ownerPrepare) || + canonicalJson(graph.prepare) !== canonicalJson(root.execution) || + opsOperationFingerprint(operation) !== root.execution.operationFingerprint || + root.execution.fingerprint !== + executionFingerprint({ + kind: operation.kind, + operationFingerprint: root.execution.operationFingerprint, + deadlineAt: root.execution.deadlineAt, + parent: root.parent, + }) || + root.parent.apiJobId !== graph.apiJobId || + root.parent.operationKey !== graph.parentOperationKey || + root.parent.fingerprint !== graph.parentFingerprint || + root.parent.deadlineAt !== graph.parentDeadlineAt || + operation.maintenanceId !== expected.maintenanceId || + operation.candidateId !== expected.candidateId || + operation.revisionId !== expected.revisionId || + operation.coordinatorMode !== expected.coordinatorMode || + request.input.planDigest !== expected.hardlinkPlanDigest || + request.input.revisionId !== expected.revisionId || + request.input.coordinatorMode !== expected.coordinatorMode || + canonicalJson(root.serviceIds) !== canonicalJson(expected.serviceIds) || + canonicalJson(graph.serviceIds) !== canonicalJson(root.serviceIds) + ) + throw Error("Original maintenance forward graph identity conflict"); + const { kind: _kind, ...binding } = operation; + if (canonicalJson(binding) !== canonicalJson(graph.binding)) + throw Error("Original maintenance graph binding conflict"); + // Private candidate digests/source metadata are deliberately NOT shared. + // These placeholders do not enter the graph digest or reservations. + const rebuilt = buildFixedConfigurationMaintenancePlan({ + installationId: installation.installationId, + prepare: root.execution, + operation, + apiParent: root.parent, + apply: first.operation as Parameters< + typeof buildFixedConfigurationMaintenancePlan + >[0]["apply"], + candidateRecordDigest: "0".repeat(64), + sourceAuthorityRevision: "not-shared", + previewDigest: "0".repeat(64), + hardlinkPlan: request.plan, + serviceIds: root.serviceIds, + recreateServiceIds: graph.steps.flatMap((step) => + step.operation.kind === "service.recreate" || + step.operation.kind === "service.recreateIsolated" + ? [step.operation.serviceId] + : [], + ), + }); + const middle = rebuilt.steps.find((step) => step.phase === "maintenance-hardlinks"), + middleReservation = rebuilt.reservations.find( + (step) => step.phase === "maintenance-hardlinks", + ); + if ( + !middle || + !middleReservation || + !rebuilt.nestedHardlinks || + graph.planDigest !== rebuilt.planDigest || + canonicalJson(graph.steps) !== canonicalJson(rebuilt.steps) || + canonicalJson(root.reservations) !== canonicalJson(rebuilt.reservations) || + parent.execution.operationKey !== middle.operationKey || + parent.execution.operationFingerprint !== middle.operationFingerprint || + parent.execution.deadlineAt > root.execution.deadlineAt || + parent.fingerprint !== parent.execution.fingerprint || + parent.execution.fingerprint !== + executionFingerprint({ + kind: middle.operation.kind, + operationFingerprint: middle.operationFingerprint, + deadlineAt: parent.execution.deadlineAt, + parent: retainedControlParent(root.execution), + }) || + grant.phase !== "hardlinks" || + grant.kind !== "generation-hardlinks" || + grant.recipient !== "data-manager" || + canonicalJson(grant.execution) !== + canonicalJson({ + ...rebuilt.nestedHardlinks.execution, + deadlineAt: grant.execution.deadlineAt, + operationFingerprint: rebuilt.nestedHardlinks.operationFingerprint, + fingerprint: grant.execution.fingerprint, + }) || + grant.execution.operationKey !== request.operationKey || + grant.execution.operationFingerprint !== controlDigest(request.input) || + grant.execution.deadlineAt !== request.requestedDeadlineAt || + grant.execution.deadlineAt > parent.deadlineAt || + grant.execution.fingerprint !== + executionFingerprint({ + kind: "generation-hardlinks", + operationFingerprint: controlDigest(request.input), + deadlineAt: grant.execution.deadlineAt, + parent, + }) || + canonicalJson(grant.serviceIds) !== canonicalJson(root.serviceIds) + ) + throw Error("Original frozen middle/sole leaf conflict"); + const decision = verifyControl( + local.publicKeys["ops-agent"], + configurationMaintenanceDecisionCommitmentDomain, + packet.decision, + configurationMaintenanceDecisionCommitmentPayloadSchema, + ), + identity = decision.identity; + if ( + identity.installationId !== installation.installationId || + canonicalJson(identity.origin) !== canonicalJson(root.origin) || + canonicalJson(identity.ownerPrepare) !== canonicalJson(root.execution) || + identity.admissionDigest !== controlDigest(packet.rootAdmission) || + identity.rootCertificateDigest !== expected.rootCertificateDigest || + identity.maintenanceId !== expected.maintenanceId || + identity.candidateId !== expected.candidateId || + identity.planDigest !== graph.planDigest || + identity.settlementId !== rebuilt.dmSettle.settlementId || + decision.operation.decision !== "continue" || + decision.operation.maintenanceId !== expected.maintenanceId || + decision.operation.candidateId !== expected.candidateId || + decision.operation.planDigest !== graph.planDigest || + decision.operation.prepareOperationKey !== root.execution.operationKey || + decision.operation.prepareFingerprint !== root.execution.operationFingerprint || + decision.apiParent.apiJobId !== graph.apiJobId || + decision.apiParent.operationKey !== graph.parentOperationKey || + decision.apiParent.fingerprint !== graph.parentFingerprint || + decision.apiParent.deadlineAt !== graph.parentDeadlineAt || + decision.execution.operationFingerprint !== opsOperationFingerprint(decision.operation) || + decision.execution.fingerprint !== + executionFingerprint({ + kind: decision.operation.kind, + operationFingerprint: decision.execution.operationFingerprint, + deadlineAt: decision.execution.deadlineAt, + parent: decision.apiParent, + }) || + decision.acceptedAt >= decision.execution.deadlineAt || + decision.result.decision?.decision !== "continue" || + decision.result.decision.decisionId !== decision.operation.decisionId || + decision.result.state !== "authorized" || + decision.result.dmFence?.state !== "prepared" + ) + throw Error("Actual original continue commitment required"); + const generation = packet.generation, + admitted = verifyControl( + local.publicKeys["ops-agent"], + "openmapx-control-execution-v2", + generation.admission, + admissionSchema, + ), + childWitness = readControlExecutionWitness( + local, + admitted.execution, + local.publicKeys["ops-agent"], + ), + terminal = verifyControl( + local.publicKeys["ops-agent"], + "openmapx-control-terminal-v1", + generation.terminal, + terminalCommitmentSchema, + ), + effect = verifyControl( + local.publicKeys["ops-agent"], + "openmapx-control-settlement-v1", + generation.effect, + settlementSchema.extend({ receipt: effectObservationSchema }), + ), + status = generation.status; + if ( + childWitness.authorityDigest !== controlDigest(generation.admission) || + admitted.installationId !== installation.installationId || + canonicalJson(admitted.origin) !== canonicalJson(root.origin) || + canonicalJson(admitted.operation) !== canonicalJson(first.operation) || + canonicalJson(admitted.parent) !== canonicalJson(retainedControlParent(root.execution)) || + admitted.execution.operationKey !== first.operationKey || + admitted.execution.operationFingerprint !== first.operationFingerprint || + admitted.execution.deadlineAt > root.execution.deadlineAt || + admitted.execution.fingerprint !== + executionFingerprint({ + kind: first.operation.kind, + operationFingerprint: first.operationFingerprint, + deadlineAt: admitted.execution.deadlineAt, + parent: admitted.parent, + }) || + terminal.phase !== "canonical" || + terminal.installationId !== installation.installationId || + canonicalJson(terminal.origin) !== canonicalJson(root.origin) || + canonicalJson(terminal.execution) !== canonicalJson(admitted.execution) || + terminal.admissionDigest !== controlDigest(generation.admission) || + terminal.terminalDigest !== controlTerminalReceiptDigest(status) || + canonicalJson(effect.identity) !== canonicalJson(admitted.execution) || + effect.outcome !== "committed" || + effect.publication !== "committed" || + effect.terminalReceiptDigest !== controlDigest(effect.receipt) || + effect.receipt.kind !== "ops-result" || + canonicalJson(parseBoundedOpsResult(first.operation.kind, effect.receipt.result)) !== + canonicalJson(status.result) || + status.state !== "succeeded" || + status.kind !== first.operation.kind || + status.operationId !== admitted.execution.executionId || + status.operationKey !== admitted.execution.operationKey || + status.fingerprint !== admitted.execution.fingerprint || + status.deadlineAt !== admitted.execution.deadlineAt || + (status.result as { outcome?: string; revisionId?: string })?.outcome !== "committed" || + (status.result as { revisionId?: string })?.revisionId !== expected.revisionId + ) + throw Error("Actual original committed generation required"); + return Object.freeze({ + witnessDigest: controlDigest(packet), + decisionId: decision.operation.decisionId, + generation: Object.freeze(admitted.execution), + }); + } finally { + directory.close(); + } +} +declare const maintenanceChildBrand: unique symbol; +export type VerifiedConfigurationMaintenanceChild = Readonly<{ [maintenanceChildBrand]: true }>; +const maintenanceChildren = new WeakMap< + VerifiedConfigurationMaintenanceChild, + { + rootDigest: string; + ordinal: number; + execution: ExecutionRef; + status: OpsJobStatus; + terminalDigest: string; + terminalAt: number; + effectSettlementDigest: string; + settlement: ChildSettlement; + } +>(); +/** The status is only a canonical preimage. The actual original admission, + * fixed phase, private effect, private canonical tuple/time and family index + * must ALL agree; a public status/result/digest cannot create this proof. */ +export function verifyConfigurationMaintenanceChildTerminal( + authority: VerifiedExecutionAuthority, + ordinal: number, + input: OpsJobStatus, + terminalAt: number, +): VerifiedConfigurationMaintenanceChild { + const root = executionAuthorityState(authority); + if ( + root.kind !== "configurationMaintenance.prepare" || + !root.origin || + root.installation.installation.role !== "ops-agent" || + root.installation.fixedChildId + ) + throw Error("Original maintenance child source required"); + const admitted = readPrivateRecord( + root.installation, + "executions", + `${root.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + name = `${uuid.parse(admitted.payload?.operation?.maintenanceId)}.json`, + raw = readPrivateRecord(root.installation, "configuration-maintenance", name), + owner = verifyControl( + root.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + root.installation, + root.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name }, + canonicalJson(raw), + ); + assertConfigurationMaintenanceDecisionSource(owner); + const step = owner.plan.steps[ordinal], + source = owner.apiControls.steps.find((value) => value.ordinal === ordinal), + slot = root.reservations.find((value) => value.ordinal === ordinal), + child = slot && lookupControlChildPhase(authority, slot); + if ( + !step || + !source?.grant || + !child || + canonicalJson(owner.plan.prepare) !== canonicalJson(root.execution) || + owner.admissionDigest !== root.receiptDigest || + canonicalJson(source.execution) !== canonicalJson(child.execution) + ) + throw Error("Original frozen maintenance child required"); + const childAuthority = restoreAdmittedControlExecution( + root.installation.installation, + child.execution, + ), + state = executionAuthorityState(childAuthority), + admission = readPrivateRecord( + root.installation, + "executions", + `${child.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: unknown; parent?: unknown } }, + canonical = readTerminalCommitment(state), + status = parseOpsJobStatus(input), + settlement = childSettlementState( + verifyControlEffectSettlement(root.installation.installation, child.execution), + ).evidence, + effect = verifyControl( + root.installation.installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord( + root.installation, + "effects", + `${child.execution.operationKey}.json`, + 65536, + ), + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + const nonCommit = + step.phase === "configuration-commit" && + effect.receipt.kind === "configuration-generation-not-committed" && + settlement.outcome === "contained" && + settlement.publication === "not_started" && + canonicalJson( + inspectConfigurationGenerationNonCommit(root.installation.installation, child.execution) + .result, + ) === canonicalJson(status.result), + committed = + settlement.outcome === "committed" && + settlement.publication === "committed" && + effect.receipt.kind === "ops-result" && + canonicalJson(parseBoundedOpsResult(step.operation.kind, effect.receipt.result)) === + canonicalJson(status.result); + if ( + canonical.phase !== "canonical" || + canonical.terminalAt !== terminalAt || + canonical.terminalDigest !== controlTerminalReceiptDigest(status) || + status.state !== "succeeded" || + status.kind !== step.operation.kind || + status.operationId !== child.execution.executionId || + status.operationKey !== child.execution.operationKey || + status.fingerprint !== child.execution.fingerprint || + status.deadlineAt !== child.execution.deadlineAt || + canonicalJson(admission.payload?.operation) !== canonicalJson(step.operation) || + canonicalJson(admission.payload?.parent) !== + canonicalJson(retainedControlParent(root.execution)) || + !(committed || nonCommit) + ) + throw Error("Actual committed maintenance child canonical/effect source required"); + if ( + step.phase === "configuration-commit" && + !nonCommit && + (!(status.result && typeof status.result === "object") || + (status.result as { outcome?: string }).outcome !== "committed" || + (status.result as { revisionId?: string }).revisionId !== owner.plan.operation.revisionId) + ) + throw Error("Original candidate generation commit required"); + assertFamilyCanonicalCompatibility( + root.installation, + root.origin, + child.execution, + canonical.terminalDigest, + canonical.terminalAt, + ); + const { diagnostics: _diagnostics, ...safe } = status, + value = Object.freeze({}) as VerifiedConfigurationMaintenanceChild; + const freezeSource = (data: T): T => { + if (data && typeof data === "object" && !Object.isFrozen(data)) { + for (const child of Object.values(data)) freezeSource(child); + Object.freeze(data); + } + return data; + }; + maintenanceChildren.set( + value, + freezeSource({ + rootDigest: root.receiptDigest, + ordinal, + execution: child.execution, + status: safe, + terminalDigest: canonical.terminalDigest, + terminalAt: canonical.terminalAt, + effectSettlementDigest: controlDigest(settlement), + settlement, + }), + ); + return value; +} +export function configurationMaintenanceChildState( + value: VerifiedConfigurationMaintenanceChild, + authority: VerifiedExecutionAuthority, +) { + const source = maintenanceChildren.get(value), + root = executionAuthorityState(authority); + if ( + !source || + source.rootDigest !== root.receiptDigest || + root.kind !== "configurationMaintenance.prepare" + ) + throw Error("Actual original maintenance child proof required"); + return source; +} const recovery = new WeakMap(); export function verifyControlRecoveryAuthority( authority: VerifiedExecutionAuthority, @@ -1855,7 +3441,8 @@ export function publishDataControlTerminalWitness( const s = executionAuthorityState(authority); if (s.installation.installation.role !== "data-manager" || s.installation.fixedChildId) throw Error("Original data journal required"); - verifyRetainedArtifactTargetRootRelease(authority); + if (s.kind === "generation-hardlinks") verifyRetainedGenerationHardlinksSettlement(authority); + else verifyRetainedArtifactTargetRootRelease(authority); const expected = controlDigest({ version: 1, status: dataExecutionStatusSchema.parse(status) }); const original = verifyControl( s.installation.installation.publicKey, @@ -2172,6 +3759,97 @@ function withControlExecutionForward(authority: VerifiedExecutionAuthority, r /** Internal publication scope. All filesystem writes complete before guards * release; callers may not await HTTP/DB or acquire a lease from this scope. */ const publicationScopes = new WeakMap(); +/** Original maintenance recovery publication only. It serializes with every + * ordinary grant, including after cancellation/expiry, and grants no dispatch. */ +export function withControlMaintenanceNoGuardPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + return withMaintenanceRecoveryPublication(authority, run); +} +function withMaintenanceRecoveryPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const s = executionAuthorityState(authority); + if ( + s.kind !== "configurationMaintenance.prepare" || + !s.origin || + s.installation.fixedChildId || + s.installation.installation.role !== "ops-agent" || + s.origin.rootKey !== s.execution.operationKey + ) + throw Error("Original maintenance issuance freeze required"); + const own = () => + withExecutionWriter(s.installation, s.execution, (directory) => { + readLifecycle(s.installation, directory, s.execution, s.installation.installation.publicKey); + return run(); + }); + const assertion = retainedParentAssertion(s); + return assertion ? withParentLifecycles(s.installation, assertion, () => own()) : own(); +} +/** Purpose-only metadata serialization for the original logical guard control. + * This creates no forward authority, child binding or dispatch ticket. The + * fixed producer still consumes its restored actual final proof. */ +export function withControlMaintenanceSettlementPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + return withMaintenanceRecoveryPublication(authority, () => { + const state = executionAuthorityState(authority), + admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + raw = readPrivateRecord( + state.installation, + "configuration-maintenance", + `${uuid.parse(admitted.payload?.operation?.maintenanceId)}.json`, + ), + owner = verifyControl( + state.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + state.installation, + state.origin!, + { + scope: "private", + namespace: "configuration-maintenance", + container: "", + name: `${owner.plan.operation.maintenanceId}.json`, + }, + canonicalJson(raw), + ); + if ( + !owner.final || + !owner.inspection || + owner.noGuard || + owner.admissionDigest !== state.receiptDigest || + canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) + ) + throw Error("Original final-only settlement publication required"); + return run(); + }); +} +/** Initial private owner publication precedes the first ordinary grant. */ +export function withControlMaintenanceInitialization( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const s = executionAuthorityState(authority); + if ( + s.kind !== "configurationMaintenance.prepare" || + s.installation.fixedChildId || + s.installation.installation.role !== "ops-agent" + ) + throw Error("Original maintenance initialization required"); + return withControlExecutionForward(authority, run); +} export function assertControlPublicationScope( authority: VerifiedExecutionAuthority, ): AnchoredDirectory { @@ -2195,6 +3873,62 @@ export function withControlDelegationPublication( s.execution, s.installation.installation.publicKey, ); + if (s.kind === "configurationMaintenance.prepare") { + const admission = readPrivateRecord( + s.installation, + "executions", + `${s.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + raw = readPrivateRecord( + s.installation, + "configuration-maintenance", + `${uuid.parse(admission.payload?.operation?.maintenanceId)}.json`, + ), + owner = verifyControl( + s.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + verifyFamilyOwnedRecord( + s.installation, + s.origin!, + { + scope: "private", + namespace: "configuration-maintenance", + container: "", + name: `${admission.payload!.operation!.maintenanceId}.json`, + }, + canonicalJson(raw), + ); + if ( + owner.admissionDigest !== s.receiptDigest || + owner.issuance !== "open" || + canonicalJson(owner.plan.prepare) !== canonicalJson(s.execution) || + canonicalJson(owner.origin) !== canonicalJson(s.origin) || + canonicalJson(owner.plan.reservations) !== canonicalJson(s.reservations) + ) + throw Error("Original maintenance issuer changed"); + assertConfigurationMaintenanceDecisionSource(owner); + if ( + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(phase) && + (canonicalJson(owner.receipt.prepare) !== canonicalJson(s.execution) || + !["authorized", "applying"].includes(owner.receipt.state) || + owner.receipt.decision?.decision !== "continue" || + owner.receipt.dmFence?.state !== "prepared" || + owner.apiControls.decide?.operation.decision !== "continue" || + !owner.apiControls.decide.receiptDelivery) + ) + throw new ControlExecutionFencedError( + "Original authorized configuration decision required", + ); + } const prepared = (s.kind === "configurationMaintenance.prepare" && [ @@ -2231,6 +3965,41 @@ export function withControlDelegationPublication( export function assertControlExecutionForward(authority: VerifiedExecutionAuthority): void { withControlExecutionForward(authority, () => {}); } +/** Only a prepared maintenance owner's finite metadata decision may use its + * terminal preparation here. This scope issues no grant/forward handle and + * does not alter ordinary execution normalization or terminal graph rules. */ +export function withControlMaintenanceDecisionPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const s = executionAuthorityState(authority); + if (s.kind !== "configurationMaintenance.prepare" || !s.origin) + throw Error("Original prepared maintenance decision owner required"); + assertOriginClockForward(s.installation, s.origin); + const own = () => + withExecutionWriter(s.installation, s.execution, (directory) => { + const current = readLifecycle( + s.installation, + directory, + s.execution, + s.installation.installation.publicKey, + ); + if (current.phase !== "settled" || Date.now() >= s.execution.deadlineAt) + throw new ControlExecutionFencedError( + "Original prepared maintenance forward decision is fenced", + ); + verifyControlPreparedTerminal(s.installation, s.execution); + return run(); + }); + const assertion = retainedParentAssertion(s); + return assertion + ? withParentLifecycles(s.installation, assertion, (forward) => { + if (!forward) + throw new ControlExecutionFencedError("Canceled ancestor prevents maintenance continue"); + return own(); + }) + : own(); +} /** Only the protocol's frozen terminal preparation edges may outlive their local worker. */ export function assertControlDelegationForward( authority: VerifiedExecutionAuthority, @@ -2858,7 +4627,22 @@ export function fixedCliArguments(operation: import("../ops/contract").OpsOperat ...(operation.regionId ? ["--region", operation.regionId] : []), ]; case "service.start": - return ["services", "start", operation.serviceId]; + return [ + "services", + "start", + operation.serviceId, + "--expected-configuration-revision", + operation.expectedConfigurationRevisionId, + ]; + case "service.update": + return [ + "services", + "update", + operation.serviceId, + "--expected-configuration-revision", + operation.expectedConfigurationRevisionId, + ...(operation.noDeps ? ["--no-deps"] : []), + ]; case "data.link": return ["data", "link"]; case "data.clean": diff --git a/packages/core/src/build-artifacts/control-child-records.ts b/packages/core/src/build-artifacts/control-child-records.ts new file mode 100644 index 000000000..798e7c7fa --- /dev/null +++ b/packages/core/src/build-artifacts/control-child-records.ts @@ -0,0 +1,138 @@ +/** Immutable phase/address/reservation data only. No admission, dispatch or guard constructors. */ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { digest } from "./control-records"; + +export type ControlRecipient = "fixed-cli" | "data-manager" | "ops-agent" | "configuration-helper"; +export type ControlChildPhase = + | "lifecycle" + | "build" + | "data-link" + | "data-clean" + | "data-workflow" + | "hardlinks" + | "stop-primary" + | "stop-staging" + | "restart-staging" + | "promote-primary" + | "recover-primary" + | "proxy-stage" + | "proxy-commit" + | "proxy-restore" + | "paired-pin-commit" + | "configuration-fence" + | "configuration-fence-settle" + | "configuration-commit" + | "maintenance-hardlinks" + | "maintenance-recreate" + | "maintenance-handoff" + | "osm" + | "transit" + | "fonts" + | "data-step" + | "configuration-handoff"; +export const childPhaseKinds: Readonly> = { + lifecycle: ["service.start", "service.update", "system.update"], + build: ["service.build", "services.buildAll"], + "data-link": ["data.link", "configuration.hardlinks.apply"], + "data-clean": ["data.clean"], + "data-workflow": [ + "data.update", + "data.downloadOsm", + "data.downloadFonts", + "data.convertOverpass", + "data.overtureSync", + "data.overtureConflate", + "data.searchIndexBuild", + "data.notablePlacesBuild", + ], + hardlinks: ["generation-hardlinks"], + "stop-primary": ["motis.primary.stop"], + "stop-staging": ["motis.staging.stop"], + "restart-staging": ["motis.staging.restart"], + "promote-primary": ["motis.primary.promote"], + "recover-primary": ["motis.primary.restart"], + "proxy-stage": ["feedProxy.validateAndReload"], + "proxy-commit": ["feedProxy.validateAndReload"], + "proxy-restore": ["feedProxy.validateAndReload"], + "paired-pin-commit": ["transitousLock.approve"], + "configuration-fence": ["configuration-maintenance.prepare"], + "configuration-fence-settle": ["configuration-maintenance.settle"], + "configuration-commit": [ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ], + "maintenance-hardlinks": ["configuration.hardlinks.apply"], + "maintenance-recreate": ["service.recreate", "service.recreateIsolated"], + "maintenance-handoff": ["configurationHandoff.prepare"], + osm: ["download-osm"], + transit: ["transit-sync"], + fonts: ["download-fonts"], + "data-step": [ + "download-osm", + "download-fonts", + "convert-overpass", + "overture-sync", + "overture-conflate", + "search-index-build", + "notable-places-build", + ], + "configuration-handoff": ["configuration-helper"], +}; +export function controlChildOperationKey( + parentKey: string, + phase: ControlChildPhase, + ordinal: number, +): string { + if ( + !/^opk1_[A-Za-z0-9_-]{16,64}$/.test(parentKey) || + !Number.isSafeInteger(ordinal) || + ordinal < 0 || + ordinal > 63 || + !Object.hasOwn(childPhaseKinds, phase) + ) + throw Error("Invalid fixed phase key"); + return `opk1_${createHash("sha256").update(`openmapx-control-child-v1\0${parentKey}\0${phase}\0${ordinal}`).digest("base64url").slice(0, 32)}`; +} +export type ControlChildReservation = { + version: 1; + phase: ControlChildPhase; + ordinal: number; + kind: string; + recipient: ControlRecipient; + serviceIds: readonly string[]; + execution: Pick; + operationFingerprint: string; +}; +const address = z + .strictObject({ + executor: executionRefSchema.shape.executor, + principal: executionRefSchema.shape.principal, + executionId: z.string(), + operationKey: executionRefSchema.shape.operationKey, + }) + .refine((v) => + v.executor === "ops-agent" + ? /^job1_[A-Za-z0-9_-]{16,64}$/.test(v.executionId) + : z.uuid().safeParse(v.executionId).success, + ); +export const controlChildReservationSchema: z.ZodType = z.lazy(() => + z.strictObject({ + version: z.literal(1), + phase: z.enum(Object.keys(childPhaseKinds) as [ControlChildPhase, ...ControlChildPhase[]]), + ordinal: z.number().int().min(0).max(63), + kind: z.string().min(1).max(128), + recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), + serviceIds: z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine((a) => a.every((v, i) => i === 0 || (a[i - 1] ?? v) < v)), + execution: address, + operationFingerprint: digest, + }), +); diff --git a/packages/core/src/build-artifacts/control-records.ts b/packages/core/src/build-artifacts/control-records.ts index 9e5600547..a2ab4ad17 100644 --- a/packages/core/src/build-artifacts/control-records.ts +++ b/packages/core/src/build-artifacts/control-records.ts @@ -1,9 +1,10 @@ /** Signed record primitives; contains no admission or retirement constructors. */ import { createHash, createPublicKey, type KeyObject, sign, verify } from "node:crypto"; import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; import { type ControlExecutionOrigin, parseControlRootOperationKey } from "../ops/control-root-key"; import type { ControlIssuer, ExecutionRef } from "../ops/execution-identity"; -import { canonicalJson } from "../ops/fingerprint"; +import type { ControlRetentionPreparationContext } from "./coordinator-pull-records"; import type { AnchoredDirectory } from "./filesystem"; declare const installationBrand: unique symbol; @@ -23,7 +24,10 @@ export interface InstallationState { dataRoot: string; registeredTargets: ReadonlySet; retentionJournal?: { - prepare?(executions: readonly { execution: ExecutionRef; digest?: string }[]): Promise; + prepare?( + executions: readonly { execution: ExecutionRef; digest?: string }[], + context?: ControlRetentionPreparationContext, + ): Promise; contained(execution: ExecutionRef, digest: string | undefined): boolean; retire( executions: readonly { execution: ExecutionRef; digest: string; terminalAt?: number }[], diff --git a/packages/core/src/build-artifacts/control-retention.test.ts b/packages/core/src/build-artifacts/control-retention.test.ts index 8898d96bd..83ab85b8a 100644 --- a/packages/core/src/build-artifacts/control-retention.test.ts +++ b/packages/core/src/build-artifacts/control-retention.test.ts @@ -8,6 +8,7 @@ import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; import { createControlRootOperationKey } from "../ops/control-root-key"; import { executionFingerprint } from "../ops/execution-identity"; import { opsOperationFingerprint } from "../ops/fingerprint"; +import { observeConfigurationSources } from "./configuration-source-witness"; import * as c from "./control-authority"; import { advanceRetentionClock } from "./retention-clock"; import { markFamilyTerminal, publishFamilyCanonicalTerminal } from "./retention-family"; @@ -138,6 +139,14 @@ it("allows only prepared configuration refinement and freezes its canonical tupl kind: "stack.render" as const, revisionId: `cfg1_${"a".repeat(43)}`, expectedCurrentRevisionId: null, + expectedSourceRevision: observeConfigurationSources({ + installation, + infraDir: root, + services: [], + authorityRevision: "retention-unit", + env: {}, + }).sourceRevision, + coordinatorMode: "require-running" as const, }; const admittedAt = Date.now(), deadlineAt = admittedAt + 60000, diff --git a/packages/core/src/build-artifacts/coordinator-pull-records.ts b/packages/core/src/build-artifacts/coordinator-pull-records.ts new file mode 100644 index 000000000..9c775f517 --- /dev/null +++ b/packages/core/src/build-artifacts/coordinator-pull-records.ts @@ -0,0 +1,130 @@ +/** Bounded copied observation, never a target owner/grant or final evidence. */ +import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { executionRefSchema } from "../ops/execution-ref"; +import { ownedServiceSnapshotSchema } from "../ops/owned-service-records"; +import { servicePullResultSchema } from "../ops/service-operation-records"; +export const coordinatorPullObservationDomain = "openmapx-coordinator-pull-observation-v2"; +const digest = z.string().regex(/^[a-f0-9]{64}$/); +export const coordinatorPullBaselineSchema = z + .strictObject({ + version: z.literal(1), + candidateId: z.uuid(), + sourceRevision: z.uuid(), + configurationRevisionId: servicePullResultSchema.shape.revisionId, + configurationAuthorityRevision: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/), + policyRevision: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/), + submissionDigest: digest, + observedAt: z.number().int().nonnegative().safe(), + configImageReference: servicePullResultSchema.shape.imageReference, + composeSourcesDigest: digest, + container: ownedServiceSnapshotSchema.refine( + (v) => v.serviceId === "data-manager" && v.state === "running", + ), + }) + .refine((v) => Buffer.byteLength(canonicalJson(v)) <= 3072); +export type CoordinatorPullBaseline = z.infer; +export const coordinatorPullWitnessPayloadSchema = z.strictObject({ + version: z.literal(2), + installationId: z.uuid(), + maintenanceId: z.uuid(), + candidateId: z.uuid(), + maintenancePrepare: executionRefSchema, + pull: executionRefSchema, + submissionDigest: digest, + operationFingerprint: digest, + apiParentDigest: digest, + sourceRevision: z.uuid(), + revisionId: servicePullResultSchema.shape.revisionId, + imageReference: servicePullResultSchema.shape.imageReference, + imageId: servicePullResultSchema.shape.imageId, + releaseSelectionDigest: servicePullResultSchema.shape.releaseSelectionDigest, + terminalReceiptDigest: digest, + effectReceiptDigest: digest, + baseline: coordinatorPullBaselineSchema, + baselineAdmissionDigest: digest, + verifiedAt: z.number().int().nonnegative().safe(), +}); +export const coordinatorPullWitnessSchema = z + .strictObject({ + payload: coordinatorPullWitnessPayloadSchema, + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), + }) + .refine((v) => Buffer.byteLength(canonicalJson(v)) <= 8192); +export type CoordinatorPullWitness = z.infer; + +/** Historical source facts, distinct from a new owner's pull witness. */ +export const stagedCoordinatorSourceSchema = z.strictObject({ + execution: executionRefSchema, + admittedAt: z.number().int().nonnegative().safe(), + baseline: coordinatorPullBaselineSchema, + admissionDigest: digest, + canonicalDigest: digest, + effectDigest: digest, + apiCopyDigest: digest, +}); +export const stagedCoordinatorObservationDomain = + "openmapx-maintenance-staged-coordinator-observation-v1"; +export const stagedCoordinatorObservationPayloadSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + ownerPrepare: executionRefSchema, + candidateId: z.uuid(), + planDigest: digest, + source: stagedCoordinatorSourceSchema, + capturedAt: z.number().int().nonnegative().safe(), +}); +export const stagedCoordinatorObservationSchema = z + .strictObject({ + payload: stagedCoordinatorObservationPayloadSchema, + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), + }) + .refine((value) => Buffer.byteLength(canonicalJson(value)) <= 8192); +export type StagedCoordinatorObservation = z.infer; + +/** Collector-owned runtime token; no structural phase flag is authority. */ +declare const retentionPreparationBrand: unique symbol; +export type ControlRetentionPreparationContext = Readonly<{ [retentionPreparationBrand]: true }>; +export const stagedCoordinatorRetirementDomain = "openmapx-staged-coordinator-retirement-v1"; +export const stagedCoordinatorRankSchema = z.strictObject({ + admittedAt: z.number().int().nonnegative().safe(), + operationKey: executionRefSchema.shape.operationKey, +}); +export const stagedCoordinatorRetirementPayloadSchema = z + .strictObject({ + version: z.literal(1), + installationId: z.uuid(), + originDigest: digest, + familyId: z.uuid(), + generation: z.uuid(), + execution: executionRefSchema, + terminalDigest: digest, + sourceAdmissionDigest: digest, + oldContainerDigest: digest, + rank: stagedCoordinatorRankSchema, + contextMode: z.enum(["preparing", "before-commit", "committed-cleanup"]), + checkNonce: z.uuid(), + manifestDigest: digest.nullable(), + checkedAt: z.number().int().nonnegative().safe(), + condition: z.enum(["not-needed", "higher-ranked-successor"]), + successor: z + .strictObject({ + execution: executionRefSchema, + admissionDigest: digest, + canonicalDigest: digest, + effectDigest: digest, + apiCopyDigest: digest, + rank: stagedCoordinatorRankSchema, + }) + .nullable(), + }) + .refine((value) => (value.condition === "not-needed") === (value.successor === null)); +export const stagedCoordinatorRetirementPacketSchema = z + .strictObject({ + payload: stagedCoordinatorRetirementPayloadSchema, + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), + }) + .refine((value) => Buffer.byteLength(canonicalJson(value)) <= 4096); +export type StagedCoordinatorRetirementPacket = z.infer< + typeof stagedCoordinatorRetirementPacketSchema +>; diff --git a/packages/core/src/build-artifacts/data-configuration-fence-records.ts b/packages/core/src/build-artifacts/data-configuration-fence-records.ts new file mode 100644 index 000000000..622fa4aec --- /dev/null +++ b/packages/core/src/build-artifacts/data-configuration-fence-records.ts @@ -0,0 +1,41 @@ +/** Exact existing DM mirror/SQL data schema; decoding creates no guard authority. */ +import { z } from "zod"; +import { + dataConfigurationFencePrepareSchema, + dataConfigurationFenceReceiptSchema, + dataConfigurationFenceSettleSchema, + legacyDataConfigurationFenceSettleSchema, +} from "../services/data-manager/contracts/configuration-maintenance"; +import { configurationSettlementSqlControlSchema } from "./configuration-fence-settlement-records"; +import { controlOriginSchema } from "./control-records"; + +export const metadataSchema = z.strictObject({ + version: z.literal(1), + guardState: z.enum(["preparing", "prepared", "settling", "settled", "recovery_required"]), + receipt: dataConfigurationFenceReceiptSchema, + input: dataConfigurationFencePrepareSchema, + // Existing signed mirror/SQL bytes remain readable; the mutation boundary above + // accepts only the new reference. Legacy presence never authorizes a new effect. + settlement: z + .union([legacyDataConfigurationFenceSettleSchema, dataConfigurationFenceSettleSchema]) + .optional(), + settlementReservation: z + .strictObject({ + version: z.literal(2), + settlementId: z.string().regex(/^[a-f0-9]{64}$/), + projectionDigest: z.string().regex(/^[a-f0-9]{64}$/), + reservationDigest: z.string().regex(/^[a-f0-9]{64}$/), + capacityReceiptDigest: z.string().regex(/^[a-f0-9]{64}$/), + }) + .optional(), + settlementControl: z.lazy(() => configurationSettlementSqlControlSchema).optional(), +}); +export const mirrorSchema = z.strictObject({ + preparedAt: z.number().int().positive().safe(), + ownerCertificateDigest: z.string().regex(/^[a-f0-9]{64}$/), + prepareAssertion: z.string().max(4096), + version: z.literal(1), + installationId: z.uuid(), + origin: controlOriginSchema, + metadata: metadataSchema, +}); diff --git a/packages/core/src/build-artifacts/data-configuration-fence-witness-records.ts b/packages/core/src/build-artifacts/data-configuration-fence-witness-records.ts new file mode 100644 index 000000000..b45d11cb8 --- /dev/null +++ b/packages/core/src/build-artifacts/data-configuration-fence-witness-records.ts @@ -0,0 +1,30 @@ +/** Original preparation witness data; schema parsing grants no authority. */ +import { z } from "zod"; +import { executionRefSchema } from "../ops/execution-ref"; +import { dataConfigurationFenceReceiptSchema } from "../services/data-manager/contracts/configuration-maintenance"; +import { controlOriginSchema } from "./control-records"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); +export const dataConfigurationFencePreparedWitnessDomain = + "openmapx-data-configuration-prepared-v2"; +export const dataConfigurationFencePreparedWitnessSchema = z.strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + ownerPrepare: executionRefSchema, + prepareExecution: executionRefSchema, + prepareAdmissionDigest: digest, + rootCertificateDigest: digest, + preparedReceipt: dataConfigurationFenceReceiptSchema.refine( + (value) => value.state === "prepared" && value.sequence === 0, + ), + preparedReceiptDigest: digest, + preparedCanonicalDigest: digest, + preparedTerminalAt: z.number().int().positive().safe(), + guardDigest: digest, + effectReceiptDigest: digest, + effectSettlementDigest: digest, +}); +export type DataConfigurationFencePreparedWitness = z.infer< + typeof dataConfigurationFencePreparedWitnessSchema +>; diff --git a/packages/core/src/build-artifacts/data-configuration-fence-witness.ts b/packages/core/src/build-artifacts/data-configuration-fence-witness.ts new file mode 100644 index 000000000..d404b57ee --- /dev/null +++ b/packages/core/src/build-artifacts/data-configuration-fence-witness.ts @@ -0,0 +1,107 @@ +import { + type DataConfigurationFencePreparedWitness, + dataConfigurationFencePreparedWitnessDomain, + dataConfigurationFencePreparedWitnessSchema, +} from "./data-configuration-fence-witness-records"; + +export { + type DataConfigurationFencePreparedWitness, + dataConfigurationFencePreparedWitnessDomain, + dataConfigurationFencePreparedWitnessSchema, +} from "./data-configuration-fence-witness-records"; + +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { + configurationSettlementDomains, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import { installationState } from "./control-authority"; +import { controlDigest, verifyControl } from "./control-records"; +import { AnchoredDirectory } from "./filesystem"; +import { + type ArtifactTargetLease, + configurationMaintenanceTargetOwner, + inspectArtifactTargetLeaseChildSettlement, +} from "./target-lease"; + +declare const verifiedPreparedBrand: unique symbol; +export type VerifiedPreparedDataConfigurationFence = Readonly<{ [verifiedPreparedBrand]: true }>; +const verified = new WeakMap< + VerifiedPreparedDataConfigurationFence, + DataConfigurationFencePreparedWitness +>(); +function freeze(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); + } + return value; +} +export function preparedDataConfigurationFenceState( + value: VerifiedPreparedDataConfigurationFence, +): DataConfigurationFencePreparedWitness { + const state = verified.get(value); + if (!state) throw Error("Verified original DM preparation required"); + return state; +} +export function verifyPublishedDataConfigurationFencePrepared( + lease: ArtifactTargetLease, + execution: ExecutionRef, +): VerifiedPreparedDataConfigurationFence { + const owner = configurationMaintenanceTargetOwner(lease), + local = installationState(owner.installation); + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(local.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + role = own(executions.directory("data-manager")); + const value = verifyControl( + local.publicKeys["data-manager"], + dataConfigurationFencePreparedWitnessDomain, + JSON.parse( + role + .read(`${executionRefSchema.parse(execution).operationKey}.data-terminal.json`, 8192) + .toString("utf8"), + ), + dataConfigurationFencePreparedWitnessSchema, + ); + const effectReceipt = { + kind: "data-configuration-fence", + receipt: value.preparedReceipt, + guardDigest: value.guardDigest, + }; + const child = inspectArtifactTargetLeaseChildSettlement(lease, execution); + if ( + value.installationId !== owner.installation.installationId || + controlDigest(value.origin) !== controlDigest(owner.origin) || + controlDigest(value.ownerPrepare) !== controlDigest(owner.execution) || + controlDigest(value.prepareExecution) !== controlDigest(execution) || + controlDigest(value.preparedReceipt.ownerPrepare) !== controlDigest(owner.execution) || + value.rootCertificateDigest !== owner.certificateDigest || + value.preparedReceipt.fenceId !== execution.executionId || + value.preparedReceiptDigest !== + settlementRecordDigest( + configurationSettlementDomains.preparedReceipt, + value.preparedReceipt, + ) || + value.preparedCanonicalDigest !== + controlDigest({ version: 1, receipt: value.preparedReceipt }) || + value.effectReceiptDigest !== controlDigest(effectReceipt) || + child?.outcome !== "committed" || + child.publication !== "committed" || + child.terminalReceiptDigest !== value.effectReceiptDigest || + value.effectSettlementDigest !== controlDigest({ ...child, receipt: effectReceipt }) + ) + throw Error("Original DM preparation source mismatch"); + const handle = Object.freeze({}) as VerifiedPreparedDataConfigurationFence; + verified.set(handle, freeze(structuredClone(value))); + return handle; + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} diff --git a/packages/core/src/build-artifacts/data-configuration-fence.ts b/packages/core/src/build-artifacts/data-configuration-fence.ts index cfde74944..5ed73a536 100644 --- a/packages/core/src/build-artifacts/data-configuration-fence.ts +++ b/packages/core/src/build-artifacts/data-configuration-fence.ts @@ -1,8 +1,16 @@ +import { metadataSchema, mirrorSchema } from "./data-configuration-fence-records"; + +export { metadataSchema, mirrorSchema } from "./data-configuration-fence-records"; + /** Fixed service-only configuration/feed guard; signatures are verified against * the anchored original agent grant, never against the shared bearer alone. */ import { z } from "zod"; -import { executionFingerprint, executionRefSchema } from "../ops/execution-identity"; -import { canonicalJson } from "../ops/fingerprint"; +import { canonicalJson } from "../ops/canonical-json"; +import { + executionFingerprint, + executionRefSchema, + type VerifiedParentExecution, +} from "../ops/execution-identity"; import { type DataConfigurationFencePrepare, type DataConfigurationFenceReceipt, @@ -12,6 +20,41 @@ import { dataConfigurationFenceSettleSchema, } from "../services/data-manager/contracts/configuration-maintenance"; import { startGenerationHardlinksExecutionSchema } from "../services/data-manager/contracts/executions"; +import { + publishedConfigurationSettlementReservationState, + type RefusedConfigurationSettlementCapacity, + refusedConfigurationSettlementCapacityState, + reserveReceivedConfigurationSettlementCapacity, + verifyPublishedConfigurationSettlementReservation, +} from "./configuration-fence-settlement"; +import { + configurationSettlementRecordDescriptors, + readConfigurationSettlementCell, + writeConfigurationSettlementCell, +} from "./configuration-fence-settlement-cells"; +import { + configurationCommittedSettlementSchema, + configurationDmCapacityCellSchema, + configurationDmNoGuardRefusalSchema, + configurationSettlementAckSchema, + configurationSettlementAdmissionSchema, + configurationSettlementAssertionSchema, + configurationSettlementCapacitySchema, + configurationSettlementContainedObservationSchema, + configurationSettlementControlCellSchema, + configurationSettlementControlEnvelopeSchema, + configurationSettlementDomains, + configurationSettlementEffectSchema, + configurationSettlementPublishedTerminalSchema, + configurationSettlementReservationProjectionSchema, + configurationSettlementSqlControlSchema, + configurationSettlementSqlSessionSchema, + configurationSettlementTerminalTupleSchema, + configurationUnusedCapacityConfirmationSchema, + configurationUnusedCapacityIntentSchema, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import { readPreparedConfigurationSettlementRetirement } from "./configuration-fence-settlement-retention"; import { beginControlEffect, type ControlInstallation, @@ -19,24 +62,37 @@ import { controlHeaders, executionAuthorityState, installationState, + readControlExecutionWitness, readPrivateRecord, requireControlJournalRetention, restoreAdmittedControlExecution, retainAdmittedControlExecution, + retainedControlAdoptionAssertion, retainedControlParent, settleExecutionLifecycle, + verifyConfigurationMaintenanceGenerationWitness, verifyControlEffectSettlement, verifyControlPeer, withPrivateEffectWriter, writePrivateRecord, } from "./control-authority"; -import { controlOriginSchema, signControl, verifyControl } from "./control-records"; +import { signControl, verifyControl } from "./control-records"; +import { + dataConfigurationFencePreparedWitnessDomain, + dataConfigurationFencePreparedWitnessSchema, +} from "./data-configuration-fence-witness-records"; import { retainedDataDatabaseTimestamp } from "./data-database-time"; +import { AnchoredDirectory, sameInode } from "./filesystem"; import { verifiedParentForArtifactDelegation } from "./peer-admission"; import { + bindOwnedConfigurationSettlementFamilySlot, + commitRefusedDataConfigurationSettlementSlotUnused, completeFamilyJournal, + familyRecordWrite, markFamilyTerminal, publishFamilyCanonicalTerminal, + readConfigurationSettlementFamilySlot, + readRefusedConfigurationSettlementCapacitySnapshot, verifyFamilyOwnedRecord, } from "./retention-family"; import { @@ -45,7 +101,9 @@ import { bindBorrowedControlExecution, closeArtifactTargetLeaseHandle, inspectArtifactDelegation, + retainedConfigurationFenceAdoption, settleArtifactTargetLeaseChild, + verifyConfigurationFenceSettlementHeldOwner, verifyRetainedArtifactTargetSettlement, } from "./target-lease"; export interface DataFenceSql { @@ -54,23 +112,15 @@ export interface DataFenceSql { parameters?: (string | number | null)[], ): Promise[]>; transaction(run: (tx: Pick) => Promise): Promise; + reserveSettlementSession?(sessionNonce: string): Promise; +} +export interface DataFenceReservedSession { + unsafe: DataFenceSql["unsafe"]; + identity: { pid: number; backendStart: string; sessionNonce: string }; + /** Called ONLY after actual COMMIT or explicit awaited ROLLBACK. An unknown + * session stays reserved and cannot execute another logical transition. */ + releaseConclusive(): void; } -const metadataSchema = z.strictObject({ - version: z.literal(1), - guardState: z.enum(["preparing", "prepared", "settling", "settled", "recovery_required"]), - receipt: dataConfigurationFenceReceiptSchema, - input: dataConfigurationFencePrepareSchema, - settlement: dataConfigurationFenceSettleSchema.optional(), -}); -const mirrorSchema = z.strictObject({ - preparedAt: z.number().int().positive().safe(), - ownerCertificateDigest: z.string().regex(/^[a-f0-9]{64}$/), - prepareAssertion: z.string().max(4096), - version: z.literal(1), - installationId: z.uuid(), - origin: controlOriginSchema, - metadata: metadataSchema, -}); const fenceTerminalSchema = z.strictObject({ version: z.literal(1), phase: z.enum(["prepared", "canonical"]), @@ -93,6 +143,26 @@ export class DataFenceBusy extends Error { super("configuration_maintenance_busy"); } } +class DataFenceCapacityRefused extends DataFenceBusy { + constructor(readonly proof: RefusedConfigurationSettlementCapacity) { + super(); + } +} +declare const preparedGenerationBrand: unique symbol; +export type VerifiedPreparedGenerationAdmission = Readonly<{ [preparedGenerationBrand]: true }>; +const preparedGenerations = new WeakMap< + VerifiedPreparedGenerationAdmission, + { + installation: ControlInstallation; + request: ReturnType; + headers: readonly string[]; + parent: VerifiedParentExecution; + guardDigest: string; + execution: ReturnType["execution"]; + } +>(); +declare const settlementPeerBrand: unique symbol; +export type VerifiedFenceSettlementPeer = Readonly<{ [settlementPeerBrand]: true }>; export function createDataConfigurationFenceStore( installation: ControlInstallation, sql: DataFenceSql, @@ -102,7 +172,986 @@ export function createDataConfigurationFenceStore( throw Error("Fixed DM fence receiver required"); let startup = "closed" as "closed" | "open" | "fenced"; const mirrors = new Map>(); - function mirror(id: string) { + type SettlementSource = { + input: DataConfigurationFenceSettle; + saved: z.infer; + owned: z.infer; + projection: z.infer; + envelope: z.infer; + signedEnvelope: ReturnType< + typeof signControl> + >; + commitment: z.infer; + commitmentDigest: string; + capacityDigest: string; + prepareExecution: z.infer; + prepareAdmissionDigest: string; + preparedReceiptDigest: string; + }; + const settlementPeers = new WeakMap(); + const receiverEffectTickets = new WeakMap(); + function shared(role: "ops-agent" | "data-manager", name: string, max: number): unknown { + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(local.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + directory = own(executions.directory(role)); + const bytes = directory.read(name, max); + if (bytes.length !== max) throw new DataFenceBusy(); + return JSON.parse(bytes.toString("utf8")); + } finally { + for (const directory of opened.reverse()) directory.close(); + } + } + async function verifyConfigurationFenceSettlementControl( + input: DataConfigurationFenceSettle, + rawHeaders: readonly string[], + ): Promise { + const headers = controlHeaders(rawHeaders); + if (!headers.delegation || headers.parent) + throw Object.assign(Error("Fixed settlement assertion required"), { statusCode: 403 }); + const assertion = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.assertion, + JSON.parse(Buffer.from(headers.delegation, "base64url").toString("utf8")), + configurationSettlementAssertionSchema, + ), + source = await loadConfigurationFenceSettlementSource(input); + if ( + assertion.settlementId !== input.settlementId || + assertion.controlEnvelopeDigest !== + settlementRecordDigest(configurationSettlementDomains.binding, source.envelope) + ) + throw Object.assign(Error("Fixed settlement assertion binding conflict"), { + statusCode: 403, + }); + const peer = Object.freeze({}) as VerifiedFenceSettlementPeer; + settlementPeers.set(peer, source); + return peer; + } + /** Common exact source loader. It reads original guard SQL/mirror/private + * preparation plus shared role-signed projection/final/binding; no sibling + * private directory, public status or caller outcome supplies authority. */ + async function loadConfigurationFenceSettlementSource( + input: DataConfigurationFenceSettle, + readSql: Pick = sql, + ): Promise { + const saved = await row(readSql, input.maintenanceId); + if (!saved || !saved.settlementReservation) throw new DataFenceBusy(); + const mirrorSource = signedMirrorData(input.maintenanceId), + owned = mirrorSource.parsed, + owner = saved.receipt.ownerPrepare, + preparedGrant = inspectArtifactDelegation({ + installation, + dataRoot: local.dataRoot, + assertion: owned.prepareAssertion, + }), + prepareAuthority = restoreAdmittedControlExecution(installation, preparedGrant.execution), + prepare = executionAuthorityState(prepareAuthority), + canonicalRaw = readPrivateRecord( + local, + "observations", + `terminal-${preparedGrant.execution.operationKey}.json`, + 8192, + ), + effectRaw = readPrivateRecord( + local, + "effects", + `${preparedGrant.execution.operationKey}.json`, + 65536, + ), + canonical = verifyControl( + installation.publicKey, + "openmapx-data-fence-terminal-v1", + canonicalRaw, + fenceTerminalSchema, + ), + effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + effectRaw, + z + .object({ + identity: executionRefSchema, + outcome: z.literal("committed"), + publication: z.literal("committed"), + terminalReceiptDigest: z.string(), + receipt: z.strictObject({ + kind: z.literal("data-configuration-fence"), + receipt: dataConfigurationFenceReceiptSchema, + guardDigest: z.string(), + }), + }) + .passthrough(), + ), + originalMetadata = metadataSchema.parse({ + version: 1, + guardState: "prepared", + receipt: effect.receipt.receipt, + input: saved.input, + settlementReservation: saved.settlementReservation, + }), + originalMirror = { ...owned, metadata: originalMetadata }; + if (!prepare.origin) throw new DataFenceBusy(); + verifyFamilyOwnedRecord( + local, + prepare.origin, + { + scope: "private", + namespace: "observations", + container: "", + name: `terminal-${preparedGrant.execution.operationKey}.json`, + }, + canonicalJson(canonicalRaw), + ); + verifyFamilyOwnedRecord( + local, + prepare.origin, + { + scope: "private", + namespace: "effects", + container: "", + name: `${preparedGrant.execution.operationKey}.json`, + }, + canonicalJson(effectRaw), + ); + if ( + prepare.kind !== "configuration-maintenance.prepare" || + preparedGrant.kind !== "configuration-maintenance.prepare" || + preparedGrant.phase !== "configuration-fence" || + preparedGrant.recipient !== "data-manager" || + preparedGrant.ownerCertificateDigest !== owned.ownerCertificateDigest || + canonicalJson(preparedGrant.owner) !== canonicalJson(owner) || + canonicalJson(preparedGrant.parent) !== canonicalJson(owner) || + canonicalJson(preparedGrant.execution) !== canonicalJson(effect.identity) || + preparedGrant.execution.executionId !== saved.receipt.fenceId || + canonical.phase !== "canonical" || + canonicalJson(canonical.execution) !== canonicalJson(preparedGrant.execution) || + canonical.admissionDigest !== prepare.receiptDigest || + canonical.terminalDigest !== dataFenceTerminalDigest(effect.receipt.receipt) || + canonical.terminalAt !== owned.preparedAt || + effect.terminalReceiptDigest !== controlDigest(effect.receipt) || + effect.receipt.guardDigest !== controlDigest(originalMirror) || + effect.receipt.receipt.state !== "prepared" || + effect.receipt.receipt.sequence !== 0 || + canonicalJson(saved.receipt) !== canonicalJson(effect.receipt.receipt) || + input.candidateId !== saved.input.candidateId || + input.planDigest !== saved.input.planDigest || + input.settlementId !== saved.settlementReservation.settlementId + ) + throw new DataFenceBusy(); + const rootWitness = readControlExecutionWitness(local, owner, local.publicKeys["ops-agent"]), + control = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.control, + shared("ops-agent", `${owner.operationKey}.configuration-settlement-control.json`, 8192), + configurationSettlementControlCellSchema.shape.payload, + ); + if ("unused" in control || !control.binding) throw new DataFenceBusy(); + const projection = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + envelope = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.control, + control.binding, + configurationSettlementControlEnvelopeSchema, + ), + commitment = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.reference, + shared("ops-agent", `${owner.operationKey}.configuration-settlement.json`, 16384), + configurationCommittedSettlementSchema, + ), + commitmentDigest = settlementRecordDigest( + configurationSettlementDomains.commitment, + commitment, + ), + identity = commitment.identity, + preparedReceiptDigest = settlementRecordDigest( + configurationSettlementDomains.preparedReceipt, + effect.receipt.receipt, + ), + envelopeDigest = settlementRecordDigest(configurationSettlementDomains.binding, envelope), + { + operationFingerprint: _operationFingerprint, + fingerprint: _fingerprint, + deadlineAt: _deadlineAt, + ...address + } = envelope.execution, + descriptors = configurationSettlementRecordDescriptors( + projection, + preparedGrant.execution.operationKey, + ).receiver, + capacityCell = configurationDmCapacityCellSchema.parse( + readConfigurationSettlementCell(local, projection, descriptors[0]!), + ); + if (!capacityCell.capacity || capacityCell.refusal || capacityCell.unused) + throw new DataFenceBusy(); + const capacity = verifyControl( + installation.publicKey, + configurationSettlementDomains.capacity, + capacityCell.capacity, + configurationSettlementCapacitySchema, + ), + capacityDigest = settlementRecordDigest(configurationSettlementDomains.capacity, capacity); + if ( + projection.installationId !== installation.installationId || + canonicalJson(projection.origin) !== canonicalJson(owned.origin) || + canonicalJson(rootWitness.origin) !== canonicalJson(owned.origin) || + rootWitness.authorityDigest !== projection.rootAdmissionDigest || + canonicalJson(rootWitness.serviceIds) !== canonicalJson(saved.input.serviceIds) || + canonicalJson(projection.ownerPrepare) !== canonicalJson(owner) || + projection.rootCertificateDigest !== owned.ownerCertificateDigest || + projection.maintenanceId !== input.maintenanceId || + projection.candidateId !== input.candidateId || + projection.planDigest !== input.planDigest || + projection.settlementId !== input.settlementId || + canonicalJson(projection.operation) !== + canonicalJson({ kind: "configuration-maintenance.settle", ...input }) || + projection.reservation.operationFingerprint !== controlDigest(projection.operation) || + settlementRecordDigest(configurationSettlementDomains.projection, projection) !== + saved.settlementReservation.projectionDigest || + projection.reservationDigest !== saved.settlementReservation.reservationDigest || + capacityDigest !== saved.settlementReservation.capacityReceiptDigest || + capacity.projectionDigest !== saved.settlementReservation.projectionDigest || + capacity.reservationDigest !== projection.reservationDigest || + capacity.settlementId !== input.settlementId || + canonicalJson(capacity.proposedExecutionAddress) !== canonicalJson(address) || + canonicalJson(projection.reservation.execution) !== canonicalJson(address) || + envelope.projectionDigest !== capacity.projectionDigest || + envelope.reservationDigest !== projection.reservationDigest || + envelope.installationId !== installation.installationId || + canonicalJson(envelope.origin) !== canonicalJson(owned.origin) || + canonicalJson(envelope.ownerPrepare) !== canonicalJson(owner) || + envelope.admissionDigest !== rootWitness.authorityDigest || + envelope.rootCertificateDigest !== owned.ownerCertificateDigest || + envelope.settlementId !== input.settlementId || + envelope.commitmentDigest !== commitmentDigest || + canonicalJson(envelope.operation) !== canonicalJson(projection.operation) || + canonicalJson(envelope.serviceIds) !== canonicalJson(saved.input.serviceIds) || + envelope.execution.operationFingerprint !== projection.reservation.operationFingerprint || + envelope.execution.fingerprint !== + executionFingerprint({ + kind: "configuration-maintenance.settle", + operationFingerprint: envelope.execution.operationFingerprint, + deadlineAt: envelope.controlDeadlineAt, + parent: retainedControlParent(owner), + }) || + identity.installationId !== installation.installationId || + canonicalJson(identity.origin) !== canonicalJson(owned.origin) || + canonicalJson(identity.ownerPrepare) !== canonicalJson(owner) || + identity.admissionDigest !== projection.rootAdmissionDigest || + identity.rootCertificateDigest !== owned.ownerCertificateDigest || + identity.maintenanceId !== input.maintenanceId || + identity.candidateId !== input.candidateId || + identity.planDigest !== input.planDigest || + identity.settlementId !== input.settlementId || + canonicalJson(commitment.final.identity) !== canonicalJson(identity) || + canonicalJson(commitment.evidence.identity) !== canonicalJson(identity) || + commitment.referenceOperationFingerprint !== projection.reservation.operationFingerprint || + commitment.ownerReceiptDigest !== + settlementRecordDigest(configurationSettlementDomains.ownerFinal, commitment.final) || + commitment.evidenceDigest !== + settlementRecordDigest(configurationSettlementDomains.finalEvidence, commitment.evidence) || + commitment.final.evidenceDigest !== commitment.evidenceDigest || + commitment.final.outcome !== commitment.evidence.outcome || + commitment.final.expectedDmSequence !== saved.receipt.sequence || + commitment.committedAt !== commitment.final.settlingAt || + canonicalJson(commitment.evidence.guard.preparedReceipt) !== + canonicalJson(effect.receipt.receipt) || + commitment.evidence.guard.fenceId !== saved.receipt.fenceId || + commitment.evidence.guard.preparedReceiptDigest !== preparedReceiptDigest || + commitment.evidence.guard.preparedCanonicalDigest !== canonical.terminalDigest || + commitment.evidence.guard.preparedTerminalAt !== canonical.terminalAt || + envelopeDigest.length !== 64 + ) + throw new DataFenceBusy(); + const source = { + input, + saved, + owned, + projection, + envelope, + signedEnvelope: control.binding, + commitment, + commitmentDigest, + capacityDigest, + prepareExecution: preparedGrant.execution, + prepareAdmissionDigest: prepare.receiptDigest, + preparedReceiptDigest, + }; + if (saved.guardState === "settled" && saved.settlementControl) { + const originalAdmission = retainedSettlementAdmission(source); + if (!originalAdmission) throw new DataFenceBusy(); + assertSettlementSqlTuple(source, saved, originalAdmission); + } else + verifyFamilyOwnedRecord( + local, + owned.origin, + { + scope: "private", + namespace: "configuration-maintenance", + container: "", + name: `${input.maintenanceId}.json`, + }, + canonicalJson(mirrorSource.raw), + ); + return source; + } + function retainedSettlementAdmission(source: SettlementSource) { + const descriptors = configurationSettlementRecordDescriptors( + source.projection, + source.prepareExecution.operationKey, + ).receiver, + raw = readConfigurationSettlementCell(local, source.projection, descriptors[1]!); + if (raw === null) return null; + const value = verifyControl( + installation.publicKey, + configurationSettlementDomains.admission, + raw, + configurationSettlementAdmissionSchema, + ), + slot = bindOwnedConfigurationSettlementFamilySlot(local, source.projection); + if ( + value.installationId !== installation.installationId || + canonicalJson(value.origin) !== canonicalJson(source.owned.origin) || + value.settlementId !== source.input.settlementId || + value.projectionDigest !== source.envelope.projectionDigest || + value.reservationDigest !== source.envelope.reservationDigest || + value.prepareExecutionDigest !== controlDigest(source.prepareExecution) || + value.prepareAdmissionDigest !== source.prepareAdmissionDigest || + value.preparedReceiptDigest !== source.preparedReceiptDigest || + value.capacityReceiptDigest !== source.capacityDigest || + value.rootCertificateDigest !== source.owned.ownerCertificateDigest || + canonicalJson(value.controlEnvelope) !== canonicalJson(source.signedEnvelope) || + value.commitmentDigest !== source.commitmentDigest || + value.executionOrdinal !== slot.executionOrdinal || + value.intentOrdinal !== slot.intentOrdinal || + value.sqlSessionNonce !== source.projection.reservation.execution.executionId || + value.boundAt < source.envelope.controlStartedAt || + value.boundAt >= source.envelope.controlDeadlineAt + ) + throw new DataFenceBusy(); + return { + value, + signature: (raw as { signature: string }).signature, + digest: settlementRecordDigest(configurationSettlementDomains.admission, value), + descriptors, + }; + } + function assertSettlementSqlTuple( + source: SettlementSource, + saved: z.infer, + admission: NonNullable>, + ) { + const control = saved.settlementControl; + if ( + !control || + saved.guardState !== "settled" || + canonicalJson(saved.settlement) !== canonicalJson(source.input) + ) + throw new DataFenceBusy(); + const tuple = control.tuple, + { terminalDigest, ...preimage } = tuple; + if ( + control.settlementId !== source.input.settlementId || + control.controlEnvelopeDigest !== + settlementRecordDigest(configurationSettlementDomains.binding, source.envelope) || + control.commitmentDigest !== source.commitmentDigest || + control.admissionDigest !== admission.digest || + control.outcome !== source.commitment.final.outcome || + control.expectedDmSequence !== source.saved.receipt.sequence || + control.preparedReceiptDigest !== source.preparedReceiptDigest || + control.sqlSession.sessionNonce !== admission.value.sqlSessionNonce || + canonicalJson(tuple.execution) !== canonicalJson(source.envelope.execution) || + tuple.settlementId !== source.input.settlementId || + tuple.commitmentDigest !== source.commitmentDigest || + tuple.fenceId !== source.saved.receipt.fenceId || + tuple.terminalAt < source.envelope.controlStartedAt || + tuple.terminalAt >= source.envelope.controlDeadlineAt || + terminalDigest !== + settlementRecordDigest(configurationSettlementDomains.terminal, preimage) || + canonicalJson(tuple.result) !== + canonicalJson({ + ...source.saved.receipt, + state: "settled", + sequence: source.saved.receipt.sequence + 1, + }) + ) + throw new DataFenceBusy(); + return control; + } + /** Publication only, from the exact atomic SQL tuple and verified original + * admission. No logical UPDATE, effect start, new time or generic adoption. */ + function publishCommittedSettlement( + source: SettlementSource, + saved: z.infer, + admission: NonNullable>, + ) { + const control = assertSettlementSqlTuple(source, saved, admission), + tuple = control.tuple, + descriptors = admission.descriptors, + priorMirror = signedMirrorData(source.input.maintenanceId).parsed; + if ( + priorMirror.metadata.settlementControl && + canonicalJson(priorMirror.metadata.settlementControl) !== canonicalJson(control) + ) + throw new DataFenceBusy(); + write({ ...priorMirror, metadata: saved }); + const committed = configurationSettlementEffectSchema.parse({ + version: 2, + origin: source.owned.origin, + settlementId: source.input.settlementId, + controlExecutionDigest: controlDigest(source.envelope.execution), + controlEnvelopeDigest: control.controlEnvelopeDigest, + admissionDigest: admission.digest, + commitmentDigest: source.commitmentDigest, + preparedReceiptDigest: source.preparedReceiptDigest, + expectedDmSequence: source.saved.receipt.sequence, + attemptedAt: control.attemptedAt, + sqlSession: control.sqlSession, + phase: "committed", + terminal: tuple, + observation: null, + }), + oldEffect = readConfigurationSettlementCell(local, source.projection, descriptors[2]!); + if (oldEffect !== null) { + const old = verifyControl( + installation.publicKey, + configurationSettlementDomains.effect, + oldEffect, + configurationSettlementEffectSchema, + ), + { + phase: _oldPhase, + terminal: _oldTerminal, + observation: _oldObservation, + ...oldIdentity + } = old, + { phase: _phase, terminal: _terminal, observation: _observation, ...identity } = committed; + if ( + canonicalJson(oldIdentity) !== canonicalJson(identity) || + old.phase === "contained_not_settled" || + (old.phase === "committed" && canonicalJson(old) !== canonicalJson(committed)) + ) + throw new DataFenceBusy(); + } + const signedEffect = signControl( + local.privateKey, + configurationSettlementDomains.effect, + committed, + ); + writeConfigurationSettlementCell(local, source.projection, descriptors[2]!, signedEffect); + const { terminal: _effectTerminal, ...effectFields } = committed; + const terminal = signControl( + local.privateKey, + configurationSettlementDomains.terminal, + configurationSettlementPublishedTerminalSchema.parse({ + version: 2, + origin: source.owned.origin, + admissionDigest: admission.digest, + tuple, + effectSettlementDigest: settlementRecordDigest( + configurationSettlementDomains.effect, + committed, + ), + effectSource: { fields: effectFields, signature: signedEffect.signature }, + admissionSource: { boundAt: admission.value.boundAt, signature: admission.signature }, + }), + ); + for (const descriptor of [descriptors[3]!, descriptors[4]!]) { + const old = readConfigurationSettlementCell(local, source.projection, descriptor); + if (old === null) + writeConfigurationSettlementCell(local, source.projection, descriptor, terminal); + else if (canonicalJson(old) !== canonicalJson(terminal)) throw new DataFenceBusy(); + } + const rawAck = shared( + "ops-agent", + `${source.saved.receipt.ownerPrepare.operationKey}.configuration-settlement-ack.json`, + 1024, + ); + if (rawAck !== null) { + const ack = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.ack, + rawAck, + configurationSettlementAckSchema, + ); + if ( + ack.installationId !== installation.installationId || + canonicalJson(ack.origin) !== canonicalJson(source.owned.origin) || + ack.settlementId !== source.input.settlementId || + ack.controlExecutionDigest !== controlDigest(source.envelope.execution) || + ack.terminalDigest !== tuple.terminalDigest || + ack.terminalAt !== tuple.terminalAt || + ack.commitmentDigest !== source.commitmentDigest + ) + throw new DataFenceBusy(); + const copied = readConfigurationSettlementCell(local, source.projection, descriptors[5]!); + if (copied === null) + writeConfigurationSettlementCell(local, source.projection, descriptors[5]!, rawAck); + else if (canonicalJson(copied) !== canonicalJson(rawAck)) throw new DataFenceBusy(); + } + startup = "open"; + return tuple.result; + } + async function recoverCommittedConfigurationFenceSettlement(input: DataConfigurationFenceSettle) { + return sql.transaction(async (tx) => { + await lock(tx); + const source = await loadConfigurationFenceSettlementSource(input, tx), + admission = retainedSettlementAdmission(source), + saved = await row(tx, input.maintenanceId, true); + if (!admission || !saved?.settlementControl || saved.guardState !== "settled") + throw new DataFenceBusy(); + return publishCommittedSettlement(source, saved, admission); + }); + } + /** Observation-only recovery of the ONE exhausted control. A prepared SQL + * row is insufficient: the original admission/effect, same nonblocking SQL + * guard/row lock and exact original session identity must agree. This never + * constructs an effect ticket, terminal tuple, ACK or guard UPDATE. */ + async function observeUnsettledConfigurationFenceSettlement( + maintenanceId: string, + ): Promise { + return sql.transaction(async (tx) => { + await lock(tx); + const saved = await row(tx, maintenanceId, true); + if ( + !saved || + saved.guardState !== "prepared" || + !saved.settlementReservation || + saved.settlement || + saved.settlementControl + ) + return null; + const control = verifyControl( + local.publicKeys["ops-agent"], + configurationSettlementDomains.control, + shared( + "ops-agent", + `${saved.receipt.ownerPrepare.operationKey}.configuration-settlement-control.json`, + 8192, + ), + configurationSettlementControlCellSchema.shape.payload, + ); + if ("unused" in control || !control.binding) return null; + const input = dataConfigurationFenceSettleSchema.parse({ + version: 2, + maintenanceId, + candidateId: saved.receipt.candidateId, + planDigest: saved.receipt.planDigest, + settlementId: saved.settlementReservation.settlementId, + }), + source = await loadConfigurationFenceSettlementSource(input, tx), + admission = retainedSettlementAdmission(source); + if (!admission) return null; + const projection = { ...saved.receipt, state: "recovery_required" as const }, + raw = readConfigurationSettlementCell(local, source.projection, admission.descriptors[2]!); + // Bound but unattempted adoption cannot regain the lost process ticket. + if (raw === null) return projection; + const effect = verifyControl( + installation.publicKey, + configurationSettlementDomains.effect, + raw, + configurationSettlementEffectSchema, + ); + if ( + canonicalJson(effect.origin) !== canonicalJson(source.owned.origin) || + effect.settlementId !== input.settlementId || + effect.controlExecutionDigest !== controlDigest(source.envelope.execution) || + effect.controlEnvelopeDigest !== + settlementRecordDigest(configurationSettlementDomains.binding, source.envelope) || + effect.admissionDigest !== admission.digest || + effect.commitmentDigest !== source.commitmentDigest || + effect.preparedReceiptDigest !== source.preparedReceiptDigest || + effect.expectedDmSequence !== saved.receipt.sequence || + effect.sqlSession.sessionNonce !== admission.value.sqlSessionNonce || + effect.attemptedAt < admission.value.boundAt || + effect.attemptedAt >= source.envelope.controlDeadlineAt || + effect.terminal || + effect.phase === "committed" + ) + throw new DataFenceBusy(); + const observationFields = { + version: 2 as const, + state: "contained_not_settled" as const, + settlementId: effect.settlementId, + controlExecutionDigest: effect.controlExecutionDigest, + controlEnvelopeDigest: effect.controlEnvelopeDigest, + commitmentDigest: effect.commitmentDigest, + preparedReceiptDigest: effect.preparedReceiptDigest, + expectedDmSequence: effect.expectedDmSequence, + attemptedAt: effect.attemptedAt, + sqlSession: effect.sqlSession, + }; + if (effect.phase === "contained_not_settled") { + const observed = configurationSettlementContainedObservationSchema.parse( + effect.observation, + ), + { observationAt, containment: _containment, ...fields } = observed; + if ( + canonicalJson(fields) !== canonicalJson(observationFields) || + observationAt < effect.attemptedAt + ) + throw new DataFenceBusy(); + if (canonicalJson(source.owned.metadata) !== canonicalJson(saved)) + throw new DataFenceBusy(); + return projection; + } + const sessions = await tx.unsafe( + "SELECT pid,backend_start::text AS backend_start,application_name FROM pg_stat_activity WHERE pid=$1::int", + [effect.sqlSession.pid], + ); + if (sessions.length > 1) throw new DataFenceBusy(); + const original = sessions[0]; + if (original) { + if ( + typeof original.backend_start !== "string" || + typeof original.application_name !== "string" + ) + throw new DataFenceBusy(); + // A renamed application_name cannot prove a live backend absent. Only + // no PID or a conclusively different backend_start excludes the exact + // original session; a changed/unverifiable nonce stays fenced too. + if (original.backend_start === effect.sqlSession.backendStart) throw new DataFenceBusy(); + } + // The row lock waited out the old transaction; that exact backend/nonce + // cannot later commit. Restore only the unchanged original prepared row. + const observation = configurationSettlementContainedObservationSchema.parse({ + ...observationFields, + observationAt: Date.now(), + containment: "original_session_absent", + }); + if (observation.observationAt < effect.attemptedAt) throw new DataFenceBusy(); + verifyConfigurationFenceSettlementHeldOwner(installation, source.owned.prepareAssertion, { + ownerPrepare: saved.receipt.ownerPrepare, + ownerCertificateDigest: source.owned.ownerCertificateDigest, + serviceIds: saved.input.serviceIds, + }); + write({ ...source.owned, metadata: saved }); + writeConfigurationSettlementCell( + local, + source.projection, + admission.descriptors[2]!, + signControl( + local.privateKey, + configurationSettlementDomains.effect, + configurationSettlementEffectSchema.parse({ + ...effect, + phase: "contained_not_settled", + observation, + }), + ), + ); + return projection; + }); + } + /** SQL's original reserved session holds the non-stealable role writer via + * the EXISTING admission advisory key. Before BEGIN it owns the session lock; + * after BEGIN it transfers to the SAME connection's transaction lock. Thus a + * crash releases kernel/DB ownership without stealing a filesystem marker, + * and an uncertain COMMIT cannot strand an extra session lock. */ + async function adoptConfigurationFenceSettlementControl(peer: VerifiedFenceSettlementPeer) { + const source = settlementPeers.get(peer); + if (!source) throw new DataFenceBusy(); + if (retainedSettlementAdmission(source)) + return { + inspect: () => recoverCommittedConfigurationFenceSettlement(source.input), + repairCommittedPublication: () => + recoverCommittedConfigurationFenceSettlement(source.input), + close: async () => {}, + }; + if (!sql.reserveSettlementSession) throw new DataFenceBusy(); + const session = await sql.reserveSettlementSession( + source.projection.reservation.execution.executionId, + ); + let sessionHeld = false, + commitAttempted = false, + handedOff = false; + const release = async () => { + if (sessionHeld) { + await session.unsafe("SELECT pg_advisory_unlock($1::int,$2::int)", [0x4f4d58, 2]); + sessionHeld = false; + } + session.releaseConclusive(); + }; + try { + const held = await session.unsafe( + "SELECT pg_try_advisory_lock($1::int,$2::int) AS held", + [0x4f4d58, 2], + ); + if (held[0]?.held !== true) { + session.releaseConclusive(); + throw new DataFenceBusy(); + } + sessionHeld = true; + const current = await loadConfigurationFenceSettlementSource(source.input, session); + if (retainedSettlementAdmission(current)) { + await release(); + return { + inspect: () => recoverCommittedConfigurationFenceSettlement(source.input), + repairCommittedPublication: () => + recoverCommittedConfigurationFenceSettlement(source.input), + close: async () => {}, + }; + } + verifyConfigurationFenceSettlementHeldOwner(installation, current.owned.prepareAssertion, { + ownerPrepare: current.saved.receipt.ownerPrepare, + ownerCertificateDigest: current.owned.ownerCertificateDigest, + serviceIds: current.saved.input.serviceIds, + }); + const slot = readConfigurationSettlementFamilySlot(local, current.projection), + boundAt = Date.now(), + descriptors = configurationSettlementRecordDescriptors( + current.projection, + current.prepareExecution.operationKey, + ).receiver; + if ( + slot.state !== "reserved" || + slot.executionOrdinal === null || + slot.intentOrdinal === null || + current.saved.guardState !== "prepared" || + current.saved.settlementControl || + current.commitmentDigest !== source.commitmentDigest || + canonicalJson(current.envelope) !== canonicalJson(source.envelope) || + boundAt >= current.envelope.controlDeadlineAt + ) + throw new DataFenceBusy(); + const identity = configurationSettlementSqlSessionSchema.parse(session.identity); + if (identity.sessionNonce !== current.projection.reservation.execution.executionId) + throw new DataFenceBusy(); + const admission = configurationSettlementAdmissionSchema.parse({ + version: 2, + purpose: "configuration-fence-settlement", + installationId: installation.installationId, + origin: current.owned.origin, + settlementId: current.input.settlementId, + projectionDigest: current.envelope.projectionDigest, + reservationDigest: current.envelope.reservationDigest, + prepareExecutionDigest: controlDigest(current.prepareExecution), + prepareAdmissionDigest: current.prepareAdmissionDigest, + preparedReceiptDigest: current.preparedReceiptDigest, + capacityReceiptDigest: current.capacityDigest, + rootCertificateDigest: current.owned.ownerCertificateDigest, + controlEnvelope: current.signedEnvelope, + commitmentDigest: current.commitmentDigest, + executionOrdinal: slot.executionOrdinal, + intentOrdinal: slot.intentOrdinal, + sqlSessionNonce: identity.sessionNonce, + boundAt, + }); + writeConfigurationSettlementCell( + local, + current.projection, + descriptors[1]!, + signControl(local.privateKey, configurationSettlementDomains.admission, admission), + ); + bindOwnedConfigurationSettlementFamilySlot(local, current.projection); + const retained = retainedSettlementAdmission(current); + if (!retained) throw new DataFenceBusy(); + const effectTicket = Object.freeze({}); + receiverEffectTickets.set(effectTicket, session); + handedOff = true; + return { + inspect: () => recoverCommittedConfigurationFenceSettlement(current.input), + repairCommittedPublication: () => + recoverCommittedConfigurationFenceSettlement(current.input), + async close() { + if (receiverEffectTickets.has(effectTicket)) { + receiverEffectTickets.delete(effectTicket); + await release(); + } + }, + async runOriginalGuardTransition() { + if (receiverEffectTickets.get(effectTicket) !== session) throw new DataFenceBusy(); + receiverEffectTickets.delete(effectTicket); + const attemptedAt = Date.now(), + attempted = configurationSettlementEffectSchema.parse({ + version: 2, + origin: current.owned.origin, + settlementId: current.input.settlementId, + controlExecutionDigest: controlDigest(current.envelope.execution), + controlEnvelopeDigest: settlementRecordDigest( + configurationSettlementDomains.binding, + current.envelope, + ), + admissionDigest: retained.digest, + commitmentDigest: current.commitmentDigest, + preparedReceiptDigest: current.preparedReceiptDigest, + expectedDmSequence: current.saved.receipt.sequence, + attemptedAt, + sqlSession: identity, + phase: "attempted", + terminal: null, + observation: null, + }); + if (attemptedAt >= current.envelope.controlDeadlineAt) { + await release(); + throw new DataFenceBusy(); + } + if ( + readConfigurationSettlementCell(local, current.projection, descriptors[2]!) !== null + ) { + await release(); + throw new DataFenceBusy(); + } + writeConfigurationSettlementCell( + local, + current.projection, + descriptors[2]!, + signControl(local.privateKey, configurationSettlementDomains.effect, attempted), + ); + try { + await session.unsafe("BEGIN"); + await lock(session); + await session.unsafe("SELECT pg_advisory_unlock($1::int,$2::int)", [0x4f4d58, 2]); + sessionHeld = false; + const saved = await row(session, current.input.maintenanceId, true); + if (canonicalJson(saved) !== canonicalJson(current.saved)) throw new DataFenceBusy(); + const active = await session.unsafe( + "SELECT id FROM data_manager.jobs WHERE kind <> 'configuration-maintenance' AND (finished_at IS NULL OR execution_state->>'publication'='unknown') LIMIT 1", + ); + if (active.length) throw new DataFenceBusy(); + verifyConfigurationFenceSettlementHeldOwner( + installation, + current.owned.prepareAssertion, + { + ownerPrepare: current.saved.receipt.ownerPrepare, + ownerCertificateDigest: current.owned.ownerCertificateDigest, + serviceIds: current.saved.input.serviceIds, + }, + ); + const [clock] = await session.unsafe( + "SELECT floor(extract(epoch FROM clock_timestamp()) * 1000)::float8 AS terminal_at", + ), + terminalAt = clock?.terminal_at; + if ( + typeof terminalAt !== "number" || + !Number.isSafeInteger(terminalAt) || + terminalAt < current.envelope.controlStartedAt || + terminalAt >= current.envelope.controlDeadlineAt || + Date.now() >= current.envelope.controlDeadlineAt + ) + throw new DataFenceBusy(); + const result = dataConfigurationFenceReceiptSchema.parse({ + ...current.saved.receipt, + state: "settled", + sequence: current.saved.receipt.sequence + 1, + }), + preimage = { + execution: current.envelope.execution, + settlementId: current.input.settlementId, + commitmentDigest: current.commitmentDigest, + fenceId: current.saved.receipt.fenceId, + result, + publication: "committed" as const, + terminalAt, + }, + tuple = configurationSettlementTerminalTupleSchema.parse({ + ...preimage, + terminalDigest: settlementRecordDigest( + configurationSettlementDomains.terminal, + preimage, + ), + }), + settlementControl = configurationSettlementSqlControlSchema.parse({ + version: 2, + settlementId: current.input.settlementId, + controlEnvelopeDigest: attempted.controlEnvelopeDigest, + commitmentDigest: current.commitmentDigest, + admissionDigest: retained.digest, + outcome: current.commitment.final.outcome, + expectedDmSequence: current.saved.receipt.sequence, + preparedReceiptDigest: current.preparedReceiptDigest, + attemptedAt, + sqlSession: identity, + tuple, + }), + metadata = metadataSchema.parse({ + ...current.saved, + guardState: "settled", + settlement: current.input, + settlementControl, + }); + write({ ...current.owned, metadata: { ...metadata, guardState: "settling" } }); + if (Date.now() >= current.envelope.controlDeadlineAt) throw new DataFenceBusy(); + const updated = await session.unsafe( + "UPDATE data_manager.jobs SET metadata=jsonb_set(coalesce(metadata,'{}'::jsonb),'{configurationMaintenance}',$2::text::jsonb),ownership_version=ownership_version+1 WHERE id=$1::uuid AND metadata->'configurationMaintenance'=$3::text::jsonb RETURNING id", + [result.fenceId, canonicalJson(metadata), canonicalJson(current.saved)], + ); + if (updated.length !== 1) throw new DataFenceBusy(); + commitAttempted = true; + await session.unsafe("COMMIT"); + session.releaseConclusive(); + return publishCommittedSettlement(current, metadata, retained); + } catch (error) { + if (!commitAttempted) { + try { + await session.unsafe("ROLLBACK"); + await release(); + const observationAt = Date.now(), + observation = { + version: 2 as const, + state: "contained_not_settled" as const, + settlementId: current.input.settlementId, + controlExecutionDigest: attempted.controlExecutionDigest, + controlEnvelopeDigest: attempted.controlEnvelopeDigest, + commitmentDigest: current.commitmentDigest, + preparedReceiptDigest: current.preparedReceiptDigest, + expectedDmSequence: current.saved.receipt.sequence, + attemptedAt, + observationAt, + sqlSession: identity, + containment: "explicit_rollback" as const, + }; + write({ ...current.owned, metadata: current.saved }); + writeConfigurationSettlementCell( + local, + current.projection, + descriptors[2]!, + signControl( + local.privateKey, + configurationSettlementDomains.effect, + configurationSettlementEffectSchema.parse({ + ...attempted, + phase: "contained_not_settled", + observation, + }), + ), + ); + } catch { + /* No conclusive rollback publication; original session stays unknown/fenced. */ + } + } + // Never return/reuse a connection following an uncertain COMMIT. + throw error; + } + }, + }; + } catch (error) { + if (!handedOff && sessionHeld) { + try { + await release(); + } catch { + /* Held unknown session remains private. */ + } + } + throw error; + } + } + /** Signature-checked data only. It is NOT indexed owner authority. Fixed + * committed-SQL repair authenticates its original fields against actual + * preparation/effect/admission/tuple sources before replacing a torn index. */ + function signedMirrorData(id: string) { const raw = readPrivateRecord( local, "configuration-maintenance", @@ -115,6 +1164,15 @@ export function createDataConfigurationFenceStore( raw, mirrorSchema, ); + if ( + parsed.installationId !== installation.installationId || + parsed.metadata.receipt.maintenanceId !== id + ) + throw Error("Configuration guard mirror identity changed"); + return { raw, parsed }; + } + function mirror(id: string) { + const { raw, parsed } = signedMirrorData(id); verifyFamilyOwnedRecord( local, parsed.origin, @@ -126,20 +1184,21 @@ export function createDataConfigurationFenceStore( }, canonicalJson(raw), ); - if ( - parsed.installationId !== installation.installationId || - parsed.metadata.receipt.maintenanceId !== id - ) - throw Error("Configuration guard mirror identity changed"); return parsed; } function write(value: z.infer) { const parsed = mirrorSchema.parse(value); + const signed = signControl(local.privateKey, "openmapx-data-configuration-fence-v1", parsed); + if ( + Buffer.byteLength(canonicalJson(signed)) > 65536 || + Buffer.byteLength(canonicalJson(parsed.metadata)) > 65536 + ) + throw Error("Configuration guard mirror/metadata capacity exceeded"); writePrivateRecord( local, "configuration-maintenance", `${parsed.metadata.receipt.maintenanceId}.json`, - signControl(local.privateKey, "openmapx-data-configuration-fence-v1", parsed), + signed, false, parsed.origin, ); @@ -152,9 +1211,301 @@ export function createDataConfigurationFenceStore( ); if (rows[0]?.held !== true) throw new DataFenceBusy(); } - async function row(tx: Pick, id: string) { + /** Fixed local projection source. Its only argument is the ORIGINAL admitted + * PREPARE authority; outcome/reason/digest/time all come from actual retained + * canonical/effect/adoption/allocator and guarded SQL/mirror observations. */ + function refusedSource(authority: import("./control-authority").VerifiedExecutionAuthority) { + const state = executionAuthorityState(authority); + if ( + state.kind !== "configuration-maintenance.prepare" || + state.installation !== local || + !state.origin + ) + throw Error("Original local DM refusal producer required"); + const original = verifyRetainedDataFenceRefusal(installation, state.execution), + adoption = retainedConfigurationFenceAdoption(authority), + assertion = retainedControlAdoptionAssertion( + installation, + { ...adoption, phase: "ready" }, + state.execution, + ), + raw = readPrivateRecord( + local, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: unknown } }, + { kind, ...fields } = z + .object({ kind: z.literal("configuration-maintenance.prepare") }) + .passthrough() + .parse(raw.payload?.operation), + input = dataConfigurationFencePrepareSchema.parse(fields), + verified = inspectGrant(input, ["x-openmapx-control-delegation", assertion], kind), + reservation = publishedConfigurationSettlementReservationState( + verifyPublishedConfigurationSettlementReservation(installation, verified.peer, input), + installation, + ), + projection = reservation.projection, + descriptors = configurationSettlementRecordDescriptors( + projection, + state.execution.operationKey, + ).receiver; + if ( + reservation.publication === "bound" || + adoption.certificateDigest !== projection.rootCertificateDigest || + original.receipt.maintenanceId !== input.maintenanceId + ) + throw Error("Original unused DM refusal binding required"); + verifyRetainedArtifactTargetSettlement( + installation, + { ...adoption, phase: "ready" }, + state.execution, + ); + return { state, original, input, reservation, projection, descriptors }; + } + async function publishDataConfigurationFenceRefusalProjection( + authority: import("./control-authority").VerifiedExecutionAuthority, + ): Promise { + const { state, original, input, reservation, projection, descriptors } = + refusedSource(authority); + await sql.transaction(async (tx) => { + await lock(tx); + if (await row(tx, input.maintenanceId)) + throw Error("A guard row cannot produce NoGuard refusal"); + try { + readPrivateRecord(local, "configuration-maintenance", `${input.maintenanceId}.json`, 65536); + throw Error("A guard mirror cannot produce NoGuard refusal"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + withPrivateEffectWriter(state, () => { + const slot = readConfigurationSettlementFamilySlot(local, projection), + cell = configurationDmCapacityCellSchema.parse( + readConfigurationSettlementCell(local, projection, descriptors[0]!), + ); + if ( + slot.executionDigest !== null || + slot.controlBindingDigest !== null || + ["bound", "allocating", "refusal_channel"].includes(slot.state) || + (original.receipt.reason === "capacity" && + (!slot.refusalConstraint || + slot.executionOrdinal !== null || + slot.intentOrdinal !== null || + !["capacity_refused", "UNUSED"].includes(slot.state))) + ) + throw Error("Conclusive original unused allocator state required"); + let capacityReceiptDigest: string | null = null; + if (cell.capacity) { + const capacity = verifyControl( + installation.publicKey, + configurationSettlementDomains.capacity, + cell.capacity, + configurationSettlementCapacitySchema, + ); + if ( + capacity.settlementId !== projection.settlementId || + capacity.projectionDigest !== reservation.projectionDigest || + capacity.reservationDigest !== projection.reservationDigest || + capacity.recordDescriptorsDigest !== slot.recordDescriptorsDigest || + capacity.reservedExecutionOrdinal !== slot.executionOrdinal || + capacity.reservedIntentOrdinal !== slot.intentOrdinal + ) + throw Error("Original DM capacity refusal preimage changed"); + capacityReceiptDigest = settlementRecordDigest( + configurationSettlementDomains.capacity, + capacity, + ); + } else if ( + original.receipt.reason !== "capacity" || + (slot.state !== "capacity_refused" && slot.state !== "UNUSED") + ) + throw Error("Original full busy capacity certificate required"); + const payload = configurationDmNoGuardRefusalSchema.parse({ + version: 2, + installationId: installation.installationId, + origin: state.origin, + ownerPrepare: input.ownerPrepare, + rootAdmissionDigest: projection.rootAdmissionDigest, + rootCertificateDigest: projection.rootCertificateDigest, + maintenanceId: input.maintenanceId, + candidateId: input.candidateId, + planDigest: input.planDigest, + settlementId: projection.settlementId, + projectionDigest: reservation.projectionDigest, + reservationDigest: projection.reservationDigest, + prepareExecution: state.execution, + prepareAdmissionDigest: state.receiptDigest, + refusalReceipt: original.receipt, + effectReceiptDigest: original.effectReceiptDigest, + effectSettlementDigest: original.effectSettlementDigest, + canonicalTerminalDigest: original.digest, + canonicalTerminalAt: original.terminalAt, + capacityReceiptDigest, + }), + signed = signControl(local.privateKey, configurationSettlementDomains.refusal, payload); + if (cell.refusal) { + if (canonicalJson(cell.refusal) !== canonicalJson(signed)) + throw Error("Original DM refusal projection conflict"); + } else { + if (cell.unused || reservation.publication !== "pending") + throw Error("Original DM refusal projection missing after closure"); + writeConfigurationSettlementCell(local, projection, descriptors[0]!, { + ...cell, + refusal: signed, + }); + } + }); + }); + } + /** No caller proof fields: original refusal -> retained intent -> exact + * local metadata/ticket CAS -> reread -> pinned-role confirmation. */ + async function markRefusedConfigurationSettlementCapacityUnused( + authority: import("./control-authority").VerifiedExecutionAuthority, + ): Promise { + const source = refusedSource(authority), + { state, original, input, reservation, projection, descriptors } = source; + const before = configurationDmCapacityCellSchema.parse( + readConfigurationSettlementCell(local, projection, descriptors[0]!), + ); + if (!before.refusal) await publishDataConfigurationFenceRefusalProjection(authority); + await sql.transaction(async (tx) => { + await lock(tx); + if (await row(tx, input.maintenanceId)) + throw Error("A guard row cannot mark settlement capacity UNUSED"); + try { + readPrivateRecord(local, "configuration-maintenance", `${input.maintenanceId}.json`, 65536); + throw Error("A guard mirror cannot mark settlement capacity UNUSED"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + withPrivateEffectWriter(state, () => { + let cell = configurationDmCapacityCellSchema.parse( + readConfigurationSettlementCell(local, projection, descriptors[0]!), + ); + if (!cell.refusal) throw Error("Original signed DM refusal required before UNUSED"); + const refusal = verifyControl( + installation.publicKey, + configurationSettlementDomains.refusal, + cell.refusal, + configurationDmNoGuardRefusalSchema, + ), + slot = readRefusedConfigurationSettlementCapacitySnapshot(local, projection); + if ( + refusal.settlementId !== projection.settlementId || + refusal.projectionDigest !== reservation.projectionDigest || + refusal.prepareAdmissionDigest !== state.receiptDigest || + canonicalJson(refusal.prepareExecution) !== canonicalJson(state.execution) || + refusal.canonicalTerminalDigest !== original.digest || + refusal.canonicalTerminalAt !== original.terminalAt || + refusal.effectReceiptDigest !== original.effectReceiptDigest || + refusal.effectSettlementDigest !== original.effectSettlementDigest || + canonicalJson(refusal.refusalReceipt) !== canonicalJson(original.receipt) + ) + throw Error("Original DM refusal source changed before UNUSED"); + const capacityDigest = cell.capacity + ? settlementRecordDigest( + configurationSettlementDomains.capacity, + verifyControl( + installation.publicKey, + configurationSettlementDomains.capacity, + cell.capacity, + configurationSettlementCapacitySchema, + ), + ) + : null; + if (capacityDigest !== refusal.capacityReceiptDigest) + throw Error("Original DM capacity certificate changed before UNUSED"); + const oldIntent = cell.unused + ? verifyControl( + installation.publicKey, + configurationSettlementDomains.unusedIntent, + cell.unused.intent, + configurationUnusedCapacityIntentSchema, + ) + : null, + intent = configurationUnusedCapacityIntentSchema.parse({ + version: 2, + installationId: installation.installationId, + origin: state.origin, + role: "data-manager", + ownerPrepareDigest: controlDigest(projection.ownerPrepare), + prepareExecutionDigest: controlDigest(state.execution), + settlementId: projection.settlementId, + projectionDigest: reservation.projectionDigest, + capacityReceiptDigest: capacityDigest, + sourceProofDigest: settlementRecordDigest( + configurationSettlementDomains.refusal, + refusal, + ), + executionOrdinal: slot.executionOrdinal, + intentOrdinal: slot.intentOrdinal, + recordDescriptorsDigest: slot.recordDescriptorsDigest, + allocatedDescriptorDigests: slot.allocatedDescriptorDigests, + createdAt: oldIntent?.createdAt ?? Date.now(), + }), + signedIntent = signControl( + local.privateKey, + configurationSettlementDomains.unusedIntent, + intent, + ); + if (Buffer.byteLength(canonicalJson(signedIntent)) > 2048) + throw Error("Original DM UNUSED intent exceeds reserved bound"); + if (oldIntent && canonicalJson(oldIntent) !== canonicalJson(intent)) + throw Error("Original DM UNUSED intent changed"); + if (!cell.unused) { + if (slot.state === "UNUSED") throw Error("Original DM UNUSED intent missing after CAS"); + cell = { ...cell, unused: { intent: signedIntent, confirmation: null } }; + writeConfigurationSettlementCell(local, projection, descriptors[0]!, cell); + } + const committed = commitRefusedDataConfigurationSettlementSlotUnused(local, projection), + intentDigest = settlementRecordDigest( + configurationSettlementDomains.unusedIntent, + intent, + ), + confirmation = configurationUnusedCapacityConfirmationSchema.parse({ + version: 2, + installationId: installation.installationId, + origin: state.origin, + role: "data-manager", + ownerPrepareDigest: controlDigest(projection.ownerPrepare), + settlementId: projection.settlementId, + projectionDigest: reservation.projectionDigest, + intentDigest, + slotStateDigest: controlDigest({ + executionOrdinal: committed.executionOrdinal, + intentOrdinal: committed.intentOrdinal, + recordDescriptorsDigest: committed.recordDescriptorsDigest, + allocatedDescriptorDigests: committed.allocatedDescriptorDigests, + state: "UNUSED", + intentDigest, + }), + confirmedAt: committed.unusedConfirmedAt, + }), + signedConfirmation = signControl( + local.privateKey, + configurationSettlementDomains.unusedConfirmation, + confirmation, + ); + if (Buffer.byteLength(canonicalJson(signedConfirmation)) > 1024) + throw Error("Original DM UNUSED confirmation exceeds reserved bound"); + cell = configurationDmCapacityCellSchema.parse( + readConfigurationSettlementCell(local, projection, descriptors[0]!), + ); + if (cell.unused?.confirmation) { + if (canonicalJson(cell.unused.confirmation) !== canonicalJson(signedConfirmation)) + throw Error("Original DM UNUSED confirmation changed"); + } else + writeConfigurationSettlementCell(local, projection, descriptors[0]!, { + ...cell, + unused: { intent: signedIntent, confirmation: signedConfirmation }, + }); + }); + }); + } + async function row(tx: Pick, id: string, forUpdate = false) { const rows = await tx.unsafe( - "SELECT id,metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND metadata->'configurationMaintenance'->'receipt'->>'maintenanceId'=$1", + "SELECT id,metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND metadata->'configurationMaintenance'->'receipt'->>'maintenanceId'=$1" + + (forUpdate ? " FOR UPDATE" : ""), [id], ); if (rows.length > 1) throw Error("Duplicate configuration fence"); @@ -164,59 +1515,274 @@ export function createDataConfigurationFenceStore( .configurationMaintenance, ); } + function withPreparedWitnessDirectory(run: (directory: AnchoredDirectory) => T): T { + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(local.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")), + role = own(executions.directory("data-manager")); + return run(role); + } finally { + for (const directory of opened.reverse()) directory.close(); + } + } + function reservePreparedWitness( + authority: import("./control-authority").VerifiedExecutionAuthority, + ): void { + const state = executionAuthorityState(authority); + if (state.kind !== "configuration-maintenance.prepare" || !state.origin) + throw Error("Original DM preparation required"); + withPreparedWitnessDirectory((directory) => { + const name = `${state.execution.operationKey}.data-terminal.json`; + try { + const prior = directory.read(name, 8192).toString("utf8"); + verifyFamilyOwnedRecord( + local, + state.origin!, + { scope: "shared", namespace: "execution", container: "data-manager", name }, + prior, + ); + if (prior !== "null") { + const published = verifyControl( + local.installation.publicKey, + dataConfigurationFencePreparedWitnessDomain, + JSON.parse(prior), + dataConfigurationFencePreparedWitnessSchema, + ); + if ( + canonicalJson(published.prepareExecution) !== canonicalJson(state.execution) || + published.prepareAdmissionDigest !== state.receiptDigest + ) + throw Error("Prepared source reservation conflict"); + } + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + familyRecordWrite( + local, + state.origin!, + { scope: "shared", namespace: "execution", container: "data-manager", name }, + "null", + (track) => directory.write(name, "null", true, track), + true, + ); + }); + } + /** The fixed local producer reads its own original SQL/mirror/canonical/effect + * sources. No caller receipt, digest, time or success flag is accepted. */ + async function publishPreparedWitness( + authority: import("./control-authority").VerifiedExecutionAuthority, + ): Promise { + const state = executionAuthorityState(authority); + if (state.kind !== "configuration-maintenance.prepare" || !state.origin) + throw Error("Original DM preparation required"); + const admitted = z + .object({ + payload: z.object({ + operation: z + .object({ kind: z.literal("configuration-maintenance.prepare") }) + .passthrough(), + }), + }) + .parse(readPrivateRecord(local, "executions", `${state.execution.operationKey}.json`, 65536)); + const { kind: _kind, ...body } = admitted.payload.operation; + const input = dataConfigurationFencePrepareSchema.parse(body); + await sql.transaction(async (tx) => { + await lock(tx); + const saved = await row(tx, input.maintenanceId), + owned = mirror(input.maintenanceId); + if ( + !saved || + !["prepared", "settled"].includes(saved.guardState) || + canonicalJson(saved) !== canonicalJson(owned.metadata) || + canonicalJson(saved.input) !== canonicalJson(input) + ) + throw Error("Original prepared guard source unavailable"); + const effect = verifyControl( + local.installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord(local, "effects", `${state.execution.operationKey}.json`, 65536), + z.strictObject({ + identity: executionRefSchema, + outcome: z.literal("committed"), + publication: z.literal("committed"), + terminalReceiptDigest: z.string().regex(/^[a-f0-9]{64}$/), + receipt: z.strictObject({ + kind: z.literal("data-configuration-fence"), + receipt: dataConfigurationFenceReceiptSchema, + guardDigest: z.string().regex(/^[a-f0-9]{64}$/), + }), + }), + ); + const receipt = effect.receipt.receipt; + const original = { + ...owned, + metadata: { + version: 1 as const, + guardState: "prepared" as const, + receipt, + input, + ...(saved.settlementReservation + ? { settlementReservation: saved.settlementReservation } + : {}), + }, + }; + const terminal = verifyControl( + local.installation.publicKey, + "openmapx-data-fence-terminal-v1", + readPrivateRecord( + local, + "observations", + `terminal-${state.execution.operationKey}.json`, + 8192, + ), + fenceTerminalSchema, + ); + const times = await tx.unsafe( + "SELECT finished_at FROM data_manager.jobs WHERE id=$1::uuid FOR UPDATE", + [receipt.fenceId], + ); + if ( + receipt.state !== "prepared" || + receipt.sequence !== 0 || + receipt.fenceId !== state.execution.executionId || + canonicalJson(effect.identity) !== canonicalJson(state.execution) || + canonicalJson({ ...saved.receipt, state: "prepared", sequence: 0 }) !== + canonicalJson(receipt) || + effect.terminalReceiptDigest !== controlDigest(effect.receipt) || + effect.receipt.guardDigest !== controlDigest(original) || + terminal.phase !== "canonical" || + terminal.admissionDigest !== state.receiptDigest || + canonicalJson(terminal.execution) !== canonicalJson(state.execution) || + terminal.terminalDigest !== dataFenceTerminalDigest(receipt) || + terminal.terminalAt !== owned.preparedAt || + !times[0] || + retainedDataDatabaseTimestamp(times[0].finished_at).getTime() !== terminal.terminalAt + ) + throw Error("Original prepared canonical evidence changed"); + verifyControlEffectSettlement(installation, state.execution); + const value = dataConfigurationFencePreparedWitnessSchema.parse({ + version: 2, + installationId: installation.installationId, + origin: state.origin, + ownerPrepare: input.ownerPrepare, + prepareExecution: state.execution, + prepareAdmissionDigest: state.receiptDigest, + rootCertificateDigest: owned.ownerCertificateDigest, + preparedReceipt: receipt, + preparedReceiptDigest: settlementRecordDigest( + configurationSettlementDomains.preparedReceipt, + receipt, + ), + preparedCanonicalDigest: terminal.terminalDigest, + preparedTerminalAt: terminal.terminalAt, + guardDigest: effect.receipt.guardDigest, + effectReceiptDigest: effect.terminalReceiptDigest, + effectSettlementDigest: controlDigest(effect), + }); + const bytes = canonicalJson( + signControl(local.privateKey, dataConfigurationFencePreparedWitnessDomain, value), + ); + if (Buffer.byteLength(bytes) > 8192) throw Error("Prepared witness capacity exceeded"); + withPreparedWitnessDirectory((directory) => { + const name = `${state.execution.operationKey}.data-terminal.json`, + prior = directory.read(name, 8192).toString("utf8"), + before = directory.fileIdentity(name); + verifyFamilyOwnedRecord( + local, + state.origin!, + { scope: "shared", namespace: "execution", container: "data-manager", name }, + prior, + ); + if (prior === bytes) return; + if (prior !== "null") throw Error("Prepared canonical witness conflict"); + familyRecordWrite( + local, + state.origin!, + { scope: "shared", namespace: "execution", container: "data-manager", name }, + bytes, + (track) => + directory.write(name, bytes, false, (inode) => { + if ( + !sameInode(before, directory.fileIdentity(name)) || + directory.read(name, 8192).toString("utf8") !== prior + ) + throw Error("Prepared witness changed during publication"); + track(inode); + }), + ); + }); + }); + } function inspectGrant( body: DataConfigurationFencePrepare | DataConfigurationFenceSettle, rawHeaders: readonly string[], kind: "configuration-maintenance.prepare" | "configuration-maintenance.settle", owner?: DataConfigurationFencePrepare, ) { - const headers = controlHeaders(rawHeaders); - if (!headers.delegation || headers.parent) throw Error("Pinned agent fence grant required"); - const grant = inspectArtifactDelegation({ - dataRoot: local.dataRoot, - assertion: headers.delegation, - installation, - }); - const operation = { kind, ...body }, - parent = "ownerPrepare" in body ? body.ownerPrepare : owner?.ownerPrepare; - if ( - grant.recipient !== "data-manager" || - grant.parent.executor !== "ops-agent" || - grant.kind !== kind || - grant.phase !== - (kind.endsWith(".prepare") ? "configuration-fence" : "configuration-fence-settle") || - !parent || - controlDigest(grant.parent) !== controlDigest(parent) || - grant.execution.operationFingerprint !== controlDigest(operation) || - grant.execution.executor !== "data-manager" || - grant.execution.fingerprint !== - executionFingerprint({ - kind, - operationFingerprint: controlDigest(operation), - deadlineAt: grant.execution.deadlineAt, - parent: retainedControlParent(grant.parent), - }) - ) - throw Error("Configuration fence grant identity conflict"); - if ("serviceIds" in body && canonicalJson(grant.serviceIds) !== canonicalJson(body.serviceIds)) - throw Error("Configuration fence target subset conflict"); - const peer = verifyControlPeer(installation, { - rawHeaders, - transportPrincipal: "shared-data-manager-bearer", - execution: grant.execution, - kind, - }); - return { - grant, - peer, - operation, - parent: verifiedParentForArtifactDelegation( + try { + const headers = controlHeaders(rawHeaders); + if (!headers.delegation || headers.parent) throw Error("Pinned agent fence grant required"); + const grant = inspectArtifactDelegation({ + dataRoot: local.dataRoot, + assertion: headers.delegation, installation, - headers.delegation, - grant.execution, - ), - assertion: headers.delegation, - }; + }); + const operation = { kind, ...body }, + parent = "ownerPrepare" in body ? body.ownerPrepare : owner?.ownerPrepare; + if ( + grant.recipient !== "data-manager" || + grant.parent.executor !== "ops-agent" || + grant.kind !== kind || + grant.phase !== + (kind.endsWith(".prepare") ? "configuration-fence" : "configuration-fence-settle") || + !parent || + controlDigest(grant.parent) !== controlDigest(parent) || + grant.execution.operationFingerprint !== controlDigest(operation) || + grant.execution.executor !== "data-manager" || + grant.execution.fingerprint !== + executionFingerprint({ + kind, + operationFingerprint: controlDigest(operation), + deadlineAt: grant.execution.deadlineAt, + parent: retainedControlParent(grant.parent), + }) + ) + throw Error("Configuration fence grant identity conflict"); + if ( + "serviceIds" in body && + canonicalJson(grant.serviceIds) !== canonicalJson(body.serviceIds) + ) + throw Error("Configuration fence target subset conflict"); + const peer = verifyControlPeer(installation, { + rawHeaders, + transportPrincipal: "shared-data-manager-bearer", + execution: grant.execution, + kind, + }); + return { + grant, + peer, + operation, + parent: verifiedParentForArtifactDelegation( + installation, + headers.delegation, + grant.execution, + ), + assertion: headers.delegation, + }; + } catch { + // Classify verification failures before any guard SQL, mirror or adoption. + throw Object.assign(Error("Pinned agent configuration fence grant rejected"), { + statusCode: 403, + }); + } } async function withLeaf( verified: ReturnType, @@ -272,6 +1838,7 @@ export function createDataConfigurationFenceStore( .parse(priorEffect).payload.receipt.kind; if (kind === "data-configuration-fence-refused") { verifyControlEffectSettlement(installation, grant.execution); + await markRefusedConfigurationSettlementCapacityUnused(authority); throw new DataFenceBusy(); } const original = verifyControl( @@ -291,12 +1858,15 @@ export function createDataConfigurationFenceStore( ); const saved = await row(sql, original.receipt.receipt.maintenanceId); if (!saved) throw new DataFenceBusy(); + if (grant.kind === "configuration-maintenance.prepare") + await publishPreparedWitness(authority); return saved.receipt; } if (grant.kind === "configuration-maintenance.prepare") requireControlJournalRetention(authority); if (grant.kind === "configuration-maintenance.prepare") assertArtifactTargetLeaseForward(lease); + if (grant.kind === "configuration-maintenance.prepare") reservePreparedWitness(authority); beginControlEffect(authority); let receipt: DataConfigurationFenceReceipt; try { @@ -305,6 +1875,8 @@ export function createDataConfigurationFenceStore( // Before any preparing mirror exists the fixed guard path has made // no logical write. A busy result can settle this exact unused leaf. if (error instanceof DataFenceBusy) { + if (error instanceof DataFenceCapacityRefused) + refusedConfigurationSettlementCapacityState(error.proof, authority); let absent = false; try { readPrivateRecord( @@ -328,7 +1900,10 @@ export function createDataConfigurationFenceStore( kind: "data-configuration-fence-refused" as const, maintenanceId: z.object({ maintenanceId: z.uuid() }).parse(verified.operation) .maintenanceId, - reason: "busy" as const, + reason: + error instanceof DataFenceCapacityRefused + ? ("capacity" as const) + : ("busy" as const), }, effect = { identity: grant.execution, @@ -392,6 +1967,8 @@ export function createDataConfigurationFenceStore( ); completeFamilyJournal(local, state.origin, grant.execution, terminalAt); } + await publishDataConfigurationFenceRefusalProjection(authority); + await markRefusedConfigurationSettlementCapacityUnused(authority); } } throw error; @@ -468,6 +2045,7 @@ export function createDataConfigurationFenceStore( stored.preparedAt, ); completeFamilyJournal(local, state.origin, grant.execution, stored.preparedAt); + await publishPreparedWitness(authority); } return receipt; } finally { @@ -489,6 +2067,86 @@ export function createDataConfigurationFenceStore( ); if (unknown.length) throw new DataFenceBusy(); } + async function inspectGenerationAdmission( + value: unknown, + rawHeaders: readonly string[], + parent: VerifiedParentExecution | undefined, + tx: Pick, + ) { + const request = startGenerationHardlinksExecutionSchema.parse(value), + headers = controlHeaders(rawHeaders); + if (!headers.delegation || headers.parent) + throw Object.assign(Error("Signed generation grant required"), { statusCode: 403 }); + const grant = inspectArtifactDelegation({ + dataRoot: local.dataRoot, + assertion: headers.delegation, + installation, + }); + if ( + grant.recipient !== "data-manager" || + grant.kind !== "generation-hardlinks" || + grant.phase !== "hardlinks" || + grant.execution.operationKey !== request.operationKey || + grant.execution.operationFingerprint !== controlDigest(request.input) || + request.input.planDigest !== controlDigest(request.plan) || + request.plan.plan.some((entry) => !grant.serviceIds.includes(entry.consumerService)) + ) + throw Object.assign(Error("Generation plan/grant binding conflict"), { statusCode: 403 }); + if (startup === "closed") throw new DataFenceBusy(); + const rows = await tx.unsafe( + "SELECT metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled' LIMIT 2", + ); + if (rows.length > 1) throw new DataFenceBusy(); + const saved = rows[0] + ? z.object({ configurationMaintenance: metadataSchema }).parse(rows[0].metadata) + .configurationMaintenance + : undefined; + if (!saved) { + if ([...mirrors.values()].some((mirror) => mirror.metadata.guardState !== "settled")) + throw new DataFenceBusy(); + return; + } + const owned = mirror(saved.receipt.maintenanceId); + if (canonicalJson(owned.metadata) !== canonicalJson(saved)) throw new DataFenceBusy(); + if ( + grant.ownerCertificateDigest !== owned.ownerCertificateDigest || + canonicalJson(grant.owner) !== canonicalJson(saved.receipt.ownerPrepare) || + grant.owner.executor !== "ops-agent" || + request.input.revisionId !== saved.receipt.revisionId || + request.input.planDigest !== saved.receipt.hardlinkPlanDigest || + request.input.coordinatorMode !== saved.input.coordinatorMode || + grant.serviceIds.some((id) => !saved.input.serviceIds.includes(id)) + ) + throw Object.assign(Error("Generation maintenance owner/revision/hash conflict"), { + statusCode: 403, + }); + if ( + saved.guardState !== "prepared" || + saved.receipt.state !== "prepared" || + saved.receipt.sequence !== 0 || + !parent + ) + throw new DataFenceBusy(); + try { + verifyConfigurationMaintenanceGenerationWitness(installation, request, parent, grant, { + maintenanceId: saved.receipt.maintenanceId, + candidateId: saved.receipt.candidateId, + ownerPrepare: saved.receipt.ownerPrepare, + revisionId: saved.receipt.revisionId, + hardlinkPlanDigest: saved.receipt.hardlinkPlanDigest!, + coordinatorMode: saved.input.coordinatorMode, + serviceIds: saved.input.serviceIds, + rootCertificateDigest: owned.ownerCertificateDigest, + }); + } catch { + throw new DataFenceBusy(); + } + const unknown = await tx.unsafe( + "SELECT id FROM data_manager.jobs WHERE kind <> 'configuration-maintenance' AND (execution_state->>'publication'='unknown' OR (execution_version IS NULL AND finished_at IS NULL AND status='interrupted')) LIMIT 1", + ); + if (unknown.length) throw new DataFenceBusy(); + return { request, grant, guardDigest: controlDigest(saved) }; + } return { async startup() { startup = "closed"; @@ -501,7 +2159,7 @@ export function createDataConfigurationFenceStore( continue; if (!/^[a-f0-9-]{36}\.json$/.test(file)) throw Error("Unexpected configuration fence file"); - const value = mirror(file.slice(0, -5)); + const value = signedMirrorData(file.slice(0, -5)).parsed; mirrors.set(value.metadata.receipt.maintenanceId, value); } } catch (error) { @@ -509,6 +2167,21 @@ export function createDataConfigurationFenceStore( } finally { directory?.close(); } + const committed = await sql.unsafe( + "SELECT metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND metadata->'configurationMaintenance'->'settlementControl' IS NOT NULL LIMIT 4097", + ); + if (committed.length > 4096) throw Error("Configuration settlement recovery capacity"); + for (const raw of committed) { + const value = z + .object({ configurationMaintenance: metadataSchema }) + .parse(raw.metadata).configurationMaintenance; + if (value.guardState === "settled" && value.settlementControl && value.settlement) { + await recoverCommittedConfigurationFenceSettlement( + dataConfigurationFenceSettleSchema.parse(value.settlement), + ); + } + } + for (const id of mirrors.keys()) mirrors.set(id, mirror(id)); const rows = await sql.unsafe( "SELECT metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled' LIMIT 2", ); @@ -517,7 +2190,9 @@ export function createDataConfigurationFenceStore( const value = z .object({ configurationMaintenance: metadataSchema }) .parse(raw.metadata).configurationMaintenance, - saved = mirrors.get(value.receipt.maintenanceId); + originalId = value.receipt.maintenanceId; + await observeUnsettledConfigurationFenceSettlement(originalId); + const saved = mirrors.get(originalId); if (!saved || canonicalJson(saved.metadata) !== canonicalJson(value)) throw Error("Configuration guard mirror/database recovery required"); } @@ -530,57 +2205,60 @@ export function createDataConfigurationFenceStore( startup = rows.length ? "fenced" : "open"; }, assertAdmission, - async assertGenerationAdmission(value: unknown, rawHeaders: readonly string[]): Promise { - const request = startGenerationHardlinksExecutionSchema.parse(value), - headers = controlHeaders(rawHeaders); - if (!headers.delegation || headers.parent) - throw Object.assign(Error("Signed generation grant required"), { statusCode: 403 }); - const grant = inspectArtifactDelegation({ - dataRoot: local.dataRoot, - assertion: headers.delegation, + async assertGenerationAdmission( + value: unknown, + rawHeaders: readonly string[], + parent?: VerifiedParentExecution, + ): Promise { + const checked = await inspectGenerationAdmission(value, rawHeaders, parent, sql); + if (!checked) return undefined; + const proof = Object.freeze({}) as VerifiedPreparedGenerationAdmission; + preparedGenerations.set(proof, { installation, + request: checked.request, + headers: Object.freeze([...rawHeaders]), + parent: parent!, + guardDigest: checked.guardDigest, + execution: checked.grant.execution, }); + return proof; + }, + async assertPreparedGenerationAdmission( + tx: Pick, + proof: VerifiedPreparedGenerationAdmission, + input: unknown, + admission: { + operationKey: string; + requestedDeadlineAt?: number; + parent?: VerifiedParentExecution; + id?: string; + principal?: string; + }, + ): Promise { + const source = preparedGenerations.get(proof); if ( - grant.recipient !== "data-manager" || - grant.kind !== "generation-hardlinks" || - grant.phase !== "hardlinks" || - grant.execution.operationKey !== request.operationKey || - grant.execution.operationFingerprint !== controlDigest(request.input) || - request.input.planDigest !== controlDigest(request.plan) || - request.plan.plan.some((entry) => !grant.serviceIds.includes(entry.consumerService)) + !source || + source.installation !== installation || + source.parent !== admission.parent || + canonicalJson(input) !== canonicalJson(source.request.input) || + admission.operationKey !== source.request.operationKey || + admission.requestedDeadlineAt !== source.request.requestedDeadlineAt || + admission.id !== source.execution.executionId || + admission.principal !== source.execution.principal ) - throw Object.assign(Error("Generation plan/grant binding conflict"), { statusCode: 403 }); - if (startup === "closed") throw new DataFenceBusy(); - const rows = await sql.unsafe( - "SELECT metadata FROM data_manager.jobs WHERE kind='configuration-maintenance' AND coalesce(metadata->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled' LIMIT 2", + throw new DataFenceBusy(); + const checked = await inspectGenerationAdmission( + source.request, + source.headers, + source.parent, + tx, ); - if (rows.length > 1) throw new DataFenceBusy(); - const saved = rows[0] - ? z.object({ configurationMaintenance: metadataSchema }).parse(rows[0].metadata) - .configurationMaintenance - : undefined; - if (!saved) { - if ([...mirrors.values()].some((mirror) => mirror.metadata.guardState !== "settled")) - throw new DataFenceBusy(); - return; - } - const owned = mirror(saved.receipt.maintenanceId); - if (canonicalJson(owned.metadata) !== canonicalJson(saved)) throw new DataFenceBusy(); if ( - grant.ownerCertificateDigest !== owned.ownerCertificateDigest || - canonicalJson(grant.owner) !== canonicalJson(saved.receipt.ownerPrepare) || - grant.owner.executor !== "ops-agent" || - request.input.revisionId !== saved.receipt.revisionId || - request.input.planDigest !== saved.receipt.hardlinkPlanDigest || - request.input.coordinatorMode !== saved.input.coordinatorMode || - grant.serviceIds.some((id) => !saved.input.serviceIds.includes(id)) + !checked || + checked.guardDigest !== source.guardDigest || + canonicalJson(checked.grant.execution) !== canonicalJson(source.execution) ) - throw Object.assign(Error("Generation maintenance owner/revision/hash conflict"), { - statusCode: 403, - }); - //01.2 owns the real frozen-step/DB-decision witness and positive - // continuation. A matching prepared receipt alone authorizes no write. - throw new DataFenceBusy(); + throw new DataFenceBusy(); }, async withAdmission(run: (tx: Pick) => Promise): Promise { return sql.transaction(async (tx) => { @@ -602,6 +2280,11 @@ export function createDataConfigurationFenceStore( return withLeaf(verified, async () => existing.receipt); } if (startup === "closed") throw new DataFenceBusy(); + const reservation = verifyPublishedConfigurationSettlementReservation( + installation, + verified.peer, + input, + ); const receipt = dataConfigurationFenceReceiptSchema.parse({ fenceId: verified.grant.execution.executionId, maintenanceId: input.maintenanceId, @@ -630,6 +2313,15 @@ export function createDataConfigurationFenceStore( metadata, }; return withLeaf(verified, async (authority) => { + const capacity = reserveReceivedConfigurationSettlementCapacity(authority, reservation); + if (!("capacity" in capacity)) throw new DataFenceCapacityRefused(capacity); + metadata.settlementReservation = { + version: 2, + settlementId: capacity.capacity.settlementId, + projectionDigest: capacity.capacity.projectionDigest, + reservationDigest: capacity.capacity.reservationDigest, + capacityReceiptDigest: capacity.capacityDigest, + }; await sql.transaction(async (tx) => { await lock(tx); await assertAdmission(tx); @@ -713,15 +2405,32 @@ export function createDataConfigurationFenceStore( witness.expiresAt - witness.issuedAt > 15000 ) throw Error("Expired or foreign fence witness"); - const saved = await row(sql, maintenanceId); + let saved = await row(sql, maintenanceId); if (!saved) return null; - const proof = mirror(maintenanceId); + let proof = + saved.guardState === "settled" && saved.settlementControl + ? signedMirrorData(maintenanceId).parsed + : mirror(maintenanceId); if ( saved.receipt.candidateId !== witness.candidateId || - proof.ownerCertificateDigest !== witness.ownerCertificateDigest || - canonicalJson(proof.metadata) !== canonicalJson(saved) + proof.ownerCertificateDigest !== witness.ownerCertificateDigest ) throw Error("Configuration guard recovery required"); + if (saved.guardState === "settled" && saved.settlementControl && saved.settlement) { + await recoverCommittedConfigurationFenceSettlement( + dataConfigurationFenceSettleSchema.parse(saved.settlement), + ); + saved = await row(sql, maintenanceId); + if (!saved) throw Error("Original settled SQL source missing"); + proof = mirror(maintenanceId); + } + const recovery = + saved.guardState === "prepared" + ? await observeUnsettledConfigurationFenceSettlement(maintenanceId) + : null; + if (recovery) proof = mirror(maintenanceId); + if (canonicalJson(proof.metadata) !== canonicalJson(saved)) + throw Error("Configuration guard recovery required"); const anchored = inspectArtifactDelegation({ dataRoot: local.dataRoot, assertion: proof.prepareAssertion, @@ -740,59 +2449,19 @@ export function createDataConfigurationFenceStore( controlDigest(saved.receipt.ownerPrepare) ) throw Error("Original fence owner required"); - return saved.receipt; + return recovery ?? saved.settlementControl?.tuple.result ?? saved.receipt; }, async settle(value: DataConfigurationFenceSettle, rawHeaders: readonly string[]) { const input = dataConfigurationFenceSettleSchema.parse(value), - saved = await row(sql, input.maintenanceId); - if (!saved) throw Object.assign(Error("Unknown configuration fence"), { statusCode: 404 }); - const verified = inspectGrant( - input, - rawHeaders, - "configuration-maintenance.settle", - saved.input, - ); - if ( - saved.receipt.candidateId !== input.candidateId || - saved.receipt.planDigest !== input.planDigest - ) - throw Error("Configuration guard settlement identity conflict"); - if (saved.guardState === "settled") { - if (canonicalJson(saved.settlement) !== canonicalJson(input)) - throw Error("Configuration guard settlement conflict"); - return withLeaf(verified, async () => saved.receipt); + peer = await verifyConfigurationFenceSettlementControl(input, rawHeaders), + adoption = await adoptConfigurationFenceSettlementControl(peer); + try { + return adoption.runOriginalGuardTransition + ? await adoption.runOriginalGuardTransition() + : await adoption.inspect(); + } finally { + await adoption.close(); } - if (saved.receipt.sequence !== input.expectedSequence) - throw Error("Configuration guard sequence conflict"); - const receipt = { - ...saved.receipt, - state: "settled" as const, - sequence: saved.receipt.sequence + 1, - }, - metadata = metadataSchema.parse({ ...saved, guardState: "settling", settlement: input }); - const owned = mirror(input.maintenanceId); - return withLeaf(verified, async () => { - await sql.transaction(async (tx) => { - await lock(tx); - const current = await row(tx, input.maintenanceId); - if (canonicalJson(current) !== canonicalJson(saved)) - throw Error("Configuration guard settlement CAS conflict"); - const active = await tx.unsafe( - "SELECT id FROM data_manager.jobs WHERE kind <> 'configuration-maintenance' AND (finished_at IS NULL OR execution_state->>'publication'='unknown') LIMIT 1", - ); - if (active.length) throw new DataFenceBusy(); - write({ ...owned, metadata }); - metadata.guardState = "settled"; - metadata.receipt = receipt; - await tx.unsafe( - "UPDATE data_manager.jobs SET metadata=$2::text::jsonb,ownership_version=ownership_version+1 WHERE id=$1::uuid", - [receipt.fenceId, canonicalJson({ configurationMaintenance: metadata })], - ); - }); - write({ ...owned, metadata }); - startup = "open"; - return receipt; - }); }, }; } @@ -852,6 +2521,9 @@ export function verifyRetainedDataFenceTerminal( guardState: "prepared" as const, receipt: effect.receipt.receipt, input: metadata.input, + ...(metadata.settlementReservation + ? { settlementReservation: metadata.settlementReservation } + : {}), }, }; if ( @@ -879,7 +2551,14 @@ export function verifyRetainedDataFenceTerminal( ) throw Error("Original logical guard terminal commitment required"); verifyControlEffectSettlement(installation, execution); - return { digest: commitment.terminalDigest, terminalAt: commitment.terminalAt }; + const control = metadata.settlementControl + ? readPreparedConfigurationSettlementRetirement(installation, proof.origin) + : null; + return { + digest: commitment.terminalDigest, + terminalAt: commitment.terminalAt, + retentionAt: Math.max(commitment.terminalAt, control?.terminalAt ?? 0), + }; } export function dataFenceRetirementTuple(raw: Record) { @@ -887,6 +2566,11 @@ export function dataFenceRetirementTuple(raw: Record) { .object({ configurationMaintenance: metadataSchema }) .parse(raw.metadata).configurationMaintenance; if (metadata.guardState !== "settled") throw Error("Unsettled guard cannot retire"); + if ( + metadata.settlementControl && + Date.now() - metadata.settlementControl.tuple.terminalAt < 72 * 3600000 + ) + throw Error("Original logical control retention floor not reached"); return { digest: dataFenceTerminalDigest({ ...metadata.receipt, state: "prepared", sequence: 0 }), terminalAt: retainedDataDatabaseTimestamp(raw.finished_at).getTime(), @@ -910,7 +2594,7 @@ export function verifyRetainedDataFenceRefusal( receipt: z.strictObject({ kind: z.literal("data-configuration-fence-refused"), maintenanceId: z.uuid(), - reason: z.literal("busy"), + reason: z.enum(["busy", "capacity"]), }), }) .passthrough(), @@ -931,5 +2615,11 @@ export function verifyRetainedDataFenceRefusal( ) throw Error("Original configuration refusal commitment required"); verifyControlEffectSettlement(installation, execution); - return { digest: commitment.terminalDigest, terminalAt: commitment.terminalAt }; + return { + digest: commitment.terminalDigest, + terminalAt: commitment.terminalAt, + receipt: effect.receipt, + effectReceiptDigest: controlDigest(effect.receipt), + effectSettlementDigest: controlDigest(effect), + }; } diff --git a/packages/core/src/build-artifacts/data-execution-candidate.ts b/packages/core/src/build-artifacts/data-execution-candidate.ts index 6230911ec..81b381b12 100644 --- a/packages/core/src/build-artifacts/data-execution-candidate.ts +++ b/packages/core/src/build-artifacts/data-execution-candidate.ts @@ -11,10 +11,12 @@ import { } from "node:fs"; import { join, sep } from "node:path"; import { z } from "zod"; -import type { OpsOperation } from "../ops/contract"; +import { canonicalJson } from "../ops/canonical-json"; +import { type OpsJobStatus, type OpsOperation, parseOpsJobStatus } from "../ops/contract"; +import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; import { retainedParentExecutionSchema } from "../ops/execution-identity"; -import { executionRefSchema } from "../ops/execution-ref"; -import { canonicalJson } from "../ops/fingerprint"; +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { opsOperationFingerprint } from "../ops/fingerprint"; // biome-ignore lint/suspicious/noImportCycles: Candidate snapshots and fixed proof publication share the original phase/authority; calls occur after schema initialization. import { inspectControlChildPublication } from "./child-phase"; // biome-ignore-start lint/suspicious/noImportCycles: Nominal execution verification is reused by the fixed producer after module initialization. @@ -29,6 +31,7 @@ import { readPrivateRecord, retainedControlParent, type VerifiedExecutionAuthority, + withPrivateEffectWriter, writePrivateRecord, } from "./control-authority"; // biome-ignore-end lint/suspicious/noImportCycles: Shared original execution verification described above. @@ -41,8 +44,11 @@ import { verifyControl, } from "./control-records"; import { AnchoredDirectory, identity, sameInode } from "./filesystem"; +// biome-ignore-start lint/suspicious/noImportCycles: Candidate provenance reads original indexed bytes; retirement checks original admissions. These reciprocal read-only verifiers run after module initialization. import { familyRecordWrite, verifyFamilyOwnedRecord } from "./retention-family"; +// biome-ignore-end lint/suspicious/noImportCycles: Exact deferred original-owner/index guard join. + const schema = z.strictObject({ version: z.literal(1), execution: executionRefSchema, @@ -564,3 +570,418 @@ export function assertDataTransitCandidateForExecution( verifyDataTransitCandidateForExecution(authority, operation, expectedDigest); assertControlExecutionForward(authority); } + +/** Fixed proxy proof helpers share the original candidate verifier and indexed + * effect writer. Their private schemas initialize here; no authority is created. */ +export const { + feedProxyValidationCompletionDigest, + reserveFeedProxyValidation, + commitFeedProxyValidationFailure, + recordFeedProxyReloadAttempted, + inspectFeedProxyValidationFailure, + publishFeedProxyCanonicalTerminal, + verifyFeedProxyCanonicalTerminal, +} = (() => { + const digest = z.string().regex(/^[a-f0-9]{64}$/); + const schema = z + .strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: digest, + candidateId: z.string().max(64), + candidateDigest: digest, + reservedAt: z.number().int().nonnegative().safe(), + phase: z.enum(["validation_pending", "validation_failed", "reload_started"]), + completedAt: z.number().int().nonnegative().safe().nullable(), + validationExitCode: z.number().int().min(1).max(255).nullable(), + completionDigest: digest.nullable(), + padding: z.string().max(4096), + }) + .superRefine((p, ctx) => { + if ( + (p.phase === "validation_failed") !== + (p.validationExitCode !== null && p.completionDigest !== null) || + (p.phase === "validation_pending") !== (p.completedAt === null) + ) + ctx.addIssue({ code: "custom", message: "Invalid fixed validation phase" }); + }); + type Proof = z.infer; + const domain = "openmapx-feed-proxy-validation-v2"; + const name = (authority: VerifiedExecutionAuthority) => + `proxy-validation-${authority.execution.operationKey}.json`; + function state(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority); + if ( + s.installation.installation.role !== "ops-agent" || + s.kind !== "feedProxy.validateAndReload" || + !s.origin + ) + throw Error("Original fixed proxy authority required"); + return s; + } + function read(authority: VerifiedExecutionAuthority) { + const s = state(authority), + raw = readPrivateRecord(s.installation, "observations", name(authority), 4096); + const p = verifyControl(s.installation.installation.publicKey, domain, raw, schema); + if ( + p.installationId !== s.installation.installation.installationId || + !sameControlOrigin(p.origin, s.origin) || + p.admissionDigest !== s.receiptDigest || + controlDigest(p.execution) !== controlDigest(s.execution) || + opsOperationFingerprint({ + kind: "feedProxy.validateAndReload", + candidateId: p.candidateId, + }) !== s.execution.operationFingerprint + ) + throw Error("Original validation proof identity changed"); + verifyFamilyOwnedRecord( + s.installation, + p.origin, + { scope: "private", namespace: "observations", container: "", name: name(authority) }, + canonicalJson(raw), + ); + return p; + } + function write(authority: VerifiedExecutionAuthority, value: Proof, exclusive = false) { + const s = state(authority), + p = { ...value, padding: "" }; + // Account the entire cell up front; transitions allocate no additional space. + const signed = signControl(s.installation.privateKey, domain, p); + const room = 4096 - Buffer.byteLength(canonicalJson(signed)); + if (room < 0) throw Error("Validation proof capacity exceeded"); + p.padding = " ".repeat(room); + const parsed = schema.parse(p); + writePrivateRecord( + s.installation, + "observations", + name(authority), + signControl(s.installation.privateKey, domain, parsed), + exclusive, + s.origin, + ); + } + function feedProxyValidationCompletionDigest( + execution: ExecutionRef, + candidateDigest: string, + nonce: string, + validationExitCode: number, + ): string { + return controlDigest({ + domain: "openmapx-feed-proxy-validation-completion-v1", + execution, + candidateDigest, + nonce, + validationExitCode, + }); + } + function reserveFeedProxyValidation( + authority: VerifiedExecutionAuthority, + candidateId: string, + candidateDigest: string, + ): void { + const s = state(authority); + assertControlExecutionForward(authority); + verifyDataTransitCandidateForExecution( + authority, + { kind: "feedProxy.validateAndReload", candidateId }, + candidateDigest, + ); + withPrivateEffectWriter(s, () => { + let raw: unknown; + try { + raw = readPrivateRecord(s.installation, "observations", name(authority), 4096); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (raw !== null && raw !== undefined) { + const prior = read(authority); + if (prior.candidateId !== candidateId || prior.candidateDigest !== candidateDigest) + throw Error("Validation candidate changed"); + const dir = witnessDirectory(s.installation); + try { + dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096); + } finally { + dir.close(); + } + return; + } + write( + authority, + schema.parse({ + version: 2, + installationId: s.installation.installation.installationId, + origin: s.origin, + execution: s.execution, + admissionDigest: s.receiptDigest, + candidateId, + candidateDigest, + reservedAt: Date.now(), + phase: "validation_pending", + completedAt: null, + validationExitCode: null, + completionDigest: null, + padding: "", + }), + true, + ); + reserveWitness(authority); + }); + } + function commitFeedProxyValidationFailure( + authority: VerifiedExecutionAuthority, + input: { candidateDigest: string; validationExitCode: number; completionDigest: string }, + ): void { + const s = state(authority); + withPrivateEffectWriter(s, () => { + const p = read(authority); + if (p.candidateDigest !== input.candidateDigest) throw Error("Foreign validation candidate"); + if (p.phase === "validation_failed") { + if ( + p.validationExitCode !== input.validationExitCode || + p.completionDigest !== input.completionDigest + ) + throw Error("Validation commitment changed"); + return; + } + if (p.phase !== "validation_pending") throw Error("Reload already attempted"); + // A new producer commitment verifies the actual invocation's bytes. An + // identical existing commitment above observes its frozen original proof, + // which remains valid after a later legitimate restore replaces files. + verifyDataTransitCandidateForExecution( + authority, + { kind: "feedProxy.validateAndReload", candidateId: p.candidateId }, + p.candidateDigest, + ); + write( + authority, + schema.parse({ ...p, ...input, phase: "validation_failed", completedAt: Date.now() }), + ); + }); + } + function recordFeedProxyReloadAttempted(authority: VerifiedExecutionAuthority): void { + const s = state(authority); + withPrivateEffectWriter(s, () => { + const p = read(authority); + if (p.phase !== "validation_pending") throw Error("Validation phase already consumed"); + verifyDataTransitCandidateForExecution( + authority, + { kind: "feedProxy.validateAndReload", candidateId: p.candidateId }, + p.candidateDigest, + ); + write(authority, { ...p, phase: "reload_started", completedAt: Date.now() }); + }); + } + /** Frozen original proof stays valid after a legitimate restore replaces files. */ + function inspectFeedProxyValidationFailure( + authority: VerifiedExecutionAuthority, + expectedDigest?: string, + ): string | null { + const p = read(authority); + if (p.phase !== "validation_failed") return null; + const proofDigest = controlDigest(p); + if (expectedDigest !== undefined && expectedDigest !== proofDigest) + throw Error("Foreign validation commitment"); + return proofDigest; + } + + const terminalSchema = z.strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: digest, + phase: z.enum(["pending", "canonical"]), + status: z.unknown().nullable(), + terminalAt: z.number().int().positive().safe().nullable(), + terminalDigest: digest.nullable(), + padding: z.string().max(4096), + }); + function witnessDirectory(local: ReturnType["installation"]) { + const data = new AnchoredDirectory(local.dataRoot), + control = data.directory(".admin-target-control"), + executions = control.directory("executions"), + role = executions.directory("ops-agent"); + return { + role, + close() { + role.close(); + executions.close(); + control.close(); + data.close(); + }, + }; + } + function writeWitness( + authority: VerifiedExecutionAuthority, + payload: z.infer, + exclusive: boolean, + ) { + const s = state(authority), + dir = witnessDirectory(s.installation), + name = `${s.execution.operationKey}.proxy-terminal.json`; + try { + const value = { ...payload, padding: "" }, + domain = "openmapx-proxy-canonical-terminal-v2"; + const room = + 4096 - + Buffer.byteLength(canonicalJson(signControl(s.installation.privateKey, domain, value))); + if (room < 0) throw Error("Proxy terminal cell capacity exceeded"); + value.padding = " ".repeat(room); + const bytes = canonicalJson(signControl(s.installation.privateKey, domain, value)); + familyRecordWrite( + s.installation, + payload.origin, + { scope: "shared", namespace: "execution", container: "ops-agent", name }, + bytes, + (before) => dir.role.write(name, bytes, exclusive, before), + exclusive, + ); + } finally { + dir.close(); + } + } + function reserveWitness(authority: VerifiedExecutionAuthority) { + const s = state(authority); + writeWitness( + authority, + terminalSchema.parse({ + version: 2, + installationId: s.installation.installation.installationId, + origin: s.origin, + execution: s.execution, + admissionDigest: s.receiptDigest, + phase: "pending", + status: null, + terminalAt: null, + terminalDigest: null, + padding: "", + }), + true, + ); + } + /** Only an already sealed original complete canonical tuple can publish this + * finite shared proxy witness. It creates no new terminal time or status. */ + function publishFeedProxyCanonicalTerminal( + authority: VerifiedExecutionAuthority, + input: OpsJobStatus, + terminalAt: number, + ) { + const s = state(authority), + status = parseOpsJobStatus(input), + terminalDigest = controlTerminalReceiptDigest(status); + const original = verifyControl( + s.installation.installation.publicKey, + "openmapx-control-terminal-v1", + readPrivateRecord( + s.installation, + "observations", + `terminal-${s.execution.operationKey}.json`, + 8192, + ), + z + .object({ + phase: z.literal("canonical"), + execution: executionRefSchema, + admissionDigest: digest, + terminalAt: z.number(), + terminalDigest: digest, + }) + .passthrough(), + ); + if ( + controlDigest(original.execution) !== controlDigest(s.execution) || + original.admissionDigest !== s.receiptDigest || + original.terminalAt !== terminalAt || + original.terminalDigest !== terminalDigest + ) + throw Error("Original canonical proxy terminal required"); + const dir = witnessDirectory(s.installation); + try { + const prior = verifyControl( + s.installation.installation.publicKey, + "openmapx-proxy-canonical-terminal-v2", + JSON.parse( + dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), + ), + terminalSchema, + ); + if ( + prior.installationId !== s.installation.installation.installationId || + prior.admissionDigest !== s.receiptDigest || + !sameControlOrigin(prior.origin, s.origin) || + controlDigest(prior.execution) !== controlDigest(s.execution) + ) + throw Error("Original reserved proxy terminal changed"); + if (prior.phase === "canonical") { + if (prior.terminalDigest !== terminalDigest || prior.terminalAt !== terminalAt) + throw Error("Proxy canonical terminal changed"); + return; + } + // Diagnostics are observational and deliberately excluded from the exact + // canonical digest. Keep the fixed shared tuple bounded. + writeWitness( + authority, + { + ...prior, + phase: "canonical", + status: { ...status, diagnostics: { tail: [], truncated: false } }, + terminalDigest, + terminalAt, + }, + false, + ); + } finally { + dir.close(); + } + } + function verifyFeedProxyCanonicalTerminal( + parent: VerifiedExecutionAuthority, + execution: ExecutionRef, + ): OpsJobStatus { + const s = executionAuthorityState(parent), + dir = witnessDirectory(s.installation); + try { + const proof = verifyControl( + s.installation.publicKeys["ops-agent"], + "openmapx-proxy-canonical-terminal-v2", + JSON.parse( + dir.role.read(`${execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), + ), + terminalSchema, + ); + if ( + proof.phase !== "canonical" || + proof.installationId !== s.installation.installation.installationId || + !sameControlOrigin(proof.origin, s.origin) || + controlDigest(proof.execution) !== controlDigest(execution) || + !proof.terminalAt + ) + throw Error("Original canonical proxy child missing"); + const status = parseOpsJobStatus(proof.status); + if ( + status.kind !== "feedProxy.validateAndReload" || + status.operationId !== execution.executionId || + status.operationKey !== execution.operationKey || + status.fingerprint !== execution.fingerprint || + status.deadlineAt !== execution.deadlineAt || + controlTerminalReceiptDigest(status) !== proof.terminalDigest + ) + throw Error("Canonical proxy status changed"); + return status; + } finally { + dir.close(); + } + } + + return { + feedProxyValidationCompletionDigest, + reserveFeedProxyValidation, + commitFeedProxyValidationFailure, + recordFeedProxyReloadAttempted, + inspectFeedProxyValidationFailure, + publishFeedProxyCanonicalTerminal, + verifyFeedProxyCanonicalTerminal, + }; +})(); diff --git a/packages/core/src/build-artifacts/data-execution-journal.ts b/packages/core/src/build-artifacts/data-execution-journal.ts index 9a36a0e0e..2928d1b15 100644 --- a/packages/core/src/build-artifacts/data-execution-journal.ts +++ b/packages/core/src/build-artifacts/data-execution-journal.ts @@ -2,8 +2,8 @@ * can mint a completion or repair an absent original terminal commitment. */ import { randomUUID } from "node:crypto"; import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; -import { canonicalJson } from "../ops/fingerprint"; import { type DataExecutionResult, type DataExecutionStatus, @@ -59,6 +59,7 @@ import { restoreGenerationHardlinksLeaseForSettlement, type TargetLeaseAuthority, verifyRetainedArtifactTargetRootRelease, + verifyRetainedGenerationHardlinksSettlement, } from "./target-lease"; export interface DataExecutionJournalDatabase { @@ -1075,7 +1076,9 @@ export function createDataExecutionControlJournal( }, async publishContainedTerminal(execution: ExecutionRef) { const { value, authority, proof } = await verifyStored(execution); - verifyRetainedArtifactTargetRootRelease(authority); + if (value.status.kind === "generation-hardlinks") + verifyRetainedGenerationHardlinksSettlement(authority); + else verifyRetainedArtifactTargetRootRelease(authority); publishDataControlTerminalWitness(authority, value.status, proof.terminalAt); }, repairTerminal: async (execution: ExecutionRef) => { diff --git a/packages/core/src/build-artifacts/feed-proxy-validation.ts b/packages/core/src/build-artifacts/feed-proxy-validation.ts index 866afb922..f6101a411 100644 --- a/packages/core/src/build-artifacts/feed-proxy-validation.ts +++ b/packages/core/src/build-artifacts/feed-proxy-validation.ts @@ -1,419 +1,10 @@ -/** Closed ops-agent producer proof for completed nginx read-validation only. */ -import { z } from "zod"; -import { type OpsJobStatus, parseOpsJobStatus } from "../ops/contract"; -import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; -import type { ExecutionRef } from "../ops/execution-ref"; -import { executionRefSchema } from "../ops/execution-ref"; -import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; -// biome-ignore-start lint/suspicious/noImportCycles: This fixed producer shares the existing nominal authority/effect writer; it is invoked only after module initialization. -import { - assertControlExecutionForward, - controlDigest, - executionAuthorityState, - readPrivateRecord, - type VerifiedExecutionAuthority, - withPrivateEffectWriter, - writePrivateRecord, -} from "./control-authority"; -// biome-ignore-end lint/suspicious/noImportCycles: Fixed live producer dependency described above. -import { - controlOriginSchema, - sameControlOrigin, - signControl, - verifyControl, -} from "./control-records"; -// biome-ignore lint/suspicious/noImportCycles: The finite producer rechecks the same retained candidate at invocation, after schemas initialize. -import { verifyDataTransitCandidateForExecution } from "./data-execution-candidate"; -import { AnchoredDirectory } from "./filesystem"; -import { familyRecordWrite, verifyFamilyOwnedRecord } from "./retention-family"; - -const digest = z.string().regex(/^[a-f0-9]{64}$/); -const schema = z - .strictObject({ - version: z.literal(2), - installationId: z.uuid(), - origin: controlOriginSchema, - execution: executionRefSchema, - admissionDigest: digest, - candidateId: z.string().max(64), - candidateDigest: digest, - reservedAt: z.number().int().nonnegative().safe(), - phase: z.enum(["validation_pending", "validation_failed", "reload_started"]), - completedAt: z.number().int().nonnegative().safe().nullable(), - validationExitCode: z.number().int().min(1).max(255).nullable(), - completionDigest: digest.nullable(), - padding: z.string().max(4096), - }) - .superRefine((p, ctx) => { - if ( - (p.phase === "validation_failed") !== - (p.validationExitCode !== null && p.completionDigest !== null) || - (p.phase === "validation_pending") !== (p.completedAt === null) - ) - ctx.addIssue({ code: "custom", message: "Invalid fixed validation phase" }); - }); -type Proof = z.infer; -const domain = "openmapx-feed-proxy-validation-v2"; -const name = (authority: VerifiedExecutionAuthority) => - `proxy-validation-${authority.execution.operationKey}.json`; -function state(authority: VerifiedExecutionAuthority) { - const s = executionAuthorityState(authority); - if ( - s.installation.installation.role !== "ops-agent" || - s.kind !== "feedProxy.validateAndReload" || - !s.origin - ) - throw Error("Original fixed proxy authority required"); - return s; -} -function read(authority: VerifiedExecutionAuthority) { - const s = state(authority), - raw = readPrivateRecord(s.installation, "observations", name(authority), 4096); - const p = verifyControl(s.installation.installation.publicKey, domain, raw, schema); - if ( - p.installationId !== s.installation.installation.installationId || - !sameControlOrigin(p.origin, s.origin) || - p.admissionDigest !== s.receiptDigest || - controlDigest(p.execution) !== controlDigest(s.execution) || - opsOperationFingerprint({ kind: "feedProxy.validateAndReload", candidateId: p.candidateId }) !== - s.execution.operationFingerprint - ) - throw Error("Original validation proof identity changed"); - verifyFamilyOwnedRecord( - s.installation, - p.origin, - { scope: "private", namespace: "observations", container: "", name: name(authority) }, - canonicalJson(raw), - ); - return p; -} -function write(authority: VerifiedExecutionAuthority, value: Proof, exclusive = false) { - const s = state(authority), - p = { ...value, padding: "" }; - // Account the entire cell up front; transitions allocate no additional space. - const signed = signControl(s.installation.privateKey, domain, p); - const room = 4096 - Buffer.byteLength(canonicalJson(signed)); - if (room < 0) throw Error("Validation proof capacity exceeded"); - p.padding = " ".repeat(room); - const parsed = schema.parse(p); - writePrivateRecord( - s.installation, - "observations", - name(authority), - signControl(s.installation.privateKey, domain, parsed), - exclusive, - s.origin, - ); -} -export function feedProxyValidationCompletionDigest( - execution: ExecutionRef, - candidateDigest: string, - nonce: string, - validationExitCode: number, -): string { - return controlDigest({ - domain: "openmapx-feed-proxy-validation-completion-v1", - execution, - candidateDigest, - nonce, - validationExitCode, - }); -} -export function reserveFeedProxyValidation( - authority: VerifiedExecutionAuthority, - candidateId: string, - candidateDigest: string, -): void { - const s = state(authority); - assertControlExecutionForward(authority); - verifyDataTransitCandidateForExecution( - authority, - { kind: "feedProxy.validateAndReload", candidateId }, - candidateDigest, - ); - withPrivateEffectWriter(s, () => { - let raw: unknown; - try { - raw = readPrivateRecord(s.installation, "observations", name(authority), 4096); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - if (raw !== null && raw !== undefined) { - const prior = read(authority); - if (prior.candidateId !== candidateId || prior.candidateDigest !== candidateDigest) - throw Error("Validation candidate changed"); - const dir = witnessDirectory(s.installation); - try { - dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096); - } finally { - dir.close(); - } - return; - } - write( - authority, - schema.parse({ - version: 2, - installationId: s.installation.installation.installationId, - origin: s.origin, - execution: s.execution, - admissionDigest: s.receiptDigest, - candidateId, - candidateDigest, - reservedAt: Date.now(), - phase: "validation_pending", - completedAt: null, - validationExitCode: null, - completionDigest: null, - padding: "", - }), - true, - ); - reserveWitness(authority); - }); -} -export function commitFeedProxyValidationFailure( - authority: VerifiedExecutionAuthority, - input: { candidateDigest: string; validationExitCode: number; completionDigest: string }, -): void { - const s = state(authority); - withPrivateEffectWriter(s, () => { - const p = read(authority); - if (p.candidateDigest !== input.candidateDigest) throw Error("Foreign validation candidate"); - if (p.phase === "validation_failed") { - if ( - p.validationExitCode !== input.validationExitCode || - p.completionDigest !== input.completionDigest - ) - throw Error("Validation commitment changed"); - return; - } - if (p.phase !== "validation_pending") throw Error("Reload already attempted"); - // A new producer commitment verifies the actual invocation's bytes. An - // identical existing commitment above observes its frozen original proof, - // which remains valid after a later legitimate restore replaces files. - verifyDataTransitCandidateForExecution( - authority, - { kind: "feedProxy.validateAndReload", candidateId: p.candidateId }, - p.candidateDigest, - ); - write( - authority, - schema.parse({ ...p, ...input, phase: "validation_failed", completedAt: Date.now() }), - ); - }); -} -export function recordFeedProxyReloadAttempted(authority: VerifiedExecutionAuthority): void { - const s = state(authority); - withPrivateEffectWriter(s, () => { - const p = read(authority); - if (p.phase !== "validation_pending") throw Error("Validation phase already consumed"); - verifyDataTransitCandidateForExecution( - authority, - { kind: "feedProxy.validateAndReload", candidateId: p.candidateId }, - p.candidateDigest, - ); - write(authority, { ...p, phase: "reload_started", completedAt: Date.now() }); - }); -} -/** Frozen original proof stays valid after a legitimate restore replaces files. */ -export function inspectFeedProxyValidationFailure( - authority: VerifiedExecutionAuthority, - expectedDigest?: string, -): string | null { - const p = read(authority); - if (p.phase !== "validation_failed") return null; - const proofDigest = controlDigest(p); - if (expectedDigest !== undefined && expectedDigest !== proofDigest) - throw Error("Foreign validation commitment"); - return proofDigest; -} - -const terminalSchema = z.strictObject({ - version: z.literal(2), - installationId: z.uuid(), - origin: controlOriginSchema, - execution: executionRefSchema, - admissionDigest: digest, - phase: z.enum(["pending", "canonical"]), - status: z.unknown().nullable(), - terminalAt: z.number().int().positive().safe().nullable(), - terminalDigest: digest.nullable(), - padding: z.string().max(4096), -}); -function witnessDirectory(local: ReturnType["installation"]) { - const data = new AnchoredDirectory(local.dataRoot), - control = data.directory(".admin-target-control"), - executions = control.directory("executions"), - role = executions.directory("ops-agent"); - return { - role, - close() { - role.close(); - executions.close(); - control.close(); - data.close(); - }, - }; -} -function writeWitness( - authority: VerifiedExecutionAuthority, - payload: z.infer, - exclusive: boolean, -) { - const s = state(authority), - dir = witnessDirectory(s.installation), - name = `${s.execution.operationKey}.proxy-terminal.json`; - try { - const value = { ...payload, padding: "" }, - domain = "openmapx-proxy-canonical-terminal-v2"; - const room = - 4096 - - Buffer.byteLength(canonicalJson(signControl(s.installation.privateKey, domain, value))); - if (room < 0) throw Error("Proxy terminal cell capacity exceeded"); - value.padding = " ".repeat(room); - const bytes = canonicalJson(signControl(s.installation.privateKey, domain, value)); - familyRecordWrite( - s.installation, - payload.origin, - { scope: "shared", namespace: "execution", container: "ops-agent", name }, - bytes, - (before) => dir.role.write(name, bytes, exclusive, before), - exclusive, - ); - } finally { - dir.close(); - } -} -function reserveWitness(authority: VerifiedExecutionAuthority) { - const s = state(authority); - writeWitness( - authority, - terminalSchema.parse({ - version: 2, - installationId: s.installation.installation.installationId, - origin: s.origin, - execution: s.execution, - admissionDigest: s.receiptDigest, - phase: "pending", - status: null, - terminalAt: null, - terminalDigest: null, - padding: "", - }), - true, - ); -} -/** Only an already sealed original complete canonical tuple can publish this - * finite shared proxy witness. It creates no new terminal time or status. */ -export function publishFeedProxyCanonicalTerminal( - authority: VerifiedExecutionAuthority, - input: OpsJobStatus, - terminalAt: number, -) { - const s = state(authority), - status = parseOpsJobStatus(input), - terminalDigest = controlTerminalReceiptDigest(status); - const original = verifyControl( - s.installation.installation.publicKey, - "openmapx-control-terminal-v1", - readPrivateRecord( - s.installation, - "observations", - `terminal-${s.execution.operationKey}.json`, - 8192, - ), - z - .object({ - phase: z.literal("canonical"), - execution: executionRefSchema, - admissionDigest: digest, - terminalAt: z.number(), - terminalDigest: digest, - }) - .passthrough(), - ); - if ( - controlDigest(original.execution) !== controlDigest(s.execution) || - original.admissionDigest !== s.receiptDigest || - original.terminalAt !== terminalAt || - original.terminalDigest !== terminalDigest - ) - throw Error("Original canonical proxy terminal required"); - const dir = witnessDirectory(s.installation); - try { - const prior = verifyControl( - s.installation.installation.publicKey, - "openmapx-proxy-canonical-terminal-v2", - JSON.parse( - dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), - ), - terminalSchema, - ); - if ( - prior.installationId !== s.installation.installation.installationId || - prior.admissionDigest !== s.receiptDigest || - !sameControlOrigin(prior.origin, s.origin) || - controlDigest(prior.execution) !== controlDigest(s.execution) - ) - throw Error("Original reserved proxy terminal changed"); - if (prior.phase === "canonical") { - if (prior.terminalDigest !== terminalDigest || prior.terminalAt !== terminalAt) - throw Error("Proxy canonical terminal changed"); - return; - } - // Diagnostics are observational and deliberately excluded from the exact - // canonical digest. Keep the fixed shared tuple bounded. - writeWitness( - authority, - { - ...prior, - phase: "canonical", - status: { ...status, diagnostics: { tail: [], truncated: false } }, - terminalDigest, - terminalAt, - }, - false, - ); - } finally { - dir.close(); - } -} -export function verifyFeedProxyCanonicalTerminal( - parent: VerifiedExecutionAuthority, - execution: ExecutionRef, -): OpsJobStatus { - const s = executionAuthorityState(parent), - dir = witnessDirectory(s.installation); - try { - const proof = verifyControl( - s.installation.publicKeys["ops-agent"], - "openmapx-proxy-canonical-terminal-v2", - JSON.parse( - dir.role.read(`${execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), - ), - terminalSchema, - ); - if ( - proof.phase !== "canonical" || - proof.installationId !== s.installation.installation.installationId || - !sameControlOrigin(proof.origin, s.origin) || - controlDigest(proof.execution) !== controlDigest(execution) || - !proof.terminalAt - ) - throw Error("Original canonical proxy child missing"); - const status = parseOpsJobStatus(proof.status); - if ( - status.kind !== "feedProxy.validateAndReload" || - status.operationId !== execution.executionId || - status.operationKey !== execution.operationKey || - status.fingerprint !== execution.fingerprint || - status.deadlineAt !== execution.deadlineAt || - controlTerminalReceiptDigest(status) !== proof.terminalDigest - ) - throw Error("Canonical proxy status changed"); - return status; - } finally { - dir.close(); - } -} +/** Stable fixed proxy proof surface; the original candidate owner supplies all implementations. */ +export { + commitFeedProxyValidationFailure, + feedProxyValidationCompletionDigest, + inspectFeedProxyValidationFailure, + publishFeedProxyCanonicalTerminal, + recordFeedProxyReloadAttempted, + reserveFeedProxyValidation, + verifyFeedProxyCanonicalTerminal, +} from "./data-execution-candidate"; diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts new file mode 100644 index 000000000..a8dfeb8c0 --- /dev/null +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -0,0 +1,143 @@ +import { expect, it } from "vitest"; +import { + createControlRootOperationKey, + executionFingerprint, + opsOperationFingerprint, +} from "../ops"; +import { controlChildOperationKey } from "./control-authority"; +import { + assertFixedConfigurationMaintenancePlan, + buildFixedConfigurationMaintenancePlan, +} from "./fixed-configuration-maintenance"; + +// Pure plan-data qualification only. These inputs confer no authority and are +// not the real producer/decision/DM positive required by the task. +function input(): Parameters[0] { + const deadlineAt = Date.now() + 60000, + root = createControlRootOperationKey({ notAfter: deadlineAt, nonce: Buffer.alloc(16, 9) }); + const parent = { + issuer: "api" as const, + apiJobId: "11111111-1111-4111-8111-111111111111", + apiOwnerVersion: 1, + operationKey: "opk1_originalApiParent0001", + fingerprint: "a".repeat(64), + deadlineAt, + attemptId: "22222222-2222-4222-8222-222222222222", + assertionDigest: "b".repeat(64), + }; + const apply = { + kind: "vault.apply" as const, + serviceId: "data-manager", + maintenanceId: "33333333-3333-4333-8333-333333333333", + candidateId: "44444444-4444-4444-8444-444444444444", + revisionId: `cfg1_${"a".repeat(43)}`, + expectedCurrentRevisionId: `cfg1_${"b".repeat(43)}`, + expectedSourceRevision: "55555555-5555-4555-8555-555555555555", + coordinatorMode: "ensure-running" as const, + }; + const operation = { + kind: "configurationMaintenance.prepare" as const, + maintenanceId: apply.maintenanceId, + candidateId: apply.candidateId, + applyOperationKey: controlChildOperationKey(root, "configuration-commit", 0), + applyOperationFingerprint: opsOperationFingerprint(apply), + revisionId: apply.revisionId, + expectedCurrentRevisionId: apply.expectedCurrentRevisionId, + expectedSourceRevision: apply.expectedSourceRevision, + coordinatorMode: apply.coordinatorMode, + followUp: "service-apply" as const, + }; + const operationFingerprint = opsOperationFingerprint(operation); + const prepare = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: "job1_originalPrepare0001", + operationKey: root, + deadlineAt, + operationFingerprint, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + parent, + }), + }; + return { + installationId: "66666666-6666-4666-8666-666666666666", + prepare, + operation, + apiParent: parent, + apply, + candidateRecordDigest: "c".repeat(64), + sourceAuthorityRevision: "registry-v1", + previewDigest: "d".repeat(64), + serviceIds: ["data-manager", "motis"], + recreateServiceIds: ["data-manager"], + hardlinkPlan: { + prune: true, + plan: [ + { + source: "data/source-osm", + target: "data/motis/osm", + consumerService: "motis", + dataType: "osm-pbf", + }, + ], + }, + }; +} +it("freezes generation, real middle-owner intent and recreation with one immutable fixed-reference settlement", () => { + const source = input(), + plan = buildFixedConfigurationMaintenancePlan(source); + expect(plan.steps.map((value) => [value.ordinal, value.phase, value.prerequisites])).toEqual([ + [0, "configuration-commit", []], + [1, "maintenance-hardlinks", [0]], + [2, "maintenance-recreate", [1]], + ]); + expect(plan.steps[0]!.operation).toEqual(source.apply); + expect(plan.steps[0]!.operationKey).toBe(source.operation.applyOperationKey); + expect(plan.reservations.map((value) => value.ordinal)).toEqual([0, 1, 2, 3, 4]); + expect(plan.reservations.map((value) => value.kind)).not.toContain("service.start"); + expect(plan.nestedHardlinks).toMatchObject({ + kind: "generation-hardlinks", + phase: "hardlinks", + ordinal: 0, + recipient: "data-manager", + }); + expect(plan.dmPrepare.planDigest).toBe(plan.planDigest); + expect(plan.dmPrepare.hardlinkPlanDigest).not.toBe(plan.planDigest); + expect(Object.keys(plan.dmSettle).sort()).toEqual([ + "candidateId", + "maintenanceId", + "planDigest", + "settlementId", + "version", + ]); + const original = JSON.stringify(plan.dmSettle); + expect(Reflect.set(plan.dmSettle, "outcome", "applied")).toBe(false); + expect(JSON.stringify(plan.dmSettle)).toBe(original); + expect(() => assertFixedConfigurationMaintenancePlan(plan, source.installationId)).not.toThrow(); + const changed = structuredClone(plan); + changed.steps[1]!.prerequisites = []; + expect(() => assertFixedConfigurationMaintenancePlan(changed, source.installationId)).toThrow(); +}); +it.each(["body", "source", "parent", "foreign-target", "protected-target", "capacity"])( + "refuses %s before admitting any plan authority", + (reason) => { + const source = input(); + if (reason === "body") source.apply.revisionId = `cfg1_${"e".repeat(43)}`; + if (reason === "source") + source.operation.expectedSourceRevision = "77777777-7777-4777-8777-777777777777"; + if (reason === "parent") source.prepare.deadlineAt++; + if (reason === "foreign-target") source.recreateServiceIds = ["foreign"]; + if (reason === "protected-target") { + source.serviceIds = ["app-api", ...source.serviceIds]; + source.recreateServiceIds = ["app-api"]; + } + if (reason === "capacity") { + source.serviceIds = Array.from({ length: 31 }, (_, i) => `svc-${String(i).padStart(2, "0")}`); + source.recreateServiceIds = [...source.serviceIds]; + } + expect(() => buildFixedConfigurationMaintenancePlan(source)).toThrow(); + }, +); diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts new file mode 100644 index 000000000..3069b8b6c --- /dev/null +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts @@ -0,0 +1,380 @@ +/** Pure closed maintenance graph construction. Plans are data; only the actual + * production admission/store may turn one into original owner authority. */ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { + type ConfigurationMaintenancePrepare, + type ConfigurationMaintenanceStep, + configurationMaintenancePrepareSchema, + configurationMaintenanceStepSchema, + protectedConfigurationServiceIdSchema, +} from "../ops/configuration-maintenance"; +import { type OpsOperation, opsOperationSchema } from "../ops/contract"; +import { + type ExecutionRef, + executionFingerprint, + executionRefSchema, + type RetainedParentExecution, + retainedParentExecutionSchema, +} from "../ops/execution-identity"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { servicePullResultSchema } from "../ops/service-operation-records"; +import { + dataConfigurationFencePrepareSchema, + dataConfigurationFenceSettleSchema, +} from "../services/data-manager/contracts/configuration-maintenance"; +import { generationHardlinksPlanSchema } from "../services/data-manager/contracts/executions"; +import { deriveConfigurationFenceSettlementId } from "./configuration-fence-settlement-records"; +import { + type ControlChildReservation, + controlChildOperationKey, + controlChildReservationSchema, +} from "./control-child-records"; +import { controlDigest } from "./control-records"; +import { generationHardlinksReservation } from "./generation-hardlinks-records"; + +const ids = z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine((values) => values.every((value, i) => i === 0 || values[i - 1]! < value)); +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const coordinatorImageSchema = z.strictObject({ + action: z.enum(["apply", "update"]), + noDeps: z.boolean(), + imageReference: servicePullResultSchema.shape.imageReference, + imageId: servicePullResultSchema.shape.imageId, + releaseSelectionDigest: servicePullResultSchema.shape.releaseSelectionDigest, + pullSubmissionDigest: digest.nullable(), +}); +export const fixedConfigurationMaintenancePlanSchema = z.strictObject({ + version: z.literal(1), + prepare: executionRefSchema, + operation: configurationMaintenancePrepareSchema, + apiParent: retainedParentExecutionSchema.refine((value) => value.issuer === "api"), + candidateRecordDigest: digest, + sourceAuthorityRevision: z.string().min(1).max(128), + previewDigest: digest, + coordinatorImage: coordinatorImageSchema.optional(), + serviceIds: ids, + planDigest: digest, + hardlinkPlan: generationHardlinksPlanSchema, + steps: z.array(configurationMaintenanceStepSchema).min(1).max(32), + reservations: z + .array(z.lazy(() => controlChildReservationSchema)) + .min(3) + .max(34), + nestedHardlinks: z.lazy(() => controlChildReservationSchema).nullable(), + dmPrepare: dataConfigurationFencePrepareSchema, + dmSettle: dataConfigurationFenceSettleSchema, +}); +export type FixedConfigurationMaintenancePlan = z.infer< + typeof fixedConfigurationMaintenancePlanSchema +>; +type GenerationOperation = Extract< + OpsOperation, + { + kind: + | "stack.render" + | "serviceSelection.apply" + | "serviceConfig.apply" + | "integrationConfig.apply" + | "vault.apply"; + } +>; +function address( + root: string, + phase: string, + ordinal: number, + executor: "ops-agent" | "data-manager", +) { + const bytes = createHash("sha256") + .update(`openmapx-fixed-configuration-maintenance-v1\0${root}\0${phase}\0${ordinal}`) + .digest(); + const executionId = + executor === "ops-agent" + ? `job1_${bytes.toString("base64url").slice(0, 32)}` + : (() => { + const value = Buffer.from(bytes.subarray(0, 16)); + value[6] = (value[6]! & 15) | 64; + value[8] = (value[8]! & 63) | 128; + const h = value.toString("hex"); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`; + })(); + return { + executor, + principal: "api" as const, + executionId, + operationKey: controlChildOperationKey( + root, + phase as ControlChildReservation["phase"], + ordinal, + ), + }; +} +function freeze(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); + } + return value; +} +export function buildFixedConfigurationMaintenancePlan(input: { + installationId: string; + prepare: ExecutionRef; + operation: ConfigurationMaintenancePrepare; + apiParent: RetainedParentExecution; + apply: GenerationOperation; + candidateRecordDigest: string; + sourceAuthorityRevision: string; + previewDigest: string; + hardlinkPlan: z.infer; + serviceIds: readonly string[]; + recreateServiceIds: readonly string[]; + coordinatorImage?: z.infer; +}): FixedConfigurationMaintenancePlan { + const prepare = executionRefSchema.parse(input.prepare), + operation = configurationMaintenancePrepareSchema.parse(input.operation), + apply = opsOperationSchema.parse(input.apply) as GenerationOperation, + serviceIds = ids.parse(input.serviceIds), + hardlinkPlan = generationHardlinksPlanSchema.parse(input.hardlinkPlan), + parent = retainedParentExecutionSchema.parse(input.apiParent); + if ( + parent.issuer !== "api" || + prepare.executor !== "ops-agent" || + prepare.principal !== "api" || + prepare.operationFingerprint !== opsOperationFingerprint(operation) || + operation.followUp === "platform-apply" || + ![ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ].includes(apply.kind) || + !("revisionId" in apply) || + !("expectedCurrentRevisionId" in apply) || + !("expectedSourceRevision" in apply) || + !("coordinatorMode" in apply) || + apply.maintenanceId !== operation.maintenanceId || + apply.candidateId !== operation.candidateId || + apply.revisionId !== operation.revisionId || + apply.expectedCurrentRevisionId !== operation.expectedCurrentRevisionId || + apply.expectedSourceRevision !== operation.expectedSourceRevision || + apply.coordinatorMode !== operation.coordinatorMode || + opsOperationFingerprint(apply) !== operation.applyOperationFingerprint || + operation.applyOperationKey !== + controlChildOperationKey(prepare.operationKey, "configuration-commit", 0) || + prepare.deadlineAt > parent.deadlineAt || + prepare.fingerprint !== + executionFingerprint({ + kind: operation.kind, + operationFingerprint: prepare.operationFingerprint, + deadlineAt: prepare.deadlineAt, + parent, + }) + ) + throw Error("Fixed maintenance candidate binding conflict"); + const recreates = [...new Set(input.recreateServiceIds)].sort(); + if ( + input.coordinatorImage && + (canonicalJson(recreates) !== '["data-manager"]' || + (input.coordinatorImage.action === "apply" && + (input.coordinatorImage.noDeps || input.coordinatorImage.pullSubmissionDigest !== null)) || + (input.coordinatorImage.action === "update" && + input.coordinatorImage.pullSubmissionDigest === null)) + ) + throw Error("Fixed coordinator image expectation conflict"); + if ( + recreates.length !== input.recreateServiceIds.length || + recreates.some( + (id) => + !serviceIds.includes(id) || protectedConfigurationServiceIdSchema.safeParse(id).success, + ) || + (operation.followUp === "render-only" && recreates.length) || + (operation.followUp === "service-apply" && !recreates.length) + ) + throw Error("Fixed maintenance continuation conflict"); + const intents: { + phase: ConfigurationMaintenanceStep["phase"]; + operation: OpsOperation; + prerequisites: number[]; + }[] = [{ phase: "configuration-commit", operation: apply, prerequisites: [] }]; + if (hardlinkPlan.plan.length) + intents.push({ + phase: "maintenance-hardlinks", + operation: { + kind: "configuration.hardlinks.apply", + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + revisionId: operation.revisionId, + coordinatorMode: operation.coordinatorMode, + }, + prerequisites: [0], + }); + for (const serviceId of recreates) + intents.push({ + phase: "maintenance-recreate", + operation: { + kind: input.coordinatorImage?.noDeps ? "service.recreateIsolated" : "service.recreate", + serviceId, + expectedConfigurationRevisionId: operation.revisionId, + maintenanceId: operation.maintenanceId, + }, + prerequisites: [intents.length - 1], + }); + if (intents.length > 32) throw Error("Fixed maintenance effect capacity exceeded"); + const { kind: _kind, ...binding } = operation; + const planDigest = createHash("sha256") + .update("openmapx-configuration-maintenance-plan-v1\0") + .update( + canonicalJson({ + version: 1, + binding, + serviceIds, + ...(input.coordinatorImage ? { coordinatorImage: input.coordinatorImage } : {}), + steps: intents.map((step, ordinal) => ({ ...step, ordinal, serviceIds })), + }), + ) + .digest("hex"); + const reservations: ControlChildReservation[] = []; + const steps = intents.map((intent, ordinal) => { + const body = + intent.phase === "configuration-commit" + ? intent.operation + : { ...intent.operation, planDigest }; + const execution = address(prepare.operationKey, intent.phase, ordinal, "ops-agent"), + operationFingerprint = opsOperationFingerprint(body as OpsOperation); + reservations.push({ + version: 1, + phase: intent.phase, + ordinal, + kind: body.kind, + recipient: "ops-agent", + serviceIds, + execution, + operationFingerprint, + }); + return configurationMaintenanceStepSchema.parse({ + phase: intent.phase, + ordinal, + operation: body, + operationKey: execution.operationKey, + operationFingerprint, + serviceIds, + prerequisites: intent.prerequisites, + }); + }); + const dmPrepare = dataConfigurationFencePrepareSchema.parse({ + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + ownerPrepare: prepare, + planDigest, + serviceIds, + revisionId: operation.revisionId, + hardlinkPlanDigest: hardlinkPlan.plan.length ? controlDigest(hardlinkPlan) : null, + coordinatorMode: operation.coordinatorMode, + }); + reservations.push({ + version: 1, + phase: "configuration-fence", + ordinal: steps.length, + kind: "configuration-maintenance.prepare", + recipient: "data-manager", + serviceIds, + execution: address(prepare.operationKey, "configuration-fence", steps.length, "data-manager"), + operationFingerprint: controlDigest({ + kind: "configuration-maintenance.prepare", + ...dmPrepare, + }), + }); + const dmSettle = dataConfigurationFenceSettleSchema.parse({ + version: 2, + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + planDigest, + settlementId: deriveConfigurationFenceSettlementId({ + installationId: input.installationId, + ownerPrepare: prepare, + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + planDigest, + }), + }); + reservations.push({ + version: 1, + phase: "configuration-fence-settle", + ordinal: steps.length + 1, + kind: "configuration-maintenance.settle", + recipient: "data-manager", + serviceIds, + execution: address( + prepare.operationKey, + "configuration-fence-settle", + steps.length + 1, + "data-manager", + ), + operationFingerprint: controlDigest({ kind: "configuration-maintenance.settle", ...dmSettle }), + }); + const middle = reservations.find((value) => value.phase === "maintenance-hardlinks"); + return freeze( + fixedConfigurationMaintenancePlanSchema.parse({ + version: 1, + prepare, + operation, + apiParent: parent, + candidateRecordDigest: input.candidateRecordDigest, + sourceAuthorityRevision: input.sourceAuthorityRevision, + previewDigest: input.previewDigest, + ...(input.coordinatorImage ? { coordinatorImage: input.coordinatorImage } : {}), + serviceIds, + planDigest, + hardlinkPlan, + steps, + reservations, + dmPrepare, + dmSettle, + nestedHardlinks: middle + ? generationHardlinksReservation({ + parent: middle.execution, + revisionId: operation.revisionId, + coordinatorMode: operation.coordinatorMode, + plan: hardlinkPlan, + serviceIds, + }) + : null, + }), + ); +} +/** Restore comparisons recompute every derived body/address from the original + * retained inputs. A schema-valid signed summary is not a frozen graph. */ +export function assertFixedConfigurationMaintenancePlan( + value: FixedConfigurationMaintenancePlan, + installationId: string, +): void { + const plan = fixedConfigurationMaintenancePlanSchema.parse(value), + first = plan.steps[0]; + if (!first || first.phase !== "configuration-commit") + throw Error("Missing fixed generation step"); + const recreateServiceIds = plan.steps.flatMap((step) => + step.operation.kind === "service.recreate" || step.operation.kind === "service.recreateIsolated" + ? [step.operation.serviceId] + : [], + ); + const rebuilt = buildFixedConfigurationMaintenancePlan({ + installationId, + prepare: plan.prepare, + operation: plan.operation, + apiParent: plan.apiParent, + apply: first.operation as GenerationOperation, + candidateRecordDigest: plan.candidateRecordDigest, + sourceAuthorityRevision: plan.sourceAuthorityRevision, + previewDigest: plan.previewDigest, + ...(plan.coordinatorImage ? { coordinatorImage: plan.coordinatorImage } : {}), + hardlinkPlan: plan.hardlinkPlan, + serviceIds: plan.serviceIds, + recreateServiceIds, + }); + if (canonicalJson(rebuilt) !== canonicalJson(plan)) + throw Error("Fixed maintenance graph changed"); +} diff --git a/packages/core/src/build-artifacts/fixed-service-lifecycle.ts b/packages/core/src/build-artifacts/fixed-service-lifecycle.ts new file mode 100644 index 000000000..0aa56ccf4 --- /dev/null +++ b/packages/core/src/build-artifacts/fixed-service-lifecycle.ts @@ -0,0 +1,172 @@ +/** One root-fixed CLI Update and its optional, root-fixed generation leaf. */ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { type OpsOperation, opsOperationSchema } from "../ops/contract"; +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; +import { generationHardlinksPlanSchema } from "../services/data-manager/contracts/executions"; +import { type ControlChildReservation, controlChildReservationSchema } from "./child-phase"; +import { + controlChildOperationKey, + executionAuthorityState, + readPrivateRecord, + type VerifiedExecutionAuthority, + writePrivateRecord, +} from "./control-authority"; +import { controlDigest, signControl, verifyControl } from "./control-records"; +import { readOwnedGenerationHardlinkPlan } from "./generation-hardlinks-owner"; + +const domain = "openmapx-fixed-service-lifecycle-v1"; +const schema = z.strictObject({ + version: z.literal(1), + parent: executionRefSchema, + operation: opsOperationSchema, + rootReservation: controlChildReservationSchema, + cliReservations: z.array(controlChildReservationSchema).max(1), + delegationId: z.uuid(), + finalRevisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), + plan: generationHardlinksPlanSchema, +}); +export type FixedServiceLifecyclePlan = z.infer; +function address(key: string, purpose: string) { + const b = createHash("sha256").update(`${domain}\0${key}\0${purpose}`).digest().subarray(0, 16); + b[6] = (b[6]! & 15) | 64; + b[8] = (b[8]! & 63) | 128; + const h = b.toString("hex"); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`; +} +export function serviceLifecycleFinalRevisionId(rootOperationKey: string): string { + if (!/^opk1_[A-Za-z0-9_-]{16,64}$/.test(rootOperationKey)) + throw Error("Lifecycle root identity required"); + return `cfg1_${createHash("sha256").update(`${domain}\0final-generation\0${rootOperationKey}`).digest("base64url")}`; +} +export function ownedServiceLifecycleFinalRevisionId( + authority: VerifiedExecutionAuthority, +): string { + const s = executionAuthorityState(authority); + if (!["service.start", "service.update"].includes(s.kind) || !s.origin) + throw Error("Original lifecycle authority required"); + return serviceLifecycleFinalRevisionId(s.origin.rootKey); +} +export function lifecycleHardlinkReservation(options: { + parent: Pick; + revisionId: string; + plan: z.infer; + serviceIds: readonly string[]; +}): ControlChildReservation | null { + if (!options.plan.plan.length) return null; + const input = { + kind: "generation-hardlinks", + revisionId: options.revisionId, + coordinatorMode: "ensure-running", + planDigest: controlDigest(options.plan), + }; + return controlChildReservationSchema.parse({ + version: 1, + phase: "hardlinks", + ordinal: 0, + kind: "generation-hardlinks", + recipient: "data-manager", + serviceIds: [...options.serviceIds].sort(), + operationFingerprint: controlDigest(input), + execution: { + executor: "data-manager", + principal: "host-cli", + executionId: address(options.parent.operationKey, "hardlinks"), + operationKey: controlChildOperationKey(options.parent.operationKey, "hardlinks", 0), + }, + }); +} +export function planFixedServiceUpdate(options: { + infraDir: string; + parent: ExecutionRef; + operation: Extract; + serviceIds: readonly string[]; +}): FixedServiceLifecyclePlan { + const operation = opsOperationSchema.parse(options.operation), + finalRevisionId = serviceLifecycleFinalRevisionId(options.parent.operationKey); + const plan = readOwnedGenerationHardlinkPlan( + options.infraDir, + options.operation.expectedConfigurationRevisionId, + ); + const child = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: address(options.parent.operationKey, "cli"), + operationKey: controlChildOperationKey(options.parent.operationKey, "lifecycle", 0), + }; + const rootReservation: ControlChildReservation = { + version: 1, + phase: "lifecycle", + ordinal: 0, + kind: operation.kind, + recipient: "fixed-cli", + serviceIds: [...options.serviceIds].sort(), + operationFingerprint: opsOperationFingerprint(operation), + execution: child, + }; + const leaf = lifecycleHardlinkReservation({ + parent: child, + revisionId: finalRevisionId, + plan, + serviceIds: options.serviceIds, + }); + return schema.parse({ + version: 1, + parent: options.parent, + operation, + rootReservation, + cliReservations: leaf ? [leaf] : [], + delegationId: address(options.parent.operationKey, "challenge"), + finalRevisionId, + plan, + }); +} +export function retainFixedServiceLifecyclePlan( + authority: VerifiedExecutionAuthority, + value: FixedServiceLifecyclePlan, +): void { + const s = executionAuthorityState(authority), + plan = schema.parse(value); + if ( + s.kind !== "service.update" || + s.installation.fixedChildId || + s.installation.installation.role !== "ops-agent" || + controlDigest(s.execution) !== controlDigest(plan.parent) || + opsOperationFingerprint(plan.operation) !== s.execution.operationFingerprint || + !s.reservations.some((slot) => canonicalJson(slot) === canonicalJson(plan.rootReservation)) + ) + throw Error("Original lifecycle admission plan required"); + writePrivateRecord( + s.installation, + "observations", + `service-lifecycle-${s.execution.operationKey}.json`, + signControl(s.installation.privateKey, domain, plan), + true, + s.origin, + ); +} +export function readFixedServiceLifecyclePlan( + authority: VerifiedExecutionAuthority, +): FixedServiceLifecyclePlan { + const s = executionAuthorityState(authority), + plan = verifyControl( + s.installation.installation.publicKey, + domain, + readPrivateRecord( + s.installation, + "observations", + `service-lifecycle-${s.execution.operationKey}.json`, + 128 * 1024, + ), + schema, + ); + if ( + s.kind !== "service.update" || + controlDigest(s.execution) !== controlDigest(plan.parent) || + opsOperationFingerprint(plan.operation) !== s.execution.operationFingerprint || + !s.reservations.some((slot) => canonicalJson(slot) === canonicalJson(plan.rootReservation)) + ) + throw Error("Original lifecycle admission plan required"); + return plan; +} diff --git a/packages/core/src/build-artifacts/generation-hardlinks-owner.ts b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts new file mode 100644 index 000000000..cafbb2e57 --- /dev/null +++ b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts @@ -0,0 +1,260 @@ +import { generationHardlinksReservation } from "./generation-hardlinks-records"; + +export { generationHardlinksReservation } from "./generation-hardlinks-records"; + +/** Fixed generation-hardlinks middle owner. It can delegate only the one frozen DM leaf. */ + +import { readFileSync } from "node:fs"; +import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { type CoordinatorMode, hardlinkApplySummarySchema } from "../ops/service-results"; +import { currentConfigurationGeneration } from "../services/configuration-generation"; +import { + dataExecutionStatusSchema, + generationHardlinksPlanSchema, +} from "../services/data-manager/contracts/executions"; +import { + DataExecutionObservationPending, + observeDataExecution, + requestDataExecutionCancellation, +} from "../services/data-manager/executions"; +import { + lookupDataExecution, + startGenerationHardlinksExecution, +} from "../services/data-manager/generated/data-manager.generated"; +import { + type DataManagerTransport, + unwrapDataManagerResponse, +} from "../services/data-manager/transport"; +import { currentConfigurationFile, GENERATED_HARDLINK_PLAN_FILE } from "../services/deployment"; +import { + bindControlChildPhase, + consumeControlChildDispatch, + controlChildReservationSchema, + issueControlChildPhase, + lookupControlChildPhaseBinding, +} from "./child-phase"; +import { + assertControlExecutionForward, + controlRolePublicKey, + executionAuthorityState, + readPrivateRecord, + type VerifiedExecutionAuthority, + writePrivateRecord, +} from "./control-authority"; +import { controlDigest, signControl, verifyControl } from "./control-records"; +import { + assertArtifactTargetLeaseExecution, + inspectArtifactTargetLeaseChildSettlement, + type TargetLeaseAuthority, +} from "./target-lease"; + +const schema = z.strictObject({ + version: z.literal(1), + parent: executionRefSchema, + revisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), + coordinatorMode: z.enum(["ensure-running", "require-running"]), + plan: generationHardlinksPlanSchema, + reservation: z.lazy(() => controlChildReservationSchema).nullable(), +}); +export type GenerationHardlinksOwnerPlan = z.infer; +export function readOwnedGenerationHardlinkPlan(infraDir: string, revisionId: string) { + if (currentConfigurationGeneration(infraDir) !== revisionId) + throw Error("Hardlink generation conflict"); + const plan = generationHardlinksPlanSchema.parse({ + plan: JSON.parse( + readFileSync(currentConfigurationFile(infraDir, GENERATED_HARDLINK_PLAN_FILE), "utf8"), + ), + prune: true, + }); + if (plan.plan.length > 256 || currentConfigurationGeneration(infraDir) !== revisionId) + throw Error("Hardlink generation conflict"); + return plan; +} +export function planGenerationHardlinksOwner(options: { + infraDir: string; + parent: ExecutionRef; + revisionId: string; + coordinatorMode: CoordinatorMode; + serviceIds: readonly string[]; +}): GenerationHardlinksOwnerPlan { + const plan = readOwnedGenerationHardlinkPlan(options.infraDir, options.revisionId); + const reservation = generationHardlinksReservation({ ...options, plan }); + return schema.parse({ + version: 1, + parent: options.parent, + revisionId: options.revisionId, + coordinatorMode: options.coordinatorMode, + plan, + reservation, + }); +} +const domain = "openmapx-generation-hardlinks-owner-v1"; +const name = (authority: VerifiedExecutionAuthority) => + `generation-hardlinks-${authority.execution.operationKey}.json`; +function validateOwner( + s: ReturnType, + plan: GenerationHardlinksOwnerPlan, +) { + const input = { + kind: "generation-hardlinks", + revisionId: plan.revisionId, + coordinatorMode: plan.coordinatorMode, + planDigest: controlDigest(plan.plan), + }; + if ( + !["configuration.hardlinks.apply", "service.start", "service.update"].includes(s.kind) || + controlDigest(plan.parent) !== controlDigest(s.execution) || + (plan.reservation + ? plan.reservation.operationFingerprint !== controlDigest(input) || + !s.reservations.some((r) => canonicalJson(r) === canonicalJson(plan.reservation)) + : s.reservations.some((r) => r.phase === "hardlinks")) + ) + throw Error("Original hardlink owner required"); +} +export function retainGenerationHardlinksOwnerPlan( + authority: VerifiedExecutionAuthority, + value: GenerationHardlinksOwnerPlan, +): void { + const s = executionAuthorityState(authority), + plan = schema.parse(value); + validateOwner(s, plan); + writePrivateRecord( + s.installation, + "observations", + name(authority), + signControl(s.installation.privateKey, domain, plan), + true, + s.origin, + ); +} +export function readGenerationHardlinksOwnerPlan( + authority: VerifiedExecutionAuthority, +): GenerationHardlinksOwnerPlan { + const s = executionAuthorityState(authority), + plan = verifyControl( + s.installation.installation.publicKey, + domain, + readPrivateRecord(s.installation, "observations", name(authority), 128 * 1024), + schema, + ); + validateOwner(s, plan); + return plan; +} +export async function executeOwnedGenerationHardlinks(options: { + authority: VerifiedExecutionAuthority; + lease: TargetLeaseAuthority; + infraDir: string; + transport: DataManagerTransport; + signal?: AbortSignal; +}) { + const { authority, lease, transport, signal } = options, + s = executionAuthorityState(authority), + plan = readGenerationHardlinksOwnerPlan(authority); + assertControlExecutionForward(authority); + assertArtifactTargetLeaseExecution(lease, authority.execution); + if ( + canonicalJson(readOwnedGenerationHardlinkPlan(options.infraDir, plan.revisionId)) !== + canonicalJson(plan.plan) + ) + throw Error("Frozen hardlink plan changed"); + if (!plan.reservation) + return hardlinkApplySummarySchema.parse({ + applied: true, + entries: 0, + linked: 0, + skipped: 0, + pruned: 0, + via: "none", + }); + const prior = lookupControlChildPhaseBinding(authority, plan.reservation); + const bound = prior?.child ?? (await bindControlChildPhase(authority, plan.reservation)); + const input = { + kind: "generation-hardlinks" as const, + revisionId: plan.revisionId, + coordinatorMode: plan.coordinatorMode, + planDigest: controlDigest(plan.plan), + }; + const identity = { + id: bound.execution.executionId, + operationKey: bound.execution.operationKey, + fingerprint: bound.execution.fingerprint, + }; + const match = (value: unknown) => { + const status = dataExecutionStatusSchema.parse(value); + if ( + status.id !== identity.id || + status.operationKey !== identity.operationKey || + status.fingerprint !== identity.fingerprint || + status.deadlineAt !== bound.execution.deadlineAt || + status.kind !== input.kind + ) + throw Error("Hardlink leaf identity conflict"); + return status; + }; + let status: ReturnType | undefined; + // A bound slot is already an attempted identity. Recovery only observes it. + if (!prior) { + const issued = await issueControlChildPhase(authority, { + phase: "hardlinks", + ordinal: 0, + childPublicKey: controlRolePublicKey(s.installation.installation, "data-manager"), + lease, + }); + consumeControlChildDispatch(issued.dispatchTicket); + try { + status = match( + unwrapDataManagerResponse( + await startGenerationHardlinksExecution( + { + operationKey: identity.operationKey, + requestedDeadlineAt: bound.execution.deadlineAt, + input, + plan: plan.plan, + }, + { transport, signal, headers: { "x-openmapx-control-delegation": issued.assertion } }, + ), + ), + ); + } catch { + /* Durable grant publication precedes POST; an ambiguous result is lookup-only. */ + } + } + try { + if (!status) + status = match( + unwrapDataManagerResponse( + await lookupDataExecution(identity.operationKey, { transport, signal }), + ), + ); + if (!["ok", "partial", "error", "canceled", "interrupted"].includes(status.status)) + status = await observeDataExecution(identity, { + transport, + signal, + observerDeadlineAt: authority.execution.deadlineAt, + }); + } catch { + if (signal?.aborted) + await requestDataExecutionCancellation(identity, { transport }).catch(() => undefined); + throw new DataExecutionObservationPending( + identity, + signal?.aborted ? "interrupted" : "transport", + ); + } + let contained = false; + const until = Math.min(Date.now() + 5000, authority.execution.deadlineAt + 30000); + do { + try { + const settlement = inspectArtifactTargetLeaseChildSettlement(lease, bound.execution); + contained = settlement?.outcome === "committed" && settlement.publication === "committed"; + } catch { + /* Public terminal publication can precede the final borrowed-node receipt. */ + } + if (contained || Date.now() >= until) break; + await new Promise((resolve) => setTimeout(resolve, 50)); + } while (!contained && Date.now() < until); + if (!contained || status.status !== "ok" || status.result?.kind !== "generation-hardlinks") + throw new DataExecutionObservationPending(identity, "transport"); + return hardlinkApplySummarySchema.parse(status.result.hardlinks); +} diff --git a/packages/core/src/build-artifacts/generation-hardlinks-records.ts b/packages/core/src/build-artifacts/generation-hardlinks-records.ts new file mode 100644 index 000000000..b5c6c7a3a --- /dev/null +++ b/packages/core/src/build-artifacts/generation-hardlinks-records.ts @@ -0,0 +1,56 @@ +/** Pure fixed hardlink reservation data; no issuer/lease/runtime dependency. */ +import { createHash } from "node:crypto"; +import type { z } from "zod"; +import type { ExecutionRef } from "../ops/execution-ref"; +import type { CoordinatorMode } from "../ops/service-results"; +import { generationHardlinksPlanSchema } from "../services/data-manager/contracts/executions"; +import { + type ControlChildReservation, + controlChildOperationKey, + controlChildReservationSchema, +} from "./control-child-records"; +import { controlDigest } from "./control-records"; + +function uuid(key: string) { + const b = createHash("sha256") + .update(`openmapx-generation-hardlinks-leaf-v1\0${key}`) + .digest() + .subarray(0, 16); + b[6] = (b[6]! & 15) | 64; + b[8] = (b[8]! & 63) | 128; + const h = b.toString("hex"); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`; +} +/** Pure fixed graph construction; this never issues a grant or an execution. */ +export function generationHardlinksReservation(options: { + parent: Pick; + revisionId: string; + coordinatorMode: CoordinatorMode; + plan: z.infer; + serviceIds: readonly string[]; +}): ControlChildReservation | null { + const plan = generationHardlinksPlanSchema.parse(options.plan), + input = { + kind: "generation-hardlinks" as const, + revisionId: options.revisionId, + coordinatorMode: options.coordinatorMode, + planDigest: controlDigest(plan), + }; + return plan.plan.length + ? controlChildReservationSchema.parse({ + version: 1, + phase: "hardlinks", + ordinal: 0, + kind: "generation-hardlinks", + recipient: "data-manager", + serviceIds: [...options.serviceIds].sort(), + operationFingerprint: controlDigest(input), + execution: { + executor: "data-manager", + principal: options.parent.executor === "host-cli" ? "host-cli" : "api", + executionId: uuid(options.parent.operationKey), + operationKey: controlChildOperationKey(options.parent.operationKey, "hardlinks", 0), + }, + }) + : null; +} diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 26210f2ea..2017b5c14 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -12,6 +12,33 @@ export { lookupControlChildPhaseAttempt, lookupControlChildPhaseBinding, } from "./child-phase"; +export { + type OwnedConfigurationSettlementReservation, + ownedConfigurationSettlementReservationState, + prepareOwnedConfigurationSettlementReservation, + publishConfigurationSettlementReservationProjection, + publishedConfigurationSettlementCapacityState, + publishedConfigurationSettlementReservationState, + reserveReceivedConfigurationSettlementCapacity, + type VerifiedPublishedConfigurationSettlementCapacity, + type VerifiedPublishedConfigurationSettlementReservation, + verifyPublishedConfigurationSettlementCapacity, + verifyPublishedConfigurationSettlementReservation, +} from "./configuration-fence-settlement"; +export { + publishedDmNoGuardRefusalState, + publishedDmUnusedCapacityState, + type VerifiedDmUnusedCapacity, + type VerifiedPublishedDmNoGuardRefusal, + verifyPublishedDataConfigurationFenceRefusal, + verifyPublishedUnusedConfigurationCapacity, +} from "./configuration-no-guard"; +export { + configurationHostFields, + observeConfigurationSources, + verifyConfigurationReleaseSourceWitness, + verifyConfigurationSourceWitness, +} from "./configuration-source-witness"; export { type ArtifactDelegationAssertion, acknowledgeControlRetention, @@ -37,21 +64,31 @@ export { controlInstallationDataRoot, controlRegisteredServiceIds, controlRolePublicKey, + executionAuthorityState, fixedCliArguments, hasRetainedControlOrigin, + inspectConfigurationPreEffectRefusal, loadControlStartup, markFeedProxyReloadAttempted, + observeCoordinatorPullForMaintenance, + observedCoordinatorPullBaseline, openControlInstallation, openFixedChildInstallation, + ownedCoordinatorPullBaseline, + ownedStagedCoordinatorBaseline, prepareFixedControlChild, provisionControlInstallation, readFixedCliSpawnIntent, + readPrivateRecord, + readVerifiedStagedCoordinatorOwnership, reconcileControlRetention, registerControlRetentionJournal, repairControlJournalTerminal, requireControlJournalRetention, restoreAdmittedControlExecution, + restoreConfigurationPreEffectTerminal, retainAdmittedControlExecution, + retainConfigurationPreEffectRefusal, retainControlEffectReceipt, retainControlPreparedTerminal, retainControlTerminalCommitment, @@ -61,19 +98,43 @@ export { retainFixedCliSpawnIntent, type VerifiedChildSettlement, type VerifiedControlPeer, + type VerifiedCoordinatorPullBaseline, type VerifiedExecutionAuthority, type VerifiedRecoveryAuthority, + type VerifiedStagedCoordinatorOwnerBaseline, + type VerifiedStagedCoordinatorOwnership, + verifiedCoordinatorPullBaselineState, verifiedRetainedControlParent, + verifiedStagedCoordinatorOwnerBaselineState, + verifiedStagedCoordinatorOwnershipState, verifyControlEffectSettlement, verifyControlPeer, verifyControlRecoveryAuthority, + verifyCoordinatorPullAdmissionSource, verifyDataControlTerminalWitness, verifyFixedCliControlPeer, + verifyMaintenanceCoordinatorPrerequisite, verifyParentExecution, verifyRetainedLocalControlPeer, + writePrivateRecord, } from "./control-authority"; +export { controlDigest, signControl, verifyControl } from "./control-records"; export { CONTROL_RETENTION_LIMITS } from "./control-retention"; -export { createDataConfigurationFenceStore, type DataFenceSql } from "./data-configuration-fence"; +export type { ControlRetentionPreparationContext } from "./coordinator-pull-records"; +export { + type CoordinatorPullBaseline, + coordinatorPullBaselineSchema, +} from "./coordinator-pull-records"; +export { + createDataConfigurationFenceStore, + type DataFenceSql, + type VerifiedPreparedGenerationAdmission, +} from "./data-configuration-fence"; +export { + preparedDataConfigurationFenceState, + type VerifiedPreparedDataConfigurationFence, + verifyPublishedDataConfigurationFencePrepared, +} from "./data-configuration-fence-witness"; export { retainedDataDatabaseTimestamp } from "./data-database-time"; export { assertDataTransitCandidateForExecution, @@ -94,6 +155,12 @@ export { reserveFeedProxyValidation, verifyFeedProxyCanonicalTerminal, } from "./feed-proxy-validation"; +export { + assertFixedConfigurationMaintenancePlan, + buildFixedConfigurationMaintenancePlan, + type FixedConfigurationMaintenancePlan, + fixedConfigurationMaintenancePlanSchema, +} from "./fixed-configuration-maintenance"; export { cliDataReservation, fixedDataBodyFingerprint, @@ -106,10 +173,32 @@ export { retainFixedSingleDataPlan, verifyCliDataTerminal, } from "./fixed-data-execution"; +export { + type FixedServiceLifecyclePlan, + lifecycleHardlinkReservation, + ownedServiceLifecycleFinalRevisionId, + planFixedServiceUpdate, + readFixedServiceLifecyclePlan, + retainFixedServiceLifecyclePlan, + serviceLifecycleFinalRevisionId, +} from "./fixed-service-lifecycle"; +export { + executeOwnedGenerationHardlinks, + type GenerationHardlinksOwnerPlan, + planGenerationHardlinksOwner, + readGenerationHardlinksOwnerPlan, + retainGenerationHardlinksOwnerPlan, +} from "./generation-hardlinks-owner"; export * from "./manifest"; export { inspectControlAdmission, inspectDataExecutionAdmission, verifiedParentForArtifactDelegation, } from "./peer-admission"; +export { + controlRetentionPreparationState, + recordStagedCoordinatorRetentionCheck, + verifyFamilyOwnedRecord, +} from "./retention-family"; export * from "./target-lease"; +export { verifyConfigurationMaintenanceRootClosure as assertConfigurationMaintenanceRootClosure } from "./target-lease"; diff --git a/packages/core/src/build-artifacts/peer-process.test.ts b/packages/core/src/build-artifacts/peer-process.test.ts index 0263f19f1..85a91a49b 100644 --- a/packages/core/src/build-artifacts/peer-process.test.ts +++ b/packages/core/src/build-artifacts/peer-process.test.ts @@ -27,6 +27,7 @@ import { } from "./control-authority"; import { acquireArtifactTargetLease, + closeArtifactTargetLeaseHandle, delegateArtifactTargetLease, releaseArtifactTargetLease, settleArtifactTargetLeaseChild, @@ -120,7 +121,11 @@ describe("separate-process physical-mount delegation", () => { const operation = role === "data-manager" ? { kind: "transit-sync" } - : { kind: "service.start", serviceId: "motis" }; + : { + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }; const kind = operation.kind; const rootRef = ref(role, kind, operation); const childOperation = @@ -287,7 +292,11 @@ it.each([false, true])( "certifies a depth-two hardlink leaf across processes (cancel intermediate: %s)", async (cancelIntermediate) => { const c = await import("./control-authority"); - const operation = { kind: "service.start" as const, serviceId: "motis" }; + const operation = { + kind: "service.start" as const, + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }; const execution = ref("ops-agent", operation.kind, operation); const cli = ref("host-cli", operation.kind, operation, execution, "lifecycle"); const plan: ControlPlannedChild = { @@ -662,7 +671,7 @@ it("keeps paired-pin approval a one-use leaf under its original activation root" await releaseArtifactTargetLease(lease); c.closeControlInstallation(installation); }); -it("settles the logical configuration-fence leaf without releasing the maintenance owner", async () => { +it("rejects a logical configuration-fence delegation without the actual maintenance producer's owner", async () => { const c = await import("./control-authority"); const operation = { kind: "configurationMaintenance.prepare", @@ -705,38 +714,24 @@ it("settles the logical configuration-fence leaf without releasing the maintenan }); const key = JSON.parse(readFileSync(join(root, "secrets/control-verifiers.json"), "utf8")) .publicKeys["data-manager"]; - const assertion = await delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }); - const envelope = JSON.parse(Buffer.from(assertion, "base64url").toString("utf8")); - const forged = Buffer.from(JSON.stringify({ ...envelope, signature: "a".repeat(86) })).toString( - "base64url", - ); - const foreign = Buffer.from( - JSON.stringify({ ...envelope, installationId: randomUUID() }), - ).toString("base64url"); - const input = { - infraDir: root, - role: "data-manager", - assertion, - execution: childRef, - kind: body.kind, - operation: body, - admittedAt: childRef.deadlineAt - 60000, - }; - expect(await child({ ...input, assertion: forged })).toMatchObject({ ok: false }); - expect(await child({ ...input, assertion: foreign })).toMatchObject({ ok: false }); - expect(await child({ ...input, settle: true })).toMatchObject({ ok: true }); - await expect(releaseArtifactTargetLease(lease)).rejects.toThrow(/settlement/); - c.retainControlEffectReceipt(authority, { - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - await settleArtifactTargetLeaseChild( - lease, - c.verifyControlEffectSettlement(installation, execution), - ); - await releaseArtifactTargetLease(lease); - c.closeControlInstallation(installation); + try { + // A bare root/plan cannot impersonate the actual maintenance producer. + // Genuine short-leaf/SQL preparation, forged/foreign controls and held-root + // acceptance run through the owned API/agent/DM PostgreSQL fixture. + await expect( + delegateArtifactTargetLease(lease, { ...plan, childPublicKey: key }), + ).rejects.toThrow(); + await expect(releaseArtifactTargetLease(lease)).rejects.toThrow(); + expect( + readFileSync( + join(root, `data/.admin-target-control/leases/${lease.leaseId}/owner.json`), + "utf8", + ), + ).toContain(execution.operationKey); + } finally { + closeArtifactTargetLeaseHandle(lease); + c.closeControlInstallation(installation); + } }); it("serializes a paused observer and a newer writer across actual processes", async () => { diff --git a/packages/core/src/build-artifacts/retention-family.ts b/packages/core/src/build-artifacts/retention-family.ts index 011b53d78..fc868743c 100644 --- a/packages/core/src/build-artifacts/retention-family.ts +++ b/packages/core/src/build-artifacts/retention-family.ts @@ -1,14 +1,68 @@ /** Fixed, role-local receipt ownership and preallocated per-origin retirement cells. */ import { createHash, randomUUID } from "node:crypto"; -import { fsyncSync, mkdirSync } from "node:fs"; +import { fsyncSync, mkdirSync, statfsSync } from "node:fs"; import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; import type { ControlExecutionOrigin } from "../ops/control-root-key"; -import type { - ControlIssuer, - ExecutionRef, - RetainedParentExecution, +import { + type ControlIssuer, + type ExecutionRef, + executionFingerprint, + type RetainedParentExecution, } from "../ops/execution-identity"; -import { canonicalJson } from "../ops/fingerprint"; +import { executionRefSchema } from "../ops/execution-ref"; +import { + type ConfigurationSettlementFamilySlot, + type ConfigurationSettlementProjection, + type ConfigurationSettlementRecordDescriptor, + configurationSettlementRecordDescriptors, +} from "./configuration-fence-settlement-descriptors"; +import { + configurationCommittedSettlementSchema, + configurationDmCapacityCellSchema, + configurationDmNoGuardRefusalSchema, + configurationIssuedPrepareRefusedProofSchema, + configurationIssuerUnusedSourceDigest, + configurationNeverIssuedProofSchema, + configurationNoGuardClosureSchema, + configurationNoGuardIntentSchema, + configurationSettlementAckSchema, + configurationSettlementAdmissionSchema, + configurationSettlementCapacitySchema, + configurationSettlementCommitmentCellSchema, + configurationSettlementControlCellSchema, + configurationSettlementControlEnvelopeSchema, + configurationSettlementDispatchSchema, + configurationSettlementDomains, + configurationSettlementEffectSchema, + configurationSettlementPublishedTerminalSchema, + configurationSettlementReservationProjectionSchema, + configurationSettlementSenderAckCellSchema, + configurationUnusedCapacityConfirmationSchema, + configurationUnusedCapacityIntentSchema, + deriveConfigurationFenceSettlementId, + configurationSettlementDomains as domains, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import { + assertConfigurationMaintenanceDecisionSource, + configurationMaintenanceOwnerRecordSchema, + configurationMaintenanceReleasedOwner, + configurationMaintenanceSettleCompletion, +} from "./configuration-maintenance-owner-records"; +// biome-ignore-start lint/suspicious/noImportCycles: Indexed completion repair and retirement require actual nominal authority from the same record owner. Parsed roots or caller flags cannot replace it; all calls are deferred. +import { + type ControlInstallation, + controlChildOperationKey, + executionAuthorityState, + installationState, + readControlExecutionWitness, + readPrivateRecord, + restoreAdmittedControlExecution, + type VerifiedExecutionAuthority, + withPrivateEffectWriter, +} from "./control-authority"; +// biome-ignore-end lint/suspicious/noImportCycles: Exact deferred original-owner/index guard join. import { controlDigest, controlOriginSchema, @@ -17,12 +71,35 @@ import { verifyControl, } from "./control-records"; import { + CONTROL_RETENTION_LIMITS, deleteRetainedRecord, finishRetainedDeletion, CONTROL_RETENTION_LIMITS as LIMITS, + writeRetainedRecord, } from "./control-retention"; +import { + type ControlRetentionPreparationContext, + coordinatorPullBaselineSchema, + type StagedCoordinatorRetirementPacket, + stagedCoordinatorRetirementDomain, + stagedCoordinatorRetirementPacketSchema, + stagedCoordinatorRetirementPayloadSchema, +} from "./coordinator-pull-records"; +import { mirrorSchema } from "./data-configuration-fence-records"; +import { + dataConfigurationFencePreparedWitnessDomain, + dataConfigurationFencePreparedWitnessSchema, +} from "./data-configuration-fence-witness-records"; import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; import { advanceRetentionClock, readRetentionClock } from "./retention-clock"; +// biome-ignore-start lint/suspicious/noImportCycles: Index repair verifies actual physical release/full closure; physical closure verifies this same index. No constructor or verifier runs at module initialization. +import { + verifyConfigurationFenceSettlementReleasedOwner, + verifyConfigurationMaintenanceRootClosure, + verifyRetainedArtifactTargetRootRelease, +} from "./target-lease"; + +// biome-ignore-end lint/suspicious/noImportCycles: Exact deferred original-owner/index guard join. const ROLES = ["ops-agent", "data-manager", "host-cli"] as const; const PRIVATE = new Set([ @@ -49,10 +126,13 @@ const PRIVATE = new Set([ "hardlinks", ]); const recordMaximum = (record: { scope: string; namespace: string }) => - record.scope === "private" && record.namespace === "hardlinks" - ? 1024 * 1024 - : LIMITS.privateRecordBytes; + record.scope === "shared" && record.namespace === "service-lifecycle" + ? 65536 + : record.scope === "private" && record.namespace === "hardlinks" + ? 1024 * 1024 + : LIMITS.privateRecordBytes; const SHARED = new Set([ + "service-lifecycle", "execution", "lease-owner", "lease-grants", @@ -135,6 +215,10 @@ interface Progress { terminalAt: number; api?: ApiDependency; apiAck?: { terminalDigest: string; proofDigest: string }; + stagedCoordinatorRevalidation?: StagedCoordinatorRetirementPacket; + /** One fixed logical control. Its unbound address counts from PREPARE onward, + * and never masquerades as an ordinary physical execution/lease member. */ + configurationSettlementSlot?: ConfigurationSettlementFamilySlot; cursor: number; sharedCursor: number; manifestDigest?: string; @@ -165,6 +249,7 @@ interface Candidate { sharedRecordSetDigest: string; leaseDigests: Record; intentDigests: Record; + stagedCoordinatorDependency?: StagedCoordinatorRetirementPacket; } const apiCopySchema = z.strictObject({ version: z.literal(1), @@ -211,6 +296,7 @@ interface Manifest { executionReceiptDigests: Record; sharedRecordSetDigest: string; localRecordSetDigests: Record; + stagedCoordinatorDependencyDigest?: string; } interface Decision { version: 1; @@ -220,6 +306,7 @@ interface Decision { manifestDigest?: string; readyDigests?: Record; cleanDigests?: Record; + stagedCoordinatorRevalidationDigest?: string; sequence: number; } interface Ack { @@ -251,6 +338,276 @@ interface Family { sharedGone?: boolean; close(): void; } +type PreparationMode = "preparing" | "before-commit" | "committed-cleanup"; +const preparationContexts = new WeakMap< + ControlRetentionPreparationContext, + { + family: Family; + sequence: number; + mode: PreparationMode; + nonce: string; + manifestDigest: string | null; + createdAt: number; + packet?: StagedCoordinatorRetirementPacket; + consumed: boolean; + } +>(); +function preparationContext(value: ControlRetentionPreparationContext) { + const state = preparationContexts.get(value); + if (!state || readProgress(state.family).sequence !== state.sequence) + throw Error("Actual current collector context required"); + checkLocalIdentity(state.family); + return state; +} +export function controlRetentionPreparationState(value: ControlRetentionPreparationContext) { + const state = preparationContext(value), + f = state.family, + p = readProgress(f); + return { + mode: state.mode, + installationId: f.meta.installationId, + role: f.state.installation.role, + origin: structuredClone(f.meta.origin), + originDigest: f.originHash, + familyId: f.meta.familyId, + generation: f.meta.generation, + executions: Object.entries(p.executions) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([digest, fact]) => ({ + digest, + execution: structuredClone(fact.execution), + terminalDigest: fact.terminalDigest ?? null, + requiresJournal: fact.requiresJournal ?? false, + })), + progressSequence: state.sequence, + manifestDigest: state.manifestDigest, + checkNonce: state.nonce, + }; +} +function verifiedStagedRetirementPacket(f: Family, p: Progress, raw: unknown) { + const packet = stagedCoordinatorRetirementPacketSchema.parse(raw), + value = verifyControl( + f.state.publicKeys["ops-agent"], + stagedCoordinatorRetirementDomain, + packet, + stagedCoordinatorRetirementPayloadSchema, + ), + fact = p.executions[controlDigest(value.execution)]; + if ( + f.state.installation.role !== "ops-agent" || + value.installationId !== f.meta.installationId || + value.originDigest !== f.originHash || + value.familyId !== f.meta.familyId || + value.generation !== f.meta.generation || + !fact || + canonicalJson(fact.execution) !== canonicalJson(value.execution) || + !fact.known || + !fact.journalContained || + value.terminalDigest !== fact.terminalDigest || + value.terminalDigest !== fact.canonicalDigest || + value.sourceAdmissionDigest !== fact.receiptDigest || + value.rank.operationKey !== value.execution.operationKey + ) + throw Error("Original staged retirement family membership changed"); + return { packet, value, fact }; +} +/** The literal fixed producer supplies a signed observation, never a caller + * boolean. This recorder binds it to the collector's exact live sequence. */ +export function recordStagedCoordinatorRetentionCheck( + context: ControlRetentionPreparationContext, + raw: unknown, +): void { + const state = preparationContext(context), + f = state.family, + p = readProgress(f); + if (state.consumed || state.mode === "committed-cleanup") + throw Error("Original unused collector check required"); + const { packet, value, fact } = verifiedStagedRetirementPacket(f, p, raw); + if ( + value.contextMode !== state.mode || + value.checkNonce !== state.nonce || + value.manifestDigest !== state.manifestDigest || + value.checkedAt < state.createdAt || + value.checkedAt > Date.now() + ) + throw Error("Collector check nonce/phase changed"); + const rawAdmission = readPrivateRecord( + f.state, + "executions", + `${value.execution.operationKey}.json`, + 65536, + ), + admitted = verifyControl( + f.state.publicKeys["ops-agent"], + "openmapx-control-execution-v2", + rawAdmission, + z + .object({ + admittedAt: z.number().int().nonnegative().safe(), + coordinatorPullBaseline: coordinatorPullBaselineSchema, + }) + .passthrough(), + ), + baseline = admitted.coordinatorPullBaseline; + if ( + controlDigest(rawAdmission) !== fact.receiptDigest || + admitted.admittedAt !== value.rank.admittedAt || + value.oldContainerDigest !== + controlDigest({ + container: baseline.container, + configImageReference: baseline.configImageReference, + composeSourcesDigest: baseline.composeSourcesDigest, + configurationRevisionId: baseline.configurationRevisionId, + }) + ) + throw Error("Actual staged retirement source changed"); + state.consumed = true; + state.packet = packet; +} +function sourcePreparationContextAllows( + context: ControlRetentionPreparationContext | undefined, + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, + sequence: number, +) { + if (!context) return false; + const local = preparationContext(context); + return ( + local.family.state === state && + local.mode !== "committed-cleanup" && + local.sequence === sequence && + local.family.originHash === controlDigest(origin) && + !!readProgress(local.family).executions[controlDigest(execution)] + ); +} +function assertStagedCommittedCleanup(f: Family, p: Progress, m: Manifest, current: Decision) { + assertPrivateRetirementProof(f, m); + const committedRaw = json(f.slot, "commit.json", 8192); + const d = + committedRaw === null + ? current + : verified( + f.state, + f.meta.origin.issuer, + "openmapx-retention-decision-v1", + committedRaw, + ); + sameFamily(f, d); + if ( + d.phase !== "committed" || + current.manifestDigest !== d.manifestDigest || + !["committed", "final"].includes(current.phase) || + (current.phase === "committed" && controlDigest(current) !== controlDigest(d)) + ) + throw Error("Original signed committed decision required"); + if ( + !["committed", "final"].includes(d.phase) || + d.manifestDigest !== controlDigest(m) || + !Number.isSafeInteger(p.cursor) || + p.cursor < 0 || + p.cursor > p.records + ) + throw Error("Original committed cleanup cursor required"); + if (f.state.installation.role !== "ops-agent") return; + const proof = verified( + f.state, + "ops-agent", + "openmapx-retention-candidate-v1", + json(f.slot, "proof.json", 65536), + ); + if (!m.stagedCoordinatorDependencyDigest) { + if ( + proof.stagedCoordinatorDependency || + p.stagedCoordinatorRevalidation || + d.stagedCoordinatorRevalidationDigest + ) + throw Error("Unbound staged committed cleanup proof"); + return; // Existing authenticated historical cleanup, never a source reader. + } + const first = verifiedStagedRetirementPacket(f, p, proof.stagedCoordinatorDependency), + last = verifiedStagedRetirementPacket(f, p, p.stagedCoordinatorRevalidation); + if ( + controlDigest(first.packet) !== m.stagedCoordinatorDependencyDigest || + controlDigest(last.packet) !== d.stagedCoordinatorRevalidationDigest || + first.value.contextMode !== "preparing" || + first.value.manifestDigest !== null || + last.value.contextMode !== "before-commit" || + last.value.manifestDigest !== controlDigest(m) || + first.value.checkNonce === last.value.checkNonce || + canonicalJson([ + first.value.execution, + first.value.terminalDigest, + first.value.sourceAdmissionDigest, + first.value.oldContainerDigest, + first.value.rank, + ]) !== + canonicalJson([ + last.value.execution, + last.value.terminalDigest, + last.value.sourceAdmissionDigest, + last.value.oldContainerDigest, + last.value.rank, + ]) + ) + throw Error("Original staged committed revalidation changed"); +} +async function prepareJournalPhase(f: Family, p: Progress, mode: PreparationMode, m?: Manifest) { + if (mode === "committed-cleanup") { + if (!m) throw Error("Committed cleanup manifest required"); + const current = + p.phase === "released" && p.sharedRelease + ? verified( + f.state, + f.meta.origin.issuer, + "openmapx-retention-decision-v1", + json(f.slot, "final.json", 8192), + ) + : decision(f); + assertStagedCommittedCleanup(f, p, m, current); + } + const token = Object.freeze({}) as ControlRetentionPreparationContext; + const state = { + family: f, + sequence: p.sequence, + mode, + nonce: randomUUID(), + manifestDigest: m ? controlDigest(m) : null, + createdAt: Date.now(), + consumed: false, + packet: undefined as StagedCoordinatorRetirementPacket | undefined, + }; + preparationContexts.set(token, state); + try { + await f.state.retentionJournal?.prepare?.( + Object.values(p.executions) + .filter((fact) => fact.requiresJournal) + .map((fact) => ({ execution: fact.execution, digest: fact.terminalDigest })), + token, + ); + preparationContext(token); + return state.packet; + } finally { + preparationContexts.delete(token); + } +} + +// One scoped read-only source observation per coordinator step. Verify outside +// family writers; inside them require the identical local indexed record set. +const preparedClosures = new WeakMap< + Family, + ReturnType & { + localRecordSetDigest: string; + } +>(); +function preparedClosure(f: Family, p: Progress) { + const proof = preparedClosures.get(f); + return p.configurationSettlementSlot?.state === "bound" && + proof && + proof.localRecordSetDigest === recordSetDigest(records(f)) + ? proof + : null; +} const DIGEST = /^[a-f0-9]{64}$/; const safeName = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,159}$/; const slotName = /^slot-\d{1,4}$/; @@ -343,8 +700,17 @@ function readProgress(f: Family): Progress { p.sharedRecords > 2048 || !Number.isSafeInteger(p.metadataBytes) || p.metadataBytes > LIMITS.familyMetadataBytes || - Object.keys(p.executions).length > LIMITS.executionsPerFamily || - Object.keys(p.intents).length > LIMITS.intentsPerFamily || + Object.keys(p.executions).length + + (p.configurationSettlementSlot?.role === "data-manager" && + p.configurationSettlementSlot.executionOrdinal !== null + ? 1 + : 0) > + LIMITS.executionsPerFamily || + Object.keys(p.intents).length + + (p.configurationSettlementSlot && p.configurationSettlementSlot.intentOrdinal !== null + ? 1 + : 0) > + LIMITS.intentsPerFamily || p.participants.some((r) => !ROLES.includes(r)) ) throw Error("Retention family progress identity/capacity mismatch"); @@ -757,6 +1123,180 @@ export function verifyFamilyOwnedRecord( family.close(); } } +/** Fixed normal coordinator source observation. Unlike ordinary indexed + * readers, the reciprocal target check runs under the configuration writer and + * must neither acquire a family writer nor change any metadata. Derive its one + * private address from nominal authority; refuse busy/changing/retiring input. */ +export function verifyCoordinatorPullAdmissionReadOnly( + authority: VerifiedExecutionAuthority, +): void { + const source = executionAuthorityState(authority), + state = source.installation; + if ( + !source.origin || + state.fixedChildId || + state.installation.role !== "ops-agent" || + source.kind !== "service.pull" || + source.execution.executor !== "ops-agent" || + source.execution.principal !== "api" + ) + throw Error("Actual normal coordinator admission required"); + const family = openFamily(state, source.origin); + try { + const unlocked = () => { + for (const [directory, name] of [ + [family.slot, ".family-writer"], + [family.shared, ".membership-writer"], + ] as const) { + let held: AnchoredDirectory; + try { + held = directory.directory(name); + } catch (error) { + if (missing(error)) continue; + throw error; + } + held.close(); + throw Error("Coordinator source index is busy"); + } + checkLocalIdentity(family); + }; + unlocked(); + const before = readProgress(family), + beforeDecision = decision(family), + address = { + scope: "private" as const, + namespace: "executions", + container: "", + name: `${source.execution.operationKey}.json`, + }, + raw = readPrivateRecord(state, address.namespace, address.name, 65536); + if ( + before.phase !== "active" || + beforeDecision.phase !== "active" || + controlDigest(raw) !== source.receiptDigest + ) + throw Error("Original active coordinator admission changed"); + z.object({ + kind: z.literal("service.pull"), + serviceId: z.literal("data-manager"), + candidateId: z.uuid(), + }) + .passthrough() + .parse((raw as { payload: { operation: unknown } }).payload.operation); + const retained = findRecord(family, address), + bytes = canonicalJson(raw); + if (!retained || retained.state !== "written" || retained.digest !== controlDigest(bytes)) + throw Error("Original coordinator admission index changed"); + const opened = recordDirectory(family, retained); + try { + if ( + !sameInode(opened.dir.fileIdentity(retained.name), retained.inode) || + controlDigest(opened.dir.read(retained.name, recordMaximum(retained)).toString("utf8")) !== + retained.digest + ) + throw Error("Original coordinator admission inode changed"); + } finally { + opened.close(); + } + if ( + controlDigest(readProgress(family)) !== controlDigest(before) || + controlDigest(decision(family)) !== controlDigest(beforeDecision) || + controlDigest(findRecord(family, address)) !== controlDigest(retained) + ) + throw Error("Coordinator source index is changing"); + unlocked(); + } finally { + family.close(); + } +} +/** Only the exact release-owner or subsequent canonical-completion preimage + * ALREADY pinned by this original index can replace the old signed owner. + * Actual retained physical release is mandatory for both finite transitions; + * normal completion still verifies full closure before publishing success. + * No generic index rollback, new effect/inspection/control body or time. */ +export function repairIndexedConfigurationMaintenanceSettleCompletion( + authority: VerifiedExecutionAuthority, +): void { + const source = executionAuthorityState(authority), + state = source.installation; + if ( + state.installation.role !== "ops-agent" || + state.fixedChildId || + source.kind !== "configurationMaintenance.prepare" || + !source.origin + ) + throw Error("Original maintenance completion recovery required"); + withPrivateEffectWriter(source, () => { + const admitted = readPrivateRecord( + state, + "executions", + `${source.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + name = `${z.uuid().parse(admitted.payload?.operation?.maintenanceId)}.json`, + raw = readPrivateRecord(state, "configuration-maintenance", name), + owner = verifyControl( + state.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + assertConfigurationMaintenanceDecisionSource(owner); + if ( + owner.admissionDigest !== source.receiptDigest || + canonicalJson(owner.origin) !== canonicalJson(source.origin) || + canonicalJson(owner.plan.prepare) !== canonicalJson(source.execution) || + !owner.final || + !owner.inspection || + owner.noGuard + ) + throw Error("Original completion source changed"); + const released = verifyRetainedArtifactTargetRootRelease(authority), + completed = + owner.receipt.state === "settling" + ? configurationMaintenanceReleasedOwner(owner) + : configurationMaintenanceSettleCompletion(owner, released.releasedAt), + bytes = canonicalJson( + signControl(state.privateKey, "openmapx-configuration-maintenance-owner-v2", completed), + ), + f = openFamily(state, source.origin!); + try { + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const address = { + scope: "private" as const, + namespace: "configuration-maintenance", + container: "", + name, + }, + retained = findRecord(f, address), + p = readProgress(f); + if ( + !retained || + retained.state !== "written" || + retained.digest !== controlDigest(bytes) || + Buffer.byteLength(bytes) > 131072 || + p.phase !== "active" || + decision(f).phase !== "active" + ) + throw Error("Exact indexed settle completion required"); + const directory = state.privateRoot.directory("configuration-maintenance"); + try { + if ( + canonicalJson(JSON.parse(directory.read(name, 131072).toString("utf8"))) !== + canonicalJson(raw) + ) + throw Error("Original completion source raced"); + directory.write(name, bytes, false, (inode) => saveEntry(f, { ...retained, inode })); + } finally { + directory.close(); + } + }); + } finally { + f.close(); + } + }); +} export function familyRecordWrite( state: InstallationState, origin: ControlExecutionOrigin, @@ -912,6 +1452,7 @@ export function registerFamilyExecution( execution: ExecutionRef, receiptDigest: string, parent?: RetainedParentExecution, + stagedCoordinatorBaseline = false, ) { reserveControlFamily(state, origin); const f = openFamily(state, origin); @@ -927,6 +1468,66 @@ export function registerFamilyExecution( } if (p.phase !== "active" || decision(f).phase !== "active") throw Error("Closed origin membership"); + if (stagedCoordinatorBaseline) { + // Unsigned prospective encoding arithmetic. Both packets occupy the + // existing cells; the collector's independent 8KiB headroom remains. + const maximum = Number.MAX_SAFE_INTEGER, + hash = "f".repeat(64), + receipts = { + ...Object.fromEntries(Object.keys(p.executions).map((key) => [key, hash])), + [id]: hash, + }, + candidate = { + version: 1, + installationId: f.meta.installationId, + familyId: f.meta.familyId, + generation: f.meta.generation, + role: f.state.installation.role, + originDigest: f.originHash, + terminalAt: maximum, + terminalSequenceDigest: hash, + executionReceiptDigests: receipts, + localRecordSetDigest: hash, + sharedRecordSetDigest: hash, + leaseDigests: Object.fromEntries(Object.keys(p.leases).map((key) => [key, hash])), + intentDigests: Object.fromEntries(Object.keys(p.intents).map((key) => [key, hash])), + }; + const width = (value: unknown) => Buffer.byteLength(canonicalJson(value)); + const futureFact = { + execution, + verifier: state.installation.publicKey, + receiptDigest: hash, + terminalDigest: hash, + settlementDigest: hash, + canonicalDigest: hash, + canonicalTerminalAt: maximum, + terminalAt: maximum, + known: true, + requiresJournal: true, + journalContained: true, + }; + const futureProgress = { + ...p, + sequence: maximum, + terminalAt: maximum, + executions: { ...p.executions, [id]: futureFact }, + ...(parent?.issuer === "api" + ? { + api: { + parent, + operationKey: execution.operationKey, + fingerprint: execution.fingerprint, + }, + } + : {}), + }; + if ( + width({ payload: candidate, signature: "" }) + 86 + 4096 + 64 > 65536 || + width(futureProgress) + 4096 + 64 + 8192 > 131072 || + p.metadataBytes > LIMITS.familyMetadataBytes + ) + throw Error("Staged coordinator retirement capacity unavailable before pull"); + } ticket(f, p, "executions", id, LIMITS.executionsPerFamily, execution); p.executions[id] = { execution, @@ -948,11 +1549,1397 @@ export function registerFamilyExecution( operationKey: execution.operationKey, fingerprint: execution.fingerprint, }; - if (p.api && controlDigest(p.api) !== controlDigest(api)) - throw Error("Origin API parent conflict"); - p.api = api; + if (p.api && controlDigest(p.api) !== controlDigest(api)) + throw Error("Origin API parent conflict"); + p.api = api; + } + progress(f, p, true); + }), + ); + } finally { + f.close(); + } +} +/** Fixed settlement reservation projection reads the actual original issuer + * membership. Caller digests/ordinals cannot create or replace a ticket. */ +export function retainedConfigurationMaintenanceFamilyMember( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, +) { + if ( + state.fixedChildId || + state.installation.role !== "ops-agent" || + origin.issuer !== "ops-agent" || + origin.rootKey !== execution.operationKey || + execution.executor !== "ops-agent" || + execution.principal !== "api" + ) + throw Error("Original maintenance issuer membership required"); + const f = openFamily(state, origin); + try { + return guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + id = controlDigest(execution), + member = p.executions[id]; + if ( + p.phase !== "active" || + decision(f).phase !== "active" || + !member || + canonicalJson(member.execution) !== canonicalJson(execution) + ) + throw Error("Original active maintenance membership unavailable"); + const tickets = f.shared.directory("executions"); + try { + for (let ordinal = 0; ordinal < LIMITS.executionsPerFamily; ordinal++) { + let ticket: { id: string; generation: string; value: unknown }; + try { + ticket = json(tickets, `slot-${ordinal}.json`, 4096); + } catch (error) { + if (missing(error)) continue; + throw error; + } + if (ticket.id !== id) continue; + if ( + ticket.generation !== f.meta.generation || + canonicalJson(ticket.value) !== canonicalJson(execution) + ) + throw Error("Original maintenance membership ticket changed"); + return { + role: "ops-agent" as const, + familyId: f.meta.familyId, + rootMemberOrdinal: ordinal, + rootExecutionDigest: id, + rootAdmissionDigest: member.receiptDigest, + }; + } + throw Error("Original maintenance issuer ticket missing"); + } finally { + tickets.close(); + } + }), + ); + } finally { + f.close(); + } +} +/** Counterparty verification reads only the indexed shared issuer source; + * the DM never needs an ops-agent-private family directory. */ +export function verifyPublishedConfigurationMaintenanceFamilyMember( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, + membership: { + role: "ops-agent"; + familyId: string; + rootMemberOrdinal: number; + rootExecutionDigest: string; + rootAdmissionDigest: string; + }, +): void { + if ( + origin.issuer !== "ops-agent" || + execution.executor !== "ops-agent" || + execution.principal !== "api" || + origin.rootKey !== execution.operationKey || + membership.role !== "ops-agent" || + !Number.isInteger(membership.rootMemberOrdinal) || + membership.rootMemberOrdinal < 0 || + membership.rootMemberOrdinal >= LIMITS.executionsPerFamily + ) + throw Error("Original shared maintenance membership required"); + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(state.dataRoot)), + control = own(root.directory(".admin-target-control")), + families = own(control.directory("retention-families")), + family = own(families.directory(originHash(origin))); + const meta = verifyControl( + state.publicKeys["ops-agent"], + "openmapx-retention-family-v1", + json(family, "identity.json", 2048), + sharedIdentitySchema, + ); + if ( + meta.installationId !== state.installation.installationId || + meta.familyId !== membership.familyId || + controlDigest(meta.origin) !== controlDigest(origin) || + !sameInode(meta.dataRootIdentity, identity(root.stat)) || + !sameInode(meta.sharedIdentity, identity(family.stat)) + ) + throw Error("Shared original family identity changed"); + const tickets = own(family.directory("executions")), + ticket = json<{ id: string; generation: string; value: unknown }>( + tickets, + `slot-${membership.rootMemberOrdinal}.json`, + 4096, + ); + if ( + ticket.generation !== meta.generation || + ticket.id !== membership.rootExecutionDigest || + ticket.id !== controlDigest(execution) || + canonicalJson(ticket.value) !== canonicalJson(execution) + ) + throw Error("Shared original issuer membership changed"); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +function settlementSlotTickets( + f: Family, + slot: ConfigurationSettlementFamilySlot, + repairAllocation: boolean, +): void { + const check = (kind: "executions" | "intents", ordinal: number, id: string) => { + const dir = f.shared.directory(kind), + expected = { + id, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role: slot.role, + settlementId: slot.settlementId, + projectionDigest: slot.projectionDigest, + address: slot.proposedExecutionAddress, + executionDigest: slot.executionDigest, + controlBindingDigest: slot.controlBindingDigest, + state: slot.state === "allocating" ? "reserved" : slot.state, + ...(slot.state === "UNUSED" ? { unusedIntentDigest: slot.unusedIntentDigest } : {}), + }, + }; + try { + let stored: unknown; + try { + stored = json(dir, `slot-${ordinal}.json`, 4096); + } catch (error) { + if (!missing(error) || !repairAllocation || slot.state !== "allocating") throw error; + // Original durable allocation transaction already fixed this ordinal + // and charged its complete width; never select a replacement ordinal. + cell(dir, `slot-${ordinal}.json`, expected, 4096, true); + stored = expected; + } + if (canonicalJson(stored) !== canonicalJson(expected)) + throw Error("Original settlement capacity ticket changed"); + } finally { + dir.close(); + } + }; + if (slot.role === "data-manager" && slot.executionOrdinal !== null) + check("executions", slot.executionOrdinal, slot.executionTicketId); + if (slot.intentOrdinal !== null) check("intents", slot.intentOrdinal, slot.intentTicketId); +} +/** Closed logical-control capacity entry. Its caller is the fixed original + * owner/verified PREPARE source, never ordinary execution registration. Tickets + * retain the original address while late execution/binding digests are null. */ +export function reserveConfigurationSettlementFamilySlot( + state: InstallationState, + projection: ConfigurationSettlementProjection, + projectionDigest: string, + prepareExecution: ExecutionRef | undefined, + descriptors: readonly ConfigurationSettlementRecordDescriptor[], + stage: "refusal-channel" | "complete" = "complete", +): ConfigurationSettlementFamilySlot { + if ( + state.fixedChildId || + !["ops-agent", "data-manager"].includes(state.installation.role) || + projection.origin.issuer !== "ops-agent" || + projection.ownerPrepare.operationKey !== projection.origin.rootKey || + projection.installationId !== state.installation.installationId || + !DIGEST.test(projectionDigest) || + descriptors.length !== 6 || + descriptors.some( + (item) => + item.role !== state.installation.role || + !safeName.test(item.name) || + !Number.isSafeInteger(item.cap) || + item.cap <= 0 || + item.cap > 16384, + ) + ) + throw Error("Original fixed settlement capacity source required"); + const f = openFamily(state, projection.origin); + try { + return guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + role = state.installation.role as "ops-agent" | "data-manager", + recordDescriptorsDigest = controlDigest(descriptors), + address = projection.reservation.execution; + if (p.configurationSettlementSlot) { + const old = p.configurationSettlementSlot; + if ( + old.settlementId !== projection.settlementId || + old.projectionDigest !== projectionDigest || + old.role !== role || + old.recordDescriptorsDigest !== recordDescriptorsDigest || + canonicalJson(old.proposedExecutionAddress) !== canonicalJson(address) + ) + throw Error("Original settlement capacity slot conflict"); + if ( + !["refusal_channel", "capacity_refused", "allocating", "reserved"].includes(old.state) + ) + throw Error("Settlement capacity already bound/closed"); + if (old.state !== "refusal_channel" || stage === "refusal-channel") { + settlementSlotTickets(f, old, true); + return structuredClone(old); + } + } + if ( + p.phase !== "active" || + decision(f).phase !== "active" || + (role === "data-manager" && !prepareExecution) || + !p.executions[ + controlDigest(role === "ops-agent" ? projection.ownerPrepare : prepareExecution!) + ] + ) + throw Error("Original settlement family member required"); + const executionTicketId = + role === "ops-agent" + ? controlDigest(projection.ownerPrepare) + : controlDigest({ + purpose: "configurationSettlementSlot", + role, + settlementId: projection.settlementId, + member: "execution", + }), + intentTicketId = controlDigest({ + purpose: "configurationSettlementSlot", + role, + settlementId: projection.settlementId, + member: "intent", + }); + if (stage === "refusal-channel") { + if ( + role !== "data-manager" || + descriptors[0]?.scope !== "shared" || + descriptors[0]?.cap !== 8192 || + !descriptors[0].name.endsWith(".configuration-settlement-capacity.json") + ) + throw Error("Fixed DM refusal channel required"); + if ( + p.metadataBytes + descriptors[0].cap > LIMITS.familyMetadataBytes || + p.records >= LIMITS.recordsPerFamily || + p.sharedRecords >= 2048 + ) + throw Error("Minimum DM refusal channel unavailable; retain original owner unknown"); + const slot: ConfigurationSettlementFamilySlot = { + version: 2, + role, + settlementId: projection.settlementId, + projectionDigest, + reservationDigest: projection.reservationDigest, + ownerPrepareDigest: controlDigest(projection.ownerPrepare), + proposedExecutionAddress: address, + executionOrdinal: null, + intentOrdinal: null, + executionTicketId, + intentTicketId, + recordDescriptorsDigest, + descriptors: structuredClone([...descriptors]), + state: "refusal_channel", + reservedMetadataBytes: descriptors[0].cap, + refusalConstraint: null, + executionDigest: null, + controlBindingDigest: null, + allocatedDescriptorDigests: [], + unusedIntentDigest: null, + unusedConfirmedAt: null, + }; + p.configurationSettlementSlot = slot; + p.metadataBytes += slot.reservedMetadataBytes; + progress(f, p, true); + return structuredClone(slot); + } + if (role === "data-manager") { + const base = p.configurationSettlementSlot, + descriptor = descriptors[0]!, + retained = findRecord(f, { ...descriptor, container: descriptor.role }); + if ( + !base || + base.state !== "refusal_channel" || + !retained || + retained.state !== "written" + ) + throw Error("Actual reserved DM refusal channel required before future capacity"); + const opened = recordDirectory(f, retained); + try { + const bytes = opened.dir.read(descriptor.name, descriptor.cap).toString("utf8"); + if ( + Buffer.byteLength(bytes) !== descriptor.cap || + controlDigest(bytes) !== retained.digest || + !sameInode(opened.dir.fileIdentity(descriptor.name), retained.inode) + ) + throw Error("Original DM refusal channel changed"); + } finally { + opened.close(); + } + base.allocatedDescriptorDigests = [controlDigest(descriptor)]; + } + const findOrdinal = (kind: "executions" | "intents", maximum: number) => { + const dir = f.shared.directory(kind); + try { + for (let ordinal = 0; ordinal < maximum; ordinal++) { + try { + json(dir, `slot-${ordinal}.json`, 4096); + } catch (error) { + if (missing(error)) return ordinal; + throw error; + } + } + return null; + } finally { + dir.close(); + } + }; + const executionOrdinal = + role === "ops-agent" + ? projection.issuerMembership.rootMemberOrdinal + : findOrdinal("executions", LIMITS.executionsPerFamily), + intentOrdinal = findOrdinal("intents", LIMITS.intentsPerFamily); + // Reserve both ticket widths and every cell's final padded width now. + const bytes = + descriptors.reduce((sum, descriptor) => sum + descriptor.cap, 0) + + (role === "data-manager" ? 8192 : 4096); + const priorBytes = p.configurationSettlementSlot?.reservedMetadataBytes ?? 0, + available = (directory: AnchoredDirectory) => { + directory.check(); + const fs = statfsSync(directory.anchor, { bigint: true }); + directory.check(); + return fs.bavail * fs.bsize; + }, + constraint: ConfigurationSettlementFamilySlot["refusalConstraint"] = + Object.keys(p.executions).length + (role === "data-manager" ? 1 : 0) > + LIMITS.executionsPerFamily || + Object.keys(p.intents).length + 1 > LIMITS.intentsPerFamily + ? "members" + : executionOrdinal === null || intentOrdinal === null + ? "tickets" + : p.metadataBytes + bytes - priorBytes > LIMITS.familyMetadataBytes + ? "metadata" + : p.records + descriptors.length - (role === "data-manager" ? 1 : 0) > + LIMITS.recordsPerFamily || + p.sharedRecords + + descriptors.filter((item) => item.scope === "shared").length - + (role === "data-manager" ? 1 : 0) > + 2048 + ? "records" + : available(f.shared) < BigInt(bytes - priorBytes) || + available(state.privateRoot) < BigInt(bytes - priorBytes) + ? "filesystem" + : null; + if (constraint) { + if (role !== "data-manager") + throw Error(`Sender settlement capacity exhausted: ${constraint}`); + const base = p.configurationSettlementSlot!; + base.state = "capacity_refused"; + base.refusalConstraint = constraint; + progress(f, p, true); + return structuredClone(base); + } + const slot: ConfigurationSettlementFamilySlot = { + version: 2, + role, + settlementId: projection.settlementId, + projectionDigest, + reservationDigest: projection.reservationDigest, + ownerPrepareDigest: controlDigest(projection.ownerPrepare), + proposedExecutionAddress: address, + executionOrdinal, + intentOrdinal, + executionTicketId, + intentTicketId, + recordDescriptorsDigest, + descriptors: structuredClone([...descriptors]), + state: "allocating", + reservedMetadataBytes: bytes, + refusalConstraint: null, + executionDigest: null, + controlBindingDigest: null, + allocatedDescriptorDigests: + p.configurationSettlementSlot?.allocatedDescriptorDigests ?? [], + unusedIntentDigest: null, + unusedConfirmedAt: null, + }; + p.configurationSettlementSlot = slot; + p.metadataBytes += bytes - priorBytes; + progress(f, p, true); + settlementSlotTickets(f, slot, true); + return structuredClone(slot); + }), + ); + } finally { + f.close(); + } +} +/** After all fixed cells exist, verify their exact index/inode/width and seal + * the original reservation. This never allocates a ticket or record. */ +export function completeConfigurationSettlementFamilyReservation( + state: InstallationState, + projection: ConfigurationSettlementProjection, +): ConfigurationSettlementFamilySlot { + const f = openFamily(state, projection.origin); + try { + return guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + slot = p.configurationSettlementSlot; + if ( + !slot || + slot.settlementId !== projection.settlementId || + !["allocating", "reserved"].includes(slot.state) || + slot.executionOrdinal === null || + slot.intentOrdinal === null || + p.phase !== "active" || + decision(f).phase !== "active" + ) + throw Error("Original unbound settlement reservation required"); + settlementSlotTickets(f, slot, false); + for (const descriptor of slot.descriptors) { + const entry = findRecord(f, { + ...descriptor, + container: descriptor.scope === "shared" ? descriptor.role : "", + }); + if (!entry || entry.state !== "written") + throw Error("Settlement record allocation incomplete"); + const source = recordDirectory(f, entry); + try { + const bytes = source.dir.read(descriptor.name, descriptor.cap).toString("utf8"); + if ( + Buffer.byteLength(bytes) !== descriptor.cap || + !sameInode(source.dir.fileIdentity(descriptor.name), entry.inode) || + controlDigest(bytes) !== entry.digest + ) + throw Error("Original settlement reserved cell changed"); + } finally { + source.close(); + } + } + const allocated = slot.descriptors.map((descriptor) => controlDigest(descriptor)).sort(); + if (slot.state === "reserved") { + if (canonicalJson(slot.allocatedDescriptorDigests) !== canonicalJson(allocated)) + throw Error("Settlement allocation set changed"); + } else { + slot.allocatedDescriptorDigests = allocated; + slot.state = "reserved"; + progress(f, p, true); + } + return structuredClone(slot); + }), + ); + } finally { + f.close(); + } +} +export function readConfigurationSettlementFamilySlot( + state: InstallationState, + projection: ConfigurationSettlementProjection, +): ConfigurationSettlementFamilySlot { + const f = openFamily(state, projection.origin); + try { + return guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + slot = p.configurationSettlementSlot; + if ( + !slot || + slot.settlementId !== projection.settlementId || + slot.reservationDigest !== projection.reservationDigest || + canonicalJson(slot.proposedExecutionAddress) !== + canonicalJson(projection.reservation.execution) + ) + throw Error("Original settlement capacity slot unavailable"); + settlementSlotTickets(f, slot, false); + return structuredClone(slot); + }); + } finally { + f.close(); + } +} +/** Fills ONLY the original PREPARE-reserved logical-control tickets. The full + * immutable binding comes from this role's actual indexed private producer + * cell, never from a supplied digest/execution/boolean. Partial ticket writes + * can be repaired only to those exact original bytes and return no dispatch. */ +export function bindOwnedConfigurationSettlementFamilySlot( + state: InstallationState, + projection: ConfigurationSettlementProjection, +): ConfigurationSettlementFamilySlot { + if (state.fixedChildId || !["ops-agent", "data-manager"].includes(state.installation.role)) + throw Error("Fixed logical settlement role required"); + const f = openFamily(state, projection.origin); + try { + return guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + slot = p.configurationSettlementSlot; + if ( + !slot || + !["reserved", "bound"].includes(slot.state) || + slot.settlementId !== projection.settlementId || + slot.projectionDigest !== + settlementRecordDigest(configurationSettlementDomains.projection, projection) || + slot.reservationDigest !== projection.reservationDigest || + canonicalJson(slot.proposedExecutionAddress) !== + canonicalJson(projection.reservation.execution) || + slot.intentOrdinal === null || + slot.executionOrdinal === null + ) + throw Error("Original logical settlement reservation required"); + const descriptor = slot.descriptors.find( + (value) => + value.scope === "private" && + value.name.endsWith(slot.role === "ops-agent" ? "-binding.json" : "-admission.json"), + ); + if (!descriptor) throw Error("Original logical binding descriptor required"); + const entry = findRecord(f, { ...descriptor, container: "" }); + if (!entry || entry.state !== "written") + throw Error("Original private binding source unavailable"); + const opened = recordDirectory(f, entry); + let raw: unknown; + try { + const bytes = opened.dir.read(descriptor.name, descriptor.cap).toString("utf8"); + if ( + Buffer.byteLength(bytes) !== descriptor.cap || + controlDigest(bytes) !== entry.digest || + !sameInode(opened.dir.fileIdentity(descriptor.name), entry.inode) + ) + throw Error("Original private binding source changed"); + raw = JSON.parse(bytes); + } finally { + opened.close(); + } + const signed = + slot.role === "ops-agent" + ? raw + : verifyControl( + state.installation.publicKey, + configurationSettlementDomains.admission, + raw, + configurationSettlementAdmissionSchema, + ).controlEnvelope, + envelope = verifyControl( + state.publicKeys["ops-agent"], + slot.role === "ops-agent" + ? configurationSettlementDomains.binding + : configurationSettlementDomains.control, + signed, + configurationSettlementControlEnvelopeSchema, + ), + executionDigest = controlDigest(envelope.execution), + bindingDigest = settlementRecordDigest(configurationSettlementDomains.binding, envelope), + { + operationFingerprint: _fingerprint, + fingerprint: _executionFingerprint, + deadlineAt: _deadline, + ...address + } = envelope.execution; + if ( + envelope.installationId !== state.installation.installationId || + canonicalJson(envelope.origin) !== canonicalJson(projection.origin) || + canonicalJson(envelope.ownerPrepare) !== canonicalJson(projection.ownerPrepare) || + envelope.admissionDigest !== projection.rootAdmissionDigest || + envelope.rootCertificateDigest !== projection.rootCertificateDigest || + envelope.settlementId !== projection.settlementId || + envelope.projectionDigest !== slot.projectionDigest || + envelope.reservationDigest !== slot.reservationDigest || + canonicalJson(envelope.operation) !== canonicalJson(projection.operation) || + canonicalJson(address) !== canonicalJson(slot.proposedExecutionAddress) || + envelope.execution.operationFingerprint !== projection.reservation.operationFingerprint || + canonicalJson(envelope.serviceIds) !== canonicalJson(projection.reservation.serviceIds) + ) + throw Error("Original logical binding identity changed"); + if (slot.state === "reserved") { + if ( + p.phase !== "active" || + decision(f).phase !== "active" || + slot.executionDigest !== null || + slot.controlBindingDigest !== null + ) + throw Error("Original logical binding no longer available"); + settlementSlotTickets(f, slot, false); + slot.state = "bound"; + slot.executionDigest = executionDigest; + slot.controlBindingDigest = bindingDigest; + progress(f, p); + } else if ( + slot.executionDigest !== executionDigest || + slot.controlBindingDigest !== bindingDigest + ) + throw Error("Immutable original logical binding changed"); + const finish = (kind: "executions" | "intents", ordinal: number, id: string) => { + const directory = f.shared.directory(kind), + base = { + id, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role: slot.role, + settlementId: slot.settlementId, + projectionDigest: slot.projectionDigest, + address: slot.proposedExecutionAddress, + executionDigest: null, + controlBindingDigest: null, + state: "reserved", + }, + }, + next = { + ...base, + value: { + ...base.value, + executionDigest, + controlBindingDigest: bindingDigest, + state: "bound", + }, + }; + try { + const existing = json(directory, `slot-${ordinal}.json`, 4096); + if (canonicalJson(existing) === canonicalJson(base)) + cell(directory, `slot-${ordinal}.json`, next, 4096); + else if (canonicalJson(existing) !== canonicalJson(next)) + throw Error("Original logical bound ticket changed"); + } finally { + directory.close(); + } + }; + if (slot.role === "data-manager") + finish("executions", slot.executionOrdinal, slot.executionTicketId); + finish("intents", slot.intentOrdinal, slot.intentTicketId); + settlementSlotTickets(f, slot, false); + return structuredClone(slot); + }), + ); + } finally { + f.close(); + } +} +/** Counterpart capacity publication is pinned-role signed. This checks its + * retained original shared member/intent preimages, without DM-private reads. */ +export function verifyPublishedDmConfigurationSettlementSlots( + state: InstallationState, + projection: ConfigurationSettlementProjection, + capacity: { + reservedExecutionOrdinal: number; + reservedIntentOrdinal: number; + projectionDigest: string; + }, +): void { + if (state.installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Original issuer capacity observer required"); + const f = openFamily(state, projection.origin); + try { + guard(f.shared, ".membership-writer", () => { + checkLocalIdentity(f); + if ( + f.meta.familyId !== projection.issuerMembership.familyId || + decision(f).phase !== "active" + ) + throw Error("Original reserved family unavailable"); + for (const [kind, ordinal, member] of [ + ["executions", capacity.reservedExecutionOrdinal, "execution"], + ["intents", capacity.reservedIntentOrdinal, "intent"], + ] as const) { + if ( + !Number.isSafeInteger(ordinal) || + ordinal < 0 || + ordinal >= (kind === "executions" ? 128 : 256) + ) + throw Error("Original DM capacity ordinal invalid"); + const dir = f.shared.directory(kind); + try { + const id = controlDigest({ + purpose: "configurationSettlementSlot", + role: "data-manager", + settlementId: projection.settlementId, + member, + }), + expected = { + id, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role: "data-manager", + settlementId: projection.settlementId, + projectionDigest: capacity.projectionDigest, + address: projection.reservation.execution, + executionDigest: null, + controlBindingDigest: null, + state: "reserved", + }, + }; + if (canonicalJson(json(dir, `slot-${ordinal}.json`, 4096)) !== canonicalJson(expected)) + throw Error("Original DM reserved capacity ticket changed"); + } finally { + dir.close(); + } + } + }); + } finally { + f.close(); + } +} +/** Read-only original DM member/intent binding. The actual signed capacity + * and already published original envelope supply every expected field; this + * creates no execution alias, membership, intent or authority handle. */ +export function verifyPublishedBoundDmConfigurationSettlementSlots( + state: InstallationState, + projection: ConfigurationSettlementProjection, + signedCapacity: unknown, +): void { + if (state.installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Original logical settlement observer required"); + const capacity = verifyControl( + state.publicKeys["data-manager"], + configurationSettlementDomains.capacity, + signedCapacity, + configurationSettlementCapacitySchema, + ), + f = openFamily(state, projection.origin); + try { + guard(f.shared, ".membership-writer", () => { + checkLocalIdentity(f); + const entry = findRecord(f, { + scope: "shared", + namespace: "execution", + container: "ops-agent", + name: `${projection.ownerPrepare.operationKey}.configuration-settlement-control.json`, + }); + if (!entry || entry.state !== "written") throw Error("Original shared control required"); + const source = recordDirectory(f, entry); + let raw: unknown; + try { + const bytes = source.dir.read(entry.name, 8192).toString("utf8"); + if ( + controlDigest(bytes) !== entry.digest || + !sameInode(source.dir.fileIdentity(entry.name), entry.inode) + ) + throw Error("Original shared control index changed"); + raw = JSON.parse(bytes); + } finally { + source.close(); + } + const cell = verifyControl( + state.installation.publicKey, + configurationSettlementDomains.control, + raw, + configurationSettlementControlCellSchema.shape.payload, + ); + if ( + !cell.binding || + "unused" in cell || + canonicalJson(cell.projection.payload) !== canonicalJson(projection) + ) + throw Error("Original bound projection required"); + const envelope = verifyControl( + state.installation.publicKey, + configurationSettlementDomains.control, + cell.binding, + configurationSettlementControlEnvelopeSchema, + ), + expectedDescriptors = configurationSettlementRecordDescriptors( + projection, + controlChildOperationKey( + projection.ownerPrepare.operationKey, + "configuration-fence", + projection.reservation.ordinal - 1, + ), + ).receiver; + if ( + f.meta.familyId !== projection.issuerMembership.familyId || + capacity.installationId !== state.installation.installationId || + canonicalJson(capacity.origin) !== canonicalJson(projection.origin) || + capacity.ownerPrepareDigest !== controlDigest(projection.ownerPrepare) || + capacity.settlementId !== projection.settlementId || + capacity.projectionDigest !== + settlementRecordDigest(configurationSettlementDomains.projection, projection) || + capacity.reservationDigest !== projection.reservationDigest || + canonicalJson(capacity.proposedExecutionAddress) !== + canonicalJson(projection.reservation.execution) || + capacity.recordDescriptorsDigest !== controlDigest(expectedDescriptors) || + capacity.maxBytes !== + expectedDescriptors.reduce((sum, descriptor) => sum + descriptor.cap, 0) + ) + throw Error("Original bound DM capacity conflict"); + for (const [kind, ordinal, member] of [ + ["executions", capacity.reservedExecutionOrdinal, "execution"], + ["intents", capacity.reservedIntentOrdinal, "intent"], + ] as const) { + const directory = f.shared.directory(kind), + id = controlDigest({ + purpose: "configurationSettlementSlot", + role: "data-manager", + settlementId: projection.settlementId, + member, + }), + expected = { + id, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role: "data-manager", + settlementId: projection.settlementId, + projectionDigest: capacity.projectionDigest, + address: projection.reservation.execution, + executionDigest: controlDigest(envelope.execution), + controlBindingDigest: settlementRecordDigest( + configurationSettlementDomains.binding, + envelope, + ), + state: "bound", + }, + }; + try { + if ( + canonicalJson(json(directory, `slot-${ordinal}.json`, 4096)) !== canonicalJson(expected) + ) + throw Error("Original bound DM ticket changed"); + } finally { + directory.close(); + } + } + }); + } finally { + f.close(); + } +} +export function verifyPublishedUnusedDmConfigurationSettlementSlots( + state: InstallationState, + projection: ConfigurationSettlementProjection, + unused: { + executionOrdinal: number | null; + intentOrdinal: number | null; + projectionDigest: string; + intentDigest: string; + }, +): void { + if (state.installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Original issuer UNUSED observer required"); + const f = openFamily(state, projection.origin); + try { + guard(f.shared, ".membership-writer", () => { + checkLocalIdentity(f); + if (f.meta.familyId !== projection.issuerMembership.familyId) + throw Error("Original UNUSED family changed"); + for (const [kind, ordinal, member, maximum] of [ + ["executions", unused.executionOrdinal, "execution", 128], + ["intents", unused.intentOrdinal, "intent", 256], + ] as const) { + const dir = f.shared.directory(kind), + id = controlDigest({ + purpose: "configurationSettlementSlot", + role: "data-manager", + settlementId: projection.settlementId, + member, + }); + try { + if (ordinal === null) { + for (let n = 0; n < maximum; n++) { + try { + if (json<{ id: string }>(dir, `slot-${n}.json`, 4096).id === id) + throw Error("Unexpected allocated DM slot in null-capacity proof"); + } catch (error) { + if (!missing(error)) throw error; + } + } + continue; + } + if (!Number.isSafeInteger(ordinal) || ordinal < 0 || ordinal >= maximum) + throw Error("Original UNUSED ordinal invalid"); + const expected = { + id, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role: "data-manager", + settlementId: projection.settlementId, + projectionDigest: unused.projectionDigest, + address: projection.reservation.execution, + executionDigest: null, + controlBindingDigest: null, + state: "UNUSED", + unusedIntentDigest: unused.intentDigest, + }, + }; + if (canonicalJson(json(dir, `slot-${ordinal}.json`, 4096)) !== canonicalJson(expected)) + throw Error("Original DM UNUSED ticket changed"); + } finally { + dir.close(); + } + } + }); + } finally { + f.close(); + } +} +/** Recovery DATA only: this cannot confirm a half-published UNUSED transition. + * The fixed CAS below must still verify source intent and every shared ticket. */ +export function readRefusedConfigurationSettlementCapacitySnapshot( + state: InstallationState, + projection: ConfigurationSettlementProjection, +): ConfigurationSettlementFamilySlot { + if (state.installation.role !== "data-manager" || state.fixedChildId) + throw Error("Original DM capacity snapshot required"); + const f = openFamily(state, projection.origin); + try { + return guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const slot = readProgress(f).configurationSettlementSlot; + if ( + !slot || + slot.settlementId !== projection.settlementId || + slot.reservationDigest !== projection.reservationDigest || + canonicalJson(slot.proposedExecutionAddress) !== + canonicalJson(projection.reservation.execution) || + !["reserved", "capacity_refused", "UNUSED"].includes(slot.state) || + slot.executionDigest !== null || + slot.controlBindingDigest !== null + ) + throw Error("Original unbound DM capacity source required"); + return structuredClone(slot); + }); + } finally { + f.close(); + } +} +/** No caller intent/digest/outcome arguments. Read the actual already-signed + * intent/refusal from the fixed indexed channel, check original private + * canonical/effect facts and null future cells, then CAS the original slots. + * Counts/ordinals remain reserved; no execution or terminal is manufactured. */ +export function commitRefusedDataConfigurationSettlementSlotUnused( + state: InstallationState, + projection: ConfigurationSettlementProjection, +): ConfigurationSettlementFamilySlot { + if (state.installation.role !== "data-manager" || state.fixedChildId) + throw Error("Original DM unused-slot producer required"); + const f = openFamily(state, projection.origin), + domainDigest = (domain: string, value: unknown) => + createHash("sha256") + .update(`${domain}\0${canonicalJson(value)}`) + .digest("hex"); + try { + return guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + slot = p.configurationSettlementSlot; + if ( + !slot || + slot.settlementId !== projection.settlementId || + slot.reservationDigest !== projection.reservationDigest || + !["reserved", "capacity_refused", "UNUSED"].includes(slot.state) || + slot.executionDigest !== null || + slot.controlBindingDigest !== null || + p.phase !== "active" || + decision(f).phase !== "active" + ) + throw Error("Original unbound DM slots required"); + const read = (descriptor: ConfigurationSettlementRecordDescriptor): unknown => { + const retained = findRecord(f, { + ...descriptor, + container: descriptor.scope === "shared" ? descriptor.role : "", + }); + if (!retained || retained.state !== "written") + throw Error("Original unused cell index missing"); + const opened = recordDirectory(f, retained); + try { + const bytes = opened.dir.read(descriptor.name, descriptor.cap).toString("utf8"); + if ( + Buffer.byteLength(bytes) !== descriptor.cap || + controlDigest(bytes) !== retained.digest || + !sameInode(opened.dir.fileIdentity(descriptor.name), retained.inode) + ) + throw Error("Original unused cell replaced"); + return JSON.parse(bytes); + } finally { + opened.close(); + } + }; + const compound = z + .object({ + refusal: z.unknown(), + unused: z.object({ intent: z.unknown() }).passthrough(), + }) + .passthrough() + .parse(read(slot.descriptors[0]!)), + hash = z.string().regex(/^[a-f0-9]{64}$/), + epoch = z.number().int().nonnegative().safe(), + ordinal = z.number().int().min(0).max(255).nullable(), + intent = verifyControl( + state.installation.publicKey, + "openmapx-configuration-unused-capacity-intent-v2", + compound.unused.intent, + z.strictObject({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + role: z.literal("data-manager"), + ownerPrepareDigest: hash, + prepareExecutionDigest: hash.nullable(), + settlementId: hash, + projectionDigest: hash, + capacityReceiptDigest: hash.nullable(), + sourceProofDigest: hash, + executionOrdinal: ordinal, + intentOrdinal: ordinal, + recordDescriptorsDigest: hash, + allocatedDescriptorDigests: z.array(hash).max(12), + createdAt: epoch, + }), + ), + refusal = verifyControl( + state.installation.publicKey, + "openmapx-dm-configuration-no-guard-refusal-v2", + compound.refusal, + z + .object({ + version: z.literal(2), + installationId: z.uuid(), + origin: controlOriginSchema, + settlementId: hash, + projectionDigest: hash, + reservationDigest: hash, + prepareExecution: executionRefSchema, + prepareAdmissionDigest: hash, + refusalReceipt: z.unknown(), + effectReceiptDigest: hash, + effectSettlementDigest: hash, + canonicalTerminalDigest: hash, + canonicalTerminalAt: epoch, + capacityReceiptDigest: hash.nullable(), + }) + .passthrough(), + ); + if ( + intent.installationId !== state.installation.installationId || + canonicalJson(intent.origin) !== canonicalJson(projection.origin) || + intent.ownerPrepareDigest !== controlDigest(projection.ownerPrepare) || + intent.prepareExecutionDigest !== controlDigest(refusal.prepareExecution) || + intent.settlementId !== slot.settlementId || + intent.projectionDigest !== slot.projectionDigest || + intent.capacityReceiptDigest !== refusal.capacityReceiptDigest || + intent.sourceProofDigest !== + domainDigest("openmapx-dm-configuration-no-guard-refusal-v2", refusal) || + intent.executionOrdinal !== slot.executionOrdinal || + intent.intentOrdinal !== slot.intentOrdinal || + intent.recordDescriptorsDigest !== slot.recordDescriptorsDigest || + canonicalJson(intent.allocatedDescriptorDigests) !== + canonicalJson(slot.allocatedDescriptorDigests) || + refusal.installationId !== state.installation.installationId || + canonicalJson(refusal.origin) !== canonicalJson(projection.origin) || + refusal.settlementId !== slot.settlementId || + refusal.projectionDigest !== slot.projectionDigest || + refusal.reservationDigest !== slot.reservationDigest || + slot.descriptors[0]?.scope !== "shared" || + slot.descriptors[0].namespace !== "execution" || + slot.descriptors[0].role !== "data-manager" || + slot.descriptors[0].name !== + `${refusal.prepareExecution.operationKey}.configuration-settlement-capacity.json` || + slot.descriptors[0].cap !== 8192 + ) + throw Error("Original DM UNUSED intent source changed"); + const fact = p.executions[controlDigest(refusal.prepareExecution)]; + if ( + !fact || + !fact.known || + !fact.journalContained || + fact.receiptDigest !== refusal.prepareAdmissionDigest || + fact.canonicalDigest !== refusal.canonicalTerminalDigest || + fact.canonicalTerminalAt !== refusal.canonicalTerminalAt + ) + throw Error("Original canonical DM refusal membership required"); + const observations = state.privateRoot.directory("observations"), + effects = state.privateRoot.directory("effects"); + try { + const canonical = verifyControl( + state.installation.publicKey, + "openmapx-data-fence-terminal-v1", + json(observations, `terminal-${refusal.prepareExecution.operationKey}.json`, 8192), + z + .object({ + phase: z.literal("canonical"), + execution: executionRefSchema, + admissionDigest: hash, + terminalDigest: hash, + terminalAt: epoch, + }) + .passthrough(), + ), + effect = verifyControl( + state.installation.publicKey, + "openmapx-control-settlement-v1", + json(effects, `${refusal.prepareExecution.operationKey}.json`, 65536), + z + .object({ + identity: executionRefSchema, + outcome: z.literal("failed_without_effect"), + publication: z.literal("not_started"), + terminalReceiptDigest: hash, + receipt: z.unknown(), + }) + .passthrough(), + ); + if ( + canonicalJson(canonical.execution) !== canonicalJson(refusal.prepareExecution) || + canonical.admissionDigest !== fact.receiptDigest || + canonical.terminalDigest !== refusal.canonicalTerminalDigest || + canonical.terminalAt !== refusal.canonicalTerminalAt || + canonicalJson(effect.identity) !== canonicalJson(refusal.prepareExecution) || + canonicalJson(effect.receipt) !== canonicalJson(refusal.refusalReceipt) || + effect.terminalReceiptDigest !== refusal.effectReceiptDigest || + controlDigest(effect.receipt) !== refusal.effectReceiptDigest || + controlDigest(effect) !== refusal.effectSettlementDigest + ) + throw Error("Original local canonical refusal preimages changed"); + } finally { + effects.close(); + observations.close(); + } + const actualAllocated = slot.descriptors.filter((descriptor) => + slot.allocatedDescriptorDigests.includes(controlDigest(descriptor)), + ); + if ( + actualAllocated.length !== slot.allocatedDescriptorDigests.length || + actualAllocated[0] !== slot.descriptors[0] + ) + throw Error("Original UNUSED descriptor set changed"); + for (const descriptor of actualAllocated.slice(1)) + if (read(descriptor) !== null) + throw Error("Bound/attempted DM control cannot become UNUSED"); + const intentDigest = domainDigest( + "openmapx-configuration-unused-capacity-intent-v2", + intent, + ); + if (slot.state === "UNUSED") { + if ( + slot.unusedIntentDigest !== intentDigest || + !Number.isSafeInteger(slot.unusedConfirmedAt) + ) + throw Error("Original UNUSED CAS changed"); + } else { + settlementSlotTickets(f, slot, false); + slot.state = "UNUSED"; + slot.unusedIntentDigest = intentDigest; + slot.unusedConfirmedAt = Date.now(); + progress(f, p); + } + const finishTicket = ( + kind: "executions" | "intents", + ordinal: number | null, + id: string, + ) => { + if (ordinal === null) return; + const dir = f.shared.directory(kind), + base = { + id, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role: slot.role, + settlementId: slot.settlementId, + projectionDigest: slot.projectionDigest, + address: slot.proposedExecutionAddress, + executionDigest: null, + controlBindingDigest: null, + state: "reserved", + }, + }, + next = { + ...base, + value: { ...base.value, state: "UNUSED", unusedIntentDigest: intentDigest }, + }; + try { + const existing = json(dir, `slot-${ordinal}.json`, 4096); + if (canonicalJson(existing) === canonicalJson(base)) + cell(dir, `slot-${ordinal}.json`, next, 4096); + else if (canonicalJson(existing) !== canonicalJson(next)) + throw Error("Original UNUSED ticket replaced"); + } finally { + dir.close(); + } + }; + finishTicket("executions", slot.executionOrdinal, slot.executionTicketId); + finishTicket("intents", slot.intentOrdinal, slot.intentTicketId); + settlementSlotTickets(f, slot, false); + return structuredClone(slot); + }), + ); + } finally { + f.close(); + } +} +/** The actual fixed issuer freeze/intent is read from its signed private owner. + * No caller closure, digest, slot list or requested outcome authorizes this CAS. */ +export function commitFrozenConfigurationSettlementSlotUnused( + state: InstallationState, + projection: ConfigurationSettlementProjection, +): ConfigurationSettlementFamilySlot { + if (state.installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Original frozen issuer required"); + const f = openFamily(state, projection.origin); + try { + return guard(f.shared, ".membership-writer", () => + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const p = readProgress(f), + slot = p.configurationSettlementSlot; + if ( + !slot || + slot.role !== "ops-agent" || + slot.settlementId !== projection.settlementId || + slot.projectionDigest !== + settlementRecordDigest(configurationSettlementDomains.projection, projection) || + !["reserved", "UNUSED"].includes(slot.state) || + slot.executionDigest !== null || + slot.controlBindingDigest !== null || + slot.intentOrdinal === null || + p.phase !== "active" || + decision(f).phase !== "active" + ) + throw Error("Original unbound issuer slot required"); + const read = ( + record: { + scope: "private" | "shared"; + namespace: string; + name: string; + container: string; + }, + cap: number, + ): unknown => { + const retained = findRecord(f, record); + if (!retained || retained.state !== "written") + throw Error("Original issuer source index missing"); + const opened = recordDirectory(f, retained); + try { + const bytes = opened.dir.read(record.name, cap).toString("utf8"); + if ( + controlDigest(bytes) !== retained.digest || + !sameInode(opened.dir.fileIdentity(record.name), retained.inode) + ) + throw Error("Original issuer source replaced"); + return JSON.parse(bytes); + } finally { + opened.close(); + } + }; + const rawOwner = read( + { + scope: "private", + namespace: "configuration-maintenance", + name: `${projection.maintenanceId}.json`, + container: "", + }, + 131072, + ), + owner = verifyControl( + state.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + rawOwner, + z + .object({ + admissionDigest: z.string(), + sequence: z.number(), + issuance: z.enum(["closing_without_guard", "closed_without_guard"]), + noGuard: configurationNoGuardIntentSchema, + guardSource: z.null(), + final: z.null(), + inspection: z.null(), + receipt: z + .object({ dmFence: z.null(), generation: z.literal("not_started") }) + .passthrough(), + }) + .passthrough(), + ), + intent = verifyControl( + state.installation.publicKey, + configurationSettlementDomains.unusedIntent, + owner.noGuard.issuerUnused.intent, + configurationNoGuardIntentSchema.shape.issuerUnused.shape.intent.shape.payload, + ), + fact = p.executions[controlDigest(projection.ownerPrepare)]; + if ( + !fact || + owner.admissionDigest !== fact.receiptDigest || + owner.sequence !== owner.noGuard.ownerSequence || + intent.role !== "ops-agent" || + intent.installationId !== state.installation.installationId || + canonicalJson(intent.origin) !== canonicalJson(projection.origin) || + intent.ownerPrepareDigest !== controlDigest(projection.ownerPrepare) || + intent.settlementId !== slot.settlementId || + intent.projectionDigest !== slot.projectionDigest || + intent.sourceProofDigest !== + configurationIssuerUnusedSourceDigest(owner.noGuard.noGuardEvidence) || + intent.executionOrdinal !== slot.executionOrdinal || + intent.intentOrdinal !== slot.intentOrdinal || + intent.recordDescriptorsDigest !== slot.recordDescriptorsDigest || + canonicalJson(intent.allocatedDescriptorDigests) !== + canonicalJson(slot.allocatedDescriptorDigests) + ) + throw Error("Original frozen issuer UNUSED source changed"); + for (const descriptor of slot.descriptors.filter( + (value) => value.scope === "private" || value.name.endsWith("-ack.json"), + )) { + if ( + read( + { ...descriptor, container: descriptor.scope === "shared" ? descriptor.role : "" }, + descriptor.cap, + ) !== null + ) + throw Error("Bound/attempted issuer control cannot become UNUSED"); + } + const intentDigest = settlementRecordDigest( + configurationSettlementDomains.unusedIntent, + intent, + ); + if (slot.state === "UNUSED") { + if ( + slot.unusedIntentDigest !== intentDigest || + !Number.isSafeInteger(slot.unusedConfirmedAt) + ) + throw Error("Original issuer UNUSED CAS conflict"); + } else { + settlementSlotTickets(f, slot, false); + slot.state = "UNUSED"; + slot.unusedIntentDigest = intentDigest; + slot.unusedConfirmedAt = Date.now(); + progress(f, p); + } + const dir = f.shared.directory("intents"), + base = { + id: slot.intentTicketId, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role: "ops-agent", + settlementId: slot.settlementId, + projectionDigest: slot.projectionDigest, + address: slot.proposedExecutionAddress, + executionDigest: null, + controlBindingDigest: null, + state: "reserved", + }, + }, + next = { + ...base, + value: { ...base.value, state: "UNUSED", unusedIntentDigest: intentDigest }, + }; + try { + const current = json(dir, `slot-${slot.intentOrdinal}.json`, 4096); + if (canonicalJson(current) === canonicalJson(base)) + cell(dir, `slot-${slot.intentOrdinal}.json`, next, 4096); + else if (canonicalJson(current) !== canonicalJson(next)) + throw Error("Original issuer UNUSED ticket changed"); + } finally { + dir.close(); } - progress(f, p, true); + settlementSlotTickets(f, slot, false); + return structuredClone(slot); }), ); } finally { @@ -1209,7 +3196,13 @@ function recordDirectory(f: Family, e: FamilyRecord): { dir: AnchoredDirectory; const opened: AnchoredDirectory[] = []; let dir: AnchoredDirectory; if (e.scope === "private") dir = f.state.privateRoot.directory(e.namespace); - else { + else if (e.namespace === "service-lifecycle") { + if (e.container !== "" || !/^opk1_[A-Za-z0-9_-]{16,64}\.json$/.test(e.name)) + throw Error("Invalid lifecycle receipt address"); + const agent = f.root.directory("ops-agent"); + opened.push(agent); + dir = agent.directory("lifecycle"); + } else { let current = f.root.directory(".admin-target-control"); opened.push(current); if (e.namespace === "execution") { @@ -1303,7 +3296,845 @@ function executionWriterPresent(f: Family, execution: ExecutionRef): boolean { for (const dir of opened.reverse()) dir.close(); } } +/** Full public two-cell/source/physical closure, plus THIS role's actual + * private canonical/UNUSED state. Never reads a sibling private directory and + * never allocates, changes a counter, or manufactures an unjoined participant. */ +function configurationNoGuardFamilyClosure( + f: Family, + p: Progress, +): { unusedExecutionTicketIds: string[]; unusedIntentTicketIds: string[] } | null { + const slot = p.configurationSettlementSlot; + if ( + !slot || + slot.state !== "UNUSED" || + slot.executionDigest !== null || + slot.controlBindingDigest !== null || + !slot.unusedIntentDigest || + slot.unusedConfirmedAt === null + ) + return null; + const domains = configurationSettlementDomains, + same = (a: unknown, b: unknown) => canonicalJson(a) === canonicalJson(b), + need = (condition: unknown, message: string) => { + if (!condition) throw Error(message); + }, + opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + const localRead = ( + address: Pick, + cap: number, + ): unknown => { + const indexed = findRecord(f, address); + need(indexed && indexed.state === "written", "Original local NoGuard source index missing"); + const source = recordDirectory(f, indexed!); + try { + const bytes = source.dir.read(address.name, cap).toString("utf8"); + need( + controlDigest(bytes) === indexed!.digest && + sameInode(source.dir.fileIdentity(address.name), indexed!.inode), + "Original local NoGuard source replaced", + ); + return JSON.parse(bytes); + } finally { + source.close(); + } + }; + try { + const controlRoot = own(f.root.directory(".admin-target-control")), + executions = own(controlRoot.directory("executions")), + sender = own(executions.directory("ops-agent")), + senderKey = f.state.publicKeys["ops-agent"], + readShared = (directory: AnchoredDirectory, name: string, cap: number) => { + const bytes = directory.read(name, cap); + need(bytes.length === cap, "Original NoGuard reserved width changed"); + return JSON.parse(bytes.toString("utf8")); + }, + control = verifyControl( + senderKey, + domains.control, + readShared(sender, `${f.meta.origin.rootKey}.configuration-settlement-control.json`, 8192), + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + senderKey, + domains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + bundle = verifyControl( + senderKey, + domains.reference, + readShared(sender, `${f.meta.origin.rootKey}.configuration-settlement.json`, 16384), + configurationSettlementCommitmentCellSchema.shape.payload, + ); + if (bundle.phase !== "unused" || !("unused" in control) || control.binding !== null) + return null; + const closure = verifyControl( + senderKey, + domains.closure, + bundle.closure, + configurationNoGuardClosureSchema, + ), + identity = closure.identity, + evidence = closure.noGuardEvidence, + projectionDigest = settlementRecordDigest(domains.projection, projection), + { closureDigest: ignoredClosureDigest, ...closureSource } = closure, + freezeDomain = evidence.kind === "never_issued" ? domains.neverIssued : domains.issuedRefused, + freeze = + evidence.kind === "never_issued" + ? verifyControl( + senderKey, + freezeDomain, + bundle.issuerFreezeProof, + configurationNeverIssuedProofSchema, + ) + : verifyControl( + senderKey, + freezeDomain, + bundle.issuerFreezeProof, + configurationIssuedPrepareRefusedProofSchema, + ), + publications = + "remainingPublicationStates" in freeze + ? freeze.remainingPublicationStates + : freeze.publicationStates, + issuerIntent = verifyControl( + senderKey, + domains.unusedIntent, + bundle.issuerUnused.intent, + configurationUnusedCapacityIntentSchema, + ), + issuerConfirmation = verifyControl( + senderKey, + domains.unusedConfirmation, + bundle.issuerUnused.confirmation, + configurationUnusedCapacityConfirmationSchema, + ), + issuerIntentDigest = settlementRecordDigest(domains.unusedIntent, issuerIntent), + issuerConfirmationDigest = settlementRecordDigest( + domains.unusedConfirmation, + issuerConfirmation, + ); + need( + projection.installationId === f.state.installation.installationId && + same(projection.origin, f.meta.origin) && + projection.ownerPrepare.operationKey === f.meta.origin.rootKey && + projection.issuerMembership.familyId === f.meta.familyId && + projection.issuerMembership.rootExecutionDigest === + controlDigest(projection.ownerPrepare) && + projection.issuerMembership.rootAdmissionDigest === projection.rootAdmissionDigest && + projection.reservationDigest === + settlementRecordDigest(domains.reservationTuple, projection.reservation) && + projection.reservation.operationFingerprint === controlDigest(projection.operation) && + projection.operation.version === 2 && + projection.operation.maintenanceId === projection.maintenanceId && + projection.operation.candidateId === projection.candidateId && + projection.operation.planDigest === projection.planDigest && + projection.operation.settlementId === projection.settlementId && + projection.settlementId === + deriveConfigurationFenceSettlementId({ + installationId: projection.installationId, + ownerPrepare: projection.ownerPrepare, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + }) && + same(identity, { + installationId: projection.installationId, + origin: projection.origin, + ownerPrepare: projection.ownerPrepare, + admissionDigest: projection.rootAdmissionDigest, + rootCertificateDigest: projection.rootCertificateDigest, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + settlementId: projection.settlementId, + }) && + bundle.settlementId === projection.settlementId && + bundle.projectionDigest === projectionDigest && + closure.closureDigest === settlementRecordDigest(domains.closure, closureSource) && + same(freeze.identity, identity) && + freeze.issuerSequence === closure.ownerSequence && + freeze.frozenAt === closure.closedAt && + settlementRecordDigest(freezeDomain, freeze) === evidence.issuerFreezeProofDigest && + publications.length > 0 && + publications[0]!.phase === "configuration-commit" && + publications[0]!.ordinal === 0 && + publications.every( + (value, index) => + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + "configuration-fence", + "configuration-fence-settle", + ].includes(value.phase) && + (index === 0 || publications[index - 1]!.ordinal < value.ordinal), + ) && + publications.some( + (value) => + value.phase === "configuration-fence-settle" && + value.ordinal === projection.reservation.ordinal && + value.operationKey === projection.reservation.execution.operationKey, + ), + "Original family NoGuard identity/freeze preimages changed", + ); + const fixedPhases = [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + "configuration-fence", + "configuration-fence-settle", + ] as const; + for (const publication of publications) { + const phase = fixedPhases.find((value) => value === publication.phase); + if ( + !phase || + publication.operationKey !== + controlChildOperationKey(projection.ownerPrepare.operationKey, phase, publication.ordinal) + ) + throw Error("Original frozen family phase correlation changed"); + } + const rootTicketDir = own(f.shared.directory("executions")), + rootTicket = json<{ id: string; generation: string; value: unknown }>( + rootTicketDir, + `slot-${projection.issuerMembership.rootMemberOrdinal}.json`, + 4096, + ); + need( + rootTicket.id === controlDigest(projection.ownerPrepare) && + rootTicket.generation === f.meta.generation && + same(rootTicket.value, projection.ownerPrepare), + "Original NoGuard issuer membership changed", + ); + const leases = Object.values(p.leases).filter( + (value) => + value.certificateDigest === identity.rootCertificateDigest && + value.ownerRole === "ops-agent", + ); + let leaseId: string, ownAdoption: { delegationId: string } | undefined; + if (f.state.installation.role === "ops-agent") { + need( + leases.length === 1, + "Original NoGuard physical root membership required before family ready", + ); + leaseId = leases[0]!.id; + } else { + const fact = Object.values(p.executions).find( + (value) => + evidence.kind === "receiver_refused" && + controlDigest(value.execution) === evidence.prepareExecutionDigest, + ); + need( + fact && evidence.kind === "receiver_refused", + "Actual original receiver member required", + ); + const adoption = verifyControl( + f.state.installation.publicKey, + "openmapx-configuration-fence-adoption-v2", + localRead( + { + scope: "private", + namespace: "execution-adoptions", + name: `${fact!.execution.operationKey}.json`, + container: "", + }, + 8192, + ), + z.strictObject({ + version: z.literal(2), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: z.string(), + leaseId: z.uuid(), + delegationId: z.uuid(), + certificateDigest: z.string(), + }), + ); + need( + same(adoption.origin, f.meta.origin) && + same(adoption.execution, fact!.execution) && + adoption.admissionDigest === fact!.receiptDigest && + adoption.certificateDigest === identity.rootCertificateDigest, + "Actual original receiver adoption changed", + ); + leaseId = adoption.leaseId; + ownAdoption = adoption; + } + const rootWitness = readControlExecutionWitness(f.state, projection.ownerPrepare, senderKey); + need( + rootWitness.kind === "configurationMaintenance.prepare" && + same(rootWitness.origin, f.meta.origin) && + rootWitness.authorityDigest === identity.admissionDigest, + "Actual original shared root admission witness required", + ); + const leaseRoot = own(controlRoot.directory("leases")), + ledger = own(leaseRoot.directory(leaseId)), + rootRaw = json<{ payload: { certificate: unknown } }>(ledger, "owner.json"), + certificate = verifyControl( + senderKey, + "openmapx-lease-certificate-v2", + rootRaw.payload.certificate, + z + .object({ + version: z.literal(2), + origin: controlOriginSchema, + installationId: z.uuid(), + leaseId: z.uuid(), + owner: executionRefSchema, + ownerPublicKey: z.string(), + dataRootIdentity: identitySchema, + serviceIds: z.array(z.string()).max(256), + }) + .passthrough(), + ), + root = verifyControl( + certificate.ownerPublicKey, + "openmapx-lease-owner-v2", + rootRaw, + z + .object({ + phase: z.literal("settled"), + certificate: z.unknown(), + origin: controlOriginSchema, + grantDigests: z.record(z.uuid(), z.string()), + settlement: z + .object({ + identity: executionRefSchema, + outcome: z.literal("failed_without_effect"), + publication: z.literal("not_started"), + terminalReceiptDigest: z.string(), + }) + .strict(), + }) + .passthrough(), + ), + release = verifyControl( + senderKey, + "openmapx-admission-slots-v1", + json(ledger, "admission.json", 4096), + z + .object({ + installationId: z.uuid(), + leaseId: z.uuid(), + certificateDigest: z.string(), + releasedAt: z.number().int().positive(), + }) + .passthrough(), + ); + need( + certificate.installationId === f.state.installation.installationId && + certificate.leaseId === leaseId && + same(certificate.origin, f.meta.origin) && + same(certificate.owner, projection.ownerPrepare) && + sameInode(certificate.dataRootIdentity, identityOfRoot(f)) && + controlDigest(rootRaw.payload.certificate) === identity.rootCertificateDigest && + same(root.origin, f.meta.origin) && + same(root.settlement.identity, projection.ownerPrepare) && + release.installationId === certificate.installationId && + release.leaseId === leaseId && + release.certificateDigest === identity.rootCertificateDigest, + "Original released NoGuard root changed", + ); + const ownReceipt = { + kind: "configuration-no-guard", + issuerFreezeProofDigest: evidence.issuerFreezeProofDigest, + issuerUnusedConfirmationDigest: issuerConfirmationDigest, + }, + ownContainmentDigest = controlDigest({ ...root.settlement, receipt: ownReceipt }); + need( + root.settlement.terminalReceiptDigest === controlDigest(ownReceipt), + "Original NoGuard root effect receipt changed", + ); + let prepareKey: string, + dmSlotProof: + | { role: "data-manager"; intentDigest: string; confirmationDigest: string } + | undefined, + dmIntent: ReturnType | undefined, + dmConfirmation: + | ReturnType + | undefined, + dmContainmentDigest: string | undefined; + if (evidence.kind === "never_issued") { + const prepare = publications.find((value) => value.phase === "configuration-fence"); + need( + prepare && + !("kind" in freeze) && + Object.keys(root.grantDigests).length === 0 && + evidence.publicationStatesDigest === + settlementRecordDigest(domains.neverIssued, freeze) && + evidence.issuerSequence === freeze.issuerSequence && + evidence.frozenAt === freeze.frozenAt && + !("kind" in freeze) && + evidence.prepareReservationDigest === freeze.prepareReservationDigest, + "Original unjoined NeverIssued family source changed", + ); + prepareKey = prepare!.operationKey; + assertNoUnissuedConfigurationReceiverPublication(f.state, prepareKey); + } else { + need( + "kind" in freeze && Object.keys(root.grantDigests).length === 1, + "Original receiver-refused root grant source changed", + ); + if (!("kind" in freeze)) throw Error("Original issued freeze required"); + const [delegationId, grantDigest] = Object.entries(root.grantDigests)[0]!, + grants = own(ledger.directory("grants")), + children = own(ledger.directory("children")), + rawGrant = json<{ grant: unknown; transition: unknown }>(grants, `${delegationId}.json`), + grant = verifyControl( + certificate.ownerPublicKey, + "openmapx-lease-grant-v2", + rawGrant.grant, + z + .object({ + origin: controlOriginSchema, + installationId: z.uuid(), + leaseId: z.uuid(), + ownerCertificateDigest: z.string(), + delegationId: z.uuid(), + parentNode: z.null(), + parent: executionRefSchema, + child: executionRefSchema, + childKind: z.literal("configuration-maintenance.prepare"), + phase: z.literal("configuration-fence"), + childPublicKey: z.string(), + recipient: z.literal("data-manager"), + ordinal: z.number().int(), + serviceIds: z.array(z.string()).max(256), + }) + .passthrough(), + ), + transition = verifyControl( + certificate.ownerPublicKey, + "openmapx-lease-intent-v1", + rawGrant.transition, + z.strictObject({ + delegationId: z.uuid(), + grantDigest: z.string(), + sequence: z.literal(1), + state: z.literal("attempted"), + }), + ), + child = verifyControl( + grant.childPublicKey, + "openmapx-lease-child-v2", + json(children, `${delegationId}.json`), + z + .object({ + origin: controlOriginSchema, + delegationId: z.uuid(), + grantDigest: z.string(), + execution: executionRefSchema, + phase: z.literal("settled"), + grantDigests: z.record(z.uuid(), z.string()), + settlement: z.strictObject({ + identity: executionRefSchema, + outcome: z.literal("failed_without_effect"), + publication: z.literal("not_started"), + terminalReceiptDigest: z.string(), + }), + }) + .passthrough(), + ); + prepareKey = grant.child.operationKey; + const receiver = own(executions.directory("data-manager")), + compound = configurationDmCapacityCellSchema.parse( + readShared(receiver, `${prepareKey}.configuration-settlement-capacity.json`, 8192), + ); + need( + compound.refusal && compound.unused?.confirmation, + "Original receiver refusal/UNUSED compound required", + ); + const receiverKey = f.state.publicKeys["data-manager"], + refusal = verifyControl( + receiverKey, + domains.refusal, + compound.refusal, + configurationDmNoGuardRefusalSchema, + ), + capacity = + compound.capacity && + verifyControl( + receiverKey, + domains.capacity, + compound.capacity, + configurationSettlementCapacitySchema, + ), + refusalDigest = settlementRecordDigest(domains.refusal, refusal), + descriptors = configurationSettlementRecordDescriptors(projection, prepareKey).receiver; + dmIntent = verifyControl( + receiverKey, + domains.unusedIntent, + compound.unused!.intent, + configurationUnusedCapacityIntentSchema, + ); + dmConfirmation = verifyControl( + receiverKey, + domains.unusedConfirmation, + compound.unused!.confirmation, + configurationUnusedCapacityConfirmationSchema, + ); + const intentDigest = settlementRecordDigest(domains.unusedIntent, dmIntent), + confirmationDigest = settlementRecordDigest(domains.unusedConfirmation, dmConfirmation), + actualDescriptors = (capacity ? descriptors : [descriptors[0]!]) + .map((value) => controlDigest(value)) + .sort(), + safeState = { + executionOrdinal: capacity?.reservedExecutionOrdinal ?? null, + intentOrdinal: capacity?.reservedIntentOrdinal ?? null, + recordDescriptorsDigest: controlDigest(descriptors), + allocatedDescriptorDigests: actualDescriptors, + state: "UNUSED", + intentDigest, + }; + const receiverWitness = readControlExecutionWitness(f.state, grant.child, receiverKey); + need( + receiverWitness.kind === "configuration-maintenance.prepare" && + same(receiverWitness.origin, f.meta.origin) && + receiverWitness.authorityDigest === refusal.prepareAdmissionDigest && + (!ownAdoption || ownAdoption.delegationId === delegationId) && + grant.ordinal === projection.reservation.ordinal - 1 && + grant.child.operationKey === + controlChildOperationKey( + projection.ownerPrepare.operationKey, + "configuration-fence", + grant.ordinal, + ) && + same(grant.serviceIds, projection.reservation.serviceIds) && + same(grant.serviceIds, certificate.serviceIds) && + grant.installationId === identity.installationId && + same(grant.origin, f.meta.origin) && + grant.leaseId === leaseId && + grant.ownerCertificateDigest === identity.rootCertificateDigest && + grant.delegationId === delegationId && + grant.childPublicKey === f.state.publicKeys["data-manager"] && + same(grant.parent, projection.ownerPrepare) && + grantDigest === controlDigest(rawGrant.grant) && + grantDigest === freeze.prepareGrantDigest && + transition.delegationId === delegationId && + transition.grantDigest === grantDigest && + same(child.origin, f.meta.origin) && + child.delegationId === delegationId && + child.grantDigest === grantDigest && + same(child.execution, grant.child) && + same(child.settlement.identity, grant.child) && + Object.keys(child.grantDigests).length === 0 && + refusal.installationId === identity.installationId && + same(refusal.origin, f.meta.origin) && + same(refusal.ownerPrepare, projection.ownerPrepare) && + refusal.rootAdmissionDigest === identity.admissionDigest && + refusal.rootCertificateDigest === identity.rootCertificateDigest && + refusal.maintenanceId === identity.maintenanceId && + refusal.candidateId === identity.candidateId && + refusal.planDigest === identity.planDigest && + refusal.settlementId === identity.settlementId && + refusal.projectionDigest === projectionDigest && + refusal.reservationDigest === projection.reservationDigest && + same(refusal.prepareExecution, grant.child) && + refusal.refusalReceipt.maintenanceId === identity.maintenanceId && + refusal.effectReceiptDigest === controlDigest(refusal.refusalReceipt) && + child.settlement.terminalReceiptDigest === refusal.effectReceiptDigest && + refusal.effectSettlementDigest === + controlDigest({ ...child.settlement, receipt: refusal.refusalReceipt }) && + refusal.canonicalTerminalDigest === + controlDigest({ version: 1, refused: refusal.refusalReceipt }) && + evidence.prepareExecutionDigest === controlDigest(grant.child) && + evidence.prepareAdmissionDigest === refusal.prepareAdmissionDigest && + evidence.refusalProjectionDigest === refusalDigest && + evidence.effectSettlementDigest === refusal.effectSettlementDigest && + evidence.canonicalTerminalDigest === refusal.canonicalTerminalDigest && + evidence.canonicalTerminalAt === refusal.canonicalTerminalAt && + evidence.capacityReceiptDigest === refusal.capacityReceiptDigest && + freeze.prepareExecutionDigest === evidence.prepareExecutionDigest && + freeze.refusalProjectionDigest === refusalDigest && + evidence.dmUnusedIntentDigest === intentDigest && + evidence.dmUnusedConfirmationDigest === confirmationDigest && + dmIntent.role === "data-manager" && + dmIntent.installationId === identity.installationId && + same(dmIntent.origin, f.meta.origin) && + dmIntent.ownerPrepareDigest === controlDigest(projection.ownerPrepare) && + dmIntent.prepareExecutionDigest === controlDigest(grant.child) && + dmIntent.settlementId === identity.settlementId && + dmIntent.projectionDigest === projectionDigest && + dmIntent.capacityReceiptDigest === refusal.capacityReceiptDigest && + dmIntent.sourceProofDigest === refusalDigest && + dmIntent.executionOrdinal === safeState.executionOrdinal && + dmIntent.intentOrdinal === safeState.intentOrdinal && + dmIntent.recordDescriptorsDigest === safeState.recordDescriptorsDigest && + same(dmIntent.allocatedDescriptorDigests, actualDescriptors) && + dmConfirmation.role === "data-manager" && + dmConfirmation.installationId === identity.installationId && + same(dmConfirmation.origin, f.meta.origin) && + dmConfirmation.ownerPrepareDigest === dmIntent.ownerPrepareDigest && + dmConfirmation.settlementId === identity.settlementId && + dmConfirmation.projectionDigest === projectionDigest && + dmConfirmation.intentDigest === intentDigest && + dmConfirmation.slotStateDigest === controlDigest(safeState) && + dmConfirmation.confirmedAt >= dmIntent.createdAt, + "Original receiver refusal/UNUSED family preimages changed", + ); + if (capacity) + need( + capacity.installationId === identity.installationId && + same(capacity.origin, f.meta.origin) && + capacity.ownerPrepareDigest === dmIntent.ownerPrepareDigest && + capacity.settlementId === identity.settlementId && + capacity.projectionDigest === projectionDigest && + capacity.reservationDigest === projection.reservationDigest && + same(capacity.proposedExecutionAddress, projection.reservation.execution) && + capacity.recordDescriptorsDigest === controlDigest(descriptors) && + capacity.maxBytes === descriptors.reduce((sum, value) => sum + value.cap, 0) && + refusal.capacityReceiptDigest === settlementRecordDigest(domains.capacity, capacity), + "Original receiver capacity family source changed", + ); + else + need( + refusal.capacityReceiptDigest === null && refusal.refusalReceipt.reason === "capacity", + "Actual capacity-null family source required", + ); + dmSlotProof = { role: "data-manager", intentDigest, confirmationDigest }; + dmContainmentDigest = refusal.effectSettlementDigest; + } + const descriptors = configurationSettlementRecordDescriptors(projection, prepareKey), + senderSlot = { + executionOrdinal: issuerIntent.executionOrdinal, + intentOrdinal: issuerIntent.intentOrdinal, + recordDescriptorsDigest: controlDigest(descriptors.sender), + allocatedDescriptorDigests: descriptors.sender.map((value) => controlDigest(value)).sort(), + state: "UNUSED", + intentDigest: issuerIntentDigest, + }; + need( + issuerIntent.role === "ops-agent" && + issuerIntent.installationId === identity.installationId && + same(issuerIntent.origin, f.meta.origin) && + issuerIntent.ownerPrepareDigest === controlDigest(projection.ownerPrepare) && + issuerIntent.settlementId === identity.settlementId && + issuerIntent.projectionDigest === projectionDigest && + issuerIntent.prepareExecutionDigest === + (evidence.kind === "receiver_refused" ? evidence.prepareExecutionDigest : null) && + issuerIntent.capacityReceiptDigest === + (evidence.kind === "receiver_refused" ? evidence.capacityReceiptDigest : null) && + issuerIntent.sourceProofDigest === configurationIssuerUnusedSourceDigest(evidence) && + issuerIntent.executionOrdinal === projection.issuerMembership.rootMemberOrdinal && + issuerIntent.intentOrdinal !== null && + issuerIntent.recordDescriptorsDigest === senderSlot.recordDescriptorsDigest && + same(issuerIntent.allocatedDescriptorDigests, senderSlot.allocatedDescriptorDigests) && + issuerConfirmation.role === "ops-agent" && + issuerConfirmation.installationId === identity.installationId && + same(issuerConfirmation.origin, f.meta.origin) && + issuerConfirmation.ownerPrepareDigest === issuerIntent.ownerPrepareDigest && + issuerConfirmation.settlementId === identity.settlementId && + issuerConfirmation.projectionDigest === projectionDigest && + issuerConfirmation.intentDigest === issuerIntentDigest && + issuerConfirmation.slotStateDigest === controlDigest(senderSlot) && + issuerConfirmation.confirmedAt >= issuerIntent.createdAt && + same( + closure.originalContainmentDigests, + [ownContainmentDigest, ...(dmContainmentDigest ? [dmContainmentDigest] : [])].sort(), + ) && + same(closure.unusedSlotDigests, [ + ...(dmSlotProof ? [dmSlotProof] : []), + { + role: "ops-agent", + intentDigest: issuerIntentDigest, + confirmationDigest: issuerConfirmationDigest, + }, + ]) && + same(control.unused, { + bundleDigest: settlementRecordDigest(domains.bundle, bundle), + closureDigest: closure.closureDigest, + issuerFreezeProofDigest: evidence.issuerFreezeProofDigest, + issuerUnusedIntentDigest: issuerIntentDigest, + issuerUnusedConfirmationDigest: issuerConfirmationDigest, + }), + "Original family issuer UNUSED/bundle linkage changed", + ); + const unusedExecutionTicketIds: string[] = [], + unusedIntentTicketIds: string[] = []; + const verifyUnusedTicket = ( + role: "ops-agent" | "data-manager", + kind: "executions" | "intents", + ordinal: number | null, + intentDigest: string, + ) => { + const id = controlDigest({ + purpose: "configurationSettlementSlot", + role, + settlementId: identity.settlementId, + member: kind === "executions" ? "execution" : "intent", + }), + directory = own(f.shared.directory(kind)); + if (ordinal === null) { + for ( + let index = 0; + index < (kind === "executions" ? LIMITS.executionsPerFamily : LIMITS.intentsPerFamily); + index++ + ) { + try { + need( + json<{ id: string }>(directory, `slot-${index}.json`, 4096).id !== id, + "Unexpected allocated NoGuard null slot", + ); + } catch (error) { + if (!missing(error)) throw error; + } + } + return; + } + const expected = { + id, + generation: f.meta.generation, + value: { + purpose: "configurationSettlementSlot", + role, + settlementId: identity.settlementId, + projectionDigest, + address: projection.reservation.execution, + executionDigest: null, + controlBindingDigest: null, + state: "UNUSED", + unusedIntentDigest: intentDigest, + }, + }; + need( + same(json(directory, `slot-${ordinal}.json`, 4096), expected), + "Original NoGuard shared UNUSED ticket changed", + ); + (kind === "executions" ? unusedExecutionTicketIds : unusedIntentTicketIds).push(id); + }; + verifyUnusedTicket("ops-agent", "intents", issuerIntent.intentOrdinal, issuerIntentDigest); + if (dmIntent && dmSlotProof) { + verifyUnusedTicket( + "data-manager", + "executions", + dmIntent.executionOrdinal, + dmSlotProof.intentDigest, + ); + verifyUnusedTicket( + "data-manager", + "intents", + dmIntent.intentOrdinal, + dmSlotProof.intentDigest, + ); + } + const localIntent = f.state.installation.role === "ops-agent" ? issuerIntent : dmIntent, + localConfirmation = + f.state.installation.role === "ops-agent" ? issuerConfirmation : dmConfirmation, + localDescriptors = + f.state.installation.role === "ops-agent" ? descriptors.sender : descriptors.receiver; + need( + localIntent && + localConfirmation && + slot.role === f.state.installation.role && + slot.projectionDigest === projectionDigest && + slot.settlementId === identity.settlementId && + slot.reservationDigest === projection.reservationDigest && + slot.ownerPrepareDigest === controlDigest(projection.ownerPrepare) && + same(slot.proposedExecutionAddress, projection.reservation.execution) && + same(slot.descriptors, localDescriptors) && + slot.recordDescriptorsDigest === localIntent!.recordDescriptorsDigest && + slot.executionOrdinal === localIntent!.executionOrdinal && + slot.intentOrdinal === localIntent!.intentOrdinal && + same(slot.allocatedDescriptorDigests, localIntent!.allocatedDescriptorDigests) && + slot.unusedIntentDigest === settlementRecordDigest(domains.unusedIntent, localIntent!) && + slot.unusedConfirmedAt === localConfirmation!.confirmedAt, + "Actual role-private NoGuard slot differs from its published source", + ); + settlementSlotTickets(f, slot, false); + for (const descriptor of localDescriptors.filter( + (value) => + slot.allocatedDescriptorDigests.includes(controlDigest(value)) && + (f.state.installation.role === "data-manager" + ? value !== descriptors.receiver[0] + : value.scope === "private" || value.name.endsWith("-ack.json")), + )) + need( + localRead( + { ...descriptor, container: descriptor.scope === "shared" ? descriptor.role : "" }, + descriptor.cap, + ) === null, + "Attempted future control cannot retire as UNUSED", + ); + if (f.state.installation.role === "ops-agent") { + const owner = verifyControl( + senderKey, + "openmapx-configuration-maintenance-owner-v2", + localRead( + { + scope: "private", + namespace: "configuration-maintenance", + container: "", + name: `${identity.maintenanceId}.json`, + }, + 131072, + ), + z + .object({ + admissionDigest: z.string(), + sequence: z.number(), + issuance: z.literal("closed_without_guard"), + guardSource: z.null(), + final: z.null(), + inspection: z.null(), + noGuard: configurationNoGuardIntentSchema, + receipt: z + .object({ dmFence: z.null(), generation: z.literal("not_started") }) + .passthrough(), + }) + .passthrough(), + ); + need( + owner.admissionDigest === identity.admissionDigest && + owner.sequence === closure.ownerSequence && + same(owner.noGuard.bundle?.payload, bundle) && + same(owner.noGuard.issuerFreezeProof, bundle.issuerFreezeProof) && + same(owner.noGuard.issuerUnused, bundle.issuerUnused), + "Actual original issuer private closure changed", + ); + } else { + const fact = Object.values(p.executions).find( + (value) => value.execution.operationKey === prepareKey, + ); + need( + fact && + evidence.kind === "receiver_refused" && + fact.receiptDigest === evidence.prepareAdmissionDigest && + fact.canonicalDigest === evidence.canonicalTerminalDigest && + fact.canonicalTerminalAt === evidence.canonicalTerminalAt && + fact.settlementDigest === evidence.effectSettlementDigest && + fact.known && + fact.journalContained, + "Actual DM original canonical refusal family facts required", + ); + need( + fact!.canonicalDigest === fact!.terminalDigest && + f.state.retentionJournal?.contained(fact!.execution, fact!.terminalDigest), + "Actual original DM refusal journal containment unavailable", + ); + } + return { unusedExecutionTicketIds, unusedIntentTicketIds }; + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +function identityOfRoot(f: Family) { + return identity(f.root.stat); +} function localProof(f: Family, p: Progress): boolean { + // A reserved logical control is a separate mandatory release/retirement + // obligation. Ordinary terminal facts cannot stand in for its closure chain. + if ( + p.configurationSettlementSlot && + !preparedClosure(f, p) && + !configurationNoGuardFamilyClosure(f, p) + ) + return false; if ( !Object.keys(p.executions).length || Object.values(p.executions).some((e) => !e.known || e.terminalAt === undefined) @@ -1476,10 +4307,16 @@ function publishAck(f: Family, p: Progress, m: Manifest, phase: Ack["phase"]) { p.phase = phase; progress(f, p); } -function publishCandidate(f: Family, p: Progress): Candidate | null { +function publishCandidate( + f: Family, + p: Progress, + staged?: StagedCoordinatorRetirementPacket, +): Candidate | null { + const prepared = preparedClosure(f, p), + terminalAt = Math.max(p.terminalAt, prepared?.terminalAt ?? 0); if ( Date.now() < f.meta.origin.notAfter || - Date.now() - p.terminalAt < LIMITS.minimumTerminalRetentionMs || + Date.now() - terminalAt < LIMITS.minimumTerminalRetentionMs || !localProof(f, p) ) return null; @@ -1533,14 +4370,22 @@ function publishCandidate(f: Family, p: Progress): Candidate | null { generation: f.meta.generation, role: f.state.installation.role, originDigest: f.originHash, - terminalAt: p.terminalAt, - terminalSequenceDigest: controlDigest( - Object.values(p.executions).map((e) => ({ + terminalAt, + terminalSequenceDigest: controlDigest([ + ...Object.values(p.executions).map((e) => ({ execution: e.execution, terminalAt: e.terminalAt, digest: e.terminalDigest ?? e.receiptDigest, })), - ), + ...(prepared + ? [ + { + logicalControlTerminalDigest: prepared.terminalDigest, + terminalAt: prepared.terminalAt, + }, + ] + : []), + ]), executionReceiptDigests, leaseDigests: Object.fromEntries( Object.values(p.leases).map((l) => [l.id, l.certificateDigest]), @@ -1548,6 +4393,7 @@ function publishCandidate(f: Family, p: Progress): Candidate | null { intentDigests: Object.fromEntries(Object.values(p.intents).map((i) => [i.id, i.childDigest])), localRecordSetDigest: recordSetDigest(records(f)), sharedRecordSetDigest: recordSetDigest(readSharedRecords(f, f.state.installation.role)), + ...(staged ? { stagedCoordinatorDependency: staged } : {}), }; cell( f.shared, @@ -1588,8 +4434,24 @@ function prepare(f: Family, p: Progress): boolean { {}, ...candidates.map((c) => c.executionReceiptDigests), ) as Record; - const executions = readAllTickets(f, "executions", LIMITS.executionsPerFamily); - const intents = readAllTickets(f, "intents", LIMITS.intentsPerFamily); + // Only the FULL source-verified disjoint closure can account an unbound + // slot without inventing an execution/terminal receipt. Never filter by a + // caller flag or a ticket's purpose/state in isolation. + const noGuard = p.configurationSettlementSlot && configurationNoGuardFamilyClosure(f, p); + const prepared = preparedClosure(f, p); + if (p.configurationSettlementSlot && !noGuard && !prepared) return false; + const excludedExecutions = noGuard + ? noGuard.unusedExecutionTicketIds + : (prepared?.logicalExecutionTicketIds ?? []), + excludedIntents = noGuard + ? noGuard.unusedIntentTicketIds + : (prepared?.logicalIntentTicketIds ?? []); + const executions = readAllTickets(f, "executions", LIMITS.executionsPerFamily).filter( + (value) => !excludedExecutions.includes(value.id), + ); + const intents = readAllTickets(f, "intents", LIMITS.intentsPerFamily).filter( + (value) => !excludedIntents.includes(value.id), + ); if ( Object.keys(receipts).length !== executions.length || executions.some((t) => !receipts[t.id]) || @@ -1652,6 +4514,13 @@ function prepare(f: Family, p: Progress): boolean { localRecordSetDigests: Object.fromEntries( candidates.map((c) => [c.role, c.localRecordSetDigest]), ), + ...(candidates.find((c) => c.role === "ops-agent")?.stagedCoordinatorDependency + ? { + stagedCoordinatorDependencyDigest: controlDigest( + candidates.find((c) => c.role === "ops-agent")!.stagedCoordinatorDependency, + ), + } + : {}), }; advanceRetentionClock(f.state, f.meta.origin.notAfter); const envelope = signed(f.state, "openmapx-retention-manifest-v1", m); @@ -2070,15 +4939,59 @@ function assertPrivateRetirementProof(f: Family, m: Manifest): void { ) ) throw Error("Private terminal proof copy changed"); + if (proof.role === "ops-agent") { + if (proof.stagedCoordinatorDependency) { + const checked = verifiedStagedRetirementPacket( + f, + readProgress(f), + proof.stagedCoordinatorDependency, + ); + if ( + checked.value.contextMode !== "preparing" || + checked.value.manifestDigest !== null || + controlDigest(checked.packet) !== m.stagedCoordinatorDependencyDigest + ) + throw Error("Staged candidate/manifest conflict"); + } else if (m.stagedCoordinatorDependencyDigest) + throw Error("Original staged candidate required"); + } } async function step(f: Family): Promise { if (f.state.fixedChildId) throw Error("Fixed children do not coordinate role-private retirement"); let p = readProgress(f); - await f.state.retentionJournal?.prepare?.( - Object.values(p.executions) - .filter((fact) => fact.requiresJournal) - .map((fact) => ({ execution: fact.execution, digest: fact.terminalDigest })), - ); + const initialDecision = + p.phase === "released" && p.sharedRelease + ? verified( + f.state, + f.meta.origin.issuer, + "openmapx-retention-decision-v1", + json(f.slot, "final.json", 8192), + ) + : decision(f); + let staged: StagedCoordinatorRetirementPacket | undefined; + if (initialDecision.phase === "committed" || initialDecision.phase === "final") { + const committedManifest = + p.phase === "released" && p.sharedRelease + ? verified( + f.state, + f.meta.origin.issuer, + "openmapx-retention-manifest-v1", + json(f.slot, "manifest.json", 32768), + ) + : manifest(f); + await prepareJournalPhase(f, p, "committed-cleanup", committedManifest); + } else if (p.phase === "active" || p.phase === "prepared") + staged = await prepareJournalPhase(f, p, "preparing"); + if ( + ["active", "prepared"].includes(p.phase) && + p.configurationSettlementSlot?.state === "bound" + ) { + const observed = readPreparedConfigurationSettlementRetirement( + f.state.installation, + f.meta.origin, + ); + preparedClosures.set(f, { ...observed, localRecordSetDigest: recordSetDigest(records(f)) }); + } checkLocalIdentity(f); if (p.phase === "released" && f.state.installation.role !== f.meta.origin.issuer) { freeFamily(f, p); @@ -2097,7 +5010,11 @@ async function step(f: Family): Promise { } const d = decision(f); if (d.phase === "active") { - guard(f.slot, ".family-writer", () => publishCandidate(f, p)); + guard(f.slot, ".family-writer", () => { + if (readProgress(f).sequence !== p.sequence) + throw Error("Collector preparation sequence changed"); + return publishCandidate(f, p, staged); + }); if (!prepare(f, p)) return false; p = readProgress(f); } @@ -2129,6 +5046,12 @@ async function step(f: Family): Promise { if (!validateAck(f, m, a, "ready")) return false; readyDigests[role] = controlDigest(a); } + const revalidation = await prepareJournalPhase(f, p, "before-commit", m); + if (m.stagedCoordinatorDependencyDigest) { + if (!revalidation) throw Error("Fresh staged retirement revalidation required"); + p.stagedCoordinatorRevalidation = revalidation; + guard(f.slot, ".family-writer", () => progress(f, p)); + } else if (revalidation) throw Error("Unbound staged retirement revalidation"); writeDecision(f, { version: 1, familyId: f.meta.familyId, @@ -2137,12 +5060,14 @@ async function step(f: Family): Promise { manifestDigest: md, readyDigests, sequence: 2, + ...(revalidation ? { stagedCoordinatorRevalidationDigest: controlDigest(revalidation) } : {}), }); } if (decision(f).phase === "prepared") return false; if (p.phase === "ready" || p.phase === "committed") { const commit = decision(f); if (commit.phase !== "committed" && commit.phase !== "final") return false; + assertStagedCommittedCleanup(f, p, m, commit); cell(f.slot, "commit.json", json(f.shared, "decision.json", 8192), 8192); advanceRetentionClock(f.state, f.meta.origin.notAfter); p.phase = "committed"; @@ -2344,6 +5269,50 @@ export function retentionApiDependency( f.close(); } } +/** Fixed staged-provenance reader. The role-signed API copy and complete + * family membership must still be active; this grants no forward authority. */ +export function readActiveStagedCoordinatorApiCopy( + state: InstallationState, + origin: ControlExecutionOrigin, + execution: ExecutionRef, + terminalDigest: string, + parent: RetainedParentExecution, + context?: ControlRetentionPreparationContext, +) { + const f = openFamily(state, origin); + try { + const p = readProgress(f), + d = decision(f), + fact = p.executions[controlDigest(execution)], + copy = apiCopy(f, p); + const preparing = sourcePreparationContextAllows(context, state, origin, execution, p.sequence); + if ( + (!preparing && (p.phase !== "active" || d.phase !== "active")) || + (preparing && + (!["active", "prepared", "ready"].includes(p.phase) || + !["active", "prepared"].includes(d.phase))) || + !fact?.known || + canonicalJson(fact.execution) !== canonicalJson(execution) || + fact.terminalDigest !== terminalDigest || + fact.canonicalDigest !== terminalDigest || + !fact.journalContained || + !copy || + copy.terminalDigest !== terminalDigest || + copy.operationKey !== execution.operationKey || + copy.fingerprint !== execution.fingerprint || + canonicalJson(p.api?.parent) !== canonicalJson(parent) + ) + throw Error("Actual active staged coordinator API copy required"); + return { + origin: f.meta.origin, + familyId: f.meta.familyId, + generation: f.meta.generation, + apiCopyDigest: controlDigest(json(f.shared, "api-ack.json", 4096)), + }; + } finally { + f.close(); + } +} export function acknowledgeRetentionDependency( state: InstallationState, origin: ControlExecutionOrigin, @@ -2490,3 +5459,599 @@ export function hasRetentionEvidence( for (const dir of opened.reverse()) dir.close(); } } + +// Fixed reserved cells and their retirement read the SAME guarded family index. +function directory( + state: InstallationState, + descriptor: ConfigurationSettlementRecordDescriptor, + create: boolean, +) { + if ( + descriptor.role !== state.installation.role || + state.fixedChildId || + descriptor.namespace !== (descriptor.scope === "private" ? "observations" : "execution") + ) + throw Error("Fixed role-local settlement descriptor required"); + const opened: AnchoredDirectory[] = [], + own = (dir: AnchoredDirectory) => { + opened.push(dir); + return dir; + }, + close = () => { + for (const dir of opened.reverse()) dir.close(); + }; + try { + if (descriptor.scope === "private") + return { dir: own(state.privateRoot.directory("observations", create, true)), close }; + const root = own(new AnchoredDirectory(state.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions", create)), + dir = own(executions.directory(descriptor.role, create)); + return { dir, close }; + } catch (error) { + close(); + throw error; + } +} +export function readConfigurationSettlementCell( + state: InstallationState, + projection: Pick, + descriptor: ConfigurationSettlementRecordDescriptor, +): unknown { + const opened = directory(state, descriptor, false), + dir = opened.dir; + try { + const bytes = dir.read(descriptor.name, descriptor.cap).toString("utf8"); + if (Buffer.byteLength(bytes) !== descriptor.cap) + throw Error("Settlement cell reserved width changed"); + verifyFamilyOwnedRecord( + state, + projection.origin, + { ...descriptor, container: descriptor.scope === "shared" ? descriptor.role : "" }, + bytes, + ); + return JSON.parse(bytes); + } finally { + opened.close(); + } +} +/** Allocation is only called by original PREPARE-time reservation. Updates + * require the existing indexed padded cell; no late namespace/index allocation. */ +export function writeConfigurationSettlementCell( + state: InstallationState, + projection: ConfigurationSettlementProjection, + descriptor: ConfigurationSettlementRecordDescriptor, + value: unknown, + allocate = false, +): void { + const opened = directory(state, descriptor, allocate), + dir = opened.dir; + try { + if (!allocate) readConfigurationSettlementCell(state, projection, descriptor); + const encoded = canonicalJson(value), + size = Buffer.byteLength(encoded); + if (size > descriptor.cap) throw Error("Settlement packet exceeds original reserved cell"); + const bytes = encoded + " ".repeat(descriptor.cap - size); + familyRecordWrite( + state, + projection.origin, + { ...descriptor, container: descriptor.scope === "shared" ? descriptor.role : "" }, + bytes, + (beforePublish) => + descriptor.scope === "private" + ? writeRetainedRecord( + dir, + descriptor.name, + bytes, + allocate, + CONTROL_RETENTION_LIMITS.recordsPerNamespace, + beforePublish, + descriptor.cap, + ) + : dir.write(descriptor.name, bytes, allocate, beforePublish), + allocate, + ); + } finally { + opened.close(); + } +} +export const configurationSettlementDescriptorDigest = ( + descriptors: readonly ConfigurationSettlementRecordDescriptor[], +) => controlDigest(descriptors); +/** A genuinely unjoined receiver has no role directory yet. Only ENOENT is + * absence; unreadable/replaced directories and any original publication deny. + * Callers still need the authentic frozen all-unissued source to use this fact. */ +export function assertNoUnissuedConfigurationReceiverPublication( + state: InstallationState, + prepareKey: string, +): void { + const opened: AnchoredDirectory[] = [], + own = (value: AnchoredDirectory) => { + opened.push(value); + return value; + }; + try { + const root = own(new AnchoredDirectory(state.dataRoot)), + control = own(root.directory(".admin-target-control")), + executions = own(control.directory("executions")); + let receiver: AnchoredDirectory; + try { + receiver = own(executions.directory("data-manager")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + for (const name of [ + `${prepareKey}.configuration-settlement-capacity.json`, + `${prepareKey}.data-terminal.json`, + `${prepareKey}.json`, + ]) { + try { + receiver.read(name, 65536); + throw Error("NeverIssued has an actual original DM publication"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} + +const same = (a: unknown, b: unknown) => canonicalJson(a) === canonicalJson(b); +function need(value: unknown, message: string): asserts value { + if (!value) throw Error(message); +} +export function readPreparedConfigurationSettlementRetirement( + installation: ControlInstallation, + origin: ControlExecutionOrigin, +) { + const local = installationState(installation), + ownRole = installation.role; + need( + !local.fixedChildId && + ["ops-agent", "data-manager"].includes(ownRole) && + origin.issuer === "ops-agent", + "Original prepared retirement role required", + ); + const root = new AnchoredDirectory(local.dataRoot), + opened: AnchoredDirectory[] = [root], + own = (dir: AnchoredDirectory) => { + opened.push(dir); + return dir; + }; + try { + const sharedRoot = own(root.directory(".admin-target-control")), + executions = own(sharedRoot.directory("executions")), + directories = { + "ops-agent": own(executions.directory("ops-agent")), + "data-manager": own(executions.directory("data-manager")), + }, + shared = (role: "ops-agent" | "data-manager", name: string, cap: number, padded = true) => { + const bytes = directories[role].read(name, cap).toString("utf8"); + need(!padded || Buffer.byteLength(bytes) === cap, "Prepared retirement cell width changed"); + if (role === ownRole) + verifyFamilyOwnedRecord( + local, + origin, + { scope: "shared", namespace: "execution", container: role, name }, + bytes, + ); + return JSON.parse(bytes); + }, + senderKey = local.publicKeys["ops-agent"], + dmKey = local.publicKeys["data-manager"], + control = verifyControl( + senderKey, + domains.control, + shared("ops-agent", `${origin.rootKey}.configuration-settlement-control.json`, 8192), + configurationSettlementControlCellSchema.shape.payload, + ); + need(control.binding && !("unused" in control), "Original prepared bound control required"); + const projection = verifyControl( + senderKey, + domains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + committed = verifyControl( + senderKey, + domains.reference, + shared("ops-agent", `${origin.rootKey}.configuration-settlement.json`, 16384), + configurationCommittedSettlementSchema, + ), + envelope = verifyControl( + senderKey, + domains.control, + control.binding, + configurationSettlementControlEnvelopeSchema, + ), + projectionDigest = settlementRecordDigest(domains.projection, projection), + commitmentDigest = settlementRecordDigest(domains.commitment, committed), + prepareKey = projection.ownerPrepare.operationKey, + prepareOrdinal = projection.reservation.ordinal - 1, + identity = committed.identity; + // The actual prepared canonical witness supplies the original DM execution; + // derive its fixed phase key, never scan or accept a caller lookup address. + const dmPrepareKey = controlChildOperationKey( + prepareKey, + "configuration-fence", + prepareOrdinal, + ), + prepared = verifyControl( + dmKey, + dataConfigurationFencePreparedWitnessDomain, + shared("data-manager", `${dmPrepareKey}.data-terminal.json`, 8192, false), + dataConfigurationFencePreparedWitnessSchema, + ), + descriptors = configurationSettlementRecordDescriptors(projection, dmPrepareKey), + capacityCell = configurationDmCapacityCellSchema.parse( + shared("data-manager", descriptors.receiver[0]!.name, descriptors.receiver[0]!.cap), + ); + need( + capacityCell.capacity && !capacityCell.refusal && !capacityCell.unused, + "Actual prepared receiver capacity required", + ); + const capacity = verifyControl( + dmKey, + domains.capacity, + capacityCell.capacity, + configurationSettlementCapacitySchema, + ), + signedTerminal = shared( + "data-manager", + descriptors.receiver[4]!.name, + descriptors.receiver[4]!.cap, + ), + terminal = verifyControl( + dmKey, + domains.terminal, + signedTerminal, + configurationSettlementPublishedTerminalSchema, + ), + tuple = terminal.tuple, + signedAck = shared("ops-agent", descriptors.sender[5]!.name, descriptors.sender[5]!.cap), + ack = verifyControl(senderKey, domains.ack, signedAck, configurationSettlementAckSchema), + { + operationFingerprint: _body, + fingerprint: _execution, + deadlineAt: _deadline, + ...address + } = envelope.execution, + { terminalDigest: _terminalDigest, ...tuplePreimage } = tuple; + need( + projection.installationId === installation.installationId && + same(projection.origin, origin) && + prepareKey === origin.rootKey && + projection.settlementId === + deriveConfigurationFenceSettlementId({ + installationId: installation.installationId, + ownerPrepare: projection.ownerPrepare, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + }) && + projection.reservationDigest === + settlementRecordDigest(domains.reservationTuple, projection.reservation) && + projection.reservation.operationFingerprint === controlDigest(projection.operation) && + same(projection.operation, { + kind: "configuration-maintenance.settle", + version: 2, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + settlementId: projection.settlementId, + }) && + same(identity, { + installationId: installation.installationId, + origin, + ownerPrepare: projection.ownerPrepare, + admissionDigest: projection.rootAdmissionDigest, + rootCertificateDigest: projection.rootCertificateDigest, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + settlementId: projection.settlementId, + }) && + same(committed.final.identity, identity) && + same(committed.evidence.identity, identity) && + committed.referenceOperationFingerprint === projection.reservation.operationFingerprint && + committed.ownerReceiptDigest === + settlementRecordDigest(domains.ownerFinal, committed.final) && + committed.evidenceDigest === + settlementRecordDigest(domains.finalEvidence, committed.evidence) && + committed.final.evidenceDigest === committed.evidenceDigest && + committed.final.outcome === committed.evidence.outcome && + committed.committedAt === committed.final.settlingAt && + envelope.installationId === installation.installationId && + same(envelope.origin, origin) && + same(envelope.ownerPrepare, projection.ownerPrepare) && + envelope.admissionDigest === projection.rootAdmissionDigest && + envelope.rootCertificateDigest === projection.rootCertificateDigest && + envelope.settlementId === projection.settlementId && + envelope.projectionDigest === projectionDigest && + envelope.reservationDigest === projection.reservationDigest && + envelope.commitmentDigest === commitmentDigest && + same(envelope.operation, projection.operation) && + same(envelope.serviceIds, projection.reservation.serviceIds) && + same(address, projection.reservation.execution) && + envelope.execution.operationFingerprint === projection.reservation.operationFingerprint && + envelope.execution.fingerprint === + executionFingerprint({ + kind: projection.operation.kind, + operationFingerprint: projection.reservation.operationFingerprint, + deadlineAt: envelope.controlDeadlineAt, + parent: { + issuer: "ops-agent", + operationKey: prepareKey, + fingerprint: projection.ownerPrepare.fingerprint, + deadlineAt: projection.ownerPrepare.deadlineAt, + execution: projection.ownerPrepare, + }, + }), + "Prepared retirement root/reference preimages changed", + ); + const rootWitness = readControlExecutionWitness(local, projection.ownerPrepare, senderKey), + slot = readConfigurationSettlementFamilySlot(local, projection); + need( + rootWitness.authorityDigest === projection.rootAdmissionDigest && + same(rootWitness.origin, origin) && + capacity.installationId === installation.installationId && + same(capacity.origin, origin) && + capacity.ownerPrepareDigest === controlDigest(projection.ownerPrepare) && + capacity.settlementId === projection.settlementId && + capacity.projectionDigest === projectionDigest && + capacity.reservationDigest === projection.reservationDigest && + same(capacity.proposedExecutionAddress, projection.reservation.execution) && + capacity.recordDescriptorsDigest === controlDigest(descriptors.receiver) && + capacity.maxBytes === descriptors.receiver.reduce((sum, value) => sum + value.cap, 0) && + slot.state === "bound" && + slot.executionDigest === controlDigest(envelope.execution) && + slot.controlBindingDigest === settlementRecordDigest(domains.binding, envelope) && + slot.projectionDigest === projectionDigest && + slot.reservationDigest === projection.reservationDigest && + same(prepared.preparedReceipt, committed.evidence.guard.preparedReceipt) && + prepared.preparedReceiptDigest === committed.evidence.guard.preparedReceiptDigest && + prepared.preparedCanonicalDigest === committed.evidence.guard.preparedCanonicalDigest && + prepared.preparedTerminalAt === committed.evidence.guard.preparedTerminalAt && + same(prepared.preparedReceipt.ownerPrepare, projection.ownerPrepare) && + prepared.preparedReceipt.maintenanceId === projection.maintenanceId && + prepared.preparedReceipt.candidateId === projection.candidateId && + prepared.preparedReceipt.planDigest === projection.planDigest && + prepared.prepareExecution.operationKey === dmPrepareKey && + committed.final.expectedDmSequence === prepared.preparedReceipt.sequence, + "Actual prepared retirement capacity/guard changed", + ); + const admissionPayload = configurationSettlementAdmissionSchema.parse({ + version: 2, + purpose: "configuration-fence-settlement", + installationId: installation.installationId, + origin, + settlementId: projection.settlementId, + projectionDigest, + reservationDigest: projection.reservationDigest, + prepareExecutionDigest: controlDigest(prepared.prepareExecution), + prepareAdmissionDigest: prepared.prepareAdmissionDigest, + preparedReceiptDigest: prepared.preparedReceiptDigest, + capacityReceiptDigest: settlementRecordDigest(domains.capacity, capacity), + rootCertificateDigest: projection.rootCertificateDigest, + controlEnvelope: control.binding, + commitmentDigest, + executionOrdinal: capacity.reservedExecutionOrdinal, + intentOrdinal: capacity.reservedIntentOrdinal, + sqlSessionNonce: envelope.execution.executionId, + boundAt: terminal.admissionSource.boundAt, + }), + signedAdmission = { + payload: admissionPayload, + signature: terminal.admissionSource.signature, + }, + admission = verifyControl( + dmKey, + domains.admission, + signedAdmission, + configurationSettlementAdmissionSchema, + ), + signedEffect = { + payload: { ...terminal.effectSource.fields, terminal: tuple }, + signature: terminal.effectSource.signature, + }, + effect = verifyControl( + dmKey, + domains.effect, + signedEffect, + configurationSettlementEffectSchema, + ); + need( + same(terminal.origin, origin) && + same(tuple.execution, envelope.execution) && + tuple.settlementId === projection.settlementId && + tuple.commitmentDigest === commitmentDigest && + tuple.fenceId === prepared.preparedReceipt.fenceId && + same(tuple.result, { + ...prepared.preparedReceipt, + state: "settled", + sequence: prepared.preparedReceipt.sequence + 1, + }) && + tuple.terminalDigest === settlementRecordDigest(domains.terminal, tuplePreimage) && + tuple.terminalAt >= envelope.controlStartedAt && + tuple.terminalAt < envelope.controlDeadlineAt && + terminal.admissionDigest === settlementRecordDigest(domains.admission, admission) && + admission.boundAt >= envelope.controlStartedAt && + admission.boundAt < envelope.controlDeadlineAt && + effect.phase === "committed" && + same(effect.origin, origin) && + effect.settlementId === projection.settlementId && + effect.controlExecutionDigest === controlDigest(envelope.execution) && + effect.controlEnvelopeDigest === settlementRecordDigest(domains.binding, envelope) && + effect.admissionDigest === terminal.admissionDigest && + effect.commitmentDigest === commitmentDigest && + effect.preparedReceiptDigest === prepared.preparedReceiptDigest && + effect.expectedDmSequence === prepared.preparedReceipt.sequence && + effect.sqlSession.sessionNonce === envelope.execution.executionId && + effect.attemptedAt >= admission.boundAt && + effect.attemptedAt < envelope.controlDeadlineAt && + terminal.effectSettlementDigest === settlementRecordDigest(domains.effect, effect) && + ack.installationId === installation.installationId && + same(ack.origin, origin) && + ack.settlementId === projection.settlementId && + ack.controlExecutionDigest === controlDigest(envelope.execution) && + ack.commitmentDigest === commitmentDigest && + ack.terminalDigest === tuple.terminalDigest && + ack.terminalAt === tuple.terminalAt, + "Actual logical control terminal/ACK required for retirement", + ); + let releasedAt: number; + if (ownRole === "ops-agent") { + const raw = readPrivateRecord( + local, + "configuration-maintenance", + `${projection.maintenanceId}.json`, + ), + owner = verifyControl( + senderKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ), + authority = restoreAdmittedControlExecution(installation, projection.ownerPrepare); + verifyFamilyOwnedRecord( + local, + origin, + { + scope: "private", + namespace: "configuration-maintenance", + container: "", + name: `${projection.maintenanceId}.json`, + }, + canonicalJson(raw), + ); + assertConfigurationMaintenanceDecisionSource(owner); + verifyConfigurationMaintenanceRootClosure(authority); + releasedAt = verifyRetainedArtifactTargetRootRelease(authority).releasedAt; + need( + same(configurationMaintenanceSettleCompletion(owner, releasedAt), owner) && + owner.apiControls.decide?.receiptDelivery && + owner.apiControls.settle?.receiptDelivery && + owner.apiControls.steps.every((value) => value.receiptDelivery), + "Original API canonical delivery required for retirement", + ); + const binding = verifyControl( + senderKey, + domains.binding, + readConfigurationSettlementCell(local, projection, descriptors.sender[1]!), + configurationSettlementControlEnvelopeSchema, + ), + dispatch = verifyControl( + senderKey, + domains.dispatch, + readConfigurationSettlementCell(local, projection, descriptors.sender[3]!), + configurationSettlementDispatchSchema, + ), + copied = configurationSettlementSenderAckCellSchema.parse( + readConfigurationSettlementCell(local, projection, descriptors.sender[4]!), + ); + need( + same(binding, envelope) && + dispatch.phase === "attempted" && + dispatch.dispatchId === envelope.dispatchId && + dispatch.controlExecutionDigest === controlDigest(envelope.execution) && + dispatch.controlEnvelopeDigest === settlementRecordDigest(domains.binding, envelope) && + dispatch.commitmentDigest === commitmentDigest && + same(copied.terminal, signedTerminal) && + same(copied.ack, signedAck), + "Actual sender original control/copy required", + ); + } else { + const raw = readPrivateRecord( + local, + "configuration-maintenance", + `${projection.maintenanceId}.json`, + 65536, + ), + mirror = verifyControl(dmKey, "openmapx-data-configuration-fence-v1", raw, mirrorSchema), + saved = mirror.metadata, + sql = saved.settlementControl; + verifyFamilyOwnedRecord( + local, + origin, + { + scope: "private", + namespace: "configuration-maintenance", + container: "", + name: `${projection.maintenanceId}.json`, + }, + canonicalJson(raw), + ); + need( + saved.guardState === "settled" && + sql && + same(saved.receipt, prepared.preparedReceipt) && + same(saved.settlement, { + version: 2, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + settlementId: projection.settlementId, + }) && + same(sql.tuple, tuple) && + sql.commitmentDigest === commitmentDigest && + sql.admissionDigest === terminal.admissionDigest && + sql.controlEnvelopeDigest === settlementRecordDigest(domains.binding, envelope) && + sql.preparedReceiptDigest === prepared.preparedReceiptDigest && + sql.expectedDmSequence === prepared.preparedReceipt.sequence && + sql.outcome === committed.final.outcome && + same(sql.sqlSession, effect.sqlSession) && + sql.attemptedAt === effect.attemptedAt && + same( + readConfigurationSettlementCell(local, projection, descriptors.receiver[1]!), + signedAdmission, + ) && + same( + readConfigurationSettlementCell(local, projection, descriptors.receiver[2]!), + signedEffect, + ) && + same( + readConfigurationSettlementCell(local, projection, descriptors.receiver[3]!), + signedTerminal, + ) && + same( + readConfigurationSettlementCell(local, projection, descriptors.receiver[5]!), + signedAck, + ), + "Actual DM SQL mirror/private terminal/ACK required", + ); + releasedAt = verifyConfigurationFenceSettlementReleasedOwner( + installation, + mirror.prepareAssertion, + { + ownerPrepare: projection.ownerPrepare, + ownerCertificateDigest: projection.rootCertificateDigest, + serviceIds: envelope.serviceIds, + }, + ); + } + return Object.freeze({ + terminalAt: Math.max(prepared.preparedTerminalAt, tuple.terminalAt, releasedAt), + terminalDigest: tuple.terminalDigest, + logicalExecutionTicketIds: [ + controlDigest({ + purpose: "configurationSettlementSlot", + role: "data-manager", + settlementId: projection.settlementId, + member: "execution", + }), + ], + logicalIntentTicketIds: ["ops-agent", "data-manager"].map((role) => + controlDigest({ + purpose: "configurationSettlementSlot", + role, + settlementId: projection.settlementId, + member: "intent", + }), + ), + }); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index 63068eb38..078678bed 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -423,7 +423,7 @@ it("rejects arbitrary caller text as effect evidence", async () => { }), ).toThrow(/evidence|receipt/); }); -it("retains terminal maintenance ownership and permits only frozen one-use continuations", async () => { +it("keeps a bare terminal maintenance summary fenced without the fixed owner source", async () => { const c = await import("./control-authority"); const a = await api(); const { writeFileSync } = await import("node:fs"); @@ -448,7 +448,24 @@ it("retains terminal maintenance ownership and permits only frozen one-use conti deadlineAt, fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), }; - const body = { kind: "service.recreate" as const, serviceId: "motis" }; + const { commitConfigurationGeneration } = await import("../services/configuration-generation"), + generation = await commitConfigurationGeneration({ + infraDir: root, + services: [], + input: { + domain: "example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + ownedServiceGenerations: [], + }); + const body = { + kind: "service.recreate" as const, + serviceId: "motis", + expectedConfigurationRevisionId: generation.revisionId, + }; const hash = opsOperationFingerprint(body); const child = { ...execution, @@ -521,48 +538,16 @@ it("retains terminal maintenance ownership and permits only frozen one-use conti childPublicKey: agent.publicKey, }), ).rejects.toThrow(); - const assertion = await a.delegateArtifactTargetLease(lease, { - ...plan, - childPublicKey: agent.publicKey, - }); + // A private journal summary and ordinary own-effect settlement are not + // the actual fixed candidate/owner/decision/guard closure source. await expect( a.delegateArtifactTargetLease(lease, { ...plan, childPublicKey: agent.publicKey }), - ).rejects.toThrow(/replay/); - const peer = c.verifyRetainedLocalControlPeer(agent, { - assertion, - execution: child, - kind: body.kind, - }); - const borrowed = await a.adoptArtifactTargetLease({ - dataRoot: join(root, "data"), - assertion, - peer, - execution: child, - serviceIds: ["motis"], - }); - const childAuthority = c.retainAdmittedControlExecution(agent, { - requestedDeadlineAt: child.deadlineAt, - execution: child, - kind: body.kind, - operation: body, - parent: c.verifiedRetainedControlParent(agent, execution), - admittedAt: deadlineAt - 60000, - serviceIds: ["motis"], - }); + ).rejects.toThrow(); + await expect( + a.settleArtifactTargetLeaseChild(lease, c.verifyControlEffectSettlement(agent, execution)), + ).rejects.toThrow(); await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(); - c.retainControlEffectReceipt(childAuthority, { - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - await a.settleArtifactTargetLeaseChild(borrowed, c.verifyControlEffectSettlement(agent, child)); - c.retainControlEffectReceipt(authority, { - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - await a.settleArtifactTargetLeaseChild(lease, c.verifyControlEffectSettlement(agent, execution)); - await a.releaseArtifactTargetLease(lease); + a.closeArtifactTargetLeaseHandle(lease); c.closeControlInstallation(agent); }); it("orders physical target ownership before a nonblocking reservation, configuration writer and release store", async () => { diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index e6e17de1b..89072313b 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -2,14 +2,64 @@ import { createPrivateKey, type KeyObject, randomUUID } from "node:crypto"; import { fsyncSync, mkdirSync } from "node:fs"; import { setTimeout as delay } from "node:timers/promises"; import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { configurationApplyResultSchema, opsOperationSchema } from "../ops/contract"; import { type ExecutionRef, executionFingerprint, executionRefSchema, } from "../ops/execution-identity"; -import { canonicalJson } from "../ops/fingerprint"; -// biome-ignore lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. -import { prepareControlChildPublication, resolveControlChildPlan } from "./child-phase"; +import { opsOperationFingerprint } from "../ops/fingerprint"; +// biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { + inspectControlChildPublication, + lookupControlChildPhaseAttempt, + lookupControlChildPhaseBinding, + prepareControlChildPublication, + resolveControlChildPlan, +} from "./child-phase"; +// biome-ignore-end lint/suspicious/noImportCycles: Same exact deferred guard import. +import { configurationSettlementRecordDescriptors } from "./configuration-fence-settlement-descriptors"; +import { + configurationChildEvidenceSchema, + configurationCommittedSettlementSchema, + configurationDmCapacityCellSchema, + configurationDmNoGuardRefusalSchema, + configurationIssuedPrepareRefusedProofSchema, + configurationIssuerUnusedSourceDigest, + configurationNeverIssuedProofSchema, + configurationNoGuardBundleSchema, + configurationNoGuardClosureSchema, + configurationNoGuardIntentSchema, + configurationOwnerFinalSchema, + configurationSettlementAckSchema, + configurationSettlementAdmissionSchema, + configurationSettlementCapacitySchema, + configurationSettlementCommitmentCellSchema, + configurationSettlementControlCellSchema, + configurationSettlementControlEnvelopeSchema, + configurationSettlementEffectSchema, + configurationSettlementPublishedTerminalSchema, + configurationSettlementReservationProjectionSchema, + configurationSettlementSenderAckCellSchema, + configurationUnusedCapacityConfirmationSchema, + configurationUnusedCapacityIntentSchema, + deriveConfigurationFenceSettlementId, + configurationSettlementDomains as domains, + settlementRecordDigest, +} from "./configuration-fence-settlement-records"; +import { + assertAppliedConfigurationMaintenanceFinalState, + assertContainedConfigurationMaintenanceFinalState, + assertUnusedConfigurationMaintenanceFinalState, + configurationMaintenanceRuntimeInspectionSchema, +} from "./configuration-maintenance-final-records"; +import { + assertConfigurationMaintenanceDecisionSource, + configurationMaintenanceDecisionCommitmentDomain, + configurationMaintenanceDecisionCommitmentPayloadSchema, + configurationMaintenanceOwnerRecordSchema, +} from "./configuration-maintenance-owner-records"; // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { type ArtifactDelegationAssertion, @@ -22,12 +72,14 @@ import { type ControlRecipient, childPhaseKinds, childSettlementState, + configurationMaintenanceChildState, controlChildOperationKey, controlPeerState, encodePrivate, executionAuthorityState, fixedControlChildId, type InstallationState, + inspectConfigurationPreEffectRefusal, installationState, newControlKey, parseControlAssertion, @@ -35,14 +87,23 @@ import { readPrivateRecord, recordLeaseControlAttempt, recoveryAuthorityState, + restoreAdmittedControlExecution, retainedControlParent, type VerifiedChildSettlement, type VerifiedControlPeer, type VerifiedExecutionAuthority, type VerifiedRecoveryAuthority, + verifyConfigurationMaintenanceChildTerminal, verifyControlEffectSettlement, verifyControlPreparedTerminal, + verifyControlRecoveryAuthority, + verifyDataControlTerminalWitness, withControlDelegationPublication, + withControlMaintenanceDecisionPublication, + withControlMaintenanceInitialization, + withControlMaintenanceNoGuardPublication, + withControlMaintenanceSettlementPublication, + withPrivateEffectWriter, writePrivateRecord, } from "./control-authority"; // biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. @@ -59,18 +120,33 @@ import { verifyControl, } from "./control-records"; import { CONTROL_RETENTION_LIMITS, reserveAuthoritySlot } from "./control-retention"; +import { + dataConfigurationFencePreparedWitnessDomain, + dataConfigurationFencePreparedWitnessSchema, +} from "./data-configuration-fence-witness-records"; import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; +import { assertFixedConfigurationMaintenancePlan } from "./fixed-configuration-maintenance"; import { assertOriginClockForward } from "./retention-clock"; +// biome-ignore-start lint/suspicious/noImportCycles: Physical closure, fixed cells and the read-only normal-pull admission observer verify the sole original family index; retirement verifies this same physical owner. These fixed joins run only after initialization. import { + assertNoUnissuedConfigurationReceiverPublication, enlistFamilyIntent, familyRecordWrite, hasLocalRetentionFamily, markFamilyLeaseReleased, + readConfigurationSettlementCell, + readConfigurationSettlementFamilySlot, registerFamilyLease, reserveControlFamily, trackFamilyDirectory, + verifyCoordinatorPullAdmissionReadOnly, + verifyFamilyOwnedRecord, + verifyPublishedBoundDmConfigurationSettlementSlots, + verifyPublishedUnusedDmConfigurationSettlementSlots, } from "./retention-family"; +// biome-ignore-end lint/suspicious/noImportCycles: Exact deferred original-owner/index guard join. + // biome-ignore lint/suspicious/noImportCycles: Type re-export for the existing shared settlement protocol. export type { ChildSettlement } from "./control-authority"; export type ArtifactTargetLease = { @@ -213,6 +289,120 @@ interface State { slots?: AdmissionSlots; } const handles = new WeakMap(); +declare const maintenanceClosureBrand: unique symbol; +export type ConfigurationMaintenanceClosureObservation = Readonly<{ + [maintenanceClosureBrand]: true; +}>; +const maintenanceClosureObservations = new WeakMap< + ConfigurationMaintenanceClosureObservation, + { + authority: VerifiedExecutionAuthority; + certificateDigest: string; + children: Array<{ execution: ExecutionRef; settlement: ChildSettlement; grantDigest: string }>; + } +>(); +export function configurationMaintenanceClosureObservationState( + value: ConfigurationMaintenanceClosureObservation, + authority: VerifiedExecutionAuthority, +) { + const source = maintenanceClosureObservations.get(value); + if (!source || source.authority !== authority) + throw Error("Actual original target-tree closure observation required"); + return source; +} +function maintenanceObservationFromTree( + s: State, + authority: VerifiedExecutionAuthority, +): ConfigurationMaintenanceClosureObservation { + const current = tree(s), + children: Array<{ execution: ExecutionRef; settlement: ChildSettlement; grantDigest: string }> = + []; + for (const [id, grant] of current.grants) { + const child = current.children.get(id); + if (!child || child.phase !== "settled" || !child.settlement || !knownOutcome(child.settlement)) + throw Error("Every original maintenance descendant must be contained"); + children.push({ + execution: grant.child, + settlement: child.settlement, + grantDigest: child.grantDigest, + }); + } + const observation = Object.freeze({}) as ConfigurationMaintenanceClosureObservation; + maintenanceClosureObservations.set(observation, { + authority, + certificateDigest: controlDigest(s.certificateEnvelope), + children, + }); + return observation; +} +function assertMaintenanceClosureFromTree(s: State, authority: VerifiedExecutionAuthority): void { + const observation = maintenanceObservationFromTree(s, authority); + try { + assertConfigurationMaintenanceRootClosure(authority, observation); + } finally { + maintenanceClosureObservations.delete(observation); + } +} +/** Fixed read-only inspector source, using the same authenticated complete tree + * as release. It creates no lease, issuer, forward capability or final proof. */ +export function observeConfigurationMaintenanceRuntimeTree( + authority: VerifiedExecutionAuthority, +): ConfigurationMaintenanceClosureObservation { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "configurationMaintenance.prepare" || + owner.installation.installation.role !== "ops-agent" || + owner.installation.fixedChildId + ) + throw Error("Original maintenance runtime tree required"); + const reference = z + .strictObject({ leaseId: uuid, certificateDigest: digest }) + .parse( + readPrivateRecord( + owner.installation, + "execution-leases", + `${owner.execution.operationKey}.json`, + ), + ), + s = openState(owner.installation.dataRoot, reference.leaseId, owner.installation, true); + try { + if ( + controlDigest(s.execution) !== controlDigest(owner.execution) || + controlDigest(s.certificateEnvelope) !== reference.certificateDigest + ) + throw Error("Original maintenance runtime certificate changed"); + return guarded(s, () => maintenanceObservationFromTree(s, authority), !s.slots?.release); + } finally { + closeState(s); + } +} +/** Public observation uses the same authenticated tree as the actual release + * paths. No caller source array/boolean can satisfy the internal predicate. */ +export function verifyConfigurationMaintenanceRootClosure( + authority: VerifiedExecutionAuthority, +): void { + const owner = executionAuthorityState(authority), + reference = z + .strictObject({ leaseId: uuid, certificateDigest: digest }) + .parse( + readPrivateRecord( + owner.installation, + "execution-leases", + `${owner.execution.operationKey}.json`, + ), + ), + s = openState(owner.installation.dataRoot, reference.leaseId, owner.installation, true); + try { + if ( + controlDigest(s.execution) !== controlDigest(owner.execution) || + controlDigest(s.certificateEnvelope) !== reference.certificateDigest + ) + throw Error("Original maintenance observation certificate changed"); + guarded(s, () => assertMaintenanceClosureFromTree(s, authority), !s.slots?.release); + } finally { + closeState(s); + } +} function required(value: T | undefined, message: string): T { if (value === undefined) throw Error(message); return value; @@ -630,6 +820,165 @@ function removeSlot(dir: AnchoredDirectory, slot: z.infer) { } dir.removeDirectory(slot.name, slot.identity); } +/** Normal coordinator staging owns no target lease. Observe the bounded shared + * index without a writer, allocation, high-water write or sibling private read. + * Partial admission/release and concurrent replacement are fail-busy. */ +export function assertCoordinatorPullLeaseCompatibility( + authority: VerifiedExecutionAuthority, +): void { + const state = executionAuthorityState(authority), + raw = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ), + admitted = verifyControl( + state.installation.installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z + .object({ + kind: z.literal("service.pull"), + execution: executionRefSchema, + operation: opsOperationSchema, + }) + .passthrough(), + ); + if ( + !state.origin || + state.installation.fixedChildId || + state.kind !== "service.pull" || + controlDigest(raw) !== state.receiptDigest || + controlDigest(admitted.execution) !== controlDigest(state.execution) || + admitted.operation.kind !== "service.pull" || + admitted.operation.serviceId !== "data-manager" || + !admitted.operation.candidateId || + opsOperationFingerprint(admitted.operation) !== state.execution.operationFingerprint + ) + throw Error("Actual normal coordinator pull required"); + verifyCoordinatorPullAdmissionReadOnly(authority); + const opened: AnchoredDirectory[] = [], + open = (parent: AnchoredDirectory, name: string) => { + const dir = parent.directory(name); + opened.push(dir); + return dir; + }, + data = new AnchoredDirectory(state.installation.dataRoot); + try { + let control: AnchoredDirectory; + try { + control = open(data, ".admin-target-control"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + data.check(); + return; + } + throw error; + } + // Before the first root, execution witnesses may exist without a target + // index. Only actual absence of both lock and retained-owner entries is safe. + if (!control.listBounded(16).includes("admission-index-v1")) { + for (const name of ["locks", "leases"]) { + let dir: AnchoredDirectory; + try { + dir = open(control, name); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + if (dir.hasEntries()) throw Error("Unindexed physical target ownership"); + } + control.check(); + return; + } + const index = open(control, "admission-index-v1"), + active = open(index, "active"), + locks = open(control, "locks"), + leases = open(control, "leases"), + ready = z.strictObject({ installationId: uuid }).parse(json(index, "ready.json", 1024)); + if (ready.installationId !== state.installation.installation.installationId) + throw Error("Foreign target index"); + const snapshot = () => + Object.fromEntries( + active + .listBounded(CONTROL_RETENTION_LIMITS.activeLeases) + .sort() + .map((name) => { + if ( + !/^slot-\d{1,4}$/.test(name) || + Number(name.slice(5)) >= CONTROL_RETENTION_LIMITS.activeLeases + ) + throw Error("Unknown active target slot"); + const dir = active.directory(name); + try { + return [ + name, + { + identity: identity(dir.stat), + generation: z + .strictObject({ generation: uuid }) + .parse(json(dir, "generation.json", 1024)).generation, + }, + ]; + } finally { + dir.close(); + } + }), + ), + before = snapshot(), + matched = new Set(); + if (locks.listBounded(256).includes("data-manager")) throw Error("Coordinator target is held"); + const names = leases.listBounded(CONTROL_RETENTION_LIMITS.leases + 3).sort(); + for (const name of names) { + if ([".retention-count", ".retention-highwater", ".retention-transaction"].includes(name)) + continue; + uuid.parse(name); + const owner = openState(state.installation.dataRoot, name, state.installation, true); + try { + const root = readRoot(owner), + slots = owner.slots; + if ( + !slots || + !validateOriginRecord(root) || + !sameControlOrigin(root.origin, owner.certificate.origin) || + controlDigest(root.certificate) !== controlDigest(owner.certificateEnvelope) + ) + throw Error("Changed physical owner"); + if (slots.release?.phase === "released" && slots.releasedAt) { + if (root.phase !== "settled" || !knownOutcome(root.settlement)) + throw Error("Unproved physical release"); + continue; + } + if (owner.certificate.serviceIds.includes("data-manager")) + throw Error("Coordinator target is unreleased"); + const slot = before[slots.active.name]; + if ( + !slot || + matched.has(slots.active.name) || + !sameInode(slot.identity, slots.active.identity) || + slot.generation !== slots.active.generation + ) + throw Error("Unindexed active physical owner"); + matched.add(slots.active.name); + } finally { + closeState(owner); + } + } + if ( + matched.size !== Object.keys(before).length || + canonicalJson(snapshot()) !== canonicalJson(before) || + canonicalJson(leases.listBounded(CONTROL_RETENTION_LIMITS.leases + 3).sort()) !== + canonicalJson(names) || + locks.listBounded(256).includes("data-manager") + ) + throw Error("Physical target index is changing"); + for (const dir of opened) dir.check(); + } finally { + for (const dir of opened.reverse()) dir.close(); + data.close(); + } +} function _releaseActiveSlot(s: State) { if (!s.slots) throw Error("Missing retained admission slots"); const index = s.control.directory("admission-index-v1"), @@ -936,6 +1285,7 @@ export function verifyFiniteControlEdge( phase === "hardlinks" && [ "service.start", + "service.update", "system.update", "service.build", "services.buildAll", @@ -1418,7 +1768,10 @@ export function bindBorrowedControlExecution( throw Error("Borrower execution authority mismatch"); s.authority = a; s.verifiedAuthority = authority; - if (a.kind === "generation-hardlinks" && a.installation.installation.role === "data-manager") { + if ( + ["generation-hardlinks", "configuration-maintenance.prepare"].includes(a.kind) && + a.installation.installation.role === "data-manager" + ) { if (!s.node || !a.origin) throw Error("Original generation adoption required"); const payload = { version: 2, @@ -1430,6 +1783,10 @@ export function bindBorrowedControlExecution( certificateDigest: controlDigest(s.certificateEnvelope), }; const name = `${a.execution.operationKey}.json`; + const domain = + a.kind === "generation-hardlinks" + ? "openmapx-generation-adoption-v2" + : "openmapx-configuration-fence-adoption-v2"; let prior: unknown; try { prior = readPrivateRecord(a.installation, "execution-adoptions", name, 8192); @@ -1439,7 +1796,7 @@ export function bindBorrowedControlExecution( if (prior !== undefined) { const saved = verifyControl( a.installation.installation.publicKey, - "openmapx-generation-adoption-v2", + domain, prior, z.unknown(), ); @@ -1450,12 +1807,49 @@ export function bindBorrowedControlExecution( a.installation, "execution-adoptions", name, - signControl(a.installation.privateKey, "openmapx-generation-adoption-v2", payload), + signControl(a.installation.privateKey, domain, payload), true, a.origin, ); } } +/** Source-only recovery address for the original normal DM PREPARE adopter. + * It allocates no lease/grant or deadline, and reads only this role's records. */ +export function retainedConfigurationFenceAdoption(authority: VerifiedExecutionAuthority) { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "configuration-maintenance.prepare" || + owner.installation.installation.role !== "data-manager" || + !owner.origin + ) + throw Error("Original normal DM fence adopter required"); + const value = verifyControl( + owner.installation.installation.publicKey, + "openmapx-configuration-fence-adoption-v2", + readPrivateRecord( + owner.installation, + "execution-adoptions", + `${owner.execution.operationKey}.json`, + 8192, + ), + z.strictObject({ + version: z.literal(2), + origin: controlOriginSchema, + execution: executionRefSchema, + admissionDigest: digest, + leaseId: uuid, + delegationId: uuid, + certificateDigest: digest, + }), + ); + if ( + canonicalJson(value.execution) !== canonicalJson(owner.execution) || + value.admissionDigest !== owner.receiptDigest || + canonicalJson(value.origin) !== canonicalJson(owner.origin) + ) + throw Error("Original normal DM fence adoption changed"); + return Object.freeze(value); +} /** Reopen only the original authenticated generation leaf for settlement. * There is no adopt, target acquire, deadline renewal or owner-release path. */ export function restoreGenerationHardlinksLeaseForSettlement( @@ -1550,6 +1944,34 @@ export function restoreGenerationHardlinksLeaseForSettlement( throw error; } } +/** Exact borrowed generation leaf containment; its parent root remains held. + * This authenticates the existing own-effect and physical terminal sources, + * never a root release, new adoption or forward capability. */ +export function verifyRetainedGenerationHardlinksSettlement( + authority: VerifiedExecutionAuthority, +): void { + const lease = restoreGenerationHardlinksLeaseForSettlement(authority), + s = stateOf(lease); + try { + guarded(s, () => { + const current = tree(s), + child = s.node && current.children.get(s.node), + effect = childSettlementState( + verifyControlEffectSettlement(s.installation.installation, s.execution), + ).evidence; + if ( + !child || + child.phase !== "settled" || + !knownOutcome(child.settlement) || + controlDigest(child.settlement) !== controlDigest(effect) || + [...current.grants.values()].some((value) => value.parentNode === s.node) + ) + throw Error("Original generation leaf containment required"); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} export async function settleArtifactTargetLeaseChild( lease: TargetLeaseAuthority, evidence: VerifiedChildSettlement, @@ -1563,6 +1985,10 @@ export async function settleArtifactTargetLeaseChild( throw Error("Foreign settlement identity"); await guarded(s, () => { const t = tree(s); + if (s.node === null && s.authority?.kind === "configurationMaintenance.prepare") { + if (!s.verifiedAuthority) throw Error("Original maintenance completion authority required"); + assertMaintenanceClosureFromTree(s, s.verifiedAuthority); + } const node = s.node === null ? t.root : required(t.children.get(s.node), "Missing retained child node"); for (const grant of t.grants.values()) { @@ -1600,6 +2026,222 @@ export function inspectArtifactTargetLeaseChildSettlement( return child.settlement; }); } +/** Source-only observation of the original held maintenance root. It creates + * no forward authority, lease, grant or renewed deadline. */ +export function configurationMaintenanceTargetOwner(lease: ArtifactTargetLease) { + const s = stateOf(lease); + return guarded(s, () => { + const current = tree(s); + if ( + s.node !== null || + s.installation.installation.role !== "ops-agent" || + s.authority?.kind !== "configurationMaintenance.prepare" || + !s.certificate.origin || + current.root.phase === "settled" || + s.slots?.release + ) + throw Error("Held original maintenance target owner required"); + return Object.freeze({ + installation: s.installation.installation, + execution: s.execution, + origin: s.certificate.origin, + admissionDigest: s.authority.receiptDigest, + certificateDigest: controlDigest(s.certificateEnvelope), + serviceIds: s.certificate.serviceIds, + }); + }); +} +/** Fixed owner-store publication order: target journal, original ancestry and + * execution publication guard, then its role-private writer. */ +export function withConfigurationMaintenanceOwnerPublication( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const s = stateOf(lease); + if ( + s.node !== null || + s.verifiedAuthority !== authority || + s.authority?.kind !== "configurationMaintenance.prepare" || + s.settlementOnly + ) + throw Error("Original live configuration maintenance owner required"); + return guarded(s, () => + withControlDelegationPublication(authority, "configuration-commit", () => + withPrivateEffectWriter(executionAuthorityState(authority), run), + ), + ); +} +export function withConfigurationMaintenanceOwnerInitialization( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const s = stateOf(lease); + if ( + s.node !== null || + s.verifiedAuthority !== authority || + s.authority?.kind !== "configurationMaintenance.prepare" || + s.settlementOnly + ) + throw Error("Original live maintenance initialization required"); + return guarded(s, () => + withControlMaintenanceInitialization(authority, () => + withPrivateEffectWriter(executionAuthorityState(authority), run), + ), + ); +} +/** Lease -> original ancestry/execution publication -> private writer. The + * callback cannot issue a grant or create a forward/replacement authority. */ +export function withConfigurationMaintenanceNoGuardPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const lease = restoreArtifactTargetLeaseForSettlement(authority), + s = stateOf(lease), + owner = executionAuthorityState(authority); + try { + if (s.node !== null || owner.kind !== "configurationMaintenance.prepare") + throw Error("Original maintenance NoGuard owner required"); + return guarded(s, () => { + if (tree(s).root.phase === "settled" || s.slots?.release) + throw Error("Original NoGuard owner already released"); + return withControlMaintenanceNoGuardPublication(authority, () => + withPrivateEffectWriter(owner, run), + ); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} +/** Fixed logical settlement metadata only, including after original expiry or + * cancellation. Ordinary descendants must already be conclusively settled. */ +export function withConfigurationMaintenanceSettlementPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const lease = restoreArtifactTargetLeaseForSettlement(authority), + state = stateOf(lease), + owner = executionAuthorityState(authority); + try { + if (state.node !== null || owner.kind !== "configurationMaintenance.prepare") + throw Error("Original settlement root required"); + return guarded(state, () => { + const current = tree(state); + if ( + current.root.phase === "settled" || + state.slots?.release || + [...current.children.values()].some( + (child) => child.phase !== "settled" || !knownOutcome(child.settlement), + ) + ) + throw Error("Original settlement root/ordinary children unresolved"); + return withControlMaintenanceSettlementPublication(authority, () => + withPrivateEffectWriter(owner, run), + ); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} +/** Publication of already proved guard observations remains possible after + * cancellation/expiry. This returns no forward lease or dispatch capability. */ +export function withConfigurationMaintenanceOwnerObservation( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const s = stateOf(lease), + owner = executionAuthorityState(authority); + if ( + s.node !== null || + s.authority?.kind !== "configurationMaintenance.prepare" || + owner.receiptDigest !== s.authority.receiptDigest || + controlDigest(owner.execution) !== controlDigest(s.execution) + ) + throw Error("Original configuration maintenance recovery owner required"); + return guarded(s, () => { + if (tree(s).root.phase === "settled" || s.slots?.release) + throw Error("Maintenance observation owner already released"); + return withPrivateEffectWriter(owner, run); + }); +} +/** Lease -> original forward ancestry/terminal-prepare -> private writer. + * This is a metadata CAS scope, not restoration of a forward lease. */ +export function withConfigurationMaintenanceDecisionPublication( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const s = stateOf(lease), + owner = executionAuthorityState(authority); + if ( + s.node !== null || + s.authority?.kind !== "configurationMaintenance.prepare" || + owner.receiptDigest !== s.authority.receiptDigest || + controlDigest(owner.execution) !== controlDigest(s.execution) + ) + throw Error("Original maintenance decision owner required"); + return guarded(s, () => { + if (tree(s).root.phase !== "held" || s.slots?.release) + throw Error("Original prepared maintenance root is fenced"); + return withControlMaintenanceDecisionPublication(authority, () => + withPrivateEffectWriter(owner, run), + ); + }); +} +/** Fixed frozen-phase metadata publication follows the same live original + * lease/ancestry/private-writer order as that phase's grant. */ +export function withConfigurationMaintenancePhasePublication( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + phase: + | "configuration-commit" + | "maintenance-hardlinks" + | "maintenance-recreate" + | "maintenance-handoff", + run: () => T, +): T { + const s = stateOf(lease), + owner = executionAuthorityState(authority); + if ( + s.node !== null || + s.authority?.kind !== "configurationMaintenance.prepare" || + s.settlementOnly || + s.verifiedAuthority !== authority + ) + throw Error("Original frozen maintenance issuer required"); + return guarded(s, () => + withControlDelegationPublication(authority, phase, () => { + if (tree(s).root.phase !== "held" || s.slots?.release) + throw Error("Original maintenance phase is fenced"); + return withPrivateEffectWriter(owner, run); + }), + ); +} +/** Original PREPARE reservation publication can restore its own retained handle, + * but the handle never escapes and its forward clock/ancestry are rechecked. */ +export function withConfigurationMaintenanceReservationPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const lease = restoreArtifactTargetLeaseForSettlement(authority), + s = stateOf(lease), + owner = executionAuthorityState(authority); + try { + if (owner.kind !== "configurationMaintenance.prepare" || s.node !== null) + throw Error("Original maintenance reservation owner required"); + return guarded(s, () => { + if (tree(s).root.phase !== "held" || s.slots?.release) + throw Error("Original pre-effect maintenance root required"); + return withControlDelegationPublication(authority, "configuration-fence", () => + withPrivateEffectWriter(owner, run), + ); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} function finishOwnedRelease(s: State): void { guarded( s, @@ -1607,6 +2249,10 @@ function finishOwnedRelease(s: State): void { const t = tree(s); if (t.root.phase !== "settled" || !knownOutcome(t.root.settlement)) throw Error("Durable own-effect settlement is required"); + if (s.authority?.kind === "configurationMaintenance.prepare") { + if (!s.verifiedAuthority) throw Error("Original maintenance release authority required"); + assertMaintenanceClosureFromTree(s, s.verifiedAuthority); + } for (const id of t.grants.keys()) { const child = t.children.get(id); if (child?.phase !== "settled" || !knownOutcome(child.settlement)) @@ -1768,7 +2414,7 @@ export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Pr /** Read-only original root release proof for a data command's parent. */ export function verifyRetainedArtifactTargetRootRelease( authority: VerifiedExecutionAuthority, -): void { +): Readonly<{ releasedAt: number }> { const a = executionAuthorityState(authority), reference = z .strictObject({ leaseId: uuid, certificateDigest: digest }) @@ -1784,6 +2430,8 @@ export function verifyRetainedArtifactTargetRootRelease( if ( controlDigest(s.execution) !== controlDigest(a.execution) || controlDigest(s.certificateEnvelope) !== reference.certificateDigest || + !Number.isSafeInteger(s.slots?.releasedAt) || + !s.slots?.releasedAt || s.slots?.release?.phase !== "released" || t.root.phase !== "settled" || controlDigest(t.root.settlement) !== controlDigest(effect) || @@ -1793,6 +2441,7 @@ export function verifyRetainedArtifactTargetRootRelease( ) ) throw Error("Original root release is not proved"); + return Object.freeze({ releasedAt: s.slots.releasedAt! }); } finally { closeState(s); } @@ -1846,6 +2495,50 @@ export function restoreArtifactTargetLeaseForSettlement( throw error; } } +/** Reopen only the actual prepared owner's already-authorized frozen phase + * issuer. Its ordinary terminal execution still cannot perform a target + * effect; only the existing finite delegation publisher consumes this handle. + * No lease/root/certificate, clock, target or reservation is allocated. */ +export function restoreConfigurationMaintenancePhaseIssuer( + authority: VerifiedExecutionAuthority, + phase: + | "configuration-commit" + | "maintenance-hardlinks" + | "maintenance-recreate" + | "maintenance-handoff", +): ArtifactTargetLease { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "configurationMaintenance.prepare" || + ![ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(phase) + ) + throw Error("Original authorized maintenance phase issuer required"); + const lease = restoreArtifactTargetLeaseForSettlement(authority), + s = stateOf(lease); + try { + guarded(s, () => + withControlDelegationPublication(authority, phase, () => { + if ( + s.node !== null || + tree(s).root.phase !== "held" || + s.slots?.release || + !owner.reservations.some((value) => value.phase === phase) + ) + throw Error("Original frozen maintenance root is fenced"); + s.settlementOnly = false; + }), + ); + return lease; + } catch (error) { + closeArtifactTargetLeaseHandle(lease); + throw error; + } +} export async function reconcileArtifactTargetLease(options: { dataRoot: string; leaseId: string; @@ -1873,6 +2566,13 @@ export async function reconcileArtifactTargetLease(options: { type: "pkcs8", format: "der", }); + if (recovery.authority.kind === "configurationMaintenance.prepare") { + s.authority = recovery.authority; + s.verifiedAuthority = restoreAdmittedControlExecution( + s.installation.installation, + recovery.authority.execution, + ); + } const t = tree(s); if ( !knownOutcome(recovery.evidence) || @@ -1887,6 +2587,10 @@ export async function reconcileArtifactTargetLease(options: { return "released"; } await guarded(s, () => { + if (s.authority?.kind === "configurationMaintenance.prepare") { + if (!s.verifiedAuthority) throw Error("Original maintenance recovery authority required"); + assertMaintenanceClosureFromTree(s, s.verifiedAuthority); + } const root = readRoot(s); root.settlement = recovery.evidence; root.phase = "settled"; @@ -1906,6 +2610,60 @@ export async function reconcileArtifactTargetLease(options: { closeState(s); } } +/** Finish only this original unused root, including its existing partial + * release cursor. No acquisition, forward authority, new clock or caller ID. */ +export async function reconcileConfigurationPreEffectRoot( + authority: VerifiedExecutionAuthority, +): Promise { + inspectConfigurationPreEffectRefusal(authority); + const state = executionAuthorityState(authority), + name = `${state.execution.operationKey}.json`; + if (!state.origin) throw Error("Original pre-effect root source required"); + let raw: unknown; + try { + raw = readPrivateRecord(state.installation, "execution-leases", name); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT" && state.kind === "service.pull") return; + throw error; + } + if (raw === null && state.kind === "service.pull") return; + if (state.kind === "service.pull") + throw Error("Normal coordinator pull cannot own a physical target root"); + const reference = z.strictObject({ leaseId: uuid, certificateDigest: digest }).parse(raw); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "execution-leases", container: "", name }, + canonicalJson(raw), + ); + const ownerName = `${reference.leaseId}.json`, + saved = readPrivateRecord(state.installation, "leases", ownerName); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "leases", container: "", name: ownerName }, + canonicalJson(saved), + ); + if ( + controlDigest((saved as { certificate: unknown }).certificate) !== reference.certificateDigest + ) + throw Error("Original pre-effect root certificate changed"); + try { + verifyRetainedArtifactTargetRootRelease(authority); + return; + } catch { + /* Only the existing exact recovery path may finish a partial release. */ + } + if ( + (await reconcileArtifactTargetLease({ + dataRoot: state.installation.dataRoot, + leaseId: reference.leaseId, + authority: verifyControlRecoveryAuthority(authority), + })) !== "released" + ) + throw Error("Original pre-effect root remains fenced"); + verifyRetainedArtifactTargetRootRelease(authority); +} export async function withArtifactTargetLease( options: { dataRoot: string; @@ -2009,6 +2767,112 @@ export function inspectArtifactDelegation(options: { closeState(s); } } +/** Read-only FIRST-use check for the fixed logical guard settlement receiver. + * It consumes the DM's ACTUAL retained preparation assertion, checks the same + * anchored physical root/all ordinary descendants, and returns no lease/peer, + * execution or forward capability. Historical settled observations omit it. */ +export function verifyConfigurationFenceSettlementHeldOwner( + installation: import("./control-authority").ControlInstallation, + originalPrepareAssertion: string, + expected: { + ownerPrepare: ExecutionRef; + ownerCertificateDigest: string; + serviceIds: readonly string[]; + }, +): void { + const local = installationState(installation), + parsed = parseControlAssertion(originalPrepareAssertion), + inspected = inspectArtifactDelegation({ + installation, + dataRoot: local.dataRoot, + assertion: originalPrepareAssertion, + }), + state = openState(local.dataRoot, parsed.leaseId, local); + try { + guarded(state, () => { + const current = tree(state); + if ( + installation.role !== "data-manager" || + local.fixedChildId || + inspected.kind !== "configuration-maintenance.prepare" || + inspected.phase !== "configuration-fence" || + inspected.recipient !== "data-manager" || + canonicalJson(inspected.parent) !== canonicalJson(expected.ownerPrepare) || + canonicalJson(inspected.owner) !== canonicalJson(expected.ownerPrepare) || + inspected.ownerCertificateDigest !== expected.ownerCertificateDigest || + canonicalJson(state.certificate.serviceIds) !== canonicalJson(expected.serviceIds) || + current.root.phase === "settled" || + state.slots?.release || + [...current.children.values()].some( + (child) => child.phase !== "settled" || !knownOutcome(child.settlement), + ) + ) + throw Error("Original logical settlement owner/ordinary children unresolved"); + }); + } finally { + closeState(state); + } +} +/** Historical retirement only: the original prepared assertion and complete + * physical tree must prove actual release. Returns no peer/lease/forward grant. */ +export function verifyConfigurationFenceSettlementReleasedOwner( + installation: import("./control-authority").ControlInstallation, + originalPrepareAssertion: string, + expected: { + ownerPrepare: ExecutionRef; + ownerCertificateDigest: string; + serviceIds: readonly string[]; + }, +): number { + const local = installationState(installation), + parsed = parseControlAssertion(originalPrepareAssertion), + state = openState(local.dataRoot, parsed.leaseId, local, true); + try { + const current = tree(state), + grant = required( + current.grants.get(parsed.delegationId), + "Original preparation grant required", + ), + parent = grant.parentNode + ? required(current.grants.get(grant.parentNode), "Original preparation parent required") + : undefined, + parentNode = grant.parentNode + ? required(current.children.get(grant.parentNode), "Original preparation adoption required") + : current.root, + { signature, ...fields } = parsed; + verifyControl( + parent?.childPublicKey ?? state.certificate.ownerPublicKey, + "openmapx-lease-assertion-v1", + { payload: fields, signature }, + assertionSchema.omit({ signature: true }), + ); + if ( + installation.role !== "data-manager" || + local.fixedChildId || + grant.childKind !== "configuration-maintenance.prepare" || + grant.phase !== "configuration-fence" || + grant.recipient !== "data-manager" || + parentNode.grantDigests[grant.delegationId] !== parsed.grantDigest || + canonicalJson(grant.parent) !== canonicalJson(expected.ownerPrepare) || + canonicalJson(state.certificate.owner) !== canonicalJson(expected.ownerPrepare) || + parsed.ownerCertificateDigest !== controlDigest(state.certificateEnvelope) || + parsed.ownerCertificateDigest !== expected.ownerCertificateDigest || + canonicalJson(state.certificate.serviceIds) !== canonicalJson(expected.serviceIds) || + current.root.phase !== "settled" || + !knownOutcome(current.root.settlement) || + state.slots?.release?.phase !== "released" || + !Number.isSafeInteger(state.slots.releasedAt) || + !state.slots.releasedAt || + [...current.children.values()].some( + (child) => child.phase !== "settled" || !knownOutcome(child.settlement), + ) + ) + throw Error("Original prepared root release required for retirement"); + return state.slots.releasedAt; + } finally { + closeState(state); + } +} export async function cancelArtifactTargetLease( lease: TargetLeaseAuthority, @@ -2070,6 +2934,29 @@ export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): v if (!node || !forwardAncestry(t, s.node) || Date.now() >= s.execution.deadlineAt) throw Error("Target lease permits observation/settlement only"); } +export function assertArtifactTargetLeaseExecution( + lease: TargetLeaseAuthority, + execution: ExecutionRef, +): void { + const state = stateOf(lease); + if (controlDigest(state.execution) !== controlDigest(execution)) + throw Error("Lifecycle lease execution mismatch"); + assertArtifactTargetLeaseForward(lease); +} + +/** Receipt-only evidence: the exact physical owner may record containment after cancellation. + * This does not authorize a forward effect, delegation, acquisition or release. */ +export function assertArtifactTargetLeaseReceiptOwner( + lease: TargetLeaseAuthority, + execution: ExecutionRef, +): void { + const s = stateOf(lease); + if (controlDigest(s.execution) !== controlDigest(execution)) + throw Error("Receipt lease execution mismatch"); + const t = tree(s), + node = s.node === null ? t.root : t.children.get(s.node); + if (!node || node.phase === "settled") throw Error("Receipt owner is already settled"); +} /** Drop local descriptors only. Durable locks, ownership and all unknown fences remain untouched. */ export function closeArtifactTargetLeaseHandle(lease: TargetLeaseAuthority): void { @@ -2102,3 +2989,1034 @@ export function verifyRetainedArtifactTargetSettlement( closeState(s); } } + +// Fixed closure and original children consume this module's one nominal tree observation. +export function readOriginalMaintenanceFinalChildren( + authority: VerifiedExecutionAuthority, + observation: ConfigurationMaintenanceClosureObservation, +) { + const state = executionAuthorityState(authority), + tree = configurationMaintenanceClosureObservationState(observation, authority), + admitted = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + name = `${admitted.payload?.operation?.maintenanceId}.json`, + raw = readPrivateRecord(state.installation, "configuration-maintenance", name), + owner = verifyControl( + state.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ); + assertConfigurationMaintenanceDecisionSource(owner); + verifyFamilyOwnedRecord( + state.installation, + owner.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name }, + canonicalJson(raw), + ); + if ( + state.kind !== "configurationMaintenance.prepare" || + !owner.guardSource || + tree.certificateDigest !== owner.rootCertificateDigest || + owner.admissionDigest !== state.receiptDigest || + canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) + ) + throw Error("Original final child owner required"); + const children: ReturnType[] = [], + allowed = new Set([controlDigest(owner.guardSource.prepareExecution)]); + for (const step of owner.plan.steps) { + const source = owner.apiControls.steps.find((value) => value.ordinal === step.ordinal), + publication = inspectControlChildPublication(authority, step); + if (!source) { + if (publication.binding !== null || publication.publication.state !== "unissued") + throw Error("Frozen child attempt has no original source"); + continue; + } + if (!source.canonical || !source.receiptDelivery) + throw Error("Original frozen child canonical delivery pending"); + const actual = configurationMaintenanceChildState( + verifyConfigurationMaintenanceChildTerminal( + authority, + step.ordinal, + source.canonical.status, + source.canonical.terminalAt, + ), + authority, + ), + physical = tree.children.find( + (value) => canonicalJson(value.execution) === canonicalJson(actual.execution), + ); + if (!physical || canonicalJson(physical.settlement) !== canonicalJson(actual.settlement)) + throw Error("Original child physical settlement changed"); + allowed.add(controlDigest(actual.execution)); + children.push( + configurationChildEvidenceSchema.parse({ + planIndex: step.ordinal, + execution: actual.execution, + terminalDigest: actual.terminalDigest, + terminalAt: actual.terminalAt, + effectSettlementDigest: actual.effectSettlementDigest, + publication: actual.settlement.publication, + }), + ); + if (step.phase === "maintenance-hardlinks" && owner.plan.nestedHardlinks) { + const child = restoreAdmittedControlExecution( + state.installation.installation, + actual.execution, + ), + binding = lookupControlChildPhaseBinding(child, owner.plan.nestedHardlinks); + if (!binding) throw Error("Original middle owner's DM leaf unavailable"); + const terminal = verifyDataControlTerminalWitness( + state.installation.installation, + binding.child.execution, + ), + physical = tree.children.find( + (value) => canonicalJson(value.execution) === canonicalJson(binding.child.execution), + ); + if ( + !physical || + physical.settlement.outcome !== "committed" || + physical.settlement.publication !== "committed" || + terminal.status.kind !== "generation-hardlinks" || + terminal.status.status !== "ok" || + terminal.status.publication !== "committed" || + terminal.status.result?.kind !== "generation-hardlinks" + ) + throw Error("Original exact DM generation leaf settlement unavailable"); + allowed.add(controlDigest(binding.child.execution)); + children.push( + configurationChildEvidenceSchema.parse({ + planIndex: step.ordinal, + execution: binding.child.execution, + terminalDigest: terminal.terminalDigest, + terminalAt: terminal.terminalAt, + effectSettlementDigest: controlDigest(physical.settlement), + publication: "committed", + }), + ); + } + } + if ( + tree.children.some((value) => !allowed.has(controlDigest(value.execution))) || + tree.children.length !== allowed.size + ) + throw Error("Unaccounted original maintenance descendant"); + children.sort( + (a, b) => + a.planIndex - b.planIndex || a.execution.operationKey.localeCompare(b.execution.operationKey), + ); + return { + ownerDigest: controlDigest(owner), + children, + complete: owner.plan.steps.every((step) => + owner.apiControls.steps.some( + (value) => value.ordinal === step.ordinal && value.canonical && value.receiptDelivery, + ), + ), + generation: children.find((value) => value.planIndex === 0), + generationResult: owner.apiControls.steps.find((value) => value.ordinal === 0)?.canonical + ? configurationApplyResultSchema.parse( + owner.apiControls.steps.find((value) => value.ordinal === 0)!.canonical!.status.result, + ) + : null, + followUpReceiptDigests: [ + ...new Set( + children.filter((value) => value.planIndex !== 0).map((value) => value.terminalDigest), + ), + ].sort(), + }; +} + +const hash = z.string().regex(/^[a-f0-9]{64}$/), + same = (a: unknown, b: unknown) => canonicalJson(a) === canonicalJson(b); +function need(value: unknown, message: string): asserts value { + if (!value) throw Error(message); +} +/** PreparedAndSettled uses the actual private inspector/final, original + * prepared physical child, full signed logical terminal/admission/effect and + * durable sender copy/ACK. Parsed prepared status alone cannot enter it. */ +function assertPreparedAndSettledClosure( + authority: VerifiedExecutionAuthority, + observation: ConfigurationMaintenanceClosureObservation, + input: unknown, +): void { + const tree = configurationMaintenanceClosureObservationState(observation, authority); + const state = executionAuthorityState(authority), + owner = configurationMaintenanceOwnerRecordSchema.parse(input); + assertConfigurationMaintenanceDecisionSource(owner); + assertFixedConfigurationMaintenancePlan(owner.plan, owner.installationId); + if ( + owner.final?.ownerFinal.payload.outcome === "applied" || + owner.final?.ownerFinal.payload.outcome === "contained" + ) { + const contained = owner.final.ownerFinal.payload.outcome === "contained"; + if (contained) assertContainedConfigurationMaintenanceFinalState(owner); + else assertAppliedConfigurationMaintenanceFinalState(owner); + const actual = readOriginalMaintenanceFinalChildren(authority, observation); + need( + (contained + ? actual.generationResult?.outcome === "not_committed" && + !actual.generationResult.cleanupFailed && + actual.children.length === 1 + : actual.complete) && + same(actual.children, owner.final.evidence.children) && + owner.final.evidence.generation.commitReceiptDigest === + (contained ? null : actual.generation?.terminalDigest) && + same(owner.final.evidence.followUp.receiptDigests, actual.followUpReceiptDigests), + "Original applied final child sources changed", + ); + } else assertUnusedConfigurationMaintenanceFinalState(owner); + need( + owner.final && + owner.inspection && + owner.guardSource && + owner.capacity && + owner.apiControls.decide?.receiptDelivery && + !owner.noGuard, + "Actual prepared final/inspection/guard source required", + ); + need( + owner.installationId === state.installation.installation.installationId && + owner.admissionDigest === state.receiptDigest && + same(owner.origin, state.origin) && + same(owner.plan.prepare, state.execution) && + owner.rootCertificateDigest === tree.certificateDigest && + same(owner.plan.reservations, state.reservations), + "Original prepared root identity changed", + ); + const final = verifyControl( + state.installation.installation.publicKey, + domains.ownerFinal, + owner.final.ownerFinal, + configurationOwnerFinalSchema, + ), + inspection = verifyControl( + state.installation.installation.publicKey, + domains.inspection, + owner.inspection, + configurationMaintenanceRuntimeInspectionSchema, + ); + verifyControl( + state.installation.installation.publicKey, + configurationMaintenanceDecisionCommitmentDomain, + owner.apiControls.decide.commitment, + configurationMaintenanceDecisionCommitmentPayloadSchema, + ); + need( + final.privateInspectionId === inspection.inspectionId && + final.evidenceDigest === owner.final.evidenceDigest, + "Original actual final/inspection changed", + ); + for (const step of final.outcome === "unused" ? owner.plan.steps : []) { + const phase = inspectControlChildPublication(authority, step); + need( + phase.binding === null && phase.publication.state === "unissued", + "Unused final has attempted frozen work", + ); + } + const prepare = state.reservations.find((value) => value.phase === "configuration-fence"), + settle = state.reservations.find((value) => value.phase === "configuration-fence-settle"); + need( + prepare && + settle && + inspectControlChildPublication(authority, settle).publication.state === "unissued", + "Original purpose-only settlement reservation required", + ); + const descriptors = configurationSettlementRecordDescriptors( + { ownerPrepare: state.execution, reservation: settle }, + prepare.execution.operationKey, + ), + control = verifyControl( + state.installation.installation.publicKey, + domains.control, + readConfigurationSettlementCell( + state.installation, + { origin: owner.origin }, + descriptors.sender[2]!, + ), + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + state.installation.installation.publicKey, + domains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + committed = verifyControl( + state.installation.installation.publicKey, + domains.reference, + readConfigurationSettlementCell(state.installation, projection, descriptors.sender[0]!), + configurationCommittedSettlementSchema, + ); + need( + control.binding && + !("unused" in control) && + same(projection.ownerPrepare, state.execution) && + projection.rootAdmissionDigest === state.receiptDigest && + projection.rootCertificateDigest === tree.certificateDigest && + same(projection.reservation, settle) && + same(projection.operation, { + kind: "configuration-maintenance.settle", + ...owner.plan.dmSettle, + }) && + projection.reservationDigest === settlementRecordDigest(domains.reservationTuple, settle) && + projection.reservation.operationFingerprint === controlDigest(projection.operation) && + projection.settlementId === final.identity.settlementId && + projection.planDigest === owner.plan.planDigest && + same(committed.identity, final.identity) && + same(committed.final, final) && + same(committed.evidence, owner.final.evidenceProjection) && + committed.evidenceDigest === owner.final.evidenceDigest && + committed.ownerReceiptDigest === owner.final.ownerReceiptDigest && + committed.committedAt === final.settlingAt && + committed.referenceOperationFingerprint === settle.operationFingerprint, + "Original committed prepared settlement changed", + ); + const envelope = verifyControl( + state.installation.installation.publicKey, + domains.control, + control.binding, + configurationSettlementControlEnvelopeSchema, + ), + binding = verifyControl( + state.installation.installation.publicKey, + domains.binding, + readConfigurationSettlementCell(state.installation, projection, descriptors.sender[1]!), + configurationSettlementControlEnvelopeSchema, + ), + capacity = verifyControl( + state.installation.publicKeys["data-manager"], + domains.capacity, + owner.capacity, + configurationSettlementCapacitySchema, + ), + commitmentDigest = settlementRecordDigest(domains.commitment, committed), + { + operationFingerprint: _op, + fingerprint: _fp, + deadlineAt: _deadline, + ...address + } = envelope.execution, + slot = readConfigurationSettlementFamilySlot(state.installation, projection); + need( + same(binding, envelope) && + envelope.installationId === owner.installationId && + same(envelope.origin, owner.origin) && + same(envelope.ownerPrepare, state.execution) && + envelope.admissionDigest === state.receiptDigest && + envelope.rootCertificateDigest === tree.certificateDigest && + envelope.settlementId === projection.settlementId && + envelope.projectionDigest === settlementRecordDigest(domains.projection, projection) && + envelope.reservationDigest === projection.reservationDigest && + envelope.commitmentDigest === commitmentDigest && + same(envelope.operation, projection.operation) && + same(address, settle.execution) && + envelope.execution.operationFingerprint === settle.operationFingerprint && + same(envelope.serviceIds, owner.plan.serviceIds) && + envelope.execution.fingerprint === + executionFingerprint({ + kind: projection.operation.kind, + operationFingerprint: settle.operationFingerprint, + deadlineAt: envelope.controlDeadlineAt, + parent: { + issuer: "ops-agent", + operationKey: state.execution.operationKey, + fingerprint: state.execution.fingerprint, + deadlineAt: state.execution.deadlineAt, + execution: state.execution, + }, + }) && + slot.state === "bound" && + slot.executionDigest === controlDigest(envelope.execution) && + slot.controlBindingDigest === settlementRecordDigest(domains.binding, envelope), + "Original prepared logical binding changed", + ); + verifyPublishedBoundDmConfigurationSettlementSlots( + state.installation, + projection, + owner.capacity, + ); + const opened: AnchoredDirectory[] = [], + own = (directory: AnchoredDirectory) => { + opened.push(directory); + return directory; + }; + try { + const root = own(new AnchoredDirectory(state.installation.dataRoot)), + shared = own(root.directory(".admin-target-control")), + executions = own(shared.directory("executions")), + dm = own(executions.directory("data-manager")), + prepared = verifyControl( + state.installation.publicKeys["data-manager"], + dataConfigurationFencePreparedWitnessDomain, + JSON.parse( + dm.read(`${prepare.execution.operationKey}.data-terminal.json`, 8192).toString("utf8"), + ), + dataConfigurationFencePreparedWitnessSchema, + ), + terminalBytes = dm.read( + `${envelope.execution.operationKey}.configuration-settlement-terminal.json`, + 4096, + ), + signedTerminal = JSON.parse(terminalBytes.toString("utf8")), + terminal = verifyControl( + state.installation.publicKeys["data-manager"], + domains.terminal, + signedTerminal, + configurationSettlementPublishedTerminalSchema, + ), + tuple = terminal.tuple, + guard = owner.guardSource, + physical = tree.children.find((value) => same(value.execution, guard.prepareExecution)); + need( + terminalBytes.length === 4096 && + same(prepared, guard) && + physical && + physical.settlement.outcome === "committed" && + physical.settlement.publication === "committed" && + physical.settlement.terminalReceiptDigest === guard.effectReceiptDigest && + guard.effectSettlementDigest === + controlDigest({ + ...physical.settlement, + receipt: { + kind: "data-configuration-fence", + receipt: guard.preparedReceipt, + guardDigest: guard.guardDigest, + }, + }), + "Original actual prepared child proof changed", + ); + const admissionPayload = configurationSettlementAdmissionSchema.parse({ + version: 2, + purpose: "configuration-fence-settlement", + installationId: owner.installationId, + origin: owner.origin, + settlementId: projection.settlementId, + projectionDigest: envelope.projectionDigest, + reservationDigest: envelope.reservationDigest, + prepareExecutionDigest: controlDigest(guard.prepareExecution), + prepareAdmissionDigest: guard.prepareAdmissionDigest, + preparedReceiptDigest: guard.preparedReceiptDigest, + capacityReceiptDigest: settlementRecordDigest(domains.capacity, capacity), + rootCertificateDigest: tree.certificateDigest, + controlEnvelope: control.binding, + commitmentDigest, + executionOrdinal: capacity.reservedExecutionOrdinal, + intentOrdinal: capacity.reservedIntentOrdinal, + sqlSessionNonce: envelope.execution.executionId, + boundAt: terminal.admissionSource.boundAt, + }), + admission = verifyControl( + state.installation.publicKeys["data-manager"], + domains.admission, + { payload: admissionPayload, signature: terminal.admissionSource.signature }, + configurationSettlementAdmissionSchema, + ), + effect = verifyControl( + state.installation.publicKeys["data-manager"], + domains.effect, + { + payload: { ...terminal.effectSource.fields, terminal: tuple }, + signature: terminal.effectSource.signature, + }, + configurationSettlementEffectSchema, + ), + { terminalDigest, ...tuplePreimage } = tuple; + need( + same(terminal.origin, owner.origin) && + same(tuple.execution, envelope.execution) && + tuple.settlementId === projection.settlementId && + tuple.commitmentDigest === commitmentDigest && + tuple.fenceId === guard.preparedReceipt.fenceId && + same(tuple.result, { + ...guard.preparedReceipt, + state: "settled", + sequence: guard.preparedReceipt.sequence + 1, + }) && + tuple.terminalAt >= envelope.controlStartedAt && + tuple.terminalAt < envelope.controlDeadlineAt && + terminalDigest === settlementRecordDigest(domains.terminal, tuplePreimage) && + terminal.admissionDigest === settlementRecordDigest(domains.admission, admission) && + admission.boundAt >= envelope.controlStartedAt && + admission.boundAt < envelope.controlDeadlineAt && + effect.phase === "committed" && + same(effect.origin, owner.origin) && + effect.settlementId === projection.settlementId && + effect.controlExecutionDigest === controlDigest(envelope.execution) && + effect.controlEnvelopeDigest === settlementRecordDigest(domains.binding, envelope) && + effect.admissionDigest === terminal.admissionDigest && + effect.commitmentDigest === commitmentDigest && + effect.preparedReceiptDigest === guard.preparedReceiptDigest && + effect.expectedDmSequence === guard.preparedReceipt.sequence && + effect.sqlSession.sessionNonce === envelope.execution.executionId && + effect.attemptedAt >= admission.boundAt && + effect.attemptedAt < envelope.controlDeadlineAt && + terminal.effectSettlementDigest === settlementRecordDigest(domains.effect, effect), + "Actual original logical guard settlement required", + ); + const copied = configurationSettlementSenderAckCellSchema.parse( + readConfigurationSettlementCell(state.installation, projection, descriptors.sender[4]!), + ), + sharedAck = readConfigurationSettlementCell( + state.installation, + projection, + descriptors.sender[5]!, + ); + need( + same(copied.terminal, signedTerminal) && copied.ack && same(copied.ack, sharedAck), + "Original durable sender copy/ACK required", + ); + const ack = verifyControl( + state.installation.installation.publicKey, + domains.ack, + sharedAck, + configurationSettlementAckSchema, + ); + need( + ack.installationId === owner.installationId && + same(ack.origin, owner.origin) && + ack.settlementId === projection.settlementId && + ack.controlExecutionDigest === controlDigest(envelope.execution) && + ack.commitmentDigest === commitmentDigest && + ack.terminalDigest === tuple.terminalDigest && + ack.terminalAt === tuple.terminalAt, + "Original exact sender settlement ACK changed", + ); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +export function assertConfigurationMaintenanceRootClosure( + authority: VerifiedExecutionAuthority, + observed: ConfigurationMaintenanceClosureObservation, +): void { + const state = executionAuthorityState(authority), + tree = configurationMaintenanceClosureObservationState(observed, authority); + need( + state.kind === "configurationMaintenance.prepare" && + state.installation.installation.role === "ops-agent" && + !state.installation.fixedChildId && + state.origin, + "Original maintenance root closure source required", + ); + const admission = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { maintenanceId?: string } } }, + name = `${z.uuid().parse(admission.payload?.operation?.maintenanceId)}.json`, + raw = readPrivateRecord(state.installation, "configuration-maintenance", name), + owner = verifyControl( + state.installation.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + z + .object({ + installationId: z.uuid(), + origin: controlOriginSchema, + admissionDigest: hash, + rootCertificateDigest: hash, + sequence: z.number(), + plan: z + .object({ + prepare: executionRefSchema, + planDigest: hash, + operation: z.object({ maintenanceId: z.uuid(), candidateId: z.uuid() }).passthrough(), + }) + .passthrough(), + receipt: z + .object({ + state: z.string(), + dmFence: z.unknown(), + generation: z.string(), + decision: z.unknown(), + }) + .passthrough(), + guardSource: z.unknown(), + final: z.unknown(), + inspection: z.unknown(), + issuance: z.string(), + noGuard: configurationNoGuardIntentSchema.nullable(), + }) + .passthrough(), + ); + verifyFamilyOwnedRecord( + state.installation, + state.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name }, + canonicalJson(raw), + ); + if (owner.guardSource !== null || owner.final !== null || owner.inspection !== null) { + need( + owner.guardSource !== null && owner.final !== null && owner.inspection !== null, + "Conclusive original maintenance guard/NoGuard closure required", + ); + assertPreparedAndSettledClosure(authority, observed, owner); + return; + } + // A genuinely never-prepared root uses the disjoint original NoGuard source. + need( + owner.receipt.state === "settled" && + owner.receipt.dmFence === null && + owner.receipt.generation === "not_started" && + owner.guardSource === null && + owner.final === null && + owner.inspection === null && + owner.issuance === "closed_without_guard" && + owner.noGuard, + "Conclusive original maintenance guard/NoGuard closure required", + ); + need( + owner.installationId === state.installation.installation.installationId && + owner.admissionDigest === state.receiptDigest && + same(owner.origin, state.origin) && + same(owner.plan.prepare, state.execution) && + owner.rootCertificateDigest === tree.certificateDigest, + "Original maintenance closure identity changed", + ); + const prepare = state.reservations.find((value) => value.phase === "configuration-fence"), + settle = state.reservations.find((value) => value.phase === "configuration-fence-settle"); + need(prepare && settle, "Original finite prepare/settlement reservations required"); + const provisional = { origin: state.origin }, + descriptors = configurationSettlementRecordDescriptors( + { ownerPrepare: state.execution, reservation: settle }, + prepare.execution.operationKey, + ), + control = verifyControl( + state.installation.installation.publicKey, + domains.control, + readConfigurationSettlementCell(state.installation, provisional, descriptors.sender[2]!), + configurationSettlementControlCellSchema.shape.payload, + ), + projection = verifyControl( + state.installation.installation.publicKey, + domains.signedProjection, + control.projection, + configurationSettlementReservationProjectionSchema, + ), + projectionDigest = settlementRecordDigest(domains.projection, projection), + bundle = verifyControl( + state.installation.installation.publicKey, + domains.reference, + readConfigurationSettlementCell(state.installation, projection, descriptors.sender[0]!), + configurationSettlementCommitmentCellSchema.shape.payload, + ); + need( + bundle.phase === "unused" && control.binding === null && "unused" in control, + "Both original NoGuard cells required", + ); + configurationNoGuardBundleSchema.parse(bundle); + const closure = verifyControl( + state.installation.installation.publicKey, + domains.closure, + bundle.closure, + configurationNoGuardClosureSchema, + ), + identity = closure.identity, + { closureDigest: _digest, ...closureSource } = closure, + intent = owner.noGuard, + slot = readConfigurationSettlementFamilySlot(state.installation, projection), + evidence = closure.noGuardEvidence; + need( + same(bundle, intent.bundle?.payload) && + bundle.settlementId === projection.settlementId && + bundle.projectionDigest === projectionDigest && + projection.installationId === owner.installationId && + same(projection.origin, state.origin) && + same(projection.ownerPrepare, state.execution) && + projection.rootAdmissionDigest === state.receiptDigest && + projection.rootCertificateDigest === tree.certificateDigest && + projection.maintenanceId === owner.plan.operation.maintenanceId && + projection.candidateId === owner.plan.operation.candidateId && + projection.planDigest === owner.plan.planDigest && + same(projection.reservation, settle) && + projection.reservationDigest === settlementRecordDigest(domains.reservationTuple, settle) && + projection.reservation.operationFingerprint === controlDigest(projection.operation) && + projection.operation.maintenanceId === projection.maintenanceId && + projection.operation.candidateId === projection.candidateId && + projection.operation.planDigest === projection.planDigest && + projection.operation.settlementId === projection.settlementId && + projection.settlementId === + deriveConfigurationFenceSettlementId({ + installationId: owner.installationId, + ownerPrepare: state.execution, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + }) && + same(identity, { + installationId: owner.installationId, + origin: state.origin, + ownerPrepare: state.execution, + admissionDigest: state.receiptDigest, + rootCertificateDigest: tree.certificateDigest, + maintenanceId: projection.maintenanceId, + candidateId: projection.candidateId, + planDigest: projection.planDigest, + settlementId: projection.settlementId, + }) && + closure.ownerSequence === owner.sequence && + closure.ownerSequence === intent.ownerSequence && + closure.closedAt === intent.closedAt && + same(evidence, intent.noGuardEvidence) && + closure.closureDigest === settlementRecordDigest(domains.closure, closureSource) && + same(bundle.issuerFreezeProof, intent.issuerFreezeProof) && + same(bundle.issuerUnused, intent.issuerUnused) && + slot.role === "ops-agent" && + slot.state === "UNUSED" && + slot.executionDigest === null && + slot.controlBindingDigest === null, + "Original NoGuard bundle/identity/source changed", + ); + const freezeDomain = + evidence.kind === "never_issued" ? domains.neverIssued : domains.issuedRefused, + freezeProof = + evidence.kind === "never_issued" + ? verifyControl( + state.installation.installation.publicKey, + freezeDomain, + bundle.issuerFreezeProof, + configurationNeverIssuedProofSchema, + ) + : verifyControl( + state.installation.installation.publicKey, + freezeDomain, + bundle.issuerFreezeProof, + configurationIssuedPrepareRefusedProofSchema, + ), + issued = evidence.kind === "receiver_refused", + expectedPublications = state.reservations + .filter((value) => !issued || value !== prepare) + .map((value) => { + const actual = inspectControlChildPublication(authority, value); + need( + actual.publication.state === "unissued" && + lookupControlChildPhaseAttempt(authority, value) === null, + "Original frozen phase was issued/attempted", + ); + return { + phase: value.phase, + ordinal: value.ordinal, + operationKey: value.execution.operationKey, + grantState: "unissued", + intentState: "not_attempted", + }; + }); + need( + same(freezeProof.identity, identity) && + freezeProof.issuerSequence === owner.sequence && + freezeProof.frozenAt === closure.closedAt && + evidence.issuerFreezeProofDigest === settlementRecordDigest(freezeDomain, freezeProof) && + same( + expectedPublications, + "remainingPublicationStates" in freezeProof + ? freezeProof.remainingPublicationStates + : freezeProof.publicationStates, + ), + "Original frozen issuer preimages changed", + ); + for (const descriptor of descriptors.sender.filter( + (value) => value.scope === "private" || value.name.endsWith("-ack.json"), + )) + need( + readConfigurationSettlementCell(state.installation, projection, descriptor) === null, + "Bound/attempted control cannot close UNUSED", + ); + const ownIntent = verifyControl( + state.installation.installation.publicKey, + domains.unusedIntent, + bundle.issuerUnused.intent, + configurationUnusedCapacityIntentSchema, + ), + ownConfirmation = verifyControl( + state.installation.installation.publicKey, + domains.unusedConfirmation, + bundle.issuerUnused.confirmation, + configurationUnusedCapacityConfirmationSchema, + ), + ownIntentDigest = settlementRecordDigest(domains.unusedIntent, ownIntent), + ownConfirmationDigest = settlementRecordDigest(domains.unusedConfirmation, ownConfirmation), + safeSlot = { + executionOrdinal: slot.executionOrdinal, + intentOrdinal: slot.intentOrdinal, + recordDescriptorsDigest: slot.recordDescriptorsDigest, + allocatedDescriptorDigests: slot.allocatedDescriptorDigests, + state: "UNUSED", + intentDigest: ownIntentDigest, + }; + need( + ownIntent.role === "ops-agent" && + ownIntent.installationId === owner.installationId && + same(ownIntent.origin, state.origin) && + ownIntent.ownerPrepareDigest === controlDigest(state.execution) && + ownIntent.settlementId === projection.settlementId && + ownIntent.projectionDigest === projectionDigest && + ownIntent.prepareExecutionDigest === + (evidence.kind === "receiver_refused" ? evidence.prepareExecutionDigest : null) && + ownIntent.capacityReceiptDigest === + (evidence.kind === "receiver_refused" ? evidence.capacityReceiptDigest : null) && + ownIntent.sourceProofDigest === configurationIssuerUnusedSourceDigest(evidence) && + ownIntent.executionOrdinal === slot.executionOrdinal && + ownIntent.intentOrdinal === slot.intentOrdinal && + ownIntent.recordDescriptorsDigest === controlDigest(descriptors.sender) && + same( + ownIntent.allocatedDescriptorDigests, + descriptors.sender.map((value) => controlDigest(value)).sort(), + ) && + slot.unusedIntentDigest === ownIntentDigest && + ownConfirmation.role === "ops-agent" && + ownConfirmation.installationId === owner.installationId && + same(ownConfirmation.origin, state.origin) && + ownConfirmation.ownerPrepareDigest === ownIntent.ownerPrepareDigest && + ownConfirmation.settlementId === projection.settlementId && + ownConfirmation.projectionDigest === projectionDigest && + ownConfirmation.intentDigest === ownIntentDigest && + ownConfirmation.slotStateDigest === controlDigest(safeSlot) && + ownConfirmation.confirmedAt === slot.unusedConfirmedAt && + ownConfirmation.confirmedAt >= ownIntent.createdAt, + "Original issuer UNUSED CAS/confirmation changed", + ); + const opened: AnchoredDirectory[] = [], + own = (value: AnchoredDirectory) => { + opened.push(value); + return value; + }; + let dmContainment: string | undefined, + dmUnused: + | { role: "data-manager"; intentDigest: string; confirmationDigest: string } + | undefined; + try { + const descriptor = descriptors.receiver[0]!; + if (evidence.kind === "never_issued") { + need( + !("kind" in freezeProof) && + tree.children.length === 0 && + evidence.prepareReservationDigest === controlDigest(prepare) && + evidence.publicationStatesDigest === + settlementRecordDigest(domains.neverIssued, freezeProof) && + evidence.issuerSequence === owner.sequence && + evidence.frozenAt === closure.closedAt, + "Original NeverIssued source changed", + ); + assertNoUnissuedConfigurationReceiverPublication( + state.installation, + prepare.execution.operationKey, + ); + } else { + need("kind" in freezeProof, "Issued preparation cannot use NeverIssued proof"); + const root = own(new AnchoredDirectory(state.installation.dataRoot)), + target = own(root.directory(".admin-target-control")), + executions = own(target.directory("executions")), + dm = own(executions.directory("data-manager")), + preparePublication = inspectControlChildPublication(authority, prepare), + attempt = preparePublication.binding?.child, + bytes = dm.read(descriptor.name, descriptor.cap), + cell = configurationDmCapacityCellSchema.parse(JSON.parse(bytes.toString("utf8"))), + key = state.installation.publicKeys["data-manager"]; + need( + bytes.length === descriptor.cap && + cell.refusal && + cell.unused?.confirmation && + attempt && + tree.children.length === 1 && + preparePublication.publication.state === "grant_prepared" && + preparePublication.publication.grantDigest === freezeProof.prepareGrantDigest, + "Actual original issued/refused DM packet required", + ); + const outbox = readPrivateRecord( + state.installation, + "attempts", + `${preparePublication.publication.delegationId}.json`, + 65536, + ) as { grant: unknown }, + refusal = verifyControl( + key, + domains.refusal, + cell.refusal, + configurationDmNoGuardRefusalSchema, + ), + capacity = + cell.capacity && + verifyControl( + key, + domains.capacity, + cell.capacity, + configurationSettlementCapacitySchema, + ), + refusalDigest = settlementRecordDigest(domains.refusal, refusal), + child = tree.children[0]!, + witness = readControlExecutionWitness(state.installation, attempt.execution, key), + unusedIntent = verifyControl( + key, + domains.unusedIntent, + cell.unused.intent, + configurationUnusedCapacityIntentSchema, + ), + unusedConfirmation = verifyControl( + key, + domains.unusedConfirmation, + cell.unused.confirmation, + configurationUnusedCapacityConfirmationSchema, + ), + unusedIntentDigest = settlementRecordDigest(domains.unusedIntent, unusedIntent), + unusedConfirmationDigest = settlementRecordDigest( + domains.unusedConfirmation, + unusedConfirmation, + ), + expectedAllocated = (capacity ? descriptors.receiver : [descriptor]) + .map((value) => controlDigest(value)) + .sort(), + dmSlot = { + executionOrdinal: capacity?.reservedExecutionOrdinal ?? null, + intentOrdinal: capacity?.reservedIntentOrdinal ?? null, + recordDescriptorsDigest: controlDigest(descriptors.receiver), + allocatedDescriptorDigests: expectedAllocated, + state: "UNUSED", + intentDigest: unusedIntentDigest, + }; + need( + child.grantDigest === freezeProof.prepareGrantDigest && + controlDigest(outbox.grant) === child.grantDigest && + same(child.execution, attempt.execution) && + child.settlement.outcome === "failed_without_effect" && + child.settlement.publication === "not_started" && + same(refusal.prepareExecution, attempt.execution) && + refusal.prepareAdmissionDigest === witness.authorityDigest && + witness.kind === "configuration-maintenance.prepare" && + same(witness.origin, state.origin) && + refusal.installationId === owner.installationId && + same(refusal.origin, state.origin) && + same(refusal.ownerPrepare, state.execution) && + refusal.rootAdmissionDigest === state.receiptDigest && + refusal.rootCertificateDigest === tree.certificateDigest && + refusal.maintenanceId === projection.maintenanceId && + refusal.candidateId === projection.candidateId && + refusal.planDigest === projection.planDigest && + refusal.settlementId === projection.settlementId && + refusal.projectionDigest === projectionDigest && + refusal.reservationDigest === projection.reservationDigest && + refusal.refusalReceipt.maintenanceId === projection.maintenanceId && + refusal.effectReceiptDigest === controlDigest(refusal.refusalReceipt) && + child.settlement.terminalReceiptDigest === refusal.effectReceiptDigest && + refusal.effectSettlementDigest === + controlDigest({ ...child.settlement, receipt: refusal.refusalReceipt }) && + refusal.canonicalTerminalDigest === + controlDigest({ version: 1, refused: refusal.refusalReceipt }) && + evidence.prepareExecutionDigest === controlDigest(attempt.execution) && + evidence.prepareAdmissionDigest === refusal.prepareAdmissionDigest && + evidence.refusalProjectionDigest === refusalDigest && + evidence.effectSettlementDigest === refusal.effectSettlementDigest && + evidence.canonicalTerminalDigest === refusal.canonicalTerminalDigest && + evidence.canonicalTerminalAt === refusal.canonicalTerminalAt && + evidence.capacityReceiptDigest === refusal.capacityReceiptDigest && + evidence.dmUnusedIntentDigest === unusedIntentDigest && + evidence.dmUnusedConfirmationDigest === unusedConfirmationDigest && + freezeProof.prepareExecutionDigest === evidence.prepareExecutionDigest && + freezeProof.refusalProjectionDigest === refusalDigest && + unusedIntent.role === "data-manager" && + unusedIntent.installationId === owner.installationId && + same(unusedIntent.origin, state.origin) && + unusedIntent.ownerPrepareDigest === controlDigest(state.execution) && + unusedIntent.prepareExecutionDigest === evidence.prepareExecutionDigest && + unusedIntent.settlementId === projection.settlementId && + unusedIntent.projectionDigest === projectionDigest && + unusedIntent.capacityReceiptDigest === refusal.capacityReceiptDigest && + unusedIntent.sourceProofDigest === refusalDigest && + unusedIntent.executionOrdinal === dmSlot.executionOrdinal && + unusedIntent.intentOrdinal === dmSlot.intentOrdinal && + unusedIntent.recordDescriptorsDigest === dmSlot.recordDescriptorsDigest && + same(unusedIntent.allocatedDescriptorDigests, expectedAllocated) && + unusedConfirmation.role === "data-manager" && + unusedConfirmation.installationId === owner.installationId && + same(unusedConfirmation.origin, state.origin) && + unusedConfirmation.ownerPrepareDigest === controlDigest(state.execution) && + unusedConfirmation.settlementId === projection.settlementId && + unusedConfirmation.projectionDigest === projectionDigest && + unusedConfirmation.intentDigest === unusedIntentDigest && + unusedConfirmation.slotStateDigest === controlDigest(dmSlot) && + unusedConfirmation.confirmedAt >= unusedIntent.createdAt, + "Actual DM refusal/UNUSED preimages changed", + ); + if (capacity) + need( + capacity.installationId === owner.installationId && + same(capacity.origin, state.origin) && + capacity.ownerPrepareDigest === controlDigest(state.execution) && + capacity.settlementId === projection.settlementId && + capacity.projectionDigest === projectionDigest && + capacity.reservationDigest === projection.reservationDigest && + same(capacity.proposedExecutionAddress, projection.reservation.execution) && + capacity.recordDescriptorsDigest === controlDigest(descriptors.receiver) && + capacity.maxBytes === descriptors.receiver.reduce((sum, value) => sum + value.cap, 0) && + refusal.capacityReceiptDigest === settlementRecordDigest(domains.capacity, capacity), + "Original DM capacity certificate changed", + ); + else + need( + refusal.capacityReceiptDigest === null && refusal.refusalReceipt.reason === "capacity", + "Actual capacity-null refusal required", + ); + verifyPublishedUnusedDmConfigurationSettlementSlots(state.installation, projection, { + executionOrdinal: dmSlot.executionOrdinal, + intentOrdinal: dmSlot.intentOrdinal, + projectionDigest, + intentDigest: unusedIntentDigest, + }); + dmContainment = refusal.effectSettlementDigest; + dmUnused = { + role: "data-manager", + intentDigest: unusedIntentDigest, + confirmationDigest: unusedConfirmationDigest, + }; + } + } finally { + for (const directory of opened.reverse()) directory.close(); + } + verifyControlEffectSettlement(state.installation.installation, state.execution); + const ownEffect = readPrivateRecord( + state.installation, + "effects", + `${state.execution.operationKey}.json`, + 65536, + ) as { + payload: { + outcome: string; + publication: string; + receipt: unknown; + }; + }; + need( + ownEffect.payload.outcome === "failed_without_effect" && + ownEffect.payload.publication === "not_started" && + same(ownEffect.payload.receipt, { + kind: "configuration-no-guard", + issuerFreezeProofDigest: evidence.issuerFreezeProofDigest, + issuerUnusedConfirmationDigest: ownConfirmationDigest, + }) && + same( + closure.originalContainmentDigests, + [controlDigest(ownEffect.payload), ...(dmContainment ? [dmContainment] : [])].sort(), + ) && + same(closure.unusedSlotDigests, [ + ...(dmUnused ? [dmUnused] : []), + { + role: "ops-agent", + intentDigest: ownIntentDigest, + confirmationDigest: ownConfirmationDigest, + }, + ]) && + same(control.unused, { + bundleDigest: settlementRecordDigest(domains.bundle, bundle), + closureDigest: closure.closureDigest, + issuerFreezeProofDigest: evidence.issuerFreezeProofDigest, + issuerUnusedIntentDigest: ownIntentDigest, + issuerUnusedConfirmationDigest: ownConfirmationDigest, + }), + "Original own containment/bundle links changed", + ); +} diff --git a/packages/core/src/ops/api-job-attempt.ts b/packages/core/src/ops/api-job-attempt.ts new file mode 100644 index 000000000..b55bf6688 --- /dev/null +++ b/packages/core/src/ops/api-job-attempt.ts @@ -0,0 +1,16 @@ +/** Exact portable decoding of the existing API HMAC parent preimage. + * Schema property order is part of that existing signed encoding. */ +import { z } from "zod"; +export const apiJobChildAttemptSchema = z.strictObject({ + version: z.literal(1), + apiJobId: z.uuid(), + apiOwnerVersion: z.number().int().positive().safe(), + parentOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + parentFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + parentDeadlineAt: z.number().int().nonnegative().safe(), + childOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + childOperationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + attemptId: z.uuid(), +}); +export type ApiJobChildAttempt = z.infer; diff --git a/packages/core/src/ops/canonical-json.ts b/packages/core/src/ops/canonical-json.ts new file mode 100644 index 000000000..38e5eda92 --- /dev/null +++ b/packages/core/src/ops/canonical-json.ts @@ -0,0 +1,12 @@ +/** Canonical JSON only; no schema, authority or journal dependencies. */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .filter(([, child]) => child !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, child]) => `${JSON.stringify(key)}:${canonicalJson(child)}`) + .join(",")}}`; + } + return JSON.stringify(value); +} diff --git a/packages/core/src/ops/client.test.ts b/packages/core/src/ops/client.test.ts index bbfed494f..8cbdf032f 100644 --- a/packages/core/src/ops/client.test.ts +++ b/packages/core/src/ops/client.test.ts @@ -42,6 +42,26 @@ class ProtocolTestClient extends OpsClient { } describe("ops client", () => { + it.each(Object.entries(OPS_PUBLIC_ERROR_MESSAGES))( + "preserves the existing canonical %s refusal class", + async (errorClass, message) => { + const client = new ProtocolTestClient({ + baseUrl: "http://127.0.0.1:3002", + tokenFile: tokenFile(), + fetch: async (_url, init) => + new Response( + JSON.stringify({ + version: 1, + requestId: JSON.parse(String(init?.body)).requestId, + ok: false, + error: { class: errorClass, message }, + }), + { status: 409 }, + ), + }); + await expect(client.execute({ kind: "docker.status" })).rejects.toMatchObject({ errorClass }); + }, + ); it("derives synchronous transport timeouts from the per-kind execution budget", () => { expect(opsRequestTimeoutMs("docker.status")).toBe(15_000); expect(opsRequestTimeoutMs("service.logs")).toBe(15_000); @@ -291,14 +311,32 @@ describe("ops client", () => { fetch, }); - await expect(client.execute({ kind: "service.pull", serviceId: "redis" })).rejects.toThrow( - "Async operation key is required", - ); + await expect( + client.execute({ + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }), + ).rejects.toThrow("Async operation key is required"); expect(fetch).not.toHaveBeenCalled(); const operationKey = client.createOperationKey(); - await client.execute({ kind: "service.pull", serviceId: "redis" }, { operationKey }); - await client.execute({ kind: "service.pull", serviceId: "redis" }, { operationKey }); + await client.execute( + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, + { operationKey }, + ); + await client.execute( + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, + { operationKey }, + ); expect(requests.map((request) => request.operationKey)).toEqual([operationKey, operationKey]); }); @@ -427,7 +465,11 @@ describe("ops client", () => { }); const admissionAbort = new AbortController(); const admission = client.execute( - { kind: "service.pull", serviceId: "redis" }, + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, { operationKey: "opk1_signalAdmission000", signal: admissionAbort.signal }, ); await vi.waitFor(() => expect(signals).toHaveLength(1)); @@ -486,7 +528,11 @@ describe("ops client", () => { for (const [stateIndex, state] of states.entries()) { await expect( client.execute( - { kind: "service.pull", serviceId: "redis" }, + { + kind: "service.pull", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, { operationKey: `opk1_retainedClient${stateIndex}00` }, ), ).resolves.toMatchObject({ state }); diff --git a/packages/core/src/ops/client.ts b/packages/core/src/ops/client.ts index c93ffe4d6..e3e4a8ecd 100644 --- a/packages/core/src/ops/client.ts +++ b/packages/core/src/ops/client.ts @@ -40,10 +40,12 @@ const ERROR_CLASSES = new Set([ "timeout", "busy", "conflict", + "configuration_prerequisite", "not_found", "not_wired", "recovery_required", "host_update_required", + "control_upgrade_required", "runtime", ]); diff --git a/packages/core/src/ops/configuration-maintenance-control-validation.ts b/packages/core/src/ops/configuration-maintenance-control-validation.ts new file mode 100644 index 000000000..3f4a0bae0 --- /dev/null +++ b/packages/core/src/ops/configuration-maintenance-control-validation.ts @@ -0,0 +1,46 @@ +/** Exact metadata-control timing validation; separate from portable receipt + * schemas so operation policy cannot become a schema initialization cycle. */ +import type { + ConfigurationMaintenanceDecision, + ConfigurationMaintenanceSettle, +} from "./configuration-maintenance"; +import { + executionFingerprint, + type RetainedExecutionAdmission, + retainedParentExecutionSchema, +} from "./execution-identity"; +import { opsOperationFingerprint } from "./fingerprint"; + +/** Data validation ONLY for the closed no-effect metadata controls. The + * receiver still requires the actual original-owner recovery capability and + * live exact API HMAC. Generic forward normalization remains unchanged. */ +export function validateRetainedConfigurationMaintenanceRecoveryControl( + operation: ConfigurationMaintenanceDecision | ConfigurationMaintenanceSettle, + input: RetainedExecutionAdmission, +): void { + if (operation.kind === "configurationMaintenance.decide" && operation.decision !== "abandon") + throw Error("Forward continue is not a recovery control"); + const parent = retainedParentExecutionSchema.parse(input.parent); + if ( + parent.issuer !== "api" || + input.executionVersion !== 2 || + input.operationFingerprint !== opsOperationFingerprint(operation) || + ![input.admittedAt, input.deadlineAt, input.requestedDeadlineAt].every( + (value) => Number.isSafeInteger(value) && value! >= 0, + ) || + !Number.isSafeInteger(input.deploymentBudgetMs) || + input.deploymentBudgetMs < 1 || + input.deploymentBudgetMs > 60000 || + input.deadlineAt <= input.admittedAt || + input.deadlineAt !== + Math.min(input.requestedDeadlineAt!, input.admittedAt + input.deploymentBudgetMs) || + input.fingerprint !== + executionFingerprint({ + kind: operation.kind, + operationFingerprint: input.operationFingerprint, + deadlineAt: input.deadlineAt, + parent, + }) + ) + throw Error("Invalid original maintenance recovery control timing"); +} diff --git a/packages/core/src/ops/configuration-maintenance.test.ts b/packages/core/src/ops/configuration-maintenance.test.ts new file mode 100644 index 000000000..76dbdd3fd --- /dev/null +++ b/packages/core/src/ops/configuration-maintenance.test.ts @@ -0,0 +1,220 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { canonicalJson } from "./canonical-json"; +import * as maintenance from "./index"; + +// Portable data vectors only: no fixture is admission or settlement authority. +function receipt(): maintenance.ConfigurationMaintenanceReceipt { + const prepare = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: "job1_originalPrepare0001", + operationKey: "opk1_originalPrepare0001", + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: 200000, + }; + const binding = { + maintenanceId: "11111111-1111-4111-8111-111111111111", + candidateId: "22222222-2222-4222-8222-222222222222", + applyOperationKey: "opk1_originalApply000001", + applyOperationFingerprint: "c".repeat(64), + expectedCurrentRevisionId: `cfg1_${"d".repeat(43)}`, + revisionId: `cfg1_${"e".repeat(43)}`, + expectedSourceRevision: "33333333-3333-4333-8333-333333333333", + coordinatorMode: "ensure-running" as const, + followUp: "service-apply" as const, + }; + return { + version: 1, + binding, + prepare, + apiJobId: "44444444-4444-4444-8444-444444444444", + parentOperationKey: "opk1_originalApiParent01", + parentFingerprint: "f".repeat(64), + parentDeadlineAt: 300000, + planDigest: "1".repeat(64), + serviceIds: ["data-manager", "motis"], + steps: [ + { + phase: "configuration-commit", + ordinal: 0, + operation: { + kind: "vault.apply", + serviceId: "data-manager", + revisionId: binding.revisionId, + expectedCurrentRevisionId: binding.expectedCurrentRevisionId, + expectedSourceRevision: binding.expectedSourceRevision, + coordinatorMode: binding.coordinatorMode, + }, + operationKey: binding.applyOperationKey, + operationFingerprint: binding.applyOperationFingerprint, + serviceIds: ["data-manager", "motis"], + prerequisites: [], + }, + ], + preflight: "current", + sequence: 2, + state: "authorized", + decision: { + decisionId: "55555555-5555-4555-8555-555555555555", + decision: "continue", + checkpointVersion: 1, + checkpointDigest: "2".repeat(64), + }, + dmFence: { + fenceId: "66666666-6666-4666-8666-666666666666", + maintenanceId: binding.maintenanceId, + candidateId: binding.candidateId, + ownerPrepare: prepare, + planDigest: "1".repeat(64), + revisionId: binding.revisionId, + hardlinkPlanDigest: "3".repeat(64), + sequence: 0, + state: "prepared", + }, + generation: "not_started", + appliedRevisionId: null, + followUp: "not_started", + }; +} +const checksum = (r: maintenance.ConfigurationMaintenanceReceipt) => + maintenance.configurationMaintenanceSettleRequestDigest(r); +describe("outer maintenance request observation checksum", () => { + it("hashes the exact approved projection/domain independent of JSON object order", () => { + const r = receipt(), + { preflight: _preflight, state: _state, ...fields } = r, + projection = { ...fields, steps: r.steps.map(({ operation: _operation, ...step }) => step) }, + expected = createHash("sha256") + .update( + "openmapx-configuration-settle-request-evidence-v1\0" + canonicalJson(projection), + "utf8", + ) + .digest("hex"); + expect(checksum(r)).toBe(expected); + expect(checksum(r)).toMatch(/^[a-f0-9]{64}$/); + const reverseKeys = (value: unknown): unknown => + Array.isArray(value) + ? value.map(reverseKeys) + : value && typeof value === "object" + ? Object.fromEntries( + Object.entries(value) + .reverse() + .map(([key, child]) => [key, reverseKeys(child)]), + ) + : value; + expect(checksum(reverseKeys(r) as typeof r)).toBe(expected); + expect(expected).not.toBe( + createHash("sha256") + .update("openmapx-configuration-final-evidence-v2\0" + canonicalJson(projection)) + .digest("hex"), + ); + }); + it.each([ + (r: maintenance.ConfigurationMaintenanceReceipt) => { + r.binding.expectedCurrentRevisionId = null; + }, + (r) => { + r.binding.candidateId = "77777777-7777-4777-8777-777777777777"; + }, + (r) => { + r.prepare.deadlineAt++; + }, + (r) => { + r.apiJobId = "77777777-7777-4777-8777-777777777777"; + }, + (r) => { + r.parentOperationKey = "opk1_differentApiParent1"; + }, + (r) => { + r.parentFingerprint = "4".repeat(64); + }, + (r) => { + r.parentDeadlineAt++; + }, + (r) => { + r.planDigest = "4".repeat(64); + }, + (r) => { + r.serviceIds = ["data-manager", "motis", "valhalla"]; + }, + (r) => { + r.steps[0]!.operationKey = "opk1_differentStep000001"; + }, + (r) => { + r.steps[0]!.operationFingerprint = "4".repeat(64); + }, + (r) => { + r.steps[0]!.serviceIds = ["data-manager"]; + }, + (r) => { + r.sequence++; + }, + (r) => { + r.decision!.checkpointVersion++; + }, + (r) => { + r.decision!.checkpointDigest = "4".repeat(64); + }, + (r) => { + r.decision = null; + }, + (r) => { + r.dmFence!.sequence++; + }, + (r) => { + r.dmFence!.hardlinkPlanDigest = null; + }, + (r) => { + r.dmFence = null; + }, + (r) => { + r.generation = "committed"; + }, + (r) => { + r.appliedRevisionId = r.binding.revisionId; + }, + (r) => { + r.followUp = "contained"; + }, + ])("changes for each included observation field (%#)", (change) => { + const r = receipt(), + original = checksum(r); + change(r); + expect(checksum(r)).not.toBe(original); + }); + it("omits only derived state/preflight and frozen operation bodies", () => { + const r = receipt(), + original = checksum(r); + r.preflight = "unavailable"; + r.state = "recovery_required"; + expect(checksum(r)).toBe(original); + r.steps[0]!.operation = { + ...r.steps[0]!.operation, + serviceId: "motis", + } as maintenance.ConfigurationMaintenanceEffect; + expect(checksum(r)).toBe(original); + }); + it("strictly rejects malformed and oversized receipts before hashing", () => { + expect(() => + checksum({ ...receipt(), callerFlag: true } as maintenance.ConfigurationMaintenanceReceipt), + ).toThrow(); + const malformed = receipt(); + malformed.steps[0]!.ordinal = 1; + expect(() => checksum(malformed)).toThrow(); + const oversized = receipt(); + oversized.steps = Array.from({ length: 32 }, (_, ordinal) => ({ + ...oversized.steps[0]!, + ordinal, + phase: ordinal === 0 ? "configuration-commit" : "maintenance-recreate", + serviceIds: [...oversized.serviceIds], + prerequisites: Array.from({ length: ordinal }, (_, index) => index), + })); + oversized.serviceIds = Array.from( + { length: 256 }, + (_, index) => `service-${String(index).padStart(3, "0")}-${"x".repeat(48)}`, + ); + for (const step of oversized.steps) step.serviceIds = [...oversized.serviceIds]; + expect(() => checksum(oversized)).toThrow(); + }); +}); diff --git a/packages/core/src/ops/configuration-maintenance.ts b/packages/core/src/ops/configuration-maintenance.ts new file mode 100644 index 000000000..0f540a105 --- /dev/null +++ b/packages/core/src/ops/configuration-maintenance.ts @@ -0,0 +1,196 @@ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { dataConfigurationFenceReceiptSchema } from "../services/data-manager/contracts/configuration-maintenance"; +import { canonicalJson } from "./canonical-json"; +import { configurationMaintenanceEffectOperationSchema } from "./configuration-operation-records"; +import { coordinatorModeSchema } from "./configuration-preparation-records"; +import type { OpsOperation } from "./contract"; +import { executionRefSchema } from "./execution-ref"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const key = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/); +const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +const ids = z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .max(256) + .refine((ids) => ids.every((id, index) => index === 0 || ids[index - 1]! < id)); +const sequence = z.number().int().nonnegative().safe(); +/** Reserved08.3 continuation targets; this task never admits that adapter. */ +export const protectedConfigurationServiceIdSchema = z.enum([ + "postgis", + "redis", + "app-web", + "app-api", + "traefik", + "ops-agent", +]); +export type ProtectedConfigurationServiceId = z.infer; +export const configurationMaintenanceBindingSchema = z.strictObject({ + maintenanceId: z.uuid(), + candidateId: z.uuid(), + applyOperationKey: key, + applyOperationFingerprint: digest, + expectedCurrentRevisionId: revision.nullable(), + revisionId: revision, + expectedSourceRevision: z.uuid(), + coordinatorMode: coordinatorModeSchema, + followUp: z.enum(["render-only", "service-apply", "platform-apply"]), +}); +export const configurationMaintenancePrepareSchema = configurationMaintenanceBindingSchema.extend({ + kind: z.literal("configurationMaintenance.prepare"), +}); +export const configurationMaintenanceIdentitySchema = z.strictObject({ + maintenanceId: z.uuid(), + candidateId: z.uuid(), + prepareOperationKey: key, + prepareFingerprint: digest, + planDigest: digest, +}); +export const configurationMaintenanceDecisionSchema = configurationMaintenanceIdentitySchema.extend( + { + kind: z.literal("configurationMaintenance.decide"), + decisionId: z.uuid(), + decision: z.enum(["continue", "abandon"]), + expectedSequence: sequence, + checkpointVersion: sequence, + checkpointDigest: digest, + }, +); +export const configurationMaintenanceSettleSchema = configurationMaintenanceIdentitySchema.extend({ + kind: z.literal("configurationMaintenance.settle"), + decisionId: z.uuid(), + expectedSequence: sequence, + evidenceDigest: digest, +}); +export type ConfigurationMaintenanceBinding = z.infer; +export type ConfigurationMaintenancePrepare = z.infer; +export type ConfigurationMaintenanceIdentity = z.infer< + typeof configurationMaintenanceIdentitySchema +>; +export type ConfigurationMaintenanceDecision = z.infer< + typeof configurationMaintenanceDecisionSchema +>; +export type ConfigurationMaintenanceSettle = z.infer; +export type ConfigurationMaintenanceEffect = Extract< + OpsOperation, + { + kind: + | "stack.render" + | "serviceSelection.apply" + | "serviceConfig.apply" + | "integrationConfig.apply" + | "vault.apply" + | "configuration.hardlinks.apply" + | "service.recreate" + | "service.recreateIsolated" + | "configurationHandoff.prepare"; + } +>; +const effectSchema: z.ZodType = + configurationMaintenanceEffectOperationSchema; +export const configurationMaintenanceStepSchema = z.strictObject({ + phase: z.enum([ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ]), + ordinal: z.number().int().min(0).max(31), + operation: effectSchema, + operationKey: key, + operationFingerprint: digest, + serviceIds: ids, + prerequisites: z.array(z.number().int().min(0).max(31)).max(31), +}); +export type ConfigurationMaintenanceStep = z.infer; +export const configurationMaintenanceReceiptSchema = z + .strictObject({ + version: z.literal(1), + binding: configurationMaintenanceBindingSchema, + prepare: executionRefSchema, + apiJobId: z.uuid(), + parentOperationKey: key, + parentFingerprint: digest, + parentDeadlineAt: sequence, + planDigest: digest, + serviceIds: ids, + steps: z.array(configurationMaintenanceStepSchema).min(1).max(32), + preflight: z.enum(["current", "drifted", "unavailable"]), + sequence, + state: z.enum([ + "preparing", + "prepared", + "authorized", + "applying", + "settling", + "settled", + "recovery_required", + ]), + decision: z + .strictObject({ + decisionId: z.uuid(), + decision: z.enum(["continue", "abandon"]), + checkpointVersion: sequence, + checkpointDigest: digest, + }) + .nullable(), + dmFence: dataConfigurationFenceReceiptSchema.nullable(), + generation: z.enum(["not_started", "committed", "not_committed", "unknown"]), + appliedRevisionId: revision.nullable(), + followUp: z.enum(["not_started", "complete", "contained", "unknown"]), + }) + .superRefine((value, context) => { + if ( + Buffer.byteLength(JSON.stringify(value)) > 65536 || + value.steps[0]?.phase !== "configuration-commit" || + value.steps.filter((step) => step.phase === "configuration-commit").length !== 1 || + value.steps.some( + (step, index) => + step.ordinal !== index || + new Set(step.prerequisites).size !== step.prerequisites.length || + step.prerequisites.some((prior) => prior >= index) || + step.serviceIds.some((id) => !value.serviceIds.includes(id)), + ) + ) + context.addIssue({ code: "custom", message: "Invalid bounded maintenance plan" }); + }); +export type ConfigurationMaintenanceReceipt = z.infer; +/** A checksum of an authenticated API observation, never final evidence or a + * nominal settlement capability. The actual owner compares it before its first + * settle-control CAS; the agent produces final-v2 evidence independently. */ +export function configurationMaintenanceSettleRequestDigest( + receipt: ConfigurationMaintenanceReceipt, +): string { + const r = configurationMaintenanceReceiptSchema.parse(receipt), + projection = { + version: 1, + binding: r.binding, + prepare: r.prepare, + apiJobId: r.apiJobId, + parentOperationKey: r.parentOperationKey, + parentFingerprint: r.parentFingerprint, + parentDeadlineAt: r.parentDeadlineAt, + planDigest: r.planDigest, + serviceIds: r.serviceIds, + steps: r.steps.map((s) => ({ + phase: s.phase, + ordinal: s.ordinal, + operationKey: s.operationKey, + operationFingerprint: s.operationFingerprint, + serviceIds: s.serviceIds, + prerequisites: s.prerequisites, + })), + sequence: r.sequence, + decision: r.decision, + dmFence: r.dmFence, + generation: r.generation, + appliedRevisionId: r.appliedRevisionId, + followUp: r.followUp, + }; + return createHash("sha256") + .update( + "openmapx-configuration-settle-request-evidence-v1\0" + canonicalJson(projection), + "utf8", + ) + .digest("hex"); +} diff --git a/packages/core/src/ops/configuration-operation-records.ts b/packages/core/src/ops/configuration-operation-records.ts new file mode 100644 index 000000000..a1c17957f --- /dev/null +++ b/packages/core/src/ops/configuration-operation-records.ts @@ -0,0 +1,97 @@ +/** Canonical existing configuration effect inputs, shared by request and + * maintenance receipt schemas. Contains no result schema/runtime policy. */ +import { z } from "zod"; +import { coordinatorModeSchema } from "./configuration-preparation-records"; + +export const serviceIdSchema = z + .string() + .min(1) + .max(64) + .regex(/^[a-z0-9][a-z0-9-]*$/); +export const stableIdSchema = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/); +export const configurationGenerationRevisionSchema = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +const configurationCondition = { + expectedSourceRevision: z.uuid(), + coordinatorMode: coordinatorModeSchema, + maintenanceId: z.uuid().optional(), + candidateId: z.uuid().optional(), +}; + +export const configurationHardlinksApplyOperationSchema = z.strictObject({ + kind: z.literal("configuration.hardlinks.apply"), + revisionId: configurationGenerationRevisionSchema, + coordinatorMode: coordinatorModeSchema, + maintenanceId: z.uuid().optional(), + candidateId: z.uuid().optional(), + planDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), +}); +export const stackRenderOperationSchema = z.strictObject({ + kind: z.literal("stack.render"), + ...configurationCondition, + revisionId: stableIdSchema, + expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), +}); +export const serviceRecreateOperationSchema = z.strictObject({ + kind: z.literal("service.recreate"), + serviceId: serviceIdSchema, + expectedConfigurationRevisionId: configurationGenerationRevisionSchema, + maintenanceId: z.uuid().optional(), + planDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), +}); +export const serviceRecreateIsolatedOperationSchema = z.strictObject({ + kind: z.literal("service.recreateIsolated"), + serviceId: serviceIdSchema, + expectedConfigurationRevisionId: configurationGenerationRevisionSchema, + maintenanceId: z.uuid().optional(), + planDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), +}); +export const serviceSelectionApplyOperationSchema = z.strictObject({ + kind: z.literal("serviceSelection.apply"), + ...configurationCondition, + revisionId: stableIdSchema, + expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), +}); +export const serviceConfigApplyOperationSchema = z.strictObject({ + kind: z.literal("serviceConfig.apply"), + ...configurationCondition, + serviceId: serviceIdSchema, + revisionId: stableIdSchema, + expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), +}); +export const integrationConfigApplyOperationSchema = z.strictObject({ + kind: z.literal("integrationConfig.apply"), + ...configurationCondition, + integrationId: serviceIdSchema, + revisionId: stableIdSchema, + expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), +}); +export const vaultApplyOperationSchema = z.strictObject({ + kind: z.literal("vault.apply"), + ...configurationCondition, + serviceId: serviceIdSchema, + revisionId: stableIdSchema, + expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), +}); +export const configurationMaintenanceEffectOperationSchema = z.discriminatedUnion("kind", [ + configurationHardlinksApplyOperationSchema, + stackRenderOperationSchema, + serviceRecreateOperationSchema, + serviceRecreateIsolatedOperationSchema, + serviceSelectionApplyOperationSchema, + serviceConfigApplyOperationSchema, + integrationConfigApplyOperationSchema, + vaultApplyOperationSchema, +]); diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts new file mode 100644 index 000000000..581ed3b89 --- /dev/null +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -0,0 +1,154 @@ +/** Portable bounded candidate records; parsing creates no authority. */ +import { z } from "zod"; +import { coordinatorLifecycleMetadataSchema } from "./coordinator-lifecycle-records"; + +const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const key = z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/); +export const coordinatorModeSchema = z.enum(["ensure-running", "require-running"]); +export const configurationSourcesInspectionSchema = z + .strictObject({ + revisionId: revision.nullable(), + sourceRevision: z.uuid(), + services: z + .array( + z.strictObject({ + serviceId: id, + fields: z + .array( + z.strictObject({ + key, + source: z.enum(["host", "none"]), + envVar: z + .string() + .regex(/^[A-Z_][A-Z0-9_]{0,127}$/) + .optional(), + }), + ) + .max(128), + }), + ) + .max(256), + }) + .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 128 * 1024); +export type ConfigurationSourcesInspection = z.infer; +export const configurationHardlinksPreviewSchema = z.strictObject({ + entries: z.number().int().min(0).max(256), + requiredServiceIds: z.array(id).max(256), + coordinatorState: z.enum(["not_required", "running", "stopped", "disabled", "unavailable"]), + previewDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +export type ConfigurationHardlinksPreview = z.infer; + +export const preparedConfigurationRecordSchema = z.strictObject({ + version: z.literal(1), + candidateId: z.uuid(), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + operation: z.unknown(), + sourceFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + expectedSourceRevision: z.uuid(), + sealed: z.string().max(699052), + privateOverlay: z.unknown(), + apiJobId: z.uuid(), + actorId: z.string().min(1).max(256), + parentOperationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + parentFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + parentDeadlineAt: z.number().int().nonnegative().safe(), + affectedServiceIds: z.array(id).max(256), + apiFollowUp: z + .enum([ + "none", + "service-apply", + "integration-reload", + "coordinator-apply", + "coordinator-update", + ]) + .optional(), + privateCoordinatorLifecycle: coordinatorLifecycleMetadataSchema.optional(), + privateCoordinatorSource: configurationSourcesInspectionSchema.optional(), + // Exact current DM credential persistence inputs retained before its helper + // prerequisite. Recovery never regenerates row IDs, encryption or a seal. + privateMaintenancePreparation: z + .strictObject({ + version: z.literal(1), + source: configurationSourcesInspectionSchema, + serviceSecrets: z + .array( + z.strictObject({ + id: z.uuid(), + serviceId: z.literal("data-manager"), + key, + ciphertext: z.string().max(350000), + iv: z.string().max(64), + tag: z.string().max(64), + createdAt: z.iso.datetime(), + updatedAt: z.iso.datetime(), + updatedBy: z.string().max(256).nullable(), + }), + ) + .max(128), + }) + .optional(), + // Private authenticated recovery data, counted in the existing 768KiB cell. + // No raw process environment, backend/provider objects or new authority. + privateIntegrationReload: z + .strictObject({ + version: z.literal(1), + services: z.array(z.unknown()).max(256), + integrations: z + .array( + z.strictObject({ + id, + directory: z.string().min(1).max(4096), + manifest: z.unknown(), + isBuiltIn: z.boolean(), + }), + ) + .max(256), + values: z.record( + id, + z.record( + key, + z.strictObject({ + value: z.unknown(), + source: z.enum(["default", "database", "vault", "config.json", "env"]), + }), + ), + ), + legacyImport: z + .strictObject({ + version: z.literal(1), + kind: z.literal("legacy-integration-config"), + integrationIds: z.array(id).min(1).max(256), + changedIntegrationIds: z.array(id).min(1).max(256), + }) + .optional(), + }) + .optional(), + maintenanceId: z.uuid().nullable(), + prepareOperationKey: z + .string() + .regex(/^opk1_[A-Za-z0-9_-]{16,64}$/) + .nullable(), + coordinatorPrerequisite: z + .strictObject({ + stepAddress: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + operation: z.strictObject({ + kind: z.literal("service.start"), + serviceId: z.literal("data-manager"), + expectedConfigurationRevisionId: revision, + }), + root: z.strictObject({ + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + requestedDeadlineAt: z.number().int().nonnegative().safe(), + parent: z.strictObject({ + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + deadlineAt: z.number().int().nonnegative().safe(), + }), + }), + }) + .nullable() + .optional(), +}); +export type PreparedConfigurationRecord = z.infer; diff --git a/packages/core/src/ops/configuration-preparation.ts b/packages/core/src/ops/configuration-preparation.ts new file mode 100644 index 000000000..1ffcb742a --- /dev/null +++ b/packages/core/src/ops/configuration-preparation.ts @@ -0,0 +1,52 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { z } from "zod"; +import { + type PreparedConfigurationRecord, + preparedConfigurationRecordSchema, +} from "./configuration-preparation-records"; +import { canonicalJson, opsOperationFingerprint } from "./fingerprint"; +import { inspectTrustedConfigurationSnapshot } from "./trusted-config"; + +export * from "./configuration-preparation-records"; + +const domain = "openmapx-prepared-configuration-candidate-v1\0"; +export function encodePreparedConfigurationRecord( + record: PreparedConfigurationRecord, + token: string, +): Buffer { + const parsed = preparedConfigurationRecordSchema.parse(record); + const mac = createHmac("sha256", token) + .update(domain) + .update(canonicalJson(parsed)) + .digest("base64url"); + const bytes = Buffer.from(`${JSON.stringify({ record: parsed, mac })}\n`); + if (bytes.length > 768 * 1024) throw new Error("Prepared configuration unavailable"); + return bytes; +} +export function inspectPreparedConfigurationRecord(bytes: Uint8Array, token: string) { + if (bytes.byteLength > 768 * 1024) throw new Error("Prepared configuration unavailable"); + const envelope = z + .strictObject({ + record: preparedConfigurationRecordSchema, + mac: z.string().regex(/^[A-Za-z0-9_-]{43}$/), + }) + .parse(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))); + const expected = createHmac("sha256", token) + .update(domain) + .update(canonicalJson(envelope.record)) + .digest(); + if (!timingSafeEqual(expected, Buffer.from(envelope.mac, "base64url"))) + throw new Error("Prepared configuration unavailable"); + const sealed = Buffer.from(envelope.record.sealed, "base64"); + if (sealed.toString("base64") !== envelope.record.sealed) + throw new Error("Prepared configuration unavailable"); + const snapshot = inspectTrustedConfigurationSnapshot(sealed, { token }); + if ( + snapshot.operationKey !== envelope.record.operationKey || + opsOperationFingerprint( + envelope.record.operation as Parameters[0], + ) !== snapshot.operationFingerprint + ) + throw new Error("Prepared configuration unavailable"); + return { record: envelope.record, sealed, snapshot }; +} diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 8b66c92cd..7eb32ec0a 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -16,6 +16,24 @@ import { configurationProjectionFingerprint } from "./fingerprint"; const issuedAt = "2026-08-23T18:00:00.000Z"; const expiresAt = "2026-08-23T18:00:20.000Z"; +const maintenanceBinding = { + maintenanceId: "11111111-1111-4111-8111-111111111111", + candidateId: "22222222-2222-4222-8222-222222222222", + applyOperationKey: "opk1_originalApply000001", + applyOperationFingerprint: "a".repeat(64), + expectedCurrentRevisionId: null, + revisionId: `cfg1_${"b".repeat(43)}`, + expectedSourceRevision: "33333333-3333-4333-8333-333333333333", + coordinatorMode: "ensure-running" as const, + followUp: "render-only" as const, +}; +const maintenanceIdentity = { + maintenanceId: maintenanceBinding.maintenanceId, + candidateId: maintenanceBinding.candidateId, + prepareOperationKey: "opk1_originalPrepare0001", + prepareFingerprint: "c".repeat(64), + planDigest: "d".repeat(64), +}; function request(operation: unknown, extra: Record = {}) { return { @@ -42,25 +60,86 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { "stack.status": { kind: "stack.status" }, "services.inspect": { kind: "services.inspect" }, "configuration.inspect": { kind: "configuration.inspect" }, + "configuration.sources.inspect": { kind: "configuration.sources.inspect" }, + "configurationMaintenance.prepare": { + kind: "configurationMaintenance.prepare", + ...maintenanceBinding, + }, + "configurationMaintenance.inspect": { + kind: "configurationMaintenance.inspect", + maintenanceId: maintenanceBinding.maintenanceId, + candidateId: maintenanceBinding.candidateId, + }, + "configurationMaintenance.decide": { + kind: "configurationMaintenance.decide", + ...maintenanceIdentity, + decisionId: "44444444-4444-4444-8444-444444444444", + decision: "abandon", + expectedSequence: 1, + checkpointVersion: 1, + checkpointDigest: "e".repeat(64), + }, + "configurationMaintenance.settle": { + kind: "configurationMaintenance.settle", + ...maintenanceIdentity, + decisionId: "44444444-4444-4444-8444-444444444444", + expectedSequence: 2, + evidenceDigest: "f".repeat(64), + }, + "configuration.hardlinks.preview": { + kind: "configuration.hardlinks.preview", + candidateId: maintenanceBinding.candidateId, + revisionId: maintenanceBinding.revisionId, + expectedCurrentRevisionId: null, + expectedSourceRevision: maintenanceBinding.expectedSourceRevision, + coordinatorMode: "ensure-running", + }, + "configuration.hardlinks.apply": { + kind: "configuration.hardlinks.apply", + revisionId: maintenanceBinding.revisionId, + coordinatorMode: "ensure-running", + }, "configuration.hardlinks.inspect": { kind: "configuration.hardlinks.inspect", revisionId: `cfg1_${"a".repeat(43)}`, }, "stack.render": { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "stack.render", revisionId: "config_20260823", }, - "stack.start": { kind: "stack.start" }, + "stack.start": { kind: "stack.start", expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}` }, "stack.stop": { kind: "stack.stop" }, - "service.start": { kind: "service.start", serviceId: "motis" }, + "service.start": { + kind: "service.start", + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "service.stop": { kind: "service.stop", serviceId: "motis" }, "service.restart": { kind: "service.restart", serviceId: "motis" }, - "service.recreate": { kind: "service.recreate", serviceId: "motis" }, - "service.recreateIsolated": { kind: "service.recreateIsolated", serviceId: "motis" }, - "service.pull": { kind: "service.pull", serviceId: "motis" }, + "service.recreate": { + kind: "service.recreate", + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, + "service.recreateIsolated": { + kind: "service.recreateIsolated", + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, + "service.pull": { + kind: "service.pull", + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "service.remove": { kind: "service.remove", serviceId: "motis" }, - "service.update": { kind: "service.update", serviceId: "motis" }, + "service.update": { + kind: "service.update", + serviceId: "motis", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "service.build": { kind: "service.build", serviceId: "motis", regionId: "europe/germany" }, "services.buildAll": { kind: "services.buildAll", failFast: true }, "service.logs": { kind: "service.logs", serviceId: "motis", tail: 200 }, @@ -136,24 +215,32 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { "communityServices.inspect": { kind: "communityServices.inspect" }, "serviceSelection.apply": { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "serviceSelection.apply", revisionId: "selection_20260823", }, "serviceSelection.inspect": { kind: "serviceSelection.inspect" }, "serviceConfig.apply": { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "serviceConfig.apply", serviceId: "motis", revisionId: "config_20260823", }, "integrationConfig.apply": { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "integrationConfig.apply", integrationId: "routing", revisionId: "config_20260823", }, "vault.apply": { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", kind: "vault.apply", serviceId: "motis", revisionId: "vault_20260823", @@ -269,6 +356,7 @@ describe("ops contract", () => { // Take the role from the policy table rather than re-deriving it from a // kind prefix: a prefix list silently omits every kind added later. const role: OpsRole = OPS_KIND_POLICIES[operation.kind].role; + expect(() => parseOpsRequest(request(operation), { role }), operation.kind).not.toThrow(); expect(parseOpsRequest(request(operation), { role }).operation).toEqual(operation); } }); diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 4150fc4e8..ecc2e4bcc 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -1,10 +1,43 @@ +import { + configurationGenerationRevisionSchema, + configurationHardlinksApplyOperationSchema, + integrationConfigApplyOperationSchema, + serviceConfigApplyOperationSchema, + serviceIdSchema, + serviceRecreateIsolatedOperationSchema, + serviceRecreateOperationSchema, + serviceSelectionApplyOperationSchema, + stableIdSchema, + stackRenderOperationSchema, + vaultApplyOperationSchema, +} from "./configuration-operation-records"; + +export { configurationGenerationRevisionSchema } from "./configuration-operation-records"; + import z from "zod/v4"; import { pairedPinApprovalInputSchema, pairedPinApprovalResultSchema, } from "./catalog-pin-approval"; +import { + configurationMaintenanceDecisionSchema, + configurationMaintenancePrepareSchema, + configurationMaintenanceReceiptSchema, + configurationMaintenanceSettleSchema, +} from "./configuration-maintenance"; +import { + configurationHardlinksPreviewSchema, + configurationSourcesInspectionSchema, + coordinatorModeSchema, +} from "./configuration-preparation-records"; import { opsFailureResultSchema, processDiagnosticsSchema } from "./job-evidence"; -import { serviceActionsSchema } from "./service-results"; +import { + type HistoricalServicePullResult, + historicalServicePullResultSchema, + servicePullOperationSchema, + servicePullResultSchema, +} from "./service-operation-records"; +import { hardlinkApplySummarySchema, serviceActionsSchema } from "./service-results"; export const OPS_PROTOCOL_VERSION = 2 as const; export const OPS_MAX_LOG_TAIL = 1_000; @@ -26,6 +59,13 @@ export const OPS_OPERATION_KINDS = [ "stack.render", "configuration.inspect", "configuration.hardlinks.inspect", + "configuration.sources.inspect", + "configuration.hardlinks.preview", + "configuration.hardlinks.apply", + "configurationMaintenance.prepare", + "configurationMaintenance.inspect", + "configurationMaintenance.decide", + "configurationMaintenance.settle", "stack.start", "stack.stop", "service.start", @@ -143,6 +183,13 @@ export const OPS_KIND_POLICIES = { "services.inspect": sync("api", 15_000, 512 * 1024), "configuration.inspect": sync("api", 15_000), "configuration.hardlinks.inspect": sync("api", 15_000, 1024 * 1024), + "configuration.sources.inspect": sync("api", 15_000, 128 * 1024), + "configuration.hardlinks.preview": sync("api", 15_000, 32 * 1024), + "configuration.hardlinks.apply": asyncEffect("api", 10 * 60_000), + "configurationMaintenance.prepare": asyncEffect("api", 3600_000, 65536), + "configurationMaintenance.inspect": sync("api", 15_000, 65536), + "configurationMaintenance.decide": asyncEffect("api", 60_000, 65536), + "configurationMaintenance.settle": asyncEffect("api", 60_000, 65536), "stack.render": asyncEffect("api", 120_000), "stack.start": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "stack.stop": asyncEffect("api", 5 * 60_000), @@ -221,16 +268,6 @@ export function opsKindRequiresControlAuthority(kind: OpsOperationKind): boolean ); } const operationKindSchema = z.enum(OPS_OPERATION_KINDS); -const serviceIdSchema = z - .string() - .min(1) - .max(64) - .regex(/^[a-z0-9][a-z0-9-]*$/); -const stableIdSchema = z - .string() - .min(1) - .max(128) - .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/); const backupIdSchema = z .string() .min(1) @@ -302,8 +339,6 @@ const kindOnly = (kind: T) => z.strictObject({ kind: const serviceOperation = (kind: T) => z.strictObject({ kind: z.literal(kind), serviceId: serviceIdSchema }); -export const configurationGenerationRevisionSchema = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); - export const opsOperationSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("controlRetention.acknowledge"), @@ -317,11 +352,25 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ kindOnly("stack.status"), kindOnly("services.inspect"), kindOnly("configuration.inspect"), + kindOnly("configuration.sources.inspect"), + configurationMaintenancePrepareSchema, z.strictObject({ - kind: z.literal("stack.render"), - revisionId: stableIdSchema, + kind: z.literal("configurationMaintenance.inspect"), + maintenanceId: z.uuid(), + candidateId: z.uuid(), + }), + configurationMaintenanceDecisionSchema, + configurationMaintenanceSettleSchema, + z.strictObject({ + kind: z.literal("configuration.hardlinks.preview"), + candidateId: z.uuid(), + revisionId: configurationGenerationRevisionSchema, expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), + expectedSourceRevision: z.uuid(), + coordinatorMode: coordinatorModeSchema, }), + configurationHardlinksApplyOperationSchema, + stackRenderOperationSchema, // The hardlink plan of the named, applied configuration generation. Fails if // another generation has been applied since. Data-manager applies it: only // its mounts confine the links and prunes to the data root. @@ -329,16 +378,28 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ kind: z.literal("configuration.hardlinks.inspect"), revisionId: configurationGenerationRevisionSchema, }), - kindOnly("stack.start"), + z.strictObject({ + kind: z.literal("stack.start"), + expectedConfigurationRevisionId: configurationGenerationRevisionSchema, + }), kindOnly("stack.stop"), - serviceOperation("service.start"), + z.strictObject({ + kind: z.literal("service.start"), + serviceId: serviceIdSchema, + expectedConfigurationRevisionId: configurationGenerationRevisionSchema, + }), serviceOperation("service.stop"), serviceOperation("service.restart"), - serviceOperation("service.recreate"), - serviceOperation("service.recreateIsolated"), - serviceOperation("service.pull"), + serviceRecreateOperationSchema, + serviceRecreateIsolatedOperationSchema, + servicePullOperationSchema, serviceOperation("service.remove"), - serviceOperation("service.update"), + z.strictObject({ + kind: z.literal("service.update"), + serviceId: serviceIdSchema, + expectedConfigurationRevisionId: configurationGenerationRevisionSchema, + noDeps: z.boolean().optional(), + }), z.strictObject({ kind: z.literal("service.build"), serviceId: serviceIdSchema, @@ -429,30 +490,11 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ extensionId: serviceIdSchema, }), kindOnly("communityServices.inspect"), - z.strictObject({ - kind: z.literal("serviceSelection.apply"), - revisionId: stableIdSchema, - expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), - }), + serviceSelectionApplyOperationSchema, kindOnly("serviceSelection.inspect"), - z.strictObject({ - kind: z.literal("serviceConfig.apply"), - serviceId: serviceIdSchema, - revisionId: stableIdSchema, - expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), - }), - z.strictObject({ - kind: z.literal("integrationConfig.apply"), - integrationId: serviceIdSchema, - revisionId: stableIdSchema, - expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), - }), - z.strictObject({ - kind: z.literal("vault.apply"), - serviceId: serviceIdSchema, - revisionId: stableIdSchema, - expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), - }), + serviceConfigApplyOperationSchema, + integrationConfigApplyOperationSchema, + vaultApplyOperationSchema, kindOnly("data.inspect"), z.strictObject({ kind: z.literal("data.downloadOsm"), regionId: regionIdSchema.optional() }), kindOnly("data.downloadFonts"), @@ -503,6 +545,28 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ kindOnly("gbfsCatalogLock.inspect"), ]); export type OpsOperation = z.infer; +export type TrustedConfigurationApplyOperation = Extract< + OpsOperation, + { + kind: + | "stack.render" + | "serviceSelection.apply" + | "serviceConfig.apply" + | "integrationConfig.apply" + | "vault.apply"; + } +>; +export function isTrustedConfigurationApplyOperation( + operation: OpsOperation, +): operation is TrustedConfigurationApplyOperation { + return [ + "stack.render", + "serviceSelection.apply", + "serviceConfig.apply", + "integrationConfig.apply", + "vault.apply", + ].includes(operation.kind); +} const opsRequestSchema = z .strictObject({ @@ -541,6 +605,7 @@ export type OpsErrorClass = | "control_upgrade_required" | "recovery_required" | "host_update_required" + | "configuration_prerequisite" | "runtime"; export const OPS_PUBLIC_ERROR_MESSAGES: Record = { authentication: "Request authentication failed", @@ -552,6 +617,8 @@ export const OPS_PUBLIC_ERROR_MESSAGES: Record = { timeout: "Operation timed out", busy: "Operations service is busy", conflict: "Operation key conflicts with an existing operation", + configuration_prerequisite: + "Apply or update Data Manager before changing instance configuration.", not_found: "Operation was not found", not_wired: "Operation is not available", control_upgrade_required: @@ -933,6 +1000,13 @@ export const OPS_RESULT_SCHEMAS = { ) .max(256), }), + "configuration.sources.inspect": configurationSourcesInspectionSchema, + "configuration.hardlinks.preview": configurationHardlinksPreviewSchema, + "configuration.hardlinks.apply": hardlinkApplySummarySchema, + "configurationMaintenance.prepare": configurationMaintenanceReceiptSchema, + "configurationMaintenance.inspect": configurationMaintenanceReceiptSchema, + "configurationMaintenance.decide": configurationMaintenanceReceiptSchema, + "configurationMaintenance.settle": configurationMaintenanceReceiptSchema, "configuration.inspect": z.strictObject({ revisionId: configurationGenerationRevisionSchema.nullable(), integrationConfigDigests: z @@ -952,9 +1026,12 @@ export const OPS_RESULT_SCHEMAS = { "service.restart": changed, "service.recreate": changed, "service.recreateIsolated": changed, - "service.pull": changed, + "service.pull": servicePullResultSchema, "service.remove": changed, - "service.update": changed, + "service.update": z.strictObject({ + changed: z.literal(true), + revisionId: configurationGenerationRevisionSchema, + }), "service.build": completed, "services.buildAll": z.strictObject({ completedServiceIds: z.array(serviceIdSchema).max(256), @@ -1197,6 +1274,19 @@ export const OPS_RESULT_SCHEMAS = { } as const satisfies Record; export type OpsResultFor = z.infer<(typeof OPS_RESULT_SCHEMAS)[K]>; +export type OpsObservedResultFor = K extends "service.pull" + ? OpsResultFor | HistoricalServicePullResult + : OpsResultFor; +/** Status observation only. Fresh execution/settlement continues to use the + * strict parseOpsResult/parseBoundedOpsResult above the runtime boundary. */ +export function parseObservedOpsResult( + kind: K, + value: unknown, +): OpsObservedResultFor { + if (kind === "service.pull" && historicalServicePullResultSchema.safeParse(value).success) + return historicalServicePullResultSchema.parse(value) as OpsObservedResultFor; + return parseBoundedOpsResult(kind, value) as OpsObservedResultFor; +} export function parseOpsResult( kind: K, value: unknown, @@ -1289,7 +1379,7 @@ export function parseOpsJobStatus(input: unknown): OpsJobStatus { if (status.state === "succeeded") { if (status.result === undefined || status.errorClass !== undefined) throw new Error("Invalid operation status"); - parseOpsResult(status.kind, status.result); + parseObservedOpsResult(status.kind, status.result); } else if (status.state === "failed" || status.state === "timed_out") { if (status.result !== undefined || status.errorClass === undefined) throw new Error("Invalid operation status"); @@ -1310,7 +1400,7 @@ export type OpsJobStatusFor = }) | (OpsJobStatusBase & { state: "succeeded"; - result: OpsResultFor; + result: OpsObservedResultFor; errorClass?: never; }) | (OpsJobStatusBase & { @@ -1330,7 +1420,7 @@ export function parseOpsJobStatusForKind( } return { ...status, - ...(status.result === undefined ? {} : { result: parseOpsResult(kind, status.result) }), + ...(status.result === undefined ? {} : { result: parseObservedOpsResult(kind, status.result) }), } as OpsJobStatusFor; } const stateEvent = z.strictObject({ @@ -1450,7 +1540,7 @@ export function opsResourceId(operation: OpsOperation): string { if ("serviceId" in operation) return operation.serviceId; if ("backupId" in operation && operation.backupId) return operation.backupId; if ("preparedRunId" in operation) return operation.preparedRunId; - if ("candidateId" in operation) return operation.candidateId; + if ("candidateId" in operation && operation.candidateId) return operation.candidateId; if ("repository" in operation) return operation.repository; if ("extensionId" in operation) return operation.extensionId; if ("integrationId" in operation) return operation.integrationId; diff --git a/packages/core/src/ops/control-retention-receipt.ts b/packages/core/src/ops/control-retention-receipt.ts index d3bcba1ac..32d748fb6 100644 --- a/packages/core/src/ops/control-retention-receipt.ts +++ b/packages/core/src/ops/control-retention-receipt.ts @@ -1,14 +1,14 @@ /** Server-capable receipt digests and the existing fixed API token proof. */ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import { z } from "zod"; -import { type OpsJobStatus, parseBoundedOpsResult, parseOpsJobStatus } from "./contract"; -import { canonicalJson } from "./fingerprint"; +import { canonicalJson } from "./canonical-json"; +import { type OpsJobStatus, parseObservedOpsResult, parseOpsJobStatus } from "./contract"; export function controlTerminalReceiptDigest(input: OpsJobStatus): string { const s = parseOpsJobStatus(input); if (s.version !== 2 || !["succeeded", "failed", "timed_out"].includes(s.state)) throw Error("A version2 terminal receipt is required"); - const result = s.result === undefined ? null : parseBoundedOpsResult(s.kind, s.result); + const result = s.result === undefined ? null : parseObservedOpsResult(s.kind, s.result); const fields = { version: 2, operationId: s.operationId, diff --git a/packages/core/src/ops/coordinator-lifecycle-records.ts b/packages/core/src/ops/coordinator-lifecycle-records.ts new file mode 100644 index 000000000..171cd49b0 --- /dev/null +++ b/packages/core/src/ops/coordinator-lifecycle-records.ts @@ -0,0 +1,59 @@ +/** Closed immutable submission data. Parsing never issues an API intent or pull proof. */ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { apiJobChildAttemptSchema } from "./api-job-attempt"; +import { canonicalJson } from "./canonical-json"; +import { servicePullOperationSchema } from "./service-operation-records"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/), + key = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/); +export const coordinatorPullSubmissionSchema = z + .strictObject({ + version: z.literal(1), + requestVersion: z.literal(2), + expectedExecutor: z.literal("ops-agent"), + expectedPrincipal: z.literal("api"), + stepAddress: key, + operation: servicePullOperationSchema.extend({ + serviceId: z.literal("data-manager"), + candidateId: z.uuid(), + }), + operationKey: key, + operationFingerprint: digest, + requestedDeadlineAt: z.number().int().nonnegative().safe(), + attempt: apiJobChildAttemptSchema, + }) + .refine( + (s) => + s.operationKey === s.attempt.childOperationKey && + s.operationFingerprint === s.attempt.childOperationFingerprint && + s.requestedDeadlineAt === s.attempt.requestedDeadlineAt && + s.requestedDeadlineAt <= s.attempt.parentDeadlineAt && + s.operationFingerprint === + createHash("sha256").update(canonicalJson(s.operation)).digest("hex"), + ); +export type CoordinatorPullSubmission = z.infer; +export const coordinatorLifecycleMetadataSchema = z.discriminatedUnion("action", [ + z.strictObject({ + version: z.literal(1), + serviceId: z.literal("data-manager"), + action: z.literal("apply"), + originalOrdinal: z.null(), + noDeps: z.literal(false), + }), + z.strictObject({ + version: z.literal(1), + serviceId: z.literal("data-manager"), + action: z.literal("update"), + originalOrdinal: z.number().int().min(0).max(255), + noDeps: z.boolean(), + pull: coordinatorPullSubmissionSchema, + }), +]); +export type CoordinatorLifecycleMetadata = z.infer; +export function coordinatorPullSubmissionDigest(submission: CoordinatorPullSubmission) { + return createHash("sha256") + .update("openmapx-coordinator-pull-submission-v1\0") + .update(canonicalJson(coordinatorPullSubmissionSchema.parse(submission))) + .digest("hex"); +} diff --git a/packages/core/src/ops/execution-identity.test.ts b/packages/core/src/ops/execution-identity.test.ts index bc13a5167..00cef83c7 100644 --- a/packages/core/src/ops/execution-identity.test.ts +++ b/packages/core/src/ops/execution-identity.test.ts @@ -4,6 +4,7 @@ import { OPS_KIND_POLICIES } from "./contract"; import { normalizeExecutionAdmission, opsExecutionBudget, + verifiedApiJobParentBinding, verifyApiJobParentExecution, } from "./execution-identity"; @@ -79,6 +80,11 @@ describe("immutable execution admission", () => { operationFingerprint: base.operationFingerprint, requestedDeadlineAt: fields.requestedDeadlineAt, }); + expect(verifiedApiJobParentBinding(parent)).toEqual(fields); + expect(Object.isFrozen(verifiedApiJobParentBinding(parent))).toBe(true); + expect(() => verifiedApiJobParentBinding({ ...parent })).toThrow( + "Actual verified API job parent binding required", + ); const result = normalizeExecutionAdmission({ ...base, requestedDeadlineAt: fields.requestedDeadlineAt, diff --git a/packages/core/src/ops/execution-identity.ts b/packages/core/src/ops/execution-identity.ts index 02fce97ba..349e4951f 100644 --- a/packages/core/src/ops/execution-identity.ts +++ b/packages/core/src/ops/execution-identity.ts @@ -1,8 +1,8 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import { z } from "zod"; +import { canonicalJson } from "./canonical-json"; import { OPS_KIND_POLICIES, type OpsOperationKind } from "./contract"; import { executionRefSchema } from "./execution-ref"; -import { canonicalJson } from "./fingerprint"; export { type ControlIssuer, type ExecutionRef, executionRefSchema } from "./execution-ref"; @@ -66,6 +66,7 @@ const futureBudgets: Readonly> = { "search-index-build": 24 * 3600_000, "notable-places-build": 24 * 3600_000, "generation-hardlinks": 10 * 60_000, + "configuration-maintenance.prepare": 60_000, "transit-maintenance": 120_000, }; /** Future policies are budgets only; they register no executable operations. */ @@ -155,6 +156,14 @@ const apiAssertionSchema = z.strictObject({ attemptId: z.uuid(), }); /** Fixed API bearer/HMAC edge only. Role-signed parent/lease verification belongs to 06.1. */ +const apiParentAssertions = new WeakMap>>(); +/** Source observation only: a parsed/restored parent cannot substitute for the + * exact live API HMAC/body binding verified on this request. */ +export function verifiedApiJobParentBinding(parent: VerifiedParentExecution | undefined) { + const actual = parent && apiParentAssertions.get(parent); + if (!actual) throw Error("Actual verified API job parent binding required"); + return actual; +} export function verifyApiJobParentExecution( assertion: string, token: string, @@ -191,6 +200,7 @@ export function verifyApiJobParentExecution( assertionDigest: createHash("sha256").update(assertion).digest("hex"), }); verifiedParents.add(parent); + apiParentAssertions.set(parent, Object.freeze(fields)); return parent as VerifiedParentExecution; } diff --git a/packages/core/src/ops/feed-credential-map.test.ts b/packages/core/src/ops/feed-credential-map.test.ts new file mode 100644 index 000000000..9eaecfa1a --- /dev/null +++ b/packages/core/src/ops/feed-credential-map.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { canonicalFeedCredentialMap } from "./feed-credential-map"; + +describe("canonical feed credential map", () => { + it("refuses non-JSON whitespace rather than repairing an invalid retained map", () => { + expect(() => canonicalFeedCredentialMap('{\u000b"a":"value"}')).toThrow(); + }); + it("accepts exactly 65536 canonical UTF-8 bytes and rejects the next byte", () => { + const values = Object.fromEntries( + Array.from({ length: 8 }, (_, index) => [`k${index}`, "x".repeat(index === 7 ? 8127 : 8192)]), + ); + const accepted = canonicalFeedCredentialMap(values); + expect(Buffer.byteLength(accepted)).toBe(65536); + values.k7 += "x"; + expect(() => canonicalFeedCredentialMap(values)).toThrow(); + }); + it("preserves orphan keys, sorts keys and counts code points independently of UTF-8 values", () => { + const key = "😀".repeat(512); + const value = canonicalFeedCredentialMap({ z: "old orphan", [key]: 'ü\\"', a: "retained" }); + expect(JSON.parse(value)).toEqual({ z: "old orphan", [key]: 'ü\\"', a: "retained" }); + expect(value.startsWith('{"a":')).toBe(true); + expect(() => canonicalFeedCredentialMap({ [key + "x"]: "" })).toThrow(); + }); + it.each([ + '{"a":"one","a":"two"}', + '{"a":"one","\\u0061":"two"}', + '{"constructor":"bad"}', + '{"a":2}', + '{"a":"\\ud800"}', + ])("refuses duplicate, unsafe, non-string and invalid-Unicode input", (input) => { + expect(() => canonicalFeedCredentialMap(input)).toThrow(); + }); + it("enforces the 1000-key and 8192-byte value bounds", () => { + expect(() => + canonicalFeedCredentialMap( + Object.fromEntries(Array.from({ length: 1001 }, (_, index) => [`k${index}`, ""])), + ), + ).toThrow(); + expect(() => canonicalFeedCredentialMap({ a: "ü".repeat(4097) })).toThrow(); + }); +}); diff --git a/packages/core/src/ops/feed-credential-map.ts b/packages/core/src/ops/feed-credential-map.ts new file mode 100644 index 000000000..31bf5718f --- /dev/null +++ b/packages/core/src/ops/feed-credential-map.ts @@ -0,0 +1,75 @@ +const rejected = () => new Error("Feed credential map exceeds its format or budget"); +const forbidden = new Set(["__proto__", "prototype", "constructor"]); +// biome-ignore lint/suspicious/noControlCharactersInRegex: JSON strings must reject literal C0 control characters. +const token = /[ \t\r\n]*("(?:[^"\\\u0000-\u001f]|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*")[ \t\r\n]*/y; + +/** Flat JSON parser intentionally retains keys until duplicates have been checked. */ +function readMap(input: string): [string, string][] { + let offset = 0; + const whitespace = () => { + while (/[ \t\r\n]/.test(input[offset] ?? "") && offset < input.length) offset++; + }; + const take = (expected: string) => { + whitespace(); + if (input[offset++] !== expected) throw rejected(); + }; + const string = () => { + token.lastIndex = offset; + const match = token.exec(input); + if (!match) throw rejected(); + offset = token.lastIndex; + return JSON.parse(match[1]!) as string; + }; + take("{"); + const entries: [string, string][] = []; + whitespace(); + if (input[offset] !== "}") { + for (;;) { + if (entries.length === 1000) throw rejected(); + const key = string(); + take(":"); + entries.push([key, string()]); + if (input[offset] !== ",") break; + offset++; + } + } + take("}"); + whitespace(); + if (offset !== input.length) throw rejected(); + return entries; +} + +/** Count every retained key, escaping and UTF-8 byte; never truncates or drops orphans. */ +export function canonicalFeedCredentialMap( + input: string | Readonly>, +): string { + try { + if (typeof input === "string" && (/[\uD800-\uDFFF]/u.test(input) || input.length > 262144)) + throw rejected(); + if (typeof input !== "string" && (!input || Array.isArray(input) || typeof input !== "object")) + throw rejected(); + const entries = typeof input === "string" ? readMap(input) : Object.entries(input); + if (entries.length > 1000) throw rejected(); + const keys = new Set(); + for (const [key, value] of entries) { + if ( + !key || + /[\uD800-\uDFFF]/u.test(key) || + [...key].length > 512 || + forbidden.has(key) || + keys.has(key) || + typeof value !== "string" || + /[\uD800-\uDFFF]/u.test(value) || + Buffer.byteLength(value, "utf8") > 8192 + ) + throw rejected(); + keys.add(key); + } + entries.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)); + const canonical = `{${entries.map(([key, value]) => `${JSON.stringify(key)}:${JSON.stringify(value)}`).join(",")}}`; + if (Buffer.byteLength(canonical, "utf8") > 65536) throw rejected(); + return canonical; + } catch { + throw rejected(); + } +} diff --git a/packages/core/src/ops/fingerprint.ts b/packages/core/src/ops/fingerprint.ts index 79532e7e9..c0414a76e 100644 --- a/packages/core/src/ops/fingerprint.ts +++ b/packages/core/src/ops/fingerprint.ts @@ -1,17 +1,8 @@ import { createHash } from "node:crypto"; +import { canonicalJson } from "./canonical-json"; import { type OpsOperation, opsOperationSchema } from "./contract"; -export function canonicalJson(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; - if (value && typeof value === "object") { - return `{${Object.entries(value as Record) - .filter(([, child]) => child !== undefined) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, child]) => `${JSON.stringify(key)}:${canonicalJson(child)}`) - .join(",")}}`; - } - return JSON.stringify(value); -} +export { canonicalJson } from "./canonical-json"; export function opsOperationFingerprint(operation: OpsOperation): string { const normalized = opsOperationSchema.parse(operation); diff --git a/packages/core/src/ops/foundation-review.test.ts b/packages/core/src/ops/foundation-review.test.ts index 77c96edd0..b2f47c180 100644 --- a/packages/core/src/ops/foundation-review.test.ts +++ b/packages/core/src/ops/foundation-review.test.ts @@ -43,14 +43,37 @@ describe("reviewed ops foundation contract", () => { it("covers every discovered API authority effect with an owner and execution budget", () => { for (const kind of requiredKinds) expect(OPS_OPERATION_KINDS).toContain(kind); expect(Object.keys(OPS_KIND_POLICIES).sort()).toEqual([...OPS_OPERATION_KINDS].sort()); - for (const policy of Object.values(OPS_KIND_POLICIES)) { + // Exact existing long-running owners, plus the approved ownership-only + // maintenance hour. Every other kind retains the original 30 minute cap. + const extendedBudgets: Record = { + "configurationMaintenance.prepare": 3600_000, + "service.build": 12 * 3600_000, + "services.buildAll": 24 * 3600_000, + "system.update": 48 * 3600_000, + "data.downloadOsm": 24 * 3600_000, + "data.downloadFonts": 24 * 3600_000, + "data.update": 24 * 3600_000, + "data.convertOverpass": 24 * 3600_000, + "data.overtureSync": 24 * 3600_000, + "data.overtureConflate": 24 * 3600_000, + "data.searchIndexBuild": 24 * 3600_000, + "data.notablePlacesBuild": 24 * 3600_000, + }; + for (const [kind, timeoutMs] of Object.entries(extendedBudgets)) + expect(OPS_KIND_POLICIES[kind as keyof typeof OPS_KIND_POLICIES].timeoutMs).toBe(timeoutMs); + for (const [kind, policy] of Object.entries(OPS_KIND_POLICIES)) { expect(policy.role === "api" || policy.role === "data-manager").toBe(true); expect(policy.execution === "sync" || policy.execution === "async").toBe(true); expect(policy.timeoutMs).toBeGreaterThan(0); - expect(policy.timeoutMs).toBeLessThanOrEqual(30 * 60_000); + expect(policy.timeoutMs).toBeLessThanOrEqual(extendedBudgets[kind] ?? 30 * 60_000); expect(policy.maxResultBytes).toBeGreaterThan(0); expect(policy.maxResultBytes).toBeLessThanOrEqual(1024 * 1024); } + expect(OPS_KIND_POLICIES["configurationMaintenance.prepare"]).toMatchObject({ + role: "api", + execution: "async", + timeoutMs: 60 * 60_000, + }); for (const kind of [ "service.logs.follow", "service.pull", diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index a96281cc5..e67652350 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -1,12 +1,19 @@ +export * from "./api-job-attempt"; export * from "./catalog-pin-approval"; export * from "./client"; +export * from "./configuration-maintenance"; +export * from "./configuration-maintenance-control-validation"; +export * from "./configuration-preparation"; export * from "./contract"; export * from "./control-retention-receipt"; export * from "./control-root-key"; +export * from "./coordinator-lifecycle-records"; export * from "./execution-identity"; +export { canonicalFeedCredentialMap } from "./feed-credential-map"; export * from "./fingerprint"; export * from "./job-evidence"; export * from "./privacy-backup"; +export * from "./service-operation-records"; export * from "./service-results"; export * from "./trusted-config"; export * from "./trusted-config-queue-lock"; diff --git a/packages/core/src/ops/owned-service-records.ts b/packages/core/src/ops/owned-service-records.ts new file mode 100644 index 000000000..3c2b4fe6e --- /dev/null +++ b/packages/core/src/ops/owned-service-records.ts @@ -0,0 +1,13 @@ +/** Portable observations only; parsing confers no ownership authority. */ +import { z } from "zod"; +export const ownedServiceSnapshotSchema = z.strictObject({ + serviceId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + containerId: z.string().regex(/^[a-f0-9]{64}$/), + state: z.enum(["running", "stopped", "created", "restarting", "paused", "unknown"]), + provenance: z.strictObject({ + project: z.string().min(1).max(128), + composeWorkingDirectory: z.string().min(1).max(4096), + composeConfigFiles: z.array(z.string().min(1).max(4096)).min(1).max(2), + imageId: z.string().regex(/^sha256:[a-f0-9]{64}$/), + }), +}); diff --git a/packages/core/src/ops/service-operation-records.ts b/packages/core/src/ops/service-operation-records.ts new file mode 100644 index 000000000..b3b9a09b1 --- /dev/null +++ b/packages/core/src/ops/service-operation-records.ts @@ -0,0 +1,31 @@ +/** Fixed service input/result data. Decoding confers no execution authority. */ +import { serviceImageReferenceSchema } from "@openmapx/extension-contracts/service"; +import { z } from "zod"; + +const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +export const servicePullOperationSchema = z.strictObject({ + kind: z.literal("service.pull"), + serviceId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + expectedConfigurationRevisionId: revision, + candidateId: z.uuid().optional(), +}); +export const servicePullResultSchema = z.strictObject({ + changed: z.literal(true), + revisionId: revision, + imageReference: serviceImageReferenceSchema, + imageId: z.string().regex(/^sha256:[a-f0-9]{64}$/), + releaseSelectionDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), +}); +export type ServicePullResult = z.infer; +/** Retained read-only data, deliberately absent from opsOperationSchema and + * fresh result validation. Neither shape can be dispatched or copied as proof. */ +export const historicalServicePullOperationSchema = z.strictObject({ + kind: z.literal("service.pull"), + serviceId: servicePullOperationSchema.shape.serviceId, +}); +export const historicalServicePullResultSchema = z.strictObject({ changed: z.boolean() }); +export type HistoricalServicePullOperation = z.infer; +export type HistoricalServicePullResult = z.infer; diff --git a/packages/core/src/ops/service-pull.test.ts b/packages/core/src/ops/service-pull.test.ts new file mode 100644 index 000000000..5715eda50 --- /dev/null +++ b/packages/core/src/ops/service-pull.test.ts @@ -0,0 +1,69 @@ +import { expect, it } from "vitest"; +import { + opsOperationSchema, + parseOpsJobStatus, + parseOpsJobStatusForKind, + parseOpsResult, +} from "./contract"; + +const revisionId = `cfg1_${"a".repeat(43)}`; +it("decodes unchanged historical pull status for observation while its old result grants no fresh pull evidence", () => { + for (const version of [1, 2]) { + const status = { + version, + operationId: "job1_legacyPullObservation0", + operationKey: "opk1_legacyPullObservation0", + kind: "service.pull", + resourceId: "redis", + state: "succeeded", + submittedAt: "2026-08-23T18:00:00.000Z", + updatedAt: "2026-08-23T18:00:01.000Z", + result: { changed: true }, + ...(version === 2 ? { admittedAt: 100, deadlineAt: 1000, fingerprint: "a".repeat(64) } : {}), + }; + expect(parseOpsJobStatus(status)).toEqual(status); + expect(parseOpsJobStatusForKind("service.pull", status)).toEqual(status); + expect(() => parseOpsResult("service.pull", status.result)).toThrow(); + } +}); +it("conditions every fresh pull on its exact current configuration and keeps candidate binding finite", () => { + expect( + opsOperationSchema.safeParse({ kind: "service.pull", serviceId: "data-manager" }).success, + ).toBe(false); + expect( + opsOperationSchema.parse({ + kind: "service.pull", + serviceId: "data-manager", + expectedConfigurationRevisionId: revisionId, + candidateId: "11111111-1111-4111-8111-111111111111", + }), + ).toEqual({ + kind: "service.pull", + serviceId: "data-manager", + expectedConfigurationRevisionId: revisionId, + candidateId: "11111111-1111-4111-8111-111111111111", + }); + expect( + opsOperationSchema.safeParse({ + kind: "service.pull", + serviceId: "data-manager", + expectedConfigurationRevisionId: revisionId, + imageReference: "caller.invalid/image:latest", + }).success, + ).toBe(false); +}); +it("requires actual bounded image and release evidence for a fresh pull result", () => { + const result = { + changed: true, + revisionId, + imageReference: "example/data-manager:2", + imageId: `sha256:${"b".repeat(64)}`, + releaseSelectionDigest: "c".repeat(64), + }; + expect(() => parseOpsResult("service.pull", result)).not.toThrow(); + expect(() => parseOpsResult("service.pull", { changed: true })).toThrow(); + expect(() => parseOpsResult("service.pull", { ...result, imageId: "cached-latest" })).toThrow(); + expect(() => + parseOpsResult("service.pull", { ...result, imageReference: "a".repeat(2049) }), + ).toThrow(); +}); diff --git a/packages/core/src/ops/service-results.ts b/packages/core/src/ops/service-results.ts index 7910bfdc2..d2df2c3a2 100644 --- a/packages/core/src/ops/service-results.ts +++ b/packages/core/src/ops/service-results.ts @@ -11,6 +11,11 @@ export const hardlinkApplySummarySchema = z.strictObject({ via: z.enum(["data-manager", "local", "none"]).optional(), }); export type HardlinkApplySummary = z.infer; +export type ServicePreparationResult = { + revisionId: string; + hardlinks: HardlinkApplySummary; + coordinatorMode: CoordinatorMode; +}; const serviceId = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); const ids = z .array(serviceId) @@ -94,7 +99,9 @@ export const serviceActionDecisionSchema = z "not_buildable", ]) .nullable(), - workflow: z.enum(["core-restart", "system-update"]).nullable(), + workflow: z + .enum(["core-restart", "system-update", "coordinator-apply", "coordinator-update"]) + .nullable(), }) .refine((decision) => decision.available diff --git a/packages/core/src/ops/trusted-config.test.ts b/packages/core/src/ops/trusted-config.test.ts index 3c39f293c..6f8ce8ab7 100644 --- a/packages/core/src/ops/trusted-config.test.ts +++ b/packages/core/src/ops/trusted-config.test.ts @@ -10,6 +10,42 @@ import { trustedConfigurationQueueFits, } from "./trusted-config"; +it("accepts exactly 524288 serialized sealed UTF-8 bytes and rejects the next byte", () => { + const input = payload(); + input.serviceSecrets = [ + { + serviceId: "motis", + values: { + ...Object.fromEntries( + Array.from({ length: 8 }, (_, i) => [`TOKEN_${i}`, "x".repeat(60000)]), + ), + PAD: "", + }, + }, + ]; + const seal = () => + sealTrustedConfigurationSnapshot({ + role: "api", + operationKey: "opk1_0123456789abcdef", + operationForRevision: (revisionId) => ({ + kind: "stack.render", + revisionId, + expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running", + }), + payload: input, + token, + issuedAtMs: now, + nonce: "nonce_0123456789abcdef", + }); + const remaining = OPS_TRUSTED_CONFIG_MAX_BYTES - seal().bytes.length; + input.serviceSecrets[0]!.values.PAD = "x".repeat(remaining); + expect(seal().bytes.length).toBe(524288); + input.serviceSecrets[0]!.values.PAD += "x"; + expect(() => seal()).toThrow(); +}); + const token = Buffer.alloc(32, 0x2a).toString("base64url"); const now = Date.parse("2026-08-24T08:00:00.000Z"); @@ -29,6 +65,26 @@ function payload(): TrustedConfigurationPayload { } describe("trusted configuration snapshot protocol", () => { + it("refuses to seal a read-only hardlink preview as a generation apply", () => { + expect(() => + sealTrustedConfigurationSnapshot({ + role: "api", + operationKey: "opk1_candidatekind123456", + token, + issuedAtMs: now, + nonce: "nonce_candidate_kind_123456", + payload: payload(), + operationForRevision: (revisionId) => ({ + kind: "configuration.hardlinks.preview", + candidateId: "031d2b04-4494-443b-8a70-5d1c44b4d951", + revisionId, + expectedCurrentRevisionId: null, + expectedSourceRevision: "d1daa4b7-1c11-47e7-88af-5442be6a3ae7", + coordinatorMode: "require-running", + }), + }), + ).toThrow(); + }); it("uses one retained queue budget for ready, claimed, and in-flight reservations", () => { expect( trustedConfigurationQueueFits({ @@ -78,6 +134,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render", revisionId, }), @@ -107,6 +165,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "serviceConfig.apply", serviceId: "redis", revisionId, @@ -139,6 +199,8 @@ describe("trusted configuration snapshot protocol", () => { { operation: { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "serviceConfig.apply" as const, serviceId: "postgis", revisionId: "cfg1_0123456789abcdef0123456789abcdef0123456789a", @@ -156,6 +218,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render", revisionId, }), @@ -184,6 +248,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId: string) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render" as const, revisionId, }), @@ -227,6 +293,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operation: { expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render", revisionId: "cfg1_0123456789abcdef0123456789abcdef0123456789a", }, @@ -251,6 +319,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render", revisionId, }), @@ -290,6 +360,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render", revisionId, }), @@ -306,6 +378,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render", revisionId, }), @@ -324,6 +398,8 @@ describe("trusted configuration snapshot protocol", () => { operationKey: "opk1_0123456789abcdef", operationForRevision: (revisionId: string) => ({ expectedCurrentRevisionId: null, + expectedSourceRevision: "11111111-1111-4111-8111-111111111111", + coordinatorMode: "ensure-running" as const, kind: "stack.render" as const, revisionId, }), diff --git a/packages/core/src/ops/trusted-config.ts b/packages/core/src/ops/trusted-config.ts index 32f370c3f..a310b478c 100644 --- a/packages/core/src/ops/trusted-config.ts +++ b/packages/core/src/ops/trusted-config.ts @@ -1,6 +1,11 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import z from "zod/v4"; -import { type OpsOperation, type OpsRole, opsOperationSchema } from "./contract"; +import { + isTrustedConfigurationApplyOperation, + type OpsOperation, + type OpsRole, + opsOperationSchema, +} from "./contract"; import { opsOperationFingerprint } from "./fingerprint"; export const OPS_TRUSTED_CONFIG_MAX_BYTES = 512 * 1024; @@ -209,9 +214,7 @@ function snapshotMac(snapshot: AuthenticatedSnapshot, token: string): string { } function revisionFromOperation(operation: OpsOperation): string | undefined { - return "revisionId" in operation && typeof operation.revisionId === "string" - ? operation.revisionId - : undefined; + return isTrustedConfigurationApplyOperation(operation) ? operation.revisionId : undefined; } function deepFreeze(value: T): T { diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index 7c15fc319..5ef7cf439 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -87,6 +87,9 @@ export interface CommitConfigurationOptions { revisionId?: string; /** Applied identity to compare under writer ownership; null means no pointer. */ expectedCurrentRevisionId?: string | null; + /** Fixed executor correlation for a retained noncommit observation. */ + operationKey?: string; + installationId?: string; signal?: AbortSignal; beforeCommit?: () => Promise; afterGenerationRename?: () => Promise; @@ -114,6 +117,43 @@ export class ConfigurationGenerationError extends Error { export interface ConfigurationGenerationWriter { commit(options: Omit): Promise; } +const liveWriters = new WeakSet(); +const writerCommits = new WeakMap(); +declare const nonCommitBrand: unique symbol; +export type OriginalConfigurationGenerationNonCommit = Readonly<{ [nonCommitBrand]: true }>; +type NonCommitSource = Readonly<{ + infraDir: string; + installationId: string; + operationKey: string; + revisionId: string; + expectedCurrentRevisionId: string; + result: Extract; +}>; +const pendingNonCommits = new WeakMap(), + nonCommitErrors = new WeakMap(), + nonCommitSources = new WeakMap(); +export function configurationGenerationNonCommitForError( + error: unknown, +): OriginalConfigurationGenerationNonCommit | undefined { + return error instanceof Error ? nonCommitErrors.get(error) : undefined; +} +export function originalConfigurationGenerationNonCommitSource( + value: OriginalConfigurationGenerationNonCommit, +): NonCommitSource { + const source = nonCommitSources.get(value); + if (!source) throw Error("Actual completed configuration writer noncommit required"); + return source; +} +export function assertConfigurationGenerationWriter(writer: ConfigurationGenerationWriter): void { + if (!liveWriters.has(writer)) throw Error("Configuration writer is not available"); +} +/** Exact evidence from this live writer, never a competing current-pointer read. */ +export function configurationGenerationWriterCommit( + writer: ConfigurationGenerationWriter, +): Readonly | undefined { + assertConfigurationGenerationWriter(writer); + return writerCommits.get(writer); +} /** Hold ownership before reading state used to construct a write, through all follow-up work. */ export async function withConfigurationGenerationWriter( @@ -127,22 +167,71 @@ export async function withConfigurationGenerationWriter( let value!: T; let active = true; let pending: Promise | undefined; + let writer: ConfigurationGenerationWriter | undefined; try { release = await acquireConfigurationOwnership(infraDir, signal); - value = await run({ + writer = { commit: (options) => { if (!active || pending) throw new Error("Configuration writer is not available"); pending = commitOwnedConfigurationGeneration({ ...options, infraDir, signal: options.signal ?? signal, - }).then((result) => { - committed = result; - return result; - }); + }) + .then((result) => { + committed = result; + writerCommits.set(writer!, result); + return result; + }) + .catch((error) => { + if ( + error instanceof ConfigurationGenerationError && + error.result.outcome === "committed" + ) + writerCommits.set(writer!, { + revisionId: error.result.revisionId, + enabledServiceIds: error.result.enabledServiceIds, + warnings: [], + }); + // Only this awaited native writer attempt, without an injected hook, + // can record conclusive nonpublication while ownership is still held. + if ( + error instanceof ConfigurationGenerationError && + error.result.outcome === "not_committed" && + !error.result.cleanupFailed && + options.revisionId && + options.expectedCurrentRevisionId && + options.operationKey && + options.installationId && + /^opk1_[A-Za-z0-9_-]{16,64}$/.test(options.operationKey) && + !options.beforeCommit && + !options.afterGenerationRename && + !options.afterCommit + ) { + try { + if (currentConfigurationGeneration(infraDir) === options.expectedCurrentRevisionId) + pendingNonCommits.set( + error, + Object.freeze({ + infraDir, + installationId: options.installationId, + operationKey: options.operationKey, + revisionId: options.revisionId, + expectedCurrentRevisionId: options.expectedCurrentRevisionId, + result: Object.freeze({ ...error.result }), + }), + ); + } catch { + /* An unreadable pointer cannot prove nonpublication. */ + } + } + throw error; + }); return pending; }, - }); + }; + liveWriters.add(writer); + value = await run(writer); } catch (error) { failure = error instanceof Error @@ -150,6 +239,7 @@ export async function withConfigurationGenerationWriter( : new Error("Configuration generation failed", { cause: error }); } active = false; + if (writer) liveWriters.delete(writer); // Even a misbehaving callback that did not await commit cannot release its fence early. try { await pending; @@ -191,10 +281,22 @@ export async function withConfigurationGenerationWriter( reason: failure instanceof ConfigurationWriterBusyError ? "busy" : "failed", cleanupFailed: false, }; - throw new ConfigurationGenerationError( + const wrapped = new ConfigurationGenerationError( { ...result, cleanupFailed: result.cleanupFailed || cleanupFailed }, failure, ); + const original = failure instanceof Error ? pendingNonCommits.get(failure) : undefined; + if ( + original && + !cleanupFailed && + wrapped.result.outcome === "not_committed" && + !wrapped.result.cleanupFailed + ) { + const proof = Object.freeze({}) as OriginalConfigurationGenerationNonCommit; + nonCommitSources.set(proof, original); + nonCommitErrors.set(wrapped, proof); + } + throw wrapped; } return value; } diff --git a/packages/core/src/services/coordinator-pull-reference.test.ts b/packages/core/src/services/coordinator-pull-reference.test.ts new file mode 100644 index 000000000..74024e047 --- /dev/null +++ b/packages/core/src/services/coordinator-pull-reference.test.ts @@ -0,0 +1,26 @@ +import { expect, it } from "vitest"; +import { coordinatorPullReferenceIdentity as identity } from "./coordinator-pull-reference"; + +it.each([ + ["example/coordinator:main", "docker.io/example/coordinator:main"], + ["ubuntu", "index.docker.io/library/ubuntu:latest"], + ["docker.io/ubuntu", "ubuntu:latest"], + ["ubuntu:Tag", "docker.io/library/ubuntu:Tag"], + [`ubuntu:ignored@sha256:${"a".repeat(64)}`, `docker.io/library/ubuntu@sha256:${"a".repeat(64)}`], +])("normalizes Docker aliases %s", (a, b) => expect(identity(a)).toBe(identity(b))); +it("keeps tags, case, explicit ports and digest identities distinct", () => { + expect(identity("ubuntu:main")).not.toBe(identity("ubuntu:other")); + expect(identity("ubuntu:Tag")).not.toBe(identity("ubuntu:tag")); + expect(identity("localhost:5000/ubuntu")).not.toBe(identity("localhost/ubuntu")); + expect(identity("ubuntu")).not.toBe(identity(`ubuntu@sha256:${"a".repeat(64)}`)); +}); +it.each([ + "example//repo", + "example/../repo", + "ubuntu:-bad", + "https://docker.io/ubuntu", + "UPPER/repo", + "a".repeat(64), + "ubuntu@sha512:abcd", + "ubuntu:tag:twice", +])("refuses unsupported normalization %s", (value) => expect(() => identity(value)).toThrow()); diff --git a/packages/core/src/services/coordinator-pull-reference.ts b/packages/core/src/services/coordinator-pull-reference.ts new file mode 100644 index 000000000..62918bf44 --- /dev/null +++ b/packages/core/src/services/coordinator-pull-reference.ts @@ -0,0 +1,38 @@ +/** Docker reference identity for the fixed sibling-pull refusal only. Never + * rewrites an operation or authorizes an image/container. Unsupported forms + * fail closed. Follows distribution/reference normalize.go, ParseDockerRef. */ +import { serviceImageReferenceSchema } from "@openmapx/extension-contracts/service"; +export function coordinatorPullReferenceIdentity(reference: string): string { + serviceImageReferenceSchema.parse(reference); + const [tagged, digest] = reference.split("@"); + if (!tagged || /^[a-f0-9]{64}$/.test(tagged)) + throw Error("Unsupported coordinator image reference"); + const colon = tagged.lastIndexOf(":"), + slash = tagged.lastIndexOf("/"), + hasTag = colon > slash; + const name = hasTag ? tagged.slice(0, colon) : tagged, + tag = hasTag ? tagged.slice(colon + 1) : "latest"; + if (!/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(tag)) + throw Error("Unsupported coordinator image tag"); + const parts = name.split("/"), + first = parts[0]!; + let registry = "docker.io", + repository = name; + if (parts.length > 1 && (first === "localhost" || /[.:]/.test(first))) { + registry = first === "index.docker.io" ? "docker.io" : first; + repository = parts.slice(1).join("/"); + } + if ( + !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)*(?::[0-9]{1,5})?$/.test( + registry, + ) + ) + throw Error("Unsupported coordinator registry"); + if (registry === "docker.io" && !repository.includes("/")) repository = `library/${repository}`; + if ( + repository.length > 255 || + !repository.split("/").every((part) => /^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*$/.test(part)) + ) + throw Error("Unsupported coordinator repository"); + return `${registry}/${repository}${digest ? `@${digest}` : `:${tag}`}`; +} diff --git a/packages/core/src/services/creation-dependencies.ts b/packages/core/src/services/creation-dependencies.ts new file mode 100644 index 000000000..c83c0a6ac --- /dev/null +++ b/packages/core/src/services/creation-dependencies.ts @@ -0,0 +1,73 @@ +/** The services Compose may create for a fixed creation command. Selection and + * data producers are deliberately separate from actual depends_on edges. */ +import type { LoadedService } from "./types"; + +const id = /^[a-z0-9][a-z0-9-]{0,63}$/; +const unavailable = () => Error("Service creation dependencies unavailable"); +function closure(targets: readonly string[], dependencies: ReadonlyMap) { + const found = new Set(), + pending = [...targets]; + for (let index = 0; index < pending.length; index++) { + const current = pending[index]!; + if (!id.test(current)) throw unavailable(); + if (found.has(current)) continue; + found.add(current); + if (found.size > 256) throw unavailable(); + const next = dependencies.get(current); + if (!next) throw unavailable(); + for (const dependency of next) if (!found.has(dependency)) pending.push(dependency); + if (pending.length > 65536) throw unavailable(); + } + return [...found].sort(); +} +export function serviceCreationDependencyIds( + services: readonly LoadedService[], + targets: readonly string[], +) { + if ( + services.length > 256 || + new Set(services.map((service) => service.manifest.id)).size !== services.length + ) + throw unavailable(); + return closure( + targets, + new Map( + services.map((service) => [ + service.manifest.id, + (service.manifest.container.dependsOn ?? []).map((dependency) => dependency.service), + ]), + ), + ); +} +export function composeCreationDependencyIds(config: unknown, targets: readonly string[]) { + const values = (config as { services?: unknown } | null)?.services; + if ( + !values || + typeof values !== "object" || + Array.isArray(values) || + Object.keys(values).length > 256 + ) + throw unavailable(); + const dependencies = new Map(); + for (const [service, value] of Object.entries(values)) { + if (!id.test(service) || !value || typeof value !== "object" || Array.isArray(value)) + throw unavailable(); + const raw = (value as { depends_on?: unknown }).depends_on; + const next = + raw === undefined + ? [] + : Array.isArray(raw) + ? raw + : raw && typeof raw === "object" + ? Object.keys(raw) + : null; + if ( + !next || + next.length > 256 || + next.some((dependency) => typeof dependency !== "string" || !id.test(dependency)) + ) + throw unavailable(); + dependencies.set(service, next as string[]); + } + return closure(targets, dependencies); +} diff --git a/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts index c98c7539a..8b037d71a 100644 --- a/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts +++ b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts @@ -30,7 +30,8 @@ export const dataConfigurationFenceReceiptSchema = z.strictObject({ sequence: dataEpochSchema, state: z.enum(["prepared", "settled", "recovery_required"]), }); -export const dataConfigurationFenceSettleSchema = z.strictObject({ +/** Historical persisted receipts only. Fresh mutation never parses this variant. */ +export const legacyDataConfigurationFenceSettleSchema = z.strictObject({ maintenanceId: z.uuid(), candidateId: z.uuid(), planDigest: dataFingerprintSchema, @@ -39,6 +40,16 @@ export const dataConfigurationFenceSettleSchema = z.strictObject({ outcome: z.enum(["unused", "applied", "contained"]), ownerReceiptDigest: dataFingerprintSchema, }); +export const dataConfigurationFenceSettleSchema = z.strictObject({ + version: z.literal(2), + maintenanceId: z.uuid(), + candidateId: z.uuid(), + planDigest: dataFingerprintSchema, + settlementId: dataFingerprintSchema, +}); +export type LegacyDataConfigurationFenceSettle = z.infer< + typeof legacyDataConfigurationFenceSettleSchema +>; export type DataConfigurationFencePrepare = z.infer; export type DataConfigurationFenceReceipt = z.infer; export type DataConfigurationFenceSettle = z.infer; diff --git a/packages/core/src/services/data-manager/generated/models/index.ts b/packages/core/src/services/data-manager/generated/models/index.ts index 822780d8e..566363d01 100644 --- a/packages/core/src/services/data-manager/generated/models/index.ts +++ b/packages/core/src/services/data-manager/generated/models/index.ts @@ -695,7 +695,7 @@ export * from "./settleDataConfigurationFence429"; export * from "./settleDataConfigurationFence500"; export * from "./settleDataConfigurationFence503"; export * from "./settleDataConfigurationFenceBody"; -export * from "./settleDataConfigurationFenceBodyOutcome"; +export * from "./settleDataConfigurationFenceBodyVersion"; export * from "./startDataExecution202"; export * from "./startDataExecution202Diagnostics"; export * from "./startDataExecution202DiagnosticsTailItem"; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts index 77b947693..d55878c5e 100644 --- a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBody.ts @@ -1,22 +1,15 @@ // biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. // Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { SettleDataConfigurationFenceBodyOutcome } from "./settleDataConfigurationFenceBodyOutcome"; +import type { SettleDataConfigurationFenceBodyVersion } from "./settleDataConfigurationFenceBodyVersion"; export type SettleDataConfigurationFenceBody = { + version: SettleDataConfigurationFenceBodyVersion; /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ maintenanceId: string; /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ candidateId: string; /** @pattern ^[a-f0-9]{64}$ */ planDigest: string; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - decisionId: string; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - expectedSequence: number; - outcome: SettleDataConfigurationFenceBodyOutcome; /** @pattern ^[a-f0-9]{64}$ */ - ownerReceiptDigest: string; + settlementId: string; }; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts deleted file mode 100644 index 864bf0301..000000000 --- a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyOutcome.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type SettleDataConfigurationFenceBodyOutcome = - (typeof SettleDataConfigurationFenceBodyOutcome)[keyof typeof SettleDataConfigurationFenceBodyOutcome]; - -export const SettleDataConfigurationFenceBodyOutcome = { - unused: "unused", - applied: "applied", - contained: "contained", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyVersion.ts b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyVersion.ts new file mode 100644 index 000000000..082f8e247 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/settleDataConfigurationFenceBodyVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type SettleDataConfigurationFenceBodyVersion = + (typeof SettleDataConfigurationFenceBodyVersion)[keyof typeof SettleDataConfigurationFenceBodyVersion]; + +export const SettleDataConfigurationFenceBodyVersion = { + NUMBER_2: 2, +} as const; diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 3a93cad5e..5ae2ab99f 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -26,6 +26,7 @@ export { serviceConfigEnvPrefix, } from "./config-resolver"; export { + assertConfigurationGenerationWriter, type CommitConfigurationOptions, type CommittedConfiguration, ConfigurationGenerationError, @@ -33,6 +34,7 @@ export { type ConfigurationInput, commitConfigurationGeneration, configurationGenerationComposeFile, + configurationGenerationWriterCommit, currentConfigurationGeneration, initializeConfigurationGenerations, inspectAppliedConfiguration, @@ -47,6 +49,11 @@ export { assertTrustedConfigurationSchema, validateTrustedConfigurationValues, } from "./configuration-schema"; +export { coordinatorPullReferenceIdentity } from "./coordinator-pull-reference"; +export { + composeCreationDependencyIds, + serviceCreationDependencyIds, +} from "./creation-dependencies"; export type { DataManagerCoverageEvidencePage, NotablePlacesBuildResult, @@ -105,6 +112,11 @@ export { type ReleaseServiceAuthorityCaptureHooks, validateReleaseServiceAuthority, } from "./release-authority-inventory"; +export { + configurationImageReference, + readServiceReleaseImageSelection, + validateExistingReleaseSelection, +} from "./release-image-selection"; export { canonicalReleaseManifest, DEFAULT_RELEASE_MANIFEST_REPOSITORY, @@ -170,6 +182,7 @@ export { parseServiceIdList, SERVICE_SELECTION_ENV, } from "./selection"; +export * from "./service-lifecycle-receipt"; export { renderTraefikDynamicConfiguration, renderTraefikDynamicYaml, diff --git a/packages/core/src/services/release-image-selection.ts b/packages/core/src/services/release-image-selection.ts new file mode 100644 index 000000000..257708db1 --- /dev/null +++ b/packages/core/src/services/release-image-selection.ts @@ -0,0 +1,102 @@ +/** Read-only eligibility shared by CLI preparation and the fixed normal pull. + * This module never resolves, writes or repairs an overlay. */ +import { createHash } from "node:crypto"; +import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { serviceImageReferenceSchema } from "@openmapx/extension-contracts/service"; +import { JSON_SCHEMA, load } from "js-yaml"; +import { canonicalJson } from "../ops/canonical-json"; +import { + parseReleaseComposeSelection, + RELEASE_IMAGE_NAMES, + RELEASE_PINNED_SERVICE_IDS, + type ReleaseChannel, + type ReleaseComposeSelection, + releaseChannel, +} from "./release-manifest"; +export function configurationImageReference(composeYaml: string, serviceId: string) { + const document = load(composeYaml, { schema: JSON_SCHEMA }) as { + services?: Record; + }; + return serviceImageReferenceSchema.parse(document?.services?.[serviceId]?.image); +} + +export function validateExistingReleaseSelection( + raw: string, + channel: ReleaseChannel = releaseChannel(), +): ReleaseComposeSelection { + const selected = parseReleaseComposeSelection(raw), + prefix = + channel.kind === "enabled" ? channel.imagePrefix : selected.images.api?.split("/api@")[0]; + if ( + !selected.release || + !prefix || + !RELEASE_IMAGE_NAMES.every( + (name) => + selected.images[name]?.startsWith(`${prefix}/${name}@`) && + serviceImageReferenceSchema.safeParse(selected.images[name]).success, + ) + ) + throw Error( + "Existing release overlay is malformed or belongs to another configured repository; explicitly reselect or clear it", + ); + return selected; +} + +export function readServiceReleaseImageSelection(infraDir: string, serviceId: string) { + const managed = (RELEASE_PINNED_SERVICE_IDS as readonly string[]).includes(serviceId), + channel = releaseChannel(), + path = join(infraDir, "docker-compose.release.yml"); + let fd: number | undefined; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1 || before.size > 128 * 1024) + throw Error("Release selection unavailable"); + const raw = readFileSync(fd), + after = fstatSync(fd), + current = lstatSync(path); + if ( + before.dev !== current.dev || + before.ino !== current.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + current.size !== after.size || + current.mtimeMs !== after.mtimeMs || + current.ctimeMs !== after.ctimeMs + ) + throw Error("Release selection changed"); + const selected = validateExistingReleaseSelection( + new TextDecoder("utf-8", { fatal: true }).decode(raw), + channel, + ); + return { + imageReference: managed + ? selected.images[ + serviceId === "app-api" + ? "api" + : serviceId === "app-web" + ? "web" + : (serviceId as keyof typeof selected.images) + ]! + : null, + releaseSelectionDigest: managed + ? createHash("sha256") + .update("openmapx-service-release-selection-v1\0") + .update(canonicalJson(selected)) + .digest("hex") + : null, + }; + } catch (error) { + if ( + fd !== undefined || + (error as NodeJS.ErrnoException).code !== "ENOENT" || + (managed && channel.kind !== "disabled") + ) + throw error; + return { imageReference: null, releaseSelectionDigest: null }; + } finally { + if (fd !== undefined) closeSync(fd); + } +} diff --git a/packages/core/src/services/service-lifecycle-receipt.test.ts b/packages/core/src/services/service-lifecycle-receipt.test.ts new file mode 100644 index 000000000..1666f4802 --- /dev/null +++ b/packages/core/src/services/service-lifecycle-receipt.test.ts @@ -0,0 +1,278 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, + restoreAdmittedControlExecution, + retainAdmittedControlExecution, +} from "../build-artifacts/control-authority"; +import { AnchoredDirectory } from "../build-artifacts/filesystem"; +import { + acquireArtifactTargetLease, + closeArtifactTargetLeaseHandle, +} from "../build-artifacts/target-lease"; +import { + createControlRootOperationKey, + executionFingerprint, + opsOperationFingerprint, +} from "../ops"; +import { withConfigurationGenerationWriter } from "./configuration-generation"; +import { + openServiceLifecycleReceipt, + readServiceLifecycleReceipt, + withServiceLifecycleReceiptAuthority, + withVerifiedLifecycleOwnership, +} from "./service-lifecycle-receipt"; + +let root: string; +const revision = `cfg1_${"a".repeat(43)}`; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-lifecycle-receipt-")); + mkdirSync(join(root, "data")); + await provisionControlInstallation(root); +}); +afterEach(() => { + vi.restoreAllMocks(); + rmSync(root, { recursive: true, force: true }); +}); +function owner() { + const installation = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["alpha", "beta"], + }); + const operation = { + kind: "service.update" as const, + serviceId: "alpha", + expectedConfigurationRevisionId: revision, + }; + const deadlineAt = Date.now() + 60_000, + operationFingerprint = opsOperationFingerprint(operation); + const execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const authority = retainAdmittedControlExecution(installation, { + operation, + execution, + kind: operation.kind, + admittedAt: deadlineAt - 60_000, + requestedDeadlineAt: deadlineAt, + serviceIds: ["alpha", "beta"], + }); + return { installation, authority, execution }; +} +function options(execution: ReturnType["execution"]) { + return { + dataRoot: join(root, "data"), + execution, + serviceIds: ["alpha"], + action: "update" as const, + noDeps: false, + coordinatorMode: "ensure-running" as const, + expectedRevisionId: revision, + seedRevisionId: revision, + }; +} +it("signs exact seed and checkpoints only with the live writer and physical target owner", async () => { + const o = owner(); + let lease: Awaited> | undefined; + try { + lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + owner: o.authority, + serviceIds: ["alpha", "beta"], + }); + const store = await withServiceLifecycleReceiptAuthority(o.authority, () => + openServiceLifecycleReceipt(options(o.execution)), + ); + const seed = await store.read(); + expect(seed.commit).toBe("seed_committed"); + await withConfigurationGenerationWriter(root, (writer) => + withVerifiedLifecycleOwnership( + { writer, lease: lease!, authority: o.authority }, + async (ownership) => { + await store.checkpoint(0, { ...seed, phase: "pins_validated", sequence: 1 }, ownership); + await expect( + store.checkpoint(0, { ...seed, phase: "pulled", sequence: 1 }, ownership), + ).rejects.toThrow(); + }, + ), + ); + expect((await readServiceLifecycleReceipt(options(o.execution))).phase).toBe("pins_validated"); + const path = join(root, "data/ops-agent/lifecycle", `${o.execution.operationKey}.json`), + original = readFileSync(path); + const tampered = JSON.parse(original.toString()); + tampered.current.payload.receipt.finalRevisionId = revision; + writeFileSync(path, JSON.stringify(tampered), { mode: 0o600 }); + await expect(readServiceLifecycleReceipt(options(o.execution))).rejects.toThrow(); + } finally { + if (lease) closeArtifactTargetLeaseHandle(lease); + closeControlInstallation(o.installation); + } +}); +it("rejects a foreign data root before creating any shared receipt", async () => { + const o = owner(), + foreign = join(root, "foreign"); + mkdirSync(foreign); + try { + await expect( + withServiceLifecycleReceiptAuthority(o.authority, () => + openServiceLifecycleReceipt({ ...options(o.execution), dataRoot: foreign }), + ), + ).rejects.toThrow(); + expect(existsSync(join(foreign, "ops-agent"))).toBe(false); + } finally { + closeControlInstallation(o.installation); + } +}); +it("rejects targets outside the original admitted closure before creating a receipt", async () => { + const o = owner(); + try { + await expect( + withServiceLifecycleReceiptAuthority(o.authority, () => + openServiceLifecycleReceipt({ ...options(o.execution), serviceIds: ["foreign"] }), + ), + ).rejects.toThrow(); + } finally { + closeControlInstallation(o.installation); + } +}); +it("refuses a final revision that the owned writer did not commit", async () => { + const o = owner(); + let lease: Awaited> | undefined; + try { + lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + owner: o.authority, + serviceIds: ["alpha", "beta"], + }); + const store = await withServiceLifecycleReceiptAuthority(o.authority, () => + openServiceLifecycleReceipt(options(o.execution)), + ); + await withConfigurationGenerationWriter(root, (writer) => + withVerifiedLifecycleOwnership( + { writer, lease: lease!, authority: o.authority }, + async (ownership) => { + const seed = await store.read(); + await store.checkpoint(0, { ...seed, sequence: 1, phase: "pins_validated" }, ownership); + await store.checkpoint(1, { ...seed, sequence: 2, phase: "pulled" }, ownership); + await expect( + store.checkpoint( + 2, + { + ...seed, + sequence: 3, + phase: "generation_committed", + commit: "final_committed", + finalRevisionId: `cfg1_${"b".repeat(43)}`, + }, + ownership, + ), + ).rejects.toThrow(); + }, + ), + ); + } finally { + if (lease) closeArtifactTargetLeaseHandle(lease); + closeControlInstallation(o.installation); + } +}); +it.each([false, true])( + "repairs only the original signed lifecycle publication after restart with shared commit=%s", + async (committed) => { + const o = owner(); + let lease: Awaited> | undefined; + let restarted: ReturnType | undefined; + try { + lease = await acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + owner: o.authority, + serviceIds: ["alpha", "beta"], + }); + const store = await withServiceLifecycleReceiptAuthority(o.authority, () => + openServiceLifecycleReceipt(options(o.execution)), + ); + const write = AnchoredDirectory.prototype.write; + let interrupted = false; + const fault = vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(function ( + this: AnchoredDirectory, + name, + bytes, + ...args + ) { + if (!interrupted) { + const value = JSON.parse(String(bytes)); + if ( + committed + ? name === `lifecycle-${o.execution.operationKey}.json` && + value.payload?.sequence === 1 && + value.payload?.pending === null + : name === `${o.execution.operationKey}.json` && + value.current?.payload?.receipt?.sequence === 1 + ) { + interrupted = true; + throw Error("Synthetic lifecycle filesystem interruption"); + } + } + return write.call(this, name, bytes, ...args); + }); + await expect( + withConfigurationGenerationWriter(root, (writer) => + withVerifiedLifecycleOwnership( + { writer, lease: lease!, authority: o.authority }, + async (ownership) => { + const seed = await store.read(); + await store.checkpoint( + 0, + { ...seed, sequence: 1, phase: "pins_validated" }, + ownership, + ); + }, + ), + ), + ).rejects.toThrow("Synthetic lifecycle filesystem interruption"); + expect(interrupted).toBe(true); + fault.mockRestore(); + closeArtifactTargetLeaseHandle(lease); + lease = undefined; + closeControlInstallation(o.installation); + restarted = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["alpha", "beta"], + }); + const authority = restoreAdmittedControlExecution(restarted, o.execution); + // An unscoped read never repairs a private high-water mark. + await expect(readServiceLifecycleReceipt(options(o.execution))).rejects.toThrow(); + const recovered = withServiceLifecycleReceiptAuthority(authority, () => + readServiceLifecycleReceipt(options(o.execution)), + ); + if (committed) { + await expect(recovered).resolves.toMatchObject({ + phase: "pins_validated", + sequence: 1, + finalRevisionId: null, + }); + await expect(readServiceLifecycleReceipt(options(o.execution))).resolves.toMatchObject({ + phase: "pins_validated", + sequence: 1, + }); + } else await expect(recovered).rejects.toThrow(); + expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(true); + } finally { + if (lease) closeArtifactTargetLeaseHandle(lease); + if (restarted) closeControlInstallation(restarted); + else closeControlInstallation(o.installation); + } + }, +); diff --git a/packages/core/src/services/service-lifecycle-receipt.ts b/packages/core/src/services/service-lifecycle-receipt.ts new file mode 100644 index 000000000..3626f0f36 --- /dev/null +++ b/packages/core/src/services/service-lifecycle-receipt.ts @@ -0,0 +1,590 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { dirname } from "node:path"; +import { z } from "zod"; +import { + closeControlInstallation, + executionAuthorityState, + fixedControlChildId, + installationState, + openControlInstallation, + readPrivateRecord, + type VerifiedExecutionAuthority, + withPrivateEffectWriter, + writePrivateRecord, +} from "../build-artifacts/control-authority"; +import { controlDigest, signControl, verifyControl } from "../build-artifacts/control-records"; +import { AnchoredDirectory, identity, sameInode } from "../build-artifacts/filesystem"; +import { familyRecordWrite } from "../build-artifacts/retention-family"; +import { + assertArtifactTargetLeaseExecution, + assertArtifactTargetLeaseReceiptOwner, + type TargetLeaseAuthority, +} from "../build-artifacts/target-lease"; +import { canonicalJson } from "../ops/canonical-json"; +import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { + type CoordinatorMode, + hardlinkApplySummarySchema, + serviceLifecycleFailureEvidenceSchema, +} from "../ops/service-results"; +import { + assertConfigurationGenerationWriter, + type ConfigurationGenerationWriter, + configurationGenerationWriterCommit, +} from "./configuration-generation"; + +const revision = z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .nullable(); +const ids = z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine((values) => values.every((value, index) => index === 0 || values[index - 1]! < value)); +export const serviceLifecycleReceiptSchema = serviceLifecycleFailureEvidenceSchema + .extend({ + version: z.literal(1), + execution: executionRefSchema, + serviceIds: ids, + action: z.enum(["start", "update"]), + noDeps: z.boolean(), + coordinatorMode: z.enum(["ensure-running", "require-running"]), + hardlinks: hardlinkApplySummarySchema.optional(), + sequence: z.number().int().nonnegative().safe(), + }) + .superRefine((value, context) => { + if ( + (value.commit === "not_committed" && (value.seedRevisionId || value.finalRevisionId)) || + (value.commit === "seed_committed" && !value.seedRevisionId) || + (value.commit === "final_committed" && !value.finalRevisionId) || + (value.phase === "complete" && + (!value.finalRevisionId || + value.commit !== "final_committed" || + value.followUp !== "complete" || + !value.hardlinks?.applied || + value.failure)) + ) + context.addIssue({ code: "custom", message: "Inconsistent lifecycle evidence" }); + }); +export type ServiceLifecycleReceipt = z.infer; +declare const ownershipBrand: unique symbol; +export type VerifiedLifecycleOwnership = Readonly<{ + writer: ConfigurationGenerationWriter; + lease: TargetLeaseAuthority; + [ownershipBrand]: true; +}>; +const ownerships = new WeakMap(); +const authorities = new AsyncLocalStorage<{ + authority: VerifiedExecutionAuthority; + child?: { execution: ExecutionRef; publicKey: string }; +}>(); + +/** Source-only scope: child identity is checked against its existing private spawn challenge. */ +export function withServiceLifecycleReceiptAuthority( + authority: VerifiedExecutionAuthority, + run: () => Promise, + child?: { execution: ExecutionRef; publicKey: string }, +): Promise { + const state = executionAuthorityState(authority); + if (child) { + const id = fixedControlChildId(state.installation, child.publicKey, authority.execution); + const spawn = z + .object({ execution: executionRefSchema }) + .parse(readPrivateRecord(state.installation, "spawns", `${id}.json`, 65536)); + if (controlDigest(spawn.execution) !== controlDigest(child.execution)) + throw Error("Lifecycle child challenge mismatch"); + } + return authorities.run({ authority, child }, run); +} +export async function withVerifiedLifecycleOwnership( + options: { + writer: ConfigurationGenerationWriter; + lease: TargetLeaseAuthority; + authority: VerifiedExecutionAuthority; + }, + run: (ownership: VerifiedLifecycleOwnership) => Promise, +): Promise { + assertConfigurationGenerationWriter(options.writer); + assertArtifactTargetLeaseExecution(options.lease, options.authority.execution); + executionAuthorityState(options.authority); + const ownership = Object.freeze({ + writer: options.writer, + lease: options.lease, + }) as VerifiedLifecycleOwnership; + ownerships.set(ownership, options.authority); + try { + return await withServiceLifecycleReceiptAuthority(options.authority, () => run(ownership)); + } finally { + ownerships.delete(ownership); + } +} +export function assertVerifiedLifecycleOwnership( + ownership: VerifiedLifecycleOwnership, +): VerifiedExecutionAuthority { + const authority = ownerships.get(ownership); + if (!authority) throw Error("Lifecycle ownership is unavailable"); + assertConfigurationGenerationWriter(ownership.writer); + assertArtifactTargetLeaseReceiptOwner(ownership.lease, authority.execution); + executionAuthorityState(authority); + return authority; +} + +const seedSchema = z.strictObject({ + version: z.literal(1), + issuer: z.enum(["ops-agent", "host-cli"]), + root: executionRefSchema, + execution: executionRefSchema, + serviceIds: ids, + action: z.enum(["start", "update"]), + noDeps: z.boolean(), + coordinatorMode: z.enum(["ensure-running", "require-running"]), + expectedRevisionId: revision, + seedRevisionId: revision, + childPublicKey: z.string().min(1).max(256), +}); +const envelopeSchema = z.strictObject({ + seed: z.strictObject({ payload: seedSchema, signature: z.string().max(128) }), + current: z.strictObject({ + payload: z.strictObject({ + seedDigest: z.string().regex(/^[a-f0-9]{64}$/), + receipt: serviceLifecycleReceiptSchema, + }), + signature: z.string().max(128), + }), +}); +const highWaterSchema = z.strictObject({ + version: z.literal(1), + execution: executionRefSchema, + seedDigest: z.string().regex(/^[a-f0-9]{64}$/), + sequence: z.number().int().nonnegative().safe(), + receiptDigest: z.string().regex(/^[a-f0-9]{64}$/), + signer: z.enum(["seed", "child"]), + pending: z + .strictObject({ + sequence: z.number().int().positive().safe(), + receiptDigest: z.string().regex(/^[a-f0-9]{64}$/), + }) + .nullable(), +}); +const seedDomain = "openmapx-service-lifecycle-seed-v1", + receiptDomain = "openmapx-service-lifecycle-receipt-v1", + highWaterDomain = "openmapx-service-lifecycle-high-water-v1"; +const highWaterName = (execution: ExecutionRef) => + `lifecycle-${executionRefSchema.parse(execution).operationKey}.json`; +function assertDataRoot(dataRoot: string, trusted: string) { + const supplied = new AnchoredDirectory(dataRoot), + expected = new AnchoredDirectory(trusted); + try { + if (!sameInode(identity(supplied.stat), identity(expected.stat))) + throw Error("Foreign lifecycle data root"); + } finally { + supplied.close(); + expected.close(); + } +} +function directories(dataRoot: string) { + const root = new AnchoredDirectory(dataRoot); + const agent = root.directory("ops-agent", true, true), + directory = agent.directory("lifecycle", true, true); + return { + directory, + close() { + directory.close(); + agent.close(); + root.close(); + }, + }; +} +function same(value: unknown, other: unknown) { + return canonicalJson(value) === canonicalJson(other); +} +function readEnvelope(dataRoot: string, execution: ExecutionRef) { + const opened = directories(dataRoot); + try { + return envelopeSchema.parse( + JSON.parse( + opened.directory + .read(`${executionRefSchema.parse(execution).operationKey}.json`, 65536) + .toString("utf8"), + ), + ); + } finally { + opened.close(); + } +} +function inspect(dataRoot: string, execution: ExecutionRef, allowRecovery = false) { + const envelope = readEnvelope(dataRoot, execution), + scope = authorities.getStore(); + let close: (() => void) | undefined; + const local = scope + ? executionAuthorityState(scope.authority).installation + : (() => { + const installation = openControlInstallation({ + infraDir: dirname(dataRoot), + role: envelope.seed.payload.issuer, + }); + close = () => closeControlInstallation(installation); + return installationState(installation); + })(); + try { + assertDataRoot(dataRoot, local.dataRoot); + const seed = verifyControl( + local.publicKeys[envelope.seed.payload.issuer], + seedDomain, + envelope.seed, + seedSchema, + ); + const seedDigest = controlDigest(envelope.seed), + publicKey = + envelope.current.payload.receipt.sequence === 0 + ? local.publicKeys[seed.issuer] + : seed.childPublicKey; + const current = verifyControl( + publicKey, + receiptDomain, + envelope.current, + envelopeSchema.shape.current.shape.payload, + ); + const raw = readPrivateRecord(local, "observations", highWaterName(execution), 8192); + const signer = (raw as { payload?: { signer?: string } }).payload?.signer; + let water = verifyControl( + signer === "child" ? seed.childPublicKey : local.publicKeys[seed.issuer], + highWaterDomain, + raw, + highWaterSchema, + ); + const receipt = current.receipt; + if ( + !same(seed.execution, execution) || + !same(receipt.execution, execution) || + !same(water.execution, execution) || + current.seedDigest !== seedDigest || + water.seedDigest !== seedDigest || + !same(seed.serviceIds, receipt.serviceIds) || + seed.action !== receipt.action || + seed.noDeps !== receipt.noDeps || + seed.coordinatorMode !== receipt.coordinatorMode || + seed.expectedRevisionId !== receipt.expectedRevisionId || + seed.seedRevisionId !== receipt.seedRevisionId + ) + throw Error("Lifecycle receipt is unavailable"); + if (water.pending && allowRecovery && scope) { + const owner = executionAuthorityState(scope.authority), + pending = water.pending; + if ( + !same(owner.execution, execution) || + owner.kind !== `service.${seed.action}` || + owner.installation.installation.publicKey !== seed.childPublicKey || + !owner.origin || + water.signer !== "child" || + pending.sequence !== water.sequence + 1 || + receipt.sequence !== pending.sequence || + controlDigest(envelope.current) !== pending.receiptDigest + ) + throw Error("Lifecycle publication requires reconciliation"); + const repaired = { + ...water, + sequence: pending.sequence, + receiptDigest: pending.receiptDigest, + pending: null, + }; + withPrivateEffectWriter(owner, () => { + if ( + controlDigest( + readPrivateRecord(local, "observations", highWaterName(execution), 8192), + ) !== controlDigest(raw) || + controlDigest(readEnvelope(dataRoot, execution)) !== controlDigest(envelope) + ) + throw Error("Lifecycle publication changed during recovery"); + writePrivateRecord( + local, + "observations", + highWaterName(execution), + signControl(local.privateKey, highWaterDomain, repaired), + false, + owner.origin, + ); + }); + water = repaired; + } + if ( + water.pending || + water.sequence !== receipt.sequence || + water.receiptDigest !== controlDigest(envelope.current) + ) + throw Error("Lifecycle receipt is unavailable"); + return { envelope, seed, receipt, water, local }; + } finally { + close?.(); + } +} +export interface ServiceLifecycleReceiptStore { + read(): Promise; + checkpoint( + expectedSequence: number, + receipt: ServiceLifecycleReceipt, + ownership: VerifiedLifecycleOwnership, + ): Promise; +} +export async function openServiceLifecycleReceipt(options: { + dataRoot: string; + execution: ExecutionRef; + serviceIds: readonly string[]; + action: "start" | "update"; + noDeps: boolean; + coordinatorMode: CoordinatorMode; + expectedRevisionId: string | null; + seedRevisionId: string | null; +}): Promise { + const scope = authorities.getStore(); + if (!scope) throw Error("Lifecycle receipt owner is required"); + const state = executionAuthorityState(scope.authority), + local = state.installation; + assertDataRoot(options.dataRoot, local.dataRoot); + if (!same(options.execution, scope.child?.execution ?? scope.authority.execution)) + throw Error("Lifecycle receipt execution mismatch"); + const admitted = readPrivateRecord( + local, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { + payload: { + operation: { + kind: string; + serviceId?: string; + expectedConfigurationRevisionId?: string; + noDeps?: boolean; + }; + }; + }; + const operation = admitted.payload.operation; + if ( + operation.kind !== `service.${options.action}` || + !operation.serviceId || + !options.serviceIds.includes(operation.serviceId) || + options.serviceIds.some((id) => !state.serviceIds.includes(id)) || + options.noDeps !== (operation.noDeps ?? false) || + options.expectedRevisionId !== operation.expectedConfigurationRevisionId || + options.coordinatorMode !== "ensure-running" + ) + throw Error("Lifecycle seed does not match admitted operation"); + const opened = directories(options.dataRoot), + name = `${executionRefSchema.parse(options.execution).operationKey}.json`; + try { + let exists = true; + try { + opened.directory.fileIdentity(name); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") exists = false; + else throw error; + } + if (!exists) { + if (local.fixedChildId) throw Error("Agent seed receipt is missing"); + try { + readPrivateRecord(local, "observations", highWaterName(options.execution), 8192); + throw Error("Lifecycle receipt initialization is unresolved"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (opened.directory.listBounded(129).length >= 128) + throw Error("Lifecycle receipt capacity unavailable"); + const serviceIds = [...new Set(options.serviceIds)].sort(); + const receipt = serviceLifecycleReceiptSchema.parse({ + version: 1, + execution: options.execution, + serviceIds, + action: options.action, + noDeps: options.noDeps, + coordinatorMode: options.coordinatorMode, + expectedRevisionId: options.expectedRevisionId, + seedRevisionId: options.seedRevisionId, + finalRevisionId: null, + phase: "prepared", + commit: options.seedRevisionId ? "seed_committed" : "not_committed", + followUp: "not_started", + sequence: 0, + }); + const seed = signControl( + local.privateKey, + seedDomain, + seedSchema.parse({ + version: 1, + issuer: local.installation.role, + root: scope.authority.execution, + execution: options.execution, + serviceIds, + action: options.action, + noDeps: options.noDeps, + coordinatorMode: options.coordinatorMode, + expectedRevisionId: options.expectedRevisionId, + seedRevisionId: options.seedRevisionId, + childPublicKey: scope.child?.publicKey ?? local.installation.publicKey, + }), + ); + const seedDigest = controlDigest(seed), + current = signControl(local.privateKey, receiptDomain, { seedDigest, receipt }); + const envelope = envelopeSchema.parse({ seed, current }), + bytes = canonicalJson(envelope); + if (Buffer.byteLength(bytes) > 65536) throw Error("Lifecycle receipt budget exceeded"); + writePrivateRecord( + local, + "observations", + highWaterName(options.execution), + signControl(local.privateKey, highWaterDomain, { + version: 1, + execution: options.execution, + seedDigest, + sequence: 0, + receiptDigest: controlDigest(current), + signer: "seed", + pending: null, + }), + true, + state.origin, + ); + if (!state.origin) throw Error("Original lifecycle retention family required"); + familyRecordWrite( + local, + state.origin, + { scope: "shared", namespace: "service-lifecycle", container: "", name }, + bytes, + (track) => opened.directory.write(name, bytes, true, track), + true, + ); + } + } finally { + opened.close(); + } + const existing = inspect(options.dataRoot, options.execution).receipt; + if ( + !same(existing.serviceIds, [...new Set(options.serviceIds)].sort()) || + existing.action !== options.action || + existing.noDeps !== options.noDeps || + existing.coordinatorMode !== options.coordinatorMode || + existing.expectedRevisionId !== options.expectedRevisionId || + existing.seedRevisionId !== options.seedRevisionId + ) + throw Error("Lifecycle receipt identity conflict"); + return { + async read() { + return readServiceLifecycleReceipt(options); + }, + async checkpoint(expectedSequence, next, ownership) { + const authority = assertVerifiedLifecycleOwnership(ownership); + const owner = executionAuthorityState(authority); + return withPrivateEffectWriter(owner, () => { + const current = inspect(options.dataRoot, options.execution); + const receipt = serviceLifecycleReceiptSchema.parse(next); + const transitions: Record< + ServiceLifecycleReceipt["phase"], + readonly ServiceLifecycleReceipt["phase"][] + > = { + prepared: ["pins_validated"], + pins_validated: [receipt.action === "update" ? "pulled" : "generation_committed"], + pulled: ["generation_committed"], + generation_committed: ["hardlinks_applied"], + hardlinks_applied: ["recreated"], + recreated: ["complete"], + complete: [], + failed: [], + unknown: [], + }; + if ( + !["failed", "unknown"].includes(receipt.phase) && + !transitions[current.receipt.phase].includes(receipt.phase) + ) + throw Error("Lifecycle phase transition conflict"); + if ( + receipt.finalRevisionId !== null && + current.receipt.finalRevisionId === null && + configurationGenerationWriterCommit(ownership.writer)?.revisionId !== + receipt.finalRevisionId + ) + throw Error("Lifecycle revision was not committed by its writer"); + if (current.receipt.commit === "final_committed" && receipt.commit !== "final_committed") + throw Error("Lifecycle commit evidence cannot regress"); + if (!["failed", "unknown"].includes(receipt.phase)) + assertArtifactTargetLeaseExecution(ownership.lease, authority.execution); + if ( + !same(authority.execution, receipt.execution) || + owner.installation.installation.publicKey !== current.seed.childPublicKey || + expectedSequence !== current.receipt.sequence || + receipt.sequence !== expectedSequence + 1 || + ["complete", "failed", "unknown"].includes(current.receipt.phase) || + !same(receipt.execution, current.receipt.execution) || + !same(receipt.serviceIds, current.receipt.serviceIds) || + receipt.action !== current.receipt.action || + receipt.noDeps !== current.receipt.noDeps || + receipt.coordinatorMode !== current.receipt.coordinatorMode || + receipt.expectedRevisionId !== current.receipt.expectedRevisionId || + receipt.seedRevisionId !== current.receipt.seedRevisionId || + (current.receipt.finalRevisionId !== null && + current.receipt.finalRevisionId !== receipt.finalRevisionId) + ) + throw Error("Lifecycle receipt checkpoint conflict"); + const signed = signControl(owner.installation.privateKey, receiptDomain, { + seedDigest: controlDigest(current.envelope.seed), + receipt, + }); + const envelope = { seed: current.envelope.seed, current: signed }, + bytes = canonicalJson(envelope); + if (Buffer.byteLength(bytes) > 65536) throw Error("Lifecycle receipt budget exceeded"); + const directory = directories(options.dataRoot); + try { + const before = directory.directory.fileIdentity(name); + const pending = { + ...current.water, + signer: "child" as const, + pending: { sequence: receipt.sequence, receiptDigest: controlDigest(signed) }, + }; + writePrivateRecord( + owner.installation, + "observations", + highWaterName(options.execution), + signControl(owner.installation.privateKey, highWaterDomain, pending), + false, + owner.origin, + ); + if (!owner.origin) throw Error("Original lifecycle retention family required"); + familyRecordWrite( + owner.installation, + owner.origin, + { scope: "shared", namespace: "service-lifecycle", container: "", name }, + bytes, + (track) => + directory.directory.write(name, bytes, false, (inode) => { + // Physical ownership was checked above, before entering the family writer. + assertConfigurationGenerationWriter(ownership.writer); + if (!sameInode(before, directory.directory.fileIdentity(name))) + throw Error("Lifecycle receipt inode changed"); + track(inode); + }), + ); + writePrivateRecord( + owner.installation, + "observations", + highWaterName(options.execution), + signControl(owner.installation.privateKey, highWaterDomain, { + ...pending, + sequence: receipt.sequence, + receiptDigest: controlDigest(signed), + pending: null, + }), + false, + owner.origin, + ); + } finally { + directory.close(); + } + }); + }, + }; +} +export async function readServiceLifecycleReceipt(options: { + dataRoot: string; + execution: ExecutionRef; +}): Promise { + return inspect(options.dataRoot, executionRefSchema.parse(options.execution), true).receipt; +} diff --git a/packages/extension-contracts/src/service.ts b/packages/extension-contracts/src/service.ts index b03ad3ff1..fa9f83780 100644 --- a/packages/extension-contracts/src/service.ts +++ b/packages/extension-contracts/src/service.ts @@ -7,6 +7,26 @@ import type { ManifestProvenance, ServiceManifest } from "./service-types"; const IMAGE_REGEX = /^[a-z0-9]([a-z0-9._\-/])*$/; const TAG_REGEX = /^[a-zA-Z0-9._-]+$/; const IMAGE_DIGEST_REGEX = /^sha256:[a-f0-9]{64}$/; +/** Bounded references derived from trusted manifest/release image fields. + * Parsing is data validation, never permission to choose or pull an image. */ +export const serviceImageReferenceSchema = z + .string() + .min(1) + .max(2048) + .refine((value) => { + const [reference = "", digest, ...extra] = value.split("@"), + colon = reference.lastIndexOf(":"), + tagged = colon > reference.lastIndexOf("/"), + repository = tagged ? reference.slice(0, colon) : reference, + tag = tagged ? reference.slice(colon + 1) : undefined, + withoutRegistryPort = repository.replace(/^([^/]+):[0-9]{1,5}\//, "$1/"); + return ( + extra.length === 0 && + IMAGE_REGEX.test(withoutRegistryPort) && + (tag === undefined || TAG_REGEX.test(tag)) && + (digest === undefined || IMAGE_DIGEST_REGEX.test(digest)) + ); + }); // Docker's container-name grammar: `[a-zA-Z0-9][a-zA-Z0-9_.-]*` (single-char // names are legal, hence `*` not `+` after the leading char). const CONTAINER_NAME_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/; diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 2e0a769b7..0a8eae504 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -26,6 +26,10 @@ ], "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", + "apps/api/src/services/__tests__/prospective-configuration-postgres.test.ts", + "apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts", + "apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts", + "apps/api/src/services/configuration-migration/legacy-integration-import.test.ts", "services/data-manager/src/jobs/execution/store-postgres.test.ts", "services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts", "services/data-manager/src/jobs/execution/runtime.test.ts", diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index b2b271bff..7c92e6407 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -36,8 +36,15 @@ "backup.list", "backup.restore", "communityServices.inspect", + "configuration.hardlinks.apply", "configuration.hardlinks.inspect", + "configuration.hardlinks.preview", "configuration.inspect", + "configuration.sources.inspect", + "configurationMaintenance.decide", + "configurationMaintenance.inspect", + "configurationMaintenance.prepare", + "configurationMaintenance.settle", "controlRetention.acknowledge", "data.clean", "data.convertOverpass", @@ -201,8 +208,16 @@ { "id": "api.configuration-hardlinks", "callSiteIds": [], - "semanticSources": ["apps/api/src/services/admin-ops.ts#applyGenerationHardlinks"], - "typedKinds": ["configuration.hardlinks.inspect"] + "semanticSources": [ + "apps/api/src/services/configuration-hardlinks.ts#applyGenerationHardlinks", + "apps/ops-agent/src/configuration-hardlinks-runtime.ts#installConfigurationHardlinksRuntime", + "apps/ops-agent/src/trusted-config-runtime.ts#installTrustedConfigurationRuntime" + ], + "typedKinds": [ + "configuration.hardlinks.inspect", + "configuration.hardlinks.apply", + "configuration.hardlinks.preview" + ] }, { "id": "api.compose-runtime", @@ -269,14 +284,16 @@ "apps/api/src/services/trusted-config-operations.ts#applyTrustedConfigurationWithEvidence", "apps/api/src/services/admin-cli.ts#validateServiceSelectionForWrite", "apps/api/src/routes/admin.ts#adminRoute", - "apps/api/src/services/service-secrets.ts#setServiceSecret" + "apps/api/src/services/service-secrets.ts#setServiceSecret", + "packages/core/src/build-artifacts/configuration-source-witness.ts#observeConfigurationSources" ], "typedKinds": [ "configuration.inspect", "serviceSelection.apply", "serviceConfig.apply", "integrationConfig.apply", - "vault.apply" + "vault.apply", + "configuration.sources.inspect" ] }, { @@ -313,6 +330,23 @@ "transitousLock.approve", "gbfsCatalogLock.inspect" ] + }, + { + "id": "api.configuration-maintenance", + "callSiteIds": [], + "semanticSources": [ + "apps/api/src/services/prospective-configuration.ts#prepareConfigurationMaintenance", + "apps/api/src/services/prospective-configuration.ts#inspectConfigurationMaintenance", + "apps/api/src/services/prospective-configuration.ts#decideConfigurationMaintenance", + "apps/api/src/services/prospective-configuration.ts#settleConfigurationMaintenance", + "apps/ops-agent/src/configuration-maintenance-store.ts#prepareConfigurationMaintenanceAdmission" + ], + "typedKinds": [ + "configurationMaintenance.prepare", + "configurationMaintenance.inspect", + "configurationMaintenance.decide", + "configurationMaintenance.settle" + ] } ], "mounts": [ diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index e729212c2..9442c9f58 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -4642,6 +4642,12 @@ "schema": { "type": "object", "properties": { + "version": { + "type": "number", + "enum": [ + 2 + ] + }, "maintenanceId": { "type": "string", "format": "uuid", @@ -4656,37 +4662,17 @@ "type": "string", "pattern": "^[a-f0-9]{64}$" }, - "decisionId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "expectedSequence": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "outcome": { - "type": "string", - "enum": [ - "unused", - "applied", - "contained" - ] - }, - "ownerReceiptDigest": { + "settlementId": { "type": "string", "pattern": "^[a-f0-9]{64}$" } }, "required": [ + "version", "maintenanceId", "candidateId", "planDigest", - "decisionId", - "expectedSequence", - "outcome", - "ownerReceiptDigest" + "settlementId" ], "additionalProperties": false } diff --git a/services/data-manager/src/jobs/execution/configuration-maintenance.test.ts b/services/data-manager/src/jobs/execution/configuration-maintenance.test.ts index da55a3f2c..6048f8894 100644 --- a/services/data-manager/src/jobs/execution/configuration-maintenance.test.ts +++ b/services/data-manager/src/jobs/execution/configuration-maintenance.test.ts @@ -129,10 +129,33 @@ describe.skipIf( await fence.startup(); const receipt = await fence.prepare(input, ["x-openmapx-control-delegation", assertion]); expect(receipt).toMatchObject({ fenceId: child.executionId, state: "prepared", sequence: 0 }); + expect( + existsSync( + join( + root, + "data/.admin-target-control/executions/data-manager", + `${child.operationKey}.data-terminal.json`, + ), + ), + ).toBe(true); const rows = await sql`SELECT status,finished_at,metadata FROM data_manager.jobs WHERE id=${child.executionId}`; expect(rows[0]?.status).toBe("ok"); expect(rows[0]?.finished_at).toBeInstanceOf(Date); + const preparedSource = control.verifyPublishedDataConfigurationFencePrepared(lease, child); + expect(control.preparedDataConfigurationFenceState(preparedSource)).toMatchObject({ + preparedReceipt: receipt, + preparedTerminalAt: (rows[0]!.finished_at as Date).getTime(), + }); + expect(() => + control.preparedDataConfigurationFenceState({} as typeof preparedSource), + ).toThrow(); + expect(() => + control.verifyPublishedDataConfigurationFencePrepared(lease, { + ...child, + executionId: randomUUID(), + }), + ).toThrow(); const store = createPostgresDataExecutionStore(sql), deadline = Date.now() + 30000, key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); @@ -463,6 +486,40 @@ async function ownedFenceFixture() { }, }; } +it.skipIf(!process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL)( + "refuses settlement capacity authority from a valid ordinary prepare grant without the original published reservation preimage", + async () => { + const f = await ownedFenceFixture(); + try { + const peer = control.verifyControlPeer(f.dm, { + rawHeaders: ["x-openmapx-control-delegation", f.assertion], + transportPrincipal: "shared-data-manager-bearer", + execution: f.child, + kind: "configuration-maintenance.prepare", + }); + expect(() => + control.verifyPublishedConfigurationSettlementReservation(f.dm, peer, f.input), + ).toThrow(); + expect(() => + control.publishedConfigurationSettlementReservationState({} as never, f.dm), + ).toThrow(); + expect(await f.sql`SELECT id FROM data_manager.jobs WHERE id=${f.child.executionId}`).toEqual( + [], + ); + expect( + existsSync( + join( + f.root, + "control-private/data-manager/configuration-maintenance", + `${f.input.maintenanceId}.json`, + ), + ), + ).toBe(false); + } finally { + await f.close(); + } + }, +); it.skipIf(!process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL)( "retains a half-written signed guard before insert and blocks a racing real admission on another connection", async () => { diff --git a/services/data-manager/src/jobs/execution/configuration-maintenance.ts b/services/data-manager/src/jobs/execution/configuration-maintenance.ts index 4b3cca84b..99dcf0e96 100644 --- a/services/data-manager/src/jobs/execution/configuration-maintenance.ts +++ b/services/data-manager/src/jobs/execution/configuration-maintenance.ts @@ -3,6 +3,7 @@ import { createDataConfigurationFenceStore, } from "@openmapx/core/build-artifacts"; import type postgres from "postgres"; +import { z } from "zod"; /** The adapter keeps every guard check and mutation on the exact transaction's * connection. It exposes no HTTP lock or caller-provided ownership boolean. */ export function createDataConfigurationMaintenance( @@ -17,6 +18,29 @@ export function createDataConfigurationMaintenance( })); return result.value; }, + async reserveSettlementSession(sessionNonce) { + z.uuid().parse(sessionNonce); + const connection = await sql.reserve(), + applicationName = `omx-config-settle:${sessionNonce}`; + // This connection is NEVER returned following an uncertain transaction. + // The fixed effect runner alone releases a conclusively finished one. + await connection.unsafe("SELECT set_config('application_name',$1,false)", [applicationName]); + const [row] = await connection.unsafe( + "SELECT pid,backend_start::text AS backend_start,application_name FROM pg_stat_activity WHERE pid=pg_backend_pid()", + ); + if ( + !row || + typeof row.pid !== "number" || + typeof row.backend_start !== "string" || + row.application_name !== applicationName + ) + throw Error("Original reserved settlement SQL identity unavailable"); + return { + identity: { pid: row.pid, backendStart: row.backend_start, sessionNonce }, + unsafe: (query, parameters) => connection.unsafe(query, parameters), + releaseConclusive: () => connection.release(), + }; + }, }); } export type DataConfigurationMaintenance = ReturnType; diff --git a/services/data-manager/src/jobs/execution/receiver.ts b/services/data-manager/src/jobs/execution/receiver.ts index 5ca16ce37..a96917f9b 100644 --- a/services/data-manager/src/jobs/execution/receiver.ts +++ b/services/data-manager/src/jobs/execution/receiver.ts @@ -69,8 +69,13 @@ export function createDataExecutionReceiver(deps: { }); if (!verified || !controlHeaders(rawHeaders).delegation) throw Object.assign(Error("Signed borrowed generation leaf required"), { statusCode: 403 }); - await configurationFence.assertGenerationAdmission(request, rawHeaders); + const preparedGeneration = await configurationFence.assertGenerationAdmission( + request, + rawHeaders, + verified.parent, + ); return runtime.admit(request.input, { + ...(preparedGeneration ? { preparedGeneration } : {}), operationKey: request.operationKey, requestedDeadlineAt: request.requestedDeadlineAt, parent: verified.parent, diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index ea30fcf6a..94f223458 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -18,6 +18,7 @@ import { settleArtifactTargetLeaseChild, type TargetLeaseAuthority, type VerifiedExecutionAuthority, + type VerifiedPreparedGenerationAdmission, verifyControlEffectSettlement, } from "@openmapx/core/build-artifacts"; import { @@ -65,6 +66,7 @@ export interface DataExecutionRuntime { admit( input: DataExecutionInput, options: { + preparedGeneration?: VerifiedPreparedGenerationAdmission; operationKey: string; requestedDeadlineAt?: number; parent?: VerifiedParentExecution; @@ -404,7 +406,8 @@ export function createDataExecutionRuntime(deps: { await owner.assertOwnedProxyRecoverySettled(record.execution); await releaseArtifactTargetLease(lease); await owner.publishContainedTerminal(record.execution); - } + } else if (record.input.kind === "generation-hardlinks") + await owner.publishContainedTerminal(record.execution); } } catch { // An I/O/CAS/containment gap is not a terminal receipt or permission to @@ -507,7 +510,8 @@ export function createDataExecutionRuntime(deps: { if (lease.mode === "owner") { await releaseArtifactTargetLease(lease); await owner.publishContainedTerminal(record.execution); - } + } else if (record.input.kind === "generation-hardlinks") + await owner.publishContainedTerminal(record.execution); } finally { closeArtifactTargetLeaseHandle(lease); } @@ -562,7 +566,8 @@ export function createDataExecutionRuntime(deps: { if (recovered.lease.mode === "owner") { await releaseArtifactTargetLease(recovered.lease); await owner.publishContainedTerminal(record.execution); - } + } else if (record.input.kind === "generation-hardlinks") + await owner.publishContainedTerminal(record.execution); } catch { try { const latest = await store.inspect(record.status.id); diff --git a/services/data-manager/src/jobs/execution/store.ts b/services/data-manager/src/jobs/execution/store.ts index 1cd41f103..debdd5e07 100644 --- a/services/data-manager/src/jobs/execution/store.ts +++ b/services/data-manager/src/jobs/execution/store.ts @@ -4,6 +4,7 @@ import { type DataExecutionControlJournal, retainedControlParent, retainedDataDatabaseTimestamp, + type VerifiedPreparedGenerationAdmission, } from "@openmapx/core/build-artifacts"; import { type DataExecutionIdentity, @@ -73,6 +74,7 @@ export interface DataExecutionRecord { finishedAt: number | null; } export interface DataExecutionAdmissionOptions { + preparedGeneration?: VerifiedPreparedGenerationAdmission; operationKey: string; requestedDeadlineAt?: number; parent?: VerifiedParentExecution; @@ -473,12 +475,32 @@ export function createPostgresDataExecutionStore( ) throw new DataExecutionConflict("A fresh dated data execution root is required"); } - return withDataAdmissionTransaction(sql, async (tx) => { + const admitTransaction = async ( + run: (tx: postgres.TransactionSql) => Promise, + ): Promise => { + if (!admission.preparedGeneration) return withDataAdmissionTransaction(sql, run); + if (input.kind !== "generation-hardlinks" || !options.configurationFence) + throw new DataConfigurationMaintenanceBusy(); + const result = await sql.begin(async (tx) => { + const held = + await tx`SELECT pg_try_advisory_xact_lock(${DATA_ADMISSION_LOCK[0]},${DATA_ADMISSION_LOCK[1]}) AS held`; + if (held[0]?.held !== true) throw new DataConfigurationMaintenanceBusy(); + await options.configurationFence!.assertPreparedGenerationAdmission( + tx, + admission.preparedGeneration!, + input, + admission, + ); + return { value: await run(tx) }; + }); + return result.value; + }; + return admitTransaction(async (tx) => { if (legacyKey) { const original = await findLegacyTransit(tx, legacyKey); if (original) return matchLegacy(original); } - await options.configurationFence?.assertAdmission(tx); + if (!admission.preparedGeneration) await options.configurationFence?.assertAdmission(tx); const retained = await find(tx, "operation_key", admission.operationKey); if (retained) return match(retained); if (options.controlJournal) { diff --git a/services/ops-agent/service.json b/services/ops-agent/service.json index 53a6e0090..fd74295de 100644 --- a/services/ops-agent/service.json +++ b/services/ops-agent/service.json @@ -31,6 +31,8 @@ "DOCKER_CONFIG": "/home/node/.docker", "OPS_AGENT_API_TOKEN_FILE": "/run/secrets/ops-agent-api-token", "OPS_AGENT_DATA_MANAGER_TOKEN_FILE": "/run/secrets/ops-agent-data-manager-token", + "DATA_MANAGER_URL": "${DATA_MANAGER_URL:-http://data-manager:4000}", + "DATA_MANAGER_AUTH_TOKEN": "${DATA_MANAGER_AUTH_TOKEN:?DATA_MANAGER_AUTH_TOKEN must be set}", "OPS_TRUSTED_CONFIG_DIR": "/var/lib/openmapx/trusted-config", "OPS_PRIVACY_BACKUP_CAPABILITY_KEY_FILE": "/run/secrets/privacy-backup-capability-key", "OPS_PRIVACY_BACKUP_COLLECTOR_ENABLED": "${OPS_PRIVACY_BACKUP_COLLECTOR_ENABLED:-false}", From ca4c7d6b641b109c2282901beff68a71ca67f76d Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 09:06:54 +0100 Subject: [PATCH 035/135] fix(admin): retain partial service batch failures --- .../__tests__/admin-job-handlers.test.ts | 89 ++++++++++ .../__tests__/job-runner-outcomes.test.ts | 15 +- .../__tests__/service-batch-result.test.ts | 42 +++++ apps/api/src/services/admin-job-handlers.ts | 89 ++++++++-- apps/api/src/services/service-batch-result.ts | 22 +++ .../src/administrative-runtime.test.ts | 162 ++++++++++++++++-- apps/ops-agent/src/administrative-runtime.ts | 51 +++++- apps/ops-agent/src/runtime.ts | 3 + apps/ops-agent/src/server.ts | 23 +++ .../cli/__tests__/cli-service-builds.test.ts | 40 +++++ packages/cli/src/lib/service-builds.ts | 89 ++++++++-- packages/core/src/ops/service-results.test.ts | 48 ++++++ packages/core/src/ops/service-results.ts | 44 ++++- 13 files changed, 663 insertions(+), 54 deletions(-) create mode 100644 apps/api/src/services/__tests__/service-batch-result.test.ts create mode 100644 apps/api/src/services/service-batch-result.ts create mode 100644 packages/core/src/ops/service-results.test.ts diff --git a/apps/api/src/services/__tests__/admin-job-handlers.test.ts b/apps/api/src/services/__tests__/admin-job-handlers.test.ts index af81848b2..f6e792de4 100644 --- a/apps/api/src/services/__tests__/admin-job-handlers.test.ts +++ b/apps/api/src/services/__tests__/admin-job-handlers.test.ts @@ -422,6 +422,91 @@ describe("admin job handlers", () => { expect(runAdminOperationMock).toHaveBeenCalledTimes(1); }); + it.each(["continue", "fail-fast", "all-failed", "unknown", "cancel"] as const)( + "checkpoints exact batch results before failure: %s", + async (mode) => { + const { ApiOpsError, OpsObservationPending } = await import("../ops-client"); + const { AdminJobFailure } = await import("../admin-job-failure"); + const controller = new AbortController(); + const pending = new OpsObservationPending( + { kind: "service.build", operationKey: "opk1_abcdefghijklmnop" }, + "transport", + ); + runAdminOperationMock.mockClear(); + runAdminOperationMock.mockImplementationOnce(async () => { + if (mode === "all-failed") throw new ApiOpsError("runtime"); + return { completed: true }; + }); + runAdminOperationMock.mockImplementationOnce(async () => { + if (mode === "unknown") throw pending; + if (mode === "cancel") { + controller.abort(); + throw new DOMException("Aborted", "AbortError"); + } + throw new ApiOpsError("runtime"); + }); + if (mode === "continue" || mode === "all-failed") + runAdminOperationMock.mockImplementationOnce(async () => { + if (mode === "all-failed") throw new ApiOpsError("runtime"); + return { completed: true }; + }); + const checkpoints: Record[] = []; + const ctx = { + jobId: "batch", + payload: { + action: "build", + serviceIds: ["valhalla", "osrm", "app-api"], + ...(mode === "fail-fast" ? {} : { continueOnError: true }), + }, + signal: controller.signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpointResult: { retained: true } as Record, + checkpoint: async (value: Record) => { + checkpoints.push(structuredClone(value)); + ctx.checkpointResult = value; + }, + }; + const failure = await handleServiceBulkJob(ctx).catch((error) => error); + if (mode === "unknown") expect(failure).toBe(pending); + else if (mode === "cancel") expect(failure.name).toBe("AbortError"); + else expect(failure).toBeInstanceOf(AdminJobFailure); + const batch = { + outcome: mode === "all-failed" ? "failed" : "partial", + completedServiceIds: + mode === "all-failed" + ? [] + : mode === "continue" + ? ["valhalla", "app-api"] + : ["valhalla"], + failedServiceIds: + mode === "all-failed" + ? ["valhalla", "osrm", "app-api"] + : mode === "unknown" || mode === "cancel" + ? [] + : ["osrm"], + skippedServiceIds: + mode === "unknown" || mode === "cancel" + ? ["app-api"] + : mode === "fail-fast" + ? ["app-api"] + : [], + }; + expect(checkpoints[0]).toMatchObject({ + retained: true, + serviceBatch: { completedServiceIds: mode === "all-failed" ? [] : ["valhalla"] }, + }); + expect(checkpoints.at(-1)).toMatchObject({ retained: true, serviceBatch: batch }); + if (mode !== "unknown" && mode !== "cancel") + expect(failure.result.serviceBatch).toEqual(batch); + expect(runAdminOperationMock.mock.calls.map((call) => call[1].serviceId)).toEqual( + mode === "continue" || mode === "all-failed" + ? ["valhalla", "osrm", "app-api"] + : ["valhalla", "osrm"], + ); + }, + ); + it("submits one typed lifecycle operation per exact service ID", async () => { runAdminOperationMock.mockClear(); const ctx = { @@ -450,6 +535,10 @@ describe("admin job handlers", () => { it("uses the bounded build-all variant and typed region options", async () => { runAdminOperationMock.mockClear(); + runAdminOperationMock.mockResolvedValueOnce({ + completedServiceIds: ["osrm"], + failedServiceIds: [], + }); const ctx = { jobId: "job", payload: { diff --git a/apps/api/src/services/__tests__/job-runner-outcomes.test.ts b/apps/api/src/services/__tests__/job-runner-outcomes.test.ts index 6b8409040..b2f74b86e 100644 --- a/apps/api/src/services/__tests__/job-runner-outcomes.test.ts +++ b/apps/api/src/services/__tests__/job-runner-outcomes.test.ts @@ -71,8 +71,14 @@ it("merges AdminJobFailure evidence into DB/SSE without publishing ownership or unsubscribe(); expect(store.writes.at(-1)?.values).toMatchObject({ status: "failed", - result: { phase: "build", _execution: execution, ...result }, }); + const patch = new PgDialect().sqlToQuery(store.writes.at(-1)?.values.result as SQL); + expect(JSON.parse(patch.params[0] as string)).toEqual({ + phase: "build", + _execution: execution, + ...result, + }); + expect(patch.sql).toContain("- 'observation'"); expect(events.at(-1)).toMatchObject({ type: "status", status: "failed", @@ -117,10 +123,13 @@ it("does not announce an old owner's completion when PostgreSQL rejects its CAS" await run("test.stale"); stop(); expect(events.some((event) => event.status === "success")).toBe(false); - const late = new PgDialect().sqlToQuery(store.writes.at(-1)?.values.result as SQL); - expect(JSON.parse(late.params[0] as string)).toEqual({ + const patches = store.writes.map((write) => + JSON.parse(new PgDialect().sqlToQuery(write.values.result as SQL).params[0] as string), + ); + expect(patches.find((patch) => patch._settled)).toEqual({ _settled: { status: "success", result: { completed: true } }, }); + expect(patches.at(-1)).toMatchObject({ observation: { state: "receipt_delivery" } }); for (const write of store.writes) { const query = new PgDialect().sqlToQuery(write.where as SQL); expect(query.sql).toContain("->'_execution'"); diff --git a/apps/api/src/services/__tests__/service-batch-result.test.ts b/apps/api/src/services/__tests__/service-batch-result.test.ts new file mode 100644 index 000000000..b54e44b5f --- /dev/null +++ b/apps/api/src/services/__tests__/service-batch-result.test.ts @@ -0,0 +1,42 @@ +import { createServiceBatchResult } from "@openmapx/core/ops"; +import { describe, expect, it, vi } from "vitest"; +import { AdminJobFailure } from "../admin-job-failure"; +import type { JobContext } from "../job-runner"; +import { finishServiceBatch } from "../service-batch-result"; + +describe("finishServiceBatch", () => { + function context() { + const ctx = { + checkpointResult: { retained: true } as Record, + checkpoint: vi.fn(async (result: Record) => { + ctx.checkpointResult = result; + }), + }; + return ctx as unknown as JobContext; + } + it("retains the checkpoint and returns only a successful nonempty batch", async () => { + const ctx = context(); + const batch = createServiceBatchResult(["alpha"], ["alpha"], [], []); + await expect(finishServiceBatch(ctx, batch)).resolves.toEqual(batch); + expect(ctx.checkpointResult).toMatchObject({ retained: true, serviceBatch: batch }); + }); + it.each(["partial", "failed", "empty"] as const)( + "checkpoints %s before throwing a typed job failure", + async (mode) => { + const ctx = context(); + const batch = + mode === "empty" + ? createServiceBatchResult([], [], [], []) + : createServiceBatchResult( + ["alpha", "beta"], + mode === "partial" ? ["alpha"] : [], + ["beta"], + mode === "failed" ? ["alpha"] : [], + ); + const error = await finishServiceBatch(ctx, batch).catch((error) => error); + expect(error).toBeInstanceOf(AdminJobFailure); + expect(error.result).toEqual(ctx.checkpointResult); + expect(error.result.serviceBatch).toEqual(batch); + }, + ); +}); diff --git a/apps/api/src/services/admin-job-handlers.ts b/apps/api/src/services/admin-job-handlers.ts index 0b8fe0d0f..9fc8c21be 100644 --- a/apps/api/src/services/admin-job-handlers.ts +++ b/apps/api/src/services/admin-job-handlers.ts @@ -1,3 +1,8 @@ +import { + createServiceBatchResult, + type OpsResultFor, + serviceBatchResultSchema, +} from "@openmapx/core/ops"; import { assertValidBackupName } from "./admin-cli"; import { JobObservationPending } from "./admin-job-failure"; import { executeAdminJobOperation } from "./admin-job-ops"; @@ -5,7 +10,8 @@ import { JobOwnershipLost } from "./admin-job-state"; import { getAdminOperation, parseAdminOperationInput } from "./admin-operation-catalog"; import { assertKnownServiceIds, assertRegion } from "./admin-operation-input"; import type { JobContext } from "./job-runner"; -import { OpsObservationPending } from "./ops-client"; +import { ApiOpsError, OpsObservationPending } from "./ops-client"; +import { checkpointServiceBatch, finishServiceBatch } from "./service-batch-result"; import { applyCoordinatorService, prepareServiceGeneration, @@ -138,25 +144,51 @@ export async function handleServiceBulkJob(ctx: JobContext): Promise; + try { + result = await executeAdminJobOperation( + ctx, + { + kind: "services.buildAll", + ...(region ? { regionId: region } : {}), + ...(payload.continueOnError === false ? { failFast: true } : {}), + }, + "admin-job.services.build-all", + ); + } catch (error) { + const batch = serviceBatchResultSchema.safeParse( + error instanceof ApiOpsError ? error.failureResult?.serviceBatch : undefined, + ); + if (batch.success) { + if (ctx.signal.aborted) await checkpointServiceBatch(ctx, batch.data); + else await finishServiceBatch(ctx, batch.data); + } + throw error; + } + const serviceBatch = await finishServiceBatch( ctx, - { - kind: "services.buildAll", - ...(region ? { regionId: region } : {}), - ...(payload.continueOnError === false ? { failFast: true } : {}), - }, - "admin-job.services.build-all", + createServiceBatchResult( + [...result.completedServiceIds, ...result.failedServiceIds], + result.completedServiceIds, + result.failedServiceIds, + [], + ), ); return { action, completedServiceIds: result.completedServiceIds, failedServiceIds: result.failedServiceIds, + serviceBatch, }; } else { + createServiceBatchResult(serviceIds, [], [], serviceIds); const completedServiceIds: string[] = []; const failedServiceIds: string[] = []; for (const [index, serviceId] of serviceIds.entries()) { + let attempted = false; try { + ctx.signal.throwIfAborted(); + attempted = true; await executeAdminJobOperation( ctx, { @@ -175,16 +207,51 @@ export async function handleServiceBulkJob(ctx: JobContext): Promise { + const serviceBatch = serviceBatchResultSchema.parse(result); + await ctx.checkpoint({ ...ctx.checkpointResult, serviceBatch }); + return serviceBatch; +} + +export async function finishServiceBatch( + ctx: JobContext, + result: ServiceBatchResult, +): Promise { + const serviceBatch = await checkpointServiceBatch(ctx, result); + if (serviceBatch.outcome !== "success") + throw new AdminJobFailure("Service batch failed", { ...ctx.checkpointResult, serviceBatch }); + return serviceBatch; +} diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index bfb7ab084..1d1c2863f 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -9,7 +9,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { OpsOperation } from "@openmapx/core/ops"; +import type { OpsOperation, ServiceBatchResult } from "@openmapx/core/ops"; import { afterEach, describe, expect, it, vi } from "vitest"; import { testExecution } from "./__fixtures__/execution"; import { @@ -23,7 +23,7 @@ import { loadConfiguredResourceAuthority, pruneBackupRetention, } from "./administrative-runtime"; -import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; +import { createUnavailableRuntime, dispatchOpsOperation, OpsExecutionFailure } from "./runtime"; const temporaryRoots: string[] = []; const privacyReleaseValidation = { @@ -49,6 +49,7 @@ function context(kind: string) { execution: testExecution(), signal: new AbortController().signal, emitLog: vi.fn(), + checkpointServiceBatch: vi.fn(async (_result: ServiceBatchResult) => {}), claim: { fingerprint: "f".repeat(64), operation: { kind } as never, @@ -396,7 +397,7 @@ describe("administrative data runtime", () => { ).toThrow("Resource authority configuration rejected"); }); - it("maps every typed variant to fixed agent-owned CLI arguments", async () => { + it("requires owned data workflows and maps only the remaining legacy CLI leaves", async () => { const rootDir = temporaryRoot(); const calls: string[][] = []; const runtime = createUnavailableRuntime(); @@ -423,20 +424,26 @@ describe("administrative data runtime", () => { { kind: "data.overtureConflate", regionId: "europe/germany", restart: true }, { kind: "data.searchIndexBuild", regionId: "europe/germany" }, ]; + const owned = new Set([ + "data.downloadOsm", + "data.downloadFonts", + "data.update", + "data.convertOverpass", + "data.overtureSync", + "data.overtureConflate", + "data.searchIndexBuild", + ]); for (const operation of operations) { - await dispatchOpsOperation(runtime, operation, context(operation.kind)); + if (owned.has(operation.kind)) + await expect( + dispatchOpsOperation(runtime, operation, context(operation.kind)), + ).rejects.toThrow("Host update required"); + else await dispatchOpsOperation(runtime, operation, context(operation.kind)); } expect(calls).toEqual([ - ["data", "download", "osm", "europe/germany"], - ["data", "download", "fonts"], - ["data", "update", "europe/germany", "--countries", "DE,AT", "--fail-fast"], - ["data", "convert", "overpass", "europe/germany"], ["data", "link"], ["data", "clean", "osm"], ["data", "generate-api-keys"], - ["data", "overture-sync", "europe/germany"], - ["data", "overture-conflate", "europe/germany", "--restart"], - ["data", "search-index", "build", "europe/germany"], ]); expect(calls.flat()).not.toContain("transitous-fixed-v1"); }); @@ -524,9 +531,16 @@ describe("administrative build runtime", () => { { kind: "services.buildAll", regionId: "europe/germany" }, context("services.buildAll"), ), - ).resolves.toEqual({ - completedServiceIds: ["nominatim"], - failedServiceIds: ["valhalla"], + ).rejects.toMatchObject({ + name: "OpsExecutionFailure", + failureResult: { + serviceBatch: { + outcome: "partial", + completedServiceIds: ["nominatim"], + failedServiceIds: ["valhalla"], + skippedServiceIds: [], + }, + }, }); expect(calls).toEqual([ ["services", "build", "nominatim", "--region", "europe/germany"], @@ -551,7 +565,17 @@ describe("administrative build runtime", () => { { kind: "services.buildAll", failFast: true }, context("services.buildAll"), ), - ).rejects.toThrow("injected build failure"); + ).rejects.toMatchObject({ + name: "OpsExecutionFailure", + failureResult: { + serviceBatch: { + outcome: "failed", + completedServiceIds: [], + failedServiceIds: ["nominatim"], + skippedServiceIds: ["valhalla"], + }, + }, + }); expect(calls).toHaveLength(1); for (const serviceId of ["redis", "disabled", "community", "unknown"]) { await expect( @@ -570,7 +594,113 @@ describe("administrative build runtime", () => { }); await expect( dispatchOpsOperation(empty, { kind: "services.buildAll" }, context("services.buildAll")), - ).resolves.toEqual({ completedServiceIds: [], failedServiceIds: [] }); + ).rejects.toMatchObject({ + failureResult: { + serviceBatch: { + outcome: "failed", + completedServiceIds: [], + failedServiceIds: [], + skippedServiceIds: [], + }, + }, + }); + }); + it.each(["continue", "fail-fast", "all-failed", "unknown", "cancel"] as const)( + "retains ordered batch evidence and stops uncertain work: %s", + async (mode) => { + const calls: string[] = []; + const controller = new AbortController(); + const runtime = createUnavailableRuntime(); + createAdministrativeRuntime(runtime, { + rootDir: temporaryRoot(), + loadBuildAuthority: async () => + ["gamma", "beta", "alpha"].map((serviceId) => ({ + serviceId, + enabled: true, + isBuiltIn: true, + buildCommand: serviceId, + })), + runFixedCli: async (args) => { + const id = args[2]!; + calls.push(id); + if (mode === "all-failed") throw Error("build failed"); + if (id === "beta") { + if (mode === "unknown") throw new OpsExecutionFailure("recovery_required", {}); + if (mode === "cancel") { + controller.abort(); + throw new DOMException("Aborted", "AbortError"); + } + throw Error("build failed"); + } + }, + }); + const ctx = { ...context("services.buildAll"), signal: controller.signal }; + const failure = await dispatchOpsOperation( + runtime, + { kind: "services.buildAll", ...(mode === "fail-fast" ? { failFast: true } : {}) }, + ctx, + ).catch((error) => error); + expect(failure).toBeInstanceOf(OpsExecutionFailure); + expect(failure.failureResult.serviceBatch).toEqual({ + outcome: mode === "all-failed" ? "failed" : "partial", + completedServiceIds: + mode === "all-failed" ? [] : mode === "continue" ? ["alpha", "gamma"] : ["alpha"], + failedServiceIds: + mode === "all-failed" + ? ["alpha", "beta", "gamma"] + : mode === "unknown" || mode === "cancel" + ? [] + : ["beta"], + skippedServiceIds: + mode === "unknown" || mode === "cancel" + ? ["gamma"] + : mode === "fail-fast" + ? ["gamma"] + : [], + }); + expect(calls).toEqual( + mode === "continue" || mode === "all-failed" + ? ["alpha", "beta", "gamma"] + : ["alpha", "beta"], + ); + expect(ctx.checkpointServiceBatch).toHaveBeenCalledTimes(calls.length); + expect(ctx.checkpointServiceBatch.mock.calls.at(-1)?.[0]).toEqual( + failure.failureResult.serviceBatch, + ); + }, + ); + + it("does not start another build when its completed prefix cannot be journaled", async () => { + const runtime = createUnavailableRuntime(); + const calls: string[] = []; + createAdministrativeRuntime(runtime, { + rootDir: temporaryRoot(), + loadBuildAuthority: async () => + ["alpha", "beta"].map((serviceId) => ({ + serviceId, + enabled: true, + isBuiltIn: true, + buildCommand: serviceId, + })), + runFixedCli: async (args) => { + calls.push(args[2]!); + }, + }); + const ctx = context("services.buildAll"); + ctx.checkpointServiceBatch.mockRejectedValueOnce(Error("journal unavailable")); + await expect( + dispatchOpsOperation(runtime, { kind: "services.buildAll" }, ctx), + ).rejects.toMatchObject({ + errorClass: "recovery_required", + failureResult: { + serviceBatch: { + completedServiceIds: ["alpha"], + failedServiceIds: [], + skippedServiceIds: ["beta"], + }, + }, + }); + expect(calls).toEqual(["alpha"]); }); }); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index b59ef05a5..9282653d2 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -39,6 +39,8 @@ import { verifyControlEffectSettlement, } from "@openmapx/core/build-artifacts"; import { + compareServiceBuildOrder, + createServiceBatchResult, type ExecutionRef, executionRefSchema, OPS_MAX_BACKUP_ID_LENGTH, @@ -1757,7 +1759,7 @@ export function createAdministrativeRuntime( typeof service.buildCommand === "string" && service.buildCommand.length > 0, ) - .sort((left, right) => left.serviceId.localeCompare(right.serviceId)); + .sort((left, right) => compareServiceBuildOrder(left.serviceId, right.serviceId)); }; runtime["service.build"] = async (operation, context) => { const buildable = await loadBuildable(); @@ -1768,18 +1770,57 @@ export function createAdministrativeRuntime( return { completed: true }; }; runtime["services.buildAll"] = async (operation, context) => { + const plan = await loadBuildable(); + const serviceIds = plan.map((service) => service.serviceId); const completedServiceIds: string[] = []; const failedServiceIds: string[] = []; - for (const service of await loadBuildable()) { - if (context.signal.aborted) throw new DOMException("Aborted", "AbortError"); + const batch = (skipped: string[]) => + createServiceBatchResult(serviceIds, completedServiceIds, failedServiceIds, skipped); + if (plan.length === 0) throw new OpsExecutionFailure("runtime", { serviceBatch: batch([]) }); + for (const [index, service] of plan.entries()) { + let attempted = false; + let stop: OpsExecutionFailure | undefined; try { + context.signal.throwIfAborted(); + attempted = true; await runFixedCli(buildArgs(service.serviceId, operation.regionId), context); completedServiceIds.push(service.serviceId); } catch (error) { - failedServiceIds.push(service.serviceId); - if (operation.failFast) throw error; + if ( + context.signal.aborted || + (error instanceof Error && error.name === "AbortError") || + (error instanceof OpsExecutionFailure && error.errorClass !== "runtime") + ) { + stop = new OpsExecutionFailure( + error instanceof OpsExecutionFailure + ? error.errorClass + : Date.now() >= context.execution.deadlineAt + ? "timeout" + : "recovery_required", + { + ...(error instanceof OpsExecutionFailure ? error.failureResult : {}), + serviceBatch: batch(serviceIds.slice(index + (attempted ? 1 : 0))), + }, + ); + } else { + failedServiceIds.push(service.serviceId); + if (operation.failFast) + stop = new OpsExecutionFailure("runtime", { + ...(error instanceof OpsExecutionFailure ? error.failureResult : {}), + serviceBatch: batch(serviceIds.slice(index + 1)), + }); + } + } + const serviceBatch = stop?.failureResult.serviceBatch ?? batch(serviceIds.slice(index + 1)); + try { + await context.checkpointServiceBatch?.(serviceBatch); + } catch { + throw new OpsExecutionFailure("recovery_required", { serviceBatch }); } + if (stop) throw stop; } + if (failedServiceIds.length > 0) + throw new OpsExecutionFailure("runtime", { serviceBatch: batch([]) }); return { completedServiceIds, failedServiceIds }; }; runtime["backup.list"] = async () => inspectBackupInventory(options.rootDir); diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index d41ebf175..29f7c762d 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -14,6 +14,7 @@ import { type OpsOperationKind, type OpsResultFor, parseOpsResult, + type ServiceBatchResult, } from "@openmapx/core/ops"; export class OpsExecutionFailure extends Error { @@ -67,6 +68,8 @@ export interface OpsExecutionContext { claim: OpsTrustedClaim; /** Unfinished operations other than this one; replacing the agent would end them. */ otherActiveOperations?(): number; + /** Persist progress before a batch can start its next effect. */ + checkpointServiceBatch?(result: ServiceBatchResult): Promise; } export interface OpsTrustedClaim { diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 8b42590c5..b3d87eab8 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -823,6 +823,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } } const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; + if (state === "succeeded" && job.operation.kind === "services.buildAll") + delete candidate.failureResult; if (authority && ["succeeded", "failed", "timed_out"].includes(state)) { if (candidate.terminalAt === undefined) throw Error("Missing terminal time"); candidate.terminalAt = retainControlTerminalCommitment( @@ -846,6 +848,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ) retainControlPreparedTerminal(authority); Object.assign(job, updates); + if (state === "succeeded" && job.operation.kind === "services.buildAll") + delete job.failureResult; job.updatedAt = updatedAt; job.state = state; if (lease && ["succeeded", "failed", "timed_out"].includes(state)) { @@ -954,6 +958,25 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif other.state === "running" || other.state === "termination_pending"), ).length, + ...(job.operation.kind === "services.buildAll" + ? { + checkpointServiceBatch: async ( + serviceBatch: import("@openmapx/core/ops").ServiceBatchResult, + ) => { + const evidence = opsFailureResultSchema.parse({ ...job.failureResult, serviceBatch }); + if ( + Buffer.byteLength(JSON.stringify(evidence)) > + OPS_KIND_POLICIES[job.operation.kind].maxResultBytes + ) + throw Error("Service batch evidence exceeds budget"); + await withAdmissionLock(async () => { + job.failureResult = evidence; + job.updatedAt = now().toISOString(); + await journal.replace([...jobsById.values()].map(persisted)); + }); + }, + } + : {}), emitLog: (stream, message) => { const boundedMessage = truncateUtf8( safeProcessDiagnostic(message), diff --git a/packages/cli/__tests__/cli-service-builds.test.ts b/packages/cli/__tests__/cli-service-builds.test.ts index 00e05487a..c8d9a263e 100644 --- a/packages/cli/__tests__/cli-service-builds.test.ts +++ b/packages/cli/__tests__/cli-service-builds.test.ts @@ -145,6 +145,46 @@ describe("buildServices", () => { expect(result.failures).toEqual([{ id: "beta", message: "beta broke" }]); }); + it.each(["all", "explicit", "empty", "cancel"] as const)( + "retains batch outcomes with the existing CLI contract: %s", + async (mode) => { + const calls: string[] = []; + const handlers: Record = Object.fromEntries( + ["alpha", "beta", "gamma"].map((id) => [ + id, + async () => { + calls.push(id); + if (id === "beta") { + if (mode === "cancel") throw new DOMException("Aborted", "AbortError"); + throw Error("beta broke"); + } + return { summary: `Built ${id}` }; + }, + ]), + ); + if (mode !== "empty") + for (const id of Object.keys(handlers)) + writeManifest(id, { ...baseManifest, id, buildCommand: `openmapx services build ${id}` }); + const result = await buildServices({ + rootDir: tmp, + mode: mode === "explicit" ? "explicit" : "all", + serviceIds: ["alpha", "beta", "gamma"], + handlers, + }).catch((error) => error); + const batch = mode === "all" ? result.serviceBatch : result.result?.serviceBatch; + expect(batch).toEqual({ + outcome: mode === "empty" ? "failed" : "partial", + completedServiceIds: + mode === "empty" ? [] : mode === "all" ? ["alpha", "gamma"] : ["alpha"], + failedServiceIds: mode === "empty" || mode === "cancel" ? [] : ["beta"], + skippedServiceIds: mode === "cancel" ? ["gamma"] : mode === "explicit" ? ["gamma"] : [], + }); + expect(calls).toEqual( + mode === "empty" ? [] : mode === "all" ? ["alpha", "beta", "gamma"] : ["alpha", "beta"], + ); + }, + ); + it("uses service-specific region env defaults when no --region is passed", async () => { process.env.OSRM_REGION = "europe/germany"; process.env.OTP_REGION = "europe/france"; diff --git a/packages/cli/src/lib/service-builds.ts b/packages/cli/src/lib/service-builds.ts index e7b11a238..0a7ec5ec0 100644 --- a/packages/cli/src/lib/service-builds.ts +++ b/packages/cli/src/lib/service-builds.ts @@ -1,5 +1,11 @@ import { statSync } from "node:fs"; import { basename } from "node:path"; +import { + compareServiceBuildOrder, + createServiceBatchResult, + SERVICE_BUILD_ORDER, + type ServiceBatchResult, +} from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; import { BUILD_ARTIFACTS, @@ -40,7 +46,7 @@ type LoadedService = coreServices.LoadedService; const SERVICE_BUILD_COMMAND_RE = /^openmapx\s+services\s+build\s+([a-z0-9-]+)$/; -export const SERVICE_BUILD_ORDER = ["osrm", "otp", "motis", "pelias", "tileserver"] as const; +export { SERVICE_BUILD_ORDER }; export interface ServiceBuildContext { rootDir: string; @@ -84,6 +90,18 @@ export interface ServiceBuildExecutionResult { plannedIds: string[]; completedIds: string[]; failures: ServiceBuildFailure[]; + serviceBatch: ServiceBatchResult; +} + +export class ServiceBuildExecutionError extends Error { + constructor( + message: string, + readonly result: ServiceBuildExecutionResult, + cause?: unknown, + ) { + super(message, { cause }); + this.name = "ServiceBuildExecutionError"; + } } export interface PlanServiceBuildsOptions { @@ -265,12 +283,7 @@ function dedupe(ids: string[]): string[] { } function compareBuildOrder(a: string, b: string): number { - const aIndex = SERVICE_BUILD_ORDER.indexOf(a as (typeof SERVICE_BUILD_ORDER)[number]); - const bIndex = SERVICE_BUILD_ORDER.indexOf(b as (typeof SERVICE_BUILD_ORDER)[number]); - const aRank = aIndex === -1 ? Number.POSITIVE_INFINITY : aIndex; - const bRank = bIndex === -1 ? Number.POSITIVE_INFINITY : bIndex; - if (aRank !== bRank) return aRank - bRank; - return a.localeCompare(b); + return compareServiceBuildOrder(a, b); } function validateBuildCommand(service: LoadedService): string { @@ -348,11 +361,23 @@ export async function buildServices( const plannedIds = plan.map((item) => item.id); const completedIds: string[] = []; const failures: ServiceBuildFailure[] = []; + const result = (skipped: string[]): ServiceBuildExecutionResult => ({ + plannedIds: [...plannedIds], + completedIds: [...completedIds], + failures: [...failures], + serviceBatch: createServiceBatchResult( + plannedIds, + completedIds, + failures.map(({ id }) => id), + skipped, + ), + }); if (plan.length === 0) { log.warn("No buildable services matched the request."); - return { plannedIds, completedIds, failures }; + throw new ServiceBuildExecutionError("No buildable services matched the request", result([])); } + result(plannedIds); // Validate the whole admitted plan before any handler runs. log.info(`Build plan: ${plannedIds.join(", ")}`); @@ -365,43 +390,71 @@ export async function buildServices( ); } - for (const item of plan) { - assertCliExecutionAvailable(); + for (const [index, item] of plan.entries()) { + let attempted = false; log.dim(`Manifest command → ${item.buildCommand}`); try { + assertCliExecutionAvailable(); const resolvedRegion = resolveBuildRegion(item.id, opts.region); if (resolvedRegion.sourceEnv) { log.dim( `${item.id}: using region "${resolvedRegion.value}" from $${resolvedRegion.sourceEnv}`, ); } - const result = await item.handler({ + attempted = true; + const built = await item.handler({ rootDir, region: resolvedRegion.value, registry, service: item.service, motisImport: opts.motisImport, }); - for (const warning of result.warnings ?? []) { + for (const warning of built.warnings ?? []) { log.warn(`${item.id}: ${warning}`); } - if (result.artifact) { - await recordBuild(rootDir, item.id, resolvedRegion.value, result.artifact); + if (built.artifact) { + await recordBuild(rootDir, item.id, resolvedRegion.value, built.artifact); } - log.ok(result.summary); + log.ok(built.summary); completedIds.push(item.id); } catch (error) { + let interrupted = + error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError"); + if ( + error && + typeof error === "object" && + (("timedOut" in error && error.timedOut === true) || + ("isCanceled" in error && error.isCanceled === true) || + ("signal" in error && typeof error.signal === "string") || + ("errorClass" in error && error.errorClass !== "runtime")) + ) + interrupted = true; + try { + assertCliExecutionAvailable(); + } catch { + interrupted = true; + } + if (interrupted) + throw new ServiceBuildExecutionError( + "Build execution interrupted", + result(plannedIds.slice(index + (attempted ? 1 : 0))), + error, + ); const failure = { id: item.id, message: (error as Error).message, }; failures.push(failure); - if (!opts.continueOnError) { - throw new Error(`${item.id} build failed: ${failure.message}`); + if (!(opts.continueOnError ?? opts.mode === "all")) { + throw new ServiceBuildExecutionError( + `${item.id} build failed: ${failure.message}`, + result(plannedIds.slice(index + 1)), + error, + ); } log.err(`${item.id} build failed: ${failure.message}`); } } - return { plannedIds, completedIds, failures }; + return result([]); } diff --git a/packages/core/src/ops/service-results.test.ts b/packages/core/src/ops/service-results.test.ts new file mode 100644 index 000000000..efb45dd5d --- /dev/null +++ b/packages/core/src/ops/service-results.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { OPS_KIND_POLICIES } from "./contract"; +import { opsFailureResultSchema } from "./job-evidence"; +import { createServiceBatchResult, serviceBatchResultSchema } from "./service-results"; + +describe("service batch evidence", () => { + it("keeps unknown attempted targets separate from skipped targets", () => { + expect(createServiceBatchResult(["alpha", "beta"], ["alpha"], [], [])).toEqual({ + outcome: "partial", + completedServiceIds: ["alpha"], + failedServiceIds: [], + skippedServiceIds: [], + }); + }); + it("rejects unadmitted, duplicate, overlapping and falsely successful results", () => { + expect(() => createServiceBatchResult(["alpha"], ["beta"], [], [])).toThrow(); + expect(() => createServiceBatchResult(["alpha"], ["alpha"], ["alpha"], [])).toThrow(); + expect(() => createServiceBatchResult(["alpha", "alpha"], ["alpha"], [], [])).toThrow(); + expect( + serviceBatchResultSchema.safeParse({ + outcome: "success", + completedServiceIds: [], + failedServiceIds: [], + skippedServiceIds: [], + }).success, + ).toBe(false); + expect( + serviceBatchResultSchema.safeParse({ + outcome: "success", + completedServiceIds: ["alpha"], + failedServiceIds: ["beta"], + skippedServiceIds: [], + }).success, + ).toBe(false); + }); + it("fits all 256 maximum-length service IDs in the build-all failure budget", () => { + const ids = Array.from( + { length: 256 }, + (_, i) => `service-${String(i).padStart(3, "0")}-${"x".repeat(52)}`, + ); + const serviceBatch = createServiceBatchResult(ids, ids.slice(0, 128), ids.slice(128), []); + const failure = opsFailureResultSchema.parse({ serviceBatch }); + expect(JSON.stringify(failure).length).toBeGreaterThan(16 * 1024); + expect(Buffer.byteLength(JSON.stringify(failure))).toBeLessThanOrEqual( + OPS_KIND_POLICIES["services.buildAll"].maxResultBytes, + ); + }); +}); diff --git a/packages/core/src/ops/service-results.ts b/packages/core/src/ops/service-results.ts index d2df2c3a2..f9c4fbd57 100644 --- a/packages/core/src/ops/service-results.ts +++ b/packages/core/src/ops/service-results.ts @@ -30,9 +30,51 @@ export const serviceBatchResultSchema = z }) .refine((x) => { const all = [...x.completedServiceIds, ...x.failedServiceIds, ...x.skippedServiceIds]; - return all.length <= 256 && new Set(all).size === all.length; + return ( + all.length <= 256 && + new Set(all).size === all.length && + (x.outcome === "success" + ? x.completedServiceIds.length > 0 && + x.failedServiceIds.length === 0 && + x.skippedServiceIds.length === 0 + : x.outcome === "partial" + ? x.completedServiceIds.length > 0 + : x.completedServiceIds.length === 0) + ); }); export type ServiceBatchResult = z.infer; + +export const SERVICE_BUILD_ORDER = ["osrm", "otp", "motis", "pelias", "tileserver"] as const; +export function compareServiceBuildOrder(a: string, b: string): number { + const rank = (id: string) => { + const index = SERVICE_BUILD_ORDER.indexOf(id as (typeof SERVICE_BUILD_ORDER)[number]); + return index < 0 ? Infinity : index; + }; + return rank(a) === rank(b) ? a.localeCompare(b) : rank(a) - rank(b); +} + +/** An attempted unknown item is omitted, never mislabeled failed or skipped. */ +export function createServiceBatchResult( + serviceIds: readonly string[], + completedServiceIds: readonly string[], + failedServiceIds: readonly string[], + skippedServiceIds: readonly string[], +): ServiceBatchResult { + const admitted = new Set(ids.parse(serviceIds)); + const all = [...completedServiceIds, ...failedServiceIds, ...skippedServiceIds]; + if (all.some((id) => !admitted.has(id))) throw Error("Unadmitted service batch target"); + return serviceBatchResultSchema.parse({ + outcome: + completedServiceIds.length === 0 + ? "failed" + : completedServiceIds.length === admitted.size + ? "success" + : "partial", + completedServiceIds, + failedServiceIds, + skippedServiceIds, + }); +} const revision = z .string() .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) From 3a2739cc4ec1d2857c79e65eaac5b6e076b2153a Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 09:37:13 +0100 Subject: [PATCH 036/135] feat(admin): add durable core service restart controls --- .../routes/__tests__/admin-services.test.ts | 32 +++ apps/api/src/routes/admin-services.ts | 22 +- apps/api/src/server.ts | 4 + .../core-service-maintenance.test.ts | 78 +++++++ .../__tests__/job-runner-initialize.test.ts | 4 + apps/api/src/services/admin-job-ops.ts | 1 + .../src/services/core-service-maintenance.ts | 44 ++++ apps/api/src/services/job-runner.ts | 1 + .../src/core-service-runtime.test.ts | 215 ++++++++++++++++++ apps/ops-agent/src/core-service-runtime.ts | 99 ++++++++ apps/ops-agent/src/policy.ts | 12 +- apps/ops-agent/src/production-runtime.test.ts | 4 +- apps/ops-agent/src/production-runtime.ts | 8 + apps/ops-agent/src/server.ts | 3 + .../src/service-action-policy.test.ts | 27 +++ apps/ops-agent/src/service-action-policy.ts | 10 +- .../src/build-artifacts/control-authority.ts | 1 + packages/core/src/ops/contract.test.ts | 5 + packages/core/src/ops/contract.ts | 12 + scripts/ops-authority-inventory.json | 9 +- 20 files changed, 581 insertions(+), 10 deletions(-) create mode 100644 apps/api/src/services/__tests__/core-service-maintenance.test.ts create mode 100644 apps/api/src/services/core-service-maintenance.ts create mode 100644 apps/ops-agent/src/core-service-runtime.test.ts create mode 100644 apps/ops-agent/src/core-service-runtime.ts diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index a04534291..c371a68c8 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -466,6 +466,38 @@ describe("GET /admin/services/:id/logs", () => { }); describe("POST /admin/services/:id/action", () => { + it.each(["app-api", "traefik"])("queues the dedicated %s restart workflow", async (id) => { + mockRegistryGet.mockReturnValueOnce({ ...MOCK_SERVICE_RUNNING, id }); + mockExecuteAndWait.mockResolvedValueOnce({ + services: [ + { serviceId: id, actions: { restart: { available: true, workflow: "core-restart" } } }, + ], + }); + const response = await app.inject({ + method: "POST", + url: `/admin/services/${id}/action`, + payload: { action: "restart" }, + }); + expect(response.statusCode).toBe(200); + expect(mockJobRunnerEnqueue).toHaveBeenCalledWith( + "service.core-restart", + { service: id }, + fakeSession.user.id, + ); + }); + + it("does not queue an unavailable core restart", async () => { + mockRegistryGet.mockReturnValueOnce({ ...MOCK_SERVICE_RUNNING, id: "app-api" }); + mockExecuteAndWait.mockResolvedValueOnce({ services: [] }); + const response = await app.inject({ + method: "POST", + url: "/admin/services/app-api/action", + payload: { action: "restart" }, + }); + expect(response.statusCode).toBe(503); + expect(mockJobRunnerEnqueue).not.toHaveBeenCalled(); + }); + it("enqueues action, writes audit log, and returns jobId", async () => { mockRegistryGet.mockReturnValueOnce(MOCK_SERVICE_RUNNING); mockGetAdminSession.mockReturnValue(fakeSession); diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index ad17b961c..fdbc9cb7d 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -187,9 +187,27 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { return { error: `Service "${id}" not found` }; } const adminSession = getAdminSession(req); - + const coreRestart = action === "restart" && (id === "app-api" || id === "traefik"); + if (coreRestart) { + try { + const inventory = await executeAndWait( + createApiOpsClient(), + { kind: "services.inspect" }, + createDurableOpsKey("admin.core-restart.inspect", `${adminSession.user.id}:${id}`), + { signal: AbortSignal.timeout(15_000) }, + ); + const decision = inventory.services.find((s) => s.serviceId === id)?.actions.restart; + if (!decision?.available || decision.workflow !== "core-restart") { + reply.status(503); + return { error: "Core service restart is not available" }; + } + } catch { + reply.status(503); + return { error: "Core service restart is not available" }; + } + } const jobId = await jobRunner.enqueue( - `service.${action}`, + coreRestart ? "service.core-restart" : `service.${action}`, { service: id }, adminSession.user.id, ); diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index cf58aa1f0..9464af5ce 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -685,6 +685,10 @@ jobRunner.register("service.restart", async (ctx) => { await serviceRestart(service, ctx); return { service, action: "restart" }; }); +jobRunner.register("service.core-restart", async (ctx) => { + const { runCoreServiceMaintenance } = await import("./services/core-service-maintenance"); + return runCoreServiceMaintenance(ctx); +}); jobRunner.register("service.apply", async (ctx) => { const service = ctx.payload.service as string; diff --git a/apps/api/src/services/__tests__/core-service-maintenance.test.ts b/apps/api/src/services/__tests__/core-service-maintenance.test.ts new file mode 100644 index 000000000..3d59db932 --- /dev/null +++ b/apps/api/src/services/__tests__/core-service-maintenance.test.ts @@ -0,0 +1,78 @@ +import { afterAll, expect, it, vi } from "vitest"; + +const env = vi.hoisted(() => { + const value = process.env.BETTER_AUTH_SECRET; + process.env.BETTER_AUTH_SECRET ??= "core-restart-unit-test-only-000000000000000000"; + return value; +}); +afterAll(() => { + if (env === undefined) delete process.env.BETTER_AUTH_SECRET; + else process.env.BETTER_AUTH_SECRET = env; +}); +const execute = vi.hoisted(() => vi.fn()); +const inspect = vi.hoisted(() => vi.fn()); +const original = vi.hoisted(() => vi.fn()); +vi.mock("../admin-job-ops", () => ({ + executeAdminJobOperation: execute, + retainedAdminJobOperation: original, +})); +vi.mock("../ops-client", () => ({ + createApiOpsClient: () => ({}), + createDurableOpsKey: (scope: string, key: string) => `${scope}:${key}`, + executeAndWait: inspect, +})); +const { runCoreServiceMaintenance } = await import("../core-service-maintenance"); +const revision = `cfg1_${"r".repeat(43)}`; +function context(service: string) { + const ctx = { + jobId: "job", + payload: { service }, + signal: new AbortController().signal, + log: vi.fn(async () => {}), + setProgress: vi.fn(async () => {}), + checkpointResult: {} as Record, + checkpoint: vi.fn(async (value: Record) => { + ctx.checkpointResult = value; + }), + }; + return ctx; +} +it.each(["app-api", "traefik"])( + "retains the exact %s restart condition before dispatch", + async (service) => { + execute.mockReset().mockResolvedValue({ changed: true, serviceId: service }); + inspect.mockReset().mockResolvedValue({ revisionId: revision }); + original.mockReset().mockReturnValue(undefined); + const ctx = context(service); + await expect(runCoreServiceMaintenance(ctx)).resolves.toEqual({ service, action: "restart" }); + expect(ctx.checkpoint).toHaveBeenCalledBefore(execute); + expect(execute).toHaveBeenCalledWith( + ctx, + { + kind: "coreService.restart", + serviceId: service, + expectedConfigurationRevisionId: revision, + }, + "admin-job.service.core-restart", + { durableIdentity: service }, + ); + original.mockReturnValue({ + kind: "coreService.restart", + serviceId: service, + expectedConfigurationRevisionId: revision, + }); + inspect.mockClear(); + await runCoreServiceMaintenance(ctx); + expect(inspect).not.toHaveBeenCalled(); + }, +); +it.each(["ops-agent", "redis", "postgis", "unknown"])( + "rejects %s without executing anything", + async (service) => { + execute.mockClear(); + inspect.mockClear(); + await expect(runCoreServiceMaintenance(context(service))).rejects.toThrow("not available"); + expect(execute).not.toHaveBeenCalled(); + expect(inspect).not.toHaveBeenCalled(); + }, +); diff --git a/apps/api/src/services/__tests__/job-runner-initialize.test.ts b/apps/api/src/services/__tests__/job-runner-initialize.test.ts index 604d2585e..f09b6cedc 100644 --- a/apps/api/src/services/__tests__/job-runner-initialize.test.ts +++ b/apps/api/src/services/__tests__/job-runner-initialize.test.ts @@ -66,8 +66,12 @@ describe("job runner restart initialization", () => { "service.start", "service.stop", "service.restart", + "service.core-restart", "service.apply", "compose.render", + "configuration.apply", + "compose.start", + "integration.config.import", "system.diagnostics", "system.update", ]); diff --git a/apps/api/src/services/admin-job-ops.ts b/apps/api/src/services/admin-job-ops.ts index a30c4d069..7353196d4 100644 --- a/apps/api/src/services/admin-job-ops.ts +++ b/apps/api/src/services/admin-job-ops.ts @@ -751,6 +751,7 @@ export async function resumeAdminJobObservation( [ "service.stop", "service.restart", + "service.core-restart", "backup.operation", "data.operation", "system.diagnostics", diff --git a/apps/api/src/services/core-service-maintenance.ts b/apps/api/src/services/core-service-maintenance.ts new file mode 100644 index 000000000..a6d52bdc3 --- /dev/null +++ b/apps/api/src/services/core-service-maintenance.ts @@ -0,0 +1,44 @@ +import { configurationGenerationRevisionSchema, type OpsOperation } from "@openmapx/core/ops"; +import { AdminJobFailure } from "./admin-job-failure"; +import { executeAdminJobOperation, retainedAdminJobOperation } from "./admin-job-ops"; +import type { JobContext } from "./job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; + +export async function runCoreServiceMaintenance(ctx: JobContext): Promise> { + const serviceId = ctx.payload.service; + if (serviceId !== "app-api" && serviceId !== "traefik") + throw new AdminJobFailure("Core service restart is not available", {}); + const address = createDurableOpsKey( + "admin-job.service.core-restart", + `${ctx.jobId}:${serviceId}`, + ); + const original = retainedAdminJobOperation(ctx, address); + let operation: Extract; + if (original) { + if (original.kind !== "coreService.restart" || original.serviceId !== serviceId) + throw new AdminJobFailure("Retained core restart conflicts", {}); + operation = original; + } else { + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("admin-job.core-restart.inspect", ctx.jobId), + { signal: ctx.signal }, + ); + const revision = configurationGenerationRevisionSchema.safeParse(current.revisionId); + if (!revision.success) throw new AdminJobFailure("No applied configuration is available", {}); + operation = { + kind: "coreService.restart" as const, + serviceId, + expectedConfigurationRevisionId: revision.data, + }; + } + await ctx.checkpoint({ + ...ctx.checkpointResult, + _completion: { operationKey: address, result: { service: serviceId, action: "restart" } }, + }); + await executeAdminJobOperation(ctx, operation, "admin-job.service.core-restart", { + durableIdentity: serviceId, + }); + return { service: serviceId, action: "restart" }; +} diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 159a5fd30..4c291e299 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -107,6 +107,7 @@ export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ "service.start", "service.stop", "service.restart", + "service.core-restart", "service.apply", "compose.render", "configuration.apply", diff --git a/apps/ops-agent/src/core-service-runtime.test.ts b/apps/ops-agent/src/core-service-runtime.test.ts new file mode 100644 index 000000000..a819db4d1 --- /dev/null +++ b/apps/ops-agent/src/core-service-runtime.test.ts @@ -0,0 +1,215 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { realpathSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ownedServiceHost } from "./__fixtures__/owned-service-host"; +import { openOpsJobJournal } from "./journal"; +import { createProductionOpsRuntime } from "./production-runtime"; +import { buildOpsAgentServer } from "./server"; + +const roots: string[] = []; +afterEach(() => { + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +const waitFor = async (condition: () => boolean) => { + const end = Date.now() + 10_000; + while (!condition()) { + if (Date.now() > end) throw Error("Restart did not reach Docker"); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +}; + +describe("production core restart workflow", () => { + it.each([ + ["app-api", false], + ["traefik", false], + ["app-api", true], + ] as const)( + "keeps %s restart independent of the API observer (failure=%s)", + async (serviceId, fail) => { + const host = await ownedServiceHost(); + roots.push(host.cleanupRoot); + await host.select(["app-api", "traefik"]); + const compose = realpathSync(host.stack.composePath); + const service = host.loaded.find((s) => s.manifest.id === serviceId)!; + host.setRows([ + { + ...host.row, + ConfigImage: services.serviceContainerImageReference(service.manifest.container), + Labels: { + ...host.row.Labels, + "com.docker.compose.service": serviceId, + "com.docker.compose.project.working_dir": dirname(compose), + "com.docker.compose.project.config_files": compose, + }, + }, + ]); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const restarts: string[][] = []; + const runtime = await createProductionOpsRuntime({ + rootDir: host.root, + stack: host.stack, + loadAuthority: host.loadAuthority, + communityRepositories: host.repositories, + docker: { + fileExists: () => true, + execFile: async (file, args, options) => { + if (args[0] === "restart") { + restarts.push([...args]); + await held; + if (fail) throw Error("Docker restart failed"); + } + return host.execFile(file, args, options); + }, + }, + }); + const installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": "d".repeat(32) }, + runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + let submissions = 0, + deletions = 0; + app.addHook("preHandler", async (req) => { + if (req.method === "DELETE") deletions++; + if ( + req.method === "POST" && + (req.body as { operation?: { kind?: string } })?.operation?.kind === "coreService.restart" + ) + submissions++; + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address() as { port: number }; + const tokenFile = join(host.root, "api-token"); + writeFileSync(tokenFile, host.token, { mode: 0o600 }); + vi.stubEnv("OPS_AGENT_URL", `http://127.0.0.1:${address.port}`); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); + vi.stubEnv("BETTER_AUTH_SECRET", "core-restart-test-only-000000000000000000000000"); + try { + const headers = { authorization: `Bearer ${host.token}`, "x-ops-protocol-version": "2" }; + const submit = async (operation: Record) => { + const deadline = Date.now() + 60_000; + return app.inject({ + method: "POST", + url: "/v1/operations", + headers, + payload: { + version: 2, + requestId: `ops1_${randomBytes(16).toString("hex")}`, + operationKey: createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20_000).toISOString(), + execution: { requestedDeadlineAt: deadline }, + operation, + }, + }); + }; + for (const kind of ["service.stop", "service.restart", "service.update"]) { + const denied = await submit({ + kind, + serviceId, + ...(kind === "service.update" + ? { + expectedConfigurationRevisionId: services.currentConfigurationGeneration( + host.infraDir, + ), + } + : {}), + }); + expect(denied.statusCode).toBe(403); + } + const stale = await submit({ + kind: "coreService.restart", + serviceId, + expectedConfigurationRevisionId: `cfg1_${"z".repeat(43)}`, + }); + expect(stale.statusCode, stale.body).toBe(202); + let staleStatus: { state?: string; errorClass?: string } = {}; + const until = Date.now() + 5_000; + while (staleStatus.state !== "failed" && Date.now() < until) { + staleStatus = ( + await app.inject({ + method: "GET", + url: `/v1/operations/${stale.json().result.operationId}`, + headers, + }) + ).json().result; + if (staleStatus.state !== "failed") + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(staleStatus).toMatchObject({ state: "failed", errorClass: "conflict" }); + expect(restarts).toEqual([]); + submissions = 0; + const { runCoreServiceMaintenance } = await import( + "../../api/src/services/core-service-maintenance" + ); + const { resumeAdminJobObservation } = await import("../../api/src/services/admin-job-ops"); + type JobContext = import("../../api/src/services/job-runner").JobContext; + const controller = new AbortController(); + const ctx: JobContext = { + jobId: randomUUID(), + jobType: "service.core-restart", + execution: { owner: randomUUID(), version: 1 }, + payload: { service: serviceId }, + signal: controller.signal, + log: vi.fn(async () => {}), + setProgress: vi.fn(async () => {}), + checkpoint: async (value) => { + ctx.checkpointResult = structuredClone(value); + }, + }; + const first = runCoreServiceMaintenance(ctx).then( + (value) => ({ value }), + (error) => ({ error }), + ); + await waitFor(() => restarts.length === 1); + controller.abort(); + expect(await first).toHaveProperty("error"); + expect(deletions).toBe(0); + const restored: JobContext = { + ...ctx, + execution: { owner: randomUUID(), version: 2 }, + signal: new AbortController().signal, + observationOnly: true, + checkpointResult: structuredClone(ctx.checkpointResult), + checkpoint: async (value) => { + restored.checkpointResult = structuredClone(value); + }, + }; + release(); + if (fail) await expect(resumeAdminJobObservation(restored)).rejects.toBeDefined(); + else + await expect(resumeAdminJobObservation(restored)).resolves.toMatchObject({ + service: serviceId, + action: "restart", + }); + expect(restarts).toEqual([["restart", "--time", "30", host.row.Id]]); + expect(submissions).toBe(1); + expect(deletions).toBe(0); + } finally { + release(); + await app.close(); + closeControlInstallation(installation); + } + }, + 30_000, + ); +}); diff --git a/apps/ops-agent/src/core-service-runtime.ts b/apps/ops-agent/src/core-service-runtime.ts new file mode 100644 index 000000000..14bab1388 --- /dev/null +++ b/apps/ops-agent/src/core-service-runtime.ts @@ -0,0 +1,99 @@ +import { + assertArtifactTargetLeaseForward, + beginControlEffect, + retainConfigurationPreEffectRefusal, +} from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import { type DockerRuntimeOptions, runContainedProcess } from "./docker-runtime"; +import type { OwnedServiceResolver } from "./owned-services"; +import { OpsExecutionFailure, OpsHostUpdateRequiredError, type OpsRuntime } from "./runtime"; +import { decideServiceAction, type ServiceAuthority } from "./service-action-policy"; + +export function installCoreServiceRuntime( + runtime: OpsRuntime, + options: { + infraDir: string; + ownedServices: OwnedServiceResolver; + loadAuthority(): Promise; + execFile?: DockerRuntimeOptions["execFile"]; + }, +): void { + const execFile = options.execFile ?? runContainedProcess; + runtime["coreService.restart"] = async (operation, context) => { + if (!context.controlAuthority || !context.ownedTargetLease) + throw new OpsHostUpdateRequiredError(); + try { + return await services.withConfigurationGenerationWriter( + options.infraDir, + async () => { + context.signal.throwIfAborted(); + assertArtifactTargetLeaseForward(context.ownedTargetLease!); + if ( + services.currentConfigurationGeneration(options.infraDir) !== + operation.expectedConfigurationRevisionId + ) + throw new OpsExecutionFailure("conflict", {}); + const authority = await options.loadAuthority(); + const target = (await options.ownedServices.inspectOwnedServices(context.signal)).find( + (s) => s.serviceId === operation.serviceId, + ); + const service = authority.services.find((s) => s.serviceId === operation.serviceId); + const decision = decideServiceAction( + { ...authority, services: service ? [service] : [] }, + target, + "restart", + ); + const encoded = context.claim.capability.values.ownedService; + if ( + !target || + !decision.available || + decision.workflow !== "core-restart" || + typeof encoded !== "string" + ) + throw new OpsExecutionFailure("authorization", {}); + const claimed = JSON.parse(encoded); + if ( + claimed.serviceId !== target.serviceId || + claimed.containerId !== target.containerId || + JSON.stringify(claimed.provenance) !== JSON.stringify(target.provenance) + ) + throw new OpsExecutionFailure("conflict", {}); + beginControlEffect(context.controlAuthority!); + await execFile("docker", ["restart", "--time", "30", target.containerId], { + signal: context.signal, + timeout: Math.max(1, context.execution.deadlineAt - Date.now()), + maxBuffer: 1024 * 1024, + }); + const after = (await options.ownedServices.inspectOwnedServices(context.signal)).find( + (s) => s.serviceId === operation.serviceId, + ); + if ( + !after || + after.containerId !== target.containerId || + JSON.stringify(after.provenance) !== JSON.stringify(target.provenance) || + after.state !== "running" + ) + throw new OpsExecutionFailure("recovery_required", {}); + return { changed: true as const, serviceId: operation.serviceId }; + }, + context.signal, + ); + } catch (error) { + // The existing source verifies actual null effect-started, original owner + // and frozen/unissued children; it refuses after Docker was dispatched. + try { + retainConfigurationPreEffectRefusal(context.controlAuthority); + } catch { + /* Unknown effects keep the original fence. */ + } + if ( + error instanceof services.ConfigurationGenerationError && + error.result.outcome === "not_committed" && + !error.result.cleanupFailed && + error.cause instanceof OpsExecutionFailure + ) + throw error.cause; + throw error; + } + }; +} diff --git a/apps/ops-agent/src/policy.ts b/apps/ops-agent/src/policy.ts index 4b4e4317d..ab83a1a5e 100644 --- a/apps/ops-agent/src/policy.ts +++ b/apps/ops-agent/src/policy.ts @@ -59,6 +59,7 @@ export function serviceActionForOperation(kind: OpsOperationKind): ServiceAction case "service.stop": return "stop"; case "service.restart": + case "coreService.restart": return "restart"; case "service.recreate": case "service.recreateIsolated": @@ -119,7 +120,7 @@ export function createRegistryResourcePolicy(options: { isBuiltIn: true, buildable: true, }; - return decideServiceAction( + const decision = decideServiceAction( { revisionId: "registry-v1", services: [service], @@ -127,7 +128,13 @@ export function createRegistryResourcePolicy(options: { }, snapshot, action, - ).available; + ); + return ( + decision.available && + (kind === "coreService.restart" + ? decision.workflow === "core-restart" + : decision.execution === "direct") + ); }, allowBackup: options.resourceAuthority?.allowBackup ?? (() => false), allowPreparedRun: () => false, @@ -365,6 +372,7 @@ export function createRegistryResourceClaimer(options: { !registeredServiceIds.has(operation.serviceId) || (!decision.available && !checkedCoordinatorUpdate) || (decision.execution !== "direct" && + !(operation.kind === "coreService.restart" && decision.workflow === "core-restart") && !isOwnedMaintenanceRecreateAdmission( owner?.maintenanceStep, operation, diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index fa1b1f73b..e1525868a 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -630,7 +630,7 @@ describe("assembled production service authority", () => { expect(host.calls.some((args) => args.includes("restart"))).toBe(false); }); - it("keeps core workflows unavailable and dependency lifecycle host-only while permitting verified core logs", async () => { + it("requires core selection and keeps dependency lifecycle host-only while permitting verified core logs", async () => { const host = await fixture(); host.row.Labels["com.docker.compose.service"] = "ops-agent"; host.row.ConfigImage = "example/ops-agent:1"; @@ -643,7 +643,7 @@ describe("assembled production service authority", () => { expect(result.services.find((s) => s.serviceId === id)?.actions.restart).toMatchObject({ available: false, execution: "workflow", - reason: "unavailable", + reason: "disabled", workflow: "core-restart", }); for (const id of ["redis", "postgis"]) diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 732cf5bce..be85a4478 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -21,6 +21,7 @@ import { assertOwnedMaintenanceRecreateRuntime, assertStagedCoordinatorReleaseWrite, } from "./configuration-maintenance-store"; +import { installCoreServiceRuntime } from "./core-service-runtime"; import { createDockerRuntime, type DockerRuntimeOptions, @@ -207,6 +208,13 @@ export async function createProductionOpsRuntime( infraDir: options.stack.infraDir, runFixedCli, }); + installCoreServiceRuntime(runtime, { + infraDir: options.stack.infraDir, + ownedServices: owned, + loadAuthority: async () => + serviceActionAuthority(await options.loadAuthority(), options.stack.infraDir), + execFile: options.docker?.execFile, + }); installServicePullRuntime(runtime, { infraDir: options.stack.infraDir, candidateSource: options.candidateSource, diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index b3d87eab8..c36256fd3 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -1234,6 +1234,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif // The fixed DM pull runtime performs its source/target-index checks // under writer -> release lock, then begins immediately before Docker. if ( + job.operation.kind !== "coreService.restart" && !( job.operation.kind === "service.pull" && job.operation.serviceId === "data-manager" && @@ -2467,6 +2468,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif [ "configurationMaintenance.prepare", "stack.start", + "coreService.restart", "service.start", "service.recreate", "service.recreateIsolated", @@ -2485,6 +2487,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif [ "stack.start", "service.start", + "coreService.restart", "service.recreate", "service.recreateIsolated", ].includes(job.operation.kind) diff --git a/apps/ops-agent/src/service-action-policy.test.ts b/apps/ops-agent/src/service-action-policy.test.ts index cade72fdd..72bfadb9c 100644 --- a/apps/ops-agent/src/service-action-policy.test.ts +++ b/apps/ops-agent/src/service-action-policy.test.ts @@ -20,6 +20,33 @@ const snapshot: OwnedServiceSnapshot = { }, }; describe("shared service action decisions", () => { + it.each(["app-api", "traefik"])( + "offers only the dedicated restart workflow for owned enabled %s", + (serviceId) => { + const current = { ...snapshot, serviceId }; + const selected = { + ...authority, + services: [{ serviceId, isBuiltIn: true, buildable: false }], + enabledServiceIds: new Set([serviceId]), + }; + expect(decideServiceAction(selected, current, "restart")).toEqual({ + available: true, + execution: "workflow", + reason: null, + workflow: "core-restart", + }); + expect( + decideServiceAction({ ...selected, enabledServiceIds: new Set() }, current, "restart") + .available, + ).toBe(false); + expect(decideServiceAction(selected, undefined, "restart").available).toBe(false); + expect( + decideServiceAction(selected, { ...current, state: "stopped" }, "restart").available, + ).toBe(false); + for (const action of ["start", "stop", "apply", "update"] as const) + expect(decideServiceAction(selected, current, action).available).toBe(false); + }, + ); it("keeps every available direct decision consistent with its portable schema", () => { for (const action of SERVICE_ACTIONS) expect( diff --git a/apps/ops-agent/src/service-action-policy.ts b/apps/ops-agent/src/service-action-policy.ts index de791afe1..5c4ff5c48 100644 --- a/apps/ops-agent/src/service-action-policy.ts +++ b/apps/ops-agent/src/service-action-policy.ts @@ -37,8 +37,14 @@ export function decideServiceAction( ? { available: true, execution: "direct", reason: null, workflow: null } : denied("not_owned"); if (controlPlane.has(service.serviceId)) { - if (action === "restart" && service.serviceId !== "ops-agent") - return denied("unavailable", "workflow", "core-restart"); + if (action === "restart" && service.serviceId !== "ops-agent") { + if (!service.isBuiltIn || !authority.enabledServiceIds.has(service.serviceId)) + return denied("disabled", "workflow", "core-restart"); + if (!snapshot) return denied("not_owned", "workflow", "core-restart"); + if (!["running", "restarting", "paused"].includes(snapshot.state)) + return denied("not_running", "workflow", "core-restart"); + return { available: true, execution: "workflow", reason: null, workflow: "core-restart" }; + } return denied("control_plane"); } if (dependencies.has(service.serviceId)) return denied("host_only", "host"); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index e43d0d788..2eb2829bd 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1734,6 +1734,7 @@ function configurationPreEffectState(authority: VerifiedExecutionAuthority) { opsOperationFingerprint(operation) !== state.execution.operationFingerprint || !( operation.kind === "service.update" || + operation.kind === "coreService.restart" || operation.kind === "stack.start" || (operation.kind === "service.start" && operation.serviceId !== "data-manager") || (operation.kind === "service.pull" && diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 7eb32ec0a..23bfc6249 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -119,6 +119,11 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { }, "service.stop": { kind: "service.stop", serviceId: "motis" }, "service.restart": { kind: "service.restart", serviceId: "motis" }, + "coreService.restart": { + kind: "coreService.restart", + serviceId: "app-api", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }, "service.recreate": { kind: "service.recreate", serviceId: "motis", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index ecc2e4bcc..6388897cb 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -71,6 +71,7 @@ export const OPS_OPERATION_KINDS = [ "service.start", "service.stop", "service.restart", + "coreService.restart", "service.recreate", "service.recreateIsolated", "service.pull", @@ -196,6 +197,7 @@ export const OPS_KIND_POLICIES = { "service.start": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "service.stop": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "service.restart": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), + "coreService.restart": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "service.recreate": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "service.recreateIsolated": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "service.pull": asyncEffect("api", 15 * 60_000), @@ -390,6 +392,11 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ }), serviceOperation("service.stop"), serviceOperation("service.restart"), + z.strictObject({ + kind: z.literal("coreService.restart"), + serviceId: z.enum(["app-api", "traefik"]), + expectedConfigurationRevisionId: configurationGenerationRevisionSchema, + }), serviceRecreateOperationSchema, serviceRecreateIsolatedOperationSchema, servicePullOperationSchema, @@ -697,6 +704,7 @@ export async function authorizeOpsResources( if ("serviceId" in operation) { if ( (NEVER_MANAGE.has(operation.serviceId) && + operation.kind !== "coreService.restart" && operation.kind !== "service.logs" && operation.kind !== "service.logs.follow") || !(await allowed(policy.allowService?.(operation.kind, operation.serviceId))) @@ -1024,6 +1032,10 @@ export const OPS_RESULT_SCHEMAS = { "service.start": changed, "service.stop": changed, "service.restart": changed, + "coreService.restart": z.strictObject({ + changed: z.literal(true), + serviceId: z.enum(["app-api", "traefik"]), + }), "service.recreate": changed, "service.recreateIsolated": changed, "service.pull": servicePullResultSchema, diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 7c92e6407..290c6ddf8 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -46,6 +46,7 @@ "configurationMaintenance.prepare", "configurationMaintenance.settle", "controlRetention.acknowledge", + "coreService.restart", "data.clean", "data.convertOverpass", "data.downloadFonts", @@ -195,14 +196,18 @@ { "id": "api.admin-cli.bulk-service", "callSiteIds": [], - "semanticSources": ["apps/api/src/services/admin-job-handlers.ts#BulkServiceAction"], + "semanticSources": [ + "apps/api/src/services/admin-job-handlers.ts#BulkServiceAction", + "apps/ops-agent/src/core-service-runtime.ts" + ], "typedKinds": [ "service.start", "service.stop", "service.restart", "service.update", "service.build", - "services.buildAll" + "services.buildAll", + "coreService.restart" ] }, { From ad667701d57092794760bd4eb09c73388a8a8cdc Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 09:46:46 +0100 Subject: [PATCH 037/135] fix(admin): preserve unknown service state and troubleshooting access --- apps/api/openapi.json | 2860 ++++++++++++----- .../__tests__/admin-services-sdk.test.ts | 50 +- .../routes/__tests__/admin-services.test.ts | 172 +- apps/api/src/routes/admin-overview.ts | 34 +- apps/api/src/routes/admin-services.ts | 58 +- apps/api/src/routes/admin-system.ts | 5 +- .../__tests__/service-inventory.test.ts | 52 + .../dawarich/__tests__/provisioning.test.ts | 19 + .../api/src/services/dawarich/provisioning.ts | 10 + apps/api/src/services/deployment-mode.ts | 10 + apps/api/src/services/service-inventory.ts | 65 + .../utils/__tests__/docker-compose.test.ts | 91 +- apps/api/src/utils/docker-compose.ts | 20 +- apps/web/src/app/admin/layout.test.tsx | 11 + apps/web/src/components/admin/AdminLayout.tsx | 2 +- .../web/src/components/admin/AdminSidebar.tsx | 10 +- .../src/lib/admin/serverDeployment.test.tsx | 12 +- apps/web/src/lib/admin/serverDeployment.ts | 4 +- .../contracts/integration-admin/overview.ts | 1 + .../src/api/contracts/service-admin.test.ts | 20 +- .../api/contracts/service-admin/services.ts | 11 + .../getAdminOverview200Services.ts | 5 + .../service-models/checkAdminServices200.ts | 2 + .../checkAdminServices200Acquisition.ts | 13 + .../service-models/getAdminService200.ts | 4 + .../getAdminService200Acquisition.ts | 13 + .../getAdminService200Actions.ts | 21 + .../getAdminService200ActionsApply.ts | 12 + ...getAdminService200ActionsApplyExecution.ts | 11 + .../getAdminService200ActionsApplyReason.ts | 16 + .../getAdminService200ActionsApplyWorkflow.ts | 13 + .../getAdminService200ActionsBuild.ts | 12 + ...getAdminService200ActionsBuildExecution.ts | 11 + .../getAdminService200ActionsBuildReason.ts | 16 + .../getAdminService200ActionsBuildWorkflow.ts | 13 + .../getAdminService200ActionsLogs.ts | 12 + .../getAdminService200ActionsLogsExecution.ts | 11 + .../getAdminService200ActionsLogsReason.ts | 16 + .../getAdminService200ActionsLogsWorkflow.ts | 13 + .../getAdminService200ActionsPull.ts | 12 + .../getAdminService200ActionsPullExecution.ts | 11 + .../getAdminService200ActionsPullReason.ts | 16 + .../getAdminService200ActionsPullWorkflow.ts | 13 + .../getAdminService200ActionsRestart.ts | 12 + ...tAdminService200ActionsRestartExecution.ts | 11 + .../getAdminService200ActionsRestartReason.ts | 16 + ...etAdminService200ActionsRestartWorkflow.ts | 13 + .../getAdminService200ActionsStart.ts | 12 + ...getAdminService200ActionsStartExecution.ts | 11 + .../getAdminService200ActionsStartReason.ts | 16 + .../getAdminService200ActionsStartWorkflow.ts | 13 + .../getAdminService200ActionsStop.ts | 12 + .../getAdminService200ActionsStopExecution.ts | 11 + .../getAdminService200ActionsStopReason.ts | 16 + .../getAdminService200ActionsStopWorkflow.ts | 13 + .../getAdminService200ActionsUpdate.ts | 12 + ...etAdminService200ActionsUpdateExecution.ts | 11 + .../getAdminService200ActionsUpdateReason.ts | 16 + ...getAdminService200ActionsUpdateWorkflow.ts | 13 + .../generated/admin/service-models/index.ts | 69 + .../service-models/listAdminServices200.ts | 2 + .../listAdminServices200Acquisition.ts | 13 + .../listAdminServices200ServicesItem.ts | 2 + ...listAdminServices200ServicesItemActions.ts | 21 + ...dminServices200ServicesItemActionsApply.ts | 12 + ...ces200ServicesItemActionsApplyExecution.ts | 11 + ...rvices200ServicesItemActionsApplyReason.ts | 16 + ...ices200ServicesItemActionsApplyWorkflow.ts | 13 + ...dminServices200ServicesItemActionsBuild.ts | 12 + ...ces200ServicesItemActionsBuildExecution.ts | 11 + ...rvices200ServicesItemActionsBuildReason.ts | 16 + ...ices200ServicesItemActionsBuildWorkflow.ts | 13 + ...AdminServices200ServicesItemActionsLogs.ts | 12 + ...ices200ServicesItemActionsLogsExecution.ts | 11 + ...ervices200ServicesItemActionsLogsReason.ts | 16 + ...vices200ServicesItemActionsLogsWorkflow.ts | 13 + ...AdminServices200ServicesItemActionsPull.ts | 12 + ...ices200ServicesItemActionsPullExecution.ts | 11 + ...ervices200ServicesItemActionsPullReason.ts | 16 + ...vices200ServicesItemActionsPullWorkflow.ts | 13 + ...inServices200ServicesItemActionsRestart.ts | 12 + ...s200ServicesItemActionsRestartExecution.ts | 11 + ...ices200ServicesItemActionsRestartReason.ts | 16 + ...es200ServicesItemActionsRestartWorkflow.ts | 13 + ...dminServices200ServicesItemActionsStart.ts | 12 + ...ces200ServicesItemActionsStartExecution.ts | 11 + ...rvices200ServicesItemActionsStartReason.ts | 16 + ...ices200ServicesItemActionsStartWorkflow.ts | 13 + ...AdminServices200ServicesItemActionsStop.ts | 12 + ...ices200ServicesItemActionsStopExecution.ts | 11 + ...ervices200ServicesItemActionsStopReason.ts | 16 + ...vices200ServicesItemActionsStopWorkflow.ts | 13 + ...minServices200ServicesItemActionsUpdate.ts | 12 + ...es200ServicesItemActionsUpdateExecution.ts | 11 + ...vices200ServicesItemActionsUpdateReason.ts | 16 + ...ces200ServicesItemActionsUpdateWorkflow.ts | 13 + .../listAdminServices200Summary.ts | 5 + packages/core/src/ops/service-results.ts | 11 +- 98 files changed, 3484 insertions(+), 1046 deletions(-) create mode 100644 apps/api/src/services/__tests__/service-inventory.test.ts create mode 100644 apps/api/src/services/deployment-mode.ts create mode 100644 apps/api/src/services/service-inventory.ts create mode 100644 packages/core/src/api/generated/admin/service-models/checkAdminServices200Acquisition.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200Acquisition.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200Actions.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApply.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuild.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogs.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPull.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestart.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStart.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStop.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdate.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200Acquisition.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActions.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApply.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuild.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogs.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPull.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestart.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStart.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStop.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopWorkflow.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdate.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateExecution.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateWorkflow.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 81dbae41e..a963529af 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -33753,6 +33753,11 @@ "minimum": 0, "maximum": 9007199254740991 }, + "unknown": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, "unhealthy": { "type": "integer", "minimum": 0, @@ -33762,6 +33767,7 @@ "required": [ "running", "stopped", + "unknown", "unhealthy" ], "additionalProperties": false @@ -34569,6 +34575,51 @@ "schema": { "type": "object", "properties": { + "acquisition": { + "anyOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "available" + ] + }, + "reason": { + "type": "null" + } + }, + "required": [ + "state", + "reason" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "unavailable" + ] + }, + "reason": { + "type": "string", + "enum": [ + "ops_unavailable" + ] + } + }, + "required": [ + "state", + "reason" + ], + "additionalProperties": false + } + ] + }, "services": { "type": "array", "items": { @@ -34809,216 +34860,704 @@ "not-running", "unknown" ] - } - }, - "required": [ - "id", - "name", - "version", - "quality", - "provides", - "consumes", - "enabled", - "isBuiltIn", - "status" - ], - "additionalProperties": false - } - }, - "summary": { - "type": "object", - "properties": { - "running": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "stopped": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "total": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "running", - "stopped", - "total" - ], - "additionalProperties": false - } - }, - "required": [ - "services", - "summary" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "401": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "403": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, + }, + "actions": { + "type": "object", + "properties": { + "start": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "stop": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "restart": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "update": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "build": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "logs": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "pull": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "apply": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + } + }, + "required": [ + "start", + "stop", + "restart", + "update", + "build", + "logs", + "pull", + "apply" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "name", + "version", + "quality", + "provides", + "consumes", + "enabled", + "isBuiltIn", + "status", + "actions" + ], + "additionalProperties": false + } + }, + "summary": { + "type": "object", + "properties": { + "running": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stopped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "unknown": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "total": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "running", + "stopped", + "unknown", + "total" + ], + "additionalProperties": false + } + }, + "required": [ + "acquisition", + "services", + "summary" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, "error": { "type": "string" }, @@ -37866,6 +38405,51 @@ "schema": { "type": "object", "properties": { + "acquisition": { + "anyOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "available" + ] + }, + "reason": { + "type": "null" + } + }, + "required": [ + "state", + "reason" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "unavailable" + ] + }, + "reason": { + "type": "string", + "enum": [ + "ops_unavailable" + ] + } + }, + "required": [ + "state", + "reason" + ], + "additionalProperties": false + } + ] + }, "statuses": { "type": "array", "items": { @@ -37901,6 +38485,7 @@ } }, "required": [ + "acquisition", "statuses", "checkedAt" ], @@ -40394,6 +40979,51 @@ "schema": { "type": "object", "properties": { + "acquisition": { + "anyOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "available" + ] + }, + "reason": { + "type": "null" + } + }, + "required": [ + "state", + "reason" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "unavailable" + ] + }, + "reason": { + "type": "string", + "enum": [ + "ops_unavailable" + ] + } + }, + "required": [ + "state", + "reason" + ], + "additionalProperties": false + } + ] + }, "manifest": { "type": "object", "properties": { @@ -40445,128 +41075,431 @@ "type": "string", "pattern": "^[a-z0-9]([a-z0-9._\\-/])*$" }, - "tag": { + "tag": { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]+$" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "containerName": { + "type": "string", + "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_.-]*$" + }, + "expose": { + "type": "array", + "items": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + }, + "networkAliases": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$" + } + }, + "command": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] + }, + "entrypoint": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] + }, + "environment": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "envFile": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "workingDir": { + "type": "string" + }, + "user": { + "type": "string" + }, + "shmSize": { + "type": "string" + }, + "capAdd": { + "type": "array", + "items": { + "type": "string" + } + }, + "capDrop": { + "type": "array", + "items": { + "type": "string" + } + }, + "devices": { + "type": "array", + "items": { + "type": "string", + "pattern": "^\\/dev\\/[a-zA-Z0-9_\\-/]+$" + } + }, + "gpu": { + "type": "object", + "properties": { + "driver": { + "type": "string" + }, + "count": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "string", + "enum": [ + "all" + ] + } + ] + }, + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + }, + "privileged": { + "type": "boolean" + }, + "networkMode": { "type": "string", - "pattern": "^[a-zA-Z0-9._-]+$" + "enum": [ + "bridge", + "host" + ] }, - "digest": { + "memory": { + "type": "string" + }, + "restart": { "type": "string", - "pattern": "^sha256:[a-f0-9]{64}$" + "enum": [ + "no", + "on-failure", + "always", + "unless-stopped" + ] }, - "containerName": { + "stopGracePeriod": { "type": "string", - "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_.-]*$" + "pattern": "^(\\d+(h|m|s|ms))+$" }, - "expose": { - "type": "array", - "items": { - "type": "integer", - "minimum": 1, - "maximum": 65535 + "healthcheck": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "http", + "tcp", + "exec" + ] + }, + "path": { + "type": "string" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "command": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] + }, + "interval": { + "type": "string" + }, + "timeout": { + "type": "string" + }, + "retries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "startPeriod": { + "type": "string" + } + }, + "required": [ + "type" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] } }, - "networkAliases": { + "dependsOn": { "type": "array", "items": { + "type": "object", + "properties": { + "service": { + "type": "string" + }, + "condition": { + "type": "string", + "enum": [ + "service_started", + "service_healthy" + ] + } + }, + "required": [ + "service" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + } + }, + "logging": { + "type": "object", + "properties": { + "driver": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "driver" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + } + }, + "required": [ + "image", + "tag" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + }, + "provides": { + "type": "array", + "items": { + "anyOf": [ + { "type": "string", - "pattern": "^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$" + "minLength": 1 + }, + { + "type": "object", + "properties": { + "capability": { + "type": "string", + "minLength": 1 + }, + "metadata": { + "type": "object", + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + } + }, + "required": [ + "capability" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + } + ] + } + }, + "consumes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "instance": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "mountAt": { + "type": "string", + "pattern": "^\\/[^\\s]+$" + }, + "targetFilename": { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]+$" + }, + "readOnly": { + "type": "boolean" + }, + "required": { + "type": "boolean" } }, - "command": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "string" - } - ] - }, - "entrypoint": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, + "required": [ + "type", + "mountAt" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ { - "type": "string" + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" } ] - }, - "environment": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "envFile": { - "type": "array", - "items": { + } + } + }, + "produces": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { "type": "string", "minLength": 1 - } - }, - "workingDir": { - "type": "string" - }, - "user": { - "type": "string" - }, - "shmSize": { - "type": "string" - }, - "capAdd": { - "type": "array", - "items": { - "type": "string" - } - }, - "capDrop": { - "type": "array", - "items": { - "type": "string" - } - }, - "devices": { - "type": "array", - "items": { + }, + "instance": { "type": "string", - "pattern": "^\\/dev\\/[a-zA-Z0-9_\\-/]+$" + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "sourceDir": { + "type": "string", + "minLength": 1 } }, - "gpu": { - "type": "object", - "properties": { - "driver": { - "type": "string" - }, - "count": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0, - "maximum": 9007199254740991 - }, - { - "type": "string", - "enum": [ - "all" - ] - } - ] - }, - "capabilities": { - "type": "array", - "items": { - "type": "string" - } + "required": [ + "type", + "sourceDir" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" } - }, + ] + } + } + }, + "selectionDependencies": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "communityNetworkAccess": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "configSchema": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "object" + ] + }, + "properties": { + "type": "object", "additionalProperties": { "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", "allOf": [ @@ -40576,109 +41509,171 @@ ] } }, - "privileged": { - "type": "boolean" - }, - "networkMode": { - "type": "string", - "enum": [ - "bridge", - "host" - ] - }, - "memory": { - "type": "string" - }, - "restart": { - "type": "string", - "enum": [ - "no", - "on-failure", - "always", - "unless-stopped" - ] - }, - "stopGracePeriod": { - "type": "string", - "pattern": "^(\\d+(h|m|s|ms))+$" + "required": { + "type": "array", + "items": { + "type": "string" + } }, - "healthcheck": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "http", - "tcp", - "exec" - ] - }, - "path": { - "type": "string" - }, - "port": { - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "command": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "string" - } - ] - }, - "interval": { - "type": "string" - }, - "timeout": { - "type": "string" - }, - "retries": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "startPeriod": { - "type": "string" + "additionalProperties": { + "type": "boolean" + } + }, + "required": [ + "type", + "properties" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + }, + "envVars": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "default": { + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" } + ] + } + } + }, + "ownsSchema": { + "type": "string", + "pattern": "^[a-z_][a-z0-9_]*$" + }, + "volumes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "pattern": "^openmapx-[a-z0-9-]+$" }, - "required": [ - "type" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } + "mountAt": { + "type": "string", + "pattern": "^\\/[^\\s]+$" + }, + "readOnly": { + "type": "boolean" + }, + "backup": { + "type": "boolean" + }, + "backupMode": { + "type": "string", + "enum": [ + "tar", + "pg_dump" ] } }, - "dependsOn": { + "required": [ + "name", + "mountAt" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + } + }, + "bindMounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "readOnly": { + "type": "boolean" + }, + "optional": { + "type": "boolean" + } + }, + "required": [ + "source", + "target" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + } + }, + "exposure": { + "type": "object", + "properties": { + "hostPorts": { "type": "array", "items": { "type": "object", "properties": { - "service": { - "type": "string" + "container": { + "type": "integer", + "minimum": 1, + "maximum": 65535 }, - "condition": { + "host": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "protocol": { "type": "string", "enum": [ - "service_started", - "service_healthy" + "tcp", + "udp" ] + }, + "bindAddress": { + "type": "string" } }, "required": [ - "service" + "container", + "host" ], "additionalProperties": { "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", @@ -40690,21 +41685,89 @@ } } }, - "logging": { + "proxy": { "type": "object", "properties": { - "driver": { - "type": "string" + "enabled": { + "type": "boolean" }, - "options": { + "host": { "type": "object", + "properties": { + "default": { + "type": "string" + }, + "configKey": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + } + }, + "required": [ + "default" + ], "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + }, + "pathPrefix": { + "type": "string", + "pattern": "^\\/[a-zA-Z0-9._\\-/]*$" + }, + "stripPrefix": { + "type": "boolean" + }, + "middleware": { + "type": "array", + "items": { "type": "string" } + }, + "authRequired": { + "type": "boolean" + }, + "priority": { + "type": "integer", + "minimum": 0, + "maximum": 1000000 + }, + "additionalRoutes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pathPrefix": { + "type": "string", + "pattern": "^\\/[a-zA-Z0-9._\\-/]*$" + }, + "path": { + "type": "string", + "pattern": "^\\/[a-zA-Z0-9._\\-/]*$" + }, + "middleware": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + } } }, "required": [ - "driver" + "enabled" ], "additionalProperties": { "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", @@ -40716,92 +41779,44 @@ } } }, - "required": [ - "image", - "tag" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - }, - "provides": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "capability": { - "type": "string", - "minLength": 1 - }, - "metadata": { - "type": "object", - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - } - }, - "required": [ - "capability" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" } ] } }, - "consumes": { - "type": "array", - "items": { + "buildCommand": { + "type": "string" + }, + "buildImages": { + "type": "object", + "propertyNames": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "additionalProperties": { "type": "object", "properties": { - "type": { - "type": "string", - "minLength": 1 - }, - "instance": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]*$" - }, - "mountAt": { + "image": { "type": "string", - "pattern": "^\\/[^\\s]+$" + "pattern": "^[a-z0-9]([a-z0-9._\\-/])*$" }, - "targetFilename": { + "tag": { "type": "string", "pattern": "^[a-zA-Z0-9._-]+$" }, - "readOnly": { - "type": "boolean" - }, - "required": { - "type": "boolean" + "digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" } }, "required": [ - "type", - "mountAt" + "image", + "tag", + "digest" ], "additionalProperties": { "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", @@ -40813,85 +41828,65 @@ } } }, - "produces": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "minLength": 1 - }, - "instance": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]*$" - }, - "sourceDir": { - "type": "string", - "minLength": 1 - } + "ui": { + "type": "object", + "properties": { + "icon": { + "type": "string" }, - "required": [ - "type", - "sourceDir" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] + "category": { + "type": "string" } + }, + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] } }, - "selectionDependencies": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]*$" - } - }, - "communityNetworkAccess": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]*$" - } - }, - "configSchema": { + "subjectData": { "type": "object", "properties": { - "type": { + "storesPersonalData": { + "type": "boolean" + }, + "strategy": { "type": "string", "enum": [ - "object" + "collector", + "operator_task", + "not_personal" ] }, - "properties": { - "type": "object", - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - }, - "required": { + "registrationIds": { + "maxItems": 64, "type": "array", "items": { - "type": "string" + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" } }, - "additionalProperties": { - "type": "boolean" + "operatorInstructions": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 2000 + }, + { + "type": "null" + } + ] } }, "required": [ - "type", - "properties" + "storesPersonalData", + "strategy", + "registrationIds", + "operatorInstructions" ], "additionalProperties": { "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", @@ -40901,350 +41896,497 @@ } ] } + } + }, + "required": [ + "id", + "name", + "version", + "quality", + "container" + ], + "additionalProperties": { + "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", + "allOf": [ + { + "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + } + ] + } + }, + "directory": { + "type": "string" + }, + "isBuiltIn": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "status": { + "type": "string", + "enum": [ + "running", + "exited", + "restarting", + "created", + "paused", + "not-running", + "unknown" + ] + }, + "actions": { + "type": "object", + "properties": { + "start": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false }, - "envVars": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "required": { - "type": "boolean" - }, - "description": { - "type": "string" - }, - "default": { - "type": "string" - } + "stop": { + "type": "object", + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] }, - "required": [ - "name" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ + "workflow": { + "anyOf": [ { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" } ] } - } - }, - "ownsSchema": { - "type": "string", - "pattern": "^[a-z_][a-z0-9_]*$" + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false }, - "volumes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "pattern": "^openmapx-[a-z0-9-]+$" - }, - "mountAt": { - "type": "string", - "pattern": "^\\/[^\\s]+$" - }, - "readOnly": { - "type": "boolean" - }, - "backup": { - "type": "boolean" - }, - "backupMode": { - "type": "string", - "enum": [ - "tar", - "pg_dump" - ] - } + "restart": { + "type": "object", + "properties": { + "available": { + "type": "boolean" }, - "required": [ - "name", - "mountAt" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" } ] - } - } - }, - "bindMounts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source": { - "type": "string", - "minLength": 1 - }, - "target": { - "type": "string", - "minLength": 1 - }, - "readOnly": { - "type": "boolean" - }, - "optional": { - "type": "boolean" - } }, - "required": [ - "source", - "target" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ + "workflow": { + "anyOf": [ { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" } ] } - } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false }, - "exposure": { + "update": { "type": "object", "properties": { - "hostPorts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "container": { - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "host": { - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "protocol": { - "type": "string", - "enum": [ - "tcp", - "udp" - ] - }, - "bindAddress": { - "type": "string" - } - }, - "required": [ - "container", - "host" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - } + "available": { + "type": "boolean" }, - "proxy": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - }, - "host": { - "type": "object", - "properties": { - "default": { - "type": "string" - }, - "configKey": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - } - }, - "required": [ - "default" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - }, - "pathPrefix": { + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { "type": "string", - "pattern": "^\\/[a-zA-Z0-9._\\-/]*$" - }, - "stripPrefix": { - "type": "boolean" - }, - "middleware": { - "type": "array", - "items": { - "type": "string" - } - }, - "authRequired": { - "type": "boolean" - }, - "priority": { - "type": "integer", - "minimum": 0, - "maximum": 1000000 + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] }, - "additionalRoutes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "pathPrefix": { - "type": "string", - "pattern": "^\\/[a-zA-Z0-9._\\-/]*$" - }, - "path": { - "type": "string", - "pattern": "^\\/[a-zA-Z0-9._\\-/]*$" - }, - "middleware": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - } + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" } - }, - "required": [ - "enabled" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } + ] } }, - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - }, - "buildCommand": { - "type": "string" + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false }, - "buildImages": { + "build": { "type": "object", - "propertyNames": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]*$" + "properties": { + "available": { + "type": "boolean" + }, + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] + } }, - "additionalProperties": { - "type": "object", - "properties": { - "image": { - "type": "string", - "pattern": "^[a-z0-9]([a-z0-9._\\-/])*$" - }, - "tag": { - "type": "string", - "pattern": "^[a-zA-Z0-9._-]+$" - }, - "digest": { - "type": "string", - "pattern": "^sha256:[a-f0-9]{64}$" - } + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false + }, + "logs": { + "type": "object", + "properties": { + "available": { + "type": "boolean" }, - "required": [ - "image", - "tag", - "digest" - ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" } ] } - } + }, + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false }, - "ui": { + "pull": { "type": "object", "properties": { - "icon": { - "type": "string" + "available": { + "type": "boolean" }, - "category": { - "type": "string" + "execution": { + "type": "string", + "enum": [ + "direct", + "workflow", + "host" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] + }, + "workflow": { + "anyOf": [ + { + "type": "string", + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] + }, + { + "type": "null" + } + ] } }, - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } + "required": [ + "available", + "execution", + "reason", + "workflow" + ], + "additionalProperties": false }, - "subjectData": { + "apply": { "type": "object", "properties": { - "storesPersonalData": { + "available": { "type": "boolean" }, - "strategy": { + "execution": { "type": "string", "enum": [ - "collector", - "operator_task", - "not_personal" + "direct", + "workflow", + "host" ] }, - "registrationIds": { - "maxItems": 64, - "type": "array", - "items": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]*$" - } + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "disabled", + "not_owned", + "not_running", + "control_plane", + "host_only", + "unavailable", + "not_buildable" + ] + }, + { + "type": "null" + } + ] }, - "operatorInstructions": { + "workflow": { "anyOf": [ { "type": "string", - "minLength": 1, - "maxLength": 2000 + "enum": [ + "core-restart", + "system-update", + "coordinator-apply", + "coordinator-update" + ] }, { "type": "null" @@ -41253,65 +42395,35 @@ } }, "required": [ - "storesPersonalData", - "strategy", - "registrationIds", - "operatorInstructions" + "available", + "execution", + "reason", + "workflow" ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } + "additionalProperties": false } }, "required": [ - "id", - "name", - "version", - "quality", - "container" + "start", + "stop", + "restart", + "update", + "build", + "logs", + "pull", + "apply" ], - "additionalProperties": { - "description": "Recursive JSON. Objects with an own __proto__ key are rejected at every depth.", - "allOf": [ - { - "$ref": "#/components/schemas/Admin_getAdminService_200___schema0" - } - ] - } - }, - "directory": { - "type": "string" - }, - "isBuiltIn": { - "type": "boolean" - }, - "enabled": { - "type": "boolean" - }, - "status": { - "type": "string", - "enum": [ - "running", - "exited", - "restarting", - "created", - "paused", - "not-running", - "unknown" - ] + "additionalProperties": false } }, "required": [ + "acquisition", "manifest", "directory", "isBuiltIn", "enabled", - "status" + "status", + "actions" ], "additionalProperties": false } diff --git a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts index eeac40a39..92770f966 100644 --- a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts @@ -5,6 +5,7 @@ import { type OpsOperationKind, type OpsSubmitResult, parseOpsJobStatusForKind, + SERVICE_ACTIONS, } from "@openmapx/core/ops"; import Fastify from "fastify"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -223,6 +224,25 @@ const execute = vi.fn( operation: OpsOperation, options: { operationKey?: string; signal?: AbortSignal }, ): Promise => { + if (operation.kind === "services.inspect") + return { + execution: "sync", + kind: operation.kind, + value: { + services: [ + { + serviceId: "motis", + state: "running", + actions: Object.fromEntries( + SERVICE_ACTIONS.map((action) => [ + action, + { available: false, execution: "direct", reason: "unavailable", workflow: null }, + ]), + ) as never, + }, + ], + }, + }; if (operation.kind === "stack.status") return { execution: "sync", @@ -354,6 +374,22 @@ async function wire(max = 1000) { } describe("actual service producers through final exported SDK", () => { + it("keeps configured deployment identity while the agent is unavailable", async () => { + vi.stubEnv("OPS_AGENT_URL", "http://ops-agent:3000"); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", "/unavailable/token"); + dockerAvailable.mockResolvedValue(false); + const { options, close } = await wire(); + try { + expect(await sdk.getAdminDeployment(options)).toMatchObject({ + status: 200, + data: { selfHosted: true, dockerAvailable: false, hostControlConfigured: true }, + }); + } finally { + await close(); + vi.unstubAllEnvs(); + } + }); + it.each([null, { count: 1 }])( "forwards overlapping intents without route-local desired writes or inverse compensation (prior %j)", async (prior) => { @@ -902,7 +938,19 @@ it("retains structured domain failures and retained credential jobs through SDK }); expect(await sdk.listAdminServices(options)).toMatchObject({ status: 200, - data: { services: [{ status: "not-running" }] }, + data: { + acquisition: { state: "unavailable", reason: "ops_unavailable" }, + summary: { running: 0, stopped: 0, unknown: 1, total: 1 }, + services: [ + { + status: "unknown", + actions: { + start: { available: false, reason: "unavailable" }, + logs: { available: false, reason: "unavailable" }, + }, + }, + ], + }, }); expect(cancelJob).not.toHaveBeenCalled(); } finally { diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index c371a68c8..a54d02e67 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -1,3 +1,4 @@ +import { SERVICE_ACTIONS } from "@openmapx/core/ops"; import type { FastifyInstance } from "fastify"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { applyTrustedConfiguration } from "../../services/trusted-config-operations"; @@ -210,18 +211,34 @@ afterEach(() => { vi.clearAllMocks(); mockRegistryGet.mockReset(); mockRegistryGet.mockReturnValue(null); + vi.mocked(applyTrustedConfiguration) + .mockReset() + .mockResolvedValue({ jobId: "job-secret", revisionId: `cfg1_${"a".repeat(43)}` } as never); }); describe("GET /admin/services", () => { it("returns running/stopped summary with normalised service fields", async () => { mockRegistryList.mockReturnValueOnce([MOCK_SERVICE_RUNNING, MOCK_SERVICE_STOPPED]); - mockDockerComposePs.mockResolvedValueOnce([{ service: "motis", state: "running" }]); + mockExecuteAndWait.mockResolvedValueOnce({ + services: [ + { + serviceId: "motis", + state: "running", + actions: Object.fromEntries( + SERVICE_ACTIONS.map((action) => [ + action, + { available: false, execution: "direct", reason: "unavailable", workflow: null }, + ]), + ), + }, + ], + }); const res = await app.inject({ method: "GET", url: "/admin/services" }); expect(res.statusCode).toBe(200); const body = res.json(); - expect(body.summary).toEqual({ running: 1, stopped: 1, total: 2 }); + expect(body.summary).toEqual({ running: 1, stopped: 1, unknown: 0, total: 2 }); expect(body.services).toHaveLength(2); const motis = body.services.find((s: { id: string }) => s.id === "motis"); expect(motis).toMatchObject({ id: "motis", status: "running", enabled: true }); @@ -231,13 +248,13 @@ describe("GET /admin/services", () => { it("returns empty summary when registry has no services", async () => { mockRegistryList.mockReturnValueOnce([]); - mockDockerComposePs.mockResolvedValueOnce([]); + mockExecuteAndWait.mockResolvedValueOnce({ services: [] }); const res = await app.inject({ method: "GET", url: "/admin/services" }); expect(res.statusCode).toBe(200); const body = res.json(); - expect(body.summary).toEqual({ running: 0, stopped: 0, total: 0 }); + expect(body.summary).toEqual({ running: 0, stopped: 0, unknown: 0, total: 0 }); expect(body.services).toHaveLength(0); }); @@ -370,7 +387,21 @@ describe("GET /admin/services/:id", () => { it("returns service detail with docker status", async () => { mockRegistryGet.mockReturnValueOnce(MOCK_SERVICE_RUNNING); - mockDockerComposePs.mockResolvedValueOnce([{ service: "motis", state: "running" }]); + mockRegistryList.mockReturnValueOnce([MOCK_SERVICE_RUNNING]); + mockExecuteAndWait.mockResolvedValueOnce({ + services: [ + { + serviceId: "motis", + state: "running", + actions: Object.fromEntries( + SERVICE_ACTIONS.map((action) => [ + action, + { available: false, execution: "direct", reason: "unavailable", workflow: null }, + ]), + ), + }, + ], + }); const res = await app.inject({ method: "GET", url: "/admin/services/motis" }); @@ -570,8 +601,22 @@ describe("GET /admin/services/selection", () => { }); describe("POST /admin/services/check", () => { - it("runs docker ps, writes audit log, and returns statuses", async () => { - mockDockerComposePs.mockResolvedValueOnce([{ service: "motis", state: "running" }]); + it("reads owned inventory, writes audit log, and returns statuses", async () => { + mockRegistryList.mockReturnValueOnce([MOCK_SERVICE_RUNNING]); + mockExecuteAndWait.mockResolvedValueOnce({ + services: [ + { + serviceId: "motis", + state: "running", + actions: Object.fromEntries( + SERVICE_ACTIONS.map((action) => [ + action, + { available: false, execution: "direct", reason: "unavailable", workflow: null }, + ]), + ), + }, + ], + }); mockGetAdminSession.mockReturnValue(fakeSession); const res = await app.inject({ method: "POST", url: "/admin/services/check" }); @@ -657,6 +702,7 @@ describe("service credentials", () => { const res = await app.inject({ method: "GET", url: "/admin/services/openconditions-ingest/config", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, }); expect(res.statusCode).toBe(200); @@ -671,7 +717,7 @@ describe("service credentials", () => { describe("POST /admin/services/:id/config", () => { it.each(["unknown", "committed", "not_committed"] as const)( - "restores saved overrides only with proven non-commit (%s)", + "leaves failed configuration reconciliation to the retained producer (%s)", async (outcome) => { const error = new TrustedConfigurationApplyError({ ...(outcome === "unknown" @@ -692,10 +738,17 @@ describe("service credentials", () => { const res = await app.inject({ method: "POST", url: "/admin/services/openconditions-ingest/config", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { config: {} }, }); expect(res.statusCode).toBe(500); - expect(mockRestoreServiceConfig).toHaveBeenCalledTimes(outcome === "not_committed" ? 1 : 0); + expect(mockRestoreServiceConfig).not.toHaveBeenCalled(); + expect(applyTrustedConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "serviceConfig.apply", + serviceId: "openconditions-ingest", + }), + ); }, ); it("persists only the route-validated update through the atomic writer", async () => { @@ -707,13 +760,27 @@ describe("service credentials", () => { const res = await app.inject({ method: "POST", url: "/admin/services/openconditions-ingest/config", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { config: { RATE_LIMIT_MAX: 240, UNKNOWN: "discarded" } }, }); expect(res.statusCode).toBe(200); - expect(mockMergeServiceConfig).toHaveBeenCalledWith("openconditions-ingest", { - RATE_LIMIT_MAX: 240, - }); + expect(applyTrustedConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "serviceConfig.apply", + overlay: expect.objectContaining({ + changes: [ + { + kind: "service-config", + serviceId: "openconditions-ingest", + set: { RATE_LIMIT_MAX: 240 }, + unset: [], + }, + ], + }), + }), + ); + expect(mockMergeServiceConfig).not.toHaveBeenCalled(); expect(mockDbSelect).not.toHaveBeenCalled(); }); }); @@ -823,37 +890,48 @@ describe("service credentials", () => { const res = await app.inject({ method: "PUT", url: "/admin/services/openconditions-ingest/credentials/NY_511_API_KEY", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { value: "secret-123" }, }); expect(res.statusCode).toBe(200); - expect(mockSetServiceSecret).toHaveBeenCalledWith( - "openconditions-ingest", - "NY_511_API_KEY", - "secret-123", - fakeSession.user.id, - ); - expect(mockJobRunnerEnqueue).toHaveBeenCalledWith( - "service.apply", - { service: "openconditions-ingest", configurationKind: "vault" }, - fakeSession.user.id, + expect(mockSetServiceSecret).not.toHaveBeenCalled(); + expect(applyTrustedConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "vault.apply", + followUp: "service-apply", + overlay: expect.objectContaining({ + changes: [ + { + kind: "service-secret", + serviceId: "openconditions-ingest", + key: "NY_511_API_KEY", + action: "set", + value: "secret-123", + }, + ], + }), + }), ); - expect(res.json()).toEqual({ ok: true, jobId: "job-123" }); expect(mockWriteAuditLog).toHaveBeenCalledWith( expect.objectContaining({ action: "service.credential.set" }), ); }); - it("PUT returns needsRender (no apply) when Docker is unavailable", async () => { + it("PUT delegates readiness checks to the retained producer during an outage", async () => { mockIsDockerAvailable.mockResolvedValue(false); const res = await app.inject({ method: "PUT", url: "/admin/services/openconditions-ingest/credentials/NY_511_API_KEY", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { value: "secret-123" }, }); - expect(res.json()).toEqual({ ok: true, needsRender: true }); + expect(res.json()).toEqual({ ok: true, jobId: "job-secret" }); + expect(applyTrustedConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ kind: "vault.apply" }), + ); expect(mockJobRunnerEnqueue).not.toHaveBeenCalled(); }); @@ -861,6 +939,7 @@ describe("service credentials", () => { const res = await app.inject({ method: "PUT", url: "/admin/services/openconditions-ingest/credentials/RATE_LIMIT_MAX", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { value: "x" }, }); @@ -872,6 +951,7 @@ describe("service credentials", () => { const res = await app.inject({ method: "PUT", url: "/admin/services/openconditions-ingest/credentials/..%2F..%2Fescaped", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { value: "x" }, }); @@ -885,6 +965,7 @@ describe("service credentials", () => { const res = await app.inject({ method: "PUT", url: "/admin/services/openconditions-ingest/credentials/NY_511_API_KEY", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { value: "secret-123" }, }); @@ -896,14 +977,26 @@ describe("service credentials", () => { const res = await app.inject({ method: "DELETE", url: "/admin/services/openconditions-ingest/credentials/NY_511_API_KEY", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, }); expect(res.statusCode).toBe(200); - expect(mockDeleteServiceSecret).toHaveBeenCalledWith("openconditions-ingest", "NY_511_API_KEY"); - expect(mockJobRunnerEnqueue).toHaveBeenCalledWith( - "service.apply", - { service: "openconditions-ingest", configurationKind: "vault" }, - fakeSession.user.id, + expect(mockDeleteServiceSecret).not.toHaveBeenCalled(); + expect(applyTrustedConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "vault.apply", + followUp: "service-apply", + overlay: expect.objectContaining({ + changes: [ + { + kind: "service-secret", + serviceId: "openconditions-ingest", + key: "NY_511_API_KEY", + action: "delete", + }, + ], + }), + }), ); expect(mockWriteAuditLog).toHaveBeenCalledWith( expect.objectContaining({ action: "service.credential.delete" }), @@ -914,6 +1007,7 @@ describe("service credentials", () => { const res = await app.inject({ method: "DELETE", url: "/admin/services/openconditions-ingest/credentials/..%2F..%2Fescaped", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, }); expect(res.statusCode).toBe(400); @@ -926,6 +1020,7 @@ describe("service credentials", () => { const res = await app.inject({ method: "DELETE", url: "/admin/services/openconditions-ingest/credentials/RATE_LIMIT_MAX", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, }); expect(res.statusCode).toBe(400); @@ -940,10 +1035,25 @@ describe("service credentials", () => { const res = await app.inject({ method: "DELETE", url: "/admin/services/openconditions-ingest/credentials/LEGACY_KEY", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, }); expect(res.statusCode).toBe(200); - expect(mockDeleteServiceSecret).toHaveBeenCalledWith("openconditions-ingest", "LEGACY_KEY"); + expect(mockDeleteServiceSecret).not.toHaveBeenCalled(); + expect(applyTrustedConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ + overlay: expect.objectContaining({ + changes: [ + { + kind: "service-secret", + serviceId: "openconditions-ingest", + key: "LEGACY_KEY", + action: "delete", + }, + ], + }), + }), + ); }); }); diff --git a/apps/api/src/routes/admin-overview.ts b/apps/api/src/routes/admin-overview.ts index 476f164f0..9d79d2e31 100644 --- a/apps/api/src/routes/admin-overview.ts +++ b/apps/api/src/routes/admin-overview.ts @@ -4,18 +4,18 @@ import { db } from "../db"; import { adminAuditLog, session, user } from "../db/schema"; import { getAllIntegrations } from "../integration-host"; import { listActivityJobs } from "../services/activity-jobs"; -import { isDockerAvailable } from "../services/admin-ops"; +import { isSelfHostedDeployment } from "../services/deployment-mode"; import { computeCredentialStatus, getIntegrationDisplayName, } from "../services/integration-credential-status"; import { getCachedHealthStatus } from "../services/integration-health"; -import { dockerComposePs, type PsEntry } from "../utils/docker-compose"; +import { readServiceInventory } from "../services/service-inventory"; import { registerIntegrationAdminRoute } from "./admin-integration-contract"; export async function adminOverviewRoute(app: FastifyInstance): Promise { registerIntegrationAdminRoute(app, "getAdminOverview", async () => { - const selfHosted = await isDockerAvailable(); + const selfHosted = isSelfHostedDeployment(); const now = new Date(); const twentyFourHoursAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000); @@ -26,7 +26,7 @@ export async function adminOverviewRoute(app: FastifyInstance): Promise { integrations, recentAudit, activityJobs, - dockerServices, + inventory, ] = await Promise.all([ db.select({ total: count() }).from(user), db.select({ total: count() }).from(user).where(eq(user.banned, true)), @@ -34,7 +34,7 @@ export async function adminOverviewRoute(app: FastifyInstance): Promise { Promise.resolve(getAllIntegrations()), db.select().from(adminAuditLog).orderBy(desc(adminAuditLog.createdAt)).limit(10), listActivityJobs({ limit: 10, offset: 0 }), - selfHosted ? dockerComposePs() : Promise.resolve([] as PsEntry[]), + selfHosted ? readServiceInventory() : Promise.resolve(null), ]); const totalUsers = userRow?.total ?? 0; @@ -55,19 +55,18 @@ export async function adminOverviewRoute(app: FastifyInstance): Promise { return cached?.status === "down"; }).length; - // Services summary (Docker only). The new `dockerComposePs` helper reports - // raw docker states; we collapse "exited"/"created"/"paused"/"not-running" - // into a single "stopped" bucket and treat "restarting" as "unhealthy". - const runningServices = dockerServices.filter((s) => s.state === "running").length; - const stoppedServices = dockerServices.filter( - (s) => s.state !== "running" && s.state !== "restarting", - ).length; + const dockerServices = + inventory?.services.map((s) => ({ service: s.id, state: s.status })) ?? []; + const runningServices = inventory?.summary.running ?? 0; + const stoppedServices = inventory?.summary.stopped ?? 0; + const unknownServices = inventory?.summary.unknown ?? 0; const unhealthyServices = dockerServices.filter((s) => s.state === "restarting").length; // System health const unhealthyCount = unhealthyIntegrations + unhealthyServices; let overallHealth: "pass" | "degraded" | "down" = "pass"; - if (unhealthyCount > 0) overallHealth = "degraded"; + if (unhealthyCount > 0 || unknownServices > 0 || inventory?.acquisition.state === "unavailable") + overallHealth = "degraded"; if ( (enabledIntegrations > 0 && unhealthyIntegrations > enabledIntegrations / 2) || (selfHosted && runningServices > 0 && unhealthyServices > runningServices) @@ -83,6 +82,14 @@ export async function adminOverviewRoute(app: FastifyInstance): Promise { target?: string; }> = []; + if (inventory?.acquisition.state === "unavailable") + attention.push({ + type: "control_plane_unavailable", + severity: "warning", + message: "Service status is unavailable. Check the operations agent.", + target: "/admin/services", + }); + // Missing credentials — flag integrations where every declared secret is // unresolved in the actual configuration cascade. const enabledIntegrationList = integrations.filter((i) => i.enabled); @@ -162,6 +169,7 @@ export async function adminOverviewRoute(app: FastifyInstance): Promise { ? { running: runningServices, stopped: stoppedServices, + unknown: unknownServices, unhealthy: unhealthyServices, } : null, diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index fdbc9cb7d..3148db6d1 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -19,6 +19,7 @@ import { DAWARICH_CREDENTIAL_MANAGED_ERROR, isProvisioningOwnedDawarichCredential, } from "../services/dawarich/credential-policy.js"; +import { hostControlConfigured, isSelfHostedDeployment } from "../services/deployment-mode"; import { createDirectAdminOpsKey, DIRECT_OPS_IDEMPOTENCY_HEADER, @@ -28,11 +29,12 @@ import { jobRunner } from "../services/job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client"; import { isSecretsConfigured } from "../services/secrets"; import { resolveServiceConfigWithSources } from "../services/service-config-resolver"; +import { readServiceInventory } from "../services/service-inventory"; import { getServiceRegistry } from "../services/service-registry"; import { listServiceSecrets } from "../services/service-secrets"; import { applyTrustedConfiguration } from "../services/trusted-config-operations"; import { writeAuditLog } from "../utils/audit-log"; -import { dockerComposeLogs, dockerComposePs } from "../utils/docker-compose"; +import { dockerComposeLogs } from "../utils/docker-compose"; import { maskSecretConfigValues } from "../utils/mask-config.js"; import { serviceActionLimit } from "../utils/rate-limit"; import { getAdminSession, requireAdmin } from "../utils/require-admin"; @@ -40,7 +42,7 @@ import { declareRouteAuth } from "../utils/route-auth.js"; import { getSecretFields, validateConfigBody } from "../utils/validate-config-body"; import { registerServiceAdminRoute } from "./admin-service-contract"; -const { getProvidedCapabilityNames, serviceConfigEnvPrefix } = coreServices; +const { serviceConfigEnvPrefix } = coreServices; const BULK_SERVICE_ACTIONS = new Set(["start", "stop", "restart", "update", "build"] as const); function hasOnlyBodyKeys(body: unknown, allowed: ReadonlySet): boolean { @@ -118,27 +120,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { // GET /admin/services — list all services from registry with docker ps status registerServiceAdminRoute(app, "listAdminServices", async () => { - const registry = getServiceRegistry(); - const ps = await dockerComposePs(); - const services = registry.list().map((s) => ({ - id: s.manifest.id, - name: s.manifest.name, - version: s.manifest.version, - description: s.manifest.description, - quality: s.manifest.quality, - // Normalised to bare strings for the admin UI; the structured-form - // metadata isn't surfaced anywhere yet (reserved slot for future - // runtime layers like region-aware routing). - provides: getProvidedCapabilityNames(s.manifest.provides), - consumes: s.manifest.consumes ?? [], - exposure: s.manifest.exposure, - enabled: s.enabled, - isBuiltIn: s.isBuiltIn, - status: ps.find((p) => p.service === s.manifest.id)?.state ?? "not-running", - })); - const running = services.filter((s) => s.status === "running").length; - const stopped = services.filter((s) => s.status !== "running").length; - return { services, summary: { running, stopped, total: services.length } }; + return readServiceInventory(); }); // GET /admin/services/:id — single service detail @@ -154,10 +136,17 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { reply.status(404); return { error: "Service not found" }; } - const ps = await dockerComposePs(); + const inventory = await readServiceInventory(); + const observed = inventory.services.find((row) => row.id === svc.manifest.id); + if (!observed) { + reply.status(503); + return { error: "Service registry changed; retry the request" }; + } return { ...svc, - status: ps.find((p) => p.service === svc?.manifest.id)?.state ?? "not-running", + status: observed.status, + actions: observed.actions, + acquisition: inventory.acquisition, }; }); @@ -723,18 +712,11 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { registerServiceAdminRoute(app, "getAdminDeployment", async () => { const { INFRA_DIR, isDockerAvailable } = await import("../services/admin-ops"); const dockerAvailable = await isDockerAvailable(); - // When the API runs inside app-api, docker-compose actions need the - // generated compose file's bind paths to resolve to real host paths. - // That only works when the operator sets OPENMAPX_HOST_DIR to the - // absolute host-side repo path and the app-api service.json bind-mounts - // `${OPENMAPX_HOST_DIR}:${OPENMAPX_HOST_DIR}`. Surface the state so the - // admin UI can warn on deploy actions that would silently no-op. const hostDir = process.env.OPENMAPX_HOST_DIR?.trim(); - const hostControlConfigured = Boolean(hostDir) && existsSync(hostDir as string); return { - selfHosted: dockerAvailable, + selfHosted: isSelfHostedDeployment(), dockerAvailable, - hostControlConfigured, + hostControlConfigured: hostControlConfigured(), hostDir: hostDir ?? null, // True once a configuration generation is applied: its render bakes the // applied services into app-api; a host-run API sees the generation. @@ -750,14 +732,18 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { // POST /admin/services/check — docker compose ps summary registerServiceAdminRoute(app, "checkAdminServices", async (request) => { const adminSession = getAdminSession(request); - const ps = await dockerComposePs(); + const inventory = await readServiceInventory(); await writeAuditLog({ actorId: adminSession.user.id, targetType: "service", action: "service.health_check", request, }); - return { statuses: ps, checkedAt: new Date().toISOString() }; + return { + acquisition: inventory.acquisition, + statuses: inventory.services.map((s) => ({ service: s.id, state: s.status })), + checkedAt: new Date().toISOString(), + }; }); // GET /admin/services/data — data inventory (OSM, builds, MOTIS health) diff --git a/apps/api/src/routes/admin-system.ts b/apps/api/src/routes/admin-system.ts index 07ee08319..56d2be91d 100644 --- a/apps/api/src/routes/admin-system.ts +++ b/apps/api/src/routes/admin-system.ts @@ -1,4 +1,5 @@ import type { FastifyInstance, FastifyReply } from "fastify"; +import { hostControlConfigured } from "../services/deployment-mode"; import { jobRunner } from "../services/job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client"; import { writeAuditLog } from "../utils/audit-log"; @@ -63,7 +64,7 @@ async function deploymentState(durableIdentity: string) { deployment: { dockerAvailable: inspection.dockerReachable, composeRendered: inspection.composeReady, - hostControlConfigured: inspection.maintenanceReady, + hostControlConfigured: hostControlConfigured(), maintenanceReady: inspection.maintenanceReady, }, inspection, @@ -73,7 +74,7 @@ async function deploymentState(durableIdentity: string) { deployment: { dockerAvailable: false, composeRendered: false, - hostControlConfigured: false, + hostControlConfigured: hostControlConfigured(), maintenanceReady: false, }, inspection: { release: {}, services: [] }, diff --git a/apps/api/src/services/__tests__/service-inventory.test.ts b/apps/api/src/services/__tests__/service-inventory.test.ts new file mode 100644 index 000000000..0c47d20f4 --- /dev/null +++ b/apps/api/src/services/__tests__/service-inventory.test.ts @@ -0,0 +1,52 @@ +import { SERVICE_ACTIONS } from "@openmapx/core/ops"; +import { afterEach, expect, it, vi } from "vitest"; +import { serviceFixture } from "../../routes/__tests__/helpers/service-admin-fixtures"; + +const inspect = vi.hoisted(() => vi.fn()); +const list = vi.hoisted(() => vi.fn()); +vi.mock("../ops-client", () => ({ + createApiOpsClient: () => ({}), + createDurableOpsKey: () => "read", + executeAndWait: inspect, +})); +vi.mock("../service-registry", () => ({ getServiceRegistry: () => ({ list }) })); +const { readServiceInventory } = await import("../service-inventory"); +const { hostControlConfigured, isSelfHostedDeployment } = await import("../deployment-mode"); +afterEach(() => vi.unstubAllEnvs()); +it("keeps known rows unknown and every action unavailable on acquisition failure", async () => { + list.mockReturnValue([serviceFixture("motis"), serviceFixture("valhalla")]); + inspect.mockRejectedValue(new Error("unavailable")); + const result = await readServiceInventory(); + expect(result.acquisition).toEqual({ state: "unavailable", reason: "ops_unavailable" }); + expect(result.summary).toEqual({ running: 0, stopped: 0, unknown: 2, total: 2 }); + for (const service of result.services) { + expect(service.status).toBe("unknown"); + expect(Object.keys(service.actions)).toHaveLength(SERVICE_ACTIONS.length); + expect(Object.values(service.actions).every((action) => !action.available)).toBe(true); + } +}); +it("distinguishes successful empty inventory from a failure and preserves unknown state", async () => { + list.mockReturnValue([serviceFixture("motis")]); + inspect.mockResolvedValue({ services: [] }); + expect(await readServiceInventory()).toMatchObject({ + acquisition: { state: "available" }, + summary: { stopped: 1, unknown: 0 }, + services: [{ status: "not-running" }], + }); + inspect.mockResolvedValue({ services: [{ serviceId: "motis", state: "unknown" }] }); + expect(await readServiceInventory()).toMatchObject({ + acquisition: { state: "available" }, + summary: { stopped: 0, unknown: 1 }, + services: [{ status: "unknown" }], + }); +}); +it("reports configured deployment intent independently of live agent acquisition or host paths", () => { + vi.stubEnv("OPS_AGENT_URL", ""); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", ""); + vi.stubEnv("OPENMAPX_APPLIED_SERVICES", ""); + expect(hostControlConfigured()).toBe(false); + vi.stubEnv("OPS_AGENT_URL", "http://ops-agent:3000"); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", "/unavailable/token"); + expect(hostControlConfigured()).toBe(true); + expect(isSelfHostedDeployment()).toBe(true); +}); diff --git a/apps/api/src/services/dawarich/__tests__/provisioning.test.ts b/apps/api/src/services/dawarich/__tests__/provisioning.test.ts index 2a0abb852..298c212d0 100644 --- a/apps/api/src/services/dawarich/__tests__/provisioning.test.ts +++ b/apps/api/src/services/dawarich/__tests__/provisioning.test.ts @@ -1115,3 +1115,22 @@ describe("rotateManagedDawarichOidcSecret", () => { expect(result.audit.hostname).toBe("history.example.test"); }); }); + +it("refuses provisioning before any mutation when runtime acquisition fails", async () => { + const h = createHarness(); + h.dependencies.getRuntimeState = vi.fn().mockRejectedValue(new Error("agent unavailable")); + await expect( + provisionManagedDawarich( + { + headers: HEADERS, + actorId: ACTOR, + controllerDomain: "maps.example.test", + publicHost: "timeline.example.test", + }, + h.dependencies, + ), + ).rejects.toThrow("agent unavailable"); + expect(h.createClient).not.toHaveBeenCalled(); + expect(h.setSecret).not.toHaveBeenCalled(); + expect(h.mergeConfig).not.toHaveBeenCalled(); +}); diff --git a/apps/api/src/services/dawarich/provisioning.ts b/apps/api/src/services/dawarich/provisioning.ts index 27cf1d3af..d69542c64 100644 --- a/apps/api/src/services/dawarich/provisioning.ts +++ b/apps/api/src/services/dawarich/provisioning.ts @@ -497,6 +497,7 @@ async function executeProvision( input: ManagedDawarichProvisioningInput, dependencies: ManagedDawarichProvisioningDependencies, ): Promise { + await dependencies.getRuntimeState(); const context = await resolveContext(input, dependencies); let clients = await matchingClients(dependencies, input.headers); if (clients.length > 1) { @@ -644,6 +645,7 @@ export async function rotateManagedDawarichOidcSecret( dependencies: ManagedDawarichProvisioningDependencies = defaultDependencies, ): Promise { return dependencies.withLock(async () => { + await dependencies.getRuntimeState(); const context = await resolveContext(input, dependencies); const clients = await matchingClients(dependencies, input.headers); if (clients.length !== 1) { @@ -708,6 +710,14 @@ async function getDefaultRuntimeState(): Promise { const installed = SERVICE_IDS.every((serviceId) => Boolean(registry.get(serviceId))); const selected = installed && SERVICE_IDS.every((serviceId) => registry.get(serviceId)?.enabled); const ps = await dockerComposePs(); + if ( + ps.some( + (row) => + SERVICE_IDS.includes(row.service as (typeof SERVICE_IDS)[number]) && + row.state === "unknown", + ) + ) + throw new ManagedDawarichProvisioningError("DAWARICH_PROVISIONING_FAILED"); const running = selected && SERVICE_IDS.every((serviceId) => diff --git a/apps/api/src/services/deployment-mode.ts b/apps/api/src/services/deployment-mode.ts new file mode 100644 index 000000000..1654b05bd --- /dev/null +++ b/apps/api/src/services/deployment-mode.ts @@ -0,0 +1,10 @@ +/** Deployment configuration describes intent, never permission or live host reachability. */ +export function hostControlConfigured(): boolean { + return Boolean(process.env.OPS_AGENT_URL?.trim() && process.env.OPS_AGENT_TOKEN_FILE?.trim()); +} +export function isSelfHostedDeployment(): boolean { + return ( + Boolean(process.env.OPS_AGENT_URL?.trim() || process.env.OPS_AGENT_TOKEN_FILE?.trim()) || + process.env.OPENMAPX_APPLIED_SERVICES !== undefined + ); +} diff --git a/apps/api/src/services/service-inventory.ts b/apps/api/src/services/service-inventory.ts new file mode 100644 index 000000000..f906b123c --- /dev/null +++ b/apps/api/src/services/service-inventory.ts @@ -0,0 +1,65 @@ +import type { AdminServiceStatus, AdminServices } from "@openmapx/core/api-contracts"; +import { type OpsResultFor, SERVICE_ACTIONS, serviceActionsSchema } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { getServiceRegistry } from "./service-registry"; + +const unavailableActions = () => + serviceActionsSchema.parse( + Object.fromEntries( + SERVICE_ACTIONS.map((action) => [ + action, + { available: false, execution: "direct", reason: "unavailable", workflow: null }, + ]), + ), + ); + +export async function readServiceInventory(): Promise { + const registered = getServiceRegistry().list(); + let inspection: OpsResultFor<"services.inspect"> | undefined; + try { + inspection = await executeAndWait( + createApiOpsClient(), + { kind: "services.inspect" }, + createDurableOpsKey("admin.services.inspect", "current"), + { signal: AbortSignal.timeout(15_000) }, + ); + } catch { + /* Preserve registry rows and unknown acquisition; never infer stopped. */ + } + const services = registered.map((service) => { + const observed = inspection?.services.find((row) => row.serviceId === service.manifest.id); + const status: AdminServiceStatus = !inspection + ? "unknown" + : !observed || observed.state === "stopped" + ? "not-running" + : observed.state; + return { + id: service.manifest.id, + name: service.manifest.name, + version: service.manifest.version, + description: service.manifest.description, + quality: service.manifest.quality, + provides: coreServices.getProvidedCapabilityNames(service.manifest.provides), + consumes: service.manifest.consumes ?? [], + exposure: service.manifest.exposure, + enabled: service.enabled, + isBuiltIn: service.isBuiltIn, + status, + actions: + status === "unknown" ? unavailableActions() : (observed?.actions ?? unavailableActions()), + }; + }); + return { + acquisition: inspection + ? { state: "available", reason: null } + : { state: "unavailable", reason: "ops_unavailable" }, + services, + summary: { + running: services.filter((s) => s.status === "running").length, + stopped: services.filter((s) => s.status !== "running" && s.status !== "unknown").length, + unknown: services.filter((s) => s.status === "unknown").length, + total: services.length, + }, + }; +} diff --git a/apps/api/src/utils/__tests__/docker-compose.test.ts b/apps/api/src/utils/__tests__/docker-compose.test.ts index e070e1455..5da6bb72a 100644 --- a/apps/api/src/utils/__tests__/docker-compose.test.ts +++ b/apps/api/src/utils/__tests__/docker-compose.test.ts @@ -47,6 +47,20 @@ describe("inspectDawarichProvisioning", () => { }); describe("typed Compose facade", () => { + it("does not report a stopped stack when control-plane acquisition fails", async () => { + const execute = vi.fn().mockRejectedValue(new Error("Agent unavailable")); + await expect(dockerComposePs({ client: clientWithExecute(execute) })).rejects.toThrow(); + }); + it("preserves an unfamiliar observed state as unknown", async () => { + const execute = vi.fn().mockResolvedValue({ + execution: "sync", + value: { services: [{ serviceId: "motis", state: "unknown" }] }, + }); + await expect(dockerComposePs({ client: clientWithExecute(execute) })).resolves.toEqual([ + { service: "motis", state: "unknown" }, + ]); + }); + it("does not submit follow work for an already disconnected response", async () => { const execute = vi.fn() as ApiOpsClient["execute"]; const out = Object.assign(new EventEmitter(), { @@ -215,7 +229,7 @@ describe("typed Compose facade", () => { delete process.env.OPS_AGENT_URL; delete process.env.OPS_AGENT_TOKEN_FILE; try { - await expect(dockerComposePs()).resolves.toEqual([]); + await expect(dockerComposePs()).rejects.toThrow(); } finally { if (previousUrl !== undefined) process.env.OPS_AGENT_URL = previousUrl; if (previousTokenFile !== undefined) process.env.OPS_AGENT_TOKEN_FILE = previousTokenFile; @@ -223,65 +237,22 @@ describe("typed Compose facade", () => { expect(execute).toHaveBeenCalledTimes(1); }); - it("waits on typed lifecycle work with the caller-owned stable key", async () => { - const operationKey = "opk1_abcdefghijklmnopqrstuvwxyzABCDEFGH123456789"; - const execute = vi.fn(async () => ({ - execution: "async" as const, - operationId: "job1_abcdefghijklmnop", - operationKey, - kind: "service.restart" as const, - state: "queued" as const, - })) as ApiOpsClient["execute"]; - const status = vi.fn(async () => ({ - version: 1 as const, - operationId: "job1_abcdefghijklmnop", - operationKey, - kind: "service.restart" as const, - resourceId: "redis", - state: "succeeded" as const, - submittedAt: "2026-08-23T10:00:00.000Z", - updatedAt: "2026-08-23T10:00:01.000Z", - result: { changed: true }, - })) as ApiOpsClient["status"]; - const client = { ...clientWithExecute(execute), status }; - - await expect( - dockerComposeAction("redis", "restart", { client, operationKey }), - ).resolves.toEqual({ exitCode: 0, stdout: "", stderr: "" }); - expect(execute).toHaveBeenCalledWith( - { kind: "service.restart", serviceId: "redis" }, - { operationKey }, - ); - }); - - it("maps ordinary and isolated recreation to two exact typed effects", async () => { - const operationKey = "opk1_abcdefghijklmnopqrstuvwxyzABCDEFGH123456789"; - const execute = vi.fn().mockResolvedValue({ - execution: "async", - operationId: "job1_abcdefghijklmnop", - operationKey, - kind: "service.recreate", - state: "succeeded", - }); - const client = clientWithExecute(execute as ApiOpsClient["execute"]); - await dockerComposeAction("redis", "recreate", { client, operationKey }); - expect(execute).toHaveBeenLastCalledWith( - { kind: "service.recreate", serviceId: "redis" }, - { operationKey }, - ); - execute.mockResolvedValueOnce({ - execution: "async", - operationId: "job1_abcdefghijklmnop", - operationKey, - kind: "service.recreateIsolated", - state: "succeeded", - } as never); - await dockerComposeAction("redis", "recreate-isolated", { client, operationKey }); - expect(execute).toHaveBeenLastCalledWith( - { kind: "service.recreateIsolated", serviceId: "redis" }, - { operationKey }, - ); - }); + it.each(["restart", "recreate", "recreate-isolated"] as const)( + "requires original administrative authority for %s", + async (action) => { + const execute = vi.fn() as ApiOpsClient["execute"]; + const result = await dockerComposeAction("motis", action, { + client: clientWithExecute(execute), + operationKey: "opk1_abcdefghijklmnopqrstuvwxyzABCDEFGH123456789", + expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + }); + expect(result).toMatchObject({ + exitCode: 1, + stderr: "Operations request failed (control_upgrade_required).", + }); + expect(execute).not.toHaveBeenCalled(); + }, + ); it("refuses stack.stop locally with fixed operator recovery guidance", async () => { const execute = vi.fn() as ApiOpsClient["execute"]; diff --git a/apps/api/src/utils/docker-compose.ts b/apps/api/src/utils/docker-compose.ts index ba2064e80..ce4ea24cc 100644 --- a/apps/api/src/utils/docker-compose.ts +++ b/apps/api/src/utils/docker-compose.ts @@ -12,7 +12,7 @@ import { export interface PsEntry { service: string; - state: "running" | "exited" | "restarting" | "created" | "paused" | "not-running"; + state: "running" | "exited" | "restarting" | "created" | "paused" | "not-running" | "unknown"; } interface OpsDependencies { @@ -36,7 +36,7 @@ function client(dependencies: OpsDependencies): ApiOpsClient { function composeState( state: OpsResultFor<"stack.status">["services"][number]["state"], ): PsEntry["state"] { - if (state === "stopped" || state === "unknown") return "not-running"; + if (state === "stopped") return "not-running"; return state; } @@ -49,16 +49,12 @@ export async function dockerStatus( } export async function dockerComposePs(dependencies: OpsDependencies = {}): Promise { - try { - const submitted = await client(dependencies).execute({ kind: "stack.status" }, {}); - if (submitted.execution !== "sync") return []; - return submitted.value.services.map((service) => ({ - service: service.serviceId, - state: composeState(service.state), - })); - } catch { - return []; - } + const submitted = await client(dependencies).execute({ kind: "stack.status" }, {}); + if (submitted.execution !== "sync") throw new ApiOpsError("runtime"); + return submitted.value.services.map((service) => ({ + service: service.serviceId, + state: composeState(service.state), + })); } export async function inspectDawarichProvisioning( diff --git a/apps/web/src/app/admin/layout.test.tsx b/apps/web/src/app/admin/layout.test.tsx index cac81889b..6a20dd79c 100644 --- a/apps/web/src/app/admin/layout.test.tsx +++ b/apps/web/src/app/admin/layout.test.tsx @@ -21,6 +21,17 @@ afterEach(() => { vi.unstubAllGlobals(); }); describe("admin layout deployment boundary", () => { + it("retains unknown deployment for an authenticated full admin during an outage", async () => { + vi.stubGlobal("fetch", async (url: string) => + new URL(url).pathname === "/api/me" + ? Response.json({ user: { name: "Admin", email: "admin@example.test", role: "admin" } }) + : Response.json({ error: "unavailable" }, { status: 503 }), + ); + const rendered = await request.run("session=admin", () => + AdminRootLayout({ children: "child" }), + ); + expect(rendered.props.selfHosted).toBeNull(); + }); it("keeps me authentication and privacy_admin deployment skip", async () => { const paths: string[] = []; vi.stubGlobal("fetch", async (url: string, init: RequestInit) => { diff --git a/apps/web/src/components/admin/AdminLayout.tsx b/apps/web/src/components/admin/AdminLayout.tsx index 385634bfb..f06a1f934 100644 --- a/apps/web/src/components/admin/AdminLayout.tsx +++ b/apps/web/src/components/admin/AdminLayout.tsx @@ -20,7 +20,7 @@ interface AdminUser { interface AdminLayoutProps { children: React.ReactNode; user: AdminUser; - selfHosted?: boolean; + selfHosted?: boolean | null; } export function AdminLayout({ children, user, selfHosted = false }: AdminLayoutProps) { diff --git a/apps/web/src/components/admin/AdminSidebar.tsx b/apps/web/src/components/admin/AdminSidebar.tsx index f0707f8df..5bbf52093 100644 --- a/apps/web/src/components/admin/AdminSidebar.tsx +++ b/apps/web/src/components/admin/AdminSidebar.tsx @@ -175,7 +175,7 @@ const NAV_GROUPS = [ interface AdminSidebarProps { open: boolean; onClose: () => void; - selfHosted?: boolean; + selfHosted?: boolean | null; role?: string; } @@ -284,7 +284,7 @@ export function AdminSidebar({ const navItems = BASE_NAV_ITEMS.filter( (item) => - (!item.selfHostedOnly || selfHosted) && + (!item.selfHostedOnly || selfHosted !== false) && (!item.privacyOnly || role === "admin" || role === "privacy_admin"), ).filter((item) => role === "admin" || item.privacyOnly); @@ -298,7 +298,7 @@ export function AdminSidebar({ return pathname === item.href || pathname.startsWith(`${item.href}/`); }; - const servicesExpanded = selfHosted && pathname.startsWith("/admin/services"); + const servicesExpanded = selfHosted !== false && pathname.startsWith("/admin/services"); const drawerContent = ( <> @@ -353,7 +353,7 @@ export function AdminSidebar({ {groupItems.map((item) => { const hasActiveChild = item.href === "/admin/services" && - selfHosted && + selfHosted !== false && SERVICES_SUB_ITEMS.some(isSubActive); return ( @@ -367,7 +367,7 @@ export function AdminSidebar({ : item.label } /> - {item.href === "/admin/services" && selfHosted && ( + {item.href === "/admin/services" && selfHosted !== false && ( {SERVICES_SUB_ITEMS.map((sub) => ( diff --git a/apps/web/src/lib/admin/serverDeployment.test.tsx b/apps/web/src/lib/admin/serverDeployment.test.tsx index d22936c06..ba670be72 100644 --- a/apps/web/src/lib/admin/serverDeployment.test.tsx +++ b/apps/web/src/lib/admin/serverDeployment.test.tsx @@ -76,9 +76,9 @@ describe("request-scoped deployment", () => { Response.json({ error: "denied" }, { status: 403 }), new Response("invalid", { headers: { "content-type": "application/json" } }), Response.json(deployment, { headers: { "content-type": "text/plain" } }), - ])("fails closed for malformed or failed deployment", async (response) => { + ])("preserves unknown for malformed or failed deployment", async (response) => { vi.stubGlobal("fetch", async () => response.clone()); - expect(await getSelfHosted("session=one")).toBe(false); + expect(await getSelfHosted("session=one")).toBe(null); }); it("aborts stalled body at 30 seconds, cancels/unlocks it and clears timers", async () => { vi.useFakeTimers(); @@ -101,14 +101,14 @@ describe("request-scoped deployment", () => { expect(settled).toBe(false); expect(canceled).toBe(false); await vi.advanceTimersByTimeAsync(1); - expect(await pending).toBe(false); + expect(await pending).toBe(null); expect(canceled).toBe(true); expect(stream.locked).toBe(false); expect((vi as typeof vi & { getTimerCount(): number }).getTimerCount()).toBe(0); }); }); -it("returns false when the deployment headers fail or time out", async () => { +it("returns unknown when the deployment headers fail or time out", async () => { vi.useFakeTimers(); let signal: AbortSignal | null | undefined; vi.stubGlobal("fetch", (_url: string, init: RequestInit) => { @@ -123,11 +123,11 @@ it("returns false when the deployment headers fail or time out", async () => { await vi.advanceTimersByTimeAsync(29_999); expect(signal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(1); - expect(await pending).toBe(false); + expect(await pending).toBe(null); expect(signal?.aborted).toBe(true); expect((vi as typeof vi & { getTimerCount(): number }).getTimerCount()).toBe(0); vi.stubGlobal("fetch", async () => { throw new Error("synthetic acquisition failure"); }); - expect(await getSelfHosted("session=one")).toBe(false); + expect(await getSelfHosted("session=one")).toBe(null); }); diff --git a/apps/web/src/lib/admin/serverDeployment.ts b/apps/web/src/lib/admin/serverDeployment.ts index 601fa7e4c..6a28b951a 100644 --- a/apps/web/src/lib/admin/serverDeployment.ts +++ b/apps/web/src/lib/admin/serverDeployment.ts @@ -4,7 +4,7 @@ import { createApiClient, serverApiUrl } from "@openmapx/core/server-api"; const DEPLOYMENT_READ_TIMEOUT_MS = 30_000; /** Request cookies belong only to this request's client, never the browser singleton. */ -export async function getSelfHosted(cookieHeader: string): Promise { +export async function getSelfHosted(cookieHeader: string): Promise { const client = createApiClient({ baseUrl: serverApiUrl(), credentials: "omit" }); try { const deployment = unwrapAdminResponse( @@ -17,6 +17,6 @@ export async function getSelfHosted(cookieHeader: string): Promise { ); return deployment.selfHosted; } catch { - return false; + return null; } } diff --git a/packages/core/src/api/contracts/integration-admin/overview.ts b/packages/core/src/api/contracts/integration-admin/overview.ts index 82b1f6063..55e9a72ec 100644 --- a/packages/core/src/api/contracts/integration-admin/overview.ts +++ b/packages/core/src/api/contracts/integration-admin/overview.ts @@ -34,6 +34,7 @@ export const adminOverviewSchema = z.strictObject({ .strictObject({ running: adminCountSchema, stopped: adminCountSchema, + unknown: adminCountSchema, unhealthy: adminCountSchema, }) .nullable(), diff --git a/packages/core/src/api/contracts/service-admin.test.ts b/packages/core/src/api/contracts/service-admin.test.ts index 01751d9c8..a661006b1 100644 --- a/packages/core/src/api/contracts/service-admin.test.ts +++ b/packages/core/src/api/contracts/service-admin.test.ts @@ -2,10 +2,18 @@ import { createRequire } from "node:module"; import { serviceManifestSchema } from "@openmapx/extension-contracts/service"; import { describe, expect, it } from "vitest"; import { OPS_RESULT_SCHEMAS } from "../../ops/contract"; +import { SERVICE_ACTIONS } from "../../ops/service-results"; import { routeJsonSchema } from "../schema"; import type { AdminOperation } from "./admin"; import * as contracts from "./index"; +const acquisition = { state: "available", reason: null }; +const actions = Object.fromEntries( + SERVICE_ACTIONS.map((action) => [ + action, + { available: false, execution: "direct", reason: "unavailable", workflow: null }, + ]), +); const at = "2026-10-21T10:00:00.000Z"; const inventory = [ ["listAdminServices", "GET", "/services"], @@ -202,6 +210,8 @@ describe("canonical complete service administration contracts", () => { }); expect(contracts.adminServiceManifestSchema.parse(raw)).toEqual(raw); const detail = { + acquisition, + actions, manifest: raw, directory: "/synthetic/service", isBuiltIn: false, @@ -268,6 +278,7 @@ describe("canonical complete service administration contracts", () => { }); it("keeps list normalized names and config JSON/source fields complete", () => { const summary = { + actions, id: raw.id, name: raw.name, version: raw.version, @@ -279,7 +290,11 @@ describe("canonical complete service administration contracts", () => { isBuiltIn: false, status: "not-running", }; - const list = { services: [summary], summary: { running: 0, stopped: 1, total: 1 } }; + const list = { + acquisition, + services: [summary], + summary: { running: 0, stopped: 1, unknown: 0, total: 1 }, + }; expect(response("listAdminServices", 200, list)).toEqual(list); const config = { schema: raw.configSchema, @@ -602,10 +617,11 @@ describe("canonical complete service administration contracts", () => { expect(response("rollbackAdminSharedMobility", 200, policy)).toEqual(policy); expect( response("checkAdminServices", 200, { + acquisition, statuses: [{ service: "sample", state: "paused" }], checkedAt: at, }), - ).toEqual({ statuses: [{ service: "sample", state: "paused" }], checkedAt: at }); + ).toEqual({ acquisition, statuses: [{ service: "sample", state: "paused" }], checkedAt: at }); const deployment = { selfHosted: true, dockerAvailable: false, diff --git a/packages/core/src/api/contracts/service-admin/services.ts b/packages/core/src/api/contracts/service-admin/services.ts index 4e86877e5..cc3e17b4c 100644 --- a/packages/core/src/api/contracts/service-admin/services.ts +++ b/packages/core/src/api/contracts/service-admin/services.ts @@ -2,6 +2,7 @@ import { credentialSetupSchema } from "@openmapx/extension-contracts/integration import { serviceManifestSchema } from "@openmapx/extension-contracts/service"; import z from "zod/v4"; +import { serviceActionsSchema } from "../../../ops/service-results"; import { adminCountSchema, adminJsonValueSchema, adminTimestampSchema } from "../admin"; import { adminJsonWire, preserveManifestJson } from "../manifest-json"; import { @@ -55,6 +56,10 @@ export const adminServiceStatusSchema = z.enum([ "not-running", "unknown", ]); +export const adminServiceAcquisitionSchema = z.union([ + z.strictObject({ state: z.literal("available"), reason: z.null() }), + z.strictObject({ state: z.literal("unavailable"), reason: z.literal("ops_unavailable") }), +]); export const adminServiceSummarySchema = z.strictObject({ id: serviceManifestSchema.shape.id, name: serviceManifestSchema.shape.name, @@ -67,21 +72,26 @@ export const adminServiceSummarySchema = z.strictObject({ enabled: z.boolean(), isBuiltIn: z.boolean(), status: adminServiceStatusSchema, + actions: serviceActionsSchema, }); export const adminServicesSchema = z.strictObject({ + acquisition: adminServiceAcquisitionSchema, services: z.array(adminServiceSummarySchema), summary: z.strictObject({ running: adminCountSchema, stopped: adminCountSchema, + unknown: adminCountSchema, total: adminCountSchema, }), }); export const adminServiceDetailSchema = z.strictObject({ + acquisition: adminServiceAcquisitionSchema, manifest: adminServiceManifestSchema, directory: z.string(), isBuiltIn: z.boolean(), enabled: z.boolean(), status: adminServiceStatusSchema, + actions: serviceActionsSchema, }); export const adminServiceConfigSourceSchema = z.enum(["default", "database", "env"]); export const adminServiceResolvedConfigSchema = z.record( @@ -153,6 +163,7 @@ export const adminServicesBulkActionSchema = z.union([ ), ]); export const adminServicesCheckedSchema = z.strictObject({ + acquisition: adminServiceAcquisitionSchema, statuses: z.array(z.strictObject({ service: z.string(), state: adminServiceStatusSchema })), checkedAt: adminTimestampSchema, }); diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminOverview200Services.ts b/packages/core/src/api/generated/admin/integration-models/getAdminOverview200Services.ts index 9c2115482..6fc53245b 100644 --- a/packages/core/src/api/generated/admin/integration-models/getAdminOverview200Services.ts +++ b/packages/core/src/api/generated/admin/integration-models/getAdminOverview200Services.ts @@ -12,6 +12,11 @@ export type GetAdminOverview200Services = { * @maximum 9007199254740991 */ stopped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + unknown: number; /** * @minimum 0 * @maximum 9007199254740991 diff --git a/packages/core/src/api/generated/admin/service-models/checkAdminServices200.ts b/packages/core/src/api/generated/admin/service-models/checkAdminServices200.ts index cb3c5e856..311ddc365 100644 --- a/packages/core/src/api/generated/admin/service-models/checkAdminServices200.ts +++ b/packages/core/src/api/generated/admin/service-models/checkAdminServices200.ts @@ -1,8 +1,10 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { CheckAdminServices200Acquisition } from "./checkAdminServices200Acquisition"; import type { CheckAdminServices200StatusesItem } from "./checkAdminServices200StatusesItem"; export type CheckAdminServices200 = { + acquisition: CheckAdminServices200Acquisition; statuses: CheckAdminServices200StatusesItem[]; /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ checkedAt: string; diff --git a/packages/core/src/api/generated/admin/service-models/checkAdminServices200Acquisition.ts b/packages/core/src/api/generated/admin/service-models/checkAdminServices200Acquisition.ts new file mode 100644 index 000000000..6edcc4c10 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/checkAdminServices200Acquisition.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CheckAdminServices200Acquisition = + | { + state: "available"; + /** @nullable */ + reason: null; + } + | { + state: "unavailable"; + reason: "ops_unavailable"; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200.ts index 029c00443..0cb4295f1 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminService200.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200.ts @@ -1,12 +1,16 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200Acquisition } from "./getAdminService200Acquisition"; +import type { GetAdminService200Actions } from "./getAdminService200Actions"; import type { GetAdminService200Manifest } from "./getAdminService200Manifest"; import type { GetAdminService200Status } from "./getAdminService200Status"; export type GetAdminService200 = { + acquisition: GetAdminService200Acquisition; manifest: GetAdminService200Manifest; directory: string; isBuiltIn: boolean; enabled: boolean; status: GetAdminService200Status; + actions: GetAdminService200Actions; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200Acquisition.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200Acquisition.ts new file mode 100644 index 000000000..26cd33b03 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200Acquisition.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200Acquisition = + | { + state: "available"; + /** @nullable */ + reason: null; + } + | { + state: "unavailable"; + reason: "ops_unavailable"; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200Actions.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200Actions.ts new file mode 100644 index 000000000..6d3dbcb02 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200Actions.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsApply } from "./getAdminService200ActionsApply"; +import type { GetAdminService200ActionsBuild } from "./getAdminService200ActionsBuild"; +import type { GetAdminService200ActionsLogs } from "./getAdminService200ActionsLogs"; +import type { GetAdminService200ActionsPull } from "./getAdminService200ActionsPull"; +import type { GetAdminService200ActionsRestart } from "./getAdminService200ActionsRestart"; +import type { GetAdminService200ActionsStart } from "./getAdminService200ActionsStart"; +import type { GetAdminService200ActionsStop } from "./getAdminService200ActionsStop"; +import type { GetAdminService200ActionsUpdate } from "./getAdminService200ActionsUpdate"; + +export type GetAdminService200Actions = { + start: GetAdminService200ActionsStart; + stop: GetAdminService200ActionsStop; + restart: GetAdminService200ActionsRestart; + update: GetAdminService200ActionsUpdate; + build: GetAdminService200ActionsBuild; + logs: GetAdminService200ActionsLogs; + pull: GetAdminService200ActionsPull; + apply: GetAdminService200ActionsApply; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApply.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApply.ts new file mode 100644 index 000000000..91418e261 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApply.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsApplyExecution } from "./getAdminService200ActionsApplyExecution"; +import type { GetAdminService200ActionsApplyReason } from "./getAdminService200ActionsApplyReason"; +import type { GetAdminService200ActionsApplyWorkflow } from "./getAdminService200ActionsApplyWorkflow"; + +export type GetAdminService200ActionsApply = { + available: boolean; + execution: GetAdminService200ActionsApplyExecution; + reason: GetAdminService200ActionsApplyReason; + workflow: GetAdminService200ActionsApplyWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyExecution.ts new file mode 100644 index 000000000..73ec3ed74 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsApplyExecution = + (typeof GetAdminService200ActionsApplyExecution)[keyof typeof GetAdminService200ActionsApplyExecution]; + +export const GetAdminService200ActionsApplyExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyReason.ts new file mode 100644 index 000000000..c1087b01a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsApplyReason = + | (typeof GetAdminService200ActionsApplyReason)[keyof typeof GetAdminService200ActionsApplyReason] + | null; + +export const GetAdminService200ActionsApplyReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyWorkflow.ts new file mode 100644 index 000000000..58711223b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsApplyWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsApplyWorkflow = + | (typeof GetAdminService200ActionsApplyWorkflow)[keyof typeof GetAdminService200ActionsApplyWorkflow] + | null; + +export const GetAdminService200ActionsApplyWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuild.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuild.ts new file mode 100644 index 000000000..b967beb99 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuild.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsBuildExecution } from "./getAdminService200ActionsBuildExecution"; +import type { GetAdminService200ActionsBuildReason } from "./getAdminService200ActionsBuildReason"; +import type { GetAdminService200ActionsBuildWorkflow } from "./getAdminService200ActionsBuildWorkflow"; + +export type GetAdminService200ActionsBuild = { + available: boolean; + execution: GetAdminService200ActionsBuildExecution; + reason: GetAdminService200ActionsBuildReason; + workflow: GetAdminService200ActionsBuildWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildExecution.ts new file mode 100644 index 000000000..88d63048c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsBuildExecution = + (typeof GetAdminService200ActionsBuildExecution)[keyof typeof GetAdminService200ActionsBuildExecution]; + +export const GetAdminService200ActionsBuildExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildReason.ts new file mode 100644 index 000000000..1bf45286f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsBuildReason = + | (typeof GetAdminService200ActionsBuildReason)[keyof typeof GetAdminService200ActionsBuildReason] + | null; + +export const GetAdminService200ActionsBuildReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildWorkflow.ts new file mode 100644 index 000000000..a1c71c9af --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsBuildWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsBuildWorkflow = + | (typeof GetAdminService200ActionsBuildWorkflow)[keyof typeof GetAdminService200ActionsBuildWorkflow] + | null; + +export const GetAdminService200ActionsBuildWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogs.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogs.ts new file mode 100644 index 000000000..45a37afa5 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogs.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsLogsExecution } from "./getAdminService200ActionsLogsExecution"; +import type { GetAdminService200ActionsLogsReason } from "./getAdminService200ActionsLogsReason"; +import type { GetAdminService200ActionsLogsWorkflow } from "./getAdminService200ActionsLogsWorkflow"; + +export type GetAdminService200ActionsLogs = { + available: boolean; + execution: GetAdminService200ActionsLogsExecution; + reason: GetAdminService200ActionsLogsReason; + workflow: GetAdminService200ActionsLogsWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsExecution.ts new file mode 100644 index 000000000..62cc2e3ae --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsLogsExecution = + (typeof GetAdminService200ActionsLogsExecution)[keyof typeof GetAdminService200ActionsLogsExecution]; + +export const GetAdminService200ActionsLogsExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsReason.ts new file mode 100644 index 000000000..439803c2f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsLogsReason = + | (typeof GetAdminService200ActionsLogsReason)[keyof typeof GetAdminService200ActionsLogsReason] + | null; + +export const GetAdminService200ActionsLogsReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsWorkflow.ts new file mode 100644 index 000000000..98f164253 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsLogsWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsLogsWorkflow = + | (typeof GetAdminService200ActionsLogsWorkflow)[keyof typeof GetAdminService200ActionsLogsWorkflow] + | null; + +export const GetAdminService200ActionsLogsWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPull.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPull.ts new file mode 100644 index 000000000..4d0f51107 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPull.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsPullExecution } from "./getAdminService200ActionsPullExecution"; +import type { GetAdminService200ActionsPullReason } from "./getAdminService200ActionsPullReason"; +import type { GetAdminService200ActionsPullWorkflow } from "./getAdminService200ActionsPullWorkflow"; + +export type GetAdminService200ActionsPull = { + available: boolean; + execution: GetAdminService200ActionsPullExecution; + reason: GetAdminService200ActionsPullReason; + workflow: GetAdminService200ActionsPullWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullExecution.ts new file mode 100644 index 000000000..16cb8e4ee --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsPullExecution = + (typeof GetAdminService200ActionsPullExecution)[keyof typeof GetAdminService200ActionsPullExecution]; + +export const GetAdminService200ActionsPullExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullReason.ts new file mode 100644 index 000000000..cd66dfdf4 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsPullReason = + | (typeof GetAdminService200ActionsPullReason)[keyof typeof GetAdminService200ActionsPullReason] + | null; + +export const GetAdminService200ActionsPullReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullWorkflow.ts new file mode 100644 index 000000000..ca93a9506 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsPullWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsPullWorkflow = + | (typeof GetAdminService200ActionsPullWorkflow)[keyof typeof GetAdminService200ActionsPullWorkflow] + | null; + +export const GetAdminService200ActionsPullWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestart.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestart.ts new file mode 100644 index 000000000..89cb054e0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestart.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsRestartExecution } from "./getAdminService200ActionsRestartExecution"; +import type { GetAdminService200ActionsRestartReason } from "./getAdminService200ActionsRestartReason"; +import type { GetAdminService200ActionsRestartWorkflow } from "./getAdminService200ActionsRestartWorkflow"; + +export type GetAdminService200ActionsRestart = { + available: boolean; + execution: GetAdminService200ActionsRestartExecution; + reason: GetAdminService200ActionsRestartReason; + workflow: GetAdminService200ActionsRestartWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartExecution.ts new file mode 100644 index 000000000..cd3b4fc3b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsRestartExecution = + (typeof GetAdminService200ActionsRestartExecution)[keyof typeof GetAdminService200ActionsRestartExecution]; + +export const GetAdminService200ActionsRestartExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartReason.ts new file mode 100644 index 000000000..0fb068bc7 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsRestartReason = + | (typeof GetAdminService200ActionsRestartReason)[keyof typeof GetAdminService200ActionsRestartReason] + | null; + +export const GetAdminService200ActionsRestartReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartWorkflow.ts new file mode 100644 index 000000000..05c460b78 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsRestartWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsRestartWorkflow = + | (typeof GetAdminService200ActionsRestartWorkflow)[keyof typeof GetAdminService200ActionsRestartWorkflow] + | null; + +export const GetAdminService200ActionsRestartWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStart.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStart.ts new file mode 100644 index 000000000..361adbcb3 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStart.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsStartExecution } from "./getAdminService200ActionsStartExecution"; +import type { GetAdminService200ActionsStartReason } from "./getAdminService200ActionsStartReason"; +import type { GetAdminService200ActionsStartWorkflow } from "./getAdminService200ActionsStartWorkflow"; + +export type GetAdminService200ActionsStart = { + available: boolean; + execution: GetAdminService200ActionsStartExecution; + reason: GetAdminService200ActionsStartReason; + workflow: GetAdminService200ActionsStartWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartExecution.ts new file mode 100644 index 000000000..84cfdf8b3 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsStartExecution = + (typeof GetAdminService200ActionsStartExecution)[keyof typeof GetAdminService200ActionsStartExecution]; + +export const GetAdminService200ActionsStartExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartReason.ts new file mode 100644 index 000000000..1c6e98153 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsStartReason = + | (typeof GetAdminService200ActionsStartReason)[keyof typeof GetAdminService200ActionsStartReason] + | null; + +export const GetAdminService200ActionsStartReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartWorkflow.ts new file mode 100644 index 000000000..b859d6f39 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStartWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsStartWorkflow = + | (typeof GetAdminService200ActionsStartWorkflow)[keyof typeof GetAdminService200ActionsStartWorkflow] + | null; + +export const GetAdminService200ActionsStartWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStop.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStop.ts new file mode 100644 index 000000000..fa6ae09ff --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStop.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsStopExecution } from "./getAdminService200ActionsStopExecution"; +import type { GetAdminService200ActionsStopReason } from "./getAdminService200ActionsStopReason"; +import type { GetAdminService200ActionsStopWorkflow } from "./getAdminService200ActionsStopWorkflow"; + +export type GetAdminService200ActionsStop = { + available: boolean; + execution: GetAdminService200ActionsStopExecution; + reason: GetAdminService200ActionsStopReason; + workflow: GetAdminService200ActionsStopWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopExecution.ts new file mode 100644 index 000000000..b1a6e6674 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsStopExecution = + (typeof GetAdminService200ActionsStopExecution)[keyof typeof GetAdminService200ActionsStopExecution]; + +export const GetAdminService200ActionsStopExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopReason.ts new file mode 100644 index 000000000..6a024b79e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsStopReason = + | (typeof GetAdminService200ActionsStopReason)[keyof typeof GetAdminService200ActionsStopReason] + | null; + +export const GetAdminService200ActionsStopReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopWorkflow.ts new file mode 100644 index 000000000..05c13ef3c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsStopWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsStopWorkflow = + | (typeof GetAdminService200ActionsStopWorkflow)[keyof typeof GetAdminService200ActionsStopWorkflow] + | null; + +export const GetAdminService200ActionsStopWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdate.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdate.ts new file mode 100644 index 000000000..c0f115582 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdate.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminService200ActionsUpdateExecution } from "./getAdminService200ActionsUpdateExecution"; +import type { GetAdminService200ActionsUpdateReason } from "./getAdminService200ActionsUpdateReason"; +import type { GetAdminService200ActionsUpdateWorkflow } from "./getAdminService200ActionsUpdateWorkflow"; + +export type GetAdminService200ActionsUpdate = { + available: boolean; + execution: GetAdminService200ActionsUpdateExecution; + reason: GetAdminService200ActionsUpdateReason; + workflow: GetAdminService200ActionsUpdateWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateExecution.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateExecution.ts new file mode 100644 index 000000000..04e74430c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsUpdateExecution = + (typeof GetAdminService200ActionsUpdateExecution)[keyof typeof GetAdminService200ActionsUpdateExecution]; + +export const GetAdminService200ActionsUpdateExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateReason.ts new file mode 100644 index 000000000..9bd1c4d0b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsUpdateReason = + | (typeof GetAdminService200ActionsUpdateReason)[keyof typeof GetAdminService200ActionsUpdateReason] + | null; + +export const GetAdminService200ActionsUpdateReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateWorkflow.ts b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateWorkflow.ts new file mode 100644 index 000000000..4926ca5f0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminService200ActionsUpdateWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminService200ActionsUpdateWorkflow = + | (typeof GetAdminService200ActionsUpdateWorkflow)[keyof typeof GetAdminService200ActionsUpdateWorkflow] + | null; + +export const GetAdminService200ActionsUpdateWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index 20c8173aa..b3f63abf2 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -9,6 +9,7 @@ export * from "./adminPreviewAdminOperationBodySchema0"; export * from "./adminRunAdminOperationBodySchema0"; export * from "./adminUpdateAdminServiceConfigBodySchema0"; export * from "./checkAdminServices200"; +export * from "./checkAdminServices200Acquisition"; export * from "./checkAdminServices200StatusesItem"; export * from "./checkAdminServices200StatusesItemState"; export * from "./checkAdminServices400"; @@ -85,6 +86,40 @@ export * from "./getAdminDeployment415"; export * from "./getAdminDeployment429"; export * from "./getAdminDeployment500"; export * from "./getAdminService200"; +export * from "./getAdminService200Acquisition"; +export * from "./getAdminService200Actions"; +export * from "./getAdminService200ActionsApply"; +export * from "./getAdminService200ActionsApplyExecution"; +export * from "./getAdminService200ActionsApplyReason"; +export * from "./getAdminService200ActionsApplyWorkflow"; +export * from "./getAdminService200ActionsBuild"; +export * from "./getAdminService200ActionsBuildExecution"; +export * from "./getAdminService200ActionsBuildReason"; +export * from "./getAdminService200ActionsBuildWorkflow"; +export * from "./getAdminService200ActionsLogs"; +export * from "./getAdminService200ActionsLogsExecution"; +export * from "./getAdminService200ActionsLogsReason"; +export * from "./getAdminService200ActionsLogsWorkflow"; +export * from "./getAdminService200ActionsPull"; +export * from "./getAdminService200ActionsPullExecution"; +export * from "./getAdminService200ActionsPullReason"; +export * from "./getAdminService200ActionsPullWorkflow"; +export * from "./getAdminService200ActionsRestart"; +export * from "./getAdminService200ActionsRestartExecution"; +export * from "./getAdminService200ActionsRestartReason"; +export * from "./getAdminService200ActionsRestartWorkflow"; +export * from "./getAdminService200ActionsStart"; +export * from "./getAdminService200ActionsStartExecution"; +export * from "./getAdminService200ActionsStartReason"; +export * from "./getAdminService200ActionsStartWorkflow"; +export * from "./getAdminService200ActionsStop"; +export * from "./getAdminService200ActionsStopExecution"; +export * from "./getAdminService200ActionsStopReason"; +export * from "./getAdminService200ActionsStopWorkflow"; +export * from "./getAdminService200ActionsUpdate"; +export * from "./getAdminService200ActionsUpdateExecution"; +export * from "./getAdminService200ActionsUpdateReason"; +export * from "./getAdminService200ActionsUpdateWorkflow"; export * from "./getAdminService200Manifest"; export * from "./getAdminService200ManifestBindMountsItem"; export * from "./getAdminService200ManifestBuildImages"; @@ -233,7 +268,41 @@ export * from "./listAdminServiceBackups415"; export * from "./listAdminServiceBackups429"; export * from "./listAdminServiceBackups500"; export * from "./listAdminServices200"; +export * from "./listAdminServices200Acquisition"; export * from "./listAdminServices200ServicesItem"; +export * from "./listAdminServices200ServicesItemActions"; +export * from "./listAdminServices200ServicesItemActionsApply"; +export * from "./listAdminServices200ServicesItemActionsApplyExecution"; +export * from "./listAdminServices200ServicesItemActionsApplyReason"; +export * from "./listAdminServices200ServicesItemActionsApplyWorkflow"; +export * from "./listAdminServices200ServicesItemActionsBuild"; +export * from "./listAdminServices200ServicesItemActionsBuildExecution"; +export * from "./listAdminServices200ServicesItemActionsBuildReason"; +export * from "./listAdminServices200ServicesItemActionsBuildWorkflow"; +export * from "./listAdminServices200ServicesItemActionsLogs"; +export * from "./listAdminServices200ServicesItemActionsLogsExecution"; +export * from "./listAdminServices200ServicesItemActionsLogsReason"; +export * from "./listAdminServices200ServicesItemActionsLogsWorkflow"; +export * from "./listAdminServices200ServicesItemActionsPull"; +export * from "./listAdminServices200ServicesItemActionsPullExecution"; +export * from "./listAdminServices200ServicesItemActionsPullReason"; +export * from "./listAdminServices200ServicesItemActionsPullWorkflow"; +export * from "./listAdminServices200ServicesItemActionsRestart"; +export * from "./listAdminServices200ServicesItemActionsRestartExecution"; +export * from "./listAdminServices200ServicesItemActionsRestartReason"; +export * from "./listAdminServices200ServicesItemActionsRestartWorkflow"; +export * from "./listAdminServices200ServicesItemActionsStart"; +export * from "./listAdminServices200ServicesItemActionsStartExecution"; +export * from "./listAdminServices200ServicesItemActionsStartReason"; +export * from "./listAdminServices200ServicesItemActionsStartWorkflow"; +export * from "./listAdminServices200ServicesItemActionsStop"; +export * from "./listAdminServices200ServicesItemActionsStopExecution"; +export * from "./listAdminServices200ServicesItemActionsStopReason"; +export * from "./listAdminServices200ServicesItemActionsStopWorkflow"; +export * from "./listAdminServices200ServicesItemActionsUpdate"; +export * from "./listAdminServices200ServicesItemActionsUpdateExecution"; +export * from "./listAdminServices200ServicesItemActionsUpdateReason"; +export * from "./listAdminServices200ServicesItemActionsUpdateWorkflow"; export * from "./listAdminServices200ServicesItemConsumesItem"; export * from "./listAdminServices200ServicesItemExposure"; export * from "./listAdminServices200ServicesItemExposureHostPortsItem"; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200.ts index 0b6190060..702941b44 100644 --- a/packages/core/src/api/generated/admin/service-models/listAdminServices200.ts +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200.ts @@ -1,9 +1,11 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200Acquisition } from "./listAdminServices200Acquisition"; import type { ListAdminServices200ServicesItem } from "./listAdminServices200ServicesItem"; import type { ListAdminServices200Summary } from "./listAdminServices200Summary"; export type ListAdminServices200 = { + acquisition: ListAdminServices200Acquisition; services: ListAdminServices200ServicesItem[]; summary: ListAdminServices200Summary; }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200Acquisition.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200Acquisition.ts new file mode 100644 index 000000000..5159a3677 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200Acquisition.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200Acquisition = + | { + state: "available"; + /** @nullable */ + reason: null; + } + | { + state: "unavailable"; + reason: "ops_unavailable"; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItem.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItem.ts index a16662dd0..53d61701b 100644 --- a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItem.ts +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItem.ts @@ -1,5 +1,6 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActions } from "./listAdminServices200ServicesItemActions"; import type { ListAdminServices200ServicesItemConsumesItem } from "./listAdminServices200ServicesItemConsumesItem"; import type { ListAdminServices200ServicesItemExposure } from "./listAdminServices200ServicesItemExposure"; import type { ListAdminServices200ServicesItemQuality } from "./listAdminServices200ServicesItemQuality"; @@ -23,4 +24,5 @@ export type ListAdminServices200ServicesItem = { enabled: boolean; isBuiltIn: boolean; status: ListAdminServices200ServicesItemStatus; + actions: ListAdminServices200ServicesItemActions; }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActions.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActions.ts new file mode 100644 index 000000000..55f079499 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActions.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsApply } from "./listAdminServices200ServicesItemActionsApply"; +import type { ListAdminServices200ServicesItemActionsBuild } from "./listAdminServices200ServicesItemActionsBuild"; +import type { ListAdminServices200ServicesItemActionsLogs } from "./listAdminServices200ServicesItemActionsLogs"; +import type { ListAdminServices200ServicesItemActionsPull } from "./listAdminServices200ServicesItemActionsPull"; +import type { ListAdminServices200ServicesItemActionsRestart } from "./listAdminServices200ServicesItemActionsRestart"; +import type { ListAdminServices200ServicesItemActionsStart } from "./listAdminServices200ServicesItemActionsStart"; +import type { ListAdminServices200ServicesItemActionsStop } from "./listAdminServices200ServicesItemActionsStop"; +import type { ListAdminServices200ServicesItemActionsUpdate } from "./listAdminServices200ServicesItemActionsUpdate"; + +export type ListAdminServices200ServicesItemActions = { + start: ListAdminServices200ServicesItemActionsStart; + stop: ListAdminServices200ServicesItemActionsStop; + restart: ListAdminServices200ServicesItemActionsRestart; + update: ListAdminServices200ServicesItemActionsUpdate; + build: ListAdminServices200ServicesItemActionsBuild; + logs: ListAdminServices200ServicesItemActionsLogs; + pull: ListAdminServices200ServicesItemActionsPull; + apply: ListAdminServices200ServicesItemActionsApply; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApply.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApply.ts new file mode 100644 index 000000000..a3360e9a9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApply.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsApplyExecution } from "./listAdminServices200ServicesItemActionsApplyExecution"; +import type { ListAdminServices200ServicesItemActionsApplyReason } from "./listAdminServices200ServicesItemActionsApplyReason"; +import type { ListAdminServices200ServicesItemActionsApplyWorkflow } from "./listAdminServices200ServicesItemActionsApplyWorkflow"; + +export type ListAdminServices200ServicesItemActionsApply = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsApplyExecution; + reason: ListAdminServices200ServicesItemActionsApplyReason; + workflow: ListAdminServices200ServicesItemActionsApplyWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyExecution.ts new file mode 100644 index 000000000..444a042a4 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsApplyExecution = + (typeof ListAdminServices200ServicesItemActionsApplyExecution)[keyof typeof ListAdminServices200ServicesItemActionsApplyExecution]; + +export const ListAdminServices200ServicesItemActionsApplyExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyReason.ts new file mode 100644 index 000000000..c02735555 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsApplyReason = + | (typeof ListAdminServices200ServicesItemActionsApplyReason)[keyof typeof ListAdminServices200ServicesItemActionsApplyReason] + | null; + +export const ListAdminServices200ServicesItemActionsApplyReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyWorkflow.ts new file mode 100644 index 000000000..41a617625 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsApplyWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsApplyWorkflow = + | (typeof ListAdminServices200ServicesItemActionsApplyWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsApplyWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsApplyWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuild.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuild.ts new file mode 100644 index 000000000..213756b4a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuild.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsBuildExecution } from "./listAdminServices200ServicesItemActionsBuildExecution"; +import type { ListAdminServices200ServicesItemActionsBuildReason } from "./listAdminServices200ServicesItemActionsBuildReason"; +import type { ListAdminServices200ServicesItemActionsBuildWorkflow } from "./listAdminServices200ServicesItemActionsBuildWorkflow"; + +export type ListAdminServices200ServicesItemActionsBuild = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsBuildExecution; + reason: ListAdminServices200ServicesItemActionsBuildReason; + workflow: ListAdminServices200ServicesItemActionsBuildWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildExecution.ts new file mode 100644 index 000000000..58dd86536 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsBuildExecution = + (typeof ListAdminServices200ServicesItemActionsBuildExecution)[keyof typeof ListAdminServices200ServicesItemActionsBuildExecution]; + +export const ListAdminServices200ServicesItemActionsBuildExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildReason.ts new file mode 100644 index 000000000..96ab36971 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsBuildReason = + | (typeof ListAdminServices200ServicesItemActionsBuildReason)[keyof typeof ListAdminServices200ServicesItemActionsBuildReason] + | null; + +export const ListAdminServices200ServicesItemActionsBuildReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildWorkflow.ts new file mode 100644 index 000000000..96e1366f9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsBuildWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsBuildWorkflow = + | (typeof ListAdminServices200ServicesItemActionsBuildWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsBuildWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsBuildWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogs.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogs.ts new file mode 100644 index 000000000..6410b6a69 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogs.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsLogsExecution } from "./listAdminServices200ServicesItemActionsLogsExecution"; +import type { ListAdminServices200ServicesItemActionsLogsReason } from "./listAdminServices200ServicesItemActionsLogsReason"; +import type { ListAdminServices200ServicesItemActionsLogsWorkflow } from "./listAdminServices200ServicesItemActionsLogsWorkflow"; + +export type ListAdminServices200ServicesItemActionsLogs = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsLogsExecution; + reason: ListAdminServices200ServicesItemActionsLogsReason; + workflow: ListAdminServices200ServicesItemActionsLogsWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsExecution.ts new file mode 100644 index 000000000..ce7350d93 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsLogsExecution = + (typeof ListAdminServices200ServicesItemActionsLogsExecution)[keyof typeof ListAdminServices200ServicesItemActionsLogsExecution]; + +export const ListAdminServices200ServicesItemActionsLogsExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsReason.ts new file mode 100644 index 000000000..00901e713 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsLogsReason = + | (typeof ListAdminServices200ServicesItemActionsLogsReason)[keyof typeof ListAdminServices200ServicesItemActionsLogsReason] + | null; + +export const ListAdminServices200ServicesItemActionsLogsReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsWorkflow.ts new file mode 100644 index 000000000..b69cb00c8 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsLogsWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsLogsWorkflow = + | (typeof ListAdminServices200ServicesItemActionsLogsWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsLogsWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsLogsWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPull.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPull.ts new file mode 100644 index 000000000..ecf1f072e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPull.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsPullExecution } from "./listAdminServices200ServicesItemActionsPullExecution"; +import type { ListAdminServices200ServicesItemActionsPullReason } from "./listAdminServices200ServicesItemActionsPullReason"; +import type { ListAdminServices200ServicesItemActionsPullWorkflow } from "./listAdminServices200ServicesItemActionsPullWorkflow"; + +export type ListAdminServices200ServicesItemActionsPull = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsPullExecution; + reason: ListAdminServices200ServicesItemActionsPullReason; + workflow: ListAdminServices200ServicesItemActionsPullWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullExecution.ts new file mode 100644 index 000000000..817a6f44a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsPullExecution = + (typeof ListAdminServices200ServicesItemActionsPullExecution)[keyof typeof ListAdminServices200ServicesItemActionsPullExecution]; + +export const ListAdminServices200ServicesItemActionsPullExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullReason.ts new file mode 100644 index 000000000..6f0b70090 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsPullReason = + | (typeof ListAdminServices200ServicesItemActionsPullReason)[keyof typeof ListAdminServices200ServicesItemActionsPullReason] + | null; + +export const ListAdminServices200ServicesItemActionsPullReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullWorkflow.ts new file mode 100644 index 000000000..1ebc07e19 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsPullWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsPullWorkflow = + | (typeof ListAdminServices200ServicesItemActionsPullWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsPullWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsPullWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestart.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestart.ts new file mode 100644 index 000000000..f6572d954 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestart.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsRestartExecution } from "./listAdminServices200ServicesItemActionsRestartExecution"; +import type { ListAdminServices200ServicesItemActionsRestartReason } from "./listAdminServices200ServicesItemActionsRestartReason"; +import type { ListAdminServices200ServicesItemActionsRestartWorkflow } from "./listAdminServices200ServicesItemActionsRestartWorkflow"; + +export type ListAdminServices200ServicesItemActionsRestart = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsRestartExecution; + reason: ListAdminServices200ServicesItemActionsRestartReason; + workflow: ListAdminServices200ServicesItemActionsRestartWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartExecution.ts new file mode 100644 index 000000000..ae19a32d8 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsRestartExecution = + (typeof ListAdminServices200ServicesItemActionsRestartExecution)[keyof typeof ListAdminServices200ServicesItemActionsRestartExecution]; + +export const ListAdminServices200ServicesItemActionsRestartExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartReason.ts new file mode 100644 index 000000000..9c0ac216f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsRestartReason = + | (typeof ListAdminServices200ServicesItemActionsRestartReason)[keyof typeof ListAdminServices200ServicesItemActionsRestartReason] + | null; + +export const ListAdminServices200ServicesItemActionsRestartReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartWorkflow.ts new file mode 100644 index 000000000..a58a642bb --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsRestartWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsRestartWorkflow = + | (typeof ListAdminServices200ServicesItemActionsRestartWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsRestartWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsRestartWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStart.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStart.ts new file mode 100644 index 000000000..ae171c239 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStart.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsStartExecution } from "./listAdminServices200ServicesItemActionsStartExecution"; +import type { ListAdminServices200ServicesItemActionsStartReason } from "./listAdminServices200ServicesItemActionsStartReason"; +import type { ListAdminServices200ServicesItemActionsStartWorkflow } from "./listAdminServices200ServicesItemActionsStartWorkflow"; + +export type ListAdminServices200ServicesItemActionsStart = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsStartExecution; + reason: ListAdminServices200ServicesItemActionsStartReason; + workflow: ListAdminServices200ServicesItemActionsStartWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartExecution.ts new file mode 100644 index 000000000..0d56f15c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsStartExecution = + (typeof ListAdminServices200ServicesItemActionsStartExecution)[keyof typeof ListAdminServices200ServicesItemActionsStartExecution]; + +export const ListAdminServices200ServicesItemActionsStartExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartReason.ts new file mode 100644 index 000000000..deb72cf8f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsStartReason = + | (typeof ListAdminServices200ServicesItemActionsStartReason)[keyof typeof ListAdminServices200ServicesItemActionsStartReason] + | null; + +export const ListAdminServices200ServicesItemActionsStartReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartWorkflow.ts new file mode 100644 index 000000000..09019c0b8 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStartWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsStartWorkflow = + | (typeof ListAdminServices200ServicesItemActionsStartWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsStartWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsStartWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStop.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStop.ts new file mode 100644 index 000000000..7905c1c2a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStop.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsStopExecution } from "./listAdminServices200ServicesItemActionsStopExecution"; +import type { ListAdminServices200ServicesItemActionsStopReason } from "./listAdminServices200ServicesItemActionsStopReason"; +import type { ListAdminServices200ServicesItemActionsStopWorkflow } from "./listAdminServices200ServicesItemActionsStopWorkflow"; + +export type ListAdminServices200ServicesItemActionsStop = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsStopExecution; + reason: ListAdminServices200ServicesItemActionsStopReason; + workflow: ListAdminServices200ServicesItemActionsStopWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopExecution.ts new file mode 100644 index 000000000..e2d0d0e98 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsStopExecution = + (typeof ListAdminServices200ServicesItemActionsStopExecution)[keyof typeof ListAdminServices200ServicesItemActionsStopExecution]; + +export const ListAdminServices200ServicesItemActionsStopExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopReason.ts new file mode 100644 index 000000000..f16f5b1cd --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsStopReason = + | (typeof ListAdminServices200ServicesItemActionsStopReason)[keyof typeof ListAdminServices200ServicesItemActionsStopReason] + | null; + +export const ListAdminServices200ServicesItemActionsStopReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopWorkflow.ts new file mode 100644 index 000000000..e1d3bae97 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsStopWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsStopWorkflow = + | (typeof ListAdminServices200ServicesItemActionsStopWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsStopWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsStopWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdate.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdate.ts new file mode 100644 index 000000000..8a162e57d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdate.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminServices200ServicesItemActionsUpdateExecution } from "./listAdminServices200ServicesItemActionsUpdateExecution"; +import type { ListAdminServices200ServicesItemActionsUpdateReason } from "./listAdminServices200ServicesItemActionsUpdateReason"; +import type { ListAdminServices200ServicesItemActionsUpdateWorkflow } from "./listAdminServices200ServicesItemActionsUpdateWorkflow"; + +export type ListAdminServices200ServicesItemActionsUpdate = { + available: boolean; + execution: ListAdminServices200ServicesItemActionsUpdateExecution; + reason: ListAdminServices200ServicesItemActionsUpdateReason; + workflow: ListAdminServices200ServicesItemActionsUpdateWorkflow; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateExecution.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateExecution.ts new file mode 100644 index 000000000..851086cc9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateExecution.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsUpdateExecution = + (typeof ListAdminServices200ServicesItemActionsUpdateExecution)[keyof typeof ListAdminServices200ServicesItemActionsUpdateExecution]; + +export const ListAdminServices200ServicesItemActionsUpdateExecution = { + direct: "direct", + workflow: "workflow", + host: "host", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateReason.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateReason.ts new file mode 100644 index 000000000..4f989f7bc --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateReason.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsUpdateReason = + | (typeof ListAdminServices200ServicesItemActionsUpdateReason)[keyof typeof ListAdminServices200ServicesItemActionsUpdateReason] + | null; + +export const ListAdminServices200ServicesItemActionsUpdateReason = { + disabled: "disabled", + not_owned: "not_owned", + not_running: "not_running", + control_plane: "control_plane", + host_only: "host_only", + unavailable: "unavailable", + not_buildable: "not_buildable", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateWorkflow.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateWorkflow.ts new file mode 100644 index 000000000..6519f4f9c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200ServicesItemActionsUpdateWorkflow.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminServices200ServicesItemActionsUpdateWorkflow = + | (typeof ListAdminServices200ServicesItemActionsUpdateWorkflow)[keyof typeof ListAdminServices200ServicesItemActionsUpdateWorkflow] + | null; + +export const ListAdminServices200ServicesItemActionsUpdateWorkflow = { + "core-restart": "core-restart", + "system-update": "system-update", + "coordinator-apply": "coordinator-apply", + "coordinator-update": "coordinator-update", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminServices200Summary.ts b/packages/core/src/api/generated/admin/service-models/listAdminServices200Summary.ts index b31248c10..d17986f50 100644 --- a/packages/core/src/api/generated/admin/service-models/listAdminServices200Summary.ts +++ b/packages/core/src/api/generated/admin/service-models/listAdminServices200Summary.ts @@ -12,6 +12,11 @@ export type ListAdminServices200Summary = { * @maximum 9007199254740991 */ stopped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + unknown: number; /** * @minimum 0 * @maximum 9007199254740991 diff --git a/packages/core/src/ops/service-results.ts b/packages/core/src/ops/service-results.ts index f9c4fbd57..0321ee332 100644 --- a/packages/core/src/ops/service-results.ts +++ b/packages/core/src/ops/service-results.ts @@ -155,4 +155,13 @@ export const serviceActionDecisionSchema = z (decision.execution !== "host" || decision.reason === "host_only"), ); export type ServiceActionDecision = z.infer; -export const serviceActionsSchema = z.record(z.enum(SERVICE_ACTIONS), serviceActionDecisionSchema); +export const serviceActionsSchema = z.strictObject({ + start: serviceActionDecisionSchema, + stop: serviceActionDecisionSchema, + restart: serviceActionDecisionSchema, + update: serviceActionDecisionSchema, + build: serviceActionDecisionSchema, + logs: serviceActionDecisionSchema, + pull: serviceActionDecisionSchema, + apply: serviceActionDecisionSchema, +}); From ab1d45b7043cbf4da416f482159eace0daf6e888 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 10:11:53 +0100 Subject: [PATCH 038/135] feat(admin): expose service capabilities and reliable outcomes --- apps/web/src/components/admin/AdminLayout.tsx | 13 +- .../components/admin/AdminSidebar.test.tsx | 75 ++++++++--- .../components/admin/activity/JobDetail.tsx | 32 ++++- .../__tests__/ActivityConsumers.test.tsx | 18 +++ .../admin/activity/activityQueries.ts | 15 ++- .../services/ServiceAdministration.test.tsx | 127 +++++++++++++++++- .../admin/services/ServiceCatalog.tsx | 72 +++++++++- .../admin/services/ServiceConfigForm.tsx | 2 +- .../admin/services/ServiceDetail.test.tsx | 11 ++ .../admin/services/ServiceDetail.tsx | 104 +++++++++++--- apps/web/src/lib/admin/adminJobEffects.tsx | 21 ++- apps/web/src/lib/admin/serviceAdmin.test.tsx | 17 ++- apps/web/src/lib/admin/serviceAdmin.ts | 23 ++++ .../src/lib/admin/serviceAdminJobs.test.tsx | 67 ++++++++- packages/core/src/api/contracts/index.ts | 1 + 15 files changed, 530 insertions(+), 68 deletions(-) diff --git a/apps/web/src/components/admin/AdminLayout.tsx b/apps/web/src/components/admin/AdminLayout.tsx index f06a1f934..b61d88986 100644 --- a/apps/web/src/components/admin/AdminLayout.tsx +++ b/apps/web/src/components/admin/AdminLayout.tsx @@ -1,6 +1,7 @@ "use client"; import Box from "@mui/material/Box"; +import CircularProgress from "@mui/material/CircularProgress"; import { useCallback, useState } from "react"; import { AdminJobEffectsProvider } from "@/lib/admin/adminJobEffects"; import { useHydrated } from "@/lib/useHydrated"; @@ -70,7 +71,17 @@ export function AdminLayout({ children, user, selfHosted = false }: AdminLayoutP py: { xs: 2, lg: 2.5 }, }} > - {children} + {hydrated ? ( + children + ) : ( + + + + )} diff --git a/apps/web/src/components/admin/AdminSidebar.test.tsx b/apps/web/src/components/admin/AdminSidebar.test.tsx index bbb583414..64de279eb 100644 --- a/apps/web/src/components/admin/AdminSidebar.test.tsx +++ b/apps/web/src/components/admin/AdminSidebar.test.tsx @@ -1,31 +1,66 @@ -// @vitest-environment jsdom - -import { createElement } from "react"; -import { expect, it, vi } from "vitest"; -import { render, screen } from "@/test"; - -vi.mock("next-intl", async () => (await import("@/test/intl")).mockNextIntl()); -vi.mock("next/navigation", () => ({ usePathname: () => "/admin/privacy-setup" })); - +import { en } from "@openmapx/i18n"; +import { NextIntlClientProvider } from "next-intl"; +import { afterEach, expect, it, vi } from "vitest"; +import { cleanup, render, screen } from "@/test"; import { AdminSidebar } from "./AdminSidebar"; -it("shows privacy setup to full administrators", () => { - render(createElement(AdminSidebar, { open: true, onClose: () => undefined, role: "admin" })); +vi.mock("next/navigation", () => ({ usePathname: () => "/admin/services" })); +afterEach(cleanup); +it.each([true, null])( + "preserves troubleshooting navigation for admin deployment %s", + (selfHosted) => { + render( + + {}} selfHosted={selfHosted} /> + , + ); + expect(screen.getAllByText("Services").length).toBeGreaterThan(0); + expect(screen.getAllByText("Maintenance").length).toBeGreaterThan(0); + }, +); +it.each(["admin", "privacy_admin"])( + "does not expose host navigation for explicitly hosted %s", + (role) => { + render( + + {}} selfHosted={false} role={role} /> + , + ); + expect(screen.queryByText("Services")).not.toBeInTheDocument(); + expect(screen.queryByText("Maintenance")).not.toBeInTheDocument(); + }, +); +it("keeps privacy-admin access restricted during unknown deployment", () => { + render( + + {}} + selfHosted={null} + {...{ role: "privacy_admin" }} + /> + , + ); + expect(screen.queryByText("Services")).not.toBeInTheDocument(); + expect(screen.queryByText("Maintenance")).not.toBeInTheDocument(); +}); - expect(screen.getAllByRole("link", { name: "privacySetup.navigationLabel" })[0]).toHaveAttribute( +it("shows privacy setup to full administrators", () => { + render( + + {}} {...{ role: "admin" }} /> + , + ); + expect(screen.getAllByRole("link", { name: "Privacy setup" })[0]).toHaveAttribute( "href", "/admin/privacy-setup", ); }); - it("does not offer privacy setup to privacy administrators", () => { render( - createElement(AdminSidebar, { - open: true, - onClose: () => undefined, - role: "privacy_admin", - }), + + {}} {...{ role: "privacy_admin" }} /> + , ); - - expect(screen.queryByRole("link", { name: "privacySetup.navigationLabel" })).toBeNull(); + expect(screen.queryByRole("link", { name: "Privacy setup" })).toBeNull(); }); diff --git a/apps/web/src/components/admin/activity/JobDetail.tsx b/apps/web/src/components/admin/activity/JobDetail.tsx index baeb94990..6579f6736 100644 --- a/apps/web/src/components/admin/activity/JobDetail.tsx +++ b/apps/web/src/components/admin/activity/JobDetail.tsx @@ -12,6 +12,7 @@ import { cancelAdminActivityJob, unwrapAdminResponse } from "@openmapx/core/api- import { type AdminActivityJobSource, adminJobObservationSchema, + serviceBatchResultSchema, } from "@openmapx/core/api-contracts"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useEffect, useRef } from "react"; @@ -20,6 +21,7 @@ import { JobStatusChip } from "../shared/JobStatusChip"; import { ACTIVITY_READ_TIMEOUT_MS, activityJobQueryOptions, + hasUnresolvedJobObservation, useActivityClient, } from "./activityQueries"; import { type JobStreamConnection, useJobEventStream } from "./useJobEventStream"; @@ -70,7 +72,9 @@ export function JobDetail({ jobId, source }: { jobId: string; source: AdminActiv refetchInterval: (query) => { if (!polling) return false; const status = query.state.data?.status; - return status && isActive(status) ? POLL_INTERVAL_MS : false; + return status && (isActive(status) || hasUnresolvedJobObservation(query.state.data?.result)) + ? POLL_INTERVAL_MS + : false; }, }); const data = stream.data ?? query.data; @@ -133,6 +137,7 @@ export function JobDetail({ jobId, source }: { jobId: string; source: AdminActiv data.result && typeof data.result === "object" && !Array.isArray(data.result) ? data.result : {}; + const batch = serviceBatchResultSchema.safeParse(result.serviceBatch); const observation = adminJobObservationSchema.safeParse(result.observation); const diagnostics = result.diagnostics && @@ -189,7 +194,7 @@ export function JobDetail({ jobId, source }: { jobId: string; source: AdminActiv > Duration: {formatDuration(data.startedAt, data.finishedAt)} - {data.cancelable && isCancelable(data.status) && ( + {data.cancelable && data.status !== "cancel_pending" && ( + + )} queueBulkAction("start")} - disabled={bulkMutation.isPending || selectedIds.size === 0} + title={bulkReason("start")} + disabled={ + bulkMutation.isPending || selectedIds.size === 0 || deniedIds("start").length > 0 + } > Start Selected @@ -485,7 +512,10 @@ function ServiceCatalogView() { variant="outlined" color="warning" onClick={() => queueBulkAction("stop")} - disabled={bulkMutation.isPending || selectedIds.size === 0} + title={bulkReason("stop")} + disabled={ + bulkMutation.isPending || selectedIds.size === 0 || deniedIds("stop").length > 0 + } > Stop Selected @@ -493,7 +523,10 @@ function ServiceCatalogView() { size="small" variant="outlined" onClick={() => queueBulkAction("restart")} - disabled={bulkMutation.isPending || selectedIds.size === 0} + title={bulkReason("restart")} + disabled={ + bulkMutation.isPending || selectedIds.size === 0 || deniedIds("restart").length > 0 + } > Restart Selected @@ -501,7 +534,10 @@ function ServiceCatalogView() { size="small" variant="outlined" onClick={() => queueBulkAction("update")} - disabled={bulkMutation.isPending || selectedIds.size === 0} + title={bulkReason("update")} + disabled={ + bulkMutation.isPending || selectedIds.size === 0 || deniedIds("update").length > 0 + } > Update selected @@ -509,7 +545,10 @@ function ServiceCatalogView() { size="small" variant="outlined" onClick={() => queueBulkAction("build")} - disabled={bulkMutation.isPending || selectedIds.size === 0} + title={bulkReason("build")} + disabled={ + bulkMutation.isPending || selectedIds.size === 0 || deniedIds("build").length > 0 + } > Build Selected @@ -517,7 +556,7 @@ function ServiceCatalogView() { size="small" variant="contained" onClick={queueBuildAll} - disabled={bulkMutation.isPending} + disabled={bulkMutation.isPending || !services.some((s) => s.actions?.build.available)} > Build All @@ -525,6 +564,22 @@ function ServiceCatalogView() { View Jobs + {selectedIds.size > 0 && + ["start", "stop", "restart", "update", "build"].some( + (action) => deniedIds(action as BulkAction).length > 0, + ) && ( + + Unavailable targets:{" "} + {["start", "stop", "restart", "update", "build"] + .map((action) => { + const ids = deniedIds(action as BulkAction); + return ids.length ? `${action}: ${ids.join(", ")}` : null; + }) + .filter(Boolean) + .join("; ")} + . Check individual service controls for the available workflow. + + )} } > + {summary && summary.unknown > 0 && ( + + {summary.unknown} service(s) have unknown status. + + )} {isError && ( [ + action, + { available: true, execution: "direct", reason: null, workflow: null }, + ]), +); +const acquisition = { state: "available", reason: null }; + vi.mock("@/integration-api/runtime/EnvProvider", () => ({ useEnv: () => ({ apiUrl: "http://admin.test" }), })); @@ -8,6 +17,7 @@ vi.mock("@/integration-api/runtime/EnvProvider", () => ({ vi.mock("@/hooks/useServices", () => ({ useServiceDetail: (serviceId: string) => ({ data: { + acquisition, manifest: { id: serviceId, name: serviceId === "dawarich-app" ? "Dawarich" : "Redis", @@ -17,6 +27,7 @@ vi.mock("@/hooks/useServices", () => ({ }, directory: "/services/example", isBuiltIn: false, + actions: availableActions, enabled: true, status: "not-running", }, diff --git a/apps/web/src/components/admin/services/ServiceDetail.tsx b/apps/web/src/components/admin/services/ServiceDetail.tsx index 2251205ee..ce0947ed8 100644 --- a/apps/web/src/components/admin/services/ServiceDetail.tsx +++ b/apps/web/src/components/admin/services/ServiceDetail.tsx @@ -23,6 +23,7 @@ import type { AdminServiceAction, AdminServiceDetail as ServiceDetailData, } from "@openmapx/core/api-contracts"; +import Link from "next/link"; import { useState } from "react"; import { useServiceAction, @@ -30,6 +31,7 @@ import { useServiceConfigSave, useServiceDetail, } from "@/hooks/useServices"; +import { serviceActionReason } from "@/lib/admin/serviceAdmin"; import { useAdminMutationOwner } from "@/lib/admin/useAdminMutationOwner"; import { useActivityClient } from "../activity/activityQueries"; import { StatusBadge } from "../integrations/StatusBadge"; @@ -386,6 +388,9 @@ function OverviewTab({ data }: { data: ServiceDetailData }) { function ConfigTab({ data }: { data: ServiceDetailData }) { const { manifest } = data; + const startAction = data.status === "running" ? "apply" : "start"; + const reason = (action: import("@openmapx/core/ops").ServiceAction) => + serviceActionReason(data.actions?.[action]); // Persisted config from `service_config` table (separate query from the // manifest itself so the config tab doesn't refetch the full LoadedService). const configQuery = useServiceConfig(manifest.id); @@ -424,15 +429,21 @@ function ConfigTab({ data }: { data: ServiceDetailData }) { onSave={async (values) => { await saveConfig.mutateAsync(values); }} - onSaveAndApply={async (values) => { - // Service configs land in the generated compose env, so applying - // a new value must go through `docker compose up -d` semantics. - // Persist first so a failed apply doesn't lose the edit. - const execution = owner; - await saveConfig.mutateAsync(values); - if (!execution.active) return; - await action.mutateAsync("start"); - }} + onSaveAndApply={ + reason(startAction) + ? undefined + : async (values) => { + // Service configs land in the generated compose env, so applying + // a new value must go through `docker compose up -d` semantics. + // Persist first so a failed apply doesn't lose the edit. + if (reason(startAction)) + throw new Error(reason(startAction) ?? "Action unavailable"); + const execution = owner; + await saveConfig.mutateAsync(values); + if (!execution.active) return; + await action.mutateAsync(startAction); + } + } /> @@ -493,7 +504,7 @@ function ServiceDetailView({ id }: ServiceDetailProps) { const [logsOpen, setLogsOpen] = useState(false); const showToast = useAdminToast(); - const { data, isLoading, isError } = useServiceDetail(id); + const { data, isLoading, isError, refetch } = useServiceDetail(id); const actionMutation = useServiceAction(id); if (isLoading) { @@ -519,9 +530,13 @@ function ServiceDetailView({ id }: ServiceDetailProps) { } const { manifest } = data; + const startAction = data.status === "running" ? "apply" : "start"; + const reason = (action: import("@openmapx/core/ops").ServiceAction) => + serviceActionReason(data.actions?.[action]); const isBusy = actionMutation.isPending; async function runAction(action: AdminServiceAction["action"]) { + if (reason(action)) return; const execution = owner; try { const result = await actionMutation.mutateAsync(action); @@ -537,7 +552,7 @@ function ServiceDetailView({ id }: ServiceDetailProps) { function handleTabChange(_: React.SyntheticEvent, value: number) { setTab(value); // Auto-open the logs drawer when the Logs tab is selected. - if (TABS[value] === "Logs") { + if (TABS[value] === "Logs" && !reason("logs")) { setLogsOpen(true); } } @@ -602,18 +617,26 @@ function ServiceDetailView({ id }: ServiceDetailProps) { gap: 1, flexWrap: "wrap", alignItems: "center", - flexShrink: 0, + flexShrink: 1, + minWidth: 0, + maxWidth: "100%", + width: { xs: "100%", sm: "auto" }, }} > - + @@ -625,18 +648,19 @@ function ServiceDetailView({ id }: ServiceDetailProps) { color="error" startIcon={isBusy ? : } onClick={() => runAction("stop")} - disabled={isBusy || data.status === "exited" || data.status === "not-running"} + disabled={isBusy || Boolean(reason("stop"))} + title={reason("stop") ?? undefined} > Stop - + @@ -646,7 +670,11 @@ function ServiceDetailView({ id }: ServiceDetailProps) { size="small" variant="outlined" startIcon={} - onClick={() => setLogsOpen(true)} + onClick={() => { + if (!reason("logs")) setLogsOpen(true); + }} + disabled={Boolean(reason("logs"))} + title={reason("logs") ?? undefined} > Logs @@ -672,6 +700,34 @@ function ServiceDetailView({ id }: ServiceDetailProps) { )} + {[ + ...new Set( + [reason(startAction), reason("stop"), reason("restart"), reason("logs")].filter(Boolean), + ), + ].length > 0 && ( + + {[ + ...new Set( + [reason(startAction), reason("stop"), reason("restart"), reason("logs")].filter( + Boolean, + ), + ), + ].join(" ")} + + )} + {data.acquisition?.state === "unavailable" && ( + + Operations agent is unavailable. Service state is unknown.{" "} + + + )} + {["app-api", "traefik", "ops-agent"].includes(manifest.id) && ( + + )} {manifest.id === "dawarich-app" && } @@ -683,7 +739,7 @@ function ServiceDetailView({ id }: ServiceDetailProps) { scrollButtons="auto" > {TABS.map((label) => ( - + ))} @@ -708,7 +764,11 @@ function ServiceDetailView({ id }: ServiceDetailProps) { variant="outlined" size="small" startIcon={} - onClick={() => setLogsOpen(true)} + onClick={() => { + if (!reason("logs")) setLogsOpen(true); + }} + disabled={Boolean(reason("logs"))} + title={reason("logs") ?? undefined} sx={{ mt: 1.5 }} > Open Logs Drawer @@ -718,7 +778,7 @@ function ServiceDetailView({ id }: ServiceDetailProps) { {tab === 4 && } setLogsOpen(false)} serviceId={id} serviceName={manifest.name} diff --git a/apps/web/src/lib/admin/adminJobEffects.tsx b/apps/web/src/lib/admin/adminJobEffects.tsx index 4d51f78c4..e5c98aa36 100644 --- a/apps/web/src/lib/admin/adminJobEffects.tsx +++ b/apps/web/src/lib/admin/adminJobEffects.tsx @@ -1,6 +1,7 @@ "use client"; import type { ApiClient } from "@openmapx/core"; +import type { AdminActivityJobDetail } from "@openmapx/core/api-contracts"; import { type QueryFunctionContext, useQueries, useQueryClient } from "@tanstack/react-query"; import { createContext, @@ -12,6 +13,7 @@ import { } from "react"; import { activityJobQueryOptions, + hasUnresolvedJobObservation, useActivityClient, } from "@/components/admin/activity/activityQueries"; import { useAdminMutationOwner } from "./useAdminMutationOwner"; @@ -53,7 +55,9 @@ export function AdminJobEffectsProvider({ children }: PropsWithChildren) { const options = activityJobQueryOptions(job.client, job.apiUrl, job.jobId, "application"); return { ...options, - queryFn: async (context: QueryFunctionContext<(string | undefined)[]>) => { + queryFn: async ( + context: QueryFunctionContext<(string | undefined)[]>, + ): Promise => { if (!options.queryFn) throw new Error("Admin job observation is unavailable"); const data = await options.queryFn(context); if (data.id !== job.jobId || data.source !== "application") @@ -61,15 +65,24 @@ export function AdminJobEffectsProvider({ children }: PropsWithChildren) { return data; }, retry: false, - refetchInterval: (query: { state: { data?: { status: string } } }) => - query.state.data && terminal.has(query.state.data.status) ? false : 1_000, + refetchInterval: (query: { state: { data?: { status: string; result?: unknown } } }) => + query.state.data && + terminal.has(query.state.data.status) && + !hasUnresolvedJobObservation(query.state.data.result) + ? false + : 1_000, }; }), }); useEffect(() => { const finished = jobs.filter((job, index) => { const data = observations[index]?.data; - return data?.id === job.jobId && data.source === "application" && terminal.has(data.status); + return ( + data?.id === job.jobId && + data.source === "application" && + terminal.has(data.status) && + !hasUnresolvedJobObservation(data.result) + ); }); if (!finished.length) return; for (const job of finished) void job.onSettled(); diff --git a/apps/web/src/lib/admin/serviceAdmin.test.tsx b/apps/web/src/lib/admin/serviceAdmin.test.tsx index 1d145edcc..61f7467cc 100644 --- a/apps/web/src/lib/admin/serviceAdmin.test.tsx +++ b/apps/web/src/lib/admin/serviceAdmin.test.tsx @@ -1,10 +1,19 @@ import { createApiClient } from "@openmapx/core"; +import { SERVICE_ACTIONS } from "@openmapx/core/ops"; import { useQueries, useQuery } from "@tanstack/react-query"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { act, cleanup, createTestQueryClient, renderHookWithQuery, waitFor } from "@/test"; import { invalidateServiceAdministration, serviceAdminQueries } from "./serviceAdmin"; let apiUrl = "http://admin.test"; +const availableActions = Object.fromEntries( + SERVICE_ACTIONS.map((action) => [ + action, + { available: true, execution: "direct", reason: null, workflow: null }, + ]), +); +const acquisition = { state: "available", reason: null }; + vi.mock("@/integration-api/runtime/EnvProvider", () => ({ useEnv: () => ({ apiUrl }) })); const clients: ReturnType[] = []; function cache() { @@ -226,8 +235,10 @@ describe("service administration query acquisition", () => { expect(new Set(keys).size).toBe(keys.length); vi.stubGlobal("fetch", (url: string) => Response.json({ + acquisition, services: [], summary: { + unknown: 0, running: 0, stopped: 0, total: new URL(url).origin === new URL(apiUrl).origin ? 1 : 2, @@ -260,7 +271,9 @@ describe("service administration query acquisition", () => { await waitFor(() => expect(qc.isFetching()).toBe(1)); unmount(); await waitFor(() => expect(canceled).toBe(true)); - vi.stubGlobal("fetch", () => Response.json({ services: [], secret: "synthetic-private" })); + vi.stubGlobal("fetch", () => + Response.json({ acquisition, services: [], secret: "synthetic-private" }), + ); await expect(qc.fetchQuery(queries().list())).rejects.toThrow("Service request failed"); expect(qc.getQueryData(queries().list().queryKey)).toBeUndefined(); }); @@ -291,9 +304,11 @@ describe("service administration query acquisition", () => { }); expect(u.pathname).toBe("/api/admin/services/selection"); return Response.json({ + acquisition, manifest: { ...manifest, name: marker }, directory: "/synthetic", isBuiltIn: true, + actions: availableActions, enabled: true, status: "running", }); diff --git a/apps/web/src/lib/admin/serviceAdmin.ts b/apps/web/src/lib/admin/serviceAdmin.ts index 52dfaed0e..e23778489 100644 --- a/apps/web/src/lib/admin/serviceAdmin.ts +++ b/apps/web/src/lib/admin/serviceAdmin.ts @@ -149,3 +149,26 @@ export async function invalidateServiceAdministration( // Default active-only refetch preserves invalidation for inactive queries too. await queryClient.invalidateQueries(filters); } + +/** Display the same decision used by the server, without inferring permission from status. */ +export function serviceActionReason( + decision: import("@openmapx/core/ops").ServiceActionDecision | undefined, +): string | null { + if (decision?.available) return null; + switch (decision?.reason) { + case "disabled": + return "Enable this service before using this action."; + case "not_owned": + return "No owned container is available for this action."; + case "not_running": + return "This service is not running."; + case "control_plane": + return "Use System Maintenance for core updates; stopping the control plane requires host access."; + case "host_only": + return "This action requires host access."; + case "not_buildable": + return "This service does not prepare build artifacts."; + default: + return "This action is currently unavailable. Check the operations agent."; + } +} diff --git a/apps/web/src/lib/admin/serviceAdminJobs.test.tsx b/apps/web/src/lib/admin/serviceAdminJobs.test.tsx index 28081997b..62e24dff4 100644 --- a/apps/web/src/lib/admin/serviceAdminJobs.test.tsx +++ b/apps/web/src/lib/admin/serviceAdminJobs.test.tsx @@ -18,6 +18,8 @@ import { serviceAdminQueries, serviceOrThrow } from "./serviceAdmin"; import { useServiceJobRegistration } from "./serviceAdminJobs"; let apiUrl = "http://admin.test"; +const acquisition = { state: "available", reason: null }; + vi.mock("@/integration-api/runtime/EnvProvider", () => ({ useEnv: () => ({ apiUrl }) })); const caches: ReturnType[] = []; function pending() { @@ -85,8 +87,9 @@ describe("admin-owned accepted service jobs", () => { async (status) => { const initial = pending(); const oldResponse = Response.json({ + acquisition, services: [], - summary: { running: 0, stopped: 2, total: 2 }, + summary: { unknown: 0, running: 0, stopped: 2, total: 2 }, }); let completed = false; let reads = 0; @@ -99,7 +102,11 @@ describe("admin-owned accepted service jobs", () => { expect(path).toBe("/api/admin/services"); signals.push(init.signal); if (++reads === 1) return initial.promise; - return Response.json({ services: [], summary: { running: 2, stopped: 0, total: 2 } }); + return Response.json({ + acquisition, + services: [], + summary: { unknown: 0, running: 2, stopped: 0, total: 2 }, + }); }); const { qc } = mount(); await waitFor(() => expect(reads).toBe(1)); @@ -126,7 +133,8 @@ describe("admin-owned accepted service jobs", () => { expect(screen.queryByText("0")).toBeNull(); expect(reads).toBe(2); for (const data of publications) - if (data !== undefined) expect(data).toMatchObject({ summary: { running: 2 } }); + if (data !== undefined) + expect(data).toMatchObject({ summary: { unknown: 0, running: 2 } }); } finally { initial.resolve(oldResponse); unsubscribe(); @@ -151,8 +159,9 @@ describe("admin-owned accepted service jobs", () => { } expect(path).toBe("/api/admin/services"); return Response.json({ + acquisition, services: [], - summary: { running: completed ? 2 : 0, stopped: 0, total: 2 }, + summary: { unknown: 0, running: completed ? 2 : 0, stopped: 0, total: 2 }, }); }); const { qc, view, rerender } = mount(); @@ -186,7 +195,11 @@ describe("admin-owned accepted service jobs", () => { ); } listReads++; - return Response.json({ services: [], summary: { running: 0, stopped: 0, total: 1 } }); + return Response.json({ + acquisition, + services: [], + summary: { unknown: 0, running: 0, stopped: 0, total: 1 }, + }); }); const { qc } = mount(); await screen.findByText("0"); @@ -210,7 +223,11 @@ describe("admin-owned accepted service jobs", () => { observed.push(url); return Response.json(job()); } - return Response.json({ services: [], summary: { running: 0, stopped: 0, total: 0 } }); + return Response.json({ + acquisition, + services: [], + summary: { unknown: 0, running: 0, stopped: 0, total: 0 }, + }); }); const { view, rerender } = mount(); await screen.findByText("0"); @@ -228,7 +245,11 @@ describe("admin-owned accepted service jobs", () => { const path = new URL(url).pathname; if (path.endsWith("/action")) return Response.json({ ok: true, jobId: "job-1" }); if (path.startsWith("/api/admin/jobs/")) return Response.json(job()); - return Response.json({ services: [], summary: { running: 0, stopped: 0, total: 0 } }); + return Response.json({ + acquisition, + services: [], + summary: { unknown: 0, running: 0, stopped: 0, total: 0 }, + }); }); const { qc } = mount(); await screen.findByText("0"); @@ -241,4 +262,36 @@ describe("admin-owned accepted service jobs", () => { expect(qc.getQueryState(second)?.isInvalidated).toBe(false); expect(qc.getQueryState(["admin", "services", apiUrl, "list"])?.isInvalidated).toBe(false); }); + it("keeps observation after a failed public status with retained unknown outcome", async () => { + let reads = 0; + vi.stubGlobal("fetch", (url: string) => { + const path = new URL(url).pathname; + if (path.endsWith("/action")) return Response.json({ ok: true, jobId: "job-1" }); + if (path.startsWith("/api/admin/jobs/")) { + reads++; + return Response.json( + job({ + status: "failed", + result: { + observation: { + state: "unknown", + reason: "transport", + nextCheckAt: "2026-10-28T09:00:00Z", + }, + }, + }), + ); + } + return Response.json({ + acquisition, + services: [], + summary: { running: 0, stopped: 0, unknown: 0, total: 0 }, + }); + }); + const { qc } = mount(); + await screen.findByText("0"); + fireEvent.click(screen.getByText("Queue")); + await waitFor(() => expect(reads).toBeGreaterThanOrEqual(2), { timeout: 3000 }); + expect(qc.getQueryState(["admin", "services", apiUrl, "list"])?.isInvalidated).toBe(false); + }); }); diff --git a/packages/core/src/api/contracts/index.ts b/packages/core/src/api/contracts/index.ts index 0551409c5..97766bc3a 100644 --- a/packages/core/src/api/contracts/index.ts +++ b/packages/core/src/api/contracts/index.ts @@ -1,3 +1,4 @@ +export { serviceBatchResultSchema } from "../../ops/service-results"; export * from "./activity-admin"; export * from "./data-manager-admin/index"; export * from "./extension-admin/index"; From 5faebfc9f7e23d25fa7220a43937a3333196ef8c Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 12:29:42 +0100 Subject: [PATCH 039/135] fix(data): await Full Update transit before graph builds --- .../__tests__/admin-data-children.test.ts | 80 +++- apps/api/src/services/admin-data-children.ts | 140 +++--- apps/ops-agent/src/administrative-runtime.ts | 8 +- .../src/configuration-maintenance-runtime.ts | 1 + .../src/data-workflow-runtime.test.ts | 140 ++++++ apps/ops-agent/src/data-workflow-runtime.ts | 152 +++++++ apps/ops-agent/src/journal.test.ts | 53 +++ apps/ops-agent/src/journal.ts | 23 + apps/ops-agent/src/runtime.ts | 2 + apps/ops-agent/src/server.ts | 53 ++- .../cli/__tests__/cli-build-staging.test.ts | 65 ++- .../__tests__/cli-coordinator-ready.test.ts | 98 +++++ .../cli-data-update-pipeline.test.ts | 158 +++++++ .../cli/__tests__/cli-data-update.test.ts | 41 ++ .../__tests__/cli-execution-deadline.test.ts | 44 +- .../fixtures/single-data-cli-process.ts | 17 +- packages/cli/src/commands/data.ts | 45 +- packages/cli/src/lib/build-staging.ts | 3 + packages/cli/src/lib/control-context.ts | 27 +- packages/cli/src/lib/coordinator-ready.ts | 46 ++ packages/cli/src/lib/data-execution.ts | 30 +- packages/cli/src/lib/data-update-command.ts | 253 +++++++++++ packages/cli/src/lib/data-update.ts | 174 ++++++++ packages/cli/src/lib/execution-deadline.ts | 6 +- packages/cli/src/lib/service-builds.ts | 5 + .../src/build-artifacts/control-authority.ts | 255 ++++++++++- .../build-artifacts/control-child-records.ts | 2 + .../build-artifacts/data-update-records.ts | 138 ++++++ .../build-artifacts/data-update-sources.ts | 107 +++++ .../core/src/build-artifacts/data-update.ts | 136 ++++++ .../build-artifacts/fixed-data-execution.ts | 16 +- .../generation-hardlinks-owner.ts | 14 +- packages/core/src/build-artifacts/index.ts | 8 + .../core/src/build-artifacts/target-lease.ts | 5 +- packages/core/src/ops/contract.ts | 3 +- .../__tests__/admin-data-workflow.test.ts | 171 +++++++ .../core/src/services/admin-data-workflow.ts | 115 +++++ packages/core/src/services/index.ts | 1 + scripts/database-test-groups.json | 1 + .../src/jobs/execution/full-update.test.ts | 416 ++++++++++++++++++ 40 files changed, 2919 insertions(+), 133 deletions(-) create mode 100644 apps/ops-agent/src/data-workflow-runtime.test.ts create mode 100644 apps/ops-agent/src/data-workflow-runtime.ts create mode 100644 packages/cli/__tests__/cli-coordinator-ready.test.ts create mode 100644 packages/cli/__tests__/cli-data-update-pipeline.test.ts create mode 100644 packages/cli/__tests__/cli-data-update.test.ts create mode 100644 packages/cli/src/lib/coordinator-ready.ts create mode 100644 packages/cli/src/lib/data-update-command.ts create mode 100644 packages/cli/src/lib/data-update.ts create mode 100644 packages/core/src/build-artifacts/data-update-records.ts create mode 100644 packages/core/src/build-artifacts/data-update-sources.ts create mode 100644 packages/core/src/build-artifacts/data-update.ts create mode 100644 packages/core/src/services/__tests__/admin-data-workflow.test.ts create mode 100644 packages/core/src/services/admin-data-workflow.ts create mode 100644 services/data-manager/src/jobs/execution/full-update.test.ts diff --git a/apps/api/src/services/__tests__/admin-data-children.test.ts b/apps/api/src/services/__tests__/admin-data-children.test.ts index 4130f1d64..0a23ee308 100644 --- a/apps/api/src/services/__tests__/admin-data-children.test.ts +++ b/apps/api/src/services/__tests__/admin-data-children.test.ts @@ -2,9 +2,20 @@ import { controlChildOperationKey } from "@openmapx/core/build-artifacts"; import { afterEach, expect, it, vi } from "vitest"; import type { JobContext } from "../job-runner"; -const fixture = vi.hoisted(() => ({ rows: [] as Record[] })); +const fixture = vi.hoisted(() => ({ + rows: [] as Record[], + batches: [] as Record[][], +})); vi.mock("../../db", () => ({ - db: { select: () => ({ from: () => ({ where: () => ({ limit: async () => fixture.rows }) }) }) }, + db: { + select: () => ({ + from: () => ({ + where: () => ({ + limit: async () => (fixture.batches.length ? fixture.batches.shift()! : fixture.rows), + }), + }), + }), + }, })); import { cancelDataManagerActivityJob } from "../activity-jobs"; @@ -46,6 +57,7 @@ afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); fixture.rows = []; + fixture.batches = []; }); it("keeps an interrupted unknown durable Activity execution explicitly cancelable", async () => { vi.stubEnv("DATA_MANAGER_URL", "https://synthetic-dm.test"); @@ -206,3 +218,67 @@ it("keeps malformed or unknown backend terminal metadata behind the finalization }), ).toBe(true); }); + +it("discovers and cancels Full Update's original nested generation leaf", async () => { + vi.stubEnv("DATA_MANAGER_URL", "https://synthetic-dm.test"); + const cliKey = controlChildOperationKey(key, "data-finalization", 1), + dataKey = controlChildOperationKey(cliKey, "hardlinks", 0), + owned = { ...child(1), operationKey: dataKey, kind: "generation-hardlinks" }, + ctx = context({ _cancelRequestedAt: new Date().toISOString() }); + fixture.batches = [ + [], + [], + [], + [ + { + id: owned.id, + kind: owned.kind, + status: "running", + operationKey: dataKey, + executionFingerprint: owned.fingerprint, + admittedAt: at, + deadlineAt: deadline, + executionVersion: 2, + executionParent: { + issuer: "host-cli", + execution: { executor: "host-cli", operationKey: cliKey }, + }, + executionState: { + cancellationRequestedAt: null, + progress: null, + diagnostics: { tail: [], truncated: false }, + publication: "not_started", + }, + }, + ], + ]; + let canceled = false; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, options?: RequestInit) => { + if (options?.method === "POST") canceled = true; + return Response.json({ + id: owned.id, + operationKey: dataKey, + fingerprint: owned.fingerprint, + admittedAt: at, + deadlineAt: deadline, + kind: owned.kind, + status: canceled ? "canceled" : "running", + cancellationRequestedAt: canceled ? new Date().toISOString() : null, + progress: null, + diagnostics: { tail: [], truncated: false }, + publication: "not_started", + }); + }), + ); + expect( + await cancelAdminDataChildren( + ctx.checkpointResult, + [{ kind: "data.update", operationKey: key }], + { context: ctx, sleep: async () => {} }, + ), + ).toBe("contained"); + expect(canceled).toBe(true); + expect(fixture.batches).toHaveLength(0); +}); diff --git a/apps/api/src/services/admin-data-children.ts b/apps/api/src/services/admin-data-children.ts index b49f2e16e..f6cf64d16 100644 --- a/apps/api/src/services/admin-data-children.ts +++ b/apps/api/src/services/admin-data-children.ts @@ -61,7 +61,9 @@ export async function cancelAdminDataChildren( ); if (!known.success) throw new JobObservationPending("invalid_identity"); const children = known.data; - const candidates = roots.filter((root) => isFixedSingleDataCommand(root.kind)); + const candidates = roots.filter( + (root) => isFixedSingleDataCommand(root.kind) || root.kind === "data.update", + ); if (!children.length && !candidates.length) return "none"; if (!options.context) return "pending"; const persist = async () => { @@ -74,67 +76,87 @@ export async function cancelAdminDataChildren( // DM receiver wrote this parent binding; an absent row remains unknown to its // agent-root reconciliation and never authorizes another POST. for (const root of candidates) { - const cliKey = controlChildOperationKey(root.operationKey, "data-workflow", 0), - key = controlChildOperationKey(cliKey, "data-step", 0); - const rows = await db - .select() - .from(dataManagerJobs) - .where(eq(dataManagerJobs.operationKey, key)) - .limit(2); - if (rows.length > 1) throw new JobObservationPending("invalid_identity"); - const row = rows[0]; - if (!row) continue; - const parent = z - .object({ - issuer: z.literal("host-cli"), - execution: z.object({ executor: z.literal("host-cli"), operationKey: z.literal(cliKey) }), - }) - .safeParse(row.executionParent); - if (row.executionVersion !== 2 || !parent.success) - throw new JobObservationPending("invalid_identity"); - const state = dataExecutionStatusSchema.parse({ - id: String(row.id), - kind: row.kind, - status: row.status, - operationKey: row.operationKey, - fingerprint: row.executionFingerprint, - admittedAt: Number(row.admittedAt), - deadlineAt: Number(row.deadlineAt), - ...z + const dataCliKey = controlChildOperationKey(root.operationKey, "data-workflow", 0), + finalizationCliKey = controlChildOperationKey(root.operationKey, "data-finalization", 1); + const addresses = + root.kind === "data.update" + ? [ + ...(["osm", "transit", "fonts"] as const).map((phase, ordinal) => ({ + parentKey: dataCliKey, + operationKey: controlChildOperationKey(dataCliKey, phase, ordinal), + })), + { + parentKey: finalizationCliKey, + operationKey: controlChildOperationKey(finalizationCliKey, "hardlinks", 0), + }, + ] + : [ + { + parentKey: dataCliKey, + operationKey: controlChildOperationKey(dataCliKey, "data-step", 0), + }, + ]; + for (const { parentKey: cliKey, operationKey: key } of addresses) { + const rows = await db + .select() + .from(dataManagerJobs) + .where(eq(dataManagerJobs.operationKey, key)) + .limit(2); + if (rows.length > 1) throw new JobObservationPending("invalid_identity"); + const row = rows[0]; + if (!row) continue; + const parent = z .object({ - cancellationRequestedAt: z.string().nullable(), - progress: z.number().nullable(), - diagnostics: z.unknown(), - publication: z.string(), - result: z.unknown().optional(), + issuer: z.literal("host-cli"), + execution: z.object({ executor: z.literal("host-cli"), operationKey: z.literal(cliKey) }), }) - .parse(row.executionState), - }); - const child = childSchema.parse({ - id: state.id, - operationKey: key, - fingerprint: state.fingerprint, - parentAgentOperationKey: root.operationKey, - kind: state.kind, - admittedAt: state.admittedAt, - deadlineAt: state.deadlineAt, - cancellationRequestedAt: null, - state: isDataExecutionTerminal(state) ? "terminal" : "observing", - status: state.status, - publication: state.publication, - }), - old = children.find((item) => item.operationKey === key); - if (old) { - if ( - old.id !== child.id || - old.fingerprint !== child.fingerprint || - old.kind !== child.kind || - old.admittedAt !== child.admittedAt || - old.deadlineAt !== child.deadlineAt || - old.parentAgentOperationKey !== root.operationKey - ) + .safeParse(row.executionParent); + if (row.executionVersion !== 2 || !parent.success) throw new JobObservationPending("invalid_identity"); - } else children.push(child); + const state = dataExecutionStatusSchema.parse({ + id: String(row.id), + kind: row.kind, + status: row.status, + operationKey: row.operationKey, + fingerprint: row.executionFingerprint, + admittedAt: Number(row.admittedAt), + deadlineAt: Number(row.deadlineAt), + ...z + .object({ + cancellationRequestedAt: z.string().nullable(), + progress: z.number().nullable(), + diagnostics: z.unknown(), + publication: z.string(), + result: z.unknown().optional(), + }) + .parse(row.executionState), + }); + const child = childSchema.parse({ + id: state.id, + operationKey: key, + fingerprint: state.fingerprint, + parentAgentOperationKey: root.operationKey, + kind: state.kind, + admittedAt: state.admittedAt, + deadlineAt: state.deadlineAt, + cancellationRequestedAt: null, + state: isDataExecutionTerminal(state) ? "terminal" : "observing", + status: state.status, + publication: state.publication, + }), + old = children.find((item) => item.operationKey === key); + if (old) { + if ( + old.id !== child.id || + old.fingerprint !== child.fingerprint || + old.kind !== child.kind || + old.admittedAt !== child.admittedAt || + old.deadlineAt !== child.deadlineAt || + old.parentAgentOperationKey !== root.operationKey + ) + throw new JobObservationPending("invalid_identity"); + } else children.push(child); + } } childrenSchema.parse(children); if (!children.length) return "none"; diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 9282653d2..80863320d 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -1,4 +1,5 @@ import { loadConfiguredResourceAuthority } from "./configured-resource-authority"; +import { installDataWorkflowRuntime } from "./data-workflow-runtime"; export { type ConfiguredResourceAuthority, @@ -1929,11 +1930,8 @@ export function createAdministrativeRuntime( await runMatchingDataCommand(operation, context); return { completed: true }; }; - // Full Update build/link finalization is the reviewed Task02.4 owner. Until - // then reject before any download rather than report partial workflow success. - runtime["data.update"] = async () => { - throw new OpsHostUpdateRequiredError(); - }; + // Full Update owns its target-free data phase and one guarded finalization phase. + installDataWorkflowRuntime(runtime, { rootDir: options.rootDir, runFixedCli }); runtime["data.convertOverpass"] = async (operation, context) => { await runMatchingDataCommand(operation, context); return { completed: true, ...(operation.regionId ? { resourceId: operation.regionId } : {}) }; diff --git a/apps/ops-agent/src/configuration-maintenance-runtime.ts b/apps/ops-agent/src/configuration-maintenance-runtime.ts index 4310f47e5..a7c7aacc7 100644 --- a/apps/ops-agent/src/configuration-maintenance-runtime.ts +++ b/apps/ops-agent/src/configuration-maintenance-runtime.ts @@ -1,6 +1,7 @@ /** Stable production maintenance runtime interface. Actual nominal producers * live beside the private owner they alone can initialize/settle. */ export { + assertStagedCoordinatorConfigurationWrite, type ConfigurationMaintenanceAdmission, configurationMaintenanceAdmissionState, consumeMaintenanceContinueObservation, diff --git a/apps/ops-agent/src/data-workflow-runtime.test.ts b/apps/ops-agent/src/data-workflow-runtime.test.ts new file mode 100644 index 000000000..757ffb8dc --- /dev/null +++ b/apps/ops-agent/src/data-workflow-runtime.test.ts @@ -0,0 +1,140 @@ +import { randomBytes } from "node:crypto"; +import { existsSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + createControlRootOperationKey, + executionFingerprint, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { afterEach, expect, it, vi } from "vitest"; +import { ownedServiceHost } from "./__fixtures__/owned-service-host"; + +const roots: string[] = []; +afterEach(() => { + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); +it("freezes distinct original data/finalization slots and refuses targets before all actual data witnesses", async () => { + const host = await ownedServiceHost({ withCoordinator: true }); + roots.push(host.cleanupRoot); + const installation = control.openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id).sort(), + }); + try { + const operation = { + kind: "data.update" as const, + regionId: "europe/germany", + countryCodes: ["DE"], + }, + admittedAt = Date.now(), + deadlineAt = admittedAt + 3600000, + operationFingerprint = opsOperationFingerprint(operation); + const parent = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${"a".repeat(24)}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const ids = host.loaded.map((s) => s.manifest.id).sort(), + plan = control.planDataUpdate({ + infraDir: host.infraDir, + parent, + operation, + serviceIds: ids, + }); + expect(plan.dataChildren.map((s) => s.ordinal)).toEqual([0, 1, 2]); + expect(plan.finalizationReservation.ordinal).toBe(1); + const authority = control.retainAdmittedControlExecution(installation, { + execution: parent, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ids, + reservations: [plan.dataReservation, plan.finalizationReservation], + }); + control.retainDataUpdatePlan(authority, plan); + expect(control.readDataUpdatePlan(authority)).toEqual(plan); + await expect( + control.acquireArtifactTargetLease({ + dataRoot: join(host.infraDir, "data"), + owner: authority, + serviceIds: ids, + }), + ).rejects.toThrow("data phase not admitted"); + expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager"))).toBe( + false, + ); + const child = await control.bindControlChildPhase(authority, plan.dataReservation), + key = control.prepareFixedControlChild(authority, plan.dataDelegationId); + control.retainFixedCliSpawnIntent(authority, { + delegationId: plan.dataDelegationId, + execution: child.execution, + operation, + args: control.fixedCliArguments(operation), + reservations: plan.dataChildren, + dataUpdatePlan: plan, + }); + const issued = await control.issueControlChildPhase(authority, { + phase: "data-workflow", + ordinal: 0, + childPublicKey: key, + }); + control.consumeControlChildDispatch(issued.dispatchTicket); + // An issued original CLI is still not a completed data prerequisite. + await expect( + control.acquireArtifactTargetLease({ + dataRoot: join(host.infraDir, "data"), + owner: authority, + serviceIds: ids, + }), + ).rejects.toThrow(); + expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager"))).toBe( + false, + ); + } finally { + control.closeControlInstallation(installation); + } +}); + +it("freezes the same host .env defaults as the registered CLI, with shell precedence", async () => { + const host = await ownedServiceHost({ withCoordinator: true }); + roots.push(host.cleanupRoot); + writeFileSync( + join(host.infraDir, ".env"), + "DOMAIN=map.example.test\nOPENMAPX_REGION=europe/germany\nTRANSITOUS_COUNTRIES=DE,FR\n", + { mode: 0o600 }, + ); + vi.stubEnv("DOMAIN", undefined); + const operation = { kind: "data.update" as const }, + deadlineAt = Date.now() + 60000, + operationFingerprint = opsOperationFingerprint(operation); + const parent = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${"b".repeat(24)}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const args = { + infraDir: host.infraDir, + parent, + operation, + serviceIds: host.loaded.map((s) => s.manifest.id).sort(), + }; + expect(control.planDataUpdate(args)).toMatchObject({ + domain: "map.example.test", + region: "europe/germany", + countries: ["DE", "FR"], + }); + vi.stubEnv("DOMAIN", "shell.example.test"); + expect(control.planDataUpdate(args).domain).toBe("shell.example.test"); +}); diff --git a/apps/ops-agent/src/data-workflow-runtime.ts b/apps/ops-agent/src/data-workflow-runtime.ts new file mode 100644 index 000000000..67abe847c --- /dev/null +++ b/apps/ops-agent/src/data-workflow-runtime.ts @@ -0,0 +1,152 @@ +/** Full Update runs its data and target-owning phases through the same real CLI. + * An attempted invocation is observation-only, including after a lost response. */ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import type { DataWorkflowReceipt, ExecutionRef, OpsOperation } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import type { FixedCliRunner } from "./administrative-runtime"; +import { assertStagedCoordinatorConfigurationWrite } from "./configuration-maintenance-runtime"; +import { OpsExecutionFailure, OpsHostUpdateRequiredError, type OpsRuntime } from "./runtime"; + +const sources = new WeakMap(); +export function prepareDataUpdateAdmission( + runtime: OpsRuntime, + parent: ExecutionRef, + operation: Extract, + serviceIds: readonly string[], +) { + const source = sources.get(runtime); + if (!source) throw new OpsHostUpdateRequiredError(); + return control.planDataUpdate({ + infraDir: join(source.rootDir, "infra/docker"), + parent, + operation, + serviceIds, + defaultRegion: process.env.OPENMAPX_REGION, + defaultCountries: process.env.TRANSITOUS_COUNTRIES, + }); +} +export function installDataWorkflowRuntime( + runtime: OpsRuntime, + options: { rootDir: string; runFixedCli: FixedCliRunner }, +) { + sources.set(runtime, options); + runtime["data.update"] = async (operation, context) => { + const authority = context.controlAuthority; + if (!authority) throw new OpsHostUpdateRequiredError(); + const infraDir = join(options.rootDir, "infra/docker"), + plan = control.readDataUpdatePlan(authority), + args = control.fixedCliArguments(operation); + const progress = await services.openAdminDataWorkflow({ + dataRoot: join(infraDir, "data"), + operationKey: authority.execution.operationKey, + fingerprint: authority.execution.fingerprint, + authority, + }); + const checkpoint = async (receipt: DataWorkflowReceipt) => { + await progress.checkpoint(receipt); + await context.checkpointDataWorkflow?.(receipt); + }; + const run = async ( + phase: "data-workflow" | "data-finalization", + lease?: control.ArtifactTargetLease, + initial?: DataWorkflowReceipt, + ) => { + const slot = phase === "data-workflow" ? plan.dataReservation : plan.finalizationReservation, + delegationId = + phase === "data-workflow" ? plan.dataDelegationId : plan.finalizationDelegationId; + const bound = await control.bindControlChildPhase(authority, slot); + let attempted = control.lookupControlChildPhaseAttempt(authority, slot), + failure: unknown; + if (!attempted) { + const key = control.prepareFixedControlChild(authority, delegationId); + control.retainFixedCliSpawnIntent(authority, { + delegationId, + execution: bound.execution, + operation, + args, + reservations: phase === "data-workflow" ? plan.dataChildren : plan.finalizationChildren, + dataUpdatePlan: plan, + ...(initial ? { dataWorkflowInitial: initial } : {}), + }); + const issued = await control.issueControlChildPhase(authority, { + phase, + ordinal: slot.ordinal, + childPublicKey: key, + ...(lease ? { lease } : {}), + }); + control.consumeControlChildDispatch(issued.dispatchTicket); + try { + await options.runFixedCli(args, { + ...context, + execution: bound.execution, + operationKind: "data.update", + ...(lease + ? { controlAssertion: issued.assertion } + : { parentAssertion: issued.assertion }), + }); + } catch (error) { + failure = error; + } + attempted = control.lookupControlChildPhaseAttempt(authority, slot); + } + if (!attempted) throw new OpsExecutionFailure("recovery_required", {}); + const installation = control.openFixedChildInstallation({ + infraDir, + assertion: attempted.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const childAuthority = control.restoreAdmittedControlExecution( + installation, + bound.execution, + ), + store = await services.openAdminDataWorkflow({ + dataRoot: join(infraDir, "data"), + operationKey: bound.execution.operationKey, + fingerprint: bound.execution.fingerprint, + authority: childAuthority, + }), + receipt = store.read(); + if (receipt) await checkpoint(receipt); + if ( + failure || + !receipt || + receipt.phase !== (phase === "data-workflow" ? "build" : "complete") + ) + throw new OpsExecutionFailure( + "recovery_required", + receipt ? { dataWorkflow: receipt } : {}, + ); + control.verifyControlEffectSettlement(installation, bound.execution); + if (lease) { + const settled = control.inspectArtifactTargetLeaseChildSettlement(lease, bound.execution); + if (settled?.outcome !== "committed" || settled.publication !== "committed") + throw new OpsExecutionFailure("recovery_required", { dataWorkflow: receipt }); + } + return receipt; + } finally { + control.closeControlInstallation(installation); + } + }; + await assertStagedCoordinatorConfigurationWrite(runtime, operation, context, context.signal); + await run("data-workflow"); + context.signal.throwIfAborted(); + const initial = await control.verifyDataUpdatePrerequisites(authority); + await assertStagedCoordinatorConfigurationWrite(runtime, operation, context, context.signal); + const lease = await control.acquireArtifactTargetLease({ + dataRoot: join(infraDir, "data"), + serviceIds: plan.dataReservation.serviceIds, + owner: authority, + signal: context.signal, + }); + context.ownDataFinalizationLease?.(lease); + try { + const receipt = await run("data-finalization", lease, initial); + return { completed: true, dataWorkflow: receipt }; + } finally { + if (!context.ownDataFinalizationLease) control.closeArtifactTargetLeaseHandle(lease); + } + }; +} diff --git a/apps/ops-agent/src/journal.test.ts b/apps/ops-agent/src/journal.test.ts index a971d2ecd..506c3f47a 100644 --- a/apps/ops-agent/src/journal.test.ts +++ b/apps/ops-agent/src/journal.test.ts @@ -417,3 +417,56 @@ it("reloads strict v2 failed evidence and rejects it on a successful record", as ]), ).rejects.toThrow(); }); +it("retains Full Update progress while running, then keeps it as interrupted evidence without granting completion", async () => { + const { normalizeExecutionAdmission } = await import("@openmapx/core/ops"); + const operation = { kind: "data.update" as const, regionId: "europe/germany" }, + operationKey = "opk1_workflowProgress123", + operationFingerprint = opsOperationFingerprint(operation), + admittedAt = Date.parse("2026-10-28T10:00:00.000Z"); + const normalized = normalizeExecutionAdmission({ + kind: operation.kind, + operationKey, + operationFingerprint, + admittedAt, + requestedDeadlineAt: admittedAt + 60000, + }); + const progressResult = { + dataWorkflow: { + version: 1 as const, + phase: "transit" as const, + children: [], + completedBuildIds: [], + failedBuildIds: [], + skippedBuildIds: [], + }, + }; + const record = { + ...job({ + operation, + operationKey, + operationFingerprint, + resourceId: "europe/germany", + ...normalized, + requestedDeadlineAt: admittedAt + 60000, + executionVersion: 2, + requestVersion: 2, + deploymentBudgetMs: 24 * 3600000, + state: "running", + submittedAt: new Date(admittedAt).toISOString(), + updatedAt: new Date(admittedAt).toISOString(), + }), + progressResult, + }; + const path = join(root(), "journal/jobs-v1.json"), + journal = await openOpsJobJournal(path); + await journal.replace([record]); + expect(journal.records()[0]).toMatchObject({ state: "running", progressResult }); + const reopened = await openOpsJobJournal(path, { now: () => new Date(admittedAt + 1000) }); + expect(reopened.records()[0]).toMatchObject({ + state: "failed", + errorClass: "recovery_required", + failureResult: progressResult, + }); + expect(reopened.records()[0]).not.toHaveProperty("progressResult"); + expect(reopened.records()[0]).not.toHaveProperty("result"); +}); diff --git a/apps/ops-agent/src/journal.ts b/apps/ops-agent/src/journal.ts index 3d8c7cfce..af48f12f7 100644 --- a/apps/ops-agent/src/journal.ts +++ b/apps/ops-agent/src/journal.ts @@ -57,6 +57,8 @@ export interface PersistedOpsJob { parent?: RetainedExecutionAdmission["parent"]; diagnostics?: ProcessDiagnostics; failureResult?: OpsFailureResult; + /** Private checkpoint, never a terminal outcome or authority. */ + progressResult?: OpsFailureResult; role: OpsRole; operation: OpsOperation; operationId: string; @@ -141,6 +143,7 @@ function parseJob(value: unknown): RetainedOpsJob { "parent", "diagnostics", "failureResult", + "progressResult", "role", "operation", "operationId", @@ -222,6 +225,23 @@ function parseJob(value: unknown): RetainedOpsJob { ) throw new Error("invalid"); if (raw.diagnostics !== undefined) processDiagnosticsSchema.parse(raw.diagnostics); + if (raw.progressResult !== undefined) { + const progress = opsFailureResultSchema.parse(raw.progressResult); + const allowed = + operation.kind === "data.update" + ? "dataWorkflow" + : operation.kind === "services.buildAll" + ? "serviceBatch" + : null; + if ( + raw.executionVersion !== 2 || + !allowed || + !["queued", "running", "termination_pending"].includes(raw.state as string) || + Object.keys(progress).some((key) => key !== allowed) || + Buffer.byteLength(JSON.stringify(progress)) > OPS_KIND_POLICIES[operation.kind].maxResultBytes + ) + throw Error("invalid"); + } if (raw.failureResult !== undefined) { if (!["failed", "timed_out"].includes(raw.state as string)) throw new Error("invalid"); const result = opsFailureResultSchema.parse(raw.failureResult); @@ -259,6 +279,7 @@ function parseJob(value: unknown): RetainedOpsJob { ) { throw new Error("invalid"); } + if (raw.progressResult === undefined) delete raw.progressResult; return raw as unknown as RetainedOpsJob; } @@ -485,6 +506,8 @@ export async function openOpsJobJournal( entry.terminationRequestedAt ?? new Date(entry.deadlineAt).toISOString(), } : {}), + ...(entry.progressResult ? { failureResult: entry.progressResult } : {}), + progressResult: undefined, state: "failed" as const, updatedAt: timestamp, terminalAt: timestamp, diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index 29f7c762d..de15ae27f 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -70,6 +70,8 @@ export interface OpsExecutionContext { otherActiveOperations?(): number; /** Persist progress before a batch can start its next effect. */ checkpointServiceBatch?(result: ServiceBatchResult): Promise; + checkpointDataWorkflow?(result: import("@openmapx/core/ops").DataWorkflowReceipt): Promise; + ownDataFinalizationLease?(lease: ArtifactTargetLease): void; } export interface OpsTrustedClaim { diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index c36256fd3..10a27fcd4 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -2,6 +2,7 @@ import { createHash, randomBytes as nodeRandomBytes, timingSafeEqual } from "nod import { acknowledgeControlRetention, acquireArtifactTargetLease, + assertArtifactTargetLeaseExecution, assertArtifactTargetLeaseForward, assertFreshControlRoot, beginControlEffect, @@ -37,6 +38,7 @@ import { retainControlPreparedTerminal, retainControlTerminalCommitment, retainControlTerminalStatus, + retainDataUpdatePlan, retainedControlAdoptionAssertion, retainFixedServiceLifecyclePlan, retainFixedSingleDataPlan, @@ -125,6 +127,7 @@ import { installCoordinatorStagedOwnershipSource, prepareCoordinatorStagedRetention, } from "./coordinator-staged-ownership"; +import { prepareDataUpdateAdmission } from "./data-workflow-runtime"; import { type DawarichSubjectExportRouteOptions, registerDawarichSubjectExportRoute, @@ -562,6 +565,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif : {}), ...(job.diagnostics ? { diagnostics: job.diagnostics } : {}), ...(job.failureResult ? { failureResult: job.failureResult } : {}), + ...(job.progressResult ? { progressResult: job.progressResult } : {}), role: job.role, operation: job.operation, operationId: job.operationId, @@ -822,7 +826,16 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (lease.mode === "owner" && unstartedConfiguration) releaseUnstartedRoot = true; } } + if (["failed", "timed_out"].includes(state) && job.progressResult) + updates = { + ...updates, + failureResult: opsFailureResultSchema.parse({ + ...job.progressResult, + ...updates.failureResult, + }), + }; const candidate: JobRecord = { ...job, ...updates, state, updatedAt }; + if (["succeeded", "failed", "timed_out"].includes(state)) delete candidate.progressResult; if (state === "succeeded" && job.operation.kind === "services.buildAll") delete candidate.failureResult; if (authority && ["succeeded", "failed", "timed_out"].includes(state)) { @@ -848,6 +861,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ) retainControlPreparedTerminal(authority); Object.assign(job, updates); + if (["succeeded", "failed", "timed_out"].includes(state)) delete job.progressResult; if (state === "succeeded" && job.operation.kind === "services.buildAll") delete job.failureResult; job.updatedAt = updatedAt; @@ -929,6 +943,28 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif artifactLease: borrowedLeaseFor(job.operationId), ownedTargetLease: ownedLeaseFor(job.operationId), controlAuthority: controlAuthorities.get(job.operationId), + ...(job.operation.kind === "data.update" + ? { + ownDataFinalizationLease( + lease: import("@openmapx/core/build-artifacts").ArtifactTargetLease, + ) { + if (controlLeases.has(job.operationId)) + throw Error("Original Full Update lease already retained"); + assertArtifactTargetLeaseExecution(lease, executionFor(job)); + controlLeases.set(job.operationId, lease); + }, + async checkpointDataWorkflow( + dataWorkflow: import("@openmapx/core/ops").DataWorkflowReceipt, + ) { + const evidence = opsFailureResultSchema.parse({ ...job.progressResult, dataWorkflow }); + await withAdmissionLock(async () => { + job.progressResult = evidence; + job.updatedAt = now().toISOString(); + await journal.replace([...jobsById.values()].map(persisted)); + }); + }, + } + : {}), ...(maintenanceControls.has(job.operationId) ? { configurationMaintenanceControl: maintenanceControls.get(job.operationId) } : {}), @@ -963,14 +999,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif checkpointServiceBatch: async ( serviceBatch: import("@openmapx/core/ops").ServiceBatchResult, ) => { - const evidence = opsFailureResultSchema.parse({ ...job.failureResult, serviceBatch }); + const evidence = opsFailureResultSchema.parse({ ...job.progressResult, serviceBatch }); if ( Buffer.byteLength(JSON.stringify(evidence)) > OPS_KIND_POLICIES[job.operation.kind].maxResultBytes ) throw Error("Service batch evidence exceeds budget"); await withAdmissionLock(async () => { - job.failureResult = evidence; + job.progressResult = evidence; job.updatedAt = now().toISOString(); await journal.replace([...jobsById.values()].map(persisted)); }); @@ -2425,6 +2461,10 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif process.env.OPENMAPX_REGION, ) : undefined; + const dataUpdatePlan = + job.operation.kind === "data.update" + ? prepareDataUpdateAdmission(runtime, executionFor(job), job.operation, targets) + : undefined; const hardlinksPlan = job.operation.kind === "configuration.hardlinks.apply" ? prepareConfigurationHardlinksAdmission( @@ -2452,6 +2492,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif parent: job.parent, serviceIds: targets, ...(fixedDataPlan ? { reservations: [fixedDataPlan.rootReservation] } : {}), + ...(dataUpdatePlan + ? { + reservations: [ + dataUpdatePlan.dataReservation, + dataUpdatePlan.finalizationReservation, + ], + } + : {}), ...(hardlinksPlan ? { reservations: hardlinksPlan.reservation ? [hardlinksPlan.reservation] : [] } : {}), @@ -2460,6 +2508,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ...(coordinatorBaseline ? { coordinatorPullBaseline: coordinatorBaseline } : {}), }); if (fixedDataPlan) retainFixedSingleDataPlan(authority, fixedDataPlan); + if (dataUpdatePlan) retainDataUpdatePlan(authority, dataUpdatePlan); if (hardlinksPlan) retainGenerationHardlinksOwnerPlan(authority, hardlinksPlan); if (lifecyclePlan) retainFixedServiceLifecyclePlan(authority, lifecyclePlan); requireControlJournalRetention(authority); diff --git a/packages/cli/__tests__/cli-build-staging.test.ts b/packages/cli/__tests__/cli-build-staging.test.ts index b4e3af9e8..9808522f1 100644 --- a/packages/cli/__tests__/cli-build-staging.test.ts +++ b/packages/cli/__tests__/cli-build-staging.test.ts @@ -2,7 +2,11 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { prepareStagedBuildDir, promoteStagedBuildDirs } from "../src/lib/build-staging"; +import { + prepareStagedBuildDir, + promoteStagedBuildDir, + promoteStagedBuildDirs, +} from "../src/lib/build-staging"; let tmp: string; @@ -66,3 +70,62 @@ describe("staged build dirs", () => { expect(readFileSync(join(aNext, "file"), "utf-8")).toBe("NEW-A"); }); }); +it("keeps staged and live bytes unchanged when the retained parent cancels before publication", async () => { + const { randomBytes, randomUUID } = await import("node:crypto"), + control = await import("@openmapx/core/build-artifacts"), + ops = await import("@openmapx/core/ops"), + { withHostDataControlContext } = await import("../src/lib/control-context"); + const ownerRoot = mkdtempSync(join(tmpdir(), "omx-canceled-build-")); + mkdirSync(join(ownerRoot, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(ownerRoot); + const installation = control.openControlInstallation({ + infraDir: ownerRoot, + role: "host-cli", + serviceIds: ["osrm"], + }); + try { + const operation = { kind: "service.build" as const, serviceId: "osrm" }, + admittedAt = Date.now(), + deadlineAt = admittedAt + 60000, + operationFingerprint = ops.opsOperationFingerprint(operation), + execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: ops.createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: ops.executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }; + const authority = control.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["osrm"], + }); + const liveDir = join(ownerRoot, "data/graph"); + mkdirSync(liveDir); + writeFileSync(join(liveDir, "old"), "old"); + const nextDir = prepareStagedBuildDir(liveDir); + writeFileSync(join(nextDir, "new"), "new"); + control.cancelControlExecution(authority); + await withHostDataControlContext(installation, authority, async () => { + expect(() => promoteStagedBuildDir(liveDir, nextDir)).toThrow(); + expect(() => prepareStagedBuildDir(liveDir)).toThrow(); + }); + expect(readFileSync(join(liveDir, "old"), "utf8")).toBe("old"); + expect(readFileSync(join(nextDir, "new"), "utf8")).toBe("new"); + } finally { + control.closeControlInstallation(installation); + rmSync(ownerRoot, { recursive: true, force: true }); + } +}); diff --git a/packages/cli/__tests__/cli-coordinator-ready.test.ts b/packages/cli/__tests__/cli-coordinator-ready.test.ts new file mode 100644 index 000000000..f8da5112c --- /dev/null +++ b/packages/cli/__tests__/cli-coordinator-ready.test.ts @@ -0,0 +1,98 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { rmSync } from "node:fs"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { createDataManagerTransport } from "@openmapx/core/data-manager-sdk"; +import * as ops from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { expect, it, vi } from "vitest"; +import { ownedServiceHost } from "../../../apps/ops-agent/src/__fixtures__/owned-service-host"; + +const io = vi.hoisted(() => ({ start: vi.fn(async () => 0) })); +vi.mock("../src/lib/docker", () => ({ dockerComposeStream: io.start })); +it("starts an unavailable coordinator without recreation under actual existing ownership", async () => { + const { ensureCliCoordinatorReady } = await import("../src/lib/coordinator-ready"); + const host = await ownedServiceHost({ withCoordinator: true }), + installation = control.openControlInstallation({ + infraDir: host.infraDir, + role: "host-cli", + serviceIds: ["data-manager", "motis"], + }); + let lease: control.ArtifactTargetLease | undefined; + try { + const operation = { + kind: "service.start" as const, + serviceId: "motis", + expectedConfigurationRevisionId: host.initial.revisionId, + }, + admittedAt = Date.now(), + deadlineAt = admittedAt + 60000, + operationFingerprint = ops.opsOperationFingerprint(operation), + execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: ops.createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: ops.executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }; + const authority = control.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["data-manager", "motis"], + }); + lease = await control.acquireArtifactTargetLease({ + dataRoot: join(host.infraDir, "data"), + owner: authority, + serviceIds: ["data-manager", "motis"], + }); + let reads = 0; + const transport = createDataManagerTransport({ + baseUrl: "http://127.0.0.1:1", + fetch: async () => { + reads++; + return reads === 1 + ? Response.json( + { + ok: false, + status: "starting", + phase: "job-reconciliation", + uptime: 0, + dataDir: "/synthetic", + }, + { status: 503 }, + ) + : Response.json({ + ok: true, + status: "ready", + phase: "complete", + uptime: 0, + dataDir: "/synthetic", + }); + }, + }); + await services.withConfigurationGenerationWriter(host.infraDir, (writer) => + services.withVerifiedLifecycleOwnership({ writer, lease: lease!, authority }, (ownership) => + ensureCliCoordinatorReady({ ownership, transport, mode: "ensure-running" }), + ), + ); + expect(io.start).toHaveBeenCalledExactlyOnceWith(["up", "-d", "--no-recreate", "data-manager"]); + expect(reads).toBe(2); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + } finally { + if (lease) control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(installation); + rmSync(host.cleanupRoot, { recursive: true, force: true }); + } +}); diff --git a/packages/cli/__tests__/cli-data-update-pipeline.test.ts b/packages/cli/__tests__/cli-data-update-pipeline.test.ts new file mode 100644 index 000000000..8202c2658 --- /dev/null +++ b/packages/cli/__tests__/cli-data-update-pipeline.test.ts @@ -0,0 +1,158 @@ +import { randomUUID } from "node:crypto"; +import { controlChildOperationKey, fixedDataBodyFingerprint } from "@openmapx/core/build-artifacts"; +import { + type DataExecutionInput, + dataExecutionStatusSchema, +} from "@openmapx/core/data-manager-contracts"; +import { type ExecutionRef, executionFingerprint } from "@openmapx/core/ops"; +import { expect, it, vi } from "vitest"; +import { type DataUpdateEffects, DataUpdateFailure, runDataUpdate } from "../src/lib/data-update"; + +const execution: ExecutionRef = { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: `opk1_${"a".repeat(43)}`, + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 3600000, +}; +const options = { region: "europe/germany", countries: ["DE"], failFast: false, execution }; +function status(step: "osm" | "transit" | "fonts", input: DataExecutionInput) { + const deadlineAt = execution.deadlineAt; + return dataExecutionStatusSchema.parse({ + id: randomUUID(), + operationKey: controlChildOperationKey( + execution.operationKey, + step, + ["osm", "transit", "fonts"].indexOf(step), + ), + kind: input.kind, + status: "ok", + publication: "committed", + admittedAt: Date.now(), + deadlineAt, + cancellationRequestedAt: null, + progress: 1, + diagnostics: { tail: [], truncated: false }, + result: + input.kind === "download-osm" + ? { kind: input.kind, region: "europe/germany", sizeBytes: 1 } + : input.kind === "transit-sync" + ? { + kind: input.kind, + finalStatus: "ok", + sourceRevision: "pinned-catalog", + countries: ["de"], + } + : { kind: "download-fonts", completed: true }, + fingerprint: executionFingerprint({ + kind: input.kind, + operationFingerprint: fixedDataBodyFingerprint(input), + deadlineAt, + parent: { + issuer: execution.executor, + execution, + operationKey: execution.operationKey, + fingerprint: execution.fingerprint, + deadlineAt: execution.deadlineAt, + }, + }), + }); +} +function effects() { + const e = { + checkpoint: vi.fn(async () => {}), + checkpointAttempt: vi.fn(async () => {}), + runDataStep: vi.fn(async (step: "osm" | "transit" | "fonts", input: DataExecutionInput) => + status(step, input), + ), + build: vi.fn(async () => ({ + outcome: "success" as const, + completedServiceIds: ["tileserver"], + failedServiceIds: [], + skippedServiceIds: [], + })), + prepare: vi.fn(async () => ({ + revisionId: `cfg1_${"r".repeat(43)}`, + coordinatorMode: "ensure-running" as const, + hardlinks: { applied: true, linked: 1, skipped: 0, pruned: 0, entries: 1 }, + })), + }; + return e; +} +it("checkpoints and completes data -> graphs -> generation/hardlinks in order", async () => { + const e = effects(); + const receipt = await runDataUpdate(options, e); + expect(e.runDataStep.mock.calls.map(([step]) => step)).toEqual(["osm", "transit", "fonts"]); + expect(e.runDataStep).toHaveBeenCalledBefore(e.build); + expect(e.build).toHaveBeenCalledBefore(e.prepare); + expect(e.build).toHaveBeenCalledWith({ region: options.region, continueOnError: true }); + expect(receipt).toMatchObject({ + phase: "complete", + completedBuildIds: ["tileserver"], + revisionId: `cfg1_${"r".repeat(43)}`, + }); + expect(e.checkpoint.mock.calls.length).toBeGreaterThan(5); +}); +it.each(["partial", "error", "interrupted", "canceled", "running"] as const)( + "never builds after transit %s", + async (outcome) => { + const e = effects(); + e.runDataStep.mockImplementation(async (step, input) => ({ + ...status(step, input), + status: step === "transit" ? outcome : "ok", + })); + await expect(runDataUpdate(options, e)).rejects.toBeInstanceOf(DataUpdateFailure); + expect(e.build).not.toHaveBeenCalled(); + expect(e.prepare).not.toHaveBeenCalled(); + expect(e.runDataStep.mock.calls.map(([step]) => step)).toEqual(["osm", "transit"]); + }, +); +it.each(["key", "fingerprint", "deadline"])("rejects a foreign transit %s", async (field) => { + const e = effects(); + e.runDataStep.mockImplementation(async (step, input) => { + const s = status(step, input); + return step !== "transit" + ? s + : { + ...s, + ...(field === "key" + ? { operationKey: `opk1_${"z".repeat(43)}` } + : field === "fingerprint" + ? { fingerprint: "f".repeat(64) } + : { deadlineAt: execution.deadlineAt + 1 }), + }; + }); + await expect(runDataUpdate(options, e)).rejects.toBeInstanceOf(DataUpdateFailure); + expect(e.build).not.toHaveBeenCalled(); +}); +it("retains a continued partial graph result after refreshing successful hardlinks", async () => { + const e = effects(); + e.build.mockResolvedValue({ + outcome: "partial", + completedServiceIds: ["tileserver"], + failedServiceIds: ["valhalla"], + skippedServiceIds: ["motis"], + }); + const error = await runDataUpdate(options, e).catch((e) => e); + expect(error).toBeInstanceOf(DataUpdateFailure); + expect(error.receipt).toMatchObject({ + phase: "link", + completedBuildIds: ["tileserver"], + failedBuildIds: ["valhalla"], + skippedBuildIds: ["motis"], + }); + expect(e.prepare).toHaveBeenCalledOnce(); +}); +it("stops before another data admission if checkpoint persistence fails", async () => { + const e = effects(); + e.checkpoint.mockImplementation(async (receipt) => { + if ((receipt as { phase: string }).phase === "transit") throw Error("Disk failure"); + }); + await expect(runDataUpdate(options, e as DataUpdateEffects)).rejects.toBeInstanceOf( + DataUpdateFailure, + ); + expect(e.runDataStep.mock.calls.map(([step]) => step)).toEqual(["osm"]); + expect(e.build).not.toHaveBeenCalled(); +}); diff --git a/packages/cli/__tests__/cli-data-update.test.ts b/packages/cli/__tests__/cli-data-update.test.ts new file mode 100644 index 000000000..a60e2bd21 --- /dev/null +++ b/packages/cli/__tests__/cli-data-update.test.ts @@ -0,0 +1,41 @@ +import { Command } from "commander"; +import { expect, it, vi } from "vitest"; + +const f = vi.hoisted(() => ({ update: vi.fn(), ok: vi.fn() })); +vi.mock("../src/lib/data-update-command", () => ({ runCliDataUpdate: f.update })); +vi.mock("../src/lib/output", () => ({ + log: { ok: f.ok, err: vi.fn(), dim: vi.fn(), info: vi.fn(), warn: vi.fn() }, + table: vi.fn(), + routeLogsToStderr: vi.fn(), +})); +const { registerDataCommands } = await import("../src/commands/data"); +it("awaits the installed Full Update workflow before printing success", async () => { + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + f.update.mockImplementation(async () => { + await held; + return { phase: "complete" }; + }); + f.ok.mockReset(); + const program = new Command(); + registerDataCommands(program); + const running = program.parseAsync( + ["data", "update", "europe/germany", "--countries", "DE", "--fail-fast"], + { from: "user" }, + ); + try { + await vi.waitFor(() => expect(f.update).toHaveBeenCalled()); + expect(f.ok).not.toHaveBeenCalled(); + expect(f.update).toHaveBeenCalledWith( + expect.objectContaining({ region: "europe/germany", countries: ["DE"], failFast: true }), + ); + release(); + await running; + expect(f.ok).toHaveBeenCalledWith("Update complete"); + } finally { + release(); + await running; + } +}); diff --git a/packages/cli/__tests__/cli-execution-deadline.test.ts b/packages/cli/__tests__/cli-execution-deadline.test.ts index 73b95583d..c39afe782 100644 --- a/packages/cli/__tests__/cli-execution-deadline.test.ts +++ b/packages/cli/__tests__/cli-execution-deadline.test.ts @@ -93,8 +93,11 @@ it("fix1 nested default build runner spends the stored remaining budget across p withCliExecutionDeadline(["services", "build", "osrm"], () => buildOsrmGraph({ rootDir: root, image: "synthetic/image:1" }), ), - ).rejects.toThrow(); - expect(readFileSync(marker, "utf8").trim().split("\n").length).toBeLessThanOrEqual(2); + ).rejects.toThrow(/Execution deadline elapsed|timed out|aborted/i); + const startedPhases = existsSync(marker) + ? readFileSync(marker, "utf8").trim().split("\n").length + : 0; + expect(startedPhases).toBeLessThanOrEqual(2); expect(readFileSync(join(root, "infra/docker/data/osrm-graph/region.osrm"), "utf8")).toBe( "previous-live", ); @@ -147,3 +150,40 @@ it("fix1 unsigned metadata can only clamp policy and cannot authorize a remote m vi.unstubAllEnvs(); } }); +it("passes the retained generation execution to its guarded receiver while legacy link stays closed", async () => { + const { vi } = await import("vitest"), + { withCliExecutionDeadline, constrainCliDataManagerTransport } = await import( + "../src/lib/execution-deadline" + ); + vi.stubEnv("OPENMAPX_CLI_DEADLINE_AT", String(Date.now() + 60000)); + const { createDataManagerTransport } = await import("@openmapx/core/data-manager-sdk"); + const request = vi.fn( + async () => + new Response(JSON.stringify({ error: "Signed generation authority required" }), { + status: 403, + headers: { "content-type": "application/json" }, + }), + ); + try { + await withCliExecutionDeadline(["data", "update"], async () => { + const transport = constrainCliDataManagerTransport( + createDataManagerTransport({ + baseUrl: "http://127.0.0.1:1", + authToken: "synthetic", + fetch: request, + }), + ); + await transport.request( + "startGenerationHardlinksExecution", + "/executions/generation-hardlinks", + { method: "POST" }, + ); + expect(request).toHaveBeenCalledOnce(); + expect(() => transport.request("link", "/link", { method: "POST" })).toThrow( + "deadline-aware receiver", + ); + }); + } finally { + vi.unstubAllEnvs(); + } +}); diff --git a/packages/cli/__tests__/fixtures/single-data-cli-process.ts b/packages/cli/__tests__/fixtures/single-data-cli-process.ts index d0026d787..fe14f086b 100644 --- a/packages/cli/__tests__/fixtures/single-data-cli-process.ts +++ b/packages/cli/__tests__/fixtures/single-data-cli-process.ts @@ -1,7 +1,9 @@ /** Actual existing data-command assembly; only unrelated command families are * omitted to keep this separate-process acceptance fixture focused. */ + import { Command } from "commander"; import { registerDataCommands } from "../../src/commands/data"; +import { fixedCliCommandExitCode } from "../../src/lib/control-context"; import { assertCliExecutionAvailable, withCliExecutionDeadline, @@ -9,7 +11,14 @@ import { const program = new Command(); registerDataCommands(program); -await withCliExecutionDeadline(process.argv.slice(2), async () => { - program.hook("preAction", () => assertCliExecutionAvailable()); - await program.parseAsync(process.argv); -}); +try { + await withCliExecutionDeadline(process.argv.slice(2), async () => { + program.hook("preAction", () => assertCliExecutionAvailable()); + await program.parseAsync(process.argv); + }); +} catch (error) { + if (!process.env.OPENMAPX_CONTROL_DELEGATION && !process.env.OPENMAPX_PARENT_EXECUTION) + throw error; + process.stderr.write("Fixed CLI command failed; inspect the original operation status.\n"); + process.exitCode = fixedCliCommandExitCode(error) ?? 1; +} diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index 3431d5982..0a2b6a086 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -8,7 +8,7 @@ import { exportBuildBundle, importBuildBundle, } from "../lib/build-artifacts"; -import { cliControlContext, completeCliCommand } from "../lib/control-context"; +import { completeCliCommand } from "../lib/control-context"; import { runCliDataStep } from "../lib/data-execution"; import { applyDataCleanup, @@ -17,6 +17,7 @@ import { pruneDataManagerStateForCleanup, } from "../lib/data-local"; import { dataManagerUrl } from "../lib/data-manager-url"; +import { runCliDataUpdate } from "../lib/data-update-command"; import { runningComposeServices } from "../lib/docker"; import { OPENMAPX_REGION_ENV, @@ -322,41 +323,17 @@ export function registerDataCommands(program: Command): void { .option("--fail-fast", "Stop build-all after the first build failure") .action( async (region: string | undefined, options: { countries?: string; failFast?: boolean }) => { - if (cliControlContext()) - throw Error("Managed Full Update requires the reviewed build/link workflow upgrade"); - const transport = createCliTransport(); - let resolvedRegion: string; try { - resolvedRegion = await runOsmDownload(transport, region); - await runTransitSync(transport, { countries: options.countries }); - await runFontDownload(transport); - } catch (err) { - log.err(`update failed: ${(err as Error).message}`); - if (!(err as Error).message.startsWith("region required")) { - dataManagerHint(); - } - completeCliCommand(1); - } - - try { - const buildResult = await buildServices({ - mode: "all", - region: resolvedRegion, - continueOnError: options.failFast !== true, + const receipt = await runCliDataUpdate({ + region, + countries: resolveTransitousCountries(options.countries).values, + failFast: options.failFast === true, + transport: createCliTransport(), }); - - await renderAndApplyHardlinks(); - - if (buildResult.failures.length > 0) { - log.err( - `update completed with ${buildResult.failures.length} build failure${buildResult.failures.length === 1 ? "" : "s"}`, - ); - completeCliCommand(1); - } - - log.ok("Update complete"); - } catch (err) { - log.err(`update failed: ${(err as Error).message}`); + if (receipt.phase === "complete") log.ok("Update complete"); + else log.info("Source data complete; preparing graph builds"); + } catch (error) { + log.err(`update failed: ${(error as Error).message}`); completeCliCommand(1); } }, diff --git a/packages/cli/src/lib/build-staging.ts b/packages/cli/src/lib/build-staging.ts index 431bb4d8d..6c18a31aa 100644 --- a/packages/cli/src/lib/build-staging.ts +++ b/packages/cli/src/lib/build-staging.ts @@ -1,4 +1,5 @@ import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs"; +import { assertCliControlForward } from "./control-context"; import { assertCliExecutionAvailable } from "./execution-deadline"; import { log } from "./output"; @@ -22,6 +23,7 @@ function previousBuildDir(liveDir: string): string { */ export function prepareStagedBuildDir(liveDir: string): string { assertCliExecutionAvailable(); + assertCliControlForward(); const nextDir = stagedBuildDir(liveDir); rmSync(previousBuildDir(liveDir), { recursive: true, force: true }); rmSync(nextDir, { recursive: true, force: true }); @@ -42,6 +44,7 @@ export interface StagedSwap { */ export function promoteStagedBuildDirs(swaps: readonly StagedSwap[]): void { assertCliExecutionAvailable(); + assertCliControlForward(); const moved: StagedSwap[] = []; const installed: StagedSwap[] = []; try { diff --git a/packages/cli/src/lib/control-context.ts b/packages/cli/src/lib/control-context.ts index d1b5566bf..0531e4918 100644 --- a/packages/cli/src/lib/control-context.ts +++ b/packages/cli/src/lib/control-context.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { adoptArtifactTargetLease, assertArtifactTargetLeaseForward, + assertControlExecutionForward, type BorrowedArtifactTargetLease, beginFixedCliExecution, bindBorrowedControlExecution, @@ -14,6 +15,7 @@ import { readFixedCliSpawnIntent, retainAdmittedControlExecution, retainControlEffectReceipt, + retainDataUpdatePlan, settleArtifactTargetLeaseChild, type VerifiedExecutionAuthority, verifiedParentForArtifactDelegation, @@ -28,6 +30,8 @@ const contexts = new AsyncLocalStorage<{ authority: VerifiedExecutionAuthority; installation: ControlInstallation; lease?: BorrowedArtifactTargetLease; + dataWorkflowInitial?: import("@openmapx/core/ops").DataWorkflowReceipt; + dataUpdatePlan?: import("@openmapx/core/build-artifacts").DataUpdatePlan; }>(); class FixedCliCommandExit extends Error { constructor(readonly code: number) { @@ -103,13 +107,25 @@ export async function withCliControlContext( plan: intent.plan, ...(intent.reservations ? { reservations: intent.reservations } : {}), }); + if (intent.dataUpdatePlan) retainDataUpdatePlan(authority, intent.dataUpdatePlan); beginFixedCliExecution(installation, execution); if (lease) bindBorrowedControlExecution(lease, authority); if (lease) assertArtifactTargetLeaseForward(lease); try { let result: T; try { - result = await contexts.run({ authority, lease, installation }, run); + result = await contexts.run( + { + authority, + lease, + installation, + ...(intent.dataUpdatePlan ? { dataUpdatePlan: intent.dataUpdatePlan } : {}), + ...(intent.dataWorkflowInitial + ? { dataWorkflowInitial: intent.dataWorkflowInitial } + : {}), + }, + run, + ); } catch (e) { if (e instanceof FixedCliCommandExit && e.code === 0) result = undefined as T; else throw e; @@ -154,3 +170,12 @@ export function withHostDataControlContext( if (contexts.getStore()) throw Error("Existing CLI control context"); return contexts.run({ installation, authority }, run); } + +/** Restriction at actual host mutation boundaries; observation remains usable + * after cancellation and cannot acquire authority through this helper. */ +export function assertCliControlForward(): void { + const context = contexts.getStore(); + if (!context) return; + assertControlExecutionForward(context.authority); + if (context.lease) assertArtifactTargetLeaseForward(context.lease); +} diff --git a/packages/cli/src/lib/coordinator-ready.ts b/packages/cli/src/lib/coordinator-ready.ts new file mode 100644 index 000000000..a05baca8a --- /dev/null +++ b/packages/cli/src/lib/coordinator-ready.ts @@ -0,0 +1,46 @@ +/** Finite non-recreating coordinator Start inside existing writer/target ownership. */ +import { + assertArtifactTargetLeaseForward, + assertControlExecutionForward, +} from "@openmapx/core/build-artifacts"; +import { + type DataManagerTransport, + getStatus, + unwrapDataManagerResponse, +} from "@openmapx/core/data-manager-sdk"; +import type { CoordinatorMode } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { dockerComposeStream } from "./docker"; +export async function ensureCliCoordinatorReady(options: { + ownership: services.VerifiedLifecycleOwnership; + transport: DataManagerTransport; + mode: CoordinatorMode; +}) { + const authority = services.assertVerifiedLifecycleOwnership(options.ownership); + if (!options.ownership.lease.serviceIds.includes("data-manager")) + throw Error("Coordinator target ownership required"); + const forward = () => { + assertControlExecutionForward(authority); + assertArtifactTargetLeaseForward(options.ownership.lease); + }; + const ready = async () => { + try { + return unwrapDataManagerResponse(await getStatus({ transport: options.transport })).ok; + } catch { + return false; + } + }; + forward(); + if (await ready()) return; + if (options.mode !== "ensure-running") throw Error("Configuration coordinator unavailable"); + forward(); + const code = await dockerComposeStream(["up", "-d", "--no-recreate", "data-manager"]); + if (code !== 0) throw Error("Configuration coordinator Start failed"); + const until = Math.min(Date.now() + 30000, authority.execution.deadlineAt); + while (!(await ready())) { + forward(); + if (Date.now() >= until) throw Error("Configuration coordinator unavailable"); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + forward(); +} diff --git a/packages/cli/src/lib/data-execution.ts b/packages/cli/src/lib/data-execution.ts index b89de8984..d8465a903 100644 --- a/packages/cli/src/lib/data-execution.ts +++ b/packages/cli/src/lib/data-execution.ts @@ -41,11 +41,13 @@ import { opsOperationFingerprint, } from "@openmapx/core/ops"; import { cliControlContext, withHostDataControlContext } from "./control-context"; +import { cliAcquisitionOptions } from "./execution-deadline"; import { repoPaths } from "./paths"; export async function executeDurableDataStep( input: DataExecutionInput, options: { operationKey: string; + phase?: "data-step" | "osm" | "transit" | "fonts"; transport: DataManagerTransport; signal?: AbortSignal; requestedDeadlineAt: number; @@ -58,7 +60,7 @@ export async function executeDurableDataStep( const context = cliControlContext(); if (!context) throw Error("Retained data command owner required"); const normalized = ordinaryDataExecutionInputSchema.parse(normalizeDataExecutionInput(input)), - slot = cliDataReservation(context.authority), + slot = cliDataReservation(context.authority, options.phase), bound = await bindControlChildPhase(context.authority, slot); if ( options.operationKey !== slot.execution.operationKey || @@ -87,7 +89,7 @@ export async function executeDurableDataStep( status: DataExecutionStatus | undefined; if (!attempt) { attempt = { - phase: "data-step", + phase: options.phase ?? "data-step", operationKey: slot.execution.operationKey, input: normalized, operationFingerprint: slot.operationFingerprint, @@ -197,32 +199,40 @@ function operationFor(input: DataExecutionInput): OpsOperation { throw Error("No existing single-command edge for this data input"); } } -async function runOwned(input: DataExecutionInput, transport: DataManagerTransport) { +export async function runOwnedDataStep( + input: DataExecutionInput, + transport: DataManagerTransport, + phase: "data-step" | "osm" | "transit" | "fonts" = "data-step", + onProgress: (status: DataExecutionStatus) => Promise = async () => {}, +) { const context = cliControlContext(); if (!context) throw Error("Data command owner required"); - const slot = cliDataReservation(context.authority), + const slot = cliDataReservation(context.authority, phase), bound = await bindControlChildPhase(context.authority, slot); const prior = readCliDataAttempt(context.authority, slot.execution.operationKey); let assertion: string | undefined; if (!prior) { const issued = await issueControlChildPhase(context.authority, { - phase: "data-step", - ordinal: 0, + phase, + ordinal: slot.ordinal, childPublicKey: controlRolePublicKey(context.installation, "data-manager"), }); assertion = issued.assertion; consumeControlChildDispatch(issued.dispatchTicket); } const result = await executeDurableDataStep(input, { + phase, + signal: cliAcquisitionOptions().signal, operationKey: slot.execution.operationKey, requestedDeadlineAt: bound.execution.deadlineAt, parentAssertion: assertion, transport, observerDeadlineAt: context.authority.execution.deadlineAt, checkpointAttempt: async (attempt) => retainCliDataAttempt(context.authority, attempt), - checkpoint: async () => {}, + checkpoint: onProgress, }); - if (result.status !== "ok") throw Error(`Data execution ${result.id} finished ${result.status}`); + if (phase === "data-step" && result.status !== "ok") + throw Error(`Data execution ${result.id} finished ${result.status}`); return result; } /** Host commands use their own dated, retained coordinator. Managed fixed CLI @@ -232,7 +242,7 @@ export async function runCliDataStep( transport: DataManagerTransport, ): Promise { const input = normalizeDataExecutionInput(value); - if (cliControlContext()) return runOwned(input, transport); + if (cliControlContext()) return runOwnedDataStep(input, transport); const operation = operationFor(input), admittedAt = Date.now(), requestedDeadlineAt = admittedAt + opsExecutionBudget(operation.kind).processMs, @@ -288,7 +298,7 @@ export async function runCliDataStep( ], }); const result = await withHostDataControlContext(installation, authority, () => - runOwned(input, transport), + runOwnedDataStep(input, transport), ); retainControlEffectReceipt(authority, { outcome: "committed", diff --git a/packages/cli/src/lib/data-update-command.ts b/packages/cli/src/lib/data-update-command.ts new file mode 100644 index 000000000..64c3652d3 --- /dev/null +++ b/packages/cli/src/lib/data-update-command.ts @@ -0,0 +1,253 @@ +/** Concrete Full Update adapter shared by standalone and fixed CLI entrypoints. */ +import { randomBytes, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import type { DataManagerTransport } from "@openmapx/core/data-manager-sdk"; +import * as ops from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { renderComposeWithWriter } from "../commands/compose"; +import { cliControlContext, withHostDataControlContext } from "./control-context"; +import { ensureCliCoordinatorReady } from "./coordinator-ready"; +import { runOwnedDataStep } from "./data-execution"; +import { DataUpdateFailure, runDataUpdate } from "./data-update"; +import { cliAcquisitionOptions } from "./execution-deadline"; +import { readGeneratedHardlinkPlan } from "./hardlinks"; +import { log } from "./output"; +import { repoPaths } from "./paths"; +import { buildServices, ServiceBuildExecutionError } from "./service-builds"; + +export async function runCliDataUpdate(options: { + region?: string; + countries: string[]; + failFast: boolean; + transport: DataManagerTransport; +}): Promise { + const paths = repoPaths(); + let context = cliControlContext(), + installation: control.ControlInstallation | undefined; + if (!context) { + mkdirSync(join(paths.infraDir, "data"), { recursive: true }); + if (!existsSync("/run/secrets/openmapx-control-verifiers")) + await control.provisionControlInstallation(paths.infraDir); + const registry = new services.ServiceRegistry({ rootDir: paths.root }); + await registry.load(); + const targets = registry + .list() + .map((s) => s.manifest.id) + .sort(); + installation = control.openControlInstallation({ + infraDir: paths.infraDir, + role: "host-cli", + serviceIds: targets, + }); + const operation = { + kind: "data.update" as const, + ...(options.region ? { regionId: options.region } : {}), + countryCodes: options.countries, + failFast: options.failFast, + }, + admittedAt = Date.now(), + deadlineAt = admittedAt + ops.opsExecutionBudget(operation.kind).processMs, + operationFingerprint = ops.opsOperationFingerprint(operation); + const execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: ops.createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: ops.executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }; + try { + const plan = control.planDataUpdate({ + infraDir: paths.infraDir, + parent: execution, + operation, + serviceIds: targets, + defaultRegion: process.env.OPENMAPX_REGION, + }); + const authority = control.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: targets, + reservations: [...plan.dataChildren, ...plan.finalizationChildren], + }); + control.retainDataUpdatePlan(authority, plan); + return await withHostDataControlContext(installation, authority, () => + runCliDataUpdate(options), + ); + } finally { + control.closeControlInstallation(installation); + } + } + const { authority } = context, + plan = control.readDataUpdatePlan(authority), + managed = context.dataUpdatePlan; + const stage = managed + ? authority.execution.operationKey === plan.dataReservation.execution.operationKey + ? "data" + : "finalize" + : undefined; + const store = await services.openAdminDataWorkflow({ + dataRoot: join(paths.infraDir, "data"), + operationKey: authority.execution.operationKey, + fingerprint: authority.execution.fingerprint, + authority, + }); + if (store.read()) throw Error("Retained Full Update permits observation only"); + let lease: control.TargetLeaseAuthority | undefined = context.lease; + try { + const receipt = await runDataUpdate( + { + region: plan.region, + countries: plan.countries, + failFast: plan.operation.kind === "data.update" && plan.operation.failFast === true, + execution: authority.execution, + signal: cliAcquisitionOptions().signal, + stage, + ...(context.dataWorkflowInitial ? { initialReceipt: context.dataWorkflowInitial } : {}), + }, + { + checkpoint: (receipt) => store.checkpoint(receipt), + checkpointAttempt: async (attempt) => control.retainCliDataAttempt(authority, attempt), + runDataStep: (phase, input) => { + let last = ""; + return runOwnedDataStep(input, options.transport, phase, async (status) => { + const message = `Full Update ${phase}: ${status.status}${status.progress === null ? "" : ` (${Math.round(status.progress * 100)}%)`}`; + if (message !== last) { + log.info(message); + last = message; + } + }); + }, + async build(input) { + control.assertDataUpdateSources(authority); + if (!lease) + lease = await control.acquireArtifactTargetLease({ + dataRoot: join(paths.infraDir, "data"), + serviceIds: control.controlRegisteredServiceIds(context.installation), + owner: authority, + signal: cliAcquisitionOptions().signal, + }); + try { + return ( + await buildServices({ + mode: "all", + region: input.region, + continueOnError: input.continueOnError, + }) + ).serviceBatch; + } catch (error) { + if (error instanceof ServiceBuildExecutionError) return error.result.serviceBatch; + throw error; + } + }, + async prepare({ coordinatorMode }) { + if (!lease) throw Error("Full Update finalization lease required"); + control.assertDataUpdateSources(authority); + control.retainGenerationHardlinksOwnerPlan(authority, { + version: 1, + parent: authority.execution, + revisionId: plan.finalRevisionId, + coordinatorMode, + plan: plan.hardlinks, + reservation: plan.finalizationChildren[0] ?? null, + }); + return services.withConfigurationGenerationWriter( + paths.infraDir, + (writer) => + services.withVerifiedLifecycleOwnership( + { writer, lease: lease!, authority }, + async (ownership) => { + control.assertDataUpdateSources(authority); + if (plan.hardlinks.plan.length) { + const registry = new services.ServiceRegistry({ rootDir: paths.root }); + await registry.load(); + const selected = services.readAppliedServiceIds( + paths.infraDir, + registry.list(), + ); + if (!selected?.has("data-manager")) + throw Error("Configuration coordinator must be enabled"); + await ensureCliCoordinatorReady({ + ownership, + transport: options.transport, + mode: coordinatorMode, + }); + } + const rendered = await renderComposeWithWriter( + { + domain: plan.domain, + expectedCurrentRevisionId: plan.revisionId, + revisionId: plan.finalRevisionId, + }, + writer, + ); + if ( + ops.canonicalJson(readGeneratedHardlinkPlan().plan) !== + ops.canonicalJson(plan.hardlinks.plan) + ) + throw Error("Full Update hardlink plan changed"); + const hardlinks = await control.executeOwnedGenerationHardlinks({ + authority, + lease: lease!, + infraDir: paths.infraDir, + transport: options.transport, + signal: cliAcquisitionOptions().signal, + }); + return { revisionId: rendered.revisionId, coordinatorMode, hardlinks }; + }, + ), + cliAcquisitionOptions().signal, + ); + }, + }, + ); + if (!managed) { + control.retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "ops-result", result: { completed: true, dataWorkflow: receipt } }, + }); + if (lease) { + await control.settleArtifactTargetLeaseChild( + lease, + control.verifyControlEffectSettlement(context.installation, authority.execution), + ); + await releaseOwned(lease); + } + } + return receipt; + } catch (error) { + if (error instanceof DataUpdateFailure) throw error; + throw new DataUpdateFailure( + store.read() ?? { + version: 1, + phase: "osm", + children: [], + completedBuildIds: [], + failedBuildIds: [], + skippedBuildIds: [], + }, + error, + ); + } finally { + if (lease && !context.lease) control.closeArtifactTargetLeaseHandle(lease); + } +} + +async function releaseOwned(lease: control.TargetLeaseAuthority) { + if (lease.mode !== "owner") throw Error("Original host Full Update root required"); + await control.releaseArtifactTargetLease(lease); +} diff --git a/packages/cli/src/lib/data-update.ts b/packages/cli/src/lib/data-update.ts new file mode 100644 index 000000000..04ec1b122 --- /dev/null +++ b/packages/cli/src/lib/data-update.ts @@ -0,0 +1,174 @@ +import { controlChildOperationKey, fixedDataBodyFingerprint } from "@openmapx/core/build-artifacts"; +import { + type DataChildAttempt, + type DataExecutionInput, + type DataExecutionStatus, + dataExecutionStatusSchema, + normalizeDataExecutionInput, + ordinaryDataExecutionInputSchema, +} from "@openmapx/core/data-manager-contracts"; +import { + type DataWorkflowReceipt, + dataWorkflowReceiptSchema, + type ExecutionRef, + executionFingerprint, + type ServiceBatchResult, + type ServicePreparationResult, + serviceBatchResultSchema, +} from "@openmapx/core/ops"; + +type DataStep = "osm" | "transit" | "fonts"; +export interface DataUpdateOptions { + region: string; + countries: string[]; + failFast: boolean; + execution: ExecutionRef; + signal?: AbortSignal; + stage?: "data" | "finalize"; + initialReceipt?: DataWorkflowReceipt; +} +export interface DataUpdateEffects { + checkpoint(receipt: DataWorkflowReceipt): Promise; + checkpointAttempt(attempt: DataChildAttempt): Promise; + /** The installed adapter journals original admission intent before its first POST. */ + runDataStep(step: DataStep, input: DataExecutionInput): Promise; + build(options: { region: string; continueOnError: boolean }): Promise; + prepare(options: { coordinatorMode: "ensure-running" }): Promise; +} +export class DataUpdateFailure extends Error { + readonly receipt: DataWorkflowReceipt; + constructor(receipt: DataWorkflowReceipt, cause: unknown) { + super("Full Update did not complete", { cause }); + this.name = "DataUpdateFailure"; + this.receipt = dataWorkflowReceiptSchema.parse(receipt); + } +} +function forward(options: DataUpdateOptions) { + options.signal?.throwIfAborted(); + if (Date.now() >= options.execution.deadlineAt) throw Error("Full Update deadline elapsed"); +} +function checkedChild( + options: DataUpdateOptions, + step: DataStep, + input: DataExecutionInput, + value: DataExecutionStatus, +) { + const status = dataExecutionStatusSchema.parse(value); + const parent = options.execution; + if ( + status.kind !== input.kind || + status.operationKey !== + controlChildOperationKey( + parent.operationKey, + step, + ["osm", "transit", "fonts"].indexOf(step), + ) || + status.deadlineAt > parent.deadlineAt || + status.fingerprint !== + executionFingerprint({ + kind: input.kind, + operationFingerprint: fixedDataBodyFingerprint(input), + deadlineAt: status.deadlineAt, + parent: { + issuer: parent.executor, + execution: parent, + operationKey: parent.operationKey, + fingerprint: parent.fingerprint, + deadlineAt: parent.deadlineAt, + }, + }) + ) + throw Error("Full Update child identity changed"); + return status; +} +/** Pure sequencing; operation authority and signed terminal verification belong to its installed effects. */ +export async function runDataUpdate( + options: DataUpdateOptions, + effects: DataUpdateEffects, +): Promise { + const receipt: DataWorkflowReceipt = options.initialReceipt + ? dataWorkflowReceiptSchema.parse(options.initialReceipt) + : { + version: 1, + phase: "osm", + children: [], + completedBuildIds: [], + failedBuildIds: [], + skippedBuildIds: [], + }; + const save = async () => { + forward(options); + await effects.checkpoint(dataWorkflowReceiptSchema.parse(receipt)); + forward(options); + }; + try { + const steps: [DataStep, DataExecutionInput][] = [ + ["osm", { kind: "download-osm", region: options.region }], + ["transit", { kind: "transit-sync", countries: options.countries }], + ["fonts", { kind: "download-fonts" }], + ]; + if ( + options.stage === "finalize" && + (receipt.phase !== "build" || + receipt.children.length !== 3 || + receipt.children.some((c) => c.status !== "succeeded")) + ) + throw Error("Full Update data phase is incomplete"); + for (const [step, raw] of options.stage === "finalize" ? [] : steps) { + receipt.phase = step; + await save(); + const input = ordinaryDataExecutionInputSchema.parse(normalizeDataExecutionInput(raw)); + const status = checkedChild(options, step, input, await effects.runDataStep(step, input)); + receipt.children.push({ + step, + jobId: status.id, + operationKey: status.operationKey, + fingerprint: status.fingerprint, + deadlineAt: status.deadlineAt, + status: + status.status === "ok" + ? "succeeded" + : status.status === "error" || status.status === "partial" + ? "failed" + : status.status, + }); + await save(); + if (status.status !== "ok" || status.publication !== "committed" || !status.result) + throw Error(`Prerequisite ${step} did not complete`); + if ( + input.kind === "download-osm" && + (status.result.kind !== "download-osm" || status.result.region !== input.region) + ) + throw Error("OSM region changed"); + if ( + input.kind === "transit-sync" && + (status.result.kind !== "transit-sync" || + status.result.finalStatus !== "ok" || + JSON.stringify([...status.result.countries].sort()) !== JSON.stringify(input.countries)) + ) + throw Error("Transit country selection changed"); + } + receipt.phase = "build"; + await save(); + if (options.stage === "data") return dataWorkflowReceiptSchema.parse(receipt); + const batch = serviceBatchResultSchema.parse( + await effects.build({ region: options.region, continueOnError: !options.failFast }), + ); + receipt.completedBuildIds = [...batch.completedServiceIds]; + receipt.failedBuildIds = [...batch.failedServiceIds]; + receipt.skippedBuildIds = [...batch.skippedServiceIds]; + await save(); + if (options.failFast && batch.outcome !== "success") throw Error("Graph build failed"); + receipt.phase = "link"; + await save(); + const prepared = await effects.prepare({ coordinatorMode: "ensure-running" }); + receipt.revisionId = prepared.revisionId; + await save(); + if (batch.outcome !== "success") throw Error("Some graph builds failed"); + receipt.phase = "complete"; + await save(); + return dataWorkflowReceiptSchema.parse(receipt); + } catch (error) { + throw new DataUpdateFailure(receipt, error); + } +} diff --git a/packages/cli/src/lib/execution-deadline.ts b/packages/cli/src/lib/execution-deadline.ts index c745323ee..cdff283dd 100644 --- a/packages/cli/src/lib/execution-deadline.ts +++ b/packages/cli/src/lib/execution-deadline.ts @@ -104,7 +104,11 @@ export function constrainCliDataManagerTransport( if ( deadlines.getStore() && !["GET", "HEAD"].includes((options?.method ?? "GET").toUpperCase()) && - !["startDataExecution", "cancelDataExecution"].includes(operationId) + ![ + "startDataExecution", + "cancelDataExecution", + "startGenerationHardlinksExecution", + ].includes(operationId) ) throw Error("Managed data-manager execution requires a deadline-aware receiver upgrade"); return transport.request(operationId, url, options); diff --git a/packages/cli/src/lib/service-builds.ts b/packages/cli/src/lib/service-builds.ts index 0a7ec5ec0..83a7c1f12 100644 --- a/packages/cli/src/lib/service-builds.ts +++ b/packages/cli/src/lib/service-builds.ts @@ -13,6 +13,7 @@ import { currentRevision, writeBuildRecord, } from "./build-artifacts"; +import { assertCliControlForward } from "./control-context"; import { runningComposeServices } from "./docker"; import { resolveBuildRegion } from "./env-defaults"; import { assertCliExecutionAvailable } from "./execution-deadline"; @@ -230,8 +231,10 @@ async function recordBuild( artifact: ServiceBuildArtifact, ): Promise { assertCliExecutionAvailable(); + assertCliControlForward(); const revision = await currentRevision(rootDir); assertCliExecutionAvailable(); + assertCliControlForward(); writeBuildRecord( { schemaVersion: 1, @@ -395,6 +398,7 @@ export async function buildServices( log.dim(`Manifest command → ${item.buildCommand}`); try { assertCliExecutionAvailable(); + assertCliControlForward(); const resolvedRegion = resolveBuildRegion(item.id, opts.region); if (resolvedRegion.sourceEnv) { log.dim( @@ -431,6 +435,7 @@ export async function buildServices( interrupted = true; try { assertCliExecutionAvailable(); + assertCliControlForward(); } catch { interrupted = true; } diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 2eb2829bd..c851daaec 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1,9 +1,12 @@ +import { dataWorkflowReceiptSchema } from "../ops/job-evidence"; import { type ControlChildPhase, type ControlRecipient, childPhaseKinds, controlChildOperationKey, } from "./control-child-records"; +import { dataUpdatePlanSchema } from "./data-update-records"; +import { dataUpdateSourcesDigest } from "./data-update-sources"; export { type ControlChildPhase, @@ -60,7 +63,7 @@ import { randomUUID, } from "node:crypto"; import { existsSync, fsyncSync, mkdirSync, realpathSync } from "node:fs"; -import { join, relative } from "node:path"; +import { dirname, join, relative } from "node:path"; import { z } from "zod"; import type { PreparedConfigurationRecord } from "../ops/configuration-preparation"; import { @@ -90,6 +93,7 @@ import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; import { servicePullResultSchema } from "../ops/service-operation-records"; import { verifiedParentOrigins, verifiedParents } from "../ops/verified-parent-internal"; import { + currentConfigurationGeneration, type OriginalConfigurationGenerationNonCommit, originalConfigurationGenerationNonCommitSource, } from "../services/configuration-generation"; @@ -105,6 +109,7 @@ import { initializeControlChildReservations, inspectControlChildPublication, lookupControlChildPhase, + lookupControlChildPhaseAttempt, prepareControlChildPublication, resolveControlChildPlan, validateControlChildReservations, @@ -1877,6 +1882,37 @@ export function restoreConfigurationPreEffectTerminal( throw Error("Original pre-effect terminal unavailable"); } /** Fixed executor adapters supply their parsed result or exact fixed-command exit; arbitrary text is never proof. */ +/** Original indexed progress is business evidence only. A success producer must + * additionally prove its actual children/target tree before committing. */ +function readDataWorkflowProgress(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority); + if (s.kind !== "data.update" || !s.origin) throw Error("Original Full Update owner required"); + const name = `data-workflow-${s.execution.operationKey}.json`, + raw = readPrivateRecord(s.installation, "observations", name, 65536); + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace: "observations", container: "", name }, + canonicalJson(raw), + ); + const body = verifyControl( + s.installation.installation.publicKey, + "openmapx-admin-data-workflow-v1", + raw, + z.strictObject({ + version: z.literal(1), + operationKey: z.string(), + fingerprint: z.string(), + receipt: dataWorkflowReceiptSchema, + }), + ); + if ( + body.operationKey !== s.execution.operationKey || + body.fingerprint !== s.execution.fingerprint + ) + throw Error("Full Update checkpoint identity changed"); + return body.receipt; +} export function retainControlEffectReceipt( authority: VerifiedExecutionAuthority, effect: { @@ -1894,6 +1930,21 @@ export function retainControlEffectReceipt( publication: ChildSettlement["publication"] = "unknown"; if (receipt.kind === "ops-result") { receipt.result = parseBoundedOpsResult(s.kind as OpsOperationKind, receipt.result); + if (s.kind === "data.update") { + const actual = readDataWorkflowProgress(authority), + result = receipt.result as { dataWorkflow?: unknown }; + if ( + actual.phase !== "complete" || + actual.children.length !== 3 || + actual.children.some((c) => c.status !== "succeeded") || + actual.failedBuildIds.length || + actual.skippedBuildIds.length || + !actual.revisionId || + canonicalJson(actual) !== canonicalJson(result.dataWorkflow) + ) + throw Error("Actual completed Full Update workflow required"); + verifyDataUpdateCompletionSource(authority, actual); + } const result = receipt.result as { outcome?: string }; if (!result.outcome || ["applied", "unchanged", "committed"].includes(result.outcome)) { outcome = "committed"; @@ -1935,7 +1986,9 @@ export function retainControlEffectReceipt( readFixedCliSpawnIntent(s.installation.installation, s.execution, receipt.args); readPrivateRecord(s.installation, "spawn-started", `${s.installation.fixedChildId}.json`); for (const slot of s.reservations.filter( - (item) => item.recipient === "data-manager" && item.phase === "data-step", + (item) => + item.recipient === "data-manager" && + ["data-step", "osm", "transit", "fonts"].includes(item.phase), )) { const binding = lookupControlChildPhase(authority, slot); if ( @@ -1945,6 +1998,24 @@ export function retainControlEffectReceipt( ) throw Error("Actual backend data containment required before fixed command success"); } + if (s.kind === "data.update") { + for (const slot of s.reservations.filter((value) => value.phase === "hardlinks")) { + const bound = lookupControlChildPhase(authority, slot); + if ( + !bound || + verifyDataControlTerminalWitness(s.installation.installation, bound.execution).status + .status !== "ok" + ) + throw Error("Actual Full Update hardlink containment required"); + } + const workflow = readDataWorkflowProgress(authority), + spawn = readFixedCliSpawnIntent(s.installation.installation, s.execution, receipt.args); + const expected = + spawn.dataUpdatePlan?.dataReservation.execution.operationKey === s.execution.operationKey + ? "build" + : "complete"; + if (workflow.phase !== expected) throw Error("Original Full Update checkpoint required"); + } outcome = "committed"; publication = "committed"; } else if (receipt.kind === "configuration-no-guard") { @@ -4566,6 +4637,8 @@ export function retainFixedCliSpawnIntent( args: readonly string[]; plan?: readonly ControlPlannedChild[]; reservations?: readonly ControlChildReservation[]; + dataUpdatePlan?: import("./data-update-records").DataUpdatePlan; + dataWorkflowInitial?: import("../ops/job-evidence").DataWorkflowReceipt; }, ): void { const s = executionAuthorityState(authority); @@ -4595,6 +4668,12 @@ export function retainFixedCliSpawnIntent( { execution: input.execution, operation, + ...(input.dataUpdatePlan + ? { dataUpdatePlan: dataUpdatePlanSchema.parse(input.dataUpdatePlan) } + : {}), + ...(input.dataWorkflowInitial + ? { dataWorkflowInitial: dataWorkflowReceiptSchema.parse(input.dataWorkflowInitial) } + : {}), args: input.args, serviceIds: plan.serviceIds, admittedAt: Date.now(), @@ -4696,6 +4775,8 @@ export function readFixedCliSpawnIntent( reservations: z.array(controlChildReservationSchema).max(64).optional(), execution: executionRefSchema, operation: opsOperationSchema, + dataUpdatePlan: dataUpdatePlanSchema.optional(), + dataWorkflowInitial: dataWorkflowReceiptSchema.optional(), args: z.array(z.string().max(4096)).max(64), serviceIds: targetSchema, admittedAt: z.number().int().nonnegative().safe(), @@ -5005,3 +5086,173 @@ export function verifyControlPreparedTerminal( d.close(); } } + +/** Target acquisition observes only its original data-phase owner and all three + * signed, physically settled DM children. No progress object is authority. */ +export async function verifyDataUpdatePrerequisites(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority); + if (s.kind !== "data.update" || s.installation.fixedChildId || !s.origin) + throw Error("Original Full Update root required"); + const name = `data-update-plan-${s.execution.operationKey}.json`, + raw = readPrivateRecord(s.installation, "observations", name, 128 * 1024); + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace: "observations", container: "", name }, + canonicalJson(raw), + ); + const plan = verifyControl( + s.installation.installation.publicKey, + "openmapx-data-update-plan-v1", + raw, + dataUpdatePlanSchema, + ), + infraDir = dirname(s.installation.dataRoot); + if ( + controlDigest(plan.parent) !== controlDigest(s.execution) || + opsOperationFingerprint(plan.operation) !== s.execution.operationFingerprint || + currentConfigurationGeneration(infraDir) !== plan.revisionId || + dataUpdateSourcesDigest(infraDir) !== plan.sourcesDigest + ) + throw Error("Original Full Update sources changed"); + let childInstallation: ControlInstallation | undefined; + try { + let dataAuthority = authority; + if (s.execution.executor === "ops-agent") { + const attempt = lookupControlChildPhaseAttempt(authority, { + phase: "data-workflow", + ordinal: 0, + }); + if (!attempt) throw Error("Full Update data phase not admitted"); + childInstallation = openFixedChildInstallation({ + infraDir, + assertion: attempt.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + dataAuthority = restoreAdmittedControlExecution(childInstallation, attempt.execution); + const settlement = childSettlementState( + verifyControlEffectSettlement(childInstallation, attempt.execution), + ).evidence; + if (settlement.outcome !== "committed" || settlement.publication !== "committed") + throw Error("Full Update data phase unsettled"); + } + const dataState = executionAuthorityState(dataAuthority), + receipt = readDataWorkflowProgress(dataAuthority); + if ( + receipt.phase !== "build" || + receipt.children.length !== 3 || + receipt.children.some((c) => c.status !== "succeeded") + ) + throw Error("Full Update data checkpoint missing"); + for (const [index, slot] of plan.dataChildren.entries()) { + const bound = lookupControlChildPhase(dataAuthority, slot); + if (!bound) throw Error("Full Update prerequisite missing"); + const child = receipt.children[index], + witness = verifyDataControlTerminalWitness( + dataState.installation.installation, + bound.execution, + ); + if ( + !child || + child.operationKey !== bound.execution.operationKey || + child.jobId !== bound.execution.executionId || + child.fingerprint !== bound.execution.fingerprint || + child.deadlineAt !== bound.execution.deadlineAt || + witness.status.status !== "ok" || + witness.status.publication !== "committed" + ) + throw Error("Full Update prerequisite did not complete"); + } + return receipt; + } finally { + if (childInstallation) closeControlInstallation(childInstallation); + } +} + +/** A signed progress file cannot mint success. Rejoin the actual original CLI + * exit sources and DM settlements, and the exact generation they published. */ +function verifyDataUpdateCompletionSource( + authority: VerifiedExecutionAuthority, + receipt: import("../ops/job-evidence").DataWorkflowReceipt, +) { + const s = executionAuthorityState(authority); + if (s.installation.fixedChildId || !s.origin) throw Error("Original Full Update root required"); + const name = `data-update-plan-${s.execution.operationKey}.json`, + raw = readPrivateRecord(s.installation, "observations", name, 128 * 1024); + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace: "observations", container: "", name }, + canonicalJson(raw), + ); + const plan = verifyControl( + s.installation.installation.publicKey, + "openmapx-data-update-plan-v1", + raw, + dataUpdatePlanSchema, + ), + infraDir = dirname(s.installation.dataRoot); + if ( + controlDigest(plan.parent) !== controlDigest(s.execution) || + opsOperationFingerprint(plan.operation) !== s.execution.operationFingerprint || + receipt.revisionId !== plan.finalRevisionId || + currentConfigurationGeneration(infraDir) !== plan.finalRevisionId || + dataUpdateSourcesDigest(infraDir) !== plan.sourcesDigest + ) + throw Error("Original Full Update completion sources changed"); + const checkData = (dataAuthority: VerifiedExecutionAuthority) => { + const dataState = executionAuthorityState(dataAuthority); + for (const [index, slot] of plan.dataChildren.entries()) { + const bound = lookupControlChildPhase(dataAuthority, slot), + child = receipt.children[index]; + if ( + !bound || + !child || + child.operationKey !== bound.execution.operationKey || + child.jobId !== bound.execution.executionId || + child.fingerprint !== bound.execution.fingerprint || + child.deadlineAt !== bound.execution.deadlineAt || + verifyDataControlTerminalWitness(dataState.installation.installation, bound.execution) + .status.status !== "ok" + ) + throw Error("Actual Full Update data completion required"); + } + }; + if (s.execution.executor === "ops-agent") { + for (const slot of [plan.dataReservation, plan.finalizationReservation]) { + const attempt = lookupControlChildPhaseAttempt(authority, slot); + if (!attempt) throw Error("Full Update fixed phase missing"); + const child = openFixedChildInstallation({ + infraDir, + assertion: attempt.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + readFixedCliSpawnIntent(child, attempt.execution, fixedCliArguments(plan.operation)); + const childAuthority = restoreAdmittedControlExecution(child, attempt.execution), + effect = childSettlementState( + verifyControlEffectSettlement(child, attempt.execution), + ).evidence; + if (effect.outcome !== "committed" || effect.publication !== "committed") + throw Error("Full Update fixed phase unsettled"); + const progress = readDataWorkflowProgress(childAuthority); + if (slot.phase === "data-workflow") checkData(childAuthority); + else if (canonicalJson(progress) !== canonicalJson(receipt)) + throw Error("Original Full Update final receipt changed"); + } finally { + closeControlInstallation(child); + } + } + } else { + checkData(authority); + for (const slot of plan.finalizationChildren) { + const bound = lookupControlChildPhase(authority, slot); + if ( + !bound || + verifyDataControlTerminalWitness(s.installation.installation, bound.execution).status + .status !== "ok" + ) + throw Error("Actual Full Update hardlink completion required"); + } + } +} diff --git a/packages/core/src/build-artifacts/control-child-records.ts b/packages/core/src/build-artifacts/control-child-records.ts index 798e7c7fa..6ece0c803 100644 --- a/packages/core/src/build-artifacts/control-child-records.ts +++ b/packages/core/src/build-artifacts/control-child-records.ts @@ -11,6 +11,7 @@ export type ControlChildPhase = | "data-link" | "data-clean" | "data-workflow" + | "data-finalization" | "hardlinks" | "stop-primary" | "stop-staging" @@ -37,6 +38,7 @@ export const childPhaseKinds: Readonly; +export function dataUpdateAddress(key: string, purpose: string) { + const b = createHash("sha256") + .update(`openmapx-data-update-v1\0${key}\0${purpose}`) + .digest() + .subarray(0, 16); + b[6] = (b[6]! & 15) | 64; + b[8] = (b[8]! & 63) | 128; + const h = b.toString("hex"); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`; +} +export function createDataUpdatePlan( + options: Pick< + DataUpdatePlan, + | "parent" + | "operation" + | "region" + | "countries" + | "domain" + | "revisionId" + | "sourcesDigest" + | "hardlinks" + > & { serviceIds: readonly string[] }, +): DataUpdatePlan { + if (options.operation.kind !== "data.update") throw Error("Full Update operation required"); + const targets = [...options.serviceIds].sort(), + root = options.parent, + opfp = opsOperationFingerprint(options.operation); + const slot = (phase: "data-workflow" | "data-finalization"): ControlChildReservation => ({ + version: 1, + phase, + ordinal: phase === "data-workflow" ? 0 : 1, + kind: "data.update", + recipient: "fixed-cli", + serviceIds: targets, + operationFingerprint: opfp, + execution: { + executor: "host-cli", + principal: root.principal, + executionId: dataUpdateAddress(root.operationKey, phase), + operationKey: controlChildOperationKey( + root.operationKey, + phase, + phase === "data-workflow" ? 0 : 1, + ), + }, + }); + const dataReservation = slot("data-workflow"), + finalizationReservation = slot("data-finalization"); + const inputs = [ + { phase: "osm" as const, input: { kind: "download-osm", region: options.region } }, + { phase: "transit" as const, input: { kind: "transit-sync", countries: options.countries } }, + { phase: "fonts" as const, input: { kind: "download-fonts" } }, + ]; + const dataParent = root.executor === "host-cli" ? root : dataReservation.execution; + const dataChildren = inputs.map(({ phase, input }, ordinal) => + controlChildReservationSchema.parse({ + version: 1, + phase, + ordinal, + kind: input.kind, + recipient: "data-manager", + serviceIds: targets, + operationFingerprint: controlDigest( + normalizeDataExecutionInput(ordinaryDataExecutionInputSchema.parse(input)), + ), + execution: { + executor: "data-manager", + principal: "host-cli", + executionId: dataUpdateAddress(root.operationKey, phase), + operationKey: controlChildOperationKey(dataParent.operationKey, phase, ordinal), + }, + }), + ); + const finalRevisionId = `cfg1_${createHash("sha256").update(`openmapx-data-update-generation-v1\0${root.operationKey}`).digest("base64url")}`; + const leaf = generationHardlinksReservation({ + parent: root.executor === "host-cli" ? root : finalizationReservation.execution, + revisionId: finalRevisionId, + coordinatorMode: "ensure-running", + plan: options.hardlinks, + serviceIds: targets, + }); + if (leaf && root.executor === "host-cli") { + leaf.ordinal = 3; + leaf.execution.operationKey = controlChildOperationKey(root.operationKey, "hardlinks", 3); + } + const { serviceIds: _targets, ...body } = options; + const plan = dataUpdatePlanSchema.parse({ + ...body, + version: 1, + dataReservation, + finalizationReservation, + dataChildren, + finalizationChildren: leaf ? [leaf] : [], + finalRevisionId, + dataDelegationId: dataUpdateAddress(root.operationKey, "data-grant"), + finalizationDelegationId: dataUpdateAddress(root.operationKey, "finalization-grant"), + }); + if (Buffer.byteLength(JSON.stringify(plan)) > 48 * 1024) + throw Error("Full Update plan exceeds fixed spawn capacity"); + return plan; +} diff --git a/packages/core/src/build-artifacts/data-update-sources.ts b/packages/core/src/build-artifacts/data-update-sources.ts new file mode 100644 index 000000000..36fdc64f5 --- /dev/null +++ b/packages/core/src/build-artifacts/data-update-sources.ts @@ -0,0 +1,107 @@ +/** Bounded source observation only. No authority, writer or grant constructors. */ +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readdirSync, + readFileSync, +} from "node:fs"; +import { join, resolve } from "node:path"; +import { parseEnv } from "node:util"; +export function dataUpdateSourcesDigest(infraDir: string): string { + const hash = createHash("sha256"); + for (const name of [".env", "service-selection.json", "docker-compose.release.yml"]) { + hash.update(name); + let fd: number | undefined; + try { + fd = openSync(join(infraDir, name), constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1 || before.size > 256 * 1024) + throw Error("Full Update source unavailable"); + const bytes = readFileSync(fd), + after = fstatSync(fd); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + before.size !== after.size + ) + throw Error("Full Update source changed"); + hash.update(bytes); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + hash.update("absent"); + } finally { + if (fd !== undefined) closeSync(fd); + } + } + const catalog = join(resolve(infraDir, "../.."), "services"); + try { + const entries = readdirSync(catalog).sort(); + if (entries.length > 256) throw Error("Full Update catalog exceeds capacity"); + for (const entry of entries) { + const directory = join(catalog, entry), + stat = lstatSync(directory); + if (stat.isSymbolicLink()) throw Error("Full Update catalog source unavailable"); + if (!stat.isDirectory()) continue; + const path = join(directory, "service.json"); + let fd: number | undefined; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1 || before.size > 128 * 1024) + throw Error("Full Update catalog source unavailable"); + const bytes = readFileSync(fd), + after = fstatSync(fd); + if ( + before.ino !== after.ino || + before.ctimeMs !== after.ctimeMs || + before.mtimeMs !== after.mtimeMs || + before.size !== after.size + ) + throw Error("Full Update catalog changed"); + hash.update(entry).update(bytes); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } finally { + if (fd !== undefined) closeSync(fd); + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + return hash.digest("hex"); +} + +/** Matches the CLI's process.loadEnvFile precedence without mutating this + * service's environment or retaining unrelated secret values in the plan. */ +export function dataUpdateHostDefaults(infraDir: string) { + let fd: number | undefined; + try { + fd = openSync(join(infraDir, ".env"), constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1 || before.size > 256 * 1024) + throw Error("Full Update host defaults unavailable"); + const bytes = readFileSync(fd), + after = fstatSync(fd); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw Error("Full Update host defaults changed"); + const env = parseEnv(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + return { domain: env.DOMAIN, region: env.OPENMAPX_REGION, countries: env.TRANSITOUS_COUNTRIES }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return {}; + } finally { + if (fd !== undefined) closeSync(fd); + } +} diff --git a/packages/core/src/build-artifacts/data-update.ts b/packages/core/src/build-artifacts/data-update.ts new file mode 100644 index 000000000..2dc246a6e --- /dev/null +++ b/packages/core/src/build-artifacts/data-update.ts @@ -0,0 +1,136 @@ +/** Fixed Full Update admission and verified prerequisite source. No public input + * can substitute for the original planned CLI or a DM terminal witness. */ +import { dirname } from "node:path"; +import { canonicalJson } from "../ops/canonical-json"; +import type { OpsOperation } from "../ops/contract"; +import type { ExecutionRef } from "../ops/execution-ref"; +import { opsOperationFingerprint } from "../ops/fingerprint"; +import { currentConfigurationGeneration } from "../services/configuration-generation"; +import { + controlDigest, + executionAuthorityState, + readPrivateRecord, + signControl, + type VerifiedExecutionAuthority, + verifyControl, + writePrivateRecord, +} from "./control-authority"; +import { + createDataUpdatePlan, + type DataUpdatePlan, + dataUpdatePlanSchema, +} from "./data-update-records"; +import { dataUpdateHostDefaults, dataUpdateSourcesDigest } from "./data-update-sources"; +import { readOwnedGenerationHardlinkPlan } from "./generation-hardlinks-owner"; +import { verifyFamilyOwnedRecord } from "./retention-family"; + +const domain = "openmapx-data-update-plan-v1"; +export function planDataUpdate(options: { + infraDir: string; + parent: ExecutionRef; + operation: Extract; + serviceIds: readonly string[]; + defaultRegion?: string; + defaultCountries?: string; +}): DataUpdatePlan { + const revisionId = currentConfigurationGeneration(options.infraDir); + if (!revisionId) throw Error("Full Update requires an applied configuration"); + const sourceBefore = dataUpdateSourcesDigest(options.infraDir), + defaults = dataUpdateHostDefaults(options.infraDir); + const countries = [ + ...new Set( + options.operation.countryCodes ?? + (options.defaultCountries ?? defaults.countries ?? "") + .split(",") + .map((v) => v.trim().toUpperCase()) + .filter(Boolean), + ), + ].sort(); + const sourcesDigest = dataUpdateSourcesDigest(options.infraDir); + if (sourceBefore !== sourcesDigest) throw Error("Full Update source changed during admission"); + return createDataUpdatePlan({ + parent: options.parent, + operation: options.operation, + domain: process.env.DOMAIN ?? defaults.domain ?? "localhost", + region: options.operation.regionId ?? options.defaultRegion ?? defaults.region ?? "", + countries, + revisionId, + sourcesDigest, + hardlinks: readOwnedGenerationHardlinkPlan(options.infraDir, revisionId), + serviceIds: options.serviceIds, + }); +} +function validate(authority: VerifiedExecutionAuthority, plan: DataUpdatePlan) { + const s = executionAuthorityState(authority); + if ( + s.kind !== "data.update" || + !s.origin || + opsOperationFingerprint(plan.operation) !== s.execution.operationFingerprint + ) + throw Error("Original Full Update authority required"); + const ownRoot = controlDigest(plan.parent) === controlDigest(s.execution); + const child = [plan.dataReservation, plan.finalizationReservation].find( + (r) => + r.execution.operationKey === s.execution.operationKey && + r.execution.executionId === s.execution.executionId, + ); + if (!ownRoot && !child) throw Error("Foreign Full Update plan"); + const expected = ownRoot + ? s.execution.executor === "host-cli" + ? [...plan.dataChildren, ...plan.finalizationChildren] + : [plan.dataReservation, plan.finalizationReservation] + : child === plan.dataReservation + ? plan.dataChildren + : plan.finalizationChildren; + if (canonicalJson(s.reservations) !== canonicalJson(expected)) + throw Error("Full Update phase plan changed"); +} +export function retainDataUpdatePlan( + authority: VerifiedExecutionAuthority, + value: DataUpdatePlan, +): void { + const plan = dataUpdatePlanSchema.parse(value); + validate(authority, plan); + const s = executionAuthorityState(authority); + writePrivateRecord( + s.installation, + "observations", + `data-update-plan-${s.execution.operationKey}.json`, + signControl(s.installation.privateKey, domain, plan), + true, + s.origin, + ); +} +export function readDataUpdatePlan(authority: VerifiedExecutionAuthority): DataUpdatePlan { + const s = executionAuthorityState(authority), + name = `data-update-plan-${s.execution.operationKey}.json`, + raw = readPrivateRecord(s.installation, "observations", name, 128 * 1024); + if (!s.origin) throw Error("Dated Full Update owner required"); + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace: "observations", container: "", name }, + canonicalJson(raw), + ); + const plan = verifyControl( + s.installation.installation.publicKey, + domain, + raw, + dataUpdatePlanSchema, + ); + validate(authority, plan); + return plan; +} +export function assertDataUpdateSources(authority: VerifiedExecutionAuthority) { + const s = executionAuthorityState(authority), + plan = readDataUpdatePlan(authority), + infraDir = dirname(s.installation.dataRoot); + if ( + currentConfigurationGeneration(infraDir) !== plan.revisionId || + dataUpdateSourcesDigest(infraDir) !== plan.sourcesDigest + ) + throw Error("Full Update configuration sources changed"); + return plan; +} + +export { verifyDataUpdatePrerequisites } from "./control-authority"; diff --git a/packages/core/src/build-artifacts/fixed-data-execution.ts b/packages/core/src/build-artifacts/fixed-data-execution.ts index 7ad9e8c1a..2cabd475d 100644 --- a/packages/core/src/build-artifacts/fixed-data-execution.ts +++ b/packages/core/src/build-artifacts/fixed-data-execution.ts @@ -216,8 +216,8 @@ export function retainCliDataAttempt( if ( !slot || !binding || - !isFixedSingleDataCommand(s.kind) || - slot.phase !== "data-step" || + !(isFixedSingleDataCommand(s.kind) || s.kind === "data.update") || + !["data-step", "osm", "transit", "fonts"].includes(slot.phase) || slot.kind !== attempt.input.kind || slot.operationFingerprint !== fixedDataBodyFingerprint(attempt.input) || attempt.operationFingerprint !== slot.operationFingerprint || @@ -267,12 +267,18 @@ export function verifyCliDataTerminal( return verifyDataControlTerminalWitness(s.installation.installation, binding.execution, status); } -export function cliDataReservation(authority: VerifiedExecutionAuthority) { +export function cliDataReservation( + authority: VerifiedExecutionAuthority, + phase: "data-step" | "osm" | "transit" | "fonts" = "data-step", +) { const s = executionAuthorityState(authority), slots = s.reservations.filter( - (slot) => slot.recipient === "data-manager" && slot.phase === "data-step", + (slot) => slot.recipient === "data-manager" && slot.phase === phase, ); - if (!isFixedSingleDataCommand(s.kind) || slots.length !== 1) + if ( + !(isFixedSingleDataCommand(s.kind) || (s.kind === "data.update" && phase !== "data-step")) || + slots.length !== 1 + ) throw Error("Matching data command reservation required"); return { ...slots[0]!, diff --git a/packages/core/src/build-artifacts/generation-hardlinks-owner.ts b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts index cafbb2e57..bb37b3d4a 100644 --- a/packages/core/src/build-artifacts/generation-hardlinks-owner.ts +++ b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts @@ -41,6 +41,7 @@ import { executionAuthorityState, readPrivateRecord, type VerifiedExecutionAuthority, + verifyDataControlTerminalWitness, writePrivateRecord, } from "./control-authority"; import { controlDigest, signControl, verifyControl } from "./control-records"; @@ -104,7 +105,9 @@ function validateOwner( planDigest: controlDigest(plan.plan), }; if ( - !["configuration.hardlinks.apply", "service.start", "service.update"].includes(s.kind) || + !["configuration.hardlinks.apply", "service.start", "service.update", "data.update"].includes( + s.kind, + ) || controlDigest(plan.parent) !== controlDigest(s.execution) || (plan.reservation ? plan.reservation.operationFingerprint !== controlDigest(input) || @@ -198,7 +201,7 @@ export async function executeOwnedGenerationHardlinks(options: { if (!prior) { const issued = await issueControlChildPhase(authority, { phase: "hardlinks", - ordinal: 0, + ordinal: plan.reservation.ordinal, childPublicKey: controlRolePublicKey(s.installation.installation, "data-manager"), lease, }); @@ -246,10 +249,15 @@ export async function executeOwnedGenerationHardlinks(options: { const until = Math.min(Date.now() + 5000, authority.execution.deadlineAt + 30000); do { try { + // The fixed DM publisher verifies physical settlement before signing this + // witness. Observe it without taking the lease writer; polling that writer + // here can itself make the publisher fail its nonblocking guard. + verifyDataControlTerminalWitness(s.installation.installation, bound.execution, status); const settlement = inspectArtifactTargetLeaseChildSettlement(lease, bound.execution); contained = settlement?.outcome === "committed" && settlement.publication === "committed"; } catch { - /* Public terminal publication can precede the final borrowed-node receipt. */ + /* SQL terminal and borrowed-node settlement precede the final shared + * contained-terminal witness. Both actual sources must be visible. */ } if (contained || Date.now() >= until) break; await new Promise((resolve) => setTimeout(resolve, 50)); diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 2017b5c14..61869ee53 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -148,6 +148,14 @@ export { type DataExecutionControlJournal, dataTerminalReceiptDigest, } from "./data-execution-journal"; +export { + assertDataUpdateSources, + planDataUpdate, + readDataUpdatePlan, + retainDataUpdatePlan, + verifyDataUpdatePrerequisites, +} from "./data-update"; +export { type DataUpdatePlan, dataUpdatePlanSchema } from "./data-update-records"; export { commitFeedProxyValidationFailure, feedProxyValidationCompletionDigest, diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index 89072313b..f036a7373 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -98,6 +98,7 @@ import { verifyControlPreparedTerminal, verifyControlRecoveryAuthority, verifyDataControlTerminalWitness, + verifyDataUpdatePrerequisites, withControlDelegationPublication, withControlMaintenanceDecisionPublication, withControlMaintenanceInitialization, @@ -1006,6 +1007,7 @@ export async function acquireArtifactTargetLease(options: { Date.now() >= authority.execution.deadlineAt ) throw Error("Target closure or execution deadline mismatch"); + if (authority.kind === "data.update") await verifyDataUpdatePrerequisites(options.owner); assertControlExecutionForward(options.owner); options.signal?.throwIfAborted(); try { @@ -1275,7 +1277,7 @@ export function verifyFiniteControlEdge( if ( role === "ops-agent" && recipient === "fixed-cli" && - ["lifecycle", "build", "data-link", "data-clean"].includes(phase) && + ["lifecycle", "build", "data-link", "data-clean", "data-finalization"].includes(phase) && childPhaseKinds[phase].includes(kind) ) return phase === "data-clean" ? [] : ["hardlinks"]; @@ -1291,6 +1293,7 @@ export function verifyFiniteControlEdge( "services.buildAll", "data.link", "configuration.hardlinks.apply", + "data.update", ].includes(kind) ) return []; diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 6388897cb..11fe8d361 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -11,6 +11,7 @@ import { stackRenderOperationSchema, vaultApplyOperationSchema, } from "./configuration-operation-records"; +import { dataWorkflowReceiptSchema } from "./job-evidence"; export { configurationGenerationRevisionSchema } from "./configuration-operation-records"; @@ -1242,7 +1243,7 @@ export const OPS_RESULT_SCHEMAS = { }), "data.downloadOsm": data, "data.downloadFonts": data, - "data.update": data, + "data.update": data.extend({ dataWorkflow: dataWorkflowReceiptSchema.optional() }), "data.convertOverpass": data, "data.link": data, "data.clean": data, diff --git a/packages/core/src/services/__tests__/admin-data-workflow.test.ts b/packages/core/src/services/__tests__/admin-data-workflow.test.ts new file mode 100644 index 000000000..471f56c33 --- /dev/null +++ b/packages/core/src/services/__tests__/admin-data-workflow.test.ts @@ -0,0 +1,171 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, + retainAdmittedControlExecution, +} from "../../build-artifacts/control-authority"; +import { + createControlRootOperationKey, + executionFingerprint, + opsOperationFingerprint, +} from "../../ops"; +import { openAdminDataWorkflow } from "../admin-data-workflow"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +async function fixture() { + const root = mkdtempSync(join(tmpdir(), "omx-update-store-")); + roots.push(root); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + const installation = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["data-manager"], + }); + const operation = { + kind: "data.update" as const, + regionId: "europe/germany", + countryCodes: ["DE"], + }; + const admittedAt = Date.now(), + deadlineAt = admittedAt + 60000, + operationFingerprint = opsOperationFingerprint(operation); + const execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + const authority = retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["data-manager"], + }); + return { + root, + installation, + execution, + authority, + options: { + dataRoot: join(root, "data"), + operationKey: execution.operationKey, + fingerprint: execution.fingerprint, + authority, + }, + }; +} +const initial = { + version: 1 as const, + phase: "osm" as const, + children: [], + completedBuildIds: [], + failedBuildIds: [], + skippedBuildIds: [], +}; +it("reopens the exact indexed workflow and refuses another owner binding", async () => { + const f = await fixture(); + try { + const store = await openAdminDataWorkflow(f.options); + expect(store.read()).toBeNull(); + await store.checkpoint(initial); + const recovered = await openAdminDataWorkflow(f.options); + expect(recovered.read()).toEqual(initial); + await expect( + openAdminDataWorkflow({ ...f.options, fingerprint: "f".repeat(64) }), + ).rejects.toThrow(); + await expect(store.checkpoint({ ...initial, phase: "complete" })).rejects.toThrow(); + expect(store.read()).toEqual(initial); + } finally { + closeControlInstallation(f.installation); + } +}); +it("rejects a byte-identical private progress file copied onto another inode", async () => { + const f = await fixture(); + try { + const store = await openAdminDataWorkflow(f.options); + await store.checkpoint(initial); + const path = join( + f.root, + "control-private/host-cli/observations", + `data-workflow-${f.execution.operationKey}.json`, + ), + bytes = readFileSync(path); + unlinkSync(path); + writeFileSync(path, bytes, { mode: 0o600 }); + expect(() => store.read()).toThrow(); + } finally { + closeControlInstallation(f.installation); + } +}); +it("refuses a successful Full Update result without its actual retained workflow", async () => { + const f = await fixture(); + try { + const { retainControlEffectReceipt } = await import("../../build-artifacts/control-authority"); + const receipt = { + ...initial, + phase: "complete" as const, + revisionId: `cfg1_${"r".repeat(43)}`, + children: ["osm", "transit", "fonts"].map((step) => ({ + step, + jobId: randomUUID(), + operationKey: `opk1_${"a".repeat(32)}`, + fingerprint: "f".repeat(64), + deadlineAt: f.execution.deadlineAt, + status: "succeeded", + })), + }; + expect(() => + retainControlEffectReceipt(f.authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "ops-result", result: { completed: true, dataWorkflow: receipt } }, + }), + ).toThrow(); + } finally { + closeControlInstallation(f.installation); + } +}); +it("does not turn signed business progress into proof that data or graph effects ran", async () => { + const f = await fixture(); + try { + const { retainControlEffectReceipt } = await import("../../build-artifacts/control-authority"); + const receipt = { + ...initial, + phase: "complete" as const, + revisionId: `cfg1_${"r".repeat(43)}`, + children: (["osm", "transit", "fonts"] as const).map((step, index) => ({ + step, + jobId: randomUUID(), + operationKey: `opk1_${String(index).repeat(32)}`, + fingerprint: "f".repeat(64), + deadlineAt: f.execution.deadlineAt, + status: "succeeded" as const, + })), + }; + const store = await openAdminDataWorkflow(f.options); + await store.checkpoint(receipt); + expect(() => + retainControlEffectReceipt(f.authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "ops-result", result: { completed: true, dataWorkflow: receipt } }, + }), + ).toThrow(); + } finally { + closeControlInstallation(f.installation); + } +}); diff --git a/packages/core/src/services/admin-data-workflow.ts b/packages/core/src/services/admin-data-workflow.ts new file mode 100644 index 000000000..30d9d919f --- /dev/null +++ b/packages/core/src/services/admin-data-workflow.ts @@ -0,0 +1,115 @@ +/** Business progress, never execution authority. Uses original indexed private + * records so retention/capacity/recovery remain owned by the control family. */ +import { z } from "zod"; +import { + controlDigest, + executionAuthorityState, + readPrivateRecord, + signControl, + type VerifiedExecutionAuthority, + verifyControl, + withPrivateEffectWriter, + writePrivateRecord, +} from "../build-artifacts/control-authority"; +import { AnchoredDirectory, identity, sameInode } from "../build-artifacts/filesystem"; +import { verifyFamilyOwnedRecord } from "../build-artifacts/retention-family"; +import { canonicalJson } from "../ops/canonical-json"; +import { type DataWorkflowReceipt, dataWorkflowReceiptSchema } from "../ops/job-evidence"; + +const domain = "openmapx-admin-data-workflow-v1"; +const bodySchema = z.strictObject({ + version: z.literal(1), + operationKey: z.string(), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + receipt: dataWorkflowReceiptSchema, +}); +const phases = ["osm", "transit", "fonts", "build", "link", "complete"] as const; +export async function openAdminDataWorkflow(options: { + dataRoot: string; + operationKey: string; + fingerprint: string; + authority: VerifiedExecutionAuthority; +}): Promise<{ + read(): DataWorkflowReceipt | null; + checkpoint(receipt: DataWorkflowReceipt): Promise; +}> { + const state = executionAuthorityState(options.authority); + if ( + state.kind !== "data.update" || + !state.origin || + state.execution.operationKey !== options.operationKey || + state.execution.fingerprint !== options.fingerprint + ) + throw Error("Original Full Update owner required"); + const root = new AnchoredDirectory(options.dataRoot), + trusted = new AnchoredDirectory(state.installation.dataRoot); + try { + if (!sameInode(identity(root.stat), identity(trusted.stat))) + throw Error("Foreign workflow data root"); + } finally { + root.close(); + trusted.close(); + } + const name = `data-workflow-${state.execution.operationKey}.json`; + const read = () => { + let raw: unknown; + try { + raw = readPrivateRecord(state.installation, "observations", name, 65536); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + verifyFamilyOwnedRecord( + state.installation, + state.origin!, + { scope: "private", namespace: "observations", container: "", name }, + canonicalJson(raw), + ); + const body = verifyControl(state.installation.installation.publicKey, domain, raw, bodySchema); + if ( + body.operationKey !== state.execution.operationKey || + body.fingerprint !== state.execution.fingerprint + ) + throw Error("Workflow owner changed"); + return body.receipt; + }; + return { + read, + async checkpoint(value) { + const receipt = dataWorkflowReceiptSchema.parse(value); + withPrivateEffectWriter(state, () => { + const prior = read(); + if (prior) { + if ( + phases.indexOf(receipt.phase) < phases.indexOf(prior.phase) || + prior.children.some( + (child, index) => controlDigest(child) !== controlDigest(receipt.children[index]), + ) || + prior.completedBuildIds.some((id) => !receipt.completedBuildIds.includes(id)) || + prior.failedBuildIds.some((id) => !receipt.failedBuildIds.includes(id)) + ) + throw Error("Workflow progress cannot be replaced or rewound"); + if (canonicalJson(prior) === canonicalJson(receipt)) return; + } + if ( + receipt.phase === "complete" && + (receipt.children.length !== 3 || + receipt.children.some((child) => child.status !== "succeeded") || + receipt.failedBuildIds.length || + receipt.skippedBuildIds.length || + !receipt.revisionId) + ) + throw Error("Incomplete Full Update cannot be reported complete"); + const signed = signControl(state.installation.privateKey, domain, { + version: 1 as const, + operationKey: state.execution.operationKey, + fingerprint: state.execution.fingerprint, + receipt, + }); + if (Buffer.byteLength(canonicalJson(signed)) > 65536) + throw Error("Workflow checkpoint exceeds retention capacity"); + writePrivateRecord(state.installation, "observations", name, signed, !prior, state.origin); + }); + }, + }; +} diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 5ae2ab99f..8143fdc8d 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -1,3 +1,4 @@ +export { openAdminDataWorkflow } from "./admin-data-workflow"; export { type CapabilityCheck, type CapabilityKind, diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 0a8eae504..2e050367e 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -34,6 +34,7 @@ "services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts", "services/data-manager/src/jobs/execution/runtime.test.ts", "services/data-manager/src/jobs/execution/fixed-cli.test.ts", + "services/data-manager/src/jobs/execution/full-update.test.ts", "services/data-manager/src/jobs/execution/runner-cancellation.test.ts", "services/data-manager/src/jobs/execution/hardlinks.test.ts", "services/data-manager/src/jobs/execution/ops-children.test.ts", diff --git a/services/data-manager/src/jobs/execution/full-update.test.ts b/services/data-manager/src/jobs/execution/full-update.test.ts new file mode 100644 index 000000000..3369dd64c --- /dev/null +++ b/services/data-manager/src/jobs/execution/full-update.test.ts @@ -0,0 +1,416 @@ +/** Wiring acceptance: real production agent/default registered CLI/HTTP/SQL and + * signed containment. Synthetic download workers and native Docker I/O keep + * this test independent of public downloads and graph-container images. */ +import { randomBytes } from "node:crypto"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey, OpsClient, type OpsOperation } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { build } from "esbuild"; +import Fastify from "fastify"; +import postgres from "postgres"; +import { expect, it } from "vitest"; +import { ownedServiceHost } from "../../../../../apps/ops-agent/src/__fixtures__/owned-service-host"; +import { configurationPolicyRevision } from "../../../../../apps/ops-agent/src/configured-resource-authority"; +import { openOpsJobJournal } from "../../../../../apps/ops-agent/src/journal"; +import { createProductionRegistryResourceClaimer } from "../../../../../apps/ops-agent/src/policy"; +import { dispatchOpsOperation } from "../../../../../apps/ops-agent/src/runtime"; +import { buildOpsAgentServer } from "../../../../../apps/ops-agent/src/server"; +import { createFileTrustedOpsDataSource } from "../../../../../apps/ops-agent/src/trusted-config-source"; +import { registerApi } from "../../api"; +import { registerAuth } from "../../auth"; +import { StateStore } from "../../state"; +import { resolveOperationsProfile } from "../transitous/operations-profile"; +import { getSingleFlightController } from "../transitous/runtime"; +import { createDataConfigurationMaintenance } from "./configuration-maintenance"; +import { createDataExecutionDispatcher } from "./dispatch"; +import { applyOwnedGenerationHardlinks } from "./hardlinks"; +import { createDataExecutionReceiver } from "./receiver"; +import { createDataExecutionRuntime } from "./runtime"; +import { createPostgresDataExecutionStore } from "./store"; +import { publishDataChange, retainDataPublication } from "./worker-scope"; + +it.skipIf( + !process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL && + process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1", +)( + "awaits the original transit child before native graph commands and generation, after a lost admission response", + async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const bundle = mkdtempSync(join(tmpdir(), "omx-full-update-bundle-")); + await build({ + entryPoints: [resolve("packages/cli/__tests__/fixtures/single-data-cli-process.ts")], + outfile: join(bundle, "cli.mjs"), + bundle: true, + platform: "node", + format: "esm", + target: "node24", + tsconfig: resolve("packages/cli/tsconfig.json"), + banner: { + js: `import {createRequire} from 'node:module';const require=createRequire(${JSON.stringify(resolve("packages/cli/package.json"))});`, + }, + }); + const host = await ownedServiceHost({ + withHardlinks: true, + extraServiceIds: ["osrm", "motis-feed-proxy", "motis-staging"], + defaultCliEntry: join(bundle, "agent.mjs"), + }), + data = join(host.infraDir, "data"), + sql = postgres(url, { max: 4 }); + const keys = [ + "OPENMAPX_ROOT_DIR", + "DATA_MANAGER_URL", + "DATA_MANAGER_AUTH_TOKEN", + "POSTGRES_PASSWORD", + "BETTER_AUTH_SECRET", + "DOMAIN", + "PATH", + ] as const, + prior = Object.fromEntries(keys.map((key) => [key, process.env[key]])); + process.env.DOMAIN = "example.test"; + process.env.OPENMAPX_ROOT_DIR = host.root; + process.env.DATA_MANAGER_AUTH_TOKEN = "synthetic-full-update-token-" + "d".repeat(32); + process.env.POSTGRES_PASSWORD = "synthetic-full-update-password-" + "p".repeat(32); + process.env.BETTER_AUTH_SECRET = "synthetic-full-update-secret-" + "s".repeat(32); + const service = host.loaded.find((s) => s.manifest.id === "osrm")!; + service.manifest.buildCommand = "openmapx services build osrm"; + writeFileSync(join(service.directory, "service.json"), JSON.stringify(service.manifest)); + const docker = join(bundle, "docker"), + commands = join(bundle, "docker-commands.jsonl"); + writeFileSync( + docker, + `#!${process.execPath}\nconst fs=require('node:fs'),path=require('node:path'),a=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(commands)},JSON.stringify(a)+'\\n');if(a[0]==='run'){const m=a[a.indexOf('-v')+1];if(m&&m.endsWith(':/data'))fs.writeFileSync(path.join(m.slice(0,-6),'region.osrm'),'synthetic-graph');}if(a.includes('ps'))process.stdout.write('');\n`, + { mode: 0o700 }, + ); + chmodSync(docker, 0o700); + process.env.PATH = `${bundle}:${process.env.PATH}`; + const dm = control.openControlInstallation({ + infraDir: host.infraDir, + role: "data-manager", + serviceIds: host.loaded.map((s) => s.manifest.id).sort(), + }), + agentInstallation = control.openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id).sort(), + }), + journal = control.createDataExecutionControlJournal(dm, sql), + fence = createDataConfigurationMaintenance(sql, dm); + const publicationFailures: Record = {}; + const publishContained = journal.publishContainedTerminal; + journal.publishContainedTerminal = async (execution) => { + try { + return await publishContained(execution); + } catch (error) { + publicationFailures[execution.operationKey] = + error instanceof Error ? (error.stack ?? error.message) : String(error); + throw error; + } + }; + await fence.startup(); + const store = createPostgresDataExecutionStore(sql, { + controlJournal: journal, + configurationFence: fence, + }), + singleFlight = getSingleFlightController(), + operationsPolicy = resolveOperationsProfile({ countries: ["de"] }); + let releaseTransit!: () => void, transitStarted!: () => void; + const held = new Promise((resolve) => { + releaseTransit = resolve; + }), + started = new Promise((resolve) => { + transitStarted = resolve; + }); + const effects: string[] = []; + const dispatcher = createDataExecutionDispatcher({ + dataDir: data, + state: new StateStore(data), + sql, + singleFlight, + operationsPolicy, + runHardlinks: async () => { + throw Error("Actual generation receiver owns its hardlink worker"); + }, + }); + const runtime = createDataExecutionRuntime({ + store, + dispatch: async (input, context) => { + effects.push(input.kind); + if (input.kind === "generation-hardlinks") return applyOwnedGenerationHardlinks(context); + if (input.kind === "download-osm") { + await publishDataChange(() => { + mkdirSync(join(data, "osm"), { recursive: true }); + writeFileSync(join(data, "osm/europe-germany.osm.pbf"), "synthetic-public-osm"); + mkdirSync(join(data, "source-osm"), { recursive: true }); + writeFileSync(join(data, "source-osm/input.pbf"), "synthetic-public-osm"); + }); + return { kind: input.kind, region: input.region, sizeBytes: 20 }; + } + if (input.kind === "transit-sync") { + transitStarted(); + await held; + context.assertForward(); + await retainDataPublication(); + return { + kind: input.kind, + countries: input.countries, + sourceRevision: "synthetic-catalog", + finalStatus: "ok", + }; + } + if (input.kind === "download-fonts") { + await retainDataPublication(); + return { kind: input.kind, completed: true }; + } + throw Error("Unexpected synthetic download worker"); + }, + }), + receiver = createDataExecutionReceiver({ + runtime, + installation: dm, + configurationFence: fence, + dispatcher, + operationsPolicy, + }), + app = Fastify(); + const responses: { status: number; payload: string }[] = []; + let admissions = 0; + app.addHook("onSend", async (request, reply, payload) => { + if (request.method === "POST" && request.url === "/executions") { + responses.push({ + status: reply.statusCode, + payload: typeof payload === "string" ? payload.slice(0, 1000) : "non-string", + }); + admissions++; + if (admissions === 2) reply.raw.destroy(); + } + return payload; + }); + registerAuth(app, process.env.DATA_MANAGER_AUTH_TOKEN, { executionReceiver: () => receiver }); + registerApi(app, { + dataDir: data, + singleFlight, + operationsPolicy, + executionReceiver: receiver, + }); + await app.listen({ host: "127.0.0.1", port: 0 }); + const address = app.server.address(); + if (!address || typeof address === "string") throw Error("DM address missing"); + process.env.DATA_MANAGER_URL = `http://127.0.0.1:${address.port}`; + const agentJournal = await openOpsJobJournal(join(data, "ops-agent/jobs-v1.json")), + token = "a".repeat(43), + tokenFile = join(bundle, "token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + let actualFailure: unknown; + const agent = buildOpsAgentServer({ + dispatch: async (op, runtime, context) => { + try { + return await dispatchOpsOperation(runtime, op, context); + } catch (error) { + actualFailure = error; + throw error; + } + }, + runtime: host.runtime, + journal: agentJournal, + controlInstallation: agentInstallation, + tokens: { api: token, "data-manager": "d".repeat(43) }, + resourceClaimer: createProductionRegistryResourceClaimer({ + services: host.loaded.map((s) => ({ + serviceId: s.manifest.id, + enabled: s.enabled, + isBuiltIn: s.isBuiltIn, + selectionAuthority: s, + })), + integrationIds: [], + loadAuthority: async () => { + const a = await host.loadAuthority(); + return { + revisionId: configurationPolicyRevision(a.revisionId), + configurationAuthorityRevision: a.revisionId, + services: a.services.map((s) => ({ + serviceId: s.manifest.id, + enabled: s.enabled, + isBuiltIn: s.isBuiltIn, + selectionAuthority: s, + })), + integrationIds: [], + }; + }, + enabledServiceIds: () => new Set(["data-manager", "motis", "osrm"]), + trustedData: createFileTrustedOpsDataSource({ + directory: host.queue, + token: host.token, + allowedUids: [process.geteuid?.() ?? 0], + expectedGid: process.getegid?.() ?? 0, + }), + ownedServices: host.owned, + resourceAuthority: { + allowRegion: (kind, id) => kind === "data.update" && id === "europe/germany", + allowCountry: (kind, id) => kind === "data.update" && id === "DE", + }, + }), + }); + await agent.listen({ host: "127.0.0.1", port: 0 }); + const agentAddress = agent.server.address(); + if (!agentAddress || typeof agentAddress === "string") throw Error("Agent address missing"); + const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agentAddress.port}`, tokenFile }); + let operationId: string | undefined, rootOperationKey: string | undefined; + try { + const operation: OpsOperation = { + kind: "data.update", + regionId: "europe/germany", + countryCodes: ["DE"], + }, + deadlineAt = Date.now() + 240000, + key = createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + admitted = await client.execute(operation, { + operationKey: key, + execution: { requestedDeadlineAt: deadlineAt }, + }); + if (admitted.execution !== "async") throw Error("Async Full Update required"); + operationId = admitted.operationId; + rootOperationKey = key; + await Promise.race([ + started, + new Promise((_, reject) => setTimeout(() => reject(Error("Transit did not start")), 60000)), + ]); + expect(effects).toEqual(["download-osm", "transit-sync"]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(host.initial.revisionId); + expect(existsSync(commands) ? readFileSync(commands, "utf8") : "").not.toContain( + "osrm-extract", + ); + expect(existsSync(join(data, ".admin-target-control/locks/osrm"))).toBe(false); + releaseTransit(); + const until = Date.now() + 90000; + let terminal = await client.status(operation.kind, operationId!); + while ( + !["succeeded", "failed", "timed_out", "terminated"].includes(terminal.state) && + !actualFailure && + Date.now() < until + ) { + await new Promise((resolve) => setTimeout(resolve, 200)); + terminal = await client.status(operation.kind, operationId!); + } + expect(terminal).toMatchObject({ + state: "succeeded", + result: { + completed: true, + dataWorkflow: { phase: "complete", completedBuildIds: ["osrm"] }, + }, + }); + expect(admissions).toBe(3); + expect(effects).toEqual([ + "download-osm", + "transit-sync", + "download-fonts", + "generation-hardlinks", + ]); + expect(readFileSync(commands, "utf8")).toContain("osrm-extract"); + expect(existsSync(join(data, "osrm-graph/region.osrm"))).toBe(true); + expect(services.currentConfigurationGeneration(host.infraDir)).not.toBe( + host.initial.revisionId, + ); + expect(existsSync(join(data, ".admin-target-control/locks/osrm"))).toBe(false); + } catch (error) { + const witnessProblems: Record = {}; + for (const row of await store.listUnsettled()) { + if ( + !existsSync( + join( + host.infraDir, + "control-private/data-manager/executions", + `${row.status.operationKey}.json`, + ), + ) + ) + continue; + try { + control.verifyDataControlTerminalWitness(agentInstallation, row.execution, row.status); + witnessProblems[row.input.kind] = "verified"; + } catch (problem) { + witnessProblems[row.input.kind] = + problem instanceof Error ? (problem.stack ?? problem.message) : String(problem); + } + } + if (operationId) + console.info( + "Full Update synthetic diagnostics", + { + effects, + admissions, + responses, + witnessProblems, + publicationFailures, + actualFailure: actualFailure instanceof Error ? actualFailure.stack : actualFailure, + rows: (await store.listUnsettled()) + .filter((row) => + existsSync( + join( + host.infraDir, + "control-private/data-manager/executions", + `${row.status.operationKey}.json`, + ), + ), + ) + .map((row) => ({ + kind: row.input.kind, + status: row.status, + finishedAt: row.finishedAt, + })), + commands: existsSync(commands) ? readFileSync(commands, "utf8") : "", + }, + await client.status("data.update", operationId), + await client.events(operationId), + ); + throw error; + } finally { + releaseTransit(); + agent.server.closeAllConnections(); + await agent.close(); + app.server.closeAllConnections(); + await app.close(); + await runtime.close(); + for (const row of await store.listUnsettled()) + if ( + existsSync( + join( + host.infraDir, + "control-private/data-manager/executions", + `${row.status.operationKey}.json`, + ), + ) || + (rootOperationKey && + row.parent && + "execution" in row.parent && + [ + control.controlChildOperationKey(rootOperationKey, "data-workflow", 0), + control.controlChildOperationKey(rootOperationKey, "data-finalization", 1), + ].includes(row.parent.execution.operationKey)) + ) + await sql`DELETE FROM data_manager.jobs WHERE id=${row.status.id} AND operation_key=${row.status.operationKey}`; + journal.close(); + control.closeControlInstallation(dm); + control.closeControlInstallation(agentInstallation); + await sql.end(); + for (const key of keys) { + if (prior[key] === undefined) delete process.env[key]; + else process.env[key] = prior[key]; + } + rmSync(host.cleanupRoot, { recursive: true, force: true }); + rmSync(bundle, { recursive: true, force: true }); + } + }, + 180000, +); From 49f56d886ef0054977a32907e95ed92236a7bff6 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 12:56:35 +0100 Subject: [PATCH 040/135] feat(auth): apply coherent account policy to every operation --- apps/api/openapi.json | 1250 +++ apps/api/src/account/auth-database.ts | 16 + .../src/account/auth-policy-postgres.test.ts | 743 ++ apps/api/src/account/auth-policy.test.ts | 13 + apps/api/src/account/auth-policy.ts | 183 + apps/api/src/account/auth-runtime.ts | 123 + .../password-verification-policy.test.ts | 13 + .../account/password-verification-policy.ts | 40 + apps/api/src/auth-policy-validation.test.ts | 22 + apps/api/src/auth.ts | 609 +- .../src/db/account-administration-schema.ts | 22 + .../db/migrations/0043_admin_auth_policy.sql | 17 + .../src/db/migrations/meta/0043_snapshot.json | 8221 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + apps/api/src/db/schema.ts | 1 + .../routes/__tests__/admin-settings.test.ts | 29 +- apps/api/src/routes/admin-accounts.ts | 59 + apps/api/src/routes/admin-settings.ts | 81 +- apps/api/src/routes/index.ts | 2 + apps/api/src/services/settings.ts | 10 +- .../core/src/api/admin-sdk-transport.test.ts | 9 +- packages/core/src/api/admin-sdk-transport.ts | 2 + packages/core/src/api/admin-sdk.ts | 2 + .../src/api/contracts/account-admin/index.ts | 36 + .../src/api/contracts/account-admin/policy.ts | 29 + packages/core/src/api/contracts/index.ts | 1 + .../admin/account-admin.generated.ts | 211 + .../account-models/applyAdminAuthPolicy200.ts | 12 + ...pplyAdminAuthPolicy200ActivationVersion.ts | 9 + .../applyAdminAuthPolicy200Active.ts | 21 + ...minAuthPolicy200ActiveActivationVersion.ts | 4 + .../applyAdminAuthPolicy200AppliesTo.ts | 9 + .../account-models/applyAdminAuthPolicy400.ts | 20 + .../account-models/applyAdminAuthPolicy401.ts | 20 + .../account-models/applyAdminAuthPolicy403.ts | 20 + .../account-models/applyAdminAuthPolicy409.ts | 20 + .../account-models/applyAdminAuthPolicy413.ts | 20 + .../account-models/applyAdminAuthPolicy415.ts | 20 + .../account-models/applyAdminAuthPolicy429.ts | 11 + .../account-models/applyAdminAuthPolicy500.ts | 20 + .../account-models/applyAdminAuthPolicy503.ts | 20 + .../applyAdminAuthPolicyBody.ts | 12 + .../applyAdminAuthPolicyBodySettings.ts | 17 + .../account-models/getAdminAuthPolicy200.ts | 11 + .../getAdminAuthPolicy200Active.ts | 21 + ...minAuthPolicy200ActiveActivationVersion.ts | 4 + .../getAdminAuthPolicy200Ownership.ts | 13 + ...cy200OwnershipEmailVerificationRequired.ts | 11 + ...AuthPolicy200OwnershipPasswordMinLength.ts | 10 + ...hPolicy200OwnershipSessionDurationHours.ts | 10 + ...AdminAuthPolicy200OwnershipTwoFaEnabled.ts | 10 + .../getAdminAuthPolicy200Pending.ts | 4 + .../account-models/getAdminAuthPolicy400.ts | 20 + .../account-models/getAdminAuthPolicy401.ts | 20 + .../account-models/getAdminAuthPolicy403.ts | 20 + .../account-models/getAdminAuthPolicy413.ts | 20 + .../account-models/getAdminAuthPolicy415.ts | 20 + .../account-models/getAdminAuthPolicy429.ts | 11 + .../account-models/getAdminAuthPolicy500.ts | 20 + .../account-models/getAdminAuthPolicy503.ts | 20 + .../generated/admin/account-models/index.ts | 36 + .../admin/operation-bindings.generated.ts | 14 + scripts/api-sdk/codegen.test.ts | 24 +- scripts/api-sdk/config.mjs | 1 + scripts/database-test-groups.json | 3 +- 65 files changed, 11986 insertions(+), 343 deletions(-) create mode 100644 apps/api/src/account/auth-database.ts create mode 100644 apps/api/src/account/auth-policy-postgres.test.ts create mode 100644 apps/api/src/account/auth-policy.test.ts create mode 100644 apps/api/src/account/auth-policy.ts create mode 100644 apps/api/src/account/auth-runtime.ts create mode 100644 apps/api/src/account/password-verification-policy.test.ts create mode 100644 apps/api/src/account/password-verification-policy.ts create mode 100644 apps/api/src/auth-policy-validation.test.ts create mode 100644 apps/api/src/db/account-administration-schema.ts create mode 100644 apps/api/src/db/migrations/0043_admin_auth_policy.sql create mode 100644 apps/api/src/db/migrations/meta/0043_snapshot.json create mode 100644 apps/api/src/routes/admin-accounts.ts create mode 100644 packages/core/src/api/contracts/account-admin/index.ts create mode 100644 packages/core/src/api/contracts/account-admin/policy.ts create mode 100644 packages/core/src/api/generated/admin/account-admin.generated.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActivationVersion.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200Active.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActiveActivationVersion.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200AppliesTo.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy400.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy401.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy403.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy409.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy413.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy415.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy429.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy500.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy503.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBody.ts create mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBodySettings.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Active.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200ActiveActivationVersion.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Ownership.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipEmailVerificationRequired.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipPasswordMinLength.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipSessionDurationHours.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipTwoFaEnabled.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Pending.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy400.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy401.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy403.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy413.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy415.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy429.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy500.ts create mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy503.ts create mode 100644 packages/core/src/api/generated/admin/account-models/index.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index a963529af..f0642bfe6 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -1624,6 +1624,10 @@ "version": "1.0.0" }, "tags": [ + { + "name": "account-admin", + "description": "Routes under /api/account-admin." + }, { "name": "activity-admin", "description": "Activity administration: application and data-manager jobs, cancellation and live application events." @@ -1782,6 +1786,1252 @@ } ], "paths": { + "/api/admin/accounts/policy": { + "get": { + "operationId": "getAdminAuthPolicy", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "active": { + "type": "object", + "properties": { + "emailVerificationRequired": { + "type": "boolean" + }, + "sessionDurationHours": { + "type": "integer", + "minimum": 1, + "maximum": 8760 + }, + "twoFaEnabled": { + "type": "boolean" + }, + "passwordMinLength": { + "type": "integer", + "minimum": 8, + "maximum": 128 + }, + "activationVersion": { + "anyOf": [ + { + "type": "number", + "enum": [ + 0 + ] + }, + { + "type": "number", + "enum": [ + 1 + ] + } + ] + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "emailVerificationRequired", + "sessionDurationHours", + "twoFaEnabled", + "passwordMinLength", + "activationVersion", + "revision" + ], + "additionalProperties": false + }, + "pending": { + "type": "object", + "additionalProperties": {} + }, + "ownership": { + "type": "object", + "properties": { + "emailVerificationRequired": { + "type": "string", + "enum": [ + "default", + "database", + "env" + ] + }, + "sessionDurationHours": { + "type": "string", + "enum": [ + "default", + "database" + ] + }, + "twoFaEnabled": { + "type": "string", + "enum": [ + "default", + "database" + ] + }, + "passwordMinLength": { + "type": "string", + "enum": [ + "default", + "database" + ] + } + }, + "required": [ + "emailVerificationRequired", + "sessionDurationHours", + "twoFaEnabled", + "passwordMinLength" + ], + "additionalProperties": false + } + }, + "required": [ + "active", + "pending", + "ownership" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "account-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/accounts/policy/apply": { + "post": { + "operationId": "applyAdminAuthPolicy", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "settings": { + "type": "object", + "properties": { + "emailVerificationRequired": { + "type": "boolean" + }, + "sessionDurationHours": { + "type": "integer", + "minimum": 1, + "maximum": 8760 + }, + "twoFaEnabled": { + "type": "boolean" + }, + "passwordMinLength": { + "type": "integer", + "minimum": 8, + "maximum": 128 + } + }, + "required": [ + "emailVerificationRequired", + "sessionDurationHours", + "twoFaEnabled", + "passwordMinLength" + ], + "additionalProperties": false + }, + "expectedRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "confirmed": { + "type": "boolean", + "enum": [ + true + ] + }, + "idempotencyKey": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + } + }, + "required": [ + "settings", + "expectedRevision", + "confirmed", + "idempotencyKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "active": { + "type": "object", + "properties": { + "emailVerificationRequired": { + "type": "boolean" + }, + "sessionDurationHours": { + "type": "integer", + "minimum": 1, + "maximum": 8760 + }, + "twoFaEnabled": { + "type": "boolean" + }, + "passwordMinLength": { + "type": "integer", + "minimum": 8, + "maximum": 128 + }, + "activationVersion": { + "anyOf": [ + { + "type": "number", + "enum": [ + 0 + ] + }, + { + "type": "number", + "enum": [ + 1 + ] + } + ] + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "emailVerificationRequired", + "sessionDurationHours", + "twoFaEnabled", + "passwordMinLength", + "activationVersion", + "revision" + ], + "additionalProperties": false + }, + "authPolicyRevision": { + "type": "string" + }, + "activationVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "appliesTo": { + "type": "string", + "enum": [ + "new_auth_operations" + ] + } + }, + "required": [ + "active", + "authPolicyRevision", + "activationVersion", + "appliesTo" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "account-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/admin/ambient-places/build": { "post": { "operationId": "postApiAdminAmbientPlacesBuild", diff --git a/apps/api/src/account/auth-database.ts b/apps/api/src/account/auth-database.ts new file mode 100644 index 000000000..0912e680c --- /dev/null +++ b/apps/api/src/account/auth-database.ts @@ -0,0 +1,16 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { db } from "../db"; +export type AuthDatabase = Pick< + typeof db, + "select" | "insert" | "update" | "delete" | "execute" | "query" | "transaction" +>; +const context = new AsyncLocalStorage(); +export function currentAuthDatabase(): AuthDatabase { + return context.getStore() ?? db; +} +export function withAuthDatabase(database: AuthDatabase, run: () => Promise): Promise { + return context.run(database, run); +} +export function isTransactionalAuthDatabase() { + return context.getStore() !== undefined; +} diff --git a/apps/api/src/account/auth-policy-postgres.test.ts b/apps/api/src/account/auth-policy-postgres.test.ts new file mode 100644 index 000000000..d7f75dda7 --- /dev/null +++ b/apps/api/src/account/auth-policy-postgres.test.ts @@ -0,0 +1,743 @@ +import { randomBytes } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + account, + authPolicyApplication, + authPolicyState, + session, + systemSettings, + user, +} from "../db/schema"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const dbDescribe = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1" ? describe : describe.skip; +dbDescribe("account policy through installed Better Auth", () => { + let fixture: Awaited>, + auth: typeof import("../auth")["auth"]; + const mail: Array<{ to: string; text: string }> = []; + beforeAll(async () => { + fixture = await createMigratedPostgresFixture(); + vi.stubEnv("BETTER_AUTH_SECRET", randomBytes(48).toString("base64url")); + vi.stubEnv("BETTER_AUTH_URL", "http://localhost:3001"); + vi.doMock("../db", () => ({ db: fixture.db, sql: fixture.sql })); + vi.doMock("../utils/email", () => ({ + sendMail: async (message: { to: string; text: string }) => { + mail.push(message); + }, + })); + await fixture.db.insert(systemSettings).values([ + { key: "emailVerificationRequired", value: false }, + { key: "twoFaEnabled", value: false }, + { key: "passwordMinLength", value: 40 }, + ]); + ({ auth } = await import("../auth")); + }, 90000); + afterAll(async () => { + await fixture?.stop(); + vi.doUnmock("../db"); + vi.doUnmock("../utils/email"); + vi.unstubAllEnvs(); + }); + it("rejects admin-created short passwords before either user or account persistence", async () => { + const email = "short-policy@example.test"; + await expect( + auth.api.createUser({ body: { name: "Synthetic", email, password: "short7!" } }), + ).rejects.toThrow(); + expect(await fixture.db.select().from(user).where(eq(user.email, email))).toHaveLength(0); + expect(await fixture.db.select().from(account)).toHaveLength(0); + }); + it("forces admin-created accounts to remain unverified despite nested input", async () => { + const created = await auth.api.createUser({ + body: { + name: "Synthetic", + email: "forced-policy@example.test", + password: "long-enough-for-secure-default", + data: { emailVerified: true }, + }, + }); + expect(created.user.emailVerified).toBe(false); + expect( + (await fixture.db.select().from(user).where(eq(user.id, created.user.id)))[0]?.emailVerified, + ).toBe(false); + }); + it("serializes the policy and enforces actual account-session authorization", async () => { + const Fastify = (await import("fastify")).default; + const { adminAccountsRoute } = await import("../routes/admin-accounts"); + const app = Fastify(); + await app.register(adminAccountsRoute, { prefix: "/api" }); + await app.ready(); + const address = await app.listen({ port: 0, host: "127.0.0.1" }); + try { + expect( + (await app.inject({ method: "GET", url: "/api/admin/accounts/policy" })).statusCode, + ).toBe(401); + expect( + ( + await app.inject({ + method: "GET", + url: "/api/admin/accounts/policy", + headers: { "x-openmapx-local-admin": "" }, + }) + ).statusCode, + ).toBe(403); + const actor = await auth.api.createUser({ + body: { + name: "Policy reader", + email: "policy-reader@example.test", + password: "reader-password", + role: "admin", + }, + }); + await fixture.db.update(user).set({ emailVerified: true }).where(eq(user.id, actor.user.id)); + const signin = await auth.api.signInEmail({ + body: { email: actor.user.email, password: "reader-password" }, + returnHeaders: true, + }); + const cookie = signin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "); + const result = await app.inject({ + method: "GET", + url: "/api/admin/accounts/policy", + headers: { cookie }, + }); + expect(result.statusCode, result.body).toBe(200); + const { getAdminAuthPolicy } = await import("@openmapx/core/api-admin-sdk"); + const { createApiClient } = await import("@openmapx/core"); + const generated = await getAdminAuthPolicy({ + client: createApiClient({ baseUrl: address }), + headers: { cookie }, + }); + expect(generated).toMatchObject({ + status: 200, + data: { active: { activationVersion: 0 }, pending: { passwordMinLength: 40 } }, + }); + + expect(result.json()).toMatchObject({ + active: { activationVersion: 0, emailVerificationRequired: true }, + pending: { passwordMinLength: 40 }, + ownership: { twoFaEnabled: "default" }, + }); + await fixture.db + .update(user) + .set({ role: "privacy_admin" }) + .where(eq(user.id, actor.user.id)); + expect( + ( + await app.inject({ + method: "GET", + url: "/api/admin/accounts/policy", + headers: { cookie }, + }) + ).statusCode, + ).toBe(403); + } finally { + await app.close(); + } + }); + it("pins one operation snapshot, explicitly activates atomically, and expires only newly issued sessions", async () => { + const { loadAuthPolicy, authPolicyView, applyAuthPolicy } = await import("./auth-policy"); + const { withAuthOperation } = await import("../auth"); + const initial = await loadAuthPolicy(); + expect(initial).toMatchObject({ + activationVersion: 0, + emailVerificationRequired: true, + twoFaEnabled: true, + passwordMinLength: 8, + sessionDurationHours: 168, + }); + expect((await authPolicyView()).pending.passwordMinLength).toBe(40); + const admin = await auth.api.createUser({ + body: { + name: "Synthetic admin", + email: "policy-admin@example.test", + password: "original-password", + role: "admin", + }, + }); + await fixture.db.update(user).set({ emailVerified: true }).where(eq(user.id, admin.user.id)); + const adminSignin = await auth.api.signInEmail({ + body: { email: admin.user.email, password: "original-password" }, + returnHeaders: true, + }); + const cookie = adminSignin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "); + const enrolled = await auth.api.enableTwoFactor({ + headers: new Headers({ cookie }), + body: { password: "original-password" }, + }); + const factor = (await fixture.db.select().from((await import("../db/schema")).twoFactor))[0]; + if (!factor) throw Error("No actual TOTP factor"); + const { symmetricDecrypt } = await import("better-auth/crypto"); + const totpSecret = await symmetricDecrypt({ + key: process.env.BETTER_AUTH_SECRET!, + data: factor.secret, + }); + await auth.api.verifyTOTP({ + headers: new Headers({ cookie }), + body: { code: (await auth.api.generateTOTP({ body: { secret: totpSecret } })).code }, + }); + if (enrolled.method !== "totp") throw Error("Unexpected enrollment method"); + expect(enrolled.backupCodes.length).toBeGreaterThan(0); + const subject = await auth.api.createUser({ + body: { + name: "Unverified", + email: "pending-policy@example.test", + password: "original-password", + }, + }); + let captured!: () => void, release!: () => void; + const entered = new Promise((r) => { + captured = r; + }), + gate = new Promise((r) => { + release = r; + }); + const old = withAuthOperation(async () => { + captured(); + await gate; + return auth.api.signInEmail({ + body: { email: subject.user.email, password: "original-password" }, + }); + }); + // Attach rejection observation before releasing the operation. + const oldResult = old.then( + () => "unexpected success", + () => "verification required", + ); + await entered; + const settings = { + emailVerificationRequired: false, + sessionDurationHours: 1, + twoFaEnabled: false, + passwordMinLength: 12, + }; + const request = { + settings, + expectedRevision: initial.revision, + confirmed: true, + idempotencyKey: "policy-activation-synthetic-0001", + }; + const applied = await applyAuthPolicy(request, admin.user.id); + release(); + expect(await oldResult).toBe("verification required"); + expect(await applyAuthPolicy(request, admin.user.id)).toEqual(applied); + await expect( + applyAuthPolicy( + { + ...request, + settings: { ...settings, passwordMinLength: 7 }, + idempotencyKey: "policy-invalid-synthetic-0001", + }, + admin.user.id, + ), + ).rejects.toThrow(); + expect(await loadAuthPolicy()).toEqual(applied); + const issuedAt = Date.now(); + const signin = await auth.api.signInEmail({ + body: { email: subject.user.email, password: "original-password" }, + returnHeaders: true, + }); + const issuedBefore = Date.now(); + expect(signin.response.user.id).toBe(subject.user.id); + const challenge = await auth.api.signInEmail({ + body: { email: admin.user.email, password: "original-password" }, + returnHeaders: true, + }); + expect(challenge.response).toMatchObject({ twoFactorRedirect: true }); + const challengeCookie = challenge.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "); + await expect( + auth.api.verifyBackupCode({ + headers: new Headers({ cookie: challengeCookie }), + body: { code: enrolled.backupCodes[0] ?? "" }, + }), + ).resolves.toHaveProperty("user"); + const ordinarySignin = await auth.api.signInEmail({ + body: { email: subject.user.email, password: "original-password" }, + returnHeaders: true, + }); + const ordinaryCookie = ordinarySignin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "); + await expect( + auth.api.enableTwoFactor({ + headers: new Headers({ + cookie: signin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "), + }), + body: { password: "original-password" }, + }), + ).rejects.toThrow(); + const stored = ( + await fixture.db.select().from(session).where(eq(session.userId, subject.user.id)) + ).find((row) => row.token === signin.response.token); + if (!stored) throw Error("Missing actual session"); + expect(stored.expiresAt.getTime()).toBeGreaterThanOrEqual(issuedAt + 3600000); + expect(stored.expiresAt.getTime()).toBeLessThanOrEqual(issuedBefore + 3600000); + expect(signin.headers.getSetCookie().some((v) => /Max-Age=3600/i.test(v))).toBe(true); + vi.useFakeTimers({ toFake: ["Date"] }); + try { + vi.setSystemTime(Date.now() + 1800000); + expect( + await auth.api.getSession({ headers: new Headers({ cookie: ordinaryCookie }) }), + ).not.toBeNull(); + expect( + (await fixture.db.select().from(session).where(eq(session.id, stored.id)))[0]?.expiresAt, + ).toEqual(stored.expiresAt); + } finally { + vi.useRealTimers(); + } + await expect( + auth.api.createUser({ + body: { name: "Short now", email: "short-active@example.test", password: "eleven-1234" }, + }), + ).rejects.toThrow(); + const response = await auth.handler( + new Request("http://localhost:3001/api/auth/sign-up/email", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + name: "Short HTTP", + email: "http-short@example.test", + password: "short", + }), + }), + ); + expect(response.status).toBe(400); + await applyAuthPolicy( + { + ...request, + settings: { ...settings, sessionDurationHours: 8 }, + expectedRevision: applied.revision, + idempotencyKey: "policy-activation-synthetic-0002", + }, + admin.user.id, + ); + expect( + (await fixture.db.select().from(session).where(eq(session.id, stored.id)))[0]?.expiresAt, + ).toEqual(stored.expiresAt); + }); + it.each([false, true])( + "email OTP reset preserves verification=%s through the installed endpoint", + async (verified) => { + const created = await auth.api.createUser({ + body: { + name: "OTP reset", + email: `reset-${verified}@example.test`, + password: "original-password", + }, + }); + if (verified) + await fixture.db + .update(user) + .set({ emailVerified: true }) + .where(eq(user.id, created.user.id)); + const otp = await auth.api.createVerificationOTP({ + body: { email: created.user.email, type: "forget-password" }, + }); + await expect( + auth.api.resetPasswordEmailOTP({ + body: { email: created.user.email, otp, password: "replacement-password" }, + }), + ).resolves.toMatchObject({ success: true }); + expect( + (await fixture.db.select().from(user).where(eq(user.id, created.user.id)))[0] + ?.emailVerified, + ).toBe(verified); + await expect( + auth.api.resetPasswordEmailOTP({ + body: { email: created.user.email, otp, password: "different-password" }, + }), + ).rejects.toThrow(); + }, + ); + it("rolls back the entire activation when transactional audit persistence fails", async () => { + const { loadAuthPolicy, applyAuthPolicy } = await import("./auth-policy"); + const [admin] = await fixture.db + .select() + .from(user) + .where(eq(user.email, "policy-admin@example.test")); + if (!admin) throw Error("Missing admin fixture"); + const before = await loadAuthPolicy(), + rows = await fixture.db.select().from(authPolicyState), + drafts = await fixture.db.select().from(systemSettings); + await fixture.db.execute( + sql`ALTER TABLE admin_audit_log ADD CONSTRAINT synthetic_policy_audit_failure CHECK (action <> 'auth.policy.apply') NOT VALID`, + ); + try { + await expect( + applyAuthPolicy( + { + settings: { + emailVerificationRequired: before.emailVerificationRequired, + sessionDurationHours: before.sessionDurationHours, + twoFaEnabled: before.twoFaEnabled, + passwordMinLength: 24, + }, + expectedRevision: before.revision, + confirmed: true, + idempotencyKey: "rollback-policy-synthetic-0001", + }, + admin.id, + ), + ).rejects.toThrow(); + expect(await loadAuthPolicy()).toEqual(before); + expect(await fixture.db.select().from(authPolicyState)).toEqual(rows); + expect(await fixture.db.select().from(systemSettings)).toEqual(drafts); + expect( + await fixture.db + .select() + .from(authPolicyApplication) + .where(eq(authPolicyApplication.key, "rollback-policy-synthetic-0001")), + ).toHaveLength(0); + } finally { + await fixture.db.execute( + sql`ALTER TABLE admin_audit_log DROP CONSTRAINT synthetic_policy_audit_failure`, + ); + } + }); + it("does not consume a reset OTP when the password transaction rolls back", async () => { + const created = await auth.api.createUser({ + body: { + name: "OTP transaction", + email: "otp-transaction@example.test", + password: "original-password", + }, + }); + const otp = await auth.api.createVerificationOTP({ + body: { email: created.user.email, type: "forget-password" }, + }); + await fixture.db.execute( + sql`ALTER TABLE account ADD CONSTRAINT synthetic_password_failure CHECK (provider_id <> 'credential') NOT VALID`, + ); + try { + await expect( + auth.api.resetPasswordEmailOTP({ + body: { email: created.user.email, otp, password: "replacement-password" }, + }), + ).rejects.toThrow(); + } finally { + await fixture.db.execute(sql`ALTER TABLE account DROP CONSTRAINT synthetic_password_failure`); + } + await expect( + auth.api.resetPasswordEmailOTP({ + body: { email: created.user.email, otp, password: "replacement-password" }, + }), + ).resolves.toMatchObject({ success: true }); + }); + + it("retains reset attempt lockout across rejected HTTP and server calls", async () => { + const created = await auth.api.createUser({ + body: { name: "OTP budget", email: "otp-budget@example.test", password: "original-password" }, + }); + const otp = await auth.api.createVerificationOTP({ + body: { email: created.user.email, type: "forget-password" }, + }), + wrong = otp === "000000" ? "999999" : "000000"; + const request = () => + new Request("http://localhost:3001/api/auth/email-otp/reset-password", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + email: created.user.email, + otp: wrong, + password: "replacement-password", + }), + }); + expect((await auth.handler(request())).status).toBe(400); + for (let i = 0; i < 2; i++) + await expect( + auth.api.resetPasswordEmailOTP({ + body: { email: created.user.email, otp: wrong, password: "replacement-password" }, + }), + ).rejects.toMatchObject({ body: { code: "INVALID_OTP" } }); + await expect( + auth.api.resetPasswordEmailOTP({ + body: { email: created.user.email, otp, password: "replacement-password" }, + }), + ).rejects.toMatchObject({ body: { code: "TOO_MANY_ATTEMPTS" } }); + }); + + it("enforces active password bounds on every installed password-writing endpoint", async () => { + const person = await auth.api.createUser({ + body: { + name: "Password bounds", + email: "bounds-policy@example.test", + password: "original-password", + role: "admin", + }, + }); + const signin = await auth.api.signInEmail({ + body: { email: person.user.email, password: "original-password" }, + returnHeaders: true, + }); + const headers = new Headers({ + cookie: signin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "), + }); + const [before] = await fixture.db + .select() + .from(account) + .where(eq(account.userId, person.user.id)); + await auth.api.requestPasswordReset({ body: { email: person.user.email } }); + const token = mail + .find((v) => v.to === person.user.email) + ?.text.match(/\/reset-password\/([^?\s]+)/)?.[1]; + if (!token) throw Error("Missing synthetic reset mail token"); + for (const password of ["eleven-1234", "x".repeat(129)]) { + await expect( + auth.api.createUser({ + body: { name: "Invalid", email: "invalid-bound@example.test", password }, + }), + ).rejects.toThrow(); + await expect( + auth.api.signUpEmail({ + body: { name: "Invalid", email: "invalid-signup@example.test", password }, + }), + ).rejects.toThrow(); + await expect( + auth.api.changePassword({ + headers, + body: { currentPassword: "original-password", newPassword: password }, + }), + ).rejects.toThrow(); + await expect( + auth.api.setUserPassword({ + headers, + body: { userId: person.user.id, newPassword: password }, + }), + ).rejects.toThrow(); + await expect( + auth.api.resetPassword({ body: { token, newPassword: password } }), + ).rejects.toThrow(); + const otp = await auth.api.createVerificationOTP({ + body: { email: person.user.email, type: "forget-password" }, + }); + await expect( + auth.api.resetPasswordEmailOTP({ body: { email: person.user.email, otp, password } }), + ).rejects.toThrow(); + } + expect( + (await fixture.db.select().from(account).where(eq(account.userId, person.user.id)))[0] + ?.password, + ).toBe(before?.password); + await expect( + auth.api.resetPassword({ body: { token, newPassword: "valid-replacement-password" } }), + ).resolves.toMatchObject({ status: true }); + const passwordless = await auth.api.createUser({ + body: { name: "Passwordless", email: "passwordless-policy@example.test" }, + }); + const otp = await auth.api.createVerificationOTP({ + body: { email: passwordless.user.email, type: "sign-in" }, + }); + const otpSession = await auth.api.signInEmailOTP({ + body: { email: passwordless.user.email, otp }, + returnHeaders: true, + }); + const passwordlessHeaders = new Headers({ + cookie: otpSession.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "), + }); + await expect( + auth.api.setPassword({ headers: passwordlessHeaders, body: { newPassword: "short" } }), + ).rejects.toThrow(); + await expect( + auth.api.setPassword({ + headers: passwordlessHeaders, + body: { newPassword: "x".repeat(129) }, + }), + ).rejects.toThrow(); + await expect( + auth.api.setPassword({ + headers: passwordlessHeaders, + body: { newPassword: "valid-new-password" }, + }), + ).resolves.toMatchObject({ status: true }); + }); + it("does not overwrite concurrent genuine verification during an OTP reset", async () => { + const created = await auth.api.createUser({ + body: { + name: "Verification race", + email: "verify-race@example.test", + password: "original-password", + }, + }); + const reset = await auth.api.createVerificationOTP({ + body: { email: created.user.email, type: "forget-password" }, + }), + verification = await auth.api.createVerificationOTP({ + body: { email: created.user.email, type: "email-verification" }, + }); + await Promise.all([ + auth.api.resetPasswordEmailOTP({ + body: { email: created.user.email, otp: reset, password: "replacement-password" }, + }), + auth.api.verifyEmailOTP({ body: { email: created.user.email, otp: verification } }), + ]); + expect( + (await fixture.db.select().from(user).where(eq(user.id, created.user.id)))[0]?.emailVerified, + ).toBe(true); + }); + + it("discloses active host ownership and fails closed on malformed active environment policy", async () => { + const { loadAuthPolicy, authPolicyView } = await import("./auth-policy"), + original = process.env.AUTH_EMAIL_VERIFICATION; + try { + vi.stubEnv("AUTH_EMAIL_VERIFICATION", "true"); + expect((await authPolicyView()).ownership.emailVerificationRequired).toBe("env"); + expect((await loadAuthPolicy()).emailVerificationRequired).toBe(true); + vi.stubEnv("AUTH_EMAIL_VERIFICATION", "not-a-boolean"); + await expect(loadAuthPolicy()).rejects.toMatchObject({ code: "AUTH_POLICY_INVALID" }); + const response = await auth.handler( + new Request("http://localhost:3001/api/auth/get-session"), + ); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ code: "AUTH_POLICY_INVALID" }); + } finally { + vi.stubEnv("AUTH_EMAIL_VERIFICATION", original); + } + }); + it("activates and replays the exact policy through the generated SDK and real admin HTTP guard", async () => { + const { loadAuthPolicy } = await import("./auth-policy"), + before = await loadAuthPolicy(); + const actor = await auth.api.createUser({ + body: { + name: "HTTP activation", + email: "http-activation@example.test", + password: "original-password", + role: "admin", + }, + }); + await fixture.db.update(user).set({ emailVerified: true }).where(eq(user.id, actor.user.id)); + const signin = await auth.api.signInEmail({ + body: { email: actor.user.email, password: "original-password" }, + returnHeaders: true, + }); + const cookie = signin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "); + const app = (await import("fastify")).default(); + await app.register((await import("../routes/admin-accounts")).adminAccountsRoute, { + prefix: "/api", + }); + const address = await app.listen({ port: 0, host: "127.0.0.1" }); + try { + const { applyAdminAuthPolicy } = await import("@openmapx/core/api-admin-sdk"), + { createApiClient } = await import("@openmapx/core"); + const options = { client: createApiClient({ baseUrl: address }), headers: { cookie } }, + input = { + settings: { + emailVerificationRequired: before.emailVerificationRequired, + twoFaEnabled: before.twoFaEnabled, + passwordMinLength: before.passwordMinLength, + sessionDurationHours: 9, + }, + expectedRevision: before.revision, + confirmed: true as const, + idempotencyKey: "policy-http-activation-synthetic-0001", + }; + const response = await applyAdminAuthPolicy(input, options); + expect(response).toMatchObject({ + status: 200, + data: { + activationVersion: 1, + appliesTo: "new_auth_operations", + active: { sessionDurationHours: 9 }, + }, + }); + expect(await applyAdminAuthPolicy(input, options)).toMatchObject({ status: 429 }); + vi.useFakeTimers({ toFake: ["Date"] }); + try { + vi.setSystemTime(Date.now() + 11000); + expect(await applyAdminAuthPolicy(input, options)).toMatchObject({ + status: 200, + data: response.data, + }); + } finally { + vi.useRealTimers(); + } + expect((await loadAuthPolicy()).sessionDurationHours).toBe(9); + } finally { + await app.close(); + } + }); + + it("does not refresh a previously issued session after its original daily refresh threshold", async () => { + const { loadAuthPolicy, applyAuthPolicy } = await import("./auth-policy"); + const actor = await auth.api.createUser({ + body: { + name: "Absolute expiry", + email: "absolute-expiry@example.test", + password: "original-password", + role: "admin", + }, + }); + await fixture.db.update(user).set({ emailVerified: true }).where(eq(user.id, actor.user.id)); + const before = await loadAuthPolicy(), + settings = { + emailVerificationRequired: before.emailVerificationRequired, + twoFaEnabled: before.twoFaEnabled, + passwordMinLength: before.passwordMinLength, + sessionDurationHours: 168, + }; + const long = await applyAuthPolicy( + { + settings, + expectedRevision: before.revision, + confirmed: true, + idempotencyKey: "absolute-long-policy-0001", + }, + actor.user.id, + ); + const signin = await auth.api.signInEmail({ + body: { email: actor.user.email, password: "original-password" }, + returnHeaders: true, + }); + const stored = ( + await fixture.db.select().from(session).where(eq(session.userId, actor.user.id)) + ).find((v) => v.token === signin.response.token); + if (!stored) throw Error("No original session"); + await applyAuthPolicy( + { + settings: { ...settings, sessionDurationHours: 1 }, + expectedRevision: long.revision, + confirmed: true, + idempotencyKey: "absolute-short-policy-0001", + }, + actor.user.id, + ); + vi.useFakeTimers({ toFake: ["Date"] }); + try { + vi.setSystemTime(Date.now() + 30 * 3600000); + const value = await auth.api.getSession({ + headers: new Headers({ + cookie: signin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "), + }), + }); + expect(value).not.toBeNull(); + expect( + (await fixture.db.select().from(session).where(eq(session.id, stored.id)))[0]?.expiresAt, + ).toEqual(stored.expiresAt); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/apps/api/src/account/auth-policy.test.ts b/apps/api/src/account/auth-policy.test.ts new file mode 100644 index 000000000..27490148e --- /dev/null +++ b/apps/api/src/account/auth-policy.test.ts @@ -0,0 +1,13 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { AuthDatabase } from "./auth-database"; +import { loadAuthPolicy } from "./auth-policy"; + +afterEach(() => vi.unstubAllEnvs()); +it("distinguishes a failed policy read from an unactivated installation", async () => { + const database = { + select: () => { + throw Error("synthetic offline database"); + }, + } as unknown as AuthDatabase; + await expect(loadAuthPolicy(database)).rejects.toMatchObject({ code: "AUTH_POLICY_UNAVAILABLE" }); +}); diff --git a/apps/api/src/account/auth-policy.ts b/apps/api/src/account/auth-policy.ts new file mode 100644 index 000000000..b6227fc9b --- /dev/null +++ b/apps/api/src/account/auth-policy.ts @@ -0,0 +1,183 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + type AuthPolicySettings, + type AuthPolicySnapshot, + authPolicyApplySchema, + authPolicySettingsSchema, +} from "@openmapx/core/api-contracts"; +import { and, eq, sql } from "drizzle-orm"; +import { + adminAuditLog, + authPolicyApplication, + authPolicyState, + systemSettings, + user, +} from "../db/schema"; +import { type AuthDatabase, currentAuthDatabase } from "./auth-database"; +export const defaultAuthPolicySettings: Readonly = Object.freeze({ + emailVerificationRequired: true, + sessionDurationHours: 168, + twoFaEnabled: true, + passwordMinLength: 8, +}); +export const authPolicyKeys = Object.freeze(Object.keys(defaultAuthPolicySettings)); +export class AuthPolicyError extends Error { + constructor( + readonly code: + | "AUTH_POLICY_INVALID" + | "AUTH_POLICY_UNAVAILABLE" + | "AUTH_POLICY_CONFLICT" + | "AUTH_POLICY_FORBIDDEN", + ) { + super(code); + } +} +const digest = (v: unknown) => createHash("sha256").update(JSON.stringify(v)).digest("hex"); +function snapshot(settings: AuthPolicySettings, activationVersion: 0 | 1): AuthPolicySnapshot { + const effective = { ...settings }; + if (activationVersion === 1 && process.env.AUTH_EMAIL_VERIFICATION !== undefined) { + const raw = process.env.AUTH_EMAIL_VERIFICATION; + if (!["true", "false", "1", "0"].includes(raw)) + throw new AuthPolicyError("AUTH_POLICY_INVALID"); + effective.emailVerificationRequired = raw === "true" || raw === "1"; + } + return Object.freeze({ + ...effective, + activationVersion, + revision: digest([ + activationVersion, + effective.emailVerificationRequired, + effective.sessionDurationHours, + effective.twoFaEnabled, + effective.passwordMinLength, + ]), + }); +} +export const unactivatedAuthPolicy = snapshot(defaultAuthPolicySettings, 0); +export async function loadAuthPolicy( + database: AuthDatabase = currentAuthDatabase(), +): Promise { + try { + const rows = await database.select().from(authPolicyState).where(eq(authPolicyState.id, 1)); + if (!rows.length) return unactivatedAuthPolicy; + const row = rows[0]; + if (!row || row.activationVersion !== 1) throw new AuthPolicyError("AUTH_POLICY_INVALID"); + const parsed = authPolicySettingsSchema.safeParse(row.settings); + if (!parsed.success) throw new AuthPolicyError("AUTH_POLICY_INVALID"); + const stored = snapshotWithoutEnv(parsed.data); + if (row.revision !== stored.revision) throw new AuthPolicyError("AUTH_POLICY_INVALID"); + return snapshot(parsed.data, 1); + } catch (error) { + if (error instanceof AuthPolicyError) throw error; + throw new AuthPolicyError("AUTH_POLICY_UNAVAILABLE"); + } +} +function snapshotWithoutEnv(s: AuthPolicySettings): AuthPolicySnapshot { + return Object.freeze({ + ...s, + activationVersion: 1, + revision: digest([ + 1, + s.emailVerificationRequired, + s.sessionDurationHours, + s.twoFaEnabled, + s.passwordMinLength, + ]), + }); +} +export async function authPolicyView(database: AuthDatabase = currentAuthDatabase()) { + const active = await loadAuthPolicy(database), + rows = await database.select().from(systemSettings); + return { + active, + pending: Object.fromEntries( + rows.filter((r) => authPolicyKeys.includes(r.key)).map((r) => [r.key, r.value]), + ), + ownership: { + emailVerificationRequired: + active.activationVersion === 1 && process.env.AUTH_EMAIL_VERIFICATION !== undefined + ? ("env" as const) + : active.activationVersion === 1 + ? ("database" as const) + : ("default" as const), + sessionDurationHours: + active.activationVersion === 1 ? ("database" as const) : ("default" as const), + twoFaEnabled: active.activationVersion === 1 ? ("database" as const) : ("default" as const), + passwordMinLength: + active.activationVersion === 1 ? ("database" as const) : ("default" as const), + }, + }; +} +export async function applyAuthPolicy( + input: unknown, + actorId: string, + database: AuthDatabase = currentAuthDatabase(), +): Promise { + const parsed = authPolicyApplySchema.safeParse(input); + if (!parsed.success) throw new AuthPolicyError("AUTH_POLICY_INVALID"); + const request = parsed.data, + intentDigest = digest(request); + return database.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(163578142, 1)`); + const [actor] = await tx + .select() + .from(user) + .where(and(eq(user.id, actorId), eq(user.banned, false))) + .for("update"); + if (actor?.role !== "admin") throw new AuthPolicyError("AUTH_POLICY_FORBIDDEN"); + const [prior] = await tx + .select() + .from(authPolicyApplication) + .where(eq(authPolicyApplication.key, request.idempotencyKey)); + if (prior) { + if (prior.actorId !== actorId || prior.intentDigest !== intentDigest) + throw new AuthPolicyError("AUTH_POLICY_CONFLICT"); + return Object.freeze(prior.snapshot); + } + const active = await loadAuthPolicy(tx); + if (active.revision !== request.expectedRevision) + throw new AuthPolicyError("AUTH_POLICY_CONFLICT"); + const result = snapshot(request.settings, 1), + stored = snapshotWithoutEnv(request.settings); + if (result.emailVerificationRequired !== request.settings.emailVerificationRequired) + throw new AuthPolicyError("AUTH_POLICY_CONFLICT"); + await tx + .insert(authPolicyState) + .values({ + id: 1, + activationVersion: 1, + settings: request.settings, + revision: stored.revision, + activatedBy: actorId, + }) + .onConflictDoUpdate({ + target: authPolicyState.id, + set: { + settings: request.settings, + revision: stored.revision, + activatedBy: actorId, + activatedAt: new Date(), + }, + }); + for (const [key, value] of Object.entries(request.settings)) + await tx + .insert(systemSettings) + .values({ key, value, updatedBy: actorId }) + .onConflictDoUpdate({ + target: systemSettings.key, + set: { value, updatedBy: actorId, updatedAt: new Date() }, + }); + await tx.insert(adminAuditLog).values({ + id: randomUUID(), + actorId, + targetType: "auth-policy", + targetId: result.revision, + action: "auth.policy.apply", + details: { revision: result.revision, activationVersion: 1 }, + }); + await tx + .insert(authPolicyApplication) + .values({ key: request.idempotencyKey, actorId, intentDigest, snapshot: result }); + return result; + }); +} diff --git a/apps/api/src/account/auth-runtime.ts b/apps/api/src/account/auth-runtime.ts new file mode 100644 index 000000000..ae8d6061c --- /dev/null +++ b/apps/api/src/account/auth-runtime.ts @@ -0,0 +1,123 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type { AuthPolicySnapshot } from "@openmapx/core/api-contracts"; +import { isAPIError } from "better-auth/api"; +import { + type AuthDatabase, + currentAuthDatabase, + isTransactionalAuthDatabase, + withAuthDatabase, +} from "./auth-database"; +import { AuthPolicyError, loadAuthPolicy, unactivatedAuthPolicy } from "./auth-policy"; +/** The facade never returns a raw mutable runtime. Nested calls keep their original snapshot/transaction. */ +export function createAuthRuntime< + T extends { + api: object; + handler: (request: Request) => Promise; + $context: Promise; + }, +>(create: (policy: AuthPolicySnapshot, database: AuthDatabase) => T) { + const local = new AsyncLocalStorage<{ + instance: T; + policy: AuthPolicySnapshot; + database: AuthDatabase; + }>(); + let cached: { revision: string; instance: T } | undefined; + const template = create(unactivatedAuthPolicy, currentAuthDatabase()); + async function withAuthOperation(run: (instance: T) => Promise): Promise { + const pinned = local.getStore(); + const database = currentAuthDatabase(); + if (pinned?.database === database) return run(pinned.instance); + const policy = pinned?.policy ?? (await loadAuthPolicy(database)), + transactional = isTransactionalAuthDatabase(); + let instance: T; + if (!transactional && cached?.revision === policy.revision) instance = cached.instance; + else { + instance = create(policy, database); + if (!transactional) cached = { revision: policy.revision, instance }; + } + return local.run({ instance, policy, database }, () => run(instance)); + } + class ResetResponseRollback extends Error { + constructor(readonly response: Response) { + super("Password reset did not commit"); + } + } + // In installed Better Auth 1.7.7 these exact errors are emitted only by + // atomicVerifyOTP, before hashing/writing the password. Commit their attempt + // accounting; roll back every later/unrecognized failure with OTP consumption. + const otpRefusal = (body: unknown, status: number) => { + const code = (body as { code?: unknown } | null)?.code; + return ( + (status === 400 && (code === "INVALID_OTP" || code === "OTP_EXPIRED")) || + (status === 403 && code === "TOO_MANY_ATTEMPTS") + ); + }; + async function entry(reset: boolean, run: () => Promise): Promise { + if (!reset || isTransactionalAuthDatabase()) return run(); + try { + const outcome = await currentAuthDatabase().transaction((tx) => + withAuthDatabase(tx, async () => { + try { + const value = await run(); + if (value instanceof Response && value.status >= 400) { + const body: unknown = await value + .clone() + .json() + .catch(() => null); + if (!otpRefusal(body, value.status)) throw new ResetResponseRollback(value); + } + return { kind: "value" as const, value }; + } catch (error) { + if (isAPIError(error) && otpRefusal(error.body, error.statusCode)) + return { kind: "error" as const, error }; + throw error; + } + }), + ); + if (outcome.kind === "error") throw outcome.error; + return outcome.value; + } catch (error) { + if (error instanceof ResetResponseRollback) return error.response as R; + throw error; + } + } + const api = new Proxy(template.api, { + get(target, key) { + const value = Reflect.get(target, key); + if (typeof value !== "function") return value; + return Object.assign( + (...args: unknown[]) => + entry(key === "resetPasswordEmailOTP", () => + withAuthOperation(async (instance) => + Reflect.apply(Reflect.get(instance.api, key), instance.api, args), + ), + ), + value, + ); + }, + }); + const auth = new Proxy(template, { + get(target, key) { + if (key === "api") return api; + if (key === "handler") + return async (request: Request) => { + try { + return await entry( + new URL(request.url).pathname === "/api/auth/email-otp/reset-password", + () => withAuthOperation((instance) => instance.handler(request)), + ); + } catch (error) { + if (error instanceof AuthPolicyError) + return Response.json( + { code: error.code, message: "Authentication policy is unavailable" }, + { status: 503 }, + ); + throw error; + } + }; + if (key === "$context") return withAuthOperation((instance) => instance.$context); + return Reflect.get(target, key); + }, + }); + return { auth, withAuthOperation }; +} diff --git a/apps/api/src/account/password-verification-policy.test.ts b/apps/api/src/account/password-verification-policy.test.ts new file mode 100644 index 000000000..9dc0f44ad --- /dev/null +++ b/apps/api/src/account/password-verification-policy.test.ts @@ -0,0 +1,13 @@ +import { expect, it } from "vitest"; +import { unactivatedAuthPolicy } from "./auth-policy"; +import { applyPasswordVerificationPolicy } from "./password-verification-policy"; + +it("refuses changed reset verification patches instead of merging emailVerified back", async () => { + const hook = applyPasswordVerificationPolicy(unactivatedAuthPolicy).userHooks.update?.before; + if (!hook) throw Error("Missing database guard"); + const context = { path: "/email-otp/reset-password" } as Parameters[1]; + await expect(hook({ emailVerified: true }, context)).resolves.toBe(false); + await expect(hook({ emailVerified: true, name: "Unexpected" }, context)).rejects.toThrow( + "Unexpected password reset verification change", + ); +}); diff --git a/apps/api/src/account/password-verification-policy.ts b/apps/api/src/account/password-verification-policy.ts new file mode 100644 index 000000000..1c03875b7 --- /dev/null +++ b/apps/api/src/account/password-verification-policy.ts @@ -0,0 +1,40 @@ +import type { AuthPolicySnapshot } from "@openmapx/core/api-contracts"; +import type { BetterAuthOptions } from "better-auth"; +import { APIError, createAuthMiddleware } from "better-auth/api"; +export function applyPasswordVerificationPolicy(policy: AuthPolicySnapshot) { + const userHooks: NonNullable["user"]> = { + create: { + before: async (user, context) => + context?.path === "/admin/create-user" + ? { data: { ...user, emailVerified: false } } + : undefined, + }, + update: { + before: async (patch, context) => { + if (context?.path !== "/email-otp/reset-password") return; + if (Object.keys(patch).length === 1 && patch.emailVerified === true) return false; + throw new APIError("BAD_REQUEST", { + code: "AUTH_POLICY_INVALID", + message: "Unexpected password reset verification change", + }); + }, + }, + }; + const before = createAuthMiddleware(async (ctx) => { + if (ctx.path === "/two-factor/enable" && !policy.twoFaEnabled) + throw new APIError("FORBIDDEN", { + code: "TWO_FACTOR_ENROLLMENT_DISABLED", + message: "Two-factor enrollment is disabled", + }); + if ( + ctx.path === "/admin/create-user" && + typeof ctx.body?.password === "string" && + (ctx.body.password.length < policy.passwordMinLength || ctx.body.password.length > 128) + ) + throw new APIError("BAD_REQUEST", { + code: "INVALID_PASSWORD", + message: "Password does not meet the instance policy", + }); + }); + return { userHooks, before }; +} diff --git a/apps/api/src/auth-policy-validation.test.ts b/apps/api/src/auth-policy-validation.test.ts new file mode 100644 index 000000000..b8f7ae199 --- /dev/null +++ b/apps/api/src/auth-policy-validation.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { matchesDeclaredType, SETTING_DEFS } from "./services/settings"; + +describe("operator authentication policy drafts", () => { + it.each([ + ["sessionDurationHours", 0], + ["sessionDurationHours", 8761], + ["sessionDurationHours", 1.5], + ["passwordMinLength", 7], + ["passwordMinLength", 129], + ])("rejects unsafe %s=%s before persistence", (key, value) => { + const definition = SETTING_DEFS.find((d) => d.key === key); + if (!definition) throw Error("Missing setting definition"); + expect(matchesDeclaredType(definition, value)).toBe(false); + }); + it.each(["emailVerificationRequired", "twoFaEnabled"])( + "displays the secure %s default until explicit activation", + (key) => { + expect(SETTING_DEFS.find((d) => d.key === key)?.default).toBe(true); + }, + ); +}); diff --git a/apps/api/src/auth.ts b/apps/api/src/auth.ts index 4614789dc..41b4a5aae 100644 --- a/apps/api/src/auth.ts +++ b/apps/api/src/auth.ts @@ -16,6 +16,9 @@ import { } from "better-auth/plugins"; import { emailHarmony } from "better-auth-harmony"; import { eq } from "drizzle-orm"; +import type { AuthDatabase } from "./account/auth-database"; +import { createAuthRuntime } from "./account/auth-runtime"; +import { applyPasswordVerificationPolicy } from "./account/password-verification-policy"; import { exchangeMapillaryAuthorizationCode, fetchMapillaryUserInfo, @@ -92,316 +95,340 @@ const providerAvatarSync = createProviderAvatarSync({ }, }); -const authOptions = { - database: drizzleAdapter(db, { - provider: "pg", - }), - baseURL: envString("BETTER_AUTH_URL", "http://localhost:3001"), - secret, - trustedOrigins: [ - ...configuredTrustedWebOrigins(), - "openmapx://", - ...(process.env.NODE_ENV === "development" - ? ["exp://", "exp://**", "exp://192.168.*.*:*/**"] - : []), - ], - appName: "OpenMapX", - user: { - deleteUser: { - enabled: true, - }, - }, - emailAndPassword: { - enabled: true, - autoSignIn: false, - requireEmailVerification: true, - async sendResetPassword({ user, url }) { - const mail = resetPasswordEmail(url); - await sendMail({ - to: user.email, - ...mail, - disclosure: { userId: user.id, operationCode: "auth.reset-password.email" }, - }); - }, - }, - emailVerification: { - sendOnSignUp: true, - autoSignInAfterVerification: true, - async sendVerificationEmail({ user, url }) { - const mail = verifyEmailEmail(url); - await sendMail({ - to: user.email, - ...mail, - disclosure: { userId: user.id, operationCode: "auth.verify-email.email" }, - }); +function createPolicyAuth( + policy: import("@openmapx/core/api-contracts").AuthPolicySnapshot, + database: AuthDatabase, +) { + const passwordPolicy = applyPasswordVerificationPolicy(policy); + const authOptions = { + database: drizzleAdapter(database, { + provider: "pg", + }), + baseURL: envString("BETTER_AUTH_URL", "http://localhost:3001"), + secret, + trustedOrigins: [ + ...configuredTrustedWebOrigins(), + "openmapx://", + ...(process.env.NODE_ENV === "development" + ? ["exp://", "exp://**", "exp://192.168.*.*:*/**"] + : []), + ], + appName: "OpenMapX", + user: { + deleteUser: { + enabled: true, + }, }, - }, - account: { - accountLinking: { + emailAndPassword: { enabled: true, - trustedProviders: ["openstreetmap", "mapillary"], - allowDifferentEmails: true, - }, - // Provider access/refresh tokens are encrypted at rest with the deployment - // auth secret. Contribution editing stores elevated OSM write scopes on - // this same account row, so a database disclosure must not hand out - // usable write tokens. Nothing may read the column directly afterwards — - // see `providerAvatarSync` and `OsmAccountService`. - encryptOAuthTokens: true, - }, - databaseHooks: { - session: { - create: { - // This hook runs after Better Auth has created the usable session. It - // intentionally records nothing for unrecognised endpoints. - after: async (createdSession, context) => { - await recordSessionFromAuthContext(createdSession, context, db); - }, + autoSignIn: false, + requireEmailVerification: policy.emailVerificationRequired, + minPasswordLength: policy.passwordMinLength, + maxPasswordLength: 128, + async sendResetPassword({ user, url }) { + const mail = resetPasswordEmail(url); + await sendMail({ + to: user.email, + ...mail, + disclosure: { userId: user.id, operationCode: "auth.reset-password.email" }, + }); }, }, - user: { - delete: userErasureHooks, + emailVerification: { + sendOnSignUp: true, + autoSignInAfterVerification: true, + async sendVerificationEmail({ user, url }) { + const mail = verifyEmailEmail(url); + await sendMail({ + to: user.email, + ...mail, + disclosure: { userId: user.id, operationCode: "auth.verify-email.email" }, + }); + }, }, account: { - create: { - after: async (account) => { - await providerAvatarSync.onAccountCreated(account.id, account.providerId, account.userId); - // An external provider exchange has completed by the time Better - // Auth persists its account row. Record only the provider and the - // stable account operation; OAuth tokens, profile payloads and - // redirect/callback values never enter the disclosure ledger. - const recipient = - account.providerId === "openstreetmap" || account.providerId === "mapillary" - ? account.providerId - : null; - if (recipient) { - await recordDataDisclosureBestEffort({ - subjectUserId: account.userId, - occurredAt: new Date(), - recipientId: recipient, - operationCode: "auth.account.link", - categoryCode: "auth-accounts", - purposeCode: "authentication", - legalBasisCode: "contract", - idempotencyKey: `auth-account-${account.id}`, - }); - } + accountLinking: { + enabled: true, + trustedProviders: ["openstreetmap", "mapillary"], + allowDifferentEmails: true, + }, + // Provider access/refresh tokens are encrypted at rest with the deployment + // auth secret. Contribution editing stores elevated OSM write scopes on + // this same account row, so a database disclosure must not hand out + // usable write tokens. Nothing may read the column directly afterwards — + // see `providerAvatarSync` and `OsmAccountService`. + encryptOAuthTokens: true, + }, + session: { + expiresIn: policy.sessionDurationHours * 3600, + disableSessionRefresh: true, + cookieCache: { enabled: false }, + }, + databaseHooks: { + session: { + create: { + // This hook runs after Better Auth has created the usable session. It + // intentionally records nothing for unrecognised endpoints. + after: async (createdSession, context) => { + await recordSessionFromAuthContext(createdSession, context, db); + }, }, }, - update: { - after: async (account) => { - // Better Auth updates the account on each OAuth sign-in and token - // refresh; re-read the picture so a changed provider avatar follows. - await providerAvatarSync.onAccountUpdated(account.id, account.providerId, account.userId); + user: { + ...passwordPolicy.userHooks, + delete: userErasureHooks, + }, + account: { + create: { + after: async (account) => { + await providerAvatarSync.onAccountCreated( + account.id, + account.providerId, + account.userId, + ); + // An external provider exchange has completed by the time Better + // Auth persists its account row. Record only the provider and the + // stable account operation; OAuth tokens, profile payloads and + // redirect/callback values never enter the disclosure ledger. + const recipient = + account.providerId === "openstreetmap" || account.providerId === "mapillary" + ? account.providerId + : null; + if (recipient) { + await recordDataDisclosureBestEffort({ + subjectUserId: account.userId, + occurredAt: new Date(), + recipientId: recipient, + operationCode: "auth.account.link", + categoryCode: "auth-accounts", + purposeCode: "authentication", + legalBasisCode: "contract", + idempotencyKey: `auth-account-${account.id}`, + }); + } + }, + }, + update: { + after: async (account) => { + // Better Auth updates the account on each OAuth sign-in and token + // refresh; re-read the picture so a changed provider avatar follows. + await providerAvatarSync.onAccountUpdated( + account.id, + account.providerId, + account.userId, + ); + }, }, }, }, - }, - hooks: { - after: auditAdminActionsHook, - }, - plugins: [ - i18n({ - defaultLocale: "en", - detection: ["cookie", "header"], - localeCookie: "NEXT_LOCALE", - translations: { - de: { - // Base error codes - USER_NOT_FOUND: "Benutzer nicht gefunden", - FAILED_TO_CREATE_USER: "Benutzer konnte nicht erstellt werden", - FAILED_TO_CREATE_SESSION: "Sitzung konnte nicht erstellt werden", - FAILED_TO_UPDATE_USER: "Benutzer konnte nicht aktualisiert werden", - FAILED_TO_GET_SESSION: "Sitzung konnte nicht abgerufen werden", - INVALID_PASSWORD: "Ungültiges Passwort", - INVALID_EMAIL: "Ungültige E-Mail-Adresse", - INVALID_EMAIL_OR_PASSWORD: "Ungültige E-Mail oder Passwort", - INVALID_USER: "Ungültiger Benutzer", - SOCIAL_ACCOUNT_ALREADY_LINKED: "Social-Konto ist bereits verknüpft", - PROVIDER_NOT_FOUND: "Anbieter nicht gefunden", - INVALID_TOKEN: "Ungültiger Token", - TOKEN_EXPIRED: "Token abgelaufen", - ID_TOKEN_NOT_SUPPORTED: "ID-Token wird nicht unterstützt", - FAILED_TO_GET_USER_INFO: "Benutzerinformationen konnten nicht abgerufen werden", - USER_EMAIL_NOT_FOUND: "E-Mail-Adresse des Benutzers nicht gefunden", - EMAIL_NOT_VERIFIED: "E-Mail-Adresse nicht bestätigt", - PASSWORD_TOO_SHORT: "Passwort zu kurz", - PASSWORD_TOO_LONG: "Passwort zu lang", - USER_ALREADY_EXISTS: "Benutzer existiert bereits.", - USER_ALREADY_EXISTS_USE_ANOTHER_EMAIL: - "Benutzer existiert bereits. Verwenden Sie eine andere E-Mail-Adresse.", - EMAIL_CAN_NOT_BE_UPDATED: "E-Mail-Adresse kann nicht geändert werden", - CREDENTIAL_ACCOUNT_NOT_FOUND: "Anmeldekonto nicht gefunden", - SESSION_EXPIRED: "Sitzung abgelaufen. Melden Sie sich erneut an.", - FAILED_TO_UNLINK_LAST_ACCOUNT: "Letztes Konto kann nicht getrennt werden", - ACCOUNT_NOT_FOUND: "Konto nicht gefunden", - USER_ALREADY_HAS_PASSWORD: - "Benutzer hat bereits ein Passwort. Geben Sie dieses an, um das Konto zu löschen.", - EMAIL_ALREADY_VERIFIED: "E-Mail-Adresse ist bereits bestätigt", - EMAIL_MISMATCH: "E-Mail-Adressen stimmen nicht überein", - SESSION_NOT_FRESH: "Sitzung ist nicht aktuell", - LINKED_ACCOUNT_ALREADY_EXISTS: "Verknüpftes Konto existiert bereits", - PASSWORD_ALREADY_SET: "Benutzer hat bereits ein Passwort", - VALIDATION_ERROR: "Validierungsfehler", - MISSING_FIELD: "Feld ist erforderlich", + hooks: { + before: passwordPolicy.before, + after: auditAdminActionsHook, + }, + plugins: [ + i18n({ + defaultLocale: "en", + detection: ["cookie", "header"], + localeCookie: "NEXT_LOCALE", + translations: { + de: { + // Base error codes + USER_NOT_FOUND: "Benutzer nicht gefunden", + FAILED_TO_CREATE_USER: "Benutzer konnte nicht erstellt werden", + FAILED_TO_CREATE_SESSION: "Sitzung konnte nicht erstellt werden", + FAILED_TO_UPDATE_USER: "Benutzer konnte nicht aktualisiert werden", + FAILED_TO_GET_SESSION: "Sitzung konnte nicht abgerufen werden", + INVALID_PASSWORD: "Ungültiges Passwort", + INVALID_EMAIL: "Ungültige E-Mail-Adresse", + INVALID_EMAIL_OR_PASSWORD: "Ungültige E-Mail oder Passwort", + INVALID_USER: "Ungültiger Benutzer", + SOCIAL_ACCOUNT_ALREADY_LINKED: "Social-Konto ist bereits verknüpft", + PROVIDER_NOT_FOUND: "Anbieter nicht gefunden", + INVALID_TOKEN: "Ungültiger Token", + TOKEN_EXPIRED: "Token abgelaufen", + ID_TOKEN_NOT_SUPPORTED: "ID-Token wird nicht unterstützt", + FAILED_TO_GET_USER_INFO: "Benutzerinformationen konnten nicht abgerufen werden", + USER_EMAIL_NOT_FOUND: "E-Mail-Adresse des Benutzers nicht gefunden", + EMAIL_NOT_VERIFIED: "E-Mail-Adresse nicht bestätigt", + PASSWORD_TOO_SHORT: "Passwort zu kurz", + PASSWORD_TOO_LONG: "Passwort zu lang", + USER_ALREADY_EXISTS: "Benutzer existiert bereits.", + USER_ALREADY_EXISTS_USE_ANOTHER_EMAIL: + "Benutzer existiert bereits. Verwenden Sie eine andere E-Mail-Adresse.", + EMAIL_CAN_NOT_BE_UPDATED: "E-Mail-Adresse kann nicht geändert werden", + CREDENTIAL_ACCOUNT_NOT_FOUND: "Anmeldekonto nicht gefunden", + SESSION_EXPIRED: "Sitzung abgelaufen. Melden Sie sich erneut an.", + FAILED_TO_UNLINK_LAST_ACCOUNT: "Letztes Konto kann nicht getrennt werden", + ACCOUNT_NOT_FOUND: "Konto nicht gefunden", + USER_ALREADY_HAS_PASSWORD: + "Benutzer hat bereits ein Passwort. Geben Sie dieses an, um das Konto zu löschen.", + EMAIL_ALREADY_VERIFIED: "E-Mail-Adresse ist bereits bestätigt", + EMAIL_MISMATCH: "E-Mail-Adressen stimmen nicht überein", + SESSION_NOT_FRESH: "Sitzung ist nicht aktuell", + LINKED_ACCOUNT_ALREADY_EXISTS: "Verknüpftes Konto existiert bereits", + PASSWORD_ALREADY_SET: "Benutzer hat bereits ein Passwort", + VALIDATION_ERROR: "Validierungsfehler", + MISSING_FIELD: "Feld ist erforderlich", - // Email OTP - OTP_EXPIRED: "Code abgelaufen", - INVALID_OTP: "Ungültiger Code", - TOO_MANY_ATTEMPTS: "Zu viele Versuche", + // Email OTP + OTP_EXPIRED: "Code abgelaufen", + INVALID_OTP: "Ungültiger Code", + TOO_MANY_ATTEMPTS: "Zu viele Versuche", - // Two-factor - OTP_NOT_ENABLED: "OTP nicht aktiviert", - OTP_HAS_EXPIRED: "OTP ist abgelaufen", - TOTP_NOT_ENABLED: "TOTP nicht aktiviert", - TWO_FACTOR_NOT_ENABLED: "Zwei-Faktor-Authentifizierung ist nicht aktiviert", - BACKUP_CODES_NOT_ENABLED: "Wiederherstellungscodes sind nicht aktiviert", - INVALID_BACKUP_CODE: "Ungültiger Wiederherstellungscode", - INVALID_CODE: "Ungültiger Code", - TOO_MANY_ATTEMPTS_REQUEST_NEW_CODE: - "Zu viele Versuche. Bitte fordern Sie einen neuen Code an.", - INVALID_TWO_FACTOR_COOKIE: "Ungültiges Zwei-Faktor-Cookie", + // Two-factor + OTP_NOT_ENABLED: "OTP nicht aktiviert", + OTP_HAS_EXPIRED: "OTP ist abgelaufen", + TOTP_NOT_ENABLED: "TOTP nicht aktiviert", + TWO_FACTOR_NOT_ENABLED: "Zwei-Faktor-Authentifizierung ist nicht aktiviert", + BACKUP_CODES_NOT_ENABLED: "Wiederherstellungscodes sind nicht aktiviert", + INVALID_BACKUP_CODE: "Ungültiger Wiederherstellungscode", + INVALID_CODE: "Ungültiger Code", + TOO_MANY_ATTEMPTS_REQUEST_NEW_CODE: + "Zu viele Versuche. Bitte fordern Sie einen neuen Code an.", + INVALID_TWO_FACTOR_COOKIE: "Ungültiges Zwei-Faktor-Cookie", - // Admin - YOU_CANNOT_BAN_YOURSELF: "Sie können sich nicht selbst sperren", - BANNED_USER: "Sie wurden von dieser Anwendung gesperrt", + // Admin + YOU_CANNOT_BAN_YOURSELF: "Sie können sich nicht selbst sperren", + BANNED_USER: "Sie wurden von dieser Anwendung gesperrt", - // Generic OAuth - SESSION_REQUIRED: "Sitzung ist erforderlich", - }, - }, - }), - admin(), - // The one mechanism by which a session established in the system browser - // becomes a session in the WebView. Two minutes and single-use because the - // token is in flight across a custom-scheme callback the OS routes, and the - // only legitimate holder redeems it immediately. Stored hashed so a database - // read never yields a usable token. - oneTimeToken({ expiresIn: 2, storeToken: "hashed" }), - // Kept for server-side compatibility with Expo's redirect handling. No Expo - // native auth client is initialised: native never holds a session. - expo(), - emailHarmony({ allowNormalizedSignin: true }), - passkey({ - rpID: envString("PASSKEY_RP_ID", "localhost"), - rpName: "OpenMapX", - origin: process.env.PASSKEY_ORIGIN - ? process.env.PASSKEY_ORIGIN.split(",").map((o) => o.trim()) - : ["http://localhost:3000"], - }), - genericOAuth({ - config: [ - { - providerId: "openstreetmap", - discoveryUrl: getOsmConfig().discoveryUrl, - // Keep the core OAuth endpoints available when OSM discovery is - // temporarily unreachable. Profile identity is still proven by the - // access token against OSM's user-details API. - authorizationUrl: getOsmConfig().webUrl("oauth2/authorize"), - tokenUrl: getOsmConfig().webUrl("oauth2/token"), - // The profile comes from OSM's authenticated user-details endpoint, - // not from Better Auth's local user mapping. Pin its immutable OSM - // numeric ID explicitly so the 1.7 account subject cannot drift. - accountSubject: ({ profile }) => String(profile.id), - clientId: envString("OSM_CLIENT_ID", ""), - clientSecret: envString("OSM_CLIENT_SECRET", ""), - // Ordinary sign-in stays minimal. Contribution write scopes are - // requested incrementally, only when someone starts contributing. - scopes: ["openid", "read_prefs"], - pkce: true, - async getUserInfo({ accessToken }) { - const osm = await fetchOsmUserDetails( - getOsmConfig().apiUrl("api/0.6/user/details.json"), - accessToken, - ); - return { - id: String(osm.id), - name: osm.display_name, - email: `${osm.id}@osm.invalid`, - emailVerified: false, - image: osm.img?.href, - }; + // Generic OAuth + SESSION_REQUIRED: "Sitzung ist erforderlich", }, }, - { - providerId: "mapillary", - authorizationUrl: "https://www.mapillary.com/connect", - tokenUrl: "https://graph.mapillary.com/token", - clientId: envString("MAPILLARY_CLIENT_ID", ""), - clientSecret: envString("MAPILLARY_CLIENT_SECRET", ""), - scopes: ["read"], - async getToken({ code, redirectURI }) { - const token = await exchangeMapillaryAuthorizationCode({ - code, - redirectUri: redirectURI, - clientId: envString("MAPILLARY_CLIENT_ID", ""), - clientSecret: envString("MAPILLARY_CLIENT_SECRET", ""), - }); - return { - accessToken: token.accessToken, - tokenType: token.tokenType, - accessTokenExpiresAt: new Date(Date.now() + token.expiresInSeconds * 1000), - }; + }), + admin(), + // The one mechanism by which a session established in the system browser + // becomes a session in the WebView. Two minutes and single-use because the + // token is in flight across a custom-scheme callback the OS routes, and the + // only legitimate holder redeems it immediately. Stored hashed so a database + // read never yields a usable token. + oneTimeToken({ expiresIn: 2, storeToken: "hashed" }), + // Kept for server-side compatibility with Expo's redirect handling. No Expo + // native auth client is initialised: native never holds a session. + expo(), + emailHarmony({ allowNormalizedSignin: true }), + passkey({ + rpID: envString("PASSKEY_RP_ID", "localhost"), + rpName: "OpenMapX", + origin: process.env.PASSKEY_ORIGIN + ? process.env.PASSKEY_ORIGIN.split(",").map((o) => o.trim()) + : ["http://localhost:3000"], + }), + genericOAuth({ + config: [ + { + providerId: "openstreetmap", + discoveryUrl: getOsmConfig().discoveryUrl, + // Keep the core OAuth endpoints available when OSM discovery is + // temporarily unreachable. Profile identity is still proven by the + // access token against OSM's user-details API. + authorizationUrl: getOsmConfig().webUrl("oauth2/authorize"), + tokenUrl: getOsmConfig().webUrl("oauth2/token"), + // The profile comes from OSM's authenticated user-details endpoint, + // not from Better Auth's local user mapping. Pin its immutable OSM + // numeric ID explicitly so the 1.7 account subject cannot drift. + accountSubject: ({ profile }) => String(profile.id), + clientId: envString("OSM_CLIENT_ID", ""), + clientSecret: envString("OSM_CLIENT_SECRET", ""), + // Ordinary sign-in stays minimal. Contribution write scopes are + // requested incrementally, only when someone starts contributing. + scopes: ["openid", "read_prefs"], + pkce: true, + async getUserInfo({ accessToken }) { + const osm = await fetchOsmUserDetails( + getOsmConfig().apiUrl("api/0.6/user/details.json"), + accessToken, + ); + return { + id: String(osm.id), + name: osm.display_name, + email: `${osm.id}@osm.invalid`, + emailVerified: false, + image: osm.img?.href, + }; + }, + }, + { + providerId: "mapillary", + authorizationUrl: "https://www.mapillary.com/connect", + tokenUrl: "https://graph.mapillary.com/token", + clientId: envString("MAPILLARY_CLIENT_ID", ""), + clientSecret: envString("MAPILLARY_CLIENT_SECRET", ""), + scopes: ["read"], + async getToken({ code, redirectURI }) { + const token = await exchangeMapillaryAuthorizationCode({ + code, + redirectUri: redirectURI, + clientId: envString("MAPILLARY_CLIENT_ID", ""), + clientSecret: envString("MAPILLARY_CLIENT_SECRET", ""), + }); + return { + accessToken: token.accessToken, + tokenType: token.tokenType, + accessTokenExpiresAt: new Date(Date.now() + token.expiresInSeconds * 1000), + }; + }, + async getUserInfo({ accessToken }) { + const data = await fetchMapillaryUserInfo(accessToken); + return { + id: data.id, + name: data.username, + email: `${data.id}@mapillary.invalid`, + emailVerified: false, + }; + }, }, - async getUserInfo({ accessToken }) { - const data = await fetchMapillaryUserInfo(accessToken); - return { - id: data.id, - name: data.username, - email: `${data.id}@mapillary.invalid`, - emailVerified: false, - }; + ], + }), + twoFactor({ + issuer: "OpenMapX", + otpOptions: { + async sendOTP({ user, otp }) { + const mail = twoFactorOtpEmail(otp); + await sendMail({ + to: user.email, + ...mail, + disclosure: { userId: user.id, operationCode: "auth.two-factor.email" }, + }); }, }, - ], - }), - twoFactor({ - issuer: "OpenMapX", - otpOptions: { - async sendOTP({ user, otp }) { - const mail = twoFactorOtpEmail(otp); - await sendMail({ - to: user.email, - ...mail, - disclosure: { userId: user.id, operationCode: "auth.two-factor.email" }, - }); + }), + emailOTP({ + async sendVerificationOTP({ email, otp, type }) { + const mail = emailOtpEmail(otp, type); + // Email OTP can be sent during account changes before a user row is + // available in this callback; only record it when Better Auth gives us + // an exact subject identifier through the typed context. + await sendMail({ to: email, ...mail }); }, - }, - }), - emailOTP({ - async sendVerificationOTP({ email, otp, type }) { - const mail = emailOtpEmail(otp, type); - // Email OTP can be sent during account changes before a user row is - // available in this callback; only record it when Better Auth gives us - // an exact subject identifier through the typed context. - await sendMail({ to: email, ...mail }); - }, - changeEmail: { - enabled: true, - }, - }), - jwt(), - oauthProvider(managedOAuthProviderOptions), - ], -} satisfies BetterAuthOptions; + changeEmail: { + enabled: true, + }, + }), + jwt(), + oauthProvider(managedOAuthProviderOptions), + ], + } satisfies BetterAuthOptions; -/** - * better-auth's own session endpoint returns the full stored session row. That - * row carries the session token, the client IP address and the user agent, - * none of which any caller reads, and the token is the value the signed - * session cookie is built from. `customSession` replaces the endpoint's - * response with the projection below, so those fields never leave the server. - * - * The options object is handed to the plugin so it keeps inferring the fields - * the enabled plugins add to the user and session models. - */ -export const auth = betterAuth({ - ...authOptions, - plugins: [ - ...authOptions.plugins, - customSession( - async ({ user, session }) => projectSessionPayload({ user, session }), - authOptions, - ), - ], -}); + /** + * better-auth's own session endpoint returns the full stored session row. That + * row carries the session token, the client IP address and the user agent, + * none of which any caller reads, and the token is the value the signed + * session cookie is built from. `customSession` replaces the endpoint's + * response with the projection below, so those fields never leave the server. + * + * The options object is handed to the plugin so it keeps inferring the fields + * the enabled plugins add to the user and session models. + */ + return betterAuth({ + ...authOptions, + plugins: [ + ...authOptions.plugins, + customSession( + async ({ user, session }) => projectSessionPayload({ user, session }), + authOptions, + ), + ], + }); +} +export const { auth, withAuthOperation } = createAuthRuntime(createPolicyAuth); diff --git a/apps/api/src/db/account-administration-schema.ts b/apps/api/src/db/account-administration-schema.ts new file mode 100644 index 000000000..c253c80eb --- /dev/null +++ b/apps/api/src/db/account-administration-schema.ts @@ -0,0 +1,22 @@ +import type { AuthPolicySettings, AuthPolicySnapshot } from "@openmapx/core/api-contracts"; +import { sql } from "drizzle-orm"; +import { check, integer, jsonb, pgTable, text, timestamp } from "drizzle-orm/pg-core"; +export const authPolicyState = pgTable( + "auth_policy_state", + { + id: integer("id").primaryKey(), + activationVersion: integer("activation_version").notNull(), + settings: jsonb("settings").$type().notNull(), + revision: text("revision").notNull(), + activatedBy: text("activated_by").notNull(), + activatedAt: timestamp("activated_at", { withTimezone: true }).defaultNow().notNull(), + }, + (t) => [check("auth_policy_singleton", sql`${t.id}=1 AND ${t.activationVersion}=1`)], +); +export const authPolicyApplication = pgTable("auth_policy_application", { + key: text("key").primaryKey(), + actorId: text("actor_id").notNull(), + intentDigest: text("intent_digest").notNull(), + snapshot: jsonb("snapshot").$type().notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), +}); diff --git a/apps/api/src/db/migrations/0043_admin_auth_policy.sql b/apps/api/src/db/migrations/0043_admin_auth_policy.sql new file mode 100644 index 000000000..608a22762 --- /dev/null +++ b/apps/api/src/db/migrations/0043_admin_auth_policy.sql @@ -0,0 +1,17 @@ +CREATE TABLE "auth_policy_application" ( + "key" text PRIMARY KEY NOT NULL, + "actor_id" text NOT NULL, + "intent_digest" text NOT NULL, + "snapshot" jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "auth_policy_state" ( + "id" integer PRIMARY KEY NOT NULL, + "activation_version" integer NOT NULL, + "settings" jsonb NOT NULL, + "revision" text NOT NULL, + "activated_by" text NOT NULL, + "activated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "auth_policy_singleton" CHECK ("auth_policy_state"."id"=1 AND "auth_policy_state"."activation_version"=1) +); diff --git a/apps/api/src/db/migrations/meta/0043_snapshot.json b/apps/api/src/db/migrations/meta/0043_snapshot.json new file mode 100644 index 000000000..9b8a3725b --- /dev/null +++ b/apps/api/src/db/migrations/meta/0043_snapshot.json @@ -0,0 +1,8221 @@ +{ + "id": "d549466e-6409-4b63-9307-604353bbbf8c", + "prevId": "ea53844f-257b-4be6-aefc-34ee5145df6c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 28d0c3b15..371b3cccc 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -302,6 +302,13 @@ "when": 1792939089650, "tag": "0042_admin_data_execution", "breakpoints": true + }, + { + "idx": 43, + "version": "7", + "when": 1793187169774, + "tag": "0043_admin_auth_policy", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 6ad8f0cb1..77655c2ac 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -1,3 +1,4 @@ +export { authPolicyApplication, authPolicyState } from "./account-administration-schema"; // Capability bindings — explicit (integration, capability) → service mappings // data-manager pipeline observability (Postgres schema "data_manager") diff --git a/apps/api/src/routes/__tests__/admin-settings.test.ts b/apps/api/src/routes/__tests__/admin-settings.test.ts index 8009377d3..e2259b81b 100644 --- a/apps/api/src/routes/__tests__/admin-settings.test.ts +++ b/apps/api/src/routes/__tests__/admin-settings.test.ts @@ -82,6 +82,8 @@ vi.mock("../../db/index.js", () => ({ db: { select: (...args: unknown[]) => mockDbSelect(...args), insert: (...args: unknown[]) => mockDbInsert(...args), + transaction: async (run: (tx: unknown) => Promise) => + run({ insert: (...args: unknown[]) => mockDbInsert(...args) }), }, })); vi.mock("../../db/schema.js", () => ({ @@ -396,22 +398,15 @@ describe("POST /admin/settings/import", () => { expect(res.json().imported).toBe(1); }); - it("imports valid values and reports invalid values as skipped", async () => { + it("rejects a malformed authentication draft atomically", async () => { const res = await app.inject({ method: "POST", url: "/admin/settings/import", payload: { settings: { sessionDurationHours: "abc", instanceName: "good" } }, }); - - expect(res.statusCode).toBe(200); - expect(res.json()).toMatchObject({ - ok: true, - imported: 1, - skipped: ["sessionDurationHours"], - }); - expect(mockWriteAuditLog).toHaveBeenCalledWith( - expect.objectContaining({ details: { imported: 1, skipped: ["sessionDurationHours"] } }), - ); + expect(res.statusCode).toBe(400); + expect(mockDbInsert).not.toHaveBeenCalled(); + expect(mockWriteAuditLog).not.toHaveBeenCalled(); }); }); @@ -498,3 +493,15 @@ describe("map settings review regressions", () => { ); }); }); + +describe("atomic authentication drafts", () => { + it("rejects an invalid auth import before writing another valid setting", async () => { + const result = await app.inject({ + method: "POST", + url: "/admin/settings/import", + payload: { settings: { instanceName: "Must not be saved", sessionDurationHours: 0 } }, + }); + expect(result.statusCode).toBe(400); + expect(mockDbInsert).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/routes/admin-accounts.ts b/apps/api/src/routes/admin-accounts.ts new file mode 100644 index 000000000..b72ab6fcd --- /dev/null +++ b/apps/api/src/routes/admin-accounts.ts @@ -0,0 +1,59 @@ +import { accountAdminOperations } from "@openmapx/core/api-contracts"; +import type { FastifyInstance } from "fastify"; +import { AuthPolicyError, applyAuthPolicy, authPolicyView } from "../account/auth-policy"; +import { serviceActionLimit } from "../utils/rate-limit"; +import { getAdminSession, requireAdmin } from "../utils/require-admin"; +import { declareRouteAuth } from "../utils/route-auth"; +import { createAdminRouteRegistrar } from "./admin-contract"; + +const register = createAdminRouteRegistrar(accountAdminOperations, { + tag: "account-admin", + validationPhase: "preHandler", +}); +export async function adminAccountsRoute(app: FastifyInstance) { + declareRouteAuth(app, "admin"); + app.addHook("preHandler", async (request, reply) => { + request.adminSession = await requireAdmin(request); + if (getAdminSession(request).user.id === "loopback") + return reply.code(403).send({ error: "A full administrator account session is required" }); + }); + const failure = (error: unknown, reply: import("fastify").FastifyReply) => { + if (error instanceof AuthPolicyError) { + const status = + error.code === "AUTH_POLICY_CONFLICT" + ? 409 + : error.code === "AUTH_POLICY_INVALID" + ? 400 + : error.code === "AUTH_POLICY_FORBIDDEN" + ? 403 + : 503; + return reply.code(status).send({ error: error.code }); + } + return reply.code(503).send({ error: "AUTH_POLICY_UNAVAILABLE" }); + }; + register(app, "getAdminAuthPolicy", async (_request, reply) => { + try { + return await authPolicyView(); + } catch (error) { + return failure(error, reply); + } + }); + register( + app, + "applyAdminAuthPolicy", + async (request, reply) => { + try { + const active = await applyAuthPolicy(request.body, getAdminSession(request).user.id); + return { + active, + authPolicyRevision: active.revision, + activationVersion: 1 as const, + appliesTo: "new_auth_operations" as const, + }; + } catch (error) { + return failure(error, reply); + } + }, + { preHandler: [serviceActionLimit.preHandler()] }, + ); +} diff --git a/apps/api/src/routes/admin-settings.ts b/apps/api/src/routes/admin-settings.ts index e61b685e7..23d8a4c81 100644 --- a/apps/api/src/routes/admin-settings.ts +++ b/apps/api/src/routes/admin-settings.ts @@ -73,23 +73,25 @@ export async function adminSettingsRoute(app: FastifyInstance) { }); } - for (const { key, value } of toWrite) { - await db - .insert(systemSettings) - .values({ - key, - value: value as never, - updatedBy: adminSession.user.id, - }) - .onConflictDoUpdate({ - target: systemSettings.key, - set: { + await db.transaction(async (transaction) => { + for (const { key, value } of toWrite) { + await transaction + .insert(systemSettings) + .values({ + key, value: value as never, - updatedAt: new Date(), updatedBy: adminSession.user.id, - }, - }); - } + }) + .onConflictDoUpdate({ + target: systemSettings.key, + set: { + value: value as never, + updatedAt: new Date(), + updatedBy: adminSession.user.id, + }, + }); + } + }); await writeAuditLog({ actorId: adminSession.user.id, @@ -110,7 +112,13 @@ export async function adminSettingsRoute(app: FastifyInstance) { await refreshDataUsePolicy(); const groups = await resolveSettings(); - return { ok: true, groups }; + return { + ok: true, + groups, + ...(toWrite.some((v) => defsByKey.get(v.key)?.group === "auth") + ? { authPolicyApplication: "pending" as const } + : {}), + }; }); app.post( @@ -236,26 +244,34 @@ export async function adminSettingsRoute(app: FastifyInstance) { let imported = 0; const skipped: string[] = []; + const toWrite: Array<{ key: string; value: unknown }> = []; for (const [key, value] of Object.entries(settings)) { if (!allowedKeys.has(key)) continue; const def = defsByKey.get(key); if (!def || !matchesDeclaredType(def, value)) { + if (def?.group === "auth") + return reply.code(400).send({ error: "Invalid authentication policy draft" }); skipped.push(key); continue; } - await db - .insert(systemSettings) - .values({ key, value: value as never, updatedBy: adminSession.user.id }) - .onConflictDoUpdate({ - target: systemSettings.key, - set: { - value: value as never, - updatedAt: new Date(), - updatedBy: adminSession.user.id, - }, - }); - imported++; + toWrite.push({ key, value }); } + await db.transaction(async (transaction) => { + for (const { key, value } of toWrite) { + await transaction + .insert(systemSettings) + .values({ key, value: value as never, updatedBy: adminSession.user.id }) + .onConflictDoUpdate({ + target: systemSettings.key, + set: { + value: value as never, + updatedAt: new Date(), + updatedBy: adminSession.user.id, + }, + }); + } + }); + imported = toWrite.length; await writeAuditLog({ actorId: adminSession.user.id, @@ -267,7 +283,14 @@ export async function adminSettingsRoute(app: FastifyInstance) { }); invalidateMapSettings(); - return { ok: true, imported, skipped }; + return { + ok: true, + imported, + skipped, + ...(toWrite.some((v) => defsByKey.get(v.key)?.group === "auth") + ? { authPolicyApplication: "pending" as const } + : {}), + }; }, ); diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index d71264b4d..db6120d06 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -1,5 +1,6 @@ import type { FastifyInstance } from "fastify"; import { adminRoute } from "./admin"; +import { adminAccountsRoute } from "./admin-accounts"; import { adminActivityRoute } from "./admin-activity"; import { adminCacheRoute } from "./admin-cache"; import { registerCapabilityBindingRoutes } from "./admin-capability-bindings"; @@ -73,6 +74,7 @@ export async function registerCoreRoutes( options: CoreRouteOptions, ): Promise { await server.register(authRoute, options); + await server.register(adminAccountsRoute, { prefix: "/api" }); await server.register(metaRoute); diff --git a/apps/api/src/services/settings.ts b/apps/api/src/services/settings.ts index 30e9e19f0..c43857256 100644 --- a/apps/api/src/services/settings.ts +++ b/apps/api/src/services/settings.ts @@ -119,13 +119,14 @@ export const SETTING_DEFS: SettingDef[] = [ description: "New users must verify their email before accessing the app.", type: "boolean", env: "AUTH_EMAIL_VERIFICATION", - default: false, + default: true, }, { group: "auth", key: "sessionDurationHours", + validate: boundedInteger(1, 8760), label: "Session Duration (hours)", - description: "How long a session stays valid after the last activity.", + description: "Absolute lifetime of new sessions. Existing sessions keep their original expiry.", type: "number", default: 168, }, @@ -133,13 +134,14 @@ export const SETTING_DEFS: SettingDef[] = [ group: "auth", key: "twoFaEnabled", label: "Enable Two-Factor Authentication", - description: "Allow users to enroll in TOTP-based 2FA.", + description: "Allow new TOTP enrollment. Existing second-factor challenges remain required.", type: "boolean", - default: false, + default: true, }, { group: "auth", key: "passwordMinLength", + validate: boundedInteger(8, 128), label: "Minimum Password Length", type: "number", default: 8, diff --git a/packages/core/src/api/admin-sdk-transport.test.ts b/packages/core/src/api/admin-sdk-transport.test.ts index 3f78848cb..d35521e6b 100644 --- a/packages/core/src/api/admin-sdk-transport.test.ts +++ b/packages/core/src/api/admin-sdk-transport.test.ts @@ -229,6 +229,7 @@ describe("pending integration family operation binding", () => { const result = await bindAdminOperation(id as keyof typeof integrationAdminOperations)(path, { client, method: operation.method, + headers: { "idempotency-key": "synthetic-intent-0001" }, }); expect(result).toMatchObject({ status: 400, data: { error: "Synthetic rejection" } }); } @@ -267,7 +268,11 @@ describe("pending integration family operation binding", () => { }); }); expect( - await bindAdminOperation(entry.id)(entry.path, { client, method: entry.method }), + await bindAdminOperation(entry.id)(entry.path, { + client, + method: entry.method, + headers: { "idempotency-key": "synthetic-intent-0001" }, + }), ).toMatchObject({ status: entry.status, data: entry.data }); } vi.stubGlobal("fetch", async () => @@ -282,7 +287,7 @@ describe("pending integration family operation binding", () => { vi.stubGlobal("fetch", async () => Response.json({ value: "forbidden" }, { status: 503 })); const error = await bindAdminOperation("setAdminIntegrationCredential")( "/api/admin/credentials/sample/apiKey", - { client, method: "PUT" }, + { client, method: "PUT", headers: { "idempotency-key": "synthetic-intent-0001" } }, ).catch((error) => error); expect(error).toBeInstanceOf(ApiContractError); expect(String(error)).not.toContain("forbidden"); diff --git a/packages/core/src/api/admin-sdk-transport.ts b/packages/core/src/api/admin-sdk-transport.ts index edb659b21..2e9444a65 100644 --- a/packages/core/src/api/admin-sdk-transport.ts +++ b/packages/core/src/api/admin-sdk-transport.ts @@ -6,6 +6,7 @@ import { apiClient, parseRetryAfter, } from "./client"; +import { accountAdminOperations } from "./contracts/account-admin"; import { activityAdminOperations } from "./contracts/activity-admin"; import type { AdminOperation } from "./contracts/admin"; import { dataManagerAdminOperations } from "./contracts/data-manager-admin"; @@ -47,6 +48,7 @@ interface AdminResponse { data: unknown; } const operations = { + ...accountAdminOperations, ...dataManagerAdminOperations, ...activityAdminOperations, ...integrationAdminOperations, diff --git a/packages/core/src/api/admin-sdk.ts b/packages/core/src/api/admin-sdk.ts index 101dd081c..50696e689 100644 --- a/packages/core/src/api/admin-sdk.ts +++ b/packages/core/src/api/admin-sdk.ts @@ -8,10 +8,12 @@ export { adminErrorMessage, unwrapAdminResponse, } from "./admin-sdk-transport"; +export * from "./contracts/account-admin"; export * from "./contracts/activity-admin"; export * from "./contracts/extension-admin"; export * from "./contracts/integration-admin"; export * from "./contracts/service-admin"; +export * from "./generated/admin/account-admin.generated"; export * from "./generated/admin/activity-admin.generated"; export * from "./generated/admin/data-manager-admin.generated"; export * from "./generated/admin/extension-admin.generated"; diff --git a/packages/core/src/api/contracts/account-admin/index.ts b/packages/core/src/api/contracts/account-admin/index.ts new file mode 100644 index 000000000..e0a9401bb --- /dev/null +++ b/packages/core/src/api/contracts/account-admin/index.ts @@ -0,0 +1,36 @@ +import z from "zod/v4"; +import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; +import { authPolicyApplySchema, authPolicySnapshotSchema, authPolicyViewSchema } from "./policy"; + +export * from "./policy"; +export const accountAdminOperations = { + getAdminAuthPolicy: { + method: "GET", + path: "/api/admin/accounts/policy", + auth: "admin", + responses: { + ...adminCommonResponses, + 200: adminJson(authPolicyViewSchema), + 503: adminJson(httpErrorSchema), + }, + }, + applyAdminAuthPolicy: { + method: "POST", + path: "/api/admin/accounts/policy/apply", + auth: "admin", + request: { body: authPolicyApplySchema }, + responses: { + ...adminCommonResponses, + 200: adminJson( + z.strictObject({ + active: authPolicySnapshotSchema, + authPolicyRevision: z.string(), + activationVersion: z.literal(1), + appliesTo: z.literal("new_auth_operations"), + }), + ), + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), + }, + }, +} as const satisfies Record; diff --git a/packages/core/src/api/contracts/account-admin/policy.ts b/packages/core/src/api/contracts/account-admin/policy.ts new file mode 100644 index 000000000..f790b4d65 --- /dev/null +++ b/packages/core/src/api/contracts/account-admin/policy.ts @@ -0,0 +1,29 @@ +import z from "zod/v4"; +export const authPolicySettingsSchema = z.strictObject({ + emailVerificationRequired: z.boolean(), + sessionDurationHours: z.number().int().min(1).max(8760), + twoFaEnabled: z.boolean(), + passwordMinLength: z.number().int().min(8).max(128), +}); +export const authPolicySnapshotSchema = authPolicySettingsSchema.extend({ + activationVersion: z.union([z.literal(0), z.literal(1)]), + revision: z.string().regex(/^[a-f0-9]{64}$/), +}); +export const authPolicyApplySchema = z.strictObject({ + settings: authPolicySettingsSchema, + expectedRevision: z.string().regex(/^[a-f0-9]{64}$/), + confirmed: z.literal(true), + idempotencyKey: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/), +}); +export const authPolicyViewSchema = z.strictObject({ + active: authPolicySnapshotSchema, + pending: z.record(z.string(), z.unknown()), + ownership: z.strictObject({ + emailVerificationRequired: z.enum(["default", "database", "env"]), + sessionDurationHours: z.enum(["default", "database"]), + twoFaEnabled: z.enum(["default", "database"]), + passwordMinLength: z.enum(["default", "database"]), + }), +}); +export type AuthPolicySettings = z.infer; +export type AuthPolicySnapshot = Readonly>; diff --git a/packages/core/src/api/contracts/index.ts b/packages/core/src/api/contracts/index.ts index 97766bc3a..1615ce8d1 100644 --- a/packages/core/src/api/contracts/index.ts +++ b/packages/core/src/api/contracts/index.ts @@ -1,4 +1,5 @@ export { serviceBatchResultSchema } from "../../ops/service-results"; +export * from "./account-admin/index"; export * from "./activity-admin"; export * from "./data-manager-admin/index"; export * from "./extension-admin/index"; diff --git a/packages/core/src/api/generated/admin/account-admin.generated.ts b/packages/core/src/api/generated/admin/account-admin.generated.ts new file mode 100644 index 000000000..ed9d907a5 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-admin.generated.ts @@ -0,0 +1,211 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { + ApplyAdminAuthPolicy200, + ApplyAdminAuthPolicy400, + ApplyAdminAuthPolicy401, + ApplyAdminAuthPolicy403, + ApplyAdminAuthPolicy409, + ApplyAdminAuthPolicy413, + ApplyAdminAuthPolicy415, + ApplyAdminAuthPolicy429, + ApplyAdminAuthPolicy500, + ApplyAdminAuthPolicy503, + ApplyAdminAuthPolicyBody, + GetAdminAuthPolicy200, + GetAdminAuthPolicy400, + GetAdminAuthPolicy401, + GetAdminAuthPolicy403, + GetAdminAuthPolicy413, + GetAdminAuthPolicy415, + GetAdminAuthPolicy429, + GetAdminAuthPolicy500, + GetAdminAuthPolicy503, +} from "./account-models"; +import { + applyAdminAuthPolicyRequest, + getAdminAuthPolicyRequest, +} from "./operation-bindings.generated"; +export type getAdminAuthPolicyResponse200 = { + data: GetAdminAuthPolicy200; + status: 200; +}; + +export type getAdminAuthPolicyResponse400 = { + data: GetAdminAuthPolicy400; + status: 400; +}; + +export type getAdminAuthPolicyResponse401 = { + data: GetAdminAuthPolicy401; + status: 401; +}; + +export type getAdminAuthPolicyResponse403 = { + data: GetAdminAuthPolicy403; + status: 403; +}; + +export type getAdminAuthPolicyResponse413 = { + data: GetAdminAuthPolicy413; + status: 413; +}; + +export type getAdminAuthPolicyResponse415 = { + data: GetAdminAuthPolicy415; + status: 415; +}; + +export type getAdminAuthPolicyResponse429 = { + data: GetAdminAuthPolicy429; + status: 429; +}; + +export type getAdminAuthPolicyResponse500 = { + data: GetAdminAuthPolicy500; + status: 500; +}; + +export type getAdminAuthPolicyResponse503 = { + data: GetAdminAuthPolicy503; + status: 503; +}; + +export type getAdminAuthPolicyResponseSuccess = getAdminAuthPolicyResponse200 & { + headers: Headers; +}; +export type getAdminAuthPolicyResponseError = ( + | getAdminAuthPolicyResponse400 + | getAdminAuthPolicyResponse401 + | getAdminAuthPolicyResponse403 + | getAdminAuthPolicyResponse413 + | getAdminAuthPolicyResponse415 + | getAdminAuthPolicyResponse429 + | getAdminAuthPolicyResponse500 + | getAdminAuthPolicyResponse503 +) & { + headers: Headers; +}; + +export type getAdminAuthPolicyResponse = + | getAdminAuthPolicyResponseSuccess + | getAdminAuthPolicyResponseError; + +export const getGetAdminAuthPolicyUrl = () => { + return `/api/admin/accounts/policy`; +}; + +export const getAdminAuthPolicy = async ( + options?: Parameters[1], +): Promise => { + return getAdminAuthPolicyRequest(getGetAdminAuthPolicyUrl(), { + ...options, + method: "GET", + }); +}; + +export type applyAdminAuthPolicyResponse200 = { + data: ApplyAdminAuthPolicy200; + status: 200; +}; + +export type applyAdminAuthPolicyResponse400 = { + data: ApplyAdminAuthPolicy400; + status: 400; +}; + +export type applyAdminAuthPolicyResponse401 = { + data: ApplyAdminAuthPolicy401; + status: 401; +}; + +export type applyAdminAuthPolicyResponse403 = { + data: ApplyAdminAuthPolicy403; + status: 403; +}; + +export type applyAdminAuthPolicyResponse409 = { + data: ApplyAdminAuthPolicy409; + status: 409; +}; + +export type applyAdminAuthPolicyResponse413 = { + data: ApplyAdminAuthPolicy413; + status: 413; +}; + +export type applyAdminAuthPolicyResponse415 = { + data: ApplyAdminAuthPolicy415; + status: 415; +}; + +export type applyAdminAuthPolicyResponse429 = { + data: ApplyAdminAuthPolicy429; + status: 429; +}; + +export type applyAdminAuthPolicyResponse500 = { + data: ApplyAdminAuthPolicy500; + status: 500; +}; + +export type applyAdminAuthPolicyResponse503 = { + data: ApplyAdminAuthPolicy503; + status: 503; +}; + +export type applyAdminAuthPolicyResponseSuccess = applyAdminAuthPolicyResponse200 & { + headers: Headers; +}; +export type applyAdminAuthPolicyResponseError = ( + | applyAdminAuthPolicyResponse400 + | applyAdminAuthPolicyResponse401 + | applyAdminAuthPolicyResponse403 + | applyAdminAuthPolicyResponse409 + | applyAdminAuthPolicyResponse413 + | applyAdminAuthPolicyResponse415 + | applyAdminAuthPolicyResponse429 + | applyAdminAuthPolicyResponse500 + | applyAdminAuthPolicyResponse503 +) & { + headers: Headers; +}; + +export type applyAdminAuthPolicyResponse = + | applyAdminAuthPolicyResponseSuccess + | applyAdminAuthPolicyResponseError; + +export const getApplyAdminAuthPolicyUrl = () => { + return `/api/admin/accounts/policy/apply`; +}; + +export const applyAdminAuthPolicy = async ( + applyAdminAuthPolicyBody: ApplyAdminAuthPolicyBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return applyAdminAuthPolicyRequest(getApplyAdminAuthPolicyUrl(), { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(applyAdminAuthPolicyBody), + }); +}; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200.ts new file mode 100644 index 000000000..a49a6068d --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminAuthPolicy200ActivationVersion } from "./applyAdminAuthPolicy200ActivationVersion"; +import type { ApplyAdminAuthPolicy200Active } from "./applyAdminAuthPolicy200Active"; +import type { ApplyAdminAuthPolicy200AppliesTo } from "./applyAdminAuthPolicy200AppliesTo"; + +export type ApplyAdminAuthPolicy200 = { + active: ApplyAdminAuthPolicy200Active; + authPolicyRevision: string; + activationVersion: ApplyAdminAuthPolicy200ActivationVersion; + appliesTo: ApplyAdminAuthPolicy200AppliesTo; +}; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActivationVersion.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActivationVersion.ts new file mode 100644 index 000000000..c7c2656ec --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActivationVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy200ActivationVersion = + (typeof ApplyAdminAuthPolicy200ActivationVersion)[keyof typeof ApplyAdminAuthPolicy200ActivationVersion]; + +export const ApplyAdminAuthPolicy200ActivationVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200Active.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200Active.ts new file mode 100644 index 000000000..66ccb9298 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200Active.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminAuthPolicy200ActiveActivationVersion } from "./applyAdminAuthPolicy200ActiveActivationVersion"; + +export type ApplyAdminAuthPolicy200Active = { + emailVerificationRequired: boolean; + /** + * @minimum 1 + * @maximum 8760 + */ + sessionDurationHours: number; + twoFaEnabled: boolean; + /** + * @minimum 8 + * @maximum 128 + */ + passwordMinLength: number; + activationVersion: (typeof ApplyAdminAuthPolicy200ActiveActivationVersion)[keyof typeof ApplyAdminAuthPolicy200ActiveActivationVersion]; + /** @pattern ^[a-f0-9]{64}$ */ + revision: string; +}; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActiveActivationVersion.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActiveActivationVersion.ts new file mode 100644 index 000000000..710724049 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActiveActivationVersion.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export const ApplyAdminAuthPolicy200ActiveActivationVersion = { NUMBER_0: 0, NUMBER_1: 1 } as const; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200AppliesTo.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200AppliesTo.ts new file mode 100644 index 000000000..6901b69bc --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200AppliesTo.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy200AppliesTo = + (typeof ApplyAdminAuthPolicy200AppliesTo)[keyof typeof ApplyAdminAuthPolicy200AppliesTo]; + +export const ApplyAdminAuthPolicy200AppliesTo = { + new_auth_operations: "new_auth_operations", +} as const; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy400.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy400.ts new file mode 100644 index 000000000..a04283fbc --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy401.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy401.ts new file mode 100644 index 000000000..8a567b214 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy403.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy403.ts new file mode 100644 index 000000000..fa7bb81f7 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy409.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy409.ts new file mode 100644 index 000000000..2c0e3e30d --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy413.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy413.ts new file mode 100644 index 000000000..2ca77280b --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy415.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy415.ts new file mode 100644 index 000000000..c36496982 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy429.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy429.ts new file mode 100644 index 000000000..8f2c27ae1 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy500.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy500.ts new file mode 100644 index 000000000..0484af635 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy503.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy503.ts new file mode 100644 index 000000000..cabd9e92f --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicy503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBody.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBody.ts new file mode 100644 index 000000000..4fcc5d2d1 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBody.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminAuthPolicyBodySettings } from "./applyAdminAuthPolicyBodySettings"; + +export type ApplyAdminAuthPolicyBody = { + settings: ApplyAdminAuthPolicyBodySettings; + /** @pattern ^[a-f0-9]{64}$ */ + expectedRevision: string; + confirmed: true; + /** @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ */ + idempotencyKey: string; +}; diff --git a/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBodySettings.ts b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBodySettings.ts new file mode 100644 index 000000000..e24632246 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBodySettings.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminAuthPolicyBodySettings = { + emailVerificationRequired: boolean; + /** + * @minimum 1 + * @maximum 8760 + */ + sessionDurationHours: number; + twoFaEnabled: boolean; + /** + * @minimum 8 + * @maximum 128 + */ + passwordMinLength: number; +}; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200.ts new file mode 100644 index 000000000..fc555170a --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminAuthPolicy200Active } from "./getAdminAuthPolicy200Active"; +import type { GetAdminAuthPolicy200Ownership } from "./getAdminAuthPolicy200Ownership"; +import type { GetAdminAuthPolicy200Pending } from "./getAdminAuthPolicy200Pending"; + +export type GetAdminAuthPolicy200 = { + active: GetAdminAuthPolicy200Active; + pending: GetAdminAuthPolicy200Pending; + ownership: GetAdminAuthPolicy200Ownership; +}; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Active.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Active.ts new file mode 100644 index 000000000..7b03e4e64 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Active.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminAuthPolicy200ActiveActivationVersion } from "./getAdminAuthPolicy200ActiveActivationVersion"; + +export type GetAdminAuthPolicy200Active = { + emailVerificationRequired: boolean; + /** + * @minimum 1 + * @maximum 8760 + */ + sessionDurationHours: number; + twoFaEnabled: boolean; + /** + * @minimum 8 + * @maximum 128 + */ + passwordMinLength: number; + activationVersion: (typeof GetAdminAuthPolicy200ActiveActivationVersion)[keyof typeof GetAdminAuthPolicy200ActiveActivationVersion]; + /** @pattern ^[a-f0-9]{64}$ */ + revision: string; +}; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200ActiveActivationVersion.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200ActiveActivationVersion.ts new file mode 100644 index 000000000..e9f778ad4 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200ActiveActivationVersion.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export const GetAdminAuthPolicy200ActiveActivationVersion = { NUMBER_0: 0, NUMBER_1: 1 } as const; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Ownership.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Ownership.ts new file mode 100644 index 000000000..64d1620e4 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Ownership.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminAuthPolicy200OwnershipEmailVerificationRequired } from "./getAdminAuthPolicy200OwnershipEmailVerificationRequired"; +import type { GetAdminAuthPolicy200OwnershipPasswordMinLength } from "./getAdminAuthPolicy200OwnershipPasswordMinLength"; +import type { GetAdminAuthPolicy200OwnershipSessionDurationHours } from "./getAdminAuthPolicy200OwnershipSessionDurationHours"; +import type { GetAdminAuthPolicy200OwnershipTwoFaEnabled } from "./getAdminAuthPolicy200OwnershipTwoFaEnabled"; + +export type GetAdminAuthPolicy200Ownership = { + emailVerificationRequired: GetAdminAuthPolicy200OwnershipEmailVerificationRequired; + sessionDurationHours: GetAdminAuthPolicy200OwnershipSessionDurationHours; + twoFaEnabled: GetAdminAuthPolicy200OwnershipTwoFaEnabled; + passwordMinLength: GetAdminAuthPolicy200OwnershipPasswordMinLength; +}; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipEmailVerificationRequired.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipEmailVerificationRequired.ts new file mode 100644 index 000000000..a3ad3a019 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipEmailVerificationRequired.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy200OwnershipEmailVerificationRequired = + (typeof GetAdminAuthPolicy200OwnershipEmailVerificationRequired)[keyof typeof GetAdminAuthPolicy200OwnershipEmailVerificationRequired]; + +export const GetAdminAuthPolicy200OwnershipEmailVerificationRequired = { + default: "default", + database: "database", + env: "env", +} as const; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipPasswordMinLength.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipPasswordMinLength.ts new file mode 100644 index 000000000..716b74323 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipPasswordMinLength.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy200OwnershipPasswordMinLength = + (typeof GetAdminAuthPolicy200OwnershipPasswordMinLength)[keyof typeof GetAdminAuthPolicy200OwnershipPasswordMinLength]; + +export const GetAdminAuthPolicy200OwnershipPasswordMinLength = { + default: "default", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipSessionDurationHours.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipSessionDurationHours.ts new file mode 100644 index 000000000..8467f7b42 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipSessionDurationHours.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy200OwnershipSessionDurationHours = + (typeof GetAdminAuthPolicy200OwnershipSessionDurationHours)[keyof typeof GetAdminAuthPolicy200OwnershipSessionDurationHours]; + +export const GetAdminAuthPolicy200OwnershipSessionDurationHours = { + default: "default", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipTwoFaEnabled.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipTwoFaEnabled.ts new file mode 100644 index 000000000..de8140ea0 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipTwoFaEnabled.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy200OwnershipTwoFaEnabled = + (typeof GetAdminAuthPolicy200OwnershipTwoFaEnabled)[keyof typeof GetAdminAuthPolicy200OwnershipTwoFaEnabled]; + +export const GetAdminAuthPolicy200OwnershipTwoFaEnabled = { + default: "default", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Pending.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Pending.ts new file mode 100644 index 000000000..d4cf9c3f2 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Pending.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy200Pending = { [key: string]: unknown }; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy400.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy400.ts new file mode 100644 index 000000000..e301dfc00 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy401.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy401.ts new file mode 100644 index 000000000..47a31a850 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy403.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy403.ts new file mode 100644 index 000000000..5f779f4d7 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy413.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy413.ts new file mode 100644 index 000000000..274b243f9 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy415.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy415.ts new file mode 100644 index 000000000..a3b1dd5d2 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy429.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy429.ts new file mode 100644 index 000000000..385e7098b --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy500.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy500.ts new file mode 100644 index 000000000..06d1d5481 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy503.ts b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy503.ts new file mode 100644 index 000000000..769fa414a --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminAuthPolicy503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/account-models/index.ts b/packages/core/src/api/generated/admin/account-models/index.ts new file mode 100644 index 000000000..b693fe901 --- /dev/null +++ b/packages/core/src/api/generated/admin/account-models/index.ts @@ -0,0 +1,36 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export * from "./applyAdminAuthPolicy200"; +export * from "./applyAdminAuthPolicy200ActivationVersion"; +export * from "./applyAdminAuthPolicy200Active"; +export * from "./applyAdminAuthPolicy200ActiveActivationVersion"; +export * from "./applyAdminAuthPolicy200AppliesTo"; +export * from "./applyAdminAuthPolicy400"; +export * from "./applyAdminAuthPolicy401"; +export * from "./applyAdminAuthPolicy403"; +export * from "./applyAdminAuthPolicy409"; +export * from "./applyAdminAuthPolicy413"; +export * from "./applyAdminAuthPolicy415"; +export * from "./applyAdminAuthPolicy429"; +export * from "./applyAdminAuthPolicy500"; +export * from "./applyAdminAuthPolicy503"; +export * from "./applyAdminAuthPolicyBody"; +export * from "./applyAdminAuthPolicyBodySettings"; +export * from "./getAdminAuthPolicy200"; +export * from "./getAdminAuthPolicy200Active"; +export * from "./getAdminAuthPolicy200ActiveActivationVersion"; +export * from "./getAdminAuthPolicy200Ownership"; +export * from "./getAdminAuthPolicy200OwnershipEmailVerificationRequired"; +export * from "./getAdminAuthPolicy200OwnershipPasswordMinLength"; +export * from "./getAdminAuthPolicy200OwnershipSessionDurationHours"; +export * from "./getAdminAuthPolicy200OwnershipTwoFaEnabled"; +export * from "./getAdminAuthPolicy200Pending"; +export * from "./getAdminAuthPolicy400"; +export * from "./getAdminAuthPolicy401"; +export * from "./getAdminAuthPolicy403"; +export * from "./getAdminAuthPolicy413"; +export * from "./getAdminAuthPolicy415"; +export * from "./getAdminAuthPolicy429"; +export * from "./getAdminAuthPolicy500"; +export * from "./getAdminAuthPolicy503"; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index 75d816f11..4f15828ef 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -14,6 +14,13 @@ export async function addAdminExtensionSourceRequest( ): Promise { return addAdminExtensionSourceBound(url, options); } +const applyAdminAuthPolicyBound = bindAdminOperation("applyAdminAuthPolicy"); +export async function applyAdminAuthPolicyRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return applyAdminAuthPolicyBound(url, options); +} const cancelAdminActivityJobBound = bindAdminOperation("cancelAdminActivityJob"); export async function cancelAdminActivityJobRequest( url: string, @@ -111,6 +118,13 @@ export async function getAdminActivityJobStreamMetricsRequest( ): Promise { return getAdminActivityJobStreamMetricsBound(url, options); } +const getAdminAuthPolicyBound = bindAdminOperation("getAdminAuthPolicy"); +export async function getAdminAuthPolicyRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminAuthPolicyBound(url, options); +} const getAdminComposePreviewBound = bindAdminOperation("getAdminComposePreview"); export async function getAdminComposePreviewRequest( url: string, diff --git a/scripts/api-sdk/codegen.test.ts b/scripts/api-sdk/codegen.test.ts index e349fc7e8..6453d29ce 100644 --- a/scripts/api-sdk/codegen.test.ts +++ b/scripts/api-sdk/codegen.test.ts @@ -73,11 +73,11 @@ function generatedConsumerDiagnostics(files: { filename: string; content: string const source: "default" | "database" | "vault" | "config.json" | "env" = response.data.resolvedConfig.field.source; } }); - setAdminIntegrationCredential("sample", "apiKey", {value: "synthetic"}, finiteOptions); + setAdminIntegrationCredential("sample", "apiKey", {value: "synthetic"}, {"idempotency-key":"synthetic-intent-0001"}, finiteOptions); // @ts-expect-error Credential bodies reject unknown fields. - setAdminIntegrationCredential("sample", "apiKey", {value: "synthetic", unexpected: true}); + setAdminIntegrationCredential("sample", "apiKey", {value: "synthetic", unexpected: true}, {"idempotency-key":"synthetic-intent-0001"}); // @ts-expect-error Credentials are strings, not numbers. - setAdminIntegrationCredential("sample", "apiKey", {value: 1}); + setAdminIntegrationCredential("sample", "apiKey", {value: 1}, {"idempotency-key":"synthetic-intent-0001"}); getAdminIntegrationHealthHistory("sample", {hours: 24}, finiteOptions); // @ts-expect-error Generated query output exposes bounded integer input, not string coercion. getAdminIntegrationHealthHistory("sample", {hours: "24"}); @@ -124,9 +124,9 @@ function generatedConsumerDiagnostics(files: { filename: string; content: string if (response.status === 200) {} if (response.status === 503) { const guidance: string = response.data.error; const ok: false = response.data.ok; } }); - setAdminServiceSelection({selectedRoots: []}); + setAdminServiceSelection({selectedRoots: []}, {"idempotency-key":"synthetic-intent-0001"}); // @ts-expect-error An absent selection cannot mean intentional deselection. - setAdminServiceSelection({}); + setAdminServiceSelection({}, {"idempotency-key":"synthetic-intent-0001"}); // @ts-expect-error Build-only fields cannot enter lifecycle actions. runAdminServicesBulkAction({action: "start", serviceIds: ["sample"], all: true}); restoreAdminServiceBackup("sample", {serviceIds: [], stopRunning: true}); @@ -668,6 +668,20 @@ describe("integration administration generation", () => { name: "hours", schema: { type: "integer", minimum: 1, maximum: 168, default: 24 }, }); + if (request && "headers" in request) { + const emitted = routeJsonSchema(request.headers, { + identity: `${id}Headers`, + io: "output", + }); + const required = emitted.required as string[] | undefined; + for (const [name, field] of Object.entries(emitted.properties as Record)) + parameters.push({ + in: "header", + name, + required: required?.includes(name) ?? false, + schema: field, + }); + } input.paths[path] ??= {}; input.paths[path][operation.method.toLowerCase()] = { operationId: id, diff --git a/scripts/api-sdk/config.mjs b/scripts/api-sdk/config.mjs index 51b00c14f..9e909c77e 100644 --- a/scripts/api-sdk/config.mjs +++ b/scripts/api-sdk/config.mjs @@ -3,6 +3,7 @@ import { emitOperationBindings, operationIds } from "../codegen/operation-bindin import { validatedResponseTypes } from "./validated-response-types.mjs"; const adminFamilies = [ + { key: "accountAdmin", tag: "account-admin", models: "account-models" }, { key: "dataManagerAdmin", tag: "data-manager-admin", models: "models", required: true }, { key: "activityAdmin", diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 2e050367e..e7549aa41 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -22,7 +22,8 @@ "apps/api/src/services/capability-bindings.test.ts", "apps/api/src/services/integration-config-store-postgres.test.ts", "apps/api/src/services/mobileAuthHandoff.test.ts", - "apps/api/src/services/service-config-writer-postgres.test.ts" + "apps/api/src/services/service-config-writer-postgres.test.ts", + "apps/api/src/account/auth-policy-postgres.test.ts" ], "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", From fc0b5a8196c957059b06716d64d727ed09f64b23 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 13:46:11 +0100 Subject: [PATCH 041/135] fix(auth): serialize account mutations and preserve the last admin --- apps/api/src/account/account-context.ts | 21 + .../api/src/account/account-mutations.test.ts | 27 + apps/api/src/account/account-mutations.ts | 203 + .../account/admin-invariant-migration.test.ts | 62 + .../account/admin-invariant-postgres.test.ts | 446 + apps/api/src/account/admin-invariant.ts | 56 + apps/api/src/account/auth-account-entry.ts | 109 + .../src/account/auth-policy-postgres.test.ts | 23 +- apps/api/src/account/auth-policy.ts | 7 +- apps/api/src/account/auth-runtime.ts | 74 +- .../account/password-verification-policy.ts | 14 +- apps/api/src/auth.ts | 9 +- .../__tests__/user-erasure-postgres.test.ts | 2 + .../src/db/account-administration-schema.ts | 14 +- .../0044_admin_account_invariant.sql | 47 + .../src/db/migrations/meta/0044_snapshot.json | 8283 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + apps/api/src/db/schema.ts | 6 +- apps/api/src/privacy/catalogue.ts | 4 +- apps/api/src/privacy/openmapx-collectors.ts | 47 + apps/api/src/privacy/session-assurance.ts | 4 +- apps/api/src/routes/privacy-roles.ts | 61 +- apps/api/src/services/user-erasure.ts | 28 +- apps/api/src/test/postgres-fixture.ts | 37 +- apps/api/src/utils/audit-log.ts | 32 +- apps/api/src/utils/auth-audit-hook.ts | 2 + scripts/check-subject-data.ts | 2 + scripts/database-test-groups.json | 4 +- 28 files changed, 9533 insertions(+), 98 deletions(-) create mode 100644 apps/api/src/account/account-context.ts create mode 100644 apps/api/src/account/account-mutations.test.ts create mode 100644 apps/api/src/account/account-mutations.ts create mode 100644 apps/api/src/account/admin-invariant-migration.test.ts create mode 100644 apps/api/src/account/admin-invariant-postgres.test.ts create mode 100644 apps/api/src/account/admin-invariant.ts create mode 100644 apps/api/src/account/auth-account-entry.ts create mode 100644 apps/api/src/db/migrations/0044_admin_account_invariant.sql create mode 100644 apps/api/src/db/migrations/meta/0044_snapshot.json diff --git a/apps/api/src/account/account-context.ts b/apps/api/src/account/account-context.ts new file mode 100644 index 000000000..3f1eacb78 --- /dev/null +++ b/apps/api/src/account/account-context.ts @@ -0,0 +1,21 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type { AuthDatabase } from "./auth-database"; +export interface AccountMutationContext { + database: AuthDatabase; + actorId: string | null; + targetId: string | null; + action: "role" | "ban" | "unban" | "delete" | "create" | "profile"; + mutationKey: string; + roleAudit: boolean; + afterCommit: Array<() => Promise>; +} +const context = new AsyncLocalStorage(); +export function currentAccountMutation() { + return context.getStore(); +} +export function withAccountMutationContext( + value: AccountMutationContext, + run: () => Promise, +) { + return context.run(value, run); +} diff --git a/apps/api/src/account/account-mutations.test.ts b/apps/api/src/account/account-mutations.test.ts new file mode 100644 index 000000000..61c1473a6 --- /dev/null +++ b/apps/api/src/account/account-mutations.test.ts @@ -0,0 +1,27 @@ +import { expect, it } from "vitest"; +import { assertAccountDeletionAllowed, mapAccountConstraint } from "./admin-invariant"; + +it("denies an erasure hook without the owning account transaction before any data access", async () => { + await expect(assertAccountDeletionAllowed("untrusted-target")).rejects.toThrow( + "Account deletion transaction required", + ); +}); +it("maps only the named database invariant and keeps database diagnostics out of the public error", () => { + let error: unknown; + try { + mapAccountConstraint({ + cause: { constraint_name: "LAST_ADMIN_REQUIRED", message: "sensitive SQL diagnostic" }, + }); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ statusCode: 409, body: { code: "LAST_ADMIN_REQUIRED" } }); + expect(JSON.stringify(error)).not.toContain("sensitive SQL diagnostic"); + const other = { cause: { constraint_name: "some_other_constraint" } }; + try { + mapAccountConstraint(other); + throw Error("Expected failure"); + } catch (caught) { + expect(caught).toBe(other); + } +}); diff --git a/apps/api/src/account/account-mutations.ts b/apps/api/src/account/account-mutations.ts new file mode 100644 index 000000000..b1267f9ae --- /dev/null +++ b/apps/api/src/account/account-mutations.ts @@ -0,0 +1,203 @@ +import { createHash, randomUUID } from "node:crypto"; +import { APIError } from "better-auth/api"; +import { and, eq, sql } from "drizzle-orm"; +import z from "zod/v4"; +import { adminAuditLog, session, user } from "../db/schema"; +import { + type AccountMutationContext, + currentAccountMutation, + withAccountMutationContext, +} from "./account-context"; +import { assertAdminCapacity, lockAdminGuard, mapAccountConstraint } from "./admin-invariant"; +import { + type AuthDatabase, + currentAuthDatabase, + isTransactionalAuthDatabase, + withAuthDatabase, +} from "./auth-database"; +export const accountRoleSchema = z.enum(["user", "admin", "privacy_admin"]); +export type AccountRole = z.infer; +export interface AccountMutationIntent { + actorId: string | null; + targetId: string | null; + action: AccountMutationContext["action"]; + mutationKey: string; + role?: AccountRole; + internalCreate?: boolean; + selfServiceDelete?: boolean; +} +class MutationResponseRollback extends Error { + constructor(readonly response: Response) { + super("Account mutation failed"); + } +} +export async function assertAccountMutationOutcome(value: T): Promise { + if (value instanceof Response && value.status >= 400) throw new MutationResponseRollback(value); + return value; +} +export async function withAccountMutation( + intent: AccountMutationIntent, + run: (tx: AuthDatabase) => Promise, + database: AuthDatabase = currentAuthDatabase(), +): Promise { + const parent = currentAccountMutation(); + const afterCommit: Array<() => Promise> = parent?.afterCommit ?? []; + const perform = async (tx: AuthDatabase) => { + await lockAdminGuard(tx); + if (intent.actorId) { + const [actor] = await tx.select().from(user).where(eq(user.id, intent.actorId)).for("update"); + if ( + !actor || + actor.banned === true || + (!(intent.action === "delete" && intent.selfServiceDelete) && actor.role !== "admin") + ) + throw new APIError("FORBIDDEN", { + code: "FULL_ADMIN_REQUIRED", + message: "An active full administrator is required", + }); + } else if (!(intent.action === "create" && intent.internalCreate)) + throw new APIError("UNAUTHORIZED", { + code: "UNAUTHORIZED", + message: "Authentication required", + }); + if (intent.targetId) { + await assertAdminCapacity( + tx, + intent.targetId, + intent.action === "delete" + ? { deleted: true } + : intent.action === "ban" + ? { banned: true } + : intent.action === "role" + ? { role: intent.role } + : {}, + ); + } + return withAuthDatabase(tx, () => + withAccountMutationContext( + { + database: tx, + actorId: intent.actorId, + targetId: intent.targetId, + action: intent.action, + mutationKey: intent.mutationKey, + roleAudit: false, + afterCommit, + }, + async () => assertAccountMutationOutcome(await run(tx)), + ), + ); + }; + if (parent) { + if ( + !isTransactionalAuthDatabase() || + parent.database !== database || + parent.actorId !== intent.actorId || + parent.targetId !== intent.targetId || + parent.action !== intent.action + ) + throw Error("Account mutation must remain in its owning transaction"); + // The original owner alone publishes completion after the physical COMMIT. + // A nested Better Auth operation shares its transaction and completion queue. + return perform(database); + } + if (isTransactionalAuthDatabase()) + throw Error("An owning account mutation is required before entering an auth transaction"); + try { + const result = await database.transaction(perform); + for (const complete of afterCommit) await complete().catch(() => {}); + return result; + } catch (error) { + if (error instanceof MutationResponseRollback) return error.response as T; + return mapAccountConstraint(error); + } +} +interface RoleChange { + actorId: string; + userId: string; + role: AccountRole; + expectedRole?: AccountRole; + idempotencyKey: string; +} +async function roleChange(input: RoleChange, tx: AuthDatabase, write: () => Promise) { + const role = accountRoleSchema.parse(input.role), + target = await assertAdminCapacity(tx, input.userId, { role }); + if (input.expectedRole !== undefined && (target.role ?? "user") !== input.expectedRole) + throw new APIError("CONFLICT", { + code: "ROLE_VERSION_CONFLICT", + message: "Account role changed; reload before retrying", + }); + const previousRole = target.role ?? "user", + changed = previousRole !== role, + context = currentAccountMutation(); + if ( + !context || + context.actorId !== input.actorId || + context.targetId !== input.userId || + context.action !== "role" + ) + throw Error("Role transaction required"); + context.roleAudit = true; + const value = await assertAccountMutationOutcome(await write()); + const [updated] = await tx.select().from(user).where(eq(user.id, input.userId)); + if (updated?.role !== role) throw Error("Role mutation did not commit its requested state"); + if (changed) { + await tx.delete(session).where(eq(session.userId, input.userId)); + await tx.insert(adminAuditLog).values({ + id: randomUUID(), + actorId: input.actorId, + targetId: input.userId, + targetType: "user", + action: "user.role.change", + details: { + previousRole, + role, + mutationDigest: createHash("sha256").update(input.idempotencyKey).digest("hex"), + }, + }); + } + return { value, change: { userId: input.userId, role, changed } }; +} +/** Both native role updates and Better Auth's authorized writer share this policy/audit transaction. */ +export async function changeAccountRole( + input: RoleChange, + database: AuthDatabase = currentAuthDatabase(), +) { + return withAccountMutation( + { + actorId: input.actorId, + targetId: input.userId, + action: "role", + mutationKey: input.idempotencyKey, + role: input.role, + }, + async (tx) => + ( + await roleChange(input, tx, () => + tx + .update(user) + .set({ role: input.role, updatedAt: new Date() }) + .where(eq(user.id, input.userId)) + .returning(), + ) + ).change, + database, + ); +} +export async function writeAuthAccountRole( + input: RoleChange, + write: () => Promise, + database: AuthDatabase = currentAuthDatabase(), +) { + return withAccountMutation( + { + actorId: input.actorId, + targetId: input.userId, + action: "role", + mutationKey: input.idempotencyKey, + role: input.role, + }, + async (tx) => (await roleChange(input, tx, write)).value, + database, + ); +} diff --git a/apps/api/src/account/admin-invariant-migration.test.ts b/apps/api/src/account/admin-invariant-migration.test.ts new file mode 100644 index 000000000..ecdc8a5fd --- /dev/null +++ b/apps/api/src/account/admin-invariant-migration.test.ts @@ -0,0 +1,62 @@ +import { fileURLToPath } from "node:url"; +import { eq } from "drizzle-orm"; +import { migrate } from "drizzle-orm/postgres-js/migrator"; +import { describe, expect, it } from "vitest"; +import { accountAdminGuard, user } from "../db/schema"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const dbDescribe = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1" ? describe : describe.skip; +dbDescribe("last-admin migration of historical roles", () => { + it("rejects unknown roles without partial activation, then normalizes recognized combinations under the write lock", async () => { + const fixture = await createMigratedPostgresFixture({ throughMigration: 43 }); + const apply = () => + migrate(fixture.db, { + migrationsFolder: fileURLToPath(new URL("../db/migrations", import.meta.url)), + }); + try { + await fixture.db.insert(user).values([ + { id: "legacy-a", name: "Synthetic A", email: "legacy-a@example.test", role: "admin,user" }, + { + id: "legacy-b", + name: "Synthetic B", + email: "legacy-b@example.test", + role: "admin,privacy_admin", + }, + { + id: "legacy-c", + name: "Synthetic C", + email: "legacy-c@example.test", + role: "privacy_admin,user", + }, + { + id: "legacy-unknown", + name: "Synthetic unknown", + email: "legacy-unknown@example.test", + role: "foreign_admin", + }, + ]); + await expect(apply()).rejects.toThrow(); + expect((await fixture.db.select().from(user).where(eq(user.id, "legacy-a")))[0]?.role).toBe( + "admin,user", + ); + await fixture.db.update(user).set({ role: "user" }).where(eq(user.id, "legacy-unknown")); + await apply(); + expect((await fixture.db.select().from(user).orderBy(user.id)).map((r) => r.role)).toEqual([ + "admin", + "admin", + "privacy_admin", + "user", + ]); + expect((await fixture.db.select().from(accountAdminGuard))[0]?.activeCount).toBe(2); + await fixture.db.update(user).set({ role: "user" }).where(eq(user.id, "legacy-a")); + await expect( + fixture.db.update(user).set({ banned: true }).where(eq(user.id, "legacy-b")), + ).rejects.toThrow(); + await expect( + fixture.db.update(user).set({ role: "admin,user" }).where(eq(user.id, "legacy-c")), + ).rejects.toThrow(); + } finally { + await fixture.stop(); + } + }, 90000); +}); diff --git a/apps/api/src/account/admin-invariant-postgres.test.ts b/apps/api/src/account/admin-invariant-postgres.test.ts new file mode 100644 index 000000000..2b6a2f0be --- /dev/null +++ b/apps/api/src/account/admin-invariant-postgres.test.ts @@ -0,0 +1,446 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { and, eq, ne, sql } from "drizzle-orm"; +import Fastify from "fastify"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + accountAdminGuard, + adminAuditLog, + authPolicyApplication, + authPolicyState, + session, + user, +} from "../db/schema"; +import { authRoute } from "../routes/auth"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const dbDescribe = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1" ? describe : describe.skip; +dbDescribe("atomic last administrator protection", () => { + let fixture: Awaited>, + auth: typeof import("../auth")["auth"], + app: ReturnType, + anchorId: string; + let directory = "", + journal = ""; + beforeAll(async () => { + fixture = await createMigratedPostgresFixture(); + directory = mkdtempSync(join(tmpdir(), "openmapx-account-guard-")); + journal = join(directory, "erasure.jsonl"); + const keyFile = join(directory, "journal.key"), + journalKey = randomBytes(32); + writeFileSync(keyFile, journalKey.toString("base64url"), { mode: 0o444 }); + (await import("@openmapx/core/erasure-journal")).initializeErasureJournal(journal, journalKey); + vi.stubEnv("BETTER_AUTH_SECRET", randomBytes(48).toString("base64url")); + vi.stubEnv("BETTER_AUTH_URL", "http://localhost:3001"); + vi.stubEnv("ERASURE_JOURNAL_KEY_FILE", keyFile); + vi.stubEnv("ERASURE_JOURNAL_KEY_UID", String(process.getuid?.() ?? 0)); + vi.stubEnv("ERASURE_JOURNAL_PATH", journal); + vi.doMock("../db", () => ({ db: fixture.db, sql: fixture.sql })); + vi.doMock("../utils/email", () => ({ sendMail: async () => {} })); + ({ auth } = await import("../auth")); + app = Fastify(); + await app.register(authRoute, { + authHandler: auth.handler, + authUiOrigin: "http://localhost:3000", + }); + await app.ready(); + const anchor = await auth.api.createUser({ + body: { + name: "Fixture anchor", + email: "anchor-admin@example.test", + password: "fixture-password-1234", + role: "admin", + }, + }); + anchorId = anchor.user.id; + }, 90000); + afterEach(async () => { + if (!fixture || !anchorId) return; + await fixture.db + .update(user) + .set({ role: "admin", banned: false }) + .where(eq(user.id, anchorId)); + await fixture.db.update(user).set({ role: "user", banned: false }).where(ne(user.id, anchorId)); + }); + afterAll(async () => { + await app?.close(); + await fixture?.stop(); + rmSync(directory, { recursive: true, force: true }); + vi.doUnmock("../db"); + vi.doUnmock("../utils/email"); + vi.unstubAllEnvs(); + }); + async function actor(label: string, role: "admin" | "user" = "admin") { + const created = await auth.api.createUser({ + body: { + name: "Synthetic account", + email: `${label}-${randomUUID()}@example.test`, + password: "fixture-password-1234", + role, + }, + }); + await fixture.db.update(user).set({ emailVerified: true }).where(eq(user.id, created.user.id)); + const signin = await auth.api.signInEmail({ + body: { email: created.user.email, password: "fixture-password-1234" }, + returnHeaders: true, + }); + return { + id: created.user.id, + cookie: signin.headers + .getSetCookie() + .map((v) => v.split(";")[0]) + .join("; "), + }; + } + const headers = (cookie: string) => ({ + host: "localhost:3001", + origin: "http://localhost:3000", + cookie, + "idempotency-key": randomUUID(), + }); + const active = () => + fixture.db + .select() + .from(user) + .where(and(eq(user.role, "admin"), eq(user.banned, false))); + it("persists a successful creation audit in the same transaction", async () => { + const rows = await fixture.db + .select() + .from(adminAuditLog) + .where(eq(adminAuditLog.action, "user.create")); + expect(rows).toHaveLength(1); + expect(JSON.stringify(rows[0]?.details)).not.toContain("anchor-admin@example.test"); + }); + it("permits only one of two concurrent full admins to demote themselves", async () => { + const a = await actor("a"), + b = await actor("b"); + await fixture.db.update(user).set({ role: "user" }).where(eq(user.id, anchorId)); + const results = await Promise.all( + [a, b].map((person) => + app.inject({ + method: "POST", + url: "/api/auth/admin/set-role", + headers: headers(person.cookie), + payload: { userId: person.id, role: "user" }, + }), + ), + ); + expect(results.filter((r) => r.statusCode === 200)).toHaveLength(1); + expect(await active()).toHaveLength(1); + }); + it("rejects last-admin deletion before any journal request or cleanup", async () => { + const a = await actor("last"); + await fixture.db.update(user).set({ role: "user" }).where(eq(user.id, anchorId)); + const before = existsSync(journal) ? readFileSync(journal, "utf8") : ""; + const result = await app.inject({ + method: "POST", + url: "/api/auth/delete-user", + headers: headers(a.cookie), + payload: { password: "fixture-password-1234" }, + }); + expect((existsSync(journal) ? readFileSync(journal, "utf8") : "").length).toBe(before.length); + expect(result.statusCode).toBe(409); + expect(result.json()).toMatchObject({ code: "LAST_ADMIN_REQUIRED" }); + expect(existsSync(journal) ? readFileSync(journal, "utf8") : "").toBe(before); + expect(await active()).toHaveLength(1); + }); + it("backs remote guards with a transactional SQL invariant", async () => { + await expect( + fixture.db.update(user).set({ role: "user" }).where(eq(user.id, anchorId)), + ).rejects.toThrow(); + expect(await active()).toHaveLength(1); + }); + it.each(["role", "banned", "emailVerified", "twoFactorEnabled", "email"])( + "rejects generic admin updates of %s", + async (field) => { + const admin = await actor("writer"), + target = await actor("target", "user"); + const value = field === "role" ? "admin" : field === "email" ? "changed@example.test" : true; + const response = await app.inject({ + method: "POST", + url: "/api/auth/admin/update-user", + headers: headers(admin.cookie), + payload: { userId: target.id, data: { [field]: value } }, + }); + expect(response.statusCode).toBe(400); + }, + ); + it.each([ + ["ban", "ban"], + ["delete", "delete"], + ["role", "ban"], + ["delete", "role"], + ] as const)("serializes concurrent %s/%s authority loss", async (first, second) => { + const a = await actor("cross-a"), + b = await actor("cross-b"); + await fixture.db.update(user).set({ role: "user" }).where(eq(user.id, anchorId)); + const run = (who: typeof a, target: string, action: "ban" | "delete" | "role") => + app.inject({ + method: "POST", + url: `/api/auth/admin/${action === "ban" ? "ban-user" : action === "delete" ? "remove-user" : "set-role"}`, + headers: headers(who.cookie), + payload: { userId: target, ...(action === "role" ? { role: "user" } : {}) }, + }); + const results = await Promise.all([run(a, b.id, first), run(b, a.id, second)]); + expect(results.filter((r) => r.statusCode === 200)).toHaveLength(1); + expect(await active()).toHaveLength(1); + expect((await fixture.db.select().from(accountAdminGuard))[0]?.activeCount).toBe(1); + }); + it("rolls back role, sessions, counter and audit together on audit failure", async () => { + const admin = await actor("audit-admin"), + target = await actor("audit-target"); + const before = (await fixture.db.select().from(accountAdminGuard))[0]?.activeCount; + await fixture.db.execute( + sql`ALTER TABLE admin_audit_log ADD CONSTRAINT synthetic_role_audit_failure CHECK (action <> 'user.role.change') NOT VALID`, + ); + try { + const result = await app.inject({ + method: "POST", + url: "/api/auth/admin/set-role", + headers: headers(admin.cookie), + payload: { userId: target.id, role: "user" }, + }); + expect(result.statusCode).toBe(500); + expect((await fixture.db.select().from(user).where(eq(user.id, target.id)))[0]?.role).toBe( + "admin", + ); + expect( + await fixture.db.select().from(session).where(eq(session.userId, target.id)), + ).not.toHaveLength(0); + expect((await fixture.db.select().from(accountAdminGuard))[0]?.activeCount).toBe(before); + } finally { + await fixture.db.execute( + sql`ALTER TABLE admin_audit_log DROP CONSTRAINT synthetic_role_audit_failure`, + ); + } + }); + it("rolls back a ban when actual session revocation fails", async () => { + const admin = await actor("ban-admin"), + target = await actor("ban-target", "user"); + await fixture.db.execute( + sql`CREATE FUNCTION synthetic_deny_session_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic session deletion failure'; END $$`, + ); + await fixture.db.execute( + sql`CREATE TRIGGER synthetic_session_failure BEFORE DELETE ON session FOR EACH ROW EXECUTE FUNCTION synthetic_deny_session_delete()`, + ); + try { + const result = await app.inject({ + method: "POST", + url: "/api/auth/admin/ban-user", + headers: headers(admin.cookie), + payload: { userId: target.id }, + }); + expect(result.statusCode).toBe(500); + expect((await fixture.db.select().from(user).where(eq(user.id, target.id)))[0]?.banned).toBe( + false, + ); + expect( + await fixture.db.select().from(session).where(eq(session.userId, target.id)), + ).not.toHaveLength(0); + expect( + await fixture.db + .select() + .from(adminAuditLog) + .where(and(eq(adminAuditLog.action, "user.ban"), eq(adminAuditLog.targetId, target.id))), + ).toHaveLength(0); + } finally { + await fixture.db.execute(sql`DROP TRIGGER synthetic_session_failure ON session`); + await fixture.db.execute(sql`DROP FUNCTION synthetic_deny_session_delete()`); + } + }); + it("keeps admitted erasure requests but never marks completion after database rollback", async () => { + const admin = await actor("delete-admin"), + target = await actor("delete-target", "user"); + const key = (await import("@openmapx/core/erasure-journal")).readErasureJournalKeyFile( + join(directory, "journal.key"), + process.getuid?.() ?? 0, + ), + read = (await import("@openmapx/core/erasure-journal")).readErasureJournal; + const before = read(journal, key); + await fixture.db.execute( + sql`ALTER TABLE admin_audit_log ADD CONSTRAINT synthetic_delete_audit_failure CHECK (action <> 'user.delete') NOT VALID`, + ); + try { + const result = await app.inject({ + method: "POST", + url: "/api/auth/admin/remove-user", + headers: headers(admin.cookie), + payload: { userId: target.id }, + }); + expect(result.statusCode).toBe(500); + expect(await fixture.db.select().from(user).where(eq(user.id, target.id))).toHaveLength(1); + const after = read(journal, key); + expect(after.requests).toHaveLength(before.requests.length + 1); + expect(after.completedReceiptIds.size).toBe(before.completedReceiptIds.size); + } finally { + await fixture.db.execute( + sql`ALTER TABLE admin_audit_log DROP CONSTRAINT synthetic_delete_audit_failure`, + ); + } + }); + it("de-links account policy attribution when an admitted ordinary account is erased", async () => { + const admin = await actor("erase-admin"), + target = await actor("erase-policy"); + const { loadAuthPolicy, applyAuthPolicy } = await import("./auth-policy"), + before = await loadAuthPolicy(); + await applyAuthPolicy( + { + settings: { + emailVerificationRequired: true, + twoFaEnabled: true, + passwordMinLength: 8, + sessionDurationHours: 168, + }, + expectedRevision: before.revision, + confirmed: true, + idempotencyKey: "synthetic-erasure-policy-0001", + }, + target.id, + ); + const [ownedApplication] = await fixture.db + .select() + .from(authPolicyApplication) + .where(eq(authPolicyApplication.actorId, target.id)); + if (!ownedApplication) throw Error("Missing original policy application"); + const { streamOpenMapxRegistrationRecords } = await import("../privacy/openmapx-collectors"); + const records = []; + for await (const row of streamOpenMapxRegistrationRecords("admin-audit-attribution", { + userId: target.id, + cutoffAt: new Date(), + database: fixture.db, + })) + records.push(row); + expect(records.some((row) => row.data.action === "auth.policy.current-owner")).toBe(true); + await fixture.db.update(user).set({ role: "user" }).where(eq(user.id, target.id)); + const result = await app.inject({ + method: "POST", + url: "/api/auth/admin/remove-user", + headers: headers(admin.cookie), + payload: { userId: target.id }, + }); + expect(result.statusCode, result.body).toBe(200); + expect((await fixture.db.select().from(authPolicyState))[0]?.activatedBy).toBeNull(); + expect( + ( + await fixture.db + .select() + .from(authPolicyApplication) + .where(eq(authPolicyApplication.key, ownedApplication.key)) + )[0]?.actorId, + ).toBeNull(); + }); + it("shares one role audit and version check across server API and the legacy privacy route", async () => { + const admin = await actor("role-admin"), + target = await actor("role-target", "user"); + const requestHeaders = new Headers(headers(admin.cookie)); + requestHeaders.set("x-openmapx-expected-role", "user"); + await auth.api.setRole({ headers: requestHeaders, body: { userId: target.id, role: "admin" } }); + await expect( + auth.api.setRole({ headers: requestHeaders, body: { userId: target.id, role: "user" } }), + ).rejects.toMatchObject({ statusCode: 409, body: { code: "ROLE_VERSION_CONFLICT" } }); + const own = await fixture.db + .select() + .from(adminAuditLog) + .where( + and(eq(adminAuditLog.targetId, target.id), eq(adminAuditLog.action, "user.role.change")), + ); + expect(own).toHaveLength(1); + expect(own[0]?.details).toMatchObject({ previousRole: "user", role: "admin" }); + expect( + await fixture.db.select().from(session).where(eq(session.userId, target.id)), + ).toHaveLength(0); + const next = await actor("privacy-target", "user"), + privacy = Fastify(); + await privacy.register((await import("../routes/privacy-roles")).privacyRolesRoute, { + prefix: "/api", + database: fixture.db, + }); + await privacy.ready(); + try { + const changed = await privacy.inject({ + method: "POST", + url: `/api/privacy/admin/roles/${next.id}/grant`, + headers: headers(admin.cookie), + payload: {}, + }); + expect(changed.statusCode, changed.body).toBe(200); + expect( + await fixture.db + .select() + .from(adminAuditLog) + .where( + and(eq(adminAuditLog.targetId, next.id), eq(adminAuditLog.action, "user.role.change")), + ), + ).toHaveLength(1); + const denied = await privacy.inject({ + method: "POST", + url: `/api/privacy/admin/roles/${target.id}/grant`, + headers: headers(admin.cookie), + payload: {}, + }); + expect(denied.statusCode).toBe(409); + } finally { + await privacy.close(); + } + }); + it("preserves ordinary self-service deletion and commits its completion only after deletion", async () => { + const target = await actor("self-service", "user"), + before = (await fixture.db.select().from(accountAdminGuard))[0]?.activeCount; + const result = await app.inject({ + method: "POST", + url: "/api/auth/delete-user", + headers: headers(target.cookie), + payload: { password: "fixture-password-1234" }, + }); + expect(result.statusCode, result.body).toBe(200); + expect(await fixture.db.select().from(user).where(eq(user.id, target.id))).toHaveLength(0); + expect((await fixture.db.select().from(accountAdminGuard))[0]?.activeCount).toBe(before); + }); + it("protects the last administrator through the installed deletion callback and its stored token", async () => { + const target = await actor("callback-admin"); + await fixture.db.update(user).set({ role: "user" }).where(eq(user.id, anchorId)); + const token = randomBytes(24).toString("hex"); + await (await auth.$context).internalAdapter.createVerificationValue({ + identifier: `delete-account-${token}`, + value: target.id, + expiresAt: new Date(Date.now() + 3600000), + }); + const before = readFileSync(journal, "utf8"); + const result = await app.inject({ + method: "GET", + url: `/api/auth/delete-user/callback?token=${token}`, + headers: headers(target.cookie), + }); + expect(result.statusCode, result.body).toBe(409); + expect(readFileSync(journal, "utf8")).toBe(before); + expect(await active()).toHaveLength(1); + }); + it("does not publish erasure completion when its owning outer account transaction rolls back", async () => { + const admin = await actor("outer-admin"), + target = await actor("outer-target", "user"); + const { readErasureJournal, readErasureJournalKeyFile } = await import( + "@openmapx/core/erasure-journal" + ), + key = readErasureJournalKeyFile(join(directory, "journal.key"), process.getuid?.() ?? 0), + before = readErasureJournal(journal, key); + const { withAccountMutation } = await import("./account-mutations"); + await expect( + withAccountMutation( + { actorId: admin.id, targetId: target.id, action: "delete", mutationKey: randomUUID() }, + async () => { + await auth.api.removeUser({ + headers: new Headers(headers(admin.cookie)), + body: { userId: target.id }, + }); + throw Error("synthetic outer transaction rollback"); + }, + ), + ).rejects.toThrow("synthetic outer transaction rollback"); + expect(await fixture.db.select().from(user).where(eq(user.id, target.id))).toHaveLength(1); + const after = readErasureJournal(journal, key); + expect(after.requests).toHaveLength(before.requests.length + 1); + expect(after.completedReceiptIds.size).toBe(before.completedReceiptIds.size); + }); +}); diff --git a/apps/api/src/account/admin-invariant.ts b/apps/api/src/account/admin-invariant.ts new file mode 100644 index 000000000..432051b35 --- /dev/null +++ b/apps/api/src/account/admin-invariant.ts @@ -0,0 +1,56 @@ +import { APIError } from "better-auth/api"; +import { eq } from "drizzle-orm"; +import { accountAdminGuard, user } from "../db/schema"; +import { currentAccountMutation } from "./account-context"; +import { type AuthDatabase, currentAuthDatabase } from "./auth-database"; +export function isActiveFullAdmin(value: { role: string | null; banned: boolean | null }) { + return value.role === "admin" && value.banned !== true; +} +export function lastAdminError() { + return new APIError("CONFLICT", { + code: "LAST_ADMIN_REQUIRED", + message: "At least one active full administrator is required", + }); +} +export async function lockAdminGuard(database: AuthDatabase) { + const [guard] = await database + .select() + .from(accountAdminGuard) + .where(eq(accountAdminGuard.id, 1)) + .for("update"); + if (!guard) throw Error("Account administration guard unavailable"); + return guard; +} +export async function assertAdminCapacity( + database: AuthDatabase, + targetId: string, + next: { role?: string; banned?: boolean; deleted?: boolean }, +) { + const guard = await lockAdminGuard(database), + [target] = await database.select().from(user).where(eq(user.id, targetId)).for("update"); + if (!target) + throw new APIError("NOT_FOUND", { code: "USER_NOT_FOUND", message: "Account not found" }); + const losing = + isActiveFullAdmin(target) && + (next.deleted === true || !isActiveFullAdmin({ ...target, ...next })); + if (losing && guard.activeCount <= 1) throw lastAdminError(); + return target; +} +export async function assertAccountDeletionAllowed(targetId: string) { + const context = currentAccountMutation(); + if (!context || context.action !== "delete" || context.database !== currentAuthDatabase()) + throw Error("Account deletion transaction required"); + if (context.targetId !== null && context.targetId !== targetId) + throw Error("Account deletion target changed"); + await assertAdminCapacity(context.database, targetId, { deleted: true }); +} +export function mapAccountConstraint(error: unknown): never { + let current = error; + for (let i = 0; i < 8 && current && typeof current === "object"; i++) { + const item = current as { constraint_name?: unknown; constraint?: unknown; cause?: unknown }; + if (item.constraint_name === "LAST_ADMIN_REQUIRED" || item.constraint === "LAST_ADMIN_REQUIRED") + throw lastAdminError(); + current = item.cause; + } + throw error; +} diff --git a/apps/api/src/account/auth-account-entry.ts b/apps/api/src/account/auth-account-entry.ts new file mode 100644 index 000000000..3e473265f --- /dev/null +++ b/apps/api/src/account/auth-account-entry.ts @@ -0,0 +1,109 @@ +import { randomUUID } from "node:crypto"; +import { APIError } from "better-auth/api"; +import { + type AccountMutationIntent, + accountRoleSchema, + withAccountMutation, + writeAuthAccountRole, +} from "./account-mutations"; + +const paths: Record< + string, + { action: AccountMutationIntent["action"]; callback?: boolean; selfService?: boolean } +> = { + createUser: { action: "create" }, + setRole: { action: "role" }, + banUser: { action: "ban" }, + unbanUser: { action: "unban" }, + removeUser: { action: "delete" }, + adminUpdateUser: { action: "profile" }, + deleteUser: { action: "delete", selfService: true }, + deleteUserCallback: { action: "delete", callback: true, selfService: true }, + "/api/auth/admin/create-user": { action: "create" }, + "/api/auth/admin/set-role": { action: "role" }, + "/api/auth/admin/ban-user": { action: "ban" }, + "/api/auth/admin/unban-user": { action: "unban" }, + "/api/auth/admin/remove-user": { action: "delete" }, + "/api/auth/admin/update-user": { action: "profile" }, + "/api/auth/delete-user": { action: "delete", selfService: true }, + "/api/auth/delete-user/callback": { action: "delete", callback: true, selfService: true }, +}; +export async function accountAuthEntry( + input: { entry: string; body: unknown; headers?: HeadersInit; internal: boolean }, + readSession: () => Promise, + run: () => Promise, +): Promise { + const descriptor = paths[input.entry]; + if (!descriptor) return run(); + const body = + input.body && typeof input.body === "object" ? (input.body as Record) : {}, + headers = new Headers(input.headers); + const mutationKey = headers.get("idempotency-key") ?? randomUUID(); + if (descriptor.action === "profile") { + const data = body.data; + if ( + !data || + typeof data !== "object" || + Object.keys(data).some((k) => k !== "name" && k !== "image") + ) + throw new APIError("BAD_REQUEST", { + code: "USE_ACCOUNT_ACTION", + message: "Use the dedicated account action for protected account fields", + }); + } + let role: import("./account-mutations").AccountRole | undefined; + if (descriptor.action === "role" || descriptor.action === "create") { + const nested = + body.data && typeof body.data === "object" + ? (body.data as Record).role + : undefined; + const candidate = body.role ?? nested ?? "user", + parsed = accountRoleSchema.safeParse(candidate); + if ( + !parsed.success || + (nested !== undefined && body.role !== undefined && nested !== body.role) + ) + throw new APIError("BAD_REQUEST", { + code: "INVALID_ROLE", + message: "Select one supported account role", + }); + role = parsed.data; + } + const raw = await readSession(), + actor = + raw && typeof raw === "object" ? (raw as { user?: { id?: unknown } }).user?.id : undefined, + actorId = typeof actor === "string" ? actor : null; + const targetId = + typeof body.userId === "string" ? body.userId : descriptor.action === "delete" ? actorId : null; + if (descriptor.action === "role") { + if (!actorId || !targetId || !role) + throw new APIError("UNAUTHORIZED", { + code: "UNAUTHORIZED", + message: "Authentication required", + }); + const expected = headers.get("x-openmapx-expected-role"); + if (expected !== null && !accountRoleSchema.safeParse(expected).success) + throw new APIError("BAD_REQUEST", { code: "INVALID_ROLE", message: "Invalid expected role" }); + return writeAuthAccountRole( + { + actorId, + userId: targetId, + role, + idempotencyKey: mutationKey, + ...(expected ? { expectedRole: accountRoleSchema.parse(expected) } : {}), + }, + run, + ); + } + return withAccountMutation( + { + actorId, + targetId, + action: descriptor.action, + mutationKey, + internalCreate: input.internal, + selfServiceDelete: descriptor.selfService, + }, + run, + ); +} diff --git a/apps/api/src/account/auth-policy-postgres.test.ts b/apps/api/src/account/auth-policy-postgres.test.ts index d7f75dda7..06517745e 100644 --- a/apps/api/src/account/auth-policy-postgres.test.ts +++ b/apps/api/src/account/auth-policy-postgres.test.ts @@ -120,6 +120,14 @@ dbDescribe("account policy through installed Better Auth", () => { pending: { passwordMinLength: 40 }, ownership: { twoFaEnabled: "default" }, }); + await auth.api.createUser({ + body: { + name: "Preserved admin", + email: "preserved-admin@example.test", + password: "original-password", + role: "admin", + }, + }); await fixture.db .update(user) .set({ role: "privacy_admin" }) @@ -220,12 +228,17 @@ dbDescribe("account policy through installed Better Auth", () => { settings, expectedRevision: initial.revision, confirmed: true, - idempotencyKey: "policy-activation-synthetic-0001", + idempotencyKey: `policy-activation-${admin.user.id}-0001`, }; const applied = await applyAuthPolicy(request, admin.user.id); release(); expect(await oldResult).toBe("verification required"); expect(await applyAuthPolicy(request, admin.user.id)).toEqual(applied); + expect( + (await fixture.db.select().from(authPolicyApplication)).every( + (row) => !row.key.includes(admin.user.id), + ), + ).toBe(true); await expect( applyAuthPolicy( { @@ -368,6 +381,7 @@ dbDescribe("account policy through installed Better Auth", () => { .from(user) .where(eq(user.email, "policy-admin@example.test")); if (!admin) throw Error("Missing admin fixture"); + const applications = await fixture.db.select().from(authPolicyApplication); const before = await loadAuthPolicy(), rows = await fixture.db.select().from(authPolicyState), drafts = await fixture.db.select().from(systemSettings); @@ -394,12 +408,7 @@ dbDescribe("account policy through installed Better Auth", () => { expect(await loadAuthPolicy()).toEqual(before); expect(await fixture.db.select().from(authPolicyState)).toEqual(rows); expect(await fixture.db.select().from(systemSettings)).toEqual(drafts); - expect( - await fixture.db - .select() - .from(authPolicyApplication) - .where(eq(authPolicyApplication.key, "rollback-policy-synthetic-0001")), - ).toHaveLength(0); + expect(await fixture.db.select().from(authPolicyApplication)).toEqual(applications); } finally { await fixture.db.execute( sql`ALTER TABLE admin_audit_log DROP CONSTRAINT synthetic_policy_audit_failure`, diff --git a/apps/api/src/account/auth-policy.ts b/apps/api/src/account/auth-policy.ts index b6227fc9b..390de085a 100644 --- a/apps/api/src/account/auth-policy.ts +++ b/apps/api/src/account/auth-policy.ts @@ -116,7 +116,8 @@ export async function applyAuthPolicy( const parsed = authPolicyApplySchema.safeParse(input); if (!parsed.success) throw new AuthPolicyError("AUTH_POLICY_INVALID"); const request = parsed.data, - intentDigest = digest(request); + intentDigest = digest(request), + applicationKey = digest([actorId, request.idempotencyKey]); return database.transaction(async (tx) => { await tx.execute(sql`SELECT pg_advisory_xact_lock(163578142, 1)`); const [actor] = await tx @@ -128,7 +129,7 @@ export async function applyAuthPolicy( const [prior] = await tx .select() .from(authPolicyApplication) - .where(eq(authPolicyApplication.key, request.idempotencyKey)); + .where(eq(authPolicyApplication.key, applicationKey)); if (prior) { if (prior.actorId !== actorId || prior.intentDigest !== intentDigest) throw new AuthPolicyError("AUTH_POLICY_CONFLICT"); @@ -177,7 +178,7 @@ export async function applyAuthPolicy( }); await tx .insert(authPolicyApplication) - .values({ key: request.idempotencyKey, actorId, intentDigest, snapshot: result }); + .values({ key: applicationKey, actorId, intentDigest, snapshot: result }); return result; }); } diff --git a/apps/api/src/account/auth-runtime.ts b/apps/api/src/account/auth-runtime.ts index ae8d6061c..771358ad7 100644 --- a/apps/api/src/account/auth-runtime.ts +++ b/apps/api/src/account/auth-runtime.ts @@ -1,6 +1,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import type { AuthPolicySnapshot } from "@openmapx/core/api-contracts"; import { isAPIError } from "better-auth/api"; +import { accountAuthEntry } from "./auth-account-entry"; import { type AuthDatabase, currentAuthDatabase, @@ -87,11 +88,45 @@ export function createAuthRuntime< if (typeof value !== "function") return value; return Object.assign( (...args: unknown[]) => - entry(key === "resetPasswordEmailOTP", () => - withAuthOperation(async (instance) => - Reflect.apply(Reflect.get(instance.api, key), instance.api, args), - ), - ), + (async () => { + const input = (args[0] ?? {}) as { + body?: unknown; + headers?: HeadersInit; + request?: Request; + asResponse?: boolean; + }; + try { + return await withAuthOperation(() => + accountAuthEntry( + { + entry: String(key), + body: input.body, + headers: input.headers, + internal: !input.headers && !input.request, + }, + () => + withAuthOperation(async (instance) => + Reflect.apply(Reflect.get(instance.api, "getSession"), instance.api, [ + { + headers: input.headers ?? new Headers(), + query: { disableCookieCache: true, disableRefresh: true }, + }, + ]), + ), + () => + entry(key === "resetPasswordEmailOTP", () => + withAuthOperation(async (instance) => + Reflect.apply(Reflect.get(instance.api, key), instance.api, args), + ), + ), + ), + ); + } catch (error) { + if (input.asResponse && isAPIError(error)) + return Response.json(error.body, { status: error.statusCode }); + throw error; + } + })(), value, ); }, @@ -102,11 +137,34 @@ export function createAuthRuntime< if (key === "handler") return async (request: Request) => { try { - return await entry( - new URL(request.url).pathname === "/api/auth/email-otp/reset-password", - () => withAuthOperation((instance) => instance.handler(request)), + const path = new URL(request.url).pathname; + const body = + request.method === "POST" + ? await request + .clone() + .json() + .catch(() => null) + : null; + return await withAuthOperation(() => + accountAuthEntry( + { entry: path, body, headers: request.headers, internal: false }, + () => + withAuthOperation(async (instance) => + Reflect.apply(Reflect.get(instance.api, "getSession"), instance.api, [ + { + headers: request.headers, + query: { disableCookieCache: true, disableRefresh: true }, + }, + ]), + ), + () => + entry(path === "/api/auth/email-otp/reset-password", () => + withAuthOperation((instance) => instance.handler(request)), + ), + ), ); } catch (error) { + if (isAPIError(error)) return Response.json(error.body, { status: error.statusCode }); if (error instanceof AuthPolicyError) return Response.json( { code: error.code, message: "Authentication policy is unavailable" }, diff --git a/apps/api/src/account/password-verification-policy.ts b/apps/api/src/account/password-verification-policy.ts index 1c03875b7..9cc475dae 100644 --- a/apps/api/src/account/password-verification-policy.ts +++ b/apps/api/src/account/password-verification-policy.ts @@ -1,13 +1,21 @@ import type { AuthPolicySnapshot } from "@openmapx/core/api-contracts"; import type { BetterAuthOptions } from "better-auth"; import { APIError, createAuthMiddleware } from "better-auth/api"; +import { accountRoleSchema } from "./account-mutations"; export function applyPasswordVerificationPolicy(policy: AuthPolicySnapshot) { const userHooks: NonNullable["user"]> = { create: { - before: async (user, context) => - context?.path === "/admin/create-user" + before: async (user, context) => { + const role = (user as typeof user & { role?: unknown }).role; + if (role !== undefined && !accountRoleSchema.safeParse(role).success) + throw new APIError("BAD_REQUEST", { + code: "INVALID_ROLE", + message: "Select one supported account role", + }); + return context?.path === "/admin/create-user" ? { data: { ...user, emailVerified: false } } - : undefined, + : undefined; + }, }, update: { before: async (patch, context) => { diff --git a/apps/api/src/auth.ts b/apps/api/src/auth.ts index 41b4a5aae..be71997e5 100644 --- a/apps/api/src/auth.ts +++ b/apps/api/src/auth.ts @@ -16,7 +16,7 @@ import { } from "better-auth/plugins"; import { emailHarmony } from "better-auth-harmony"; import { eq } from "drizzle-orm"; -import type { AuthDatabase } from "./account/auth-database"; +import { type AuthDatabase, currentAuthDatabase } from "./account/auth-database"; import { createAuthRuntime } from "./account/auth-runtime"; import { applyPasswordVerificationPolicy } from "./account/password-verification-policy"; import { @@ -25,7 +25,6 @@ import { fetchOsmUserDetails, } from "./auth-provider-http"; import { resolveBetterAuthSecret } from "./auth-secret"; -import { db } from "./db"; import { user as userTable } from "./db/schema"; import { managedOAuthProviderOptions } from "./managed-oauth-provider"; import { recordDataDisclosureBestEffort } from "./privacy/disclosures.js"; @@ -84,14 +83,14 @@ const providerAvatarSync = createProviderAvatarSync({ }, fetchProviderImage, async getCurrentImage(userId) { - const [currentUser] = await db + const [currentUser] = await currentAuthDatabase() .select({ image: userTable.image }) .from(userTable) .where(eq(userTable.id, userId)); return currentUser?.image; }, async setUserImage(userId, image) { - await db.update(userTable).set({ image }).where(eq(userTable.id, userId)); + await currentAuthDatabase().update(userTable).set({ image }).where(eq(userTable.id, userId)); }, }); @@ -170,7 +169,7 @@ function createPolicyAuth( // This hook runs after Better Auth has created the usable session. It // intentionally records nothing for unrecognised endpoints. after: async (createdSession, context) => { - await recordSessionFromAuthContext(createdSession, context, db); + await recordSessionFromAuthContext(createdSession, context, currentAuthDatabase()); }, }, }, diff --git a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts index d402876da..613a3dc6a 100644 --- a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts +++ b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts @@ -49,6 +49,8 @@ describe.skipIf(skipDatabase)("user erasure constraints with PostgreSQL", () => "account.user_id:c", "admin_audit_log.actor_id:n", "admin_job.created_by:n", + "auth_policy_application.actor_id:n", + "auth_policy_state.activated_by:n", "data_disclosure_event.user_id:n", "data_export_reauthentication.initiating_admin_user_id:n", "data_export_reauthentication.user_id:n", diff --git a/apps/api/src/db/account-administration-schema.ts b/apps/api/src/db/account-administration-schema.ts index c253c80eb..a873ae576 100644 --- a/apps/api/src/db/account-administration-schema.ts +++ b/apps/api/src/db/account-administration-schema.ts @@ -1,6 +1,7 @@ import type { AuthPolicySettings, AuthPolicySnapshot } from "@openmapx/core/api-contracts"; import { sql } from "drizzle-orm"; import { check, integer, jsonb, pgTable, text, timestamp } from "drizzle-orm/pg-core"; +import { user } from "./auth-schema"; export const authPolicyState = pgTable( "auth_policy_state", { @@ -8,15 +9,24 @@ export const authPolicyState = pgTable( activationVersion: integer("activation_version").notNull(), settings: jsonb("settings").$type().notNull(), revision: text("revision").notNull(), - activatedBy: text("activated_by").notNull(), + activatedBy: text("activated_by").references(() => user.id, { onDelete: "set null" }), activatedAt: timestamp("activated_at", { withTimezone: true }).defaultNow().notNull(), }, (t) => [check("auth_policy_singleton", sql`${t.id}=1 AND ${t.activationVersion}=1`)], ); export const authPolicyApplication = pgTable("auth_policy_application", { key: text("key").primaryKey(), - actorId: text("actor_id").notNull(), + actorId: text("actor_id").references(() => user.id, { onDelete: "set null" }), intentDigest: text("intent_digest").notNull(), snapshot: jsonb("snapshot").$type().notNull(), createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), }); + +export const accountAdminGuard = pgTable( + "account_admin_guard", + { id: integer("id").primaryKey(), activeCount: integer("active_count").notNull() }, + (t) => [ + check("account_admin_guard_singleton", sql`${t.id}=1`), + check("account_admin_guard_nonnegative", sql`${t.activeCount}>=0`), + ], +); diff --git a/apps/api/src/db/migrations/0044_admin_account_invariant.sql b/apps/api/src/db/migrations/0044_admin_account_invariant.sql new file mode 100644 index 000000000..52aca547b --- /dev/null +++ b/apps/api/src/db/migrations/0044_admin_account_invariant.sql @@ -0,0 +1,47 @@ +CREATE TABLE "account_admin_guard" ( + "id" integer PRIMARY KEY NOT NULL, + "active_count" integer NOT NULL, + CONSTRAINT "account_admin_guard_singleton" CHECK ("account_admin_guard"."id"=1), + CONSTRAINT "account_admin_guard_nonnegative" CHECK ("account_admin_guard"."active_count">=0) +); +--> statement-breakpoint +ALTER TABLE "auth_policy_application" ALTER COLUMN "actor_id" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "auth_policy_state" ALTER COLUMN "activated_by" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "auth_policy_application" ADD CONSTRAINT "auth_policy_application_actor_id_user_id_fk" FOREIGN KEY ("actor_id") REFERENCES "public"."user"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "auth_policy_state" ADD CONSTRAINT "auth_policy_state_activated_by_user_id_fk" FOREIGN KEY ("activated_by") REFERENCES "public"."user"("id") ON DELETE set null ON UPDATE no action; +--> statement-breakpoint +-- Drizzle emits the schema above; this custom SQL installs the transactional invariant. +LOCK TABLE "user" IN SHARE ROW EXCLUSIVE MODE; +--> statement-breakpoint +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM "user" u CROSS JOIN LATERAL unnest(string_to_array(coalesce(u.role,'user'),',')) token WHERE btrim(token) NOT IN ('user','admin','privacy_admin')) THEN + RAISE EXCEPTION 'Unrecognized historical account role'; + END IF; +END $$; +--> statement-breakpoint +WITH normalized AS ( + SELECT id, ARRAY(SELECT btrim(token) FROM unnest(string_to_array(coalesce(role,'user'),',')) token) AS roles FROM "user" +) +UPDATE "user" u SET role=CASE WHEN 'admin'=ANY(n.roles) THEN 'admin' WHEN 'privacy_admin'=ANY(n.roles) THEN 'privacy_admin' ELSE 'user' END +FROM normalized n WHERE u.id=n.id; +--> statement-breakpoint +ALTER TABLE "user" ADD CONSTRAINT openmapx_canonical_account_role CHECK (role IS NULL OR role IN ('user','admin','privacy_admin')); +--> statement-breakpoint +INSERT INTO account_admin_guard(id,active_count) SELECT 1,count(*)::integer FROM "user" WHERE role='admin' AND banned IS NOT TRUE; +--> statement-breakpoint +CREATE FUNCTION openmapx_preserve_admin_capacity() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE old_active integer:=0; new_active integer:=0; delta integer; +BEGIN + IF TG_OP <> 'INSERT' THEN old_active:=CASE WHEN OLD.role='admin' AND OLD.banned IS NOT TRUE THEN 1 ELSE 0 END; END IF; + IF TG_OP <> 'DELETE' THEN new_active:=CASE WHEN NEW.role='admin' AND NEW.banned IS NOT TRUE THEN 1 ELSE 0 END; END IF; + delta:=new_active-old_active; + IF delta <> 0 THEN + UPDATE account_admin_guard SET active_count=active_count+delta WHERE id=1 AND (delta>0 OR active_count+delta>0); + IF NOT FOUND THEN RAISE EXCEPTION USING ERRCODE='23514',CONSTRAINT='LAST_ADMIN_REQUIRED',MESSAGE='LAST_ADMIN_REQUIRED'; END IF; + END IF; + IF TG_OP='DELETE' THEN RETURN OLD; END IF; + RETURN NEW; +END $$; +--> statement-breakpoint +CREATE TRIGGER openmapx_admin_capacity BEFORE INSERT OR UPDATE OF role,banned OR DELETE ON "user" FOR EACH ROW EXECUTE FUNCTION openmapx_preserve_admin_capacity(); diff --git a/apps/api/src/db/migrations/meta/0044_snapshot.json b/apps/api/src/db/migrations/meta/0044_snapshot.json new file mode 100644 index 000000000..bc31a9e04 --- /dev/null +++ b/apps/api/src/db/migrations/meta/0044_snapshot.json @@ -0,0 +1,8283 @@ +{ + "id": "e1901a3d-3001-4fc1-b4a9-1f0c8e34b932", + "prevId": "d549466e-6409-4b63-9307-604353bbbf8c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_admin_guard": { + "name": "account_admin_guard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "active_count": { + "name": "active_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_admin_guard_singleton": { + "name": "account_admin_guard_singleton", + "value": "\"account_admin_guard\".\"id\"=1" + }, + "account_admin_guard_nonnegative": { + "name": "account_admin_guard_nonnegative", + "value": "\"account_admin_guard\".\"active_count\">=0" + } + }, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_application_actor_id_user_id_fk": { + "name": "auth_policy_application_actor_id_user_id_fk", + "tableFrom": "auth_policy_application", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_state_activated_by_user_id_fk": { + "name": "auth_policy_state_activated_by_user_id_fk", + "tableFrom": "auth_policy_state", + "tableTo": "user", + "columnsFrom": [ + "activated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 371b3cccc..d51695fe0 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -309,6 +309,13 @@ "when": 1793187169774, "tag": "0043_admin_auth_policy", "breakpoints": true + }, + { + "idx": 44, + "version": "7", + "when": 1793189162695, + "tag": "0044_admin_account_invariant", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 77655c2ac..0aedbd7bd 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -1,4 +1,8 @@ -export { authPolicyApplication, authPolicyState } from "./account-administration-schema"; +export { + accountAdminGuard, + authPolicyApplication, + authPolicyState, +} from "./account-administration-schema"; // Capability bindings — explicit (integration, capability) → service mappings // data-manager pipeline observability (Postgres schema "data_manager") diff --git a/apps/api/src/privacy/catalogue.ts b/apps/api/src/privacy/catalogue.ts index 952ea7d2b..ade2ef834 100644 --- a/apps/api/src/privacy/catalogue.ts +++ b/apps/api/src/privacy/catalogue.ts @@ -335,7 +335,7 @@ const registrationDefinitions: SubjectDataRegistration[] = [ }, { id: "admin-audit-attribution", - version: 1, + version: 2, category: "administrative-accountability", source: "openmapx-db", strategy: "collector", @@ -343,6 +343,8 @@ const registrationDefinitions: SubjectDataRegistration[] = [ locators: [ { type: "actor_id", field: "admin_audit_log.actor_id" }, { type: "target_id", field: "admin_audit_log.target_id" }, + { type: "actor_id", field: "auth_policy_application.actor_id" }, + { type: "actor_id", field: "auth_policy_state.activated_by" }, ], purposes: ["legal-accountability", "security-and-abuse-prevention"], legalBases: ["legal-obligation", "legitimate-interest"], diff --git a/apps/api/src/privacy/openmapx-collectors.ts b/apps/api/src/privacy/openmapx-collectors.ts index fbf9c1cff..6b4caad4e 100644 --- a/apps/api/src/privacy/openmapx-collectors.ts +++ b/apps/api/src/privacy/openmapx-collectors.ts @@ -4,6 +4,8 @@ import { account, adminAuditLog, adminJob, + authPolicyApplication, + authPolicyState, dataDisclosureEvent, dataExportReauthentication, dataManagerOfflinePackageArtifactReferences, @@ -761,6 +763,51 @@ export async function* streamOpenMapxRegistrationRecords( }), }, ); + yield* emit( + database + .select({ id: authPolicyApplication.key, createdAt: authPolicyApplication.createdAt }) + .from(authPolicyApplication) + .where( + and( + eq(authPolicyApplication.actorId, context.userId), + lte(authPolicyApplication.createdAt, cutoff), + ), + ) + .orderBy(asc(authPolicyApplication.key)), + { + map: (row) => ({ + recordId: `policy-application-${pseudonymousIdentifier(String(row.id))}`, + actor: "subject", + action: "auth.policy.apply", + createdAt: row.createdAt, + }), + }, + ); + yield* emit( + database + .select({ + id: authPolicyState.id, + revision: authPolicyState.revision, + createdAt: authPolicyState.activatedAt, + }) + .from(authPolicyState) + .where( + and( + eq(authPolicyState.activatedBy, context.userId), + lte(authPolicyState.activatedAt, cutoff), + ), + ) + .orderBy(asc(authPolicyState.id)), + { + map: (row) => ({ + recordId: `policy-owner-${pseudonymousIdentifier(String(row.id))}`, + actor: "subject", + action: "auth.policy.current-owner", + revision: row.revision, + createdAt: row.createdAt, + }), + }, + ); return; case "admin-jobs-attribution": yield* emit( diff --git a/apps/api/src/privacy/session-assurance.ts b/apps/api/src/privacy/session-assurance.ts index 1831892db..2b7e9e81a 100644 --- a/apps/api/src/privacy/session-assurance.ts +++ b/apps/api/src/privacy/session-assurance.ts @@ -34,7 +34,7 @@ export interface SessionAssuranceDatabase { } export async function recordSessionAuthAssurance(input: { - database?: typeof defaultDb; + database?: Pick; sessionId: string; userId: string; method: SessionAuthMethod; @@ -82,7 +82,7 @@ export async function getSessionAuthAssurance( export async function recordSessionFromAuthContext( session: { id: string; userId: string; createdAt?: Date }, context: { path?: string; body?: unknown } | null | undefined, - database: typeof defaultDb = defaultDb, + database: Pick = defaultDb, ): Promise { const method = assuranceMethodForAuthPath(context?.path); if (!method) return false; diff --git a/apps/api/src/routes/privacy-roles.ts b/apps/api/src/routes/privacy-roles.ts index 7ae19827a..e726424ab 100644 --- a/apps/api/src/routes/privacy-roles.ts +++ b/apps/api/src/routes/privacy-roles.ts @@ -1,9 +1,10 @@ -import { createHash, randomUUID } from "node:crypto"; -import { and, eq, isNull, or } from "drizzle-orm"; +import { isAPIError } from "better-auth/api"; +import { eq } from "drizzle-orm"; import type { FastifyPluginAsync } from "fastify"; import z from "zod/v4"; +import { accountRoleSchema, changeAccountRole } from "../account/account-mutations"; import { db as defaultDb } from "../db/index.js"; -import { adminAuditLog, session, user } from "../db/schema.js"; +import { user } from "../db/schema.js"; import { getAdminSession, requireAdmin } from "../utils/require-admin.js"; import { declareRouteAuth } from "../utils/route-auth.js"; @@ -79,40 +80,26 @@ export const privacyRolesRoute: FastifyPluginAsync = a if (target.role === "admin") return reply.status(409).send({ code: "FULL_ADMIN_PROTECTED" }); if ((target.role ?? "user") === desired) return reply.send({ userId: target.id, role: desired, changed: false }); - const previousRole = target.role ?? "user"; - await database.transaction(async (tx) => { - const updated = await tx - .update(user) - .set({ role: desired, updatedAt: new Date() }) - .where( - and( - eq(user.id, targetId), - or(isNull(user.role), eq(user.role, "user"), eq(user.role, "privacy_admin")), - ), - ) - .returning({ id: user.id }); - if (!updated[0]) throw new Error("ROLE_VERSION_CONFLICT"); - // Role changes invalidate all existing sessions (and their recorded - // authentication assurance) before the new least-privilege session is - // used. This prevents a stale session from retaining old authority. - await tx.delete(session).where(eq(session.userId, targetId)); - await tx.insert(adminAuditLog).values({ - id: randomUUID(), - actorId: actor.user.id, - targetId, - targetType: "user", - action: desired === "privacy_admin" ? "privacy.role.grant" : "privacy.role.revoke", - details: { - role: desired, - previousRole, - mutationDigest: createHash("sha256").update(key).digest("hex").slice(0, 32), - }, - ipAddress: null, - userAgent: "privacy-role-admin", - createdAt: new Date(), - }); - }); - return reply.send({ userId: target.id, role: desired, changed: true }); + try { + return reply.send( + await changeAccountRole( + { + actorId: actor.user.id, + userId: target.id, + role: desired, + expectedRole: accountRoleSchema.parse(target.role ?? "user"), + idempotencyKey: key, + }, + database, + ), + ); + } catch (error) { + if (isAPIError(error)) + return reply + .status(error.statusCode) + .send({ code: error.body?.code ?? "ROLE_MUTATION_FAILED" }); + throw error; + } } fastify.post<{ Params: { userId: string }; Body: unknown }>( diff --git a/apps/api/src/services/user-erasure.ts b/apps/api/src/services/user-erasure.ts index 0bb6c4361..75534501b 100644 --- a/apps/api/src/services/user-erasure.ts +++ b/apps/api/src/services/user-erasure.ts @@ -10,7 +10,9 @@ import { readErasureJournalKeyFile, } from "@openmapx/core/erasure-journal"; import { and, eq, inArray, sql } from "drizzle-orm"; -import { db } from "../db"; +import { currentAccountMutation } from "../account/account-context"; +import { assertAccountDeletionAllowed } from "../account/admin-invariant"; +import { currentAuthDatabase } from "../account/auth-database"; import { dataExportArtifact, dataSubjectRequest, @@ -99,7 +101,7 @@ export async function cleanupResidualUserData(user: ErasureUser): Promise // is still available. A missing store/object is idempotent, but an unsafe // path or failed delete aborts account deletion so we never report an // erasure while a deliverable archive remains readable. - const requests = await db + const requests = await currentAuthDatabase() .select({ id: dataSubjectRequest.id, state: dataSubjectRequest.state }) .from(dataSubjectRequest) .where(eq(dataSubjectRequest.userId, user.id)); @@ -112,18 +114,18 @@ export async function cleanupResidualUserData(user: ErasureUser): Promise const deletionRequestIds = requestIds.filter((id) => !retainedActiveRequestIds.has(id)); if (deletionRequestIds.length) { const [artifacts, attachments, sourceSnapshots] = await Promise.all([ - db + currentAuthDatabase() .select({ id: dataExportArtifact.id, storageKey: dataExportArtifact.storageKey }) .from(dataExportArtifact) .where(inArray(dataExportArtifact.requestId, deletionRequestIds)), - db + currentAuthDatabase() .select({ id: dataSubjectRequestAttachment.id, storageKey: dataSubjectRequestAttachment.storageKey, }) .from(dataSubjectRequestAttachment) .where(inArray(dataSubjectRequestAttachment.requestId, deletionRequestIds)), - db + currentAuthDatabase() .select({ id: dataSubjectRequestSourceSnapshot.id, storageKey: dataSubjectRequestSourceSnapshot.storageKey, @@ -137,7 +139,7 @@ export async function cleanupResidualUserData(user: ErasureUser): Promise else await deletePrivacyCiphertext(object.storageKey); } const erasedAt = new Date(); - await db.transaction(async (tx) => { + await currentAuthDatabase().transaction(async (tx) => { if (artifacts.length) { await tx .update(dataExportArtifact) @@ -194,7 +196,7 @@ export async function cleanupResidualUserData(user: ErasureUser): Promise } const [emailIdentifier, changeEmailIdentifier] = erasureVerificationIdentifiers(user); const accountErasedAt = new Date(); - await db.transaction(async (tx) => { + await currentAuthDatabase().transaction(async (tx) => { await tx .update(dataSubjectRequest) .set({ @@ -302,7 +304,15 @@ async function deletePrivacyCiphertext(storageKey: string): Promise { } export const userErasureHooks = createUserErasureHooks({ - request: async (userId) => appendErasureRequest(journalPath(), readJournalKey(), userId), + request: async (userId) => { + await assertAccountDeletionAllowed(userId); + return appendErasureRequest(journalPath(), readJournalKey(), userId); + }, cleanup: cleanupResidualUserData, - complete: async (receiptId) => appendErasureCompleted(journalPath(), readJournalKey(), receiptId), + complete: async (receiptId) => { + const complete = () => appendErasureCompleted(journalPath(), readJournalKey(), receiptId); + const context = currentAccountMutation(); + if (context) context.afterCommit.push(complete); + else await complete(); + }, }); diff --git a/apps/api/src/test/postgres-fixture.ts b/apps/api/src/test/postgres-fixture.ts index 5508beb72..ff142b981 100644 --- a/apps/api/src/test/postgres-fixture.ts +++ b/apps/api/src/test/postgres-fixture.ts @@ -1,3 +1,6 @@ +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { PostgreSqlContainer } from "@testcontainers/postgresql"; import { drizzle } from "drizzle-orm/postgres-js"; @@ -6,7 +9,7 @@ import postgres from "postgres"; import * as schema from "../db/schema"; /** Never consumes DATABASE_URL: all migrations and requests use a disposable database. */ -export async function createMigratedPostgresFixture() { +export async function createMigratedPostgresFixture(options: { throughMigration?: number } = {}) { const container = await new PostgreSqlContainer( "ghcr.io/baosystems/postgis:18-3.6@sha256:8784f86b3e0e046859c96ea25eb8b10f0b801fd891b99a82b82f28ac4a643db8", ) @@ -17,18 +20,42 @@ export async function createMigratedPostgresFixture() { .start(); const sql = postgres(container.getConnectionUri(), { max: 4 }); const db = drizzle(sql, { schema }); + let temporaryMigrations: string | undefined; const stop = async () => { try { await sql.end({ timeout: 2 }); } finally { - await container.stop(); + try { + await container.stop(); + } finally { + if (temporaryMigrations) await rm(temporaryMigrations, { recursive: true, force: true }); + } } }; try { await sql`CREATE EXTENSION IF NOT EXISTS postgis`; - await migrate(db, { - migrationsFolder: fileURLToPath(new URL("../db/migrations", import.meta.url)), - }); + let migrationsFolder = fileURLToPath(new URL("../db/migrations", import.meta.url)); + if (options.throughMigration !== undefined) { + const journal = JSON.parse( + await readFile(join(migrationsFolder, "meta/_journal.json"), "utf8"), + ) as { entries: { idx: number; tag: string }[] }; + if ( + !Number.isInteger(options.throughMigration) || + !journal.entries.some((e) => e.idx === options.throughMigration) + ) + throw Error("Invalid fixture migration boundary"); + journal.entries = journal.entries.filter((e) => e.idx <= options.throughMigration!); + temporaryMigrations = await mkdtemp(join(tmpdir(), "openmapx-migration-fixture-")); + await mkdir(join(temporaryMigrations, "meta")); + await writeFile(join(temporaryMigrations, "meta/_journal.json"), JSON.stringify(journal)); + for (const entry of journal.entries) + await copyFile( + join(migrationsFolder, `${entry.tag}.sql`), + join(temporaryMigrations, `${entry.tag}.sql`), + ); + migrationsFolder = temporaryMigrations; + } + await migrate(db, { migrationsFolder }); return { sql, db, stop }; } catch (error) { await stop(); diff --git a/apps/api/src/utils/audit-log.ts b/apps/api/src/utils/audit-log.ts index 69b0dd42c..6e3af15ad 100644 --- a/apps/api/src/utils/audit-log.ts +++ b/apps/api/src/utils/audit-log.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import type { FastifyRequest } from "fastify"; -import { db } from "../db"; +import { currentAccountMutation } from "../account/account-context"; +import { currentAuthDatabase } from "../account/auth-database"; import { adminAuditLog } from "../db/schema"; import { appLogger } from "../services/app-logger.js"; import { safeErrorClass, sanitizeLogMetadata } from "./safe-log-fields.js"; @@ -29,20 +30,23 @@ export async function writeAuditLog(entry: AuditLogEntry): Promise { const userAgent = isLoopback ? `${baseUa ?? "unknown"} (loopback)` : baseUa; try { - await db.insert(adminAuditLog).values({ - id: randomUUID(), - actorId: isLoopback ? null : (entry.actorId ?? null), - targetId: entry.targetId ?? null, - targetType: entry.targetType ?? null, - action: entry.action, - details: - entry.details === null || entry.details === undefined - ? null - : sanitizeLogMetadata(entry.details), - ipAddress: entry.request?.ip ?? null, - userAgent, - }); + await currentAuthDatabase() + .insert(adminAuditLog) + .values({ + id: randomUUID(), + actorId: isLoopback ? null : (entry.actorId ?? null), + targetId: entry.targetId ?? null, + targetType: entry.targetType ?? null, + action: entry.action, + details: + entry.details === null || entry.details === undefined + ? null + : { ...sanitizeLogMetadata(entry.details) }, + ipAddress: entry.request?.ip ?? null, + userAgent, + }); } catch (err) { + if (currentAccountMutation()) throw err; // Audit log failures must never break the main operation appLogger.add({ level: "error", diff --git a/apps/api/src/utils/auth-audit-hook.ts b/apps/api/src/utils/auth-audit-hook.ts index d1f2c57d9..d6ff13159 100644 --- a/apps/api/src/utils/auth-audit-hook.ts +++ b/apps/api/src/utils/auth-audit-hook.ts @@ -1,4 +1,5 @@ import { createAuthMiddleware } from "better-auth/api"; +import { currentAccountMutation } from "../account/account-context"; import { writeAuditLog } from "./audit-log"; // Map admin-plugin endpoint paths to (action, target-type) tuples for audit @@ -45,6 +46,7 @@ function pickString(value: unknown): string | undefined { // Extracted from the `hooks.after` middleware so it can be tested directly; the // middleware below delegates to it with the real better-auth context. export async function handleAdminAuditEvent(ctx: AdminAuditCtx): Promise { + if (ctx.path === "/admin/set-role" && currentAccountMutation()?.roleAudit) return; const route = ADMIN_AUDIT_ROUTES[ctx.path]; if (!route) return; diff --git a/scripts/check-subject-data.ts b/scripts/check-subject-data.ts index e2c982ee9..d4d0f4dc6 100644 --- a/scripts/check-subject-data.ts +++ b/scripts/check-subject-data.ts @@ -14,6 +14,8 @@ export const DIRECT_USER_FK_CLASSIFICATIONS: Record = { "account.user_id": "auth-accounts", "data_disclosure_event.user_id": "disclosure-events", "admin_audit_log.actor_id": "admin-audit-attribution", + "auth_policy_application.actor_id": "admin-audit-attribution", + "auth_policy_state.activated_by": "admin-audit-attribution", "admin_job.created_by": "admin-jobs-attribution", "installed_extension.installed_by": "installed-component-attribution", "installed_integration.installed_by": "installed-component-attribution", diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index e7549aa41..696282272 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -23,7 +23,9 @@ "apps/api/src/services/integration-config-store-postgres.test.ts", "apps/api/src/services/mobileAuthHandoff.test.ts", "apps/api/src/services/service-config-writer-postgres.test.ts", - "apps/api/src/account/auth-policy-postgres.test.ts" + "apps/api/src/account/auth-policy-postgres.test.ts", + "apps/api/src/account/admin-invariant-postgres.test.ts", + "apps/api/src/account/admin-invariant-migration.test.ts" ], "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", From 02861c8a8cac8ff93d9a10e6c9cf89007a128f38 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 14:11:57 +0100 Subject: [PATCH 042/135] fix(admin): unwrap account operations before reporting success --- .../components/admin/users/BanUserDialog.tsx | 25 +- .../admin/users/CreateUserDialog.tsx | 44 +-- .../admin/users/UserActions.test.tsx | 251 ++++++++++++++++++ .../src/components/admin/users/UserDetail.tsx | 176 ++++++------ .../src/components/admin/users/UserList.tsx | 200 ++++++++------ apps/web/src/lib/admin/accountAdmin.test.ts | 186 +++++++++++++ apps/web/src/lib/admin/accountAdmin.ts | 233 ++++++++++++++++ packages/core/src/api/privacy.ts | 14 +- 8 files changed, 930 insertions(+), 199 deletions(-) create mode 100644 apps/web/src/components/admin/users/UserActions.test.tsx create mode 100644 apps/web/src/lib/admin/accountAdmin.test.ts create mode 100644 apps/web/src/lib/admin/accountAdmin.ts diff --git a/apps/web/src/components/admin/users/BanUserDialog.tsx b/apps/web/src/components/admin/users/BanUserDialog.tsx index cbfec9a8b..7c4954af2 100644 --- a/apps/web/src/components/admin/users/BanUserDialog.tsx +++ b/apps/web/src/components/admin/users/BanUserDialog.tsx @@ -7,9 +7,8 @@ import DialogContent from "@mui/material/DialogContent"; import DialogContentText from "@mui/material/DialogContentText"; import DialogTitle from "@mui/material/DialogTitle"; import TextField from "@mui/material/TextField"; -import { authClient } from "@openmapx/core"; -import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useState } from "react"; +import { useAccountAction } from "@/lib/admin/accountAdmin"; import { useAdminToast } from "../shared/AdminToast"; interface BanUserDialogProps { @@ -18,26 +17,23 @@ interface BanUserDialogProps { } export function BanUserDialog({ user, onClose }: BanUserDialogProps) { - const qc = useQueryClient(); const showToast = useAdminToast(); const [reason, setReason] = useState(""); - const ban = useMutation({ - mutationFn: () => - authClient.admin.banUser({ - userId: user.id, - banReason: reason.trim() || undefined, - }), + const ban = useAccountAction(user.id, { onSuccess: () => { - void qc.invalidateQueries({ queryKey: ["admin", "users"] }); showToast(`${user.name} has been banned`); onClose(); }, - onError: (err) => showToast((err as Error).message || "Failed to ban user", "error"), + onError: (error) => showToast(error.message, "error"), }); + const close = () => { + ban.invalidateOwner(); + onClose(); + }; return ( - + Ban {user.name}? setReason(e.target.value)} @@ -63,14 +60,14 @@ export function BanUserDialog({ user, onClose }: BanUserDialogProps) { )} - diff --git a/apps/web/src/components/admin/users/CreateUserDialog.tsx b/apps/web/src/components/admin/users/CreateUserDialog.tsx index c93bc3198..219e5252c 100644 --- a/apps/web/src/components/admin/users/CreateUserDialog.tsx +++ b/apps/web/src/components/admin/users/CreateUserDialog.tsx @@ -13,9 +13,8 @@ import MenuItem from "@mui/material/MenuItem"; import Select from "@mui/material/Select"; import Stack from "@mui/material/Stack"; import TextField from "@mui/material/TextField"; -import { authClient } from "@openmapx/core"; -import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useState } from "react"; +import { useAccountAction } from "@/lib/admin/accountAdmin"; import { useAdminToast } from "../shared/AdminToast"; interface CreateUserDialogProps { @@ -24,7 +23,6 @@ interface CreateUserDialogProps { } export function CreateUserDialog({ open, onClose }: CreateUserDialogProps) { - const qc = useQueryClient(); const showToast = useAdminToast(); const [name, setName] = useState(""); const [email, setEmail] = useState(""); @@ -32,17 +30,9 @@ export function CreateUserDialog({ open, onClose }: CreateUserDialogProps) { const [role, setRole] = useState<"user" | "admin">("user"); const [sendWelcome, setSendWelcome] = useState(true); - const create = useMutation({ - mutationFn: () => - authClient.admin.createUser({ - name, - email, - password, - role, - data: { sendWelcomeEmail: sendWelcome }, - }), + const create = useAccountAction("create", { + viewKey: String(open), onSuccess: () => { - void qc.invalidateQueries({ queryKey: ["admin", "users"] }); showToast("User created successfully"); onClose(); setName(""); @@ -50,13 +40,17 @@ export function CreateUserDialog({ open, onClose }: CreateUserDialogProps) { setPassword(""); setRole("user"); }, - onError: (err) => showToast((err as Error).message || "Failed to create user", "error"), + onError: (error) => showToast(error.message, "error"), }); + const close = () => { + create.invalidateOwner(); + onClose(); + }; const valid = name.trim() && email.trim() && password.length >= 8; return ( - + Create User setName(e.target.value)} @@ -74,6 +69,7 @@ export function CreateUserDialog({ open, onClose }: CreateUserDialogProps) { autoFocus /> Role setRole(e.target.value)} - disabled={isSelf} + disabled={isSelf || updateUser.isPending} > User Admin @@ -189,7 +162,13 @@ function ProfileTab({ user, isSelf }: { user: AdminUser; isSelf: boolean }) { variant="contained" size="small" sx={{ alignSelf: "flex-start" }} - onClick={() => updateUser.mutate()} + onClick={() => + updateUser.mutate(async (client) => { + if (name !== user.name) await client.updateName(user.id, name); + if (role !== (user.role ?? "user")) + await client.changeRole(user.id, role, user.role ?? "user"); + }) + } disabled={updateUser.isPending} > Save Changes @@ -257,7 +236,7 @@ function ProfileTab({ user, isSelf }: { user: AdminUser; isSelf: boolean }) { startIcon={} variant="outlined" size="small" - onClick={() => impersonate.mutate()} + onClick={impersonate.mutate} disabled={impersonate.isPending} > Impersonate @@ -268,7 +247,7 @@ function ProfileTab({ user, isSelf }: { user: AdminUser; isSelf: boolean }) { startIcon={} variant="outlined" size="small" - onClick={() => unban.mutate()} + onClick={() => unban.mutate((client) => client.unban(user.id))} disabled={unban.isPending} > Unban @@ -307,7 +286,7 @@ function ProfileTab({ user, isSelf }: { user: AdminUser; isSelf: boolean }) { confirmLabel="Delete" confirmColor="error" loading={deleteUser.isPending} - onConfirm={() => deleteUser.mutate()} + onConfirm={() => deleteUser.mutate((client) => client.remove(user.id))} onCancel={() => setDeleteOpen(false)} /> @@ -315,25 +294,15 @@ function ProfileTab({ user, isSelf }: { user: AdminUser; isSelf: boolean }) { } function SessionsTab({ userId }: { userId: string }) { - const qc = useQueryClient(); - - const { data, isLoading } = useQuery({ - queryKey: ["admin", "user-sessions", userId], - queryFn: async () => { - const res = await authClient.admin.listUserSessions({ userId }); - return (res.data?.sessions ?? []) as unknown as SessionRecord[]; - }, - }); - - const revoke = useMutation({ - mutationFn: (sessionToken: string) => authClient.admin.revokeUserSession({ sessionToken }), - onSuccess: () => qc.invalidateQueries({ queryKey: ["admin", "user-sessions", userId] }), - }); - - const revokeAll = useMutation({ - mutationFn: () => authClient.admin.revokeUserSessions({ userId }), - onSuccess: () => qc.invalidateQueries({ queryKey: ["admin", "user-sessions", userId] }), + const account = useAccountAdmin(); + const { data, isLoading, error } = useQuery({ + queryKey: account.keys.sessions(userId), + enabled: Boolean(account.actorId), + retry: false, + queryFn: () => account.client.sessions(userId), }); + const revoke = useAccountAction(userId), + revokeAll = useAccountAction(userId); const sessions = data ?? []; const { paged, paginationProps } = useClientPagination(sessions); @@ -346,6 +315,11 @@ function SessionsTab({ userId }: { userId: string }) { gap: 2, }} > + {(error || revoke.error || revokeAll.error) && ( + + {error ? "Could not load sessions." : (revoke.error || revokeAll.error)?.message} + + )} revokeAll.mutate()} + onClick={() => revokeAll.mutate((client) => client.revokeAll(userId))} disabled={revokeAll.isPending || (data?.length ?? 0) === 0} > Revoke All Sessions @@ -385,7 +359,7 @@ function SessionsTab({ userId }: { userId: string }) { py: 2, }} > - No active sessions + {error ? "Unable to load sessions" : "No active sessions"} @@ -416,7 +390,7 @@ function SessionsTab({ userId }: { userId: string }) { + + ) : ( + + edit(() => setName(e.target.value))} + size="small" + fullWidth + autoFocus + slotProps={{ htmlInput: { maxLength: 256 } }} + /> + edit(() => setEmail(e.target.value))} + size="small" + fullWidth + slotProps={{ htmlInput: { maxLength: 320 } }} + /> + setRole(e.target.value as "user" | "admin")} - > - User - Admin - - - edit(() => setPassword(e.target.value))} + size="small" + fullWidth + slotProps={{ htmlInput: { maxLength: 128 } }} + helperText="Optional. Leave empty to send a one-use password setup link. The instance password policy applies." + /> + + Role + + + edit(() => setSendInvitation(e.target.checked))} + size="small" + /> + } + label="Send welcome email" + /> + {!password && ( + + Without an initial password, a setup invitation is required. Verification and + password setup are sent separately. + + )} + {create.error && {create.error.message}} + + )} - - + + {!created && ( + + )} ); diff --git a/apps/web/src/components/admin/users/UserActions.test.tsx b/apps/web/src/components/admin/users/UserActions.test.tsx index 5d813a3f9..64a084c9d 100644 --- a/apps/web/src/components/admin/users/UserActions.test.tsx +++ b/apps/web/src/components/admin/users/UserActions.test.tsx @@ -41,6 +41,14 @@ vi.mock("@openmapx/core", async () => ({ }, }, })); +vi.mock("@openmapx/core/api-admin-sdk", async () => ({ + ...(await vi.importActual( + "@openmapx/core/api-admin-sdk", + )), + createAdminAccount: mocks.create, + changeAdminAccountRole: mocks.role, + getAdminAccountDelivery: vi.fn(async () => ({ status: 200, data: { latest: null } })), +})); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ useEnv: () => ({ apiUrl: "http://admin-a.test" }), })); @@ -69,7 +77,8 @@ beforeEach(() => { "impersonate", ] as const) mocks[key].mockResolvedValue({ - data: null, + status: 403, + data: { error: "FORBIDDEN" }, error: { code: "FORBIDDEN", message: "synthetic private error" }, }); mocks.list.mockResolvedValue({ @@ -77,7 +86,8 @@ beforeEach(() => { error: { code: "SERVICE_UNAVAILABLE", message: "synthetic provider diagnostic" }, }); mocks.create.mockResolvedValue({ - data: null, + status: 400, + data: { error: "INVALID_PASSWORD" }, error: { code: "INVALID_PASSWORD", message: "synthetic provider diagnostic" }, }); mocks.ban.mockResolvedValue({ @@ -203,13 +213,10 @@ it.each(["list", "detail"])( if (view === "detail") fireEvent.click(screen.getByRole("button", { name: "Save Changes" })); await waitFor(() => expect(mocks.role).toHaveBeenCalledTimes(1)); expect(mocks.role).toHaveBeenCalledWith( - expect.objectContaining({ - userId: subject.id, - role: "admin", - fetchOptions: expect.objectContaining({ - headers: expect.objectContaining({ "x-openmapx-expected-role": "user" }), - }), - }), + subject.id, + { role: "admin", expectedRole: "user" }, + { "idempotency-key": expect.any(String) }, + expect.objectContaining({ client: expect.any(Object) }), ); expect(mocks.update).not.toHaveBeenCalled(); if (view === "list") @@ -244,8 +251,29 @@ it("a closed and reopened create dialog keeps new inputs when the old request co view.rerender(child(false)); view.rerender(child(true)); fireEvent.change(screen.getByLabelText("Full Name"), { target: { value: "Second" } }); - await act(async () => resolve({ data: { user: subject }, error: null })); + await act(async () => + resolve({ + status: 200, + data: { + userId: subject.id, + created: true, + verificationDelivery: "queued", + passwordSetupDelivery: "queued", + }, + }), + ); expect(screen.getByLabelText("Full Name")).toHaveValue("Second"); expect(close).not.toHaveBeenCalled(); expect(mocks.toast).not.toHaveBeenCalled(); }); + +it("offers the privacy role for an existing administrator through the same canonical writer", async () => { + mocks.list.mockResolvedValue({ + data: { users: [{ ...subject, role: "admin" }], total: 2 }, + error: null, + }); + mount(); + await screen.findByText(subject.email); + fireEvent.mouseDown(screen.getByRole("combobox", { name: "Role" })); + expect(screen.getByRole("option", { name: "Privacy admin" })).toBeInTheDocument(); +}); diff --git a/apps/web/src/components/admin/users/UserDetail.tsx b/apps/web/src/components/admin/users/UserDetail.tsx index 013118d11..58a68c9ae 100644 --- a/apps/web/src/components/admin/users/UserDetail.tsx +++ b/apps/web/src/components/admin/users/UserDetail.tsx @@ -39,6 +39,7 @@ import { AdminPageHeader } from "../shared/AdminPageHeader"; import { AdminTablePagination } from "../shared/AdminTablePagination"; import { ConfirmDialog } from "../shared/ConfirmDialog"; import { useClientPagination } from "../shared/tableHooks"; +import { AccountRecoveryActions } from "./AccountRecoveryActions"; import { BanUserDialog } from "./BanUserDialog"; interface AdminUser { @@ -154,7 +155,7 @@ function ProfileTab({ > User Admin - {user.role !== "admin" && Privacy admin} + Privacy admin {dirty && ( @@ -224,6 +225,8 @@ function ProfileTab({ + + User Admin - {role !== "admin" && Privacy admin} + Privacy admin ); @@ -131,13 +131,6 @@ function ActionsMenu({ }, onError: (error) => showToast(error.message, "error"), }); - const resetPassword = useAccountAction(user.id, { - onSuccess: () => { - setAnchor(null); - showToast(`Password reset requested for ${user.name}`); - }, - onError: (error) => showToast(error.message, "error"), - }); const deleteUser = useAccountAction(user.id, { onSuccess: () => { setConfirmDelete(false); @@ -203,17 +196,14 @@ function ActionsMenu({ ) )} - resetPassword.mutate((client) => - client.resetPassword(user.email, window.location.origin), - ) - } - disabled={resetPassword.isPending} + component={Link} + href={`/admin/users/${user.id}#account-recovery`} + onClick={() => setAnchor(null)} > - Send Password Reset + Account recovery… {!isSelf && !isLastAdmin && ( ({ }, setPrivacyAdminRole: state.methods.privacy, })); +vi.mock("@openmapx/core/api-admin-sdk", async () => ({ + ...(await vi.importActual( + "@openmapx/core/api-admin-sdk", + )), + createAdminAccount: state.methods.createUser, + changeAdminAccountRole: state.methods.setRole, + sendAdminAccountPasswordReset: state.methods.reset, +})); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ useEnv: () => ({ apiUrl: state.apiUrl }), })); @@ -75,6 +83,7 @@ it.each<[string, (c: ReturnType) => Promise) => c.updateName("a", "Synthetic")], @@ -86,7 +95,13 @@ it.each<[string, (c: ReturnType) => Promise) => - c.resetPassword("a@example.test", "http://ui.test"), + c.recover( + "reset-password", + "a", + { confirmed: true }, + { challengeId: "11111111-1111-4111-8111-111111111111" }, + crypto.randomUUID(), + ), ], ["revoke", (c: ReturnType) => c.revoke("synthetic-token")], ["revoke all", (c: ReturnType) => c.revokeAll("a")], @@ -107,14 +122,18 @@ it("captures origin and canonical role expectation for every write", async () => state.apiUrl = "http://api-b.test"; await expect(client.changeRole("a", "admin", "user")).rejects.toThrow(); expect(state.methods.setRole).toHaveBeenCalledWith( - expect.objectContaining({ - role: "admin", - fetchOptions: expect.objectContaining({ - baseURL: "http://api-a.test/api/auth", - headers: expect.objectContaining({ "x-openmapx-expected-role": "user" }), - }), - }), + "a", + { role: "admin", expectedRole: "user" }, + { "idempotency-key": expect.any(String) }, + { client: expect.any(Object) }, ); + const options = state.methods.setRole.mock.calls[0]?.[3] as + | { client: { connectionTarget: (path: string) => unknown } } + | undefined; + expect(options?.client.connectionTarget("/api/admin/accounts")).toEqual({ + url: "http://api-a.test/api/admin/accounts", + withCredentials: true, + }); expect(state.methods.updateUser).not.toHaveBeenCalled(); }); it.each(["target", "actor", "origin", "unmount"])( diff --git a/apps/web/src/lib/admin/accountAdmin.ts b/apps/web/src/lib/admin/accountAdmin.ts index 9264b0ff4..62c2246d7 100644 --- a/apps/web/src/lib/admin/accountAdmin.ts +++ b/apps/web/src/lib/admin/accountAdmin.ts @@ -1,11 +1,27 @@ "use client"; -import { authClient, createApiClient, setPrivacyAdminRole } from "@openmapx/core"; +import { authClient, createApiClient } from "@openmapx/core"; +import * as accountSdk from "@openmapx/core/api-admin-sdk"; +import { + type AdminReauthProofRef, + type AuthPolicySettings, + accountDeliverySchema, + accountProvisioningResultSchema, + accountRoleChangeSchema, + assistedAccountPasswordSchema, + assistedAccountVerifySchema, + authPolicyViewSchema, +} from "@openmapx/core/api-contracts"; import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useLayoutEffect, useMemo, useRef } from "react"; import { useEnv } from "@/integration-api/runtime/EnvProvider"; import { browserAdminApiBase } from "./browserAdminApiBase"; const messages: Record = { + ACCOUNT_MAIL_RATE_LIMITED: "Too many mail requests. Wait an hour before requesting another.", + ACCOUNT_EMAIL_CHANGED: "This account's email changed. Refresh and confirm its current address.", + ACCOUNT_ACTION_CONFLICT: "This request changed. Refresh before trying again.", + REAUTHENTICATION_REQUIRED: "Confirm a fresh administrator sign-in for this action.", + AUTH_POLICY_CONFLICT: "The active policy changed. Reload before applying your changes.", LAST_ADMIN_REQUIRED: "At least one active administrator must remain.", ROLE_VERSION_CONFLICT: "This account changed. Refresh before changing its role.", INVALID_PASSWORD: "The password does not meet the instance policy.", @@ -40,10 +56,20 @@ export function accountError(error: unknown) { (error as { payload?: { code?: unknown } } | null)?.payload?.code, ); } +export type AccountRecoveryAction = + | "invitation" + | "verification-email" + | "verify" + | "reset-password" + | "password"; +function accountResponse(result: { status: number; data: unknown }) { + if (result.status !== 200 || !result.data) + throw new AccountAdminError((result.data as { error?: unknown } | null)?.error); + return result.data; +} /** Capture the actual auth methods and origin once; an in-flight request never follows a replacement client. */ export function createAccountAdminClient(apiUrl: string) { const admin = authClient.admin, - reset = authClient.requestPasswordReset, baseURL = browserAdminApiBase(apiUrl), client = createApiClient({ baseUrl: baseURL, credentials: "include" }); const fetchOptions = (headers?: Record) => ({ @@ -77,8 +103,75 @@ export function createAccountAdminClient(apiUrl: string) { await admin.listUserSessions({ userId, fetchOptions: fetchOptions() }), ).sessions; }, - async create(input: Parameters[0]) { - return unwrapAuthResult(await admin.createUser({ ...input, fetchOptions: writes() })); + async create( + input: Parameters[0], + key = crypto.randomUUID(), + ) { + return accountProvisioningResultSchema.parse( + accountResponse( + await accountSdk.createAdminAccount(input, { "idempotency-key": key }, { client }), + ), + ); + }, + async delivery(userId: string) { + return accountDeliverySchema.parse( + accountResponse(await accountSdk.getAdminAccountDelivery(userId, { client })), + ); + }, + async policy() { + return authPolicyViewSchema.parse( + accountResponse(await accountSdk.getAdminAuthPolicy({ client })), + ); + }, + async applyPolicy(settings: AuthPolicySettings, expectedRevision: string, key: string) { + accountResponse( + await accountSdk.applyAdminAuthPolicy( + { settings, expectedRevision, confirmed: true, idempotencyKey: key }, + { client }, + ), + ); + }, + async recover( + action: AccountRecoveryAction, + userId: string, + body: unknown, + proof: AdminReauthProofRef, + key: string, + ) { + const headers = { "idempotency-key": key, "x-openmapx-reauth-challenge": proof.challengeId }; + const response = + action === "verify" + ? await accountSdk.verifyAdminAccount( + userId, + assistedAccountVerifySchema.parse(body), + headers, + { client }, + ) + : action === "password" + ? await accountSdk.setAdminAccountPassword( + userId, + assistedAccountPasswordSchema.parse(body), + headers, + { client }, + ) + : action === "reset-password" + ? await accountSdk.sendAdminAccountPasswordReset( + userId, + { confirmed: true }, + headers, + { client }, + ) + : action === "verification-email" + ? await accountSdk.sendAdminAccountVerificationEmail( + userId, + { confirmed: true }, + headers, + { client }, + ) + : await accountSdk.inviteAdminAccount(userId, { confirmed: true }, headers, { + client, + }); + return accountProvisioningResultSchema.parse(accountResponse(response)); }, async updateName(userId: string, name: string) { return unwrapAuthResult( @@ -86,16 +179,14 @@ export function createAccountAdminClient(apiUrl: string) { ); }, async changeRole(userId: string, role: string, expectedRole: string) { - if (!["user", "admin", "privacy_admin"].includes(role)) - throw new AccountAdminError("INVALID_ROLE"); - if (role === "privacy_admin" || (expectedRole === "privacy_admin" && role === "user")) - return setPrivacyAdminRole(userId, role as "privacy_admin" | "user", { client }); - return unwrapAuthResult( - await admin.setRole({ + const body = accountRoleChangeSchema.parse({ role, expectedRole }); + return accountResponse( + await accountSdk.changeAdminAccountRole( userId, - role: role as "user" | "admin", - fetchOptions: writes({ "x-openmapx-expected-role": expectedRole }), - }), + body, + { "idempotency-key": crypto.randomUUID() }, + { client }, + ), ); }, async ban(userId: string, banReason?: string) { @@ -110,9 +201,6 @@ export function createAccountAdminClient(apiUrl: string) { async impersonate(userId: string) { return unwrapAuthResult(await admin.impersonateUser({ userId, fetchOptions: writes() })); }, - async resetPassword(email: string, redirectTo: string) { - return unwrapAuthResult(await reset({ email, redirectTo, fetchOptions: writes() })); - }, async revoke(sessionToken: string) { return unwrapAuthResult( await admin.revokeUserSession({ sessionToken, fetchOptions: writes() }), @@ -143,6 +231,8 @@ export function useAccountAdmin() { count: [...root, "count"], detail: (id: string) => [...root, "detail", id], sessions: (id: string) => [...root, "sessions", id], + delivery: (id: string) => [...root, "delivery", id], + policy: [...root, "policy"], }, }; } @@ -194,6 +284,8 @@ export function useAccountAction( void qc.invalidateQueries({ queryKey: [...root, "count"] }); void qc.invalidateQueries({ queryKey: [...root, "detail", request.targetId] }); void qc.invalidateQueries({ queryKey: [...root, "sessions", request.targetId] }); + void qc.invalidateQueries({ queryKey: [...root, "delivery", request.targetId] }); + void qc.invalidateQueries({ queryKey: [...root, "policy"] }); if (owns(request.token)) { void qc.invalidateQueries({ queryKey: ["admin", "users"] }); void qc.invalidateQueries({ queryKey: ["admin", "user", request.targetId] }); diff --git a/apps/web/src/lib/admin/accountAdminTransport.test.ts b/apps/web/src/lib/admin/accountAdminTransport.test.ts new file mode 100644 index 000000000..f60d02bff --- /dev/null +++ b/apps/web/src/lib/admin/accountAdminTransport.test.ts @@ -0,0 +1,69 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { createAccountAdminClient } from "./accountAdmin"; + +const calls: { url: string; init: RequestInit }[] = []; +vi.mock("@openmapx/core", async () => ({ + ...(await vi.importActual("@openmapx/core")), + authClient: { admin: {} }, +})); +beforeEach(() => { + calls.length = 0; + vi.stubGlobal( + "fetch", + vi.fn(async (...args: unknown[]) => { + const [url, init] = args as [string, RequestInit]; + calls.push({ url: String(url), init }); + return new Response( + JSON.stringify({ + userId: "subject-a", + created: false, + verificationDelivery: "queued", + passwordSetupDelivery: "queued", + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }), + ); +}); +afterEach(() => vi.unstubAllGlobals()); +it.each(["invitation", "verification-email", "verify", "reset-password", "password"] as const)( + "uses real generated transport for %s with bounded proof and stable idempotency headers", + async (action) => { + const client = createAccountAdminClient("https://api.example.test"), + key = "test-idempotency-key-0001", + challengeId = "11111111-1111-4111-8111-111111111111", + body = + action === "verify" + ? { + confirmed: true, + expectedEmail: "subject@example.test", + reasonCode: "identity_verified_out_of_band", + } + : action === "password" + ? { confirmed: true, password: "synthetic-password", reasonCode: "account_recovery" } + : { confirmed: true }; + await client.recover(action, "subject-a", body, { challengeId }, key); + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe(`https://api.example.test/api/admin/accounts/subject-a/${action}`); + expect(new Headers(calls[0]?.init.headers).get("idempotency-key")).toBe(key); + expect(new Headers(calls[0]?.init.headers).get("x-openmapx-reauth-challenge")).toBe( + challengeId, + ); + expect(calls[0]?.init.credentials).toBe("include"); + expect(JSON.parse(String(calls[0]?.init.body))).toEqual(body); + }, +); +it("uses the actual generated create transport with the selected invitation value", async () => { + await createAccountAdminClient("https://api.example.test").create( + { + name: "Subject", + email: "subject@example.test", + password: "synthetic-password", + role: "user", + sendInvitation: false, + }, + "test-idempotency-key-0002", + ); + expect(calls[0]?.url).toBe("https://api.example.test/api/admin/accounts"); + expect(JSON.parse(String(calls[0]?.init.body))).toMatchObject({ sendInvitation: false }); +}); diff --git a/docs/docs/administration/users-and-access.md b/docs/docs/administration/users-and-access.md index f9120a089..a8921c330 100644 --- a/docs/docs/administration/users-and-access.md +++ b/docs/docs/administration/users-and-access.md @@ -21,12 +21,14 @@ edited there directly. Every account has a small, fixed set of fields: a name, an email, a verified flag, an optional avatar, a **role**, and moderation state (banned, ban reason, -ban expiry). There are exactly two roles: +ban expiry). The three roles are: - **`user`** — the default. Can use the app and manage their own account, but never sees `/admin`. - **`admin`** — everything a user can do, plus the full operations surface. - There are no finer-grained permissions; admin is all-or-nothing. + This role manages instance configuration and accounts. +- **`privacy_admin`** — access to the dedicated privacy case workflow; it does + not grant general instance or account administration. The role is the single gate in front of the [admin panel](./admin-panel.md). In the browser the `/admin` route checks it before rendering, and on the API every @@ -76,11 +78,11 @@ when the web app is down — which is exactly when you tend to need them. :::note[Why the CLI works without logging in] -The CLI reaches admin endpoints over the host's loopback interface, which is -treated as trusted. In production you gate that with a token; the mechanics and -the safer multi-tenant setup are covered in -[Admin panel → The local admin escape hatch](./admin-panel.md). Set it up before -exposing the API. +These bootstrap user commands run through the host's generated Compose +configuration and database container. Protect host access. Other CLI operations +can use the local admin API escape hatch described in +[Admin panel](./admin-panel.md); that path cannot substitute for the fresh +account-session confirmation required by assisted recovery. ::: @@ -119,9 +121,33 @@ updated timestamps, verification and two-factor state, and any ban reason. ### Reset a password -From a user's row menu, **Send Password Reset** emails them a reset link — the -account owner sets the new password, so you never see or handle it. (This needs -a working email provider.) +Open **Users → Create User** to provision an account. You may supply an initial +password or leave it empty to send a one-use setup link. The welcome checkbox +requests separate verification and password-setup emails; setup is required if +no initial password was supplied. Passwords are never included in the message. +Creating or resetting an account does not mark its email verified. + +The created account remains visible if mail fails. **Latest administrative mail +request** distinguishes queued, accepted by the mail service, failed and unknown +handoffs. Acceptance does not prove delivery. Interrupted sends remain unknown +and are never automatically resent; check with the recipient before making an +explicit new request. Use **View account and recovery actions** rather than +creating the same account again. + +Open **Account recovery** from a user's row or profile for an invitation, +verification email, password-reset link, assisted verification or temporary +password. Review the selected account, confirm the action, and complete a fresh +administrator sign-in with its configured second factor. After returning from +sign-in, reopen the original action and review it before confirming. Passwords +are deliberately not carried across that navigation. Assisted verification +requires the current email and a recorded reason; use it only after verifying +identity outside this flow. Password replacement revokes existing sessions and +reset links while preserving the verification flag. Mail actions are limited +to three per target and twenty per administrator per hour. + +Setup links expire, are one-use, and are replaced by an explicit new setup +invitation. A separate verification email proves ownership of the email address. +A working email provider is required for both; see **Settings → Email**. ### Sessions @@ -130,8 +156,9 @@ agent, and creation and expiry times. **Revoke** ends one session; **Revoke All Sessions** signs the user out everywhere at once. Reach for this when an account may be compromised, or to force a re-login after a role or password change. -How long sessions last is set globally by **Session Duration** under **Settings -→ Authentication** — seven days by default — measured from the last activity. +New sessions have an absolute lifetime set by **Settings → Authentication +policy** — seven days by default. Activity does not extend that deadline. +Existing sessions keep their original expiry after a policy change. ### Ban and unban @@ -175,7 +202,7 @@ The default. New users register with an email and password, and — when verification is required — confirm their address before they can sign in. Password reset runs over a one-time code sent by email, and so does changing an account's email. Whether verification is enforced and the minimum password -length are set under **Settings → Authentication**. +length are set under **Settings → Authentication policy**. This path depends on a configured email provider for verification and reset messages. See **Settings → Email** and @@ -210,12 +237,28 @@ linked accounts from their own account settings, not from the admin panel. ### Two-factor authentication -When enabled under **Settings → Authentication**, users can add a second factor +When new enrollment is allowed under **Settings → Authentication policy**, users can add a second factor to their account: a TOTP authenticator app, with backup codes for recovery. Email one-time codes are also available. Two-factor is opt-in per user — the setting controls whether anyone can enroll, not a blanket requirement — and the user list flags accounts that have it turned on. +## Effective authentication policy + +The policy panel shows the active values, their ownership source, activation +version and revision. Before explicit version-1 activation, safe defaults stay +active: email verification required, 168-hour sessions, new 2FA enrollment +allowed and an eight-character password minimum. Existing legacy settings are +shown as a separate draft. Loading that draft only edits the form; **Activate +authentication policy** applies all four reviewed values atomically. + +Session lifetime accepts 1–8760 whole hours and password minimum accepts 8–128 +characters. Passwords have a maximum of 128 characters. Turning off new 2FA +enrollment leaves existing second factors and sign-in challenges in place. +`AUTH_EMAIL_VERIFICATION`, when active, owns the verification setting and locks +that field in the panel. A conflicting or failed apply preserves the active +policy; reload before retrying a changed revision. + ## Where to go next - **[Admin panel](./admin-panel.md)** — how `/admin` is gated and the loopback From aa9014d3d0213a6acb4cb939d01773f1f53559a4 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 17:48:22 +0100 Subject: [PATCH 046/135] fix(admin): apply capability bindings through verified reloads --- apps/api/openapi.json | 957 +- .../__tests__/user-erasure-postgres.test.ts | 1 + .../db/integration-administration-schema.ts | 29 + .../0047_admin_integration_application.sql | 22 + .../src/db/migrations/meta/0047_snapshot.json | 8933 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + apps/api/src/db/schema.ts | 1 + ...tegration-host.binding-application.test.ts | 196 + apps/api/src/integration-host.ts | 20 +- apps/api/src/privacy/catalogue.ts | 1 + apps/api/src/privacy/openmapx-collectors.ts | 27 + .../__tests__/admin-integrations-sdk.test.ts | 83 +- .../src/routes/admin-capability-bindings.ts | 190 +- .../administrative-configuration-context.ts | 9 + ...ministrative-configuration-writer-owner.ts | 14 + apps/api/src/services/capability-bindings.ts | 25 +- .../configuration-commit-participant.ts | 4 +- .../integration-application-postgres.test.ts | 47 + .../services/integration-application.test.ts | 380 + .../src/services/integration-application.ts | 63 + .../integration-binding-participant.ts | 357 + .../src/services/integration-binding-state.ts | 76 + .../services/integration-runtime-evidence.ts | 34 + apps/api/src/services/job-runner.ts | 66 +- .../src/services/prospective-configuration.ts | 273 +- .../test/configuration-postgres-fixture.ts | 2 + .../src/__fixtures__/owned-service-host.ts | 5 +- .../integrations/CapabilityBindingPicker.tsx | 3 + .../integrations/RequiredServicesPanel.tsx | 110 +- .../IntegrationAdminConsumers.test.tsx | 89 +- .../src/lib/admin/integrationAdmin.test.tsx | 38 +- apps/web/src/lib/admin/integrationAdmin.ts | 25 +- .../integrations-administration.md | 16 +- .../api/contracts/integration-admin.test.ts | 33 +- .../integration-admin/application.ts | 39 + .../api/contracts/integration-admin/common.ts | 13 +- .../api/contracts/integration-admin/index.ts | 1 + .../contracts/integration-admin/operations.ts | 22 +- .../admin/integration-admin.generated.ts | 31 +- .../deleteAdminIntegrationBinding200.ts | 22 +- ...eleteAdminIntegrationBinding200Database.ts | 12 + ...leteAdminIntegrationBinding200HostApply.ts | 34 + .../deleteAdminIntegrationBinding200Reason.ts | 13 + ...deleteAdminIntegrationBinding200Runtime.ts | 12 + .../deleteAdminIntegrationBinding409.ts | 20 + .../deleteAdminIntegrationBinding503.ts | 20 + .../deleteAdminIntegrationBindingBody.ts | 6 +- .../deleteAdminIntegrationBindingHeaders.ts | 9 + .../admin/integration-models/index.ts | 20 + .../listAdminIntegrationBindings200.ts | 8 + ...inIntegrationBindings200LastApplication.ts | 26 + ...ationBindings200LastApplicationDatabase.ts | 12 + ...tionBindings200LastApplicationHostApply.ts | 34 + ...grationBindings200LastApplicationReason.ts | 13 + ...rationBindings200LastApplicationRuntime.ts | 12 + .../listAdminIntegrationBindings200Runtime.ts | 11 + ...inIntegrationBindings200RuntimeBindings.ts | 4 + .../setAdminIntegrationBinding200.ts | 22 +- .../setAdminIntegrationBinding200Database.ts | 12 + .../setAdminIntegrationBinding200HostApply.ts | 34 + .../setAdminIntegrationBinding200Reason.ts | 13 + .../setAdminIntegrationBinding200Runtime.ts | 12 + .../setAdminIntegrationBinding409.ts | 20 + .../setAdminIntegrationBindingBody.ts | 8 +- .../setAdminIntegrationBindingHeaders.ts | 9 + .../ops/configuration-preparation-records.ts | 1 + scripts/check-subject-data.ts | 1 + scripts/database-test-groups.json | 4 +- 68 files changed, 12479 insertions(+), 187 deletions(-) create mode 100644 apps/api/src/db/integration-administration-schema.ts create mode 100644 apps/api/src/db/migrations/0047_admin_integration_application.sql create mode 100644 apps/api/src/db/migrations/meta/0047_snapshot.json create mode 100644 apps/api/src/integration-host.binding-application.test.ts create mode 100644 apps/api/src/services/integration-application-postgres.test.ts create mode 100644 apps/api/src/services/integration-application.test.ts create mode 100644 apps/api/src/services/integration-application.ts create mode 100644 apps/api/src/services/integration-binding-participant.ts create mode 100644 apps/api/src/services/integration-binding-state.ts create mode 100644 apps/api/src/services/integration-runtime-evidence.ts create mode 100644 packages/core/src/api/contracts/integration-admin/application.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Database.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200HostApply.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Reason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Runtime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding503.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingHeaders.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplication.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationDatabase.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationHostApply.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationReason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationRuntime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200Runtime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200RuntimeBindings.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Database.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200HostApply.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Reason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Runtime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingHeaders.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index b98e2e3a0..515797338 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -34096,10 +34096,265 @@ ], "additionalProperties": false } + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runtime": { + "anyOf": [ + { + "type": "object", + "properties": { + "generationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "null" + ] + } + }, + "configRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "generationId", + "bindings", + "configRevision" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "lastApplication": { + "anyOf": [ + { + "type": "object", + "properties": { + "operationId": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "integrationId": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "database": { + "type": "string", + "enum": [ + "retained", + "restored", + "conflict", + "unknown" + ] + }, + "runtime": { + "type": "string", + "enum": [ + "applied", + "pending", + "failed", + "unknown" + ] + }, + "runtimeGenerationId": { + "anyOf": [ + { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "type": "string", + "enum": [ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required" + ] + }, + "hostApply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "operationId", + "integrationId", + "desiredRevision", + "database", + "runtime", + "runtimeGenerationId" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] } }, "required": [ - "bindings" + "bindings", + "revision", + "desiredRevision", + "runtime", + "lastApplication" ], "additionalProperties": false } @@ -34527,6 +34782,15 @@ "in": "path", "name": "capability", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { @@ -34536,12 +34800,31 @@ "schema": { "type": "object", "properties": { + "expectedServiceId": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + { + "type": "null" + } + ] + }, + "expectedRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "serviceId": { "type": "string", - "minLength": 1 + "minLength": 1, + "maxLength": 64 } }, "required": [ + "expectedServiceId", + "expectedRevision", "serviceId" ], "additionalProperties": false @@ -34557,49 +34840,234 @@ "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", + "operationId": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "integrationId": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "database": { + "type": "string", "enum": [ - true + "retained", + "restored", + "conflict", + "unknown" ] - } - }, - "required": [ - "ok" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" + }, + "runtime": { + "type": "string", + "enum": [ + "applied", + "pending", + "failed", + "unknown" + ] + }, + "runtimeGenerationId": { + "anyOf": [ + { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" }, - "jobId": { - "type": "string" + { + "type": "null" } - }, - "required": [ - "error" - ], + ] + }, + "reason": { + "type": "string", + "enum": [ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required" + ] + }, + "hostApply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "operationId", + "integrationId", + "desiredRevision", + "database", + "runtime", + "runtimeGenerationId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], "additionalProperties": false }, { @@ -34752,6 +35220,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -35051,15 +35579,45 @@ "in": "path", "name": "capability", "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true } ], "requestBody": { - "required": false, + "required": true, "content": { "application/json": { "schema": { "type": "object", - "properties": {}, + "properties": { + "expectedServiceId": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + { + "type": "null" + } + ] + }, + "expectedRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "expectedServiceId", + "expectedRevision" + ], "additionalProperties": false } } @@ -35073,15 +35631,200 @@ "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", + "operationId": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "integrationId": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "database": { + "type": "string", "enum": [ - true + "retained", + "restored", + "conflict", + "unknown" + ] + }, + "runtime": { + "type": "string", + "enum": [ + "applied", + "pending", + "failed", + "unknown" + ] + }, + "runtimeGenerationId": { + "anyOf": [ + { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "type": "string", + "enum": [ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required" + ] + }, + "hostApply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } ] } }, "required": [ - "ok" + "operationId", + "integrationId", + "desiredRevision", + "database", + "runtime", + "runtimeGenerationId" ], "additionalProperties": false } @@ -35268,6 +36011,66 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -35480,6 +36283,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } }, "tags": [ diff --git a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts index 598843da9..0a4e5d15e 100644 --- a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts +++ b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts @@ -68,6 +68,7 @@ describe.skipIf(skipDatabase)("user erasure constraints with PostgreSQL", () => "data_subject_request_task.assigned_to:n", "installed_extension.installed_by:n", "installed_integration.installed_by:n", + "integration_application_receipt.actor_id:n", "integration_secret.updated_by:n", "labeled_place.user_id:c", "mangrove_keypair.user_id:c", diff --git a/apps/api/src/db/integration-administration-schema.ts b/apps/api/src/db/integration-administration-schema.ts new file mode 100644 index 000000000..23b7d2337 --- /dev/null +++ b/apps/api/src/db/integration-administration-schema.ts @@ -0,0 +1,29 @@ +import type { IntegrationApplyReceipt } from "@openmapx/core/api-contracts"; +import { index, jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core"; +import { user } from "./auth-schema"; +/** Retained across unbinding, so a delete/reinsert cannot restore an old CAS token. */ +export const integrationBindingRevision = pgTable("integration_binding_revision", { + integrationId: text("integration_id").primaryKey(), + version: uuid("version").defaultRandom().notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(), +}); +/** Inserted in the same candidate transaction as the desired binding. */ +export const integrationApplicationReceipt = pgTable( + "integration_application_receipt", + { + operationId: text("operation_id").primaryKey(), + actorId: text("actor_id").references(() => user.id, { onDelete: "set null" }), + integrationId: text("integration_id").notNull(), + capability: text("capability").notNull(), + serviceId: text("service_id"), + candidateId: uuid("candidate_id").notNull(), + intentDigest: text("intent_digest").notNull(), + savedBindingRevision: text("saved_binding_revision").notNull(), + result: jsonb("result").$type().notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), + }, + (t) => [ + index("integration_application_target_time_idx").on(t.integrationId, t.createdAt), + index("integration_application_actor_idx").on(t.actorId), + ], +); diff --git a/apps/api/src/db/migrations/0047_admin_integration_application.sql b/apps/api/src/db/migrations/0047_admin_integration_application.sql new file mode 100644 index 000000000..b49fc3de3 --- /dev/null +++ b/apps/api/src/db/migrations/0047_admin_integration_application.sql @@ -0,0 +1,22 @@ +CREATE TABLE "integration_application_receipt" ( + "operation_id" text PRIMARY KEY NOT NULL, + "actor_id" text, + "integration_id" text NOT NULL, + "capability" text NOT NULL, + "service_id" text, + "candidate_id" uuid NOT NULL, + "intent_digest" text NOT NULL, + "saved_binding_revision" text NOT NULL, + "result" jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "integration_binding_revision" ( + "integration_id" text PRIMARY KEY NOT NULL, + "version" uuid DEFAULT gen_random_uuid() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "integration_application_receipt" ADD CONSTRAINT "integration_application_receipt_actor_id_user_id_fk" FOREIGN KEY ("actor_id") REFERENCES "public"."user"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "integration_application_target_time_idx" ON "integration_application_receipt" USING btree ("integration_id","created_at");--> statement-breakpoint +CREATE INDEX "integration_application_actor_idx" ON "integration_application_receipt" USING btree ("actor_id"); \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0047_snapshot.json b/apps/api/src/db/migrations/meta/0047_snapshot.json new file mode 100644 index 000000000..b1a6402f9 --- /dev/null +++ b/apps/api/src/db/migrations/meta/0047_snapshot.json @@ -0,0 +1,8933 @@ +{ + "id": "615f75f9-83ef-45de-a7be-63ec7174bae3", + "prevId": "d279e5e1-ae88-41e3-89d9-5e4f6579d86a", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_action_receipt": { + "name": "account_action_receipt", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mail_requested": { + "name": "mail_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_action_actor_time_idx": { + "name": "account_action_actor_time_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_action_target_time_idx": { + "name": "account_action_target_time_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_action_receipt_actor_id_user_id_fk": { + "name": "account_action_receipt_actor_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "account_action_receipt_target_id_user_id_fk": { + "name": "account_action_receipt_target_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_admin_guard": { + "name": "account_admin_guard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "active_count": { + "name": "active_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_admin_guard_singleton": { + "name": "account_admin_guard_singleton", + "value": "\"account_admin_guard\".\"id\"=1" + }, + "account_admin_guard_nonnegative": { + "name": "account_admin_guard_nonnegative", + "value": "\"account_admin_guard\".\"active_count\">=0" + } + }, + "isRLSEnabled": false + }, + "public.account_mail_outbox": { + "name": "account_mail_outbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_mail_action_kind_uq": { + "name": "account_mail_action_kind_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_mail_pending_idx": { + "name": "account_mail_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_mail_outbox_request_key_account_action_receipt_key_fk": { + "name": "account_mail_outbox_request_key_account_action_receipt_key_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "account_action_receipt", + "columnsFrom": [ + "request_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "account_mail_outbox_user_id_user_id_fk": { + "name": "account_mail_outbox_user_id_user_id_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_mail_state_check": { + "name": "account_mail_state_check", + "value": "\"account_mail_outbox\".\"state\" in ('queued','sending','sent','failed','unknown')" + }, + "account_mail_kind_check": { + "name": "account_mail_kind_check", + "value": "\"account_mail_outbox\".\"kind\" in ('verification','password_setup')" + } + }, + "isRLSEnabled": false + }, + "public.admin_action_reauthentication": { + "name": "admin_action_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admin_reauth_actor_expiry_idx": { + "name": "admin_reauth_actor_expiry_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admin_reauth_request_uq": { + "name": "admin_reauth_request_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_action_reauthentication_actor_id_user_id_fk": { + "name": "admin_action_reauthentication_actor_id_user_id_fk", + "tableFrom": "admin_action_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "admin_reauth_state_check": { + "name": "admin_reauth_state_check", + "value": "\"admin_action_reauthentication\".\"state\" in ('pending','completed','consumed','superseded','expired')" + }, + "admin_reauth_purpose_check": { + "name": "admin_reauth_purpose_check", + "value": "\"admin_action_reauthentication\".\"purpose\" in ('account.verify','account.password.set','account.reset.send','account.invite','credentials.export','credentials.import','transit.credentials','platform.domain.apply')" + } + }, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_application_actor_id_user_id_fk": { + "name": "auth_policy_application_actor_id_user_id_fk", + "tableFrom": "auth_policy_application", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_state_activated_by_user_id_fk": { + "name": "auth_policy_state_activated_by_user_id_fk", + "tableFrom": "auth_policy_state", + "tableTo": "user", + "columnsFrom": [ + "activated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_independent_password_login": { + "name": "origin_independent_password_login", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_application_receipt": { + "name": "integration_application_receipt", + "schema": "", + "columns": { + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "saved_binding_revision": { + "name": "saved_binding_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_application_target_time_idx": { + "name": "integration_application_target_time_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_application_actor_idx": { + "name": "integration_application_actor_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_application_receipt_actor_id_user_id_fk": { + "name": "integration_application_receipt_actor_id_user_id_fk", + "tableFrom": "integration_application_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_binding_revision": { + "name": "integration_binding_revision", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 412c8a606..dae076538 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -330,6 +330,13 @@ "when": 1793195693647, "tag": "0046_admin_account_provisioning", "breakpoints": true + }, + { + "idx": 47, + "version": "7", + "when": 1793203302497, + "tag": "0047_admin_integration_application", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 3c7b4a72f..1df84157c 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -65,6 +65,7 @@ export { } from "./installed-extension-schema"; // Installed community integrations export { installedIntegration } from "./installed-integration-schema"; +export * from "./integration-administration-schema"; // Integration config export { integrationConfig } from "./integration-config-schema"; // Integration secrets diff --git a/apps/api/src/integration-host.binding-application.test.ts b/apps/api/src/integration-host.binding-application.test.ts new file mode 100644 index 000000000..83deb74ec --- /dev/null +++ b/apps/api/src/integration-host.binding-application.test.ts @@ -0,0 +1,196 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; +import Fastify from "fastify"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + selected: "engine-a" as string | null, + root: "", + failBindings: false, + registry: null as InstanceType | null, +})); +vi.mock("./redis", () => ({ redis: null })); +vi.mock("./db", () => { + const rows = (): unknown => + Object.assign(Promise.resolve([]), { where: rows, limit: async () => [] }); + return { db: { select: () => ({ from: rows }) }, sql: { unsafe: async () => [] } }; +}); +vi.mock("./services/capability-bindings", () => ({ + loadAllBindingsByIntegration: async () => { + if (state.failBindings) throw Error("Synthetic unavailable binding query"); + return new Map([ + ["binding-test", new Map(state.selected ? [["routing-engine", state.selected]] : [])], + ]); + }, + listBindingsForIntegration: async () => + state.selected ? [{ capability: "routing-engine", serviceId: state.selected }] : [], + setBinding: async (_key: unknown, id: string) => { + state.selected = id; + }, + removeBinding: async () => { + state.selected = null; + }, +})); +vi.mock("@openmapx/core/server", async (original) => ({ + ...(await original()), + findRepoRoot: () => state.root, +})); +vi.mock("./utils/require-auth", () => ({ + requireAuth: async () => { + throw Error("Synthetic route has no login"); + }, +})); +vi.mock("./utils/require-admin", () => ({ + requireAdmin: async () => ({ + user: { id: "synthetic-admin", role: "admin" }, + session: { id: "synthetic-session" }, + }), + getAdminSession: () => ({ user: { id: "synthetic-admin" } }), +})); +vi.mock("./services/attribution", () => ({ + AttributionIndex: { + init: async () => ({ setIntegrationManifests() {}, reload: async () => {}, close() {} }), + }, + defaultMotisLicenseFile: () => null, + getAttributionIndex: () => null, + setAttributionIndex() {}, +})); +vi.mock("./services/provider-health/registry", () => ({ + getProviderHealth: () => null, + ProviderHealth: { init: async () => ({ close() {} }) }, + setProviderHealth() {}, +})); +vi.mock("./services/integration-health", () => ({ + executeAllIntegrationHealthChecks: async () => [], + getCachedIntegrationHealthSnapshot: () => ({ updatedAt: null, results: [] }), +})); + +import { initIntegrations, shutdownIntegrations } from "./integration-host"; +import { registerCapabilityBindingRoutes } from "./routes/admin-capability-bindings"; +import { integrationsRoute } from "./routes/integrations"; + +let root: string, app: ReturnType; +beforeEach(async () => { + state.selected = "engine-a"; + state.failBindings = false; + vi.stubEnv("OPS_AGENT_URL", ""); + vi.stubEnv(services.APPLIED_SERVICES_ENV, "engine-a,engine-b"); + root = mkdtempSync(join(tmpdir(), "openmapx-binding-runtime-")); + state.root = root; + for (const id of ["engine-a", "engine-b"]) { + const dir = join(root, "services", id); + mkdirSync(dir, { recursive: true }); + writeFileSync( + join(dir, "service.json"), + JSON.stringify({ + id, + name: id, + version: "1.0.0", + quality: "built-in", + container: { image: "example.invalid/synthetic", tag: "1", expose: [8080] }, + provides: ["routing-engine", "unused-capability"], + }), + ); + } + const registryModule = await import("./services/service-registry"); + await registryModule.initServiceRegistry(); + state.registry = registryModule.getServiceRegistry(); + expect(state.registry.list()).toHaveLength(2); + const directory = join(root, "integrations", "binding-test"); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id: "binding-test", + version: "1.0.0", + author: "Test", + license: "MIT", + domains: ["knowledge"], + quality: "built-in", + requires: [{ capability: "routing-engine" }], + backend: { routes: true }, + }), + ); + writeFileSync( + join(directory, "index.ts"), + `export function setup(ctx){const captured=ctx.getRequiredService('routing-engine');ctx.registerRoute('GET','/bound',async(_req,reply)=>reply.send({serviceId:captured?.serviceId??null}));}`, + ); + app = Fastify(); + await app.register(integrationsRoute, { prefix: "/api" }); + await initIntegrations(app, [join(root, "integrations")]); + await registerCapabilityBindingRoutes(app); + await app.ready(); +}); +afterEach(async () => { + await shutdownIntegrations(); + await app?.close(); + if (root) rmSync(root, { recursive: true, force: true }); + state.registry = null; + vi.unstubAllEnvs(); +}); +it("activates a saved capability binding in the actual trusted provider that captured its requirement", async () => { + expect((await app.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "engine-a", + }); + state.selected = "engine-b"; + await (await import("./integration-host")).reloadIntegrations(); + expect((await app.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "engine-b", + }); +}); + +it("keeps the last provider active when a reload cannot acquire binding authority", async () => { + const host = await import("./integration-host"); + state.failBindings = true; + await expect(host.reloadIntegrations()).rejects.toThrow(); + expect((await app.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "engine-a", + }); +}); +it("exposes the actual activated generation without leaking configuration values", async () => { + const host = await import("./integration-host"), + inspect = (host as unknown as { inspectIntegrationRuntime: (id: string) => unknown }) + .inspectIntegrationRuntime; + expect(typeof inspect).toBe("function"); + expect(inspect("binding-test")).toMatchObject({ + generationId: expect.any(String), + configRevision: expect.any(String), + bindings: { "routing-engine": "engine-a" }, + }); + expect(inspect("missing")).toBeNull(); +}); + +it("rejects a binding for a capability the integration never declared", async () => { + (await import("./utils/rate-limit")).serviceActionLimit.reset(); + const result = await app.inject({ + method: "POST", + url: "/api/admin/integrations/binding-test/bindings/unused-capability", + headers: { "idempotency-key": "binding-undeclared-test-0001" }, + payload: { + serviceId: "engine-b", + expectedServiceId: "engine-a", + expectedRevision: "a".repeat(64), + }, + }); + expect(result.statusCode).toBe(400); + expect(state.selected).toBe("engine-a"); +}); +it("rejects a disabled provider before saving a desired binding", async () => { + (await import("./utils/rate-limit")).serviceActionLimit.reset(); + state.registry?.applyEnabledIds(new Set(["engine-a"])); + (await import("./utils/rate-limit")).serviceActionLimit.reset(); + const result = await app.inject({ + method: "POST", + url: "/api/admin/integrations/binding-test/bindings/routing-engine", + headers: { "idempotency-key": "binding-disabled-test-0001" }, + payload: { + serviceId: "engine-b", + expectedServiceId: "engine-a", + expectedRevision: "a".repeat(64), + }, + }); + expect(result.statusCode).toBe(400); + expect(state.selected).toBe("engine-a"); +}); diff --git a/apps/api/src/integration-host.ts b/apps/api/src/integration-host.ts index 175812f6e..d2501b027 100644 --- a/apps/api/src/integration-host.ts +++ b/apps/api/src/integration-host.ts @@ -1,4 +1,4 @@ -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; @@ -69,6 +69,10 @@ import { executeAllIntegrationHealthChecks, getCachedIntegrationHealthSnapshot, } from "./services/integration-health"; +import { + integrationRuntimeEvidence, + stageIntegrationRuntimeEvidence, +} from "./services/integration-runtime-evidence"; import { getMetricsRecorder } from "./services/metrics/recorder"; import { cursorCodecFromEnvironment } from "./services/opaque-cursor"; import { @@ -91,6 +95,11 @@ const integrations = new Map(); export type IntegrationDirectoryInput = string | { directory: string; isBuiltIn: boolean }; type NormalizedIntegrationDirectory = { directory: string; isBuiltIn: boolean }; +/** A read of the current host registry, never a projection of saved bindings. */ +export function inspectIntegrationRuntime(integrationId: string) { + return integrationRuntimeEvidence(integrations.get(integrationId)); +} + // Stored for reload support // biome-ignore lint/suspicious/noExplicitAny: accept any Fastify logger variant let _fastify: FastifyInstance | null = null; @@ -770,6 +779,7 @@ export async function initIntegrations( fastify: FastifyInstance, integrationDirs: IntegrationDirectoryInput[], ): Promise { + const generationId = randomUUID(); _fastify = fastify; _integrationDirs = normalizeIntegrationDirs(integrationDirs); resetIntegrationRoutes(); @@ -968,6 +978,7 @@ export async function initIntegrations( } } + stageIntegrationRuntimeEvidence(integration, generationId); activationScope.complete(); integrations.set(id, integration); log.info(`Integration ${id} v${manifest.version ?? "unknown"} loaded successfully`); @@ -1190,6 +1201,7 @@ export async function reloadIntegrations( async function doReloadIntegrations( context?: AdministrativeConfigurationReadContext, ): Promise { + const generationId = randomUUID(); if (!_fastify) throw new Error("Integration host not initialized"); if (context) assertAdministrativeConfigurationRead(context); @@ -1224,10 +1236,9 @@ async function doReloadIntegrations( // Reload capability bindings and service registry for requires: resolution. let reloadBindings = new Map>(); try { - reloadBindings = await loadAllBindingsByIntegration(); + reloadBindings = await loadAllBindingsByIntegration(context); } catch { - if (context) throw Error("Captured integration bindings unavailable"); - _fastify.log.debug("Capability bindings unavailable during reload"); + throw Error("Captured integration bindings unavailable"); } let reloadRegistry: ReturnType | null = null; @@ -1364,6 +1375,7 @@ async function doReloadIntegrations( } } + stageIntegrationRuntimeEvidence(integration, generationId); next.set(id, integration); log.info(`Integration ${id} v${manifest.version ?? "unknown"} reloaded successfully`); stagedEventBus.emit({ type: "integration.loaded", integrationId: id }); diff --git a/apps/api/src/privacy/catalogue.ts b/apps/api/src/privacy/catalogue.ts index c09965979..d85e5995d 100644 --- a/apps/api/src/privacy/catalogue.ts +++ b/apps/api/src/privacy/catalogue.ts @@ -348,6 +348,7 @@ const registrationDefinitions: SubjectDataRegistration[] = [ { type: "actor_id", field: "admin_audit_log.actor_id" }, { type: "target_id", field: "admin_audit_log.target_id" }, { type: "actor_id", field: "auth_policy_application.actor_id" }, + { type: "actor_id", field: "integration_application_receipt.actor_id" }, { type: "actor_id", field: "account_action_receipt.actor_id" }, { type: "target_id", field: "account_action_receipt.target_id" }, { type: "actor_id", field: "auth_policy_state.activated_by" }, diff --git a/apps/api/src/privacy/openmapx-collectors.ts b/apps/api/src/privacy/openmapx-collectors.ts index 517f357f7..cb1c0505c 100644 --- a/apps/api/src/privacy/openmapx-collectors.ts +++ b/apps/api/src/privacy/openmapx-collectors.ts @@ -23,6 +23,7 @@ import { dataSubjectRequestTask, installedExtension, installedIntegration, + integrationApplicationReceipt, integrationSecret, labeledPlace, mangroveKeypair, @@ -885,6 +886,32 @@ export async function* streamOpenMapxRegistrationRecords( }), }, ); + yield* emit( + database + .select({ + operationId: integrationApplicationReceipt.operationId, + integrationId: integrationApplicationReceipt.integrationId, + capability: integrationApplicationReceipt.capability, + createdAt: integrationApplicationReceipt.createdAt, + database: sql`${integrationApplicationReceipt.result}->>'database'`, + runtime: sql`${integrationApplicationReceipt.result}->>'runtime'`, + }) + .from(integrationApplicationReceipt) + .where( + and( + eq(integrationApplicationReceipt.actorId, context.userId), + lte(integrationApplicationReceipt.createdAt, cutoff), + ), + ) + .orderBy(asc(integrationApplicationReceipt.operationId)), + { + policyCodes: ["rights-of-others", "foreign-identifiers-redacted"], + map: ({ operationId, ...row }) => ({ + recordId: `integration-application-${pseudonymousIdentifier(operationId)}`, + ...row, + }), + }, + ); return; case "admin-jobs-attribution": yield* emit( diff --git a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts index c2267a7b0..32fabb3c4 100644 --- a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts @@ -9,7 +9,7 @@ import { integrationConfigDbFixture } from "../../services/__tests__/helpers/int import type { IntegrationImportPorts } from "../../services/integration-config-import"; import { TrustedConfigurationApplyError } from "../../services/trusted-config-outcome"; import { makeCsrfGuardHook } from "../../utils/csrf"; -import { healthCheckSweepLimit, RateLimiter } from "../../utils/rate-limit"; +import { healthCheckSweepLimit, RateLimiter, serviceActionLimit } from "../../utils/rate-limit"; import { installAdminRouteMocks } from "./admin-test-helpers"; import { administrationSdkFetch } from "./helpers/administration-sdk-fetch"; @@ -18,8 +18,15 @@ const { requireAdmin: mockRequireAdmin, session: fakeSession } = installAdminRou const mockJobRunnerEnqueue = vi.fn().mockResolvedValue("job-main-123"); const mockJobRunnerCancel = vi.fn().mockResolvedValue(true); const mockImmediateImport = vi.fn(); +const mockImmediateBinding = vi.fn(); +const expectedBinding = { expectedServiceId: null, expectedRevision: "a".repeat(64) }; +const bindingHeaders = () => { + serviceActionLimit.reset(); + return { "idempotency-key": crypto.randomUUID() }; +}; vi.mock("../../services/job-runner.js", () => ({ jobRunner: { + applyImmediateIntegrationBinding: (...args: unknown[]) => mockImmediateBinding(...args), enqueue: (...args: unknown[]) => mockJobRunnerEnqueue(...args), cancel: (...args: unknown[]) => mockJobRunnerCancel(...args), importImmediateIntegrationConfiguration: (...args: unknown[]) => mockImmediateImport(...args), @@ -85,6 +92,11 @@ vi.mock("../../db/index.js", () => ({ vi.mock("../../db/schema.js", () => ({ user: { id: "id", banned: "banned" }, session: { id: "id", userId: "userId", updatedAt: "updatedAt" }, + integrationApplicationReceipt: { + integrationId: "integration_id", + createdAt: "created_at", + result: "result", + }, adminAuditLog: { id: "id", action: "action", @@ -124,7 +136,7 @@ const MOCK_INTEGRATION = { quality: "built-in", dependencies: [], - requires: [], + requires: [{ capability: "routing" }], configSchema: { type: "object", @@ -147,6 +159,7 @@ const mockReloadIntegrations = vi const mockResolveConfigWithSources = vi.fn().mockResolvedValue({}); vi.mock("../../integration-host.js", () => ({ + inspectIntegrationRuntime: () => null, getAllIntegrations: (...args: unknown[]) => mockGetAllIntegrations(...args), getIntegration: (...args: unknown[]) => mockGetIntegration(...args), reloadIntegrations: (...args: unknown[]) => mockReloadIntegrations(...args), @@ -197,6 +210,19 @@ vi.mock("../../services/admin-ops.js", () => ({ isDockerAvailable: vi.fn().mockResolvedValue(false), })); +vi.mock("../../services/administrative-configuration-writer", () => ({ + withAdministrativeConfigurationRead: async ( + _options: unknown, + run: (read: unknown) => Promise, + ) => run({}), +})); +vi.mock("../../services/integration-binding-state", () => ({ + capturedBindingState: () => ({ + bindings: { routing: "fixture-service" }, + revision: "a".repeat(64), + desiredRevision: "a".repeat(64), + }), +})); const setBinding = vi.fn(); const removeBinding = vi.fn(); const getRegistry = vi.fn(); @@ -257,7 +283,17 @@ beforeEach(() => { mockIsSecretsConfigured.mockReset().mockReturnValue(true); getRegistry.mockReset().mockReturnValue({ get: (id: string) => - id === "fixture-service" ? { manifest: { id, provides: ["routing"] } } : undefined, + id === "fixture-service" + ? { enabled: true, manifest: { id, provides: ["routing"] } } + : undefined, + }); + mockImmediateBinding.mockReset().mockResolvedValue({ + operationId: "binding-sdk-fixture", + integrationId: "geocoding-maptiler", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "applied", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", }); setBinding.mockReset().mockResolvedValue(undefined); removeBinding.mockReset().mockResolvedValue(undefined); @@ -349,8 +385,21 @@ describe("actual integration producers through generated SDK", () => { () => sdk.reloadAdminIntegration(id, undefined, options), () => sdk.listAdminIntegrationBindings(id, options), () => - sdk.setAdminIntegrationBinding(id, "routing", { serviceId: "fixture-service" }, options), - () => sdk.deleteAdminIntegrationBinding(id, "routing", undefined, options), + sdk.setAdminIntegrationBinding( + id, + "routing", + { serviceId: "fixture-service", ...expectedBinding }, + bindingHeaders(), + options, + ), + () => + sdk.deleteAdminIntegrationBinding( + id, + "routing", + expectedBinding, + bindingHeaders(), + options, + ), () => sdk.listAdminCredentials(options), () => sdk.getAdminIntegrationCredentials(id, options), () => { @@ -541,25 +590,34 @@ describe("actual integration producers through generated SDK", () => { await sdk.setAdminIntegrationBinding( "fixture", "routing", - { serviceId: "fixture-service" }, + { serviceId: "fixture-service", ...expectedBinding }, + bindingHeaders(), { client }, ) ).status, ).toBe(403); mockRequireAdmin.mockResolvedValue(fakeSession); - setBinding.mockRejectedValueOnce(Error("synthetic-secret")); + mockImmediateBinding.mockRejectedValueOnce(Error("synthetic-secret")); const failed = await sdk.setAdminIntegrationBinding( "fixture", "routing", - { serviceId: "fixture-service" }, + { serviceId: "fixture-service", ...expectedBinding }, + bindingHeaders(), { client }, ); expect(failed.status).toBe(500); expect(JSON.stringify(failed)).not.toContain("synthetic-secret"); - removeBinding.mockRejectedValueOnce(Error("synthetic-secret")); + mockImmediateBinding.mockRejectedValueOnce(Error("synthetic-secret")); expect( - (await sdk.deleteAdminIntegrationBinding("fixture", "routing", undefined, { client })) - .status, + ( + await sdk.deleteAdminIntegrationBinding( + "fixture", + "routing", + expectedBinding, + bindingHeaders(), + { client }, + ) + ).status, ).toBe(500); getRegistry.mockImplementationOnce(() => { throw Error("unavailable"); @@ -569,7 +627,8 @@ describe("actual integration producers through generated SDK", () => { await sdk.setAdminIntegrationBinding( "fixture", "routing", - { serviceId: "fixture-service" }, + { serviceId: "fixture-service", ...expectedBinding }, + bindingHeaders(), { client }, ) ).status, diff --git a/apps/api/src/routes/admin-capability-bindings.ts b/apps/api/src/routes/admin-capability-bindings.ts index 8836568f3..2026466e7 100644 --- a/apps/api/src/routes/admin-capability-bindings.ts +++ b/apps/api/src/routes/admin-capability-bindings.ts @@ -1,98 +1,164 @@ -import { adminIntegrationBindingSetSchema } from "@openmapx/core/api-contracts"; +import { + adminIntegrationBindingDeleteSchema, + adminIntegrationBindingSetSchema, +} from "@openmapx/core/api-contracts"; import { services as coreServices } from "@openmapx/core/server"; +import { desc, eq } from "drizzle-orm"; import type { FastifyInstance } from "fastify"; +import { db } from "../db"; +import { integrationApplicationReceipt } from "../db/schema"; +import { getIntegration, inspectIntegrationRuntime } from "../integration-host"; +import { AdminJobFailure, JobObservationPending } from "../services/admin-job-failure"; +import { withAdministrativeConfigurationRead } from "../services/administrative-configuration-writer"; import { - listBindingsForIntegration, - removeBinding, - setBinding, -} from "../services/capability-bindings"; + createDirectAdminOpsKey, + parseDirectOpsIdempotency, +} from "../services/direct-ops-idempotency"; +import { IntegrationBindingConflict } from "../services/integration-binding-participant"; +import { capturedBindingState } from "../services/integration-binding-state"; +import { jobRunner } from "../services/job-runner"; import { getServiceRegistry } from "../services/service-registry"; -import { requireAdmin } from "../utils/require-admin"; +import { serviceActionLimit } from "../utils/rate-limit"; +import { getAdminSession, requireAdmin } from "../utils/require-admin"; import { declareRouteAuth } from "../utils/route-auth"; import { registerIntegrationAdminRoute } from "./admin-integration-contract"; -const { getProvidedCapabilityNames } = coreServices; - -export async function registerCapabilityBindingRoutes( - // biome-ignore lint/suspicious/noExplicitAny: accept any Fastify logger variant - app: FastifyInstance, -): Promise { +export async function registerCapabilityBindingRoutes(app: FastifyInstance): Promise { await app.register( async (child) => { declareRouteAuth(child, "admin"); - // Preserve the existing POST framework-body-before-handler-auth boundary. child.addHook("preValidation", async (request, reply) => { - if ( - request.method === "POST" && - !adminIntegrationBindingSetSchema.safeParse(request.body).success - ) + const schema = + request.method === "POST" + ? adminIntegrationBindingSetSchema + : request.method === "DELETE" + ? adminIntegrationBindingDeleteSchema + : null; + if (schema && !schema.safeParse(request.body).success) return reply.code(400).send({ error: "Invalid administration body" }); }); - + const guard = { + beforeValidation: async ( + request: Parameters[0], + reply: Parameters>[1], + ) => { + request.adminSession = await requireAdmin(request); + await serviceActionLimit.preHandler()(request, reply); + }, + }; registerIntegrationAdminRoute( child, "listAdminIntegrationBindings", - async (req, _reply) => { - const rows = await listBindingsForIntegration(req.params.integrationId); - return { bindings: rows }; - }, + async (req) => + withAdministrativeConfigurationRead({}, async (read) => { + const desired = capturedBindingState(read, req.params.integrationId), + [last] = await db + .select({ result: integrationApplicationReceipt.result }) + .from(integrationApplicationReceipt) + .where(eq(integrationApplicationReceipt.integrationId, req.params.integrationId)) + .orderBy(desc(integrationApplicationReceipt.createdAt)) + .limit(1); + return { + bindings: Object.entries(desired.bindings).map(([capability, serviceId]) => ({ + capability, + serviceId, + })), + revision: desired.revision, + desiredRevision: desired.desiredRevision, + runtime: inspectIntegrationRuntime(req.params.integrationId), + lastApplication: last?.result ?? null, + }; + }), { beforeValidation: async (request) => { - await requireAdmin(request); + request.adminSession = await requireAdmin(request); }, }, ); - - registerIntegrationAdminRoute( - child, - "setAdminIntegrationBinding", - async (req, reply) => { + const apply = async ( + req: { + params: { integrationId: string; capability: string }; + body: { expectedServiceId: string | null; expectedRevision: string; serviceId?: string }; + headers: Record; + }, + actorId: string, + ) => { + const integration = getIntegration(req.params.integrationId); + if ( + !integration?.manifest.requires?.some( + (entry) => entry.capability === req.params.capability, + ) + ) + throw new IntegrationBindingConflict("BINDING_CAPABILITY_NOT_DECLARED"); + if (req.body.serviceId) { let registry: ReturnType; try { registry = getServiceRegistry(); } catch { - reply.status(503); - return { error: "Service registry not available" }; - } - const svc = registry.get(req.body.serviceId); - if (!svc) { - reply.status(400); - return { error: `Unknown service: ${req.body.serviceId}` }; + throw new JobObservationPending("transport"); } - if (!getProvidedCapabilityNames(svc.manifest.provides).includes(req.params.capability)) { - reply.status(400); - return { - error: `Service ${svc.manifest.id} does not provide capability ${req.params.capability}`, - }; + const service = registry.get(req.body.serviceId); + if ( + !service?.enabled || + !coreServices + .getProvidedCapabilityNames(service.manifest.provides) + .includes(req.params.capability) + ) + throw new IntegrationBindingConflict("BINDING_SERVICE_UNAVAILABLE"); + } + return jobRunner.applyImmediateIntegrationBinding({ + actorId, + operationKey: createDirectAdminOpsKey( + actorId, + parseDirectOpsIdempotency(req.headers["idempotency-key"]), + ), + input: { ...req.params, ...req.body, serviceId: req.body.serviceId ?? null }, + }); + }; + const errors = ( + error: unknown, + reply: { code: (status: number) => { send: (body: { error: string }) => unknown } }, + ) => { + if (error instanceof IntegrationBindingConflict) + return reply + .code( + error.code === "BINDING_VERSION_CONFLICT" || error.code === "BINDING_SOURCE_CHANGED" + ? 409 + : 400, + ) + .send({ error: error.code }); + if (error instanceof AdminJobFailure && error.result.errorCode === "conflict") + return reply.code(409).send({ error: "Retained binding intent conflicts" }); + if (error instanceof JobObservationPending) + return reply.code(503).send({ + error: + "Binding saved or awaiting reconciliation; inspect its operation status before retrying", + }); + throw error; + }; + registerIntegrationAdminRoute( + child, + "setAdminIntegrationBinding", + async (req, reply) => { + try { + return await apply(req, getAdminSession(req).user.id); + } catch (error) { + return errors(error, reply); } - await setBinding( - { integrationId: req.params.integrationId, capability: req.params.capability }, - req.body.serviceId, - ); - return { ok: true }; - }, - { - beforeValidation: async (request) => { - await requireAdmin(request); - }, }, + guard, ); - registerIntegrationAdminRoute( child, "deleteAdminIntegrationBinding", - async (req, _reply) => { - await removeBinding({ - integrationId: req.params.integrationId, - capability: req.params.capability, - }); - return { ok: true }; - }, - { - beforeValidation: async (request) => { - await requireAdmin(request); - }, + async (req, reply) => { + try { + return await apply(req, getAdminSession(req).user.id); + } catch (error) { + return errors(error, reply); + } }, + guard, ); }, { prefix: "/api" }, diff --git a/apps/api/src/services/administrative-configuration-context.ts b/apps/api/src/services/administrative-configuration-context.ts index fb4cb96c8..609cec0f8 100644 --- a/apps/api/src/services/administrative-configuration-context.ts +++ b/apps/api/src/services/administrative-configuration-context.ts @@ -13,6 +13,15 @@ export interface AdministrativeConfigurationReadContext { readonly integrationRows: readonly (typeof schema.integrationConfig.$inferSelect)[]; readonly serviceSecretRows: readonly (typeof schema.serviceSecret.$inferSelect)[]; readonly integrationSecretRows: readonly (typeof schema.integrationSecret.$inferSelect)[]; + /** Actual transaction rows include PostgreSQL xmin for delete/reinsert CAS. */ + readonly bindingRows?: readonly (typeof schema.capabilityBinding.$inferSelect & { + _rowVersion?: string; + })[]; + readonly bindingRevisionRows?: readonly (typeof schema.integrationBindingRevision.$inferSelect & { + _rowVersion?: string; + })[]; + /** Original candidate's proposed map, used only for its captured reload. */ + readonly integrationBindingSnapshot?: Readonly>>>; /** Only the committed-candidate reload scope supplies an already resolved snapshot. */ readonly integrationReloadSnapshot?: Readonly< Record>> diff --git a/apps/api/src/services/administrative-configuration-writer-owner.ts b/apps/api/src/services/administrative-configuration-writer-owner.ts index 406c71ce1..544dafa9f 100644 --- a/apps/api/src/services/administrative-configuration-writer-owner.ts +++ b/apps/api/src/services/administrative-configuration-writer-owner.ts @@ -154,7 +154,21 @@ async function capture( _rowVersion: query`xmin::text`, }) .from(schema.integrationSecret); + const bindingRows = await tx + .select({ + ...getTableColumns(schema.capabilityBinding), + _rowVersion: query`xmin::text`, + }) + .from(schema.capabilityBinding); + const bindingRevisionRows = await tx + .select({ + ...getTableColumns(schema.integrationBindingRevision), + _rowVersion: query`xmin::text`, + }) + .from(schema.integrationBindingRevision); const context = openAdministrativeConfigurationRead({ + bindingRows, + bindingRevisionRows, services, integrations, env: Object.freeze({ ...process.env }), diff --git a/apps/api/src/services/capability-bindings.ts b/apps/api/src/services/capability-bindings.ts index 722dd7271..117a55e14 100644 --- a/apps/api/src/services/capability-bindings.ts +++ b/apps/api/src/services/capability-bindings.ts @@ -1,6 +1,10 @@ import { and, eq } from "drizzle-orm"; import { db } from "../db"; import { capabilityBinding } from "../db/schema"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./administrative-configuration-context"; export interface BindingKey { integrationId: string; @@ -58,7 +62,26 @@ export async function removeBinding(key: BindingKey): Promise { ); } -export async function loadAllBindingsByIntegration(): Promise>> { +export async function loadAllBindingsByIntegration( + context?: AdministrativeConfigurationReadContext, +): Promise>> { + if (context) { + assertAdministrativeConfigurationRead(context); + if (context.integrationBindingSnapshot) + return new Map( + Object.entries(context.integrationBindingSnapshot).map(([id, bindings]) => [ + id, + new Map(Object.entries(bindings)), + ]), + ); + if (!context.bindingRows) throw Error("Captured binding authority unavailable"); + const out = new Map>(); + for (const row of context.bindingRows) { + if (!out.has(row.integrationId)) out.set(row.integrationId, new Map()); + out.get(row.integrationId)?.set(row.capability, row.serviceId); + } + return out; + } const rows = await db .select({ integrationId: capabilityBinding.integrationId, diff --git a/apps/api/src/services/configuration-commit-participant.ts b/apps/api/src/services/configuration-commit-participant.ts index d841f4c17..bcacb93fa 100644 --- a/apps/api/src/services/configuration-commit-participant.ts +++ b/apps/api/src/services/configuration-commit-participant.ts @@ -16,9 +16,11 @@ export async function withConfigurationCommitParticipant( const { isLegacyIntegrationImportParticipant } = await import( "./configuration-migration/legacy-integration-import-intent" ); + const { isIntegrationBindingParticipant } = await import("./integration-binding-participant"); // Exact installed built-in issuer, never a registration or caller callback. if ( - !isLegacyIntegrationImportParticipant(participant, writer) || + (!isLegacyIntegrationImportParticipant(participant, writer) && + !isIntegrationBindingParticipant(participant, writer)) || participant.actorId !== writer.actorId || participant.operationKey !== writer.operationKey || (state.participant && state.participant !== participant) diff --git a/apps/api/src/services/integration-application-postgres.test.ts b/apps/api/src/services/integration-application-postgres.test.ts new file mode 100644 index 000000000..f15e8c668 --- /dev/null +++ b/apps/api/src/services/integration-application-postgres.test.ts @@ -0,0 +1,47 @@ +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { capabilityBinding, integrationBindingRevision } from "../db/schema"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const databaseTests = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1" ? describe : describe.skip; +databaseTests("integration application binding source with migrated PostgreSQL", () => { + let fixture: Awaited>; + beforeAll(async () => { + fixture = await createMigratedPostgresFixture(); + vi.doMock("../db", () => ({ db: fixture.db, sql: fixture.sql })); + }, 90000); + afterAll(async () => { + await fixture?.stop(); + vi.doUnmock("../db"); + }); + it("detects a delete/reinsert ABA in the exact read version while preserving the semantic desired revision", async () => { + const { readBindingState } = await import("./integration-binding-state"), + integrationId = "binding-aba"; + const empty = await readBindingState(integrationId, fixture.db); + await fixture.db + .insert(capabilityBinding) + .values({ integrationId, capability: "routing-engine", serviceId: "engine-a" }); + const first = await readBindingState(integrationId, fixture.db); + expect(first.revision).not.toBe(empty.revision); + expect(first.bindings).toEqual({ "routing-engine": "engine-a" }); + await fixture.db + .delete(capabilityBinding) + .where(eq(capabilityBinding.integrationId, integrationId)); + await fixture.db + .insert(capabilityBinding) + .values({ integrationId, capability: "routing-engine", serviceId: "engine-a" }); + const second = await readBindingState(integrationId, fixture.db); + expect(second.desiredRevision).toBe(first.desiredRevision); + expect(second.revision).not.toBe(first.revision); + }); + it("retains a changed revision when an explicit binding is removed back to automatic selection", async () => { + const { readBindingState } = await import("./integration-binding-state"), + integrationId = "binding-empty-version"; + const before = await readBindingState(integrationId, fixture.db); + await fixture.db.insert(integrationBindingRevision).values({ integrationId }); + const after = await readBindingState(integrationId, fixture.db); + expect(after.bindings).toEqual({}); + expect(after.desiredRevision).toBe(before.desiredRevision); + expect(after.revision).not.toBe(before.revision); + }); +}); diff --git a/apps/api/src/services/integration-application.test.ts b/apps/api/src/services/integration-application.test.ts new file mode 100644 index 000000000..c46efaab0 --- /dev/null +++ b/apps/api/src/services/integration-application.test.ts @@ -0,0 +1,380 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import Fastify from "fastify"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { ownedServiceHost } from "../../../ops-agent/src/__fixtures__/owned-service-host"; +import { openOpsJobJournal } from "../../../ops-agent/src/journal"; +import { buildOpsAgentServer } from "../../../ops-agent/src/server"; +import { createConfigurationPostgresFixture } from "../test/configuration-postgres-fixture"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const state = vi.hoisted(() => ({ + fixture: null as unknown as Awaited>, + root: "", +})); +vi.mock("../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); +vi.mock("@openmapx/core/server", async (original) => ({ + ...(await original()), + findRepoRoot: () => state.root, +})); +// Infrastructure-only ports; provider activation, API, SQL writer, candidate, +// production registry/agent, signed receipts and target runtime remain actual. +vi.mock("../redis", () => ({ redis: null })); +vi.mock("./attribution", () => ({ + AttributionIndex: { + init: async () => ({ setIntegrationManifests() {}, reload: async () => {}, close() {} }), + }, + defaultMotisLicenseFile: () => null, + getAttributionIndex: () => null, + setAttributionIndex() {}, +})); +vi.mock("./provider-health/registry", () => ({ + getProviderHealth: () => null, + ProviderHealth: { init: async () => ({ close() {} }) }, + setProviderHealth() {}, +})); +vi.mock("./integration-health", () => ({ + executeAllIntegrationHealthChecks: async () => [], + getCachedIntegrationHealthSnapshot: () => ({ updatedAt: null, results: [] }), +})); +describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "actual binding candidate / API / agent / PostgreSQL / captured provider", + () => { + let raceAfterPreview = false; + let migrated: Awaited>, + host: Awaited>, + api: ReturnType, + agent: ReturnType, + installation: ReturnType; + const headers = { "x-openmapx-local-admin": "binding-owned-admin" }; + beforeAll(async () => { + migrated = await createMigratedPostgresFixture(); + }, 90000); + afterAll(async () => { + await migrated?.stop(); + }); + beforeEach(async () => { + vi.stubEnv("OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL", migrated.connectionUri); + state.fixture = await createConfigurationPostgresFixture(); + raceAfterPreview = false; + host = await ownedServiceHost({ + extraServiceIds: ["engine-b"], + capabilities: { motis: ["routing-engine"], "engine-b": ["routing-engine"] }, + }); + state.root = host.root; + const root = join(host.root, "integrations"), + directory = join(root, "binding-test"); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id: "binding-test", + version: "1.0.0", + author: "Synthetic fixture", + license: "MIT", + domains: ["knowledge"], + quality: "built-in", + requires: [{ capability: "routing-engine" }], + backend: { routes: true }, + }), + ); + writeFileSync( + join(directory, "index.ts"), + `export function setup(ctx){const captured=ctx.getRequiredService('routing-engine');ctx.registerRoute('GET','/bound',async(_req,reply)=>reply.send({serviceId:captured?.serviceId??null}));}`, + ); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + agent = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + controlInstallation: installation, + }); + agent.addHook("onSend", async (request, _reply, payload) => { + if ( + raceAfterPreview && + (request.body as { operation?: { kind?: string } })?.operation?.kind === + "configuration.hardlinks.preview" + ) { + raceAfterPreview = false; + await state.fixture + .sql`UPDATE capability_binding SET updated_at=now() WHERE integration_id='binding-test'`; + } + return payload; + }); + const baseUrl = await agent.listen({ host: "127.0.0.1", port: 0 }), + tokenFile = join(host.root, "api-token"); + writeFileSync(tokenFile, host.token, { mode: 0o600 }); + for (const [key, value] of Object.entries({ + OPS_AGENT_URL: baseUrl, + OPS_AGENT_TOKEN_FILE: tokenFile, + OPS_TRUSTED_CONFIG_DIR: host.queue, + OPS_TRUSTED_CONFIG_UID: String(statSync(host.queue).uid), + OPS_TRUSTED_CONFIG_GID: String(statSync(host.queue).gid), + OPENMAPX_SECRETS_KEY: "71".repeat(32), + OPENMAPX_LOCAL_ADMIN_TOKEN: "binding-owned-admin", + OPENMAPX_DISABLE_LOCALHOST_AUTH: "0", + BETTER_AUTH_SECRET: "binding-owned-synthetic-auth-secret-only", + BETTER_AUTH_URL: "http://localhost:3001", + [services.APPLIED_SERVICES_ENV]: "motis,engine-b", + })) + vi.stubEnv(key, value); + await state.fixture + .sql`INSERT INTO capability_binding(integration_id,capability,service_id) VALUES ('binding-test','routing-engine','motis')`; + await (await import("./service-registry")).initServiceRegistry(); + api = Fastify(); + await api.register((await import("../routes/integrations")).integrationsRoute, { + prefix: "/api", + }); + await (await import("../integration-host")).initIntegrations(api, [ + { directory: root, isBuiltIn: true }, + ]); + await (await import("../routes/admin-capability-bindings")).registerCapabilityBindingRoutes( + api, + ); + await api.ready(); + }, 30000); + afterEach(async () => { + vi.restoreAllMocks(); + await (await import("../integration-host")).shutdownIntegrations(); + await api?.close(); + await agent?.close(); + if (installation) closeControlInstallation(installation); + await state.fixture?.stop(); + if (host) rmSync(host.root, { recursive: true, force: true }); + vi.unstubAllEnvs(); + }); + async function desired() { + const response = await api.inject({ + url: "/api/admin/integrations/binding-test/bindings", + headers, + }); + expect(response.statusCode).toBe(200); + return response.json(); + } + async function set(body: unknown, key = `binding-${randomUUID()}`) { + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + return api.inject({ + method: "POST", + url: "/api/admin/integrations/binding-test/bindings/routing-engine", + headers: { ...headers, "idempotency-key": key }, + payload: body, + }); + } + it("saves and activates B once, rejects stale CAS, replays the same result, then unbinds to automatic resolution", async () => { + expect((await api.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "motis", + }); + const before = await desired(), + key = `binding-${randomUUID()}`, + body = { + serviceId: "engine-b", + expectedServiceId: "motis", + expectedRevision: before.revision, + }; + const result = await set(body, key); + expect(result.statusCode, result.body).toBe(200); + expect(result.json()).toMatchObject({ + database: "retained", + runtime: "applied", + runtimeGenerationId: expect.any(String), + }); + expect((await api.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "engine-b", + }); + expect((await set(body, key)).json()).toEqual(result.json()); + const staleKey = `binding-stale-${randomUUID()}`; + expect((await set({ ...body, serviceId: "motis" }, staleKey)).statusCode).toBe(409); + expect((await set({ ...body, serviceId: "motis" }, staleKey)).statusCode).toBe(409); + const current = await desired(); + expect(current.runtime.bindings["routing-engine"]).toBe("engine-b"); + expect(current.revision).not.toBe(before.revision); + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + const removed = await api.inject({ + method: "DELETE", + url: "/api/admin/integrations/binding-test/bindings/routing-engine", + headers: { ...headers, "idempotency-key": `binding-${randomUUID()}` }, + payload: { expectedServiceId: "engine-b", expectedRevision: current.revision }, + }); + expect(removed.statusCode, removed.body).toBe(200); + expect(removed.json().runtime).toBe("applied"); + expect((await desired()).bindings).toEqual([]); + }, 90000); + it("retains B and the old live A after a strict reload failure, allowing an explicit new apply", async () => { + const before = await desired(), + bindings = await import("./capability-bindings"); + vi.spyOn(bindings, "loadAllBindingsByIntegration").mockRejectedValueOnce( + Error("Synthetic unavailable binding query"), + ); + const failed = await set({ + serviceId: "engine-b", + expectedServiceId: "motis", + expectedRevision: before.revision, + }); + expect(failed.statusCode, failed.body).toBe(200); + expect(failed.json()).toMatchObject({ + database: "retained", + runtime: "failed", + reason: "reload_failed", + runtimeGenerationId: before.runtime.generationId, + }); + expect((await api.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "motis", + }); + const saved = await desired(); + expect(saved.bindings).toEqual([{ capability: "routing-engine", serviceId: "engine-b" }]); + const retry = await set({ + serviceId: "engine-b", + expectedServiceId: "engine-b", + expectedRevision: saved.revision, + }); + expect(retry.statusCode, retry.body).toBe(200); + expect(retry.json().runtime).toBe("applied"); + }, 90000); + it("serializes two actual writer connections and rejects the stale loser without undoing the winner", async () => { + const before = await desired(), + { jobRunner } = await import("./job-runner"), + { createDirectAdminOpsKey } = await import("./direct-ops-idempotency"); + const request = (serviceId: string) => + jobRunner.applyImmediateIntegrationBinding({ + actorId: "loopback", + operationKey: createDirectAdminOpsKey("loopback", `binding-race-${randomUUID()}`), + input: { + integrationId: "binding-test", + capability: "routing-engine", + expectedServiceId: "motis", + expectedRevision: before.revision, + serviceId, + }, + }); + const results = await Promise.allSettled([request("engine-b"), request("motis")]); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + expect(results.filter((r) => r.status === "rejected")).toHaveLength(1); + const winner = results.find((r) => r.status === "fulfilled"); + expect(winner?.status === "fulfilled" && winner.value.runtime).toBe("applied"); + const current = await desired(); + expect(current.revision).not.toBe(before.revision); + expect(await state.fixture.sql`SELECT * FROM integration_application_receipt`).toHaveLength( + 1, + ); + }, 90000); + it("observes an already active generation after completion publication fails, without another setup", async () => { + const before = await desired(), + key = `binding-recovery-${randomUUID()}`, + body = { + serviceId: "engine-b", + expectedServiceId: "motis", + expectedRevision: before.revision, + }; + const hostModule = await import("../integration-host"), + reload = vi.spyOn(hostModule, "reloadIntegrations"); + await state.fixture.sql.unsafe( + `CREATE FUNCTION reject_binding_completion() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.result#>>'{integrationReload,state}'='complete' THEN RAISE EXCEPTION 'synthetic completion unavailable'; END IF; RETURN NEW; END $$`, + ); + await state.fixture.sql.unsafe( + `CREATE TRIGGER reject_binding_completion BEFORE UPDATE ON admin_job FOR EACH ROW EXECUTE FUNCTION reject_binding_completion()`, + ); + const interrupted = await set(body, key); + expect(interrupted.statusCode).toBe(503); + expect((await api.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "engine-b", + }); + expect(reload).toHaveBeenCalledTimes(1); + const original = await state.fixture + .sql`SELECT id,result FROM admin_job WHERE type='integration.binding.apply'`; + expect(original).toHaveLength(1); + expect(original[0]!.result.integrationReload.state).toBe("attempted"); + await state.fixture.sql.unsafe(`DROP TRIGGER reject_binding_completion ON admin_job`); + // The original call has returned. Expire only its fixture observation lease; + // keep the exact nonce, candidate, source and all production claim checks. + await state.fixture + .sql`UPDATE admin_job SET result=jsonb_set(result,'{_observationLeaseUntil}',to_jsonb('2000-01-01T00:00:00.000Z'::text)) WHERE type='integration.binding.apply'`; + const resumed = await set(body, key); + const diagnostic = await state.fixture + .sql`SELECT status,result->'_configurationCandidate'->>'phase' AS phase,result->'_execution' AS execution,result->'observation' AS observation,result->'integrationReload' AS reload,error FROM admin_job WHERE type='integration.binding.apply'`; + expect(resumed.statusCode, JSON.stringify(diagnostic)).toBe(200); + expect(resumed.json().runtime).toBe("applied"); + expect(reload).toHaveBeenCalledTimes(1); + const rows = await state.fixture + .sql`SELECT id,status,result FROM admin_job WHERE type='integration.binding.apply'`; + expect(rows).toHaveLength(1); + expect(rows[0]!.id).toBe(original[0]!.id); + expect(rows[0]!.status).toBe("success"); + expect(rows[0]!.result._execution.version).toBe(2); + }, 90000); + it("keeps disabled integrations pending instead of claiming an active provider", async () => { + await state.fixture + .sql`INSERT INTO integration_config (id,integration_id,config,created_at,updated_at) VALUES (${randomUUID()},'binding-test','{"enabled":false}'::jsonb,now(),now())`; + await (await import("../integration-host")).reloadIntegrations(); + const before = await desired(); + expect(before.runtime).toBeNull(); + const result = await set({ + serviceId: "engine-b", + expectedServiceId: "motis", + expectedRevision: before.revision, + }); + expect(result.statusCode, result.body).toBe(200); + expect(result.json()).toMatchObject({ + database: "retained", + runtime: "pending", + reason: "integration_disabled", + runtimeGenerationId: null, + }); + expect((await desired()).bindings).toEqual([ + { capability: "routing-engine", serviceId: "engine-b" }, + ]); + }, 90000); + it("rejects a real out-of-protocol row race after preview without saving the proposed binding", async () => { + const before = await desired(); + raceAfterPreview = true; + const key = `binding-row-race-${randomUUID()}`, + body = { + serviceId: "engine-b", + expectedServiceId: "motis", + expectedRevision: before.revision, + }; + const rejected = await set(body, key); + expect(rejected.statusCode, rejected.body).toBe(409); + expect(raceAfterPreview).toBe(false); + expect((await desired()).bindings).toEqual([ + { capability: "routing-engine", serviceId: "motis" }, + ]); + expect(await state.fixture.sql`SELECT * FROM integration_application_receipt`).toHaveLength( + 0, + ); + expect((await set(body, key)).statusCode).toBe(409); + }, 90000); + it("rolls back binding, revision and application receipt together on an actual SQL constraint failure", async () => { + const before = await desired(); + await state.fixture + .sql`ALTER TABLE integration_application_receipt ADD CONSTRAINT fixture_reject_binding CHECK (capability <> 'routing-engine')`; + const refused = await set({ + serviceId: "engine-b", + expectedServiceId: "motis", + expectedRevision: before.revision, + }); + expect(refused.statusCode).not.toBe(200); + expect((await desired()).revision).toBe(before.revision); + expect(await state.fixture.sql`SELECT * FROM integration_application_receipt`).toHaveLength( + 0, + ); + expect((await api.inject("/api/integrations/binding-test/bound")).json()).toEqual({ + serviceId: "motis", + }); + }, 90000); + }, +); diff --git a/apps/api/src/services/integration-application.ts b/apps/api/src/services/integration-application.ts new file mode 100644 index 000000000..0a4e17d4f --- /dev/null +++ b/apps/api/src/services/integration-application.ts @@ -0,0 +1,63 @@ +/** Fixed binding coordinator in the existing candidate/writer transaction. */ +import type { IntegrationApplyReceipt } from "@openmapx/core/api-contracts"; +import { JobObservationPending } from "./admin-job-failure"; +import { + withAdministrativeConfigurationJob, + withAdministrativeConfigurationWriter, + withConfigurationCommitParticipant, +} from "./administrative-configuration-writer"; +import { + type IntegrationBindingInput, + integrationBindingParticipation, + prepareIntegrationBindingChange, +} from "./integration-binding-participant"; +import type { JobContext } from "./job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { + completeBindingActivation, + persistAndApplyProspectiveConfiguration, + prepareProspectiveConfiguration, +} from "./prospective-configuration"; +import { completeTrustedConfigurationApply } from "./trusted-config-outcome"; + +export async function runBindingChange( + ctx: JobContext, + options: { input: IntegrationBindingInput; actorId: string; operationKey: string }, +): Promise { + return withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { + actorId: options.actorId, + operationKey: options.operationKey, + integrationIds: [options.input.integrationId], + signal: ctx.signal, + }, + async (writer) => { + const intent = await prepareIntegrationBindingChange(options.input, writer), + scope = integrationBindingParticipation(intent, writer); + return withConfigurationCommitParticipant(writer, scope.participant, async () => { + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("configuration.inspect", options.operationKey), + { signal: ctx.signal }, + ); + const candidate = await prepareProspectiveConfiguration({ + actorId: options.actorId, + operationKey: options.operationKey, + expectedCurrentRevisionId: current.revisionId, + overlay: scope.overlay, + signal: ctx.signal, + }); + const receipt = await persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + signal: ctx.signal, + }); + if (!completeTrustedConfigurationApply(receipt.application)) + throw new JobObservationPending("recovery_required"); + return completeBindingActivation(candidate, receipt.application); + }); + }, + ), + ); +} diff --git a/apps/api/src/services/integration-binding-participant.ts b/apps/api/src/services/integration-binding-participant.ts new file mode 100644 index 000000000..532bdd606 --- /dev/null +++ b/apps/api/src/services/integration-binding-participant.ts @@ -0,0 +1,357 @@ +/** Sole built-in issuer for binding changes in the existing candidate transaction. */ +import { createHash, randomUUID } from "node:crypto"; +import { canonicalJson } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { and, eq, sql } from "drizzle-orm"; +import z from "zod/v4"; +import { + adminAuditLog, + capabilityBinding, + integrationApplicationReceipt, + integrationBindingRevision, +} from "../db/schema"; +import { + type AdministrativeConfigurationWriter, + administrativeConfigurationState, + type ConfigurationCommitParticipant, + type ConfigurationWriteTransaction, + withAdministrativeConfigurationRead, +} from "./administrative-configuration-writer-owner"; +import { + bindingContentRevision, + capturedBindingState, + readBindingState, +} from "./integration-binding-state"; +import type { + ProspectiveConfigurationOverlay, + ValidatedConfigurationCandidate, +} from "./prospective-configuration"; +export const integrationBindingInputSchema = z.strictObject({ + integrationId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + capability: z + .string() + .min(1) + .max(128) + .refine((value) => !["__proto__", "constructor", "prototype"].includes(value)), + serviceId: z.string().min(1).max(64).nullable(), + expectedServiceId: z.string().min(1).max(64).nullable(), + expectedRevision: z.string().regex(/^[a-f0-9]{64}$/), +}); +export type IntegrationBindingInput = z.infer; +export class IntegrationBindingConflict extends Error { + constructor(readonly code = "BINDING_VERSION_CONFLICT") { + super(code); + } +} +const digest = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex"); +const intentBrand: unique symbol = Symbol("integration binding"); +export interface IntegrationBindingIntent { + readonly [intentBrand]: true; + readonly integrationId: string; +} +interface State { + writer: AdministrativeConfigurationWriter; + overlay: ProspectiveConfigurationOverlay; + participant: ConfigurationCommitParticipant; + input: IntegrationBindingInput; + before: ReturnType; + registry: string; + committed: boolean; +} +const intents = new WeakMap(), + issued = new WeakMap(); +function active(writer: AdministrativeConfigurationWriter) { + return administrativeConfigurationState(writer); +} +export function isIntegrationBindingParticipant( + participant: ConfigurationCommitParticipant, + writer: AdministrativeConfigurationWriter, +) { + return issued.get(participant)?.writer === writer; +} +export function assertIntegrationBindingOverlay( + writer: AdministrativeConfigurationWriter, + overlay: unknown, +) { + const state = active(writer), + owned = state.participant && issued.get(state.participant); + if (!owned || owned.writer !== writer || canonicalJson(owned.overlay) !== canonicalJson(overlay)) + throw Error("Binding participant unavailable"); +} +export function integrationBindingParticipation( + intent: IntegrationBindingIntent, + writer: AdministrativeConfigurationWriter, +) { + const state = intents.get(intent); + if (!state || state.writer !== writer) throw Error("Binding intent unavailable"); + active(writer); + return { overlay: state.overlay, participant: state.participant }; +} +export async function prepareIntegrationBindingChange( + raw: unknown, + writer: AdministrativeConfigurationWriter, +): Promise { + const input = integrationBindingInputSchema.parse(raw); + active(writer); + if (!writer.integrationIds.includes(input.integrationId)) + throw Error("Binding integration lock unavailable"); + return withAdministrativeConfigurationRead({}, async (read) => { + const integration = read.integrations.find((item) => item.id === input.integrationId), + service = input.serviceId + ? read.services.find((item) => item.manifest.id === input.serviceId) + : undefined; + if (!integration?.manifest.requires?.some((item) => item.capability === input.capability)) + throw new IntegrationBindingConflict("BINDING_CAPABILITY_NOT_DECLARED"); + if ( + input.serviceId && + (!service?.enabled || + !services.getProvidedCapabilityNames(service.manifest.provides).includes(input.capability)) + ) + throw new IntegrationBindingConflict("BINDING_SERVICE_UNAVAILABLE"); + const before = capturedBindingState(read, input.integrationId); + if ( + (before.bindings[input.capability] ?? null) !== input.expectedServiceId || + before.revision !== input.expectedRevision + ) + throw new IntegrationBindingConflict(); + const nextBindings = { ...before.bindings }; + if (input.serviceId) nextBindings[input.capability] = input.serviceId; + else delete nextBindings[input.capability]; + const change = { + kind: "integration-binding" as const, + ...input, + nextVersion: randomUUID(), + desiredRevision: bindingContentRevision(input.integrationId, nextBindings), + }, + overlay = Object.freeze({ + purpose: "integration-binding" as const, + coordinatorMode: "ensure-running" as const, + changes: Object.freeze([Object.freeze(change)]), + }) as unknown as ProspectiveConfigurationOverlay, + registry = digest({ + integration: { + id: integration.id, + directory: integration.directory, + manifest: integration.manifest, + enabled: integration.enabled, + }, + service: service ? { manifest: service.manifest, enabled: service.enabled } : null, + }); + let state: State; + const verify = async ( + actual: AdministrativeConfigurationWriter, + candidate?: ValidatedConfigurationCandidate, + ) => { + if ( + actual !== writer || + issued.get(state.participant) !== state || + (candidate && candidate.operationKey !== writer.operationKey) + ) + throw Error("Binding participant unavailable"); + active(writer); + await withAdministrativeConfigurationRead({}, async (current) => { + const i = current.integrations.find((item) => item.id === input.integrationId), + s = input.serviceId + ? current.services.find((item) => item.manifest.id === input.serviceId) + : undefined; + if ( + digest({ + integration: i + ? { id: i.id, directory: i.directory, manifest: i.manifest, enabled: i.enabled } + : null, + service: s ? { manifest: s.manifest, enabled: s.enabled } : null, + }) !== registry + ) + throw new IntegrationBindingConflict("BINDING_SOURCE_CHANGED"); + if ( + !state.committed && + capturedBindingState(current, input.integrationId).revision !== before.revision + ) + throw new IntegrationBindingConflict(); + }); + }; + const participant = Object.freeze({ + kind: "integration-binding", + mode: "apply", + actorId: writer.actorId, + operationKey: writer.operationKey, + managedServiceSecretKeys: Object.freeze([]), + authorize: (actual: AdministrativeConfigurationWriter) => verify(actual), + verify: ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + ) => verify(actual, candidate), + commit: async ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + tx: ConfigurationWriteTransaction, + ) => { + if (state.committed || active(actual).connection !== tx) + throw Error("Original binding transaction required"); + await verify(actual, candidate); + const prior = before.rows.find((row) => row.capability === input.capability), + one = (rows: unknown[]) => { + if (rows.length !== 1) throw new IntegrationBindingConflict(); + }; + if (input.serviceId) { + if (prior) + one( + await tx + .update(capabilityBinding) + .set({ serviceId: input.serviceId, updatedAt: new Date() }) + .where( + and( + and( + eq(capabilityBinding.integrationId, input.integrationId), + eq(capabilityBinding.capability, input.capability), + ), + sql`xmin::text=${prior._rowVersion}`, + ), + ) + .returning({ id: capabilityBinding.integrationId }), + ); + else + one( + await tx + .insert(capabilityBinding) + .values({ + integrationId: input.integrationId, + capability: input.capability, + serviceId: input.serviceId, + }) + .onConflictDoNothing() + .returning({ id: capabilityBinding.integrationId }), + ); + } else if (prior) + one( + await tx + .delete(capabilityBinding) + .where( + and( + and( + eq(capabilityBinding.integrationId, input.integrationId), + eq(capabilityBinding.capability, input.capability), + ), + sql`xmin::text=${prior._rowVersion}`, + ), + ) + .returning({ id: capabilityBinding.integrationId }), + ); + if (before.version) + one( + await tx + .update(integrationBindingRevision) + .set({ version: change.nextVersion, updatedAt: new Date() }) + .where( + and( + eq(integrationBindingRevision.integrationId, input.integrationId), + eq(integrationBindingRevision.version, before.version.version), + sql`xmin::text=${before.version._rowVersion}`, + ), + ) + .returning({ id: integrationBindingRevision.integrationId }), + ); + else + one( + await tx + .insert(integrationBindingRevision) + .values({ integrationId: input.integrationId, version: change.nextVersion }) + .onConflictDoNothing() + .returning({ id: integrationBindingRevision.integrationId }), + ); + const saved = await readBindingState(input.integrationId, tx); + if (saved.desiredRevision !== change.desiredRevision) + throw new IntegrationBindingConflict(); + const actorId = ["loopback", "internal"].includes(writer.actorId) ? null : writer.actorId; + await tx.insert(integrationApplicationReceipt).values({ + operationId: writer.operationKey, + actorId, + integrationId: input.integrationId, + capability: input.capability, + serviceId: input.serviceId, + candidateId: candidate.candidateId, + intentDigest: digest(change), + savedBindingRevision: saved.revision, + result: { + operationId: writer.operationKey, + integrationId: input.integrationId, + desiredRevision: saved.desiredRevision, + database: "retained", + runtime: "pending", + runtimeGenerationId: null, + ...(!integration.enabled ? { reason: "integration_disabled" as const } : {}), + }, + }); + await tx.insert(adminAuditLog).values({ + id: randomUUID(), + actorId, + targetType: "integration", + targetId: input.integrationId, + action: input.serviceId ? "integration.binding.set" : "integration.binding.remove", + details: { + capability: input.capability, + previousServiceId: input.expectedServiceId, + serviceId: input.serviceId, + operationDigest: digest(writer.operationKey), + }, + }); + state.committed = true; + }, + }) as unknown as ConfigurationCommitParticipant; + state = { writer, overlay, participant, input, before, registry, committed: false }; + const intent = Object.freeze({ + [intentBrand]: true as const, + integrationId: input.integrationId, + }); + intents.set(intent, state); + issued.set(participant, state); + return intent; + }); +} +/** A verifier only: no write, participant reconstruction or mutation admission. */ +export async function assertSavedIntegrationBinding( + candidate: ValidatedConfigurationCandidate, + rawOverlay: unknown, + tx: ConfigurationWriteTransaction, +) { + const overlay = z + .strictObject({ + purpose: z.literal("integration-binding"), + coordinatorMode: z.enum(["ensure-running", "require-running"]), + changes: z + .array( + integrationBindingInputSchema.extend({ + kind: z.literal("integration-binding"), + nextVersion: z.uuid(), + desiredRevision: z.string().regex(/^[a-f0-9]{64}$/), + }), + ) + .length(1), + }) + .parse(rawOverlay); + const change = overlay.changes[0]; + if ( + overlay.purpose !== "integration-binding" || + overlay.changes.length !== 1 || + change?.kind !== "integration-binding" + ) + throw Error("Original binding overlay required"); + const [receipt] = await tx + .select() + .from(integrationApplicationReceipt) + .where(eq(integrationApplicationReceipt.operationId, candidate.operationKey)); + if ( + !receipt || + receipt.candidateId !== candidate.candidateId || + receipt.integrationId !== change.integrationId || + receipt.intentDigest !== digest(change) + ) + throw Error("Original binding commit receipt unavailable"); + const saved = await readBindingState(change.integrationId, tx); + if ( + saved.revision !== receipt.savedBindingRevision || + saved.desiredRevision !== change.desiredRevision + ) + throw new IntegrationBindingConflict("BINDING_SOURCE_CHANGED"); + return receipt; +} diff --git a/apps/api/src/services/integration-binding-state.ts b/apps/api/src/services/integration-binding-state.ts new file mode 100644 index 000000000..b8997ccd4 --- /dev/null +++ b/apps/api/src/services/integration-binding-state.ts @@ -0,0 +1,76 @@ +/** Desired binding fingerprints. Values are service IDs, never credentials. */ +import { createHash } from "node:crypto"; +import { canonicalJson } from "@openmapx/core/ops"; +import { eq, getTableColumns, sql } from "drizzle-orm"; +import { db } from "../db"; +import { capabilityBinding, integrationBindingRevision } from "../db/schema"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./administrative-configuration-context"; +export type BindingRow = typeof capabilityBinding.$inferSelect & { _rowVersion?: string }; +export type BindingVersion = typeof integrationBindingRevision.$inferSelect & { + _rowVersion?: string; +}; +const digest = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex"); +export function bindingState( + integrationId: string, + allRows: readonly BindingRow[], + allVersions: readonly BindingVersion[], +) { + const rows = allRows + .filter((row) => row.integrationId === integrationId) + .sort((a, b) => a.capability.localeCompare(b.capability)), + versions = allVersions.filter((row) => row.integrationId === integrationId); + if (versions.length > 1 || new Set(rows.map((row) => row.capability)).size !== rows.length) + throw Error("Binding authority unavailable"); + for (const row of [...rows, ...versions]) + if (typeof row._rowVersion !== "string" || !/^\d+$/.test(row._rowVersion)) + throw Error("Binding row version unavailable"); + const bindings = Object.fromEntries(rows.map((row) => [row.capability, row.serviceId])); + return { + rows, + bindings, + version: versions[0] ?? null, + revision: digest({ + integrationId, + rows: rows.map((row) => ({ + capability: row.capability, + serviceId: row.serviceId, + rowVersion: row._rowVersion, + })), + version: versions[0] ?? null, + }), + desiredRevision: bindingContentRevision(integrationId, bindings), + }; +} +export const bindingContentRevision = ( + integrationId: string, + bindings: Readonly>, +) => digest({ integrationId, bindings }); +export function capturedBindingState( + context: AdministrativeConfigurationReadContext, + integrationId: string, +) { + assertAdministrativeConfigurationRead(context); + if (!context.bindingRows || !context.bindingRevisionRows) + throw Error("Captured binding authority unavailable"); + return bindingState(integrationId, context.bindingRows, context.bindingRevisionRows); +} +export async function readBindingState( + integrationId: string, + database: Pick = db, +) { + const rows = await database + .select({ ...getTableColumns(capabilityBinding), _rowVersion: sql`xmin::text` }) + .from(capabilityBinding) + .where(eq(capabilityBinding.integrationId, integrationId)); + const versions = await database + .select({ + ...getTableColumns(integrationBindingRevision), + _rowVersion: sql`xmin::text`, + }) + .from(integrationBindingRevision) + .where(eq(integrationBindingRevision.integrationId, integrationId)); + return bindingState(integrationId, rows, versions); +} diff --git a/apps/api/src/services/integration-runtime-evidence.ts b/apps/api/src/services/integration-runtime-evidence.ts new file mode 100644 index 000000000..fc9d593b7 --- /dev/null +++ b/apps/api/src/services/integration-runtime-evidence.ts @@ -0,0 +1,34 @@ +/** Host-owned observations of successfully staged provider generations. */ +import { createHmac, randomBytes } from "node:crypto"; +import type { IntegrationRuntimeEvidence } from "@openmapx/core/api-contracts"; +import { canonicalJson } from "@openmapx/core/ops"; +import type { LoadedIntegration } from "@openmapx/integration-framework"; + +// An ephemeral keyed fingerprint prevents the administrative projection from +// becoming an offline oracle for low-entropy configuration or credential values. +const revisionKey = randomBytes(32); +const observations = new WeakMap(); +export const integrationConfigRevision = (config: Record) => + createHmac("sha256", revisionKey).update(canonicalJson(config)).digest("hex"); +export function stageIntegrationRuntimeEvidence( + integration: LoadedIntegration, + generationId: string, +): void { + const bindings: Record = {}; + for (const requirement of integration.manifest.requires ?? []) { + const key = requirement.capability ?? requirement.service; + if (key) bindings[key] = integration.resolvedServices?.[key]?.serviceId ?? null; + } + observations.set(integration, { + generationId, + bindings, + configRevision: integrationConfigRevision(integration.config), + }); +} +export function integrationRuntimeEvidence( + integration: LoadedIntegration | undefined, +): IntegrationRuntimeEvidence | null { + if (!integration?.enabled || integration.blockedCode) return null; + const value = observations.get(integration); + return value ? structuredClone(value) : null; +} diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 4c291e299..5b4694860 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -2,6 +2,7 @@ import { createHmac, randomUUID } from "node:crypto"; import { adminIntegrationConfigImportedSchema, adminIntegrationImportFailureSchema, + integrationApplyReceiptSchema, } from "@openmapx/core/api-contracts"; import { canonicalJson, hardlinkApplySummarySchema, readOpsTokenFile } from "@openmapx/core/ops"; import { and, asc, eq, inArray, sql } from "drizzle-orm"; @@ -34,6 +35,7 @@ import { renewAdminJobClaim, } from "./admin-job-state"; import { coordinatorLifecycleOperationKey } from "./coordinator-lifecycle"; +import { IntegrationBindingConflict as RetainedBindingConflict } from "./integration-binding-participant"; import { type JobStatusEvent, jobEventBus, TERMINAL_JOB_STATUSES } from "./job-events"; import { ApiOpsError, OpsObservationPending } from "./ops-client"; import { appApiRestartCheckpoint } from "./system-update-state"; @@ -113,6 +115,7 @@ export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ "configuration.apply", "compose.start", "integration.config.import", + "integration.binding.apply", "system.diagnostics", "system.update", ] as const; @@ -129,7 +132,12 @@ function hasRetainedImmediateConfiguration(job: { const result = job.result as Record | null, payload = job.payload as Record | null; return ( - ["service.apply", "configuration.apply", "integration.config.import"].includes(job.type) && + [ + "service.apply", + "configuration.apply", + "integration.config.import", + "integration.binding.apply", + ].includes(job.type) && result?._immediateJobId === job.id && typeof result._immediateActorId === "string" && typeof result._immediateIntentDigest === "string" && @@ -200,7 +208,12 @@ class AdminJobRunner { private async immediate( options: { - type: "configuration.apply" | "service.apply" | "compose.start" | "integration.config.import"; + type: + | "configuration.apply" + | "service.apply" + | "compose.start" + | "integration.config.import" + | "integration.binding.apply"; actorId?: string; operationKey: string; intent: unknown; @@ -275,9 +288,12 @@ class AdminJobRunner { if ( !["running", "cancel_pending"].includes(job.status) || !checkpoint._configurationCandidate || - !["service.apply", "configuration.apply", "integration.config.import"].includes( - options.type, - ) + ![ + "service.apply", + "configuration.apply", + "integration.config.import", + "integration.binding.apply", + ].includes(options.type) ) throw new JobObservationPending("recovery_required"); const claimed = await claimAdminJob(job, {}, true); @@ -381,6 +397,11 @@ class AdminJobRunner { ) ) throw new AdminJobFailure("Original configuration did not apply", result); + if ( + result.integrationBinding && + integrationApplyReceiptSchema.parse(result.integrationBinding).runtime === "failed" + ) + throw new AdminJobFailure("Binding provider did not activate", result); return result; } @@ -534,6 +555,41 @@ class AdminJobRunner { ); } + applyImmediateIntegrationBinding(options: { + input: import("./integration-binding-participant").IntegrationBindingInput; + actorId: string; + operationKey: string; + signal?: AbortSignal; + }) { + return this.immediate( + { ...options, type: "integration.binding.apply", intent: options.input }, + async (ctx) => { + const { runBindingChange } = await import("./integration-application"); + try { + const value = await runBindingChange(ctx, options); + return { + value, + result: { integrationBinding: value }, + failed: value.runtime === "failed", + }; + } catch (error) { + const { IntegrationBindingConflict } = await import("./integration-binding-participant"); + if (error instanceof IntegrationBindingConflict) + await ctx.checkpoint({ + ...ctx.checkpointResult, + integrationBindingConflict: error.code, + }); + throw error; + } + }, + (result) => { + if (typeof result.integrationBindingConflict === "string") + throw new RetainedBindingConflict(result.integrationBindingConflict); + return integrationApplyReceiptSchema.parse(result.integrationBinding); + }, + ); + } + /** Publish a status transition so live viewers see it without polling. */ private announce( jobId: string, diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 4dac09640..454546aa6 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -2,6 +2,10 @@ import { createHash, createHmac, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { lstat, open, unlink } from "node:fs/promises"; import { join } from "node:path"; +import { + type IntegrationApplyReceipt, + integrationApplyReceiptSchema, +} from "@openmapx/core/api-contracts"; import { controlChildOperationKey } from "@openmapx/core/build-artifacts"; import { acquireTrustedConfigurationQueueLock, @@ -39,7 +43,9 @@ import { services as coreServices } from "@openmapx/core/server"; import { and, eq, sql } from "drizzle-orm"; import { z } from "zod"; import * as schema from "../db/schema"; +import { integrationApplicationReceipt } from "../db/schema"; import { resolveIntegrationConfigWithSourcesStrict } from "../integration-config"; +import { getIntegration, inspectIntegrationRuntime } from "../integration-host"; import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; import { deliverCoordinatorPullSubmission, @@ -102,11 +108,16 @@ import { closeUnissuedCoordinatorPull, coordinatorLifecycleOperationKey, } from "./coordinator-lifecycle"; +import { + assertIntegrationBindingOverlay, + assertSavedIntegrationBinding, +} from "./integration-binding-participant"; +import { integrationConfigRevision } from "./integration-runtime-evidence"; import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; import { encrypt } from "./secrets"; import { resolveServiceConfigWithSourcesStrict } from "./service-config-resolver"; -import { getServiceRegistry } from "./service-registry"; +import { getServiceRegistry, resolveRequiresForIntegration } from "./service-registry"; import { completeTrustedConfigurationApply, parseTrustedConfigurationApplyOutcome, @@ -159,6 +170,7 @@ const overlaySchema = z.strictObject({ "service-secret", "integration-config", "integration-secret", + "integration-binding", "migration", "platform", "transit-credentials", @@ -177,6 +189,16 @@ const overlaySchema = z.strictObject({ integrationId: id, ...secretDelete, }), + z.strictObject({ + kind: z.literal("integration-binding"), + integrationId: id, + capability: z.string().min(1).max(128), + serviceId: id.nullable(), + expectedServiceId: id.nullable(), + expectedRevision: z.string().regex(/^[a-f0-9]{64}$/), + nextVersion: z.uuid(), + desiredRevision: z.string().regex(/^[a-f0-9]{64}$/), + }), z.strictObject({ kind: z.literal("selection"), selectedRoots: z.array(id).max(256) }), z.strictObject({ kind: z.literal("platform"), @@ -234,6 +256,9 @@ interface CandidateState { | "integrationRows" | "serviceSecretRows" | "integrationSecretRows" + | "bindingRows" + | "bindingRevisionRows" + | "integrationBindingSnapshot" > | null; sourceRows: Pick< AdministrativeConfigurationReadContext, @@ -304,6 +329,8 @@ async function verifyConfigurationCommitParticipant( } if (state.overlay.purpose === "migration") assertLegacyIntegrationImportOverlay(writer, state.overlay); + if (state.overlay.purpose === "integration-binding") + assertIntegrationBindingOverlay(writer, state.overlay); await owner.participant?.verify(candidate, writer); } export function assertCommittedConfigurationCandidate( @@ -1063,9 +1090,12 @@ export async function retireSettledProspectiveConfiguration( if ( queuedCoordinator || (job.checkpointResult?._immediateJobId === job.jobId && - ["service.apply", "configuration.apply", "integration.config.import"].includes( - job.jobType ?? "", - )) + [ + "service.apply", + "configuration.apply", + "integration.config.import", + "integration.binding.apply", + ].includes(job.jobType ?? "")) ) { const original = job.checkpointResult; if (!original) throw new JobObservationPending("recovery_required"); @@ -1078,6 +1108,7 @@ export async function retireSettledProspectiveConfiguration( ? { coordinatorLifecycle: original.coordinatorLifecycle } : {}), ...(original.integrationReload ? { integrationReload: original.integrationReload } : {}), + ...(original.integrationBinding ? { integrationBinding: original.integrationBinding } : {}), ...(provenance ? { integrationImport: { @@ -1306,6 +1337,7 @@ export async function withProspectiveConfigurationRead( integrations: state.rows.integrations, env: state.rows.env, integrationReloadSnapshot: state.integrationReloadSnapshot, + integrationBindingSnapshot: state.record.privateIntegrationReload?.bindings, }); try { return await withAdministrativeConfigurationReadScope(context, () => run(context)); @@ -1323,7 +1355,7 @@ export async function reloadProspectiveIntegrations(candidate: ValidatedConfigur if (!recoveredCommittedCandidates.has(candidate)) return withProspectiveConfigurationRead(candidate, (read) => reloadIntegrations(read)); const retained = state.record.privateIntegrationReload; - if (state.record.apiFollowUp !== "integration-reload" || !retained) + if (state.record.apiFollowUp !== "integration-reload" || !retained || !retained.bindings) throw new JobObservationPending("recovery_required"); return withAdministrativeConfigurationRead({}, async (read) => { if ( @@ -1344,6 +1376,7 @@ export async function reloadProspectiveIntegrations(candidate: ValidatedConfigur const captured = openAdministrativeConfigurationRead({ ...read, integrationReloadSnapshot: retained.values, + integrationBindingSnapshot: retained.bindings, }); try { return await withAdministrativeConfigurationReadScope(captured, () => @@ -1541,6 +1574,8 @@ async function sourceFingerprint( env: context.env, files: administrativeConfigurationFileObservations(context), serviceRows: context.serviceRows, + bindingRows: context.bindingRows, + bindingRevisionRows: context.bindingRevisionRows, integrationRows: context.integrationRows, serviceSecretRows: context.serviceSecretRows, integrationSecretRows: context.integrationSecretRows, @@ -2224,7 +2259,9 @@ async function restoreRetainedOrdinaryCandidate(): Promise | undefined; @@ -2664,6 +2707,7 @@ function fieldDefinition(schema: Record | undefined, key: strin function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWriter): void { const state = administrativeConfigurationState(writer); if (overlay.purpose === "migration") assertLegacyIntegrationImportOverlay(writer, overlay); + if (overlay.purpose === "integration-binding") assertIntegrationBindingOverlay(writer, overlay); if (overlay.purpose === "migration" && state.participant?.kind !== "configuration-migration") throw Error("Configuration commit participant unavailable"); if ( @@ -2683,11 +2727,13 @@ function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWr if ("integrationId" in change && !writer.integrationIds.includes(change.integrationId)) throw Error("Configuration integration lock unavailable"); const keys = - "set" in change - ? [...Object.keys(change.set), ...change.unset] - : "key" in change - ? [change.key] - : ["selection"]; + change.kind === "integration-binding" + ? [change.capability] + : "set" in change + ? [...Object.keys(change.set), ...change.unset] + : "key" in change + ? [change.key] + : ["selection"]; for (const key of keys) { const identity = `${change.kind.startsWith("integration") ? "integration" : "service"}:${target}:${key}`; if (fields.has(identity)) throw Error("Conflicting configuration overlay"); @@ -2708,7 +2754,11 @@ function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWr } } const serviceIds = new Set( - overlay.changes.flatMap((change) => ("serviceId" in change ? [change.serviceId] : [])), + overlay.changes.flatMap((change) => + change.kind !== "integration-binding" && "serviceId" in change && change.serviceId + ? [change.serviceId] + : [], + ), ); const integrationIds = new Set( overlay.changes.flatMap((change) => ("integrationId" in change ? [change.integrationId] : [])), @@ -2719,7 +2769,9 @@ function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWr (overlay.changes.length !== 1 || overlay.changes[0]?.kind !== "selection")) || (["service-config", "service-secret", "transit-credentials"].includes(overlay.purpose) && (serviceIds.size !== 1 || integrationIds.size > 0)) || - (["integration-config", "integration-secret"].includes(overlay.purpose) && + (["integration-config", "integration-secret", "integration-binding"].includes( + overlay.purpose, + ) && (integrationIds.size !== 1 || serviceIds.size > 0)) ) throw Error("Configuration purpose conflicts with overlay"); @@ -2728,6 +2780,7 @@ function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWr "service-secret": "service-secret", "integration-config": "integration-config", "integration-secret": "integration-secret", + "integration-binding": "integration-binding", "transit-credentials": "service-secret", selection: "selection", } as const; @@ -2745,6 +2798,18 @@ async function mergeOverlay( services: context.services, integrations: context.integrations, env: context.env, + bindingRows: context.bindingRows, + bindingRevisionRows: context.bindingRevisionRows, + integrationBindingSnapshot: Object.fromEntries( + [...new Set((context.bindingRows ?? []).map((row) => row.integrationId))].map((id) => [ + id, + Object.fromEntries( + (context.bindingRows ?? []) + .filter((row) => row.integrationId === id) + .map((row) => [row.capability, row.serviceId]), + ), + ]), + ), serviceRows: structuredClone([...context.serviceRows]), integrationRows: structuredClone([...context.integrationRows]), serviceSecretRows: structuredClone([...context.serviceSecretRows]), @@ -2752,6 +2817,13 @@ async function mergeOverlay( }; const now = new Date(); for (const change of overlay.changes) { + if (change.kind === "integration-binding") { + const bindings = rows.integrationBindingSnapshot[change.integrationId] ?? {}; + if (change.serviceId) bindings[change.capability] = change.serviceId; + else delete bindings[change.capability]; + rows.integrationBindingSnapshot[change.integrationId] = bindings; + continue; + } if (change.kind === "platform" || change.kind === "selection") continue; const service = "serviceId" in change @@ -2951,7 +3023,9 @@ export async function prepareProspectiveConfiguration(input: { : [ ...new Set( overlay.changes.flatMap((change) => - "serviceId" in change ? [change.serviceId] : [], + change.kind !== "integration-binding" && "serviceId" in change && change.serviceId + ? [change.serviceId] + : [], ), ), ].sort(); @@ -3133,6 +3207,7 @@ export async function prepareProspectiveConfiguration(input: { isBuiltIn: i.isBuiltIn, })), values: integrationReloadSnapshot, + bindings: merged.integrationBindingSnapshot, ...(overlay.purpose === "migration" ? { legacyImport: ( @@ -3333,7 +3408,12 @@ function savedRowIntents(state: CandidateState) { } const seen = new Set(); return state.overlay.changes.flatMap((change) => { - if (change.kind === "selection" || change.kind === "platform") return []; + if ( + change.kind === "selection" || + change.kind === "platform" || + change.kind === "integration-binding" + ) + return []; const ownerId = "serviceId" in change ? change.serviceId : change.integrationId; const secretKey = "key" in change ? change.key : undefined, identity = `${change.kind}:${ownerId}:${secretKey ?? ""}`; @@ -4366,6 +4446,8 @@ async function applyCommittedOrdinaryCandidate( throw new JobObservationPending("recovery_required"); const rows = z.array(configurationSavedRowSchema).max(512).parse(retained.rowVersions); await assertConfigurationSavedRows(tx, rows); + if (state.overlay.purpose === "integration-binding") + await assertSavedIntegrationBinding(candidate, state.overlay, tx); committed = result; return rows; }); @@ -5413,3 +5495,164 @@ const configurationMaintenanceProtocol = Object.freeze({ prepareConfigurationMaintenance, inspectConfigurationMaintenance, }); + +/** Observation-only after an interrupted attempt; never replay unknown setup. */ +export async function completeBindingActivation( + candidate: ValidatedConfigurationCandidate, + hostApply: IntegrationApplyReceipt["hostApply"], +): Promise { + const state = validatedConfigurationCandidateState(candidate), + { job } = administrativeConfigurationState(state.writer), + change = state.overlay.changes[0]; + if ( + state.overlay.purpose !== "integration-binding" || + state.overlay.changes.length !== 1 || + change?.kind !== "integration-binding" + ) + throw Error("Original binding candidate required"); + const retained = state.record.privateIntegrationReload, + retainedManifest = retained?.integrations.find((i) => i.id === change.integrationId)?.manifest, + values = retained?.values[change.integrationId]; + const manifest = getIntegration(change.integrationId)?.manifest, + loadedServices = getServiceRegistry().list(); + if ( + !retained?.bindings || + !manifest || + !values || + canonicalJson(manifest) !== canonicalJson(retainedManifest) || + canonicalJson(loadedServices) !== canonicalJson(retained.services) + ) + throw new JobObservationPending("recovery_required"); + const config = Object.fromEntries( + Object.entries(values).map(([key, entry]) => [key, entry.value]), + ); + const resolved = resolveRequiresForIntegration({ + manifestId: change.integrationId, + requires: manifest.requires, + loadedServices, + bindings: new Map(Object.entries(retained.bindings[change.integrationId] ?? {})), + }); + const expected = Object.fromEntries( + (manifest.requires ?? []).map((req) => { + const key = req.capability ?? req.service!; + return [key, resolved.get(key)?.serviceId ?? null]; + }), + ); + const matches = () => { + const live = inspectIntegrationRuntime(change.integrationId); + return live && + live.configRevision === integrationConfigRevision(config) && + canonicalJson(live.bindings) === canonicalJson(expected) + ? live + : null; + }; + let receipt = await withConfigurationWriteTransaction( + state.writer, + async (tx) => (await assertSavedIntegrationBinding(candidate, state.overlay, tx)).result, + ); + const original = job.checkpointResult?.integrationReload as Record | undefined; + if (original?.candidateId && original.candidateId !== candidate.candidateId) + throw new JobObservationPending("recovery_required"); + if (original?.state !== "complete") { + const before = inspectIntegrationRuntime(change.integrationId); + let failed = false; + if (!original) { + await job.checkpoint({ + ...job.checkpointResult, + integrationReload: { + state: "attempted", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }, + }); + try { + await reloadProspectiveIntegrations(candidate); + } catch { + failed = true; + } + } + const live = matches(), + disabled = + config.enabled === false && getIntegration(change.integrationId)?.enabled === false; + if (live) + receipt = { + ...receipt, + runtime: "applied", + runtimeGenerationId: live.generationId, + reason: undefined, + }; + else if (disabled) + receipt = { + ...receipt, + runtime: "pending", + runtimeGenerationId: null, + reason: "integration_disabled", + }; + else if ( + failed && + before && + inspectIntegrationRuntime(change.integrationId)?.generationId === before.generationId + ) + receipt = { + ...receipt, + runtime: "failed", + runtimeGenerationId: before.generationId, + reason: "reload_failed", + }; + else { + await withConfigurationWriteTransaction(state.writer, async (tx) => { + await assertSavedIntegrationBinding(candidate, state.overlay, tx); + await tx + .update(integrationApplicationReceipt) + .set({ + result: { + ...receipt, + runtime: "unknown", + reason: "reconciliation_required", + hostApply, + }, + }) + .where( + and( + eq(integrationApplicationReceipt.operationId, candidate.operationKey), + eq(integrationApplicationReceipt.candidateId, candidate.candidateId), + ), + ); + }); + throw new JobObservationPending("recovery_required"); + } + receipt = integrationApplyReceiptSchema.parse({ ...receipt, hostApply }); + await withConfigurationWriteTransaction(state.writer, async (tx) => { + await assertSavedIntegrationBinding(candidate, state.overlay, tx); + await tx + .update(integrationApplicationReceipt) + .set({ result: receipt }) + .where( + and( + eq(integrationApplicationReceipt.operationId, candidate.operationKey), + eq(integrationApplicationReceipt.candidateId, candidate.candidateId), + ), + ); + }); + await job.checkpoint({ + ...job.checkpointResult, + integrationBinding: receipt, + _configurationCandidate: { + ...(job.checkpointResult?._configurationCandidate as Record), + phase: "settled", + }, + integrationReload: { + state: "complete", + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + result: { + message: "Binding activation observed", + reloaded: receipt.runtime === "applied" ? 1 : 0, + enabled: getIntegration(change.integrationId)?.enabled ? 1 : 0, + }, + }, + }); + } + await retireSettledProspectiveConfiguration(candidate); + return integrationApplyReceiptSchema.parse(receipt); +} diff --git a/apps/api/src/test/configuration-postgres-fixture.ts b/apps/api/src/test/configuration-postgres-fixture.ts index 6fae6ac0b..fefd97aaf 100644 --- a/apps/api/src/test/configuration-postgres-fixture.ts +++ b/apps/api/src/test/configuration-postgres-fixture.ts @@ -18,6 +18,8 @@ export async function createConfigurationPostgresFixture() { "admin_job_log", "admin_audit_log", "capability_binding", + "integration_application_receipt", + "integration_binding_revision", "service_config", "integration_config", "service_secret", diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index bdbd070b0..ec4c9a367 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -59,7 +59,8 @@ export async function ownedServiceHost(options: { extraServiceIds?:string[]; coo name: id, version: "1.0.0", quality: "built-in", - container: {...(options.imageReferences?.[id] ?? { image: `example/${id}`, tag: "1" }), + ...(options.capabilities?.[id]?{provides:options.capabilities[id]}:{}), + container: {...(options.capabilities?.[id]?{expose:[8080]}:{}),...(options.imageReferences?.[id] ?? { image: `example/${id}`, tag: "1" }), ...(options.coordinatorDependency && id==="motis"?{dependsOn:[{service:"data-manager"}]}:{})}, ...(options.schemas?.[id] ? { configSchema: options.schemas[id] } : {}), ...(options.oversizedMaintenancePlan && id==="motis" ? { diff --git a/apps/web/src/components/admin/integrations/CapabilityBindingPicker.tsx b/apps/web/src/components/admin/integrations/CapabilityBindingPicker.tsx index 6b17db15d..508d27e8d 100644 --- a/apps/web/src/components/admin/integrations/CapabilityBindingPicker.tsx +++ b/apps/web/src/components/admin/integrations/CapabilityBindingPicker.tsx @@ -13,15 +13,18 @@ export function CapabilityBindingPicker({ providers, value, onChange, + disabled = false, }: { capability: string; providers: CapabilityProvider[]; value: string | null; onChange: (serviceId: string | null) => void; + disabled?: boolean; }) { return ( void; + disabled: boolean; + activeServiceId: string | null; }) { const providers = services.filter((s) => s.provides.includes(req.capability)); return ( + + Saved: {currentBinding ?? "automatic"} + Active: {activeServiceId ?? "unresolved"} + + {providers.length === 0 && ( No installed services provide this capability @@ -138,20 +152,67 @@ export function RequiredServicesPanel({ const { data: bindingsData, error: bindingsError } = useQuery( admin.queries.bindings(integrationId), ); + const pendingIntent = useRef<{ signature: string; key: string } | null>(null); const setBinding = useMutation({ mutationKey: ["integration-binding", admin.apiUrl, integrationId], retry: false, - mutationFn: async ({ - capability, - serviceId, - }: { + mutationFn: async (input: { + apiUrl: string; + integrationId: string; capability: string; serviceId: string | null; + expectedServiceId: string | null; + expectedRevision: string; + operationId: string; }) => { - await admin.setBinding(integrationId, capability, serviceId); + const value = await admin.setBinding( + input.integrationId, + input.capability, + input.serviceId, + input, + ); + if (pendingIntent.current?.key === input.operationId) pendingIntent.current = null; void admin.refresh(); + return value; }, }); + const owned = + setBinding.variables?.integrationId === integrationId && + setBinding.variables?.apiUrl === admin.apiUrl; + const observedReceipt = + owned && setBinding.data ? setBinding.data : bindingsData?.lastApplication; + const receipt = + observedReceipt && + observedReceipt.desiredRevision === bindingsData?.desiredRevision && + (observedReceipt.runtime === "unknown" || + observedReceipt.runtimeGenerationId === (bindingsData?.runtime?.generationId ?? null)) + ? observedReceipt + : null; + const busy = owned && setBinding.isPending; + function bind(capability: string, serviceId: string | null) { + if (!bindingsData?.revision || busy) return; + const expectedServiceId = + bindingsData.bindings.find((b) => b.capability === capability)?.serviceId ?? null; + const signature = JSON.stringify([ + admin.apiUrl, + integrationId, + capability, + serviceId, + expectedServiceId, + bindingsData.revision, + ]); + if (pendingIntent.current?.signature !== signature) + pendingIntent.current = { signature, key: crypto.randomUUID() }; + setBinding.mutate({ + apiUrl: admin.apiUrl, + integrationId, + capability, + serviceId, + expectedServiceId, + expectedRevision: bindingsData.revision, + operationId: pendingIntent.current.key, + }); + } if (requires.length === 0) return null; @@ -175,9 +236,36 @@ export function RequiredServicesPanel({ gap: 1.5, }} > - {(setBinding.error || bindingsError) && ( - {(setBinding.error || bindingsError)?.message} + {((owned && setBinding.error) || bindingsError) && ( + + {((owned && setBinding.error) || bindingsError)?.message} + + )} + {receipt && ( + + {receipt.runtime === "applied" + ? "Binding saved and active." + : receipt.reason === "integration_disabled" + ? "Binding saved. Integration is disabled; activation is pending." + : receipt.runtime === "failed" + ? "Binding saved. Reload failed; previous provider remains active." + : "Binding status requires reconciliation. Review the saved and live provider before retrying."} + + )} + {!bindingsData?.runtime && ( + Runtime unavailable or integration disabled )} + {requires.map((r, idx) => { const key = r.service ?? r.capability ?? `req-${idx}`; if (r.service) { @@ -197,7 +285,9 @@ export function RequiredServicesPanel({ req={r as RequireEntry & { capability: string }} services={services} currentBinding={bindings.get(capability) ?? null} - onBind={(serviceId) => setBinding.mutate({ capability, serviceId })} + onBind={(serviceId) => bind(capability, serviceId)} + disabled={busy || !bindingsData?.revision || !!bindingsError} + activeServiceId={bindingsData?.runtime?.bindings[capability] ?? null} /> ); } diff --git a/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx b/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx index c87c8eb5b..c5b415668 100644 --- a/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx +++ b/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx @@ -200,7 +200,14 @@ describe("integration administration browser behavior", () => { if (init.method && init.method !== "GET") return pending; if (url.pathname === "/api/admin/integrations") return Response.json([summary]); if (url.pathname === "/api/admin/integrations/demo") return Response.json(detail); - if (url.pathname.endsWith("/bindings")) return Response.json({ bindings: [] }); + if (url.pathname.endsWith("/bindings")) + return Response.json({ + bindings: [], + revision: "a".repeat(64), + desiredRevision: "a".repeat(64), + runtime: null, + lastApplication: null, + }); if (url.pathname.endsWith("/env-vars")) return Response.json({ integrations: [ @@ -426,6 +433,82 @@ describe("integration administration browser behavior", () => { }, ); + it.each(["desired", "generation"])( + "does not label an older successful receipt as current (%s)", + async (mode) => { + transport(() => + Response.json({ + bindings: [{ capability: "search /places", serviceId: "local" }], + revision: "b".repeat(64), + desiredRevision: (mode === "desired" ? "b" : "a").repeat(64), + runtime: { + generationId: + mode === "generation" + ? "22222222-2222-4222-8222-222222222222" + : "11111111-1111-4111-8111-111111111111", + bindings: { "search /places": "old-provider" }, + configRevision: "a".repeat(64), + }, + lastApplication: { + operationId: "old-binding-operation", + integrationId: "demo", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "applied", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", + }, + }), + ); + mount( + , + ); + expect(await screen.findByText(/Active: old-provider/)).toBeInTheDocument(); + expect(screen.queryByText("Binding saved and active.")).not.toBeInTheDocument(); + }, + ); + it("shows saved and live bindings independently when activation fails", async () => { + transport((_url, init) => { + if (init.method && init.method !== "GET") { + expect(JSON.parse(String(init.body))).toEqual({ + serviceId: "local", + expectedServiceId: null, + expectedRevision: "a".repeat(64), + }); + return Response.json({ + operationId: "opk1_syntheticbindingoperation", + integrationId: "demo", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "failed", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", + reason: "reload_failed", + }); + } + return Response.json({ + bindings: [], + revision: "a".repeat(64), + desiredRevision: "a".repeat(64), + runtime: { + generationId: "11111111-1111-4111-8111-111111111111", + bindings: { "search /places": "old-provider" }, + configRevision: "a".repeat(64), + }, + lastApplication: null, + }); + }); + mount( + , + ); + expect(await screen.findByText(/Active: old-provider/)).toBeInTheDocument(); + await userEvent.click(screen.getByRole("combobox")); + await userEvent.click(await screen.findByRole("option", { name: "Local (local)" })); + expect(await screen.findByText(/Binding saved.*Reload failed/)).toBeInTheDocument(); + expect(screen.getByText(/Active: old-provider/)).toBeInTheDocument(); + }); + it.each(["POST", "DELETE"])( "rejects binding %s without success invalidation or retry", async (method) => { @@ -433,6 +516,10 @@ describe("integration administration browser behavior", () => { if (init.method === "GET" || !init.method) { expect(url.pathname).toBe("/api/admin/integrations/demo/bindings"); return Response.json({ + revision: "a".repeat(64), + desiredRevision: "a".repeat(64), + runtime: null, + lastApplication: null, bindings: method === "DELETE" ? [{ capability: "search /places", serviceId: "local" }] : [], }); diff --git a/apps/web/src/lib/admin/integrationAdmin.test.tsx b/apps/web/src/lib/admin/integrationAdmin.test.tsx index 57129deb7..90cfcb84d 100644 --- a/apps/web/src/lib/admin/integrationAdmin.test.tsx +++ b/apps/web/src/lib/admin/integrationAdmin.test.tsx @@ -79,7 +79,7 @@ describe("integration query boundaries and reload observation", () => { await expect(run[kind]()).rejects.toThrow(); }); expect(requests).toHaveLength(1); - const init = requests[0]!.init; + const init = requests[0]?.init; expect(new Headers(init.headers).get("idempotency-key")).toMatch( /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/, ); @@ -93,6 +93,42 @@ describe("integration query boundaries and reload observation", () => { ); }, ); + it.each(["POST", "DELETE"])( + "sends binding %s CAS and the retained intent through the actual generated client", + async (method) => { + const requests = fakeFetch(() => + Response.json({ error: "Binding version changed" }, { status: 409 }), + ); + const { result } = renderHookWithQuery(() => useIntegrationAdmin(), { + queryClient: client(), + }); + const expected = { + expectedServiceId: "old", + expectedRevision: "a".repeat(64), + operationId: "binding-retained-header-0001", + }; + await act(async () => { + await expect( + result.current.setBinding( + "demo", + "search /places", + method === "POST" ? "new" : null, + expected, + ), + ).rejects.toThrow(/Binding version changed/); + }); + expect(requests).toHaveLength(1); + expect(requests[0]?.init.method).toBe(method); + expect(new Headers(requests[0]?.init.headers).get("idempotency-key")).toBe( + expected.operationId, + ); + expect(JSON.parse(String(requests[0]?.init.body))).toEqual({ + expectedServiceId: "old", + expectedRevision: expected.expectedRevision, + ...(method === "POST" ? { serviceId: "new" } : {}), + }); + }, + ); it("keeps full raw manifest and canonical config values without response casts", async () => { const manifest = { id: "demo", diff --git a/apps/web/src/lib/admin/integrationAdmin.ts b/apps/web/src/lib/admin/integrationAdmin.ts index 8c025b3ea..e10a94ce3 100644 --- a/apps/web/src/lib/admin/integrationAdmin.ts +++ b/apps/web/src/lib/admin/integrationAdmin.ts @@ -167,12 +167,27 @@ export function useIntegrationAdmin() { { client }, ), ), - setBinding: (id: string, capability: string, serviceId: string | null) => - integrationOrThrow( + setBinding: ( + id: string, + capability: string, + serviceId: string | null, + expected: { + expectedServiceId: string | null; + expectedRevision: string; + operationId: string; + }, + ) => { + const { operationId, expectedServiceId, expectedRevision } = expected, + body = { expectedServiceId, expectedRevision }, + headers = { "idempotency-key": operationId }; + return integrationOrThrow( serviceId === null - ? sdk.deleteAdminIntegrationBinding(id, capability, undefined, { client }) - : sdk.setAdminIntegrationBinding(id, capability, { serviceId }, { client }), - ), + ? sdk.deleteAdminIntegrationBinding(id, capability, body, headers, { client }) + : sdk.setAdminIntegrationBinding(id, capability, { ...body, serviceId }, headers, { + client, + }), + ); + }, importConfigs: async (input: unknown) => { const parsed = sdk.adminIntegrationConfigImportSchema.safeParse(input); if (!parsed.success) diff --git a/docs/docs/administration/integrations-administration.md b/docs/docs/administration/integrations-administration.md index 381a8a5cf..ab7851695 100644 --- a/docs/docs/administration/integrations-administration.md +++ b/docs/docs/administration/integrations-administration.md @@ -172,10 +172,18 @@ let auto-select or the public fallback take over. The picker only offers services that genuinely provide the capability — a service that doesn't is rejected. -A binding is a stored `(integration, capability) → service` mapping. The -integration host loads all bindings once at startup and re-reads them on reload, -so after changing a binding, reload the integration (or restart the API) for it -to take effect. Like every admin action, setting a binding is audit-logged. +A binding is a stored `(integration, capability) → service` mapping. Saving it +also applies the trusted configuration and reloads the integration. The panel +shows **Saved** and **Active** separately: a saved choice is only active after +the host has activated that exact provider generation. If reload fails, the +previous provider remains active and the desired choice stays saved. Use +**Apply saved binding** to make a new explicit attempt. Disabled integrations +remain pending until enabled. **Refresh binding status** reads the retained +operation outcome without repeating the write. + +Changes check the previously observed mapping and revision, so a stale page +cannot overwrite another administrator's choice. Refresh after a conflict. +Binding changes and their retained outcomes are audit-logged. :::note[Bindings are per integration, not global] The same capability can be bound to different services for different diff --git a/packages/core/src/api/contracts/integration-admin.test.ts b/packages/core/src/api/contracts/integration-admin.test.ts index a216b57c7..985905354 100644 --- a/packages/core/src/api/contracts/integration-admin.test.ts +++ b/packages/core/src/api/contracts/integration-admin.test.ts @@ -115,7 +115,7 @@ const overview = { systemHealth: { status: "degraded", unhealthyCount: 1 }, users: { total: 2, active24h: 1, banned: 0 }, integrations: { total: 3, enabled: 2, unhealthy: 1, unconfigured: 0 }, - services: { running: 1, stopped: 2, unhealthy: 0 }, + services: { running: 1, stopped: 2, unhealthy: 0, unknown: 0 }, attention: [ { type: "health_check_failed", severity: "error", message: "Probe failed", target: "sample" }, ], @@ -235,9 +235,28 @@ describe("integration administration canonical producer contracts", () => { reloadAdminIntegration: { ok: true, jobId: "job" }, listAdminIntegrationBindings: { bindings: [{ capability: "routing-engine", serviceId: "router" }], + revision: "a".repeat(64), + desiredRevision: "a".repeat(64), + runtime: null, + lastApplication: null, + }, + setAdminIntegrationBinding: { + operationId: "binding-test-operation", + integrationId: "sample", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "applied", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", + }, + deleteAdminIntegrationBinding: { + operationId: "binding-test-operation", + integrationId: "sample", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "pending", + runtimeGenerationId: null, + reason: "integration_disabled", }, - setAdminIntegrationBinding: { ok: true }, - deleteAdminIntegrationBinding: { ok: true }, listAdminCredentials: { credentials: [ { @@ -520,13 +539,19 @@ describe("integration administration canonical producer contracts", () => { "reloadAdminIntegration", "reloadAdminIntegrations", "runAdminIntegrationHealthChecks", - "deleteAdminIntegrationBinding", "deleteAdminIntegrationCredential", ] as const) { expect(ops[id].request.body.safeParse(undefined).success).toBe(true); expect(ops[id].request.body.safeParse({}).success).toBe(true); expect(ops[id].request.body.safeParse({ extra: true }).success).toBe(false); } + expect(ops.deleteAdminIntegrationBinding.request.body.safeParse(undefined).success).toBe(false); + expect( + ops.deleteAdminIntegrationBinding.request.body.safeParse({ + expectedServiceId: null, + expectedRevision: "a".repeat(64), + }).success, + ).toBe(true); for (const body of [{}, [], null, { enabled: true }]) expect(ops.patchAdminIntegrationConfig.request.body.safeParse(body).success).toBe(false); }); diff --git a/packages/core/src/api/contracts/integration-admin/application.ts b/packages/core/src/api/contracts/integration-admin/application.ts new file mode 100644 index 000000000..4d79b1612 --- /dev/null +++ b/packages/core/src/api/contracts/integration-admin/application.ts @@ -0,0 +1,39 @@ +import z from "zod/v4"; +import { configurationApplyResultSchema } from "../../../ops/contract"; +/** Live host evidence, independent of the desired database binding. */ +export const integrationRuntimeEvidenceSchema = z.strictObject({ + generationId: z.uuid(), + bindings: z.record(z.string(), z.string().nullable()), + configRevision: z.string().regex(/^[a-f0-9]{64}$/), +}); +export type IntegrationRuntimeEvidence = z.infer; +export const integrationHostApplyEvidenceSchema = z.union([ + ...configurationApplyResultSchema.options.map((schema) => + schema.extend({ publisherCleanupFailed: z.boolean(), registryUpdateFailed: z.boolean() }), + ), + z.strictObject({ + outcome: z.literal("unknown"), + reason: z.literal("transport_or_result"), + publisherCleanupFailed: z.boolean(), + registryUpdateFailed: z.boolean(), + }), +]); +export const integrationApplyReceiptSchema = z.strictObject({ + operationId: z.string().min(1).max(256), + integrationId: z.string().min(1).max(64), + desiredRevision: z.string().regex(/^[a-f0-9]{64}$/), + database: z.enum(["retained", "restored", "conflict", "unknown"]), + runtime: z.enum(["applied", "pending", "failed", "unknown"]), + runtimeGenerationId: z.uuid().nullable(), + reason: z + .enum([ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required", + ]) + .optional(), + hostApply: integrationHostApplyEvidenceSchema.optional(), +}); +export type IntegrationApplyReceipt = z.infer; diff --git a/packages/core/src/api/contracts/integration-admin/common.ts b/packages/core/src/api/contracts/integration-admin/common.ts index e0c432649..ceb30b78b 100644 --- a/packages/core/src/api/contracts/integration-admin/common.ts +++ b/packages/core/src/api/contracts/integration-admin/common.ts @@ -6,6 +6,7 @@ import z from "zod/v4"; import { configurationApplyResultSchema } from "../../../ops/contract"; import { adminCountSchema, adminJsonValueSchema, adminTimestampSchema } from "../admin"; import { adminJsonWire, preserveManifestJson } from "../manifest-json"; +import { integrationApplyReceiptSchema, integrationRuntimeEvidenceSchema } from "./application"; import { adminIntegrationConfigSourceSchema, adminIntegrationCredentialSchema, @@ -149,8 +150,18 @@ export const adminIntegrationBindingSchema = z.strictObject({ }); export const adminIntegrationBindingsSchema = z.strictObject({ bindings: z.array(adminIntegrationBindingSchema), + revision: z.string().regex(/^[a-f0-9]{64}$/), + desiredRevision: z.string().regex(/^[a-f0-9]{64}$/), + runtime: integrationRuntimeEvidenceSchema.nullable(), + lastApplication: integrationApplyReceiptSchema.nullable(), +}); +export const adminIntegrationBindingDeleteSchema = z.strictObject({ + expectedServiceId: z.string().min(1).max(64).nullable(), + expectedRevision: z.string().regex(/^[a-f0-9]{64}$/), +}); +export const adminIntegrationBindingSetSchema = adminIntegrationBindingDeleteSchema.extend({ + serviceId: z.string().min(1).max(64), }); -export const adminIntegrationBindingSetSchema = z.strictObject({ serviceId: z.string().min(1) }); export const adminIntegrationEmptyMutationSchema = z.strictObject({}).optional(); export const adminIntegrationIdempotencyHeadersSchema = z.object({ "idempotency-key": z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/), diff --git a/packages/core/src/api/contracts/integration-admin/index.ts b/packages/core/src/api/contracts/integration-admin/index.ts index 7fb4bf350..c5dfcb7c8 100644 --- a/packages/core/src/api/contracts/integration-admin/index.ts +++ b/packages/core/src/api/contracts/integration-admin/index.ts @@ -1,3 +1,4 @@ +export * from "./application"; export * from "./common"; export * from "./credentials"; export * from "./health"; diff --git a/packages/core/src/api/contracts/integration-admin/operations.ts b/packages/core/src/api/contracts/integration-admin/operations.ts index 462ee1364..930f3aa33 100644 --- a/packages/core/src/api/contracts/integration-admin/operations.ts +++ b/packages/core/src/api/contracts/integration-admin/operations.ts @@ -1,6 +1,8 @@ import z from "zod/v4"; import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; +import { integrationApplyReceiptSchema } from "./application"; import { + adminIntegrationBindingDeleteSchema, adminIntegrationBindingParamsSchema, adminIntegrationBindingSetSchema, adminIntegrationBindingsSchema, @@ -50,7 +52,7 @@ const reloaded = { 200: adminJson(adminIntegrationReloadResultSchema), 409: adminJson(httpErrorSchema), }; -const ok = adminJson(z.strictObject({ ok: z.literal(true) })); +const _ok = adminJson(z.strictObject({ ok: z.literal(true) })); /** Complete integration/credential/overview family. Authentication metadata never enforces access. */ export const integrationAdminOperations = { @@ -175,9 +177,15 @@ export const integrationAdminOperations = { auth: "admin", request: { params: adminIntegrationBindingParamsSchema, + headers: adminIntegrationIdempotencyHeadersSchema, body: adminIntegrationBindingSetSchema, }, - responses: { ...common, 200: ok, 503: adminJson(httpErrorSchema) }, + responses: { + ...common, + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), + 200: adminJson(integrationApplyReceiptSchema), + }, }, deleteAdminIntegrationBinding: { method: "DELETE", @@ -185,9 +193,15 @@ export const integrationAdminOperations = { auth: "admin", request: { params: adminIntegrationBindingParamsSchema, - body: adminIntegrationEmptyMutationSchema, + headers: adminIntegrationIdempotencyHeadersSchema, + body: adminIntegrationBindingDeleteSchema, + }, + responses: { + ...common, + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), + 200: adminJson(integrationApplyReceiptSchema), }, - responses: { ...common, 200: ok }, }, listAdminCredentials: { method: "GET", diff --git a/packages/core/src/api/generated/admin/integration-admin.generated.ts b/packages/core/src/api/generated/admin/integration-admin.generated.ts index c656d0b3e..c36181954 100644 --- a/packages/core/src/api/generated/admin/integration-admin.generated.ts +++ b/packages/core/src/api/generated/admin/integration-admin.generated.ts @@ -5,11 +5,14 @@ import type { DeleteAdminIntegrationBinding400, DeleteAdminIntegrationBinding401, DeleteAdminIntegrationBinding403, + DeleteAdminIntegrationBinding409, DeleteAdminIntegrationBinding413, DeleteAdminIntegrationBinding415, DeleteAdminIntegrationBinding429, DeleteAdminIntegrationBinding500, + DeleteAdminIntegrationBinding503, DeleteAdminIntegrationBindingBody, + DeleteAdminIntegrationBindingHeaders, DeleteAdminIntegrationCredential200, DeleteAdminIntegrationCredential400, DeleteAdminIntegrationCredential401, @@ -186,12 +189,14 @@ import type { SetAdminIntegrationBinding400, SetAdminIntegrationBinding401, SetAdminIntegrationBinding403, + SetAdminIntegrationBinding409, SetAdminIntegrationBinding413, SetAdminIntegrationBinding415, SetAdminIntegrationBinding429, SetAdminIntegrationBinding500, SetAdminIntegrationBinding503, SetAdminIntegrationBindingBody, + SetAdminIntegrationBindingHeaders, SetAdminIntegrationCredential200, SetAdminIntegrationCredential400, SetAdminIntegrationCredential401, @@ -2052,6 +2057,11 @@ export type setAdminIntegrationBindingResponse403 = { status: 403; }; +export type setAdminIntegrationBindingResponse409 = { + data: SetAdminIntegrationBinding409; + status: 409; +}; + export type setAdminIntegrationBindingResponse413 = { data: SetAdminIntegrationBinding413; status: 413; @@ -2084,6 +2094,7 @@ export type setAdminIntegrationBindingResponseError = ( | setAdminIntegrationBindingResponse400 | setAdminIntegrationBindingResponse401 | setAdminIntegrationBindingResponse403 + | setAdminIntegrationBindingResponse409 | setAdminIntegrationBindingResponse413 | setAdminIntegrationBindingResponse415 | setAdminIntegrationBindingResponse429 @@ -2105,6 +2116,7 @@ export const setAdminIntegrationBinding = async ( integrationId: string, capability: string, setAdminIntegrationBindingBody: SetAdminIntegrationBindingBody, + headers: SetAdminIntegrationBindingHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -2131,7 +2143,7 @@ export const setAdminIntegrationBinding = async ( { ...options, method: "POST", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(setAdminIntegrationBindingBody), }, ); @@ -2157,6 +2169,11 @@ export type deleteAdminIntegrationBindingResponse403 = { status: 403; }; +export type deleteAdminIntegrationBindingResponse409 = { + data: DeleteAdminIntegrationBinding409; + status: 409; +}; + export type deleteAdminIntegrationBindingResponse413 = { data: DeleteAdminIntegrationBinding413; status: 413; @@ -2177,6 +2194,11 @@ export type deleteAdminIntegrationBindingResponse500 = { status: 500; }; +export type deleteAdminIntegrationBindingResponse503 = { + data: DeleteAdminIntegrationBinding503; + status: 503; +}; + export type deleteAdminIntegrationBindingResponseSuccess = deleteAdminIntegrationBindingResponse200 & { headers: Headers; @@ -2185,10 +2207,12 @@ export type deleteAdminIntegrationBindingResponseError = ( | deleteAdminIntegrationBindingResponse400 | deleteAdminIntegrationBindingResponse401 | deleteAdminIntegrationBindingResponse403 + | deleteAdminIntegrationBindingResponse409 | deleteAdminIntegrationBindingResponse413 | deleteAdminIntegrationBindingResponse415 | deleteAdminIntegrationBindingResponse429 | deleteAdminIntegrationBindingResponse500 + | deleteAdminIntegrationBindingResponse503 ) & { headers: Headers; }; @@ -2204,7 +2228,8 @@ export const getDeleteAdminIntegrationBindingUrl = (integrationId: string, capab export const deleteAdminIntegrationBinding = async ( integrationId: string, capability: string, - deleteAdminIntegrationBindingBody?: DeleteAdminIntegrationBindingBody, + deleteAdminIntegrationBindingBody: DeleteAdminIntegrationBindingBody, + headers: DeleteAdminIntegrationBindingHeaders, options?: Parameters[1], ): Promise => { const getHeaders = ( @@ -2231,7 +2256,7 @@ export const deleteAdminIntegrationBinding = async ( { ...options, method: "DELETE", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, body: JSON.stringify(deleteAdminIntegrationBindingBody), }, ); diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200.ts index ab996d145..f085609ab 100644 --- a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200.ts +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200.ts @@ -1,6 +1,26 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { DeleteAdminIntegrationBinding200Database } from "./deleteAdminIntegrationBinding200Database"; +import type { DeleteAdminIntegrationBinding200HostApply } from "./deleteAdminIntegrationBinding200HostApply"; +import type { DeleteAdminIntegrationBinding200Reason } from "./deleteAdminIntegrationBinding200Reason"; +import type { DeleteAdminIntegrationBinding200Runtime } from "./deleteAdminIntegrationBinding200Runtime"; export type DeleteAdminIntegrationBinding200 = { - ok: true; + /** + * @minLength 1 + * @maxLength 256 + */ + operationId: string; + /** + * @minLength 1 + * @maxLength 64 + */ + integrationId: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + database: DeleteAdminIntegrationBinding200Database; + runtime: DeleteAdminIntegrationBinding200Runtime; + runtimeGenerationId: string | null; + reason?: DeleteAdminIntegrationBinding200Reason; + hostApply?: DeleteAdminIntegrationBinding200HostApply; }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Database.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Database.ts new file mode 100644 index 000000000..78ff54654 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Database.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationBinding200Database = + (typeof DeleteAdminIntegrationBinding200Database)[keyof typeof DeleteAdminIntegrationBinding200Database]; + +export const DeleteAdminIntegrationBinding200Database = { + retained: "retained", + restored: "restored", + conflict: "conflict", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200HostApply.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200HostApply.ts new file mode 100644 index 000000000..9774901c7 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200HostApply.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationBinding200HostApply = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Reason.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Reason.ts new file mode 100644 index 000000000..b3f71dbca --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Reason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationBinding200Reason = + (typeof DeleteAdminIntegrationBinding200Reason)[keyof typeof DeleteAdminIntegrationBinding200Reason]; + +export const DeleteAdminIntegrationBinding200Reason = { + integration_disabled: "integration_disabled", + reload_failed: "reload_failed", + ownership_changed: "ownership_changed", + authority_unavailable: "authority_unavailable", + reconciliation_required: "reconciliation_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Runtime.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Runtime.ts new file mode 100644 index 000000000..06ab68a4b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding200Runtime.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationBinding200Runtime = + (typeof DeleteAdminIntegrationBinding200Runtime)[keyof typeof DeleteAdminIntegrationBinding200Runtime]; + +export const DeleteAdminIntegrationBinding200Runtime = { + applied: "applied", + pending: "pending", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts new file mode 100644 index 000000000..da51a4527 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationBinding409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding503.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding503.ts new file mode 100644 index 000000000..7dcfa7d75 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationBinding503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingBody.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingBody.ts index 9464f7ea2..bce8b17f7 100644 --- a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingBody.ts +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingBody.ts @@ -1,4 +1,8 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. -export type DeleteAdminIntegrationBindingBody = { [key: string]: unknown }; +export type DeleteAdminIntegrationBindingBody = { + expectedServiceId: string | null; + /** @pattern ^[a-f0-9]{64}$ */ + expectedRevision: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingHeaders.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingHeaders.ts new file mode 100644 index 000000000..5d026f21e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBindingHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationBindingHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/index.ts b/packages/core/src/api/generated/admin/integration-models/index.ts index 5df2d4dfd..18b9e8433 100644 --- a/packages/core/src/api/generated/admin/integration-models/index.ts +++ b/packages/core/src/api/generated/admin/integration-models/index.ts @@ -8,14 +8,21 @@ export * from "./adminImportAdminIntegrationConfigsBodySchema0"; export * from "./adminListAdminIntegrationEnvVars200Schema0"; export * from "./adminPatchAdminIntegrationConfigBodySchema0"; export * from "./deleteAdminIntegrationBinding200"; +export * from "./deleteAdminIntegrationBinding200Database"; +export * from "./deleteAdminIntegrationBinding200HostApply"; +export * from "./deleteAdminIntegrationBinding200Reason"; +export * from "./deleteAdminIntegrationBinding200Runtime"; export * from "./deleteAdminIntegrationBinding400"; export * from "./deleteAdminIntegrationBinding401"; export * from "./deleteAdminIntegrationBinding403"; +export * from "./deleteAdminIntegrationBinding409"; export * from "./deleteAdminIntegrationBinding413"; export * from "./deleteAdminIntegrationBinding415"; export * from "./deleteAdminIntegrationBinding429"; export * from "./deleteAdminIntegrationBinding500"; +export * from "./deleteAdminIntegrationBinding503"; export * from "./deleteAdminIntegrationBindingBody"; +export * from "./deleteAdminIntegrationBindingHeaders"; export * from "./deleteAdminIntegrationCredential200"; export * from "./deleteAdminIntegrationCredential400"; export * from "./deleteAdminIntegrationCredential401"; @@ -195,6 +202,13 @@ export * from "./listAdminCredentials429"; export * from "./listAdminCredentials500"; export * from "./listAdminIntegrationBindings200"; export * from "./listAdminIntegrationBindings200BindingsItem"; +export * from "./listAdminIntegrationBindings200LastApplication"; +export * from "./listAdminIntegrationBindings200LastApplicationDatabase"; +export * from "./listAdminIntegrationBindings200LastApplicationHostApply"; +export * from "./listAdminIntegrationBindings200LastApplicationReason"; +export * from "./listAdminIntegrationBindings200LastApplicationRuntime"; +export * from "./listAdminIntegrationBindings200Runtime"; +export * from "./listAdminIntegrationBindings200RuntimeBindings"; export * from "./listAdminIntegrationBindings400"; export * from "./listAdminIntegrationBindings401"; export * from "./listAdminIntegrationBindings403"; @@ -268,15 +282,21 @@ export * from "./runAdminIntegrationHealthChecks429"; export * from "./runAdminIntegrationHealthChecks500"; export * from "./runAdminIntegrationHealthChecksBody"; export * from "./setAdminIntegrationBinding200"; +export * from "./setAdminIntegrationBinding200Database"; +export * from "./setAdminIntegrationBinding200HostApply"; +export * from "./setAdminIntegrationBinding200Reason"; +export * from "./setAdminIntegrationBinding200Runtime"; export * from "./setAdminIntegrationBinding400"; export * from "./setAdminIntegrationBinding401"; export * from "./setAdminIntegrationBinding403"; +export * from "./setAdminIntegrationBinding409"; export * from "./setAdminIntegrationBinding413"; export * from "./setAdminIntegrationBinding415"; export * from "./setAdminIntegrationBinding429"; export * from "./setAdminIntegrationBinding500"; export * from "./setAdminIntegrationBinding503"; export * from "./setAdminIntegrationBindingBody"; +export * from "./setAdminIntegrationBindingHeaders"; export * from "./setAdminIntegrationCredential200"; export * from "./setAdminIntegrationCredential400"; export * from "./setAdminIntegrationCredential401"; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200.ts index 0d8a7bdcc..5db36452b 100644 --- a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200.ts +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200.ts @@ -1,7 +1,15 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. import type { ListAdminIntegrationBindings200BindingsItem } from "./listAdminIntegrationBindings200BindingsItem"; +import type { ListAdminIntegrationBindings200LastApplication } from "./listAdminIntegrationBindings200LastApplication"; +import type { ListAdminIntegrationBindings200Runtime } from "./listAdminIntegrationBindings200Runtime"; export type ListAdminIntegrationBindings200 = { bindings: ListAdminIntegrationBindings200BindingsItem[]; + /** @pattern ^[a-f0-9]{64}$ */ + revision: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + runtime: ListAdminIntegrationBindings200Runtime; + lastApplication: ListAdminIntegrationBindings200LastApplication; }; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplication.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplication.ts new file mode 100644 index 000000000..534bd8f93 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplication.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminIntegrationBindings200LastApplicationDatabase } from "./listAdminIntegrationBindings200LastApplicationDatabase"; +import type { ListAdminIntegrationBindings200LastApplicationHostApply } from "./listAdminIntegrationBindings200LastApplicationHostApply"; +import type { ListAdminIntegrationBindings200LastApplicationReason } from "./listAdminIntegrationBindings200LastApplicationReason"; +import type { ListAdminIntegrationBindings200LastApplicationRuntime } from "./listAdminIntegrationBindings200LastApplicationRuntime"; + +export type ListAdminIntegrationBindings200LastApplication = { + /** + * @minLength 1 + * @maxLength 256 + */ + operationId: string; + /** + * @minLength 1 + * @maxLength 64 + */ + integrationId: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + database: ListAdminIntegrationBindings200LastApplicationDatabase; + runtime: ListAdminIntegrationBindings200LastApplicationRuntime; + runtimeGenerationId: string | null; + reason?: ListAdminIntegrationBindings200LastApplicationReason; + hostApply?: ListAdminIntegrationBindings200LastApplicationHostApply; +} | null; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationDatabase.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationDatabase.ts new file mode 100644 index 000000000..09cdc75d5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationDatabase.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminIntegrationBindings200LastApplicationDatabase = + (typeof ListAdminIntegrationBindings200LastApplicationDatabase)[keyof typeof ListAdminIntegrationBindings200LastApplicationDatabase]; + +export const ListAdminIntegrationBindings200LastApplicationDatabase = { + retained: "retained", + restored: "restored", + conflict: "conflict", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationHostApply.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationHostApply.ts new file mode 100644 index 000000000..8327e2848 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationHostApply.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminIntegrationBindings200LastApplicationHostApply = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationReason.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationReason.ts new file mode 100644 index 000000000..7746469a2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminIntegrationBindings200LastApplicationReason = + (typeof ListAdminIntegrationBindings200LastApplicationReason)[keyof typeof ListAdminIntegrationBindings200LastApplicationReason]; + +export const ListAdminIntegrationBindings200LastApplicationReason = { + integration_disabled: "integration_disabled", + reload_failed: "reload_failed", + ownership_changed: "ownership_changed", + authority_unavailable: "authority_unavailable", + reconciliation_required: "reconciliation_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationRuntime.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationRuntime.ts new file mode 100644 index 000000000..a144a9931 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200LastApplicationRuntime.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminIntegrationBindings200LastApplicationRuntime = + (typeof ListAdminIntegrationBindings200LastApplicationRuntime)[keyof typeof ListAdminIntegrationBindings200LastApplicationRuntime]; + +export const ListAdminIntegrationBindings200LastApplicationRuntime = { + applied: "applied", + pending: "pending", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200Runtime.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200Runtime.ts new file mode 100644 index 000000000..c27eae929 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200Runtime.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminIntegrationBindings200RuntimeBindings } from "./listAdminIntegrationBindings200RuntimeBindings"; + +export type ListAdminIntegrationBindings200Runtime = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + generationId: string; + bindings: ListAdminIntegrationBindings200RuntimeBindings; + /** @pattern ^[a-f0-9]{64}$ */ + configRevision: string; +} | null; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200RuntimeBindings.ts b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200RuntimeBindings.ts new file mode 100644 index 000000000..07a6b2dbc --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminIntegrationBindings200RuntimeBindings.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminIntegrationBindings200RuntimeBindings = { [key: string]: string | null }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200.ts index b3563cf2e..ea09368ca 100644 --- a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200.ts +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200.ts @@ -1,6 +1,26 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { SetAdminIntegrationBinding200Database } from "./setAdminIntegrationBinding200Database"; +import type { SetAdminIntegrationBinding200HostApply } from "./setAdminIntegrationBinding200HostApply"; +import type { SetAdminIntegrationBinding200Reason } from "./setAdminIntegrationBinding200Reason"; +import type { SetAdminIntegrationBinding200Runtime } from "./setAdminIntegrationBinding200Runtime"; export type SetAdminIntegrationBinding200 = { - ok: true; + /** + * @minLength 1 + * @maxLength 256 + */ + operationId: string; + /** + * @minLength 1 + * @maxLength 64 + */ + integrationId: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + database: SetAdminIntegrationBinding200Database; + runtime: SetAdminIntegrationBinding200Runtime; + runtimeGenerationId: string | null; + reason?: SetAdminIntegrationBinding200Reason; + hostApply?: SetAdminIntegrationBinding200HostApply; }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Database.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Database.ts new file mode 100644 index 000000000..0e96f11d3 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Database.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationBinding200Database = + (typeof SetAdminIntegrationBinding200Database)[keyof typeof SetAdminIntegrationBinding200Database]; + +export const SetAdminIntegrationBinding200Database = { + retained: "retained", + restored: "restored", + conflict: "conflict", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200HostApply.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200HostApply.ts new file mode 100644 index 000000000..dd04f3b85 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200HostApply.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationBinding200HostApply = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Reason.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Reason.ts new file mode 100644 index 000000000..a014c936a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Reason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationBinding200Reason = + (typeof SetAdminIntegrationBinding200Reason)[keyof typeof SetAdminIntegrationBinding200Reason]; + +export const SetAdminIntegrationBinding200Reason = { + integration_disabled: "integration_disabled", + reload_failed: "reload_failed", + ownership_changed: "ownership_changed", + authority_unavailable: "authority_unavailable", + reconciliation_required: "reconciliation_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Runtime.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Runtime.ts new file mode 100644 index 000000000..48353688c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding200Runtime.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationBinding200Runtime = + (typeof SetAdminIntegrationBinding200Runtime)[keyof typeof SetAdminIntegrationBinding200Runtime]; + +export const SetAdminIntegrationBinding200Runtime = { + applied: "applied", + pending: "pending", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts new file mode 100644 index 000000000..7cbd49f7e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationBinding409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingBody.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingBody.ts index 0012eac35..3a48005f1 100644 --- a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingBody.ts +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingBody.ts @@ -2,6 +2,12 @@ // Generated by pnpm sdk:generate. Do not edit. export type SetAdminIntegrationBindingBody = { - /** @minLength 1 */ + expectedServiceId: string | null; + /** @pattern ^[a-f0-9]{64}$ */ + expectedRevision: string; + /** + * @minLength 1 + * @maxLength 64 + */ serviceId: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingHeaders.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingHeaders.ts new file mode 100644 index 000000000..db8c9a89d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBindingHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationBindingHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts index 581ed3b89..09c42b999 100644 --- a/packages/core/src/ops/configuration-preparation-records.ts +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -115,6 +115,7 @@ export const preparedConfigurationRecordSchema = z.strictObject({ }), ), ), + bindings: z.record(id, z.record(z.string().min(1).max(128), id)).optional(), legacyImport: z .strictObject({ version: z.literal(1), diff --git a/scripts/check-subject-data.ts b/scripts/check-subject-data.ts index c69194828..169c9292a 100644 --- a/scripts/check-subject-data.ts +++ b/scripts/check-subject-data.ts @@ -22,6 +22,7 @@ export const DIRECT_USER_FK_CLASSIFICATIONS: Record = { "admin_job.created_by": "admin-jobs-attribution", "installed_extension.installed_by": "installed-component-attribution", "installed_integration.installed_by": "installed-component-attribution", + "integration_application_receipt.actor_id": "admin-audit-attribution", "integration_secret.updated_by": "secret-update-attribution", "labeled_place.user_id": "labeled-places", "mangrove_keypair.user_id": "mangrove-keypairs", diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 7de2ecc57..b07680606 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -28,7 +28,9 @@ "apps/api/src/account/admin-invariant-migration.test.ts", "apps/api/src/account/admin-reauthentication-postgres.test.ts", "apps/api/src/account/admin-reauthentication-http-postgres.test.ts", - "apps/api/src/account/account-provisioning-postgres.test.ts" + "apps/api/src/account/account-provisioning-postgres.test.ts", + "apps/api/src/services/integration-application-postgres.test.ts", + "apps/api/src/services/integration-application.test.ts" ], "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", From d721c425dded1da76ea26b80937b741307f939ce Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 20:02:16 +0100 Subject: [PATCH 047/135] fix(admin): enforce effective configuration ownership --- apps/api/openapi.json | 3982 ++++++++++++++--- apps/api/src/integration-config.test.ts | 26 +- apps/api/src/integration-config.ts | 6 +- .../__tests__/admin-integrations-sdk.test.ts | 74 +- .../src/routes/admin-integration-contract.ts | 24 +- .../routes/admin-integration-credentials.ts | 54 +- apps/api/src/routes/admin-integrations.ts | 19 +- apps/api/src/routes/admin-service-contract.ts | 10 + apps/api/src/routes/admin-services.ts | 51 +- .../coordinator-lifecycle-postgres.test.ts | 10 +- .../prospective-configuration.test.ts | 2 + .../legacy-integration-import.ts | 9 +- .../configuration-ownership-view.test.ts | 132 + .../services/configuration-ownership-view.ts | 191 + .../services/integration-application.test.ts | 377 +- .../src/services/integration-application.ts | 4 +- .../integration-credential-application.ts | 44 + .../integration-credential-status.test.ts | 73 +- .../services/integration-credential-status.ts | 78 +- apps/api/src/services/job-runner.ts | 39 +- .../services/managed-service-credentials.ts | 12 + .../src/services/prospective-configuration.ts | 200 +- .../trusted-configuration-producer.ts | 17 + .../src/__fixtures__/owned-service-host.ts | 4 +- .../admin/integrations/BulkConfigure.tsx | 33 +- .../admin/integrations/ConfigSchemaForm.tsx | 8 +- .../admin/integrations/IntegrationDetail.tsx | 62 +- .../integrations/SetCredentialDialog.tsx | 50 +- .../IntegrationAdminConsumers.test.tsx | 96 +- .../admin/integrations/__tests__/fixtures.ts | 1 + .../admin/services/ServiceConfigForm.tsx | 8 +- .../admin/services/ServiceCredentials.tsx | 37 +- .../admin/services/ServiceDetail.tsx | 1 + .../admin/shared/SchemaConfigForm.tsx | 46 +- .../SchemaConfigForm.ownership.test.tsx | 27 + .../src/lib/admin/integrationAdmin.test.tsx | 7 +- apps/web/src/lib/admin/integrationAdmin.ts | 23 +- .../api/contracts/configuration-ownership.ts | 25 + packages/core/src/api/contracts/index.ts | 1 + .../api/contracts/integration-admin.test.ts | 36 +- .../api/contracts/integration-admin/common.ts | 8 + .../integration-admin/credentials.ts | 11 +- .../contracts/integration-admin/operations.ts | 67 +- .../src/api/contracts/service-admin/index.ts | 27 +- .../api/contracts/service-admin/services.ts | 12 +- .../admin/integration-admin.generated.ts | 9 +- .../deleteAdminIntegrationBinding409.ts | 18 + .../deleteAdminIntegrationCredential200.ts | 25 +- ...teAdminIntegrationCredential200Database.ts | 12 + ...eAdminIntegrationCredential200HostApply.ts | 34 + ...leteAdminIntegrationCredential200Reason.ts | 13 + ...eteAdminIntegrationCredential200Runtime.ts | 12 + .../deleteAdminIntegrationCredential409.ts | 18 + .../deleteAdminIntegrationCredential503.ts | 20 + .../deleteAdminIntegrationCredentialBody.ts | 5 +- .../disableAdminIntegration409.ts | 18 + .../enableAdminIntegration409.ts | 18 + .../getAdminIntegration200.ts | 8 + .../getAdminIntegration200Application.ts | 26 + ...tAdminIntegration200ApplicationDatabase.ts | 12 + ...AdminIntegration200ApplicationHostApply.ts | 34 + ...getAdminIntegration200ApplicationReason.ts | 13 + ...etAdminIntegration200ApplicationRuntime.ts | 12 + ...AdminIntegration200CredentialStatusItem.ts | 2 + ...gration200CredentialStatusItemOwnership.ts | 17 + ...00CredentialStatusItemOwnershipAdoption.ts | 10 + ...0CredentialStatusItemOwnershipManagedBy.ts | 10 + ...n200CredentialStatusItemOwnershipReason.ts | 13 + ...n200CredentialStatusItemOwnershipSource.ts | 13 + .../getAdminIntegration200Ownership.ts | 19 + ...getAdminIntegration200OwnershipAdoption.ts | 10 + ...etAdminIntegration200OwnershipManagedBy.ts | 10 + .../getAdminIntegration200OwnershipReason.ts | 13 + .../getAdminIntegration200OwnershipSource.ts | 13 + .../getAdminIntegration200Runtime.ts | 11 + .../getAdminIntegration200RuntimeBindings.ts | 4 + .../getAdminIntegrationCredentials200.ts | 6 + ...minIntegrationCredentials200Application.ts | 26 + ...rationCredentials200ApplicationDatabase.ts | 12 + ...ationCredentials200ApplicationHostApply.ts | 34 + ...egrationCredentials200ApplicationReason.ts | 13 + ...grationCredentials200ApplicationRuntime.ts | 12 + ...ntegrationCredentials200CredentialsItem.ts | 2 + ...nCredentials200CredentialsItemOwnership.ts | 17 + ...ials200CredentialsItemOwnershipAdoption.ts | 10 + ...als200CredentialsItemOwnershipManagedBy.ts | 10 + ...ntials200CredentialsItemOwnershipReason.ts | 13 + ...ntials200CredentialsItemOwnershipSource.ts | 13 + ...etAdminIntegrationCredentials200Runtime.ts | 11 + ...ntegrationCredentials200RuntimeBindings.ts | 4 + .../importAdminIntegrationConfigs409.ts | 18 + .../admin/integration-models/index.ts | 38 + .../patchAdminIntegrationConfig409.ts | 18 + .../setAdminIntegrationBinding409.ts | 18 + .../setAdminIntegrationCredential200.ts | 25 +- ...etAdminIntegrationCredential200Database.ts | 12 + ...tAdminIntegrationCredential200HostApply.ts | 34 + .../setAdminIntegrationCredential200Reason.ts | 13 + ...setAdminIntegrationCredential200Runtime.ts | 12 + .../setAdminIntegrationCredential409.ts | 18 + .../setAdminIntegrationCredentialBody.ts | 2 + .../deleteAdminServiceCredential409.ts | 16 +- .../getAdminServiceConfig200.ts | 2 + .../getAdminServiceConfig200Ownership.ts | 19 + ...tAdminServiceConfig200OwnershipAdoption.ts | 10 + ...AdminServiceConfig200OwnershipManagedBy.ts | 10 + ...getAdminServiceConfig200OwnershipReason.ts | 13 + ...getAdminServiceConfig200OwnershipSource.ts | 13 + ...minServiceConfig200ResolvedConfigSource.ts | 1 + ...minServiceCredentials200CredentialsItem.ts | 2 + ...eCredentials200CredentialsItemManagedBy.ts | 1 + ...eCredentials200CredentialsItemOwnership.ts | 17 + ...ials200CredentialsItemOwnershipAdoption.ts | 10 + ...als200CredentialsItemOwnershipManagedBy.ts | 10 + ...ntials200CredentialsItemOwnershipReason.ts | 13 + ...ntials200CredentialsItemOwnershipSource.ts | 13 + ...viceCredentials200CredentialsItemSource.ts | 4 + .../generated/admin/service-models/index.ts | 10 + .../restoreAdminServiceBackup409.ts | 14 + .../setAdminServiceCredential409.ts | 16 +- .../setAdminServiceSelection409.ts | 14 + .../service-models/startAdminCompose409.ts | 14 + .../streamAdminServiceLogs409.ts | 14 + .../updateAdminServiceConfig409.ts | 14 + 124 files changed, 6216 insertions(+), 901 deletions(-) create mode 100644 apps/api/src/services/configuration-ownership-view.test.ts create mode 100644 apps/api/src/services/configuration-ownership-view.ts create mode 100644 apps/api/src/services/integration-credential-application.ts create mode 100644 apps/api/src/services/managed-service-credentials.ts create mode 100644 apps/web/src/components/admin/shared/__tests__/SchemaConfigForm.ownership.test.tsx create mode 100644 packages/core/src/api/contracts/configuration-ownership.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Database.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200HostApply.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Reason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Runtime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential503.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Application.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationDatabase.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationHostApply.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationReason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationRuntime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnership.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipAdoption.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipManagedBy.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipReason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipSource.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Ownership.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipAdoption.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipManagedBy.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipReason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipSource.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Runtime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegration200RuntimeBindings.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Application.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationDatabase.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationHostApply.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationReason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationRuntime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnership.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipAdoption.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipReason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipSource.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Runtime.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200RuntimeBindings.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Database.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200HostApply.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Reason.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Runtime.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200Ownership.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipAdoption.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipManagedBy.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipSource.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnership.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipAdoption.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipManagedBy.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipReason.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipSource.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 515797338..e8cee00fd 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -13713,51 +13713,138 @@ "schema": { "anyOf": [ { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false }, - "jobId": { - "type": "string" + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false } - }, - "required": [ - "error" - ], - "additionalProperties": false + ] }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] }, "error": { "type": "string" }, - "message": { - "type": "string" - }, - "code": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "statusCode", + "code", "error", - "message" + "ownership" ], "additionalProperties": false } @@ -14736,6 +14823,69 @@ "missing" ] }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + }, "sharedSecretName": { "type": "string" }, @@ -14801,6 +14951,253 @@ "additionalProperties": false } }, + "configurationRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runtime": { + "anyOf": [ + { + "type": "object", + "properties": { + "generationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "null" + ] + } + }, + "configRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "generationId", + "bindings", + "configRevision" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "application": { + "anyOf": [ + { + "type": "object", + "properties": { + "operationId": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "integrationId": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "database": { + "type": "string", + "enum": [ + "retained", + "restored", + "conflict", + "unknown" + ] + }, + "runtime": { + "type": "string", + "enum": [ + "applied", + "pending", + "failed", + "unknown" + ] + }, + "runtimeGenerationId": { + "anyOf": [ + { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "type": "string", + "enum": [ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required" + ] + }, + "hostApply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "operationId", + "integrationId", + "desiredRevision", + "database", + "runtime", + "runtimeGenerationId" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, "secretsConfigured": { "type": "boolean" } @@ -14809,6 +15206,9 @@ "integrationId", "name", "credentials", + "configurationRevision", + "runtime", + "application", "secretsConfigured" ], "additionalProperties": false @@ -15857,11 +16257,16 @@ "schema": { "type": "object", "properties": { + "expectedRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "value": { "type": "string" } }, "required": [ + "expectedRevision", "value" ], "additionalProperties": false @@ -15877,31 +16282,200 @@ "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", + "operationId": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "integrationId": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "database": { + "type": "string", "enum": [ - true + "retained", + "restored", + "conflict", + "unknown" ] }, - "message": { - "type": "string" + "runtime": { + "type": "string", + "enum": [ + "applied", + "pending", + "failed", + "unknown" + ] }, - "reloaded": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "runtimeGenerationId": { + "anyOf": [ + { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + { + "type": "null" + } + ] }, - "enabled": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "reason": { + "type": "string", + "enum": [ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required" + ] + }, + "hostApply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] } }, "required": [ - "ok", - "message", - "reloaded", - "enabled" + "operationId", + "integrationId", + "desiredRevision", + "database", + "runtime", + "runtimeGenerationId" ], "additionalProperties": false } @@ -16154,52 +16728,158 @@ "application/json": { "schema": { "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, { "type": "object", "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", + "code": { + "type": "string", "enum": [ - false + "CONFIGURATION_SOURCE_OWNED" ] }, - "message": { + "error": { "type": "string" }, - "jobId": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "error" + "code", + "error", + "ownership" ], "additionalProperties": false }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] }, "error": { "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" } }, "required": [ - "statusCode", - "error", - "message" + "code", + "error" ], "additionalProperties": false } @@ -16539,12 +17219,20 @@ } ], "requestBody": { - "required": false, + "required": true, "content": { "application/json": { "schema": { "type": "object", - "properties": {}, + "properties": { + "expectedRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "expectedRevision" + ], "additionalProperties": false } } @@ -16558,31 +17246,200 @@ "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", + "operationId": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "integrationId": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "database": { + "type": "string", "enum": [ - true + "retained", + "restored", + "conflict", + "unknown" ] }, - "message": { - "type": "string" + "runtime": { + "type": "string", + "enum": [ + "applied", + "pending", + "failed", + "unknown" + ] }, - "reloaded": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "runtimeGenerationId": { + "anyOf": [ + { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + { + "type": "null" + } + ] }, - "enabled": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "reason": { + "type": "string", + "enum": [ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required" + ] + }, + "hostApply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] } }, "required": [ - "ok", - "message", - "reloaded", - "enabled" + "operationId", + "integrationId", + "desiredRevision", + "database", + "runtime", + "runtimeGenerationId" ], "additionalProperties": false } @@ -16835,52 +17692,158 @@ "application/json": { "schema": { "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, { "type": "object", "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", + "code": { + "type": "string", "enum": [ - false + "CONFIGURATION_SOURCE_OWNED" ] }, - "message": { + "error": { "type": "string" }, - "jobId": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "error" + "code", + "error", + "ownership" ], "additionalProperties": false }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] }, "error": { "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" } }, "required": [ - "statusCode", - "error", - "message" + "code", + "error" ], "additionalProperties": false } @@ -17101,6 +18064,66 @@ } } } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } } }, "tags": [ @@ -28039,6 +29062,108 @@ "additionalProperties": false } ] + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] + }, + "error": { + "type": "string" + }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "code", + "error", + "ownership" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] + }, + "error": { + "type": "string" + } + }, + "required": [ + "code", + "error" + ], + "additionalProperties": false } ] } @@ -29989,6 +31114,319 @@ "additionalProperties": false } }, + "ownership": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "configurationRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runtime": { + "anyOf": [ + { + "type": "object", + "properties": { + "generationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "bindings": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "null" + ] + } + }, + "configRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "generationId", + "bindings", + "configRevision" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "application": { + "anyOf": [ + { + "type": "object", + "properties": { + "operationId": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "integrationId": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "desiredRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "database": { + "type": "string", + "enum": [ + "retained", + "restored", + "conflict", + "unknown" + ] + }, + "runtime": { + "type": "string", + "enum": [ + "applied", + "pending", + "failed", + "unknown" + ] + }, + "runtimeGenerationId": { + "anyOf": [ + { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "type": "string", + "enum": [ + "integration_disabled", + "reload_failed", + "ownership_changed", + "authority_unavailable", + "reconciliation_required" + ] + }, + "hostApply": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "operationId", + "integrationId", + "desiredRevision", + "database", + "runtime", + "runtimeGenerationId" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, "dependencyStatus": { "type": "array", "items": { @@ -30037,6 +31475,69 @@ "missing" ] }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + }, "sharedSecretName": { "type": "string" }, @@ -30943,52 +32444,158 @@ "application/json": { "schema": { "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, { "type": "object", "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", + "code": { + "type": "string", "enum": [ - false + "CONFIGURATION_SOURCE_OWNED" ] }, - "message": { + "error": { "type": "string" }, - "jobId": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "error" + "code", + "error", + "ownership" ], "additionalProperties": false }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] }, "error": { "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" } }, "required": [ - "statusCode", - "error", - "message" + "code", + "error" ], "additionalProperties": false } @@ -31209,149 +32816,89 @@ } } } - } - }, - "tags": [ - "integration-admin" - ], - "x-openmapx-auth": "admin" - } - }, - "/api/admin/integrations/{id}/disable": { - "post": { - "operationId": "disableAdminIntegration", - "parameters": [ - { - "schema": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]*$" - }, - "in": "path", - "name": "id", - "required": true - }, - { - "schema": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" - }, - "in": "header", - "name": "idempotency-key", - "required": true - } - ], - "requestBody": { - "required": false, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": {}, - "additionalProperties": false - } - } - } - }, - "responses": { - "200": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] - }, - "message": { - "type": "string" - }, - "reloaded": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "enabled": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "ok", - "message", - "reloaded", - "enabled" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } + } + }, + "tags": [ + "integration-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/integrations/{id}/disable": { + "post": { + "operationId": "disableAdminIntegration", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "in": "path", + "name": "id", + "required": true }, - "401": { + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {}, + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "message": { + "type": "string" + }, + "reloaded": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "enabled": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "ok", + "message", + "reloaded", + "enabled" + ], + "additionalProperties": false + } + } + } + }, + "400": { "description": "Response", "content": { "application/json": { @@ -31411,7 +32958,7 @@ } } }, - "403": { + "401": { "description": "Response", "content": { "application/json": { @@ -31471,7 +33018,7 @@ } } }, - "404": { + "403": { "description": "Response", "content": { "application/json": { @@ -31531,7 +33078,7 @@ } } }, - "409": { + "404": { "description": "Response", "content": { "application/json": { @@ -31591,6 +33138,172 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] + }, + "error": { + "type": "string" + }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "code", + "error", + "ownership" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] + }, + "error": { + "type": "string" + } + }, + "required": [ + "code", + "error" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -32125,66 +33838,172 @@ } } }, - "409": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] + }, + "error": { + "type": "string" + }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "code", + "error", + "ownership" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] + }, + "error": { + "type": "string" + } + }, + "required": [ + "code", + "error" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -35226,52 +37045,158 @@ "application/json": { "schema": { "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, { "type": "object", "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", + "code": { + "type": "string", "enum": [ - false + "CONFIGURATION_SOURCE_OWNED" ] }, - "message": { + "error": { "type": "string" }, - "jobId": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "error" + "code", + "error", + "ownership" ], "additionalProperties": false }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] }, "error": { "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" } }, "required": [ - "statusCode", - "error", - "message" + "code", + "error" ], "additionalProperties": false } @@ -36017,52 +37942,158 @@ "application/json": { "schema": { "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, { "type": "object", "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", + "code": { + "type": "string", "enum": [ - false + "CONFIGURATION_SOURCE_OWNED" ] }, - "message": { + "error": { "type": "string" }, - "jobId": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "error" + "code", + "error", + "ownership" ], "additionalProperties": false }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_VERSION_CONFLICT" + ] }, "error": { "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" } }, "required": [ - "statusCode", - "error", - "message" + "code", + "error" ], "additionalProperties": false } @@ -46019,142 +48050,82 @@ } } } - } - }, - "tags": [ - "service-admin" - ], - "x-openmapx-auth": "admin" - } - }, - "/api/admin/services/backups/{name}/restore": { - "post": { - "operationId": "restoreAdminServiceBackup", - "parameters": [ - { - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 128, - "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$" - }, - "in": "path", - "name": "name", - "required": true - } - ], - "requestBody": { - "required": false, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "serviceIds": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - }, - "stopRunning": { - "type": "boolean" - } - }, - "additionalProperties": false - } - } - } - }, - "responses": { - "200": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] - }, - "jobId": { - "type": "string", - "minLength": 1 - } - }, - "required": [ - "ok", - "jobId" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/services/backups/{name}/restore": { + "post": { + "operationId": "restoreAdminServiceBackup", + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$" + }, + "in": "path", + "name": "name", + "required": true + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "serviceIds": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "stopRunning": { + "type": "boolean" + } + }, + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "jobId": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "ok", + "jobId" + ], + "additionalProperties": false + } + } + } }, - "401": { + "400": { "description": "Response", "content": { "application/json": { @@ -46214,7 +48185,7 @@ } } }, - "403": { + "401": { "description": "Response", "content": { "application/json": { @@ -46274,7 +48245,7 @@ } } }, - "404": { + "403": { "description": "Response", "content": { "application/json": { @@ -46334,7 +48305,7 @@ } } }, - "409": { + "404": { "description": "Response", "content": { "application/json": { @@ -46394,6 +48365,153 @@ } } }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] + }, + "error": { + "type": "string" + }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "code", + "error", + "ownership" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -49381,66 +51499,153 @@ } } }, - "409": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] + }, + "error": { + "type": "string" + }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "code", + "error", + "ownership" + ], + "additionalProperties": false + } + ] + } + } + } + }, "413": { "description": "Response", "content": { @@ -52359,6 +54564,7 @@ "enum": [ "default", "database", + "vault", "env" ] } @@ -52372,6 +54578,72 @@ }, "envPrefix": { "type": "string" + }, + "ownership": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } } }, "required": [ @@ -53181,51 +55453,138 @@ "schema": { "anyOf": [ { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false }, - "jobId": { - "type": "string" + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false } - }, - "required": [ - "error" - ], - "additionalProperties": false + ] }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] }, "error": { "type": "string" }, - "message": { - "type": "string" - }, - "code": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "statusCode", + "code", "error", - "message" + "ownership" ], "additionalProperties": false } @@ -53541,14 +55900,82 @@ "source": { "type": "string", "enum": [ + "default", + "database", "vault", + "config.json", + "env", "missing" ] }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + }, "managedBy": { "type": "string", "enum": [ - "dawarich-provisioning" + "dawarich-provisioning", + "transit.credentials" ] }, "updatedAt": { @@ -54400,7 +56827,8 @@ "code": { "type": "string", "enum": [ - "DAWARICH_CREDENTIAL_MANAGED" + "DAWARICH_CREDENTIAL_MANAGED", + "transit_credentials_managed" ] }, "error": { @@ -54465,6 +56893,89 @@ "additionalProperties": false } ] + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] + }, + "error": { + "type": "string" + }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "code", + "error", + "ownership" + ], + "additionalProperties": false } ] } @@ -55039,7 +57550,8 @@ "code": { "type": "string", "enum": [ - "DAWARICH_CREDENTIAL_MANAGED" + "DAWARICH_CREDENTIAL_MANAGED", + "transit_credentials_managed" ] }, "error": { @@ -55104,6 +57616,89 @@ "additionalProperties": false } ] + }, + { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] + }, + "error": { + "type": "string" + }, + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "code", + "error", + "ownership" + ], + "additionalProperties": false } ] } @@ -55622,51 +58217,138 @@ "schema": { "anyOf": [ { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false }, - "jobId": { - "type": "string" + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false } - }, - "required": [ - "error" - ], - "additionalProperties": false + ] }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "code": { + "type": "string", + "enum": [ + "CONFIGURATION_SOURCE_OWNED" + ] }, "error": { "type": "string" }, - "message": { - "type": "string" - }, - "code": { - "type": "string" + "ownership": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "default", + "database", + "vault", + "config.json", + "env" + ] + }, + "writable": { + "type": "boolean" + }, + "secret": { + "type": "boolean" + }, + "adoption": { + "type": "string", + "enum": [ + "direct", + "host_action_required" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "enum": [ + "host_environment", + "host_file", + "vault_only", + "managed_workflow" + ] + }, + { + "type": "null" + } + ] + }, + "envVar": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" + }, + "managedBy": { + "type": "string", + "enum": [ + "transit.credentials", + "dawarich-provisioning" + ] + } + }, + "required": [ + "source", + "writable", + "secret", + "adoption", + "reason" + ], + "additionalProperties": false } }, "required": [ - "statusCode", + "code", "error", - "message" + "ownership" ], "additionalProperties": false } diff --git a/apps/api/src/integration-config.test.ts b/apps/api/src/integration-config.test.ts index 2cef39a09..309fd34a5 100644 --- a/apps/api/src/integration-config.test.ts +++ b/apps/api/src/integration-config.test.ts @@ -6,7 +6,7 @@ */ import { describe, expect, it } from "vitest"; -import { coerceEnvValue, isEnvValuePresent } from "./integration-config"; +import { coerceEnvValue, isEnvValuePresent, warnInvalidConfig } from "./integration-config"; describe("coerceEnvValue", () => { it("coerces integer and number types to numbers", () => { @@ -57,3 +57,27 @@ describe("isEnvValuePresent", () => { expect(isEnvValuePresent(" ")).toBe(true); }); }); + +describe("configuration diagnostics", () => { + it("never logs a rejected secret enum value or its allowed credential bodies", () => { + const messages: string[] = []; + warnInvalidConfig( + { + id: "synthetic", + version: "1", + domains: [], + configSchema: { + properties: { + TOKEN: { type: "string", "x-openmapx-secret": true, enum: ["allowed-private"] }, + }, + }, + } as Parameters[0], + { TOKEN: "rejected-private" }, + "synthetic", + (message) => messages.push(message), + ); + expect(messages).toHaveLength(1); + expect(messages.join("\n")).not.toContain("rejected-private"); + expect(messages.join("\n")).not.toContain("allowed-private"); + }); +}); diff --git a/apps/api/src/integration-config.ts b/apps/api/src/integration-config.ts index e914d101e..942084ec2 100644 --- a/apps/api/src/integration-config.ts +++ b/apps/api/src/integration-config.ts @@ -289,9 +289,9 @@ export function warnInvalidConfig( warn(`Integration ${id}: missing required config key "${key}"`); } if (def.enum && config[key] !== undefined && !def.enum.includes(config[key])) { - warn( - `Integration ${id}: config "${key}" value "${config[key]}" not in allowed values: ${def.enum.join(", ")}`, - ); + // Values and enum members can both contain credentials. Diagnostics + // identify only the field, including for ordinary provider settings. + warn(`Integration ${id}: config "${key}" is not an allowed value`); } } } diff --git a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts index 32fabb3c4..37981b879 100644 --- a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { createApiClient } from "@openmapx/core"; import { integrationAdminOperations } from "@openmapx/core/api-contracts"; import { licensingFromDeclaration } from "@openmapx/mobility-core/licensing"; @@ -19,6 +20,7 @@ const mockJobRunnerEnqueue = vi.fn().mockResolvedValue("job-main-123"); const mockJobRunnerCancel = vi.fn().mockResolvedValue(true); const mockImmediateImport = vi.fn(); const mockImmediateBinding = vi.fn(); +const mockImmediateCredential = vi.fn(); const expectedBinding = { expectedServiceId: null, expectedRevision: "a".repeat(64) }; const bindingHeaders = () => { serviceActionLimit.reset(); @@ -26,6 +28,7 @@ const bindingHeaders = () => { }; vi.mock("../../services/job-runner.js", () => ({ jobRunner: { + applyImmediateIntegrationCredential: (...args: unknown[]) => mockImmediateCredential(...args), applyImmediateIntegrationBinding: (...args: unknown[]) => mockImmediateBinding(...args), enqueue: (...args: unknown[]) => mockJobRunnerEnqueue(...args), cancel: (...args: unknown[]) => mockJobRunnerCancel(...args), @@ -166,6 +169,30 @@ vi.mock("../../integration-host.js", () => ({ resolveConfigWithSources: (...args: unknown[]) => mockResolveConfigWithSources(...args), })); +vi.mock("../../services/integration-credential-status", async (original) => { + const view = async () => { + const resolved = await mockResolveConfigWithSources(); + return { + revision: "a".repeat(64), + resolved, + ownership: {}, + credentials: [ + { + key: "apiKey", + title: "API Key", + source: resolved.apiKey?.source ?? "missing", + updatedBy: null, + }, + ], + }; + }; + return { + ...(await original()), + inspectIntegrationCredentialView: view, + computeCredentialStatus: async () => (await view()).credentials, + latestIntegrationApplication: async () => null, + }; +}); // Health services const mockGetCachedHealthStatus = vi.fn().mockReturnValue(null); const mockExecuteIntegrationHealthCheck = vi.fn().mockResolvedValue([]); @@ -287,6 +314,14 @@ beforeEach(() => { ? { enabled: true, manifest: { id, provides: ["routing"] } } : undefined, }); + mockImmediateCredential.mockReset().mockResolvedValue({ + operationId: "credential-sdk-fixture", + integrationId: "geocoding-maptiler", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "applied", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", + }); mockImmediateBinding.mockReset().mockResolvedValue({ operationId: "binding-sdk-fixture", integrationId: "geocoding-maptiler", @@ -416,7 +451,7 @@ describe("actual integration producers through generated SDK", () => { sdk.setAdminIntegrationCredential( id, "apiKey", - { value: "synthetic-credential" }, + { value: "synthetic-credential", expectedRevision: "a".repeat(64) }, { "idempotency-key": crypto.randomUUID() }, options, ), @@ -424,8 +459,8 @@ describe("actual integration producers through generated SDK", () => { sdk.deleteAdminIntegrationCredential( id, "apiKey", + { expectedRevision: "a".repeat(64) }, { "idempotency-key": crypto.randomUUID() }, - undefined, options, ), () => sdk.getAdminOverview(options), @@ -664,7 +699,7 @@ describe("actual integration producers through generated SDK", () => { await sdk.setAdminIntegrationCredential( id, "apiKey", - { value: "synthetic" }, + { value: "synthetic", expectedRevision: "a".repeat(64) }, { "idempotency-key": crypto.randomUUID() }, { client }, ), @@ -916,6 +951,39 @@ describe("actual integration producers through generated SDK", () => { await app.close(); } }); + it.each(["set", "delete"] as const)( + "reports %s credential reconciliation as unavailable rather than an internal error", + async (action) => { + const { JobObservationPending } = await import("../../services/admin-job-failure"); + mockImmediateCredential.mockRejectedValue(new JobObservationPending("recovery_required")); + const sdk = await import("@openmapx/core/api-admin-sdk"), + { app, client } = await wire(); + try { + const headers = { "idempotency-key": randomUUID() }, + body = { expectedRevision: "a".repeat(64) }; + const result = + action === "set" + ? await sdk.setAdminIntegrationCredential( + "geocoding-maptiler", + "apiKey", + { ...body, value: "synthetic" }, + headers, + { client }, + ) + : await sdk.deleteAdminIntegrationCredential( + "geocoding-maptiler", + "apiKey", + body, + headers, + { client }, + ); + expect(result.status).toBe(503); + expect(result.data).toMatchObject({ error: expect.stringMatching(/reconciliation/) }); + } finally { + await app.close(); + } + }, + ); it("retains real cookie CSRF and the public rate floor", async () => { const app = Fastify(); const limiter = new RateLimiter({ max: 2, windowMs: 60000 }); diff --git a/apps/api/src/routes/admin-integration-contract.ts b/apps/api/src/routes/admin-integration-contract.ts index 1b302f410..bc13163dc 100644 --- a/apps/api/src/routes/admin-integration-contract.ts +++ b/apps/api/src/routes/admin-integration-contract.ts @@ -1,5 +1,12 @@ import { integrationAdminOperations } from "@openmapx/core/api-contracts"; -import { ConfigurationPrerequisiteFailure } from "../services/admin-job-failure"; +import { + ConfigurationPrerequisiteFailure, + JobObservationPending, +} from "../services/admin-job-failure"; +import { + ConfigurationSourceOwned, + IntegrationConfigurationConflict, +} from "../services/configuration-ownership-view"; import { createAdminRouteRegistrar } from "./admin-contract"; const register = createAdminRouteRegistrar(integrationAdminOperations, { @@ -17,6 +24,21 @@ export const registerIntegrationAdminRoute: typeof register = (app, operation, h try { return await handler(request, reply); } catch (error) { + if ( + error instanceof JobObservationPending && + (operation === "setAdminIntegrationCredential" || + operation === "deleteAdminIntegrationCredential") + ) + return reply.code(503).send({ + error: + "Credential saved or awaiting reconciliation; inspect its operation status before retrying", + }); + if (error instanceof IntegrationConfigurationConflict) + return reply.code(409).send({ code: error.code, error: error.message }); + if (error instanceof ConfigurationSourceOwned) + return reply + .code(409) + .send({ code: error.code, error: error.message, ownership: error.ownership }); if (error instanceof ConfigurationPrerequisiteFailure) return reply.code(409).send({ error: error.message }); return reply.code(500).send({ error: "Integration administration failed" }); diff --git a/apps/api/src/routes/admin-integration-credentials.ts b/apps/api/src/routes/admin-integration-credentials.ts index e15c0224e..6dcbbf418 100644 --- a/apps/api/src/routes/admin-integration-credentials.ts +++ b/apps/api/src/routes/admin-integration-credentials.ts @@ -1,5 +1,5 @@ import type { FastifyInstance } from "fastify"; -import { getAllIntegrations, getIntegration } from "../integration-host"; +import { getAllIntegrations, getIntegration, inspectIntegrationRuntime } from "../integration-host"; import { createDirectAdminOpsKey, DIRECT_OPS_IDEMPOTENCY_HEADER, @@ -8,10 +8,12 @@ import { import { computeCredentialStatus, getIntegrationDisplayName, + inspectIntegrationCredentialView, + latestIntegrationApplication, } from "../services/integration-credential-status"; import { executeIntegrationHealthCheck } from "../services/integration-health"; +import { jobRunner } from "../services/job-runner"; import { isSecretsConfigured, listSecrets } from "../services/secrets"; -import { applyTrustedConfiguration } from "../services/trusted-config-operations"; import { writeAuditLog } from "../utils/audit-log"; import { getAdminSession } from "../utils/require-admin"; import { getSecretFields } from "../utils/validate-config-body"; @@ -47,11 +49,16 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr const integration = getIntegration(request.params.integrationId); if (!integration) return reply.status(404).send({ error: "Integration not found" }); - const credentialStatus = await computeCredentialStatus(integration); + const view = await inspectIntegrationCredentialView(integration); + const credentialStatus = view.credentials; + const application = await latestIntegrationApplication(integration.id); return { integrationId: integration.id, name: getIntegrationDisplayName(integration), credentials: credentialStatus, + configurationRevision: view.revision, + runtime: inspectIntegrationRuntime(integration.id), + application, secretsConfigured: isSecretsConfigured(), }; }); @@ -85,19 +92,17 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr }); } - const applied = await applyTrustedConfiguration({ - kind: "integrationConfig.apply", + const applied = await jobRunner.applyImmediateIntegrationCredential({ integrationId, + key, + action: "set", + value, + expectedRevision: request.body.expectedRevision, actorId: adminSession.user.id, operationKey: createDirectAdminOpsKey( adminSession.user.id, parseDirectOpsIdempotency(request.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), ), - overlay: { - purpose: "integration-secret", - coordinatorMode: "ensure-running", - changes: [{ kind: "integration-secret", integrationId, key, action: "set", value }], - }, }); await writeAuditLog({ @@ -105,17 +110,11 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr targetId: integrationId, targetType: "integration", action: "credential.set", - details: { key }, + details: { key, database: applied.database, runtime: applied.runtime }, request, }); - // Reload so the running integration picks up the new secret. An - // integration captures its resolved config (incl. vault secrets) once at - // `setup(ctx)` load time, so without this a freshly-set key never reaches - // the provider until the next restart. Mirrors the config-PATCH endpoint. - const reloadResult = applied.integrationReload; - if (!reloadResult) throw Error("Captured integration reload unavailable"); - return { ok: true, ...reloadResult }; + return applied; }); // DELETE /admin/credentials/:integrationId/:key @@ -126,19 +125,16 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr const adminSession = getAdminSession(request); - const applied = await applyTrustedConfiguration({ - kind: "integrationConfig.apply", + const applied = await jobRunner.applyImmediateIntegrationCredential({ integrationId, + key, + action: "delete", + expectedRevision: request.body.expectedRevision, actorId: adminSession.user.id, operationKey: createDirectAdminOpsKey( adminSession.user.id, parseDirectOpsIdempotency(request.headers[DIRECT_OPS_IDEMPOTENCY_HEADER]), ), - overlay: { - purpose: "integration-secret", - coordinatorMode: "ensure-running", - changes: [{ kind: "integration-secret", integrationId, key, action: "delete" }], - }, }); await writeAuditLog({ @@ -146,15 +142,11 @@ export async function adminIntegrationCredentialsRoute(app: FastifyInstance): Pr targetId: integrationId, targetType: "integration", action: "credential.delete", - details: { key }, + details: { key, database: applied.database, runtime: applied.runtime }, request, }); - // Reload so the running integration drops the removed secret (see the - // set-secret handler above). - const reloadResult = applied.integrationReload; - if (!reloadResult) throw Error("Captured integration reload unavailable"); - return { ok: true, ...reloadResult }; + return applied; }); // POST /admin/credentials/:integrationId/test — test credentials via health check diff --git a/apps/api/src/routes/admin-integrations.ts b/apps/api/src/routes/admin-integrations.ts index 3da8ffce0..9a55c1689 100644 --- a/apps/api/src/routes/admin-integrations.ts +++ b/apps/api/src/routes/admin-integrations.ts @@ -3,7 +3,7 @@ import { integrationEnvVarName } from "@openmapx/integration-framework"; import type { FastifyInstance } from "fastify"; import { db } from "../db"; import { integrationConfig } from "../db/schema"; -import { getAllIntegrations, getIntegration, resolveConfigWithSources } from "../integration-host"; +import { getAllIntegrations, getIntegration, inspectIntegrationRuntime } from "../integration-host"; import { createDirectAdminOpsKey, DIRECT_OPS_IDEMPOTENCY_HEADER, @@ -18,6 +18,8 @@ import { import { computeCredentialStatus, getIntegrationDisplayName, + inspectIntegrationCredentialView, + latestIntegrationApplication, } from "../services/integration-credential-status"; import { executeAllIntegrationHealthChecks, @@ -146,13 +148,16 @@ export async function adminIntegrationsRoute(app: FastifyInstance): Promise { const dep = getIntegration(depId); @@ -186,6 +191,10 @@ export async function adminIntegrationsRoute(app: FastifyInstance): Promise { // Reduce declared-secret fields to the "***" configured sentinel so a // credential never reaches the Config tab, mirroring the integration // detail route. - const resolvedConfig = maskSecretConfigValues( - await resolveServiceConfigWithSources({ - id: svc.manifest.id, - configSchema: svc.manifest.configSchema, - }), - svc.manifest.configSchema, - ); + const snapshot = await inspectServiceConfiguration(svc.manifest.id); + const resolvedConfig = maskSecretConfigValues(snapshot.resolved, svc.manifest.configSchema); return { schema: svc.manifest.configSchema ?? null, resolvedConfig, envPrefix: serviceConfigEnvPrefix(svc.manifest.id), + ownership: snapshot.ownership, }; }); @@ -452,6 +452,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { return { error: "Service not found" }; } const fields = getSecretFields(svc.manifest.configSchema); + const snapshot = await inspectServiceConfiguration(svc.manifest.id); const vault = await listServiceSecrets(svc.manifest.id); const vaultByKey = new Map(vault.map((v) => [v.key, v])); return { @@ -464,10 +465,14 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { title: f.title, description: f.description, setup: f.setup, - source: v ? ("vault" as const) : ("missing" as const), - managedBy: isProvisioningOwnedDawarichCredential(svc.manifest.id, f.key) - ? DAWARICH_CREDENTIAL_MANAGED_BY - : undefined, + source: + snapshot.resolved[f.key]?.value === undefined || + snapshot.resolved[f.key]?.value === null || + snapshot.resolved[f.key]?.value === "" + ? ("missing" as const) + : snapshot.resolved[f.key]!.source, + ownership: snapshot.ownership[f.key], + managedBy: managedServiceCredential(svc.manifest.id, f.key), updatedAt: v?.updatedAt?.toISOString(), updatedBy: v?.updatedBy ?? null, }; @@ -489,6 +494,11 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { reply.status(400); return { error: "Invalid credential key" }; } + if (isManagedTransitCredential(svc.manifest.id, req.params.key)) + return reply.code(409).send({ + code: TRANSIT_CREDENTIAL_MANAGED_CODE, + error: "Use the pinned transit feed credential workflow.", + }); if (isProvisioningOwnedDawarichCredential(svc.manifest.id, req.params.key)) { reply.status(409); return { @@ -553,6 +563,11 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { if (!svc) return reply.code(404).send({ error: "Service not found" }); if (!isValidSecretKey(key)) return reply.code(400).send({ error: "Invalid credential key" }); + if (isManagedTransitCredential(svc.manifest.id, key)) + return reply.code(409).send({ + code: TRANSIT_CREDENTIAL_MANAGED_CODE, + error: "Use the pinned transit feed credential workflow.", + }); if (isProvisioningOwnedDawarichCredential(svc.manifest.id, key)) return reply.code(409).send({ code: DAWARICH_CREDENTIAL_MANAGED_CODE, @@ -576,6 +591,11 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { reply.status(400); return { error: "Invalid credential key" }; } + if (isManagedTransitCredential(svc.manifest.id, req.params.key)) + return reply.code(409).send({ + code: TRANSIT_CREDENTIAL_MANAGED_CODE, + error: "Use the pinned transit feed credential workflow.", + }); if (isProvisioningOwnedDawarichCredential(svc.manifest.id, req.params.key)) { reply.status(409); return { @@ -638,6 +658,11 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { if (!svc) return reply.code(404).send({ error: "Service not found" }); if (!isValidSecretKey(key)) return reply.code(400).send({ error: "Invalid credential key" }); + if (isManagedTransitCredential(svc.manifest.id, key)) + return reply.code(409).send({ + code: TRANSIT_CREDENTIAL_MANAGED_CODE, + error: "Use the pinned transit feed credential workflow.", + }); if (isProvisioningOwnedDawarichCredential(svc.manifest.id, key)) return reply.code(409).send({ code: DAWARICH_CREDENTIAL_MANAGED_CODE, diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts index 60ec9f65f..c7ef18b1c 100644 --- a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -1125,7 +1125,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( sdk.setAdminIntegrationCredential( "alpha", "TOKEN", - { value: "synthetic-new-secret" }, + { value: "synthetic-new-secret", expectedRevision: "a".repeat(64) }, headers, options, ), @@ -1133,7 +1133,13 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( [ "integration credential delete", (headers) => - sdk.deleteAdminIntegrationCredential("alpha", "TOKEN", headers, undefined, options), + sdk.deleteAdminIntegrationCredential( + "alpha", + "TOKEN", + { expectedRevision: "a".repeat(64) }, + headers, + options, + ), ], [ "integration enable", diff --git a/apps/api/src/services/__tests__/prospective-configuration.test.ts b/apps/api/src/services/__tests__/prospective-configuration.test.ts index ab8a6f70f..ff82bb052 100644 --- a/apps/api/src/services/__tests__/prospective-configuration.test.ts +++ b/apps/api/src/services/__tests__/prospective-configuration.test.ts @@ -34,6 +34,8 @@ vi.mock("../../db/schema", () => ({ integrationConfig: { name: "integration_config" }, serviceSecret: { name: "service_secret", serviceId: "service_id" }, integrationSecret: { name: "integration_secret", integrationId: "integration_id" }, + capabilityBinding: { name: "capability_binding" }, + integrationBindingRevision: { name: "integration_binding_revision" }, })); vi.mock("../../integration-host", () => ({ getAllIntegrations: () => [] })); vi.mock("../service-registry", () => ({ diff --git a/apps/api/src/services/configuration-migration/legacy-integration-import.ts b/apps/api/src/services/configuration-migration/legacy-integration-import.ts index 9025d5cb6..21f6e63fa 100644 --- a/apps/api/src/services/configuration-migration/legacy-integration-import.ts +++ b/apps/api/src/services/configuration-migration/legacy-integration-import.ts @@ -1,5 +1,6 @@ /** Existing ordinary integration batch adapter; its sole issuer is the * adjacent built-in intent module, with no future workflow registration. */ + import type { AdminIntegrationConfigImported, AdminIntegrationImportFailure, @@ -11,6 +12,10 @@ import { withAdministrativeConfigurationWriter, withConfigurationCommitParticipant, } from "../administrative-configuration-writer"; +import { + ConfigurationSourceOwned, + IntegrationConfigurationConflict, +} from "../configuration-ownership-view"; import { completeConfigurationApply, prepareIntegrationImport } from "../integration-config-import"; import type { JobContext } from "../job-runner"; import { @@ -181,7 +186,9 @@ export async function runLegacyIntegrationImport( if ( error instanceof JobObservationPending || error instanceof OpsObservationPending || - error instanceof ConfigurationPrerequisiteFailure + error instanceof ConfigurationPrerequisiteFailure || + error instanceof ConfigurationSourceOwned || + error instanceof IntegrationConfigurationConflict ) throw error; if (progress.phase === "reload") diff --git a/apps/api/src/services/configuration-ownership-view.test.ts b/apps/api/src/services/configuration-ownership-view.test.ts new file mode 100644 index 000000000..79ec19eb3 --- /dev/null +++ b/apps/api/src/services/configuration-ownership-view.test.ts @@ -0,0 +1,132 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + closeAdministrativeConfigurationRead, + openAdministrativeConfigurationRead, + withAdministrativeConfigurationReadScope, +} from "./administrative-configuration-context"; +import { + integrationOwnershipSnapshot, + projectFieldOwnership, +} from "./configuration-ownership-view"; +import { encrypt } from "./secrets"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); + vi.unstubAllEnvs(); +}); +describe("effective configuration ownership metadata", () => { + it.each(["default", "database", "vault", "config.json", "env"] as const)( + "projects %s without values", + (source) => { + for (const secret of [false, true]) { + const result = projectFieldOwnership({ source, secret, envVar: "SYNTHETIC_TOKEN" }); + expect(result.writable).toBe(!["env", "config.json"].includes(source)); + expect(result.secret).toBe(secret); + expect(result.reason).toBe( + source === "env" + ? "host_environment" + : source === "config.json" + ? "host_file" + : secret + ? "vault_only" + : null, + ); + expect(Object.keys(result)).not.toContain("value"); + } + }, + ); + it("reads the actual strict cascade, preserves empty env as unset, and locks a file winner", async () => { + vi.stubEnv("OPENMAPX_SECRETS_KEY", "71".repeat(32)); + const root = mkdtempSync(join(tmpdir(), "openmapx-ownership-")); + roots.push(root); + mkdirSync(join(root, "fixture")); + const directory = join(root, "fixture"), + now = new Date(), + configSchema = { + properties: { + TOKEN: { type: "string", "x-openmapx-secret": true, default: "default-private" }, + REGION: { type: "string", default: "default" }, + }, + }; + const context = openAdministrativeConfigurationRead({ + services: [], + integrations: [ + { + id: "fixture", + manifest: { id: "fixture", version: "1", domains: [], configSchema }, + directory, + config: {}, + enabled: true, + isBuiltIn: true, + strings: {}, + providers: new Map(), + shutdownHandlers: [], + }, + ], + env: { INTEGRATION_FIXTURE_TOKEN: "" }, + serviceRows: [], + serviceSecretRows: [], + integrationRows: [ + { + id: "row", + integrationId: "fixture", + config: { REGION: "saved" }, + createdAt: now, + updatedAt: now, + }, + ], + integrationSecretRows: [ + { + id: "secret", + integrationId: "fixture", + key: "TOKEN", + ...encrypt("vault-private"), + createdAt: now, + updatedAt: now, + updatedBy: null, + }, + ], + }); + try { + await withAdministrativeConfigurationReadScope(context, async () => { + const first = await integrationOwnershipSnapshot("fixture", context); + expect(first.ownership.TOKEN).toMatchObject({ + source: "vault", + writable: true, + reason: "vault_only", + }); + writeFileSync(join(directory, "config.json"), JSON.stringify({ TOKEN: "file-private" })); + const next = openAdministrativeConfigurationRead({ ...context }); + let second: Awaited>; + try { + second = await withAdministrativeConfigurationReadScope(next, () => + integrationOwnershipSnapshot("fixture", next), + ); + } finally { + closeAdministrativeConfigurationRead(next); + } + expect(second.ownership.TOKEN).toMatchObject({ + source: "config.json", + writable: false, + adoption: "host_action_required", + }); + expect(JSON.stringify(second.ownership)).not.toContain("private"); + }); + } finally { + closeAdministrativeConfigurationRead(context); + } + }); + it("keeps managed credentials outside generic mutations", () => { + expect( + projectFieldOwnership({ source: "vault", secret: true, managedBy: "transit.credentials" }), + ).toMatchObject({ + writable: false, + managedBy: "transit.credentials", + reason: "managed_workflow", + }); + }); +}); diff --git a/apps/api/src/services/configuration-ownership-view.ts b/apps/api/src/services/configuration-ownership-view.ts new file mode 100644 index 000000000..957d44abd --- /dev/null +++ b/apps/api/src/services/configuration-ownership-view.ts @@ -0,0 +1,191 @@ +import { randomUUID } from "node:crypto"; +import { + type ConfigurationOwnership, + configurationSourceOwnedSchema, + configurationVersionConflictSchema, +} from "@openmapx/core/api-contracts"; +import type { ConfigurationSourcesInspection } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; +import { integrationEnvVarName } from "@openmapx/integration-framework"; +import { resolveIntegrationConfigWithSourcesStrict } from "../integration-config"; +import { + type AdministrativeConfigurationReadContext, + assertAdministrativeConfigurationRead, +} from "./administrative-configuration-context"; +import { integrationConfigRevision } from "./integration-runtime-evidence"; +import { managedServiceCredential } from "./managed-service-credentials"; +import { resolveServiceConfigWithSourcesStrict } from "./service-config-resolver"; +import { resolveServiceVaultSecretsStrict } from "./service-secrets"; +export class IntegrationConfigurationConflict extends Error { + readonly code = "CONFIGURATION_VERSION_CONFLICT"; + constructor() { + super("Configuration changed. Refresh before applying a new credential intent."); + } +} +export class ConfigurationSourceOwned extends Error { + readonly code = "CONFIGURATION_SOURCE_OWNED"; + constructor(readonly ownership: ConfigurationOwnership) { + super( + "This field is controlled by host configuration. Remove the host override, refresh, then explicitly save or rotate it.", + ); + } +} +export function projectFieldOwnership(input: { + source: ConfigurationOwnership["source"]; + secret: boolean; + envVar?: string; + managedBy?: ConfigurationOwnership["managedBy"]; + readOnly?: boolean; +}): ConfigurationOwnership { + const host = input.source === "env" || input.source === "config.json"; + return { + source: input.source, + secret: input.secret, + writable: !host && !input.managedBy && !input.readOnly, + adoption: host || input.readOnly ? "host_action_required" : "direct", + reason: host + ? input.source === "env" + ? "host_environment" + : "host_file" + : input.managedBy || input.readOnly + ? "managed_workflow" + : input.secret + ? "vault_only" + : null, + ...(input.source === "env" && input.envVar ? { envVar: input.envVar } : {}), + ...(input.managedBy ? { managedBy: input.managedBy } : {}), + }; +} +const definitions = (schema: Record | undefined) => + (schema?.properties ?? schema ?? {}) as Record< + string, + { "x-openmapx-secret"?: boolean; readOnly?: boolean } + >; +export async function integrationOwnershipSnapshot( + integrationId: string, + context: AdministrativeConfigurationReadContext, +) { + assertAdministrativeConfigurationRead(context); + const integration = context.integrations.find((i) => i.id === integrationId); + if (!integration) throw Error("Integration unavailable"); + const resolved = await resolveIntegrationConfigWithSourcesStrict( + integration.manifest, + integration.directory, + context, + ), + fields: ReturnType = { + ...definitions(integration.manifest.configSchema), + enabled: {}, + }; + const ownership = Object.fromEntries( + Object.entries(fields) + .filter(([key]) => !["type", "properties"].includes(key)) + .map(([key, definition]) => [ + key, + projectFieldOwnership({ + source: resolved[key]?.source ?? "default", + secret: definition["x-openmapx-secret"] === true, + readOnly: definition.readOnly, + envVar: integrationEnvVarName(integrationId, key), + }), + ]), + ); + const revision = integrationConfigRevision({ + integrationId, + resolved, + rows: context.integrationRows.filter((row) => row.integrationId === integrationId), + vault: context.integrationSecretRows.filter((row) => row.integrationId === integrationId), + }); + return { ownership, resolved, revision }; +} +export async function inspectIntegrationConfiguration(integrationId: string) { + const { withAdministrativeConfigurationRead } = await import( + "./administrative-configuration-writer-owner" + ); + return withAdministrativeConfigurationRead({}, async (read) => + integrationOwnershipSnapshot(integrationId, read), + ); +} +export async function serviceOwnershipSnapshot( + serviceId: string, + context: AdministrativeConfigurationReadContext, + source: ConfigurationSourcesInspection, +) { + assertAdministrativeConfigurationRead(context); + const service = context.services.find((s) => s.manifest.id === serviceId); + if (!service) throw Error("Service unavailable"); + const resolved = await resolveServiceConfigWithSourcesStrict( + { + id: serviceId, + configSchema: service.manifest.configSchema, + containerEnv: service.manifest.container.environment, + }, + context, + ), + vault = await resolveServiceVaultSecretsStrict(serviceId, context), + properties = definitions(service.manifest.configSchema); + const ownership: Record = {}; + for (const [key, definition] of Object.entries(properties)) { + if (["type", "properties"].includes(key)) continue; + const secret = definition["x-openmapx-secret"] === true, + host = source.services + .find((s) => s.serviceId === serviceId) + ?.fields.find((field) => field.key === key); + if (secret && Object.hasOwn(vault, key) && resolved[key]?.source !== "env") + resolved[key] = { value: vault[key], source: "vault" }; + if (host?.source === "host") resolved[key] = { value: null, source: "env" }; + ownership[key] = projectFieldOwnership({ + source: resolved[key]?.source ?? "default", + secret, + readOnly: definition.readOnly, + envVar: + host?.envVar ?? `${coreServices.serviceConfigEnvPrefix(serviceId)}${key.toUpperCase()}`, + managedBy: managedServiceCredential(serviceId, key), + }); + } + // Even an undeclared retained managed key remains protected. + for (const key of Object.keys(vault)) + if (!ownership[key] && managedServiceCredential(serviceId, key)) + ownership[key] = projectFieldOwnership({ + source: "vault", + secret: true, + managedBy: managedServiceCredential(serviceId, key), + }); + return { ownership, resolved }; +} +export async function inspectServiceConfiguration(serviceId: string) { + const { withAdministrativeConfigurationRead } = await import( + "./administrative-configuration-writer-owner" + ); + const { createApiOpsClient, createDurableOpsKey, executeAndWait } = await import("./ops-client"); + return withAdministrativeConfigurationRead({}, async (read) => { + const source = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.sources.inspect" }, + createDurableOpsKey("configuration.sources.inspect", randomUUID()), + ); + return serviceOwnershipSnapshot(serviceId, read, source); + }); +} + +export async function inspectIntegrationOwnership(id: string) { + return (await inspectIntegrationConfiguration(id)).ownership; +} +export async function inspectServiceOwnership(id: string) { + return (await inspectServiceConfiguration(id)).ownership; +} + +export function configurationRejection(error: unknown) { + return error instanceof ConfigurationSourceOwned + ? { code: error.code, error: error.message, ownership: error.ownership } + : error instanceof IntegrationConfigurationConflict + ? { code: error.code, error: error.message } + : null; +} +export function restoreConfigurationRejection(value: unknown): Error { + const owned = configurationSourceOwnedSchema.safeParse(value); + if (owned.success) return new ConfigurationSourceOwned(owned.data.ownership); + if (configurationVersionConflictSchema.safeParse(value).success) + return new IntegrationConfigurationConflict(); + return Error("Configuration rejection unavailable"); +} diff --git a/apps/api/src/services/integration-application.test.ts b/apps/api/src/services/integration-application.test.ts index c46efaab0..c5292a8f9 100644 --- a/apps/api/src/services/integration-application.test.ts +++ b/apps/api/src/services/integration-application.test.ts @@ -51,6 +51,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "actual binding candidate / API / agent / PostgreSQL / captured provider", () => { let raceAfterPreview = false; + let driftAfterCommit = false; let migrated: Awaited>, host: Awaited>, api: ReturnType, @@ -63,14 +64,37 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( afterAll(async () => { await migrated?.stop(); }); - beforeEach(async () => { + beforeEach(async (context) => { + vi.stubEnv("BETTER_AUTH_SECRET", "binding-owned-synthetic-auth-secret-only"); + vi.stubEnv("BETTER_AUTH_URL", "http://localhost:3001"); vi.stubEnv("OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL", migrated.connectionUri); state.fixture = await createConfigurationPostgresFixture(); raceAfterPreview = false; + driftAfterCommit = false; host = await ownedServiceHost({ - extraServiceIds: ["engine-b"], + extraServiceIds: [ + "engine-b", + ...(context.task.name.includes("managed transit") ? ["data-manager"] : []), + ], capabilities: { motis: ["routing-engine"], "engine-b": ["routing-engine"] }, + schemas: { + motis: { + type: "object", + properties: { + REGION: { type: "string", default: "default" }, + TOKEN: { type: "string", "x-openmapx-secret": true }, + OPENMAPX_MEMORY_MIB: { type: "integer", minimum: 128 }, + }, + }, + }, + resources: { motis: { memory: "${MOTIS_SLOT_MEMORY:-4g}" } }, }); + for (const service of host.loaded) { + const checked = services.validateServiceManifest(service.manifest, { + firstParty: service.isBuiltIn, + }); + expect(checked.valid, JSON.stringify(checked.errors)).toBe(true); + } state.root = host.root; const root = join(host.root, "integrations"), directory = join(root, "binding-test"); @@ -86,11 +110,17 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( quality: "built-in", requires: [{ capability: "routing-engine" }], backend: { routes: true }, + configSchema: { + properties: { + TOKEN: { type: "string", "x-openmapx-secret": true, default: "old-synthetic-token" }, + ENDPOINT: { type: "string", default: "old-endpoint" }, + }, + }, }), ); writeFileSync( join(directory, "index.ts"), - `export function setup(ctx){const captured=ctx.getRequiredService('routing-engine');ctx.registerRoute('GET','/bound',async(_req,reply)=>reply.send({serviceId:captured?.serviceId??null}));}`, + `export function setup(ctx){const captured=ctx.getRequiredService('routing-engine');ctx.registerRoute('GET','/bound',async(_req,reply)=>reply.send({serviceId:captured?.serviceId??null}));const token=ctx.config.TOKEN;ctx.registerRoute('GET','/credential',async(_req,reply)=>reply.send({rotated:token==='new-synthetic-token'}));}`, ); installation = openControlInstallation({ infraDir: host.infraDir, @@ -114,6 +144,21 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await state.fixture .sql`UPDATE capability_binding SET updated_at=now() WHERE integration_id='binding-test'`; } + if ( + driftAfterCommit && + request.method === "GET" && + request.url.startsWith("/v1/operations/") && + typeof payload === "string" + ) { + const response = JSON.parse(payload); + if (response.result?.result?.outcome === "committed") { + driftAfterCommit = false; + writeFileSync( + join(directory, "config.json"), + JSON.stringify({ TOKEN: "host-drift-token" }), + ); + } + } return payload; }); const baseUrl = await agent.listen({ host: "127.0.0.1", port: 0 }), @@ -146,6 +191,19 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await (await import("../routes/admin-capability-bindings")).registerCapabilityBindingRoutes( api, ); + api.addHook("preHandler", async (request: import("fastify").FastifyRequest) => { + if (request.url.startsWith("/api/admin/")) + request.adminSession = await (await import("../utils/require-admin")).requireAdmin( + request, + ); + }); + await api.register( + (await import("../routes/admin-integration-credentials")).adminIntegrationCredentialsRoute, + { prefix: "/api" }, + ); + await api.register((await import("../routes/admin-services")).adminServicesRoute, { + prefix: "/api", + }); await api.ready(); }, 30000); afterEach(async () => { @@ -358,6 +416,319 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ); expect((await set(body, key)).statusCode).toBe(409); }, 90000); + it("returns an applied credential receipt for the actual provider without exposing the value", async () => { + const revision = ( + await ( + await import("./configuration-ownership-view") + ).inspectIntegrationConfiguration("binding-test") + ).revision; + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + const result = await api.inject({ + method: "PUT", + url: "/api/admin/credentials/binding-test/TOKEN", + headers: { ...headers, "idempotency-key": `credential-${randomUUID()}` }, + payload: { value: "new-synthetic-token", expectedRevision: revision }, + }); + expect(result.statusCode, result.body).toBe(200); + expect(result.json()).toMatchObject({ + database: "retained", + runtime: "applied", + runtimeGenerationId: expect.any(String), + }); + expect(result.body).not.toContain("new-synthetic-token"); + expect((await api.inject("/api/integrations/binding-test/credential")).json()).toEqual({ + rotated: true, + }); + }, 90000); + it("recovers credential activation observation without replaying provider setup", async () => { + const before = await ( + await api.inject({ url: "/api/admin/credentials/binding-test", headers }) + ).json(), + key = randomUUID(), + reload = vi.spyOn(await import("../integration-host"), "reloadIntegrations"); + await state.fixture.sql.unsafe( + `CREATE FUNCTION reject_credential_completion() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.result#>>'{integrationReload,state}'='complete' THEN RAISE EXCEPTION 'synthetic completion unavailable'; END IF; RETURN NEW; END $$`, + ); + await state.fixture.sql.unsafe( + `CREATE TRIGGER reject_credential_completion BEFORE UPDATE ON admin_job FOR EACH ROW EXECUTE FUNCTION reject_credential_completion()`, + ); + const interrupted = await credential("PUT", before.configurationRevision, key); + expect(interrupted.statusCode, interrupted.body).toBe(503); + expect((await api.inject("/api/integrations/binding-test/credential")).json()).toEqual({ + rotated: true, + }); + expect(reload).toHaveBeenCalledTimes(1); + const original = await state.fixture + .sql`SELECT id,result FROM admin_job WHERE type='configuration.apply'`; + expect(original).toHaveLength(1); + expect(original[0]!.result.integrationReload.state).toBe("attempted"); + await state.fixture.sql.unsafe(`DROP TRIGGER reject_credential_completion ON admin_job`); + await state.fixture + .sql`UPDATE admin_job SET result=jsonb_set(result,'{_observationLeaseUntil}',to_jsonb('2000-01-01T00:00:00.000Z'::text)) WHERE type='configuration.apply'`; + const recovered = await credential("PUT", before.configurationRevision, key); + expect(recovered.statusCode, recovered.body).toBe(200); + expect(recovered.json().runtime).toBe("applied"); + expect(reload).toHaveBeenCalledTimes(1); + const current = await state.fixture + .sql`SELECT id,status FROM admin_job WHERE type='configuration.apply'`; + expect(current).toEqual([{ id: original[0]!.id, status: "success" }]); + }, 90000); + it("locks real service host fields and resource knobs, then permits explicit adoption after removal", async () => { + vi.stubEnv("SERVICE_MOTIS_REGION", "host-region"); + vi.stubEnv("SERVICE_MOTIS_TOKEN", "host-private"); + writeFileSync(join(host.infraDir, ".env"), "MOTIS_SLOT_MEMORY=8g\n"); + const view = await api.inject({ url: "/api/admin/services/motis/config", headers }); + expect(view.statusCode, view.body).toBe(200); + expect(view.json().ownership.REGION).toMatchObject({ source: "env", writable: false }); + expect(view.json().ownership.OPENMAPX_MEMORY_MIB).toMatchObject({ + source: "env", + writable: false, + envVar: "MOTIS_SLOT_MEMORY", + }); + expect(view.body).not.toContain("host-private"); + for (const config of [{ REGION: "new" }, { OPENMAPX_MEMORY_MIB: 1024 }]) { + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + const refused = await api.inject({ + method: "POST", + url: "/api/admin/services/motis/config", + headers: { ...headers, "idempotency-key": `ownership-${randomUUID()}` }, + payload: { config }, + }); + expect(refused.statusCode, refused.body).toBe(409); + expect(refused.json()).toMatchObject({ + code: "CONFIGURATION_SOURCE_OWNED", + ownership: { writable: false }, + }); + } + const key = `ownership-${randomUUID()}`; + for (const method of ["PUT", "DELETE"] as const) { + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + const refused = await api.inject({ + method, + url: "/api/admin/services/motis/credentials/TOKEN", + headers: { ...headers, "idempotency-key": method + key }, + ...(method === "PUT" ? { payload: { value: "new-private" } } : {}), + }); + expect(refused.statusCode, refused.body).toBe(409); + } + expect(await state.fixture.sql`SELECT * FROM service_config`).toHaveLength(0); + expect(await state.fixture.sql`SELECT * FROM service_secret`).toHaveLength(0); + vi.stubEnv("SERVICE_MOTIS_REGION", ""); + vi.stubEnv("SERVICE_MOTIS_TOKEN", ""); + rmSync(join(host.infraDir, ".env")); + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + const adopted = await api.inject({ + method: "POST", + url: "/api/admin/services/motis/config", + headers: { ...headers, "idempotency-key": `adopt-${randomUUID()}` }, + payload: { config: { REGION: "new" } }, + }); + expect(adopted.statusCode, adopted.body).toBe(200); + expect( + (await state.fixture.sql`SELECT config FROM service_config WHERE service_id='motis'`)[0]! + .config, + ).toEqual({ REGION: "new" }); + }, 90000); + async function credentials() { + const response = await api.inject({ url: "/api/admin/credentials/binding-test", headers }); + expect(response.statusCode, response.body).toBe(200); + return response.json(); + } + async function credential(method: "PUT" | "DELETE", revision: string, key = randomUUID()) { + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + return api.inject({ + method, + url: "/api/admin/credentials/binding-test/TOKEN", + headers: { ...headers, "idempotency-key": key }, + payload: { + expectedRevision: revision, + ...(method === "PUT" ? { value: "new-synthetic-token" } : {}), + }, + }); + } + it.each(["env", "config.json"] as const)( + "refuses %s-shadowed vault changes without writes, then explicitly rotates and deletes", + async (source) => { + const directory = join(host.root, "integrations/binding-test"), + path = join(directory, "config.json"); + if (source === "env") vi.stubEnv("INTEGRATION_BINDING_TEST_TOKEN", "host-private-token"); + else writeFileSync(path, JSON.stringify({ TOKEN: "host-private-token" })); + const before = await credentials(), + reload = vi.spyOn(await import("../integration-host"), "reloadIntegrations"); + expect( + before.credentials.find((field: { key: string }) => field.key === "TOKEN"), + ).toMatchObject({ + source, + ownership: { writable: false, adoption: "host_action_required" }, + }); + expect(JSON.stringify(before)).not.toContain("host-private-token"); + for (const method of ["PUT", "DELETE"] as const) { + const key = randomUUID(), + rejected = await credential(method, before.configurationRevision, key); + expect(rejected.statusCode, rejected.body).toBe(409); + expect(rejected.json()).toMatchObject({ code: "CONFIGURATION_SOURCE_OWNED" }); + expect((await credential(method, before.configurationRevision, key)).statusCode).toBe( + 409, + ); + } + expect(await state.fixture.sql`SELECT * FROM integration_secret`).toHaveLength(0); + expect(await state.fixture.sql`SELECT * FROM integration_config`).toHaveLength(0); + expect(reload).not.toHaveBeenCalled(); + if (source === "env") vi.stubEnv("INTEGRATION_BINDING_TEST_TOKEN", ""); + else rmSync(path); + const refreshed = await credentials(); + const applied = await credential("PUT", refreshed.configurationRevision); + expect(applied.statusCode, applied.body).toBe(200); + expect(applied.json().runtime).toBe("applied"); + expect((await api.inject("/api/integrations/binding-test/credential")).json()).toEqual({ + rotated: true, + }); + const current = await credentials(); + expect(current.credentials[0]).toMatchObject({ + source: "vault", + ownership: { writable: true }, + }); + const removed = await credential("DELETE", current.configurationRevision); + expect(removed.statusCode, removed.body).toBe(200); + expect(removed.json().runtime).toBe("applied"); + expect((await api.inject("/api/integrations/binding-test/credential")).json()).toEqual({ + rotated: false, + }); + expect(await state.fixture.sql`SELECT * FROM integration_secret`).toHaveLength(0); + }, + 90000, + ); + it("does not report applied when a host file changes after committed generation", async () => { + const before = await credentials(); + driftAfterCommit = true; + const result = await credential("PUT", before.configurationRevision); + expect(driftAfterCommit).toBe(false); + expect(result.statusCode, result.body).toBe(200); + expect(result.json()).toMatchObject({ + database: "retained", + runtime: "failed", + reason: "reload_failed", + }); + expect((await api.inject("/api/integrations/binding-test/credential")).json()).toEqual({ + rotated: false, + }); + expect(await state.fixture.sql`SELECT * FROM integration_secret`).toHaveLength(1); + expect(result.body).not.toContain("new-synthetic-token"); + expect(result.body).not.toContain("host-drift-token"); + }, 90000); + it.each(["service-query", "vault", "file"] as const)( + "refuses unavailable strict %s acquisition before admission", + async (failure) => { + const run = vi.spyOn(host.runtime, "stack.render"), + path = join(host.root, "integrations/binding-test/config.json"); + if (failure === "service-query") + await state.fixture.sql`ALTER TABLE service_config RENAME TO unavailable_service_config`; + if (failure === "file") writeFileSync(path, "{malformed"); + if (failure === "vault") { + const encrypted = (await import("./secrets")).encrypt("private-value"); + await state.fixture + .sql`INSERT INTO integration_secret(id,integration_id,key,ciphertext,iv,tag) VALUES (${randomUUID()},'binding-test','TOKEN','invalid',${encrypted.iv},${encrypted.tag})`; + } + try { + expect(await state.fixture.sql`SELECT * FROM capability_binding`).toHaveLength(1); + const rejected = await credential("PUT", "a".repeat(64)); + expect(rejected.statusCode).toBeGreaterThanOrEqual(400); + expect(await state.fixture.sql`SELECT * FROM integration_config`).toHaveLength(0); + expect(await state.fixture.sql`SELECT * FROM integration_secret`).toHaveLength( + failure === "vault" ? 1 : 0, + ); + expect(run).not.toHaveBeenCalled(); + } finally { + if (failure === "service-query") + await state.fixture + .sql`ALTER TABLE unavailable_service_config RENAME TO service_config`; + } + }, + 90000, + ); + it("rejects managed transit writes before malformed body validation, including an undeclared key", async () => { + for (const method of ["PUT", "DELETE"] as const) { + (await import("../utils/rate-limit")).serviceActionLimit.reset(); + const result = await api.inject({ + method, + url: "/api/admin/services/data-manager/credentials/TRANSITOUS_API_KEYS", + headers: { ...headers, "idempotency-key": randomUUID() }, + payload: { unexpected: true }, + }); + expect(result.statusCode, result.body).toBe(409); + expect(result.json()).toMatchObject({ code: "transit_credentials_managed" }); + } + expect(await state.fixture.sql`SELECT * FROM service_secret`).toHaveLength(0); + }); + it("serializes binding and mixed import or refuses the busy writer without a partial candidate", async () => { + const before = await desired(), + { jobRunner } = await import("./job-runner"), + { createDirectAdminOpsKey } = await import("./direct-ops-idempotency"); + const results = await Promise.allSettled([ + jobRunner.applyImmediateIntegrationBinding({ + actorId: "loopback", + operationKey: createDirectAdminOpsKey("loopback", randomUUID()), + input: { + integrationId: "binding-test", + capability: "routing-engine", + expectedServiceId: "motis", + expectedRevision: before.revision, + serviceId: "engine-b", + }, + }), + jobRunner.importImmediateIntegrationConfiguration({ + actorId: "loopback", + operationKey: createDirectAdminOpsKey("loopback", randomUUID()), + body: { + integrations: [{ integrationId: "binding-test", config: { ENDPOINT: "new-endpoint" } }], + }, + }), + ]); + expect( + results.filter((result) => result.status === "fulfilled").length, + ).toBeGreaterThanOrEqual(1); + for (const [index, result] of results.entries()) + if (result.status === "rejected") { + expect(result.reason).toBeInstanceOf( + (await import("./admin-job-failure")).JobObservationPending, + ); + const type = index === 0 ? "integration.binding.apply" : "integration.config.import"; + const rows = await state.fixture.sql`SELECT result FROM admin_job WHERE type=${type}`; + expect(rows.length).toBeLessThanOrEqual(1); + for (const row of rows) expect(row.result._configurationCandidate).toBeUndefined(); + } + if (results[0]!.status === "rejected") { + expect((await desired()).bindings).toEqual([ + { capability: "routing-engine", serviceId: "motis" }, + ]); + const current = await desired(), + applied = await set({ + serviceId: "engine-b", + expectedServiceId: "motis", + expectedRevision: current.revision, + }); + expect(applied.statusCode, applied.body).toBe(200); + expect(applied.json().runtime).toBe("applied"); + } + if (results[1]!.status === "rejected") { + expect(await state.fixture.sql`SELECT * FROM integration_config`).toHaveLength(0); + const imported = await jobRunner.importImmediateIntegrationConfiguration({ + actorId: "loopback", + operationKey: createDirectAdminOpsKey("loopback", randomUUID()), + body: { + integrations: [{ integrationId: "binding-test", config: { ENDPOINT: "new-endpoint" } }], + }, + }); + expect(imported.ok).toBe(true); + } + expect((await desired()).runtime.bindings["routing-engine"]).toBe("engine-b"); + expect( + ( + await state.fixture + .sql`SELECT config FROM integration_config WHERE integration_id='binding-test'` + )[0]!.config, + ).toMatchObject({ ENDPOINT: "new-endpoint" }); + }, 90000); it("rolls back binding, revision and application receipt together on an actual SQL constraint failure", async () => { const before = await desired(); await state.fixture diff --git a/apps/api/src/services/integration-application.ts b/apps/api/src/services/integration-application.ts index 0a4e17d4f..b2333ea9e 100644 --- a/apps/api/src/services/integration-application.ts +++ b/apps/api/src/services/integration-application.ts @@ -14,7 +14,7 @@ import { import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; import { - completeBindingActivation, + completeIntegrationActivation, persistAndApplyProspectiveConfiguration, prepareProspectiveConfiguration, } from "./prospective-configuration"; @@ -55,7 +55,7 @@ export async function runBindingChange( }); if (!completeTrustedConfigurationApply(receipt.application)) throw new JobObservationPending("recovery_required"); - return completeBindingActivation(candidate, receipt.application); + return completeIntegrationActivation(candidate, receipt.application); }); }, ), diff --git a/apps/api/src/services/integration-credential-application.ts b/apps/api/src/services/integration-credential-application.ts new file mode 100644 index 000000000..19ad6f110 --- /dev/null +++ b/apps/api/src/services/integration-credential-application.ts @@ -0,0 +1,44 @@ +/** Fixed current credential producer; all writes remain in the existing candidate. */ +import type { IntegrationApplyReceipt } from "@openmapx/core/api-contracts"; +import { JobObservationPending } from "./admin-job-failure"; +import type { JobContext } from "./job-runner"; +import { applyTrustedConfigurationWithEvidence } from "./trusted-configuration-producer"; +export interface IntegrationCredentialIntent { + integrationId: string; + key: string; + action: "set" | "delete"; + value?: string; + expectedRevision: string; + operationKey: string; + actorId: string; +} +export async function runIntegrationCredentialChange( + ctx: JobContext, + input: IntegrationCredentialIntent, +): Promise { + const common = { + kind: "integration-secret" as const, + integrationId: input.integrationId, + key: input.key, + expectedRevision: input.expectedRevision, + }; + const evidence = await applyTrustedConfigurationWithEvidence({ + kind: "integrationConfig.apply", + integrationId: input.integrationId, + actorId: input.actorId, + operationKey: input.operationKey, + ctx, + signal: ctx.signal, + overlay: { + purpose: "integration-secret", + coordinatorMode: "ensure-running", + changes: [ + input.action === "set" + ? { ...common, action: "set", value: input.value ?? "" } + : { ...common, action: "delete" }, + ], + }, + }); + if (!evidence.integrationApplication) throw new JobObservationPending("recovery_required"); + return evidence.integrationApplication; +} diff --git a/apps/api/src/services/integration-credential-status.test.ts b/apps/api/src/services/integration-credential-status.test.ts index c2364e168..3dca3ea31 100644 --- a/apps/api/src/services/integration-credential-status.test.ts +++ b/apps/api/src/services/integration-credential-status.test.ts @@ -4,23 +4,64 @@ import { join } from "node:path"; import type { IntegrationManifest } from "@openmapx/integration-framework"; import { afterEach, expect, it, vi } from "vitest"; -const database = { apiKey: "synthetic-db" }; -vi.mock("../db", () => ({ - db: { - select: () => ({ - from: () => ({ where: () => ({ limit: async () => [{ config: database }] }) }), - }), +import { + closeAdministrativeConfigurationRead, + openAdministrativeConfigurationRead, + withAdministrativeConfigurationReadScope, +} from "./administrative-configuration-context"; +import { encrypt } from "./secrets"; + +let installed: { id: string; directory: string; manifest: IntegrationManifest }; +vi.mock("./administrative-configuration-writer-owner", () => ({ + withAdministrativeConfigurationRead: async ( + _options: unknown, + run: (context: ReturnType) => Promise, + ) => { + const date = new Date("2026-10-20T00:00:00Z"), + context = openAdministrativeConfigurationRead({ + services: [], + integrations: [ + { + ...installed, + enabled: true, + isBuiltIn: true, + config: {}, + strings: {}, + providers: new Map(), + shutdownHandlers: [], + }, + ], + env: { ...process.env }, + serviceRows: [], + serviceSecretRows: [], + integrationRows: [ + { + id: "row", + integrationId: "fixture", + config: { apiKey: "synthetic-db" }, + createdAt: date, + updatedAt: date, + }, + ], + integrationSecretRows: [ + { + id: "secret", + integrationId: "fixture", + key: "apiKey", + ...encrypt("synthetic-vault"), + updatedAt: date, + createdAt: date, + updatedBy: "fixture-actor", + }, + ], + }); + try { + return await withAdministrativeConfigurationReadScope(context, () => run(context)); + } finally { + closeAdministrativeConfigurationRead(context); + } }, })); -vi.mock("../services/secrets", () => ({ - resolveVaultSecrets: async () => ({ apiKey: "synthetic-vault" }), - listSecrets: async () => [ - { key: "apiKey", updatedAt: new Date("2026-10-20T00:00:00Z"), updatedBy: "fixture-actor" }, - ], -})); -vi.mock("../integration-host", async () => ({ - resolveConfigWithSources: (await import("../integration-config")).resolveConfigWithSources, -})); const { computeCredentialStatus } = await import("./integration-credential-status"); const roots: string[] = []; afterEach(() => { @@ -43,6 +84,8 @@ it("uses the actual env > file > vault cascade and keeps separate vault audit wi }, } as IntegrationManifest, }; + installed = integration; + vi.stubEnv("OPENMAPX_SECRETS_KEY", "71".repeat(32)); vi.stubEnv("INTEGRATION_FIXTURE_APIKEY", "synthetic-env"); writeFileSync(join(directory, "config.json"), JSON.stringify({ apiKey: "synthetic-file" })); expect((await computeCredentialStatus(integration))[0]).toMatchObject({ diff --git a/apps/api/src/services/integration-credential-status.ts b/apps/api/src/services/integration-credential-status.ts index f1e616264..463a2cc3c 100644 --- a/apps/api/src/services/integration-credential-status.ts +++ b/apps/api/src/services/integration-credential-status.ts @@ -1,8 +1,6 @@ import type { AdminIntegrationCredential } from "@openmapx/core/api-contracts"; import type { IntegrationManifest } from "@openmapx/integration-framework"; -import { resolveConfigWithSources } from "../integration-host"; import { getSecretFields } from "../utils/validate-config-body"; -import { listSecrets } from "./secrets"; export function getIntegrationDisplayName(integration: { id: string; @@ -13,32 +11,62 @@ export function getIntegrationDisplayName(integration: { return typeof name === "string" ? name : (integration.manifest.name ?? integration.id); } /** Values never leave this service. Stored-vault audit is independent of the winning layer. */ +export async function inspectIntegrationCredentialView(integration: { + id: string; + directory: string; + manifest: IntegrationManifest; +}) { + const { withAdministrativeConfigurationRead } = await import( + "./administrative-configuration-writer-owner" + ); + const { integrationOwnershipSnapshot } = await import("./configuration-ownership-view"); + return withAdministrativeConfigurationRead({}, async (read) => { + const snapshot = await integrationOwnershipSnapshot(integration.id, read), + vault = read.integrationSecretRows.filter((row) => row.integrationId === integration.id), + byKey = new Map(vault.map((row) => [row.key, row])); + const credentials: AdminIntegrationCredential[] = getSecretFields( + integration.manifest.configSchema, + ).map((field) => { + const stored = byKey.get(field.key), + effective = snapshot.resolved[field.key]; + return { + ...field, + source: + effective && + effective.value !== undefined && + effective.value !== null && + effective.value !== "" + ? effective.source + : "missing", + ownership: snapshot.ownership[field.key], + updatedAt: stored?.updatedAt.toISOString(), + updatedBy: stored?.updatedBy ?? null, + }; + }); + return { ...snapshot, credentials }; + }); +} export async function computeCredentialStatus(integration: { id: string; directory: string; manifest: IntegrationManifest; }): Promise { - const fields = getSecretFields(integration.manifest.configSchema); - if (!fields.length) return []; - const [resolved, vault] = await Promise.all([ - resolveConfigWithSources(integration.manifest, integration.directory), - listSecrets(integration.id), - ]); - const byKey = new Map(vault.map((entry) => [entry.key, entry])); - return fields.map((field) => { - const stored = byKey.get(field.key); - const effective = resolved[field.key]; - return { - ...field, - source: - effective && - effective.value !== undefined && - effective.value !== null && - effective.value !== "" - ? effective.source - : "missing", - updatedAt: stored?.updatedAt?.toISOString(), - updatedBy: stored?.updatedBy ?? null, - }; - }); + return (await inspectIntegrationCredentialView(integration)).credentials; +} + +export async function latestIntegrationApplication(integrationId: string) { + const [{ db }, { adminJob }, { sql, desc }, { integrationApplyReceiptSchema }] = + await Promise.all([ + import("../db"), + import("../db/schema"), + import("drizzle-orm"), + import("@openmapx/core/api-contracts"), + ]); + const [last] = await db + .select({ application: sql`${adminJob.result}->'integrationApplication'` }) + .from(adminJob) + .where(sql`${adminJob.result}->'integrationApplication'->>'integrationId'=${integrationId}`) + .orderBy(desc(adminJob.createdAt), desc(adminJob.id)) + .limit(1); + return last ? integrationApplyReceiptSchema.parse(last.application) : null; } diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 5b4694860..8141cb274 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -34,6 +34,10 @@ import { readJobExecution, renewAdminJobClaim, } from "./admin-job-state"; +import { + configurationRejection, + restoreConfigurationRejection, +} from "./configuration-ownership-view"; import { coordinatorLifecycleOperationKey } from "./coordinator-lifecycle"; import { IntegrationBindingConflict as RetainedBindingConflict } from "./integration-binding-participant"; import { type JobStatusEvent, jobEventBus, TERMINAL_JOB_STATUSES } from "./job-events"; @@ -284,7 +288,11 @@ class AdminJobRunner { errorCode: "conflict", }); if (!(job as { _fresh?: boolean })._fresh) { - if (["success", "failed"].includes(job.status)) return restore(checkpoint); + if (["success", "failed"].includes(job.status)) { + if (checkpoint._configurationRejection) + throw restoreConfigurationRejection(checkpoint._configurationRejection); + return restore(checkpoint); + } if ( !["running", "cancel_pending"].includes(job.status) || !checkpoint._configurationCandidate || @@ -333,6 +341,9 @@ class AdminJobRunner { return result.result; } catch (error) { failure = error; + const rejection = configurationRejection(error); + if (rejection) + await ctx.checkpoint({ ...ctx.checkpointResult, _configurationRejection: rejection }); throw error; } }, @@ -398,8 +409,10 @@ class AdminJobRunner { ) throw new AdminJobFailure("Original configuration did not apply", result); if ( - result.integrationBinding && - integrationApplyReceiptSchema.parse(result.integrationBinding).runtime === "failed" + (result.integrationBinding || result.integrationApplication) && + integrationApplyReceiptSchema.parse( + result.integrationBinding ?? result.integrationApplication, + ).runtime === "failed" ) throw new AdminJobFailure("Binding provider did not activate", result); return result; @@ -555,6 +568,26 @@ class AdminJobRunner { ); } + applyImmediateIntegrationCredential( + options: import("./integration-credential-application").IntegrationCredentialIntent, + ) { + return this.immediate( + { ...options, type: "configuration.apply", intent: { ...options, operationKey: undefined } }, + async (ctx) => { + const { runIntegrationCredentialChange } = await import( + "./integration-credential-application" + ); + const value = await runIntegrationCredentialChange(ctx, options); + return { + value, + result: { integrationApplication: value }, + failed: value.runtime === "failed", + }; + }, + (result) => integrationApplyReceiptSchema.parse(result.integrationApplication), + ); + } + applyImmediateIntegrationBinding(options: { input: import("./integration-binding-participant").IntegrationBindingInput; actorId: string; diff --git a/apps/api/src/services/managed-service-credentials.ts b/apps/api/src/services/managed-service-credentials.ts new file mode 100644 index 000000000..df155903d --- /dev/null +++ b/apps/api/src/services/managed-service-credentials.ts @@ -0,0 +1,12 @@ +import { isProvisioningOwnedDawarichCredential } from "./dawarich/credential-policy"; +export const TRANSIT_CREDENTIAL_MANAGED_BY = "transit.credentials" as const; +export const TRANSIT_CREDENTIAL_MANAGED_CODE = "transit_credentials_managed" as const; +export const isManagedTransitCredential = (serviceId: string, key: string) => + serviceId === "data-manager" && key === "TRANSITOUS_API_KEYS"; +export function managedServiceCredential(serviceId: string, key: string) { + return isManagedTransitCredential(serviceId, key) + ? TRANSIT_CREDENTIAL_MANAGED_BY + : isProvisioningOwnedDawarichCredential(serviceId, key) + ? ("dawarich-provisioning" as const) + : undefined; +} diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 454546aa6..c8a18614c 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -101,6 +101,12 @@ import { completedCoordinatorMaintenanceSchema, } from "./configuration-maintenance-step-records"; import { assertLegacyIntegrationImportOverlay } from "./configuration-migration/legacy-integration-import-intent"; +import { + ConfigurationSourceOwned, + IntegrationConfigurationConflict, + integrationOwnershipSnapshot, + projectFieldOwnership, +} from "./configuration-ownership-view"; import { assertCoordinatorLifecycleIntent, assertRetainedCoordinatorLifecycleProvenance, @@ -112,6 +118,7 @@ import { assertIntegrationBindingOverlay, assertSavedIntegrationBinding, } from "./integration-binding-participant"; +import { capturedBindingState } from "./integration-binding-state"; import { integrationConfigRevision } from "./integration-runtime-evidence"; import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; @@ -183,11 +190,23 @@ const overlaySchema = z.strictObject({ z.strictObject({ kind: z.literal("integration-config"), integrationId: id, ...ordinary }), z.strictObject({ kind: z.literal("service-secret"), serviceId: id, ...secretSet }), z.strictObject({ kind: z.literal("service-secret"), serviceId: id, ...secretDelete }), - z.strictObject({ kind: z.literal("integration-secret"), integrationId: id, ...secretSet }), + z.strictObject({ + kind: z.literal("integration-secret"), + integrationId: id, + ...secretSet, + expectedRevision: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }), z.strictObject({ kind: z.literal("integration-secret"), integrationId: id, ...secretDelete, + expectedRevision: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), }), z.strictObject({ kind: z.literal("integration-binding"), @@ -1109,6 +1128,9 @@ export async function retireSettledProspectiveConfiguration( : {}), ...(original.integrationReload ? { integrationReload: original.integrationReload } : {}), ...(original.integrationBinding ? { integrationBinding: original.integrationBinding } : {}), + ...(original.integrationApplication + ? { integrationApplication: original.integrationApplication } + : {}), ...(provenance ? { integrationImport: { @@ -1348,7 +1370,42 @@ export async function withProspectiveConfigurationRead( /** The finite reload continuation. Recovery consumes the authenticated resolved * values, with exact registry metadata revalidation; it never reconstructs the * candidate from current desired rows or captures unrelated process secrets. */ +async function assertCurrentIntegrationReloadSources(candidate: ValidatedConfigurationCandidate) { + const state = validatedConfigurationCandidateState(candidate), + retained = state.record.privateIntegrationReload; + if (!retained?.bindings) throw new JobObservationPending("recovery_required"); + const expectedBindings = retained.bindings; + await withAdministrativeConfigurationRead({}, async (read) => { + if ( + canonicalJson(retained.services) !== canonicalJson(read.services) || + canonicalJson(retained.integrations) !== + canonicalJson( + read.integrations.map((i) => ({ + id: i.id, + directory: i.directory, + manifest: i.manifest, + isBuiltIn: i.isBuiltIn, + })), + ) + ) + throw Error("Integration source changed"); + for (const integration of read.integrations) { + const resolved = await resolveIntegrationConfigWithSourcesStrict( + integration.manifest, + integration.directory, + read, + ); + if ( + canonicalJson(resolved) !== canonicalJson(retained.values[integration.id]) || + canonicalJson(capturedBindingState(read, integration.id).bindings) !== + canonicalJson(expectedBindings[integration.id] ?? {}) + ) + throw Error("Integration source changed"); + } + }); +} export async function reloadProspectiveIntegrations(candidate: ValidatedConfigurationCandidate) { + await assertCurrentIntegrationReloadSources(candidate); const state = validatedConfigurationCandidateState(candidate); assertCommittedConfigurationCandidate(candidate); const { reloadIntegrations } = await import("../integration-host"); @@ -2569,9 +2626,20 @@ export async function resumeRetainedConfigurationCandidate(): Promise change.kind === "integration-secret" && change.expectedRevision, + )) + ) { + const applied = await completeIntegrationActivation(candidate, receipt.application); + return { + configuration: receipt.application, + [state.overlay.purpose === "integration-binding" + ? "integrationBinding" + : "integrationApplication"]: applied, + }; } // An interrupted activation is not permission to invoke setup again. Only // an unattempted reload may begin; a durable completion can be copied. @@ -2849,6 +2917,13 @@ async function mergeOverlay( integration!.directory, context, ); + if ( + change.kind === "integration-secret" && + change.expectedRevision && + (await integrationOwnershipSnapshot(change.integrationId, context)).revision !== + change.expectedRevision + ) + throw new IntegrationConfigurationConflict(); const changedKeys = "set" in change ? [...Object.keys(change.set), ...change.unset] : [change.key]; for (const key of changedKeys) { @@ -2868,7 +2943,13 @@ async function mergeOverlay( .find((item) => item.serviceId === service.manifest.id) ?.fields.find((field) => field.key === key); if (host?.source === "host" || ["env", "config.json"].includes(resolved[key]?.source ?? "")) - throw Error("Configuration field is host-owned"); + throw new ConfigurationSourceOwned( + projectFieldOwnership({ + source: host?.source === "host" ? "env" : resolved[key]!.source, + secret: definition["x-openmapx-secret"] === true, + envVar: host?.envVar, + }), + ); } if (change.kind === "service-config") { let row = rows.serviceRows.find((row) => row.serviceId === change.serviceId); @@ -5497,19 +5578,25 @@ const configurationMaintenanceProtocol = Object.freeze({ }); /** Observation-only after an interrupted attempt; never replay unknown setup. */ -export async function completeBindingActivation( +export async function completeIntegrationActivation( candidate: ValidatedConfigurationCandidate, hostApply: IntegrationApplyReceipt["hostApply"], ): Promise { const state = validatedConfigurationCandidateState(candidate), { job } = administrativeConfigurationState(state.writer), change = state.overlay.changes[0]; + const binding = state.overlay.purpose === "integration-binding"; if ( - state.overlay.purpose !== "integration-binding" || state.overlay.changes.length !== 1 || - change?.kind !== "integration-binding" + !change || + !( + (binding && change.kind === "integration-binding") || + (state.overlay.purpose === "integration-secret" && + change.kind === "integration-secret" && + change.expectedRevision) + ) ) - throw Error("Original binding candidate required"); + throw Error("Original integration candidate required"); const retained = state.record.privateIntegrationReload, retainedManifest = retained?.integrations.find((i) => i.id === change.integrationId)?.manifest, values = retained?.values[change.integrationId]; @@ -5546,10 +5633,46 @@ export async function completeBindingActivation( ? live : null; }; - let receipt = await withConfigurationWriteTransaction( - state.writer, - async (tx) => (await assertSavedIntegrationBinding(candidate, state.overlay, tx)).result, - ); + let receipt: IntegrationApplyReceipt = binding + ? await withConfigurationWriteTransaction( + state.writer, + async (tx) => (await assertSavedIntegrationBinding(candidate, state.overlay, tx)).result, + ) + : integrationApplyReceiptSchema.parse( + job.checkpointResult?.integrationApplication ?? { + operationId: candidate.operationKey, + integrationId: change.integrationId, + desiredRevision: await withAdministrativeConfigurationRead( + {}, + async (read) => + (await integrationOwnershipSnapshot(change.integrationId, read)).revision, + ), + database: "retained", + runtime: "pending", + runtimeGenerationId: null, + }, + ); + if ( + receipt.operationId !== candidate.operationKey || + receipt.integrationId !== change.integrationId + ) + throw new JobObservationPending("recovery_required"); + async function saveReceipt(next: IntegrationApplyReceipt) { + if (binding) + await withConfigurationWriteTransaction(state.writer, async (tx) => { + await assertSavedIntegrationBinding(candidate, state.overlay, tx); + await tx + .update(integrationApplicationReceipt) + .set({ result: next }) + .where( + and( + eq(integrationApplicationReceipt.operationId, candidate.operationKey), + eq(integrationApplicationReceipt.candidateId, candidate.candidateId), + ), + ); + }); + else await job.checkpoint({ ...job.checkpointResult, integrationApplication: next }); + } const original = job.checkpointResult?.integrationReload as Record | undefined; if (original?.candidateId && original.candidateId !== candidate.candidateId) throw new JobObservationPending("recovery_required"); @@ -5557,6 +5680,7 @@ export async function completeBindingActivation( const before = inspectIntegrationRuntime(change.integrationId); let failed = false; if (!original) { + if (!binding) await saveReceipt(receipt); await job.checkpoint({ ...job.checkpointResult, integrationReload: { @@ -5571,9 +5695,17 @@ export async function completeBindingActivation( failed = true; } } - const live = matches(), + let sourceCurrent = true; + try { + await assertCurrentIntegrationReloadSources(candidate); + } catch { + sourceCurrent = false; + } + const live = sourceCurrent ? matches() : null, disabled = - config.enabled === false && getIntegration(change.integrationId)?.enabled === false; + sourceCurrent && + config.enabled === false && + getIntegration(change.integrationId)?.enabled === false; if (live) receipt = { ...receipt, @@ -5600,43 +5732,19 @@ export async function completeBindingActivation( reason: "reload_failed", }; else { - await withConfigurationWriteTransaction(state.writer, async (tx) => { - await assertSavedIntegrationBinding(candidate, state.overlay, tx); - await tx - .update(integrationApplicationReceipt) - .set({ - result: { - ...receipt, - runtime: "unknown", - reason: "reconciliation_required", - hostApply, - }, - }) - .where( - and( - eq(integrationApplicationReceipt.operationId, candidate.operationKey), - eq(integrationApplicationReceipt.candidateId, candidate.candidateId), - ), - ); + await saveReceipt({ + ...receipt, + runtime: "unknown", + reason: "reconciliation_required", + hostApply, }); throw new JobObservationPending("recovery_required"); } receipt = integrationApplyReceiptSchema.parse({ ...receipt, hostApply }); - await withConfigurationWriteTransaction(state.writer, async (tx) => { - await assertSavedIntegrationBinding(candidate, state.overlay, tx); - await tx - .update(integrationApplicationReceipt) - .set({ result: receipt }) - .where( - and( - eq(integrationApplicationReceipt.operationId, candidate.operationKey), - eq(integrationApplicationReceipt.candidateId, candidate.candidateId), - ), - ); - }); + await saveReceipt(receipt); await job.checkpoint({ ...job.checkpointResult, - integrationBinding: receipt, + [binding ? "integrationBinding" : "integrationApplication"]: receipt, _configurationCandidate: { ...(job.checkpointResult?._configurationCandidate as Record), phase: "settled", diff --git a/apps/api/src/services/trusted-configuration-producer.ts b/apps/api/src/services/trusted-configuration-producer.ts index 419a64b83..987772c64 100644 --- a/apps/api/src/services/trusted-configuration-producer.ts +++ b/apps/api/src/services/trusted-configuration-producer.ts @@ -1,3 +1,4 @@ +import type { IntegrationApplyReceipt } from "@openmapx/core/api-contracts"; import { type CoordinatorMode, canonicalJson, @@ -75,6 +76,7 @@ export async function applyTrustedConfiguration( enabledServiceIds: string[]; jobId?: string; integrationReload?: CapturedIntegrationReloadResult; + integrationApplication?: IntegrationApplyReceipt; }> { return completeTrustedConfigurationApplication( options, @@ -91,6 +93,7 @@ export async function applyTrustedConfigurationWithEvidence( outcome: TrustedConfigurationApplyOutcome; jobId?: string; integrationReload?: CapturedIntegrationReloadResult; + integrationApplication?: IntegrationApplyReceipt; }> { if (options.ctx && hasRetainedAdminJobOperation(options.ctx, options.operationKey)) { const outcome = configurationApplyResultSchema.parse( @@ -199,6 +202,20 @@ export async function applyTrustedConfigurationWithEvidence( ); await completeProspectiveServiceCredential(candidate); } + if ( + candidateState.overlay.purpose === "integration-secret" && + candidateState.overlay.changes.some( + (change) => change.kind === "integration-secret" && change.expectedRevision, + ) && + completeTrustedConfigurationApply(receipt.application) + ) { + const { completeIntegrationActivation } = await import("./prospective-configuration"); + const integrationApplication = await completeIntegrationActivation( + candidate, + receipt.application, + ); + return { payload, previousApplied, outcome: receipt.application, integrationApplication }; + } if ( candidateState.record.apiFollowUp === "integration-reload" && completeTrustedConfigurationApply(receipt.application) diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index ec4c9a367..bc0d48005 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -60,7 +60,7 @@ export async function ownedServiceHost(options: { capabilities?:Record )} @@ -262,6 +263,7 @@ function IntegrationPanel({ data }: { data: IntegrationDetail }) { @@ -274,10 +276,12 @@ function IntegrationPanel({ data }: { data: IntegrationDetail }) { function CredentialsTable({ integrationId, secretsConfigured, + configurationRevision, credentials, }: { integrationId: string; secretsConfigured: boolean; + configurationRevision?: string; credentials: CredentialStatus[]; }) { const admin = useIntegrationAdmin(); @@ -290,9 +294,19 @@ function CredentialsTable({ retry: false, mutationFn: async (key: string) => { try { - await admin.deleteCredential(integrationId, key); + const applied = await admin.deleteCredential( + integrationId, + key, + configurationRevision ?? "", + ); void admin.refresh(); - if (owner.active) showToast("Credential deleted"); + if (owner.active) + showToast( + applied.runtime === "applied" + ? "Credential deleted and active" + : "Credential removal saved; provider activation is pending or failed", + applied.runtime === "applied" ? "success" : "info", + ); } catch (err) { if (owner.active) showToast(err instanceof Error ? err.message : "Delete failed", "error"); throw err; @@ -388,7 +402,12 @@ function CredentialsTable({ size="small" variant="outlined" onClick={() => setDialogField(cred)} - disabled={!secretsConfigured} + disabled={ + !secretsConfigured || + !configurationRevision || + cred.ownership?.writable === false || + ["env", "config.json"].includes(cred.source) + } > {hasVaultCredential(cred) ? "Rotate" : "Set"} @@ -398,7 +417,12 @@ function CredentialsTable({ size="small" color="error" onClick={() => deleteMutation.mutate(cred.key)} - disabled={deleteMutation.isPending} + disabled={ + deleteMutation.isPending || + !configurationRevision || + cred.ownership?.writable === false || + ["env", "config.json"].includes(cred.source) + } > @@ -417,6 +441,7 @@ function CredentialsTable({ onClose={() => setDialogField(null)} integrationId={integrationId} credentialKey={dialogField.key} + expectedRevision={configurationRevision} title={dialogField.title} description={dialogField.description} setup={dialogField.setup} diff --git a/apps/web/src/components/admin/integrations/ConfigSchemaForm.tsx b/apps/web/src/components/admin/integrations/ConfigSchemaForm.tsx index 3141e505f..ebbd0804b 100644 --- a/apps/web/src/components/admin/integrations/ConfigSchemaForm.tsx +++ b/apps/web/src/components/admin/integrations/ConfigSchemaForm.tsx @@ -1,6 +1,9 @@ "use client"; -import type { AdminIntegrationResolvedConfig } from "@openmapx/core/api-contracts"; +import type { + AdminIntegrationResolvedConfig, + ConfigurationOwnership, +} from "@openmapx/core/api-contracts"; import { useMutation } from "@tanstack/react-query"; import { useIntegrationAdmin } from "@/lib/admin/integrationAdmin"; import { useAdminMutationOwner } from "@/lib/admin/useAdminMutationOwner"; @@ -14,6 +17,7 @@ interface ConfigSchemaFormProps { schema: Record | undefined; /** Resolved config values with source annotations. */ resolvedConfig: AdminIntegrationResolvedConfig; + ownership?: Record; /** Called after a successful save + reload. */ onSaved?: () => void; } @@ -29,6 +33,7 @@ export function ConfigSchemaForm({ integrationId, schema, resolvedConfig, + ownership, onSaved, }: ConfigSchemaFormProps) { const admin = useIntegrationAdmin(); @@ -49,6 +54,7 @@ export function ConfigSchemaForm({ key={JSON.stringify([admin.apiUrl, integrationId])} schema={schema} resolvedConfig={resolvedConfig} + ownership={ownership} excludeKeys={excludedKeys} onSave={async (diff) => { await saveMutation.mutateAsync(diff); diff --git a/apps/web/src/components/admin/integrations/IntegrationDetail.tsx b/apps/web/src/components/admin/integrations/IntegrationDetail.tsx index d6b74f928..7e55d69ea 100644 --- a/apps/web/src/components/admin/integrations/IntegrationDetail.tsx +++ b/apps/web/src/components/admin/integrations/IntegrationDetail.tsx @@ -431,6 +431,7 @@ function ConfigTab({ integrationId={integrationId} schema={data.manifest.configSchema} resolvedConfig={data.resolvedConfig} + ownership={data.ownership} /> @@ -469,9 +470,19 @@ function CredentialsTab({ retry: false, mutationFn: async (key: string) => { try { - await admin.deleteCredential(integrationId, key); + const result = await admin.deleteCredential( + integrationId, + key, + data.configurationRevision ?? "", + ); void admin.refresh(); - if (owner.active) showToast(`Credential "${key}" deleted`); + if (owner.active) + showToast( + result.runtime === "applied" + ? `Credential "${key}" deleted and active` + : "Credential removal saved; provider activation is pending or failed", + result.runtime === "applied" ? "success" : "info", + ); } catch (err) { if (owner.active) showToast(`Failed to delete "${key}"`, "error"); throw err; @@ -479,6 +490,13 @@ function CredentialsTab({ }, }); + const application = + data.application && + data.application.desiredRevision === data.configurationRevision && + (data.application.runtime === "unknown" || + data.application.runtimeGenerationId === (data.runtime?.generationId ?? null)) + ? data.application + : null; const secretCredentials = data.credentialStatus; const { paged: pagedSecrets, paginationProps: secretsPagination } = useClientPagination(secretCredentials); @@ -497,6 +515,23 @@ function CredentialsTab({ gap: 2, }} > + {application && ( + + {application.runtime === "applied" + ? "The saved credential change is active." + : application.runtime === "failed" + ? "The credential change is saved, but provider reload failed. The previous provider remains active." + : "The credential change is saved or awaiting reconciliation. Review the operation before retrying."} + + )} {!data.secretsConfigured && secretCredentials.length > 0 && ( Vault not configured — secrets cannot be stored. Set OPENMAPX_SECRETS_KEY{" "} @@ -582,6 +617,14 @@ function CredentialsTab({ + {(cred.ownership?.writable === false || + ["env", "config.json"].includes(cred.source)) && ( + + Host controlled + {cred.ownership?.envVar ? ` (${cred.ownership.envVar})` : ""}. Remove + the override on the host, refresh, then rotate. + + )} {cred.updatedAt ? ( @@ -618,7 +661,12 @@ function CredentialsTab({ size="small" variant="outlined" onClick={() => setDialogField(cred)} - disabled={!data.secretsConfigured} + disabled={ + !data.secretsConfigured || + !data.configurationRevision || + cred.ownership?.writable === false || + ["env", "config.json"].includes(cred.source) + } > {hasVaultCredential(cred) ? "Rotate" : "Set"} @@ -628,7 +676,12 @@ function CredentialsTab({ size="small" color="error" onClick={() => deleteMutation.mutate(cred.key)} - disabled={deleteMutation.isPending} + disabled={ + deleteMutation.isPending || + !data.configurationRevision || + cred.ownership?.writable === false || + ["env", "config.json"].includes(cred.source) + } > @@ -651,6 +704,7 @@ function CredentialsTab({ onClose={() => setDialogField(null)} integrationId={integrationId} credentialKey={dialogField.key} + expectedRevision={data.configurationRevision} title={dialogField.title} description={dialogField.description} setup={dialogField.setup} diff --git a/apps/web/src/components/admin/integrations/SetCredentialDialog.tsx b/apps/web/src/components/admin/integrations/SetCredentialDialog.tsx index 38c04264b..61c977955 100644 --- a/apps/web/src/components/admin/integrations/SetCredentialDialog.tsx +++ b/apps/web/src/components/admin/integrations/SetCredentialDialog.tsx @@ -2,6 +2,7 @@ import VisibilityIcon from "@mui/icons-material/Visibility"; import VisibilityOffIcon from "@mui/icons-material/VisibilityOff"; +import Alert from "@mui/material/Alert"; import Button from "@mui/material/Button"; import CircularProgress from "@mui/material/CircularProgress"; import Dialog from "@mui/material/Dialog"; @@ -13,9 +14,10 @@ import InputAdornment from "@mui/material/InputAdornment"; import Stack from "@mui/material/Stack"; import TextField from "@mui/material/TextField"; import Typography from "@mui/material/Typography"; +import type { IntegrationApplyReceipt } from "@openmapx/core/api-contracts"; import type { CredentialSetup } from "@openmapx/integration-framework"; import { useMutation } from "@tanstack/react-query"; -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { useIntegrationAdmin } from "@/lib/admin/integrationAdmin"; import { useAdminMutationOwner } from "@/lib/admin/useAdminMutationOwner"; import { CredentialSetupGuide } from "./CredentialSetupGuide"; @@ -28,6 +30,7 @@ interface SetCredentialDialogProps { title: string; description?: string; setup?: CredentialSetup; + expectedRevision?: string; } export function SetCredentialDialog({ @@ -38,6 +41,7 @@ export function SetCredentialDialog({ title, description, setup, + expectedRevision, }: SetCredentialDialogProps) { const admin = useIntegrationAdmin(); const owner = useAdminMutationOwner( @@ -45,22 +49,49 @@ export function SetCredentialDialog({ JSON.stringify([integrationId, credentialKey, open]), ); + const draft = useRef<{ scope: string; revision?: string; value: string; key: string } | null>( + null, + ); + const scope = JSON.stringify([admin.apiUrl, integrationId, credentialKey, open]); + if (draft.current?.scope !== scope) + draft.current = { scope, revision: expectedRevision, value: "", key: crypto.randomUUID() }; + const [receipt, setReceipt] = useState(null); const [value, setValue] = useState(""); const [showValue, setShowValue] = useState(false); const [error, setError] = useState(null); + useEffect(() => { + if (draft.current?.scope !== scope) return; + setReceipt(null); + setValue(""); + setError(null); + setShowValue(false); + }, [scope]); + const mutation = useMutation({ mutationKey: ["integration-credential", admin.apiUrl, integrationId, credentialKey, open], retry: false, mutationFn: async () => { try { - await admin.setCredential(integrationId, credentialKey, value); + if (!draft.current?.revision) throw Error("Refresh credential metadata before saving."); + const revision = draft.current.revision; + if (draft.current.value !== value) + draft.current = { ...draft.current, value, key: crypto.randomUUID() }; + const applied = await admin.setCredential( + integrationId, + credentialKey, + value, + revision, + draft.current.key, + ); void admin.refresh(); if (!owner.active) return; setValue(""); + if (draft.current?.scope === scope) draft.current.value = ""; setShowValue(false); setError(null); - onClose(); + if (applied.runtime === "applied") onClose(); + else setReceipt(applied); } catch (err) { if (owner.active) setError(err instanceof Error ? err.message : "Failed to save credential"); @@ -96,6 +127,13 @@ export function SetCredentialDialog({ pt: 1, }} > + {receipt && ( + + {receipt.runtime === "failed" + ? "Credential saved. Provider reload failed; the previous provider remains active." + : "Credential saved. Activation is pending or requires reconciliation. Review the operation before retrying."} + + )} {description && ( setValue(e.target.value)} - disabled={mutation.isPending} + disabled={mutation.isPending || !!receipt} error={!!error} helperText={error} autoComplete="new-password" @@ -141,12 +179,12 @@ export function SetCredentialDialog({ + )} {cred.updatedAt && ( Updated {new Date(cred.updatedAt).toLocaleString()} @@ -177,12 +190,16 @@ function ServiceCredentialsView({ serviceId }: { serviceId: string }) { )} - {cred.managedBy !== "dawarich-provisioning" && ( + {!cred.managedBy && ( diff --git a/apps/web/src/components/admin/services/ServiceDetail.tsx b/apps/web/src/components/admin/services/ServiceDetail.tsx index ce0947ed8..9635fb76f 100644 --- a/apps/web/src/components/admin/services/ServiceDetail.tsx +++ b/apps/web/src/components/admin/services/ServiceDetail.tsx @@ -426,6 +426,7 @@ function ConfigTab({ data }: { data: ServiceDetailData }) { schema={manifest.configSchema} resolvedConfig={configQuery.data?.resolvedConfig ?? {}} envPrefix={configQuery.data?.envPrefix} + ownership={configQuery.data?.ownership} onSave={async (values) => { await saveConfig.mutateAsync(values); }} diff --git a/apps/web/src/components/admin/shared/SchemaConfigForm.tsx b/apps/web/src/components/admin/shared/SchemaConfigForm.tsx index 2c90f2f7c..dd33079eb 100644 --- a/apps/web/src/components/admin/shared/SchemaConfigForm.tsx +++ b/apps/web/src/components/admin/shared/SchemaConfigForm.tsx @@ -15,6 +15,7 @@ import Switch from "@mui/material/Switch"; import TextField from "@mui/material/TextField"; import Tooltip from "@mui/material/Tooltip"; import Typography from "@mui/material/Typography"; +import type { ConfigurationOwnership } from "@openmapx/core/api-contracts"; import { type CredentialSetup, readCredentialSetup } from "@openmapx/integration-framework"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { CredentialSetupGuide } from "../integrations/CredentialSetupGuide"; @@ -148,6 +149,7 @@ export interface SchemaConfigFormProps { excludeKeys?: readonly string[]; /** Optional env-var-prefix hint banner (services use `SERVICE__`). */ envPrefix?: string; + ownership?: Record; /** Persist the changed-keys diff. Throw to surface an inline error. */ onSave: (diff: Record) => Promise; /** Optional second action that also applies the change. */ @@ -167,6 +169,7 @@ export function SchemaConfigForm({ resolvedConfig, excludeKeys, envPrefix, + ownership, onSave, onSaveAndApply, applyLabel = "Save & apply", @@ -182,12 +185,16 @@ export function SchemaConfigForm({ const initial: Record = {}; for (const field of fields) { const entry = resolvedConfig[field.key]; - if (entry?.source !== "env") { + if ( + entry?.source !== "env" && + entry?.source !== "config.json" && + ownership?.[field.key]?.writable !== false + ) { initial[field.key] = toFormValue(field, entry?.value ?? field.default ?? ""); } } return initial; - }, [fields, resolvedConfig]); + }, [fields, resolvedConfig, ownership]); const [values, setValues] = useState>(computeInitialValues); const [pending, setPending] = useState<"save" | "apply" | null>(null); @@ -215,19 +222,26 @@ export function SchemaConfigForm({ const orig: Record = {}; for (const field of fields) { const entry = resolvedConfig[field.key]; - if (entry?.source !== "env") { + if ( + entry?.source !== "env" && + entry?.source !== "config.json" && + ownership?.[field.key]?.writable !== false + ) { orig[field.key] = toFormValue(field, entry?.value ?? field.default ?? ""); } } return orig; - }, [fields, resolvedConfig]); + }, [fields, resolvedConfig, ownership]); const changedKeys = useMemo( () => Object.keys(values).filter( - (k) => comparable(values[k]) !== comparable(originalValues[k] ?? ""), + (k) => + !["env", "config.json"].includes(resolvedConfig[k]?.source ?? "") && + ownership?.[k]?.writable !== false && + comparable(values[k]) !== comparable(originalValues[k] ?? ""), ), - [values, originalValues], + [values, originalValues, resolvedConfig, ownership], ); const saving = pending !== null; @@ -290,7 +304,10 @@ export function SchemaConfigForm({ {fields.map((field) => { const entry = resolvedConfig[field.key]; const source: ConfigFieldSource = entry?.source ?? "default"; - const isEnvOverridden = source === "env"; + const isEnvOverridden = + source === "env" || + source === "config.json" || + ownership?.[field.key]?.writable === false; const displayValue = isEnvOverridden ? toFormValue(field, entry?.value) : values[field.key]; return ( @@ -310,7 +327,13 @@ export function SchemaConfigForm({ {isEnvOverridden && ( )} + {isEnvOverridden && ( + + {source === "config.json" + ? "Controlled by the host config file. Remove the host override, refresh, then save." + : `Controlled by the host environment${ownership?.[field.key]?.envVar ? ` (${ownership[field.key].envVar})` : ""}. Remove the override, refresh, then save.`} + + )} {field.setup && } {field.type === "boolean" ? ( { + const save = vi.fn(); + render( + { + save(diff); + }} + />, + ); + const fields = screen.getAllByRole("textbox"); + expect(fields[0]).toBeDisabled(); + await userEvent.clear(fields[1]!); + await userEvent.type(fields[1]!, "new"); + await userEvent.click(screen.getByRole("button", { name: /^Save/ })); + expect(save).toHaveBeenCalledWith({ region: "new" }); + expect(await screen.findByText(/Controlled by the host config file/i)).toBeInTheDocument(); +}); diff --git a/apps/web/src/lib/admin/integrationAdmin.test.tsx b/apps/web/src/lib/admin/integrationAdmin.test.tsx index 90cfcb84d..72d5f6504 100644 --- a/apps/web/src/lib/admin/integrationAdmin.test.tsx +++ b/apps/web/src/lib/admin/integrationAdmin.test.tsx @@ -67,8 +67,9 @@ describe("integration query boundaries and reload observation", () => { await act(async () => { const run = { patch: () => result.current.patchConfig("demo", { endpoint: "synthetic" }), - set: () => result.current.setCredential("demo", "TOKEN", "synthetic-private"), - delete: () => result.current.deleteCredential("demo", "TOKEN"), + set: () => + result.current.setCredential("demo", "TOKEN", "synthetic-private", "a".repeat(64)), + delete: () => result.current.deleteCredential("demo", "TOKEN", "a".repeat(64)), enable: () => result.current.toggle("demo", true), disable: () => result.current.toggle("demo", false), import: () => @@ -88,7 +89,7 @@ describe("integration query boundaries and reload observation", () => { kind === "patch" ? { endpoint: "synthetic" } : kind === "set" - ? { value: "synthetic-private" } + ? { value: "synthetic-private", expectedRevision: "a".repeat(64) } : { integrations: [{ integrationId: "demo", config: { endpoint: "synthetic" } }] }, ); }, diff --git a/apps/web/src/lib/admin/integrationAdmin.ts b/apps/web/src/lib/admin/integrationAdmin.ts index e10a94ce3..b03af1ff7 100644 --- a/apps/web/src/lib/admin/integrationAdmin.ts +++ b/apps/web/src/lib/admin/integrationAdmin.ts @@ -147,23 +147,34 @@ export function useIntegrationAdmin() { { client }, ), ), - setCredential: (id: string, key: string, value: string) => + setCredential: ( + id: string, + key: string, + value: string, + expectedRevision: string, + operationId = crypto.randomUUID(), + ) => integrationOrThrow( sdk.setAdminIntegrationCredential( id, key, - { value }, - { "idempotency-key": crypto.randomUUID() }, + { value, expectedRevision }, + { "idempotency-key": operationId }, { client }, ), ), - deleteCredential: (id: string, key: string) => + deleteCredential: ( + id: string, + key: string, + expectedRevision: string, + operationId = crypto.randomUUID(), + ) => integrationOrThrow( sdk.deleteAdminIntegrationCredential( id, key, - { "idempotency-key": crypto.randomUUID() }, - undefined, + { expectedRevision }, + { "idempotency-key": operationId }, { client }, ), ), diff --git a/packages/core/src/api/contracts/configuration-ownership.ts b/packages/core/src/api/contracts/configuration-ownership.ts new file mode 100644 index 000000000..d06ffb731 --- /dev/null +++ b/packages/core/src/api/contracts/configuration-ownership.ts @@ -0,0 +1,25 @@ +import z from "zod/v4"; +/** Editable ownership metadata only. Never host paths, values or secret hashes. */ +export const configurationOwnershipSchema = z.strictObject({ + source: z.enum(["default", "database", "vault", "config.json", "env"]), + writable: z.boolean(), + secret: z.boolean(), + adoption: z.enum(["direct", "host_action_required"]), + reason: z.enum(["host_environment", "host_file", "vault_only", "managed_workflow"]).nullable(), + envVar: z + .string() + .regex(/^[A-Z_][A-Z0-9_]{0,127}$/) + .optional(), + managedBy: z.enum(["transit.credentials", "dawarich-provisioning"]).optional(), +}); +export type ConfigurationOwnership = z.infer; +export const configurationSourceOwnedSchema = z.strictObject({ + code: z.literal("CONFIGURATION_SOURCE_OWNED"), + error: z.string(), + ownership: configurationOwnershipSchema, +}); + +export const configurationVersionConflictSchema = z.strictObject({ + code: z.literal("CONFIGURATION_VERSION_CONFLICT"), + error: z.string(), +}); diff --git a/packages/core/src/api/contracts/index.ts b/packages/core/src/api/contracts/index.ts index 1615ce8d1..580917a18 100644 --- a/packages/core/src/api/contracts/index.ts +++ b/packages/core/src/api/contracts/index.ts @@ -1,6 +1,7 @@ export { serviceBatchResultSchema } from "../../ops/service-results"; export * from "./account-admin/index"; export * from "./activity-admin"; +export * from "./configuration-ownership"; export * from "./data-manager-admin/index"; export * from "./extension-admin/index"; export * from "./integration-admin/index"; diff --git a/packages/core/src/api/contracts/integration-admin.test.ts b/packages/core/src/api/contracts/integration-admin.test.ts index 985905354..537956c64 100644 --- a/packages/core/src/api/contracts/integration-admin.test.ts +++ b/packages/core/src/api/contracts/integration-admin.test.ts @@ -275,6 +275,9 @@ describe("integration administration canonical producer contracts", () => { name: "Sample", credentials: [credential], secretsConfigured: true, + configurationRevision: "a".repeat(64), + runtime: null, + application: null, }, testAdminIntegrationCredentials: { ok: true, @@ -284,8 +287,23 @@ describe("integration administration canonical producer contracts", () => { responseTime: 15, error: null, }, - setAdminIntegrationCredential: reload, - deleteAdminIntegrationCredential: reload, + setAdminIntegrationCredential: { + operationId: "test-credential", + integrationId: "sample", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "applied", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", + }, + deleteAdminIntegrationCredential: { + operationId: "test-credential", + integrationId: "sample", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "pending", + runtimeGenerationId: null, + reason: "integration_disabled", + }, getAdminOverview: overview, }; for (const [id, fixture] of Object.entries(fixtures)) @@ -510,8 +528,14 @@ describe("integration administration canonical producer contracts", () => { }); it("rejects unknown mutation keys and malformed credential/binding/no-body shapes", () => { const ops = contracts.integrationAdminOperations; - expect(ops.setAdminIntegrationCredential.request.body.parse({ value: " synthetic " })).toEqual({ + expect( + ops.setAdminIntegrationCredential.request.body.parse({ + value: " synthetic ", + expectedRevision: "a".repeat(64), + }), + ).toEqual({ value: " synthetic ", + expectedRevision: "a".repeat(64), }); for (const value of ["", " ", null, 1]) expect(ops.setAdminIntegrationCredential.request.body.safeParse({ value }).success).toBe( @@ -539,7 +563,6 @@ describe("integration administration canonical producer contracts", () => { "reloadAdminIntegration", "reloadAdminIntegrations", "runAdminIntegrationHealthChecks", - "deleteAdminIntegrationCredential", ] as const) { expect(ops[id].request.body.safeParse(undefined).success).toBe(true); expect(ops[id].request.body.safeParse({}).success).toBe(true); @@ -552,6 +575,11 @@ describe("integration administration canonical producer contracts", () => { expectedRevision: "a".repeat(64), }).success, ).toBe(true); + expect( + ops.deleteAdminIntegrationCredential.request.body.safeParse({ + expectedRevision: "a".repeat(64), + }).success, + ).toBe(true); for (const body of [{}, [], null, { enabled: true }]) expect(ops.patchAdminIntegrationConfig.request.body.safeParse(body).success).toBe(false); }); diff --git a/packages/core/src/api/contracts/integration-admin/common.ts b/packages/core/src/api/contracts/integration-admin/common.ts index ceb30b78b..63e836028 100644 --- a/packages/core/src/api/contracts/integration-admin/common.ts +++ b/packages/core/src/api/contracts/integration-admin/common.ts @@ -5,6 +5,7 @@ import { import z from "zod/v4"; import { configurationApplyResultSchema } from "../../../ops/contract"; import { adminCountSchema, adminJsonValueSchema, adminTimestampSchema } from "../admin"; +import { configurationOwnershipSchema } from "../configuration-ownership"; import { adminJsonWire, preserveManifestJson } from "../manifest-json"; import { integrationApplyReceiptSchema, integrationRuntimeEvidenceSchema } from "./application"; import { @@ -64,6 +65,13 @@ export const adminIntegrationDetailSchema = adminIntegrationSummarySchema.extend documentation: integrationManifestSchema.shape.documentation, manifest: adminIntegrationManifestSchema, resolvedConfig: adminIntegrationResolvedConfigSchema, + ownership: z.record(z.string(), configurationOwnershipSchema).optional(), + configurationRevision: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + runtime: integrationRuntimeEvidenceSchema.nullable().optional(), + application: integrationApplyReceiptSchema.nullable().optional(), dependencyStatus: z.array( z.strictObject({ id: z.string(), loaded: z.boolean(), enabled: z.boolean() }), ), diff --git a/packages/core/src/api/contracts/integration-admin/credentials.ts b/packages/core/src/api/contracts/integration-admin/credentials.ts index 9ccc4fbb0..a980a7a63 100644 --- a/packages/core/src/api/contracts/integration-admin/credentials.ts +++ b/packages/core/src/api/contracts/integration-admin/credentials.ts @@ -1,6 +1,8 @@ import { credentialSetupSchema } from "@openmapx/extension-contracts/integration"; import z from "zod/v4"; import { adminCountSchema, adminTimestampSchema } from "../admin"; +import { configurationOwnershipSchema } from "../configuration-ownership"; +import { integrationApplyReceiptSchema, integrationRuntimeEvidenceSchema } from "./application"; import { adminIntegrationHealthStatusSchema } from "./health"; /** Winning effective layer; vault audit metadata is retained independently. */ @@ -25,6 +27,7 @@ export const adminIntegrationCredentialSchema = z.strictObject({ title: z.string(), description: z.string().optional(), source: adminIntegrationCredentialSourceSchema, + ownership: configurationOwnershipSchema.optional(), sharedSecretName: z.string().optional(), setup: adminIntegrationCredentialSetupSchema.optional(), updatedAt: adminTimestampSchema.optional(), @@ -46,9 +49,15 @@ export const adminIntegrationCredentialsSchema = z.strictObject({ integrationId: z.string(), name: z.string(), credentials: z.array(adminIntegrationCredentialSchema), + configurationRevision: z.string().regex(/^[a-f0-9]{64}$/), + runtime: integrationRuntimeEvidenceSchema.nullable(), + application: integrationApplyReceiptSchema.nullable(), secretsConfigured: z.boolean(), }); -export const adminIntegrationCredentialSetSchema = z.strictObject({ +export const adminIntegrationCredentialDeleteSchema = z.strictObject({ + expectedRevision: z.string().regex(/^[a-f0-9]{64}$/), +}); +export const adminIntegrationCredentialSetSchema = adminIntegrationCredentialDeleteSchema.extend({ // Check whitespace without transforming the actual credential bytes. value: z.string().refine((value) => value.trim().length > 0, "value must be a non-empty string"), }); diff --git a/packages/core/src/api/contracts/integration-admin/operations.ts b/packages/core/src/api/contracts/integration-admin/operations.ts index 930f3aa33..d1281edf1 100644 --- a/packages/core/src/api/contracts/integration-admin/operations.ts +++ b/packages/core/src/api/contracts/integration-admin/operations.ts @@ -1,5 +1,9 @@ import z from "zod/v4"; import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; +import { + configurationSourceOwnedSchema, + configurationVersionConflictSchema, +} from "../configuration-ownership"; import { integrationApplyReceiptSchema } from "./application"; import { adminIntegrationBindingDeleteSchema, @@ -27,6 +31,7 @@ import { } from "./common"; import { adminCredentialsSchema, + adminIntegrationCredentialDeleteSchema, adminIntegrationCredentialSetSchema, adminIntegrationCredentialsSchema, adminIntegrationCredentialTestRequestSchema, @@ -50,7 +55,9 @@ const mutation = { const reloaded = { ...missing, 200: adminJson(adminIntegrationReloadResultSchema), - 409: adminJson(httpErrorSchema), + 409: adminJson( + z.union([httpErrorSchema, configurationSourceOwnedSchema, configurationVersionConflictSchema]), + ), }; const _ok = adminJson(z.strictObject({ ok: z.literal(true) })); @@ -93,7 +100,14 @@ export const integrationAdminOperations = { ...common, 400: configErrors, 200: adminJson(adminIntegrationConfigImportedSchema), - 409: adminJson(z.union([adminIntegrationImportFailureSchema, httpErrorSchema])), + 409: adminJson( + z.union([ + adminIntegrationImportFailureSchema, + httpErrorSchema, + configurationSourceOwnedSchema, + configurationVersionConflictSchema, + ]), + ), }, }, runAdminIntegrationHealthChecks: { @@ -123,7 +137,13 @@ export const integrationAdminOperations = { ...missing, 400: configErrors, 200: adminJson(adminIntegrationConfigUpdatedSchema), - 409: adminJson(httpErrorSchema), + 409: adminJson( + z.union([ + httpErrorSchema, + configurationSourceOwnedSchema, + configurationVersionConflictSchema, + ]), + ), }, }, enableAdminIntegration: { @@ -182,7 +202,13 @@ export const integrationAdminOperations = { }, responses: { ...common, - 409: adminJson(httpErrorSchema), + 409: adminJson( + z.union([ + httpErrorSchema, + configurationSourceOwnedSchema, + configurationVersionConflictSchema, + ]), + ), 503: adminJson(httpErrorSchema), 200: adminJson(integrationApplyReceiptSchema), }, @@ -198,7 +224,13 @@ export const integrationAdminOperations = { }, responses: { ...common, - 409: adminJson(httpErrorSchema), + 409: adminJson( + z.union([ + httpErrorSchema, + configurationSourceOwnedSchema, + configurationVersionConflictSchema, + ]), + ), 503: adminJson(httpErrorSchema), 200: adminJson(integrationApplyReceiptSchema), }, @@ -236,7 +268,15 @@ export const integrationAdminOperations = { body: adminIntegrationCredentialSetSchema, }, responses: { - ...reloaded, + ...missing, + 200: adminJson(integrationApplyReceiptSchema), + 409: adminJson( + z.union([ + httpErrorSchema, + configurationSourceOwnedSchema, + configurationVersionConflictSchema, + ]), + ), 503: adminJson(z.union([httpErrorSchema, adminIntegrationVaultUnavailableSchema])), }, }, @@ -247,9 +287,20 @@ export const integrationAdminOperations = { request: { headers: adminIntegrationIdempotencyHeadersSchema, params: adminIntegrationCredentialParamsSchema, - body: adminIntegrationEmptyMutationSchema, + body: adminIntegrationCredentialDeleteSchema, + }, + responses: { + ...missing, + 200: adminJson(integrationApplyReceiptSchema), + 409: adminJson( + z.union([ + httpErrorSchema, + configurationSourceOwnedSchema, + configurationVersionConflictSchema, + ]), + ), + 503: adminJson(httpErrorSchema), }, - responses: reloaded, }, getAdminOverview: { method: "GET", diff --git a/packages/core/src/api/contracts/service-admin/index.ts b/packages/core/src/api/contracts/service-admin/index.ts index 73ce7f622..78e879dde 100644 --- a/packages/core/src/api/contracts/service-admin/index.ts +++ b/packages/core/src/api/contracts/service-admin/index.ts @@ -7,6 +7,7 @@ import { adminText, httpErrorSchema, } from "../admin"; +import { configurationSourceOwnedSchema } from "../configuration-ownership"; import { adminServiceAcceptedSchema, adminServiceCredentialParamsSchema, @@ -113,7 +114,7 @@ export const serviceAdminOperations = { responses: { ...adminCommonResponses, 200: adminJson(adminServiceOkSchema), - 409: adminJson(httpErrorSchema), + 409: adminJson(z.union([httpErrorSchema, configurationSourceOwnedSchema])), 404: adminJson(httpErrorSchema), 400: adminJson(z.union([httpErrorSchema, adminServiceValidationErrorSchema])), }, @@ -142,7 +143,13 @@ export const serviceAdminOperations = { ...adminCommonResponses, 200: adminJson(adminServiceCredentialChangedSchema), 404: adminJson(httpErrorSchema), - 409: adminJson(z.union([adminServiceCredentialManagedSchema, httpErrorSchema])), + 409: adminJson( + z.union([ + adminServiceCredentialManagedSchema, + httpErrorSchema, + configurationSourceOwnedSchema, + ]), + ), }, }, deleteAdminServiceCredential: { @@ -158,7 +165,13 @@ export const serviceAdminOperations = { ...adminCommonResponses, 200: adminJson(adminServiceCredentialChangedSchema), 404: adminJson(httpErrorSchema), - 409: adminJson(z.union([adminServiceCredentialManagedSchema, httpErrorSchema])), + 409: adminJson( + z.union([ + adminServiceCredentialManagedSchema, + httpErrorSchema, + configurationSourceOwnedSchema, + ]), + ), }, }, streamAdminServiceLogs: { @@ -174,7 +187,7 @@ export const serviceAdminOperations = { ...adminCommonResponses, 200: adminStream("text/plain"), 404: adminJson(httpErrorSchema), - 409: adminJson(httpErrorSchema), + 409: adminJson(z.union([httpErrorSchema, configurationSourceOwnedSchema])), 503: adminJson(httpErrorSchema), }, }, @@ -231,7 +244,7 @@ export const serviceAdminOperations = { responses: { ...adminCommonResponses, 200: adminJson(adminServiceSelectionUpdatedSchema), - 409: adminJson(httpErrorSchema), + 409: adminJson(z.union([httpErrorSchema, configurationSourceOwnedSchema])), 503: adminJson(httpErrorSchema), }, }, @@ -257,7 +270,7 @@ export const serviceAdminOperations = { ...adminCommonResponses, 200: adminJson(adminServiceAcceptedSchema), 404: adminJson(httpErrorSchema), - 409: adminJson(httpErrorSchema), + 409: adminJson(z.union([httpErrorSchema, configurationSourceOwnedSchema])), }, }, deleteAdminServiceBackup: { @@ -298,7 +311,7 @@ export const serviceAdminOperations = { responses: { ...adminCommonResponses, 200: adminJson(adminServiceComposeStartedSchema), - 409: adminJson(httpErrorSchema), + 409: adminJson(z.union([httpErrorSchema, configurationSourceOwnedSchema])), 400: adminJson(z.union([httpErrorSchema, adminServiceFailureSchema])), }, }, diff --git a/packages/core/src/api/contracts/service-admin/services.ts b/packages/core/src/api/contracts/service-admin/services.ts index cc3e17b4c..c995f92c7 100644 --- a/packages/core/src/api/contracts/service-admin/services.ts +++ b/packages/core/src/api/contracts/service-admin/services.ts @@ -1,9 +1,9 @@ import { credentialSetupSchema } from "@openmapx/extension-contracts/integration"; - import { serviceManifestSchema } from "@openmapx/extension-contracts/service"; import z from "zod/v4"; import { serviceActionsSchema } from "../../../ops/service-results"; import { adminCountSchema, adminJsonValueSchema, adminTimestampSchema } from "../admin"; +import { configurationOwnershipSchema } from "../configuration-ownership"; import { adminJsonWire, preserveManifestJson } from "../manifest-json"; import { adminServiceAcceptedSchema, @@ -93,7 +93,7 @@ export const adminServiceDetailSchema = z.strictObject({ status: adminServiceStatusSchema, actions: serviceActionsSchema, }); -export const adminServiceConfigSourceSchema = z.enum(["default", "database", "env"]); +export const adminServiceConfigSourceSchema = z.enum(["default", "database", "vault", "env"]); export const adminServiceResolvedConfigSchema = z.record( z.string(), z.strictObject({ value: adminJsonValueSchema, source: adminServiceConfigSourceSchema }), @@ -102,6 +102,7 @@ export const adminServiceConfigSchema = z.strictObject({ schema: adminServiceJsonObjectSchema.nullable(), resolvedConfig: adminServiceResolvedConfigSchema, envPrefix: z.string(), + ownership: z.record(z.string(), configurationOwnershipSchema).optional(), }); export const adminServiceConfigUpdateSchema = z.strictObject({ config: adminServiceJsonObjectSchema, @@ -114,8 +115,9 @@ export const adminServiceCredentialSchema = z.strictObject({ title: z.string(), description: z.string().optional(), setup: adminServiceCredentialSetupSchema.optional(), - source: z.enum(["vault", "missing"]), - managedBy: z.literal("dawarich-provisioning").optional(), + source: z.enum(["default", "database", "vault", "config.json", "env", "missing"]), + ownership: configurationOwnershipSchema.optional(), + managedBy: z.enum(["dawarich-provisioning", "transit.credentials"]).optional(), updatedAt: adminTimestampSchema.optional(), updatedBy: z.string().nullable(), }); @@ -132,7 +134,7 @@ export const adminServiceCredentialChangedSchema = z.union([ z.strictObject({ ok: z.literal(true), needsRender: z.literal(true) }), ]); export const adminServiceCredentialManagedSchema = z.strictObject({ - code: z.literal("DAWARICH_CREDENTIAL_MANAGED"), + code: z.enum(["DAWARICH_CREDENTIAL_MANAGED", "transit_credentials_managed"]), error: z.string(), }); export const adminServiceActionSchema = z.strictObject({ diff --git a/packages/core/src/api/generated/admin/integration-admin.generated.ts b/packages/core/src/api/generated/admin/integration-admin.generated.ts index c36181954..2df913cbf 100644 --- a/packages/core/src/api/generated/admin/integration-admin.generated.ts +++ b/packages/core/src/api/generated/admin/integration-admin.generated.ts @@ -23,6 +23,7 @@ import type { DeleteAdminIntegrationCredential415, DeleteAdminIntegrationCredential429, DeleteAdminIntegrationCredential500, + DeleteAdminIntegrationCredential503, DeleteAdminIntegrationCredentialBody, DeleteAdminIntegrationCredentialHeaders, DisableAdminIntegration200, @@ -674,6 +675,11 @@ export type deleteAdminIntegrationCredentialResponse500 = { status: 500; }; +export type deleteAdminIntegrationCredentialResponse503 = { + data: DeleteAdminIntegrationCredential503; + status: 503; +}; + export type deleteAdminIntegrationCredentialResponseSuccess = deleteAdminIntegrationCredentialResponse200 & { headers: Headers; @@ -688,6 +694,7 @@ export type deleteAdminIntegrationCredentialResponseError = ( | deleteAdminIntegrationCredentialResponse415 | deleteAdminIntegrationCredentialResponse429 | deleteAdminIntegrationCredentialResponse500 + | deleteAdminIntegrationCredentialResponse503 ) & { headers: Headers; }; @@ -703,8 +710,8 @@ export const getDeleteAdminIntegrationCredentialUrl = (integrationId: string, ke export const deleteAdminIntegrationCredential = async ( integrationId: string, key: string, + deleteAdminIntegrationCredentialBody: DeleteAdminIntegrationCredentialBody, headers: DeleteAdminIntegrationCredentialHeaders, - deleteAdminIntegrationCredentialBody?: DeleteAdminIntegrationCredentialBody, options?: Parameters[1], ): Promise => { const getHeaders = ( diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts index da51a4527..e76f9a6a3 100644 --- a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationBinding409.ts @@ -17,4 +17,22 @@ export type DeleteAdminIntegrationBinding409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200.ts index 9edb34d17..cd4559e58 100644 --- a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200.ts +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200.ts @@ -1,17 +1,26 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { DeleteAdminIntegrationCredential200Database } from "./deleteAdminIntegrationCredential200Database"; +import type { DeleteAdminIntegrationCredential200HostApply } from "./deleteAdminIntegrationCredential200HostApply"; +import type { DeleteAdminIntegrationCredential200Reason } from "./deleteAdminIntegrationCredential200Reason"; +import type { DeleteAdminIntegrationCredential200Runtime } from "./deleteAdminIntegrationCredential200Runtime"; export type DeleteAdminIntegrationCredential200 = { - ok: true; - message: string; /** - * @minimum 0 - * @maximum 9007199254740991 + * @minLength 1 + * @maxLength 256 */ - reloaded: number; + operationId: string; /** - * @minimum 0 - * @maximum 9007199254740991 + * @minLength 1 + * @maxLength 64 */ - enabled: number; + integrationId: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + database: DeleteAdminIntegrationCredential200Database; + runtime: DeleteAdminIntegrationCredential200Runtime; + runtimeGenerationId: string | null; + reason?: DeleteAdminIntegrationCredential200Reason; + hostApply?: DeleteAdminIntegrationCredential200HostApply; }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Database.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Database.ts new file mode 100644 index 000000000..3a9d1cbf4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Database.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationCredential200Database = + (typeof DeleteAdminIntegrationCredential200Database)[keyof typeof DeleteAdminIntegrationCredential200Database]; + +export const DeleteAdminIntegrationCredential200Database = { + retained: "retained", + restored: "restored", + conflict: "conflict", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200HostApply.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200HostApply.ts new file mode 100644 index 000000000..f044cfe62 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200HostApply.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationCredential200HostApply = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Reason.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Reason.ts new file mode 100644 index 000000000..d978dbdcc --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Reason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationCredential200Reason = + (typeof DeleteAdminIntegrationCredential200Reason)[keyof typeof DeleteAdminIntegrationCredential200Reason]; + +export const DeleteAdminIntegrationCredential200Reason = { + integration_disabled: "integration_disabled", + reload_failed: "reload_failed", + ownership_changed: "ownership_changed", + authority_unavailable: "authority_unavailable", + reconciliation_required: "reconciliation_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Runtime.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Runtime.ts new file mode 100644 index 000000000..e0f8139f3 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential200Runtime.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationCredential200Runtime = + (typeof DeleteAdminIntegrationCredential200Runtime)[keyof typeof DeleteAdminIntegrationCredential200Runtime]; + +export const DeleteAdminIntegrationCredential200Runtime = { + applied: "applied", + pending: "pending", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts index d3c7393b5..fce9f0122 100644 --- a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential409.ts @@ -17,4 +17,22 @@ export type DeleteAdminIntegrationCredential409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential503.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential503.ts new file mode 100644 index 000000000..816934eb4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredential503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminIntegrationCredential503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialBody.ts b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialBody.ts index 41aef4317..dea36d62a 100644 --- a/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialBody.ts +++ b/packages/core/src/api/generated/admin/integration-models/deleteAdminIntegrationCredentialBody.ts @@ -1,4 +1,7 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. -export type DeleteAdminIntegrationCredentialBody = { [key: string]: unknown }; +export type DeleteAdminIntegrationCredentialBody = { + /** @pattern ^[a-f0-9]{64}$ */ + expectedRevision: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts index cf1036277..5f5c2b1d5 100644 --- a/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts +++ b/packages/core/src/api/generated/admin/integration-models/disableAdminIntegration409.ts @@ -17,4 +17,22 @@ export type DisableAdminIntegration409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts index 95900167d..b1e2fd3b5 100644 --- a/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts +++ b/packages/core/src/api/generated/admin/integration-models/enableAdminIntegration409.ts @@ -17,4 +17,22 @@ export type EnableAdminIntegration409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200.ts index 5168fc319..d0d9ab98f 100644 --- a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200.ts +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200.ts @@ -1,13 +1,16 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegration200Application } from "./getAdminIntegration200Application"; import type { GetAdminIntegration200CredentialStatusItem } from "./getAdminIntegration200CredentialStatusItem"; import type { GetAdminIntegration200DependencyStatusItem } from "./getAdminIntegration200DependencyStatusItem"; import type { GetAdminIntegration200Health } from "./getAdminIntegration200Health"; import type { GetAdminIntegration200Infrastructure } from "./getAdminIntegration200Infrastructure"; import type { GetAdminIntegration200Manifest } from "./getAdminIntegration200Manifest"; +import type { GetAdminIntegration200Ownership } from "./getAdminIntegration200Ownership"; import type { GetAdminIntegration200Quality } from "./getAdminIntegration200Quality"; import type { GetAdminIntegration200RequiresItem } from "./getAdminIntegration200RequiresItem"; import type { GetAdminIntegration200ResolvedConfig } from "./getAdminIntegration200ResolvedConfig"; +import type { GetAdminIntegration200Runtime } from "./getAdminIntegration200Runtime"; export type GetAdminIntegration200 = { id: string; @@ -30,6 +33,11 @@ export type GetAdminIntegration200 = { documentation?: string; manifest: GetAdminIntegration200Manifest; resolvedConfig: GetAdminIntegration200ResolvedConfig; + ownership?: GetAdminIntegration200Ownership; + /** @pattern ^[a-f0-9]{64}$ */ + configurationRevision?: string; + runtime?: GetAdminIntegration200Runtime; + application?: GetAdminIntegration200Application; dependencyStatus: GetAdminIntegration200DependencyStatusItem[]; credentialStatus: GetAdminIntegration200CredentialStatusItem[]; secretsConfigured: boolean; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Application.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Application.ts new file mode 100644 index 000000000..ce3a77c7a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Application.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegration200ApplicationDatabase } from "./getAdminIntegration200ApplicationDatabase"; +import type { GetAdminIntegration200ApplicationHostApply } from "./getAdminIntegration200ApplicationHostApply"; +import type { GetAdminIntegration200ApplicationReason } from "./getAdminIntegration200ApplicationReason"; +import type { GetAdminIntegration200ApplicationRuntime } from "./getAdminIntegration200ApplicationRuntime"; + +export type GetAdminIntegration200Application = { + /** + * @minLength 1 + * @maxLength 256 + */ + operationId: string; + /** + * @minLength 1 + * @maxLength 64 + */ + integrationId: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + database: GetAdminIntegration200ApplicationDatabase; + runtime: GetAdminIntegration200ApplicationRuntime; + runtimeGenerationId: string | null; + reason?: GetAdminIntegration200ApplicationReason; + hostApply?: GetAdminIntegration200ApplicationHostApply; +} | null; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationDatabase.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationDatabase.ts new file mode 100644 index 000000000..bc5cde749 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationDatabase.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200ApplicationDatabase = + (typeof GetAdminIntegration200ApplicationDatabase)[keyof typeof GetAdminIntegration200ApplicationDatabase]; + +export const GetAdminIntegration200ApplicationDatabase = { + retained: "retained", + restored: "restored", + conflict: "conflict", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationHostApply.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationHostApply.ts new file mode 100644 index 000000000..6667a2bd0 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationHostApply.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200ApplicationHostApply = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationReason.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationReason.ts new file mode 100644 index 000000000..64e3ae582 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200ApplicationReason = + (typeof GetAdminIntegration200ApplicationReason)[keyof typeof GetAdminIntegration200ApplicationReason]; + +export const GetAdminIntegration200ApplicationReason = { + integration_disabled: "integration_disabled", + reload_failed: "reload_failed", + ownership_changed: "ownership_changed", + authority_unavailable: "authority_unavailable", + reconciliation_required: "reconciliation_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationRuntime.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationRuntime.ts new file mode 100644 index 000000000..0f8b43821 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200ApplicationRuntime.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200ApplicationRuntime = + (typeof GetAdminIntegration200ApplicationRuntime)[keyof typeof GetAdminIntegration200ApplicationRuntime]; + +export const GetAdminIntegration200ApplicationRuntime = { + applied: "applied", + pending: "pending", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItem.ts index 4ef08ed91..ec42a7906 100644 --- a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItem.ts +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItem.ts @@ -1,5 +1,6 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegration200CredentialStatusItemOwnership } from "./getAdminIntegration200CredentialStatusItemOwnership"; import type { GetAdminIntegration200CredentialStatusItemSetup } from "./getAdminIntegration200CredentialStatusItemSetup"; import type { GetAdminIntegration200CredentialStatusItemSource } from "./getAdminIntegration200CredentialStatusItemSource"; @@ -8,6 +9,7 @@ export type GetAdminIntegration200CredentialStatusItem = { title: string; description?: string; source: GetAdminIntegration200CredentialStatusItemSource; + ownership?: GetAdminIntegration200CredentialStatusItemOwnership; sharedSecretName?: string; setup?: GetAdminIntegration200CredentialStatusItemSetup; /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnership.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnership.ts new file mode 100644 index 000000000..82db45045 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnership.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegration200CredentialStatusItemOwnershipAdoption } from "./getAdminIntegration200CredentialStatusItemOwnershipAdoption"; +import type { GetAdminIntegration200CredentialStatusItemOwnershipManagedBy } from "./getAdminIntegration200CredentialStatusItemOwnershipManagedBy"; +import type { GetAdminIntegration200CredentialStatusItemOwnershipReason } from "./getAdminIntegration200CredentialStatusItemOwnershipReason"; +import type { GetAdminIntegration200CredentialStatusItemOwnershipSource } from "./getAdminIntegration200CredentialStatusItemOwnershipSource"; + +export type GetAdminIntegration200CredentialStatusItemOwnership = { + source: GetAdminIntegration200CredentialStatusItemOwnershipSource; + writable: boolean; + secret: boolean; + adoption: GetAdminIntegration200CredentialStatusItemOwnershipAdoption; + reason: GetAdminIntegration200CredentialStatusItemOwnershipReason; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: GetAdminIntegration200CredentialStatusItemOwnershipManagedBy; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipAdoption.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipAdoption.ts new file mode 100644 index 000000000..826654190 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipAdoption.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200CredentialStatusItemOwnershipAdoption = + (typeof GetAdminIntegration200CredentialStatusItemOwnershipAdoption)[keyof typeof GetAdminIntegration200CredentialStatusItemOwnershipAdoption]; + +export const GetAdminIntegration200CredentialStatusItemOwnershipAdoption = { + direct: "direct", + host_action_required: "host_action_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipManagedBy.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipManagedBy.ts new file mode 100644 index 000000000..2e886a6fc --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipManagedBy.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200CredentialStatusItemOwnershipManagedBy = + (typeof GetAdminIntegration200CredentialStatusItemOwnershipManagedBy)[keyof typeof GetAdminIntegration200CredentialStatusItemOwnershipManagedBy]; + +export const GetAdminIntegration200CredentialStatusItemOwnershipManagedBy = { + transitcredentials: "transit.credentials", + "dawarich-provisioning": "dawarich-provisioning", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipReason.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipReason.ts new file mode 100644 index 000000000..bcfd2e4e2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200CredentialStatusItemOwnershipReason = + | (typeof GetAdminIntegration200CredentialStatusItemOwnershipReason)[keyof typeof GetAdminIntegration200CredentialStatusItemOwnershipReason] + | null; + +export const GetAdminIntegration200CredentialStatusItemOwnershipReason = { + host_environment: "host_environment", + host_file: "host_file", + vault_only: "vault_only", + managed_workflow: "managed_workflow", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipSource.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipSource.ts new file mode 100644 index 000000000..965fe78a9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200CredentialStatusItemOwnershipSource.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200CredentialStatusItemOwnershipSource = + (typeof GetAdminIntegration200CredentialStatusItemOwnershipSource)[keyof typeof GetAdminIntegration200CredentialStatusItemOwnershipSource]; + +export const GetAdminIntegration200CredentialStatusItemOwnershipSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Ownership.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Ownership.ts new file mode 100644 index 000000000..6e0c873e5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Ownership.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegration200OwnershipAdoption } from "./getAdminIntegration200OwnershipAdoption"; +import type { GetAdminIntegration200OwnershipManagedBy } from "./getAdminIntegration200OwnershipManagedBy"; +import type { GetAdminIntegration200OwnershipReason } from "./getAdminIntegration200OwnershipReason"; +import type { GetAdminIntegration200OwnershipSource } from "./getAdminIntegration200OwnershipSource"; + +export type GetAdminIntegration200Ownership = { + [key: string]: { + source: GetAdminIntegration200OwnershipSource; + writable: boolean; + secret: boolean; + adoption: GetAdminIntegration200OwnershipAdoption; + reason: GetAdminIntegration200OwnershipReason; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: GetAdminIntegration200OwnershipManagedBy; + }; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipAdoption.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipAdoption.ts new file mode 100644 index 000000000..d384f8aaa --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipAdoption.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200OwnershipAdoption = + (typeof GetAdminIntegration200OwnershipAdoption)[keyof typeof GetAdminIntegration200OwnershipAdoption]; + +export const GetAdminIntegration200OwnershipAdoption = { + direct: "direct", + host_action_required: "host_action_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipManagedBy.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipManagedBy.ts new file mode 100644 index 000000000..3313558eb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipManagedBy.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200OwnershipManagedBy = + (typeof GetAdminIntegration200OwnershipManagedBy)[keyof typeof GetAdminIntegration200OwnershipManagedBy]; + +export const GetAdminIntegration200OwnershipManagedBy = { + transitcredentials: "transit.credentials", + "dawarich-provisioning": "dawarich-provisioning", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipReason.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipReason.ts new file mode 100644 index 000000000..54e059198 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200OwnershipReason = + | (typeof GetAdminIntegration200OwnershipReason)[keyof typeof GetAdminIntegration200OwnershipReason] + | null; + +export const GetAdminIntegration200OwnershipReason = { + host_environment: "host_environment", + host_file: "host_file", + vault_only: "vault_only", + managed_workflow: "managed_workflow", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipSource.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipSource.ts new file mode 100644 index 000000000..3a16581ce --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200OwnershipSource.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200OwnershipSource = + (typeof GetAdminIntegration200OwnershipSource)[keyof typeof GetAdminIntegration200OwnershipSource]; + +export const GetAdminIntegration200OwnershipSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Runtime.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Runtime.ts new file mode 100644 index 000000000..20a6f81ad --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200Runtime.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegration200RuntimeBindings } from "./getAdminIntegration200RuntimeBindings"; + +export type GetAdminIntegration200Runtime = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + generationId: string; + bindings: GetAdminIntegration200RuntimeBindings; + /** @pattern ^[a-f0-9]{64}$ */ + configRevision: string; +} | null; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200RuntimeBindings.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200RuntimeBindings.ts new file mode 100644 index 000000000..d07fddf9e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegration200RuntimeBindings.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegration200RuntimeBindings = { [key: string]: string | null }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200.ts index 156d448e2..68d0d851b 100644 --- a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200.ts +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200.ts @@ -1,10 +1,16 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegrationCredentials200Application } from "./getAdminIntegrationCredentials200Application"; import type { GetAdminIntegrationCredentials200CredentialsItem } from "./getAdminIntegrationCredentials200CredentialsItem"; +import type { GetAdminIntegrationCredentials200Runtime } from "./getAdminIntegrationCredentials200Runtime"; export type GetAdminIntegrationCredentials200 = { integrationId: string; name: string; credentials: GetAdminIntegrationCredentials200CredentialsItem[]; + /** @pattern ^[a-f0-9]{64}$ */ + configurationRevision: string; + runtime: GetAdminIntegrationCredentials200Runtime; + application: GetAdminIntegrationCredentials200Application; secretsConfigured: boolean; }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Application.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Application.ts new file mode 100644 index 000000000..96255bbcf --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Application.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegrationCredentials200ApplicationDatabase } from "./getAdminIntegrationCredentials200ApplicationDatabase"; +import type { GetAdminIntegrationCredentials200ApplicationHostApply } from "./getAdminIntegrationCredentials200ApplicationHostApply"; +import type { GetAdminIntegrationCredentials200ApplicationReason } from "./getAdminIntegrationCredentials200ApplicationReason"; +import type { GetAdminIntegrationCredentials200ApplicationRuntime } from "./getAdminIntegrationCredentials200ApplicationRuntime"; + +export type GetAdminIntegrationCredentials200Application = { + /** + * @minLength 1 + * @maxLength 256 + */ + operationId: string; + /** + * @minLength 1 + * @maxLength 64 + */ + integrationId: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + database: GetAdminIntegrationCredentials200ApplicationDatabase; + runtime: GetAdminIntegrationCredentials200ApplicationRuntime; + runtimeGenerationId: string | null; + reason?: GetAdminIntegrationCredentials200ApplicationReason; + hostApply?: GetAdminIntegrationCredentials200ApplicationHostApply; +} | null; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationDatabase.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationDatabase.ts new file mode 100644 index 000000000..d0ab55091 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationDatabase.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200ApplicationDatabase = + (typeof GetAdminIntegrationCredentials200ApplicationDatabase)[keyof typeof GetAdminIntegrationCredentials200ApplicationDatabase]; + +export const GetAdminIntegrationCredentials200ApplicationDatabase = { + retained: "retained", + restored: "restored", + conflict: "conflict", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationHostApply.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationHostApply.ts new file mode 100644 index 000000000..8f9eec435 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationHostApply.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200ApplicationHostApply = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationReason.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationReason.ts new file mode 100644 index 000000000..4fc8e1f8c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200ApplicationReason = + (typeof GetAdminIntegrationCredentials200ApplicationReason)[keyof typeof GetAdminIntegrationCredentials200ApplicationReason]; + +export const GetAdminIntegrationCredentials200ApplicationReason = { + integration_disabled: "integration_disabled", + reload_failed: "reload_failed", + ownership_changed: "ownership_changed", + authority_unavailable: "authority_unavailable", + reconciliation_required: "reconciliation_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationRuntime.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationRuntime.ts new file mode 100644 index 000000000..82c01ba18 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200ApplicationRuntime.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200ApplicationRuntime = + (typeof GetAdminIntegrationCredentials200ApplicationRuntime)[keyof typeof GetAdminIntegrationCredentials200ApplicationRuntime]; + +export const GetAdminIntegrationCredentials200ApplicationRuntime = { + applied: "applied", + pending: "pending", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItem.ts index d511b20c7..c2ec55ee6 100644 --- a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItem.ts +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItem.ts @@ -1,5 +1,6 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegrationCredentials200CredentialsItemOwnership } from "./getAdminIntegrationCredentials200CredentialsItemOwnership"; import type { GetAdminIntegrationCredentials200CredentialsItemSetup } from "./getAdminIntegrationCredentials200CredentialsItemSetup"; import type { GetAdminIntegrationCredentials200CredentialsItemSource } from "./getAdminIntegrationCredentials200CredentialsItemSource"; @@ -8,6 +9,7 @@ export type GetAdminIntegrationCredentials200CredentialsItem = { title: string; description?: string; source: GetAdminIntegrationCredentials200CredentialsItemSource; + ownership?: GetAdminIntegrationCredentials200CredentialsItemOwnership; sharedSecretName?: string; setup?: GetAdminIntegrationCredentials200CredentialsItemSetup; /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnership.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnership.ts new file mode 100644 index 000000000..375e3c251 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnership.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegrationCredentials200CredentialsItemOwnershipAdoption } from "./getAdminIntegrationCredentials200CredentialsItemOwnershipAdoption"; +import type { GetAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy } from "./getAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy"; +import type { GetAdminIntegrationCredentials200CredentialsItemOwnershipReason } from "./getAdminIntegrationCredentials200CredentialsItemOwnershipReason"; +import type { GetAdminIntegrationCredentials200CredentialsItemOwnershipSource } from "./getAdminIntegrationCredentials200CredentialsItemOwnershipSource"; + +export type GetAdminIntegrationCredentials200CredentialsItemOwnership = { + source: GetAdminIntegrationCredentials200CredentialsItemOwnershipSource; + writable: boolean; + secret: boolean; + adoption: GetAdminIntegrationCredentials200CredentialsItemOwnershipAdoption; + reason: GetAdminIntegrationCredentials200CredentialsItemOwnershipReason; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: GetAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipAdoption.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipAdoption.ts new file mode 100644 index 000000000..8bd58239f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipAdoption.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200CredentialsItemOwnershipAdoption = + (typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipAdoption)[keyof typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipAdoption]; + +export const GetAdminIntegrationCredentials200CredentialsItemOwnershipAdoption = { + direct: "direct", + host_action_required: "host_action_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy.ts new file mode 100644 index 000000000..9b1f3f322 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy = + (typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy)[keyof typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy]; + +export const GetAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy = { + transitcredentials: "transit.credentials", + "dawarich-provisioning": "dawarich-provisioning", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipReason.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipReason.ts new file mode 100644 index 000000000..9a6f0c346 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200CredentialsItemOwnershipReason = + | (typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipReason)[keyof typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipReason] + | null; + +export const GetAdminIntegrationCredentials200CredentialsItemOwnershipReason = { + host_environment: "host_environment", + host_file: "host_file", + vault_only: "vault_only", + managed_workflow: "managed_workflow", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipSource.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipSource.ts new file mode 100644 index 000000000..3d2d7ee4c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200CredentialsItemOwnershipSource.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200CredentialsItemOwnershipSource = + (typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipSource)[keyof typeof GetAdminIntegrationCredentials200CredentialsItemOwnershipSource]; + +export const GetAdminIntegrationCredentials200CredentialsItemOwnershipSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Runtime.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Runtime.ts new file mode 100644 index 000000000..2fb75b365 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200Runtime.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminIntegrationCredentials200RuntimeBindings } from "./getAdminIntegrationCredentials200RuntimeBindings"; + +export type GetAdminIntegrationCredentials200Runtime = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + generationId: string; + bindings: GetAdminIntegrationCredentials200RuntimeBindings; + /** @pattern ^[a-f0-9]{64}$ */ + configRevision: string; +} | null; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200RuntimeBindings.ts b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200RuntimeBindings.ts new file mode 100644 index 000000000..55197ea09 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminIntegrationCredentials200RuntimeBindings.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminIntegrationCredentials200RuntimeBindings = { [key: string]: string | null }; diff --git a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts index fccd3b6f3..276b92ecb 100644 --- a/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts +++ b/packages/core/src/api/generated/admin/integration-models/importAdminIntegrationConfigs409.ts @@ -88,4 +88,22 @@ export type ImportAdminIntegrationConfigs409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/index.ts b/packages/core/src/api/generated/admin/integration-models/index.ts index 18b9e8433..41021ba30 100644 --- a/packages/core/src/api/generated/admin/integration-models/index.ts +++ b/packages/core/src/api/generated/admin/integration-models/index.ts @@ -24,6 +24,10 @@ export * from "./deleteAdminIntegrationBinding503"; export * from "./deleteAdminIntegrationBindingBody"; export * from "./deleteAdminIntegrationBindingHeaders"; export * from "./deleteAdminIntegrationCredential200"; +export * from "./deleteAdminIntegrationCredential200Database"; +export * from "./deleteAdminIntegrationCredential200HostApply"; +export * from "./deleteAdminIntegrationCredential200Reason"; +export * from "./deleteAdminIntegrationCredential200Runtime"; export * from "./deleteAdminIntegrationCredential400"; export * from "./deleteAdminIntegrationCredential401"; export * from "./deleteAdminIntegrationCredential403"; @@ -33,6 +37,7 @@ export * from "./deleteAdminIntegrationCredential413"; export * from "./deleteAdminIntegrationCredential415"; export * from "./deleteAdminIntegrationCredential429"; export * from "./deleteAdminIntegrationCredential500"; +export * from "./deleteAdminIntegrationCredential503"; export * from "./deleteAdminIntegrationCredentialBody"; export * from "./deleteAdminIntegrationCredentialHeaders"; export * from "./disableAdminIntegration200"; @@ -70,7 +75,17 @@ export * from "./exportAdminIntegrationConfigs415"; export * from "./exportAdminIntegrationConfigs429"; export * from "./exportAdminIntegrationConfigs500"; export * from "./getAdminIntegration200"; +export * from "./getAdminIntegration200Application"; +export * from "./getAdminIntegration200ApplicationDatabase"; +export * from "./getAdminIntegration200ApplicationHostApply"; +export * from "./getAdminIntegration200ApplicationReason"; +export * from "./getAdminIntegration200ApplicationRuntime"; export * from "./getAdminIntegration200CredentialStatusItem"; +export * from "./getAdminIntegration200CredentialStatusItemOwnership"; +export * from "./getAdminIntegration200CredentialStatusItemOwnershipAdoption"; +export * from "./getAdminIntegration200CredentialStatusItemOwnershipManagedBy"; +export * from "./getAdminIntegration200CredentialStatusItemOwnershipReason"; +export * from "./getAdminIntegration200CredentialStatusItemOwnershipSource"; export * from "./getAdminIntegration200CredentialStatusItemSetup"; export * from "./getAdminIntegration200CredentialStatusItemSetupEmail"; export * from "./getAdminIntegration200CredentialStatusItemSource"; @@ -116,10 +131,17 @@ export * from "./getAdminIntegration200ManifestQuality"; export * from "./getAdminIntegration200ManifestRequiresItem"; export * from "./getAdminIntegration200ManifestSubjectData"; export * from "./getAdminIntegration200ManifestSubjectDataStrategy"; +export * from "./getAdminIntegration200Ownership"; +export * from "./getAdminIntegration200OwnershipAdoption"; +export * from "./getAdminIntegration200OwnershipManagedBy"; +export * from "./getAdminIntegration200OwnershipReason"; +export * from "./getAdminIntegration200OwnershipSource"; export * from "./getAdminIntegration200Quality"; export * from "./getAdminIntegration200RequiresItem"; export * from "./getAdminIntegration200ResolvedConfig"; export * from "./getAdminIntegration200ResolvedConfigSource"; +export * from "./getAdminIntegration200Runtime"; +export * from "./getAdminIntegration200RuntimeBindings"; export * from "./getAdminIntegration400"; export * from "./getAdminIntegration401"; export * from "./getAdminIntegration403"; @@ -129,10 +151,22 @@ export * from "./getAdminIntegration415"; export * from "./getAdminIntegration429"; export * from "./getAdminIntegration500"; export * from "./getAdminIntegrationCredentials200"; +export * from "./getAdminIntegrationCredentials200Application"; +export * from "./getAdminIntegrationCredentials200ApplicationDatabase"; +export * from "./getAdminIntegrationCredentials200ApplicationHostApply"; +export * from "./getAdminIntegrationCredentials200ApplicationReason"; +export * from "./getAdminIntegrationCredentials200ApplicationRuntime"; export * from "./getAdminIntegrationCredentials200CredentialsItem"; +export * from "./getAdminIntegrationCredentials200CredentialsItemOwnership"; +export * from "./getAdminIntegrationCredentials200CredentialsItemOwnershipAdoption"; +export * from "./getAdminIntegrationCredentials200CredentialsItemOwnershipManagedBy"; +export * from "./getAdminIntegrationCredentials200CredentialsItemOwnershipReason"; +export * from "./getAdminIntegrationCredentials200CredentialsItemOwnershipSource"; export * from "./getAdminIntegrationCredentials200CredentialsItemSetup"; export * from "./getAdminIntegrationCredentials200CredentialsItemSetupEmail"; export * from "./getAdminIntegrationCredentials200CredentialsItemSource"; +export * from "./getAdminIntegrationCredentials200Runtime"; +export * from "./getAdminIntegrationCredentials200RuntimeBindings"; export * from "./getAdminIntegrationCredentials400"; export * from "./getAdminIntegrationCredentials401"; export * from "./getAdminIntegrationCredentials403"; @@ -298,6 +332,10 @@ export * from "./setAdminIntegrationBinding503"; export * from "./setAdminIntegrationBindingBody"; export * from "./setAdminIntegrationBindingHeaders"; export * from "./setAdminIntegrationCredential200"; +export * from "./setAdminIntegrationCredential200Database"; +export * from "./setAdminIntegrationCredential200HostApply"; +export * from "./setAdminIntegrationCredential200Reason"; +export * from "./setAdminIntegrationCredential200Runtime"; export * from "./setAdminIntegrationCredential400"; export * from "./setAdminIntegrationCredential401"; export * from "./setAdminIntegrationCredential403"; diff --git a/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts index bce415243..1a815071e 100644 --- a/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts +++ b/packages/core/src/api/generated/admin/integration-models/patchAdminIntegrationConfig409.ts @@ -17,4 +17,22 @@ export type PatchAdminIntegrationConfig409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts index 7cbd49f7e..e6a02ac77 100644 --- a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationBinding409.ts @@ -17,4 +17,22 @@ export type SetAdminIntegrationBinding409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200.ts index fc9927728..c20cc56ea 100644 --- a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200.ts +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200.ts @@ -1,17 +1,26 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { SetAdminIntegrationCredential200Database } from "./setAdminIntegrationCredential200Database"; +import type { SetAdminIntegrationCredential200HostApply } from "./setAdminIntegrationCredential200HostApply"; +import type { SetAdminIntegrationCredential200Reason } from "./setAdminIntegrationCredential200Reason"; +import type { SetAdminIntegrationCredential200Runtime } from "./setAdminIntegrationCredential200Runtime"; export type SetAdminIntegrationCredential200 = { - ok: true; - message: string; /** - * @minimum 0 - * @maximum 9007199254740991 + * @minLength 1 + * @maxLength 256 */ - reloaded: number; + operationId: string; /** - * @minimum 0 - * @maximum 9007199254740991 + * @minLength 1 + * @maxLength 64 */ - enabled: number; + integrationId: string; + /** @pattern ^[a-f0-9]{64}$ */ + desiredRevision: string; + database: SetAdminIntegrationCredential200Database; + runtime: SetAdminIntegrationCredential200Runtime; + runtimeGenerationId: string | null; + reason?: SetAdminIntegrationCredential200Reason; + hostApply?: SetAdminIntegrationCredential200HostApply; }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Database.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Database.ts new file mode 100644 index 000000000..2ce07eac0 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Database.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationCredential200Database = + (typeof SetAdminIntegrationCredential200Database)[keyof typeof SetAdminIntegrationCredential200Database]; + +export const SetAdminIntegrationCredential200Database = { + retained: "retained", + restored: "restored", + conflict: "conflict", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200HostApply.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200HostApply.ts new file mode 100644 index 000000000..04ce08bc1 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200HostApply.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationCredential200HostApply = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Reason.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Reason.ts new file mode 100644 index 000000000..3dbdd1391 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Reason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationCredential200Reason = + (typeof SetAdminIntegrationCredential200Reason)[keyof typeof SetAdminIntegrationCredential200Reason]; + +export const SetAdminIntegrationCredential200Reason = { + integration_disabled: "integration_disabled", + reload_failed: "reload_failed", + ownership_changed: "ownership_changed", + authority_unavailable: "authority_unavailable", + reconciliation_required: "reconciliation_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Runtime.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Runtime.ts new file mode 100644 index 000000000..a8cec5fd8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential200Runtime.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type SetAdminIntegrationCredential200Runtime = + (typeof SetAdminIntegrationCredential200Runtime)[keyof typeof SetAdminIntegrationCredential200Runtime]; + +export const SetAdminIntegrationCredential200Runtime = { + applied: "applied", + pending: "pending", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts index 71bd170f9..bab331603 100644 --- a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredential409.ts @@ -17,4 +17,22 @@ export type SetAdminIntegrationCredential409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; + } + | { + code: "CONFIGURATION_VERSION_CONFLICT"; + error: string; }; diff --git a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialBody.ts b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialBody.ts index 44e4c4057..f7462cc36 100644 --- a/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialBody.ts +++ b/packages/core/src/api/generated/admin/integration-models/setAdminIntegrationCredentialBody.ts @@ -2,5 +2,7 @@ // Generated by pnpm sdk:generate. Do not edit. export type SetAdminIntegrationCredentialBody = { + /** @pattern ^[a-f0-9]{64}$ */ + expectedRevision: string; value: string; }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts index 3ef2f0d84..273210231 100644 --- a/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminServiceCredential409.ts @@ -3,7 +3,7 @@ export type DeleteAdminServiceCredential409 = | { - code: "DAWARICH_CREDENTIAL_MANAGED"; + code: "DAWARICH_CREDENTIAL_MANAGED" | "transit_credentials_managed"; error: string; } | { @@ -21,4 +21,18 @@ export type DeleteAdminServiceCredential409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200.ts index b71da9615..202cf7fed 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200.ts @@ -1,5 +1,6 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminServiceConfig200Ownership } from "./getAdminServiceConfig200Ownership"; import type { GetAdminServiceConfig200ResolvedConfig } from "./getAdminServiceConfig200ResolvedConfig"; import type { GetAdminServiceConfig200Schema } from "./getAdminServiceConfig200Schema"; @@ -7,4 +8,5 @@ export type GetAdminServiceConfig200 = { schema: GetAdminServiceConfig200Schema; resolvedConfig: GetAdminServiceConfig200ResolvedConfig; envPrefix: string; + ownership?: GetAdminServiceConfig200Ownership; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200Ownership.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200Ownership.ts new file mode 100644 index 000000000..3641d6b68 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200Ownership.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminServiceConfig200OwnershipAdoption } from "./getAdminServiceConfig200OwnershipAdoption"; +import type { GetAdminServiceConfig200OwnershipManagedBy } from "./getAdminServiceConfig200OwnershipManagedBy"; +import type { GetAdminServiceConfig200OwnershipReason } from "./getAdminServiceConfig200OwnershipReason"; +import type { GetAdminServiceConfig200OwnershipSource } from "./getAdminServiceConfig200OwnershipSource"; + +export type GetAdminServiceConfig200Ownership = { + [key: string]: { + source: GetAdminServiceConfig200OwnershipSource; + writable: boolean; + secret: boolean; + adoption: GetAdminServiceConfig200OwnershipAdoption; + reason: GetAdminServiceConfig200OwnershipReason; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: GetAdminServiceConfig200OwnershipManagedBy; + }; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipAdoption.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipAdoption.ts new file mode 100644 index 000000000..85eedcf5b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipAdoption.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceConfig200OwnershipAdoption = + (typeof GetAdminServiceConfig200OwnershipAdoption)[keyof typeof GetAdminServiceConfig200OwnershipAdoption]; + +export const GetAdminServiceConfig200OwnershipAdoption = { + direct: "direct", + host_action_required: "host_action_required", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipManagedBy.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipManagedBy.ts new file mode 100644 index 000000000..9987400ef --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipManagedBy.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceConfig200OwnershipManagedBy = + (typeof GetAdminServiceConfig200OwnershipManagedBy)[keyof typeof GetAdminServiceConfig200OwnershipManagedBy]; + +export const GetAdminServiceConfig200OwnershipManagedBy = { + transitcredentials: "transit.credentials", + "dawarich-provisioning": "dawarich-provisioning", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipReason.ts new file mode 100644 index 000000000..4116c5169 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceConfig200OwnershipReason = + | (typeof GetAdminServiceConfig200OwnershipReason)[keyof typeof GetAdminServiceConfig200OwnershipReason] + | null; + +export const GetAdminServiceConfig200OwnershipReason = { + host_environment: "host_environment", + host_file: "host_file", + vault_only: "vault_only", + managed_workflow: "managed_workflow", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipSource.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipSource.ts new file mode 100644 index 000000000..c28e9b9d0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200OwnershipSource.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceConfig200OwnershipSource = + (typeof GetAdminServiceConfig200OwnershipSource)[keyof typeof GetAdminServiceConfig200OwnershipSource]; + +export const GetAdminServiceConfig200OwnershipSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200ResolvedConfigSource.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200ResolvedConfigSource.ts index 0b389a368..05ae3bc06 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200ResolvedConfigSource.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceConfig200ResolvedConfigSource.ts @@ -7,5 +7,6 @@ export type GetAdminServiceConfig200ResolvedConfigSource = export const GetAdminServiceConfig200ResolvedConfigSource = { default: "default", database: "database", + vault: "vault", env: "env", } as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItem.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItem.ts index 61a2f0c82..1f1257107 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItem.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItem.ts @@ -1,6 +1,7 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. import type { GetAdminServiceCredentials200CredentialsItemManagedBy } from "./getAdminServiceCredentials200CredentialsItemManagedBy"; +import type { GetAdminServiceCredentials200CredentialsItemOwnership } from "./getAdminServiceCredentials200CredentialsItemOwnership"; import type { GetAdminServiceCredentials200CredentialsItemSetup } from "./getAdminServiceCredentials200CredentialsItemSetup"; import type { GetAdminServiceCredentials200CredentialsItemSource } from "./getAdminServiceCredentials200CredentialsItemSource"; @@ -10,6 +11,7 @@ export type GetAdminServiceCredentials200CredentialsItem = { description?: string; setup?: GetAdminServiceCredentials200CredentialsItemSetup; source: GetAdminServiceCredentials200CredentialsItemSource; + ownership?: GetAdminServiceCredentials200CredentialsItemOwnership; managedBy?: GetAdminServiceCredentials200CredentialsItemManagedBy; /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ updatedAt?: string; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemManagedBy.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemManagedBy.ts index b66ea0ba2..32e0c8a33 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemManagedBy.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemManagedBy.ts @@ -6,4 +6,5 @@ export type GetAdminServiceCredentials200CredentialsItemManagedBy = export const GetAdminServiceCredentials200CredentialsItemManagedBy = { "dawarich-provisioning": "dawarich-provisioning", + transitcredentials: "transit.credentials", } as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnership.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnership.ts new file mode 100644 index 000000000..55fbbdf8b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnership.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminServiceCredentials200CredentialsItemOwnershipAdoption } from "./getAdminServiceCredentials200CredentialsItemOwnershipAdoption"; +import type { GetAdminServiceCredentials200CredentialsItemOwnershipManagedBy } from "./getAdminServiceCredentials200CredentialsItemOwnershipManagedBy"; +import type { GetAdminServiceCredentials200CredentialsItemOwnershipReason } from "./getAdminServiceCredentials200CredentialsItemOwnershipReason"; +import type { GetAdminServiceCredentials200CredentialsItemOwnershipSource } from "./getAdminServiceCredentials200CredentialsItemOwnershipSource"; + +export type GetAdminServiceCredentials200CredentialsItemOwnership = { + source: GetAdminServiceCredentials200CredentialsItemOwnershipSource; + writable: boolean; + secret: boolean; + adoption: GetAdminServiceCredentials200CredentialsItemOwnershipAdoption; + reason: GetAdminServiceCredentials200CredentialsItemOwnershipReason; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: GetAdminServiceCredentials200CredentialsItemOwnershipManagedBy; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipAdoption.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipAdoption.ts new file mode 100644 index 000000000..b358c462f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipAdoption.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceCredentials200CredentialsItemOwnershipAdoption = + (typeof GetAdminServiceCredentials200CredentialsItemOwnershipAdoption)[keyof typeof GetAdminServiceCredentials200CredentialsItemOwnershipAdoption]; + +export const GetAdminServiceCredentials200CredentialsItemOwnershipAdoption = { + direct: "direct", + host_action_required: "host_action_required", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipManagedBy.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipManagedBy.ts new file mode 100644 index 000000000..4d22b3543 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipManagedBy.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceCredentials200CredentialsItemOwnershipManagedBy = + (typeof GetAdminServiceCredentials200CredentialsItemOwnershipManagedBy)[keyof typeof GetAdminServiceCredentials200CredentialsItemOwnershipManagedBy]; + +export const GetAdminServiceCredentials200CredentialsItemOwnershipManagedBy = { + transitcredentials: "transit.credentials", + "dawarich-provisioning": "dawarich-provisioning", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipReason.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipReason.ts new file mode 100644 index 000000000..da9485760 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceCredentials200CredentialsItemOwnershipReason = + | (typeof GetAdminServiceCredentials200CredentialsItemOwnershipReason)[keyof typeof GetAdminServiceCredentials200CredentialsItemOwnershipReason] + | null; + +export const GetAdminServiceCredentials200CredentialsItemOwnershipReason = { + host_environment: "host_environment", + host_file: "host_file", + vault_only: "vault_only", + managed_workflow: "managed_workflow", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipSource.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipSource.ts new file mode 100644 index 000000000..d35264b3d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemOwnershipSource.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceCredentials200CredentialsItemOwnershipSource = + (typeof GetAdminServiceCredentials200CredentialsItemOwnershipSource)[keyof typeof GetAdminServiceCredentials200CredentialsItemOwnershipSource]; + +export const GetAdminServiceCredentials200CredentialsItemOwnershipSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemSource.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemSource.ts index 8e0f4783c..d52f88437 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemSource.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceCredentials200CredentialsItemSource.ts @@ -5,6 +5,10 @@ export type GetAdminServiceCredentials200CredentialsItemSource = (typeof GetAdminServiceCredentials200CredentialsItemSource)[keyof typeof GetAdminServiceCredentials200CredentialsItemSource]; export const GetAdminServiceCredentials200CredentialsItemSource = { + default: "default", + database: "database", vault: "vault", + configjson: "config.json", + env: "env", missing: "missing", } as const; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index b3f63abf2..0e38d53c9 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -164,6 +164,11 @@ export * from "./getAdminService429"; export * from "./getAdminService500"; export * from "./getAdminService503"; export * from "./getAdminServiceConfig200"; +export * from "./getAdminServiceConfig200Ownership"; +export * from "./getAdminServiceConfig200OwnershipAdoption"; +export * from "./getAdminServiceConfig200OwnershipManagedBy"; +export * from "./getAdminServiceConfig200OwnershipReason"; +export * from "./getAdminServiceConfig200OwnershipSource"; export * from "./getAdminServiceConfig200ResolvedConfig"; export * from "./getAdminServiceConfig200ResolvedConfigSource"; export * from "./getAdminServiceConfig200Schema"; @@ -178,6 +183,11 @@ export * from "./getAdminServiceConfig500"; export * from "./getAdminServiceCredentials200"; export * from "./getAdminServiceCredentials200CredentialsItem"; export * from "./getAdminServiceCredentials200CredentialsItemManagedBy"; +export * from "./getAdminServiceCredentials200CredentialsItemOwnership"; +export * from "./getAdminServiceCredentials200CredentialsItemOwnershipAdoption"; +export * from "./getAdminServiceCredentials200CredentialsItemOwnershipManagedBy"; +export * from "./getAdminServiceCredentials200CredentialsItemOwnershipReason"; +export * from "./getAdminServiceCredentials200CredentialsItemOwnershipSource"; export * from "./getAdminServiceCredentials200CredentialsItemSetup"; export * from "./getAdminServiceCredentials200CredentialsItemSetupEmail"; export * from "./getAdminServiceCredentials200CredentialsItemSource"; diff --git a/packages/core/src/api/generated/admin/service-models/restoreAdminServiceBackup409.ts b/packages/core/src/api/generated/admin/service-models/restoreAdminServiceBackup409.ts index 2b6b66dab..4a00e23d2 100644 --- a/packages/core/src/api/generated/admin/service-models/restoreAdminServiceBackup409.ts +++ b/packages/core/src/api/generated/admin/service-models/restoreAdminServiceBackup409.ts @@ -17,4 +17,18 @@ export type RestoreAdminServiceBackup409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; }; diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts index 58860e71d..a0f3ae069 100644 --- a/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts +++ b/packages/core/src/api/generated/admin/service-models/setAdminServiceCredential409.ts @@ -3,7 +3,7 @@ export type SetAdminServiceCredential409 = | { - code: "DAWARICH_CREDENTIAL_MANAGED"; + code: "DAWARICH_CREDENTIAL_MANAGED" | "transit_credentials_managed"; error: string; } | { @@ -21,4 +21,18 @@ export type SetAdminServiceCredential409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; }; diff --git a/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts index 9379a8a6b..63d94a000 100644 --- a/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts +++ b/packages/core/src/api/generated/admin/service-models/setAdminServiceSelection409.ts @@ -17,4 +17,18 @@ export type SetAdminServiceSelection409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; }; diff --git a/packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts b/packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts index 86900a70c..2d1e5c78a 100644 --- a/packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts +++ b/packages/core/src/api/generated/admin/service-models/startAdminCompose409.ts @@ -17,4 +17,18 @@ export type StartAdminCompose409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; }; diff --git a/packages/core/src/api/generated/admin/service-models/streamAdminServiceLogs409.ts b/packages/core/src/api/generated/admin/service-models/streamAdminServiceLogs409.ts index edaf7b60b..ddcad1dfd 100644 --- a/packages/core/src/api/generated/admin/service-models/streamAdminServiceLogs409.ts +++ b/packages/core/src/api/generated/admin/service-models/streamAdminServiceLogs409.ts @@ -17,4 +17,18 @@ export type StreamAdminServiceLogs409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; }; diff --git a/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts index 53a8ce991..f6e4f118b 100644 --- a/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts +++ b/packages/core/src/api/generated/admin/service-models/updateAdminServiceConfig409.ts @@ -17,4 +17,18 @@ export type UpdateAdminServiceConfig409 = error: string; message: string; code?: string; + } + | { + code: "CONFIGURATION_SOURCE_OWNED"; + error: string; + ownership: { + source: "default" | "database" | "vault" | "config.json" | "env"; + writable: boolean; + secret: boolean; + adoption: "direct" | "host_action_required"; + reason: "host_environment" | "host_file" | "vault_only" | "managed_workflow" | null; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: "transit.credentials" | "dawarich-provisioning"; + }; }; From 633317c54184d192593d7656e03e49f67a2269f2 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 20:58:35 +0100 Subject: [PATCH 048/135] fix(admin): make transit source disabling reversible --- apps/api/openapi.json | 797 ++++++++++++++- .../__tests__/data-manager-sources.test.ts | 53 +- apps/api/src/routes/data-manager.ts | 25 + .../IntegrationAdminConsumers.test.tsx | 2 +- .../services/TransitSourcesSection.test.ts | 3 + .../admin/services/TransitSourcesSection.tsx | 135 ++- .../TransitSourcesSection.workflow.test.tsx | 91 +- .../administration/services-administration.md | 6 +- .../data-manager-admin/contracts.test.ts | 6 +- .../data-manager-admin/operations.ts | 17 +- .../admin/data-manager-admin.generated.ts | 123 +++ ...ApiDataManagerTransitSourcesSourceId409.ts | 5 + ...piDataManagerTransitSourcesSourceIdBody.ts | 2 + .../getApiDataManagerTransitSources200.ts | 2 + ...DataManagerTransitSources200SourcesItem.ts | 6 + ...agerTransitSources200SourcesItemActions.ts | 10 + ...ransitSources200SourcesItemDesiredState.ts | 11 + .../src/api/generated/admin/models/index.ts | 14 + .../postApiDataManagerTransitSources409.ts | 5 + .../postApiDataManagerTransitSourcesBody.ts | 2 + ...ManagerTransitSourcesSourceIdDisable202.ts | 9 + ...rTransitSourcesSourceIdDisable202Status.ts | 9 + ...ManagerTransitSourcesSourceIdDisable400.ts | 20 + ...ManagerTransitSourcesSourceIdDisable401.ts | 20 + ...ManagerTransitSourcesSourceIdDisable403.ts | 20 + ...ManagerTransitSourcesSourceIdDisable404.ts | 20 + ...ManagerTransitSourcesSourceIdDisable409.ts | 31 + ...ManagerTransitSourcesSourceIdDisable413.ts | 20 + ...ManagerTransitSourcesSourceIdDisable415.ts | 20 + ...ManagerTransitSourcesSourceIdDisable429.ts | 11 + ...ManagerTransitSourcesSourceIdDisable500.ts | 20 + ...anagerTransitSourcesSourceIdDisableBody.ts | 9 + ...aManagerTransitSourcesSourceIdEnable409.ts | 5 + ...ManagerTransitSourcesSourceIdEnableBody.ts | 2 + .../admin/operation-bindings.generated.ts | 9 + .../__fixtures__/reviewed-surface.json | 42 +- .../data-manager/contracts/contracts.test.ts | 4 +- .../data-manager/contracts/executions.ts | 18 +- .../data-manager/contracts/operations.ts | 9 +- .../data-manager/contracts/transit.ts | 38 +- .../generated/data-manager.generated.ts | 122 +++ .../generated/models/addTransitSource409.ts | 5 + .../generated/models/addTransitSourceBody.ts | 2 + .../models/disableTransitSource202.ts | 9 + .../models/disableTransitSource202Status.ts | 9 + .../models/disableTransitSource400.ts | 20 + .../models/disableTransitSource401.ts | 20 + .../models/disableTransitSource404.ts | 20 + .../models/disableTransitSource409.ts | 31 + .../models/disableTransitSource413.ts | 20 + .../models/disableTransitSource415.ts | 20 + .../models/disableTransitSource429.ts | 20 + .../models/disableTransitSource500.ts | 20 + .../models/disableTransitSource503.ts | 20 + .../models/disableTransitSourceBody.ts | 11 + .../models/enableTransitSource409.ts | 5 + .../models/enableTransitSourceBody.ts | 2 + .../generated/models/getTransitSources200.ts | 2 + .../models/getTransitSources200SourcesItem.ts | 6 + .../getTransitSources200SourcesItemActions.ts | 10 + ...ransitSources200SourcesItemDesiredState.ts | 11 + .../data-manager/generated/models/index.ts | 14 + .../models/removeTransitSource409.ts | 5 + .../models/removeTransitSourceBody.ts | 2 + .../models/startDataExecutionBodyInput.ts | 28 + .../generated/operation-bindings.generated.ts | 7 + .../src/services/data-manager/sdk.test.ts | 4 +- scripts/database-test-groups.json | 1 + services/data-manager/__tests__/api.test.ts | 58 +- .../__tests__/openapi-freshness.test.ts | 2 +- .../__tests__/transit-source-actions.test.ts | 204 ++++ .../__tests__/transit-sources.test.ts | 6 +- services/data-manager/openapi.json | 943 ++++++++++++++++++ .../data-manager/scripts/openapi/generate.ts | 9 +- services/data-manager/src/api.ts | 116 ++- .../src/jobs/execution/dispatch.ts | 7 + .../source-mutation-postgres.test.ts | 208 ++++ .../src/jobs/execution/source-mutation.ts | 40 + .../data-manager/src/jobs/execution/store.ts | 16 +- services/data-manager/src/transit-sources.ts | 222 ++++- 80 files changed, 3792 insertions(+), 136 deletions(-) create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemActions.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemDesiredState.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202Status.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable400.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable401.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable403.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable404.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable409.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable413.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable415.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable429.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable500.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisableBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource202.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource202Status.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource404.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSource503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/disableTransitSourceBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemActions.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemDesiredState.ts create mode 100644 services/data-manager/__tests__/transit-source-actions.test.ts create mode 100644 services/data-manager/src/jobs/execution/source-mutation-postgres.test.ts create mode 100644 services/data-manager/src/jobs/execution/source-mutation.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index e8cee00fd..f13f246b7 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -68167,6 +68167,10 @@ "schema": { "type": "object", "properties": { + "sourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "sources": { "type": "array", "items": { @@ -68207,6 +68211,45 @@ "requested": { "type": "boolean" }, + "sourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "desiredState": { + "type": "string", + "enum": [ + "enabled", + "disabled", + "removed" + ] + }, + "actions": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "disable": { + "type": "boolean" + }, + "remove": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "enable", + "disable", + "remove", + "reason" + ], + "additionalProperties": false + }, "active": { "type": "boolean" }, @@ -68277,6 +68320,9 @@ "origin", "license", "requested", + "sourceRevision", + "desiredState", + "actions", "active", "lifecycle" ], @@ -68297,6 +68343,7 @@ } }, "required": [ + "sourceRevision", "sources", "total", "limit", @@ -68835,6 +68882,10 @@ "schema": { "type": "object", "properties": { + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "idempotencyKey": { "type": "string", "maxLength": 200 @@ -68878,6 +68929,7 @@ } }, "required": [ + "expectedSourceRevision", "region", "name", "url", @@ -69247,6 +69299,32 @@ "existingJobId" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false } ] } @@ -69497,8 +69575,15 @@ "idempotencyKey": { "type": "string", "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" } }, + "required": [ + "expectedSourceRevision" + ], "additionalProperties": false } } @@ -69863,6 +69948,32 @@ "existingJobId" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false } ] } @@ -70089,9 +70200,9 @@ "x-openmapx-auth": "service" } }, - "/api/data-manager/transit/sources/{sourceId}/enable": { + "/api/data-manager/transit/sources/{sourceId}/disable": { "post": { - "operationId": "postApiDataManagerTransitSourcesSourceIdEnable", + "operationId": "postApiDataManagerTransitSourcesSourceIdDisable", "parameters": [ { "schema": { @@ -70113,8 +70224,15 @@ "idempotencyKey": { "type": "string", "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" } }, + "required": [ + "expectedSourceRevision" + ], "additionalProperties": false } } @@ -70479,6 +70597,681 @@ "existingJobId" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "service" + } + }, + "/api/data-manager/transit/sources/{sourceId}/enable": { + "post": { + "operationId": "postApiDataManagerTransitSourcesSourceIdEnable", + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "sourceId", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "expectedSourceRevision" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "sourceId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "started" + ] + } + }, + "required": [ + "jobId", + "sourceId", + "status" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] + }, + "existingJobId": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "ok", + "reason", + "existingJobId" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false } ] } diff --git a/apps/api/src/routes/__tests__/data-manager-sources.test.ts b/apps/api/src/routes/__tests__/data-manager-sources.test.ts index 97ab48fb4..593f4d62a 100644 --- a/apps/api/src/routes/__tests__/data-manager-sources.test.ts +++ b/apps/api/src/routes/__tests__/data-manager-sources.test.ts @@ -72,6 +72,7 @@ describe("transit source proxy", () => { url: "/data-manager/transit/sources", headers: { authorization: "Bearer service-token" }, payload: { + expectedSourceRevision: "a".repeat(64), region: "de", name: "demo", url: "https://example.test/demo.zip", @@ -89,6 +90,56 @@ describe("transit source proxy", () => { ); }); + it("forwards Disable as its own encoded mutation with the original revision and identity", async () => { + fetchMock.mockImplementation(async (url, init) => { + expect(String(url)).toBe( + "https://data-manager.test:4000/transit/sources/operator%3Ade%3Alocal/disable", + ); + expect(JSON.parse(init.body)).toEqual({ + expectedSourceRevision: "a".repeat(64), + idempotencyKey: "source-disable-fixture", + triggeredBy: "service-token", + }); + return Response.json( + { jobId: "disable-job", sourceId: "operator:de:local", status: "started" }, + { status: 202 }, + ); + }); + const result = await app.inject({ + method: "POST", + url: "/data-manager/transit/sources/operator%3Ade%3Alocal/disable", + headers: { authorization: "Bearer service-token" }, + payload: { expectedSourceRevision: "a".repeat(64), idempotencyKey: "source-disable-fixture" }, + }); + expect(result.statusCode, result.body).toBe(202); + expect(audit).toHaveBeenCalledWith( + expect.objectContaining({ + action: "transit.source.disable", + details: { jobId: "disable-job" }, + }), + ); + }); + it("preserves unavailable-action codes without a successful removal audit", async () => { + fetchMock.mockResolvedValue( + Response.json( + { + ok: false, + error: "Catalog sources cannot be removed", + code: "source_action_not_allowed", + }, + { status: 409 }, + ), + ); + const result = await app.inject({ + method: "DELETE", + url: "/data-manager/transit/sources/catalog%3Ade%3Alocal", + headers: { authorization: "Bearer service-token" }, + payload: { expectedSourceRevision: "a".repeat(64) }, + }); + expect(result.statusCode, result.body).toBe(409); + expect(result.json()).toMatchObject({ code: "source_action_not_allowed" }); + expect(audit).not.toHaveBeenCalled(); + }); it("propagates a conflict without writing a successful-mutation audit", async () => { fetchMock.mockResolvedValue( new Response( @@ -103,7 +154,7 @@ describe("transit source proxy", () => { method: "DELETE", url: "/data-manager/transit/sources/catalog%3Ade%3Avbb", headers: { authorization: "Bearer service-token" }, - payload: {}, + payload: { expectedSourceRevision: "a".repeat(64) }, }); expect(response.statusCode).toBe(409); expect(audit).not.toHaveBeenCalled(); diff --git a/apps/api/src/routes/data-manager.ts b/apps/api/src/routes/data-manager.ts index e1b0a7dba..2898307ed 100644 --- a/apps/api/src/routes/data-manager.ts +++ b/apps/api/src/routes/data-manager.ts @@ -382,6 +382,31 @@ export async function dataManagerRoute(app: FastifyInstance): Promise { return reply.code(result.status).send(result.data); }, ); + registerDataManagerAdminRoute( + app, + "postApiDataManagerTransitSourcesSourceIdDisable", + async (req, reply) => { + const auth = await authenticateDataManager(req); + const triggeredBy = auth.kind === "session" ? `user:${auth.userId}` : "service-token"; + const result = await dm.disableTransitSource( + req.params.sourceId, + { ...req.body, triggeredBy }, + dataManagerOptions(), + ); + if (result.status === 202) { + const body = result.data; + await writeAuditLog({ + actorId: auth.kind === "session" ? auth.userId : null, + targetId: req.params.sourceId, + targetType: "transit-source", + action: "transit.source.disable", + details: { jobId: body.jobId ?? null }, + request: req, + }); + } + return reply.code(result.status).send(result.data); + }, + ); registerDataManagerAdminRoute(app, "postApiDataManagerTransitSync", async (req, reply) => { const auth = await authenticateDataManager(req); diff --git a/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx b/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx index 78f830f85..e2029c5d2 100644 --- a/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx +++ b/apps/web/src/components/admin/integrations/__tests__/IntegrationAdminConsumers.test.tsx @@ -855,7 +855,7 @@ describe("integration administration browser behavior", () => { expect(screen.getByLabelText("Token")).toHaveValue(""); expect(screen.getByRole("button", { name: "Save Credential" })).toBeDisabled(); await userEvent.click(screen.getByRole("button", { name: "Done" })); - expect(close).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledTimes(1); }); it("retains the original revision and idempotency key for an unchanged rejected credential draft", async () => { const calls = transport(() => diff --git a/apps/web/src/components/admin/services/TransitSourcesSection.test.ts b/apps/web/src/components/admin/services/TransitSourcesSection.test.ts index 9af54469c..475ab7c40 100644 --- a/apps/web/src/components/admin/services/TransitSourcesSection.test.ts +++ b/apps/web/src/components/admin/services/TransitSourcesSection.test.ts @@ -3,6 +3,9 @@ import { sourceStateLabel, type TransitSource } from "./TransitSourcesSection"; function source(overrides: Partial): TransitSource { return { + sourceRevision: "a".repeat(64), + desiredState: "enabled", + actions: { enable: false, disable: true, remove: false, reason: null }, id: "de-test", region: "de", name: "Test", diff --git a/apps/web/src/components/admin/services/TransitSourcesSection.tsx b/apps/web/src/components/admin/services/TransitSourcesSection.tsx index 05f8d3444..680bdc8a5 100644 --- a/apps/web/src/components/admin/services/TransitSourcesSection.tsx +++ b/apps/web/src/components/admin/services/TransitSourcesSection.tsx @@ -26,9 +26,10 @@ import * as sdk from "@openmapx/core/api-admin-sdk"; import type { TransitSource } from "@openmapx/core/api-contracts"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import Link from "next/link"; -import { useMemo, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { formatBytes } from "@/lib/storageFormat"; import { AdminTablePagination } from "../shared/AdminTablePagination"; +import { ConfirmDialog } from "../shared/ConfirmDialog"; import { TableEmptyState } from "../shared/TableEmptyState"; import { TableSearchField, TableToolbar } from "../shared/TableToolbar"; import { useClientPagination, useTextFilter } from "../shared/tableHooks"; @@ -36,6 +37,9 @@ import { useClientPagination, useTextFilter } from "../shared/tableHooks"; export type { TransitSource }; export function sourceStateLabel(source: TransitSource): string { + if (source.desiredState === "removed") + return source.active ? "Removed · active pending sync" : "Removed"; + if (source.desiredState === "disabled" && source.active) return "Disabled · active pending sync"; if (source.lifecycle === "removal-pending") return "Removal pending · active"; if (source.lifecycle === "update-pending") return "Update pending · active"; if (source.lifecycle === "add-pending") return "Add pending · not active"; @@ -68,11 +72,27 @@ function formatDate(value: string): string { } export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { + return ; +} +function TransitSourcesView({ apiUrl }: { apiUrl: string }) { const queryClient = useQueryClient(); const client = useMemo( () => createApiClient({ baseUrl: apiUrl, credentials: "include" }), [apiUrl], ); + const [removing, setRemoving] = useState(null); + const identities = useRef(new Map()); + const operationId = (intent: unknown) => { + const fingerprint = JSON.stringify(intent); + let key = identities.current.get(fingerprint); + if (!key) { + key = crypto.randomUUID(); + if (identities.current.size >= 128) + identities.current.delete(identities.current.keys().next().value!); + identities.current.set(fingerprint, key); + } + return key; + }; const [toast, setToast] = useState(null); const [dialogOpen, setDialogOpen] = useState(false); const [url, setUrl] = useState(""); @@ -85,7 +105,7 @@ export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { const [lastJobId, setLastJobId] = useState(null); const sourcesQuery = useQuery({ - queryKey: ["admin", "transit", "sources"], + queryKey: ["admin", "transit", apiUrl, "sources"], queryFn: async ({ signal }) => sdk.unwrapAdminResponse( await sdk.getApiDataManagerTransitSources({ limit: 500 }, { client, signal }), @@ -93,13 +113,13 @@ export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { refetchInterval: 30_000, }); const stateQuery = useQuery({ - queryKey: ["admin", "transit", "state"], + queryKey: ["admin", "transit", apiUrl, "state"], queryFn: async ({ signal }) => sdk.unwrapAdminResponse(await sdk.getApiDataManagerTransitState({ client, signal })), refetchInterval: 10_000, }); const catalogQuery = useQuery({ - queryKey: ["admin", "transit", "catalog", catalogSearch], + queryKey: ["admin", "transit", apiUrl, "catalog", catalogSearch], enabled: dialogOpen && catalogSearch.trim().length >= 2, queryFn: async ({ signal }) => sdk.unwrapAdminResponse( @@ -113,13 +133,25 @@ export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { const reportJob = (jobId: string, message: string) => { setLastJobId(jobId); setToast(message); - void queryClient.invalidateQueries({ queryKey: ["admin", "transit"] }); + void queryClient.invalidateQueries({ queryKey: ["admin", "transit", apiUrl] }); }; const addMutation = useMutation({ + retry: false, mutationFn: async () => sdk.unwrapAdminResponse( await sdk.postApiDataManagerTransitSources( { + expectedSourceRevision: sourcesQuery.data!.sourceRevision, + idempotencyKey: operationId({ + action: "add", + url, + name, + region, + licenseSpdx, + licenseUrl, + attribution, + revision: sourcesQuery.data?.sourceRevision, + }), url: url.trim(), name: name.trim(), region: region.trim(), @@ -145,16 +177,36 @@ export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { onError: (error) => setToast(sdk.adminErrorMessage(error, error.message)), }); const sourceMutation = useMutation({ - mutationFn: async ({ sourceId, method }: { sourceId: string; method: "DELETE" | "POST" }) => - sdk.unwrapAdminResponse( - await (method === "DELETE" - ? sdk.deleteApiDataManagerTransitSourcesSourceId(sourceId, {}, { client }) - : sdk.postApiDataManagerTransitSourcesSourceIdEnable(sourceId, {}, { client })), - ), - onSuccess: ({ jobId, sourceId }) => reportJob(jobId, `${sourceId} queued as ${jobId}.`), + retry: false, + mutationFn: async ({ + sourceId, + action, + revision, + }: { + sourceId: string; + action: "enable" | "disable" | "remove"; + revision: string; + }) => { + const body = { + expectedSourceRevision: revision, + idempotencyKey: operationId({ sourceId, action, revision }), + }; + return sdk.unwrapAdminResponse( + await (action === "remove" + ? sdk.deleteApiDataManagerTransitSourcesSourceId(sourceId, body, { client }) + : action === "disable" + ? sdk.postApiDataManagerTransitSourcesSourceIdDisable(sourceId, body, { client }) + : sdk.postApiDataManagerTransitSourcesSourceIdEnable(sourceId, body, { client })), + ); + }, + onSuccess: ({ jobId, sourceId }) => { + setRemoving(null); + reportJob(jobId, `${sourceId} queued as ${jobId}.`); + }, onError: (error) => setToast(sdk.adminErrorMessage(error, error.message)), }); const syncMutation = useMutation({ + retry: false, mutationFn: async () => sdk.unwrapAdminResponse(await sdk.postApiDataManagerTransitSync({}, { client })), onSuccess: ({ jobId }) => reportJob(jobId, `Transit sync queued as ${jobId}.`), @@ -219,7 +271,7 @@ export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { - @@ -420,15 +486,19 @@ export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { {source.lastImportedAt ? formatDate(source.lastImportedAt) : "—"} - + + {source.actions.remove && ( + + )} )) @@ -444,6 +524,23 @@ export function TransitSourcesSection({ apiUrl }: { apiUrl: string }) { + setRemoving(null)} + onConfirm={() => { + if (removing && !sourceMutation.isPending) + sourceMutation.mutate({ + sourceId: removing.id, + action: "remove", + revision: removing.sourceRevision, + }); + }} + /> { }); function reads(input: string) { if (input.endsWith("/transit/state")) return Response.json(state); - return Response.json({ sources: [source], total: 1, limit: 500, offset: 0 }); + return Response.json({ + sourceRevision: "a".repeat(64), + sources: [source], + total: 1, + limit: 500, + offset: 0, + }); } describe("transit source administration workflow", () => { @@ -54,11 +63,14 @@ describe("transit source administration workflow", () => { const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, initValue) => { const init = initValue as RequestInit | undefined; expect(init?.credentials).toBe("include"); - if (init?.method === "DELETE") { + if (init?.method === "POST") { expect(String(input)).toBe( - "https://api.example.test/api/data-manager/transit/sources/catalog%3Ade%3Avbb", + "https://api.example.test/api/data-manager/transit/sources/catalog%3Ade%3Avbb/disable", ); - expect(init.body).toBe("{}"); + expect(JSON.parse(init.body as string)).toMatchObject({ + expectedSourceRevision: "a".repeat(64), + idempotencyKey: expect.any(String), + }); return Response.json( { jobId: "job-visible", sourceId: source.id, status: "started" }, { status: 202 }, @@ -71,14 +83,16 @@ describe("transit source administration workflow", () => { fireEvent.click(await screen.findByRole("button", { name: "Disable" })); expect(await screen.findByText("Source change queued as job-visible.")).toBeInTheDocument(); expect(await screen.findByText("catalog:de:vbb queued as job-visible.")).toBeInTheDocument(); - expect(invalidate).toHaveBeenCalledWith({ queryKey: ["admin", "transit"] }); + expect(invalidate).toHaveBeenCalledWith({ + queryKey: ["admin", "transit", "https://api.example.test"], + }); expect( - fetch.mock.calls.some(([, init]) => (init as RequestInit | undefined)?.method === "DELETE"), + fetch.mock.calls.some(([, init]) => (init as RequestInit | undefined)?.method === "POST"), ).toBe(true); }); it("does not report a queued job or successful invalidation for a conflict", async () => { vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => - (init as RequestInit | undefined)?.method === "DELETE" + (init as RequestInit | undefined)?.method === "POST" ? Response.json( { ok: false, reason: "in-flight", existingJobId: "running" }, { status: 409 }, @@ -92,6 +106,65 @@ describe("transit source administration workflow", () => { expect(screen.queryByText(/Source change queued as/)).not.toBeInTheDocument(); expect(invalidate).not.toHaveBeenCalled(); }); + it("confirms the exact operator source name before removing its saved metadata", async () => { + const row = { + ...source, + id: "operator:de:local", + name: "Local operator", + origin: "operator", + actions: { enable: false, disable: true, remove: true, reason: null }, + }, + mutations: RequestInit[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, initValue) => { + const init = initValue as RequestInit | undefined; + if (init?.method === "DELETE") { + mutations.push(init); + expect(JSON.parse(init.body as string).expectedSourceRevision).toBe("a".repeat(64)); + return Response.json( + { jobId: "remove-job", sourceId: row.id, status: "started" }, + { status: 202 }, + ); + } + if (String(input).endsWith("/transit/state")) return Response.json(state); + return Response.json({ + sourceRevision: "a".repeat(64), + sources: [row], + total: 1, + limit: 500, + offset: 0, + }); + }); + view(); + fireEvent.click(await screen.findByRole("button", { name: "Remove" })); + expect(await screen.findByText(/Remove Local operator\?/)).toBeInTheDocument(); + expect(mutations).toHaveLength(0); + fireEvent.click(screen.getByRole("button", { name: "Remove source" })); + await waitFor(() => expect(mutations).toHaveLength(1)); + }); + it("shows desired-disabled but still active after failure and trusts server action decisions", async () => { + const row = { + ...source, + requested: false, + desiredState: "disabled", + lifecycle: "removal-pending", + actions: { enable: true, disable: false, remove: false, reason: null }, + }; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => + String(input).endsWith("/transit/state") + ? Response.json(state) + : Response.json({ + sourceRevision: "a".repeat(64), + sources: [row], + total: 1, + limit: 500, + offset: 0, + }), + ); + view(); + expect(await screen.findByText("Disabled · active pending sync")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Enable" })).not.toBeDisabled(); + expect(screen.queryByRole("button", { name: "Remove" })).not.toBeInTheDocument(); + }); it("gates catalog search on the dialog and two trimmed characters while retaining its raw key", async () => { const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { if (String(input).includes("/transit/catalog")) { @@ -123,7 +196,9 @@ describe("transit source administration workflow", () => { fireEvent.change(search, { target: { value: " de " } }); expect(await screen.findByRole("button", { name: "Enable VBB (de)" })).toBeInTheDocument(); await waitFor(() => - expect(client.getQueryData(["admin", "transit", "catalog", " de "])).toMatchObject({ + expect( + client.getQueryData(["admin", "transit", "https://api.example.test", "catalog", " de "]), + ).toMatchObject({ total: 1, }), ); diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index f33792f7b..a687b4789 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -192,7 +192,11 @@ metadata tracked by the data-manager: candidate: the prior healthy MOTIS dataset stays live until the complete candidate passes validation and promotion. -Catalog sources can be disabled and later re-enabled. An operator source is +Catalog and operator sources can be disabled and later re-enabled. Disable +retains the saved URL, attribution and license. The separate, confirmed Remove +action is available only for operator sources and deletes their saved metadata; +enabling cannot restore a removed source. Catalog entries can be disabled but +cannot be removed from their pinned catalog. An operator source is defined by a region, a safe display name, its HTTPS URL, attribution text, and either an SPDX license identifier or license URL. Adding, disabling, enabling, or manually syncing a source queues an asynchronous job; the row links to the diff --git a/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts b/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts index 4c55fc831..cd47da419 100644 --- a/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts +++ b/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts @@ -9,10 +9,10 @@ import { } from "./index"; describe("application data-manager administration contracts", () => { - it("describes all 25 actual BFF operations with the real session-only distinctions", () => { - expect(Object.keys(dataManagerAdminOperations)).toHaveLength(25); + it("describes all 26 actual BFF operations with the real session-only distinctions", () => { + expect(Object.keys(dataManagerAdminOperations)).toHaveLength(26); const operations = Object.values(dataManagerAdminOperations); - expect(new Set(operations.map(({ method, path }) => `${method} ${path}`)).size).toBe(25); + expect(new Set(operations.map(({ method, path }) => `${method} ${path}`)).size).toBe(26); expect(operations.filter(({ auth }) => auth === "admin")).toHaveLength(11); expect(operations.every(({ path }) => path.startsWith("/api/"))).toBe(true); expect(dataManagerAdminOperations.getApiDataManagerTransitState.auth).toBe("service"); diff --git a/packages/core/src/api/contracts/data-manager-admin/operations.ts b/packages/core/src/api/contracts/data-manager-admin/operations.ts index af863d27a..1aa040372 100644 --- a/packages/core/src/api/contracts/data-manager-admin/operations.ts +++ b/packages/core/src/api/contracts/data-manager-admin/operations.ts @@ -18,6 +18,7 @@ import { transitConflictSchema, transitMutationRequestSchema, transitScopeRejectedSchema, + transitSourceActionErrorSchema, transitSourceListSchema, transitSourceMutationResultSchema, transitSourcesQuerySchema, @@ -52,10 +53,9 @@ const service = "service" as const; const admin = "admin" as const; const idParams = z.strictObject({ id: z.string().min(1) }); const sourceParams = z.strictObject({ sourceId: z.string().min(1) }); -export const adminSourceMutationRequestSchema = transitMutationRequestSchema - .unwrap() - .omit({ triggeredBy: true }) - .optional(); +export const adminSourceMutationRequestSchema = transitMutationRequestSchema.omit({ + triggeredBy: true, +}); export const adminAddTransitSourceRequestSchema = addTransitSourceRequestSchema.omit({ triggeredBy: true, }); @@ -70,7 +70,7 @@ const mutationResponses = { ...common, 202: json(transitSourceMutationResultSchema), 404: json(httpErrorSchema), - 409: json(z.union([httpErrorSchema, transitConflictSchema])), + 409: json(z.union([httpErrorSchema, transitConflictSchema, transitSourceActionErrorSchema])), }; const ambientAccepted = { ...common, @@ -162,6 +162,13 @@ export const dataManagerAdminOperations = { request: { params: sourceParams, body: adminSourceMutationRequestSchema }, responses: mutationResponses, }, + postApiDataManagerTransitSourcesSourceIdDisable: { + method: "POST", + path: "/api/data-manager/transit/sources/:sourceId/disable", + auth: service, + request: { params: sourceParams, body: adminSourceMutationRequestSchema }, + responses: mutationResponses, + }, postApiDataManagerTransitSync: { method: "POST", path: "/api/data-manager/transit/sync", diff --git a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts index 5383cb312..9e8f1fc9e 100644 --- a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts +++ b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts @@ -223,6 +223,17 @@ import type { PostApiDataManagerTransitSources429, PostApiDataManagerTransitSources500, PostApiDataManagerTransitSourcesBody, + PostApiDataManagerTransitSourcesSourceIdDisable202, + PostApiDataManagerTransitSourcesSourceIdDisable400, + PostApiDataManagerTransitSourcesSourceIdDisable401, + PostApiDataManagerTransitSourcesSourceIdDisable403, + PostApiDataManagerTransitSourcesSourceIdDisable404, + PostApiDataManagerTransitSourcesSourceIdDisable409, + PostApiDataManagerTransitSourcesSourceIdDisable413, + PostApiDataManagerTransitSourcesSourceIdDisable415, + PostApiDataManagerTransitSourcesSourceIdDisable429, + PostApiDataManagerTransitSourcesSourceIdDisable500, + PostApiDataManagerTransitSourcesSourceIdDisableBody, PostApiDataManagerTransitSourcesSourceIdEnable202, PostApiDataManagerTransitSourcesSourceIdEnable400, PostApiDataManagerTransitSourcesSourceIdEnable401, @@ -270,6 +281,7 @@ import { postApiDataManagerTransitBumpTransitousRefRequest, postApiDataManagerTransitRestartMotisRequest, postApiDataManagerTransitSourcesRequest, + postApiDataManagerTransitSourcesSourceIdDisableRequest, postApiDataManagerTransitSourcesSourceIdEnableRequest, postApiDataManagerTransitSyncRequest, } from "./operation-bindings.generated"; @@ -2322,6 +2334,117 @@ export const deleteApiDataManagerTransitSourcesSourceId = async ( ); }; +export type postApiDataManagerTransitSourcesSourceIdDisableResponse202 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable202; + status: 202; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse400 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable400; + status: 400; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse401 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable401; + status: 401; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse403 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable403; + status: 403; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse404 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable404; + status: 404; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse409 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable409; + status: 409; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse413 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable413; + status: 413; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse415 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable415; + status: 415; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse429 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable429; + status: 429; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse500 = { + data: PostApiDataManagerTransitSourcesSourceIdDisable500; + status: 500; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponseSuccess = + postApiDataManagerTransitSourcesSourceIdDisableResponse202 & { + headers: Headers; + }; +export type postApiDataManagerTransitSourcesSourceIdDisableResponseError = ( + | postApiDataManagerTransitSourcesSourceIdDisableResponse400 + | postApiDataManagerTransitSourcesSourceIdDisableResponse401 + | postApiDataManagerTransitSourcesSourceIdDisableResponse403 + | postApiDataManagerTransitSourcesSourceIdDisableResponse404 + | postApiDataManagerTransitSourcesSourceIdDisableResponse409 + | postApiDataManagerTransitSourcesSourceIdDisableResponse413 + | postApiDataManagerTransitSourcesSourceIdDisableResponse415 + | postApiDataManagerTransitSourcesSourceIdDisableResponse429 + | postApiDataManagerTransitSourcesSourceIdDisableResponse500 +) & { + headers: Headers; +}; + +export type postApiDataManagerTransitSourcesSourceIdDisableResponse = + | postApiDataManagerTransitSourcesSourceIdDisableResponseSuccess + | postApiDataManagerTransitSourcesSourceIdDisableResponseError; + +export const getPostApiDataManagerTransitSourcesSourceIdDisableUrl = (sourceId: string) => { + return `/api/data-manager/transit/sources/${encodeURIComponent(String(sourceId))}/disable`; +}; + +export const postApiDataManagerTransitSourcesSourceIdDisable = async ( + sourceId: string, + postApiDataManagerTransitSourcesSourceIdDisableBody: PostApiDataManagerTransitSourcesSourceIdDisableBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return postApiDataManagerTransitSourcesSourceIdDisableRequest( + getPostApiDataManagerTransitSourcesSourceIdDisableUrl(sourceId), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(postApiDataManagerTransitSourcesSourceIdDisableBody), + }, + ); +}; + export type postApiDataManagerTransitSourcesSourceIdEnableResponse202 = { data: PostApiDataManagerTransitSourcesSourceIdEnable202; status: 202; diff --git a/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceId409.ts b/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceId409.ts index 1836d1d8b..9bc3197ca 100644 --- a/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceId409.ts +++ b/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceId409.ts @@ -23,4 +23,9 @@ export type DeleteApiDataManagerTransitSourcesSourceId409 = reason: "in-flight" | "duplicate-idempotency-key"; /** @nullable */ existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; }; diff --git a/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceIdBody.ts b/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceIdBody.ts index 511cdcae4..f70e026c5 100644 --- a/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceIdBody.ts +++ b/packages/core/src/api/generated/admin/models/deleteApiDataManagerTransitSourcesSourceIdBody.ts @@ -4,4 +4,6 @@ export type DeleteApiDataManagerTransitSourcesSourceIdBody = { /** @maxLength 200 */ idempotencyKey?: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200.ts index 4a3739082..b0e3c1d3a 100644 --- a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200.ts +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200.ts @@ -3,6 +3,8 @@ import type { GetApiDataManagerTransitSources200SourcesItem } from "./getApiDataManagerTransitSources200SourcesItem"; export type GetApiDataManagerTransitSources200 = { + /** @pattern ^[a-f0-9]{64}$ */ + sourceRevision: string; sources: GetApiDataManagerTransitSources200SourcesItem[]; /** @minimum 0 */ total: number; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItem.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItem.ts index 02c6bc383..cff27287d 100644 --- a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItem.ts +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItem.ts @@ -1,6 +1,8 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetApiDataManagerTransitSources200SourcesItemActions } from "./getApiDataManagerTransitSources200SourcesItemActions"; import type { GetApiDataManagerTransitSources200SourcesItemArtifact } from "./getApiDataManagerTransitSources200SourcesItemArtifact"; +import type { GetApiDataManagerTransitSources200SourcesItemDesiredState } from "./getApiDataManagerTransitSources200SourcesItemDesiredState"; import type { GetApiDataManagerTransitSources200SourcesItemFormat } from "./getApiDataManagerTransitSources200SourcesItemFormat"; import type { GetApiDataManagerTransitSources200SourcesItemLicense } from "./getApiDataManagerTransitSources200SourcesItemLicense"; import type { GetApiDataManagerTransitSources200SourcesItemLifecycle } from "./getApiDataManagerTransitSources200SourcesItemLifecycle"; @@ -15,6 +17,10 @@ export type GetApiDataManagerTransitSources200SourcesItem = { originUrl?: string; license: GetApiDataManagerTransitSources200SourcesItemLicense; requested: boolean; + /** @pattern ^[a-f0-9]{64}$ */ + sourceRevision: string; + desiredState: GetApiDataManagerTransitSources200SourcesItemDesiredState; + actions: GetApiDataManagerTransitSources200SourcesItemActions; active: boolean; activeEpoch?: string; artifact?: GetApiDataManagerTransitSources200SourcesItemArtifact; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemActions.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemActions.ts new file mode 100644 index 000000000..7bab61c00 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemActions.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitSources200SourcesItemActions = { + enable: boolean; + disable: boolean; + remove: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemDesiredState.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemDesiredState.ts new file mode 100644 index 000000000..c3f4e0501 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSources200SourcesItemDesiredState.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitSources200SourcesItemDesiredState = + (typeof GetApiDataManagerTransitSources200SourcesItemDesiredState)[keyof typeof GetApiDataManagerTransitSources200SourcesItemDesiredState]; + +export const GetApiDataManagerTransitSources200SourcesItemDesiredState = { + enabled: "enabled", + disabled: "disabled", + removed: "removed", +} as const; diff --git a/packages/core/src/api/generated/admin/models/index.ts b/packages/core/src/api/generated/admin/models/index.ts index 802dd0d2b..2f70a6ed9 100644 --- a/packages/core/src/api/generated/admin/models/index.ts +++ b/packages/core/src/api/generated/admin/models/index.ts @@ -167,7 +167,9 @@ export * from "./getApiDataManagerTransitJobsId500"; export * from "./getApiDataManagerTransitJobsParams"; export * from "./getApiDataManagerTransitSources200"; export * from "./getApiDataManagerTransitSources200SourcesItem"; +export * from "./getApiDataManagerTransitSources200SourcesItemActions"; export * from "./getApiDataManagerTransitSources200SourcesItemArtifact"; +export * from "./getApiDataManagerTransitSources200SourcesItemDesiredState"; export * from "./getApiDataManagerTransitSources200SourcesItemFormat"; export * from "./getApiDataManagerTransitSources200SourcesItemLicense"; export * from "./getApiDataManagerTransitSources200SourcesItemLifecycle"; @@ -294,6 +296,18 @@ export * from "./postApiDataManagerTransitSources429"; export * from "./postApiDataManagerTransitSources500"; export * from "./postApiDataManagerTransitSourcesBody"; export * from "./postApiDataManagerTransitSourcesBodyLicense"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable202"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable202Status"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable400"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable401"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable403"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable404"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable409"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable413"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable415"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable429"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisable500"; +export * from "./postApiDataManagerTransitSourcesSourceIdDisableBody"; export * from "./postApiDataManagerTransitSourcesSourceIdEnable202"; export * from "./postApiDataManagerTransitSourcesSourceIdEnable202Status"; export * from "./postApiDataManagerTransitSourcesSourceIdEnable400"; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSources409.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSources409.ts index 20d47e719..8c1ce0917 100644 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSources409.ts +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSources409.ts @@ -23,4 +23,9 @@ export type PostApiDataManagerTransitSources409 = reason: "in-flight" | "duplicate-idempotency-key"; /** @nullable */ existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesBody.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesBody.ts index ed0a86e6e..fe475ab40 100644 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesBody.ts +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesBody.ts @@ -3,6 +3,8 @@ import type { PostApiDataManagerTransitSourcesBodyLicense } from "./postApiDataManagerTransitSourcesBodyLicense"; export type PostApiDataManagerTransitSourcesBody = { + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; /** @maxLength 200 */ idempotencyKey?: string; /** @minLength 1 */ diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202.ts new file mode 100644 index 000000000..486f618ae --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PostApiDataManagerTransitSourcesSourceIdDisable202Status } from "./postApiDataManagerTransitSourcesSourceIdDisable202Status"; + +export type PostApiDataManagerTransitSourcesSourceIdDisable202 = { + jobId: string; + sourceId: string; + status: PostApiDataManagerTransitSourcesSourceIdDisable202Status; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202Status.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202Status.ts new file mode 100644 index 000000000..433e469ac --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable202Status.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable202Status = + (typeof PostApiDataManagerTransitSourcesSourceIdDisable202Status)[keyof typeof PostApiDataManagerTransitSourcesSourceIdDisable202Status]; + +export const PostApiDataManagerTransitSourcesSourceIdDisable202Status = { + started: "started", +} as const; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable400.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable400.ts new file mode 100644 index 000000000..1d72c2bc0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable401.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable401.ts new file mode 100644 index 000000000..a61b267bb --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable403.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable403.ts new file mode 100644 index 000000000..c97dbfe8e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable404.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable404.ts new file mode 100644 index 000000000..271111a5d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable409.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable409.ts new file mode 100644 index 000000000..3b156089d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable409.ts @@ -0,0 +1,31 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + } + | { + ok: false; + reason: "in-flight" | "duplicate-idempotency-key"; + /** @nullable */ + existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable413.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable413.ts new file mode 100644 index 000000000..b0303a85d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable415.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable415.ts new file mode 100644 index 000000000..521174780 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable429.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable429.ts new file mode 100644 index 000000000..d7455929d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable500.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable500.ts new file mode 100644 index 000000000..995e288c2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisable500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisable500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisableBody.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisableBody.ts new file mode 100644 index 000000000..f7a5bdf3c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdDisableBody.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSourcesSourceIdDisableBody = { + /** @maxLength 200 */ + idempotencyKey?: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnable409.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnable409.ts index a3946689b..e4e274486 100644 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnable409.ts +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnable409.ts @@ -23,4 +23,9 @@ export type PostApiDataManagerTransitSourcesSourceIdEnable409 = reason: "in-flight" | "duplicate-idempotency-key"; /** @nullable */ existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnableBody.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnableBody.ts index f057604f6..37c0942dc 100644 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnableBody.ts +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSourcesSourceIdEnableBody.ts @@ -4,4 +4,6 @@ export type PostApiDataManagerTransitSourcesSourceIdEnableBody = { /** @maxLength 200 */ idempotencyKey?: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; }; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index 22374788c..0ed5d8f8d 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -539,6 +539,15 @@ export async function postApiDataManagerTransitSourcesRequest( ): Promise { return postApiDataManagerTransitSourcesBound(url, options); } +const postApiDataManagerTransitSourcesSourceIdDisableBound = bindAdminOperation( + "postApiDataManagerTransitSourcesSourceIdDisable", +); +export async function postApiDataManagerTransitSourcesSourceIdDisableRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return postApiDataManagerTransitSourcesSourceIdDisableBound(url, options); +} const postApiDataManagerTransitSourcesSourceIdEnableBound = bindAdminOperation( "postApiDataManagerTransitSourcesSourceIdEnable", ); diff --git a/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json b/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json index 6d33a0373..ecd1a028d 100644 --- a/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json +++ b/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json @@ -1,7 +1,7 @@ { - "revision": "815a24294c283fc829f3a372a1ae21618442cca0", - "explicitOperations": 48, - "note": "Static inventory. Expands two literal ambient actions and imported relay constant. Excludes Fastify automatic HEAD aliases; binary HEAD routes explicitly included. No route handler was invoked.", + "revision": "d721c425dded1da76ea26b80937b741307f939ce", + "explicitOperations": 57, + "note": "Static inventory. Expands two literal ambient actions and imported relay constant. Excludes Fastify automatic HEAD aliases; binary HEAD routes explicitly included. No route handler was invoked. Updated by explicit review for the eight previously installed durable execution/fence routes and the new reversible source-disable route; baseline revision precedes the latter.", "operations": [ { "method": "POST", @@ -194,6 +194,42 @@ { "method": "POST", "path": "/transit/sync" + }, + { + "method": "POST", + "path": "/executions" + }, + { + "method": "GET", + "path": "/executions/:id" + }, + { + "method": "GET", + "path": "/executions/by-key/:operationKey" + }, + { + "method": "POST", + "path": "/executions/:id/cancel" + }, + { + "method": "POST", + "path": "/executions/configuration-maintenance/prepare" + }, + { + "method": "GET", + "path": "/executions/configuration-maintenance/:maintenanceId" + }, + { + "method": "POST", + "path": "/executions/configuration-maintenance/:maintenanceId/settle" + }, + { + "method": "POST", + "path": "/executions/generation-hardlinks" + }, + { + "method": "POST", + "path": "/transit/sources/:sourceId/disable" } ] } diff --git a/packages/core/src/services/data-manager/contracts/contracts.test.ts b/packages/core/src/services/data-manager/contracts/contracts.test.ts index 97b3aeeab..6348d4a00 100644 --- a/packages/core/src/services/data-manager/contracts/contracts.test.ts +++ b/packages/core/src/services/data-manager/contracts/contracts.test.ts @@ -12,8 +12,8 @@ import { } from "./index"; describe("data-manager wire contracts", () => { - it("owns all 48 explicit operations with two named glyph HEAD/GET contracts", () => { - expect(Object.keys(dataManagerOperations)).toHaveLength(48); + it("owns all 57 explicit operations with two named glyph HEAD/GET contracts", () => { + expect(Object.keys(dataManagerOperations)).toHaveLength(57); expect(dataManagerOperations.getOfflineGlyph.path).toBe( "/offline/packages/glyphs/:version/:fontstack/:range.pbf", ); diff --git a/packages/core/src/services/data-manager/contracts/executions.ts b/packages/core/src/services/data-manager/contracts/executions.ts index cce6f7138..f94f66911 100644 --- a/packages/core/src/services/data-manager/contracts/executions.ts +++ b/packages/core/src/services/data-manager/contracts/executions.ts @@ -4,6 +4,7 @@ import { processDiagnosticsSchema } from "../../../ops/job-evidence"; import { hardlinkApplySummarySchema } from "../../../ops/service-results"; import { regionSchema } from "./common"; import { hardlinkEntrySchema } from "./datasets"; +import { transitDesiredSourceMutationSchema } from "./transit"; export const dataOperationKeySchema = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/); export const dataFingerprintSchema = z.string().regex(/^[a-f0-9]{64}$/); @@ -17,7 +18,11 @@ const ordinaryInputs = [ regional("download-osm"), z.strictObject({ kind: z.literal("download-fonts") }), regional("convert-overpass"), - z.strictObject({ kind: z.literal("transit-sync"), countries }), + z.strictObject({ + kind: z.literal("transit-sync"), + countries, + sourceMutation: transitDesiredSourceMutationSchema.optional(), + }), regional("overture-pull"), regional("overture-ingest"), regional("overture-extract"), @@ -52,6 +57,17 @@ export const dataExecutionInputSchema = z.union([ export type DataExecutionInput = z.output; export function normalizeDataExecutionInput(value: unknown): DataExecutionInput { const input = dataExecutionInputSchema.parse(value); + if ( + input.kind === "transit-sync" && + input.sourceMutation && + new TextEncoder().encode(JSON.stringify(input.sourceMutation)).byteLength > 16 * 1024 + ) + throw Object.assign( + new Error( + "Transit source metadata is too large (maximum 16 KiB). Use a license URL and shorter attribution.", + ), + { statusCode: 413 }, + ); return input.kind === "transit-sync" ? { ...input, diff --git a/packages/core/src/services/data-manager/contracts/operations.ts b/packages/core/src/services/data-manager/contracts/operations.ts index d6611e9c7..fc7ec6613 100644 --- a/packages/core/src/services/data-manager/contracts/operations.ts +++ b/packages/core/src/services/data-manager/contracts/operations.ts @@ -34,7 +34,7 @@ const sourceParams = z.strictObject({ sourceId: z.string().min(1) }); const sourceMutationErrors = { ...dataMutationErrors, 404: json(httpErrorSchema), - 409: json(z.union([httpErrorSchema, t.transitConflictSchema])), + 409: json(z.union([httpErrorSchema, t.transitConflictSchema, t.transitSourceActionErrorSchema])), }; const offlineErrors = { ...dataMutationErrors, @@ -378,6 +378,13 @@ export const dataManagerOperations = { request: { params: sourceParams, body: t.transitMutationRequestSchema }, responses: { ...sourceMutationErrors, 202: json(t.transitSourceMutationResultSchema) }, }, + disableTransitSource: { + method: "POST", + path: "/transit/sources/:sourceId/disable", + auth: service, + request: { params: sourceParams, body: t.transitMutationRequestSchema }, + responses: { ...sourceMutationErrors, 202: json(t.transitSourceMutationResultSchema) }, + }, preflightTransit: { method: "POST", path: "/transit/preflight", diff --git a/packages/core/src/services/data-manager/contracts/transit.ts b/packages/core/src/services/data-manager/contracts/transit.ts index 22fe59092..e6e032750 100644 --- a/packages/core/src/services/data-manager/contracts/transit.ts +++ b/packages/core/src/services/data-manager/contracts/transit.ts @@ -26,6 +26,14 @@ export const transitCatalogSourceSchema = z .omit({ origin: true, requested: true }); export const transitSourceRowSchema = z.strictObject({ ...sourceShape, + sourceRevision: z.string().regex(/^[a-f0-9]{64}$/), + desiredState: z.enum(["enabled", "disabled", "removed"]), + actions: z.strictObject({ + enable: z.boolean(), + disable: z.boolean(), + remove: z.boolean(), + reason: z.string().nullable(), + }), active: z.boolean(), activeEpoch: z.string().optional(), artifact: z @@ -43,6 +51,7 @@ export const transitSourceRowSchema = z.strictObject({ lifecycle: transitSourceLifecycleSchema, }); export const transitSourceListSchema = z.strictObject({ + sourceRevision: z.string().regex(/^[a-f0-9]{64}$/), sources: z.array(transitSourceRowSchema), total: countSchema, limit: countSchema, @@ -84,8 +93,13 @@ const mutationShape = { idempotencyKey: z.string().max(200).optional(), triggeredBy: z.string().max(200).optional(), }; -export const transitMutationRequestSchema = z.strictObject(mutationShape).optional(); +const expectedSourceRevision = z.string().regex(/^[a-f0-9]{64}$/); +export const transitMutationRequestSchema = z.strictObject({ + ...mutationShape, + expectedSourceRevision, +}); export const addTransitSourceRequestSchema = z.strictObject({ + expectedSourceRevision, ...mutationShape, region: z.string().min(1), name: z.string().min(1), @@ -98,11 +112,33 @@ export const addTransitSourceRequestSchema = z.strictObject({ publisherUrl: z.string().optional(), }), }); +export const transitDesiredSourceMutationSchema = z.discriminatedUnion("action", [ + z.strictObject({ + action: z.literal("add"), + expectedSourceRevision: expectedSourceRevision, + source: addTransitSourceRequestSchema.omit({ + idempotencyKey: true, + triggeredBy: true, + expectedSourceRevision: true, + }), + }), + z.strictObject({ + action: z.enum(["enable", "disable", "remove"]), + sourceId: z.string().min(1), + expectedSourceRevision: expectedSourceRevision, + }), +]); +export type TransitDesiredSourceMutation = z.infer; export const transitSourceMutationResultSchema = z.strictObject({ jobId: z.string(), sourceId: z.string(), status: z.literal("started"), }); +export const transitSourceActionErrorSchema = z.strictObject({ + ok: z.literal(false), + error: z.string(), + code: z.literal("source_action_not_allowed"), +}); export const transitConflictSchema = z.strictObject({ ok: z.literal(false), reason: z.enum(["in-flight", "duplicate-idempotency-key"]), diff --git a/packages/core/src/services/data-manager/generated/data-manager.generated.ts b/packages/core/src/services/data-manager/generated/data-manager.generated.ts index b0ea8f53e..663c97287 100644 --- a/packages/core/src/services/data-manager/generated/data-manager.generated.ts +++ b/packages/core/src/services/data-manager/generated/data-manager.generated.ts @@ -94,6 +94,17 @@ import type { ConvertOverpass500, ConvertOverpass503, ConvertOverpassBody, + DisableTransitSource202, + DisableTransitSource400, + DisableTransitSource401, + DisableTransitSource404, + DisableTransitSource409, + DisableTransitSource413, + DisableTransitSource415, + DisableTransitSource429, + DisableTransitSource500, + DisableTransitSource503, + DisableTransitSourceBody, DiscardAmbientPlaces200, DiscardAmbientPlaces400, DiscardAmbientPlaces401, @@ -529,6 +540,7 @@ import { cancelDataExecutionRequest, conflateOvertureRequest, convertOverpassRequest, + disableTransitSourceRequest, discardAmbientPlacesRequest, downloadFontsRequest, downloadOsmRequest, @@ -3679,6 +3691,116 @@ export const enableTransitSource = async ( ); }; +export type disableTransitSourceResponse202 = { + data: DisableTransitSource202; + status: 202; +}; + +export type disableTransitSourceResponse400 = { + data: DisableTransitSource400; + status: 400; +}; + +export type disableTransitSourceResponse401 = { + data: DisableTransitSource401; + status: 401; +}; + +export type disableTransitSourceResponse404 = { + data: DisableTransitSource404; + status: 404; +}; + +export type disableTransitSourceResponse409 = { + data: DisableTransitSource409; + status: 409; +}; + +export type disableTransitSourceResponse413 = { + data: DisableTransitSource413; + status: 413; +}; + +export type disableTransitSourceResponse415 = { + data: DisableTransitSource415; + status: 415; +}; + +export type disableTransitSourceResponse429 = { + data: DisableTransitSource429; + status: 429; +}; + +export type disableTransitSourceResponse500 = { + data: DisableTransitSource500; + status: 500; +}; + +export type disableTransitSourceResponse503 = { + data: DisableTransitSource503; + status: 503; +}; + +export type disableTransitSourceResponseSuccess = disableTransitSourceResponse202 & { + headers: Headers; +}; +export type disableTransitSourceResponseError = ( + | disableTransitSourceResponse400 + | disableTransitSourceResponse401 + | disableTransitSourceResponse404 + | disableTransitSourceResponse409 + | disableTransitSourceResponse413 + | disableTransitSourceResponse415 + | disableTransitSourceResponse429 + | disableTransitSourceResponse500 + | disableTransitSourceResponse503 +) & { + headers: Headers; +}; + +export type disableTransitSourceResponse = + | disableTransitSourceResponseSuccess + | disableTransitSourceResponseError; + +export const getDisableTransitSourceUrl = (sourceId: string) => { + return `/transit/sources/${encodeURIComponent(String(sourceId))}/disable`; +}; + +export const disableTransitSource = async ( + sourceId: string, + disableTransitSourceBody: DisableTransitSourceBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return disableTransitSourceRequest( + getDisableTransitSourceUrl(sourceId), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(disableTransitSourceBody), + }, + ); +}; + export type preflightTransitResponse200 = { data: PreflightTransit200; status: 200; diff --git a/packages/core/src/services/data-manager/generated/models/addTransitSource409.ts b/packages/core/src/services/data-manager/generated/models/addTransitSource409.ts index 54d13fc33..e06e69cce 100644 --- a/packages/core/src/services/data-manager/generated/models/addTransitSource409.ts +++ b/packages/core/src/services/data-manager/generated/models/addTransitSource409.ts @@ -23,4 +23,9 @@ export type AddTransitSource409 = reason: "in-flight" | "duplicate-idempotency-key"; /** @nullable */ existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; }; diff --git a/packages/core/src/services/data-manager/generated/models/addTransitSourceBody.ts b/packages/core/src/services/data-manager/generated/models/addTransitSourceBody.ts index ac2280b6d..b18686894 100644 --- a/packages/core/src/services/data-manager/generated/models/addTransitSourceBody.ts +++ b/packages/core/src/services/data-manager/generated/models/addTransitSourceBody.ts @@ -3,6 +3,8 @@ import type { AddTransitSourceBodyLicense } from "./addTransitSourceBodyLicense"; export type AddTransitSourceBody = { + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; /** @maxLength 200 */ idempotencyKey?: string; /** @maxLength 200 */ diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource202.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource202.ts new file mode 100644 index 000000000..fcda2c805 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource202.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { DisableTransitSource202Status } from "./disableTransitSource202Status"; + +export type DisableTransitSource202 = { + jobId: string; + sourceId: string; + status: DisableTransitSource202Status; +}; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource202Status.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource202Status.ts new file mode 100644 index 000000000..e2d855cb6 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource202Status.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource202Status = + (typeof DisableTransitSource202Status)[keyof typeof DisableTransitSource202Status]; + +export const DisableTransitSource202Status = { + started: "started", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource400.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource400.ts new file mode 100644 index 000000000..9384115e1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource401.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource401.ts new file mode 100644 index 000000000..4c922b224 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource404.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource404.ts new file mode 100644 index 000000000..3a34e8e47 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource409.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource409.ts new file mode 100644 index 000000000..417eba094 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource409.ts @@ -0,0 +1,31 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + } + | { + ok: false; + reason: "in-flight" | "duplicate-idempotency-key"; + /** @nullable */ + existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource413.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource413.ts new file mode 100644 index 000000000..d94176ec5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource415.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource415.ts new file mode 100644 index 000000000..9ea53f281 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource429.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource429.ts new file mode 100644 index 000000000..140e06e84 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource500.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource500.ts new file mode 100644 index 000000000..a76fb1fcf --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSource503.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSource503.ts new file mode 100644 index 000000000..cff9d4016 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSource503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSource503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/disableTransitSourceBody.ts b/packages/core/src/services/data-manager/generated/models/disableTransitSourceBody.ts new file mode 100644 index 000000000..757f3d996 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/disableTransitSourceBody.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DisableTransitSourceBody = { + /** @maxLength 200 */ + idempotencyKey?: string; + /** @maxLength 200 */ + triggeredBy?: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/enableTransitSource409.ts b/packages/core/src/services/data-manager/generated/models/enableTransitSource409.ts index 1b00d6236..c6025fdeb 100644 --- a/packages/core/src/services/data-manager/generated/models/enableTransitSource409.ts +++ b/packages/core/src/services/data-manager/generated/models/enableTransitSource409.ts @@ -23,4 +23,9 @@ export type EnableTransitSource409 = reason: "in-flight" | "duplicate-idempotency-key"; /** @nullable */ existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; }; diff --git a/packages/core/src/services/data-manager/generated/models/enableTransitSourceBody.ts b/packages/core/src/services/data-manager/generated/models/enableTransitSourceBody.ts index 32b3807a7..aa0325964 100644 --- a/packages/core/src/services/data-manager/generated/models/enableTransitSourceBody.ts +++ b/packages/core/src/services/data-manager/generated/models/enableTransitSourceBody.ts @@ -6,4 +6,6 @@ export type EnableTransitSourceBody = { idempotencyKey?: string; /** @maxLength 200 */ triggeredBy?: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; }; diff --git a/packages/core/src/services/data-manager/generated/models/getTransitSources200.ts b/packages/core/src/services/data-manager/generated/models/getTransitSources200.ts index 7e8de9c13..c57880025 100644 --- a/packages/core/src/services/data-manager/generated/models/getTransitSources200.ts +++ b/packages/core/src/services/data-manager/generated/models/getTransitSources200.ts @@ -3,6 +3,8 @@ import type { GetTransitSources200SourcesItem } from "./getTransitSources200SourcesItem"; export type GetTransitSources200 = { + /** @pattern ^[a-f0-9]{64}$ */ + sourceRevision: string; sources: GetTransitSources200SourcesItem[]; /** @minimum 0 */ total: number; diff --git a/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItem.ts b/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItem.ts index 5e7f9ef4d..71239c705 100644 --- a/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItem.ts +++ b/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItem.ts @@ -1,6 +1,8 @@ // biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. // Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { GetTransitSources200SourcesItemActions } from "./getTransitSources200SourcesItemActions"; import type { GetTransitSources200SourcesItemArtifact } from "./getTransitSources200SourcesItemArtifact"; +import type { GetTransitSources200SourcesItemDesiredState } from "./getTransitSources200SourcesItemDesiredState"; import type { GetTransitSources200SourcesItemFormat } from "./getTransitSources200SourcesItemFormat"; import type { GetTransitSources200SourcesItemLicense } from "./getTransitSources200SourcesItemLicense"; import type { GetTransitSources200SourcesItemLifecycle } from "./getTransitSources200SourcesItemLifecycle"; @@ -15,6 +17,10 @@ export type GetTransitSources200SourcesItem = { originUrl?: string; license: GetTransitSources200SourcesItemLicense; requested: boolean; + /** @pattern ^[a-f0-9]{64}$ */ + sourceRevision: string; + desiredState: GetTransitSources200SourcesItemDesiredState; + actions: GetTransitSources200SourcesItemActions; active: boolean; activeEpoch?: string; artifact?: GetTransitSources200SourcesItemArtifact; diff --git a/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemActions.ts b/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemActions.ts new file mode 100644 index 000000000..1e29c0531 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemActions.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetTransitSources200SourcesItemActions = { + enable: boolean; + disable: boolean; + remove: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemDesiredState.ts b/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemDesiredState.ts new file mode 100644 index 000000000..aef8c63d7 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getTransitSources200SourcesItemDesiredState.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetTransitSources200SourcesItemDesiredState = + (typeof GetTransitSources200SourcesItemDesiredState)[keyof typeof GetTransitSources200SourcesItemDesiredState]; + +export const GetTransitSources200SourcesItemDesiredState = { + enabled: "enabled", + disabled: "disabled", + removed: "removed", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/index.ts b/packages/core/src/services/data-manager/generated/models/index.ts index 566363d01..a810df769 100644 --- a/packages/core/src/services/data-manager/generated/models/index.ts +++ b/packages/core/src/services/data-manager/generated/models/index.ts @@ -107,6 +107,18 @@ export * from "./convertOverpass503"; export * from "./convertOverpassBody"; export * from "./dataManagerGetTransitCatalogResponse200Schema0"; export * from "./dataManagerGetTransitSourcesResponse200Schema0"; +export * from "./disableTransitSource202"; +export * from "./disableTransitSource202Status"; +export * from "./disableTransitSource400"; +export * from "./disableTransitSource401"; +export * from "./disableTransitSource404"; +export * from "./disableTransitSource409"; +export * from "./disableTransitSource413"; +export * from "./disableTransitSource415"; +export * from "./disableTransitSource429"; +export * from "./disableTransitSource500"; +export * from "./disableTransitSource503"; +export * from "./disableTransitSourceBody"; export * from "./discardAmbientPlaces200"; export * from "./discardAmbientPlaces400"; export * from "./discardAmbientPlaces401"; @@ -423,7 +435,9 @@ export * from "./getTransitProfile429"; export * from "./getTransitProfile500"; export * from "./getTransitSources200"; export * from "./getTransitSources200SourcesItem"; +export * from "./getTransitSources200SourcesItemActions"; export * from "./getTransitSources200SourcesItemArtifact"; +export * from "./getTransitSources200SourcesItemDesiredState"; export * from "./getTransitSources200SourcesItemFormat"; export * from "./getTransitSources200SourcesItemLicense"; export * from "./getTransitSources200SourcesItemLifecycle"; diff --git a/packages/core/src/services/data-manager/generated/models/removeTransitSource409.ts b/packages/core/src/services/data-manager/generated/models/removeTransitSource409.ts index d6225ee8a..01a43e18f 100644 --- a/packages/core/src/services/data-manager/generated/models/removeTransitSource409.ts +++ b/packages/core/src/services/data-manager/generated/models/removeTransitSource409.ts @@ -23,4 +23,9 @@ export type RemoveTransitSource409 = reason: "in-flight" | "duplicate-idempotency-key"; /** @nullable */ existingJobId: string | null; + } + | { + ok: false; + error: string; + code: "source_action_not_allowed"; }; diff --git a/packages/core/src/services/data-manager/generated/models/removeTransitSourceBody.ts b/packages/core/src/services/data-manager/generated/models/removeTransitSourceBody.ts index 3ec6db253..72525e0ca 100644 --- a/packages/core/src/services/data-manager/generated/models/removeTransitSourceBody.ts +++ b/packages/core/src/services/data-manager/generated/models/removeTransitSourceBody.ts @@ -6,4 +6,6 @@ export type RemoveTransitSourceBody = { idempotencyKey?: string; /** @maxLength 200 */ triggeredBy?: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts index d42f11e8f..22ed0d142 100644 --- a/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts +++ b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts @@ -30,6 +30,34 @@ export type StartDataExecutionBodyInput = * @items.pattern ^[A-Za-z]{2,3}$ */ countries: string[]; + sourceMutation?: + | { + action: "add"; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; + source: { + /** @minLength 1 */ + region: string; + /** @minLength 1 */ + name: string; + url: string; + license: { + spdxIdentifier?: string; + url?: string; + /** @minLength 1 */ + attribution: string; + publisher?: string; + publisherUrl?: string; + }; + }; + } + | { + action: "enable" | "disable" | "remove"; + /** @minLength 1 */ + sourceId: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSourceRevision: string; + }; } | { kind: "overture-pull"; diff --git a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts index 6ed4cfd27..51ebf6e5b 100644 --- a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts +++ b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts @@ -65,6 +65,13 @@ export async function convertOverpassRequest( ): Promise { return convertOverpassBound(url, options); } +const disableTransitSourceBound = bindDataManagerOperation("disableTransitSource"); +export async function disableTransitSourceRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return disableTransitSourceBound(url, options); +} const discardAmbientPlacesBound = bindDataManagerOperation("discardAmbientPlaces"); export async function discardAmbientPlacesRequest( url: string, diff --git a/packages/core/src/services/data-manager/sdk.test.ts b/packages/core/src/services/data-manager/sdk.test.ts index 37defc8a8..d938edbff 100644 --- a/packages/core/src/services/data-manager/sdk.test.ts +++ b/packages/core/src/services/data-manager/sdk.test.ts @@ -19,8 +19,8 @@ describe("generated complete internal data-manager SDK", () => { Extract>, { status: 416 }>["data"] >().toEqualTypeOf(); }); - it("has a generated callable for all 48 explicit operations", () => { - expect(Object.keys(dataManagerOperations)).toHaveLength(48); + it("has a generated callable for all 57 explicit operations", () => { + expect(Object.keys(dataManagerOperations)).toHaveLength(57); for (const key of Object.keys(dataManagerOperations)) expect(sdk).toHaveProperty(key, expect.any(Function)); }); diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index b07680606..22cf93f88 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -41,6 +41,7 @@ "services/data-manager/src/jobs/execution/store-postgres.test.ts", "services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts", "services/data-manager/src/jobs/execution/runtime.test.ts", + "services/data-manager/src/jobs/execution/source-mutation-postgres.test.ts", "services/data-manager/src/jobs/execution/fixed-cli.test.ts", "services/data-manager/src/jobs/execution/full-update.test.ts", "services/data-manager/src/jobs/execution/runner-cancellation.test.ts", diff --git a/services/data-manager/__tests__/api.test.ts b/services/data-manager/__tests__/api.test.ts index 52c4c6d12..7a9dc7dd8 100644 --- a/services/data-manager/__tests__/api.test.ts +++ b/services/data-manager/__tests__/api.test.ts @@ -1,3 +1,5 @@ +import { transitSourceRevision } from "../src/transit-sources.js"; + vi.mock("../src/jobs/execution/legacy-mutation.js", () => ({ beginLegacyCatalogPreflight: async () => {}, rejectLegacyMissingCatalog: async () => {}, @@ -16,7 +18,7 @@ vi.mock("../src/jobs/persistence.js", async () => ({ finalizeJobRow: vi.fn(async () => {}), })); -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { SafeDownloadOptions } from "@openmapx/core/utils/safe-download"; @@ -422,6 +424,10 @@ describe("data-manager API", () => { payload: { region: "de", name: "operator-feed", + expectedSourceRevision: transitSourceRevision({ + catalogDir: join(dataDir, ".transitous-catalog"), + overlayPath: join(dataDir, "overrides/feeds-overlay.json"), + }), url: "https://operator.example/feed.zip", license: { spdxIdentifier: "CC-BY-4.0", attribution: "Operator" }, }, @@ -430,6 +436,37 @@ describe("data-manager API", () => { expect(existsSync(overlayPath)).toBe(false); }); + it("rejects oversized source metadata before reserving a job or admitting work", async () => { + const dataDir = createDataDir(); + sourceCatalog(dataDir); + const singleFlight: SingleFlightController = { + tryStartSync: vi.fn(), + reserveOwned: vi.fn(() => true), + markSyncFinished: vi.fn(), + getInflight: () => null, + }, + admitLocal = vi.fn(); + const app = createTestApp(); + registerApi(app, { dataDir, singleFlight, executionReceiver: receiverFixture({ admitLocal }) }); + const response = await app.inject({ + method: "POST", + url: "/transit/sources", + payload: { + region: "de", + name: "bounded", + url: "https://operator.example/feed.zip", + license: { spdxIdentifier: "CC0-1.0", attribution: "ü".repeat(8192) }, + expectedSourceRevision: transitSourceRevision({ + catalogDir: join(dataDir, ".transitous-catalog"), + overlayPath: join(dataDir, "overrides/feeds-overlay.json"), + }), + }, + }); + expect(response.statusCode, response.body).toBe(413); + expect(singleFlight.tryStartSync).not.toHaveBeenCalled(); + expect(admitLocal).not.toHaveBeenCalled(); + expect(existsSync(join(dataDir, "overrides/feeds-overlay.json"))).toBe(false); + }); it("returns a reserved visible job before launching an accepted source mutation", async () => { const dataDir = createDataDir(); sourceCatalog(dataDir); @@ -455,6 +492,10 @@ describe("data-manager API", () => { payload: { region: "de", name: "operator-feed", + expectedSourceRevision: transitSourceRevision({ + catalogDir: join(dataDir, ".transitous-catalog"), + overlayPath: join(dataDir, "overrides/feeds-overlay.json"), + }), url: "https://operator.example/feed.zip", license: { spdxIdentifier: "CC-BY-4.0", attribution: "Operator" }, }, @@ -466,10 +507,17 @@ describe("data-manager API", () => { status: "started", }); expect(events).toEqual(["job-inserted", "durable-admitted"]); - const overlay = JSON.parse( - readFileSync(join(dataDir, "overrides", "feeds-overlay.json"), "utf-8"), - ) as { version: number }; - expect(overlay.version).toBe(3); + expect(existsSync(join(dataDir, "overrides/feeds-overlay.json"))).toBe(false); + expect(admitLocal).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "transit-sync", + sourceMutation: expect.objectContaining({ + action: "add", + source: expect.objectContaining({ name: "operator-feed" }), + }), + }), + expect.objectContaining({ sourceJobId: "job-visible" }), + ); }); it.each([ diff --git a/services/data-manager/__tests__/openapi-freshness.test.ts b/services/data-manager/__tests__/openapi-freshness.test.ts index 84caab56a..2cf90330d 100644 --- a/services/data-manager/__tests__/openapi-freshness.test.ts +++ b/services/data-manager/__tests__/openapi-freshness.test.ts @@ -8,7 +8,7 @@ it("exports every actual data-manager route with stable IDs, auth and payload co expect(generated).toBe(await readFile(DOCUMENT_PATH, "utf8")); const document = JSON.parse(generated); const operations = Object.values(document.paths).flatMap((path) => Object.values(path as object)); - expect(operations).toHaveLength(48); + expect(operations).toHaveLength(57); for (const value of operations) { const operation = value as { operationId: keyof typeof dataManagerOperations; diff --git a/services/data-manager/__tests__/transit-source-actions.test.ts b/services/data-manager/__tests__/transit-source-actions.test.ts new file mode 100644 index 000000000..a06ac0414 --- /dev/null +++ b/services/data-manager/__tests__/transit-source-actions.test.ts @@ -0,0 +1,204 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { readFeedOverlay, writeFeedOverlayAtomic } from "../src/jobs/transitous-feeds-overlay.js"; +import { + listTransitSources, + prepareEnableTransitSource, + prepareRemoveTransitSource, +} from "../src/transit-sources.js"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function fixture() { + const dataDir = mkdtempSync(join(tmpdir(), "openmapx-source-actions-")); + roots.push(dataDir); + const catalogDir = join(dataDir, "catalog"), + overlayPath = join(dataDir, "overlay.json"); + mkdirSync(join(catalogDir, "feeds"), { recursive: true }); + writeFileSync( + join(catalogDir, "feeds/de.json"), + JSON.stringify({ + sources: [ + { + name: "catalog", + spec: "gtfs", + type: "http", + url: "https://catalog.example.test/feed.zip", + }, + ], + }), + ); + const source = { + spec: "gtfs" as const, + type: "http" as const, + origin: "operator" as const, + region: "de", + name: "local", + url: "https://operator.example.test/feed.zip", + license: { + spdxIdentifier: "CC-BY-4.0", + attribution: "Synthetic operator", + publisher: "Operator", + }, + }, + sourceId = "operator:de:local"; + writeFeedOverlayAtomic(overlayPath, { + version: 3, + sources: [source], + patches: [{ sourceId, skip: true }], + quarantine: [], + }); + return { dataDir, catalogDir, overlayPath, source, sourceId }; +} +it("re-enables a retained disabled operator source without losing its URL, attribution or licence", () => { + const fx = fixture(); + expect(listTransitSources(fx).sources.find((row) => row.id === fx.sourceId)?.requested).toBe( + false, + ); + prepareEnableTransitSource({ ...fx, countries: ["de"] }).persist(); + expect(readFeedOverlay(fx.overlayPath)?.sources).toEqual([fx.source]); + expect(listTransitSources(fx).sources.find((row) => row.id === fx.sourceId)?.requested).toBe( + true, + ); +}); +it("removes an operator source together with its patches and quarantine, leaving unrelated records", () => { + const fx = fixture(), + overlay = readFeedOverlay(fx.overlayPath)!; + writeFeedOverlayAtomic(fx.overlayPath, { + ...overlay, + patches: [...overlay.patches, { sourceId: "catalog:de:catalog", skip: false }], + quarantine: [ + { + sourceId: fx.sourceId, + reason: "fixture", + firstSeen: "2026-10-01", + lastChecked: "2026-10-01", + }, + ], + }); + prepareRemoveTransitSource(fx).persist(); + expect(readFeedOverlay(fx.overlayPath)).toEqual({ + version: 3, + sources: [], + patches: [{ sourceId: "catalog:de:catalog", skip: false }], + quarantine: [], + }); +}); +it("refuses catalog removal without changing desired state", () => { + const fx = fixture(), + before = readFileSync(fx.overlayPath); + expect(() => prepareRemoveTransitSource({ ...fx, sourceId: "catalog:de:catalog" })).toThrow(); + expect(readFileSync(fx.overlayPath)).toEqual(before); +}); + +it("does not overwrite an unrelated source edit made after preparation", () => { + const fx = fixture(), + prepared = prepareEnableTransitSource(fx), + overlay = readFeedOverlay(fx.overlayPath)!; + writeFeedOverlayAtomic(fx.overlayPath, { + ...overlay, + patches: [...overlay.patches, { sourceId: "catalog:de:catalog", skip: true }], + }); + const changed = readFileSync(fx.overlayPath); + expect(() => prepared.persist()).toThrow(); + expect(readFileSync(fx.overlayPath)).toEqual(changed); +}); + +it("retains a closed desired-source mutation in the durable sync identity", async () => { + const { normalizeDataExecutionInput } = await import("@openmapx/core/data-manager-contracts"); + const mutation = { + action: "disable", + sourceId: "operator:de:local", + expectedSourceRevision: "a".repeat(64), + }; + expect( + normalizeDataExecutionInput({ + kind: "transit-sync", + countries: ["de"], + sourceMutation: mutation, + }), + ).toMatchObject({ sourceMutation: mutation }); +}); + +it("keeps exact source provenance across disable and enable materialization", async () => { + const fx = fixture(), + { prepareSetTransitSourceEnabled } = await import("../src/transit-sources.js"), + { applyFeedOverlay } = await import("../src/jobs/transitous-feeds-overlay.js"); + for (const enabled of [true, false, true]) { + prepareSetTransitSourceEnabled({ ...fx, enabled }).persist(); + const feeds = [{ region: "de", sources: [] as Record[] }]; + applyFeedOverlay(feeds, readFeedOverlay(fx.overlayPath)!); + const source = feeds[0]!.sources[0]!; + expect(source).toMatchObject({ + url: fx.source.url, + skip: !enabled, + license: { + "spdx-identifier": "CC-BY-4.0", + "attribution-text": "Synthetic operator", + publisher: "Operator", + }, + }); + expect(listTransitSources(fx).sources.find((row) => row.id === fx.sourceId)).toMatchObject({ + desiredState: enabled ? "enabled" : "disabled", + actions: { enable: !enabled, disable: enabled, remove: true }, + }); + } +}); +it("allows a retained out-of-scope source to be disabled or removed, but not enabled", async () => { + const fx = fixture(), + { prepareSetTransitSourceEnabled } = await import("../src/transit-sources.js"); + prepareSetTransitSourceEnabled({ ...fx, countries: ["ch"], enabled: false }).persist(); + expect( + listTransitSources({ ...fx, countries: ["ch"] }).sources.find((row) => row.id === fx.sourceId), + ).toMatchObject({ actions: { enable: false, disable: false, remove: true } }); + expect(() => prepareEnableTransitSource({ ...fx, countries: ["ch"] })).toThrow(/outside/); + prepareRemoveTransitSource({ ...fx, countries: ["ch"] }).persist(); + expect(() => prepareEnableTransitSource(fx)).toThrow(/not found/); +}); + +it("keeps a quarantined colliding operator visible and removable without materializing it", () => { + const fx = fixture(), + source = { ...fx.source, name: "catalog", url: "https://catalog.example.test/feed.zip" }, + sourceId = "operator:de:catalog"; + writeFeedOverlayAtomic(fx.overlayPath, { + version: 3, + sources: [source], + patches: [], + quarantine: [ + { sourceId, reason: "catalog collision", firstSeen: "2026-10-01", lastChecked: "2026-10-01" }, + ], + }); + expect(listTransitSources(fx).sources.find((row) => row.id === sourceId)).toMatchObject({ + requested: false, + desiredState: "disabled", + actions: { enable: false, remove: true }, + }); + prepareRemoveTransitSource({ ...fx, sourceId }).persist(); + expect(readFeedOverlay(fx.overlayPath)?.sources).toEqual([]); +}); + +it("bounds the UTF-8 source intent before creating a durable execution", async () => { + const { normalizeDataExecutionInput } = await import("@openmapx/core/data-manager-contracts"), + mutation = { + action: "add", + expectedSourceRevision: "a".repeat(64), + source: { + region: "de", + name: "bounded", + url: "https://operator.example.test/feed.zip", + license: { spdxIdentifier: "CC0-1.0", attribution: "ü".repeat(8192) }, + }, + }; + expect(JSON.stringify(mutation).length).toBeLessThan(16384); + expect(() => + normalizeDataExecutionInput({ + kind: "transit-sync", + countries: ["de"], + sourceMutation: mutation, + }), + ).toThrow(/metadata is too large/); +}); diff --git a/services/data-manager/__tests__/transit-sources.test.ts b/services/data-manager/__tests__/transit-sources.test.ts index 655d8d8a7..f8b8ee1f2 100644 --- a/services/data-manager/__tests__/transit-sources.test.ts +++ b/services/data-manager/__tests__/transit-sources.test.ts @@ -9,6 +9,7 @@ import { prepareAddTransitSource, prepareEnableTransitSource, prepareRemoveTransitSource, + prepareSetTransitSourceEnabled, } from "../src/transit-sources.js"; let tmp: string | undefined; @@ -142,8 +143,9 @@ describe("transit source lifecycle", () => { it("persists catalog disable/re-enable and operator removal as v3 state", () => { const fx = fixture(); prepareAddTransitSource({ ...fx, source: operatorSource() }).persist(); - prepareRemoveTransitSource({ + prepareSetTransitSourceEnabled({ ...fx, + enabled: false, sourceId: "catalog:de:active-feed", }).persist(); expect(listTransitSources({ ...fx }).sources).toContainEqual( @@ -244,6 +246,6 @@ describe("transit source lifecycle", () => { ).toThrow(/country scope/); expect(() => prepareEnableTransitSource({ ...fx, countries: ["de"], sourceId: "catalog:fr:fr-feed" }), - ).toThrow(/not found in the configured scope/); + ).toThrow(/outside the configured country scope/); }); }); diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 9442c9f58..cc94b96b9 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -170,6 +170,107 @@ "type": "string", "pattern": "^[A-Za-z]{2,3}$" } + }, + "sourceMutation": { + "oneOf": [ + { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "add" + ] + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "source": { + "type": "object", + "properties": { + "region": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri" + }, + "license": { + "type": "object", + "properties": { + "spdxIdentifier": { + "type": "string" + }, + "url": { + "type": "string" + }, + "attribution": { + "type": "string", + "minLength": 1 + }, + "publisher": { + "type": "string" + }, + "publisherUrl": { + "type": "string" + } + }, + "required": [ + "attribution" + ], + "additionalProperties": false + } + }, + "required": [ + "region", + "name", + "url", + "license" + ], + "additionalProperties": false + } + }, + "required": [ + "action", + "expectedSourceRevision", + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "enable", + "disable", + "remove" + ] + }, + "sourceId": { + "type": "string", + "minLength": 1 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "action", + "sourceId", + "expectedSourceRevision" + ], + "additionalProperties": false + } + ] } }, "required": [ @@ -26069,6 +26170,10 @@ "schema": { "type": "object", "properties": { + "sourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "sources": { "type": "array", "items": { @@ -26109,6 +26214,45 @@ "requested": { "type": "boolean" }, + "sourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "desiredState": { + "type": "string", + "enum": [ + "enabled", + "disabled", + "removed" + ] + }, + "actions": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "disable": { + "type": "boolean" + }, + "remove": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "enable", + "disable", + "remove", + "reason" + ], + "additionalProperties": false + }, "active": { "type": "boolean" }, @@ -26179,6 +26323,9 @@ "origin", "license", "requested", + "sourceRevision", + "desiredState", + "actions", "active", "lifecycle" ], @@ -26199,6 +26346,7 @@ } }, "required": [ + "sourceRevision", "sources", "total", "limit", @@ -26712,6 +26860,10 @@ "schema": { "type": "object", "properties": { + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "idempotencyKey": { "type": "string", "maxLength": 200 @@ -26759,6 +26911,7 @@ } }, "required": [ + "expectedSourceRevision", "region", "name", "url", @@ -27078,6 +27231,32 @@ "existingJobId" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false } ] } @@ -27982,8 +28161,15 @@ "triggeredBy": { "type": "string", "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" } }, + "required": [ + "expectedSourceRevision" + ], "additionalProperties": false } } @@ -28309,6 +28495,32 @@ "existingJobId" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false } ] } @@ -28647,8 +28859,15 @@ "triggeredBy": { "type": "string", "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" } }, + "required": [ + "expectedSourceRevision" + ], "additionalProperties": false } } @@ -28974,6 +29193,730 @@ "existingJobId" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/transit/sources/{sourceId}/disable": { + "post": { + "operationId": "disableTransitSource", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "triggeredBy": { + "type": "string", + "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "expectedSourceRevision" + ], + "additionalProperties": false + } + } + } + }, + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "sourceId", + "required": true + } + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "disableTransitSource", + "x-openmapx-auth": "service", + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "sourceId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "started" + ] + } + }, + "required": [ + "jobId", + "sourceId", + "status" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] + }, + "existingJobId": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "ok", + "reason", + "existingJobId" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false } ] } diff --git a/services/data-manager/scripts/openapi/generate.ts b/services/data-manager/scripts/openapi/generate.ts index 3002636fe..31d1a3510 100644 --- a/services/data-manager/scripts/openapi/generate.ts +++ b/services/data-manager/scripts/openapi/generate.ts @@ -1,4 +1,5 @@ import { readFile, writeFile } from "node:fs/promises"; +import { dataManagerOperations } from "@openmapx/core/data-manager-contracts"; import { DOCUMENT_PATH, generateDocument } from "./document.js"; try { @@ -8,10 +9,14 @@ try { throw new Error( "Data-manager OpenAPI differs. Run pnpm data-manager:openapi:generate and commit the result.", ); - console.log("Data-manager OpenAPI is current (48 operations)."); + console.log( + `Data-manager OpenAPI is current (${Object.keys(dataManagerOperations).length} operations).`, + ); } else { await writeFile(DOCUMENT_PATH, document); - console.log("Wrote services/data-manager/openapi.json (48 operations)."); + console.log( + `Wrote services/data-manager/openapi.json (${Object.keys(dataManagerOperations).length} operations).`, + ); } } catch (error) { console.error(error instanceof Error ? error.message : error); diff --git a/services/data-manager/src/api.ts b/services/data-manager/src/api.ts index 8050f4eeb..b536b9b64 100644 --- a/services/data-manager/src/api.ts +++ b/services/data-manager/src/api.ts @@ -10,6 +10,8 @@ import { type DataExecutionInput, type DataExecutionStatus, isDataExecutionTerminal, + normalizeDataExecutionInput, + type TransitDesiredSourceMutation, } from "@openmapx/core/data-manager-contracts"; import { feedState } from "@openmapx/db-schema"; import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; @@ -80,6 +82,7 @@ import { prepareAddTransitSource, prepareEnableTransitSource, prepareRemoveTransitSource, + prepareSetTransitSourceEnabled, type TransitFeedStateEvidence, TransitSourceError, } from "./transit-sources.js"; @@ -495,8 +498,8 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { triggeredBy: string; idempotencyKey?: string; sourceId: string; - action: "add" | "remove" | "enable"; - persist: () => void; + action: "add" | "remove" | "enable" | "disable"; + mutation: TransitDesiredSourceMutation; }): Promise< { ok: true; jobId: string } | { ok: false; status: number; body: Record } > { @@ -506,6 +509,11 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { status: 503, body: { ok: false, error: "Durable data receiver is unavailable" }, }; + const input = normalizeDataExecutionInput({ + kind: "transit-sync", + countries: operationsPolicy.countries, + sourceMutation: options.mutation, + }); const start = await singleFlight.tryStartSync({ trigger: "api", triggeredBy: options.triggeredBy, @@ -529,26 +537,14 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }, }; } - try { - // The synchronous single-flight reservation and visible job row exist - // before the atomic desired-state rename. - options.persist(); - } catch (error) { - await finalizeJobRow(start.jobId, "error"); - singleFlight.markSyncFinished(); - return { - ok: false, - status: 500, - body: { ok: false, error: (error as Error).message, jobId: start.jobId }, - }; - } + // Desired state is written only by the fixed dispatcher under this sync's + // original admitted target authority, never by the HTTP observer. singleFlight.markSyncFinished(start.jobId); try { - const admitted = await opts.executionReceiver.admitLocal( - { kind: "transit-sync", countries: operationsPolicy.countries }, - { sourceJobId: start.jobId }, - ); - await finalizeJobRow(start.jobId, "ok"); + const admitted = await opts.executionReceiver.admitLocal(input, { + sourceJobId: start.jobId, + triggeredBy: options.triggeredBy, + }); return { ok: true, jobId: admitted.id }; } catch (error) { await finalizeJobRow(start.jobId, "error"); @@ -653,6 +649,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { catalogDir, overlayPath, countries: operationsPolicy.countries, + expectedSourceRevision: req.body.expectedSourceRevision, source: { spec: "gtfs", type: "http", @@ -665,14 +662,27 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }); } catch (error) { const status = error instanceof TransitSourceError ? error.statusCode : 400; - return reply.code(status).send({ ok: false, error: (error as Error).message }); + return reply.code(status).send({ + ok: false, + error: (error as Error).message, + ...(error instanceof TransitSourceError && error.code ? { code: error.code } : {}), + }); } const result = await reserveSourceMutation({ triggeredBy: req.body.triggeredBy ?? "api", idempotencyKey: req.body.idempotencyKey, sourceId: mutation.sourceId, action: "add", - persist: mutation.persist, + mutation: { + action: "add", + expectedSourceRevision: req.body.expectedSourceRevision, + source: { + region: req.body.region, + name: req.body.name, + url: req.body.url, + license: req.body.license, + }, + }, }); if (!result.ok) return reply.code(result.status).send(result.body); return reply @@ -687,18 +697,27 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { catalogDir, overlayPath, countries: operationsPolicy.countries, + expectedSourceRevision: req.body.expectedSourceRevision, sourceId: req.params.sourceId, }); } catch (error) { const status = error instanceof TransitSourceError ? error.statusCode : 400; - return reply.code(status).send({ ok: false, error: (error as Error).message }); + return reply.code(status).send({ + ok: false, + error: (error as Error).message, + ...(error instanceof TransitSourceError && error.code ? { code: error.code } : {}), + }); } const result = await reserveSourceMutation({ triggeredBy: req.body?.triggeredBy ?? "api", idempotencyKey: req.body?.idempotencyKey, sourceId: mutation.sourceId, action: "remove", - persist: mutation.persist, + mutation: { + action: "remove", + sourceId: mutation.sourceId, + expectedSourceRevision: req.body.expectedSourceRevision, + }, }); if (!result.ok) return reply.code(result.status).send(result.body); return reply @@ -713,18 +732,63 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { catalogDir, overlayPath, countries: operationsPolicy.countries, + expectedSourceRevision: req.body.expectedSourceRevision, sourceId: req.params.sourceId, }); } catch (error) { const status = error instanceof TransitSourceError ? error.statusCode : 400; - return reply.code(status).send({ ok: false, error: (error as Error).message }); + return reply.code(status).send({ + ok: false, + error: (error as Error).message, + ...(error instanceof TransitSourceError && error.code ? { code: error.code } : {}), + }); } const result = await reserveSourceMutation({ triggeredBy: req.body?.triggeredBy ?? "api", idempotencyKey: req.body?.idempotencyKey, sourceId: mutation.sourceId, action: "enable", - persist: mutation.persist, + mutation: { + action: "enable", + sourceId: mutation.sourceId, + expectedSourceRevision: req.body.expectedSourceRevision, + }, + }); + if (!result.ok) return reply.code(result.status).send(result.body); + return reply + .code(202) + .send({ jobId: result.jobId, sourceId: mutation.sourceId, status: "started" }); + }); + + registerDataManagerRoute(app, "disableTransitSource", async (req, reply) => { + let mutation: ReturnType; + try { + mutation = prepareSetTransitSourceEnabled({ + catalogDir, + overlayPath, + countries: operationsPolicy.countries, + expectedSourceRevision: req.body.expectedSourceRevision, + sourceId: req.params.sourceId, + enabled: false, + }); + } catch (error) { + const status = error instanceof TransitSourceError ? error.statusCode : 400; + return reply.code(status).send({ + ok: false, + error: (error as Error).message, + ...(error instanceof TransitSourceError && error.code ? { code: error.code } : {}), + }); + } + const result = await reserveSourceMutation({ + triggeredBy: req.body?.triggeredBy ?? "api", + idempotencyKey: req.body?.idempotencyKey, + sourceId: mutation.sourceId, + action: "disable", + mutation: { + action: "disable", + sourceId: mutation.sourceId, + expectedSourceRevision: req.body.expectedSourceRevision, + }, }); if (!result.ok) return reply.code(result.status).send(result.body); return reply diff --git a/services/data-manager/src/jobs/execution/dispatch.ts b/services/data-manager/src/jobs/execution/dispatch.ts index 12bec1c27..9b7e8cab1 100644 --- a/services/data-manager/src/jobs/execution/dispatch.ts +++ b/services/data-manager/src/jobs/execution/dispatch.ts @@ -47,6 +47,7 @@ import type { SingleFlightController } from "../transitous/single-flight.js"; import { dataExecutionDiagnostic } from "./diagnostics.js"; import { runOwnedDataOpsOperation } from "./ops-children.js"; import type { DataExecutionContext } from "./runtime.js"; +import { publishOwnedTransitSourceMutation } from "./source-mutation.js"; import { DataPublicationUnknown, publishDataChange, @@ -158,6 +159,12 @@ export function createDataExecutionDispatcher(deps: { jobChildLogger({ job: "transit-sync", jobId: context.execution.executionId }), ); try { + await publishOwnedTransitSourceMutation( + input, + context, + deps.operationsPolicy, + deps.dataDir, + ); const ctx = await publishDataChange(() => buildJobContext({ dataDir: deps.dataDir, diff --git a/services/data-manager/src/jobs/execution/source-mutation-postgres.test.ts b/services/data-manager/src/jobs/execution/source-mutation-postgres.test.ts new file mode 100644 index 000000000..0e4d82feb --- /dev/null +++ b/services/data-manager/src/jobs/execution/source-mutation-postgres.test.ts @@ -0,0 +1,208 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { expect, it } from "vitest"; +import { transitSourceRevision } from "../../transit-sources.js"; +import { resolveOperationsProfile } from "../transitous/operations-profile.js"; +import { readFeedOverlay, writeFeedOverlayAtomic } from "../transitous-feeds-overlay.js"; +import { createDataExecutionRuntime } from "./runtime.js"; +import { publishOwnedTransitSourceMutation } from "./source-mutation.js"; +import { createPostgresDataExecutionStore } from "./store.js"; + +it.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "publishes source intent only under its original durable sync root and preserves it on replay", + async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned migrated PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + drizzle(sql); + const root = mkdtempSync(join(tmpdir(), "omx-source-owner-")), + dataRoot = join(root, "data"), + catalogDir = join(dataRoot, ".transitous-catalog"), + overlayPath = join(dataRoot, "overrides/feeds-overlay.json"); + mkdirSync(join(catalogDir, "feeds"), { recursive: true }); + mkdirSync(join(dataRoot, "overrides")); + writeFileSync(join(catalogDir, "feeds/de.json"), JSON.stringify({ sources: [] })); + writeFeedOverlayAtomic(overlayPath, { + version: 3, + sources: [ + { + spec: "gtfs", + type: "http", + origin: "operator", + region: "de", + name: "local", + url: "https://operator.example.test/feed.zip", + license: { spdxIdentifier: "CC0-1.0", attribution: "Synthetic operator" }, + }, + ], + patches: [], + quarantine: [], + }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; + } + const held = deferred(), + started = deferred(); + let writes = 0; + const input = { + kind: "transit-sync" as const, + countries: ["de"], + sourceMutation: { + action: "disable" as const, + sourceId: "operator:de:local", + expectedSourceRevision: transitSourceRevision({ catalogDir, overlayPath }), + }, + }; + const runtime = createDataExecutionRuntime({ + store, + dispatch: async (value, context) => { + if (value.kind === "download-fonts") { + started.resolve(); + await held.promise; + return { kind: "download-fonts", completed: true }; + } + if (value.kind !== "transit-sync") throw Error("Unexpected fixture input"); + const before = readFileSync(overlayPath); + await expect( + publishOwnedTransitSourceMutation( + { ...value, sourceMutation: { ...input.sourceMutation, action: "remove" } }, + context, + resolveOperationsProfile({ countries: ["de"] }), + dataRoot, + ), + ).rejects.toThrow("Original transit mutation identity"); + expect(readFileSync(overlayPath)).toEqual(before); + await publishOwnedTransitSourceMutation( + value, + context, + resolveOperationsProfile({ countries: ["de"] }), + dataRoot, + ); + writes++; + return { + kind: "transit-sync", + sourceRevision: "synthetic-pinned-catalog", + countries: ["de"], + finalStatus: "ok", + }; + }, + }); + const deadline = Date.now() + 60000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + let id: string | undefined; + const keys = [key], + sourceReservationId = randomUUID(); + try { + const fontKey = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }), + blockedKey = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + keys.push(fontKey, blockedKey); + const competing = await runtime.admit( + { kind: "download-fonts" }, + { operationKey: fontKey, requestedDeadlineAt: deadline }, + ); + await started.promise; + const originalBytes = readFileSync(overlayPath); + await expect( + runtime.admit(input, { operationKey: blockedKey, requestedDeadlineAt: deadline }), + ).rejects.toThrow(); + expect(writes).toBe(0); + expect(readFileSync(overlayPath)).toEqual(originalBytes); + held.resolve(); + await expect + .poll(async () => (await runtime.inspect(competing.id))?.status, { timeout: 15000 }) + .toBe("ok"); + await sql`INSERT INTO data_manager.jobs(id,kind,status,metadata) VALUES (${sourceReservationId},'transit-source-disable','running',${JSON.stringify({ action: "disable", sourceId: "operator:de:local" })}::text::jsonb)`; + const admitted = await runtime.admit(input, { + operationKey: key, + requestedDeadlineAt: deadline, + sourceJobId: sourceReservationId, + }); + id = admitted.id; + await expect + .poll(async () => (await runtime.inspect(id!))?.status, { timeout: 15000 }) + .toBe("ok"); + expect(readFeedOverlay(overlayPath)?.patches).toEqual([ + { sourceId: "operator:de:local", skip: true }, + ]); + expect( + (await sql`SELECT status FROM data_manager.jobs WHERE id=${sourceReservationId}`)[0] + ?.status, + ).toBe("ok"); + const bytes = readFileSync(overlayPath); + expect( + (await runtime.admit(input, { operationKey: key, requestedDeadlineAt: deadline })).id, + ).toBe(id); + expect(writes).toBe(1); + expect(readFileSync(overlayPath)).toEqual(bytes); + for (const action of ["enable", "remove"] as const) { + const nextKey = createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }); + keys.push(nextKey); + const next = { + ...input, + sourceMutation: { + ...input.sourceMutation, + action, + expectedSourceRevision: transitSourceRevision({ catalogDir, overlayPath }), + }, + }; + const status = await runtime.admit(next, { + operationKey: nextKey, + requestedDeadlineAt: deadline, + }); + await expect + .poll(async () => (await runtime.inspect(status.id))?.status, { timeout: 15000 }) + .toBe("ok"); + const observed = await store.inspect(status.id); + expect(observed?.input).toEqual(next); + if (action === "enable") + expect(readFeedOverlay(overlayPath)).toMatchObject({ + sources: [ + { + url: "https://operator.example.test/feed.zip", + license: { spdxIdentifier: "CC0-1.0", attribution: "Synthetic operator" }, + }, + ], + patches: [{ sourceId: "operator:de:local", skip: false }], + }); + else + expect(readFeedOverlay(overlayPath)).toEqual({ + version: 3, + sources: [], + patches: [], + quarantine: [], + }); + } + expect(writes).toBe(3); + } finally { + held.resolve(); + await runtime.close(); + await sql`DELETE FROM data_manager.jobs WHERE operation_key IN ${sql(keys)} OR id=${sourceReservationId}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }, + 45000, +); diff --git a/services/data-manager/src/jobs/execution/source-mutation.ts b/services/data-manager/src/jobs/execution/source-mutation.ts new file mode 100644 index 000000000..4c03c97fe --- /dev/null +++ b/services/data-manager/src/jobs/execution/source-mutation.ts @@ -0,0 +1,40 @@ +import { join } from "node:path"; +import { assertArtifactTargetLeaseForward, controlDigest } from "@openmapx/core/build-artifacts"; +import type { DataExecutionInput } from "@openmapx/core/data-manager-contracts"; +import { prepareTransitSourceMutation } from "../../transit-sources.js"; +import { resolveTransitOverlayPath } from "../transitous/internal.js"; +import type { MotisOperationsPolicy } from "../transitous/operations-profile.js"; +import type { DataExecutionContext } from "./runtime.js"; +import { dataExecutionWorkerContext, publishDataChange } from "./worker-scope.js"; +/** Fixed first phase of the original transit-sync, not an HTTP-side mutation. */ +export async function publishOwnedTransitSourceMutation( + input: Extract, + context: DataExecutionContext, + policy: MotisOperationsPolicy, + dataDir: string, +): Promise { + if (!input.sourceMutation) return; + if (dataExecutionWorkerContext() !== context) throw Error("Owned transit worker required"); + context.assertForward(); + assertArtifactTargetLeaseForward(context.targetAuthority); + const retained = await context.record(); + if ( + retained.input.kind !== "transit-sync" || + controlDigest(retained.input) !== controlDigest(input) || + controlDigest(retained.execution) !== controlDigest(context.execution) + ) + throw Error("Original transit mutation identity required"); + const mutation = prepareTransitSourceMutation( + { + catalogDir: join(dataDir, ".transitous-catalog"), + overlayPath: resolveTransitOverlayPath(dataDir), + countries: policy.countries, + }, + input.sourceMutation, + ); + await publishDataChange(() => { + context.assertForward(); + assertArtifactTargetLeaseForward(context.targetAuthority); + mutation.persist(); + }); +} diff --git a/services/data-manager/src/jobs/execution/store.ts b/services/data-manager/src/jobs/execution/store.ts index debdd5e07..b8e652560 100644 --- a/services/data-manager/src/jobs/execution/store.ts +++ b/services/data-manager/src/jobs/execution/store.ts @@ -709,7 +709,21 @@ export function createPostgresDataExecutionStore( const result = await sql.begin(async (tx) => { const current = await find(tx, "id", id, true); if (!current || current.version !== version) return null; - return update(tx, current, status, current.children, terminalAt); + const saved = await update(tx, current, status, current.children, terminalAt); + if (saved && current.input.kind === "transit-sync" && current.input.sourceMutation) { + // The original source reservation is a linked observer, not a second + // worker. Settle it atomically with the actual contained sync result. + const linked = canonicalJson({ + id: current.status.id, + operationKey: current.execution.operationKey, + fingerprint: current.execution.fingerprint, + }); + await tx`UPDATE data_manager.jobs SET status=${status.status},finished_at=${new Date(terminalAt).toISOString()}::text::timestamptz + WHERE execution_version IS NULL AND finished_at IS NULL + AND kind=${`transit-source-${current.input.sourceMutation.action}`} + AND metadata->'durableExecution'=${linked}::text::jsonb`; + } + return saved; }); if (result) await options.controlJournal.publishTerminal(result.execution); return result; diff --git a/services/data-manager/src/transit-sources.ts b/services/data-manager/src/transit-sources.ts index 9d3c3650b..22bb25c16 100644 --- a/services/data-manager/src/transit-sources.ts +++ b/services/data-manager/src/transit-sources.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { @@ -53,6 +54,7 @@ export class TransitSourceError extends Error { constructor( message: string, readonly statusCode: 400 | 404 | 409 = 400, + readonly code?: "source_action_not_allowed", ) { super(message); } @@ -122,6 +124,21 @@ function desiredSources( ): DesiredSource[] { const feeds = rawCatalog(catalogDir); applyFeedOverlay(feeds, overlay); + const quarantined = new Set(overlay.quarantine.map((entry) => entry.sourceId)); + // Project retained quarantined operators in isolation: their very reason + // for quarantine may be a collision with the pinned catalog. They remain + // visible for removal without ever being materialized into that catalog. + const retained: FeedFile[] = []; + applyFeedOverlay(retained, { + version: 3, + sources: overlay.sources.filter( + (source) => + source.spec === "gtfs" && quarantined.has(operatorSourceId(source.region, source.name)), + ), + patches: [], + quarantine: [], + }); + feeds.push(...retained); const atlasSpecIndex = buildTransitlandAtlasSpecIndex(catalogDir); const scope = (countries ?? []).map((country) => country.toLowerCase()); const sources: DesiredSource[] = feeds.flatMap((feed) => { @@ -142,7 +159,7 @@ function desiredSources( origin: entry.origin, originUrl: entry.originUrl, license: structuredClone(entry.license ?? {}), - requested: entry.requested, + requested: entry.requested && !quarantined.has(entry.sourceId), }, ]; }); @@ -178,6 +195,35 @@ function stableStringify(value: unknown): string { return JSON.stringify(value) ?? "null"; } +export function transitSourceRevision(options: { + catalogDir: string; + overlayPath: string; +}): string { + return createHash("sha256") + .update( + stableStringify({ + catalog: rawCatalog(options.catalogDir), + overlay: readOverlay(options.overlayPath), + }), + ) + .digest("hex"); +} +function guardedOverlayPersistence( + options: { catalogDir: string; overlayPath: string; expectedSourceRevision?: string }, + candidate: FeedOverlay, +): () => void { + const revision = transitSourceRevision(options); + if (options.expectedSourceRevision && options.expectedSourceRevision !== revision) + throw new TransitSourceError("Desired transit sources changed. Refresh before applying.", 409); + return () => { + if (transitSourceRevision(options) !== revision) + throw new TransitSourceError( + "Desired transit sources changed. Refresh before applying.", + 409, + ); + persistOverlay(options.overlayPath, candidate); + }; +} function equivalentDesiredActive( desired: DesiredSource, active: TransitSourceManifestRecord, @@ -203,12 +249,18 @@ export function listTransitSources(options: { feedStates?: TransitFeedStateEvidence[]; query?: TransitSourceListQuery; }): TransitSourceListResult { - const desired = desiredSources( - options.catalogDir, - readOverlay(options.overlayPath), - options.countries, + const overlay = readOverlay(options.overlayPath), + sourceRevision = transitSourceRevision(options), + active = activeEvidence(options.dataDir); + const scope = (options.countries ?? []).map((country) => country.toLowerCase()); + const desired = desiredSources(options.catalogDir, overlay).filter( + (source) => + source.origin === "operator" || + scope.length === 0 || + scope.includes(countryOfRegion(source.region)) || + active.sources.has(source.id) || + overlay.patches.some((patch) => patch.sourceId === source.id), ); - const active = activeEvidence(options.dataDir); const byId = new Map( desired.map((source) => [source.id, source]), ); @@ -252,6 +304,24 @@ export function listTransitSources(options: { ...(base.originUrl ? { originUrl: base.originUrl } : {}), license: transitLicenseSchema.parse(base.license ?? {}), requested, + sourceRevision, + desiredState: !desiredSource ? "removed" : requested ? "enabled" : "disabled", + actions: { + enable: + !!desiredSource && + !requested && + (scope.length === 0 || scope.includes(countryOfRegion(base.region))) && + !overlay.quarantine.some((entry) => entry.sourceId === id), + disable: !!desiredSource && requested, + remove: desiredSource?.origin === "operator", + reason: !desiredSource + ? "Source metadata is no longer retained." + : overlay.quarantine.some((entry) => entry.sourceId === id) + ? "Source is quarantined." + : scope.length > 0 && !scope.includes(countryOfRegion(base.region)) + ? "Outside configured country scope." + : null, + }, active: isActive, ...(isActive && active.epoch ? { activeEpoch: active.epoch } : {}), ...(activeSource @@ -287,7 +357,7 @@ export function listTransitSources(options: { const total = rows.length; const limit = Math.min(Math.max(Math.floor(query.limit ?? 100), 1), 500); const offset = Math.max(Math.floor(query.offset ?? 0), 0); - return { sources: rows.slice(offset, offset + limit), total, limit, offset }; + return { sourceRevision, sources: rows.slice(offset, offset + limit), total, limit, offset }; } function validateOverlayAgainstCatalog(catalogDir: string, overlay: FeedOverlay): void { @@ -304,6 +374,7 @@ export function prepareAddTransitSource(options: { overlayPath: string; source: FeedOverlayGtfsSource; countries?: string[]; + expectedSourceRevision?: string; }): { sourceId: string; persist: () => void } { if (options.countries?.length) { const scope = options.countries.map((country) => country.toLowerCase()); @@ -315,6 +386,7 @@ export function prepareAddTransitSource(options: { ); } } + const initialRevision = transitSourceRevision(options); const current = readOverlay(options.overlayPath); const candidate = parseFeedOverlay({ ...current, @@ -322,7 +394,13 @@ export function prepareAddTransitSource(options: { }); validateOverlayAgainstCatalog(options.catalogDir, candidate); const sourceId = operatorSourceId(options.source.region.toLowerCase(), options.source.name); - return { sourceId, persist: () => persistOverlay(options.overlayPath, candidate) }; + return { + sourceId, + persist: guardedOverlayPersistence( + { ...options, expectedSourceRevision: options.expectedSourceRevision ?? initialRevision }, + candidate, + ), + }; } export function prepareRemoveTransitSource(options: { @@ -330,77 +408,123 @@ export function prepareRemoveTransitSource(options: { overlayPath: string; sourceId: string; countries?: string[]; + expectedSourceRevision?: string; }): { sourceId: string; persist: () => void } { + const initialRevision = transitSourceRevision(options); const current = readOverlay(options.overlayPath); - let candidate: FeedOverlay; - if (options.sourceId.startsWith("operator:")) { - const sources = current.sources.filter( - (source) => - source.spec !== "gtfs" || operatorSourceId(source.region, source.name) !== options.sourceId, - ); - if (sources.length === current.sources.length) { - throw new TransitSourceError(`Operator source ${options.sourceId} not found`, 404); - } - candidate = { ...current, sources }; - } else if (options.sourceId.startsWith("catalog:")) { - const known = desiredSources(options.catalogDir, current, options.countries).some( - (source) => source.origin === "catalog" && source.id === options.sourceId, + if (!options.sourceId.startsWith("operator:")) { + throw new TransitSourceError( + "Pinned catalog sources can be disabled, but cannot be removed", + 409, + "source_action_not_allowed", ); - if (!known) { - throw new TransitSourceError( - `Catalog source ${options.sourceId} not found in the configured scope`, - 404, - ); - } - candidate = { - ...current, - patches: [ - ...current.patches.filter((patch) => patch.sourceId !== options.sourceId), - { sourceId: options.sourceId, skip: true }, - ], - }; - } else { - throw new TransitSourceError(`Unsupported transit source identity ${options.sourceId}`); } + const sources = current.sources.filter( + (source) => + source.spec !== "gtfs" || operatorSourceId(source.region, source.name) !== options.sourceId, + ); + if (sources.length === current.sources.length) + throw new TransitSourceError(`Operator source ${options.sourceId} not found`, 404); + const candidate: FeedOverlay = { + ...current, + sources, + patches: current.patches.filter((patch) => patch.sourceId !== options.sourceId), + quarantine: current.quarantine.filter((entry) => entry.sourceId !== options.sourceId), + }; const validated = parseFeedOverlay(candidate); return { sourceId: options.sourceId, - persist: () => persistOverlay(options.overlayPath, validated), + persist: guardedOverlayPersistence( + { ...options, expectedSourceRevision: options.expectedSourceRevision ?? initialRevision }, + validated, + ), }; } -export function prepareEnableTransitSource(options: { +export function prepareSetTransitSourceEnabled(options: { catalogDir: string; overlayPath: string; sourceId: string; + enabled: boolean; countries?: string[]; + expectedSourceRevision?: string; }): { sourceId: string; persist: () => void } { - if (!options.sourceId.startsWith("catalog:")) { - throw new TransitSourceError("Only pinned catalog sources can be re-enabled"); - } + const initialRevision = transitSourceRevision(options); const current = readOverlay(options.overlayPath); - const known = desiredSources(options.catalogDir, current, options.countries).some( - (source) => source.origin === "catalog" && source.id === options.sourceId, + // Look outside the configured scope for deactivation. A retained source must + // remain removable/disableable after an operator narrows the deployment. + const known = desiredSources(options.catalogDir, current).find( + (source) => source.id === options.sourceId, ); - if (!known) { + if (!known) throw new TransitSourceError(`Transit source ${options.sourceId} not found`, 404); + if (options.enabled && current.quarantine.some((entry) => entry.sourceId === options.sourceId)) throw new TransitSourceError( - `Catalog source ${options.sourceId} not found in the configured scope`, - 404, + "Source is quarantined; repair it before enabling", + 409, + "source_action_not_allowed", + ); + if ( + options.enabled && + options.countries?.length && + !options.countries + .map((country) => country.toLowerCase()) + .includes(countryOfRegion(known.region)) + ) + throw new TransitSourceError( + "Source is outside the configured country scope", + 409, + "source_action_not_allowed", ); - } const candidate = parseFeedOverlay({ ...current, patches: [ ...current.patches.filter((patch) => patch.sourceId !== options.sourceId), - { sourceId: options.sourceId, skip: false }, + { sourceId: options.sourceId, skip: !options.enabled }, ], }); + validateOverlayAgainstCatalog(options.catalogDir, candidate); return { sourceId: options.sourceId, - persist: () => persistOverlay(options.overlayPath, candidate), + persist: guardedOverlayPersistence( + { ...options, expectedSourceRevision: options.expectedSourceRevision ?? initialRevision }, + candidate, + ), }; } +export function prepareEnableTransitSource(options: { + catalogDir: string; + overlayPath: string; + sourceId: string; + countries?: string[]; + expectedSourceRevision?: string; +}): { sourceId: string; persist: () => void } { + return prepareSetTransitSourceEnabled({ ...options, enabled: true }); +} +/** Called by the fixed durable transit-sync dispatcher after original target admission. */ +export function prepareTransitSourceMutation( + options: { catalogDir: string; overlayPath: string; countries?: string[] }, + input: import("@openmapx/core/data-manager-contracts").TransitDesiredSourceMutation, +) { + if (input.action === "add") + return prepareAddTransitSource({ + ...options, + expectedSourceRevision: input.expectedSourceRevision, + source: { ...input.source, spec: "gtfs", type: "http", origin: "operator" }, + }); + if (input.action === "remove") + return prepareRemoveTransitSource({ + ...options, + sourceId: input.sourceId, + expectedSourceRevision: input.expectedSourceRevision, + }); + return prepareSetTransitSourceEnabled({ + ...options, + sourceId: input.sourceId, + enabled: input.action === "enable", + expectedSourceRevision: input.expectedSourceRevision, + }); +} export function listPinnedTransitCatalog( catalogDir: string, countries?: string[], From 42a7964f9b8eaf28a6c2749ab8608e3b1d1f8e19 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 21:30:46 +0100 Subject: [PATCH 049/135] refactor(transit): establish paired catalog pin authority --- apps/ops-agent/src/catalog-pins.test.ts | 101 ++++ apps/ops-agent/src/index.ts | 13 + apps/ops-agent/src/server.ts | 1 + .../src/transitous-lock-runtime.test.ts | 34 +- apps/ops-agent/src/transitous-lock-runtime.ts | 37 +- .../cli/__tests__/cli-transit-sources.test.ts | 33 +- .../cli/__tests__/transitous-bump.test.ts | 5 +- packages/cli/src/commands/data.ts | 50 +- packages/cli/src/commands/transitous-bump.ts | 86 ++-- packages/core/package.json | 8 + .../src/build-artifacts/control-authority.ts | 49 +- .../src/build-artifacts/retention-family.ts | 28 ++ .../core/src/build-artifacts/target-lease.ts | 14 + packages/core/src/catalog-pins/bootstrap.ts | 88 ++++ packages/core/src/catalog-pins/generation.ts | 32 ++ packages/core/src/catalog-pins/index.ts | 5 + packages/core/src/catalog-pins/schemas.ts | 67 +++ packages/core/src/catalog-pins/store.test.ts | 384 +++++++++++++++ packages/core/src/catalog-pins/store.ts | 443 ++++++++++++++++++ packages/core/src/catalog-pins/types.ts | 42 ++ packages/core/src/ops/contract.test.ts | 4 + packages/core/src/ops/contract.ts | 21 +- scripts/ops-authority-inventory.json | 8 +- .../__tests__/catalog-pair-read.test.ts | 25 + .../__tests__/gbfs-catalog-lock.test.ts | 2 +- .../__tests__/transitous-lock.test.ts | 11 +- .../__tests__/transitous/compile-gbfs.test.ts | 41 +- .../__tests__/transitous/prepare.test.ts | 67 +-- .../data-manager/src/gbfs-catalog-lock.ts | 20 +- .../src/jobs/transitous/compile-gbfs.ts | 5 +- .../src/jobs/transitous/prepare.ts | 37 +- .../data-manager/src/jobs/transitous/types.ts | 2 + services/data-manager/src/transitous-lock.ts | 22 +- 33 files changed, 1592 insertions(+), 193 deletions(-) create mode 100644 apps/ops-agent/src/catalog-pins.test.ts create mode 100644 packages/core/src/catalog-pins/bootstrap.ts create mode 100644 packages/core/src/catalog-pins/generation.ts create mode 100644 packages/core/src/catalog-pins/index.ts create mode 100644 packages/core/src/catalog-pins/schemas.ts create mode 100644 packages/core/src/catalog-pins/store.test.ts create mode 100644 packages/core/src/catalog-pins/store.ts create mode 100644 packages/core/src/catalog-pins/types.ts create mode 100644 services/data-manager/__tests__/catalog-pair-read.test.ts diff --git a/apps/ops-agent/src/catalog-pins.test.ts b/apps/ops-agent/src/catalog-pins.test.ts new file mode 100644 index 000000000..6563b77c6 --- /dev/null +++ b/apps/ops-agent/src/catalog-pins.test.ts @@ -0,0 +1,101 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { createCatalogPinSet, initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; +import { OpsClient } from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { createRegistryResourcePolicy, denyAllTrustedOpsData } from "./policy"; +import { createUnavailableRuntime } from "./runtime"; +import { buildOpsAgentServer } from "./server"; +import { createTransitousLockRuntime } from "./transitous-lock-runtime"; + +it("serializes the largest legal paired pins through the actual HTTP server/client for both read roles", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-pins-http-")), + infra = join(root, "infra/docker"); + mkdirSync(join(infra, "data"), { recursive: true, mode: 0o700 }); + const apiToken = Buffer.alloc(32, 1).toString("base64url"), + dmToken = Buffer.alloc(32, 2).toString("base64url"); + const t = { + ref: `${"a".repeat(215)}@${"b".repeat(40)}`, + submodules: Object.fromEntries( + Array.from({ length: 128 }, (_, n) => [ + `${"x".repeat(253)}${String(n).padStart(3, "0")}`, + "c".repeat(40), + ]), + ), + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "x".repeat(128), + comment: "💠".repeat(256), + }; + const commit = "d".repeat(40), + g = { + schemaVersion: 1 as const, + source: "mobilitydata-gbfs" as const, + commit, + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${commit}/systems.csv`, + sha256: "e".repeat(64), + lockedAt: t.lockedAt, + lockedBy: t.lockedBy, + }; + writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(t)); + writeFileSync(join(infra, "gbfs-catalog.lock.json"), JSON.stringify(g)); + await provisionControlInstallation(infra); + const installation = openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["motis"], + }); + const runtime = createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: root }); + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dmToken }, + runtime, + resourcePolicy: createRegistryResourcePolicy({ + serviceIds: ["motis"], + integrationIds: [], + trustedData: denyAllTrustedOpsData, + }), + }); + try { + await initializeManagedCatalogPins({ rootDir: root, installation }); + const expected = createCatalogPinSet(t, g); + const baseUrl = await app.listen({ host: "127.0.0.1", port: 0 }); + for (const [role, token] of [ + ["api", apiToken], + ["data-manager", dmToken], + ]) { + const tokenFile = join(root, `${role}.token`); + writeFileSync(tokenFile, token, { mode: 0o600 }); + const client = new OpsClient({ baseUrl, tokenFile }); + const response = await client.execute({ kind: "catalogPins.inspect" }); + expect(response).toEqual({ + execution: "sync", + kind: "catalogPins.inspect", + value: { active: expected, catalogRevisionId: expected.generationId }, + }); + expect(Buffer.byteLength(JSON.stringify(response))).toBeGreaterThan(38000); + if (role === "api") + await expect( + client.execute({ + kind: "transitousLock.propose", + ref: t.ref, + submodules: {}, + lockedBy: "fixture", + }), + ).rejects.toMatchObject({ errorClass: "authorization" }); + else + await expect(client.execute({ kind: "configuration.inspect" })).rejects.toMatchObject({ + errorClass: "authorization", + }); + } + expect(existsSync(join(infra, "transitous.lock.proposed.json"))).toBe(false); + } finally { + await app.close(); + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } +}, 30000); diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 9b969c882..019728534 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -1,6 +1,7 @@ import { constants, existsSync } from "node:fs"; import { open } from "node:fs/promises"; import { loadControlStartup } from "@openmapx/core/build-artifacts"; +import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; import { readOpsTokenFile } from "@openmapx/core/ops"; import { repoPaths, services } from "@openmapx/core/server"; import { @@ -135,6 +136,18 @@ async function main(): Promise { container: existsSync("/run/secrets/openmapx-control-verifiers"), }); if (controlStartup.reason) console.error(controlStartup.reason); + if (controlStartup.installation) { + try { + await initializeManagedCatalogPins({ + rootDir: config.rootDir, + installation: controlStartup.installation, + }); + } catch { + console.error( + "Paired catalog initialization unavailable; preserve original ownership and reconcile before transit operations", + ); + } + } const terminalClaimCleanup = await initializeTerminalClaimCleanup( config.trustedConfigDirectory, { diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 10a27fcd4..909e79caa 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -201,6 +201,7 @@ const RELEASE_RECOVERY_READ_KINDS = new Set([ "data.inspect", "valhalla.traffic.inspect", "postgis.capacity.inspect", + "catalogPins.inspect", "transitousLock.inspect", "gbfsCatalogLock.inspect", ]); diff --git a/apps/ops-agent/src/transitous-lock-runtime.test.ts b/apps/ops-agent/src/transitous-lock-runtime.test.ts index 493b4b713..2f4d55aeb 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.test.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.test.ts @@ -1,6 +1,12 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; import { afterEach, describe, expect, it, vi } from "vitest"; import { testExecution } from "./__fixtures__/execution"; import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; @@ -122,6 +128,32 @@ describe("agent-owned Transitous catalog lock", () => { comment: "pinned", }), ); + const commit = "b".repeat(40); + writeFileSync( + join(rootDir, "infra/docker/gbfs-catalog.lock.json"), + JSON.stringify({ + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit, + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${commit}/systems.csv`, + sha256: "c".repeat(64), + lockedAt: "2026-01-01T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + const infra = join(rootDir, "infra/docker"); + mkdirSync(join(infra, "data"), { mode: 0o700 }); + await provisionControlInstallation(infra); + const installation = openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["motis"], + }); + try { + await initializeManagedCatalogPins({ rootDir, installation }); + } finally { + closeControlInstallation(installation); + } const result = (await dispatchOpsOperation( runtimeFor(rootDir), { kind: "transitousLock.inspect" }, @@ -135,6 +167,6 @@ describe("agent-owned Transitous catalog lock", () => { writeFileSync(lockPath(rootDir), `{"ref":"${"x".repeat(300_000)}"}`); await expect( dispatchOpsOperation(runtimeFor(rootDir), { kind: "transitousLock.inspect" }, context()), - ).rejects.toThrow(/too large/i); + ).rejects.toThrow(/not wired/i); }); }); diff --git a/apps/ops-agent/src/transitous-lock-runtime.ts b/apps/ops-agent/src/transitous-lock-runtime.ts index 17e6b745b..95552ebea 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.ts @@ -12,6 +12,7 @@ import { writeFileSync, } from "node:fs"; import { dirname, join } from "node:path"; +import { catalogRevisionId, readCatalogPinSet } from "@openmapx/core/catalog-pins"; import type { OpsResultFor } from "@openmapx/core/ops"; import { OpsNotWiredError, type OpsRuntime } from "./runtime"; @@ -98,29 +99,31 @@ export function createTransitousLockRuntime( options: TransitousLockRuntimeOptions, ): OpsRuntime { const now = options.now ?? (() => new Date()); - const lockPath = join(options.rootDir, LOCK_RELATIVE_PATH); const proposedPath = join(options.rootDir, PROPOSED_RELATIVE_PATH); + const pair = () => { + const active = readCatalogPinSet(options.rootDir); + if ( + !active && + (existsSync(join(options.rootDir, LOCK_RELATIVE_PATH)) || + existsSync(join(options.rootDir, GBFS_LOCK_RELATIVE_PATH))) + ) + throw new OpsNotWiredError(); + return active; + }; + runtime["catalogPins.inspect"] = async () => { + const active = pair(); + return { active, catalogRevisionId: active ? catalogRevisionId(active) : null }; + }; runtime["transitousLock.inspect"] = async () => ({ - active: readLockAt(lockPath), + active: pair()?.transitous ?? null, proposed: readLockAt(proposedPath), }); - runtime["gbfsCatalogLock.inspect"] = async () => { - const path = join(options.rootDir, GBFS_LOCK_RELATIVE_PATH); - if (!existsSync(path)) throw new Error("GBFS catalog lock is missing"); - const raw = readFileSync(path, "utf-8"); - if (Buffer.byteLength(raw, "utf8") > MAX_LOCK_BYTES) { - throw new Error("GBFS catalog lock is too large"); - } - const parsed = JSON.parse(raw) as Record; - return { - commit: String(parsed.commit ?? ""), - url: String(parsed.url ?? ""), - sha256: String(parsed.sha256 ?? ""), - lockedAt: String(parsed.lockedAt ?? ""), - lockedBy: String(parsed.lockedBy ?? ""), - }; + const active = pair(); + if (!active) throw new OpsNotWiredError(); + const { commit, url, sha256, lockedAt, lockedBy } = active.gbfs; + return { commit, url, sha256, lockedAt, lockedBy }; }; runtime["transitousLock.propose"] = async (operation) => { diff --git a/packages/cli/__tests__/cli-transit-sources.test.ts b/packages/cli/__tests__/cli-transit-sources.test.ts index 18d14c6be..afdc03869 100644 --- a/packages/cli/__tests__/cli-transit-sources.test.ts +++ b/packages/cli/__tests__/cli-transit-sources.test.ts @@ -1,7 +1,37 @@ import { Command } from "commander"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { registerDataCommands } from "../src/commands/data"; +afterEach(() => vi.restoreAllMocks()); +it.each(["enable", "remove", "disable"])( + "reads and sends the exact desired revision when the user requests source %s", + async (action) => { + const revision = "a".repeat(64), + requests: Array<{ url: string; method: string; body: unknown }> = []; + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = String(input), + method = init?.method ?? "GET"; + requests.push({ url, method, body: init?.body ? JSON.parse(String(init.body)) : null }); + return Response.json( + method === "GET" + ? { sourceRevision: revision, sources: [], total: 0, limit: 50, offset: 0 } + : { jobId: "fixture-job", sourceId: "operator:de:demo", status: "started" }, + { status: method === "GET" ? 200 : 202 }, + ); + }); + const program = new Command(); + program.exitOverride(); + registerDataCommands(program); + await program.parseAsync(["data", "source", action, "operator:de:demo"], { from: "user" }); + expect(requests).toHaveLength(2); + expect(requests[0].url).not.toContain("transport="); + expect(requests[1].body).toEqual({ expectedSourceRevision: revision }); + expect(requests[1].method).toBe(action === "remove" ? "DELETE" : "POST"); + if (action !== "remove") expect(requests[1].url).toContain(`/${action}`); + }, +); + describe("transit source CLI", () => { it("exposes lifecycle commands without direct feed import/remove commands", () => { const program = new Command(); @@ -23,6 +53,7 @@ describe("transit source CLI", () => { "list", "add", "remove", + "disable", "enable", ]); }); diff --git a/packages/cli/__tests__/transitous-bump.test.ts b/packages/cli/__tests__/transitous-bump.test.ts index 3faf21539..5b55d9377 100644 --- a/packages/cli/__tests__/transitous-bump.test.ts +++ b/packages/cli/__tests__/transitous-bump.test.ts @@ -63,7 +63,7 @@ describe("Transitous combined catalog bump", () => { expect(combinedPinsAreCurrent(TRANSITOUS, GBFS, "a".repeat(40), "f".repeat(40))).toBe(false); }); - it("restores both previous locks if replacement fails midway", () => { + it("refuses unvalidated direct activation before replacing either compatibility file", () => { temporaryRoot = mkdtempSync(join(tmpdir(), "openmapx-bump-lockset-")); const directory = join(temporaryRoot, "infra", "docker"); mkdirSync(directory, { recursive: true }); @@ -80,7 +80,8 @@ describe("Transitous combined catalog bump", () => { return renameSync(from, to); }, }), - ).toThrow(/previous pins restored/); + ).toThrow(/validated paired approval/); + expect(renameCalls).toBe(0); expect(readFileSync(transitousPath, "utf-8")).toBe("old-transitous\n"); expect(readFileSync(gbfsPath, "utf-8")).toBe("old-gbfs\n"); }); diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index 0a2b6a086..86b70f528 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -598,6 +598,9 @@ export function registerDataCommands(program: Command): void { const result = dm.unwrapDataManagerResponse( await dm.addTransitSource( { + expectedSourceRevision: dm.unwrapDataManagerResponse( + await dm.getTransitSources(undefined, { transport: createCliTransport() }), + ).sourceRevision, url, name: options.name, region: options.region, @@ -621,11 +624,19 @@ export function registerDataCommands(program: Command): void { source .command("remove ") - .description("Disable a desired source and start a transactional sync") + .description("Remove operator source metadata and start a transactional sync") .action(async (sourceId: string) => { try { const result = dm.unwrapDataManagerResponse( - await dm.removeTransitSource(sourceId, {}, { transport: createCliTransport() }), + await dm.removeTransitSource( + sourceId, + { + expectedSourceRevision: dm.unwrapDataManagerResponse( + await dm.getTransitSources(undefined, { transport: createCliTransport() }), + ).sourceRevision, + }, + { transport: createCliTransport() }, + ), ); log.ok(`Source ${result.sourceId} removal queued: jobId=${result.jobId}`); } catch (err) { @@ -635,13 +646,44 @@ export function registerDataCommands(program: Command): void { } }); + source + .command("disable ") + .description("Reversibly disable a source and start a transactional sync") + .action(async (sourceId: string) => { + try { + const expectedSourceRevision = dm.unwrapDataManagerResponse( + await dm.getTransitSources(undefined, { transport: createCliTransport() }), + ).sourceRevision; + const result = dm.unwrapDataManagerResponse( + await dm.disableTransitSource( + sourceId, + { expectedSourceRevision }, + { transport: createCliTransport() }, + ), + ); + log.ok(`Source ${result.sourceId} disable queued: jobId=${result.jobId}`); + } catch (err) { + log.err(`source disable failed: ${(err as Error).message}`); + dataManagerHint(); + completeCliCommand(1); + } + }); + source .command("enable ") - .description("Enable a catalog source and start a transactional sync") + .description("Re-enable a retained source and start a transactional sync") .action(async (sourceId: string) => { try { const result = dm.unwrapDataManagerResponse( - await dm.enableTransitSource(sourceId, {}, { transport: createCliTransport() }), + await dm.enableTransitSource( + sourceId, + { + expectedSourceRevision: dm.unwrapDataManagerResponse( + await dm.getTransitSources(undefined, { transport: createCliTransport() }), + ).sourceRevision, + }, + { transport: createCliTransport() }, + ), ); log.ok(`Source ${result.sourceId} enable queued: jobId=${result.jobId}`); } catch (err) { diff --git a/packages/cli/src/commands/transitous-bump.ts b/packages/cli/src/commands/transitous-bump.ts index e31520966..f723925e0 100644 --- a/packages/cli/src/commands/transitous-bump.ts +++ b/packages/cli/src/commands/transitous-bump.ts @@ -1,18 +1,26 @@ import { createHash } from "node:crypto"; -import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { existsSync, readFileSync, renameSync, type unlinkSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { createInterface } from "node:readline/promises"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { + type CatalogPinSet, + initializeManagedCatalogPins, + readCatalogPinSet, +} from "@openmapx/core/catalog-pins"; import { parseMobilityDataGbfsCsv } from "@openmapx/transitous-core"; import type { Command } from "commander"; import { execa } from "execa"; import { decodeGbfsCatalogLock, type GbfsCatalogLock, - readGbfsCatalogLock, } from "../../../../services/data-manager/src/gbfs-catalog-lock"; import { parseRefShaPair, - readTransitousLock, type TransitousLock, } from "../../../../services/data-manager/src/transitous-lock"; import { log } from "../lib/output"; @@ -103,41 +111,11 @@ export function writeCombinedCatalogLocks( remove?: typeof unlinkSync; } = {}, ): void { - const write = operations.write ?? writeFileSync; - const rename = operations.rename ?? renameSync; - const remove = operations.remove ?? unlinkSync; - const transitousPath = join(repoRoot, "infra", "docker", "transitous.lock.json"); - const gbfsPath = join(repoRoot, "infra", "docker", "gbfs-catalog.lock.json"); - const suffix = `.candidate-${process.pid}`; - const stagedTransitous = `${transitousPath}${suffix}`; - const stagedGbfs = `${gbfsPath}${suffix}`; - const previousTransitous = existsSync(transitousPath) ? readFileSync(transitousPath) : null; - const previousGbfs = existsSync(gbfsPath) ? readFileSync(gbfsPath) : null; - let transitousReplaced = false; - let gbfsReplaced = false; - try { - write(stagedTransitous, transitousLockJson(transitous), "utf-8"); - write(stagedGbfs, gbfsLockJson(gbfs), "utf-8"); - rename(stagedTransitous, transitousPath); - transitousReplaced = true; - rename(stagedGbfs, gbfsPath); - gbfsReplaced = true; - } catch (error) { - if (transitousReplaced) { - if (previousTransitous) write(transitousPath, previousTransitous); - else if (existsSync(transitousPath)) remove(transitousPath); - } - if (gbfsReplaced) { - if (previousGbfs) write(gbfsPath, previousGbfs); - else if (existsSync(gbfsPath)) remove(gbfsPath); - } - for (const path of [stagedTransitous, stagedGbfs]) { - if (existsSync(path)) remove(path); - } - throw new Error( - `Combined catalog lock update failed; previous pins restored: ${(error as Error).message}`, - ); - } + void repoRoot; + void transitous; + void gbfs; + void operations; + throw new Error("Catalog pin activation requires validated paired approval"); } /** Write a reviewable pin-set proposal without mutating active lockfiles. */ @@ -372,7 +350,20 @@ export function registerTransitousCommands(program: Command): void { "transitland-atlas", ); - const existing = readTransitousLock(paths.root); + if (!existsSync(join(paths.infraDir, "secrets/control-verifiers.json"))) + await provisionControlInstallation(paths.infraDir); + const installation = openControlInstallation({ + infraDir: paths.infraDir, + role: "host-cli", + serviceIds: ["motis"], + }); + let active: CatalogPinSet | null; + try { + active = await initializeManagedCatalogPins({ rootDir: paths.root, installation }); + } finally { + closeControlInstallation(installation); + } + const existing = active?.transitous ?? null; const lockedBy = await resolveLockedBy(); let gbfsCandidate: Awaited>; try { @@ -383,12 +374,7 @@ export function registerTransitousCommands(program: Command): void { ); process.exit(1); } - let existingGbfs: GbfsCatalogLock | null = null; - try { - existingGbfs = readGbfsCatalogLock(paths.root); - } catch { - // First combined bump creates it after confirmation. - } + const existingGbfs = active?.gbfs ?? null; if (combinedPinsAreCurrent(existing, existingGbfs, newSha, gbfsCandidate.lock.commit)) { log.ok( `Already pinned to Transitous ${newSha.slice(0, 12)} and GBFS ${gbfsCandidate.lock.commit.slice(0, 12)} — nothing to do.`, @@ -448,13 +434,11 @@ export function registerTransitousCommands(program: Command): void { .description("Print the current Transitous lockfile contents") .action(() => { const paths = repoPaths(); - const lockPath = join(paths.infraDir, "transitous.lock.json"); - if (!existsSync(lockPath)) { - log.warn( - "No infra/docker/transitous.lock.json found. Run `pnpm openmapx transitous bump` to create one.", - ); + const pins = readCatalogPinSet(paths.root); + if (!pins) { + log.warn("Paired catalog pins are not initialized; run the managed bump workflow"); return; } - console.log(readFileSync(lockPath, "utf-8")); + console.log(JSON.stringify(pins, null, 2)); }); } diff --git a/packages/core/package.json b/packages/core/package.json index 31fd9a126..e97180f1b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -201,6 +201,14 @@ "./ops/control-root-key": { "types": "./src/ops/control-root-key.ts", "default": "./src/ops/control-root-key.ts" + }, + "./catalog-pins": { + "types": "./src/catalog-pins/index.ts", + "default": "./src/catalog-pins/index.ts" + }, + "./catalog-pins/types": { + "types": "./src/catalog-pins/types.ts", + "default": "./src/catalog-pins/types.ts" } }, "scripts": { diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index c851daaec..fd24717ed 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -503,6 +503,7 @@ export function writePrivateRecord( "execution-adoptions", "configuration-maintenance", "hardlinks", + "catalog-commits", ].includes(namespace) ) throw Error("Unknown fixed authority namespace"); @@ -1044,6 +1045,7 @@ export function retainAdmittedControlExecution( "notable-places-build", "catalog.activate", "catalog.validate", + "catalog-pins.initialize", "artifactTransfer.import", "artifactTransfer.export", "artifactTransfer.validate", @@ -1104,6 +1106,18 @@ export function retainAdmittedControlExecution( plan, reservations, }); + if ( + input.kind === "catalog-pins.initialize" && + (state.fixedChildId || + input.parent || + plan.length || + reservations.length || + serviceIds.length !== 1 || + serviceIds[0] !== "motis" || + !["ops-agent", "host-cli"].includes(installation.role) || + controlDigest(input.operation) !== controlDigest({ kind: "catalog-pins.initialize" })) + ) + throw Error("Fixed local catalog bootstrap only"); const budget = (() => { try { return opsExecutionMaximumMs(input.kind); @@ -1116,6 +1130,7 @@ export function retainAdmittedControlExecution( ].includes(input.kind) ) return 3600000; + if (input.kind === "catalog-pins.initialize") return 60000; if ( ["configuration-maintenance.prepare", "configuration-maintenance.settle"].includes( input.kind, @@ -1512,6 +1527,7 @@ const settlementSchema = z.strictObject({ terminalReceiptDigest: digest, }); const effectObservationSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("catalog-pins-initialized"), generationId: digest.nullable() }), z.strictObject({ kind: z.literal("configuration-generation-not-committed"), revisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), @@ -1928,7 +1944,38 @@ export function retainControlEffectReceipt( const receipt = parsed.data; let outcome: ChildSettlement["outcome"] = "unknown", publication: ChildSettlement["publication"] = "unknown"; - if (receipt.kind === "ops-result") { + if (receipt.kind === "catalog-pins-initialized") { + if ( + s.kind !== "catalog-pins.initialize" || + s.installation.fixedChildId || + !s.origin || + !["ops-agent", "host-cli"].includes(s.installation.installation.role) + ) + throw Error("Original local catalog initialization required"); + const source = verifyControl( + s.installation.installation.publicKey, + "openmapx-catalog-pin-initialization-v1", + readPrivateRecord( + s.installation, + "catalog-commits", + `initialization-${s.execution.operationKey}.json`, + 65536, + ), + z.strictObject({ + version: z.literal(1), + execution: executionRefSchema, + generationId: digest.nullable(), + pointerDigest: digest.nullable(), + }), + ); + if ( + controlDigest(source.execution) !== controlDigest(s.execution) || + source.generationId !== receipt.generationId + ) + throw Error("Catalog initialization source mismatch"); + outcome = "committed"; + publication = "committed"; + } else if (receipt.kind === "ops-result") { receipt.result = parseBoundedOpsResult(s.kind as OpsOperationKind, receipt.result); if (s.kind === "data.update") { const actual = readDataWorkflowProgress(authority), diff --git a/packages/core/src/build-artifacts/retention-family.ts b/packages/core/src/build-artifacts/retention-family.ts index fc868743c..eac86fc14 100644 --- a/packages/core/src/build-artifacts/retention-family.ts +++ b/packages/core/src/build-artifacts/retention-family.ts @@ -124,6 +124,7 @@ const PRIVATE = new Set([ "execution-adoptions", "configuration-maintenance", "hardlinks", + "catalog-commits", ]); const recordMaximum = (record: { scope: string; namespace: string }) => record.scope === "shared" && record.namespace === "service-lifecycle" @@ -1123,6 +1124,33 @@ export function verifyFamilyOwnedRecord( family.close(); } } +/** Catalog recovery may finish only the exact terminal cache preimage already + * indexed before a lost rename. This does not authorize pointer publication. */ +export function verifyCatalogPinReceiptRepairPreimage( + state: InstallationState, + origin: ControlExecutionOrigin, + operationKey: string, + bytes: string, +): void { + if (!/^opk1_[A-Za-z0-9_-]{16,64}$/.test(operationKey) || Buffer.byteLength(bytes) > 65536) + throw Error("Invalid catalog receipt repair"); + const f = openFamily(state, origin); + try { + guard(f.slot, ".family-writer", () => { + checkLocalIdentity(f); + const record = findRecord(f, { + scope: "private", + namespace: "catalog-commits", + name: `${operationKey}.json`, + container: "", + }); + if (!record || record.digest !== controlDigest(bytes) || record.state !== "written") + throw Error("Original indexed catalog receipt preimage required"); + }); + } finally { + f.close(); + } +} /** Fixed normal coordinator source observation. Unlike ordinary indexed * readers, the reciprocal target check runs under the configuration writer and * must neither acquire a family writer nor change any metadata. Derive its one diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index f036a7373..ecde3cecb 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -2937,6 +2937,20 @@ export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): v if (!node || !forwardAncestry(t, s.node) || Date.now() >= s.execution.deadlineAt) throw Error("Target lease permits observation/settlement only"); } +/** Fixed catalog-store adapter; caller DTOs cannot construct physical ownership. */ +export function catalogPinTargetOwner(lease: TargetLeaseAuthority) { + assertArtifactTargetLeaseForward(lease); + const s = stateOf(lease); + if (!s.certificate.serviceIds.includes("motis") || !s.certificate.origin) + throw Error("MOTIS target authority required for catalog pins"); + if (!["ops-agent", "host-cli"].includes(s.installation.installation.role)) + throw Error("Catalog pins are owned by the agent or standalone host CLI"); + return { + installation: s.installation.installation, + execution: s.execution, + origin: s.certificate.origin, + }; +} export function assertArtifactTargetLeaseExecution( lease: TargetLeaseAuthority, execution: ExecutionRef, diff --git a/packages/core/src/catalog-pins/bootstrap.ts b/packages/core/src/catalog-pins/bootstrap.ts new file mode 100644 index 000000000..b9b83004b --- /dev/null +++ b/packages/core/src/catalog-pins/bootstrap.ts @@ -0,0 +1,88 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { + beginControlEffect, + controlDigest, + executionAuthorityState, + installationState, + retainAdmittedControlExecution, + retainControlEffectReceipt, + verifyControlEffectSettlement, +} from "../build-artifacts/control-authority"; +import type { ControlInstallation } from "../build-artifacts/control-records"; +import { + type ArtifactTargetLease, + acquireArtifactTargetLease, + closeArtifactTargetLeaseHandle, + releaseArtifactTargetLease, + settleArtifactTargetLeaseChild, +} from "../build-artifacts/target-lease"; +import { createControlRootOperationKey } from "../ops/control-root-key"; +import { executionFingerprint } from "../ops/execution-identity"; +import { + initializeCatalogPinSet, + preflightCatalogPinInitialization, + readCatalogPinSet, +} from "./store"; +/** Local startup/CLI bootstrap only: no HTTP operation, child edge or activation. */ +export async function initializeManagedCatalogPins(options: { + rootDir: string; + installation: ControlInstallation; +}) { + const current = readCatalogPinSet(options.rootDir); + if (current) return current; + if (!preflightCatalogPinInitialization(options.rootDir)) return null; + const role = options.installation.role; + if (role !== "ops-agent" && role !== "host-cli") + throw Error("Catalog initialization is host-owned"); + const operation = { kind: "catalog-pins.initialize" }, + admittedAt = Date.now(), + deadlineAt = admittedAt + 60000, + hash = controlDigest(operation); + const authority = retainAdmittedControlExecution(options.installation, { + execution: { + executor: role, + principal: role === "ops-agent" ? "api" : "host-cli", + executionId: role === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint: hash, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: hash, + deadlineAt, + }), + }, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis"], + }); + let lease: ArtifactTargetLease | undefined; + try { + lease = await acquireArtifactTargetLease({ + dataRoot: installationState(options.installation).dataRoot, + serviceIds: ["motis"], + owner: authority, + }); + beginControlEffect(authority); + const pins = await initializeCatalogPinSet({ rootDir: options.rootDir, lease }); + retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "catalog-pins-initialized", generationId: pins?.generationId ?? null }, + }); + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement( + options.installation, + executionAuthorityState(authority).execution, + ), + ); + await releaseArtifactTargetLease(lease); + lease = undefined; + return pins; + } finally { + if (lease) closeArtifactTargetLeaseHandle(lease); + } +} diff --git a/packages/core/src/catalog-pins/generation.ts b/packages/core/src/catalog-pins/generation.ts new file mode 100644 index 000000000..ef887ab55 --- /dev/null +++ b/packages/core/src/catalog-pins/generation.ts @@ -0,0 +1,32 @@ +import { createHash } from "node:crypto"; +import { canonicalJson } from "../ops/canonical-json"; +import { + catalogPinSetSchema, + gbfsPinSchema, + MAX_CATALOG_PIN_BYTES, + transitousPinSchema, +} from "./schemas"; +import type { CatalogPinSet, GbfsCatalogLock, TransitousLock } from "./types"; +export function parseCatalogPinSet(value: unknown): CatalogPinSet { + const pair = catalogPinSetSchema.parse(value), + { generationId, ...body } = pair; + if (createHash("sha256").update(canonicalJson(body)).digest("hex") !== generationId) + throw Error("Catalog pin generation hash mismatch"); + if (Buffer.byteLength(canonicalJson(pair)) > MAX_CATALOG_PIN_BYTES) + throw Error("Catalog pin generation is too large"); + return pair; +} +export function createCatalogPinSet( + transitous: TransitousLock, + gbfs: GbfsCatalogLock, +): CatalogPinSet { + const body = { + schemaVersion: 1 as const, + transitous: transitousPinSchema.parse(transitous), + gbfs: gbfsPinSchema.parse(gbfs), + }; + return parseCatalogPinSet({ + ...body, + generationId: createHash("sha256").update(canonicalJson(body)).digest("hex"), + }); +} diff --git a/packages/core/src/catalog-pins/index.ts b/packages/core/src/catalog-pins/index.ts new file mode 100644 index 000000000..8cba2c995 --- /dev/null +++ b/packages/core/src/catalog-pins/index.ts @@ -0,0 +1,5 @@ +export * from "./bootstrap"; +export * from "./generation"; +export * from "./schemas"; +export * from "./store"; +export * from "./types"; diff --git a/packages/core/src/catalog-pins/schemas.ts b/packages/core/src/catalog-pins/schemas.ts new file mode 100644 index 000000000..28f88f4de --- /dev/null +++ b/packages/core/src/catalog-pins/schemas.ts @@ -0,0 +1,67 @@ +import { z } from "zod"; +import { executionRefSchema } from "../ops/execution-ref"; +export const MAX_CATALOG_PIN_BYTES = 512 * 1024; +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const stamp = z.iso.datetime({ offset: true }); +export const transitousPinSchema = z.strictObject({ + ref: z + .string() + .max(256) + .regex(/^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$/), + submodules: z + .record(z.string().min(1).max(256), z.string().regex(/^[a-f0-9]{40}$/)) + .refine((v) => Object.keys(v).length <= 128), + lockedAt: stamp, + lockedBy: z.string().min(1).max(128), + comment: z.string().max(512).optional(), +}); +export const gbfsPinSchema = z + .strictObject({ + schemaVersion: z.literal(1), + source: z.literal("mobilitydata-gbfs"), + commit: z.string().regex(/^[a-f0-9]{40}$/), + url: z.string().max(256), + sha256: digest, + lockedAt: stamp, + lockedBy: z.string().min(1).max(128), + }) + .refine( + (v) => v.url === `https://raw.githubusercontent.com/MobilityData/gbfs/${v.commit}/systems.csv`, + "GBFS URL must pin its exact commit", + ); +export const catalogPinSetSchema = z.strictObject({ + schemaVersion: z.literal(1), + generationId: digest, + transitous: transitousPinSchema, + gbfs: gbfsPinSchema, +}); +export const catalogPinCommitIdentitySchema = z.strictObject({ + commitId: z.uuid(), + execution: executionRefSchema, + proposalId: z.uuid(), + validationId: z.uuid(), + diffDigest: digest, +}); +export const catalogPinCommitReceiptSchema = z.strictObject({ + version: z.literal(1), + identity: catalogPinCommitIdentitySchema, + previousGenerationId: digest.nullable(), + generationId: digest, + phase: z.enum(["prepared", "committed", "not_committed", "unknown"]), + committedAt: stamp.nullable(), +}); +export const catalogPinPointerSchema = z.strictObject({ + version: z.literal(1), + generationId: digest, + commitId: z.uuid(), + operationKey: executionRefSchema.shape.operationKey, + fingerprint: digest, + committedAt: stamp, +}); +export function decodeLegacyPin(bytes: Buffer, schema: z.ZodType): T { + const value: unknown = JSON.parse(bytes.toString("utf8")); + if (!value || typeof value !== "object" || Array.isArray(value)) + throw Error("Catalog lock must be an object"); + const { $schema: _compatibility, ...pin } = value as Record; + return schema.parse(pin); +} diff --git a/packages/core/src/catalog-pins/store.test.ts b/packages/core/src/catalog-pins/store.test.ts new file mode 100644 index 000000000..e544a8f9e --- /dev/null +++ b/packages/core/src/catalog-pins/store.test.ts @@ -0,0 +1,384 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import * as fs from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readSync: vi.fn(actual.readSync), renameSync: vi.fn(actual.renameSync) }; +}); + +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { + type ControlInstallation, + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, + retainAdmittedControlExecution, +} from "../build-artifacts/control-authority"; +import { + type ArtifactTargetLease, + acquireArtifactTargetLease, + closeArtifactTargetLeaseHandle, +} from "../build-artifacts/target-lease"; +import { createControlRootOperationKey } from "../ops/control-root-key"; +import { executionFingerprint } from "../ops/execution-identity"; +import { opsOperationFingerprint } from "../ops/fingerprint"; +import { createCatalogPinSet } from "./generation"; +import { + initializeCatalogPinSet, + publishCatalogPinSet, + readCatalogPinCommit, + readCatalogPinSet, +} from "./store"; +import type { CatalogPinCommitIdentity } from "./types"; + +const nativeFs = await vi.importActual("node:fs"); +let root: string, docker: string, installation: ControlInstallation, lease: ArtifactTargetLease; +const transitous = { + ref: `main@${"a".repeat(40)}`, + submodules: { "transitland-atlas": "b".repeat(40) }, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", +}; +const gbfs = { + schemaVersion: 1 as const, + source: "mobilitydata-gbfs" as const, + commit: "c".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"c".repeat(40)}/systems.csv`, + sha256: "d".repeat(64), + lockedAt: transitous.lockedAt, + lockedBy: "fixture", +}; +function legacy() { + writeFileSync( + join(docker, "transitous.lock.json"), + JSON.stringify({ $schema: "legacy", ...transitous }), + ); + writeFileSync( + join(docker, "gbfs-catalog.lock.json"), + JSON.stringify({ $schema: "legacy", ...gbfs }), + ); +} +function identity(): CatalogPinCommitIdentity { + return { + commitId: randomUUID(), + execution: lease.owner, + proposalId: randomUUID(), + validationId: randomUUID(), + diffDigest: "e".repeat(64), + }; +} +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-catalog-pair-")); + docker = join(root, "infra/docker"); + mkdirSync(join(docker, "data"), { recursive: true, mode: 0o700 }); + await provisionControlInstallation(docker); + installation = openControlInstallation({ + infraDir: docker, + role: "ops-agent", + serviceIds: ["motis"], + }); + const operation = { kind: "service.build" as const, serviceId: "motis" }, + hash = opsOperationFingerprint(operation), + admittedAt = Date.now(), + deadlineAt = admittedAt + 60000; + const authority = retainAdmittedControlExecution(installation, { + execution: { + executor: "ops-agent", + principal: "api", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint: hash, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint: hash, + deadlineAt, + }), + }, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis"], + }); + lease = await acquireArtifactTargetLease({ + dataRoot: join(docker, "data"), + serviceIds: ["motis"], + owner: authority, + }); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.mocked(fs.renameSync).mockImplementation(nativeFs.renameSync); + vi.mocked(fs.readSync).mockImplementation(nativeFs.readSync); + closeArtifactTargetLeaseHandle(lease); + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); +}); +it("imports both exact legacy pins once, ignoring later compatibility-file changes", async () => { + legacy(); + expect(readCatalogPinSet(root)).toBeNull(); + const first = await initializeCatalogPinSet({ rootDir: root, lease }); + expect(first).toEqual(createCatalogPinSet(transitous, gbfs)); + const ptr = readFileSync(join(docker, "catalog-pins/current.json")); + writeFileSync(join(docker, "gbfs-catalog.lock.json"), "corrupt"); + expect(await initializeCatalogPinSet({ rootDir: root, lease })).toEqual(first); + expect(readCatalogPinSet(root)).toEqual(first); + expect(readFileSync(join(docker, "catalog-pins/current.json"))).toEqual(ptr); +}); +it("does not initialize from missing or malformed halves", async () => { + expect(await initializeCatalogPinSet({ rootDir: root, lease })).toBeNull(); + legacy(); + fs.unlinkSync(join(docker, "gbfs-catalog.lock.json")); + await expect(initializeCatalogPinSet({ rootDir: root, lease })).rejects.toThrow(/Both/); + for (const patch of [ + { sha256: "bad" }, + { commit: "abcd" }, + { url: "https://example.test/mutable" }, + { lockedAt: "yesterday" }, + ]) { + writeFileSync(join(docker, "gbfs-catalog.lock.json"), JSON.stringify({ ...gbfs, ...patch })); + await expect(initializeCatalogPinSet({ rootDir: root, lease })).rejects.toThrow(); + } + expect(existsSync(join(docker, "catalog-pins/current.json"))).toBe(false); +}); +it("refuses symlinked locks and replacements between descriptor checks", async () => { + legacy(); + renameSync(join(docker, "transitous.lock.json"), join(docker, "saved.json")); + symlinkSync("saved.json", join(docker, "transitous.lock.json")); + await expect(initializeCatalogPinSet({ rootDir: root, lease })).rejects.toThrow(); + fs.unlinkSync(join(docker, "transitous.lock.json")); + renameSync(join(docker, "saved.json"), join(docker, "transitous.lock.json")); + const read = vi.mocked(fs.readSync).getMockImplementation()!; + let seen = 0; + const target = fs.statSync(join(docker, "transitous.lock.json")); + vi.spyOn(fs, "readSync").mockImplementation(((...args: Parameters) => { + const result = Reflect.apply(read, fs, args); + if (fs.fstatSync(args[0]).ino === target.ino && seen++ === 0) { + const path = join(docker, "transitous.lock.json"); + renameSync(path, `${path}.old`); + writeFileSync(path, JSON.stringify(transitous)); + } + return result; + }) as typeof read); + await expect(initializeCatalogPinSet({ rootDir: root, lease })).rejects.toThrow(/changed/); + expect(existsSync(join(docker, "catalog-pins/current.json"))).toBe(false); +}); +it("uses an attempt-bound CAS and retains its receipt after a later pointer change", async () => { + legacy(); + const old = await initializeCatalogPinSet({ rootDir: root, lease }); + if (!old) throw Error("missing fixture pair"); + const next = createCatalogPinSet({ ...transitous, ref: `main@${"f".repeat(40)}` }, gbfs), + commit = identity(); + const receipt = await publishCatalogPinSet({ + rootDir: root, + lease, + pins: next, + expectedGenerationId: old.generationId, + commit, + }); + expect(receipt.phase).toBe("committed"); + expect(readCatalogPinSet(root)).toEqual(next); + expect(await readCatalogPinCommit({ rootDir: root, commit })).toEqual(receipt); + expect( + await publishCatalogPinSet({ + rootDir: root, + lease, + pins: next, + expectedGenerationId: old.generationId, + commit, + }), + ).toEqual(receipt); + await expect( + publishCatalogPinSet({ + rootDir: root, + lease, + pins: old, + expectedGenerationId: next.generationId, + commit, + }), + ).rejects.toThrow(/intent conflict/); + const pointerPath = join(docker, "catalog-pins/current.json"), + pointer = JSON.parse(readFileSync(pointerPath, "utf8")); + writeFileSync(pointerPath, JSON.stringify({ ...pointer, commitId: randomUUID() }), { + mode: 0o600, + }); + expect(await readCatalogPinCommit({ rootDir: root, commit })).toEqual(receipt); +}); +it("recovers rename-before-receipt loss from the exact marker without rewriting the pointer", async () => { + legacy(); + const old = await initializeCatalogPinSet({ rootDir: root, lease }); + if (!old) throw Error("missing fixture pair"); + const next = createCatalogPinSet({ ...transitous, comment: "reviewed" }, gbfs), + commit = identity(); + const rename = nativeFs.renameSync; + let committed = false; + vi.spyOn(fs, "renameSync").mockImplementation((from, to) => { + if (String(to).endsWith("/current.json")) { + rename(from, to); + committed = true; + return; + } + if ( + committed && + fs.statSync(dirname(String(to))).ino === + fs.statSync(join(docker, "control-private/ops-agent/catalog-commits")).ino + ) + throw Error("receipt rename lost"); + rename(from, to); + }); + await expect( + publishCatalogPinSet({ + rootDir: root, + lease, + pins: next, + expectedGenerationId: old.generationId, + commit, + }), + ).rejects.toThrow(/lost/); + vi.restoreAllMocks(); + vi.mocked(fs.renameSync).mockImplementation(nativeFs.renameSync); + const pointerBytes = readFileSync(join(docker, "catalog-pins/current.json")), + receipt = await readCatalogPinCommit({ rootDir: root, commit }); + expect(receipt?.phase).toBe("committed"); + expect( + await publishCatalogPinSet({ + rootDir: root, + lease, + pins: next, + expectedGenerationId: old.generationId, + commit, + }), + ).toEqual(receipt); + expect(readFileSync(join(docker, "catalog-pins/current.json"))).toEqual(pointerBytes); +}); +it("never falls back to legacy files when a selected generation is corrupt", async () => { + legacy(); + const pins = await initializeCatalogPinSet({ rootDir: root, lease }); + if (!pins) throw Error("missing fixture pair"); + writeFileSync(join(docker, `catalog-pins/generations/${pins.generationId}.json`), "{}"); + expect(() => readCatalogPinSet(root)).toThrow(); +}); +it("bootstraps an actual local initialization root and releases its MOTIS ownership", async () => { + const other = mkdtempSync(join(tmpdir(), "omx-catalog-startup-")), + infra = join(other, "infra/docker"); + mkdirSync(join(infra, "data"), { recursive: true, mode: 0o700 }); + await provisionControlInstallation(infra); + const local = openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["motis"], + }); + try { + writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(transitous)); + writeFileSync(join(infra, "gbfs-catalog.lock.json"), JSON.stringify(gbfs)); + const { initializeManagedCatalogPins } = await import("./bootstrap"); + const pins = await initializeManagedCatalogPins({ rootDir: other, installation: local }); + expect(pins).toEqual(createCatalogPinSet(transitous, gbfs)); + expect(existsSync(join(infra, "data/.admin-target-control/locks/motis"))).toBe(false); + expect(await initializeManagedCatalogPins({ rootDir: other, installation: local })).toEqual( + pins, + ); + } finally { + closeControlInstallation(local); + rmSync(other, { recursive: true, force: true }); + } +}); +it("refuses a byte-identical receipt copied to a foreign inode", async () => { + legacy(); + const old = await initializeCatalogPinSet({ rootDir: root, lease }); + if (!old) throw Error("missing fixture pair"); + const pair = createCatalogPinSet({ ...transitous, comment: "copy check" }, gbfs), + commit = identity(); + await publishCatalogPinSet({ + rootDir: root, + lease, + pins: pair, + expectedGenerationId: old.generationId, + commit, + }); + const path = join( + docker, + `control-private/ops-agent/catalog-commits/${commit.execution.operationKey}.json`, + ), + bytes = readFileSync(path); + renameSync(path, `${path}.saved`); + writeFileSync(path, bytes, { mode: 0o600 }); + await expect(readCatalogPinCommit({ rootDir: root, commit })).rejects.toThrow( + /identity|inode|retained/i, + ); +}); +it("retains a conclusive noncommit when the live pointer rename fails before publication", async () => { + legacy(); + const old = await initializeCatalogPinSet({ rootDir: root, lease }); + if (!old) throw Error("missing fixture pair"); + const pair = createCatalogPinSet({ ...transitous, comment: "rename refused" }, gbfs), + commit = identity(); + vi.mocked(fs.renameSync).mockImplementation((from, to) => { + if (String(to).endsWith("/current.json")) throw Error("pointer rename refused"); + nativeFs.renameSync(from, to); + }); + await expect( + publishCatalogPinSet({ + rootDir: root, + lease, + pins: pair, + expectedGenerationId: old.generationId, + commit, + }), + ).rejects.toThrow(/refused/); + vi.mocked(fs.renameSync).mockImplementation(nativeFs.renameSync); + expect((await readCatalogPinCommit({ rootDir: root, commit }))?.phase).toBe("not_committed"); + expect(readCatalogPinSet(root)).toEqual(old); + expect(existsSync(join(docker, "catalog-pins/pending.json"))).toBe(false); +}); +it("does not acquire a startup target root for a malformed legacy pair", async () => { + const other = mkdtempSync(join(tmpdir(), "omx-catalog-invalid-")), + infra = join(other, "infra/docker"); + mkdirSync(join(infra, "data"), { recursive: true, mode: 0o700 }); + await provisionControlInstallation(infra); + const local = openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["motis"], + }); + try { + writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(transitous)); + writeFileSync(join(infra, "gbfs-catalog.lock.json"), "invalid"); + const { initializeManagedCatalogPins } = await import("./bootstrap"); + await expect( + initializeManagedCatalogPins({ rootDir: other, installation: local }), + ).rejects.toThrow(); + expect(existsSync(join(infra, "data/.admin-target-control/locks/motis"))).toBe(false); + } finally { + closeControlInstallation(local); + rmSync(other, { recursive: true, force: true }); + } +}); +it("refuses to initialize another checkout with this checkout's MOTIS lease", async () => { + const other = mkdtempSync(join(tmpdir(), "omx-catalog-foreign-")); + try { + const dir = join(other, "infra/docker"); + mkdirSync(join(dir, "data"), { recursive: true, mode: 0o700 }); + writeFileSync(join(dir, "transitous.lock.json"), JSON.stringify(transitous)); + writeFileSync(join(dir, "gbfs-catalog.lock.json"), JSON.stringify(gbfs)); + await expect(initializeCatalogPinSet({ rootDir: other, lease })).rejects.toThrow( + /root|foreign|checkout/i, + ); + expect(existsSync(join(dir, "catalog-pins/current.json"))).toBe(false); + } finally { + rmSync(other, { recursive: true, force: true }); + } +}); diff --git a/packages/core/src/catalog-pins/store.ts b/packages/core/src/catalog-pins/store.ts new file mode 100644 index 000000000..b0cccffb9 --- /dev/null +++ b/packages/core/src/catalog-pins/store.ts @@ -0,0 +1,443 @@ +import { randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import { + closeControlInstallation, + installationState, + openControlInstallation, + readPrivateRecord, + retainedExecutionOrigin, + writePrivateRecord, +} from "../build-artifacts/control-authority"; +import { controlDigest, signControl, verifyControl } from "../build-artifacts/control-records"; +import { AnchoredDirectory, identity, sameInode } from "../build-artifacts/filesystem"; +import { + verifyCatalogPinReceiptRepairPreimage, + verifyFamilyOwnedRecord, +} from "../build-artifacts/retention-family"; +import { + assertArtifactTargetLeaseExecution, + catalogPinTargetOwner, + type TargetLeaseAuthority, +} from "../build-artifacts/target-lease"; +import { canonicalJson } from "../ops/canonical-json"; +import { createCatalogPinSet, parseCatalogPinSet } from "./generation"; +import { + catalogPinCommitIdentitySchema, + catalogPinCommitReceiptSchema, + catalogPinPointerSchema, + decodeLegacyPin, + gbfsPinSchema, + MAX_CATALOG_PIN_BYTES, + transitousPinSchema, +} from "./schemas"; +import type { CatalogPinCommitIdentity, CatalogPinCommitReceipt, CatalogPinSet } from "./types"; + +const DOMAIN = "openmapx-catalog-pin-commit-v1"; +const journalSchema = z.strictObject({ + version: z.literal(1), + receipt: catalogPinCommitReceiptSchema, + previous: catalogPinPointerSchema.nullable(), + next: catalogPinPointerSchema, +}); +const missing = (error: unknown) => (error as NodeJS.ErrnoException).code === "ENOENT"; +function optionalRead( + dir: AnchoredDirectory, + name: string, + max: number, + privateMode = true, +): Buffer | null { + try { + return dir.read(name, max, privateMode); + } catch (error) { + if (missing(error)) return null; + throw error; + } +} +function directories( + rootDir: string, + create: boolean, + run: (docker: AnchoredDirectory, pins: AnchoredDirectory | null) => T, +): T { + const opened: AnchoredDirectory[] = []; + try { + let dir = new AnchoredDirectory(rootDir); + opened.push(dir); + for (const part of ["infra", "docker"]) { + dir = dir.directory(part); + opened.push(dir); + } + const docker = dir; + let pins: AnchoredDirectory; + try { + pins = docker.directory("catalog-pins", create, true); + opened.push(pins); + } catch (error) { + if (!create && missing(error)) return run(docker, null); + throw error; + } + return run(docker, pins); + } finally { + for (const dir of opened.reverse()) dir.close(); + } +} +function pointer(dir: AnchoredDirectory) { + const bytes = optionalRead(dir, "current.json", 4096); + return bytes ? catalogPinPointerSchema.parse(JSON.parse(bytes.toString("utf8"))) : null; +} +function selected(pins: AnchoredDirectory): CatalogPinSet | null { + const current = pointer(pins); + if (!current) return null; + const generations = pins.directory("generations"); + try { + const pair = parseCatalogPinSet( + JSON.parse( + generations.read(`${current.generationId}.json`, MAX_CATALOG_PIN_BYTES).toString("utf8"), + ), + ); + if (pair.generationId !== current.generationId) + throw Error("Catalog pin pointer/generation mismatch"); + return pair; + } finally { + generations.close(); + } +} +export function readCatalogPinSet(rootDir: string): CatalogPinSet | null { + return directories(rootDir, false, (_docker, pins) => (pins ? selected(pins) : null)); +} +function assertCatalogRoot(rootDir: string, installation: Parameters[0]) { + const expected = new AnchoredDirectory(installationState(installation).dataRoot); + let actual: AnchoredDirectory | undefined; + try { + actual = new AnchoredDirectory(join(rootDir, "infra/docker/data")); + if (!sameInode(identity(expected.stat), identity(actual.stat))) + throw Error("Foreign catalog checkout/data root"); + } finally { + actual?.close(); + expected.close(); + } +} +function writeGeneration(pins: AnchoredDirectory, pair: CatalogPinSet) { + const generations = pins.directory("generations", true, true); + try { + const name = `${pair.generationId}.json`, + existing = optionalRead(generations, name, MAX_CATALOG_PIN_BYTES); + if (existing) { + if ( + canonicalJson(parseCatalogPinSet(JSON.parse(existing.toString("utf8")))) !== + canonicalJson(pair) + ) + throw Error("Immutable catalog generation changed"); + } else generations.write(name, canonicalJson(pair), true); + } finally { + generations.close(); + } +} +export function preflightCatalogPinInitialization(rootDir: string): boolean { + return directories(rootDir, false, (docker, _pins) => { + const transitous = optionalRead(docker, "transitous.lock.json", 256 * 1024, false), + gbfs = optionalRead(docker, "gbfs-catalog.lock.json", 256 * 1024, false); + if (!transitous && !gbfs) return false; + if (!transitous || !gbfs) + throw Error("Both legacy catalog locks are required for paired initialization"); + createCatalogPinSet( + decodeLegacyPin(transitous, transitousPinSchema), + decodeLegacyPin(gbfs, gbfsPinSchema), + ); + return true; + }); +} +/** Bootstrap is separate from validation/activation and never creates an approval receipt. */ +export async function initializeCatalogPinSet(options: { + rootDir: string; + lease: TargetLeaseAuthority; +}): Promise { + const owner = catalogPinTargetOwner(options.lease); + assertCatalogRoot(options.rootDir, owner.installation); + return directories(options.rootDir, true, (docker, pins) => { + if (!pins) throw Error("Catalog pin directory unavailable"); + const record = (pair: CatalogPinSet | null) => { + const state = installationState(owner.installation), + marker = pointer(pins); + writePrivateRecord( + state, + "catalog-commits", + `initialization-${owner.execution.operationKey}.json`, + signControl(state.privateKey, "openmapx-catalog-pin-initialization-v1", { + version: 1, + execution: owner.execution, + generationId: pair?.generationId ?? null, + pointerDigest: marker ? controlDigest(marker) : null, + }), + false, + owner.origin, + ); + return pair; + }; + const current = selected(pins); + if (current) return record(current); + const transitous = optionalRead(docker, "transitous.lock.json", 256 * 1024, false), + gbfs = optionalRead(docker, "gbfs-catalog.lock.json", 256 * 1024, false); + if (!transitous && !gbfs) return record(null); + if (!transitous || !gbfs) + throw Error("Both legacy catalog locks are required for paired initialization"); + const tIdentity = docker.fileIdentity("transitous.lock.json", false), + gIdentity = docker.fileIdentity("gbfs-catalog.lock.json", false); + const pair = createCatalogPinSet( + decodeLegacyPin(transitous, transitousPinSchema), + decodeLegacyPin(gbfs, gbfsPinSchema), + ); + writeGeneration(pins, pair); + catalogPinTargetOwner(options.lease); + if ( + canonicalJson(tIdentity) !== + canonicalJson(docker.fileIdentity("transitous.lock.json", false)) || + canonicalJson(gIdentity) !== + canonicalJson(docker.fileIdentity("gbfs-catalog.lock.json", false)) || + !transitous.equals(docker.read("transitous.lock.json", 256 * 1024, false)) || + !gbfs.equals(docker.read("gbfs-catalog.lock.json", 256 * 1024, false)) + ) + throw Error("Legacy catalog locks changed during initialization"); + pins.write( + "current.json", + canonicalJson( + catalogPinPointerSchema.parse({ + version: 1, + generationId: pair.generationId, + commitId: randomUUID(), + operationKey: owner.execution.operationKey, + fingerprint: owner.execution.fingerprint, + committedAt: new Date().toISOString(), + }), + ), + ); + return record(pair); + }); +} +function readJournal(rootDir: string, commit: CatalogPinCommitIdentity) { + if (commit.execution.executor !== "ops-agent") throw Error("Agent-owned catalog commit required"); + const installation = openControlInstallation({ + infraDir: join(rootDir, "infra/docker"), + role: "ops-agent", + serviceIds: ["motis"], + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const state = installationState(installation), + origin = retainedExecutionOrigin(state, commit.execution); + if (!origin) throw Error("Retained catalog execution origin required"); + const read = (name: string) => { + const raw = readPrivateRecord(state, "catalog-commits", name, 65536); + verifyFamilyOwnedRecord( + state, + origin, + { scope: "private", namespace: "catalog-commits", name, container: "" }, + canonicalJson(raw), + ); + const journal = verifyControl(state.installation.publicKey, DOMAIN, raw, journalSchema); + if (canonicalJson(journal.receipt.identity) !== canonicalJson(commit)) + throw Error("Catalog commit identity conflict"); + return journal; + }; + let intent: z.infer; + try { + intent = read(`${commit.execution.operationKey}.intent.json`); + } catch (error) { + if (missing(error)) return null; + throw error; + } + if (intent.receipt.phase !== "prepared") throw Error("Invalid immutable catalog intent"); + const name = `${commit.execution.operationKey}.json`; + try { + const receipt = read(name); + if (canonicalJson({ ...receipt, receipt: intent.receipt }) !== canonicalJson(intent)) + throw Error("Catalog receipt intent mismatch"); + return receipt; + } catch (error) { + if (missing(error)) return intent; + // The immutable intent and exact pointer prove publication. A copied + // terminal receipt is refused; only the indexed, not-yet-published + // terminal preimage can replace the original prepared cache. + const raw = readPrivateRecord(state, "catalog-commits", name, 65536); + const prior = verifyControl(state.installation.publicKey, DOMAIN, raw, journalSchema); + if (canonicalJson(prior) !== canonicalJson(intent)) throw error; + const marker = directories(rootDir, false, (_docker, pins) => (pins ? pointer(pins) : null)); + if (canonicalJson(marker) !== canonicalJson(intent.next)) throw error; + const repaired = { + ...intent, + receipt: { + ...intent.receipt, + phase: "committed" as const, + committedAt: intent.next.committedAt, + }, + }; + const signed = signControl(state.privateKey, DOMAIN, repaired), + bytes = canonicalJson(signed); + verifyCatalogPinReceiptRepairPreimage(state, origin, commit.execution.operationKey, bytes); + writePrivateRecord(state, "catalog-commits", name, signed, false, origin); + return read(name); + } + } finally { + closeControlInstallation(installation); + } +} +function reconcile( + journal: z.infer, + current: z.infer | null, +): CatalogPinCommitReceipt { + if (journal.receipt.phase === "committed" || journal.receipt.phase === "not_committed") + return journal.receipt; + if (canonicalJson(current) === canonicalJson(journal.next)) + return { ...journal.receipt, phase: "committed", committedAt: journal.next.committedAt }; + return { ...journal.receipt, phase: "unknown", committedAt: null }; +} +export async function readCatalogPinCommit(options: { + rootDir: string; + commit: CatalogPinCommitIdentity; +}): Promise { + const commit = catalogPinCommitIdentitySchema.parse(options.commit), + journal = readJournal(options.rootDir, commit); + if (!journal) return null; + return directories(options.rootDir, false, (_docker, pins) => { + const receipt = reconcile(journal, pins ? pointer(pins) : null); + if (receipt.phase === "committed" && journal.receipt.phase !== "committed") { + const installation = openControlInstallation({ + infraDir: join(options.rootDir, "infra/docker"), + role: "ops-agent", + serviceIds: ["motis"], + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const state = installationState(installation), + origin = retainedExecutionOrigin(state, commit.execution); + if (!origin) throw Error("Original catalog receipt origin required"); + writePrivateRecord( + state, + "catalog-commits", + `${commit.execution.operationKey}.json`, + signControl(state.privateKey, DOMAIN, { ...journal, receipt }), + false, + origin, + ); + } finally { + closeControlInstallation(installation); + } + } + if (pins && (receipt.phase === "committed" || receipt.phase === "not_committed")) { + const pending = optionalRead(pins, "pending.json", 65536); + if (pending) { + if ( + canonicalJson(JSON.parse(pending.toString("utf8"))) !== + canonicalJson({ identity: commit, generationId: receipt.generationId }) + ) + throw Error("Foreign pending catalog commit"); + // Only attempt metadata is repaired; current.json is never rewritten. + pins.removeFile("pending.json"); + } + } + return receipt; + }); +} +/** Local CAS only. The agent's validation-bound activation is its only production caller. */ +export async function publishCatalogPinSet(options: { + rootDir: string; + lease: TargetLeaseAuthority; + pins: CatalogPinSet; + expectedGenerationId: string | null; + commit: CatalogPinCommitIdentity; +}): Promise { + const pair = parseCatalogPinSet(options.pins), + commit = catalogPinCommitIdentitySchema.parse(options.commit); + const owner = catalogPinTargetOwner(options.lease); + assertCatalogRoot(options.rootDir, owner.installation); + if (owner.installation.role !== "ops-agent") + throw Error("Only the agent may activate catalog pins"); + assertArtifactTargetLeaseExecution(options.lease, commit.execution); + const state = installationState(owner.installation); + const save = (journal: z.infer, intent = false) => { + const signed = signControl(state.privateKey, DOMAIN, journal); + if (Buffer.byteLength(canonicalJson(signed)) > 65536) + throw Error("Catalog commit receipt is too large"); + writePrivateRecord( + state, + "catalog-commits", + `${commit.execution.operationKey}${intent ? ".intent" : ""}.json`, + signed, + intent, + owner.origin, + ); + }; + return directories(options.rootDir, true, (_docker, store) => { + if (!store) throw Error("Catalog pin store unavailable"); + const current = pointer(store), + retained = readJournal(options.rootDir, commit); + if (retained) { + if ( + retained.receipt.generationId !== pair.generationId || + retained.receipt.previousGenerationId !== options.expectedGenerationId + ) + throw Error("Catalog commit intent conflict"); + const receipt = reconcile(retained, current); + save({ ...retained, receipt }); + const pending = optionalRead(store, "pending.json", 65536); + if (receipt.phase === "committed" && pending) { + if ( + canonicalJson(JSON.parse(pending.toString("utf8"))) !== + canonicalJson({ identity: commit, generationId: pair.generationId }) + ) + throw Error("Foreign pending catalog commit"); + store.removeFile("pending.json"); + } + return receipt; + } + // A pending attempt is a global fence, not permission to copy or replay its pointer. + if (optionalRead(store, "pending.json", 65536)) + throw Error("Catalog pin publication requires reconciliation"); + if ((current?.generationId ?? null) !== options.expectedGenerationId) + throw Error("Catalog pin generation changed"); + const next = catalogPinPointerSchema.parse({ + version: 1, + generationId: pair.generationId, + commitId: commit.commitId, + operationKey: commit.execution.operationKey, + fingerprint: commit.execution.fingerprint, + committedAt: new Date().toISOString(), + }); + const receipt: CatalogPinCommitReceipt = { + version: 1, + identity: commit, + previousGenerationId: options.expectedGenerationId, + generationId: pair.generationId, + phase: "prepared", + committedAt: null, + }; + const journal = { version: 1 as const, receipt, previous: current, next }; + save(journal, true); + save(journal); + store.write( + "pending.json", + canonicalJson({ identity: commit, generationId: pair.generationId }), + true, + ); + try { + writeGeneration(store, pair); + catalogPinTargetOwner(options.lease); + if (canonicalJson(pointer(store)) !== canonicalJson(current)) + throw Error("Catalog pin pointer changed"); + store.write("current.json", canonicalJson(next)); + const committed = { ...receipt, phase: "committed" as const, committedAt: next.committedAt }; + save({ ...journal, receipt: committed }); + store.removeFile("pending.json"); + return committed; + } catch (error) { + // The live synchronous writer has unwound. Exact unchanged predecessor + // proves nonpublication; a foreign/missing/changed pointer proves nothing. + if (canonicalJson(pointer(store)) === canonicalJson(current)) { + save({ ...journal, receipt: { ...receipt, phase: "not_committed" } }); + store.removeFile("pending.json"); + } + throw error; + } + }); +} diff --git a/packages/core/src/catalog-pins/types.ts b/packages/core/src/catalog-pins/types.ts new file mode 100644 index 000000000..d38450c21 --- /dev/null +++ b/packages/core/src/catalog-pins/types.ts @@ -0,0 +1,42 @@ +import type { ExecutionRef } from "../ops/execution-ref"; + +export interface TransitousLock { + ref: string; + submodules: Record; + lockedAt: string; + lockedBy: string; + comment?: string; +} +export interface GbfsCatalogLock { + schemaVersion: 1; + source: "mobilitydata-gbfs"; + commit: string; + url: string; + sha256: string; + lockedAt: string; + lockedBy: string; +} +export interface CatalogPinSet { + schemaVersion: 1; + generationId: string; + transitous: TransitousLock; + gbfs: GbfsCatalogLock; +} +export interface CatalogPinCommitIdentity { + commitId: string; + execution: ExecutionRef; + proposalId: string; + validationId: string; + diffDigest: string; +} +export interface CatalogPinCommitReceipt { + version: 1; + identity: CatalogPinCommitIdentity; + previousGenerationId: string | null; + generationId: string; + phase: "prepared" | "committed" | "not_committed" | "unknown"; + committedAt: string | null; +} +export function catalogRevisionId(pins: CatalogPinSet): string { + return pins.generationId; +} diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 23bfc6249..838101094 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -289,6 +289,7 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { }, "valhalla.traffic.refreshWaysToEdges": { kind: "valhalla.traffic.refreshWaysToEdges" }, "postgis.capacity.inspect": { kind: "postgis.capacity.inspect" }, + "catalogPins.inspect": { kind: "catalogPins.inspect" }, "transitousLock.inspect": { kind: "transitousLock.inspect" }, "transitousLock.propose": { kind: "transitousLock.propose", @@ -699,6 +700,9 @@ describe("ops contract", () => { ); it("authorizes roles from the operation kind without inspecting operation arguments", () => { + expect(authorizeOpsOperation("api", "catalogPins.inspect")).toBe(true); + expect(authorizeOpsOperation("data-manager", "catalogPins.inspect")).toBe(true); + expect(authorizeOpsOperation("data-manager", "configuration.inspect")).toBe(false); expect(authorizeOpsOperation("api", "service.start")).toBe(true); expect(authorizeOpsOperation("api", "motis.primary.restart")).toBe(false); expect(authorizeOpsOperation("data-manager", "motis.primary.restart")).toBe(true); diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 11fe8d361..aa649a085 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -1,3 +1,4 @@ +import { catalogPinSetSchema } from "../catalog-pins/schemas"; import { configurationGenerationRevisionSchema, configurationHardlinksApplyOperationSchema, @@ -131,6 +132,7 @@ export const OPS_OPERATION_KINDS = [ "valhalla.traffic.maintain", "valhalla.traffic.refreshWaysToEdges", "postgis.capacity.inspect", + "catalogPins.inspect", "transitousLock.inspect", "transitousLock.propose", "transitousLock.approve", @@ -257,6 +259,7 @@ export const OPS_KIND_POLICIES = { "valhalla.traffic.maintain": asyncEffect("data-manager", 30 * 60_000), "valhalla.traffic.refreshWaysToEdges": asyncEffect("data-manager", 30 * 60_000), "postgis.capacity.inspect": sync("data-manager", 30_000), + "catalogPins.inspect": sync("api", 30_000, 768 * 1024), "transitousLock.inspect": sync("data-manager", 30_000), "transitousLock.propose": sync("data-manager", 30_000), "transitousLock.approve": asyncEffect("data-manager", 120_000, 4096), @@ -541,6 +544,7 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ kindOnly("postgis.capacity.inspect"), // The catalog lock lives under the repository's `infra/docker/`, which only // the operations agent may write. Callers name a pinned ref, never a path. + kindOnly("catalogPins.inspect"), kindOnly("transitousLock.inspect"), z.strictObject({ kind: z.literal("transitousLock.propose"), @@ -595,7 +599,10 @@ const opsRequestSchema = z export type OpsRequest = z.infer; export function authorizeOpsOperation(role: OpsRole, kind: OpsOperationKind): boolean { - return OPS_KIND_POLICIES[kind].role === role; + return ( + OPS_KIND_POLICIES[kind].role === role || + (role === "data-manager" && kind === "catalogPins.inspect") + ); } export type OpsErrorClass = @@ -1271,6 +1278,18 @@ export const OPS_RESULT_SCHEMAS = { }), "valhalla.traffic.refreshWaysToEdges": changed, "postgis.capacity.inspect": z.strictObject({ availableBytes: z.number().int().nonnegative() }), + "catalogPins.inspect": z + .strictObject({ + active: catalogPinSetSchema.nullable(), + catalogRevisionId: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + }) + .refine( + (value) => value.catalogRevisionId === (value.active?.generationId ?? null), + "Catalog pair/revision mismatch", + ), "transitousLock.inspect": z.strictObject({ active: transitousLockRecordSchema.nullable(), proposed: transitousLockRecordSchema.nullable(), diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 290c6ddf8..b368877f4 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -35,6 +35,7 @@ "backup.delete", "backup.list", "backup.restore", + "catalogPins.inspect", "communityServices.inspect", "configuration.hardlinks.apply", "configuration.hardlinks.inspect", @@ -107,10 +108,10 @@ "transitousLock.approve", "transitousLock.inspect", "transitousLock.propose", + "valhalla.traffic.disable", "valhalla.traffic.inspect", "valhalla.traffic.maintain", - "valhalla.traffic.refreshWaysToEdges", - "valhalla.traffic.disable" + "valhalla.traffic.refreshWaysToEdges" ] }, "trustedDataSourceKinds": [ @@ -333,7 +334,8 @@ "transitousLock.inspect", "transitousLock.propose", "transitousLock.approve", - "gbfsCatalogLock.inspect" + "gbfsCatalogLock.inspect", + "catalogPins.inspect" ] }, { diff --git a/services/data-manager/__tests__/catalog-pair-read.test.ts b/services/data-manager/__tests__/catalog-pair-read.test.ts new file mode 100644 index 000000000..714d215ef --- /dev/null +++ b/services/data-manager/__tests__/catalog-pair-read.test.ts @@ -0,0 +1,25 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { readTransitousLock } from "../src/transitous-lock.js"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots) rmSync(root, { recursive: true, force: true }); +}); +it("never treats one legacy Transitous file as an authoritative complete catalog pair", () => { + const root = mkdtempSync(join(tmpdir(), "omx-pins-read-")); + roots.push(root); + mkdirSync(join(root, "infra/docker"), { recursive: true }); + writeFileSync( + join(root, "infra/docker/transitous.lock.json"), + JSON.stringify({ + ref: `main@${"a".repeat(40)}`, + submodules: {}, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + expect(() => readTransitousLock(root)).toThrow(/paired|initializ|catalog/i); +}); diff --git a/services/data-manager/__tests__/gbfs-catalog-lock.test.ts b/services/data-manager/__tests__/gbfs-catalog-lock.test.ts index d3e3a742e..d2b80fcb6 100644 --- a/services/data-manager/__tests__/gbfs-catalog-lock.test.ts +++ b/services/data-manager/__tests__/gbfs-catalog-lock.test.ts @@ -28,7 +28,7 @@ describe("GBFS catalog lock", () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-gbfs-lock-")); mkdirSync(join(tmp, "infra", "docker"), { recursive: true }); writeGbfsCatalogLock(tmp, LOCK); - expect(readGbfsCatalogLock(tmp)).toEqual(LOCK); + expect(() => readGbfsCatalogLock(tmp!)).toThrow(/not initialized/); expect(readFileSync(join(tmp, "infra", "docker", "gbfs-catalog.lock.json"), "utf-8")).toContain( LOCK.sha256, ); diff --git a/services/data-manager/__tests__/transitous-lock.test.ts b/services/data-manager/__tests__/transitous-lock.test.ts index e29a87f7c..04a375973 100644 --- a/services/data-manager/__tests__/transitous-lock.test.ts +++ b/services/data-manager/__tests__/transitous-lock.test.ts @@ -49,7 +49,7 @@ describe("parseRefShaPair", () => { describe("readTransitousLock / writeTransitousLock", () => { it("returns null when the file is absent", () => { const root = makeRepoRoot(); - expect(readTransitousLock(root)).toBeNull(); + expect(() => readTransitousLock(root)).toThrow(/not initialized/); }); it("roundtrips a lockfile through write + read", () => { @@ -64,8 +64,7 @@ describe("readTransitousLock / writeTransitousLock", () => { comment: "test fixture", }; writeTransitousLock(root, lock); - const loaded = readTransitousLock(root); - expect(loaded).toEqual(lock); + expect(() => readTransitousLock(root)).toThrow(/not initialized/); const onDisk = JSON.parse( readFileSync(join(root, TRANSITOUS_LOCK_RELATIVE_PATH), "utf-8"), ) as Record; @@ -87,7 +86,7 @@ describe("readTransitousLock / writeTransitousLock", () => { }; writeTransitousLock(root, active); writeTransitousLockProposal(root, proposed); - expect(readTransitousLock(root)).toEqual(active); + expect(() => readTransitousLock(root)).toThrow(/not initialized/); expect(readTransitousLockProposal(root)).toEqual(proposed); expect(readFileSync(join(root, TRANSITOUS_PROPOSED_LOCK_RELATIVE_PATH), "utf-8")).toContain( proposed.ref, @@ -97,12 +96,12 @@ describe("readTransitousLock / writeTransitousLock", () => { it("throws on a malformed lockfile", () => { const root = makeRepoRoot(); writeFileSync(join(root, TRANSITOUS_LOCK_RELATIVE_PATH), "not json"); - expect(() => readTransitousLock(root)).toThrow(/parse/); + expect(() => readTransitousLock(root)).toThrow(/not initialized/); }); it("throws when required fields are missing", () => { const root = makeRepoRoot(); writeFileSync(join(root, TRANSITOUS_LOCK_RELATIVE_PATH), JSON.stringify({ submodules: {} })); - expect(() => readTransitousLock(root)).toThrow(/"ref"/); + expect(() => readTransitousLock(root)).toThrow(/not initialized/); }); }); diff --git a/services/data-manager/__tests__/transitous/compile-gbfs.test.ts b/services/data-manager/__tests__/transitous/compile-gbfs.test.ts index c38e74218..7b869db4e 100644 --- a/services/data-manager/__tests__/transitous/compile-gbfs.test.ts +++ b/services/data-manager/__tests__/transitous/compile-gbfs.test.ts @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; +import type { GbfsCatalogLock } from "@openmapx/core/catalog-pins/types"; import { afterEach, describe, expect, it, vi } from "vitest"; vi.mock("node:dns/promises", () => ({ lookup: vi.fn() })); @@ -22,15 +24,6 @@ import { StateStore } from "../../src/state.js"; // The GBFS catalog lock lives under the repository's `infra/docker/`, which is // agent-owned. Tests hand the same pinned record through the typed operation. const gbfsLock: { value: Record | null } = { value: null }; -vi.mock("../../src/ops-client.js", () => ({ - runOpsOperation: vi.fn(async (operation: { kind: string }) => { - if (operation.kind === "gbfsCatalogLock.inspect") { - if (!gbfsLock.value) throw new Error("GBFS catalog lock is missing"); - return gbfsLock.value; - } - return { changed: true }; - }), -})); const originalFetch = globalThis.fetch; const originalEnabled = process.env.MOTIS_GBFS_CATALOG_ENABLED; @@ -163,6 +156,16 @@ describe("bounded GBFS candidate validation", () => { now: () => "2026-01-01T00:00:00Z", }); ctx.state.catalogDir = catalog; + if (gbfsLock.value) + ctx.state.catalogPins = createCatalogPinSet( + { + ref: `main@${"a".repeat(40)}`, + submodules: {}, + lockedAt: "2026-01-01T00:00:00Z", + lockedBy: "fixture", + }, + gbfsLock.value as unknown as GbfsCatalogLock, + ); const result = await run(ctx); @@ -226,6 +229,16 @@ describe("bounded GBFS candidate validation", () => { now: () => "2026-01-01T00:00:00Z", }); ctx.state.catalogDir = catalog; + if (gbfsLock.value) + ctx.state.catalogPins = createCatalogPinSet( + { + ref: `main@${"a".repeat(40)}`, + submodules: {}, + lockedAt: "2026-01-01T00:00:00Z", + lockedBy: "fixture", + }, + gbfsLock.value as unknown as GbfsCatalogLock, + ); const result = await run(ctx); @@ -292,6 +305,16 @@ describe("bounded GBFS candidate validation", () => { now: () => "2026-01-01T00:00:00Z", }); ctx.state.catalogDir = catalog; + if (gbfsLock.value) + ctx.state.catalogPins = createCatalogPinSet( + { + ref: `main@${"a".repeat(40)}`, + submodules: {}, + lockedAt: "2026-01-01T00:00:00Z", + lockedBy: "fixture", + }, + gbfsLock.value as unknown as GbfsCatalogLock, + ); const result = await run(ctx); expect(result.status).toBe("ok"); expect(readFileSync(join(catalog, "feeds", "de.json"), "utf-8")).toContain("openmapx-demo"); diff --git a/services/data-manager/__tests__/transitous/prepare.test.ts b/services/data-manager/__tests__/transitous/prepare.test.ts index 0661fe545..30c286397 100644 --- a/services/data-manager/__tests__/transitous/prepare.test.ts +++ b/services/data-manager/__tests__/transitous/prepare.test.ts @@ -1,6 +1,7 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; import { afterEach, describe, expect, it, vi } from "vitest"; import { buildJobContext } from "../../src/jobs/transitous/pipeline.js"; import { run as prepareRun } from "../../src/jobs/transitous/prepare.js"; @@ -16,8 +17,17 @@ const PINNED_LOCK = { }; vi.mock("../../src/ops-client.js", () => ({ runOpsOperation: vi.fn(async (operation: { kind: string }) => { - if (operation.kind === "transitousLock.inspect") { - return { active: PINNED_LOCK, proposed: null }; + if (operation.kind === "catalogPins.inspect") { + const active = createCatalogPinSet(PINNED_LOCK, { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "b".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"b".repeat(40)}/systems.csv`, + sha256: "c".repeat(64), + lockedAt: PINNED_LOCK.lockedAt, + lockedBy: "test", + }); + return { active, catalogRevisionId: active.generationId }; } if (operation.kind === "gbfsCatalogLock.inspect") { return { @@ -144,7 +154,7 @@ describe("prepare stage", () => { // Fail closed: an unpinned or unverifiable catalog must never reach the // upstream execution step. expect(result.status).toBe("error"); - expect(String(result.message)).toMatch(/lock/i); + expect(String(result.message)).toMatch(/pin|lock|agent/i); expect(ran).not.toContain("python3"); }); @@ -174,43 +184,20 @@ describe("prepare stage", () => { expect(warnings.some((w) => w.includes("no repoRoot supplied"))).toBe(true); }); - it("resets the catalog to the PROPOSED lock when ctx.useProposedLock is set", async () => { - tmp = mkdtempSync(join(tmpdir(), "openmapx-prepare-proposed-")); - const dataDir = join(tmp, "data"); - const catalogDir = join(dataDir, ".transitous-catalog"); - mkdirSync(join(catalogDir, ".git"), { recursive: true }); - mkdirSync(join(catalogDir, "feeds"), { recursive: true }); - - const ACTIVE = "a".repeat(40); - const PROPOSED = "d".repeat(40); - const { runOpsOperation } = await import("../../src/ops-client.js"); - vi.mocked(runOpsOperation).mockImplementationOnce( - async () => - ({ - active: { ref: `main@${ACTIVE}`, submodules: {}, lockedAt: "x", lockedBy: "test" }, - proposed: { ref: `main@${PROPOSED}`, submodules: {}, lockedAt: "x", lockedBy: "test" }, - }) as never, - ); - - const calls: string[] = []; - const runner = async (command: string, args: string[]) => { - calls.push(`${command} ${args.join(" ")}`); - }; - + it("refuses a ref-only proposal before upstream execution", async () => { + tmp = mkdtempSync(join(tmpdir(), "openmapx-proposed-closed-")); + const runner = vi.fn(async () => {}); const ctx = buildJobContext({ - dataDir, - store: new StateStore(dataDir), + dataDir: tmp, + store: new StateStore(tmp), useProposedLock: true, runner, - now: () => "2026-05-01T00:00:00.000Z", }); - - const result = await prepareRun(ctx); - expect(result.status).toBe("ok"); - expect(calls.some((c) => c.includes(`checkout --force ${PROPOSED}`))).toBe(true); - expect(calls.some((c) => c.includes(ACTIVE))).toBe(false); - // The capability snapshot later records the lock this run enforced. - expect(ctx.state.transitousLock?.ref).toBe(`main@${PROPOSED}`); + expect(await prepareRun(ctx)).toMatchObject({ + status: "error", + message: expect.stringContaining("Paired proposal"), + }); + expect(runner).not.toHaveBeenCalled(); }); it("resets to the ACTIVE lock by default (no proposal enforcement)", async () => { @@ -222,14 +209,6 @@ describe("prepare stage", () => { const ACTIVE = "a".repeat(40); const PROPOSED = "d".repeat(40); - const { runOpsOperation } = await import("../../src/ops-client.js"); - vi.mocked(runOpsOperation).mockImplementationOnce( - async () => - ({ - active: { ref: `main@${ACTIVE}`, submodules: {}, lockedAt: "x", lockedBy: "test" }, - proposed: { ref: `main@${PROPOSED}`, submodules: {}, lockedAt: "x", lockedBy: "test" }, - }) as never, - ); const calls: string[] = []; const runner = async (command: string, args: string[]) => { diff --git a/services/data-manager/src/gbfs-catalog-lock.ts b/services/data-manager/src/gbfs-catalog-lock.ts index 2a628c8aa..4f0d88af1 100644 --- a/services/data-manager/src/gbfs-catalog-lock.ts +++ b/services/data-manager/src/gbfs-catalog-lock.ts @@ -1,18 +1,12 @@ -import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { atomicWriteJsonSync } from "./utils/atomic-write.js"; export const GBFS_CATALOG_LOCK_RELATIVE_PATH = "infra/docker/gbfs-catalog.lock.json"; -export interface GbfsCatalogLock { - schemaVersion: 1; - source: "mobilitydata-gbfs"; - commit: string; - url: string; - sha256: string; - lockedAt: string; - lockedBy: string; -} +export type { GbfsCatalogLock } from "@openmapx/core/catalog-pins/types"; + +import { readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import type { GbfsCatalogLock } from "@openmapx/core/catalog-pins/types"; export function decodeGbfsCatalogLock( value: unknown, @@ -47,9 +41,9 @@ export function decodeGbfsCatalogLock( } export function readGbfsCatalogLock(repoRoot: string): GbfsCatalogLock { - const path = join(repoRoot, GBFS_CATALOG_LOCK_RELATIVE_PATH); - if (!existsSync(path)) throw new Error(`GBFS catalog lock missing at ${path}`); - return decodeGbfsCatalogLock(JSON.parse(readFileSync(path, "utf-8")), path); + const pins = readCatalogPinSet(repoRoot); + if (!pins) throw Error("Paired catalog pins are not initialized"); + return pins.gbfs; } export function writeGbfsCatalogLock(repoRoot: string, lock: GbfsCatalogLock): void { diff --git a/services/data-manager/src/jobs/transitous/compile-gbfs.ts b/services/data-manager/src/jobs/transitous/compile-gbfs.ts index 981f88885..6ca327ed1 100644 --- a/services/data-manager/src/jobs/transitous/compile-gbfs.ts +++ b/services/data-manager/src/jobs/transitous/compile-gbfs.ts @@ -10,7 +10,6 @@ import { parseMobilityDataGbfsCsv, } from "@openmapx/transitous-core"; import pMap from "p-map"; -import { runOpsOperation } from "../../ops-client.js"; import { atomicWriteJsonSync } from "../../utils/atomic-write.js"; import { scrubSecrets } from "../../utils/scrub-secrets.js"; import { readFeedOverlay } from "../transitous-feeds-overlay.js"; @@ -288,7 +287,9 @@ export const run: StageFn = async (ctx) => { try { // The catalog lock lives under the repository's `infra/docker/`, which only // the operations agent may read on this service's behalf. - const lock = await runOpsOperation({ kind: "gbfsCatalogLock.inspect" }); + const lock = ctx.state.catalogPins?.gbfs; + if (!lock) + throw Error("Preparation must capture the paired catalog pins before GBFS compilation"); const response = await fetch(lock.url, { headers: { "User-Agent": "Open-MapX-gbfs-compiler" }, }); diff --git a/services/data-manager/src/jobs/transitous/prepare.ts b/services/data-manager/src/jobs/transitous/prepare.ts index f2df6d018..f07a30b58 100644 --- a/services/data-manager/src/jobs/transitous/prepare.ts +++ b/services/data-manager/src/jobs/transitous/prepare.ts @@ -1,4 +1,5 @@ import { existsSync, mkdirSync } from "node:fs"; +import { parseCatalogPinSet } from "@openmapx/core/catalog-pins"; import { ensureCatalog } from "@openmapx/transitous-core"; import { runOpsOperation } from "../../ops-client.js"; import { parseRefShaPair, type TransitousLock } from "../../transitous-lock.js"; @@ -21,6 +22,16 @@ export const run: StageFn = async (ctx) => { const startedAt = ctx.now(); const start = Date.now(); try { + if (ctx.useProposedLock) + throw Error("Paired proposal validation requires the reviewed activation workflow"); + const pins = await runOpsOperation({ kind: "catalogPins.inspect" }); + if (!pins.active || pins.catalogRevisionId !== pins.active.generationId) + throw Error("Paired catalog pins are unavailable"); + try { + ctx.state.catalogPins = parseCatalogPinSet(pins.active); + } catch { + throw Error("Paired catalog pin response is invalid"); + } mkdirSync(ctx.dataDir, { recursive: true }); const catalogDir = ctx.catalogDir; @@ -40,7 +51,7 @@ export const run: StageFn = async (ctx) => { catalogDir, ctx.runner, ctx.logger, - ctx.useProposedLock ?? false, + ctx.state.catalogPins.transitous, ); ensureTransitousWorkdirs(catalogDir, ctx.outDir, ctx.downloadsDir); pruneOrphanedGtfsDatasets(ctx); @@ -113,29 +124,9 @@ async function enforceTransitousLock( catalogDir: string, runner: JobContext["runner"], logger: JobLogger, - useProposedLock: boolean, + pinned: TransitousLock, ): Promise { - let lock: TransitousLock | null; - try { - // The lock is agent-owned; data-manager reads it through the typed - // operation rather than bind-mounting the host checkout. The auto-bump - // canary validates the proposed ref in the staging slot before it is - // activated; every other run enforces the active lock. - const slots = await runOpsOperation({ kind: "transitousLock.inspect" }); - lock = useProposedLock ? (slots.proposed ?? slots.active) : slots.active; - } catch (error) { - throw new Error( - `transitous-pipeline: refusing to run — the catalog lock could not be read (${(error as Error).message})`, - ); - } - if (!lock) { - throw new Error( - "transitous-pipeline: refusing to run — no infra/docker/transitous.lock.json is pinned (run `pnpm openmapx transitous bump`)", - ); - } - if (useProposedLock) { - logger.info(`transitous-pipeline: enforcing PROPOSED lock ${lock.ref} (auto-bump canary)`); - } + const lock = pinned; let sha: string; try { ({ sha } = parseRefShaPair(lock.ref)); diff --git a/services/data-manager/src/jobs/transitous/types.ts b/services/data-manager/src/jobs/transitous/types.ts index fe46ec43b..c939f98df 100644 --- a/services/data-manager/src/jobs/transitous/types.ts +++ b/services/data-manager/src/jobs/transitous/types.ts @@ -1,3 +1,4 @@ +import type { CatalogPinSet } from "@openmapx/core/catalog-pins/types"; import type { TransitSource } from "@openmapx/transitous-core"; import type { DatasetMetadata, StateStore } from "../../state.js"; import type { TransitousLock } from "../../transitous-lock.js"; @@ -118,6 +119,7 @@ export interface JobState { proxyTransaction?: ProxyTransactionState; /** The catalog lock prepare enforced; the capability snapshot records it. */ transitousLock?: TransitousLock; + catalogPins?: CatalogPinSet; gbfsCompilation?: { output: string; healthy: number; diff --git a/services/data-manager/src/transitous-lock.ts b/services/data-manager/src/transitous-lock.ts index 9039aca10..7b11b4c2b 100644 --- a/services/data-manager/src/transitous-lock.ts +++ b/services/data-manager/src/transitous-lock.ts @@ -7,25 +7,19 @@ import { join } from "node:path"; * unrelated upstream `feeds/*.json` schema change doesn't silently break our * fetch pipeline. */ -export interface TransitousLock { - ref: string; // e.g. "main@" - submodules: Record; - lockedAt: string; - lockedBy: string; - comment?: string; -} +export type { TransitousLock } from "@openmapx/core/catalog-pins/types"; + +import { readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import type { TransitousLock } from "@openmapx/core/catalog-pins/types"; export const TRANSITOUS_LOCK_RELATIVE_PATH = "infra/docker/transitous.lock.json"; export const TRANSITOUS_PROPOSED_LOCK_RELATIVE_PATH = "infra/docker/transitous.lock.proposed.json"; -/** - * Read the lockfile from `/infra/docker/transitous.lock.json`. - * Returns `null` if the file does not exist; the caller is expected to log a - * warning and continue (existing deployments must keep working before the - * first `transitous bump` run). - */ +/** Read the initialized paired generation. Compatibility files never provide active authority. */ export function readTransitousLock(repoRoot: string): TransitousLock | null { - return readTransitousLockAt(repoRoot, TRANSITOUS_LOCK_RELATIVE_PATH); + const pins = readCatalogPinSet(repoRoot); + if (!pins) throw new Error("Paired catalog pins are not initialized"); + return pins.transitous; } function readTransitousLockAt(repoRoot: string, relativePath: string): TransitousLock | null { From 830ff437bd49103014d27893e6e73b18f45d4993 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 22:44:17 +0100 Subject: [PATCH 050/135] feat(admin): page transit sources and preview scoped sync --- apps/api/openapi.json | 4173 ++++++++++++----- .../data-manager-admin-contract.test.ts | 19 +- .../__tests__/data-manager-sources.test.ts | 23 + .../__tests__/data-manager-sync.test.ts | 13 +- apps/api/src/routes/data-manager.ts | 23 +- .../services/transit-catalog/index.test.ts | 177 +- .../api/src/services/transit-catalog/index.ts | 75 +- .../api/src/services/transit-catalog/types.ts | 2 + .../(full-admin)/transit/CurrentJobCard.tsx | 41 +- .../admin/services/TransitSourcesSection.tsx | 141 +- .../TransitSourcesSection.workflow.test.tsx | 177 +- .../admin/services/TransitSyncDialog.test.tsx | 196 + .../admin/services/TransitSyncDialog.tsx | 228 + apps/web/src/lib/admin/transitHooks.test.tsx | 13 +- apps/web/src/lib/admin/transitHooks.ts | 3 +- .../administration/services-administration.md | 14 + .../cli/__tests__/cli-env-defaults.test.ts | 10 + packages/cli/src/lib/env-defaults.ts | 12 +- packages/core/src/api/admin-sdk.test.ts | 37 +- .../data-manager-admin/contracts.test.ts | 10 +- .../data-manager-admin/operations.ts | 34 +- .../contracts/data-manager-admin/transit.ts | 6 + .../admin/data-manager-admin.generated.ts | 221 + .../getApiDataManagerTransitCatalog200.ts | 10 + ...DataManagerTransitCatalog200SourcesItem.ts | 3 + .../getApiDataManagerTransitCatalogParams.ts | 10 + .../getApiDataManagerTransitProfile200.ts | 7 + ...etApiDataManagerTransitProfile200Policy.ts | 26 + ...nagerTransitProfile200PolicyAcquisition.ts | 10 + ...gerTransitProfile200PolicyGbfsSelection.ts | 11 + ...taManagerTransitProfile200PolicyProfile.ts | 11 + ...TransitProfile200PolicyResourceEnvelope.ts | 12 + .../getApiDataManagerTransitProfile400.ts | 20 + .../getApiDataManagerTransitProfile401.ts | 20 + .../getApiDataManagerTransitProfile403.ts | 20 + .../getApiDataManagerTransitProfile413.ts | 20 + .../getApiDataManagerTransitProfile415.ts | 20 + .../getApiDataManagerTransitProfile429.ts | 11 + .../getApiDataManagerTransitProfile500.ts | 20 + .../getApiDataManagerTransitProfile503.ts | 20 + .../getApiDataManagerTransitSourcesParams.ts | 1 + .../src/api/generated/admin/models/index.ts | 37 +- .../postApiDataManagerTransitSync409.ts | 34 +- .../postApiDataManagerTransitSync409Reason.ts | 10 - .../postApiDataManagerTransitSyncBody.ts | 15 +- ...stApiDataManagerTransitSyncPreflight200.ts | 15 + ...ataManagerTransitSyncPreflight200Result.ts | 14 + ...erTransitSyncPreflight200ResultCapacity.ts | 17 + ...erTransitSyncPreflight200ResultEstimate.ts | 23 + ...nsitSyncPreflight200ResultEstimateBasis.ts | 10 + ...agerTransitSyncPreflight200ResultPolicy.ts | 26 + ...SyncPreflight200ResultPolicyAcquisition.ts | 10 + ...ncPreflight200ResultPolicyGbfsSelection.ts | 11 + ...nsitSyncPreflight200ResultPolicyProfile.ts | 11 + ...reflight200ResultPolicyResourceEnvelope.ts | 12 + ...stApiDataManagerTransitSyncPreflight400.ts | 20 + ...stApiDataManagerTransitSyncPreflight401.ts | 20 + ...stApiDataManagerTransitSyncPreflight403.ts | 20 + ...stApiDataManagerTransitSyncPreflight409.ts | 8 + ...iDataManagerTransitSyncPreflight409Code.ts | 9 + ...stApiDataManagerTransitSyncPreflight413.ts | 20 + ...stApiDataManagerTransitSyncPreflight415.ts | 20 + ...stApiDataManagerTransitSyncPreflight422.ts | 20 + ...stApiDataManagerTransitSyncPreflight429.ts | 11 + ...stApiDataManagerTransitSyncPreflight500.ts | 20 + ...stApiDataManagerTransitSyncPreflight503.ts | 20 + ...tApiDataManagerTransitSyncPreflightBody.ts | 11 + .../admin/operation-bindings.generated.ts | 16 + .../__fixtures__/reviewed-surface.json | 10 +- .../data-manager/contracts/contracts.test.ts | 4 +- .../data-manager/contracts/executions.ts | 3 +- .../data-manager/contracts/operations.ts | 15 +- .../data-manager/contracts/transit.ts | 30 + .../generated/data-manager.generated.ts | 121 + .../models/getTransitSourcesParams.ts | 1 + .../data-manager/generated/models/index.ts | 21 + .../models/inspectTransitSyncPreflight200.ts | 15 + .../inspectTransitSyncPreflight200Result.ts | 14 + ...ctTransitSyncPreflight200ResultCapacity.ts | 17 + ...ctTransitSyncPreflight200ResultEstimate.ts | 23 + ...nsitSyncPreflight200ResultEstimateBasis.ts | 10 + ...pectTransitSyncPreflight200ResultPolicy.ts | 26 + ...SyncPreflight200ResultPolicyAcquisition.ts | 10 + ...ncPreflight200ResultPolicyGbfsSelection.ts | 11 + ...nsitSyncPreflight200ResultPolicyProfile.ts | 11 + ...reflight200ResultPolicyResourceEnvelope.ts | 12 + .../models/inspectTransitSyncPreflight400.ts | 20 + .../models/inspectTransitSyncPreflight401.ts | 20 + .../models/inspectTransitSyncPreflight409.ts | 8 + .../inspectTransitSyncPreflight409Code.ts | 9 + .../models/inspectTransitSyncPreflight413.ts | 20 + .../models/inspectTransitSyncPreflight415.ts | 20 + .../models/inspectTransitSyncPreflight422.ts | 20 + .../models/inspectTransitSyncPreflight429.ts | 20 + .../models/inspectTransitSyncPreflight500.ts | 20 + .../models/inspectTransitSyncPreflight503.ts | 20 + .../models/inspectTransitSyncPreflightBody.ts | 11 + .../models/startDataExecutionBodyInput.ts | 10 + .../generated/models/syncTransit409.ts | 4 + .../generated/models/syncTransitBody.ts | 4 + .../generated/operation-bindings.generated.ts | 7 + .../src/services/data-manager/sdk.test.ts | 4 +- services/data-manager/__tests__/api.test.ts | 70 + .../__tests__/openapi-freshness.test.ts | 2 +- .../transitous/admin-preflight.test.ts | 148 + .../__tests__/transitous/preflight.test.ts | 18 + services/data-manager/openapi.json | 1068 ++++- services/data-manager/src/api.ts | 71 +- .../data-manager/src/jobs/execution/api.ts | 7 + .../src/jobs/execution/dispatch.ts | 16 +- .../legacy-admission-postgres.test.ts | 210 +- .../src/jobs/execution/receiver.ts | 32 +- .../src/jobs/execution/runtime.ts | 2 + .../data-manager/src/jobs/execution/store.ts | 7 + .../src/jobs/transitous/admin-preflight.ts | 198 + .../src/jobs/transitous/operations-profile.ts | 16 +- .../src/jobs/transitous/preflight.ts | 14 +- services/data-manager/src/service.ts | 7 +- services/data-manager/src/transit-sources.ts | 5 +- 119 files changed, 7289 insertions(+), 1533 deletions(-) create mode 100644 apps/web/src/components/admin/services/TransitSyncDialog.test.tsx create mode 100644 apps/web/src/components/admin/services/TransitSyncDialog.tsx create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200Policy.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyAcquisition.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyGbfsSelection.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyProfile.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyResourceEnvelope.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile400.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile401.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile403.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile413.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile415.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile429.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile500.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile503.ts delete mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409Reason.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200Result.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultCapacity.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimate.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimateBasis.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicy.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyProfile.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight400.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight401.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight403.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409Code.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight413.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight415.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight422.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight429.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight500.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight503.ts create mode 100644 packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflightBody.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200Result.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultCapacity.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimate.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimateBasis.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicy.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyAcquisition.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyGbfsSelection.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyProfile.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyResourceEnvelope.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409Code.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight422.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflightBody.ts create mode 100644 services/data-manager/__tests__/transitous/admin-preflight.test.ts create mode 100644 services/data-manager/src/jobs/transitous/admin-preflight.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index f13f246b7..631502f93 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -65007,6 +65007,26 @@ "get": { "operationId": "getApiDataManagerTransitCatalog", "parameters": [ + { + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 500 + }, + "in": "query", + "name": "limit", + "required": false + }, + { + "schema": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "in": "query", + "name": "offset", + "required": false + }, { "schema": { "type": "string" @@ -65032,6 +65052,16 @@ "schema": { "type": "object", "properties": { + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, "sources": { "type": "array", "items": { @@ -65040,6 +65070,18 @@ "id": { "type": "string" }, + "installedSourceId": { + "type": [ + "string", + "null" + ] + }, + "discoveryIds": { + "type": "array", + "items": { + "type": "string" + } + }, "name": { "type": "string" }, @@ -65448,6 +65490,8 @@ }, "required": [ "id", + "installedSourceId", + "discoveryIds", "name", "source", "countryCode", @@ -65463,6 +65507,8 @@ } }, "required": [ + "limit", + "offset", "sources", "total" ], @@ -67608,9 +67654,9 @@ "x-openmapx-auth": "service" } }, - "/api/data-manager/transit/restart-motis": { - "post": { - "operationId": "postApiDataManagerTransitRestartMotis", + "/api/data-manager/transit/profile": { + "get": { + "operationId": "getApiDataManagerTransitProfile", "responses": { "200": { "description": "Response", @@ -67619,22 +67665,122 @@ "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] - }, - "status": { - "type": "string", - "enum": [ - "restart-initiated" - ] + "policy": { + "type": "object", + "properties": { + "profile": { + "type": "string", + "enum": [ + "regional-assisted", + "regional-sovereign", + "planet" + ] + }, + "countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "feedAllowList": { + "type": "array", + "items": { + "type": "string" + } + }, + "acquisition": { + "type": "string", + "enum": [ + "mirror", + "build" + ] + }, + "artifactOrigins": { + "type": "array", + "items": { + "type": "string" + } + }, + "originDownloadsRequired": { + "type": "boolean" + }, + "hostedRuntimeFallbackAllowed": { + "type": "boolean" + }, + "osmInput": { + "type": "string" + }, + "gbfsSelection": { + "type": "string", + "enum": [ + "explicit-countries", + "explicit-feeds", + "allow-listed-batches" + ] + }, + "maxFeedCount": { + "type": "number", + "minimum": 0 + }, + "retentionGenerations": { + "type": "number", + "minimum": 0 + }, + "resourceEnvelope": { + "type": "object", + "properties": { + "minimumMemoryGb": { + "type": "number", + "minimum": 0 + }, + "minimumCpu": { + "type": "number", + "minimum": 0 + }, + "minimumDiskGb": { + "type": "number", + "minimum": 0 + }, + "operatorCapacityRequired": { + "type": "boolean" + } + }, + "required": [ + "minimumMemoryGb", + "minimumCpu", + "minimumDiskGb", + "operatorCapacityRequired" + ], + "additionalProperties": false + }, + "updateCadenceHours": { + "type": "number", + "minimum": 0 + }, + "experimental": { + "type": "boolean" + } + }, + "required": [ + "profile", + "countries", + "feedAllowList", + "acquisition", + "artifactOrigins", + "originDownloadsRequired", + "hostedRuntimeFallbackAllowed", + "gbfsSelection", + "maxFeedCount", + "retentionGenerations", + "resourceEnvelope", + "updateCadenceHours", + "experimental" + ], + "additionalProperties": false } }, "required": [ - "ok", - "status" + "policy" ], "additionalProperties": false } @@ -68101,64 +68247,9 @@ "x-openmapx-auth": "service" } }, - "/api/data-manager/transit/sources": { - "get": { - "operationId": "getApiDataManagerTransitSources", - "parameters": [ - { - "schema": { - "type": "string" - }, - "in": "query", - "name": "search", - "required": false - }, - { - "schema": { - "type": "string", - "enum": [ - "active", - "add-pending", - "update-pending", - "removal-pending", - "disabled", - "failed", - "stale" - ] - }, - "in": "query", - "name": "lifecycle", - "required": false - }, - { - "schema": { - "type": "string", - "enum": [ - "catalog", - "operator" - ] - }, - "in": "query", - "name": "origin", - "required": false - }, - { - "schema": { - "type": "number" - }, - "in": "query", - "name": "limit", - "required": false - }, - { - "schema": { - "type": "number" - }, - "in": "query", - "name": "offset", - "required": false - } - ], + "/api/data-manager/transit/restart-motis": { + "post": { + "operationId": "postApiDataManagerTransitRestartMotis", "responses": { "200": { "description": "Response", @@ -68167,222 +68258,57 @@ "schema": { "type": "object", "properties": { - "sourceRevision": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "status": { "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "sources": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "region": { - "type": "string" - }, - "name": { - "type": "string" - }, - "format": { - "type": "string", - "enum": [ - "gtfs", - "netex" - ] - }, - "origin": { - "type": "string", - "enum": [ - "catalog", - "operator" - ] - }, - "originUrl": { - "type": "string" - }, - "license": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/Admin_getApiDataManagerTransitSources_200___schema0" - } - }, - "requested": { - "type": "boolean" - }, - "sourceRevision": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "desiredState": { - "type": "string", - "enum": [ - "enabled", - "disabled", - "removed" - ] - }, - "actions": { - "type": "object", - "properties": { - "enable": { - "type": "boolean" - }, - "disable": { - "type": "boolean" - }, - "remove": { - "type": "boolean" - }, - "reason": { - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "enable", - "disable", - "remove", - "reason" - ], - "additionalProperties": false - }, - "active": { - "type": "boolean" - }, - "activeEpoch": { - "type": "string" - }, - "artifact": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "sha256": { - "type": "string" - }, - "sizeBytes": { - "type": "number", - "minimum": 0 - }, - "retrievedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - } - }, - "required": [ - "path", - "sha256", - "sizeBytes", - "retrievedAt" - ], - "additionalProperties": false - }, - "lastFetchedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "lastImportedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "validationStatus": { - "type": "string" - }, - "validationMessage": { - "type": "string" - }, - "lifecycle": { - "type": "string", - "enum": [ - "active", - "add-pending", - "update-pending", - "removal-pending", - "disabled", - "failed", - "stale" - ] - } - }, - "required": [ - "id", - "region", - "name", - "format", - "origin", - "license", - "requested", - "sourceRevision", - "desiredState", - "actions", - "active", - "lifecycle" - ], - "additionalProperties": false - } - }, - "total": { - "type": "number", - "minimum": 0 - }, - "limit": { - "type": "number", - "minimum": 0 - }, - "offset": { - "type": "number", - "minimum": 0 - } - }, - "required": [ - "sourceRevision", - "sources", - "total", - "limit", - "offset" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false + "enum": [ + "restart-initiated" + ] + } + }, + "required": [ + "ok", + "status" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false }, { "type": "object", @@ -68534,7 +68460,7 @@ } } }, - "404": { + "413": { "description": "Response", "content": { "application/json": { @@ -68594,7 +68520,7 @@ } } }, - "409": { + "415": { "description": "Response", "content": { "application/json": { @@ -68654,7 +68580,40 @@ } } }, - "413": { + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { "description": "Response", "content": { "application/json": { @@ -68714,7 +68673,7 @@ } } }, - "415": { + "503": { "description": "Response", "content": { "application/json": { @@ -68773,24 +68732,1389 @@ } } } - }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^\\d+$" - } - } - }, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "error": { - "type": "string" + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "service" + } + }, + "/api/data-manager/transit/sources": { + "get": { + "operationId": "getApiDataManagerTransitSources", + "parameters": [ + { + "schema": { + "type": "string" + }, + "in": "query", + "name": "country", + "required": false + }, + { + "schema": { + "type": "string" + }, + "in": "query", + "name": "search", + "required": false + }, + { + "schema": { + "type": "string", + "enum": [ + "active", + "add-pending", + "update-pending", + "removal-pending", + "disabled", + "failed", + "stale" + ] + }, + "in": "query", + "name": "lifecycle", + "required": false + }, + { + "schema": { + "type": "string", + "enum": [ + "catalog", + "operator" + ] + }, + "in": "query", + "name": "origin", + "required": false + }, + { + "schema": { + "type": "number" + }, + "in": "query", + "name": "limit", + "required": false + }, + { + "schema": { + "type": "number" + }, + "in": "query", + "name": "offset", + "required": false + } + ], + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "region": { + "type": "string" + }, + "name": { + "type": "string" + }, + "format": { + "type": "string", + "enum": [ + "gtfs", + "netex" + ] + }, + "origin": { + "type": "string", + "enum": [ + "catalog", + "operator" + ] + }, + "originUrl": { + "type": "string" + }, + "license": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/Admin_getApiDataManagerTransitSources_200___schema0" + } + }, + "requested": { + "type": "boolean" + }, + "sourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "desiredState": { + "type": "string", + "enum": [ + "enabled", + "disabled", + "removed" + ] + }, + "actions": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "disable": { + "type": "boolean" + }, + "remove": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "enable", + "disable", + "remove", + "reason" + ], + "additionalProperties": false + }, + "active": { + "type": "boolean" + }, + "activeEpoch": { + "type": "string" + }, + "artifact": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "sizeBytes": { + "type": "number", + "minimum": 0 + }, + "retrievedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "path", + "sha256", + "sizeBytes", + "retrievedAt" + ], + "additionalProperties": false + }, + "lastFetchedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lastImportedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "validationStatus": { + "type": "string" + }, + "validationMessage": { + "type": "string" + }, + "lifecycle": { + "type": "string", + "enum": [ + "active", + "add-pending", + "update-pending", + "removal-pending", + "disabled", + "failed", + "stale" + ] + } + }, + "required": [ + "id", + "region", + "name", + "format", + "origin", + "license", + "requested", + "sourceRevision", + "desiredState", + "actions", + "active", + "lifecycle" + ], + "additionalProperties": false + } + }, + "total": { + "type": "number", + "minimum": 0 + }, + "limit": { + "type": "number", + "minimum": 0 + }, + "offset": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "sourceRevision", + "sources", + "total", + "limit", + "offset" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "service" + }, + "post": { + "operationId": "postApiDataManagerTransitSources", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "region": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri" + }, + "license": { + "type": "object", + "properties": { + "spdxIdentifier": { + "type": "string" + }, + "url": { + "type": "string" + }, + "attribution": { + "type": "string", + "minLength": 1 + }, + "publisher": { + "type": "string" + }, + "publisherUrl": { + "type": "string" + } + }, + "required": [ + "attribution" + ], + "additionalProperties": false + } + }, + "required": [ + "expectedSourceRevision", + "region", + "name", + "url", + "license" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "sourceId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "started" + ] + } + }, + "required": [ + "jobId", + "sourceId", + "status" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] + }, + "existingJobId": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "ok", + "reason", + "existingJobId" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" }, "retryAfter": { "type": "integer", @@ -68872,68 +70196,40 @@ "data-manager-admin" ], "x-openmapx-auth": "service" - }, - "post": { - "operationId": "postApiDataManagerTransitSources", + } + }, + "/api/data-manager/transit/sources/{sourceId}": { + "delete": { + "operationId": "deleteApiDataManagerTransitSourcesSourceId", + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "sourceId", + "required": true + } + ], "requestBody": { "required": true, "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "expectedSourceRevision": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "idempotencyKey": { - "type": "string", - "maxLength": 200 - }, - "region": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "url": { - "type": "string", - "format": "uri" - }, - "license": { - "type": "object", - "properties": { - "spdxIdentifier": { - "type": "string" - }, - "url": { - "type": "string" - }, - "attribution": { - "type": "string", - "minLength": 1 - }, - "publisher": { - "type": "string" - }, - "publisherUrl": { - "type": "string" - } - }, - "required": [ - "attribution" - ], - "additionalProperties": false + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" } }, "required": [ - "expectedSourceRevision", - "region", - "name", - "url", - "license" + "expectedSourceRevision" ], "additionalProperties": false } @@ -69551,9 +70847,9 @@ "x-openmapx-auth": "service" } }, - "/api/data-manager/transit/sources/{sourceId}": { - "delete": { - "operationId": "deleteApiDataManagerTransitSourcesSourceId", + "/api/data-manager/transit/sources/{sourceId}/disable": { + "post": { + "operationId": "postApiDataManagerTransitSourcesSourceIdDisable", "parameters": [ { "schema": { @@ -69620,58 +70916,358 @@ } } }, - "400": { + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { "description": "Response", "content": { "application/json": { "schema": { "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + }, { "type": "object", "properties": { - "error": { - "type": "string" - }, "ok": { "type": "boolean", "enum": [ false ] }, - "message": { - "type": "string" + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] }, - "jobId": { - "type": "string" + "existingJobId": { + "type": [ + "string", + "null" + ] } }, "required": [ - "error" + "ok", + "reason", + "existingJobId" ], "additionalProperties": false }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "ok": { + "type": "boolean", + "enum": [ + false + ] }, "error": { "type": "string" }, - "message": { - "type": "string" - }, "code": { - "type": "string" + "type": "string", + "enum": [ + "source_action_not_allowed" + ] } }, "required": [ - "statusCode", + "ok", "error", - "message" + "code" ], "additionalProperties": false } @@ -69680,7 +71276,7 @@ } } }, - "401": { + "413": { "description": "Response", "content": { "application/json": { @@ -69740,7 +71336,7 @@ } } }, - "403": { + "415": { "description": "Response", "content": { "application/json": { @@ -69800,7 +71396,40 @@ } } }, - "404": { + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { "description": "Response", "content": { "application/json": { @@ -69859,119 +71488,135 @@ } } } - }, - "409": { + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "service" + } + }, + "/api/data-manager/transit/sources/{sourceId}/enable": { + "post": { + "operationId": "postApiDataManagerTransitSourcesSourceIdEnable", + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "sourceId", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "expectedSourceRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "expectedSourceRevision" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { "description": "Response", "content": { "application/json": { "schema": { - "anyOf": [ - { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "sourceId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "started" ] - }, + } + }, + "required": [ + "jobId", + "sourceId", + "status" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ { "type": "object", "properties": { + "error": { + "type": "string" + }, "ok": { "type": "boolean", "enum": [ false ] }, - "reason": { - "type": "string", - "enum": [ - "in-flight", - "duplicate-idempotency-key" - ] + "message": { + "type": "string" }, - "existingJobId": { - "type": [ - "string", - "null" - ] + "jobId": { + "type": "string" } }, "required": [ - "ok", - "reason", - "existingJobId" + "error" ], "additionalProperties": false }, { "type": "object", "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, "error": { "type": "string" }, + "message": { + "type": "string" + }, "code": { - "type": "string", - "enum": [ - "source_action_not_allowed" - ] + "type": "string" } }, "required": [ - "ok", + "statusCode", "error", - "code" + "message" ], "additionalProperties": false } @@ -69980,7 +71625,7 @@ } } }, - "413": { + "401": { "description": "Response", "content": { "application/json": { @@ -70040,7 +71685,7 @@ } } }, - "415": { + "403": { "description": "Response", "content": { "application/json": { @@ -70100,40 +71745,7 @@ } } }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^\\d+$" - } - } - }, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "retryAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "error", - "retryAfter" - ], - "additionalProperties": false - } - } - } - }, - "500": { + "404": { "description": "Response", "content": { "application/json": { @@ -70192,84 +71804,128 @@ } } } - } - }, - "tags": [ - "data-manager-admin" - ], - "x-openmapx-auth": "service" - } - }, - "/api/data-manager/transit/sources/{sourceId}/disable": { - "post": { - "operationId": "postApiDataManagerTransitSourcesSourceIdDisable", - "parameters": [ - { - "schema": { - "type": "string", - "minLength": 1 - }, - "in": "path", - "name": "sourceId", - "required": true - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "idempotencyKey": { - "type": "string", - "maxLength": 200 - }, - "expectedSourceRevision": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - } - }, - "required": [ - "expectedSourceRevision" - ], - "additionalProperties": false - } - } - } - }, - "responses": { - "202": { + }, + "409": { "description": "Response", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "jobId": { - "type": "string" + "anyOf": [ + { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] }, - "sourceId": { - "type": "string" + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "reason": { + "type": "string", + "enum": [ + "in-flight", + "duplicate-idempotency-key" + ] + }, + "existingJobId": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "ok", + "reason", + "existingJobId" + ], + "additionalProperties": false }, - "status": { - "type": "string", - "enum": [ - "started" - ] + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "source_action_not_allowed" + ] + } + }, + "required": [ + "ok", + "error", + "code" + ], + "additionalProperties": false } - }, - "required": [ - "jobId", - "sourceId", - "status" - ], - "additionalProperties": false + ] } } } }, - "400": { + "413": { "description": "Response", "content": { "application/json": { @@ -70329,7 +71985,7 @@ } } }, - "401": { + "415": { "description": "Response", "content": { "application/json": { @@ -70389,7 +72045,40 @@ } } }, - "403": { + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { "description": "Response", "content": { "application/json": { @@ -70448,8 +72137,140 @@ } } } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "service" + } + }, + "/api/data-manager/transit/state": { + "get": { + "operationId": "getApiDataManagerTransitState", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "transitousRef": { + "type": [ + "string", + "null" + ] + }, + "transitousLockedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "transitousLockedBy": { + "type": [ + "string", + "null" + ] + }, + "lastSyncAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "lastSyncStatus": { + "type": [ + "string", + "null" + ] + }, + "currentJob": { + "anyOf": [ + { + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "startedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "jobId", + "startedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "feedCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "feeds": { + "type": "object", + "properties": { + "byRegion": { + "type": "object", + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "byStatus": { + "type": "object", + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "byRegion", + "byStatus" + ], + "additionalProperties": false + } + }, + "required": [ + "transitousRef", + "transitousLockedAt", + "transitousLockedBy", + "lastSyncAt", + "lastSyncStatus", + "currentJob", + "feedCount", + "feeds" + ], + "additionalProperties": false + } + } + } }, - "404": { + "400": { "description": "Response", "content": { "application/json": { @@ -70509,118 +72330,118 @@ } } }, - "409": { + "401": { "description": "Response", "content": { "application/json": { "schema": { "anyOf": [ { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false + "type": "object", + "properties": { + "error": { + "type": "string" }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" } - ] + }, + "required": [ + "error" + ], + "additionalProperties": false }, { "type": "object", "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, "ok": { "type": "boolean", "enum": [ false ] }, - "reason": { - "type": "string", - "enum": [ - "in-flight", - "duplicate-idempotency-key" - ] + "message": { + "type": "string" }, - "existingJobId": { - "type": [ - "string", - "null" - ] + "jobId": { + "type": "string" } }, "required": [ - "ok", - "reason", - "existingJobId" + "error" ], "additionalProperties": false }, { "type": "object", "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, "error": { "type": "string" }, + "message": { + "type": "string" + }, "code": { - "type": "string", - "enum": [ - "source_action_not_allowed" - ] + "type": "string" } }, "required": [ - "ok", + "statusCode", "error", - "code" + "message" ], "additionalProperties": false } @@ -70849,20 +72670,9 @@ "x-openmapx-auth": "service" } }, - "/api/data-manager/transit/sources/{sourceId}/enable": { + "/api/data-manager/transit/sync": { "post": { - "operationId": "postApiDataManagerTransitSourcesSourceIdEnable", - "parameters": [ - { - "schema": { - "type": "string", - "minLength": 1 - }, - "in": "path", - "name": "sourceId", - "required": true - } - ], + "operationId": "postApiDataManagerTransitSync", "requestBody": { "required": true, "content": { @@ -70870,17 +72680,33 @@ "schema": { "type": "object", "properties": { - "idempotencyKey": { + "countries": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + }, + "preflightId": { "type": "string", - "maxLength": 200 + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" }, - "expectedSourceRevision": { + "fingerprint": { "type": "string", "pattern": "^[a-f0-9]{64}$" + }, + "idempotencyKey": { + "type": "string", + "maxLength": 200 } }, "required": [ - "expectedSourceRevision" + "countries", + "preflightId", + "fingerprint" ], "additionalProperties": false } @@ -70895,10 +72721,13 @@ "schema": { "type": "object", "properties": { - "jobId": { - "type": "string" + "ok": { + "type": "boolean", + "enum": [ + true + ] }, - "sourceId": { + "jobId": { "type": "string" }, "status": { @@ -70909,8 +72738,8 @@ } }, "required": [ + "ok", "jobId", - "sourceId", "status" ], "additionalProperties": false @@ -71098,66 +72927,6 @@ } } }, - "404": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, "409": { "description": "Response", "content": { @@ -71227,109 +72996,42 @@ ] }, "reason": { - "type": "string", - "enum": [ - "in-flight", - "duplicate-idempotency-key" - ] - }, - "existingJobId": { - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "ok", - "reason", - "existingJobId" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "error": { - "type": "string" - }, - "code": { - "type": "string", - "enum": [ - "source_action_not_allowed" - ] - } - }, - "required": [ - "ok", - "error", - "code" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "413": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", + "type": "string", "enum": [ - false + "in-flight", + "duplicate-idempotency-key" ] }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" + "existingJobId": { + "type": [ + "string", + "null" + ] } }, "required": [ - "error" + "ok", + "reason", + "existingJobId" ], "additionalProperties": false }, { "type": "object", "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, "error": { "type": "string" }, - "message": { - "type": "string" - }, "code": { - "type": "string" + "type": "string", + "enum": [ + "preflight_stale" + ] } }, "required": [ - "statusCode", "error", - "message" + "code" ], "additionalProperties": false } @@ -71338,7 +73040,7 @@ } } }, - "415": { + "413": { "description": "Response", "content": { "application/json": { @@ -71398,40 +73100,7 @@ } } }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^\\d+$" - } - } - }, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "retryAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "error", - "retryAfter" - ], - "additionalProperties": false - } - } - } - }, - "500": { + "415": { "description": "Response", "content": { "application/json": { @@ -71490,200 +73159,67 @@ } } } - } - }, - "tags": [ - "data-manager-admin" - ], - "x-openmapx-auth": "service" - } - }, - "/api/data-manager/transit/state": { - "get": { - "operationId": "getApiDataManagerTransitState", - "responses": { - "200": { + }, + "422": { "description": "Response", "content": { "application/json": { "schema": { "type": "object", "properties": { - "transitousRef": { - "type": [ - "string", - "null" - ] - }, - "transitousLockedAt": { - "anyOf": [ - { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - { - "type": "null" - } - ] - }, - "transitousLockedBy": { - "type": [ - "string", - "null" - ] - }, - "lastSyncAt": { - "anyOf": [ - { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - { - "type": "null" - } - ] - }, - "lastSyncStatus": { - "type": [ - "string", - "null" - ] - }, - "currentJob": { - "anyOf": [ - { - "type": "object", - "properties": { - "jobId": { - "type": "string" - }, - "startedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - } - }, - "required": [ - "jobId", - "startedAt" - ], - "additionalProperties": false - }, - { - "type": "null" - } + "ok": { + "type": "boolean", + "enum": [ + false ] }, - "feedCount": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "feeds": { - "type": "object", - "properties": { - "byRegion": { - "type": "object", - "additionalProperties": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "byStatus": { - "type": "object", - "additionalProperties": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - } - }, - "required": [ - "byRegion", - "byStatus" - ], - "additionalProperties": false + "reason": { + "type": "string" } }, "required": [ - "transitousRef", - "transitousLockedAt", - "transitousLockedBy", - "lastSyncAt", - "lastSyncStatus", - "currentJob", - "feedCount", - "feeds" + "ok", + "reason" ], "additionalProperties": false } } } }, - "400": { + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, "content": { "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } - ] + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false } } } }, - "401": { + "500": { "description": "Response", "content": { "application/json": { @@ -71742,68 +73278,326 @@ } } } - }, - "403": { + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "service" + } + }, + "/api/data-manager/transit/sync/preflight": { + "post": { + "operationId": "postApiDataManagerTransitSyncPreflight", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "countries": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "countries" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { "description": "Response", "content": { "application/json": { "schema": { - "anyOf": [ - { + "type": "object", + "properties": { + "preflightId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowedCountries": { + "type": "array", + "items": { + "type": "string" + } + }, + "result": { "type": "object", "properties": { - "error": { - "type": "string" - }, "ok": { - "type": "boolean", - "enum": [ - false - ] + "type": "boolean" }, - "message": { - "type": "string" + "policy": { + "type": "object", + "properties": { + "profile": { + "type": "string", + "enum": [ + "regional-assisted", + "regional-sovereign", + "planet" + ] + }, + "countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "feedAllowList": { + "type": "array", + "items": { + "type": "string" + } + }, + "acquisition": { + "type": "string", + "enum": [ + "mirror", + "build" + ] + }, + "artifactOrigins": { + "type": "array", + "items": { + "type": "string" + } + }, + "originDownloadsRequired": { + "type": "boolean" + }, + "hostedRuntimeFallbackAllowed": { + "type": "boolean" + }, + "osmInput": { + "type": "string" + }, + "gbfsSelection": { + "type": "string", + "enum": [ + "explicit-countries", + "explicit-feeds", + "allow-listed-batches" + ] + }, + "maxFeedCount": { + "type": "number", + "minimum": 0 + }, + "retentionGenerations": { + "type": "number", + "minimum": 0 + }, + "resourceEnvelope": { + "type": "object", + "properties": { + "minimumMemoryGb": { + "type": "number", + "minimum": 0 + }, + "minimumCpu": { + "type": "number", + "minimum": 0 + }, + "minimumDiskGb": { + "type": "number", + "minimum": 0 + }, + "operatorCapacityRequired": { + "type": "boolean" + } + }, + "required": [ + "minimumMemoryGb", + "minimumCpu", + "minimumDiskGb", + "operatorCapacityRequired" + ], + "additionalProperties": false + }, + "updateCadenceHours": { + "type": "number", + "minimum": 0 + }, + "experimental": { + "type": "boolean" + } + }, + "required": [ + "profile", + "countries", + "feedAllowList", + "acquisition", + "artifactOrigins", + "originDownloadsRequired", + "hostedRuntimeFallbackAllowed", + "gbfsSelection", + "maxFeedCount", + "retentionGenerations", + "resourceEnvelope", + "updateCadenceHours", + "experimental" + ], + "additionalProperties": false }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "estimate": { + "type": "object", + "properties": { + "feedCount": { + "type": "number", + "minimum": 0 + }, + "measuredCompressedBytes": { + "type": "number", + "minimum": 0 + }, + "compressedBytes": { + "type": "number", + "minimum": 0 + }, + "expandedGtfsBytes": { + "type": "number", + "minimum": 0 + }, + "importedMotisBytes": { + "type": "number", + "minimum": 0 + }, + "osmBytes": { + "type": "number", + "minimum": 0 + }, + "proxyCacheBytes": { + "type": "number", + "minimum": 0 + }, + "requiredDiskBytes": { + "type": "number", + "minimum": 0 + }, + "basis": { + "type": "string", + "enum": [ + "measured-and-conservative", + "conservative-defaults" + ] + } + }, + "required": [ + "feedCount", + "measuredCompressedBytes", + "compressedBytes", + "expandedGtfsBytes", + "importedMotisBytes", + "osmBytes", + "proxyCacheBytes", + "requiredDiskBytes", + "basis" + ], + "additionalProperties": false }, - "error": { - "type": "string" + "capacity": { + "type": "object", + "properties": { + "freeDiskBytes": { + "type": "number", + "minimum": 0 + }, + "freeInodes": { + "type": "number", + "minimum": 0 + }, + "slotMemoryGb": { + "type": "number", + "minimum": 0 + }, + "slotCpu": { + "type": "number", + "minimum": 0 + }, + "fileDescriptorLimit": { + "type": "number", + "minimum": 0 + }, + "buildTimeoutHours": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "freeDiskBytes", + "slotMemoryGb", + "slotCpu", + "fileDescriptorLimit", + "buildTimeoutHours" + ], + "additionalProperties": false }, - "message": { - "type": "string" + "blockers": { + "type": "array", + "items": { + "type": "string" + } }, - "code": { - "type": "string" + "warnings": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ - "statusCode", - "error", - "message" + "ok", + "policy", + "estimate", + "capacity", + "blockers", + "warnings" ], "additionalProperties": false } - ] + }, + "required": [ + "preflightId", + "expiresAt", + "fingerprint", + "countries", + "allowedCountries", + "result" + ], + "additionalProperties": false } } } }, - "413": { + "400": { "description": "Response", "content": { "application/json": { @@ -71863,7 +73657,7 @@ } } }, - "415": { + "401": { "description": "Response", "content": { "application/json": { @@ -71923,40 +73717,7 @@ } } }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^\\d+$" - } - } - }, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "retryAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "error", - "retryAfter" - ], - "additionalProperties": false - } - } - } - }, - "500": { + "403": { "description": "Response", "content": { "application/json": { @@ -72015,77 +73776,34 @@ } } } - } - }, - "tags": [ - "data-manager-admin" - ], - "x-openmapx-auth": "service" - } - }, - "/api/data-manager/transit/sync": { - "post": { - "operationId": "postApiDataManagerTransitSync", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "idempotencyKey": { - "type": "string", - "maxLength": 200 - }, - "countries": { - "maxItems": 250, - "type": "array", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,31}$" - } - } - }, - "additionalProperties": false - } - } - } - }, - "responses": { - "202": { + }, + "409": { "description": "Response", "content": { "application/json": { "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] - }, - "jobId": { + "error": { "type": "string" }, - "status": { + "code": { "type": "string", "enum": [ - "started" + "preflight_stale" ] } }, "required": [ - "ok", - "jobId", - "status" + "error", + "code" ], "additionalProperties": false } } } }, - "400": { + "413": { "description": "Response", "content": { "application/json": { @@ -72145,7 +73863,7 @@ } } }, - "401": { + "415": { "description": "Response", "content": { "application/json": { @@ -72205,7 +73923,7 @@ } } }, - "403": { + "422": { "description": "Response", "content": { "application/json": { @@ -72265,104 +73983,40 @@ } } }, - "409": { + "429": { "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, "content": { "application/json": { "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "reason": { - "type": "string", - "enum": [ - "in-flight", - "duplicate-idempotency-key" - ] + "error": { + "type": "string" }, - "existingJobId": { - "type": [ - "string", - "null" - ] + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "required": [ - "ok", - "reason", - "existingJobId" + "error", + "retryAfter" ], "additionalProperties": false } } } }, - "413": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "415": { + "500": { "description": "Response", "content": { "application/json": { @@ -72422,66 +74076,7 @@ } } }, - "422": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "reason": { - "type": "string" - } - }, - "required": [ - "ok", - "reason" - ], - "additionalProperties": false - } - } - } - }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^\\d+$" - } - } - }, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "retryAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "error", - "retryAfter" - ], - "additionalProperties": false - } - } - } - }, - "500": { + "503": { "description": "Response", "content": { "application/json": { diff --git a/apps/api/src/routes/__tests__/data-manager-admin-contract.test.ts b/apps/api/src/routes/__tests__/data-manager-admin-contract.test.ts index c2551bde6..02efe88c7 100644 --- a/apps/api/src/routes/__tests__/data-manager-admin-contract.test.ts +++ b/apps/api/src/routes/__tests__/data-manager-admin-contract.test.ts @@ -8,6 +8,11 @@ import { administrationSdkFetch } from "./helpers/administration-sdk-fetch"; afterEach(() => vi.unstubAllGlobals()); +const reviewed = { + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), +}; describe("application administration contract registration", () => { it("registers the canonical relative path and documents its actual auth", async () => { const app = Fastify(); @@ -94,7 +99,7 @@ describe("application administration contract registration", () => { ); vi.stubGlobal("fetch", administrationSdkFetch(app)); const client = createApiClient({ baseUrl: "https://app.test", credentials: "include" }); - const oversizedBody = { idempotencyKey: "x".repeat(200) }; + const oversizedBody = { ...reviewed, idempotencyKey: "x".repeat(200) }; try { const media = await baseline.inject({ method: "POST", @@ -111,10 +116,10 @@ describe("application administration contract registration", () => { expect(media.statusCode).toBe(415); expect(oversized.statusCode).toBe(413); await expect( - postApiDataManagerTransitSync( - {}, - { client, headers: { "Content-Type": "application/xml" } }, - ), + postApiDataManagerTransitSync(reviewed, { + client, + headers: { "Content-Type": "application/xml" }, + }), ).resolves.toMatchObject({ status: 415, data: media.json() }); await expect(postApiDataManagerTransitSync(oversizedBody, { client })).resolves.toMatchObject( { status: 413, data: oversized.json() }, @@ -146,7 +151,7 @@ describe("application administration contract registration", () => { const response = await app.inject({ method: "POST", url: "/data-manager/transit/sync", - payload: { countries: ["de"] }, + payload: reviewed, }); expect(response.statusCode).toBe(409); expect(response.json()).toEqual({ ok: false, reason: "in-flight", existingJobId: null }); @@ -193,7 +198,7 @@ describe("shared registration response kinds and ordering", () => { expect(calls).toEqual(["before validation"]); calls.length = 0; expect( - (await app.inject({ method: "POST", url: "/data-manager/transit/sync", payload: {} })) + (await app.inject({ method: "POST", url: "/data-manager/transit/sync", payload: reviewed })) .statusCode, ).toBe(202); expect(calls).toEqual(["before validation", "inherited guard", "route guard", "handler"]); diff --git a/apps/api/src/routes/__tests__/data-manager-sources.test.ts b/apps/api/src/routes/__tests__/data-manager-sources.test.ts index 593f4d62a..fd68f3042 100644 --- a/apps/api/src/routes/__tests__/data-manager-sources.test.ts +++ b/apps/api/src/routes/__tests__/data-manager-sources.test.ts @@ -160,3 +160,26 @@ describe("transit source proxy", () => { expect(audit).not.toHaveBeenCalled(); }); }); +it("pages the deduplicated catalog without losing entries beyond500", async () => { + const { searchTransitCatalog } = await import("../../services/transit-catalog/index.js"); + vi.mocked(searchTransitCatalog).mockResolvedValueOnce( + Array.from({ length: 601 }, (_, n) => ({ + id: `catalog:de:${String(n).padStart(4, "0")}`, + installedSourceId: `catalog:de:${String(n).padStart(4, "0")}`, + discoveryIds: [], + name: `Source ${String(n).padStart(4, "0")}`, + source: "transitous" as const, + countryCode: "de", + url: `https://example.test/${n}.zip`, + })), + ); + const response = await app.inject({ + method: "GET", + url: "/data-manager/transit/catalog?offset=550&limit=50&country=de", + headers: { authorization: "Bearer service-token" }, + }); + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ total: 601, limit: 50, offset: 550 }); + expect(response.json().sources[0].installedSourceId).toBe("catalog:de:0550"); + expect(response.json().sources).toHaveLength(50); +}); diff --git a/apps/api/src/routes/__tests__/data-manager-sync.test.ts b/apps/api/src/routes/__tests__/data-manager-sync.test.ts index 7bd7f62b3..f06c3bfca 100644 --- a/apps/api/src/routes/__tests__/data-manager-sync.test.ts +++ b/apps/api/src/routes/__tests__/data-manager-sync.test.ts @@ -36,6 +36,11 @@ beforeAll(() => { vi.stubGlobal("fetch", fetchMock); }); +const reviewed = { + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), +}; let app: FastifyInstance; beforeAll(async () => { @@ -76,7 +81,7 @@ describe("POST /data-manager/transit/sync", () => { method: "POST", url: "/data-manager/transit/sync", headers: { authorization: "Bearer service-token" }, - payload: { idempotencyKey: "abc-123" }, + payload: { ...reviewed, idempotencyKey: "abc-123" }, }); expect(res.statusCode).toBe(202); @@ -110,7 +115,7 @@ describe("POST /data-manager/transit/sync", () => { method: "POST", url: "/data-manager/transit/sync", headers: { authorization: "Bearer service-token" }, - payload: {}, + payload: reviewed, }); expect(res.statusCode).toBe(409); @@ -128,7 +133,7 @@ describe("POST /data-manager/transit/sync", () => { method: "POST", url: "/data-manager/transit/sync", headers: { authorization: "Bearer service-token" }, - payload: {}, + payload: reviewed, }); expect(res.statusCode).toBe(500); @@ -150,7 +155,7 @@ describe("POST /data-manager/transit/sync", () => { method: "POST", url: "/data-manager/transit/sync", headers: { authorization: "Bearer service-token" }, - payload: {}, + payload: reviewed, }); expect(res.statusCode).toBe(500); diff --git a/apps/api/src/routes/data-manager.ts b/apps/api/src/routes/data-manager.ts index 2898307ed..e5585fdd4 100644 --- a/apps/api/src/routes/data-manager.ts +++ b/apps/api/src/routes/data-manager.ts @@ -146,7 +146,14 @@ export async function dataManagerRoute(app: FastifyInstance): Promise { try { const { searchTransitCatalog } = await import("../services/transit-catalog/index.js"); const sources = await searchTransitCatalog(req.query.search, req.query.country); - return { sources, total: sources.length }; + const limit = req.query.limit ?? 50, + offset = req.query.offset ?? 0; + return { + sources: sources.slice(offset, offset + limit), + total: sources.length, + limit, + offset, + }; } catch (error) { return reply.code(502).send({ error: (error as Error).message }); } @@ -408,6 +415,18 @@ export async function dataManagerRoute(app: FastifyInstance): Promise { }, ); + registerDataManagerAdminRoute(app, "getApiDataManagerTransitProfile", async (_req, reply) => { + const result = await dm.getTransitProfile(dataManagerOptions()); + return reply.code(result.status).send(result.data); + }); + registerDataManagerAdminRoute( + app, + "postApiDataManagerTransitSyncPreflight", + async (req, reply) => { + const result = await dm.inspectTransitSyncPreflight(req.body, dataManagerOptions()); + return reply.code(result.status).send(result.data); + }, + ); registerDataManagerAdminRoute(app, "postApiDataManagerTransitSync", async (req, reply) => { const auth = await authenticateDataManager(req); @@ -417,6 +436,8 @@ export async function dataManagerRoute(app: FastifyInstance): Promise { { idempotencyKey: body.idempotencyKey, countries: body.countries, + preflightId: body.preflightId, + fingerprint: body.fingerprint, triggeredBy, }, dataManagerOptions(), diff --git a/apps/api/src/services/transit-catalog/index.test.ts b/apps/api/src/services/transit-catalog/index.test.ts index 1086cabbf..fc9db436e 100644 --- a/apps/api/src/services/transit-catalog/index.test.ts +++ b/apps/api/src/services/transit-catalog/index.test.ts @@ -1,9 +1,14 @@ import { licensingFromDeclaration } from "@openmapx/mobility-core/licensing"; import { afterEach, describe, expect, it, vi } from "vitest"; -vi.mock("../../integration-host.js", () => ({ getIntegrationsByDomain: () => [] })); +const discovery = vi.hoisted(() => ({ integrations: [] as unknown[] })); +vi.mock("../../integration-host.js", () => ({ + getIntegrationsByDomain: () => discovery.integrations, +})); afterEach(() => { + vi.restoreAllMocks(); + discovery.integrations = []; vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); @@ -60,3 +65,173 @@ describe("installed transit catalog", () => { expect(fetchMock).toHaveBeenCalledTimes(1); }); }); + +it("enriches discovery duplicates without losing installed identity or licensing", async () => { + discovery.integrations = [ + { + providers: new Map([ + [ + "gtfs-catalog", + [ + { + id: "fixture", + listFeeds: async () => [ + { + id: "mobilitydb:123", + source: "mobilitydb", + name: "Discovery name", + countryCode: "de", + url: "https://example.test/demo.zip", + licensing: licensingFromDeclaration("CC0-1.0", undefined), + mdbId: "123", + }, + ], + }, + ], + ], + ]), + }, + ]; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ + sources: [ + { + id: "catalog:de:Demo", + region: "de", + name: "Installed Demo", + format: "gtfs", + originUrl: "https://example.test/demo.zip", + license: { + "spdx-identifier": "CC-BY-4.0", + "attribution-text": "Installed operator credit", + }, + }, + ], + }), + ), + ); + const { searchTransitCatalog } = await import("./index.js"); + expect(await searchTransitCatalog()).toEqual([ + expect.objectContaining({ + id: "catalog:de:Demo", + installedSourceId: "catalog:de:Demo", + source: "transitous", + name: "Installed Demo", + discoveryIds: ["mobilitydb:123"], + licensing: { + ...licensingFromDeclaration("CC-BY-4.0", undefined), + bindings: [ + { + ...licensingFromDeclaration("CC-BY-4.0", undefined).bindings[0], + notice: "Installed operator credit", + }, + ], + }, + }), + ]); +}); +it("retains installed rows when optional discovery is unavailable", async () => { + discovery.integrations = [ + { + providers: new Map([ + [ + "gtfs-catalog", + [ + { + id: "fixture", + listFeeds: async () => { + throw Error("fixture unavailable"); + }, + }, + ], + ], + ]), + }, + ]; + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ + sources: [ + { + id: "catalog:de:Demo", + region: "de", + name: "Demo", + format: "gtfs", + originUrl: "https://example.test/demo.zip", + license: {}, + }, + ], + }), + ), + ); + const { searchTransitCatalog } = await import("./index.js"); + expect(await searchTransitCatalog()).toEqual([ + expect.objectContaining({ + id: "catalog:de:Demo", + installedSourceId: "catalog:de:Demo", + discoveryIds: [], + }), + ]); +}); +it.each([ + { id: "broken", source: "mobilitydb", countryCode: "de", url: "https://example.test/feed.zip" }, + { + id: "credential", + source: "mobilitydb", + name: "Private URL", + countryCode: "de", + url: "https://example.test/feed.zip?api_key=synthetic-credential", + }, +])( + "does not let malformed or credential-bearing discovery erase or expose installed rows", + async (discovered) => { + discovery.integrations = [ + { + providers: new Map([ + ["gtfs-catalog", [{ id: "fixture", listFeeds: async () => [discovered] }]], + ]), + }, + ]; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ + sources: [ + { + id: "catalog:de:Demo", + region: "de", + name: "Demo", + format: "gtfs", + originUrl: "https://example.test/demo.zip", + license: {}, + }, + ], + }), + ), + ); + const { getTransitCatalog } = await import("./index.js"); + expect((await getTransitCatalog()).map((row) => row.id)).toEqual(["catalog:de:Demo"]); + }, +); +it("keeps source601 searchable after building the full installed catalog", async () => { + const sources = Array.from({ length: 601 }, (_, n) => ({ + id: `catalog:de:${String(n).padStart(4, "0")}`, + region: "de", + name: `Source ${String(n).padStart(4, "0")}`, + format: "gtfs", + originUrl: `https://example.test/${n}.zip`, + license: {}, + })); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ sources })), + ); + const { searchTransitCatalog } = await import("./index.js"); + expect((await searchTransitCatalog("Source 0600", "DE")).map((row) => row.id)).toEqual([ + "catalog:de:0600", + ]); +}); diff --git a/apps/api/src/services/transit-catalog/index.ts b/apps/api/src/services/transit-catalog/index.ts index bda76cb73..77b6bf1a9 100644 --- a/apps/api/src/services/transit-catalog/index.ts +++ b/apps/api/src/services/transit-catalog/index.ts @@ -1,9 +1,11 @@ import { normalizeProducerUrl } from "@integrations/transit-mobility-database"; +import { transitCatalogFeedSchema } from "@openmapx/core/api-contracts"; import { createDataManagerTransport, getTransitCatalog as readPinnedTransitCatalog, unwrapDataManagerResponse, } from "@openmapx/core/data-manager-sdk"; +import { DIAGNOSTIC_CREDENTIAL_KEY_SOURCE } from "@openmapx/core/scrub-secrets"; import { envString } from "@openmapx/core/server-env"; import type { GtfsCatalogProvider } from "@openmapx/integration-framework"; import { licensingFromDeclaration } from "@openmapx/mobility-core/licensing"; @@ -22,6 +24,12 @@ async function installedCatalog(): Promise { if (!source.originUrl) return []; const spdx = source.license?.["spdx-identifier"]; const licenseUrl = source.license?.url; + const licensing = licensingFromDeclaration( + typeof spdx === "string" ? spdx : undefined, + typeof licenseUrl === "string" ? licenseUrl : undefined, + ); + const notice = source.license?.["attribution-text"]; + if (typeof notice === "string" && licensing.bindings[0]) licensing.bindings[0].notice = notice; return [ { id: source.id, @@ -29,22 +37,42 @@ async function installedCatalog(): Promise { source: "transitous" as const, countryCode: source.region.split(/[.-]/)[0]?.toLowerCase() ?? source.region, url: source.originUrl, - licensing: licensingFromDeclaration( - typeof spdx === "string" ? spdx : undefined, - typeof licenseUrl === "string" ? licenseUrl : undefined, - ), + licensing, }, ]; }); } +const discoveryFeedSchema = transitCatalogFeedSchema + .omit({ installedSourceId: true, discoveryIds: true }) + .strip(); +const credentialQuery = new RegExp(`^(?:${DIAGNOSTIC_CREDENTIAL_KEY_SOURCE}|authorization)$`, "i"); +function credentialFreeDiscoveryUrl(raw: string): boolean { + try { + const url = new URL(raw); + return ( + ["http:", "https:"].includes(url.protocol) && + !url.username && + !url.password && + ![...url.searchParams.keys()].some((key) => credentialQuery.test(key)) + ); + } catch { + return false; + } +} async function discoveryCatalog(): Promise { const feeds: TransitCatalogFeed[] = []; for (const integration of getIntegrationsByDomain("gtfs-catalog")) { const providers = (integration.providers.get("gtfs-catalog") ?? []) as GtfsCatalogProvider[]; for (const provider of providers) { try { - feeds.push(...((await provider.listFeeds()) as TransitCatalogFeed[])); + const candidates = await provider.listFeeds(); + if (!Array.isArray(candidates)) continue; + for (const candidate of candidates) { + const parsed = discoveryFeedSchema.safeParse(candidate); + if (parsed.success && credentialFreeDiscoveryUrl(parsed.data.url)) + feeds.push(parsed.data); + } } catch (error) { console.warn(`[transit-catalog] provider "${provider.id}" failed:`, error); } @@ -64,19 +92,38 @@ export async function getTransitCatalog(): Promise { return []; }), ]); - const discoveredUrls = new Set( - discovered - .map((feed) => normalizeProducerUrl(feed.url)) - .filter((url): url is string => Boolean(url)), - ); const byId = new Map(); + const byUrl = new Map(); for (const feed of installed) { + const row = { ...feed, installedSourceId: feed.id, discoveryIds: [] as string[] }; + byId.set(feed.id, row); const normalized = normalizeProducerUrl(feed.url); - if (normalized && discoveredUrls.has(normalized)) continue; - byId.set(feed.id, feed); + if (normalized) byUrl.set(normalized, [...(byUrl.get(normalized) ?? []), row]); } - for (const feed of discovered) byId.set(feed.id, feed); - return [...byId.values()].sort((a, b) => a.name.localeCompare(b.name)); + for (const feed of [...discovered].sort((a, b) => a.id.localeCompare(b.id))) { + const normalized = normalizeProducerUrl(feed.url), + matches = normalized ? byUrl.get(normalized) : undefined; + if (matches?.length) { + for (const row of matches) { + row.discoveryIds = [...new Set([...(row.discoveryIds ?? []), feed.id])].sort(); + // Discovery can enrich acquisition metadata, never installed provenance. + row.mdbId ??= feed.mdbId; + row.bbox ??= feed.bbox; + row.isOfficial ??= feed.isOfficial; + row.latestDatasetUrl ??= feed.latestDatasetUrl; + row.latestDatasetHash ??= feed.latestDatasetHash; + row.latestDatasetDownloadedAt ??= feed.latestDatasetDownloadedAt; + } + continue; + } + if (byId.has(feed.id)) continue; + const row = { ...feed, installedSourceId: null, discoveryIds: [feed.id] }; + byId.set(feed.id, row); + if (normalized) byUrl.set(normalized, [row]); + } + return [...byId.values()].sort( + (a, b) => a.name.localeCompare(b.name) || a.id.localeCompare(b.id), + ); } export async function searchTransitCatalog( diff --git a/apps/api/src/services/transit-catalog/types.ts b/apps/api/src/services/transit-catalog/types.ts index 8c9dba93f..a8530a94b 100644 --- a/apps/api/src/services/transit-catalog/types.ts +++ b/apps/api/src/services/transit-catalog/types.ts @@ -3,6 +3,8 @@ import type { Licensing } from "@openmapx/mobility-core/licensing"; export interface TransitCatalogFeed { id: string; + installedSourceId?: string | null; + discoveryIds?: string[]; name: string; source: "transitous" | "mobilitydb" | "manual"; countryCode: string; diff --git a/apps/web/src/app/admin/(full-admin)/transit/CurrentJobCard.tsx b/apps/web/src/app/admin/(full-admin)/transit/CurrentJobCard.tsx index 7fc333c09..bffe0076e 100644 --- a/apps/web/src/app/admin/(full-admin)/transit/CurrentJobCard.tsx +++ b/apps/web/src/app/admin/(full-admin)/transit/CurrentJobCard.tsx @@ -13,14 +13,16 @@ import Paper from "@mui/material/Paper"; import Stack from "@mui/material/Stack"; import Typography from "@mui/material/Typography"; import { adminErrorMessage } from "@openmapx/core/api-admin-sdk"; -import type { ReactNode } from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { type ReactNode, useState } from "react"; +import { TransitSyncDialog } from "@/components/admin/services/TransitSyncDialog"; import { useAdminToast } from "@/components/admin/shared/AdminToast"; import { formatTransitAdminTimestamp } from "@/components/admin/shared/adminTimestamp"; import { JobStatusChip } from "@/components/admin/shared/JobStatusChip"; +import { useEnv } from "@/integration-api/runtime/EnvProvider"; import { type TransitStateSummary, useRestartMotis, - useSyncTransit, useTransitJobDetail, } from "@/lib/admin/transitHooks"; @@ -93,27 +95,14 @@ function RunningJobDetail({ jobId }: { jobId: string }) { export function CurrentJobCard({ state }: { state: TransitStateSummary }) { const showToast = useAdminToast(); - const syncMutation = useSyncTransit(); + const [syncReview, setSyncReview] = useState(false); + const { apiUrl } = useEnv(); + const qc = useQueryClient(); const restartMutation = useRestartMotis(); const inflight = state.currentJob; - const onSync = () => { - syncMutation.mutate( - {}, - { - onSuccess: (result) => { - if (result?.jobId) showToast(`Sync queued (${result.jobId.slice(0, 8)}…)`, "success"); - else showToast("Sync queued", "success"); - }, - onError: (err) => - showToast( - adminErrorMessage(err, err instanceof Error ? err.message : "Failed to queue sync"), - "error", - ), - }, - ); - }; + const onSync = () => setSyncReview(true); const onRestart = () => { restartMutation.mutate(undefined, { @@ -128,6 +117,18 @@ export function CurrentJobCard({ state }: { state: TransitStateSummary }) { return ( + {syncReview && ( + setSyncReview(false)} + onQueued={(jobId) => { + setSyncReview(false); + showToast(`Sync queued (${jobId.slice(0, 8)}…)`, "success"); + void qc.invalidateQueries({ queryKey: ["admin", "transit"] }); + }} + /> + )} + } onClick={onSync} - disabled={syncMutation.isPending} + disabled={syncReview} > Run sync now diff --git a/apps/web/src/components/admin/services/TransitSourcesSection.tsx b/apps/web/src/components/admin/services/TransitSourcesSection.tsx index 680bdc8a5..36066e02b 100644 --- a/apps/web/src/components/admin/services/TransitSourcesSection.tsx +++ b/apps/web/src/components/admin/services/TransitSourcesSection.tsx @@ -32,7 +32,8 @@ import { AdminTablePagination } from "../shared/AdminTablePagination"; import { ConfirmDialog } from "../shared/ConfirmDialog"; import { TableEmptyState } from "../shared/TableEmptyState"; import { TableSearchField, TableToolbar } from "../shared/TableToolbar"; -import { useClientPagination, useTextFilter } from "../shared/tableHooks"; +import { useServerPagination } from "../shared/tableHooks"; +import { TransitSyncDialog } from "./TransitSyncDialog"; export type { TransitSource }; @@ -95,6 +96,7 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { }; const [toast, setToast] = useState(null); const [dialogOpen, setDialogOpen] = useState(false); + const [syncDialogOpen, setSyncDialogOpen] = useState(false); const [url, setUrl] = useState(""); const [name, setName] = useState(""); const [region, setRegion] = useState(""); @@ -102,13 +104,35 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { const [licenseUrl, setLicenseUrl] = useState(""); const [attribution, setAttribution] = useState(""); const [catalogSearch, setCatalogSearch] = useState(""); + const catalogPagination = useServerPagination(50); const [lastJobId, setLastJobId] = useState(null); + const [query, setSearch] = useState(""); + const pagination = useServerPagination(50); + const setQuery = (value: string) => { + setSearch(value); + pagination.setPage(0); + }; const sourcesQuery = useQuery({ - queryKey: ["admin", "transit", apiUrl, "sources"], + queryKey: [ + "admin", + "transit", + apiUrl, + "sources", + query.trim(), + pagination.offset, + pagination.rowsPerPage, + ], queryFn: async ({ signal }) => sdk.unwrapAdminResponse( - await sdk.getApiDataManagerTransitSources({ limit: 500 }, { client, signal }), + await sdk.getApiDataManagerTransitSources( + { + search: query.trim() || undefined, + limit: pagination.rowsPerPage, + offset: pagination.offset, + }, + { client, signal }, + ), ), refetchInterval: 30_000, }); @@ -119,12 +143,24 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { refetchInterval: 10_000, }); const catalogQuery = useQuery({ - queryKey: ["admin", "transit", apiUrl, "catalog", catalogSearch], + queryKey: [ + "admin", + "transit", + apiUrl, + "catalog", + catalogSearch, + catalogPagination.offset, + catalogPagination.rowsPerPage, + ], enabled: dialogOpen && catalogSearch.trim().length >= 2, queryFn: async ({ signal }) => sdk.unwrapAdminResponse( await sdk.getApiDataManagerTransitCatalog( - { search: catalogSearch.trim() }, + { + search: catalogSearch.trim(), + limit: catalogPagination.rowsPerPage, + offset: catalogPagination.offset, + }, { client, signal }, ), ), @@ -205,24 +241,12 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { }, onError: (error) => setToast(sdk.adminErrorMessage(error, error.message)), }); - const syncMutation = useMutation({ - retry: false, - mutationFn: async () => - sdk.unwrapAdminResponse(await sdk.postApiDataManagerTransitSync({}, { client })), - onSuccess: ({ jobId }) => reportJob(jobId, `Transit sync queued as ${jobId}.`), - onError: (error) => setToast(sdk.adminErrorMessage(error, error.message)), - }); const sources = sourcesQuery.data?.sources ?? []; - const { query, setQuery, filtered } = useTextFilter(sources, (source) => - [source.id, source.name, source.region, source.origin, source.originUrl] - .filter(Boolean) - .join(" "), - ); - const { paged, paginationProps } = useClientPagination(filtered); + const paged = sources; + const paginationProps = { ...pagination.paginationProps, count: sourcesQuery.data?.total ?? 0 }; const syncJob = stateQuery.data?.currentJob ?? null; - const mutationPending = - addMutation.isPending || sourceMutation.isPending || syncMutation.isPending; + const mutationPending = addMutation.isPending || sourceMutation.isPending || syncDialogOpen; const actionsDisabled = syncJob !== null || mutationPending; const disabledReason = syncJob ? `Transit sync ${syncJob.jobId} is active. Open Activity to monitor it.` @@ -260,7 +284,7 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { variant="outlined" size="small" disabled={actionsDisabled} - onClick={() => syncMutation.mutate()} + onClick={() => setSyncDialogOpen(true)} > Sync sources @@ -295,6 +319,16 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { : `Source change queued as ${lastJobId}.`} )} + {syncDialogOpen && ( + setSyncDialogOpen(false)} + onQueued={(jobId) => { + setSyncDialogOpen(false); + reportJob(jobId, `Transit sync queued as ${jobId}.`); + }} + /> + )} setDialogOpen(false)} @@ -334,6 +368,8 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { /> setAttribution(event.target.value)} fullWidth @@ -363,32 +399,61 @@ function TransitSourcesView({ apiUrl }: { apiUrl: string }) { setCatalogSearch(event.target.value)} + onChange={(event) => { + setCatalogSearch(event.target.value); + catalogPagination.setPage(0); + }} fullWidth size="small" /> - {catalogQuery.data?.sources - .filter((source) => source.source === "transitous") - .slice(0, 5) - .map((source) => ( - - ))} + } else { + setUrl(source.url); + setName(source.name); + setRegion(source.countryCode); + const bindings = source.licensing?.bindings ?? [], + license = bindings.length === 1 ? bindings[0]?.license : undefined; + setLicenseSpdx(license?.kind === "spdx" ? license.expression : ""); + setLicenseUrl(license?.url ?? ""); + setAttribution( + bindings + .flatMap((binding) => [ + binding.subject, + binding.notice, + ...(binding.conditions ?? []), + ]) + .filter(Boolean) + .join("\n"), + ); + } + }} + > + {source.installedSourceId ? "Enable" : "Use"} {source.name} ({source.countryCode}) + + ))} + diff --git a/apps/web/src/components/admin/services/TransitSourcesSection.workflow.test.tsx b/apps/web/src/components/admin/services/TransitSourcesSection.workflow.test.tsx index ae862f42a..ff9c3c9d0 100644 --- a/apps/web/src/components/admin/services/TransitSourcesSection.workflow.test.tsx +++ b/apps/web/src/components/admin/services/TransitSourcesSection.workflow.test.tsx @@ -173,6 +173,8 @@ describe("transit source administration workflow", () => { sources: [ { id: source.id, + installedSourceId: source.id, + discoveryIds: [], name: source.name, source: "transitous", countryCode: "de", @@ -180,6 +182,8 @@ describe("transit source administration workflow", () => { }, ], total: 1, + limit: 50, + offset: 0, }); } return reads(String(input)); @@ -197,10 +201,181 @@ describe("transit source administration workflow", () => { expect(await screen.findByRole("button", { name: "Enable VBB (de)" })).toBeInTheDocument(); await waitFor(() => expect( - client.getQueryData(["admin", "transit", "https://api.example.test", "catalog", " de "]), + client.getQueryData([ + "admin", + "transit", + "https://api.example.test", + "catalog", + " de ", + 0, + 50, + ]), ).toMatchObject({ total: 1, }), ); }); }); +it("finds source601 through a server query instead of filtering the first500", async () => { + const rows = Array.from({ length: 601 }, (_, n) => ({ + ...source, + id: `catalog:de:${n}`, + name: `Source ${String(n).padStart(4, "0")}`, + })); + const requests: URL[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/transit/state")) return Response.json(state); + requests.push(url); + const search = url.searchParams.get("search")?.toLowerCase(), + matching = search ? rows.filter((r) => r.name.toLowerCase().includes(search)) : rows; + const limit = Number(url.searchParams.get("limit") ?? 50), + offset = Number(url.searchParams.get("offset") ?? 0); + return Response.json({ + sourceRevision: "a".repeat(64), + sources: matching.slice(offset, offset + limit), + total: matching.length, + limit, + offset, + }); + }); + view(); + await screen.findByText("Source 0000"); + fireEvent.click(screen.getByRole("button", { name: "Go to next page" })); + await screen.findByText("Source 0050"); + fireEvent.change(screen.getByPlaceholderText(/Search sources by name/), { + target: { value: "Source 0600" }, + }); + expect(await screen.findByText("Source 0600")).toBeInTheDocument(); + expect( + requests.some( + (url) => + url.searchParams.get("search") === "Source 0600" && url.searchParams.get("offset") === "0", + ), + ).toBe(true); + fireEvent.change(screen.getByPlaceholderText(/Search sources by name/), { + target: { value: "" }, + }); + expect(await screen.findByText("Source 0000")).toBeInTheDocument(); + expect(screen.queryByText("Source 0050")).not.toBeInTheDocument(); +}); +it("opens a country review before submitting a transit sync", async () => { + const fetch = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input) => reads(String(input))); + view(); + await screen.findByText("VBB"); + fireEvent.click(screen.getByRole("button", { name: "Sync sources" })); + expect(await screen.findByRole("dialog", { name: "Review transit sync" })).toBeInTheDocument(); + expect( + fetch.mock.calls.some( + ([url, init]) => + String(url).endsWith("/transit/sync") && + (init as RequestInit | undefined)?.method === "POST", + ), + ).toBe(false); +}); +it("uses a discovery-only result to fill Add and retains its declared license and credit", async () => { + const mutations: unknown[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + if ((init as RequestInit | undefined)?.method === "POST") { + mutations.push(input); + return Response.json({}); + } + if (String(input).includes("/catalog")) + return Response.json({ + sources: [ + { + id: "mobilitydb:demo", + installedSourceId: null, + discoveryIds: ["mobilitydb:demo"], + name: "Demo", + source: "mobilitydb", + countryCode: "de", + url: "https://example.test/demo.zip", + licensing: { + version: 1, + bindings: [ + { + scope: "data", + subject: "Demo publisher", + license: { kind: "spdx", expression: "CC-BY-4.0" }, + notice: "Credit Demo", + }, + ], + }, + }, + ], + total: 1, + limit: 50, + offset: 0, + }); + return reads(String(input)); + }); + view(); + await screen.findByText("VBB"); + fireEvent.click(screen.getByRole("button", { name: "Add source" })); + fireEvent.change(screen.getByRole("textbox", { name: "Search installed catalog (optional)" }), { + target: { value: "Demo" }, + }); + fireEvent.click(await screen.findByRole("button", { name: "Use Demo (de)" })); + expect(screen.getByRole("textbox", { name: "Schedule URL" })).toHaveValue( + "https://example.test/demo.zip", + ); + expect(screen.getByRole("textbox", { name: "SPDX license" })).toHaveValue("CC-BY-4.0"); + expect(screen.getByRole("textbox", { name: "Attribution" })).toHaveValue( + "Demo publisher\nCredit Demo", + ); + expect(mutations).toEqual([]); +}); + +it("ignores old origin and out-of-order search pages", async () => { + let old: ((response: Response) => void) | undefined; + let earlier: ((response: Response) => void) | undefined; + const page = (name: string) => + Response.json({ + sourceRevision: "a".repeat(64), + sources: [{ ...source, name }], + total: 1, + limit: 50, + offset: 0, + }); + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/state")) return Response.json(state); + if (url.hostname === "old.example.test") + return new Promise((resolve) => { + old = resolve; + }); + if (url.searchParams.get("search") === "earlier") + return new Promise((resolve) => { + earlier = resolve; + }); + return page(url.searchParams.get("search") === "latest" ? "Latest page" : "Current instance"); + }); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + clients.push(client); + const renderView = (apiUrl: string) => ( + + + + ); + const rendered = render(renderView("https://old.example.test")); + await waitFor(() => expect(old).toBeTypeOf("function")); + rendered.rerender(renderView("https://new.example.test")); + await screen.findByText("Current instance"); + old?.(page("Old instance")); + fireEvent.change(screen.getByPlaceholderText(/Search sources by name/), { + target: { value: "earlier" }, + }); + await waitFor(() => expect(earlier).toBeTypeOf("function")); + fireEvent.change(screen.getByPlaceholderText(/Search sources by name/), { + target: { value: "latest" }, + }); + await screen.findByText("Latest page"); + earlier?.(page("Earlier page")); + await waitFor(() => expect(client.isFetching()).toBe(0)); + expect(screen.getByText("Latest page")).toBeInTheDocument(); + expect(screen.queryByText("Earlier page")).not.toBeInTheDocument(); + expect(screen.queryByText("Old instance")).not.toBeInTheDocument(); +}); diff --git a/apps/web/src/components/admin/services/TransitSyncDialog.test.tsx b/apps/web/src/components/admin/services/TransitSyncDialog.test.tsx new file mode 100644 index 000000000..c10376ef6 --- /dev/null +++ b/apps/web/src/components/admin/services/TransitSyncDialog.test.tsx @@ -0,0 +1,196 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, expect, it, vi } from "vitest"; +import { TransitSyncDialog } from "./TransitSyncDialog"; + +const clients: QueryClient[] = []; +afterEach(() => { + cleanup(); + clients.splice(0).forEach((c) => { + c.clear(); + }); + vi.restoreAllMocks(); +}); +const policy = { + profile: "regional-assisted", + countries: ["de", "at"], + feedAllowList: [], + acquisition: "mirror", + artifactOrigins: ["https://api.transitous.org"], + originDownloadsRequired: false, + hostedRuntimeFallbackAllowed: true, + gbfsSelection: "explicit-countries", + maxFeedCount: 500, + retentionGenerations: 2, + resourceEnvelope: { + minimumMemoryGb: 16, + minimumCpu: 4, + minimumDiskGb: 40, + operatorCapacityRequired: false, + }, + updateCadenceHours: 24, + experimental: false, +}; +function preview(countries: string[], ok: boolean) { + return { + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), + expiresAt: new Date(Date.now() + 300000).toISOString(), + countries, + allowedCountries: ["de", "at"], + result: { + ok, + policy: { ...policy, countries }, + estimate: { + feedCount: 1, + measuredCompressedBytes: 0, + compressedBytes: 1, + expandedGtfsBytes: 1, + importedMotisBytes: 1, + osmBytes: 0, + proxyCacheBytes: 1, + requiredDiskBytes: 10, + basis: "conservative-defaults", + }, + capacity: { + freeDiskBytes: ok ? 100 : 0, + slotMemoryGb: 16, + slotCpu: 4, + fileDescriptorLimit: 65536, + buildTimeoutHours: 12, + }, + blockers: ok ? [] : ["insufficient disk"], + warnings: ["size estimate uses conservative defaults"], + }, + }; +} +function view() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + clients.push(client); + const queued = vi.fn(); + render( + + {}} onQueued={queued} /> + , + ); + return queued; +} +it("shows server blockers and never queues a blocked preview", async () => { + const submissions: unknown[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + if (String(input).endsWith("/profile")) return Response.json({ policy }); + if (String(input).endsWith("/preflight")) + return Response.json( + preview(JSON.parse(String((init as RequestInit | undefined)?.body)).countries, false), + ); + submissions.push(input); + return Response.json({}); + }); + view(); + await screen.findByLabelText("DE"); + fireEvent.click(screen.getByRole("button", { name: "Preview sync" })); + expect(await screen.findByText("insufficient disk")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Queue sync" })).toBeDisabled(); + expect(submissions).toEqual([]); +}); +it("submits only the reviewed subset and retains its exact intent on a failed response", async () => { + const submitted: Record[] = [], + previews: string[][] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + if (String(input).endsWith("/profile")) return Response.json({ policy }); + const body = JSON.parse(String((init as RequestInit | undefined)?.body)); + if (String(input).endsWith("/preflight")) { + previews.push(body.countries); + return Response.json(preview(body.countries, true)); + } + submitted.push(body); + return submitted.length === 1 + ? Response.json({ error: "fixture unavailable" }, { status: 503 }) + : Response.json({ ok: true, jobId: "queued-1", status: "started" }, { status: 202 }); + }); + const queued = view(); + await screen.findByLabelText("AT"); + fireEvent.click(screen.getByLabelText("AT")); + fireEvent.click(screen.getByRole("button", { name: "Preview sync" })); + await waitFor(() => + expect(screen.getByRole("button", { name: "Queue sync" })).not.toBeDisabled(), + ); + fireEvent.click(screen.getByRole("button", { name: "Queue sync" })); + await screen.findByText(/Check Activity/); + fireEvent.click(screen.getByRole("button", { name: "Queue sync" })); + await waitFor(() => expect(queued).toHaveBeenCalledWith("queued-1")); + expect(previews).toEqual([["de"]]); + expect(submitted).toHaveLength(2); + expect(submitted[1]).toEqual(submitted[0]); + expect(submitted[0]).toMatchObject({ + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), + idempotencyKey: expect.any(String), + }); +}); +it("invalidates a ready preview when the country selection changes", async () => { + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => + String(input).endsWith("/profile") + ? Response.json({ policy }) + : Response.json( + preview(JSON.parse(String((init as RequestInit | undefined)?.body)).countries, true), + ), + ); + view(); + await screen.findByLabelText("AT"); + fireEvent.click(screen.getByRole("button", { name: "Preview sync" })); + await waitFor(() => + expect(screen.getByRole("button", { name: "Queue sync" })).not.toBeDisabled(), + ); + fireEvent.click(screen.getByLabelText("AT")); + expect(screen.getByRole("button", { name: "Queue sync" })).toBeDisabled(); +}); +it("preserves a typed separator while choosing an explicit planet subset", async () => { + vi.spyOn(globalThis, "fetch").mockImplementation(async () => + Response.json({ policy: { ...policy, profile: "planet", countries: [] } }), + ); + view(); + const input = await screen.findByRole("textbox", { name: "Country codes" }); + fireEvent.change(input, { target: { value: "de," } }); + expect(input).toHaveValue("de,"); +}); +it("does not associate a late sync acceptance with a newly selected instance", async () => { + let deliver: ((value: Response) => void) | undefined; + const queued = vi.fn(), + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + clients.push(client); + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + if (String(input).endsWith("/profile")) return Response.json({ policy }); + if (String(input).endsWith("/preflight")) + return Response.json( + preview(JSON.parse(String((init as RequestInit | undefined)?.body)).countries, true), + ); + return new Promise((resolve) => { + deliver = resolve; + }); + }); + const screenView = render( + + {}} onQueued={queued} /> + , + ); + await screen.findByLabelText("DE"); + fireEvent.click(screen.getByRole("button", { name: "Preview sync" })); + await waitFor(() => + expect(screen.getByRole("button", { name: "Queue sync" })).not.toBeDisabled(), + ); + fireEvent.click(screen.getByRole("button", { name: "Queue sync" })); + await waitFor(() => expect(deliver).toBeDefined()); + screenView.rerender( + + {}} onQueued={queued} /> + , + ); + deliver?.( + Response.json({ ok: true, jobId: "old-instance-job", status: "started" }, { status: 202 }), + ); + await waitFor(() => expect(client.isMutating()).toBe(0)); + expect(screen.getByRole("button", { name: "Queue sync" })).toBeDisabled(); + expect(queued).not.toHaveBeenCalled(); +}); diff --git a/apps/web/src/components/admin/services/TransitSyncDialog.tsx b/apps/web/src/components/admin/services/TransitSyncDialog.tsx new file mode 100644 index 000000000..0baffd652 --- /dev/null +++ b/apps/web/src/components/admin/services/TransitSyncDialog.tsx @@ -0,0 +1,228 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Button from "@mui/material/Button"; +import Checkbox from "@mui/material/Checkbox"; +import Dialog from "@mui/material/Dialog"; +import DialogActions from "@mui/material/DialogActions"; +import DialogContent from "@mui/material/DialogContent"; +import DialogTitle from "@mui/material/DialogTitle"; +import FormControlLabel from "@mui/material/FormControlLabel"; +import Stack from "@mui/material/Stack"; +import TextField from "@mui/material/TextField"; +import Typography from "@mui/material/Typography"; +import { createApiClient } from "@openmapx/core"; +import * as sdk from "@openmapx/core/api-admin-sdk"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useEffect, useMemo, useRef, useState } from "react"; + +type Preview = Extract< + Awaited>, + { status: 200 } +>["data"]; +type Props = { apiUrl: string; onClose(): void; onQueued(jobId: string): void }; +export function TransitSyncDialog(props: Props) { + return ; +} +function TransitSyncView({ apiUrl, onClose, onQueued }: Props) { + const alive = useRef(true); + useEffect(() => { + alive.current = true; + return () => { + alive.current = false; + }; + }, []); + const [manualCountries, setManualCountries] = useState(""); + const client = useMemo( + () => createApiClient({ baseUrl: apiUrl, credentials: "include" }), + [apiUrl], + ); + const profile = useQuery({ + queryKey: ["admin", "transit", apiUrl, "profile"], + queryFn: async ({ signal }) => + sdk.unwrapAdminResponse(await sdk.getApiDataManagerTransitProfile({ client, signal })), + }); + const [selection, setSelection] = useState(null), + [preview, setPreview] = useState(null), + [expired, setExpired] = useState(false), + [error, setError] = useState(null); + const selected = [...new Set(selection ?? profile.data?.policy.countries ?? [])].sort(), + selectionKey = JSON.stringify(selected), + current = useRef(selectionKey), + key = useRef(null); + current.current = selectionKey; + useEffect(() => { + if (!preview) return; + const timer = setTimeout( + () => setExpired(true), + Math.max(0, Date.parse(preview.expiresAt) - Date.now()), + ); + return () => clearTimeout(timer); + }, [preview]); + const change = (countries: string[]) => { + setSelection(countries); + setPreview(null); + setExpired(false); + setError(null); + key.current = null; + }; + const inspect = useMutation({ + retry: false, + mutationFn: async (countries: string[]) => ({ + selectionKey: JSON.stringify(countries), + value: sdk.unwrapAdminResponse( + await sdk.postApiDataManagerTransitSyncPreflight({ countries }, { client }), + ), + }), + onSuccess: ({ selectionKey, value }) => { + if (!alive.current || current.current !== selectionKey) return; + setPreview(value); + setExpired(false); + setError(null); + key.current = null; + }, + onError: (e) => setError(sdk.adminErrorMessage(e, e.message)), + }); + const submit = useMutation({ + retry: false, + mutationFn: async () => { + if ( + !preview || + expired || + Date.parse(preview.expiresAt) <= Date.now() || + !preview.result.ok || + JSON.stringify(preview.countries) !== current.current + ) + throw Error("Preview the current country selection again"); + key.current ??= crypto.randomUUID(); + return sdk.unwrapAdminResponse( + await sdk.postApiDataManagerTransitSync( + { + countries: preview.countries, + preflightId: preview.preflightId, + fingerprint: preview.fingerprint, + idempotencyKey: key.current, + }, + { client }, + ), + ); + }, + onSuccess: ({ jobId }) => { + if (alive.current) onQueued(jobId); + }, + onError: (e) => setError(sdk.adminErrorMessage(e, e.message)), + }); + const busy = inspect.isPending || submit.isPending; + return ( + { + if (!busy) onClose(); + }} + fullWidth + maxWidth="sm" + > + Review transit sync + + + + Select the countries to synchronize. Capacity and input estimates come from your + instance. + + {profile.isPending && Loading instance scope…} + {profile.error && ( + + {sdk.adminErrorMessage(profile.error, profile.error.message)} + + )} + + {profile.data?.policy.countries.map((country) => ( + + change( + checked ? [...selected, country] : selected.filter((c) => c !== country), + ) + } + /> + } + /> + ))} + + {profile.data?.policy.profile === "planet" && ( + { + setManualCountries(event.target.value); + change( + event.target.value + .toLowerCase() + .split(/[\s,]+/) + .filter(Boolean), + ); + }} + helperText="Enter ISO country codes separated by commas" + /> + )} + {error && ( + + {error} Check Activity if an earlier submission may already be running. + + )} + {preview && ( + <> + + Countries: {preview.countries.map((c) => c.toUpperCase()).join(", ")} + + + {preview.result.estimate.feedCount} feeds · estimate:{" "} + {preview.result.estimate.basis.replaceAll("-", " ")} + + + Disk required: {Math.ceil(preview.result.estimate.requiredDiskBytes / 1024 ** 3)}{" "} + GiB · available: {Math.floor(preview.result.capacity.freeDiskBytes / 1024 ** 3)} GiB + + {preview.result.blockers.map((reason) => ( + + {reason} + + ))} + {preview.result.warnings.map((reason) => ( + + {reason} + + ))} + {expired && ( + This preview expired. Refresh it before queuing. + )} + + )} + + + + + + + + + ); +} diff --git a/apps/web/src/lib/admin/transitHooks.test.tsx b/apps/web/src/lib/admin/transitHooks.test.tsx index 97b397841..d7b12b834 100644 --- a/apps/web/src/lib/admin/transitHooks.test.tsx +++ b/apps/web/src/lib/admin/transitHooks.test.tsx @@ -57,6 +57,11 @@ afterEach(() => { vi.restoreAllMocks(); }); +const reviewed = { + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), +}; describe("transit administration acquisition", () => { it("rejects a malformed successful state instead of caching an asserted DTO", async () => { vi.spyOn(globalThis, "fetch").mockImplementation(async () => Response.json({})); @@ -73,7 +78,7 @@ describe("transit administration acquisition", () => { const invalidate = vi.spyOn(client, "invalidateQueries"); const { result } = renderHook(() => useSyncTransit(), { wrapper }); await act(async () => { - await expect(result.current.mutateAsync({})).rejects.toThrow(); + await expect(result.current.mutateAsync(reviewed)).rejects.toThrow(); }); expect(invalidate).not.toHaveBeenCalled(); }); @@ -140,7 +145,7 @@ describe("transit administration acquisition", () => { vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, initValue) => { const init = initValue as RequestInit | undefined; expect(JSON.parse(init?.body as string)).toEqual({ - countries: ["de"], + ...reviewed, idempotencyKey: "attempt-1", }); return Response.json({ ok: true, jobId: "job-1", status: "started" }, { status: 202 }); @@ -150,7 +155,7 @@ describe("transit administration acquisition", () => { const { result } = renderHook(() => useSyncTransit(), { wrapper }); await act(async () => { await expect( - result.current.mutateAsync({ countries: ["de"], idempotencyKey: "attempt-1" }), + result.current.mutateAsync({ ...reviewed, idempotencyKey: "attempt-1" }), ).resolves.toMatchObject({ jobId: "job-1" }); }); expect(invalidate.mock.calls).toEqual([ @@ -169,7 +174,7 @@ describe("transit administration acquisition", () => { const invalidate = vi.spyOn(client, "invalidateQueries"); const { result } = renderHook(() => useSyncTransit(), { wrapper }); await act(async () => { - await expect(result.current.mutateAsync({})).rejects.toMatchObject({ status: 409 }); + await expect(result.current.mutateAsync(reviewed)).rejects.toMatchObject({ status: 409 }); }); expect(invalidate).not.toHaveBeenCalled(); }); diff --git a/apps/web/src/lib/admin/transitHooks.ts b/apps/web/src/lib/admin/transitHooks.ts index 3995a55fb..d273827ba 100644 --- a/apps/web/src/lib/admin/transitHooks.ts +++ b/apps/web/src/lib/admin/transitHooks.ts @@ -123,7 +123,8 @@ export function useSyncTransit() { const client = useAdminClient(); const qc = useQueryClient(); return useMutation({ - mutationFn: (input: SyncTransitInput = {}) => + retry: false, + mutationFn: (input: SyncTransitInput) => adminOrThrow(sdk.postApiDataManagerTransitSync(input, { client })), onSuccess: () => { void qc.invalidateQueries({ queryKey: ["admin", "transit", "state"] }); diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index a687b4789..74d2493cb 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -202,6 +202,20 @@ either an SPDX license identifier or license URL. Adding, disabling, enabling, or manually syncing a source queues an asynchronous job; the row links to the activity log and disables conflicting actions while that job is active. +Source search and pagination run on the server, so entries beyond the first +500 remain reachable. Optional discovery enriches installed catalog entries +without replacing their IDs or licensing; discovery-only entries populate the +Add form for review rather than pretending they are already enabled. + +**Sync sources** and **Run sync now** open a country review. Select a subset of +the instance's allowed countries, then preview its source count, conservative +size estimate, capacity blockers and warnings. The preview expires after five +minutes and is checked again before admission and worker execution. Browser +supplied capacity values cannot authorize a sync. Slot limits come from the +instance configuration; an unavailable measurement blocks the preview. +Queuing reports an Activity job, not a completed import. A retry of an already +accepted intent observes its original job even if the preview cache was lost. + MOTIS is the only compiled static-schedule runtime. Postgres stores the data-manager's source, job, validation, manifest, and promotion metadata; it does not contain imported GTFS schedules. diff --git a/packages/cli/__tests__/cli-env-defaults.test.ts b/packages/cli/__tests__/cli-env-defaults.test.ts index 4b7390709..36e5fff4f 100644 --- a/packages/cli/__tests__/cli-env-defaults.test.ts +++ b/packages/cli/__tests__/cli-env-defaults.test.ts @@ -67,3 +67,13 @@ describe("env defaults", () => { }); }); }); + +it("uses finite deployment feed countries when the country environment is omitted", () => { + const env = { MOTIS_FEED_ALLOW_LIST: "de_demo,de_other,at_local,invalid" }; + expect(resolveTransitousCountries(undefined, env)).toEqual({ + values: ["at", "de"], + sourceEnv: "MOTIS_FEED_ALLOW_LIST", + }); + expect(resolveTransitousCountries("ch", env)).toEqual({ values: ["ch"] }); + expect(resolveTransitousCountries(undefined, {})).toEqual({ values: [] }); +}); diff --git a/packages/cli/src/lib/env-defaults.ts b/packages/cli/src/lib/env-defaults.ts index 38735ff2b..b89631af6 100644 --- a/packages/cli/src/lib/env-defaults.ts +++ b/packages/cli/src/lib/env-defaults.ts @@ -88,7 +88,17 @@ export function resolveTransitousCountries( } const envValue = normalizeScalar(env[TRANSITOUS_COUNTRIES_ENV]); - if (!envValue) return { values: [] }; + if (!envValue) { + const values = [ + ...new Set( + parseCsvList(env.MOTIS_FEED_ALLOW_LIST).flatMap((id) => { + const match = /^([a-z]{2,3})_/.exec(id.toLowerCase()); + return match ? [match[1]!] : []; + }), + ), + ].sort(); + return values.length ? { values, sourceEnv: "MOTIS_FEED_ALLOW_LIST" } : { values: [] }; + } return { values: parseCsvList(envValue), sourceEnv: TRANSITOUS_COUNTRIES_ENV, diff --git a/packages/core/src/api/admin-sdk.test.ts b/packages/core/src/api/admin-sdk.test.ts index 475a4a2fc..f5437354d 100644 --- a/packages/core/src/api/admin-sdk.test.ts +++ b/packages/core/src/api/admin-sdk.test.ts @@ -6,8 +6,8 @@ import { dataManagerAdminOperations } from "./contracts/data-manager-admin"; const client = createApiClient({ baseUrl: "https://app.example", credentials: "include" }); afterEach(() => vi.unstubAllGlobals()); describe("complete generated administration operations", () => { - it("exposes all25canonical operation keys", () => { - expect(Object.keys(dataManagerAdminOperations)).toHaveLength(25); + it("exposes all28canonical operation keys", () => { + expect(Object.keys(dataManagerAdminOperations)).toHaveLength(28); for (const key of Object.keys(dataManagerAdminOperations)) expect(sdk).toHaveProperty(key, expect.any(Function)); }); @@ -22,13 +22,38 @@ describe("complete generated administration operations", () => { ) .mockResolvedValueOnce(Response.json({ ok: false, reason: "out-of-scope" }, { status: 422 })); vi.stubGlobal("fetch", fetch); - const accepted = await sdk.postApiDataManagerTransitSync({ countries: ["de"] }, { client }); + const accepted = await sdk.postApiDataManagerTransitSync( + { + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), + }, + { client }, + ); expect(sdk.unwrapAdminResponse(accepted).jobId).toBe("job"); - expect(JSON.parse(fetch.mock.calls[0]?.[1].body)).toEqual({ countries: ["de"] }); - const conflict = await sdk.postApiDataManagerTransitSync({}, { client }); + expect(JSON.parse(fetch.mock.calls[0]?.[1].body)).toMatchObject({ + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), + }); + const conflict = await sdk.postApiDataManagerTransitSync( + { + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), + }, + { client }, + ); expect(conflict.status).toBe(409); expect(() => sdk.unwrapAdminResponse(conflict)).toThrow(ApiClientError); - const rejected = await sdk.postApiDataManagerTransitSync({}, { client }); + const rejected = await sdk.postApiDataManagerTransitSync( + { + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), + }, + { client }, + ); expect(rejected.status).toBe(422); }); it.each([400, 401, 403, 404, 429, 500])( diff --git a/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts b/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts index cd47da419..cbfc22963 100644 --- a/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts +++ b/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts @@ -9,10 +9,10 @@ import { } from "./index"; describe("application data-manager administration contracts", () => { - it("describes all 26 actual BFF operations with the real session-only distinctions", () => { - expect(Object.keys(dataManagerAdminOperations)).toHaveLength(26); + it("describes all 28 actual BFF operations with the real session-only distinctions", () => { + expect(Object.keys(dataManagerAdminOperations)).toHaveLength(28); const operations = Object.values(dataManagerAdminOperations); - expect(new Set(operations.map(({ method, path }) => `${method} ${path}`)).size).toBe(26); + expect(new Set(operations.map(({ method, path }) => `${method} ${path}`)).size).toBe(28); expect(operations.filter(({ auth }) => auth === "admin")).toHaveLength(11); expect(operations.every(({ path }) => path.startsWith("/api/"))).toBe(true); expect(dataManagerAdminOperations.getApiDataManagerTransitState.auth).toBe("service"); @@ -83,6 +83,8 @@ describe("application data-manager administration contracts", () => { sources: [ { id: "catalog", + installedSourceId: null, + discoveryIds: ["catalog"], name: "Feed", source: "mobilitydb", countryCode: "de", @@ -93,6 +95,8 @@ describe("application data-manager administration contracts", () => { }, ], total: 1, + limit: 50, + offset: 0, }; expect(transitCatalogResponseSchema.parse(result)).toEqual(result); expect( diff --git a/packages/core/src/api/contracts/data-manager-admin/operations.ts b/packages/core/src/api/contracts/data-manager-admin/operations.ts index 1aa040372..aa6547016 100644 --- a/packages/core/src/api/contracts/data-manager-admin/operations.ts +++ b/packages/core/src/api/contracts/data-manager-admin/operations.ts @@ -17,12 +17,16 @@ import { transitBumpResultSchema, transitConflictSchema, transitMutationRequestSchema, + transitPreflightSelectionSchema, + transitPreflightStaleSchema, + transitProfileSchema, transitScopeRejectedSchema, transitSourceActionErrorSchema, transitSourceListSchema, transitSourceMutationResultSchema, transitSourcesQuerySchema, transitSyncAcceptedSchema, + transitSyncPreflightSchema, transitSyncRequestSchema, } from "../../../services/data-manager/contracts/transit"; import { @@ -59,10 +63,11 @@ export const adminSourceMutationRequestSchema = transitMutationRequestSchema.omi export const adminAddTransitSourceRequestSchema = addTransitSourceRequestSchema.omit({ triggeredBy: true, }); -export const adminTransitSyncRequestSchema = transitSyncRequestSchema - .unwrap() - .omit({ triggeredBy: true }) - .optional(); +export const adminTransitSyncRequestSchema = transitPreflightSelectionSchema.extend({ + preflightId: z.uuid(), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + idempotencyKey: z.string().max(200).optional(), +}); export const adminTransitBumpRequestSchema = z .strictObject({ branch: z.string().max(128).optional(), force: z.boolean().optional() }) .optional(); @@ -169,6 +174,25 @@ export const dataManagerAdminOperations = { request: { params: sourceParams, body: adminSourceMutationRequestSchema }, responses: mutationResponses, }, + getApiDataManagerTransitProfile: { + method: "GET", + path: "/api/data-manager/transit/profile", + auth: service, + responses: { ...common, 200: json(transitProfileSchema), 503: json(httpErrorSchema) }, + }, + postApiDataManagerTransitSyncPreflight: { + method: "POST", + path: "/api/data-manager/transit/sync/preflight", + auth: service, + request: { body: transitPreflightSelectionSchema }, + responses: { + ...common, + 200: json(transitSyncPreflightSchema), + 409: json(transitPreflightStaleSchema), + 422: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, postApiDataManagerTransitSync: { method: "POST", path: "/api/data-manager/transit/sync", @@ -177,7 +201,7 @@ export const dataManagerAdminOperations = { responses: { ...common, 202: json(transitSyncAcceptedSchema), - 409: json(transitConflictSchema), + 409: json(z.union([httpErrorSchema, transitConflictSchema, transitPreflightStaleSchema])), 422: json(transitScopeRejectedSchema), }, }, diff --git a/packages/core/src/api/contracts/data-manager-admin/transit.ts b/packages/core/src/api/contracts/data-manager-admin/transit.ts index 06a03413c..635f58efc 100644 --- a/packages/core/src/api/contracts/data-manager-admin/transit.ts +++ b/packages/core/src/api/contracts/data-manager-admin/transit.ts @@ -91,11 +91,15 @@ export const transitJobsQuerySchema = z.strictObject({ kind: z.string().optional(), }); export const transitCatalogQuerySchema = z.strictObject({ + limit: z.coerce.number().int().min(1).max(500).optional(), + offset: z.coerce.number().int().nonnegative().optional(), search: z.string().optional(), country: z.string().optional(), }); export const transitCatalogFeedSchema = z.strictObject({ id: z.string(), + installedSourceId: z.string().nullable(), + discoveryIds: z.array(z.string()), name: z.string(), source: z.enum(["transitous", "mobilitydb", "manual"]), countryCode: z.string(), @@ -114,6 +118,8 @@ export const transitCatalogFeedSchema = z.strictObject({ dataType: z.string().optional(), }); export const transitCatalogResponseSchema = z.strictObject({ + limit: count, + offset: count, sources: z.array(transitCatalogFeedSchema), total: count, }); diff --git a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts index 9e8f1fc9e..230129448 100644 --- a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts +++ b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts @@ -110,6 +110,15 @@ import type { GetApiDataManagerTransitJobsId429, GetApiDataManagerTransitJobsId500, GetApiDataManagerTransitJobsParams, + GetApiDataManagerTransitProfile200, + GetApiDataManagerTransitProfile400, + GetApiDataManagerTransitProfile401, + GetApiDataManagerTransitProfile403, + GetApiDataManagerTransitProfile413, + GetApiDataManagerTransitProfile415, + GetApiDataManagerTransitProfile429, + GetApiDataManagerTransitProfile500, + GetApiDataManagerTransitProfile503, GetApiDataManagerTransitSources200, GetApiDataManagerTransitSources400, GetApiDataManagerTransitSources401, @@ -256,6 +265,18 @@ import type { PostApiDataManagerTransitSync429, PostApiDataManagerTransitSync500, PostApiDataManagerTransitSyncBody, + PostApiDataManagerTransitSyncPreflight200, + PostApiDataManagerTransitSyncPreflight400, + PostApiDataManagerTransitSyncPreflight401, + PostApiDataManagerTransitSyncPreflight403, + PostApiDataManagerTransitSyncPreflight409, + PostApiDataManagerTransitSyncPreflight413, + PostApiDataManagerTransitSyncPreflight415, + PostApiDataManagerTransitSyncPreflight422, + PostApiDataManagerTransitSyncPreflight429, + PostApiDataManagerTransitSyncPreflight500, + PostApiDataManagerTransitSyncPreflight503, + PostApiDataManagerTransitSyncPreflightBody, } from "./models"; import { deleteApiDataManagerTransitSourcesSourceIdRequest, @@ -270,6 +291,7 @@ import { getApiDataManagerTransitFeedsRequest, getApiDataManagerTransitJobsIdRequest, getApiDataManagerTransitJobsRequest, + getApiDataManagerTransitProfileRequest, getApiDataManagerTransitSourcesRequest, getApiDataManagerTransitStateRequest, postApiAdminAmbientPlacesBuildRequest, @@ -283,6 +305,7 @@ import { postApiDataManagerTransitSourcesRequest, postApiDataManagerTransitSourcesSourceIdDisableRequest, postApiDataManagerTransitSourcesSourceIdEnableRequest, + postApiDataManagerTransitSyncPreflightRequest, postApiDataManagerTransitSyncRequest, } from "./operation-bindings.generated"; export type postApiAdminAmbientPlacesBuildResponse202 = { @@ -1928,6 +1951,88 @@ export const getApiDataManagerTransitJobsId = async ( ); }; +export type getApiDataManagerTransitProfileResponse200 = { + data: GetApiDataManagerTransitProfile200; + status: 200; +}; + +export type getApiDataManagerTransitProfileResponse400 = { + data: GetApiDataManagerTransitProfile400; + status: 400; +}; + +export type getApiDataManagerTransitProfileResponse401 = { + data: GetApiDataManagerTransitProfile401; + status: 401; +}; + +export type getApiDataManagerTransitProfileResponse403 = { + data: GetApiDataManagerTransitProfile403; + status: 403; +}; + +export type getApiDataManagerTransitProfileResponse413 = { + data: GetApiDataManagerTransitProfile413; + status: 413; +}; + +export type getApiDataManagerTransitProfileResponse415 = { + data: GetApiDataManagerTransitProfile415; + status: 415; +}; + +export type getApiDataManagerTransitProfileResponse429 = { + data: GetApiDataManagerTransitProfile429; + status: 429; +}; + +export type getApiDataManagerTransitProfileResponse500 = { + data: GetApiDataManagerTransitProfile500; + status: 500; +}; + +export type getApiDataManagerTransitProfileResponse503 = { + data: GetApiDataManagerTransitProfile503; + status: 503; +}; + +export type getApiDataManagerTransitProfileResponseSuccess = + getApiDataManagerTransitProfileResponse200 & { + headers: Headers; + }; +export type getApiDataManagerTransitProfileResponseError = ( + | getApiDataManagerTransitProfileResponse400 + | getApiDataManagerTransitProfileResponse401 + | getApiDataManagerTransitProfileResponse403 + | getApiDataManagerTransitProfileResponse413 + | getApiDataManagerTransitProfileResponse415 + | getApiDataManagerTransitProfileResponse429 + | getApiDataManagerTransitProfileResponse500 + | getApiDataManagerTransitProfileResponse503 +) & { + headers: Headers; +}; + +export type getApiDataManagerTransitProfileResponse = + | getApiDataManagerTransitProfileResponseSuccess + | getApiDataManagerTransitProfileResponseError; + +export const getGetApiDataManagerTransitProfileUrl = () => { + return `/api/data-manager/transit/profile`; +}; + +export const getApiDataManagerTransitProfile = async ( + options?: Parameters[1], +): Promise => { + return getApiDataManagerTransitProfileRequest( + getGetApiDataManagerTransitProfileUrl(), + { + ...options, + method: "GET", + }, + ); +}; + export type postApiDataManagerTransitRestartMotisResponse200 = { data: PostApiDataManagerTransitRestartMotis200; status: 200; @@ -2741,3 +2846,119 @@ export const postApiDataManagerTransitSync = async ( }, ); }; + +export type postApiDataManagerTransitSyncPreflightResponse200 = { + data: PostApiDataManagerTransitSyncPreflight200; + status: 200; +}; + +export type postApiDataManagerTransitSyncPreflightResponse400 = { + data: PostApiDataManagerTransitSyncPreflight400; + status: 400; +}; + +export type postApiDataManagerTransitSyncPreflightResponse401 = { + data: PostApiDataManagerTransitSyncPreflight401; + status: 401; +}; + +export type postApiDataManagerTransitSyncPreflightResponse403 = { + data: PostApiDataManagerTransitSyncPreflight403; + status: 403; +}; + +export type postApiDataManagerTransitSyncPreflightResponse409 = { + data: PostApiDataManagerTransitSyncPreflight409; + status: 409; +}; + +export type postApiDataManagerTransitSyncPreflightResponse413 = { + data: PostApiDataManagerTransitSyncPreflight413; + status: 413; +}; + +export type postApiDataManagerTransitSyncPreflightResponse415 = { + data: PostApiDataManagerTransitSyncPreflight415; + status: 415; +}; + +export type postApiDataManagerTransitSyncPreflightResponse422 = { + data: PostApiDataManagerTransitSyncPreflight422; + status: 422; +}; + +export type postApiDataManagerTransitSyncPreflightResponse429 = { + data: PostApiDataManagerTransitSyncPreflight429; + status: 429; +}; + +export type postApiDataManagerTransitSyncPreflightResponse500 = { + data: PostApiDataManagerTransitSyncPreflight500; + status: 500; +}; + +export type postApiDataManagerTransitSyncPreflightResponse503 = { + data: PostApiDataManagerTransitSyncPreflight503; + status: 503; +}; + +export type postApiDataManagerTransitSyncPreflightResponseSuccess = + postApiDataManagerTransitSyncPreflightResponse200 & { + headers: Headers; + }; +export type postApiDataManagerTransitSyncPreflightResponseError = ( + | postApiDataManagerTransitSyncPreflightResponse400 + | postApiDataManagerTransitSyncPreflightResponse401 + | postApiDataManagerTransitSyncPreflightResponse403 + | postApiDataManagerTransitSyncPreflightResponse409 + | postApiDataManagerTransitSyncPreflightResponse413 + | postApiDataManagerTransitSyncPreflightResponse415 + | postApiDataManagerTransitSyncPreflightResponse422 + | postApiDataManagerTransitSyncPreflightResponse429 + | postApiDataManagerTransitSyncPreflightResponse500 + | postApiDataManagerTransitSyncPreflightResponse503 +) & { + headers: Headers; +}; + +export type postApiDataManagerTransitSyncPreflightResponse = + | postApiDataManagerTransitSyncPreflightResponseSuccess + | postApiDataManagerTransitSyncPreflightResponseError; + +export const getPostApiDataManagerTransitSyncPreflightUrl = () => { + return `/api/data-manager/transit/sync/preflight`; +}; + +export const postApiDataManagerTransitSyncPreflight = async ( + postApiDataManagerTransitSyncPreflightBody: PostApiDataManagerTransitSyncPreflightBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return postApiDataManagerTransitSyncPreflightRequest( + getPostApiDataManagerTransitSyncPreflightUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(postApiDataManagerTransitSyncPreflightBody), + }, + ); +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200.ts index 8ead57f9c..e693b22c3 100644 --- a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200.ts +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200.ts @@ -3,6 +3,16 @@ import type { GetApiDataManagerTransitCatalog200SourcesItem } from "./getApiDataManagerTransitCatalog200SourcesItem"; export type GetApiDataManagerTransitCatalog200 = { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + limit: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + offset: number; sources: GetApiDataManagerTransitCatalog200SourcesItem[]; /** * @minimum 0 diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200SourcesItem.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200SourcesItem.ts index 2c2ad0a7a..d54cc519e 100644 --- a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200SourcesItem.ts +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalog200SourcesItem.ts @@ -5,6 +5,9 @@ import type { GetApiDataManagerTransitCatalog200SourcesItemSource } from "./getA export type GetApiDataManagerTransitCatalog200SourcesItem = { id: string; + /** @nullable */ + installedSourceId: string | null; + discoveryIds: string[]; name: string; source: GetApiDataManagerTransitCatalog200SourcesItemSource; countryCode: string; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalogParams.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalogParams.ts index f75ac9086..9eb2d1f70 100644 --- a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalogParams.ts +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitCatalogParams.ts @@ -2,6 +2,16 @@ // Generated by pnpm sdk:generate. Do not edit. export type GetApiDataManagerTransitCatalogParams = { + /** + * @minimum 1 + * @maximum 500 + */ + limit?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + offset?: number; search?: string; country?: string; }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200.ts new file mode 100644 index 000000000..ab0aaea8e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetApiDataManagerTransitProfile200Policy } from "./getApiDataManagerTransitProfile200Policy"; + +export type GetApiDataManagerTransitProfile200 = { + policy: GetApiDataManagerTransitProfile200Policy; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200Policy.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200Policy.ts new file mode 100644 index 000000000..8e7546ae0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200Policy.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetApiDataManagerTransitProfile200PolicyAcquisition } from "./getApiDataManagerTransitProfile200PolicyAcquisition"; +import type { GetApiDataManagerTransitProfile200PolicyGbfsSelection } from "./getApiDataManagerTransitProfile200PolicyGbfsSelection"; +import type { GetApiDataManagerTransitProfile200PolicyProfile } from "./getApiDataManagerTransitProfile200PolicyProfile"; +import type { GetApiDataManagerTransitProfile200PolicyResourceEnvelope } from "./getApiDataManagerTransitProfile200PolicyResourceEnvelope"; + +export type GetApiDataManagerTransitProfile200Policy = { + profile: GetApiDataManagerTransitProfile200PolicyProfile; + countries: string[]; + feedAllowList: string[]; + acquisition: GetApiDataManagerTransitProfile200PolicyAcquisition; + artifactOrigins: string[]; + originDownloadsRequired: boolean; + hostedRuntimeFallbackAllowed: boolean; + osmInput?: string; + gbfsSelection: GetApiDataManagerTransitProfile200PolicyGbfsSelection; + /** @minimum 0 */ + maxFeedCount: number; + /** @minimum 0 */ + retentionGenerations: number; + resourceEnvelope: GetApiDataManagerTransitProfile200PolicyResourceEnvelope; + /** @minimum 0 */ + updateCadenceHours: number; + experimental: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyAcquisition.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyAcquisition.ts new file mode 100644 index 000000000..f30b87c98 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyAcquisition.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile200PolicyAcquisition = + (typeof GetApiDataManagerTransitProfile200PolicyAcquisition)[keyof typeof GetApiDataManagerTransitProfile200PolicyAcquisition]; + +export const GetApiDataManagerTransitProfile200PolicyAcquisition = { + mirror: "mirror", + build: "build", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyGbfsSelection.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyGbfsSelection.ts new file mode 100644 index 000000000..11ecdf6cd --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyGbfsSelection.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile200PolicyGbfsSelection = + (typeof GetApiDataManagerTransitProfile200PolicyGbfsSelection)[keyof typeof GetApiDataManagerTransitProfile200PolicyGbfsSelection]; + +export const GetApiDataManagerTransitProfile200PolicyGbfsSelection = { + "explicit-countries": "explicit-countries", + "explicit-feeds": "explicit-feeds", + "allow-listed-batches": "allow-listed-batches", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyProfile.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyProfile.ts new file mode 100644 index 000000000..3d7213593 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyProfile.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile200PolicyProfile = + (typeof GetApiDataManagerTransitProfile200PolicyProfile)[keyof typeof GetApiDataManagerTransitProfile200PolicyProfile]; + +export const GetApiDataManagerTransitProfile200PolicyProfile = { + "regional-assisted": "regional-assisted", + "regional-sovereign": "regional-sovereign", + planet: "planet", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyResourceEnvelope.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyResourceEnvelope.ts new file mode 100644 index 000000000..74d2e706e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile200PolicyResourceEnvelope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile200PolicyResourceEnvelope = { + /** @minimum 0 */ + minimumMemoryGb: number; + /** @minimum 0 */ + minimumCpu: number; + /** @minimum 0 */ + minimumDiskGb: number; + operatorCapacityRequired: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile400.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile400.ts new file mode 100644 index 000000000..b791ec4d1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile401.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile401.ts new file mode 100644 index 000000000..106fc3244 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile403.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile403.ts new file mode 100644 index 000000000..71861bd5f --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile413.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile413.ts new file mode 100644 index 000000000..b6e85d12c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile415.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile415.ts new file mode 100644 index 000000000..db9a2b254 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile429.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile429.ts new file mode 100644 index 000000000..5cd8a7625 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile500.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile500.ts new file mode 100644 index 000000000..e90ab6094 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile503.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile503.ts new file mode 100644 index 000000000..789546072 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitProfile503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerTransitProfile503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSourcesParams.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSourcesParams.ts index 470ee1969..46e3e59ba 100644 --- a/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSourcesParams.ts +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerTransitSourcesParams.ts @@ -4,6 +4,7 @@ import type { GetApiDataManagerTransitSourcesLifecycle } from "./getApiDataManag import type { GetApiDataManagerTransitSourcesOrigin } from "./getApiDataManagerTransitSourcesOrigin"; export type GetApiDataManagerTransitSourcesParams = { + country?: string; search?: string; lifecycle?: GetApiDataManagerTransitSourcesLifecycle; origin?: GetApiDataManagerTransitSourcesOrigin; diff --git a/packages/core/src/api/generated/admin/models/index.ts b/packages/core/src/api/generated/admin/models/index.ts index 2f70a6ed9..b32e15daf 100644 --- a/packages/core/src/api/generated/admin/models/index.ts +++ b/packages/core/src/api/generated/admin/models/index.ts @@ -165,6 +165,20 @@ export * from "./getApiDataManagerTransitJobsId415"; export * from "./getApiDataManagerTransitJobsId429"; export * from "./getApiDataManagerTransitJobsId500"; export * from "./getApiDataManagerTransitJobsParams"; +export * from "./getApiDataManagerTransitProfile200"; +export * from "./getApiDataManagerTransitProfile200Policy"; +export * from "./getApiDataManagerTransitProfile200PolicyAcquisition"; +export * from "./getApiDataManagerTransitProfile200PolicyGbfsSelection"; +export * from "./getApiDataManagerTransitProfile200PolicyProfile"; +export * from "./getApiDataManagerTransitProfile200PolicyResourceEnvelope"; +export * from "./getApiDataManagerTransitProfile400"; +export * from "./getApiDataManagerTransitProfile401"; +export * from "./getApiDataManagerTransitProfile403"; +export * from "./getApiDataManagerTransitProfile413"; +export * from "./getApiDataManagerTransitProfile415"; +export * from "./getApiDataManagerTransitProfile429"; +export * from "./getApiDataManagerTransitProfile500"; +export * from "./getApiDataManagerTransitProfile503"; export * from "./getApiDataManagerTransitSources200"; export * from "./getApiDataManagerTransitSources200SourcesItem"; export * from "./getApiDataManagerTransitSources200SourcesItemActions"; @@ -326,10 +340,31 @@ export * from "./postApiDataManagerTransitSync400"; export * from "./postApiDataManagerTransitSync401"; export * from "./postApiDataManagerTransitSync403"; export * from "./postApiDataManagerTransitSync409"; -export * from "./postApiDataManagerTransitSync409Reason"; export * from "./postApiDataManagerTransitSync413"; export * from "./postApiDataManagerTransitSync415"; export * from "./postApiDataManagerTransitSync422"; export * from "./postApiDataManagerTransitSync429"; export * from "./postApiDataManagerTransitSync500"; export * from "./postApiDataManagerTransitSyncBody"; +export * from "./postApiDataManagerTransitSyncPreflight200"; +export * from "./postApiDataManagerTransitSyncPreflight200Result"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultCapacity"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultEstimate"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultEstimateBasis"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultPolicy"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultPolicyProfile"; +export * from "./postApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope"; +export * from "./postApiDataManagerTransitSyncPreflight400"; +export * from "./postApiDataManagerTransitSyncPreflight401"; +export * from "./postApiDataManagerTransitSyncPreflight403"; +export * from "./postApiDataManagerTransitSyncPreflight409"; +export * from "./postApiDataManagerTransitSyncPreflight409Code"; +export * from "./postApiDataManagerTransitSyncPreflight413"; +export * from "./postApiDataManagerTransitSyncPreflight415"; +export * from "./postApiDataManagerTransitSyncPreflight422"; +export * from "./postApiDataManagerTransitSyncPreflight429"; +export * from "./postApiDataManagerTransitSyncPreflight500"; +export * from "./postApiDataManagerTransitSyncPreflight503"; +export * from "./postApiDataManagerTransitSyncPreflightBody"; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409.ts index f93040aa1..2a3b88937 100644 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409.ts +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409.ts @@ -1,10 +1,30 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. -import type { PostApiDataManagerTransitSync409Reason } from "./postApiDataManagerTransitSync409Reason"; -export type PostApiDataManagerTransitSync409 = { - ok: false; - reason: PostApiDataManagerTransitSync409Reason; - /** @nullable */ - existingJobId: string | null; -}; +export type PostApiDataManagerTransitSync409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + } + | { + ok: false; + reason: "in-flight" | "duplicate-idempotency-key"; + /** @nullable */ + existingJobId: string | null; + } + | { + error: string; + code: "preflight_stale"; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409Reason.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409Reason.ts deleted file mode 100644 index adf0fd8da..000000000 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSync409Reason.ts +++ /dev/null @@ -1,10 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. -// Generated by pnpm sdk:generate. Do not edit. - -export type PostApiDataManagerTransitSync409Reason = - (typeof PostApiDataManagerTransitSync409Reason)[keyof typeof PostApiDataManagerTransitSync409Reason]; - -export const PostApiDataManagerTransitSync409Reason = { - "in-flight": "in-flight", - "duplicate-idempotency-key": "duplicate-idempotency-key", -} as const; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncBody.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncBody.ts index 8687925c2..00cb669f2 100644 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncBody.ts +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncBody.ts @@ -2,11 +2,16 @@ // Generated by pnpm sdk:generate. Do not edit. export type PostApiDataManagerTransitSyncBody = { - /** @maxLength 200 */ - idempotencyKey?: string; /** - * @maxItems 250 - * @items.pattern ^[A-Za-z][A-Za-z0-9_-]{0,31}$ + * @minItems 1 + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ */ - countries?: string[]; + countries: string[]; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + preflightId: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** @maxLength 200 */ + idempotencyKey?: string; }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200.ts new file mode 100644 index 000000000..0eaea1d1d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PostApiDataManagerTransitSyncPreflight200Result } from "./postApiDataManagerTransitSyncPreflight200Result"; + +export type PostApiDataManagerTransitSyncPreflight200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + preflightId: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + countries: string[]; + allowedCountries: string[]; + result: PostApiDataManagerTransitSyncPreflight200Result; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200Result.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200Result.ts new file mode 100644 index 000000000..e55befbff --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200Result.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PostApiDataManagerTransitSyncPreflight200ResultCapacity } from "./postApiDataManagerTransitSyncPreflight200ResultCapacity"; +import type { PostApiDataManagerTransitSyncPreflight200ResultEstimate } from "./postApiDataManagerTransitSyncPreflight200ResultEstimate"; +import type { PostApiDataManagerTransitSyncPreflight200ResultPolicy } from "./postApiDataManagerTransitSyncPreflight200ResultPolicy"; + +export type PostApiDataManagerTransitSyncPreflight200Result = { + ok: boolean; + policy: PostApiDataManagerTransitSyncPreflight200ResultPolicy; + estimate: PostApiDataManagerTransitSyncPreflight200ResultEstimate; + capacity: PostApiDataManagerTransitSyncPreflight200ResultCapacity; + blockers: string[]; + warnings: string[]; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultCapacity.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultCapacity.ts new file mode 100644 index 000000000..5df1646db --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultCapacity.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight200ResultCapacity = { + /** @minimum 0 */ + freeDiskBytes: number; + /** @minimum 0 */ + freeInodes?: number; + /** @minimum 0 */ + slotMemoryGb: number; + /** @minimum 0 */ + slotCpu: number; + /** @minimum 0 */ + fileDescriptorLimit: number; + /** @minimum 0 */ + buildTimeoutHours: number; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimate.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimate.ts new file mode 100644 index 000000000..a957b5c8e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimate.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PostApiDataManagerTransitSyncPreflight200ResultEstimateBasis } from "./postApiDataManagerTransitSyncPreflight200ResultEstimateBasis"; + +export type PostApiDataManagerTransitSyncPreflight200ResultEstimate = { + /** @minimum 0 */ + feedCount: number; + /** @minimum 0 */ + measuredCompressedBytes: number; + /** @minimum 0 */ + compressedBytes: number; + /** @minimum 0 */ + expandedGtfsBytes: number; + /** @minimum 0 */ + importedMotisBytes: number; + /** @minimum 0 */ + osmBytes: number; + /** @minimum 0 */ + proxyCacheBytes: number; + /** @minimum 0 */ + requiredDiskBytes: number; + basis: PostApiDataManagerTransitSyncPreflight200ResultEstimateBasis; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimateBasis.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimateBasis.ts new file mode 100644 index 000000000..522cab015 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultEstimateBasis.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight200ResultEstimateBasis = + (typeof PostApiDataManagerTransitSyncPreflight200ResultEstimateBasis)[keyof typeof PostApiDataManagerTransitSyncPreflight200ResultEstimateBasis]; + +export const PostApiDataManagerTransitSyncPreflight200ResultEstimateBasis = { + "measured-and-conservative": "measured-and-conservative", + "conservative-defaults": "conservative-defaults", +} as const; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicy.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicy.ts new file mode 100644 index 000000000..bc08749f6 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicy.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PostApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition } from "./postApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition"; +import type { PostApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection } from "./postApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection"; +import type { PostApiDataManagerTransitSyncPreflight200ResultPolicyProfile } from "./postApiDataManagerTransitSyncPreflight200ResultPolicyProfile"; +import type { PostApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope } from "./postApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope"; + +export type PostApiDataManagerTransitSyncPreflight200ResultPolicy = { + profile: PostApiDataManagerTransitSyncPreflight200ResultPolicyProfile; + countries: string[]; + feedAllowList: string[]; + acquisition: PostApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition; + artifactOrigins: string[]; + originDownloadsRequired: boolean; + hostedRuntimeFallbackAllowed: boolean; + osmInput?: string; + gbfsSelection: PostApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection; + /** @minimum 0 */ + maxFeedCount: number; + /** @minimum 0 */ + retentionGenerations: number; + resourceEnvelope: PostApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope; + /** @minimum 0 */ + updateCadenceHours: number; + experimental: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition.ts new file mode 100644 index 000000000..4f978f8ad --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition = + (typeof PostApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition)[keyof typeof PostApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition]; + +export const PostApiDataManagerTransitSyncPreflight200ResultPolicyAcquisition = { + mirror: "mirror", + build: "build", +} as const; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection.ts new file mode 100644 index 000000000..ad11974cd --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection = + (typeof PostApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection)[keyof typeof PostApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection]; + +export const PostApiDataManagerTransitSyncPreflight200ResultPolicyGbfsSelection = { + "explicit-countries": "explicit-countries", + "explicit-feeds": "explicit-feeds", + "allow-listed-batches": "allow-listed-batches", +} as const; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyProfile.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyProfile.ts new file mode 100644 index 000000000..ecb88f171 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyProfile.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight200ResultPolicyProfile = + (typeof PostApiDataManagerTransitSyncPreflight200ResultPolicyProfile)[keyof typeof PostApiDataManagerTransitSyncPreflight200ResultPolicyProfile]; + +export const PostApiDataManagerTransitSyncPreflight200ResultPolicyProfile = { + "regional-assisted": "regional-assisted", + "regional-sovereign": "regional-sovereign", + planet: "planet", +} as const; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope.ts new file mode 100644 index 000000000..0617acf60 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight200ResultPolicyResourceEnvelope = { + /** @minimum 0 */ + minimumMemoryGb: number; + /** @minimum 0 */ + minimumCpu: number; + /** @minimum 0 */ + minimumDiskGb: number; + operatorCapacityRequired: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight400.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight400.ts new file mode 100644 index 000000000..b76c08142 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight401.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight401.ts new file mode 100644 index 000000000..174f28db3 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight403.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight403.ts new file mode 100644 index 000000000..b672e3058 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409.ts new file mode 100644 index 000000000..7f2db1414 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PostApiDataManagerTransitSyncPreflight409Code } from "./postApiDataManagerTransitSyncPreflight409Code"; + +export type PostApiDataManagerTransitSyncPreflight409 = { + error: string; + code: PostApiDataManagerTransitSyncPreflight409Code; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409Code.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409Code.ts new file mode 100644 index 000000000..c6985dd1a --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight409Code.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight409Code = + (typeof PostApiDataManagerTransitSyncPreflight409Code)[keyof typeof PostApiDataManagerTransitSyncPreflight409Code]; + +export const PostApiDataManagerTransitSyncPreflight409Code = { + preflight_stale: "preflight_stale", +} as const; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight413.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight413.ts new file mode 100644 index 000000000..d2c96df45 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight415.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight415.ts new file mode 100644 index 000000000..19fd0707d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight422.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight422.ts new file mode 100644 index 000000000..d66be2b0e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight429.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight429.ts new file mode 100644 index 000000000..478fc5a33 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight500.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight500.ts new file mode 100644 index 000000000..eb1b8123c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight503.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight503.ts new file mode 100644 index 000000000..84d5e9b3b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflight503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflight503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflightBody.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflightBody.ts new file mode 100644 index 000000000..99f879418 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitSyncPreflightBody.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PostApiDataManagerTransitSyncPreflightBody = { + /** + * @minItems 1 + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; +}; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index 0ed5d8f8d..3b5351938 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -343,6 +343,13 @@ export async function getApiDataManagerTransitJobsIdRequest( ): Promise { return getApiDataManagerTransitJobsIdBound(url, options); } +const getApiDataManagerTransitProfileBound = bindAdminOperation("getApiDataManagerTransitProfile"); +export async function getApiDataManagerTransitProfileRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getApiDataManagerTransitProfileBound(url, options); +} const getApiDataManagerTransitSourcesBound = bindAdminOperation("getApiDataManagerTransitSources"); export async function getApiDataManagerTransitSourcesRequest( url: string, @@ -564,6 +571,15 @@ export async function postApiDataManagerTransitSyncRequest( ): Promise { return postApiDataManagerTransitSyncBound(url, options); } +const postApiDataManagerTransitSyncPreflightBound = bindAdminOperation( + "postApiDataManagerTransitSyncPreflight", +); +export async function postApiDataManagerTransitSyncPreflightRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return postApiDataManagerTransitSyncPreflightBound(url, options); +} const previewAdminExtensionInstallBound = bindAdminOperation("previewAdminExtensionInstall"); export async function previewAdminExtensionInstallRequest( url: string, diff --git a/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json b/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json index ecd1a028d..cf084fff4 100644 --- a/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json +++ b/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json @@ -1,7 +1,7 @@ { - "revision": "d721c425dded1da76ea26b80937b741307f939ce", - "explicitOperations": 57, - "note": "Static inventory. Expands two literal ambient actions and imported relay constant. Excludes Fastify automatic HEAD aliases; binary HEAD routes explicitly included. No route handler was invoked. Updated by explicit review for the eight previously installed durable execution/fence routes and the new reversible source-disable route; baseline revision precedes the latter.", + "revision": "42a7964f9b8eaf28a6c2749ab8608e3b1d1f8e19", + "explicitOperations": 58, + "note": "Static inventory. Expands two literal ambient actions and imported relay constant. Excludes Fastify automatic HEAD aliases; binary HEAD routes explicitly included. No route handler was invoked. Updated by explicit review for the eight previously installed durable execution/fence routes and the new reversible source-disable route; baseline revision precedes the latter. Task05.2 explicitly adds the country-only server preflight route; parser, paging and metadata changes retain existing route identities.", "operations": [ { "method": "POST", @@ -230,6 +230,10 @@ { "method": "POST", "path": "/transit/sources/:sourceId/disable" + }, + { + "method": "POST", + "path": "/transit/sync/preflight" } ] } diff --git a/packages/core/src/services/data-manager/contracts/contracts.test.ts b/packages/core/src/services/data-manager/contracts/contracts.test.ts index 6348d4a00..d7e3fcb92 100644 --- a/packages/core/src/services/data-manager/contracts/contracts.test.ts +++ b/packages/core/src/services/data-manager/contracts/contracts.test.ts @@ -12,8 +12,8 @@ import { } from "./index"; describe("data-manager wire contracts", () => { - it("owns all 57 explicit operations with two named glyph HEAD/GET contracts", () => { - expect(Object.keys(dataManagerOperations)).toHaveLength(57); + it("owns all 58 explicit operations with two named glyph HEAD/GET contracts", () => { + expect(Object.keys(dataManagerOperations)).toHaveLength(58); expect(dataManagerOperations.getOfflineGlyph.path).toBe( "/offline/packages/glyphs/:version/:fontstack/:range.pbf", ); diff --git a/packages/core/src/services/data-manager/contracts/executions.ts b/packages/core/src/services/data-manager/contracts/executions.ts index f94f66911..dc02b7596 100644 --- a/packages/core/src/services/data-manager/contracts/executions.ts +++ b/packages/core/src/services/data-manager/contracts/executions.ts @@ -4,7 +4,7 @@ import { processDiagnosticsSchema } from "../../../ops/job-evidence"; import { hardlinkApplySummarySchema } from "../../../ops/service-results"; import { regionSchema } from "./common"; import { hardlinkEntrySchema } from "./datasets"; -import { transitDesiredSourceMutationSchema } from "./transit"; +import { transitDesiredSourceMutationSchema, transitPreflightBindingSchema } from "./transit"; export const dataOperationKeySchema = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/); export const dataFingerprintSchema = z.string().regex(/^[a-f0-9]{64}$/); @@ -22,6 +22,7 @@ const ordinaryInputs = [ kind: z.literal("transit-sync"), countries, sourceMutation: transitDesiredSourceMutationSchema.optional(), + preflight: transitPreflightBindingSchema.optional(), }), regional("overture-pull"), regional("overture-ingest"), diff --git a/packages/core/src/services/data-manager/contracts/operations.ts b/packages/core/src/services/data-manager/contracts/operations.ts index fc7ec6613..849b0a711 100644 --- a/packages/core/src/services/data-manager/contracts/operations.ts +++ b/packages/core/src/services/data-manager/contracts/operations.ts @@ -396,6 +396,19 @@ export const dataManagerOperations = { 422: json(t.transitPreflightSchema.extend({ ok: z.literal(false) })), }, }, + inspectTransitSyncPreflight: { + method: "POST", + path: "/transit/sync/preflight", + auth: service, + request: { body: t.transitPreflightSelectionSchema }, + responses: { + ...serviceErrors, + 200: json(t.transitSyncPreflightSchema), + 409: json(t.transitPreflightStaleSchema), + 422: json(httpErrorSchema), + 503: json(httpErrorSchema), + }, + }, syncTransit: { method: "POST", path: "/transit/sync", @@ -404,7 +417,7 @@ export const dataManagerOperations = { responses: { ...dataMutationErrors, 202: json(t.transitSyncAcceptedSchema), - 409: json(z.union([httpErrorSchema, t.transitConflictSchema])), + 409: json(z.union([httpErrorSchema, t.transitConflictSchema, t.transitPreflightStaleSchema])), 422: json(t.transitScopeRejectedSchema), }, }, diff --git a/packages/core/src/services/data-manager/contracts/transit.ts b/packages/core/src/services/data-manager/contracts/transit.ts index e6e032750..f1e58a5e2 100644 --- a/packages/core/src/services/data-manager/contracts/transit.ts +++ b/packages/core/src/services/data-manager/contracts/transit.ts @@ -83,6 +83,7 @@ export const transitPolicySchema = z.strictObject({ }); export const transitProfileSchema = z.strictObject({ policy: transitPolicySchema }); export const transitSourcesQuerySchema = z.strictObject({ + country: z.string().optional(), search: z.string().optional(), lifecycle: transitSourceLifecycleSchema.optional(), origin: z.enum(["catalog", "operator"]).optional(), @@ -144,9 +145,30 @@ export const transitConflictSchema = z.strictObject({ reason: z.enum(["in-flight", "duplicate-idempotency-key"]), existingJobId: z.string().nullable(), }); +export const transitPreflightBindingSchema = z.strictObject({ + preflightId: z.uuid(), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + expiresAt: timestampSchema, + sourceFingerprint: z.string().regex(/^[a-f0-9]{64}$/), +}); +export const transitPreflightSelectionSchema = z.strictObject({ + countries: z + .array(z.string().regex(/^[A-Za-z]{2,3}$/)) + .min(1) + .max(256), +}); +export const transitPreflightStaleSchema = z.strictObject({ + error: z.string(), + code: z.literal("preflight_stale"), +}); export const transitSyncRequestSchema = z .strictObject({ ...mutationShape, + preflightId: z.uuid().optional(), + fingerprint: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), countries: z .array(z.string().regex(/^[A-Za-z][A-Za-z0-9_-]{0,31}$/)) .max(250) @@ -202,6 +224,14 @@ export const transitPreflightSchema = z.strictObject({ blockers: z.array(z.string()), warnings: z.array(z.string()), }); +export const transitSyncPreflightSchema = z.strictObject({ + preflightId: z.uuid(), + expiresAt: timestampSchema, + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + countries: z.array(z.string()), + allowedCountries: z.array(z.string()), + result: transitPreflightSchema, +}); const safeGitRef = z .string() .regex(/^[A-Za-z0-9][A-Za-z0-9._/-]*$/) diff --git a/packages/core/src/services/data-manager/generated/data-manager.generated.ts b/packages/core/src/services/data-manager/generated/data-manager.generated.ts index 663c97287..8ba4dd45b 100644 --- a/packages/core/src/services/data-manager/generated/data-manager.generated.ts +++ b/packages/core/src/services/data-manager/generated/data-manager.generated.ts @@ -340,6 +340,17 @@ import type { InspectDataConfigurationFence429, InspectDataConfigurationFence500, InspectDataConfigurationFence503, + InspectTransitSyncPreflight200, + InspectTransitSyncPreflight400, + InspectTransitSyncPreflight401, + InspectTransitSyncPreflight409, + InspectTransitSyncPreflight413, + InspectTransitSyncPreflight415, + InspectTransitSyncPreflight422, + InspectTransitSyncPreflight429, + InspectTransitSyncPreflight500, + InspectTransitSyncPreflight503, + InspectTransitSyncPreflightBody, Link200, Link400, Link401, @@ -570,6 +581,7 @@ import { headOfflinePackageArchiveRequest, ingestOvertureRequest, inspectDataConfigurationFenceRequest, + inspectTransitSyncPreflightRequest, linkRequest, lookupDataExecutionRequest, preflightTransitRequest, @@ -5401,6 +5413,115 @@ export const conflateOverture = async ( }); }; +export type inspectTransitSyncPreflightResponse200 = { + data: InspectTransitSyncPreflight200; + status: 200; +}; + +export type inspectTransitSyncPreflightResponse400 = { + data: InspectTransitSyncPreflight400; + status: 400; +}; + +export type inspectTransitSyncPreflightResponse401 = { + data: InspectTransitSyncPreflight401; + status: 401; +}; + +export type inspectTransitSyncPreflightResponse409 = { + data: InspectTransitSyncPreflight409; + status: 409; +}; + +export type inspectTransitSyncPreflightResponse413 = { + data: InspectTransitSyncPreflight413; + status: 413; +}; + +export type inspectTransitSyncPreflightResponse415 = { + data: InspectTransitSyncPreflight415; + status: 415; +}; + +export type inspectTransitSyncPreflightResponse422 = { + data: InspectTransitSyncPreflight422; + status: 422; +}; + +export type inspectTransitSyncPreflightResponse429 = { + data: InspectTransitSyncPreflight429; + status: 429; +}; + +export type inspectTransitSyncPreflightResponse500 = { + data: InspectTransitSyncPreflight500; + status: 500; +}; + +export type inspectTransitSyncPreflightResponse503 = { + data: InspectTransitSyncPreflight503; + status: 503; +}; + +export type inspectTransitSyncPreflightResponseSuccess = inspectTransitSyncPreflightResponse200 & { + headers: Headers; +}; +export type inspectTransitSyncPreflightResponseError = ( + | inspectTransitSyncPreflightResponse400 + | inspectTransitSyncPreflightResponse401 + | inspectTransitSyncPreflightResponse409 + | inspectTransitSyncPreflightResponse413 + | inspectTransitSyncPreflightResponse415 + | inspectTransitSyncPreflightResponse422 + | inspectTransitSyncPreflightResponse429 + | inspectTransitSyncPreflightResponse500 + | inspectTransitSyncPreflightResponse503 +) & { + headers: Headers; +}; + +export type inspectTransitSyncPreflightResponse = + | inspectTransitSyncPreflightResponseSuccess + | inspectTransitSyncPreflightResponseError; + +export const getInspectTransitSyncPreflightUrl = () => { + return `/transit/sync/preflight`; +}; + +export const inspectTransitSyncPreflight = async ( + inspectTransitSyncPreflightBody: InspectTransitSyncPreflightBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return inspectTransitSyncPreflightRequest( + getInspectTransitSyncPreflightUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(inspectTransitSyncPreflightBody), + }, + ); +}; + export type syncTransitResponse202 = { data: SyncTransit202; status: 202; diff --git a/packages/core/src/services/data-manager/generated/models/getTransitSourcesParams.ts b/packages/core/src/services/data-manager/generated/models/getTransitSourcesParams.ts index d50bd89f8..71cfaa93c 100644 --- a/packages/core/src/services/data-manager/generated/models/getTransitSourcesParams.ts +++ b/packages/core/src/services/data-manager/generated/models/getTransitSourcesParams.ts @@ -4,6 +4,7 @@ import type { GetTransitSourcesLifecycle } from "./getTransitSourcesLifecycle"; import type { GetTransitSourcesOrigin } from "./getTransitSourcesOrigin"; export type GetTransitSourcesParams = { + country?: string; search?: string; lifecycle?: GetTransitSourcesLifecycle; origin?: GetTransitSourcesOrigin; diff --git a/packages/core/src/services/data-manager/generated/models/index.ts b/packages/core/src/services/data-manager/generated/models/index.ts index a810df769..2f5ca7d3c 100644 --- a/packages/core/src/services/data-manager/generated/models/index.ts +++ b/packages/core/src/services/data-manager/generated/models/index.ts @@ -478,6 +478,27 @@ export * from "./inspectDataConfigurationFence415"; export * from "./inspectDataConfigurationFence429"; export * from "./inspectDataConfigurationFence500"; export * from "./inspectDataConfigurationFence503"; +export * from "./inspectTransitSyncPreflight200"; +export * from "./inspectTransitSyncPreflight200Result"; +export * from "./inspectTransitSyncPreflight200ResultCapacity"; +export * from "./inspectTransitSyncPreflight200ResultEstimate"; +export * from "./inspectTransitSyncPreflight200ResultEstimateBasis"; +export * from "./inspectTransitSyncPreflight200ResultPolicy"; +export * from "./inspectTransitSyncPreflight200ResultPolicyAcquisition"; +export * from "./inspectTransitSyncPreflight200ResultPolicyGbfsSelection"; +export * from "./inspectTransitSyncPreflight200ResultPolicyProfile"; +export * from "./inspectTransitSyncPreflight200ResultPolicyResourceEnvelope"; +export * from "./inspectTransitSyncPreflight400"; +export * from "./inspectTransitSyncPreflight401"; +export * from "./inspectTransitSyncPreflight409"; +export * from "./inspectTransitSyncPreflight409Code"; +export * from "./inspectTransitSyncPreflight413"; +export * from "./inspectTransitSyncPreflight415"; +export * from "./inspectTransitSyncPreflight422"; +export * from "./inspectTransitSyncPreflight429"; +export * from "./inspectTransitSyncPreflight500"; +export * from "./inspectTransitSyncPreflight503"; +export * from "./inspectTransitSyncPreflightBody"; export * from "./link200"; export * from "./link400"; export * from "./link401"; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200.ts new file mode 100644 index 000000000..aa5b4a9a1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectTransitSyncPreflight200Result } from "./inspectTransitSyncPreflight200Result"; + +export type InspectTransitSyncPreflight200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + preflightId: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + countries: string[]; + allowedCountries: string[]; + result: InspectTransitSyncPreflight200Result; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200Result.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200Result.ts new file mode 100644 index 000000000..fba215555 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200Result.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectTransitSyncPreflight200ResultCapacity } from "./inspectTransitSyncPreflight200ResultCapacity"; +import type { InspectTransitSyncPreflight200ResultEstimate } from "./inspectTransitSyncPreflight200ResultEstimate"; +import type { InspectTransitSyncPreflight200ResultPolicy } from "./inspectTransitSyncPreflight200ResultPolicy"; + +export type InspectTransitSyncPreflight200Result = { + ok: boolean; + policy: InspectTransitSyncPreflight200ResultPolicy; + estimate: InspectTransitSyncPreflight200ResultEstimate; + capacity: InspectTransitSyncPreflight200ResultCapacity; + blockers: string[]; + warnings: string[]; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultCapacity.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultCapacity.ts new file mode 100644 index 000000000..0d6721777 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultCapacity.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight200ResultCapacity = { + /** @minimum 0 */ + freeDiskBytes: number; + /** @minimum 0 */ + freeInodes?: number; + /** @minimum 0 */ + slotMemoryGb: number; + /** @minimum 0 */ + slotCpu: number; + /** @minimum 0 */ + fileDescriptorLimit: number; + /** @minimum 0 */ + buildTimeoutHours: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimate.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimate.ts new file mode 100644 index 000000000..550abd997 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimate.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectTransitSyncPreflight200ResultEstimateBasis } from "./inspectTransitSyncPreflight200ResultEstimateBasis"; + +export type InspectTransitSyncPreflight200ResultEstimate = { + /** @minimum 0 */ + feedCount: number; + /** @minimum 0 */ + measuredCompressedBytes: number; + /** @minimum 0 */ + compressedBytes: number; + /** @minimum 0 */ + expandedGtfsBytes: number; + /** @minimum 0 */ + importedMotisBytes: number; + /** @minimum 0 */ + osmBytes: number; + /** @minimum 0 */ + proxyCacheBytes: number; + /** @minimum 0 */ + requiredDiskBytes: number; + basis: InspectTransitSyncPreflight200ResultEstimateBasis; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimateBasis.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimateBasis.ts new file mode 100644 index 000000000..8e6e766bd --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultEstimateBasis.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight200ResultEstimateBasis = + (typeof InspectTransitSyncPreflight200ResultEstimateBasis)[keyof typeof InspectTransitSyncPreflight200ResultEstimateBasis]; + +export const InspectTransitSyncPreflight200ResultEstimateBasis = { + "measured-and-conservative": "measured-and-conservative", + "conservative-defaults": "conservative-defaults", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicy.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicy.ts new file mode 100644 index 000000000..347588846 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicy.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectTransitSyncPreflight200ResultPolicyAcquisition } from "./inspectTransitSyncPreflight200ResultPolicyAcquisition"; +import type { InspectTransitSyncPreflight200ResultPolicyGbfsSelection } from "./inspectTransitSyncPreflight200ResultPolicyGbfsSelection"; +import type { InspectTransitSyncPreflight200ResultPolicyProfile } from "./inspectTransitSyncPreflight200ResultPolicyProfile"; +import type { InspectTransitSyncPreflight200ResultPolicyResourceEnvelope } from "./inspectTransitSyncPreflight200ResultPolicyResourceEnvelope"; + +export type InspectTransitSyncPreflight200ResultPolicy = { + profile: InspectTransitSyncPreflight200ResultPolicyProfile; + countries: string[]; + feedAllowList: string[]; + acquisition: InspectTransitSyncPreflight200ResultPolicyAcquisition; + artifactOrigins: string[]; + originDownloadsRequired: boolean; + hostedRuntimeFallbackAllowed: boolean; + osmInput?: string; + gbfsSelection: InspectTransitSyncPreflight200ResultPolicyGbfsSelection; + /** @minimum 0 */ + maxFeedCount: number; + /** @minimum 0 */ + retentionGenerations: number; + resourceEnvelope: InspectTransitSyncPreflight200ResultPolicyResourceEnvelope; + /** @minimum 0 */ + updateCadenceHours: number; + experimental: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyAcquisition.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyAcquisition.ts new file mode 100644 index 000000000..8a5e520e9 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyAcquisition.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight200ResultPolicyAcquisition = + (typeof InspectTransitSyncPreflight200ResultPolicyAcquisition)[keyof typeof InspectTransitSyncPreflight200ResultPolicyAcquisition]; + +export const InspectTransitSyncPreflight200ResultPolicyAcquisition = { + mirror: "mirror", + build: "build", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyGbfsSelection.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyGbfsSelection.ts new file mode 100644 index 000000000..86de59c7d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyGbfsSelection.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight200ResultPolicyGbfsSelection = + (typeof InspectTransitSyncPreflight200ResultPolicyGbfsSelection)[keyof typeof InspectTransitSyncPreflight200ResultPolicyGbfsSelection]; + +export const InspectTransitSyncPreflight200ResultPolicyGbfsSelection = { + "explicit-countries": "explicit-countries", + "explicit-feeds": "explicit-feeds", + "allow-listed-batches": "allow-listed-batches", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyProfile.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyProfile.ts new file mode 100644 index 000000000..b90279ade --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyProfile.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight200ResultPolicyProfile = + (typeof InspectTransitSyncPreflight200ResultPolicyProfile)[keyof typeof InspectTransitSyncPreflight200ResultPolicyProfile]; + +export const InspectTransitSyncPreflight200ResultPolicyProfile = { + "regional-assisted": "regional-assisted", + "regional-sovereign": "regional-sovereign", + planet: "planet", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyResourceEnvelope.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyResourceEnvelope.ts new file mode 100644 index 000000000..aa75138bd --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight200ResultPolicyResourceEnvelope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight200ResultPolicyResourceEnvelope = { + /** @minimum 0 */ + minimumMemoryGb: number; + /** @minimum 0 */ + minimumCpu: number; + /** @minimum 0 */ + minimumDiskGb: number; + operatorCapacityRequired: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight400.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight400.ts new file mode 100644 index 000000000..f4e774971 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight401.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight401.ts new file mode 100644 index 000000000..852cdc858 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409.ts new file mode 100644 index 000000000..633c4bbcf --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectTransitSyncPreflight409Code } from "./inspectTransitSyncPreflight409Code"; + +export type InspectTransitSyncPreflight409 = { + error: string; + code: InspectTransitSyncPreflight409Code; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409Code.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409Code.ts new file mode 100644 index 000000000..1fcb32e96 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight409Code.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight409Code = + (typeof InspectTransitSyncPreflight409Code)[keyof typeof InspectTransitSyncPreflight409Code]; + +export const InspectTransitSyncPreflight409Code = { + preflight_stale: "preflight_stale", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight413.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight413.ts new file mode 100644 index 000000000..310efdd56 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight415.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight415.ts new file mode 100644 index 000000000..d80f8409a --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight422.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight422.ts new file mode 100644 index 000000000..4f030a881 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight429.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight429.ts new file mode 100644 index 000000000..b8be891eb --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight500.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight500.ts new file mode 100644 index 000000000..894e6834e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight503.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight503.ts new file mode 100644 index 000000000..d40f39503 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflight503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflight503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflightBody.ts b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflightBody.ts new file mode 100644 index 000000000..552dadddf --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectTransitSyncPreflightBody.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectTransitSyncPreflightBody = { + /** + * @minItems 1 + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; +}; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts index 22ed0d142..ab88a5345 100644 --- a/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts +++ b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts @@ -58,6 +58,16 @@ export type StartDataExecutionBodyInput = /** @pattern ^[a-f0-9]{64}$ */ expectedSourceRevision: string; }; + preflight?: { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + preflightId: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourceFingerprint: string; + }; } | { kind: "overture-pull"; diff --git a/packages/core/src/services/data-manager/generated/models/syncTransit409.ts b/packages/core/src/services/data-manager/generated/models/syncTransit409.ts index d6ee57704..fb8ec7a36 100644 --- a/packages/core/src/services/data-manager/generated/models/syncTransit409.ts +++ b/packages/core/src/services/data-manager/generated/models/syncTransit409.ts @@ -23,4 +23,8 @@ export type SyncTransit409 = reason: "in-flight" | "duplicate-idempotency-key"; /** @nullable */ existingJobId: string | null; + } + | { + error: string; + code: "preflight_stale"; }; diff --git a/packages/core/src/services/data-manager/generated/models/syncTransitBody.ts b/packages/core/src/services/data-manager/generated/models/syncTransitBody.ts index ae6595048..ea542365a 100644 --- a/packages/core/src/services/data-manager/generated/models/syncTransitBody.ts +++ b/packages/core/src/services/data-manager/generated/models/syncTransitBody.ts @@ -6,6 +6,10 @@ export type SyncTransitBody = { idempotencyKey?: string; /** @maxLength 200 */ triggeredBy?: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + preflightId?: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint?: string; /** * @maxItems 250 * @items.pattern ^[A-Za-z][A-Za-z0-9_-]{0,31}$ diff --git a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts index 51ebf6e5b..2f5053ad0 100644 --- a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts +++ b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts @@ -277,6 +277,13 @@ export async function inspectDataConfigurationFenceRequest( ): Promise { return inspectDataConfigurationFenceBound(url, options); } +const inspectTransitSyncPreflightBound = bindDataManagerOperation("inspectTransitSyncPreflight"); +export async function inspectTransitSyncPreflightRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return inspectTransitSyncPreflightBound(url, options); +} const linkBound = bindDataManagerOperation("link"); export async function linkRequest(url: string, options?: DataManagerRequestOptions): Promise { return linkBound(url, options); diff --git a/packages/core/src/services/data-manager/sdk.test.ts b/packages/core/src/services/data-manager/sdk.test.ts index d938edbff..d7cdae976 100644 --- a/packages/core/src/services/data-manager/sdk.test.ts +++ b/packages/core/src/services/data-manager/sdk.test.ts @@ -19,8 +19,8 @@ describe("generated complete internal data-manager SDK", () => { Extract>, { status: 416 }>["data"] >().toEqualTypeOf(); }); - it("has a generated callable for all 57 explicit operations", () => { - expect(Object.keys(dataManagerOperations)).toHaveLength(57); + it("has a generated callable for all 58 explicit operations", () => { + expect(Object.keys(dataManagerOperations)).toHaveLength(58); for (const key of Object.keys(dataManagerOperations)) expect(sdk).toHaveProperty(key, expect.any(Function)); }); diff --git a/services/data-manager/__tests__/api.test.ts b/services/data-manager/__tests__/api.test.ts index 7a9dc7dd8..e608cae15 100644 --- a/services/data-manager/__tests__/api.test.ts +++ b/services/data-manager/__tests__/api.test.ts @@ -548,6 +548,76 @@ describe("data-manager API", () => { expect(singleFlight.tryStartSync).not.toHaveBeenCalled(); }); + it.each([{ countries: [] }, { countries: ["de"], freeDiskBytes: 10 ** 15, feedCount: 1 }])( + "POST /transit/sync/preflight refuses an invalid selection or caller capacity %j", + async (payload) => { + const app = createTestApp(); + registerApi(app, { + dataDir: createDataDir(), + operationsPolicy: resolveOperationsProfile({ countries: ["de", "at"] }), + }); + const response = await app.inject({ + method: "POST", + url: "/transit/sync/preflight", + payload, + }); + expect(response.statusCode).toBe(400); + }, + ); + + it("observes an already admitted reviewed intent without replaying an expired preview", async () => { + const app = createTestApp(), + status = dataReceipt({ kind: "transit-sync", status: "running", publication: "unknown" }), + admitLocal = vi.fn(async () => status); + registerApi(app, { + dataDir: createDataDir(), + operationsPolicy: resolveOperationsProfile({ countries: ["de"] }), + executionReceiver: receiverFixture({ + admitLocal, + observeReviewedTransit: async () => status, + }), + }); + const response = await app.inject({ + method: "POST", + url: "/transit/sync", + payload: { + countries: ["de"], + preflightId: "00000000-0000-4000-8000-000000000001", + fingerprint: "a".repeat(64), + idempotencyKey: "original-intent", + }, + }); + expect(response.statusCode).toBe(202); + expect(response.json().jobId).toBe(status.id); + expect(admitLocal).not.toHaveBeenCalled(); + }); + + it("does not disclose private acquisition diagnostics from transit preview failures", async () => { + const ops = await import("../src/ops-client.js"), + spy = vi + .spyOn(ops, "runOpsOperation") + .mockRejectedValueOnce( + Error("private acquisition /private/operator-file synthetic-secret"), + ); + try { + const app = createTestApp(); + registerApi(app, { + dataDir: createDataDir(), + operationsPolicy: resolveOperationsProfile({ countries: ["de"] }), + }); + const response = await app.inject({ + method: "POST", + url: "/transit/sync/preflight", + payload: { countries: ["de"] }, + }); + expect(response.statusCode).toBe(503); + expect(response.body).not.toContain("operator-file"); + expect(response.body).not.toContain("synthetic-secret"); + } finally { + spy.mockRestore(); + } + }); + it("POST /transit/sync starts an in-scope country", async () => { const dataDir = createDataDir(); const singleFlight: SingleFlightController = { diff --git a/services/data-manager/__tests__/openapi-freshness.test.ts b/services/data-manager/__tests__/openapi-freshness.test.ts index 2cf90330d..a4b27a9c3 100644 --- a/services/data-manager/__tests__/openapi-freshness.test.ts +++ b/services/data-manager/__tests__/openapi-freshness.test.ts @@ -8,7 +8,7 @@ it("exports every actual data-manager route with stable IDs, auth and payload co expect(generated).toBe(await readFile(DOCUMENT_PATH, "utf8")); const document = JSON.parse(generated); const operations = Object.values(document.paths).flatMap((path) => Object.values(path as object)); - expect(operations).toHaveLength(57); + expect(operations).toHaveLength(58); for (const value of operations) { const operation = value as { operationId: keyof typeof dataManagerOperations; diff --git a/services/data-manager/__tests__/transitous/admin-preflight.test.ts b/services/data-manager/__tests__/transitous/admin-preflight.test.ts new file mode 100644 index 000000000..c55374a3e --- /dev/null +++ b/services/data-manager/__tests__/transitous/admin-preflight.test.ts @@ -0,0 +1,148 @@ +import * as fs from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { + createTransitPreflightController, + normalizeTransitPreflightCountries, + verifyTransitPreflightBinding, +} from "../../src/jobs/transitous/admin-preflight.js"; +import { resolveOperationsProfile } from "../../src/jobs/transitous/operations-profile.js"; + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, statfsSync: vi.fn(actual.statfsSync) }; +}); +const pins = vi.hoisted(() => ({ value: null as unknown })); +vi.mock("../../src/ops-client.js", () => ({ runOpsOperation: async () => pins.value })); +let root: string; +const policy = () => resolveOperationsProfile({ countries: ["de", "at"] }); +const makePins = (ref = "a") => + createCatalogPinSet( + { + ref: `main@${ref.repeat(40)}`, + submodules: {}, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }, + { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "b".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"b".repeat(40)}/systems.csv`, + sha256: "c".repeat(64), + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }, + ); +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "omx-admin-preflight-")); + mkdirSync(join(root, ".transitous-catalog/feeds"), { recursive: true }); + writeFileSync( + join(root, ".transitous-catalog/feeds/de.json"), + JSON.stringify({ + sources: [ + { name: "catalog", spec: "gtfs", type: "http", url: "https://example.test/feed.zip" }, + ], + }), + ); + const active = makePins(); + pins.value = { active, catalogRevisionId: active.generationId }; + for (const [key, value] of Object.entries({ + MOTIS_SLOT_MEMORY_GB: "16", + MOTIS_SLOT_CPU: "4", + MOTIS_FILE_DESCRIPTOR_LIMIT: "65536", + MOTIS_BUILD_TIMEOUT_HOURS: "12", + })) + vi.stubEnv(key, value); + vi.mocked(fs.statfsSync).mockReturnValue({ + bsize: 4096, + bavail: 2 ** 30, + ffree: 10 ** 9, + } as fs.StatsFs); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); +}); +it("normalizes a checked country subset without accepting empty or unauthorized selections", () => { + expect(normalizeTransitPreflightCountries(["DE", "de"], policy())).toEqual(["de"]); + expect(() => normalizeTransitPreflightCountries([], policy())).toThrow(/Select/); + expect(() => normalizeTransitPreflightCountries(["de", "us"], policy())).toThrow(/scope/); +}); +it("derives a bounded preview from actual desired sources, pins and capacity", async () => { + const controller = createTransitPreflightController({ dataDir: root, policy: policy() }); + const preview = await controller.inspect(["DE", "de"]); + expect(preview.countries).toEqual(["de"]); + expect(preview.result.ok).toBe(true); + expect(preview.result.estimate.feedCount).toBe(1); + expect(preview.result.capacity.freeDiskBytes).toBe(2 ** 42); + const binding = await controller.verify({ + countries: ["de"], + preflightId: preview.preflightId, + fingerprint: preview.fingerprint, + }); + await expect( + verifyTransitPreflightBinding({ dataDir: root, policy: policy() }, ["de"], binding), + ).resolves.toBeUndefined(); +}); +it.each(["catalog", "pins", "capacity", "expiry"])( + "refuses stale %s evidence at admission and worker execution", + async (change) => { + const options = { dataDir: root, policy: policy() }, + controller = createTransitPreflightController(options), + preview = await controller.inspect(["de"]); + const request = { + countries: ["de"], + preflightId: preview.preflightId, + fingerprint: preview.fingerprint, + }, + binding = await controller.verify(request); + if (change === "catalog") + writeFileSync( + join(root, ".transitous-catalog/feeds/de.json"), + JSON.stringify({ sources: [] }), + ); + if (change === "pins") { + const active = makePins("d"); + pins.value = { active, catalogRevisionId: active.generationId }; + } + if (change === "capacity") + vi.mocked(fs.statfsSync).mockReturnValue({ bsize: 4096, bavail: 1, ffree: 1 } as fs.StatsFs); + if (change === "expiry") vi.spyOn(Date, "now").mockReturnValue(Date.now() + 301000); + await expect(controller.verify(request)).rejects.toMatchObject({ code: "preflight_stale" }); + await expect(verifyTransitPreflightBinding(options, ["de"], binding)).rejects.toMatchObject({ + code: "preflight_stale", + }); + }, +); +it("blocks unknown filesystem capacity and rejects invented preview identities", async () => { + vi.mocked(fs.statfsSync).mockImplementation(() => { + throw Error("fixture unavailable"); + }); + const controller = createTransitPreflightController({ dataDir: root, policy: policy() }), + preview = await controller.inspect(["de"]); + expect(preview.result.ok).toBe(false); + expect(preview.result.capacity.freeDiskBytes).toBe(0); + await expect( + controller.verify({ + countries: ["de"], + preflightId: preview.preflightId, + fingerprint: "0".repeat(64), + }), + ).rejects.toMatchObject({ code: "preflight_stale" }); +}); +it("supports a country implied by a finite feed allow-list while retaining the feed constraint", async () => { + const limited = resolveOperationsProfile({ countries: [], feedAllowList: ["de_catalog"] }); + expect(normalizeTransitPreflightCountries(["DE"], limited)).toEqual(["de"]); + const preview = await createTransitPreflightController({ + dataDir: root, + policy: limited, + }).inspect(["de"]); + expect(preview.allowedCountries).toEqual(["de"]); + expect(preview.result.ok).toBe(true); + expect(() => normalizeTransitPreflightCountries(["at"], limited)).toThrow(/scope/); +}); diff --git a/services/data-manager/__tests__/transitous/preflight.test.ts b/services/data-manager/__tests__/transitous/preflight.test.ts index 3c8c60dac..9f4ddd834 100644 --- a/services/data-manager/__tests__/transitous/preflight.test.ts +++ b/services/data-manager/__tests__/transitous/preflight.test.ts @@ -181,3 +181,21 @@ describe("MOTIS capacity preflight", () => { expect(result.warnings.join(" ")).toMatch(/conservative defaults/); }); }); +it("treats the actual schedule and artifact identities as aliases of one allowed feed", () => { + const policy = resolveOperationsProfile({ countries: ["de"], feedAllowList: ["de_demo_test"] }); + const result = runMotisPreflight({ + policy, + feedCount: 1, + osmAvailable: false, + capacity: { + freeDiskBytes: 10 ** 15, + slotMemoryGb: 16, + slotCpu: 4, + fileDescriptorLimit: 65536, + buildTimeoutHours: 12, + }, + selectedFeedIds: ["de_demo_test", "de_demo.test"], + selectedSources: [{ id: "de_demo_test", artifactId: "de_demo.test" }], + }); + expect(result.blockers.filter((message) => message.includes("allow-list"))).toEqual([]); +}); diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index cc94b96b9..86e39923d 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -271,6 +271,36 @@ "additionalProperties": false } ] + }, + "preflight": { + "type": "object", + "properties": { + "preflightId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "sourceFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "preflightId", + "fingerprint", + "expiresAt", + "sourceFingerprint" + ], + "additionalProperties": false } }, "required": [ @@ -26098,6 +26128,14 @@ "data-manager" ], "parameters": [ + { + "schema": { + "type": "string" + }, + "in": "query", + "name": "country", + "required": false + }, { "schema": { "type": "string" @@ -42073,9 +42111,9 @@ } } }, - "/transit/sync": { + "/transit/sync/preflight": { "post": { - "operationId": "syncTransit", + "operationId": "inspectTransitSyncPreflight", "tags": [ "data-manager" ], @@ -42086,23 +42124,19 @@ "schema": { "type": "object", "properties": { - "idempotencyKey": { - "type": "string", - "maxLength": 200 - }, - "triggeredBy": { - "type": "string", - "maxLength": 200 - }, "countries": { - "maxItems": 250, + "minItems": 1, + "maxItems": 256, "type": "array", "items": { "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,31}$" + "pattern": "^[A-Za-z]{2,3}$" } } }, + "required": [ + "countries" + ], "additionalProperties": false } } @@ -42116,103 +42150,958 @@ "serviceToken": [] } ], - "x-openmapx-contract-id": "syncTransit", + "x-openmapx-contract-id": "inspectTransitSyncPreflight", "x-openmapx-auth": "service", "responses": { - "202": { + "200": { "description": "Response", "content": { "application/json": { "schema": { "type": "object", "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] + "preflightId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" }, - "jobId": { - "type": "string" + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" }, - "status": { + "fingerprint": { "type": "string", - "enum": [ - "started" - ] - } - }, - "required": [ - "ok", - "jobId", - "status" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { + "pattern": "^[a-f0-9]{64}$" + }, + "countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowedCountries": { + "type": "array", + "items": { + "type": "string" + } + }, + "result": { "type": "object", "properties": { - "error": { - "type": "string" - }, "ok": { - "type": "boolean", - "enum": [ - false - ] + "type": "boolean" }, - "message": { - "type": "string" + "policy": { + "type": "object", + "properties": { + "profile": { + "type": "string", + "enum": [ + "regional-assisted", + "regional-sovereign", + "planet" + ] + }, + "countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "feedAllowList": { + "type": "array", + "items": { + "type": "string" + } + }, + "acquisition": { + "type": "string", + "enum": [ + "mirror", + "build" + ] + }, + "artifactOrigins": { + "type": "array", + "items": { + "type": "string" + } + }, + "originDownloadsRequired": { + "type": "boolean" + }, + "hostedRuntimeFallbackAllowed": { + "type": "boolean" + }, + "osmInput": { + "type": "string" + }, + "gbfsSelection": { + "type": "string", + "enum": [ + "explicit-countries", + "explicit-feeds", + "allow-listed-batches" + ] + }, + "maxFeedCount": { + "type": "number", + "minimum": 0 + }, + "retentionGenerations": { + "type": "number", + "minimum": 0 + }, + "resourceEnvelope": { + "type": "object", + "properties": { + "minimumMemoryGb": { + "type": "number", + "minimum": 0 + }, + "minimumCpu": { + "type": "number", + "minimum": 0 + }, + "minimumDiskGb": { + "type": "number", + "minimum": 0 + }, + "operatorCapacityRequired": { + "type": "boolean" + } + }, + "required": [ + "minimumMemoryGb", + "minimumCpu", + "minimumDiskGb", + "operatorCapacityRequired" + ], + "additionalProperties": false + }, + "updateCadenceHours": { + "type": "number", + "minimum": 0 + }, + "experimental": { + "type": "boolean" + } + }, + "required": [ + "profile", + "countries", + "feedAllowList", + "acquisition", + "artifactOrigins", + "originDownloadsRequired", + "hostedRuntimeFallbackAllowed", + "gbfsSelection", + "maxFeedCount", + "retentionGenerations", + "resourceEnvelope", + "updateCadenceHours", + "experimental" + ], + "additionalProperties": false }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 + "estimate": { + "type": "object", + "properties": { + "feedCount": { + "type": "number", + "minimum": 0 + }, + "measuredCompressedBytes": { + "type": "number", + "minimum": 0 + }, + "compressedBytes": { + "type": "number", + "minimum": 0 + }, + "expandedGtfsBytes": { + "type": "number", + "minimum": 0 + }, + "importedMotisBytes": { + "type": "number", + "minimum": 0 + }, + "osmBytes": { + "type": "number", + "minimum": 0 + }, + "proxyCacheBytes": { + "type": "number", + "minimum": 0 + }, + "requiredDiskBytes": { + "type": "number", + "minimum": 0 + }, + "basis": { + "type": "string", + "enum": [ + "measured-and-conservative", + "conservative-defaults" + ] + } + }, + "required": [ + "feedCount", + "measuredCompressedBytes", + "compressedBytes", + "expandedGtfsBytes", + "importedMotisBytes", + "osmBytes", + "proxyCacheBytes", + "requiredDiskBytes", + "basis" + ], + "additionalProperties": false }, - "error": { - "type": "string" + "capacity": { + "type": "object", + "properties": { + "freeDiskBytes": { + "type": "number", + "minimum": 0 + }, + "freeInodes": { + "type": "number", + "minimum": 0 + }, + "slotMemoryGb": { + "type": "number", + "minimum": 0 + }, + "slotCpu": { + "type": "number", + "minimum": 0 + }, + "fileDescriptorLimit": { + "type": "number", + "minimum": 0 + }, + "buildTimeoutHours": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "freeDiskBytes", + "slotMemoryGb", + "slotCpu", + "fileDescriptorLimit", + "buildTimeoutHours" + ], + "additionalProperties": false }, - "message": { - "type": "string" + "blockers": { + "type": "array", + "items": { + "type": "string" + } }, - "code": { - "type": "string" + "warnings": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ - "statusCode", - "error", - "message" + "ok", + "policy", + "estimate", + "capacity", + "blockers", + "warnings" ], "additionalProperties": false } - ] + }, + "required": [ + "preflightId", + "expiresAt", + "fingerprint", + "countries", + "allowedCountries", + "result" + ], + "additionalProperties": false } } } }, - "401": { + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "preflight_stale" + ] + } + }, + "required": [ + "error", + "code" + ], + "additionalProperties": false + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/transit/sync": { + "post": { + "operationId": "syncTransit", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "maxLength": 200 + }, + "triggeredBy": { + "type": "string", + "maxLength": 200 + }, + "preflightId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "countries": { + "maxItems": 250, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,31}$" + } + } + }, + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "syncTransit", + "x-openmapx-auth": "service", + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "jobId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "started" + ] + } + }, + "required": [ + "ok", + "jobId", + "status" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { "description": "Response", "content": { "application/json": { @@ -42360,6 +43249,25 @@ "existingJobId" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "code": { + "type": "string", + "enum": [ + "preflight_stale" + ] + } + }, + "required": [ + "error", + "code" + ], + "additionalProperties": false } ] } diff --git a/services/data-manager/src/api.ts b/services/data-manager/src/api.ts index b536b9b64..bf097e621 100644 --- a/services/data-manager/src/api.ts +++ b/services/data-manager/src/api.ts @@ -38,6 +38,7 @@ import { } from "./jobs/search-index/operation-lock.js"; import { createSearchIndexRuntimeState, getSearchIndexStatus } from "./jobs/search-index/state.js"; import type { BakePredictedResult } from "./jobs/traffic/bake-predicted.js"; +import { createTransitPreflightController } from "./jobs/transitous/admin-preflight.js"; import { CatalogBumpError, candidateMatchesLock, @@ -49,6 +50,7 @@ import { type MotisOperationsPolicy, publicOperationsPolicy, resolveOperationsProfileFromEnv, + transitAllowedCountries, } from "./jobs/transitous/operations-profile.js"; import { getOperatorFeedRelayStore, @@ -417,6 +419,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { resolveOperationsProfileFromEnv(process.env, { allowEmptyRegional: true }); const catalogDir = join(dataDir, ".transitous-catalog"); const overlayPath = resolveTransitOverlayPath(dataDir); + const transitPreflight = createTransitPreflightController({ dataDir, policy: operationsPolicy }); const operatorFeedRelay = opts.operatorFeedRelay ?? getOperatorFeedRelayStore(); if (!opts.operatorFeedRelay) { operatorFeedRelay.setAuditSink((event) => { @@ -511,7 +514,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { }; const input = normalizeDataExecutionInput({ kind: "transit-sync", - countries: operationsPolicy.countries, + countries: transitAllowedCountries(operationsPolicy), sourceMutation: options.mutation, }); const start = await singleFlight.tryStartSync({ @@ -523,7 +526,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { source: "source-mutation", action: options.action, sourceId: options.sourceId, - countries: operationsPolicy.countries, + countries: transitAllowedCountries(operationsPolicy), }, }); if (!start.ok) { @@ -617,7 +620,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { dataDir, catalogDir, overlayPath, - countries: operationsPolicy.countries, + countries: transitAllowedCountries(operationsPolicy), feedStates, query: { search: req.query.search, @@ -635,7 +638,9 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { registerDataManagerRoute(app, "getTransitCatalog", async (_req, reply) => { try { - return { sources: listPinnedTransitCatalog(catalogDir, operationsPolicy.countries) }; + return { + sources: listPinnedTransitCatalog(catalogDir, transitAllowedCountries(operationsPolicy)), + }; } catch (error) { const status = error instanceof TransitSourceError ? error.statusCode : 500; return reply.code(status).send({ ok: false, error: (error as Error).message }); @@ -648,7 +653,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { mutation = prepareAddTransitSource({ catalogDir, overlayPath, - countries: operationsPolicy.countries, + countries: transitAllowedCountries(operationsPolicy), expectedSourceRevision: req.body.expectedSourceRevision, source: { spec: "gtfs", @@ -696,7 +701,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { mutation = prepareRemoveTransitSource({ catalogDir, overlayPath, - countries: operationsPolicy.countries, + countries: transitAllowedCountries(operationsPolicy), expectedSourceRevision: req.body.expectedSourceRevision, sourceId: req.params.sourceId, }); @@ -731,7 +736,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { mutation = prepareEnableTransitSource({ catalogDir, overlayPath, - countries: operationsPolicy.countries, + countries: transitAllowedCountries(operationsPolicy), expectedSourceRevision: req.body.expectedSourceRevision, sourceId: req.params.sourceId, }); @@ -766,7 +771,7 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { mutation = prepareSetTransitSourceEnabled({ catalogDir, overlayPath, - countries: operationsPolicy.countries, + countries: transitAllowedCountries(operationsPolicy), expectedSourceRevision: req.body.expectedSourceRevision, sourceId: req.params.sourceId, enabled: false, @@ -980,12 +985,60 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { // POST /transit/sync — fire-and-forget Transitous pipeline trigger. Honours // the single-flight lock + idempotency key. apps/api proxies user-facing // sync requests here. + registerDataManagerRoute(app, "inspectTransitSyncPreflight", async (req, reply) => { + try { + return await transitPreflight.inspect(req.body.countries); + } catch (error) { + const e = error as Error & { statusCode?: number; code?: string }, + known = e.code === "preflight_stale", + status = known ? (e.statusCode ?? 409) : 503; + return reply.code(status).send({ + error: known + ? e.message + : "Transit preview inputs or capacity are unavailable. Check instance status.", + ...(status === 409 ? { code: "preflight_stale" } : {}), + }); + } + }); registerDataManagerRoute(app, "syncTransit", async (req, reply) => { try { + const countries = req.body?.countries ?? transitAllowedCountries(operationsPolicy); + const reviewed = req.body?.preflightId || req.body?.fingerprint; + if (req.body?.idempotencyKey && reviewed) { + const original = await opts.executionReceiver?.observeReviewedTransit?.({ + key: req.body.idempotencyKey, + countries, + preflightId: req.body.preflightId ?? "", + fingerprint: req.body.fingerprint ?? "", + triggeredBy: req.body.triggeredBy, + }); + if (original) + return reply.code(202).send({ ok: true, jobId: original.id, status: "started" }); + } + let preflight: + | import("./jobs/transitous/admin-preflight.js").TransitPreflightBinding + | undefined; + if (reviewed) { + try { + preflight = await transitPreflight.verify({ + countries, + preflightId: req.body?.preflightId ?? "", + fingerprint: req.body?.fingerprint ?? "", + }); + } catch (error) { + const e = error as Error & { code?: string }; + return e.code === "preflight_stale" + ? reply.code(409).send({ error: e.message, code: "preflight_stale" }) + : reply.code(503).send({ + error: "Transit preview is unavailable. Check instance status before retrying.", + }); + } + } const admitted = await admitLegacyData( { kind: "transit-sync", - countries: req.body?.countries ?? operationsPolicy.countries, + countries, + ...(preflight ? { preflight } : {}), }, { triggeredBy: req.body?.triggeredBy, legacyTransitKey: req.body?.idempotencyKey }, ); diff --git a/services/data-manager/src/jobs/execution/api.ts b/services/data-manager/src/jobs/execution/api.ts index 6f3704f13..5f5329ee1 100644 --- a/services/data-manager/src/jobs/execution/api.ts +++ b/services/data-manager/src/jobs/execution/api.ts @@ -14,6 +14,13 @@ import { registerDataManagerRoute } from "../../route-contract.js"; /** The receiver performs signed admission/adoption, separately from bearer * authentication. A missing receiver never authorizes a compatibility worker. */ export interface DataExecutionReceiver { + observeReviewedTransit?(request: { + key: string; + countries: string[]; + preflightId: string; + fingerprint: string; + triggeredBy?: string; + }): Promise; admitLocal( input: DataExecutionInput, options?: { sourceJobId?: string; triggeredBy?: string; legacyTransitKey?: string }, diff --git a/services/data-manager/src/jobs/execution/dispatch.ts b/services/data-manager/src/jobs/execution/dispatch.ts index 9b7e8cab1..7bef24bc7 100644 --- a/services/data-manager/src/jobs/execution/dispatch.ts +++ b/services/data-manager/src/jobs/execution/dispatch.ts @@ -39,7 +39,11 @@ import { fingerprintDataset, updateCurrentSearchIndexFingerprint, } from "../search-index/state.js"; -import type { MotisOperationsPolicy } from "../transitous/operations-profile.js"; +import { verifyTransitPreflightBinding } from "../transitous/admin-preflight.js"; +import { + type MotisOperationsPolicy, + transitAllowedCountries, +} from "../transitous/operations-profile.js"; import { getOperatorFeedRelayStore } from "../transitous/operator-feed-relay.js"; import { makePersistingOnStageComplete } from "../transitous/persistence.js"; import { buildJobContext, runTransitousPipeline } from "../transitous/pipeline.js"; @@ -148,9 +152,17 @@ export function createDataExecutionDispatcher(deps: { return { kind: input.kind, region: input.region, sizeBytes: result.sizeBytes }; } case "transit-sync": { + if (input.preflight) + await verifyTransitPreflightBinding( + { dataDir: deps.dataDir, policy: deps.operationsPolicy }, + input.countries, + input.preflight, + ); if ( deps.operationsPolicy.profile !== "planet" && - input.countries.some((country) => !deps.operationsPolicy.countries.includes(country)) + input.countries.some( + (country) => !transitAllowedCountries(deps.operationsPolicy).includes(country), + ) ) throw Error("Countries outside configured operations profile"); if (!deps.singleFlight.reserveOwned(context.execution.executionId)) diff --git a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts index 69eecf8d9..9822f30be 100644 --- a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts +++ b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts @@ -16,6 +16,7 @@ import { reconcileLegacyCatalogPreflightRejections } from "./legacy-mutation"; const enabled = Boolean(process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL) || process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"; +let repository: string; let root: string, installation: control.ControlInstallation, fence: ReturnType; @@ -26,8 +27,9 @@ beforeAll(async () => { process.env.DATABASE_URL !== process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL ) throw Error("Explicit production adapter must use the verified owned synthetic DB"); - root = mkdtempSync(join(tmpdir(), "omx-fix1-legacy-")); - mkdirSync(join(root, "data"), { mode: 0o700 }); + repository = mkdtempSync(join(tmpdir(), "omx-fix1-legacy-")); + root = join(repository, "infra/docker"); + mkdirSync(join(root, "data"), { recursive: true, mode: 0o700 }); await control.provisionControlInstallation(root); installation = control.openControlInstallation({ infraDir: root, @@ -42,7 +44,7 @@ afterAll(async () => { if (!enabled) return; control.closeControlInstallation(installation); await rawSql.end(); - rmSync(root, { recursive: true, force: true }); + rmSync(repository, { recursive: true, force: true }); }); it.skipIf(!enabled)( "I4 records actual missing-catalog preflight as terminal error and permits the first subsequent admission", @@ -240,7 +242,8 @@ import { createDataExecutionReceiver } from "./receiver"; import { createDataExecutionRuntime } from "./runtime"; import { createPostgresDataExecutionStore } from "./store"; -async function legacyFixture(held = false) { +async function legacyFixture(held = false, realDispatch = false) { + let workerError: unknown; const journal = control.createDataExecutionControlJournal(installation, rawSql); const store = createPostgresDataExecutionStore(rawSql, { controlJournal: journal, @@ -253,9 +256,17 @@ async function legacyFixture(held = false) { }); const runtime = createDataExecutionRuntime({ store, - dispatch: async (input) => { + dispatch: async (input, context) => { dispatches++; if (held) await gate; + if (realDispatch) { + try { + return await dispatcher.dispatch(input, context); + } catch (error) { + workerError = error; + throw error; + } + } if (input.kind !== "transit-sync") throw Error("This admission fixture selects transit only"); return { kind: input.kind, @@ -303,6 +314,7 @@ async function legacyFixture(held = false) { store, release, dispatches: () => dispatches, + workerError: () => workerError, async close() { release(); await runtime.close(); @@ -536,3 +548,191 @@ it.skipIf(!enabled)( }, 45000, ); +it.skipIf(!enabled)( + "observes the original reviewed transit intent after loss of the API preview cache without another dispatch", + async () => { + const f = await legacyFixture(true), + key = `reviewed-observe-${randomUUID()}`; + try { + const preflight = { + preflightId: randomUUID(), + fingerprint: "a".repeat(64), + sourceFingerprint: "b".repeat(64), + expiresAt: new Date(Date.now() + 300000).toISOString(), + }; + // This held-worker fixture proves SQL identity/observation, not production + // capacity validation or downloads. The real dispatcher is checked separately. + const admitted = await f.receiver.admitLocal( + { kind: "transit-sync", countries: ["de"], preflight }, + { legacyTransitKey: key, triggeredBy: "fixture-reviewer" }, + ); + const body = { + countries: ["DE"], + idempotencyKey: key, + preflightId: preflight.preflightId, + fingerprint: preflight.fingerprint, + triggeredBy: "fixture-reviewer", + }; + const retry = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: body, + }); + expect(retry.statusCode).toBe(202); + expect(retry.json().jobId).toBe(admitted.id); + const [row] = + await rawSql`SELECT count(*)::int AS count FROM data_manager.jobs WHERE idempotency_key=${key}`; + expect(row?.count).toBe(1); + const wrong = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { ...body, fingerprint: "c".repeat(64) }, + }); + expect(wrong.statusCode).toBe(409); + const actor = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { ...body, triggeredBy: "another-reviewer" }, + }); + expect(actor.statusCode).toBe(409); + expect(f.dispatches()).toBeLessThanOrEqual(1); + } finally { + await f.close(); + } + }, + 30000, +); +it.skipIf(!enabled)( + "rechecks reviewed sources in the actual durable dispatcher before any pipeline effects", + async () => { + const { initializeManagedCatalogPins } = await import("@openmapx/core/catalog-pins"); + const { buildOpsAgentServer } = await import("../../../../../apps/ops-agent/src/server"); + const { createUnavailableRuntime } = await import("../../../../../apps/ops-agent/src/runtime"); + const { createTransitousLockRuntime } = await import( + "../../../../../apps/ops-agent/src/transitous-lock-runtime" + ); + const { createRegistryResourcePolicy, denyAllTrustedOpsData } = await import( + "../../../../../apps/ops-agent/src/policy" + ); + const agent = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + const f = await legacyFixture(true, true), + envNames = [ + "MOTIS_SLOT_MEMORY_GB", + "MOTIS_SLOT_CPU", + "MOTIS_FILE_DESCRIPTOR_LIMIT", + "MOTIS_BUILD_TIMEOUT_HOURS", + "TRANSITOUS_FEEDS_OVERLAY_PATH", + ] as const, + old = envNames.map((name) => process.env[name]); + const token = Buffer.alloc(32, 5).toString("base64url"); + const server = buildOpsAgentServer({ + tokens: { api: Buffer.alloc(32, 6).toString("base64url"), "data-manager": token }, + controlInstallation: agent, + runtime: createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: repository }), + resourcePolicy: createRegistryResourcePolicy({ + serviceIds: ["motis"], + integrationIds: [], + trustedData: denyAllTrustedOpsData, + }), + }); + try { + writeFileSync( + join(root, "transitous.lock.json"), + JSON.stringify({ + ref: `main@${"a".repeat(40)}`, + submodules: {}, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + const commit = "b".repeat(40); + writeFileSync( + join(root, "gbfs-catalog.lock.json"), + JSON.stringify({ + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit, + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${commit}/systems.csv`, + sha256: "c".repeat(64), + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + await initializeManagedCatalogPins({ rootDir: repository, installation: agent }); + const baseUrl = await server.listen({ host: "127.0.0.1", port: 0 }), + tokenFile = join(root, "preview-token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + resetOpsClient(new OpsClient({ baseUrl, tokenFile })); + for (const [name, value] of Object.entries({ + MOTIS_SLOT_MEMORY_GB: "16", + MOTIS_SLOT_CPU: "4", + MOTIS_FILE_DESCRIPTOR_LIMIT: "65536", + MOTIS_BUILD_TIMEOUT_HOURS: "12", + TRANSITOUS_FEEDS_OVERLAY_PATH: join(root, "preview-overlay.json"), + })) + process.env[name] = value; + const catalog = join(root, "data/.transitous-catalog"); + mkdirSync(join(catalog, "feeds"), { recursive: true }); + mkdirSync(join(catalog, ".git"), { recursive: true }); + writeFileSync( + join(catalog, "feeds/de.json"), + JSON.stringify({ + sources: [ + { name: "demo", spec: "gtfs", type: "http", url: "https://example.test/demo.zip" }, + ], + }), + ); + const preview = await f.app.inject({ + method: "POST", + url: "/transit/sync/preflight", + headers, + payload: { countries: ["de"] }, + }); + expect(preview.statusCode, preview.body).toBe(200); + expect(preview.json().result.ok, preview.body).toBe(true); + const reviewed = preview.json(), + response = await f.app.inject({ + method: "POST", + url: "/transit/sync", + headers, + payload: { + countries: ["de"], + preflightId: reviewed.preflightId, + fingerprint: reviewed.fingerprint, + idempotencyKey: `reviewed-worker-${randomUUID()}`, + }, + }); + expect(response.statusCode, response.body).toBe(202); + const id = response.json().jobId; + writeFileSync(join(catalog, "feeds/de.json"), JSON.stringify({ sources: [] })); + f.release(); + await expect + .poll(async () => (await f.runtime.inspect(id))?.status, { timeout: 15000 }) + .toBe("error"); + expect(f.workerError()).toMatchObject({ code: "preflight_stale" }); + expect(f.dispatches()).toBe(1); + expect((await f.runtime.inspect(id))?.publication).toBe("not_started"); + expect( + (await import("node:fs")).existsSync(join(root, "data/.admin-target-control/locks/motis")), + ).toBe(false); + } finally { + f.release(); + await f.close(); + await server.close(); + control.closeControlInstallation(agent); + resetOpsClient(); + envNames.forEach((name, n) => { + if (old[n] === undefined) delete process.env[name]; + else process.env[name] = old[n]; + }); + } + }, + 45000, +); diff --git a/services/data-manager/src/jobs/execution/receiver.ts b/services/data-manager/src/jobs/execution/receiver.ts index a96917f9b..26c9ae43f 100644 --- a/services/data-manager/src/jobs/execution/receiver.ts +++ b/services/data-manager/src/jobs/execution/receiver.ts @@ -8,14 +8,21 @@ import { normalizeDataExecutionInput, startGenerationHardlinksExecutionSchema, } from "@openmapx/core/data-manager-contracts"; -import { createControlRootOperationKey, opsExecutionBudget } from "@openmapx/core/ops"; -import type { MotisOperationsPolicy } from "../transitous/operations-profile.js"; +import { + canonicalJson, + createControlRootOperationKey, + opsExecutionBudget, +} from "@openmapx/core/ops"; +import { + type MotisOperationsPolicy, + transitAllowedCountries, +} from "../transitous/operations-profile.js"; import type { DataExecutionReceiver } from "./api.js"; import type { DataConfigurationMaintenance } from "./configuration-maintenance.js"; import type { createDataExecutionDispatcher } from "./dispatch.js"; import { transitChildCandidateId } from "./ops-children.js"; import type { DataExecutionRuntime } from "./runtime.js"; -import { dataExecutionInputFingerprint } from "./store.js"; +import { DataExecutionConflict, dataExecutionInputFingerprint } from "./store.js"; export function createDataExecutionReceiver(deps: { runtime: DataExecutionRuntime; installation: ControlInstallation; @@ -30,7 +37,9 @@ export function createDataExecutionReceiver(deps: { input.kind === "transit-sync" && deps.operationsPolicy.profile !== "planet" && (input.countries.length === 0 || - input.countries.some((country) => !deps.operationsPolicy.countries.includes(country))) + input.countries.some( + (country) => !transitAllowedCountries(deps.operationsPolicy).includes(country), + )) ) throw Object.assign(Error("Countries outside configured operations profile"), { statusCode: 422, @@ -38,6 +47,21 @@ export function createDataExecutionReceiver(deps: { return input; }; return { + async observeReviewedTransit(request) { + const record = await runtime.lookupLegacyTransit?.(request.key); + if (!record) return null; + const input = record.input, + countries = [...new Set(request.countries.map((c) => c.toLowerCase()))].sort(); + if ( + input.kind !== "transit-sync" || + canonicalJson(input.countries) !== canonicalJson(countries) || + record.triggeredBy !== (request.triggeredBy ?? null) || + input.preflight?.preflightId !== request.preflightId || + input.preflight.fingerprint !== request.fingerprint + ) + throw new DataExecutionConflict("Legacy transit key body or actor conflict"); + return runtime.inspect(record.status.id); + }, async admit(value, options) { const input = validate(value); if (input.kind === "generation-hardlinks" || input.kind === "transit-maintenance") diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index 94f223458..de5902fdc 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -81,6 +81,7 @@ export interface DataExecutionRuntime { ): Promise; inspect(id: string): Promise; lookup(key: string): Promise; + lookupLegacyTransit?(key: string): Promise; requestCancel(identity: DataExecutionIdentity): Promise; reconcile(): Promise; close(): Promise; @@ -483,6 +484,7 @@ export function createDataExecutionRuntime(deps: { }, inspect: async (id) => (await store.inspect(id))?.status ?? null, lookup: async (key) => (await store.lookup(key))?.status ?? null, + lookupLegacyTransit: async (key) => (await store.lookupLegacyTransit?.(key)) ?? null, requestCancel: cancel, async reconcile() { const owner = requireJournal(); diff --git a/services/data-manager/src/jobs/execution/store.ts b/services/data-manager/src/jobs/execution/store.ts index b8e652560..5ad3a4d6b 100644 --- a/services/data-manager/src/jobs/execution/store.ts +++ b/services/data-manager/src/jobs/execution/store.ts @@ -113,6 +113,7 @@ export interface DataExecutionStore { ): Promise<{ created: boolean; record: DataExecutionRecord }>; inspect(id: string): Promise; lookup(operationKey: string): Promise; + lookupLegacyTransit?(key: string): Promise; checkpoint( id: string, expectedVersion: number, @@ -567,6 +568,12 @@ export function createPostgresDataExecutionStore( throw error; }); }, + async lookupLegacyTransit(key) { + const row = await findLegacyTransit(sql, z.string().max(200).parse(key)); + if (!row) return null; + if (row.execution_version !== 2) throw new LegacyTransitDuplicate(row.id); + return decode(row); + }, inspect: (id) => find(sql, "id", id), lookup: (key) => find(sql, "operation_key", key), async checkpoint(id, version, value) { diff --git a/services/data-manager/src/jobs/transitous/admin-preflight.ts b/services/data-manager/src/jobs/transitous/admin-preflight.ts new file mode 100644 index 000000000..979907c65 --- /dev/null +++ b/services/data-manager/src/jobs/transitous/admin-preflight.ts @@ -0,0 +1,198 @@ +import { createHash, randomUUID } from "node:crypto"; +import { lstatSync, statfsSync } from "node:fs"; +import { isAbsolute, join } from "node:path"; +import { parseCatalogPinSet } from "@openmapx/core/catalog-pins"; +import type { TransitSourceRow } from "@openmapx/core/data-manager-contracts"; +import { canonicalJson } from "@openmapx/core/ops"; +import { runOpsOperation } from "../../ops-client.js"; +import { listTransitSources, transitSourceRevision } from "../../transit-sources.js"; +import { countryOfRegion, resolveTransitOverlayPath, sourceIdForFailure } from "./internal.js"; +import { type MotisOperationsPolicy, transitAllowedCountries } from "./operations-profile.js"; +import { runMotisPreflight } from "./preflight.js"; + +const hash = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex"); +const fail = (message: string, statusCode = 409) => + Object.assign(Error(message), { statusCode, code: "preflight_stale" }); +export type TransitPreflightBinding = { + preflightId: string; + fingerprint: string; + expiresAt: string; + sourceFingerprint: string; +}; +export interface TransitPreflightOptions { + dataDir: string; + policy: MotisOperationsPolicy; +} +export function normalizeTransitPreflightCountries( + countries: readonly string[], + policy: MotisOperationsPolicy, +): string[] { + const selected = [...new Set(countries.map((c) => c.toLowerCase()))].sort(); + if (!selected.length || selected.length > 256 || selected.some((c) => !/^([a-z]{2,3})$/.test(c))) + throw fail("Select one or more valid countries", 400); + const allowed = new Set(transitAllowedCountries(policy)); + if (policy.profile !== "planet" && selected.some((c) => !allowed.has(c))) + throw fail("Selected countries exceed the instance transit scope", 422); + return selected; +} +function configuredNumber(name: string) { + const value = Number(process.env[name]); + return Number.isFinite(value) && value > 0 ? value : 0; +} +export async function collectTransitPreflight( + options: TransitPreflightOptions, + countries: readonly string[], + signal?: AbortSignal, +) { + signal?.throwIfAborted(); + const selected = normalizeTransitPreflightCountries(countries, options.policy); + const catalogDir = join(options.dataDir, ".transitous-catalog"), + overlayPath = resolveTransitOverlayPath(options.dataDir); + const revision = transitSourceRevision({ catalogDir, overlayPath }); + const pins = await runOpsOperation({ kind: "catalogPins.inspect" }); + if (!pins.active) throw fail("Paired catalog pins are unavailable", 503); + const generation = parseCatalogPinSet(pins.active).generationId; + if (pins.catalogRevisionId !== generation) + throw fail("Paired catalog revision is unavailable", 503); + const sources: TransitSourceRow[] = []; + for (const country of selected) { + for (let offset = 0; ; offset += 500) { + signal?.throwIfAborted(); + const page = listTransitSources({ + dataDir: options.dataDir, + catalogDir, + overlayPath, + countries: options.policy.countries, + query: { country, limit: 500, offset }, + }); + if (page.sourceRevision !== revision) + throw fail("Transit sources changed; refresh preflight"); + sources.push(...page.sources.filter((s) => s.requested)); + if (offset + page.limit >= page.total) break; + } + } + const resourceConfig = { + slotMemoryGb: configuredNumber("MOTIS_SLOT_MEMORY_GB"), + slotCpu: configuredNumber("MOTIS_SLOT_CPU"), + fileDescriptorLimit: configuredNumber("MOTIS_FILE_DESCRIPTOR_LIMIT"), + buildTimeoutHours: configuredNumber("MOTIS_BUILD_TIMEOUT_HOURS"), + }; + let filesystem: ReturnType | undefined; + try { + filesystem = statfsSync(options.dataDir); + } catch { + /* Unknown space is zero, never a caller override. */ + } + const capacity = { + ...resourceConfig, + freeDiskBytes: filesystem ? Number(filesystem.bavail) * Number(filesystem.bsize) : 0, + ...(filesystem ? { freeInodes: Number(filesystem.ffree) } : {}), + }; + const osmInput = options.policy.osmInput; + let osmBytes: number | undefined; + if (osmInput && !isAbsolute(osmInput) && !osmInput.split(/[\\/]/).includes("..")) { + for (const dir of ["gtfs", "motis"]) { + try { + const stat = lstatSync(join(options.dataDir, dir, osmInput)); + if (stat.isFile()) { + osmBytes = stat.size; + break; + } + } catch { + /* Missing input stays unavailable. */ + } + } + } + const result = runMotisPreflight({ + policy: { ...options.policy, countries: selected }, + feedCount: sources.length, + measuredCompressedBytes: sources.reduce((total, s) => total + (s.artifact?.sizeBytes ?? 0), 0), + osmBytes, + osmAvailable: osmBytes !== undefined, + capacity, + selectedFeedIds: sources.map((s) => s.id), + selectedSources: sources.map((s) => ({ + id: sourceIdForFailure(countryOfRegion(s.region), s.name, s.region), + artifactId: `${s.region}_${s.name}`.toLowerCase(), + originUrl: s.originUrl, + license: s.license, + })), + }); + const after = await runOpsOperation({ kind: "catalogPins.inspect" }); + if ( + after.catalogRevisionId !== generation || + transitSourceRevision({ catalogDir, overlayPath }) !== revision + ) + throw fail("Transit sources or catalog pins changed; refresh preflight"); + return { + countries: selected, + allowedCountries: transitAllowedCountries(options.policy), + sourceFingerprint: hash({ + countries: selected, + generation, + revision, + policy: options.policy, + resourceConfig, + }), + result, + }; +} +/** Preview state is bounded and advisory. Restart/expiry requires a new preview; + * only normal durable execution admission and its live lease can authorize work. */ +export function createTransitPreflightController(options: TransitPreflightOptions) { + const previews = new Map< + string, + { fingerprint: string; binding: TransitPreflightBinding; countries: string[] } + >(); + return { + async inspect(countries: string[], signal?: AbortSignal) { + const snapshot = await collectTransitPreflight(options, countries, signal), + preflightId = randomUUID(), + expiresAt = new Date(Date.now() + 5 * 60_000).toISOString(); + for (const [id, preview] of previews) + if (Date.parse(preview.binding.expiresAt) <= Date.now()) previews.delete(id); + if (previews.size >= 512) + throw fail("Too many active transit previews; retry after expiry", 503); + const value = { + preflightId, + expiresAt, + countries: snapshot.countries, + allowedCountries: snapshot.allowedCountries, + result: snapshot.result, + }; + const fingerprint = hash(value), + binding = { + preflightId, + fingerprint, + expiresAt, + sourceFingerprint: snapshot.sourceFingerprint, + }; + previews.set(preflightId, { fingerprint, binding, countries: snapshot.countries }); + return { ...value, fingerprint }; + }, + async verify(request: { countries: string[]; preflightId: string; fingerprint: string }) { + const countries = normalizeTransitPreflightCountries(request.countries, options.policy), + preview = previews.get(request.preflightId); + if ( + !preview || + preview.fingerprint !== request.fingerprint || + canonicalJson(countries) !== canonicalJson(preview.countries) || + Date.parse(preview.binding.expiresAt) <= Date.now() + ) + throw fail("Transit preflight expired or changed; preview again"); + await verifyTransitPreflightBinding(options, countries, preview.binding); + return preview.binding; + }, + }; +} +export async function verifyTransitPreflightBinding( + options: TransitPreflightOptions, + countries: string[], + binding: TransitPreflightBinding, +) { + if (Date.parse(binding.expiresAt) <= Date.now()) + throw fail("Transit preflight expired; preview again"); + const current = await collectTransitPreflight(options, countries); + if (current.sourceFingerprint !== binding.sourceFingerprint || !current.result.ok) + throw fail("Transit inputs or available capacity changed; preview again"); +} diff --git a/services/data-manager/src/jobs/transitous/operations-profile.ts b/services/data-manager/src/jobs/transitous/operations-profile.ts index 2d7bc3fb4..aaa12b5e7 100644 --- a/services/data-manager/src/jobs/transitous/operations-profile.ts +++ b/services/data-manager/src/jobs/transitous/operations-profile.ts @@ -166,6 +166,20 @@ export function resolveOperationsProfileFromEnv( }); } +/** Effective country scope for a finite legacy feed allow-list. This does + * not select feeds: the preflight independently checks each exact feed ID. */ +export function transitAllowedCountries(policy: MotisOperationsPolicy): string[] { + if (policy.countries.length) + return [...new Set(policy.countries.map((c) => c.toLowerCase()))].sort(); + return [ + ...new Set( + policy.feedAllowList.flatMap((id) => { + const country = /^([a-z]{2,3})_/.exec(id.toLowerCase())?.[1]; + return country ? [country] : []; + }), + ), + ].sort(); +} export function publicOperationsPolicy(policy: MotisOperationsPolicy): MotisOperationsPolicy { - return structuredClone(policy); + return { ...structuredClone(policy), countries: transitAllowedCountries(policy) }; } diff --git a/services/data-manager/src/jobs/transitous/preflight.ts b/services/data-manager/src/jobs/transitous/preflight.ts index 8887c1adc..8c4463e8a 100644 --- a/services/data-manager/src/jobs/transitous/preflight.ts +++ b/services/data-manager/src/jobs/transitous/preflight.ts @@ -48,6 +48,7 @@ export interface RunMotisPreflightInput { sourceUrls?: string[]; selectedSources?: Array<{ id: string; + artifactId?: string; originUrl?: string; license?: Record; }>; @@ -107,7 +108,17 @@ export function runMotisPreflight(input: RunMotisPreflightInput): MotisPreflight } if (input.policy.feedAllowList.length > 0) { const allowed = new Set(input.policy.feedAllowList); - const outside = (input.selectedFeedIds ?? []).filter((id) => !allowed.has(id.toLowerCase())); + const captured = input.selectedSources ?? []; + const outside = + captured.length && captured.every((source) => source.artifactId) + ? captured + .filter( + (source) => + !allowed.has(source.id.toLowerCase()) && + !allowed.has(source.artifactId!.toLowerCase()), + ) + .map((source) => source.id) + : (input.selectedFeedIds ?? []).filter((id) => !allowed.has(id.toLowerCase())); if (outside.length > 0) blockers.push(`feeds outside allow-list: ${outside.sort().join(", ")}`); } if (input.policy.profile === "regional-sovereign") { @@ -230,6 +241,7 @@ export const run: StageFn = async (ctx) => { selectedSources: (ctx.state.selectedFeedFiles ?? []).flatMap((feed) => feed.activeScheduleSources.map((source) => ({ id: source.id, + artifactId: `${source.region}_${source.name}`.toLowerCase(), originUrl: source.originUrl, license: source.license, })), diff --git a/services/data-manager/src/service.ts b/services/data-manager/src/service.ts index 1bbc5e8a1..5364f75a5 100644 --- a/services/data-manager/src/service.ts +++ b/services/data-manager/src/service.ts @@ -30,7 +30,10 @@ import { reconcileOrphanedJobs } from "./jobs/reconcile.js"; import { createNotablePlacesOperationLock } from "./jobs/search-index/operation-lock.js"; import { bakePredicted } from "./jobs/traffic/bake-predicted.js"; import { fetchCoveredWayIds } from "./jobs/traffic/covered-ways.js"; -import { resolveOperationsProfileFromEnv } from "./jobs/transitous/operations-profile.js"; +import { + resolveOperationsProfileFromEnv, + transitAllowedCountries, +} from "./jobs/transitous/operations-profile.js"; import { getSingleFlightController } from "./jobs/transitous/runtime.js"; import { rootLogger } from "./logger.js"; import { OfflinePackageGenerator } from "./offline-packages/generator.js"; @@ -196,7 +199,7 @@ async function start(): Promise { "Data mutation admission remains closed pending retained ownership recovery", ); } - const countries = operationsPolicy.countries; + const countries = transitAllowedCountries(operationsPolicy); const initialized = await initializeRequiredSubsystems({ readiness, initializeOfflineStorage: async () => { diff --git a/services/data-manager/src/transit-sources.ts b/services/data-manager/src/transit-sources.ts index 22bb25c16..c5d91f814 100644 --- a/services/data-manager/src/transit-sources.ts +++ b/services/data-manager/src/transit-sources.ts @@ -344,6 +344,8 @@ export function listTransitSources(options: { }; }); const query = options.query ?? {}; + const country = query.country?.toLowerCase(); + if (country) rows = rows.filter((row) => countryOfRegion(row.region) === country); if (query.origin) rows = rows.filter((source) => source.origin === query.origin); if (query.lifecycle) rows = rows.filter((source) => source.lifecycle === query.lifecycle); if (query.search?.trim()) { @@ -354,8 +356,9 @@ export function listTransitSources(options: { ), ); } + rows.sort((a, b) => a.name.localeCompare(b.name) || a.id.localeCompare(b.id)); const total = rows.length; - const limit = Math.min(Math.max(Math.floor(query.limit ?? 100), 1), 500); + const limit = Math.min(Math.max(Math.floor(query.limit ?? 50), 1), 500); const offset = Math.max(Math.floor(query.offset ?? 0), 0); return { sourceRevision, sources: rows.slice(offset, offset + limit), total, limit, offset }; } From 783555f646bdddd8c8a55ef8cd9349b7f54442a0 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 28 Oct 2026 23:06:29 +0100 Subject: [PATCH 051/135] feat(config): retain canonical platform ownership across renders --- .../routes/__tests__/admin-compose.test.ts | 145 +++++++++++------- apps/api/src/routes/admin-compose.ts | 11 +- .../prospective-configuration.test.ts | 13 ++ apps/api/src/services/configuration-input.ts | 15 +- .../src/services/prospective-configuration.ts | 20 +-- apps/ops-agent/src/administrative-runtime.ts | 17 +- .../src/data-workflow-runtime.test.ts | 27 ++++ apps/ops-agent/src/data-workflow-runtime.ts | 18 ++- apps/ops-agent/src/server.ts | 6 +- .../administration/services-administration.md | 15 ++ packages/cli/__tests__/cli-compose.test.ts | 69 +++++++++ packages/cli/src/commands/compose.ts | 17 +- packages/cli/src/commands/data.ts | 4 +- packages/cli/src/commands/services.ts | 8 +- packages/cli/src/lib/env-defaults.ts | 16 +- packages/cli/src/lib/service-lifecycle.ts | 1 - .../configuration-source-witness.test.ts | 44 ++++++ .../configuration-source-witness.ts | 13 +- .../core/src/build-artifacts/data-update.ts | 22 ++- .../ops/configuration-preparation-records.ts | 2 + packages/core/src/ops/contract.ts | 2 + packages/core/src/ops/trusted-config.ts | 2 + .../configuration-generation.test.ts | 38 +++++ .../__tests__/platform-configuration.test.ts | 79 ++++++++++ .../core/src/services/compose-renderer.ts | 24 +++ .../src/services/configuration-generation.ts | 36 +++++ packages/core/src/services/index.ts | 8 + .../src/services/platform-configuration.ts | 98 ++++++++++++ .../core/src/services/release-manifest.ts | 72 +++------ .../core/src/services/release-selection.ts | 54 +++++++ 30 files changed, 740 insertions(+), 156 deletions(-) create mode 100644 packages/core/src/services/__tests__/platform-configuration.test.ts create mode 100644 packages/core/src/services/platform-configuration.ts create mode 100644 packages/core/src/services/release-selection.ts diff --git a/apps/api/src/routes/__tests__/admin-compose.test.ts b/apps/api/src/routes/__tests__/admin-compose.test.ts index 7b21ed520..06aa0a90f 100644 --- a/apps/api/src/routes/__tests__/admin-compose.test.ts +++ b/apps/api/src/routes/__tests__/admin-compose.test.ts @@ -1,30 +1,31 @@ import Fastify from "fastify"; import { beforeEach, describe, expect, it, vi } from "vitest"; -const composeAction = vi.fn(); -const applyGenerationHardlinks = vi - .fn() - .mockResolvedValue({ applied: true, linked: 0, skipped: 0, pruned: 0, entries: 0 }); -const REVISION = `cfg1_${"r".repeat(43)}`; -const renderAndPersistCompose = vi.fn().mockResolvedValue({ revisionId: REVISION }); +const f = vi.hoisted(() => ({ start: vi.fn(), inspect: vi.fn(), build: vi.fn(), render: vi.fn() })); const adminA = { user: { id: "admin-a" } }; - vi.mock("@openmapx/core/server", () => ({ repoPaths: () => ({ infraDir: "/trusted/infra" }), - services: { renderConfiguration: vi.fn() }, + services: { renderConfiguration: (...args: unknown[]) => f.render(...args) }, +})); +vi.mock("../../services/job-runner", () => ({ + jobRunner: { startImmediateCompose: (...args: unknown[]) => f.start(...args) }, +})); +vi.mock("../../services/ops-client", () => ({ + createApiOpsClient: () => ({}), + createDurableOpsKey: () => `opk1_${"p".repeat(43)}`, + executeAndWait: (...args: unknown[]) => f.inspect(...args), })); -vi.mock("../../services/admin-ops", () => ({ - applyGenerationHardlinks: (...args: unknown[]) => applyGenerationHardlinks(...args), - renderAndPersistCompose: (...args: unknown[]) => renderAndPersistCompose(...args), +vi.mock("../../services/desired-selection", () => ({ + readDesiredSelection: async () => ({ roots: ["alpha"], source: "file" }), })); -vi.mock("../../services/desired-selection", () => ({ readDesiredSelection: vi.fn() })); vi.mock("../../services/trusted-config-operations", () => ({ - buildConfigurationInput: vi.fn(), - integrationSchemas: vi.fn(), + buildConfigurationInput: (...args: unknown[]) => f.build(...args), + integrationSchemas: () => new Map(), +})); +vi.mock("../../services/service-registry", () => ({ + getServiceRegistry: () => ({ list: () => [] }), })); -vi.mock("../../services/service-registry", () => ({ getServiceRegistry: vi.fn() })); vi.mock("../../utils/docker-compose", () => ({ - dockerComposeAction: (...args: unknown[]) => composeAction(...args), STACK_STOP_GUIDANCE: "Stack shutdown is unavailable from the web API because it would stop the operations agent. Run the documented host shutdown command instead.", })); @@ -34,49 +35,81 @@ vi.mock("../../utils/require-admin", () => ({ })); vi.mock("../../utils/route-auth", () => ({ declareRouteAuth: vi.fn() })); +beforeEach(() => { + vi.clearAllMocks(); + f.start.mockResolvedValue({ + ok: true, + stdout: "", + hardlinks: { applied: true, linked: 0, skipped: 0, pruned: 0, entries: 0 }, + }); +}); +async function app() { + const a = Fastify({ logger: false }); + const { registerAdminComposeRoutes } = await import("../admin-compose.js"); + await a.register(registerAdminComposeRoutes, { prefix: "/api" }); + return a; +} describe("admin Compose routes", () => { - beforeEach(() => { - composeAction.mockClear(); - applyGenerationHardlinks.mockClear(); - renderAndPersistCompose.mockClear(); + it("requires caller-retained idempotency and retains the original job intent across an ambiguous retry", async () => { + const a = await app(); + try { + expect((await a.inject({ method: "POST", url: "/api/admin/compose/up" })).statusCode).toBe( + 400, + ); + expect(f.start).not.toHaveBeenCalled(); + const headers = { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }; + f.start.mockRejectedValueOnce(Error("synthetic response loss")); + expect( + (await a.inject({ method: "POST", url: "/api/admin/compose/up", headers })).statusCode, + ).toBe(500); + expect( + (await a.inject({ method: "POST", url: "/api/admin/compose/up", headers })).statusCode, + ).toBe(200); + const original = f.start.mock.calls[0]?.[0]; + expect(original).toMatchObject({ + actorId: "admin-a", + operationKey: expect.stringMatching(/^opk1_[A-Za-z0-9_-]{43}$/), + }); + expect(f.start.mock.calls[1]?.[0]).toEqual(original); + } finally { + await a.close(); + } }); - it("requires caller-retained idempotency and reuses the same agent key across an ambiguous retry", async () => { - composeAction.mockResolvedValue({ exitCode: 0, stdout: "", stderr: "" }); - const { registerAdminComposeRoutes } = await import("../admin-compose.js"); - const app = Fastify({ logger: false }); - await app.register(registerAdminComposeRoutes, { prefix: "/api" }); - - const missing = await app.inject({ method: "POST", url: "/api/admin/compose/up" }); - expect(missing.statusCode).toBe(400); - expect(composeAction).not.toHaveBeenCalled(); - - const headers = { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }; - await app.inject({ method: "POST", url: "/api/admin/compose/up", headers }); - await app.inject({ method: "POST", url: "/api/admin/compose/up", headers }); - const firstKey = composeAction.mock.calls[0]?.[2]?.operationKey; - expect(firstKey).toMatch(/^opk1_[A-Za-z0-9_-]{43}$/); - expect(composeAction.mock.calls[1]?.[2]?.operationKey).toBe(firstKey); - expect(applyGenerationHardlinks.mock.calls[0]?.[0]?.operationIdentity).toBe( - applyGenerationHardlinks.mock.calls[1]?.[0]?.operationIdentity, - ); - // The links follow the generation this request just applied. - expect(applyGenerationHardlinks.mock.calls[0]?.[0]?.revisionId).toBe(REVISION); - await app.close(); + it("refuses stack shutdown and never queues a job", async () => { + const a = await app(); + try { + const response = await a.inject({ method: "POST", url: "/api/admin/compose/down" }); + expect(response.statusCode).toBe(503); + expect(response.json()).toMatchObject({ + ok: false, + error: expect.stringContaining("host shutdown"), + }); + expect(f.start).not.toHaveBeenCalled(); + } finally { + await a.close(); + } }); - - it("refuses stack shutdown with a non-success status and never submits stack.stop", async () => { - const { registerAdminComposeRoutes } = await import("../admin-compose.js"); - const app = Fastify({ logger: false }); - await app.register(registerAdminComposeRoutes, { prefix: "/api" }); - const response = await app.inject({ method: "POST", url: "/api/admin/compose/down" }); - - expect(response.statusCode).toBe(503); - expect(response.json()).toEqual({ - ok: false, - error: - "Stack shutdown is unavailable from the web API because it would stop the operations agent. Run the documented host shutdown command instead.", + it("uses read-only platform inspection for preview without allocating a source witness", async () => { + const platform = { version: 1, domain: { source: "database", value: "adopted.example.test" } }; + f.inspect.mockResolvedValue({ revisionId: null, integrationConfigDigests: [], platform }); + f.build.mockResolvedValue({ + domain: "adopted.example.test", + platform, + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], }); - expect(composeAction).not.toHaveBeenCalled(); - await app.close(); + f.render.mockReturnValue({ rendered: { composeYaml: "services: {}\n" } }); + const a = await app(); + try { + const response = await a.inject("/api/admin/compose/preview"); + expect(response.statusCode).toBe(200); + expect(response.body).toBe("services: {}\n"); + expect(f.inspect.mock.calls[0]?.[1]).toEqual({ kind: "configuration.inspect" }); + expect(f.build).toHaveBeenCalledWith(undefined, platform); + expect(f.start).not.toHaveBeenCalled(); + } finally { + await a.close(); + } }); }); diff --git a/apps/api/src/routes/admin-compose.ts b/apps/api/src/routes/admin-compose.ts index 7136616dd..054d955d3 100644 --- a/apps/api/src/routes/admin-compose.ts +++ b/apps/api/src/routes/admin-compose.ts @@ -7,6 +7,7 @@ import { parseDirectOpsIdempotency, } from "../services/direct-ops-idempotency"; import { jobRunner } from "../services/job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client"; import { getServiceRegistry } from "../services/service-registry"; import { buildConfigurationInput, integrationSchemas } from "../services/trusted-config-operations"; import { STACK_STOP_GUIDANCE } from "../utils/docker-compose"; @@ -33,11 +34,19 @@ export async function registerAdminComposeRoutes( return { error: "Service registry not available" }; } const desired = await readDesiredSelection(); + const source = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("configuration.inspect", "compose-preview"), + ); const { rendered } = services.renderConfiguration({ infraDir: repoPaths().infraDir, services: registry.list(), integrationSchemas: integrationSchemas(), - input: { ...(await buildConfigurationInput()), selectedRoots: desired.roots }, + input: { + ...(await buildConfigurationInput(undefined, source.platform)), + selectedRoots: desired.roots, + }, allowMissingSelected: desired.source === "default", }); reply.header("Content-Type", "text/yaml; charset=utf-8"); diff --git a/apps/api/src/services/__tests__/prospective-configuration.test.ts b/apps/api/src/services/__tests__/prospective-configuration.test.ts index ff82bb052..abd8b366f 100644 --- a/apps/api/src/services/__tests__/prospective-configuration.test.ts +++ b/apps/api/src/services/__tests__/prospective-configuration.test.ts @@ -104,3 +104,16 @@ it("rejects a foreign still-live read context inside another coherent read", asy }); }); }); + +it("preserves the agent-observed platform projection when collecting unrelated desired configuration", async () => { + const platform = { + version: 1 as const, + domain: { source: "database" as const, value: "adopted.example.test" }, + backupRetentionDays: { source: "database" as const, value: 45 }, + }; + vi.stubEnv("DOMAIN", "host.example.test"); + const payload = await buildConfigurationInput(undefined, platform); + expect(payload.domain).toBe("adopted.example.test"); + expect(payload.platform).toEqual(platform); + expect(payload.serviceConfigs).toHaveLength(3); +}); diff --git a/apps/api/src/services/configuration-input.ts b/apps/api/src/services/configuration-input.ts index 3b35b8f4b..6f2b56009 100644 --- a/apps/api/src/services/configuration-input.ts +++ b/apps/api/src/services/configuration-input.ts @@ -3,7 +3,11 @@ import { type TrustedConfigurationPayload, trustedConfigurationPayloadSchema, } from "@openmapx/core/ops"; -import { envString } from "@openmapx/core/server-env"; +import { services as coreServices } from "@openmapx/core/server"; + +type PlatformConfiguration = coreServices.PlatformConfiguration; +const { platformConfigurationSchema, resolvePlatformConfiguration } = coreServices; + import { resolveIntegrationConfigWithSourcesStrict } from "../integration-config"; import { type AdministrativeConfigurationReadContext, @@ -21,8 +25,12 @@ import { resolveServiceVaultSecretsStrict } from "./service-secrets"; */ export async function buildConfigurationInput( context?: AdministrativeConfigurationReadContext, + platform?: PlatformConfiguration, ): Promise { - if (!context) return withAdministrativeConfigurationRead({}, buildConfigurationInput); + if (!context) + return withAdministrativeConfigurationRead({}, (read) => + buildConfigurationInput(read, platform), + ); assertAdministrativeConfigurationRead(context); const services = context.services; const serviceConfigMap = await resolveAllServiceConfigsStrict( @@ -66,7 +74,8 @@ export async function buildConfigurationInput( }), ); return trustedConfigurationPayloadSchema.parse({ - domain: envString("DOMAIN", "localhost"), + domain: resolvePlatformConfiguration({ saved: platform, env: process.env }).values.domain, + ...(platform ? { platform: platformConfigurationSchema.parse(platform) } : {}), serviceConfigs: services.map((service) => { const envKeys = serviceConfigMap.envKeys.get(service.manifest.id); return { diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index c8a18614c..b8d26d2a3 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -145,14 +145,8 @@ import { export type BoundedConfigValue = TrustedConfigurationPayload["serviceConfigs"][number]["values"][string]; -export type PlatformField = { source: "host" } | { source: "database"; value: T }; -export type PlatformConfigurationPatch = { - domain?: PlatformField; - acmeEmail?: PlatformField; - releaseSelector?: PlatformField; - defaultRegion?: PlatformField; - backupRetentionDays?: PlatformField; -}; +export type PlatformField = coreServices.PlatformField; +export type PlatformConfigurationPatch = Omit; const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); const key = z .string() @@ -221,13 +215,7 @@ const overlaySchema = z.strictObject({ z.strictObject({ kind: z.literal("selection"), selectedRoots: z.array(id).max(256) }), z.strictObject({ kind: z.literal("platform"), - patch: z.strictObject({ - domain: z.unknown().optional(), - acmeEmail: z.unknown().optional(), - releaseSelector: z.unknown().optional(), - defaultRegion: z.unknown().optional(), - backupRetentionDays: z.unknown().optional(), - }), + patch: coreServices.platformConfigurationSchema.omit({ version: true }), }), ]), ) @@ -3065,7 +3053,7 @@ export async function prepareProspectiveConfiguration(input: { const selection = overlay.changes.find((change) => change.kind === "selection"); const payload = trustedConfigurationPayloadSchema.parse({ ...(await withAdministrativeConfigurationReadScope(read, () => - buildConfigurationInput(read), + buildConfigurationInput(read, source.platform), )), ...(selection?.kind === "selection" ? { selectedRoots: selection.selectedRoots } : {}), }); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 80863320d..974ac5874 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -1,5 +1,5 @@ import { loadConfiguredResourceAuthority } from "./configured-resource-authority"; -import { installDataWorkflowRuntime } from "./data-workflow-runtime"; +import { installDataWorkflowRuntime, resolveDataDefaultRegion } from "./data-workflow-runtime"; export { type ConfiguredResourceAuthority, @@ -1689,6 +1689,11 @@ export function createDefaultFixedCli( return async (args, options) => { const timeout = remainingCliTime(options); const env = { ...process.env }; + const platform = coreServices.readAppliedPlatformConfiguration(join(_rootDir, "infra/docker")); + if (platform?.defaultRegion?.source === "database") + env.OPENMAPX_REGION = + coreServices.resolvePlatformConfiguration({ saved: platform, env }).values.defaultRegion ?? + undefined; for (const key of [ "OPENMAPX_CONTROL_EXECUTION", "OPENMAPX_CONTROL_DELEGATION", @@ -1767,7 +1772,10 @@ export function createAdministrativeRuntime( if (!buildable.some((service) => service.serviceId === operation.serviceId)) { throw new Error("Build authority rejected"); } - await runFixedCli(buildArgs(operation.serviceId, operation.regionId), context); + await runFixedCli( + buildArgs(operation.serviceId, operation.regionId ?? resolveDataDefaultRegion(runtime)), + context, + ); return { completed: true }; }; runtime["services.buildAll"] = async (operation, context) => { @@ -1784,7 +1792,10 @@ export function createAdministrativeRuntime( try { context.signal.throwIfAborted(); attempted = true; - await runFixedCli(buildArgs(service.serviceId, operation.regionId), context); + await runFixedCli( + buildArgs(service.serviceId, operation.regionId ?? resolveDataDefaultRegion(runtime)), + context, + ); completedServiceIds.push(service.serviceId); } catch (error) { if ( diff --git a/apps/ops-agent/src/data-workflow-runtime.test.ts b/apps/ops-agent/src/data-workflow-runtime.test.ts index 757ffb8dc..701a0428a 100644 --- a/apps/ops-agent/src/data-workflow-runtime.test.ts +++ b/apps/ops-agent/src/data-workflow-runtime.test.ts @@ -138,3 +138,30 @@ it("freezes the same host .env defaults as the registered CLI, with shell preced vi.stubEnv("DOMAIN", "shell.example.test"); expect(control.planDataUpdate(args).domain).toBe("shell.example.test"); }); + +it("uses adopted default regions without expanding the deployment region authority", async () => { + const host = await ownedServiceHost({ withCoordinator: true }); + roots.push(host.cleanupRoot); + const { services } = await import("@openmapx/core/server"); + const { createUnavailableRuntime } = await import("./runtime"); + const { installDataWorkflowRuntime, resolveDataDefaultRegion } = await import( + "./data-workflow-runtime" + ); + const runtime = createUnavailableRuntime(); + installDataWorkflowRuntime(runtime, { rootDir: host.root, runFixedCli: vi.fn() }); + await services.commitConfigurationGeneration({ + infraDir: host.infraDir, + services: host.loaded, + input: { + ...services.readAppliedConfiguration(host.infraDir), + domain: "fixture.example.test", + selectedRoots: host.loaded.map((s) => s.manifest.id), + platform: { version: 1, defaultRegion: { source: "database", value: "europe/france" } }, + }, + }); + vi.stubEnv("OPENMAPX_REGION", "europe/germany"); + vi.stubEnv("OPENMAPX_ALLOWED_REGIONS", "europe/germany"); + expect(() => resolveDataDefaultRegion(runtime)).toThrow(/allow-list/); + vi.stubEnv("OPENMAPX_ALLOWED_REGIONS", "europe/germany,europe/france"); + expect(resolveDataDefaultRegion(runtime)).toBe("europe/france"); +}); diff --git a/apps/ops-agent/src/data-workflow-runtime.ts b/apps/ops-agent/src/data-workflow-runtime.ts index 67abe847c..2abf8641a 100644 --- a/apps/ops-agent/src/data-workflow-runtime.ts +++ b/apps/ops-agent/src/data-workflow-runtime.ts @@ -7,9 +7,25 @@ import type { DataWorkflowReceipt, ExecutionRef, OpsOperation } from "@openmapx/ import { services } from "@openmapx/core/server"; import type { FixedCliRunner } from "./administrative-runtime"; import { assertStagedCoordinatorConfigurationWrite } from "./configuration-maintenance-runtime"; +import { loadConfiguredResourceAuthority } from "./configured-resource-authority"; import { OpsExecutionFailure, OpsHostUpdateRequiredError, type OpsRuntime } from "./runtime"; const sources = new WeakMap(); +export function resolveDataDefaultRegion(runtime: OpsRuntime): string | undefined { + const source = sources.get(runtime); + if (!source) return process.env.OPENMAPX_REGION; + const saved = services.readAppliedPlatformConfiguration(join(source.rootDir, "infra/docker")); + const region = + services.resolvePlatformConfiguration({ saved, env: process.env }).values.defaultRegion ?? + undefined; + if ( + saved?.defaultRegion?.source === "database" && + region && + !loadConfiguredResourceAuthority().regions.has(region) + ) + throw Error("Applied default region exceeds the deployment region allow-list"); + return region; +} export function prepareDataUpdateAdmission( runtime: OpsRuntime, parent: ExecutionRef, @@ -23,7 +39,7 @@ export function prepareDataUpdateAdmission( parent, operation, serviceIds, - defaultRegion: process.env.OPENMAPX_REGION, + defaultRegion: operation.regionId === undefined ? resolveDataDefaultRegion(runtime) : undefined, defaultCountries: process.env.TRANSITOUS_COUNTRIES, }); } diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 909e79caa..1cc308814 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -127,7 +127,7 @@ import { installCoordinatorStagedOwnershipSource, prepareCoordinatorStagedRetention, } from "./coordinator-staged-ownership"; -import { prepareDataUpdateAdmission } from "./data-workflow-runtime"; +import { prepareDataUpdateAdmission, resolveDataDefaultRegion } from "./data-workflow-runtime"; import { type DawarichSubjectExportRouteOptions, registerDawarichSubjectExportRoute, @@ -2459,7 +2459,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif executionFor(job), job.operation, targets, - process.env.OPENMAPX_REGION, + "regionId" in job.operation && job.operation.regionId === undefined + ? resolveDataDefaultRegion(runtime) + : undefined, ) : undefined; const dataUpdatePlan = diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index 74d2493cb..34a7130d6 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -174,6 +174,21 @@ apply the data hardlink plan, and start or stop the whole enabled stack. There's no compose file to hand-edit anywhere — it's generated, every time. To re-render from the CLI, run `pnpm openmapx compose render`. +## Platform ownership + +Applied configuration generations retain a versioned, non-secret ownership +projection for the instance domain, ACME contact email, release selector, default +region and backup retention. Unadopted fields continue to use their host variables. +An explicit database-owned value survives later CLI rendering; a CLI `--domain` +argument affects that invocation and keeps the saved ownership intact. Returning +a field to host ownership removes its saved value. Service and integration +configuration keep their existing environment and file precedence. + +The default region never expands the operations agent's deployment allow-list. +A default outside `OPENMAPX_ALLOWED_REGIONS` is refused; explicit operation +regions remain subject to the same deployment policy. This projection contains +no platform credentials or arbitrary environment keys. + ## Data inventory `/admin/services/data` is the inventory and control surface for the open data the diff --git a/packages/cli/__tests__/cli-compose.test.ts b/packages/cli/__tests__/cli-compose.test.ts index 0525cb424..889587701 100644 --- a/packages/cli/__tests__/cli-compose.test.ts +++ b/packages/cli/__tests__/cli-compose.test.ts @@ -678,3 +678,72 @@ describe("renderComposeForRepo and what the admin panel applied", () => { expect(yaml).not.toContain("from-env"); }); }); + +it("keeps adopted platform slots across CLI renders and an invocation domain override", async () => { + writeManifest("alpha", { + ...baseManifest, + id: "alpha", + container: { + ...baseManifest.container, + environment: { + PUBLIC_ORIGIN: "https://${DOMAIN:-localhost}", + OPENMAPX_REGION: "${OPENMAPX_REGION:-}", + CONTACT: "${ACME_EMAIL:-admin@example.com}", + }, + }, + }); + await renderComposeForRepo({ rootDir: tmp, services: ["alpha"] }); + const registry = new coreServices.ServiceRegistry({ rootDir: tmp }); + await registry.load(); + const platform = { + version: 1 as const, + domain: { source: "database" as const, value: "adopted.example.test" }, + acmeEmail: { source: "database" as const, value: "operator@example.test" }, + defaultRegion: { source: "database" as const, value: "europe/germany" }, + }; + await coreServices.commitConfigurationGeneration({ + infraDir: join(tmp, "infra/docker"), + services: registry.list(), + input: { + domain: "adopted.example.test", + platform, + selectedRoots: ["alpha"], + serviceConfigs: [], + serviceSecrets: [], + integrationConfigs: [], + }, + }); + const previousRoot = process.env.OPENMAPX_ROOT_DIR; + const previousRegion = process.env.OPENMAPX_REGION; + try { + process.env.OPENMAPX_ROOT_DIR = tmp; + process.env.OPENMAPX_REGION = "europe/france"; + const { resolveOsmRegion } = await import("../src/lib/env-defaults"); + expect(resolveOsmRegion(undefined)).toEqual({ value: "europe/germany" }); + expect(resolveOsmRegion("europe/austria")).toEqual({ value: "europe/austria" }); + } finally { + if (previousRoot === undefined) delete process.env.OPENMAPX_ROOT_DIR; + else process.env.OPENMAPX_ROOT_DIR = previousRoot; + if (previousRegion === undefined) delete process.env.OPENMAPX_REGION; + else process.env.OPENMAPX_REGION = previousRegion; + } + await renderComposeForRepo({ rootDir: tmp, services: ["alpha"] }); + expect(readFileSync(generatedCompose(), "utf8")).toContain( + "PUBLIC_ORIGIN: https://adopted.example.test", + ); + expect(readFileSync(generatedCompose(), "utf8")).toContain("OPENMAPX_REGION: europe/germany"); + expect(coreServices.readAppliedPlatformConfiguration(join(tmp, "infra/docker"))).toEqual( + platform, + ); + await renderComposeForRepo({ rootDir: tmp, services: ["alpha"], domain: "once.example.test" }); + expect(readFileSync(generatedCompose(), "utf8")).toContain( + "PUBLIC_ORIGIN: https://once.example.test", + ); + expect(coreServices.readAppliedPlatformConfiguration(join(tmp, "infra/docker"))).toEqual( + platform, + ); + await renderComposeForRepo({ rootDir: tmp, services: ["alpha"] }); + expect(readFileSync(generatedCompose(), "utf8")).toContain( + "PUBLIC_ORIGIN: https://adopted.example.test", + ); +}); diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index d868dc19f..5e150cde4 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -49,7 +49,7 @@ const { export interface RenderRepoOptions { rootDir?: string; - domain: string; + domain?: string; services?: string[]; expectedCurrentRevisionId?: string | null; revisionId?: string; @@ -300,7 +300,12 @@ export async function renderComposeWithWriter( services, integrationSchemas, input: { - domain: opts.domain, + domain: coreServices.resolvePlatformConfiguration({ + saved: saved.platform, + env: process.env, + overrides: opts.domain === undefined ? {} : { domain: opts.domain }, + }).values.domain, + ...(saved.platform ? { platform: saved.platform } : {}), selectedRoots: applied.requestedIds, serviceConfigs, integrationConfigs: saved.integrationConfigs.filter((entry) => @@ -348,13 +353,13 @@ export function registerComposeCommands(program: Command): void { compose .command("render") .description("Apply the service selection and configuration as a new configuration generation") - .option("--domain ", "Public domain", process.env.DOMAIN ?? "localhost") + .option("--domain ", "Public domain (default: applied ownership or host)") .option("--services ", "Comma/space-separated root service ids for this render") .option( "--preset ", "Comma/space-separated preset names (app, routing, transit, pelias, nominatim, photon, overpass, tiles, martin, proxy, dev)", ) - .action(async (options: { domain: string; services?: string; preset?: string }) => { + .action(async (options: { domain?: string; services?: string; preset?: string }) => { try { const services = combineServiceSelection(options.services, options.preset); const r = await renderComposeForRepo({ domain: options.domain, services }); @@ -374,13 +379,13 @@ export function registerComposeCommands(program: Command): void { compose .command("up") .description("Start the stack via generated compose") - .option("--domain ", "Public domain", process.env.DOMAIN ?? "localhost") + .option("--domain ", "Public domain (default: applied ownership or host)") .option("--services ", "Comma/space-separated root service ids for this run") .option( "--preset ", "Comma/space-separated preset names (app, routing, transit, pelias, nominatim, photon, overpass, tiles, martin, proxy, dev)", ) - .action(async (options: { domain: string; services?: string; preset?: string }) => { + .action(async (options: { domain?: string; services?: string; preset?: string }) => { try { const services = combineServiceSelection(options.services, options.preset); const r = await renderComposeForRepo({ domain: options.domain, services }); diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index 86b70f528..d93e004a7 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -187,7 +187,7 @@ async function runFontDownload(transport: ReturnType) } async function renderAndApplyHardlinks(): Promise { - const rendered = await renderComposeForRepo({ domain: process.env.DOMAIN ?? "localhost" }); + const rendered = await renderComposeForRepo({}); for (const warning of rendered.selectionWarnings) log.warn(warning); const linked = await applyGeneratedHardlinks({ prune: true, requirePlan: true }); log.ok( @@ -384,7 +384,7 @@ export function registerDataCommands(program: Command): void { // Always render first so the plan file reflects the current // service-selection. Running against a stale plan silently links // dirs the operator no longer wants. - const rendered = await renderComposeForRepo({ domain: process.env.DOMAIN ?? "localhost" }); + const rendered = await renderComposeForRepo({}); for (const warning of rendered.selectionWarnings) log.warn(warning); const result = await applyGeneratedHardlinks({ prune: true, requirePlan: true }); log.ok( diff --git a/packages/cli/src/commands/services.ts b/packages/cli/src/commands/services.ts index eadc13633..01545278d 100644 --- a/packages/cli/src/commands/services.ts +++ b/packages/cli/src/commands/services.ts @@ -290,9 +290,7 @@ export function registerServicesCommands(program: Command): void { continueOnError: options.failFast !== true, }); if (result.completedIds.length > 0) { - const rendered = await renderComposeForRepo({ - domain: process.env.DOMAIN ?? "localhost", - }); + const rendered = await renderComposeForRepo({}); for (const warning of rendered.selectionWarnings) log.warn(warning); const linked = await applyGeneratedHardlinks({ prune: true, requirePlan: true }); log.ok( @@ -340,9 +338,7 @@ export function registerServicesCommands(program: Command): void { motisImport: options.import === true, }); if (result.completedIds.length > 0) { - const rendered = await renderComposeForRepo({ - domain: process.env.DOMAIN ?? "localhost", - }); + const rendered = await renderComposeForRepo({}); for (const warning of rendered.selectionWarnings) log.warn(warning); const linked = await applyGeneratedHardlinks({ prune: true, requirePlan: true }); log.ok( diff --git a/packages/cli/src/lib/env-defaults.ts b/packages/cli/src/lib/env-defaults.ts index b89631af6..470250fbb 100644 --- a/packages/cli/src/lib/env-defaults.ts +++ b/packages/cli/src/lib/env-defaults.ts @@ -1,3 +1,4 @@ +import { services as coreServices, repoPaths } from "@openmapx/core/server"; export const OPENMAPX_REGION_ENV = "OPENMAPX_REGION"; export const TRANSITOUS_COUNTRIES_ENV = "TRANSITOUS_COUNTRIES"; @@ -51,7 +52,20 @@ export function resolveRegionFromEnv( ): ResolvedEnvDefault { const direct = normalizeScalar(explicit); if (direct) return { value: direct }; - return resolveFromEnvNames(envNames, env); + for (const name of envNames) { + if (name === OPENMAPX_REGION_ENV && env === process.env) { + const saved = coreServices.readAppliedPlatformConfiguration(repoPaths().infraDir); + if (saved?.defaultRegion?.source === "database") + return { + value: + coreServices.resolvePlatformConfiguration({ saved, env }).values.defaultRegion ?? + undefined, + }; + } + const resolved = resolveFromEnvNames([name], env); + if (resolved.value) return resolved; + } + return {}; } export function resolveBuildRegion( diff --git a/packages/cli/src/lib/service-lifecycle.ts b/packages/cli/src/lib/service-lifecycle.ts index c296299ca..d19c00391 100644 --- a/packages/cli/src/lib/service-lifecycle.ts +++ b/packages/cli/src/lib/service-lifecycle.ts @@ -357,7 +357,6 @@ export async function runCliServiceLifecycle(options: { async prepare({ ownership: owned, coordinatorMode }) { const rendered = await renderComposeWithWriter( { - domain: process.env.DOMAIN ?? "localhost", expectedCurrentRevisionId: expected, revisionId: finalRevisionId, }, diff --git a/packages/core/src/build-artifacts/configuration-source-witness.test.ts b/packages/core/src/build-artifacts/configuration-source-witness.test.ts index 04e916d71..577edc13c 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.test.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.test.ts @@ -2,6 +2,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, expect, it } from "vitest"; +import { commitConfigurationGeneration } from "../services/configuration-generation"; import { writeServiceSelectionFile } from "../services/deployment"; import { PRIVACY_BACKUP_COLLECTOR_IMAGE_ENV, @@ -217,3 +218,46 @@ it("includes declared flat-schema host fields in the same private source witness closeControlInstallation(installation); } }); + +it("projects only validated non-secret platform ownership under the original revision witness", async () => { + const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: [], + }); + try { + const platform = { + version: 1 as const, + domain: { source: "database" as const, value: "adopted.example.test" }, + }; + const applied = await commitConfigurationGeneration({ + infraDir: root, + services: [], + input: { + domain: "adopted.example.test", + platform, + selectedRoots: [], + serviceConfigs: [], + serviceSecrets: [], + integrationConfigs: [], + }, + }); + const options = { + installation, + infraDir: root, + services: [] as LoadedService[], + authorityRevision: "v1", + env: { BETTER_AUTH_SECRET: "not-public" }, + }; + const observed = observeConfigurationSources(options); + expect(observed.platform).toEqual(platform); + expect(JSON.stringify(observed)).not.toContain("not-public"); + verifyConfigurationSourceWitness({ + ...options, + sourceRevision: observed.sourceRevision, + expectedCurrentRevisionId: applied.revisionId, + }); + } finally { + closeControlInstallation(installation); + } +}); diff --git a/packages/core/src/build-artifacts/configuration-source-witness.ts b/packages/core/src/build-artifacts/configuration-source-witness.ts index 5ec684afb..df5084c0c 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.ts @@ -9,7 +9,10 @@ import { configurationSourcesInspectionSchema, } from "../ops/configuration-preparation"; import { serviceConfigEnvPrefix } from "../services/config-resolver"; -import { currentConfigurationGeneration } from "../services/configuration-generation"; +import { + currentConfigurationGeneration, + readAppliedPlatformConfiguration, +} from "../services/configuration-generation"; import { SERVICE_SELECTION_FILE } from "../services/deployment"; import type { LoadedService } from "../services/types"; import { @@ -143,7 +146,12 @@ function releaseSource(infraDir: string) { } } function currentSources(infraDir: string, env: NodeJS.ProcessEnv) { - return { ...hostEnv(infraDir, env), release: releaseSource(infraDir) }; + const platform = readAppliedPlatformConfiguration(infraDir); + return { + ...hostEnv(infraDir, env), + release: releaseSource(infraDir), + ...(platform ? { platform } : {}), + }; } /** The renderer's supported resource interpolation is a finite mapping. */ @@ -213,6 +221,7 @@ export function observeConfigurationSources(options: { continue; const inspection = configurationSourcesInspectionSchema.parse({ revisionId, + ...(observed.platform ? { platform: observed.platform } : {}), sourceRevision: randomUUID(), services: configurationHostFields(options.services, observed.env), }); diff --git a/packages/core/src/build-artifacts/data-update.ts b/packages/core/src/build-artifacts/data-update.ts index 2dc246a6e..192cd4070 100644 --- a/packages/core/src/build-artifacts/data-update.ts +++ b/packages/core/src/build-artifacts/data-update.ts @@ -5,7 +5,11 @@ import { canonicalJson } from "../ops/canonical-json"; import type { OpsOperation } from "../ops/contract"; import type { ExecutionRef } from "../ops/execution-ref"; import { opsOperationFingerprint } from "../ops/fingerprint"; -import { currentConfigurationGeneration } from "../services/configuration-generation"; +import { + currentConfigurationGeneration, + readAppliedPlatformConfiguration, +} from "../services/configuration-generation"; +import { resolvePlatformConfiguration } from "../services/platform-configuration"; import { controlDigest, executionAuthorityState, @@ -46,13 +50,25 @@ export function planDataUpdate(options: { .filter(Boolean), ), ].sort(); + const saved = readAppliedPlatformConfiguration(options.infraDir); + const platform = resolvePlatformConfiguration({ + saved, + env: { DOMAIN: defaults.domain, OPENMAPX_REGION: defaults.region, ...process.env }, + }).values; const sourcesDigest = dataUpdateSourcesDigest(options.infraDir); + if (revisionId !== currentConfigurationGeneration(options.infraDir)) + throw Error("Full Update configuration changed during admission"); if (sourceBefore !== sourcesDigest) throw Error("Full Update source changed during admission"); return createDataUpdatePlan({ parent: options.parent, operation: options.operation, - domain: process.env.DOMAIN ?? defaults.domain ?? "localhost", - region: options.operation.regionId ?? options.defaultRegion ?? defaults.region ?? "", + domain: platform.domain, + region: + options.operation.regionId ?? + (saved?.defaultRegion?.source === "database" + ? platform.defaultRegion + : (options.defaultRegion ?? platform.defaultRegion)) ?? + "", countries, revisionId, sourcesDigest, diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts index 09c42b999..3238236bb 100644 --- a/packages/core/src/ops/configuration-preparation-records.ts +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -1,5 +1,6 @@ /** Portable bounded candidate records; parsing creates no authority. */ import { z } from "zod"; +import { platformConfigurationSchema } from "../services/platform-configuration"; import { coordinatorLifecycleMetadataSchema } from "./coordinator-lifecycle-records"; const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); @@ -9,6 +10,7 @@ export const coordinatorModeSchema = z.enum(["ensure-running", "require-running" export const configurationSourcesInspectionSchema = z .strictObject({ revisionId: revision.nullable(), + platform: platformConfigurationSchema.optional(), sourceRevision: z.uuid(), services: z .array( diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index aa649a085..540d9e1ba 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -1,4 +1,5 @@ import { catalogPinSetSchema } from "../catalog-pins/schemas"; +import { platformConfigurationSchema } from "../services/platform-configuration"; import { configurationGenerationRevisionSchema, configurationHardlinksApplyOperationSchema, @@ -1025,6 +1026,7 @@ export const OPS_RESULT_SCHEMAS = { "configurationMaintenance.settle": configurationMaintenanceReceiptSchema, "configuration.inspect": z.strictObject({ revisionId: configurationGenerationRevisionSchema.nullable(), + platform: platformConfigurationSchema.optional(), integrationConfigDigests: z .array( z.strictObject({ diff --git a/packages/core/src/ops/trusted-config.ts b/packages/core/src/ops/trusted-config.ts index a310b478c..6bcc3814e 100644 --- a/packages/core/src/ops/trusted-config.ts +++ b/packages/core/src/ops/trusted-config.ts @@ -1,5 +1,6 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import z from "zod/v4"; +import { platformConfigurationSchema } from "../services/platform-configuration"; import { isTrustedConfigurationApplyOperation, type OpsOperation, @@ -125,6 +126,7 @@ function uniqueBy(values: readonly T[], identity: (value: T) => string): bool export const trustedConfigurationPayloadSchema = z .strictObject({ domain, + platform: platformConfigurationSchema.optional(), // A new selection, carried by `serviceSelection.apply` only; every other // apply renders the operator's selection as it stands. selectedRoots: z.array(serviceId).max(256).optional(), diff --git a/packages/core/src/services/__tests__/configuration-generation.test.ts b/packages/core/src/services/__tests__/configuration-generation.test.ts index 80c9a7dd7..9be75f46b 100644 --- a/packages/core/src/services/__tests__/configuration-generation.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation.test.ts @@ -325,3 +325,41 @@ describe("stack compose arguments", () => { ]); }); }); + +it("retains closed platform ownership through a generation without exporting secrets", async () => { + const { infraDir } = checkout(); + const platform = { + version: 1 as const, + domain: { source: "database" as const, value: "adopted.example.test" }, + defaultRegion: { source: "database" as const, value: "europe/germany" }, + }; + await commitConfigurationGeneration({ + infraDir, + services, + input: input({ domain: "adopted.example.test", platform }), + }); + expect(readAppliedConfiguration(infraDir).platform).toEqual(platform); + const config = JSON.parse( + readFileSync(join(infraDir, ".trusted-config-current/resolved-config.generated.json"), "utf8"), + ); + expect(config.platform).toEqual(platform); + expect(config).not.toHaveProperty("serviceSecrets"); +}); + +it("keeps host-owned Compose interpolation equivalent to legacy generations", () => { + const { infraDir } = checkout(); + const manifest = service("alpha"); + manifest.manifest.container.environment = { + ORIGIN: "https://${DOMAIN:-localhost}", + ACME: "${ACME_EMAIL:-admin@example.com}", + }; + const before = renderConfiguration({ infraDir, services: [manifest], input: input() }); + const after = renderConfiguration({ + infraDir, + services: [manifest], + input: input({ + platform: { version: 1, domain: { source: "host" }, acmeEmail: { source: "host" } }, + }), + }); + expect(after.rendered.composeYaml).toBe(before.rendered.composeYaml); +}); diff --git a/packages/core/src/services/__tests__/platform-configuration.test.ts b/packages/core/src/services/__tests__/platform-configuration.test.ts new file mode 100644 index 000000000..5b849f302 --- /dev/null +++ b/packages/core/src/services/__tests__/platform-configuration.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import { + platformConfigurationSchema, + resolvePlatformConfiguration, +} from "../platform-configuration"; + +describe("platform ownership", () => { + it("keeps legacy host defaults and excludes unrelated credentials", () => { + expect(resolvePlatformConfiguration({ env: { BETTER_AUTH_SECRET: "never-copy" } })).toEqual({ + values: { + domain: "localhost", + acmeEmail: "admin@example.com", + releaseSelector: "main", + defaultRegion: null, + backupRetentionDays: 30, + }, + sources: { + domain: "host", + acmeEmail: "host", + releaseSelector: "host", + defaultRegion: "host", + backupRetentionDays: "host", + }, + warnings: [], + }); + }); + it("retains explicit adoption, invocation override and return to host", () => { + const saved = platformConfigurationSchema.parse({ + version: 1, + domain: { source: "database", value: "maps.example.test" }, + defaultRegion: { source: "database", value: "europe/germany" }, + }); + const env = { DOMAIN: "host.example.test", OPENMAPX_REGION: "europe/france" }; + expect(resolvePlatformConfiguration({ saved, env }).values).toMatchObject({ + domain: "maps.example.test", + defaultRegion: "europe/germany", + }); + expect( + resolvePlatformConfiguration({ saved, env, overrides: { domain: "once.example.test" } }), + ).toMatchObject({ values: { domain: "once.example.test" }, sources: { domain: "invocation" } }); + expect( + resolvePlatformConfiguration({ saved: { ...saved, domain: { source: "host" } }, env }).values + .domain, + ).toBe("host.example.test"); + expect( + platformConfigurationSchema.safeParse({ + version: 1, + domain: { source: "host", value: "stale.test" }, + }).success, + ).toBe(false); + }); + it.each([ + ["domain", "https://maps.test"], + ["domain", "maps.test:443"], + ["domain", "a..test"], + ["domain", "-maps.test"], + ["domain", "map_test"], + ["acmeEmail", "not-email"], + ["releaseSelector", "latest"], + ["defaultRegion", "../outside"], + ["defaultRegion", "https://data.test"], + ["backupRetentionDays", 0], + ["backupRetentionDays", 36501], + ])("rejects malformed %s", (key, value) => { + expect( + platformConfigurationSchema.safeParse({ version: 1, [key]: { source: "database", value } }) + .success, + ).toBe(false); + }); + it("rejects unknown fields and invalid host configuration instead of exporting a secret map", () => { + expect( + platformConfigurationSchema.safeParse({ + version: 1, + redisPassword: { source: "database", value: "secret" }, + }).success, + ).toBe(false); + expect(() => resolvePlatformConfiguration({ env: { DOMAIN: "bad/path" } })).toThrow(); + }); +}); diff --git a/packages/core/src/services/compose-renderer.ts b/packages/core/src/services/compose-renderer.ts index 9d8da7452..9b1a550c8 100644 --- a/packages/core/src/services/compose-renderer.ts +++ b/packages/core/src/services/compose-renderer.ts @@ -157,6 +157,8 @@ function hardlinkTargetDir(consumerId: string, c: ServiceConsumes): string { } export interface RenderContext { + platform?: import("./platform-configuration").PlatformConfiguration; + platformValues?: import("./platform-configuration").PlatformValues; domain?: string; /** * Per-consumer-entry mount paths keyed by the same `` / `/` @@ -533,6 +535,28 @@ export function renderServiceSnippet( : [...c.envFile]; } if (c.environment) snippet.environment = { ...c.environment }; + // Only known platform slots in audited built-in manifests may be adopted. + // Host-owned slots keep their original Compose interpolation byte-for-byte. + if (service.isBuiltIn && snippet.environment && ctx.platform && ctx.platformValues) { + const slots = { + DOMAIN: "domain", + ACME_EMAIL: "acmeEmail", + OPENMAPX_RELEASE: "releaseSelector", + OPENMAPX_REGION: "defaultRegion", + BACKUP_RETENTION_DAYS: "backupRetentionDays", + } as const; + for (const [key, value] of Object.entries(snippet.environment)) { + snippet.environment[key] = value.replace( + /\$\{(DOMAIN|ACME_EMAIL|OPENMAPX_RELEASE|OPENMAPX_REGION|BACKUP_RETENTION_DAYS)(?::-[^}]*)?\}/g, + (reference, variable: keyof typeof slots) => { + const field = slots[variable]; + return ctx.platform?.[field]?.source === "database" + ? escapeComposeInterpolation(String(ctx.platformValues?.[field] ?? "")) + : reference; + }, + ); + } + } // Overlay operator-resolved config onto the manifest's baseline environment. // Resolved values win over manifest defaults (that's the whole point of // admin/env config). Keys with `undefined`/`null` values are skipped so a diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index 5ef7cf439..edde022d8 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -52,6 +52,11 @@ import { readConfigurationGenerationBase, SERVICE_SELECTION_FILE, } from "./deployment"; +import { + type PlatformConfiguration, + platformConfigurationSchema, + resolvePlatformConfiguration, +} from "./platform-configuration"; import { buildAppApiServiceEnv, expandServiceSelection } from "./selection"; import { renderTraefikDynamicConfiguration, renderTraefikDynamicYaml } from "./traefik-renderer"; import type { LoadedService } from "./types"; @@ -694,6 +699,7 @@ export function readAppliedConfiguration( const resolved = JSON.parse(readFileSync(join(generation, RESOLVED_CONFIG_FILE), "utf-8")) as { serviceConfigs?: ConfigurationInput["serviceConfigs"]; integrationConfigs?: ConfigurationInput["integrationConfigs"]; + platform?: PlatformConfiguration; }; const serviceSecrets: ConfigurationInput["serviceSecrets"] = []; const secretsRoot = join(generation, GENERATED_SECRETS_DIRNAME); @@ -710,9 +716,26 @@ export function readAppliedConfiguration( serviceConfigs: resolved.serviceConfigs ?? [], integrationConfigs: resolved.integrationConfigs ?? [], serviceSecrets, + ...(resolved.platform + ? { platform: platformConfigurationSchema.parse(resolved.platform) } + : {}), }; } +/** Read only non-secret platform metadata, never service secret files. */ +export function readAppliedPlatformConfiguration( + infraDir: string, +): PlatformConfiguration | undefined { + const active = currentConfigurationGeneration(infraDir); + if (!active) return undefined; + const resolved = JSON.parse( + readFileSync(join(infraDir, CONFIGURATION_GENERATIONS, active, RESOLVED_CONFIG_FILE), "utf8"), + ); + return resolved.platform === undefined + ? undefined + : platformConfigurationSchema.parse(resolved.platform); +} + /** Call under writer ownership so the pointer and its projections are one observation. * Only the resolved integration file is read: service secrets never enter this result. */ @@ -735,6 +758,9 @@ export function inspectAppliedConfiguration( }); return { revisionId, + ...(resolved.platform + ? { platform: platformConfigurationSchema.parse(resolved.platform) } + : {}), integrationConfigDigests: integrationConfigs.map(({ integrationId, values }) => ({ integrationId, digest: configurationProjectionFingerprint(values), @@ -1099,6 +1125,15 @@ export function renderConfiguration(options: RenderConfigurationOptions) { ); const context: RenderContext = { domain: input.domain, + ...(input.platform + ? { + platform: input.platform, + platformValues: resolvePlatformConfiguration({ + saved: input.platform, + overrides: { domain: input.domain }, + }).values, + } + : {}), // Runtime supplies a fixed generation alias at this same directory depth. // Keep these bytes compatible with existing current/retained generations. composeOutDir: join(infraDir, CURRENT_CONFIGURATION), @@ -1173,6 +1208,7 @@ async function commitOwnedConfigurationGeneration( `${JSON.stringify({ serviceConfigs: input.serviceConfigs, integrationConfigs: input.integrationConfigs, + ...(input.platform ? { platform: platformConfigurationSchema.parse(input.platform) } : {}), })}\n`, 0o600, ); diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 8143fdc8d..a67c62008 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -41,6 +41,7 @@ export { inspectAppliedConfiguration, type RenderConfigurationOptions, readAppliedConfiguration, + readAppliedPlatformConfiguration, readAppliedServiceIds, renderConfiguration, retainedContainerComposeFile, @@ -104,6 +105,13 @@ export { serviceManifestSchema, validateServiceManifest, } from "./manifest-schema"; +export { + type PlatformConfiguration, + type PlatformField, + type PlatformValues, + platformConfigurationSchema, + resolvePlatformConfiguration, +} from "./platform-configuration"; export { ServiceRegistry, type ServiceRegistryOptions } from "./registry"; export { captureReleaseServiceAuthority, diff --git a/packages/core/src/services/platform-configuration.ts b/packages/core/src/services/platform-configuration.ts new file mode 100644 index 000000000..f585dccca --- /dev/null +++ b/packages/core/src/services/platform-configuration.ts @@ -0,0 +1,98 @@ +/** Closed non-secret ownership projection carried by a configuration generation. */ +import z from "zod/v4"; +import { normalizeReleaseSelection } from "./release-selection"; + +export type PlatformField = { source: "host" } | { source: "database"; value: T }; +const field = (value: T) => + z.discriminatedUnion("source", [ + z.strictObject({ source: z.literal("host") }), + z.strictObject({ source: z.literal("database"), value }), + ]); +const domain = z + .string() + .max(253) + .refine( + (value) => + value === "localhost" || + value.split(".").every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(label)), + "Expected a hostname without scheme, port or path", + ); +const email = z.email().max(320); +const releaseSelector = z + .string() + .max(128) + .refine((value) => { + try { + normalizeReleaseSelection(value); + return true; + } catch { + return false; + } + }, "Invalid release selector"); +const region = z + .string() + .min(1) + .max(128) + .regex(/^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$/); +const retention = z.number().int().min(1).max(36500); +export const platformConfigurationSchema = z.strictObject({ + version: z.literal(1), + domain: field(domain).optional(), + acmeEmail: field(email).optional(), + releaseSelector: field(releaseSelector).optional(), + defaultRegion: field(region).optional(), + backupRetentionDays: field(retention).optional(), +}); +export type PlatformConfiguration = z.infer; +export type PlatformValues = { + domain: string; + acmeEmail: string; + releaseSelector: string; + defaultRegion: string | null; + backupRetentionDays: number; +}; +export function resolvePlatformConfiguration({ + saved, + env = {}, + overrides = {}, +}: { + saved?: PlatformConfiguration; + env?: Readonly>; + overrides?: Partial; +}) { + const ownership = saved ? platformConfigurationSchema.parse(saved) : undefined; + const host: PlatformValues = { + domain: env.DOMAIN || "localhost", + acmeEmail: env.ACME_EMAIL || "admin@example.com", + releaseSelector: env.OPENMAPX_RELEASE || "main", + defaultRegion: env.OPENMAPX_REGION || null, + backupRetentionDays: Number(env.BACKUP_RETENTION_DAYS || 30), + }; + const values = { ...host }; + const sources = {} as Record; + for (const key of Object.keys(host) as (keyof PlatformValues)[]) { + const adopted = ownership?.[key]; + const value = + overrides[key] !== undefined + ? overrides[key] + : adopted?.source === "database" + ? adopted.value + : host[key]; + sources[key] = + overrides[key] !== undefined + ? "invocation" + : adopted?.source === "database" + ? "database" + : "host"; + // Validate only the effective value; an explicitly adopted field supersedes its host slot. + const schema = { + domain, + acmeEmail: email, + releaseSelector, + defaultRegion: region.nullable(), + backupRetentionDays: retention, + }[key]; + Object.assign(values, { [key]: schema.parse(value) }); + } + return { values, sources, warnings: [] as string[] }; +} diff --git a/packages/core/src/services/release-manifest.ts b/packages/core/src/services/release-manifest.ts index b3af7235a..5af9718f6 100644 --- a/packages/core/src/services/release-manifest.ts +++ b/packages/core/src/services/release-manifest.ts @@ -1,3 +1,22 @@ +import { + DEFAULT_RELEASE_MANIFEST_REPOSITORY, + imagePrefixOf, + normalizeReleaseSelection, + PRODUCT_VERSION_PATTERN, + RELEASE_REPOSITORY_ENV, + RELEASE_SELECTOR_ENV, + type ReleaseSelection, +} from "./release-selection"; + +export { + DEFAULT_RELEASE_MANIFEST_REPOSITORY, + normalizeReleaseSelection, + PRODUCT_VERSION_PATTERN, + RELEASE_REPOSITORY_ENV, + RELEASE_SELECTOR_ENV, + type ReleaseSelection, +} from "./release-selection"; + /** * The aggregate OpenMapX release manifest published by * `.github/workflows/docker.yml` as `//release-manifest:main`. @@ -54,21 +73,6 @@ export const RELEASE_IMAGE_NAMES = [ ] as const; const DIGEST = "sha256:[a-f0-9]{64}"; -export const RELEASE_SELECTOR_ENV = "OPENMAPX_RELEASE"; -export const RELEASE_REPOSITORY_ENV = "OPENMAPX_RELEASE_MANIFEST_REPOSITORY"; -export const DEFAULT_RELEASE_MANIFEST_REPOSITORY = "ghcr.io/openmapx/release-manifest"; -export const PRODUCT_VERSION_PATTERN = - /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)-(alpha|beta|rc)\.(0|[1-9]\d*)$/; -export type ReleaseSelection = - | { kind: "local" } - | { - kind: "main" | "preview" | "version" | "digest"; - selector: string; - manifestImage: string; - manifestRepository: string; - imagePrefix: string; - expectedVersion?: string; - }; export type ReleaseChannel = | { kind: "disabled" } | { @@ -80,44 +84,6 @@ export type ReleaseChannel = function envValue(name: string): string | undefined { return process.env?.[name]; } -function imagePrefixOf(image: string): string { - return image.slice(0, image.lastIndexOf("/")); -} -export function normalizeReleaseSelection( - selector: string = "main", - manifestRepository: string = DEFAULT_RELEASE_MANIFEST_REPOSITORY, -): ReleaseSelection { - if ( - !/^[a-z0-9][a-z0-9.-]*(?::[0-9]+)?\/[a-z0-9][a-z0-9._/-]*\/release-manifest$/.test( - manifestRepository, - ) || - manifestRepository.split("/").some((part) => part === "." || part === ".." || part === "") - ) - throw new Error( - `${RELEASE_REPOSITORY_ENV} must be a registry/namespace/release-manifest repository without a tag or digest`, - ); - if (selector === "local") return { kind: "local" }; - const kind = - selector === "main" || selector === "preview" - ? selector - : PRODUCT_VERSION_PATTERN.test(selector) - ? "version" - : /^sha256:[a-f0-9]{64}$/.test(selector) - ? "digest" - : undefined; - if (!kind) - throw new Error( - `Unsupported ${RELEASE_SELECTOR_ENV}: ${selector}. Choose main, preview, an exact prerelease version/digest, or local.`, - ); - return { - kind, - selector, - manifestRepository, - manifestImage: `${manifestRepository}${kind === "digest" ? "@" : ":"}${selector}`, - imagePrefix: imagePrefixOf(manifestRepository), - ...(kind === "version" ? { expectedVersion: selector } : {}), - }; -} export function releaseSelection(): ReleaseSelection { return normalizeReleaseSelection( envValue(RELEASE_SELECTOR_ENV), diff --git a/packages/core/src/services/release-selection.ts b/packages/core/src/services/release-selection.ts new file mode 100644 index 000000000..dfe352cf9 --- /dev/null +++ b/packages/core/src/services/release-selection.ts @@ -0,0 +1,54 @@ +/** Pure release selection shared by runtime validation and browser-safe schemas. */ +export const RELEASE_SELECTOR_ENV = "OPENMAPX_RELEASE"; +export const RELEASE_REPOSITORY_ENV = "OPENMAPX_RELEASE_MANIFEST_REPOSITORY"; +export const DEFAULT_RELEASE_MANIFEST_REPOSITORY = "ghcr.io/openmapx/release-manifest"; +export const PRODUCT_VERSION_PATTERN = + /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)-(alpha|beta|rc)\.(0|[1-9]\d*)$/; +export type ReleaseSelection = + | { kind: "local" } + | { + kind: "main" | "preview" | "version" | "digest"; + selector: string; + manifestImage: string; + manifestRepository: string; + imagePrefix: string; + expectedVersion?: string; + }; +export function imagePrefixOf(image: string): string { + return image.slice(0, image.lastIndexOf("/")); +} +export function normalizeReleaseSelection( + selector: string = "main", + manifestRepository: string = DEFAULT_RELEASE_MANIFEST_REPOSITORY, +): ReleaseSelection { + if ( + !/^[a-z0-9][a-z0-9.-]*(?::[0-9]+)?\/[a-z0-9][a-z0-9._/-]*\/release-manifest$/.test( + manifestRepository, + ) || + manifestRepository.split("/").some((part) => part === "." || part === ".." || part === "") + ) + throw new Error( + `${RELEASE_REPOSITORY_ENV} must be a registry/namespace/release-manifest repository without a tag or digest`, + ); + if (selector === "local") return { kind: "local" }; + const kind = + selector === "main" || selector === "preview" + ? selector + : PRODUCT_VERSION_PATTERN.test(selector) + ? "version" + : /^sha256:[a-f0-9]{64}$/.test(selector) + ? "digest" + : undefined; + if (!kind) + throw new Error( + `Unsupported ${RELEASE_SELECTOR_ENV}: ${selector}. Choose main, preview, an exact prerelease version/digest, or local.`, + ); + return { + kind, + selector, + manifestRepository, + manifestImage: `${manifestRepository}${kind === "digest" ? "@" : ":"}${selector}`, + imagePrefix: imagePrefixOf(manifestRepository), + ...(kind === "version" ? { expectedVersion: selector } : {}), + }; +} From 0a4bb2d11e6efff61375aa6e9a8a49f51f2158c7 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 02:07:42 +0100 Subject: [PATCH 052/135] feat(admin): manage pinned transit feed credentials safely --- apps/api/openapi.json | 2058 +++- .../account/admin-invariant-postgres.test.ts | 63 + apps/api/src/account/admin-invariant.ts | 33 +- .../admin-reauthentication-postgres.test.ts | 17 + .../api/src/account/admin-reauthentication.ts | 50 +- .../__tests__/user-erasure-postgres.test.ts | 1 + .../0048_admin_transit_credentials.sql | 22 + .../src/db/migrations/meta/0048_snapshot.json | 9093 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + apps/api/src/db/schema.ts | 1 + .../db/transit-credential-preview-schema.ts | 34 + apps/api/src/privacy/catalogue.ts | 1 + apps/api/src/privacy/openmapx-collectors.ts | 22 + .../routes/__tests__/admin-operations.test.ts | 7 +- .../admin-transit-credentials.test.ts | 231 + .../src/routes/admin-transit-credentials.ts | 112 + apps/api/src/routes/index.ts | 2 + .../__tests__/admin-job-handlers.test.ts | 13 +- .../native-transit-credentials-api.ts | 31 + apps/api/src/services/activity-retention.ts | 10 +- .../src/services/admin-operation-catalog.ts | 21 - .../configuration-commit-participant.ts | 4 +- apps/api/src/services/configuration-input.ts | 12 + apps/api/src/services/job-runner.ts | 51 +- .../src/services/prospective-configuration.ts | 100 +- .../transit-credential-application.ts | 41 + .../transit-credential-authorization.ts | 94 + ...ransit-credential-preview-postgres.test.ts | 149 + .../transit-credential-preview.test.ts | 22 + .../services/transit-credential-preview.ts | 205 + .../services/transit-credential-recovery.ts | 83 + .../services/transit-credential-retention.ts | 31 + .../services/transit-credential-revision.ts | 15 + .../transit-feed-credential-participant.ts | 521 + .../transit-feed-credentials-postgres.test.ts | 612 ++ .../src/services/transit-feed-credentials.ts | 77 + .../test/configuration-postgres-fixture.ts | 16 +- .../src/__fixtures__/owned-data-manager.ts | 1 + .../__fixtures__/pinned-feed-credentials.ts | 16 + .../src/administrative-runtime.test.ts | 5 +- apps/ops-agent/src/administrative-runtime.ts | 10 +- .../src/configuration-maintenance-store.ts | 22 +- apps/ops-agent/src/index.ts | 3 +- apps/ops-agent/src/production-runtime.ts | 8 + apps/ops-agent/src/server.ts | 1 + .../src/transit-credentials-runtime.test.ts | 172 + .../src/transit-credentials-runtime.ts | 231 + .../transit/TransitPipelinePage.tsx | 4 + .../TransitCredentialsSection.test.tsx | 178 + .../services/TransitCredentialsSection.tsx | 431 + apps/web/src/lib/admin/accountAdmin.ts | 2 + .../administration/services-administration.md | 24 + packages/cli/__tests__/cli-compose.test.ts | 12 +- .../__tests__/cli-transitous-api-keys.test.ts | 248 +- packages/cli/src/commands/compose.ts | 3 + packages/cli/src/commands/data.ts | 6 +- packages/cli/src/lib/transitous-api-keys.ts | 425 +- packages/core/package.json | 1 + .../api/contracts/data-manager-admin/index.ts | 2 + .../data-manager-admin/operations.ts | 2 + .../data-manager-admin/transit-credentials.ts | 119 + .../api/contracts/service-admin/operations.ts | 1 - .../admin/data-manager-admin.generated.ts | 342 + .../models/applyAdminTransitCredentials200.ts | 9 + ...AdminTransitCredentials200Configuration.ts | 34 + .../models/applyAdminTransitCredentials400.ts | 20 + .../models/applyAdminTransitCredentials401.ts | 20 + .../models/applyAdminTransitCredentials403.ts | 20 + .../models/applyAdminTransitCredentials409.ts | 20 + .../models/applyAdminTransitCredentials413.ts | 20 + .../models/applyAdminTransitCredentials415.ts | 20 + .../models/applyAdminTransitCredentials429.ts | 11 + .../models/applyAdminTransitCredentials500.ts | 20 + .../models/applyAdminTransitCredentials503.ts | 20 + .../applyAdminTransitCredentialsBody.ts | 13 + .../models/getAdminTransitCredentials200.ts | 20 + ...etAdminTransitCredentials200EntriesItem.ts | 13 + .../getAdminTransitCredentials200Owner.ts | 12 + .../models/getAdminTransitCredentials400.ts | 20 + .../models/getAdminTransitCredentials401.ts | 20 + .../models/getAdminTransitCredentials403.ts | 20 + .../models/getAdminTransitCredentials409.ts | 20 + .../models/getAdminTransitCredentials413.ts | 20 + .../models/getAdminTransitCredentials415.ts | 20 + .../models/getAdminTransitCredentials429.ts | 11 + .../models/getAdminTransitCredentials500.ts | 20 + .../models/getAdminTransitCredentials503.ts | 20 + .../src/api/generated/admin/models/index.ts | 38 + .../previewAdminTransitCredentials200.ts | 29 + ...ewAdminTransitCredentials200ChangesItem.ts | 12 + ...nTransitCredentials200ChangesItemAction.ts | 10 + .../previewAdminTransitCredentials400.ts | 20 + .../previewAdminTransitCredentials401.ts | 20 + .../previewAdminTransitCredentials403.ts | 20 + .../previewAdminTransitCredentials409.ts | 20 + .../previewAdminTransitCredentials413.ts | 20 + .../previewAdminTransitCredentials415.ts | 20 + .../previewAdminTransitCredentials429.ts | 11 + .../previewAdminTransitCredentials500.ts | 20 + .../previewAdminTransitCredentials503.ts | 20 + .../previewAdminTransitCredentialsBody.ts | 18 + ...wAdminTransitCredentialsBodyChangesItem.ts | 22 + .../admin/operation-bindings.generated.ts | 21 + .../listAdminOperations200OperationsItemId.ts | 1 - ...nfiguration-maintenance-forward-records.ts | 2 + .../src/build-artifacts/control-authority.ts | 74 + .../fixed-configuration-maintenance.test.ts | 37 + packages/core/src/build-artifacts/index.ts | 1 + .../local-credential-template.test.ts | 93 + .../local-credential-template.ts | 75 + packages/core/src/ops/contract.test.ts | 8 + packages/core/src/ops/contract.ts | 25 + packages/core/src/ops/feed-credential-map.ts | 77 +- .../core/src/ops/transit-credentials.test.ts | 32 + packages/core/src/ops/trusted-config.ts | 5 + .../configuration-generation.test.ts | 23 + .../src/services/configuration-generation.ts | 21 + packages/core/src/services/index.ts | 2 + .../feed-credential-requirements.test.ts | 55 + .../__tests__/feed-credentials.test.ts | 109 + .../__tests__/pinned-feed-credentials.test.ts | 111 + packages/transitous-core/package.json | 10 + .../src/feed-credential-requirements.ts | 124 + .../transitous-core/src/feed-credentials.ts | 185 + packages/transitous-core/src/index.ts | 13 + .../src/pinned-feed-credentials.ts | 90 + pnpm-lock.yaml | 3 + .../admin-transit-feed-credentials.md | 45 + scripts/check-subject-data.ts | 1 + scripts/database-test-groups.json | 4 +- scripts/ops-authority-inventory.json | 21 +- .../__tests__/transitous/internal.test.ts | 24 + services/data-manager/service.json | 13 +- .../src/jobs/transitous/internal.ts | 49 +- 134 files changed, 17777 insertions(+), 451 deletions(-) create mode 100644 apps/api/src/db/migrations/0048_admin_transit_credentials.sql create mode 100644 apps/api/src/db/migrations/meta/0048_snapshot.json create mode 100644 apps/api/src/db/transit-credential-preview-schema.ts create mode 100644 apps/api/src/routes/__tests__/admin-transit-credentials.test.ts create mode 100644 apps/api/src/routes/admin-transit-credentials.ts create mode 100644 apps/api/src/services/__tests__/fixtures/native-transit-credentials-api.ts create mode 100644 apps/api/src/services/transit-credential-application.ts create mode 100644 apps/api/src/services/transit-credential-authorization.ts create mode 100644 apps/api/src/services/transit-credential-preview-postgres.test.ts create mode 100644 apps/api/src/services/transit-credential-preview.test.ts create mode 100644 apps/api/src/services/transit-credential-preview.ts create mode 100644 apps/api/src/services/transit-credential-recovery.ts create mode 100644 apps/api/src/services/transit-credential-retention.ts create mode 100644 apps/api/src/services/transit-credential-revision.ts create mode 100644 apps/api/src/services/transit-feed-credential-participant.ts create mode 100644 apps/api/src/services/transit-feed-credentials-postgres.test.ts create mode 100644 apps/api/src/services/transit-feed-credentials.ts create mode 100644 apps/ops-agent/src/__fixtures__/pinned-feed-credentials.ts create mode 100644 apps/ops-agent/src/transit-credentials-runtime.test.ts create mode 100644 apps/ops-agent/src/transit-credentials-runtime.ts create mode 100644 apps/web/src/components/admin/services/TransitCredentialsSection.test.tsx create mode 100644 apps/web/src/components/admin/services/TransitCredentialsSection.tsx create mode 100644 packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200Configuration.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials400.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials401.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials403.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials409.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials413.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials415.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials429.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials500.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials503.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentialsBody.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials200EntriesItem.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials200Owner.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials400.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials401.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials403.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials409.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials413.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials415.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials429.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials500.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials503.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItem.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItemAction.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials400.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials401.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials403.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials409.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials413.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials415.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials429.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials500.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials503.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBodyChangesItem.ts create mode 100644 packages/core/src/build-artifacts/local-credential-template.test.ts create mode 100644 packages/core/src/build-artifacts/local-credential-template.ts create mode 100644 packages/core/src/ops/transit-credentials.test.ts create mode 100644 packages/transitous-core/__tests__/feed-credential-requirements.test.ts create mode 100644 packages/transitous-core/__tests__/feed-credentials.test.ts create mode 100644 packages/transitous-core/__tests__/pinned-feed-credentials.test.ts create mode 100644 packages/transitous-core/src/feed-credential-requirements.ts create mode 100644 packages/transitous-core/src/feed-credentials.ts create mode 100644 packages/transitous-core/src/pinned-feed-credentials.ts create mode 100644 release-notes/admin-transit-feed-credentials.md diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 631502f93..1ca998e6d 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -41603,7 +41603,6 @@ "convert-overpass", "link", "clean", - "generate-api-keys", "overture-sync", "overture-conflate", "search-index-build", @@ -59927,6 +59926,2063 @@ "x-openmapx-auth": "admin" } }, + "/api/admin/transit/credentials": { + "get": { + "operationId": "getAdminTransitCredentials", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configurationRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "owner": { + "type": "string", + "enum": [ + "vault", + "file", + "environment", + "unset" + ] + }, + "editable": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + }, + "appliedCredentialsRevision": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "pendingApplication": { + "type": "boolean" + }, + "entries": { + "maxItems": 11000, + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "required": { + "type": "boolean" + }, + "configured": { + "type": "boolean" + }, + "orphaned": { + "type": "boolean" + } + }, + "required": [ + "key", + "required", + "configured", + "orphaned" + ], + "additionalProperties": false + } + } + }, + "required": [ + "catalogRevisionId", + "credentialsRevision", + "configurationRevisionId", + "owner", + "editable", + "reason", + "appliedCredentialsRevision", + "pendingApplication", + "entries" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + }, + "patch": { + "operationId": "applyAdminTransitCredentials", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "previewId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "intentFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "transitProofToken": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "previewId", + "intentFingerprint", + "operationKey", + "transitProofToken" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configuration": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "credentialsRevision", + "configuration" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/transit/credentials/preview": { + "post": { + "operationId": "previewAdminTransitCredentials", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "expectedCredentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "changes": { + "minItems": 1, + "maxItems": 100, + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "action": { + "type": "string", + "enum": [ + "set" + ] + }, + "value": { + "type": "string", + "maxLength": 8192 + } + }, + "required": [ + "key", + "action", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "action": { + "type": "string", + "enum": [ + "delete" + ] + } + }, + "required": [ + "key", + "action" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "catalogRevisionId", + "expectedCredentialsRevision", + "expectedCurrentRevisionId", + "operationKey", + "changes" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "previewId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "intentFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "changes": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "action": { + "type": "string", + "enum": [ + "set", + "delete" + ] + } + }, + "required": [ + "key", + "action" + ], + "additionalProperties": false + } + }, + "preservedCount": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "mapBytes": { + "type": "integer", + "minimum": 2, + "maximum": 65536 + } + }, + "required": [ + "previewId", + "intentFingerprint", + "catalogRevisionId", + "credentialsRevision", + "expectedCurrentRevisionId", + "expiresAt", + "changes", + "preservedCount", + "mapBytes" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/ambient-places/manifest": { "get": { "operationId": "getApiAmbientPlacesManifest", diff --git a/apps/api/src/account/admin-invariant-postgres.test.ts b/apps/api/src/account/admin-invariant-postgres.test.ts index 2b6a2f0be..21d538e51 100644 --- a/apps/api/src/account/admin-invariant-postgres.test.ts +++ b/apps/api/src/account/admin-invariant-postgres.test.ts @@ -8,9 +8,11 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest import { accountAdminGuard, adminAuditLog, + adminJob, authPolicyApplication, authPolicyState, session, + transitCredentialPreview, user, } from "../db/schema"; import { authRoute } from "../routes/auth"; @@ -105,6 +107,7 @@ dbDescribe("atomic last administrator protection", () => { .select() .from(user) .where(and(eq(user.role, "admin"), eq(user.banned, false))); + it("persists a successful creation audit in the same transaction", async () => { const rows = await fixture.db .select() @@ -443,4 +446,64 @@ dbDescribe("atomic last administrator protection", () => { expect(after.requests).toHaveLength(before.requests.length + 1); expect(after.completedReceiptIds.size).toBe(before.completedReceiptIds.size); }); + it("keeps a running administrator's private credential input until its operation settles", async () => { + const owner = await actor("pending-credential-owner"), + deleter = await actor("credential-eraser"), + jobId = randomUUID(), + previewId = randomUUID(), + operationKey = `opk1_${randomUUID().replaceAll("-", "")}`; + vi.stubEnv("OPENMAPX_SECRETS_KEY", "73".repeat(32)); + const { encrypt } = await import("../services/secrets"), + encrypted = encrypt("synthetic private proposal"); + await fixture.db.insert(adminJob).values({ + id: jobId, + type: "configuration.apply", + status: "running", + createdBy: owner.id, + payload: { operationKey }, + result: { _immediateActorId: owner.id }, + }); + await fixture.db.insert(transitCredentialPreview).values({ + id: previewId, + actorId: owner.id, + operationKey, + intentFingerprint: "a".repeat(64), + ...encrypted, + expiresAt: new Date(Date.now() + 600000), + }); + const journalBefore = readFileSync(journal, "utf8"), + response = await app.inject({ + method: "POST", + url: "/api/auth/admin/remove-user", + headers: headers(deleter.cookie), + payload: { userId: owner.id }, + }); + expect(response.statusCode).toBe(409); + expect(response.json().code).toBe("ADMIN_OPERATION_IN_PROGRESS"); + expect(readFileSync(journal, "utf8")).toBe(journalBefore); + expect(await fixture.db.select().from(user).where(eq(user.id, owner.id))).toHaveLength(1); + expect( + await fixture.db + .select() + .from(transitCredentialPreview) + .where(eq(transitCredentialPreview.id, previewId)), + ).toHaveLength(1); + await fixture.db + .update(adminJob) + .set({ status: "failed", finishedAt: new Date() }) + .where(eq(adminJob.id, jobId)); + const settled = await app.inject({ + method: "POST", + url: "/api/auth/admin/remove-user", + headers: headers(deleter.cookie), + payload: { userId: owner.id }, + }); + expect(settled.statusCode).toBe(200); + expect( + await fixture.db + .select() + .from(transitCredentialPreview) + .where(eq(transitCredentialPreview.id, previewId)), + ).toHaveLength(0); + }, 30000); }); diff --git a/apps/api/src/account/admin-invariant.ts b/apps/api/src/account/admin-invariant.ts index 432051b35..aea8f282a 100644 --- a/apps/api/src/account/admin-invariant.ts +++ b/apps/api/src/account/admin-invariant.ts @@ -1,6 +1,7 @@ import { APIError } from "better-auth/api"; -import { eq } from "drizzle-orm"; -import { accountAdminGuard, user } from "../db/schema"; +import { and, eq, or, sql } from "drizzle-orm"; +import { accountAdminGuard, adminJob, user } from "../db/schema"; +import { noPendingReceiptDelivery, noUnsettledManagedOperation } from "../services/admin-job-state"; import { currentAccountMutation } from "./account-context"; import { type AuthDatabase, currentAuthDatabase } from "./auth-database"; export function isActiveFullAdmin(value: { role: string | null; banned: boolean | null }) { @@ -43,6 +44,34 @@ export async function assertAccountDeletionAllowed(targetId: string) { if (context.targetId !== null && context.targetId !== targetId) throw Error("Account deletion target changed"); await assertAdminCapacity(context.database, targetId, { deleted: true }); + // That existing guard holds the target user FOR UPDATE. New actor-attributed + // job inserts need the FK's KEY SHARE lock, so none can slip past this check + // before deletion commits. Pending private input must survive cancellation. + const [pending] = await context.database + .select({ id: adminJob.id }) + .from(adminJob) + .where( + and( + or( + eq(adminJob.createdBy, targetId), + sql`${adminJob.result}->>'_immediateActorId' = ${targetId}`, + ), + or( + sql`${adminJob.status} NOT IN ('success','failed','canceled')`, + sql`${adminJob.finishedAt} IS NULL`, + sql`NOT (${noPendingReceiptDelivery()})`, + sql`NOT (${noUnsettledManagedOperation()})`, + sql`${adminJob.result}->'_configurationCandidate' IS NOT NULL AND coalesce(${adminJob.result}->'_configurationCandidate'->>'phase','unknown') NOT IN ('settled','discarded')`, + ), + ), + ) + .limit(1); + if (pending) + throw new APIError("CONFLICT", { + code: "ADMIN_OPERATION_IN_PROGRESS", + message: + "Finish or cancel this administrator's instance operations in Activity before deleting the account", + }); } export function mapAccountConstraint(error: unknown): never { let current = error; diff --git a/apps/api/src/account/admin-reauthentication-postgres.test.ts b/apps/api/src/account/admin-reauthentication-postgres.test.ts index 6c65e8dbb..2b2cd6d78 100644 --- a/apps/api/src/account/admin-reauthentication-postgres.test.ts +++ b/apps/api/src/account/admin-reauthentication-postgres.test.ts @@ -109,6 +109,23 @@ dbDescribe("durable purpose-bound administrator action proof", () => { await fixture.db.update(user).set({ twoFactorEnabled: true }).where(eq(user.id, c.actorId)); await expect(finish(c)).rejects.toThrow(); }); + it("checks a purpose proof inside a real read-only candidate snapshot without consuming or locking it", async () => { + const c = await ceremony(), + input = await finish(c); + await fixture.db.transaction((tx) => service.check(input, tx), { + isolationLevel: "repeatable read", + accessMode: "read only", + }); + const [row] = await fixture.db + .select() + .from(adminActionReauthentication) + .where(eq(adminActionReauthentication.id, c.proof.challengeId)); + expect(row?.state).toBe("completed"); + await fixture.db.transaction((tx) => service.consume(input, tx)); + await expect( + fixture.db.transaction((tx) => service.check(input, tx), { accessMode: "read only" }), + ).rejects.toThrow(); + }); it("consumes once and binds nonce, purpose and target", async () => { const c = await ceremony(), input = await finish(c); diff --git a/apps/api/src/account/admin-reauthentication.ts b/apps/api/src/account/admin-reauthentication.ts index e5d6dcc48..1b0c9bf2a 100644 --- a/apps/api/src/account/admin-reauthentication.ts +++ b/apps/api/src/account/admin-reauthentication.ts @@ -41,13 +41,18 @@ export class AdminActionReauthentication { private readonly database: AuthDatabase = currentAuthDatabase(), private readonly now: () => Date = () => new Date(), ) {} - private async live(sessionId: string, tx: AuthDatabase, requireAssurance = true) { - const [row] = await tx + private async live( + sessionId: string, + tx: AuthDatabase, + requireAssurance = true, + lockRows = true, + ) { + const sessionQuery = tx .select({ session: sessions, user }) .from(sessions) .innerJoin(user, eq(sessions.userId, user.id)) - .where(eq(sessions.id, sessionId)) - .for("share"); + .where(eq(sessions.id, sessionId)); + const [row] = await (lockRows ? sessionQuery.for("share") : sessionQuery); if ( !row || row.session.expiresAt <= this.now() || @@ -56,11 +61,11 @@ export class AdminActionReauthentication { row.user.banned === true ) deny(); - const [assurance] = await tx + const assuranceQuery = tx .select() .from(sessionAuthAssurance) - .where(eq(sessionAuthAssurance.sessionId, sessionId)) - .for("share"); + .where(eq(sessionAuthAssurance.sessionId, sessionId)); + const [assurance] = await (lockRows ? assuranceQuery.for("share") : assuranceQuery); if ( requireAssurance && (!assurance || assurance.userId !== row.user.id || assurance.authenticatedAt > this.now()) @@ -68,10 +73,12 @@ export class AdminActionReauthentication { deny(); return { ...row, assurance }; } - private async policy(tx: AuthDatabase) { + private async policy(tx: AuthDatabase, lockRows = true) { // The table lock also protects the not-yet-activated singleton from an INSERT. - await tx.execute(sql`LOCK TABLE auth_policy_state IN SHARE MODE`); - await tx.select().from(authPolicyState).where(eq(authPolicyState.id, 1)).for("share"); + if (lockRows) { + await tx.execute(sql`LOCK TABLE auth_policy_state IN SHARE MODE`); + await tx.select().from(authPolicyState).where(eq(authPolicyState.id, 1)).for("share"); + } return loadAuthPolicy(tx); } async start(input: { @@ -185,6 +192,7 @@ export class AdminActionReauthentication { sessionId: string, tx: AuthDatabase, states: ReadonlyArray, + lockRows = true, ) { if ( !row || @@ -194,7 +202,7 @@ export class AdminActionReauthentication { row.startingSessionId === sessionId ) deny(); - const live = await this.live(sessionId, tx), + const live = await this.live(sessionId, tx, true, lockRows), a = live.assurance; if (!a || live.user.id !== row.actorId || a.authenticatedAt <= row.createdAt) deny(); const proof: ReauthenticationSession = { @@ -208,7 +216,7 @@ export class AdminActionReauthentication { if ( row.purpose === "platform.domain.apply" && (!isDomainRecoveryAssuranceSufficient(proof, a.originIndependentPasswordLogin) || - (await this.policy(tx)).revision !== row.authPolicyRevision) + (await this.policy(tx, lockRows)).revision !== row.authPolicyRevision) ) deny(); return proof; @@ -257,16 +265,20 @@ export class AdminActionReauthentication { const purposes = new Set(input.proofs.map((p) => p.purpose)); if (!purposes.has("credentials.import") || !purposes.has("transit.credentials")) deny(); } - await this.live(input.sessionId, tx); + await this.live(input.sessionId, tx, true, consume); for (const ref of [...input.proofs].sort((a, b) => a.challengeId.localeCompare(b.challengeId), )) { - const [row] = await tx - .select() - .from(challenge) - .where(eq(challenge.id, ref.challengeId)) - .for("update"); - const proof = await this.validate(row, input.nonce, input.sessionId, tx, ["completed"]); + const query = tx.select().from(challenge).where(eq(challenge.id, ref.challengeId)); + const [row] = await (consume ? query.for("update") : query); + const proof = await this.validate( + row, + input.nonce, + input.sessionId, + tx, + ["completed"], + consume, + ); if ( !row || row.completedSessionId !== input.sessionId || diff --git a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts index 0a4e5d15e..39178246f 100644 --- a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts +++ b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts @@ -86,6 +86,7 @@ describe.skipIf(skipDatabase)("user erasure constraints with PostgreSQL", () => "session.user_id:c", "session_auth_assurance.user_id:n", "share_link.user_id:c", + "transit_credential_preview.actor_id:c", "two_factor.user_id:c", ]); }); diff --git a/apps/api/src/db/migrations/0048_admin_transit_credentials.sql b/apps/api/src/db/migrations/0048_admin_transit_credentials.sql new file mode 100644 index 000000000..440b84d49 --- /dev/null +++ b/apps/api/src/db/migrations/0048_admin_transit_credentials.sql @@ -0,0 +1,22 @@ +CREATE TABLE "transit_credential_preview" ( + "id" uuid PRIMARY KEY NOT NULL, + "actor_id" text NOT NULL, + "operation_key" text NOT NULL, + "intent_fingerprint" text NOT NULL, + "ciphertext" text NOT NULL, + "iv" text NOT NULL, + "tag" text NOT NULL, + "authorization_ciphertext" text, + "authorization_iv" text, + "authorization_tag" text, + "candidate_id" uuid, + "credentials_revision" text, + "consumed_at" timestamp with time zone, + "expires_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "transit_credential_preview_bounds" CHECK (octet_length("transit_credential_preview"."ciphertext") <= 1500000 and length("transit_credential_preview"."intent_fingerprint") = 64 and length("transit_credential_preview"."iv") = 16 and length("transit_credential_preview"."tag") = 24) +); +--> statement-breakpoint +ALTER TABLE "transit_credential_preview" ADD CONSTRAINT "transit_credential_preview_actor_id_user_id_fk" FOREIGN KEY ("actor_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "transit_credential_preview_actor_key_uq" ON "transit_credential_preview" USING btree ("actor_id","operation_key");--> statement-breakpoint +CREATE INDEX "transit_credential_preview_expiry_idx" ON "transit_credential_preview" USING btree ("expires_at"); \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0048_snapshot.json b/apps/api/src/db/migrations/meta/0048_snapshot.json new file mode 100644 index 000000000..8a3e7cc09 --- /dev/null +++ b/apps/api/src/db/migrations/meta/0048_snapshot.json @@ -0,0 +1,9093 @@ +{ + "id": "8ee8e5bb-6745-4202-ba28-f7579b4c192f", + "prevId": "615f75f9-83ef-45de-a7be-63ec7174bae3", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_action_receipt": { + "name": "account_action_receipt", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mail_requested": { + "name": "mail_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_action_actor_time_idx": { + "name": "account_action_actor_time_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_action_target_time_idx": { + "name": "account_action_target_time_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_action_receipt_actor_id_user_id_fk": { + "name": "account_action_receipt_actor_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "account_action_receipt_target_id_user_id_fk": { + "name": "account_action_receipt_target_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_admin_guard": { + "name": "account_admin_guard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "active_count": { + "name": "active_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_admin_guard_singleton": { + "name": "account_admin_guard_singleton", + "value": "\"account_admin_guard\".\"id\"=1" + }, + "account_admin_guard_nonnegative": { + "name": "account_admin_guard_nonnegative", + "value": "\"account_admin_guard\".\"active_count\">=0" + } + }, + "isRLSEnabled": false + }, + "public.account_mail_outbox": { + "name": "account_mail_outbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_mail_action_kind_uq": { + "name": "account_mail_action_kind_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_mail_pending_idx": { + "name": "account_mail_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_mail_outbox_request_key_account_action_receipt_key_fk": { + "name": "account_mail_outbox_request_key_account_action_receipt_key_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "account_action_receipt", + "columnsFrom": [ + "request_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "account_mail_outbox_user_id_user_id_fk": { + "name": "account_mail_outbox_user_id_user_id_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_mail_state_check": { + "name": "account_mail_state_check", + "value": "\"account_mail_outbox\".\"state\" in ('queued','sending','sent','failed','unknown')" + }, + "account_mail_kind_check": { + "name": "account_mail_kind_check", + "value": "\"account_mail_outbox\".\"kind\" in ('verification','password_setup')" + } + }, + "isRLSEnabled": false + }, + "public.admin_action_reauthentication": { + "name": "admin_action_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admin_reauth_actor_expiry_idx": { + "name": "admin_reauth_actor_expiry_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admin_reauth_request_uq": { + "name": "admin_reauth_request_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_action_reauthentication_actor_id_user_id_fk": { + "name": "admin_action_reauthentication_actor_id_user_id_fk", + "tableFrom": "admin_action_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "admin_reauth_state_check": { + "name": "admin_reauth_state_check", + "value": "\"admin_action_reauthentication\".\"state\" in ('pending','completed','consumed','superseded','expired')" + }, + "admin_reauth_purpose_check": { + "name": "admin_reauth_purpose_check", + "value": "\"admin_action_reauthentication\".\"purpose\" in ('account.verify','account.password.set','account.reset.send','account.invite','credentials.export','credentials.import','transit.credentials','platform.domain.apply')" + } + }, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_application_actor_id_user_id_fk": { + "name": "auth_policy_application_actor_id_user_id_fk", + "tableFrom": "auth_policy_application", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_state_activated_by_user_id_fk": { + "name": "auth_policy_state_activated_by_user_id_fk", + "tableFrom": "auth_policy_state", + "tableTo": "user", + "columnsFrom": [ + "activated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_independent_password_login": { + "name": "origin_independent_password_login", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transit_credential_preview": { + "name": "transit_credential_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_ciphertext": { + "name": "authorization_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_iv": { + "name": "authorization_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_tag": { + "name": "authorization_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "credentials_revision": { + "name": "credentials_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "transit_credential_preview_actor_key_uq": { + "name": "transit_credential_preview_actor_key_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transit_credential_preview_expiry_idx": { + "name": "transit_credential_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "transit_credential_preview_actor_id_user_id_fk": { + "name": "transit_credential_preview_actor_id_user_id_fk", + "tableFrom": "transit_credential_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "transit_credential_preview_bounds": { + "name": "transit_credential_preview_bounds", + "value": "octet_length(\"transit_credential_preview\".\"ciphertext\") <= 1500000 and length(\"transit_credential_preview\".\"intent_fingerprint\") = 64 and length(\"transit_credential_preview\".\"iv\") = 16 and length(\"transit_credential_preview\".\"tag\") = 24" + } + }, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_application_receipt": { + "name": "integration_application_receipt", + "schema": "", + "columns": { + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "saved_binding_revision": { + "name": "saved_binding_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_application_target_time_idx": { + "name": "integration_application_target_time_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_application_actor_idx": { + "name": "integration_application_actor_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_application_receipt_actor_id_user_id_fk": { + "name": "integration_application_receipt_actor_id_user_id_fk", + "tableFrom": "integration_application_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_binding_revision": { + "name": "integration_binding_revision", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index dae076538..44b1eff07 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -337,6 +337,13 @@ "when": 1793203302497, "tag": "0047_admin_integration_application", "breakpoints": true + }, + { + "idx": 48, + "version": "7", + "when": 1793227919397, + "tag": "0048_admin_transit_credentials", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 1df84157c..c8ee464f1 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -117,6 +117,7 @@ export { type PersonalTimelineConnectionRow, personalTimelineConnection, } from "./timeline-connection-schema"; +export { transitCredentialPreview } from "./transit-credential-preview-schema"; // Personal vehicle garage and parked-location state export { type ParkedLocationRow, diff --git a/apps/api/src/db/transit-credential-preview-schema.ts b/apps/api/src/db/transit-credential-preview-schema.ts new file mode 100644 index 000000000..1d4c7335a --- /dev/null +++ b/apps/api/src/db/transit-credential-preview-schema.ts @@ -0,0 +1,34 @@ +/** Private encrypted transit credential proposals. Never exposed as admin-job payloads. */ +import { sql } from "drizzle-orm"; +import { check, index, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"; +import { user } from "./auth-schema"; +export const transitCredentialPreview = pgTable( + "transit_credential_preview", + { + id: uuid("id").primaryKey(), + actorId: text("actor_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + operationKey: text("operation_key").notNull(), + intentFingerprint: text("intent_fingerprint").notNull(), + ciphertext: text("ciphertext").notNull(), + iv: text("iv").notNull(), + tag: text("tag").notNull(), + authorizationCiphertext: text("authorization_ciphertext"), + authorizationIv: text("authorization_iv"), + authorizationTag: text("authorization_tag"), + candidateId: uuid("candidate_id"), + credentialsRevision: text("credentials_revision"), + consumedAt: timestamp("consumed_at", { withTimezone: true }), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), + }, + (t) => [ + uniqueIndex("transit_credential_preview_actor_key_uq").on(t.actorId, t.operationKey), + index("transit_credential_preview_expiry_idx").on(t.expiresAt), + check( + "transit_credential_preview_bounds", + sql`octet_length(${t.ciphertext}) <= 1500000 and length(${t.intentFingerprint}) = 64 and length(${t.iv}) = 16 and length(${t.tag}) = 24`, + ), + ], +); diff --git a/apps/api/src/privacy/catalogue.ts b/apps/api/src/privacy/catalogue.ts index d85e5995d..5e40134e6 100644 --- a/apps/api/src/privacy/catalogue.ts +++ b/apps/api/src/privacy/catalogue.ts @@ -427,6 +427,7 @@ const registrationDefinitions: SubjectDataRegistration[] = [ locators: [ { type: "updater_id", field: "integration_secret.updated_by" }, { type: "updater_id", field: "service_secret.updated_by" }, + { type: "actor_id", field: "transit_credential_preview.actor_id" }, ], purposes: ["service-provisioning", "security-and-abuse-prevention"], legalBases: ["contract", "legitimate-interest"], diff --git a/apps/api/src/privacy/openmapx-collectors.ts b/apps/api/src/privacy/openmapx-collectors.ts index cb1c0505c..2b904ee41 100644 --- a/apps/api/src/privacy/openmapx-collectors.ts +++ b/apps/api/src/privacy/openmapx-collectors.ts @@ -43,6 +43,7 @@ import { session, shareLink, systemSettings, + transitCredentialPreview, twoFactor, user, verification, @@ -1011,6 +1012,27 @@ export async function* streamOpenMapxRegistrationRecords( .orderBy(asc(serviceSecret.id)), { policyCodes: ["secret-value-redacted"] }, ); + yield* emit( + database + .select({ + id: transitCredentialPreview.id, + createdAt: transitCredentialPreview.createdAt, + expiresAt: transitCredentialPreview.expiresAt, + consumedAt: transitCredentialPreview.consumedAt, + }) + .from(transitCredentialPreview) + .where( + and( + eq(transitCredentialPreview.actorId, context.userId), + lte(transitCredentialPreview.createdAt, cutoff), + ), + ) + .orderBy(asc(transitCredentialPreview.id)), + { + policyCodes: ["secret-value-redacted"], + map: ({ id, ...row }) => ({ ...row, proposalDigest: pseudonymousIdentifier(id) }), + }, + ); return; case "system-setting-attribution": yield* emit( diff --git a/apps/api/src/routes/__tests__/admin-operations.test.ts b/apps/api/src/routes/__tests__/admin-operations.test.ts index 93df43246..e1155a318 100644 --- a/apps/api/src/routes/__tests__/admin-operations.test.ts +++ b/apps/api/src/routes/__tests__/admin-operations.test.ts @@ -12,7 +12,8 @@ const mockJobRunnerEnqueue = vi.fn().mockResolvedValue("job-123"); vi.mock("../../services/job-runner.js", () => ({ jobRunner: { enqueue: (...args: unknown[]) => mockJobRunnerEnqueue(...args) }, })); -vi.mock("@openmapx/core/server", () => ({ +vi.mock("@openmapx/core/server", async (importOriginal) => ({ + ...(await importOriginal()), findRepoRoot: () => "/repo", })); vi.mock("../../services/service-registry.js", () => ({ @@ -139,7 +140,7 @@ describe("POST /admin/operations/:id/run", () => { ); }); - it("rejects caller URL/output/argv/environment before fixed API-key generation", async () => { + it("rejects caller URL/output/argv/environment for the retired remote API-key generator", async () => { const response = await app.inject({ method: "POST", url: "/admin/operations/generate-api-keys/run", @@ -150,7 +151,7 @@ describe("POST /admin/operations/:id/run", () => { environment: { NODE_OPTIONS: "--require=/tmp/payload" }, }, }); - expect(response.statusCode).toBe(400); + expect(response.statusCode).toBe(404); expect(mockJobRunnerEnqueue).not.toHaveBeenCalled(); expect(mockWriteAuditLog).not.toHaveBeenCalled(); }); diff --git a/apps/api/src/routes/__tests__/admin-transit-credentials.test.ts b/apps/api/src/routes/__tests__/admin-transit-credentials.test.ts new file mode 100644 index 000000000..fd7a77fb5 --- /dev/null +++ b/apps/api/src/routes/__tests__/admin-transit-credentials.test.ts @@ -0,0 +1,231 @@ +import { randomUUID } from "node:crypto"; +import { createApiClient } from "@openmapx/core"; +import { + applyAdminTransitCredentials, + getAdminTransitCredentials, + previewAdminTransitCredentials, + unwrapAdminResponse, +} from "@openmapx/core/api-admin-sdk"; +import Fastify, { type FastifyInstance } from "fastify"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { mockAdminSession } from "../../test/auth"; + +const mocks = vi.hoisted(() => ({ + session: null as unknown, + inspect: vi.fn(), + preview: vi.fn(), + bind: vi.fn(), + apply: vi.fn(), +})); +vi.mock("../../auth", () => ({ auth: { api: { getSession: async () => mocks.session } } })); +vi.mock("../../services/transit-feed-credentials", () => ({ + inspectFeedCredentials: mocks.inspect, + previewFeedCredentialChanges: mocks.preview, +})); +vi.mock("../../services/transit-credential-authorization", () => ({ + bindTransitCredentialAuthorization: mocks.bind, +})); +vi.mock("../../services/job-runner", () => ({ + jobRunner: { applyImmediateTransitCredentials: mocks.apply }, +})); +vi.mock("../../services/ops-client", () => ({ + createApiOpsClient: () => ({}), + createDurableOpsKey: () => `opk1_${"a".repeat(32)}`, + executeAndWait: async () => ({ catalogRevisionId: "a".repeat(64) }), +})); + +import { ADMIN_REAUTH_COOKIE_NAME } from "../../account/admin-reauthentication"; +import { TransitCredentialConflict } from "../../services/transit-credential-preview"; +import { adminTransitCredentialsRoute } from "../admin-transit-credentials"; + +let app: FastifyInstance, client: ReturnType; +const body = () => ({ + catalogRevisionId: "a".repeat(64), + expectedCredentialsRevision: "b".repeat(64), + expectedCurrentRevisionId: null, + operationKey: `opk1_${"c".repeat(32)}`, + changes: [{ key: "de/A", action: "set" as const, value: "private-input" }], +}); +beforeEach(async () => { + vi.stubEnv("OPENMAPX_DISABLE_LOCALHOST_AUTH", "1"); + vi.clearAllMocks(); + mocks.session = mockAdminSession(); + mocks.inspect.mockResolvedValue({ + catalogRevisionId: "a".repeat(64), + credentialsRevision: "b".repeat(64), + configurationRevisionId: null, + owner: "unset", + editable: true, + reason: null, + appliedCredentialsRevision: null, + pendingApplication: false, + entries: [{ key: "de/A", required: true, configured: false, orphaned: false }], + }); + mocks.preview.mockResolvedValue({ + previewId: randomUUID(), + intentFingerprint: "d".repeat(64), + catalogRevisionId: "a".repeat(64), + credentialsRevision: "b".repeat(64), + expectedCurrentRevisionId: null, + expiresAt: new Date(Date.now() + 600000).toISOString(), + changes: [{ key: "de/A", action: "set" }], + preservedCount: 1, + mapBytes: 24, + }); + mocks.bind.mockResolvedValue(undefined); + mocks.apply.mockResolvedValue({ + credentialsRevision: "e".repeat(64), + configuration: { + outcome: "committed", + revisionId: `cfg1_${"f".repeat(43)}`, + enabledServiceIds: ["data-manager"], + followUp: "complete", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }, + }); + app = Fastify(); + await app.register(adminTransitCredentialsRoute, { prefix: "/api" }); + const baseUrl = await app.listen({ host: "127.0.0.1", port: 0 }); + client = createApiClient({ baseUrl }); +}); +afterEach(async () => { + await app?.close(); + vi.unstubAllEnvs(); +}); +it("uses the generated SDK over real HTTP and serializes only masked metadata and reviewed key actions", async () => { + expect(unwrapAdminResponse(await getAdminTransitCredentials({ client }))).toMatchObject({ + configurationRevisionId: null, + owner: "unset", + }); + const preview = unwrapAdminResponse(await previewAdminTransitCredentials(body(), { client })); + expect(JSON.stringify(preview)).not.toContain("private-input"); + expect(mocks.preview).toHaveBeenCalledWith( + expect.objectContaining({ actorId: "test-admin-id", changes: body().changes }), + ); + const response = await applyAdminTransitCredentials( + { + previewId: preview.previewId, + intentFingerprint: preview.intentFingerprint, + operationKey: body().operationKey, + transitProofToken: randomUUID(), + }, + { + client, + headers: { + cookie: `${ADMIN_REAUTH_COOKIE_NAME}=${Buffer.alloc(32, 9).toString("base64url")}`, + }, + }, + ); + expect(unwrapAdminResponse(response)).toMatchObject({ credentialsRevision: "e".repeat(64) }); + expect(mocks.bind).toHaveBeenCalledWith( + expect.objectContaining({ actorId: "test-admin-id", sessionId: "test-session-id" }), + ); + expect(mocks.apply).toHaveBeenCalledOnce(); +}); +it.each([ + null, + { ...mockAdminSession(), user: { ...mockAdminSession().user, role: "support_admin" } }, +])("enforces the actual full-admin guard before credential reads or writes", async (session) => { + mocks.session = session; + const response = await app.inject({ + method: "POST", + url: "/api/admin/transit/credentials/preview", + payload: body(), + }); + expect(response.statusCode).toBe(session ? 403 : 401); + expect(mocks.preview).not.toHaveBeenCalled(); + expect(mocks.inspect).not.toHaveBeenCalled(); +}); +it.each(["impersonated", "loopback"])("refuses mutation from %s authority", async (mode) => { + const session = mockAdminSession(); + if (mode === "loopback") session.user.id = "loopback"; + else session.session.impersonatedBy = "another-admin"; + mocks.session = session; + expect( + ( + await app.inject({ + method: "POST", + url: "/api/admin/transit/credentials/preview", + payload: body(), + }) + ).statusCode, + ).toBe(403); + expect(mocks.preview).not.toHaveBeenCalled(); +}); +it("rejects substitution fields and missing nonce before binding or applying", async () => { + const input = { + previewId: randomUUID(), + intentFingerprint: "d".repeat(64), + operationKey: body().operationKey, + transitProofToken: randomUUID(), + }; + expect( + ( + await app.inject({ + method: "PATCH", + url: "/api/admin/transit/credentials", + payload: { ...input, changes: body().changes }, + }) + ).statusCode, + ).toBe(400); + expect( + (await app.inject({ method: "PATCH", url: "/api/admin/transit/credentials", payload: input })) + .statusCode, + ).toBe(409); + expect(mocks.bind).not.toHaveBeenCalled(); + expect(mocks.apply).not.toHaveBeenCalled(); +}); +it("returns sanitized conflicts and failures with no-store headers", async () => { + mocks.preview.mockRejectedValueOnce(new TransitCredentialConflict()); + let response = await app.inject({ + method: "POST", + url: "/api/admin/transit/credentials/preview", + payload: body(), + }); + expect(response.statusCode).toBe(409); + expect(response.headers["cache-control"]).toContain("no-store"); + mocks.preview.mockRejectedValueOnce(Error("private-input /private/host/path")); + response = await app.inject({ + method: "POST", + url: "/api/admin/transit/credentials/preview", + payload: body(), + }); + expect(response.statusCode).toBe(503); + expect(response.body).not.toContain("private-input"); + expect(response.body).not.toContain("/private/"); +}); +it("bounds the actual parser and shared per-admin mutation rate", async () => { + expect( + ( + await app.inject({ + method: "POST", + url: "/api/admin/transit/credentials/preview", + payload: { + ...body(), + changes: [{ key: "de/A", action: "set", value: "x".repeat(1024 * 1024) }], + }, + }) + ).statusCode, + ).toBe(413); + for (let i = 0; i < 12; i++) + expect( + ( + await app.inject({ + method: "POST", + url: "/api/admin/transit/credentials/preview", + payload: body(), + }) + ).statusCode, + ).toBe(200); + expect( + ( + await app.inject({ + method: "POST", + url: "/api/admin/transit/credentials/preview", + payload: body(), + }) + ).statusCode, + ).toBe(429); +}); diff --git a/apps/api/src/routes/admin-transit-credentials.ts b/apps/api/src/routes/admin-transit-credentials.ts new file mode 100644 index 000000000..a28dac41d --- /dev/null +++ b/apps/api/src/routes/admin-transit-credentials.ts @@ -0,0 +1,112 @@ +import { transitCredentialOperations } from "@openmapx/core/api-contracts"; +import type { FastifyInstance } from "fastify"; +import { AdminReauthenticationError } from "../account/admin-reauthentication"; +import { JobObservationPending } from "../services/admin-job-failure"; +import { jobRunner } from "../services/job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client"; +import { bindTransitCredentialAuthorization } from "../services/transit-credential-authorization"; +import { TransitCredentialConflict } from "../services/transit-credential-preview"; +import { + inspectFeedCredentials, + previewFeedCredentialChanges, +} from "../services/transit-feed-credentials"; +import { RateLimiter } from "../utils/rate-limit"; +import { getAdminSession, requireAdmin } from "../utils/require-admin"; +import { declareRouteAuth } from "../utils/route-auth"; +import { createAdminRouteRegistrar } from "./admin-contract"; +import { adminReauthNonce } from "./admin-reauthentication"; + +const register = createAdminRouteRegistrar(transitCredentialOperations, { + tag: "data-manager-admin", + validationPhase: "preHandler", +}); +export async function adminTransitCredentialsRoute(app: FastifyInstance) { + declareRouteAuth(app, "admin"); + app.addHook("preHandler", async (request) => { + request.adminSession = await requireAdmin(request); + }); + app.addHook("onRequest", async (_request, reply) => { + reply.header("Cache-Control", "private, no-store"); + }); + app.addHook("onRoute", (route) => { + if (route.method === "POST" || route.method === "PATCH") route.bodyLimit = 1024 * 1024; + }); + const limits = new RateLimiter({ + max: 12, + windowMs: 60_000, + keyFn: (request) => getAdminSession(request).user.id, + }); + app.addHook("onClose", async () => limits.destroy()); + const failure = (error: unknown) => ({ + status: + error instanceof TransitCredentialConflict || error instanceof AdminReauthenticationError + ? 409 + : 503, + error: + error instanceof TransitCredentialConflict || error instanceof AdminReauthenticationError + ? error.code + : error instanceof JobObservationPending + ? "Credential application is pending reconciliation; inspect Activity before retrying" + : "Transit credential administration is unavailable", + }); + register(app, "getAdminTransitCredentials", async (_request, reply) => { + try { + const pair = await executeAndWait( + createApiOpsClient(), + { kind: "catalogPins.inspect" }, + createDurableOpsKey("transit.credentials.current", "current"), + ); + if (!pair.catalogRevisionId) + return reply + .code(503) + .send({ error: "Select and activate catalog pins before managing feed credentials" }); + return await inspectFeedCredentials({ catalogRevisionId: pair.catalogRevisionId }); + } catch (error) { + const result = failure(error); + return reply.code(result.status).send({ error: result.error }); + } + }); + register( + app, + "previewAdminTransitCredentials", + async (request, reply) => { + const actor = getAdminSession(request); + if (actor.user.id === "loopback" || actor.session.impersonatedBy) + return reply.code(403).send({ error: "A real full administrator session is required" }); + try { + return await previewFeedCredentialChanges({ ...request.body, actorId: actor.user.id }); + } catch (error) { + const result = failure(error); + return reply.code(result.status).send({ error: result.error }); + } + }, + { preHandler: limits.preHandler() }, + ); + register( + app, + "applyAdminTransitCredentials", + async (request, reply) => { + const actor = getAdminSession(request), + nonce = adminReauthNonce(request.headers.cookie); + if (actor.user.id === "loopback" || actor.session.impersonatedBy) + return reply.code(403).send({ error: "A real full administrator session is required" }); + if (!nonce) return reply.code(409).send({ error: "REAUTHENTICATION_FAILED" }); + try { + await bindTransitCredentialAuthorization({ + ...request.body, + actorId: actor.user.id, + sessionId: actor.session.id, + nonce: nonce.toString("base64url"), + }); + return await jobRunner.applyImmediateTransitCredentials({ + ...request.body, + actorId: actor.user.id, + }); + } catch (error) { + const result = failure(error); + return reply.code(result.status).send({ error: result.error }); + } + }, + { preHandler: limits.preHandler() }, + ); +} diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index d4559faec..6820fc739 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -14,6 +14,7 @@ import { adminReauthenticationRoute } from "./admin-reauthentication"; import { adminServicesRoute } from "./admin-services"; import { adminSettingsRoute } from "./admin-settings"; import { adminSystemRoute } from "./admin-system"; +import { adminTransitCredentialsRoute } from "./admin-transit-credentials"; import { ambientPlacesRoute } from "./ambient-places"; import { attributionRoute } from "./attribution"; import { authRoute } from "./auth"; @@ -117,6 +118,7 @@ export async function registerCoreRoutes( await server.register(statusRoute, { prefix: "/api" }); await server.register(adminRoute, { prefix: "/api" }); await server.register(adminServicesRoute, { prefix: "/api" }); + await server.register(adminTransitCredentialsRoute, { prefix: "/api" }); await server.register(adminOperationsRoute, { prefix: "/api" }); await server.register(adminCoverageRoute, { prefix: "/api" }); await server.register(adminActivityRoute, { prefix: "/api" }); diff --git a/apps/api/src/services/__tests__/admin-job-handlers.test.ts b/apps/api/src/services/__tests__/admin-job-handlers.test.ts index f6e792de4..c4efc6a73 100644 --- a/apps/api/src/services/__tests__/admin-job-handlers.test.ts +++ b/apps/api/src/services/__tests__/admin-job-handlers.test.ts @@ -20,7 +20,8 @@ const runAdminOperationMock = vi.hoisted(() => // Stub the heavy import chain so the argv-guard tests don't bring in the real // service registry, postgres client, or CLI runner. We only exercise pure // validation helpers. -vi.mock("@openmapx/core/server", () => ({ +vi.mock("@openmapx/core/server", async (importOriginal) => ({ + ...(await importOriginal()), findRepoRoot: () => "/repo", })); vi.mock("@openmapx/core", () => ({ @@ -335,16 +336,12 @@ describe("admin job handlers", () => { }, }), ), - ).rejects.toThrow(/Invalid input/); + ).rejects.toThrow(); expect(runAdminOperationMock).not.toHaveBeenCalled(); const ctx = dataCtx({ operation: "generate-api-keys", version: 1, input: {} }); - await handleDataOperationJob(ctx); - expect(runAdminOperationMock).toHaveBeenCalledWith( - ctx, - { kind: "data.generateApiKeys", catalogRevisionId: "transitous-fixed-v1" }, - "admin-job.data.generate-api-keys", - ); + await expect(handleDataOperationJob(ctx)).rejects.toThrow(); + expect(runAdminOperationMock).not.toHaveBeenCalled(); }); }); diff --git a/apps/api/src/services/__tests__/fixtures/native-transit-credentials-api.ts b/apps/api/src/services/__tests__/fixtures/native-transit-credentials-api.ts new file mode 100644 index 000000000..6cd13081d --- /dev/null +++ b/apps/api/src/services/__tests__/fixtures/native-transit-credentials-api.ts @@ -0,0 +1,31 @@ +/** Native, synthetic acceptance driver. Input and output contain opaque identities only. */ +import { sql } from "../../../db"; +import { jobRunner } from "../../job-runner"; +import { initServiceRegistry } from "../../service-registry"; + +let bytes = ""; +for await (const chunk of process.stdin) bytes += chunk; +const input = JSON.parse(bytes) as { + actorId: string; + previewId: string; + intentFingerprint: string; + operationKey: string; + reconcile?: boolean; +}; +let exit = 0; +try { + await initServiceRegistry({ strict: true }); + if (input.reconcile) await jobRunner.reconcile({ limit: 1 }); + else await jobRunner.applyImmediateTransitCredentials(input); + const rows = + await sql`SELECT status FROM admin_job WHERE payload->>'operationKey'=${input.operationKey}`; + if (rows.length !== 1 || rows[0]!.status !== "success") exit = 2; + process.stdout.write(JSON.stringify({ status: rows[0]?.status ?? "missing" }) + "\n"); +} catch { + exit = 2; + process.stdout.write(JSON.stringify({ status: "unresolved" }) + "\n"); +} finally { + await new Promise((resolve) => setTimeout(resolve, 0)); + await sql.end({ timeout: 2 }); + process.exit(exit); +} diff --git a/apps/api/src/services/activity-retention.ts b/apps/api/src/services/activity-retention.ts index 33c0e7b48..0c0f8f283 100644 --- a/apps/api/src/services/activity-retention.ts +++ b/apps/api/src/services/activity-retention.ts @@ -6,6 +6,7 @@ import { withPreservationRetentionLock, } from "../privacy/preservation.js"; import { noPendingReceiptDelivery, noUnsettledManagedOperation } from "./admin-job-state"; +import { pruneTransitCredentialPreviews } from "./transit-credential-retention"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -67,8 +68,9 @@ export async function pruneCompletedJobs(days: number): Promise { } const cutoff = new Date(Date.now() - days * MS_PER_DAY); // adminJobLog has ON DELETE CASCADE so deleting parent rows clears logs. - const result = await withPreservationRetentionLock(db, (tx) => - tx + const result = await withPreservationRetentionLock(db, async (tx) => { + await pruneTransitCredentialPreviews(tx); + return tx .delete(adminJob) .where( and( @@ -83,8 +85,8 @@ export async function pruneCompletedJobs(days: number): Promise { }), ), ) - .returning({ id: adminJob.id }), - ); + .returning({ id: adminJob.id }); + }); return result.length; } diff --git a/apps/api/src/services/admin-operation-catalog.ts b/apps/api/src/services/admin-operation-catalog.ts index 63f29c70d..3dfb41682 100644 --- a/apps/api/src/services/admin-operation-catalog.ts +++ b/apps/api/src/services/admin-operation-catalog.ts @@ -23,7 +23,6 @@ export type AdminOperationId = | "convert-overpass" | "link" | "clean" - | "generate-api-keys" | "overture-sync" | "overture-conflate" | "search-index-build" @@ -107,8 +106,6 @@ const requiredRegionField = (helpText: string): AdminOperationField => ({ const withRegion = (label: string, regionId: string | undefined) => regionId ? `${label} ${regionId}` : `${label} (configured default region)`; -const TRANSITOUS_API_KEY_CATALOG_REVISION = "transitous-fixed-v1"; - export const ADMIN_OPERATIONS: readonly AnyAdminOperationDefinition[] = [ defineOperation({ id: "download-osm", @@ -241,24 +238,6 @@ export const ADMIN_OPERATIONS: readonly AnyAdminOperationDefinition[] = [ "Services depending on that data need a rebuild afterwards", ], }), - defineOperation({ - id: "generate-api-keys", - version: 1, - group: "transit", - title: "Generate Transitous API-Key Template", - description: "Writes the API-key template for the pinned Transitous catalog revision.", - risk: "normal", - input: z.strictObject({}), - fields: [], - defaults: {}, - effect: () => ({ - kind: "data.generateApiKeys", - catalogRevisionId: TRANSITOUS_API_KEY_CATALOG_REVISION, - }), - preview: () => [ - `Generate the API-key template for catalog revision ${TRANSITOUS_API_KEY_CATALOG_REVISION}`, - ], - }), defineOperation({ id: "overture-sync", version: 1, diff --git a/apps/api/src/services/configuration-commit-participant.ts b/apps/api/src/services/configuration-commit-participant.ts index bcacb93fa..c92ae6925 100644 --- a/apps/api/src/services/configuration-commit-participant.ts +++ b/apps/api/src/services/configuration-commit-participant.ts @@ -17,10 +17,12 @@ export async function withConfigurationCommitParticipant( "./configuration-migration/legacy-integration-import-intent" ); const { isIntegrationBindingParticipant } = await import("./integration-binding-participant"); + const { isTransitCredentialParticipant } = await import("./transit-feed-credential-participant"); // Exact installed built-in issuer, never a registration or caller callback. if ( (!isLegacyIntegrationImportParticipant(participant, writer) && - !isIntegrationBindingParticipant(participant, writer)) || + !isIntegrationBindingParticipant(participant, writer) && + !isTransitCredentialParticipant(participant, writer)) || participant.actorId !== writer.actorId || participant.operationKey !== writer.operationKey || (state.participant && state.participant !== participant) diff --git a/apps/api/src/services/configuration-input.ts b/apps/api/src/services/configuration-input.ts index 6f2b56009..291bec10d 100644 --- a/apps/api/src/services/configuration-input.ts +++ b/apps/api/src/services/configuration-input.ts @@ -17,6 +17,7 @@ import { import { projectIntegrationConfig } from "./integration-config-projection"; import { resolveAllServiceConfigsStrict } from "./service-config-resolver"; import { resolveServiceVaultSecretsStrict } from "./service-secrets"; +import { transitCredentialRevision } from "./transit-credential-revision"; /** * Everything a render needs from the database, for every registered service: @@ -86,5 +87,16 @@ export async function buildConfigurationInput( }), integrationConfigs, serviceSecrets, + ...(context.serviceSecretRows.some( + (row) => row.serviceId === "data-manager" && row.key === "TRANSITOUS_API_KEYS", + ) + ? { + transitCredentialsRevision: transitCredentialRevision( + context.serviceSecretRows.find( + (row) => row.serviceId === "data-manager" && row.key === "TRANSITOUS_API_KEYS", + ), + ), + } + : {}), }); } diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 8141cb274..b1fdf8402 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -382,12 +382,15 @@ class AdminJobRunner { "./prospective-configuration" ), integrationIds = await retainedConfigurationIntegrationIds(ctx, actorId, operationKey), - result = await withAdministrativeConfigurationJob(ctx, () => + originalResult = await withAdministrativeConfigurationJob(ctx, () => withAdministrativeConfigurationWriter( { actorId, operationKey, integrationIds, signal: ctx.signal }, () => resumeRetainedConfigurationCandidate(), ), - ); + ), + result = await ( + await import("./transit-credential-recovery") + ).recoveredTransitCredentialResult(ctx, originalResult); const [canceled] = await db .select() .from(adminJob) @@ -588,6 +591,50 @@ class AdminJobRunner { ); } + applyImmediateTransitCredentials(options: { + actorId: string; + previewId: string; + intentFingerprint: string; + operationKey: string; + signal?: AbortSignal; + }) { + return this.immediate( + { + ...options, + type: "configuration.apply", + intent: { + purpose: "transit-credentials", + previewId: options.previewId, + intentFingerprint: options.intentFingerprint, + }, + }, + async (ctx) => { + const { runTransitCredentialChange } = await import("./transit-credential-application"); + const value = await runTransitCredentialChange(ctx, options); + return { + value, + result: { configuration: value.configuration, transitCredentials: value }, + failed: !completeTrustedConfigurationApply(value.configuration), + }; + }, + (result) => { + const value = result.transitCredentials as + | { credentialsRevision?: unknown; configuration?: unknown } + | undefined; + if ( + !value || + typeof value.credentialsRevision !== "string" || + !/^[a-f0-9]{64}$/.test(value.credentialsRevision) + ) + throw new JobObservationPending("recovery_required"); + return { + credentialsRevision: value.credentialsRevision, + configuration: parseTrustedConfigurationApplyOutcome(value.configuration), + }; + }, + ); + } + applyImmediateIntegrationBinding(options: { input: import("./integration-binding-participant").IntegrationBindingInput; actorId: string; diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index b8d26d2a3..0aa630c01 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -1793,7 +1793,13 @@ async function restoreRetainedMaintenanceCandidate(): Promise change.kind !== "service-secret" || change.serviceId !== "data-manager", ) @@ -1993,6 +1999,19 @@ async function restoreRetainedMaintenanceCandidate(): Promise row.serviceId === change.serviceId && row.key === change.key, + ) + : undefined; rows.serviceSecretRows = rows.serviceSecretRows.filter( (row) => row.serviceId !== change.serviceId || row.key !== change.key, ); if (change.action === "set") rows.serviceSecretRows.push({ - id: randomUUID(), + id: managedPrior?.id ?? randomUUID(), serviceId: change.serviceId, key: change.key, ...encrypt(change.value), @@ -3022,6 +3054,12 @@ export async function prepareProspectiveConfiguration(input: { : null; input.signal?.throwIfAborted(); const overlay = overlaySchema.parse(input.overlay); + if (overlay.purpose === "transit-credentials") { + const { assertTransitCredentialOverlay } = await import( + "./transit-feed-credential-participant" + ); + assertTransitCredentialOverlay(writer, overlay); + } validateOverlay(overlay, writer); if ( coordinator && @@ -3328,6 +3366,7 @@ export async function prepareProspectiveConfiguration(input: { ); await retainPreparedTrustedConfigurationCandidate(publisher, record); const state = validatedConfigurationCandidateState(candidate); + let preflightStage: "preview" | "coordinator" | "capacity" = "preview"; try { state.preview = await executeAndWait( createApiOpsClient(), @@ -3342,9 +3381,12 @@ export async function prepareProspectiveConfiguration(input: { createDurableOpsKey("configuration.hardlinks.preview", input.operationKey), { signal: input.signal }, ); + preflightStage = "coordinator"; if ( overlay.coordinatorMode === "require-running" && - (state.preview.entries || coordinator) && + (state.preview.entries || + coordinator || + (maintenanceId && input.followUp === "service-apply")) && state.preview.coordinatorState !== "running" ) throw Error("Configuration coordinator prerequisite unavailable"); @@ -3356,6 +3398,7 @@ export async function prepareProspectiveConfiguration(input: { !["running", "stopped"].includes(state.preview.coordinatorState)) ) throw Error("Configuration coordinator prerequisite unavailable"); + preflightStage = "capacity"; assertMaintenanceIntentCapacity(state, owner.job.checkpointResult); } catch (error) { if (coordinator?.action === "update") @@ -3374,6 +3417,11 @@ export async function prepareProspectiveConfiguration(input: { } throw new AdminJobFailure("Configuration candidate preflight failed", { errorCode: "configuration_preflight_failed", + configurationPreflight: { + stage: preflightStage, + coordinatorState: state.preview?.coordinatorState ?? null, + errorClass: error instanceof ApiOpsError ? error.errorClass : "local_validation", + }, }); } candidateStages.set(candidate, "prepared"); @@ -3603,7 +3651,11 @@ async function readSavedRowVersions( return receipts; } -async function persistOverlay(tx: ConfigurationWriteTransaction, state: CandidateState) { +async function persistOverlay( + tx: ConfigurationWriteTransaction, + state: CandidateState, + candidate: ValidatedConfigurationCandidate, +) { if (!state.rows || !state.sourceRows) throw Error("Original prospective desired state required"); const version = (row: { id: string } | undefined) => { const value = (row as { _rowVersion?: unknown } | undefined)?._rowVersion; @@ -3670,8 +3722,15 @@ async function persistOverlay(tx: ConfigurationWriteTransaction, state: Candidat owner.participant?.managedServiceSecretKeys.some( (entry) => entry.serviceId === change.serviceId && entry.key === change.key, ) - ) + ) { + const { transitParticipantPatch } = await import("./transit-feed-credential-participant"); + await persistPreparedFeedCredentialPatch({ + patch: transitParticipantPatch(owner.participant, state.writer), + candidate, + writer: state.writer, + }); continue; + } const prior = state.sourceRows.serviceSecretRows.find( (row) => row.serviceId === change.serviceId && row.key === change.key, ); @@ -3769,7 +3828,8 @@ async function verifyCurrentPreview( if ( preview.previewDigest !== state.preview?.previewDigest || (state.overlay.coordinatorMode === "require-running" && - preview.entries && + (preview.entries || + (state.record.maintenanceId && state.record.apiFollowUp === "service-apply")) && preview.coordinatorState !== "running") ) rejectUnissuedCandidate(candidate, "Configuration preview changed"); @@ -4084,7 +4144,7 @@ export async function commitProspectiveConfigurationMaintenance( ) throw Error("Original prepared maintenance preflight changed"); await owner.participant?.commit(candidate, state.writer, tx); - const rowVersions = await persistOverlay(tx, state), + const rowVersions = await persistOverlay(tx, state, candidate), body = { kind: "configurationMaintenance.decide" as const, maintenanceId: record.maintenanceId!, @@ -4206,7 +4266,15 @@ export async function persistAndApplyProspectiveConfiguration( owner.participant?.mode === "preview" ) throw Error("Configuration candidate persistence refused"); - if (["migration", "transit-credentials"].includes(state.overlay.purpose) && !owner.participant) + if ( + ["migration", "transit-credentials"].includes(state.overlay.purpose) && + !owner.participant && + !( + state.overlay.purpose === "transit-credentials" && + stage === "committed" && + recoveredCommittedCandidates.has(candidate) + ) + ) throw Error("Configuration commit participant unavailable"); if (!state.record.maintenanceId && stage === "committed") return applyCommittedOrdinaryCandidate(candidate, options.signal); @@ -4443,7 +4511,7 @@ export async function persistAndApplyProspectiveConfiguration( await verifyConfigurationCommitParticipant(candidate, state.writer); await verifyCurrentPreview(candidate, state, options.signal); await owner.participant?.commit(candidate, state.writer, tx); - rowVersions = await persistOverlay(tx, state); + rowVersions = await persistOverlay(tx, state, candidate); const rows = await tx .update(schema.adminJob) .set({ @@ -5752,3 +5820,17 @@ export async function completeIntegrationActivation( await retireSettledProspectiveConfiguration(candidate); return integrationApplyReceiptSchema.parse(receipt); } + +/** Fixed managed-key dispatch over THIS producer's nominal candidate, inside its original transaction. */ +export async function persistPreparedFeedCredentialPatch(input: { + patch: import("./transit-feed-credential-participant").PreparedFeedCredentialPatch; + candidate: ValidatedConfigurationCandidate; + writer: AdministrativeConfigurationWriter; +}): Promise<{ credentialsRevision: string }> { + const state = validatedConfigurationCandidateState(input.candidate); + if (state.writer !== input.writer) throw Error("Original transit credential candidate required"); + const { persistPreparedFeedCredentialRow } = await import( + "./transit-feed-credential-participant" + ); + return persistPreparedFeedCredentialRow({ ...input, candidateState: state }); +} diff --git a/apps/api/src/services/transit-credential-application.ts b/apps/api/src/services/transit-credential-application.ts new file mode 100644 index 000000000..2d4d26ba6 --- /dev/null +++ b/apps/api/src/services/transit-credential-application.ts @@ -0,0 +1,41 @@ +import { + withAdministrativeConfigurationJob, + withAdministrativeConfigurationWriter, +} from "./administrative-configuration-writer-owner"; +import type { JobContext } from "./job-runner"; +import { readTransitCredentialAuthorization } from "./transit-credential-authorization"; +import type { FeedCredentialApplyResult } from "./transit-feed-credentials"; +import { + applyFeedCredentialChanges, + withTransitCredentialProofContext, +} from "./transit-feed-credentials"; +export async function runTransitCredentialChange( + ctx: JobContext, + input: { actorId: string; previewId: string; intentFingerprint: string; operationKey: string }, +): Promise { + const binding = await readTransitCredentialAuthorization(input); + return withTransitCredentialProofContext( + { + actorId: binding.actorId, + sessionId: binding.sessionId, + nonce: Buffer.from(binding.nonce, "base64url"), + }, + () => + withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { + actorId: input.actorId, + operationKey: input.operationKey, + integrationIds: [], + signal: ctx.signal, + }, + () => + applyFeedCredentialChanges({ + ...input, + authorization: { kind: "direct", transitProofToken: binding.transitProofToken }, + signal: ctx.signal, + }), + ), + ), + ); +} diff --git a/apps/api/src/services/transit-credential-authorization.ts b/apps/api/src/services/transit-credential-authorization.ts new file mode 100644 index 000000000..d13a90d42 --- /dev/null +++ b/apps/api/src/services/transit-credential-authorization.ts @@ -0,0 +1,94 @@ +/** Private authenticated request binding, distinct from public job metadata. */ +import { canonicalJson } from "@openmapx/core/ops"; +import { and, eq, sql } from "drizzle-orm"; +import { AdminActionReauthentication } from "../account/admin-reauthentication"; +import { db } from "../db"; +import { transitCredentialPreview } from "../db/schema"; +import { decrypt, encrypt } from "./secrets"; +import { readFeedCredentialPreview, TransitCredentialConflict } from "./transit-credential-preview"; +export interface TransitCredentialRequestBinding { + actorId: string; + previewId: string; + intentFingerprint: string; + operationKey: string; + transitProofToken: string; + sessionId: string; + nonce: string; +} +export async function bindTransitCredentialAuthorization(input: TransitCredentialRequestBinding) { + return db.transaction(async (tx) => { + const stored = await readFeedCredentialPreview(input, tx); + if (stored.row.authorizationCiphertext) { + const original = decode(stored.row); + if ( + original.transitProofToken !== input.transitProofToken || + original.actorId !== input.actorId + ) + throw new TransitCredentialConflict(); + if (!stored.row.consumedAt && canonicalJson(original) !== canonicalJson(input)) + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_AUTHORIZATION_CHANGED"); + return; + } + if (stored.row.expiresAt <= new Date() || stored.row.consumedAt) + throw new TransitCredentialConflict(); + await new AdminActionReauthentication(tx).check( + { + challengeId: input.transitProofToken, + purpose: "transit.credentials", + targetId: stored.private.preview.catalogRevisionId, + intentFingerprint: input.intentFingerprint, + sessionId: input.sessionId, + nonce: Buffer.from(input.nonce, "base64url"), + }, + tx, + ); + const bytes = encrypt(canonicalJson(input)); + const marked = await tx + .update(transitCredentialPreview) + .set({ + authorizationCiphertext: bytes.ciphertext, + authorizationIv: bytes.iv, + authorizationTag: bytes.tag, + }) + .where( + and( + eq(transitCredentialPreview.id, input.previewId), + sql`${transitCredentialPreview.authorizationCiphertext} is null`, + ), + ) + .returning({ id: transitCredentialPreview.id }); + if (marked.length !== 1) throw new TransitCredentialConflict(); + }); +} +function decode( + row: typeof transitCredentialPreview.$inferSelect, +): TransitCredentialRequestBinding { + try { + if (!row.authorizationCiphertext || !row.authorizationIv || !row.authorizationTag) + throw new Error(); + const input: TransitCredentialRequestBinding = JSON.parse( + decrypt(row.authorizationCiphertext, row.authorizationIv, row.authorizationTag), + ); + if ( + input.actorId !== row.actorId || + input.previewId !== row.id || + input.intentFingerprint !== row.intentFingerprint || + input.operationKey !== row.operationKey || + typeof input.sessionId !== "string" || + !/^[A-Za-z0-9_-]{43}$/.test(input.nonce) || + typeof input.transitProofToken !== "string" + ) + throw new Error(); + return input; + } catch { + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_AUTHORIZATION_UNAVAILABLE"); + } +} +export async function readTransitCredentialAuthorization(input: { + actorId: string; + previewId: string; + intentFingerprint: string; + operationKey: string; +}) { + return decode((await readFeedCredentialPreview(input)).row); +} diff --git a/apps/api/src/services/transit-credential-preview-postgres.test.ts b/apps/api/src/services/transit-credential-preview-postgres.test.ts new file mode 100644 index 000000000..604a6ff90 --- /dev/null +++ b/apps/api/src/services/transit-credential-preview-postgres.test.ts @@ -0,0 +1,149 @@ +import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const state = vi.hoisted(() => ({ + fixture: null as unknown as Awaited>, +})); +vi.mock("../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); + +import { transitCredentialPreview, user } from "../db/schema"; +import { + readFeedCredentialPreview, + retainFeedCredentialPreview, +} from "./transit-credential-preview"; + +const enabled = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"; +beforeAll(async () => { + if (!enabled) return; + vi.stubEnv("OPENMAPX_SECRETS_KEY", "57".repeat(32)); + state.fixture = await createMigratedPostgresFixture(); + await state.fixture.db.insert(user).values({ + id: "transit-preview-admin", + name: "Fixture", + email: "transit-preview@example.test", + emailVerified: true, + role: "admin", + createdAt: new Date(), + updatedAt: new Date(), + }); +}, 90000); +afterAll(async () => { + await state.fixture?.stop(); + vi.unstubAllEnvs(); +}); +it.skipIf(!enabled)( + "retains encrypted immutable actor-bound changes with a value-free replayable public preview", + async () => { + const input = { + actorId: "transit-preview-admin", + operationKey: "preview-immutable-1", + catalogRevisionId: "a".repeat(64), + credentialsRevision: "b".repeat(64), + expectedCurrentRevisionId: null, + changes: [ + { key: "de/required", action: "set" as const, value: "private-transit-credential" }, + ], + preservedCount: 2, + mapBytes: 53, + }; + const preview = await retainFeedCredentialPreview(input); + expect(JSON.stringify(preview)).not.toContain(input.changes[0]!.value); + const [row] = await state.fixture.db.select().from(transitCredentialPreview); + expect(JSON.stringify(row)).not.toContain(input.changes[0]!.value); + expect(await retainFeedCredentialPreview(input)).toEqual(preview); + expect( + ( + await readFeedCredentialPreview({ + actorId: input.actorId, + previewId: preview.previewId, + intentFingerprint: preview.intentFingerprint, + operationKey: input.operationKey, + }) + ).private.changes, + ).toEqual(input.changes); + await expect( + retainFeedCredentialPreview({ + ...input, + changes: [{ ...input.changes[0]!, value: "substitution" }], + }), + ).rejects.toThrow("TRANSIT_CREDENTIALS_CHANGED"); + await expect( + readFeedCredentialPreview({ + actorId: "foreign", + previewId: preview.previewId, + intentFingerprint: preview.intentFingerprint, + operationKey: input.operationKey, + }), + ).rejects.toThrow("PREVIEW_UNAVAILABLE"); + }, + 30000, +); +it.skipIf(!enabled)( + "reclaims expired completed proposals while retaining bound unresolved ones", + async () => { + const { adminJob } = await import("../db/schema"), + { eq, inArray } = await import("drizzle-orm"), + { encrypt } = await import("./secrets"), + { randomUUID } = await import("node:crypto"); + const make = (operationKey: string) => + retainFeedCredentialPreview({ + actorId: "transit-preview-admin", + operationKey, + catalogRevisionId: "a".repeat(64), + credentialsRevision: "b".repeat(64), + expectedCurrentRevisionId: null, + changes: [{ key: "de/A", action: "set", value: "synthetic-private" }], + preservedCount: 0, + mapBytes: 29, + }); + const completedKey = `opk1_${randomUUID().replaceAll("-", "")}`, + pendingKey = `opk1_${randomUUID().replaceAll("-", "")}`, + completed = await make(completedKey), + pending = await make(pendingKey), + expired = new Date(Date.now() - 4 * 86400000), + binding = encrypt("{}"); + for (const preview of [completed, pending]) + await state.fixture.db + .update(transitCredentialPreview) + .set({ + expiresAt: expired, + authorizationCiphertext: binding.ciphertext, + authorizationIv: binding.iv, + authorizationTag: binding.tag, + }) + .where(eq(transitCredentialPreview.id, preview.previewId)); + await state.fixture.db.insert(adminJob).values([ + { + id: randomUUID(), + type: "configuration.apply", + status: "failed", + createdBy: "transit-preview-admin", + finishedAt: expired, + payload: { operationKey: completedKey }, + result: { _immediateActorId: "transit-preview-admin" }, + }, + { + id: randomUUID(), + type: "configuration.apply", + status: "running", + createdBy: "transit-preview-admin", + payload: { operationKey: pendingKey }, + result: { _immediateActorId: "transit-preview-admin" }, + }, + ]); + await make(`opk1_${randomUUID().replaceAll("-", "")}`); + const rows = await state.fixture.db + .select({ id: transitCredentialPreview.id }) + .from(transitCredentialPreview) + .where(inArray(transitCredentialPreview.id, [completed.previewId, pending.previewId])); + expect(rows).toEqual([{ id: pending.previewId }]); + }, + 30000, +); diff --git a/apps/api/src/services/transit-credential-preview.test.ts b/apps/api/src/services/transit-credential-preview.test.ts new file mode 100644 index 000000000..209f1e07a --- /dev/null +++ b/apps/api/src/services/transit-credential-preview.test.ts @@ -0,0 +1,22 @@ +import { expect, it } from "vitest"; +import { feedCredentialIntent } from "./transit-credential-preview"; + +it("public intent binds opaque preview identity and actions, never hashes a credential value", () => { + const base = { + previewId: "opaque-random-id", + actorId: "admin", + catalogRevisionId: "catalog", + credentialsRevision: "random-encrypted-row", + expectedCurrentRevisionId: null, + changes: [{ key: "de/a", action: "set" as const, value: "first" }], + }; + expect(feedCredentialIntent(base)).toBe( + feedCredentialIntent({ ...base, changes: [{ ...base.changes[0]!, value: "second" }] }), + ); + expect(feedCredentialIntent(base)).not.toBe( + feedCredentialIntent({ ...base, previewId: "another-opaque-random-id" }), + ); + expect(feedCredentialIntent(base)).not.toBe( + feedCredentialIntent({ ...base, changes: [{ key: "de/a", action: "delete" }] }), + ); +}); diff --git a/apps/api/src/services/transit-credential-preview.ts b/apps/api/src/services/transit-credential-preview.ts new file mode 100644 index 000000000..6120b97a0 --- /dev/null +++ b/apps/api/src/services/transit-credential-preview.ts @@ -0,0 +1,205 @@ +/** Private immutable encrypted proposals; public intents contain no value-derived hashes. */ +import { createHash, randomUUID } from "node:crypto"; +import { canonicalJson } from "@openmapx/core/ops"; +import type { FeedCredentialChange } from "@openmapx/transitous-core/feed-credentials"; +import { and, eq, lt, sql } from "drizzle-orm"; +import { db } from "../db"; +import { transitCredentialPreview } from "../db/schema"; +import { withPreservationRetentionLock } from "../privacy/preservation"; +import { decrypt, encrypt } from "./secrets"; +import { pruneTransitCredentialPreviews } from "./transit-credential-retention"; +export class TransitCredentialConflict extends Error { + constructor(readonly code = "TRANSIT_CREDENTIALS_CHANGED") { + super(code); + } +} +export interface FeedCredentialChangePreview { + previewId: string; + intentFingerprint: string; + catalogRevisionId: string; + credentialsRevision: string; + expectedCurrentRevisionId: string | null; + expiresAt: string; + changes: Array<{ key: string; action: "set" | "delete" }>; + preservedCount: number; + mapBytes: number; +} +export interface PrivateFeedCredentialPreview { + version: 1; + actorId: string; + operationKey: string; + preview: FeedCredentialChangePreview; + changes: FeedCredentialChange[]; +} +export function feedCredentialIntent(input: { + previewId: string; + actorId: string; + catalogRevisionId: string; + credentialsRevision: string; + expectedCurrentRevisionId: string | null; + changes: readonly FeedCredentialChange[]; +}): string { + return createHash("sha256") + .update( + canonicalJson({ + purpose: "transit.credentials", + previewId: input.previewId, + actorId: input.actorId, + catalogRevisionId: input.catalogRevisionId, + credentialsRevision: input.credentialsRevision, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + changes: input.changes + .map(({ key, action }) => ({ key, action })) + .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)), + }), + ) + .digest("hex"); +} +function decoded(row: typeof transitCredentialPreview.$inferSelect): PrivateFeedCredentialPreview { + try { + const value: PrivateFeedCredentialPreview = JSON.parse( + decrypt(row.ciphertext, row.iv, row.tag), + ); + if ( + value.version !== 1 || + value.actorId !== row.actorId || + value.operationKey !== row.operationKey || + value.preview.previewId !== row.id || + value.preview.intentFingerprint !== row.intentFingerprint || + value.preview.expiresAt !== row.expiresAt.toISOString() || + feedCredentialIntent({ + ...value.preview, + previewId: row.id, + actorId: row.actorId, + credentialsRevision: value.preview.credentialsRevision, + changes: value.changes, + }) !== row.intentFingerprint + ) + throw new Error(); + return value; + } catch { + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_PREVIEW_UNAVAILABLE"); + } +} +export async function retainFeedCredentialPreview( + input: Omit & { + catalogRevisionId: string; + credentialsRevision: string; + expectedCurrentRevisionId: string | null; + preservedCount: number; + mapBytes: number; + }, +): Promise { + return withPreservationRetentionLock(db, async (tx) => { + await tx.execute(sql`select pg_advisory_xact_lock(71954,1)`); + await pruneTransitCredentialPreviews(tx); + const [existing] = await tx + .select() + .from(transitCredentialPreview) + .where( + and( + eq(transitCredentialPreview.actorId, input.actorId), + eq(transitCredentialPreview.operationKey, input.operationKey), + ), + ) + .for("share"); + if (existing) { + const value = decoded(existing); + if ( + existing.expiresAt <= new Date() || + canonicalJson({ + catalogRevisionId: value.preview.catalogRevisionId, + credentialsRevision: value.preview.credentialsRevision, + expectedCurrentRevisionId: value.preview.expectedCurrentRevisionId, + changes: value.changes, + }) !== + canonicalJson({ + catalogRevisionId: input.catalogRevisionId, + credentialsRevision: input.credentialsRevision, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + changes: input.changes, + }) + ) + throw new TransitCredentialConflict(); + return value.preview; + } + const [{ total, actor }] = await tx + .select({ + total: sql`count(*)::int`, + actor: sql`count(*) filter (where ${transitCredentialPreview.actorId} = ${input.actorId})::int`, + }) + .from(transitCredentialPreview); + if (total >= 4096 || actor >= 100) + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_PREVIEW_CAPACITY"); + const previewId = randomUUID(), + expiresAt = new Date(Date.now() + 10 * 60_000).toISOString(); + const preview: FeedCredentialChangePreview = { + previewId, + intentFingerprint: feedCredentialIntent({ ...input, previewId }), + catalogRevisionId: input.catalogRevisionId, + credentialsRevision: input.credentialsRevision, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + expiresAt, + changes: input.changes.map(({ key, action }) => ({ key, action })), + preservedCount: input.preservedCount, + mapBytes: input.mapBytes, + }; + const privateValue: PrivateFeedCredentialPreview = { + version: 1, + actorId: input.actorId, + operationKey: input.operationKey, + preview, + changes: input.changes, + }; + const plaintext = canonicalJson(privateValue); + if (Buffer.byteLength(plaintext) > 1024 * 1024) + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_PREVIEW_CAPACITY"); + await tx.insert(transitCredentialPreview).values({ + id: previewId, + actorId: input.actorId, + operationKey: input.operationKey, + intentFingerprint: preview.intentFingerprint, + expiresAt: new Date(expiresAt), + ...encrypt(plaintext), + }); + return preview; + }); +} +export async function readFeedCredentialPreview( + input: { actorId: string; previewId: string; intentFingerprint: string; operationKey: string }, + connection: Pick = db, +) { + const [row] = await connection + .select() + .from(transitCredentialPreview) + .where( + and( + eq(transitCredentialPreview.id, input.previewId), + eq(transitCredentialPreview.actorId, input.actorId), + eq(transitCredentialPreview.operationKey, input.operationKey), + eq(transitCredentialPreview.intentFingerprint, input.intentFingerprint), + ), + ) + .for("share"); + if (!row) throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_PREVIEW_UNAVAILABLE"); + return { row, private: decoded(row) }; +} + +/** Private recovery lookup. The caller must still verify the original job/candidate. */ +export async function readRetainedFeedCredentialPreview( + input: { actorId: string; operationKey: string }, + connection: Pick = db, +) { + const rows = await connection + .select() + .from(transitCredentialPreview) + .where( + and( + eq(transitCredentialPreview.actorId, input.actorId), + eq(transitCredentialPreview.operationKey, input.operationKey), + ), + ) + .limit(2); + if (rows.length !== 1) return undefined; + return { row: rows[0]!, private: decoded(rows[0]!) }; +} diff --git a/apps/api/src/services/transit-credential-recovery.ts b/apps/api/src/services/transit-credential-recovery.ts new file mode 100644 index 000000000..4410d3dd7 --- /dev/null +++ b/apps/api/src/services/transit-credential-recovery.ts @@ -0,0 +1,83 @@ +/** Observation/continuation of one already saved nominal managed-map candidate. + * This never constructs a participant, consumes a proof, saves, encrypts or seals. */ +import { createHmac } from "node:crypto"; +import { canonicalJson, readOpsTokenFile } from "@openmapx/core/ops"; +import type { db } from "../db"; +import { JobObservationPending } from "./admin-job-failure"; +import type { JobContext } from "./job-runner"; +import type { ProspectiveConfigurationOverlay } from "./prospective-configuration"; +import { readRetainedFeedCredentialPreview } from "./transit-credential-preview"; + +export async function assertSavedTransitCredentialCandidate(input: { + actorId: string; + operationKey: string; + candidateId: string; + revision: string | undefined; + overlay: ProspectiveConfigurationOverlay; + connection: Pick; +}) { + const retained = await readRetainedFeedCredentialPreview(input, input.connection), + change = input.overlay.changes[0]; + if ( + !retained || + !retained.row.consumedAt || + retained.row.candidateId !== input.candidateId || + retained.row.credentialsRevision !== input.revision || + input.overlay.purpose !== "transit-credentials" || + input.overlay.coordinatorMode !== "require-running" || + input.overlay.changes.length !== 1 || + change?.kind !== "service-secret" || + change.serviceId !== "data-manager" || + change.key !== "TRANSITOUS_API_KEYS" || + change.action !== "set" + ) + throw new JobObservationPending("recovery_required"); + const map: Record = JSON.parse(change.value); + if ( + retained.private.changes.some((selected) => + selected.action === "set" + ? map[selected.key] !== selected.value + : Object.hasOwn(map, selected.key), + ) + ) + throw new JobObservationPending("recovery_required"); +} +export async function recoveredTransitCredentialResult( + ctx: JobContext, + result: Record, +): Promise> { + const actorId = ctx.checkpointResult?._immediateActorId, + operationKey = ctx.payload.operationKey; + if (typeof actorId !== "string" || typeof operationKey !== "string") return result; + const retained = await readRetainedFeedCredentialPreview({ actorId, operationKey }); + if (!retained) return result; + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""), + digest = createHmac("sha256", token) + .update("openmapx-immediate-configuration-intent-v1\0") + .update( + canonicalJson({ + purpose: "transit-credentials", + previewId: retained.row.id, + intentFingerprint: retained.row.intentFingerprint, + }), + ) + .digest("hex"); + if (digest !== ctx.checkpointResult?._immediateIntentDigest) + throw new JobObservationPending("invalid_identity"); + const checkpoint = ctx.checkpointResult?._configurationCandidate as + | { candidateId?: string } + | undefined; + if ( + retained.row.consumedAt && + (retained.row.candidateId !== checkpoint?.candidateId || !retained.row.credentialsRevision) + ) + throw new JobObservationPending("invalid_identity"); + return { + ...result, + transitCredentials: { + credentialsRevision: + retained.row.credentialsRevision ?? retained.private.preview.credentialsRevision, + configuration: result.configuration, + }, + }; +} diff --git a/apps/api/src/services/transit-credential-retention.ts b/apps/api/src/services/transit-credential-retention.ts new file mode 100644 index 000000000..c5ba05547 --- /dev/null +++ b/apps/api/src/services/transit-credential-retention.ts @@ -0,0 +1,31 @@ +/** Reclaim only expired inert proposals or proven terminal original jobs. + * Unknown jobs and any undelivered/unsettled effects keep their private inputs. */ +import { and, lt, or, sql } from "drizzle-orm"; +import type { db } from "../db"; +import { adminJob, transitCredentialPreview as preview } from "../db/schema"; +import { preservationAllowsDeletion } from "../privacy/preservation"; +import { noPendingReceiptDelivery, noUnsettledManagedOperation } from "./admin-job-state"; +export async function pruneTransitCredentialPreviews( + connection: Pick, + now = new Date(), +) { + const matchingJob = sql`${adminJob.payload}->>'operationKey' = ${preview.operationKey} AND ${adminJob.result}->>'_immediateActorId' = ${preview.actorId}`; + const rows = await connection + .delete(preview) + .where( + and( + lt(preview.expiresAt, now), + preservationAllowsDeletion({ + registrationId: "secret-update-attribution", + candidateTimestamp: preview.createdAt, + subjectIds: [preview.actorId], + }), + or( + sql`${preview.consumedAt} IS NULL AND NOT EXISTS (SELECT 1 FROM ${adminJob} WHERE ${matchingJob})`, + sql`EXISTS (SELECT 1 FROM ${adminJob} WHERE ${matchingJob} AND ${adminJob.status} IN ('success','failed','canceled') AND ${adminJob.finishedAt} < ${new Date(now.getTime() - 72 * 3600000).toISOString()}::timestamptz AND ${noPendingReceiptDelivery()} AND ${noUnsettledManagedOperation()} AND (${preview.consumedAt} IS NULL OR (${adminJob.result}->'_configurationCandidate'->>'candidateId' = ${preview.candidateId}::text AND ${adminJob.result}->'transitCredentials'->>'credentialsRevision' = ${preview.credentialsRevision})))`, + ), + ), + ) + .returning({ id: preview.id }); + return rows.length; +} diff --git a/apps/api/src/services/transit-credential-revision.ts b/apps/api/src/services/transit-credential-revision.ts new file mode 100644 index 000000000..3e0175de5 --- /dev/null +++ b/apps/api/src/services/transit-credential-revision.ts @@ -0,0 +1,15 @@ +/** Randomized encrypted-row identity; never hash the plaintext map. */ +import { createHash } from "node:crypto"; +export function transitCredentialRevision( + row: { id: string; ciphertext: string; iv: string; tag: string } | undefined, +): string { + return createHash("sha256") + .update( + JSON.stringify( + row + ? ["transit-credentials-v1", row.id, row.ciphertext, row.iv, row.tag] + : ["transit-credentials-v1", "absent"], + ), + ) + .digest("hex"); +} diff --git a/apps/api/src/services/transit-feed-credential-participant.ts b/apps/api/src/services/transit-feed-credential-participant.ts new file mode 100644 index 000000000..d4e8f054f --- /dev/null +++ b/apps/api/src/services/transit-feed-credential-participant.ts @@ -0,0 +1,521 @@ +/** One finite managed-map adapter in the existing configuration writer/transaction. */ + +import { AsyncLocalStorage } from "node:async_hooks"; +import { randomUUID } from "node:crypto"; +import { canonicalJson } from "@openmapx/core/ops"; +import { + type FeedCredentialChange, + parseFeedCredentialMapJson, + validateFeedCredentialMap, +} from "@openmapx/transitous-core/feed-credentials"; +import { and, eq, sql } from "drizzle-orm"; +import { AdminActionReauthentication } from "../account/admin-reauthentication"; +import type { AuthDatabase } from "../account/auth-database"; +import { adminAuditLog, serviceSecret, transitCredentialPreview } from "../db/schema"; +import { + type AdministrativeConfigurationWriter, + administrativeConfigurationState, + type ConfigurationCommitParticipant, + type ConfigurationWriteTransaction, + currentAdministrativeConfigurationWriter, + withAdministrativeConfigurationRead, +} from "./administrative-configuration-writer-owner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import type { + ProspectiveConfigurationOverlay, + ValidatedConfigurationCandidate, +} from "./prospective-configuration"; +import { decrypt } from "./secrets"; +import { + type FeedCredentialChangePreview, + readFeedCredentialPreview, + retainFeedCredentialPreview, + TransitCredentialConflict, +} from "./transit-credential-preview"; +import { transitCredentialRevision } from "./transit-credential-revision"; +import type { TrustedConfigurationApplyOutcome } from "./trusted-config-outcome"; + +export type { FeedCredentialChangePreview } from "./transit-credential-preview"; + +const SERVICE = "data-manager" as const, + KEY = "TRANSITOUS_API_KEYS" as const; +export type FeedCredentialAuthorization = + | { kind: "direct"; transitProofToken: string } + | { + kind: "credential-import"; + transitProofToken: string; + importProofToken: string; + importPreviewId: string; + importFingerprint: string; + }; +export interface FeedCredentialApplyResult { + credentialsRevision: string; + configuration: TrustedConfigurationApplyOutcome; +} +interface ServerProofContext { + actorId: string; + sessionId: string; + nonce: Buffer; +} +const proofContexts = new AsyncLocalStorage(); +/** Set only by authenticated HTTP/job adapters, never from a request body. */ +export function withTransitCredentialProofContext( + context: ServerProofContext, + run: () => Promise, +) { + return proofContexts.run(context, run); +} +async function inspect(catalogRevisionId: string, identity: string, signal?: AbortSignal) { + return executeAndWait( + createApiOpsClient(), + { kind: "transitCredentials.inspect", catalogRevisionId }, + createDurableOpsKey("transit.credentials.inspect", identity), + { signal }, + ); +} +function rowFrom(read: { serviceSecretRows: readonly (typeof serviceSecret.$inferSelect)[] }) { + return read.serviceSecretRows.find((row) => row.serviceId === SERVICE && row.key === KEY); +} +function values(row: typeof serviceSecret.$inferSelect | undefined) { + return row ? parseFeedCredentialMapJson(decrypt(row.ciphertext, row.iv, row.tag)) : {}; +} +export async function inspectFeedCredentials(input: { + catalogRevisionId: string; + signal?: AbortSignal; +}) { + const host = await inspect( + input.catalogRevisionId, + `read:${input.catalogRevisionId}`, + input.signal, + ); + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("transit.credentials.configuration", input.catalogRevisionId), + { signal: input.signal }, + ); + return withAdministrativeConfigurationRead({}, async (read) => { + const row = rowFrom(read), + map = values(row), + revision = transitCredentialRevision(row); + if (host.owner === "vault" && !row) + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_UNAVAILABLE"); + const keys = [ + ...new Set([...host.requiredKeys, ...Object.keys(map), ...host.retainedKeys]), + ].sort(); + const configured = new Set( + host.editable ? Object.keys(map).filter((key) => map[key] !== "") : host.configuredKeys, + ); + return { + catalogRevisionId: input.catalogRevisionId, + credentialsRevision: + host.owner === "file" ? (host.hostCredentialsRevision ?? revision) : revision, + configurationRevisionId: current.revisionId, + owner: host.owner === "unset" && row ? ("vault" as const) : host.owner, + editable: host.editable, + reason: host.reason, + appliedCredentialsRevision: host.appliedCredentialsRevision, + pendingApplication: Boolean(row) && host.appliedCredentialsRevision !== revision, + entries: keys.map((key) => ({ + key, + required: host.requiredKeys.includes(key), + configured: configured.has(key), + orphaned: !host.requiredKeys.includes(key), + })), + }; + }); +} +interface ChangeInput { + actorId: string; + catalogRevisionId: string; + expectedCredentialsRevision: string; + expectedCurrentRevisionId: string | null; + changes: FeedCredentialChange[]; + operationKey: string; + signal?: AbortSignal; +} +async function validateInput(input: ChangeInput) { + const host = await inspect(input.catalogRevisionId, input.operationKey, input.signal); + if (!host.editable) throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_HOST_OWNED"); + return withAdministrativeConfigurationRead({}, async (read) => { + const row = rowFrom(read), + revision = transitCredentialRevision(row), + existing = values(row); + if (revision !== input.expectedCredentialsRevision) throw new TransitCredentialConflict(); + for (const change of input.changes) + if (!Object.hasOwn(existing, change.key) && !host.requiredKeys.includes(change.key)) + throw new TransitCredentialConflict("TRANSIT_CREDENTIAL_KEY_NOT_IN_CATALOG"); + const next = validateFeedCredentialMap(existing, input.changes); + return { host, row, revision, existing, next }; + }); +} +export async function previewFeedCredentialChanges( + input: ChangeInput, +): Promise { + const value = await validateInput(input); + const current = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.inspect" }, + createDurableOpsKey("transit.credentials.preview", input.operationKey), + { signal: input.signal }, + ); + if (current.revisionId !== input.expectedCurrentRevisionId) throw new TransitCredentialConflict(); + return retainFeedCredentialPreview({ + ...input, + credentialsRevision: value.revision, + preservedCount: Object.keys(value.existing).filter( + (key) => !input.changes.some((change) => change.key === key), + ).length, + mapBytes: value.next.utf8Bytes, + }); +} +declare const patchBrand: unique symbol; +export interface PreparedFeedCredentialPatch { + readonly [patchBrand]: true; + readonly serviceId: typeof SERVICE; + readonly key: typeof KEY; + readonly catalogRevisionId: string; + readonly expectedCredentialsRevision: string; + readonly expectedCurrentRevisionId: string | null; + readonly importPreviewId: string; + readonly operationKey: string; + readonly overlayChange: Extract< + ProspectiveConfigurationOverlay["changes"][number], + { kind: "service-secret" } + >; +} +interface PatchState { + writer: AdministrativeConfigurationWriter; + input: ChangeInput; + patch: PreparedFeedCredentialPatch; + row: typeof serviceSecret.$inferSelect | undefined; + authorization?: FeedCredentialAuthorization; + proof?: ServerProofContext; + preview?: FeedCredentialChangePreview; + committedRevision?: string; + candidateId?: string; +} +const patches = new WeakMap(), + participants = new WeakMap(); +export async function prepareFeedCredentialImportPatch( + input: ChangeInput & { importPreviewId: string }, +): Promise { + const writer = currentAdministrativeConfigurationWriter(); + if (writer.actorId !== input.actorId || writer.operationKey !== input.operationKey) + throw new TransitCredentialConflict(); + const value = await validateInput(input); + const overlayChange = Object.freeze({ + kind: "service-secret" as const, + serviceId: SERVICE, + key: KEY, + action: "set" as const, + value: value.next.canonicalJson, + }); + const patch = Object.freeze({ + serviceId: SERVICE, + key: KEY, + catalogRevisionId: input.catalogRevisionId, + expectedCredentialsRevision: input.expectedCredentialsRevision, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + importPreviewId: input.importPreviewId, + operationKey: input.operationKey, + overlayChange, + }) as PreparedFeedCredentialPatch; + patches.set(patch, { + writer, + input: structuredClone({ ...input, signal: undefined }), + patch, + row: value.row, + }); + return patch; +} +function owned(patch: PreparedFeedCredentialPatch, writer: AdministrativeConfigurationWriter) { + const state = patches.get(patch); + if ( + !state || + state.writer !== writer || + writer.actorId !== state.input.actorId || + writer.operationKey !== patch.operationKey + ) + throw new TransitCredentialConflict(); + administrativeConfigurationState(writer); + return state; +} +function proofInput(state: PatchState) { + const authorization = state.authorization, + context = state.proof, + preview = state.preview; + if (!authorization || !context || !preview) + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_PROOF_REQUIRED"); + const transit = { + challengeId: authorization.transitProofToken, + purpose: "transit.credentials" as const, + targetId: state.input.catalogRevisionId, + intentFingerprint: preview.intentFingerprint, + }; + return { + nonce: context.nonce, + sessionId: context.sessionId, + proofs: + authorization.kind === "direct" + ? [transit] + : [ + transit, + { + challengeId: authorization.importProofToken, + purpose: "credentials.import" as const, + targetId: authorization.importPreviewId, + intentFingerprint: authorization.importFingerprint, + }, + ], + }; +} +async function proof(state: PatchState, consume: boolean, tx: AuthDatabase) { + const input = proofInput(state), + verifier = new AdminActionReauthentication(tx); + if (input.proofs.length === 1) { + const single = { ...input.proofs[0]!, nonce: input.nonce, sessionId: input.sessionId }; + await (consume ? verifier.consume(single, tx) : verifier.check(single, tx)); + } else await (consume ? verifier.consumeMany(input, tx) : verifier.checkMany(input, tx)); +} +/** 04.6 calls this once before preparing its whole mixed candidate; no nested participant. */ +export async function authorizePreparedFeedCredentialPatch(input: { + patch: PreparedFeedCredentialPatch; + writer: AdministrativeConfigurationWriter; + authorization: FeedCredentialAuthorization; + preview: FeedCredentialChangePreview; +}) { + const state = owned(input.patch, input.writer), + context = proofContexts.getStore(); + if ( + !context || + context.actorId !== state.input.actorId || + input.preview.catalogRevisionId !== state.input.catalogRevisionId || + input.preview.credentialsRevision !== state.input.expectedCredentialsRevision || + input.preview.expectedCurrentRevisionId !== state.input.expectedCurrentRevisionId || + (input.authorization.kind === "direct" + ? input.preview.previewId + : input.authorization.importPreviewId) !== state.patch.importPreviewId + ) + throw new TransitCredentialConflict(); + const stored = await readFeedCredentialPreview( + { + actorId: state.input.actorId, + previewId: input.preview.previewId, + intentFingerprint: input.preview.intentFingerprint, + operationKey: state.input.operationKey, + }, + administrativeConfigurationState(input.writer).connection, + ); + if ( + stored.row.consumedAt || + stored.row.expiresAt <= new Date() || + canonicalJson(stored.private.preview) !== canonicalJson(input.preview) || + canonicalJson(stored.private.changes) !== canonicalJson(state.input.changes) + ) + throw new TransitCredentialConflict(); + state.authorization = Object.freeze({ ...input.authorization }); + state.proof = { + actorId: context.actorId, + sessionId: context.sessionId, + nonce: Buffer.from(context.nonce), + }; + state.preview = Object.freeze({ ...input.preview }); + await proof(state, false, administrativeConfigurationState(input.writer).connection); +} +export function isTransitCredentialParticipant( + participant: ConfigurationCommitParticipant, + writer: AdministrativeConfigurationWriter, +) { + return participants.get(participant)?.writer === writer; +} +export function assertTransitCredentialOverlay( + writer: AdministrativeConfigurationWriter, + overlay: unknown, +) { + const owner = administrativeConfigurationState(writer), + state = owner.participant && participants.get(owner.participant); + if ( + !state || + state.writer !== writer || + canonicalJson(overlay) !== + canonicalJson({ + purpose: "transit-credentials", + coordinatorMode: "require-running", + changes: [state.patch.overlayChange], + }) + ) + throw new TransitCredentialConflict(); +} +export function transitCredentialParticipation( + patch: PreparedFeedCredentialPatch, + writer: AdministrativeConfigurationWriter, +) { + const state = owned(patch, writer); + if (!state.authorization) + throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_PROOF_REQUIRED"); + const verify = async (candidate?: ValidatedConfigurationCandidate) => { + owned(patch, writer); + if (candidate && candidate.operationKey !== writer.operationKey) + throw new TransitCredentialConflict(); + const current = await inspect( + patch.catalogRevisionId, + writer.operationKey, + administrativeConfigurationState(writer).job.signal, + ); + if (!current.editable) throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_HOST_OWNED"); + await withAdministrativeConfigurationRead({}, async (read) => { + if ( + transitCredentialRevision(rowFrom(read)) !== + (state.committedRevision ?? patch.expectedCredentialsRevision) + ) + throw new TransitCredentialConflict(); + }); + if (!state.committedRevision) + await proof(state, false, administrativeConfigurationState(writer).connection); + }; + const participant = Object.freeze({ + kind: "transit-credentials", + mode: "apply", + actorId: writer.actorId, + operationKey: writer.operationKey, + managedServiceSecretKeys: Object.freeze([{ serviceId: SERVICE, key: KEY }]), + authorize: async () => verify(), + verify: async ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + ) => { + if (actual !== writer) throw new TransitCredentialConflict(); + await verify(candidate); + }, + commit: async ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + tx: ConfigurationWriteTransaction, + ) => { + if (actual !== writer || administrativeConfigurationState(writer).connection !== tx) + throw new TransitCredentialConflict(); + await verify(candidate); + }, + }) as unknown as ConfigurationCommitParticipant; + participants.set(participant, state); + return { + overlay: { + purpose: "transit-credentials", + coordinatorMode: "require-running", + changes: [patch.overlayChange], + } as ProspectiveConfigurationOverlay, + participant, + }; +} +export async function persistPreparedFeedCredentialRow(input: { + patch: PreparedFeedCredentialPatch; + candidate: ValidatedConfigurationCandidate; + writer: AdministrativeConfigurationWriter; + candidateState: { + writer: AdministrativeConfigurationWriter; + rows: { serviceSecretRows: readonly (typeof serviceSecret.$inferSelect)[] } | null; + sourceRows: { serviceSecretRows: readonly (typeof serviceSecret.$inferSelect)[] } | null; + payload: { transitCredentialsRevision?: string }; + overlay: ProspectiveConfigurationOverlay; + }; +}): Promise<{ credentialsRevision: string }> { + const state = owned(input.patch, input.writer), + owner = administrativeConfigurationState(input.writer); + if ( + !owner.transactionActive || + owner.transactionMode !== "write" || + !owner.participant?.managedServiceSecretKeys.some( + (item) => item.serviceId === SERVICE && item.key === KEY, + ) || + state.committedRevision + ) + throw new TransitCredentialConflict(); + const { candidateState: candidate } = input; + const next = candidate.rows?.serviceSecretRows.find( + (row) => row.serviceId === SERVICE && row.key === KEY, + ), + before = candidate.sourceRows?.serviceSecretRows.find( + (row) => row.serviceId === SERVICE && row.key === KEY, + ); + if ( + candidate.writer !== input.writer || + input.candidate.operationKey !== input.patch.operationKey || + !next || + transitCredentialRevision(before) !== input.patch.expectedCredentialsRevision || + input.patch.overlayChange.action !== "set" || + decrypt(next.ciphertext, next.iv, next.tag) !== input.patch.overlayChange.value || + candidate.payload.transitCredentialsRevision !== transitCredentialRevision(next) || + !candidate.overlay.changes.some( + (change) => canonicalJson(change) === canonicalJson(input.patch.overlayChange), + ) + ) + throw new TransitCredentialConflict(); + const tx = owner.connection; + await proof(state, true, tx); + const revision = transitCredentialRevision(next); + const changed = before + ? await tx + .update(serviceSecret) + .set({ + ciphertext: next.ciphertext, + iv: next.iv, + tag: next.tag, + updatedBy: input.writer.actorId, + updatedAt: new Date(), + }) + .where( + and( + eq(serviceSecret.id, before.id), + sql`xmin::text = ${(before as typeof before & { _rowVersion?: string })._rowVersion}`, + ), + ) + .returning({ id: serviceSecret.id }) + : await tx + .insert(serviceSecret) + .values({ ...next, updatedBy: input.writer.actorId }) + .onConflictDoNothing() + .returning({ id: serviceSecret.id }); + if (changed.length !== 1) throw new TransitCredentialConflict(); + const marked = await tx + .update(transitCredentialPreview) + .set({ + consumedAt: new Date(), + candidateId: input.candidate.candidateId, + credentialsRevision: revision, + }) + .where( + and( + eq(transitCredentialPreview.id, state.preview!.previewId), + eq(transitCredentialPreview.actorId, input.writer.actorId), + eq(transitCredentialPreview.intentFingerprint, state.preview!.intentFingerprint), + sql`${transitCredentialPreview.consumedAt} is null`, + sql`${transitCredentialPreview.expiresAt} > now()`, + ), + ) + .returning({ id: transitCredentialPreview.id }); + if (marked.length !== 1) throw new TransitCredentialConflict(); + await tx.insert(adminAuditLog).values({ + id: randomUUID(), + action: "transit.credentials.update", + actorId: input.writer.actorId, + targetType: "transit.credentials", + targetId: state.input.catalogRevisionId, + details: { changes: state.input.changes.map(({ key, action }) => ({ key, action })) }, + }); + // This is proposed commit state only; the actual persistence wrapper owns SQL + // rollback/unknown handling and must not retry a consumed decision as fresh work. + state.committedRevision = revision; + state.candidateId = input.candidate.candidateId; + return { credentialsRevision: revision }; +} + +export function transitParticipantPatch( + participant: ConfigurationCommitParticipant, + writer: AdministrativeConfigurationWriter, +): PreparedFeedCredentialPatch { + const state = participants.get(participant); + if (!state || state.writer !== writer) throw new TransitCredentialConflict(); + owned(state.patch, writer); + return state.patch; +} diff --git a/apps/api/src/services/transit-feed-credentials-postgres.test.ts b/apps/api/src/services/transit-feed-credentials-postgres.test.ts new file mode 100644 index 000000000..53d960570 --- /dev/null +++ b/apps/api/src/services/transit-feed-credentials-postgres.test.ts @@ -0,0 +1,612 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { ownedDataManager } from "../../../ops-agent/src/__fixtures__/owned-data-manager"; +import { ownedServiceHost } from "../../../ops-agent/src/__fixtures__/owned-service-host"; +import { installPinnedCredentialCatalog } from "../../../ops-agent/src/__fixtures__/pinned-feed-credentials"; +import { openOpsJobJournal } from "../../../ops-agent/src/journal"; +import { buildOpsAgentServer } from "../../../ops-agent/src/server"; +import { createConfigurationPostgresFixture } from "../test/configuration-postgres-fixture"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const state = vi.hoisted(() => ({ + fixture: null as unknown as Awaited>, + services: [] as services.LoadedService[], +})); +vi.mock("../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); +vi.mock("../integration-host", () => ({ getAllIntegrations: () => [] })); +vi.mock("./service-registry", () => ({ + getServiceRegistry: () => ({ + list: () => state.services, + get: (id: string) => state.services.find((service) => service.manifest.id === id), + applyEnabledIds: (ids: Set) => { + for (const service of state.services) service.enabled = ids.has(service.manifest.id); + }, + }), +})); + +import { AdminActionReauthentication } from "../account/admin-reauthentication"; +import * as schema from "../db/schema"; +import { jobRunner } from "./job-runner"; +import { decrypt, encrypt } from "./secrets"; +import { bindTransitCredentialAuthorization } from "./transit-credential-authorization"; +import { transitCredentialRevision } from "./transit-credential-revision"; +import { inspectFeedCredentials, previewFeedCredentialChanges } from "./transit-feed-credentials"; + +const enabled = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"; +describe.skipIf(!enabled)( + "actual pinned managed credentials / original configuration maintenance / PostgreSQL", + () => { + let pauseKind = "vault.apply"; + let faults = false; + const dropped = new Set(); + let stopCoordinator: () => void; + let nativeRecovery = false; + let releaseAdmission: (() => void) | undefined; + const children: ReturnType[] = []; + let ackReplies: unknown[] = []; + let wire: unknown[] = []; + let journal: Awaited>; + + let migrated: Awaited>, + host: Awaited>, + agent: ReturnType, + installation: ReturnType, + dm: Awaited>; + beforeAll(async () => { + migrated = await createMigratedPostgresFixture(); + }, 90000); + afterAll(async () => { + await migrated?.stop(); + }); + beforeEach(async (context) => { + faults = context.task.name.startsWith("consumes one real"); + dropped.clear(); + nativeRecovery = context.task.name.includes("killed native"); + pauseKind = context.task.name.includes("phase=prepared") + ? "configurationMaintenance.prepare" + : "vault.apply"; + vi.stubEnv("OPENMAPX_RELEASE", "local"); + vi.stubEnv("OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL", migrated.connectionUri); + vi.stubEnv("OPENMAPX_SECRETS_KEY", "71".repeat(32)); + state.fixture = await createConfigurationPostgresFixture({ withAccountAdministration: true }); + const schemas = Object.fromEntries( + ["motis", "ops-agent", "app-api", "traefik", "postgis", "redis", "data-manager"].map( + (id) => [ + id, + { + type: "object", + properties: { TRANSITOUS_API_KEYS: { type: "string", "x-openmapx-secret": true } }, + }, + ], + ), + ); + host = await ownedServiceHost({ + schemas, + actualComposeConfig: true, + withCoordinator: true, + withHardlinks: !context.task.name.includes("empty-plan"), + }); + state.services = host.loaded; + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((service) => service.manifest.id), + }); + await installPinnedCredentialCatalog(host.root, installation); + journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + agent = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + ackReplies = []; + wire = []; + agent.addHook("preHandler", async (request) => { + const kind = (request.body as { operation?: { kind?: string } } | undefined)?.operation + ?.kind; + if (kind) wire.push({ kind, phase: "entered", at: Date.now() }); + }); + agent.addHook("onSend", async (request, reply, payload) => { + if ((request.body as { operation?: { kind?: string } } | undefined)?.operation?.kind) { + const body = typeof payload === "string" ? JSON.parse(payload) : {}; + const kind = (request.body as { operation: { kind: string } }).operation.kind; + wire.push({ + kind, + phase: "response", + status: reply.statusCode, + errorClass: body.error?.class, + at: Date.now(), + }); + if ( + faults && + [ + "configurationMaintenance.prepare", + "configurationMaintenance.decide", + "configurationMaintenance.settle", + ].includes(kind) && + reply.statusCode === 202 && + !dropped.has(kind) + ) { + dropped.add(kind); + reply.raw.destroy(); + } + if (kind === "controlRetention.acknowledge") + ackReplies.push({ + status: reply.statusCode, + code: body.error?.class, + message: body.error?.message, + }); + } + if ( + nativeRecovery && + (request.body as { operation?: { kind?: string } } | undefined)?.operation?.kind === + pauseKind && + reply.statusCode === 202 + ) + await new Promise((resolve) => { + releaseAdmission = resolve; + }); + return payload; + }); + const url = await agent.listen({ host: "127.0.0.1", port: 0 }), + token = join(host.root, "api-token"); + writeFileSync(token, host.token, { mode: 0o600 }); + for (const [key, value] of Object.entries({ + OPS_AGENT_URL: url, + OPS_AGENT_TOKEN_FILE: token, + OPS_TRUSTED_CONFIG_DIR: host.queue, + OPS_TRUSTED_CONFIG_UID: String(statSync(host.queue).uid), + OPS_TRUSTED_CONFIG_GID: String(statSync(host.queue).gid), + })) + vi.stubEnv(key, value); + dm = await ownedDataManager({ + sql: state.fixture.sql, + infraDir: host.infraDir, + serviceIds: host.loaded.map((service) => service.manifest.id), + losePrepareResponse: faults, + loseHardlinkResponse: faults, + observeSettlementSql: true, + loseSettlementCommitAcknowledgement: faults, + }); + vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", dm.token); + const dmRow = { + ...host.row, + Id: "c".repeat(64), + ConfigImage: "example/data-manager:1", + Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, + }; + host.setRows([host.row, dmRow]); + stopCoordinator = () => host.setRows([host.row, { ...dmRow, State: "exited" }]); + host.setCreatingProcess(async (args) => { + if (!args.includes("--force-recreate")) return; + const revision = services.currentConfigurationGeneration(host.infraDir)!; + const compose = services.stackComposeArgs(host.stack), + options = { + signal: new AbortController().signal, + timeout: 15000, + maxBuffer: 1024 * 1024, + }; + const config = JSON.parse( + (await host.execFile("docker", [...compose, "config", "--format", "json"], options)) + .stdout, + ), + hash = ( + await host.execFile("docker", [...compose, "config", "--hash", "data-manager"], options) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!; + const row = { + ...dmRow, + Id: "d".repeat(64), + Labels: { + ...dmRow.Labels, + "com.docker.compose.project.working_dir": join( + host.infraDir, + ".trusted-config-generations", + revision, + ), + "com.docker.compose.project.config_files": + services.configurationGenerationComposeFile(host.infraDir, revision), + "com.docker.compose.config-hash": hash, + }, + }, + service = config.services["data-manager"]; + host.setRows([host.row, row]); + host.setRuntimeInspections({ + [row.Id]: { + Id: row.Id, + Image: row.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: row.ConfigImage, + Labels: row.Labels, + Env: Object.entries(service.environment ?? {}).map( + ([key, value]) => `${key}=${value}`, + ), + }, + Mounts: [ + ...(service.volumes ?? []).map( + (value: { source: string; target: string; type: string; read_only?: boolean }) => ({ + Type: value.type, + Source: value.source, + Destination: value.target, + RW: !value.read_only, + }), + ), + ...(service.secrets ?? []).map((value: { source: string; target: string }) => ({ + Type: "bind", + Source: config.secrets[value.source].file, + Destination: `/run/secrets/${value.target}`, + RW: false, + })), + ], + }, + }); + }); + const source = join(host.infraDir, "data/source-osm"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "fixture.pbf"), "public-osm-fixture"); + }, 90000); + afterEach(async () => { + releaseAdmission?.(); + releaseAdmission = undefined; + for (const child of children) + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + children.length = 0; + await dm?.close(); + await agent?.close(); + if (installation) closeControlInstallation(installation); + await state.fixture?.stop(); + if (host) rmSync(host.root, { recursive: true, force: true }); + vi.unstubAllEnvs(); + state.services = []; + }); + async function authorizedInput(options: { seed?: boolean } = {}) { + const actor = randomUUID(); + await state.fixture.db.insert(schema.user).values({ + id: actor, + name: "Synthetic admin", + email: `${actor}@example.test`, + emailVerified: true, + role: "admin", + }); + const login = async () => { + const id = randomUUID(), + at = new Date(); + await state.fixture.db.insert(schema.session).values({ + id, + userId: actor, + token: randomUUID(), + expiresAt: new Date(Date.now() + 3600000), + createdAt: at, + updatedAt: at, + }); + await state.fixture.db.insert(schema.sessionAuthAssurance).values({ + sessionId: id, + userId: actor, + method: "password", + authenticatedAt: at, + originIndependentPasswordLogin: true, + }); + return { + id, + userId: actor, + authenticatedAt: at, + method: "password" as const, + mfaConfigured: false, + }; + }; + const old = await login(), + operationKey = `opk1_${randomUUID().replaceAll("-", "")}`; + const { readCatalogPinSet } = await import("@openmapx/core/catalog-pins"); + const catalog = readCatalogPinSet(host.root)!.generationId; + const prior = { + id: randomUUID(), + serviceId: "data-manager", + key: "TRANSITOUS_API_KEYS", + ...encrypt('{"orphan":" untouched "}'), + createdAt: new Date(), + updatedAt: new Date(), + updatedBy: null, + }; + if (options.seed !== false) await state.fixture.db.insert(schema.serviceSecret).values(prior); + const [currentRow] = await state.fixture.db.select().from(schema.serviceSecret); + const preview = await previewFeedCredentialChanges({ + actorId: actor, + catalogRevisionId: catalog, + expectedCredentialsRevision: transitCredentialRevision(currentRow), + expectedCurrentRevisionId: services.currentConfigurationGeneration(host.infraDir), + changes: [{ key: "de/A", action: "set", value: "synthetic-private-key" }], + operationKey, + }); + const reauth = new AdminActionReauthentication(state.fixture.db), + challenge = await reauth.start({ + actorId: actor, + startingSessionId: old.id, + purpose: "transit.credentials", + targetId: catalog, + intentFingerprint: preview.intentFingerprint, + }); + await new Promise((resolve) => setTimeout(resolve, 2)); + const fresh = await login(); + await reauth.complete({ + challengeId: challenge.challengeId, + nonce: challenge.nonce, + session: fresh, + }); + const input = { + actorId: actor, + previewId: preview.previewId, + intentFingerprint: preview.intentFingerprint, + operationKey, + }; + await bindTransitCredentialAuthorization({ + ...input, + transitProofToken: challenge.challengeId, + sessionId: fresh.id, + nonce: challenge.nonce.toString("base64url"), + }); + return { input, catalog, challenge, preview, sessionId: fresh.id, nonce: challenge.nonce }; + } + it("consumes one real purpose proof and commits a preserved map through one original guard, recreation and active mounted revision", async () => { + const { input, catalog, challenge } = await authorizedInput(); + const competing = await authorizedInput({ seed: false }), + applying = jobRunner.applyImmediateTransitCredentials(input); + const until = Date.now() + 30000; + while (dm.preparePosts === 0) { + if (Date.now() > until) throw Error("Original preparation not observed"); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + await expect(jobRunner.applyImmediateTransitCredentials(competing.input)).rejects.toThrow(); + const result = await applying; + const [unusedProof] = await state.fixture.db + .select() + .from(schema.adminActionReauthentication) + .where(eq(schema.adminActionReauthentication.id, competing.challenge.challengeId)); + expect(unusedProof?.state).toBe("completed"); + expect(result.configuration).toMatchObject({ outcome: "committed", followUp: "complete" }); + const [row] = await state.fixture.db.select().from(schema.serviceSecret); + expect(JSON.parse(decrypt(row!.ciphertext, row!.iv, row!.tag))).toEqual({ + orphan: " untouched ", + "de/A": "synthetic-private-key", + }); + expect(result.credentialsRevision).toBe(transitCredentialRevision(row)); + const [proof] = await state.fixture.db + .select() + .from(schema.adminActionReauthentication) + .where(eq(schema.adminActionReauthentication.id, challenge.challengeId)); + expect(proof?.state).toBe("consumed"); + expect(dm.preparePosts).toBe(1); + expect(dm.posts).toBe(1); + expect(dm.settlementSqlIo).toEqual({ updates: 1, commits: 1 }); + expect(dropped.size).toBe(3); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(1); + expect(services.readAppliedTransitCredentialsRevision(host.infraDir)).toBe( + result.credentialsRevision, + ); + const view = await inspectFeedCredentials({ catalogRevisionId: catalog }); + expect(view).toMatchObject({ + appliedCredentialsRevision: result.credentialsRevision, + pendingApplication: false, + }); + expect(JSON.stringify(view)).not.toContain("synthetic-private-key"); + expect(await jobRunner.applyImmediateTransitCredentials(input)).toEqual(result); + expect(dm.preparePosts).toBe(1); + const [job] = await state.fixture.db.select().from(schema.adminJob); + expect(job?.status).toBe("success"); + const logs = await state.fixture.db.select().from(schema.adminJobLog), + audit = await state.fixture.db.select().from(schema.adminAuditLog); + expect(JSON.stringify({ logs, audit })).not.toContain("synthetic-private-key"); + expect(JSON.stringify({ payload: job?.payload, result: job?.result })).not.toContain( + "synthetic-private-key", + ); + }, 120000); + it.each(["prepared", "saved"])( + "recovers a killed native credential producer without consuming or applying twice (phase=%s)", + async (phase) => { + const { input } = await authorizedInput(), + database = new URL(migrated.connectionUri); + database.searchParams.set("search_path", state.fixture.schema); + const loader = createRequire(new URL("../../package.json", import.meta.url)).resolve( + "tsx/esm", + ), + script = fileURLToPath( + new URL("./__tests__/fixtures/native-transit-credentials-api.ts", import.meta.url), + ); + const start = (reconcile = false) => { + const child = spawn(process.execPath, ["--import", loader, script], { + cwd: host.root, + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + DATABASE_URL: database.toString(), + OPENMAPX_ROOT_DIR: host.root, + NODE_ENV: "test", + BETTER_AUTH_SECRET: "synthetic-transit-native-auth-fixture", + }, + }); + children.push(child); + let output = ""; + child.stdout!.on("data", (chunk) => { + output += chunk; + }); + child.stderr!.on("data", () => {}); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code) => resolve(code)); + }); + child.stdin!.end(JSON.stringify({ ...input, reconcile })); + return { child, exited, output: () => output }; + }; + const waitFor = async (check: () => boolean | Promise, ms = 90000) => { + const deadline = Date.now() + ms; + while (!(await check())) { + if (Date.now() > deadline) throw Error("Original credential checkpoint not observed"); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + }; + const first = start(); + await waitFor( + () => + Boolean(releaseAdmission) && + journal + .records() + .some( + (record) => record.operation.kind === pauseKind && record.state === "succeeded", + ), + ); + const [before] = await state.fixture.db.select().from(schema.adminJob), + [proof] = await state.fixture.db.select().from(schema.adminActionReauthentication), + [secret] = await state.fixture.db.select().from(schema.serviceSecret); + expect(proof?.state).toBe(phase === "saved" ? "consumed" : "completed"); + expect( + (before!.result as { _configurationCandidate: { phase: string } })._configurationCandidate + .phase, + ).toBe(phase); + first.child.kill("SIGKILL"); + await first.exited; + releaseAdmission?.(); + releaseAdmission = undefined; + const lease = (before!.result as { _observationLeaseUntil: string })._observationLeaseUntil; + await new Promise((resolve) => + setTimeout(resolve, Math.max(0, Date.parse(lease) - Date.now() + 100)), + ); + const replacement = start(true); + const code = await replacement.exited; + console.info( + "Native recovery result", + JSON.stringify({ code, output: replacement.output() }), + ); + expect(code).toBe(phase === "saved" ? 0 : 2); + const [after] = await state.fixture.db.select().from(schema.adminJob), + [afterProof] = await state.fixture.db.select().from(schema.adminActionReauthentication), + [afterSecret] = await state.fixture.db.select().from(schema.serviceSecret); + expect(after?.id).toBe(before!.id); + expect(after?.status).toBe(phase === "saved" ? "success" : "failed"); + expect(afterProof?.consumedAt).toEqual(proof?.consumedAt); + expect(afterSecret).toEqual(secret); + expect(dm.preparePosts).toBe(1); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength( + phase === "saved" ? 1 : 0, + ); + if (phase === "prepared") { + const replay = await jobRunner.applyImmediateTransitCredentials(input); + expect(replay.configuration.outcome).toBe("not_committed"); + } + expect( + (after!.result as { transitCredentials: { credentialsRevision: string } }) + .transitCredentials.credentialsRevision, + ).toBe(transitCredentialRevision(secret)); + }, + 240000, + ); + + it("refuses an empty-plan credential follow-up when the original DM is stopped, before proof consumption or admission", async () => { + const { input, challenge } = await authorizedInput(); + stopCoordinator(); + const [before] = await state.fixture.db.select().from(schema.serviceSecret); + await expect(jobRunner.applyImmediateTransitCredentials(input)).rejects.toThrow(); + expect( + wire.filter( + (value) => (value as { kind: string }).kind === "configurationMaintenance.prepare", + ), + ).toHaveLength(0); + const [after] = await state.fixture.db.select().from(schema.serviceSecret), + [proof] = await state.fixture.db + .select() + .from(schema.adminActionReauthentication) + .where(eq(schema.adminActionReauthentication.id, challenge.challengeId)); + expect(after).toEqual(before); + expect(proof?.state).toBe("completed"); + expect(dm.preparePosts).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + }, 90000); + it("rejects a nominal patch whose values differ from its immutable reviewed private preview", async () => { + const { input, catalog, challenge, preview, sessionId, nonce } = await authorizedInput(); + const { withAdministrativeConfigurationJob, withAdministrativeConfigurationWriter } = + await import("./administrative-configuration-writer-owner"); + const { + withTransitCredentialProofContext, + prepareFeedCredentialImportPatch, + authorizePreparedFeedCredentialPatch, + } = await import("./transit-feed-credential-participant"); + const attempted = Promise.withResolvers(); + jobRunner.register("test.feed-credential-review", async (ctx) => { + try { + await withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { actorId: input.actorId, operationKey: input.operationKey, integrationIds: [] }, + async (writer) => + withTransitCredentialProofContext( + { actorId: input.actorId, sessionId, nonce }, + async () => { + const patch = await prepareFeedCredentialImportPatch({ + actorId: input.actorId, + catalogRevisionId: catalog, + expectedCredentialsRevision: preview.credentialsRevision, + expectedCurrentRevisionId: preview.expectedCurrentRevisionId, + changes: [{ key: "de/A", action: "set", value: "synthetic-private-kex" }], + importPreviewId: preview.previewId, + operationKey: input.operationKey, + }); + await authorizePreparedFeedCredentialPatch({ + patch, + writer, + authorization: { kind: "direct", transitProofToken: challenge.challengeId }, + preview, + }); + }, + ), + ), + ); + attempted.resolve(); + return {}; + } catch (error) { + attempted.reject(error); + throw error; + } + }); + const id = await jobRunner.enqueue("test.feed-credential-review", {}, input.actorId); + const outcome = await attempted.promise.then( + () => null, + (error) => error, + ); + await vi.waitFor(async () => { + const [job] = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + expect(["success", "failed"]).toContain(job?.status); + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(outcome).toBeInstanceOf(Error); + expect((outcome as Error).message).toBe("TRANSIT_CREDENTIALS_CHANGED"); + const [proof] = await state.fixture.db + .select() + .from(schema.adminActionReauthentication) + .where(eq(schema.adminActionReauthentication.id, challenge.challengeId)); + expect(proof?.state).toBe("completed"); + expect(dm.preparePosts).toBe(0); + expect(host.calls.filter((args) => args.includes("--force-recreate"))).toHaveLength(0); + }, 90000); + }, +); diff --git a/apps/api/src/services/transit-feed-credentials.ts b/apps/api/src/services/transit-feed-credentials.ts new file mode 100644 index 000000000..f388d90b0 --- /dev/null +++ b/apps/api/src/services/transit-feed-credentials.ts @@ -0,0 +1,77 @@ +/** Public orchestration over the sole finite participant and original candidate writer. */ +import { + administrativeConfigurationState, + currentAdministrativeConfigurationWriter, +} from "./administrative-configuration-writer-owner"; +import { readFeedCredentialPreview, TransitCredentialConflict } from "./transit-credential-preview"; +import { + authorizePreparedFeedCredentialPatch, + type FeedCredentialApplyResult, + type FeedCredentialAuthorization, + prepareFeedCredentialImportPatch, + transitCredentialParticipation, +} from "./transit-feed-credential-participant"; + +export { persistPreparedFeedCredentialPatch } from "./prospective-configuration"; +export * from "./transit-feed-credential-participant"; +export async function applyFeedCredentialChanges(input: { + actorId: string; + previewId: string; + intentFingerprint: string; + operationKey: string; + authorization: FeedCredentialAuthorization; + signal?: AbortSignal; +}): Promise { + const writer = currentAdministrativeConfigurationWriter(), + stored = await readFeedCredentialPreview( + input, + administrativeConfigurationState(writer).connection, + ); + if (stored.row.expiresAt <= new Date() || stored.row.consumedAt) + throw new TransitCredentialConflict(); + const p = stored.private, + patch = await prepareFeedCredentialImportPatch({ + actorId: input.actorId, + catalogRevisionId: p.preview.catalogRevisionId, + expectedCredentialsRevision: p.preview.credentialsRevision, + expectedCurrentRevisionId: p.preview.expectedCurrentRevisionId, + changes: p.changes, + operationKey: input.operationKey, + importPreviewId: input.previewId, + signal: input.signal, + }); + await authorizePreparedFeedCredentialPatch({ + patch, + writer, + authorization: input.authorization, + preview: p.preview, + }); + const scope = transitCredentialParticipation(patch, writer); + const { withConfigurationCommitParticipant } = await import("./configuration-commit-participant"); + return withConfigurationCommitParticipant(writer, scope.participant, async () => { + const { prepareProspectiveConfiguration, persistAndApplyProspectiveConfiguration } = + await import("./prospective-configuration"); + const candidate = await prepareProspectiveConfiguration({ + actorId: input.actorId, + operationKey: input.operationKey, + expectedCurrentRevisionId: p.preview.expectedCurrentRevisionId, + overlay: scope.overlay, + followUp: "service-apply", + signal: input.signal, + }); + const result = await persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "require-running", + signal: input.signal, + }); + return { + credentialsRevision: + ( + await readFeedCredentialPreview( + input, + administrativeConfigurationState(writer).connection, + ) + ).row.credentialsRevision ?? patch.expectedCredentialsRevision, + configuration: result.application, + }; + }); +} diff --git a/apps/api/src/test/configuration-postgres-fixture.ts b/apps/api/src/test/configuration-postgres-fixture.ts index fefd97aaf..4b7145f0c 100644 --- a/apps/api/src/test/configuration-postgres-fixture.ts +++ b/apps/api/src/test/configuration-postgres-fixture.ts @@ -5,7 +5,9 @@ import * as schema from "../db/schema"; /** An explicitly supplied owned/migrated synthetic database. No migrations run. * Only empty fixture copies of the current configuration/job tables are created and removed. */ -export async function createConfigurationPostgresFixture() { +export async function createConfigurationPostgresFixture( + options: { withAccountAdministration?: boolean } = {}, +) { const url = process.env.OPENMAPX_ADMIN_JOB_TEST_DATABASE_URL; if (!url) throw Error("Explicit owned synthetic PostgreSQL connection required"); const name = `task012_${randomUUID().replaceAll("-", "")}`; @@ -24,6 +26,18 @@ export async function createConfigurationPostgresFixture() { "integration_config", "service_secret", "integration_secret", + "transit_credential_preview", + ...(options.withAccountAdministration + ? [ + "user", + "session", + "session_auth_assurance", + "auth_policy_state", + "admin_action_reauthentication", + "data_subject_request", + "data_subject_request_preservation", + ] + : []), ]) await control.unsafe( `CREATE TABLE "${name}"."${table}" (LIKE public."${table}" INCLUDING ALL)`, diff --git a/apps/ops-agent/src/__fixtures__/owned-data-manager.ts b/apps/ops-agent/src/__fixtures__/owned-data-manager.ts index 51dbe28a7..d47892eb5 100644 --- a/apps/ops-agent/src/__fixtures__/owned-data-manager.ts +++ b/apps/ops-agent/src/__fixtures__/owned-data-manager.ts @@ -139,6 +139,7 @@ export async function ownedDataManager(options: { }, get settlementResponseStatuses(){return [...settlementResponseStatuses];}, get hardlinkResponseStatuses() { return replies.map(reply => reply.status); }, + get hardlinkResponses() { return structuredClone(replies); }, reconcileRetention: () => reconcileControlRetention(installation, { maxFamilies: 64 }), async diagnostics() { return { maintenanceReplies, replies: replies.map(reply => ({ status: reply.status, error: (reply.body as { error?: unknown } | null)?.error })), executions: await Promise.all([...keys].map(async key => { const value = await store.lookup(key); return value && { status: value.status.status, publication: value.status.publication, diagnostics: value.status.diagnostics }; })) }; }, diff --git a/apps/ops-agent/src/__fixtures__/pinned-feed-credentials.ts b/apps/ops-agent/src/__fixtures__/pinned-feed-credentials.ts new file mode 100644 index 000000000..75c4678fc --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/pinned-feed-credentials.ts @@ -0,0 +1,16 @@ +/** Real local Git repositories and actual paired-pin bootstrap, no upstream network. */ +import { execFileSync } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { ControlInstallation } from "@openmapx/core/build-artifacts"; +import { initializeManagedCatalogPins, readCatalogPinSet } from "@openmapx/core/catalog-pins"; +export async function installPinnedCredentialCatalog(root: string, installation: ControlInstallation) { + const infraDir=join(root,"infra/docker"),catalog=join(infraDir,"data/.transitous-catalog"),atlas=join(catalog,"transitland-atlas"); + mkdirSync(join(catalog,"feeds"),{recursive:true});mkdirSync(join(atlas,"feeds"),{recursive:true});mkdirSync(join(root,"services/motis/tools/transitous"),{recursive:true}); + const git=(cwd:string,args:string[])=>execFileSync("git",args,{cwd,encoding:"utf8",stdio:["ignore","pipe","pipe"],env:{...process.env,GIT_AUTHOR_NAME:"fixture",GIT_AUTHOR_EMAIL:"fixture@example.test",GIT_COMMITTER_NAME:"fixture",GIT_COMMITTER_EMAIL:"fixture@example.test"}}).trim(); + git(atlas,["init"]);writeFileSync(join(atlas,"feeds/a.dmfr.json"),JSON.stringify({feeds:[{id:"auth",authorization:{type:"api-key"}}]}));git(atlas,["add","."]);git(atlas,["commit","-m","Atlas"]);const atlasSha=git(atlas,["rev-parse","HEAD"]); + git(catalog,["init"]);writeFileSync(join(catalog,"feeds/de.json"),JSON.stringify({sources:[{name:"A","transitland-atlas-id":"auth"}]}));git(catalog,["add","feeds"]);git(catalog,["update-index","--add","--cacheinfo",`160000,${atlasSha},transitland-atlas`]);git(catalog,["commit","-m","Transitous"]);const sha=git(catalog,["rev-parse","HEAD"]); + writeFileSync(join(infraDir,"transitous.lock.json"),JSON.stringify({ref:`main@${sha}`,submodules:{"transitland-atlas":atlasSha},lockedAt:"2026-10-23T00:00:00.000Z",lockedBy:"fixture"}));const gbfs="b".repeat(40); + writeFileSync(join(infraDir,"gbfs-catalog.lock.json"),JSON.stringify({schemaVersion:1,source:"mobilitydata-gbfs",commit:gbfs,url:`https://raw.githubusercontent.com/MobilityData/gbfs/${gbfs}/systems.csv`,sha256:"c".repeat(64),lockedAt:"2026-10-23T00:00:00.000Z",lockedBy:"fixture"})); + await initializeManagedCatalogPins({rootDir:root,installation});return readCatalogPinSet(root)!; +} diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index 1d1c2863f..02976641c 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -438,12 +438,15 @@ describe("administrative data runtime", () => { await expect( dispatchOpsOperation(runtime, operation, context(operation.kind)), ).rejects.toThrow("Host update required"); + else if (operation.kind === "data.generateApiKeys") + await expect( + dispatchOpsOperation(runtime, operation, context(operation.kind)), + ).rejects.toThrow("Typed adapter is not wired"); else await dispatchOpsOperation(runtime, operation, context(operation.kind)); } expect(calls).toEqual([ ["data", "link"], ["data", "clean", "osm"], - ["data", "generate-api-keys"], ]); expect(calls.flat()).not.toContain("transitous-fixed-v1"); }); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 974ac5874..24ef5d894 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -1961,12 +1961,10 @@ export function createAdministrativeRuntime( }); return { completed: true, resourceId: operation.dataTypeId }; }; - runtime["data.generateApiKeys"] = async (_operation, context) => { - await runFixedCli(["data", "generate-api-keys"], { - ...context, - operationKind: context.claim.operation.kind, - }); - return { completed: true }; + // Historical decoding is retained; remote admission is forbidden. Never run + // the host-owned file command from a browser/job, including old pending work. + runtime["data.generateApiKeys"] = async () => { + throw new OpsNotWiredError(); }; runtime["data.overtureSync"] = async (operation, context) => { await runMatchingDataCommand(operation, context); diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index 84fdf168d..5ab30be77 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -961,7 +961,7 @@ export async function assertPlannedCoordinatorConfigurationWrite( !overlay || typeof overlay !== "object" || Array.isArray(overlay) || - overlay.purpose !== "service-secret" || + !["service-secret", "transit-credentials"].includes(String(overlay.purpose)) || !Array.isArray(overlay.changes) || !overlay.changes.length || !overlay.changes.every( @@ -974,6 +974,26 @@ export async function assertPlannedCoordinatorConfigurationWrite( ) ) throw Error("Fixed planned DM recreation producer required"); + if (overlay.purpose === "transit-credentials") { + const change = overlay.changes[0] as Record | undefined; + if ( + original.coordinatorMode !== "require-running" || + overlay.changes.length !== 1 || + change?.key !== "TRANSITOUS_API_KEYS" || + change.action !== "set" || + typeof change.value !== "string" || + candidate.payload.serviceSecrets.find((entry) => entry.serviceId === "data-manager")?.values + .TRANSITOUS_API_KEYS !== change.value || + !candidate.payload.transitCredentialsRevision + ) + throw Error("Fixed managed transit credential producer required"); + } else if ( + overlay.changes.some( + (change) => (change as Record).key === "TRANSITOUS_API_KEYS", + ) + ) { + throw Error("Managed transit credential producer required"); + } } /** Actual publication is stricter than the read-only planned eligibility. */ export async function assertOwnedCoordinatorConfigurationWrite( diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 019728534..5c8a5a870 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -262,8 +262,7 @@ async function main(): Promise { loadConfiguredResourceAuthority().countries.has(countryCode), allowDataType: async (dataTypeId) => inspectDataTypeAuthority((await authorityLoader()).services, dataTypeId), - allowCatalogRevision: (kind, revisionId) => - kind === "data.generateApiKeys" && revisionId === "transitous-fixed-v1", + allowCatalogRevision: () => false, allowRelease: (_kind, releaseId) => inspectReleaseAuthority(config.rootDir, releaseId), allowUpdateJobId: (updateJobId) => /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(updateJobId), diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index be85a4478..433110224 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -43,6 +43,7 @@ import { installServicePullRuntime, } from "./service-pull-runtime"; import { resolveBootstrapEnabledServiceIds } from "./startup-authority"; +import { installTransitCredentialsRuntime } from "./transit-credentials-runtime"; import { createTransitousLockRuntime } from "./transitous-lock-runtime"; import { installTrustedConfigurationRuntime, @@ -229,6 +230,13 @@ export async function createProductionOpsRuntime( createDefaultReleaseEffects(options.rootDir, runFixedCli), ); createTransitousLockRuntime(runtime, { rootDir: options.rootDir }); + installTransitCredentialsRuntime(runtime, { + rootDir: options.rootDir, + sourceToken: options.candidateSource?.token, + stack: options.stack, + ownedServices: owned, + execFile: options.docker?.execFile, + }); createAdministrativeRuntime(runtime, { rootDir: options.rootDir, runFixedCli, diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 1cc308814..ff8bc3a48 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -202,6 +202,7 @@ const RELEASE_RECOVERY_READ_KINDS = new Set([ "valhalla.traffic.inspect", "postgis.capacity.inspect", "catalogPins.inspect", + "transitCredentials.inspect", "transitousLock.inspect", "gbfsCatalogLock.inspect", ]); diff --git a/apps/ops-agent/src/transit-credentials-runtime.test.ts b/apps/ops-agent/src/transit-credentials-runtime.test.ts new file mode 100644 index 000000000..efd5addd0 --- /dev/null +++ b/apps/ops-agent/src/transit-credentials-runtime.test.ts @@ -0,0 +1,172 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { initializeManagedCatalogPins, readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import { afterEach, expect, it } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; +import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; +import { installTransitCredentialsRuntime } from "./transit-credentials-runtime"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +const git = (cwd: string, args: string[]) => + execFileSync("git", args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + GIT_AUTHOR_NAME: "fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }).trim(); +async function fixture() { + const root = mkdtempSync(join(tmpdir(), "omx-transit-credentials-runtime-")); + roots.push(root); + const infraDir = join(root, "infra/docker"), + catalog = join(infraDir, "data/.transitous-catalog"), + atlas = join(catalog, "transitland-atlas"), + local = join(root, "services/motis/tools/transitous"); + mkdirSync(join(catalog, "feeds"), { recursive: true }); + mkdirSync(join(atlas, "feeds"), { recursive: true }); + mkdirSync(local, { recursive: true }); + git(atlas, ["init"]); + writeFileSync( + join(atlas, "feeds/a.dmfr.json"), + JSON.stringify({ feeds: [{ id: "auth", authorization: { type: "api-key" } }] }), + ); + git(atlas, ["add", "."]); + git(atlas, ["commit", "-m", "Atlas"]); + const atlasSha = git(atlas, ["rev-parse", "HEAD"]); + git(catalog, ["init"]); + writeFileSync( + join(catalog, "feeds/de.json"), + JSON.stringify({ sources: [{ name: "A", "transitland-atlas-id": "auth" }] }), + ); + git(catalog, ["add", "feeds"]); + git(catalog, ["update-index", "--add", "--cacheinfo", `160000,${atlasSha},transitland-atlas`]); + git(catalog, ["commit", "-m", "Transitous"]); + const sha = git(catalog, ["rev-parse", "HEAD"]); + writeFileSync( + join(infraDir, "transitous.lock.json"), + JSON.stringify({ + ref: `main@${sha}`, + submodules: { "transitland-atlas": atlasSha }, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + const gbfs = "b".repeat(40); + writeFileSync( + join(infraDir, "gbfs-catalog.lock.json"), + JSON.stringify({ + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: gbfs, + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${gbfs}/systems.csv`, + sha256: "c".repeat(64), + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + await provisionControlInstallation(infraDir); + const installation = openControlInstallation({ + infraDir, + role: "ops-agent", + serviceIds: ["motis", "data-manager"], + }); + try { + await initializeManagedCatalogPins({ rootDir: root, installation }); + } finally { + closeControlInstallation(installation); + } + const operation = { + kind: "transitCredentials.inspect" as const, + catalogRevisionId: readCatalogPinSet(root)!.generationId, + }; + let environment: Record = {}; + const runtime = createUnavailableRuntime(); + installTransitCredentialsRuntime(runtime, { + rootDir: root, + sourceToken: "synthetic-private-inspection-token", + stack: { + infraDir, + composePath: join(infraDir, "docker-compose.generated.yml"), + composeReleasePath: join(infraDir, "docker-compose.release.yml"), + }, + ownedServices: { inspectOwnedServices: async () => [] }, + execFile: async (_file, args) => { + expect(args).toContain("config"); + return { + stdout: JSON.stringify({ services: { "data-manager": { environment } } }), + stderr: "", + }; + }, + }); + const inspect = () => + dispatchOpsOperation(runtime, operation, { + execution: testExecution(), + signal: new AbortController().signal, + emitLog: () => {}, + claim: { + operation, + fingerprint: "a".repeat(64), + source: "registry", + capability: { revisionId: "fixture", values: {} }, + }, + }); + return { + root, + local, + inspect, + setEnv: (value: Record) => { + environment = value; + }, + }; +} +it("discovers pinned keys and masks host-file values without adopting or modifying them", async () => { + const f = await fixture(); + expect(await f.inspect()).toMatchObject({ + requiredKeys: ["de/A"], + owner: "unset", + editable: true, + configuredKeys: [], + }); + const file = join(f.local, "api-keys.json"), + bytes = '{"de/A":"sensitive","orphan":"still-sensitive","blank/orphan":""}'; + writeFileSync(file, bytes, { mode: 0o600 }); + const result = await f.inspect(); + expect(result).toMatchObject({ + owner: "file", + editable: false, + configuredKeys: ["de/A", "orphan"], + retainedKeys: ["blank/orphan", "de/A", "orphan"], + }); + expect(JSON.stringify(result)).not.toContain("sensitive"); + expect(readFileSync(file, "utf8")).toBe(bytes); + expect(result.hostCredentialsRevision).toMatch(/^[a-f0-9]{64}$/); + expect((await f.inspect()).hostCredentialsRevision).toBe(result.hostCredentialsRevision); + writeFileSync(file, '{"de/A":"rotated","orphan":"still-sensitive"}', { mode: 0o600 }); + expect((await f.inspect()).hostCredentialsRevision).not.toBe(result.hostCredentialsRevision); + f.setEnv({ TRANSITOUS_API_KEYS_PATH: "/operator/custom.json" }); + expect(await f.inspect()).toMatchObject({ + owner: "environment", + editable: false, + configuredKeys: [], + }); +}, 60_000); +it("refuses malformed existing files rather than treating the vault as unset", async () => { + const f = await fixture(); + writeFileSync(join(f.local, "api-keys.json"), '{"key":"first","key":"last"}', { mode: 0o600 }); + await expect(f.inspect()).rejects.toThrow(); +}, 60_000); diff --git a/apps/ops-agent/src/transit-credentials-runtime.ts b/apps/ops-agent/src/transit-credentials-runtime.ts new file mode 100644 index 000000000..7e0b0825c --- /dev/null +++ b/apps/ops-agent/src/transit-credentials-runtime.ts @@ -0,0 +1,231 @@ +/** Fixed read-only catalog/credential inspection. Host paths and values never leave the agent. */ +import { createHmac } from "node:crypto"; +import { lstatSync, realpathSync } from "node:fs"; +import { join } from "node:path"; +import { catalogRevisionId, readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import type { OpsResultFor } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { inspectPinnedFeedCredentialKeys, TRANSITOUS_CATALOG_DIR } from "@openmapx/transitous-core"; +import { parseFeedCredentialMapJson } from "@openmapx/transitous-core/feed-credentials"; +import { AnchoredDirectory } from "../../../packages/core/src/build-artifacts/filesystem"; +import { type DockerRuntimeOptions, runContainedProcess } from "./docker-runtime"; +import type { OwnedServiceResolver } from "./owned-services"; +import { OpsAgentBusyError, OpsNotWiredError, type OpsRuntime } from "./runtime"; + +const MANAGED = "/run/secrets/TRANSITOUS_API_KEYS"; +const DEFAULT = "/config/transitous/api-keys.json"; +export function installTransitCredentialsRuntime( + runtime: OpsRuntime, + options: { + rootDir: string; + sourceToken?: string; + stack: services.StackPaths; + ownedServices: OwnedServiceResolver; + execFile?: DockerRuntimeOptions["execFile"]; + }, +) { + let inspecting = false; + const docker = options.execFile ?? runContainedProcess; + runtime["transitCredentials.inspect"] = async (operation, context) => { + if (inspecting) throw new OpsAgentBusyError(); + inspecting = true; + try { + const pair = readCatalogPinSet(options.rootDir); + if (!pair || catalogRevisionId(pair) !== operation.catalogRevisionId) + throw new OpsNotWiredError(); + const requiredKeys = await inspectPinnedFeedCredentialKeys({ + catalogDir: join(options.stack.infraDir, "data", TRANSITOUS_CATALOG_DIR), + transitousCommit: pair.transitous.ref.split("@").at(-1)!, + atlasCommit: pair.transitous.submodules["transitland-atlas"]!, + git: async (args, cwd) => + ( + await runContainedProcess("git", ["-C", cwd, ...args], { + signal: context.signal, + timeout: 30_000, + maxBuffer: 5 * 1024 * 1024, + }) + ).stdout, + }); + // Configuration is collected through the existing fixed Compose argument builder. + const config = JSON.parse( + ( + await docker( + "docker", + [...services.stackComposeArgs(options.stack), "config", "--format", "json"], + { signal: context.signal, timeout: 30_000, maxBuffer: 1024 * 1024 }, + ) + ).stdout, + ); + const env = config.services?.["data-manager"]?.environment ?? {}; + let owner: OpsResultFor<"transitCredentials.inspect">["owner"] = "unset"; + let reason: string | null = null, + configuredKeys: string[] = [], + retainedKeys: string[] = [], + appliedCredentialsRevision: string | null = null, + hostCredentialsRevision: string | null = null; + const withDirectory = ( + components: string[], + read: (directory: AnchoredDirectory) => T, + ): T => { + const opened: AnchoredDirectory[] = []; + try { + let directory = new AnchoredDirectory(options.rootDir); + opened.push(directory); + for (const component of components) { + directory = directory.directory(component); + opened.push(directory); + } + return read(directory); + } finally { + for (const directory of opened.reverse()) directory.close(); + } + }; + const readMap = (components: string[]) => { + const parents = components.slice(0, -1), + file = components.at(-1)!; + return withDirectory(parents, (directory) => { + const bytes = directory.read(file, 1024 * 1024, false); + if (!options.sourceToken) throw new OpsNotWiredError(); + hostCredentialsRevision = createHmac("sha256", options.sourceToken) + .update("openmapx-host-transit-credentials-v1\0") + .update(JSON.stringify(components)) + .update("\0") + .update(bytes) + .digest("hex"); + const values = parseFeedCredentialMapJson( + new TextDecoder("utf-8", { fatal: true }).decode(bytes), + ); + retainedKeys = Object.keys(values).sort(); + return values; + }); + }; + if (env.TRANSITOUS_API_KEYS_PATH) { + owner = "environment"; + reason = + "TRANSITOUS_API_KEYS_PATH is managed on the host. Update that file on the host or explicitly remove the override."; + } else if (env.TRANSITOUS_API_KEYS_FILE && env.TRANSITOUS_API_KEYS_FILE !== MANAGED) { + owner = "environment"; + reason = "TRANSITOUS_API_KEYS_FILE is managed on the host."; + } else if (env.TRANSITOUS_API_KEYS_FILE === MANAGED) { + owner = "vault"; + const revision = services.currentConfigurationGeneration(options.stack.infraDir); + if (!revision) throw new OpsNotWiredError(); + const components = [ + "infra", + "docker", + services.CONFIGURATION_GENERATIONS, + revision, + services.GENERATED_SECRETS_DIRNAME, + "data-manager", + "TRANSITOUS_API_KEYS", + ]; + const values = readMap(components); + configuredKeys = Object.keys(values) + .filter((key) => values[key] !== "") + .sort(); + const owned = (await options.ownedServices.inspectOwnedServices(context.signal)).filter( + (service) => service.serviceId === "data-manager", + ); + if (owned.length === 1 && owned[0]!.state === "running") { + const observed = JSON.parse( + ( + await docker( + "docker", + ["container", "inspect", "--format", "{{json .}}", owned[0]!.containerId], + { + signal: context.signal, + timeout: 15_000, + maxBuffer: 256 * 1024, + }, + ) + ).stdout, + ); + const environment: string[] = observed?.Config?.Env ?? []; + const mounts: { Source?: string; Destination?: string; RW?: boolean }[] = + observed?.Mounts ?? []; + const mount = mounts.filter((item) => item.Destination === MANAGED && item.RW === false); + const expected = join(options.rootDir, ...components), + canonicalExpected = join(realpathSync(options.rootDir), ...components), + stableBase = services.configurationGenerationBasePath(options.stack.infraDir, revision), + canonicalStableBase = services.configurationGenerationBasePath( + realpathSync(options.stack.infraDir), + revision, + ), + allowedSources = [ + expected, + canonicalExpected, + ...[stableBase, canonicalStableBase].map((base) => + join( + base, + services.GENERATED_SECRETS_DIRNAME, + "data-manager", + "TRANSITOUS_API_KEYS", + ), + ), + ]; + if ( + mount.length === 1 && + mount[0]!.Source && + allowedSources.includes(mount[0]!.Source) && + environment.includes(`TRANSITOUS_API_KEYS_FILE=${MANAGED}`) && + !environment.some( + (item) => + item.startsWith("TRANSITOUS_API_KEYS_PATH=") && + item !== "TRANSITOUS_API_KEYS_PATH=", + ) + ) { + const a = lstatSync(expected), + b = lstatSync(mount[0]!.Source!); + if ( + a.isFile() && + b.isFile() && + !b.isSymbolicLink() && + a.dev === b.dev && + a.ino === b.ino && + realpathSync(expected) === realpathSync(mount[0]!.Source!) + ) + appliedCredentialsRevision = services.readAppliedTransitCredentialsRevision( + options.stack.infraDir, + ); + } + } + if (services.currentConfigurationGeneration(options.stack.infraDir) !== revision) + throw new OpsNotWiredError(); + } else { + try { + const values = readMap(["services", "motis", "tools", "transitous", "api-keys.json"]); + owner = "file"; + configuredKeys = Object.keys(values) + .filter((key) => values[key] !== "") + .sort(); + reason = `The existing ${DEFAULT} file is host-owned. Manage it on the host; it is never adopted automatically.`; + } catch (error) { + const entries = withDirectory(["services", "motis", "tools", "transitous"], (directory) => + directory.listBounded(1024), + ); + if (entries.includes("api-keys.json")) throw error; + owner = "unset"; + } + } + const current = readCatalogPinSet(options.rootDir); + if (!current || catalogRevisionId(current) !== operation.catalogRevisionId) + throw new OpsNotWiredError(); + return { + catalogRevisionId: operation.catalogRevisionId, + requiredKeys, + owner, + editable: owner === "vault" || owner === "unset", + reason, + configuredKeys, + retainedKeys, + appliedCredentialsRevision, + hostCredentialsRevision: owner === "file" ? hostCredentialsRevision : null, + }; + } catch (error) { + if (error instanceof OpsAgentBusyError || error instanceof OpsNotWiredError) throw error; + throw new OpsNotWiredError(); + } finally { + inspecting = false; + } + }; +} diff --git a/apps/web/src/app/admin/(full-admin)/transit/TransitPipelinePage.tsx b/apps/web/src/app/admin/(full-admin)/transit/TransitPipelinePage.tsx index ed173f233..6d00ffa60 100644 --- a/apps/web/src/app/admin/(full-admin)/transit/TransitPipelinePage.tsx +++ b/apps/web/src/app/admin/(full-admin)/transit/TransitPipelinePage.tsx @@ -8,6 +8,7 @@ import IconButton from "@mui/material/IconButton"; import Stack from "@mui/material/Stack"; import Tooltip from "@mui/material/Tooltip"; import { useQueryClient } from "@tanstack/react-query"; +import { TransitCredentialsSection } from "@/components/admin/services/TransitCredentialsSection"; import { AdminPageHeader } from "@/components/admin/shared/AdminPageHeader"; import { useTransitState } from "@/lib/admin/transitHooks"; import { CurrentJobCard } from "./CurrentJobCard"; @@ -43,6 +44,9 @@ export function TransitPipelinePage() { } /> + + + {isLoading && ( ({ + get: vi.fn(), + preview: vi.fn(), + apply: vi.fn(), + apiUrl: "http://api.test", + actorId: "admin-a", +})); +vi.mock("@/lib/admin/accountAdmin", () => ({ + useAccountAdmin: () => ({ apiUrl: mocks.apiUrl, actorId: mocks.actorId }), +})); +vi.mock("@openmapx/core/api-admin-sdk", async () => ({ + ...(await vi.importActual( + "@openmapx/core/api-admin-sdk", + )), + getAdminTransitCredentials: mocks.get, + previewAdminTransitCredentials: mocks.preview, + applyAdminTransitCredentials: mocks.apply, +})); +vi.mock("../users/AdminReauthenticationDialog", () => ({ + AdminReauthenticationDialog: ({ + onCompleted, + }: { + onCompleted: (proofs: { challengeId: string }[]) => void; + }) => ( + + ), +})); +const reply = (data: unknown) => ({ status: 200, data, headers: new Headers() }); +const view = { + catalogRevisionId: "a".repeat(64), + credentialsRevision: "b".repeat(64), + configurationRevisionId: `cfg1_${"c".repeat(43)}`, + owner: "vault", + editable: true, + reason: null, + appliedCredentialsRevision: "b".repeat(64), + pendingApplication: false, + entries: [ + { key: "de/A", required: true, configured: true, orphaned: false }, + { key: "old/retained", required: false, configured: true, orphaned: true }, + ], +}; +beforeEach(() => { + vi.clearAllMocks(); + sessionStorage.clear(); + mocks.get.mockResolvedValue(reply(view)); + mocks.apiUrl = "http://api.test"; + mocks.actorId = "admin-a"; +}); +afterEach(cleanup); +function mount() { + const query = createTestQueryClient(); + const result = render( + + + , + ); + return { ...result, query }; +} +it("shows key IDs and orphan/owner metadata with blank password fields", async () => { + mount(); + expect(await screen.findByText("de/A")).toBeInTheDocument(); + expect(screen.getByText("old/retained")).toBeInTheDocument(); + expect(screen.getByText("Orphaned")).toBeInTheDocument(); + expect(screen.getByLabelText("New value for de/A")).toHaveValue(""); + expect(mocks.preview).not.toHaveBeenCalled(); +}); +it("clears submitted values on a failed preview and never keeps them in a query/mutation cache", async () => { + mocks.preview.mockRejectedValue(new Error("network unavailable")); + const { query } = mount(); + const input = await screen.findByLabelText("New value for de/A"); + fireEvent.change(input, { target: { value: "synthetic-private-ui-key" } }); + fireEvent.click(screen.getByRole("button", { name: "Review credential changes" })); + await waitFor(() => expect(mocks.preview).toHaveBeenCalledTimes(1)); + expect(input).toHaveValue(""); + expect( + JSON.stringify( + query + .getQueryCache() + .getAll() + .map((entry) => entry.state.data), + ), + ).not.toContain("synthetic-private-ui-key"); + expect(query.getMutationCache().getAll()).toHaveLength(0); + expect(JSON.stringify(sessionStorage)).not.toContain("synthetic-private-ui-key"); + expect(await screen.findByText(/Could not prepare credential changes/)).toBeInTheDocument(); +}); +it("keeps host-owned values read-only and requires explicit deletion confirmation", async () => { + mocks.get.mockResolvedValue( + reply({ ...view, owner: "file", editable: false, reason: "Manage the existing host file." }), + ); + mount(); + expect(await screen.findByText("Manage the existing host file.")).toBeInTheDocument(); + expect(screen.queryByLabelText("New value for de/A")).not.toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: "Review credential changes" }), + ).not.toBeInTheDocument(); +}); +it("does not stage deletion until its confirmation is accepted", async () => { + mount(); + await screen.findByText("old/retained"); + const review = screen.getByRole("button", { name: "Review credential changes" }); + fireEvent.click(screen.getByRole("button", { name: "Delete old/retained" })); + expect(review).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Confirm deletion" })); + expect((review as HTMLButtonElement).disabled).toBe(false); + expect(mocks.apply).not.toHaveBeenCalled(); +}); +const reviewedReply = () => + reply({ + previewId: "22222222-2222-4222-8222-222222222222", + intentFingerprint: "d".repeat(64), + catalogRevisionId: view.catalogRevisionId, + credentialsRevision: view.credentialsRevision, + expectedCurrentRevisionId: view.configurationRevisionId, + expiresAt: new Date(Date.now() + 600000).toISOString(), + changes: [{ key: "de/A", action: "set" }], + preservedCount: 1, + mapBytes: 27, + }); +it("does not apply a completed proof after the observed map changes while the proof dialog is open", async () => { + mocks.preview.mockResolvedValue(reviewedReply()); + const { query } = mount(); + fireEvent.change(await screen.findByLabelText("New value for de/A"), { + target: { value: "synthetic-secret" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Review credential changes" })); + fireEvent.click( + await screen.findByRole("button", { name: "Confirm administrator sign-in and apply" }), + ); + query.setQueryData(["admin", "transit", "credentials", mocks.apiUrl, mocks.actorId], { + ...view, + credentialsRevision: "e".repeat(64), + }); + await screen.findByText(/The catalog, configuration or credentials changed/); + fireEvent.click(screen.getByRole("button", { name: "Test confirmed sign-in" })); + await waitFor(() => expect(mocks.apply).not.toHaveBeenCalled()); +}); +it("discards a late preview response when the actor changes", async () => { + let resolve!: (value: ReturnType) => void; + mocks.preview.mockReturnValue( + new Promise((done) => { + resolve = done; + }), + ); + const mounted = mount(); + fireEvent.change(await screen.findByLabelText("New value for de/A"), { + target: { value: "synthetic-secret" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Review credential changes" })); + await waitFor(() => expect(mocks.preview).toHaveBeenCalledTimes(1)); + mocks.actorId = "admin-b"; + mounted.rerender( + + + , + ); + await screen.findByLabelText("New value for de/A"); + expect((mocks.preview.mock.calls[0]![1] as { signal: AbortSignal }).signal.aborted).toBe(true); + resolve(reviewedReply()); + await waitFor(() => + expect( + screen.queryByRole("button", { name: "Confirm administrator sign-in and apply" }), + ).not.toBeInTheDocument(), + ); + expect(screen.getByLabelText("New value for de/A")).toHaveValue(""); + expect(sessionStorage.length).toBe(0); +}); diff --git a/apps/web/src/components/admin/services/TransitCredentialsSection.tsx b/apps/web/src/components/admin/services/TransitCredentialsSection.tsx new file mode 100644 index 000000000..5c115d2ff --- /dev/null +++ b/apps/web/src/components/admin/services/TransitCredentialsSection.tsx @@ -0,0 +1,431 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Box from "@mui/material/Box"; +import Button from "@mui/material/Button"; +import Chip from "@mui/material/Chip"; +import CircularProgress from "@mui/material/CircularProgress"; +import Dialog from "@mui/material/Dialog"; +import DialogActions from "@mui/material/DialogActions"; +import DialogContent from "@mui/material/DialogContent"; +import DialogTitle from "@mui/material/DialogTitle"; +import Paper from "@mui/material/Paper"; +import Stack from "@mui/material/Stack"; +import Table from "@mui/material/Table"; +import TableBody from "@mui/material/TableBody"; +import TableCell from "@mui/material/TableCell"; +import TableHead from "@mui/material/TableHead"; +import TablePagination from "@mui/material/TablePagination"; +import TableRow from "@mui/material/TableRow"; +import TextField from "@mui/material/TextField"; +import Typography from "@mui/material/Typography"; +import { createApiClient } from "@openmapx/core"; +import * as sdk from "@openmapx/core/api-admin-sdk"; +import { + type FeedCredentialPreviewInput, + feedCredentialPreviewSchema, +} from "@openmapx/core/api-contracts"; + +type FeedCredentialChange = FeedCredentialPreviewInput["changes"][number]; + +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { useAccountAdmin } from "@/lib/admin/accountAdmin"; +import { browserAdminApiBase } from "@/lib/admin/browserAdminApiBase"; +import { AdminReauthenticationDialog } from "../users/AdminReauthenticationDialog"; + +type Preview = ReturnType; +type Reviewed = { preview: Preview; operationKey: string }; +function restored(key: string): Reviewed | null { + if (typeof window === "undefined") return null; + try { + const raw = sessionStorage.getItem(key); + if (!raw) return null; + const value = JSON.parse(raw); + if ( + Object.keys(value).sort().join(",") !== "operationKey,preview" || + !/^opk1_[A-Za-z0-9_-]{16,64}$/.test(value.operationKey) + ) + return null; + const preview = feedCredentialPreviewSchema.parse(value.preview); + return Date.parse(preview.expiresAt) > Date.now() + ? { preview, operationKey: value.operationKey } + : null; + } catch { + return null; + } +} +export function TransitCredentialsSection() { + const { apiUrl, actorId } = useAccountAdmin(); + return ( + + ); +} +function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: string }) { + const client = useMemo( + () => createApiClient({ baseUrl: browserAdminApiBase(apiUrl), credentials: "include" }), + [apiUrl], + ); + const queryClient = useQueryClient(), + query = useQuery({ + queryKey: ["admin", "transit", "credentials", apiUrl, actorId], + queryFn: async ({ signal }) => + sdk.unwrapAdminResponse(await sdk.getAdminTransitCredentials({ client, signal })), + gcTime: 0, + enabled: Boolean(actorId), + }); + const storageKey = `openmapx-transit-credential-preview:${JSON.stringify([apiUrl, actorId])}`; + const [drafts, setDrafts] = useState>({}), + [deletions, setDeletions] = useState([]), + [deleting, setDeleting] = useState(null), + [reviewed, setReviewed] = useState(null), + [proving, setProving] = useState(false), + [busy, setBusy] = useState(false), + [error, setError] = useState(null), + [success, setSuccess] = useState(null), + [page, setPage] = useState(0); + const epoch = useRef(0), + request = useRef(null); + useEffect(() => { + setReviewed(restored(storageKey)); + return () => { + epoch.current++; + request.current?.abort(); + }; + }, [storageKey]); + const invalidateReview = () => { + epoch.current++; + request.current?.abort(); + setReviewed(null); + setProving(false); + try { + sessionStorage.removeItem(storageKey); + } catch { + /* No secret is stored; private drafts still clear. */ + } + setError(null); + setSuccess(null); + }; + const changes: FeedCredentialChange[] = [ + ...Object.entries(drafts) + .filter(([, value]) => value !== "") + .map(([key, value]) => ({ key, action: "set" as const, value })), + ...deletions.map((key) => ({ key, action: "delete" as const })), + ]; + const preview = async () => { + const data = query.data; + if (!data || !data.editable || !changes.length || changes.length > 100) return; + invalidateReview(); + const identity = epoch.current, + controller = new AbortController(); + request.current = controller; + setBusy(true); + const operationKey = `opk1_${crypto.randomUUID().replaceAll("-", "")}`; + const body = { + catalogRevisionId: data.catalogRevisionId, + expectedCredentialsRevision: data.credentialsRevision, + expectedCurrentRevisionId: data.configurationRevisionId, + changes, + operationKey, + }; + // No secret-bearing mutation variables enter React Query, storage or review state. + setDrafts({}); + setDeletions([]); + try { + const value = sdk.unwrapAdminResponse( + await sdk.previewAdminTransitCredentials(body, { client, signal: controller.signal }), + ); + if (identity !== epoch.current) return; + const next = { preview: feedCredentialPreviewSchema.parse(value), operationKey }; + setReviewed(next); + sessionStorage.setItem(storageKey, JSON.stringify(next)); + } catch { + if (identity === epoch.current) + setError( + "Could not prepare credential changes. Refresh and enter the intended changes again.", + ); + } finally { + if (identity === epoch.current) setBusy(false); + } + }; + const apply = async (challengeId: string) => { + const selected = reviewed, + current = query.data; + if (!selected || !current) return; + if ( + !current.editable || + selected.preview.catalogRevisionId !== current.catalogRevisionId || + selected.preview.credentialsRevision !== current.credentialsRevision || + selected.preview.expectedCurrentRevisionId !== current.configurationRevisionId || + Date.parse(selected.preview.expiresAt) <= Date.now() + ) { + invalidateReview(); + setError("The catalog, configuration or credentials changed. Prepare a new review."); + return; + } + setProving(false); + const identity = epoch.current, + controller = new AbortController(); + request.current = controller; + setBusy(true); + setError(null); + try { + const result = sdk.unwrapAdminResponse( + await sdk.applyAdminTransitCredentials( + { + previewId: selected.preview.previewId, + intentFingerprint: selected.preview.intentFingerprint, + operationKey: selected.operationKey, + transitProofToken: challengeId, + }, + { client, signal: controller.signal }, + ), + ); + if (identity !== epoch.current) return; + setSuccess( + result.configuration.outcome === "committed" && result.configuration.followUp === "complete" + ? "Feed credentials applied." + : "Credential application needs attention. Inspect the original operation in Activity.", + ); + setReviewed(null); + try { + sessionStorage.removeItem(storageKey); + } catch { + /* No secret is stored; private drafts still clear. */ + } + await queryClient.invalidateQueries({ queryKey: ["admin", "transit"] }); + } catch { + if (identity === epoch.current) { + setError( + "Credential application did not complete. Inspect Activity before preparing another change.", + ); + setReviewed(null); + try { + sessionStorage.removeItem(storageKey); + } catch { + /* No secret is stored; private drafts still clear. */ + } + } + } finally { + if (identity === epoch.current) { + setDrafts({}); + setDeletions([]); + setBusy(false); + } + } + }; + const data = query.data, + stale = + reviewed && + data && + (reviewed.preview.catalogRevisionId !== data.catalogRevisionId || + reviewed.preview.credentialsRevision !== data.credentialsRevision || + reviewed.preview.expectedCurrentRevisionId !== data.configurationRevisionId || + Date.parse(reviewed.preview.expiresAt) <= Date.now()); + const entries = data?.entries ?? [], + effectivePage = Math.min(page, Math.max(0, Math.ceil(entries.length / 25) - 1)); + return ( + + + Transit feed credentials + + Values stay masked. Blank inputs keep the existing value; deletion is a separate, + confirmed action. Changes use the selected catalog pins. + + {query.isLoading && } + {query.isError && ( + + Feed credential inspection is unavailable. Refresh after restoring instance + connectivity. + + )} + {data && ( + <> + + + + + {!data.editable && ( + + {data.reason ?? "Manage these credentials on the host."} + + )} + {data.pendingApplication && ( + + Saved credentials have not been verified in the running Data Manager mount. Inspect + Activity for the original application. + + )} + {error && {error}} + {success && {success}} + + + + + Key + Status + {data.editable && ( + <> + New value + Action + + )} + + + + {entries.slice(effectivePage * 25, effectivePage * 25 + 25).map((entry) => ( + + {entry.key} + + + + {entry.required && } + {entry.orphaned && } + {deletions.includes(entry.key) && ( + + )} + + + {data.editable && ( + <> + + { + invalidateReview(); + setDrafts((current) => ({ + ...current, + [entry.key]: event.target.value, + })); + setDeletions((current) => + current.filter((key) => key !== entry.key), + ); + }} + /> + + + + + + )} + + ))} + +
+ + setPage(next)} + /> + {changes.length > 100 && ( + Review at most 100 key changes at a time. + )} + {data.editable && ( + + )} + + )} + {reviewed && ( + + {stale + ? "The catalog, configuration or credentials changed. Prepare a new review." + : `Review ${reviewed.preview.changes.length} key changes. Existing unselected values are preserved.`} + + {reviewed.preview.changes.map((change) => ( + + {change.action === "delete" ? "Delete" : "Set"}: {change.key} + + ))} + + + + + + + )} + {deleting && ( + setDeleting(null)}> + Delete feed credential? + + + {deleting} will be removed from the managed map. Other keys will be preserved. + + + + + + + + )} + {proving && reviewed && ( + setProving(false)} + onCompleted={(proofs) => { + const proof = proofs[0]; + if (proof) void apply(proof.challengeId); + }} + /> + )} + + + ); +} diff --git a/apps/web/src/lib/admin/accountAdmin.ts b/apps/web/src/lib/admin/accountAdmin.ts index 62c2246d7..cdffe79f2 100644 --- a/apps/web/src/lib/admin/accountAdmin.ts +++ b/apps/web/src/lib/admin/accountAdmin.ts @@ -22,6 +22,8 @@ const messages: Record = { ACCOUNT_ACTION_CONFLICT: "This request changed. Refresh before trying again.", REAUTHENTICATION_REQUIRED: "Confirm a fresh administrator sign-in for this action.", AUTH_POLICY_CONFLICT: "The active policy changed. Reload before applying your changes.", + ADMIN_OPERATION_IN_PROGRESS: + "Finish or cancel this administrator’s instance operations in Activity before deleting the account.", LAST_ADMIN_REQUIRED: "At least one active administrator must remain.", ROLE_VERSION_CONFLICT: "This account changed. Refresh before changing its role.", INVALID_PASSWORD: "The password does not meet the instance policy.", diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index 34a7130d6..f0fc2d451 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -330,3 +330,27 @@ health snapshots and the running job log, see [Monitoring](./monitoring.md). styles, and the build/link pipeline behind the data inventory. - **[Configuration](../install/configuration.md)** — the `.env` file, secrets, and how the admin config forms relate to environment variables. + +## Transit feed credentials + +Use **Transit → Transit feed credentials** to manage the encrypted +`data-manager/TRANSITOUS_API_KEYS` map. The panel displays key names and whether +values are configured, required by the active catalog, or retained as orphaned +entries. It never returns existing values. A blank input preserves the existing +value; deletion is a separate confirmed action. + +Review changes before confirming a fresh administrator sign-in. The review is +bound to the current catalog pins, credential revision and configuration. If any +changes, prepare a new review. The complete operation saves the map and updates +the running Data Manager's credential mount under one maintenance reservation. +A pending-application warning means the running mount has not been verified; use +Activity to inspect the original operation. A retry observes the retained work +rather than applying it twice. Finish or cancel an administrator's unresolved +instance operations before deleting their account; those operations retain the +private inputs needed for recovery. + +File and environment overrides remain host-owned. The panel explains that +ownership and does not silently adopt their values. The host-local +`openmapx data generate-api-keys` command creates or updates a template from the +active pins while preserving all existing values and orphaned keys. The previous +browser generator operation is no longer available. diff --git a/packages/cli/__tests__/cli-compose.test.ts b/packages/cli/__tests__/cli-compose.test.ts index 889587701..181ab7133 100644 --- a/packages/cli/__tests__/cli-compose.test.ts +++ b/packages/cli/__tests__/cli-compose.test.ts @@ -619,7 +619,7 @@ describe("renderComposeForRepo and what the admin panel applied", () => { }; /** An admin apply that saved config and a vault secret for ingest without enabling it. */ - async function adminApply() { + async function adminApply(transitCredentialsRevision?: string) { writeManifest("alpha", { ...baseManifest, id: "alpha" }); writeManifest("ingest", ingest); const registry = new coreServices.ServiceRegistry({ rootDir: tmp }); @@ -629,6 +629,7 @@ describe("renderComposeForRepo and what the admin panel applied", () => { services: registry.list(), input: { domain: "example.com", + ...(transitCredentialsRevision ? { transitCredentialsRevision } : {}), selectedRoots: ["alpha"], serviceConfigs: [{ serviceId: "ingest", values: { MODE: "saved" } }], integrationConfigs: [], @@ -641,6 +642,15 @@ describe("renderComposeForRepo and what the admin panel applied", () => { delete process.env.SERVICE_INGEST_MODE; }); + it("preserves the opaque transit credential version during the actual CLI render", async () => { + const revision = "c".repeat(64); + await adminApply(revision); + await renderComposeForRepo({ rootDir: tmp, domain: "example.com", services: ["alpha"] }); + expect(coreServices.readAppliedTransitCredentialsRevision(join(tmp, "infra/docker"))).toBe( + revision, + ); + }); + it("keeps saved config and vault secrets, also for a service it enables later", async () => { await adminApply(); diff --git a/packages/cli/__tests__/cli-transitous-api-keys.test.ts b/packages/cli/__tests__/cli-transitous-api-keys.test.ts index cc2cd946a..3c74d9b67 100644 --- a/packages/cli/__tests__/cli-transitous-api-keys.test.ts +++ b/packages/cli/__tests__/cli-transitous-api-keys.test.ts @@ -1,6 +1,13 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { type CommandRunner, @@ -58,114 +65,141 @@ afterEach(() => { }); describe("generateTransitousApiKeys", () => { - it("clones the Transitous catalog and preserves existing filled keys", async () => { - const outputPath = join(tmp, "services", "motis", "tools", "transitous", "api-keys.json"); - writeFileSync( - outputPath, - JSON.stringify( - { - "us-ca/NeedsKey": "preserved-secret", - "legacy/no-longer-needed": "stale-value", + it("requires active paired pins without refreshing upstream or changing existing values", async () => { + const output = join(tmp, "services/motis/tools/transitous/api-keys.json"), + before = JSON.stringify({ "retained/orphan": " exact value\n" }); + writeFileSync(output, before); + let calls = 0; + await expect( + generateTransitousApiKeys({ + rootDir: tmp, + runner: async () => { + calls++; }, - null, - 2, - ), - "utf-8", - ); - - const catalogDir = join(tmp, "infra", "docker", "data", ".transitous-catalog"); - const calls: Array<{ command: string; args: string[]; cwd?: string }> = []; - const runner: CommandRunner = async (command, args, opts) => { - calls.push({ command, args, cwd: opts.cwd }); - if (command === "git" && args.includes("clone")) { - const targetDir = args.at(-1); - if (typeof targetDir === "string") { - writeTransitousCatalog(targetDir); - } - } - }; - - const result = await generateTransitousApiKeys({ rootDir: tmp, runner }); - - expect(result.requiredCount).toBe(2); - expect(result.preservedCount).toBe(1); - expect(result.droppedCount).toBe(1); - expect(result.outputPath).toBe(outputPath); - expect(JSON.parse(readFileSync(outputPath, "utf-8")) as Record).toStrictEqual({ - "ca-qc/QuebecAgency": "", - "us-ca/NeedsKey": "preserved-secret", - }); - expect(calls).toEqual([ - { - command: "git", - args: [ - "-c", - `safe.directory=${catalogDir}`, - "clone", - "--depth", - "1", - "--recurse-submodules", - "--shallow-submodules", - "--", - DEFAULT_TRANSITOUS_REPO_URL, - catalogDir, - ], - cwd: join(tmp, "infra", "docker", "data"), - }, - ]); + }), + ).rejects.toThrow(/pins/); + expect(calls).toBe(0); + expect(readFileSync(output, "utf8")).toBe(before); }); + it("refuses mutable repository overrides before any acquisition or output", async () => { + let calls = 0; + await expect( + generateTransitousApiKeys({ + rootDir: tmp, + transitousRepoUrl: "https://example.test/upstream", + runner: async () => { + calls++; + }, + }), + ).rejects.toThrow(/overrides/); + expect(calls).toBe(0); + }); +}); - it("uses an existing cached catalog and continues when git pull fails", async () => { - const catalogDir = join(tmp, "infra", "docker", "data", ".transitous-catalog"); - mkdirSync(join(catalogDir, ".git"), { recursive: true }); - writeTransitousCatalog(catalogDir); - - const safe = `safe.directory=${catalogDir}`; - const calls: Array<{ command: string; args: string[]; cwd?: string }> = []; - const runner: CommandRunner = async (command, args, opts) => { - calls.push({ command, args, cwd: opts.cwd }); - if (command === "git" && args.includes("pull")) { - throw new Error("network unavailable"); - } - }; - - const result = await generateTransitousApiKeys({ rootDir: tmp, runner }); - - expect(result.requiredCount).toBe(2); - expect(result.preservedCount).toBe(0); - expect(result.droppedCount).toBe(0); - const outputPath = join(tmp, "services", "motis", "tools", "transitous", "api-keys.json"); - expect(JSON.parse(readFileSync(outputPath, "utf-8")) as Record).toStrictEqual({ - "ca-qc/QuebecAgency": "", - "us-ca/NeedsKey": "", - }); - expect(calls).toEqual([ - { - command: "git", - args: ["-c", safe, "-C", catalogDir, "reset", "--hard", "HEAD"], - cwd: catalogDir, - }, - { - command: "git", - args: ["-c", safe, "-C", catalogDir, "pull", "--ff-only"], - cwd: join(tmp, "infra", "docker", "data"), +it("uses actual active pin A while upstream is B and preserves unrelated credential bytes", async () => { + const git = (cwd: string, args: string[]) => + execFileSync("git", ["-c", "protocol.file.allow=always", ...args], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + GIT_AUTHOR_NAME: "OpenMapX fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "OpenMapX fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", }, - { - command: "git", - args: [ - "-c", - safe, - "-C", - catalogDir, - "submodule", - "update", - "--init", - "--checkout", - "--depth", - "1", - ], - cwd: join(tmp, "infra", "docker", "data"), - }, - ]); + }).trim(); + const upstream = join(tmp, "upstream"), + cache = join(tmp, "infra/docker/data/.transitous-catalog"); + mkdirSync(upstream); + writeTransitousCatalog(upstream); + git(upstream, ["init"]); + const atlas = join(upstream, "transitland-atlas"); + git(atlas, ["init"]); + git(atlas, ["add", "."]); + git(atlas, ["commit", "-m", "Atlas A"]); + const atlasSha = git(atlas, ["rev-parse", "HEAD"]); + writeFileSync( + join(upstream, ".gitmodules"), + `[submodule "transitland-atlas"]\npath = transitland-atlas\nurl = ${atlas}\n`, + ); + writeFileSync( + join(upstream, "feeds/de.json"), + JSON.stringify({ sources: [{ name: "PinnedA", "transitland-atlas-id": "atlas-auth-1" }] }), + ); + git(upstream, ["add", "feeds", ".gitmodules"]); + git(upstream, ["update-index", "--add", "--cacheinfo", `160000,${atlasSha},transitland-atlas`]); + git(upstream, ["commit", "-m", "Catalog A"]); + const sha = git(upstream, ["rev-parse", "HEAD"]); + mkdirSync(join(tmp, "infra/docker/data"), { recursive: true }); + git(tmp, ["clone", "--recurse-submodules", "--", upstream, cache]); + const infra = join(tmp, "infra/docker"); + writeFileSync( + join(infra, "transitous.lock.json"), + JSON.stringify({ + ref: `main@${sha}`, + submodules: { "transitland-atlas": atlasSha }, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + writeFileSync( + join(infra, "gbfs-catalog.lock.json"), + JSON.stringify({ + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "a".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"a".repeat(40)}/systems.csv`, + sha256: "b".repeat(64), + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + await provisionControlInstallation(infra); + const installation = openControlInstallation({ + infraDir: infra, + role: "host-cli", + serviceIds: ["motis"], }); -}); + try { + await initializeManagedCatalogPins({ rootDir: tmp, installation }); + } finally { + closeControlInstallation(installation); + } + writeFileSync( + join(upstream, "feeds/de.json"), + JSON.stringify({ sources: [{ name: "UpstreamB", "transitland-atlas-id": "atlas-auth-1" }] }), + ); + git(upstream, ["add", "feeds"]); + git(upstream, ["commit", "-m", "Catalog B"]); + const output = join(tmp, "services/motis/tools/transitous/api-keys.json"); + writeFileSync(output, JSON.stringify({ "retained/orphan": " exact value\n" })); + const options = { + rootDir: tmp, + runner: async (command: string, args: string[], options: { cwd?: string }) => { + expect(command).toBe("git"); + git(options.cwd ?? tmp, args); + }, + }; + const result = await generateTransitousApiKeys(options); + expect(result).toMatchObject({ + requiredCount: 3, + preservedCount: 1, + droppedCount: 0, + orphanedKeys: ["retained/orphan"], + }); + expect(lstatSync(output).mode & 0o777).toBe(0o600); + const values = JSON.parse(readFileSync(output, "utf8")); + expect(values).toHaveProperty("de/PinnedA", ""); + expect(values).not.toHaveProperty("de/UpstreamB"); + expect(values["retained/orphan"]).toBe(" exact value\n"); + expect(git(cache, ["rev-parse", "HEAD"])).toBe(sha); + const valid = readFileSync(output, "utf8"), + malformed = '{"key":"one","key":"two"}'; + writeFileSync(output, malformed); + await expect(generateTransitousApiKeys(options)).rejects.toThrow(/Feed credential/); + expect(readFileSync(output, "utf8")).toBe(malformed); + writeFileSync(output, valid); + await expect(generateTransitousApiKeys(options)).resolves.toMatchObject({ droppedCount: 0 }); +}, 45000); diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index 5e150cde4..78ca5be4d 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -306,6 +306,9 @@ export async function renderComposeWithWriter( overrides: opts.domain === undefined ? {} : { domain: opts.domain }, }).values.domain, ...(saved.platform ? { platform: saved.platform } : {}), + ...(saved.transitCredentialsRevision + ? { transitCredentialsRevision: saved.transitCredentialsRevision } + : {}), selectedRoots: applied.requestedIds, serviceConfigs, integrationConfigs: saved.integrationConfigs.filter((entry) => diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index d93e004a7..71dd73490 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -885,16 +885,14 @@ export function registerDataCommands(program: Command): void { .description( "Generate Transitous API-key template at services/motis/tools/transitous/api-keys.json", ) - .option("--repo-url ", "Override Transitous catalog git URL") .option( "--output ", "Override output path (default: services/motis/tools/transitous/api-keys.json)", ) - .action(async (options: { repoUrl?: string; output?: string }) => { + .action(async (options: { output?: string }) => { try { - log.dim("Syncing Transitous catalog and scanning feeds that require API keys..."); + log.dim("Reading pinned Transitous catalog and scanning feeds that require API keys..."); const result = await generateTransitousApiKeys({ - transitousRepoUrl: options.repoUrl, outputPath: options.output, }); const preserved = diff --git a/packages/cli/src/lib/transitous-api-keys.ts b/packages/cli/src/lib/transitous-api-keys.ts index ba44cf544..344dc11c9 100644 --- a/packages/cli/src/lib/transitous-api-keys.ts +++ b/packages/cli/src/lib/transitous-api-keys.ts @@ -1,219 +1,288 @@ -import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { randomBytes, randomUUID } from "node:crypto"; +import { + closeSync, + constants, + existsSync, + fchmodSync, + fstatSync, + lstatSync, + openSync, + readSync, + realpathSync, +} from "node:fs"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import { createControlRootOperationKey, executionFingerprint } from "@openmapx/core/ops"; +import { fsyncDirectory } from "@openmapx/core/release-files"; import { type CommandRunner, DEFAULT_TRANSITOUS_REPO_URL, - ensureCatalog, + inspectPinnedFeedCredentialKeys, + mergeFeedCredentialTemplate, + parseFeedCredentialMapJson, TRANSITOUS_CATALOG_DIR, - type TransitousFeedFile, } from "@openmapx/transitous-core"; import { execa } from "execa"; +import { cliControlContext } from "./control-context"; import { cliProcessOptions } from "./execution-deadline"; import { repoPaths } from "./paths"; +import { + type PreparedPlatformFileReplacement, + preparePlatformFileReplacement, +} from "./platform-secret-files"; export type { CommandRunner }; export { DEFAULT_TRANSITOUS_REPO_URL }; - -interface TransitousAtlasFeed { - id?: string; - authorization?: unknown; -} - -interface TransitousAtlasFile { - feeds?: TransitousAtlasFeed[]; -} - export interface GenerateTransitousApiKeysOptions { rootDir?: string; transitousRepoUrl?: string; outputPath?: string; runner?: CommandRunner; } - export interface GenerateTransitousApiKeysResult { outputPath: string; catalogDir: string; requiredCount: number; preservedCount: number; droppedCount: number; + orphanedKeys: string[]; + catalogRevisionId: string; } - -async function defaultRunner( - command: string, - args: string[], - opts: { cwd?: string; stdio?: "inherit" | "pipe" }, -): Promise { - await execa(command, args, { +async function nativeGit(args: readonly string[], cwd: string): Promise { + const result = await execa("git", [...args], { ...cliProcessOptions(), - cwd: opts.cwd, - stdio: opts.stdio ?? "pipe", + cwd, + stdin: "ignore", + maxBuffer: args.includes("ls-tree") ? 5 * 1024 * 1024 : 16384, }); + return result.stdout; } - -/** Defense-in-depth: refuse to write outside the monorepo root. */ -function assertInsideRoot(candidate: string, root: string): string { - const rel = relative(root, candidate); - if (rel.startsWith("..") || isAbsolute(rel)) { - throw new Error(`--output must resolve to a path inside ${root} (got ${candidate})`); +function outputInsideRoot(root: string, output: string) { + const rootStat = lstatSync(root); + if ( + !rootStat.isDirectory() || + rootStat.isSymbolicLink() || + rootStat.uid !== process.geteuid?.() || + (rootStat.mode & 0o022) !== 0 + ) + throw Error("Repository root ownership is unavailable"); + const value = relative(root, output); + if (!value || value.startsWith("..") || isAbsolute(value)) + throw Error("--output must be inside the repository"); + let current = root; + for (const part of value.split(/[\\/]/).slice(0, -1)) { + current = join(current, part); + const stat = lstatSync(current); + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.uid !== process.geteuid?.() || + (stat.mode & 0o022) !== 0 + ) + throw Error("Output parents must be existing owned directories"); } - return candidate; + const parent = realpathSync(dirname(output)), + base = realpathSync(root); + if (parent !== base && !parent.startsWith(`${base}/`)) + throw Error("Output parent leaves the repository"); + return value.split(/[\\/]/).join("/"); } - -function listFilesRecursively(dir: string, suffix: string): string[] { - if (!existsSync(dir)) return []; - const files: string[] = []; - const stack = [dir]; - - while (stack.length > 0) { - const current = stack.pop(); - if (!current) continue; - const entries = readdirSync(current, { withFileTypes: true }); - for (const entry of entries) { - const path = join(current, entry.name); - if (entry.isDirectory()) { - stack.push(path); - } else if (entry.isFile() && entry.name.endsWith(suffix)) { - files.push(path); - } +function existingSnapshot(path: string): { contents: string; identity: string } | null { + let fd: number | undefined; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const before = fstatSync(fd, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.uid !== BigInt(process.geteuid?.() ?? -1) || + before.size > 1048576n + ) + throw Error("Credential file is unavailable or oversized"); + const buffer = Buffer.alloc(Number(before.size) + 1); + let count = 0; + while (count < buffer.length) { + const n = readSync(fd, buffer, count, buffer.length - count, null); + if (!n) break; + count += n; } + const after = fstatSync(fd, { bigint: true }), + current = lstatSync(path, { bigint: true }); + const identity = (stat: typeof before) => + [stat.dev, stat.ino, stat.size, stat.mtimeNs, stat.ctimeNs].map(String).join(":"); + if ( + BigInt(count) !== before.size || + identity(before) !== identity(after) || + identity(before) !== identity(current) + ) + throw Error("Credential file changed during reading"); + return { + contents: new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, count)), + identity: identity(before), + }; + } catch (error) { + if (fd === undefined && (error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw Error("Credential file is unavailable or changed"); + } finally { + if (fd !== undefined) closeSync(fd); } - - return files.sort((a, b) => a.localeCompare(b)); } - -function collectAtlasAuthorisedFeedIds(catalogDir: string): Set { - const atlasFeedsDir = join(catalogDir, "transitland-atlas", "feeds"); - if (!existsSync(atlasFeedsDir)) { - throw new Error( - `Transitous catalog missing transitland-atlas feed directory: ${atlasFeedsDir}`, +export async function generateTransitousApiKeys( + options: GenerateTransitousApiKeysOptions = {}, +): Promise { + if ( + cliControlContext() || + process.env.OPENMAPX_CONTROL_EXECUTION || + process.env.OPENMAPX_CONTROL_DELEGATION || + process.env.OPENMAPX_PARENT_EXECUTION + ) + throw Error("Credential templates require an explicit local host command"); + if (options.transitousRepoUrl && options.transitousRepoUrl !== DEFAULT_TRANSITOUS_REPO_URL) + throw Error( + "Credential discovery uses the active paired pin and cached catalog; repository overrides are unavailable", ); - } - - const authorisedFeedIds = new Set(); - const dmfrFiles = listFilesRecursively(atlasFeedsDir, ".dmfr.json"); - for (const filePath of dmfrFiles) { - let payload: TransitousAtlasFile; - try { - payload = JSON.parse(readFileSync(filePath, "utf-8")) as TransitousAtlasFile; - } catch { - continue; - } - for (const feed of payload.feeds ?? []) { - if (typeof feed.id !== "string" || feed.id.length === 0) continue; - if (feed.authorization === undefined || feed.authorization === null) continue; - authorisedFeedIds.add(feed.id); - } - } - return authorisedFeedIds; -} - -function collectRequiredApiKeys(catalogDir: string, authorisedFeedIds: Set): string[] { - const feedsDir = join(catalogDir, "feeds"); - if (!existsSync(feedsDir)) { - throw new Error(`Transitous catalog missing feed directory: ${feedsDir}`); - } - - const keys = new Set(); - const feedFiles = readdirSync(feedsDir) - .filter((name) => name.endsWith(".json")) - .sort((a, b) => a.localeCompare(b)); - - for (const fileName of feedFiles) { - const region = fileName.replace(/\.json$/i, ""); - let feedFile: TransitousFeedFile; - try { - feedFile = JSON.parse(readFileSync(join(feedsDir, fileName), "utf-8")) as TransitousFeedFile; - } catch { - continue; - } - - for (const source of feedFile.sources ?? []) { - const atlasId = source["transitland-atlas-id"]; - if (typeof atlasId !== "string" || atlasId.length === 0) continue; - if (!authorisedFeedIds.has(atlasId)) continue; - if (typeof source["api-key"] === "string" && source["api-key"].trim().length > 0) continue; - if (typeof source["url-override"] === "string" && source["url-override"].trim().length > 0) - continue; - if (typeof source.name !== "string" || source.name.trim().length === 0) continue; - keys.add(`${region}/${source.name}`); - } - } - - return [...keys].sort((a, b) => a.localeCompare(b)); -} - -function readExistingApiKeys(outputPath: string): Record { - if (!existsSync(outputPath)) return {}; - let payload: unknown; + const paths = repoPaths(options.rootDir), + outputPath = options.outputPath + ? resolve(paths.root, options.outputPath) + : join(paths.root, "services/motis/tools/transitous/api-keys.json"), + outputRelative = outputInsideRoot(paths.root, outputPath); + let pins: ReturnType; try { - payload = JSON.parse(readFileSync(outputPath, "utf-8")); - } catch (error) { - throw new Error( - `Failed to parse existing api-keys file at ${outputPath}: ${(error as Error).message}`, + pins = readCatalogPinSet(paths.root); + } catch { + throw Error( + "Active paired catalog pins are unavailable; initialize or repair the instance first", ); } - if (!payload || typeof payload !== "object" || Array.isArray(payload)) { - throw new Error(`Expected api-keys file to contain a JSON object: ${outputPath}`); - } - - const existing: Record = {}; - for (const [key, value] of Object.entries(payload)) { - if (key.startsWith("_")) continue; - if (typeof value !== "string") continue; - const trimmed = value.trim(); - if (!trimmed) continue; - existing[key] = trimmed; - } - return existing; -} - -export async function generateTransitousApiKeys( - opts: GenerateTransitousApiKeysOptions = {}, -): Promise { - const runner = opts.runner ?? defaultRunner; - const paths = repoPaths(opts.rootDir); - const dataDir = join(paths.infraDir, "data"); - const outputPath = opts.outputPath - ? assertInsideRoot(resolve(paths.root, opts.outputPath), paths.root) - : join(paths.root, "services", "motis", "tools", "transitous", "api-keys.json"); - - mkdirSync(dataDir, { recursive: true }); - const catalogDir = await ensureCatalog({ - dataDir, - catalogDir: resolve(dataDir, TRANSITOUS_CATALOG_DIR), - repoUrl: opts.transitousRepoUrl ?? DEFAULT_TRANSITOUS_REPO_URL, - runner, - reset: true, - }); - - const existing = readExistingApiKeys(outputPath); - const requiredKeys = collectRequiredApiKeys( + if (!pins) + throw Error( + "Active paired catalog pins are required; initialize the instance before generating credentials", + ); + const sha = pins.transitous.ref.split("@").at(-1), + atlas = pins.transitous.submodules["transitland-atlas"]; + if (!sha || !atlas) throw Error("Exact Transitous and atlas pins are required"); + const before = existingSnapshot(outputPath); + const existing = before ? parseFeedCredentialMapJson(before.contents) : {}; + const catalogDir = join(paths.infraDir, "data", TRANSITOUS_CATALOG_DIR); + const required = await inspectPinnedFeedCredentialKeys({ catalogDir, - collectAtlasAuthorisedFeedIds(catalogDir), - ); - const requiredSet = new Set(requiredKeys); - const droppedCount = Object.keys(existing).filter((key) => !requiredSet.has(key)).length; - - const rendered: Record = {}; - let preservedCount = 0; - for (const key of requiredKeys) { - const preserved = existing[key]; - if (preserved) { - rendered[key] = preserved; - preservedCount++; - continue; + transitousCommit: sha, + atlasCommit: atlas, + git: async (args, cwd) => { + if (options.runner && !args.includes("rev-parse") && !args.includes("ls-tree")) { + await options.runner("git", [...args], { cwd, stdio: "pipe" }); + return ""; + } + return nativeGit(args, cwd); + }, + }); + const merged = mergeFeedCredentialTemplate(existing, required); + // Serialize the map by code-unit key order using the shared canonical encoder. + const { canonicalFeedCredentialMap } = await import("@openmapx/transitous-core"); + const contents = `${canonicalFeedCredentialMap(merged.values).canonicalJson}\n`; + if (!existsSync(join(paths.infraDir, "secrets/control-verifiers.json"))) + await control.provisionControlInstallation(paths.infraDir); + const installation = control.openControlInstallation({ + infraDir: paths.infraDir, + role: "host-cli", + serviceIds: ["motis"], + }); + let lease: control.ArtifactTargetLease | undefined; + let authority: control.VerifiedExecutionAuthority | undefined; + let began = false; + let replacement: PreparedPlatformFileReplacement | undefined; + try { + const operation = { + kind: "transit.credentials.template", + catalogRevisionId: pins.generationId, + outputPath: outputRelative, + fileMac: "0".repeat(64), + }; + operation.fileMac = control.localFeedCredentialTemplateMac(installation, operation, contents); + const admittedAt = Date.now(), + deadlineAt = admittedAt + 300000, + operationFingerprint = control.controlDigest(operation); + const execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }; + authority = control.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis"], + }); + lease = await control.acquireArtifactTargetLease({ + dataRoot: join(paths.infraDir, "data"), + serviceIds: ["motis"], + owner: authority, + }); + if (readCatalogPinSet(paths.root)?.generationId !== pins.generationId) + throw Error("Catalog pins changed; inspect the new pin before retrying"); + if (existingSnapshot(outputPath)?.identity !== before?.identity) + throw Error("Credential file changed; review before retrying"); + control.assertArtifactTargetLeaseForward(lease); + control.beginControlEffect(authority); + began = true; + replacement = preparePlatformFileReplacement(outputPath, contents, { + temporaryFileOps: { chmod: (fd) => fchmodSync(fd, 0o600) }, + }); + if (existingSnapshot(outputPath)?.identity !== before?.identity) + throw Error("Credential file changed; refusing to overwrite it"); + replacement.assertTargetUnchanged(); + replacement.commit(); + fsyncDirectory(dirname(outputPath)); + control.retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "feed-credential-template-written", fileMac: operation.fileMac }, + }); + await control.settleArtifactTargetLeaseChild( + lease, + control.verifyControlEffectSettlement(installation, execution), + ); + await control.releaseArtifactTargetLease(lease); + lease = undefined; + return { + outputPath, + catalogDir, + requiredCount: merged.requiredCount, + preservedCount: merged.preservedCount, + droppedCount: 0, + orphanedKeys: merged.orphanedKeys, + catalogRevisionId: pins.generationId, + }; + } catch (error) { + if (authority && !began) { + control.retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + if (lease) { + await control.settleArtifactTargetLeaseChild( + lease, + control.verifyControlEffectSettlement( + installation, + control.executionAuthorityState(authority).execution, + ), + ); + await control.releaseArtifactTargetLease(lease); + lease = undefined; + } } - rendered[key] = ""; + throw error; + } finally { + replacement?.cleanup(); + if (lease) control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(installation); } - - mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, `${JSON.stringify(rendered, null, 2)}\n`, "utf-8"); - - return { - outputPath, - catalogDir, - requiredCount: requiredKeys.length, - preservedCount, - droppedCount, - }; } diff --git a/packages/core/package.json b/packages/core/package.json index e97180f1b..da8e73b5c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -249,6 +249,7 @@ "@openmapx/air-quality": "workspace:*", "@openmapx/extension-contracts": "workspace:^", "@openmapx/mobility-core": "workspace:^", + "@openmapx/transitous-core": "workspace:^", "@tanstack/react-query": "^5.104.1", "@turf/helpers": "^7.4.0", "@turf/nearest-point-on-line": "^7.4.0", diff --git a/packages/core/src/api/contracts/data-manager-admin/index.ts b/packages/core/src/api/contracts/data-manager-admin/index.ts index 624178968..792a366cc 100644 --- a/packages/core/src/api/contracts/data-manager-admin/index.ts +++ b/packages/core/src/api/contracts/data-manager-admin/index.ts @@ -8,3 +8,5 @@ export * from "./common"; export * from "./operations"; export * from "./providers"; export * from "./transit"; + +export * from "./transit-credentials"; diff --git a/packages/core/src/api/contracts/data-manager-admin/operations.ts b/packages/core/src/api/contracts/data-manager-admin/operations.ts index aa6547016..e68aed9e4 100644 --- a/packages/core/src/api/contracts/data-manager-admin/operations.ts +++ b/packages/core/src/api/contracts/data-manager-admin/operations.ts @@ -52,6 +52,7 @@ import { transitJobsResponseSchema, transitStateSummarySchema, } from "./transit"; +import { transitCredentialOperations } from "./transit-credentials"; const service = "service" as const; const admin = "admin" as const; @@ -91,6 +92,7 @@ const ambientUpdated = { }; /** Application API operations. Service credentials, private acquisition and internal endpoints are not browser contracts. */ export const dataManagerAdminOperations = { + ...transitCredentialOperations, getApiDataManagerRoadConditionsPolicy: { method: "GET", path: "/api/data-manager/road-conditions/policy", diff --git a/packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts b/packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts new file mode 100644 index 000000000..23c600993 --- /dev/null +++ b/packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts @@ -0,0 +1,119 @@ +import { canonicalFeedCredentialMap } from "@openmapx/transitous-core/feed-credentials"; +import z from "zod/v4"; +import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; +import { integrationHostApplyEvidenceSchema } from "../integration-admin/application"; + +const revision = z.string().regex(/^[a-f0-9]{64}$/); +const currentRevision = z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .nullable(); +const operationKey = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/); +const key = z + .string() + .min(1) + .max(1024) + .refine((value) => { + try { + canonicalFeedCredentialMap({ [value]: "" }); + return true; + } catch { + return false; + } + }, "Invalid feed credential key"); +const value = z + .string() + .max(8192) + .refine((value) => { + try { + canonicalFeedCredentialMap({ validation: value }); + return true; + } catch { + return false; + } + }, "Credential exceeds its UTF-8 limit"); +export const feedCredentialChangeSchema = z.discriminatedUnion("action", [ + z.strictObject({ key, action: z.literal("set"), value }), + z.strictObject({ key, action: z.literal("delete") }), +]); +export const feedCredentialPreviewInputSchema = z.strictObject({ + catalogRevisionId: revision, + expectedCredentialsRevision: revision, + expectedCurrentRevisionId: currentRevision, + operationKey, + changes: z + .array(feedCredentialChangeSchema) + .min(1) + .max(100) + .refine((values) => new Set(values.map((value) => value.key)).size === values.length), +}); +export const feedCredentialApplyInputSchema = z.strictObject({ + previewId: z.uuid(), + intentFingerprint: revision, + operationKey, + transitProofToken: z.uuid(), +}); +export const feedCredentialPreviewSchema = z.strictObject({ + previewId: z.uuid(), + intentFingerprint: revision, + catalogRevisionId: revision, + credentialsRevision: revision, + expectedCurrentRevisionId: currentRevision, + expiresAt: z.iso.datetime(), + changes: z.array(z.strictObject({ key, action: z.enum(["set", "delete"]) })).max(100), + preservedCount: z.number().int().nonnegative().max(10000), + mapBytes: z.number().int().min(2).max(65536), +}); +export const transitCredentialsSchema = z.strictObject({ + catalogRevisionId: revision, + credentialsRevision: revision, + configurationRevisionId: currentRevision, + owner: z.enum(["vault", "file", "environment", "unset"]), + editable: z.boolean(), + reason: z.string().nullable(), + appliedCredentialsRevision: revision.nullable(), + pendingApplication: z.boolean(), + entries: z + .array( + z.strictObject({ + key, + required: z.boolean(), + configured: z.boolean(), + orphaned: z.boolean(), + }), + ) + .max(11000), +}); +export const feedCredentialApplyResultSchema = z.strictObject({ + credentialsRevision: revision, + configuration: integrationHostApplyEvidenceSchema, +}); +const responses = { + ...adminCommonResponses, + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), +}; +export const transitCredentialOperations = { + getAdminTransitCredentials: { + method: "GET", + path: "/api/admin/transit/credentials", + auth: "admin", + responses: { ...responses, 200: adminJson(transitCredentialsSchema) }, + }, + previewAdminTransitCredentials: { + method: "POST", + path: "/api/admin/transit/credentials/preview", + auth: "admin", + request: { body: feedCredentialPreviewInputSchema }, + responses: { ...responses, 200: adminJson(feedCredentialPreviewSchema) }, + }, + applyAdminTransitCredentials: { + method: "PATCH", + path: "/api/admin/transit/credentials", + auth: "admin", + request: { body: feedCredentialApplyInputSchema }, + responses: { ...responses, 200: adminJson(feedCredentialApplyResultSchema) }, + }, +} as const satisfies Record; +export type TransitCredentials = z.infer; +export type FeedCredentialPreviewInput = z.infer; diff --git a/packages/core/src/api/contracts/service-admin/operations.ts b/packages/core/src/api/contracts/service-admin/operations.ts index 5a4ff4c42..5a9ed4205 100644 --- a/packages/core/src/api/contracts/service-admin/operations.ts +++ b/packages/core/src/api/contracts/service-admin/operations.ts @@ -8,7 +8,6 @@ export const adminOperationIdSchema = z.enum([ "convert-overpass", "link", "clean", - "generate-api-keys", "overture-sync", "overture-conflate", "search-index-build", diff --git a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts index 230129448..7baa407f1 100644 --- a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts +++ b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts @@ -1,6 +1,17 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. import type { + ApplyAdminTransitCredentials200, + ApplyAdminTransitCredentials400, + ApplyAdminTransitCredentials401, + ApplyAdminTransitCredentials403, + ApplyAdminTransitCredentials409, + ApplyAdminTransitCredentials413, + ApplyAdminTransitCredentials415, + ApplyAdminTransitCredentials429, + ApplyAdminTransitCredentials500, + ApplyAdminTransitCredentials503, + ApplyAdminTransitCredentialsBody, DeleteApiDataManagerTransitSourcesSourceId202, DeleteApiDataManagerTransitSourcesSourceId400, DeleteApiDataManagerTransitSourcesSourceId401, @@ -12,6 +23,16 @@ import type { DeleteApiDataManagerTransitSourcesSourceId429, DeleteApiDataManagerTransitSourcesSourceId500, DeleteApiDataManagerTransitSourcesSourceIdBody, + GetAdminTransitCredentials200, + GetAdminTransitCredentials400, + GetAdminTransitCredentials401, + GetAdminTransitCredentials403, + GetAdminTransitCredentials409, + GetAdminTransitCredentials413, + GetAdminTransitCredentials415, + GetAdminTransitCredentials429, + GetAdminTransitCredentials500, + GetAdminTransitCredentials503, GetApiAdminAmbientPlacesStatus200, GetApiAdminAmbientPlacesStatus400, GetApiAdminAmbientPlacesStatus401, @@ -277,9 +298,22 @@ import type { PostApiDataManagerTransitSyncPreflight500, PostApiDataManagerTransitSyncPreflight503, PostApiDataManagerTransitSyncPreflightBody, + PreviewAdminTransitCredentials200, + PreviewAdminTransitCredentials400, + PreviewAdminTransitCredentials401, + PreviewAdminTransitCredentials403, + PreviewAdminTransitCredentials409, + PreviewAdminTransitCredentials413, + PreviewAdminTransitCredentials415, + PreviewAdminTransitCredentials429, + PreviewAdminTransitCredentials500, + PreviewAdminTransitCredentials503, + PreviewAdminTransitCredentialsBody, } from "./models"; import { + applyAdminTransitCredentialsRequest, deleteApiDataManagerTransitSourcesSourceIdRequest, + getAdminTransitCredentialsRequest, getApiAdminAmbientPlacesStatusRequest, getApiDataManagerNotablePlacesStatusRequest, getApiDataManagerOvertureStatusRequest, @@ -307,6 +341,7 @@ import { postApiDataManagerTransitSourcesSourceIdEnableRequest, postApiDataManagerTransitSyncPreflightRequest, postApiDataManagerTransitSyncRequest, + previewAdminTransitCredentialsRequest, } from "./operation-bindings.generated"; export type postApiAdminAmbientPlacesBuildResponse202 = { data: PostApiAdminAmbientPlacesBuild202; @@ -918,6 +953,313 @@ export const getApiAdminAmbientPlacesStatus = async ( ); }; +export type getAdminTransitCredentialsResponse200 = { + data: GetAdminTransitCredentials200; + status: 200; +}; + +export type getAdminTransitCredentialsResponse400 = { + data: GetAdminTransitCredentials400; + status: 400; +}; + +export type getAdminTransitCredentialsResponse401 = { + data: GetAdminTransitCredentials401; + status: 401; +}; + +export type getAdminTransitCredentialsResponse403 = { + data: GetAdminTransitCredentials403; + status: 403; +}; + +export type getAdminTransitCredentialsResponse409 = { + data: GetAdminTransitCredentials409; + status: 409; +}; + +export type getAdminTransitCredentialsResponse413 = { + data: GetAdminTransitCredentials413; + status: 413; +}; + +export type getAdminTransitCredentialsResponse415 = { + data: GetAdminTransitCredentials415; + status: 415; +}; + +export type getAdminTransitCredentialsResponse429 = { + data: GetAdminTransitCredentials429; + status: 429; +}; + +export type getAdminTransitCredentialsResponse500 = { + data: GetAdminTransitCredentials500; + status: 500; +}; + +export type getAdminTransitCredentialsResponse503 = { + data: GetAdminTransitCredentials503; + status: 503; +}; + +export type getAdminTransitCredentialsResponseSuccess = getAdminTransitCredentialsResponse200 & { + headers: Headers; +}; +export type getAdminTransitCredentialsResponseError = ( + | getAdminTransitCredentialsResponse400 + | getAdminTransitCredentialsResponse401 + | getAdminTransitCredentialsResponse403 + | getAdminTransitCredentialsResponse409 + | getAdminTransitCredentialsResponse413 + | getAdminTransitCredentialsResponse415 + | getAdminTransitCredentialsResponse429 + | getAdminTransitCredentialsResponse500 + | getAdminTransitCredentialsResponse503 +) & { + headers: Headers; +}; + +export type getAdminTransitCredentialsResponse = + | getAdminTransitCredentialsResponseSuccess + | getAdminTransitCredentialsResponseError; + +export const getGetAdminTransitCredentialsUrl = () => { + return `/api/admin/transit/credentials`; +}; + +export const getAdminTransitCredentials = async ( + options?: Parameters[1], +): Promise => { + return getAdminTransitCredentialsRequest( + getGetAdminTransitCredentialsUrl(), + { + ...options, + method: "GET", + }, + ); +}; + +export type applyAdminTransitCredentialsResponse200 = { + data: ApplyAdminTransitCredentials200; + status: 200; +}; + +export type applyAdminTransitCredentialsResponse400 = { + data: ApplyAdminTransitCredentials400; + status: 400; +}; + +export type applyAdminTransitCredentialsResponse401 = { + data: ApplyAdminTransitCredentials401; + status: 401; +}; + +export type applyAdminTransitCredentialsResponse403 = { + data: ApplyAdminTransitCredentials403; + status: 403; +}; + +export type applyAdminTransitCredentialsResponse409 = { + data: ApplyAdminTransitCredentials409; + status: 409; +}; + +export type applyAdminTransitCredentialsResponse413 = { + data: ApplyAdminTransitCredentials413; + status: 413; +}; + +export type applyAdminTransitCredentialsResponse415 = { + data: ApplyAdminTransitCredentials415; + status: 415; +}; + +export type applyAdminTransitCredentialsResponse429 = { + data: ApplyAdminTransitCredentials429; + status: 429; +}; + +export type applyAdminTransitCredentialsResponse500 = { + data: ApplyAdminTransitCredentials500; + status: 500; +}; + +export type applyAdminTransitCredentialsResponse503 = { + data: ApplyAdminTransitCredentials503; + status: 503; +}; + +export type applyAdminTransitCredentialsResponseSuccess = + applyAdminTransitCredentialsResponse200 & { + headers: Headers; + }; +export type applyAdminTransitCredentialsResponseError = ( + | applyAdminTransitCredentialsResponse400 + | applyAdminTransitCredentialsResponse401 + | applyAdminTransitCredentialsResponse403 + | applyAdminTransitCredentialsResponse409 + | applyAdminTransitCredentialsResponse413 + | applyAdminTransitCredentialsResponse415 + | applyAdminTransitCredentialsResponse429 + | applyAdminTransitCredentialsResponse500 + | applyAdminTransitCredentialsResponse503 +) & { + headers: Headers; +}; + +export type applyAdminTransitCredentialsResponse = + | applyAdminTransitCredentialsResponseSuccess + | applyAdminTransitCredentialsResponseError; + +export const getApplyAdminTransitCredentialsUrl = () => { + return `/api/admin/transit/credentials`; +}; + +export const applyAdminTransitCredentials = async ( + applyAdminTransitCredentialsBody: ApplyAdminTransitCredentialsBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return applyAdminTransitCredentialsRequest( + getApplyAdminTransitCredentialsUrl(), + { + ...options, + method: "PATCH", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(applyAdminTransitCredentialsBody), + }, + ); +}; + +export type previewAdminTransitCredentialsResponse200 = { + data: PreviewAdminTransitCredentials200; + status: 200; +}; + +export type previewAdminTransitCredentialsResponse400 = { + data: PreviewAdminTransitCredentials400; + status: 400; +}; + +export type previewAdminTransitCredentialsResponse401 = { + data: PreviewAdminTransitCredentials401; + status: 401; +}; + +export type previewAdminTransitCredentialsResponse403 = { + data: PreviewAdminTransitCredentials403; + status: 403; +}; + +export type previewAdminTransitCredentialsResponse409 = { + data: PreviewAdminTransitCredentials409; + status: 409; +}; + +export type previewAdminTransitCredentialsResponse413 = { + data: PreviewAdminTransitCredentials413; + status: 413; +}; + +export type previewAdminTransitCredentialsResponse415 = { + data: PreviewAdminTransitCredentials415; + status: 415; +}; + +export type previewAdminTransitCredentialsResponse429 = { + data: PreviewAdminTransitCredentials429; + status: 429; +}; + +export type previewAdminTransitCredentialsResponse500 = { + data: PreviewAdminTransitCredentials500; + status: 500; +}; + +export type previewAdminTransitCredentialsResponse503 = { + data: PreviewAdminTransitCredentials503; + status: 503; +}; + +export type previewAdminTransitCredentialsResponseSuccess = + previewAdminTransitCredentialsResponse200 & { + headers: Headers; + }; +export type previewAdminTransitCredentialsResponseError = ( + | previewAdminTransitCredentialsResponse400 + | previewAdminTransitCredentialsResponse401 + | previewAdminTransitCredentialsResponse403 + | previewAdminTransitCredentialsResponse409 + | previewAdminTransitCredentialsResponse413 + | previewAdminTransitCredentialsResponse415 + | previewAdminTransitCredentialsResponse429 + | previewAdminTransitCredentialsResponse500 + | previewAdminTransitCredentialsResponse503 +) & { + headers: Headers; +}; + +export type previewAdminTransitCredentialsResponse = + | previewAdminTransitCredentialsResponseSuccess + | previewAdminTransitCredentialsResponseError; + +export const getPreviewAdminTransitCredentialsUrl = () => { + return `/api/admin/transit/credentials/preview`; +}; + +export const previewAdminTransitCredentials = async ( + previewAdminTransitCredentialsBody: PreviewAdminTransitCredentialsBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return previewAdminTransitCredentialsRequest( + getPreviewAdminTransitCredentialsUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(previewAdminTransitCredentialsBody), + }, + ); +}; + export type getApiDataManagerNotablePlacesStatusResponse200 = { data: GetApiDataManagerNotablePlacesStatus200; status: 200; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200.ts new file mode 100644 index 000000000..3b668ce4f --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminTransitCredentials200Configuration } from "./applyAdminTransitCredentials200Configuration"; + +export type ApplyAdminTransitCredentials200 = { + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + configuration: ApplyAdminTransitCredentials200Configuration; +}; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200Configuration.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200Configuration.ts new file mode 100644 index 000000000..d2c6177d2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials200Configuration.ts @@ -0,0 +1,34 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials200Configuration = + | { + outcome: "committed"; + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + revisionId: string; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + * @items.pattern ^[a-z0-9][a-z0-9-]*$ + */ + enabledServiceIds: string[]; + followUp: "complete" | "failed"; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "not_committed"; + reason: "conflict" | "busy" | "failed"; + currentRevisionId?: string | null; + cleanupFailed: boolean; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + } + | { + outcome: "unknown"; + reason: "transport_or_result"; + publisherCleanupFailed: boolean; + registryUpdateFailed: boolean; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials400.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials400.ts new file mode 100644 index 000000000..8356d8c6e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials401.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials401.ts new file mode 100644 index 000000000..e4bdacb84 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials403.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials403.ts new file mode 100644 index 000000000..0195449b2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials409.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials409.ts new file mode 100644 index 000000000..a2309eb73 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials413.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials413.ts new file mode 100644 index 000000000..faa2859cc --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials415.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials415.ts new file mode 100644 index 000000000..35ea0dab4 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials429.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials429.ts new file mode 100644 index 000000000..cbe3086d3 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials500.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials500.ts new file mode 100644 index 000000000..f9dafa514 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials503.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials503.ts new file mode 100644 index 000000000..e5b133574 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentialsBody.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentialsBody.ts new file mode 100644 index 000000000..b32648bf3 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentialsBody.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentialsBody = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** @pattern ^[a-f0-9]{64}$ */ + intentFingerprint: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + transitProofToken: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts new file mode 100644 index 000000000..42522dd61 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitCredentials200EntriesItem } from "./getAdminTransitCredentials200EntriesItem"; +import type { GetAdminTransitCredentials200Owner } from "./getAdminTransitCredentials200Owner"; + +export type GetAdminTransitCredentials200 = { + /** @pattern ^[a-f0-9]{64}$ */ + catalogRevisionId: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + configurationRevisionId: string | null; + owner: GetAdminTransitCredentials200Owner; + editable: boolean; + /** @nullable */ + reason: string | null; + appliedCredentialsRevision: string | null; + pendingApplication: boolean; + /** @maxItems 11000 */ + entries: GetAdminTransitCredentials200EntriesItem[]; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200EntriesItem.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200EntriesItem.ts new file mode 100644 index 000000000..4a51046c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200EntriesItem.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials200EntriesItem = { + /** + * @minLength 1 + * @maxLength 1024 + */ + key: string; + required: boolean; + configured: boolean; + orphaned: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200Owner.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200Owner.ts new file mode 100644 index 000000000..d50c56bae --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200Owner.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials200Owner = + (typeof GetAdminTransitCredentials200Owner)[keyof typeof GetAdminTransitCredentials200Owner]; + +export const GetAdminTransitCredentials200Owner = { + vault: "vault", + file: "file", + environment: "environment", + unset: "unset", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials400.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials400.ts new file mode 100644 index 000000000..9bba914b5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials401.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials401.ts new file mode 100644 index 000000000..ae17eec43 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials403.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials403.ts new file mode 100644 index 000000000..f334c0ed2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials409.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials409.ts new file mode 100644 index 000000000..f91c4b421 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials413.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials413.ts new file mode 100644 index 000000000..7d54739f7 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials415.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials415.ts new file mode 100644 index 000000000..cc34c1a38 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials429.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials429.ts new file mode 100644 index 000000000..6feccd482 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials500.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials500.ts new file mode 100644 index 000000000..7b17cb7c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials503.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials503.ts new file mode 100644 index 000000000..77ae8e7a7 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/index.ts b/packages/core/src/api/generated/admin/models/index.ts index b32e15daf..af8f270f4 100644 --- a/packages/core/src/api/generated/admin/models/index.ts +++ b/packages/core/src/api/generated/admin/models/index.ts @@ -6,6 +6,18 @@ export * from "./adminGetApiDataManagerTransitJobsId200Schema0"; export * from "./adminGetApiDataManagerTransitJobsId200Schema1"; export * from "./adminGetApiDataManagerTransitJobsId200Schema2"; export * from "./adminGetApiDataManagerTransitSources200Schema0"; +export * from "./applyAdminTransitCredentials200"; +export * from "./applyAdminTransitCredentials200Configuration"; +export * from "./applyAdminTransitCredentials400"; +export * from "./applyAdminTransitCredentials401"; +export * from "./applyAdminTransitCredentials403"; +export * from "./applyAdminTransitCredentials409"; +export * from "./applyAdminTransitCredentials413"; +export * from "./applyAdminTransitCredentials415"; +export * from "./applyAdminTransitCredentials429"; +export * from "./applyAdminTransitCredentials500"; +export * from "./applyAdminTransitCredentials503"; +export * from "./applyAdminTransitCredentialsBody"; export * from "./deleteApiDataManagerTransitSourcesSourceId202"; export * from "./deleteApiDataManagerTransitSourcesSourceId202Status"; export * from "./deleteApiDataManagerTransitSourcesSourceId400"; @@ -18,6 +30,18 @@ export * from "./deleteApiDataManagerTransitSourcesSourceId415"; export * from "./deleteApiDataManagerTransitSourcesSourceId429"; export * from "./deleteApiDataManagerTransitSourcesSourceId500"; export * from "./deleteApiDataManagerTransitSourcesSourceIdBody"; +export * from "./getAdminTransitCredentials200"; +export * from "./getAdminTransitCredentials200EntriesItem"; +export * from "./getAdminTransitCredentials200Owner"; +export * from "./getAdminTransitCredentials400"; +export * from "./getAdminTransitCredentials401"; +export * from "./getAdminTransitCredentials403"; +export * from "./getAdminTransitCredentials409"; +export * from "./getAdminTransitCredentials413"; +export * from "./getAdminTransitCredentials415"; +export * from "./getAdminTransitCredentials429"; +export * from "./getAdminTransitCredentials500"; +export * from "./getAdminTransitCredentials503"; export * from "./getApiAdminAmbientPlacesStatus200"; export * from "./getApiAdminAmbientPlacesStatus200Active"; export * from "./getApiAdminAmbientPlacesStatus200ActiveAttributionPrecision"; @@ -368,3 +392,17 @@ export * from "./postApiDataManagerTransitSyncPreflight429"; export * from "./postApiDataManagerTransitSyncPreflight500"; export * from "./postApiDataManagerTransitSyncPreflight503"; export * from "./postApiDataManagerTransitSyncPreflightBody"; +export * from "./previewAdminTransitCredentials200"; +export * from "./previewAdminTransitCredentials200ChangesItem"; +export * from "./previewAdminTransitCredentials200ChangesItemAction"; +export * from "./previewAdminTransitCredentials400"; +export * from "./previewAdminTransitCredentials401"; +export * from "./previewAdminTransitCredentials403"; +export * from "./previewAdminTransitCredentials409"; +export * from "./previewAdminTransitCredentials413"; +export * from "./previewAdminTransitCredentials415"; +export * from "./previewAdminTransitCredentials429"; +export * from "./previewAdminTransitCredentials500"; +export * from "./previewAdminTransitCredentials503"; +export * from "./previewAdminTransitCredentialsBody"; +export * from "./previewAdminTransitCredentialsBodyChangesItem"; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts new file mode 100644 index 000000000..9eaae3336 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts @@ -0,0 +1,29 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminTransitCredentials200ChangesItem } from "./previewAdminTransitCredentials200ChangesItem"; + +export type PreviewAdminTransitCredentials200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** @pattern ^[a-f0-9]{64}$ */ + intentFingerprint: string; + /** @pattern ^[a-f0-9]{64}$ */ + catalogRevisionId: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + expectedCurrentRevisionId: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + /** @maxItems 100 */ + changes: PreviewAdminTransitCredentials200ChangesItem[]; + /** + * @minimum 0 + * @maximum 10000 + */ + preservedCount: number; + /** + * @minimum 2 + * @maximum 65536 + */ + mapBytes: number; +}; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItem.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItem.ts new file mode 100644 index 000000000..fcb345aea --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItem.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminTransitCredentials200ChangesItemAction } from "./previewAdminTransitCredentials200ChangesItemAction"; + +export type PreviewAdminTransitCredentials200ChangesItem = { + /** + * @minLength 1 + * @maxLength 1024 + */ + key: string; + action: PreviewAdminTransitCredentials200ChangesItemAction; +}; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItemAction.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItemAction.ts new file mode 100644 index 000000000..4bc185d9e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200ChangesItemAction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials200ChangesItemAction = + (typeof PreviewAdminTransitCredentials200ChangesItemAction)[keyof typeof PreviewAdminTransitCredentials200ChangesItemAction]; + +export const PreviewAdminTransitCredentials200ChangesItemAction = { + set: "set", + delete: "delete", +} as const; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials400.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials400.ts new file mode 100644 index 000000000..9b2fb4153 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials401.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials401.ts new file mode 100644 index 000000000..87a097b6c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials403.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials403.ts new file mode 100644 index 000000000..cbbc14cca --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials409.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials409.ts new file mode 100644 index 000000000..d1a75b1fe --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials413.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials413.ts new file mode 100644 index 000000000..06c26dfca --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials415.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials415.ts new file mode 100644 index 000000000..c94a1bbe1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials429.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials429.ts new file mode 100644 index 000000000..713396172 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials500.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials500.ts new file mode 100644 index 000000000..85e1abbb5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials503.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials503.ts new file mode 100644 index 000000000..63427b4dd --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts new file mode 100644 index 000000000..8d90ff05e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminTransitCredentialsBodyChangesItem } from "./previewAdminTransitCredentialsBodyChangesItem"; + +export type PreviewAdminTransitCredentialsBody = { + /** @pattern ^[a-f0-9]{64}$ */ + catalogRevisionId: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedCredentialsRevision: string; + expectedCurrentRevisionId: string | null; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** + * @minItems 1 + * @maxItems 100 + */ + changes: PreviewAdminTransitCredentialsBodyChangesItem[]; +}; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBodyChangesItem.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBodyChangesItem.ts new file mode 100644 index 000000000..670f98af6 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBodyChangesItem.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentialsBodyChangesItem = + | { + /** + * @minLength 1 + * @maxLength 1024 + */ + key: string; + action: "set"; + /** @maxLength 8192 */ + value: string; + } + | { + /** + * @minLength 1 + * @maxLength 1024 + */ + key: string; + action: "delete"; + }; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index 3b5351938..f2493872a 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -21,6 +21,13 @@ export async function applyAdminAuthPolicyRequest( ): Promise { return applyAdminAuthPolicyBound(url, options); } +const applyAdminTransitCredentialsBound = bindAdminOperation("applyAdminTransitCredentials"); +export async function applyAdminTransitCredentialsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return applyAdminTransitCredentialsBound(url, options); +} const cancelAdminActivityJobBound = bindAdminOperation("cancelAdminActivityJob"); export async function cancelAdminActivityJobRequest( url: string, @@ -260,6 +267,13 @@ export async function getAdminSharedMobilityRequest( ): Promise { return getAdminSharedMobilityBound(url, options); } +const getAdminTransitCredentialsBound = bindAdminOperation("getAdminTransitCredentials"); +export async function getAdminTransitCredentialsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminTransitCredentialsBound(url, options); +} const getApiAdminAmbientPlacesStatusBound = bindAdminOperation("getApiAdminAmbientPlacesStatus"); export async function getApiAdminAmbientPlacesStatusRequest( url: string, @@ -594,6 +608,13 @@ export async function previewAdminOperationRequest( ): Promise { return previewAdminOperationBound(url, options); } +const previewAdminTransitCredentialsBound = bindAdminOperation("previewAdminTransitCredentials"); +export async function previewAdminTransitCredentialsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return previewAdminTransitCredentialsBound(url, options); +} const provisionAdminDawarichBound = bindAdminOperation("provisionAdminDawarich"); export async function provisionAdminDawarichRequest( url: string, diff --git a/packages/core/src/api/generated/admin/service-models/listAdminOperations200OperationsItemId.ts b/packages/core/src/api/generated/admin/service-models/listAdminOperations200OperationsItemId.ts index 67bd042ed..f755fda62 100644 --- a/packages/core/src/api/generated/admin/service-models/listAdminOperations200OperationsItemId.ts +++ b/packages/core/src/api/generated/admin/service-models/listAdminOperations200OperationsItemId.ts @@ -11,7 +11,6 @@ export const ListAdminOperations200OperationsItemId = { "convert-overpass": "convert-overpass", link: "link", clean: "clean", - "generate-api-keys": "generate-api-keys", "overture-sync": "overture-sync", "overture-conflate": "overture-conflate", "search-index-build": "search-index-build", diff --git a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts index d68b25b7c..321fb9af1 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts @@ -4,6 +4,7 @@ import { configurationMaintenanceReceiptSchema } from "../ops/configuration-main import { type OpsJobStatus, parseOpsJobStatus } from "../ops/contract"; import { executionRefSchema } from "../ops/execution-ref"; import { configurationMaintenanceDecisionCommitmentSchema } from "./configuration-maintenance-owner-records"; +import { fixedConfigurationMaintenancePlanSchema } from "./fixed-configuration-maintenance"; export const configurationMaintenanceForwardDomain = "openmapx-configuration-maintenance-forward-v1"; @@ -21,6 +22,7 @@ export const configurationMaintenanceForwardGraphSchema = z.strictObject({ parentFingerprint: digest, parentDeadlineAt: z.number().int().positive().safe(), planDigest: digest, + coordinatorImage: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.coordinatorImage), serviceIds: z.lazy(() => configurationMaintenanceReceiptSchema.shape.serviceIds), steps: z.lazy(() => configurationMaintenanceReceiptSchema.shape.steps), }); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index fd24717ed..99669eae3 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -7,6 +7,12 @@ import { } from "./control-child-records"; import { dataUpdatePlanSchema } from "./data-update-records"; import { dataUpdateSourcesDigest } from "./data-update-sources"; +import { + LOCAL_CREDENTIAL_TEMPLATE_KIND, + localCredentialTemplateFileMac, + localCredentialTemplateOperationSchema, + localCredentialTemplateReceiptSchema, +} from "./local-credential-template"; export { type ControlChildPhase, @@ -1046,6 +1052,7 @@ export function retainAdmittedControlExecution( "catalog.activate", "catalog.validate", "catalog-pins.initialize", + LOCAL_CREDENTIAL_TEMPLATE_KIND, "artifactTransfer.import", "artifactTransfer.export", "artifactTransfer.validate", @@ -1118,6 +1125,20 @@ export function retainAdmittedControlExecution( controlDigest(input.operation) !== controlDigest({ kind: "catalog-pins.initialize" })) ) throw Error("Fixed local catalog bootstrap only"); + if (input.kind === LOCAL_CREDENTIAL_TEMPLATE_KIND) { + localCredentialTemplateOperationSchema.parse(input.operation); + if ( + state.fixedChildId || + input.parent || + plan.length || + reservations.length || + installation.role !== "host-cli" || + execution.principal !== "host-cli" || + serviceIds.length !== 1 || + serviceIds[0] !== "motis" + ) + throw Error("Fixed local host credential template only"); + } const budget = (() => { try { return opsExecutionMaximumMs(input.kind); @@ -1131,6 +1152,7 @@ export function retainAdmittedControlExecution( ) return 3600000; if (input.kind === "catalog-pins.initialize") return 60000; + if (input.kind === LOCAL_CREDENTIAL_TEMPLATE_KIND) return 300000; if ( ["configuration-maintenance.prepare", "configuration-maintenance.settle"].includes( input.kind, @@ -1527,6 +1549,7 @@ const settlementSchema = z.strictObject({ terminalReceiptDigest: digest, }); const effectObservationSchema = z.discriminatedUnion("kind", [ + localCredentialTemplateReceiptSchema, z.strictObject({ kind: z.literal("catalog-pins-initialized"), generationId: digest.nullable() }), z.strictObject({ kind: z.literal("configuration-generation-not-committed"), @@ -1975,6 +1998,46 @@ export function retainControlEffectReceipt( throw Error("Catalog initialization source mismatch"); outcome = "committed"; publication = "committed"; + } else if (receipt.kind === "feed-credential-template-written") { + if ( + s.kind !== LOCAL_CREDENTIAL_TEMPLATE_KIND || + !s.origin || + s.installation.fixedChildId || + s.installation.installation.role !== "host-cli" + ) + throw Error("Original local credential template required"); + const name = `${s.execution.operationKey}.json`, + started = readPrivateRecord(s.installation, "effect-started", name); + if ( + !started || + controlDigest((started as { execution: unknown }).execution) !== + controlDigest(s.execution) || + (started as { receiptDigest: unknown }).receiptDigest !== s.receiptDigest + ) + throw Error("Original started credential template required"); + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace: "effect-started", container: "", name }, + canonicalJson(started), + ); + const admitted = readPrivateRecord(s.installation, "executions", name, 65536) as { + payload: { operation: unknown }; + }; + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace: "executions", container: "", name }, + canonicalJson(admitted), + ); + const operation = localCredentialTemplateOperationSchema.parse(admitted.payload.operation); + if ( + receipt.fileMac !== operation.fileMac || + localCredentialTemplateFileMac(s.installation, operation) !== operation.fileMac + ) + throw Error("Credential template file evidence changed"); + outcome = "committed"; + publication = "committed"; } else if (receipt.kind === "ops-result") { receipt.result = parseBoundedOpsResult(s.kind as OpsOperationKind, receipt.result); if (s.kind === "data.update") { @@ -2896,6 +2959,7 @@ export function publishConfigurationMaintenanceForwardWitness( parentFingerprint: receipt.parentFingerprint, parentDeadlineAt: receipt.parentDeadlineAt, planDigest: receipt.planDigest, + ...(owner.plan.coordinatorImage ? { coordinatorImage: owner.plan.coordinatorImage } : {}), serviceIds: receipt.serviceIds, steps: receipt.steps, }); @@ -3093,6 +3157,7 @@ export function verifyConfigurationMaintenanceGenerationWitness( candidateRecordDigest: "0".repeat(64), sourceAuthorityRevision: "not-shared", previewDigest: "0".repeat(64), + ...(graph.coordinatorImage ? { coordinatorImage: graph.coordinatorImage } : {}), hardlinkPlan: request.plan, serviceIds: root.serviceIds, recreateServiceIds: graph.steps.flatMap((step) => @@ -5303,3 +5368,12 @@ function verifyDataUpdateCompletionSource( } } } + +/** Opaque local file version, never a public hash of a credential value. */ +export function localFeedCredentialTemplateMac( + installation: ControlInstallation, + operation: unknown, + contents?: string, +) { + return localCredentialTemplateFileMac(installationState(installation), operation, contents); +} diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts index a8dfeb8c0..f2ecc1aa7 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -141,3 +141,40 @@ it.each(["body", "source", "parent", "foreign-target", "protected-target", "capa expect(() => buildFixedConfigurationMaintenancePlan(source)).toThrow(); }, ); + +it("retains the frozen coordinator image in the cross-role graph used to reconstruct nonempty hardlinks", async () => { + const { configurationMaintenanceForwardGraphSchema } = await import( + "./configuration-maintenance-forward-records" + ); + const source = input(); + const coordinatorImage = { + action: "apply" as const, + noDeps: false, + imageReference: "example/data-manager:1", + imageId: `sha256:${"e".repeat(64)}`, + releaseSelectionDigest: "f".repeat(64), + pullSubmissionDigest: null, + }; + const plan = buildFixedConfigurationMaintenancePlan({ ...source, coordinatorImage }); + const { kind: _kind, ...binding } = plan.operation; + const graph = configurationMaintenanceForwardGraphSchema.parse({ + version: 1, + binding, + prepare: plan.prepare, + apiJobId: plan.apiParent.apiJobId, + parentOperationKey: plan.apiParent.operationKey, + parentFingerprint: plan.apiParent.fingerprint, + parentDeadlineAt: plan.apiParent.deadlineAt, + planDigest: plan.planDigest, + serviceIds: plan.serviceIds, + steps: plan.steps, + coordinatorImage, + }); + expect(graph.coordinatorImage).toEqual(coordinatorImage); + const rebuilt = buildFixedConfigurationMaintenancePlan({ + ...source, + coordinatorImage: graph.coordinatorImage, + }); + expect(rebuilt.planDigest).toBe(plan.planDigest); + expect(rebuilt.steps).toEqual(plan.steps); +}); diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 61869ee53..8f86c8a8f 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -69,6 +69,7 @@ export { hasRetainedControlOrigin, inspectConfigurationPreEffectRefusal, loadControlStartup, + localFeedCredentialTemplateMac, markFeedProxyReloadAttempted, observeCoordinatorPullForMaintenance, observedCoordinatorPullBaseline, diff --git a/packages/core/src/build-artifacts/local-credential-template.test.ts b/packages/core/src/build-artifacts/local-credential-template.test.ts new file mode 100644 index 000000000..03a3aded7 --- /dev/null +++ b/packages/core/src/build-artifacts/local-credential-template.test.ts @@ -0,0 +1,93 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { createControlRootOperationKey, executionFingerprint, opsOperationSchema } from "../ops"; +import { + closeControlInstallation, + controlDigest, + localFeedCredentialTemplateMac, + openControlInstallation, + provisionControlInstallation, + retainAdmittedControlExecution, +} from "./control-authority"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +const operation = { + kind: "transit.credentials.template", + catalogRevisionId: "a".repeat(64), + outputPath: "services/motis/tools/transitous/api-keys.json", + fileMac: "b".repeat(64), +}; +async function fixture(role: "host-cli" | "ops-agent" | "data-manager") { + const root = mkdtempSync(join(tmpdir(), "omx-local-credential-owner-")); + roots.push(root); + mkdirSync(join(root, "data")); + await provisionControlInstallation(root); + return openControlInstallation({ infraDir: root, role, serviceIds: ["motis", "valhalla"] }); +} +function input( + executor: "host-cli" | "ops-agent" | "data-manager", + principal: "host-cli" | "api" | "data-manager" = "host-cli", + serviceIds = ["motis"], +) { + const admittedAt = Date.now(), + deadlineAt = admittedAt + 300000, + operationFingerprint = controlDigest(operation); + return { + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds, + execution: { + executor, + principal, + executionId: + executor === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + deadlineAt, + operationFingerprint, + fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), + }, + }; +} +it("does not expose a credential template as an HTTP operation", () => { + expect(opsOperationSchema.safeParse(operation).success).toBe(false); +}); +it.each(["ops-agent", "data-manager"] as const)( + "refuses the %s executor and its opaque file MAC oracle", + async (role) => { + const installation = await fixture(role); + try { + expect(() => + retainAdmittedControlExecution( + installation, + input(role, role === "ops-agent" ? "api" : "data-manager"), + ), + ).toThrow(/local/); + expect(() => localFeedCredentialTemplateMac(installation, operation, "{}\n")).toThrow( + /Local host/, + ); + } finally { + closeControlInstallation(installation); + } + }, +); +it("requires local host intent and exactly the MOTIS closure", async () => { + const installation = await fixture("host-cli"); + try { + expect(() => retainAdmittedControlExecution(installation, input("host-cli", "api"))).toThrow( + /local/, + ); + expect(() => + retainAdmittedControlExecution(installation, input("host-cli", "host-cli", ["valhalla"])), + ).toThrow(/local/); + } finally { + closeControlInstallation(installation); + } +}); diff --git a/packages/core/src/build-artifacts/local-credential-template.ts b/packages/core/src/build-artifacts/local-credential-template.ts new file mode 100644 index 000000000..388eef046 --- /dev/null +++ b/packages/core/src/build-artifacts/local-credential-template.ts @@ -0,0 +1,75 @@ +/** Fixed host-local template evidence. Not an HTTP operation or delegation edge. */ +import { createHmac } from "node:crypto"; +import { join } from "node:path"; +import z from "zod/v4"; +import type { InstallationState } from "./control-records"; +import { AnchoredDirectory } from "./filesystem"; +export const LOCAL_CREDENTIAL_TEMPLATE_KIND = "transit.credentials.template"; +const digest = z.string().regex(/^[a-f0-9]{64}$/); +export const localCredentialTemplateOperationSchema = z.strictObject({ + kind: z.literal(LOCAL_CREDENTIAL_TEMPLATE_KIND), + catalogRevisionId: digest, + outputPath: z + .string() + .min(1) + .max(1024) + .refine( + (value) => + !value.startsWith("/") && + !value.includes("\\") && + value + .split("/") + .every( + (part) => + part !== "" && + part !== "." && + part !== ".." && + ![...part].some((char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127), + ), + ), + fileMac: digest, +}); +export const localCredentialTemplateReceiptSchema = z.strictObject({ + kind: z.literal("feed-credential-template-written"), + fileMac: digest, +}); +export function localCredentialTemplateFileMac( + state: InstallationState, + raw: unknown, + contents?: string, +): string { + const operation = localCredentialTemplateOperationSchema.parse(raw); + if (state.fixedChildId || state.installation.role !== "host-cli") + throw Error("Local host credential template required"); + let bytes: Buffer; + if (contents !== undefined) { + bytes = Buffer.from(contents, "utf8"); + if (bytes.length > 65537) throw Error("Credential template file exceeds budget"); + } else { + const opened: AnchoredDirectory[] = []; + try { + let directory = new AnchoredDirectory(join(state.dataRoot, "../../..")); + opened.push(directory); + const parts = operation.outputPath.split("/"), + file = parts.pop(); + if (!file) throw Error("Credential template file unavailable"); + for (const part of parts) { + directory = directory.directory(part); + opened.push(directory); + } + bytes = directory.read(file, 65537, true); + } finally { + for (const directory of opened.reverse()) directory.close(); + } + } + return createHmac("sha256", state.privateKey.export({ type: "pkcs8", format: "der" })) + .update("openmapx-local-feed-credential-template-v1\0") + .update(state.installation.installationId) + .update("\0") + .update(operation.catalogRevisionId) + .update("\0") + .update(operation.outputPath) + .update("\0") + .update(bytes) + .digest("hex"); +} diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 838101094..5cbd171dc 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -290,6 +290,10 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { "valhalla.traffic.refreshWaysToEdges": { kind: "valhalla.traffic.refreshWaysToEdges" }, "postgis.capacity.inspect": { kind: "postgis.capacity.inspect" }, "catalogPins.inspect": { kind: "catalogPins.inspect" }, + "transitCredentials.inspect": { + kind: "transitCredentials.inspect", + catalogRevisionId: "a".repeat(64), + }, "transitousLock.inspect": { kind: "transitousLock.inspect" }, "transitousLock.propose": { kind: "transitousLock.propose", @@ -362,6 +366,10 @@ describe("ops contract", () => { // Take the role from the policy table rather than re-deriving it from a // kind prefix: a prefix list silently omits every kind added later. const role: OpsRole = OPS_KIND_POLICIES[operation.kind].role; + if (operation.kind === "data.generateApiKeys") { + expect(() => parseOpsRequest(request(operation), { role })).toThrow(/not permitted/); + continue; + } expect(() => parseOpsRequest(request(operation), { role }), operation.kind).not.toThrow(); expect(parseOpsRequest(request(operation), { role }).operation).toEqual(operation); } diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 540d9e1ba..9aaa51351 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -134,6 +134,7 @@ export const OPS_OPERATION_KINDS = [ "valhalla.traffic.refreshWaysToEdges", "postgis.capacity.inspect", "catalogPins.inspect", + "transitCredentials.inspect", "transitousLock.inspect", "transitousLock.propose", "transitousLock.approve", @@ -260,6 +261,7 @@ export const OPS_KIND_POLICIES = { "valhalla.traffic.maintain": asyncEffect("data-manager", 30 * 60_000), "valhalla.traffic.refreshWaysToEdges": asyncEffect("data-manager", 30 * 60_000), "postgis.capacity.inspect": sync("data-manager", 30_000), + "transitCredentials.inspect": sync("api", 120_000, 768 * 1024), "catalogPins.inspect": sync("api", 30_000, 768 * 1024), "transitousLock.inspect": sync("data-manager", 30_000), "transitousLock.propose": sync("data-manager", 30_000), @@ -546,6 +548,10 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ // The catalog lock lives under the repository's `infra/docker/`, which only // the operations agent may write. Callers name a pinned ref, never a path. kindOnly("catalogPins.inspect"), + z.strictObject({ + kind: z.literal("transitCredentials.inspect"), + catalogRevisionId: z.string().regex(/^[a-f0-9]{64}$/), + }), kindOnly("transitousLock.inspect"), z.strictObject({ kind: z.literal("transitousLock.propose"), @@ -600,6 +606,8 @@ const opsRequestSchema = z export type OpsRequest = z.infer; export function authorizeOpsOperation(role: OpsRole, kind: OpsOperationKind): boolean { + // Decode retained historical records, but never admit a new remote file writer. + if (kind === "data.generateApiKeys") return false; return ( OPS_KIND_POLICIES[kind].role === role || (role === "data-manager" && kind === "catalogPins.inspect") @@ -1280,6 +1288,23 @@ export const OPS_RESULT_SCHEMAS = { }), "valhalla.traffic.refreshWaysToEdges": changed, "postgis.capacity.inspect": z.strictObject({ availableBytes: z.number().int().nonnegative() }), + "transitCredentials.inspect": z.strictObject({ + catalogRevisionId: z.string().regex(/^[a-f0-9]{64}$/), + requiredKeys: z.array(z.string().min(1).max(1024)).max(1000), + hostCredentialsRevision: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + owner: z.enum(["vault", "file", "environment", "unset"]), + editable: z.boolean(), + reason: z.string().max(512).nullable(), + configuredKeys: z.array(z.string().min(1).max(1024)).max(10000), + retainedKeys: z.array(z.string().min(1).max(1024)).max(10000), + appliedCredentialsRevision: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + }), "catalogPins.inspect": z .strictObject({ active: catalogPinSetSchema.nullable(), diff --git a/packages/core/src/ops/feed-credential-map.ts b/packages/core/src/ops/feed-credential-map.ts index 31bf5718f..e4ffd5956 100644 --- a/packages/core/src/ops/feed-credential-map.ts +++ b/packages/core/src/ops/feed-credential-map.ts @@ -1,75 +1,16 @@ -const rejected = () => new Error("Feed credential map exceeds its format or budget"); -const forbidden = new Set(["__proto__", "prototype", "constructor"]); -// biome-ignore lint/suspicious/noControlCharactersInRegex: JSON strings must reject literal C0 control characters. -const token = /[ \t\r\n]*("(?:[^"\\\u0000-\u001f]|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*")[ \t\r\n]*/y; - -/** Flat JSON parser intentionally retains keys until duplicates have been checked. */ -function readMap(input: string): [string, string][] { - let offset = 0; - const whitespace = () => { - while (/[ \t\r\n]/.test(input[offset] ?? "") && offset < input.length) offset++; - }; - const take = (expected: string) => { - whitespace(); - if (input[offset++] !== expected) throw rejected(); - }; - const string = () => { - token.lastIndex = offset; - const match = token.exec(input); - if (!match) throw rejected(); - offset = token.lastIndex; - return JSON.parse(match[1]!) as string; - }; - take("{"); - const entries: [string, string][] = []; - whitespace(); - if (input[offset] !== "}") { - for (;;) { - if (entries.length === 1000) throw rejected(); - const key = string(); - take(":"); - entries.push([key, string()]); - if (input[offset] !== ",") break; - offset++; - } - } - take("}"); - whitespace(); - if (offset !== input.length) throw rejected(); - return entries; -} - -/** Count every retained key, escaping and UTF-8 byte; never truncates or drops orphans. */ +/** Compatibility projection for the original trusted-configuration callers. + * One portable implementation owns JSON parsing, ordering and final budgets. */ +import { + parseFeedCredentialMapJson, + canonicalFeedCredentialMap as validate, +} from "@openmapx/transitous-core/feed-credentials"; export function canonicalFeedCredentialMap( input: string | Readonly>, ): string { try { - if (typeof input === "string" && (/[\uD800-\uDFFF]/u.test(input) || input.length > 262144)) - throw rejected(); - if (typeof input !== "string" && (!input || Array.isArray(input) || typeof input !== "object")) - throw rejected(); - const entries = typeof input === "string" ? readMap(input) : Object.entries(input); - if (entries.length > 1000) throw rejected(); - const keys = new Set(); - for (const [key, value] of entries) { - if ( - !key || - /[\uD800-\uDFFF]/u.test(key) || - [...key].length > 512 || - forbidden.has(key) || - keys.has(key) || - typeof value !== "string" || - /[\uD800-\uDFFF]/u.test(value) || - Buffer.byteLength(value, "utf8") > 8192 - ) - throw rejected(); - keys.add(key); - } - entries.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)); - const canonical = `{${entries.map(([key, value]) => `${JSON.stringify(key)}:${JSON.stringify(value)}`).join(",")}}`; - if (Buffer.byteLength(canonical, "utf8") > 65536) throw rejected(); - return canonical; + return validate(typeof input === "string" ? parseFeedCredentialMapJson(input) : input) + .canonicalJson; } catch { - throw rejected(); + throw Error("Feed credential map exceeds its format or budget"); } } diff --git a/packages/core/src/ops/transit-credentials.test.ts b/packages/core/src/ops/transit-credentials.test.ts new file mode 100644 index 000000000..f571e5826 --- /dev/null +++ b/packages/core/src/ops/transit-credentials.test.ts @@ -0,0 +1,32 @@ +import { expect, it } from "vitest"; +import { authorizeOpsOperation, opsOperationSchema, parseOpsResult } from "./contract"; + +it("offers a read-only API inspection tied to one exact catalog, without credential values", () => { + const input = { kind: "transitCredentials.inspect", catalogRevisionId: "a".repeat(64) }; + expect(opsOperationSchema.safeParse(input).success).toBe(true); + expect(authorizeOpsOperation("api", "transitCredentials.inspect")).toBe(true); + expect(authorizeOpsOperation("data-manager", "transitCredentials.inspect")).toBe(false); + expect(opsOperationSchema.safeParse({ ...input, path: "/etc/passwd" }).success).toBe(false); + const result = { + catalogRevisionId: input.catalogRevisionId, + requiredKeys: ["de/provider"], + owner: "unset", + editable: true, + reason: null, + configuredKeys: [], + retainedKeys: [], + appliedCredentialsRevision: null, + hostCredentialsRevision: null, + }; + expect(parseOpsResult("transitCredentials.inspect", result)).toEqual(result); + expect(() => + parseOpsResult("transitCredentials.inspect", { + ...result, + values: { "de/provider": "sensitive" }, + }), + ).toThrow(); +}); +it("denies fresh execution of the retired remote file generator for every caller role", () => { + expect(authorizeOpsOperation("api", "data.generateApiKeys")).toBe(false); + expect(authorizeOpsOperation("data-manager", "data.generateApiKeys")).toBe(false); +}); diff --git a/packages/core/src/ops/trusted-config.ts b/packages/core/src/ops/trusted-config.ts index 6bcc3814e..872e78c3c 100644 --- a/packages/core/src/ops/trusted-config.ts +++ b/packages/core/src/ops/trusted-config.ts @@ -127,6 +127,11 @@ export const trustedConfigurationPayloadSchema = z .strictObject({ domain, platform: platformConfigurationSchema.optional(), + // Opaque encrypted-row identity, never a digest of a credential value. + transitCredentialsRevision: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), // A new selection, carried by `serviceSelection.apply` only; every other // apply renders the operator's selection as it stands. selectedRoots: z.array(serviceId).max(256).optional(), diff --git a/packages/core/src/services/__tests__/configuration-generation.test.ts b/packages/core/src/services/__tests__/configuration-generation.test.ts index 9be75f46b..cfc112178 100644 --- a/packages/core/src/services/__tests__/configuration-generation.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation.test.ts @@ -79,6 +79,29 @@ function input(overrides: Partial = {}): ConfigurationInput } describe("configuration generations", () => { + it("carries opaque managed credential revision through unrelated CLI renders", async () => { + const { infraDir } = checkout(); + const revision = "b".repeat(64); + await commitConfigurationGeneration({ + infraDir, + services, + input: input({ transitCredentialsRevision: revision } as Partial), + }); + expect(readAppliedConfiguration(infraDir)).toHaveProperty( + "transitCredentialsRevision", + revision, + ); + await commitConfigurationGeneration({ + infraDir, + services, + input: input(readAppliedConfiguration(infraDir)), + }); + expect(readAppliedConfiguration(infraDir)).toHaveProperty( + "transitCredentialsRevision", + revision, + ); + }); + it("gives each application a fresh identity even when content repeats", async () => { const { root, infraDir } = checkout(); const first = await commitConfigurationGeneration({ infraDir, services, input: input() }); diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index edde022d8..d0d8b9253 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -700,6 +700,7 @@ export function readAppliedConfiguration( serviceConfigs?: ConfigurationInput["serviceConfigs"]; integrationConfigs?: ConfigurationInput["integrationConfigs"]; platform?: PlatformConfiguration; + transitCredentialsRevision?: string; }; const serviceSecrets: ConfigurationInput["serviceSecrets"] = []; const secretsRoot = join(generation, GENERATED_SECRETS_DIRNAME); @@ -716,12 +717,29 @@ export function readAppliedConfiguration( serviceConfigs: resolved.serviceConfigs ?? [], integrationConfigs: resolved.integrationConfigs ?? [], serviceSecrets, + ...(resolved.transitCredentialsRevision + ? { transitCredentialsRevision: resolved.transitCredentialsRevision } + : {}), ...(resolved.platform ? { platform: platformConfigurationSchema.parse(resolved.platform) } : {}), }; } +/** Non-secret encrypted-row revision of the applied managed map. */ +export function readAppliedTransitCredentialsRevision(infraDir: string): string | null { + const active = currentConfigurationGeneration(infraDir); + if (!active) return null; + const resolved = JSON.parse( + readFileSync(join(infraDir, CONFIGURATION_GENERATIONS, active, RESOLVED_CONFIG_FILE), "utf8"), + ); + return resolved.transitCredentialsRevision === undefined + ? null + : (trustedConfigurationPayloadSchema.shape.transitCredentialsRevision.parse( + resolved.transitCredentialsRevision, + ) ?? null); +} + /** Read only non-secret platform metadata, never service secret files. */ export function readAppliedPlatformConfiguration( infraDir: string, @@ -1208,6 +1226,9 @@ async function commitOwnedConfigurationGeneration( `${JSON.stringify({ serviceConfigs: input.serviceConfigs, integrationConfigs: input.integrationConfigs, + ...(input.transitCredentialsRevision + ? { transitCredentialsRevision: input.transitCredentialsRevision } + : {}), ...(input.platform ? { platform: platformConfigurationSchema.parse(input.platform) } : {}), })}\n`, 0o600, diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index a67c62008..eb1c824d6 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -43,6 +43,7 @@ export { readAppliedConfiguration, readAppliedPlatformConfiguration, readAppliedServiceIds, + readAppliedTransitCredentialsRevision, renderConfiguration, retainedContainerComposeFile, withConfigurationGenerationWriter, @@ -71,6 +72,7 @@ export { export { CONFIGURATION_GENERATIONS, CURRENT_CONFIGURATION, + configurationGenerationBasePath, currentConfigurationFile, type DesiredSelection, type DesiredSelectionSource, diff --git a/packages/transitous-core/__tests__/feed-credential-requirements.test.ts b/packages/transitous-core/__tests__/feed-credential-requirements.test.ts new file mode 100644 index 000000000..7cd87c3c8 --- /dev/null +++ b/packages/transitous-core/__tests__/feed-credential-requirements.test.ts @@ -0,0 +1,55 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { collectRequiredFeedCredentialKeys } from "../src/feed-credential-requirements"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function catalog() { + const root = mkdtempSync(join(tmpdir(), "omx-credential-catalog-")); + roots.push(root); + mkdirSync(join(root, "feeds")); + mkdirSync(join(root, "transitland-atlas/feeds"), { recursive: true }); + writeFileSync( + join(root, "transitland-atlas/feeds/a.dmfr.json"), + JSON.stringify({ feeds: [{ id: "auth", authorization: { type: "api-key" } }, { id: "open" }] }), + ); + return root; +} +it("discovers requirements only from the supplied isolated catalog and atlas", () => { + const a = catalog(), + b = catalog(); + writeFileSync( + join(a, "feeds/de.json"), + JSON.stringify({ + sources: [ + { name: "A", "transitland-atlas-id": "auth" }, + { name: "Open", "transitland-atlas-id": "open" }, + { name: "Inline", "transitland-atlas-id": "auth", "api-key": "already" }, + { + name: "Override", + "transitland-atlas-id": "auth", + "url-override": "https://example.test", + }, + ], + }), + ); + writeFileSync( + join(b, "feeds/de.json"), + JSON.stringify({ sources: [{ name: "B", "transitland-atlas-id": "auth" }] }), + ); + expect(collectRequiredFeedCredentialKeys(a)).toEqual(["de/A"]); + expect(collectRequiredFeedCredentialKeys(b)).toEqual(["de/B"]); +}); +it("refuses missing, malformed or linked catalog inputs rather than erasing requirements", () => { + const root = catalog(); + expect(() => collectRequiredFeedCredentialKeys(join(root, "missing"))).toThrow(); + writeFileSync(join(root, "feeds/de.json"), "invalid"); + expect(() => collectRequiredFeedCredentialKeys(root)).toThrow(/Catalog/); + rmSync(join(root, "feeds/de.json")); + symlinkSync(join(root, "transitland-atlas/feeds/a.dmfr.json"), join(root, "feeds/de.json")); + expect(() => collectRequiredFeedCredentialKeys(root)).toThrow(/Catalog/); +}); diff --git a/packages/transitous-core/__tests__/feed-credentials.test.ts b/packages/transitous-core/__tests__/feed-credentials.test.ts new file mode 100644 index 000000000..3ea53f591 --- /dev/null +++ b/packages/transitous-core/__tests__/feed-credentials.test.ts @@ -0,0 +1,109 @@ +import { expect, it, vi } from "vitest"; +import { + canonicalFeedCredentialMap, + mergeFeedCredentialTemplate, + parseFeedCredentialMapJson, + validateFeedCredentialMap, +} from "../src/feed-credentials"; + +it("preserves retained values and orphaned keys without trimming or silent deletion", () => { + const result = mergeFeedCredentialTemplate( + { "de/required": " secret\n", orphan: "retained", empty: "" }, + ["de/required", "at/new"], + ); + expect(result.values).toEqual({ + "de/required": " secret\n", + orphan: "retained", + empty: "", + "at/new": "", + }); + expect(result.orphanedKeys).toEqual(["empty", "orphan"]); + expect(result.droppedCount).toBe(0); + expect( + validateFeedCredentialMap(result.values, [{ key: "at/new", action: "set", value: " next " }]) + .values.orphan, + ).toBe("retained"); +}); +it("sorts canonical keys by code-unit order including integer-looking keys", () => { + expect(canonicalFeedCredentialMap({ "2": "b", "10": "a", z: " ", a: "\n" }).canonicalJson).toBe( + '{"10":"a","2":"b","a":"\\n","z":" "}', + ); +}); +it.each([ + '{"a":"one","a":"two"}', + '{"a":"one","\\u0061":"two"}', + "[]", + '{"a":null}', + '{"__proto__":"bad"}', + '{"a":"\\ud800"}', + '{"\\ud800":"v"}', + '{"a":"v",}', +])("refuses malformed/duplicate/unsafe input without leaking its body: %s", (raw) => { + expect(() => parseFeedCredentialMapJson(raw)).toThrow(/Feed credential/); +}); +it("validates the final aggregate using UTF-8 and re-escaped canonical bytes", () => { + const values = Object.fromEntries( + Array.from({ length: 8 }, (_, n) => [String(n), "x".repeat(8192)]), + ); + expect(() => canonicalFeedCredentialMap(values)).toThrow(/budget/); + const fixed = validateFeedCredentialMap(values, [{ key: "7", action: "delete" }]); + expect(fixed.entryCount).toBe(7); + expect(fixed.utf8Bytes).toBe(new TextEncoder().encode(fixed.canonicalJson).byteLength); + expect(() => canonicalFeedCredentialMap({ a: "ü".repeat(4097) })).toThrow(/budget/); + expect(() => canonicalFeedCredentialMap({ a: '"'.repeat(8193) })).toThrow(/budget/); + const thousand = Object.fromEntries(Array.from({ length: 1000 }, (_, n) => [`key${n}`, ""])); + expect(canonicalFeedCredentialMap(thousand).entryCount).toBe(1000); + expect(() => canonicalFeedCredentialMap({ ...thousand, extra: "" })).toThrow(/budget/); +}); +it("rejects duplicate mutations, malformed Unicode and overlong keys", () => { + expect(() => + validateFeedCredentialMap({ a: "v" }, [ + { key: "a", action: "delete" }, + { key: "a", action: "set", value: "v" }, + ]), + ).toThrow(); + expect(() => canonicalFeedCredentialMap({ ["ü".repeat(513)]: "v" })).toThrow(); + expect(canonicalFeedCredentialMap({ ["😀".repeat(512)]: "v" }).entryCount).toBe(1); + expect(() => canonicalFeedCredentialMap({ a: "\ud800" })).toThrow(); +}); +it("lets a legacy aggregate be corrected only when the complete merged map is valid", () => { + const raw = JSON.stringify( + Object.fromEntries(Array.from({ length: 9 }, (_, n) => [String(n), "x".repeat(8192)])), + ); + const existing = parseFeedCredentialMapJson(raw); + expect(() => validateFeedCredentialMap(existing, [{ key: "0", action: "delete" }])).toThrow(); + expect( + validateFeedCredentialMap(existing, [ + { key: "0", action: "delete" }, + { key: "1", action: "delete" }, + ]).entryCount, + ).toBe(7); +}); + +it("accepts exactly65536 canonical bytes and refuses65537 after escaping", () => { + const values: Record = Object.fromEntries( + Array.from({ length: 8 }, (_, n) => [String(n), n === 7 ? "" : "x".repeat(8192)]), + ); + const base = new TextEncoder().encode(JSON.stringify(values)).byteLength; + values["7"] = "x".repeat(65536 - base); + expect(canonicalFeedCredentialMap(values).utf8Bytes).toBe(65536); + values["7"] += "x"; + expect(() => canonicalFeedCredentialMap(values)).toThrow(/budget/); + expect(() => + canonicalFeedCredentialMap( + Object.fromEntries(Array.from({ length: 8 }, (_, n) => [String(n), '"'.repeat(8192)])), + ), + ).toThrow(/budget/); +}); + +it("keeps the portable data validator independent of DOM encoding globals", async () => { + vi.stubGlobal("TextEncoder", undefined); + vi.resetModules(); + try { + const module = await import("../src/feed-credentials"); + expect(module.canonicalFeedCredentialMap({ emoji: "😀", accent: "ü" }).utf8Bytes).toBe(30); + } finally { + vi.unstubAllGlobals(); + vi.resetModules(); + } +}); diff --git a/packages/transitous-core/__tests__/pinned-feed-credentials.test.ts b/packages/transitous-core/__tests__/pinned-feed-credentials.test.ts new file mode 100644 index 000000000..077e165d3 --- /dev/null +++ b/packages/transitous-core/__tests__/pinned-feed-credentials.test.ts @@ -0,0 +1,111 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { inspectPinnedFeedCredentialKeys } from "../src/pinned-feed-credentials"; + +const roots: string[] = []; +afterEach(() => { + for (const r of roots.splice(0)) rmSync(r, { recursive: true, force: true }); +}); +const run = (cwd: string, args: readonly string[]) => + execFileSync("git", [...args], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + GIT_AUTHOR_NAME: "OpenMapX fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "OpenMapX fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }).trim(); +function fixture() { + const root = mkdtempSync(join(tmpdir(), "omx-pinned-credential-fixture-")); + roots.push(root); + const atlas = join(root, "transitland-atlas"); + mkdirSync(join(root, "feeds")); + mkdirSync(join(atlas, "feeds"), { recursive: true }); + run(root, ["init"]); + run(atlas, ["init"]); + writeFileSync( + join(atlas, "feeds/a.dmfr.json"), + JSON.stringify({ feeds: [{ id: "auth", authorization: { type: "api-key" } }] }), + ); + run(atlas, ["add", "."]); + run(atlas, ["commit", "-m", "Atlas A"]); + const atlasCommit = run(atlas, ["rev-parse", "HEAD"]); + writeFileSync( + join(root, "feeds/de.json"), + JSON.stringify({ sources: [{ name: "A", "transitland-atlas-id": "auth" }] }), + ); + run(root, ["add", "feeds"]); + run(root, ["update-index", "--add", "--cacheinfo", `160000,${atlasCommit},transitland-atlas`]); + run(root, ["commit", "-m", "Transitous A"]); + const transitousCommit = run(root, ["rev-parse", "HEAD"]); + writeFileSync( + join(root, "feeds/de.json"), + JSON.stringify({ sources: [{ name: "B", "transitland-atlas-id": "auth" }] }), + ); + run(root, ["add", "feeds"]); + run(root, ["commit", "-m", "Transitous B"]); + return { root, atlasCommit, transitousCommit }; +} +it("reads actual pin A while active cache is at B without resetting/pulling the cache", async () => { + const f = fixture(), + before = readFileSync(join(f.root, "feeds/de.json")), + head = run(f.root, ["rev-parse", "HEAD"]), + calls: readonly string[][] = []; + const keys = await inspectPinnedFeedCredentialKeys({ + catalogDir: f.root, + transitousCommit: f.transitousCommit, + atlasCommit: f.atlasCommit, + git: async (args, cwd) => { + (calls as string[][]).push([...args]); + return run(cwd, args); + }, + }); + expect(keys).toEqual(["de/A"]); + expect(readFileSync(join(f.root, "feeds/de.json"))).toEqual(before); + expect(run(f.root, ["rev-parse", "HEAD"])).toBe(head); + expect(calls.flat()).not.toContain("pull"); + expect(calls.flat()).not.toContain("reset"); +}); +it("rejects a foreign atlas pin or unavailable commit without touching the cache", async () => { + const f = fixture(), + head = run(f.root, ["rev-parse", "HEAD"]); + await expect( + inspectPinnedFeedCredentialKeys({ + catalogDir: f.root, + transitousCommit: f.transitousCommit, + atlasCommit: "b".repeat(40), + git: async (args, cwd) => run(cwd, args), + }), + ).rejects.toThrow(/unavailable/); + expect(run(f.root, ["rev-parse", "HEAD"])).toBe(head); +}); + +it.each(["oversize", "symlink"])("refuses %s repository content before checkout", async (kind) => { + const f = fixture(); + if (kind === "oversize") + writeFileSync(join(f.root, "large.dat"), Buffer.alloc(4 * 1024 * 1024 + 1, 120)); + else symlinkSync("../../outside", join(f.root, "feeds/linked.json")); + run(f.root, ["add", "."]); + run(f.root, ["commit", "-m", "Rejected content"]); + const sha = run(f.root, ["rev-parse", "HEAD"]); + let checkouts = 0; + await expect( + inspectPinnedFeedCredentialKeys({ + catalogDir: f.root, + transitousCommit: sha, + atlasCommit: f.atlasCommit, + git: async (args, cwd) => { + if (args.includes("checkout")) checkouts++; + return run(cwd, args); + }, + }), + ).rejects.toThrow(/unavailable/); + expect(checkouts).toBe(0); +}); diff --git a/packages/transitous-core/package.json b/packages/transitous-core/package.json index 97ff297c2..27d568de5 100644 --- a/packages/transitous-core/package.json +++ b/packages/transitous-core/package.json @@ -18,5 +18,15 @@ "@types/node": "^26.6.5", "typescript": "^6.0.3", "vitest": "^5.0.3" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + }, + "./feed-credentials": { + "types": "./src/feed-credentials.ts", + "default": "./src/feed-credentials.ts" + } } } diff --git a/packages/transitous-core/src/feed-credential-requirements.ts b/packages/transitous-core/src/feed-credential-requirements.ts new file mode 100644 index 000000000..ac1979ac5 --- /dev/null +++ b/packages/transitous-core/src/feed-credential-requirements.ts @@ -0,0 +1,124 @@ +/** Read-only requirements from an already isolated, pin-verified checkout. + * Pin resolution and checkout ownership belong to the caller. */ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readdirSync, + readFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { isWellFormedFeedCredentialText } from "./feed-credentials"; + +const unavailable = () => new Error("Catalog credential requirements are unavailable"); +const decoder = new TextDecoder("utf-8", { fatal: true }); +export function collectRequiredFeedCredentialKeys(catalogDir: string): string[] { + let bytes = 0, + files = 0; + const directory = (path: string) => { + const stat = lstatSync(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw unavailable(); + }; + const json = (path: string): Record => { + let fd: number | undefined; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd); + files += 1; + bytes += before.size; + if ( + !before.isFile() || + before.nlink !== 1 || + before.size > 4 * 1024 * 1024 || + files > 50000 || + bytes > 128 * 1024 * 1024 + ) + throw unavailable(); + const raw = readFileSync(fd), + after = fstatSync(fd), + current = lstatSync(path); + if ( + raw.length !== before.size || + before.ino !== current.ino || + before.dev !== current.dev || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw unavailable(); + const value: unknown = JSON.parse(decoder.decode(raw)); + if (!value || typeof value !== "object" || Array.isArray(value)) throw unavailable(); + return value as Record; + } catch { + throw unavailable(); + } finally { + if (fd !== undefined) closeSync(fd); + } + }; + const atlas = join(catalogDir, "transitland-atlas", "feeds"), + feedDir = join(catalogDir, "feeds"), + authorized = new Set(), + required = new Set(); + try { + directory(catalogDir); + directory(join(catalogDir, "transitland-atlas")); + directory(atlas); + directory(feedDir); + const pending = [atlas]; + let dirs = 0; + while (pending.length) { + const path = pending.pop(); + if (!path) throw unavailable(); + if (++dirs > 10000) throw unavailable(); + directory(path); + const entries = readdirSync(path, { withFileTypes: true }); + if (entries.length > 50000) throw unavailable(); + for (const entry of entries) { + if (entry.isSymbolicLink()) throw unavailable(); + const file = join(path, entry.name); + if (entry.isDirectory()) pending.push(file); + else if (entry.isFile() && entry.name.endsWith(".dmfr.json")) { + const value = json(file); + if (value.feeds !== undefined && !Array.isArray(value.feeds)) throw unavailable(); + for (const feed of (value.feeds ?? []) as Record[]) { + if (!feed || typeof feed !== "object") throw unavailable(); + if ( + typeof feed.id === "string" && + feed.id.length && + feed.authorization !== undefined && + feed.authorization !== null + ) + authorized.add(feed.id); + } + } + } + } + const feeds = readdirSync(feedDir, { withFileTypes: true }); + if (feeds.length > 50000) throw unavailable(); + for (const entry of feeds) { + if (entry.isSymbolicLink()) throw unavailable(); + if (!entry.isFile() || !entry.name.endsWith(".json")) continue; + const value = json(join(feedDir, entry.name)); + if (value.sources !== undefined && !Array.isArray(value.sources)) throw unavailable(); + for (const source of (value.sources ?? []) as Record[]) { + if (!source || typeof source !== "object") throw unavailable(); + const id = source["transitland-atlas-id"]; + if (typeof id !== "string" || !authorized.has(id)) continue; + if (typeof source["api-key"] === "string" && source["api-key"].trim()) continue; + if (typeof source["url-override"] === "string" && source["url-override"].trim()) continue; + if ( + typeof source.name !== "string" || + !source.name.trim() || + !isWellFormedFeedCredentialText(source.name) + ) + throw unavailable(); + required.add(`${entry.name.slice(0, -5)}/${source.name}`); + } + } + return [...required].sort(); + } catch { + throw unavailable(); + } +} diff --git a/packages/transitous-core/src/feed-credentials.ts b/packages/transitous-core/src/feed-credentials.ts new file mode 100644 index 000000000..d8ba0488e --- /dev/null +++ b/packages/transitous-core/src/feed-credentials.ts @@ -0,0 +1,185 @@ +/** Pure credential data validation. These values confer no authority; callers + * must separately check catalog membership, ownership, proofs and revisions. */ +export type FeedCredentialChange = + | { key: string; action: "set"; value: string } + | { key: string; action: "delete" }; +export type ValidatedFeedCredentialMap = { + values: Readonly>; + canonicalJson: string; + utf8Bytes: number; + entryCount: number; +}; +function utf8BytesOf(value: string) { + let bytes = 0; + for (const character of value) { + const code = character.codePointAt(0) ?? 0; + bytes += code <= 0x7f ? 1 : code <= 0x7ff ? 2 : code <= 0xffff ? 3 : 4; + } + return bytes; +} +// With Unicode regex semantics a valid surrogate pair is one astral code point; +// only lone surrogate code units match this range. Works with the package's ES target. +export const isWellFormedFeedCredentialText = (value: string) => !/[\uD800-\uDFFF]/u.test(value); +const invalid = () => new Error("Feed credential map is invalid"); +const budget = () => new Error("Feed credential map exceeds its budget"); +const unsafe = new Set(["__proto__", "constructor", "prototype"]); +function keyIsValid(key: string) { + return ( + isWellFormedFeedCredentialText(key) && + key.length > 0 && + [...key].length <= 512 && + !unsafe.has(key) + ); +} +function entriesOf(value: unknown): [string, string][] { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + ![null, Object.prototype].includes(Object.getPrototypeOf(value)) || + Object.getOwnPropertySymbols(value).length + ) + throw invalid(); + return Object.entries(Object.getOwnPropertyDescriptors(value)).map(([key, descriptor]) => { + if ( + !keyIsValid(key) || + !("value" in descriptor) || + typeof descriptor.value !== "string" || + !isWellFormedFeedCredentialText(descriptor.value) + ) + throw invalid(); + return [key, descriptor.value]; + }); +} +export function canonicalFeedCredentialMap( + values: Readonly>, +): ValidatedFeedCredentialMap { + const entries = entriesOf(values).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + if (entries.length > 1000 || entries.some(([, value]) => utf8BytesOf(value) > 8192)) + throw budget(); + // JSON.stringify(object) reorders integer-looking keys numerically; serialize + // the sorted entries explicitly so every key uses the same code-unit order. + const canonicalJson = `{${entries.map(([key, value]) => `${JSON.stringify(key)}:${JSON.stringify(value)}`).join(",")}}`; + const utf8Bytes = utf8BytesOf(canonicalJson); + if (utf8Bytes > 65536) throw budget(); + return { + values: Object.freeze(Object.fromEntries(entries)), + canonicalJson, + utf8Bytes, + entryCount: entries.length, + }; +} +export function parseFeedCredentialMapJson(json: string): Readonly> { + // A bounded legacy aggregate may be read for explicit correction; final-map + // limits are enforced after merging, including every unchanged/orphaned key. + if ( + typeof json !== "string" || + !isWellFormedFeedCredentialText(json) || + utf8BytesOf(json) > 1048576 + ) + throw invalid(); + let cursor = 0; + const whitespace = () => { + while (cursor < json.length && /[\t\r\n ]/.test(json[cursor] ?? "")) cursor++; + }; + const string = (): string => { + const start = cursor; + if (json[cursor++] !== '"') throw invalid(); + let escaped = false; + while (cursor < json.length) { + const char = json[cursor++]; + if (char === '"' && !escaped) { + try { + const value: unknown = JSON.parse(json.slice(start, cursor)); + if (typeof value !== "string" || !isWellFormedFeedCredentialText(value)) throw invalid(); + return value; + } catch { + throw invalid(); + } + } + escaped = char === "\\" && !escaped; + } + throw invalid(); + }; + const retained = new Map(); + whitespace(); + if (json[cursor++] !== "{") throw invalid(); + whitespace(); + if (json[cursor] !== "}") { + for (;;) { + const key = string(); + if (!keyIsValid(key) || retained.has(key)) throw invalid(); + whitespace(); + if (json[cursor++] !== ":") throw invalid(); + whitespace(); + const value = string(); + retained.set(key, value); + whitespace(); + if (json[cursor] !== ",") break; + cursor++; + whitespace(); + } + } + if (json[cursor++] !== "}") throw invalid(); + whitespace(); + if (cursor !== json.length) throw invalid(); + return Object.freeze(Object.fromEntries(retained)); +} +export function validateFeedCredentialMap( + existing: Readonly>, + changes: readonly FeedCredentialChange[], +): ValidatedFeedCredentialMap { + const values = Object.fromEntries(entriesOf(existing)); + if (!Array.isArray(changes) || changes.length > 100) throw invalid(); + const seen = new Set(); + for (const change of changes) { + if ( + !change || + typeof change !== "object" || + typeof change.key !== "string" || + !keyIsValid(change.key) || + seen.has(change.key) + ) + throw invalid(); + seen.add(change.key); + if (change.action === "set") { + if ( + Object.keys(change).some((key) => !["key", "action", "value"].includes(key)) || + typeof change.value !== "string" || + !isWellFormedFeedCredentialText(change.value) + ) + throw invalid(); + values[change.key] = change.value; + } else if (change.action === "delete") { + if (Object.keys(change).some((key) => !["key", "action"].includes(key))) throw invalid(); + delete values[change.key]; + } else throw invalid(); + } + return canonicalFeedCredentialMap(values); +} +export function mergeFeedCredentialTemplate( + existing: Readonly>, + required: readonly string[], +) { + if ( + !Array.isArray(required) || + required.some((key) => typeof key !== "string" || !keyIsValid(key)) + ) + throw invalid(); + const values = Object.fromEntries(entriesOf(existing)), + keys = [...new Set(required)].sort(), + requiredKeys = new Set(keys); + const orphanedKeys = Object.keys(values) + .filter((key) => !requiredKeys.has(key)) + .sort(); + const preservedCount = Object.keys(values).length; + for (const key of keys) if (!Object.hasOwn(values, key)) values[key] = ""; + const validated = canonicalFeedCredentialMap(values); + return { + values: validated.values, + requiredCount: keys.length, + preservedCount, + orphanedKeys, + droppedCount: 0 as const, + }; +} diff --git a/packages/transitous-core/src/index.ts b/packages/transitous-core/src/index.ts index cec6a4e7b..6c0c04110 100644 --- a/packages/transitous-core/src/index.ts +++ b/packages/transitous-core/src/index.ts @@ -16,6 +16,15 @@ export { TRANSITOUS_DOWNLOADS_DIR, type TransitSource, } from "./constants.js"; +export { collectRequiredFeedCredentialKeys } from "./feed-credential-requirements"; +export { + canonicalFeedCredentialMap, + type FeedCredentialChange, + mergeFeedCredentialTemplate, + parseFeedCredentialMapJson, + type ValidatedFeedCredentialMap, + validateFeedCredentialMap, +} from "./feed-credentials"; export type { TransitousFeedFile, TransitousFeedSource } from "./feed-source.js"; export { isSafeFeedSourceName } from "./feed-source.js"; export { @@ -44,6 +53,10 @@ export { type MotisConfigExpectations, parseMotisConfigExpectations, } from "./motis-config.js"; +export { + inspectPinnedFeedCredentialKeys, + type PinnedCredentialGitRunner, +} from "./pinned-feed-credentials"; export type { CommandRunner, TransitousLogger } from "./runner.js"; export { type PruneUnresolvableSourcesOptions, diff --git a/packages/transitous-core/src/pinned-feed-credentials.ts b/packages/transitous-core/src/pinned-feed-credentials.ts new file mode 100644 index 000000000..ef637cea7 --- /dev/null +++ b/packages/transitous-core/src/pinned-feed-credentials.ts @@ -0,0 +1,90 @@ +/** Local read-only clone of two content-addressed cached repositories. No fetch, + * reset, pull or submodule mutation of the shared active working tree. */ +import { lstatSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { collectRequiredFeedCredentialKeys } from "./feed-credential-requirements"; +export type PinnedCredentialGitRunner = (args: readonly string[], cwd: string) => Promise; +export async function inspectPinnedFeedCredentialKeys(input: { + catalogDir: string; + transitousCommit: string; + atlasCommit: string; + git: PinnedCredentialGitRunner; +}): Promise { + const unavailable = () => new Error("Pinned catalog credential requirements are unavailable"); + if (!/^[a-f0-9]{40}$/.test(input.transitousCommit) || !/^[a-f0-9]{40}$/.test(input.atlasCommit)) + throw unavailable(); + for (const path of [input.catalogDir, join(input.catalogDir, "transitland-atlas")]) { + const stat = lstatSync(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw unavailable(); + } + const root = mkdtempSync(join(tmpdir(), "openmapx-feed-credential-inspect-")), + checkout = join(root, "catalog"); + try { + await input.git( + ["clone", "--shared", "--no-hardlinks", "--no-checkout", "--", input.catalogDir, checkout], + root, + ); + let bytes = 0, + files = 0; + const checkTree = async (repository: string, commit: string) => { + const tree = await input.git(["-C", repository, "ls-tree", "-r", "-l", commit], root); + if (Buffer.byteLength(tree) > 5 * 1024 * 1024) throw unavailable(); + for (const line of tree.split("\n")) { + if (!line) continue; + if (/^160000 commit [a-f0-9]{40}\s+-\t/.test(line)) continue; + const match = /^(100644|100755) blob [a-f0-9]{40}\s+(\d+)\t/.exec(line); + if (!match) throw unavailable(); + const size = Number(match[2]); + bytes += size; + files++; + if ( + !Number.isSafeInteger(size) || + size > 4 * 1024 * 1024 || + bytes > 128 * 1024 * 1024 || + files > 50000 + ) + throw unavailable(); + } + }; + await checkTree(checkout, input.transitousCommit); + const entry = ( + await input.git( + ["-C", checkout, "ls-tree", input.transitousCommit, "--", "transitland-atlas"], + root, + ) + ).trim(); + if (entry !== `160000 commit ${input.atlasCommit}\ttransitland-atlas`) throw unavailable(); + const atlas = join(checkout, "transitland-atlas"); + await input.git( + [ + "clone", + "--shared", + "--no-hardlinks", + "--no-checkout", + "--", + join(input.catalogDir, "transitland-atlas"), + atlas, + ], + root, + ); + await checkTree(atlas, input.atlasCommit); + await input.git( + ["-C", checkout, "checkout", "--detach", "--force", input.transitousCommit], + root, + ); + if ( + (await input.git(["-C", checkout, "rev-parse", "HEAD"], root)).trim() !== + input.transitousCommit + ) + throw unavailable(); + await input.git(["-C", atlas, "checkout", "--detach", "--force", input.atlasCommit], root); + if ((await input.git(["-C", atlas, "rev-parse", "HEAD"], root)).trim() !== input.atlasCommit) + throw unavailable(); + return collectRequiredFeedCredentialKeys(checkout); + } catch { + throw unavailable(); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e6299ce5..bb7a1da78 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2536,6 +2536,9 @@ importers: '@openmapx/mobility-core': specifier: workspace:^ version: link:../mobility-core + '@openmapx/transitous-core': + specifier: workspace:^ + version: link:../transitous-core '@tanstack/react-query': specifier: ^5.104.1 version: 5.104.1(react@19.3.0) diff --git a/release-notes/admin-transit-feed-credentials.md b/release-notes/admin-transit-feed-credentials.md new file mode 100644 index 000000000..360433476 --- /dev/null +++ b/release-notes/admin-transit-feed-credentials.md @@ -0,0 +1,45 @@ +--- +impacts: [breaking, migration, operator-action] +--- + +## What changes + +The Transit administration page manages a masked map of per-feed credentials +against the active Transitous and atlas pins. Changes require a reviewed preview +and fresh administrator sign-in. Unselected and obsolete keys are preserved; +deleting a key requires an explicit selection. The same guarded operation saves +the encrypted map and recreates an already running Data Manager to consume it. + +Migration `0048_admin_transit_credentials` adds encrypted, temporary credential +proposals. Completed operations retain recovery inputs for at least 72 hours; +unresolved operations retain their inputs until they can be reconciled. Values +are excluded from public previews, job metadata and account data exports. + +The remote `generate-api-keys` operation is retired. The local +`openmapx data generate-api-keys` command remains available, reads the active pins +and preserves existing values and orphaned keys without updating the shared +catalog checkout. + +## Who is affected + +Operators managing Transitous feed credentials, including installations using +`TRANSITOUS_API_KEYS_PATH`, a custom `TRANSITOUS_API_KEYS_FILE`, or an existing +`/config/transitous/api-keys.json` file. + +## Required action + +Apply the normal API migrations when upgrading and upgrade the API, CLI and +ops-agent together. Use **Administration → Transit → Transit feed credentials** +for vault-managed credentials. Data Manager must already be running; this action +does not start a stopped coordinator. Inspect Activity before retrying an +unresolved operation. + +Existing files and environment overrides remain host-owned and read-only in the +panel. Continue managing them on the host unless deliberately changing their +ownership. No file is imported, replaced or deleted automatically. Integrations +that called the retired remote generator must use the credential workflow or +explicit host-local CLI management instead. + +Account deletion also waits for the administrator's unfinished instance +operations. Finish or cancel those operations in Activity first, so their +private recovery inputs remain available until settlement. diff --git a/scripts/check-subject-data.ts b/scripts/check-subject-data.ts index 169c9292a..3162ace23 100644 --- a/scripts/check-subject-data.ts +++ b/scripts/check-subject-data.ts @@ -37,6 +37,7 @@ export const DIRECT_USER_FK_CLASSIFICATIONS: Record = { "personal_vehicle.user_id": "personal-vehicles", "saved_list.user_id": "saved-lists", "service_secret.updated_by": "secret-update-attribution", + "transit_credential_preview.actor_id": "secret-update-attribution", "session.user_id": "auth-sessions", "admin_action_reauthentication.actor_id": "auth-sessions", "share_link.user_id": "share-links", diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 22cf93f88..9047a62e8 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -60,7 +60,9 @@ "services/data-manager/__tests__/overture/runtime-postgres.test.ts", "services/data-manager/__tests__/overture/schema-postgres.test.ts", "services/data-manager/__tests__/search-index/build-postgres.test.ts", - "services/data-manager/__tests__/search-index/schema-postgres.test.ts" + "services/data-manager/__tests__/search-index/schema-postgres.test.ts", + "apps/api/src/services/transit-credential-preview-postgres.test.ts", + "apps/api/src/services/transit-feed-credentials-postgres.test.ts" ], "export-restore": [ "apps/api/src/privacy/controller-wide-export-postgres.test.ts", diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index b368877f4..d6b033b3e 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -94,6 +94,7 @@ "system.diagnostics", "system.inspect", "system.update", + "transitCredentials.inspect", "vault.apply" ], "data-manager": [ @@ -139,7 +140,6 @@ "convert-overpass", "link", "clean", - "generate-api-keys", "overture-sync", "overture-conflate", "search-index-build", @@ -180,7 +180,6 @@ "data.convertOverpass", "data.link", "data.clean", - "data.generateApiKeys", "data.overtureSync", "data.overtureConflate", "data.searchIndexBuild", @@ -354,6 +353,24 @@ "configurationMaintenance.decide", "configurationMaintenance.settle" ] + }, + { + "id": "api.transit-credentials", + "callSiteIds": [], + "semanticSources": [ + "apps/api/src/services/transit-feed-credential-participant.ts#inspect", + "apps/ops-agent/src/transit-credentials-runtime.ts#installTransitCredentialsRuntime" + ], + "typedKinds": ["transitCredentials.inspect"] + }, + { + "id": "ops.retired-feed-credential-generator", + "callSiteIds": [], + "semanticSources": [ + "packages/core/src/ops/contract.ts#authorizeOpsOperation", + "apps/ops-agent/src/administrative-runtime.ts#createAdministrativeRuntime" + ], + "typedKinds": ["data.generateApiKeys"] } ], "mounts": [ diff --git a/services/data-manager/__tests__/transitous/internal.test.ts b/services/data-manager/__tests__/transitous/internal.test.ts index ff35f4016..f58748f70 100644 --- a/services/data-manager/__tests__/transitous/internal.test.ts +++ b/services/data-manager/__tests__/transitous/internal.test.ts @@ -373,3 +373,27 @@ describe("pruneUnresolvableSources", () => { expect(readSources(catalogDir).find((s) => s.name === "MBTA")?.skip).toBeUndefined(); }); }); + +describe("canonical credential overlay consumption", () => { + it("preserves the exact configured value rather than trimming it", () => { + const catalog = makeCatalog([{ name: "A", "transitland-atlas-id": "auth" }]); + const file = join(catalog, "keys.json"); + writeFileSync(file, JSON.stringify({ "us/A": " exact secret " })); + expect(applyApiKeysOverlay(catalog, file)).toBe(1); + expect( + JSON.parse(readFileSync(join(catalog, "feeds/us.json"), "utf8")).sources[0]["api-key"], + ).toBe(" exact secret "); + }); + it.each(['{"us/A":"one","us/A":"two"}', '{"us/A":42}', "[]", "{broken"])( + "rejects malformed maps without changing any feed: %s", + (bytes) => { + const catalog = makeCatalog([{ name: "A", "transitland-atlas-id": "auth" }]); + const file = join(catalog, "keys.json"), + feed = join(catalog, "feeds/us.json"), + before = readFileSync(feed, "utf8"); + writeFileSync(file, bytes); + expect(() => applyApiKeysOverlay(catalog, file)).toThrow(); + expect(readFileSync(feed, "utf8")).toBe(before); + }, + ); +}); diff --git a/services/data-manager/service.json b/services/data-manager/service.json index d4a7c4bc6..fbcb86f9e 100644 --- a/services/data-manager/service.json +++ b/services/data-manager/service.json @@ -23,7 +23,7 @@ "PORT": "4000", "HOST": "0.0.0.0", "LOG_LEVEL": "${LOG_LEVEL:-}", - "DATA_MANAGER_AUTH_TOKEN": "${DATA_MANAGER_AUTH_TOKEN:?DATA_MANAGER_AUTH_TOKEN must be set — generate with: openssl rand -hex 32}", + "DATA_MANAGER_AUTH_TOKEN": "${DATA_MANAGER_AUTH_TOKEN:?DATA_MANAGER_AUTH_TOKEN must be set \u2014 generate with: openssl rand -hex 32}", "DATABASE_URL": "postgresql://postgres:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set}@postgis:5432/openmapx", "OPS_AGENT_URL": "${OPS_AGENT_URL:-http://ops-agent:4300}", "OPS_AGENT_TOKEN_FILE": "/run/secrets/ops-agent-data-manager-token", @@ -227,5 +227,16 @@ }, "ui": { "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "TRANSITOUS_API_KEYS": { + "type": "string", + "title": "Transit feed credentials", + "description": "Managed by the dedicated Transit credentials page.", + "x-openmapx-secret": true + } + } } } diff --git a/services/data-manager/src/jobs/transitous/internal.ts b/services/data-manager/src/jobs/transitous/internal.ts index 6297c2b34..12cc5225c 100644 --- a/services/data-manager/src/jobs/transitous/internal.ts +++ b/services/data-manager/src/jobs/transitous/internal.ts @@ -5,6 +5,7 @@ import { fchmodSync, fstatSync, ftruncateSync, + lstatSync, mkdirSync, openSync, readdirSync, @@ -28,6 +29,10 @@ import { type TransitousFeedFile, type TransitousFeedSource, } from "@openmapx/transitous-core"; +import { + canonicalFeedCredentialMap, + parseFeedCredentialMapJson, +} from "@openmapx/transitous-core/feed-credentials"; import { execa } from "execa"; import { dataWorkerSignal } from "../execution/worker-scope.js"; import type { TransitousScriptRunner } from "./script-runner.js"; @@ -65,7 +70,12 @@ export const DEFAULT_TRANSITOUS_API_KEYS_PATH = "/config/transitous/api-keys.jso export function resolveTransitousApiKeysPath(explicit?: string): string { // `||`, not `??`: compose renders an unset variable as "", which must fall // through to the mounted default. - return explicit || process.env.TRANSITOUS_API_KEYS_PATH || DEFAULT_TRANSITOUS_API_KEYS_PATH; + return ( + explicit || + process.env.TRANSITOUS_API_KEYS_PATH || + process.env.TRANSITOUS_API_KEYS_FILE || + DEFAULT_TRANSITOUS_API_KEYS_PATH + ); } /** @@ -210,16 +220,43 @@ export function writePrivateCatalogFile(feedPath: string, content: string): void export function applyApiKeysOverlay(catalogDir: string, overlayPath: string): number { if (!existsSync(overlayPath)) return 0; - let overlay: Record; + const before = lstatSync(overlayPath); + if (!before.isFile() || before.isSymbolicLink() || before.size > 65536) + throw Error("Invalid feed credential map"); + const descriptor = openSync( + overlayPath, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + let overlay: Readonly>; try { - overlay = JSON.parse(readFileSync(overlayPath, "utf-8")) as Record; - } catch { - return 0; + const opened = fstatSync(descriptor); + if ( + !opened.isFile() || + opened.dev !== before.dev || + opened.ino !== before.ino || + opened.size > 65536 + ) + throw Error("Feed credential map changed"); + overlay = canonicalFeedCredentialMap( + parseFeedCredentialMapJson(readFileSync(descriptor, "utf8")), + ).values; + const after = fstatSync(descriptor), + current = lstatSync(overlayPath); + if ( + opened.dev !== current.dev || + opened.ino !== current.ino || + opened.size !== after.size || + opened.mtimeMs !== after.mtimeMs || + opened.ctimeMs !== after.ctimeMs + ) + throw Error("Feed credential map changed"); + } finally { + closeSync(descriptor); } let applied = 0; for (const [keyPath, rawValue] of Object.entries(overlay)) { - const apiKey = rawValue.trim(); + const apiKey = rawValue; if (!apiKey) continue; const slashIndex = keyPath.indexOf("/"); if (slashIndex <= 0) continue; From 5e6999f24f3f838b94804457ee93dec42932b599 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 07:42:52 +0100 Subject: [PATCH 053/135] feat(admin): review and activate validated catalog pin pairs --- apps/api/openapi.json | 8774 ++++++++++++++++- .../admin-transit-credentials.test.ts | 21 +- .../__tests__/admin-transit-pins.test.ts | 255 + .../src/routes/admin-transit-credentials.ts | 15 +- apps/api/src/routes/admin-transit-pins.ts | 294 + apps/api/src/routes/index.ts | 2 + .../activity-catalog-postgres.test.ts | 62 + apps/api/src/services/activity-jobs.ts | 2 +- .../transit-credential-preview.test.ts | 17 + .../services/transit-credential-preview.ts | 7 + .../services/transit-credential-revision.ts | 10 +- .../transit-feed-credential-participant.ts | 28 +- .../src/services/transit-feed-credentials.ts | 1 + apps/api/src/services/transit-pin-requests.ts | 28 + .../src/__fixtures__/owned-service-host.ts | 4 +- .../src/catalog-approval-runtime.test.ts | 39 + .../ops-agent/src/catalog-approval-runtime.ts | 349 + apps/ops-agent/src/catalog-pins.test.ts | 47 +- .../src/catalog-validation-inputs.ts | 161 + apps/ops-agent/src/docker-runtime.test.ts | 13 +- apps/ops-agent/src/docker-runtime.ts | 7 +- apps/ops-agent/src/production-runtime.ts | 9 + apps/ops-agent/src/server.ts | 1703 ++-- .../staged-coordinator-request-process.mjs | 69 +- .../src/transit-credentials-runtime.test.ts | 63 +- .../src/transit-credentials-runtime.ts | 162 +- .../src/transitous-lock-runtime.test.ts | 117 +- apps/ops-agent/src/transitous-lock-runtime.ts | 110 +- .../(full-admin)/transit/LockfileCard.tsx | 653 +- .../transit/__tests__/LockfileCard.test.tsx | 218 + .../TransitCredentialsSection.test.tsx | 2 +- .../services/TransitCredentialsSection.tsx | 41 +- apps/web/src/lib/admin/transitPinHooks.ts | 173 + .../motis-operations-profiles.md | 38 +- docs/docs/developer/cli-reference.md | 28 +- .../__tests__/cli-transitous-api-keys.test.ts | 45 +- .../cli/__tests__/transitous-bump.test.ts | 95 +- packages/cli/src/commands/data.ts | 4 +- packages/cli/src/commands/transitous-bump.ts | 371 +- packages/cli/src/lib/transitous-api-keys.ts | 30 +- .../api/contracts/data-manager-admin/index.ts | 1 + .../data-manager-admin/operations.ts | 2 + .../data-manager-admin/transit-credentials.ts | 5 + .../data-manager-admin/transit-pins.ts | 78 + .../admin/data-manager-admin.generated.ts | 830 +- .../models/activateAdminTransitPins202.ts | 7 + .../models/activateAdminTransitPins400.ts | 20 + .../models/activateAdminTransitPins401.ts | 20 + .../models/activateAdminTransitPins403.ts | 20 + .../models/activateAdminTransitPins404.ts | 20 + .../models/activateAdminTransitPins409.ts | 20 + .../models/activateAdminTransitPins413.ts | 20 + .../models/activateAdminTransitPins415.ts | 20 + .../models/activateAdminTransitPins422.ts | 20 + .../models/activateAdminTransitPins429.ts | 11 + .../models/activateAdminTransitPins500.ts | 20 + .../models/activateAdminTransitPins503.ts | 20 + .../models/activateAdminTransitPinsBody.ts | 11 + .../models/activateAdminTransitPinsHeaders.ts | 9 + .../adminGetAdminTransitPins200Schema0.ts | 10 + .../adminGetAdminTransitPins200Schema1.ts | 10 + .../models/applyAdminTransitCredentials404.ts | 20 + .../cancelAdminTransitPinExecution200.ts | 33 + ...lAdminTransitPinExecution200Diagnostics.ts | 9 + ...ansitPinExecution200DiagnosticsTailItem.ts | 8 + ...inExecution200DiagnosticsTailItemStream.ts | 10 + .../cancelAdminTransitPinExecution200Kind.ts | 23 + ...lAdminTransitPinExecution200Publication.ts | 11 + ...cancelAdminTransitPinExecution200Result.ts | 375 + ...cancelAdminTransitPinExecution200Status.ts | 16 + .../cancelAdminTransitPinExecution400.ts | 20 + .../cancelAdminTransitPinExecution401.ts | 20 + .../cancelAdminTransitPinExecution403.ts | 20 + .../cancelAdminTransitPinExecution404.ts | 20 + .../cancelAdminTransitPinExecution409.ts | 20 + .../cancelAdminTransitPinExecution413.ts | 20 + .../cancelAdminTransitPinExecution415.ts | 20 + .../cancelAdminTransitPinExecution422.ts | 20 + .../cancelAdminTransitPinExecution429.ts | 11 + .../cancelAdminTransitPinExecution500.ts | 20 + .../cancelAdminTransitPinExecution503.ts | 20 + .../cancelAdminTransitPinExecutionBody.ts | 9 + .../models/getAdminTransitCredentials200.ts | 2 + .../models/getAdminTransitCredentials404.ts | 20 + .../getAdminTransitCredentialsParams.ts | 9 + .../models/getAdminTransitPinExecution200.ts | 33 + ...tAdminTransitPinExecution200Diagnostics.ts | 9 + ...ansitPinExecution200DiagnosticsTailItem.ts | 8 + ...inExecution200DiagnosticsTailItemStream.ts | 10 + .../getAdminTransitPinExecution200Kind.ts | 23 + ...tAdminTransitPinExecution200Publication.ts | 11 + .../getAdminTransitPinExecution200Result.ts | 375 + .../getAdminTransitPinExecution200Status.ts | 16 + .../models/getAdminTransitPinExecution400.ts | 20 + .../models/getAdminTransitPinExecution401.ts | 20 + .../models/getAdminTransitPinExecution403.ts | 20 + .../models/getAdminTransitPinExecution404.ts | 20 + .../models/getAdminTransitPinExecution409.ts | 20 + .../models/getAdminTransitPinExecution413.ts | 20 + .../models/getAdminTransitPinExecution415.ts | 20 + .../models/getAdminTransitPinExecution422.ts | 20 + .../models/getAdminTransitPinExecution429.ts | 11 + .../models/getAdminTransitPinExecution500.ts | 20 + .../models/getAdminTransitPinExecution503.ts | 20 + .../admin/models/getAdminTransitPins200.ts | 27 + .../models/getAdminTransitPins200Active.ts | 13 + .../getAdminTransitPins200ActiveGbfs.ts | 22 + ...inTransitPins200ActiveGbfsSchemaVersion.ts | 9 + .../getAdminTransitPins200ActiveGbfsSource.ts | 9 + ...tAdminTransitPins200ActiveSchemaVersion.ts | 9 + .../getAdminTransitPins200ActiveTransitous.ts | 21 + ...ransitPins200ActiveTransitousSubmodules.ts | 4 + .../models/getAdminTransitPins200Diff.ts | 21 + .../getAdminTransitPins200DiffRowsItem.ts | 18 + ...getAdminTransitPins200DiffRowsItemAfter.ts | 7 + ...etAdminTransitPins200DiffRowsItemBefore.ts | 7 + ...etAdminTransitPins200DiffRowsItemChange.ts | 11 + ...minTransitPins200DiffRowsItemFieldsItem.ts | 14 + ...TransitPins200DiffRowsItemLicenseStatus.ts | 10 + .../models/getAdminTransitPins200Execution.ts | 33 + ...AdminTransitPins200ExecutionDiagnostics.ts | 9 + ...nsitPins200ExecutionDiagnosticsTailItem.ts | 8 + ...ns200ExecutionDiagnosticsTailItemStream.ts | 10 + .../getAdminTransitPins200ExecutionKind.ts | 23 + ...AdminTransitPins200ExecutionPublication.ts | 11 + .../getAdminTransitPins200ExecutionResult.ts | 375 + .../getAdminTransitPins200ExecutionStatus.ts | 16 + .../models/getAdminTransitPins200Page.ts | 19 + ...getAdminTransitPins200PageProposalsItem.ts | 24 + ...minTransitPins200PageProposalsItemState.ts | 14 + .../models/getAdminTransitPins200Proposal.ts | 18 + .../getAdminTransitPins200ProposalPins.ts | 13 + .../getAdminTransitPins200ProposalPinsGbfs.ts | 22 + ...sitPins200ProposalPinsGbfsSchemaVersion.ts | 9 + ...minTransitPins200ProposalPinsGbfsSource.ts | 9 + ...TransitPins200ProposalPinsSchemaVersion.ts | 9 + ...minTransitPins200ProposalPinsTransitous.ts | 21 + ...Pins200ProposalPinsTransitousSubmodules.ts | 4 + .../getAdminTransitPins200ProposalState.ts | 14 + .../getAdminTransitPins200Validation.ts | 78 + .../getAdminTransitPins200ValidationInputs.ts | 29 + ...nsitPins200ValidationInputsStagingState.ts | 14 + .../admin/models/getAdminTransitPins400.ts | 20 + .../admin/models/getAdminTransitPins401.ts | 20 + .../admin/models/getAdminTransitPins403.ts | 20 + .../admin/models/getAdminTransitPins404.ts | 20 + .../admin/models/getAdminTransitPins409.ts | 20 + .../admin/models/getAdminTransitPins413.ts | 20 + .../admin/models/getAdminTransitPins415.ts | 20 + .../admin/models/getAdminTransitPins422.ts | 20 + .../admin/models/getAdminTransitPins429.ts | 11 + .../admin/models/getAdminTransitPins500.ts | 20 + .../admin/models/getAdminTransitPins503.ts | 20 + .../admin/models/getAdminTransitPinsParams.ts | 19 + .../src/api/generated/admin/models/index.ts | 137 + ...iDataManagerTransitBumpTransitousRef200.ts | 4 + .../previewAdminTransitCredentials200.ts | 2 + .../previewAdminTransitCredentials404.ts | 20 + .../previewAdminTransitCredentialsBody.ts | 2 + .../models/proposeAdminTransitPins200.ts | 9 + .../models/proposeAdminTransitPins400.ts | 20 + .../models/proposeAdminTransitPins401.ts | 20 + .../models/proposeAdminTransitPins403.ts | 20 + .../models/proposeAdminTransitPins404.ts | 20 + .../models/proposeAdminTransitPins409.ts | 20 + .../models/proposeAdminTransitPins413.ts | 20 + .../models/proposeAdminTransitPins415.ts | 20 + .../models/proposeAdminTransitPins422.ts | 20 + .../models/proposeAdminTransitPins429.ts | 11 + .../models/proposeAdminTransitPins500.ts | 20 + .../models/proposeAdminTransitPins503.ts | 20 + .../models/proposeAdminTransitPinsBody.ts | 11 + .../models/proposeAdminTransitPinsHeaders.ts | 9 + .../models/validateAdminTransitPins202.ts | 7 + .../models/validateAdminTransitPins400.ts | 20 + .../models/validateAdminTransitPins401.ts | 20 + .../models/validateAdminTransitPins403.ts | 20 + .../models/validateAdminTransitPins404.ts | 20 + .../models/validateAdminTransitPins409.ts | 20 + .../models/validateAdminTransitPins413.ts | 20 + .../models/validateAdminTransitPins415.ts | 20 + .../models/validateAdminTransitPins422.ts | 20 + .../models/validateAdminTransitPins429.ts | 11 + .../models/validateAdminTransitPins500.ts | 20 + .../models/validateAdminTransitPins503.ts | 20 + .../models/validateAdminTransitPinsBody.ts | 10 + .../models/validateAdminTransitPinsHeaders.ts | 9 + .../admin/operation-bindings.generated.ts | 42 + .../catalog-control-edges.test.ts | 42 + .../src/build-artifacts/control-authority.ts | 6 +- .../src/build-artifacts/copy-tree.test.ts | 159 + .../core/src/build-artifacts/copy-tree.ts | 391 + .../data-execution-candidate.ts | 121 +- .../build-artifacts/data-execution-journal.ts | 232 +- .../src/build-artifacts/peer-admission.ts | 6 + .../core/src/build-artifacts/target-lease.ts | 97 + packages/core/src/catalog-pins/index.ts | 4 + .../core/src/catalog-pins/proposal-schemas.ts | 38 + packages/core/src/catalog-pins/proposals.ts | 175 + .../core/src/catalog-pins/record-files.ts | 132 + .../core/src/catalog-pins/review-records.ts | 45 + .../core/src/catalog-pins/review-schemas.ts | 147 + packages/core/src/ops/contract.test.ts | 25 +- packages/core/src/ops/contract.ts | 42 +- packages/core/src/ops/feed-credential-map.ts | 4 + packages/core/src/ops/index.ts | 5 +- .../contracts/catalog-review.test.ts | 29 + .../data-manager/contracts/catalog-review.ts | 85 + .../data-manager/contracts/executions.ts | 28 +- .../services/data-manager/contracts/index.ts | 1 + .../data-manager/contracts/operations.ts | 8 + .../data-manager/contracts/transit.ts | 3 + .../generated/data-manager.generated.ts | 103 + .../models/cancelDataExecution200Kind.ts | 2 + .../models/cancelDataExecution200Result.ts | 20 + ...rInspectCatalogReviewResponse200Schema0.ts | 10 + ...rInspectCatalogReviewResponse200Schema1.ts | 10 + .../models/getDataExecution200Kind.ts | 2 + .../models/getDataExecution200Result.ts | 20 + .../data-manager/generated/models/index.ts | 49 + .../models/inspectCatalogReview200.ts | 27 + .../models/inspectCatalogReview200Active.ts | 13 + .../inspectCatalogReview200ActiveGbfs.ts | 22 + ...CatalogReview200ActiveGbfsSchemaVersion.ts | 9 + ...inspectCatalogReview200ActiveGbfsSource.ts | 9 + ...pectCatalogReview200ActiveSchemaVersion.ts | 9 + ...inspectCatalogReview200ActiveTransitous.ts | 21 + ...alogReview200ActiveTransitousSubmodules.ts | 4 + .../models/inspectCatalogReview200Diff.ts | 21 + .../inspectCatalogReview200DiffRowsItem.ts | 18 + ...nspectCatalogReview200DiffRowsItemAfter.ts | 7 + ...spectCatalogReview200DiffRowsItemBefore.ts | 7 + ...spectCatalogReview200DiffRowsItemChange.ts | 11 + ...tCatalogReview200DiffRowsItemFieldsItem.ts | 14 + ...talogReview200DiffRowsItemLicenseStatus.ts | 10 + .../inspectCatalogReview200Execution.ts | 33 + ...ectCatalogReview200ExecutionDiagnostics.ts | 9 + ...ogReview200ExecutionDiagnosticsTailItem.ts | 8 + ...ew200ExecutionDiagnosticsTailItemStream.ts | 10 + .../inspectCatalogReview200ExecutionKind.ts | 23 + ...ectCatalogReview200ExecutionPublication.ts | 11 + .../inspectCatalogReview200ExecutionResult.ts | 375 + .../inspectCatalogReview200ExecutionStatus.ts | 16 + .../models/inspectCatalogReview200Page.ts | 19 + ...nspectCatalogReview200PageProposalsItem.ts | 24 + ...tCatalogReview200PageProposalsItemState.ts | 14 + .../models/inspectCatalogReview200Proposal.ts | 18 + .../inspectCatalogReview200ProposalPins.ts | 13 + ...inspectCatalogReview200ProposalPinsGbfs.ts | 22 + ...gReview200ProposalPinsGbfsSchemaVersion.ts | 9 + ...tCatalogReview200ProposalPinsGbfsSource.ts | 9 + ...talogReview200ProposalPinsSchemaVersion.ts | 9 + ...tCatalogReview200ProposalPinsTransitous.ts | 21 + ...view200ProposalPinsTransitousSubmodules.ts | 4 + .../inspectCatalogReview200ProposalState.ts | 14 + .../inspectCatalogReview200Validation.ts | 78 + ...inspectCatalogReview200ValidationInputs.ts | 29 + ...ogReview200ValidationInputsStagingState.ts | 14 + .../models/inspectCatalogReview400.ts | 20 + .../models/inspectCatalogReview401.ts | 20 + .../models/inspectCatalogReview409.ts | 20 + .../models/inspectCatalogReview413.ts | 20 + .../models/inspectCatalogReview415.ts | 20 + .../models/inspectCatalogReview429.ts | 20 + .../models/inspectCatalogReview500.ts | 20 + .../models/inspectCatalogReview503.ts | 20 + .../models/inspectCatalogReviewParams.ts | 19 + .../models/lookupDataExecution200Kind.ts | 2 + .../models/lookupDataExecution200Result.ts | 20 + .../generated/models/proposeTransitBump200.ts | 4 + .../models/proposeTransitBumpBody.ts | 2 + .../models/startDataExecution202Kind.ts | 2 + .../models/startDataExecution202Result.ts | 20 + .../models/startDataExecutionBodyInput.ts | 21 + ...tartGenerationHardlinksExecution202Kind.ts | 2 + ...rtGenerationHardlinksExecution202Result.ts | 20 + .../generated/operation-bindings.generated.ts | 7 + .../__tests__/catalog-diff.test.ts | 270 + .../feed-credential-requirements.test.ts | 29 + .../__tests__/pinned-feed-credentials.test.ts | 49 + packages/transitous-core/src/catalog-diff.ts | 409 + .../transitous-core/src/catalog-resolution.ts | 116 + .../src/feed-credential-requirements.ts | 20 +- packages/transitous-core/src/index.ts | 29 + .../transitous-core/src/operations-profile.ts | 185 + .../src/pinned-feed-credentials.ts | 39 +- scripts/database-test-groups.json | 5 +- scripts/ops-authority-inventory.json | 1 + .../__tests__/api-transit-bump.test.ts | 8 +- services/data-manager/__tests__/cron.test.ts | 95 +- .../fixtures/catalog-data-manager-process.mjs | 90 + .../__tests__/ops-client-ownership.test.ts | 31 +- .../transitous/assemble-staging.test.ts | 9 + .../__tests__/transitous/candidate.test.ts | 19 +- .../transitous/catalog-review.test.ts | 385 + .../catalog-validation-current.test.ts | 177 + .../catalog-workflow-postgres.test.ts | 970 ++ .../__tests__/transitous/fixtures/tiny-osm.ts | 3 + .../transitous/pipeline-live-motis.test.ts | 5 +- .../__tests__/transitous/pipeline.test.ts | 30 + .../__tests__/transitous/promote.test.ts | 13 +- services/data-manager/openapi.json | 3446 ++++++- services/data-manager/src/api.ts | 111 +- services/data-manager/src/cron.ts | 50 +- .../catalog-child-reservations.test.ts | 62 + .../src/jobs/execution/dispatch.ts | 52 + .../src/jobs/execution/ops-children.test.ts | 1 + .../src/jobs/execution/ops-children.ts | 104 +- .../src/jobs/execution/receiver.ts | 2 +- .../src/jobs/execution/runtime.ts | 96 +- .../src/jobs/transitous/assemble-staging.ts | 19 +- .../src/jobs/transitous/candidate.ts | 196 +- .../transitous/catalog-activation-record.ts | 100 + .../src/jobs/transitous/catalog-activation.ts | 193 + .../src/jobs/transitous/catalog-review.ts | 399 + .../transitous/catalog-rollback-retention.ts | 211 + .../src/jobs/transitous/catalog-selection.ts | 93 + .../transitous/catalog-validation-current.ts | 77 + .../src/jobs/transitous/catalog-validation.ts | 246 + .../src/jobs/transitous/isolate-producer.ts | 61 + .../src/jobs/transitous/motis-health.ts | 6 +- .../src/jobs/transitous/motis-import.ts | 22 +- .../src/jobs/transitous/operations-profile.ts | 196 +- .../src/jobs/transitous/pipeline.ts | 13 +- .../src/jobs/transitous/prepare.ts | 21 +- .../src/jobs/transitous/promote.ts | 10 +- .../src/jobs/transitous/proxy-transaction.ts | 4 +- .../src/jobs/transitous/slot-state.ts | 23 + .../data-manager/src/jobs/transitous/types.ts | 1 + services/data-manager/src/service.ts | 10 + 330 files changed, 28087 insertions(+), 2620 deletions(-) create mode 100644 apps/api/src/routes/__tests__/admin-transit-pins.test.ts create mode 100644 apps/api/src/routes/admin-transit-pins.ts create mode 100644 apps/api/src/services/activity-catalog-postgres.test.ts create mode 100644 apps/api/src/services/transit-pin-requests.ts create mode 100644 apps/ops-agent/src/catalog-approval-runtime.test.ts create mode 100644 apps/ops-agent/src/catalog-approval-runtime.ts create mode 100644 apps/ops-agent/src/catalog-validation-inputs.ts create mode 100644 apps/web/src/app/admin/(full-admin)/transit/__tests__/LockfileCard.test.tsx create mode 100644 apps/web/src/lib/admin/transitPinHooks.ts create mode 100644 packages/core/src/api/contracts/data-manager-admin/transit-pins.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins202.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins400.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins401.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins403.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins404.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins409.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins413.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins415.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins422.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins429.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins500.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPins503.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPinsBody.ts create mode 100644 packages/core/src/api/generated/admin/models/activateAdminTransitPinsHeaders.ts create mode 100644 packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema0.ts create mode 100644 packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema1.ts create mode 100644 packages/core/src/api/generated/admin/models/applyAdminTransitCredentials404.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Diagnostics.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItem.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItemStream.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Kind.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Publication.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Result.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Status.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution400.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution401.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution403.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution404.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution409.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution413.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution415.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution422.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution429.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution500.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution503.ts create mode 100644 packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecutionBody.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentials404.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitCredentialsParams.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Diagnostics.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItem.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItemStream.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Kind.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Publication.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Result.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Status.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution400.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution401.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution403.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution404.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution409.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution413.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution415.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution422.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution429.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution500.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinExecution503.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200Active.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfs.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSchemaVersion.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSource.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveSchemaVersion.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitous.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitousSubmodules.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200Diff.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItem.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemAfter.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemBefore.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemChange.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemFieldsItem.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemLicenseStatus.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200Execution.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnostics.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItem.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItemStream.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionKind.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionPublication.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionResult.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionStatus.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200Page.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItem.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItemState.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200Proposal.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPins.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfs.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSchemaVersion.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSource.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsSchemaVersion.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitous.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitousSubmodules.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalState.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200Validation.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputs.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputsStagingState.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins400.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins401.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins403.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins404.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins409.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins413.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins415.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins422.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins429.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins500.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPins503.ts create mode 100644 packages/core/src/api/generated/admin/models/getAdminTransitPinsParams.ts create mode 100644 packages/core/src/api/generated/admin/models/previewAdminTransitCredentials404.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins200.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins400.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins401.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins403.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins404.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins409.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins413.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins415.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins422.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins429.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins500.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPins503.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPinsBody.ts create mode 100644 packages/core/src/api/generated/admin/models/proposeAdminTransitPinsHeaders.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins202.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins400.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins401.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins403.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins404.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins409.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins413.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins415.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins422.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins429.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins500.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPins503.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPinsBody.ts create mode 100644 packages/core/src/api/generated/admin/models/validateAdminTransitPinsHeaders.ts create mode 100644 packages/core/src/build-artifacts/catalog-control-edges.test.ts create mode 100644 packages/core/src/build-artifacts/copy-tree.test.ts create mode 100644 packages/core/src/build-artifacts/copy-tree.ts create mode 100644 packages/core/src/catalog-pins/proposal-schemas.ts create mode 100644 packages/core/src/catalog-pins/proposals.ts create mode 100644 packages/core/src/catalog-pins/record-files.ts create mode 100644 packages/core/src/catalog-pins/review-records.ts create mode 100644 packages/core/src/catalog-pins/review-schemas.ts create mode 100644 packages/core/src/services/data-manager/contracts/catalog-review.test.ts create mode 100644 packages/core/src/services/data-manager/contracts/catalog-review.ts create mode 100644 packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema0.ts create mode 100644 packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema1.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Active.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfs.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSchemaVersion.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSource.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveSchemaVersion.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitous.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitousSubmodules.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Diff.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemAfter.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemBefore.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemChange.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemFieldsItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemLicenseStatus.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Execution.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnostics.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItemStream.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionKind.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionPublication.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionResult.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionStatus.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Page.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItem.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItemState.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Proposal.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPins.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfs.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSchemaVersion.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSource.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsSchemaVersion.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitous.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitousSubmodules.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalState.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Validation.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputs.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputsStagingState.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview409.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReview503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectCatalogReviewParams.ts create mode 100644 packages/transitous-core/__tests__/catalog-diff.test.ts create mode 100644 packages/transitous-core/src/catalog-diff.ts create mode 100644 packages/transitous-core/src/catalog-resolution.ts create mode 100644 packages/transitous-core/src/operations-profile.ts create mode 100644 services/data-manager/__tests__/fixtures/catalog-data-manager-process.mjs create mode 100644 services/data-manager/__tests__/transitous/catalog-review.test.ts create mode 100644 services/data-manager/__tests__/transitous/catalog-validation-current.test.ts create mode 100644 services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts create mode 100644 services/data-manager/__tests__/transitous/fixtures/tiny-osm.ts create mode 100644 services/data-manager/src/jobs/execution/catalog-child-reservations.test.ts create mode 100644 services/data-manager/src/jobs/transitous/catalog-activation-record.ts create mode 100644 services/data-manager/src/jobs/transitous/catalog-activation.ts create mode 100644 services/data-manager/src/jobs/transitous/catalog-review.ts create mode 100644 services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts create mode 100644 services/data-manager/src/jobs/transitous/catalog-selection.ts create mode 100644 services/data-manager/src/jobs/transitous/catalog-validation-current.ts create mode 100644 services/data-manager/src/jobs/transitous/catalog-validation.ts create mode 100644 services/data-manager/src/jobs/transitous/isolate-producer.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 1ca998e6d..b87b4577a 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -1279,6 +1279,62 @@ } ] }, + "Admin_getAdminTransitPins_200___schema0": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/Admin_getAdminTransitPins_200___schema0" + } + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/Admin_getAdminTransitPins_200___schema0" + } + } + ] + }, + "Admin_getAdminTransitPins_200___schema1": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/Admin_getAdminTransitPins_200___schema1" + } + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/Admin_getAdminTransitPins_200___schema1" + } + } + ] + }, "Admin_previewAdminOperation_200___schema0": { "anyOf": [ { @@ -59929,6 +59985,18 @@ "/api/admin/transit/credentials": { "get": { "operationId": "getAdminTransitCredentials", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "query", + "name": "proposalId", + "required": false + } + ], "responses": { "200": { "description": "Response", @@ -59941,6 +60009,11 @@ "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, "credentialsRevision": { "type": "string", "pattern": "^[a-f0-9]{64}$" @@ -60215,7 +60288,7 @@ } } }, - "409": { + "404": { "description": "Response", "content": { "application/json": { @@ -60275,7 +60348,7 @@ } } }, - "413": { + "409": { "description": "Response", "content": { "application/json": { @@ -60335,7 +60408,7 @@ } } }, - "415": { + "413": { "description": "Response", "content": { "application/json": { @@ -60395,40 +60468,7 @@ } } }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^\\d+$" - } - } - }, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "retryAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "error", - "retryAfter" - ], - "additionalProperties": false - } - } - } - }, - "500": { + "415": { "description": "Response", "content": { "application/json": { @@ -60488,7 +60528,40 @@ } } }, - "503": { + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { "description": "Response", "content": { "application/json": { @@ -60547,207 +60620,7913 @@ } } } - } - }, - "tags": [ - "data-manager-admin" - ], - "x-openmapx-auth": "admin" - }, - "patch": { - "operationId": "applyAdminTransitCredentials", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "previewId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "intentFingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "operationKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "transitProofToken": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - } - }, - "required": [ - "previewId", - "intentFingerprint", - "operationKey", - "transitProofToken" - ], - "additionalProperties": false - } - } - } - }, - "responses": { - "200": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "credentialsRevision": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "configuration": { - "anyOf": [ - { - "type": "object", - "properties": { - "outcome": { - "type": "string", - "enum": [ - "committed" - ] - }, - "revisionId": { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - "enabledServiceIds": { - "maxItems": 256, - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9][a-z0-9-]*$" - } - }, - "followUp": { - "type": "string", - "enum": [ - "complete", - "failed" - ] - }, - "cleanupFailed": { - "type": "boolean" - }, - "publisherCleanupFailed": { - "type": "boolean" - }, - "registryUpdateFailed": { - "type": "boolean" - } - }, - "required": [ - "outcome", - "revisionId", - "enabledServiceIds", - "followUp", - "cleanupFailed", - "publisherCleanupFailed", - "registryUpdateFailed" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "outcome": { - "type": "string", - "enum": [ - "not_committed" - ] - }, - "reason": { - "type": "string", - "enum": [ - "conflict", - "busy", - "failed" - ] - }, - "currentRevisionId": { - "anyOf": [ - { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - { - "type": "null" - } - ] - }, - "cleanupFailed": { - "type": "boolean" - }, - "publisherCleanupFailed": { - "type": "boolean" - }, - "registryUpdateFailed": { - "type": "boolean" - } - }, - "required": [ - "outcome", - "reason", - "cleanupFailed", - "publisherCleanupFailed", - "registryUpdateFailed" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "outcome": { - "type": "string", - "enum": [ - "unknown" - ] - }, - "reason": { - "type": "string", - "enum": [ - "transport_or_result" - ] - }, - "publisherCleanupFailed": { - "type": "boolean" - }, - "registryUpdateFailed": { - "type": "boolean" - } - }, - "required": [ - "outcome", - "reason", - "publisherCleanupFailed", - "registryUpdateFailed" - ], - "additionalProperties": false - } - ] - } - }, - "required": [ - "credentialsRevision", - "configuration" - ], - "additionalProperties": false - } - } - } }, - "400": { + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + }, + "patch": { + "operationId": "applyAdminTransitCredentials", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "previewId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "intentFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "transitProofToken": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "previewId", + "intentFingerprint", + "operationKey", + "transitProofToken" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configuration": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "committed" + ] + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "enabledServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9-]*$" + } + }, + "followUp": { + "type": "string", + "enum": [ + "complete", + "failed" + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "revisionId", + "enabledServiceIds", + "followUp", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "not_committed" + ] + }, + "reason": { + "type": "string", + "enum": [ + "conflict", + "busy", + "failed" + ] + }, + "currentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "cleanupFailed": { + "type": "boolean" + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "cleanupFailed", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "reason": { + "type": "string", + "enum": [ + "transport_or_result" + ] + }, + "publisherCleanupFailed": { + "type": "boolean" + }, + "registryUpdateFailed": { + "type": "boolean" + } + }, + "required": [ + "outcome", + "reason", + "publisherCleanupFailed", + "registryUpdateFailed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "credentialsRevision", + "configuration" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/transit/credentials/preview": { + "post": { + "operationId": "previewAdminTransitCredentials", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedCredentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "changes": { + "minItems": 1, + "maxItems": 100, + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "action": { + "type": "string", + "enum": [ + "set" + ] + }, + "value": { + "type": "string", + "maxLength": 8192 + } + }, + "required": [ + "key", + "action", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "action": { + "type": "string", + "enum": [ + "delete" + ] + } + }, + "required": [ + "key", + "action" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "catalogRevisionId", + "expectedCredentialsRevision", + "expectedCurrentRevisionId", + "operationKey", + "changes" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "previewId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "intentFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "changes": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "action": { + "type": "string", + "enum": [ + "set", + "delete" + ] + } + }, + "required": [ + "key", + "action" + ], + "additionalProperties": false + } + }, + "preservedCount": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "mapBytes": { + "type": "integer", + "minimum": 2, + "maximum": 65536 + } + }, + "required": [ + "previewId", + "intentFingerprint", + "catalogRevisionId", + "credentialsRevision", + "expectedCurrentRevisionId", + "expiresAt", + "changes", + "preservedCount", + "mapBytes" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/transit/pins": { + "get": { + "operationId": "getAdminTransitPins", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "query", + "name": "proposalId", + "required": false + }, + { + "schema": { + "default": 0, + "type": "integer", + "minimum": 0, + "maximum": 4096 + }, + "in": "query", + "name": "offset", + "required": false + }, + { + "schema": { + "default": 0, + "type": "integer", + "minimum": 0, + "maximum": 200000 + }, + "in": "query", + "name": "diffOffset", + "required": false + } + ], + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "active": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "generationId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transitous": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$" + }, + "submodules": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "additionalProperties": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "comment": { + "type": "string", + "maxLength": 512 + } + }, + "required": [ + "ref", + "submodules", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + }, + "gbfs": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "source": { + "type": "string", + "enum": [ + "mobilitydata-gbfs" + ] + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "url": { + "type": "string", + "maxLength": 256 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "required": [ + "schemaVersion", + "source", + "commit", + "url", + "sha256", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "generationId", + "transitous", + "gbfs" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "allowedCountries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{2,3}$" + } + }, + "catalogRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "page": { + "type": "object", + "properties": { + "proposals": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "baseGenerationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "string", + "enum": [ + "proposed", + "validating", + "validated", + "rejected", + "activating", + "activated" + ] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transitousRef": { + "type": "string", + "maxLength": 256 + }, + "atlasCommit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "gbfsCommit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "proposalId", + "baseGenerationId", + "diffDigest", + "sourcePolicyRevision", + "state", + "createdAt", + "catalogRevisionId", + "transitousRef", + "atlasCommit", + "gbfsCommit" + ], + "additionalProperties": false + } + }, + "total": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "nextOffset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "proposals", + "total", + "offset", + "nextOffset" + ], + "additionalProperties": false + }, + "proposal": { + "anyOf": [ + { + "type": "object", + "properties": { + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "baseGenerationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "pins": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "generationId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transitous": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$" + }, + "submodules": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "additionalProperties": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "comment": { + "type": "string", + "maxLength": 512 + } + }, + "required": [ + "ref", + "submodules", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + }, + "gbfs": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "source": { + "type": "string", + "enum": [ + "mobilitydata-gbfs" + ] + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "url": { + "type": "string", + "maxLength": 256 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "required": [ + "schemaVersion", + "source", + "commit", + "url", + "sha256", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "generationId", + "transitous", + "gbfs" + ], + "additionalProperties": false + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "string", + "enum": [ + "proposed", + "validating", + "validated", + "rejected", + "activating", + "activated" + ] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "proposalId", + "baseGenerationId", + "pins", + "diffDigest", + "sourcePolicyRevision", + "state", + "createdAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "diff": { + "anyOf": [ + { + "type": "object", + "properties": { + "digest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "total": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rows": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "region": { + "type": "string" + }, + "source": { + "type": "string" + }, + "change": { + "type": "string", + "enum": [ + "added", + "removed", + "changed" + ] + }, + "fields": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "endpoint", + "origin", + "license", + "attribution", + "eligibility", + "metadata" + ] + } + }, + "licenseStatus": { + "type": "string", + "enum": [ + "declared", + "unknown" + ] + }, + "before": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/Admin_getAdminTransitPins_200___schema0" + } + }, + { + "type": "null" + } + ] + }, + "after": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/Admin_getAdminTransitPins_200___schema1" + } + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "id", + "region", + "source", + "change", + "fields", + "licenseStatus", + "before", + "after" + ], + "additionalProperties": false + } + }, + "nextOffset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "digest", + "total", + "offset", + "rows", + "nextOffset" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "validation": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "properties": { + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "pinDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "overlayDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{2,3}$" + } + }, + "completedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "slot": { + "type": "string", + "enum": [ + "A", + "B" + ] + }, + "epoch": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "manifestHash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "imageDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed" + ] + } + }, + "required": [ + "validationId", + "proposalId", + "pinDigest", + "diffDigest", + "sourcePolicyRevision", + "overlayDigest", + "credentialsRevision", + "countries", + "completedAt", + "expiresAt", + "slot", + "epoch", + "manifestHash", + "imageDigest", + "status" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "pinDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "overlayDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{2,3}$" + } + }, + "completedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "status": { + "type": "string", + "enum": [ + "failed" + ] + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "candidate": { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "A", + "B" + ] + }, + "epoch": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "manifestHash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "imageDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "required": [ + "slot", + "epoch", + "manifestHash", + "imageDigest" + ], + "additionalProperties": false + } + }, + "required": [ + "validationId", + "proposalId", + "pinDigest", + "diffDigest", + "sourcePolicyRevision", + "overlayDigest", + "credentialsRevision", + "countries", + "completedAt", + "expiresAt", + "status", + "reason" + ], + "additionalProperties": false + } + ] + }, + { + "type": "null" + } + ] + }, + "execution": { + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "catalog.validate", + "catalog.activate", + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "validationInputs": { + "anyOf": [ + { + "type": "object", + "properties": { + "configurationRevisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "overlayDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "imageDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "stagingState": { + "type": "string", + "enum": [ + "running", + "stopped", + "created", + "restarting", + "paused", + "unknown" + ] + }, + "requiredKeys": { + "maxItems": 1000, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + }, + "configuredKeys": { + "maxItems": 10000, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + } + }, + "required": [ + "configurationRevisionId", + "credentialsRevision", + "sourcePolicyRevision", + "overlayDigest", + "imageDigest", + "stagingState", + "requiredKeys", + "configuredKeys" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "validationEligible": { + "type": "boolean" + }, + "blocker": { + "anyOf": [ + { + "type": "string", + "maxLength": 2048 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "active", + "allowedCountries", + "catalogRevisionId", + "page", + "proposal", + "diff", + "validation", + "execution", + "validationInputs", + "validationEligible", + "blocker" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/transit/pins/executions/{id}": { + "get": { + "operationId": "getAdminTransitPinExecution", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "id", + "required": true + } + ], + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "catalog.validate", + "catalog.activate", + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + }, + "delete": { + "operationId": "cancelAdminTransitPinExecution", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "id", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "operationKey", + "fingerprint" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "catalog.validate", + "catalog.activate", + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/transit/pins/proposals": { + "post": { + "operationId": "proposeAdminTransitPins", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^opk1_t1_[0-9a-f]{14}_[A-Za-z0-9_-]{22}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "branch": { + "default": "main", + "type": "string", + "minLength": 1, + "maxLength": 215, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*$" + } + }, + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "generationId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "proposalId", + "generationId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/transit/pins/proposals/{proposalId}/activate": { + "post": { + "operationId": "activateAdminTransitPins", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "proposalId", + "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^opk1_t1_[0-9a-f]{14}_[A-Za-z0-9_-]{22}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedGenerationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "acknowledgeLicenseChanges": { + "type": "boolean" + } + }, + "required": [ + "validationId", + "expectedGenerationId", + "diffDigest", + "acknowledgeLicenseChanges" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "jobId" + ], + "additionalProperties": false + } + } + } + }, + "400": { "description": "Response", "content": { "application/json": { @@ -60927,6 +68706,66 @@ } } }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "409": { "description": "Response", "content": { @@ -61107,6 +68946,66 @@ } } }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, "429": { "description": "Response", "headers": { @@ -61259,211 +69158,205 @@ } } } - } - }, - "tags": [ - "data-manager-admin" - ], - "x-openmapx-auth": "admin" - } - }, - "/api/admin/transit/credentials/preview": { - "post": { - "operationId": "previewAdminTransitCredentials", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "catalogRevisionId": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "expectedCredentialsRevision": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "expectedCurrentRevisionId": { - "anyOf": [ - { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - { - "type": "null" - } - ] - }, - "operationKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "changes": { - "minItems": 1, - "maxItems": 100, - "type": "array", - "items": { - "oneOf": [ - { - "type": "object", - "properties": { - "key": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "action": { - "type": "string", - "enum": [ - "set" - ] - }, - "value": { - "type": "string", - "maxLength": 8192 - } - }, - "required": [ - "key", - "action", - "value" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "key": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "action": { - "type": "string", - "enum": [ - "delete" - ] - } - }, - "required": [ - "key", - "action" - ], - "additionalProperties": false - } - ] - } - } - }, - "required": [ - "catalogRevisionId", - "expectedCredentialsRevision", - "expectedCurrentRevisionId", - "operationKey", - "changes" - ], - "additionalProperties": false - } - } - } - }, - "responses": { - "200": { + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/transit/pins/proposals/{proposalId}/validate": { + "post": { + "operationId": "validateAdminTransitPins", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "proposalId", + "required": true + }, + { + "schema": { + "type": "string", + "pattern": "^opk1_t1_[0-9a-f]{14}_[A-Za-z0-9_-]{22}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "countries" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "jobId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { "description": "Response", "content": { "application/json": { "schema": { - "type": "object", - "properties": { - "previewId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "intentFingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "catalogRevisionId": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "credentialsRevision": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "expectedCurrentRevisionId": { - "anyOf": [ - { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" }, - { - "type": "null" + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" } - ] - }, - "expiresAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "required": [ + "error" + ], + "additionalProperties": false }, - "changes": { - "maxItems": 100, - "type": "array", - "items": { - "type": "object", - "properties": { - "key": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "action": { - "type": "string", - "enum": [ - "set", - "delete" - ] - } + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 }, - "required": [ - "key", - "action" - ], - "additionalProperties": false - } - }, - "preservedCount": { - "type": "integer", - "minimum": 0, - "maximum": 10000 - }, - "mapBytes": { - "type": "integer", - "minimum": 2, - "maximum": 65536 + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false } - }, - "required": [ - "previewId", - "intentFingerprint", - "catalogRevisionId", - "credentialsRevision", - "expectedCurrentRevisionId", - "expiresAt", - "changes", - "preservedCount", - "mapBytes" - ], - "additionalProperties": false + ] } } } }, - "400": { + "403": { "description": "Response", "content": { "application/json": { @@ -61523,7 +69416,7 @@ } } }, - "401": { + "404": { "description": "Response", "content": { "application/json": { @@ -61583,7 +69476,7 @@ } } }, - "403": { + "409": { "description": "Response", "content": { "application/json": { @@ -61643,7 +69536,7 @@ } } }, - "409": { + "413": { "description": "Response", "content": { "application/json": { @@ -61703,7 +69596,7 @@ } } }, - "413": { + "415": { "description": "Response", "content": { "application/json": { @@ -61763,7 +69656,7 @@ } } }, - "415": { + "422": { "description": "Response", "content": { "application/json": { @@ -66501,6 +74394,15 @@ true ] }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "ref": { "type": "string" }, @@ -66526,6 +74428,8 @@ "ok", "unchanged", "proposed", + "proposalId", + "catalogRevisionId", "ref", "previousRef", "submoduleSha", diff --git a/apps/api/src/routes/__tests__/admin-transit-credentials.test.ts b/apps/api/src/routes/__tests__/admin-transit-credentials.test.ts index fd7a77fb5..d7eb9f26d 100644 --- a/apps/api/src/routes/__tests__/admin-transit-credentials.test.ts +++ b/apps/api/src/routes/__tests__/admin-transit-credentials.test.ts @@ -12,6 +12,7 @@ import { mockAdminSession } from "../../test/auth"; const mocks = vi.hoisted(() => ({ session: null as unknown, + proposalId: "00000000-0000-4000-8000-000000000001", inspect: vi.fn(), preview: vi.fn(), bind: vi.fn(), @@ -31,7 +32,10 @@ vi.mock("../../services/job-runner", () => ({ vi.mock("../../services/ops-client", () => ({ createApiOpsClient: () => ({}), createDurableOpsKey: () => `opk1_${"a".repeat(32)}`, - executeAndWait: async () => ({ catalogRevisionId: "a".repeat(64) }), + executeAndWait: async (_client: unknown, operation: { kind: string }) => + operation.kind === "transitousLock.inspect" + ? { proposal: { proposalId: mocks.proposalId, pins: { generationId: "e".repeat(64) } } } + : { catalogRevisionId: "a".repeat(64) }, })); import { ADMIN_REAUTH_COOKIE_NAME } from "../../account/admin-reauthentication"; @@ -95,7 +99,9 @@ afterEach(async () => { vi.unstubAllEnvs(); }); it("uses the generated SDK over real HTTP and serializes only masked metadata and reviewed key actions", async () => { - expect(unwrapAdminResponse(await getAdminTransitCredentials({ client }))).toMatchObject({ + expect( + unwrapAdminResponse(await getAdminTransitCredentials(undefined, { client })), + ).toMatchObject({ configurationRevisionId: null, owner: "unset", }); @@ -229,3 +235,14 @@ it("bounds the actual parser and shared per-admin mutation rate", async () => { ).statusCode, ).toBe(429); }); + +it("selects credentials for the exact verified proposed pair before activation", async () => { + const response = await fetch( + `http://127.0.0.1:${(app.server.address() as import("node:net").AddressInfo).port}/api/admin/transit/credentials?proposalId=${mocks.proposalId}`, + ); + expect(response.status).toBe(200); + expect(mocks.inspect).toHaveBeenCalledWith({ + catalogRevisionId: "e".repeat(64), + proposalId: mocks.proposalId, + }); +}); diff --git a/apps/api/src/routes/__tests__/admin-transit-pins.test.ts b/apps/api/src/routes/__tests__/admin-transit-pins.test.ts new file mode 100644 index 000000000..d9647a355 --- /dev/null +++ b/apps/api/src/routes/__tests__/admin-transit-pins.test.ts @@ -0,0 +1,255 @@ +/** BFF/generated-SDK transport/auth proof. DM HTTP is an I/O fixture, not a signed host authority. */ +import { createHash, randomUUID } from "node:crypto"; +import { createApiClient } from "@openmapx/core"; +import * as sdk from "@openmapx/core/api-admin-sdk"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; +import { + type DataExecutionStatus, + normalizeDataExecutionInput, + startDataExecutionSchema, +} from "@openmapx/core/data-manager-contracts"; +import { + canonicalJson, + createControlRootOperationKey, + executionFingerprint, +} from "@openmapx/core/ops"; +import Fastify, { type FastifyInstance } from "fastify"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { mockAdminSession } from "../../test/auth"; + +const state = vi.hoisted(() => ({ session: null as unknown, audit: vi.fn() })); +vi.mock("../../auth", () => ({ auth: { api: { getSession: async () => state.session } } })); +vi.mock("../../utils/audit-log", () => ({ writeAuditLog: state.audit })); + +import { adminTransitPinsRoute } from "../admin-transit-pins"; + +let app: FastifyInstance, + backend: FastifyInstance, + client: ReturnType, + posts = 0, + loseNext = false; +const rows = new Map(), + proposals = new Map }>(); +function proposal(id: string, branch: string) { + const lockedAt = new Date().toISOString(), + pins = createCatalogPinSet( + { + ref: `${branch}@${"a".repeat(40)}`, + submodules: { "transitland-atlas": "b".repeat(40) }, + lockedAt, + lockedBy: "fixture", + }, + { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "c".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"c".repeat(40)}/systems.csv`, + sha256: "d".repeat(64), + lockedAt, + lockedBy: "fixture", + }, + ); + return { + proposalId: id, + baseGenerationId: null, + pins, + diffDigest: "e".repeat(64), + sourcePolicyRevision: "f".repeat(64), + state: "proposed" as const, + createdAt: lockedAt, + }; +} +const key = () => + createControlRootOperationKey({ + notAfter: Date.now() + 120000, + nonce: crypto.getRandomValues(new Uint8Array(16)), + }); +beforeEach(async () => { + vi.stubEnv("OPENMAPX_DISABLE_LOCALHOST_AUTH", "1"); + vi.stubEnv("DATA_MANAGER_AUTH_TOKEN", "component-service-token"); + state.session = mockAdminSession(); + state.audit.mockReset(); + rows.clear(); + proposals.clear(); + posts = 0; + loseNext = false; + backend = Fastify({ forceCloseConnections: true }); + backend.addHook("preHandler", async (request, reply) => { + if (request.headers.authorization !== "Bearer component-service-token") + return reply.code(401).send({ error: "authentication" }); + }); + backend.get<{ Params: { operationKey: string } }>( + "/executions/by-key/:operationKey", + async (request, reply) => + rows.get(request.params.operationKey) ?? reply.code(404).send({ error: "unknown" }), + ); + backend.get<{ Params: { id: string } }>( + "/executions/:id", + async (request, reply) => + [...rows.values()].find((row) => row.id === request.params.id) ?? + reply.code(404).send({ error: "unknown" }), + ); + backend.post("/executions", async (request, reply) => { + posts++; + const body = startDataExecutionSchema.parse(request.body), + input = normalizeDataExecutionInput(body.input), + deadlineAt = body.requestedDeadlineAt!; + const row: DataExecutionStatus = { + id: randomUUID(), + kind: input.kind, + operationKey: body.operationKey, + fingerprint: executionFingerprint({ + kind: input.kind, + operationFingerprint: createHash("sha256").update(canonicalJson(input)).digest("hex"), + deadlineAt, + }), + admittedAt: Date.now(), + deadlineAt, + status: "queued", + progress: null, + diagnostics: { tail: [], truncated: false }, + publication: "not_started", + cancellationRequestedAt: null, + }; + rows.set(row.operationKey, row); + if (loseNext) { + loseNext = false; + reply.hijack(); + reply.raw.destroy(); + return; + } + return reply.code(202).send(row); + }); + backend.post<{ Params: { id: string } }>("/executions/:id/cancel", async (request, reply) => { + const row = [...rows.values()].find((value) => value.id === request.params.id); + if (!row) return reply.code(404).send({ error: "unknown" }); + const canceled = { + ...row, + status: "cancel_pending" as const, + cancellationRequestedAt: new Date().toISOString(), + }; + rows.set(row.operationKey, canceled); + return canceled; + }); + backend.get<{ Querystring: { proposalId?: string } }>("/transit/pins", async (request) => ({ + active: null, + catalogRevisionId: null, + allowedCountries: ["de"], + page: { proposals: [], total: 0, offset: 0, nextOffset: null }, + proposal: request.query.proposalId + ? (proposals.get(request.query.proposalId)?.proposal ?? null) + : null, + diff: null, + validation: null, + execution: null, + validationInputs: null, + validationEligible: false, + blocker: null, + })); + backend.post<{ Body: { proposalId: string; branch: string } }>( + "/transit/bump", + async (request) => { + posts++; + const selected = proposal(request.body.proposalId, request.body.branch); + proposals.set(selected.proposalId, { branch: request.body.branch, proposal: selected }); + return { + ok: true, + unchanged: false, + proposed: true, + proposalId: selected.proposalId, + catalogRevisionId: selected.pins.generationId, + ref: selected.pins.transitous.ref, + previousRef: null, + submoduleSha: "b".repeat(40), + lockedAt: selected.createdAt, + lockedBy: "fixture", + }; + }, + ); + vi.stubEnv("DATA_MANAGER_URL", await backend.listen({ host: "127.0.0.1", port: 0 })); + app = Fastify({ forceCloseConnections: true }); + await app.register(adminTransitPinsRoute, { prefix: "/api" }); + client = createApiClient({ baseUrl: await app.listen({ host: "127.0.0.1", port: 0 }) }); +}); +afterEach(async () => { + await app?.close(); + await backend?.close(); + vi.unstubAllEnvs(); +}); +it("uses the real generated SDK and reuses the exact admitted job instead of submitting it twice", async () => { + const proposalId = randomUUID(), + headers = { "idempotency-key": key() }; + const first = await sdk.validateAdminTransitPins( + proposalId, + { countries: ["DE", "de"] }, + headers, + { client }, + ); + expect(first.status).toBe(202); + expect( + sdk.unwrapAdminResponse( + await sdk.validateAdminTransitPins(proposalId, { countries: ["DE", "de"] }, headers, { + client, + }), + ), + ).toEqual(sdk.unwrapAdminResponse(first)); + expect(posts).toBe(1); + expect( + (await sdk.validateAdminTransitPins(proposalId, { countries: ["at"] }, headers, { client })) + .status, + ).toBe(409); + expect(posts).toBe(1); +}); +it("recovers a lost admission response by lookup only and cancels that exact identity", async () => { + loseNext = true; + const accepted = sdk.unwrapAdminResponse( + await sdk.validateAdminTransitPins( + randomUUID(), + { countries: ["de"] }, + { "idempotency-key": key() }, + { client }, + ), + ); + expect(posts).toBe(1); + const row = [...rows.values()][0]!; + expect(accepted.jobId).toBe(row.id); + const canceled = sdk.unwrapAdminResponse( + await sdk.cancelAdminTransitPinExecution( + row.id, + { operationKey: row.operationKey, fingerprint: row.fingerprint }, + { client }, + ), + ); + expect(canceled.status).toBe("cancel_pending"); + expect(posts).toBe(1); +}); +it("retains proposal identity and refuses a changed branch on the same key", async () => { + const headers = { "idempotency-key": key() }, + first = sdk.unwrapAdminResponse( + await sdk.proposeAdminTransitPins({ branch: "main" }, headers, { client }), + ); + expect( + sdk.unwrapAdminResponse( + await sdk.proposeAdminTransitPins({ branch: "main" }, headers, { client }), + ), + ).toEqual(first); + expect(posts).toBe(1); + expect( + (await sdk.proposeAdminTransitPins({ branch: "different" }, headers, { client })).status, + ).toBe(409); + expect(posts).toBe(1); +}); +it("keeps authentication/role/header denial ahead of the DM side effect", async () => { + state.session = null; + expect((await sdk.getAdminTransitPins(undefined, { client })).status).toBe(401); + state.session = { ...mockAdminSession(), user: { ...mockAdminSession().user, role: "user" } }; + expect((await sdk.getAdminTransitPins(undefined, { client })).status).toBe(403); + state.session = mockAdminSession(); + const response = await app.inject({ + method: "POST", + url: `/api/admin/transit/pins/proposals/${randomUUID()}/validate`, + payload: { countries: ["de"] }, + }); + expect(response.statusCode).toBe(400); + expect(posts).toBe(0); +}); diff --git a/apps/api/src/routes/admin-transit-credentials.ts b/apps/api/src/routes/admin-transit-credentials.ts index a28dac41d..2d7c29ab9 100644 --- a/apps/api/src/routes/admin-transit-credentials.ts +++ b/apps/api/src/routes/admin-transit-credentials.ts @@ -49,8 +49,21 @@ export async function adminTransitCredentialsRoute(app: FastifyInstance) { ? "Credential application is pending reconciliation; inspect Activity before retrying" : "Transit credential administration is unavailable", }); - register(app, "getAdminTransitCredentials", async (_request, reply) => { + register(app, "getAdminTransitCredentials", async (request, reply) => { try { + if (request.query.proposalId) { + const selected = await executeAndWait( + createApiOpsClient(), + { kind: "transitousLock.inspect", proposalId: request.query.proposalId, offset: 0 }, + createDurableOpsKey("transit.credentials.proposal", request.query.proposalId), + ); + if (!selected.proposal) + return reply.code(404).send({ error: "Catalog proposal is unavailable" }); + return await inspectFeedCredentials({ + catalogRevisionId: selected.proposal.pins.generationId, + proposalId: selected.proposal.proposalId, + }); + } const pair = await executeAndWait( createApiOpsClient(), { kind: "catalogPins.inspect" }, diff --git a/apps/api/src/routes/admin-transit-pins.ts b/apps/api/src/routes/admin-transit-pins.ts new file mode 100644 index 000000000..f403f63de --- /dev/null +++ b/apps/api/src/routes/admin-transit-pins.ts @@ -0,0 +1,294 @@ +import { createHash } from "node:crypto"; +import { transitPinOperations } from "@openmapx/core/api-contracts"; +import { + type DataExecutionInput, + type DataExecutionStatus, + normalizeDataExecutionInput, +} from "@openmapx/core/data-manager-contracts"; +import * as dm from "@openmapx/core/data-manager-sdk"; +import { canonicalJson, executionFingerprint, opsExecutionBudget } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; +import type { FastifyInstance } from "fastify"; +import { + TransitPinRequestConflict, + transitPinRequestIdentity, +} from "../services/transit-pin-requests"; +import { writeAuditLog } from "../utils/audit-log"; +import { RateLimiter } from "../utils/rate-limit"; +import { getAdminSession, requireAdmin } from "../utils/require-admin"; +import { declareRouteAuth } from "../utils/route-auth"; +import { createAdminRouteRegistrar } from "./admin-contract"; + +const register = createAdminRouteRegistrar(transitPinOperations, { + tag: "data-manager-admin", + validationPhase: "preHandler", +}); +function transport(long = false) { + return dm.createDataManagerTransport({ + baseUrl: envString("DATA_MANAGER_URL", "http://localhost:4000"), + authToken: envString("DATA_MANAGER_AUTH_TOKEN", ""), + requestTimeoutMs: long ? 120000 : 15000, + maxJsonBytes: 8 * 1024 * 1024, + }); +} +type CatalogInput = Extract; +function matched( + status: DataExecutionStatus, + input: CatalogInput, + identity: ReturnType, +) { + const operationFingerprint = createHash("sha256") + .update(canonicalJson(normalizeDataExecutionInput(input))) + .digest("hex"); + if ( + status.operationKey !== identity.operationKey || + status.kind !== input.kind || + status.deadlineAt > identity.deadlineAt || + status.fingerprint !== + executionFingerprint({ + kind: input.kind, + operationFingerprint, + deadlineAt: status.deadlineAt, + }) + ) + throw new TransitPinRequestConflict( + "This request identity belongs to a different catalog operation", + ); + return status; +} +async function submit( + input: CatalogInput, + actor: string, + key: unknown, +): Promise { + const identity = transitPinRequestIdentity(actor, key), + connection = transport(); + const existing = await dm.lookupDataExecution(identity.operationKey, { transport: connection }); + if (existing.status !== 404) + return matched(dm.unwrapDataManagerResponse(existing), input, identity); + if ( + identity.deadlineAt <= Date.now() || + identity.deadlineAt > Date.now() + opsExecutionBudget(input.kind).processMs + 5000 + ) + throw new TransitPinRequestConflict( + "Catalog request expired or exceeds its time budget. Refresh review and check the device clock.", + ); + try { + return matched( + dm.unwrapDataManagerResponse( + await dm.startDataExecution( + { input, operationKey: identity.operationKey, requestedDeadlineAt: identity.deadlineAt }, + { transport: connection }, + ), + ), + input, + identity, + ); + } catch (error) { + // An accepted root can outlive a lost admission response. Read its original + // address; never resubmit the mutation in this observation continuation. + if (error instanceof dm.DataManagerHttpError) throw error; + const retained = await dm.lookupDataExecution(identity.operationKey, { transport: connection }); + if (retained.status !== 404) + return matched(dm.unwrapDataManagerResponse(retained), input, identity); + throw error; + } +} +export async function adminTransitPinsRoute(app: FastifyInstance): Promise { + declareRouteAuth(app, "admin"); + app.addHook("preHandler", async (request) => { + request.adminSession = await requireAdmin(request); + }); + app.addHook("onRequest", async (_request, reply) => { + reply.header("Cache-Control", "private, no-store"); + }); + const limits = new RateLimiter({ + max: 12, + windowMs: 60000, + keyFn: (request) => getAdminSession(request).user.id, + }); + app.addHook("onClose", async () => limits.destroy()); + const failure = (error: unknown) => ({ + status: + error instanceof TransitPinRequestConflict + ? 409 + : error instanceof dm.DataManagerHttpError && [400, 404, 409, 422].includes(error.status) + ? error.status + : 503, + error: + error instanceof TransitPinRequestConflict + ? error.message + : "Catalog operation is unavailable. Inspect the retained transit job before retrying.", + }); + register(app, "getAdminTransitPins", async (request, reply) => { + try { + return dm.unwrapDataManagerResponse( + await dm.inspectCatalogReview(request.query, { transport: transport() }), + ); + } catch (error) { + const rejected = failure(error); + return reply.code(rejected.status).send({ error: rejected.error }); + } + }); + register( + app, + "proposeAdminTransitPins", + async (request, reply) => { + const actor = getAdminSession(request).user.id; + try { + const identity = transitPinRequestIdentity(actor, request.headers["idempotency-key"]); + const retained = dm.unwrapDataManagerResponse( + await dm.inspectCatalogReview( + { proposalId: identity.proposalId }, + { transport: transport() }, + ), + ); + if (retained.proposal) { + if (retained.proposal.pins.transitous.ref.split("@")[0] !== request.body.branch) + throw new TransitPinRequestConflict( + "This proposal identity belongs to a different branch", + ); + return { + proposalId: retained.proposal.proposalId, + generationId: retained.proposal.pins.generationId, + }; + } + if (identity.deadlineAt <= Date.now() || identity.deadlineAt > Date.now() + 3600000 + 5000) + throw new TransitPinRequestConflict( + "Proposal request expired or exceeds its time budget", + ); + // The immutable resolver retains this exact ID/body before its agent POST; + // duplicate delivery reads that original source rather than refetching it. + const created = dm.unwrapDataManagerResponse( + await dm.proposeTransitBump( + { branch: request.body.branch, proposalId: identity.proposalId }, + { transport: transport(true) }, + ), + ); + if (created.unchanged) + throw new TransitPinRequestConflict("Refresh review to select the current catalog pair"); + await writeAuditLog({ + actorId: actor, + targetType: "transit", + targetId: created.proposalId, + action: "transit.catalog.propose", + details: { branch: request.body.branch, generationId: created.catalogRevisionId }, + request, + }); + return { proposalId: created.proposalId, generationId: created.catalogRevisionId }; + } catch (error) { + const rejected = failure(error); + return reply.code(rejected.status).send({ error: rejected.error }); + } + }, + { beforeValidation: limits.preHandler() }, + ); + register( + app, + "validateAdminTransitPins", + async (request, reply) => { + const actor = getAdminSession(request).user.id; + try { + const job = await submit( + { + kind: "catalog.validate", + proposalId: request.params.proposalId, + countries: request.body.countries, + }, + actor, + request.headers["idempotency-key"], + ); + await writeAuditLog({ + actorId: actor, + targetType: "transit", + targetId: request.params.proposalId, + action: "transit.catalog.validate", + details: { jobId: job.id, countries: request.body.countries }, + request, + }); + return reply.code(202).send({ jobId: job.id }); + } catch (error) { + const rejected = failure(error); + return reply.code(rejected.status).send({ error: rejected.error }); + } + }, + { beforeValidation: limits.preHandler() }, + ); + register( + app, + "activateAdminTransitPins", + async (request, reply) => { + const actor = getAdminSession(request).user.id; + try { + const job = await submit( + { kind: "catalog.activate", proposalId: request.params.proposalId, ...request.body }, + actor, + request.headers["idempotency-key"], + ); + await writeAuditLog({ + actorId: actor, + targetType: "transit", + targetId: request.params.proposalId, + action: "transit.catalog.activate", + details: { + jobId: job.id, + validationId: request.body.validationId, + diffDigest: request.body.diffDigest, + }, + request, + }); + return reply.code(202).send({ jobId: job.id }); + } catch (error) { + const rejected = failure(error); + return reply.code(rejected.status).send({ error: rejected.error }); + } + }, + { beforeValidation: limits.preHandler() }, + ); + register( + app, + "cancelAdminTransitPinExecution", + async (request, reply) => { + try { + const connection = transport(), + current = dm.unwrapDataManagerResponse( + await dm.getDataExecution(request.params.id, { transport: connection }), + ); + if ( + !["catalog.validate", "catalog.activate"].includes(current.kind) || + current.operationKey !== request.body.operationKey || + current.fingerprint !== request.body.fingerprint + ) + throw new TransitPinRequestConflict("Catalog execution identity changed"); + const canceled = dm.unwrapDataManagerResponse( + await dm.cancelDataExecution(request.params.id, request.body, { transport: connection }), + ); + await writeAuditLog({ + actorId: getAdminSession(request).user.id, + targetType: "transit", + targetId: current.id, + action: "transit.catalog.cancel", + request, + }); + return canceled; + } catch (error) { + const rejected = failure(error); + return reply.code(rejected.status).send({ error: rejected.error }); + } + }, + { beforeValidation: limits.preHandler() }, + ); + register(app, "getAdminTransitPinExecution", async (request, reply) => { + try { + const current = dm.unwrapDataManagerResponse( + await dm.getDataExecution(request.params.id, { transport: transport() }), + ); + if (!["catalog.validate", "catalog.activate"].includes(current.kind)) + return reply.code(404).send({ error: "Catalog execution is unavailable" }); + return current; + } catch (error) { + const rejected = failure(error); + return reply.code(rejected.status).send({ error: rejected.error }); + } + }); +} diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index 6820fc739..8869d00fc 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -15,6 +15,7 @@ import { adminServicesRoute } from "./admin-services"; import { adminSettingsRoute } from "./admin-settings"; import { adminSystemRoute } from "./admin-system"; import { adminTransitCredentialsRoute } from "./admin-transit-credentials"; +import { adminTransitPinsRoute } from "./admin-transit-pins"; import { ambientPlacesRoute } from "./ambient-places"; import { attributionRoute } from "./attribution"; import { authRoute } from "./auth"; @@ -119,6 +120,7 @@ export async function registerCoreRoutes( await server.register(adminRoute, { prefix: "/api" }); await server.register(adminServicesRoute, { prefix: "/api" }); await server.register(adminTransitCredentialsRoute, { prefix: "/api" }); + await server.register(adminTransitPinsRoute, { prefix: "/api" }); await server.register(adminOperationsRoute, { prefix: "/api" }); await server.register(adminCoverageRoute, { prefix: "/api" }); await server.register(adminActivityRoute, { prefix: "/api" }); diff --git a/apps/api/src/services/activity-catalog-postgres.test.ts b/apps/api/src/services/activity-catalog-postgres.test.ts new file mode 100644 index 000000000..ac483ae46 --- /dev/null +++ b/apps/api/src/services/activity-catalog-postgres.test.ts @@ -0,0 +1,62 @@ +/** Actual SQL Activity projection; admitted rows here are inert and never dispatch. */ +import { randomBytes, randomUUID } from "node:crypto"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { expect, it, vi } from "vitest"; +import { createPostgresDataExecutionStore } from "../../../../services/data-manager/src/jobs/execution/store"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; + +const state = vi.hoisted(() => ({ + fixture: null as Awaited> | null, +})); +vi.mock("../db", () => ({ + get db() { + if (!state.fixture) throw Error("Owned database not installed"); + return state.fixture.db; + }, +})); +it.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "offers exact durable catalog jobs for cancellation in both Activity list and detail", + async () => { + const fixture = await createMigratedPostgresFixture(); + state.fixture = fixture; + try { + const store = createPostgresDataExecutionStore(fixture.sql), + deadline = Date.now() + 60000, + proposalId = randomUUID(); + const inputs = [ + { kind: "catalog.validate" as const, proposalId, countries: ["de"] }, + { + kind: "catalog.activate" as const, + proposalId, + validationId: randomUUID(), + expectedGenerationId: null, + diffDigest: "d".repeat(64), + acknowledgeLicenseChanges: true, + }, + ]; + const { listActivityJobs, getActivityJob } = await import("./activity-jobs"); + for (const input of inputs) { + const { record } = await store.admit(input, { + operationKey: createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + requestedDeadlineAt: deadline, + }); + const listed = await listActivityJobs({ filter: "active", limit: 100, offset: 0 }); + expect(listed.jobs.find((job) => job.id === record.status.id)).toMatchObject({ + source: "data-manager", + type: input.kind, + cancelable: true, + }); + expect(await getActivityJob(record.status.id, "data-manager")).toMatchObject({ + cancelable: true, + }); + } + } finally { + state.fixture = null; + await fixture.stop(); + } + }, + 60000, +); diff --git a/apps/api/src/services/activity-jobs.ts b/apps/api/src/services/activity-jobs.ts index a462f7185..57d9f04bc 100644 --- a/apps/api/src/services/activity-jobs.ts +++ b/apps/api/src/services/activity-jobs.ts @@ -175,7 +175,7 @@ function dataManagerSelect(filter?: ActivityJobFilter) { startedAt: dataManagerJobs.startedAt, finishedAt: dataManagerJobs.finishedAt, cancelable: - sql`${dataManagerJobs.executionVersion}=2 AND (${dataManagerJobs.status} IN ('queued','running','cancel_pending') OR ${dataManagerRecoveryPending()}) AND ${dataManagerJobs.kind} IN ('download-osm','download-fonts','convert-overpass','transit-sync','overture-pull','overture-ingest','overture-extract','overture-conflate','overture-sync','search-index-build','notable-places-build','generation-hardlinks','transit-maintenance') AND ${dataManagerJobs.operationKey} IS NOT NULL AND ${dataManagerJobs.executionFingerprint} IS NOT NULL`.as( + sql`${dataManagerJobs.executionVersion}=2 AND (${dataManagerJobs.status} IN ('queued','running','cancel_pending') OR ${dataManagerRecoveryPending()}) AND ${dataManagerJobs.kind} IN ('download-osm','download-fonts','convert-overpass','transit-sync','overture-pull','overture-ingest','overture-extract','overture-conflate','overture-sync','search-index-build','notable-places-build','generation-hardlinks','transit-maintenance','catalog.validate','catalog.activate') AND ${dataManagerJobs.operationKey} IS NOT NULL AND ${dataManagerJobs.executionFingerprint} IS NOT NULL`.as( "cancelable", ), }) diff --git a/apps/api/src/services/transit-credential-preview.test.ts b/apps/api/src/services/transit-credential-preview.test.ts index 209f1e07a..6cbbef92f 100644 --- a/apps/api/src/services/transit-credential-preview.test.ts +++ b/apps/api/src/services/transit-credential-preview.test.ts @@ -20,3 +20,20 @@ it("public intent binds opaque preview identity and actions, never hashes a cred feedCredentialIntent({ ...base, changes: [{ key: "de/a", action: "delete" }] }), ); }); + +it("binds the proposed catalog selector without changing legacy active-preview intent", () => { + const base = { + previewId: "opaque", + actorId: "admin", + catalogRevisionId: "catalog", + credentialsRevision: "private-row-version", + expectedCurrentRevisionId: null, + changes: [{ key: "de/a", action: "set" as const, value: "private-value" }], + }; + const first = { ...base, proposalId: "00000000-0000-4000-8000-000000000001" }; + expect(feedCredentialIntent(first)).not.toBe(feedCredentialIntent(base)); + expect(feedCredentialIntent(first)).not.toBe( + feedCredentialIntent({ ...first, proposalId: "00000000-0000-4000-8000-000000000002" }), + ); + expect(feedCredentialIntent({ ...base, proposalId: undefined })).toBe(feedCredentialIntent(base)); +}); diff --git a/apps/api/src/services/transit-credential-preview.ts b/apps/api/src/services/transit-credential-preview.ts index 6120b97a0..c7ff9761d 100644 --- a/apps/api/src/services/transit-credential-preview.ts +++ b/apps/api/src/services/transit-credential-preview.ts @@ -17,6 +17,7 @@ export interface FeedCredentialChangePreview { previewId: string; intentFingerprint: string; catalogRevisionId: string; + proposalId?: string; credentialsRevision: string; expectedCurrentRevisionId: string | null; expiresAt: string; @@ -35,6 +36,7 @@ export function feedCredentialIntent(input: { previewId: string; actorId: string; catalogRevisionId: string; + proposalId?: string; credentialsRevision: string; expectedCurrentRevisionId: string | null; changes: readonly FeedCredentialChange[]; @@ -46,6 +48,7 @@ export function feedCredentialIntent(input: { previewId: input.previewId, actorId: input.actorId, catalogRevisionId: input.catalogRevisionId, + ...(input.proposalId ? { proposalId: input.proposalId } : {}), credentialsRevision: input.credentialsRevision, expectedCurrentRevisionId: input.expectedCurrentRevisionId, changes: input.changes @@ -84,6 +87,7 @@ function decoded(row: typeof transitCredentialPreview.$inferSelect): PrivateFeed export async function retainFeedCredentialPreview( input: Omit & { catalogRevisionId: string; + proposalId?: string; credentialsRevision: string; expectedCurrentRevisionId: string | null; preservedCount: number; @@ -109,12 +113,14 @@ export async function retainFeedCredentialPreview( existing.expiresAt <= new Date() || canonicalJson({ catalogRevisionId: value.preview.catalogRevisionId, + ...(value.preview.proposalId ? { proposalId: value.preview.proposalId } : {}), credentialsRevision: value.preview.credentialsRevision, expectedCurrentRevisionId: value.preview.expectedCurrentRevisionId, changes: value.changes, }) !== canonicalJson({ catalogRevisionId: input.catalogRevisionId, + ...(input.proposalId ? { proposalId: input.proposalId } : {}), credentialsRevision: input.credentialsRevision, expectedCurrentRevisionId: input.expectedCurrentRevisionId, changes: input.changes, @@ -137,6 +143,7 @@ export async function retainFeedCredentialPreview( previewId, intentFingerprint: feedCredentialIntent({ ...input, previewId }), catalogRevisionId: input.catalogRevisionId, + ...(input.proposalId ? { proposalId: input.proposalId } : {}), credentialsRevision: input.credentialsRevision, expectedCurrentRevisionId: input.expectedCurrentRevisionId, expiresAt, diff --git a/apps/api/src/services/transit-credential-revision.ts b/apps/api/src/services/transit-credential-revision.ts index 3e0175de5..e3e7ef5b7 100644 --- a/apps/api/src/services/transit-credential-revision.ts +++ b/apps/api/src/services/transit-credential-revision.ts @@ -1,15 +1,11 @@ /** Randomized encrypted-row identity; never hash the plaintext map. */ import { createHash } from "node:crypto"; +import { TRANSIT_CREDENTIALS_ABSENT_REVISION } from "@openmapx/core/ops"; export function transitCredentialRevision( row: { id: string; ciphertext: string; iv: string; tag: string } | undefined, ): string { + if (!row) return TRANSIT_CREDENTIALS_ABSENT_REVISION; return createHash("sha256") - .update( - JSON.stringify( - row - ? ["transit-credentials-v1", row.id, row.ciphertext, row.iv, row.tag] - : ["transit-credentials-v1", "absent"], - ), - ) + .update(JSON.stringify(["transit-credentials-v1", row.id, row.ciphertext, row.iv, row.tag])) .digest("hex"); } diff --git a/apps/api/src/services/transit-feed-credential-participant.ts b/apps/api/src/services/transit-feed-credential-participant.ts index d4e8f054f..560fc4baf 100644 --- a/apps/api/src/services/transit-feed-credential-participant.ts +++ b/apps/api/src/services/transit-feed-credential-participant.ts @@ -65,10 +65,19 @@ export function withTransitCredentialProofContext( ) { return proofContexts.run(context, run); } -async function inspect(catalogRevisionId: string, identity: string, signal?: AbortSignal) { +async function inspect( + catalogRevisionId: string, + identity: string, + signal?: AbortSignal, + proposalId?: string, +) { return executeAndWait( createApiOpsClient(), - { kind: "transitCredentials.inspect", catalogRevisionId }, + { + kind: "transitCredentials.inspect", + catalogRevisionId, + ...(proposalId ? { proposalId } : {}), + }, createDurableOpsKey("transit.credentials.inspect", identity), { signal }, ); @@ -81,12 +90,14 @@ function values(row: typeof serviceSecret.$inferSelect | undefined) { } export async function inspectFeedCredentials(input: { catalogRevisionId: string; + proposalId?: string; signal?: AbortSignal; }) { const host = await inspect( input.catalogRevisionId, `read:${input.catalogRevisionId}`, input.signal, + input.proposalId, ); const current = await executeAndWait( createApiOpsClient(), @@ -108,6 +119,7 @@ export async function inspectFeedCredentials(input: { ); return { catalogRevisionId: input.catalogRevisionId, + ...(input.proposalId ? { proposalId: input.proposalId } : {}), credentialsRevision: host.owner === "file" ? (host.hostCredentialsRevision ?? revision) : revision, configurationRevisionId: current.revisionId, @@ -128,6 +140,7 @@ export async function inspectFeedCredentials(input: { interface ChangeInput { actorId: string; catalogRevisionId: string; + proposalId?: string; expectedCredentialsRevision: string; expectedCurrentRevisionId: string | null; changes: FeedCredentialChange[]; @@ -135,7 +148,12 @@ interface ChangeInput { signal?: AbortSignal; } async function validateInput(input: ChangeInput) { - const host = await inspect(input.catalogRevisionId, input.operationKey, input.signal); + const host = await inspect( + input.catalogRevisionId, + input.operationKey, + input.signal, + input.proposalId, + ); if (!host.editable) throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_HOST_OWNED"); return withAdministrativeConfigurationRead({}, async (read) => { const row = rowFrom(read), @@ -175,6 +193,7 @@ export interface PreparedFeedCredentialPatch { readonly serviceId: typeof SERVICE; readonly key: typeof KEY; readonly catalogRevisionId: string; + readonly proposalId?: string; readonly expectedCredentialsRevision: string; readonly expectedCurrentRevisionId: string | null; readonly importPreviewId: string; @@ -215,6 +234,7 @@ export async function prepareFeedCredentialImportPatch( serviceId: SERVICE, key: KEY, catalogRevisionId: input.catalogRevisionId, + ...(input.proposalId ? { proposalId: input.proposalId } : {}), expectedCredentialsRevision: input.expectedCredentialsRevision, expectedCurrentRevisionId: input.expectedCurrentRevisionId, importPreviewId: input.importPreviewId, @@ -291,6 +311,7 @@ export async function authorizePreparedFeedCredentialPatch(input: { !context || context.actorId !== state.input.actorId || input.preview.catalogRevisionId !== state.input.catalogRevisionId || + input.preview.proposalId !== state.input.proposalId || input.preview.credentialsRevision !== state.input.expectedCredentialsRevision || input.preview.expectedCurrentRevisionId !== state.input.expectedCurrentRevisionId || (input.authorization.kind === "direct" @@ -362,6 +383,7 @@ export function transitCredentialParticipation( patch.catalogRevisionId, writer.operationKey, administrativeConfigurationState(writer).job.signal, + state.input.proposalId, ); if (!current.editable) throw new TransitCredentialConflict("TRANSIT_CREDENTIALS_HOST_OWNED"); await withAdministrativeConfigurationRead({}, async (read) => { diff --git a/apps/api/src/services/transit-feed-credentials.ts b/apps/api/src/services/transit-feed-credentials.ts index f388d90b0..2c538fd3f 100644 --- a/apps/api/src/services/transit-feed-credentials.ts +++ b/apps/api/src/services/transit-feed-credentials.ts @@ -33,6 +33,7 @@ export async function applyFeedCredentialChanges(input: { patch = await prepareFeedCredentialImportPatch({ actorId: input.actorId, catalogRevisionId: p.preview.catalogRevisionId, + ...(p.preview.proposalId ? { proposalId: p.preview.proposalId } : {}), expectedCredentialsRevision: p.preview.credentialsRevision, expectedCurrentRevisionId: p.preview.expectedCurrentRevisionId, changes: p.changes, diff --git a/apps/api/src/services/transit-pin-requests.ts b/apps/api/src/services/transit-pin-requests.ts new file mode 100644 index 000000000..f6c7a1535 --- /dev/null +++ b/apps/api/src/services/transit-pin-requests.ts @@ -0,0 +1,28 @@ +/** Request identity only. These values authorize no principal, lease, parent or forward effect. */ +import { createHash } from "node:crypto"; +import { createControlRootOperationKey, parseControlRootOperationKey } from "@openmapx/core/ops"; +export class TransitPinRequestConflict extends Error {} +export function transitPinRequestIdentity(actorId: string, key: unknown) { + if (typeof key !== "string") + throw new TransitPinRequestConflict("A single catalog request identity is required"); + let parsed: ReturnType; + try { + parsed = parseControlRootOperationKey(key); + } catch { + throw new TransitPinRequestConflict( + "Refresh catalog review to create a supported request identity", + ); + } + const bytes = createHash("sha256") + .update(`openmapx-admin-catalog-request-v1\0${actorId}\0${key}`) + .digest(); + const hex = bytes.toString("hex"); + return { + deadlineAt: parsed.notAfter, + operationKey: createControlRootOperationKey({ + notAfter: parsed.notAfter, + nonce: bytes.subarray(0, 16), + }), + proposalId: `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20, 32)}`, + }; +} diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index bc0d48005..53ecb2c8f 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -60,7 +60,7 @@ export async function ownedServiceHost(options: { resources?:Record { + const runtime = createUnavailableRuntime(); + installCatalogApprovalRuntime(runtime, { rootDir: "/unused" }); + await expect( + runtime[operation.kind](operation, { + signal: new AbortController().signal, + } as OpsExecutionContext), + ).rejects.toThrow(/Original reviewed activation/); + await expect( + runtime[operation.kind](operation, { + artifactLease: { mode: "borrowed", owner: { executor: "data-manager" } }, + controlAuthority: {}, + signal: new AbortController().signal, + } as OpsExecutionContext), + ).rejects.toThrow(/Unverified execution authority/); +}); +it("gives one immutable approval attempt a stable distinct commit marker", () => { + const key = "opk1_aaaaaaaaaaaaaaaa", + fingerprint = "b".repeat(64); + expect(catalogApprovalCommitId(key, fingerprint)).toMatch( + /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-8[a-f0-9]{3}-[a-f0-9]{12}$/, + ); + expect(catalogApprovalCommitId(key, fingerprint)).toBe(catalogApprovalCommitId(key, fingerprint)); + expect(catalogApprovalCommitId(key, "c".repeat(64))).not.toBe( + catalogApprovalCommitId(key, fingerprint), + ); +}); diff --git a/apps/ops-agent/src/catalog-approval-runtime.ts b/apps/ops-agent/src/catalog-approval-runtime.ts new file mode 100644 index 000000000..94e855a68 --- /dev/null +++ b/apps/ops-agent/src/catalog-approval-runtime.ts @@ -0,0 +1,349 @@ +/** The only production pin publisher: independent evidence, original promotion and original borrowed leaf. */ +import { createHash } from "node:crypto"; +import { realpathSync } from "node:fs"; +import { join, sep } from "node:path"; +import type { + TargetLeaseAuthority, + VerifiedExecutionAuthority, +} from "@openmapx/core/build-artifacts"; +import { + assertControlExecutionForward, + controlDigest, + executionAuthorityState, + inspectArtifactDelegation, + inspectDataTransitCandidate, + restoreAdmittedControlExecution, + retainedControlAdoptionAssertion, + retainedControlParent, + verifyControl, + verifyControlEffectSettlement, + verifyDataControlTerminalWitness, + verifyRetainedArtifactTargetSettlement, +} from "@openmapx/core/build-artifacts"; +import { + catalogApprovalAdmissionSchema, + catalogPromotionEffectSchema, + catalogRuntimeInputsSchema, + publishCatalogPinSet, + readCatalogActivation, + readCatalogPinCommit, + readCatalogPinSet, + readCatalogProposal, + readCatalogValidation, + readCurrentCatalogValidation, + sameCatalogRuntimeInputs, + scopeCatalogRuntimeInputs, +} from "@openmapx/core/catalog-pins"; +import { + canonicalJson, + type OpsOperation, + opsOperationFingerprint, + parseOpsResult, +} from "@openmapx/core/ops"; +import { + childSettlementState, + readPrivateRecord, +} from "../../../packages/core/src/build-artifacts/control-authority"; +import { originRecordDomain } from "../../../packages/core/src/build-artifacts/control-records"; +import { verifyDataTransitCandidateForExecution } from "../../../packages/core/src/build-artifacts/data-execution-candidate"; +import { AnchoredDirectory } from "../../../packages/core/src/build-artifacts/filesystem"; +import type { OpsRuntime } from "./runtime"; + +const installedRoots = new WeakMap(); + +/** Lookup-only original publication recovery. This never invokes the handler, + * reevaluates eligibility, acquires a root, or changes the active pointer. */ +export async function readCommittedCatalogApproval( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + lease: TargetLeaseAuthority, + operation: Extract, +) { + const rootDir = installedRoots.get(runtime); + if (!rootDir) return null; + const own = executionAuthorityState(authority); + if ( + own.kind !== operation.kind || + own.execution.operationFingerprint !== opsOperationFingerprint(operation) || + lease.mode !== "borrowed" || + lease.owner.executor !== "data-manager" + ) + throw Error("Original catalog approval borrower required"); + const activation = readCatalogActivation( + own.installation.installation, + operation.proposalId, + lease.owner.executionId, + ); + if ( + controlDigest(activation.approval) !== controlDigest(own.execution) || + controlDigest(activation.parent) !== controlDigest(lease.owner) + ) + throw Error("Original catalog approval source changed"); + const receipt = await readCatalogPinCommit({ + rootDir, + commit: { + commitId: catalogApprovalCommitId(own.execution.operationKey, own.execution.fingerprint), + execution: own.execution, + proposalId: operation.proposalId, + validationId: operation.validationId, + diffDigest: operation.diffDigest, + }, + }); + if (receipt?.phase !== "committed" || !receipt.committedAt) return null; + const proposal = readCatalogProposal({ + rootDir, + installation: own.installation.installation, + proposalId: operation.proposalId, + }); + if ( + !proposal || + receipt.previousGenerationId !== operation.expectedGenerationId || + receipt.generationId !== proposal.pins.generationId || + activation.proposalDigest !== controlDigest(proposal) + ) + throw Error("Original committed catalog result changed"); + return { + terminalAt: Date.parse(receipt.committedAt), + result: { + outcome: "activated" as const, + proposalId: operation.proposalId, + validationId: operation.validationId, + previousGenerationId: operation.expectedGenerationId, + generationId: receipt.generationId, + commitId: receipt.identity.commitId, + committedAt: receipt.committedAt, + }, + }; +} +export function catalogApprovalCommitId(operationKey: string, fingerprint: string): string { + const value = createHash("sha256") + .update(`openmapx-paired-pin-commit-v1\0${operationKey}\0${fingerprint}`) + .digest("hex"); + return `${value.slice(0, 8)}-${value.slice(8, 12)}-4${value.slice(13, 16)}-8${value.slice(17, 20)}-${value.slice(20, 32)}`; +} +export function installCatalogApprovalRuntime( + runtime: OpsRuntime, + options: { rootDir: string }, +): void { + installedRoots.set(runtime, options.rootDir); + runtime["transitousLock.approve"] = async (operation, context) => { + const authority = context.controlAuthority, + lease = context.artifactLease; + if ( + !authority || + !lease || + lease.mode !== "borrowed" || + lease.owner.executor !== "data-manager" || + !operation.acknowledgeLicenseChanges + ) + throw Error("Original reviewed activation borrower required"); + assertControlExecutionForward(authority); + const own = executionAuthorityState(authority), + installation = own.installation.installation; + if ( + own.kind !== operation.kind || + controlDigest(context.execution) !== controlDigest(own.execution) + ) + throw Error("Original approval identity required"); + const activation = readCatalogActivation( + installation, + operation.proposalId, + lease.owner.executionId, + ), + proposal = readCatalogProposal({ + rootDir: options.rootDir, + installation, + proposalId: operation.proposalId, + }), + validation = readCatalogValidation( + installation, + operation.proposalId, + operation.validationId, + ), + passed = validation.validation; + if ( + !proposal || + passed.status !== "passed" || + activation.validationId !== operation.validationId || + controlDigest(activation.parent) !== controlDigest(lease.owner) || + controlDigest(activation.approval) !== controlDigest(context.execution) || + activation.proposalDigest !== controlDigest(proposal) || + activation.validationDigest !== controlDigest(validation) || + validation.proposalDigest !== controlDigest(proposal) || + passed.pinDigest !== proposal.pins.generationId || + passed.diffDigest !== operation.diffDigest || + proposal.diffDigest !== operation.diffDigest || + activation.diffDigest !== operation.diffDigest || + proposal.baseGenerationId !== operation.expectedGenerationId || + Date.now() >= Date.parse(passed.expiresAt) || + Date.parse(passed.completedAt) > Date.now() || + Date.parse(passed.expiresAt) > Date.parse(passed.completedAt) + 86400000 + ) + throw Error("Catalog approval evidence changed or expired"); + const currentValidation = readCurrentCatalogValidation(installation); + if ( + currentValidation?.selected?.proposalId !== operation.proposalId || + currentValidation.selected.validationId !== operation.validationId || + currentValidation.selected.validationDigest !== controlDigest(validation) + ) + throw Error("Validation was invalidated before slot reuse"); + const validated = verifyDataControlTerminalWitness(installation, validation.sourceExecution); + if ( + validated.status.status !== "ok" || + validated.status.result?.kind !== "catalog.validate" || + validated.status.result.status !== "passed" || + validated.status.result.validationId !== passed.validationId + ) + throw Error("Successful original validation completion required"); + const admission = verifyControl( + installation.publicKey, + originRecordDomain( + "openmapx-control-execution", + readPrivateRecord( + own.installation, + "executions", + `${context.execution.operationKey}.json`, + 65536, + ), + ), + readPrivateRecord( + own.installation, + "executions", + `${context.execution.operationKey}.json`, + 65536, + ), + catalogApprovalAdmissionSchema, + ); + if ( + canonicalJson(admission.parent) !== canonicalJson(retainedControlParent(activation.parent)) || + canonicalJson(admission.operation) !== canonicalJson(operation) + ) + throw Error("Approval differs from its actual parent admission"); + const promoted = restoreAdmittedControlExecution(installation, activation.promotion.execution), + promotion = executionAuthorityState(promoted); + if ( + promotion.kind !== "motis.primary.promote" || + activation.promotion.adoption.leaseId !== lease.leaseId + ) + throw Error("Promotion must belong to this exact original physical root"); + const assertion = retainedControlAdoptionAssertion( + installation, + activation.promotion.adoption, + activation.promotion.execution, + ), + grant = inspectArtifactDelegation({ + dataRoot: own.installation.dataRoot, + installation, + assertion, + }); + if ( + grant.phase !== "promote-primary" || + controlDigest(grant.parent) !== controlDigest(activation.parent) || + controlDigest(grant.owner) !== controlDigest(lease.owner) + ) + throw Error("Original promotion grant required"); + verifyRetainedArtifactTargetSettlement( + installation, + activation.promotion.adoption, + activation.promotion.execution, + ); + const settlement = childSettlementState( + verifyControlEffectSettlement(installation, activation.promotion.execution), + ).evidence; + if ( + settlement.outcome !== "committed" || + settlement.publication !== "committed" || + settlement.terminalReceiptDigest !== activation.promotion.terminalReceiptDigest + ) + throw Error("Original successful promotion settlement required"); + const effect = verifyControl( + installation.publicKey, + "openmapx-control-settlement-v1", + readPrivateRecord( + own.installation, + "effects", + `${activation.promotion.execution.operationKey}.json`, + 65536, + ), + catalogPromotionEffectSchema, + ); + if (parseOpsResult("motis.primary.promote", effect.receipt.result).activeRunId !== passed.epoch) + throw Error("Promotion activated a different candidate"); + const selected = await runtime["transitousLock.inspect"]( + { kind: "transitousLock.inspect", proposalId: operation.proposalId, offset: 0 }, + context, + ); + if ( + !selected.validationInputs || + !sameCatalogRuntimeInputs( + validation.runtimeInputs, + scopeCatalogRuntimeInputs( + catalogRuntimeInputsSchema.parse(selected.validationInputs), + passed.countries, + ), + ) || + !sameCatalogRuntimeInputs( + activation.runtimeInputs, + scopeCatalogRuntimeInputs( + catalogRuntimeInputsSchema.parse(selected.validationInputs), + passed.countries, + ), + ) + ) + throw Error("Effective policy, image or credential mount changed"); + const rootPath = realpathSync(own.installation.dataRoot), + livePath = realpathSync(join(rootPath, "motis/live")); + if (!livePath.startsWith(rootPath + sep)) + throw Error("Live candidate escaped its installation"); + const live = new AnchoredDirectory(livePath); + try { + const bytes = live.read("motis-candidate-manifest.json", 32 * 1024 * 1024, false); + if (createHash("sha256").update(bytes).digest("hex") !== passed.manifestHash) + throw Error("Activated manifest differs from validation"); + // The host independently rejoins its actual promotion's signed candidate + // publication and current inode/size/mtime/ctime inventory. Re-reading + // multi-hundred-GB datasets inside a 120-second pin leaf is not viable; + // the owned DM validation/activation worker hashes their complete bytes. + const promotedBody = { kind: "motis.primary.promote" as const, preparedRunId: passed.epoch }; + const candidate = inspectDataTransitCandidate( + installation, + activation.promotion.execution.operationKey, + promotedBody, + ); + verifyDataTransitCandidateForExecution(promoted, promotedBody, candidate.digest); + } finally { + live.close(); + } + const commit = { + commitId: catalogApprovalCommitId( + context.execution.operationKey, + context.execution.fingerprint, + ), + execution: context.execution, + proposalId: operation.proposalId, + validationId: operation.validationId, + diffDigest: operation.diffDigest, + }; + const current = readCatalogPinSet(options.rootDir); + if ((current?.generationId ?? null) !== operation.expectedGenerationId) + throw Error("Active catalog pair changed"); + const receipt = await publishCatalogPinSet({ + rootDir: options.rootDir, + lease, + pins: proposal.pins, + expectedGenerationId: operation.expectedGenerationId, + commit, + }); + if (receipt.phase !== "committed" || !receipt.committedAt) + throw Error("Catalog pin publication requires exact attempt recovery"); + return { + outcome: "activated", + proposalId: operation.proposalId, + validationId: operation.validationId, + previousGenerationId: operation.expectedGenerationId, + generationId: receipt.generationId, + commitId: commit.commitId, + committedAt: receipt.committedAt, + }; + }; +} diff --git a/apps/ops-agent/src/catalog-pins.test.ts b/apps/ops-agent/src/catalog-pins.test.ts index 6563b77c6..c9234fa37 100644 --- a/apps/ops-agent/src/catalog-pins.test.ts +++ b/apps/ops-agent/src/catalog-pins.test.ts @@ -24,7 +24,7 @@ it("serializes the largest legal paired pins through the actual HTTP server/clie ref: `${"a".repeat(215)}@${"b".repeat(40)}`, submodules: Object.fromEntries( Array.from({ length: 128 }, (_, n) => [ - `${"x".repeat(253)}${String(n).padStart(3, "0")}`, + n === 0 ? "transitland-atlas" : `${"x".repeat(253)}${String(n).padStart(3, "0")}`, "c".repeat(40), ]), ), @@ -82,15 +82,52 @@ it("serializes the largest legal paired pins through the actual HTTP server/clie await expect( client.execute({ kind: "transitousLock.propose", - ref: t.ref, - submodules: {}, - lockedBy: "fixture", + proposalId: "00000000-0000-4000-8000-000000000001", + baseGenerationId: expected.generationId, + pins: expected, + diffDigest: "e".repeat(64), + sourcePolicyRevision: "f".repeat(64), }), ).rejects.toMatchObject({ errorClass: "authorization" }); - else + else { await expect(client.execute({ kind: "configuration.inspect" })).rejects.toMatchObject({ errorClass: "authorization", }); + const changedCommit = "f".repeat(40), + candidate = createCatalogPinSet(t, { + ...g, + commit: changedCommit, + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${changedCommit}/systems.csv`, + }); + const proposalId = "00000000-0000-4000-8000-000000000002"; + const retained = await client.execute({ + kind: "transitousLock.propose", + proposalId, + baseGenerationId: expected.generationId, + pins: candidate, + diffDigest: "e".repeat(64), + sourcePolicyRevision: "f".repeat(64), + }); + expect(retained).toMatchObject({ + execution: "sync", + value: { proposalId, proposed: true, catalogRevisionId: candidate.generationId }, + }); + const reviewed = await client.execute({ + kind: "transitousLock.inspect", + proposalId, + offset: 0, + }); + expect(reviewed).toMatchObject({ + execution: "sync", + value: { + proposal: { proposalId, state: "proposed", pins: candidate }, + page: { total: 1, nextOffset: null }, + }, + }); + expect(await client.execute({ kind: "catalogPins.inspect" })).toMatchObject({ + value: { active: expected }, + }); + } } expect(existsSync(join(infra, "transitous.lock.proposed.json"))).toBe(false); } finally { diff --git a/apps/ops-agent/src/catalog-validation-inputs.ts b/apps/ops-agent/src/catalog-validation-inputs.ts new file mode 100644 index 000000000..2d1ffa59d --- /dev/null +++ b/apps/ops-agent/src/catalog-validation-inputs.ts @@ -0,0 +1,161 @@ +/** Fixed, read-only host observations required by catalog validation/publication. */ +import { join } from "node:path"; +import { catalogRuntimeInputsSchema } from "@openmapx/core/catalog-pins"; +import { canonicalJson } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { + catalogCompilerSettings, + catalogReviewInputDigests, + resolveOperationsProfileFromEnv, +} from "@openmapx/transitous-core"; +import { AnchoredDirectory } from "../../../packages/core/src/build-artifacts/filesystem"; +import { safeArtifactPath } from "../../../packages/core/src/build-artifacts/manifest"; +import { parseFeedOverlay } from "../../../services/data-manager/src/jobs/transitous-feeds-overlay"; +import { type DockerRuntimeOptions, runContainedProcess } from "./docker-runtime"; +import type { OwnedServiceResolver } from "./owned-services"; +import type { OpsRuntime } from "./runtime"; +import { inspectAgentPinnedCredentialKeys } from "./transit-credentials-runtime"; +export function installCatalogValidationInputsRuntime( + runtime: OpsRuntime, + options: { + rootDir: string; + stack: services.StackPaths; + ownedServices: OwnedServiceResolver; + execFile?: DockerRuntimeOptions["execFile"]; + }, +) { + const inspect = runtime["transitousLock.inspect"], + docker = options.execFile ?? runContainedProcess; + runtime["transitousLock.inspect"] = async (operation, context) => { + const result = await inspect(operation, context); + if (!result.proposal) return result; + try { + const generation = services.currentConfigurationGeneration(options.stack.infraDir); + if (!generation) throw Error("Apply the instance configuration before catalog validation"); + const compose = JSON.parse( + ( + await docker( + "docker", + [...services.stackComposeArgs(options.stack), "config", "--format", "json"], + { signal: context.signal, timeout: 30000, maxBuffer: 1024 * 1024 }, + ) + ).stdout, + ); + const environment = compose.services?.["data-manager"]?.environment; + if (!environment || typeof environment !== "object" || Array.isArray(environment)) + throw Error("Data Manager policy is unavailable"); + const policy = resolveOperationsProfileFromEnv(environment); + if ( + environment.MOTIS_TWO_SLOT !== "true" || + environment.MOTIS_GBFS_CATALOG_ENABLED !== "true" + ) + throw Error("Enable paired two-slot catalog validation"); + const owned = await options.ownedServices.inspectOwnedServices(context.signal), + dm = owned.filter( + (entry) => entry.serviceId === "data-manager" && entry.state === "running", + ), + primary = owned.filter((entry) => entry.serviceId === "motis" && entry.state === "running"), + staging = owned.filter((entry) => entry.serviceId === "motis-staging"); + if ( + dm.length !== 1 || + primary.length !== 1 || + staging.length !== 1 || + primary[0]!.provenance.imageId !== staging[0]!.provenance.imageId + ) + throw Error( + "Run the owned Data Manager and primary MOTIS with the same configured staging image before validation", + ); + const observed = JSON.parse( + ( + await docker( + "docker", + ["container", "inspect", "--format", "{{json .}}", dm[0]!.containerId], + { signal: context.signal, timeout: 15000, maxBuffer: 256 * 1024 }, + ) + ).stdout, + ); + const runningEnvironment = Object.fromEntries( + (observed?.Config?.Env ?? []).map((entry: string) => { + const equals = entry.indexOf("="); + return [entry.slice(0, equals), entry.slice(equals + 1)]; + }), + ); + if ( + runningEnvironment.MOTIS_TWO_SLOT !== "true" || + runningEnvironment.MOTIS_GBFS_CATALOG_ENABLED !== "true" + ) + throw Error("Apply paired catalog configuration before validation"); + if ( + canonicalJson(resolveOperationsProfileFromEnv(runningEnvironment)) !== + canonicalJson(policy) || + canonicalJson(catalogCompilerSettings(runningEnvironment)) !== + canonicalJson(catalogCompilerSettings(environment)) + ) + throw Error("Apply pending Data Manager policy changes before validation"); + if ( + environment.TRANSITOUS_FEEDS_OVERLAY_PATH && + !String(environment.TRANSITOUS_FEEDS_OVERLAY_PATH).startsWith("/data/") + ) + throw Error("The host-owned source overlay cannot be verified"); + const relative = environment.TRANSITOUS_FEEDS_OVERLAY_PATH + ? String(environment.TRANSITOUS_FEEDS_OVERLAY_PATH).replace(/^\/data\//, "") + : "overrides/feeds-overlay.json"; + if (!safeArtifactPath(relative)) + throw Error("The host-owned source overlay cannot be verified"); + const opened: AnchoredDirectory[] = []; + let overlay = null; + try { + let root = new AnchoredDirectory(join(options.stack.infraDir, "data")); + opened.push(root); + const parts = relative.split("/"); + for (const name of parts.slice(0, -1)) { + root = root.directory(name); + opened.push(root); + } + overlay = parseFeedOverlay( + JSON.parse(root.read(parts.at(-1)!, 4 * 1024 * 1024, false).toString("utf8")), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } finally { + for (const root of opened.reverse()) root.close(); + } + const credentials = await runtime["transitCredentials.inspect"]( + { + kind: "transitCredentials.inspect", + proposalId: result.proposal.proposalId, + catalogRevisionId: result.proposal.pins.generationId, + }, + context, + ); + if (!credentials.consumedCredentialsRevision) + throw Error("The running Data Manager credential mount cannot be verified"); + const requiredKeys = await inspectAgentPinnedCredentialKeys( + options.stack.infraDir, + result.proposal.pins, + context.signal, + { countries: policy.countries, patches: overlay?.patches ?? [] }, + ); + if (services.currentConfigurationGeneration(options.stack.infraDir) !== generation) + throw Error("Configuration changed during catalog inspection"); + return { + ...result, + validationInputs: catalogRuntimeInputsSchema.parse({ + configurationRevisionId: generation, + credentialsRevision: credentials.consumedCredentialsRevision, + ...catalogReviewInputDigests(policy, overlay, catalogCompilerSettings(environment)), + imageDigest: staging[0]!.provenance.imageId, + stagingState: staging[0]!.state, + requiredKeys, + configuredKeys: credentials.configuredKeys, + }), + }; + } catch { + return { + ...result, + validationBlocker: + "Catalog validation needs a verified current policy, required credentials, running Data Manager/primary MOTIS and matching owned staging image. Apply pending configuration and inspect Services.", + }; + } + }; +} diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index be5c6f6e8..1857865e6 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -1067,8 +1067,17 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done }); await dispatchOpsOperation(runtime, { kind: "motis.staging.restart" }, context()); await dispatchOpsOperation(runtime, { kind: "motis.primary.stop" }, context()); - expect(calls).toEqual([ - ["restart", "motis-staging"], + await dispatchOpsOperation(runtime, { kind: "motis.primary.restart" }, context()); + expect(calls.at(-1)!.slice(-5)).toEqual(["up", "-d", "--force-recreate", "--no-deps", "motis"]); + expect(calls[0]).toContain("compose"); + expect(calls[0]!.slice(-5)).toEqual([ + "up", + "-d", + "--force-recreate", + "--no-deps", + "motis-staging", + ]); + expect(calls.slice(1, -1)).toEqual([ ["container", "ls", "-a", "--filter", "name=^/motis$", "--format", "{{.Names}}\t{{.State}}"], ["stop", "motis"], ["container", "ls", "-a", "--filter", "name=^/motis$", "--format", "{{.Names}}\t{{.State}}"], diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 17015e55f..184df1ef8 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -1069,10 +1069,13 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { }; runtime["motis.staging.restart"] = (_operation, context) => - changed(["restart", "motis-staging"], context); + changed( + [...composePrefix(), "up", "-d", "--force-recreate", "--no-deps", "motis-staging"], + context, + ); runtime["motis.staging.stop"] = (_operation, context) => stopMotis("motis-staging", context); runtime["motis.primary.restart"] = (_operation, context) => - changed(["restart", "motis"], context); + changed([...composePrefix(), "up", "-d", "--force-recreate", "--no-deps", "motis"], context); runtime["motis.primary.promote"] = async (operation, context) => { assertTransitCandidateEffect(operation, context); // Recreate rather than restart: Docker resolves a bind-mount symlink when diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 433110224..34b7b01fe 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -10,6 +10,8 @@ import { type FixedCliRunner, } from "./administrative-runtime"; import { isTerminalHandoff, readAgentHandoff } from "./agent-handoff"; +import { installCatalogApprovalRuntime } from "./catalog-approval-runtime"; +import { installCatalogValidationInputsRuntime } from "./catalog-validation-inputs"; import { type CommunityRepositoryStore, installCommunityRepositoryRuntime, @@ -237,6 +239,13 @@ export async function createProductionOpsRuntime( ownedServices: owned, execFile: options.docker?.execFile, }); + installCatalogValidationInputsRuntime(runtime, { + rootDir: options.rootDir, + stack: options.stack, + ownedServices: owned, + execFile: options.docker?.execFile, + }); + installCatalogApprovalRuntime(runtime, { rootDir: options.rootDir }); createAdministrativeRuntime(runtime, { rootDir: options.rootDir, runFixedCli, diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index ff8bc3a48..0fe4f47ce 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -95,6 +95,7 @@ import { } from "@openmapx/core/ops"; import Fastify, { type FastifyInstance } from "fastify"; import { inspectConfigurationGenerationNonCommit } from "../../../packages/core/src/build-artifacts/control-authority"; +import { readCommittedCatalogApproval } from "./catalog-approval-runtime"; import { prepareConfigurationHardlinksAdmission } from "./configuration-hardlinks-runtime"; import { configurationMaintenanceAdmissionState, @@ -742,6 +743,30 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } const lease = controlLeases.get(job.operationId); const authority = controlAuthorities.get(job.operationId); + if ( + authority && + lease && + job.operation.kind === "transitousLock.approve" && + ["succeeded", "failed", "timed_out"].includes(state) + ) { + const published = await readCommittedCatalogApproval( + runtime, + authority, + lease, + job.operation, + ); + if (published) { + state = "succeeded"; + updatedAt = new Date(published.terminalAt).toISOString(); + updates = { + ...updates, + result: published.result, + terminalAt: published.terminalAt, + errorClass: undefined, + failureResult: undefined, + }; + } + } let releaseUnstartedRoot = false; if (authority && state === "termination_pending") cancelControlExecution(authority); if (lease && state === "termination_pending") await cancelArtifactTargetLease(lease); @@ -1735,6 +1760,44 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ); controlLeases.set(job.operationId, lease); controlAuthorities.set(job.operationId, authority); + if (job.operation.kind === "transitousLock.approve") { + const published = await readCommittedCatalogApproval( + runtime, + authority, + lease, + job.operation, + ); + if (published) { + retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "ops-result", result: published.result }, + }); + const restored: JobRecord = { + ...job, + state: "succeeded", + result: published.result, + errorClass: undefined, + failureResult: undefined, + terminalAt: published.terminalAt, + updatedAt: new Date(published.terminalAt).toISOString(), + }; + // A previously sealed, different canonical tuple refuses here; + // an unknown secondary journal never overrides that source. + restored.terminalAt = retainControlTerminalCommitment( + authority, + statusFor(restored, 2, true), + published.terminalAt, + ); + await journal.replace( + [...jobsById.values()].map((value) => + persisted(value === job ? restored : value), + ), + ); + Object.assign(job, restored); + changed = true; + } + } await settleArtifactTargetLeaseChild( lease, verifyControlEffectSettlement(options.controlInstallation, execution), @@ -1839,898 +1902,920 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif await reconcileControlRetention(options.controlInstallation).catch(() => undefined); }); - app.post("/v1/operations", async (request, reply) => { - const startedAt = Date.now(); - const requestId = requestIdFromBody(request.body); - const role = resolveRole(request.headers.authorization, options.tokens); - if (!role) return reply.code(401).send(redactedOpsError(requestId, "authentication")); - let kind = operationKindFromBody(request.body); - let resourceId: string = kind; - try { - const parsed = parseOpsRequest(request.body, { role }); - const operation = immutableSnapshot(parsed.operation); - kind = operation.kind; - resourceId = opsResourceId(operation); - const nowMs = now().getTime(); - const timing = validateTiming(parsed.issuedAt, parsed.expiresAt, nowMs); - replay.accept(parsed.requestId, timing.expiresAtMs, nowMs); - const policy = OPS_KIND_POLICIES[operation.kind]; - // Read-only historical records can neither reserve a fresh normal key - // nor turn an unresolved old effect into permission for another writer. - if ( - historicalPulls.some((job) => job.role === role && job.operationKey === parsed.operationKey) - ) - throw new RequestGuardError("conflict", 409); - if ( - opsKindRequiresControlAuthority(operation.kind) && - historicalPulls.some( - (job) => - ["queued", "running", "termination_pending"].includes(job.state) || - job.errorClass === "recovery_required", + app.post( + "/v1/operations", + { bodyLimit: options.bodyLimit ?? 772 * 1024 }, + async (request, reply) => { + const startedAt = Date.now(); + const requestId = requestIdFromBody(request.body); + const role = resolveRole(request.headers.authorization, options.tokens); + if (!role) return reply.code(401).send(redactedOpsError(requestId, "authentication")); + let kind = operationKindFromBody(request.body); + let resourceId: string = kind; + try { + const parsed = parseOpsRequest(request.body, { role }); + const requestLimit = + parsed.operation.kind === "transitousLock.propose" + ? 772 * 1024 + : (options.bodyLimit ?? DEFAULT_BODY_LIMIT); + if (Buffer.byteLength(JSON.stringify(request.body), "utf8") > requestLimit) + throw new RequestGuardError("validation", 413); + const operation = immutableSnapshot(parsed.operation); + kind = operation.kind; + resourceId = opsResourceId(operation); + const nowMs = now().getTime(); + const timing = validateTiming(parsed.issuedAt, parsed.expiresAt, nowMs); + replay.accept(parsed.requestId, timing.expiresAtMs, nowMs); + const policy = OPS_KIND_POLICIES[operation.kind]; + // Read-only historical records can neither reserve a fresh normal key + // nor turn an unresolved old effect into permission for another writer. + if ( + historicalPulls.some( + (job) => job.role === role && job.operationKey === parsed.operationKey, + ) ) - ) - throw new RequestGuardError("recovery_required", 409); - const operationFingerprint = opsOperationFingerprint(operation); - if (operation.kind === "controlRetention.acknowledge") { - if (role !== "api" || !options.controlInstallation) - throw new RequestGuardError("authorization", 403); - try { - const existing = jobsByKey.get(`api:${operation.operationKey}`); - if ( - existing && - controlTerminalReceiptDigest(statusFor(existing, 2)) !== operation.terminalDigest + throw new RequestGuardError("conflict", 409); + if ( + opsKindRequiresControlAuthority(operation.kind) && + historicalPulls.some( + (job) => + ["queued", "running", "termination_pending"].includes(job.state) || + job.errorClass === "recovery_required", ) - throw Error("Current terminal digest conflict"); - const phase = - existing && - (isTrustedConfigurationApplyOperation(existing.operation) || - existing.operation.kind === "configuration.hardlinks.apply" || - existing.operation.kind === "service.recreate" || - existing.operation.kind === "service.recreateIsolated") && - existing.operation.maintenanceId; - if (phase && existing.terminalAt !== undefined) - recordOwnedConfigurationMaintenanceStepTerminal( - options.controlInstallation, - existing.operation, - executionFor(existing), - statusFor(existing, 2), - existing.terminalAt, - ); - const value = phase - ? acknowledgeConfigurationMaintenanceStep( + ) + throw new RequestGuardError("recovery_required", 409); + const operationFingerprint = opsOperationFingerprint(operation); + if (operation.kind === "controlRetention.acknowledge") { + if (role !== "api" || !options.controlInstallation) + throw new RequestGuardError("authorization", 403); + try { + const existing = jobsByKey.get(`api:${operation.operationKey}`); + if ( + existing && + controlTerminalReceiptDigest(statusFor(existing, 2)) !== operation.terminalDigest + ) + throw Error("Current terminal digest conflict"); + const phase = + existing && + (isTrustedConfigurationApplyOperation(existing.operation) || + existing.operation.kind === "configuration.hardlinks.apply" || + existing.operation.kind === "service.recreate" || + existing.operation.kind === "service.recreateIsolated") && + existing.operation.maintenanceId; + if (phase && existing.terminalAt !== undefined) + recordOwnedConfigurationMaintenanceStepTerminal( options.controlInstallation, - operation, - options.tokens.api, - ) - : operation.originRootKey !== operation.operationKey && - (existing?.operation.kind === "configurationMaintenance.decide" || - existing?.operation.kind === "configurationMaintenance.settle") - ? acknowledgeConfigurationMaintenanceControl( + existing.operation, + executionFor(existing), + statusFor(existing, 2), + existing.terminalAt, + ); + const value = phase + ? acknowledgeConfigurationMaintenanceStep( options.controlInstallation, operation, options.tokens.api, ) - : acknowledgeControlRetention( - options.controlInstallation, - operation, - options.tokens.api, - ); - return reply - .code(200) - .send(opsSuccess(parsed.requestId, { execution: "sync", kind: operation.kind, value })); - } catch { - throw new RequestGuardError("recovery_required", 409); + : operation.originRootKey !== operation.operationKey && + (existing?.operation.kind === "configurationMaintenance.decide" || + existing?.operation.kind === "configurationMaintenance.settle") + ? acknowledgeConfigurationMaintenanceControl( + options.controlInstallation, + operation, + options.tokens.api, + ) + : acknowledgeControlRetention( + options.controlInstallation, + operation, + options.tokens.api, + ); + return reply + .code(200) + .send( + opsSuccess(parsed.requestId, { execution: "sync", kind: operation.kind, value }), + ); + } catch { + throw new RequestGuardError("recovery_required", 409); + } } - } - if ( - operation.kind === "service.pull" && - operation.serviceId === "data-manager" && - !operation.candidateId - ) - throw new OpsExecutionFailure("configuration_prerequisite", {}); - let parent: VerifiedParentExecution | undefined; - let configurationMaintenanceControl: - | import("./configuration-maintenance-store").ConfigurationMaintenanceControlOwner - | undefined; - let maintenanceStepAdmission: - | Awaited> - | undefined; - let controlAdmission: - | ReturnType - | Awaited> - | undefined; - const parentHeader = request.headers["x-openmapx-parent-execution"]; - const delegationHeader = request.headers["x-openmapx-control-delegation"]; - // Approval is exclusively a borrowed activation leaf. Installing its - // future validation runtime must never enable a bearer-only root path. - if (operation.kind === "transitousLock.approve" && delegationHeader === undefined) - throw new RequestGuardError("authorization", 403); - try { - readControlHeaders(request.raw.rawHeaders); - } catch { - throw new RequestGuardError("authorization", 403); - } - if (delegationHeader !== undefined) { if ( - !options.controlInstallation || - role !== "data-manager" || - parsed.version !== 2 || - parentHeader !== undefined + operation.kind === "service.pull" && + operation.serviceId === "data-manager" && + !operation.candidateId ) + throw new OpsExecutionFailure("configuration_prerequisite", {}); + let parent: VerifiedParentExecution | undefined; + let configurationMaintenanceControl: + | import("./configuration-maintenance-store").ConfigurationMaintenanceControlOwner + | undefined; + let maintenanceStepAdmission: + | Awaited> + | undefined; + let controlAdmission: + | ReturnType + | Awaited> + | undefined; + const parentHeader = request.headers["x-openmapx-parent-execution"]; + const delegationHeader = request.headers["x-openmapx-control-delegation"]; + // Approval is exclusively a borrowed activation leaf. Installing its + // future validation runtime must never enable a bearer-only root path. + if (operation.kind === "transitousLock.approve" && delegationHeader === undefined) throw new RequestGuardError("authorization", 403); try { - controlAdmission = inspectControlAdmission(options.controlInstallation, { - rawHeaders: request.raw.rawHeaders, - transportPrincipal: "data-manager", - kind, - operation, - operationKey: parsed.operationKey, - operationFingerprint, - requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, - }); - parent = controlAdmission.parent; - } catch { - throw new RequestGuardError("authorization", 403); - } - } - if (parsed.version === 1 && (parentHeader !== undefined || delegationHeader !== undefined)) - throw new RequestGuardError("authorization", 403); - if (parentHeader !== undefined) { - if (role !== "api" || typeof parentHeader !== "string") - throw new RequestGuardError("authorization", 403); - try { - parent = verifyApiJobParentExecution(parentHeader, options.tokens.api, { - operationKey: parsed.operationKey, - operationFingerprint, - requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, - }); + readControlHeaders(request.raw.rawHeaders); } catch { throw new RequestGuardError("authorization", 403); } - } - if ( - operation.kind === "configurationMaintenance.inspect" || - operation.kind === "configurationMaintenance.decide" || - operation.kind === "configurationMaintenance.settle" - ) { - if (!options.controlInstallation || !parent) - throw new RequestGuardError("authorization", 403); - try { - configurationMaintenanceControl = admitConfigurationMaintenanceControlOwner( - options.controlInstallation, - operation, - parent, - { + if (delegationHeader !== undefined) { + if ( + !options.controlInstallation || + role !== "data-manager" || + parsed.version !== 2 || + parentHeader !== undefined + ) + throw new RequestGuardError("authorization", 403); + try { + controlAdmission = inspectControlAdmission(options.controlInstallation, { + rawHeaders: request.raw.rawHeaders, + transportPrincipal: "data-manager", + kind, + operation, operationKey: parsed.operationKey, + operationFingerprint, requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, - }, - ); - } catch (error) { - if (error instanceof ConfigurationMaintenanceControlUnauthorized) + }); + parent = controlAdmission.parent; + } catch { throw new RequestGuardError("authorization", 403); - throw new RequestGuardError( - (error as NodeJS.ErrnoException).code === "ENOENT" ? "not_found" : "recovery_required", - (error as NodeJS.ErrnoException).code === "ENOENT" ? 404 : 409, - ); + } } - } - if ( - (isTrustedConfigurationApplyOperation(operation) || - operation.kind === "configuration.hardlinks.apply" || - operation.kind === "service.recreate" || - operation.kind === "service.recreateIsolated") && - operation.maintenanceId - ) { - if ( - !options.controlInstallation || - role !== "api" || - !parent || - typeof parentHeader !== "string" - ) + if (parsed.version === 1 && (parentHeader !== undefined || delegationHeader !== undefined)) throw new RequestGuardError("authorization", 403); - try { - maintenanceStepAdmission = await prepareOwnedConfigurationMaintenanceStepAdmission( - runtime, - options.controlInstallation, - operation, - parent, - { + if (parentHeader !== undefined) { + if (role !== "api" || typeof parentHeader !== "string") + throw new RequestGuardError("authorization", 403); + try { + parent = verifyApiJobParentExecution(parentHeader, options.tokens.api, { operationKey: parsed.operationKey, + operationFingerprint, requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, - parentAssertion: parentHeader, - signal: AbortSignal.timeout(15000), - }, - ); - controlAdmission = maintenanceStepAdmission; - parent = maintenanceStepAdmission.parent; - } catch (error) { - if (error instanceof ConfigurationMaintenanceControlUnauthorized) + }); + } catch { throw new RequestGuardError("authorization", 403); - if (error instanceof ConfigurationMaintenanceControlConflict) - throw new RequestGuardError("recovery_required", 409); - throw error; + } } - } - const normalize = ( - originalAdmittedAt: number, - deploymentBudgetMs = Math.min( - policy.timeoutMs, - options.operationTimeoutMs ?? policy.timeoutMs, - ), - ) => { - try { - if (maintenanceStepAdmission) - return { - admittedAt: maintenanceStepAdmission.boundAt, - deadlineAt: maintenanceStepAdmission.execution.deadlineAt, - fingerprint: maintenanceStepAdmission.execution.fingerprint, - }; - if ( - operation.kind === "configurationMaintenance.inspect" && - configurationMaintenanceControl - ) - return normalizeConfigurationMaintenanceInspection( - configurationMaintenanceControl, + if ( + operation.kind === "configurationMaintenance.inspect" || + operation.kind === "configurationMaintenance.decide" || + operation.kind === "configurationMaintenance.settle" + ) { + if (!options.controlInstallation || !parent) + throw new RequestGuardError("authorization", 403); + try { + configurationMaintenanceControl = admitConfigurationMaintenanceControlOwner( + options.controlInstallation, operation, - parsed.operationKey, - originalAdmittedAt, - deploymentBudgetMs, + parent, + { + operationKey: parsed.operationKey, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + }, + ); + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + throw new RequestGuardError( + (error as NodeJS.ErrnoException).code === "ENOENT" + ? "not_found" + : "recovery_required", + (error as NodeJS.ErrnoException).code === "ENOENT" ? 404 : 409, ); + } + } + if ( + (isTrustedConfigurationApplyOperation(operation) || + operation.kind === "configuration.hardlinks.apply" || + operation.kind === "service.recreate" || + operation.kind === "service.recreateIsolated") && + operation.maintenanceId + ) { if ( - configurationMaintenanceControl && - (operation.kind === "configurationMaintenance.settle" || - (operation.kind === "configurationMaintenance.decide" && - operation.decision === "abandon")) + !options.controlInstallation || + role !== "api" || + !parent || + typeof parentHeader !== "string" ) - return normalizeConfigurationMaintenanceRecoveryControl( - configurationMaintenanceControl, + throw new RequestGuardError("authorization", 403); + try { + maintenanceStepAdmission = await prepareOwnedConfigurationMaintenanceStepAdmission( + runtime, + options.controlInstallation, operation, - parsed.operationKey, - originalAdmittedAt, - deploymentBudgetMs, + parent, + { + operationKey: parsed.operationKey, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + parentAssertion: parentHeader, + signal: AbortSignal.timeout(15000), + }, ); - return normalizeExecutionAdmission({ - kind, - operationKey: parsed.operationKey, - operationFingerprint, - admittedAt: originalAdmittedAt, - requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, - parent, - deploymentBudgetMs, - }); - } catch { - throw new RequestGuardError("validation", 400); - } - }; - let normalized: ReturnType; - const fingerprint = operationFingerprint; // Existing sealed claims bind the body, never execution timing. - const matchesRetained = (job: JobRecord): boolean => { - try { - return job.executionVersion === 2 - ? job.fingerprint === - normalize(job.admittedAt as number, job.deploymentBudgetMs).fingerprint - : job.fingerprint === operationFingerprint && parsed.execution === undefined && !parent; - } catch { - return false; + controlAdmission = maintenanceStepAdmission; + parent = maintenanceStepAdmission.parent; + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + if (error instanceof ConfigurationMaintenanceControlConflict) + throw new RequestGuardError("recovery_required", 409); + throw error; + } } - }; - if (policy.execution === "async") { - await pruneJobs(nowMs); - const ownerKey = `${role}:${parsed.operationKey}`; - const existing = jobsByKey.get(ownerKey); - if (existing) { - if (!matchesRetained(existing)) throw new RequestGuardError("conflict", 409); - return reply - .code(202) - .send(opsSuccess(parsed.requestId, admissionFor(existing, parsed.version))); + const normalize = ( + originalAdmittedAt: number, + deploymentBudgetMs = Math.min( + policy.timeoutMs, + options.operationTimeoutMs ?? policy.timeoutMs, + ), + ) => { + try { + if (maintenanceStepAdmission) + return { + admittedAt: maintenanceStepAdmission.boundAt, + deadlineAt: maintenanceStepAdmission.execution.deadlineAt, + fingerprint: maintenanceStepAdmission.execution.fingerprint, + }; + if ( + operation.kind === "configurationMaintenance.inspect" && + configurationMaintenanceControl + ) + return normalizeConfigurationMaintenanceInspection( + configurationMaintenanceControl, + operation, + parsed.operationKey, + originalAdmittedAt, + deploymentBudgetMs, + ); + if ( + configurationMaintenanceControl && + (operation.kind === "configurationMaintenance.settle" || + (operation.kind === "configurationMaintenance.decide" && + operation.decision === "abandon")) + ) + return normalizeConfigurationMaintenanceRecoveryControl( + configurationMaintenanceControl, + operation, + parsed.operationKey, + originalAdmittedAt, + deploymentBudgetMs, + ); + return normalizeExecutionAdmission({ + kind, + operationKey: parsed.operationKey, + operationFingerprint, + admittedAt: originalAdmittedAt, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + parent, + deploymentBudgetMs, + }); + } catch { + throw new RequestGuardError("validation", 400); + } + }; + let normalized: ReturnType; + const fingerprint = operationFingerprint; // Existing sealed claims bind the body, never execution timing. + const matchesRetained = (job: JobRecord): boolean => { + try { + return job.executionVersion === 2 + ? job.fingerprint === + normalize(job.admittedAt as number, job.deploymentBudgetMs).fingerprint + : job.fingerprint === operationFingerprint && + parsed.execution === undefined && + !parent; + } catch { + return false; + } + }; + if (policy.execution === "async") { + await pruneJobs(nowMs); + const ownerKey = `${role}:${parsed.operationKey}`; + const existing = jobsByKey.get(ownerKey); + if (existing) { + if (!matchesRetained(existing)) throw new RequestGuardError("conflict", 409); + return reply + .code(202) + .send(opsSuccess(parsed.requestId, admissionFor(existing, parsed.version))); + } + if (options.controlInstallation) { + try { + await reconcileControlRetention(options.controlInstallation, { maxFamilies: 1 }); + } catch { + throw new RequestGuardError("recovery_required", 409); + } + } + if (jobsById.size + historicalPulls.length >= maxJobEntries) { + throw new RequestGuardError("busy", 429); + } + // A resumed agent update re-attaches above; nothing new may start. + if (operation.kind !== "agent.update" && options.replacementInProgress?.()) { + throw new RequestGuardError("busy", 409); + } } - if (options.controlInstallation) { + if (options.releaseRecovery?.ready === false && !RELEASE_RECOVERY_READ_KINDS.has(kind)) + throw new RequestGuardError("recovery_required", 409); + if ( + opsKindRequiresControlAuthority(operation.kind) && + !controlAdmission && + !configurationMaintenanceControl + ) { + if (!options.controlInstallation) throw new RequestGuardError("recovery_required", 409); + try { + const key = parseControlRootOperationKey(parsed.operationKey); + if (key.notAfter !== parsed.execution?.requestedDeadlineAt) throw Error(); + } catch { + throw new RequestGuardError("control_upgrade_required", 409); + } try { - await reconcileControlRetention(options.controlInstallation, { maxFamilies: 1 }); + assertFreshControlRoot( + options.controlInstallation, + parsed.operationKey, + parsed.execution?.requestedDeadlineAt, + ); } catch { throw new RequestGuardError("recovery_required", 409); } } - if (jobsById.size + historicalPulls.length >= maxJobEntries) { - throw new RequestGuardError("busy", 429); - } - // A resumed agent update re-attaches above; nothing new may start. - if (operation.kind !== "agent.update" && options.replacementInProgress?.()) { - throw new RequestGuardError("busy", 409); - } - } - if (options.releaseRecovery?.ready === false && !RELEASE_RECOVERY_READ_KINDS.has(kind)) - throw new RequestGuardError("recovery_required", 409); - if ( - opsKindRequiresControlAuthority(operation.kind) && - !controlAdmission && - !configurationMaintenanceControl - ) { - if (!options.controlInstallation) throw new RequestGuardError("recovery_required", 409); + normalized = normalize(nowMs); + if (active >= maxConcurrency) throw new RequestGuardError("busy", 429); + active += 1; + const claimController = new AbortController(); + const claimExecution = Promise.resolve().then(() => + resourceClaimer.claim(operation, fingerprint, claimController.signal, { + role, + operationKey: parsed.operationKey, + ...(maintenanceStepAdmission ? { maintenanceStep: maintenanceStepAdmission } : {}), + }), + ); + let claimTimer: ReturnType | undefined; + let claim: Awaited>; try { - const key = parseControlRootOperationKey(parsed.operationKey); - if (key.notAfter !== parsed.execution?.requestedDeadlineAt) throw Error(); - } catch { - throw new RequestGuardError("control_upgrade_required", 409); + claim = await Promise.race([ + claimExecution, + new Promise((_resolve, reject) => { + claimTimer = setTimeout(() => { + claimController.abort(); + reject(new RequestGuardError("timeout", 504)); + }, options.claimTimeoutMs ?? DEFAULT_CLAIM_TIMEOUT_MS); + }), + ]); + if (claimTimer !== undefined) clearTimeout(claimTimer); + } catch (error) { + if (claimTimer !== undefined) clearTimeout(claimTimer); + if (error instanceof RequestGuardError && error.errorClass === "timeout") { + void claimExecution.then( + (lateClaim) => { + void lateClaim?.admission?.rollback().catch(() => undefined); + active -= 1; + audit(role, kind, resourceId, "late_completion", startedAt, "timeout"); + }, + () => { + active -= 1; + audit(role, kind, resourceId, "terminated", startedAt, "timeout"); + }, + ); + audit(role, kind, resourceId, "timeout", startedAt, "timeout"); + return reply.code(504).send(redactedOpsError(parsed.requestId, "timeout")); + } + active -= 1; + throw error; } + let claimAdmission: OpsTrustedClaim["admission"]; + let coordinatorCapture: CoordinatorPullOwnershipObservation | undefined; try { - assertFreshControlRoot( - options.controlInstallation, - parsed.operationKey, - parsed.execution?.requestedDeadlineAt, - ); + if ( + !claim || + claim.fingerprint !== fingerprint || + opsOperationFingerprint(claim.operation as OpsOperation) !== fingerprint || + !claim.capability || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(claim.capability.revisionId) || + !claim.capability.values || + typeof claim.capability.values !== "object" || + Array.isArray(claim.capability.values) || + Object.keys(claim.capability.values).length > 64 || + Object.entries(claim.capability.values).some( + ([key, value]) => + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(key) || + (value !== null && !["string", "number", "boolean"].includes(typeof value)) || + (typeof value === "number" && !Number.isFinite(value)), + ) || + Buffer.byteLength(JSON.stringify(claim.capability), "utf8") > 512 * 1024 + ) { + throw new OpsContractError("authorization", "Operation resource is not permitted"); + } + if (operation.kind === "service.pull" && operation.candidateId) + coordinatorCapture = consumeCoordinatorPullOwnershipCapture(claim, { + operationFingerprint: fingerprint, + candidateId: operation.candidateId, + policyRevision: claim.capability.revisionId, + }); + claimAdmission = claim.admission; + claim = immutableSnapshot({ + fingerprint: claim.fingerprint, + operation: claim.operation, + source: claim.source, + capability: claim.capability, + }); } catch { - throw new RequestGuardError("recovery_required", 409); - } - } - normalized = normalize(nowMs); - if (active >= maxConcurrency) throw new RequestGuardError("busy", 429); - active += 1; - const claimController = new AbortController(); - const claimExecution = Promise.resolve().then(() => - resourceClaimer.claim(operation, fingerprint, claimController.signal, { - role, - operationKey: parsed.operationKey, - ...(maintenanceStepAdmission ? { maintenanceStep: maintenanceStepAdmission } : {}), - }), - ); - let claimTimer: ReturnType | undefined; - let claim: Awaited>; - try { - claim = await Promise.race([ - claimExecution, - new Promise((_resolve, reject) => { - claimTimer = setTimeout(() => { - claimController.abort(); - reject(new RequestGuardError("timeout", 504)); - }, options.claimTimeoutMs ?? DEFAULT_CLAIM_TIMEOUT_MS); - }), - ]); - if (claimTimer !== undefined) clearTimeout(claimTimer); - } catch (error) { - if (claimTimer !== undefined) clearTimeout(claimTimer); - if (error instanceof RequestGuardError && error.errorClass === "timeout") { - void claimExecution.then( - (lateClaim) => { - void lateClaim?.admission?.rollback().catch(() => undefined); - active -= 1; - audit(role, kind, resourceId, "late_completion", startedAt, "timeout"); - }, - () => { - active -= 1; - audit(role, kind, resourceId, "terminated", startedAt, "timeout"); - }, - ); - audit(role, kind, resourceId, "timeout", startedAt, "timeout"); - return reply.code(504).send(redactedOpsError(parsed.requestId, "timeout")); - } - active -= 1; - throw error; - } - let claimAdmission: OpsTrustedClaim["admission"]; - let coordinatorCapture: CoordinatorPullOwnershipObservation | undefined; - try { - if ( - !claim || - claim.fingerprint !== fingerprint || - opsOperationFingerprint(claim.operation as OpsOperation) !== fingerprint || - !claim.capability || - !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(claim.capability.revisionId) || - !claim.capability.values || - typeof claim.capability.values !== "object" || - Array.isArray(claim.capability.values) || - Object.keys(claim.capability.values).length > 64 || - Object.entries(claim.capability.values).some( - ([key, value]) => - !/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(key) || - (value !== null && !["string", "number", "boolean"].includes(typeof value)) || - (typeof value === "number" && !Number.isFinite(value)), - ) || - Buffer.byteLength(JSON.stringify(claim.capability), "utf8") > 512 * 1024 - ) { + await claim?.admission?.rollback().catch(() => undefined); + active -= 1; throw new OpsContractError("authorization", "Operation resource is not permitted"); } - if (operation.kind === "service.pull" && operation.candidateId) - coordinatorCapture = consumeCoordinatorPullOwnershipCapture(claim, { - operationFingerprint: fingerprint, - candidateId: operation.candidateId, - policyRevision: claim.capability.revisionId, - }); - claimAdmission = claim.admission; - claim = immutableSnapshot({ - fingerprint: claim.fingerprint, - operation: claim.operation, - source: claim.source, - capability: claim.capability, - }); - } catch { - await claim?.admission?.rollback().catch(() => undefined); - active -= 1; - throw new OpsContractError("authorization", "Operation resource is not permitted"); - } - if (policy.execution === "async") { - const ownerKey = `${role}:${parsed.operationKey}`; - const admission = await withAdmissionLock(async () => { - const retained = jobsByKey.get(ownerKey); - if (retained) { - if (!matchesRetained(retained)) { + if (policy.execution === "async") { + const ownerKey = `${role}:${parsed.operationKey}`; + const admission = await withAdmissionLock(async () => { + const retained = jobsByKey.get(ownerKey); + if (retained) { + if (!matchesRetained(retained)) { + throw new RequestGuardError("conflict", 409); + } + return { type: "retained", job: retained } as const; + } + if (jobsById.size + historicalPulls.length >= maxJobEntries) + throw new RequestGuardError("busy", 429); + if (normalized.deadlineAt <= now().getTime()) + throw new RequestGuardError("timeout", 504); + const timestamp = new Date(normalized.admittedAt).toISOString(); + let operationId: string | undefined = controlAdmission?.execution.executionId; + if (operationId && jobsById.has(operationId)) throw new RequestGuardError("conflict", 409); + try { + for (let attempt = 0; !operationId && attempt < MAX_JOB_ID_ATTEMPTS; attempt += 1) { + const candidate = `job1_${Buffer.from(randomBytes(18)).toString("base64url")}`; + if (OPERATION_ID_PATTERN.test(candidate) && !jobsById.has(candidate)) { + operationId = candidate; + break; + } + } + } catch { + throw new RequestGuardError("runtime", 500); } - return { type: "retained", job: retained } as const; - } - if (jobsById.size + historicalPulls.length >= maxJobEntries) - throw new RequestGuardError("busy", 429); - if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); - const timestamp = new Date(normalized.admittedAt).toISOString(); - let operationId: string | undefined = controlAdmission?.execution.executionId; - if (operationId && jobsById.has(operationId)) - throw new RequestGuardError("conflict", 409); - try { - for (let attempt = 0; !operationId && attempt < MAX_JOB_ID_ATTEMPTS; attempt += 1) { - const candidate = `job1_${Buffer.from(randomBytes(18)).toString("base64url")}`; - if (OPERATION_ID_PATTERN.test(candidate) && !jobsById.has(candidate)) { - operationId = candidate; - break; + if (!operationId) throw new RequestGuardError("busy", 429); + const job: JobRecord = { + ...(controlAdmission + ? { + controlAdoption: { + leaseId: controlAdmission.leaseId, + delegationId: controlAdmission.delegationId, + phase: "pending-adoption" as const, + }, + } + : {}), + role, + operation, + operationId, + operationKey: parsed.operationKey, + executionVersion: 2, + requestVersion: parsed.version, + ...normalized, + operationFingerprint, + requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, + parent, + deploymentBudgetMs: maintenanceStepAdmission + ? opsExecutionBudget(operation.kind).processMs + : Math.min(policy.timeoutMs, options.operationTimeoutMs ?? policy.timeoutMs), + resourceId, + state: "queued", + submittedAt: timestamp, + updatedAt: timestamp, + cursor: 0, + events: [], + eventBytes: 0, + eventsTruncated: false, + }; + if ( + configurationMaintenanceControl && + operation.kind === "configurationMaintenance.decide" + ) { + try { + if ( + job.admittedAt === undefined || + job.requestedDeadlineAt === undefined || + typeof parentHeader !== "string" + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual decision request source required", + ); + const continueObservation = + operation.decision === "continue" + ? await prepareMaintenanceContinueObservation( + runtime, + configurationMaintenanceControlOwnerState( + configurationMaintenanceControl, + operation, + job.operationKey, + ).authority, + AbortSignal.any([ + claimController.signal, + AbortSignal.timeout(Math.max(1, job.deadlineAt! - Date.now())), + ]), + ) + : undefined; + retainConfigurationMaintenanceDecisionControl( + configurationMaintenanceControl, + operation, + executionFor(job), + { + admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, + parentAssertion: parentHeader, + requestVersion: job.requestVersion!, + deploymentBudgetMs: job.deploymentBudgetMs!, + }, + continueObservation, + ); + maintenanceControls.set(job.operationId, configurationMaintenanceControl); + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + if (error instanceof ConfigurationMaintenanceControlConflict) + throw new RequestGuardError("conflict", 409); + throw error; } } - } catch { - throw new RequestGuardError("runtime", 500); - } - if (!operationId) throw new RequestGuardError("busy", 429); - const job: JobRecord = { - ...(controlAdmission - ? { - controlAdoption: { - leaseId: controlAdmission.leaseId, - delegationId: controlAdmission.delegationId, - phase: "pending-adoption" as const, + if ( + configurationMaintenanceControl && + operation.kind === "configurationMaintenance.settle" + ) { + try { + if ( + job.admittedAt === undefined || + job.requestedDeadlineAt === undefined || + typeof parentHeader !== "string" + ) + throw new ConfigurationMaintenanceControlUnauthorized( + "Actual settlement request source required", + ); + retainConfigurationMaintenanceSettleControl( + configurationMaintenanceControl, + operation, + executionFor(job), + { + admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, + parentAssertion: parentHeader, + requestVersion: job.requestVersion!, + deploymentBudgetMs: job.deploymentBudgetMs!, }, - } - : {}), - role, - operation, - operationId, - operationKey: parsed.operationKey, - executionVersion: 2, - requestVersion: parsed.version, - ...normalized, - operationFingerprint, - requestedDeadlineAt: parsed.execution?.requestedDeadlineAt, - parent, - deploymentBudgetMs: maintenanceStepAdmission - ? opsExecutionBudget(operation.kind).processMs - : Math.min(policy.timeoutMs, options.operationTimeoutMs ?? policy.timeoutMs), - resourceId, - state: "queued", - submittedAt: timestamp, - updatedAt: timestamp, - cursor: 0, - events: [], - eventBytes: 0, - eventsTruncated: false, - }; - if ( - configurationMaintenanceControl && - operation.kind === "configurationMaintenance.decide" - ) { - try { - if ( - job.admittedAt === undefined || - job.requestedDeadlineAt === undefined || - typeof parentHeader !== "string" - ) - throw new ConfigurationMaintenanceControlUnauthorized( - "Actual decision request source required", ); - const continueObservation = - operation.decision === "continue" - ? await prepareMaintenanceContinueObservation( - runtime, - configurationMaintenanceControlOwnerState( - configurationMaintenanceControl, - operation, - job.operationKey, - ).authority, - AbortSignal.any([ - claimController.signal, - AbortSignal.timeout(Math.max(1, job.deadlineAt! - Date.now())), - ]), - ) - : undefined; - retainConfigurationMaintenanceDecisionControl( - configurationMaintenanceControl, - operation, - executionFor(job), - { - admittedAt: job.admittedAt, - requestedDeadlineAt: job.requestedDeadlineAt, - parentAssertion: parentHeader, - requestVersion: job.requestVersion!, - deploymentBudgetMs: job.deploymentBudgetMs!, - }, - continueObservation, - ); - maintenanceControls.set(job.operationId, configurationMaintenanceControl); - } catch (error) { - if (error instanceof ConfigurationMaintenanceControlUnauthorized) - throw new RequestGuardError("authorization", 403); - if (error instanceof ConfigurationMaintenanceControlConflict) - throw new RequestGuardError("conflict", 409); - throw error; + maintenanceControls.set(job.operationId, configurationMaintenanceControl); + } catch (error) { + if (error instanceof ConfigurationMaintenanceControlUnauthorized) + throw new RequestGuardError("authorization", 403); + if (error instanceof ConfigurationMaintenanceControlConflict) + throw new RequestGuardError("conflict", 409); + throw error; + } } - } - if ( - configurationMaintenanceControl && - operation.kind === "configurationMaintenance.settle" - ) { - try { - if ( - job.admittedAt === undefined || - job.requestedDeadlineAt === undefined || - typeof parentHeader !== "string" - ) - throw new ConfigurationMaintenanceControlUnauthorized( - "Actual settlement request source required", + let coordinatorBaseline: + | Awaited> + | undefined; + if (job.operation.kind === "service.pull" && job.operation.candidateId) { + if (!options.controlInstallation || !coordinatorCapture) + throw new RequestGuardError("authorization", 403); + try { + coordinatorBaseline = await prepareCoordinatorPullBaseline( + runtime, + options.controlInstallation, + { + operation: job.operation, + execution: executionFor(job), + parent: job.parent, + requestedDeadlineAt: job.requestedDeadlineAt, + }, + coordinatorCapture, ); - retainConfigurationMaintenanceSettleControl( - configurationMaintenanceControl, - operation, - executionFor(job), - { - admittedAt: job.admittedAt, - requestedDeadlineAt: job.requestedDeadlineAt, - parentAssertion: parentHeader, - requestVersion: job.requestVersion!, - deploymentBudgetMs: job.deploymentBudgetMs!, - }, - ); - maintenanceControls.set(job.operationId, configurationMaintenanceControl); - } catch (error) { - if (error instanceof ConfigurationMaintenanceControlUnauthorized) + } catch { throw new RequestGuardError("authorization", 403); - if (error instanceof ConfigurationMaintenanceControlConflict) - throw new RequestGuardError("conflict", 409); - throw error; - } - } - let coordinatorBaseline: - | Awaited> - | undefined; - if (job.operation.kind === "service.pull" && job.operation.candidateId) { - if (!options.controlInstallation || !coordinatorCapture) - throw new RequestGuardError("authorization", 403); - try { - coordinatorBaseline = await prepareCoordinatorPullBaseline( - runtime, - options.controlInstallation, - { - operation: job.operation, - execution: executionFor(job), - parent: job.parent, - requestedDeadlineAt: job.requestedDeadlineAt, - }, - coordinatorCapture, - ); - } catch { - throw new RequestGuardError("authorization", 403); + } } - } - await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); - jobsById.set(job.operationId, job); - jobsByKey.set(ownerKey, job); - if ( - !controlAdmission && - !configurationMaintenanceControl && - opsKindRequiresControlAuthority(operation.kind) - ) { - try { - if (!options.controlInstallation || job.admittedAt === undefined) - throw Error("Control trust unavailable"); - const maintenanceAdmission = - job.operation.kind === "configurationMaintenance.prepare" - ? await prepareConfigurationMaintenanceAdmission(runtime, { - installation: options.controlInstallation, - execution: executionFor(job), - operation: job.operation, - apiParent: - job.parent ?? - (() => { - throw Error("Original retained API maintenance parent required"); - })(), - signal: claimController.signal, - }) - : undefined; - const maintenancePlan = - maintenanceAdmission && - configurationMaintenanceAdmissionState(maintenanceAdmission).plan; - const targets = - maintenancePlan?.serviceIds ?? - ("serviceId" in operation && - ![ - "service.start", - "service.recreate", - "service.recreateIsolated", - "service.update", - ].includes(operation.kind) - ? [operation.serviceId] - : controlRegisteredServiceIds(options.controlInstallation)); - const fixedDataPlan = isFixedSingleDataCommand(job.operation.kind) - ? planFixedSingleDataCommand( - executionFor(job), - job.operation, - targets, - "regionId" in job.operation && job.operation.regionId === undefined - ? resolveDataDefaultRegion(runtime) - : undefined, - ) - : undefined; - const dataUpdatePlan = - job.operation.kind === "data.update" - ? prepareDataUpdateAdmission(runtime, executionFor(job), job.operation, targets) - : undefined; - const hardlinksPlan = - job.operation.kind === "configuration.hardlinks.apply" - ? prepareConfigurationHardlinksAdmission( - runtime, + await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); + jobsById.set(job.operationId, job); + jobsByKey.set(ownerKey, job); + if ( + !controlAdmission && + !configurationMaintenanceControl && + opsKindRequiresControlAuthority(operation.kind) + ) { + try { + if (!options.controlInstallation || job.admittedAt === undefined) + throw Error("Control trust unavailable"); + const maintenanceAdmission = + job.operation.kind === "configurationMaintenance.prepare" + ? await prepareConfigurationMaintenanceAdmission(runtime, { + installation: options.controlInstallation, + execution: executionFor(job), + operation: job.operation, + apiParent: + job.parent ?? + (() => { + throw Error("Original retained API maintenance parent required"); + })(), + signal: claimController.signal, + }) + : undefined; + const maintenancePlan = + maintenanceAdmission && + configurationMaintenanceAdmissionState(maintenanceAdmission).plan; + const targets = + maintenancePlan?.serviceIds ?? + ("serviceId" in operation && + ![ + "service.start", + "service.recreate", + "service.recreateIsolated", + "service.update", + ].includes(operation.kind) + ? [operation.serviceId] + : controlRegisteredServiceIds(options.controlInstallation)); + const fixedDataPlan = isFixedSingleDataCommand(job.operation.kind) + ? planFixedSingleDataCommand( executionFor(job), job.operation, targets, + "regionId" in job.operation && job.operation.regionId === undefined + ? resolveDataDefaultRegion(runtime) + : undefined, ) : undefined; - const lifecyclePlan = - job.operation.kind === "service.update" - ? prepareServiceUpdateAdmission( + const dataUpdatePlan = + job.operation.kind === "data.update" + ? prepareDataUpdateAdmission(runtime, executionFor(job), job.operation, targets) + : undefined; + const hardlinksPlan = + job.operation.kind === "configuration.hardlinks.apply" + ? prepareConfigurationHardlinksAdmission( + runtime, + executionFor(job), + job.operation, + targets, + ) + : undefined; + const lifecyclePlan = + job.operation.kind === "service.update" + ? prepareServiceUpdateAdmission( + runtime, + executionFor(job), + job.operation, + targets, + ) + : undefined; + const authority = retainAdmittedControlExecution(options.controlInstallation, { + execution: executionFor(job), + operation: job.operation, + kind: job.operation.kind, + admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, + parent: job.parent, + serviceIds: targets, + ...(fixedDataPlan ? { reservations: [fixedDataPlan.rootReservation] } : {}), + ...(dataUpdatePlan + ? { + reservations: [ + dataUpdatePlan.dataReservation, + dataUpdatePlan.finalizationReservation, + ], + } + : {}), + ...(hardlinksPlan + ? { reservations: hardlinksPlan.reservation ? [hardlinksPlan.reservation] : [] } + : {}), + ...(lifecyclePlan ? { reservations: [lifecyclePlan.rootReservation] } : {}), + ...(maintenancePlan ? { reservations: maintenancePlan.reservations } : {}), + ...(coordinatorBaseline ? { coordinatorPullBaseline: coordinatorBaseline } : {}), + }); + if (fixedDataPlan) retainFixedSingleDataPlan(authority, fixedDataPlan); + if (dataUpdatePlan) retainDataUpdatePlan(authority, dataUpdatePlan); + if (hardlinksPlan) retainGenerationHardlinksOwnerPlan(authority, hardlinksPlan); + if (lifecyclePlan) retainFixedServiceLifecyclePlan(authority, lifecyclePlan); + requireControlJournalRetention(authority); + controlAuthorities.set(job.operationId, authority); + if ( + [ + "configurationMaintenance.prepare", + "stack.start", + "coreService.restart", + "service.start", + "service.recreate", + "service.recreateIsolated", + "service.update", + "configuration.hardlinks.apply", + "valhalla.traffic.maintain", + "valhalla.traffic.refreshWaysToEdges", + ].includes(job.operation.kind) + ) { + const lease = await acquireArtifactTargetLease({ + dataRoot: controlInstallationDataRoot(options.controlInstallation), + serviceIds: + maintenancePlan || + hardlinksPlan || + lifecyclePlan || + [ + "stack.start", + "service.start", + "coreService.restart", + "service.recreate", + "service.recreateIsolated", + ].includes(job.operation.kind) + ? targets + : ["valhalla"], + owner: authority, + }); + controlLeases.set(job.operationId, lease); + if (maintenanceAdmission) { + retainOwnedConfigurationMaintenance(authority, lease, maintenanceAdmission); + publishConfigurationSettlementReservationProjection( + authority, + maintenancePlan!.operation.maintenanceId, + ); + } + } + } catch (error) { + forceRecoveryRequired(job); + await journal.replace([...jobsById.values()].map(persisted)); + throw error; + } + } + if (controlAdmission) { + const adoption = { + leaseId: controlAdmission.leaseId, + delegationId: controlAdmission.delegationId, + }; + try { + if (!options.controlInstallation || job.admittedAt === undefined) + throw Error("Missing durable control admission context"); + const lease = await controlAdmission.adopt(executionFor(job)); + const authority = retainAdmittedControlExecution(options.controlInstallation, { + execution: executionFor(job), + kind: job.operation.kind, + operation: job.operation, + admittedAt: job.admittedAt, + requestedDeadlineAt: job.requestedDeadlineAt, + parent: job.parent, + serviceIds: lease.serviceIds, + ...(maintenanceStepAdmission + ? { reservations: maintenanceStepAdmission.reservations } + : {}), + }); + if (maintenanceStepAdmission) bindBorrowedControlExecution(lease, authority); + if ( + maintenanceStepAdmission && + job.operation.kind === "configuration.hardlinks.apply" + ) + retainGenerationHardlinksOwnerPlan( + authority, + prepareConfigurationHardlinksAdmission( runtime, executionFor(job), job.operation, - targets, - ) - : undefined; - const authority = retainAdmittedControlExecution(options.controlInstallation, { - execution: executionFor(job), - operation: job.operation, - kind: job.operation.kind, - admittedAt: job.admittedAt, - requestedDeadlineAt: job.requestedDeadlineAt, - parent: job.parent, - serviceIds: targets, - ...(fixedDataPlan ? { reservations: [fixedDataPlan.rootReservation] } : {}), - ...(dataUpdatePlan - ? { - reservations: [ - dataUpdatePlan.dataReservation, - dataUpdatePlan.finalizationReservation, - ], - } - : {}), - ...(hardlinksPlan - ? { reservations: hardlinksPlan.reservation ? [hardlinksPlan.reservation] : [] } - : {}), - ...(lifecyclePlan ? { reservations: [lifecyclePlan.rootReservation] } : {}), - ...(maintenancePlan ? { reservations: maintenancePlan.reservations } : {}), - ...(coordinatorBaseline ? { coordinatorPullBaseline: coordinatorBaseline } : {}), - }); - if (fixedDataPlan) retainFixedSingleDataPlan(authority, fixedDataPlan); - if (dataUpdatePlan) retainDataUpdatePlan(authority, dataUpdatePlan); - if (hardlinksPlan) retainGenerationHardlinksOwnerPlan(authority, hardlinksPlan); - if (lifecyclePlan) retainFixedServiceLifecyclePlan(authority, lifecyclePlan); - requireControlJournalRetention(authority); - controlAuthorities.set(job.operationId, authority); - if ( - [ - "configurationMaintenance.prepare", - "stack.start", - "coreService.restart", - "service.start", - "service.recreate", - "service.recreateIsolated", - "service.update", - "configuration.hardlinks.apply", - "valhalla.traffic.maintain", - "valhalla.traffic.refreshWaysToEdges", - ].includes(job.operation.kind) - ) { - const lease = await acquireArtifactTargetLease({ - dataRoot: controlInstallationDataRoot(options.controlInstallation), - serviceIds: - maintenancePlan || - hardlinksPlan || - lifecyclePlan || - [ - "stack.start", - "service.start", - "coreService.restart", - "service.recreate", - "service.recreateIsolated", - ].includes(job.operation.kind) - ? targets - : ["valhalla"], - owner: authority, - }); + lease.serviceIds, + ), + ); + requireControlJournalRetention(authority); controlLeases.set(job.operationId, lease); - if (maintenanceAdmission) { - retainOwnedConfigurationMaintenance(authority, lease, maintenanceAdmission); - publishConfigurationSettlementReservationProjection( - authority, - maintenancePlan!.operation.maintenanceId, + controlAuthorities.set(job.operationId, authority); + job.controlAdoption = { ...adoption, phase: "ready" }; + await journal.replace([...jobsById.values()].map(persisted)); + } catch (error) { + // The receiver may already have adopted. This identity remains + // observable as recovery-required and is never dispatched again. + job.controlAdoption = { ...adoption, phase: "recovery_required" }; + forceRecoveryRequired(job); + try { + await journal.replace([...jobsById.values()].map(persisted)); + } catch (persistence) { + throw new AggregateError( + [error, persistence], + "Control adoption requires reconciliation", ); } + throw error; } - } catch (error) { - forceRecoveryRequired(job); - await journal.replace([...jobsById.values()].map(persisted)); - throw error; } + return { type: "new", job } as const; + }).catch(async (error) => { + await claimAdmission?.rollback().catch(() => undefined); + active -= 1; + throw error instanceof RequestGuardError + ? error + : new RequestGuardError("runtime", 500); + }); + if (admission.type === "retained") { + await claimAdmission?.rollback().catch(() => undefined); + active -= 1; + return reply + .code(202) + .send(opsSuccess(parsed.requestId, admissionFor(admission.job, parsed.version))); } - if (controlAdmission) { - const adoption = { - leaseId: controlAdmission.leaseId, - delegationId: controlAdmission.delegationId, - }; - try { - if (!options.controlInstallation || job.admittedAt === undefined) - throw Error("Missing durable control admission context"); - const lease = await controlAdmission.adopt(executionFor(job)); - const authority = retainAdmittedControlExecution(options.controlInstallation, { - execution: executionFor(job), - kind: job.operation.kind, - operation: job.operation, - admittedAt: job.admittedAt, - requestedDeadlineAt: job.requestedDeadlineAt, - parent: job.parent, - serviceIds: lease.serviceIds, - ...(maintenanceStepAdmission - ? { reservations: maintenanceStepAdmission.reservations } - : {}), - }); - if (maintenanceStepAdmission) bindBorrowedControlExecution(lease, authority); - if ( - maintenanceStepAdmission && - job.operation.kind === "configuration.hardlinks.apply" - ) - retainGenerationHardlinksOwnerPlan( - authority, - prepareConfigurationHardlinksAdmission( - runtime, - executionFor(job), - job.operation, - lease.serviceIds, - ), - ); - requireControlJournalRetention(authority); - controlLeases.set(job.operationId, lease); - controlAuthorities.set(job.operationId, authority); - job.controlAdoption = { ...adoption, phase: "ready" }; - await journal.replace([...jobsById.values()].map(persisted)); - } catch (error) { - // The receiver may already have adopted. This identity remains - // observable as recovery-required and is never dispatched again. - job.controlAdoption = { ...adoption, phase: "recovery_required" }; - forceRecoveryRequired(job); - try { - await journal.replace([...jobsById.values()].map(persisted)); - } catch (persistence) { - throw new AggregateError( - [error, persistence], - "Control adoption requires reconciliation", - ); - } - throw error; - } + const { job } = admission; + try { + if (normalized.deadlineAt <= now().getTime()) + throw new RequestGuardError("timeout", 504); + await claimAdmission?.commit(); + } catch { + stickyRecoveryRequired.add(job.operationId); + await transition(job, "failed", { + result: undefined, + errorClass: "recovery_required", + terminalAt: now().getTime(), + }).catch(() => forceRecoveryRequired(job)); + await claimAdmission?.release().catch(() => undefined); + active -= 1; + throw new RequestGuardError("runtime", 500); } - return { type: "new", job } as const; - }).catch(async (error) => { - await claimAdmission?.rollback().catch(() => undefined); - active -= 1; - throw error instanceof RequestGuardError ? error : new RequestGuardError("runtime", 500); - }); - if (admission.type === "retained") { - await claimAdmission?.rollback().catch(() => undefined); - active -= 1; + await runAsync(job, startedAt, claim, claimAdmission); return reply .code(202) - .send(opsSuccess(parsed.requestId, admissionFor(admission.job, parsed.version))); + .send(opsSuccess(parsed.requestId, admissionFor(job, parsed.version))); } - const { job } = admission; + const controller = new AbortController(); try { if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); await claimAdmission?.commit(); } catch { - stickyRecoveryRequired.add(job.operationId); - await transition(job, "failed", { - result: undefined, - errorClass: "recovery_required", - terminalAt: now().getTime(), - }).catch(() => forceRecoveryRequired(job)); - await claimAdmission?.release().catch(() => undefined); active -= 1; throw new RequestGuardError("runtime", 500); } - await runAsync(job, startedAt, claim, claimAdmission); - return reply - .code(202) - .send(opsSuccess(parsed.requestId, admissionFor(job, parsed.version))); - } - const controller = new AbortController(); - try { - if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); - await claimAdmission?.commit(); - } catch { - active -= 1; - throw new RequestGuardError("runtime", 500); - } - const timeoutMs = Math.max(0, normalized.deadlineAt - now().getTime()); - const execution = Promise.resolve().then(() => { - if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); - return dispatch(operation, runtime, { - signal: controller.signal, - ...(configurationMaintenanceControl ? { configurationMaintenanceControl } : {}), - execution: Object.freeze({ - executor: "ops-agent", - principal: role, - executionId: `job1_${Buffer.from(randomBytes(18)).toString("base64url")}`, - operationKey: parsed.operationKey, - operationFingerprint, - fingerprint: normalized.fingerprint, - deadlineAt: normalized.deadlineAt, - }), - emitLog: () => undefined, - claim, - }); - }); - let timeout: ReturnType | undefined; - try { - const timeoutResult = new Promise((_resolve, reject) => { - timeout = setTimeout(() => { - controller.abort(); - reject(new RequestGuardError("timeout", 504)); - }, timeoutMs); + const timeoutMs = Math.max(0, normalized.deadlineAt - now().getTime()); + const execution = Promise.resolve().then(() => { + if (normalized.deadlineAt <= now().getTime()) throw new RequestGuardError("timeout", 504); + return dispatch(operation, runtime, { + signal: controller.signal, + ...(configurationMaintenanceControl ? { configurationMaintenanceControl } : {}), + execution: Object.freeze({ + executor: "ops-agent", + principal: role, + executionId: `job1_${Buffer.from(randomBytes(18)).toString("base64url")}`, + operationKey: parsed.operationKey, + operationFingerprint, + fingerprint: normalized.fingerprint, + deadlineAt: normalized.deadlineAt, + }), + emitLog: () => undefined, + claim, + }); }); - const rawResult = await Promise.race([execution, timeoutResult]); - clearTimeout(timeout); - const value = parseBoundedOpsResult(operation.kind, rawResult); - audit(role, kind, resourceId, "success", startedAt); - const envelope = opsSuccess(parsed.requestId, { execution: "sync", kind, value }); - if (Buffer.byteLength(JSON.stringify(envelope), "utf8") > OPS_MAX_HTTP_RESPONSE_BYTES) { - throw new RequestGuardError("runtime", 500); + let timeout: ReturnType | undefined; + try { + const timeoutResult = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(); + reject(new RequestGuardError("timeout", 504)); + }, timeoutMs); + }); + const rawResult = await Promise.race([execution, timeoutResult]); + clearTimeout(timeout); + const value = parseBoundedOpsResult(operation.kind, rawResult); + audit(role, kind, resourceId, "success", startedAt); + const envelope = opsSuccess(parsed.requestId, { execution: "sync", kind, value }); + if (Buffer.byteLength(JSON.stringify(envelope), "utf8") > OPS_MAX_HTTP_RESPONSE_BYTES) { + throw new RequestGuardError("runtime", 500); + } + active -= 1; + return reply.code(200).send(envelope); + } catch (error) { + if (timeout !== undefined) clearTimeout(timeout); + if (error instanceof RequestGuardError && error.errorClass === "timeout") { + void execution.then( + () => { + active -= 1; + audit(role, kind, resourceId, "late_completion", startedAt, "timeout"); + }, + () => { + active -= 1; + audit(role, kind, resourceId, "terminated", startedAt, "timeout"); + }, + ); + audit(role, kind, resourceId, "timeout", startedAt, "timeout"); + return reply.code(504).send(redactedOpsError(parsed.requestId, "timeout")); + } + active -= 1; + throw error; } - active -= 1; - return reply.code(200).send(envelope); } catch (error) { - if (timeout !== undefined) clearTimeout(timeout); - if (error instanceof RequestGuardError && error.errorClass === "timeout") { - void execution.then( - () => { - active -= 1; - audit(role, kind, resourceId, "late_completion", startedAt, "timeout"); - }, - () => { - active -= 1; - audit(role, kind, resourceId, "terminated", startedAt, "timeout"); - }, - ); - audit(role, kind, resourceId, "timeout", startedAt, "timeout"); - return reply.code(504).send(redactedOpsError(parsed.requestId, "timeout")); - } - active -= 1; - throw error; + const classification = classify(error); + audit( + role, + kind, + resourceId, + classification.errorClass === "authorization" ? "denied" : "error", + startedAt, + classification.errorClass, + ); + return reply + .code(classification.statusCode) + .send(redactedOpsError(requestId, classification.errorClass, error)); } - } catch (error) { - const classification = classify(error); - audit( - role, - kind, - resourceId, - classification.errorClass === "authorization" ? "denied" : "error", - startedAt, - classification.errorClass, - ); - return reply - .code(classification.statusCode) - .send(redactedOpsError(requestId, classification.errorClass, error)); - } - }); + }, + ); app.get("/v1/operation-keys/:operationKey", async (request, reply) => { const requestId = validRequestId(request.headers["x-ops-request-id"]); diff --git a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs index 4da060e1a..c33319a86 100644 --- a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs +++ b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs @@ -7,8 +7,10 @@ import { closeControlInstallation, openControlInstallation } from "@openmapx/cor import { services } from "@openmapx/core/server"; import { CommunityRepositoryStore } from "../community-repositories.ts"; import { configurationPolicyRevision } from "../configured-resource-authority.ts"; +import { createDataTransitTrustedOpsDataSource } from "../data-transit-resource-source.ts"; import { openOpsJobJournal } from "../journal.ts"; import { createProductionRegistryResourceClaimer } from "../policy.ts"; +import { safeProcessDiagnostic } from "../process-output.ts"; import { createProductionOpsRuntime, productionOwnedServices } from "../production-runtime.ts"; import { buildOpsAgentServer } from "../server.ts"; import { createFileTrustedOpsDataSource } from "../trusted-config-source.ts"; @@ -131,6 +133,22 @@ const runtime = await createProductionOpsRuntime({ }, }, }); +// Observe the original handler; all authority checks and results remain native. +if (input.catalogFault) { + const approve = runtime["transitousLock.approve"]; + runtime["transitousLock.approve"] = async (...args) => { + try { + return await approve(...args); + } catch (error) { + console.error( + safeProcessDiagnostic( + error instanceof Error ? (error.stack ?? error.message) : String(error), + ), + ); + throw error; + } + }; +} const policyAuthority = async () => { const a = await loadAuthority(); return { @@ -152,22 +170,51 @@ const installation = openControlInstallation({ serviceIds: input.serviceIds, }); const journal = await openOpsJobJournal(join(input.infraDir, "data/ops-agent/jobs-v1.json")); +if (["pointer", "cancel-pointer", "before-pointer"].includes(input.catalogFault)) { + const rename = fs.renameSync; + let stopped = false; + fs.renameSync = (from, to) => { + let matching = false; + if (String(to).endsWith("/current.json")) { + const actual = fs.statSync(dirname(String(to))); + const expected = fs.statSync(join(input.infraDir, "catalog-pins")); + matching = actual.dev === expected.dev && actual.ino === expected.ino; + } + if (matching && !stopped && input.catalogFault === "before-pointer") { + stopped = true; + fs.writeSync(1, JSON.stringify({ event: "catalog-pointer-before-commit" }) + "\n"); + process.kill(process.pid, "SIGSTOP"); + } + const result = rename(from, to); + if (matching && !stopped) { + stopped = true; + fs.writeSync(1, JSON.stringify({ event: "catalog-pointer-committed" }) + "\n"); + process.kill(process.pid, "SIGSTOP"); + } + return result; + }; + syncBuiltinESMExports(); +} + const app = buildOpsAgentServer({ runtime, journal, controlInstallation: installation, - operationTimeoutMs: 180000, + operationTimeoutMs: input.operationTimeoutMs ?? 180000, tokens: { api: input.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, resourceClaimer: createProductionRegistryResourceClaimer({ services: authority.services, integrationIds: authority.integrationIds, loadAuthority: policyAuthority, - trustedData: createFileTrustedOpsDataSource({ - directory: input.queue, - token: input.token, - allowedUids: [fs.statSync(input.queue).uid], - expectedGid: fs.statSync(input.queue).gid, - }), + trustedData: createDataTransitTrustedOpsDataSource( + installation, + createFileTrustedOpsDataSource({ + directory: input.queue, + token: input.token, + allowedUids: [fs.statSync(input.queue).uid], + expectedGid: fs.statSync(input.queue).gid, + }), + ), ownedServices: productionOwnedServices(runtime), enabledServiceIds: (ss) => services.readAppliedServiceIds( @@ -196,9 +243,15 @@ app.addHook("onSend", async (request, reply, payload) => { ...(reply.statusCode >= 400 ? { errorClass: JSON.parse(payload).error?.class } : {}), }) + "\n", ); + if ( + ["pointer", "cancel-pointer", "before-pointer"].includes(input.catalogFault) && + kind === "transitousLock.approve" && + reply.statusCode === 202 + ) + request.raw.socket.destroy(); return payload; }); -const url = await app.listen({ host: "127.0.0.1", port: 0 }); +const url = await app.listen({ host: "127.0.0.1", port: input.listenPort ?? 0 }); const owned = await productionOwnedServices(runtime).inspectOwnedServices( new AbortController().signal, ); diff --git a/apps/ops-agent/src/transit-credentials-runtime.test.ts b/apps/ops-agent/src/transit-credentials-runtime.test.ts index efd5addd0..c598dcef3 100644 --- a/apps/ops-agent/src/transit-credentials-runtime.test.ts +++ b/apps/ops-agent/src/transit-credentials-runtime.test.ts @@ -1,5 +1,5 @@ import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -95,6 +95,8 @@ async function fixture() { catalogRevisionId: readCatalogPinSet(root)!.generationId, }; let environment: Record = {}; + let running = false; + let consumer: Record = { Config: { Env: [] }, Mounts: [] }; const runtime = createUnavailableRuntime(); installTransitCredentialsRuntime(runtime, { rootDir: root, @@ -104,8 +106,26 @@ async function fixture() { composePath: join(infraDir, "docker-compose.generated.yml"), composeReleasePath: join(infraDir, "docker-compose.release.yml"), }, - ownedServices: { inspectOwnedServices: async () => [] }, + ownedServices: { + inspectOwnedServices: async () => + running + ? [ + { + serviceId: "data-manager", + containerId: "dm-owned", + state: "running", + provenance: { + project: "fixture", + composeWorkingDirectory: infraDir, + composeConfigFiles: [], + imageId: "sha256:" + "a".repeat(64), + }, + }, + ] + : [], + }, execFile: async (_file, args) => { + if (args[0] === "container") return { stdout: JSON.stringify(consumer), stderr: "" }; expect(args).toContain("config"); return { stdout: JSON.stringify({ services: { "data-manager": { environment } } }), @@ -129,6 +149,10 @@ async function fixture() { root, local, inspect, + setConsumer: (value: Record) => { + running = true; + consumer = value; + }, setEnv: (value: Record) => { environment = value; }, @@ -170,3 +194,38 @@ it("refuses malformed existing files rather than treating the vault as unset", a writeFileSync(join(f.local, "api-keys.json"), '{"key":"first","key":"last"}', { mode: 0o600 }); await expect(f.inspect()).rejects.toThrow(); }, 60_000); + +it("binds the host credential revision only to the running consumer's exact read-only mount", async () => { + const f = await fixture(); + writeFileSync(join(f.local, "api-keys.json"), '{"de/A":"private"}', { mode: 0o600 }); + const own = { + Config: { Env: [] }, + Mounts: [{ Source: f.local, Destination: "/config/transitous", RW: false }], + }; + f.setConsumer(own); + const observed = await f.inspect(); + expect(observed).toMatchObject({ + owner: "file", + consumedCredentialsRevision: observed.hostCredentialsRevision, + }); + expect(observed.consumedCredentialsRevision).toMatch(/^[a-f0-9]{64}$/); + f.setConsumer({ + ...own, + Mounts: [{ Source: f.local, Destination: "/config/transitous", RW: true }], + }); + expect((await f.inspect()).consumedCredentialsRevision).toBeNull(); + f.setConsumer({ ...own, Config: { Env: ["TRANSITOUS_API_KEYS_PATH=/other/map.json"] } }); + expect((await f.inspect()).consumedCredentialsRevision).toBeNull(); +}, 60_000); + +it("invalidates the host source version after an identical atomic template replacement", async () => { + const f = await fixture(), + target = join(f.local, "api-keys.json"), + bytes = '{"de/A":"private"}'; + writeFileSync(target, bytes, { mode: 0o600 }); + const before = await f.inspect(); + writeFileSync(target + ".next", bytes, { mode: 0o600 }); + renameSync(target + ".next", target); + const after = await f.inspect(); + expect(after.hostCredentialsRevision).not.toBe(before.hostCredentialsRevision); +}, 60_000); diff --git a/apps/ops-agent/src/transit-credentials-runtime.ts b/apps/ops-agent/src/transit-credentials-runtime.ts index 7e0b0825c..acfe54b8a 100644 --- a/apps/ops-agent/src/transit-credentials-runtime.ts +++ b/apps/ops-agent/src/transit-credentials-runtime.ts @@ -2,8 +2,14 @@ import { createHmac } from "node:crypto"; import { lstatSync, realpathSync } from "node:fs"; import { join } from "node:path"; -import { catalogRevisionId, readCatalogPinSet } from "@openmapx/core/catalog-pins"; -import type { OpsResultFor } from "@openmapx/core/ops"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { + type CatalogPinSet, + catalogRevisionId, + readCatalogPinSet, + readCatalogProposal, +} from "@openmapx/core/catalog-pins"; +import { type OpsResultFor, TRANSIT_CREDENTIALS_ABSENT_REVISION } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { inspectPinnedFeedCredentialKeys, TRANSITOUS_CATALOG_DIR } from "@openmapx/transitous-core"; import { parseFeedCredentialMapJson } from "@openmapx/transitous-core/feed-credentials"; @@ -14,6 +20,27 @@ import { OpsAgentBusyError, OpsNotWiredError, type OpsRuntime } from "./runtime" const MANAGED = "/run/secrets/TRANSITOUS_API_KEYS"; const DEFAULT = "/config/transitous/api-keys.json"; +export function inspectAgentPinnedCredentialKeys( + infraDir: string, + pins: CatalogPinSet, + signal: AbortSignal, + scope?: Parameters[0]["scope"], +) { + return inspectPinnedFeedCredentialKeys({ + catalogDir: join(infraDir, "data", TRANSITOUS_CATALOG_DIR), + transitousCommit: pins.transitous.ref.split("@").at(-1)!, + atlasCommit: pins.transitous.submodules["transitland-atlas"]!, + scope, + git: async (args, cwd) => + ( + await runContainedProcess("git", ["-C", cwd, ...args], { + signal, + timeout: 30000, + maxBuffer: 5 * 1024 * 1024, + }) + ).stdout, + }); +} export function installTransitCredentialsRuntime( runtime: OpsRuntime, options: { @@ -30,22 +57,35 @@ export function installTransitCredentialsRuntime( if (inspecting) throw new OpsAgentBusyError(); inspecting = true; try { - const pair = readCatalogPinSet(options.rootDir); + const select = () => { + const active = readCatalogPinSet(options.rootDir); + if (!operation.proposalId) return { pair: active, base: active?.generationId ?? null }; + const installation = openControlInstallation({ + infraDir: options.stack.infraDir, + role: "ops-agent", + }); + try { + const proposal = readCatalogProposal({ + rootDir: options.rootDir, + installation, + proposalId: operation.proposalId, + }); + if (!proposal || proposal.baseGenerationId !== (active?.generationId ?? null)) + throw new OpsNotWiredError(); + return { pair: proposal.pins, base: proposal.baseGenerationId }; + } finally { + closeControlInstallation(installation); + } + }; + const selected = select(), + pair = selected.pair; if (!pair || catalogRevisionId(pair) !== operation.catalogRevisionId) throw new OpsNotWiredError(); - const requiredKeys = await inspectPinnedFeedCredentialKeys({ - catalogDir: join(options.stack.infraDir, "data", TRANSITOUS_CATALOG_DIR), - transitousCommit: pair.transitous.ref.split("@").at(-1)!, - atlasCommit: pair.transitous.submodules["transitland-atlas"]!, - git: async (args, cwd) => - ( - await runContainedProcess("git", ["-C", cwd, ...args], { - signal: context.signal, - timeout: 30_000, - maxBuffer: 5 * 1024 * 1024, - }) - ).stdout, - }); + const requiredKeys = await inspectAgentPinnedCredentialKeys( + options.stack.infraDir, + pair, + context.signal, + ); // Configuration is collected through the existing fixed Compose argument builder. const config = JSON.parse( ( @@ -84,12 +124,26 @@ export function installTransitCredentialsRuntime( const parents = components.slice(0, -1), file = components.at(-1)!; return withDirectory(parents, (directory) => { + const before = lstatSync(directory.name(file), { bigint: true }); const bytes = directory.read(file, 1024 * 1024, false); + const after = lstatSync(directory.name(file), { bigint: true }); + const fields = ["dev", "ino", "size", "mtimeNs", "ctimeNs"] as const; + if (fields.some((field) => before[field] !== after[field])) throw new OpsNotWiredError(); if (!options.sourceToken) throw new OpsNotWiredError(); hostCredentialsRevision = createHmac("sha256", options.sourceToken) .update("openmapx-host-transit-credentials-v1\0") .update(JSON.stringify(components)) .update("\0") + .update( + JSON.stringify([ + String(after.dev), + String(after.ino), + String(after.size), + String(after.mtimeNs), + String(after.ctimeNs), + ]), + ) + .update("\0") .update(bytes) .digest("hex"); const values = parseFeedCredentialMapJson( @@ -207,8 +261,79 @@ export function installTransitCredentialsRuntime( owner = "unset"; } } - const current = readCatalogPinSet(options.rootDir); - if (!current || catalogRevisionId(current) !== operation.catalogRevisionId) + let consumedCredentialsRevision: string | null = + owner === "vault" ? appliedCredentialsRevision : null; + if (owner === "file" || owner === "unset") { + const initialHostRevision = hostCredentialsRevision; + const owned = (await options.ownedServices.inspectOwnedServices(context.signal)).filter( + (service) => service.serviceId === "data-manager" && service.state === "running", + ); + if (owned.length === 1) { + const observed = JSON.parse( + ( + await docker( + "docker", + ["container", "inspect", "--format", "{{json .}}", owned[0]!.containerId], + { signal: context.signal, timeout: 15000, maxBuffer: 256 * 1024 }, + ) + ).stdout, + ); + const environment: string[] = observed?.Config?.Env ?? [], + mounts: { Source?: string; Destination?: string; RW?: boolean }[] = + observed?.Mounts ?? []; + const matches = mounts.filter( + (mount) => + mount.Destination === "/config/transitous" && + mount.RW === false && + typeof mount.Source === "string", + ); + if ( + matches.length === 1 && + !environment.some( + (value) => + value.startsWith("TRANSITOUS_API_KEYS_PATH=") && + value !== "TRANSITOUS_API_KEYS_PATH=", + ) && + !environment.some( + (value) => + value.startsWith("TRANSITOUS_API_KEYS_FILE=") && + value !== "TRANSITOUS_API_KEYS_FILE=" && + value !== `TRANSITOUS_API_KEYS_FILE=${DEFAULT}`, + ) + ) { + const components = ["services", "motis", "tools", "transitous"], + expectedDirectory = join(options.rootDir, ...components), + mountedDirectory = matches[0]!.Source!; + const source = lstatSync(expectedDirectory), + target = lstatSync(mountedDirectory); + if ( + source.isDirectory() && + target.isDirectory() && + !target.isSymbolicLink() && + source.dev === target.dev && + source.ino === target.ino && + realpathSync(expectedDirectory) === realpathSync(mountedDirectory) + ) { + if (owner === "file") { + readMap([...components, "api-keys.json"]); + if (hostCredentialsRevision === initialHostRevision) + consumedCredentialsRevision = hostCredentialsRevision; + } else if ( + !withDirectory(components, (directory) => directory.listBounded(1024)).includes( + "api-keys.json", + ) + ) + consumedCredentialsRevision = TRANSIT_CREDENTIALS_ABSENT_REVISION; + } + } + } + } + const current = select(); + if ( + !current.pair || + catalogRevisionId(current.pair) !== operation.catalogRevisionId || + current.base !== selected.base + ) throw new OpsNotWiredError(); return { catalogRevisionId: operation.catalogRevisionId, @@ -219,6 +344,7 @@ export function installTransitCredentialsRuntime( configuredKeys, retainedKeys, appliedCredentialsRevision, + consumedCredentialsRevision, hostCredentialsRevision: owner === "file" ? hostCredentialsRevision : null, }; } catch (error) { diff --git a/apps/ops-agent/src/transitous-lock-runtime.test.ts b/apps/ops-agent/src/transitous-lock-runtime.test.ts index 2f4d55aeb..b6762e7eb 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.test.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.test.ts @@ -6,7 +6,8 @@ import { openControlInstallation, provisionControlInstallation, } from "@openmapx/core/build-artifacts"; -import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; +import { createCatalogPinSet, initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; +import { opsOperationSchema } from "@openmapx/core/ops"; import { afterEach, describe, expect, it, vi } from "vitest"; import { testExecution } from "./__fixtures__/execution"; import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; @@ -57,6 +58,92 @@ afterEach(async () => { }); describe("agent-owned Transitous catalog lock", () => { + it("retains a paired GBFS-only proposal without changing active pins", async () => { + const rootDir = root(), + infra = join(rootDir, "infra/docker"); + mkdirSync(join(infra, "data"), { mode: 0o700 }); + await provisionControlInstallation(infra); + const pins = createCatalogPinSet( + { + ref: REF, + submodules: { "transitland-atlas": "c".repeat(40) }, + lockedAt: "2026-08-25T00:00:00.000Z", + lockedBy: "fixture", + }, + { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "d".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"d".repeat(40)}/systems.csv`, + sha256: "e".repeat(64), + lockedAt: "2026-08-25T00:00:00.000Z", + lockedBy: "fixture", + }, + ); + const operation = opsOperationSchema.parse({ + kind: "transitousLock.propose", + proposalId: "00000000-0000-4000-8000-000000000001", + baseGenerationId: null, + pins, + diffDigest: "f".repeat(64), + sourcePolicyRevision: "a".repeat(64), + }); + const result = await dispatchOpsOperation(runtimeFor(rootDir), operation, context()); + expect(result).toMatchObject({ + proposalId: "00000000-0000-4000-8000-000000000001", + proposed: true, + catalogRevisionId: pins.generationId, + }); + expect(existsSync(lockPath(rootDir))).toBe(false); + const retained = JSON.parse( + readFileSync( + join(infra, "catalog-pins/proposals/00000000-0000-4000-8000-000000000001.json"), + "utf8", + ), + ); + expect(retained.payload.proposal.pins.gbfs.commit).toBe("d".repeat(40)); + expect(retained.signature).toMatch(/^[A-Za-z0-9_-]{86}$/); + const originalBytes = readFileSync( + join(infra, "catalog-pins/proposals/00000000-0000-4000-8000-000000000001.json"), + "utf8", + ); + await expect(dispatchOpsOperation(runtimeFor(rootDir), operation, context())).resolves.toEqual( + result, + ); + const changedPins = createCatalogPinSet(pins.transitous, { + ...pins.gbfs, + commit: "f".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"f".repeat(40)}/systems.csv`, + }); + await expect( + dispatchOpsOperation( + runtimeFor(rootDir), + opsOperationSchema.parse({ ...operation, pins: changedPins }), + context(), + ), + ).rejects.toThrow(/Immutable catalog proposal changed/); + expect( + readFileSync( + join(infra, "catalog-pins/proposals/00000000-0000-4000-8000-000000000001.json"), + "utf8", + ), + ).toBe(originalBytes); + await expect( + dispatchOpsOperation( + runtimeFor(rootDir), + opsOperationSchema.parse({ + ...operation, + proposalId: "00000000-0000-4000-8000-000000000002", + baseGenerationId: "b".repeat(64), + }), + context(), + ), + ).rejects.toThrow(/base changed/); + expect( + existsSync(join(infra, "catalog-pins/proposals/00000000-0000-4000-8000-000000000002.json")), + ).toBe(false); + }); + it("reports both slots as absent on a fresh checkout", async () => { const rootDir = root(); await expect( @@ -64,36 +151,20 @@ describe("agent-owned Transitous catalog lock", () => { ).resolves.toEqual({ active: null, proposed: null }); }); - it("writes a proposal without activating anything", async () => { - const rootDir = root(); - const result = await dispatchOpsOperation( - runtimeFor(rootDir), - { + it("refuses legacy one-pin proposal bodies", () => { + expect( + opsOperationSchema.safeParse({ kind: "transitousLock.propose", ref: REF, submodules: { "transitland-atlas": "c".repeat(40) }, lockedBy: "auto-bump", - }, - context(), - ); - - expect(result).toEqual({ ref: REF, proposed: true }); - // A proposal must never become the active pin on its own. - expect(existsSync(lockPath(rootDir))).toBe(false); - expect(JSON.parse(readFileSync(proposedPath(rootDir), "utf-8"))).toMatchObject({ - ref: REF, - lockedBy: "auto-bump", - }); + }).success, + ).toBe(false); }); it("keeps approval closed until durable paired-pin publication is installed", async () => { const rootDir = root(); const runtime = runtimeFor(rootDir); - await dispatchOpsOperation( - runtime, - { kind: "transitousLock.propose", ref: REF, submodules: {}, lockedBy: "auto-bump" }, - context(), - ); for (const proposalId of [ "00000000-0000-4000-8000-000000000001", "00000000-0000-4000-8000-000000000002", @@ -113,7 +184,7 @@ describe("agent-owned Transitous catalog lock", () => { ), ).rejects.toThrow(/paired-pin/); expect(existsSync(lockPath(rootDir))).toBe(false); - expect(existsSync(proposedPath(rootDir))).toBe(true); + expect(existsSync(proposedPath(rootDir))).toBe(false); }); it("reads back an existing active lock", async () => { diff --git a/apps/ops-agent/src/transitous-lock-runtime.ts b/apps/ops-agent/src/transitous-lock-runtime.ts index 95552ebea..c72de4e10 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.ts @@ -1,18 +1,13 @@ -import { randomUUID } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; import { - closeSync, - constants, - existsSync, - fsyncSync, - mkdirSync, - openSync, - readFileSync, - renameSync, - unlinkSync, - writeFileSync, -} from "node:fs"; -import { dirname, join } from "node:path"; -import { catalogRevisionId, readCatalogPinSet } from "@openmapx/core/catalog-pins"; + catalogRevisionId, + listCatalogProposals, + readCatalogPinSet, + readCatalogProposal, + retainCatalogProposal, +} from "@openmapx/core/catalog-pins"; import type { OpsResultFor } from "@openmapx/core/ops"; import { OpsNotWiredError, type OpsRuntime } from "./runtime"; @@ -54,33 +49,6 @@ function readLockAt(path: string): LockRecord { } as LockRecord; } -/** Write-then-rename so a reader never observes a half-written lock. */ -function atomicWriteJson(path: string, value: unknown): void { - mkdirSync(dirname(path), { recursive: true }); - const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`; - const descriptor = openSync( - temporary, - constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, - 0o600, - ); - try { - writeFileSync(descriptor, `${JSON.stringify(value, null, 2)}\n`, "utf-8"); - fsyncSync(descriptor); - } finally { - closeSync(descriptor); - } - try { - renameSync(temporary, path); - } catch (error) { - try { - unlinkSync(temporary); - } catch { - // The rename already failed; surface that, not the cleanup. - } - throw error; - } -} - export interface TransitousLockRuntimeOptions { rootDir: string; now?: () => Date; @@ -115,10 +83,34 @@ export function createTransitousLockRuntime( const active = pair(); return { active, catalogRevisionId: active ? catalogRevisionId(active) : null }; }; - runtime["transitousLock.inspect"] = async () => ({ - active: pair()?.transitous ?? null, - proposed: readLockAt(proposedPath), - }); + runtime["transitousLock.inspect"] = async (operation) => { + const base = { active: pair()?.transitous ?? null, proposed: readLockAt(proposedPath) }; + if (operation.proposalId === undefined && operation.offset === undefined) return base; + const installation = openControlInstallation({ + infraDir: join(options.rootDir, "infra/docker"), + role: "ops-agent", + }); + try { + return { + ...base, + proposal: operation.proposalId + ? readCatalogProposal({ + rootDir: options.rootDir, + installation, + proposalId: operation.proposalId, + }) + : null, + page: listCatalogProposals({ + rootDir: options.rootDir, + installation, + offset: operation.offset, + }), + }; + } finally { + closeControlInstallation(installation); + } + }; + runtime["gbfsCatalogLock.inspect"] = async () => { const active = pair(); if (!active) throw new OpsNotWiredError(); @@ -126,15 +118,27 @@ export function createTransitousLockRuntime( return { commit, url, sha256, lockedAt, lockedBy }; }; - runtime["transitousLock.propose"] = async (operation) => { - atomicWriteJson(proposedPath, { - ref: operation.ref, - submodules: operation.submodules, - lockedAt: now().toISOString(), - lockedBy: operation.lockedBy, - ...(operation.comment ? { comment: operation.comment } : {}), + runtime["transitousLock.propose"] = async ({ kind: _kind, ...proposal }, context) => { + const installation = openControlInstallation({ + infraDir: join(options.rootDir, "infra/docker"), + role: "ops-agent", }); - return { ref: operation.ref, proposed: true as const }; + try { + const retained = await retainCatalogProposal({ + rootDir: options.rootDir, + installation, + proposal, + now: now(), + signal: context.signal, + }); + return { + proposalId: retained.proposalId, + catalogRevisionId: retained.pins.generationId, + proposed: true as const, + }; + } finally { + closeControlInstallation(installation); + } }; // The legacy ref-only approval cannot prove paired generation/pointer ownership. diff --git a/apps/web/src/app/admin/(full-admin)/transit/LockfileCard.tsx b/apps/web/src/app/admin/(full-admin)/transit/LockfileCard.tsx index 79a2da4f8..82811ff4d 100644 --- a/apps/web/src/app/admin/(full-admin)/transit/LockfileCard.tsx +++ b/apps/web/src/app/admin/(full-admin)/transit/LockfileCard.tsx @@ -1,168 +1,541 @@ "use client"; - -import LockIcon from "@mui/icons-material/Lock"; -import LockOpenIcon from "@mui/icons-material/LockOpen"; import RefreshIcon from "@mui/icons-material/Refresh"; import Alert from "@mui/material/Alert"; import Box from "@mui/material/Box"; import Button from "@mui/material/Button"; +import Checkbox from "@mui/material/Checkbox"; import Chip from "@mui/material/Chip"; +import CircularProgress from "@mui/material/CircularProgress"; +import FormControlLabel from "@mui/material/FormControlLabel"; +import MenuItem from "@mui/material/MenuItem"; import Paper from "@mui/material/Paper"; import Stack from "@mui/material/Stack"; import TextField from "@mui/material/TextField"; import Typography from "@mui/material/Typography"; import { adminErrorMessage } from "@openmapx/core/api-admin-sdk"; -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; +import { TransitCredentialsSection } from "@/components/admin/services/TransitCredentialsSection"; import { useAdminToast } from "@/components/admin/shared/AdminToast"; import { formatTransitAdminTimestamp } from "@/components/admin/shared/adminTimestamp"; import { ConfirmDialog } from "@/components/admin/shared/ConfirmDialog"; +import { useAccountAdmin } from "@/lib/admin/accountAdmin"; import type { TransitStateSummary } from "@/lib/admin/transitHooks"; -import { useBumpTransitous } from "@/lib/admin/transitHooks"; - -export function LockfileCard({ state }: { state: TransitStateSummary }) { - const hasLock = !!state.transitousRef; - const [branch, setBranch] = useState("main"); - const [confirmOpen, setConfirmOpen] = useState(false); - const bump = useBumpTransitous(); - const showToast = useAdminToast(); +import { + transitPinExecutionPending, + useActivateTransitPins, + useCancelTransitPinExecution, + useProposeTransitPins, + useTransitPinExecution, + useTransitPins, + useValidateTransitPins, +} from "@/lib/admin/transitPinHooks"; +export function LockfileCard(_props: { state: TransitStateSummary }) { + const { apiUrl, actorId } = useAccountAdmin(); + return ; +} +function CatalogPinView() { + const [branch, setBranch] = useState("main"), + [proposalId, setProposalId] = useState(), + [offset, setOffset] = useState(0), + [diffOffset, setDiffOffset] = useState(0), + [history, setHistory] = useState([]), + [countries, setCountries] = useState(""), + [ackScope, setAckScope] = useState(null), + [confirmScope, setConfirmScope] = useState(null), + [ownedJobId, setOwnedJobId] = useState(null); + const touched = useRef(false), + alive = useRef(true), + announced = useRef(null); + const query = useTransitPins({ proposalId, offset, diffOffset }), + propose = useProposeTransitPins(), + validate = useValidateTransitPins(), + activate = useActivateTransitPins(), + cancel = useCancelTransitPinExecution(), + own = useTransitPinExecution(ownedJobId), + toast = useAdminToast(); + useEffect(() => { + alive.current = true; + return () => { + alive.current = false; + }; + }, []); + const data = query.data, + configuredCountries = data?.allowedCountries.join(",") ?? ""; + useEffect(() => { + if (!touched.current) setCountries(configuredCountries); + }, [configuredCountries]); + const reviewScope = JSON.stringify([ + proposalId, + data?.diff?.digest, + data?.validation?.validationId, + data?.validationEligible, + ]); + const ack = ackScope === reviewScope, + confirm = confirmScope === reviewScope && ack; + const setAck = (value: boolean) => setAckScope(value ? reviewScope : null); + const setConfirm = (value: boolean) => setConfirmScope(value ? reviewScope : null); + useEffect(() => { + const result = own.data; + if (!result || transitPinExecutionPending(result) || announced.current === result.id) return; + announced.current = result.id; + if ( + result.status === "ok" && + result.result?.kind === "catalog.activate" && + result.result.outcome === "activated" + ) + toast("Validated catalog pair activated", "success"); + else if ( + result.status === "ok" && + result.result?.kind === "catalog.validate" && + result.result.status === "passed" + ) + toast("Validation passed; review and acknowledge the diff before activation", "success"); + }, [own.data, toast]); + const execution = ownedJobId ? own.data : data?.execution, + observing = Boolean(ownedJobId && !own.data), + busy = + observing || + transitPinExecutionPending(execution) || + transitPinExecutionPending(data?.execution), + submitting = propose.isPending || validate.isPending || activate.isPending || cancel.isPending; + const selectedCountries = [ + ...new Set( + countries + .split(",") + .map((value) => value.trim().toLowerCase()) + .filter(Boolean), + ), + ].sort(), + invalidCountries = + selectedCountries.length > 256 || + selectedCountries.some((value) => !/^[a-z]{2,3}$/.test(value)); + const errors = Object.entries({ + propose: propose.error, + validate: validate.error, + activate: activate.error, + cancel: cancel.error, + }).filter(([, error]) => Boolean(error)); + const choose = (id: string) => { + setProposalId(id || undefined); + setDiffOffset(0); + setHistory([]); + setAck(false); + setConfirm(false); + propose.reset(); + validate.reset(); + activate.reset(); + cancel.reset(); + }; return ( - - {hasLock ? ( - - ) : ( - - )} - + - Transitous lock - - - - - - {!hasLock && ( - - No infra/docker/transitous.lock.json found. Run the bump CLI to pin the - catalog to a specific upstream ref before syncing. - - )} - - - - Ref - - - {state.transitousRef ?? "—"} + + Catalog pins - - - } + onClick={() => { + void query.refetch(); + if (ownedJobId) void own.refetch(); }} > - Locked at - - - {formatTransitAdminTimestamp(state.transitousLockedAt)} - - - - + + {query.isLoading && } + {query.isError && ( + + Catalog review is unavailable. Restore instance connectivity and refresh. + + )} + {data && ( + <> + + + + Transitous: {data.active?.transitous.ref ?? "Not initialized"} + + + Atlas: {data.active?.transitous.submodules["transitland-atlas"] ?? "—"} + + + GBFS: {data.active?.gbfs.commit ?? "—"} + + + Active pins stay selected while a proposal is validated. + + + + setBranch(event.target.value)} + /> + + + choose(event.target.value)} + > + Select a proposal + {data.proposal && + !data.page.proposals.some( + (entry) => entry.proposalId === data.proposal?.proposalId, + ) && ( + + {data.proposal.pins.transitous.ref} · {data.proposal.proposalId.slice(0, 8)} + + )} + {data.page.proposals.map((entry) => ( + + {entry.transitousRef} · {formatTransitAdminTimestamp(entry.createdAt)} ·{" "} + {entry.proposalId.slice(0, 8)} + + ))} + + {data.page.total > 100 && ( + + + + {offset + 1}–{offset + data.page.proposals.length} of {data.page.total} + + + + )} + {data.proposal && ( + <> + + + + Transitous: {data.proposal.pins.transitous.ref} + + + Atlas: {data.proposal.pins.transitous.submodules["transitland-atlas"] ?? "—"} + + + GBFS: {data.proposal.pins.gbfs.commit} + + + {data.diff && ( + + + Source and license review · {data.diff.total} changes + + + Exact diff: {data.diff.digest} + + {data.diff.rows.map((row) => ( + + + {row.change}: {row.region} / {row.source} · {row.fields.join(", ")} · + license {row.licenseStatus === "unknown" ? "unknown" : "declared"} + + + {JSON.stringify({ before: row.before, after: row.after }, null, 2)} + + + ))} + {data.diff.total === 0 && ( + + No source metadata changes in this pair. + + )} + + + + {data.diff.total + ? `${data.diff.offset + 1}–${data.diff.offset + data.diff.rows.length}` + : "0"}{" "} + of {data.diff.total} + + + + + )} + {data.blocker && {data.blocker}} + + { + touched.current = true; + setCountries(event.target.value); + setAck(false); + }} + /> + + + {data.validation && ( + + {data.validation.status === "passed" + ? `Validation ${data.validationEligible ? "passed" : "is no longer eligible for activation"}. Countries: ${data.validation.countries.join(", ") || "planet"}. Valid until ${formatTransitAdminTimestamp(data.validation.expiresAt)}.` + : data.validation.reason} + + )} + {data.validation?.status === "passed" && ( + + Tested slot {data.validation.slot} · epoch {data.validation.epoch} · image{" "} + {data.validation.imageDigest} · manifest {data.validation.manifestHash} + + )} + setAck(event.target.checked)} + disabled={!data.validationEligible || busy || submitting} + /> + } + label="I reviewed this exact source and licensing diff, including entries with unknown licensing." + /> + + + + )} + + )} + {(observing || execution) && ( + - Locked by - - {state.transitousLockedBy ?? "—"} - + + Catalog job {ownedJobId ?? execution?.id}:{" "} + {execution?.status ?? "connecting to retained execution"} + + {execution?.progress != null && ( + {Math.round(execution.progress * 100)}% + )} + {Array.from( + new Map( + execution?.diagnostics.tail + .slice(-3) + .map((line) => [`${line.stream}:${line.message}`, line]), + ).values(), + ).map((line) => ( + + {line.message} + + ))} + {execution && transitPinExecutionPending(execution) && ( + + )} + {execution?.publication === "unknown" && ( + + Outcome is still being reconciled. New conflicting work stays blocked. + + )} + + )} + {own.isError && ( + + The job observer disconnected. Its retained execution continues; refresh to reconnect. + + )} + {errors.map(([kind, error]) => ( + + {adminErrorMessage( + error, + "Catalog request failed. Inspect its retained job before retrying.", + )} + + ))} - - Pin changes are explicit, admin-only, rate-limited, and written to the audit log. Run a - transit sync after reviewing the new catalog diff. - - setBranch(event.target.value)} - sx={{ mt: 1.5, maxWidth: 280 }} - /> setConfirmOpen(false)} - onConfirm={() => - bump.mutate(branch.trim(), { - onSuccess: () => { - showToast("Transitous pin updated"); - setConfirmOpen(false); - }, - onError: (error) => - showToast( - adminErrorMessage( - error, - error instanceof Error ? error.message : "Pin update failed", - ), - "error", - ), - }) - } + loading={activate.isPending} + onCancel={() => setConfirm(false)} + onConfirm={async () => { + const selected = data?.proposal, + validation = data?.validation, + diff = data?.diff; + if (!selected || !validation || !diff || !ack || !data?.validationEligible) return; + try { + const result = await activate.mutateAsync({ + proposalId: selected.proposalId, + validationId: validation.validationId, + expectedGenerationId: selected.baseGenerationId, + diffDigest: diff.digest, + acknowledgeLicenseChanges: true, + }); + if (alive.current) { + setOwnedJobId(result.value.jobId); + setConfirm(false); + setAck(false); + } + } catch { + /* Keep exact request identity and show error. */ + } + }} /> ); diff --git a/apps/web/src/app/admin/(full-admin)/transit/__tests__/LockfileCard.test.tsx b/apps/web/src/app/admin/(full-admin)/transit/__tests__/LockfileCard.test.tsx new file mode 100644 index 000000000..1eded737a --- /dev/null +++ b/apps/web/src/app/admin/(full-admin)/transit/__tests__/LockfileCard.test.tsx @@ -0,0 +1,218 @@ +import { QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { cleanup, createTestQueryClient, fireEvent, render, screen, waitFor } from "@/test"; +import { LockfileCard } from "../LockfileCard"; + +const mocks = vi.hoisted(() => ({ + get: vi.fn(), + propose: vi.fn(), + validate: vi.fn(), + activate: vi.fn(), + status: vi.fn(), + cancel: vi.fn(), + toast: vi.fn(), + apiUrl: "http://api.test", + actorId: "admin-a", + eligible: true, + publication: "committed", + validationStatus: "passed", +})); +vi.mock("@/lib/admin/accountAdmin", () => ({ + useAccountAdmin: () => ({ apiUrl: mocks.apiUrl, actorId: mocks.actorId }), +})); +vi.mock("@/components/admin/shared/AdminToast", () => ({ useAdminToast: () => mocks.toast })); +vi.mock("@/components/admin/services/TransitCredentialsSection", () => ({ + TransitCredentialsSection: ({ proposalId }: { proposalId?: string }) => ( +
Credentials for {proposalId ?? "active"}
+ ), +})); +vi.mock("@openmapx/core/api-admin-sdk", async (original) => ({ + ...(await original()), + getAdminTransitPins: mocks.get, + proposeAdminTransitPins: mocks.propose, + validateAdminTransitPins: mocks.validate, + activateAdminTransitPins: mocks.activate, + getAdminTransitPinExecution: mocks.status, + cancelAdminTransitPinExecution: mocks.cancel, +})); +const proposalId = "00000000-0000-4000-8000-000000000001", + validationId = "00000000-0000-4000-8000-000000000002", + jobId = "00000000-0000-4000-8000-000000000003"; +const reply = (data: unknown, status = 200) => ({ data, status, headers: new Headers() }); +const proposal = { + proposalId, + baseGenerationId: "a".repeat(64), + diffDigest: "d".repeat(64), + sourcePolicyRevision: "s".repeat(64), + pins: { + generationId: "b".repeat(64), + transitous: { ref: "main@proposed", submodules: { "transitland-atlas": "atlas-proposed" } }, + gbfs: { commit: "gbfs-proposed" }, + }, +}; +function review(selected: boolean) { + return { + active: { + transitous: { ref: "main@active", submodules: { "transitland-atlas": "atlas-active" } }, + gbfs: { commit: "gbfs-active" }, + }, + catalogRevisionId: "a".repeat(64), + allowedCountries: ["de"], + page: { + proposals: [ + { proposalId, transitousRef: "main@proposed", createdAt: "2026-10-23T00:00:00Z" }, + ], + total: 1, + nextOffset: null, + }, + proposal: selected ? proposal : null, + diff: selected + ? { + digest: proposal.diffDigest, + rows: [ + { + id: "feed", + change: "changed", + region: "de", + source: "bus", + fields: ["license"], + licenseStatus: "unknown", + before: { url: "https://old.test" }, + after: { url: "https://new.test" }, + }, + ], + total: 1, + offset: 0, + nextOffset: null, + } + : null, + validationInputs: {}, + validation: selected + ? { + status: mocks.validationStatus, + reason: "Staging import failed", + validationId, + countries: ["de"], + slot: "B", + epoch: validationId, + manifestHash: "e".repeat(64), + imageDigest: "sha256:test", + expiresAt: "2026-10-30T00:00:00Z", + } + : null, + validationEligible: mocks.eligible, + blocker: null, + execution: + mocks.publication === "unknown" + ? { id: jobId, status: "interrupted", publication: "unknown", diagnostics: { tail: [] } } + : null, + }; +} +beforeEach(() => { + vi.clearAllMocks(); + mocks.apiUrl = "http://api.test"; + mocks.actorId = "admin-a"; + mocks.eligible = true; + mocks.publication = "committed"; + mocks.validationStatus = "passed"; + mocks.get.mockImplementation(async (query) => + reply( + review( + Boolean(query && typeof query === "object" && "proposalId" in query && query.proposalId), + ), + ), + ); + mocks.propose.mockResolvedValue( + reply({ proposalId, catalogRevisionId: proposal.pins.generationId }), + ); + mocks.validate.mockResolvedValue(reply({ jobId }, 202)); + mocks.activate.mockResolvedValue(reply({ jobId }, 202)); + mocks.status.mockResolvedValue( + reply({ + id: jobId, + operationKey: "retained", + fingerprint: "f".repeat(64), + status: "running", + publication: "committed", + diagnostics: { tail: [], truncated: false }, + }), + ); +}); +afterEach(cleanup); +function mount() { + const client = createTestQueryClient(); + const element = () => ( + + + + ); + const view = render(element()); + return { ...view, client, rerenderView: () => view.rerender(element()) }; +} +async function selectProposal() { + fireEvent.mouseDown(await screen.findByLabelText("Proposal")); + fireEvent.click(await screen.findByRole("option", { name: /main@proposed/ })); + await screen.findByText("Credentials for " + proposalId); +} +it("creates a paired proposal without announcing an active catalog change", async () => { + mount(); + fireEvent.click(await screen.findByRole("button", { name: "Create proposal" })); + await waitFor(() => expect(mocks.propose).toHaveBeenCalledTimes(1)); + expect(mocks.propose.mock.calls[0]?.[0]).toEqual({ branch: "main" }); + const headers = mocks.propose.mock.calls[0]?.[1] as Record; + expect(headers["idempotency-key"]?.startsWith("opk")).toBe(true); + await waitFor(() => + expect(mocks.toast).toHaveBeenCalledWith( + "Proposal created; active pins are unchanged", + "success", + ), + ); + expect(screen.getByText("Transitous: main@active")).toBeInTheDocument(); + expect(mocks.activate).not.toHaveBeenCalled(); +}); +it("requires the exact diff acknowledgement and final confirmation before activation", async () => { + mount(); + await selectProposal(); + const button = screen.getByRole("button", { name: "Activate validated pair" }); + expect(button).toBeDisabled(); + expect(screen.getByText(/license unknown/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole("checkbox")); + fireEvent.click(button); + expect(mocks.activate).not.toHaveBeenCalled(); + fireEvent.click(screen.getAllByRole("button", { name: "Activate validated pair" }).at(-1)!); + await waitFor(() => expect(mocks.activate).toHaveBeenCalledTimes(1)); + expect(mocks.activate.mock.calls[0]?.slice(0, 2)).toEqual([ + proposalId, + { + validationId, + expectedGenerationId: proposal.baseGenerationId, + diffDigest: proposal.diffDigest, + acknowledgeLicenseChanges: true, + }, + ]); + expect(mocks.toast).not.toHaveBeenCalledWith("Validated catalog pair activated", "success"); + expect(await screen.findByText(/Catalog job .*running/)).toBeInTheDocument(); +}); +it("blocks activation for failed evidence and unresolved publication", async () => { + mocks.eligible = false; + mocks.validationStatus = "failed"; + mocks.publication = "unknown"; + mount(); + await selectProposal(); + expect(screen.getByText("Staging import failed")).toBeInTheDocument(); + expect(screen.getByRole("checkbox")).toBeDisabled(); + expect(screen.getByRole("button", { name: "Activate validated pair" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Validate inactive slot" })).toBeDisabled(); + expect(mocks.activate).not.toHaveBeenCalled(); +}); +it("does not cancel a remote job when its observer unmounts or the account changes", async () => { + const view = mount(); + await selectProposal(); + fireEvent.click(screen.getByRole("button", { name: "Validate inactive slot" })); + await screen.findByText(/Catalog job .*running/); + mocks.actorId = "admin-b"; + view.rerenderView(); + await waitFor(() => expect(screen.queryByText(/Catalog job .*running/)).not.toBeInTheDocument()); + view.unmount(); + expect(mocks.cancel).not.toHaveBeenCalled(); +}); diff --git a/apps/web/src/components/admin/services/TransitCredentialsSection.test.tsx b/apps/web/src/components/admin/services/TransitCredentialsSection.test.tsx index c93aa67c2..72b2ab5d5 100644 --- a/apps/web/src/components/admin/services/TransitCredentialsSection.test.tsx +++ b/apps/web/src/components/admin/services/TransitCredentialsSection.test.tsx @@ -138,7 +138,7 @@ it("does not apply a completed proof after the observed map changes while the pr fireEvent.click( await screen.findByRole("button", { name: "Confirm administrator sign-in and apply" }), ); - query.setQueryData(["admin", "transit", "credentials", mocks.apiUrl, mocks.actorId], { + query.setQueryData(["admin", "transit", "credentials", mocks.apiUrl, mocks.actorId, "active"], { ...view, credentialsRevision: "e".repeat(64), }); diff --git a/apps/web/src/components/admin/services/TransitCredentialsSection.tsx b/apps/web/src/components/admin/services/TransitCredentialsSection.tsx index 5c115d2ff..c07cfc6f0 100644 --- a/apps/web/src/components/admin/services/TransitCredentialsSection.tsx +++ b/apps/web/src/components/admin/services/TransitCredentialsSection.tsx @@ -54,30 +54,44 @@ function restored(key: string): Reviewed | null { return null; } } -export function TransitCredentialsSection() { +export function TransitCredentialsSection({ proposalId }: { proposalId?: string } = {}) { const { apiUrl, actorId } = useAccountAdmin(); return ( ); } -function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: string }) { +function TransitCredentialView({ + apiUrl, + actorId, + proposalId, +}: { + apiUrl: string; + actorId: string; + proposalId?: string; +}) { const client = useMemo( () => createApiClient({ baseUrl: browserAdminApiBase(apiUrl), credentials: "include" }), [apiUrl], ); const queryClient = useQueryClient(), query = useQuery({ - queryKey: ["admin", "transit", "credentials", apiUrl, actorId], + queryKey: ["admin", "transit", "credentials", apiUrl, actorId, proposalId ?? "active"], queryFn: async ({ signal }) => - sdk.unwrapAdminResponse(await sdk.getAdminTransitCredentials({ client, signal })), + sdk.unwrapAdminResponse( + await sdk.getAdminTransitCredentials(proposalId ? { proposalId } : undefined, { + client, + signal, + }), + ), gcTime: 0, enabled: Boolean(actorId), }); - const storageKey = `openmapx-transit-credential-preview:${JSON.stringify([apiUrl, actorId])}`; + const storageKey = `openmapx-transit-credential-preview:${JSON.stringify(proposalId ? [apiUrl, actorId, proposalId] : [apiUrl, actorId])}`; const [drafts, setDrafts] = useState>({}), [deletions, setDeletions] = useState([]), [deleting, setDeleting] = useState(null), @@ -126,6 +140,7 @@ function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: s const operationKey = `opk1_${crypto.randomUUID().replaceAll("-", "")}`; const body = { catalogRevisionId: data.catalogRevisionId, + ...(proposalId ? { proposalId } : {}), expectedCredentialsRevision: data.credentialsRevision, expectedCurrentRevisionId: data.configurationRevisionId, changes, @@ -158,6 +173,7 @@ function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: s if ( !current.editable || selected.preview.catalogRevisionId !== current.catalogRevisionId || + selected.preview.proposalId !== current.proposalId || selected.preview.credentialsRevision !== current.credentialsRevision || selected.preview.expectedCurrentRevisionId !== current.configurationRevisionId || Date.parse(selected.preview.expiresAt) <= Date.now() @@ -187,7 +203,7 @@ function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: s if (identity !== epoch.current) return; setSuccess( result.configuration.outcome === "committed" && result.configuration.followUp === "complete" - ? "Feed credentials applied." + ? "Credential map applied to Data Manager. Validate the proposal or sync the active pins to refresh feed routing." : "Credential application needs attention. Inspect the original operation in Activity.", ); setReviewed(null); @@ -196,7 +212,12 @@ function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: s } catch { /* No secret is stored; private drafts still clear. */ } - await queryClient.invalidateQueries({ queryKey: ["admin", "transit"] }); + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: ["admin", "transit", "credentials", apiUrl, actorId], + }), + queryClient.invalidateQueries({ queryKey: ["admin", "transit", "pins", apiUrl, actorId] }), + ]); } catch { if (identity === epoch.current) { setError( @@ -222,6 +243,7 @@ function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: s reviewed && data && (reviewed.preview.catalogRevisionId !== data.catalogRevisionId || + reviewed.preview.proposalId !== data.proposalId || reviewed.preview.credentialsRevision !== data.credentialsRevision || reviewed.preview.expectedCurrentRevisionId !== data.configurationRevisionId || Date.parse(reviewed.preview.expiresAt) <= Date.now()); @@ -233,7 +255,8 @@ function TransitCredentialView({ apiUrl, actorId }: { apiUrl: string; actorId: s Transit feed credentials Values stay masked. Blank inputs keep the existing value; deletion is a separate, - confirmed action. Changes use the selected catalog pins. + confirmed action. Changes use the selected catalog pins. Validation requires credentials + only for its selected countries and enabled sources. {query.isLoading && } {query.isError && ( diff --git a/apps/web/src/lib/admin/transitPinHooks.ts b/apps/web/src/lib/admin/transitPinHooks.ts new file mode 100644 index 000000000..13de932b1 --- /dev/null +++ b/apps/web/src/lib/admin/transitPinHooks.ts @@ -0,0 +1,173 @@ +"use client"; +import { createApiClient } from "@openmapx/core"; +import * as sdk from "@openmapx/core/api-admin-sdk"; +import { createControlRootOperationKey } from "@openmapx/core/ops/control-root-key"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useEffect, useMemo, useRef } from "react"; +import { useAccountAdmin } from "./accountAdmin"; +import { browserAdminApiBase } from "./browserAdminApiBase"; + +function useScope() { + const { apiUrl, actorId } = useAccountAdmin(); + const client = useMemo( + () => createApiClient({ baseUrl: browserAdminApiBase(apiUrl), credentials: "include" }), + [apiUrl], + ); + return { apiUrl, actorId, client }; +} +export function useTransitPins(query: { + proposalId?: string; + offset?: number; + diffOffset?: number; +}) { + const { apiUrl, actorId, client } = useScope(); + return useQuery({ + queryKey: [ + "admin", + "transit", + "pins", + apiUrl, + actorId, + query.proposalId ?? null, + query.offset ?? 0, + query.diffOffset ?? 0, + ], + enabled: Boolean(actorId), + gcTime: 0, + queryFn: async ({ signal }) => + sdk.unwrapAdminResponse(await sdk.getAdminTransitPins(query, { client, signal })), + refetchInterval: (current) => + current.state.data?.execution && + ["queued", "running", "cancel_pending"].includes(current.state.data.execution.status) + ? 5000 + : 60000, + }); +} +function useCatalogAction( + budgetMs: number, + execute: ( + input: Input, + key: string, + options: { client: ReturnType; signal: AbortSignal }, + ) => Promise, +) { + const { apiUrl, actorId, client } = useScope(), + qc = useQueryClient(); + const identities = useRef(new Map()), + observers = useRef(new Set()); + useEffect( + () => () => { + for (const observer of observers.current) observer.abort(); + }, + [], + ); + return useMutation({ + retry: false, + gcTime: 0, + mutationFn: async (input: Input) => { + if (!actorId) throw Error("Administrator session required"); + const address = JSON.stringify([apiUrl, actorId, input]), + controller = new AbortController(); + const key = + identities.current.get(address) ?? + createControlRootOperationKey({ + notAfter: Date.now() + budgetMs, + nonce: crypto.getRandomValues(new Uint8Array(16)), + }); + identities.current.set(address, key); + observers.current.add(controller); + try { + const value = await execute(input, key, { client, signal: controller.signal }); + identities.current.delete(address); + return { value, apiUrl, actorId }; + } finally { + observers.current.delete(controller); + } + }, + onSuccess: (result) => { + void qc.invalidateQueries({ + queryKey: ["admin", "transit", "pins", result.apiUrl, result.actorId], + }); + }, + }); +} +export function useProposeTransitPins() { + return useCatalogAction(3600000, async (input: { branch: string }, key, options) => + sdk.unwrapAdminResponse( + await sdk.proposeAdminTransitPins(input, { "idempotency-key": key }, options), + ), + ); +} +export function useValidateTransitPins() { + return useCatalogAction( + 72 * 3600000, + async (input: { proposalId: string; countries: string[] }, key, options) => + sdk.unwrapAdminResponse( + await sdk.validateAdminTransitPins( + input.proposalId, + { countries: input.countries }, + { "idempotency-key": key }, + options, + ), + ), + ); +} +export function useActivateTransitPins() { + type Input = { + proposalId: string; + validationId: string; + expectedGenerationId: string | null; + diffDigest: string; + acknowledgeLicenseChanges: boolean; + }; + return useCatalogAction(3600000, async (input: Input, key, options) => { + const { proposalId, ...body } = input; + return sdk.unwrapAdminResponse( + await sdk.activateAdminTransitPins(proposalId, body, { "idempotency-key": key }, options), + ); + }); +} +export function useCancelTransitPinExecution() { + return useCatalogAction( + 120000, + async (input: { id: string; operationKey: string; fingerprint: string }, _key, options) => { + const { id, ...body } = input; + return sdk.unwrapAdminResponse(await sdk.cancelAdminTransitPinExecution(id, body, options)); + }, + ); +} +export function transitPinExecutionPending( + value: { status: string; publication: string } | null | undefined, +): boolean { + return Boolean( + value && + (["queued", "running", "cancel_pending"].includes(value.status) || + value.publication === "unknown"), + ); +} +export function useTransitPinExecution(id: string | null) { + const { apiUrl, actorId, client } = useScope(), + qc = useQueryClient(), + completed = useRef(null); + const query = useQuery({ + queryKey: ["admin", "transit", "pin-execution", apiUrl, actorId, id], + enabled: Boolean(actorId && id), + gcTime: 0, + retry: false, + queryFn: async ({ signal }) => { + if (!id) throw Error("Catalog execution required"); + return sdk.unwrapAdminResponse(await sdk.getAdminTransitPinExecution(id, { client, signal })); + }, + refetchInterval: (current) => + !current.state.data || transitPinExecutionPending(current.state.data) ? 5000 : false, + }); + useEffect(() => { + const value = query.data; + if (!value || transitPinExecutionPending(value)) return; + const key = `${value.id}:${value.fingerprint}:${value.status}`; + if (completed.current === key) return; + completed.current = key; + void qc.invalidateQueries({ queryKey: ["admin", "transit", "pins", apiUrl, actorId] }); + }, [query.data, apiUrl, actorId, qc]); + return query; +} diff --git a/docs/docs/administration/motis-operations-profiles.md b/docs/docs/administration/motis-operations-profiles.md index c3a038b77..9dd8f821c 100644 --- a/docs/docs/administration/motis-operations-profiles.md +++ b/docs/docs/administration/motis-operations-profiles.md @@ -73,7 +73,43 @@ Recovery drill: ## Pin upgrades -`openmapx transitous bump` and `POST /transit/bump` write `*.proposed.json`; they do not mutate active pins. Review catalog/feed/license diffs, build the inactive slot, and run the timetable, rental, planner, attribution, and response-contract canaries. API activation requires typing the exact proposed ref at `/transit/bump/approve`. Keep previous pins with the previous healthy slot. +`openmapx transitous bump` and `POST /transit/bump` resolve immutable paired +Transitous/Atlas and GBFS proposals; they never change active pins. In +**Admin → Transit → Catalog pins**, review the paginated source, country, URL, +attribution and licensing diff. Unknown licensing is shown explicitly. + +Configure credentials against the proposed pair before validation. Values remain +masked, and existing orphaned keys are preserved. Regional validation requires +credentials only for the selected countries and enabled sources. Planet validation +checks all enabled sources. + +**Validate inactive slot** builds the exact proposal through the functional MOTIS +probes, restores the current proxy and stops staging. Validation is valid for at +most 24 hours and binds the source policy, overlay, countries, consumed credentials, +image, slot, epoch and candidate manifest. Changed inputs require new validation. +The previous healthy inactive generation is archived before its slot is reused; +insufficient retention capacity blocks validation. Candidate directories must be +owned by the data-manager and must not be writable by other users. + +After validation, acknowledge the exact diff and confirm **Activate validated pair**. +The operation keeps its original target fence through promotion, atomic pair +publication and child settlement. CLI and scheduled bumps cannot activate proposals; +legacy raw-ref approval is closed. A disconnected observer does not cancel work. +Cancellation requests wait for actual effect containment. An unknown outcome keeps +conflicting work blocked; reconnect to the original job instead of submitting a +second attempt or manually replacing its pointers. Original signed commit receipts +support recovery after either process restarts. + +The actual catalog-worker/agent canary additionally runs the pinned MOTIS image +against disposable PostgreSQL and uniquely named containers: + +```bash +OPENMAPX_RUN_DATABASE_TESTS=1 OPENMAPX_CATALOG_LIVE_MOTIS=1 pnpm exec vitest run --project node services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts +``` + +Pre-pull the repository-pinned MOTIS image before enabling this opt-in check. The +test removes only its own containers; Docker registry/ownership I/O is a fixture +bridge, while MOTIS import, health and functional routing use the actual image. Mutable tags such as `latest` are prohibited. MOTIS currently uses the repository-wide pinned release. diff --git a/docs/docs/developer/cli-reference.md b/docs/docs/developer/cli-reference.md index f3f1f0841..6a7455b9e 100644 --- a/docs/docs/developer/cli-reference.md +++ b/docs/docs/developer/cli-reference.md @@ -269,19 +269,21 @@ probe. ## `transitous` -Manage the pin of the [Transitous](https://github.com/public-transport/transitous) -GTFS catalog that the data-manager consumes, recorded in -`infra/docker/transitous.lock.json`. - -| Command | Description | -| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `transitous show` | Print the current lockfile contents. | -| `transitous bump` | Fetch the catalog, summarize the feed changes, and update the lockfile. Flags: `--yes` (skip the confirmation prompt), `--branch ` (default `main`). | - -`transitous bump` reads from a catalog clone the data-manager maintains, so run a -transit sync once (`pnpm openmapx data sync`) before the first bump. -After bumping, restart the data-manager to pick up the new ref, or wait for the -next scheduled sync. +Manage the paired [Transitous](https://github.com/public-transport/transitous), +Transitland Atlas and MobilityData GBFS catalog pins used by the data-manager. +The managed pair is selected atomically under `infra/docker/catalog-pins/`. + +| Command | Description | +| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `transitous show` | Print the currently selected pair, including its generation ID. | +| `transitous bump` | Ask the installed data-manager to resolve a paired proposal. Flags: `--yes` (skip confirmation), `--branch ` (default `main`). Active pins stay unchanged. | + +Review the proposal in **Admin → Transit → Catalog pins**. Inspect the source, +country, URL and licensing changes; configure any newly required feed credentials; +then validate the inactive MOTIS slot. Activation requires passing, unexpired +validation, acknowledgement of the exact diff and final confirmation. It promotes +the tested dataset and selects its pair under the same retained operation. +Scheduled auto-bumps also create proposals and cannot approve them. ## `transit-registry` diff --git a/packages/cli/__tests__/cli-transitous-api-keys.test.ts b/packages/cli/__tests__/cli-transitous-api-keys.test.ts index 3c74d9b67..3414f1b1b 100644 --- a/packages/cli/__tests__/cli-transitous-api-keys.test.ts +++ b/packages/cli/__tests__/cli-transitous-api-keys.test.ts @@ -7,7 +7,12 @@ import { openControlInstallation, provisionControlInstallation, } from "@openmapx/core/build-artifacts"; -import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; +import { + createCatalogPinSet, + initializeManagedCatalogPins, + readCatalogPinSet, + retainCatalogProposal, +} from "@openmapx/core/catalog-pins"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { type CommandRunner, @@ -202,4 +207,42 @@ it("uses actual active pin A while upstream is B and preserves unrelated credent expect(readFileSync(output, "utf8")).toBe(malformed); writeFileSync(output, valid); await expect(generateTransitousApiKeys(options)).resolves.toMatchObject({ droppedCount: 0 }); + const proposedSha = git(upstream, ["rev-parse", "HEAD"]); + git(cache, ["fetch", "origin", proposedSha]); + const agent = openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["motis"], + }); + const proposalId = "00000000-0000-4000-8000-000000000003"; + const active = readCatalogPinSet(tmp)!; + const pins = createCatalogPinSet( + { ...active.transitous, ref: `main@${proposedSha}` }, + active.gbfs, + ); + try { + await retainCatalogProposal({ + rootDir: tmp, + installation: agent, + proposal: { + proposalId, + baseGenerationId: active.generationId, + pins, + diffDigest: "e".repeat(64), + sourcePolicyRevision: "f".repeat(64), + }, + }); + } finally { + closeControlInstallation(agent); + } + await expect(generateTransitousApiKeys({ ...options, proposalId })).resolves.toMatchObject({ + catalogRevisionId: pins.generationId, + droppedCount: 0, + }); + const proposedValues = JSON.parse(readFileSync(output, "utf8")); + expect(proposedValues).toHaveProperty("de/UpstreamB", ""); + expect(proposedValues).toHaveProperty("de/PinnedA", ""); + expect(proposedValues["retained/orphan"]).toBe(" exact value\n"); + expect(readCatalogPinSet(tmp)?.generationId).toBe(active.generationId); + expect(git(cache, ["rev-parse", "HEAD"])).toBe(sha); }, 45000); diff --git a/packages/cli/__tests__/transitous-bump.test.ts b/packages/cli/__tests__/transitous-bump.test.ts index 5b55d9377..992c3592c 100644 --- a/packages/cli/__tests__/transitous-bump.test.ts +++ b/packages/cli/__tests__/transitous-bump.test.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -58,6 +59,46 @@ describe("Transitous combined catalog bump", () => { expect(Object.fromEntries(result.countryCounts)).toEqual({ de: 1, at: 1 }); }); + it("includes same-name endpoint and attribution changes in the review", () => { + const oldFeeds = new Map([ + [ + "de", + { + sources: [ + { + name: "BVG", + url: "https://old.test/feed", + attribution: { text: "Original operator" }, + }, + ], + }, + ], + ]); + const newFeeds = new Map([ + [ + "de", + { + sources: [ + { name: "BVG", url: "https://new.test/feed", attribution: { text: "New operator" } }, + ], + }, + ], + ]); + expect(diffFeedFiles(oldFeeds, newFeeds).modifiedRegions).toEqual(["de"]); + }); + + it("bounds upstream commit metadata before it can become a proposal", async () => { + const fetchStub = async (input: string | URL | Request) => + String(input).includes("api.github.com") + ? new Response(" ".repeat(65536) + JSON.stringify({ sha: "e".repeat(40) })) + : new Response( + "Country Code,Name,Location,System ID,URL,Auto-Discovery URL,Supported Versions,Authentication Info URL\n", + ); + await expect(resolveGbfsCandidate("fixture", fetchStub as typeof fetch)).rejects.toThrow( + /byte limit/, + ); + }); + it("detects a no-op only when both compatible pins match", () => { expect(combinedPinsAreCurrent(TRANSITOUS, GBFS, "a".repeat(40), "c".repeat(40))).toBe(true); expect(combinedPinsAreCurrent(TRANSITOUS, GBFS, "a".repeat(40), "f".repeat(40))).toBe(false); @@ -105,22 +146,62 @@ describe("Transitous combined catalog bump", () => { expect(summary.licenseChanges).toEqual([{ region: "de", source: "BVG" }]); }); - it("writes paired proposals without changing active pins", () => { + it("refuses obsolete unsigned proposal file writes", () => { temporaryRoot = mkdtempSync(join(tmpdir(), "openmapx-bump-proposal-")); const directory = join(temporaryRoot, "infra", "docker"); mkdirSync(directory, { recursive: true }); writeFileSync(join(directory, "transitous.lock.json"), "active-transitous\n"); writeFileSync(join(directory, "gbfs-catalog.lock.json"), "active-gbfs\n"); - writeCombinedCatalogLockProposal(temporaryRoot, TRANSITOUS, GBFS); + expect(() => writeCombinedCatalogLockProposal(temporaryRoot!, TRANSITOUS, GBFS)).toThrow( + /installed paired review/, + ); expect(readFileSync(join(directory, "transitous.lock.json"), "utf-8")).toBe( "active-transitous\n", ); expect(readFileSync(join(directory, "gbfs-catalog.lock.json"), "utf-8")).toBe("active-gbfs\n"); - expect(readFileSync(join(directory, "transitous.lock.proposed.json"), "utf-8")).toContain( - TRANSITOUS.ref, - ); - expect(readFileSync(join(directory, "gbfs-catalog.lock.proposed.json"), "utf-8")).toContain( - GBFS.commit, + }); +}); + +it("delegates a CLI bump to the actual paired proposal HTTP operation", async () => { + const calls: Array<{ url: string; body: unknown }> = []; + const server = createServer(async (request, response) => { + let body = ""; + for await (const bytes of request) body += bytes; + calls.push({ url: request.url!, body: JSON.parse(body) }); + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify({ + ok: true, + unchanged: false, + proposed: true, + proposalId: "00000000-0000-4000-8000-000000000001", + catalogRevisionId: "e".repeat(64), + ref: TRANSITOUS.ref, + previousRef: TRANSITOUS.ref, + submoduleSha: TRANSITOUS.submodules["transitland-atlas"], + lockedAt: TRANSITOUS.lockedAt, + lockedBy: "catalog-review", + }), ); }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = (server.address() as import("node:net").AddressInfo).port; + const previous = process.env.DATA_MANAGER_URL; + process.env.DATA_MANAGER_URL = `http://127.0.0.1:${port}`; + try { + const { proposeReviewedCatalog } = await import("../src/commands/transitous-bump"); + const result = await proposeReviewedCatalog("main"); + expect(result).toMatchObject({ + proposed: true, + proposalId: "00000000-0000-4000-8000-000000000001", + }); + expect(calls).toEqual([{ url: "/transit/bump", body: { branch: "main" } }]); + } finally { + if (previous === undefined) delete process.env.DATA_MANAGER_URL; + else process.env.DATA_MANAGER_URL = previous; + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } }); diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index 71dd73490..2383ecffd 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -889,11 +889,13 @@ export function registerDataCommands(program: Command): void { "--output ", "Override output path (default: services/motis/tools/transitous/api-keys.json)", ) - .action(async (options: { output?: string }) => { + .option("--proposal ", "Use this retained paired proposal instead of active pins") + .action(async (options: { output?: string; proposal?: string }) => { try { log.dim("Reading pinned Transitous catalog and scanning feeds that require API keys..."); const result = await generateTransitousApiKeys({ outputPath: options.output, + proposalId: options.proposal, }); const preserved = result.preservedCount > 0 diff --git a/packages/cli/src/commands/transitous-bump.ts b/packages/cli/src/commands/transitous-bump.ts index f723925e0..69c007e52 100644 --- a/packages/cli/src/commands/transitous-bump.ts +++ b/packages/cli/src/commands/transitous-bump.ts @@ -1,72 +1,21 @@ -import { createHash } from "node:crypto"; -import { existsSync, readFileSync, renameSync, type unlinkSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import type { renameSync, unlinkSync, writeFileSync } from "node:fs"; import { createInterface } from "node:readline/promises"; -import { - closeControlInstallation, - openControlInstallation, - provisionControlInstallation, -} from "@openmapx/core/build-artifacts"; -import { - type CatalogPinSet, - initializeManagedCatalogPins, - readCatalogPinSet, -} from "@openmapx/core/catalog-pins"; -import { parseMobilityDataGbfsCsv } from "@openmapx/transitous-core"; +import { readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import { transitBumpResultSchema } from "@openmapx/core/data-manager-contracts"; +import * as dm from "@openmapx/core/data-manager-sdk"; +import { diffCatalogFeeds, resolveGbfsCatalogCandidate } from "@openmapx/transitous-core"; import type { Command } from "commander"; -import { execa } from "execa"; -import { - decodeGbfsCatalogLock, - type GbfsCatalogLock, -} from "../../../../services/data-manager/src/gbfs-catalog-lock"; +import type { GbfsCatalogLock } from "../../../../services/data-manager/src/gbfs-catalog-lock"; import { parseRefShaPair, type TransitousLock, } from "../../../../services/data-manager/src/transitous-lock"; +import { dataManagerUrl } from "../lib/data-manager-url"; +import { cliAcquisitionOptions, constrainCliDataManagerTransport } from "../lib/execution-deadline"; import { log } from "../lib/output"; import { repoPaths } from "../lib/paths"; -const TRANSITOUS_CATALOG_DIR = ".transitous-catalog"; -const MOBILITYDATA_REPO_API = "https://api.github.com/repos/MobilityData/gbfs/commits/master"; - -export async function resolveGbfsCandidate( - lockedBy: string, - fetchImpl: typeof fetch = fetch, - now: () => Date = () => new Date(), -): Promise<{ lock: GbfsCatalogLock; countryCounts: Map }> { - const commitResponse = await fetchImpl(MOBILITYDATA_REPO_API, { - headers: { "User-Agent": "Open-MapX-transitous-bump" }, - }); - if (!commitResponse.ok) - throw new Error(`MobilityData commit lookup failed: HTTP ${commitResponse.status}`); - const commitJson = (await commitResponse.json()) as { sha?: unknown }; - if (typeof commitJson.sha !== "string" || !/^[0-9a-f]{40}$/.test(commitJson.sha)) { - throw new Error("MobilityData commit lookup returned an invalid SHA"); - } - const url = `https://raw.githubusercontent.com/MobilityData/gbfs/${commitJson.sha}/systems.csv`; - const csvResponse = await fetchImpl(url, { - headers: { "User-Agent": "Open-MapX-transitous-bump" }, - }); - if (!csvResponse.ok) - throw new Error(`MobilityData systems.csv failed: HTTP ${csvResponse.status}`); - const csv = await csvResponse.text(); - const countryCounts = new Map(); - for (const row of parseMobilityDataGbfsCsv(csv)) { - countryCounts.set(row.countryCode, (countryCounts.get(row.countryCode) ?? 0) + 1); - } - return { - lock: { - schemaVersion: 1, - source: "mobilitydata-gbfs", - commit: commitJson.sha, - url, - sha256: createHash("sha256").update(csv).digest("hex"), - lockedAt: now().toISOString(), - lockedBy, - }, - countryCounts, - }; -} +export const resolveGbfsCandidate = resolveGbfsCatalogCandidate; export function combinedPinsAreCurrent( transitous: TransitousLock | null, @@ -80,26 +29,6 @@ export function combinedPinsAreCurrent( ); } -function transitousLockJson(lock: TransitousLock): string { - return `${JSON.stringify( - { - $schema: "./transitous.lock.schema.json", - ref: lock.ref, - submodules: lock.submodules, - lockedAt: lock.lockedAt, - lockedBy: lock.lockedBy, - ...(lock.comment ? { comment: lock.comment } : {}), - }, - null, - 2, - )}\n`; -} - -function gbfsLockJson(lock: GbfsCatalogLock): string { - decodeGbfsCatalogLock(lock); - return `${JSON.stringify({ $schema: "./gbfs-catalog.lock.schema.json", ...lock }, null, 2)}\n`; -} - /** Stage and replace the compatible Transitous/MobilityData pin set together. */ export function writeCombinedCatalogLocks( repoRoot: string, @@ -124,13 +53,10 @@ export function writeCombinedCatalogLockProposal( transitous: TransitousLock, gbfs: GbfsCatalogLock, ): void { - const transitousPath = join(repoRoot, "infra", "docker", "transitous.lock.proposed.json"); - const gbfsPath = join(repoRoot, "infra", "docker", "gbfs-catalog.lock.proposed.json"); - const suffix = `.candidate-${process.pid}`; - writeFileSync(`${transitousPath}${suffix}`, transitousLockJson(transitous), "utf-8"); - writeFileSync(`${gbfsPath}${suffix}`, gbfsLockJson(gbfs), "utf-8"); - renameSync(`${transitousPath}${suffix}`, transitousPath); - renameSync(`${gbfsPath}${suffix}`, gbfsPath); + void repoRoot; + void transitous; + void gbfs; + throw new Error("Catalog proposals require the installed paired review workflow"); } export interface FeedFile { @@ -146,146 +72,11 @@ export interface FeedDiffSummary { licenseChanges: Array<{ region: string; source: string }>; } -function gitArgs(catalogDir: string, ...rest: string[]): string[] { - return ["-c", `safe.directory=${catalogDir}`, "-C", catalogDir, ...rest]; -} - -async function runGit(catalogDir: string, args: string[]): Promise { - const result = await execa("git", gitArgs(catalogDir, ...args), { stdio: "pipe" }); - return result.stdout.trim(); -} - -async function listFeedsAtRevision( - catalogDir: string, - sha: string, -): Promise> { - const out = new Map(); - let listing: string; - try { - listing = await runGit(catalogDir, ["ls-tree", "--name-only", `${sha}:feeds`]); - } catch { - return out; - } - for (const line of listing.split("\n")) { - const name = line.trim(); - if (!name.endsWith(".json")) continue; - const region = name.replace(/\.json$/i, ""); - let raw: string; - try { - raw = await runGit(catalogDir, ["show", `${sha}:feeds/${name}`]); - } catch { - continue; - } - try { - out.set(region, JSON.parse(raw) as FeedFile); - } catch { - // Skip malformed feed files — we don't want a single broken JSON file to - // block a catalog bump on otherwise fine regions. - } - } - return out; -} - export function diffFeedFiles( oldFeeds: Map, newFeeds: Map, ): FeedDiffSummary { - const oldRegions = new Set(oldFeeds.keys()); - const newRegions = new Set(newFeeds.keys()); - const addedRegions = [...newRegions].filter((region) => !oldRegions.has(region)).sort(); - const removedRegions = [...oldRegions].filter((region) => !newRegions.has(region)).sort(); - const modifiedRegions: string[] = []; - let addedSources = 0; - let removedSources = 0; - const licenseChanges: FeedDiffSummary["licenseChanges"] = []; - - for (const region of newRegions) { - if (!oldRegions.has(region)) { - addedSources += (newFeeds.get(region)?.sources ?? []).length; - continue; - } - const oldNames = new Set( - (oldFeeds.get(region)?.sources ?? []) - .map((source) => source.name) - .filter((name): name is string => typeof name === "string"), - ); - const newNames = new Set( - (newFeeds.get(region)?.sources ?? []) - .map((source) => source.name) - .filter((name): name is string => typeof name === "string"), - ); - let regionChanged = false; - for (const name of newNames) { - if (!oldNames.has(name)) { - addedSources++; - regionChanged = true; - } - } - for (const name of oldNames) { - if (!newNames.has(name)) { - removedSources++; - regionChanged = true; - } - } - const oldByName = new Map( - (oldFeeds.get(region)?.sources ?? []).flatMap((source) => - source.name ? [[source.name, source] as const] : [], - ), - ); - for (const source of newFeeds.get(region)?.sources ?? []) { - if (!source.name || !oldByName.has(source.name)) continue; - const previous = oldByName.get(source.name); - if (JSON.stringify(previous?.license ?? null) !== JSON.stringify(source.license ?? null)) { - licenseChanges.push({ region, source: source.name }); - regionChanged = true; - } - } - if (regionChanged) modifiedRegions.push(region); - } - for (const region of removedRegions) { - removedSources += (oldFeeds.get(region)?.sources ?? []).length; - } - modifiedRegions.sort(); - return { - addedRegions, - removedRegions, - modifiedRegions, - addedSources, - removedSources, - licenseChanges: licenseChanges.sort((a, b) => - `${a.region}/${a.source}`.localeCompare(`${b.region}/${b.source}`), - ), - }; -} - -function printDiffSummary(summary: FeedDiffSummary): void { - log.info( - ` added regions: ${summary.addedRegions.length}${summary.addedRegions.length > 0 ? ` (${summary.addedRegions.join(", ")})` : ""}`, - ); - log.info( - ` removed regions: ${summary.removedRegions.length}${summary.removedRegions.length > 0 ? ` (${summary.removedRegions.join(", ")})` : ""}`, - ); - log.info( - ` modified regions: ${summary.modifiedRegions.length}${summary.modifiedRegions.length > 0 ? ` (${summary.modifiedRegions.join(", ")})` : ""}`, - ); - log.info(` added sources: ${summary.addedSources}`); - log.info(` removed sources: ${summary.removedSources}`); - log.info( - ` license changes: ${summary.licenseChanges.length}${summary.licenseChanges.length > 0 ? ` (${summary.licenseChanges.map((entry) => `${entry.region}/${entry.source}`).join(", ")})` : ""}`, - ); -} - -async function readSubmoduleSha(catalogDir: string, ref: string, path: string): Promise { - // `git ls-tree ` returns a line like: - // 160000 commit \t - // for a submodule entry. We pull that SHA out explicitly because - // `rev-parse :` resolves the tree, not the gitlink target. - const output = await runGit(catalogDir, ["ls-tree", ref, path]); - const match = output.match(/^\d+\s+commit\s+([0-9a-f]{40})\s/i); - if (!match) { - throw new Error(`Could not resolve submodule SHA for "${path}" at ${ref}`); - } - return match[1]; + return diffCatalogFeeds(oldFeeds, newFeeds); } async function promptConfirm(question: string): Promise { @@ -302,15 +93,17 @@ async function promptConfirm(question: string): Promise { } } -async function resolveLockedBy(): Promise { - try { - const result = await execa("git", ["config", "user.email"], { stdio: "pipe" }); - const email = result.stdout.trim(); - if (email) return email; - } catch { - // ignore — fall through to env var - } - return process.env.USER ?? "unknown"; +export async function proposeReviewedCatalog(branch: string) { + const transport = constrainCliDataManagerTransport( + dm.createDataManagerTransport({ + ...cliAcquisitionOptions(), + baseUrl: dataManagerUrl(), + authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), + }), + ); + return transitBumpResultSchema.parse( + dm.unwrapDataManagerResponse(await dm.proposeTransitBump({ branch }, { transport })), + ); } export function registerTransitousCommands(program: Command): void { @@ -318,115 +111,23 @@ export function registerTransitousCommands(program: Command): void { transitous .command("bump") - .description( - "Fetch upstream pins, summarize feed changes, and write a reviewable inactive-slot proposal", - ) + .description("Create a paired Transitous/GBFS proposal through the installed Data Manager") .option("--yes", "Skip the interactive confirmation prompt", false) .option("--branch ", "Branch to track (default: main)", "main") .action(async (options: { yes: boolean; branch: string }) => { - const paths = repoPaths(); - const catalogDir = join(paths.infraDir, "data", TRANSITOUS_CATALOG_DIR); - if (!existsSync(join(catalogDir, ".git"))) { - log.err( - `Transitous catalog not found at ${catalogDir}. Run the data-manager once (e.g. via \`pnpm openmapx data download gtfs\`) so it clones the catalog, then retry.`, - ); - process.exit(1); - } - - log.dim(`Fetching origin/${options.branch} in ${catalogDir}...`); - try { - await execa("git", gitArgs(catalogDir, "fetch", "origin", options.branch), { - stdio: "pipe", - }); - } catch (err) { - log.err(`git fetch failed: ${(err as Error).message}`); - process.exit(1); - } - - const newSha = await runGit(catalogDir, ["rev-parse", `origin/${options.branch}`]); - const newSubmoduleSha = await readSubmoduleSha( - catalogDir, - `origin/${options.branch}`, - "transitland-atlas", - ); - - if (!existsSync(join(paths.infraDir, "secrets/control-verifiers.json"))) - await provisionControlInstallation(paths.infraDir); - const installation = openControlInstallation({ - infraDir: paths.infraDir, - role: "host-cli", - serviceIds: ["motis"], - }); - let active: CatalogPinSet | null; - try { - active = await initializeManagedCatalogPins({ rootDir: paths.root, installation }); - } finally { - closeControlInstallation(installation); - } - const existing = active?.transitous ?? null; - const lockedBy = await resolveLockedBy(); - let gbfsCandidate: Awaited>; - try { - gbfsCandidate = await resolveGbfsCandidate(lockedBy); - } catch (error) { - log.err( - `GBFS registry candidate failed validation; preserving all current pins: ${(error as Error).message}`, - ); - process.exit(1); - } - const existingGbfs = active?.gbfs ?? null; - if (combinedPinsAreCurrent(existing, existingGbfs, newSha, gbfsCandidate.lock.commit)) { - log.ok( - `Already pinned to Transitous ${newSha.slice(0, 12)} and GBFS ${gbfsCandidate.lock.commit.slice(0, 12)} — nothing to do.`, - ); + if ( + !options.yes && + !(await promptConfirm("Create a paired catalog proposal for review? [y/N]")) + ) return; - } - - log.info(`New ref: ${options.branch}@${newSha}`); - log.info(`New transitland-atlas: ${newSubmoduleSha}`); - log.info(`New MobilityData GBFS registry: ${gbfsCandidate.lock.commit}`); - log.info( - ` GBFS systems: ${[...gbfsCandidate.countryCounts.values()].reduce((sum, count) => sum + count, 0)} across ${gbfsCandidate.countryCounts.size} countries; DACH DE=${gbfsCandidate.countryCounts.get("de") ?? 0}, AT=${gbfsCandidate.countryCounts.get("at") ?? 0}, CH=${gbfsCandidate.countryCounts.get("ch") ?? 0}`, - ); - - if (existing) { - const previousSha = parseRefShaPair(existing.ref).sha; - log.info(`Diff vs current pin ${previousSha.slice(0, 12)}:`); - const [oldFeeds, newFeeds] = await Promise.all([ - listFeedsAtRevision(catalogDir, previousSha), - listFeedsAtRevision(catalogDir, newSha), - ]); - printDiffSummary(diffFeedFiles(oldFeeds, newFeeds)); - } else { - log.dim("(no previous pin — printing full feed inventory)"); - const feeds = await listFeedsAtRevision(catalogDir, newSha); - const sourceCount = [...feeds.values()].reduce( - (sum, feed) => sum + (feed.sources?.length ?? 0), - 0, + const proposal = await proposeReviewedCatalog(options.branch); + if (!proposal.unchanged) { + log.ok(`Proposal created: ${proposal.proposalId}`); + log.info(`Transitous: ${proposal.ref}`); + log.dim( + "Review the complete Transitous/GBFS diff in Admin → Transit, then validate the inactive MOTIS slot before activation.", ); - log.info(` regions: ${feeds.size}, sources: ${sourceCount}`); } - - if (!options.yes) { - const ok = await promptConfirm("Write this pin set as an inactive-slot proposal? [y/N]"); - if (!ok) { - log.info("Aborted — no lockfile changes."); - return; - } - } - - const lock: TransitousLock = { - ref: `${options.branch}@${newSha}`, - submodules: { "transitland-atlas": newSubmoduleSha }, - lockedAt: new Date().toISOString(), - lockedBy, - comment: - "Pinned commit of public-transport/transitous consumed by services/data-manager. Bump via `pnpm openmapx transitous bump`.", - }; - writeCombinedCatalogLockProposal(paths.root, lock, gbfsCandidate.lock); - - log.ok(`Proposed Transitous and GBFS catalog lock set under ${join("infra", "docker")}`); - log.dim("Review diffs and validate an inactive MOTIS slot before activating the proposal."); }); transitous diff --git a/packages/cli/src/lib/transitous-api-keys.ts b/packages/cli/src/lib/transitous-api-keys.ts index 344dc11c9..438084e31 100644 --- a/packages/cli/src/lib/transitous-api-keys.ts +++ b/packages/cli/src/lib/transitous-api-keys.ts @@ -12,7 +12,7 @@ import { } from "node:fs"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; -import { readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import { readCatalogPinSet, readCatalogProposal } from "@openmapx/core/catalog-pins"; import { createControlRootOperationKey, executionFingerprint } from "@openmapx/core/ops"; import { fsyncDirectory } from "@openmapx/core/release-files"; import { @@ -39,6 +39,7 @@ export interface GenerateTransitousApiKeysOptions { transitousRepoUrl?: string; outputPath?: string; runner?: CommandRunner; + proposalId?: string; } export interface GenerateTransitousApiKeysResult { outputPath: string; @@ -159,6 +160,26 @@ export async function generateTransitousApiKeys( throw Error( "Active paired catalog pins are required; initialize the instance before generating credentials", ); + const activeGenerationId = pins.generationId; + if (options.proposalId) { + const reader = control.openControlInstallation({ + infraDir: paths.infraDir, + role: "host-cli", + serviceIds: ["motis"], + }); + try { + const proposal = readCatalogProposal({ + rootDir: paths.root, + installation: reader, + proposalId: options.proposalId, + }); + if (!proposal || proposal.baseGenerationId !== activeGenerationId) + throw Error("Proposal is missing or based on different active pins"); + pins = proposal.pins; + } finally { + control.closeControlInstallation(reader); + } + } const sha = pins.transitous.ref.split("@").at(-1), atlas = pins.transitous.submodules["transitland-atlas"]; if (!sha || !atlas) throw Error("Exact Transitous and atlas pins are required"); @@ -225,7 +246,12 @@ export async function generateTransitousApiKeys( serviceIds: ["motis"], owner: authority, }); - if (readCatalogPinSet(paths.root)?.generationId !== pins.generationId) + if ( + readCatalogPinSet(paths.root)?.generationId !== activeGenerationId || + (options.proposalId && + readCatalogProposal({ rootDir: paths.root, installation, proposalId: options.proposalId }) + ?.pins.generationId !== pins.generationId) + ) throw Error("Catalog pins changed; inspect the new pin before retrying"); if (existingSnapshot(outputPath)?.identity !== before?.identity) throw Error("Credential file changed; review before retrying"); diff --git a/packages/core/src/api/contracts/data-manager-admin/index.ts b/packages/core/src/api/contracts/data-manager-admin/index.ts index 792a366cc..6351d2faa 100644 --- a/packages/core/src/api/contracts/data-manager-admin/index.ts +++ b/packages/core/src/api/contracts/data-manager-admin/index.ts @@ -10,3 +10,4 @@ export * from "./providers"; export * from "./transit"; export * from "./transit-credentials"; +export * from "./transit-pins"; diff --git a/packages/core/src/api/contracts/data-manager-admin/operations.ts b/packages/core/src/api/contracts/data-manager-admin/operations.ts index e68aed9e4..89a5e9534 100644 --- a/packages/core/src/api/contracts/data-manager-admin/operations.ts +++ b/packages/core/src/api/contracts/data-manager-admin/operations.ts @@ -53,6 +53,7 @@ import { transitStateSummarySchema, } from "./transit"; import { transitCredentialOperations } from "./transit-credentials"; +import { transitPinOperations } from "./transit-pins"; const service = "service" as const; const admin = "admin" as const; @@ -93,6 +94,7 @@ const ambientUpdated = { /** Application API operations. Service credentials, private acquisition and internal endpoints are not browser contracts. */ export const dataManagerAdminOperations = { ...transitCredentialOperations, + ...transitPinOperations, getApiDataManagerRoadConditionsPolicy: { method: "GET", path: "/api/data-manager/road-conditions/policy", diff --git a/packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts b/packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts index 23c600993..8b58db40d 100644 --- a/packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts +++ b/packages/core/src/api/contracts/data-manager-admin/transit-credentials.ts @@ -38,6 +38,7 @@ export const feedCredentialChangeSchema = z.discriminatedUnion("action", [ ]); export const feedCredentialPreviewInputSchema = z.strictObject({ catalogRevisionId: revision, + proposalId: z.uuid().optional(), expectedCredentialsRevision: revision, expectedCurrentRevisionId: currentRevision, operationKey, @@ -57,6 +58,7 @@ export const feedCredentialPreviewSchema = z.strictObject({ previewId: z.uuid(), intentFingerprint: revision, catalogRevisionId: revision, + proposalId: z.uuid().optional(), credentialsRevision: revision, expectedCurrentRevisionId: currentRevision, expiresAt: z.iso.datetime(), @@ -66,6 +68,7 @@ export const feedCredentialPreviewSchema = z.strictObject({ }); export const transitCredentialsSchema = z.strictObject({ catalogRevisionId: revision, + proposalId: z.uuid().optional(), credentialsRevision: revision, configurationRevisionId: currentRevision, owner: z.enum(["vault", "file", "environment", "unset"]), @@ -90,6 +93,7 @@ export const feedCredentialApplyResultSchema = z.strictObject({ }); const responses = { ...adminCommonResponses, + 404: adminJson(httpErrorSchema), 409: adminJson(httpErrorSchema), 503: adminJson(httpErrorSchema), }; @@ -98,6 +102,7 @@ export const transitCredentialOperations = { method: "GET", path: "/api/admin/transit/credentials", auth: "admin", + request: { query: z.object({ proposalId: z.uuid().optional() }) }, responses: { ...responses, 200: adminJson(transitCredentialsSchema) }, }, previewAdminTransitCredentials: { diff --git a/packages/core/src/api/contracts/data-manager-admin/transit-pins.ts b/packages/core/src/api/contracts/data-manager-admin/transit-pins.ts new file mode 100644 index 000000000..f61982d95 --- /dev/null +++ b/packages/core/src/api/contracts/data-manager-admin/transit-pins.ts @@ -0,0 +1,78 @@ +import z from "zod/v4"; +import { + catalogActivateRequestSchema, + catalogProposalCreateSchema, + catalogReviewQuerySchema, + catalogReviewSnapshotSchema, + catalogValidateRequestSchema, +} from "../../../services/data-manager/contracts/catalog-review"; +import { dataExecutionStatusSchema } from "../../../services/data-manager/contracts/executions"; +import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; + +const responses = { + ...adminCommonResponses, + 404: adminJson(httpErrorSchema), + 409: adminJson(httpErrorSchema), + 422: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), +}; +const headers = z.object({ + "idempotency-key": z.string().regex(/^opk1_t1_[0-9a-f]{14}_[A-Za-z0-9_-]{22}$/), +}); +const params = z.strictObject({ proposalId: z.uuid() }); +const accepted = z.strictObject({ jobId: z.uuid() }); +export const transitPinOperations = { + getAdminTransitPins: { + method: "GET", + path: "/api/admin/transit/pins", + auth: "admin", + request: { query: catalogReviewQuerySchema }, + responses: { ...responses, 200: adminJson(catalogReviewSnapshotSchema) }, + }, + proposeAdminTransitPins: { + method: "POST", + path: "/api/admin/transit/pins/proposals", + auth: "admin", + request: { headers, body: catalogProposalCreateSchema }, + responses: { + ...responses, + 200: adminJson( + z.strictObject({ proposalId: z.uuid(), generationId: z.string().regex(/^[a-f0-9]{64}$/) }), + ), + }, + }, + validateAdminTransitPins: { + method: "POST", + path: "/api/admin/transit/pins/proposals/:proposalId/validate", + auth: "admin", + request: { params, headers, body: catalogValidateRequestSchema }, + responses: { ...responses, 202: adminJson(accepted) }, + }, + activateAdminTransitPins: { + method: "POST", + path: "/api/admin/transit/pins/proposals/:proposalId/activate", + auth: "admin", + request: { params, headers, body: catalogActivateRequestSchema }, + responses: { ...responses, 202: adminJson(accepted) }, + }, + cancelAdminTransitPinExecution: { + method: "DELETE", + path: "/api/admin/transit/pins/executions/:id", + auth: "admin", + request: { + params: z.strictObject({ id: z.uuid() }), + body: z.strictObject({ + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + }), + }, + responses: { ...responses, 200: adminJson(dataExecutionStatusSchema) }, + }, + getAdminTransitPinExecution: { + method: "GET", + path: "/api/admin/transit/pins/executions/:id", + auth: "admin", + request: { params: z.strictObject({ id: z.uuid() }) }, + responses: { ...responses, 200: adminJson(dataExecutionStatusSchema) }, + }, +} as const satisfies Record; diff --git a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts index 7baa407f1..adbe2e88f 100644 --- a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts +++ b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts @@ -1,10 +1,25 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. import type { + ActivateAdminTransitPins202, + ActivateAdminTransitPins400, + ActivateAdminTransitPins401, + ActivateAdminTransitPins403, + ActivateAdminTransitPins404, + ActivateAdminTransitPins409, + ActivateAdminTransitPins413, + ActivateAdminTransitPins415, + ActivateAdminTransitPins422, + ActivateAdminTransitPins429, + ActivateAdminTransitPins500, + ActivateAdminTransitPins503, + ActivateAdminTransitPinsBody, + ActivateAdminTransitPinsHeaders, ApplyAdminTransitCredentials200, ApplyAdminTransitCredentials400, ApplyAdminTransitCredentials401, ApplyAdminTransitCredentials403, + ApplyAdminTransitCredentials404, ApplyAdminTransitCredentials409, ApplyAdminTransitCredentials413, ApplyAdminTransitCredentials415, @@ -12,6 +27,19 @@ import type { ApplyAdminTransitCredentials500, ApplyAdminTransitCredentials503, ApplyAdminTransitCredentialsBody, + CancelAdminTransitPinExecution200, + CancelAdminTransitPinExecution400, + CancelAdminTransitPinExecution401, + CancelAdminTransitPinExecution403, + CancelAdminTransitPinExecution404, + CancelAdminTransitPinExecution409, + CancelAdminTransitPinExecution413, + CancelAdminTransitPinExecution415, + CancelAdminTransitPinExecution422, + CancelAdminTransitPinExecution429, + CancelAdminTransitPinExecution500, + CancelAdminTransitPinExecution503, + CancelAdminTransitPinExecutionBody, DeleteApiDataManagerTransitSourcesSourceId202, DeleteApiDataManagerTransitSourcesSourceId400, DeleteApiDataManagerTransitSourcesSourceId401, @@ -27,12 +55,39 @@ import type { GetAdminTransitCredentials400, GetAdminTransitCredentials401, GetAdminTransitCredentials403, + GetAdminTransitCredentials404, GetAdminTransitCredentials409, GetAdminTransitCredentials413, GetAdminTransitCredentials415, GetAdminTransitCredentials429, GetAdminTransitCredentials500, GetAdminTransitCredentials503, + GetAdminTransitCredentialsParams, + GetAdminTransitPinExecution200, + GetAdminTransitPinExecution400, + GetAdminTransitPinExecution401, + GetAdminTransitPinExecution403, + GetAdminTransitPinExecution404, + GetAdminTransitPinExecution409, + GetAdminTransitPinExecution413, + GetAdminTransitPinExecution415, + GetAdminTransitPinExecution422, + GetAdminTransitPinExecution429, + GetAdminTransitPinExecution500, + GetAdminTransitPinExecution503, + GetAdminTransitPins200, + GetAdminTransitPins400, + GetAdminTransitPins401, + GetAdminTransitPins403, + GetAdminTransitPins404, + GetAdminTransitPins409, + GetAdminTransitPins413, + GetAdminTransitPins415, + GetAdminTransitPins422, + GetAdminTransitPins429, + GetAdminTransitPins500, + GetAdminTransitPins503, + GetAdminTransitPinsParams, GetApiAdminAmbientPlacesStatus200, GetApiAdminAmbientPlacesStatus400, GetApiAdminAmbientPlacesStatus401, @@ -302,6 +357,7 @@ import type { PreviewAdminTransitCredentials400, PreviewAdminTransitCredentials401, PreviewAdminTransitCredentials403, + PreviewAdminTransitCredentials404, PreviewAdminTransitCredentials409, PreviewAdminTransitCredentials413, PreviewAdminTransitCredentials415, @@ -309,11 +365,43 @@ import type { PreviewAdminTransitCredentials500, PreviewAdminTransitCredentials503, PreviewAdminTransitCredentialsBody, + ProposeAdminTransitPins200, + ProposeAdminTransitPins400, + ProposeAdminTransitPins401, + ProposeAdminTransitPins403, + ProposeAdminTransitPins404, + ProposeAdminTransitPins409, + ProposeAdminTransitPins413, + ProposeAdminTransitPins415, + ProposeAdminTransitPins422, + ProposeAdminTransitPins429, + ProposeAdminTransitPins500, + ProposeAdminTransitPins503, + ProposeAdminTransitPinsBody, + ProposeAdminTransitPinsHeaders, + ValidateAdminTransitPins202, + ValidateAdminTransitPins400, + ValidateAdminTransitPins401, + ValidateAdminTransitPins403, + ValidateAdminTransitPins404, + ValidateAdminTransitPins409, + ValidateAdminTransitPins413, + ValidateAdminTransitPins415, + ValidateAdminTransitPins422, + ValidateAdminTransitPins429, + ValidateAdminTransitPins500, + ValidateAdminTransitPins503, + ValidateAdminTransitPinsBody, + ValidateAdminTransitPinsHeaders, } from "./models"; import { + activateAdminTransitPinsRequest, applyAdminTransitCredentialsRequest, + cancelAdminTransitPinExecutionRequest, deleteApiDataManagerTransitSourcesSourceIdRequest, getAdminTransitCredentialsRequest, + getAdminTransitPinExecutionRequest, + getAdminTransitPinsRequest, getApiAdminAmbientPlacesStatusRequest, getApiDataManagerNotablePlacesStatusRequest, getApiDataManagerOvertureStatusRequest, @@ -342,6 +430,8 @@ import { postApiDataManagerTransitSyncPreflightRequest, postApiDataManagerTransitSyncRequest, previewAdminTransitCredentialsRequest, + proposeAdminTransitPinsRequest, + validateAdminTransitPinsRequest, } from "./operation-bindings.generated"; export type postApiAdminAmbientPlacesBuildResponse202 = { data: PostApiAdminAmbientPlacesBuild202; @@ -973,6 +1063,11 @@ export type getAdminTransitCredentialsResponse403 = { status: 403; }; +export type getAdminTransitCredentialsResponse404 = { + data: GetAdminTransitCredentials404; + status: 404; +}; + export type getAdminTransitCredentialsResponse409 = { data: GetAdminTransitCredentials409; status: 409; @@ -1010,6 +1105,7 @@ export type getAdminTransitCredentialsResponseError = ( | getAdminTransitCredentialsResponse400 | getAdminTransitCredentialsResponse401 | getAdminTransitCredentialsResponse403 + | getAdminTransitCredentialsResponse404 | getAdminTransitCredentialsResponse409 | getAdminTransitCredentialsResponse413 | getAdminTransitCredentialsResponse415 @@ -1024,15 +1120,28 @@ export type getAdminTransitCredentialsResponse = | getAdminTransitCredentialsResponseSuccess | getAdminTransitCredentialsResponseError; -export const getGetAdminTransitCredentialsUrl = () => { - return `/api/admin/transit/credentials`; +export const getGetAdminTransitCredentialsUrl = (params?: GetAdminTransitCredentialsParams) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 + ? `/api/admin/transit/credentials?${stringifiedParams}` + : `/api/admin/transit/credentials`; }; export const getAdminTransitCredentials = async ( + params?: GetAdminTransitCredentialsParams, options?: Parameters[1], ): Promise => { return getAdminTransitCredentialsRequest( - getGetAdminTransitCredentialsUrl(), + getGetAdminTransitCredentialsUrl(params), { ...options, method: "GET", @@ -1060,6 +1169,11 @@ export type applyAdminTransitCredentialsResponse403 = { status: 403; }; +export type applyAdminTransitCredentialsResponse404 = { + data: ApplyAdminTransitCredentials404; + status: 404; +}; + export type applyAdminTransitCredentialsResponse409 = { data: ApplyAdminTransitCredentials409; status: 409; @@ -1098,6 +1212,7 @@ export type applyAdminTransitCredentialsResponseError = ( | applyAdminTransitCredentialsResponse400 | applyAdminTransitCredentialsResponse401 | applyAdminTransitCredentialsResponse403 + | applyAdminTransitCredentialsResponse404 | applyAdminTransitCredentialsResponse409 | applyAdminTransitCredentialsResponse413 | applyAdminTransitCredentialsResponse415 @@ -1170,6 +1285,11 @@ export type previewAdminTransitCredentialsResponse403 = { status: 403; }; +export type previewAdminTransitCredentialsResponse404 = { + data: PreviewAdminTransitCredentials404; + status: 404; +}; + export type previewAdminTransitCredentialsResponse409 = { data: PreviewAdminTransitCredentials409; status: 409; @@ -1208,6 +1328,7 @@ export type previewAdminTransitCredentialsResponseError = ( | previewAdminTransitCredentialsResponse400 | previewAdminTransitCredentialsResponse401 | previewAdminTransitCredentialsResponse403 + | previewAdminTransitCredentialsResponse404 | previewAdminTransitCredentialsResponse409 | previewAdminTransitCredentialsResponse413 | previewAdminTransitCredentialsResponse415 @@ -1260,6 +1381,709 @@ export const previewAdminTransitCredentials = async ( ); }; +export type getAdminTransitPinsResponse200 = { + data: GetAdminTransitPins200; + status: 200; +}; + +export type getAdminTransitPinsResponse400 = { + data: GetAdminTransitPins400; + status: 400; +}; + +export type getAdminTransitPinsResponse401 = { + data: GetAdminTransitPins401; + status: 401; +}; + +export type getAdminTransitPinsResponse403 = { + data: GetAdminTransitPins403; + status: 403; +}; + +export type getAdminTransitPinsResponse404 = { + data: GetAdminTransitPins404; + status: 404; +}; + +export type getAdminTransitPinsResponse409 = { + data: GetAdminTransitPins409; + status: 409; +}; + +export type getAdminTransitPinsResponse413 = { + data: GetAdminTransitPins413; + status: 413; +}; + +export type getAdminTransitPinsResponse415 = { + data: GetAdminTransitPins415; + status: 415; +}; + +export type getAdminTransitPinsResponse422 = { + data: GetAdminTransitPins422; + status: 422; +}; + +export type getAdminTransitPinsResponse429 = { + data: GetAdminTransitPins429; + status: 429; +}; + +export type getAdminTransitPinsResponse500 = { + data: GetAdminTransitPins500; + status: 500; +}; + +export type getAdminTransitPinsResponse503 = { + data: GetAdminTransitPins503; + status: 503; +}; + +export type getAdminTransitPinsResponseSuccess = getAdminTransitPinsResponse200 & { + headers: Headers; +}; +export type getAdminTransitPinsResponseError = ( + | getAdminTransitPinsResponse400 + | getAdminTransitPinsResponse401 + | getAdminTransitPinsResponse403 + | getAdminTransitPinsResponse404 + | getAdminTransitPinsResponse409 + | getAdminTransitPinsResponse413 + | getAdminTransitPinsResponse415 + | getAdminTransitPinsResponse422 + | getAdminTransitPinsResponse429 + | getAdminTransitPinsResponse500 + | getAdminTransitPinsResponse503 +) & { + headers: Headers; +}; + +export type getAdminTransitPinsResponse = + | getAdminTransitPinsResponseSuccess + | getAdminTransitPinsResponseError; + +export const getGetAdminTransitPinsUrl = (params?: GetAdminTransitPinsParams) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 + ? `/api/admin/transit/pins?${stringifiedParams}` + : `/api/admin/transit/pins`; +}; + +export const getAdminTransitPins = async ( + params?: GetAdminTransitPinsParams, + options?: Parameters[1], +): Promise => { + return getAdminTransitPinsRequest( + getGetAdminTransitPinsUrl(params), + { + ...options, + method: "GET", + }, + ); +}; + +export type getAdminTransitPinExecutionResponse200 = { + data: GetAdminTransitPinExecution200; + status: 200; +}; + +export type getAdminTransitPinExecutionResponse400 = { + data: GetAdminTransitPinExecution400; + status: 400; +}; + +export type getAdminTransitPinExecutionResponse401 = { + data: GetAdminTransitPinExecution401; + status: 401; +}; + +export type getAdminTransitPinExecutionResponse403 = { + data: GetAdminTransitPinExecution403; + status: 403; +}; + +export type getAdminTransitPinExecutionResponse404 = { + data: GetAdminTransitPinExecution404; + status: 404; +}; + +export type getAdminTransitPinExecutionResponse409 = { + data: GetAdminTransitPinExecution409; + status: 409; +}; + +export type getAdminTransitPinExecutionResponse413 = { + data: GetAdminTransitPinExecution413; + status: 413; +}; + +export type getAdminTransitPinExecutionResponse415 = { + data: GetAdminTransitPinExecution415; + status: 415; +}; + +export type getAdminTransitPinExecutionResponse422 = { + data: GetAdminTransitPinExecution422; + status: 422; +}; + +export type getAdminTransitPinExecutionResponse429 = { + data: GetAdminTransitPinExecution429; + status: 429; +}; + +export type getAdminTransitPinExecutionResponse500 = { + data: GetAdminTransitPinExecution500; + status: 500; +}; + +export type getAdminTransitPinExecutionResponse503 = { + data: GetAdminTransitPinExecution503; + status: 503; +}; + +export type getAdminTransitPinExecutionResponseSuccess = getAdminTransitPinExecutionResponse200 & { + headers: Headers; +}; +export type getAdminTransitPinExecutionResponseError = ( + | getAdminTransitPinExecutionResponse400 + | getAdminTransitPinExecutionResponse401 + | getAdminTransitPinExecutionResponse403 + | getAdminTransitPinExecutionResponse404 + | getAdminTransitPinExecutionResponse409 + | getAdminTransitPinExecutionResponse413 + | getAdminTransitPinExecutionResponse415 + | getAdminTransitPinExecutionResponse422 + | getAdminTransitPinExecutionResponse429 + | getAdminTransitPinExecutionResponse500 + | getAdminTransitPinExecutionResponse503 +) & { + headers: Headers; +}; + +export type getAdminTransitPinExecutionResponse = + | getAdminTransitPinExecutionResponseSuccess + | getAdminTransitPinExecutionResponseError; + +export const getGetAdminTransitPinExecutionUrl = (id: string) => { + return `/api/admin/transit/pins/executions/${encodeURIComponent(String(id))}`; +}; + +export const getAdminTransitPinExecution = async ( + id: string, + options?: Parameters[1], +): Promise => { + return getAdminTransitPinExecutionRequest( + getGetAdminTransitPinExecutionUrl(id), + { + ...options, + method: "GET", + }, + ); +}; + +export type cancelAdminTransitPinExecutionResponse200 = { + data: CancelAdminTransitPinExecution200; + status: 200; +}; + +export type cancelAdminTransitPinExecutionResponse400 = { + data: CancelAdminTransitPinExecution400; + status: 400; +}; + +export type cancelAdminTransitPinExecutionResponse401 = { + data: CancelAdminTransitPinExecution401; + status: 401; +}; + +export type cancelAdminTransitPinExecutionResponse403 = { + data: CancelAdminTransitPinExecution403; + status: 403; +}; + +export type cancelAdminTransitPinExecutionResponse404 = { + data: CancelAdminTransitPinExecution404; + status: 404; +}; + +export type cancelAdminTransitPinExecutionResponse409 = { + data: CancelAdminTransitPinExecution409; + status: 409; +}; + +export type cancelAdminTransitPinExecutionResponse413 = { + data: CancelAdminTransitPinExecution413; + status: 413; +}; + +export type cancelAdminTransitPinExecutionResponse415 = { + data: CancelAdminTransitPinExecution415; + status: 415; +}; + +export type cancelAdminTransitPinExecutionResponse422 = { + data: CancelAdminTransitPinExecution422; + status: 422; +}; + +export type cancelAdminTransitPinExecutionResponse429 = { + data: CancelAdminTransitPinExecution429; + status: 429; +}; + +export type cancelAdminTransitPinExecutionResponse500 = { + data: CancelAdminTransitPinExecution500; + status: 500; +}; + +export type cancelAdminTransitPinExecutionResponse503 = { + data: CancelAdminTransitPinExecution503; + status: 503; +}; + +export type cancelAdminTransitPinExecutionResponseSuccess = + cancelAdminTransitPinExecutionResponse200 & { + headers: Headers; + }; +export type cancelAdminTransitPinExecutionResponseError = ( + | cancelAdminTransitPinExecutionResponse400 + | cancelAdminTransitPinExecutionResponse401 + | cancelAdminTransitPinExecutionResponse403 + | cancelAdminTransitPinExecutionResponse404 + | cancelAdminTransitPinExecutionResponse409 + | cancelAdminTransitPinExecutionResponse413 + | cancelAdminTransitPinExecutionResponse415 + | cancelAdminTransitPinExecutionResponse422 + | cancelAdminTransitPinExecutionResponse429 + | cancelAdminTransitPinExecutionResponse500 + | cancelAdminTransitPinExecutionResponse503 +) & { + headers: Headers; +}; + +export type cancelAdminTransitPinExecutionResponse = + | cancelAdminTransitPinExecutionResponseSuccess + | cancelAdminTransitPinExecutionResponseError; + +export const getCancelAdminTransitPinExecutionUrl = (id: string) => { + return `/api/admin/transit/pins/executions/${encodeURIComponent(String(id))}`; +}; + +export const cancelAdminTransitPinExecution = async ( + id: string, + cancelAdminTransitPinExecutionBody: CancelAdminTransitPinExecutionBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return cancelAdminTransitPinExecutionRequest( + getCancelAdminTransitPinExecutionUrl(id), + { + ...options, + method: "DELETE", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(cancelAdminTransitPinExecutionBody), + }, + ); +}; + +export type proposeAdminTransitPinsResponse200 = { + data: ProposeAdminTransitPins200; + status: 200; +}; + +export type proposeAdminTransitPinsResponse400 = { + data: ProposeAdminTransitPins400; + status: 400; +}; + +export type proposeAdminTransitPinsResponse401 = { + data: ProposeAdminTransitPins401; + status: 401; +}; + +export type proposeAdminTransitPinsResponse403 = { + data: ProposeAdminTransitPins403; + status: 403; +}; + +export type proposeAdminTransitPinsResponse404 = { + data: ProposeAdminTransitPins404; + status: 404; +}; + +export type proposeAdminTransitPinsResponse409 = { + data: ProposeAdminTransitPins409; + status: 409; +}; + +export type proposeAdminTransitPinsResponse413 = { + data: ProposeAdminTransitPins413; + status: 413; +}; + +export type proposeAdminTransitPinsResponse415 = { + data: ProposeAdminTransitPins415; + status: 415; +}; + +export type proposeAdminTransitPinsResponse422 = { + data: ProposeAdminTransitPins422; + status: 422; +}; + +export type proposeAdminTransitPinsResponse429 = { + data: ProposeAdminTransitPins429; + status: 429; +}; + +export type proposeAdminTransitPinsResponse500 = { + data: ProposeAdminTransitPins500; + status: 500; +}; + +export type proposeAdminTransitPinsResponse503 = { + data: ProposeAdminTransitPins503; + status: 503; +}; + +export type proposeAdminTransitPinsResponseSuccess = proposeAdminTransitPinsResponse200 & { + headers: Headers; +}; +export type proposeAdminTransitPinsResponseError = ( + | proposeAdminTransitPinsResponse400 + | proposeAdminTransitPinsResponse401 + | proposeAdminTransitPinsResponse403 + | proposeAdminTransitPinsResponse404 + | proposeAdminTransitPinsResponse409 + | proposeAdminTransitPinsResponse413 + | proposeAdminTransitPinsResponse415 + | proposeAdminTransitPinsResponse422 + | proposeAdminTransitPinsResponse429 + | proposeAdminTransitPinsResponse500 + | proposeAdminTransitPinsResponse503 +) & { + headers: Headers; +}; + +export type proposeAdminTransitPinsResponse = + | proposeAdminTransitPinsResponseSuccess + | proposeAdminTransitPinsResponseError; + +export const getProposeAdminTransitPinsUrl = () => { + return `/api/admin/transit/pins/proposals`; +}; + +export const proposeAdminTransitPins = async ( + proposeAdminTransitPinsBody: ProposeAdminTransitPinsBody, + headers: ProposeAdminTransitPinsHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return proposeAdminTransitPinsRequest( + getProposeAdminTransitPinsUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(proposeAdminTransitPinsBody), + }, + ); +}; + +export type activateAdminTransitPinsResponse202 = { + data: ActivateAdminTransitPins202; + status: 202; +}; + +export type activateAdminTransitPinsResponse400 = { + data: ActivateAdminTransitPins400; + status: 400; +}; + +export type activateAdminTransitPinsResponse401 = { + data: ActivateAdminTransitPins401; + status: 401; +}; + +export type activateAdminTransitPinsResponse403 = { + data: ActivateAdminTransitPins403; + status: 403; +}; + +export type activateAdminTransitPinsResponse404 = { + data: ActivateAdminTransitPins404; + status: 404; +}; + +export type activateAdminTransitPinsResponse409 = { + data: ActivateAdminTransitPins409; + status: 409; +}; + +export type activateAdminTransitPinsResponse413 = { + data: ActivateAdminTransitPins413; + status: 413; +}; + +export type activateAdminTransitPinsResponse415 = { + data: ActivateAdminTransitPins415; + status: 415; +}; + +export type activateAdminTransitPinsResponse422 = { + data: ActivateAdminTransitPins422; + status: 422; +}; + +export type activateAdminTransitPinsResponse429 = { + data: ActivateAdminTransitPins429; + status: 429; +}; + +export type activateAdminTransitPinsResponse500 = { + data: ActivateAdminTransitPins500; + status: 500; +}; + +export type activateAdminTransitPinsResponse503 = { + data: ActivateAdminTransitPins503; + status: 503; +}; + +export type activateAdminTransitPinsResponseSuccess = activateAdminTransitPinsResponse202 & { + headers: Headers; +}; +export type activateAdminTransitPinsResponseError = ( + | activateAdminTransitPinsResponse400 + | activateAdminTransitPinsResponse401 + | activateAdminTransitPinsResponse403 + | activateAdminTransitPinsResponse404 + | activateAdminTransitPinsResponse409 + | activateAdminTransitPinsResponse413 + | activateAdminTransitPinsResponse415 + | activateAdminTransitPinsResponse422 + | activateAdminTransitPinsResponse429 + | activateAdminTransitPinsResponse500 + | activateAdminTransitPinsResponse503 +) & { + headers: Headers; +}; + +export type activateAdminTransitPinsResponse = + | activateAdminTransitPinsResponseSuccess + | activateAdminTransitPinsResponseError; + +export const getActivateAdminTransitPinsUrl = (proposalId: string) => { + return `/api/admin/transit/pins/proposals/${encodeURIComponent(String(proposalId))}/activate`; +}; + +export const activateAdminTransitPins = async ( + proposalId: string, + activateAdminTransitPinsBody: ActivateAdminTransitPinsBody, + headers: ActivateAdminTransitPinsHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return activateAdminTransitPinsRequest( + getActivateAdminTransitPinsUrl(proposalId), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(activateAdminTransitPinsBody), + }, + ); +}; + +export type validateAdminTransitPinsResponse202 = { + data: ValidateAdminTransitPins202; + status: 202; +}; + +export type validateAdminTransitPinsResponse400 = { + data: ValidateAdminTransitPins400; + status: 400; +}; + +export type validateAdminTransitPinsResponse401 = { + data: ValidateAdminTransitPins401; + status: 401; +}; + +export type validateAdminTransitPinsResponse403 = { + data: ValidateAdminTransitPins403; + status: 403; +}; + +export type validateAdminTransitPinsResponse404 = { + data: ValidateAdminTransitPins404; + status: 404; +}; + +export type validateAdminTransitPinsResponse409 = { + data: ValidateAdminTransitPins409; + status: 409; +}; + +export type validateAdminTransitPinsResponse413 = { + data: ValidateAdminTransitPins413; + status: 413; +}; + +export type validateAdminTransitPinsResponse415 = { + data: ValidateAdminTransitPins415; + status: 415; +}; + +export type validateAdminTransitPinsResponse422 = { + data: ValidateAdminTransitPins422; + status: 422; +}; + +export type validateAdminTransitPinsResponse429 = { + data: ValidateAdminTransitPins429; + status: 429; +}; + +export type validateAdminTransitPinsResponse500 = { + data: ValidateAdminTransitPins500; + status: 500; +}; + +export type validateAdminTransitPinsResponse503 = { + data: ValidateAdminTransitPins503; + status: 503; +}; + +export type validateAdminTransitPinsResponseSuccess = validateAdminTransitPinsResponse202 & { + headers: Headers; +}; +export type validateAdminTransitPinsResponseError = ( + | validateAdminTransitPinsResponse400 + | validateAdminTransitPinsResponse401 + | validateAdminTransitPinsResponse403 + | validateAdminTransitPinsResponse404 + | validateAdminTransitPinsResponse409 + | validateAdminTransitPinsResponse413 + | validateAdminTransitPinsResponse415 + | validateAdminTransitPinsResponse422 + | validateAdminTransitPinsResponse429 + | validateAdminTransitPinsResponse500 + | validateAdminTransitPinsResponse503 +) & { + headers: Headers; +}; + +export type validateAdminTransitPinsResponse = + | validateAdminTransitPinsResponseSuccess + | validateAdminTransitPinsResponseError; + +export const getValidateAdminTransitPinsUrl = (proposalId: string) => { + return `/api/admin/transit/pins/proposals/${encodeURIComponent(String(proposalId))}/validate`; +}; + +export const validateAdminTransitPins = async ( + proposalId: string, + validateAdminTransitPinsBody: ValidateAdminTransitPinsBody, + headers: ValidateAdminTransitPinsHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return validateAdminTransitPinsRequest( + getValidateAdminTransitPinsUrl(proposalId), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(validateAdminTransitPinsBody), + }, + ); +}; + export type getApiDataManagerNotablePlacesStatusResponse200 = { data: GetApiDataManagerNotablePlacesStatus200; status: 200; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins202.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins202.ts new file mode 100644 index 000000000..d139fd53d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins202.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; +}; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins400.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins400.ts new file mode 100644 index 000000000..d7de872bb --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins401.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins401.ts new file mode 100644 index 000000000..73ebdcc2e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins403.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins403.ts new file mode 100644 index 000000000..107ec2ac2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins404.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins404.ts new file mode 100644 index 000000000..236b48275 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins409.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins409.ts new file mode 100644 index 000000000..f1651e5e1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins413.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins413.ts new file mode 100644 index 000000000..035701066 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins415.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins415.ts new file mode 100644 index 000000000..3c6be1b02 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins422.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins422.ts new file mode 100644 index 000000000..f874999b1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins429.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins429.ts new file mode 100644 index 000000000..9233fbaea --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins500.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins500.ts new file mode 100644 index 000000000..6fbd2b33a --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPins503.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPins503.ts new file mode 100644 index 000000000..0e5386d97 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPins503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPins503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPinsBody.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPinsBody.ts new file mode 100644 index 000000000..6c15df9ca --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPinsBody.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPinsBody = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + expectedGenerationId: string | null; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + acknowledgeLicenseChanges: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/activateAdminTransitPinsHeaders.ts b/packages/core/src/api/generated/admin/models/activateAdminTransitPinsHeaders.ts new file mode 100644 index 000000000..0e63a2b8e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/activateAdminTransitPinsHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ActivateAdminTransitPinsHeaders = { + /** + * @pattern ^opk1_t1_[0-9a-f]{14}_[A-Za-z0-9_-]{22}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema0.ts b/packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema0.ts new file mode 100644 index 000000000..223065848 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema0.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdminGetAdminTransitPins200Schema0 = + | string + | number + | boolean + | null + | (AdminGetAdminTransitPins200Schema0 | null)[] + | { [key: string]: AdminGetAdminTransitPins200Schema0 | null }; diff --git a/packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema1.ts b/packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema1.ts new file mode 100644 index 000000000..91668c6c1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/adminGetAdminTransitPins200Schema1.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdminGetAdminTransitPins200Schema1 = + | string + | number + | boolean + | null + | (AdminGetAdminTransitPins200Schema1 | null)[] + | { [key: string]: AdminGetAdminTransitPins200Schema1 | null }; diff --git a/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials404.ts b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials404.ts new file mode 100644 index 000000000..2566fdcc6 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/applyAdminTransitCredentials404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminTransitCredentials404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200.ts new file mode 100644 index 000000000..5e02615e7 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { CancelAdminTransitPinExecution200Diagnostics } from "./cancelAdminTransitPinExecution200Diagnostics"; +import type { CancelAdminTransitPinExecution200Kind } from "./cancelAdminTransitPinExecution200Kind"; +import type { CancelAdminTransitPinExecution200Publication } from "./cancelAdminTransitPinExecution200Publication"; +import type { CancelAdminTransitPinExecution200Result } from "./cancelAdminTransitPinExecution200Result"; +import type { CancelAdminTransitPinExecution200Status } from "./cancelAdminTransitPinExecution200Status"; + +export type CancelAdminTransitPinExecution200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: CancelAdminTransitPinExecution200Kind; + status: CancelAdminTransitPinExecution200Status; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: CancelAdminTransitPinExecution200Diagnostics; + publication: CancelAdminTransitPinExecution200Publication; + result?: CancelAdminTransitPinExecution200Result; +}; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Diagnostics.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Diagnostics.ts new file mode 100644 index 000000000..51dc29203 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Diagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { CancelAdminTransitPinExecution200DiagnosticsTailItem } from "./cancelAdminTransitPinExecution200DiagnosticsTailItem"; + +export type CancelAdminTransitPinExecution200Diagnostics = { + /** @maxItems 256 */ + tail: CancelAdminTransitPinExecution200DiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItem.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItem.ts new file mode 100644 index 000000000..4d12bffbe --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { CancelAdminTransitPinExecution200DiagnosticsTailItemStream } from "./cancelAdminTransitPinExecution200DiagnosticsTailItemStream"; + +export type CancelAdminTransitPinExecution200DiagnosticsTailItem = { + stream: CancelAdminTransitPinExecution200DiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItemStream.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItemStream.ts new file mode 100644 index 000000000..cde4b0f68 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200DiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution200DiagnosticsTailItemStream = + (typeof CancelAdminTransitPinExecution200DiagnosticsTailItemStream)[keyof typeof CancelAdminTransitPinExecution200DiagnosticsTailItemStream]; + +export const CancelAdminTransitPinExecution200DiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Kind.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Kind.ts new file mode 100644 index 000000000..8a274264f --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Kind.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution200Kind = + (typeof CancelAdminTransitPinExecution200Kind)[keyof typeof CancelAdminTransitPinExecution200Kind]; + +export const CancelAdminTransitPinExecution200Kind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Publication.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Publication.ts new file mode 100644 index 000000000..b6d41a511 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Publication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution200Publication = + (typeof CancelAdminTransitPinExecution200Publication)[keyof typeof CancelAdminTransitPinExecution200Publication]; + +export const CancelAdminTransitPinExecution200Publication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Result.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Result.ts new file mode 100644 index 000000000..0081a32f7 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Result.ts @@ -0,0 +1,375 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution200Result = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Status.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Status.ts new file mode 100644 index 000000000..363995a75 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution200Status.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution200Status = + (typeof CancelAdminTransitPinExecution200Status)[keyof typeof CancelAdminTransitPinExecution200Status]; + +export const CancelAdminTransitPinExecution200Status = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution400.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution400.ts new file mode 100644 index 000000000..2261b46e6 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution401.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution401.ts new file mode 100644 index 000000000..f7bafaf96 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution403.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution403.ts new file mode 100644 index 000000000..a1239a8b7 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution404.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution404.ts new file mode 100644 index 000000000..9e848df26 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution409.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution409.ts new file mode 100644 index 000000000..676e1cca1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution413.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution413.ts new file mode 100644 index 000000000..75fe79862 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution415.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution415.ts new file mode 100644 index 000000000..5b713d98c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution422.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution422.ts new file mode 100644 index 000000000..60b6bdf5d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution429.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution429.ts new file mode 100644 index 000000000..870a8ba74 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution500.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution500.ts new file mode 100644 index 000000000..2cdd141a8 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution503.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution503.ts new file mode 100644 index 000000000..1337680a0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecution503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecution503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecutionBody.ts b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecutionBody.ts new file mode 100644 index 000000000..43247546f --- /dev/null +++ b/packages/core/src/api/generated/admin/models/cancelAdminTransitPinExecutionBody.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CancelAdminTransitPinExecutionBody = { + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts index 42522dd61..d56356c79 100644 --- a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials200.ts @@ -6,6 +6,8 @@ import type { GetAdminTransitCredentials200Owner } from "./getAdminTransitCreden export type GetAdminTransitCredentials200 = { /** @pattern ^[a-f0-9]{64}$ */ catalogRevisionId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId?: string; /** @pattern ^[a-f0-9]{64}$ */ credentialsRevision: string; configurationRevisionId: string | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentials404.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials404.ts new file mode 100644 index 000000000..372351949 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentials404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentials404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitCredentialsParams.ts b/packages/core/src/api/generated/admin/models/getAdminTransitCredentialsParams.ts new file mode 100644 index 000000000..805c97ece --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitCredentialsParams.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitCredentialsParams = { + /** + * @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + */ + proposalId?: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200.ts new file mode 100644 index 000000000..30298e6c0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPinExecution200Diagnostics } from "./getAdminTransitPinExecution200Diagnostics"; +import type { GetAdminTransitPinExecution200Kind } from "./getAdminTransitPinExecution200Kind"; +import type { GetAdminTransitPinExecution200Publication } from "./getAdminTransitPinExecution200Publication"; +import type { GetAdminTransitPinExecution200Result } from "./getAdminTransitPinExecution200Result"; +import type { GetAdminTransitPinExecution200Status } from "./getAdminTransitPinExecution200Status"; + +export type GetAdminTransitPinExecution200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: GetAdminTransitPinExecution200Kind; + status: GetAdminTransitPinExecution200Status; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: GetAdminTransitPinExecution200Diagnostics; + publication: GetAdminTransitPinExecution200Publication; + result?: GetAdminTransitPinExecution200Result; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Diagnostics.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Diagnostics.ts new file mode 100644 index 000000000..27e28508b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Diagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPinExecution200DiagnosticsTailItem } from "./getAdminTransitPinExecution200DiagnosticsTailItem"; + +export type GetAdminTransitPinExecution200Diagnostics = { + /** @maxItems 256 */ + tail: GetAdminTransitPinExecution200DiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItem.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItem.ts new file mode 100644 index 000000000..b11ebbb37 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPinExecution200DiagnosticsTailItemStream } from "./getAdminTransitPinExecution200DiagnosticsTailItemStream"; + +export type GetAdminTransitPinExecution200DiagnosticsTailItem = { + stream: GetAdminTransitPinExecution200DiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItemStream.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItemStream.ts new file mode 100644 index 000000000..3c28232ec --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200DiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution200DiagnosticsTailItemStream = + (typeof GetAdminTransitPinExecution200DiagnosticsTailItemStream)[keyof typeof GetAdminTransitPinExecution200DiagnosticsTailItemStream]; + +export const GetAdminTransitPinExecution200DiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Kind.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Kind.ts new file mode 100644 index 000000000..b4f5d5fc5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Kind.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution200Kind = + (typeof GetAdminTransitPinExecution200Kind)[keyof typeof GetAdminTransitPinExecution200Kind]; + +export const GetAdminTransitPinExecution200Kind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Publication.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Publication.ts new file mode 100644 index 000000000..e9318d70c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Publication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution200Publication = + (typeof GetAdminTransitPinExecution200Publication)[keyof typeof GetAdminTransitPinExecution200Publication]; + +export const GetAdminTransitPinExecution200Publication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Result.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Result.ts new file mode 100644 index 000000000..647f6272a --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Result.ts @@ -0,0 +1,375 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution200Result = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Status.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Status.ts new file mode 100644 index 000000000..57faa809c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution200Status.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution200Status = + (typeof GetAdminTransitPinExecution200Status)[keyof typeof GetAdminTransitPinExecution200Status]; + +export const GetAdminTransitPinExecution200Status = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution400.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution400.ts new file mode 100644 index 000000000..4c838d730 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution401.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution401.ts new file mode 100644 index 000000000..5d75a4825 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution403.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution403.ts new file mode 100644 index 000000000..1024734bf --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution404.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution404.ts new file mode 100644 index 000000000..52c0453e5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution409.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution409.ts new file mode 100644 index 000000000..34ab59f7a --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution413.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution413.ts new file mode 100644 index 000000000..5ed6feae5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution415.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution415.ts new file mode 100644 index 000000000..16d151d94 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution422.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution422.ts new file mode 100644 index 000000000..91744e0eb --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution429.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution429.ts new file mode 100644 index 000000000..b5535e91f --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution500.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution500.ts new file mode 100644 index 000000000..9742c9ada --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution503.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution503.ts new file mode 100644 index 000000000..991a08a60 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinExecution503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinExecution503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200.ts new file mode 100644 index 000000000..9cf66ccda --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200.ts @@ -0,0 +1,27 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200Active } from "./getAdminTransitPins200Active"; +import type { GetAdminTransitPins200Diff } from "./getAdminTransitPins200Diff"; +import type { GetAdminTransitPins200Execution } from "./getAdminTransitPins200Execution"; +import type { GetAdminTransitPins200Page } from "./getAdminTransitPins200Page"; +import type { GetAdminTransitPins200Proposal } from "./getAdminTransitPins200Proposal"; +import type { GetAdminTransitPins200Validation } from "./getAdminTransitPins200Validation"; +import type { GetAdminTransitPins200ValidationInputs } from "./getAdminTransitPins200ValidationInputs"; + +export type GetAdminTransitPins200 = { + active: GetAdminTransitPins200Active; + /** + * @maxItems 256 + * @items.pattern ^[a-z]{2,3}$ + */ + allowedCountries: string[]; + catalogRevisionId: string | null; + page: GetAdminTransitPins200Page; + proposal: GetAdminTransitPins200Proposal; + diff: GetAdminTransitPins200Diff; + validation: GetAdminTransitPins200Validation; + execution: GetAdminTransitPins200Execution; + validationInputs: GetAdminTransitPins200ValidationInputs; + validationEligible: boolean; + blocker: string | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200Active.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Active.ts new file mode 100644 index 000000000..7f0015609 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Active.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ActiveGbfs } from "./getAdminTransitPins200ActiveGbfs"; +import type { GetAdminTransitPins200ActiveSchemaVersion } from "./getAdminTransitPins200ActiveSchemaVersion"; +import type { GetAdminTransitPins200ActiveTransitous } from "./getAdminTransitPins200ActiveTransitous"; + +export type GetAdminTransitPins200Active = { + schemaVersion: GetAdminTransitPins200ActiveSchemaVersion; + /** @pattern ^[a-f0-9]{64}$ */ + generationId: string; + transitous: GetAdminTransitPins200ActiveTransitous; + gbfs: GetAdminTransitPins200ActiveGbfs; +} | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfs.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfs.ts new file mode 100644 index 000000000..b97d39782 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfs.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ActiveGbfsSchemaVersion } from "./getAdminTransitPins200ActiveGbfsSchemaVersion"; +import type { GetAdminTransitPins200ActiveGbfsSource } from "./getAdminTransitPins200ActiveGbfsSource"; + +export type GetAdminTransitPins200ActiveGbfs = { + schemaVersion: GetAdminTransitPins200ActiveGbfsSchemaVersion; + source: GetAdminTransitPins200ActiveGbfsSource; + /** @pattern ^[a-f0-9]{40}$ */ + commit: string; + /** @maxLength 256 */ + url: string; + /** @pattern ^[a-f0-9]{64}$ */ + sha256: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSchemaVersion.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSchemaVersion.ts new file mode 100644 index 000000000..0d07b31a4 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ActiveGbfsSchemaVersion = + (typeof GetAdminTransitPins200ActiveGbfsSchemaVersion)[keyof typeof GetAdminTransitPins200ActiveGbfsSchemaVersion]; + +export const GetAdminTransitPins200ActiveGbfsSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSource.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSource.ts new file mode 100644 index 000000000..c2a751471 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveGbfsSource.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ActiveGbfsSource = + (typeof GetAdminTransitPins200ActiveGbfsSource)[keyof typeof GetAdminTransitPins200ActiveGbfsSource]; + +export const GetAdminTransitPins200ActiveGbfsSource = { + "mobilitydata-gbfs": "mobilitydata-gbfs", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveSchemaVersion.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveSchemaVersion.ts new file mode 100644 index 000000000..2f4bec49c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ActiveSchemaVersion = + (typeof GetAdminTransitPins200ActiveSchemaVersion)[keyof typeof GetAdminTransitPins200ActiveSchemaVersion]; + +export const GetAdminTransitPins200ActiveSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitous.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitous.ts new file mode 100644 index 000000000..0efb27ff0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitous.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ActiveTransitousSubmodules } from "./getAdminTransitPins200ActiveTransitousSubmodules"; + +export type GetAdminTransitPins200ActiveTransitous = { + /** + * @maxLength 256 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$ + */ + ref: string; + submodules: GetAdminTransitPins200ActiveTransitousSubmodules; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; + /** @maxLength 512 */ + comment?: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitousSubmodules.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitousSubmodules.ts new file mode 100644 index 000000000..260e23937 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ActiveTransitousSubmodules.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ActiveTransitousSubmodules = { [key: string]: string }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200Diff.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Diff.ts new file mode 100644 index 000000000..442c68c3b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Diff.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200DiffRowsItem } from "./getAdminTransitPins200DiffRowsItem"; + +export type GetAdminTransitPins200Diff = { + /** @pattern ^[a-f0-9]{64}$ */ + digest: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + total: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + offset: number; + /** @maxItems 100 */ + rows: GetAdminTransitPins200DiffRowsItem[]; + nextOffset: number | null; +} | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItem.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItem.ts new file mode 100644 index 000000000..537fbea23 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItem.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200DiffRowsItemAfter } from "./getAdminTransitPins200DiffRowsItemAfter"; +import type { GetAdminTransitPins200DiffRowsItemBefore } from "./getAdminTransitPins200DiffRowsItemBefore"; +import type { GetAdminTransitPins200DiffRowsItemChange } from "./getAdminTransitPins200DiffRowsItemChange"; +import type { GetAdminTransitPins200DiffRowsItemFieldsItem } from "./getAdminTransitPins200DiffRowsItemFieldsItem"; +import type { GetAdminTransitPins200DiffRowsItemLicenseStatus } from "./getAdminTransitPins200DiffRowsItemLicenseStatus"; + +export type GetAdminTransitPins200DiffRowsItem = { + id: string; + region: string; + source: string; + change: GetAdminTransitPins200DiffRowsItemChange; + fields: GetAdminTransitPins200DiffRowsItemFieldsItem[]; + licenseStatus: GetAdminTransitPins200DiffRowsItemLicenseStatus; + before: GetAdminTransitPins200DiffRowsItemBefore; + after: GetAdminTransitPins200DiffRowsItemAfter; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemAfter.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemAfter.ts new file mode 100644 index 000000000..6a46c1c30 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemAfter.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminGetAdminTransitPins200Schema1 } from "./adminGetAdminTransitPins200Schema1"; + +export type GetAdminTransitPins200DiffRowsItemAfter = { + [key: string]: AdminGetAdminTransitPins200Schema1 | null; +} | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemBefore.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemBefore.ts new file mode 100644 index 000000000..ba3505253 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemBefore.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminGetAdminTransitPins200Schema0 } from "./adminGetAdminTransitPins200Schema0"; + +export type GetAdminTransitPins200DiffRowsItemBefore = { + [key: string]: AdminGetAdminTransitPins200Schema0 | null; +} | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemChange.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemChange.ts new file mode 100644 index 000000000..ca58bf66d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemChange.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200DiffRowsItemChange = + (typeof GetAdminTransitPins200DiffRowsItemChange)[keyof typeof GetAdminTransitPins200DiffRowsItemChange]; + +export const GetAdminTransitPins200DiffRowsItemChange = { + added: "added", + removed: "removed", + changed: "changed", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemFieldsItem.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemFieldsItem.ts new file mode 100644 index 000000000..b495b329d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemFieldsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200DiffRowsItemFieldsItem = + (typeof GetAdminTransitPins200DiffRowsItemFieldsItem)[keyof typeof GetAdminTransitPins200DiffRowsItemFieldsItem]; + +export const GetAdminTransitPins200DiffRowsItemFieldsItem = { + endpoint: "endpoint", + origin: "origin", + license: "license", + attribution: "attribution", + eligibility: "eligibility", + metadata: "metadata", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemLicenseStatus.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemLicenseStatus.ts new file mode 100644 index 000000000..f709d13c6 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200DiffRowsItemLicenseStatus.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200DiffRowsItemLicenseStatus = + (typeof GetAdminTransitPins200DiffRowsItemLicenseStatus)[keyof typeof GetAdminTransitPins200DiffRowsItemLicenseStatus]; + +export const GetAdminTransitPins200DiffRowsItemLicenseStatus = { + declared: "declared", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200Execution.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Execution.ts new file mode 100644 index 000000000..6a66e6f62 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Execution.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ExecutionDiagnostics } from "./getAdminTransitPins200ExecutionDiagnostics"; +import type { GetAdminTransitPins200ExecutionKind } from "./getAdminTransitPins200ExecutionKind"; +import type { GetAdminTransitPins200ExecutionPublication } from "./getAdminTransitPins200ExecutionPublication"; +import type { GetAdminTransitPins200ExecutionResult } from "./getAdminTransitPins200ExecutionResult"; +import type { GetAdminTransitPins200ExecutionStatus } from "./getAdminTransitPins200ExecutionStatus"; + +export type GetAdminTransitPins200Execution = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: GetAdminTransitPins200ExecutionKind; + status: GetAdminTransitPins200ExecutionStatus; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: GetAdminTransitPins200ExecutionDiagnostics; + publication: GetAdminTransitPins200ExecutionPublication; + result?: GetAdminTransitPins200ExecutionResult; +} | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnostics.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnostics.ts new file mode 100644 index 000000000..75a1862e0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ExecutionDiagnosticsTailItem } from "./getAdminTransitPins200ExecutionDiagnosticsTailItem"; + +export type GetAdminTransitPins200ExecutionDiagnostics = { + /** @maxItems 256 */ + tail: GetAdminTransitPins200ExecutionDiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItem.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItem.ts new file mode 100644 index 000000000..ff6981cd6 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ExecutionDiagnosticsTailItemStream } from "./getAdminTransitPins200ExecutionDiagnosticsTailItemStream"; + +export type GetAdminTransitPins200ExecutionDiagnosticsTailItem = { + stream: GetAdminTransitPins200ExecutionDiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItemStream.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItemStream.ts new file mode 100644 index 000000000..c11e72a60 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionDiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ExecutionDiagnosticsTailItemStream = + (typeof GetAdminTransitPins200ExecutionDiagnosticsTailItemStream)[keyof typeof GetAdminTransitPins200ExecutionDiagnosticsTailItemStream]; + +export const GetAdminTransitPins200ExecutionDiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionKind.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionKind.ts new file mode 100644 index 000000000..e265f61bd --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionKind.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ExecutionKind = + (typeof GetAdminTransitPins200ExecutionKind)[keyof typeof GetAdminTransitPins200ExecutionKind]; + +export const GetAdminTransitPins200ExecutionKind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionPublication.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionPublication.ts new file mode 100644 index 000000000..ca36c6206 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionPublication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ExecutionPublication = + (typeof GetAdminTransitPins200ExecutionPublication)[keyof typeof GetAdminTransitPins200ExecutionPublication]; + +export const GetAdminTransitPins200ExecutionPublication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionResult.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionResult.ts new file mode 100644 index 000000000..7878ebc7d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionResult.ts @@ -0,0 +1,375 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ExecutionResult = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionStatus.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionStatus.ts new file mode 100644 index 000000000..ebcd5e004 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ExecutionStatus.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ExecutionStatus = + (typeof GetAdminTransitPins200ExecutionStatus)[keyof typeof GetAdminTransitPins200ExecutionStatus]; + +export const GetAdminTransitPins200ExecutionStatus = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200Page.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Page.ts new file mode 100644 index 000000000..3d20211f3 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Page.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200PageProposalsItem } from "./getAdminTransitPins200PageProposalsItem"; + +export type GetAdminTransitPins200Page = { + /** @maxItems 100 */ + proposals: GetAdminTransitPins200PageProposalsItem[]; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + total: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + offset: number; + nextOffset: number | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItem.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItem.ts new file mode 100644 index 000000000..753cfabf1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItem.ts @@ -0,0 +1,24 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200PageProposalsItemState } from "./getAdminTransitPins200PageProposalsItemState"; + +export type GetAdminTransitPins200PageProposalsItem = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + baseGenerationId: string | null; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + state: GetAdminTransitPins200PageProposalsItemState; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + createdAt: string; + /** @pattern ^[a-f0-9]{64}$ */ + catalogRevisionId: string; + /** @maxLength 256 */ + transitousRef: string; + /** @pattern ^[a-f0-9]{40}$ */ + atlasCommit: string; + /** @pattern ^[a-f0-9]{40}$ */ + gbfsCommit: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItemState.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItemState.ts new file mode 100644 index 000000000..ca36dd132 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200PageProposalsItemState.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200PageProposalsItemState = + (typeof GetAdminTransitPins200PageProposalsItemState)[keyof typeof GetAdminTransitPins200PageProposalsItemState]; + +export const GetAdminTransitPins200PageProposalsItemState = { + proposed: "proposed", + validating: "validating", + validated: "validated", + rejected: "rejected", + activating: "activating", + activated: "activated", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200Proposal.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Proposal.ts new file mode 100644 index 000000000..afd97ffc2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Proposal.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ProposalPins } from "./getAdminTransitPins200ProposalPins"; +import type { GetAdminTransitPins200ProposalState } from "./getAdminTransitPins200ProposalState"; + +export type GetAdminTransitPins200Proposal = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + baseGenerationId: string | null; + pins: GetAdminTransitPins200ProposalPins; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + state: GetAdminTransitPins200ProposalState; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + createdAt: string; +} | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPins.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPins.ts new file mode 100644 index 000000000..bf963ec6d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPins.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ProposalPinsGbfs } from "./getAdminTransitPins200ProposalPinsGbfs"; +import type { GetAdminTransitPins200ProposalPinsSchemaVersion } from "./getAdminTransitPins200ProposalPinsSchemaVersion"; +import type { GetAdminTransitPins200ProposalPinsTransitous } from "./getAdminTransitPins200ProposalPinsTransitous"; + +export type GetAdminTransitPins200ProposalPins = { + schemaVersion: GetAdminTransitPins200ProposalPinsSchemaVersion; + /** @pattern ^[a-f0-9]{64}$ */ + generationId: string; + transitous: GetAdminTransitPins200ProposalPinsTransitous; + gbfs: GetAdminTransitPins200ProposalPinsGbfs; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfs.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfs.ts new file mode 100644 index 000000000..0503bd76a --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfs.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ProposalPinsGbfsSchemaVersion } from "./getAdminTransitPins200ProposalPinsGbfsSchemaVersion"; +import type { GetAdminTransitPins200ProposalPinsGbfsSource } from "./getAdminTransitPins200ProposalPinsGbfsSource"; + +export type GetAdminTransitPins200ProposalPinsGbfs = { + schemaVersion: GetAdminTransitPins200ProposalPinsGbfsSchemaVersion; + source: GetAdminTransitPins200ProposalPinsGbfsSource; + /** @pattern ^[a-f0-9]{40}$ */ + commit: string; + /** @maxLength 256 */ + url: string; + /** @pattern ^[a-f0-9]{64}$ */ + sha256: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSchemaVersion.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSchemaVersion.ts new file mode 100644 index 000000000..f0ef29ffb --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ProposalPinsGbfsSchemaVersion = + (typeof GetAdminTransitPins200ProposalPinsGbfsSchemaVersion)[keyof typeof GetAdminTransitPins200ProposalPinsGbfsSchemaVersion]; + +export const GetAdminTransitPins200ProposalPinsGbfsSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSource.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSource.ts new file mode 100644 index 000000000..e79e0e7c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsGbfsSource.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ProposalPinsGbfsSource = + (typeof GetAdminTransitPins200ProposalPinsGbfsSource)[keyof typeof GetAdminTransitPins200ProposalPinsGbfsSource]; + +export const GetAdminTransitPins200ProposalPinsGbfsSource = { + "mobilitydata-gbfs": "mobilitydata-gbfs", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsSchemaVersion.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsSchemaVersion.ts new file mode 100644 index 000000000..0c81afcd2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ProposalPinsSchemaVersion = + (typeof GetAdminTransitPins200ProposalPinsSchemaVersion)[keyof typeof GetAdminTransitPins200ProposalPinsSchemaVersion]; + +export const GetAdminTransitPins200ProposalPinsSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitous.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitous.ts new file mode 100644 index 000000000..a690add4d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitous.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ProposalPinsTransitousSubmodules } from "./getAdminTransitPins200ProposalPinsTransitousSubmodules"; + +export type GetAdminTransitPins200ProposalPinsTransitous = { + /** + * @maxLength 256 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$ + */ + ref: string; + submodules: GetAdminTransitPins200ProposalPinsTransitousSubmodules; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; + /** @maxLength 512 */ + comment?: string; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitousSubmodules.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitousSubmodules.ts new file mode 100644 index 000000000..30d4c0c93 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalPinsTransitousSubmodules.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ProposalPinsTransitousSubmodules = { [key: string]: string }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalState.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalState.ts new file mode 100644 index 000000000..c2f566e71 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ProposalState.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ProposalState = + (typeof GetAdminTransitPins200ProposalState)[keyof typeof GetAdminTransitPins200ProposalState]; + +export const GetAdminTransitPins200ProposalState = { + proposed: "proposed", + validating: "validating", + validated: "validated", + rejected: "rejected", + activating: "activating", + activated: "activated", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200Validation.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Validation.ts new file mode 100644 index 000000000..a0a58d27c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200Validation.ts @@ -0,0 +1,78 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200Validation = + | { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^[a-f0-9]{64}$ */ + pinDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + overlayDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[a-z]{2,3}$ + */ + countries: string[]; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + completedAt: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + slot: "A" | "B"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + epoch: string; + /** @pattern ^[a-f0-9]{64}$ */ + manifestHash: string; + /** @pattern ^sha256:[a-f0-9]{64}$ */ + imageDigest: string; + status: "passed"; + } + | { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^[a-f0-9]{64}$ */ + pinDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + overlayDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[a-z]{2,3}$ + */ + countries: string[]; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + completedAt: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + status: "failed"; + /** + * @minLength 1 + * @maxLength 1024 + */ + reason: string; + candidate?: { + slot: "A" | "B"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + epoch: string; + /** @pattern ^[a-f0-9]{64}$ */ + manifestHash: string; + /** @pattern ^sha256:[a-f0-9]{64}$ */ + imageDigest: string; + }; + } + | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputs.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputs.ts new file mode 100644 index 000000000..11676193b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputs.ts @@ -0,0 +1,29 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminTransitPins200ValidationInputsStagingState } from "./getAdminTransitPins200ValidationInputsStagingState"; + +export type GetAdminTransitPins200ValidationInputs = { + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + configurationRevisionId: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + overlayDigest: string; + /** @pattern ^sha256:[a-f0-9]{64}$ */ + imageDigest: string; + stagingState: GetAdminTransitPins200ValidationInputsStagingState; + /** + * @maxItems 1000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + requiredKeys: string[]; + /** + * @maxItems 10000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + configuredKeys: string[]; +} | null; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputsStagingState.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputsStagingState.ts new file mode 100644 index 000000000..d21e2ef99 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins200ValidationInputsStagingState.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins200ValidationInputsStagingState = + (typeof GetAdminTransitPins200ValidationInputsStagingState)[keyof typeof GetAdminTransitPins200ValidationInputsStagingState]; + +export const GetAdminTransitPins200ValidationInputsStagingState = { + running: "running", + stopped: "stopped", + created: "created", + restarting: "restarting", + paused: "paused", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins400.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins400.ts new file mode 100644 index 000000000..75c91759c --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins401.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins401.ts new file mode 100644 index 000000000..881eb9983 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins403.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins403.ts new file mode 100644 index 000000000..ef8cbef23 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins404.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins404.ts new file mode 100644 index 000000000..8e169a238 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins409.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins409.ts new file mode 100644 index 000000000..f7f8f264f --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins413.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins413.ts new file mode 100644 index 000000000..9f951957a --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins415.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins415.ts new file mode 100644 index 000000000..f1cfe0ca0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins422.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins422.ts new file mode 100644 index 000000000..52025e622 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins429.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins429.ts new file mode 100644 index 000000000..1512913c6 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins500.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins500.ts new file mode 100644 index 000000000..1e5444a8b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPins503.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPins503.ts new file mode 100644 index 000000000..216f8af4b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPins503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPins503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getAdminTransitPinsParams.ts b/packages/core/src/api/generated/admin/models/getAdminTransitPinsParams.ts new file mode 100644 index 000000000..7070742c2 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getAdminTransitPinsParams.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminTransitPinsParams = { + /** + * @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + */ + proposalId?: string; + /** + * @minimum 0 + * @maximum 4096 + */ + offset?: number; + /** + * @minimum 0 + * @maximum 200000 + */ + diffOffset?: number; +}; diff --git a/packages/core/src/api/generated/admin/models/index.ts b/packages/core/src/api/generated/admin/models/index.ts index af8f270f4..b8f2445fb 100644 --- a/packages/core/src/api/generated/admin/models/index.ts +++ b/packages/core/src/api/generated/admin/models/index.ts @@ -1,6 +1,22 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +export * from "./activateAdminTransitPins202"; +export * from "./activateAdminTransitPins400"; +export * from "./activateAdminTransitPins401"; +export * from "./activateAdminTransitPins403"; +export * from "./activateAdminTransitPins404"; +export * from "./activateAdminTransitPins409"; +export * from "./activateAdminTransitPins413"; +export * from "./activateAdminTransitPins415"; +export * from "./activateAdminTransitPins422"; +export * from "./activateAdminTransitPins429"; +export * from "./activateAdminTransitPins500"; +export * from "./activateAdminTransitPins503"; +export * from "./activateAdminTransitPinsBody"; +export * from "./activateAdminTransitPinsHeaders"; +export * from "./adminGetAdminTransitPins200Schema0"; +export * from "./adminGetAdminTransitPins200Schema1"; export * from "./adminGetApiDataManagerTransitJobs200Schema0"; export * from "./adminGetApiDataManagerTransitJobsId200Schema0"; export * from "./adminGetApiDataManagerTransitJobsId200Schema1"; @@ -11,6 +27,7 @@ export * from "./applyAdminTransitCredentials200Configuration"; export * from "./applyAdminTransitCredentials400"; export * from "./applyAdminTransitCredentials401"; export * from "./applyAdminTransitCredentials403"; +export * from "./applyAdminTransitCredentials404"; export * from "./applyAdminTransitCredentials409"; export * from "./applyAdminTransitCredentials413"; export * from "./applyAdminTransitCredentials415"; @@ -18,6 +35,26 @@ export * from "./applyAdminTransitCredentials429"; export * from "./applyAdminTransitCredentials500"; export * from "./applyAdminTransitCredentials503"; export * from "./applyAdminTransitCredentialsBody"; +export * from "./cancelAdminTransitPinExecution200"; +export * from "./cancelAdminTransitPinExecution200Diagnostics"; +export * from "./cancelAdminTransitPinExecution200DiagnosticsTailItem"; +export * from "./cancelAdminTransitPinExecution200DiagnosticsTailItemStream"; +export * from "./cancelAdminTransitPinExecution200Kind"; +export * from "./cancelAdminTransitPinExecution200Publication"; +export * from "./cancelAdminTransitPinExecution200Result"; +export * from "./cancelAdminTransitPinExecution200Status"; +export * from "./cancelAdminTransitPinExecution400"; +export * from "./cancelAdminTransitPinExecution401"; +export * from "./cancelAdminTransitPinExecution403"; +export * from "./cancelAdminTransitPinExecution404"; +export * from "./cancelAdminTransitPinExecution409"; +export * from "./cancelAdminTransitPinExecution413"; +export * from "./cancelAdminTransitPinExecution415"; +export * from "./cancelAdminTransitPinExecution422"; +export * from "./cancelAdminTransitPinExecution429"; +export * from "./cancelAdminTransitPinExecution500"; +export * from "./cancelAdminTransitPinExecution503"; +export * from "./cancelAdminTransitPinExecutionBody"; export * from "./deleteApiDataManagerTransitSourcesSourceId202"; export * from "./deleteApiDataManagerTransitSourcesSourceId202Status"; export * from "./deleteApiDataManagerTransitSourcesSourceId400"; @@ -36,12 +73,83 @@ export * from "./getAdminTransitCredentials200Owner"; export * from "./getAdminTransitCredentials400"; export * from "./getAdminTransitCredentials401"; export * from "./getAdminTransitCredentials403"; +export * from "./getAdminTransitCredentials404"; export * from "./getAdminTransitCredentials409"; export * from "./getAdminTransitCredentials413"; export * from "./getAdminTransitCredentials415"; export * from "./getAdminTransitCredentials429"; export * from "./getAdminTransitCredentials500"; export * from "./getAdminTransitCredentials503"; +export * from "./getAdminTransitCredentialsParams"; +export * from "./getAdminTransitPinExecution200"; +export * from "./getAdminTransitPinExecution200Diagnostics"; +export * from "./getAdminTransitPinExecution200DiagnosticsTailItem"; +export * from "./getAdminTransitPinExecution200DiagnosticsTailItemStream"; +export * from "./getAdminTransitPinExecution200Kind"; +export * from "./getAdminTransitPinExecution200Publication"; +export * from "./getAdminTransitPinExecution200Result"; +export * from "./getAdminTransitPinExecution200Status"; +export * from "./getAdminTransitPinExecution400"; +export * from "./getAdminTransitPinExecution401"; +export * from "./getAdminTransitPinExecution403"; +export * from "./getAdminTransitPinExecution404"; +export * from "./getAdminTransitPinExecution409"; +export * from "./getAdminTransitPinExecution413"; +export * from "./getAdminTransitPinExecution415"; +export * from "./getAdminTransitPinExecution422"; +export * from "./getAdminTransitPinExecution429"; +export * from "./getAdminTransitPinExecution500"; +export * from "./getAdminTransitPinExecution503"; +export * from "./getAdminTransitPins200"; +export * from "./getAdminTransitPins200Active"; +export * from "./getAdminTransitPins200ActiveGbfs"; +export * from "./getAdminTransitPins200ActiveGbfsSchemaVersion"; +export * from "./getAdminTransitPins200ActiveGbfsSource"; +export * from "./getAdminTransitPins200ActiveSchemaVersion"; +export * from "./getAdminTransitPins200ActiveTransitous"; +export * from "./getAdminTransitPins200ActiveTransitousSubmodules"; +export * from "./getAdminTransitPins200Diff"; +export * from "./getAdminTransitPins200DiffRowsItem"; +export * from "./getAdminTransitPins200DiffRowsItemAfter"; +export * from "./getAdminTransitPins200DiffRowsItemBefore"; +export * from "./getAdminTransitPins200DiffRowsItemChange"; +export * from "./getAdminTransitPins200DiffRowsItemFieldsItem"; +export * from "./getAdminTransitPins200DiffRowsItemLicenseStatus"; +export * from "./getAdminTransitPins200Execution"; +export * from "./getAdminTransitPins200ExecutionDiagnostics"; +export * from "./getAdminTransitPins200ExecutionDiagnosticsTailItem"; +export * from "./getAdminTransitPins200ExecutionDiagnosticsTailItemStream"; +export * from "./getAdminTransitPins200ExecutionKind"; +export * from "./getAdminTransitPins200ExecutionPublication"; +export * from "./getAdminTransitPins200ExecutionResult"; +export * from "./getAdminTransitPins200ExecutionStatus"; +export * from "./getAdminTransitPins200Page"; +export * from "./getAdminTransitPins200PageProposalsItem"; +export * from "./getAdminTransitPins200PageProposalsItemState"; +export * from "./getAdminTransitPins200Proposal"; +export * from "./getAdminTransitPins200ProposalPins"; +export * from "./getAdminTransitPins200ProposalPinsGbfs"; +export * from "./getAdminTransitPins200ProposalPinsGbfsSchemaVersion"; +export * from "./getAdminTransitPins200ProposalPinsGbfsSource"; +export * from "./getAdminTransitPins200ProposalPinsSchemaVersion"; +export * from "./getAdminTransitPins200ProposalPinsTransitous"; +export * from "./getAdminTransitPins200ProposalPinsTransitousSubmodules"; +export * from "./getAdminTransitPins200ProposalState"; +export * from "./getAdminTransitPins200Validation"; +export * from "./getAdminTransitPins200ValidationInputs"; +export * from "./getAdminTransitPins200ValidationInputsStagingState"; +export * from "./getAdminTransitPins400"; +export * from "./getAdminTransitPins401"; +export * from "./getAdminTransitPins403"; +export * from "./getAdminTransitPins404"; +export * from "./getAdminTransitPins409"; +export * from "./getAdminTransitPins413"; +export * from "./getAdminTransitPins415"; +export * from "./getAdminTransitPins422"; +export * from "./getAdminTransitPins429"; +export * from "./getAdminTransitPins500"; +export * from "./getAdminTransitPins503"; +export * from "./getAdminTransitPinsParams"; export * from "./getApiAdminAmbientPlacesStatus200"; export * from "./getApiAdminAmbientPlacesStatus200Active"; export * from "./getApiAdminAmbientPlacesStatus200ActiveAttributionPrecision"; @@ -398,6 +506,7 @@ export * from "./previewAdminTransitCredentials200ChangesItemAction"; export * from "./previewAdminTransitCredentials400"; export * from "./previewAdminTransitCredentials401"; export * from "./previewAdminTransitCredentials403"; +export * from "./previewAdminTransitCredentials404"; export * from "./previewAdminTransitCredentials409"; export * from "./previewAdminTransitCredentials413"; export * from "./previewAdminTransitCredentials415"; @@ -406,3 +515,31 @@ export * from "./previewAdminTransitCredentials500"; export * from "./previewAdminTransitCredentials503"; export * from "./previewAdminTransitCredentialsBody"; export * from "./previewAdminTransitCredentialsBodyChangesItem"; +export * from "./proposeAdminTransitPins200"; +export * from "./proposeAdminTransitPins400"; +export * from "./proposeAdminTransitPins401"; +export * from "./proposeAdminTransitPins403"; +export * from "./proposeAdminTransitPins404"; +export * from "./proposeAdminTransitPins409"; +export * from "./proposeAdminTransitPins413"; +export * from "./proposeAdminTransitPins415"; +export * from "./proposeAdminTransitPins422"; +export * from "./proposeAdminTransitPins429"; +export * from "./proposeAdminTransitPins500"; +export * from "./proposeAdminTransitPins503"; +export * from "./proposeAdminTransitPinsBody"; +export * from "./proposeAdminTransitPinsHeaders"; +export * from "./validateAdminTransitPins202"; +export * from "./validateAdminTransitPins400"; +export * from "./validateAdminTransitPins401"; +export * from "./validateAdminTransitPins403"; +export * from "./validateAdminTransitPins404"; +export * from "./validateAdminTransitPins409"; +export * from "./validateAdminTransitPins413"; +export * from "./validateAdminTransitPins415"; +export * from "./validateAdminTransitPins422"; +export * from "./validateAdminTransitPins429"; +export * from "./validateAdminTransitPins500"; +export * from "./validateAdminTransitPins503"; +export * from "./validateAdminTransitPinsBody"; +export * from "./validateAdminTransitPinsHeaders"; diff --git a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitBumpTransitousRef200.ts b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitBumpTransitousRef200.ts index d1fd9c764..f5178f54c 100644 --- a/packages/core/src/api/generated/admin/models/postApiDataManagerTransitBumpTransitousRef200.ts +++ b/packages/core/src/api/generated/admin/models/postApiDataManagerTransitBumpTransitousRef200.ts @@ -13,6 +13,10 @@ export type PostApiDataManagerTransitBumpTransitousRef200 = ok: true; unchanged: false; proposed: true; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^[a-f0-9]{64}$ */ + catalogRevisionId: string; ref: string; /** @nullable */ previousRef: string | null; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts index 9eaae3336..8ae8d40e9 100644 --- a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials200.ts @@ -9,6 +9,8 @@ export type PreviewAdminTransitCredentials200 = { intentFingerprint: string; /** @pattern ^[a-f0-9]{64}$ */ catalogRevisionId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId?: string; /** @pattern ^[a-f0-9]{64}$ */ credentialsRevision: string; expectedCurrentRevisionId: string | null; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials404.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials404.ts new file mode 100644 index 000000000..5aaf6dbce --- /dev/null +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentials404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminTransitCredentials404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts index 8d90ff05e..4d318064b 100644 --- a/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts +++ b/packages/core/src/api/generated/admin/models/previewAdminTransitCredentialsBody.ts @@ -5,6 +5,8 @@ import type { PreviewAdminTransitCredentialsBodyChangesItem } from "./previewAdm export type PreviewAdminTransitCredentialsBody = { /** @pattern ^[a-f0-9]{64}$ */ catalogRevisionId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId?: string; /** @pattern ^[a-f0-9]{64}$ */ expectedCredentialsRevision: string; expectedCurrentRevisionId: string | null; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins200.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins200.ts new file mode 100644 index 000000000..6599fc5d7 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins200.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^[a-f0-9]{64}$ */ + generationId: string; +}; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins400.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins400.ts new file mode 100644 index 000000000..23adfd34d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins401.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins401.ts new file mode 100644 index 000000000..b36d9dd91 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins403.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins403.ts new file mode 100644 index 000000000..8cd87f1de --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins404.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins404.ts new file mode 100644 index 000000000..dd383e3fd --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins409.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins409.ts new file mode 100644 index 000000000..51c6aeec1 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins413.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins413.ts new file mode 100644 index 000000000..4e8203db5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins415.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins415.ts new file mode 100644 index 000000000..9ab6c9cba --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins422.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins422.ts new file mode 100644 index 000000000..81329ee0e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins429.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins429.ts new file mode 100644 index 000000000..c749d9ae5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins500.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins500.ts new file mode 100644 index 000000000..4ad139cf8 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPins503.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins503.ts new file mode 100644 index 000000000..b7ec8b08b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPins503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPins503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPinsBody.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPinsBody.ts new file mode 100644 index 000000000..739c89bb5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPinsBody.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPinsBody = { + /** + * @minLength 1 + * @maxLength 215 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._/-]*$ + */ + branch?: string; +}; diff --git a/packages/core/src/api/generated/admin/models/proposeAdminTransitPinsHeaders.ts b/packages/core/src/api/generated/admin/models/proposeAdminTransitPinsHeaders.ts new file mode 100644 index 000000000..ae0b7555f --- /dev/null +++ b/packages/core/src/api/generated/admin/models/proposeAdminTransitPinsHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ProposeAdminTransitPinsHeaders = { + /** + * @pattern ^opk1_t1_[0-9a-f]{14}_[A-Za-z0-9_-]{22}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins202.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins202.ts new file mode 100644 index 000000000..8fede62a0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins202.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; +}; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins400.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins400.ts new file mode 100644 index 000000000..c631d81c5 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins401.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins401.ts new file mode 100644 index 000000000..4d36c21b3 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins403.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins403.ts new file mode 100644 index 000000000..393b5a391 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins404.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins404.ts new file mode 100644 index 000000000..22be554f4 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins409.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins409.ts new file mode 100644 index 000000000..b310ca451 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins413.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins413.ts new file mode 100644 index 000000000..24b5fad59 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins415.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins415.ts new file mode 100644 index 000000000..860956225 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins422.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins422.ts new file mode 100644 index 000000000..41dd91048 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins429.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins429.ts new file mode 100644 index 000000000..c598767fa --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins500.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins500.ts new file mode 100644 index 000000000..96bea38f0 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPins503.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPins503.ts new file mode 100644 index 000000000..930e1c295 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPins503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPins503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPinsBody.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPinsBody.ts new file mode 100644 index 000000000..20fd3ded4 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPinsBody.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPinsBody = { + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; +}; diff --git a/packages/core/src/api/generated/admin/models/validateAdminTransitPinsHeaders.ts b/packages/core/src/api/generated/admin/models/validateAdminTransitPinsHeaders.ts new file mode 100644 index 000000000..bbcea2612 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/validateAdminTransitPinsHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ValidateAdminTransitPinsHeaders = { + /** + * @pattern ^opk1_t1_[0-9a-f]{14}_[A-Za-z0-9_-]{22}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index f2493872a..eaa31f9ac 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -7,6 +7,13 @@ import { bindAdminStreamOperation, } from "../../admin-sdk-transport"; +const activateAdminTransitPinsBound = bindAdminOperation("activateAdminTransitPins"); +export async function activateAdminTransitPinsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return activateAdminTransitPinsBound(url, options); +} const addAdminExtensionSourceBound = bindAdminOperation("addAdminExtensionSource"); export async function addAdminExtensionSourceRequest( url: string, @@ -35,6 +42,13 @@ export async function cancelAdminActivityJobRequest( ): Promise { return cancelAdminActivityJobBound(url, options); } +const cancelAdminTransitPinExecutionBound = bindAdminOperation("cancelAdminTransitPinExecution"); +export async function cancelAdminTransitPinExecutionRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return cancelAdminTransitPinExecutionBound(url, options); +} const changeAdminAccountRoleBound = bindAdminOperation("changeAdminAccountRole"); export async function changeAdminAccountRoleRequest( url: string, @@ -274,6 +288,20 @@ export async function getAdminTransitCredentialsRequest( ): Promise { return getAdminTransitCredentialsBound(url, options); } +const getAdminTransitPinExecutionBound = bindAdminOperation("getAdminTransitPinExecution"); +export async function getAdminTransitPinExecutionRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminTransitPinExecutionBound(url, options); +} +const getAdminTransitPinsBound = bindAdminOperation("getAdminTransitPins"); +export async function getAdminTransitPinsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminTransitPinsBound(url, options); +} const getApiAdminAmbientPlacesStatusBound = bindAdminOperation("getApiAdminAmbientPlacesStatus"); export async function getApiAdminAmbientPlacesStatusRequest( url: string, @@ -615,6 +643,13 @@ export async function previewAdminTransitCredentialsRequest( ): Promise { return previewAdminTransitCredentialsBound(url, options); } +const proposeAdminTransitPinsBound = bindAdminOperation("proposeAdminTransitPins"); +export async function proposeAdminTransitPinsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return proposeAdminTransitPinsBound(url, options); +} const provisionAdminDawarichBound = bindAdminOperation("provisionAdminDawarich"); export async function provisionAdminDawarichRequest( url: string, @@ -813,6 +848,13 @@ export async function updateAdminServiceConfigRequest( ): Promise { return updateAdminServiceConfigBound(url, options); } +const validateAdminTransitPinsBound = bindAdminOperation("validateAdminTransitPins"); +export async function validateAdminTransitPinsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return validateAdminTransitPinsBound(url, options); +} const verifyAdminAccountBound = bindAdminOperation("verifyAdminAccount"); export async function verifyAdminAccountRequest( url: string, diff --git a/packages/core/src/build-artifacts/catalog-control-edges.test.ts b/packages/core/src/build-artifacts/catalog-control-edges.test.ts new file mode 100644 index 000000000..3bf8a182b --- /dev/null +++ b/packages/core/src/build-artifacts/catalog-control-edges.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { verifyFiniteControlEdge } from "./target-lease"; + +describe("finite catalog workflow edges", () => { + it("allows validation's staging and restoration leaves without promotion/activation", () => { + for (const phase of [ + "stop-staging", + "restart-staging", + "proxy-stage", + "proxy-restore", + ] as const) + expect(() => + verifyFiniteControlEdge("data-manager", "catalog.validate", phase, "ops-agent"), + ).not.toThrow(); + for (const phase of [ + "promote-primary", + "stop-primary", + "recover-primary", + "proxy-commit", + "paired-pin-commit", + ] as const) + expect(() => + verifyFiniteControlEdge("data-manager", "catalog.validate", phase, "ops-agent"), + ).toThrow(); + }); + it("keeps paired commit specific to activation and the intended recipient", () => { + expect(() => + verifyFiniteControlEdge("data-manager", "catalog.activate", "paired-pin-commit", "ops-agent"), + ).not.toThrow(); + expect(() => + verifyFiniteControlEdge("data-manager", "transit-sync", "paired-pin-commit", "ops-agent"), + ).toThrow(); + expect(() => + verifyFiniteControlEdge( + "data-manager", + "catalog.activate", + "paired-pin-commit", + "data-manager", + ), + ).toThrow(); + }); +}); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 99669eae3..7a07bf988 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -2056,7 +2056,11 @@ export function retainControlEffectReceipt( verifyDataUpdateCompletionSource(authority, actual); } const result = receipt.result as { outcome?: string }; - if (!result.outcome || ["applied", "unchanged", "committed"].includes(result.outcome)) { + if ( + !result.outcome || + ["applied", "unchanged", "committed"].includes(result.outcome) || + (s.kind === "transitousLock.approve" && result.outcome === "activated") + ) { outcome = "committed"; publication = "committed"; } diff --git a/packages/core/src/build-artifacts/copy-tree.test.ts b/packages/core/src/build-artifacts/copy-tree.test.ts new file mode 100644 index 000000000..1bb753905 --- /dev/null +++ b/packages/core/src/build-artifacts/copy-tree.test.ts @@ -0,0 +1,159 @@ +import { + linkSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { copyArtifactTree, inspectArtifactTree, isolateArtifactHardlinks } from "./copy-tree"; +import { AnchoredDirectory } from "./filesystem"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function directories() { + const root = mkdtempSync(join(tmpdir(), "omx-copy-tree-")); + roots.push(root); + mkdirSync(join(root, "source")); + mkdirSync(join(root, "target"), { mode: 0o700 }); + return { + root, + source: new AnchoredDirectory(join(root, "source")), + target: new AnchoredDirectory(join(root, "target"), undefined, true), + }; +} +it("copies a complete bounded tree with actual byte digests and separate inodes", async () => { + const f = directories(); + try { + mkdirSync(join(f.root, "source", "empty")); + mkdirSync(join(f.root, "source", "nested")); + writeFileSync(join(f.root, "source/nested/file"), "retained"); + const limits = { maximumEntries: 10, maximumBytes: 100, maximumManifestBytes: 4096 }, + inventory = inspectArtifactTree(f.source, limits); + const copied = await copyArtifactTree({ + source: f.source, + destination: f.target, + expectedInventory: inventory.metadataDigest, + ...limits, + }); + expect(copied.files).toEqual([ + { + path: "nested/file", + sizeBytes: 8, + sha256: "ca67bd59cc4116ac80db5f64cbf4747f26ba929c85e8cc689a6123368d08719b", + }, + ]); + expect(readFileSync(join(f.root, "target/nested/file"), "utf8")).toBe("retained"); + expect(copied.directories).toEqual(["empty", "nested"]); + expect(lstatSync(join(f.root, "source/nested/file")).ino).not.toBe( + lstatSync(join(f.root, "target/nested/file")).ino, + ); + } finally { + f.source.close(); + f.target.close(); + } +}); +it("refuses symlinks, exhausted budgets and source drift before copying", async () => { + const f = directories(); + try { + writeFileSync(join(f.root, "source/file"), "before"); + const limits = { maximumEntries: 10, maximumBytes: 100, maximumManifestBytes: 4096 }, + original = inspectArtifactTree(f.source, limits); + writeFileSync(join(f.root, "source/file"), "after"); + await expect( + copyArtifactTree({ + source: f.source, + destination: f.target, + expectedInventory: original.metadataDigest, + ...limits, + }), + ).rejects.toThrow(/changed/); + expect(f.target.list()).toEqual([]); + expect(() => inspectArtifactTree(f.source, { ...limits, maximumBytes: 1 })).toThrow(/budget/); + expect(() => inspectArtifactTree(f.source, { ...limits, maximumEntries: 0 })).toThrow(/budget/); + symlinkSync(join(f.root, "source/file"), join(f.root, "source/link")); + expect(() => inspectArtifactTree(f.source, limits)).toThrow(/symlink|special/); + } finally { + f.source.close(); + f.target.close(); + } +}); +it("isolates shared producer inodes before writes while preserving active bytes and inode", async () => { + const f = directories(); + try { + writeFileSync(join(f.root, "source/file"), "original feed"); + utimesSync( + join(f.root, "source/file"), + new Date("2020-01-01T00:00:00Z"), + new Date("2020-01-01T00:00:00Z"), + ); + writeFileSync(join(f.root, "source/unshared"), "local"); + linkSync(join(f.root, "source/file"), join(f.root, "target/live")); + const active = lstatSync(join(f.root, "target/live")).ino; + const local = lstatSync(join(f.root, "source/unshared")).ino; + const result = await isolateArtifactHardlinks({ + source: f.source, + minimumFreeBytes: 0, + maximumEntries: 10, + maximumBytes: 100, + maximumManifestBytes: 4096, + }); + expect(result).toEqual({ isolated: 1, canceled: false }); + expect(lstatSync(join(f.root, "source/file")).ino).not.toBe(active); + expect(lstatSync(join(f.root, "source/file")).mtimeMs).toBeCloseTo( + Date.parse("2020-01-01T00:00:00Z"), + 1, + ); + expect(lstatSync(join(f.root, "source/unshared")).ino).toBe(local); + writeFileSync(join(f.root, "source/file"), "new download"); + expect(readFileSync(join(f.root, "target/live"), "utf8")).toBe("original feed"); + expect(lstatSync(join(f.root, "target/live")).ino).toBe(active); + expect( + ( + await isolateArtifactHardlinks({ + source: f.source, + minimumFreeBytes: 0, + maximumEntries: 10, + maximumBytes: 100, + maximumManifestBytes: 4096, + }) + ).isolated, + ).toBe(0); + } finally { + f.source.close(); + f.target.close(); + } +}); +it("cancels a bounded in-flight isolation without replacing the producer or leaving a temporary writer", async () => { + const f = directories(); + try { + writeFileSync(join(f.root, "source/file"), Buffer.alloc(16 * 1024 * 1024, 42)); + linkSync(join(f.root, "source/file"), join(f.root, "target/live")); + const active = lstatSync(join(f.root, "target/live")).ino; + const controller = new AbortController(); + const run = isolateArtifactHardlinks({ + source: f.source, + minimumFreeBytes: 0, + signal: controller.signal, + maximumEntries: 10, + maximumBytes: 32 * 1024 * 1024, + maximumManifestBytes: 4096, + }); + controller.abort(); + expect(await run).toEqual({ isolated: 0, canceled: true }); + expect(lstatSync(join(f.root, "source/file")).ino).toBe(active); + expect(f.source.list()).toEqual(["file"]); + expect(readFileSync(join(f.root, "target/live")).length).toBe(16 * 1024 * 1024); + } finally { + f.source.close(); + f.target.close(); + } +}); diff --git a/packages/core/src/build-artifacts/copy-tree.ts b/packages/core/src/build-artifacts/copy-tree.ts new file mode 100644 index 000000000..379e530f7 --- /dev/null +++ b/packages/core/src/build-artifacts/copy-tree.ts @@ -0,0 +1,391 @@ +/** Bounded data-only filesystem copying. No admission, signer or effect authority. */ +import { createHash, randomUUID } from "node:crypto"; +import { + type BigIntStats, + constants, + fsyncSync, + lstatSync, + type ReadStream, + renameSync, + statfsSync, + unlinkSync, + type WriteStream, +} from "node:fs"; +import { open } from "node:fs/promises"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { type AnchoredDirectory, identity, sameInode } from "./filesystem"; +import { artifactFingerprint, safeArtifactPath } from "./manifest"; +export interface ArtifactTreeLimits { + maximumEntries: number; + maximumBytes: number; + maximumManifestBytes: number; +} +interface FileObservation { + path: string; + sizeBytes: number; + dev: string; + ino: string; + mtimeNs: string; + ctimeNs: string; +} +export interface ArtifactTreeInventory { + files: FileObservation[]; + directories: string[]; + totalBytes: number; + metadataDigest: string; +} +function checkLimits(limits: ArtifactTreeLimits) { + if ( + !Number.isSafeInteger(limits.maximumEntries) || + limits.maximumEntries < 0 || + limits.maximumEntries > 100000 || + !Number.isSafeInteger(limits.maximumBytes) || + limits.maximumBytes < 0 || + !Number.isSafeInteger(limits.maximumManifestBytes) || + limits.maximumManifestBytes < 0 || + limits.maximumManifestBytes > 64 * 1024 * 1024 + ) + throw Error("Invalid artifact tree budget"); +} +function observation(path: string, stat: BigIntStats): FileObservation { + return { + path, + sizeBytes: Number(stat.size), + dev: String(stat.dev), + ino: String(stat.ino), + mtimeNs: String(stat.mtimeNs), + ctimeNs: String(stat.ctimeNs), + }; +} + +export function inspectArtifactTree( + source: AnchoredDirectory, + limits: ArtifactTreeLimits, +): ArtifactTreeInventory { + checkLimits(limits); + source.check(); + const files: FileObservation[] = [], + directories: string[] = []; + let totalBytes = 0, + manifestBytes = 128; + const count = () => { + if ( + files.length + directories.length > limits.maximumEntries || + totalBytes > limits.maximumBytes || + !Number.isSafeInteger(totalBytes) || + manifestBytes > limits.maximumManifestBytes + ) + throw Error("Artifact tree exceeds its budget"); + }; + const scan = (directory: AnchoredDirectory, prefix: string, depth: number) => { + if (depth > 64) throw Error("Artifact tree exceeds its depth budget"); + for (const name of directory.listBounded(limits.maximumEntries + 1).sort()) { + const path = prefix ? `${prefix}/${name}` : name; + if (!safeArtifactPath(path)) throw Error("Unsafe artifact path"); + const stat = lstatSync(directory.name(name), { bigint: true }); + if (stat.isSymbolicLink()) throw Error("Artifact tree contains a symlink"); + if (stat.isDirectory()) { + directories.push(path); + manifestBytes += Buffer.byteLength(JSON.stringify(path)) + 1; + count(); + const child = directory.directory(name); + try { + scan(child, path, depth + 1); + } finally { + child.close(); + } + } else if (stat.isFile()) { + const file = observation(path, stat); + files.push(file); + totalBytes += file.sizeBytes; + manifestBytes += Buffer.byteLength(JSON.stringify(file)) + 1; + count(); + } else throw Error("Artifact tree contains a special file"); + } + directory.check(); + }; + scan(source, "", 0); + const metadataDigest = createHash("sha256") + .update(JSON.stringify({ directories, files })) + .digest("hex"); + return { files, directories, totalBytes, metadataDigest }; +} +function parent(root: AnchoredDirectory, path: string, create: boolean) { + const parts = path.split("/"), + opened: AnchoredDirectory[] = []; + let directory = root; + try { + for (const name of parts.slice(0, -1)) { + directory = directory.directory(name, create, create ? true : undefined); + opened.push(directory); + } + return { + directory, + name: parts.at(-1)!, + close: () => { + for (const dir of opened.reverse()) dir.close(); + }, + }; + } catch (error) { + for (const dir of opened.reverse()) dir.close(); + throw error; + } +} + +/** Detach producer files before a build can rewrite inputs shared with a live generation. + * Only the producer directory entry changes; existing consumers retain their inode/bytes. + * Cancellation returns only after all owned writes and temporary-file cleanup settle. */ +export async function isolateArtifactHardlinks( + input: ArtifactTreeLimits & { + source: AnchoredDirectory; + minimumFreeBytes: number; + signal?: AbortSignal; + }, +): Promise<{ isolated: number; canceled: boolean }> { + if (!Number.isSafeInteger(input.minimumFreeBytes) || input.minimumFreeBytes < 0) + throw Error("Invalid isolation capacity floor"); + const inventory = inspectArtifactTree(input.source, input); + const shared = inventory.files.filter((file) => { + const location = parent(input.source, file.path, false); + try { + const current = lstatSync(location.directory.name(location.name), { bigint: true }); + if (JSON.stringify(observation(file.path, current)) !== JSON.stringify(file)) + throw Error("Producer input changed before isolation"); + return current.nlink > 1n; + } finally { + location.close(); + } + }); + const bytes = shared.reduce((sum, file) => sum + file.sizeBytes, 0); + if (bytes > 0) { + const capacity = statfsSync(input.source.anchor, { bigint: true }); + if ( + capacity.bavail * capacity.bsize < + BigInt(Math.max(bytes, input.minimumFreeBytes)) + 64n * 1024n * 1024n + ) + throw Error("Insufficient capacity to isolate shared producer inputs"); + } + let isolated = 0; + for (const file of shared) { + if (input.signal?.aborted) return { isolated, canceled: true }; + const location = parent(input.source, file.path, false); + const temporary = `.isolate-${randomUUID()}`; + let reader: Awaited> | undefined, + writer: Awaited> | undefined, + created = false; + try { + reader = await open( + location.directory.name(location.name), + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const before = await reader.stat({ bigint: true }); + if ( + !before.isFile() || + JSON.stringify(observation(file.path, before)) !== JSON.stringify(file) + ) + throw Error("Producer input changed before isolation"); + writer = await open( + location.directory.name(temporary), + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + Number(before.mode & 0o755n), + ); + created = true; + const buffer = Buffer.alloc(1024 * 1024); + let total = 0; + while (total < file.sizeBytes) { + if (input.signal?.aborted) return { isolated, canceled: true }; + const { bytesRead } = await reader.read( + buffer, + 0, + Math.min(buffer.length, file.sizeBytes - total), + null, + ); + if (!bytesRead) throw Error("Producer input truncated during isolation"); + let written = 0; + while (written < bytesRead) { + const result = await writer.write(buffer, written, bytesRead - written, null); + if (!result.bytesWritten) throw Error("Producer isolation write did not advance"); + written += result.bytesWritten; + } + total += bytesRead; + } + // Detachment is not a download. Keep the producer's original data age + // so upstream freshness/cache decisions cannot mistake it for new data. + await writer.utimes(Number(before.atimeNs) / 1e9, Number(before.mtimeNs) / 1e9); + await writer.sync(); + if (input.signal?.aborted) return { isolated, canceled: true }; + if ( + JSON.stringify(observation(file.path, await reader.stat({ bigint: true }))) !== + JSON.stringify(file) || + JSON.stringify( + observation( + file.path, + lstatSync(location.directory.name(location.name), { bigint: true }), + ), + ) !== JSON.stringify(file) + ) + throw Error("Producer input changed during isolation"); + location.directory.check(); + renameSync(location.directory.name(temporary), location.directory.name(location.name)); + created = false; + fsyncSync(location.directory.fd); + isolated++; + } finally { + try { + await reader?.close(); + } finally { + try { + await writer?.close(); + } finally { + try { + if (created) { + unlinkSync(location.directory.name(temporary)); + fsyncSync(location.directory.fd); + } + } finally { + location.close(); + } + } + } + } + } + return { isolated, canceled: input.signal?.aborted ?? false }; +} +export async function copyArtifactTree( + input: ArtifactTreeLimits & { + source: AnchoredDirectory; + destination: AnchoredDirectory; + expectedInventory: string; + signal?: AbortSignal; + }, +) { + const inventory = inspectArtifactTree(input.source, input); + if (inventory.metadataDigest !== input.expectedInventory) + throw Error("Artifact source changed before copy"); + input.destination.check(); + if ( + sameInode(identity(input.source.stat), identity(input.destination.stat)) || + input.destination.hasEntries() + ) + throw Error("Empty separate artifact destination required"); + const files: Array<{ path: string; sizeBytes: number; sha256: string }> = []; + for (const path of inventory.directories) { + const target = parent(input.destination, path + "/.directory", true); + target.close(); + } + for (const file of inventory.files) { + input.signal?.throwIfAborted(); + const source = parent(input.source, file.path, false), + target = parent(input.destination, file.path, true); + let reader: Awaited> | undefined, + writer: Awaited> | undefined, + readStream: ReadStream | undefined, + writeStream: WriteStream | undefined; + try { + reader = await open( + source.directory.name(source.name), + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const before = await reader.stat({ bigint: true }); + if ( + !before.isFile() || + JSON.stringify(observation(file.path, before)) !== JSON.stringify(file) + ) + throw Error("Artifact source inode changed"); + writer = await open( + target.directory.name(target.name), + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ); + const hash = createHash("sha256"); + readStream = reader.createReadStream({ autoClose: false }); + writeStream = writer.createWriteStream({ autoClose: false }); + await pipeline( + readStream, + new Transform({ + transform(chunk, _encoding, done) { + hash.update(chunk); + done(null, chunk); + }, + }), + writeStream, + { signal: input.signal }, + ); + await writer.sync(); + if ( + JSON.stringify(observation(file.path, await reader.stat({ bigint: true }))) !== + JSON.stringify(file) + ) + throw Error("Artifact source changed during copy"); + source.directory.check(); + target.directory.check(); + fsyncSync(target.directory.fd); + files.push({ path: file.path, sizeBytes: file.sizeBytes, sha256: hash.digest("hex") }); + } finally { + readStream?.destroy(); + writeStream?.destroy(); + try { + await reader?.close(); + } finally { + try { + await writer?.close(); + } finally { + source.close(); + target.close(); + } + } + } + } + if (inspectArtifactTree(input.source, input).metadataDigest !== input.expectedInventory) + throw Error("Artifact source changed during copy"); + fsyncSync(input.destination.fd); + return { + files, + directories: inventory.directories, + totalBytes: inventory.totalBytes, + fingerprint: artifactFingerprint(files), + }; +} +/** Hash a regular artifact through a stable descriptor with bounded, abortable reads. */ +export async function hashArtifactFile( + root: AnchoredDirectory, + path: string, + signal?: AbortSignal, +) { + if (!safeArtifactPath(path)) throw Error("Unsafe artifact path"); + const source = parent(root, path, false); + let reader: Awaited> | undefined; + try { + signal?.throwIfAborted(); + reader = await open( + source.directory.name(source.name), + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const before = await reader.stat({ bigint: true }); + if (!before.isFile() || before.size > BigInt(Number.MAX_SAFE_INTEGER)) + throw Error("Regular bounded artifact required"); + const hash = createHash("sha256"), + buffer = Buffer.alloc(1024 * 1024); + let total = 0; + for (;;) { + signal?.throwIfAborted(); + const { bytesRead } = await reader.read(buffer, 0, buffer.length, null); + if (!bytesRead) break; + total += bytesRead; + if (BigInt(total) > before.size) throw Error("Artifact grew while hashing"); + hash.update(buffer.subarray(0, bytesRead)); + } + if ( + BigInt(total) !== before.size || + JSON.stringify(observation(path, before)) !== + JSON.stringify(observation(path, await reader.stat({ bigint: true }))) + ) + throw Error("Artifact changed while hashing"); + source.directory.check(); + return { path, sizeBytes: total, sha256: hash.digest("hex") }; + } finally { + await reader?.close(); + source.close(); + } +} diff --git a/packages/core/src/build-artifacts/data-execution-candidate.ts b/packages/core/src/build-artifacts/data-execution-candidate.ts index 81b381b12..28e4d5268 100644 --- a/packages/core/src/build-artifacts/data-execution-candidate.ts +++ b/packages/core/src/build-artifacts/data-execution-candidate.ts @@ -11,7 +11,9 @@ import { } from "node:fs"; import { join, sep } from "node:path"; import { z } from "zod"; +import { readCatalogProposalFile, readCatalogValidationFile } from "../catalog-pins/record-files"; import { canonicalJson } from "../ops/canonical-json"; +import { pairedPinApprovalInputSchema } from "../ops/catalog-pin-approval"; import { type OpsJobStatus, type OpsOperation, parseOpsJobStatus } from "../ops/contract"; import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; import { retainedParentExecutionSchema } from "../ops/execution-identity"; @@ -54,6 +56,13 @@ const schema = z.strictObject({ execution: executionRefSchema, phase: z.enum(["promote-primary", "proxy-stage", "proxy-commit", "proxy-restore"]), ordinal: z.number().int().min(0).max(31), + catalogActivation: z + .strictObject({ + proposalId: z.uuid(), + validationId: z.uuid(), + validationDigest: z.string().regex(/^[a-f0-9]{64}$/), + }) + .optional(), candidateId: z.string(), directory: z.strictObject({ dev: z.string(), ino: z.string() }), consumer: z @@ -226,6 +235,98 @@ function proxyConsumer( mounted.close(); } } +function catalogRecordSource(state: ReturnType) { + return { + dataRoot: state.dataRoot, + installationId: state.installation.installationId, + dataManagerPublicKey: state.publicKeys["data-manager"], + agentPublicKey: state.publicKeys["ops-agent"], + }; +} +function catalogActivationCandidate(s: ReturnType) { + if (s.kind !== "catalog.activate") return undefined; + const raw = readPrivateRecord( + s.installation, + "executions", + `${s.execution.operationKey}.json`, + 65536, + ), + admitted = verifyControl( + s.installation.installation.publicKey, + originRecordDomain("openmapx-control-execution", raw), + raw, + z + .object({ + kind: z.literal("catalog.activate"), + operation: z.record(z.string(), z.unknown()), + }) + .passthrough(), + ); + if ( + controlDigest(raw) !== s.receiptDigest || + controlDigest(admitted.operation) !== s.execution.operationFingerprint || + admitted.operation.kind !== "catalog.activate" + ) + throw Error("Original activation body required"); + const input = pairedPinApprovalInputSchema.parse({ + ...admitted.operation, + kind: "transitousLock.approve", + }); + const validation = readCatalogValidationFile( + catalogRecordSource(s.installation), + input.proposalId, + input.validationId, + ); + if ( + validation.validation.status !== "passed" || + validation.validation.diffDigest !== input.diffDigest || + !input.acknowledgeLicenseChanges + ) + throw Error("Passing reviewed validation required"); + return { + proposalId: input.proposalId, + validationId: input.validationId, + validationDigest: controlDigest(validation), + }; +} +function verifyCatalogActivationCandidate( + installation: ControlInstallation, + candidate: z.infer, +) { + const selected = candidate.catalogActivation; + if (!selected) return candidate.execution.executionId; + const validation = readCatalogValidationFile( + catalogRecordSource(installationState(installation)), + selected.proposalId, + selected.validationId, + ), + proposal = readCatalogProposalFile( + catalogRecordSource(installationState(installation)), + selected.proposalId, + ); + if ( + !proposal || + validation.validation.status !== "passed" || + controlDigest(validation) !== selected.validationDigest || + controlDigest(proposal) !== validation.proposalDigest || + validation.validation.pinDigest !== proposal.pins.generationId || + controlDigest({ + kind: "catalog.activate", + proposalId: selected.proposalId, + validationId: selected.validationId, + expectedGenerationId: proposal.baseGenerationId, + diffDigest: proposal.diffDigest, + acknowledgeLicenseChanges: true, + }) !== candidate.execution.operationFingerprint + ) + throw Error("Activation candidate differs from its original validation/proposal/body"); + if ( + candidate.phase === "promote-primary" && + candidate.files[0]?.sha256 !== validation.validation.manifestHash + ) + throw Error("Promotion manifest differs from its passed validation"); + return selected.validationId; +} function snapshot( authority: VerifiedExecutionAuthority, phase: ControlChildPhase, @@ -239,9 +340,12 @@ function snapshot( if ( !slot || s.installation.installation.role !== "data-manager" || - !["transit-sync", "transit-maintenance"].includes(s.kind) + !["transit-sync", "transit-maintenance", "catalog.validate", "catalog.activate"].includes( + s.kind, + ) ) throw Error("Original data candidate reservation required"); + const activation = catalogActivationCandidate(s); const proxy = ["proxy-stage", "proxy-commit", "proxy-restore"].includes(phase); if (!proxy && phase !== "promote-primary") return null; const relative = proxy ? "motis-feed-proxy" : `motis/${preparing ? "staging" : "live"}`; @@ -269,15 +373,20 @@ function snapshot( execution: s.execution, phase, ordinal, + ...(activation ? { catalogActivation: activation } : {}), candidateId: proxy ? `ctrl-${createHash("sha256").update(`openmapx-transit-candidate-v1\0${s.execution.operationKey}\0${phase}\0${ordinal}`).digest("hex").slice(0, 32)}` - : s.execution.executionId, + : (activation?.validationId ?? s.execution.executionId), directory: identity(directory.stat), files: retainedFiles.map(({ name, inode, sha256 }) => ({ name, inode, sha256 })), consumer: proxy ? proxyConsumer(directory, retainedFiles[0]!) : null, artifactsDigest: proxy ? null - : candidateArtifacts(directory, retainedFiles[0]!.bytes, s.execution.executionId), + : candidateArtifacts( + directory, + retainedFiles[0]!.bytes, + activation?.validationId ?? s.execution.executionId, + ), }; } finally { directory.close(); @@ -464,12 +573,13 @@ export function inspectDataTransitCandidate( candidate.execution.executor !== "data-manager" ) throw Error("Original candidate parent/phase/body changed"); + const expectedEpoch = verifyCatalogActivationCandidate(installation, candidate); const promotion = operation.kind === "motis.primary.promote"; if ( promotion ? candidate.phase !== "promote-primary" || candidate.candidateId !== operation.preparedRunId || - candidate.candidateId !== candidate.execution.executionId || + candidate.candidateId !== expectedEpoch || canonicalJson(proof.serviceIds) !== canonicalJson(["motis"]) : !["proxy-stage", "proxy-commit", "proxy-restore"].includes(candidate.phase) || candidate.candidateId !== operation.candidateId || @@ -509,8 +619,7 @@ export function inspectDataTransitCandidate( if ( !manifestBytes || candidate.consumer !== null || - candidateArtifacts(confined, manifestBytes, candidate.execution.executionId) !== - candidate.artifactsDigest + candidateArtifacts(confined, manifestBytes, expectedEpoch) !== candidate.artifactsDigest ) throw Error("Prepared candidate artifacts changed"); } else if ( diff --git a/packages/core/src/build-artifacts/data-execution-journal.ts b/packages/core/src/build-artifacts/data-execution-journal.ts index 2928d1b15..ffc9b98e6 100644 --- a/packages/core/src/build-artifacts/data-execution-journal.ts +++ b/packages/core/src/build-artifacts/data-execution-journal.ts @@ -3,10 +3,15 @@ import { randomUUID } from "node:crypto"; import { z } from "zod"; import { canonicalJson } from "../ops/canonical-json"; +import { + pairedPinApprovalInputSchema, + pairedPinApprovalResultSchema, +} from "../ops/catalog-pin-approval"; import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; import { type DataExecutionResult, type DataExecutionStatus, + dataExecutionInputSchema, dataExecutionResultSchema, dataExecutionStatusSchema, generationHardlinksPlanSchema, @@ -19,6 +24,7 @@ import { import { beginControlEffect, type ControlInstallation, + cancelControlExecution, controlDigest, executionAuthorityState, installationState, @@ -51,10 +57,13 @@ import { completeFamilyJournal, markFamilyTerminal, publishFamilyCanonicalTerminal, + verifyFamilyOwnedRecord, } from "./retention-family"; import { + assertRetainedCatalogActivationChildrenSettled, closeArtifactTargetLeaseHandle, inspectArtifactTargetLeaseChildSettlement, + inspectRetainedOwnedArtifactTargetChildSettlement, restoreArtifactTargetLeaseForSettlement, restoreGenerationHardlinksLeaseForSettlement, type TargetLeaseAuthority, @@ -92,7 +101,7 @@ const workerSchema = z admissionDigest: z.string().regex(/^[a-f0-9]{64}$/), workerId: z.uuid(), containedAt: z.number().int().positive().safe(), - outcome: z.enum(["returned", "partial", "failed"]), + outcome: z.enum(["returned", "partial", "failed", "catalog-activation-committed"]), publication: z.enum(["not_started", "committed", "unknown"]), childrenDigest: z .string() @@ -267,8 +276,7 @@ export function createDataExecutionControlJournal( ); const intent = recoveryIntent(authority); let required = intent !== null || children.some((child) => child.phase === "proxy-restore"); - let lease: TargetLeaseAuthority | undefined; - try { + { for (const child of children.filter((child) => ["proxy-stage", "proxy-commit", "proxy-restore"].includes(child.phase), )) { @@ -285,8 +293,10 @@ export function createDataExecutionControlJournal( ) throw Error("Original proxy phase changed"); if (child.state !== "terminal" || !child.execution) continue; - lease ??= restoreArtifactTargetLeaseForSettlement(authority); - const shared = inspectArtifactTargetLeaseChildSettlement(lease, child.execution); + const shared = inspectRetainedOwnedArtifactTargetChildSettlement( + authority, + child.execution, + ); const canonical = verifyFeedProxyCanonicalTerminal(authority, child.execution); if ( restore && @@ -312,8 +322,7 @@ export function createDataExecutionControlJournal( child.execution.operationKey !== restore.execution.operationKey ) throw Error("Original proxy restore obligation remains pending"); - lease ??= restoreArtifactTargetLeaseForSettlement(authority); - const shared = inspectArtifactTargetLeaseChildSettlement(lease, child.execution), + const shared = inspectRetainedOwnedArtifactTargetChildSettlement(authority, child.execution), canonical = verifyFeedProxyCanonicalTerminal(authority, child.execution); const result = z .strictObject({ candidateId: z.string(), reloaded: z.literal(true) }) @@ -330,10 +339,108 @@ export function createDataExecutionControlJournal( ) throw Error("Original successful proxy restore is not proved"); return "settled"; - } finally { - if (lease) closeArtifactTargetLeaseHandle(lease); } } + /** Only the final fixed paired leaf can close a killed activation worker. + * Its untrusted result is accepted solely by the exact signed physical receipt. + * No phase is dispatched and no public status alone proves completion. */ + async function catalogActivationCompletion(authority: VerifiedExecutionAuthority) { + const state = executionAuthorityState(authority), + value = await load(state.execution); + const input = dataExecutionInputSchema.parse(value.row.execution_input); + if ( + state.kind !== "catalog.activate" || + input.kind !== "catalog.activate" || + !input.acknowledgeLicenseChanges || + controlDigest(input) !== state.execution.operationFingerprint || + !state.origin + ) + throw Error("Original reviewed catalog activation required"); + assertRetainedCatalogActivationChildrenSettled(authority); + const children = value.row.execution_children.map((raw) => + z + .object({ + phase: z.string(), + ordinal: z.number(), + state: z.literal("terminal"), + operationKey: z.string(), + operationFingerprint: z.string(), + parentExecution: executionRefSchema, + execution: executionRefSchema, + operation: z.unknown(), + approvalResult: z.unknown().optional(), + }) + .passthrough() + .parse(raw), + ); + for (const child of children) { + const slot = state.reservations.find( + (slot) => slot.phase === child.phase && slot.ordinal === child.ordinal, + ); + if ( + !slot || + slot.execution.operationKey !== child.operationKey || + slot.operationFingerprint !== child.operationFingerprint || + controlDigest(child.operation) !== child.operationFingerprint || + controlDigest(child.parentExecution) !== controlDigest(state.execution) || + !inspectRetainedOwnedArtifactTargetChildSettlement(authority, child.execution) + ) + throw Error("Original settled activation child required"); + } + const matches = children.filter( + (child) => child.phase === "paired-pin-commit" && child.ordinal === 0, + ); + if (matches.length !== 1) throw Error("Original paired approval required"); + const child = matches[0]!, + operation = pairedPinApprovalInputSchema.parse(child.operation), + result = pairedPinApprovalResultSchema.parse(child.approvalResult); + const physical = inspectRetainedOwnedArtifactTargetChildSettlement(authority, child.execution); + if ( + !physical || + physical.outcome !== "committed" || + physical.publication !== "committed" || + physical.terminalReceiptDigest !== controlDigest({ kind: "ops-result", result }) || + operation.proposalId !== input.proposalId || + operation.validationId !== input.validationId || + operation.diffDigest !== input.diffDigest || + operation.expectedGenerationId !== input.expectedGenerationId || + !operation.acknowledgeLicenseChanges || + result.proposalId !== input.proposalId || + result.validationId !== input.validationId || + result.previousGenerationId !== input.expectedGenerationId + ) + throw Error("Actual committed paired approval evidence required"); + const recovery = await assertOwnedProxyRecoverySettled(state.execution), + stamp = Date.parse(result.committedAt), + digest = controlDigest({ + execution: state.execution, + approval: child.execution, + receipt: physical.terminalReceiptDigest, + }); + if (stamp < value.status.admittedAt || stamp > Date.now()) + throw Error("Original catalog commit time changed"); + return workerSchema.parse({ + version: 1, + installationId: installation.installationId, + origin: state.origin, + execution: state.execution, + admissionDigest: state.receiptDigest, + workerId: `${digest.slice(0, 8)}-${digest.slice(8, 12)}-4${digest.slice(13, 16)}-8${digest.slice(17, 20)}-${digest.slice(20, 32)}`, + containedAt: stamp, + outcome: "catalog-activation-committed", + publication: "committed", + childrenDigest: controlDigest(value.row.execution_children), + proxyRecovery: recovery, + result: { + kind: "catalog.activate", + proposalId: input.proposalId, + validationId: input.validationId, + outcome: "activated", + generationId: result.generationId, + reason: null, + }, + }); + } function readProof(authority: VerifiedExecutionAuthority) { const s = executionAuthorityState(authority); const proof = verifyControl( @@ -559,7 +666,11 @@ export function createDataExecutionControlJournal( }, pendingRunnerAttempts(authority: VerifiedExecutionAuthority) { const state = executionAuthorityState(authority); - if (state.installation !== local || state.kind !== "transit-sync" || !state.origin) + if ( + state.installation !== local || + !["transit-sync", "catalog.validate"].includes(state.kind) || + !state.origin + ) throw Error("Original transit worker required"); try { const saved = verifyControl( @@ -584,7 +695,11 @@ export function createDataExecutionControlJournal( retainRunnerAttempt(authority: VerifiedExecutionAuthority, request: unknown): void { const state = executionAuthorityState(authority), body = transitousRunnerRequestSchema.parse(request); - if (state.installation !== local || state.kind !== "transit-sync" || !state.origin) + if ( + state.installation !== local || + !["transit-sync", "catalog.validate"].includes(state.kind) || + !state.origin + ) throw Error("Original transit worker required"); withPrivateEffectWriter(state, () => { const name = `runners-${state.execution.operationKey}.json`; @@ -815,20 +930,25 @@ export function createDataExecutionControlJournal( error = Error("Remote runner containment remains unknown"); } if (!state.origin) throw Error("Dated contained worker origin required"); - const proof = workerSchema.parse({ - version: 1, - installationId: installation.installationId, - origin: state.origin, - execution: state.execution, - admissionDigest: state.receiptDigest, - workerId, - containedAt: Date.now(), - outcome: result ? (partial ? "partial" : "returned") : "failed", - publication, - childrenDigest, - proxyRecovery: recovery, - ...(result ? { result } : {}), - }); + const proof = + state.kind === "catalog.activate" && + result?.kind === "catalog.activate" && + result.outcome === "activated" + ? await catalogActivationCompletion(authority) + : workerSchema.parse({ + version: 1, + installationId: installation.installationId, + origin: state.origin, + execution: state.execution, + admissionDigest: state.receiptDigest, + workerId, + containedAt: Date.now(), + outcome: result ? (partial ? "partial" : "returned") : "failed", + publication, + childrenDigest, + proxyRecovery: recovery, + ...(result ? { result } : {}), + }); // Only the live producer, after awaiting the actual dispatcher promise, // creates this immutable completion. Recovery only reads its signature. writePrivateRecord( @@ -842,7 +962,7 @@ export function createDataExecutionControlJournal( const completion = Object.freeze({ [completionBrand]: true as const }); completions.set(completion, { authority, - workerId, + workerId: proof.workerId, publication, childrenDigest, partial, @@ -850,7 +970,34 @@ export function createDataExecutionControlJournal( error, ...(lease ? { lease } : {}), }); - return { completion, result, error, partial }; + return { completion, result, error, partial, publication }; + }, + async repairCommittedCatalogActivation(execution: ExecutionRef): Promise { + const authority = restoreAdmittedControlExecution(installation, execution), + state = executionAuthorityState(authority); + if (state.kind !== "catalog.activate" || !state.origin) + throw Error("Original catalog activation owner required"); + // This closes all future issuance under the original owner, even when its + // deadline passed. Every already-issued physical child must be settled. + cancelControlExecution(authority); + const proof = await catalogActivationCompletion(authority), + name = `worker-${execution.operationKey}.json`; + const signed = signControl(local.privateKey, "openmapx-data-worker-contained-v1", proof); + try { + const retained = readPrivateRecord(local, "observations", name, 65536); + if (canonicalJson(retained) !== canonicalJson(signed)) + throw Error("Original worker completion already differs"); + verifyFamilyOwnedRecord( + local, + state.origin, + { scope: "private", namespace: "observations", name, container: "" }, + canonicalJson(retained), + ); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + writePrivateRecord(local, "observations", name, signed, true, state.origin); }, async restoreCompletion(execution: ExecutionRef) { const value = await load(execution), @@ -862,6 +1009,20 @@ export function createDataExecutionControlJournal( readPrivateRecord(local, "observations", `worker-${execution.operationKey}.json`, 65536), workerSchema, ); + if (proof.outcome === "catalog-activation-committed") { + if ( + !state.origin || + canonicalJson(proof) !== canonicalJson(await catalogActivationCompletion(authority)) + ) + throw Error("Original final approval completion changed"); + const name = `worker-${execution.operationKey}.json`; + verifyFamilyOwnedRecord( + local, + state.origin, + { scope: "private", namespace: "observations", name, container: "" }, + canonicalJson(readPrivateRecord(local, "observations", name, 65536)), + ); + } if ( proof.installationId !== installation.installationId || proof.admissionDigest !== state.receiptDigest || @@ -923,8 +1084,9 @@ export function createDataExecutionControlJournal( throw Error("Contained worker/current child union changed"); if ( !s.origin || + owned.publication === "unknown" || + terminal.publication !== owned.publication || !isDataExecutionTerminal(terminal) || - terminal.publication === "unknown" || terminal.id !== value.status.id || terminal.operationKey !== value.status.operationKey || terminal.fingerprint !== value.status.fingerprint || @@ -943,11 +1105,15 @@ export function createDataExecutionControlJournal( if ( owned.result && terminal.status !== - (owned.result.kind === "transit-sync" - ? owned.result.finalStatus - : owned.partial - ? "partial" - : "ok") + (owned.result.kind === "catalog.validate" && owned.result.status === "failed" + ? "error" + : owned.result.kind === "catalog.activate" && owned.result.outcome !== "activated" + ? "error" + : owned.result.kind === "transit-sync" + ? owned.result.finalStatus + : owned.partial + ? "partial" + : "ok") ) throw Error("Terminal status differs from actual dispatcher outcome"); if (terminal.status === "ok" && owned.error !== undefined) diff --git a/packages/core/src/build-artifacts/peer-admission.ts b/packages/core/src/build-artifacts/peer-admission.ts index 8c24c7c20..addf9890a 100644 --- a/packages/core/src/build-artifacts/peer-admission.ts +++ b/packages/core/src/build-artifacts/peer-admission.ts @@ -36,6 +36,7 @@ export function inspectControlAdmission( case "motis.primary.stop": case "motis.primary.restart": case "motis.primary.promote": + case "transitousLock.approve": return ["motis"]; case "motis.staging.stop": case "motis.staging.restart": @@ -67,6 +68,11 @@ export function inspectControlAdmission( inspected.execution.executor !== installation.role ) throw Error("Signed immutable admission mismatch"); + if ( + operation.kind === "transitousLock.approve" && + (inspected.phase !== "paired-pin-commit" || inspected.owner.executor !== "data-manager") + ) + throw Error("Original paired activation leaf required"); if (targets.some((id) => !state.registeredTargets.has(id) || !inspected.serviceIds.includes(id))) throw Error("Grant does not contain operation-derived target closure"); const peer = verifyControlPeer(installation, { diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index ecde3cecb..acff8cab7 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -1255,6 +1255,14 @@ export function verifyFiniteControlEdge( recipient: ControlRecipient, ): string[] { if (role === "data-manager") { + if (kind === "catalog.validate") { + if ( + recipient !== "ops-agent" || + !["stop-staging", "restart-staging", "proxy-stage", "proxy-restore"].includes(phase) + ) + throw Error("Unauthorized catalog validation child edge"); + return []; + } if ( !["transit-sync", "catalog.activate", "transit-maintenance"].includes(kind) || recipient !== "ops-agent" || @@ -2029,6 +2037,95 @@ export function inspectArtifactTargetLeaseChildSettlement( return child.settlement; }); } + +/** Observe an original owned child's signed settlement after its root released + * physical locks. This returns data only: no lease/issuer/forward capability. */ +function retainedOwnedTargetState(authority: VerifiedExecutionAuthority) { + const owner = executionAuthorityState(authority), + reference = z + .strictObject({ leaseId: uuid, certificateDigest: digest }) + .parse( + readPrivateRecord( + owner.installation, + "execution-leases", + `${owner.execution.operationKey}.json`, + ), + ), + saved = z + .strictObject({ + certificate: signedCertificateSchema, + privateKey: z.string(), + executionReceiptDigest: digest, + }) + .parse(readPrivateRecord(owner.installation, "leases", `${reference.leaseId}.json`)), + s = openState(owner.installation.dataRoot, reference.leaseId, owner.installation, true); + try { + if ( + controlDigest(s.execution) !== controlDigest(owner.execution) || + controlDigest(s.certificateEnvelope) !== reference.certificateDigest || + controlDigest(saved.certificate) !== reference.certificateDigest || + saved.executionReceiptDigest !== owner.receiptDigest || + !sameControlOrigin(owner.origin, s.certificate.origin) + ) + throw Error("Original retained target owner changed"); + return s; + } catch (error) { + closeState(s); + throw error; + } +} +/** Fixed terminal-only source check for a recovered catalog activation. */ +export function assertRetainedCatalogActivationChildrenSettled( + authority: VerifiedExecutionAuthority, +): void { + const owner = executionAuthorityState(authority); + if (owner.kind !== "catalog.activate" || owner.installation.installation.role !== "data-manager") + throw Error("Original catalog activation owner required"); + const s = retainedOwnedTargetState(authority); + try { + guarded( + s, + () => { + const current = tree(s); + for (const grant of current.grants.values()) { + const child = current.children.get(grant.delegationId); + if (child?.phase !== "settled" || !knownOutcome(child.settlement)) + throw Error("Catalog child tree remains unsettled"); + } + }, + !s.slots?.release, + ); + } finally { + closeState(s); + } +} +export function inspectRetainedOwnedArtifactTargetChildSettlement( + authority: VerifiedExecutionAuthority, + execution: ExecutionRef, +): ChildSettlement | null { + const s = retainedOwnedTargetState(authority); + try { + return guarded( + s, + () => { + const current = tree(s), + grants = [...current.grants.values()].filter( + (grant) => + grant.parentNode === null && controlDigest(grant.child) === controlDigest(execution), + ); + if (grants.length !== 1) throw Error("Exact original child grant required"); + const child = current.children.get(grants[0]!.delegationId); + if (!child || child.phase !== "settled" || !knownOutcome(child.settlement)) return null; + if (controlDigest(child.execution) !== controlDigest(execution)) + throw Error("Original child identity changed"); + return child.settlement; + }, + !s.slots?.release, + ); + } finally { + closeState(s); + } +} /** Source-only observation of the original held maintenance root. It creates * no forward authority, lease, grant or renewed deadline. */ export function configurationMaintenanceTargetOwner(lease: ArtifactTargetLease) { diff --git a/packages/core/src/catalog-pins/index.ts b/packages/core/src/catalog-pins/index.ts index 8cba2c995..34af12f80 100644 --- a/packages/core/src/catalog-pins/index.ts +++ b/packages/core/src/catalog-pins/index.ts @@ -1,5 +1,9 @@ export * from "./bootstrap"; export * from "./generation"; +export * from "./proposal-schemas"; +export * from "./proposals"; +export * from "./review-records"; +export * from "./review-schemas"; export * from "./schemas"; export * from "./store"; export * from "./types"; diff --git a/packages/core/src/catalog-pins/proposal-schemas.ts b/packages/core/src/catalog-pins/proposal-schemas.ts new file mode 100644 index 000000000..56fcd386f --- /dev/null +++ b/packages/core/src/catalog-pins/proposal-schemas.ts @@ -0,0 +1,38 @@ +/** Portable review identity, with no caller-authored validation assertion. */ +import z from "zod/v4"; +import { catalogPinSetSchema } from "./schemas"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); +export const catalogProposalInputSchema = z.strictObject({ + proposalId: z.uuid(), + baseGenerationId: digest.nullable(), + pins: catalogPinSetSchema, + diffDigest: digest, + sourcePolicyRevision: digest, +}); +export const catalogProposalSchema = catalogProposalInputSchema.extend({ + state: z.enum(["proposed", "validating", "validated", "rejected", "activating", "activated"]), + createdAt: z.iso.datetime({ offset: true }), +}); +export const catalogProposalEnvelopePayloadSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + proposal: catalogProposalSchema, +}); +export type CatalogProposalInput = z.infer; +export type CatalogProposal = z.infer; +export const catalogProposalSummarySchema = catalogProposalSchema + .pick({ + proposalId: true, + baseGenerationId: true, + diffDigest: true, + sourcePolicyRevision: true, + state: true, + createdAt: true, + }) + .extend({ + catalogRevisionId: digest, + transitousRef: z.string().max(256), + atlasCommit: z.string().regex(/^[a-f0-9]{40}$/), + gbfsCommit: z.string().regex(/^[a-f0-9]{40}$/), + }); diff --git a/packages/core/src/catalog-pins/proposals.ts b/packages/core/src/catalog-pins/proposals.ts new file mode 100644 index 000000000..75a020403 --- /dev/null +++ b/packages/core/src/catalog-pins/proposals.ts @@ -0,0 +1,175 @@ +/** Immutable agent-signed proposals; retaining one never selects active pins. */ +import { join } from "node:path"; +import { type ControlInstallation, installationState } from "../build-artifacts/control-authority"; +import { controlDigest, signControl, verifyControl } from "../build-artifacts/control-records"; +import { AnchoredDirectory, identity, sameInode } from "../build-artifacts/filesystem"; +import { canonicalJson } from "../ops/canonical-json"; +import { withConfigurationGenerationWriter } from "../services/configuration-generation"; +import { parseCatalogPinSet } from "./generation"; +import { + type CatalogProposal, + type CatalogProposalInput, + catalogProposalEnvelopePayloadSchema, + catalogProposalInputSchema, +} from "./proposal-schemas"; +import { readCatalogPinSet } from "./store"; + +const DOMAIN = "openmapx-catalog-proposal-v1"; +export const MAX_CATALOG_PROPOSAL_BYTES = 768 * 1024; +function withProposalDirectory( + rootDir: string, + installation: ControlInstallation, + create: boolean, + run: (directory: AnchoredDirectory) => T, +): T { + const state = installationState(installation), + opened: AnchoredDirectory[] = []; + try { + const expected = new AnchoredDirectory(state.dataRoot); + opened.push(expected); + const actual = new AnchoredDirectory(join(rootDir, "infra/docker/data")); + opened.push(actual); + if (!sameInode(identity(expected.stat), identity(actual.stat))) + throw Error("Foreign catalog proposal root"); + let directory = new AnchoredDirectory(rootDir); + opened.push(directory); + for (const name of ["infra", "docker", "catalog-pins", "proposals"]) { + directory = directory.directory( + name, + create && ["catalog-pins", "proposals"].includes(name), + ["catalog-pins", "proposals"].includes(name), + ); + opened.push(directory); + } + return run(directory); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +function decode(installation: ControlInstallation, bytes: Buffer): CatalogProposal { + const state = installationState(installation), + value = verifyControl( + state.publicKeys["ops-agent"], + DOMAIN, + JSON.parse(bytes.toString("utf8")), + catalogProposalEnvelopePayloadSchema, + ); + if (value.installationId !== installation.installationId || value.proposal.state !== "proposed") + throw Error("Foreign catalog proposal"); + parseCatalogPinSet(value.proposal.pins); + return value.proposal; +} +export function readCatalogProposal(input: { + rootDir: string; + installation: ControlInstallation; + proposalId: string; +}): CatalogProposal | null { + const id = catalogProposalInputSchema.shape.proposalId.parse(input.proposalId); + try { + return withProposalDirectory(input.rootDir, input.installation, false, (directory) => + decode(input.installation, directory.read(`${id}.json`, MAX_CATALOG_PROPOSAL_BYTES)), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} +export async function retainCatalogProposal(input: { + rootDir: string; + installation: ControlInstallation; + proposal: CatalogProposalInput; + now?: Date; + signal?: AbortSignal; +}): Promise { + const state = installationState(input.installation); + if (input.installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Catalog proposal requires the original operations agent"); + const proposal = catalogProposalInputSchema.parse(input.proposal); + parseCatalogPinSet(proposal.pins); + if (!proposal.pins.transitous.submodules["transitland-atlas"]) + throw Error("Catalog proposal atlas pin missing"); + return withConfigurationGenerationWriter( + join(input.rootDir, "infra/docker"), + async () => + withProposalDirectory(input.rootDir, input.installation, true, (directory) => { + const name = `${proposal.proposalId}.json`; + let existing: CatalogProposal | null = null; + try { + existing = decode(input.installation, directory.read(name, MAX_CATALOG_PROPOSAL_BYTES)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (existing) { + const { state: _state, createdAt: _time, ...original } = existing; + if (controlDigest(original) !== controlDigest(proposal)) + throw Error("Immutable catalog proposal changed"); + return existing; + } + if ((readCatalogPinSet(input.rootDir)?.generationId ?? null) !== proposal.baseGenerationId) + throw Error("Catalog proposal base changed"); + if (directory.listBounded(4096).length >= 4096) + throw Error("Catalog proposal capacity reached"); + const retained: CatalogProposal = { + ...proposal, + state: "proposed", + createdAt: (input.now ?? new Date()).toISOString(), + }; + const bytes = canonicalJson( + signControl(state.privateKey, DOMAIN, { + version: 1, + installationId: input.installation.installationId, + proposal: retained, + }), + ); + if (Buffer.byteLength(bytes, "utf8") > MAX_CATALOG_PROPOSAL_BYTES) + throw Error("Catalog proposal exceeds its byte limit"); + directory.write(name, bytes); + return retained; + }), + input.signal, + ); +} +/** Bounded metadata only; selecting a proposal separately re-verifies its signature. */ +export function listCatalogProposals(input: { + rootDir: string; + installation: ControlInstallation; + offset?: number; +}) { + const offset = input.offset ?? 0; + if (!Number.isSafeInteger(offset) || offset < 0 || offset > 4096) + throw Error("Invalid catalog proposal offset"); + try { + return withProposalDirectory(input.rootDir, input.installation, false, (directory) => { + const ids = directory.listBounded(4096).sort(); + const proposals = ids.slice(offset, offset + 100).map((name) => { + const id = catalogProposalInputSchema.shape.proposalId.parse(name.replace(/\.json$/, "")); + if (name !== `${id}.json`) throw Error("Unexpected catalog proposal record"); + const proposal = decode( + input.installation, + directory.read(name, MAX_CATALOG_PROPOSAL_BYTES), + ); + if (proposal.proposalId !== id) throw Error("Catalog proposal filename mismatch"); + const { pins, ...metadata } = proposal; + const atlasCommit = pins.transitous.submodules["transitland-atlas"]; + if (!atlasCommit) throw Error("Catalog proposal atlas pin missing"); + return { + ...metadata, + catalogRevisionId: pins.generationId, + transitousRef: pins.transitous.ref, + atlasCommit, + gbfsCommit: pins.gbfs.commit, + }; + }); + return { + proposals, + total: ids.length, + offset, + nextOffset: offset + proposals.length < ids.length ? offset + proposals.length : null, + }; + }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return { proposals: [], total: 0, offset, nextOffset: null }; + throw error; + } +} diff --git a/packages/core/src/catalog-pins/record-files.ts b/packages/core/src/catalog-pins/record-files.ts new file mode 100644 index 000000000..552228bf5 --- /dev/null +++ b/packages/core/src/catalog-pins/record-files.ts @@ -0,0 +1,132 @@ +/** Data-only fixed signed record readers. Callers obtain their source from the installed nominal role. */ +import { dirname } from "node:path"; +import z from "zod/v4"; +import { verifyControl } from "../build-artifacts/control-records"; +import { AnchoredDirectory } from "../build-artifacts/filesystem"; +import { parseCatalogPinSet } from "./generation"; +import { catalogProposalEnvelopePayloadSchema } from "./proposal-schemas"; +import { + catalogActivationRecordSchema, + catalogValidationRecordSchema, + currentCatalogValidationSchema, +} from "./review-schemas"; +export const CATALOG_VALIDATION_DOMAIN = "openmapx-catalog-validation-v1"; +export const CATALOG_ACTIVATION_DOMAIN = "openmapx-catalog-activation-v1"; +type RecordSource = { + dataRoot: string; + installationId: string; + dataManagerPublicKey: string; + agentPublicKey: string; +}; +export function withCatalogProposalDataDirectory( + source: Pick, + proposalId: string, + run: (directory: AnchoredDirectory) => T, +): T { + z.uuid().parse(proposalId); + const opened: AnchoredDirectory[] = []; + try { + let directory = new AnchoredDirectory(source.dataRoot); + opened.push(directory); + for (const name of ["motis", "proposals", proposalId]) { + directory = directory.directory(name, false, name !== "motis"); + opened.push(directory); + } + return run(directory); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +export function readCatalogValidationFile( + source: RecordSource, + proposalId: string, + validationId: string, +) { + z.uuid().parse(validationId); + const record = withCatalogProposalDataDirectory(source, proposalId, (directory) => + verifyControl( + source.dataManagerPublicKey, + CATALOG_VALIDATION_DOMAIN, + JSON.parse(directory.read(`validation-${validationId}.json`, 64 * 1024).toString("utf8")), + catalogValidationRecordSchema, + ), + ); + if ( + record.installationId !== source.installationId || + record.validation.proposalId !== proposalId || + record.validation.validationId !== validationId + ) + throw Error("Foreign catalog validation"); + return record; +} +export function readCatalogActivationFile( + source: RecordSource, + proposalId: string, + executionId: string, +) { + z.uuid().parse(executionId); + const record = withCatalogProposalDataDirectory(source, proposalId, (directory) => + verifyControl( + source.dataManagerPublicKey, + CATALOG_ACTIVATION_DOMAIN, + JSON.parse(directory.read(`activation-${executionId}.json`, 64 * 1024).toString("utf8")), + catalogActivationRecordSchema, + ), + ); + if ( + record.installationId !== source.installationId || + record.proposalId !== proposalId || + record.parent.executionId !== executionId + ) + throw Error("Foreign catalog activation"); + return record; +} +export function readCatalogProposalFile(source: RecordSource, proposalId: string) { + z.uuid().parse(proposalId); + const infra = new AnchoredDirectory(dirname(source.dataRoot)); + let pins: AnchoredDirectory | undefined, proposals: AnchoredDirectory | undefined; + try { + pins = infra.directory("catalog-pins", false, true); + proposals = pins.directory("proposals", false, true); + const record = verifyControl( + source.agentPublicKey, + "openmapx-catalog-proposal-v1", + JSON.parse(proposals.read(`${proposalId}.json`, 768 * 1024).toString("utf8")), + catalogProposalEnvelopePayloadSchema, + ); + if ( + record.installationId !== source.installationId || + record.proposal.proposalId !== proposalId + ) + throw Error("Foreign catalog proposal"); + parseCatalogPinSet(record.proposal.pins); + return record.proposal; + } finally { + proposals?.close(); + pins?.close(); + infra.close(); + } +} +export const CURRENT_CATALOG_VALIDATION_DOMAIN = "openmapx-current-catalog-validation-v1"; +export function readCurrentCatalogValidationFile(source: RecordSource) { + const root = new AnchoredDirectory(source.dataRoot); + let motis: AnchoredDirectory | undefined; + try { + motis = root.directory("motis"); + const record = verifyControl( + source.dataManagerPublicKey, + CURRENT_CATALOG_VALIDATION_DOMAIN, + JSON.parse(motis.read("catalog-validation-current.json", 8192).toString("utf8")), + currentCatalogValidationSchema, + ); + if (record.installationId !== source.installationId) + throw Error("Foreign current catalog validation"); + return record; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } finally { + motis?.close(); + root.close(); + } +} diff --git a/packages/core/src/catalog-pins/review-records.ts b/packages/core/src/catalog-pins/review-records.ts new file mode 100644 index 000000000..06b0f6488 --- /dev/null +++ b/packages/core/src/catalog-pins/review-records.ts @@ -0,0 +1,45 @@ +/** Fixed cross-role reads obtain paths and public keys from the installed nominal role. */ +import { type ControlInstallation, installationState } from "../build-artifacts/control-authority"; +import { + readCatalogActivationFile, + readCatalogValidationFile, + readCurrentCatalogValidationFile, + withCatalogProposalDataDirectory, +} from "./record-files"; + +export { CATALOG_ACTIVATION_DOMAIN, CATALOG_VALIDATION_DOMAIN } from "./record-files"; + +function source(installation: ControlInstallation) { + const state = installationState(installation); + return { + dataRoot: state.dataRoot, + installationId: installation.installationId, + dataManagerPublicKey: state.publicKeys["data-manager"], + agentPublicKey: state.publicKeys["ops-agent"], + }; +} +export function withCatalogProposalDirectory( + installation: ControlInstallation, + proposalId: string, + run: Parameters>[2], +): T { + return withCatalogProposalDataDirectory(source(installation), proposalId, run); +} +export function readCatalogValidation( + installation: ControlInstallation, + proposalId: string, + validationId: string, +) { + return readCatalogValidationFile(source(installation), proposalId, validationId); +} +export function readCatalogActivation( + installation: ControlInstallation, + proposalId: string, + executionId: string, +) { + return readCatalogActivationFile(source(installation), proposalId, executionId); +} + +export function readCurrentCatalogValidation(installation: ControlInstallation) { + return readCurrentCatalogValidationFile(source(installation)); +} diff --git a/packages/core/src/catalog-pins/review-schemas.ts b/packages/core/src/catalog-pins/review-schemas.ts new file mode 100644 index 000000000..36065b611 --- /dev/null +++ b/packages/core/src/catalog-pins/review-schemas.ts @@ -0,0 +1,147 @@ +/** Portable signed review evidence. A failed build never fabricates candidate evidence. */ +import z from "zod/v4"; +import { executionRefSchema } from "../ops/execution-ref"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/), + stamp = z.iso.datetime({ offset: true }); +export const catalogRuntimeInputsSchema = z.strictObject({ + configurationRevisionId: z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/), + credentialsRevision: digest, + sourcePolicyRevision: digest, + overlayDigest: digest, + imageDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + stagingState: z.enum(["running", "stopped", "created", "restarting", "paused", "unknown"]), + requiredKeys: z.array(z.string().min(1).max(1024)).max(1000), + configuredKeys: z.array(z.string().min(1).max(1024)).max(10000), +}); +/** Narrow only already pin/policy-verified required IDs to this immutable job's + * country scope. Configured IDs and the consumed mount revision remain exact. */ +export function scopeCatalogRuntimeInputs( + inputs: z.infer, + countries: readonly string[], +) { + return { + ...inputs, + requiredKeys: inputs.requiredKeys.filter( + (key) => + !countries.length || countries.includes(key.split("/")[0]!.split("-")[0]!.toLowerCase()), + ), + }; +} +const common = { + validationId: z.uuid(), + proposalId: z.uuid(), + pinDigest: digest, + diffDigest: digest, + sourcePolicyRevision: digest, + overlayDigest: digest, + credentialsRevision: digest, + countries: z.array(z.string().regex(/^[a-z]{2,3}$/)).max(256), + completedAt: stamp, + expiresAt: stamp, +}; +const candidate = { + slot: z.enum(["A", "B"]), + epoch: z.uuid(), + manifestHash: digest, + imageDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/), +}; +export const catalogValidationSchema = z.discriminatedUnion("status", [ + z.strictObject({ ...common, ...candidate, status: z.literal("passed") }), + z.strictObject({ + ...common, + status: z.literal("failed"), + reason: z.string().min(1).max(1024), + candidate: z.strictObject(candidate).optional(), + }), +]); +export const catalogValidationRecordSchema = z + .strictObject({ + version: z.literal(1), + installationId: z.uuid(), + sourceExecution: executionRefSchema, + proposalDigest: digest, + runtimeInputs: catalogRuntimeInputsSchema, + validation: catalogValidationSchema, + }) + .refine( + (record) => + record.sourceExecution.executor === "data-manager" && + record.sourceExecution.executionId === record.validation.validationId, + "Original validation execution required", + ); +export const catalogActivationRecordSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + parent: executionRefSchema, + proposalId: z.uuid(), + validationId: z.uuid(), + proposalDigest: digest, + validationDigest: digest, + diffDigest: digest, + runtimeInputs: catalogRuntimeInputsSchema, + promotion: z.strictObject({ + execution: executionRefSchema, + terminalReceiptDigest: digest, + adoption: z.strictObject({ + leaseId: z.uuid(), + delegationId: z.uuid(), + phase: z.literal("ready"), + }), + }), + approval: executionRefSchema, +}); +export type CatalogValidation = z.infer; +export type CatalogRuntimeInputs = z.infer; +/** Staging is intentionally stopped/recreated by validation and activation. All consumed inputs remain fixed. */ +export function sameCatalogRuntimeInputs( + before: CatalogRuntimeInputs, + after: CatalogRuntimeInputs, +): boolean { + const keys = [ + "configurationRevisionId", + "credentialsRevision", + "sourcePolicyRevision", + "overlayDigest", + "imageDigest", + ] as const; + return ( + keys.every((key) => before[key] === after[key]) && + JSON.stringify([...before.requiredKeys].sort()) === + JSON.stringify([...after.requiredKeys].sort()) && + JSON.stringify([...before.configuredKeys].sort()) === + JSON.stringify([...after.configuredKeys].sort()) + ); +} +/** Fixed host-source codecs; these validate retained bytes and grant no authority. */ +export const catalogApprovalAdmissionSchema = z + .object({ parent: z.unknown(), operation: z.unknown() }) + .passthrough(); +export const catalogPromotionEffectSchema = z + .object({ receipt: z.strictObject({ kind: z.literal("ops-result"), result: z.unknown() }) }) + .passthrough(); +const catalogArtifactSchema = z.strictObject({ + path: z.string().min(1).max(4096), + sha256: digest, + sizeBytes: z.number().int().nonnegative().safe(), +}); +export const catalogCandidateManifestSchema = z.object({ + epoch: z.uuid(), + artifacts: z.strictObject({ + config: catalogArtifactSchema, + license: catalogArtifactSchema, + proxyConfig: catalogArtifactSchema, + proxyVars: catalogArtifactSchema, + sourceManifest: catalogArtifactSchema, + sourceIndex: catalogArtifactSchema.optional(), + datasets: z.array(catalogArtifactSchema).min(1).max(16384), + }), +}); +export const currentCatalogValidationSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + sourceExecution: executionRefSchema, + selected: z + .strictObject({ proposalId: z.uuid(), validationId: z.uuid(), validationDigest: digest }) + .nullable(), +}); diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 5cbd171dc..3cc09cdee 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { createCatalogPinSet } from "../catalog-pins/generation"; import { authorizeOpsOperation, OPS_KIND_POLICIES, @@ -297,9 +298,27 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { "transitousLock.inspect": { kind: "transitousLock.inspect" }, "transitousLock.propose": { kind: "transitousLock.propose", - ref: `main@${"a".repeat(40)}`, - submodules: { "transitland-atlas": "b".repeat(40) }, - lockedBy: "fixture-actor", + proposalId: "00000000-0000-4000-8000-000000000001", + baseGenerationId: null, + pins: createCatalogPinSet( + { + ref: `main@${"a".repeat(40)}`, + submodules: { "transitland-atlas": "b".repeat(40) }, + lockedAt: issuedAt, + lockedBy: "fixture-actor", + }, + { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "c".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"c".repeat(40)}/systems.csv`, + sha256: "d".repeat(64), + lockedAt: issuedAt, + lockedBy: "fixture-actor", + }, + ), + diffDigest: "e".repeat(64), + sourcePolicyRevision: "f".repeat(64), }, "transitousLock.approve": { kind: "transitousLock.approve", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 9aaa51351..57ef71ae7 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -1,3 +1,9 @@ +import { + catalogProposalInputSchema, + catalogProposalSchema, + catalogProposalSummarySchema, +} from "../catalog-pins/proposal-schemas"; +import { catalogRuntimeInputsSchema } from "../catalog-pins/review-schemas"; import { catalogPinSetSchema } from "../catalog-pins/schemas"; import { platformConfigurationSchema } from "../services/platform-configuration"; import { @@ -263,7 +269,7 @@ export const OPS_KIND_POLICIES = { "postgis.capacity.inspect": sync("data-manager", 30_000), "transitCredentials.inspect": sync("api", 120_000, 768 * 1024), "catalogPins.inspect": sync("api", 30_000, 768 * 1024), - "transitousLock.inspect": sync("data-manager", 30_000), + "transitousLock.inspect": sync("data-manager", 30_000, 768 * 1024), "transitousLock.propose": sync("data-manager", 30_000), "transitousLock.approve": asyncEffect("data-manager", 120_000, 4096), "gbfsCatalogLock.inspect": sync("data-manager", 30_000), @@ -550,15 +556,17 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ kindOnly("catalogPins.inspect"), z.strictObject({ kind: z.literal("transitCredentials.inspect"), + proposalId: z.uuid().optional(), catalogRevisionId: z.string().regex(/^[a-f0-9]{64}$/), }), - kindOnly("transitousLock.inspect"), + z.strictObject({ + kind: z.literal("transitousLock.inspect"), + proposalId: z.uuid().optional(), + offset: z.number().int().min(0).max(4096).optional(), + }), z.strictObject({ kind: z.literal("transitousLock.propose"), - ref: transitousRefSchema, - submodules: transitousSubmodulesSchema, - lockedBy: actorLabelSchema, - comment: z.string().min(1).max(512).optional(), + ...catalogProposalInputSchema.shape, }), pairedPinApprovalInputSchema, kindOnly("gbfsCatalogLock.inspect"), @@ -1289,6 +1297,11 @@ export const OPS_RESULT_SCHEMAS = { "valhalla.traffic.refreshWaysToEdges": changed, "postgis.capacity.inspect": z.strictObject({ availableBytes: z.number().int().nonnegative() }), "transitCredentials.inspect": z.strictObject({ + consumedCredentialsRevision: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable() + .optional(), catalogRevisionId: z.string().regex(/^[a-f0-9]{64}$/), requiredKeys: z.array(z.string().min(1).max(1024)).max(1000), hostCredentialsRevision: z @@ -1320,8 +1333,23 @@ export const OPS_RESULT_SCHEMAS = { "transitousLock.inspect": z.strictObject({ active: transitousLockRecordSchema.nullable(), proposed: transitousLockRecordSchema.nullable(), + proposal: catalogProposalSchema.nullable().optional(), + validationInputs: catalogRuntimeInputsSchema.optional(), + validationBlocker: z.string().max(1024).optional(), + page: z + .strictObject({ + proposals: z.array(catalogProposalSummarySchema).max(100), + total: z.number().int().min(0).max(4096), + offset: z.number().int().min(0).max(4096), + nextOffset: z.number().int().min(0).max(4096).nullable(), + }) + .optional(), + }), + "transitousLock.propose": z.strictObject({ + proposalId: z.uuid(), + catalogRevisionId: z.string().regex(/^[a-f0-9]{64}$/), + proposed: z.literal(true), }), - "transitousLock.propose": z.strictObject({ ref: transitousRefSchema, proposed: z.literal(true) }), "transitousLock.approve": pairedPinApprovalResultSchema, "gbfsCatalogLock.inspect": z.strictObject({ commit: z.string().regex(/^[0-9a-f]{40}$/), diff --git a/packages/core/src/ops/feed-credential-map.ts b/packages/core/src/ops/feed-credential-map.ts index e4ffd5956..eb3d0c21f 100644 --- a/packages/core/src/ops/feed-credential-map.ts +++ b/packages/core/src/ops/feed-credential-map.ts @@ -14,3 +14,7 @@ export function canonicalFeedCredentialMap( throw Error("Feed credential map exceeds its format or budget"); } } + +/** Identity of no managed row, never a digest of a secret. */ +export const TRANSIT_CREDENTIALS_ABSENT_REVISION = + "7f1e896e62b3c13fa40bcc4df28f107dab28409a292d5796d1bc91388af92c07"; diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index e67652350..79a2b74a0 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -9,7 +9,10 @@ export * from "./control-retention-receipt"; export * from "./control-root-key"; export * from "./coordinator-lifecycle-records"; export * from "./execution-identity"; -export { canonicalFeedCredentialMap } from "./feed-credential-map"; +export { + canonicalFeedCredentialMap, + TRANSIT_CREDENTIALS_ABSENT_REVISION, +} from "./feed-credential-map"; export * from "./fingerprint"; export * from "./job-evidence"; export * from "./privacy-backup"; diff --git a/packages/core/src/services/data-manager/contracts/catalog-review.test.ts b/packages/core/src/services/data-manager/contracts/catalog-review.test.ts new file mode 100644 index 000000000..28c72aeda --- /dev/null +++ b/packages/core/src/services/data-manager/contracts/catalog-review.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { normalizeDataExecutionInput } from "./executions"; + +describe("catalog durable input identities", () => { + const proposalId = "00000000-0000-4000-8000-000000000001", + validationId = "00000000-0000-4000-8000-000000000002"; + it("retains exact proposal and normalized countries for validation", () => { + expect( + normalizeDataExecutionInput({ + kind: "catalog.validate", + proposalId, + countries: ["DE", "at", "de"], + }), + ).toEqual({ kind: "catalog.validate", proposalId, countries: ["at", "de"] }); + }); + it("activates only an exact reviewed pair/evidence identity with no raw refs or passed flags", () => { + const input = { + kind: "catalog.activate", + proposalId, + validationId, + expectedGenerationId: null, + diffDigest: "a".repeat(64), + acknowledgeLicenseChanges: true, + }; + expect(normalizeDataExecutionInput(input)).toEqual(input); + expect(() => normalizeDataExecutionInput({ ...input, passed: true })).toThrow(); + expect(() => normalizeDataExecutionInput({ ...input, ref: "main" })).toThrow(); + }); +}); diff --git a/packages/core/src/services/data-manager/contracts/catalog-review.ts b/packages/core/src/services/data-manager/contracts/catalog-review.ts new file mode 100644 index 000000000..7d28c476d --- /dev/null +++ b/packages/core/src/services/data-manager/contracts/catalog-review.ts @@ -0,0 +1,85 @@ +import z from "zod/v4"; +import { + catalogProposalSchema, + catalogProposalSummarySchema, +} from "../../../catalog-pins/proposal-schemas"; +import { + catalogRuntimeInputsSchema, + catalogValidationSchema, +} from "../../../catalog-pins/review-schemas"; +import { catalogPinSetSchema } from "../../../catalog-pins/schemas"; +import { dataExecutionStatusSchema } from "./executions"; + +const count = z.number().int().nonnegative().safe(); +export const catalogReviewQuerySchema = z.object({ + proposalId: z.uuid().optional(), + offset: z.coerce.number().int().min(0).max(4096).default(0), + diffOffset: z.coerce.number().int().min(0).max(200000).default(0), +}); +const change = z.strictObject({ + id: z.string(), + region: z.string(), + source: z.string(), + change: z.enum(["added", "removed", "changed"]), + fields: z.array( + z.enum(["endpoint", "origin", "license", "attribution", "eligibility", "metadata"]), + ), + licenseStatus: z.enum(["declared", "unknown"]), + before: z.record(z.string(), z.json()).nullable(), + after: z.record(z.string(), z.json()).nullable(), +}); +export const catalogReviewPageSchema = z.strictObject({ + digest: z.string().regex(/^[a-f0-9]{64}$/), + total: count, + offset: count, + rows: z.array(change).max(100), + nextOffset: count.nullable(), +}); +export const catalogReviewSnapshotSchema = z.strictObject({ + active: catalogPinSetSchema.nullable(), + allowedCountries: z.array(z.string().regex(/^[a-z]{2,3}$/)).max(256), + catalogRevisionId: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + page: z.strictObject({ + proposals: z.array(catalogProposalSummarySchema).max(100), + total: count, + offset: count, + nextOffset: count.nullable(), + }), + proposal: catalogProposalSchema.nullable(), + diff: catalogReviewPageSchema.nullable(), + validation: catalogValidationSchema.nullable(), + execution: dataExecutionStatusSchema.nullable(), + validationInputs: catalogRuntimeInputsSchema.nullable(), + validationEligible: z.boolean(), + blocker: z.string().max(2048).nullable(), +}); +export const catalogProposalCreateSchema = z.strictObject({ + branch: z + .string() + .min(1) + .max(215) + .regex(/^[A-Za-z0-9][A-Za-z0-9._/-]*$/) + .refine( + (value) => + !value.includes("..") && + !value.includes("@{") && + !value.endsWith("/") && + !value.endsWith(".lock"), + ) + .default("main"), +}); +export const catalogValidateRequestSchema = z.strictObject({ + countries: z.array(z.string().regex(/^[A-Za-z]{2,3}$/)).max(256), +}); +export const catalogActivateRequestSchema = z.strictObject({ + validationId: z.uuid(), + expectedGenerationId: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + diffDigest: z.string().regex(/^[a-f0-9]{64}$/), + acknowledgeLicenseChanges: z.boolean(), +}); diff --git a/packages/core/src/services/data-manager/contracts/executions.ts b/packages/core/src/services/data-manager/contracts/executions.ts index dc02b7596..b0690b8ac 100644 --- a/packages/core/src/services/data-manager/contracts/executions.ts +++ b/packages/core/src/services/data-manager/contracts/executions.ts @@ -15,6 +15,15 @@ const regional = (kind: K) => z.strictObject({ kind: z.literal(kind), region: regionSchema }); const countries = z.array(z.string().regex(/^[A-Za-z]{2,3}$/)).max(256); const ordinaryInputs = [ + z.strictObject({ kind: z.literal("catalog.validate"), proposalId: z.uuid(), countries }), + z.strictObject({ + kind: z.literal("catalog.activate"), + proposalId: z.uuid(), + validationId: z.uuid(), + expectedGenerationId: dataFingerprintSchema.nullable(), + diffDigest: dataFingerprintSchema, + acknowledgeLicenseChanges: z.boolean(), + }), regional("download-osm"), z.strictObject({ kind: z.literal("download-fonts") }), regional("convert-overpass"), @@ -69,7 +78,7 @@ export function normalizeDataExecutionInput(value: unknown): DataExecutionInput ), { statusCode: 413 }, ); - return input.kind === "transit-sync" + return input.kind === "transit-sync" || input.kind === "catalog.validate" ? { ...input, countries: [...new Set(input.countries.map((country) => country.toLowerCase()))].sort(), @@ -95,6 +104,21 @@ const metricsResult = (kind: K) => }); export const dataExecutionResultSchema = z .discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("catalog.validate"), + proposalId: z.uuid(), + validationId: z.uuid(), + status: z.enum(["passed", "failed"]), + expiresAt: z.iso.datetime({ offset: true }), + }), + z.strictObject({ + kind: z.literal("catalog.activate"), + proposalId: z.uuid(), + validationId: z.uuid(), + outcome: z.enum(["activated", "not_activated", "unknown"]), + generationId: dataFingerprintSchema.nullable(), + reason: z.string().max(1024).nullable(), + }), regional("download-osm").extend({ sizeBytes: dataEpochSchema, md5: z @@ -140,6 +164,8 @@ export const dataExecutionStatusSchema = z admittedAt: dataEpochSchema, deadlineAt: dataEpochSchema, kind: z.enum([ + "catalog.validate", + "catalog.activate", "download-osm", "download-fonts", "convert-overpass", diff --git a/packages/core/src/services/data-manager/contracts/index.ts b/packages/core/src/services/data-manager/contracts/index.ts index 2c586e71a..ba58ba4bb 100644 --- a/packages/core/src/services/data-manager/contracts/index.ts +++ b/packages/core/src/services/data-manager/contracts/index.ts @@ -1,4 +1,5 @@ export * from "./ambient"; +export * from "./catalog-review"; export * from "./common"; export * from "./configuration-maintenance"; export * from "./datasets"; diff --git a/packages/core/src/services/data-manager/contracts/operations.ts b/packages/core/src/services/data-manager/contracts/operations.ts index 849b0a711..558a84ae4 100644 --- a/packages/core/src/services/data-manager/contracts/operations.ts +++ b/packages/core/src/services/data-manager/contracts/operations.ts @@ -1,6 +1,7 @@ import z from "zod/v4"; import { dataManagerCoverageEvidencePageSchema } from "../../../coverage/schemas"; import * as a from "./ambient"; +import * as cr from "./catalog-review"; import { acceptedSchema, type DataManagerOperation, @@ -431,6 +432,13 @@ export const dataManagerOperations = { 503: json(httpErrorSchema), }, }, + inspectCatalogReview: { + method: "GET", + path: "/transit/pins", + auth: service, + request: { query: cr.catalogReviewQuerySchema }, + responses: { ...dataMutationErrors, 200: json(cr.catalogReviewSnapshotSchema) }, + }, proposeTransitBump: { method: "POST", path: "/transit/bump", diff --git a/packages/core/src/services/data-manager/contracts/transit.ts b/packages/core/src/services/data-manager/contracts/transit.ts index f1e58a5e2..67bb50455 100644 --- a/packages/core/src/services/data-manager/contracts/transit.ts +++ b/packages/core/src/services/data-manager/contracts/transit.ts @@ -241,6 +241,7 @@ const safeGitRef = z export const transitBumpRequestSchema = z .strictObject({ branch: safeGitRef.optional(), + proposalId: z.uuid().optional(), force: z.boolean().optional(), lockedBy: z.string().optional(), }) @@ -256,6 +257,8 @@ export const transitBumpResultSchema = z.union([ ok: z.literal(true), unchanged: z.literal(false), proposed: z.literal(true), + proposalId: z.uuid(), + catalogRevisionId: z.string().regex(/^[a-f0-9]{64}$/), ref: z.string(), previousRef: z.string().nullable(), submoduleSha: z.string(), diff --git a/packages/core/src/services/data-manager/generated/data-manager.generated.ts b/packages/core/src/services/data-manager/generated/data-manager.generated.ts index 8ba4dd45b..91c940bac 100644 --- a/packages/core/src/services/data-manager/generated/data-manager.generated.ts +++ b/packages/core/src/services/data-manager/generated/data-manager.generated.ts @@ -330,6 +330,16 @@ import type { IngestOverture500, IngestOverture503, IngestOvertureBody, + InspectCatalogReview200, + InspectCatalogReview400, + InspectCatalogReview401, + InspectCatalogReview409, + InspectCatalogReview413, + InspectCatalogReview415, + InspectCatalogReview429, + InspectCatalogReview500, + InspectCatalogReview503, + InspectCatalogReviewParams, InspectDataConfigurationFence200, InspectDataConfigurationFence400, InspectDataConfigurationFence401, @@ -580,6 +590,7 @@ import { headOfflineGlyphRequest, headOfflinePackageArchiveRequest, ingestOvertureRequest, + inspectCatalogReviewRequest, inspectDataConfigurationFenceRequest, inspectTransitSyncPreflightRequest, linkRequest, @@ -5702,6 +5713,98 @@ export const restartMotis = async ( }); }; +export type inspectCatalogReviewResponse200 = { + data: InspectCatalogReview200; + status: 200; +}; + +export type inspectCatalogReviewResponse400 = { + data: InspectCatalogReview400; + status: 400; +}; + +export type inspectCatalogReviewResponse401 = { + data: InspectCatalogReview401; + status: 401; +}; + +export type inspectCatalogReviewResponse409 = { + data: InspectCatalogReview409; + status: 409; +}; + +export type inspectCatalogReviewResponse413 = { + data: InspectCatalogReview413; + status: 413; +}; + +export type inspectCatalogReviewResponse415 = { + data: InspectCatalogReview415; + status: 415; +}; + +export type inspectCatalogReviewResponse429 = { + data: InspectCatalogReview429; + status: 429; +}; + +export type inspectCatalogReviewResponse500 = { + data: InspectCatalogReview500; + status: 500; +}; + +export type inspectCatalogReviewResponse503 = { + data: InspectCatalogReview503; + status: 503; +}; + +export type inspectCatalogReviewResponseSuccess = inspectCatalogReviewResponse200 & { + headers: Headers; +}; +export type inspectCatalogReviewResponseError = ( + | inspectCatalogReviewResponse400 + | inspectCatalogReviewResponse401 + | inspectCatalogReviewResponse409 + | inspectCatalogReviewResponse413 + | inspectCatalogReviewResponse415 + | inspectCatalogReviewResponse429 + | inspectCatalogReviewResponse500 + | inspectCatalogReviewResponse503 +) & { + headers: Headers; +}; + +export type inspectCatalogReviewResponse = + | inspectCatalogReviewResponseSuccess + | inspectCatalogReviewResponseError; + +export const getInspectCatalogReviewUrl = (params?: InspectCatalogReviewParams) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 ? `/transit/pins?${stringifiedParams}` : `/transit/pins`; +}; + +export const inspectCatalogReview = async ( + params?: InspectCatalogReviewParams, + options?: Parameters[1], +): Promise => { + return inspectCatalogReviewRequest( + getInspectCatalogReviewUrl(params), + { + ...options, + method: "GET", + }, + ); +}; + export type proposeTransitBumpResponse200 = { data: ProposeTransitBump200; status: 200; diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts index f3dd04605..fcb640613 100644 --- a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Kind.ts @@ -5,6 +5,8 @@ export type CancelDataExecution200Kind = (typeof CancelDataExecution200Kind)[keyof typeof CancelDataExecution200Kind]; export const CancelDataExecution200Kind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", "download-osm": "download-osm", "download-fonts": "download-fonts", "convert-overpass": "convert-overpass", diff --git a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts index 9704165ad..d09d3ab5a 100644 --- a/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts +++ b/packages/core/src/services/data-manager/generated/models/cancelDataExecution200Result.ts @@ -2,6 +2,26 @@ // Generated by pnpm data-manager:sdk:generate. Do not edit. export type CancelDataExecution200Result = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } | { kind: "download-osm"; /** diff --git a/packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema0.ts b/packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema0.ts new file mode 100644 index 000000000..1548e1cde --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema0.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DataManagerInspectCatalogReviewResponse200Schema0 = + | string + | number + | boolean + | null + | (DataManagerInspectCatalogReviewResponse200Schema0 | null)[] + | { [key: string]: DataManagerInspectCatalogReviewResponse200Schema0 | null }; diff --git a/packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema1.ts b/packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema1.ts new file mode 100644 index 000000000..4f1bcf1a0 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/dataManagerInspectCatalogReviewResponse200Schema1.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type DataManagerInspectCatalogReviewResponse200Schema1 = + | string + | number + | boolean + | null + | (DataManagerInspectCatalogReviewResponse200Schema1 | null)[] + | { [key: string]: DataManagerInspectCatalogReviewResponse200Schema1 | null }; diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts index f432111b3..f489a5565 100644 --- a/packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200Kind.ts @@ -5,6 +5,8 @@ export type GetDataExecution200Kind = (typeof GetDataExecution200Kind)[keyof typeof GetDataExecution200Kind]; export const GetDataExecution200Kind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", "download-osm": "download-osm", "download-fonts": "download-fonts", "convert-overpass": "convert-overpass", diff --git a/packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts b/packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts index cae13c06e..5104813d8 100644 --- a/packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts +++ b/packages/core/src/services/data-manager/generated/models/getDataExecution200Result.ts @@ -2,6 +2,26 @@ // Generated by pnpm data-manager:sdk:generate. Do not edit. export type GetDataExecution200Result = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } | { kind: "download-osm"; /** diff --git a/packages/core/src/services/data-manager/generated/models/index.ts b/packages/core/src/services/data-manager/generated/models/index.ts index 2f5ca7d3c..e2830a192 100644 --- a/packages/core/src/services/data-manager/generated/models/index.ts +++ b/packages/core/src/services/data-manager/generated/models/index.ts @@ -107,6 +107,8 @@ export * from "./convertOverpass503"; export * from "./convertOverpassBody"; export * from "./dataManagerGetTransitCatalogResponse200Schema0"; export * from "./dataManagerGetTransitSourcesResponse200Schema0"; +export * from "./dataManagerInspectCatalogReviewResponse200Schema0"; +export * from "./dataManagerInspectCatalogReviewResponse200Schema1"; export * from "./disableTransitSource202"; export * from "./disableTransitSource202Status"; export * from "./disableTransitSource400"; @@ -464,6 +466,53 @@ export * from "./ingestOverture429"; export * from "./ingestOverture500"; export * from "./ingestOverture503"; export * from "./ingestOvertureBody"; +export * from "./inspectCatalogReview200"; +export * from "./inspectCatalogReview200Active"; +export * from "./inspectCatalogReview200ActiveGbfs"; +export * from "./inspectCatalogReview200ActiveGbfsSchemaVersion"; +export * from "./inspectCatalogReview200ActiveGbfsSource"; +export * from "./inspectCatalogReview200ActiveSchemaVersion"; +export * from "./inspectCatalogReview200ActiveTransitous"; +export * from "./inspectCatalogReview200ActiveTransitousSubmodules"; +export * from "./inspectCatalogReview200Diff"; +export * from "./inspectCatalogReview200DiffRowsItem"; +export * from "./inspectCatalogReview200DiffRowsItemAfter"; +export * from "./inspectCatalogReview200DiffRowsItemBefore"; +export * from "./inspectCatalogReview200DiffRowsItemChange"; +export * from "./inspectCatalogReview200DiffRowsItemFieldsItem"; +export * from "./inspectCatalogReview200DiffRowsItemLicenseStatus"; +export * from "./inspectCatalogReview200Execution"; +export * from "./inspectCatalogReview200ExecutionDiagnostics"; +export * from "./inspectCatalogReview200ExecutionDiagnosticsTailItem"; +export * from "./inspectCatalogReview200ExecutionDiagnosticsTailItemStream"; +export * from "./inspectCatalogReview200ExecutionKind"; +export * from "./inspectCatalogReview200ExecutionPublication"; +export * from "./inspectCatalogReview200ExecutionResult"; +export * from "./inspectCatalogReview200ExecutionStatus"; +export * from "./inspectCatalogReview200Page"; +export * from "./inspectCatalogReview200PageProposalsItem"; +export * from "./inspectCatalogReview200PageProposalsItemState"; +export * from "./inspectCatalogReview200Proposal"; +export * from "./inspectCatalogReview200ProposalPins"; +export * from "./inspectCatalogReview200ProposalPinsGbfs"; +export * from "./inspectCatalogReview200ProposalPinsGbfsSchemaVersion"; +export * from "./inspectCatalogReview200ProposalPinsGbfsSource"; +export * from "./inspectCatalogReview200ProposalPinsSchemaVersion"; +export * from "./inspectCatalogReview200ProposalPinsTransitous"; +export * from "./inspectCatalogReview200ProposalPinsTransitousSubmodules"; +export * from "./inspectCatalogReview200ProposalState"; +export * from "./inspectCatalogReview200Validation"; +export * from "./inspectCatalogReview200ValidationInputs"; +export * from "./inspectCatalogReview200ValidationInputsStagingState"; +export * from "./inspectCatalogReview400"; +export * from "./inspectCatalogReview401"; +export * from "./inspectCatalogReview409"; +export * from "./inspectCatalogReview413"; +export * from "./inspectCatalogReview415"; +export * from "./inspectCatalogReview429"; +export * from "./inspectCatalogReview500"; +export * from "./inspectCatalogReview503"; +export * from "./inspectCatalogReviewParams"; export * from "./inspectDataConfigurationFence200"; export * from "./inspectDataConfigurationFence200OwnerPrepare"; export * from "./inspectDataConfigurationFence200OwnerPrepareExecutor"; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200.ts new file mode 100644 index 000000000..275646405 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200.ts @@ -0,0 +1,27 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200Active } from "./inspectCatalogReview200Active"; +import type { InspectCatalogReview200Diff } from "./inspectCatalogReview200Diff"; +import type { InspectCatalogReview200Execution } from "./inspectCatalogReview200Execution"; +import type { InspectCatalogReview200Page } from "./inspectCatalogReview200Page"; +import type { InspectCatalogReview200Proposal } from "./inspectCatalogReview200Proposal"; +import type { InspectCatalogReview200Validation } from "./inspectCatalogReview200Validation"; +import type { InspectCatalogReview200ValidationInputs } from "./inspectCatalogReview200ValidationInputs"; + +export type InspectCatalogReview200 = { + active: InspectCatalogReview200Active; + /** + * @maxItems 256 + * @items.pattern ^[a-z]{2,3}$ + */ + allowedCountries: string[]; + catalogRevisionId: string | null; + page: InspectCatalogReview200Page; + proposal: InspectCatalogReview200Proposal; + diff: InspectCatalogReview200Diff; + validation: InspectCatalogReview200Validation; + execution: InspectCatalogReview200Execution; + validationInputs: InspectCatalogReview200ValidationInputs; + validationEligible: boolean; + blocker: string | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Active.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Active.ts new file mode 100644 index 000000000..6f070d379 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Active.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ActiveGbfs } from "./inspectCatalogReview200ActiveGbfs"; +import type { InspectCatalogReview200ActiveSchemaVersion } from "./inspectCatalogReview200ActiveSchemaVersion"; +import type { InspectCatalogReview200ActiveTransitous } from "./inspectCatalogReview200ActiveTransitous"; + +export type InspectCatalogReview200Active = { + schemaVersion: InspectCatalogReview200ActiveSchemaVersion; + /** @pattern ^[a-f0-9]{64}$ */ + generationId: string; + transitous: InspectCatalogReview200ActiveTransitous; + gbfs: InspectCatalogReview200ActiveGbfs; +} | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfs.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfs.ts new file mode 100644 index 000000000..f8fcb0ee0 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfs.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ActiveGbfsSchemaVersion } from "./inspectCatalogReview200ActiveGbfsSchemaVersion"; +import type { InspectCatalogReview200ActiveGbfsSource } from "./inspectCatalogReview200ActiveGbfsSource"; + +export type InspectCatalogReview200ActiveGbfs = { + schemaVersion: InspectCatalogReview200ActiveGbfsSchemaVersion; + source: InspectCatalogReview200ActiveGbfsSource; + /** @pattern ^[a-f0-9]{40}$ */ + commit: string; + /** @maxLength 256 */ + url: string; + /** @pattern ^[a-f0-9]{64}$ */ + sha256: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSchemaVersion.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSchemaVersion.ts new file mode 100644 index 000000000..684903f70 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ActiveGbfsSchemaVersion = + (typeof InspectCatalogReview200ActiveGbfsSchemaVersion)[keyof typeof InspectCatalogReview200ActiveGbfsSchemaVersion]; + +export const InspectCatalogReview200ActiveGbfsSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSource.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSource.ts new file mode 100644 index 000000000..95a8ee351 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveGbfsSource.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ActiveGbfsSource = + (typeof InspectCatalogReview200ActiveGbfsSource)[keyof typeof InspectCatalogReview200ActiveGbfsSource]; + +export const InspectCatalogReview200ActiveGbfsSource = { + "mobilitydata-gbfs": "mobilitydata-gbfs", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveSchemaVersion.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveSchemaVersion.ts new file mode 100644 index 000000000..d6e8b307b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ActiveSchemaVersion = + (typeof InspectCatalogReview200ActiveSchemaVersion)[keyof typeof InspectCatalogReview200ActiveSchemaVersion]; + +export const InspectCatalogReview200ActiveSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitous.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitous.ts new file mode 100644 index 000000000..addc5a9a1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitous.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ActiveTransitousSubmodules } from "./inspectCatalogReview200ActiveTransitousSubmodules"; + +export type InspectCatalogReview200ActiveTransitous = { + /** + * @maxLength 256 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$ + */ + ref: string; + submodules: InspectCatalogReview200ActiveTransitousSubmodules; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; + /** @maxLength 512 */ + comment?: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitousSubmodules.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitousSubmodules.ts new file mode 100644 index 000000000..ea9fd94d1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ActiveTransitousSubmodules.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ActiveTransitousSubmodules = { [key: string]: string }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Diff.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Diff.ts new file mode 100644 index 000000000..0dd071a2b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Diff.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200DiffRowsItem } from "./inspectCatalogReview200DiffRowsItem"; + +export type InspectCatalogReview200Diff = { + /** @pattern ^[a-f0-9]{64}$ */ + digest: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + total: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + offset: number; + /** @maxItems 100 */ + rows: InspectCatalogReview200DiffRowsItem[]; + nextOffset: number | null; +} | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItem.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItem.ts new file mode 100644 index 000000000..41b38f596 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItem.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200DiffRowsItemAfter } from "./inspectCatalogReview200DiffRowsItemAfter"; +import type { InspectCatalogReview200DiffRowsItemBefore } from "./inspectCatalogReview200DiffRowsItemBefore"; +import type { InspectCatalogReview200DiffRowsItemChange } from "./inspectCatalogReview200DiffRowsItemChange"; +import type { InspectCatalogReview200DiffRowsItemFieldsItem } from "./inspectCatalogReview200DiffRowsItemFieldsItem"; +import type { InspectCatalogReview200DiffRowsItemLicenseStatus } from "./inspectCatalogReview200DiffRowsItemLicenseStatus"; + +export type InspectCatalogReview200DiffRowsItem = { + id: string; + region: string; + source: string; + change: InspectCatalogReview200DiffRowsItemChange; + fields: InspectCatalogReview200DiffRowsItemFieldsItem[]; + licenseStatus: InspectCatalogReview200DiffRowsItemLicenseStatus; + before: InspectCatalogReview200DiffRowsItemBefore; + after: InspectCatalogReview200DiffRowsItemAfter; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemAfter.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemAfter.ts new file mode 100644 index 000000000..c97d7072e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemAfter.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { DataManagerInspectCatalogReviewResponse200Schema1 } from "./dataManagerInspectCatalogReviewResponse200Schema1"; + +export type InspectCatalogReview200DiffRowsItemAfter = { + [key: string]: DataManagerInspectCatalogReviewResponse200Schema1 | null; +} | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemBefore.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemBefore.ts new file mode 100644 index 000000000..3c4f8d298 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemBefore.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { DataManagerInspectCatalogReviewResponse200Schema0 } from "./dataManagerInspectCatalogReviewResponse200Schema0"; + +export type InspectCatalogReview200DiffRowsItemBefore = { + [key: string]: DataManagerInspectCatalogReviewResponse200Schema0 | null; +} | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemChange.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemChange.ts new file mode 100644 index 000000000..0ffc59054 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemChange.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200DiffRowsItemChange = + (typeof InspectCatalogReview200DiffRowsItemChange)[keyof typeof InspectCatalogReview200DiffRowsItemChange]; + +export const InspectCatalogReview200DiffRowsItemChange = { + added: "added", + removed: "removed", + changed: "changed", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemFieldsItem.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemFieldsItem.ts new file mode 100644 index 000000000..cea890d53 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemFieldsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200DiffRowsItemFieldsItem = + (typeof InspectCatalogReview200DiffRowsItemFieldsItem)[keyof typeof InspectCatalogReview200DiffRowsItemFieldsItem]; + +export const InspectCatalogReview200DiffRowsItemFieldsItem = { + endpoint: "endpoint", + origin: "origin", + license: "license", + attribution: "attribution", + eligibility: "eligibility", + metadata: "metadata", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemLicenseStatus.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemLicenseStatus.ts new file mode 100644 index 000000000..1489a8c12 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200DiffRowsItemLicenseStatus.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200DiffRowsItemLicenseStatus = + (typeof InspectCatalogReview200DiffRowsItemLicenseStatus)[keyof typeof InspectCatalogReview200DiffRowsItemLicenseStatus]; + +export const InspectCatalogReview200DiffRowsItemLicenseStatus = { + declared: "declared", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Execution.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Execution.ts new file mode 100644 index 000000000..0b8fe735c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Execution.ts @@ -0,0 +1,33 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ExecutionDiagnostics } from "./inspectCatalogReview200ExecutionDiagnostics"; +import type { InspectCatalogReview200ExecutionKind } from "./inspectCatalogReview200ExecutionKind"; +import type { InspectCatalogReview200ExecutionPublication } from "./inspectCatalogReview200ExecutionPublication"; +import type { InspectCatalogReview200ExecutionResult } from "./inspectCatalogReview200ExecutionResult"; +import type { InspectCatalogReview200ExecutionStatus } from "./inspectCatalogReview200ExecutionStatus"; + +export type InspectCatalogReview200Execution = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + id: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + admittedAt: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; + kind: InspectCatalogReview200ExecutionKind; + status: InspectCatalogReview200ExecutionStatus; + cancellationRequestedAt: string | null; + progress: number | null; + diagnostics: InspectCatalogReview200ExecutionDiagnostics; + publication: InspectCatalogReview200ExecutionPublication; + result?: InspectCatalogReview200ExecutionResult; +} | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnostics.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnostics.ts new file mode 100644 index 000000000..f7db80219 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnostics.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ExecutionDiagnosticsTailItem } from "./inspectCatalogReview200ExecutionDiagnosticsTailItem"; + +export type InspectCatalogReview200ExecutionDiagnostics = { + /** @maxItems 256 */ + tail: InspectCatalogReview200ExecutionDiagnosticsTailItem[]; + truncated: boolean; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItem.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItem.ts new file mode 100644 index 000000000..26729927f --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ExecutionDiagnosticsTailItemStream } from "./inspectCatalogReview200ExecutionDiagnosticsTailItemStream"; + +export type InspectCatalogReview200ExecutionDiagnosticsTailItem = { + stream: InspectCatalogReview200ExecutionDiagnosticsTailItemStream; + message: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItemStream.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItemStream.ts new file mode 100644 index 000000000..f21c61a1a --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionDiagnosticsTailItemStream.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ExecutionDiagnosticsTailItemStream = + (typeof InspectCatalogReview200ExecutionDiagnosticsTailItemStream)[keyof typeof InspectCatalogReview200ExecutionDiagnosticsTailItemStream]; + +export const InspectCatalogReview200ExecutionDiagnosticsTailItemStream = { + stdout: "stdout", + stderr: "stderr", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionKind.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionKind.ts new file mode 100644 index 000000000..6541bd202 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionKind.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ExecutionKind = + (typeof InspectCatalogReview200ExecutionKind)[keyof typeof InspectCatalogReview200ExecutionKind]; + +export const InspectCatalogReview200ExecutionKind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", + "download-osm": "download-osm", + "download-fonts": "download-fonts", + "convert-overpass": "convert-overpass", + "transit-sync": "transit-sync", + "overture-pull": "overture-pull", + "overture-ingest": "overture-ingest", + "overture-extract": "overture-extract", + "overture-conflate": "overture-conflate", + "overture-sync": "overture-sync", + "search-index-build": "search-index-build", + "notable-places-build": "notable-places-build", + "generation-hardlinks": "generation-hardlinks", + "transit-maintenance": "transit-maintenance", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionPublication.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionPublication.ts new file mode 100644 index 000000000..8838088b6 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionPublication.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ExecutionPublication = + (typeof InspectCatalogReview200ExecutionPublication)[keyof typeof InspectCatalogReview200ExecutionPublication]; + +export const InspectCatalogReview200ExecutionPublication = { + not_started: "not_started", + committed: "committed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionResult.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionResult.ts new file mode 100644 index 000000000..2032f168a --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionResult.ts @@ -0,0 +1,375 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ExecutionResult = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } + | { + kind: "download-osm"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^[a-f0-9]{32}$ */ + md5?: string; + } + | { + kind: "download-fonts"; + completed: true; + } + | { + kind: "convert-overpass"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + } + | { + kind: "transit-sync"; + finalStatus: "ok" | "partial" | "error"; + /** + * @minLength 1 + * @maxLength 512 + */ + sourceRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "overture-pull"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-ingest"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-extract"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-conflate"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "overture-sync"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + /** + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + release?: string; + } + | { + kind: "search-index-build"; + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "notable-places-build"; + metrics: { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + processed?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + inserted?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + updated?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped?: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes?: number; + }; + } + | { + kind: "generation-hardlinks"; + hardlinks: { + applied: boolean; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + linked: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + skipped: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + pruned: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + entries: number; + via?: "data-manager" | "local" | "none"; + }; + } + | { + kind: "transit-maintenance"; + action: "restart-primary" | "reload-feed-proxy"; + changed: true; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionStatus.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionStatus.ts new file mode 100644 index 000000000..39e1ef530 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ExecutionStatus.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ExecutionStatus = + (typeof InspectCatalogReview200ExecutionStatus)[keyof typeof InspectCatalogReview200ExecutionStatus]; + +export const InspectCatalogReview200ExecutionStatus = { + queued: "queued", + running: "running", + cancel_pending: "cancel_pending", + ok: "ok", + partial: "partial", + error: "error", + canceled: "canceled", + interrupted: "interrupted", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Page.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Page.ts new file mode 100644 index 000000000..311a7f388 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Page.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200PageProposalsItem } from "./inspectCatalogReview200PageProposalsItem"; + +export type InspectCatalogReview200Page = { + /** @maxItems 100 */ + proposals: InspectCatalogReview200PageProposalsItem[]; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + total: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + offset: number; + nextOffset: number | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItem.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItem.ts new file mode 100644 index 000000000..55c965d68 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItem.ts @@ -0,0 +1,24 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200PageProposalsItemState } from "./inspectCatalogReview200PageProposalsItemState"; + +export type InspectCatalogReview200PageProposalsItem = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + baseGenerationId: string | null; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + state: InspectCatalogReview200PageProposalsItemState; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + createdAt: string; + /** @pattern ^[a-f0-9]{64}$ */ + catalogRevisionId: string; + /** @maxLength 256 */ + transitousRef: string; + /** @pattern ^[a-f0-9]{40}$ */ + atlasCommit: string; + /** @pattern ^[a-f0-9]{40}$ */ + gbfsCommit: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItemState.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItemState.ts new file mode 100644 index 000000000..cb43f25a1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200PageProposalsItemState.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200PageProposalsItemState = + (typeof InspectCatalogReview200PageProposalsItemState)[keyof typeof InspectCatalogReview200PageProposalsItemState]; + +export const InspectCatalogReview200PageProposalsItemState = { + proposed: "proposed", + validating: "validating", + validated: "validated", + rejected: "rejected", + activating: "activating", + activated: "activated", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Proposal.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Proposal.ts new file mode 100644 index 000000000..d37595d5d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Proposal.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ProposalPins } from "./inspectCatalogReview200ProposalPins"; +import type { InspectCatalogReview200ProposalState } from "./inspectCatalogReview200ProposalState"; + +export type InspectCatalogReview200Proposal = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + baseGenerationId: string | null; + pins: InspectCatalogReview200ProposalPins; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + state: InspectCatalogReview200ProposalState; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + createdAt: string; +} | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPins.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPins.ts new file mode 100644 index 000000000..473059e03 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPins.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ProposalPinsGbfs } from "./inspectCatalogReview200ProposalPinsGbfs"; +import type { InspectCatalogReview200ProposalPinsSchemaVersion } from "./inspectCatalogReview200ProposalPinsSchemaVersion"; +import type { InspectCatalogReview200ProposalPinsTransitous } from "./inspectCatalogReview200ProposalPinsTransitous"; + +export type InspectCatalogReview200ProposalPins = { + schemaVersion: InspectCatalogReview200ProposalPinsSchemaVersion; + /** @pattern ^[a-f0-9]{64}$ */ + generationId: string; + transitous: InspectCatalogReview200ProposalPinsTransitous; + gbfs: InspectCatalogReview200ProposalPinsGbfs; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfs.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfs.ts new file mode 100644 index 000000000..36e8393fd --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfs.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ProposalPinsGbfsSchemaVersion } from "./inspectCatalogReview200ProposalPinsGbfsSchemaVersion"; +import type { InspectCatalogReview200ProposalPinsGbfsSource } from "./inspectCatalogReview200ProposalPinsGbfsSource"; + +export type InspectCatalogReview200ProposalPinsGbfs = { + schemaVersion: InspectCatalogReview200ProposalPinsGbfsSchemaVersion; + source: InspectCatalogReview200ProposalPinsGbfsSource; + /** @pattern ^[a-f0-9]{40}$ */ + commit: string; + /** @maxLength 256 */ + url: string; + /** @pattern ^[a-f0-9]{64}$ */ + sha256: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSchemaVersion.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSchemaVersion.ts new file mode 100644 index 000000000..33ff02f7c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ProposalPinsGbfsSchemaVersion = + (typeof InspectCatalogReview200ProposalPinsGbfsSchemaVersion)[keyof typeof InspectCatalogReview200ProposalPinsGbfsSchemaVersion]; + +export const InspectCatalogReview200ProposalPinsGbfsSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSource.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSource.ts new file mode 100644 index 000000000..1c94b798a --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsGbfsSource.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ProposalPinsGbfsSource = + (typeof InspectCatalogReview200ProposalPinsGbfsSource)[keyof typeof InspectCatalogReview200ProposalPinsGbfsSource]; + +export const InspectCatalogReview200ProposalPinsGbfsSource = { + "mobilitydata-gbfs": "mobilitydata-gbfs", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsSchemaVersion.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsSchemaVersion.ts new file mode 100644 index 000000000..740334f79 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsSchemaVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ProposalPinsSchemaVersion = + (typeof InspectCatalogReview200ProposalPinsSchemaVersion)[keyof typeof InspectCatalogReview200ProposalPinsSchemaVersion]; + +export const InspectCatalogReview200ProposalPinsSchemaVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitous.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitous.ts new file mode 100644 index 000000000..cbe2193e9 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitous.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ProposalPinsTransitousSubmodules } from "./inspectCatalogReview200ProposalPinsTransitousSubmodules"; + +export type InspectCatalogReview200ProposalPinsTransitous = { + /** + * @maxLength 256 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$ + */ + ref: string; + submodules: InspectCatalogReview200ProposalPinsTransitousSubmodules; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + lockedAt: string; + /** + * @minLength 1 + * @maxLength 128 + */ + lockedBy: string; + /** @maxLength 512 */ + comment?: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitousSubmodules.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitousSubmodules.ts new file mode 100644 index 000000000..af17aab75 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalPinsTransitousSubmodules.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ProposalPinsTransitousSubmodules = { [key: string]: string }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalState.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalState.ts new file mode 100644 index 000000000..229c79f9a --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ProposalState.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ProposalState = + (typeof InspectCatalogReview200ProposalState)[keyof typeof InspectCatalogReview200ProposalState]; + +export const InspectCatalogReview200ProposalState = { + proposed: "proposed", + validating: "validating", + validated: "validated", + rejected: "rejected", + activating: "activating", + activated: "activated", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Validation.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Validation.ts new file mode 100644 index 000000000..7aa052603 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200Validation.ts @@ -0,0 +1,78 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200Validation = + | { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^[a-f0-9]{64}$ */ + pinDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + overlayDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[a-z]{2,3}$ + */ + countries: string[]; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + completedAt: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + slot: "A" | "B"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + epoch: string; + /** @pattern ^[a-f0-9]{64}$ */ + manifestHash: string; + /** @pattern ^sha256:[a-f0-9]{64}$ */ + imageDigest: string; + status: "passed"; + } + | { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^[a-f0-9]{64}$ */ + pinDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + overlayDigest: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + /** + * @maxItems 256 + * @items.pattern ^[a-z]{2,3}$ + */ + countries: string[]; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + completedAt: string; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + status: "failed"; + /** + * @minLength 1 + * @maxLength 1024 + */ + reason: string; + candidate?: { + slot: "A" | "B"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + epoch: string; + /** @pattern ^[a-f0-9]{64}$ */ + manifestHash: string; + /** @pattern ^sha256:[a-f0-9]{64}$ */ + imageDigest: string; + }; + } + | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputs.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputs.ts new file mode 100644 index 000000000..a74b370eb --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputs.ts @@ -0,0 +1,29 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { InspectCatalogReview200ValidationInputsStagingState } from "./inspectCatalogReview200ValidationInputsStagingState"; + +export type InspectCatalogReview200ValidationInputs = { + /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ + configurationRevisionId: string; + /** @pattern ^[a-f0-9]{64}$ */ + credentialsRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + sourcePolicyRevision: string; + /** @pattern ^[a-f0-9]{64}$ */ + overlayDigest: string; + /** @pattern ^sha256:[a-f0-9]{64}$ */ + imageDigest: string; + stagingState: InspectCatalogReview200ValidationInputsStagingState; + /** + * @maxItems 1000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + requiredKeys: string[]; + /** + * @maxItems 10000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + configuredKeys: string[]; +} | null; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputsStagingState.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputsStagingState.ts new file mode 100644 index 000000000..ea5380cbe --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview200ValidationInputsStagingState.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview200ValidationInputsStagingState = + (typeof InspectCatalogReview200ValidationInputsStagingState)[keyof typeof InspectCatalogReview200ValidationInputsStagingState]; + +export const InspectCatalogReview200ValidationInputsStagingState = { + running: "running", + stopped: "stopped", + created: "created", + restarting: "restarting", + paused: "paused", + unknown: "unknown", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview400.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview400.ts new file mode 100644 index 000000000..c5df40c1d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview401.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview401.ts new file mode 100644 index 000000000..8dcecb559 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview409.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview409.ts new file mode 100644 index 000000000..04f45072c --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview413.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview413.ts new file mode 100644 index 000000000..a3bb895c8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview415.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview415.ts new file mode 100644 index 000000000..a4d5e37bd --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview429.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview429.ts new file mode 100644 index 000000000..12f6ad0dc --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview500.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview500.ts new file mode 100644 index 000000000..318a94c6b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReview503.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview503.ts new file mode 100644 index 000000000..bfeab11a6 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReview503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReview503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/inspectCatalogReviewParams.ts b/packages/core/src/services/data-manager/generated/models/inspectCatalogReviewParams.ts new file mode 100644 index 000000000..c15e9deb6 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectCatalogReviewParams.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectCatalogReviewParams = { + /** + * @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + */ + proposalId?: string; + /** + * @minimum 0 + * @maximum 4096 + */ + offset?: number; + /** + * @minimum 0 + * @maximum 200000 + */ + diffOffset?: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts index df4b40d12..801692e8c 100644 --- a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Kind.ts @@ -5,6 +5,8 @@ export type LookupDataExecution200Kind = (typeof LookupDataExecution200Kind)[keyof typeof LookupDataExecution200Kind]; export const LookupDataExecution200Kind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", "download-osm": "download-osm", "download-fonts": "download-fonts", "convert-overpass": "convert-overpass", diff --git a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts index 84fdcb84d..240022004 100644 --- a/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts +++ b/packages/core/src/services/data-manager/generated/models/lookupDataExecution200Result.ts @@ -2,6 +2,26 @@ // Generated by pnpm data-manager:sdk:generate. Do not edit. export type LookupDataExecution200Result = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } | { kind: "download-osm"; /** diff --git a/packages/core/src/services/data-manager/generated/models/proposeTransitBump200.ts b/packages/core/src/services/data-manager/generated/models/proposeTransitBump200.ts index e10db0298..1db429581 100644 --- a/packages/core/src/services/data-manager/generated/models/proposeTransitBump200.ts +++ b/packages/core/src/services/data-manager/generated/models/proposeTransitBump200.ts @@ -13,6 +13,10 @@ export type ProposeTransitBump200 = ok: true; unchanged: false; proposed: true; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^[a-f0-9]{64}$ */ + catalogRevisionId: string; ref: string; /** @nullable */ previousRef: string | null; diff --git a/packages/core/src/services/data-manager/generated/models/proposeTransitBumpBody.ts b/packages/core/src/services/data-manager/generated/models/proposeTransitBumpBody.ts index 2df606658..121990d04 100644 --- a/packages/core/src/services/data-manager/generated/models/proposeTransitBumpBody.ts +++ b/packages/core/src/services/data-manager/generated/models/proposeTransitBumpBody.ts @@ -4,6 +4,8 @@ export type ProposeTransitBumpBody = { /** @pattern ^[A-Za-z0-9][A-Za-z0-9._/-]*$ */ branch?: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId?: string; force?: boolean; lockedBy?: string; }; diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts index ae2e69451..fdf562980 100644 --- a/packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202Kind.ts @@ -5,6 +5,8 @@ export type StartDataExecution202Kind = (typeof StartDataExecution202Kind)[keyof typeof StartDataExecution202Kind]; export const StartDataExecution202Kind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", "download-osm": "download-osm", "download-fonts": "download-fonts", "convert-overpass": "convert-overpass", diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts b/packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts index d907d3531..88c12fd0c 100644 --- a/packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts +++ b/packages/core/src/services/data-manager/generated/models/startDataExecution202Result.ts @@ -2,6 +2,26 @@ // Generated by pnpm data-manager:sdk:generate. Do not edit. export type StartDataExecution202Result = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } | { kind: "download-osm"; /** diff --git a/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts index ab88a5345..cddb78f53 100644 --- a/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts +++ b/packages/core/src/services/data-manager/generated/models/startDataExecutionBodyInput.ts @@ -2,6 +2,27 @@ // Generated by pnpm data-manager:sdk:generate. Do not edit. export type StartDataExecutionBodyInput = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** + * @maxItems 256 + * @items.pattern ^[A-Za-z]{2,3}$ + */ + countries: string[]; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + expectedGenerationId: string | null; + /** @pattern ^[a-f0-9]{64}$ */ + diffDigest: string; + acknowledgeLicenseChanges: boolean; + } | { kind: "download-osm"; /** diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts index 59eca469b..7089eae84 100644 --- a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Kind.ts @@ -5,6 +5,8 @@ export type StartGenerationHardlinksExecution202Kind = (typeof StartGenerationHardlinksExecution202Kind)[keyof typeof StartGenerationHardlinksExecution202Kind]; export const StartGenerationHardlinksExecution202Kind = { + catalogvalidate: "catalog.validate", + catalogactivate: "catalog.activate", "download-osm": "download-osm", "download-fonts": "download-fonts", "convert-overpass": "convert-overpass", diff --git a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts index 6fa714b92..f9a4ac939 100644 --- a/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts +++ b/packages/core/src/services/data-manager/generated/models/startGenerationHardlinksExecution202Result.ts @@ -2,6 +2,26 @@ // Generated by pnpm data-manager:sdk:generate. Do not edit. export type StartGenerationHardlinksExecution202Result = + | { + kind: "catalog.validate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + status: "passed" | "failed"; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; + } + | { + kind: "catalog.activate"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proposalId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + validationId: string; + outcome: "activated" | "not_activated" | "unknown"; + generationId: string | null; + reason: string | null; + } | { kind: "download-osm"; /** diff --git a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts index 2f5053ad0..ffa5d1f85 100644 --- a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts +++ b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts @@ -268,6 +268,13 @@ export async function ingestOvertureRequest( ): Promise { return ingestOvertureBound(url, options); } +const inspectCatalogReviewBound = bindDataManagerOperation("inspectCatalogReview"); +export async function inspectCatalogReviewRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return inspectCatalogReviewBound(url, options); +} const inspectDataConfigurationFenceBound = bindDataManagerOperation( "inspectDataConfigurationFence", ); diff --git a/packages/transitous-core/__tests__/catalog-diff.test.ts b/packages/transitous-core/__tests__/catalog-diff.test.ts new file mode 100644 index 000000000..21b0b0076 --- /dev/null +++ b/packages/transitous-core/__tests__/catalog-diff.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from "vitest"; +import { + catalogDiffPage, + diffCatalogFeeds, + parseCatalogFeedFile, + readCatalogFeedSnapshot, + reviewCatalogPair, +} from "../src/catalog-diff"; + +describe("canonical catalog review", () => { + it("reviews endpoints, origins, structured licensing, attribution and policy", () => { + const before = new Map([ + [ + "de", + { + sources: [ + { + name: "one", + url: "https://old.test", + license: { spdx: "CC-BY-4.0", attribution: { text: "Before" } }, + skip: false, + "transitland-atlas-id": "old", + }, + ], + }, + ], + ]); + const after = new Map([ + [ + "de", + { + sources: [ + { + name: "one", + url: "https://new.test", + license: { spdx: "CC-BY-4.0", attribution: { text: "After" } }, + skip: true, + "transitland-atlas-id": "new", + }, + ], + }, + ], + ]); + const review = diffCatalogFeeds(before, after); + expect(review.changes).toHaveLength(1); + expect(review.changes[0]).toMatchObject({ + id: "de/one", + change: "changed", + fields: ["endpoint", "origin", "license", "eligibility"], + }); + expect(review.requiresLicenseAcknowledgment).toBe(true); + expect(review.digest).not.toBe(diffCatalogFeeds(before, before).digest); + }); + it("does not mistake metadata property order for a changed license", () => { + const left = new Map([ + [ + "de", + { sources: [{ name: "one", license: { name: "License", url: "https://license.test" } }] }, + ], + ]); + const right = new Map([ + [ + "de", + { sources: [{ license: { url: "https://license.test", name: "License" }, name: "one" }] }, + ], + ]); + expect(diffCatalogFeeds(left, right).total).toBe(0); + expect(diffCatalogFeeds(left, right).digest).toBe(diffCatalogFeeds(right, left).digest); + }); + it("paginates every source beyond 100 without changing the complete review digest", () => { + const sources = Array.from({ length: 211 }, (_, index) => ({ + name: `source-${String(index).padStart(3, "0")}`, + })); + const review = diffCatalogFeeds(new Map(), new Map([["at", { sources }]])); + expect(review.addedRegions).toEqual(["at"]); + expect(review.requiresLicenseAcknowledgment).toBe(true); + const pages = [ + catalogDiffPage(review), + catalogDiffPage(review, 100), + catalogDiffPage(review, 200), + ]; + expect(pages.map((page) => page.rows.length)).toEqual([100, 100, 11]); + expect(new Set(pages.flatMap((page) => page.rows.map((row) => row.id))).size).toBe(211); + expect(pages.every((page) => page.total === 211 && page.digest === review.digest)).toBe(true); + expect(pages[2]!.nextOffset).toBeNull(); + expect(() => catalogDiffPage(review, -1)).toThrow(); + expect(() => catalogDiffPage(review, 212)).toThrow(); + }); + it("keeps removed sources and explicitly unknown licenses in the review", () => { + const review = diffCatalogFeeds( + new Map([["de", { sources: [{ name: "removed" }] }]]), + new Map([["at", { sources: [{ name: "added" }] }]]), + ); + expect(review).toMatchObject({ + removedRegions: ["de"], + addedRegions: ["at"], + addedSources: 1, + removedSources: 1, + total: 2, + }); + expect(review.changes.every((row) => row.licenseStatus === "unknown")).toBe(true); + }); + it.each([ + '{"sources":1}', + '{"sources":[null]}', + '{"sources":[{"name":"a"},{"name":"a"}]}', + '{"sources":[{"url":"https://one.test"}]}', + "not json", + ])("refuses malformed/ambiguous source metadata instead of skipping it: %s", (raw) => { + expect(() => parseCatalogFeedFile(raw)).toThrow(); + }); + it("reads only the named commit and refuses an unreadable or malformed file", async () => { + const sha = "a".repeat(40), + calls: string[][] = []; + const git = async (args: string[]) => { + calls.push(args); + return args[0] === "ls-tree" + ? "de.json\0at.json\0" + : JSON.stringify({ + sources: [{ name: args[1]!.includes("de.json") ? "German" : "Austrian" }], + }); + }; + expect([...(await readCatalogFeedSnapshot(sha, git)).keys()]).toEqual(["at", "de"]); + expect(calls.every((args) => args.at(-1)!.startsWith(sha + ":feeds"))).toBe(true); + await expect(readCatalogFeedSnapshot("main", git)).rejects.toThrow(/exact commit/); + await expect( + readCatalogFeedSnapshot(sha, async (args) => + args[0] === "ls-tree" ? "de.json\0" : "invalid json", + ), + ).rejects.toThrow(); + await expect( + readCatalogFeedSnapshot(sha, async () => { + throw Error("Object is missing"); + }), + ).rejects.toThrow(/Object is missing/); + }); + + it("reviews GBFS-only URL/eligibility changes, retaining excluded registry rows", () => { + const row = { + countryCode: "de", + name: "Bike", + location: "Berlin", + systemId: "bike", + operatorUrl: "https://operator.test", + discoveryUrl: "https://old.test/gbfs.json", + supportedVersions: ["2.3"], + }; + const feeds = new Map([ + ["de", { sources: [{ name: "bus", spec: "gtfs", url: "https://bus.test" }] }], + ]); + const input = { + previousFeeds: feeds, + nextFeeds: feeds, + previousGbfsRows: [row], + nextGbfsRows: [ + { ...row, discoveryUrl: "https://new.test/gbfs.json" }, + { ...row, systemId: "excluded", countryCode: "at" }, + ], + countries: ["de"], + quarantine: [], + maxAdditions: 500, + }; + const review = reviewCatalogPair(input); + expect(review.total).toBe(3); + expect(review.changes.every((change) => change.region.startsWith("mobilitydata."))).toBe(true); + expect( + review.changes.some((change) => JSON.stringify(change.after).includes('"reason":"country"')), + ).toBe(true); + expect(review.changes.every((change) => change.licenseStatus === "unknown")).toBe(true); + expect(review.requiresLicenseAcknowledgment).toBe(true); + expect(review.digest).not.toBe(reviewCatalogPair({ ...input, nextGbfsRows: [row] }).digest); + }); +}); + +it("invalidates review policy when compiler flags or failure tolerance change", async () => { + const { catalogCompilerSettings, catalogReviewInputDigests } = await import( + "../src/catalog-diff" + ); + const before = catalogReviewInputDigests( + {}, + null, + catalogCompilerSettings({ MOTIS_TWO_SLOT: "true", MOTIS_GBFS_CATALOG_ENABLED: "true" }), + ); + expect( + catalogReviewInputDigests( + {}, + null, + catalogCompilerSettings({ + MOTIS_TWO_SLOT: "true", + MOTIS_GBFS_CATALOG_ENABLED: "true", + MOTIS_GBFS_CATALOG_MAX_FAILURE_RATIO: "0.9", + }), + ).sourcePolicyRevision, + ).not.toBe(before.sourcePolicyRevision); + expect( + catalogReviewInputDigests( + {}, + null, + catalogCompilerSettings({ MOTIS_TWO_SLOT: "false", MOTIS_GBFS_CATALOG_ENABLED: "true" }), + ).sourcePolicyRevision, + ).not.toBe(before.sourcePolicyRevision); +}); + +it("bounds each full-metadata review page by bytes without omitting subsequent rows", () => { + const oldFeeds = new Map([ + [ + "de", + { + sources: ["a", "b", "c"].map((name) => ({ + name, + url: `https://old.test/${name}`, + attribution: { text: "x".repeat(1024 * 1024) }, + })), + }, + ], + ]); + const nextFeeds = new Map([ + [ + "de", + { + sources: ["a", "b", "c"].map((name) => ({ + name, + url: `https://new.test/${name}`, + attribution: { text: "x".repeat(1024 * 1024) }, + })), + }, + ], + ]); + const diff = diffCatalogFeeds(oldFeeds, nextFeeds), + first = catalogDiffPage(diff); + expect(first.rows).toHaveLength(1); + expect(first.nextOffset).toBe(1); + expect(catalogDiffPage(diff, first.nextOffset!).rows[0]!.id).toBe(diff.changes[1]!.id); + expect(Buffer.byteLength(JSON.stringify(first))).toBeLessThan(3 * 1024 * 1024); +}); + +it("reviews a pinned Atlas license change even when Transitous feed JSON is unchanged", async () => { + const oldAtlas = new Map([ + ["atlas.fixture", { sources: [{ name: "f-bus", license: { spdx_identifier: "CC0-1.0" } }] }], + ]); + const nextAtlas = new Map([ + [ + "atlas.fixture", + { + sources: [ + { name: "f-bus", license: { spdx_identifier: "CC-BY-4.0", attribution: "Operator" } }, + ], + }, + ], + ]); + const feeds = new Map([["de", { sources: [{ name: "bus", "transitland-atlas-id": "f-bus" }] }]]); + const diff = reviewCatalogPair({ + previousFeeds: feeds, + nextFeeds: feeds, + previousAtlas: oldAtlas, + nextAtlas, + previousGbfsRows: [], + nextGbfsRows: [], + countries: ["de"], + quarantine: [], + maxAdditions: 50, + }); + expect(diff.total).toBe(1); + expect(diff.changes[0]!.fields).toContain("license"); + expect(diff.changes[0]!.after?.license).toEqual({ + spdx_identifier: "CC-BY-4.0", + attribution: "Operator", + }); + expect(diff.requiresLicenseAcknowledgment).toBe(true); +}); diff --git a/packages/transitous-core/__tests__/feed-credential-requirements.test.ts b/packages/transitous-core/__tests__/feed-credential-requirements.test.ts index 7cd87c3c8..74af30171 100644 --- a/packages/transitous-core/__tests__/feed-credential-requirements.test.ts +++ b/packages/transitous-core/__tests__/feed-credential-requirements.test.ts @@ -53,3 +53,32 @@ it("refuses missing, malformed or linked catalog inputs rather than erasing requ symlinkSync(join(root, "transitland-atlas/feeds/a.dmfr.json"), join(root, "feeds/de.json")); expect(() => collectRequiredFeedCredentialKeys(root)).toThrow(/Catalog/); }); +it("checks enabled validation countries without losing the complete template inventory", () => { + const root = catalog(); + writeFileSync( + join(root, "feeds/de.json"), + JSON.stringify({ + sources: [ + { name: "A", "transitland-atlas-id": "auth" }, + { name: "C", "transitland-atlas-id": "auth", skip: true }, + ], + }), + ); + writeFileSync( + join(root, "feeds/at.json"), + JSON.stringify({ sources: [{ name: "B", "transitland-atlas-id": "auth" }] }), + ); + expect(collectRequiredFeedCredentialKeys(root)).toEqual(["at/B", "de/A", "de/C"]); + expect(collectRequiredFeedCredentialKeys(root, { countries: ["de"], patches: [] })).toEqual([ + "de/A", + ]); + expect( + collectRequiredFeedCredentialKeys(root, { + countries: ["de"], + patches: [ + { sourceId: "catalog:de:A", skip: true }, + { sourceId: "catalog:de:C", skip: false }, + ], + }), + ).toEqual(["de/C"]); +}); diff --git a/packages/transitous-core/__tests__/pinned-feed-credentials.test.ts b/packages/transitous-core/__tests__/pinned-feed-credentials.test.ts index 077e165d3..1ca1784d8 100644 --- a/packages/transitous-core/__tests__/pinned-feed-credentials.test.ts +++ b/packages/transitous-core/__tests__/pinned-feed-credentials.test.ts @@ -109,3 +109,52 @@ it.each(["oversize", "symlink"])("refuses %s repository content before checkout" ).rejects.toThrow(/unavailable/); expect(checkouts).toBe(0); }); + +it("inspects a newly fetched Atlas pin in a shallow cache without moving either shared HEAD", async () => { + const f = fixture(), + atlas = join(f.root, "transitland-atlas"), + remote = join(f.root, "atlas-remote.git"), + work = join(f.root, "atlas-work"); + run(f.root, ["clone", "--bare", atlas, remote]); + run(f.root, ["clone", remote, work]); + writeFileSync( + join(work, "feeds/a.dmfr.json"), + JSON.stringify({ + feeds: [ + { id: "auth", authorization: { type: "api-key" }, license: { name: "New pinned terms" } }, + ], + }), + ); + run(work, ["add", "."]); + run(work, ["commit", "-m", "Atlas B"]); + const next = run(work, ["rev-parse", "HEAD"]); + run(work, ["push", "origin", "HEAD"]); + run(atlas, ["fetch", "--depth=1", "--no-tags", remote, next]); + run(f.root, ["update-index", "--cacheinfo", `160000,${next},transitland-atlas`]); + run(f.root, ["commit", "-m", "New Atlas pin"]); + const head = run(f.root, ["rev-parse", "HEAD"]), + atlasHead = run(atlas, ["rev-parse", "HEAD"]); + const failures: Array<{ args: readonly string[]; stderr: string }> = []; + try { + expect( + await inspectPinnedFeedCredentialKeys({ + catalogDir: f.root, + transitousCommit: head, + atlasCommit: next, + git: async (args, cwd) => { + try { + return run(cwd, args); + } catch (error) { + failures.push({ args, stderr: String((error as { stderr?: unknown }).stderr ?? "") }); + throw error; + } + }, + }), + ).toEqual(["de/B"]); + } catch (error) { + console.info("Synthetic Git failure only", JSON.stringify(failures)); + throw error; + } + expect(run(f.root, ["rev-parse", "HEAD"])).toBe(head); + expect(run(atlas, ["rev-parse", "HEAD"])).toBe(atlasHead); +}); diff --git a/packages/transitous-core/src/catalog-diff.ts b/packages/transitous-core/src/catalog-diff.ts new file mode 100644 index 000000000..54c076bf9 --- /dev/null +++ b/packages/transitous-core/src/catalog-diff.ts @@ -0,0 +1,409 @@ +/** One deterministic source review shared by CLI and remote catalog proposals. */ +import { createHash } from "node:crypto"; +import type { TransitousFeedFile } from "./feed-source"; +import { + compileGbfsCatalog, + type ExistingGbfsSource, + type GbfsQuarantineEntry, + type MobilityDataGbfsRow, +} from "./gbfs-catalog"; + +export type CatalogChangeField = + | "endpoint" + | "origin" + | "license" + | "attribution" + | "eligibility" + | "metadata"; +export interface CatalogSourceChange { + id: string; + region: string; + source: string; + change: "added" | "removed" | "changed"; + fields: CatalogChangeField[]; + licenseStatus: "declared" | "unknown"; + before: Record | null; + after: Record | null; +} +export interface CatalogFeedDiff { + digest: string; + total: number; + changes: CatalogSourceChange[]; + addedRegions: string[]; + removedRegions: string[]; + modifiedRegions: string[]; + addedSources: number; + removedSources: number; + licenseChanges: Array<{ region: string; source: string }>; + requiresLicenseAcknowledgment: boolean; +} +function canonical(value: unknown, depth = 0): string { + if (depth > 32) throw Error("Catalog metadata is too deeply nested"); + if (value === null || typeof value === "string" || typeof value === "boolean") + return JSON.stringify(value); + if (typeof value === "number" && Number.isFinite(value)) return JSON.stringify(value); + if (Array.isArray(value)) + return `[${value.map((entry) => canonical(entry, depth + 1)).join(",")}]`; + if (value && typeof value === "object") + return `{${Object.keys(value) + .sort() + .map( + (key) => + `${JSON.stringify(key)}:${canonical((value as Record)[key], depth + 1)}`, + ) + .join(",")}}`; + throw Error("Catalog metadata must be JSON"); +} +function object(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} +/** Unreadable/malformed catalog entries may never disappear from a review. */ +export function parseCatalogFeedFile(raw: string): TransitousFeedFile { + if (Buffer.byteLength(raw, "utf8") > 4 * 1024 * 1024) + throw Error("Catalog feed file exceeds its byte limit"); + const value: unknown = JSON.parse(raw); + if (!object(value) || (value.sources !== undefined && !Array.isArray(value.sources))) + throw Error("Invalid catalog feed file"); + canonical(value); + const names = new Set(); + for (const source of value.sources ?? []) { + if ( + !object(source) || + typeof source.name !== "string" || + !source.name || + source.name.length > 512 || + names.has(source.name) + ) + throw Error("Invalid or duplicate catalog source identity"); + names.add(source.name); + } + return value as TransitousFeedFile; +} +function sources(feeds: ReadonlyMap) { + const entries = new Map< + string, + { region: string; source: string; data: Record } + >(); + let bytes = 0; + for (const [region, feed] of feeds) { + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,255}$/.test(region)) + throw Error("Invalid catalog region identity"); + const raw = canonical(feed); + bytes += Buffer.byteLength(raw, "utf8"); + if (bytes > 128 * 1024 * 1024) throw Error("Catalog review exceeds its byte limit"); + for (const source of parseCatalogFeedFile(raw).sources ?? []) { + const id = `${region}/${source.name}`; + if (entries.has(id) || entries.size >= 100000) + throw Error("Catalog review identity/entry limit"); + entries.set(id, { + region, + source: source.name!, + data: source as unknown as Record, + }); + } + } + return entries; +} +const fieldKeys: Record, string[]> = { + endpoint: ["url", "url-override", "spec", "type", "api-key"], + origin: ["openmapx-origin", "origin", "transitland-atlas-id", "mdb-id"], + license: ["license"], + attribution: ["attribution", "attributions", "credit", "credits"], + eligibility: ["skip", "skip-reason", "policy", "eligible"], +}; +export function diffCatalogFeeds( + previous: ReadonlyMap, + next: ReadonlyMap, +): CatalogFeedDiff { + const before = sources(previous), + after = sources(next), + changes: CatalogSourceChange[] = []; + for (const id of [...new Set([...before.keys(), ...after.keys()])].sort()) { + const old = before.get(id), + current = after.get(id), + a = old?.data ?? null, + b = current?.data ?? null; + if (canonical(a) === canonical(b)) continue; + const selected = current ?? old!; + const fields: CatalogChangeField[] = []; + for (const [field, keys] of Object.entries(fieldKeys)) { + if (keys.some((key) => canonical(a?.[key] ?? null) !== canonical(b?.[key] ?? null))) + fields.push(field as CatalogChangeField); + } + if (!fields.length) fields.push("metadata"); + const license = b?.license ?? a?.license; + changes.push({ + id, + region: selected.region, + source: selected.source, + change: !old ? "added" : !current ? "removed" : "changed", + fields, + licenseStatus: object(license) && Object.keys(license).length ? "declared" : "unknown", + before: a, + after: b, + }); + } + const addedRegions = [...next.keys()].filter((region) => !previous.has(region)).sort(), + removedRegions = [...previous.keys()].filter((region) => !next.has(region)).sort(); + const licenseChanges = changes + .filter( + (row) => + row.change === "changed" && + (row.fields.includes("license") || row.fields.includes("attribution")), + ) + .map(({ region, source }) => ({ region, source })); + return { + digest: createHash("sha256").update(canonical(changes)).digest("hex"), + total: changes.length, + changes, + addedRegions, + removedRegions, + modifiedRegions: [ + ...new Set( + changes + .filter((row) => previous.has(row.region) && next.has(row.region)) + .map((row) => row.region), + ), + ].sort(), + addedSources: changes.filter((row) => row.change === "added").length, + removedSources: changes.filter((row) => row.change === "removed").length, + licenseChanges, + requiresLicenseAcknowledgment: changes.some( + (row) => + row.fields.includes("license") || + row.fields.includes("attribution") || + (row.change === "added" && row.licenseStatus === "unknown"), + ), + }; +} +export function catalogDiffPage(diff: CatalogFeedDiff, offset = 0) { + if (!Number.isSafeInteger(offset) || offset < 0 || offset > diff.total) + throw Error("Invalid catalog review offset"); + const rows: CatalogSourceChange[] = []; + let bytes = 4096; + for (const row of diff.changes.slice(offset, offset + 100)) { + const size = Buffer.byteLength(canonical(row), "utf8") + 1; + if (size > 3 * 1024 * 1024 - 4096) + throw Error("Catalog review row exceeds its full-metadata byte limit"); + if (bytes + size > 3 * 1024 * 1024) break; + rows.push(row); + bytes += size; + } + return { + digest: diff.digest, + total: diff.total, + offset, + rows, + nextOffset: offset + rows.length < diff.total ? offset + rows.length : null, + }; +} +/** Read one exact Git object tree; never fall back to working-tree/remote HEAD. */ +export async function readCatalogFeedSnapshot( + commit: string, + git: (args: string[]) => Promise, +): Promise> { + if (!/^[a-f0-9]{40}$/.test(commit)) throw Error("Catalog review requires an exact commit"); + const listing = await git(["ls-tree", "-z", "--name-only", `${commit}:feeds`]); + if (Buffer.byteLength(listing, "utf8") > 16 * 1024 * 1024) + throw Error("Catalog listing exceeds its byte limit"); + const names = listing.split("\0").filter(Boolean); + if (names.length > 50000) throw Error("Catalog listing exceeds its entry limit"); + const feeds = new Map(); + let bytes = 0; + for (const name of names.sort()) { + if (!name.endsWith(".json")) continue; + const region = name.slice(0, -5); + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,255}$/.test(region) || feeds.has(region)) + throw Error("Invalid catalog feed filename"); + const raw = await git(["show", `${commit}:feeds/${name}`]); + bytes += Buffer.byteLength(raw, "utf8"); + if (bytes > 128 * 1024 * 1024) throw Error("Catalog snapshot exceeds its byte limit"); + feeds.set(region, parseCatalogFeedFile(raw)); + } + return feeds; +} + +/** Read full feed-owned Atlas metadata at the exact gitlink, including license changes. */ +export async function readAtlasFeedSnapshot( + commit: string, + git: (args: string[]) => Promise, +): Promise> { + if (!/^[a-f0-9]{40}$/.test(commit)) throw Error("Atlas review requires an exact commit"); + const listing = await git(["ls-tree", "-r", "-z", "--name-only", commit, "--", "feeds"]); + if (Buffer.byteLength(listing, "utf8") > 16 * 1024 * 1024) + throw Error("Atlas listing exceeds its byte limit"); + const names = listing.split("\0").filter(Boolean); + if (names.length > 50000) throw Error("Atlas listing exceeds its entry limit"); + const result = new Map(), + ids = new Set(); + let bytes = 0; + for (const name of names.sort()) { + if (!name.endsWith(".dmfr.json")) continue; + if ( + !name.startsWith("feeds/") || + name.length > 4096 || + name.split("/").some((part) => !part || part === "." || part === "..") + ) + throw Error("Invalid Atlas feed filename"); + const raw = await git(["show", `${commit}:${name}`]); + bytes += Buffer.byteLength(raw, "utf8"); + if (Buffer.byteLength(raw, "utf8") > 4 * 1024 * 1024 || bytes > 128 * 1024 * 1024) + throw Error("Atlas review exceeds its byte limit"); + const document: unknown = JSON.parse(raw); + canonical(document); + if (!object(document) || !Array.isArray(document.feeds)) + throw Error("Malformed Atlas feed metadata"); + const sources = document.feeds.map((feed: unknown) => { + if ( + !object(feed) || + typeof feed.id !== "string" || + !feed.id || + feed.id.length > 512 || + ids.has(feed.id) || + ids.size >= 100000 + ) + throw Error("Invalid or duplicate Atlas feed identity"); + ids.add(feed.id); + return { + name: feed.id, + origin: "transitland-atlas", + metadata: feed, + ...(object(feed.license) ? { license: feed.license } : {}), + }; + }); + result.set(`atlas.${createHash("sha256").update(name).digest("hex").slice(0, 32)}`, { + sources, + }); + } + return result; +} + +/** Include every GBFS registry row, including exclusions/conflicts/quarantine. */ +export function reviewCatalogPair(input: { + previousFeeds: ReadonlyMap; + nextFeeds: ReadonlyMap; + previousAtlas?: ReadonlyMap; + nextAtlas?: ReadonlyMap; + previousGbfsRows: MobilityDataGbfsRow[]; + nextGbfsRows: MobilityDataGbfsRow[]; + countries: string[]; + quarantine: GbfsQuarantineEntry[]; + maxAdditions: number; +}): CatalogFeedDiff { + const withGbfs = ( + feeds: ReadonlyMap, + rows: MobilityDataGbfsRow[], + ) => { + const existingSources: ExistingGbfsSource[] = []; + for (const [region, feed] of feeds) + for (const source of feed.sources ?? []) { + if (source.spec === "gbfs" && source.url && source.name) + existingSources.push({ + region, + name: source.name, + discoveryUrl: source.url, + sourceId: source["openmapx-source-id"], + provenance: "transitous", + }); + } + if (rows.length > 100000) throw Error("GBFS review exceeds its entry limit"); + const compiled = compileGbfsCatalog({ + rows, + countries: input.countries, + quarantine: input.quarantine, + maxAdditions: input.maxAdditions, + existingSources, + }); + const grouped = new Map(); + for (const entry of compiled.sourceIndex) { + const group = grouped.get(entry.sourceId); + if (group) group.push(entry); + else grouped.set(entry.sourceId, [entry]); + } + const result = new Map(feeds); + for (const [sourceId, entries] of grouped) { + const first = entries[0]!, + region = `mobilitydata.${/^[a-z]{2,3}$/.test(first.country) ? first.country : "unknown"}`; + if (feeds.has(region)) throw Error("Catalog/GBFS namespace collision"); + const previous = result.get(region)?.sources ?? []; + const source = { + name: sourceId, + spec: "gbfs" as const, + type: "url" as const, + url: first.discoveryUrl, + origin: "mobilitydata", + eligibility: entries.map((row) => ({ + status: row.status, + reason: row.exclusionReason ?? null, + preferredSourceId: row.preferredSourceId ?? null, + })), + registry: JSON.parse(JSON.stringify(entries)).sort((a: unknown, b: unknown) => + canonical(a) < canonical(b) ? -1 : canonical(a) > canonical(b) ? 1 : 0, + ), + }; + result.set(region, { sources: [...previous, source] }); + } + return result; + }; + const withAtlas = ( + feeds: ReadonlyMap, + atlas: ReadonlyMap | undefined, + ) => { + const result = new Map(feeds), + referenced = new Set( + [...feeds.values()].flatMap((feed) => + (feed.sources ?? []) + .map((source) => source["transitland-atlas-id"]) + .filter((id): id is string => typeof id === "string"), + ), + ); + for (const [region, metadata] of atlas ?? []) { + const sources = (metadata.sources ?? []).filter( + (source) => source.name && referenced.has(source.name), + ); + if (!sources.length) continue; + if (result.has(region)) throw Error("Catalog/Atlas namespace collision"); + result.set(region, { sources }); + } + return result; + }; + return diffCatalogFeeds( + withGbfs(withAtlas(input.previousFeeds, input.previousAtlas), input.previousGbfsRows), + withGbfs(withAtlas(input.nextFeeds, input.nextAtlas), input.nextGbfsRows), + ); +} + +/** Policy and overlay metadata share one canonical encoding across DM and host. */ +export function catalogCompilerSettings( + env: Record, +): Record { + const defaults = { + MOTIS_TWO_SLOT: "false", + MOTIS_GBFS_CATALOG_ENABLED: "false", + MOTIS_GBFS_CATALOG_MAX_ADDITIONS: "50", + MOTIS_GBFS_CATALOG_TIMEOUT_MS: "5000", + MOTIS_GBFS_CATALOG_CONCURRENCY: "4", + MOTIS_GBFS_CATALOG_MAX_FAILURE_RATIO: "0.35", + }; + return Object.fromEntries( + Object.entries(defaults).map(([key, fallback]) => [key, env[key] ?? fallback]), + ); +} +export function catalogReviewInputDigests( + operationsProfile: object, + overlay: { patches: unknown[]; quarantine: unknown[] } | null, + compilerSettings = catalogCompilerSettings({}), +) { + return { + overlayDigest: createHash("sha256").update(canonical(overlay)).digest("hex"), + sourcePolicyRevision: createHash("sha256") + .update( + canonical({ + compilerSettings, + operationsProfile: JSON.parse(JSON.stringify(operationsProfile)), + sourcePolicy: { patches: overlay?.patches ?? [], quarantine: overlay?.quarantine ?? [] }, + }), + ) + .digest("hex"), + }; +} diff --git a/packages/transitous-core/src/catalog-resolution.ts b/packages/transitous-core/src/catalog-resolution.ts new file mode 100644 index 000000000..e94ec59ab --- /dev/null +++ b/packages/transitous-core/src/catalog-resolution.ts @@ -0,0 +1,116 @@ +/** Fixed upstream resolution shared by proposal producers; no active pin writes. */ +import { createHash } from "node:crypto"; +import { type MobilityDataGbfsRow, parseMobilityDataGbfsCsv } from "./gbfs-catalog"; +export interface ResolvedGbfsCatalogPin { + schemaVersion: 1; + source: "mobilitydata-gbfs"; + commit: string; + url: string; + sha256: string; + lockedAt: string; + lockedBy: string; +} +const COMMIT_URL = "https://api.github.com/repos/MobilityData/gbfs/commits/master"; +async function boundedText(response: Response, maximum: number): Promise { + if (!response.ok) throw Error(`MobilityData acquisition failed: HTTP ${response.status}`); + if (!response.body) throw Error("MobilityData response body is missing"); + const reader = response.body.getReader(), + chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + size += chunk.value.byteLength; + if (size > maximum) throw Error("MobilityData response exceeds its byte limit"); + chunks.push(chunk.value); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (error) { + await reader.cancel().catch(() => {}); + throw error; + } finally { + reader.releaseLock(); + } +} +function options(signal?: AbortSignal): RequestInit { + return { + headers: { "User-Agent": "Open-MapX-transitous-bump" }, + signal: signal + ? AbortSignal.any([signal, AbortSignal.timeout(30000)]) + : AbortSignal.timeout(30000), + redirect: "error", + }; +} +export async function fetchPinnedGbfsCatalog( + pin: ResolvedGbfsCatalogPin, + fetchImpl: typeof fetch = fetch, + signal?: AbortSignal, +): Promise { + if ( + !/^[a-f0-9]{40}$/.test(pin.commit) || + pin.url !== `https://raw.githubusercontent.com/MobilityData/gbfs/${pin.commit}/systems.csv` + ) + throw Error("GBFS acquisition requires the exact pinned upstream URL"); + const csv = await boundedText(await fetchImpl(pin.url, options(signal)), 32 * 1024 * 1024); + if (createHash("sha256").update(csv).digest("hex") !== pin.sha256) + throw Error("Pinned GBFS content digest changed"); + return csv; +} +export async function resolveGbfsCatalogCandidate( + lockedBy: string, + fetchImpl: typeof fetch = fetch, + now: () => Date = () => new Date(), + signal?: AbortSignal, +): Promise<{ + lock: ResolvedGbfsCatalogPin; + countryCounts: Map; + rows: MobilityDataGbfsRow[]; +}> { + const commit: unknown = JSON.parse( + await boundedText(await fetchImpl(COMMIT_URL, options(signal)), 65536), + ); + if ( + !commit || + typeof commit !== "object" || + !("sha" in commit) || + typeof commit.sha !== "string" || + !/^[a-f0-9]{40}$/.test(commit.sha) + ) + throw Error("MobilityData commit lookup returned an invalid SHA"); + const url = `https://raw.githubusercontent.com/MobilityData/gbfs/${commit.sha}/systems.csv`; + const csv = await boundedText(await fetchImpl(url, options(signal)), 32 * 1024 * 1024), + countryCounts = new Map(); + const rows = parseCatalogGbfsRows(csv); + for (const row of rows) + countryCounts.set(row.countryCode, (countryCounts.get(row.countryCode) ?? 0) + 1); + return { + lock: { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: commit.sha, + url, + sha256: createHash("sha256").update(csv).digest("hex"), + lockedAt: now().toISOString(), + lockedBy, + }, + countryCounts, + rows, + }; +} + +export function parseCatalogGbfsRows(csv: string): MobilityDataGbfsRow[] { + let separators = 0; + for (const character of csv) + if ((character === "\n" || character === "\r") && ++separators > 200002) + throw Error("GBFS registry exceeds its entry limit"); + const rows = parseMobilityDataGbfsCsv(csv); + if (rows.length > 100000) throw Error("GBFS registry exceeds its entry limit"); + return rows; +} diff --git a/packages/transitous-core/src/feed-credential-requirements.ts b/packages/transitous-core/src/feed-credential-requirements.ts index ac1979ac5..c694f5144 100644 --- a/packages/transitous-core/src/feed-credential-requirements.ts +++ b/packages/transitous-core/src/feed-credential-requirements.ts @@ -14,7 +14,14 @@ import { isWellFormedFeedCredentialText } from "./feed-credentials"; const unavailable = () => new Error("Catalog credential requirements are unavailable"); const decoder = new TextDecoder("utf-8", { fatal: true }); -export function collectRequiredFeedCredentialKeys(catalogDir: string): string[] { +export interface FeedCredentialRequirementScope { + countries: readonly string[]; + patches: readonly { sourceId: string; skip: boolean }[]; +} +export function collectRequiredFeedCredentialKeys( + catalogDir: string, + scope?: FeedCredentialRequirementScope, +): string[] { let bytes = 0, files = 0; const directory = (path: string) => { @@ -114,6 +121,17 @@ export function collectRequiredFeedCredentialKeys(catalogDir: string): string[] !isWellFormedFeedCredentialText(source.name) ) throw unavailable(); + if (scope) { + const region = entry.name.slice(0, -5), + country = region.split("-")[0]!.toLowerCase(); + if (scope.countries.length && !scope.countries.includes(country)) continue; + const sourceId = + typeof source["openmapx-source-id"] === "string" + ? source["openmapx-source-id"] + : `catalog:${region}:${source.name}`; + const patch = scope.patches.find((patch) => patch.sourceId === sourceId); + if (patch?.skip ?? source.skip === true) continue; + } required.add(`${entry.name.slice(0, -5)}/${source.name}`); } } diff --git a/packages/transitous-core/src/index.ts b/packages/transitous-core/src/index.ts index 6c0c04110..104a2bb86 100644 --- a/packages/transitous-core/src/index.ts +++ b/packages/transitous-core/src/index.ts @@ -4,6 +4,24 @@ export { resetCatalog, safeDirArgs, } from "./catalog.js"; +export { + type CatalogFeedDiff, + type CatalogSourceChange, + catalogCompilerSettings, + catalogDiffPage, + catalogReviewInputDigests, + diffCatalogFeeds, + parseCatalogFeedFile, + readAtlasFeedSnapshot, + readCatalogFeedSnapshot, + reviewCatalogPair, +} from "./catalog-diff"; +export { + fetchPinnedGbfsCatalog, + parseCatalogGbfsRows, + type ResolvedGbfsCatalogPin, + resolveGbfsCatalogCandidate, +} from "./catalog-resolution"; export { DEFAULT_TRANSITOUS_REPO_URL, GTFSCLEAN_COMMIT, @@ -53,6 +71,17 @@ export { type MotisConfigExpectations, parseMotisConfigExpectations, } from "./motis-config.js"; +export { + isTransitousHostname, + MOTIS_OPERATIONS_PROFILES, + type MotisOperationsPolicy, + type MotisOperationsProfileName, + publicOperationsPolicy, + type ResolveOperationsProfileInput, + resolveOperationsProfile, + resolveOperationsProfileFromEnv, + transitAllowedCountries, +} from "./operations-profile"; export { inspectPinnedFeedCredentialKeys, type PinnedCredentialGitRunner, diff --git a/packages/transitous-core/src/operations-profile.ts b/packages/transitous-core/src/operations-profile.ts new file mode 100644 index 000000000..279ff03c6 --- /dev/null +++ b/packages/transitous-core/src/operations-profile.ts @@ -0,0 +1,185 @@ +import type { TransitSource } from "./constants.js"; + +export const MOTIS_OPERATIONS_PROFILES = [ + "regional-assisted", + "regional-sovereign", + "planet", +] as const; + +export type MotisOperationsProfileName = (typeof MOTIS_OPERATIONS_PROFILES)[number]; + +export interface MotisOperationsPolicy { + profile: MotisOperationsProfileName; + countries: string[]; + feedAllowList: string[]; + acquisition: TransitSource; + artifactOrigins: string[]; + originDownloadsRequired: boolean; + hostedRuntimeFallbackAllowed: boolean; + osmInput?: string; + gbfsSelection: "explicit-countries" | "explicit-feeds" | "allow-listed-batches"; + maxFeedCount: number; + retentionGenerations: number; + resourceEnvelope: { + minimumMemoryGb: number; + minimumCpu: number; + minimumDiskGb: number; + operatorCapacityRequired: boolean; + }; + updateCadenceHours: number; + experimental: boolean; +} + +export interface ResolveOperationsProfileInput { + profile?: string; + countries?: string[]; + feedAllowList?: string[]; + source?: TransitSource; + artifactBaseUrl?: string; + osmInput?: string; + confirmPlanet?: boolean; + maxFeedCount?: number; + retentionGenerations?: number; + allowEmptyRegional?: boolean; +} + +function normalized(values: string[] | undefined): string[] { + return [ + ...new Set((values ?? []).map((value) => value.trim().toLowerCase()).filter(Boolean)), + ].sort(); +} + +function positiveInteger(value: number | undefined, fallback: number, label: string): number { + const resolved = value ?? fallback; + if (!Number.isInteger(resolved) || resolved < 1) + throw new Error(`${label} must be a positive integer`); + return resolved; +} + +/** Match DNS labels, including a fully qualified hostname's optional root dot. */ +export function isTransitousHostname(hostname: string): boolean { + const normalizedHostname = hostname.toLowerCase().replace(/\.$/, ""); + return normalizedHostname === "transitous.org" || normalizedHostname.endsWith(".transitous.org"); +} + +function artifactUrl(url: string | undefined): URL | undefined { + if (!url) return undefined; + try { + return new URL(url); + } catch { + throw new Error(`Invalid Transitous artifact URL: ${url}`); + } +} + +export function resolveOperationsProfile( + input: ResolveOperationsProfileInput, +): MotisOperationsPolicy { + const profile = (input.profile ?? "regional-assisted") as MotisOperationsProfileName; + if (!MOTIS_OPERATIONS_PROFILES.includes(profile)) { + throw new Error(`Unsupported MOTIS operations profile: ${input.profile}`); + } + const countries = normalized(input.countries); + const feedAllowList = normalized(input.feedAllowList); + const source = input.source ?? (profile === "regional-assisted" ? "mirror" : "build"); + const artifact = artifactUrl(input.artifactBaseUrl); + const origin = artifact?.origin; + + if ( + profile !== "planet" && + countries.length === 0 && + feedAllowList.length === 0 && + !input.allowEmptyRegional + ) { + throw new Error( + `${profile} requires explicit countries or a feed allow-list; [] never means planet`, + ); + } + if (profile === "planet" && input.confirmPlanet !== true) { + throw new Error("planet profile requires MOTIS_PLANET_CONFIRM=true"); + } + if (profile === "regional-sovereign") { + if (source !== "build") throw new Error("regional-sovereign requires origin build acquisition"); + if (artifact && isTransitousHostname(artifact.hostname)) { + throw new Error("regional-sovereign prohibits Transitous artifact domains"); + } + } + + const assisted = profile === "regional-assisted"; + const planet = profile === "planet"; + return { + profile, + countries, + feedAllowList, + acquisition: source, + artifactOrigins: origin ? [origin] : assisted ? ["https://api.transitous.org"] : [], + originDownloadsRequired: !assisted, + hostedRuntimeFallbackAllowed: assisted, + osmInput: input.osmInput?.trim() || undefined, + gbfsSelection: planet + ? "allow-listed-batches" + : feedAllowList.length > 0 + ? "explicit-feeds" + : "explicit-countries", + maxFeedCount: positiveInteger(input.maxFeedCount, planet ? 10_000 : 500, "maxFeedCount"), + retentionGenerations: positiveInteger( + input.retentionGenerations, + planet ? 3 : 2, + "retentionGenerations", + ), + resourceEnvelope: { + minimumMemoryGb: planet ? 64 : 16, + minimumCpu: planet ? 8 : 4, + minimumDiskGb: planet ? 500 : 40, + operatorCapacityRequired: planet, + }, + updateCadenceHours: planet ? 48 : 24, + experimental: planet, + }; +} + +function csv(value: string | undefined): string[] { + return (value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean); +} + +export function resolveOperationsProfileFromEnv( + env: NodeJS.ProcessEnv = process.env, + overrides: Partial = {}, +): MotisOperationsPolicy { + const number = (name: string): number | undefined => { + const value = Number(env[name]); + return Number.isFinite(value) ? value : undefined; + }; + return resolveOperationsProfile({ + profile: env.MOTIS_OPERATIONS_PROFILE, + countries: csv(env.TRANSITOUS_COUNTRIES), + feedAllowList: csv(env.MOTIS_FEED_ALLOW_LIST), + source: (env.TRANSIT_SOURCE as TransitSource | undefined) ?? undefined, + artifactBaseUrl: env.TRANSITOUS_ARTIFACT_BASE_URL, + osmInput: env.MOTIS_OSM_REGION ?? env.MOTIS_OSM_FILE, + confirmPlanet: env.MOTIS_PLANET_CONFIRM === "true", + maxFeedCount: number("MOTIS_MAX_FEED_COUNT"), + retentionGenerations: number("MOTIS_RETENTION_GENERATIONS"), + ...overrides, + }); +} + +/** Effective country scope for a finite legacy feed allow-list. This does + * not select feeds: the preflight independently checks each exact feed ID. */ +export function transitAllowedCountries(policy: MotisOperationsPolicy): string[] { + if (policy.countries.length) + return [...new Set(policy.countries.map((c) => c.toLowerCase()))].sort(); + return [ + ...new Set( + policy.feedAllowList.flatMap((id) => { + const country = /^([a-z]{2,3})_/.exec(id.toLowerCase())?.[1]; + return country ? [country] : []; + }), + ), + ].sort(); +} +export function publicOperationsPolicy(policy: MotisOperationsPolicy): MotisOperationsPolicy { + return { ...structuredClone(policy), countries: transitAllowedCountries(policy) }; +} diff --git a/packages/transitous-core/src/pinned-feed-credentials.ts b/packages/transitous-core/src/pinned-feed-credentials.ts index ef637cea7..b38abe74c 100644 --- a/packages/transitous-core/src/pinned-feed-credentials.ts +++ b/packages/transitous-core/src/pinned-feed-credentials.ts @@ -1,15 +1,19 @@ -/** Local read-only clone of two content-addressed cached repositories. No fetch, - * reset, pull or submodule mutation of the shared active working tree. */ +/** Read two content-addressed cached repositories using disposable checkouts. + * No upstream fetch, reset, pull or submodule mutation of the shared cache. */ import { lstatSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { collectRequiredFeedCredentialKeys } from "./feed-credential-requirements"; +import { + collectRequiredFeedCredentialKeys, + type FeedCredentialRequirementScope, +} from "./feed-credential-requirements"; export type PinnedCredentialGitRunner = (args: readonly string[], cwd: string) => Promise; export async function inspectPinnedFeedCredentialKeys(input: { catalogDir: string; transitousCommit: string; atlasCommit: string; git: PinnedCredentialGitRunner; + scope?: FeedCredentialRequirementScope; }): Promise { const unavailable = () => new Error("Pinned catalog credential requirements are unavailable"); if (!/^[a-f0-9]{40}$/.test(input.transitousCommit) || !/^[a-f0-9]{40}$/.test(input.atlasCommit)) @@ -21,10 +25,36 @@ export async function inspectPinnedFeedCredentialKeys(input: { const root = mkdtempSync(join(tmpdir(), "openmapx-feed-credential-inspect-")), checkout = join(root, "catalog"); try { + const ensureCachedObject = async (repository: string, source: string, commit: string) => { + // A shallow local clone may omit a fetched-but-not-checked-out pin. Copy + // that exact existing object through the local Git transport, never a + // configured upstream URL or a mutable branch. Only the scratch changes. + await input.git(["-C", source, "cat-file", "-e", `${commit}^{commit}`], root); + try { + await input.git(["-C", repository, "cat-file", "-e", `${commit}^{commit}`], root); + } catch { + await input.git( + [ + "-C", + repository, + "-c", + "protocol.file.allow=always", + "fetch", + "--no-tags", + "--depth=1", + "--", + source, + commit, + ], + root, + ); + } + }; await input.git( ["clone", "--shared", "--no-hardlinks", "--no-checkout", "--", input.catalogDir, checkout], root, ); + await ensureCachedObject(checkout, input.catalogDir, input.transitousCommit); let bytes = 0, files = 0; const checkTree = async (repository: string, commit: string) => { @@ -68,6 +98,7 @@ export async function inspectPinnedFeedCredentialKeys(input: { ], root, ); + await ensureCachedObject(atlas, join(input.catalogDir, "transitland-atlas"), input.atlasCommit); await checkTree(atlas, input.atlasCommit); await input.git( ["-C", checkout, "checkout", "--detach", "--force", input.transitousCommit], @@ -81,7 +112,7 @@ export async function inspectPinnedFeedCredentialKeys(input: { await input.git(["-C", atlas, "checkout", "--detach", "--force", input.atlasCommit], root); if ((await input.git(["-C", atlas, "rev-parse", "HEAD"], root)).trim() !== input.atlasCommit) throw unavailable(); - return collectRequiredFeedCredentialKeys(checkout); + return collectRequiredFeedCredentialKeys(checkout, input.scope); } catch { throw unavailable(); } finally { diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 9047a62e8..3edc713ce 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -62,7 +62,10 @@ "services/data-manager/__tests__/search-index/build-postgres.test.ts", "services/data-manager/__tests__/search-index/schema-postgres.test.ts", "apps/api/src/services/transit-credential-preview-postgres.test.ts", - "apps/api/src/services/transit-feed-credentials-postgres.test.ts" + "apps/api/src/services/transit-feed-credentials-postgres.test.ts", + "services/data-manager/__tests__/transitous/catalog-review.test.ts", + "services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts", + "apps/api/src/services/activity-catalog-postgres.test.ts" ], "export-restore": [ "apps/api/src/privacy/controller-wide-export-postgres.test.ts", diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index d6b033b3e..bd0a11a99 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -17,6 +17,7 @@ "services/data-manager/src/jobs/transitous/catalog-bump.ts|runner|1|git|transitous-runner|transitous.catalog-status", "services/data-manager/src/jobs/transitous/catalog-bump.ts|runner|2|git|transitous-runner|transitous.catalog-diff", "services/data-manager/src/jobs/transitous/catalog-bump.ts|runner|3|git|transitous-runner|transitous.catalog-proposal", + "services/data-manager/src/jobs/transitous/catalog-review.ts|execa|1|git|transitous-runner|transitous.catalog-diff", "services/data-manager/src/jobs/transitous/internal.ts|execa|1|command|transitous-runner|transitous.fixed-runner", "services/data-manager/src/jobs/transitous/internal.ts|execa|2|git|transitous-runner|transitous.git-read", "services/data-manager/src/jobs/transitous/prepare.ts|runner|1|git|transitous-runner|transitous.checkout-fetch", diff --git a/services/data-manager/__tests__/api-transit-bump.test.ts b/services/data-manager/__tests__/api-transit-bump.test.ts index 62e5a9790..50d998645 100644 --- a/services/data-manager/__tests__/api-transit-bump.test.ts +++ b/services/data-manager/__tests__/api-transit-bump.test.ts @@ -44,7 +44,7 @@ describe("data-manager /transit/bump branch validation", () => { }, ); - it("accepts a normal branch name (passes validation, fails later on missing catalog)", async () => { + it("accepts a normal branch name and reports the missing installed paired resolver", async () => { const app = createTestApp(); registerApi(app, { dataDir: join(fixtureRoot, "data") }); const res = await app.inject({ @@ -52,9 +52,7 @@ describe("data-manager /transit/bump branch validation", () => { url: "/transit/bump", payload: { branch: "release/2026-06" }, }); - // Passes the ref check, then hits the catalog-not-cloned guard (409), - // proving the validator did not reject a legitimate ref. - expect(res.statusCode).toBe(409); - expect(JSON.parse(res.body).error).toBe("catalog-not-cloned"); + expect(res.statusCode).toBe(503); + expect(JSON.parse(res.body).error).toBe("Paired catalog resolver is unavailable"); }); }); diff --git a/services/data-manager/__tests__/cron.test.ts b/services/data-manager/__tests__/cron.test.ts index 94a2a569c..b226f54fb 100644 --- a/services/data-manager/__tests__/cron.test.ts +++ b/services/data-manager/__tests__/cron.test.ts @@ -291,15 +291,12 @@ describe("setupCron", () => { key: "autoBumpCron", configure: (work) => ({ autoBumpCronExpression: everySecond, - resolveBumpCandidate: async () => { - await work(); - return { - branch: "main", - ref: `main@${"a".repeat(40)}`, - transitousSha: "a".repeat(40), - transitlandAtlasSha: "b".repeat(40), - }; - }, + catalogReview: { + resolveCatalogProposal: async () => { + await work(); + return { proposalId: "00000000-0000-4000-8000-000000000001" }; + }, + } as unknown as CronSetupOptions["catalogReview"], }), }, { @@ -697,13 +694,11 @@ describe("setupCron", () => { handles.stop(); }); - it("retains the proposal without running a canary or unvalidated activation", async () => { + it("creates a paired proposal even when the Transitous SHA is unchanged, and never activates", async () => { seedLocks(); - const runBumpPipeline = vi.fn(async (_jobId: string) => ({ - jobId: "j", - results: [], - finalStatus: "ok" as const, - })); + const resolveCatalogProposal = vi.fn(async () => ({ + proposalId: "00000000-0000-4000-8000-000000000001", + })) as unknown as NonNullable["resolveCatalogProposal"]; const handles = setupCron({ dataDir, countries: [], @@ -714,23 +709,21 @@ describe("setupCron", () => { feedProxyReloadCronExpression: "disabled", stalenessCheckCronExpression: "disabled", autoBumpCronExpression: "0 0 1 1 *", - resolveBumpCandidate: async () => newCandidate, + catalogReview: { resolveCatalogProposal }, }); - await handles.runAutoBumpNow(); - expect(runBumpPipeline).not.toHaveBeenCalled(); - const active = activeLock(); - expect(active.ref).toBe(`main@${ACTIVE}`); - expect(active.submodules["transitland-atlas"]).toBe(ATLAS_OLD); - expect(proposalExists()).toBe(true); - handles.stop(); + try { + const { runOpsOperation } = await import("../src/ops-client.js"); + vi.mocked(runOpsOperation).mockClear(); + await handles.runAutoBumpNow(); + expect(resolveCatalogProposal).toHaveBeenCalledOnce(); + expect(runOpsOperation).not.toHaveBeenCalled(); + expect(activeLock().ref).toBe(`main@${ACTIVE}`); + } finally { + handles.stop(); + } }); - - it("keeps the current pin and proposal while validation-bound activation is unavailable", async () => { + it("refuses scheduled proposals when the installed paired resolver is unavailable", async () => { seedLocks(); - const createIssue = vi.fn(async (_t: string, _b: string) => "https://x/issues/1"); - const runBumpPipeline = vi.fn(async (_jobId: string) => { - throw new Error('motis-health probe "rentals" failed: empty'); - }); const handles = setupCron({ dataDir, countries: [], @@ -742,45 +735,13 @@ describe("setupCron", () => { stalenessCheckCronExpression: "disabled", autoBumpCronExpression: "0 0 1 1 *", resolveBumpCandidate: async () => newCandidate, - - githubIssueSink: { findOpenIssueByTitle: async () => null, createIssue }, - }); - await handles.runAutoBumpNow(); - expect(activeLock().ref).toBe(`main@${ACTIVE}`); - expect(proposalExists()).toBe(true); - expect(runBumpPipeline).not.toHaveBeenCalled(); - expect(createIssue).not.toHaveBeenCalled(); - handles.stop(); - }); - - it("does nothing when the active pin already matches upstream", async () => { - seedLocks(); - const runBumpPipeline = vi.fn(async (_jobId: string) => ({ - jobId: "j", - results: [], - finalStatus: "ok" as const, - })); - const handles = setupCron({ - dataDir, - countries: [], - store: {} as never, - singleFlight: makeController(), - logger: { info: () => {}, warn: () => {}, error: () => {} }, - syncCronExpression: "disabled", - feedProxyReloadCronExpression: "disabled", - stalenessCheckCronExpression: "disabled", - autoBumpCronExpression: "0 0 1 1 *", - resolveBumpCandidate: async () => ({ - branch: "main", - ref: `main@${ACTIVE}`, - transitousSha: ACTIVE, - transitlandAtlasSha: ATLAS_OLD, - }), }); - await handles.runAutoBumpNow(); - expect(runBumpPipeline).not.toHaveBeenCalled(); - expect(proposalExists()).toBe(false); - handles.stop(); + try { + await expect(handles.runAutoBumpNow()).rejects.toThrow(/paired catalog resolver/i); + expect(proposalExists()).toBe(false); + } finally { + handles.stop(); + } }); }); diff --git a/services/data-manager/__tests__/fixtures/catalog-data-manager-process.mjs b/services/data-manager/__tests__/fixtures/catalog-data-manager-process.mjs new file mode 100644 index 000000000..8d9fb6cde --- /dev/null +++ b/services/data-manager/__tests__/fixtures/catalog-data-manager-process.mjs @@ -0,0 +1,90 @@ +/** Native owning DM assembly. Only exact pinned public CSV acquisition is fixture IO. */ +import { createInterface } from "node:readline"; + +const lines = createInterface({ input: process.stdin }); +const input = JSON.parse((await lines[Symbol.asyncIterator]().next()).value); +lines.close(); +Object.assign(process.env, input.environment, { + DATABASE_URL: input.databaseUrl, + OPS_AGENT_URL: input.agentUrl, + OPS_AGENT_TOKEN_FILE: input.tokenFile, +}); +const fetchOriginal = globalThis.fetch; +globalThis.fetch = (url, options) => + /^https:\/\/raw\.githubusercontent\.com\/MobilityData\/gbfs\/[bc]{40}\/systems\.csv$/.test( + String(url), + ) + ? Promise.resolve(new Response(input.csv, { headers: { "content-type": "text/csv" } })) + : fetchOriginal(url, options); +const [ + { openControlInstallation, createDataExecutionControlJournal }, + { rawSql, sql }, + { createDataConfigurationMaintenance }, + { createPostgresDataExecutionStore }, + { createDataExecutionRuntime }, + { createDataExecutionReceiver }, + { createDataExecutionDispatcher }, + { getSingleFlightController }, + { StateStore }, + { resolveOperationsProfileFromEnv }, + { registerAuth }, + { registerApi }, + { default: Fastify }, +] = await Promise.all([ + import("@openmapx/core/build-artifacts"), + import("../../src/db/index.ts"), + import("../../src/jobs/execution/configuration-maintenance.ts"), + import("../../src/jobs/execution/store.ts"), + import("../../src/jobs/execution/runtime.ts"), + import("../../src/jobs/execution/receiver.ts"), + import("../../src/jobs/execution/dispatch.ts"), + import("../../src/jobs/transitous/runtime.ts"), + import("../../src/state.ts"), + import("../../src/jobs/transitous/operations-profile.ts"), + import("../../src/auth.ts"), + import("../../src/api.ts"), + import("fastify"), +]); +const installation = openControlInstallation({ + infraDir: input.infraDir, + role: "data-manager", + serviceIds: input.serviceIds, +}); +const journal = createDataExecutionControlJournal(installation, rawSql); +const fence = createDataConfigurationMaintenance(rawSql, installation); +await fence.startup(); +const store = createPostgresDataExecutionStore(rawSql, { + controlJournal: journal, + configurationFence: fence, +}); +const policy = resolveOperationsProfileFromEnv(), + singleFlight = getSingleFlightController(); +const dispatcher = createDataExecutionDispatcher({ + dataDir: input.dataDir, + state: new StateStore(input.dataDir), + sql, + singleFlight, + operationsPolicy: policy, + runHardlinks: async () => { + throw Error("No generation in catalog fixture"); + }, +}); +const runtime = createDataExecutionRuntime({ store, dispatch: dispatcher.dispatch }); +const receiver = createDataExecutionReceiver({ + runtime, + installation, + configurationFence: fence, + dispatcher, + operationsPolicy: policy, +}); +const app = Fastify(); +registerAuth(app, input.token, { executionReceiver: () => receiver }); +registerApi(app, { + dataDir: input.dataDir, + singleFlight, + operationsPolicy: policy, + executionReceiver: receiver, +}); +await runtime.reconcile(); +const url = await app.listen({ host: "127.0.0.1", port: input.listenPort ?? 0 }); +process.stdout.write(JSON.stringify({ event: "ready", url }) + "\n"); diff --git a/services/data-manager/__tests__/ops-client-ownership.test.ts b/services/data-manager/__tests__/ops-client-ownership.test.ts index 90577cc04..cce87711f 100644 --- a/services/data-manager/__tests__/ops-client-ownership.test.ts +++ b/services/data-manager/__tests__/ops-client-ownership.test.ts @@ -1,6 +1,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; import { OpsClient } from "@openmapx/core/ops"; import { expect, it } from "vitest"; import { resetOpsClient, runOpsOperation } from "../src/ops-client.js"; @@ -12,9 +13,27 @@ it("refuses an unowned synchronous proposal before the actual client POST", asyn let posts = 0; const operation = { kind: "transitousLock.propose" as const, - ref: `main@${"a".repeat(40)}`, - submodules: { "transitland-atlas": "b".repeat(40) }, - lockedBy: "public-fixture", + proposalId: "00000000-0000-4000-8000-000000000001", + baseGenerationId: null, + diffDigest: "d".repeat(64), + sourcePolicyRevision: "e".repeat(64), + pins: createCatalogPinSet( + { + ref: `main@${"a".repeat(40)}`, + submodules: { "transitland-atlas": "b".repeat(40) }, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }, + { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "c".repeat(40), + sha256: "d".repeat(64), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"c".repeat(40)}/systems.csv`, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }, + ), }; resetOpsClient( new OpsClient({ @@ -37,7 +56,11 @@ it("refuses an unowned synchronous proposal before the actual client POST", asyn result: { execution: "sync", kind: operation.kind, - value: { ref: operation.ref, proposed: true }, + value: { + proposalId: operation.proposalId, + catalogRevisionId: operation.pins.generationId, + proposed: true, + }, }, }); }, diff --git a/services/data-manager/__tests__/transitous/assemble-staging.test.ts b/services/data-manager/__tests__/transitous/assemble-staging.test.ts index d58fd23a9..1a7950f5b 100644 --- a/services/data-manager/__tests__/transitous/assemble-staging.test.ts +++ b/services/data-manager/__tests__/transitous/assemble-staging.test.ts @@ -217,6 +217,15 @@ function makeCtx(dataDir: string) { // --------------------------------------------------------------------------- describe("assemble-staging run()", () => { + it("keeps candidate metadata immutable when the producer rewrites its next config", async () => { + const fx = setupFixture({ withConfig: true, feeds: ["feed.gtfs.zip"] }); + expect((await run(makeCtx(fx.dataDir))).status).toBe("ok"); + for (const name of ["config.yml", "license.json", TRANSIT_SOURCE_MANIFEST_FILENAME]) { + const original = readFileSync(join(fx.stagingDir, name)); + writeFileSync(join(fx.outDir, name), "next producer generation"); + expect(readFileSync(join(fx.stagingDir, name))).toEqual(original); + } + }); it("returns status 'skipped' when config.yml is absent from outDir", async () => { const fx = setupFixture({}); const result = await run(makeCtx(fx.dataDir)); diff --git a/services/data-manager/__tests__/transitous/candidate.test.ts b/services/data-manager/__tests__/transitous/candidate.test.ts index 706173e03..096aab435 100644 --- a/services/data-manager/__tests__/transitous/candidate.test.ts +++ b/services/data-manager/__tests__/transitous/candidate.test.ts @@ -1,11 +1,14 @@ +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { CANDIDATE_PROXY_DIRNAME, createCandidateManifest, parseMotisConfigExpectations, + sha256File, verifyCandidateManifest, } from "../../src/jobs/transitous/candidate.js"; import { resolveOperationsProfile } from "../../src/jobs/transitous/operations-profile.js"; @@ -98,3 +101,17 @@ describe("candidate manifest", () => { expect(() => verifyCandidateManifest(dir)).toThrow(/hash mismatch: license\.json/); }); }); + +vi.mock("node:fs", async (original) => { + const actual = await original(); + return { ...actual, readFileSync: vi.fn(actual.readFileSync) }; +}); +it("hashes datasets without loading a whole file into one buffer", () => { + tmp = mkdtempSync(join(tmpdir(), "omx-candidate-stream-")); + const file = join(tmp, "large.gtfs.zip"), + bytes = Buffer.alloc(8 * 1024 * 1024, 7); + writeFileSync(file, bytes); + vi.mocked(fs.readFileSync).mockClear(); + expect(sha256File(file).sha256).toBe(createHash("sha256").update(bytes).digest("hex")); + expect(fs.readFileSync).not.toHaveBeenCalledWith(file); +}); diff --git a/services/data-manager/__tests__/transitous/catalog-review.test.ts b/services/data-manager/__tests__/transitous/catalog-review.test.ts new file mode 100644 index 000000000..4e308d5d0 --- /dev/null +++ b/services/data-manager/__tests__/transitous/catalog-review.test.ts @@ -0,0 +1,385 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + closeControlInstallation, + createDataExecutionControlJournal, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { initializeManagedCatalogPins, readCatalogPinSet } from "@openmapx/core/catalog-pins"; +import { createControlRootOperationKey, OpsClient } from "@openmapx/core/ops"; +import { + feedState, + jobStages, + jobs, + offlinePackageArtifactReferences, + offlinePackageJobOwners, + offlinePackageJobs, +} from "@openmapx/db-schema"; +import { inspectPinnedFeedCredentialKeys } from "@openmapx/transitous-core"; +import { drizzle } from "drizzle-orm/postgres-js"; +import Fastify from "fastify"; +import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { createMigratedPostgresFixture } from "../../../../apps/api/src/test/postgres-fixture"; +import { + createRegistryResourcePolicy, + denyAllTrustedOpsData, +} from "../../../../apps/ops-agent/src/policy"; +import { createUnavailableRuntime } from "../../../../apps/ops-agent/src/runtime"; +import { buildOpsAgentServer } from "../../../../apps/ops-agent/src/server"; +import { createTransitousLockRuntime } from "../../../../apps/ops-agent/src/transitous-lock-runtime"; +import { registerApi } from "../../src/api"; +import { registerAuth } from "../../src/auth"; +import { installDataMutationAdmission } from "../../src/jobs/execution/admission"; +import { createDataConfigurationMaintenance } from "../../src/jobs/execution/configuration-maintenance"; +import { createDataExecutionDispatcher } from "../../src/jobs/execution/dispatch"; +import { createDataExecutionReceiver } from "../../src/jobs/execution/receiver"; +import { createDataExecutionRuntime } from "../../src/jobs/execution/runtime"; +import { createPostgresDataExecutionStore } from "../../src/jobs/execution/store"; +import { createCatalogReviewService } from "../../src/jobs/transitous/catalog-review"; +import { resolveOperationsProfile } from "../../src/jobs/transitous/operations-profile"; +import { createSingleFlightController } from "../../src/jobs/transitous/single-flight"; +import { resetOpsClient } from "../../src/ops-client"; +import { StateStore } from "../../src/state"; + +const state = vi.hoisted(() => ({ + fixture: null as unknown as Awaited>, + requests: [] as string[], +})); +vi.mock("../../src/db/index", () => ({ + get sql() { + return state.fixture.sql; + }, + get rawSql() { + return state.fixture.sql; + }, + get db() { + return state.fixture.db; + }, +})); +const oldCsv = + "Country Code,Name,Location,System ID,URL,Auto-Discovery URL,Supported Versions,Authentication Info URL\nDE,Bike,Berlin,bike,https://operator.test,https://old.test/gbfs.json,2.3,\n"; +const newCsv = oldCsv.replace("https://old.test/gbfs.json", "https://new.test/gbfs.json"); +vi.mock("@openmapx/core/utils/safe-download", async (original) => ({ + ...(await original()), + safeFetchTextResponse: async (url: string) => { + state.requests.push(url); + const text = url.includes("api.github.com") + ? JSON.stringify({ sha: "c".repeat(40) }) + : url.includes("b".repeat(40)) + ? oldCsv + : newCsv; + return { text, status: 200, headers: new Headers(), finalUrl: url }; + }, +})); +const enabled = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"; +let root = "", + agent: ReturnType, + agentInstallation: ReturnType, + dmInstallation: ReturnType; +beforeAll(async () => { + if (enabled) state.fixture = await createMigratedPostgresFixture(); +}, 90000); +afterAll(async () => { + resetOpsClient(); + await agent?.close(); + if (dmInstallation) closeControlInstallation(dmInstallation); + if (agentInstallation) closeControlInstallation(agentInstallation); + if (root) rmSync(root, { recursive: true, force: true }); + if (state.fixture) await state.fixture.stop(); +}); +it.skipIf(!enabled)( + "resolves a real-Git GBFS-only proposal through actual agent HTTP and retains exact replay", + async () => { + root = mkdtempSync(join(tmpdir(), "omx-catalog-review-")); + const infra = join(root, "infra/docker"), + dataDir = join(infra, "data"), + catalog = join(dataDir, ".transitous-catalog"); + mkdirSync(join(catalog, "feeds"), { recursive: true }); + mkdirSync(join(catalog, "transitland-atlas"), { recursive: true }); + const git = (cwd: string, args: string[]) => + execFileSync("git", args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + GIT_AUTHOR_NAME: "fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }).trim(); + const atlas = join(catalog, "transitland-atlas"); + git(atlas, ["init", "--initial-branch=main"]); + mkdirSync(join(atlas, "feeds")); + writeFileSync( + join(atlas, "feeds/bus.dmfr.json"), + JSON.stringify({ feeds: [{ id: "auth", license: { spdx_identifier: "CC0-1.0" } }] }), + ); + writeFileSync(join(atlas, "README"), "Atlas fixture"); + git(atlas, ["add", "."]); + git(atlas, ["commit", "-m", "Atlas"]); + const atlasSha = git(atlas, ["rev-parse", "HEAD"]); + git(catalog, ["init", "--initial-branch=main"]); + writeFileSync( + join(catalog, "feeds/de.json"), + JSON.stringify({ + sources: [ + { + name: "bus", + spec: "gtfs", + url: "https://bus.test", + license: { name: "Test terms" }, + "transitland-atlas-id": "auth", + }, + ], + }), + ); + git(catalog, ["add", "feeds"]); + git(catalog, ["update-index", "--add", "--cacheinfo", `160000,${atlasSha},transitland-atlas`]); + git(catalog, ["commit", "-m", "Catalog"]); + const sha = git(catalog, ["rev-parse", "HEAD"]); + const remote = join(root, "upstream.git"); + git(root, ["clone", "--bare", catalog, remote]); + git(catalog, ["remote", "add", "origin", remote]); + await provisionControlInstallation(infra); + agentInstallation = openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["data-manager", "motis", "motis-staging", "motis-feed-proxy"], + }); + dmInstallation = openControlInstallation({ + infraDir: infra, + role: "data-manager", + serviceIds: ["data-manager", "motis", "motis-staging", "motis-feed-proxy"], + }); + writeFileSync( + join(infra, "transitous.lock.json"), + JSON.stringify({ + ref: `main@${sha}`, + submodules: { "transitland-atlas": atlasSha }, + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + writeFileSync( + join(infra, "gbfs-catalog.lock.json"), + JSON.stringify({ + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "b".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"b".repeat(40)}/systems.csv`, + sha256: createHash("sha256").update(oldCsv).digest("hex"), + lockedAt: "2026-10-23T00:00:00.000Z", + lockedBy: "fixture", + }), + ); + await initializeManagedCatalogPins({ rootDir: root, installation: agentInstallation }); + const originalPins = readCatalogPinSet(root), + token = "t".repeat(43), + tokenFile = join(root, "dm-token"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + agent = buildOpsAgentServer({ + tokens: { api: "a".repeat(43), "data-manager": token }, + runtime: createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: root }), + resourcePolicy: createRegistryResourcePolicy({ + serviceIds: ["motis"], + integrationIds: [], + trustedData: denyAllTrustedOpsData, + }), + }); + resetOpsClient( + new OpsClient({ baseUrl: await agent.listen({ host: "127.0.0.1", port: 0 }), tokenFile }), + ); + const fence = createDataConfigurationMaintenance(state.fixture.sql, dmInstallation); + await fence.startup(); + installDataMutationAdmission(fence); + const reviewer = createCatalogReviewService({ + dataDir, + installation: dmInstallation, + operationsPolicy: resolveOperationsProfile({ + profile: "regional-assisted", + countries: ["de"], + }), + }); + const proposalId = "00000000-0000-4000-8000-000000000001", + proposal = await reviewer.resolveCatalogProposal({ branch: "main", proposalId }); + expect(proposal.pins.transitous.ref).toBe(`main@${sha}`); + expect(proposal.pins.gbfs.commit).toBe("c".repeat(40)); + expect(proposal.baseGenerationId).toBe(originalPins!.generationId); + expect(readCatalogPinSet(root)).toEqual(originalPins); + expect(git(catalog, ["rev-parse", "HEAD"])).toBe(sha); + const page = reviewer.reviewPage(proposalId); + expect(page.digest).toBe(proposal.diffDigest); + expect(page.total).toBe(2); + expect(page.rows.every((row) => row.region === "mobilitydata.de")).toBe(true); + const requests = state.requests.length, + retainedPath = join(infra, "catalog-pins/proposals", `${proposalId}.json`), + bytes = readFileSync(retainedPath, "utf8"); + expect(await reviewer.resolveCatalogProposal({ branch: "main", proposalId })).toEqual(proposal); + expect(state.requests).toHaveLength(requests); + expect(readFileSync(retainedPath, "utf8")).toBe(bytes); + expect( + await state.fixture.sql`SELECT status FROM data_manager.jobs WHERE kind='transit-proposal'`, + ).toEqual([{ status: "ok" }, { status: "ok" }]); + const atlasRemote = join(root, "atlas-upstream.git"), + atlasWork = join(root, "atlas-upstream-work"), + catalogWork = join(root, "catalog-upstream-work"); + git(root, ["clone", "--bare", atlas, atlasRemote]); + git(atlas, ["remote", "add", "origin", atlasRemote]); + git(root, ["clone", atlasRemote, atlasWork]); + writeFileSync( + join(atlasWork, "feeds/bus.dmfr.json"), + JSON.stringify({ + feeds: [ + { + id: "auth", + authorization: { type: "api-key" }, + license: { spdx_identifier: "CC-BY-4.0", attribution: "Operator" }, + }, + ], + }), + ); + git(atlasWork, ["add", "."]); + git(atlasWork, ["commit", "-m", "Changed pinned license and credential requirement"]); + git(atlasWork, ["push", "origin", "main"]); + const nextAtlas = git(atlasWork, ["rev-parse", "HEAD"]); + expect(() => git(atlas, ["cat-file", "-e", `${nextAtlas}^{commit}`])).toThrow(); + git(root, ["clone", remote, catalogWork]); + git(catalogWork, ["checkout", "-b", "next"]); + git(catalogWork, ["update-index", "--cacheinfo", `160000,${nextAtlas},transitland-atlas`]); + git(catalogWork, ["commit", "-m", "Next Atlas gitlink"]); + git(catalogWork, ["push", "origin", "next"]); + const nextProposal = await reviewer.resolveCatalogProposal({ + branch: "next", + proposalId: "00000000-0000-4000-8000-000000000002", + }); + expect(nextProposal.pins.transitous.submodules["transitland-atlas"]).toBe(nextAtlas); + expect(git(atlas, ["rev-parse", "HEAD"])).toBe(atlasSha); + expect(git(catalog, ["rev-parse", "HEAD"])).toBe(sha); + expect(git(atlas, ["cat-file", "-t", nextAtlas])).toBe("commit"); + expect( + await inspectPinnedFeedCredentialKeys({ + catalogDir: catalog, + transitousCommit: nextProposal.pins.transitous.ref.split("@").at(-1)!, + atlasCommit: nextAtlas, + git: async (args, cwd) => git(cwd, [...args]), + }), + ).toEqual(["de/bus"]); + expect( + reviewer + .reviewPage(nextProposal.proposalId) + .rows.some((row) => row.region.startsWith("atlas.") && row.fields.includes("license")), + ).toBe(true); + expect(readCatalogPinSet(root)).toEqual(originalPins); + + // Actual route/auth/codecs plus real durable SQL/runtime/dispatcher. This + // negative validation deliberately has no installed host prerequisites; + // it must finish as error without fabricating passed evidence or pins. + const policy = resolveOperationsProfile({ profile: "regional-assisted", countries: ["de"] }); + const singleFlight = createSingleFlightController({ + db: drizzle(state.fixture.sql, { + schema: { + jobs, + jobStages, + feedState, + offlinePackageJobs, + offlinePackageJobOwners, + offlinePackageArtifactReferences, + }, + }), + admissionSql: state.fixture.sql, + }); + const dispatcher = createDataExecutionDispatcher({ + dataDir, + state: new StateStore(dataDir), + sql: state.fixture.sql, + singleFlight, + operationsPolicy: policy, + runHardlinks: async () => { + throw Error("Unexpected generation effect"); + }, + }); + const store = createPostgresDataExecutionStore(state.fixture.sql, { + controlJournal: createDataExecutionControlJournal(dmInstallation, state.fixture.sql), + configurationFence: fence, + }); + const runtime = createDataExecutionRuntime({ store, dispatch: dispatcher.dispatch }); + const receiver = createDataExecutionReceiver({ + runtime, + installation: dmInstallation, + configurationFence: fence, + dispatcher, + operationsPolicy: policy, + }); + const api = Fastify(), + serviceToken = "catalog-service-fixture-token"; + registerAuth(api, serviceToken, { executionReceiver: () => receiver }); + registerApi(api, { + dataDir, + operationsPolicy: policy, + singleFlight, + catalogReview: reviewer, + executionReceiver: receiver, + }); + const headers = { authorization: `Bearer ${serviceToken}` }; + const oldTwoSlot = process.env.MOTIS_TWO_SLOT; + process.env.MOTIS_TWO_SLOT = "false"; + try { + const read = await api.inject({ + method: "GET", + url: `/transit/pins?proposalId=${proposalId}`, + headers, + }); + expect(read.statusCode).toBe(200); + expect(read.json()).toMatchObject({ + catalogRevisionId: originalPins!.generationId, + proposal: { proposalId }, + diff: { total: 2 }, + validationEligible: false, + execution: null, + }); + const deadline = Date.now() + 120000, + key = createControlRootOperationKey({ notAfter: deadline, nonce: Buffer.alloc(16, 9) }); + const accepted = await api.inject({ + method: "POST", + url: "/executions", + headers, + payload: { + input: { kind: "catalog.validate", proposalId, countries: ["de"] }, + operationKey: key, + requestedDeadlineAt: deadline, + }, + }); + expect(accepted.statusCode).toBe(202); + const id = accepted.json().id; + await vi.waitFor(async () => expect((await receiver.inspect(id))?.status).toBe("error"), { + timeout: 10000, + interval: 50, + }); + const failed = await api.inject({ + method: "GET", + url: `/transit/pins?proposalId=${proposalId}`, + headers, + }); + expect(failed.statusCode).toBe(200); + expect(failed.json()).toMatchObject({ + validationEligible: false, + validation: null, + execution: { id, kind: "catalog.validate", status: "error" }, + }); + expect(readCatalogPinSet(root)).toEqual(originalPins); + expect(await receiver.lookup(key)).toMatchObject({ id, status: "error" }); + } finally { + if (oldTwoSlot === undefined) delete process.env.MOTIS_TWO_SLOT; + else process.env.MOTIS_TWO_SLOT = oldTwoSlot; + await api.close(); + await runtime.close(); + } + }, + 90000, +); diff --git a/services/data-manager/__tests__/transitous/catalog-validation-current.test.ts b/services/data-manager/__tests__/transitous/catalog-validation-current.test.ts new file mode 100644 index 000000000..690e14086 --- /dev/null +++ b/services/data-manager/__tests__/transitous/catalog-validation-current.test.ts @@ -0,0 +1,177 @@ +/** Signed pointer/nominal-owner component; this does not fabricate a completed validation worker. */ +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + acquireArtifactTargetLease, + closeArtifactTargetLeaseHandle, + closeControlInstallation, + controlDigest, + openControlInstallation, + provisionControlInstallation, + retainAdmittedControlExecution, + signControl, +} from "@openmapx/core/build-artifacts"; +import { + CATALOG_VALIDATION_DOMAIN, + catalogValidationRecordSchema, + readCurrentCatalogValidation, + withCatalogProposalDirectory, +} from "@openmapx/core/catalog-pins"; +import { + canonicalJson, + createControlRootOperationKey, + executionFingerprint, +} from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { installationState } from "../../../../packages/core/src/build-artifacts/control-authority"; +import { publishCatalogValidationCurrent } from "../../src/jobs/transitous/catalog-validation-current"; + +async function fixture(kind: "catalog.validate" | "transit-sync" | "download-fonts") { + const root = mkdtempSync(join(tmpdir(), "omx-catalog-current-")), + dataDir = join(root, "data"), + proposalId = randomUUID(), + id = randomUUID(); + mkdirSync(join(dataDir, "motis/proposals", proposalId), { recursive: true, mode: 0o700 }); + await provisionControlInstallation(root); + const ids = ["data-manager", "motis", "motis-feed-proxy", "motis-staging"], + installation = openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ids, + }); + const input = + kind === "catalog.validate" + ? { kind, proposalId, countries: ["de"] } + : kind === "transit-sync" + ? { kind, countries: ["de"] } + : { kind }; + const deadlineAt = Date.now() + 120000, + operationFingerprint = controlDigest(input), + execution = { + executor: "data-manager" as const, + principal: "data-manager" as const, + executionId: id, + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind, operationFingerprint, deadlineAt }), + }; + const authority = retainAdmittedControlExecution(installation, { + execution, + kind, + operation: input, + admittedAt: Date.now(), + requestedDeadlineAt: deadlineAt, + serviceIds: ids, + }), + lease = await acquireArtifactTargetLease({ + dataRoot: dataDir, + owner: authority, + serviceIds: ids, + }); + return { + root, + dataDir, + proposalId, + installation, + execution, + authority, + lease, + close() { + closeArtifactTargetLeaseHandle(lease); + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + }, + }; +} +it("selects only original signed validation bytes, then invalidates before reuse", async () => { + const f = await fixture("catalog.validate"); + try { + const completedAt = new Date().toISOString(), + d = "a".repeat(64), + inputs = { + configurationRevisionId: `cfg1_${"a".repeat(43)}`, + credentialsRevision: d, + sourcePolicyRevision: d, + overlayDigest: d, + imageDigest: `sha256:${d}`, + stagingState: "stopped", + requiredKeys: [], + configuredKeys: [], + }; + const record = catalogValidationRecordSchema.parse({ + version: 1, + installationId: f.installation.installationId, + sourceExecution: f.execution, + proposalDigest: d, + runtimeInputs: inputs, + validation: { + status: "passed", + validationId: f.execution.executionId, + proposalId: f.proposalId, + pinDigest: d, + diffDigest: d, + sourcePolicyRevision: d, + overlayDigest: d, + credentialsRevision: d, + countries: ["de"], + slot: "B", + epoch: f.execution.executionId, + manifestHash: d, + imageDigest: `sha256:${d}`, + completedAt, + expiresAt: new Date(Date.parse(completedAt) + 86400000).toISOString(), + }, + }); + withCatalogProposalDirectory(f.installation, f.proposalId, (dir) => + dir.write( + `validation-${f.execution.executionId}.json`, + canonicalJson( + signControl( + installationState(f.installation).privateKey, + CATALOG_VALIDATION_DOMAIN, + record, + ), + ), + ), + ); + publishCatalogValidationCurrent(f.authority, f.lease, f.proposalId); + expect(readCurrentCatalogValidation(f.installation)?.selected).toEqual({ + proposalId: f.proposalId, + validationId: f.execution.executionId, + validationDigest: controlDigest(record), + }); + publishCatalogValidationCurrent(f.authority, f.lease); + expect(readCurrentCatalogValidation(f.installation)?.selected).toBeNull(); + } finally { + f.close(); + } +}); +it("permits ordinary owned transit invalidation but not a different kind or a forged nominal owner", async () => { + const ordinary = await fixture("transit-sync"); + try { + publishCatalogValidationCurrent(ordinary.authority, ordinary.lease); + expect(readCurrentCatalogValidation(ordinary.installation)?.sourceExecution).toEqual( + ordinary.execution, + ); + expect(() => + publishCatalogValidationCurrent({} as typeof ordinary.authority, ordinary.lease), + ).toThrow(/Unverified/); + expect(() => + publishCatalogValidationCurrent(ordinary.authority, ordinary.lease, ordinary.proposalId), + ).toThrow(/Only the original validation/); + } finally { + ordinary.close(); + } + const foreign = await fixture("download-fonts"); + try { + expect(() => publishCatalogValidationCurrent(foreign.authority, foreign.lease)).toThrow( + /Original complete transit/, + ); + expect(readCurrentCatalogValidation(foreign.installation)).toBeNull(); + } finally { + foreign.close(); + } +}); diff --git a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts new file mode 100644 index 000000000..ac0a3cf6d --- /dev/null +++ b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts @@ -0,0 +1,970 @@ +/** Actual durable workers + PostgreSQL + a separate production agent. Only + * external Docker/MOTIS IO and pinned upstream CSV bytes are synthetic here. */ +import { execFile as execFileCallback, execFileSync, spawn } from "node:child_process"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { + cpSync, + existsSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { promisify } from "node:util"; +import * as control from "@openmapx/core/build-artifacts"; +import { + createCatalogPinSet, + initializeManagedCatalogPins, + readCatalogPinSet, + readCatalogValidation, +} from "@openmapx/core/catalog-pins"; +import * as sdk from "@openmapx/core/data-manager-sdk"; +import { createControlRootOperationKey, OpsClient } from "@openmapx/core/ops"; +import { + feedState, + jobStages, + jobs, + offlinePackageArtifactReferences, + offlinePackageJobOwners, + offlinePackageJobs, +} from "@openmapx/db-schema"; +import { applyHardlinkPlan } from "@openmapx/hardlinks"; +import { renderFeedProxyNginxConfig } from "@openmapx/motis-feed-proxy-config"; +import { + catalogCompilerSettings, + catalogReviewInputDigests, + resolveOperationsProfileFromEnv, +} from "@openmapx/transitous-core"; +import { drizzle } from "drizzle-orm/postgres-js"; +import Fastify from "fastify"; +import { expect, it, vi } from "vitest"; +import { createMigratedPostgresFixture } from "../../../../apps/api/src/test/postgres-fixture"; +import { ownedServiceHost } from "../../../../apps/ops-agent/src/__fixtures__/owned-service-host"; +import { runContainedProcess } from "../../../../apps/ops-agent/src/docker-runtime"; +import { buildTransitousRunnerServer } from "../../../../apps/transitous-runner/src/server"; +import { ensureMotisSlotLayout } from "../../src/jobs/transitous/slot-state"; +import { buildTinyGtfsFeeds } from "./fixtures/build-tiny-gtfs"; +import { BERLIN_TINY_OSM_PBF_BASE64 } from "./fixtures/tiny-osm"; + +const state = vi.hoisted(() => ({ + db: null as unknown as Awaited>, +})); +vi.mock("../../src/db/index", () => ({ + get db() { + return state.db.db; + }, + get sql() { + return state.db.sql; + }, + get rawSql() { + return state.db.sql; + }, +})); +const csv = + "Country Code,Name,Location,System ID,URL,Auto-Discovery URL,Supported Versions,Authentication Info URL\n"; +vi.mock("@openmapx/core/utils/safe-download", async (original) => ({ + ...(await original()), + safeFetchTextResponse: async (url: string) => ({ + text: csv, + status: 200, + headers: new Headers(), + finalUrl: url, + }), +})); +const enabled = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"; +const execFile = promisify(execFileCallback); +const MOTIS_IMAGE = + "ghcr.io/motis-project/motis:2.11.3@sha256:7126a80f8d1537426655a14a121e395e9d56451c36144f3a1ce5c3e4deb38f64"; +let motisPort = 0; + +it + .skipIf(!enabled) + .each([ + "none", + "pointer", + "before-pointer", + "dm-pointer", + "cancel-pointer", + "runner-refusal", + ...(process.env.OPENMAPX_CATALOG_LIVE_MOTIS === "1" ? (["live"] as const) : []), + ] as const)( + "validates then activates one exact pair through actual workers / separate production agent (%s)", + async (fault) => { + state.db = await createMigratedPostgresFixture(); + const motis = Fastify(), + bridge = Fastify(); + let agent: ReturnType | undefined, + host: Awaited> | undefined, + dm: ReturnType | undefined, + runner: ReturnType | undefined, + installation: control.ControlInstallation | undefined, + agentInstallation: control.ControlInstallation | undefined, + journal: ReturnType | undefined, + runtime: + | Awaited< + ReturnType< + typeof import("../../src/jobs/execution/runtime")["createDataExecutionRuntime"] + > + > + | undefined; + const children: ReturnType[] = []; + const liveNames = new Set(), + livePorts = new Map(); + const containerPrefix = `openmapx-catalog-${randomUUID()}`; + async function removeLive(id: string) { + const name = `${containerPrefix}-${id}`; + if (liveNames.has(name)) { + await execFile("docker", ["rm", "-f", name]); + liveNames.delete(name); + } + livePorts.delete(id); + } + try { + motis.all("/*", async (request, reply) => { + if (fault === "live") { + const [id, ...path] = request.url.slice(1).split("/"), + port = livePorts.get(id!); + if (!port) return reply.code(503).send({ error: "Owned MOTIS is stopped" }); + try { + const response = await fetch(`http://127.0.0.1:${port}/${path.join("/")}`, { + signal: AbortSignal.timeout(5000), + }); + return reply + .code(response.status) + .type(response.headers.get("content-type") ?? "application/json") + .send(await response.text()); + } catch { + return reply.code(503).send({ error: "Owned MOTIS is importing" }); + } + } + return request.url.includes("/health") + ? { rt: true, gbfs: true } + : request.url.includes("/map/initial") + ? { lat: 52.525, lon: 13.369, zoom: 10, serverConfig: {} } + : request.url.includes("/map/stops") + ? [] + : { itineraries: [{}], direct: [], requestParameters: {}, debugOutput: {} }; + }); + const fetchOriginal = globalThis.fetch; + let lostCancellationResponse = false; + vi.stubGlobal( + "fetch", + async (input: Parameters[0], init?: Parameters[1]) => { + const url = String(input); + if ( + /^https:\/\/raw\.githubusercontent\.com\/MobilityData\/gbfs\/[bc]{40}\/systems\.csv$/.test( + url, + ) + ) + return new Response(csv, { headers: { "content-type": "text/csv" } }); + const response = await fetchOriginal(input, init); + if ( + fault === "cancel-pointer" && + init?.method === "POST" && + /\/executions\/[^/]+\/cancel$/.test(url) && + !lostCancellationResponse + ) { + lostCancellationResponse = true; + throw new TypeError("Fixture dropped the actual cancellation response"); + } + return response; + }, + ); + const motisUrl = await motis.listen({ host: "127.0.0.1", port: motisPort }); + motisPort = Number(new URL(motisUrl).port); + const environment = { + MOTIS_TWO_SLOT: "true", + MOTIS_GBFS_CATALOG_ENABLED: "true", + MOTIS_OPERATIONS_PROFILE: "regional-assisted", + TRANSITOUS_COUNTRIES: "de", + TRANSIT_SOURCE: "build", + }; + for (const [key, value] of Object.entries({ + ...environment, + MOTIS_URL: `${motisUrl}/primary`, + MOTIS_STAGING_URL: `${motisUrl}/staging`, + MOTIS_HEALTH_PLAN_TO_LAT: "52.521", + MOTIS_HEALTH_PLAN_TO_LNG: "13.413", + MOTIS_IMPORT_TIMEOUT_MS: "60000", + GIT_ALLOW_PROTOCOL: "file", + })) + vi.stubEnv(key, value); + host = await ownedServiceHost({ + withCoordinator: true, + extraServiceIds: ["motis-staging", "motis-feed-proxy"], + environment: { "data-manager": environment }, + actualComposeConfig: true, + }); + const f = host, + data = join(f.infraDir, "data"), + catalog = join(data, ".transitous-catalog"), + atlas = join(catalog, "transitland-atlas"); + const feeds = await buildTinyGtfsFeeds(join(f.root, "tiny-feeds")); + mkdirSync(join(catalog, "feeds"), { recursive: true }); + mkdirSync(join(atlas, "feeds"), { recursive: true }); + mkdirSync(join(f.root, "services/motis/tools/transitous"), { recursive: true }); + cpSync( + resolve("services/data-manager/__tests__/transitous/fixtures/stub-catalog-scripts"), + join(catalog, "src"), + { recursive: true }, + ); + const git = (cwd: string, args: string[]) => + execFileSync("git", args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + GIT_AUTHOR_NAME: "fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }).trim(); + git(atlas, ["init", "--initial-branch=main"]); + writeFileSync( + join(atlas, "feeds/feed.dmfr.json"), + '{"feeds":[{"id":"f-outside","authorization":{"type":"query_param","param_name":"key"}}]}', + ); + git(atlas, ["add", "."]); + git(atlas, ["commit", "-m", "Atlas"]); + if (fault === "live") + writeFileSync( + join(catalog, "berlin-tiny.osm.pbf"), + Buffer.from(BERLIN_TINY_OSM_PBF_BASE64, "base64"), + ); + const atlasSha = git(atlas, ["rev-parse", "HEAD"]), + feed = feeds.entries.find((entry) => entry.region === "de")!; + git(catalog, ["init", "--initial-branch=main"]); + writeFileSync( + join(catalog, "feeds/de.json"), + JSON.stringify({ + sources: [ + { + name: "demo", + type: "url", + url: `file://${feed.zipPath}`, + license: { name: "Synthetic test terms" }, + }, + ], + }), + ); + writeFileSync( + join(catalog, "feeds/ca.json"), + JSON.stringify({ sources: [{ name: "outside", "transitland-atlas-id": "f-outside" }] }), + ); + writeFileSync( + join(catalog, ".gitmodules"), + `[submodule "transitland-atlas"]\npath = transitland-atlas\nurl = ${atlas}\n`, + ); + git(catalog, [ + "add", + "feeds", + "src", + ".gitmodules", + ...(fault === "live" ? ["berlin-tiny.osm.pbf"] : []), + ]); + git(catalog, [ + "update-index", + "--add", + "--cacheinfo", + `160000,${atlasSha},transitland-atlas`, + ]); + git(catalog, ["commit", "-m", "Catalog"]); + const sha = git(catalog, ["rev-parse", "HEAD"]), + remote = join(f.root, "upstream.git"); + git(f.root, ["clone", "--bare", catalog, remote]); + git(catalog, ["remote", "add", "origin", remote]); + git(catalog, ["fetch", "origin"]); + git(catalog, ["branch", "--set-upstream-to=origin/main", "main"]); + vi.stubEnv("TRANSITOUS_REPO_URL", remote); + const lockedAt = new Date().toISOString(), + oldGbfs = "b".repeat(40), + nextGbfs = "c".repeat(40); + writeFileSync( + join(f.infraDir, "transitous.lock.json"), + JSON.stringify({ + ref: `main@${sha}`, + submodules: { "transitland-atlas": atlasSha }, + lockedAt, + lockedBy: "fixture", + }), + ); + const gbfs = { + schemaVersion: 1 as const, + source: "mobilitydata-gbfs" as const, + commit: oldGbfs, + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${oldGbfs}/systems.csv`, + sha256: createHash("sha256").update(csv).digest("hex"), + lockedAt, + lockedBy: "fixture", + }; + writeFileSync(join(f.infraDir, "gbfs-catalog.lock.json"), JSON.stringify(gbfs)); + const serviceIds = f.loaded.map((service) => service.manifest.id); + installation = control.openControlInstallation({ + infraDir: f.infraDir, + role: "data-manager", + serviceIds, + }); + agentInstallation = control.openControlInstallation({ + infraDir: f.infraDir, + role: "ops-agent", + serviceIds, + }); + await initializeManagedCatalogPins({ rootDir: f.root, installation: agentInstallation }); + const active = readCatalogPinSet(f.root)!; + mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); + writeFileSync( + join(data, "motis-feed-proxy/conf/default.conf"), + renderFeedProxyNginxConfig({}), + ); + writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "{}\n"); + applyHardlinkPlan( + [ + { + source: "motis-feed-proxy/conf", + target: "motis-feed-proxy/motis-feed-proxy-config", + consumerService: "motis-feed-proxy", + dataType: "motis-feed-proxy-config", + }, + ], + { rootDir: data }, + ); + const rows = ["motis", "data-manager", "motis-staging", "motis-feed-proxy"].map( + (id, index) => ({ + ...structuredClone(f.row), + Id: String(index + 1).repeat(64), + ConfigImage: `example/${id}:1`, + Labels: { ...f.row.Labels, "com.docker.compose.service": id }, + }), + ); + f.setRows(rows); + f.setRuntimeInspections( + Object.fromEntries( + rows.map((row) => [ + row.Id, + { + Config: { + Env: + row.Labels["com.docker.compose.service"] === "data-manager" + ? Object.entries(environment).map(([key, value]) => `${key}=${value}`) + : [], + }, + Mounts: + row.Labels["com.docker.compose.service"] === "data-manager" + ? [ + { + Source: join(f.root, "services/motis/tools/transitous"), + Destination: "/config/transitous", + RW: false, + }, + ] + : [], + }, + ]), + ), + ); + const calls: string[][] = []; + bridge.post("/docker", async (request) => { + const { args } = request.body as { args: string[] }; + calls.push(args); + const filter = args.find((value) => value.startsWith("name=^/")); + if (args[0] === "container" && args[1] === "ls" && filter) { + const id = filter.slice(7, -1); + const row = rows.find((row) => row.Labels["com.docker.compose.service"] === id); + return { stdout: row ? `${id}\t${row.State}\n` : "", stderr: "" }; + } + if (args[0] === "stop") { + if (fault === "live") await removeLive(args[1] === "motis" ? "primary" : "staging"); + const row = rows.find((row) => row.Labels["com.docker.compose.service"] === args[1]); + if (row) row.State = "exited"; + return { stdout: "", stderr: "" }; + } + if (args[0] === "compose" && args.includes("up")) { + if (fault === "live") { + const id = args.at(-1) === "motis" ? "primary" : "staging", + name = `${containerPrefix}-${id}`; + await removeLive(id); + const mount = realpathSync( + join(data, id === "primary" ? "motis/live" : "motis/staging"), + ); + liveNames.add(name); + await execFile( + "docker", + [ + "run", + "-d", + "--name", + name, + "--memory=1g", + "--cpus=2", + "--user", + `${process.getuid!()}:${process.getgid!()}`, + "-p", + "127.0.0.1::8080", + "--mount", + `type=bind,src=${mount},dst=/motis-data`, + "--workdir", + "/motis-data", + "--entrypoint", + "/bin/sh", + MOTIS_IMAGE, + "-c", + "/motis import && exec /motis server", + ], + { timeout: 30000 }, + ); + const { stdout } = await execFile("docker", ["port", name, "8080/tcp"]); + livePorts.set(id, stdout.trim().split(":").at(-1)!); + } + const row = rows.find((row) => row.Labels["com.docker.compose.service"] === args.at(-1)); + if (row) row.State = "running"; + } + if (args[0] === "exec") { + // Execute the real fixed validation shell/nonce protocol, substituting + // only the nginx binary normally provided by Docker. + const bin = join(f.root, "fixture-bin"); + mkdirSync(bin, { recursive: true }); + writeFileSync(join(bin, "nginx"), "#!/bin/sh\nexit 0\n", { mode: 0o700 }); + return runContainedProcess( + "/bin/sh", + [ + "-c", + 'PATH="$1:$PATH"; export PATH; shift; exec "$@"', + "docker-io", + bin, + ...args.slice(2), + ], + { signal: AbortSignal.timeout(30000), timeout: 30000, maxBuffer: 65536 }, + ); + } + return f.execFile("docker", args, { + signal: AbortSignal.timeout(30000), + timeout: 30000, + maxBuffer: 1024 * 1024, + }); + }); + const dockerBridge = (await bridge.listen({ host: "127.0.0.1", port: 0 })) + "/docker"; + const require = createRequire(import.meta.url), + loader = require.resolve("tsx/esm"); + async function startAgent(catalogFault: string, listenPort = 0) { + const child = spawn( + process.execPath, + [ + "--import", + loader, + resolve("apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs"), + ], + { stdio: ["pipe", "pipe", "pipe"] }, + ); + children.push(child); + let stderr = ""; + child.stderr!.on("data", (chunk) => { + stderr += String(chunk); + }); + const lines = createInterface({ input: child.stdout! }), + iterator = lines[Symbol.asyncIterator](); + child.stdin!.end( + JSON.stringify({ + root: f.root, + infraDir: f.infraDir, + queue: f.queue, + stack: f.stack, + serviceIds, + token: f.token, + dockerBridge, + catalogFault: catalogFault === "dm-pointer" ? "pointer" : catalogFault, + listenPort, + operationTimeoutMs: 3600000, + }), + ); + const events: Record[] = []; + let readyResolve!: (value: string) => void, + readyReject!: (reason: unknown) => void, + stoppedResolve!: () => void; + const ready = { + promise: new Promise((resolve, reject) => { + readyResolve = resolve; + readyReject = reject; + }), + resolve: (value: string) => readyResolve(value), + reject: (reason: unknown) => readyReject(reason), + }, + stopped = { + promise: new Promise((resolve) => { + stoppedResolve = resolve; + }), + resolve: () => stoppedResolve(), + }; + void (async () => { + for await (const line of { [Symbol.asyncIterator]: () => iterator }) { + const event = JSON.parse(line); + events.push(event); + if (event.event === "ready") ready.resolve(event.url); + if ( + ["catalog-pointer-committed", "catalog-pointer-before-commit"].includes( + String(event.event), + ) + ) + stopped.resolve(); + } + })().catch(ready.reject); + child.once("exit", (code) => { + ready.reject(Error(`Native agent exited ${code}: ${stderr}`)); + }); + const url = await Promise.race([ + ready.promise, + new Promise((_, reject) => + setTimeout( + () => reject(Error(`Native agent startup timeout: ${stderr}`)), + 30000, + ).unref(), + ), + ]); + return { + child, + url, + stopped: stopped.promise, + events, + get stderr() { + return stderr; + }, + }; + } + let processAgent = await startAgent(fault); + agent = processAgent.child; + const { resetOpsClient } = await import("../../src/ops-client"); + const tokenFile = join(f.root, "dm-token"), + token = Buffer.alloc(32, 23).toString("base64url"); + writeFileSync(tokenFile, token, { mode: 0o600 }); + resetOpsClient(new OpsClient({ baseUrl: processAgent.url, tokenFile })); + const ops = new OpsClient({ baseUrl: processAgent.url, tokenFile }); + const pins = createCatalogPinSet(active.transitous, { + ...gbfs, + commit: nextGbfs, + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${nextGbfs}/systems.csv`, + }); + const proposalId = randomUUID(), + policy = resolveOperationsProfileFromEnv(environment), + source = catalogReviewInputDigests(policy, null, catalogCompilerSettings(environment)); + const proposalDeadline = Date.now() + 60000; + await ops.execute( + { + kind: "transitousLock.propose", + proposalId, + baseGenerationId: active.generationId, + pins, + diffDigest: "d".repeat(64), + sourcePolicyRevision: source.sourcePolicyRevision, + }, + { + operationKey: createControlRootOperationKey({ + notAfter: proposalDeadline, + nonce: randomBytes(16), + }), + execution: { requestedDeadlineAt: proposalDeadline }, + }, + ); + mkdirSync(join(data, "motis/proposals", proposalId), { recursive: true, mode: 0o700 }); + const keyText = randomBytes(32).toString("hex"), + capabilityKey = Buffer.from(keyText), + keyFile = join(f.root, "runner-key"); + writeFileSync(keyFile, keyText, { mode: 0o600 }); + let runnerRefused = false; + runner = buildTransitousRunnerServer({ + catalogDir: catalog, + stagingDir: join(data, "gtfs"), + capabilityKey: fault === "runner-refusal" ? randomBytes(64) : capabilityKey, + }); + runner.addHook("onSend", async (_request, reply, payload) => { + if (reply.statusCode === 401) runnerRefused = true; + return payload; + }); + vi.stubEnv("TRANSITOUS_RUNNER_URL", await runner.listen({ host: "127.0.0.1", port: 0 })); + vi.stubEnv("TRANSITOUS_RUNNER_CAPABILITY_KEY_FILE", keyFile); + const { createDataConfigurationMaintenance } = await import( + "../../src/jobs/execution/configuration-maintenance" + ), + { createPostgresDataExecutionStore } = await import("../../src/jobs/execution/store"), + { createDataExecutionRuntime } = await import("../../src/jobs/execution/runtime"), + { createDataExecutionReceiver } = await import("../../src/jobs/execution/receiver"), + { createDataExecutionDispatcher } = await import("../../src/jobs/execution/dispatch"), + { createSingleFlightController } = await import("../../src/jobs/transitous/single-flight"), + { StateStore } = await import("../../src/state"), + { registerAuth } = await import("../../src/auth"), + { registerApi } = await import("../../src/api"); + journal = control.createDataExecutionControlJournal(installation, state.db.sql); + const fence = createDataConfigurationMaintenance(state.db.sql, installation); + await fence.startup(); + const store = createPostgresDataExecutionStore(state.db.sql, { + controlJournal: journal, + configurationFence: fence, + }), + singleFlight = createSingleFlightController({ + db: drizzle(state.db.sql, { + schema: { + jobs, + jobStages, + feedState, + offlinePackageJobs, + offlinePackageJobOwners, + offlinePackageArtifactReferences, + }, + }), + }), + dispatcher = createDataExecutionDispatcher({ + dataDir: data, + state: new StateStore(data), + sql: state.db.sql, + singleFlight, + operationsPolicy: policy, + runHardlinks: async () => { + throw Error("No generation operation in catalog workflow"); + }, + }); + runtime = createDataExecutionRuntime({ store, dispatch: dispatcher.dispatch }); + const receiver = createDataExecutionReceiver({ + runtime, + installation, + configurationFence: fence, + dispatcher, + operationsPolicy: policy, + }); + dm = Fastify(); + registerAuth(dm, token, { executionReceiver: () => receiver }); + registerApi(dm, { + dataDir: data, + singleFlight, + operationsPolicy: policy, + executionReceiver: receiver, + }); + let dmUrl = await dm.listen({ host: "127.0.0.1", port: 0 }), + transport = sdk.createDataManagerTransport({ baseUrl: dmUrl, authToken: token }); + async function startNativeDm(listenPort = 0) { + const child = spawn( + process.execPath, + [ + "--import", + loader, + resolve("services/data-manager/__tests__/fixtures/catalog-data-manager-process.mjs"), + ], + { stdio: ["pipe", "pipe", "pipe"] }, + ); + children.push(child); + let stderr = ""; + child.stderr!.on("data", (value) => { + stderr += String(value); + }); + child.stdin!.end( + JSON.stringify({ + databaseUrl: state.db.connectionUri, + agentUrl: processAgent.url, + tokenFile, + infraDir: f.infraDir, + dataDir: data, + serviceIds, + token, + csv, + listenPort, + environment: Object.fromEntries( + Object.entries(process.env).filter(([key]) => + /^(MOTIS|TRANSITOUS|TRANSIT_SOURCE|GIT_ALLOW_PROTOCOL)/.test(key), + ), + ), + }) + "\n", + ); + const lines = createInterface({ input: child.stdout! }); + const ready = new Promise((resolve, reject) => { + child.once("exit", (code) => reject(Error(`Native DM exited ${code}: ${stderr}`))); + void (async () => { + for await (const line of lines) { + const event = JSON.parse(line); + if (event.event === "ready") resolve(event.url); + else stderr += line + "\n"; + } + })().catch(reject); + }); + const url = await Promise.race([ + ready, + new Promise((_, reject) => + setTimeout(() => reject(Error(`Native DM startup timeout: ${stderr}`)), 60000).unref(), + ), + ]); + return { + child, + url, + get stderr() { + return stderr; + }, + }; + } + let nativeDm: Awaited> | undefined; + if (fault === "dm-pointer" || fault === "runner-refusal" || fault === "before-pointer") { + await runtime.close(); + nativeDm = await startNativeDm(); + transport = sdk.createDataManagerTransport({ baseUrl: nativeDm.url, authToken: token }); + } + async function terminal(id: string) { + let status = sdk.unwrapDataManagerResponse(await sdk.getDataExecution(id, { transport })); + const until = Date.now() + 90000; + while ( + ["queued", "running", "cancel_pending"].includes(status.status) && + Date.now() < until && + !( + status.publication === "unknown" && + (fault === "runner-refusal" || + status.diagnostics.tail.some((line) => line.stream === "stderr")) + ) + ) { + await new Promise((resolve) => setTimeout(resolve, 100)); + status = sdk.unwrapDataManagerResponse(await sdk.getDataExecution(id, { transport })); + if ( + processAgent.events.some( + (event) => event.event === "operation" && Number(event.status) >= 400, + ) + ) + return status; + } + return status; + } + ensureMotisSlotLayout(data); + const deadline = Date.now() + 3600000; + const validation = sdk.unwrapDataManagerResponse( + await sdk.startDataExecution( + { + input: { kind: "catalog.validate", proposalId, countries: ["de"] }, + operationKey: createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + requestedDeadlineAt: deadline, + }, + { transport }, + ), + ); + const validated = await terminal(validation.id); + if (fault === "runner-refusal") { + await vi.waitFor(() => expect(runnerRefused).toBe(true), { timeout: 30000 }); + expect(validated.publication).toBe("unknown"); + expect(["running", "interrupted", "cancel_pending"]).toContain(validated.status); + const [row] = await state.db + .sql`SELECT finished_at FROM data_manager.jobs WHERE id=${validation.id}`; + expect(row!.finished_at).toBeNull(); + expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(true); + return; + } + expect( + validated, + JSON.stringify({ + status: validated, + agent: processAgent.events, + stages: await state.db + .sql`SELECT stage,status,message,artifacts FROM data_manager.job_stages WHERE job_id=${validation.id}`, + }), + ).toMatchObject({ status: "ok", result: { kind: "catalog.validate", status: "passed" } }); + const proof = readCatalogValidation(installation, proposalId, validation.id); + expect(proof.validation.status).toBe("passed"); + expect(readCatalogPinSet(f.root)).toEqual(active); + // Like the review eligibility reader, require the real immutable contained + // completion, not merely the earlier visible SQL terminal projection. + await vi.waitFor( + () => + expect(() => + control.verifyDataControlTerminalWitness(installation!, proof.sourceExecution), + ).not.toThrow(), + { timeout: 10000, interval: 100 }, + ); + const completedValidation = await store.inspect(validation.id); + const child = completedValidation!.children.find((child) => child.state === "terminal"); + if (!child || child.state !== "terminal") + throw Error("Expected actual settled validation child"); + const originalOwner = control.restoreAdmittedControlExecution( + installation, + proof.sourceExecution, + ); + expect( + control.inspectRetainedOwnedArtifactTargetChildSettlement(originalOwner, child.execution) + ?.outcome, + ).toBe("committed"); + expect(() => + control.inspectRetainedOwnedArtifactTargetChildSettlement(originalOwner, { + ...child.execution, + executionId: randomUUID(), + }), + ).toThrow(/original child/); + expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(false); + const validationBytes = readFileSync( + join(data, "motis/proposals", proposalId, `validation-${validation.id}.json`), + ); + const originalPointer = readFileSync(join(f.infraDir, "catalog-pins/current.json")); + const activation = sdk.unwrapDataManagerResponse( + await sdk.startDataExecution( + { + input: { + kind: "catalog.activate", + proposalId, + validationId: validation.id, + expectedGenerationId: active.generationId, + diffDigest: "d".repeat(64), + acknowledgeLicenseChanges: true, + }, + operationKey: createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + requestedDeadlineAt: deadline, + }, + { transport }, + ), + ); + if ( + fault === "pointer" || + fault === "dm-pointer" || + fault === "cancel-pointer" || + fault === "before-pointer" + ) { + await Promise.race([ + processAgent.stopped, + new Promise((_, reject) => + setTimeout(() => reject(Error("Pin pointer fault not reached")), 90000).unref(), + ), + ]); + expect(readCatalogPinSet(f.root)?.generationId).toBe( + fault === "before-pointer" ? active.generationId : pins.generationId, + ); + if (fault === "cancel-pointer") { + const cancelResponse = sdk + .cancelDataExecution( + activation.id, + { operationKey: activation.operationKey, fingerprint: activation.fingerprint }, + { transport }, + ) + .then( + () => false, + () => true, + ); + await vi.waitFor( + async () => + expect( + sdk.unwrapDataManagerResponse( + await sdk.getDataExecution(activation.id, { transport }), + ).cancellationRequestedAt, + ).not.toBeNull(), + { timeout: 15000 }, + ); + expect(await cancelResponse).toBe(true); + expect(lostCancellationResponse).toBe(true); + expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(true); + } + if (nativeDm) { + nativeDm.child.kill("SIGKILL"); + await new Promise((resolve) => nativeDm!.child.once("exit", resolve)); + } + agent!.kill("SIGKILL"); + await new Promise((resolve) => agent!.once("exit", resolve)); + processAgent = await startAgent("none", Number(new URL(processAgent.url).port)); + agent = processAgent.child; + resetOpsClient(new OpsClient({ baseUrl: processAgent.url, tokenFile })); + if (nativeDm) { + nativeDm = await startNativeDm(Number(new URL(nativeDm.url).port)); + transport = sdk.createDataManagerTransport({ baseUrl: nativeDm.url, authToken: token }); + } + + // Existing pending child addresses retain the original agent endpoint. + // The same owned loopback port is restored below by the native fixture. + } + if (fault === "before-pointer") { + const retained = await store.inspect(activation.id), + approval = retained!.children.find((child) => child.phase === "paired-pin-commit"); + if (!approval || approval.state === "reserved" || approval.state === "bound") + throw Error("Original attempted approval required"); + expect(await ops.status("transitousLock.approve", approval.proposedId)).toMatchObject({ + state: "failed", + errorClass: "recovery_required", + }); + expect(readFileSync(join(f.infraDir, "catalog-pins/current.json"))).toEqual( + originalPointer, + ); + expect(processAgent.events.some((event) => event.kind === "transitousLock.approve")).toBe( + false, + ); + const pending = sdk.unwrapDataManagerResponse( + await sdk.getDataExecution(activation.id, { transport }), + ); + expect(pending.publication).toBe("unknown"); + await sdk.cancelDataExecution( + activation.id, + { operationKey: activation.operationKey, fingerprint: activation.fingerprint }, + { transport }, + ); + const [row] = await state.db + .sql`SELECT finished_at FROM data_manager.jobs WHERE id=${activation.id}`; + expect(row!.finished_at).toBeNull(); + expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(true); + const next = await sdk.startDataExecution( + { + input: { kind: "catalog.validate", proposalId, countries: ["de"] }, + operationKey: createControlRootOperationKey({ + notAfter: deadline, + nonce: randomBytes(16), + }), + requestedDeadlineAt: deadline, + }, + { transport }, + ); + expect(next.status).toBe(409); + return; + } + const activated = await terminal(activation.id); + expect( + activated, + JSON.stringify({ + status: activated, + agent: processAgent.events, + agentDiagnostics: processAgent.stderr, + nativeDmDiagnostics: nativeDm?.stderr, + }), + ).toMatchObject({ + status: "ok", + result: { outcome: "activated", generationId: pins.generationId }, + }); + expect(readCatalogPinSet(f.root)?.generationId).toBe(pins.generationId); + expect( + readFileSync(join(data, "motis/proposals", proposalId, `validation-${validation.id}.json`)), + ).toEqual(validationBytes); + expect( + calls.filter( + (args) => args[0] === "compose" && args.includes("up") && args.at(-1) === "motis", + ), + ).toHaveLength(1); + } finally { + await runtime?.close(); + for (const id of ["primary", "staging"]) await removeLive(id); + for (const child of children) { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + await new Promise((resolve) => child.once("exit", resolve)); + } + } + if (dm) { + dm.server.closeAllConnections(); + await dm.close(); + } + await runner?.close(); + await bridge.close(); + await motis.close(); + journal?.close(); + if (installation) control.closeControlInstallation(installation); + if (agentInstallation) control.closeControlInstallation(agentInstallation); + if (host) rmSync(host.cleanupRoot, { recursive: true, force: true }); + await state.db.stop(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + } + }, + 240000, +); diff --git a/services/data-manager/__tests__/transitous/fixtures/tiny-osm.ts b/services/data-manager/__tests__/transitous/fixtures/tiny-osm.ts new file mode 100644 index 000000000..832b9ed2a --- /dev/null +++ b/services/data-manager/__tests__/transitous/fixtures/tiny-osm.ts @@ -0,0 +1,3 @@ +/** Generated from the adjacent hand-authored berlin-tiny.osm; bounded shared MOTIS fixture. */ +export const BERLIN_TINY_OSM_PBF_BASE64 = + "AAAADQoJT1NNSGVhZGVyGFUQSBpReJzjkuJoWLR3RbJAw7I37SkSDd9OHW5nVmjY8nlyO7MSn39xbnByRmpuom6YgZ6ZEpdLal5xql9+SmpxEyNvfnFuZmmuvqGeoaWeAQAnRhqIAAAACwoHT1NNRGF0YRhoENcBGmN4nONi4WLgYhA6xyh0mpFLgAkK2KC0VhSXACMaEBJgQANSGCJKGCJaAkxoIk5iC7Z//cIM5f3/wAJleRk2TJ9T37BVYsE/rgfbeA/M4gGiBe/5Hxzl+H85Eat4EIZ1APJ9I7YAAAAMCgdPU01EYXRhGJ0BEOgBGpcBeJzjyuBi4GLPyEzPKE+s5GLJS8xN5eIuSi3OTEnNK8lMzOHid7VU8MsvKslQCC4pSk0tASoHCQXnlyKEBIEC4anFJQrO+Xl5qckl+UVgIddEZCGhGikFjlQhJkYmKSZmFiUuDkYBBgkGBQYNVicOJigAqkiDqmBDVSEGUyHFkQ5VwY6igolNBCiXAZXjQJXjEwEA2NEpHA=="; diff --git a/services/data-manager/__tests__/transitous/pipeline-live-motis.test.ts b/services/data-manager/__tests__/transitous/pipeline-live-motis.test.ts index ab18ba81d..ced6e0947 100644 --- a/services/data-manager/__tests__/transitous/pipeline-live-motis.test.ts +++ b/services/data-manager/__tests__/transitous/pipeline-live-motis.test.ts @@ -148,11 +148,12 @@ const { motisLocalInstance, motisLocalReachabilityInstance } = createTransitMoti const FEED_PROXY_SERVICE = "motis-feed-proxy"; const GBFS_FIXTURE_SERVICE = "gbfs-fixture"; const GBFS_FIXTURE_PORT = 18_083; + // The existing human-readable XML is the source of this 400-byte PBF. MOTIS // 2.11's street importer accepts OSM PBF only; keeping the tiny generated // artifact as base64 avoids committing an opaque binary fixture. -const BERLIN_TINY_OSM_PBF_BASE64 = - "AAAADQoJT1NNSGVhZGVyGFUQSBpReJzjkuJoWLR3RbJAw7I37SkSDd9OHW5nVmjY8nlyO7MSn39xbnByRmpuom6YgZ6ZEpdLal5xql9+SmpxEyNvfnFuZmmuvqGeoaWeAQAnRhqIAAAACwoHT1NNRGF0YRhoENcBGmN4nONi4WLgYhA6xyh0mpFLgAkK2KC0VhSXACMaEBJgQANSGCJKGCJaAkxoIk5iC7Z//cIM5f3/wAJleRk2TJ9T37BVYsE/rgfbeA/M4gGiBe/5Hxzl+H85Eat4EIZ1APJ9I7YAAAAMCgdPU01EYXRhGJ0BEOgBGpcBeJzjyuBi4GLPyEzPKE+s5GLJS8xN5eIuSi3OTEnNK8lMzOHid7VU8MsvKslQCC4pSk0tASoHCQXnlyKEBIEC4anFJQrO+Xl5qckl+UVgIddEZCGhGikFjlQhJkYmKSZmFiUuDkYBBgkGBQYNVicOJigAqkiDqmBDVSEGUyHFkQ5VwY6igolNBCiXAZXjQJXjEwEA2NEpHA=="; +import { BERLIN_TINY_OSM_PBF_BASE64 } from "./fixtures/tiny-osm"; + // Generous wall-clock ceiling for the whole lifecycle scenario. It performs a // successful promotion, imports a deliberately rejected candidate, and then // performs a second successful promotion, so a tight budget would flake on a diff --git a/services/data-manager/__tests__/transitous/pipeline.test.ts b/services/data-manager/__tests__/transitous/pipeline.test.ts index 5343c82a4..8fddbe1cd 100644 --- a/services/data-manager/__tests__/transitous/pipeline.test.ts +++ b/services/data-manager/__tests__/transitous/pipeline.test.ts @@ -1,6 +1,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; import { afterEach, describe, expect, it, vi } from "vitest"; import { buildJobContext, runTransitousPipeline } from "../../src/jobs/transitous/pipeline.js"; import type { StageName, StageResult } from "../../src/jobs/transitous/types.js"; @@ -17,6 +18,18 @@ const PINNED_LOCK = { }; vi.mock("../../src/ops-client.js", () => ({ runOpsOperation: vi.fn(async (operation: { kind: string }) => { + if (operation.kind === "catalogPins.inspect") { + const pins = createCatalogPinSet(PINNED_LOCK, { + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "b".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"b".repeat(40)}/systems.csv`, + sha256: "c".repeat(64), + lockedAt: PINNED_LOCK.lockedAt, + lockedBy: "fixture", + }); + return { active: pins, catalogRevisionId: pins.generationId }; + } if (operation.kind === "transitousLock.inspect") { return { active: PINNED_LOCK, proposed: null }; } @@ -59,6 +72,23 @@ const ORDERED_STAGES: StageName[] = [ ]; describe("runTransitousPipeline orchestrator", () => { + it("refuses a skipped critical stage when validating a reviewed catalog", async () => { + tmp = mkdtempSync(join(tmpdir(), "openmapx-catalog-critical-")); + const ctx = buildJobContext({ + dataDir: tmp, + store: new StateStore(tmp), + countries: ["de"], + runner: async () => {}, + }); + await expect( + runTransitousPipeline(ctx, { + startAt: "assemble-staging", + stopAt: "motis-health", + requireCompleteStages: true, + }), + ).rejects.toThrow(/assemble-staging.*skipped/); + }); + it("stops at a failed owner checkpoint before the next stage can mutate", async () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-pipeline-owner-")); mkdirSync(join(tmp, ".transitous-catalog", ".git"), { recursive: true }); diff --git a/services/data-manager/__tests__/transitous/promote.test.ts b/services/data-manager/__tests__/transitous/promote.test.ts index 82fc0d7b1..e5ffc785e 100644 --- a/services/data-manager/__tests__/transitous/promote.test.ts +++ b/services/data-manager/__tests__/transitous/promote.test.ts @@ -616,7 +616,7 @@ describe("promote stage", () => { it("preserves staging directory permissions for the next import", async () => { const fx = setupFixture({ staging: true, current: true }); - chmodSync(fx.stagingDir, 0o777); + chmodSync(fx.stagingDir, 0o750); globalThis.fetch = vi.fn(async (input: unknown) => { const url = typeof input === "string" ? input : (input as Request | URL).toString(); return jsonResponse(successfulBody(url)); @@ -630,7 +630,16 @@ describe("promote stage", () => { ); expect(result.status).toBe("ok"); - expect(statSync(fx.stagingDir).mode & 0o777).toBe(0o777); + expect(statSync(fx.stagingDir).mode & 0o777).toBe(0o750); + }); + + it("rejects a world-writable candidate before any host effect", async () => { + const fx = setupFixture({ staging: true, current: true }); + chmodSync(fx.stagingDir, 0o777); + const runner = vi.fn(async () => {}); + const result = await promoteRun(makeCtx({ dataDir: fx.dataDir, runner })); + expect(result.status).toBe("error"); + expect(runner).not.toHaveBeenCalled(); }); it("rejects a candidate whose epoch duplicates the active dataset", async () => { diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 86e39923d..1a2df70b3 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -72,6 +72,62 @@ } } ] + }, + "DataManager_inspectCatalogReview_response_200___schema0": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataManager_inspectCatalogReview_response_200___schema0" + } + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DataManager_inspectCatalogReview_response_200___schema0" + } + } + ] + }, + "DataManager_inspectCatalogReview_response_200___schema1": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataManager_inspectCatalogReview_response_200___schema1" + } + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DataManager_inspectCatalogReview_response_200___schema1" + } + } + ] } } }, @@ -95,6 +151,84 @@ }, "input": { "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "proposalId", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedGenerationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "acknowledgeLicenseChanges": { + "type": "boolean" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "expectedGenerationId", + "diffDigest", + "acknowledgeLicenseChanges" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -520,6 +654,8 @@ "kind": { "type": "string", "enum": [ + "catalog.validate", + "catalog.activate", "download-osm", "download-fonts", "convert-overpass", @@ -619,6 +755,107 @@ }, "result": { "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -1968,6 +2205,8 @@ "kind": { "type": "string", "enum": [ + "catalog.validate", + "catalog.activate", "download-osm", "download-fonts", "convert-overpass", @@ -2067,6 +2306,107 @@ }, "result": { "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -5628,6 +5968,8 @@ "kind": { "type": "string", "enum": [ + "catalog.validate", + "catalog.activate", "download-osm", "download-fonts", "convert-overpass", @@ -5727,6 +6069,107 @@ }, "result": { "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -6969,6 +7412,8 @@ "kind": { "type": "string", "enum": [ + "catalog.validate", + "catalog.activate", "download-osm", "download-fonts", "convert-overpass", @@ -7068,6 +7513,107 @@ }, "result": { "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -8335,6 +8881,8 @@ "kind": { "type": "string", "enum": [ + "catalog.validate", + "catalog.activate", "download-osm", "download-fonts", "convert-overpass", @@ -8440,29 +8988,38 @@ "kind": { "type": "string", "enum": [ - "download-osm" + "catalog.validate" ] }, - "region": { + "proposalId": { "type": "string", - "minLength": 1, - "maxLength": 512, - "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" }, - "sizeBytes": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" }, - "md5": { + "status": { "type": "string", - "pattern": "^[a-f0-9]{32}$" + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" } }, "required": [ "kind", - "region", - "sizeBytes" + "proposalId", + "validationId", + "status", + "expiresAt" ], "additionalProperties": false }, @@ -8472,19 +9029,57 @@ "kind": { "type": "string", "enum": [ - "download-fonts" + "catalog.activate" ] }, - "completed": { - "type": "boolean", + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", "enum": [ - true + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } ] } }, "required": [ "kind", - "completed" + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" ], "additionalProperties": false }, @@ -8494,7 +9089,7 @@ "kind": { "type": "string", "enum": [ - "convert-overpass" + "download-osm" ] }, "region": { @@ -8507,6 +9102,10 @@ "type": "integer", "minimum": 0, "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" } }, "required": [ @@ -8522,36 +9121,19 @@ "kind": { "type": "string", "enum": [ - "transit-sync" + "download-fonts" ] }, - "finalStatus": { - "type": "string", + "completed": { + "type": "boolean", "enum": [ - "ok", - "partial", - "error" + true ] - }, - "sourceRevision": { - "type": "string", - "minLength": 1, - "maxLength": 512 - }, - "countries": { - "maxItems": 256, - "type": "array", - "items": { - "type": "string", - "pattern": "^[A-Za-z]{2,3}$" - } } }, "required": [ "kind", - "finalStatus", - "sourceRevision", - "countries" + "completed" ], "additionalProperties": false }, @@ -8561,7 +9143,7 @@ "kind": { "type": "string", "enum": [ - "overture-pull" + "convert-overpass" ] }, "region": { @@ -8570,47 +9152,16 @@ "maxLength": 512, "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" }, - "metrics": { - "type": "object", - "properties": { - "processed": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "inserted": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "updated": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "skipped": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "sizeBytes": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "additionalProperties": false - }, - "release": { - "type": "string", - "maxLength": 128, - "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "required": [ "kind", "region", - "metrics" + "sizeBytes" ], "additionalProperties": false }, @@ -8620,115 +9171,36 @@ "kind": { "type": "string", "enum": [ - "overture-ingest" + "transit-sync" ] }, - "region": { - "type": "string", - "minLength": 1, - "maxLength": 512, - "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" - }, - "metrics": { - "type": "object", - "properties": { - "processed": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "inserted": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "updated": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "skipped": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "sizeBytes": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "additionalProperties": false - }, - "release": { - "type": "string", - "maxLength": 128, - "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" - } - }, - "required": [ - "kind", - "region", - "metrics" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "kind": { + "finalStatus": { "type": "string", "enum": [ - "overture-extract" + "ok", + "partial", + "error" ] }, - "region": { + "sourceRevision": { "type": "string", "minLength": 1, - "maxLength": 512, - "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" - }, - "metrics": { - "type": "object", - "properties": { - "processed": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "inserted": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "updated": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "skipped": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "sizeBytes": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "additionalProperties": false + "maxLength": 512 }, - "release": { - "type": "string", - "maxLength": 128, - "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } } }, "required": [ "kind", - "region", - "metrics" + "finalStatus", + "sourceRevision", + "countries" ], "additionalProperties": false }, @@ -8738,7 +9210,184 @@ "kind": { "type": "string", "enum": [ - "overture-conflate" + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" ] }, "region": { @@ -44150,35 +44799,46 @@ } } }, - "/transit/bump": { - "post": { - "operationId": "proposeTransitBump", + "/transit/pins": { + "get": { + "operationId": "inspectCatalogReview", "tags": [ "data-manager" ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "branch": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*$" - }, - "force": { - "type": "boolean" - }, - "lockedBy": { - "type": "string" - } - }, - "additionalProperties": false - } - } + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "query", + "name": "proposalId", + "required": false + }, + { + "schema": { + "default": 0, + "type": "integer", + "minimum": 0, + "maximum": 4096 + }, + "in": "query", + "name": "offset", + "required": false + }, + { + "schema": { + "default": 0, + "type": "integer", + "minimum": 0, + "maximum": 200000 + }, + "in": "query", + "name": "diffOffset", + "required": false } - }, + ], "security": [ { "serviceBearer": [] @@ -44187,7 +44847,7 @@ "serviceToken": [] } ], - "x-openmapx-contract-id": "proposeTransitBump", + "x-openmapx-contract-id": "inspectCatalogReview", "x-openmapx-auth": "service", "responses": { "200": { @@ -44195,60 +44855,2358 @@ "content": { "application/json": { "schema": { - "anyOf": [ - { + "type": "object", + "properties": { + "active": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "generationId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transitous": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$" + }, + "submodules": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "additionalProperties": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "comment": { + "type": "string", + "maxLength": 512 + } + }, + "required": [ + "ref", + "submodules", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + }, + "gbfs": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "source": { + "type": "string", + "enum": [ + "mobilitydata-gbfs" + ] + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "url": { + "type": "string", + "maxLength": 256 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "required": [ + "schemaVersion", + "source", + "commit", + "url", + "sha256", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "generationId", + "transitous", + "gbfs" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "allowedCountries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{2,3}$" + } + }, + "catalogRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "page": { "type": "object", "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] + "proposals": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "baseGenerationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "string", + "enum": [ + "proposed", + "validating", + "validated", + "rejected", + "activating", + "activated" + ] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transitousRef": { + "type": "string", + "maxLength": 256 + }, + "atlasCommit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "gbfsCommit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "proposalId", + "baseGenerationId", + "diffDigest", + "sourcePolicyRevision", + "state", + "createdAt", + "catalogRevisionId", + "transitousRef", + "atlasCommit", + "gbfsCommit" + ], + "additionalProperties": false + } }, - "unchanged": { - "type": "boolean", - "enum": [ - true - ] + "total": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 }, - "ref": { - "type": "string" + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 }, - "previousRef": { - "type": [ - "string", - "null" + "nextOffset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } ] } }, "required": [ - "ok", - "unchanged", - "ref", - "previousRef" + "proposals", + "total", + "offset", + "nextOffset" ], "additionalProperties": false }, - { - "type": "object", - "properties": { - "ok": { - "type": "boolean", - "enum": [ - true - ] - }, - "unchanged": { - "type": "boolean", - "enum": [ - false - ] - }, - "proposed": { - "type": "boolean", - "enum": [ - true - ] + "proposal": { + "anyOf": [ + { + "type": "object", + "properties": { + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "baseGenerationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "pins": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "generationId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transitous": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*@[0-9a-f]{40}$" + }, + "submodules": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "additionalProperties": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "comment": { + "type": "string", + "maxLength": 512 + } + }, + "required": [ + "ref", + "submodules", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + }, + "gbfs": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "source": { + "type": "string", + "enum": [ + "mobilitydata-gbfs" + ] + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "url": { + "type": "string", + "maxLength": 256 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lockedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "lockedBy": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "required": [ + "schemaVersion", + "source", + "commit", + "url", + "sha256", + "lockedAt", + "lockedBy" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "generationId", + "transitous", + "gbfs" + ], + "additionalProperties": false + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "string", + "enum": [ + "proposed", + "validating", + "validated", + "rejected", + "activating", + "activated" + ] + }, + "createdAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "proposalId", + "baseGenerationId", + "pins", + "diffDigest", + "sourcePolicyRevision", + "state", + "createdAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "diff": { + "anyOf": [ + { + "type": "object", + "properties": { + "digest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "total": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rows": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "region": { + "type": "string" + }, + "source": { + "type": "string" + }, + "change": { + "type": "string", + "enum": [ + "added", + "removed", + "changed" + ] + }, + "fields": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "endpoint", + "origin", + "license", + "attribution", + "eligibility", + "metadata" + ] + } + }, + "licenseStatus": { + "type": "string", + "enum": [ + "declared", + "unknown" + ] + }, + "before": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DataManager_inspectCatalogReview_response_200___schema0" + } + }, + { + "type": "null" + } + ] + }, + "after": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DataManager_inspectCatalogReview_response_200___schema1" + } + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "id", + "region", + "source", + "change", + "fields", + "licenseStatus", + "before", + "after" + ], + "additionalProperties": false + } + }, + "nextOffset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "digest", + "total", + "offset", + "rows", + "nextOffset" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "validation": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "properties": { + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "pinDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "overlayDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{2,3}$" + } + }, + "completedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "slot": { + "type": "string", + "enum": [ + "A", + "B" + ] + }, + "epoch": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "manifestHash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "imageDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed" + ] + } + }, + "required": [ + "validationId", + "proposalId", + "pinDigest", + "diffDigest", + "sourcePolicyRevision", + "overlayDigest", + "credentialsRevision", + "countries", + "completedAt", + "expiresAt", + "slot", + "epoch", + "manifestHash", + "imageDigest", + "status" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "pinDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "diffDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "overlayDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{2,3}$" + } + }, + "completedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "status": { + "type": "string", + "enum": [ + "failed" + ] + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "candidate": { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "A", + "B" + ] + }, + "epoch": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "manifestHash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "imageDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "required": [ + "slot", + "epoch", + "manifestHash", + "imageDigest" + ], + "additionalProperties": false + } + }, + "required": [ + "validationId", + "proposalId", + "pinDigest", + "diffDigest", + "sourcePolicyRevision", + "overlayDigest", + "credentialsRevision", + "countries", + "completedAt", + "expiresAt", + "status", + "reason" + ], + "additionalProperties": false + } + ] + }, + { + "type": "null" + } + ] + }, + "execution": { + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "admittedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "kind": { + "type": "string", + "enum": [ + "catalog.validate", + "catalog.activate", + "download-osm", + "download-fonts", + "convert-overpass", + "transit-sync", + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-sync", + "search-index-build", + "notable-places-build", + "generation-hardlinks", + "transit-maintenance" + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "cancel_pending", + "ok", + "partial", + "error", + "canceled", + "interrupted" + ] + }, + "cancellationRequestedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + { + "type": "null" + } + ] + }, + "progress": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "diagnostics": { + "type": "object", + "properties": { + "tail": { + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "stream", + "message" + ], + "additionalProperties": false + } + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "tail", + "truncated" + ], + "additionalProperties": false + }, + "publication": { + "type": "string", + "enum": [ + "not_started", + "committed", + "unknown" + ] + }, + "result": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.validate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "status", + "expiresAt" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "catalog.activate" + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "validationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "outcome": { + "type": "string", + "enum": [ + "activated", + "not_activated", + "unknown" + ] + }, + "generationId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "proposalId", + "validationId", + "outcome", + "generationId", + "reason" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-osm" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "download-fonts" + ] + }, + "completed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "completed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "convert-overpass" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "kind", + "region", + "sizeBytes" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-sync" + ] + }, + "finalStatus": { + "type": "string", + "enum": [ + "ok", + "partial", + "error" + ] + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "countries": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z]{2,3}$" + } + } + }, + "required": [ + "kind", + "finalStatus", + "sourceRevision", + "countries" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-pull" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-ingest" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-extract" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-conflate" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "overture-sync" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "release": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "search-index-build" + ] + }, + "region": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "region", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "notable-places-build" + ] + }, + "metrics": { + "type": "object", + "properties": { + "processed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inserted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + } + }, + "required": [ + "kind", + "metrics" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "generation-hardlinks" + ] + }, + "hardlinks": { + "type": "object", + "properties": { + "applied": { + "type": "boolean" + }, + "linked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "pruned": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "via": { + "type": "string", + "enum": [ + "data-manager", + "local", + "none" + ] + } + }, + "required": [ + "applied", + "linked", + "skipped", + "pruned", + "entries" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "hardlinks" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "transit-maintenance" + ] + }, + "action": { + "type": "string", + "enum": [ + "restart-primary", + "reload-feed-proxy" + ] + }, + "changed": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "kind", + "action", + "changed" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "id", + "operationKey", + "fingerprint", + "admittedAt", + "deadlineAt", + "kind", + "status", + "cancellationRequestedAt", + "progress", + "diagnostics", + "publication" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "validationInputs": { + "anyOf": [ + { + "type": "object", + "properties": { + "configurationRevisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "credentialsRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "overlayDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "imageDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "stagingState": { + "type": "string", + "enum": [ + "running", + "stopped", + "created", + "restarting", + "paused", + "unknown" + ] + }, + "requiredKeys": { + "maxItems": 1000, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + }, + "configuredKeys": { + "maxItems": 10000, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + } + }, + "required": [ + "configurationRevisionId", + "credentialsRevision", + "sourcePolicyRevision", + "overlayDigest", + "imageDigest", + "stagingState", + "requiredKeys", + "configuredKeys" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "validationEligible": { + "type": "boolean" + }, + "blocker": { + "anyOf": [ + { + "type": "string", + "maxLength": 2048 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "active", + "allowedCountries", + "catalogRevisionId", + "page", + "proposal", + "diff", + "validation", + "execution", + "validationInputs", + "validationEligible", + "blocker" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, + "/transit/bump": { + "post": { + "operationId": "proposeTransitBump", + "tags": [ + "data-manager" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "branch": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*$" + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "force": { + "type": "boolean" + }, + "lockedBy": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "proposeTransitBump", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "unchanged": { + "type": "boolean", + "enum": [ + true + ] + }, + "ref": { + "type": "string" + }, + "previousRef": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "ok", + "unchanged", + "ref", + "previousRef" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "unchanged": { + "type": "boolean", + "enum": [ + false + ] + }, + "proposed": { + "type": "boolean", + "enum": [ + true + ] + }, + "proposalId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "catalogRevisionId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" }, "ref": { "type": "string" @@ -44275,6 +47233,8 @@ "ok", "unchanged", "proposed", + "proposalId", + "catalogRevisionId", "ref", "previousRef", "submoduleSha", diff --git a/services/data-manager/src/api.ts b/services/data-manager/src/api.ts index bf097e621..ae1634328 100644 --- a/services/data-manager/src/api.ts +++ b/services/data-manager/src/api.ts @@ -13,7 +13,8 @@ import { normalizeDataExecutionInput, type TransitDesiredSourceMutation, } from "@openmapx/core/data-manager-contracts"; -import { feedState } from "@openmapx/db-schema"; +import { feedState, jobs } from "@openmapx/db-schema"; +import { and, desc, sql as drizzleSql, eq, inArray } from "drizzle-orm"; import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; import type postgres from "postgres"; import { type CoverageApiOptions, registerCoverageEvidenceRoute } from "./coverage/api.js"; @@ -39,12 +40,7 @@ import { import { createSearchIndexRuntimeState, getSearchIndexStatus } from "./jobs/search-index/state.js"; import type { BakePredictedResult } from "./jobs/traffic/bake-predicted.js"; import { createTransitPreflightController } from "./jobs/transitous/admin-preflight.js"; -import { - CatalogBumpError, - candidateMatchesLock, - lockFromCandidate, - resolveCatalogBumpCandidate, -} from "./jobs/transitous/catalog-bump.js"; +import { CatalogBumpError } from "./jobs/transitous/catalog-bump.js"; import { resolveTransitOverlayPath } from "./jobs/transitous/internal.js"; import { type MotisOperationsPolicy, @@ -90,6 +86,10 @@ import { } from "./transit-sources.js"; export interface ApiOptions { + catalogReview?: Pick< + ReturnType, + "resolveCatalogProposal" | "reviewState" + >; executionReceiver?: DataExecutionReceiver; dataDir?: string; /** @@ -1070,55 +1070,72 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { return undefined; }); + registerDataManagerRoute(app, "inspectCatalogReview", async (req, reply) => { + if (!opts.catalogReview || !opts.executionReceiver) + return reply.code(503).send({ error: "Durable catalog review is unavailable" }); + const proposalId = req.query.proposalId; + const matching = proposalId + ? and( + eq(jobs.executionVersion, 2), + drizzleSql`${jobs.executionInput}->>'proposalId' = ${proposalId}`, + ) + : undefined; + const [executions, validations] = proposalId + ? await Promise.all([ + db + .select({ id: jobs.id }) + .from(jobs) + .where(and(matching, inArray(jobs.kind, ["catalog.validate", "catalog.activate"]))) + .orderBy(desc(jobs.startedAt), desc(jobs.id)) + .limit(1), + db + .select({ id: jobs.id }) + .from(jobs) + .where(and(matching, eq(jobs.kind, "catalog.validate"))) + .orderBy(desc(jobs.startedAt), desc(jobs.id)) + .limit(1), + ]) + : [[], []]; + const latestValidation = validations[0], + latest = executions[0]; + const snapshot = await opts.catalogReview.reviewState({ + ...req.query, + ...(latestValidation ? { validationId: latestValidation.id } : {}), + }); + return { + ...snapshot, + execution: latest ? await opts.executionReceiver.inspect(latest.id) : null, + }; + }); + // POST /transit/bump — propose a new pin set. It never activates the lock; // operators must review diffs/build the inactive slot before approval. registerDataManagerRoute(app, "proposeTransitBump", async (req, reply) => { - const branch = req.body?.branch?.trim() || "main"; - const force = req.body?.force === true; - const lockedBy = req.body?.lockedBy?.trim() || "api"; - - const catalogDir = join(dataDir, ".transitous-catalog"); - + if (!opts.catalogReview) + return reply.code(503).send({ error: "Paired catalog resolver is unavailable" }); try { - return await withLegacyDataMutation("transit-proposal", async () => { - const candidate = await resolveCatalogBumpCandidate({ catalogDir, branch }); - const { active: existing } = await runOpsOperation({ kind: "transitousLock.inspect" }); - if (candidateMatchesLock(candidate, existing) && !force) - return { - ok: true, - unchanged: true, - ref: candidate.ref, - previousRef: existing?.ref ?? null, - }; - const lock = lockFromCandidate( - candidate, - lockedBy, - "Pinned commit of public-transport/transitous consumed by services/data-manager. Bumped via POST /transit/bump.", - ); - await runOpsOperation({ - kind: "transitousLock.propose", - ref: lock.ref, - submodules: lock.submodules, - lockedBy: lock.lockedBy, - ...(lock.comment ? { comment: lock.comment } : {}), - }); - return { - ok: true, - unchanged: false, - proposed: true, - ref: lock.ref, - previousRef: existing?.ref ?? null, - submoduleSha: candidate.transitlandAtlasSha, - lockedAt: lock.lockedAt, - lockedBy: lock.lockedBy, - }; + const previous = await runOpsOperation({ kind: "catalogPins.inspect" }); + const proposal = await opts.catalogReview.resolveCatalogProposal({ + branch: req.body?.branch?.trim() || "main", + proposalId: req.body?.proposalId, }); + return { + ok: true, + unchanged: false, + proposed: true, + proposalId: proposal.proposalId, + catalogRevisionId: proposal.pins.generationId, + ref: proposal.pins.transitous.ref, + previousRef: previous.active?.transitous.ref ?? null, + submoduleSha: proposal.pins.transitous.submodules["transitland-atlas"]!, + lockedAt: proposal.pins.transitous.lockedAt, + lockedBy: proposal.pins.transitous.lockedBy, + }; } catch (err) { if (err instanceof CatalogBumpError) { const status = err.code === "catalog-not-cloned" ? 409 : err.code === "git-fetch-failed" ? 502 : 500; - reply.code(status); - return { ok: false, error: err.code, message: err.message }; + return reply.code(status).send({ error: err.code }); } throw err; } diff --git a/services/data-manager/src/cron.ts b/services/data-manager/src/cron.ts index 8d5929aed..0f83c2c61 100644 --- a/services/data-manager/src/cron.ts +++ b/services/data-manager/src/cron.ts @@ -77,11 +77,7 @@ import { type WriteLiveTrafficDeps, type WriteLiveTrafficResult, } from "./jobs/traffic/write-live.js"; -import { - type CatalogBumpCandidate, - lockFromCandidate, - resolveCatalogBumpCandidate, -} from "./jobs/transitous/catalog-bump.js"; +import type { CatalogBumpCandidate } from "./jobs/transitous/catalog-bump.js"; import { fetchWithTimeout } from "./jobs/transitous/motis-probe.js"; import type { MotisOperationsPolicy } from "./jobs/transitous/operations-profile.js"; import type { SingleFlightController } from "./jobs/transitous/single-flight.js"; @@ -156,6 +152,10 @@ export interface CronLogger { } export interface CronSetupOptions { + catalogReview?: Pick< + ReturnType, + "resolveCatalogProposal" + >; executionReceiver?: DataExecutionReceiver; dataDir: string; countries: string[]; @@ -542,40 +542,14 @@ export function setupCron(options: CronSetupOptions): CronHandles { const runStalenessCheck = options.runStalenessCheck ?? defaultStalenessCheck; - const catalogDir = join(options.dataDir, ".transitous-catalog"); - const resolveBumpCandidate = - options.resolveBumpCandidate ?? ((o) => resolveCatalogBumpCandidate(o)); - - /** - * Auto-bump the pinned Transitous catalog behind the pipeline's own canary. - * Resolve upstream → propose the candidate → build it into the staging slot - * and run the functional-probe canary → activate the pin only if the whole - * pipeline (incl. the live promote) succeeds. On rejection the active pin is - * untouched, the proposal is retained for review, and a failure alert fires. - */ + /** Scheduled checks create paired proposals only. Activation always requires review. */ const runAutoBump = async (): Promise => { - await withLegacyDataMutation("transit-proposal", async () => { - const candidate = await resolveBumpCandidate({ - catalogDir, - branch: process.env.TRANSITOUS_AUTO_BUMP_BRANCH ?? "master", - }); - const { active } = await runOpsOperation({ kind: "transitousLock.inspect" }); - if (active?.ref === candidate.ref) return; - const lock = lockFromCandidate( - candidate, - "data-manager-auto-bump", - "Proposed by the scheduled catalog check; validation-bound activation is required", - ); - await runOpsOperation({ - kind: "transitousLock.propose", - ref: lock.ref, - submodules: lock.submodules, - lockedBy: lock.lockedBy, - ...(lock.comment ? { comment: lock.comment } : {}), - }); - log.info( - "transitous-auto-bump: proposal retained; validation-bound activation is unavailable", - ); + if (!options.catalogReview) throw Error("Installed paired catalog resolver is unavailable"); + const proposal = await options.catalogReview.resolveCatalogProposal({ + branch: process.env.TRANSITOUS_AUTO_BUMP_BRANCH ?? "master", + }); + log.info("transitous-auto-bump: paired proposal retained for administrator review", { + proposalId: proposal.proposalId, }); }; diff --git a/services/data-manager/src/jobs/execution/catalog-child-reservations.test.ts b/services/data-manager/src/jobs/execution/catalog-child-reservations.test.ts new file mode 100644 index 000000000..88e6381ce --- /dev/null +++ b/services/data-manager/src/jobs/execution/catalog-child-reservations.test.ts @@ -0,0 +1,62 @@ +import { randomUUID } from "node:crypto"; +import type { ExecutionRef } from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { reserveTransitChildOperations } from "./ops-children"; + +const execution: ExecutionRef = { + executor: "data-manager", + principal: "api", + executionId: randomUUID(), + operationKey: "opk1_catalogReservedRoot0001", + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 3600000, +}; +const proposalId = randomUUID(), + validationId = randomUUID(); +it("freezes validation's two distinct stops, staging restart and proxy restoration only", () => { + const slots = reserveTransitChildOperations(execution, { + kind: "catalog.validate", + proposalId, + countries: ["de"], + }); + expect(slots.map((slot) => [slot.phase, slot.ordinal])).toEqual([ + ["stop-staging", 0], + ["restart-staging", 1], + ["proxy-stage", 2], + ["proxy-restore", 3], + ["stop-staging", 4], + ]); + expect(new Set(slots.map((slot) => slot.operationKey)).size).toBe(5); + expect( + slots.some((slot) => slot.phase === "promote-primary" || slot.phase === "paired-pin-commit"), + ).toBe(false); +}); +it("freezes exact validation promotion and paired commit ordinal zero under one activation root", () => { + const slots = reserveTransitChildOperations(execution, { + kind: "catalog.activate", + proposalId, + validationId, + expectedGenerationId: null, + diffDigest: "c".repeat(64), + acknowledgeLicenseChanges: true, + }); + expect(slots[0]).toMatchObject({ + phase: "paired-pin-commit", + ordinal: 0, + serviceIds: ["motis"], + operation: { + kind: "transitousLock.approve", + proposalId, + validationId, + expectedGenerationId: null, + diffDigest: "c".repeat(64), + acknowledgeLicenseChanges: true, + }, + }); + expect(slots.find((slot) => slot.phase === "promote-primary")?.operation).toEqual({ + kind: "motis.primary.promote", + preparedRunId: validationId, + }); + expect(new Set(slots.map((slot) => slot.ordinal)).size).toBe(slots.length); +}); diff --git a/services/data-manager/src/jobs/execution/dispatch.ts b/services/data-manager/src/jobs/execution/dispatch.ts index 7bef24bc7..517fd3571 100644 --- a/services/data-manager/src/jobs/execution/dispatch.ts +++ b/services/data-manager/src/jobs/execution/dispatch.ts @@ -40,6 +40,10 @@ import { updateCurrentSearchIndexFingerprint, } from "../search-index/state.js"; import { verifyTransitPreflightBinding } from "../transitous/admin-preflight.js"; +import { runCatalogActivation } from "../transitous/catalog-activation.js"; +import { runCatalogValidation } from "../transitous/catalog-validation.js"; +import { publishCatalogValidationCurrent } from "../transitous/catalog-validation-current.js"; +import { isolateOwnedTransitProducer } from "../transitous/isolate-producer.js"; import { type MotisOperationsPolicy, transitAllowedCountries, @@ -171,6 +175,13 @@ export function createDataExecutionDispatcher(deps: { jobChildLogger({ job: "transit-sync", jobId: context.execution.executionId }), ); try { + await publishDataChange(() => + publishCatalogValidationCurrent(context.authority, context.targetAuthority), + ); + await isolateOwnedTransitProducer( + context, + deps.operationsPolicy.resourceEnvelope.minimumDiskGb * 1024 ** 3, + ); await publishOwnedTransitSourceMutation( input, context, @@ -220,6 +231,47 @@ export function createDataExecutionDispatcher(deps: { deps.singleFlight.markSyncFinished(context.execution.executionId); } } + case "catalog.validate": { + if (!deps.singleFlight.reserveOwned(context.execution.executionId)) + throw Error("Transit reservation is busy"); + try { + const validation = await runCatalogValidation({ + context, + dataDir: deps.dataDir, + store: deps.state, + operationsPolicy: deps.operationsPolicy, + }); + finish(context.execution.executionId, { + event: "done", + ok: validation.status === "passed", + proposalId: validation.proposalId, + validationId: validation.validationId, + }); + return { + kind: input.kind, + proposalId: validation.proposalId, + validationId: validation.validationId, + status: validation.status, + expiresAt: validation.expiresAt, + }; + } finally { + deps.singleFlight.markSyncFinished(context.execution.executionId); + } + } + case "catalog.activate": { + if (!deps.singleFlight.reserveOwned(context.execution.executionId)) + throw Error("Transit reservation is busy"); + try { + return await runCatalogActivation({ + context, + dataDir: deps.dataDir, + store: deps.state, + operationsPolicy: deps.operationsPolicy, + }); + } finally { + deps.singleFlight.markSyncFinished(context.execution.executionId); + } + } case "overture-pull": { const path = await pullOverture({ ...common, region: input.region }); finish(context.execution.executionId, { event: "done", ok: true, path }); diff --git a/services/data-manager/src/jobs/execution/ops-children.test.ts b/services/data-manager/src/jobs/execution/ops-children.test.ts index 8102e1fab..9e24692e3 100644 --- a/services/data-manager/src/jobs/execution/ops-children.test.ts +++ b/services/data-manager/src/jobs/execution/ops-children.test.ts @@ -243,6 +243,7 @@ it.each(["promotion", "reload"] as const)( rmSync(root, { recursive: true, force: true }); } }, + 60000, ); import { existsSync } from "node:fs"; diff --git a/services/data-manager/src/jobs/execution/ops-children.ts b/services/data-manager/src/jobs/execution/ops-children.ts index 6a889b88b..5b8d528a7 100644 --- a/services/data-manager/src/jobs/execution/ops-children.ts +++ b/services/data-manager/src/jobs/execution/ops-children.ts @@ -10,8 +10,10 @@ import { executionRefSchema, type OpsOperation, opsOperationFingerprint, + pairedPinApprovalResultSchema, } from "@openmapx/core/ops"; import z from "zod/v4"; +import { retainCatalogActivationRecord } from "../transitous/catalog-activation-record.js"; export const DATA_EFFECT_KINDS = [ "motis.primary.stop", "motis.staging.stop", @@ -119,6 +121,7 @@ export const dataOpsChildAttemptSchema = z ...attempted, state: z.literal("terminal"), acceptedId: receiverId, + approvalResult: pairedPinApprovalResultSchema.optional(), publication: z.enum(["not_started", "committed"]), settlement: z.strictObject({ outcome: z.enum(["contained", "committed", "failed_without_effect"]), @@ -133,6 +136,15 @@ export const dataOpsChildAttemptSchema = z }), ]) .superRefine((child, ctx) => { + if ( + child.state === "terminal" && + child.approvalResult && + child.operation.kind !== "transitousLock.approve" + ) + ctx.addIssue({ + code: "custom", + message: "Only paired approval retains its bounded committed result", + }); if (child.proxyRecovery && child.phase !== "proxy-restore") ctx.addIssue({ code: "custom", @@ -199,7 +211,90 @@ export function reserveTransitChildOperations( ): DataOpsChildReserved[] { const entries: Array<{ phase: DataOpsChildPhase; operation: OpsOperation; targets: string[] }> = []; - if (input.kind === "transit-sync") { + if (input.kind === "catalog.validate") { + entries.push( + { + phase: "stop-staging", + operation: { kind: "motis.staging.stop" }, + targets: ["motis-staging"], + }, + { + phase: "restart-staging", + operation: { kind: "motis.staging.restart" }, + targets: ["motis-staging"], + }, + { + phase: "proxy-stage", + operation: { + kind: "feedProxy.validateAndReload", + candidateId: transitChildCandidateId(execution, "proxy-stage", 2), + }, + targets: ["motis-feed-proxy"], + }, + { + phase: "proxy-restore", + operation: { + kind: "feedProxy.validateAndReload", + candidateId: transitChildCandidateId(execution, "proxy-restore", 3), + }, + targets: ["motis-feed-proxy"], + }, + { + phase: "stop-staging", + operation: { kind: "motis.staging.stop" }, + targets: ["motis-staging"], + }, + ); + } else if (input.kind === "catalog.activate") { + entries.push( + { + phase: "paired-pin-commit", + operation: { + kind: "transitousLock.approve", + proposalId: input.proposalId, + validationId: input.validationId, + expectedGenerationId: input.expectedGenerationId, + diffDigest: input.diffDigest, + acknowledgeLicenseChanges: input.acknowledgeLicenseChanges, + }, + targets: ["motis"], + }, + { phase: "stop-primary", operation: { kind: "motis.primary.stop" }, targets: ["motis"] }, + { + phase: "promote-primary", + operation: { kind: "motis.primary.promote", preparedRunId: input.validationId }, + targets: ["motis"], + }, + { + phase: "recover-primary", + operation: { kind: "motis.primary.restart" }, + targets: ["motis"], + }, + ); + for (const phase of ["proxy-stage", "proxy-commit", "proxy-restore"] as const) { + const ordinal = entries.length; + entries.push({ + phase, + operation: { + kind: "feedProxy.validateAndReload", + candidateId: transitChildCandidateId(execution, phase, ordinal), + }, + targets: ["motis-feed-proxy"], + }); + } + entries.push( + { + phase: "stop-staging", + operation: { kind: "motis.staging.stop" }, + targets: ["motis-staging"], + }, + { + phase: "restart-staging", + operation: { kind: "motis.staging.restart" }, + targets: ["motis-staging"], + }, + ); + } else if (input.kind === "transit-sync") { entries.push( { phase: "stop-primary", operation: { kind: "motis.primary.stop" }, targets: ["motis"] }, { @@ -401,6 +496,9 @@ async function observeChild( state: "terminal", acceptedId: status.operationId, publication: evidence.publication, + ...(child.operation.kind === "transitousLock.approve" && status.state === "succeeded" + ? { approvalResult: pairedPinApprovalResultSchema.parse(status.result) } + : {}), settlement: { outcome: evidence.outcome, terminalReceiptDigest: evidence.terminalReceiptDigest, @@ -463,8 +561,6 @@ export async function runOwnedDataOpsOperation( canonicalJson(slot.serviceIds) !== canonicalJson(options.serviceIds) ) throw Error("Operation differs from original data phase reservation"); - if (operation.kind === "transitousLock.approve") - throw Error("Paired pin approval requires the validation-bound activation runtime"); let child: DataOpsChildAttempt | undefined = record.children.find( (value) => value.operationKey === slot.operationKey, ); @@ -497,6 +593,8 @@ export async function runOwnedDataOpsOperation( await context.checkpointChild(bound); child = bound; context.assertForward(); + if (operation.kind === "transitousLock.approve") + await retainCatalogActivationRecord(context, exact.execution); if (assertOwnedDataTransitCandidate(context.authority, slot) !== candidateDigest) throw Error("Owned candidate changed before grant"); const issued = await issueControlChildPhase(context.authority, { diff --git a/services/data-manager/src/jobs/execution/receiver.ts b/services/data-manager/src/jobs/execution/receiver.ts index 26c9ae43f..af51f0243 100644 --- a/services/data-manager/src/jobs/execution/receiver.ts +++ b/services/data-manager/src/jobs/execution/receiver.ts @@ -34,7 +34,7 @@ export function createDataExecutionReceiver(deps: { const validate = (value: Parameters[0]) => { const input = normalizeDataExecutionInput(value); if ( - input.kind === "transit-sync" && + (input.kind === "transit-sync" || input.kind === "catalog.validate") && deps.operationsPolicy.profile !== "planet" && (input.countries.length === 0 || input.countries.some( diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index de5902fdc..25a0a6396 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -30,7 +30,9 @@ import { isDataExecutionTerminal, } from "@openmapx/core/data-manager-contracts"; import type { ExecutionRef, VerifiedParentExecution } from "@openmapx/core/ops"; +import { jobChildLogger } from "../../logger.js"; import { cancelAndSettleRetainedTransitousScripts } from "../transitous/script-runner.js"; +import { dataExecutionDiagnostic } from "./diagnostics.js"; import { cancelAndSettleOwnedDataOpsChildren, type DataOpsChildAttempt, @@ -87,7 +89,7 @@ export interface DataExecutionRuntime { close(): Promise; } export function dataExecutionTargetIds(input: DataExecutionInput): string[] { - return input.kind === "transit-sync" + return ["transit-sync", "catalog.validate", "catalog.activate"].includes(input.kind) ? ["data-manager", "motis", "motis-feed-proxy", "motis-staging"] : input.kind === "transit-maintenance" ? input.action === "restart-primary" @@ -350,6 +352,15 @@ export function createDataExecutionRuntime(deps: { return result; }, async (error) => { + const diagnostic = dataExecutionDiagnostic( + error instanceof Error ? error.message : "Data worker failed", + ); + await context.checkpoint({ + diagnostics: { + tail: [{ stream: "stderr", message: diagnostic.message }], + truncated: diagnostic.truncated, + }, + }); await settleChildren(); throw error; }, @@ -373,21 +384,31 @@ export function createDataExecutionRuntime(deps: { throw Error("Dispatcher result kind mismatch"); const canceled = current.status.cancellationRequestedAt !== null; const status = result - ? result.kind === "transit-sync" - ? result.finalStatus - : contained.partial - ? "partial" - : "ok" + ? result.kind === "catalog.validate" && result.status === "failed" + ? "error" + : result.kind === "catalog.activate" && result.outcome !== "activated" + ? "error" + : result.kind === "transit-sync" + ? result.finalStatus + : contained.partial + ? "partial" + : "ok" : canceled ? "canceled" : "error"; const patch: DataExecutionPatch = { status, - publication: result ? "committed" : current.status.publication, + publication: contained.publication, ...(result ? { result } : {}), progress: result ? 1 : current.status.progress, }; - if (patch.publication === "unknown") return { record: current, value: undefined }; + if (patch.publication === "unknown") { + const saved = await store.checkpoint(current.status.id, current.version, { + publication: "unknown", + }); + if (!saved) throw new DataExecutionOwnershipLost("Data unknown-outcome CAS lost"); + return { record: saved, value: undefined }; + } const saved = await store.finalize( current.status.id, current.version, @@ -410,9 +431,22 @@ export function createDataExecutionRuntime(deps: { } else if (record.input.kind === "generation-hardlinks") await owner.publishContainedTerminal(record.execution); } - } catch { + } catch (error) { // An I/O/CAS/containment gap is not a terminal receipt or permission to // release. Keep original identities and reconcile by observation only. + jobChildLogger({ job: "data-execution", jobId: record.status.id }).error( + { + reason: dataExecutionDiagnostic( + error instanceof Error ? error.message : "Unknown containment failure", + ).message, + trace: dataExecutionDiagnostic( + error instanceof Error + ? (error.stack ?? error.message) + : "Unknown containment failure", + ).message, + }, + "Data execution requires original receipt reconciliation", + ); try { await context.checkpoint({ publication: "unknown" }); } catch {} @@ -529,7 +563,22 @@ export function createDataExecutionRuntime(deps: { // Recover authentic observation authority without assuming that the // killed local producer ever reached worker completion publication. observation = observationContext(record); - recovered = await owner.restoreCompletion(record.execution); + try { + recovered = await owner.restoreCompletion(record.execution); + } catch (error) { + if ( + record.input.kind !== "catalog.activate" || + (error as NodeJS.ErrnoException).code !== "ENOENT" + ) + throw error; + // Final approval is the fixed graph's last effect. Reconcile only + // its original address and authenticated, fully settled child tree. + cancelControlExecution(observation.authority); + await cancelArtifactTargetLease(observation.targetAuthority); + if ((await cancelAndSettleOwnedDataOpsChildren(observation)) !== "settled") throw error; + await owner.repairCommittedCatalogActivation(record.execution); + recovered = await owner.restoreCompletion(record.execution); + } cancelControlExecution(recovered.authority); await cancelArtifactTargetLease(recovered.lease); const context = observation; @@ -541,11 +590,15 @@ export function createDataExecutionRuntime(deps: { const current = await context.record(), result = recovered.result, status = result - ? result.kind === "transit-sync" - ? result.finalStatus - : recovered.partial - ? "partial" - : "ok" + ? result.kind === "catalog.validate" && result.status === "failed" + ? "error" + : result.kind === "catalog.activate" && result.outcome !== "activated" + ? "error" + : result.kind === "transit-sync" + ? result.finalStatus + : recovered.partial + ? "partial" + : "ok" : current.status.cancellationRequestedAt ? "canceled" : "error"; @@ -570,7 +623,18 @@ export function createDataExecutionRuntime(deps: { await owner.publishContainedTerminal(record.execution); } else if (record.input.kind === "generation-hardlinks") await owner.publishContainedTerminal(record.execution); - } catch { + } catch (error) { + jobChildLogger({ job: "data-execution-recovery", jobId: record.status.id }).warn( + { + reason: dataExecutionDiagnostic( + error instanceof Error ? error.message : "Original recovery evidence unavailable", + ).message, + trace: dataExecutionDiagnostic( + error instanceof Error ? (error.stack ?? error.message) : "", + ).message, + }, + "Original execution remains fenced", + ); try { const latest = await store.inspect(record.status.id); if (latest && !isDataExecutionTerminal(latest.status)) { diff --git a/services/data-manager/src/jobs/transitous/assemble-staging.ts b/services/data-manager/src/jobs/transitous/assemble-staging.ts index 40168374c..ea65ede2b 100644 --- a/services/data-manager/src/jobs/transitous/assemble-staging.ts +++ b/services/data-manager/src/jobs/transitous/assemble-staging.ts @@ -9,7 +9,7 @@ import { rmSync, } from "node:fs"; import { join } from "node:path"; -import { CANDIDATE_PROXY_DIRNAME, createCandidateManifest } from "./candidate.js"; +import { CANDIDATE_PROXY_DIRNAME, createCandidateManifestAsync } from "./candidate.js"; import { GTFS_ARCHIVE_RE } from "./internal.js"; import { readTransitSourceManifest, TRANSIT_SOURCE_MANIFEST_FILENAME } from "./source-manifest.js"; import type { StageFn, StageResult } from "./types.js"; @@ -49,6 +49,12 @@ function linkOrCopy(src: string, dest: string): void { } } +/** Metadata is rewritten in place by catalog tools; it must never share candidate inodes. */ +function copyMetadata(src: string, dest: string): void { + if (existsSync(dest)) rmSync(dest, { force: true }); + copyFileSync(src, dest); +} + /** Extract the GTFS archive filenames + OSM extract a MOTIS `config.yml` references. */ export function parseConfigInputs(configText: string): { gtfs: string[]; osm?: string } { const gtfs = new Set(); @@ -92,7 +98,7 @@ export const run: StageFn = async (ctx) => { mkdirSync(stagingDir, { recursive: true }); // config.yml — the import-time entrypoint. - linkOrCopy(configSrc, join(stagingDir, "config.yml")); + copyMetadata(configSrc, join(stagingDir, "config.yml")); // GTFS/NeTEx archives the config references — hardlinked from the build output. const linkedFeeds: string[] = []; @@ -148,10 +154,10 @@ export const run: StageFn = async (ctx) => { if (!existsSync(licenseSrc)) { return finish("error", `required attribution artifact missing at ${licenseSrc}`); } - linkOrCopy(licenseSrc, join(stagingDir, "license.json")); + copyMetadata(licenseSrc, join(stagingDir, "license.json")); const sourceIndexSrc = join(outDir, "gbfs-source-index.json"); if (existsSync(sourceIndexSrc)) { - linkOrCopy(sourceIndexSrc, join(stagingDir, "gbfs-source-index.json")); + copyMetadata(sourceIndexSrc, join(stagingDir, "gbfs-source-index.json")); } const transitSourceManifest = join(outDir, TRANSIT_SOURCE_MANIFEST_FILENAME); if (!existsSync(transitSourceManifest)) { @@ -160,7 +166,7 @@ export const run: StageFn = async (ctx) => { `required transit source manifest missing at ${transitSourceManifest}`, ); } - linkOrCopy(transitSourceManifest, join(stagingDir, TRANSIT_SOURCE_MANIFEST_FILENAME)); + copyMetadata(transitSourceManifest, join(stagingDir, TRANSIT_SOURCE_MANIFEST_FILENAME)); const proxyCandidateSrc = join(outDir, CANDIDATE_PROXY_DIRNAME); if (!existsSync(proxyCandidateSrc)) { @@ -205,11 +211,12 @@ export const run: StageFn = async (ctx) => { ); } - const manifest = createCandidateManifest( + const manifest = await createCandidateManifestAsync( stagingDir, ctx.jobId, ctx.now(), ctx.operationsPolicy, + ctx.abortSignal, ); const status = missingFeeds.length > 0 || osmSource === "missing" ? "partial" : "ok"; diff --git a/services/data-manager/src/jobs/transitous/candidate.ts b/services/data-manager/src/jobs/transitous/candidate.ts index c8dee9b49..c093a917b 100644 --- a/services/data-manager/src/jobs/transitous/candidate.ts +++ b/services/data-manager/src/jobs/transitous/candidate.ts @@ -1,10 +1,25 @@ import { createHash } from "node:crypto"; -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { + closeSync, + constants, + existsSync, + fstatSync, + lstatSync, + openSync, + readdirSync, + readFileSync, + readSync, + realpathSync, + statSync, +} from "node:fs"; import { basename, join } from "node:path"; import { type MotisConfigExpectations, parseMotisConfigExpectations, } from "@openmapx/transitous-core"; +import { hashArtifactFile } from "../../../../../packages/core/src/build-artifacts/copy-tree"; +import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { safeArtifactPath } from "../../../../../packages/core/src/build-artifacts/manifest"; import { atomicWriteJsonSync } from "../../utils/atomic-write.js"; import { GTFS_ARCHIVE_RE } from "./internal.js"; import type { MotisOperationsPolicy } from "./operations-profile.js"; @@ -56,12 +71,42 @@ export interface MotisCandidateManifest { export { parseMotisConfigExpectations } from "@openmapx/transitous-core"; export function sha256File(path: string, relativeTo?: string): CandidateArtifactHash { - const bytes = readFileSync(path); - return { - path: relativeTo ? path.slice(relativeTo.length + 1) : basename(path), - sha256: createHash("sha256").update(bytes).digest("hex"), - sizeBytes: bytes.byteLength, - }; + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = fstatSync(fd, { bigint: true }); + if (!before.isFile() || before.size > BigInt(Number.MAX_SAFE_INTEGER)) + throw Error("Regular bounded candidate artifact required"); + const hash = createHash("sha256"), + buffer = Buffer.alloc(1024 * 1024); + let total = 0; + for (;;) { + const count = readSync(fd, buffer, 0, buffer.length, null); + if (!count) break; + total += count; + if (BigInt(total) > before.size) throw Error("Candidate artifact changed while hashing"); + hash.update(buffer.subarray(0, count)); + } + const after = fstatSync(fd, { bigint: true }), + named = lstatSync(path, { bigint: true }); + if ( + BigInt(total) !== before.size || + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + named.dev !== before.dev || + named.ino !== before.ino + ) + throw Error("Candidate artifact changed while hashing"); + return { + path: relativeTo ? path.slice(relativeTo.length + 1) : basename(path), + sha256: hash.digest("hex"), + sizeBytes: total, + }; + } finally { + closeSync(fd); + } } function parseFloatSetting(name: string, fallback: number): number { @@ -107,12 +152,18 @@ export function resolveCandidateCanary(): MotisCandidateManifest["canary"] { return canary; } -export function createCandidateManifest( +function createCandidateManifestWithHashes( stagingDir: string, epoch: string, createdAt: string, - operationsPolicy?: MotisOperationsPolicy, + operationsPolicy: MotisOperationsPolicy | undefined, + hashes: ReadonlyMap, ): MotisCandidateManifest { + const hash = (path: string, _relativeTo: string) => { + const result = hashes.get(path); + if (!result) throw Error("Candidate artifact hash missing"); + return result; + }; const configPath = join(stagingDir, "config.yml"); const licensePath = join(stagingDir, "license.json"); const proxyRoot = join(stagingDir, CANDIDATE_PROXY_DIRNAME); @@ -126,7 +177,7 @@ export function createCandidateManifest( const datasets = readdirSync(stagingDir) .filter((entry) => GTFS_ARCHIVE_RE.test(entry)) .sort() - .map((entry) => sha256File(join(stagingDir, entry), stagingDir)); + .map((entry) => hash(join(stagingDir, entry), stagingDir)); if (datasets.length === 0) throw new Error("Candidate contains no timetable datasets"); const sourceIndexPath = join(stagingDir, "gbfs-source-index.json"); const sourceManifestPath = join(stagingDir, TRANSIT_SOURCE_MANIFEST_FILENAME); @@ -141,15 +192,13 @@ export function createCandidateManifest( expectations: parseMotisConfigExpectations(readFileSync(configPath, "utf-8")), canary: resolveCandidateCanary(), artifacts: { - config: sha256File(configPath, stagingDir), - license: sha256File(licensePath, stagingDir), - proxyConfig: sha256File(proxyConfigPath, stagingDir), - proxyVars: sha256File(proxyVarsPath, stagingDir), + config: hash(configPath, stagingDir), + license: hash(licensePath, stagingDir), + proxyConfig: hash(proxyConfigPath, stagingDir), + proxyVars: hash(proxyVarsPath, stagingDir), datasets, - ...(existsSync(sourceIndexPath) - ? { sourceIndex: sha256File(sourceIndexPath, stagingDir) } - : {}), - sourceManifest: sha256File(sourceManifestPath, stagingDir), + ...(existsSync(sourceIndexPath) ? { sourceIndex: hash(sourceIndexPath, stagingDir) } : {}), + sourceManifest: hash(sourceManifestPath, stagingDir), }, }; const output = join(stagingDir, CANDIDATE_MANIFEST_FILENAME); @@ -157,9 +206,81 @@ export function createCandidateManifest( return manifest; } +function candidateArtifactPaths(stagingDir: string): string[] { + const required = [ + "config.yml", + "license.json", + `${CANDIDATE_PROXY_DIRNAME}/conf/default.conf`, + `${CANDIDATE_PROXY_DIRNAME}/feed-proxy-vars.json`, + TRANSIT_SOURCE_MANIFEST_FILENAME, + ]; + for (const name of required) + if (!existsSync(join(stagingDir, name)) || !statSync(join(stagingDir, name)).isFile()) + throw Error(`Required candidate artifact missing: ${name}`); + if (statSync(join(stagingDir, "config.yml")).size > 32 * 1024 * 1024) + throw Error("Candidate configuration exceeds its byte budget"); + const datasets = readdirSync(stagingDir) + .filter((entry) => GTFS_ARCHIVE_RE.test(entry)) + .sort(); + if (!datasets.length || datasets.length > 16384) + throw Error("Candidate contains no timetable datasets or exceeds its entry budget"); + return [ + ...required, + ...datasets, + ...(existsSync(join(stagingDir, "gbfs-source-index.json")) ? ["gbfs-source-index.json"] : []), + ]; +} +export function createCandidateManifest( + stagingDir: string, + epoch: string, + createdAt: string, + operationsPolicy?: MotisOperationsPolicy, +): MotisCandidateManifest { + const hashes = new Map( + candidateArtifactPaths(stagingDir).map((name) => [ + join(stagingDir, name), + sha256File(join(stagingDir, name), stagingDir), + ]), + ); + return createCandidateManifestWithHashes(stagingDir, epoch, createdAt, operationsPolicy, hashes); +} +export async function createCandidateManifestAsync( + stagingDir: string, + epoch: string, + createdAt: string, + operationsPolicy?: MotisOperationsPolicy, + signal?: AbortSignal, +): Promise { + const root = new AnchoredDirectory(realpathSync(stagingDir)); + try { + const hashes = new Map(); + for (const name of candidateArtifactPaths(stagingDir)) + hashes.set(join(stagingDir, name), await hashArtifactFile(root, name, signal)); + signal?.throwIfAborted(); + return createCandidateManifestWithHashes( + stagingDir, + epoch, + createdAt, + operationsPolicy, + hashes, + ); + } finally { + root.close(); + } +} export function readCandidateManifest(stagingDir: string): MotisCandidateManifest { - const path = join(stagingDir, CANDIDATE_MANIFEST_FILENAME); - const manifest = JSON.parse(readFileSync(path, "utf-8")) as MotisCandidateManifest; + const path = join(stagingDir, CANDIDATE_MANIFEST_FILENAME), + root = new AnchoredDirectory(realpathSync(stagingDir)); + let manifest: MotisCandidateManifest; + try { + manifest = JSON.parse( + root.read(CANDIDATE_MANIFEST_FILENAME, 32 * 1024 * 1024, false).toString("utf8"), + ); + } catch { + throw Error("Unsupported or malformed candidate manifest"); + } finally { + root.close(); + } if ( manifest.schemaVersion !== 1 || !manifest.epoch || @@ -190,3 +311,38 @@ export function verifyCandidateManifest(stagingDir: string): MotisCandidateManif } return manifest; } + +export async function verifyCandidateManifestAsync( + stagingDir: string, + signal?: AbortSignal, +): Promise { + const manifest = readCandidateManifest(stagingDir), + root = new AnchoredDirectory(realpathSync(stagingDir)); + try { + if ( + !Array.isArray(manifest.artifacts.datasets) || + manifest.artifacts.datasets.length === 0 || + manifest.artifacts.datasets.length > 16384 + ) + throw Error("Candidate dataset inventory is invalid"); + const artifacts = [ + manifest.artifacts.config, + manifest.artifacts.license, + manifest.artifacts.proxyConfig, + manifest.artifacts.proxyVars, + ...manifest.artifacts.datasets, + ...(manifest.artifacts.sourceIndex ? [manifest.artifacts.sourceIndex] : []), + manifest.artifacts.sourceManifest, + ]; + for (const expected of artifacts) { + if (!expected || typeof expected.path !== "string" || !safeArtifactPath(expected.path)) + throw Error("Candidate artifact path is invalid"); + const current = await hashArtifactFile(root, expected.path, signal); + if (current.sha256 !== expected.sha256 || current.sizeBytes !== expected.sizeBytes) + throw Error(`Candidate artifact hash mismatch: ${expected.path}`); + } + return manifest; + } finally { + root.close(); + } +} diff --git a/services/data-manager/src/jobs/transitous/catalog-activation-record.ts b/services/data-manager/src/jobs/transitous/catalog-activation-record.ts new file mode 100644 index 000000000..3f476ade2 --- /dev/null +++ b/services/data-manager/src/jobs/transitous/catalog-activation-record.ts @@ -0,0 +1,100 @@ +/** The fixed approval child receives the actual already-settled promotion, never a caller assertion. */ +import { + controlDigest, + executionAuthorityState, + inspectArtifactTargetLeaseChildSettlement, + signControl, +} from "@openmapx/core/build-artifacts"; +import { + CATALOG_ACTIVATION_DOMAIN, + catalogActivationRecordSchema, + readCatalogValidation, + scopeCatalogRuntimeInputs, + withCatalogProposalDirectory, +} from "@openmapx/core/catalog-pins"; +import { canonicalJson, type ExecutionRef } from "@openmapx/core/ops"; +import { runOpsOperation } from "../../ops-client.js"; +import type { DataExecutionContext } from "../execution/runtime.js"; +import { publishDataChange } from "../execution/worker-scope.js"; +export async function retainCatalogActivationRecord( + context: DataExecutionContext, + approval: ExecutionRef, +): Promise { + context.assertForward(); + const record = await context.record(); + if (record.input.kind !== "catalog.activate") throw Error("Original activation input required"); + const input = record.input, + selected = await runOpsOperation( + { kind: "transitousLock.inspect", proposalId: input.proposalId, offset: 0 }, + { signal: context.signal }, + ); + const validation = readCatalogValidation( + context.installation, + input.proposalId, + input.validationId, + ); + const promotion = record.children.find((child) => child.phase === "promote-primary"); + if ( + !selected.proposal || + !selected.validationInputs || + !promotion || + promotion.state !== "terminal" || + promotion.settlement.outcome !== "committed" || + !record.reservations.some( + (child) => + child.phase === "paired-pin-commit" && + child.proposedId === approval.executionId && + child.operationFingerprint === approval.operationFingerprint, + ) + ) + throw Error("Original settled promotion and fixed approval reservation required"); + const physical = inspectArtifactTargetLeaseChildSettlement( + context.targetAuthority, + promotion.execution, + ); + if ( + physical?.outcome !== "committed" || + physical.terminalReceiptDigest !== promotion.settlement.terminalReceiptDigest + ) + throw Error("Original promotion physical settlement required"); + const body = catalogActivationRecordSchema.parse({ + version: 1, + installationId: context.installation.installationId, + parent: context.execution, + proposalId: input.proposalId, + validationId: input.validationId, + proposalDigest: controlDigest(selected.proposal), + validationDigest: controlDigest(validation), + diffDigest: input.diffDigest, + runtimeInputs: scopeCatalogRuntimeInputs( + selected.validationInputs, + validation.validation.countries, + ), + promotion: { + execution: promotion.execution, + terminalReceiptDigest: physical.terminalReceiptDigest, + adoption: { + leaseId: promotion.leaseId, + delegationId: promotion.delegationId, + phase: "ready", + }, + }, + approval, + }); + const owner = executionAuthorityState(context.authority); + await publishDataChange(() => + withCatalogProposalDirectory(context.installation, input.proposalId, (directory) => { + const name = `activation-${context.execution.executionId}.json`, + bytes = canonicalJson( + signControl(owner.installation.privateKey, CATALOG_ACTIVATION_DOMAIN, body), + ); + try { + const previous = directory.read(name, 128 * 1024, true).toString("utf8"); + if (previous !== bytes) throw Error("Original immutable catalog activation record changed"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + directory.write(name, bytes, true); + } + }), + ); +} diff --git a/services/data-manager/src/jobs/transitous/catalog-activation.ts b/services/data-manager/src/jobs/transitous/catalog-activation.ts new file mode 100644 index 000000000..5b852d900 --- /dev/null +++ b/services/data-manager/src/jobs/transitous/catalog-activation.ts @@ -0,0 +1,193 @@ +/** Activate exactly one reviewed candidate under the existing finite root and child reservations. */ +import { createHash } from "node:crypto"; +import { realpathSync } from "node:fs"; +import { controlDigest, verifyDataControlTerminalWitness } from "@openmapx/core/build-artifacts"; +import { + catalogProposalSchema, + catalogRuntimeInputsSchema, + readCatalogValidation, + readCurrentCatalogValidation, + sameCatalogRuntimeInputs, + scopeCatalogRuntimeInputs, +} from "@openmapx/core/catalog-pins"; +import type { DataExecutionResult } from "@openmapx/core/data-manager-contracts"; +import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { asJobLogger, jobChildLogger } from "../../logger.js"; +import { runOpsOperation } from "../../ops-client.js"; +import type { StateStore } from "../../state.js"; +import { dataExecutionDiagnostic } from "../execution/diagnostics.js"; +import { DataOpsObservationPending, runOwnedDataOpsOperation } from "../execution/ops-children.js"; +import type { DataExecutionContext } from "../execution/runtime.js"; +import { DataPublicationUnknown, publishDataChange } from "../execution/worker-scope.js"; +import { CANDIDATE_MANIFEST_FILENAME, verifyCandidateManifestAsync } from "./candidate.js"; +import { run as checkHealth } from "./motis-health.js"; +import type { MotisOperationsPolicy } from "./operations-profile.js"; +import { runTransitOwnedPhase } from "./owned-ops.js"; +import { makePersistingOnStageComplete } from "./persistence.js"; +import { buildJobContext } from "./pipeline.js"; +import { run as promote } from "./promote.js"; +import { rollbackProxyTransaction, run as stageProxy } from "./proxy-transaction.js"; +import { readMotisSlotState } from "./slot-state.js"; +export async function runCatalogActivation(options: { + context: DataExecutionContext; + dataDir: string; + store: StateStore; + operationsPolicy: MotisOperationsPolicy; +}): Promise> { + const { context, dataDir } = options, + record = await context.record(); + if (record.input.kind !== "catalog.activate" || !record.input.acknowledgeLicenseChanges) + throw Error("Acknowledge the complete catalog diff before activation"); + const input = record.input, + validation = readCatalogValidation(context.installation, input.proposalId, input.validationId), + passed = validation.validation; + if ( + passed.status !== "passed" || + Date.now() >= Date.parse(passed.expiresAt) || + Date.parse(passed.completedAt) > Date.now() || + Date.parse(passed.expiresAt) > Date.parse(passed.completedAt) + 86400000 + ) + throw Error("Catalog validation is failed or expired"); + const currentValidation = readCurrentCatalogValidation(context.installation); + if ( + currentValidation?.selected?.proposalId !== input.proposalId || + currentValidation.selected.validationId !== input.validationId || + currentValidation.selected.validationDigest !== controlDigest(validation) + ) + throw Error("Validation was invalidated before slot reuse; validate again"); + const terminal = verifyDataControlTerminalWitness( + context.installation, + validation.sourceExecution, + ); + if ( + terminal.status.status !== "ok" || + terminal.status.result?.kind !== "catalog.validate" || + terminal.status.result.status !== "passed" || + terminal.status.result.validationId !== input.validationId + ) + throw Error("Original successful validation completion required"); + const selected = await runOpsOperation( + { kind: "transitousLock.inspect", proposalId: input.proposalId, offset: 0 }, + { signal: context.signal }, + ), + proposal = catalogProposalSchema.parse(selected.proposal), + active = await runOpsOperation({ kind: "catalogPins.inspect" }, { signal: context.signal }); + if ( + controlDigest(proposal) !== validation.proposalDigest || + proposal.pins.generationId !== passed.pinDigest || + proposal.diffDigest !== input.diffDigest || + proposal.baseGenerationId !== input.expectedGenerationId || + active.catalogRevisionId !== input.expectedGenerationId || + !selected.validationInputs || + !sameCatalogRuntimeInputs( + validation.runtimeInputs, + scopeCatalogRuntimeInputs( + catalogRuntimeInputsSchema.parse(selected.validationInputs), + passed.countries, + ), + ) + ) + throw Error("Catalog validation inputs changed; validate again"); + const initial = readMotisSlotState(dataDir); + if (!initial || initial.activeSlot === passed.slot) + throw Error("Validated inactive slot is no longer available"); + const logger = asJobLogger( + jobChildLogger({ job: "catalog.activate", jobId: context.execution.executionId }), + ); + const ctx = await publishDataChange(() => + buildJobContext({ + dataDir, + store: options.store, + countries: passed.countries, + source: options.operationsPolicy.acquisition, + operationsPolicy: { ...options.operationsPolicy, countries: passed.countries }, + jobId: context.execution.executionId, + executionContext: context, + abortSignal: context.signal, + logger, + onStageComplete: async (stage) => { + await makePersistingOnStageComplete(context.execution.executionId, logger)(stage); + const diagnostic = dataExecutionDiagnostic(stage.message ?? stage.stage); + await context.checkpoint({ + diagnostics: { + tail: [{ stream: "stdout", message: diagnostic.message }], + truncated: diagnostic.truncated, + }, + }); + }, + }), + ); + if (!ctx.slotLayout) throw Error("Paired activation requires the two-slot layout"); + const manifest = await verifyCandidateManifestAsync(ctx.motisStagingDataDir, context.signal), + directory = new AnchoredDirectory(realpathSync(ctx.motisStagingDataDir)); + try { + if ( + manifest.epoch !== passed.epoch || + createHash("sha256") + .update(directory.read(CANDIDATE_MANIFEST_FILENAME, 32 * 1024 * 1024, false)) + .digest("hex") !== passed.manifestHash + ) + throw Error("Validated candidate was replaced"); + } finally { + directory.close(); + } + ctx.state.catalogPins = proposal.pins; + ctx.state.transitousLock = proposal.pins.transitous; + try { + const proxy = await stageProxy(ctx); + await ctx.onStageComplete(proxy); + await context.checkpoint({ progress: 0.2 }); + if (proxy.status !== "ok") throw Error("Candidate proxy preparation failed"); + await runTransitOwnedPhase(ctx, "restart-staging"); + const health = await checkHealth(ctx); + await ctx.onStageComplete(health); + await context.checkpoint({ progress: 0.5 }); + if (health.status !== "ok") throw Error("Validated candidate no longer passes staging probes"); + const result = await promote(ctx); + await ctx.onStageComplete(result); + await context.checkpoint({ progress: 0.8 }); + if (result.status !== "ok") throw Error("Candidate promotion did not complete"); + } catch (error) { + if (error instanceof DataOpsObservationPending || error instanceof DataPublicationUnknown) + throw error; + context.assertForward(); + await rollbackProxyTransaction(ctx); + await runTransitOwnedPhase(ctx, "stop-staging"); + const current = readMotisSlotState(dataDir); + if ( + current?.activeSlot !== initial.activeSlot || + current?.datasetEpoch !== initial.datasetEpoch + ) + throw new DataOpsObservationPending(context.execution.operationKey); + return { + kind: input.kind, + proposalId: input.proposalId, + validationId: input.validationId, + outcome: "not_activated", + generationId: input.expectedGenerationId, + reason: + "Candidate probes or promotion failed; the previous active slot and catalog pins remain selected.", + }; + } + const slot = record.reservations.find( + (child) => child.phase === "paired-pin-commit" && child.ordinal === 0, + ); + if (!slot || slot.operation.kind !== "transitousLock.approve") + throw Error("Original paired publication reservation required"); + // From here the dataset is active. Any missing pin receipt is unknown and + // fenced, never reported as rolled back or retried as a different attempt. + const committed = await runOwnedDataOpsOperation(slot.operation, { + context, + phase: slot.phase, + ordinal: slot.ordinal, + serviceIds: slot.serviceIds, + }); + return { + kind: input.kind, + proposalId: input.proposalId, + validationId: input.validationId, + outcome: "activated", + generationId: committed.generationId, + reason: null, + }; +} diff --git a/services/data-manager/src/jobs/transitous/catalog-review.ts b/services/data-manager/src/jobs/transitous/catalog-review.ts new file mode 100644 index 000000000..b07341773 --- /dev/null +++ b/services/data-manager/src/jobs/transitous/catalog-review.ts @@ -0,0 +1,399 @@ +/** Paired proposal resolution. Validation and activation consume the same retained intent. */ +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, realpathSync } from "node:fs"; +import { join } from "node:path"; +import { + type ControlInstallation, + controlDigest, + signControl, + verifyControl, + verifyDataControlTerminalWitness, +} from "@openmapx/core/build-artifacts"; +import { + type CatalogProposal, + catalogProposalInputSchema, + catalogProposalSchema, + createCatalogPinSet, + parseCatalogPinSet, + readCatalogValidation, + readCurrentCatalogValidation, + sameCatalogRuntimeInputs, + scopeCatalogRuntimeInputs, +} from "@openmapx/core/catalog-pins"; +import { canonicalJson } from "@openmapx/core/ops"; +import { safeFetchTextResponse } from "@openmapx/core/utils/safe-download"; +import { + type CatalogFeedDiff, + catalogCompilerSettings, + catalogDiffPage, + catalogReviewInputDigests, + fetchPinnedGbfsCatalog, + parseCatalogGbfsRows, + readAtlasFeedSnapshot, + readCatalogFeedSnapshot, + resolveGbfsCatalogCandidate, + reviewCatalogPair, +} from "@openmapx/transitous-core"; +import { execa } from "execa"; +import z from "zod/v4"; +import { installationState } from "../../../../../packages/core/src/build-artifacts/control-authority"; +import { + AnchoredDirectory, + identity, + sameInode, +} from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { runOpsOperation } from "../../ops-client.js"; +import { withLegacyDataMutation } from "../execution/legacy-mutation.js"; +import { readFeedOverlay } from "../transitous-feeds-overlay.js"; +import { CANDIDATE_MANIFEST_FILENAME } from "./candidate.js"; +import { + CatalogBumpError, + lockFromCandidate, + resolveCatalogBumpCandidate, +} from "./catalog-bump.js"; +import { resolveTransitOverlayPath } from "./internal.js"; +import { type MotisOperationsPolicy, transitAllowedCountries } from "./operations-profile.js"; +import { readMotisSlotState } from "./slot-state.js"; + +const DOMAIN = "openmapx-catalog-resolution-v1"; +const MAX_DIFF_BYTES = 64 * 1024 * 1024; +const resolutionSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + branch: z + .string() + .max(215) + .regex(/^[A-Za-z0-9][A-Za-z0-9._/-]*$/), + proposal: catalogProposalInputSchema, + overlayDigest: z.string().regex(/^[a-f0-9]{64}$/), + createdAt: z.iso.datetime({ offset: true }), +}); +const missing = (error: unknown) => (error as NodeJS.ErrnoException).code === "ENOENT"; +const safeUpstreamFetch: typeof fetch = async (input, init) => { + const response = await safeFetchTextResponse(String(input), { + maxBytes: String(input).includes("api.github.com") ? 65536 : 32 * 1024 * 1024, + timeoutMs: 30000, + signal: init?.signal ?? undefined, + maxRedirects: 0, + allowPrivateHosts: [], + headers: Object.fromEntries(new Headers(init?.headers)), + }); + if (response.finalUrl !== String(input)) throw Error("Catalog upstream changed its fixed URL"); + return new Response(response.text, { status: response.status, headers: response.headers }); +}; + +/** Dependencies are the literal installed service, not values or callbacks from a request. */ +export function createCatalogReviewService(options: { + dataDir: string; + installation: ControlInstallation; + operationsPolicy: MotisOperationsPolicy; +}) { + const state = installationState(options.installation); + if (options.installation.role !== "data-manager" || state.fixedChildId) + throw Error("Original Data Manager installation required"); + const expected = new AnchoredDirectory(state.dataRoot), + actual = new AnchoredDirectory(options.dataDir); + try { + if (!sameInode(identity(expected.stat), identity(actual.stat))) + throw Error("Foreign catalog review data root"); + } finally { + expected.close(); + actual.close(); + } + function directory( + proposalId: string, + create: boolean, + run: (directory: AnchoredDirectory) => T, + ): T { + const id = catalogProposalInputSchema.shape.proposalId.parse(proposalId), + opened: AnchoredDirectory[] = []; + try { + let root = new AnchoredDirectory(options.dataDir); + opened.push(root); + for (const name of ["motis", "proposals", id]) { + root = root.directory(name, create, name !== "motis"); + opened.push(root); + } + return run(root); + } finally { + for (const root of opened.reverse()) root.close(); + } + } + function sourceSnapshot() { + const overlay = readFeedOverlay(resolveTransitOverlayPath(options.dataDir)); + return { + overlay, + ...catalogReviewInputDigests( + options.operationsPolicy, + overlay, + catalogCompilerSettings(process.env), + ), + }; + } + function retainedResolution(proposalId: string) { + try { + return directory(proposalId, false, (dir) => { + const value = verifyControl( + state.installation.publicKey, + DOMAIN, + JSON.parse(dir.read("resolution.json", 768 * 1024).toString("utf8")), + resolutionSchema, + ); + if ( + value.installationId !== options.installation.installationId || + value.proposal.proposalId !== proposalId + ) + throw Error("Foreign catalog resolution"); + parseCatalogPinSet(value.proposal.pins); + const bytes = dir.read("diff.json", MAX_DIFF_BYTES); + if (createHash("sha256").update(bytes).digest("hex") !== value.proposal.diffDigest) + throw Error("Retained catalog diff changed"); + return value; + }); + } catch (error) { + if (missing(error)) return null; + throw error; + } + } + async function readProposal(proposalId: string): Promise { + const result = await runOpsOperation({ kind: "transitousLock.inspect", proposalId, offset: 0 }); + const proposal = catalogProposalSchema.parse(result.proposal); + if (proposal.proposalId !== proposalId) throw Error("Catalog proposal identity changed"); + parseCatalogPinSet(proposal.pins); + return proposal; + } + async function resolveCatalogProposal(input: { + branch: string; + signal?: AbortSignal; + proposalId?: string; + }): Promise { + const branch = resolutionSchema.shape.branch.parse(input.branch), + proposalId = input.proposalId ?? randomUUID(); + return withLegacyDataMutation("transit-proposal", async () => { + input.signal?.throwIfAborted(); + let resolution = retainedResolution(proposalId); + if (resolution && resolution.branch !== branch) + throw Error("Original catalog branch changed"); + if (!resolution) { + const active = await runOpsOperation( + { kind: "catalogPins.inspect" }, + { signal: input.signal }, + ), + snapshot = sourceSnapshot(); + const catalogDir = join(options.dataDir, ".transitous-catalog"); + const candidate = await resolveCatalogBumpCandidate({ catalogDir, branch }); + const gbfs = await resolveGbfsCatalogCandidate( + "catalog-review", + safeUpstreamFetch, + () => new Date(), + input.signal, + ); + const pins = createCatalogPinSet( + lockFromCandidate( + candidate, + "catalog-review", + "Proposed for explicit source/license review", + ), + gbfs.lock, + ); + const gitAt = async (directory: string, args: string[]) => + ( + await execa("git", ["-c", `safe.directory=${directory}`, "-C", directory, ...args], { + cancelSignal: input.signal, + timeout: 30000, + maxBuffer: 16 * 1024 * 1024, + }) + ).stdout; + const git = (args: string[]) => gitAt(catalogDir, args), + atlasDir = join(catalogDir, "transitland-atlas"); + const atlasCommits = [ + ...new Set([ + candidate.transitlandAtlasSha, + ...(active.active ? [active.active.transitous.submodules["transitland-atlas"]!] : []), + ]), + ]; + if (!existsSync(join(atlasDir, ".git"))) + throw new CatalogBumpError( + "submodule-cache-unavailable", + "Initialize the pinned catalog and Atlas cache before proposing updates", + ); + for (const commit of atlasCommits) { + try { + await gitAt(atlasDir, ["cat-file", "-e", `${commit}^{commit}`]); + } catch (error) { + if (![1, 128].includes((error as { exitCode?: number }).exitCode ?? -1)) throw error; + await gitAt(atlasDir, ["fetch", "--no-tags", "--depth=1", "origin", commit]); + await gitAt(atlasDir, ["cat-file", "-e", `${commit}^{commit}`]); + } + } + const [previousFeeds, nextFeeds, previousGbfs] = await Promise.all([ + active.active + ? readCatalogFeedSnapshot(active.active.transitous.ref.split("@").at(-1)!, git) + : Promise.resolve(new Map()), + readCatalogFeedSnapshot(candidate.transitousSha, git), + active.active + ? fetchPinnedGbfsCatalog(active.active.gbfs, safeUpstreamFetch, input.signal) + : Promise.resolve(null), + ]); + const [previousAtlas, nextAtlas] = await Promise.all([ + active.active + ? readAtlasFeedSnapshot( + active.active.transitous.submodules["transitland-atlas"]!, + (args) => gitAt(atlasDir, args), + ) + : Promise.resolve(new Map()), + readAtlasFeedSnapshot(candidate.transitlandAtlasSha, (args) => gitAt(atlasDir, args)), + ]); + const diff = reviewCatalogPair({ + previousAtlas, + nextAtlas, + previousFeeds, + nextFeeds, + previousGbfsRows: previousGbfs ? parseCatalogGbfsRows(previousGbfs) : [], + nextGbfsRows: gbfs.rows, + countries: transitAllowedCountries(options.operationsPolicy), + quarantine: snapshot.overlay?.quarantine ?? [], + maxAdditions: Number.parseInt( + catalogCompilerSettings(process.env).MOTIS_GBFS_CATALOG_MAX_ADDITIONS!, + 10, + ), + }); + // Preflight every complete row before retaining a proposal that the + // bounded review endpoint could not actually display. + for (let offset = 0; offset < diff.total; ) { + const page = catalogDiffPage(diff, offset); + if (page.nextOffset === null) break; + offset = page.nextOffset; + } + const bytes = canonicalJson(diff); + if (Buffer.byteLength(bytes, "utf8") > MAX_DIFF_BYTES) + throw Error("Catalog review exceeds its byte limit"); + if (canonicalJson(snapshot) !== canonicalJson(sourceSnapshot())) + throw Error("Catalog source policy changed during resolution"); + resolution = resolutionSchema.parse({ + version: 1, + installationId: options.installation.installationId, + branch, + proposal: { + proposalId, + baseGenerationId: active.catalogRevisionId, + pins, + diffDigest: createHash("sha256").update(bytes).digest("hex"), + sourcePolicyRevision: snapshot.sourcePolicyRevision, + }, + overlayDigest: snapshot.overlayDigest, + createdAt: new Date().toISOString(), + }); + directory(proposalId, true, (dir) => { + dir.write("diff.json", bytes, true); + const envelope = canonicalJson(signControl(state.privateKey, DOMAIN, resolution)); + if (Buffer.byteLength(envelope, "utf8") > 768 * 1024) + throw Error("Catalog resolution exceeds its byte limit"); + dir.write("resolution.json", envelope, true); + }); + } + await runOpsOperation( + { kind: "transitousLock.propose", ...resolution.proposal }, + { signal: input.signal }, + ); + return readProposal(proposalId); + }); + } + function reviewPage(proposalId: string, offset = 0) { + const resolution = retainedResolution(proposalId); + if (!resolution) throw Error("Catalog proposal resolution is unavailable"); + return directory(proposalId, false, (dir) => ({ + ...catalogDiffPage( + JSON.parse(dir.read("diff.json", MAX_DIFF_BYTES).toString("utf8")) as CatalogFeedDiff, + offset, + ), + digest: resolution.proposal.diffDigest, + })); + } + async function reviewState(query: { + proposalId?: string; + offset?: number; + diffOffset?: number; + validationId?: string; + }) { + const [host, active] = await Promise.all([ + runOpsOperation({ + kind: "transitousLock.inspect", + ...(query.proposalId ? { proposalId: query.proposalId } : {}), + offset: query.offset ?? 0, + }), + runOpsOperation({ kind: "catalogPins.inspect" }), + ]); + const proposal = host.proposal ?? null, + diff = proposal ? reviewPage(proposal.proposalId, query.diffOffset ?? 0) : null; + let validation: import("@openmapx/core/catalog-pins").CatalogValidation | null = null, + validationEligible = false, + blocker = host.validationBlocker ?? null; + if (proposal && query.validationId) { + try { + const retained = readCatalogValidation( + options.installation, + proposal.proposalId, + query.validationId, + ); + validation = retained.validation; + if (validation.status === "passed") { + const completed = verifyDataControlTerminalWitness( + options.installation, + retained.sourceExecution, + ), + pointer = readCurrentCatalogValidation(options.installation), + slots = readMotisSlotState(options.dataDir); + if ( + completed.status.status !== "ok" || + completed.status.result?.kind !== "catalog.validate" || + completed.status.result.status !== "passed" || + pointer?.selected?.validationId !== validation.validationId || + pointer.selected.validationDigest !== controlDigest(retained) || + Date.now() >= Date.parse(validation.expiresAt) || + controlDigest(proposal) !== retained.proposalDigest || + proposal.baseGenerationId !== active.catalogRevisionId || + !host.validationInputs || + !sameCatalogRuntimeInputs( + retained.runtimeInputs, + scopeCatalogRuntimeInputs(host.validationInputs, validation.countries), + ) || + !slots || + slots.activeSlot === validation.slot + ) + throw Error("Catalog validation is stale, incomplete or already activated"); + const candidate = new AnchoredDirectory( + realpathSync(join(options.dataDir, "motis", "slots", validation.slot)), + ); + try { + if ( + createHash("sha256") + .update(candidate.read(CANDIDATE_MANIFEST_FILENAME, 32 * 1024 * 1024, false)) + .digest("hex") !== validation.manifestHash + ) + throw Error("Validated slot changed"); + } finally { + candidate.close(); + } + validationEligible = true; + } else blocker = validation.reason; + } catch { + blocker ??= + "Validation is incomplete, stale or its slot was reused. Run a fresh validation before activation."; + } + } + return { + ...active, + allowedCountries: transitAllowedCountries(options.operationsPolicy), + page: host.page ?? { proposals: [], total: 0, offset: query.offset ?? 0, nextOffset: null }, + proposal, + diff, + validation, + execution: null, + validationInputs: host.validationInputs ?? null, + validationEligible, + blocker, + }; + } + return { resolveCatalogProposal, readProposal, reviewPage, sourceSnapshot, reviewState }; +} diff --git a/services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts b/services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts new file mode 100644 index 000000000..6dafd7107 --- /dev/null +++ b/services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts @@ -0,0 +1,211 @@ +/** Preserve the complete inactive healthy generation before validation can reuse it. */ +import { createHash } from "node:crypto"; +import { mkdirSync, statfsSync } from "node:fs"; +import { join } from "node:path"; +import { + assertArtifactTargetLeaseForward, + assertControlExecutionForward, + controlDigest, + executionAuthorityState, + inspectArtifactTargetLeaseChildSettlement, + signControl, +} from "@openmapx/core/build-artifacts"; +import { canonicalJson } from "@openmapx/core/ops"; +import z from "zod/v4"; +import { + copyArtifactTree, + inspectArtifactTree, +} from "../../../../../packages/core/src/build-artifacts/copy-tree"; +import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { safeArtifactPath } from "../../../../../packages/core/src/build-artifacts/manifest"; +import { atomicWriteJsonSync } from "../../utils/atomic-write"; +import type { DataExecutionContext } from "../execution/runtime"; +import { dataExecutionInputFingerprint } from "../execution/store"; +import { publishDataChange } from "../execution/worker-scope"; +import { CANDIDATE_MANIFEST_FILENAME } from "./candidate"; +import { readMotisSlotState } from "./slot-state"; + +const artifact = z.strictObject({ + path: z + .string() + .max(4096) + .refine((value) => safeArtifactPath(value)), + sizeBytes: z.number().int().nonnegative().safe(), + sha256: z.string().regex(/^[a-f0-9]{64}$/), +}); +const previousManifest = z.object({ + schemaVersion: z.literal(1), + epoch: z.string().min(1).max(512), + artifacts: z.object({ + config: artifact, + license: artifact, + proxyConfig: artifact, + proxyVars: artifact, + sourceManifest: artifact, + sourceIndex: artifact.optional(), + datasets: z.array(artifact).min(1).max(16384), + }), +}); +export async function retainPreviousHealthyCatalogSlot( + context: DataExecutionContext, + dataDir: string, + proposalId: string, + minimumDiskBytes: number, +): Promise { + const owner = executionAuthorityState(context.authority), + record = await context.record(); + assertControlExecutionForward(context.authority); + assertArtifactTargetLeaseForward(context.targetAuthority); + if ( + owner.kind !== "catalog.validate" || + record.input.kind !== "catalog.validate" || + record.input.proposalId !== proposalId || + dataExecutionInputFingerprint(record.input) !== owner.execution.operationFingerprint || + context.targetAuthority.mode !== "owner" || + !["motis", "motis-staging", "motis-feed-proxy"].every((id) => + context.targetAuthority.serviceIds.includes(id), + ) || + !Number.isSafeInteger(minimumDiskBytes) || + minimumDiskBytes < 0 + ) + throw Error("Original complete catalog validation owner required"); + const state = readMotisSlotState(dataDir); + if (!state) throw Error("Initialize the two-slot MOTIS layout before catalog validation"); + const inactive = state.activeSlot === "A" ? "B" : "A"; + if (state.previousHealthySlot !== inactive) return; + const stop = record.children.find( + (child) => child.phase === "stop-staging" && child.ordinal === 0 && child.state === "terminal", + ); + if ( + !stop || + stop.state !== "terminal" || + inspectArtifactTargetLeaseChildSettlement(context.targetAuthority, stop.execution)?.outcome !== + "committed" + ) + throw Error( + "Stop staging under the original validation reservation before preserving rollback data", + ); + const opened: AnchoredDirectory[] = [], + limits = { + maximumEntries: 100000, + maximumBytes: Number.MAX_SAFE_INTEGER, + maximumManifestBytes: 64 * 1024 * 1024, + }; + const directory = (root: AnchoredDirectory, name: string, create = false, strict = false) => { + const next = root.directory(name, create, strict); + opened.push(next); + return next; + }; + try { + const data = new AnchoredDirectory(owner.installation.dataRoot); + opened.push(data); + const motis = directory(data, "motis"), + slots = directory(motis, "slots"), + source = directory(slots, inactive); + const expected = new AnchoredDirectory(dataDir); + opened.push(expected); + if (expected.stat.dev !== data.stat.dev || expected.stat.ino !== data.stat.ino) + throw Error("Foreign catalog retention root"); + const manifest = previousManifest.parse( + JSON.parse( + source.read(CANDIDATE_MANIFEST_FILENAME, 32 * 1024 * 1024, false).toString("utf8"), + ), + ), + inventory = inspectArtifactTree(source, limits), + capacity = statfsSync(data.anchor, { bigint: true }); + const artifacts = [ + manifest.artifacts.config, + manifest.artifacts.license, + manifest.artifacts.proxyConfig, + manifest.artifacts.proxyVars, + manifest.artifacts.sourceManifest, + ...manifest.artifacts.datasets, + ...(manifest.artifacts.sourceIndex ? [manifest.artifacts.sourceIndex] : []), + ]; + for (const expected of artifacts) + if ( + !inventory.files.some( + (file) => file.path === expected.path && file.sizeBytes === expected.sizeBytes, + ) + ) + throw Error("Rollback generation is incomplete"); + if ( + capacity.bavail * capacity.bsize < + BigInt(Math.max(inventory.totalBytes, minimumDiskBytes)) + 64n * 1024n * 1024n + ) + throw Error( + "Insufficient capacity to retain the previous healthy generation before validation", + ); + const proposals = directory(motis, "proposals", false, true), + base = directory(proposals, proposalId, false, true), + basename = `rollback-${context.execution.executionId}`; + await publishDataChange(async () => { + mkdirSync(base.name(basename), { mode: 0o700 }); + const destination = directory(base, basename, false, true); + const copied = await copyArtifactTree({ + source, + destination, + expectedInventory: inventory.metadataDigest, + signal: context.signal, + ...limits, + }); + const copiedByPath = new Map(copied.files.map((file) => [file.path, file])); + for (const expected of artifacts) { + const actual = copiedByPath.get(expected.path); + if (!actual || actual.sha256 !== expected.sha256 || actual.sizeBytes !== expected.sizeBytes) + throw Error("Rollback generation artifact changed"); + } + const manifestHash = copied.files.find( + (file) => file.path === CANDIDATE_MANIFEST_FILENAME, + )?.sha256; + if (!manifestHash) throw Error("Retained rollback manifest is missing"); + const bytes = canonicalJson({ + files: copied.files, + directories: copied.directories, + totalBytes: copied.totalBytes, + }), + filesDigest = createHash("sha256").update(bytes).digest("hex"); + if (Buffer.byteLength(bytes, "utf8") > limits.maximumManifestBytes) + throw Error("Rollback manifest exceeds its budget"); + base.write(`${basename}.files.json`, bytes, true); + base.write( + `${basename}.json`, + canonicalJson( + signControl(owner.installation.privateKey, "openmapx-catalog-rollback-v1", { + version: 1, + execution: context.execution, + proposalId, + slot: inactive, + epoch: manifest.epoch, + manifestHash, + filesDigest, + totalBytes: copied.totalBytes, + fileCount: copied.files.length, + }), + ), + true, + ); + context.assertForward(); + if (controlDigest(readMotisSlotState(dataDir)) !== controlDigest(state)) + throw Error("MOTIS slot state changed before rollback retention"); + atomicWriteJsonSync( + motis.name("slot-state.json"), + { + ...state, + previousHealthySlot: undefined, + retainedRollback: { + proposalId, + validationId: context.execution.executionId, + slot: inactive, + epoch: manifest.epoch, + manifestHash, + filesDigest, + }, + }, + { durability: "full" }, + ); + }); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} diff --git a/services/data-manager/src/jobs/transitous/catalog-selection.ts b/services/data-manager/src/jobs/transitous/catalog-selection.ts new file mode 100644 index 000000000..413f5b42d --- /dev/null +++ b/services/data-manager/src/jobs/transitous/catalog-selection.ts @@ -0,0 +1,93 @@ +/** A proposal becomes a pipeline input only through its actual admitted owner. */ +import { + assertControlExecutionForward, + controlDigest, + executionAuthorityState, +} from "@openmapx/core/build-artifacts"; +import { + type CatalogProposal, + type CatalogRuntimeInputs, + catalogProposalSchema, + catalogRuntimeInputsSchema, + parseCatalogPinSet, +} from "@openmapx/core/catalog-pins"; +import { runOpsOperation } from "../../ops-client.js"; +import type { DataExecutionContext } from "../execution/runtime.js"; +import { dataExecutionInputFingerprint } from "../execution/store.js"; +import type { JobContext } from "./types.js"; + +declare const selectionBrand: unique symbol; +export type ReviewedCatalogSelection = Readonly<{ + proposalId: string; + catalogRevisionId: string; + [selectionBrand]: true; +}>; +const selections = new WeakMap< + object, + { context: DataExecutionContext; proposal: CatalogProposal; runtimeInputs?: CatalogRuntimeInputs } +>(); +export async function loadOwnedCatalogSelection( + context: DataExecutionContext, + proposalId: string, +): Promise { + const owner = executionAuthorityState(context.authority), + record = await context.record(); + assertControlExecutionForward(context.authority); + context.assertForward(); + if ( + owner.kind !== "catalog.validate" || + record.input.kind !== "catalog.validate" || + record.input.proposalId !== proposalId || + dataExecutionInputFingerprint(record.input) !== owner.execution.operationFingerprint || + controlDigest(owner.execution) !== controlDigest(context.execution) + ) + throw Error("Original catalog validation execution required"); + const [selected, active] = await Promise.all([ + runOpsOperation( + { kind: "transitousLock.inspect", proposalId, offset: 0 }, + { signal: context.signal }, + ), + runOpsOperation({ kind: "catalogPins.inspect" }, { signal: context.signal }), + ]); + const proposal = catalogProposalSchema.parse(selected.proposal); + parseCatalogPinSet(proposal.pins); + if (proposal.proposalId !== proposalId || proposal.baseGenerationId !== active.catalogRevisionId) + throw Error("Catalog proposal base changed"); + const handle = Object.freeze({ + proposalId, + catalogRevisionId: proposal.pins.generationId, + }) as ReviewedCatalogSelection; + selections.set(handle, { + context, + proposal, + ...(selected.validationInputs + ? { runtimeInputs: catalogRuntimeInputsSchema.parse(selected.validationInputs) } + : {}), + }); + return handle; +} +export function selectedCatalogPins(ctx: JobContext) { + const selected = ctx.reviewedCatalog && selections.get(ctx.reviewedCatalog); + if ( + !selected || + selected.context !== ctx.executionContext || + selected.context.execution.executionId !== ctx.jobId + ) + throw Error("Verified owned catalog proposal required"); + assertControlExecutionForward(selected.context.authority); + selected.context.assertForward(); + return parseCatalogPinSet(structuredClone(selected.proposal.pins)); +} + +export function ownedCatalogSelectionEvidence( + handle: ReviewedCatalogSelection, + context: DataExecutionContext, +) { + const selected = selections.get(handle); + if (!selected || selected.context !== context) throw Error("Original catalog selection required"); + assertControlExecutionForward(context.authority); + return { + proposal: structuredClone(selected.proposal), + runtimeInputs: selected.runtimeInputs && structuredClone(selected.runtimeInputs), + }; +} diff --git a/services/data-manager/src/jobs/transitous/catalog-validation-current.ts b/services/data-manager/src/jobs/transitous/catalog-validation-current.ts new file mode 100644 index 000000000..9f696b37b --- /dev/null +++ b/services/data-manager/src/jobs/transitous/catalog-validation-current.ts @@ -0,0 +1,77 @@ +/** Invalidate before slot reuse. A restored old manifest cannot resurrect an old reviewed validation. */ +import { + assertArtifactTargetLeaseExecution, + assertArtifactTargetLeaseForward, + assertControlExecutionForward, + controlDigest, + executionAuthorityState, + signControl, + type TargetLeaseAuthority, + type VerifiedExecutionAuthority, +} from "@openmapx/core/build-artifacts"; +import { currentCatalogValidationSchema, readCatalogValidation } from "@openmapx/core/catalog-pins"; +import { canonicalJson } from "@openmapx/core/ops"; +import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { CURRENT_CATALOG_VALIDATION_DOMAIN } from "../../../../../packages/core/src/catalog-pins/record-files"; +export function publishCatalogValidationCurrent( + authority: VerifiedExecutionAuthority, + lease: TargetLeaseAuthority, + proposalId?: string, +): void { + const owner = executionAuthorityState(authority); + assertControlExecutionForward(authority); + assertArtifactTargetLeaseForward(lease); + assertArtifactTargetLeaseExecution(lease, owner.execution); + if ( + owner.installation.installation.role !== "data-manager" || + lease.mode !== "owner" || + !["transit-sync", "catalog.validate"].includes(owner.kind) || + !["motis", "motis-staging", "motis-feed-proxy"].every((id) => lease.serviceIds.includes(id)) + ) + throw Error("Original complete transit/validation owner required"); + let selected = null; + if (proposalId !== undefined) { + if (owner.kind !== "catalog.validate") + throw Error("Only the original validation may select its evidence"); + const validation = readCatalogValidation( + owner.installation.installation, + proposalId, + owner.execution.executionId, + ); + if ( + validation.validation.status !== "passed" || + controlDigest(validation.sourceExecution) !== controlDigest(owner.execution) || + controlDigest({ + kind: "catalog.validate", + proposalId, + countries: validation.validation.countries, + }) !== owner.execution.operationFingerprint + ) + throw Error("Original passing validation evidence required"); + selected = { + proposalId, + validationId: owner.execution.executionId, + validationDigest: controlDigest(validation), + }; + } + const body = currentCatalogValidationSchema.parse({ + version: 1, + installationId: owner.installation.installation.installationId, + sourceExecution: owner.execution, + selected, + }); + const data = new AnchoredDirectory(owner.installation.dataRoot); + let motis: AnchoredDirectory | undefined; + try { + motis = data.directory("motis", true); + motis.write( + "catalog-validation-current.json", + canonicalJson( + signControl(owner.installation.privateKey, CURRENT_CATALOG_VALIDATION_DOMAIN, body), + ), + ); + } finally { + motis?.close(); + data.close(); + } +} diff --git a/services/data-manager/src/jobs/transitous/catalog-validation.ts b/services/data-manager/src/jobs/transitous/catalog-validation.ts new file mode 100644 index 000000000..41c30bd98 --- /dev/null +++ b/services/data-manager/src/jobs/transitous/catalog-validation.ts @@ -0,0 +1,246 @@ +/** The admitted validation root owns the inactive slot through proxy restoration and final stop. */ +import { createHash } from "node:crypto"; +import { realpathSync } from "node:fs"; +import { + controlDigest, + executionAuthorityState, + signControl, +} from "@openmapx/core/build-artifacts"; +import { + CATALOG_VALIDATION_DOMAIN, + type CatalogRuntimeInputs, + type CatalogValidation, + catalogRuntimeInputsSchema, + catalogValidationRecordSchema, + sameCatalogRuntimeInputs, + scopeCatalogRuntimeInputs, + withCatalogProposalDirectory, +} from "@openmapx/core/catalog-pins"; +import { canonicalJson } from "@openmapx/core/ops"; +import { catalogCompilerSettings, catalogReviewInputDigests } from "@openmapx/transitous-core"; +import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { asJobLogger, jobChildLogger } from "../../logger.js"; +import { runOpsOperation } from "../../ops-client.js"; +import type { StateStore } from "../../state.js"; +import { dataExecutionDiagnostic } from "../execution/diagnostics.js"; +import { DataOpsObservationPending, runOwnedDataOpsOperation } from "../execution/ops-children.js"; +import type { DataExecutionContext } from "../execution/runtime.js"; +import { DataPublicationUnknown, publishDataChange } from "../execution/worker-scope.js"; +import { readFeedOverlay } from "../transitous-feeds-overlay.js"; +import { CANDIDATE_MANIFEST_FILENAME, verifyCandidateManifestAsync } from "./candidate.js"; +import { retainPreviousHealthyCatalogSlot } from "./catalog-rollback-retention.js"; +import { loadOwnedCatalogSelection, ownedCatalogSelectionEvidence } from "./catalog-selection.js"; +import { publishCatalogValidationCurrent } from "./catalog-validation-current.js"; +import { resolveTransitOverlayPath } from "./internal.js"; +import { isolateOwnedTransitProducer } from "./isolate-producer.js"; +import type { MotisOperationsPolicy } from "./operations-profile.js"; +import { makePersistingOnStageComplete } from "./persistence.js"; +import { buildJobContext, runTransitousPipeline, stagePolicyFor } from "./pipeline.js"; +import { rollbackProxyTransaction } from "./proxy-transaction.js"; +import { readMotisSlotState } from "./slot-state.js"; + +export function unchangedCatalogRuntimeInputs( + before: CatalogRuntimeInputs, + after: CatalogRuntimeInputs, +): boolean { + return sameCatalogRuntimeInputs(before, after); +} +export async function runCatalogValidation(options: { + context: DataExecutionContext; + dataDir: string; + store: StateStore; + operationsPolicy: MotisOperationsPolicy; +}): Promise { + const { context, dataDir, operationsPolicy } = options, + record = await context.record(); + if (record.input.kind !== "catalog.validate") + throw Error("Original catalog validation input required"); + if (process.env.MOTIS_TWO_SLOT !== "true" || process.env.MOTIS_GBFS_CATALOG_ENABLED !== "true") + throw Error("Enable and apply two-slot MOTIS and the paired GBFS compiler before validation"); + const input = record.input, + selection = await loadOwnedCatalogSelection(context, input.proposalId), + evidence = ownedCatalogSelectionEvidence(selection, context); + if (!evidence.runtimeInputs) + throw Error("Verified catalog runtime inputs required before validation"); + const runtimeInputs = scopeCatalogRuntimeInputs(evidence.runtimeInputs, input.countries); + if (runtimeInputs.requiredKeys.some((key) => !runtimeInputs.configuredKeys.includes(key))) + throw Error( + "Configure the credentials required by the selected validation countries and enabled sources", + ); + if (Buffer.byteLength(canonicalJson(runtimeInputs), "utf8") > 48 * 1024) + throw Error("Catalog input evidence exceeds its retained budget"); + const source = catalogReviewInputDigests( + operationsPolicy, + readFeedOverlay(resolveTransitOverlayPath(dataDir)), + catalogCompilerSettings(process.env), + ); + if ( + source.sourcePolicyRevision !== evidence.proposal.sourcePolicyRevision || + source.sourcePolicyRevision !== runtimeInputs.sourcePolicyRevision || + source.overlayDigest !== runtimeInputs.overlayDigest + ) + throw Error("Catalog policy changed; create a fresh proposal"); + const initialSlots = readMotisSlotState(dataDir); + if (!initialSlots) throw Error("Initialize the two-slot MOTIS layout before validation"); + const slot = initialSlots.activeSlot === "A" ? "B" : "A"; + const child = async (ordinal: number) => { + const reserved = record.reservations.find((entry) => entry.ordinal === ordinal); + if (!reserved || reserved.operation.kind !== "motis.staging.stop") + throw Error("Original validation stop reservation required"); + await runOwnedDataOpsOperation(reserved.operation, { + context, + phase: reserved.phase, + ordinal, + serviceIds: reserved.serviceIds, + }); + }; + await child(0); + await publishDataChange(() => + publishCatalogValidationCurrent(context.authority, context.targetAuthority), + ); + await retainPreviousHealthyCatalogSlot( + context, + dataDir, + input.proposalId, + operationsPolicy.resourceEnvelope.minimumDiskGb * 1024 ** 3, + ); + await isolateOwnedTransitProducer( + context, + operationsPolicy.resourceEnvelope.minimumDiskGb * 1024 ** 3, + ); + const logger = asJobLogger( + jobChildLogger({ job: "catalog.validate", jobId: context.execution.executionId }), + ); + const stages = stagePolicyFor(operationsPolicy.acquisition); + const ctx = await publishDataChange(() => + buildJobContext({ + dataDir, + store: options.store, + countries: input.countries, + source: operationsPolicy.acquisition, + operationsPolicy: { ...operationsPolicy, countries: input.countries }, + jobId: context.execution.executionId, + reviewedCatalog: selection, + executionContext: context, + abortSignal: context.signal, + logger, + onStageComplete: async (stage) => { + await makePersistingOnStageComplete(context.execution.executionId, logger)(stage); + const diagnostic = dataExecutionDiagnostic(stage.message ?? stage.stage); + await context.checkpoint({ + progress: Math.min( + 0.95, + (stages.findIndex((entry) => entry.name === stage.stage) + 1) / stages.length, + ), + diagnostics: { + tail: [{ stream: "stdout", message: diagnostic.message }], + truncated: diagnostic.truncated, + }, + }); + }, + }), + ); + let failure = false; + try { + await runTransitousPipeline(ctx, { stopAt: "motis-health", requireCompleteStages: true }); + } catch (error) { + if (error instanceof DataOpsObservationPending || error instanceof DataPublicationUnknown) + throw error; + context.assertForward(); + failure = true; + } + // Restoration is part of validation, not best-effort cleanup. An uncertain + // child throws and keeps this root fenced; it cannot publish passing evidence. + await rollbackProxyTransaction(ctx); + await child(4); + context.assertForward(); + const fresh = await runOpsOperation( + { kind: "transitousLock.inspect", proposalId: input.proposalId, offset: 0 }, + { signal: context.signal }, + ); + if ( + controlDigest(fresh.proposal) !== controlDigest(evidence.proposal) || + !fresh.validationInputs || + !unchangedCatalogRuntimeInputs( + runtimeInputs, + scopeCatalogRuntimeInputs( + catalogRuntimeInputsSchema.parse(fresh.validationInputs), + input.countries, + ), + ) + ) + throw Error("Catalog inputs changed during validation"); + if (readMotisSlotState(dataDir)?.activeSlot !== initialSlots.activeSlot) + throw Error("Validation changed the active MOTIS slot"); + const completedAt = new Date().toISOString(), + common = { + validationId: context.execution.executionId, + proposalId: input.proposalId, + pinDigest: evidence.proposal.pins.generationId, + diffDigest: evidence.proposal.diffDigest, + sourcePolicyRevision: runtimeInputs.sourcePolicyRevision, + overlayDigest: runtimeInputs.overlayDigest, + credentialsRevision: runtimeInputs.credentialsRevision, + countries: input.countries, + completedAt, + expiresAt: new Date(Date.parse(completedAt) + 24 * 60 * 60 * 1000).toISOString(), + }; + let validation: CatalogValidation; + if (failure) + validation = { + ...common, + status: "failed", + reason: + "A required import or functional probe did not complete. Inspect the validation job and create a new validation after resolving its failure.", + }; + else { + const manifest = await verifyCandidateManifestAsync(ctx.motisStagingDataDir, context.signal); + if ( + manifest.epoch !== context.execution.executionId || + ctx.slotLayout?.record.activeSlot !== initialSlots.activeSlot + ) + throw Error("Validation candidate identity changed"); + const directory = new AnchoredDirectory(realpathSync(ctx.motisStagingDataDir)); + let manifestHash: string; + try { + manifestHash = createHash("sha256") + .update(directory.read(CANDIDATE_MANIFEST_FILENAME, 32 * 1024 * 1024, false)) + .digest("hex"); + } finally { + directory.close(); + } + validation = { + ...common, + status: "passed", + slot, + epoch: manifest.epoch, + manifestHash, + imageDigest: runtimeInputs.imageDigest, + }; + } + const owner = executionAuthorityState(context.authority), + retained = catalogValidationRecordSchema.parse({ + version: 1, + installationId: context.installation.installationId, + sourceExecution: context.execution, + proposalDigest: controlDigest(evidence.proposal), + runtimeInputs, + validation, + }); + await publishDataChange(() => + withCatalogProposalDirectory(context.installation, input.proposalId, (directory) => + directory.write( + `validation-${context.execution.executionId}.json`, + canonicalJson( + signControl(owner.installation.privateKey, CATALOG_VALIDATION_DOMAIN, retained), + ), + true, + ), + ), + ); + if (validation.status === "passed") + await publishDataChange(() => + publishCatalogValidationCurrent(context.authority, context.targetAuthority, input.proposalId), + ); + return validation; +} diff --git a/services/data-manager/src/jobs/transitous/isolate-producer.ts b/services/data-manager/src/jobs/transitous/isolate-producer.ts new file mode 100644 index 000000000..8c154327f --- /dev/null +++ b/services/data-manager/src/jobs/transitous/isolate-producer.ts @@ -0,0 +1,61 @@ +/** The original transit owner detaches mutable build inputs from active/rollback generations. */ +import { + assertArtifactTargetLeaseExecution, + assertArtifactTargetLeaseForward, + controlDigest, + executionAuthorityState, +} from "@openmapx/core/build-artifacts"; +import { isolateArtifactHardlinks } from "../../../../../packages/core/src/build-artifacts/copy-tree"; +import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import type { DataExecutionContext } from "../execution/runtime"; +import { dataExecutionWorkerContext, publishDataChange } from "../execution/worker-scope"; + +export async function isolateOwnedTransitProducer( + context: DataExecutionContext, + minimumFreeBytes: number, +): Promise { + if (dataExecutionWorkerContext() !== context) throw Error("Original transit worker required"); + context.assertForward(); + const owner = executionAuthorityState(context.authority), + record = await context.record(); + assertArtifactTargetLeaseForward(context.targetAuthority); + assertArtifactTargetLeaseExecution(context.targetAuthority, context.execution); + if ( + owner.installation.installation.role !== "data-manager" || + !["transit-sync", "catalog.validate"].includes(owner.kind) || + controlDigest(record.execution) !== controlDigest(owner.execution) || + controlDigest(record.input) !== owner.execution.operationFingerprint || + context.targetAuthority.mode !== "owner" || + !["motis", "motis-staging", "motis-feed-proxy"].every((id) => + context.targetAuthority.serviceIds.includes(id), + ) + ) + throw Error("Original complete transit target owner required"); + const data = new AnchoredDirectory(owner.installation.dataRoot); + let producer: AnchoredDirectory | undefined; + try { + try { + producer = data.directory("gtfs"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + const source = producer; + await publishDataChange(() => + isolateArtifactHardlinks({ + source, + minimumFreeBytes, + signal: context.signal, + maximumEntries: 100000, + maximumBytes: Number.MAX_SAFE_INTEGER, + maximumManifestBytes: 64 * 1024 * 1024, + }), + ); + // A canceled copy has settled and only identical producer bytes were published. + // Record that containment before the original cancellation stops the pipeline. + context.assertForward(); + } finally { + producer?.close(); + data.close(); + } +} diff --git a/services/data-manager/src/jobs/transitous/motis-health.ts b/services/data-manager/src/jobs/transitous/motis-health.ts index 869948e00..6fb1fae25 100644 --- a/services/data-manager/src/jobs/transitous/motis-health.ts +++ b/services/data-manager/src/jobs/transitous/motis-health.ts @@ -1,5 +1,5 @@ import { existsSync, readdirSync } from "node:fs"; -import { verifyCandidateManifest } from "./candidate.js"; +import { verifyCandidateManifestAsync } from "./candidate.js"; import { runFunctionalProbes, writeCapabilitySnapshot } from "./functional-probes.js"; import { healthUrl } from "./motis-endpoints.js"; import { parseIntEnv, pollUntilHealthy } from "./motis-probe.js"; @@ -36,7 +36,7 @@ export const run: StageFn = async (ctx) => { } try { - const manifest = verifyCandidateManifest(ctx.motisStagingDataDir); + const manifest = await verifyCandidateManifestAsync(ctx.motisStagingDataDir, ctx.abortSignal); const baseUrl = process.env.MOTIS_STAGING_URL ?? DEFAULT_STAGING_URL; const importDeadline = Date.now() + parseIntEnv("MOTIS_IMPORT_TIMEOUT_MS", DEFAULT_IMPORT_TIMEOUT_MS); @@ -51,7 +51,7 @@ export const run: StageFn = async (ctx) => { } // Re-verify after the long import wait: no candidate input may change once probing begins. - verifyCandidateManifest(ctx.motisStagingDataDir); + await verifyCandidateManifestAsync(ctx.motisStagingDataDir, ctx.abortSignal); const report = await runFunctionalProbes(baseUrl, manifest, Date.now() + FUNCTIONAL_BUDGET_MS); if (!report.ok) { return finish( diff --git a/services/data-manager/src/jobs/transitous/motis-import.ts b/services/data-manager/src/jobs/transitous/motis-import.ts index f5731078b..1f37669cd 100644 --- a/services/data-manager/src/jobs/transitous/motis-import.ts +++ b/services/data-manager/src/jobs/transitous/motis-import.ts @@ -1,6 +1,6 @@ import { existsSync, writeFileSync } from "node:fs"; import { join } from "node:path"; -import { verifyCandidateManifest } from "./candidate.js"; +import { verifyCandidateManifestAsync } from "./candidate.js"; import { IMPORT_MARKER_FILE } from "./internal.js"; import { STAGING_CONTAINER as STAGING_CONTAINER_NAME } from "./motis-containers.js"; import { assertTransitForward, preserveTransitPending, runTransitOwnedPhase } from "./owned-ops.js"; @@ -58,21 +58,13 @@ export const run: StageFn = async (ctx) => { message: `staging config not generated at ${expectedConfig}`, } satisfies StageResult; } - const manifest = verifyCandidateManifest(ctx.motisStagingDataDir); + const manifest = await verifyCandidateManifestAsync(ctx.motisStagingDataDir, ctx.abortSignal); - // Re-run the staging container's import+serve entrypoint against the freshly - // assembled config. `docker restart` covers every state the container can be - // in: running (re-import the new data), stopped (start it), or blocked in its - // wait-for-config loop (the restart picks up the now-present config.yml). The - // import itself is owned by the container entrypoint; completion is verified - // downstream by motis-health, so we return as soon as the restart is issued. - // - // Creating the container is NOT our job: motis-staging is part of the - // rendered stack (plain bind-mount, pinned container name), brought up by the - // deploy step. A missing container is an operator/selection error we surface - // rather than paper over with `docker compose up` — the data-manager image - // ships only a static docker CLI (no compose plugin), so that path can't run - // here anyway. + // Recreate the staging container against the newly selected inactive slot. + // Docker resolves a bind source once at container creation: a restart would + // retain an old slot after a previous promotion changed the staging alias. + // The agent uses the existing rendered service and --no-deps so this cannot + // start the control plane or dependency containers. try { // Restarting a container is host authority; the agent owns the name. await runTransitOwnedPhase(ctx, "restart-staging"); diff --git a/services/data-manager/src/jobs/transitous/operations-profile.ts b/services/data-manager/src/jobs/transitous/operations-profile.ts index aaa12b5e7..49ce80e67 100644 --- a/services/data-manager/src/jobs/transitous/operations-profile.ts +++ b/services/data-manager/src/jobs/transitous/operations-profile.ts @@ -1,185 +1,11 @@ -import type { TransitSource } from "./types.js"; - -export const MOTIS_OPERATIONS_PROFILES = [ - "regional-assisted", - "regional-sovereign", - "planet", -] as const; - -export type MotisOperationsProfileName = (typeof MOTIS_OPERATIONS_PROFILES)[number]; - -export interface MotisOperationsPolicy { - profile: MotisOperationsProfileName; - countries: string[]; - feedAllowList: string[]; - acquisition: TransitSource; - artifactOrigins: string[]; - originDownloadsRequired: boolean; - hostedRuntimeFallbackAllowed: boolean; - osmInput?: string; - gbfsSelection: "explicit-countries" | "explicit-feeds" | "allow-listed-batches"; - maxFeedCount: number; - retentionGenerations: number; - resourceEnvelope: { - minimumMemoryGb: number; - minimumCpu: number; - minimumDiskGb: number; - operatorCapacityRequired: boolean; - }; - updateCadenceHours: number; - experimental: boolean; -} - -export interface ResolveOperationsProfileInput { - profile?: string; - countries?: string[]; - feedAllowList?: string[]; - source?: TransitSource; - artifactBaseUrl?: string; - osmInput?: string; - confirmPlanet?: boolean; - maxFeedCount?: number; - retentionGenerations?: number; - allowEmptyRegional?: boolean; -} - -function normalized(values: string[] | undefined): string[] { - return [ - ...new Set((values ?? []).map((value) => value.trim().toLowerCase()).filter(Boolean)), - ].sort(); -} - -function positiveInteger(value: number | undefined, fallback: number, label: string): number { - const resolved = value ?? fallback; - if (!Number.isInteger(resolved) || resolved < 1) - throw new Error(`${label} must be a positive integer`); - return resolved; -} - -/** Match DNS labels, including a fully qualified hostname's optional root dot. */ -export function isTransitousHostname(hostname: string): boolean { - const normalizedHostname = hostname.toLowerCase().replace(/\.$/, ""); - return normalizedHostname === "transitous.org" || normalizedHostname.endsWith(".transitous.org"); -} - -function artifactUrl(url: string | undefined): URL | undefined { - if (!url) return undefined; - try { - return new URL(url); - } catch { - throw new Error(`Invalid Transitous artifact URL: ${url}`); - } -} - -export function resolveOperationsProfile( - input: ResolveOperationsProfileInput, -): MotisOperationsPolicy { - const profile = (input.profile ?? "regional-assisted") as MotisOperationsProfileName; - if (!MOTIS_OPERATIONS_PROFILES.includes(profile)) { - throw new Error(`Unsupported MOTIS operations profile: ${input.profile}`); - } - const countries = normalized(input.countries); - const feedAllowList = normalized(input.feedAllowList); - const source = input.source ?? (profile === "regional-assisted" ? "mirror" : "build"); - const artifact = artifactUrl(input.artifactBaseUrl); - const origin = artifact?.origin; - - if ( - profile !== "planet" && - countries.length === 0 && - feedAllowList.length === 0 && - !input.allowEmptyRegional - ) { - throw new Error( - `${profile} requires explicit countries or a feed allow-list; [] never means planet`, - ); - } - if (profile === "planet" && input.confirmPlanet !== true) { - throw new Error("planet profile requires MOTIS_PLANET_CONFIRM=true"); - } - if (profile === "regional-sovereign") { - if (source !== "build") throw new Error("regional-sovereign requires origin build acquisition"); - if (artifact && isTransitousHostname(artifact.hostname)) { - throw new Error("regional-sovereign prohibits Transitous artifact domains"); - } - } - - const assisted = profile === "regional-assisted"; - const planet = profile === "planet"; - return { - profile, - countries, - feedAllowList, - acquisition: source, - artifactOrigins: origin ? [origin] : assisted ? ["https://api.transitous.org"] : [], - originDownloadsRequired: !assisted, - hostedRuntimeFallbackAllowed: assisted, - osmInput: input.osmInput?.trim() || undefined, - gbfsSelection: planet - ? "allow-listed-batches" - : feedAllowList.length > 0 - ? "explicit-feeds" - : "explicit-countries", - maxFeedCount: positiveInteger(input.maxFeedCount, planet ? 10_000 : 500, "maxFeedCount"), - retentionGenerations: positiveInteger( - input.retentionGenerations, - planet ? 3 : 2, - "retentionGenerations", - ), - resourceEnvelope: { - minimumMemoryGb: planet ? 64 : 16, - minimumCpu: planet ? 8 : 4, - minimumDiskGb: planet ? 500 : 40, - operatorCapacityRequired: planet, - }, - updateCadenceHours: planet ? 48 : 24, - experimental: planet, - }; -} - -function csv(value: string | undefined): string[] { - return (value ?? "") - .split(",") - .map((item) => item.trim()) - .filter(Boolean); -} - -export function resolveOperationsProfileFromEnv( - env: NodeJS.ProcessEnv = process.env, - overrides: Partial = {}, -): MotisOperationsPolicy { - const number = (name: string): number | undefined => { - const value = Number(env[name]); - return Number.isFinite(value) ? value : undefined; - }; - return resolveOperationsProfile({ - profile: env.MOTIS_OPERATIONS_PROFILE, - countries: csv(env.TRANSITOUS_COUNTRIES), - feedAllowList: csv(env.MOTIS_FEED_ALLOW_LIST), - source: (env.TRANSIT_SOURCE as TransitSource | undefined) ?? undefined, - artifactBaseUrl: env.TRANSITOUS_ARTIFACT_BASE_URL, - osmInput: env.MOTIS_OSM_REGION ?? env.MOTIS_OSM_FILE, - confirmPlanet: env.MOTIS_PLANET_CONFIRM === "true", - maxFeedCount: number("MOTIS_MAX_FEED_COUNT"), - retentionGenerations: number("MOTIS_RETENTION_GENERATIONS"), - ...overrides, - }); -} - -/** Effective country scope for a finite legacy feed allow-list. This does - * not select feeds: the preflight independently checks each exact feed ID. */ -export function transitAllowedCountries(policy: MotisOperationsPolicy): string[] { - if (policy.countries.length) - return [...new Set(policy.countries.map((c) => c.toLowerCase()))].sort(); - return [ - ...new Set( - policy.feedAllowList.flatMap((id) => { - const country = /^([a-z]{2,3})_/.exec(id.toLowerCase())?.[1]; - return country ? [country] : []; - }), - ), - ].sort(); -} -export function publicOperationsPolicy(policy: MotisOperationsPolicy): MotisOperationsPolicy { - return { ...structuredClone(policy), countries: transitAllowedCountries(policy) }; -} +export { + isTransitousHostname, + MOTIS_OPERATIONS_PROFILES, + type MotisOperationsPolicy, + type MotisOperationsProfileName, + publicOperationsPolicy, + type ResolveOperationsProfileInput, + resolveOperationsProfile, + resolveOperationsProfileFromEnv, + transitAllowedCountries, +} from "@openmapx/transitous-core"; diff --git a/services/data-manager/src/jobs/transitous/pipeline.ts b/services/data-manager/src/jobs/transitous/pipeline.ts index 96f6fb399..4fd1c053d 100644 --- a/services/data-manager/src/jobs/transitous/pipeline.ts +++ b/services/data-manager/src/jobs/transitous/pipeline.ts @@ -100,6 +100,8 @@ export function stagePolicyFor(source: TransitSource): ReadonlyArray<{ } export interface RunPipelineOptions { + /** Reviewed validation requires every selected critical stage to finish completely. */ + requireCompleteStages?: boolean; startAt?: StageName; stopAt?: StageName; } @@ -149,9 +151,14 @@ export async function runTransitousPipeline( const result = await entry.run(ctx); results.push(result); await ctx.onStageComplete(result); - if (result.status === "error" && entry.criticality === "critical") { + if ( + entry.criticality === "critical" && + (result.status === "error" || (options.requireCompleteStages && result.status !== "ok")) + ) { hardStopError = new Error( - result.error?.message ?? result.message ?? `${entry.name} failed`, + options.requireCompleteStages && result.status !== "error" + ? `${entry.name} ${result.status}; reviewed validation requires a complete stage` + : (result.error?.message ?? result.message ?? `${entry.name} failed`), ); if (result.error?.stack) hardStopError.stack = result.error.stack; break; @@ -209,6 +216,7 @@ function aggregateFinalStatus(results: StageResult[]): StageStatus { } export interface BuildJobContextOptions { + reviewedCatalog?: import("./catalog-selection.js").ReviewedCatalogSelection; executionContext?: import("../execution/runtime.js").DataExecutionContext; dataDir: string; store: StateStore; @@ -266,6 +274,7 @@ export function buildJobContext(opts: BuildJobContextOptions): JobContext { process.env.MOTIS_TWO_SLOT === "true" ? ensureMotisSlotLayout(opts.dataDir) : undefined; return { executionContext: opts.executionContext, + reviewedCatalog: opts.reviewedCatalog, jobId: opts.jobId ?? randomUUID(), dataDir: opts.dataDir, catalogDir, diff --git a/services/data-manager/src/jobs/transitous/prepare.ts b/services/data-manager/src/jobs/transitous/prepare.ts index f07a30b58..adbf8f845 100644 --- a/services/data-manager/src/jobs/transitous/prepare.ts +++ b/services/data-manager/src/jobs/transitous/prepare.ts @@ -3,6 +3,7 @@ import { parseCatalogPinSet } from "@openmapx/core/catalog-pins"; import { ensureCatalog } from "@openmapx/transitous-core"; import { runOpsOperation } from "../../ops-client.js"; import { parseRefShaPair, type TransitousLock } from "../../transitous-lock.js"; +import { selectedCatalogPins } from "./catalog-selection.js"; import { DEFAULT_TRANSITOUS_REPO_URL, ensureTransitousWorkdirs, @@ -22,15 +23,19 @@ export const run: StageFn = async (ctx) => { const startedAt = ctx.now(); const start = Date.now(); try { - if (ctx.useProposedLock) + if (ctx.useProposedLock && !ctx.reviewedCatalog) throw Error("Paired proposal validation requires the reviewed activation workflow"); - const pins = await runOpsOperation({ kind: "catalogPins.inspect" }); - if (!pins.active || pins.catalogRevisionId !== pins.active.generationId) - throw Error("Paired catalog pins are unavailable"); - try { - ctx.state.catalogPins = parseCatalogPinSet(pins.active); - } catch { - throw Error("Paired catalog pin response is invalid"); + if (ctx.reviewedCatalog) { + ctx.state.catalogPins = selectedCatalogPins(ctx); + } else { + const pins = await runOpsOperation({ kind: "catalogPins.inspect" }); + if (!pins.active || pins.catalogRevisionId !== pins.active.generationId) + throw Error("Paired catalog pins are unavailable"); + try { + ctx.state.catalogPins = parseCatalogPinSet(pins.active); + } catch { + throw Error("Paired catalog pin response is invalid"); + } } mkdirSync(ctx.dataDir, { recursive: true }); diff --git a/services/data-manager/src/jobs/transitous/promote.ts b/services/data-manager/src/jobs/transitous/promote.ts index 3dc84c082..d34a05af2 100644 --- a/services/data-manager/src/jobs/transitous/promote.ts +++ b/services/data-manager/src/jobs/transitous/promote.ts @@ -15,7 +15,7 @@ import { CANDIDATE_MANIFEST_FILENAME, type MotisCandidateManifest, readCandidateManifest, - verifyCandidateManifest, + verifyCandidateManifestAsync, } from "./candidate.js"; import { recordPromotedSource } from "./feed-state-writer.js"; import { runFunctionalProbes, verifyCapabilitySnapshot } from "./functional-probes.js"; @@ -144,10 +144,8 @@ function assertCandidateEpochIsNew( const liveManifestPath = join(currentDir, CANDIDATE_MANIFEST_FILENAME); if (!existsSync(liveManifestPath)) return; - // Read (not hash-verify) the live manifest: live artifacts are hardlinks - // into out/, which the next run rewrites in place, so re-hashing the live - // dir would fail exactly when a source set legitimately changed. Only the - // epoch matters here. + // This guard compares generation identities. Candidate integrity is checked + // separately; producer inputs are detached before a build can rewrite them. const liveManifest = readCandidateManifest(currentDir); if (liveManifest.epoch === candidateEpoch) { throw new Error(`Candidate epoch ${candidateEpoch} duplicates the active dataset epoch`); @@ -456,7 +454,7 @@ export const run: StageFn = async (ctx) => { } satisfies StageResult; } - const manifest = verifyCandidateManifest(stagingDir); + const manifest = await verifyCandidateManifestAsync(stagingDir, ctx.abortSignal); await retainTransitPhaseCandidate(ctx, "promote-primary"); assertCandidateEpochIsNew(ctx, currentDir, manifest.epoch); const sourceManifest = readTransitSourceManifest( diff --git a/services/data-manager/src/jobs/transitous/proxy-transaction.ts b/services/data-manager/src/jobs/transitous/proxy-transaction.ts index 5ecaeb0d1..ea8e1db13 100644 --- a/services/data-manager/src/jobs/transitous/proxy-transaction.ts +++ b/services/data-manager/src/jobs/transitous/proxy-transaction.ts @@ -13,7 +13,7 @@ import type { FreshDataChildReservation } from "../execution/store.js"; import { CANDIDATE_MANIFEST_FILENAME, CANDIDATE_PROXY_DIRNAME, - verifyCandidateManifest, + verifyCandidateManifestAsync, } from "./candidate.js"; import { assertTransitForward, @@ -185,7 +185,7 @@ export const run: StageFn = async (ctx) => { if (!existsSync(join(ctx.motisStagingDataDir, CANDIDATE_MANIFEST_FILENAME))) { return finish("skipped", "no assembled MOTIS candidate; feed-proxy transaction not needed"); } - const manifest = verifyCandidateManifest(ctx.motisStagingDataDir); + const manifest = await verifyCandidateManifestAsync(ctx.motisStagingDataDir, ctx.abortSignal); const candidateRoot = join(ctx.motisStagingDataDir, CANDIDATE_PROXY_DIRNAME); const candidateConfigPath = join(candidateRoot, CONFIG_RELATIVE_PATH); const candidateVarsPath = join(candidateRoot, VARS_FILENAME); diff --git a/services/data-manager/src/jobs/transitous/slot-state.ts b/services/data-manager/src/jobs/transitous/slot-state.ts index f2258e018..0c7fd2c18 100644 --- a/services/data-manager/src/jobs/transitous/slot-state.ts +++ b/services/data-manager/src/jobs/transitous/slot-state.ts @@ -11,14 +11,32 @@ import { symlinkSync, } from "node:fs"; import { dirname, join, relative, resolve } from "node:path"; +import z from "zod/v4"; import { atomicWriteJsonSync } from "../../utils/atomic-write.js"; +const rollbackRecordSchema = z.strictObject({ + proposalId: z.uuid(), + validationId: z.uuid(), + slot: z.enum(["A", "B"]), + epoch: z.string().min(1).max(512), + manifestHash: z.string().regex(/^[a-f0-9]{64}$/), + filesDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); + export type MotisSlot = "A" | "B"; export interface MotisSlotRecord { schemaVersion: 1; activeSlot: MotisSlot; previousHealthySlot?: MotisSlot; + retainedRollback?: { + proposalId: string; + validationId: string; + slot: MotisSlot; + epoch: string; + manifestHash: string; + filesDigest: string; + }; datasetEpoch?: string; manifestHash?: string; imageDigest?: string; @@ -38,10 +56,14 @@ const other = (slot: MotisSlot): MotisSlot => (slot === "A" ? "B" : "A"); function readRecord(path: string): MotisSlotRecord | null { if (!existsSync(path)) return null; + if (statSync(path).size > 64 * 1024) throw Error("MOTIS slot state exceeds size limit"); const parsed = JSON.parse(readFileSync(path, "utf-8")) as MotisSlotRecord; if (parsed.schemaVersion !== 1 || !["A", "B"].includes(parsed.activeSlot)) { throw new Error(`malformed MOTIS slot state at ${path}`); } + if (parsed.previousHealthySlot !== undefined && !["A", "B"].includes(parsed.previousHealthySlot)) + throw Error("Malformed previous healthy slot"); + if (parsed.retainedRollback !== undefined) rollbackRecordSchema.parse(parsed.retainedRollback); return parsed; } @@ -131,6 +153,7 @@ export function commitMotisSlotActivation( schemaVersion: 1, activeSlot: next.activeSlot, previousHealthySlot: layout.record.activeSlot, + ...(layout.record.retainedRollback ? { retainedRollback: layout.record.retainedRollback } : {}), datasetEpoch: next.datasetEpoch, manifestHash: next.manifestHash, imageDigest: next.imageDigest, diff --git a/services/data-manager/src/jobs/transitous/types.ts b/services/data-manager/src/jobs/transitous/types.ts index c939f98df..3e75ba626 100644 --- a/services/data-manager/src/jobs/transitous/types.ts +++ b/services/data-manager/src/jobs/transitous/types.ts @@ -141,6 +141,7 @@ export interface ProxyTransactionState { export interface JobContext { executionContext?: import("../execution/runtime.js").DataExecutionContext; + reviewedCatalog?: import("./catalog-selection.js").ReviewedCatalogSelection; /** ULID/UUID */ jobId: string; /** Absolute path, typically `infra/docker/data`. */ diff --git a/services/data-manager/src/service.ts b/services/data-manager/src/service.ts index 5364f75a5..5f5bf0559 100644 --- a/services/data-manager/src/service.ts +++ b/services/data-manager/src/service.ts @@ -30,6 +30,7 @@ import { reconcileOrphanedJobs } from "./jobs/reconcile.js"; import { createNotablePlacesOperationLock } from "./jobs/search-index/operation-lock.js"; import { bakePredicted } from "./jobs/traffic/bake-predicted.js"; import { fetchCoveredWayIds } from "./jobs/traffic/covered-ways.js"; +import { createCatalogReviewService } from "./jobs/transitous/catalog-review.js"; import { resolveOperationsProfileFromEnv, transitAllowedCountries, @@ -96,6 +97,13 @@ const executionStore = createPostgresDataExecutionStore(rawSql, { controlJournal: executionJournal, configurationFence, }); +const catalogReview = controlStartup.installation + ? createCatalogReviewService({ + dataDir, + installation: controlStartup.installation, + operationsPolicy, + }) + : undefined; const dispatcher = createDataExecutionDispatcher({ dataDir, state: dataState, @@ -119,6 +127,7 @@ if (controlStartup.installation && configurationFence) }); registerApi(app, { + catalogReview, executionReceiver, searchIndexRuntimeState: dispatcher.searchIndexRuntimeState, dataDir, @@ -209,6 +218,7 @@ async function start(): Promise { reconcileJobs: reconcileOrphanedJobs, setupCronSchedulers: () => setupCron({ + catalogReview, dataDir, executionReceiver, countries, From 5cc9e174bc7fa197dc337dfd5dc002d04399205d Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 08:29:53 +0100 Subject: [PATCH 054/135] feat(admin): expose guarded overture recovery --- apps/api/openapi.json | 627 ++++++++++++++++++ .../data-manager-search-index.test.ts | 47 ++ apps/api/src/routes/data-manager.ts | 15 + .../__tests__/admin-job-handlers.test.ts | 39 +- .../__tests__/admin-operation-catalog.test.ts | 44 +- .../src/services/admin-operation-catalog.ts | 80 ++- .../src/administrative-runtime.test.ts | 6 + apps/ops-agent/src/administrative-runtime.ts | 14 +- .../services/OvertureMaintenance.test.tsx | 5 + .../admin/services/OvertureMaintenance.tsx | 202 +++++- .../OvertureMaintenance.workflow.test.tsx | 123 ++++ .../admin/services/ServiceWorkflows.test.tsx | 48 +- docs/docs/developer/cli-reference.md | 2 +- docs/docs/features/overture-places.md | 11 + docs/docs/install/preparing-data.md | 4 +- .../__tests__/cli-overture-recovery.test.ts | 42 ++ packages/cli/src/commands/data.ts | 26 +- .../api/contracts/data-manager-admin/index.ts | 5 +- .../data-manager-admin/operations.ts | 19 +- .../admin/data-manager-admin.generated.ts | 108 +++ .../getApiDataManagerOvertureRecovery200.ts | 14 + ...piDataManagerOvertureRecovery200Actions.ts | 15 + ...rtureRecovery200ActionsOvertureConflate.ts | 8 + ...ertureRecovery200ActionsOvertureExtract.ts | 8 + ...vertureRecovery200ActionsOvertureIngest.ts | 8 + ...rOvertureRecovery200ActionsOverturePull.ts | 8 + ...ertureRecovery200ActionsOvertureRestart.ts | 8 + .../getApiDataManagerOvertureRecovery400.ts | 20 + .../getApiDataManagerOvertureRecovery401.ts | 20 + .../getApiDataManagerOvertureRecovery403.ts | 20 + .../getApiDataManagerOvertureRecovery413.ts | 20 + .../getApiDataManagerOvertureRecovery415.ts | 20 + .../getApiDataManagerOvertureRecovery429.ts | 11 + .../getApiDataManagerOvertureRecovery500.ts | 20 + .../getApiDataManagerOvertureRecovery503.ts | 20 + ...getApiDataManagerOvertureRecoveryParams.ts | 11 + .../src/api/generated/admin/models/index.ts | 16 + .../admin/operation-bindings.generated.ts | 9 + .../src/build-artifacts/child-phase.test.ts | 46 +- .../src/build-artifacts/control-authority.ts | 6 + .../build-artifacts/control-child-records.ts | 6 + .../build-artifacts/fixed-data-execution.ts | 9 + packages/core/src/ops/contract.test.ts | 3 + packages/core/src/ops/contract.ts | 15 + .../data-manager/contracts/operations.ts | 11 + .../data-manager/contracts/overture.ts | 20 + .../generated/data-manager.generated.ts | 98 +++ .../models/getOvertureRecovery200.ts | 14 + .../models/getOvertureRecovery200Actions.ts | 15 + ...rtureRecovery200ActionsOvertureConflate.ts | 8 + ...ertureRecovery200ActionsOvertureExtract.ts | 8 + ...vertureRecovery200ActionsOvertureIngest.ts | 8 + ...tOvertureRecovery200ActionsOverturePull.ts | 8 + ...ertureRecovery200ActionsOvertureRestart.ts | 8 + .../models/getOvertureRecovery400.ts | 20 + .../models/getOvertureRecovery401.ts | 20 + .../models/getOvertureRecovery413.ts | 20 + .../models/getOvertureRecovery415.ts | 20 + .../models/getOvertureRecovery429.ts | 20 + .../models/getOvertureRecovery500.ts | 20 + .../models/getOvertureRecovery503.ts | 20 + .../models/getOvertureRecoveryParams.ts | 11 + .../data-manager/generated/models/index.ts | 15 + .../generated/operation-bindings.generated.ts | 7 + scripts/database-test-groups.json | 1 + scripts/ops-authority-inventory.json | 6 + .../__tests__/overture/local-inputs.test.ts | 93 +++ .../overture/recovery-postgres.test.ts | 145 ++++ services/data-manager/openapi.json | 612 +++++++++++++++++ services/data-manager/src/api.ts | 11 + .../src/jobs/execution/dispatch.ts | 26 +- .../data-manager/src/jobs/overture/ingest.ts | 9 +- .../src/jobs/overture/local-inputs.ts | 95 +++ .../src/jobs/overture/operation-lock.ts | 11 + .../src/jobs/overture/recovery.ts | 60 ++ .../data-manager/src/jobs/overture/stac.ts | 2 + 76 files changed, 3171 insertions(+), 79 deletions(-) create mode 100644 apps/web/src/components/admin/services/OvertureMaintenance.workflow.test.tsx create mode 100644 packages/cli/__tests__/cli-overture-recovery.test.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200Actions.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureConflate.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureExtract.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureIngest.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOverturePull.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureRestart.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery400.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery401.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery403.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery413.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery415.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery429.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery500.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery503.ts create mode 100644 packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecoveryParams.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery200.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery200Actions.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureConflate.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureExtract.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureIngest.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOverturePull.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureRestart.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery400.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery401.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery413.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery415.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery429.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery500.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecovery503.ts create mode 100644 packages/core/src/services/data-manager/generated/models/getOvertureRecoveryParams.ts create mode 100644 services/data-manager/__tests__/overture/local-inputs.test.ts create mode 100644 services/data-manager/__tests__/overture/recovery-postgres.test.ts create mode 100644 services/data-manager/src/jobs/overture/local-inputs.ts create mode 100644 services/data-manager/src/jobs/overture/recovery.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index b87b4577a..f3c70d5ec 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -70655,6 +70655,633 @@ "x-openmapx-auth": "admin" } }, + "/api/data-manager/overture/recovery": { + "get": { + "operationId": "getApiDataManagerOvertureRecovery", + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "in": "query", + "name": "region", + "required": true + } + ], + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "region": { + "type": "string" + }, + "busy": { + "type": "boolean" + }, + "osmAvailable": { + "type": "boolean" + }, + "downloadedRelease": { + "type": [ + "string", + "null" + ] + }, + "installedRelease": { + "type": [ + "string", + "null" + ] + }, + "actions": { + "type": "object", + "properties": { + "overture-pull": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-ingest": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-extract": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-conflate": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-restart": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-restart" + ], + "additionalProperties": false + } + }, + "required": [ + "region", + "busy", + "osmAvailable", + "downloadedRelease", + "installedRelease", + "actions" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "data-manager-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/data-manager/overture/status": { "get": { "operationId": "getApiDataManagerOvertureStatus", diff --git a/apps/api/src/routes/__tests__/data-manager-search-index.test.ts b/apps/api/src/routes/__tests__/data-manager-search-index.test.ts index 062b19dbe..fdc1719b9 100644 --- a/apps/api/src/routes/__tests__/data-manager-search-index.test.ts +++ b/apps/api/src/routes/__tests__/data-manager-search-index.test.ts @@ -135,3 +135,50 @@ describe("GET /data-manager/notable-places/status", () => { expect(absent.statusCode).toBe(404); }); }); + +describe("GET /data-manager/overture/recovery", () => { + it("keeps operator prerequisites unavailable to service tokens", async () => { + const response = await app.inject({ + method: "GET", + url: "/data-manager/overture/recovery?region=europe/germany", + headers: { authorization: "Bearer service-token" }, + }); + expect(response.statusCode).toBe(403); + expect(fetchMock).not.toHaveBeenCalled(); + }); + it("proxies the exact region through the generated service contract without caching", async () => { + fetchMock.mockResolvedValueOnce( + Response.json({ + region: "europe/germany", + busy: false, + osmAvailable: false, + downloadedRelease: null, + installedRelease: null, + actions: Object.fromEntries( + [ + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-restart", + ].map((id) => [ + id, + { + allowed: id === "overture-pull", + reason: id === "overture-pull" ? null : "Missing inputs", + }, + ]), + ), + }), + ); + const response = await app.inject({ + method: "GET", + url: "/data-manager/overture/recovery?region=europe/germany", + }); + expect(response.statusCode).toBe(200); + expect(response.headers["cache-control"]).toBe("no-store"); + expect(String(fetchMock.mock.calls[0]?.[0])).toBe( + "https://data-manager.test:4000/overture/recovery?region=europe%2Fgermany", + ); + }); +}); diff --git a/apps/api/src/routes/data-manager.ts b/apps/api/src/routes/data-manager.ts index e5585fdd4..2e7b66f3a 100644 --- a/apps/api/src/routes/data-manager.ts +++ b/apps/api/src/routes/data-manager.ts @@ -552,6 +552,21 @@ export async function dataManagerRoute(app: FastifyInstance): Promise { }, ); + registerDataManagerAdminRoute(app, "getApiDataManagerOvertureRecovery", async (req, reply) => { + const auth = await authenticateDataManager(req); + if (auth.kind === "token") + return reply + .code(403) + .send({ error: "Overture recovery inspection requires an admin session" }); + const response = await dm.getOvertureRecovery( + { region: req.query.region }, + dataManagerOptions(), + ); + reply.header("cache-control", "no-store"); + reply.code(response.status); + return response.data; + }); + registerDataManagerAdminRoute(app, "getApiDataManagerOvertureStatus", async (req, reply) => { const auth = await authenticateDataManager(req); if (auth.kind === "token") { diff --git a/apps/api/src/services/__tests__/admin-job-handlers.test.ts b/apps/api/src/services/__tests__/admin-job-handlers.test.ts index c4efc6a73..9ea55daa4 100644 --- a/apps/api/src/services/__tests__/admin-job-handlers.test.ts +++ b/apps/api/src/services/__tests__/admin-job-handlers.test.ts @@ -214,7 +214,11 @@ describe("admin job handlers", () => { }); expect(response.statusCode).toBe(200); const payload = enqueue.mock.calls[0][1]; - expect(payload).toEqual({ operation: id, version: 1, input: raw }); + expect(payload).toEqual({ + operation: id, + version: id === "overture-conflate" ? 2 : 1, + input: raw, + }); const ctx = dataCtx(payload); await handleDataOperationJob(ctx); expect(runAdminOperationMock.mock.calls).toEqual([[ctx, effect, `admin-job.data.${id}`]]); @@ -301,7 +305,7 @@ describe("admin job handlers", () => { runAdminOperationMock.mockClear(); await expect( handleDataOperationJob( - dataCtx({ operation: "overture-conflate", version: 1, input: { region: "../etc" } }), + dataCtx({ operation: "overture-conflate", version: 2, input: { region: "../etc" } }), ), ).rejects.toThrow(/region/); await expect( @@ -310,6 +314,37 @@ describe("admin job handlers", () => { expect(runAdminOperationMock).not.toHaveBeenCalled(); }); + it("rejects old conflation jobs and forwards both reviewed version2 modes exactly", async () => { + runAdminOperationMock.mockClear(); + await expect( + handleDataOperationJob( + dataCtx({ + operation: "overture-conflate", + version: 1, + input: { region: "europe/germany", restart: true }, + }), + ), + ).rejects.toThrow(/catalog version 1; current is 2/); + expect(runAdminOperationMock).not.toHaveBeenCalled(); + for (const restart of [false, true]) { + const ctx = dataCtx({ + operation: "overture-conflate", + version: 2, + input: { region: "europe/germany", restart }, + }); + await handleDataOperationJob(ctx); + expect(runAdminOperationMock).toHaveBeenLastCalledWith( + ctx, + { + kind: "data.overtureConflate", + regionId: "europe/germany", + ...(restart ? { restart: true } : {}), + }, + "admin-job.data.overture-conflate", + ); + } + }); + it("rejects unknown operations and stale catalog versions", async () => { runAdminOperationMock.mockClear(); await expect( diff --git a/apps/api/src/services/__tests__/admin-operation-catalog.test.ts b/apps/api/src/services/__tests__/admin-operation-catalog.test.ts index eac6c710a..64606082c 100644 --- a/apps/api/src/services/__tests__/admin-operation-catalog.test.ts +++ b/apps/api/src/services/__tests__/admin-operation-catalog.test.ts @@ -46,8 +46,10 @@ describe("validateAdminOperationCatalog", () => { "convert-overpass", "link", "clean", - "generate-api-keys", "overture-sync", + "overture-pull", + "overture-ingest", + "overture-extract", "overture-conflate", "search-index-build", "notable-places-build", @@ -132,11 +134,6 @@ describe("shipped operation effects", () => { ], ["link", {}, { kind: "data.link" }], ["clean", { target: "osm" }, { kind: "data.clean", dataTypeId: "osm" }], - [ - "generate-api-keys", - {}, - { kind: "data.generateApiKeys", catalogRevisionId: "transitous-fixed-v1" }, - ], [ "overture-sync", { region: "europe/germany" }, @@ -175,7 +172,6 @@ describe("shipped operation effects", () => { ["search-index-build", { region: "Europe/Germany" }], ["notable-places-build", { endpoint: "https://attacker.example/sparql" }], ["download-fonts", { region: "europe/germany" }], - ["generate-api-keys", { repoUrl: "https://attacker.example", output: "/etc/passwd" }], ["update", { region: "europe/germany", argv: ["--privileged"] }], ])("%s rejects %j", (id, raw) => { const operation = getAdminOperation(id); @@ -211,3 +207,37 @@ describe("describeAdminOperationCatalog", () => { expect(clean?.defaults).toEqual({ target: "all" }); }); }); + +describe("guarded Overture recovery catalogue", () => { + it("requires destructive confirmation and a new version for both link modes", () => { + expect(getAdminOperation("overture-conflate")).toMatchObject({ + version: 2, + risk: "destructive", + confirmation: { title: expect.any(String), message: expect.any(String) }, + }); + }); + it.each([ + ["overture-pull", "data.overturePull"], + ["overture-ingest", "data.overtureIngest"], + ["overture-extract", "data.overtureExtract"], + ])("offers the fixed %s command without caller paths or releases", (id, kind) => { + const operation = getAdminOperation(id); + expect(operation).toBeDefined(); + if (!operation) return; + const parsed = parseAdminOperationInput(operation, { region: "europe/germany" }); + expect(parsed.ok).toBe(true); + if (parsed.ok) + expect(operation.effect(parsed.input)).toEqual({ kind, regionId: "europe/germany" }); + expect( + parseAdminOperationInput(operation, { + region: "europe/germany", + path: "/tmp/x", + releaseUrl: "https://example.test/x", + }).ok, + ).toBe(false); + }); +}); + +it("keeps remote credential-template generation retired", () => { + expect(getAdminOperation("generate-api-keys")).toBeUndefined(); +}); diff --git a/apps/api/src/services/admin-operation-catalog.ts b/apps/api/src/services/admin-operation-catalog.ts index 3dfb41682..fe0b32cf6 100644 --- a/apps/api/src/services/admin-operation-catalog.ts +++ b/apps/api/src/services/admin-operation-catalog.ts @@ -23,6 +23,9 @@ export type AdminOperationId = | "convert-overpass" | "link" | "clean" + | "overture-pull" + | "overture-ingest" + | "overture-extract" | "overture-sync" | "overture-conflate" | "search-index-build" @@ -253,12 +256,85 @@ export const ADMIN_OPERATIONS: readonly AnyAdminOperationDefinition[] = [ preview: (input) => [`Sync Overture Places for ${input.region}`], }), defineOperation({ - id: "overture-conflate", + id: "overture-pull", + version: 1, + group: "overture", + title: "Download release partition", + description: + "Download the current release partition for this region without changing installed Places.", + risk: "normal", + + input: z.strictObject({ region: regionSchema }), + fields: [requiredRegionField("Region for this recovery step.")], + defaults: { region: "" }, + example: { region: "europe/germany" }, + effect: (input) => ({ kind: "data.overturePull", regionId: input.region }), + preview: (input) => [ + "Download the current release partition for this region without changing installed Places.", + `Region: ${input.region}`, + "Prerequisites are rechecked under the existing Overture operation lease at execution.", + ], + }), + defineOperation({ + id: "overture-ingest", + version: 1, + group: "overture", + title: "Ingest downloaded partition", + description: + "Validate and atomically install the latest completed local release partition for this region.", + risk: "destructive", + confirmation: { + title: "Ingest downloaded partition", + message: + "Validate and atomically install the latest completed local release partition for this region.", + }, + input: z.strictObject({ region: regionSchema }), + fields: [requiredRegionField("Region for this recovery step.")], + defaults: { region: "" }, + example: { region: "europe/germany" }, + effect: (input) => ({ kind: "data.overtureIngest", regionId: input.region }), + preview: (input) => [ + "Validate and atomically install the latest completed local release partition for this region.", + `Region: ${input.region}`, + "Prerequisites are rechecked under the existing Overture operation lease at execution.", + ], + }), + defineOperation({ + id: "overture-extract", version: 1, group: "overture", + title: "Extract OSM features", + description: + "Replace the OSM extraction workspace for the installed region using its downloaded PBF.", + risk: "destructive", + confirmation: { + title: "Extract OSM features", + message: + "Replace the OSM extraction workspace for the installed region using its downloaded PBF.", + }, + input: z.strictObject({ region: regionSchema }), + fields: [requiredRegionField("Region for this recovery step.")], + defaults: { region: "" }, + example: { region: "europe/germany" }, + effect: (input) => ({ kind: "data.overtureExtract", regionId: input.region }), + preview: (input) => [ + "Replace the OSM extraction workspace for the installed region using its downloaded PBF.", + `Region: ${input.region}`, + "Prerequisites are rechecked under the existing Overture operation lease at execution.", + ], + }), + defineOperation({ + id: "overture-conflate", + version: 2, + group: "overture", title: "Overture Conflation", description: "Links Overture places to OSM features for a region.", - risk: "normal", + risk: "destructive", + confirmation: { + title: "Rebuild Overture links", + message: + "Resume or restart the selected region's leased link workspace. Restart discards its saved progress; published Places remain installed.", + }, input: z.strictObject({ region: regionSchema, restart: z.boolean().default(false) }), fields: [ requiredRegionField("Region whose Overture data is conflated."), diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index 02976641c..e4c344f19 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -421,6 +421,9 @@ describe("administrative data runtime", () => { { kind: "data.clean", dataTypeId: "osm" }, { kind: "data.generateApiKeys", catalogRevisionId: "transitous-fixed-v1" }, { kind: "data.overtureSync", regionId: "europe/germany" }, + { kind: "data.overturePull", regionId: "europe/germany" }, + { kind: "data.overtureIngest", regionId: "europe/germany" }, + { kind: "data.overtureExtract", regionId: "europe/germany" }, { kind: "data.overtureConflate", regionId: "europe/germany", restart: true }, { kind: "data.searchIndexBuild", regionId: "europe/germany" }, ]; @@ -430,6 +433,9 @@ describe("administrative data runtime", () => { "data.update", "data.convertOverpass", "data.overtureSync", + "data.overturePull", + "data.overtureIngest", + "data.overtureExtract", "data.overtureConflate", "data.searchIndexBuild", ]); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 24ef5d894..f4a73ae3b 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -110,7 +110,7 @@ const BACKUP_STORE_LOCK_NAME = ".backup-store.lock"; const MAX_RELEASE_STORE_BYTES = MAX_RELEASE_STORE_ENTRIES * MAX_RELEASE_MANIFEST_BYTES; const BACKUP_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; const SERVICE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/; -const REGION_ID = /^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$/; +const _REGION_ID = /^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$/; const DATA_TYPE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/; const DATA_TYPE_ALIASES: Readonly> = { osm: ["osm-pbf"], @@ -1970,6 +1970,18 @@ export function createAdministrativeRuntime( await runMatchingDataCommand(operation, context); return { completed: true, resourceId: operation.regionId }; }; + runtime["data.overturePull"] = async (operation, context) => { + await runMatchingDataCommand(operation, context); + return { completed: true, resourceId: operation.regionId }; + }; + runtime["data.overtureIngest"] = async (operation, context) => { + await runMatchingDataCommand(operation, context); + return { completed: true, resourceId: operation.regionId }; + }; + runtime["data.overtureExtract"] = async (operation, context) => { + await runMatchingDataCommand(operation, context); + return { completed: true, resourceId: operation.regionId }; + }; runtime["data.overtureConflate"] = async (operation, context) => { await runMatchingDataCommand(operation, context); return { completed: true, resourceId: operation.regionId }; diff --git a/apps/web/src/components/admin/services/OvertureMaintenance.test.tsx b/apps/web/src/components/admin/services/OvertureMaintenance.test.tsx index 7fdccaf01..095ca8dc9 100644 --- a/apps/web/src/components/admin/services/OvertureMaintenance.test.tsx +++ b/apps/web/src/components/admin/services/OvertureMaintenance.test.tsx @@ -31,6 +31,11 @@ describe("OvertureMaintenance", () => { expect(markup).toContain("europe/germany"); expect(markup).toContain("Full sync"); expect(markup).toContain("Resume links"); + expect(markup).toContain("Restart links from scratch"); + expect(markup).toContain("Advanced recovery"); + expect(markup).toContain("Download release partition"); + expect(markup).toContain("Ingest downloaded partition"); + expect(markup).toContain("Extract OSM features"); expect(markup).toContain("completed"); expect(markup).not.toContain("completed · complete"); }); diff --git a/apps/web/src/components/admin/services/OvertureMaintenance.tsx b/apps/web/src/components/admin/services/OvertureMaintenance.tsx index 31782077c..b3070c128 100644 --- a/apps/web/src/components/admin/services/OvertureMaintenance.tsx +++ b/apps/web/src/components/admin/services/OvertureMaintenance.tsx @@ -1,8 +1,12 @@ "use client"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; import MergeIcon from "@mui/icons-material/Merge"; import RefreshIcon from "@mui/icons-material/Refresh"; import SyncIcon from "@mui/icons-material/Sync"; +import Accordion from "@mui/material/Accordion"; +import AccordionDetails from "@mui/material/AccordionDetails"; +import AccordionSummary from "@mui/material/AccordionSummary"; import Alert from "@mui/material/Alert"; import Box from "@mui/material/Box"; import Button from "@mui/material/Button"; @@ -14,18 +18,47 @@ import TextField from "@mui/material/TextField"; import Typography from "@mui/material/Typography"; import { createApiClient } from "@openmapx/core"; import * as sdk from "@openmapx/core/api-admin-sdk"; -import type { OvertureStatus } from "@openmapx/core/api-contracts"; +import type { OvertureRecoveryStatus, OvertureStatus } from "@openmapx/core/api-contracts"; import { useMutation, useQuery } from "@tanstack/react-query"; import Link from "next/link"; import { useMemo, useState } from "react"; import { SERVICE_READ_TIMEOUT_MS } from "@/lib/admin/serviceAdmin"; import { useServiceJobRegistration } from "@/lib/admin/serviceAdminJobs"; import { useAdminMutationOwner } from "@/lib/admin/useAdminMutationOwner"; -import { runAdminOperation } from "../operations/adminOperationsApi"; +import { + type AdminOperationFormValues, + previewAdminOperation, + runAdminOperation, +} from "../operations/adminOperationsApi"; import { useAdminToast } from "../shared/AdminToast"; import { ConfirmDialog } from "../shared/ConfirmDialog"; -type Operation = "overture-sync" | "overture-conflate"; +type Operation = + | "overture-sync" + | "overture-conflate" + | "overture-restart" + | "overture-pull" + | "overture-ingest" + | "overture-extract"; +const operationLabels: Record = { + "overture-sync": "Full sync", + "overture-conflate": "Resume links", + "overture-restart": "Restart links from scratch", + "overture-pull": "Download release partition", + "overture-ingest": "Ingest downloaded partition", + "overture-extract": "Extract OSM features", +}; +function operationRequest(name: Operation, region: string) { + return { + id: name === "overture-restart" ? "overture-conflate" : name, + input: { + region, + ...(["overture-restart", "overture-conflate"].includes(name) + ? { restart: name === "overture-restart" } + : {}), + } as AdminOperationFormValues, + }; +} export function overtureProgress( status: @@ -107,7 +140,13 @@ function OvertureMaintenanceView({ apiUrl }: { apiUrl: string }) { [apiUrl], ); const [region, setRegion] = useState(""); - const [confirmOperation, setConfirmOperation] = useState(null); + const [prepared, setPrepared] = useState<{ + name: Operation; + region: string; + input: AdminOperationFormValues; + preview: Awaited>; + } | null>(null); + const confirmOperation = prepared?.name ?? null; const statusQuery = useQuery({ queryKey: overtureStatusQueryKey(apiUrl), @@ -123,7 +162,38 @@ function OvertureMaintenanceView({ apiUrl }: { apiUrl: string }) { refetchInterval: (query) => (query.state.data?.status === "running" ? 10_000 : 60_000), }); + const regionValid = + /^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$/.test(region.trim()) && region.trim().length <= 512; + const recoveryQuery = useQuery({ + queryKey: [...overtureStatusQueryKey(apiUrl), "recovery", region.trim()], + enabled: regionValid, + queryFn: async ({ signal }) => + sdk.unwrapAdminResponse( + await sdk.getApiDataManagerOvertureRecovery( + { region: region.trim() }, + { client, signal, timeoutMs: SERVICE_READ_TIMEOUT_MS }, + ), + ), + refetchInterval: (query) => (query.state.data?.busy ? 10000 : 30000), + }); + const owner = useAdminMutationOwner(client, JSON.stringify([region, confirmOperation])); + const prepare = useMutation({ + retry: false, + mutationFn: async (name: Operation) => { + const scope = owner, + selectedRegion = region.trim(), + request = operationRequest(name, selectedRegion); + try { + const preview = await previewAdminOperation(client, request.id, request.input); + if (scope.active) + setPrepared({ name, region: selectedRegion, input: request.input, preview }); + } catch (error) { + if (scope.active) + showToast(error instanceof Error ? error.message : "Preview failed", "error"); + } + }, + }); const operation = useMutation({ retry: false, mutationFn: async (operationName: Operation) => { @@ -133,11 +203,14 @@ function OvertureMaintenanceView({ apiUrl }: { apiUrl: string }) { refreshKeys: [overtureStatusQueryKey(apiUrl)], } as const; try { - const jobId = await runAdminOperation(client, operationName, { region: region.trim() }); + if (!prepared || prepared.name !== operationName || prepared.region !== region.trim()) + throw Error("The operation preview changed. Review it again."); + const request = operationRequest(operationName, prepared.region); + const jobId = await runAdminOperation(client, request.id, prepared.input); register({ client, apiUrl, jobId, effect }); if (execution.active) { showToast(`Queued ${operationName} (${jobId})`); - setConfirmOperation(null); + setPrepared(null); } } catch (error) { if (execution.active) @@ -149,6 +222,21 @@ function OvertureMaintenanceView({ apiUrl }: { apiUrl: string }) { const status = statusQuery.data; const progress = overtureProgress(status); + const recovery = recoveryQuery.data; + const busy = recovery?.busy === true || (status?.status === "running" && !status.stalled); + const pending = operation.isPending || prepare.isPending; + const canRun = (name: Operation) => + regionValid && + !pending && + !busy && + Boolean(recovery) && + (name === "overture-sync" || recovery?.actions?.[name]?.allowed === true); + const leaseReason = recovery?.busy + ? "Another Overture operation holds the workspace lease." + : status?.status === "running" && !status.stalled + ? "The Overture link worker is still running." + : null; + const visiblePhase = status?.status === "completed" && status.phase === "complete" ? undefined : status?.phase; @@ -174,7 +262,13 @@ function OvertureMaintenanceView({ apiUrl }: { apiUrl: string }) { durable OSM↔GERS link index. - + {leaseReason && ( + + {leaseReason} + + )} + {regionValid && recoveryQuery.isError && ( + + Prerequisite inspection is unavailable. Restore connectivity and refresh before queuing + work. + + )} + + {regionValid && !busy && recovery?.actions?.["overture-conflate"]?.reason && ( + + {recovery.actions["overture-conflate"].reason} + + )} + + }> + Advanced recovery + + + + Run an individual recovery step. Full sync remains the usual workflow. Inputs are + checked again under the existing workspace lease. + + {recovery && ( + + Completed local partition: {recovery.downloadedRelease ?? "none"} · OSM PBF:{" "} + {recovery.osmAvailable ? "available" : "missing"} · Installed release:{" "} + {recovery.installedRelease ?? "none"} + + )} + + {(["overture-pull", "overture-ingest", "overture-extract"] as const).map((name) => ( + + + {!busy && recovery?.actions?.[name]?.reason && ( + + {recovery.actions[name].reason} + + )} + + ))} + + + + setConfirmOperation(null)} + onCancel={() => setPrepared(null)} onConfirm={() => { if (confirmOperation) operation.mutate(confirmOperation); }} diff --git a/apps/web/src/components/admin/services/OvertureMaintenance.workflow.test.tsx b/apps/web/src/components/admin/services/OvertureMaintenance.workflow.test.tsx new file mode 100644 index 000000000..31d3bff49 --- /dev/null +++ b/apps/web/src/components/admin/services/OvertureMaintenance.workflow.test.tsx @@ -0,0 +1,123 @@ +/** Real UI state/queries; mocked DTOs are presentation evidence, not authority. */ +import { QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { cleanup, createTestQueryClient, fireEvent, render, screen, waitFor } from "@/test"; +import { OvertureMaintenance } from "./OvertureMaintenance"; + +const mocks = vi.hoisted(() => ({ + preview: vi.fn(), + run: vi.fn(), + register: vi.fn(), + recovery: vi.fn(), + busy: false, +})); +vi.mock("@/lib/admin/serviceAdminJobs", () => ({ + useServiceJobRegistration: () => mocks.register, +})); +vi.mock("@/lib/admin/useAdminMutationOwner", () => ({ + useAdminMutationOwner: () => ({ active: true }), +})); +vi.mock("../shared/AdminToast", () => ({ useAdminToast: () => vi.fn() })); +vi.mock("../operations/adminOperationsApi", () => ({ + previewAdminOperation: mocks.preview, + runAdminOperation: mocks.run, +})); +vi.mock("@openmapx/core/api-admin-sdk", async (original) => ({ + ...(await original()), + getApiDataManagerOvertureStatus: async () => ({ status: 404, data: { error: "Not installed" } }), + getApiDataManagerOvertureRecovery: mocks.recovery, +})); +beforeEach(() => { + vi.clearAllMocks(); + mocks.busy = false; + mocks.recovery.mockImplementation(async () => ({ + status: 200, + data: { + region: "europe/germany", + busy: mocks.busy, + downloadedRelease: "2026-10-21.0", + installedRelease: "2026-10-21.0", + osmAvailable: true, + actions: Object.fromEntries( + [ + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-restart", + ].map((id) => [ + id, + { + allowed: !mocks.busy, + reason: mocks.busy ? "Another Overture operation holds the workspace lease." : null, + }, + ]), + ), + }, + })); + mocks.preview.mockImplementation(async (_client, id, input) => ({ + ok: true, + input, + preview: [`Server-authored preview for ${id}`], + risk: "destructive", + confirmation: { + title: "Confirm guarded recovery", + message: "This changes the selected workspace.", + }, + })); + mocks.run.mockResolvedValue("synthetic-job-dto"); +}); +afterEach(cleanup); +function mount() { + render( + + + , + ); + fireEvent.change(screen.getByLabelText("Region"), { target: { value: "europe/germany" } }); +} +it.each([ + ["Restart links from scratch", true, "Restart links"], + ["Resume links", false, "Resume"], +] as const)( + "previews %s and submits only after explicit confirmation", + async (label, restart, confirm) => { + mount(); + const button = await screen.findByRole("button", { name: label }); + await waitFor(() => expect(button).not.toBeDisabled()); + fireEvent.click(button); + await screen.findByText(/Server-authored preview/); + expect(mocks.preview.mock.calls[0]?.[1]).toBe("overture-conflate"); + expect(mocks.preview.mock.calls[0]?.[2]).toEqual({ region: "europe/germany", restart }); + expect(mocks.run).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: confirm })); + await waitFor(() => expect(mocks.run).toHaveBeenCalledTimes(1)); + expect(mocks.run.mock.calls[0]?.[1]).toBe("overture-conflate"); + expect(mocks.run.mock.calls[0]?.[2]).toEqual({ region: "europe/germany", restart }); + expect(mocks.register).toHaveBeenCalledTimes(1); + }, +); +it("shows the busy lease reason and disables conflicting actions", async () => { + mocks.busy = true; + mount(); + await screen.findByText("Another Overture operation holds the workspace lease."); + expect(screen.getByRole("button", { name: "Full sync" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Restart links from scratch" })).toBeDisabled(); +}); +it.each([ + ["Download release partition", "overture-pull"], + ["Ingest downloaded partition", "overture-ingest"], + ["Extract OSM features", "overture-extract"], +] as const)("confirms the fixed advanced %s action", async (label, id) => { + mount(); + fireEvent.click(screen.getByRole("button", { name: "Advanced recovery" })); + const button = await screen.findByRole("button", { name: label }); + await waitFor(() => expect(button).not.toBeDisabled()); + fireEvent.click(button); + await screen.findByText(/Server-authored preview/); + expect(mocks.run).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Run operation" })); + await waitFor(() => expect(mocks.run).toHaveBeenCalledTimes(1)); + expect(mocks.run.mock.calls[0]?.[1]).toBe(id); + expect(mocks.run.mock.calls[0]?.[2]).toEqual({ region: "europe/germany" }); +}); diff --git a/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx b/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx index a17e51b0c..b7cc96b6d 100644 --- a/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx +++ b/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx @@ -151,6 +151,23 @@ function setup( vi.stubGlobal("fetch", async (input: string, init: RequestInit = {}) => { const url = new URL(input); requests.push({ url, init }); + if (url.pathname.endsWith("/overture/recovery")) + return Response.json({ + region: "europe/germany", + busy: false, + osmAvailable: true, + downloadedRelease: "2026-10-21.0", + installedRelease: "2026-10-21.0", + actions: Object.fromEntries( + [ + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-restart", + ].map((id) => [id, { allowed: true, reason: null }]), + ), + }); const result = handler?.(url, init); if (result) return result; if (url.pathname.startsWith("/api/admin/jobs/")) @@ -340,7 +357,7 @@ function embedded(kind: string, origin = apiUrl) { if (kind === "search-index-build") return ; return ; } -async function queueEmbedded(kind: string) { +async function queueEmbedded(kind: string, confirmPreview = true) { if (kind.startsWith("overture") || kind === "search-index-build") { const region = await screen.findByLabelText("Region"); fireEvent.change(region, { target: { value: "europe/germany" } }); @@ -353,14 +370,19 @@ async function queueEmbedded(kind: string) { : kind === "search-index-build" ? "Build index" : "Build now"; - fireEvent.click(await screen.findByRole("button", { name: queue })); + const queueButton = await screen.findByRole("button", { name: queue }); + await waitFor(() => expect(queueButton).not.toBeDisabled()); + fireEvent.click(queueButton); + if (!confirmPreview) return; const confirm = kind === "overture-sync" - ? "Start full sync" + ? "Run operation" : kind === "overture-conflate" ? "Resume" : "Start build"; - fireEvent.click(await screen.findByRole("button", { name: confirm })); + fireEvent.click( + await within(await screen.findByRole("dialog")).findByRole("button", { name: confirm }), + ); } describe("embedded operation callers", () => { it.each(["success", "failed", "canceled"] as const)( @@ -488,9 +510,11 @@ describe("embedded operation callers", () => { fireEvent.change(await screen.findByLabelText("Region"), { target: { value: "new/region" }, }); - fireEvent.click(await screen.findByRole("button", { name: queue })); + const queueButton = await screen.findByRole("button", { name: queue }); + await waitFor(() => expect(queueButton).not.toBeDisabled()); + fireEvent.click(queueButton); await act(async () => deferred.resolve(Response.json({ ok: true, jobId: "job-1" }))); - expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(await screen.findByRole("dialog")).toBeInTheDocument(); expect(toast).not.toHaveBeenCalled(); expect(calls.filter((r) => r.url.pathname.includes("/jobs/"))).toHaveLength(0); }, @@ -564,7 +588,7 @@ describe("backup dialog and input ownership", () => { target: { value: "new-service" }, }); await act(async () => deferred.resolve(Response.json({ ok: true, jobId: "job-1" }))); - expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(await screen.findByRole("dialog")).toBeInTheDocument(); if (kind === "restore") expect(screen.getByLabelText("Service IDs (optional)")).toHaveValue("new-service"); expect(toast).not.toHaveBeenCalled(); @@ -808,7 +832,7 @@ describe("wire intent, validation, polling and policy", () => { expect(calls.find((r) => r.init.method === "DELETE")?.url.pathname).toBe( "/api/admin/services/backups/unsafe%2Fname%20%25%3F", ); - expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(await screen.findByRole("dialog")).toBeInTheDocument(); }); it.each([ "create", @@ -859,7 +883,7 @@ describe("wire intent, validation, polling and policy", () => { fireEvent.click((await screen.findAllByLabelText("Fan-out rollback"))[0] as HTMLElement); } else { mount(embedded(kind)); - await queueEmbedded(kind); + await queueEmbedded(kind, !kind.startsWith("overture")); } if (kind === "policy") await screen.findByText("Synthetic refused"); else await waitFor(() => expect(toast).toHaveBeenCalledWith("Synthetic refused", "error")); @@ -955,7 +979,7 @@ describe("pending form inputs and malformed operation acceptance", () => { target: { value: "replacement" }, }); await act(async () => deferred.resolve(Response.json({ ok: true, jobId: "job-1" }))); - expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(await screen.findByRole("dialog")).toBeInTheDocument(); expect(screen.getByLabelText("Service IDs (optional)")).toHaveValue("replacement"); expect(toast).not.toHaveBeenCalled(); }); @@ -967,7 +991,7 @@ describe("pending form inputs and malformed operation acceptance", () => { fireEvent.click(screen.getByRole("button", { name: "Queue Update data" })); fireEvent.click(await screen.findByRole("button", { name: "Confirm" })); await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.any(String), "error")); - expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(await screen.findByRole("dialog")).toBeInTheDocument(); expect(toast).not.toHaveBeenCalledWith("invented"); }, ); @@ -1173,7 +1197,7 @@ it.each(["restore", "delete"])( screen.getByRole("button", { name: kind === "restore" ? "Restore" : "Delete" }), ); await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.any(String), "error")); - expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(await screen.findByRole("dialog")).toBeInTheDocument(); expect(calls.filter((r) => r.url.pathname.includes("/jobs/"))).toHaveLength(0); }, ); diff --git a/docs/docs/developer/cli-reference.md b/docs/docs/developer/cli-reference.md index 6a7455b9e..53154a0ac 100644 --- a/docs/docs/developer/cli-reference.md +++ b/docs/docs/developer/cli-reference.md @@ -132,7 +132,7 @@ hardlinks, authenticated feeds — lives in | `data generate-api-keys` | Generate the Transitous API-key template for feeds that require keys. Flags: `--repo-url `, `--output `. | | `data overture-sync [region]` | Run the release-pinned regional refresh: pull and atomic ingest, then start the independently retryable OSM↔GERS link rebuild. | | `data overture-pull [region]` | Resolve exact Overture Places assets through STAC, pull a regional Parquet snapshot, and write its validated release contract. | -| `data overture-ingest [region]` | Verify the regional release contract and ingest its Overture Places snapshot into PostGIS. | +| `data overture-ingest [region]` | Verify and ingest the newest completed local regional Places snapshot into PostGIS. Pull a partition first; recovery does not discover a different upstream release. | | `data overture-conflate [region]` | Resume the installed release's durable extraction, scoring, exact assignment, or publication phase. Flag: `--restart` deliberately discards saved phases and starts from extraction. | | `data overture-status` | Show the installed release, durable phase, cursors, counts, timings, heartbeat age, and last error. | | `data overture-extract [region]` | Stream OSM POIs from the local PBF and atomically replace `overture_places.osm_pois` (diagnostic/manual use). | diff --git a/docs/docs/features/overture-places.md b/docs/docs/features/overture-places.md index ce531d4f6..2bab23097 100644 --- a/docs/docs/features/overture-places.md +++ b/docs/docs/features/overture-places.md @@ -208,6 +208,17 @@ Discard saved progress and deliberately restart from OSM extraction with: pnpm openmapx data overture-conflate europe/germany/berlin --restart ``` +The admin panel offers the same controls under **Services → Data workflows → +Overture Places**. **Full sync** remains the usual workflow. **Resume links** and +**Restart links from scratch** show a server preview before confirmation; +restart discards the selected resumable workspace, preserving published Places. +**Advanced recovery** exposes download, ingest and OSM extraction individually. +Ingest selects the newest completed local partition for the selected region, +without discovering a newer upstream release. Extraction and restart require +that region's installed Places and OSM PBF. The panel shows missing prerequisites +and disables conflicting actions while the shared Overture lease is held. +Workers check these inputs again under the same lease when they execute. + Inspect the installed release, phase, cursors, counts, timings, heartbeat age, and last error with: diff --git a/docs/docs/install/preparing-data.md b/docs/docs/install/preparing-data.md index ccfcee12c..ba7563b17 100644 --- a/docs/docs/install/preparing-data.md +++ b/docs/docs/install/preparing-data.md @@ -270,10 +270,10 @@ pnpm openmapx data notable-places status # Regional or planet Overture Places synchronization and conflation pnpm openmapx data overture-sync [region] # end-to-end pull, ingest, and conflation pnpm openmapx data overture-pull [region] # pull Parquet partitions from STAC catalog -pnpm openmapx data overture-ingest [region] # ingest raw Places into database staging +pnpm openmapx data overture-ingest [region] # ingest the newest completed local partition pnpm openmapx data overture-conflate [region] # rebuild OSM <-> Overture GERS entity links pnpm openmapx data overture-status # inspect active and staging Overture snapshots -pnpm openmapx data overture-extract # extract regional boundaries from OSM PBF +pnpm openmapx data overture-extract [region] # extract regional POIs from OSM PBF ``` For global ambient maps, explicitly prepare `planet` with the OSM download, diff --git a/packages/cli/__tests__/cli-overture-recovery.test.ts b/packages/cli/__tests__/cli-overture-recovery.test.ts new file mode 100644 index 000000000..4a3e214e9 --- /dev/null +++ b/packages/cli/__tests__/cli-overture-recovery.test.ts @@ -0,0 +1,42 @@ +/** CLI delegation only; DTO mocks provide no execution/host authority. */ +import { Command } from "commander"; +import { afterEach, expect, it, vi } from "vitest"; + +const step = vi.hoisted(() => vi.fn()); +vi.mock("../src/lib/data-execution", async (original) => ({ + ...(await original()), + runCliDataStep: step, +})); + +import { registerDataCommands } from "../src/commands/data"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); +}); +it.each(["overture-pull", "overture-ingest", "overture-extract"] as const)( + "routes %s through the original durable data-step owner", + async (kind) => { + step.mockResolvedValue({ id: "synthetic-observer-dto" }); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(globalThis, "fetch").mockImplementation( + async (url) => + new Response( + `${JSON.stringify({ event: "done", ok: true, ...(String(url).includes("/overture/pull") ? { path: "/synthetic/pull" } : {}) })}\n`, + { + headers: { + "content-type": "application/x-ndjson", + "cache-control": "no-cache", + "x-accel-buffering": "no", + }, + }, + ), + ); + const app = new Command(); + app.exitOverride(); + registerDataCommands(app); + await app.parseAsync(["data", kind, "europe/germany"], { from: "user" }); + expect(step).toHaveBeenCalledTimes(1); + expect(step.mock.calls[0]?.[0]).toEqual({ kind, region: "europe/germany" }); + }, +); diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index 2383ecffd..52b6e9279 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -783,13 +783,7 @@ export function registerDataCommands(program: Command): void { const resolvedRegion = region || process.env.OPENMAPX_REGION || "europe/germany/berlin"; try { log.dim(`Pulling Overture places for "${resolvedRegion}"…`); - await dm.pullOvertureWithProgress( - { region: resolvedRegion }, - { - onProgress: ({ message }) => log.dim(message), - transport, - }, - ); + await runCliDataStep({ kind: "overture-pull", region: resolvedRegion }, transport); log.ok(`Overture pull complete for "${resolvedRegion}"`); } catch (err) { log.err(`overture-pull failed: ${(err as Error).message}`); @@ -800,19 +794,13 @@ export function registerDataCommands(program: Command): void { data .command("overture-ingest [region]") - .description("Ingest Overture places parquet into PostGIS for a region") + .description("Ingest the latest completed local Overture partition into PostGIS for a region") .action(async (region: string | undefined) => { const transport = createCliTransport(); const resolvedRegion = region || process.env.OPENMAPX_REGION || "europe/germany/berlin"; try { log.dim(`Ingesting Overture places for "${resolvedRegion}"…`); - await dm.ingestOvertureWithProgress( - { region: resolvedRegion }, - { - onProgress: ({ message }) => log.dim(message), - transport, - }, - ); + await runCliDataStep({ kind: "overture-ingest", region: resolvedRegion }, transport); log.ok(`Overture ingest complete for "${resolvedRegion}"`); } catch (err) { log.err(`overture-ingest failed: ${(err as Error).message}`); @@ -865,13 +853,7 @@ export function registerDataCommands(program: Command): void { const resolvedRegion = region || process.env.OPENMAPX_REGION || "europe/germany/berlin"; try { log.dim(`Extracting OSM POIs for "${resolvedRegion}"…`); - await dm.extractOvertureWithProgress( - { region: resolvedRegion }, - { - onProgress: ({ message }) => log.dim(message), - transport, - }, - ); + await runCliDataStep({ kind: "overture-extract", region: resolvedRegion }, transport); log.ok(`OSM POI extraction complete for "${resolvedRegion}"`); } catch (err) { log.err(`overture-extract failed: ${(err as Error).message}`); diff --git a/packages/core/src/api/contracts/data-manager-admin/index.ts b/packages/core/src/api/contracts/data-manager-admin/index.ts index 6351d2faa..be5b76f34 100644 --- a/packages/core/src/api/contracts/data-manager-admin/index.ts +++ b/packages/core/src/api/contracts/data-manager-admin/index.ts @@ -3,7 +3,10 @@ export type { NotablePlacesStatus, SearchIndexStatus, } from "../../../services/data-manager/contracts/indexes"; -export type { OvertureStatus } from "../../../services/data-manager/contracts/overture"; +export type { + OvertureRecoveryStatus, + OvertureStatus, +} from "../../../services/data-manager/contracts/overture"; export * from "./common"; export * from "./operations"; export * from "./providers"; diff --git a/packages/core/src/api/contracts/data-manager-admin/operations.ts b/packages/core/src/api/contracts/data-manager-admin/operations.ts index 89a5e9534..d2f655269 100644 --- a/packages/core/src/api/contracts/data-manager-admin/operations.ts +++ b/packages/core/src/api/contracts/data-manager-admin/operations.ts @@ -5,13 +5,20 @@ import { ambientPlacesStatusSchema, ambientRegionSchema, } from "../../../services/data-manager/contracts/ambient"; -import { acceptedSchema, okSchema } from "../../../services/data-manager/contracts/common"; +import { + acceptedSchema, + okSchema, + regionBodySchema, +} from "../../../services/data-manager/contracts/common"; import { uncachedResponseHeadersSchema } from "../../../services/data-manager/contracts/headers"; import { notablePlacesStatusSchema, searchIndexStatusSchema, } from "../../../services/data-manager/contracts/indexes"; -import { overtureStatusSchema } from "../../../services/data-manager/contracts/overture"; +import { + overtureRecoverySchema, + overtureStatusSchema, +} from "../../../services/data-manager/contracts/overture"; import { addTransitSourceRequestSchema, transitBumpResultSchema, @@ -27,7 +34,6 @@ import { transitSourcesQuerySchema, transitSyncAcceptedSchema, transitSyncPreflightSchema, - transitSyncRequestSchema, } from "../../../services/data-manager/contracts/transit"; import { adminCommonResponses as common, @@ -262,6 +268,13 @@ export const dataManagerAdminOperations = { 502: json(httpErrorSchema), }, }, + getApiDataManagerOvertureRecovery: { + method: "GET", + path: "/api/data-manager/overture/recovery", + auth: admin, + request: { query: regionBodySchema }, + responses: { ...common, 200: json(overtureRecoverySchema), 503: json(httpErrorSchema) }, + }, getApiDataManagerOvertureStatus: { method: "GET", path: "/api/data-manager/overture/status", diff --git a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts index adbe2e88f..290fa72c9 100644 --- a/packages/core/src/api/generated/admin/data-manager-admin.generated.ts +++ b/packages/core/src/api/generated/admin/data-manager-admin.generated.ts @@ -106,6 +106,16 @@ import type { GetApiDataManagerNotablePlacesStatus415, GetApiDataManagerNotablePlacesStatus429, GetApiDataManagerNotablePlacesStatus500, + GetApiDataManagerOvertureRecovery200, + GetApiDataManagerOvertureRecovery400, + GetApiDataManagerOvertureRecovery401, + GetApiDataManagerOvertureRecovery403, + GetApiDataManagerOvertureRecovery413, + GetApiDataManagerOvertureRecovery415, + GetApiDataManagerOvertureRecovery429, + GetApiDataManagerOvertureRecovery500, + GetApiDataManagerOvertureRecovery503, + GetApiDataManagerOvertureRecoveryParams, GetApiDataManagerOvertureStatus200, GetApiDataManagerOvertureStatus400, GetApiDataManagerOvertureStatus401, @@ -404,6 +414,7 @@ import { getAdminTransitPinsRequest, getApiAdminAmbientPlacesStatusRequest, getApiDataManagerNotablePlacesStatusRequest, + getApiDataManagerOvertureRecoveryRequest, getApiDataManagerOvertureStatusRequest, getApiDataManagerProvidersIdRequest, getApiDataManagerProvidersRequest, @@ -2166,6 +2177,103 @@ export const getApiDataManagerNotablePlacesStatus = async ( ); }; +export type getApiDataManagerOvertureRecoveryResponse200 = { + data: GetApiDataManagerOvertureRecovery200; + status: 200; +}; + +export type getApiDataManagerOvertureRecoveryResponse400 = { + data: GetApiDataManagerOvertureRecovery400; + status: 400; +}; + +export type getApiDataManagerOvertureRecoveryResponse401 = { + data: GetApiDataManagerOvertureRecovery401; + status: 401; +}; + +export type getApiDataManagerOvertureRecoveryResponse403 = { + data: GetApiDataManagerOvertureRecovery403; + status: 403; +}; + +export type getApiDataManagerOvertureRecoveryResponse413 = { + data: GetApiDataManagerOvertureRecovery413; + status: 413; +}; + +export type getApiDataManagerOvertureRecoveryResponse415 = { + data: GetApiDataManagerOvertureRecovery415; + status: 415; +}; + +export type getApiDataManagerOvertureRecoveryResponse429 = { + data: GetApiDataManagerOvertureRecovery429; + status: 429; +}; + +export type getApiDataManagerOvertureRecoveryResponse500 = { + data: GetApiDataManagerOvertureRecovery500; + status: 500; +}; + +export type getApiDataManagerOvertureRecoveryResponse503 = { + data: GetApiDataManagerOvertureRecovery503; + status: 503; +}; + +export type getApiDataManagerOvertureRecoveryResponseSuccess = + getApiDataManagerOvertureRecoveryResponse200 & { + headers: Headers; + }; +export type getApiDataManagerOvertureRecoveryResponseError = ( + | getApiDataManagerOvertureRecoveryResponse400 + | getApiDataManagerOvertureRecoveryResponse401 + | getApiDataManagerOvertureRecoveryResponse403 + | getApiDataManagerOvertureRecoveryResponse413 + | getApiDataManagerOvertureRecoveryResponse415 + | getApiDataManagerOvertureRecoveryResponse429 + | getApiDataManagerOvertureRecoveryResponse500 + | getApiDataManagerOvertureRecoveryResponse503 +) & { + headers: Headers; +}; + +export type getApiDataManagerOvertureRecoveryResponse = + | getApiDataManagerOvertureRecoveryResponseSuccess + | getApiDataManagerOvertureRecoveryResponseError; + +export const getGetApiDataManagerOvertureRecoveryUrl = ( + params: GetApiDataManagerOvertureRecoveryParams, +) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 + ? `/api/data-manager/overture/recovery?${stringifiedParams}` + : `/api/data-manager/overture/recovery`; +}; + +export const getApiDataManagerOvertureRecovery = async ( + params: GetApiDataManagerOvertureRecoveryParams, + options?: Parameters[1], +): Promise => { + return getApiDataManagerOvertureRecoveryRequest( + getGetApiDataManagerOvertureRecoveryUrl(params), + { + ...options, + method: "GET", + }, + ); +}; + export type getApiDataManagerOvertureStatusResponse200 = { data: GetApiDataManagerOvertureStatus200; status: 200; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200.ts new file mode 100644 index 000000000..52f49c00d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetApiDataManagerOvertureRecovery200Actions } from "./getApiDataManagerOvertureRecovery200Actions"; + +export type GetApiDataManagerOvertureRecovery200 = { + region: string; + busy: boolean; + osmAvailable: boolean; + /** @nullable */ + downloadedRelease: string | null; + /** @nullable */ + installedRelease: string | null; + actions: GetApiDataManagerOvertureRecovery200Actions; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200Actions.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200Actions.ts new file mode 100644 index 000000000..9b5338592 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200Actions.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetApiDataManagerOvertureRecovery200ActionsOvertureConflate } from "./getApiDataManagerOvertureRecovery200ActionsOvertureConflate"; +import type { GetApiDataManagerOvertureRecovery200ActionsOvertureExtract } from "./getApiDataManagerOvertureRecovery200ActionsOvertureExtract"; +import type { GetApiDataManagerOvertureRecovery200ActionsOvertureIngest } from "./getApiDataManagerOvertureRecovery200ActionsOvertureIngest"; +import type { GetApiDataManagerOvertureRecovery200ActionsOverturePull } from "./getApiDataManagerOvertureRecovery200ActionsOverturePull"; +import type { GetApiDataManagerOvertureRecovery200ActionsOvertureRestart } from "./getApiDataManagerOvertureRecovery200ActionsOvertureRestart"; + +export type GetApiDataManagerOvertureRecovery200Actions = { + "overture-pull": GetApiDataManagerOvertureRecovery200ActionsOverturePull; + "overture-ingest": GetApiDataManagerOvertureRecovery200ActionsOvertureIngest; + "overture-extract": GetApiDataManagerOvertureRecovery200ActionsOvertureExtract; + "overture-conflate": GetApiDataManagerOvertureRecovery200ActionsOvertureConflate; + "overture-restart": GetApiDataManagerOvertureRecovery200ActionsOvertureRestart; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureConflate.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureConflate.ts new file mode 100644 index 000000000..ed1ac5091 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureConflate.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery200ActionsOvertureConflate = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureExtract.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureExtract.ts new file mode 100644 index 000000000..2dffe9d9e --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureExtract.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery200ActionsOvertureExtract = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureIngest.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureIngest.ts new file mode 100644 index 000000000..86ff60803 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureIngest.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery200ActionsOvertureIngest = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOverturePull.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOverturePull.ts new file mode 100644 index 000000000..93edb9361 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOverturePull.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery200ActionsOverturePull = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureRestart.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureRestart.ts new file mode 100644 index 000000000..c1f622518 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery200ActionsOvertureRestart.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery200ActionsOvertureRestart = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery400.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery400.ts new file mode 100644 index 000000000..973a6937b --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery401.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery401.ts new file mode 100644 index 000000000..56388ee7d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery403.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery403.ts new file mode 100644 index 000000000..61cd37778 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery413.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery413.ts new file mode 100644 index 000000000..8db139b1d --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery415.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery415.ts new file mode 100644 index 000000000..ff651f419 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery429.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery429.ts new file mode 100644 index 000000000..2d6bd6650 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery500.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery500.ts new file mode 100644 index 000000000..8b8414cdc --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery503.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery503.ts new file mode 100644 index 000000000..831295e09 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecovery503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecovery503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecoveryParams.ts b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecoveryParams.ts new file mode 100644 index 000000000..500a3b9e3 --- /dev/null +++ b/packages/core/src/api/generated/admin/models/getApiDataManagerOvertureRecoveryParams.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetApiDataManagerOvertureRecoveryParams = { + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; +}; diff --git a/packages/core/src/api/generated/admin/models/index.ts b/packages/core/src/api/generated/admin/models/index.ts index b8f2445fb..5acef9e71 100644 --- a/packages/core/src/api/generated/admin/models/index.ts +++ b/packages/core/src/api/generated/admin/models/index.ts @@ -185,6 +185,22 @@ export * from "./getApiDataManagerNotablePlacesStatus413"; export * from "./getApiDataManagerNotablePlacesStatus415"; export * from "./getApiDataManagerNotablePlacesStatus429"; export * from "./getApiDataManagerNotablePlacesStatus500"; +export * from "./getApiDataManagerOvertureRecovery200"; +export * from "./getApiDataManagerOvertureRecovery200Actions"; +export * from "./getApiDataManagerOvertureRecovery200ActionsOvertureConflate"; +export * from "./getApiDataManagerOvertureRecovery200ActionsOvertureExtract"; +export * from "./getApiDataManagerOvertureRecovery200ActionsOvertureIngest"; +export * from "./getApiDataManagerOvertureRecovery200ActionsOverturePull"; +export * from "./getApiDataManagerOvertureRecovery200ActionsOvertureRestart"; +export * from "./getApiDataManagerOvertureRecovery400"; +export * from "./getApiDataManagerOvertureRecovery401"; +export * from "./getApiDataManagerOvertureRecovery403"; +export * from "./getApiDataManagerOvertureRecovery413"; +export * from "./getApiDataManagerOvertureRecovery415"; +export * from "./getApiDataManagerOvertureRecovery429"; +export * from "./getApiDataManagerOvertureRecovery500"; +export * from "./getApiDataManagerOvertureRecovery503"; +export * from "./getApiDataManagerOvertureRecoveryParams"; export * from "./getApiDataManagerOvertureStatus200"; export * from "./getApiDataManagerOvertureStatus200Phase"; export * from "./getApiDataManagerOvertureStatus200Status"; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index eaa31f9ac..d2a3f6507 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -318,6 +318,15 @@ export async function getApiDataManagerNotablePlacesStatusRequest( ): Promise { return getApiDataManagerNotablePlacesStatusBound(url, options); } +const getApiDataManagerOvertureRecoveryBound = bindAdminOperation( + "getApiDataManagerOvertureRecovery", +); +export async function getApiDataManagerOvertureRecoveryRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getApiDataManagerOvertureRecoveryBound(url, options); +} const getApiDataManagerOvertureStatusBound = bindAdminOperation("getApiDataManagerOvertureStatus"); export async function getApiDataManagerOvertureStatusRequest( url: string, diff --git a/packages/core/src/build-artifacts/child-phase.test.ts b/packages/core/src/build-artifacts/child-phase.test.ts index 91a793e45..9458ae551 100644 --- a/packages/core/src/build-artifacts/child-phase.test.ts +++ b/packages/core/src/build-artifacts/child-phase.test.ts @@ -503,7 +503,7 @@ it("binds one same-slot identity across actual filesystem processes", async () = const lines = createInterface({ input: process.stdout }), next = lines[Symbol.asyncIterator]().next(), exited = new Promise((resolve) => process.once("close", () => resolve())); - process.stdin.end(JSON.stringify({ root: f.root, execution: f.execution }) + "\n"); + process.stdin.end(`${JSON.stringify({ root: f.root, execution: f.execution })}\n`); const result = await next; await exited; lines.close(); @@ -621,7 +621,9 @@ it.each(["cell", "aggregate"] as const)( 30000, ); -async function deadlineFixture() { +async function deadlineFixture( + operation: import("../ops/contract").OpsOperation = { kind: "data.downloadFonts" }, +) { const root = mkdtempSync(join(tmpdir(), "omx-fixed-phase-ancestry-")); mkdirSync(join(root, "data"), { mode: 0o700 }); await control.provisionControlInstallation(root); @@ -636,7 +638,6 @@ async function deadlineFixture() { role: "data-manager", serviceIds: services, }), - operation = { kind: "data.downloadFonts" as const }, operationFingerprint = opsOperationFingerprint(operation), deadlineAt = Date.now() + 60000, execution: ExecutionRef = { @@ -705,6 +706,7 @@ async function deadlineFixture() { authority, execution, cliAuthority, + cliExecution: bound.execution, slot: plan.cliReservations[0]!, close() { control.closeControlInstallation(cli); @@ -774,3 +776,41 @@ it.each(["cancel-first", "grant-first"] as const)( }, 30000, ); + +it.each(["Pull", "Ingest", "Extract"] as const)( + "binds Overture %s to one original fixed CLI/data command and rejects substitution", + async (suffix) => { + const operation = { kind: `data.overture${suffix}` as const, regionId: "europe/germany" }; + const f = await deadlineFixture(operation); + try { + expect(control.fixedCliArguments(operation)).toEqual([ + "data", + `overture-${suffix.toLowerCase()}`, + "europe/germany", + ]); + const original = await control.bindControlChildPhase(f.cliAuthority, f.slot); + expect(original.execution.operationFingerprint).toBe( + control.fixedDataBodyFingerprint(control.fixedDataCommandInput(operation)), + ); + expect(() => + control.readFixedCliSpawnIntent(f.cli, f.cliExecution, [ + "data", + "overture-sync", + "europe/france", + ]), + ).toThrow(); + const issued = await control.issueControlChildPhase(f.cliAuthority, { + ...f.slot, + childPublicKey: f.dm.publicKey, + }); + control.consumeControlChildDispatch(issued.dispatchTicket); + expect(() => control.consumeControlChildDispatch(issued.dispatchTicket)).toThrow(); + expect(control.lookupControlChildPhaseAttempt(f.cliAuthority, f.slot)?.execution).toEqual( + original.execution, + ); + } finally { + f.close(); + } + }, + 30000, +); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 7a07bf988..57ca3d8f6 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -4870,6 +4870,12 @@ export function fixedCliArguments(operation: import("../ops/contract").OpsOperat return ["data", "convert", "overpass", ...(operation.regionId ? [operation.regionId] : [])]; case "data.overtureSync": return ["data", "overture-sync", operation.regionId]; + case "data.overturePull": + return ["data", "overture-pull", operation.regionId]; + case "data.overtureIngest": + return ["data", "overture-ingest", operation.regionId]; + case "data.overtureExtract": + return ["data", "overture-extract", operation.regionId]; case "data.searchIndexBuild": return ["data", "search-index", "build", operation.regionId]; case "data.notablePlacesBuild": diff --git a/packages/core/src/build-artifacts/control-child-records.ts b/packages/core/src/build-artifacts/control-child-records.ts index 6ece0c803..ecb00828b 100644 --- a/packages/core/src/build-artifacts/control-child-records.ts +++ b/packages/core/src/build-artifacts/control-child-records.ts @@ -45,6 +45,9 @@ export const childPhaseKinds: Readonly = { catalogRevisionId: "catalog_20260823", }, "data.overtureSync": { kind: "data.overtureSync", regionId: "europe/germany" }, + "data.overturePull": { kind: "data.overturePull", regionId: "europe/germany" }, + "data.overtureIngest": { kind: "data.overtureIngest", regionId: "europe/germany" }, + "data.overtureExtract": { kind: "data.overtureExtract", regionId: "europe/germany" }, "data.overtureConflate": { kind: "data.overtureConflate", regionId: "europe/germany", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 57ef71ae7..82c831197 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -125,6 +125,9 @@ export const OPS_OPERATION_KINDS = [ "data.clean", "data.generateApiKeys", "data.overtureSync", + "data.overturePull", + "data.overtureIngest", + "data.overtureExtract", "data.overtureConflate", "data.searchIndexBuild", "data.notablePlacesBuild", @@ -253,6 +256,9 @@ export const OPS_KIND_POLICIES = { "data.clean": asyncEffect("api", 10 * 60_000), "data.generateApiKeys": asyncEffect("api", 10 * 60_000), "data.overtureSync": asyncEffect("api", 24 * 3600_000), + "data.overturePull": asyncEffect("api", 24 * 3600_000), + "data.overtureIngest": asyncEffect("api", 24 * 3600_000), + "data.overtureExtract": asyncEffect("api", 24 * 3600_000), "data.overtureConflate": asyncEffect("api", 24 * 3600_000), "data.searchIndexBuild": asyncEffect("api", 24 * 3600_000), "data.notablePlacesBuild": asyncEffect("api", 24 * 3600_000), @@ -529,6 +535,9 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("data.clean"), dataTypeId: stableIdSchema }), z.strictObject({ kind: z.literal("data.generateApiKeys"), catalogRevisionId: stableIdSchema }), z.strictObject({ kind: z.literal("data.overtureSync"), regionId: regionIdSchema }), + z.strictObject({ kind: z.literal("data.overturePull"), regionId: regionIdSchema }), + z.strictObject({ kind: z.literal("data.overtureIngest"), regionId: regionIdSchema }), + z.strictObject({ kind: z.literal("data.overtureExtract"), regionId: regionIdSchema }), z.strictObject({ kind: z.literal("data.overtureConflate"), regionId: regionIdSchema, @@ -806,6 +815,9 @@ export async function authorizeOpsResources( return true; } case "data.overtureSync": + case "data.overturePull": + case "data.overtureIngest": + case "data.overtureExtract": case "data.overtureConflate": case "data.searchIndexBuild": return allowed(policy.allowRegion?.(operation.kind, operation.regionId)); @@ -1274,6 +1286,9 @@ export const OPS_RESULT_SCHEMAS = { "data.clean": data, "data.generateApiKeys": data, "data.overtureSync": data, + "data.overturePull": data, + "data.overtureIngest": data, + "data.overtureExtract": data, "data.overtureConflate": data, "data.searchIndexBuild": data, "data.notablePlacesBuild": data, diff --git a/packages/core/src/services/data-manager/contracts/operations.ts b/packages/core/src/services/data-manager/contracts/operations.ts index 558a84ae4..2098d9ae5 100644 --- a/packages/core/src/services/data-manager/contracts/operations.ts +++ b/packages/core/src/services/data-manager/contracts/operations.ts @@ -272,6 +272,17 @@ export const dataManagerOperations = { 200: ndjson(i.notablePlacesEventSchema, h.ndjsonResponseHeadersSchema), }, }, + getOvertureRecovery: { + method: "GET", + path: "/overture/recovery", + auth: service, + request: { query: regionBodySchema }, + responses: { + ...serviceErrors, + 200: json(o.overtureRecoverySchema), + 503: json(httpErrorSchema), + }, + }, getOvertureStatus: { method: "GET", path: "/overture/status", diff --git a/packages/core/src/services/data-manager/contracts/overture.ts b/packages/core/src/services/data-manager/contracts/overture.ts index 52141ac35..ed41700ff 100644 --- a/packages/core/src/services/data-manager/contracts/overture.ts +++ b/packages/core/src/services/data-manager/contracts/overture.ts @@ -103,3 +103,23 @@ export type OvertureStatus = z.infer; export type OvertureSyncResult = z.infer; export type OvertureConflateEvent = z.infer; export type OvertureConflateResult = Extract; + +const recoveryActionSchema = z.strictObject({ + allowed: z.boolean(), + reason: z.string().nullable(), +}); +export const overtureRecoverySchema = z.strictObject({ + region: z.string(), + busy: z.boolean(), + osmAvailable: z.boolean(), + downloadedRelease: z.string().nullable(), + installedRelease: z.string().nullable(), + actions: z.strictObject({ + "overture-pull": recoveryActionSchema, + "overture-ingest": recoveryActionSchema, + "overture-extract": recoveryActionSchema, + "overture-conflate": recoveryActionSchema, + "overture-restart": recoveryActionSchema, + }), +}); +export type OvertureRecoveryStatus = z.infer; diff --git a/packages/core/src/services/data-manager/generated/data-manager.generated.ts b/packages/core/src/services/data-manager/generated/data-manager.generated.ts index 91c940bac..f292d414c 100644 --- a/packages/core/src/services/data-manager/generated/data-manager.generated.ts +++ b/packages/core/src/services/data-manager/generated/data-manager.generated.ts @@ -265,6 +265,15 @@ import type { GetOfflinePackageManifest429, GetOfflinePackageManifest500, GetOfflinePackageManifest503, + GetOvertureRecovery200, + GetOvertureRecovery400, + GetOvertureRecovery401, + GetOvertureRecovery413, + GetOvertureRecovery415, + GetOvertureRecovery429, + GetOvertureRecovery500, + GetOvertureRecovery503, + GetOvertureRecoveryParams, GetOvertureStatus200, GetOvertureStatus400, GetOvertureStatus401, @@ -580,6 +589,7 @@ import { getOfflinePackageCapabilityRequest, getOfflinePackageJobRequest, getOfflinePackageManifestRequest, + getOvertureRecoveryRequest, getOvertureStatusRequest, getSearchIndexStatusRequest, getStatusRequest, @@ -4958,6 +4968,94 @@ export const pullOverture = async ( }); }; +export type getOvertureRecoveryResponse200 = { + data: GetOvertureRecovery200; + status: 200; +}; + +export type getOvertureRecoveryResponse400 = { + data: GetOvertureRecovery400; + status: 400; +}; + +export type getOvertureRecoveryResponse401 = { + data: GetOvertureRecovery401; + status: 401; +}; + +export type getOvertureRecoveryResponse413 = { + data: GetOvertureRecovery413; + status: 413; +}; + +export type getOvertureRecoveryResponse415 = { + data: GetOvertureRecovery415; + status: 415; +}; + +export type getOvertureRecoveryResponse429 = { + data: GetOvertureRecovery429; + status: 429; +}; + +export type getOvertureRecoveryResponse500 = { + data: GetOvertureRecovery500; + status: 500; +}; + +export type getOvertureRecoveryResponse503 = { + data: GetOvertureRecovery503; + status: 503; +}; + +export type getOvertureRecoveryResponseSuccess = getOvertureRecoveryResponse200 & { + headers: Headers; +}; +export type getOvertureRecoveryResponseError = ( + | getOvertureRecoveryResponse400 + | getOvertureRecoveryResponse401 + | getOvertureRecoveryResponse413 + | getOvertureRecoveryResponse415 + | getOvertureRecoveryResponse429 + | getOvertureRecoveryResponse500 + | getOvertureRecoveryResponse503 +) & { + headers: Headers; +}; + +export type getOvertureRecoveryResponse = + | getOvertureRecoveryResponseSuccess + | getOvertureRecoveryResponseError; + +export const getGetOvertureRecoveryUrl = (params: GetOvertureRecoveryParams) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 + ? `/overture/recovery?${stringifiedParams}` + : `/overture/recovery`; +}; + +export const getOvertureRecovery = async ( + params: GetOvertureRecoveryParams, + options?: Parameters[1], +): Promise => { + return getOvertureRecoveryRequest( + getGetOvertureRecoveryUrl(params), + { + ...options, + method: "GET", + }, + ); +}; + export type getOvertureStatusResponse200 = { data: GetOvertureStatus200; status: 200; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200.ts new file mode 100644 index 000000000..038335556 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { GetOvertureRecovery200Actions } from "./getOvertureRecovery200Actions"; + +export type GetOvertureRecovery200 = { + region: string; + busy: boolean; + osmAvailable: boolean; + /** @nullable */ + downloadedRelease: string | null; + /** @nullable */ + installedRelease: string | null; + actions: GetOvertureRecovery200Actions; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200Actions.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200Actions.ts new file mode 100644 index 000000000..63a76e747 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200Actions.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { GetOvertureRecovery200ActionsOvertureConflate } from "./getOvertureRecovery200ActionsOvertureConflate"; +import type { GetOvertureRecovery200ActionsOvertureExtract } from "./getOvertureRecovery200ActionsOvertureExtract"; +import type { GetOvertureRecovery200ActionsOvertureIngest } from "./getOvertureRecovery200ActionsOvertureIngest"; +import type { GetOvertureRecovery200ActionsOverturePull } from "./getOvertureRecovery200ActionsOverturePull"; +import type { GetOvertureRecovery200ActionsOvertureRestart } from "./getOvertureRecovery200ActionsOvertureRestart"; + +export type GetOvertureRecovery200Actions = { + "overture-pull": GetOvertureRecovery200ActionsOverturePull; + "overture-ingest": GetOvertureRecovery200ActionsOvertureIngest; + "overture-extract": GetOvertureRecovery200ActionsOvertureExtract; + "overture-conflate": GetOvertureRecovery200ActionsOvertureConflate; + "overture-restart": GetOvertureRecovery200ActionsOvertureRestart; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureConflate.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureConflate.ts new file mode 100644 index 000000000..4834356d5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureConflate.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery200ActionsOvertureConflate = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureExtract.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureExtract.ts new file mode 100644 index 000000000..084c2b4b8 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureExtract.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery200ActionsOvertureExtract = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureIngest.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureIngest.ts new file mode 100644 index 000000000..b212a167b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureIngest.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery200ActionsOvertureIngest = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOverturePull.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOverturePull.ts new file mode 100644 index 000000000..5046721d5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOverturePull.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery200ActionsOverturePull = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureRestart.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureRestart.ts new file mode 100644 index 000000000..b1180b1d4 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery200ActionsOvertureRestart.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery200ActionsOvertureRestart = { + allowed: boolean; + /** @nullable */ + reason: string | null; +}; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery400.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery400.ts new file mode 100644 index 000000000..892b41f0d --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery401.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery401.ts new file mode 100644 index 000000000..60aa376f9 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery413.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery413.ts new file mode 100644 index 000000000..3035973a1 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery415.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery415.ts new file mode 100644 index 000000000..2732a79bf --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery429.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery429.ts new file mode 100644 index 000000000..81bfa3a6e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery500.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery500.ts new file mode 100644 index 000000000..d16c5df35 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecovery503.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery503.ts new file mode 100644 index 000000000..016717cd3 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecovery503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecovery503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/services/data-manager/generated/models/getOvertureRecoveryParams.ts b/packages/core/src/services/data-manager/generated/models/getOvertureRecoveryParams.ts new file mode 100644 index 000000000..35656791b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/getOvertureRecoveryParams.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type GetOvertureRecoveryParams = { + /** + * @minLength 1 + * @maxLength 512 + * @pattern ^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$ + */ + region: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/index.ts b/packages/core/src/services/data-manager/generated/models/index.ts index e2830a192..6edd484c1 100644 --- a/packages/core/src/services/data-manager/generated/models/index.ts +++ b/packages/core/src/services/data-manager/generated/models/index.ts @@ -376,6 +376,21 @@ export * from "./getOfflinePackageManifest415"; export * from "./getOfflinePackageManifest429"; export * from "./getOfflinePackageManifest500"; export * from "./getOfflinePackageManifest503"; +export * from "./getOvertureRecovery200"; +export * from "./getOvertureRecovery200Actions"; +export * from "./getOvertureRecovery200ActionsOvertureConflate"; +export * from "./getOvertureRecovery200ActionsOvertureExtract"; +export * from "./getOvertureRecovery200ActionsOvertureIngest"; +export * from "./getOvertureRecovery200ActionsOverturePull"; +export * from "./getOvertureRecovery200ActionsOvertureRestart"; +export * from "./getOvertureRecovery400"; +export * from "./getOvertureRecovery401"; +export * from "./getOvertureRecovery413"; +export * from "./getOvertureRecovery415"; +export * from "./getOvertureRecovery429"; +export * from "./getOvertureRecovery500"; +export * from "./getOvertureRecovery503"; +export * from "./getOvertureRecoveryParams"; export * from "./getOvertureStatus200"; export * from "./getOvertureStatus200Phase"; export * from "./getOvertureStatus200Status"; diff --git a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts index ffa5d1f85..51910b1c9 100644 --- a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts +++ b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts @@ -198,6 +198,13 @@ export async function getOfflinePackageManifestRequest( ): Promise { return getOfflinePackageManifestBound(url, options); } +const getOvertureRecoveryBound = bindDataManagerOperation("getOvertureRecovery"); +export async function getOvertureRecoveryRequest( + url: string, + options?: DataManagerRequestOptions, +): Promise { + return getOvertureRecoveryBound(url, options); +} const getOvertureStatusBound = bindDataManagerOperation("getOvertureStatus"); export async function getOvertureStatusRequest( url: string, diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 3edc713ce..500f9dd1d 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -58,6 +58,7 @@ "services/data-manager/__tests__/offline-package-postgres-accounting.test.ts", "services/data-manager/__tests__/overture/global-conflation-postgres.test.ts", "services/data-manager/__tests__/overture/runtime-postgres.test.ts", + "services/data-manager/__tests__/overture/recovery-postgres.test.ts", "services/data-manager/__tests__/overture/schema-postgres.test.ts", "services/data-manager/__tests__/search-index/build-postgres.test.ts", "services/data-manager/__tests__/search-index/schema-postgres.test.ts", diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index bd0a11a99..4d012a9c1 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -59,6 +59,9 @@ "data.notablePlacesBuild", "data.overtureConflate", "data.overtureSync", + "data.overturePull", + "data.overtureIngest", + "data.overtureExtract", "data.searchIndexBuild", "data.update", "dawarich.provisioning.inspect", @@ -182,6 +185,9 @@ "data.link", "data.clean", "data.overtureSync", + "data.overturePull", + "data.overtureIngest", + "data.overtureExtract", "data.overtureConflate", "data.searchIndexBuild", "data.notablePlacesBuild", diff --git a/services/data-manager/__tests__/overture/local-inputs.test.ts b/services/data-manager/__tests__/overture/local-inputs.test.ts new file mode 100644 index 000000000..31163d80b --- /dev/null +++ b/services/data-manager/__tests__/overture/local-inputs.test.ts @@ -0,0 +1,93 @@ +/** Producer-contract metadata inspection, not a parquet ingestion proof. */ +import { mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { captureOvertureCatalogue } from "../../src/jobs/overture/attribution-capture"; +import { inspectOvertureLocalInputs } from "../../src/jobs/overture/local-inputs"; +import { buildOverturePullContract, pullContractPath } from "../../src/jobs/overture/stac"; + +const database = vi.hoisted(() => ({ sql: Object.assign(vi.fn(), { unsafe: vi.fn() }) })); +vi.mock("../../src/db/index", () => ({ sql: database.sql, rawSql: database.sql, db: {} })); +const roots: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function root() { + const path = mkdtempSync(join(tmpdir(), "omx-overture-inputs-")); + roots.push(path); + return path; +} +function snapshot(dataDir: string, release: string) { + const region = "europe/germany", + directory = join(dataDir, "overture", release); + mkdirSync(directory, { recursive: true }); + const file = join(directory, "europe-germany.parquet"); + writeFileSync(file, "metadata fixture, not a real parquet"); + const contract = buildOverturePullContract( + { + release, + stacVersion: "1.1.0", + collectionUrl: `https://stac.overturemaps.org/${release}/places/place/collection.json`, + assets: [ + { + itemId: "00001", + bbox: [5, 30, 20, 60], + numRows: 200, + href: `https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/${release}/theme=places/type=place/part-00001-immutable.parquet`, + }, + ], + selectedAssetRows: 200, + }, + region, + { west: 5, south: 30, east: 20, north: 60 }, + file, + { rowCount: 42, contributors: ["meta", "Overture"] }, + undefined, + captureOvertureCatalogue("2026-10-13T00:00:00.000Z"), + ); + writeFileSync(pullContractPath(dataDir, release, region), JSON.stringify(contract)); + return file; +} +it("selects the newest completed local region contract, ignoring partial pulls without creating files", () => { + const path = root(); + snapshot(path, "2026-10-01.0"); + const newest = snapshot(path, "2026-10-21.0"); + mkdirSync(join(path, "overture/2026-10-29.0")); + writeFileSync(join(path, "overture/2026-10-29.0/europe-germany.parquet"), "incomplete"); + mkdirSync(join(path, "osm")); + writeFileSync(join(path, "osm/europe-germany.osm.pbf"), "PBF fixture"); + expect(inspectOvertureLocalInputs(path, "europe/germany")).toEqual({ + downloadedRelease: "2026-10-21.0", + osmAvailable: true, + }); + expect(inspectOvertureLocalInputs(path, "europe/france")).toEqual({ + downloadedRelease: null, + osmAvailable: false, + }); + writeFileSync(newest, "changed size"); + expect(inspectOvertureLocalInputs(path, "europe/germany").downloadedRelease).toBe("2026-10-01.0"); +}); +it("does not follow a release directory symlink or accept traversal regions", () => { + const path = root(), + outside = root(); + mkdirSync(join(path, "overture")); + symlinkSync(outside, join(path, "overture/2026-10-21.0")); + expect(() => inspectOvertureLocalInputs(path, "europe/germany")).toThrow(); + expect(() => inspectOvertureLocalInputs(path, "../outside")).toThrow(); +}); +it("refuses standalone ingest without a completed local download before network or database work", async () => { + const path = root(), + fetch = vi + .spyOn(globalThis, "fetch") + .mockRejectedValue(Error("Unexpected network release discovery")); + const { ingestOverture } = await import("../../src/jobs/overture/ingest"); + await expect(ingestOverture({ dataDir: path, region: "europe/germany" })).rejects.toThrow( + /completed local Overture partition/, + ); + expect(fetch).not.toHaveBeenCalled(); + expect(database.sql.unsafe).not.toHaveBeenCalled(); + expect(readdirSync(path)).toEqual([]); +}); diff --git a/services/data-manager/__tests__/overture/recovery-postgres.test.ts b/services/data-manager/__tests__/overture/recovery-postgres.test.ts new file mode 100644 index 000000000..3d2c070fb --- /dev/null +++ b/services/data-manager/__tests__/overture/recovery-postgres.test.ts @@ -0,0 +1,145 @@ +/** Real authenticated read-only recovery inspection, owned PostgreSQL and lock. */ +import { mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Fastify from "fastify"; +import { expect, it, vi } from "vitest"; +import { createMigratedPostgresFixture } from "../../../../apps/api/src/test/postgres-fixture"; + +const state = vi.hoisted(() => ({ + fixture: null as Awaited> | null, +})); +vi.mock("../../src/db/index", () => ({ + get db() { + return state.fixture?.db; + }, + get sql() { + return state.fixture?.sql; + }, + get rawSql() { + return state.fixture?.sql; + }, +})); +it.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "reports missing inputs and the actual Overture lease without creating files", + async () => { + const fixture = await createMigratedPostgresFixture(); + state.fixture = fixture; + const root = mkdtempSync(join(tmpdir(), "omx-overture-recovery-")), + app = Fastify(); + try { + const { registerApi } = await import("../../src/api"), + { registerAuth } = await import("../../src/auth"); + const { OVERTURE_LOCK_NAMESPACE, OVERTURE_LOCK_KEY } = await import( + "../../src/jobs/overture/operation-lock" + ); + registerAuth(app, "synthetic-recovery-token"); + registerApi(app, { dataDir: root }); + const request = () => + app.inject({ + method: "GET", + url: "/overture/recovery?region=europe/germany", + headers: { authorization: "Bearer synthetic-recovery-token" }, + }); + const response = await request(); + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + region: "europe/germany", + busy: false, + downloadedRelease: null, + osmAvailable: false, + actions: { + "overture-pull": { allowed: true }, + "overture-ingest": { allowed: false }, + "overture-extract": { allowed: false }, + "overture-restart": { allowed: false }, + }, + }); + const held = await fixture.sql.reserve(); + try { + await held`SELECT pg_advisory_lock(${OVERTURE_LOCK_NAMESPACE}, ${OVERTURE_LOCK_KEY})`; + const busy = await request(); + expect(busy.statusCode).toBe(200); + expect(busy.json()).toMatchObject({ + busy: true, + actions: { + "overture-pull": { allowed: false }, + "overture-ingest": { allowed: false }, + "overture-extract": { allowed: false }, + "overture-conflate": { allowed: false }, + }, + }); + } finally { + await held`SELECT pg_advisory_unlock(${OVERTURE_LOCK_NAMESPACE}, ${OVERTURE_LOCK_KEY})`; + held.release(); + } + const { assertOvertureRecoveryInputs } = await import("../../src/jobs/overture/recovery"); + const { withOvertureOperationLock } = await import("../../src/jobs/overture/operation-lock"); + const { buildSchemaDDL } = await import("../../src/jobs/overture/schema"); + await fixture.sql.unsafe(buildSchemaDDL("overture_places")); + await fixture.sql.unsafe( + `INSERT INTO overture_places.places (gers_id, geom, release) VALUES ('published-place', ST_SetSRID(ST_MakePoint(13,52),4326), '2026-10-21.0')`, + ); + await fixture.sql.unsafe( + `INSERT INTO overture_places.conflation_state (release,region,place_count,status) VALUES ('2026-10-21.0','europe/germany',1,'pending')`, + ); + for (const kind of ["overture-ingest", "overture-extract", "overture-conflate"] as const) { + await expect( + withOvertureOperationLock(() => + assertOvertureRecoveryInputs(root, "europe/germany", kind, true), + ), + ).rejects.toThrow(/Download/); + } + await expect( + withOvertureOperationLock(() => + assertOvertureRecoveryInputs(root, "europe/france", "overture-extract"), + ), + ).rejects.toThrow(/Ingest this region/); + expect( + await fixture.sql.unsafe("SELECT gers_id, release FROM overture_places.places"), + ).toEqual([expect.objectContaining({ gers_id: "published-place", release: "2026-10-21.0" })]); + expect( + ( + await fixture.sql.unsafe( + "SELECT status, attempt_count FROM overture_places.conflation_state", + ) + )[0], + ).toMatchObject({ status: "pending", attempt_count: 0 }); + await fixture.sql.unsafe("UPDATE overture_places.conflation_state SET status='completed'"); + await expect( + withOvertureOperationLock(() => + assertOvertureRecoveryInputs(root, "europe/germany", "overture-conflate", false), + ), + ).resolves.toBeNull(); + await expect( + withOvertureOperationLock(() => + assertOvertureRecoveryInputs(root, "europe/germany", "overture-conflate", true), + ), + ).rejects.toThrow(/OSM PBF/); + const failedRead = vi + .spyOn(fixture.sql, "unsafe") + .mockRejectedValueOnce(Error("Synthetic lock-read outage")); + try { + const unavailable = await request(); + expect(unavailable.statusCode).toBe(503); + expect(unavailable.json()).toEqual({ + ok: false, + error: "Overture prerequisite inspection is unavailable", + }); + } finally { + failedRead.mockRestore(); + } + expect(readdirSync(root)).toEqual([]); + expect( + (await app.inject({ method: "GET", url: "/overture/recovery?region=europe/germany" })) + .statusCode, + ).toBe(401); + } finally { + await app.close(); + rmSync(root, { recursive: true, force: true }); + state.fixture = null; + await fixture.stop(); + } + }, + 60000, +); diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 1a2df70b3..cb5ff224f 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -39266,6 +39266,618 @@ } } }, + "/overture/recovery": { + "get": { + "operationId": "getOvertureRecovery", + "tags": [ + "data-manager" + ], + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 512, + "pattern": "^[a-z][a-z0-9_-]*(?:\\/[a-z][a-z0-9_-]*)*$" + }, + "in": "query", + "name": "region", + "required": true + } + ], + "security": [ + { + "serviceBearer": [] + }, + { + "serviceToken": [] + } + ], + "x-openmapx-contract-id": "getOvertureRecovery", + "x-openmapx-auth": "service", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "region": { + "type": "string" + }, + "busy": { + "type": "boolean" + }, + "osmAvailable": { + "type": "boolean" + }, + "downloadedRelease": { + "type": [ + "string", + "null" + ] + }, + "installedRelease": { + "type": [ + "string", + "null" + ] + }, + "actions": { + "type": "object", + "properties": { + "overture-pull": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-ingest": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-extract": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-conflate": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + }, + "overture-restart": { + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "reason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "allowed", + "reason" + ], + "additionalProperties": false + } + }, + "required": [ + "overture-pull", + "overture-ingest", + "overture-extract", + "overture-conflate", + "overture-restart" + ], + "additionalProperties": false + } + }, + "required": [ + "region", + "busy", + "osmAvailable", + "downloadedRelease", + "installedRelease", + "actions" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^[1-9]\\d*$" + }, + "description": "Required response header" + } + }, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + } + } + }, "/overture/status": { "get": { "operationId": "getOvertureStatus", diff --git a/services/data-manager/src/api.ts b/services/data-manager/src/api.ts index ae1634328..511113aa8 100644 --- a/services/data-manager/src/api.ts +++ b/services/data-manager/src/api.ts @@ -33,6 +33,7 @@ import { type NotablePlacesRuntimeState, } from "./jobs/notable-places/state.js"; import { getOvertureConflationState } from "./jobs/overture/rebuild-links.js"; +import { inspectOvertureRecovery } from "./jobs/overture/recovery.js"; import { createNotablePlacesOperationLock, type SearchIndexOperationLock, @@ -943,6 +944,16 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { await observeLegacyData(admitted, reply, "pullOverture"); }); + registerDataManagerRoute(app, "getOvertureRecovery", async (req, reply) => { + reply.header("cache-control", "no-store"); + try { + return await inspectOvertureRecovery(dataDir, req.query.region); + } catch { + reply.code(503); + return { ok: false, error: "Overture prerequisite inspection is unavailable" }; + } + }); + registerDataManagerRoute(app, "getOvertureStatus", async (_req, reply) => { const state = await getOvertureConflationState(); if (!state) { diff --git a/services/data-manager/src/jobs/execution/dispatch.ts b/services/data-manager/src/jobs/execution/dispatch.ts index 517fd3571..f89b076ed 100644 --- a/services/data-manager/src/jobs/execution/dispatch.ts +++ b/services/data-manager/src/jobs/execution/dispatch.ts @@ -24,6 +24,7 @@ import { getOvertureConflationState, rebuildOvertureLinksUnlocked, } from "../overture/rebuild-links.js"; +import { assertOvertureRecoveryInputs } from "../overture/recovery.js"; import { finalizeOvertureReleaseFiles, overtureReleaseRetentionFromEnv, @@ -273,20 +274,39 @@ export function createDataExecutionDispatcher(deps: { } } case "overture-pull": { - const path = await pullOverture({ ...common, region: input.region }); + const path = await withOvertureOperationLock(() => + pullOverture({ ...common, region: input.region }), + ); finish(context.execution.executionId, { event: "done", ok: true, path }); return { kind: input.kind, region: input.region, metrics: {} }; } case "overture-ingest": - await withOvertureOperationLock(() => ingestOverture({ ...common, region: input.region })); + await withOvertureOperationLock(async () => { + const release = await assertOvertureRecoveryInputs( + deps.dataDir, + input.region, + "overture-ingest", + ); + if (!release) throw Error("Completed local Overture release required"); + await ingestOverture({ ...common, region: input.region, release }); + }); finish(context.execution.executionId, { event: "done", ok: true }); return { kind: input.kind, region: input.region, metrics: {} }; case "overture-extract": - await withOvertureOperationLock(() => extractOsmPois({ ...common, region: input.region })); + await withOvertureOperationLock(async () => { + await assertOvertureRecoveryInputs(deps.dataDir, input.region, "overture-extract"); + await extractOsmPois({ ...common, region: input.region }); + }); finish(context.execution.executionId, { event: "done", ok: true }); return { kind: input.kind, region: input.region, metrics: {} }; case "overture-conflate": { const result = await withOvertureOperationLock(async () => { + await assertOvertureRecoveryInputs( + deps.dataDir, + input.region, + "overture-conflate", + input.restart === true, + ); const installed = await getOvertureConflationState(); if (!installed) throw Error("No installed Overture release"); const result = await rebuildOvertureLinksUnlocked({ diff --git a/services/data-manager/src/jobs/overture/ingest.ts b/services/data-manager/src/jobs/overture/ingest.ts index 88d0f81e0..337e88488 100644 --- a/services/data-manager/src/jobs/overture/ingest.ts +++ b/services/data-manager/src/jobs/overture/ingest.ts @@ -11,6 +11,7 @@ import { import { assertOverturePostgresCapacity, estimateOvertureIngestBytes } from "./capacity.js"; import { duckDbSqlLiteral, runDuckDbScript } from "./duckdb.js"; import { validateOvertureQuality } from "./eval/quality-gate.js"; +import { inspectOvertureLocalInputs } from "./local-inputs.js"; import { regionSlug, resolveOvertureRelease } from "./pull.js"; import { assertValidOvertureSchema, buildPlacesIndexesDDL, buildSchemaDDL } from "./schema.js"; import { readOverturePullContract } from "./stac.js"; @@ -138,7 +139,13 @@ export async function activateOvertureStagingSchema( } export async function ingestOverture(opts: IngestOvertureOptions): Promise { - const release = await resolveOvertureRelease(opts.release); + const localRelease = + opts.release ?? inspectOvertureLocalInputs(opts.dataDir, opts.region).downloadedRelease; + if (!localRelease) + throw Error( + "Download a completed local Overture partition for this region before ingesting it.", + ); + const release = await resolveOvertureRelease(localRelease); const slug = regionSlug(opts.region); const parquetPath = join(opts.dataDir, "overture", release, `${slug}.parquet`); const schema = "overture_places"; diff --git a/services/data-manager/src/jobs/overture/local-inputs.ts b/services/data-manager/src/jobs/overture/local-inputs.ts new file mode 100644 index 000000000..edc85f21f --- /dev/null +++ b/services/data-manager/src/jobs/overture/local-inputs.ts @@ -0,0 +1,95 @@ +/** Bounded local producer metadata. Polling never hashes/parses the parquet body. */ +import { closeSync, constants, fstatSync, lstatSync, openSync, realpathSync } from "node:fs"; +import { regionSchema } from "@openmapx/core/data-manager-contracts"; +import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { assertValidOvertureRelease, validateOverturePullContract } from "./stac.js"; + +function availableFile(directory: AnchoredDirectory, name: string): boolean { + let fd: number | undefined; + try { + fd = openSync(directory.name(name), constants.O_RDONLY | constants.O_NOFOLLOW); + const stat = fstatSync(fd); + return stat.isFile() && stat.size > 0; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } finally { + if (fd !== undefined) closeSync(fd); + } +} +export function inspectOvertureLocalInputs(dataDir: string, selectedRegion: string) { + const region = regionSchema.parse(selectedRegion), + slug = region.replaceAll("/", "-"), + root = new AnchoredDirectory(realpathSync(dataDir), undefined, false, true); + const opened: AnchoredDirectory[] = [root]; + try { + let osmAvailable = false, + downloadedRelease: string | null = null, + metadataBytes = 0; + try { + const osm = new AnchoredDirectory(root.name("osm"), root, false, true); + opened.push(osm); + osmAvailable = availableFile(osm, `${slug}.osm.pbf`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + try { + const directory = new AnchoredDirectory(root.name("overture"), root, false, true); + opened.push(directory); + const releases = directory + .listBounded(512) + .filter((name) => { + try { + assertValidOvertureRelease(name); + return true; + } catch { + return false; + } + }) + .sort((a, b) => b.localeCompare(a, "en", { numeric: true })); + for (const release of releases) { + const snapshot = new AnchoredDirectory(directory.name(release), directory, false, true); + let fd: number | undefined; + try { + const filename = `${slug}.parquet`; + if (!availableFile(snapshot, filename)) continue; + fd = openSync(snapshot.name(filename), constants.O_RDONLY | constants.O_NOFOLLOW); + const before = fstatSync(fd, { bigint: true }); + const bytes = snapshot.read(`${slug}.contract.json`, 16 * 1024 * 1024, false); + metadataBytes += bytes.length; + if (metadataBytes > 64 * 1024 * 1024) + throw Object.assign(Error("Overture inspection metadata budget exceeded"), { + code: "E2BIG", + }); + const contract = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + validateOverturePullContract(contract, dataDir, release, region); + const after = fstatSync(fd, { bigint: true }), + named = lstatSync(snapshot.name(filename), { bigint: true }); + snapshot.check(); + if ( + before.dev !== named.dev || + before.ino !== named.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs + ) + throw Error("Overture partition changed during inspection"); + downloadedRelease = release; + break; + } catch (error) { + // Incomplete/malformed pulls cannot enable ingest. No content is exposed. + const code = (error as NodeJS.ErrnoException).code; + if (code && code !== "ENOENT") throw error; + } finally { + if (fd !== undefined) closeSync(fd); + snapshot.close(); + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + return { osmAvailable, downloadedRelease }; + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} diff --git a/services/data-manager/src/jobs/overture/operation-lock.ts b/services/data-manager/src/jobs/overture/operation-lock.ts index 69dbc55dc..b31a2c073 100644 --- a/services/data-manager/src/jobs/overture/operation-lock.ts +++ b/services/data-manager/src/jobs/overture/operation-lock.ts @@ -26,3 +26,14 @@ export async function withOvertureOperationLock(operation: () => Promise): } } } + +/** Read the existing two-key lease without acquiring it or creating a worker. */ +export async function overtureOperationBusy(): Promise { + const [row] = await sql.unsafe<{ busy: boolean }[]>( + `SELECT EXISTS (SELECT 1 FROM pg_locks WHERE locktype='advisory' + AND database=(SELECT oid FROM pg_database WHERE datname=current_database()) + AND classid::bigint=$1::bigint AND objid::bigint=$2::bigint AND objsubid=2 AND granted) AS busy`, + [OVERTURE_LOCK_NAMESPACE, OVERTURE_LOCK_KEY], + ); + return row?.busy === true; +} diff --git a/services/data-manager/src/jobs/overture/recovery.ts b/services/data-manager/src/jobs/overture/recovery.ts new file mode 100644 index 000000000..b081a45bb --- /dev/null +++ b/services/data-manager/src/jobs/overture/recovery.ts @@ -0,0 +1,60 @@ +/** Readiness is advisory data; the existing worker lock and checks enforce writes. */ +import type { OvertureRecoveryStatus } from "@openmapx/core/data-manager-contracts"; +import { inspectOvertureLocalInputs } from "./local-inputs.js"; +import { overtureOperationBusy } from "./operation-lock.js"; +import { getOvertureConflationState } from "./rebuild-links.js"; +export async function inspectOvertureRecovery( + dataDir: string, + region: string, +): Promise { + const local = inspectOvertureLocalInputs(dataDir, region), + [busy, installed] = await Promise.all([overtureOperationBusy(), getOvertureConflationState()]); + const matching = installed?.region === region; + const action = (reason: string | null) => ({ + allowed: !busy && reason === null, + reason: busy ? "Another Overture operation holds the workspace lease." : reason, + }); + const needInstalled = matching ? null : "Ingest this region's Places partition first."; + const needOsm = local.osmAvailable ? null : "Download this region's OSM PBF first."; + return { + region, + busy, + ...local, + installedRelease: installed?.release ?? null, + actions: { + "overture-pull": action(null), + "overture-ingest": action( + local.downloadedRelease + ? null + : "Download a completed release partition for this region first.", + ), + "overture-extract": action(needInstalled ?? needOsm), + "overture-conflate": action( + needInstalled ?? (installed?.status === "completed" ? null : needOsm), + ), + "overture-restart": action(needInstalled ?? needOsm), + }, + }; +} +export async function assertOvertureRecoveryInputs( + dataDir: string, + region: string, + kind: "overture-ingest" | "overture-extract" | "overture-conflate", + restart = false, +): Promise { + const local = inspectOvertureLocalInputs(dataDir, region); + if (kind === "overture-ingest") { + if (!local.downloadedRelease) + throw Error("Download a completed release partition for this region before ingesting it."); + return local.downloadedRelease; + } + const installed = await getOvertureConflationState(); + if (installed?.region !== region) + throw Error("Ingest this region's Places partition before rebuilding its OSM workspace."); + if ( + !local.osmAvailable && + (kind === "overture-extract" || restart || installed.status !== "completed") + ) + throw Error("Download this region's OSM PBF before rebuilding its workspace."); + return null; +} diff --git a/services/data-manager/src/jobs/overture/stac.ts b/services/data-manager/src/jobs/overture/stac.ts index f2db72179..3bb826cc1 100644 --- a/services/data-manager/src/jobs/overture/stac.ts +++ b/services/data-manager/src/jobs/overture/stac.ts @@ -499,3 +499,5 @@ export function readOverturePullContract( } return assertPullContract(parsed, dataDir, release, region); } + +export { assertPullContract as validateOverturePullContract }; From 5c5f02a216ec197612454db9062839669be92538 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 09:14:19 +0100 Subject: [PATCH 055/135] fix(admin): derive data readiness from artifact evidence --- apps/api/openapi.json | 233 +++++++++++++-- .../__tests__/admin-services-sdk.test.ts | 21 +- .../helpers/service-admin-fixtures.ts | 16 +- apps/api/src/routes/admin-services.ts | 6 +- .../src/administrative-runtime.test.ts | 71 ++--- apps/ops-agent/src/administrative-runtime.ts | 2 +- apps/ops-agent/src/data-inventory.test.ts | 268 ++++++++++++++++++ apps/ops-agent/src/data-inventory.ts | 91 +----- apps/ops-agent/src/osm-inventory.ts | 109 +++++++ apps/ops-agent/src/prepared-inventory.ts | 246 ++++++++++++++++ .../services/DataInventorySection.test.tsx | 59 ++++ .../admin/services/DataInventorySection.tsx | 154 ++++++++++ .../admin/services/DataWorkflowsPage.tsx | 223 +-------------- .../admin/services/ServiceWorkflows.test.tsx | 78 ++++- apps/web/src/lib/admin/serviceAdmin.ts | 6 +- docs/docs/install/preparing-data.md | 20 ++ .../cli/__tests__/cli-build-artifacts.test.ts | 37 +++ .../cli/__tests__/cli-build-evidence.test.ts | 74 +++++ packages/cli/src/lib/build-artifacts.ts | 23 +- packages/cli/src/lib/service-builds.ts | 106 +++++-- .../src/api/contracts/service-admin.test.ts | 34 ++- .../src/api/contracts/service-admin/data.ts | 4 + .../src/api/contracts/service-admin/index.ts | 2 + .../admin/service-admin.generated.ts | 29 +- .../getAdminServiceData200BuildsItem.ts | 11 + ...tAdminServiceData200BuildsItemReadiness.ts | 13 + ...inServiceData200BuildsItemRuntimeImages.ts | 4 + ...Data200BuildsItemSourceFingerprintsItem.ts | 9 + .../getAdminServiceData200Osm.ts | 23 +- .../getAdminServiceData200OsmItemsItem.ts | 24 ++ ...viceData200OsmItemsItemFingerprintState.ts | 11 + .../getAdminServiceDataParams.ts | 15 + .../generated/admin/service-models/index.ts | 6 + .../core/src/build-artifacts/data-evidence.ts | 130 +++++++++ packages/core/src/build-artifacts/index.ts | 2 + packages/core/src/build-artifacts/manifest.ts | 4 + .../build-artifacts/prepared-evidence.test.ts | 50 ++++ .../src/build-artifacts/prepared-evidence.ts | 195 +++++++++++++ packages/core/src/ops/contract.test.ts | 56 +++- packages/core/src/ops/contract.ts | 56 +++- .../__tests__/osm-fingerprints.test.ts | 112 ++++++++ .../data-manager/src/jobs/download-osm.ts | 46 +-- .../data-manager/src/jobs/osm-fingerprints.ts | 44 +++ 43 files changed, 2266 insertions(+), 457 deletions(-) create mode 100644 apps/ops-agent/src/data-inventory.test.ts create mode 100644 apps/ops-agent/src/osm-inventory.ts create mode 100644 apps/ops-agent/src/prepared-inventory.ts create mode 100644 apps/web/src/components/admin/services/DataInventorySection.test.tsx create mode 100644 apps/web/src/components/admin/services/DataInventorySection.tsx create mode 100644 packages/cli/__tests__/cli-build-evidence.test.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemReadiness.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemRuntimeImages.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemSourceFingerprintsItem.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItem.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItemFingerprintState.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceDataParams.ts create mode 100644 packages/core/src/build-artifacts/data-evidence.ts create mode 100644 packages/core/src/build-artifacts/prepared-evidence.test.ts create mode 100644 packages/core/src/build-artifacts/prepared-evidence.ts create mode 100644 services/data-manager/__tests__/osm-fingerprints.test.ts create mode 100644 services/data-manager/src/jobs/osm-fingerprints.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index f3c70d5ec..375a49bae 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -49834,6 +49834,28 @@ "/api/admin/services/data": { "get": { "operationId": "getAdminServiceData", + "parameters": [ + { + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 500 + }, + "in": "query", + "name": "osmLimit", + "required": false + }, + { + "schema": { + "type": "integer", + "minimum": 0, + "maximum": 1000000 + }, + "in": "query", + "name": "osmOffset", + "required": false + } + ], "responses": { "200": { "description": "Response", @@ -49845,32 +49867,100 @@ "osm": { "type": "object", "properties": { - "found": { - "type": "boolean" - }, - "filename": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "pattern": "^[^/\\\\]+$" + "items": { + "maxItems": 500, + "type": "array", + "items": { + "type": "object", + "properties": { + "artifactId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "filename": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[^/\\\\]+$" + }, + "region": { + "anyOf": [ + { + "type": "string", + "maxLength": 512 + }, + { + "type": "null" + } + ] + }, + "sizeBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "modifiedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "sourceFingerprint": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "fingerprintState": { + "type": "string", + "enum": [ + "verified", + "missing", + "stale" + ] + } + }, + "required": [ + "artifactId", + "filename", + "region", + "sizeBytes", + "modifiedAt", + "sourceFingerprint", + "fingerprintState" + ], + "additionalProperties": false + } }, - "sizeBytes": { + "total": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, - "modifiedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 500 }, - "region": { - "type": "string", - "maxLength": 2048 + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "unavailable": { + "type": "boolean" } }, "required": [ - "found" + "items", + "total", + "limit", + "offset", + "unavailable" ], "additionalProperties": false }, @@ -49902,11 +49992,118 @@ "type": "string", "format": "date-time", "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "readiness": { + "type": "string", + "enum": [ + "ready", + "missing", + "incomplete", + "stale", + "unknown" + ] + }, + "reason": { + "anyOf": [ + { + "type": "string", + "maxLength": 2048 + }, + { + "type": "null" + } + ] + }, + "sizeBytes": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "null" + } + ] + }, + "sourceFingerprints": { + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "sha256" + ], + "additionalProperties": false + } + }, + "buildRecordVersion": { + "anyOf": [ + { + "anyOf": [ + { + "type": "number", + "enum": [ + 1 + ] + }, + { + "type": "number", + "enum": [ + 2 + ] + } + ] + }, + { + "type": "null" + } + ] + }, + "region": { + "anyOf": [ + { + "type": "string", + "maxLength": 2048 + }, + { + "type": "null" + } + ] + }, + "runtimeImages": { + "type": "object", + "propertyNames": { + "type": "string", + "maxLength": 64 + }, + "additionalProperties": { + "type": "string", + "maxLength": 2048 + } } }, "required": [ "target", - "built" + "built", + "readiness", + "reason", + "sizeBytes", + "sourceFingerprints", + "buildRecordVersion", + "region", + "runtimeImages" ], "additionalProperties": false } diff --git a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts index 92770f966..0133f678f 100644 --- a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts @@ -548,6 +548,23 @@ describe("actual service producers through final exported SDK", () => { await close(); } }); + it("forwards only bounded OSM pagination through the actual route and generated SDK", async () => { + const { options, close } = await wire(); + try { + const result = await sdk.getAdminServiceData({ osmLimit: 2, osmOffset: 50 }, options); + expect(result.status).toBe(200); + expect(execute).toHaveBeenLastCalledWith( + expect.objectContaining({ kind: "data.inspect", osmLimit: 2, osmOffset: 50 }), + expect.anything(), + ); + const before = execute.mock.calls.length; + expect((await sdk.getAdminServiceData({ osmLimit: 501 }, options)).status).toBe(400); + expect(execute.mock.calls.length).toBe(before); + } finally { + await close(); + } + }); + it("preserves complete manifests, omitted defaults, configuration, timestamps and inventory", async () => { const { options, close } = await wire(); try { @@ -577,7 +594,7 @@ describe("actual service producers through final exported SDK", () => { ], }, }); - expect(await sdk.getAdminServiceData(options)).toMatchObject({ + expect(await sdk.getAdminServiceData(undefined, options)).toMatchObject({ status: 200, data: { ...serviceDataFixture, fetchedAt: expect.any(String) }, }); @@ -929,7 +946,7 @@ it("retains structured domain failures and retained credential jobs through SDK data: { code: "DAWARICH_CREDENTIAL_MANAGED", error: expect.any(String) }, }); execute.mockRejectedValueOnce(Error("synthetic-secret must be redacted")); - const failed = await sdk.getAdminServiceData(options); + const failed = await sdk.getAdminServiceData(undefined, options); expect(failed).toMatchObject({ status: 500, data: { error: "Service administration failed" } }); expect(() => sdk.unwrapAdminResponse(failed)).toThrow("API request failed with status 500"); expect(JSON.stringify(failed)).not.toContain("synthetic-secret"); diff --git a/apps/api/src/routes/__tests__/helpers/service-admin-fixtures.ts b/apps/api/src/routes/__tests__/helpers/service-admin-fixtures.ts index b72a2412c..67426fd72 100644 --- a/apps/api/src/routes/__tests__/helpers/service-admin-fixtures.ts +++ b/apps/api/src/routes/__tests__/helpers/service-admin-fixtures.ts @@ -13,8 +13,20 @@ export const serviceSelectionFixture: AdminServiceSelection = { selectionFilePath: "/synthetic/selection.json", }; export const serviceDataFixture: OpsResultFor<"data.inspect"> = { - osm: { found: false }, - builds: [{ target: "valhalla", built: false }], + osm: { items: [], total: 0, limit: 50, offset: 0, unavailable: false }, + builds: [ + { + target: "valhalla", + built: false, + readiness: "missing", + reason: "No prepared output", + sizeBytes: null, + sourceFingerprints: [], + buildRecordVersion: null, + region: null, + runtimeImages: {}, + }, + ], motisTransitous: { configFound: false, datasetCount: 0, diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index c592524d6..79a52caf0 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -772,11 +772,11 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { }); // GET /admin/services/data — data inventory (OSM, builds, MOTIS health) - registerServiceAdminRoute(app, "getAdminServiceData", async () => { + registerServiceAdminRoute(app, "getAdminServiceData", async (request) => { const inventory = await executeAndWait( createApiOpsClient(), - { kind: "data.inspect" }, - createDurableOpsKey("admin.data.inspect", "current"), + { kind: "data.inspect", ...request.query }, + createDurableOpsKey("admin.data.inspect", JSON.stringify(request.query)), { signal: AbortSignal.timeout(30_000) }, ); return { diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index e4c344f19..550c792f6 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -457,41 +457,44 @@ describe("administrative data runtime", () => { expect(calls.flat()).not.toContain("transitous-fixed-v1"); }); - it.runIf(process.platform === "linux")( - "returns bounded data inventory from the agent-owned root without a host path", - async () => { - const rootDir = temporaryRoot(); - const dataRoot = join(rootDir, "infra", "docker", "data"); - mkdirSync(join(dataRoot, "osm"), { recursive: true }); - mkdirSync(join(dataRoot, "valhalla"), { recursive: true }); - writeFileSync(join(dataRoot, "osm", "germany-latest.osm.pbf"), "pbf", { mode: 0o600 }); - const runtime = createUnavailableRuntime(); - createAdministrativeRuntime(runtime, { - rootDir, - runFixedCli: async () => undefined, - }); + it("returns bounded data inventory from the agent-owned root without a host path", async () => { + const rootDir = temporaryRoot(); + const dataRoot = join(rootDir, "infra", "docker", "data"); + mkdirSync(join(dataRoot, "osm"), { recursive: true }); + mkdirSync(join(dataRoot, "valhalla"), { recursive: true }); + writeFileSync(join(dataRoot, "osm", "germany-latest.osm.pbf"), "pbf", { mode: 0o600 }); + const runtime = createUnavailableRuntime(); + createAdministrativeRuntime(runtime, { + rootDir, + runFixedCli: async () => undefined, + }); - const result = await dispatchOpsOperation( - runtime, - { kind: "data.inspect" }, - context("data.inspect"), - ); - expect(result.osm).toMatchObject({ - found: true, - filename: "germany-latest.osm.pbf", - sizeBytes: 3, - region: "germany", - }); - expect(result.builds).toContainEqual( - expect.objectContaining({ target: "valhalla", built: true }), - ); - expect(result.motisTransitous).toMatchObject({ - configFound: false, - capabilityState: "missing", - }); - expect(JSON.stringify(result)).not.toContain(rootDir); - }, - ); + const result = await dispatchOpsOperation( + runtime, + { kind: "data.inspect" }, + context("data.inspect"), + ); + expect(result.osm).toMatchObject({ + items: [ + { + filename: "germany-latest.osm.pbf", + sizeBytes: 3, + region: null, + fingerprintState: "missing", + }, + ], + total: 1, + unavailable: false, + }); + expect(result.builds).toContainEqual( + expect.objectContaining({ target: "valhalla", built: false, readiness: "unknown" }), + ); + expect(result.motisTransitous).toMatchObject({ + configFound: false, + capabilityState: "missing", + }); + expect(JSON.stringify(result)).not.toContain(rootDir); + }); }); describe("administrative build runtime", () => { diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index f4a73ae3b..901523e2b 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -1893,7 +1893,7 @@ export function createAdministrativeRuntime( } return { backupId: operation.backupId }; }; - runtime["data.inspect"] = async () => inspectDataInventory(options.rootDir); + runtime["data.inspect"] = async (operation) => inspectDataInventory(options.rootDir, operation); async function runMatchingDataCommand( operation: import("@openmapx/core/ops").OpsOperation, context: import("./runtime").OpsExecutionContext, diff --git a/apps/ops-agent/src/data-inventory.test.ts b/apps/ops-agent/src/data-inventory.test.ts new file mode 100644 index 000000000..93c46be1d --- /dev/null +++ b/apps/ops-agent/src/data-inventory.test.ts @@ -0,0 +1,268 @@ +/** Readiness must be metadata-backed, never directory presence. */ +import { mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { inspectDataInventory } from "./data-inventory"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function fixture() { + const root = mkdtempSync(join(tmpdir(), "omx-data-inventory-")); + roots.push(root); + const data = join(root, "infra/docker/data"); + mkdirSync(join(data, "osm"), { recursive: true }); + return { root, data }; +} +it("pages every extract with stable IDs and does not call unrecorded bytes verified", async () => { + const { root, data } = fixture(); + writeFileSync(join(data, "osm/z.osm.pbf"), "z"); + writeFileSync(join(data, "osm/a.osm.pbf"), "aa"); + const first = await inspectDataInventory(root, { osmLimit: 1, osmOffset: 0 }), + second = await inspectDataInventory(root, { osmLimit: 1, osmOffset: 1 }); + expect(first.osm).toMatchObject({ + total: 2, + offset: 0, + limit: 1, + unavailable: false, + items: [{ filename: "a.osm.pbf", fingerprintState: "missing", sourceFingerprint: null }], + }); + expect(second.osm.items[0]?.filename).toBe("z.osm.pbf"); + expect(second.osm.items[0]?.artifactId).not.toBe(first.osm.items[0]?.artifactId); +}); +it("reports empty prepared directories incomplete and database-backed stores unknown", async () => { + const { root, data } = fixture(); + for (const p of [ + "osrm-graph", + "otp-graph", + "tile-mbtiles", + "motis/live", + "nominatim", + "photon", + "overpass", + ]) + mkdirSync(join(data, p), { recursive: true }); + const inventory = await inspectDataInventory(root); + for (const target of ["osrm", "otp", "tiles", "motis"]) + expect(inventory.builds.find((x) => x.target === target)).toMatchObject({ + built: false, + readiness: "incomplete", + }); + for (const target of ["nominatim", "photon", "overpass"]) + expect(inventory.builds.find((x) => x.target === target)).toMatchObject({ + built: false, + readiness: "unknown", + }); +}); +it("reports legacy build records as unknown rather than verified readiness", async () => { + const { root, data } = fixture(); + mkdirSync(join(data, "otp-graph")); + writeFileSync(join(data, "otp-graph/graph.obj"), "graph"); + mkdirSync(join(data, ".openmapx-builds")); + writeFileSync( + join(data, ".openmapx-builds/otp.json"), + JSON.stringify({ + schemaVersion: 1, + service: "otp", + builtAt: new Date().toISOString(), + runtimeImages: { otp: "example.test/otp@sha256:" + "a".repeat(64) }, + toolImages: {}, + }), + ); + expect((await inspectDataInventory(root)).builds.find((x) => x.target === "otp")).toMatchObject({ + readiness: "unknown", + buildRecordVersion: 1, + built: false, + }); +}); +it("surfaces unsafe OSM directories as unavailable, without exposing host paths", async () => { + const { root, data } = fixture(); + renameSync(join(data, "osm"), join(data, "saved")); + writeFileSync(join(data, "osm"), "not a directory"); + const result = await inspectDataInventory(root); + expect(result.osm).toMatchObject({ unavailable: true, items: [] }); + expect(JSON.stringify(result)).not.toContain(root); +}); +it.each(["output", "source"] as const)( + "invalidates same-size %s replacement without hashing on poll", + async (changed) => { + const { root, data } = fixture(); + const { capturePreparedBuildEvidence, hashDataFile, osmArtifactId } = await import( + "@openmapx/core/build-artifacts" + ); + mkdirSync(join(data, "osrm-graph")); + for (const file of [ + "region.osrm", + "region.osrm.partition", + "region.osrm.cells", + "region.osrm.mldgr", + ]) + writeFileSync(join(data, "osrm-graph", file), "graph bytes"); + const path = "osm/europe-germany.osm.pbf"; + writeFileSync(join(data, path), "pbf bytes"); + const source = await hashDataFile(join(data, path)); + const record = await capturePreparedBuildEvidence({ + dataRoot: data, + record: { + schemaVersion: 2, + service: "osrm", + builtAt: new Date().toISOString(), + region: "europe/germany", + sourcePbf: { name: "europe-germany.osm.pbf", sizeBytes: source.sizeBytes }, + runtimeImages: { osrm: "example.test/osrm@sha256:" + "a".repeat(64) }, + toolImages: {}, + sourceFingerprints: [ + { + id: osmArtifactId("europe-germany.osm.pbf"), + sha256: source.sha256, + sizeBytes: source.sizeBytes, + }, + ], + artifactFingerprint: "0".repeat(64), + }, + inputs: [ + { + path, + sha256: source.sha256, + device: source.device, + inode: source.inode, + sizeBytes: source.sizeBytes, + modifiedAtMs: source.modifiedAtMs, + changedAtMs: source.changedAtMs, + }, + ], + }); + writeFileSync(join(data, ".openmapx-builds/osrm.json"), JSON.stringify(record)); + expect( + (await inspectDataInventory(root)).builds.find((x) => x.target === "osrm"), + ).toMatchObject({ + built: true, + readiness: "ready", + buildRecordVersion: 2, + region: "europe/germany", + }); + const replaced = join(data, changed === "output" ? "osrm-graph/region.osrm" : path), + bytes = changed === "output" ? "other bytes" : "new bytes"; + writeFileSync(replaced + ".replacement", bytes); + renameSync(replaced + ".replacement", replaced); + expect( + (await inspectDataInventory(root)).builds.find((x) => x.target === "osrm"), + ).toMatchObject({ built: false, readiness: "stale" }); + }, +); +it("invalidates a download fingerprint after same-size inode replacement", async () => { + const { root, data } = fixture(); + const { fileIdentity, osmArtifactId } = await import("@openmapx/core/build-artifacts"); + const { lstatSync } = await import("node:fs"); + const name = "europe-germany.osm.pbf", + path = join(data, "osm", name); + writeFileSync(path, "before"); + writeFileSync( + path + ".openmapx.json", + JSON.stringify({ + schemaVersion: 1, + artifactId: osmArtifactId(name), + region: "europe/germany", + sha256: "c".repeat(64), + ...fileIdentity(lstatSync(path, { bigint: true })), + }), + ); + expect((await inspectDataInventory(root)).osm.items[0]).toMatchObject({ + fingerprintState: "verified", + sourceFingerprint: "c".repeat(64), + }); + writeFileSync(path + ".replacement", "after!"); + renameSync(path + ".replacement", path); + expect((await inspectDataInventory(root)).osm.items[0]).toMatchObject({ + fingerprintState: "stale", + sourceFingerprint: null, + }); +}); +it("does not silently truncate extracts at the previous first page", async () => { + const { root, data } = fixture(); + for (let i = 0; i < 501; i++) + writeFileSync(join(data, "osm", `${String(i).padStart(4, "0")}.osm.pbf`), "x"); + const page = await inspectDataInventory(root, { osmOffset: 500, osmLimit: 50 }); + expect(page.osm).toMatchObject({ total: 501, offset: 500, unavailable: false }); + expect(page.osm.items.map((x) => x.filename)).toEqual(["0500.osm.pbf"]); +}); +it("checks changed source evidence even outside the displayed OSM page", async () => { + const { root, data } = fixture(); + const { capturePreparedBuildEvidence, fileIdentity, osmArtifactId } = await import( + "@openmapx/core/build-artifacts" + ); + const { lstatSync } = await import("node:fs"); + mkdirSync(join(data, "otp-graph")); + writeFileSync(join(data, "otp-graph/graph.obj"), "graph"); + for (let i = 0; i < 50; i++) writeFileSync(join(data, "osm", `a${i}.osm.pbf`), "x"); + const name = "zz.osm.pbf", + path = join(data, "osm", name); + writeFileSync(path, "new source"); + writeFileSync( + path + ".openmapx.json", + JSON.stringify({ + schemaVersion: 1, + artifactId: osmArtifactId(name), + region: "zz", + sha256: "d".repeat(64), + ...fileIdentity(lstatSync(path, { bigint: true })), + }), + ); + const record = await capturePreparedBuildEvidence({ + dataRoot: data, + record: { + schemaVersion: 2, + service: "otp", + builtAt: new Date().toISOString(), + sourcePbf: { name, sizeBytes: 10 }, + runtimeImages: { otp: "example.test/otp" }, + toolImages: {}, + sourceFingerprints: [{ id: osmArtifactId(name), sha256: "e".repeat(64), sizeBytes: 10 }], + artifactFingerprint: "0".repeat(64), + }, + }); + writeFileSync(join(data, ".openmapx-builds/otp.json"), JSON.stringify(record)); + const inventory = await inspectDataInventory(root); + expect(inventory.osm.items).toHaveLength(50); + expect(inventory.osm.items.some((item) => item.filename === name)).toBe(false); + expect(inventory.builds.find((item) => item.target === "otp")).toMatchObject({ + built: false, + readiness: "stale", + }); +}); +it("reports a disappeared listed extract as unavailable, not an empty successful scan", async () => { + const { root, data } = fixture(), + file = join(data, "osm/vanishing.osm.pbf"); + writeFileSync(file, "bytes"); + const { AnchoredDirectory } = await import( + "../../../packages/core/src/build-artifacts/filesystem" + ); + const { unlinkSync } = await import("node:fs"); + const { vi } = await import("vitest"); + const list = AnchoredDirectory.prototype.listBounded; + const spy = vi.spyOn(AnchoredDirectory.prototype, "listBounded").mockImplementation(function ( + this: InstanceType, + maximum: number, + ) { + const names = list.call(this, maximum); + if (this.path.endsWith("/osm") && names.includes("vanishing.osm.pbf")) unlinkSync(file); + return names; + }); + try { + expect((await inspectDataInventory(root)).osm).toMatchObject({ unavailable: true, items: [] }); + } finally { + spy.mockRestore(); + } +}); +it("preserves the legacy top-level extract fallback when managed OSM storage is absent", async () => { + const { root, data } = fixture(); + rmSync(join(data, "osm"), { recursive: true }); + writeFileSync(join(data, "legacy-latest.osm.pbf"), "legacy"); + expect((await inspectDataInventory(root)).osm).toMatchObject({ + unavailable: false, + total: 1, + items: [{ filename: "legacy-latest.osm.pbf", fingerprintState: "missing", region: null }], + }); +}); diff --git a/apps/ops-agent/src/data-inventory.ts b/apps/ops-agent/src/data-inventory.ts index 51bb58312..26a5cc699 100644 --- a/apps/ops-agent/src/data-inventory.ts +++ b/apps/ops-agent/src/data-inventory.ts @@ -6,11 +6,9 @@ import { type MotisConfigExpectations, parseMotisConfigExpectations, } from "@openmapx/transitous-core"; -import { - listDescriptorAnchoredDirectory, - readDescriptorAnchoredUtf8, - statDescriptorAnchoredEntry, -} from "./descriptor-file"; +import { readDescriptorAnchoredUtf8 } from "./descriptor-file"; +import { inspectOsmInventory } from "./osm-inventory"; +import { inspectPreparedInventory } from "./prepared-inventory"; const MAX_STATUS_FILE_BYTES = 1024 * 1024; const MAX_DIRECTORY_ENTRIES = 1024; @@ -20,19 +18,6 @@ const MAX_TEXT = 2_048; type DataInventory = OpsResultFor<"data.inspect">; type MotisInventory = DataInventory["motisTransitous"]; -const BUILT_PRODUCT_DIRS = { - valhalla: "valhalla", - osrm: "osrm-graph", - otp: "otp-graph", - motis: "motis/live", - motisFeedProxy: "motis-feed-proxy", - tiles: "tile-mbtiles", - pelias: "pelias", - nominatim: "nominatim", - photon: "photon", - overpass: "overpass", -} as const; - interface MobilityCapabilitySnapshot { schemaVersion: 1; testedAt: string; @@ -107,66 +92,6 @@ function countBoundedStrings(value: unknown): number { .length; } -function inspectOsm(dataRoot: string): DataInventory["osm"] { - const osmDir = join(dataRoot, "osm"); - const directory = isDirectory(osmDir) ? osmDir : dataRoot; - if (!isDirectory(directory)) return { found: false }; - let entries: ReturnType; - try { - const child = relative(dataRoot, directory); - entries = listDescriptorAnchoredDirectory(dataRoot, child ? child.split(sep) : [], { - maximumEntries: MAX_DIRECTORY_ENTRIES, - }); - } catch { - return { found: false }; - } - const name = entries - .filter( - (entry) => - entry.type === "file" && entry.name.length <= 255 && entry.name.endsWith(".osm.pbf"), - ) - .map((entry) => entry.name) - .sort((left, right) => left.localeCompare(right))[0]; - if (!name) return { found: false }; - // Read the metadata through the same descriptor-anchored walk that chose the - // name. Reopening `join(directory, name)` would let a rename between - // enumeration and stat fabricate the reported size and timestamp. - const child = relative(dataRoot, directory); - let metadata: ReturnType; - try { - metadata = statDescriptorAnchoredEntry(dataRoot, [...(child ? child.split(sep) : []), name]); - } catch { - return { found: false }; - } - if (metadata.type !== "file") return { found: false }; - return { - found: true, - filename: name, - sizeBytes: metadata.sizeBytes, - modifiedAt: new Date(metadata.modifiedAtMs).toISOString(), - region: name - .replace(/-latest\.osm\.pbf$/, "") - .replace(/\.osm\.pbf$/, "") - .slice(0, MAX_TEXT), - }; -} - -function inspectBuilds(dataRoot: string): DataInventory["builds"] { - return Object.entries(BUILT_PRODUCT_DIRS).map(([target, relativePath]) => { - const key = target as keyof typeof BUILT_PRODUCT_DIRS; - try { - // Descriptor-anchored so that a renamed or hardlinked path cannot - // fabricate a built status or timestamp. - const metadata = statDescriptorAnchoredEntry(dataRoot, relativePath.split("/")); - return metadata.type === "directory" - ? { target: key, built: true, builtAt: new Date(metadata.modifiedAtMs).toISOString() } - : { target: key, built: false }; - } catch { - return { target: key, built: false }; - } - }); -} - function missingGbfsCatalog(): MotisInventory["gbfsCatalog"] { return { state: "missing", @@ -489,12 +414,16 @@ async function inspectMotis( }; } -export async function inspectDataInventory(rootDir: string): Promise { +export async function inspectDataInventory( + rootDir: string, + options: { osmLimit?: number; osmOffset?: number } = {}, +): Promise { const infraDir = join(rootDir, "infra", "docker"); const dataRoot = join(infraDir, "data"); + const osm = inspectOsmInventory(dataRoot, options); return { - osm: inspectOsm(dataRoot), - builds: inspectBuilds(dataRoot), + osm, + builds: inspectPreparedInventory(dataRoot, osm), motisTransitous: await inspectMotis(rootDir, infraDir, dataRoot), }; } diff --git a/apps/ops-agent/src/osm-inventory.ts b/apps/ops-agent/src/osm-inventory.ts new file mode 100644 index 000000000..88c422dc0 --- /dev/null +++ b/apps/ops-agent/src/osm-inventory.ts @@ -0,0 +1,109 @@ +import { + inspectDataFile, + osmArtifactId, + osmSourceEvidenceSchema, + sameDataFile, + withDataDirectory, +} from "@openmapx/core/build-artifacts"; +import type { OpsResultFor } from "@openmapx/core/ops"; + +type Inventory = OpsResultFor<"data.inspect">["osm"]; +export function inspectOsmInventory( + dataRoot: string, + options: { osmLimit?: number; osmOffset?: number } = {}, +): Inventory { + const limit = options.osmLimit ?? 50, + offset = options.osmOffset ?? 0; + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 500 || + !Number.isSafeInteger(offset) || + offset < 0 || + offset > 1000000 + ) + throw Error("Invalid OSM inventory page"); + const empty = (unavailable: boolean): Inventory => ({ + items: [], + total: 0, + limit, + offset, + unavailable, + }); + let enumerating = false; + const read = (location: string): Inventory => + withDataDirectory(dataRoot, location, (directory) => { + enumerating = true; + const files = directory + .listBounded(10000) + .filter((name) => name.length <= 255 && name.endsWith(".osm.pbf")) + .sort(); + const items = files.slice(offset, offset + limit).map((filename) => { + const metadata = inspectDataFile(dataRoot, location ? `${location}/${filename}` : filename); + let fingerprintState: "verified" | "missing" | "stale" = "missing", + sourceFingerprint: string | null = null, + region: string | null = null; + let bytes: Buffer | undefined; + try { + if (Buffer.byteLength(`${filename}.openmapx.json`) <= 255) + bytes = directory.read(`${filename}.openmapx.json`, 16384, false); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + try { + if (!bytes) + return { + artifactId: osmArtifactId(filename), + filename, + region, + sizeBytes: metadata.sizeBytes, + modifiedAt: new Date(metadata.modifiedAtMs).toISOString(), + sourceFingerprint, + fingerprintState, + }; + const evidence = osmSourceEvidenceSchema.parse(JSON.parse(bytes.toString("utf8"))); + fingerprintState = "stale"; + if ( + evidence.artifactId === osmArtifactId(filename) && + filename === `${evidence.region.replaceAll("/", "-")}.osm.pbf` && + sameDataFile(metadata, { + device: evidence.device, + inode: evidence.inode, + sizeBytes: evidence.sizeBytes, + modifiedAtMs: evidence.modifiedAtMs, + changedAtMs: evidence.changedAtMs, + }) + ) { + fingerprintState = "verified"; + sourceFingerprint = evidence.sha256; + region = evidence.region; + } + } catch (error) { + fingerprintState = "stale"; + } + return { + artifactId: osmArtifactId(filename), + filename, + region, + sizeBytes: metadata.sizeBytes, + modifiedAt: new Date(metadata.modifiedAtMs).toISOString(), + sourceFingerprint, + fingerprintState, + }; + }); + directory.check(); + return { items, total: files.length, limit, offset, unavailable: false }; + }); + try { + return read("osm"); + } catch (error) { + if (!enumerating && (error as NodeJS.ErrnoException).code === "ENOENT") { + try { + return read(""); + } catch (fallback) { + return empty(enumerating || (fallback as NodeJS.ErrnoException).code !== "ENOENT"); + } + } + return empty(true); + } +} diff --git a/apps/ops-agent/src/prepared-inventory.ts b/apps/ops-agent/src/prepared-inventory.ts new file mode 100644 index 000000000..ccaaf13d3 --- /dev/null +++ b/apps/ops-agent/src/prepared-inventory.ts @@ -0,0 +1,246 @@ +import { + artifactFingerprint, + BUILD_RECORD_DIR, + buildArtifactSpec, + buildRecordSchema, + incompletePreparedLayout, + inspectDataFile, + osmArtifactId, + PREPARED_EVIDENCE_BYTES, + preparedArtifactPath, + preparedBuildEvidenceSchema, + sameDataFile, + verifyPreparedMotisManifest, + withDataDirectory, +} from "@openmapx/core/build-artifacts"; +import type { OpsResultFor } from "@openmapx/core/ops"; +import { inspectOsmInventory } from "./osm-inventory"; + +type Build = OpsResultFor<"data.inspect">["builds"][number]; +const targets = { + valhalla: ["valhalla", null], + osrm: ["osrm-graph", "osrm"], + otp: ["otp-graph", "otp"], + motis: ["motis/live", "motis"], + motisFeedProxy: ["motis-feed-proxy", null], + tiles: ["tile-mbtiles", "tileserver"], + pelias: ["pelias", "pelias"], + nominatim: ["nominatim", null], + photon: ["photon", null], + overpass: ["overpass", null], +} as const; +function readRecord(dataRoot: string, name: string, max = 16384) { + return withDataDirectory(dataRoot, BUILD_RECORD_DIR, (dir) => + JSON.parse(dir.read(name, max, false).toString("utf8")), + ); +} +export function inspectPreparedInventory( + dataRoot: string, + osm: OpsResultFor<"data.inspect">["osm"], +): Build[] { + return Object.entries(targets).map(([target, [path, service]]) => { + const row: Build = { + target: target as Build["target"], + built: false, + readiness: "missing", + reason: "No prepared output directory.", + sizeBytes: null, + sourceFingerprints: [], + buildRecordVersion: null, + region: null, + runtimeImages: {}, + }; + if ( + !service || + ["pelias", "nominatim", "photon", "overpass", "valhalla", "motisFeedProxy"].includes(target) + ) { + row.readiness = "unknown"; + row.reason = + target === "valhalla" + ? "Valhalla is maintained by its runtime; dataset metadata alone does not verify routing readiness." + : "Authoritative runtime or database readiness evidence is required."; + return row; + } + try { + const physical = preparedArtifactPath(dataRoot, path); + withDataDirectory(dataRoot, physical, (dir) => dir.check()); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + row.readiness = "unknown"; + row.reason = "Prepared output could not be inspected safely."; + } + return row; + } + row.readiness = "incomplete"; + row.reason = "Prepared output has no verified build evidence."; + try { + const record = buildRecordSchema.parse(readRecord(dataRoot, `${service}.json`)); + if (record.service !== service) throw Error("Build service mismatch"); + row.buildRecordVersion = record.schemaVersion; + if ( + (record.region?.length ?? 0) > 2048 || + Object.values(record.runtimeImages).some((image) => image.length > 2048) || + (record.schemaVersion === 2 && + record.sourceFingerprints.some((source) => source.id.length > 2048)) + ) { + row.readiness = "unknown"; + row.reason = "Build metadata exceeds the supported status field limits."; + return row; + } + row.region = record.region ?? null; + row.runtimeImages = record.runtimeImages; + row.builtAt = record.builtAt; + if (record.schemaVersion === 1) { + row.readiness = "unknown"; + row.reason = + "Legacy build record has no source fingerprints; rebuild or import verified data."; + return row; + } + row.sourceFingerprints = record.sourceFingerprints.map(({ id, sha256 }) => ({ id, sha256 })); + if ( + record.schemaVersion === 2 && + new Set(record.sourceFingerprints.map((source) => source.id)).size !== + record.sourceFingerprints.length + ) + throw Error("Duplicate source fingerprint identity"); + const evidence = preparedBuildEvidenceSchema.parse( + readRecord(dataRoot, `${service}.evidence.json`, PREPARED_EVIDENCE_BYTES), + ); + if (new Set(evidence.files.map((file) => file.path)).size !== evidence.files.length) + throw Error("Duplicate recorded output path"); + if ( + JSON.stringify(evidence.record) !== JSON.stringify(record) || + record.artifactFingerprint !== artifactFingerprint(evidence.files) + ) + throw Error("Build evidence mismatch"); + if (service === "motis") { + const valid = withDataDirectory( + dataRoot, + preparedArtifactPath(dataRoot, "motis/live"), + (dir) => + verifyPreparedMotisManifest( + JSON.parse( + dir.read("motis-candidate-manifest.json", 1024 * 1024, false).toString("utf8"), + ), + evidence.files, + ), + ); + if (!valid) { + row.reason = "MOTIS candidate manifest does not match recorded artifacts."; + return row; + } + } + const missing = incompletePreparedLayout(service, evidence.files); + if (missing) { + row.reason = missing; + return row; + } + const layout = buildArtifactSpec(service); + if ( + evidence.files.some( + (file) => !layout.dirs.some((dir) => file.path.startsWith(dir + "/")), + ) || + evidence.inputs.some( + (file) => + file.path !== `osm/${record.sourcePbf?.name}` || + !record.sourceFingerprints.some( + (source) => + source.id === osmArtifactId(record.sourcePbf?.name ?? "") && + source.sha256 === file.sha256, + ), + ) + ) + throw Error("Unexpected prepared evidence layout"); + for (const file of evidence.files) { + const now = inspectDataFile(dataRoot, preparedArtifactPath(dataRoot, file.path)); + if ( + !sameDataFile(now, { + device: file.device, + inode: file.inode, + sizeBytes: file.sizeBytes, + modifiedAtMs: file.modifiedAtMs, + changedAtMs: file.changedAtMs, + }) + ) { + row.readiness = "stale"; + row.reason = "Prepared output or its source changed since the build or import."; + return row; + } + } + for (const input of evidence.inputs) { + try { + if ( + !sameDataFile(inspectDataFile(dataRoot, input.path), { + device: input.device, + inode: input.inode, + sizeBytes: input.sizeBytes, + modifiedAtMs: input.modifiedAtMs, + changedAtMs: input.changedAtMs, + }) + ) { + row.readiness = "stale"; + row.reason = "The source extract changed since this build."; + return row; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + row.sizeBytes = evidence.files.reduce((sum, file) => sum + file.sizeBytes, 0); + if (!Number.isSafeInteger(row.sizeBytes)) + throw Error("Prepared size exceeds supported inventory"); + if (!record.sourceFingerprints.length || !Object.keys(record.runtimeImages).length) { + row.readiness = "unknown"; + row.reason = "Source fingerprints and runtime image bindings are required."; + return row; + } + let local = osm; + if (record.sourcePbf && !osm.items.some((item) => item.filename === record.sourcePbf?.name)) { + try { + const index = withDataDirectory(dataRoot, "osm", (directory) => + directory + .listBounded(10000) + .filter((name) => name.endsWith(".osm.pbf")) + .sort() + .indexOf(record.sourcePbf?.name ?? ""), + ); + if (index >= 0) local = inspectOsmInventory(dataRoot, { osmLimit: 1, osmOffset: index }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + if (local.unavailable) { + row.readiness = "unknown"; + row.reason = "The source inventory is unavailable."; + return row; + } + for (const source of record.sourceFingerprints) { + const current = local.items.find((item) => item.artifactId === source.id); + if ( + current && + current.fingerprintState === "verified" && + current.sourceFingerprint !== source.sha256 + ) { + row.readiness = "stale"; + row.reason = "The source extract has changed since this build."; + return row; + } + if (current && current.fingerprintState !== "verified" && !evidence.inputs.length) { + row.readiness = current.fingerprintState === "stale" ? "stale" : "unknown"; + row.reason = "Current source evidence is missing or stale."; + return row; + } + } + row.readiness = "ready"; + row.reason = null; + row.built = true; + return row; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + row.readiness = "unknown"; + row.reason = "Build metadata or output could not be verified safely."; + } + return row; + } + }); +} diff --git a/apps/web/src/components/admin/services/DataInventorySection.test.tsx b/apps/web/src/components/admin/services/DataInventorySection.test.tsx new file mode 100644 index 000000000..5e676d594 --- /dev/null +++ b/apps/web/src/components/admin/services/DataInventorySection.test.tsx @@ -0,0 +1,59 @@ +import type { AdminServiceData } from "@openmapx/core/api-contracts"; +import { afterEach, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen } from "@/test"; +import { DataInventorySection } from "./DataInventorySection"; + +afterEach(cleanup); +const osm: AdminServiceData["osm"] = { + items: [ + { + artifactId: "a".repeat(64), + filename: "europe-germany.osm.pbf", + region: "europe/germany", + sizeBytes: 1024, + modifiedAt: "2026-10-29T08:00:00Z", + sourceFingerprint: "b".repeat(64), + fingerprintState: "verified", + }, + ], + limit: 1, + offset: 0, + total: 2, + unavailable: false, +}; +const builds: AdminServiceData["builds"] = [ + { + target: "osrm", + built: false, + readiness: "stale", + reason: "The source extract changed.", + sizeBytes: 1024, + sourceFingerprints: [{ id: "a".repeat(64), sha256: "b".repeat(64) }], + buildRecordVersion: 2, + region: "europe/germany", + runtimeImages: { osrm: "example.test/osrm@sha256:" + "c".repeat(64) }, + }, +]; +it("shows all source/readiness evidence and pages extracts", () => { + const page = vi.fn(); + render(); + expect(screen.getByText("europe-germany.osm.pbf")).toBeInTheDocument(); + expect(screen.getByText("The source extract changed.")).toBeInTheDocument(); + expect(screen.getByText(/example.test\/osrm/)).toBeInTheDocument(); + expect(screen.getByText(/Build record v2/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Next extracts" })); + expect(page).toHaveBeenCalledWith(1); + expect(screen.getByRole("button", { name: "Previous extracts" })).toBeDisabled(); +}); +it("does not present an unavailable scan as empty or ready", () => { + render( + , + ); + expect(screen.getByRole("alert")).toHaveTextContent(/could not be inspected completely/); + expect(screen.queryByText("No OSM extracts are available.")).toBeNull(); + expect(screen.queryByRole("button", { name: "Next extracts" })).toBeNull(); +}); diff --git a/apps/web/src/components/admin/services/DataInventorySection.tsx b/apps/web/src/components/admin/services/DataInventorySection.tsx new file mode 100644 index 000000000..226f300d7 --- /dev/null +++ b/apps/web/src/components/admin/services/DataInventorySection.tsx @@ -0,0 +1,154 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Box from "@mui/material/Box"; +import Button from "@mui/material/Button"; +import Chip from "@mui/material/Chip"; +import Paper from "@mui/material/Paper"; +import Stack from "@mui/material/Stack"; +import Typography from "@mui/material/Typography"; +import type { AdminServiceData } from "@openmapx/core/api-contracts"; +import { formatBytes } from "@/lib/storageFormat"; + +const targetLabels: Record = { + valhalla: "Valhalla", + osrm: "OSRM", + otp: "OpenTripPlanner", + motis: "MOTIS", + motisFeedProxy: "MOTIS feed proxy", + tiles: "Tile server", + pelias: "Pelias", + nominatim: "Nominatim", + photon: "Photon", + overpass: "Overpass", +}; +export function DataInventorySection({ + osm, + builds, + onPageChange, +}: { + osm: AdminServiceData["osm"]; + builds: AdminServiceData["builds"]; + onPageChange: (offset: number) => void; +}) { + return ( + + + + OSM extracts + + + Download and build workflows record fingerprints. Refreshing inventory never hashes + dataset contents. + + {osm.unavailable ? ( + + The OSM inventory could not be inspected completely. Restore access before relying on + readiness. + + ) : ( + <> + {osm.total === 0 ? ( + + No OSM extracts are available. + + ) : ( + osm.items.map((item) => ( + + + {item.filename} + + + + {item.region ?? "Region not recorded"} ·{" "} + {formatBytes(item.sizeBytes)} · {new Date(item.modifiedAt).toLocaleString()} + + + SHA-256: {item.sourceFingerprint ?? "No current fingerprint evidence"} + + + )) + )} + + + + {osm.items.length + ? `${osm.offset + 1}–${Math.min(osm.offset + osm.items.length, osm.total)} of ${osm.total}` + : `0 on this page (${osm.total} total)`} + + + + + )} + + + + Prepared service data + + + Ready requires recorded source and output evidence. Legacy records and database-backed + services require their authoritative checks. + + {builds.map((build) => ( + + + + {targetLabels[build.target] ?? build.target} + + + + + {build.reason ?? "Recorded outputs and source evidence are current."} + + + {build.region ?? "Region not recorded"} ·{" "} + {build.sizeBytes === null ? "Size unknown" : formatBytes(build.sizeBytes)} ·{" "} + {build.buildRecordVersion === null + ? "No build record" + : `Build record v${build.buildRecordVersion}`} + + {Object.entries(build.runtimeImages).map(([id, image]) => ( + + {id}: {image} + + ))} + {build.sourceFingerprints.map((source) => ( + + Source {source.id}: {source.sha256} + + ))} + + ))} + + + ); +} diff --git a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx index 3abc22ba2..c37935e07 100644 --- a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx +++ b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx @@ -1,15 +1,11 @@ "use client"; import BuildIcon from "@mui/icons-material/Build"; -import CheckCircleIcon from "@mui/icons-material/CheckCircle"; -import HelpOutlineIcon from "@mui/icons-material/HelpOutlined"; import RefreshIcon from "@mui/icons-material/Refresh"; import StorageIcon from "@mui/icons-material/Storage"; import Alert from "@mui/material/Alert"; import Box from "@mui/material/Box"; import Button from "@mui/material/Button"; -import Card from "@mui/material/Card"; -import CardContent from "@mui/material/CardContent"; import Chip from "@mui/material/Chip"; import CircularProgress from "@mui/material/CircularProgress"; import FormControlLabel from "@mui/material/FormControlLabel"; @@ -40,13 +36,12 @@ import type { AdminOperationContract } from "../operations/adminOperationsApi"; import { AdminPageHeader } from "../shared/AdminPageHeader"; import { useAdminToast } from "../shared/AdminToast"; import { AmbientPlacesMaintenance } from "./AmbientPlacesMaintenance"; +import { DataInventorySection } from "./DataInventorySection"; import { NotablePlacesMaintenance, notablePlacesStatusQueryKey } from "./NotablePlacesMaintenance"; import { OvertureMaintenance, overtureStatusQueryKey } from "./OvertureMaintenance"; import { SearchIndexMaintenance, searchIndexStatusQueryKey } from "./SearchIndexMaintenance"; import { TransitSourcesSection } from "./TransitSourcesSection"; -type OsmInfo = AdminServiceData["osm"]; -type BuildStatus = AdminServiceData["builds"][number]; type MotisTransitousStatus = AdminServiceData["motisTransitous"]; /** Overture and search operations have their own maintenance sections below. */ @@ -66,19 +61,6 @@ function formatDate(iso: string): string { }); } -const BUILD_LABELS: Record = { - valhalla: "Valhalla", - osrm: "OSRM", - otp: "OpenTripPlanner", - motis: "MOTIS", - motisFeedProxy: "MOTIS Feed Proxy", - tiles: "Tile Server", - pelias: "Pelias", - nominatim: "Nominatim", - photon: "Photon", - overpass: "Overpass", -}; - function DataOperationsSection({ apiUrl }: { apiUrl: string }) { const showToast = useAdminToast(); const { client } = useActivityClient(); @@ -174,203 +156,6 @@ function DataOperationsSection({ apiUrl }: { apiUrl: string }) { ); } -function OsmSection({ osm }: { osm: OsmInfo }) { - return ( - - - - - OSM Planet Data - - - {osm.found ? ( - - - - - File - - - {osm.filename} - - - {osm.region && ( - - - Region - - {osm.region} - - )} - {osm.sizeBytes && ( - - - Size - - {formatBytes(osm.sizeBytes)} - - )} - {osm.modifiedAt && ( - - - Downloaded - - {formatDate(osm.modifiedAt)} - - )} - - } - label="OSM PBF present" - color="success" - size="small" - sx={{ width: "fit-content" }} - /> - - ) : ( - - No OSM PBF file found in the data directory. Queue an OSM download above. - - )} - - ); -} - -function BuildsSection({ builds }: { builds: BuildStatus[] }) { - return ( - - - - - Service Build Inventory - - - Read-only view of which services have populated their data dirs - - - - {builds.map((b) => ( - - - - - {b.built ? ( - - ) : ( - - )} - - {BUILD_LABELS[b.target] ?? b.target} - - - {b.builtAt && ( - - Built {formatDate(b.builtAt)} - - )} - {!b.built && ( - - Not built - - )} - - - - ))} - - - Each service builds its own indexes/graphs on first start. Trigger rebuilds from the service - catalog or queue data update/build operations. - - - ); -} - function motisProxyModeColor( mode: MotisTransitousStatus["feedProxyMode"], ): "default" | "success" | "warning" | "error" { @@ -596,9 +381,10 @@ export function DataWorkflowsPage() { } function DataWorkflowsView() { const { apiUrl, client } = useActivityClient(); + const [osmOffset, setOsmOffset] = useState(0); const { data, isLoading, isError, refetch, isFetching } = useQuery({ - ...serviceAdminQueries(client, apiUrl).data(), + ...serviceAdminQueries(client, apiUrl).data({ osmLimit: 50, osmOffset }), refetchInterval: 30_000, }); @@ -642,7 +428,7 @@ function DataWorkflowsView() { - + @@ -650,7 +436,6 @@ function DataWorkflowsView() { - ); diff --git a/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx b/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx index b7cc96b6d..9c2315f77 100644 --- a/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx +++ b/apps/web/src/components/admin/services/ServiceWorkflows.test.tsx @@ -36,13 +36,35 @@ const at = "2026-10-21T10:00:00.000Z"; const inventory = { fetchedAt: at, osm: { - found: true, - filename: "synthetic.osm.pbf", - sizeBytes: 42, - modifiedAt: at, - region: "test", + items: [ + { + artifactId: "a".repeat(64), + filename: "synthetic.osm.pbf", + region: "test", + sizeBytes: 42, + modifiedAt: at, + sourceFingerprint: null, + fingerprintState: "missing", + }, + ], + total: 1, + limit: 50, + offset: 0, + unavailable: false, }, - builds: [{ target: "motis", built: true, builtAt: at }], + builds: [ + { + target: "motis", + built: false, + readiness: "unknown", + reason: "Synthetic readiness unavailable", + sizeBytes: null, + sourceFingerprints: [], + buildRecordVersion: null, + region: null, + runtimeImages: {}, + }, + ], motisTransitous: { configFound: true, datasetCount: 1, @@ -142,7 +164,7 @@ class Boundary extends Component<{ children: ReactNode }, { crashed: boolean }> function InventoryStatus() { const { client, apiUrl } = useActivityClient(); const { data } = useQuery(serviceAdminQueries(client, apiUrl).data()); - return {data?.osm.region}; + return {data?.osm.items[0]?.region}; } function setup( handler?: (url: URL, init: RequestInit) => Response | Promise | undefined, @@ -618,7 +640,13 @@ describe("accepted jobs refresh concrete workflows", () => { if (url.pathname.endsWith("/services/data")) return Response.json({ ...inventory, - osm: { ...inventory.osm, region: `generation-${done.size}` }, + osm: { + ...inventory.osm, + items: inventory.osm.items.map((item) => ({ + ...item, + region: `generation-${done.size}`, + })), + }, }); return undefined; }); @@ -684,7 +712,13 @@ describe("accepted jobs refresh concrete workflows", () => { if (url.pathname.endsWith("/services/data")) return Response.json({ ...inventory, - osm: { ...inventory.osm, region: changed ? "should-not-refresh" : "original" }, + osm: { + ...inventory.osm, + items: inventory.osm.items.map((item) => ({ + ...item, + region: changed ? "should-not-refresh" : "original", + })), + }, }); return undefined; }); @@ -715,7 +749,13 @@ describe("accepted jobs refresh concrete workflows", () => { if (url.pathname.endsWith("/services/data")) return Response.json({ ...inventory, - osm: { ...inventory.osm, region: done ? "refreshed" : "original" }, + osm: { + ...inventory.osm, + items: inventory.osm.items.map((item) => ({ + ...item, + region: done ? "refreshed" : "original", + })), + }, }); return undefined; }); @@ -744,7 +784,13 @@ describe("accepted jobs refresh concrete workflows", () => { const calls = setup((url) => { if (url.pathname.endsWith("/run")) return deferred.promise; if (url.pathname.endsWith("/services/data")) - return Response.json({ ...inventory, osm: { ...inventory.osm, region: url.origin } }); + return Response.json({ + ...inventory, + osm: { + ...inventory.osm, + items: inventory.osm.items.map((item) => ({ ...item, region: url.origin })), + }, + }); return undefined; }); const { view, rerender } = mount( @@ -930,7 +976,7 @@ describe("wire intent, validation, polling and policy", () => { }); const { qc } = mount(); await screen.findByText("1 schedule dataset(s)"); - expect(qc.getQueryData(["admin", "services", apiUrl, "data"])).toEqual(inventory); + expect(qc.getQueryData(["admin", "services", apiUrl, "data", "50", "0"])).toEqual(inventory); expect(await screen.findByText(/called motis; skipped fallback/)).toBeInTheDocument(); const toggle = screen.getAllByLabelText("Fan-out rollback")[0] as HTMLElement; expect(toggle).not.toBeChecked(); @@ -947,7 +993,13 @@ describe("wire intent, validation, polling and policy", () => { let reads = 0; setup((url) => url.pathname.endsWith("/services/data") - ? Response.json({ ...inventory, osm: { ...inventory.osm, region: `poll-${++reads}` } }) + ? Response.json({ + ...inventory, + osm: { + ...inventory.osm, + items: inventory.osm.items.map((item) => ({ ...item, region: `poll-${++reads}` })), + }, + }) : undefined, ); mount(); diff --git a/apps/web/src/lib/admin/serviceAdmin.ts b/apps/web/src/lib/admin/serviceAdmin.ts index e23778489..3de1eeeb8 100644 --- a/apps/web/src/lib/admin/serviceAdmin.ts +++ b/apps/web/src/lib/admin/serviceAdmin.ts @@ -73,10 +73,10 @@ export function serviceAdminQueries(client: ApiClient, apiUrl: string) { queryKey: [...key, "backups"], queryFn: ({ signal }) => serviceOrThrow(sdk.listAdminServiceBackups(options(signal))), }), - data: () => + data: (page: { osmLimit?: number; osmOffset?: number } = {}) => queryOptions({ - queryKey: [...key, "data"], - queryFn: ({ signal }) => serviceOrThrow(sdk.getAdminServiceData(options(signal))), + queryKey: [...key, "data", String(page.osmLimit ?? 50), String(page.osmOffset ?? 0)], + queryFn: ({ signal }) => serviceOrThrow(sdk.getAdminServiceData(page, options(signal))), }), policy: () => queryOptions({ diff --git a/docs/docs/install/preparing-data.md b/docs/docs/install/preparing-data.md index ba7563b17..ede1b796a 100644 --- a/docs/docs/install/preparing-data.md +++ b/docs/docs/install/preparing-data.md @@ -283,6 +283,26 @@ source builds. This is a sized preparation workload, not a boot-time task. See t [global publication runbook](../developer/ambient-places-publication.md#planet-preparation-restart-and-deployment) for ordered commands, persistent checkpoints, resource budgets and recovery. +## Inspecting prepared data + +**Services → Data workflows** lists every local OSM extract with pagination, +its recorded region, size and source fingerprint. Downloads record SHA-256 +evidence during their checksum pass. Builds and imports record their output +hashes and local file identities; refreshing this page does not hash datasets. + +Prepared service data distinguishes **ready**, **missing**, **incomplete**, +**stale** and **unknown**. Empty directories do not establish readiness. A +changed source or output invalidates its recorded evidence, including a +replacement with the same byte size. Legacy build records without source hashes +remain unknown until rebuilt or imported with current evidence. MOTIS also +requires its candidate manifest and imported outputs. + +These checks describe prepared artifacts, rather than live routing or database +health. Services such as Valhalla, Pelias, Nominatim, Photon and Overpass remain +unknown when authoritative runtime evidence is unavailable. An inspection or +permission failure is shown as unavailable; it is never presented as an empty +successful inventory. + ## Sharing data with hardlinks Several services consume the same files. Rather than copy a multi-gigabyte extract diff --git a/packages/cli/__tests__/cli-build-artifacts.test.ts b/packages/cli/__tests__/cli-build-artifacts.test.ts index 035fa0acc..8895bbbf3 100644 --- a/packages/cli/__tests__/cli-build-artifacts.test.ts +++ b/packages/cli/__tests__/cli-build-artifacts.test.ts @@ -97,6 +97,43 @@ afterEach(() => { }); describe("build bundles", () => { + it("retains imported source provenance with fresh local output identities", async () => { + seedOsrmBuild(buildHost); + const { artifactFingerprint } = await import("@openmapx/core/build-artifacts"); + const initial = await exportBuildBundle({ + serviceId: "osrm", + bundle: join(tmp, "first.tar"), + rootDir: buildHost, + }); + const sourceFingerprints = [{ id: "a".repeat(64), sha256: "b".repeat(64), sizeBytes: 3 }]; + writeBuildRecord( + { + ...osrmRecord(), + schemaVersion: 2, + sourceFingerprints, + artifactFingerprint: artifactFingerprint(initial.files), + }, + buildHost, + ); + const path = join(tmp, "v2.tar"); + await exportBuildBundle({ serviceId: "osrm", bundle: path, rootDir: buildHost }); + await importBuildBundle({ bundle: path, rootDir: serveHost, runningServices: noneRunning }); + expect(readBuildRecord("osrm", serveHost)).toMatchObject({ + schemaVersion: 2, + sourceFingerprints, + }); + const evidence = JSON.parse( + readFileSync(join(dataDir(serveHost), ".openmapx-builds/osrm.evidence.json"), "utf8"), + ); + expect(evidence.record.sourceFingerprints).toEqual(sourceFingerprints); + expect( + evidence.files.find((file: { path: string }) => file.path === "osrm-graph/region.osrm"), + ).toMatchObject({ + sizeBytes: 5, + inode: String(lstatSync(join(dataDir(serveHost), "osrm-graph/region.osrm")).ino), + }); + }); + it("round-trips a build to another host, dropping staged inputs and recording provenance", async () => { seedOsrmBuild(buildHost); const bundle = join(tmp, "osrm.tar"); diff --git a/packages/cli/__tests__/cli-build-evidence.test.ts b/packages/cli/__tests__/cli-build-evidence.test.ts new file mode 100644 index 000000000..bcd495121 --- /dev/null +++ b/packages/cli/__tests__/cli-build-evidence.test.ts @@ -0,0 +1,74 @@ +/** Actual built-in OSRM builder/staged publication/recording; external command I/O only is substituted. */ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; + +const commands = vi.hoisted(() => ({ calls: [] as string[][] })); +vi.mock("execa", () => ({ + execa: async (command: string, args: string[]) => { + commands.calls.push([command, ...args]); + if (command === "docker" && args[0] === "run") { + const volume = args[args.indexOf("-v") + 1]; + if (!volume) throw Error("Missing actual builder mount"); + const directory = volume.slice(0, -6); + for (const name of [ + "region.osrm", + "region.osrm.partition", + "region.osrm.cells", + "region.osrm.mldgr", + ]) + writeFileSync(join(directory, name), `synthetic external OSRM output ${name}`); + } + return { stdout: "", stderr: "", exitCode: command === "git" ? 1 : 0 }; + }, +})); +it("records the actual default builder's source hash and local output evidence", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-default-build-evidence-")); + try { + mkdirSync(join(root, "infra/docker/data/osm"), { recursive: true }); + mkdirSync(join(root, "services/osrm"), { recursive: true }); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + writeFileSync( + join(root, "services/osrm/service.json"), + JSON.stringify({ + id: "osrm", + name: "OSRM", + version: "1.0.0", + quality: "built-in", + buildCommand: "openmapx services build osrm", + container: { image: "example.test/osrm", tag: "1" }, + }), + ); + writeFileSync(join(root, "infra/docker/data/osm/europe-germany.osm.pbf"), "source fixture"); + const { buildServices } = await import("../src/lib/service-builds"); + await buildServices({ + rootDir: root, + mode: "explicit", + serviceIds: ["osrm"], + region: "europe/germany", + }); + const record = JSON.parse( + readFileSync(join(root, "infra/docker/data/.openmapx-builds/osrm.json"), "utf8"), + ), + evidence = JSON.parse( + readFileSync(join(root, "infra/docker/data/.openmapx-builds/osrm.evidence.json"), "utf8"), + ); + expect(record).toMatchObject({ + schemaVersion: 2, + service: "osrm", + sourceFingerprints: [{ sha256: createHash("sha256").update("source fixture").digest("hex") }], + }); + expect(evidence.record).toEqual(record); + expect(evidence.files.map((f: { path: string }) => f.path)).toContain( + "osrm-graph/region.osrm.mldgr", + ); + expect(evidence.inputs).toHaveLength(1); + expect(commands.calls.filter((call) => call[0] === "docker" && call[1] === "run")).toHaveLength( + 3, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/packages/cli/src/lib/build-artifacts.ts b/packages/cli/src/lib/build-artifacts.ts index fb16ae2a0..977971add 100644 --- a/packages/cli/src/lib/build-artifacts.ts +++ b/packages/cli/src/lib/build-artifacts.ts @@ -8,7 +8,6 @@ import { readFileSync, realpathSync, rmSync, - writeFileSync, } from "node:fs"; import { dirname, join, relative } from "node:path"; import { Writable } from "node:stream"; @@ -18,10 +17,13 @@ import { type BuildRecordV2, type BundleManifest, buildArtifactSpec, + buildRecordSchema, + capturePreparedBuildEvidence, DEFAULT_ARTIFACT_ARCHIVE_LIMITS, safeArtifactPath, runtimeImageMismatches as sharedRuntimeImageMismatches, validateBuildBundle, + withDataDirectory, writeBuildBundle, } from "@openmapx/core/build-artifacts"; import { execa } from "execa"; @@ -56,8 +58,16 @@ function buildRecordPath(dataRoot: string, serviceId: string): string { export function writeBuildRecord(record: BuildRecord | BuildRecordV2, rootDir?: string): string { const path = buildRecordPath(dataRootFor(rootDir), record.service); - mkdirSync(join(dataRootFor(rootDir), BUILD_RECORD_DIR), { recursive: true }); - writeFileSync(path, `${JSON.stringify(record, null, 2)}\n`, "utf-8"); + const checked = buildRecordSchema.parse(record); + mkdirSync(dataRootFor(rootDir), { recursive: true }); + withDataDirectory(dataRootFor(rootDir), "", (root) => { + const directory = root.directory(BUILD_RECORD_DIR, true); + try { + directory.write(`${checked.service}.json`, `${JSON.stringify(checked)}\n`); + } finally { + directory.close(); + } + }); return path; } @@ -184,6 +194,13 @@ export async function importBuildBundle(opts: ImportBuildBundleOptions): Promise assertCliExecutionAvailable(); promoteStagedBuildDirs(swaps); const { dirs: _dirs, directories: _directories, files: _files, ...record } = manifest; + if (record.schemaVersion === 2) + await capturePreparedBuildEvidence({ + dataRoot, + record, + verifiedFiles: manifest.files, + signal: cliAcquisitionOptions().signal, + }); writeBuildRecord(record, opts.rootDir); return manifest; } finally { diff --git a/packages/cli/src/lib/service-builds.ts b/packages/cli/src/lib/service-builds.ts index 83a7c1f12..930532f5c 100644 --- a/packages/cli/src/lib/service-builds.ts +++ b/packages/cli/src/lib/service-builds.ts @@ -1,5 +1,12 @@ import { statSync } from "node:fs"; -import { basename } from "node:path"; +import { basename, join, relative } from "node:path"; +import { + capturePreparedBuildEvidence, + hashDataFile, + inspectDataFile, + osmArtifactId, + sameDataFile, +} from "@openmapx/core/build-artifacts"; import { compareServiceBuildOrder, createServiceBatchResult, @@ -16,8 +23,9 @@ import { import { assertCliControlForward } from "./control-context"; import { runningComposeServices } from "./docker"; import { resolveBuildRegion } from "./env-defaults"; -import { assertCliExecutionAvailable } from "./execution-deadline"; +import { assertCliExecutionAvailable, cliAcquisitionOptions } from "./execution-deadline"; import { buildMotisData, importMotisData } from "./motis-data"; +import { resolveOsmPbf } from "./osm-pbf"; import { buildOsrmGraph } from "./osrm-graph"; import { buildOtpGraph } from "./otp-graph"; import { log } from "./output"; @@ -217,7 +225,7 @@ const BUILT_IN_SERVICE_BUILD_HANDLERS: Record = { summary: `Built TileServer MBTiles → ${result.mbtilesPath}`, artifact: { sourcePbf: result.sourcePbf, - runtimeImages: {}, + runtimeImages: { tileserver: serviceImage(service) }, toolImages: { planetiler: image }, }, }; @@ -229,28 +237,79 @@ async function recordBuild( serviceId: string, region: string | undefined, artifact: ServiceBuildArtifact, + input?: { path: string; evidence: Awaited> }, ): Promise { assertCliExecutionAvailable(); assertCliControlForward(); const revision = await currentRevision(rootDir); assertCliExecutionAvailable(); assertCliControlForward(); - writeBuildRecord( - { - schemaVersion: 1, - service: serviceId, - ...(region ? { region } : {}), - sourcePbf: { - name: basename(artifact.sourcePbf), - sizeBytes: statSync(artifact.sourcePbf).size, - }, - builtAt: new Date().toISOString(), - ...(revision ? { revision } : {}), - runtimeImages: artifact.runtimeImages, - toolImages: artifact.toolImages, + const record: BuildRecord = { + schemaVersion: 1, + service: serviceId, + ...(region ? { region } : {}), + sourcePbf: { name: basename(artifact.sourcePbf), sizeBytes: statSync(artifact.sourcePbf).size }, + builtAt: new Date().toISOString(), + ...(revision ? { revision } : {}), + runtimeImages: artifact.runtimeImages, + toolImages: artifact.toolImages, + }; + if (!input) { + writeBuildRecord(record, rootDir); + return; + } + if (input.path !== artifact.sourcePbf) throw Error("Build source differs from captured input"); + const dataRoot = join(repoPaths(rootDir).infraDir, "data"), + path = relative(dataRoot, input.path), + now = inspectDataFile(dataRoot, path); + if ( + now.device !== input.evidence.device || + now.inode !== input.evidence.inode || + now.sizeBytes !== input.evidence.sizeBytes || + now.modifiedAtMs !== input.evidence.modifiedAtMs + ) + throw Error("Build source changed while preparing outputs"); + let evidence = input.evidence; + if ( + !sameDataFile(now, { + device: evidence.device, + inode: evidence.inode, + sizeBytes: evidence.sizeBytes, + modifiedAtMs: evidence.modifiedAtMs, + changedAtMs: evidence.changedAtMs, + }) + ) { + const checked = await hashDataFile(input.path, cliAcquisitionOptions().signal); + if (checked.sha256 !== evidence.sha256) + throw Error("Build source hash changed while preparing outputs"); + evidence = checked; + } + const sha256 = evidence.sha256, + sourceFingerprints = [ + { id: osmArtifactId(basename(input.path)), sha256, sizeBytes: evidence.sizeBytes }, + ]; + const version2 = await capturePreparedBuildEvidence({ + dataRoot, + record: { + ...record, + schemaVersion: 2, + sourceFingerprints, + artifactFingerprint: "0".repeat(64), }, - rootDir, - ); + signal: cliAcquisitionOptions().signal, + inputs: [ + { + path, + sha256, + device: evidence.device, + inode: evidence.inode, + sizeBytes: evidence.sizeBytes, + modifiedAtMs: evidence.modifiedAtMs, + changedAtMs: evidence.changedAtMs, + }, + ], + }); + writeBuildRecord(version2, rootDir); } function getServiceBuildHandlers( @@ -405,6 +464,15 @@ export async function buildServices( `${item.id}: using region "${resolvedRegion.value}" from $${resolvedRegion.sourceEnv}`, ); } + const sourcePath = opts.handlers?.[item.id] + ? null + : resolveOsmPbf(join(repoPaths(rootDir).infraDir, "data"), resolvedRegion.value, item.id); + const input = sourcePath + ? { + path: sourcePath, + evidence: await hashDataFile(sourcePath, cliAcquisitionOptions().signal), + } + : undefined; attempted = true; const built = await item.handler({ rootDir, @@ -417,7 +485,7 @@ export async function buildServices( log.warn(`${item.id}: ${warning}`); } if (built.artifact) { - await recordBuild(rootDir, item.id, resolvedRegion.value, built.artifact); + await recordBuild(rootDir, item.id, resolvedRegion.value, built.artifact, input); } log.ok(built.summary); completedIds.push(item.id); diff --git a/packages/core/src/api/contracts/service-admin.test.ts b/packages/core/src/api/contracts/service-admin.test.ts index a661006b1..409ac3849 100644 --- a/packages/core/src/api/contracts/service-admin.test.ts +++ b/packages/core/src/api/contracts/service-admin.test.ts @@ -118,13 +118,35 @@ const dawarich = { }; const data = { osm: { - found: true, - filename: "synthetic.osm.pbf", - sizeBytes: 42, - modifiedAt: at, - region: "test", + items: [ + { + artifactId: "a".repeat(64), + filename: "synthetic.osm.pbf", + region: "test", + sizeBytes: 42, + modifiedAt: at, + sourceFingerprint: null, + fingerprintState: "missing", + }, + ], + total: 1, + limit: 50, + offset: 0, + unavailable: false, }, - builds: [{ target: "motis", built: true, builtAt: at }], + builds: [ + { + target: "motis", + built: false, + readiness: "unknown", + reason: "Synthetic readiness unavailable", + sizeBytes: null, + sourceFingerprints: [], + buildRecordVersion: null, + region: null, + runtimeImages: {}, + }, + ], motisTransitous: { configFound: true, datasetCount: 1, diff --git a/packages/core/src/api/contracts/service-admin/data.ts b/packages/core/src/api/contracts/service-admin/data.ts index 5ce33a02d..fd00060d7 100644 --- a/packages/core/src/api/contracts/service-admin/data.ts +++ b/packages/core/src/api/contracts/service-admin/data.ts @@ -3,6 +3,10 @@ import { OPS_RESULT_SCHEMAS } from "../../../ops/contract"; import { adminCountSchema, adminTimestampSchema } from "../admin"; import { adminServiceIdListSchema } from "./common"; +export const adminServiceDataQuerySchema = z.strictObject({ + osmLimit: z.coerce.number().int().min(1).max(500).optional(), + osmOffset: z.coerce.number().int().min(0).max(1000000).optional(), +}); export const adminServiceDataSchema = OPS_RESULT_SCHEMAS["data.inspect"].extend({ fetchedAt: adminTimestampSchema, }); diff --git a/packages/core/src/api/contracts/service-admin/index.ts b/packages/core/src/api/contracts/service-admin/index.ts index 78e879dde..fea3e00d0 100644 --- a/packages/core/src/api/contracts/service-admin/index.ts +++ b/packages/core/src/api/contracts/service-admin/index.ts @@ -23,6 +23,7 @@ import { adminServiceBackupParamsSchema, adminServiceBackupRestoreSchema, adminServiceBackupsSchema, + adminServiceDataQuerySchema, adminServiceDataSchema, adminServiceSelectionSchema, adminServiceSelectionSetSchema, @@ -221,6 +222,7 @@ export const serviceAdminOperations = { method: "GET", path: "/api/admin/services/data", auth: "admin", + request: { query: adminServiceDataQuerySchema }, responses: { ...adminCommonResponses, 200: adminJson(adminServiceDataSchema) }, }, getAdminServiceSelection: { diff --git a/packages/core/src/api/generated/admin/service-admin.generated.ts b/packages/core/src/api/generated/admin/service-admin.generated.ts index 62db3aaa4..d5c020d7c 100644 --- a/packages/core/src/api/generated/admin/service-admin.generated.ts +++ b/packages/core/src/api/generated/admin/service-admin.generated.ts @@ -137,6 +137,7 @@ import type { GetAdminServiceData415, GetAdminServiceData429, GetAdminServiceData500, + GetAdminServiceDataParams, GetAdminServiceSelection200, GetAdminServiceSelection400, GetAdminServiceSelection401, @@ -1980,17 +1981,33 @@ export type getAdminServiceDataResponse = | getAdminServiceDataResponseSuccess | getAdminServiceDataResponseError; -export const getGetAdminServiceDataUrl = () => { - return `/api/admin/services/data`; +export const getGetAdminServiceDataUrl = (params?: GetAdminServiceDataParams) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 + ? `/api/admin/services/data?${stringifiedParams}` + : `/api/admin/services/data`; }; export const getAdminServiceData = async ( + params?: GetAdminServiceDataParams, options?: Parameters[1], ): Promise => { - return getAdminServiceDataRequest(getGetAdminServiceDataUrl(), { - ...options, - method: "GET", - }); + return getAdminServiceDataRequest( + getGetAdminServiceDataUrl(params), + { + ...options, + method: "GET", + }, + ); }; export type getAdminServiceSelectionResponse200 = { diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItem.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItem.ts index 7fbbb1781..6711cba09 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItem.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItem.ts @@ -1,5 +1,8 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminServiceData200BuildsItemReadiness } from "./getAdminServiceData200BuildsItemReadiness"; +import type { GetAdminServiceData200BuildsItemRuntimeImages } from "./getAdminServiceData200BuildsItemRuntimeImages"; +import type { GetAdminServiceData200BuildsItemSourceFingerprintsItem } from "./getAdminServiceData200BuildsItemSourceFingerprintsItem"; import type { GetAdminServiceData200BuildsItemTarget } from "./getAdminServiceData200BuildsItemTarget"; export type GetAdminServiceData200BuildsItem = { @@ -7,4 +10,12 @@ export type GetAdminServiceData200BuildsItem = { built: boolean; /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ builtAt?: string; + readiness: GetAdminServiceData200BuildsItemReadiness; + reason: string | null; + sizeBytes: number | null; + /** @maxItems 10000 */ + sourceFingerprints: GetAdminServiceData200BuildsItemSourceFingerprintsItem[]; + buildRecordVersion: 1 | 2 | null; + region: string | null; + runtimeImages: GetAdminServiceData200BuildsItemRuntimeImages; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemReadiness.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemReadiness.ts new file mode 100644 index 000000000..1f5ae8834 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemReadiness.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceData200BuildsItemReadiness = + (typeof GetAdminServiceData200BuildsItemReadiness)[keyof typeof GetAdminServiceData200BuildsItemReadiness]; + +export const GetAdminServiceData200BuildsItemReadiness = { + ready: "ready", + missing: "missing", + incomplete: "incomplete", + stale: "stale", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemRuntimeImages.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemRuntimeImages.ts new file mode 100644 index 000000000..028889f4f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemRuntimeImages.ts @@ -0,0 +1,4 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceData200BuildsItemRuntimeImages = { [key: string]: string }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemSourceFingerprintsItem.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemSourceFingerprintsItem.ts new file mode 100644 index 000000000..336bd5ccd --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200BuildsItemSourceFingerprintsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceData200BuildsItemSourceFingerprintsItem = { + /** @maxLength 2048 */ + id: string; + /** @pattern ^[a-f0-9]{64}$ */ + sha256: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200Osm.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200Osm.ts index 36328faa8..82e5d4dbb 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200Osm.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200Osm.ts @@ -1,21 +1,24 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminServiceData200OsmItemsItem } from "./getAdminServiceData200OsmItemsItem"; export type GetAdminServiceData200Osm = { - found: boolean; + /** @maxItems 500 */ + items: GetAdminServiceData200OsmItemsItem[]; /** - * @minLength 1 - * @maxLength 255 - * @pattern ^[^/\\]+$ + * @minimum 0 + * @maximum 9007199254740991 + */ + total: number; + /** + * @minimum 1 + * @maximum 500 */ - filename?: string; + limit: number; /** * @minimum 0 * @maximum 9007199254740991 */ - sizeBytes?: number; - /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ - modifiedAt?: string; - /** @maxLength 2048 */ - region?: string; + offset: number; + unavailable: boolean; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItem.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItem.ts new file mode 100644 index 000000000..d84687497 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItem.ts @@ -0,0 +1,24 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminServiceData200OsmItemsItemFingerprintState } from "./getAdminServiceData200OsmItemsItemFingerprintState"; + +export type GetAdminServiceData200OsmItemsItem = { + /** @pattern ^[a-f0-9]{64}$ */ + artifactId: string; + /** + * @minLength 1 + * @maxLength 255 + * @pattern ^[^/\\]+$ + */ + filename: string; + region: string | null; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + sizeBytes: number; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + modifiedAt: string; + sourceFingerprint: string | null; + fingerprintState: GetAdminServiceData200OsmItemsItemFingerprintState; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItemFingerprintState.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItemFingerprintState.ts new file mode 100644 index 000000000..407415d0d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmItemsItemFingerprintState.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceData200OsmItemsItemFingerprintState = + (typeof GetAdminServiceData200OsmItemsItemFingerprintState)[keyof typeof GetAdminServiceData200OsmItemsItemFingerprintState]; + +export const GetAdminServiceData200OsmItemsItemFingerprintState = { + verified: "verified", + missing: "missing", + stale: "stale", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceDataParams.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceDataParams.ts new file mode 100644 index 000000000..47e9f82d6 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceDataParams.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceDataParams = { + /** + * @minimum 1 + * @maximum 500 + */ + osmLimit?: number; + /** + * @minimum 0 + * @maximum 1000000 + */ + osmOffset?: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index 0e38d53c9..cb305cd69 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -201,6 +201,9 @@ export * from "./getAdminServiceCredentials429"; export * from "./getAdminServiceCredentials500"; export * from "./getAdminServiceData200"; export * from "./getAdminServiceData200BuildsItem"; +export * from "./getAdminServiceData200BuildsItemReadiness"; +export * from "./getAdminServiceData200BuildsItemRuntimeImages"; +export * from "./getAdminServiceData200BuildsItemSourceFingerprintsItem"; export * from "./getAdminServiceData200BuildsItemTarget"; export * from "./getAdminServiceData200MotisTransitous"; export * from "./getAdminServiceData200MotisTransitousActiveSlot"; @@ -217,6 +220,8 @@ export * from "./getAdminServiceData200MotisTransitousOperationsProfile"; export * from "./getAdminServiceData200MotisTransitousPreflightState"; export * from "./getAdminServiceData200MotisTransitousPreviousHealthySlot"; export * from "./getAdminServiceData200Osm"; +export * from "./getAdminServiceData200OsmItemsItem"; +export * from "./getAdminServiceData200OsmItemsItemFingerprintState"; export * from "./getAdminServiceData400"; export * from "./getAdminServiceData401"; export * from "./getAdminServiceData403"; @@ -224,6 +229,7 @@ export * from "./getAdminServiceData413"; export * from "./getAdminServiceData415"; export * from "./getAdminServiceData429"; export * from "./getAdminServiceData500"; +export * from "./getAdminServiceDataParams"; export * from "./getAdminServiceSelection200"; export * from "./getAdminServiceSelection200Source"; export * from "./getAdminServiceSelection400"; diff --git a/packages/core/src/build-artifacts/data-evidence.ts b/packages/core/src/build-artifacts/data-evidence.ts new file mode 100644 index 000000000..9469ea3cf --- /dev/null +++ b/packages/core/src/build-artifacts/data-evidence.ts @@ -0,0 +1,130 @@ +/** Producer-written data evidence. Status readers inspect metadata, never hash dataset bodies. */ +import { createHash } from "node:crypto"; +import type { BigIntStats, Stats } from "node:fs"; +import { closeSync, constants, createReadStream, fstatSync, lstatSync, openSync } from "node:fs"; +import { z } from "zod"; +import { AnchoredDirectory } from "./filesystem"; +import { safeArtifactPath } from "./manifest"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/), + size = z.number().nonnegative().safe(); +export const dataFileIdentitySchema = z.strictObject({ + device: z.string().regex(/^\d+$/), + inode: z.string().regex(/^\d+$/), + sizeBytes: size, + modifiedAtMs: z.number().finite().nonnegative(), + changedAtMs: z.number().finite().nonnegative(), +}); +export type DataFileIdentity = z.infer; +export const osmSourceEvidenceSchema = z.strictObject({ + schemaVersion: z.literal(1), + artifactId: digest, + region: z.string().min(1).max(512), + sha256: digest, + ...dataFileIdentitySchema.shape, +}); +export const osmArtifactId = (sourceId: string) => + createHash("sha256").update(`openmapx-osm-v1\0${sourceId}`).digest("hex"); +export function fileIdentity(stat: Stats | BigIntStats): DataFileIdentity { + if (!Number.isSafeInteger(Number(stat.size))) + throw Error("Data file size exceeds supported inventory"); + return { + device: String(stat.dev), + inode: String(stat.ino), + sizeBytes: Number(stat.size), + modifiedAtMs: "mtimeNs" in stat ? Number(stat.mtimeNs) / 1e6 : stat.mtimeMs, + changedAtMs: "ctimeNs" in stat ? Number(stat.ctimeNs) / 1e6 : stat.ctimeMs, + }; +} +export function sameDataFile(left: DataFileIdentity, right: DataFileIdentity) { + return Object.keys(left).every( + (key) => left[key as keyof DataFileIdentity] === right[key as keyof DataFileIdentity], + ); +} +export async function hashDataFile(path: string, signal?: AbortSignal) { + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = fstatSync(fd, { bigint: true }); + if (!before.isFile() || before.nlink < 1) throw Error("Regular data file required"); + const sha = createHash("sha256"), + md5 = createHash("md5"); + for await (const chunk of createReadStream("", { fd, autoClose: false, signal })) { + sha.update(chunk); + md5.update(chunk); + } + const after = fstatSync(fd, { bigint: true }), + named = lstatSync(path, { bigint: true }); + if ( + !sameDataFile(fileIdentity(before), fileIdentity(after)) || + !sameDataFile(fileIdentity(after), fileIdentity(named)) + ) + throw Error("Data file changed while hashing"); + return { sha256: sha.digest("hex"), md5: md5.digest("hex"), ...fileIdentity(after) }; + } finally { + closeSync(fd); + } +} +export function withDataDirectory( + dataRoot: string, + relative: string, + read: (directory: AnchoredDirectory) => T, +): T { + if (relative && !safeArtifactPath(relative)) throw Error("Invalid data layout"); + const root = new AnchoredDirectory(dataRoot, undefined, false, true), + opened = [root]; + let current = root; + try { + for (const component of relative ? relative.split("/") : []) { + current = new AnchoredDirectory(current.name(component), current, false, true); + opened.push(current); + } + return read(current); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} +export function inspectDataFile(dataRoot: string, relative: string): DataFileIdentity { + const parts = relative.split("/"), + name = parts.pop(); + if (!name || !safeArtifactPath(relative)) throw Error("Invalid data file"); + return withDataDirectory(dataRoot, parts.join("/"), (directory) => { + const path = directory.name(name), + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = fstatSync(fd, { bigint: true }); + if (!before.isFile() || before.nlink < 1 || (before.mode & 0o022n) !== 0n) + throw Error("Unsafe data file"); + directory.check(); + const after = fstatSync(fd, { bigint: true }), + named = lstatSync(path, { bigint: true }); + if ( + !sameDataFile(fileIdentity(before), fileIdentity(after)) || + !sameDataFile(fileIdentity(after), fileIdentity(named)) + ) + throw Error("Data file changed during inspection"); + return fileIdentity(after); + } finally { + closeSync(fd); + } + }); +} + +export async function withDataDirectoryAsync( + dataRoot: string, + relative: string, + read: (directory: AnchoredDirectory) => Promise, +): Promise { + if (relative && !safeArtifactPath(relative)) throw Error("Invalid data layout"); + const root = new AnchoredDirectory(dataRoot, undefined, false, true), + opened = [root]; + let current = root; + try { + for (const component of relative ? relative.split("/") : []) { + current = new AnchoredDirectory(current.name(component), current, false, true); + opened.push(current); + } + return await read(current); + } finally { + for (const directory of opened.reverse()) directory.close(); + } +} diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 8f86c8a8f..398185350 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -137,6 +137,7 @@ export { verifyPublishedDataConfigurationFencePrepared, } from "./data-configuration-fence-witness"; export { retainedDataDatabaseTimestamp } from "./data-database-time"; +export * from "./data-evidence"; export { assertDataTransitCandidateForExecution, assertOwnedDataTransitCandidate, @@ -204,6 +205,7 @@ export { inspectDataExecutionAdmission, verifiedParentForArtifactDelegation, } from "./peer-admission"; +export * from "./prepared-evidence"; export { controlRetentionPreparationState, recordStagedCoordinatorRetentionCheck, diff --git a/packages/core/src/build-artifacts/manifest.ts b/packages/core/src/build-artifacts/manifest.ts index 24d749d3b..68f79f4af 100644 --- a/packages/core/src/build-artifacts/manifest.ts +++ b/packages/core/src/build-artifacts/manifest.ts @@ -79,6 +79,10 @@ const v2 = { credentialRequirements: z.array(string).optional(), runtimeMaterialization: z.literal("motis-target-credentials").optional(), }; +export const buildRecordSchema = z.discriminatedUnion("schemaVersion", [ + z.strictObject({ schemaVersion: z.literal(1), ...common }), + z.strictObject({ schemaVersion: z.literal(2), ...common, ...v2 }), +]); const layout = { dirs: z.array(string), directories: z.array(string), diff --git a/packages/core/src/build-artifacts/prepared-evidence.test.ts b/packages/core/src/build-artifacts/prepared-evidence.test.ts new file mode 100644 index 000000000..2d4ebd503 --- /dev/null +++ b/packages/core/src/build-artifacts/prepared-evidence.test.ts @@ -0,0 +1,50 @@ +import { expect, it } from "vitest"; +import { incompletePreparedLayout, verifyPreparedMotisManifest } from "./prepared-evidence"; + +it("requires every recorded MOTIS source artifact to match the actual candidate manifest", () => { + const artifact = (path: string) => ({ path, sizeBytes: 3, sha256: "a".repeat(64) }), + artifacts = { + config: artifact("config.yml"), + license: artifact("license.json"), + proxyConfig: artifact(".openmapx-feed-proxy/default.conf"), + proxyVars: artifact(".openmapx-feed-proxy/vars.json"), + sourceManifest: artifact("transit-source-manifest.json"), + datasets: [artifact("gtfs.zip")], + }; + const manifest = { schemaVersion: 1, epoch: "candidate-epoch", artifacts }; + const { datasets, ...fixed } = artifacts; + const files = [...Object.values(fixed), ...datasets].map((value) => ({ + ...value, + path: `motis/live/${value.path}`, + })); + expect(verifyPreparedMotisManifest(manifest, files)).toBe(true); + expect( + verifyPreparedMotisManifest( + manifest, + files.map((file, index) => (index === 0 ? { ...file, sha256: "b".repeat(64) } : file)), + ), + ).toBe(false); + expect( + verifyPreparedMotisManifest( + { ...manifest, artifacts: { ...artifacts, datasets: [artifact("../outside")] } }, + files, + ), + ).toBe(false); + expect(verifyPreparedMotisManifest({ schemaVersion: 1, epoch: "incomplete" }, files)).toBe(false); +}); +it("requires concrete runtime companions instead of nonempty directories", () => { + expect( + incompletePreparedLayout("osrm", [{ path: "osrm-graph/region.osrm", sizeBytes: 3 }]), + ).toMatch(/companions/); + expect(incompletePreparedLayout("otp", [{ path: "otp-graph/input.pbf", sizeBytes: 3 }])).toMatch( + /graph.obj/, + ); + expect( + incompletePreparedLayout("tileserver", [{ path: "tile-mbtiles/tiles.mbtiles", sizeBytes: 0 }]), + ).toMatch(/MBTiles/); + expect( + incompletePreparedLayout("motis", [ + { path: "motis/live/motis-candidate-manifest.json", sizeBytes: 20 }, + ]), + ).toMatch(/imported data/); +}); diff --git a/packages/core/src/build-artifacts/prepared-evidence.ts b/packages/core/src/build-artifacts/prepared-evidence.ts new file mode 100644 index 000000000..fcc3f8f75 --- /dev/null +++ b/packages/core/src/build-artifacts/prepared-evidence.ts @@ -0,0 +1,195 @@ +/** Local output identity binding; portable build records never contain host identities. */ +import { lstatSync, realpathSync } from "node:fs"; +import { join, relative } from "node:path"; +import { z } from "zod"; +import { + dataFileIdentitySchema, + hashDataFile, + inspectDataFile, + sameDataFile, + withDataDirectory, + withDataDirectoryAsync, +} from "./data-evidence"; +import { + artifactFingerprint, + BUILD_RECORD_DIR, + type BuildRecord, + type BuildRecordV2, + type BundleFile, + buildArtifactSpec, + buildRecordSchema, + safeArtifactPath, +} from "./manifest"; + +const fileSchema = z.strictObject({ + path: z.string().min(1).max(4096), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + ...dataFileIdentitySchema.shape, +}); +export const preparedBuildEvidenceSchema = z.strictObject({ + schemaVersion: z.literal(1), + record: buildRecordSchema, + files: z.array(fileSchema).max(10000), + inputs: z.array(fileSchema).max(10000).default([]), +}); +export const PREPARED_EVIDENCE_BYTES = 16 * 1024 * 1024; +export function preparedArtifactPath(dataRoot: string, path: string): string { + if (!safeArtifactPath(path)) throw Error("Invalid prepared path"); + // The sole existing managed alias is MOTIS' fixed A/B slot selector. + if (path === "motis/live" || path.startsWith("motis/live/")) { + const real = relative(realpathSync(dataRoot), realpathSync(join(dataRoot, "motis/live"))); + if (real !== "motis/live" && !/^motis\/slots\/[AB]$/.test(real)) + throw Error("Untrusted MOTIS artifact alias"); + return real + path.slice("motis/live".length); + } + return path; +} +function excluded(path: string, dir: boolean, patterns: readonly string[]) { + return patterns.some((pattern) => + pattern.startsWith("*.") + ? path.endsWith(pattern.slice(1)) + : pattern.endsWith("/") + ? (dir ? path + "/" : path).startsWith(pattern) + : path === pattern, + ); +} +export function incompletePreparedLayout( + service: string, + files: readonly { path: string; sizeBytes: number }[], +): string | null { + const has = (p: string) => files.some((file) => file.path === p && file.sizeBytes > 0); + if ( + service === "osrm" && + !["region.osrm", "region.osrm.partition", "region.osrm.cells", "region.osrm.mldgr"].every((n) => + has(`osrm-graph/${n}`), + ) + ) + return "The OSRM graph or required MLD companions are missing."; + if (service === "otp" && !has("otp-graph/graph.obj")) + return "The OTP graph.obj output is missing."; + if (service === "tileserver" && !has("tile-mbtiles/tiles.mbtiles")) + return "The tile MBTiles output is missing."; + if ( + service === "motis" && + (!has("motis/live/motis-candidate-manifest.json") || + !files.some((file) => file.path.startsWith("motis/live/data/") && file.sizeBytes > 0)) + ) + return "A verified MOTIS manifest and imported data output are required."; + return files.some((file) => file.sizeBytes > 0) ? null : "Prepared output files are missing."; +} +export async function capturePreparedBuildEvidence(options: { + dataRoot: string; + record: BuildRecord | BuildRecordV2; + signal?: AbortSignal; + verifiedFiles?: readonly BundleFile[]; + inputs?: z.infer[]; +}): Promise { + const { dataRoot, signal } = options, + spec = buildArtifactSpec(options.record.service), + files: z.infer[] = [], + known = new Map(options.verifiedFiles?.map((f) => [f.path, f])); + let entries = 0; + async function visit(path: string) { + signal?.throwIfAborted(); + if (++entries > 10000) throw Error("Prepared metadata capacity exceeded"); + const physical = preparedArtifactPath(dataRoot, path), + stat = withDataDirectory(dataRoot, physical.split("/").slice(0, -1).join("/"), (dir) => + lstatSync(dir.name(physical.split("/").at(-1) ?? "")), + ); + if (stat.isSymbolicLink()) throw Error("Prepared artifact symlink refused"); + if (stat.isDirectory()) { + if (excluded(path, true, spec.exclude)) return; + const names = withDataDirectory(dataRoot, physical, (dir) => dir.listBounded(10000)); + for (const name of names) await visit(`${path}/${name}`); + } else { + if (excluded(path, false, spec.exclude)) return; + const metadata = inspectDataFile(dataRoot, physical), + previous = known.get(path); + if (previous && previous.sizeBytes !== metadata.sizeBytes) + throw Error("Imported output size changed before recording"); + const sha256 = + previous?.sha256 ?? + (await withDataDirectoryAsync(dataRoot, physical.split("/").slice(0, -1).join("/"), (dir) => + hashDataFile(dir.name(physical.split("/").at(-1) ?? ""), signal), + )); + const hash = typeof sha256 === "string" ? sha256 : sha256.sha256; + const after = inspectDataFile(dataRoot, physical); + if ( + !sameDataFile(metadata, after) || + (typeof sha256 !== "string" && + !sameDataFile(after, { + device: sha256.device, + inode: sha256.inode, + sizeBytes: sha256.sizeBytes, + modifiedAtMs: sha256.modifiedAtMs, + changedAtMs: sha256.changedAtMs, + })) + ) + throw Error("Prepared output changed before evidence publication"); + files.push({ path, sha256: hash, ...after }); + if (files.length > 10000) throw Error("Prepared evidence capacity exceeded"); + } + } + for (const dir of spec.dirs) await visit(dir); + const sourceFingerprints = + options.record.schemaVersion === 2 ? options.record.sourceFingerprints : []; + const record = buildRecordSchema.parse({ + ...options.record, + schemaVersion: 2, + sourceFingerprints, + artifactFingerprint: artifactFingerprint(files), + }) as BuildRecordV2; + const evidence = preparedBuildEvidenceSchema.parse({ + schemaVersion: 1, + record, + files, + inputs: options.inputs ?? [], + }); + const bytes = JSON.stringify(evidence); + if (Buffer.byteLength(bytes) > PREPARED_EVIDENCE_BYTES) + throw Error("Prepared evidence metadata capacity exceeded"); + withDataDirectory(dataRoot, "", (root) => { + const records = root.directory(BUILD_RECORD_DIR, true); + try { + records.write(`${record.service}.evidence.json`, bytes); + } finally { + records.close(); + } + }); + return record; +} + +const motisArtifact = z.object({ + path: z.string().min(1).max(4096), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + sizeBytes: z.number().nonnegative().safe(), +}); +const motisManifest = z.object({ + schemaVersion: z.literal(1), + epoch: z.string().min(1).max(512), + artifacts: z.object({ + config: motisArtifact, + license: motisArtifact, + proxyConfig: motisArtifact, + proxyVars: motisArtifact, + sourceManifest: motisArtifact, + sourceIndex: motisArtifact.optional(), + datasets: z.array(motisArtifact).min(1).max(10000), + }), +}); +/** Match the existing candidate's own hashed artifact inventory, not its filename alone. */ +export function verifyPreparedMotisManifest(value: unknown, files: readonly BundleFile[]): boolean { + const parsed = motisManifest.safeParse(value); + if (!parsed.success) return false; + const { datasets, sourceIndex, ...required } = parsed.data.artifacts; + return [...Object.values(required), ...datasets, ...(sourceIndex ? [sourceIndex] : [])].every( + (artifact) => + safeArtifactPath(artifact.path) && + files.some( + (file) => + file.path === `motis/live/${artifact.path}` && + file.sizeBytes === artifact.sizeBytes && + file.sha256 === artifact.sha256, + ), + ); +} diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index f15167ccc..b78642b7e 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -9,6 +9,7 @@ import { type OpsOperation, type OpsRole, opsResourceId, + parseBoundedOpsResult, parseOpsRequest, parseOpsResult, redactedOpsError, @@ -585,7 +586,7 @@ describe("ops contract", () => { ).toBe("release-1"); }); - it("keeps data inventory kind-only and validates its bounded typed result", () => { + it("accepts bounded data inventory paging and validates its typed result", () => { expect(parseOpsRequest(request({ kind: "data.inspect" }), { role: "api" }).operation).toEqual({ kind: "data.inspect", }); @@ -598,13 +599,35 @@ describe("ops contract", () => { } const inventory = { osm: { - found: true, - filename: "germany-latest.osm.pbf", - sizeBytes: 42, - modifiedAt: "2026-08-23T18:00:00.000Z", - region: "germany", + items: [ + { + artifactId: "a".repeat(64), + filename: "synthetic.osm.pbf", + region: "test", + sizeBytes: 42, + modifiedAt: "2026-08-23T18:00:00.000Z", + sourceFingerprint: null, + fingerprintState: "missing", + }, + ], + total: 1, + limit: 50, + offset: 0, + unavailable: false, }, - builds: [{ target: "valhalla", built: true, builtAt: "2026-08-23T18:00:00.000Z" }], + builds: [ + { + target: "valhalla", + built: false, + readiness: "unknown", + reason: "Synthetic readiness unavailable", + sizeBytes: null, + sourceFingerprints: [], + buildRecordVersion: null, + region: null, + runtimeImages: {}, + }, + ], motisTransitous: { configFound: false, datasetCount: 0, @@ -647,6 +670,25 @@ describe("ops contract", () => { }, }; expect(parseOpsResult("data.inspect", inventory)).toEqual(inventory); + const fullPage = { + ...inventory, + osm: { + items: Array.from({ length: 500 }, (_, i) => ({ + artifactId: i.toString(16).padStart(64, "0"), + filename: "f".repeat(220) + ".osm.pbf", + region: "r".repeat(220), + sizeBytes: 42, + modifiedAt: "2026-08-23T18:00:00.000Z", + sourceFingerprint: "a".repeat(64), + fingerprintState: "verified", + })), + total: 500, + offset: 0, + limit: 500, + unavailable: false, + }, + }; + expect(() => parseBoundedOpsResult("data.inspect", fullPage)).not.toThrow(); expect(() => parseOpsResult("data.inspect", { ...inventory, dataRoot: "/host/data" }), ).toThrow(); diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 82c831197..b330524d0 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -247,7 +247,7 @@ export const OPS_KIND_POLICIES = { "serviceConfig.apply": asyncEffect("api", 2 * 60_000), "integrationConfig.apply": asyncEffect("api", 2 * 60_000), "vault.apply": asyncEffect("api", 2 * 60_000), - "data.inspect": sync("api", 30_000, 256 * 1024), + "data.inspect": sync("api", 30_000, OPS_MAX_RESULT_BYTES), "data.downloadOsm": asyncEffect("api", 24 * 3600_000), "data.downloadFonts": asyncEffect("api", 24 * 3600_000), "data.update": asyncEffect("api", 24 * 3600_000), @@ -521,7 +521,11 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ serviceConfigApplyOperationSchema, integrationConfigApplyOperationSchema, vaultApplyOperationSchema, - kindOnly("data.inspect"), + z.strictObject({ + kind: z.literal("data.inspect"), + osmLimit: z.number().int().min(1).max(500).optional(), + osmOffset: z.number().int().min(0).max(1000000).optional(), + }), z.strictObject({ kind: z.literal("data.downloadOsm"), regionId: regionIdSchema.optional() }), kindOnly("data.downloadFonts"), z.strictObject({ @@ -1245,16 +1249,30 @@ export const OPS_RESULT_SCHEMAS = { "vault.apply": revision, "data.inspect": z.strictObject({ osm: z.strictObject({ - found: z.boolean(), - filename: z - .string() - .min(1) - .max(255) - .regex(/^[^/\\]+$/) - .optional(), - sizeBytes: nonnegativeSafeInteger.optional(), - modifiedAt: timestampSchema.optional(), - region: boundedInventoryString.optional(), + items: z + .array( + z.strictObject({ + artifactId: z.string().regex(/^[a-f0-9]{64}$/), + filename: z + .string() + .min(1) + .max(255) + .regex(/^[^/\\]+$/), + region: z.string().max(512).nullable(), + sizeBytes: nonnegativeSafeInteger, + modifiedAt: timestampSchema, + sourceFingerprint: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + fingerprintState: z.enum(["verified", "missing", "stale"]), + }), + ) + .max(500), + total: nonnegativeSafeInteger, + limit: z.number().int().min(1).max(500), + offset: nonnegativeSafeInteger, + unavailable: z.boolean(), }), builds: z .array( @@ -1273,6 +1291,20 @@ export const OPS_RESULT_SCHEMAS = { ]), built: z.boolean(), builtAt: timestampSchema.optional(), + readiness: z.enum(["ready", "missing", "incomplete", "stale", "unknown"]), + reason: boundedInventoryString.nullable(), + sizeBytes: nonnegativeSafeInteger.nullable(), + sourceFingerprints: z + .array( + z.strictObject({ + id: boundedInventoryString, + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }), + ) + .max(10000), + buildRecordVersion: z.union([z.literal(1), z.literal(2)]).nullable(), + region: boundedInventoryString.nullable(), + runtimeImages: z.record(z.string().max(64), boundedInventoryString), }), ) .max(10), diff --git a/services/data-manager/__tests__/osm-fingerprints.test.ts b/services/data-manager/__tests__/osm-fingerprints.test.ts new file mode 100644 index 000000000..2d74ff343 --- /dev/null +++ b/services/data-manager/__tests__/osm-fingerprints.test.ts @@ -0,0 +1,112 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { downloadOsm } from "../src/jobs/download-osm"; +import { StateStore } from "../src/state"; + +it("publishes SHA256 metadata inside the same snapshot publication lock", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-osm-evidence-")); + let publishedUnderLock = false; + try { + await downloadOsm({ + region: "europe/germany", + dataDir: root, + store: new StateStore(root), + verifyChecksum: false, + withPublishLock: async (publish) => { + publish(); + publishedUnderLock = existsSync(join(root, "osm/europe-germany.osm.pbf.openmapx.json")); + }, + downloadImpl: async (_url, target, options) => { + mkdirSync(join(root, "osm"), { recursive: true }); + const tmp = target + ".test-part"; + writeFileSync(tmp, "fixture pbf bytes"); + await options.beforePublish?.(tmp); + await options.withPublishLock?.(() => renameSync(tmp, target)); + return { published: true }; + }, + }); + expect(publishedUnderLock).toBe(true); + const evidence = JSON.parse( + readFileSync(join(root, "osm/europe-germany.osm.pbf.openmapx.json"), "utf8"), + ); + expect(evidence).toMatchObject({ + schemaVersion: 1, + region: "europe/germany", + sha256: createHash("sha256").update("fixture pbf bytes").digest("hex"), + sizeBytes: 17, + }); + expect(evidence.artifactId).toMatch(/^[a-f0-9]{64}$/); + expect(evidence.device).toMatch(/^\d+$/); + expect(evidence.inode).toMatch(/^\d+$/); + } finally { + rmSync(root, { recursive: true, force: true }); + vi.restoreAllMocks(); + } +}); +it("does not bind a 304 checksum to a replacement inode at publication", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-osm-304-evidence-")); + mkdirSync(join(root, "osm")); + const target = join(root, "osm/europe-germany.osm.pbf"); + writeFileSync(target, "old bytes"); + try { + await expect( + downloadOsm({ + region: "europe/germany", + dataDir: root, + store: new StateStore(root), + verifyChecksum: false, + downloadImpl: async () => ({ published: false }), + withPublishLock: async (publish) => { + writeFileSync(target + ".replacement", "new bytes"); + renameSync(target + ".replacement", target); + publish(); + }, + }), + ).rejects.toThrow(/source changed/); + expect(existsSync(target + ".openmapx.json")).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); +it("reports failed sidecar publication without leaving verified evidence", async () => { + const root = mkdtempSync(join(tmpdir(), "omx-osm-sidecar-fail-")); + const { AnchoredDirectory } = await import( + "../../../packages/core/src/build-artifacts/filesystem" + ); + const write = vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(() => { + throw Error("Synthetic metadata publication failure"); + }); + try { + await expect( + downloadOsm({ + region: "europe/germany", + dataDir: root, + store: new StateStore(root), + verifyChecksum: false, + downloadImpl: async (_url, target, options) => { + mkdirSync(join(root, "osm"), { recursive: true }); + writeFileSync(target + ".part", "new bytes"); + await options.beforePublish?.(target + ".part"); + await options.withPublishLock?.(() => renameSync(target + ".part", target)); + return { published: true }; + }, + }), + ).rejects.toThrow(/metadata publication failure/); + expect(existsSync(join(root, "osm/europe-germany.osm.pbf"))).toBe(true); + expect(existsSync(join(root, "osm/europe-germany.osm.pbf.openmapx.json"))).toBe(false); + } finally { + write.mockRestore(); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/services/data-manager/src/jobs/download-osm.ts b/services/data-manager/src/jobs/download-osm.ts index 6f91e2ef4..6666c6982 100644 --- a/services/data-manager/src/jobs/download-osm.ts +++ b/services/data-manager/src/jobs/download-osm.ts @@ -1,6 +1,6 @@ -import { createHash } from "node:crypto"; -import { createReadStream, mkdirSync, statSync } from "node:fs"; +import { mkdirSync, statSync } from "node:fs"; import { join } from "node:path"; +import { type DataFileIdentity, hashDataFile } from "@openmapx/core/build-artifacts"; import { execa } from "execa"; import type { DatasetMetadata, StateStore } from "../state.js"; import { @@ -8,6 +8,7 @@ import { type CurlAtomicResult, curlAtomicWithResult, } from "./atomic-download.js"; +import { publishOsmFingerprint } from "./osm-fingerprints.js"; export function resolveOsmUrl(region: string): string { if (!region) throw new Error("region is required"); @@ -51,16 +52,6 @@ export function resolveOsmMd5Url(osmUrl: string): string { return `${osmUrl}.md5`; } -function computeMd5(filePath: string, signal?: AbortSignal): Promise { - return new Promise((resolve, reject) => { - const hash = createHash("md5"); - const stream = createReadStream(filePath, { signal }); - stream.on("data", (chunk) => hash.update(chunk)); - stream.on("end", () => resolve(hash.digest("hex"))); - stream.on("error", reject); - }); -} - async function fetchExpectedMd5(url: string, signal?: AbortSignal): Promise { try { const { stdout } = await execa("curl", ["-fsSL", url], { @@ -115,6 +106,8 @@ export async function downloadOsm(opts: DownloadOsmOptions): Promise { + const commit = () => { + publish(); + if (!sha256 || !source) throw Error("OSM source checksum missing"); + publishOsmFingerprint(targetPath, opts.region, sha256, source, true); + }; + if (opts.withPublishLock) await opts.withPublishLock(commit); + else commit(); + }, beforePublish: async (tempPath) => { + const checksums = await hashDataFile(tempPath, opts.signal); + sha256 = checksums.sha256; + source = checksums; if (expected) { - const actual = await computeMd5(tempPath, opts.signal); + const actual = checksums.md5; if (actual !== expected) { throw new Error( `OSM PBF checksum mismatch for ${opts.region}: expected ${expected}, got ${actual}. The download is corrupt — re-run to retry.`, @@ -140,14 +144,20 @@ export async function downloadOsm(opts: DownloadOsmOptions): Promise + publishOsmFingerprint(targetPath, opts.region, checksums.sha256, checksums); + if (opts.withPublishLock) await opts.withPublishLock(record); + else record(); } const sizeBytes = statSync(targetPath).size; diff --git a/services/data-manager/src/jobs/osm-fingerprints.ts b/services/data-manager/src/jobs/osm-fingerprints.ts new file mode 100644 index 000000000..412760483 --- /dev/null +++ b/services/data-manager/src/jobs/osm-fingerprints.ts @@ -0,0 +1,44 @@ +/** Called only by the downloader, inside its snapshot publication lock. */ + +import { lstatSync } from "node:fs"; +import { basename, dirname } from "node:path"; +import { + type DataFileIdentity, + fileIdentity, + osmArtifactId, + osmSourceEvidenceSchema, +} from "@openmapx/core/build-artifacts"; +import { AnchoredDirectory } from "../../../../packages/core/src/build-artifacts/filesystem"; +export function publishOsmFingerprint( + path: string, + region: string, + sha256: string, + source: DataFileIdentity, + renamed = false, +) { + const directory = new AnchoredDirectory(dirname(path), undefined, false, true); + try { + const stat = lstatSync(directory.name(basename(path)), { bigint: true }); + if (!stat.isFile()) throw Error("Published OSM file required"); + const current = fileIdentity(stat); + for (const key of [ + "device", + "inode", + "sizeBytes", + "modifiedAtMs", + ...(renamed ? [] : ["changedAtMs"]), + ] as (keyof DataFileIdentity)[]) + if (source[key] !== current[key]) + throw Error("OSM source changed before evidence publication"); + const evidence = osmSourceEvidenceSchema.parse({ + schemaVersion: 1, + artifactId: osmArtifactId(basename(path)), + region, + sha256, + ...fileIdentity(stat), + }); + directory.write(`${basename(path)}.openmapx.json`, JSON.stringify(evidence)); + } finally { + directory.close(); + } +} From f32850774d6581eaf23004af7759b332772c7d2c Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 10:25:34 +0100 Subject: [PATCH 056/135] feat(admin): stream authenticated artifact stages --- apps/api/openapi.json | 5323 +++++++++++++++++ .../admin-artifact-transfers.test.ts | 206 + .../src/routes/admin-artifact-transfers.ts | 102 + apps/api/src/routes/admin-byte-response.ts | 53 + apps/api/src/routes/admin-contract.ts | 37 +- apps/api/src/routes/index.ts | 2 + apps/api/src/server-wiring.ts | 3 + apps/api/src/services/artifact-transfers.ts | 125 + apps/ops-agent/package.json | 3 +- .../ops-agent/src/artifact-transfer/errors.ts | 9 + .../src/artifact-transfer/ingress.test.ts | 183 + .../src/artifact-transfer/policy.test.ts | 15 + .../ops-agent/src/artifact-transfer/policy.ts | 90 + .../src/artifact-transfer/routes.test.ts | 114 + .../ops-agent/src/artifact-transfer/routes.ts | 164 + .../src/artifact-transfer/runtime.ts | 17 + .../src/artifact-transfer/store.test.ts | 313 + apps/ops-agent/src/artifact-transfer/store.ts | 687 +++ apps/ops-agent/src/index.ts | 26 + apps/ops-agent/src/server.ts | 44 +- docs/docs/developer/api-surface.md | 22 + docs/docs/install/preparing-data.md | 48 + .../services-traefik-renderer.test.ts | 21 + packages/core/src/api/admin-byte-reader.ts | 80 + .../core/src/api/admin-byte-transport.test.ts | 278 + packages/core/src/api/admin-sdk-transport.ts | 136 +- packages/core/src/api/admin-sdk.ts | 4 + packages/core/src/api/client.ts | 48 +- packages/core/src/api/contracts/admin.ts | 30 + .../service-admin/artifact-transfers.ts | 102 + .../src/api/contracts/service-admin/index.ts | 5 + .../admin/operation-bindings.generated.ts | 48 + .../admin/service-admin.generated.ts | 871 ++- .../createAdminArtifactTransfer200.ts | 23 + .../createAdminArtifactTransfer200Cleanup.ts | 11 + ...createAdminArtifactTransfer200Direction.ts | 10 + .../createAdminArtifactTransfer200Purpose.ts | 11 + .../createAdminArtifactTransfer200State.ts | 17 + .../createAdminArtifactTransfer400.ts | 20 + .../createAdminArtifactTransfer401.ts | 20 + .../createAdminArtifactTransfer403.ts | 20 + .../createAdminArtifactTransfer404.ts | 20 + .../createAdminArtifactTransfer409.ts | 20 + .../createAdminArtifactTransfer410.ts | 20 + .../createAdminArtifactTransfer413.ts | 20 + .../createAdminArtifactTransfer415.ts | 20 + .../createAdminArtifactTransfer422.ts | 20 + .../createAdminArtifactTransfer429.ts | 20 + .../createAdminArtifactTransfer500.ts | 20 + .../createAdminArtifactTransfer503.ts | 20 + .../createAdminArtifactTransfer507.ts | 20 + .../createAdminArtifactTransferBody.ts | 16 + ...reateAdminArtifactTransferBodyDirection.ts | 9 + .../createAdminArtifactTransferBodyPurpose.ts | 11 + .../createAdminArtifactTransferHeaders.ts | 10 + .../deleteAdminArtifactTransfer200.ts | 6 + .../deleteAdminArtifactTransfer400.ts | 20 + .../deleteAdminArtifactTransfer401.ts | 20 + .../deleteAdminArtifactTransfer403.ts | 20 + .../deleteAdminArtifactTransfer404.ts | 20 + .../deleteAdminArtifactTransfer409.ts | 20 + .../deleteAdminArtifactTransfer410.ts | 20 + .../deleteAdminArtifactTransfer413.ts | 20 + .../deleteAdminArtifactTransfer415.ts | 20 + .../deleteAdminArtifactTransfer422.ts | 20 + .../deleteAdminArtifactTransfer429.ts | 20 + .../deleteAdminArtifactTransfer500.ts | 20 + .../deleteAdminArtifactTransfer503.ts | 20 + .../deleteAdminArtifactTransfer507.ts | 20 + ...downloadAdminArtifactTransferContent400.ts | 20 + ...downloadAdminArtifactTransferContent401.ts | 20 + ...downloadAdminArtifactTransferContent403.ts | 20 + ...downloadAdminArtifactTransferContent404.ts | 20 + ...downloadAdminArtifactTransferContent409.ts | 20 + ...downloadAdminArtifactTransferContent410.ts | 20 + ...downloadAdminArtifactTransferContent413.ts | 20 + ...downloadAdminArtifactTransferContent415.ts | 20 + ...downloadAdminArtifactTransferContent422.ts | 20 + ...downloadAdminArtifactTransferContent429.ts | 20 + ...downloadAdminArtifactTransferContent500.ts | 20 + ...downloadAdminArtifactTransferContent503.ts | 20 + ...downloadAdminArtifactTransferContent507.ts | 20 + .../getAdminArtifactTransfer200.ts | 23 + .../getAdminArtifactTransfer200Cleanup.ts | 11 + .../getAdminArtifactTransfer200Direction.ts | 10 + .../getAdminArtifactTransfer200Purpose.ts | 11 + .../getAdminArtifactTransfer200State.ts | 17 + .../getAdminArtifactTransfer400.ts | 20 + .../getAdminArtifactTransfer401.ts | 20 + .../getAdminArtifactTransfer403.ts | 20 + .../getAdminArtifactTransfer404.ts | 20 + .../getAdminArtifactTransfer409.ts | 20 + .../getAdminArtifactTransfer410.ts | 20 + .../getAdminArtifactTransfer413.ts | 20 + .../getAdminArtifactTransfer415.ts | 20 + .../getAdminArtifactTransfer422.ts | 20 + .../getAdminArtifactTransfer429.ts | 20 + .../getAdminArtifactTransfer500.ts | 20 + .../getAdminArtifactTransfer503.ts | 20 + .../getAdminArtifactTransfer507.ts | 20 + .../generated/admin/service-models/index.ts | 105 + .../listAdminArtifactTransfers200.ts | 23 + .../listAdminArtifactTransfers200ItemsItem.ts | 23 + ...minArtifactTransfers200ItemsItemCleanup.ts | 11 + ...nArtifactTransfers200ItemsItemDirection.ts | 10 + ...minArtifactTransfers200ItemsItemPurpose.ts | 11 + ...AdminArtifactTransfers200ItemsItemState.ts | 17 + .../listAdminArtifactTransfers400.ts | 20 + .../listAdminArtifactTransfers401.ts | 20 + .../listAdminArtifactTransfers403.ts | 20 + .../listAdminArtifactTransfers404.ts | 20 + .../listAdminArtifactTransfers409.ts | 20 + .../listAdminArtifactTransfers410.ts | 20 + .../listAdminArtifactTransfers413.ts | 20 + .../listAdminArtifactTransfers415.ts | 20 + .../listAdminArtifactTransfers422.ts | 20 + .../listAdminArtifactTransfers429.ts | 20 + .../listAdminArtifactTransfers500.ts | 20 + .../listAdminArtifactTransfers503.ts | 20 + .../listAdminArtifactTransfers507.ts | 20 + .../listAdminArtifactTransfersParams.ts | 15 + .../uploadAdminArtifactTransferContent200.ts | 23 + ...dAdminArtifactTransferContent200Cleanup.ts | 11 + ...dminArtifactTransferContent200Direction.ts | 10 + ...dAdminArtifactTransferContent200Purpose.ts | 11 + ...oadAdminArtifactTransferContent200State.ts | 17 + .../uploadAdminArtifactTransferContent400.ts | 20 + .../uploadAdminArtifactTransferContent401.ts | 20 + .../uploadAdminArtifactTransferContent403.ts | 20 + .../uploadAdminArtifactTransferContent404.ts | 20 + .../uploadAdminArtifactTransferContent409.ts | 20 + .../uploadAdminArtifactTransferContent410.ts | 20 + .../uploadAdminArtifactTransferContent413.ts | 20 + .../uploadAdminArtifactTransferContent415.ts | 20 + .../uploadAdminArtifactTransferContent422.ts | 20 + .../uploadAdminArtifactTransferContent429.ts | 20 + .../uploadAdminArtifactTransferContent500.ts | 20 + .../uploadAdminArtifactTransferContent503.ts | 20 + .../uploadAdminArtifactTransferContent507.ts | 20 + packages/core/src/api/index.ts | 2 + packages/core/src/build-artifacts/index.ts | 1 + .../src/ops/artifact-transfer-capability.ts | 66 + .../src/ops/artifact-transfer-client.test.ts | 164 + .../core/src/ops/artifact-transfer-client.ts | 220 + .../core/src/ops/artifact-transfer.test.ts | 7 + packages/core/src/ops/artifact-transfer.ts | 150 + packages/core/src/ops/client.ts | 60 + packages/core/src/ops/contract.test.ts | 61 + packages/core/src/ops/contract.ts | 21 + packages/core/src/ops/index.ts | 3 + pnpm-lock.yaml | 3 + scripts/api-sdk/codegen.test.ts | 34 +- scripts/api-sdk/config.mjs | 83 +- scripts/ops-authority-inventory.json | 28 +- services/ops-agent/service.json | 13 +- services/traefik/config/traefik.yml | 7 + 156 files changed, 12131 insertions(+), 44 deletions(-) create mode 100644 apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts create mode 100644 apps/api/src/routes/admin-artifact-transfers.ts create mode 100644 apps/api/src/routes/admin-byte-response.ts create mode 100644 apps/api/src/services/artifact-transfers.ts create mode 100644 apps/ops-agent/src/artifact-transfer/errors.ts create mode 100644 apps/ops-agent/src/artifact-transfer/ingress.test.ts create mode 100644 apps/ops-agent/src/artifact-transfer/policy.test.ts create mode 100644 apps/ops-agent/src/artifact-transfer/policy.ts create mode 100644 apps/ops-agent/src/artifact-transfer/routes.test.ts create mode 100644 apps/ops-agent/src/artifact-transfer/routes.ts create mode 100644 apps/ops-agent/src/artifact-transfer/runtime.ts create mode 100644 apps/ops-agent/src/artifact-transfer/store.test.ts create mode 100644 apps/ops-agent/src/artifact-transfer/store.ts create mode 100644 packages/core/src/api/admin-byte-reader.ts create mode 100644 packages/core/src/api/admin-byte-transport.test.ts create mode 100644 packages/core/src/api/contracts/service-admin/artifact-transfers.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Cleanup.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Direction.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Purpose.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200State.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer404.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer410.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer422.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer507.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBody.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyDirection.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyPurpose.ts create mode 100644 packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferHeaders.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer404.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer410.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer422.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer507.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent404.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent410.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent422.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent507.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Cleanup.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Direction.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Purpose.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200State.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer404.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer410.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer422.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer507.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItem.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemCleanup.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemDirection.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemPurpose.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemState.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers404.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers410.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers422.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers507.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfersParams.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Cleanup.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Direction.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Purpose.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200State.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent404.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent410.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent422.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent507.ts create mode 100644 packages/core/src/ops/artifact-transfer-capability.ts create mode 100644 packages/core/src/ops/artifact-transfer-client.test.ts create mode 100644 packages/core/src/ops/artifact-transfer-client.ts create mode 100644 packages/core/src/ops/artifact-transfer.test.ts create mode 100644 packages/core/src/ops/artifact-transfer.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 375a49bae..8fcdb3a2f 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -12497,6 +12497,5329 @@ "x-openmapx-auth": "admin" } }, + "/api/admin/artifact-transfers": { + "get": { + "operationId": "listAdminArtifactTransfers", + "parameters": [ + { + "schema": { + "default": 50, + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "in": "query", + "name": "limit", + "required": false + }, + { + "schema": { + "default": 0, + "type": "integer", + "minimum": 0, + "maximum": 1000000 + }, + "in": "query", + "name": "offset", + "required": false + } + ], + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "items": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "stageId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "purpose": { + "type": "string", + "enum": [ + "prepared-artifact", + "instance-migration", + "integration-artifact" + ] + }, + "direction": { + "type": "string", + "enum": [ + "upload", + "download" + ] + }, + "state": { + "type": "string", + "enum": [ + "created", + "uploading", + "uploaded", + "validating", + "ready", + "claimed", + "consumed", + "failed", + "expired" + ] + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sha256": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "cleanup": { + "type": "string", + "enum": [ + "pending", + "complete", + "failed" + ] + } + }, + "required": [ + "stageId", + "purpose", + "direction", + "state", + "bytes", + "sha256", + "expiresAt", + "cleanup" + ], + "additionalProperties": false + } + }, + "total": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "limit": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "offset": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + } + }, + "required": [ + "items", + "total", + "limit", + "offset" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "507": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + }, + "post": { + "operationId": "createAdminArtifactTransfer", + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "enum": [ + "prepared-artifact", + "instance-migration", + "integration-artifact" + ] + }, + "direction": { + "type": "string", + "enum": [ + "upload" + ] + }, + "declaredBytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "expectedSha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "purpose", + "direction", + "declaredBytes" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "stageId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "purpose": { + "type": "string", + "enum": [ + "prepared-artifact", + "instance-migration", + "integration-artifact" + ] + }, + "direction": { + "type": "string", + "enum": [ + "upload", + "download" + ] + }, + "state": { + "type": "string", + "enum": [ + "created", + "uploading", + "uploaded", + "validating", + "ready", + "claimed", + "consumed", + "failed", + "expired" + ] + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sha256": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "cleanup": { + "type": "string", + "enum": [ + "pending", + "complete", + "failed" + ] + } + }, + "required": [ + "stageId", + "purpose", + "direction", + "state", + "bytes", + "sha256", + "expiresAt", + "cleanup" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "507": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/artifact-transfers/{stageId}": { + "get": { + "operationId": "getAdminArtifactTransfer", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "stageId", + "required": true + } + ], + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "stageId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "purpose": { + "type": "string", + "enum": [ + "prepared-artifact", + "instance-migration", + "integration-artifact" + ] + }, + "direction": { + "type": "string", + "enum": [ + "upload", + "download" + ] + }, + "state": { + "type": "string", + "enum": [ + "created", + "uploading", + "uploaded", + "validating", + "ready", + "claimed", + "consumed", + "failed", + "expired" + ] + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sha256": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "cleanup": { + "type": "string", + "enum": [ + "pending", + "complete", + "failed" + ] + } + }, + "required": [ + "stageId", + "purpose", + "direction", + "state", + "bytes", + "sha256", + "expiresAt", + "cleanup" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "507": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + }, + "delete": { + "operationId": "deleteAdminArtifactTransfer", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "stageId", + "required": true + } + ], + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "deleted": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "deleted" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "507": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/artifact-transfers/{stageId}/content": { + "get": { + "operationId": "downloadAdminArtifactTransferContent", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "stageId", + "required": true + } + ], + "responses": { + "200": { + "description": "Response", + "headers": { + "content-type": { + "schema": { + "type": "string", + "const": "application/octet-stream" + } + }, + "content-length": { + "schema": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + } + }, + "content-disposition": { + "schema": { + "type": "string", + "const": "attachment; filename=\"openmapx-artifact.tar\"" + } + }, + "x-artifact-sha256": { + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "cache-control": { + "schema": { + "type": "string", + "const": "no-store" + } + }, + "x-content-type-options": { + "schema": { + "type": "string", + "const": "nosniff" + } + } + }, + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "507": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin", + "x-openmapx-native-download": true + }, + "put": { + "operationId": "uploadAdminArtifactTransferContent", + "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "path", + "name": "stageId", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "stageId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "purpose": { + "type": "string", + "enum": [ + "prepared-artifact", + "instance-migration", + "integration-artifact" + ] + }, + "direction": { + "type": "string", + "enum": [ + "upload", + "download" + ] + }, + "state": { + "type": "string", + "enum": [ + "created", + "uploading", + "uploaded", + "validating", + "ready", + "claimed", + "consumed", + "failed", + "expired" + ] + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sha256": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "cleanup": { + "type": "string", + "enum": [ + "pending", + "complete", + "failed" + ] + } + }, + "required": [ + "stageId", + "purpose", + "direction", + "state", + "bytes", + "sha256", + "expiresAt", + "cleanup" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "404": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "410": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "422": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "507": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/admin/audit": { "get": { "operationId": "getApiAdminAudit", diff --git a/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts b/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts new file mode 100644 index 000000000..46dd32378 --- /dev/null +++ b/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts @@ -0,0 +1,206 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import cors from "@fastify/cors"; +import { createApiClient } from "@openmapx/core"; +import { + createAdminArtifactTransfer, + downloadAdminArtifactTransferContent, + getAdminArtifactTransfer, + listAdminArtifactTransfers, + uploadAdminArtifactTransferContent, +} from "@openmapx/core/api-admin-sdk"; +import Fastify from "fastify"; +import { afterEach, expect, it, vi } from "vitest"; +import { installArtifactTransferRuntime } from "../../../../ops-agent/src/artifact-transfer/runtime"; +import { ArtifactTransferStore } from "../../../../ops-agent/src/artifact-transfer/store"; +import { createUnavailableRuntime } from "../../../../ops-agent/src/runtime"; +import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; +import { corsOptions } from "../../server-wiring"; +import { adminArtifactTransfersRoute } from "../admin-artifact-transfers"; + +// Authentication storage only is synthetic. Actual full-admin/session-owner guard and both HTTP brokers run unchanged. +vi.mock("../../auth", () => ({ + auth: { + api: { + getSession: async ({ headers }: { headers: Headers }) => { + const who = headers.get("x-fixture-session"); + return who + ? { user: { id: who, role: who === "member" ? "user" : "admin" }, session: { id: who } } + : null; + }, + }, + }, +})); +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const close of cleanups.splice(0).reverse()) await close(); + vi.unstubAllEnvs(); +}); +it("generated SDK streams through real admin/API and typed/raw agent paths, enforcing owner, CSRF and media", async () => { + const root = await mkdtemp(join(tmpdir(), "artifact-admin-")); + cleanups.push(() => rm(root, { recursive: true, force: true })); + const token = "a".repeat(43); + const tokenFile = join(root, "token"); + await writeFile(tokenFile, token, { mode: 0o600 }); + const store = new ArtifactTransferStore({ + dataRoot: join(root, "data"), + privateRoot: join(root, "private"), + token, + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 10000 }), + }); + const runtime = createUnavailableRuntime(); + installArtifactTransferRuntime(runtime, store); + const audit: unknown[] = []; + const agent = buildOpsAgentServer({ + audit: (event) => audit.push(event), + tokens: { api: token, "data-manager": "b".repeat(43) }, + runtime, + artifactTransfers: store, + resourcePolicy: { allowGlobal: () => true }, + }); + cleanups.push(async () => { + await agent.close(); + store.close(); + }); + const agentUrl = await agent.listen({ host: "127.0.0.1", port: 0 }); + vi.stubEnv("OPS_AGENT_URL", agentUrl); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); + vi.stubEnv("OPENMAPX_DISABLE_LOCALHOST_AUTH", "1"); + const app = Fastify(); + await app.register(cors, corsOptions(["https://web.example"])); + await app.register(adminArtifactTransfersRoute, { prefix: "/api" }); + cleanups.push(() => app.close()); + const url = await app.listen({ host: "127.0.0.1", port: 0 }); + const client = createApiClient({ + baseUrl: url, + credentials: "omit", + headerInterceptor: () => ({ "x-fixture-session": "admin", Origin: "https://web.example" }), + }); + const bytes = new Uint8Array([0, 255, 192, 128]); + const result = await createAdminArtifactTransfer( + { purpose: "prepared-artifact", direction: "upload", declaredBytes: 4 }, + { "idempotency-key": "binary-test" }, + { client }, + ); + expect(result.status, JSON.stringify({ result: result.data, audit })).toBe(200); + if (result.status !== 200) throw Error("No stage"); + const stage = result.data; + const uploaded = await uploadAdminArtifactTransferContent( + stage.stageId, + new Blob([bytes], { type: "text/plain" }), + { client }, + ); + expect(uploaded.status).toBe(200); + expect(uploaded.headers.get("access-control-allow-origin")).toBe("https://web.example"); + expect(uploaded.headers.get("access-control-expose-headers")).toContain("X-Artifact-Sha256"); + if (uploaded.status !== 200) throw Error("No upload"); + expect(uploaded.data).toMatchObject({ + bytes: 4, + state: "uploaded", + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + expect((await getAdminArtifactTransfer(stage.stageId, { client })).status).toBe(200); + const other = createApiClient({ + baseUrl: url, + headerInterceptor: () => ({ "x-fixture-session": "other" }), + }); + expect((await getAdminArtifactTransfer(stage.stageId, { client: other })).status).toBe(404); + const noSession = await fetch(`${url}/api/admin/artifact-transfers`); + expect(noSession.status).toBe(401); + const member = await fetch(`${url}/api/admin/artifact-transfers`, { + headers: { "x-fixture-session": "member" }, + }); + expect(member.status).toBe(403); + expect( + ( + await fetch(`${url}/api/admin/artifact-transfers/${stage.stageId}/content`, { + method: "PUT", + headers: { + "x-fixture-session": "admin", + cookie: "session=x", + origin: "https://foreign.invalid", + "content-type": "application/octet-stream", + }, + body: bytes, + }) + ).status, + ).toBe(403); + expect( + ( + await fetch(`${url}/api/admin/artifact-transfers/${stage.stageId}/content`, { + method: "PUT", + headers: { "x-fixture-session": "admin", "content-type": "text/plain" }, + body: bytes, + }) + ).status, + ).toBe(415); + expect((await listAdminArtifactTransfers({ limit: 50, offset: 0 }, { client })).status).toBe(200); + let callbacks = 0; + const premature = await downloadAdminArtifactTransferContent(stage.stageId, { + client, + onByteChunk: () => { + callbacks++; + }, + }); + expect(premature.status).toBe(409); + expect(callbacks).toBe(0); +}, 30000); + +it("canonical byte-send adapter and generated download preserve non-UTF8 EOF and headers", async () => { + const { createAdminRouteRegistrar } = await import("../admin-contract"); + const { sendAdminByteResponse } = await import("../admin-byte-response"); + const { artifactTransferAdminOperations } = await import("@openmapx/core/api-contracts"); + const app = Fastify(); + cleanups.push(() => app.close()); + let canceled = false; + await app.register( + async (scoped) => { + const register = createAdminRouteRegistrar(artifactTransferAdminOperations, { + tag: "service-admin", + validationPhase: "preHandler", + }); + register(scoped, "downloadAdminArtifactTransferContent", async (_req, reply) => { + const body = new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array([0, 255, 192, 128])); + c.close(); + }, + cancel() { + canceled = true; + }, + }); + await sendAdminByteResponse(reply, { + body, + signal: new AbortController().signal, + headers: new Headers({ + "content-type": "application/octet-stream", + "content-length": "4", + "content-disposition": 'attachment; filename="openmapx-artifact.tar"', + "x-artifact-sha256": "a".repeat(64), + "cache-control": "no-store", + "x-content-type-options": "nosniff", + }), + }); + }); + }, + { prefix: "/api" }, + ); + const url = await app.listen({ host: "127.0.0.1", port: 0 }); + const client = createApiClient({ baseUrl: url, credentials: "omit" }); + const bytes: number[] = []; + const result = await downloadAdminArtifactTransferContent( + "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + { + client, + onByteChunk: (chunk) => { + bytes.push(...chunk); + }, + }, + ); + expect(result.status).toBe(200); + expect(result.data).toBeUndefined(); + expect(bytes).toEqual([0, 255, 192, 128]); + expect(canceled).toBe(false); +}); diff --git a/apps/api/src/routes/admin-artifact-transfers.ts b/apps/api/src/routes/admin-artifact-transfers.ts new file mode 100644 index 000000000..f0cac3d22 --- /dev/null +++ b/apps/api/src/routes/admin-artifact-transfers.ts @@ -0,0 +1,102 @@ +import { Readable } from "node:stream"; +import { artifactTransferAdminOperations } from "@openmapx/core/api-contracts"; +import type { FastifyInstance } from "fastify"; +import { AdminArtifactTransferError, AdminArtifactTransfers } from "../services/artifact-transfers"; +import { configuredTrustedWebOrigins, makeCsrfGuardHook } from "../utils/csrf"; +import { getAdminSession, requireAdmin } from "../utils/require-admin"; +import { declareRouteAuth } from "../utils/route-auth"; +import { sendAdminByteResponse } from "./admin-byte-response"; +import { createAdminRouteRegistrar } from "./admin-contract"; + +const register = createAdminRouteRegistrar(artifactTransferAdminOperations, { + tag: "service-admin", + validationPhase: "preHandler", +}); +export async function adminArtifactTransfersRoute(app: FastifyInstance): Promise { + declareRouteAuth(app, "admin"); + // onRequest runs before the scoped octet parser can expose body bytes. + app.addHook("onRequest", makeCsrfGuardHook(configuredTrustedWebOrigins())); + app.addHook("onRequest", async (request) => { + request.adminSession = await requireAdmin(request); + }); + app.addContentTypeParser("application/octet-stream", (_request, payload, done) => + done(null, payload), + ); + app.setErrorHandler((error, _request, reply) => { + if (reply.raw.headersSent) { + reply.raw.destroy(); + return; + } + const status = + error instanceof AdminArtifactTransferError + ? error.statusCode + : (error as { statusCode?: number }).statusCode; + return reply + .code( + status && [400, 401, 403, 404, 409, 410, 413, 415, 422, 429, 503, 507].includes(status) + ? status + : 503, + ) + .send({ + error: + error instanceof AdminArtifactTransferError + ? error.message + : "Artifact transfer unavailable", + }); + }); + const broker = () => new AdminArtifactTransfers(); + const owner = (request: Parameters[0]) => + getAdminSession(request).user.id; + register(app, "listAdminArtifactTransfers", (req) => + broker().list(owner(req), req.query.limit, req.query.offset), + ); + register(app, "createAdminArtifactTransfer", (req) => + broker().create(owner(req), String(req.headers["idempotency-key"]), req.body), + ); + register(app, "getAdminArtifactTransfer", (req) => + broker().inspect(owner(req), req.params.stageId), + ); + register(app, "deleteAdminArtifactTransfer", (req) => + broker().delete(owner(req), req.params.stageId), + ); + register(app, "uploadAdminArtifactTransferContent", async (req, reply) => { + if ( + req.headers["content-type"]?.split(";")[0].trim().toLowerCase() !== "application/octet-stream" + ) + return reply.code(415).send({ error: "Artifact upload requires application/octet-stream" }); + const controller = new AbortController(); + const aborted = () => { + if (!reply.raw.writableFinished) controller.abort(); + }; + req.raw.on("aborted", aborted); + reply.raw.on("close", aborted); + try { + return await broker().upload( + owner(req), + req.params.stageId, + Readable.toWeb(req.body as Readable) as ReadableStream, + controller.signal, + req.headers["content-length"] === undefined + ? undefined + : Number(req.headers["content-length"]), + ); + } finally { + req.raw.off("aborted", aborted); + reply.raw.off("close", aborted); + } + }); + register(app, "downloadAdminArtifactTransferContent", async (req, reply) => { + const controller = new AbortController(); + const aborted = () => { + if (!reply.raw.writableFinished) controller.abort(); + }; + reply.raw.on("close", aborted); + try { + await broker().download(owner(req), req.params.stageId, controller.signal, (input) => + sendAdminByteResponse(reply, input), + ); + } finally { + reply.raw.off("close", aborted); + } + }); +} diff --git a/apps/api/src/routes/admin-byte-response.ts b/apps/api/src/routes/admin-byte-response.ts new file mode 100644 index 000000000..fa6e248f7 --- /dev/null +++ b/apps/api/src/routes/admin-byte-response.ts @@ -0,0 +1,53 @@ +import { Readable } from "node:stream"; +import { finished } from "node:stream/promises"; +import type { AdminResponseContract } from "@openmapx/core/api-contracts"; +import type { FastifyReply } from "fastify"; + +const contracts = new WeakMap>(); +const sending = new WeakSet(); +export function bindAdminByteResponse( + reply: FastifyReply, + responses: Record, +): void { + contracts.set(reply, responses); +} +export function isAdminByteResponse(reply: FastifyReply): boolean { + return sending.has(reply); +} +/** Sole canonical byte-send adapter. Its promise owns destination completion/abort. */ +export async function sendAdminByteResponse( + reply: FastifyReply, + input: { + body: ReadableStream; + signal: AbortSignal; + headers: Headers; + status?: number; + }, +): Promise { + const status = input.status ?? 200; + const contract = contracts.get(reply)?.[status]; + if ( + contract?.kind !== "bytes" || + input.headers.get("content-type")?.split(";")[0] !== contract.mediaType || + (contract.headers && !contract.headers.safeParse(Object.fromEntries(input.headers)).success) + ) + throw Error("Invalid administration byte response"); + input.signal.throwIfAborted(); + reply.code(status); + input.headers.forEach((v, k) => { + reply.header(k, v); + }); + sending.add(reply); + const source = Readable.fromWeb( + input.body as import("node:stream/web").ReadableStream, + { signal: input.signal }, + ); + const completion = finished(reply.raw, { signal: input.signal, cleanup: true }); + reply.send(source); + try { + await completion; + } finally { + source.destroy(); + sending.delete(reply); + } +} diff --git a/apps/api/src/routes/admin-contract.ts b/apps/api/src/routes/admin-contract.ts index f0a009474..9fd349785 100644 --- a/apps/api/src/routes/admin-contract.ts +++ b/apps/api/src/routes/admin-contract.ts @@ -1,5 +1,10 @@ import { STATUS_CODES } from "node:http"; -import type { AdminOperation, AdminResponseContract } from "@openmapx/core/api-contracts"; +import type {} from "@fastify/swagger"; +import { + type AdminOperation, + type AdminResponseContract, + assertAdminOperationContract, +} from "@openmapx/core/api-contracts"; import { routeJsonSchema } from "@openmapx/core/api-schema"; import type { FastifyContextConfig, @@ -10,6 +15,7 @@ import type { RouteGenericInterface, } from "fastify"; import type z from "zod/v4"; +import { bindAdminByteResponse, isAdminByteResponse } from "./admin-byte-response"; type Registry = Record; type ContractPart = R[K] extends { @@ -47,6 +53,7 @@ export function createAdminRouteRegistrar( options: RegistrationOptions = {}, ): void { const operation = registry[operationId]; + assertAdminOperationContract(operation); const request = ("request" in operation ? operation.request : {}) as Record; const responses = operation.responses as Record; const convert = (validator: z.ZodType, identity: string, io: "input" | "output") => @@ -60,7 +67,9 @@ export function createAdminRouteRegistrar( const body = "schema" in contract ? convert(contract.schema, `${operationId}_${status}`, "output") - : { type: "string" }; + : contract.kind === "bytes" + ? { type: "string", format: "binary" } + : { type: "string" }; const media = contract.kind === "json" ? "application/json" @@ -132,9 +141,21 @@ export function createAdminRouteRegistrar( } as FastifyContextConfig & { auth: "admin" | "service"; adminOptionalBody?: boolean }, schema: { operationId, + ...(Object.values(responses).some((c) => c.kind === "bytes" && c.nativeDownload) + ? { "x-openmapx-native-download": true } + : {}), tags: [family.tag], ...requestSchemas, response: responseSchemas, + ...(operation.requestBody + ? { + body: { type: "string", format: "binary" }, + consumes: [operation.requestBody.mediaType], + } + : {}), + }, + onRequest: async (_request, reply) => { + bindAdminByteResponse(reply, responses); }, validatorCompiler: () => (value) => ({ value }), preValidation: family.validationPhase === "preValidation" ? validateRequest : undefined, @@ -149,6 +170,10 @@ export function createAdminRouteRegistrar( ({ httpStatus }) => (payload) => { const status = Number(httpStatus); + if (responses[status]?.kind === "bytes") + throw new Error( + "Administration byte response must be streamed through its send adapter", + ); let wire: unknown; if (payload instanceof Error) { const error = payload as Error & { error?: unknown; code?: unknown }; @@ -181,6 +206,14 @@ export function createAdminRouteRegistrar( if (!contract.headers.safeParse(normalized).success) throw new Error("Invalid administration response headers"); } + if (contract.kind === "bytes") { + if ( + !isAdminByteResponse(reply) || + reply.getHeader("content-type")?.toString().split(";")[0] !== contract.mediaType + ) + throw Error("Administration bytes require the canonical send adapter"); + return payload; + } if (contract.kind === "stream") throw new Error("Administration streams require a raw response"); if (contract.kind === "empty") { diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index 8869d00fc..3785f2479 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -2,6 +2,7 @@ import type { FastifyInstance } from "fastify"; import { adminRoute } from "./admin"; import { adminAccountsRoute } from "./admin-accounts"; import { adminActivityRoute } from "./admin-activity"; +import { adminArtifactTransfersRoute } from "./admin-artifact-transfers"; import { adminCacheRoute } from "./admin-cache"; import { registerCapabilityBindingRoutes } from "./admin-capability-bindings"; import { registerAdminComposeRoutes } from "./admin-compose"; @@ -119,6 +120,7 @@ export async function registerCoreRoutes( await server.register(statusRoute, { prefix: "/api" }); await server.register(adminRoute, { prefix: "/api" }); await server.register(adminServicesRoute, { prefix: "/api" }); + await server.register(adminArtifactTransfersRoute, { prefix: "/api" }); await server.register(adminTransitCredentialsRoute, { prefix: "/api" }); await server.register(adminTransitPinsRoute, { prefix: "/api" }); await server.register(adminOperationsRoute, { prefix: "/api" }); diff --git a/apps/api/src/server-wiring.ts b/apps/api/src/server-wiring.ts index e51c7cd9b..5c5c419fb 100644 --- a/apps/api/src/server-wiring.ts +++ b/apps/api/src/server-wiring.ts @@ -190,6 +190,9 @@ export function corsOptions(trustedWebOrigins: readonly string[] = configuredTru // Browser clients need these response headers when the web app and API are // on different origins (the default local-development topology). exposedHeaders: [ + "Content-Length", + "Content-Disposition", + "X-Artifact-Sha256", "X-Tile-Source", "Accept-Ranges", "Content-Range", diff --git a/apps/api/src/services/artifact-transfers.ts b/apps/api/src/services/artifact-transfers.ts new file mode 100644 index 000000000..841a372f7 --- /dev/null +++ b/apps/api/src/services/artifact-transfers.ts @@ -0,0 +1,125 @@ +import { + ArtifactStageTransportError, + type OpsClient, + OpsClientError, + type OpsOperation, + type OpsOperationKind, + type OpsResultFor, + type TransferStage, +} from "@openmapx/core/ops"; +import { createApiOpsClient, createDurableOpsKey } from "./ops-client"; +export class AdminArtifactTransferError extends Error { + constructor(readonly statusCode: number) { + super("Artifact transfer unavailable; inspect the retained stage before retrying"); + } +} +export class AdminArtifactTransfers { + constructor(private readonly client: OpsClient = createApiOpsClient()) {} + private async execute( + operation: Extract, + ): Promise> { + try { + const result = await this.client.execute(operation); + if (!("execution" in result) || result.execution !== "sync") throw Error(); + return result.value; + } catch (error) { + if (error instanceof OpsClientError) + throw new AdminArtifactTransferError( + error.status && [404, 409, 410, 413, 415, 422, 429, 503, 507].includes(error.status) + ? error.status + : 503, + ); + throw error; + } + } + list(ownerId: string, limit = 50, offset = 0) { + return this.execute({ kind: "artifactTransfer.list", ownerId, limit, offset }); + } + async inspect(ownerId: string, stageId: string): Promise { + const list = await this.list(ownerId, 100, 0); + const found = list.items.find((s) => s.stageId === stageId); + if (!found) throw new AdminArtifactTransferError(404); + return this.execute({ + kind: "artifactTransfer.inspect", + stageId, + ownerId, + purpose: found.purpose, + }); + } + create( + ownerId: string, + key: string, + input: { + purpose: "prepared-artifact" | "instance-migration" | "integration-artifact"; + direction: "upload"; + declaredBytes: number; + expectedSha256?: string; + }, + ) { + return this.execute({ + kind: "artifactTransfer.create", + ...input, + ownerId, + operationKey: createDurableOpsKey("artifact-transfer.create", `${ownerId}:${key}`), + }); + } + async delete(ownerId: string, stageId: string) { + return this.execute({ kind: "artifactTransfer.delete", stageId, ownerId }); + } + async upload( + ownerId: string, + stageId: string, + body: ReadableStream, + signal: AbortSignal, + contentLength?: number, + ) { + const stage = await this.inspect(ownerId, stageId); + try { + return await this.client.uploadArtifactStage({ + stageId, + ownerId, + purpose: stage.purpose, + body, + contentLength, + signal, + executionDeadlineAt: new Date( + Math.min(Date.parse(stage.expiresAt), Date.now() + 72 * 3600_000), + ).toISOString(), + }); + } catch (error) { + if (error instanceof ArtifactStageTransportError) + throw new AdminArtifactTransferError(error.status); + throw error; + } + } + async download( + ownerId: string, + stageId: string, + signal: AbortSignal, + consume: (input: { + body: ReadableStream; + signal: AbortSignal; + headers: Headers; + }) => Promise, + ): Promise { + const stage = await this.inspect(ownerId, stageId); + try { + return await this.client.consumeArtifactStage( + { + stageId, + ownerId, + purpose: stage.purpose, + signal, + executionDeadlineAt: new Date( + Math.min(Date.parse(stage.expiresAt), Date.now() + 72 * 3600_000), + ).toISOString(), + }, + consume, + ); + } catch (error) { + if (error instanceof ArtifactStageTransportError) + throw new AdminArtifactTransferError(error.status); + throw error; + } + } +} diff --git a/apps/ops-agent/package.json b/apps/ops-agent/package.json index d9a0e0414..2ca7e1bbc 100644 --- a/apps/ops-agent/package.json +++ b/apps/ops-agent/package.json @@ -14,7 +14,8 @@ "dependencies": { "@openmapx/core": "workspace:^", "@openmapx/transitous-core": "workspace:^", - "fastify": "^5.12.7" + "fastify": "^5.12.7", + "zod": "^4.6.5" }, "devDependencies": { "@types/node": "^26.6.5", diff --git a/apps/ops-agent/src/artifact-transfer/errors.ts b/apps/ops-agent/src/artifact-transfer/errors.ts new file mode 100644 index 000000000..002b4a18a --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/errors.ts @@ -0,0 +1,9 @@ +export class ArtifactTransferError extends Error { + constructor( + readonly statusCode: 404 | 409 | 410 | 413 | 415 | 422 | 429 | 503 | 507, + message: string, + ) { + super(message); + this.name = "ArtifactTransferError"; + } +} diff --git a/apps/ops-agent/src/artifact-transfer/ingress.test.ts b/apps/ops-agent/src/artifact-transfer/ingress.test.ts new file mode 100644 index 000000000..3ee885c2c --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/ingress.test.ts @@ -0,0 +1,183 @@ +/** Opt-in exact-image ingress check; synthetic bytes only. No host Docker socket is mounted. */ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { promisify } from "node:util"; +import { createApiClient } from "@openmapx/core"; +import { uploadAdminArtifactTransferContent } from "@openmapx/core/api-admin-sdk"; +import Fastify from "fastify"; +import { expect, it, vi } from "vitest"; +import { adminArtifactTransfersRoute } from "../../../api/src/routes/admin-artifact-transfers"; +import { createUnavailableRuntime } from "../runtime"; +import { buildOpsAgentServer } from "../server"; +import { installArtifactTransferRuntime } from "./runtime"; +import { ArtifactTransferStore } from "./store"; + +const authFixture = vi.hoisted(() => ({ token: "" })); +vi.mock("../../../api/src/auth", () => ({ + auth: { + api: { + getSession: async ({ headers }: { headers: Headers }) => + headers.get("authorization") === `Bearer ${authFixture.token}` + ? { user: { id: "ingress-admin", role: "admin" }, session: { id: "ingress" } } + : null, + }, + }, +})); +const run = promisify(execFile); +it.skipIf(process.env.RUN_ARTIFACT_INGRESS_TESTS !== "1")( + "ships a nonbuffering 72h entrypoint and streams beyond the old 30s agent body deadline", + async () => { + const root = await mkdtemp(join(tmpdir(), "openmapx-artifact-ingress-")); + const name = `openmapx-artifact-ingress-${randomUUID().slice(0, 8)}`; + let containerId: string | undefined; + const token = "a".repeat(43); + authFixture.token = randomUUID(); + const tokenFile = join(root, "token"); + await writeFile(tokenFile, token, { mode: 0o600 }); + const store = new ArtifactTransferStore({ + dataRoot: join(root, "data"), + privateRoot: join(root, "private"), + token, + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 10000 }), + }); + const runtime = createUnavailableRuntime(); + installArtifactTransferRuntime(runtime, store); + const agent = buildOpsAgentServer({ + tokens: { api: token, "data-manager": "b".repeat(43) }, + runtime, + artifactTransfers: store, + resourcePolicy: { allowGlobal: () => true }, + }); + const api = Fastify(); + const prior = { + OPS_AGENT_URL: process.env.OPS_AGENT_URL, + OPS_AGENT_TOKEN_FILE: process.env.OPS_AGENT_TOKEN_FILE, + OPENMAPX_DISABLE_LOCALHOST_AUTH: process.env.OPENMAPX_DISABLE_LOCALHOST_AUTH, + }; + try { + process.env.OPS_AGENT_URL = await agent.listen({ host: "127.0.0.1", port: 0 }); + process.env.OPS_AGENT_TOKEN_FILE = tokenFile; + process.env.OPENMAPX_DISABLE_LOCALHOST_AUTH = "1"; + await api.register(adminArtifactTransfersRoute, { prefix: "/api" }); + await api.listen({ host: "0.0.0.0", port: 0 }); + const address = api.server.address(); + if (!address || typeof address === "string") throw Error("No owned API address"); + const shipped = await readFile(resolve("services/traefik/config/traefik.yml"), "utf8"); + expect(shipped).toContain("readTimeout: 72h"); + expect(shipped).toContain("writeTimeout: 72h"); + expect(shipped).not.toContain("buffering:"); + // Exercise the shipped static timeout block over HTTP; remove only TLS/ACME/QUIC setup to avoid certificate-network effects in this local fixture. + const configuration = shipped + .replace(/ {4}http3:\n {6}advertisedPort: 443\n/, "") + .replace(/ {6}tls:\n {8}certResolver: letsencrypt\n/, "") + .replace(/\ncertificatesResolvers:\n[\s\S]*?(?=\n# Routing is generated)/, ""); + await writeFile(join(root, "traefik.yml"), configuration); + await mkdir(join(root, "dynamic")); + await writeFile( + join(root, "dynamic", "routes.yml"), + `http:\n routers:\n upload:\n rule: PathPrefix(\`/api/admin/artifact-transfers\`)\n entryPoints: [websecure]\n service: api\n services:\n api:\n loadBalancer:\n servers:\n - url: http://host.docker.internal:${address.port}\n middlewares:\n hsts:\n headers:\n stsSeconds: 31536000\n`, + ); + containerId = ( + await run("docker", [ + "run", + "--detach", + "--rm", + "--name", + name, + "--label", + "openmapx.test=artifact-ingress", + "--memory", + "256m", + "-p", + "127.0.0.1::443", + "--mount", + `type=bind,source=${root}/traefik.yml,target=/etc/traefik/traefik.yml,readonly`, + "--mount", + `type=bind,source=${root}/dynamic,target=/etc/traefik/dynamic,readonly`, + "traefik@sha256:575fa15b135078fe5e50aa847987d96dbddd7b093c172429618404df73f3fa7c", + ]) + ).stdout.trim(); + const mapping = (await run("docker", ["port", containerId, "443/tcp"])).stdout.trim(); + const port = Number(mapping.split(":").at(-1)); + const client = createApiClient({ + baseUrl: `http://127.0.0.1:${port}`, + credentials: "omit", + headerInterceptor: () => ({ Authorization: `Bearer ${authFixture.token}` }), + }); + let reachable = false; + for (let attempt = 0; attempt < 30; attempt++) { + try { + const response = await fetch(`http://127.0.0.1:${port}/api/admin/artifact-transfers`, { + headers: { Authorization: `Bearer ${authFixture.token}` }, + signal: AbortSignal.timeout(1000), + }); + if (response.status === 200) { + reachable = true; + break; + } + } catch {} + await delay(200); + } + expect(reachable, (await run("docker", ["logs", containerId])).stderr).toBe(true); + const stage = await store.createStage({ + ownerId: "ingress-admin", + purpose: "prepared-artifact", + direction: "upload", + declaredBytes: 36, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + }); + class SlowBlob extends Blob { + override stream(): ReadableStream> { + let sent = 0; + return new ReadableStream( + { + async pull(c) { + if (sent === 36) { + c.close(); + return; + } + await delay(1000); + c.enqueue(new Uint8Array([sent++])); + }, + }, + { highWaterMark: 0 }, + ); + } + } + const started = Date.now(); + const upload = uploadAdminArtifactTransferContent( + stage.stageId, + new SlowBlob([new Uint8Array(36)]), + { client, timeoutMs: 60000 }, + ); + await delay(6000); + const inFlight = await store.getStage({ + stageId: stage.stageId, + ownerId: "ingress-admin", + purpose: stage.purpose, + }); + expect(inFlight.bytes).toBeGreaterThan(0); + expect(inFlight.bytes).toBeLessThan(36); + const result = await upload; + expect(Date.now() - started).toBeGreaterThan(30000); + expect(result.status).toBe(200); + if (result.status === 200) + expect(result.data).toMatchObject({ state: "uploaded", bytes: 36 }); + } finally { + if (containerId) await run("docker", ["rm", "--force", containerId]).catch(() => {}); + await api.close(); + await agent.close(); + store.close(); + for (const [k, v] of Object.entries(prior)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + await rm(root, { recursive: true, force: true }); + } + }, + 120000, +); diff --git a/apps/ops-agent/src/artifact-transfer/policy.test.ts b/apps/ops-agent/src/artifact-transfer/policy.test.ts new file mode 100644 index 000000000..d2c44dba4 --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/policy.test.ts @@ -0,0 +1,15 @@ +import { expect, it } from "vitest"; +import { artifactTransferPolicySchema, DEFAULT_TRANSFER_POLICY, purposeByteLimit } from "./policy"; + +it("uses ceilings, purpose caps and explicit memory accounting", () => { + expect(DEFAULT_TRANSFER_POLICY.maxUploadBytes).toBe(1024 ** 4); + expect(purposeByteLimit(DEFAULT_TRANSFER_POLICY, "integration-artifact")).toBe(200 * 1024 ** 2); + expect(purposeByteLimit(DEFAULT_TRANSFER_POLICY, "instance-migration")).toBe(64 * 1024 ** 2); + expect( + artifactTransferPolicySchema.safeParse({ maxManifestBytes: 256 * 1024 ** 2 }).success, + ).toBe(false); + expect(artifactTransferPolicySchema.safeParse({ maxUploadBytes: 9 * 1024 ** 4 }).success).toBe( + false, + ); + expect(artifactTransferPolicySchema.safeParse({ unknown: true }).success).toBe(false); +}); diff --git a/apps/ops-agent/src/artifact-transfer/policy.ts b/apps/ops-agent/src/artifact-transfer/policy.ts new file mode 100644 index 000000000..2f0fc4c1b --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/policy.ts @@ -0,0 +1,90 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import z from "zod/v4"; + +const TiB = 1024 ** 4; +const integer = (max: number) => z.number().int().positive().safe().max(max); +export const artifactTransferPolicySchema = z + .strictObject({ + maxUploadBytes: integer(8 * TiB).default(TiB), + maxExpandedBytes: integer(16 * TiB).default(2 * TiB), + maxStageBytes: integer(32 * TiB).default(4 * TiB), + maxEntryBytes: integer(16 * TiB).default(2 * TiB), + maxEntries: integer(1_000_000).default(250_000), + maxManifestBytes: integer(256 * 1024 ** 2).default(64 * 1024 ** 2), + maxPathBytes: integer(4096).default(4096), + maxPathDepth: integer(64).default(64), + maxExtendedHeaderBytes: integer(16384).default(16384), + maxExpansionRatio: integer(1000).default(1000), + expansionRatioThresholdBytes: integer(64 * 1024 ** 2).default(64 * 1024 ** 2), + maxConcurrentArchiveJobs: integer(4).default(1), + maxConcurrentTransfers: integer(4).default(2), + maxConcurrentTransfersPerPrincipal: integer(4).default(1), + maxRetainedStagesPerPrincipal: integer(100).default(4), + idleMs: integer(5 * 60_000).default(5 * 60_000), + transferMs: integer(7 * 86400_000).default(72 * 3600_000), + stageMs: integer(8 * 86400_000).default(96 * 3600_000), + downloadRetentionMs: integer(24 * 3600_000).default(24 * 3600_000), + memoryBudgetBytes: integer(16 * 1024 ** 3).default(1024 ** 3), + }) + .refine( + (p) => + p.maxEntryBytes <= p.maxExpandedBytes && + p.maxUploadBytes <= p.maxStageBytes && + p.stageMs > p.transferMs && + p.maxConcurrentTransfersPerPrincipal <= p.maxConcurrentTransfers && + p.maxConcurrentArchiveJobs <= p.maxConcurrentTransfers, + ) + .refine( + (p) => + p.memoryBudgetBytes >= + p.maxManifestBytes * 4 + p.maxEntries * 2048 + p.maxConcurrentTransfers * 8 * 1024 ** 2, + "Transfer metadata exceeds declared memory budget", + ); +export type ArtifactTransferPolicy = z.infer; +export const DEFAULT_TRANSFER_POLICY = artifactTransferPolicySchema.parse({}); +export function transferPolicyRevision(policy: ArtifactTransferPolicy): string { + return createHash("sha256").update(JSON.stringify(policy)).digest("hex"); +} +function availableMemoryBytes(): number { + const constrained = process.constrainedMemory?.() ?? 0; + return constrained > 0 ? constrained : Infinity; +} +function checkedPolicy( + policy: ArtifactTransferPolicy, +): { policy: ArtifactTransferPolicy; revision: string } | { reason: string } { + if (policy.memoryBudgetBytes > availableMemoryBytes()) + return { + reason: + "Artifact transfer policy exceeds the agent memory allocation; lower limits or raise the container memory allocation", + }; + return { policy, revision: transferPolicyRevision(policy) }; +} +export function loadArtifactTransferPolicy( + path = process.env.OPENMAPX_ARTIFACT_TRANSFER_POLICY_FILE ?? + "/run/openmapx/artifact-transfer-policy.json", +): { policy: ArtifactTransferPolicy; revision: string } | { reason: string } { + try { + const parsed = z + .strictObject({ artifactTransfer: artifactTransferPolicySchema }) + .parse(JSON.parse(readFileSync(path, "utf8"))); + return checkedPolicy(parsed.artifactTransfer); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return checkedPolicy(DEFAULT_TRANSFER_POLICY); + return { + reason: + "Invalid artifact transfer policy; correct the deployment policy and restart the operations agent", + }; + } +} +export function purposeByteLimit(policy: ArtifactTransferPolicy, purpose: string): number { + return Math.min( + policy.maxUploadBytes, + purpose === "integration-artifact" + ? 200 * 1024 ** 2 + : purpose === "instance-migration" + ? 64 * 1024 ** 2 + : policy.maxUploadBytes, + ); +} diff --git a/apps/ops-agent/src/artifact-transfer/routes.test.ts b/apps/ops-agent/src/artifact-transfer/routes.test.ts new file mode 100644 index 000000000..dbba4077b --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/routes.test.ts @@ -0,0 +1,114 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { issueArtifactStreamCapability } from "@openmapx/core/ops"; +import Fastify from "fastify"; +import { afterEach, expect, it } from "vitest"; +import { registerArtifactTransferRoutes } from "./routes"; +import { ArtifactTransferStore } from "./store"; + +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const close of cleanups.splice(0).reverse()) await close(); +}); +it("authenticates before streaming, preserves binary bytes, refuses purpose and attempt replay", async () => { + const root = await mkdtemp(join(tmpdir(), "artifact-http-")); + cleanups.push(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, "data"), { mode: 0o700 }); + await mkdir(join(root, "private"), { mode: 0o700 }); + const token = "a".repeat(43); + const store = new ArtifactTransferStore({ + dataRoot: join(root, "data"), + privateRoot: join(root, "private"), + token, + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 10000 }), + }); + const app = Fastify(); + registerArtifactTransferRoutes(app, { store, apiToken: token }); + cleanups.push(async () => { + await app.close(); + store.close(); + }); + const stage = await store.createStage({ + ownerId: "admin", + purpose: "prepared-artifact", + direction: "upload", + declaredBytes: 4, + operationKey: `opk1_${"a".repeat(20)}`, + }); + const input = { + version: 1 as const, + stageId: stage.stageId, + ownerId: "admin", + purpose: stage.purpose, + direction: "upload" as const, + byteCap: 4, + attemptId: randomUUID(), + issuedAt: Date.now(), + expiresAt: Date.now() + 60000, + deadlineAt: Date.now() + 10000, + }; + const cap = issueArtifactStreamCapability(token, input); + const path = `/v1/artifact-transfers/${stage.stageId}/content`; + const body = Buffer.from([0, 255, 192, 128]); + expect( + ( + await app.inject({ + method: "PUT", + url: path, + payload: body, + headers: { "Content-Type": "application/octet-stream" }, + }) + ).statusCode, + ).toBe(401); + expect( + ( + await app.inject({ + method: "PUT", + url: path, + payload: body, + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/octet-stream", + "x-openmapx-artifact-capability": issueArtifactStreamCapability(token, { + ...input, + ownerId: "foreign", + }), + }, + }) + ).statusCode, + ).toBe(404); + expect( + ( + await app.inject({ + method: "PUT", + url: path, + payload: body, + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/octet-stream", + "x-openmapx-artifact-capability": issueArtifactStreamCapability(token, { + ...input, + purpose: "integration-artifact", + }), + }, + }) + ).statusCode, + ).toBe(404); + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/octet-stream", + "x-openmapx-artifact-capability": cap, + }; + const result = await app.inject({ method: "PUT", url: path, payload: body, headers }); + expect(result.statusCode).toBe(200); + expect(result.json()).toMatchObject({ + state: "uploaded", + sha256: createHash("sha256").update(body).digest("hex"), + bytes: 4, + }); + expect((await app.inject({ method: "PUT", url: path, payload: body, headers })).statusCode).toBe( + 409, + ); +}); diff --git a/apps/ops-agent/src/artifact-transfer/routes.ts b/apps/ops-agent/src/artifact-transfer/routes.ts new file mode 100644 index 000000000..a0b6e726b --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/routes.ts @@ -0,0 +1,164 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { verifyArtifactStreamCapability } from "@openmapx/core/ops"; +import type { FastifyInstance } from "fastify"; +import { ArtifactTransferError } from "./errors"; +import type { ArtifactTransferStore } from "./store"; +export function registerArtifactTransferRoutes( + app: FastifyInstance, + options: { store: ArtifactTransferStore; apiToken: string; admissionsOpen?: () => boolean }, +): void { + app.register(async (scoped) => { + scoped.addContentTypeParser("application/octet-stream", (_request, payload, done) => + done(null, payload), + ); + scoped.addHook("onRequest", async (request, reply) => { + const candidate = createHash("sha256") + .update(request.headers.authorization ?? "") + .digest(); + const expected = createHash("sha256").update(`Bearer ${options.apiToken}`).digest(); + if (!timingSafeEqual(candidate, expected)) + return reply.code(401).send({ error: "Artifact stream authentication required" }); + if (options.admissionsOpen?.() === false) + return reply.code(503).send({ error: "Artifact transfer admission is draining" }); + try { + const capability = verifyArtifactStreamCapability( + options.apiToken, + String(request.headers["x-openmapx-artifact-capability"] ?? ""), + ); + const params = request.params as { stageId: string }; + if ( + capability.stageId !== params.stageId || + capability.direction !== (request.method === "PUT" ? "upload" : "download") + ) + throw Error(); + await options.store.getStage({ + stageId: capability.stageId, + ownerId: capability.ownerId, + purpose: capability.purpose, + }); + } catch (error) { + return reply.code(error instanceof ArtifactTransferError ? error.statusCode : 403).send({ + error: + error instanceof ArtifactTransferError + ? error.message + : "Artifact stream capability refused", + }); + } + }); + scoped.setErrorHandler((error, _request, reply) => { + if (reply.raw.headersSent) { + reply.raw.destroy(); + return; + } + return reply + .code( + error instanceof ArtifactTransferError + ? error.statusCode + : (error as { statusCode?: number }).statusCode === 415 + ? 415 + : 503, + ) + .send({ + error: + error instanceof ArtifactTransferError ? error.message : "Artifact stream unavailable", + }); + }); + scoped.put( + "/v1/artifact-transfers/:stageId/content", + { config: { artifactTransferBytes: true } }, + async (request, reply) => { + if ( + request.headers["content-type"]?.split(";")[0].trim().toLowerCase() !== + "application/octet-stream" + ) + return reply + .code(415) + .send({ error: "Artifact upload requires application/octet-stream" }); + const capability = verifyArtifactStreamCapability( + options.apiToken, + String(request.headers["x-openmapx-artifact-capability"]), + ); + const abort = new AbortController(); + const disconnected = () => { + if (!reply.raw.writableFinished) abort.abort(); + }; + request.raw.on("aborted", disconnected); + reply.raw.on("close", disconnected); + try { + const stage = await options.store.getStage({ + stageId: capability.stageId, + ownerId: capability.ownerId, + purpose: capability.purpose, + }); + if (stage.bytes > capability.byteCap) + throw new ArtifactTransferError(413, "Artifact stream exceeds capability limit"); + const source = request.body as Readable; + const bounded = async function* () { + let bytes = 0; + for await (const chunk of source) { + bytes += chunk.length; + if (bytes > capability.byteCap) + throw new ArtifactTransferError(413, "Artifact stream exceeds capability limit"); + yield chunk; + } + }; + const contentLength = + request.headers["content-length"] === undefined + ? undefined + : Number(request.headers["content-length"]); + const result = await options.store.upload({ + ...capability, + body: Readable.toWeb(Readable.from(bounded())) as ReadableStream, + contentLength, + signal: abort.signal, + }); + return reply + .header("Cache-Control", "no-store") + .header("X-Content-Type-Options", "nosniff") + .send(result); + } finally { + request.raw.off("aborted", disconnected); + reply.raw.off("close", disconnected); + } + }, + ); + scoped.get( + "/v1/artifact-transfers/:stageId/content", + { config: { artifactTransferBytes: true } }, + async (request, reply) => { + const capability = verifyArtifactStreamCapability( + options.apiToken, + String(request.headers["x-openmapx-artifact-capability"]), + ); + const abort = new AbortController(); + const disconnected = () => { + if (!reply.raw.writableFinished) abort.abort(); + }; + reply.raw.on("close", disconnected); + try { + await options.store.openDownload( + { ...capability, signal: abort.signal }, + async ({ body, headers, signal }) => { + if (Number(headers.get("Content-Length")) > capability.byteCap) + throw new ArtifactTransferError(413, "Artifact stream exceeds capability limit"); + headers.forEach((value, name) => { + reply.header(name, value); + }); + reply.hijack(); + reply.raw.writeHead(200, Object.fromEntries(headers.entries())); + await pipeline( + Readable.fromWeb(body as import("node:stream/web").ReadableStream), + reply.raw, + { signal }, + ); + }, + ); + } finally { + reply.raw.off("close", disconnected); + } + }, + ); + }); +} diff --git a/apps/ops-agent/src/artifact-transfer/runtime.ts b/apps/ops-agent/src/artifact-transfer/runtime.ts new file mode 100644 index 000000000..bb253e8f4 --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/runtime.ts @@ -0,0 +1,17 @@ +import type { OpsRuntime } from "../runtime"; +import type { ArtifactTransferStore } from "./store"; +export function installArtifactTransferRuntime( + runtime: OpsRuntime, + store: ArtifactTransferStore, +): void { + runtime["artifactTransfer.create"] = ({ kind: _, ...input }) => store.createStage(input); + runtime["artifactTransfer.list"] = (op) => store.listStages(op); + runtime["artifactTransfer.inspect"] = (op) => store.getStage(op); + runtime["artifactTransfer.delete"] = async (op) => { + await store.deleteStage(op); + return { deleted: true }; + }; + runtime["artifactTransfer.claim"] = (op) => store.claimStage(op); + runtime["artifactTransfer.inspectClaim"] = (op) => store.inspectClaim(op); + runtime["artifactTransfer.settle"] = (op) => store.settleStageClaim(op); +} diff --git a/apps/ops-agent/src/artifact-transfer/store.test.ts b/apps/ops-agent/src/artifact-transfer/store.test.ts new file mode 100644 index 000000000..26237b222 --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/store.test.ts @@ -0,0 +1,313 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rename, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { ArtifactTransferStore } from "./store"; + +const roots: string[] = []; +const stores: ArtifactTransferStore[] = []; +afterEach(async () => { + for (const store of stores.splice(0)) store.close(); + await Promise.all(roots.splice(0).map((p) => rm(p, { recursive: true, force: true }))); +}); +async function fixture(free = 2 * 1024 ** 3) { + const root = await mkdtemp(join(tmpdir(), "openmapx-transfer-")); + roots.push(root); + await mkdir(join(root, "data"), { mode: 0o700 }); + await mkdir(join(root, "private"), { mode: 0o700 }); + const store = new ArtifactTransferStore({ + dataRoot: join(root, "data"), + privateRoot: join(root, "private"), + token: "test-api-token", + capacity: () => ({ bytes: free, inodes: 100000 }), + }); + stores.push(store); + return { root, store }; +} +const ownerId = "admin-one"; +const input = () => ({ + ownerId, + purpose: "prepared-artifact" as const, + direction: "upload" as const, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, +}); +const stream = (bytes: Uint8Array) => + new ReadableStream({ + start(c) { + c.enqueue(bytes); + c.close(); + }, + }); +it("streams exact non-UTF8 bytes, journals digest and never calls an upload ready", async () => { + const { store } = await fixture(); + const bytes = new Uint8Array([0, 255, 192, 128]); + const request = { ...input(), declaredBytes: 4 }; + const stage = await store.createStage(request); + expect(await store.createStage(request)).toEqual(stage); + const result = await store.upload({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body: stream(bytes), + }); + expect(result.state).toBe("uploaded"); + expect(result.sha256).toBe(createHash("sha256").update(bytes).digest("hex")); + expect(result.bytes).toBe(4); + expect(await store.getStage({ stageId: stage.stageId, ownerId, purpose: stage.purpose })).toEqual( + result, + ); + await expect( + store.getStage({ stageId: stage.stageId, ownerId: "foreign", purpose: stage.purpose }), + ).rejects.toMatchObject({ statusCode: 404 }); + await expect( + store.openDownload( + { + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 1000, + }, + async () => {}, + ), + ).rejects.toMatchObject({ statusCode: 409 }); +}); +it("refuses unknown space and reserves actual small uploads, not expansion ceiling", async () => { + const { store } = await fixture(); + const stage = await store.createStage({ ...input(), declaredBytes: 2 * 1024 ** 2 }); + expect(stage.state).toBe("created"); + expect(await store.reservedBytes()).toBe(2 * 1024 ** 2); + const { store: small } = await fixture(512); + await expect(small.createStage({ ...input(), declaredBytes: 4 })).rejects.toMatchObject({ + statusCode: 507, + }); + const { root } = await fixture(); + const unavailable = new ArtifactTransferStore({ + dataRoot: join(root, "data"), + privateRoot: join(root, "private"), + token: "test-api-token", + capacity: () => { + throw Error("unknown"); + }, + }); + stores.push(unavailable); + await expect(unavailable.createStage(input())).rejects.toMatchObject({ statusCode: 507 }); +}); +it("foreign requests do not pull; length/digest lies and disconnect cannot finalize", async () => { + const { store } = await fixture(); + let pulled = 0; + const body = () => + new ReadableStream( + { + pull(c) { + pulled++; + c.enqueue(new Uint8Array(5)); + c.close(); + }, + }, + { highWaterMark: 0 }, + ); + const stage = await store.createStage({ ...input(), declaredBytes: 4 }); + await expect( + store.upload({ + stageId: stage.stageId, + ownerId: "foreign", + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body: body(), + }), + ).rejects.toMatchObject({ statusCode: 404 }); + expect(pulled).toBe(0); + await expect( + store.upload({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body: body(), + }), + ).rejects.toMatchObject({ statusCode: 413 }); + expect( + (await store.getStage({ stageId: stage.stageId, ownerId, purpose: stage.purpose })).state, + ).toBe("failed"); + const wrong = await store.createStage({ + ...input(), + declaredBytes: 4, + expectedSha256: "a".repeat(64), + }); + await expect( + store.upload({ + stageId: wrong.stageId, + ownerId, + purpose: wrong.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body: stream(new Uint8Array(4)), + }), + ).rejects.toMatchObject({ statusCode: 422 }); +}); +it("protects accepted writer against simultaneous upload and delete, then awaits cancellation", async () => { + const { store } = await fixture(); + const stage = await store.createStage({ ...input(), declaredBytes: 4 }); + let controller!: ReadableStreamDefaultController; + const body = new ReadableStream({ + start(c) { + controller = c; + }, + }); + const abort = new AbortController(); + const first = store.upload({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body, + signal: abort.signal, + }); + await new Promise((r) => setTimeout(r, 10)); + expect(store.activeStreams).toBe(1); + await expect(store.deleteStage({ stageId: stage.stageId, ownerId })).rejects.toMatchObject({ + statusCode: 409, + }); + await expect( + store.upload({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body: stream(new Uint8Array(4)), + }), + ).rejects.toMatchObject({ statusCode: 409 }); + abort.abort(); + await expect(first).rejects.toBeDefined(); + expect(store.activeStreams).toBe(0); + expect(body.locked).toBe(false); + expect(() => controller.close()).toThrow(); +}); +it("refuses replaced stage roots and unavailable purpose consumers", async () => { + const { store, root } = await fixture(); + const stage = await store.createStage({ ...input(), declaredBytes: 4 }); + await store.upload({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body: stream(new Uint8Array(4)), + }); + await expect( + store.claimStage({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + use: "validate-build", + expectedSha256: stage.sha256 ?? "0".repeat(64), + identity: { + executionId: `job1_${"a".repeat(20)}`, + operationKey: `opk1_${"a".repeat(20)}`, + fingerprint: "a".repeat(64), + executionDeadlineAt: new Date(Date.now() + 10000).toISOString(), + }, + }), + ).rejects.toMatchObject({ statusCode: 409 }); + const location = join(root, "data", stage.stageId); + await rename(location, `${location}.old`); + await symlink(`${location}.old`, location); + await expect( + store.getStage({ stageId: stage.stageId, ownerId, purpose: stage.purpose }), + ).rejects.toBeDefined(); + expect(await readFile(join(`${location}.old`, "content"))).toHaveLength(4); +}); +it("admits a declared 600 GiB stage using its actual reservation", async () => { + const { store } = await fixture(2 * 1024 ** 4); + const stage = await store.createStage({ ...input(), declaredBytes: 600 * 1024 ** 3 }); + expect(stage.bytes).toBe(0); + expect(await store.reservedBytes()).toBe(600 * 1024 ** 3); +}); +it("streams generated 2 GiB with bounded pull-ahead and no materialized body", async () => { + const { store } = await fixture(5 * 1024 ** 3); + const length = 2 * 1024 ** 3, + chunk = new Uint8Array(1024 ** 2); + let emitted = 0, + maximumAhead = 0; + const stage = await store.createStage({ ...input(), declaredBytes: length }); + const body = new ReadableStream( + { + async pull(controller) { + if (emitted === length) { + controller.close(); + return; + } + const observed = await store.getStage({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + }); + emitted += chunk.byteLength; + maximumAhead = Math.max(maximumAhead, emitted - observed.bytes); + controller.enqueue(chunk); + }, + }, + { highWaterMark: 1 }, + ); + const result = await store.upload({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 120000, + body, + }); + expect(result.bytes).toBe(length); + expect(result.state).toBe("uploaded"); + expect(maximumAhead).toBeLessThanOrEqual(8 * 1024 ** 2); + expect(store.activeStreams).toBe(0); + await store.deleteStage({ stageId: stage.stageId, ownerId }); + expect(await store.reservedBytes()).toBe(0); +}, 120000); +it("closes new admissions during the existing drain without dropping the active stream", async () => { + const { root } = await fixture(); + let open = true; + const store = new ArtifactTransferStore({ + dataRoot: join(root, "data"), + privateRoot: join(root, "private"), + token: "test-api-token", + admissionsOpen: () => open, + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 10000 }), + }); + stores.push(store); + const stage = await store.createStage({ ...input(), declaredBytes: 1 }); + const abort = new AbortController(); + const body = new ReadableStream( + { pull: () => new Promise(() => {}) }, + { highWaterMark: 0 }, + ); + const pending = store + .upload({ + stageId: stage.stageId, + ownerId, + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 10000, + body, + signal: abort.signal, + }) + .catch((error) => error); + await new Promise((r) => setTimeout(r, 10)); + open = false; + await expect(store.createStage(input())).rejects.toMatchObject({ statusCode: 503 }); + expect(store.activeStreams).toBe(1); + expect( + (await store.getStage({ stageId: stage.stageId, ownerId, purpose: stage.purpose })).state, + ).toBe("uploading"); + abort.abort(); + expect(await pending).toBeInstanceOf(Error); + expect(store.activeStreams).toBe(0); +}); diff --git a/apps/ops-agent/src/artifact-transfer/store.ts b/apps/ops-agent/src/artifact-transfer/store.ts new file mode 100644 index 000000000..bfd3a803c --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/store.ts @@ -0,0 +1,687 @@ +import { createHash, createHmac, hkdfSync, randomUUID, timingSafeEqual } from "node:crypto"; +import { + constants as C, + closeSync, + createReadStream, + fstatSync, + fsyncSync, + mkdirSync, + openSync, + statfsSync, + writeSync, +} from "node:fs"; +import { type FileHandle, open } from "node:fs/promises"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; +import { + type ArtifactStageClaim, + type ClaimTransferStage, + type CreateTransferStage, + canonicalJson, + createTransferStageSchema, + type StageClaimIdentity, + type StageClaimSettlement, + type TransferPurpose, + type TransferStage, + transferStageSchema, +} from "@openmapx/core/ops"; +import z from "zod/v4"; +import { ArtifactTransferError } from "./errors"; +import { + type ArtifactTransferPolicy, + DEFAULT_TRANSFER_POLICY, + purposeByteLimit, + transferPolicyRevision, +} from "./policy"; + +const id = z.uuid(); +const metaSchema = z.strictObject({ + stage: transferStageSchema, + ownerId: z.string().min(1).max(128), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + createdAt: z.number().safe().int(), + policyRevision: z.string().regex(/^[a-f0-9]{64}$/), + declaredBytes: z.number().safe().int().nonnegative().nullable(), + expectedSha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + reservedBytes: z.number().safe().int().nonnegative(), + directory: z.strictObject({ dev: z.string(), ino: z.string() }), + content: z + .strictObject({ + dev: z.string(), + ino: z.string(), + size: z.string(), + mtimeNs: z.string(), + ctimeNs: z.string(), + }) + .nullable(), + attempts: z + .array( + z.strictObject({ + attemptId: z.uuid(), + openedAt: z.number().int().safe(), + deadlineAt: z.number().int().safe(), + }), + ) + .max(32), +}); +type Meta = z.infer; +type StreamInput = { + stageId: string; + ownerId: string; + purpose: TransferPurpose; + attemptId: string; + deadlineAt: number; + signal?: AbortSignal; +}; +function failed( + status: ConstructorParameters[0], + message: string, +): never { + throw new ArtifactTransferError(status, message); +} +function absent(error: unknown) { + return (error as NodeJS.ErrnoException).code === "ENOENT"; +} +function same(a: { dev: string; ino: string }, b: { dev: bigint; ino: bigint }) { + return a.dev === String(b.dev) && a.ino === String(b.ino); +} + +/** One filesystem store for metadata, reservations and body ownership. Public IDs never become paths without UUID validation. */ +export class ArtifactTransferStore { + readonly policy: ArtifactTransferPolicy; + readonly revision: string; + private readonly key: Buffer; + private readonly data: AnchoredDirectory; + private readonly records: AnchoredDirectory; + private readonly streams = new Map(); + private readonly capacity: () => { bytes: number; inodes: number }; + private readonly now: () => number; + constructor(options: { + dataRoot: string; + privateRoot: string; + token: string; + policy?: ArtifactTransferPolicy; + capacity?: () => { bytes: number; inodes: number }; + now?: () => number; + admissionsOpen?: () => boolean; + }) { + this.policy = options.policy ?? DEFAULT_TRANSFER_POLICY; + this.revision = transferPolicyRevision(this.policy); + this.now = options.now ?? Date.now; + this.admissionsOpen = options.admissionsOpen ?? (() => true); + this.key = Buffer.from( + hkdfSync("sha256", options.token, "openmapx-artifact-transfer-v1", "stage-metadata", 32), + ); + mkdirSync(options.dataRoot, { recursive: true, mode: 0o700 }); + mkdirSync(options.privateRoot, { recursive: true, mode: 0o700 }); + this.data = new AnchoredDirectory(options.dataRoot, undefined, true); + this.records = new AnchoredDirectory(options.privateRoot, undefined, true); + this.capacity = + options.capacity ?? + (() => { + const fs = statfsSync(this.data.anchor, { bigint: true }); + const bytes = Number(fs.bavail * fs.bsize), + inodes = Number(fs.ffree); + if (!Number.isSafeInteger(bytes) || !Number.isSafeInteger(inodes)) + throw Error("Unknown capacity"); + return { bytes, inodes }; + }); + } + private readonly admissionsOpen: () => boolean; + get activeStreams() { + return this.streams.size; + } + close() { + if (this.streams.size) throw Error("Transfer streams still owned"); + this.records.close(); + this.data.close(); + this.key.fill(0); + } + private admit() { + if (!this.admissionsOpen()) failed(503, "Artifact transfer admission is draining"); + } + private lock(run: () => T): T { + let guard: AnchoredDirectory; + try { + mkdirSync(this.records.name("writer"), { mode: 0o700 }); + guard = this.records.directory("writer"); + } catch { + failed(503, "Artifact transfer state is busy or unavailable"); + } + try { + return run(); + } finally { + const stat = { dev: String(guard.stat.dev), ino: String(guard.stat.ino) }; + guard.close(); + this.records.removeDirectory("writer", stat); + } + } + private load(stageId: string): Meta { + id.parse(stageId); + try { + const outer = z + .strictObject({ body: z.string().max(30000), mac: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse(JSON.parse(this.records.read(`${stageId}.json`, 65536).toString("utf8"))); + const actual = createHmac("sha256", this.key).update(outer.body).digest(); + if (!timingSafeEqual(actual, Buffer.from(outer.mac, "hex"))) + throw Error("Stage metadata authentication failed"); + const meta = metaSchema.parse(JSON.parse(outer.body)); + if (meta.stage.stageId !== stageId) throw Error("Stage identity changed"); + return meta; + } catch (error) { + if (absent(error)) failed(404, "Artifact stage not found"); + throw error; + } + } + private save(meta: Meta) { + const body = canonicalJson(metaSchema.parse(meta)); + this.records.write( + `${meta.stage.stageId}.json`, + JSON.stringify({ body, mac: createHmac("sha256", this.key).update(body).digest("hex") }), + ); + } + private all(): Meta[] { + return this.records + .listBounded(4097) + .filter((name) => /^[a-f0-9-]{36}\.json$/.test(name)) + .map((name) => this.load(name.slice(0, -5))); + } + private directory(meta: Meta): AnchoredDirectory { + const directory = this.data.directory(meta.stage.stageId, false, true); + if (!same(meta.directory, directory.stat)) { + directory.close(); + failed(409, "Artifact stage storage changed"); + } + return directory; + } + private owned(stageId: string, ownerId: string, purpose?: TransferPurpose): Meta { + const meta = this.load(stageId); + if (meta.ownerId !== ownerId || (purpose !== undefined && meta.stage.purpose !== purpose)) + failed(404, "Artifact stage not found"); + const directory = this.directory(meta); + directory.close(); + return meta; + } + private alive(meta: Meta) { + if (Date.parse(meta.stage.expiresAt) <= this.now()) failed(410, "Artifact stage expired"); + if (meta.policyRevision !== this.revision) + failed(409, "Artifact transfer policy changed; create a new stage"); + } + private checkCapacity(additional: number, records = this.all()) { + let available: { bytes: number; inodes: number }; + try { + available = this.capacity(); + } catch { + failed(507, "Artifact transfer free capacity is unavailable"); + } + const pending = records.reduce( + (sum, m) => sum + Math.max(0, m.reservedBytes - m.stage.bytes), + 0, + ); + const headroom = Math.max(1024 ** 3, Math.ceil(additional * 0.05)); + if ( + !Number.isSafeInteger(available.bytes) || + !Number.isSafeInteger(available.inodes) || + available.bytes < additional + pending + headroom || + available.inodes < records.length * 3 + 16 + ) + failed(507, "Insufficient artifact transfer capacity"); + if ( + records.reduce((sum, m) => sum + m.reservedBytes, 0) + additional > + this.policy.maxStageBytes + ) + failed(507, "Artifact stage quota exceeded"); + } + async reservedBytes() { + return this.lock(() => this.all().reduce((sum, m) => sum + m.reservedBytes, 0)); + } + async createStage(raw: CreateTransferStage): Promise { + const input = createTransferStageSchema.parse(raw); + this.admit(); + return this.lock(() => { + const records = this.all(); + const existing = records.find( + (m) => m.ownerId === input.ownerId && m.operationKey === input.operationKey, + ); + if (existing) { + if ( + existing.stage.purpose !== input.purpose || + existing.stage.direction !== input.direction || + existing.declaredBytes !== (input.declaredBytes ?? null) || + existing.expectedSha256 !== (input.expectedSha256 ?? null) + ) + failed(409, "Artifact stage key conflicts with retained intent"); + return existing.stage; + } + const owned = records.filter((m) => m.ownerId === input.ownerId); + if (owned.filter((m) => m.createdAt > this.now() - 60000).length >= 10) + failed(429, "Artifact stage admission rate exceeded"); + if ( + owned.filter((m) => m.stage.cleanup !== "complete").length >= + this.policy.maxRetainedStagesPerPrincipal + ) + failed(429, "Artifact stage retention quota exceeded"); + if (records.length >= 4096) failed(507, "Artifact stage metadata capacity exceeded"); + const limit = purposeByteLimit(this.policy, input.purpose); + const reserved = input.declaredBytes ?? Math.min(1024 ** 3, limit); + if (reserved > limit) failed(413, "Artifact stage exceeds purpose limit"); + this.checkCapacity(reserved, records); + const stage: TransferStage = { + stageId: randomUUID(), + purpose: input.purpose, + direction: input.direction, + state: "created", + bytes: 0, + sha256: null, + expiresAt: new Date(this.now() + this.policy.stageMs).toISOString(), + cleanup: "pending", + }; + const directory = this.data.directory(stage.stageId, true, true); + try { + this.save({ + stage, + ownerId: input.ownerId, + operationKey: input.operationKey, + createdAt: this.now(), + policyRevision: this.revision, + declaredBytes: input.declaredBytes ?? null, + expectedSha256: input.expectedSha256 ?? null, + reservedBytes: reserved, + directory: { dev: String(directory.stat.dev), ino: String(directory.stat.ino) }, + content: null, + attempts: [], + }); + } finally { + directory.close(); + } + return stage; + }); + } + async getStage(input: { + stageId: string; + ownerId: string; + purpose: TransferPurpose; + }): Promise { + return this.lock(() => this.owned(input.stageId, input.ownerId, input.purpose).stage); + } + async listStages(input: { ownerId: string; limit: number; offset: number }) { + return this.lock(() => { + const records = this.all() + .filter((m) => m.ownerId === input.ownerId && m.stage.cleanup !== "complete") + .sort((a, b) => b.createdAt - a.createdAt); + return { + items: records.slice(input.offset, input.offset + input.limit).map((m) => m.stage), + total: records.length, + limit: input.limit, + offset: input.offset, + }; + }); + } + async deleteStage(input: { stageId: string; ownerId: string }): Promise { + this.lock(() => { + const meta = this.owned(input.stageId, input.ownerId); + if ( + meta.stage.state === "claimed" || + meta.stage.state === "validating" || + meta.stage.state === "uploading" || + this.streams.has(input.stageId) + ) + failed(409, "Artifact stage is owned by unfinished work"); + const dir = this.directory(meta); + try { + for (const name of dir.listBounded(4)) { + if (name !== "content") failed(409, "Unrecognized artifact stage content"); + dir.fileIdentity(name); + dir.removeFile(name); + } + meta.stage.cleanup = "complete"; + meta.stage.state = "expired"; + meta.reservedBytes = 0; + meta.content = null; + this.save(meta); + } catch (error) { + meta.stage.cleanup = "failed"; + this.save(meta); + throw error; + } finally { + dir.close(); + } + }); + } + private start( + input: StreamInput, + upload: boolean, + ): { meta: Meta; controller: AbortController; done: () => void; touch: () => void } { + this.admit(); + const meta = this.lock(() => { + const meta = this.owned(input.stageId, input.ownerId, input.purpose); + this.alive(meta); + id.parse(input.attemptId); + if (!Number.isSafeInteger(input.deadlineAt) || input.deadlineAt <= this.now()) + failed(410, "Artifact stream deadline expired"); + if (meta.attempts.some((attempt) => attempt.attemptId === input.attemptId)) + failed(409, "Artifact stream attempt was already consumed"); + if (meta.attempts.length >= 32) failed(429, "Artifact stream attempt limit exceeded"); + if (this.streams.has(input.stageId)) + failed(409, "Artifact stage already has an active stream"); + if ( + this.streams.size >= this.policy.maxConcurrentTransfers || + [...this.streams.values()].filter((s) => s.ownerId === input.ownerId).length >= + this.policy.maxConcurrentTransfersPerPrincipal + ) + failed(429, "Artifact stream concurrency limit exceeded"); + if ( + upload + ? meta.stage.direction !== "upload" || meta.stage.state !== "created" + : meta.stage.direction !== "download" || meta.stage.state !== "ready" + ) + failed(409, "Artifact stage is not available for this stream"); + meta.attempts.push({ + attemptId: input.attemptId, + openedAt: this.now(), + deadlineAt: Math.min( + input.deadlineAt, + this.now() + this.policy.transferMs, + Date.parse(meta.stage.expiresAt), + ), + }); + if (upload) meta.stage.state = "uploading"; + this.save(meta); + return meta; + }); + const controller = new AbortController(); + this.streams.set(input.stageId, { ownerId: input.ownerId, controller }); + let idle: ReturnType; + const touch = () => { + clearTimeout(idle); + idle = setTimeout( + () => + controller.abort(new ArtifactTransferError(410, "Artifact stream idle deadline expired")), + this.policy.idleMs, + ); + idle.unref?.(); + }; + const duration = Math.min( + this.policy.transferMs, + input.deadlineAt - this.now(), + Date.parse(meta.stage.expiresAt) - this.now(), + ); + const total = setTimeout( + () => controller.abort(new ArtifactTransferError(410, "Artifact stream deadline expired")), + Math.max(1, duration), + ); + total.unref?.(); + const abort = () => controller.abort(input.signal?.reason); + input.signal?.addEventListener("abort", abort, { once: true }); + if (input.signal?.aborted) abort(); + touch(); + return { + meta, + controller, + touch, + done: () => { + clearTimeout(total); + clearTimeout(idle); + input.signal?.removeEventListener("abort", abort); + this.streams.delete(input.stageId); + }, + }; + } + async upload( + input: StreamInput & { body: ReadableStream; contentLength?: number }, + ): Promise { + const lifetime = this.start(input, true); + let fd: number | undefined, + dir: AnchoredDirectory | undefined, + reader: ReadableStreamDefaultReader | undefined; + let complete = false; + const meta = lifetime.meta; + const expectedBytes = meta.declaredBytes ?? input.contentLength; + try { + if ( + input.contentLength !== undefined && + (!Number.isSafeInteger(input.contentLength) || + input.contentLength < 0 || + input.contentLength > purposeByteLimit(this.policy, meta.stage.purpose)) + ) + failed(413, "Artifact content length exceeds purpose limit"); + if ( + input.contentLength !== undefined && + meta.declaredBytes !== null && + input.contentLength !== meta.declaredBytes + ) + failed(422, "Artifact length does not match admitted stage"); + dir = this.directory(meta); + fd = openSync(dir.name("content"), C.O_WRONLY | C.O_CREAT | C.O_EXCL | C.O_NOFOLLOW, 0o600); + reader = input.body.getReader(); + const hash = createHash("sha256"); + let bytes = 0; + while (true) { + lifetime.controller.signal.throwIfAborted(); + const part = await this.readChunk(reader, lifetime.controller.signal); + if (part.done) break; + if (part.value.byteLength > 8 * 1024 ** 2) + failed(413, "Artifact stream chunk exceeds queue limit"); + const next = bytes + part.value.byteLength; + const limit = expectedBytes ?? purposeByteLimit(this.policy, meta.stage.purpose); + if (!Number.isSafeInteger(next) || next > limit) + failed(413, "Artifact stream exceeds admitted byte limit"); + if (next > meta.reservedBytes) + this.lock(() => { + const current = this.load(meta.stage.stageId); + const reserved = Math.min(limit, Math.ceil(next / 1024 ** 3) * 1024 ** 3); + this.checkCapacity(reserved - current.reservedBytes); + meta.reservedBytes = reserved; + current.reservedBytes = reserved; + this.save(current); + }); + let written = 0; + while (written < part.value.byteLength) + written += writeSync(fd, part.value, written, part.value.byteLength - written); + hash.update(part.value); + bytes = next; + meta.stage.bytes = bytes; + this.lock(() => { + this.checkCapacity(0); + this.save(meta); + }); + lifetime.touch(); + } + lifetime.controller.signal.throwIfAborted(); + if (expectedBytes !== undefined && bytes !== expectedBytes) + failed(422, "Artifact stream length mismatch"); + const sha256 = hash.digest("hex"); + if (meta.expectedSha256 !== null && sha256 !== meta.expectedSha256) + failed(422, "Artifact stream digest mismatch"); + fsyncSync(fd); + const stat = fstatSync(fd, { bigint: true }); + if (!stat.isFile() || stat.nlink !== 1n || stat.size !== BigInt(bytes)) + failed(409, "Artifact content changed"); + meta.content = { + dev: String(stat.dev), + ino: String(stat.ino), + size: String(stat.size), + mtimeNs: String(stat.mtimeNs), + ctimeNs: String(stat.ctimeNs), + }; + meta.stage.sha256 = sha256; + meta.stage.state = "uploaded"; + meta.reservedBytes = bytes; + this.lock(() => { + dir!.check(); + dir!.fileIdentity("content"); + this.save(meta); + }); + complete = true; + return meta.stage; + } catch (error) { + meta.stage.state = "failed"; + try { + this.lock(() => this.save(meta)); + } catch {} + throw error; + } finally { + try { + if (reader) { + if (!complete) await reader.cancel(lifetime.controller.signal.reason).catch(() => {}); + reader.releaseLock(); + } + } finally { + if (fd !== undefined) closeSync(fd); + dir?.close(); + lifetime.done(); + } + } + } + private async readChunk(reader: ReadableStreamDefaultReader, signal: AbortSignal) { + let abort: () => void = () => {}; + try { + return await Promise.race([ + reader.read(), + new Promise((_, reject) => { + abort = () => reject(signal.reason ?? Error("Artifact stream aborted")); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + }), + ]); + } finally { + signal.removeEventListener("abort", abort); + } + } + async openDownload( + input: StreamInput, + consume: (input: { + body: ReadableStream; + signal: AbortSignal; + headers: Headers; + }) => Promise, + ): Promise { + const lifetime = this.start(input, false); + let file: FileHandle | undefined, + dir: AnchoredDirectory | undefined, + body: ReadableStream | undefined; + try { + if (!lifetime.meta.stage.sha256) failed(409, "Artifact download digest is unavailable"); + const sha256 = lifetime.meta.stage.sha256; + dir = this.directory(lifetime.meta); + file = await open(dir.name("content"), C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + const stat = await file.stat({ bigint: true }); + if ( + !lifetime.meta.content || + !stat.isFile() || + stat.nlink !== 1n || + !same(lifetime.meta.content, stat) || + String(stat.size) !== lifetime.meta.content.size || + String(stat.mtimeNs) !== lifetime.meta.content.mtimeNs || + String(stat.ctimeNs) !== lifetime.meta.content.ctimeNs + ) + failed(409, "Artifact content changed"); + const source = createReadStream("", { + fd: file.fd, + autoClose: false, + highWaterMark: 1024 * 1024, + signal: lifetime.controller.signal, + }); + const upstream = (Readable.toWeb(source) as ReadableStream).getReader(); + let eof = false; + body = new ReadableStream( + { + pull: async (controller) => { + try { + const chunk = await this.readChunk(upstream, lifetime.controller.signal); + if (chunk.done) { + eof = true; + controller.close(); + upstream.releaseLock(); + } else { + lifetime.touch(); + controller.enqueue(chunk.value); + } + } catch (error) { + controller.error(error); + } + }, + cancel: async (reason) => { + await upstream.cancel(reason).catch(() => {}); + upstream.releaseLock(); + }, + }, + { highWaterMark: 0 }, + ); + const result = await consume({ + body, + signal: lifetime.controller.signal, + headers: new Headers({ + "Content-Type": "application/octet-stream", + "Content-Length": String(stat.size), + "Content-Disposition": 'attachment; filename="openmapx-artifact.tar"', + "X-Artifact-Sha256": sha256, + "Cache-Control": "no-store", + "X-Content-Type-Options": "nosniff", + }), + }); + lifetime.controller.signal.throwIfAborted(); + if (!eof) failed(409, "Artifact download consumer did not complete"); + const after = await file.stat({ bigint: true }); + if ( + after.size !== stat.size || + after.mtimeNs !== stat.mtimeNs || + after.ctimeNs !== stat.ctimeNs + ) + failed(409, "Artifact content changed during download"); + dir.check(); + return result; + } finally { + try { + if (body && !body.locked) await body.cancel().catch(() => {}); + await file?.close(); + } finally { + dir?.close(); + lifetime.done(); + } + } + } + async claimStage(input: ClaimTransferStage): Promise { + this.admit(); + this.lock(() => { + const meta = this.owned(input.stageId, input.ownerId, input.purpose); + this.alive(meta); + }); + failed(409, "Artifact purpose consumer is not installed"); + } + async inspectClaim(input: { + stageId: string; + ownerId: string; + identity: StageClaimIdentity; + }): Promise<{ claim: ArtifactStageClaim; settlement: StageClaimSettlement | null } | null> { + this.lock(() => this.owned(input.stageId, input.ownerId)); + return null; + } + async settleStageClaim(_input: { + claim: ArtifactStageClaim; + settlement: StageClaimSettlement; + }): Promise { + failed(409, "Artifact purpose consumer is not installed"); + } + async withStageFile( + _input: { claim: ArtifactStageClaim; signal: AbortSignal }, + _run: (file: FileHandle, signal: AbortSignal) => Promise, + ): Promise { + failed(409, "Artifact purpose consumer is not installed"); + } + async withClaimedIntegrationArtifact( + _input: { claim: ArtifactStageClaim; signal: AbortSignal }, + _run: (input: { + source: AsyncIterable; + signal: AbortSignal; + bytes: number; + sha256: string; + }) => Promise, + ): Promise { + failed(409, "Artifact purpose consumer is not installed"); + } +} diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 5c8a5a870..dc813a0e1 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -1,5 +1,6 @@ import { constants, existsSync } from "node:fs"; import { open } from "node:fs/promises"; +import { join } from "node:path"; import { loadControlStartup } from "@openmapx/core/build-artifacts"; import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; import { readOpsTokenFile } from "@openmapx/core/ops"; @@ -13,6 +14,9 @@ import { pruneBackupRetention, } from "./administrative-runtime"; import { isTerminalHandoff, readAgentHandoff } from "./agent-handoff"; +import { loadArtifactTransferPolicy } from "./artifact-transfer/policy"; +import { installArtifactTransferRuntime } from "./artifact-transfer/runtime"; +import { ArtifactTransferStore } from "./artifact-transfer/store"; import { startOpsAgentServer } from "./bootstrap"; import { CommunityRepositoryStore } from "./community-repositories"; import { loadOpsAgentConfig } from "./config"; @@ -194,6 +198,27 @@ async function main(): Promise { ownerGid: gid, }, }); + const replacementInProgress = () => { + try { + const handoff = readAgentHandoff(config.rootDir); + return handoff !== null && !isTerminalHandoff(handoff.phase); + } catch { + return true; + } + }; + const transferPolicy = loadArtifactTransferPolicy(); + const artifactTransfers = + "policy" in transferPolicy + ? new ArtifactTransferStore({ + dataRoot: join(paths.infraDir, "data", ".openmapx-transfers"), + privateRoot: join(config.trustedConfigDirectory, "artifact-transfers"), + token: apiToken, + policy: transferPolicy.policy, + admissionsOpen: () => !replacementInProgress(), + }) + : undefined; + if (artifactTransfers) installArtifactTransferRuntime(runtime, artifactTransfers); + else if ("reason" in transferPolicy) console.error(transferPolicy.reason); const releaseEffects = productionReleaseEffects(runtime); const retentionDays = Number(process.env.BACKUP_RETENTION_DAYS?.trim() || "30"); if (!Number.isSafeInteger(retentionDays) || retentionDays <= 0 || retentionDays > 36_500) { @@ -272,6 +297,7 @@ async function main(): Promise { controlInstallation: controlStartup.installation, terminalClaimCleanup, runtime, + artifactTransfers, replacementInProgress: () => { try { const handoff = readAgentHandoff(config.rootDir); diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 0fe4f47ce..171289c46 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -95,6 +95,9 @@ import { } from "@openmapx/core/ops"; import Fastify, { type FastifyInstance } from "fastify"; import { inspectConfigurationGenerationNonCommit } from "../../../packages/core/src/build-artifacts/control-authority"; +import { ArtifactTransferError } from "./artifact-transfer/errors"; +import { registerArtifactTransferRoutes } from "./artifact-transfer/routes"; +import type { ArtifactTransferStore } from "./artifact-transfer/store"; import { readCommittedCatalogApproval } from "./catalog-approval-runtime"; import { prepareConfigurationHardlinksAdmission } from "./configuration-hardlinks-runtime"; import { @@ -222,6 +225,7 @@ export interface OpsAuditEvent { } export interface BuildOpsAgentServerOptions { + artifactTransfers?: ArtifactTransferStore; controlInstallation?: ControlInstallation; terminalClaimCleanup?: VerifiedTerminalClaimCleanup; tokens: Record; @@ -375,6 +379,18 @@ function immutableSnapshot(value: T): T { } function classify(error: unknown): { errorClass: OpsErrorClass; statusCode: number } { + if (error instanceof ArtifactTransferError) + return { + errorClass: + error.statusCode === 404 + ? "not_found" + : error.statusCode === 503 + ? "runtime" + : error.statusCode === 429 + ? "busy" + : "conflict", + statusCode: error.statusCode, + }; if (error instanceof OpsContractError) { return { errorClass: error.errorClass, @@ -408,9 +424,27 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const app = Fastify({ logger: false, bodyLimit: options.bodyLimit ?? DEFAULT_BODY_LIMIT, - requestTimeout: DEFAULT_ADMISSION_TIMEOUT_MS, + requestTimeout: 0, connectionTimeout: DEFAULT_ADMISSION_TIMEOUT_MS, }); + app.addHook("onRequest", async (request, reply) => { + const streaming = + (request.routeOptions.config as { artifactTransferBytes?: boolean }).artifactTransferBytes === + true; + request.raw.socket.setTimeout(streaming ? 5 * 60_000 : DEFAULT_ADMISSION_TIMEOUT_MS); + if (streaming) return; + const timer = setTimeout(() => request.raw.destroy(), DEFAULT_ADMISSION_TIMEOUT_MS); + timer.unref?.(); + const clear = () => { + clearTimeout(timer); + request.raw.off("end", clear); + request.raw.off("aborted", clear); + reply.raw.off("close", clear); + }; + request.raw.once("end", clear); + request.raw.once("aborted", clear); + reply.raw.once("close", clear); + }); if (options.dawarichExport) { registerDawarichSubjectExportRoute(app, { ...options.dawarichExport, @@ -423,6 +457,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif apiToken: options.tokens.api, }); } + if (options.artifactTransfers) + registerArtifactTransferRoutes(app, { + store: options.artifactTransfers, + apiToken: options.tokens.api, + admissionsOpen: () => !options.replacementInProgress?.(), + }); const runtime = options.runtime ?? createUnavailableRuntime(); const dispatch = options.dispatch ?? @@ -1020,7 +1060,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif (other.state === "queued" || other.state === "running" || other.state === "termination_pending"), - ).length, + ).length + (options.artifactTransfers?.activeStreams ?? 0), ...(job.operation.kind === "services.buildAll" ? { checkpointServiceBatch: async ( diff --git a/docs/docs/developer/api-surface.md b/docs/docs/developer/api-surface.md index bf3e373a5..876147a9a 100644 --- a/docs/docs/developer/api-surface.md +++ b/docs/docs/developer/api-surface.md @@ -561,3 +561,25 @@ The CLI prints queued 202 job IDs without adding an Activity watcher. Since install/update acquire another preview, a preceding explicit `ext preview` does not bypass the one-preview-per-30-seconds bucket; 429 ends that command without writing or automatically retrying. + +## Administration byte transfers + +Declare opaque byte bodies with `AdminOperation.requestBody` and successful byte +responses with `adminBytes`. They are mutually exclusive with a JSON request +schema. The registrar emits `format: binary`, validates success media/headers and +requires the shared byte-send adapter; application handlers must not JSON-encode +or buffer a large stream. Authentication and cookie Origin checks run before the +scoped octet-stream parser exposes upload bytes. + +The generated service-admin clients accept an unchanged `Blob` for uploads and a +required `onByteChunk(chunk, signal)` callback for downloads. Each callback must +finish its sink cleanup when aborted. The client waits for that cleanup and EOF; +a returned response never transfers ownership of a raw response body. Error JSON +is bounded separately to 64 KiB. + +A generated native download target constructs a browser-owned navigation only +for the actual same origin, compatible cookie credentials and no configured +header interceptor. It performs no download and cannot provide custom headers, +progress or cancellation. Other configured clients keep using the callback-based +byte client with a supported file sink; do not silently change their credentials +or buffer a large response into a Blob. diff --git a/docs/docs/install/preparing-data.md b/docs/docs/install/preparing-data.md index ede1b796a..4c16bc2b2 100644 --- a/docs/docs/install/preparing-data.md +++ b/docs/docs/install/preparing-data.md @@ -444,3 +444,51 @@ downloads and build outputs are reproducible — you can always re-fetch and reb the RAM and disk you'll need before you start downloading. - **[How it works](../overview/how-it-works.md)** — the service/integration model and how the data-manager fits into a running deployment. + +## Transfer limits and storage + +Artifact transfers use private stages managed by the operations agent. Uploading +bytes does not validate or install them. The server records the received size and +SHA-256; an interrupted or ambiguous upload must be inspected before starting a +new transfer. Deleting a stage cannot interrupt an unfinished installation. + +The default limits are 1 TiB uploaded, 2 TiB expanded and 4 TiB across retained +stages, with two simultaneous transfers and one per administrator. Integration +archives have a separate 200 MiB limit; encrypted instance migration files have a +64 MiB limit. Disk admission reserves the actual declared upload size, not the +expansion ceiling, and requires at least 1 GiB of additional free headroom. Unknown +free capacity blocks admission. A successful upload stays `uploaded` until its +purpose-specific validator accepts it. + +Operators can supply `infra/docker/artifact-transfer-policy.json`, mounted +read-only into the agent. Its optional `artifactTransfer` object accepts the +published deployment limits. For example, a smaller installation can use: + +```json +{ + "artifactTransfer": { + "maxUploadBytes": 17179869184, + "maxExpandedBytes": 34359738368, + "maxEntryBytes": 34359738368, + "maxStageBytes": 68719476736, + "maxEntries": 10000, + "maxManifestBytes": 8388608, + "maxConcurrentTransfers": 1, + "maxConcurrentTransfersPerPrincipal": 1, + "maxConcurrentArchiveJobs": 1, + "memoryBudgetBytes": 268435456 + } +} +``` + +The supported agent manifest allocates 1 GiB of memory. Larger metadata or +concurrency limits require a matching declared and actual memory allocation; +invalid or incompatible policy disables transfers rather than silently weakening +limits. A missing policy file uses the defaults. + +Streams have a five-minute idle limit and a 72-hour total limit, shortened by an +owning job's earlier deadline. Stages expire after 96 hours, while unfinished or +unknown publication work retains its ownership. The shipped Traefik HTTPS +entrypoint permits 72-hour reads and writes without request buffering. These are +entrypoint-wide settings; ordinary API body limits and authentication still apply. +Custom ingress deployments must preserve streaming and compatible timeouts. diff --git a/packages/core/src/__tests__/services-traefik-renderer.test.ts b/packages/core/src/__tests__/services-traefik-renderer.test.ts index 15bb21e4a..0b97c0422 100644 --- a/packages/core/src/__tests__/services-traefik-renderer.test.ts +++ b/packages/core/src/__tests__/services-traefik-renderer.test.ts @@ -159,3 +159,24 @@ describe("generated Traefik file configuration", () => { ).toBeNull(); }); }); + +it("keeps large-transfer entrypoint deadlines static and generated routes unbuffered", () => { + const shipped = parseYaml( + readFileSync(join(REPO_ROOT, "services/traefik/config/traefik.yml"), "utf8"), + ) as { + entryPoints: { websecure: { transport: { respondingTimeouts: Record } } }; + }; + expect(shipped.entryPoints.websecure.transport.respondingTimeouts).toEqual({ + readTimeout: "72h", + writeTimeout: "72h", + idleTimeout: "5m", + }); + const dynamic = renderTraefikDynamicConfiguration( + loadBuiltIns().map((s) => s.manifest), + { + domain: DOMAIN, + resolveProxyHost: (candidate) => resolveProxyHost(candidate, { domain: DOMAIN }), + }, + ); + expect(JSON.stringify(dynamic)).not.toContain('"buffering"'); +}); diff --git a/packages/core/src/api/admin-byte-reader.ts b/packages/core/src/api/admin-byte-reader.ts new file mode 100644 index 000000000..a37f2b893 --- /dev/null +++ b/packages/core/src/api/admin-byte-reader.ts @@ -0,0 +1,80 @@ +/** Incremental byte consumption: callbacks and their cleanup stay inside request ownership. */ +import { ApiRequestAbortedError } from "./client"; +import { ApiContractError } from "./sdk-transport"; +export interface AdminByteConsumer { + onByteChunk: (chunk: Uint8Array, signal: AbortSignal) => void | Promise; + onByteHeaders?: (headers: Headers, signal: AbortSignal) => void | Promise; +} +async function readOwned(pending: Promise, signal: AbortSignal): Promise { + if (signal.aborted) { + void pending.catch(() => {}); + signal.throwIfAborted(); + } + let abort = () => {}; + try { + return await Promise.race([ + pending, + new Promise((_resolve, reject) => { + abort = () => reject(signal.reason); + signal.addEventListener("abort", abort, { once: true }); + }), + ]); + } finally { + signal.removeEventListener("abort", abort); + } +} +export async function consumeAdminBytes( + response: Response, + parent: AbortSignal, + consumer: AdminByteConsumer, +): Promise { + if (!response.body) throw new ApiContractError(response.status); + const length = response.headers.get("content-length"); + if (length !== null && (!/^\d+$/.test(length) || !Number.isSafeInteger(Number(length)))) + throw new ApiContractError(response.status); + const expected = length === null ? null : Number(length), + controller = new AbortController(), + signal = controller.signal; + const cancel = () => controller.abort(parent.reason); + if (parent.aborted) cancel(); + else parent.addEventListener("abort", cancel, { once: true }); + let idle: ReturnType | undefined, + completed = false, + bytes = 0; + const touch = () => { + if (idle !== undefined) clearTimeout(idle); + idle = setTimeout(() => controller.abort(new ApiRequestAbortedError("timeout")), 5 * 60_000); + }; + const reader = response.body.getReader(); + try { + touch(); + signal.throwIfAborted(); + await consumer.onByteHeaders?.(response.headers, signal); + signal.throwIfAborted(); + touch(); + for (;;) { + const { done, value } = await readOwned(reader.read(), signal); + if (done) break; + if (value.byteLength > 8 * 1024 * 1024) throw new ApiContractError(response.status); + bytes += value.byteLength; + if (!Number.isSafeInteger(bytes) || (expected !== null && bytes > expected)) + throw new ApiContractError(response.status); + touch(); + // Never race an active sink with abort: it must settle its own cleanup first. + await consumer.onByteChunk(value, signal); + signal.throwIfAborted(); + touch(); + } + signal.throwIfAborted(); + if (expected !== null && bytes !== expected) throw new ApiContractError(response.status); + completed = true; + } catch (error) { + if (signal.aborted && !parent.aborted) throw new ApiRequestAbortedError("timeout"); + throw error; + } finally { + if (idle !== undefined) clearTimeout(idle); + parent.removeEventListener("abort", cancel); + if (!completed) await reader.cancel().catch(() => {}); + reader.releaseLock(); + } +} diff --git a/packages/core/src/api/admin-byte-transport.test.ts b/packages/core/src/api/admin-byte-transport.test.ts new file mode 100644 index 000000000..92ddeab7b --- /dev/null +++ b/packages/core/src/api/admin-byte-transport.test.ts @@ -0,0 +1,278 @@ +/** Canonical bytes stay bytes and remain owned until each sink callback settles. */ +import { afterEach, expect, it, vi } from "vitest"; +import z from "zod/v4"; +import { createAdminOperationBinder } from "./admin-sdk-transport"; +import { ApiRequestAbortedError, createApiClient } from "./client"; + +const download = { + method: "GET", + path: "/api/admin/artifact-transfers/fixture/content", + auth: "admin", + responses: { + 200: { + kind: "bytes", + mediaType: "application/octet-stream", + headers: z.object({ "content-length": z.string().regex(/^\d+$/) }), + }, + 409: { kind: "json", schema: z.object({ error: z.string() }) }, + }, +} as const; +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); +}); +it("passes non-UTF8 octets unchanged through the configured client", async () => { + const bytes = new Uint8Array([0, 255, 192, 128]), + chunks: number[][] = []; + const fetch = vi.fn().mockResolvedValue( + new Response(bytes, { + headers: { "content-type": "application/octet-stream", "content-length": "4" }, + }), + ); + vi.stubGlobal("fetch", fetch); + const client = createApiClient({ + baseUrl: "https://api.example", + credentials: "omit", + headerInterceptor: () => ({ "x-client": "retained" }), + }); + const result = await createAdminOperationBinder({ download })("download")(download.path, { + client, + onByteChunk: async (chunk) => { + chunks.push(Array.from(chunk)); + }, + }); + expect(result).toMatchObject({ status: 200, data: undefined }); + expect(chunks).toEqual([[0, 255, 192, 128]]); + expect(fetch.mock.calls[0]?.[1]).toMatchObject({ credentials: "omit" }); + expect(new Headers(fetch.mock.calls[0]?.[1].headers).get("x-client")).toBe("retained"); +}); +it("refuses an interceptor's changed byte request media type before fetch", async () => { + const fetch = vi.fn().mockResolvedValue(Response.json({ ok: true })); + vi.stubGlobal("fetch", fetch); + const client = createApiClient({ + baseUrl: "https://api.example", + headerInterceptor: () => ({ "content-type": "application/json" }), + }); + await expect( + client.request( + "/api/admin/artifact-transfers/fixture/content", + { method: "PUT", body: new Blob([new Uint8Array([0, 255])]) }, + async (response) => response.status, + { + headers: { "content-type": "application/octet-stream" }, + expectedRequestMediaType: "application/octet-stream", + }, + ), + ).rejects.toThrow(); + expect(fetch).not.toHaveBeenCalled(); +}); +it("constructs only a same-origin browser-owned native target", () => { + vi.stubGlobal("window", { location: { origin: "https://app.example" } }); + const client = createApiClient({ baseUrl: "https://app.example", credentials: "include" }); + expect(client.downloadTarget(download.path)).toEqual({ + url: "https://app.example" + download.path, + ownership: "browser", + }); + for (const cfg of [ + { baseUrl: "https://other.example", credentials: "include" as const }, + { baseUrl: "https://app.example", credentials: "omit" as const }, + { + baseUrl: "https://app.example", + credentials: "include" as const, + headerInterceptor: () => ({}), + }, + ]) + expect(() => createApiClient(cfg).downloadTarget(download.path)).toThrow(); + expect(() => client.downloadTarget("//foreign.example/file")).toThrow(); + expect(() => client.downloadTarget(download.path + "#fragment")).toThrow(); +}); +it.each(["aborted", "timeout"] as const)( + "awaits active sink cleanup and upstream cancellation after %s", + async (mode) => { + vi.useFakeTimers(); + let entered = false, + cleanupStarted = false, + cleanupFinished = false, + canceled = false, + settled = false; + let release = () => {}; + const cleanup = new Promise((resolve) => { + release = resolve; + }); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([0, 255])); + }, + cancel() { + canceled = true; + }, + }); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response(body, { + headers: { "content-type": "application/octet-stream", "content-length": "2" }, + }), + ), + ); + const owner = new AbortController(), + client = createApiClient({ baseUrl: "https://api.example" }); + const pending = createAdminOperationBinder({ download })("download")(download.path, { + client, + signal: owner.signal, + ...(mode === "timeout" ? { timeoutMs: 20 } : {}), + onByteChunk: async (_chunk, signal) => { + entered = true; + await new Promise((resolve) => + signal.addEventListener( + "abort", + () => { + cleanupStarted = true; + void cleanup.then(() => { + cleanupFinished = true; + resolve(); + }); + }, + { once: true }, + ), + ); + }, + }) + .catch((error) => error) + .finally(() => { + settled = true; + }); + await vi.advanceTimersByTimeAsync(0); + expect(entered).toBe(true); + if (mode === "timeout") await vi.advanceTimersByTimeAsync(21); + else owner.abort(); + await vi.advanceTimersByTimeAsync(0); + expect(cleanupStarted).toBe(true); + expect(settled).toBe(false); + release(); + await vi.advanceTimersByTimeAsync(0); + expect(await pending).toBeInstanceOf(ApiRequestAbortedError); + expect(cleanupFinished).toBe(true); + expect(canceled).toBe(true); + expect(body.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }, +); +it("rejects ambiguous binary declarations and native POST targets at binding", () => { + expect(() => + createAdminOperationBinder({ + bad: { + ...download, + requestBody: { kind: "bytes", mediaType: "application/octet-stream" }, + request: { body: z.object({ value: z.string() }) }, + }, + })("bad"), + ).toThrow(); + expect(() => + createAdminOperationBinder({ + bad: { + ...download, + method: "POST", + responses: { 200: { ...download.responses[200], nativeDownload: true } }, + }, + })("bad"), + ).toThrow(); +}); +it("refuses malformed success headers and bounded non-JSON/oversized error responses before callbacks", async () => { + const consume = vi.fn(); + const client = createApiClient({ baseUrl: "https://api.example" }); + const bound = createAdminOperationBinder({ download })("download"); + for (const response of [ + new Response(new Uint8Array([1]), { + headers: { "content-type": "application/octet-stream", "content-length": "invalid" }, + }), + new Response("x".repeat(65537), { + status: 409, + headers: { "content-type": "application/json" }, + }), + new Response("private-error", { status: 409, headers: { "content-type": "text/plain" } }), + ]) { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(response)); + await expect( + bound(download.path, { client, onByteChunk: consume, maxErrorBytes: 10 * 1024 ** 2 }), + ).rejects.toThrow(); + } + expect(consume).not.toHaveBeenCalled(); +}); +it("keeps invalid-header rejection owned until upstream cancellation finishes", async () => { + let entered = false, + settled = false; + let release = () => {}; + const cleanup = new Promise((resolve) => { + release = resolve; + }); + const body = new ReadableStream({ + cancel: async () => { + entered = true; + await cleanup; + }, + }); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response(body, { + headers: { "content-type": "application/octet-stream", "content-length": "invalid" }, + }), + ), + ); + const pending = createAdminOperationBinder({ download })("download")(download.path, { + client: createApiClient({ baseUrl: "https://api.example" }), + onByteChunk: () => {}, + }) + .catch((error) => error) + .finally(() => { + settled = true; + }); + await vi.waitFor(() => expect(entered).toBe(true)); + const early = settled; + release(); + await pending; + expect(early).toBe(false); +}); +it("generated native targets validate UUID params and refuse unsupported headers without fetch", async () => { + const { getDownloadAdminArtifactTransferContentTarget } = await import( + "./generated/admin/service-admin.generated" + ); + vi.stubGlobal("window", { location: { origin: "https://app.example" } }); + const fetch = vi.fn(); + vi.stubGlobal("fetch", fetch); + const client = createApiClient({ baseUrl: "https://app.example", credentials: "include" }); + const stageId = "91a78b3e-208b-4ba4-9ddf-870fbdf1da22"; + expect(await getDownloadAdminArtifactTransferContentTarget(stageId, { client })).toEqual({ + url: `https://app.example/api/admin/artifact-transfers/${stageId}/content`, + ownership: "browser", + }); + await expect( + getDownloadAdminArtifactTransferContentTarget("invalid", { client }), + ).rejects.toThrow(); + await expect( + getDownloadAdminArtifactTransferContentTarget(stageId, { + client, + headers: { Authorization: "not-supported" } as never, + }), + ).rejects.toMatchObject({ reason: "headers" }); + expect(fetch).not.toHaveBeenCalled(); +}); +it("caps byte-upload success JSON at 64 KiB even when the caller asks for more", async () => { + const operation = { + method: "PUT", + path: "/api/admin/artifact-transfers/fixture/content", + auth: "admin", + requestBody: { kind: "bytes", mediaType: "application/octet-stream" }, + responses: { 200: { kind: "json", schema: z.object({ padding: z.string() }) } }, + } as const; + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ padding: "x".repeat(65537) }))); + await expect( + createAdminOperationBinder({ upload: operation })("upload")(operation.path, { + client: createApiClient({ baseUrl: "https://api.example" }), + method: "PUT", + body: new Blob([new Uint8Array([0])]), + maxJsonBytes: 1024 ** 2, + }), + ).rejects.toThrow(); +}); diff --git a/packages/core/src/api/admin-sdk-transport.ts b/packages/core/src/api/admin-sdk-transport.ts index 2e9444a65..79ef02f72 100644 --- a/packages/core/src/api/admin-sdk-transport.ts +++ b/packages/core/src/api/admin-sdk-transport.ts @@ -1,14 +1,17 @@ import { readBoundedJsonResponse, readBoundedResponseText } from "../utils/fetchJson"; +import { consumeAdminBytes } from "./admin-byte-reader"; import { type ApiClient, ApiClientError, + ApiNativeDownloadUnavailableError, + ApiRequestAbortedError, type ApiRequestOptions, apiClient, parseRetryAfter, } from "./client"; import { accountAdminOperations } from "./contracts/account-admin"; import { activityAdminOperations } from "./contracts/activity-admin"; -import type { AdminOperation } from "./contracts/admin"; +import { type AdminOperation, assertAdminOperationContract } from "./contracts/admin"; import { dataManagerAdminOperations } from "./contracts/data-manager-admin"; import { extensionAdminOperations } from "./contracts/extension-admin"; import { integrationAdminOperations } from "./contracts/integration-admin"; @@ -23,7 +26,27 @@ export type AdminSdkRequestOptions = Omit maxErrorBytes?: number; /** Called within ApiClient's abort/deadline lifetime; never receives a raw Response. */ onTextChunk?: (chunk: string) => void | Promise; + onByteChunk?: (chunk: Uint8Array, signal: AbortSignal) => void | Promise; + onByteHeaders?: (headers: Headers, signal: AbortSignal) => void | Promise; }; +export type AdminBinaryUploadRequestOptions = Omit< + AdminSdkRequestOptions, + "body" | "onTextChunk" | "keepalive" +>; +export type AdminByteResponseRequestOptions = Omit< + AdminSdkRequestOptions, + "body" | "onTextChunk" | "keepalive" +> & { + onByteChunk: (chunk: Uint8Array, signal: AbortSignal) => void | Promise; + onByteHeaders?: (headers: Headers, signal: AbortSignal) => void | Promise; +}; +export type { AdminNativeDownloadTarget } from "./client"; +export type AdminNativeDownloadTargetOptions = { + client?: ApiClient; + signal?: AbortSignal; + method?: "GET"; + headers?: Readonly>; +}; /** Required consumer for incremental plain text; successful envelope data is undefined. */ export type AdminTextStreamRequestOptions = AdminSdkRequestOptions & { onTextChunk: NonNullable; @@ -143,6 +166,7 @@ export function createAdminOperationBinder { const operation: AdminOperation = registry[operationId]; + assertAdminOperationContract(operation); return async ( url: string, options: AdminSdkRequestOptions = {}, @@ -157,6 +181,8 @@ export function createAdminOperationBinder contract.kind === "bytes", + ), + byteRequest = operation.requestBody?.kind === "bytes", + binary = byteRequest || byteResponse; + if (byteResponse && (!onByteChunk || body !== undefined || onTextChunk || keepalive)) + throw new Error("Byte response requires an owned byte consumer"); + if (byteRequest && (!(body instanceof Blob) || onTextChunk || keepalive)) + throw new Error("Binary upload requires the original Blob body"); + const effectiveTimeout = binary ? (timeoutMs ?? 72 * 3600_000) : timeoutMs; + if ( + binary && + (!Number.isFinite(effectiveTimeout) || + !effectiveTimeout || + effectiveTimeout < 0 || + effectiveTimeout > 7 * 24 * 3600_000) + ) + throw new Error("Invalid byte transfer deadline"); // SSE is deliberately browser-owned and cannot enter the finite/JSON reader. if ( Object.values(operation.responses).some( @@ -177,18 +221,27 @@ export function createAdminOperationBinder => { const contract = operation.responses[response.status]; - if (!contract) throw new ApiContractError(response.status); + if (!contract) { + if (binary && response.body && !response.body.locked) + await response.body.cancel().catch(() => {}); + throw new ApiContractError(response.status); + } try { if ( contract.headers && @@ -213,7 +266,11 @@ export function createAdminOperationBinder {}); + if (effectiveSignal.aborted || error instanceof ApiRequestAbortedError) throw error; throw new ApiContractError(response.status); } }, - { signal, timeoutMs, cache, headers: requestHeaders, acceptedStatuses: "all" }, + { + signal, + timeoutMs: effectiveTimeout, + cache, + headers: requestHeaders, + acceptedStatuses: "all", + ...(byteRequest ? { expectedRequestMediaType: "application/octet-stream" as const } : {}), + }, ); return result as T; }; @@ -310,3 +376,59 @@ export function adminErrorMessage(error: unknown, fallback: string): string { } return fallback; } + +type ByteOperationId = { + [K in OperationId]: Extract< + (typeof operations)[K]["responses"][keyof (typeof operations)[K]["responses"]], + { kind: "bytes" } + > extends never + ? never + : K; +}[OperationId]; +type BinaryUploadOperationId = { + [K in OperationId]: (typeof operations)[K] extends { requestBody: { kind: "bytes" } } ? K : never; +}[OperationId]; +export function bindAdminByteOperation(operationId: ByteOperationId) { + return bindAdminOperation(operationId); +} +export function bindAdminBinaryUploadOperation(operationId: BinaryUploadOperationId) { + return bindAdminOperation(operationId); +} +export function bindAdminDownloadTarget(operationId: ByteOperationId) { + const operation: AdminOperation = operations[operationId]; + if ( + operation.method !== "GET" || + !Object.values(operation.responses).some((c) => c.kind === "bytes" && c.nativeDownload) + ) + throw Error("Operation cannot construct a native download"); + return async (url: string, options: AdminNativeDownloadTargetOptions = {}) => { + const supported = new Set(["client", "signal", "method", "headers"]); + if (Reflect.ownKeys(options).some((k) => typeof k !== "string" || !supported.has(k))) + throw new ApiNativeDownloadUnavailableError("headers"); + if (!matchesBoundRequest(url, options.method ?? "GET", operation)) + throw new ApiNativeDownloadUnavailableError("origin"); + const parsedUrl = new URL(url, "https://api.invalid"), + template = operation.path.split("/"), + segments = parsedUrl.pathname.split("/"); + const params = Object.fromEntries( + template.flatMap((part, index) => + part.startsWith(":") ? [[part.slice(1), decodeURIComponent(segments[index])]] : [], + ), + ); + if (operation.request?.params && !operation.request.params.safeParse(params).success) + throw new ApiNativeDownloadUnavailableError("origin"); + if (!operation.request?.query && parsedUrl.search) + throw new ApiNativeDownloadUnavailableError("origin"); + if (options.headers && Reflect.ownKeys(options.headers).length) + throw new ApiNativeDownloadUnavailableError("headers"); + if ( + operation.request?.query && + !operation.request.query.safeParse( + Object.fromEntries(new URL(url, "https://api.invalid").searchParams), + ).success + ) + throw new ApiNativeDownloadUnavailableError("origin"); + options.signal?.throwIfAborted(); + return (options.client ?? apiClient).downloadTarget(url); + }; +} diff --git a/packages/core/src/api/admin-sdk.ts b/packages/core/src/api/admin-sdk.ts index 50696e689..255cf0199 100644 --- a/packages/core/src/api/admin-sdk.ts +++ b/packages/core/src/api/admin-sdk.ts @@ -1,8 +1,12 @@ /** Browser administration bindings use application API credentials, never service tokens. */ export { + type AdminBinaryUploadRequestOptions, + type AdminByteResponseRequestOptions, type AdminEventStreamTarget, type AdminEventStreamTargetOptions, + type AdminNativeDownloadTarget, + type AdminNativeDownloadTargetOptions, type AdminSdkRequestOptions, type AdminTextStreamRequestOptions, adminErrorMessage, diff --git a/packages/core/src/api/client.ts b/packages/core/src/api/client.ts index fa562ad68..ce95941e8 100644 --- a/packages/core/src/api/client.ts +++ b/packages/core/src/api/client.ts @@ -190,6 +190,16 @@ export interface ApiRequestOptions { export interface ApiResponseOptions extends ApiRequestOptions { /** SDK-owned status validation; `all` passes every response to the consumer. */ acceptedStatuses?: readonly number[] | "all"; + /** SDK-owned check after all configured header sources have merged. */ + expectedRequestMediaType?: "application/octet-stream"; +} + +export type AdminNativeDownloadTarget = { url: string; ownership: "browser" }; +export class ApiNativeDownloadUnavailableError extends Error { + constructor(readonly reason: "runtime" | "origin" | "credentials" | "headers" | "interceptor") { + super("Native downloads are unavailable for this API client."); + this.name = "ApiNativeDownloadUnavailableError"; + } } /** @@ -238,6 +248,36 @@ export class ApiClient { return url.toString(); } + /** A browser navigation owns the subsequent download; it cannot reproduce client headers. */ + downloadTarget(path: string): AdminNativeDownloadTarget { + if (typeof window === "undefined") throw new ApiNativeDownloadUnavailableError("runtime"); + const cfg = this.config(); + if (cfg.headerInterceptor !== undefined) + throw new ApiNativeDownloadUnavailableError("interceptor"); + if (cfg.credentials !== "include" && cfg.credentials !== "same-origin") + throw new ApiNativeDownloadUnavailableError("credentials"); + try { + const origin = new URL(window.location.origin), + base = new URL(cfg.baseUrl || "/", origin); + if ( + !/^https?:$/.test(base.protocol) || + base.origin !== origin.origin || + base.username || + base.password || + base.hash || + !path.startsWith("/") || + path.startsWith("//") + ) + throw Error(); + const target = new URL(path, base); + if (target.origin !== origin.origin || target.username || target.password || target.hash) + throw Error(); + return { url: target.toString(), ownership: "browser" }; + } catch { + throw new ApiNativeDownloadUnavailableError("origin"); + } + } + /** Native EventSource cannot omit same-origin cookies or send custom headers. */ connectionTarget(path: string): { url: string; withCredentials: boolean } { const cfg = this.config(); @@ -275,7 +315,7 @@ export class ApiClient { private async send( url: string, init: RequestInit, - options: ApiRequestOptions, + options: ApiResponseOptions, consume: (response: Response, signal: AbortSignal) => Promise, ): Promise { const cfg = this.config(); @@ -303,6 +343,12 @@ export class ApiClient { headers.set(key, value); }); } + if ( + options.expectedRequestMediaType && + headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== + options.expectedRequestMediaType + ) + throw new TypeError("Request media type does not match its declared byte contract"); response = await fetch(url, { ...init, ...(options.cache === undefined ? {} : { cache: options.cache }), diff --git a/packages/core/src/api/contracts/admin.ts b/packages/core/src/api/contracts/admin.ts index 3297a047a..be310c449 100644 --- a/packages/core/src/api/contracts/admin.ts +++ b/packages/core/src/api/contracts/admin.ts @@ -48,6 +48,12 @@ export const adminStream = (mediaType: string, headers?: z.ZodType) => ({ mediaType, ...(headers ? { headers } : {}), }); +export const adminBytes = (mediaType: string, headers?: z.ZodType, nativeDownload?: true) => ({ + kind: "bytes" as const, + mediaType, + ...(headers ? { headers } : {}), + ...(nativeDownload ? { nativeDownload } : {}), +}); export const adminCommonResponses = { 400: adminJson(httpErrorSchema), 401: adminJson(httpErrorSchema), @@ -61,11 +67,35 @@ export type AdminResponseContract = | { kind: "json"; schema: z.ZodType; headers?: z.ZodType } | { kind: "text"; schema: z.ZodType; mediaType: string; headers?: z.ZodType } | { kind: "empty"; headers?: z.ZodType } + | { kind: "bytes"; mediaType: string; headers?: z.ZodType; nativeDownload?: true } | { kind: "stream"; mediaType: string; headers?: z.ZodType }; export interface AdminOperation { method: "GET" | "POST" | "DELETE" | "PUT" | "PATCH"; path: `/api/${string}`; auth: "service" | "admin"; request?: { body?: z.ZodType; query?: z.ZodType; params?: z.ZodType; headers?: z.ZodType }; + requestBody?: { kind: "bytes"; mediaType: "application/octet-stream" }; responses: Record; } + +/** Reject malformed first-party descriptors before registration or SDK acquisition. */ +export function assertAdminOperationContract(operation: AdminOperation): void { + if ( + operation.requestBody && + (operation.request?.body || + operation.requestBody.kind !== "bytes" || + operation.requestBody.mediaType !== "application/octet-stream") + ) + throw Error("Ambiguous administration binary request"); + for (const [status, contract] of Object.entries(operation.responses)) { + if ( + "nativeDownload" in contract && + (contract.nativeDownload !== true || + contract.kind !== "bytes" || + operation.method !== "GET" || + Number(status) < 200 || + Number(status) >= 300) + ) + throw Error("Native downloads require a successful GET byte response"); + } +} diff --git a/packages/core/src/api/contracts/service-admin/artifact-transfers.ts b/packages/core/src/api/contracts/service-admin/artifact-transfers.ts new file mode 100644 index 000000000..3abb82d96 --- /dev/null +++ b/packages/core/src/api/contracts/service-admin/artifact-transfers.ts @@ -0,0 +1,102 @@ +import z from "zod/v4"; +import { transferPurposeSchema, transferStageSchema } from "../../../ops/artifact-transfer"; +import { type AdminOperation, adminBytes, adminJson, httpErrorSchema } from "../admin"; + +const params = z.strictObject({ stageId: z.uuid() }); +export const adminTransferCreateSchema = z.strictObject({ + purpose: transferPurposeSchema, + direction: z.literal("upload"), + declaredBytes: z.number().safe().int().nonnegative(), + expectedSha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), +}); +export const adminTransferPageSchema = z.strictObject({ + limit: z.coerce.number().int().min(1).max(100).default(50), + offset: z.coerce.number().safe().int().nonnegative().max(1_000_000).default(0), +}); +export const artifactDownloadHeadersSchema = z.object({ + "content-type": z.literal("application/octet-stream"), + "content-length": z.string().regex(/^(0|[1-9][0-9]*)$/), + "content-disposition": z.literal('attachment; filename="openmapx-artifact.tar"'), + "x-artifact-sha256": z.string().regex(/^[a-f0-9]{64}$/), + "cache-control": z.literal("no-store"), + "x-content-type-options": z.literal("nosniff"), +}); +const errors = { + 400: adminJson(httpErrorSchema), + 401: adminJson(httpErrorSchema), + 403: adminJson(httpErrorSchema), + 404: adminJson(httpErrorSchema), + 409: adminJson(httpErrorSchema), + 410: adminJson(httpErrorSchema), + 413: adminJson(httpErrorSchema), + 415: adminJson(httpErrorSchema), + 422: adminJson(httpErrorSchema), + 429: adminJson(httpErrorSchema), + 500: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), + 507: adminJson(httpErrorSchema), +}; +export const artifactTransferAdminOperations = { + listAdminArtifactTransfers: { + method: "GET", + path: "/api/admin/artifact-transfers", + auth: "admin", + request: { query: adminTransferPageSchema }, + responses: { + 200: adminJson( + z.strictObject({ + items: z.array(transferStageSchema).max(100), + total: z.number().safe().int().nonnegative(), + limit: z.number().int(), + offset: z.number().int(), + }), + ), + ...errors, + }, + }, + createAdminArtifactTransfer: { + method: "POST", + path: "/api/admin/artifact-transfers", + auth: "admin", + request: { + body: adminTransferCreateSchema, + headers: z.object({ "idempotency-key": z.string().min(1).max(128) }), + }, + responses: { 200: adminJson(transferStageSchema), ...errors }, + }, + getAdminArtifactTransfer: { + method: "GET", + path: "/api/admin/artifact-transfers/:stageId", + auth: "admin", + request: { params }, + responses: { 200: adminJson(transferStageSchema), ...errors }, + }, + deleteAdminArtifactTransfer: { + method: "DELETE", + path: "/api/admin/artifact-transfers/:stageId", + auth: "admin", + request: { params }, + responses: { 200: adminJson(z.strictObject({ deleted: z.literal(true) })), ...errors }, + }, + uploadAdminArtifactTransferContent: { + method: "PUT", + path: "/api/admin/artifact-transfers/:stageId/content", + auth: "admin", + request: { params }, + requestBody: { kind: "bytes", mediaType: "application/octet-stream" }, + responses: { 200: adminJson(transferStageSchema), ...errors }, + }, + downloadAdminArtifactTransferContent: { + method: "GET", + path: "/api/admin/artifact-transfers/:stageId/content", + auth: "admin", + request: { params }, + responses: { + 200: adminBytes("application/octet-stream", artifactDownloadHeadersSchema, true), + ...errors, + }, + }, +} as const satisfies Record; diff --git a/packages/core/src/api/contracts/service-admin/index.ts b/packages/core/src/api/contracts/service-admin/index.ts index fea3e00d0..9f667fef6 100644 --- a/packages/core/src/api/contracts/service-admin/index.ts +++ b/packages/core/src/api/contracts/service-admin/index.ts @@ -1,3 +1,7 @@ +import { artifactTransferAdminOperations } from "./artifact-transfers"; + +export * from "./artifact-transfers"; + import z from "zod/v4"; import { type AdminOperation, @@ -74,6 +78,7 @@ export * from "./workflows"; /** Complete service/workflow family. Authentication metadata never grants access. */ export const serviceAdminOperations = { + ...artifactTransferAdminOperations, listAdminServices: { method: "GET", path: "/api/admin/services", diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index d2a3f6507..eb7d2983a 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -3,6 +3,8 @@ import { type AdminEventStreamTargetOptions, type AdminSdkRequestOptions, + bindAdminBinaryUploadOperation, + bindAdminByteOperation, bindAdminOperation, bindAdminStreamOperation, } from "../../admin-sdk-transport"; @@ -77,6 +79,13 @@ export async function createAdminAccountRequest( ): Promise { return createAdminAccountBound(url, options); } +const createAdminArtifactTransferBound = bindAdminOperation("createAdminArtifactTransfer"); +export async function createAdminArtifactTransferRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return createAdminArtifactTransferBound(url, options); +} const createAdminServiceBackupBound = bindAdminOperation("createAdminServiceBackup"); export async function createAdminServiceBackupRequest( url: string, @@ -84,6 +93,13 @@ export async function createAdminServiceBackupRequest( ): Promise { return createAdminServiceBackupBound(url, options); } +const deleteAdminArtifactTransferBound = bindAdminOperation("deleteAdminArtifactTransfer"); +export async function deleteAdminArtifactTransferRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return deleteAdminArtifactTransferBound(url, options); +} const deleteAdminIntegrationBindingBound = bindAdminOperation("deleteAdminIntegrationBinding"); export async function deleteAdminIntegrationBindingRequest( url: string, @@ -130,6 +146,15 @@ export async function disableAdminIntegrationRequest( ): Promise { return disableAdminIntegrationBound(url, options); } +const downloadAdminArtifactTransferContentBound = bindAdminByteOperation( + "downloadAdminArtifactTransferContent", +); +export async function downloadAdminArtifactTransferContentRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return downloadAdminArtifactTransferContentBound(url, options); +} const enableAdminIntegrationBound = bindAdminOperation("enableAdminIntegration"); export async function enableAdminIntegrationRequest( url: string, @@ -167,6 +192,13 @@ export async function getAdminActivityJobStreamMetricsRequest( ): Promise { return getAdminActivityJobStreamMetricsBound(url, options); } +const getAdminArtifactTransferBound = bindAdminOperation("getAdminArtifactTransfer"); +export async function getAdminArtifactTransferRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminArtifactTransferBound(url, options); +} const getAdminAuthPolicyBound = bindAdminOperation("getAdminAuthPolicy"); export async function getAdminAuthPolicyRequest( url: string, @@ -443,6 +475,13 @@ export async function listAdminActivityJobsRequest( ): Promise { return listAdminActivityJobsBound(url, options); } +const listAdminArtifactTransfersBound = bindAdminOperation("listAdminArtifactTransfers"); +export async function listAdminArtifactTransfersRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return listAdminArtifactTransfersBound(url, options); +} const listAdminCredentialsBound = bindAdminOperation("listAdminCredentials"); export async function listAdminCredentialsRequest( url: string, @@ -857,6 +896,15 @@ export async function updateAdminServiceConfigRequest( ): Promise { return updateAdminServiceConfigBound(url, options); } +const uploadAdminArtifactTransferContentBound = bindAdminBinaryUploadOperation( + "uploadAdminArtifactTransferContent", +); +export async function uploadAdminArtifactTransferContentRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return uploadAdminArtifactTransferContentBound(url, options); +} const validateAdminTransitPinsBound = bindAdminOperation("validateAdminTransitPins"); export async function validateAdminTransitPinsRequest( url: string, diff --git a/packages/core/src/api/generated/admin/service-admin.generated.ts b/packages/core/src/api/generated/admin/service-admin.generated.ts index d5c020d7c..d24f6354b 100644 --- a/packages/core/src/api/generated/admin/service-admin.generated.ts +++ b/packages/core/src/api/generated/admin/service-admin.generated.ts @@ -1,11 +1,23 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. -import type { AdminTextStreamRequestOptions } from "../../admin-sdk-transport"; + +import type { + AdminBinaryUploadRequestOptions, + AdminByteResponseRequestOptions, + AdminNativeDownloadTarget, + AdminNativeDownloadTargetOptions, + AdminTextStreamRequestOptions, +} from "../../admin-sdk-transport"; +import { bindAdminDownloadTarget } from "../../admin-sdk-transport"; import { checkAdminServicesRequest, + createAdminArtifactTransferRequest, createAdminServiceBackupRequest, + deleteAdminArtifactTransferRequest, deleteAdminServiceBackupRequest, deleteAdminServiceCredentialRequest, + downloadAdminArtifactTransferContentRequest, + getAdminArtifactTransferRequest, getAdminComposePreviewRequest, getAdminDawarichRequest, getAdminDeploymentRequest, @@ -15,6 +27,7 @@ import { getAdminServiceRequest, getAdminServiceSelectionRequest, getAdminSharedMobilityRequest, + listAdminArtifactTransfersRequest, listAdminOperationsRequest, listAdminServiceBackupsRequest, listAdminServicesRequest, @@ -32,6 +45,7 @@ import { stopAdminComposeRequest, streamAdminServiceLogsRequest, updateAdminServiceConfigRequest, + uploadAdminArtifactTransferContentRequest, } from "./operation-bindings.generated"; import type { CheckAdminServices200, @@ -43,6 +57,22 @@ import type { CheckAdminServices429, CheckAdminServices500, CheckAdminServicesBody, + CreateAdminArtifactTransfer200, + CreateAdminArtifactTransfer400, + CreateAdminArtifactTransfer401, + CreateAdminArtifactTransfer403, + CreateAdminArtifactTransfer404, + CreateAdminArtifactTransfer409, + CreateAdminArtifactTransfer410, + CreateAdminArtifactTransfer413, + CreateAdminArtifactTransfer415, + CreateAdminArtifactTransfer422, + CreateAdminArtifactTransfer429, + CreateAdminArtifactTransfer500, + CreateAdminArtifactTransfer503, + CreateAdminArtifactTransfer507, + CreateAdminArtifactTransferBody, + CreateAdminArtifactTransferHeaders, CreateAdminServiceBackup200, CreateAdminServiceBackup400, CreateAdminServiceBackup401, @@ -52,6 +82,20 @@ import type { CreateAdminServiceBackup429, CreateAdminServiceBackup500, CreateAdminServiceBackupBody, + DeleteAdminArtifactTransfer200, + DeleteAdminArtifactTransfer400, + DeleteAdminArtifactTransfer401, + DeleteAdminArtifactTransfer403, + DeleteAdminArtifactTransfer404, + DeleteAdminArtifactTransfer409, + DeleteAdminArtifactTransfer410, + DeleteAdminArtifactTransfer413, + DeleteAdminArtifactTransfer415, + DeleteAdminArtifactTransfer422, + DeleteAdminArtifactTransfer429, + DeleteAdminArtifactTransfer500, + DeleteAdminArtifactTransfer503, + DeleteAdminArtifactTransfer507, DeleteAdminServiceBackup200, DeleteAdminServiceBackup400, DeleteAdminServiceBackup401, @@ -74,6 +118,33 @@ import type { DeleteAdminServiceCredential500, DeleteAdminServiceCredentialBody, DeleteAdminServiceCredentialHeaders, + DownloadAdminArtifactTransferContent400, + DownloadAdminArtifactTransferContent401, + DownloadAdminArtifactTransferContent403, + DownloadAdminArtifactTransferContent404, + DownloadAdminArtifactTransferContent409, + DownloadAdminArtifactTransferContent410, + DownloadAdminArtifactTransferContent413, + DownloadAdminArtifactTransferContent415, + DownloadAdminArtifactTransferContent422, + DownloadAdminArtifactTransferContent429, + DownloadAdminArtifactTransferContent500, + DownloadAdminArtifactTransferContent503, + DownloadAdminArtifactTransferContent507, + GetAdminArtifactTransfer200, + GetAdminArtifactTransfer400, + GetAdminArtifactTransfer401, + GetAdminArtifactTransfer403, + GetAdminArtifactTransfer404, + GetAdminArtifactTransfer409, + GetAdminArtifactTransfer410, + GetAdminArtifactTransfer413, + GetAdminArtifactTransfer415, + GetAdminArtifactTransfer422, + GetAdminArtifactTransfer429, + GetAdminArtifactTransfer500, + GetAdminArtifactTransfer503, + GetAdminArtifactTransfer507, GetAdminComposePreview400, GetAdminComposePreview401, GetAdminComposePreview403, @@ -155,6 +226,21 @@ import type { GetAdminSharedMobility415, GetAdminSharedMobility429, GetAdminSharedMobility500, + ListAdminArtifactTransfers200, + ListAdminArtifactTransfers400, + ListAdminArtifactTransfers401, + ListAdminArtifactTransfers403, + ListAdminArtifactTransfers404, + ListAdminArtifactTransfers409, + ListAdminArtifactTransfers410, + ListAdminArtifactTransfers413, + ListAdminArtifactTransfers415, + ListAdminArtifactTransfers422, + ListAdminArtifactTransfers429, + ListAdminArtifactTransfers500, + ListAdminArtifactTransfers503, + ListAdminArtifactTransfers507, + ListAdminArtifactTransfersParams, ListAdminOperations200, ListAdminOperations400, ListAdminOperations401, @@ -331,7 +417,790 @@ import type { UpdateAdminServiceConfig500, UpdateAdminServiceConfigBody, UpdateAdminServiceConfigHeaders, + UploadAdminArtifactTransferContent200, + UploadAdminArtifactTransferContent400, + UploadAdminArtifactTransferContent401, + UploadAdminArtifactTransferContent403, + UploadAdminArtifactTransferContent404, + UploadAdminArtifactTransferContent409, + UploadAdminArtifactTransferContent410, + UploadAdminArtifactTransferContent413, + UploadAdminArtifactTransferContent415, + UploadAdminArtifactTransferContent422, + UploadAdminArtifactTransferContent429, + UploadAdminArtifactTransferContent500, + UploadAdminArtifactTransferContent503, + UploadAdminArtifactTransferContent507, } from "./service-models"; +export type listAdminArtifactTransfersResponse200 = { + data: ListAdminArtifactTransfers200; + status: 200; +}; + +export type listAdminArtifactTransfersResponse400 = { + data: ListAdminArtifactTransfers400; + status: 400; +}; + +export type listAdminArtifactTransfersResponse401 = { + data: ListAdminArtifactTransfers401; + status: 401; +}; + +export type listAdminArtifactTransfersResponse403 = { + data: ListAdminArtifactTransfers403; + status: 403; +}; + +export type listAdminArtifactTransfersResponse404 = { + data: ListAdminArtifactTransfers404; + status: 404; +}; + +export type listAdminArtifactTransfersResponse409 = { + data: ListAdminArtifactTransfers409; + status: 409; +}; + +export type listAdminArtifactTransfersResponse410 = { + data: ListAdminArtifactTransfers410; + status: 410; +}; + +export type listAdminArtifactTransfersResponse413 = { + data: ListAdminArtifactTransfers413; + status: 413; +}; + +export type listAdminArtifactTransfersResponse415 = { + data: ListAdminArtifactTransfers415; + status: 415; +}; + +export type listAdminArtifactTransfersResponse422 = { + data: ListAdminArtifactTransfers422; + status: 422; +}; + +export type listAdminArtifactTransfersResponse429 = { + data: ListAdminArtifactTransfers429; + status: 429; +}; + +export type listAdminArtifactTransfersResponse500 = { + data: ListAdminArtifactTransfers500; + status: 500; +}; + +export type listAdminArtifactTransfersResponse503 = { + data: ListAdminArtifactTransfers503; + status: 503; +}; + +export type listAdminArtifactTransfersResponse507 = { + data: ListAdminArtifactTransfers507; + status: 507; +}; + +export type listAdminArtifactTransfersResponseSuccess = listAdminArtifactTransfersResponse200 & { + headers: Headers; +}; +export type listAdminArtifactTransfersResponseError = ( + | listAdminArtifactTransfersResponse400 + | listAdminArtifactTransfersResponse401 + | listAdminArtifactTransfersResponse403 + | listAdminArtifactTransfersResponse404 + | listAdminArtifactTransfersResponse409 + | listAdminArtifactTransfersResponse410 + | listAdminArtifactTransfersResponse413 + | listAdminArtifactTransfersResponse415 + | listAdminArtifactTransfersResponse422 + | listAdminArtifactTransfersResponse429 + | listAdminArtifactTransfersResponse500 + | listAdminArtifactTransfersResponse503 + | listAdminArtifactTransfersResponse507 +) & { + headers: Headers; +}; + +export type listAdminArtifactTransfersResponse = + | listAdminArtifactTransfersResponseSuccess + | listAdminArtifactTransfersResponseError; + +export const getListAdminArtifactTransfersUrl = (params?: ListAdminArtifactTransfersParams) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 + ? `/api/admin/artifact-transfers?${stringifiedParams}` + : `/api/admin/artifact-transfers`; +}; + +export const listAdminArtifactTransfers = async ( + params?: ListAdminArtifactTransfersParams, + options?: Parameters[1], +): Promise => { + return listAdminArtifactTransfersRequest( + getListAdminArtifactTransfersUrl(params), + { + ...options, + method: "GET", + }, + ); +}; + +export type createAdminArtifactTransferResponse200 = { + data: CreateAdminArtifactTransfer200; + status: 200; +}; + +export type createAdminArtifactTransferResponse400 = { + data: CreateAdminArtifactTransfer400; + status: 400; +}; + +export type createAdminArtifactTransferResponse401 = { + data: CreateAdminArtifactTransfer401; + status: 401; +}; + +export type createAdminArtifactTransferResponse403 = { + data: CreateAdminArtifactTransfer403; + status: 403; +}; + +export type createAdminArtifactTransferResponse404 = { + data: CreateAdminArtifactTransfer404; + status: 404; +}; + +export type createAdminArtifactTransferResponse409 = { + data: CreateAdminArtifactTransfer409; + status: 409; +}; + +export type createAdminArtifactTransferResponse410 = { + data: CreateAdminArtifactTransfer410; + status: 410; +}; + +export type createAdminArtifactTransferResponse413 = { + data: CreateAdminArtifactTransfer413; + status: 413; +}; + +export type createAdminArtifactTransferResponse415 = { + data: CreateAdminArtifactTransfer415; + status: 415; +}; + +export type createAdminArtifactTransferResponse422 = { + data: CreateAdminArtifactTransfer422; + status: 422; +}; + +export type createAdminArtifactTransferResponse429 = { + data: CreateAdminArtifactTransfer429; + status: 429; +}; + +export type createAdminArtifactTransferResponse500 = { + data: CreateAdminArtifactTransfer500; + status: 500; +}; + +export type createAdminArtifactTransferResponse503 = { + data: CreateAdminArtifactTransfer503; + status: 503; +}; + +export type createAdminArtifactTransferResponse507 = { + data: CreateAdminArtifactTransfer507; + status: 507; +}; + +export type createAdminArtifactTransferResponseSuccess = createAdminArtifactTransferResponse200 & { + headers: Headers; +}; +export type createAdminArtifactTransferResponseError = ( + | createAdminArtifactTransferResponse400 + | createAdminArtifactTransferResponse401 + | createAdminArtifactTransferResponse403 + | createAdminArtifactTransferResponse404 + | createAdminArtifactTransferResponse409 + | createAdminArtifactTransferResponse410 + | createAdminArtifactTransferResponse413 + | createAdminArtifactTransferResponse415 + | createAdminArtifactTransferResponse422 + | createAdminArtifactTransferResponse429 + | createAdminArtifactTransferResponse500 + | createAdminArtifactTransferResponse503 + | createAdminArtifactTransferResponse507 +) & { + headers: Headers; +}; + +export type createAdminArtifactTransferResponse = + | createAdminArtifactTransferResponseSuccess + | createAdminArtifactTransferResponseError; + +export const getCreateAdminArtifactTransferUrl = () => { + return `/api/admin/artifact-transfers`; +}; + +export const createAdminArtifactTransfer = async ( + createAdminArtifactTransferBody: CreateAdminArtifactTransferBody, + headers: CreateAdminArtifactTransferHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return createAdminArtifactTransferRequest( + getCreateAdminArtifactTransferUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(createAdminArtifactTransferBody), + }, + ); +}; + +export type getAdminArtifactTransferResponse200 = { + data: GetAdminArtifactTransfer200; + status: 200; +}; + +export type getAdminArtifactTransferResponse400 = { + data: GetAdminArtifactTransfer400; + status: 400; +}; + +export type getAdminArtifactTransferResponse401 = { + data: GetAdminArtifactTransfer401; + status: 401; +}; + +export type getAdminArtifactTransferResponse403 = { + data: GetAdminArtifactTransfer403; + status: 403; +}; + +export type getAdminArtifactTransferResponse404 = { + data: GetAdminArtifactTransfer404; + status: 404; +}; + +export type getAdminArtifactTransferResponse409 = { + data: GetAdminArtifactTransfer409; + status: 409; +}; + +export type getAdminArtifactTransferResponse410 = { + data: GetAdminArtifactTransfer410; + status: 410; +}; + +export type getAdminArtifactTransferResponse413 = { + data: GetAdminArtifactTransfer413; + status: 413; +}; + +export type getAdminArtifactTransferResponse415 = { + data: GetAdminArtifactTransfer415; + status: 415; +}; + +export type getAdminArtifactTransferResponse422 = { + data: GetAdminArtifactTransfer422; + status: 422; +}; + +export type getAdminArtifactTransferResponse429 = { + data: GetAdminArtifactTransfer429; + status: 429; +}; + +export type getAdminArtifactTransferResponse500 = { + data: GetAdminArtifactTransfer500; + status: 500; +}; + +export type getAdminArtifactTransferResponse503 = { + data: GetAdminArtifactTransfer503; + status: 503; +}; + +export type getAdminArtifactTransferResponse507 = { + data: GetAdminArtifactTransfer507; + status: 507; +}; + +export type getAdminArtifactTransferResponseSuccess = getAdminArtifactTransferResponse200 & { + headers: Headers; +}; +export type getAdminArtifactTransferResponseError = ( + | getAdminArtifactTransferResponse400 + | getAdminArtifactTransferResponse401 + | getAdminArtifactTransferResponse403 + | getAdminArtifactTransferResponse404 + | getAdminArtifactTransferResponse409 + | getAdminArtifactTransferResponse410 + | getAdminArtifactTransferResponse413 + | getAdminArtifactTransferResponse415 + | getAdminArtifactTransferResponse422 + | getAdminArtifactTransferResponse429 + | getAdminArtifactTransferResponse500 + | getAdminArtifactTransferResponse503 + | getAdminArtifactTransferResponse507 +) & { + headers: Headers; +}; + +export type getAdminArtifactTransferResponse = + | getAdminArtifactTransferResponseSuccess + | getAdminArtifactTransferResponseError; + +export const getGetAdminArtifactTransferUrl = (stageId: string) => { + return `/api/admin/artifact-transfers/${encodeURIComponent(String(stageId))}`; +}; + +export const getAdminArtifactTransfer = async ( + stageId: string, + options?: Parameters[1], +): Promise => { + return getAdminArtifactTransferRequest( + getGetAdminArtifactTransferUrl(stageId), + { + ...options, + method: "GET", + }, + ); +}; + +export type deleteAdminArtifactTransferResponse200 = { + data: DeleteAdminArtifactTransfer200; + status: 200; +}; + +export type deleteAdminArtifactTransferResponse400 = { + data: DeleteAdminArtifactTransfer400; + status: 400; +}; + +export type deleteAdminArtifactTransferResponse401 = { + data: DeleteAdminArtifactTransfer401; + status: 401; +}; + +export type deleteAdminArtifactTransferResponse403 = { + data: DeleteAdminArtifactTransfer403; + status: 403; +}; + +export type deleteAdminArtifactTransferResponse404 = { + data: DeleteAdminArtifactTransfer404; + status: 404; +}; + +export type deleteAdminArtifactTransferResponse409 = { + data: DeleteAdminArtifactTransfer409; + status: 409; +}; + +export type deleteAdminArtifactTransferResponse410 = { + data: DeleteAdminArtifactTransfer410; + status: 410; +}; + +export type deleteAdminArtifactTransferResponse413 = { + data: DeleteAdminArtifactTransfer413; + status: 413; +}; + +export type deleteAdminArtifactTransferResponse415 = { + data: DeleteAdminArtifactTransfer415; + status: 415; +}; + +export type deleteAdminArtifactTransferResponse422 = { + data: DeleteAdminArtifactTransfer422; + status: 422; +}; + +export type deleteAdminArtifactTransferResponse429 = { + data: DeleteAdminArtifactTransfer429; + status: 429; +}; + +export type deleteAdminArtifactTransferResponse500 = { + data: DeleteAdminArtifactTransfer500; + status: 500; +}; + +export type deleteAdminArtifactTransferResponse503 = { + data: DeleteAdminArtifactTransfer503; + status: 503; +}; + +export type deleteAdminArtifactTransferResponse507 = { + data: DeleteAdminArtifactTransfer507; + status: 507; +}; + +export type deleteAdminArtifactTransferResponseSuccess = deleteAdminArtifactTransferResponse200 & { + headers: Headers; +}; +export type deleteAdminArtifactTransferResponseError = ( + | deleteAdminArtifactTransferResponse400 + | deleteAdminArtifactTransferResponse401 + | deleteAdminArtifactTransferResponse403 + | deleteAdminArtifactTransferResponse404 + | deleteAdminArtifactTransferResponse409 + | deleteAdminArtifactTransferResponse410 + | deleteAdminArtifactTransferResponse413 + | deleteAdminArtifactTransferResponse415 + | deleteAdminArtifactTransferResponse422 + | deleteAdminArtifactTransferResponse429 + | deleteAdminArtifactTransferResponse500 + | deleteAdminArtifactTransferResponse503 + | deleteAdminArtifactTransferResponse507 +) & { + headers: Headers; +}; + +export type deleteAdminArtifactTransferResponse = + | deleteAdminArtifactTransferResponseSuccess + | deleteAdminArtifactTransferResponseError; + +export const getDeleteAdminArtifactTransferUrl = (stageId: string) => { + return `/api/admin/artifact-transfers/${encodeURIComponent(String(stageId))}`; +}; + +export const deleteAdminArtifactTransfer = async ( + stageId: string, + options?: Parameters[1], +): Promise => { + return deleteAdminArtifactTransferRequest( + getDeleteAdminArtifactTransferUrl(stageId), + { + ...options, + method: "DELETE", + }, + ); +}; + +export type downloadAdminArtifactTransferContentResponse200 = { + data: undefined; + status: 200; +}; + +export type downloadAdminArtifactTransferContentResponse400 = { + data: DownloadAdminArtifactTransferContent400; + status: 400; +}; + +export type downloadAdminArtifactTransferContentResponse401 = { + data: DownloadAdminArtifactTransferContent401; + status: 401; +}; + +export type downloadAdminArtifactTransferContentResponse403 = { + data: DownloadAdminArtifactTransferContent403; + status: 403; +}; + +export type downloadAdminArtifactTransferContentResponse404 = { + data: DownloadAdminArtifactTransferContent404; + status: 404; +}; + +export type downloadAdminArtifactTransferContentResponse409 = { + data: DownloadAdminArtifactTransferContent409; + status: 409; +}; + +export type downloadAdminArtifactTransferContentResponse410 = { + data: DownloadAdminArtifactTransferContent410; + status: 410; +}; + +export type downloadAdminArtifactTransferContentResponse413 = { + data: DownloadAdminArtifactTransferContent413; + status: 413; +}; + +export type downloadAdminArtifactTransferContentResponse415 = { + data: DownloadAdminArtifactTransferContent415; + status: 415; +}; + +export type downloadAdminArtifactTransferContentResponse422 = { + data: DownloadAdminArtifactTransferContent422; + status: 422; +}; + +export type downloadAdminArtifactTransferContentResponse429 = { + data: DownloadAdminArtifactTransferContent429; + status: 429; +}; + +export type downloadAdminArtifactTransferContentResponse500 = { + data: DownloadAdminArtifactTransferContent500; + status: 500; +}; + +export type downloadAdminArtifactTransferContentResponse503 = { + data: DownloadAdminArtifactTransferContent503; + status: 503; +}; + +export type downloadAdminArtifactTransferContentResponse507 = { + data: DownloadAdminArtifactTransferContent507; + status: 507; +}; + +export type downloadAdminArtifactTransferContentResponseSuccess = + downloadAdminArtifactTransferContentResponse200 & { + headers: Headers; + }; +export type downloadAdminArtifactTransferContentResponseError = ( + | downloadAdminArtifactTransferContentResponse400 + | downloadAdminArtifactTransferContentResponse401 + | downloadAdminArtifactTransferContentResponse403 + | downloadAdminArtifactTransferContentResponse404 + | downloadAdminArtifactTransferContentResponse409 + | downloadAdminArtifactTransferContentResponse410 + | downloadAdminArtifactTransferContentResponse413 + | downloadAdminArtifactTransferContentResponse415 + | downloadAdminArtifactTransferContentResponse422 + | downloadAdminArtifactTransferContentResponse429 + | downloadAdminArtifactTransferContentResponse500 + | downloadAdminArtifactTransferContentResponse503 + | downloadAdminArtifactTransferContentResponse507 +) & { + headers: Headers; +}; + +export type downloadAdminArtifactTransferContentResponse = + | downloadAdminArtifactTransferContentResponseSuccess + | downloadAdminArtifactTransferContentResponseError; + +const downloadAdminArtifactTransferContentClient = (() => { + const getDownloadAdminArtifactTransferContentUrl = (stageId: string) => { + return `/api/admin/artifact-transfers/${encodeURIComponent(String(stageId))}/content`; + }; + + const downloadAdminArtifactTransferContent = async ( + stageId: string, + options?: Parameters[1], + ): Promise => { + return downloadAdminArtifactTransferContentRequest( + getDownloadAdminArtifactTransferContentUrl(stageId), + { + ...options, + method: "GET", + }, + ); + }; + + return { downloadAdminArtifactTransferContent, getDownloadAdminArtifactTransferContentUrl }; +})(); +export const getDownloadAdminArtifactTransferContentUrl = + downloadAdminArtifactTransferContentClient.getDownloadAdminArtifactTransferContentUrl; +type downloadAdminArtifactTransferContentArguments = Parameters< + typeof downloadAdminArtifactTransferContentClient.downloadAdminArtifactTransferContent +>; +export const downloadAdminArtifactTransferContent: ( + stageId: downloadAdminArtifactTransferContentArguments[0], + options: AdminByteResponseRequestOptions, +) => ReturnType< + typeof downloadAdminArtifactTransferContentClient.downloadAdminArtifactTransferContent +> = downloadAdminArtifactTransferContentClient.downloadAdminArtifactTransferContent; +export const getDownloadAdminArtifactTransferContentTarget = ( + stageId: downloadAdminArtifactTransferContentArguments[0], + options?: AdminNativeDownloadTargetOptions, +): Promise => + bindAdminDownloadTarget("downloadAdminArtifactTransferContent")( + getDownloadAdminArtifactTransferContentUrl(stageId), + options, + ); + +export type uploadAdminArtifactTransferContentResponse200 = { + data: UploadAdminArtifactTransferContent200; + status: 200; +}; + +export type uploadAdminArtifactTransferContentResponse400 = { + data: UploadAdminArtifactTransferContent400; + status: 400; +}; + +export type uploadAdminArtifactTransferContentResponse401 = { + data: UploadAdminArtifactTransferContent401; + status: 401; +}; + +export type uploadAdminArtifactTransferContentResponse403 = { + data: UploadAdminArtifactTransferContent403; + status: 403; +}; + +export type uploadAdminArtifactTransferContentResponse404 = { + data: UploadAdminArtifactTransferContent404; + status: 404; +}; + +export type uploadAdminArtifactTransferContentResponse409 = { + data: UploadAdminArtifactTransferContent409; + status: 409; +}; + +export type uploadAdminArtifactTransferContentResponse410 = { + data: UploadAdminArtifactTransferContent410; + status: 410; +}; + +export type uploadAdminArtifactTransferContentResponse413 = { + data: UploadAdminArtifactTransferContent413; + status: 413; +}; + +export type uploadAdminArtifactTransferContentResponse415 = { + data: UploadAdminArtifactTransferContent415; + status: 415; +}; + +export type uploadAdminArtifactTransferContentResponse422 = { + data: UploadAdminArtifactTransferContent422; + status: 422; +}; + +export type uploadAdminArtifactTransferContentResponse429 = { + data: UploadAdminArtifactTransferContent429; + status: 429; +}; + +export type uploadAdminArtifactTransferContentResponse500 = { + data: UploadAdminArtifactTransferContent500; + status: 500; +}; + +export type uploadAdminArtifactTransferContentResponse503 = { + data: UploadAdminArtifactTransferContent503; + status: 503; +}; + +export type uploadAdminArtifactTransferContentResponse507 = { + data: UploadAdminArtifactTransferContent507; + status: 507; +}; + +export type uploadAdminArtifactTransferContentResponseSuccess = + uploadAdminArtifactTransferContentResponse200 & { + headers: Headers; + }; +export type uploadAdminArtifactTransferContentResponseError = ( + | uploadAdminArtifactTransferContentResponse400 + | uploadAdminArtifactTransferContentResponse401 + | uploadAdminArtifactTransferContentResponse403 + | uploadAdminArtifactTransferContentResponse404 + | uploadAdminArtifactTransferContentResponse409 + | uploadAdminArtifactTransferContentResponse410 + | uploadAdminArtifactTransferContentResponse413 + | uploadAdminArtifactTransferContentResponse415 + | uploadAdminArtifactTransferContentResponse422 + | uploadAdminArtifactTransferContentResponse429 + | uploadAdminArtifactTransferContentResponse500 + | uploadAdminArtifactTransferContentResponse503 + | uploadAdminArtifactTransferContentResponse507 +) & { + headers: Headers; +}; + +export type uploadAdminArtifactTransferContentResponse = + | uploadAdminArtifactTransferContentResponseSuccess + | uploadAdminArtifactTransferContentResponseError; + +const uploadAdminArtifactTransferContentClient = (() => { + const getUploadAdminArtifactTransferContentUrl = (stageId: string) => { + return `/api/admin/artifact-transfers/${encodeURIComponent(String(stageId))}/content`; + }; + + const uploadAdminArtifactTransferContent = async ( + stageId: string, + uploadAdminArtifactTransferContentBody: Blob, + options?: Parameters[1], + ): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return uploadAdminArtifactTransferContentRequest( + getUploadAdminArtifactTransferContentUrl(stageId), + { + ...options, + method: "PUT", + headers: { "Content-Type": "application/octet-stream", ...getHeaders(options?.headers) }, + body: uploadAdminArtifactTransferContentBody, + }, + ); + }; + + return { uploadAdminArtifactTransferContent, getUploadAdminArtifactTransferContentUrl }; +})(); +export const getUploadAdminArtifactTransferContentUrl = + uploadAdminArtifactTransferContentClient.getUploadAdminArtifactTransferContentUrl; +type uploadAdminArtifactTransferContentArguments = Parameters< + typeof uploadAdminArtifactTransferContentClient.uploadAdminArtifactTransferContent +>; +export const uploadAdminArtifactTransferContent: ( + stageId: uploadAdminArtifactTransferContentArguments[0], + uploadAdminArtifactTransferContentBody: uploadAdminArtifactTransferContentArguments[1], + options?: AdminBinaryUploadRequestOptions, +) => ReturnType< + typeof uploadAdminArtifactTransferContentClient.uploadAdminArtifactTransferContent +> = uploadAdminArtifactTransferContentClient.uploadAdminArtifactTransferContent; + export type stopAdminComposeResponse400 = { data: StopAdminCompose400; status: 400; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200.ts new file mode 100644 index 000000000..ad1259dc6 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { CreateAdminArtifactTransfer200Cleanup } from "./createAdminArtifactTransfer200Cleanup"; +import type { CreateAdminArtifactTransfer200Direction } from "./createAdminArtifactTransfer200Direction"; +import type { CreateAdminArtifactTransfer200Purpose } from "./createAdminArtifactTransfer200Purpose"; +import type { CreateAdminArtifactTransfer200State } from "./createAdminArtifactTransfer200State"; + +export type CreateAdminArtifactTransfer200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + stageId: string; + purpose: CreateAdminArtifactTransfer200Purpose; + direction: CreateAdminArtifactTransfer200Direction; + state: CreateAdminArtifactTransfer200State; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + bytes: number; + sha256: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + cleanup: CreateAdminArtifactTransfer200Cleanup; +}; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Cleanup.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Cleanup.ts new file mode 100644 index 000000000..695f23e7e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Cleanup.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer200Cleanup = + (typeof CreateAdminArtifactTransfer200Cleanup)[keyof typeof CreateAdminArtifactTransfer200Cleanup]; + +export const CreateAdminArtifactTransfer200Cleanup = { + pending: "pending", + complete: "complete", + failed: "failed", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Direction.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Direction.ts new file mode 100644 index 000000000..d3c4a5411 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Direction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer200Direction = + (typeof CreateAdminArtifactTransfer200Direction)[keyof typeof CreateAdminArtifactTransfer200Direction]; + +export const CreateAdminArtifactTransfer200Direction = { + upload: "upload", + download: "download", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Purpose.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Purpose.ts new file mode 100644 index 000000000..193cde87f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200Purpose.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer200Purpose = + (typeof CreateAdminArtifactTransfer200Purpose)[keyof typeof CreateAdminArtifactTransfer200Purpose]; + +export const CreateAdminArtifactTransfer200Purpose = { + "prepared-artifact": "prepared-artifact", + "instance-migration": "instance-migration", + "integration-artifact": "integration-artifact", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200State.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200State.ts new file mode 100644 index 000000000..044df9027 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer200State.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer200State = + (typeof CreateAdminArtifactTransfer200State)[keyof typeof CreateAdminArtifactTransfer200State]; + +export const CreateAdminArtifactTransfer200State = { + created: "created", + uploading: "uploading", + uploaded: "uploaded", + validating: "validating", + ready: "ready", + claimed: "claimed", + consumed: "consumed", + failed: "failed", + expired: "expired", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer400.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer400.ts new file mode 100644 index 000000000..8fa6ad092 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer401.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer401.ts new file mode 100644 index 000000000..d3e7b2e3f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer403.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer403.ts new file mode 100644 index 000000000..90598c777 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer404.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer404.ts new file mode 100644 index 000000000..5e34af050 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer409.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer409.ts new file mode 100644 index 000000000..ffc8691d1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer410.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer410.ts new file mode 100644 index 000000000..a7d2f5eac --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer413.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer413.ts new file mode 100644 index 000000000..f1cd9c7b5 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer415.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer415.ts new file mode 100644 index 000000000..71766731e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer422.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer422.ts new file mode 100644 index 000000000..c2ae1bee1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer429.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer429.ts new file mode 100644 index 000000000..899305d56 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer500.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer500.ts new file mode 100644 index 000000000..61d0871d9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer503.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer503.ts new file mode 100644 index 000000000..f7394aa0d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer507.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer507.ts new file mode 100644 index 000000000..e261e8654 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransfer507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransfer507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBody.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBody.ts new file mode 100644 index 000000000..68d5f4ba1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBody.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { CreateAdminArtifactTransferBodyDirection } from "./createAdminArtifactTransferBodyDirection"; +import type { CreateAdminArtifactTransferBodyPurpose } from "./createAdminArtifactTransferBodyPurpose"; + +export type CreateAdminArtifactTransferBody = { + purpose: CreateAdminArtifactTransferBodyPurpose; + direction: CreateAdminArtifactTransferBodyDirection; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + declaredBytes: number; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSha256?: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyDirection.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyDirection.ts new file mode 100644 index 000000000..2ad63fefe --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyDirection.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransferBodyDirection = + (typeof CreateAdminArtifactTransferBodyDirection)[keyof typeof CreateAdminArtifactTransferBodyDirection]; + +export const CreateAdminArtifactTransferBodyDirection = { + upload: "upload", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyPurpose.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyPurpose.ts new file mode 100644 index 000000000..3c50f150b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferBodyPurpose.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransferBodyPurpose = + (typeof CreateAdminArtifactTransferBodyPurpose)[keyof typeof CreateAdminArtifactTransferBodyPurpose]; + +export const CreateAdminArtifactTransferBodyPurpose = { + "prepared-artifact": "prepared-artifact", + "instance-migration": "instance-migration", + "integration-artifact": "integration-artifact", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferHeaders.ts b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferHeaders.ts new file mode 100644 index 000000000..d5facd61f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/createAdminArtifactTransferHeaders.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type CreateAdminArtifactTransferHeaders = { + /** + * @minLength 1 + * @maxLength 128 + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer200.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer200.ts new file mode 100644 index 000000000..1c096692d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer200.ts @@ -0,0 +1,6 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer200 = { + deleted: true; +}; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer400.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer400.ts new file mode 100644 index 000000000..c8b5fc6c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer401.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer401.ts new file mode 100644 index 000000000..6d9114bda --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer403.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer403.ts new file mode 100644 index 000000000..5399af1c2 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer404.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer404.ts new file mode 100644 index 000000000..68385c28c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer409.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer409.ts new file mode 100644 index 000000000..6298a5ec8 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer410.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer410.ts new file mode 100644 index 000000000..9c1f550ba --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer413.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer413.ts new file mode 100644 index 000000000..0463c6c6f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer415.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer415.ts new file mode 100644 index 000000000..636d3e1d1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer422.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer422.ts new file mode 100644 index 000000000..479d513bf --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer429.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer429.ts new file mode 100644 index 000000000..3bf6594f4 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer500.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer500.ts new file mode 100644 index 000000000..666103462 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer503.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer503.ts new file mode 100644 index 000000000..cd4a0f60c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer507.ts b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer507.ts new file mode 100644 index 000000000..7230c1ea0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/deleteAdminArtifactTransfer507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DeleteAdminArtifactTransfer507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent400.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent400.ts new file mode 100644 index 000000000..e6bea6e2d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent401.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent401.ts new file mode 100644 index 000000000..d19343834 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent403.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent403.ts new file mode 100644 index 000000000..6d65b59ce --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent404.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent404.ts new file mode 100644 index 000000000..d39786f0a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent409.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent409.ts new file mode 100644 index 000000000..d24431c0e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent410.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent410.ts new file mode 100644 index 000000000..87eadc2a6 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent413.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent413.ts new file mode 100644 index 000000000..2fa3238d0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent415.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent415.ts new file mode 100644 index 000000000..cba33f76d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent422.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent422.ts new file mode 100644 index 000000000..9ff257ba5 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent429.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent429.ts new file mode 100644 index 000000000..cba4cd3fe --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent500.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent500.ts new file mode 100644 index 000000000..519da06da --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent503.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent503.ts new file mode 100644 index 000000000..87dff4964 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent507.ts b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent507.ts new file mode 100644 index 000000000..776ae4a29 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/downloadAdminArtifactTransferContent507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type DownloadAdminArtifactTransferContent507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200.ts new file mode 100644 index 000000000..ddf4d6ea0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminArtifactTransfer200Cleanup } from "./getAdminArtifactTransfer200Cleanup"; +import type { GetAdminArtifactTransfer200Direction } from "./getAdminArtifactTransfer200Direction"; +import type { GetAdminArtifactTransfer200Purpose } from "./getAdminArtifactTransfer200Purpose"; +import type { GetAdminArtifactTransfer200State } from "./getAdminArtifactTransfer200State"; + +export type GetAdminArtifactTransfer200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + stageId: string; + purpose: GetAdminArtifactTransfer200Purpose; + direction: GetAdminArtifactTransfer200Direction; + state: GetAdminArtifactTransfer200State; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + bytes: number; + sha256: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + cleanup: GetAdminArtifactTransfer200Cleanup; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Cleanup.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Cleanup.ts new file mode 100644 index 000000000..15310651f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Cleanup.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer200Cleanup = + (typeof GetAdminArtifactTransfer200Cleanup)[keyof typeof GetAdminArtifactTransfer200Cleanup]; + +export const GetAdminArtifactTransfer200Cleanup = { + pending: "pending", + complete: "complete", + failed: "failed", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Direction.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Direction.ts new file mode 100644 index 000000000..b322cd025 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Direction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer200Direction = + (typeof GetAdminArtifactTransfer200Direction)[keyof typeof GetAdminArtifactTransfer200Direction]; + +export const GetAdminArtifactTransfer200Direction = { + upload: "upload", + download: "download", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Purpose.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Purpose.ts new file mode 100644 index 000000000..f3903cf7c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200Purpose.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer200Purpose = + (typeof GetAdminArtifactTransfer200Purpose)[keyof typeof GetAdminArtifactTransfer200Purpose]; + +export const GetAdminArtifactTransfer200Purpose = { + "prepared-artifact": "prepared-artifact", + "instance-migration": "instance-migration", + "integration-artifact": "integration-artifact", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200State.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200State.ts new file mode 100644 index 000000000..28c898bde --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer200State.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer200State = + (typeof GetAdminArtifactTransfer200State)[keyof typeof GetAdminArtifactTransfer200State]; + +export const GetAdminArtifactTransfer200State = { + created: "created", + uploading: "uploading", + uploaded: "uploaded", + validating: "validating", + ready: "ready", + claimed: "claimed", + consumed: "consumed", + failed: "failed", + expired: "expired", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer400.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer400.ts new file mode 100644 index 000000000..720aad68c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer401.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer401.ts new file mode 100644 index 000000000..4074fd0ba --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer403.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer403.ts new file mode 100644 index 000000000..405f7b221 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer404.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer404.ts new file mode 100644 index 000000000..647ee72d0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer409.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer409.ts new file mode 100644 index 000000000..6f0c9a5ea --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer410.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer410.ts new file mode 100644 index 000000000..d3f7eccfa --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer413.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer413.ts new file mode 100644 index 000000000..92f3850e0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer415.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer415.ts new file mode 100644 index 000000000..11a08be33 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer422.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer422.ts new file mode 100644 index 000000000..f88cf4019 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer429.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer429.ts new file mode 100644 index 000000000..289bc11cd --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer500.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer500.ts new file mode 100644 index 000000000..247d2b2ed --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer503.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer503.ts new file mode 100644 index 000000000..65f8b593c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer507.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer507.ts new file mode 100644 index 000000000..af81835aa --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransfer507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransfer507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index cb305cd69..d50c14ec0 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -20,6 +20,28 @@ export * from "./checkAdminServices415"; export * from "./checkAdminServices429"; export * from "./checkAdminServices500"; export * from "./checkAdminServicesBody"; +export * from "./createAdminArtifactTransfer200"; +export * from "./createAdminArtifactTransfer200Cleanup"; +export * from "./createAdminArtifactTransfer200Direction"; +export * from "./createAdminArtifactTransfer200Purpose"; +export * from "./createAdminArtifactTransfer200State"; +export * from "./createAdminArtifactTransfer400"; +export * from "./createAdminArtifactTransfer401"; +export * from "./createAdminArtifactTransfer403"; +export * from "./createAdminArtifactTransfer404"; +export * from "./createAdminArtifactTransfer409"; +export * from "./createAdminArtifactTransfer410"; +export * from "./createAdminArtifactTransfer413"; +export * from "./createAdminArtifactTransfer415"; +export * from "./createAdminArtifactTransfer422"; +export * from "./createAdminArtifactTransfer429"; +export * from "./createAdminArtifactTransfer500"; +export * from "./createAdminArtifactTransfer503"; +export * from "./createAdminArtifactTransfer507"; +export * from "./createAdminArtifactTransferBody"; +export * from "./createAdminArtifactTransferBodyDirection"; +export * from "./createAdminArtifactTransferBodyPurpose"; +export * from "./createAdminArtifactTransferHeaders"; export * from "./createAdminServiceBackup200"; export * from "./createAdminServiceBackup400"; export * from "./createAdminServiceBackup401"; @@ -29,6 +51,20 @@ export * from "./createAdminServiceBackup415"; export * from "./createAdminServiceBackup429"; export * from "./createAdminServiceBackup500"; export * from "./createAdminServiceBackupBody"; +export * from "./deleteAdminArtifactTransfer200"; +export * from "./deleteAdminArtifactTransfer400"; +export * from "./deleteAdminArtifactTransfer401"; +export * from "./deleteAdminArtifactTransfer403"; +export * from "./deleteAdminArtifactTransfer404"; +export * from "./deleteAdminArtifactTransfer409"; +export * from "./deleteAdminArtifactTransfer410"; +export * from "./deleteAdminArtifactTransfer413"; +export * from "./deleteAdminArtifactTransfer415"; +export * from "./deleteAdminArtifactTransfer422"; +export * from "./deleteAdminArtifactTransfer429"; +export * from "./deleteAdminArtifactTransfer500"; +export * from "./deleteAdminArtifactTransfer503"; +export * from "./deleteAdminArtifactTransfer507"; export * from "./deleteAdminServiceBackup200"; export * from "./deleteAdminServiceBackup400"; export * from "./deleteAdminServiceBackup401"; @@ -51,6 +87,37 @@ export * from "./deleteAdminServiceCredential429"; export * from "./deleteAdminServiceCredential500"; export * from "./deleteAdminServiceCredentialBody"; export * from "./deleteAdminServiceCredentialHeaders"; +export * from "./downloadAdminArtifactTransferContent400"; +export * from "./downloadAdminArtifactTransferContent401"; +export * from "./downloadAdminArtifactTransferContent403"; +export * from "./downloadAdminArtifactTransferContent404"; +export * from "./downloadAdminArtifactTransferContent409"; +export * from "./downloadAdminArtifactTransferContent410"; +export * from "./downloadAdminArtifactTransferContent413"; +export * from "./downloadAdminArtifactTransferContent415"; +export * from "./downloadAdminArtifactTransferContent422"; +export * from "./downloadAdminArtifactTransferContent429"; +export * from "./downloadAdminArtifactTransferContent500"; +export * from "./downloadAdminArtifactTransferContent503"; +export * from "./downloadAdminArtifactTransferContent507"; +export * from "./getAdminArtifactTransfer200"; +export * from "./getAdminArtifactTransfer200Cleanup"; +export * from "./getAdminArtifactTransfer200Direction"; +export * from "./getAdminArtifactTransfer200Purpose"; +export * from "./getAdminArtifactTransfer200State"; +export * from "./getAdminArtifactTransfer400"; +export * from "./getAdminArtifactTransfer401"; +export * from "./getAdminArtifactTransfer403"; +export * from "./getAdminArtifactTransfer404"; +export * from "./getAdminArtifactTransfer409"; +export * from "./getAdminArtifactTransfer410"; +export * from "./getAdminArtifactTransfer413"; +export * from "./getAdminArtifactTransfer415"; +export * from "./getAdminArtifactTransfer422"; +export * from "./getAdminArtifactTransfer429"; +export * from "./getAdminArtifactTransfer500"; +export * from "./getAdminArtifactTransfer503"; +export * from "./getAdminArtifactTransfer507"; export * from "./getAdminComposePreview400"; export * from "./getAdminComposePreview401"; export * from "./getAdminComposePreview403"; @@ -255,6 +322,26 @@ export * from "./getAdminSharedMobility413"; export * from "./getAdminSharedMobility415"; export * from "./getAdminSharedMobility429"; export * from "./getAdminSharedMobility500"; +export * from "./listAdminArtifactTransfers200"; +export * from "./listAdminArtifactTransfers200ItemsItem"; +export * from "./listAdminArtifactTransfers200ItemsItemCleanup"; +export * from "./listAdminArtifactTransfers200ItemsItemDirection"; +export * from "./listAdminArtifactTransfers200ItemsItemPurpose"; +export * from "./listAdminArtifactTransfers200ItemsItemState"; +export * from "./listAdminArtifactTransfers400"; +export * from "./listAdminArtifactTransfers401"; +export * from "./listAdminArtifactTransfers403"; +export * from "./listAdminArtifactTransfers404"; +export * from "./listAdminArtifactTransfers409"; +export * from "./listAdminArtifactTransfers410"; +export * from "./listAdminArtifactTransfers413"; +export * from "./listAdminArtifactTransfers415"; +export * from "./listAdminArtifactTransfers422"; +export * from "./listAdminArtifactTransfers429"; +export * from "./listAdminArtifactTransfers500"; +export * from "./listAdminArtifactTransfers503"; +export * from "./listAdminArtifactTransfers507"; +export * from "./listAdminArtifactTransfersParams"; export * from "./listAdminOperations200"; export * from "./listAdminOperations200OperationsItem"; export * from "./listAdminOperations200OperationsItemConfirmation"; @@ -518,3 +605,21 @@ export * from "./updateAdminServiceConfig500"; export * from "./updateAdminServiceConfigBody"; export * from "./updateAdminServiceConfigBodyConfig"; export * from "./updateAdminServiceConfigHeaders"; +export * from "./uploadAdminArtifactTransferContent200"; +export * from "./uploadAdminArtifactTransferContent200Cleanup"; +export * from "./uploadAdminArtifactTransferContent200Direction"; +export * from "./uploadAdminArtifactTransferContent200Purpose"; +export * from "./uploadAdminArtifactTransferContent200State"; +export * from "./uploadAdminArtifactTransferContent400"; +export * from "./uploadAdminArtifactTransferContent401"; +export * from "./uploadAdminArtifactTransferContent403"; +export * from "./uploadAdminArtifactTransferContent404"; +export * from "./uploadAdminArtifactTransferContent409"; +export * from "./uploadAdminArtifactTransferContent410"; +export * from "./uploadAdminArtifactTransferContent413"; +export * from "./uploadAdminArtifactTransferContent415"; +export * from "./uploadAdminArtifactTransferContent422"; +export * from "./uploadAdminArtifactTransferContent429"; +export * from "./uploadAdminArtifactTransferContent500"; +export * from "./uploadAdminArtifactTransferContent503"; +export * from "./uploadAdminArtifactTransferContent507"; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200.ts new file mode 100644 index 000000000..4dcf0815a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminArtifactTransfers200ItemsItem } from "./listAdminArtifactTransfers200ItemsItem"; + +export type ListAdminArtifactTransfers200 = { + /** @maxItems 100 */ + items: ListAdminArtifactTransfers200ItemsItem[]; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + total: number; + /** + * @minimum -9007199254740991 + * @maximum 9007199254740991 + */ + limit: number; + /** + * @minimum -9007199254740991 + * @maximum 9007199254740991 + */ + offset: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItem.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItem.ts new file mode 100644 index 000000000..d716819a4 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItem.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminArtifactTransfers200ItemsItemCleanup } from "./listAdminArtifactTransfers200ItemsItemCleanup"; +import type { ListAdminArtifactTransfers200ItemsItemDirection } from "./listAdminArtifactTransfers200ItemsItemDirection"; +import type { ListAdminArtifactTransfers200ItemsItemPurpose } from "./listAdminArtifactTransfers200ItemsItemPurpose"; +import type { ListAdminArtifactTransfers200ItemsItemState } from "./listAdminArtifactTransfers200ItemsItemState"; + +export type ListAdminArtifactTransfers200ItemsItem = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + stageId: string; + purpose: ListAdminArtifactTransfers200ItemsItemPurpose; + direction: ListAdminArtifactTransfers200ItemsItemDirection; + state: ListAdminArtifactTransfers200ItemsItemState; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + bytes: number; + sha256: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + cleanup: ListAdminArtifactTransfers200ItemsItemCleanup; +}; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemCleanup.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemCleanup.ts new file mode 100644 index 000000000..ef628342c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemCleanup.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers200ItemsItemCleanup = + (typeof ListAdminArtifactTransfers200ItemsItemCleanup)[keyof typeof ListAdminArtifactTransfers200ItemsItemCleanup]; + +export const ListAdminArtifactTransfers200ItemsItemCleanup = { + pending: "pending", + complete: "complete", + failed: "failed", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemDirection.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemDirection.ts new file mode 100644 index 000000000..2706e3224 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemDirection.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers200ItemsItemDirection = + (typeof ListAdminArtifactTransfers200ItemsItemDirection)[keyof typeof ListAdminArtifactTransfers200ItemsItemDirection]; + +export const ListAdminArtifactTransfers200ItemsItemDirection = { + upload: "upload", + download: "download", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemPurpose.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemPurpose.ts new file mode 100644 index 000000000..d49107f12 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemPurpose.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers200ItemsItemPurpose = + (typeof ListAdminArtifactTransfers200ItemsItemPurpose)[keyof typeof ListAdminArtifactTransfers200ItemsItemPurpose]; + +export const ListAdminArtifactTransfers200ItemsItemPurpose = { + "prepared-artifact": "prepared-artifact", + "instance-migration": "instance-migration", + "integration-artifact": "integration-artifact", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemState.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemState.ts new file mode 100644 index 000000000..976b6d061 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers200ItemsItemState.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers200ItemsItemState = + (typeof ListAdminArtifactTransfers200ItemsItemState)[keyof typeof ListAdminArtifactTransfers200ItemsItemState]; + +export const ListAdminArtifactTransfers200ItemsItemState = { + created: "created", + uploading: "uploading", + uploaded: "uploaded", + validating: "validating", + ready: "ready", + claimed: "claimed", + consumed: "consumed", + failed: "failed", + expired: "expired", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers400.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers400.ts new file mode 100644 index 000000000..e73949ebf --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers401.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers401.ts new file mode 100644 index 000000000..c799492cc --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers403.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers403.ts new file mode 100644 index 000000000..99495636e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers404.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers404.ts new file mode 100644 index 000000000..fb2e39bad --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers409.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers409.ts new file mode 100644 index 000000000..ded7a4214 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers410.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers410.ts new file mode 100644 index 000000000..befa7e6a9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers413.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers413.ts new file mode 100644 index 000000000..380b3bdc2 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers415.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers415.ts new file mode 100644 index 000000000..ae766a3cd --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers422.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers422.ts new file mode 100644 index 000000000..0f901dd4b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers429.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers429.ts new file mode 100644 index 000000000..a52dbf5bf --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers500.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers500.ts new file mode 100644 index 000000000..c15f337c1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers503.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers503.ts new file mode 100644 index 000000000..cb2ce7f7e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers507.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers507.ts new file mode 100644 index 000000000..1f57be8fe --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfers507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfers507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfersParams.ts b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfersParams.ts new file mode 100644 index 000000000..3c98ee154 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/listAdminArtifactTransfersParams.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminArtifactTransfersParams = { + /** + * @minimum 1 + * @maximum 100 + */ + limit?: number; + /** + * @minimum 0 + * @maximum 1000000 + */ + offset?: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200.ts new file mode 100644 index 000000000..4586a6fa9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200.ts @@ -0,0 +1,23 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { UploadAdminArtifactTransferContent200Cleanup } from "./uploadAdminArtifactTransferContent200Cleanup"; +import type { UploadAdminArtifactTransferContent200Direction } from "./uploadAdminArtifactTransferContent200Direction"; +import type { UploadAdminArtifactTransferContent200Purpose } from "./uploadAdminArtifactTransferContent200Purpose"; +import type { UploadAdminArtifactTransferContent200State } from "./uploadAdminArtifactTransferContent200State"; + +export type UploadAdminArtifactTransferContent200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + stageId: string; + purpose: UploadAdminArtifactTransferContent200Purpose; + direction: UploadAdminArtifactTransferContent200Direction; + state: UploadAdminArtifactTransferContent200State; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + bytes: number; + sha256: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + cleanup: UploadAdminArtifactTransferContent200Cleanup; +}; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Cleanup.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Cleanup.ts new file mode 100644 index 000000000..a476c406b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Cleanup.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent200Cleanup = + (typeof UploadAdminArtifactTransferContent200Cleanup)[keyof typeof UploadAdminArtifactTransferContent200Cleanup]; + +export const UploadAdminArtifactTransferContent200Cleanup = { + pending: "pending", + complete: "complete", + failed: "failed", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Direction.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Direction.ts new file mode 100644 index 000000000..507c444f0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Direction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent200Direction = + (typeof UploadAdminArtifactTransferContent200Direction)[keyof typeof UploadAdminArtifactTransferContent200Direction]; + +export const UploadAdminArtifactTransferContent200Direction = { + upload: "upload", + download: "download", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Purpose.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Purpose.ts new file mode 100644 index 000000000..fe31b1810 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200Purpose.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent200Purpose = + (typeof UploadAdminArtifactTransferContent200Purpose)[keyof typeof UploadAdminArtifactTransferContent200Purpose]; + +export const UploadAdminArtifactTransferContent200Purpose = { + "prepared-artifact": "prepared-artifact", + "instance-migration": "instance-migration", + "integration-artifact": "integration-artifact", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200State.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200State.ts new file mode 100644 index 000000000..10f280a1c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent200State.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent200State = + (typeof UploadAdminArtifactTransferContent200State)[keyof typeof UploadAdminArtifactTransferContent200State]; + +export const UploadAdminArtifactTransferContent200State = { + created: "created", + uploading: "uploading", + uploaded: "uploaded", + validating: "validating", + ready: "ready", + claimed: "claimed", + consumed: "consumed", + failed: "failed", + expired: "expired", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent400.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent400.ts new file mode 100644 index 000000000..1ff57446b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent401.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent401.ts new file mode 100644 index 000000000..606eab37f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent403.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent403.ts new file mode 100644 index 000000000..9a6e232ff --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent404.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent404.ts new file mode 100644 index 000000000..649e16330 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent409.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent409.ts new file mode 100644 index 000000000..2e91ccffb --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent410.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent410.ts new file mode 100644 index 000000000..bcecd9f7b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent413.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent413.ts new file mode 100644 index 000000000..3ef198265 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent415.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent415.ts new file mode 100644 index 000000000..81427e382 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent422.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent422.ts new file mode 100644 index 000000000..7a3a8055f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent429.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent429.ts new file mode 100644 index 000000000..672ebb35f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent500.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent500.ts new file mode 100644 index 000000000..fb1775dc2 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent503.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent503.ts new file mode 100644 index 000000000..04e87a77a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent507.ts b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent507.ts new file mode 100644 index 000000000..6aa9198c8 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/uploadAdminArtifactTransferContent507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type UploadAdminArtifactTransferContent507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts index 6f3ff5771..93e2d21c1 100644 --- a/packages/core/src/api/index.ts +++ b/packages/core/src/api/index.ts @@ -1,7 +1,9 @@ export { + type AdminNativeDownloadTarget, ApiClient, type ApiClientConfig, ApiClientError, + ApiNativeDownloadUnavailableError, ApiRequestAbortedError, apiClient, apiUrl, diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index 398185350..e70dfb6d9 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -165,6 +165,7 @@ export { reserveFeedProxyValidation, verifyFeedProxyCanonicalTerminal, } from "./feed-proxy-validation"; +export { AnchoredDirectory } from "./filesystem"; export { assertFixedConfigurationMaintenancePlan, buildFixedConfigurationMaintenancePlan, diff --git a/packages/core/src/ops/artifact-transfer-capability.ts b/packages/core/src/ops/artifact-transfer-capability.ts new file mode 100644 index 000000000..f01a31905 --- /dev/null +++ b/packages/core/src/ops/artifact-transfer-capability.ts @@ -0,0 +1,66 @@ +/** Server-only API↔agent bearer proof. Never send this capability to a browser. */ +import { createHmac, hkdfSync, timingSafeEqual } from "node:crypto"; +import z from "zod/v4"; +import { artifactStageClaimSchema, transferPurposeSchema } from "./artifact-transfer"; +import { canonicalJson } from "./canonical-json"; +export const artifactStreamCapabilitySchema = z.strictObject({ + version: z.literal(1), + stageId: z.uuid(), + ownerId: z.string().min(1).max(128), + purpose: transferPurposeSchema, + direction: z.enum(["upload", "download"]), + byteCap: z.number().int().safe().nonnegative(), + claim: artifactStageClaimSchema.optional(), + attemptId: z.uuid(), + issuedAt: z.number().int().safe(), + expiresAt: z.number().int().safe(), + deadlineAt: z.number().int().safe(), +}); +export type ArtifactStreamCapability = z.infer; +function key(token: string) { + return Buffer.from( + hkdfSync("sha256", token, "openmapx-artifact-transfer-v1", "stream-capability", 32), + ); +} +export function issueArtifactStreamCapability( + token: string, + input: ArtifactStreamCapability, +): string { + const value = artifactStreamCapabilitySchema.parse(input); + if ( + value.expiresAt <= value.issuedAt || + value.expiresAt - value.issuedAt > 60000 || + value.deadlineAt <= value.issuedAt || + value.deadlineAt - value.issuedAt > 7 * 86400_000 + ) + throw Error("Invalid artifact stream lifetime"); + const body = Buffer.from(canonicalJson(value)).toString("base64url"); + return `atcap1.${body}.${createHmac("sha256", key(token)).update(`artifact-transfer-v1.${body}`).digest("base64url")}`; +} +export function verifyArtifactStreamCapability( + token: string, + input: string, + now = Date.now(), +): ArtifactStreamCapability { + if (input.length > 8192) throw Error("Invalid artifact stream capability"); + const match = /^atcap1\.([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]{43})$/.exec(input); + if (!match) throw Error("Invalid artifact stream capability"); + const actual = createHmac("sha256", key(token)) + .update(`artifact-transfer-v1.${match[1]}`) + .digest(); + const supplied = Buffer.from(match[2], "base64url"); + if (supplied.length !== actual.length || !timingSafeEqual(supplied, actual)) + throw Error("Invalid artifact stream capability"); + const value = artifactStreamCapabilitySchema.parse( + JSON.parse(Buffer.from(match[1], "base64url").toString("utf8")), + ); + if ( + now < value.issuedAt - 5000 || + now >= value.expiresAt || + value.expiresAt - value.issuedAt > 60000 || + value.deadlineAt <= now || + value.deadlineAt - value.issuedAt > 7 * 86400_000 + ) + throw Error("Expired artifact stream capability"); + return value; +} diff --git a/packages/core/src/ops/artifact-transfer-client.test.ts b/packages/core/src/ops/artifact-transfer-client.test.ts new file mode 100644 index 000000000..567ce760f --- /dev/null +++ b/packages/core/src/ops/artifact-transfer-client.test.ts @@ -0,0 +1,164 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { TransferStage } from "./artifact-transfer"; +import { + issueArtifactStreamCapability, + verifyArtifactStreamCapability, +} from "./artifact-transfer-capability"; +import { ArtifactStageTransportError, withArtifactStream } from "./artifact-transfer-client"; + +const stage: TransferStage = { + stageId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + purpose: "prepared-artifact", + direction: "download", + state: "ready", + bytes: 2, + sha256: "a".repeat(64), + expiresAt: "2099-01-01T00:00:00.000Z", + cleanup: "pending", +}; +afterEach(() => vi.useRealTimers()); +it.each(["abort", "deadline"])("raw ops download owns sink cleanup through %s", async (mode) => { + vi.useFakeTimers(); + let entered = false, + cleaned = false, + canceled = false, + settled = false; + let release = () => {}; + const cleanup = new Promise((resolve) => { + release = resolve; + }); + const abort = new AbortController(); + const source = new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array([0, 255])); + }, + cancel() { + canceled = true; + }, + }); + const pending = withArtifactStream( + { + baseUrl: "http://agent", + token: "a".repeat(43), + fetch: vi.fn().mockResolvedValue( + new Response(source, { + headers: { + "content-type": "application/octet-stream", + "content-length": "2", + "x-artifact-sha256": stage.sha256!, + }, + }), + ), + now: Date.now(), + stage, + input: { + stageId: stage.stageId, + ownerId: "admin", + purpose: stage.purpose, + signal: abort.signal, + executionDeadlineAt: new Date(Date.now() + 20).toISOString(), + }, + }, + async ({ body, signal }) => { + const reader = body.getReader(); + try { + await reader.read(); + entered = true; + await new Promise((resolve) => + signal.addEventListener( + "abort", + () => { + void cleanup.then(() => { + cleaned = true; + resolve(); + }); + }, + { once: true }, + ), + ); + } finally { + await reader.cancel(); + reader.releaseLock(); + } + }, + ) + .catch((error) => error) + .finally(() => { + settled = true; + }); + await vi.advanceTimersByTimeAsync(0); + expect(entered).toBe(true); + if (mode === "abort") abort.abort(); + else await vi.advanceTimersByTimeAsync(21); + await vi.advanceTimersByTimeAsync(0); + expect(settled).toBe(false); + release(); + await vi.advanceTimersByTimeAsync(0); + expect(await pending).toBeInstanceOf(Error); + expect(cleaned).toBe(true); + expect(canceled).toBe(true); + expect(source.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); +}); +it("refuses early callback completion and bounds error JSON", async () => { + const input = { + stageId: stage.stageId, + ownerId: "admin", + purpose: stage.purpose, + executionDeadlineAt: new Date(Date.now() + 10000).toISOString(), + }; + const options = { baseUrl: "http://agent", token: "a".repeat(43), now: Date.now(), stage, input }; + await expect( + withArtifactStream( + { + ...options, + fetch: vi.fn().mockResolvedValue( + new Response(new Uint8Array([0, 255]), { + headers: { + "content-type": "application/octet-stream", + "content-length": "2", + "x-artifact-sha256": stage.sha256!, + }, + }), + ), + }, + async () => {}, + ), + ).rejects.toMatchObject({ status: 409 }); + await expect( + withArtifactStream( + { + ...options, + fetch: vi.fn().mockResolvedValue( + new Response("x".repeat(65537), { + status: 409, + headers: { "content-type": "application/json" }, + }), + ), + }, + async () => {}, + ), + ).rejects.toBeInstanceOf(ArtifactStageTransportError); +}); +it("domain-separates the short-lived capability and rejects key substitution, expiry and alteration", () => { + const now = Date.now(); + const input = { + version: 1 as const, + stageId: stage.stageId, + ownerId: "admin", + purpose: stage.purpose, + direction: "download" as const, + byteCap: 2, + attemptId: "81a78b3e-208b-4ba4-9ddf-870fbdf1da22", + issuedAt: now, + expiresAt: now + 60000, + deadlineAt: now + 100000, + }; + const cap = issueArtifactStreamCapability("token", input); + expect(verifyArtifactStreamCapability("token", cap, now)).toEqual(input); + expect(() => verifyArtifactStreamCapability("other", cap, now)).toThrow(); + expect(() => verifyArtifactStreamCapability("token", cap, now + 60001)).toThrow(); + expect(() => + verifyArtifactStreamCapability("token", cap.replace("atcap1.", "pbcap1."), now), + ).toThrow(); +}); diff --git a/packages/core/src/ops/artifact-transfer-client.ts b/packages/core/src/ops/artifact-transfer-client.ts new file mode 100644 index 000000000..e9441f133 --- /dev/null +++ b/packages/core/src/ops/artifact-transfer-client.ts @@ -0,0 +1,220 @@ +import { randomUUID } from "node:crypto"; +import { + type ArtifactStageClaim, + type TransferPurpose, + type TransferStage, + transferStageSchema, +} from "./artifact-transfer"; +import { issueArtifactStreamCapability } from "./artifact-transfer-capability"; +export type ArtifactStageStreamInput = { + stageId: string; + ownerId: string; + purpose: TransferPurpose; + signal?: AbortSignal; + executionDeadlineAt: string; +}; +export class ArtifactStageTransportError extends Error { + constructor(readonly status: number) { + super("Artifact transfer failed; inspect the retained stage before retrying"); + this.name = "ArtifactStageTransportError"; + } +} +async function boundedJson(response: Response): Promise { + if (response.headers.get("Content-Type")?.split(";")[0] !== "application/json") + throw new ArtifactStageTransportError(503); + const reader = response.body?.getReader(); + if (!reader) throw new ArtifactStageTransportError(503); + let text = "", + bytes = 0; + const decoder = new TextDecoder(); + try { + while (true) { + const part = await reader.read(); + if (part.done) break; + bytes += part.value.byteLength; + if (bytes > 65536) throw new ArtifactStageTransportError(503); + text += decoder.decode(part.value, { stream: true }); + } + return JSON.parse(text + decoder.decode()); + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } +} +export async function withArtifactStream( + options: { + baseUrl: string; + token: string; + fetch: typeof globalThis.fetch; + now: number; + parentSignal?: AbortSignal; + stage: TransferStage; + input: ArtifactStageStreamInput & { claim?: ArtifactStageClaim }; + body?: ReadableStream; + contentLength?: number; + }, + consume?: (input: { + body: ReadableStream; + signal: AbortSignal; + headers: Headers; + }) => Promise, +): Promise { + const { stage, input } = options; + const requested = Date.parse(input.executionDeadlineAt); + const deadlineAt = Math.min(requested, options.now + 72 * 3600_000, Date.parse(stage.expiresAt)); + if (!Number.isSafeInteger(deadlineAt) || deadlineAt <= options.now) + throw new ArtifactStageTransportError(410); + const controller = new AbortController(); + const signals = [input.signal, options.parentSignal].filter((s): s is AbortSignal => !!s); + const abort = () => controller.abort(signals.find((s) => s.aborted)?.reason); + for (const signal of signals) { + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + } + const timer = setTimeout(() => controller.abort(), deadlineAt - options.now); + timer.unref?.(); + let idle: ReturnType | undefined; + const touch = () => { + clearTimeout(idle); + idle = setTimeout(() => controller.abort(), 5 * 60_000); + idle.unref?.(); + }; + touch(); + let response: Response | undefined; + const readers: Array> = []; + let downloadEof = false; + const ownBody = ( + body: ReadableStream, + expected: number | undefined, + isDownload = false, + ) => { + const reader = body.getReader(); + readers.push(reader); + let count = 0; + return new ReadableStream( + { + pull: async (destination) => { + try { + controller.signal.throwIfAborted(); + const part = await reader.read(); + controller.signal.throwIfAborted(); + if (part.done) { + if (expected !== undefined && count !== expected) + throw new ArtifactStageTransportError(422); + if (isDownload) downloadEof = true; + destination.close(); + return; + } + count += part.value.byteLength; + if ( + part.value.byteLength > 8 * 1024 ** 2 || + !Number.isSafeInteger(count) || + (expected !== undefined && count > expected) + ) + throw new ArtifactStageTransportError(413); + touch(); + destination.enqueue(part.value); + } catch (error) { + destination.error(error); + controller.abort(error); + } + }, + cancel: (reason) => reader.cancel(reason), + }, + { highWaterMark: 0 }, + ); + }; + const stopReaders = () => { + for (const reader of readers) void reader.cancel(controller.signal.reason).catch(() => {}); + }; + controller.signal.addEventListener("abort", stopReaders, { once: true }); + try { + const cap = issueArtifactStreamCapability(options.token, { + version: 1, + stageId: stage.stageId, + ownerId: input.ownerId, + purpose: input.purpose, + direction: options.body ? "upload" : "download", + byteCap: options.body + ? (options.contentLength ?? + (input.purpose === "integration-artifact" + ? 200 * 1024 ** 2 + : input.purpose === "instance-migration" + ? 64 * 1024 ** 2 + : 1024 ** 4)) + : stage.bytes, + ...(input.claim ? { claim: input.claim } : {}), + attemptId: randomUUID(), + issuedAt: options.now, + expiresAt: options.now + 60000, + deadlineAt, + }); + controller.signal.throwIfAborted(); + response = await options.fetch( + `${options.baseUrl}/v1/artifact-transfers/${stage.stageId}/content`, + { + method: options.body ? "PUT" : "GET", + headers: { + Authorization: `Bearer ${options.token}`, + "x-openmapx-artifact-capability": cap, + ...(options.body + ? { + "Content-Type": "application/octet-stream", + ...(options.contentLength === undefined + ? {} + : { "Content-Length": String(options.contentLength) }), + } + : {}), + }, + body: options.body ? ownBody(options.body, options.contentLength) : undefined, + signal: controller.signal, + redirect: "error", + ...(options.body ? { duplex: "half" } : {}), + } as RequestInit, + ); + if (!response.ok) { + await boundedJson(response); + throw new ArtifactStageTransportError(response.status); + } + if (options.body) { + const result = transferStageSchema.parse(await boundedJson(response)); + if ( + result.stageId !== stage.stageId || + result.purpose !== stage.purpose || + result.state !== "uploaded" + ) + throw new ArtifactStageTransportError(503); + return result; + } + const headers = response.headers; + if ( + headers.get("Content-Type")?.split(";")[0] !== "application/octet-stream" || + headers.get("Content-Length") !== String(stage.bytes) || + headers.get("X-Artifact-Sha256") !== stage.sha256 || + !response.body || + !consume + ) + throw new ArtifactStageTransportError(503); + const result = await consume({ + body: ownBody(response.body, stage.bytes, true), + headers, + signal: controller.signal, + }); + controller.signal.throwIfAborted(); + if (!downloadEof) throw new ArtifactStageTransportError(409); + return result; + } finally { + try { + for (const reader of readers) { + await reader.cancel(controller.signal.reason).catch(() => {}); + reader.releaseLock(); + } + if (response?.body && !response.body.locked) await response.body.cancel().catch(() => {}); + } finally { + clearTimeout(timer); + clearTimeout(idle); + controller.signal.removeEventListener("abort", stopReaders); + for (const signal of signals) signal.removeEventListener("abort", abort); + } + } +} diff --git a/packages/core/src/ops/artifact-transfer.test.ts b/packages/core/src/ops/artifact-transfer.test.ts new file mode 100644 index 000000000..c530acd87 --- /dev/null +++ b/packages/core/src/ops/artifact-transfer.test.ts @@ -0,0 +1,7 @@ +import { expect, it } from "vitest"; +import { OPS_OPERATION_KINDS } from "./contract"; + +it("installs only the finite API stage operations, including owner-scoped listing", () => { + for (const action of ["create", "list", "inspect", "delete", "claim", "inspectClaim", "settle"]) + expect(OPS_OPERATION_KINDS).toContain(`artifactTransfer.${action}`); +}); diff --git a/packages/core/src/ops/artifact-transfer.ts b/packages/core/src/ops/artifact-transfer.ts new file mode 100644 index 000000000..658cef5e7 --- /dev/null +++ b/packages/core/src/ops/artifact-transfer.ts @@ -0,0 +1,150 @@ +/** Portable metadata only. No paths, credentials or file bodies cross the ops envelope. */ +import z from "zod/v4"; + +const count = z.number().int().nonnegative().safe(); +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const owner = z.string().min(1).max(128); +const key = z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/); +export const transferPurposeSchema = z.enum([ + "prepared-artifact", + "instance-migration", + "integration-artifact", +]); +export const transferDirectionSchema = z.enum(["upload", "download"]); +export const transferStageSchema = z.strictObject({ + stageId: z.uuid(), + purpose: transferPurposeSchema, + direction: transferDirectionSchema, + state: z.enum([ + "created", + "uploading", + "uploaded", + "validating", + "ready", + "claimed", + "consumed", + "failed", + "expired", + ]), + bytes: count, + sha256: digest.nullable(), + expiresAt: z.iso.datetime(), + cleanup: z.enum(["pending", "complete", "failed"]), +}); +export const stageClaimUseSchema = z.enum([ + "inspect-integration", + "install-integration", + "validate-build", + "import-build", + "read-migration", +]); +export const stageClaimIdentitySchema = z.strictObject({ + executionId: z.string().regex(/^job1_[A-Za-z0-9_-]{16,64}$/), + operationKey: key, + fingerprint: digest, + executionDeadlineAt: z.iso.datetime(), +}); +export const stageInspectionBindingSchema = z.strictObject({ + inspectionId: z.uuid(), + stageId: z.uuid(), + sha256: digest, + bytes: count, + policyRevision: digest, + validationFingerprint: digest, + expiresAt: z.iso.datetime(), +}); +export const artifactStageClaimSchema = z.strictObject({ + claimId: z.uuid(), + stageId: z.uuid(), + ownerId: owner, + purpose: transferPurposeSchema, + use: stageClaimUseSchema, + identity: stageClaimIdentitySchema, + sha256: digest, + bytes: count, +}); +export const stageClaimSettlementSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("validated"), inspection: stageInspectionBindingSchema }), + z.strictObject({ kind: z.literal("released"), evidence: z.enum(["no_effect", "restored"]) }), + z.strictObject({ kind: z.literal("consumed"), receiptId: z.string().min(1).max(256) }), + z.strictObject({ kind: z.literal("failed"), receiptId: z.string().min(1).max(256) }), + z.strictObject({ kind: z.literal("unknown"), receiptId: z.string().min(1).max(256) }), +]); +export const createTransferStageSchema = z.strictObject({ + purpose: transferPurposeSchema, + direction: transferDirectionSchema, + ownerId: owner, + declaredBytes: count.optional(), + expectedSha256: digest.optional(), + operationKey: key, +}); +export const inspectTransferStageSchema = z.strictObject({ + stageId: z.uuid(), + ownerId: owner, + purpose: transferPurposeSchema, +}); +export const claimTransferStageSchema = inspectTransferStageSchema.extend({ + use: stageClaimUseSchema, + identity: stageClaimIdentitySchema, + expectedSha256: digest, + inspection: stageInspectionBindingSchema.optional(), +}); +export const artifactTransferOperationSchemas = [ + createTransferStageSchema.extend({ kind: z.literal("artifactTransfer.create") }), + z.strictObject({ + kind: z.literal("artifactTransfer.list"), + ownerId: owner, + limit: z.number().int().min(1).max(100).default(50), + offset: count.default(0), + }), + inspectTransferStageSchema.extend({ kind: z.literal("artifactTransfer.inspect") }), + z.strictObject({ kind: z.literal("artifactTransfer.delete"), stageId: z.uuid(), ownerId: owner }), + claimTransferStageSchema.extend({ kind: z.literal("artifactTransfer.claim") }), + z.strictObject({ + kind: z.literal("artifactTransfer.inspectClaim"), + stageId: z.uuid(), + ownerId: owner, + identity: stageClaimIdentitySchema, + }), + z.strictObject({ + kind: z.literal("artifactTransfer.settle"), + claim: artifactStageClaimSchema, + settlement: stageClaimSettlementSchema, + }), +] as const; +export const artifactTransferResultSchemas = { + "artifactTransfer.create": transferStageSchema, + "artifactTransfer.list": z.strictObject({ + items: z.array(transferStageSchema).max(100), + total: count, + limit: count, + offset: count, + }), + "artifactTransfer.inspect": transferStageSchema, + "artifactTransfer.delete": z.strictObject({ deleted: z.literal(true) }), + "artifactTransfer.claim": artifactStageClaimSchema, + "artifactTransfer.inspectClaim": z + .strictObject({ + claim: artifactStageClaimSchema, + settlement: stageClaimSettlementSchema.nullable(), + }) + .nullable(), + "artifactTransfer.settle": transferStageSchema, +} as const; +export type TransferPurpose = z.infer; +export type TransferDirection = z.infer; +export type TransferStage = z.infer; +export type StageClaimUse = z.infer; +export type StageClaimIdentity = z.infer; +export type StageInspectionBinding = z.infer; +export type ArtifactStageClaim = z.infer; +export type StageClaimSettlement = z.infer; +export type CreateTransferStage = z.infer; +export type ClaimTransferStage = z.infer; +export function stagePurposeAllowsUse(purpose: TransferPurpose, use: StageClaimUse): boolean { + return purpose === "integration-artifact" + ? ["inspect-integration", "install-integration"].includes(use) + : purpose === "prepared-artifact" + ? ["validate-build", "import-build"].includes(use) + : use === "read-migration"; +} diff --git a/packages/core/src/ops/client.ts b/packages/core/src/ops/client.ts index e3e4a8ecd..55b886187 100644 --- a/packages/core/src/ops/client.ts +++ b/packages/core/src/ops/client.ts @@ -1,6 +1,8 @@ import { randomBytes as nodeRandomBytes } from "node:crypto"; import { constants } from "node:fs"; import { open } from "node:fs/promises"; +import type { ArtifactStageClaim, TransferStage } from "./artifact-transfer"; +import { type ArtifactStageStreamInput, withArtifactStream } from "./artifact-transfer-client"; import { OPS_KIND_POLICIES, OPS_MAX_HTTP_RESPONSE_BYTES, @@ -250,6 +252,64 @@ export class OpsClient { this.randomBytes = options.randomBytes ?? nodeRandomBytes; } + async uploadArtifactStage( + input: ArtifactStageStreamInput & { body: ReadableStream; contentLength?: number }, + ): Promise { + const observation = await this.execute( + { + kind: "artifactTransfer.inspect", + stageId: input.stageId, + ownerId: input.ownerId, + purpose: input.purpose, + }, + { signal: input.signal }, + ); + if (!("execution" in observation) || observation.execution !== "sync") + throw new OpsClientError("runtime"); + return (await withArtifactStream({ + baseUrl: this.baseUrl, + token: await readOpsTokenFile(this.tokenFile), + fetch: this.fetchImpl, + now: this.now().getTime(), + parentSignal: this.signal, + stage: observation.value, + input, + body: input.body, + contentLength: input.contentLength, + })) as TransferStage; + } + async consumeArtifactStage( + input: ArtifactStageStreamInput & { claim?: ArtifactStageClaim }, + consume: (input: { + body: ReadableStream; + signal: AbortSignal; + headers: Headers; + }) => Promise, + ): Promise { + const observation = await this.execute( + { + kind: "artifactTransfer.inspect", + stageId: input.stageId, + ownerId: input.ownerId, + purpose: input.purpose, + }, + { signal: input.signal }, + ); + if (!("execution" in observation) || observation.execution !== "sync") + throw new OpsClientError("runtime"); + return (await withArtifactStream( + { + baseUrl: this.baseUrl, + token: await readOpsTokenFile(this.tokenFile), + fetch: this.fetchImpl, + now: this.now().getTime(), + parentSignal: this.signal, + stage: observation.value, + input, + }, + consume, + )) as T; + } createOperationKey(): string { return `opk1_${Buffer.from(this.randomBytes(18)).toString("base64url")}`; } diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index b78642b7e..23ab22f0b 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -49,7 +49,68 @@ function request(operation: unknown, extra: Record = {}) { }; } +const transferIdentity = { + executionId: "job1_abcdefghijklmnop", + operationKey: "opk1_abcdefghijklmnop", + fingerprint: "a".repeat(64), + executionDeadlineAt: "2026-10-29T00:00:00.000Z", +}; +const transferClaim = { + claimId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + stageId: "81a78b3e-208b-4ba4-9ddf-870fbdf1da22", + ownerId: "admin", + purpose: "prepared-artifact" as const, + use: "validate-build" as const, + identity: transferIdentity, + sha256: "a".repeat(64), + bytes: 4, +}; const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { + "artifactTransfer.create": { + kind: "artifactTransfer.create", + ownerId: "admin", + purpose: "prepared-artifact", + direction: "upload", + operationKey: "opk1_abcdefghijklmnop", + declaredBytes: 4, + }, + "artifactTransfer.list": { + kind: "artifactTransfer.list", + ownerId: "admin", + limit: 50, + offset: 0, + }, + "artifactTransfer.inspect": { + kind: "artifactTransfer.inspect", + stageId: transferClaim.stageId, + ownerId: "admin", + purpose: "prepared-artifact", + }, + "artifactTransfer.delete": { + kind: "artifactTransfer.delete", + stageId: transferClaim.stageId, + ownerId: "admin", + }, + "artifactTransfer.claim": { + kind: "artifactTransfer.claim", + stageId: transferClaim.stageId, + ownerId: "admin", + purpose: "prepared-artifact", + use: "validate-build", + identity: transferIdentity, + expectedSha256: "a".repeat(64), + }, + "artifactTransfer.inspectClaim": { + kind: "artifactTransfer.inspectClaim", + stageId: transferClaim.stageId, + ownerId: "admin", + identity: transferIdentity, + }, + "artifactTransfer.settle": { + kind: "artifactTransfer.settle", + claim: transferClaim, + settlement: { kind: "unknown", receiptId: "retained" }, + }, "controlRetention.acknowledge": { kind: "controlRetention.acknowledge", originRootKey: "opk1_abcdefghijklmnop", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index b330524d0..403b750ee 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -6,6 +6,10 @@ import { import { catalogRuntimeInputsSchema } from "../catalog-pins/review-schemas"; import { catalogPinSetSchema } from "../catalog-pins/schemas"; import { platformConfigurationSchema } from "../services/platform-configuration"; +import { + artifactTransferOperationSchemas, + artifactTransferResultSchemas, +} from "./artifact-transfer"; import { configurationGenerationRevisionSchema, configurationHardlinksApplyOperationSchema, @@ -61,6 +65,13 @@ export const OPS_MAX_BACKUP_ID_LENGTH = 128; export const OPS_MAX_BACKUP_INVENTORY_ENTRIES = 128; export const OPS_OPERATION_KINDS = [ + "artifactTransfer.create", + "artifactTransfer.list", + "artifactTransfer.inspect", + "artifactTransfer.delete", + "artifactTransfer.claim", + "artifactTransfer.inspectClaim", + "artifactTransfer.settle", "controlRetention.acknowledge", "docker.status", "stack.status", @@ -192,6 +203,14 @@ const asyncEffect = ( const LIFECYCLE_TIMEOUT_MS = 10 * 60_000 + 30_000; export const OPS_KIND_POLICIES = { + "artifactTransfer.create": sync("api", 30_000), + "artifactTransfer.list": sync("api", 30_000), + "artifactTransfer.inspect": sync("api", 30_000), + "artifactTransfer.delete": sync("api", 30_000), + "artifactTransfer.claim": sync("api", 30_000), + "artifactTransfer.inspectClaim": sync("api", 30_000), + "artifactTransfer.settle": sync("api", 30_000), + "controlRetention.acknowledge": sync("api", 15_000), "docker.status": sync("api", 5_000), "stack.status": sync("api", 15_000, 256 * 1024), @@ -361,6 +380,7 @@ const serviceOperation = (kind: T) => z.strictObject({ kind: z.literal(kind), serviceId: serviceIdSchema }); export const opsOperationSchema = z.discriminatedUnion("kind", [ + ...artifactTransferOperationSchemas, z.strictObject({ kind: z.literal("controlRetention.acknowledge"), originRootKey: operationKeySchema, @@ -1007,6 +1027,7 @@ const pinnedImageReference = z .regex(/^[a-z0-9][a-z0-9._:/-]*@sha256:[a-f0-9]{64}$/); export const OPS_RESULT_SCHEMAS = { + ...artifactTransferResultSchemas, "controlRetention.acknowledge": z.strictObject({ acknowledged: z.literal(true) }), "docker.status": z.strictObject({ reachable: z.boolean(), diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index 79a2b74a0..635c46836 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -1,4 +1,7 @@ export * from "./api-job-attempt"; +export * from "./artifact-transfer"; +export * from "./artifact-transfer-capability"; +export { ArtifactStageTransportError } from "./artifact-transfer-client"; export * from "./catalog-pin-approval"; export * from "./client"; export * from "./configuration-maintenance"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb7a1da78..fe6060020 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -541,6 +541,9 @@ importers: fastify: specifier: ^5.12.7 version: 5.12.7 + zod: + specifier: ^4.6.5 + version: 4.6.5 devDependencies: '@types/node': specifier: ^26.6.5 diff --git a/scripts/api-sdk/codegen.test.ts b/scripts/api-sdk/codegen.test.ts index 6453d29ce..709801f4e 100644 --- a/scripts/api-sdk/codegen.test.ts +++ b/scripts/api-sdk/codegen.test.ts @@ -73,7 +73,7 @@ function generatedConsumerDiagnostics(files: { filename: string; content: string const source: "default" | "database" | "vault" | "config.json" | "env" = response.data.resolvedConfig.field.source; } }); - setAdminIntegrationCredential("sample", "apiKey", {value: "synthetic"}, {"idempotency-key":"synthetic-intent-0001"}, finiteOptions); + setAdminIntegrationCredential("sample", "apiKey", {value: "synthetic",expectedRevision:"cfg1_synthetic"}, {"idempotency-key":"synthetic-intent-0001"}, finiteOptions); // @ts-expect-error Credential bodies reject unknown fields. setAdminIntegrationCredential("sample", "apiKey", {value: "synthetic", unexpected: true}, {"idempotency-key":"synthetic-intent-0001"}); // @ts-expect-error Credentials are strings, not numbers. @@ -99,7 +99,7 @@ function generatedConsumerDiagnostics(files: { filename: string; content: string virtual.set( filename, `${virtual.get(filename)} - import { streamAdminServiceLogs, getAdminService, getAdminComposePreview, stopAdminCompose, setAdminServiceSelection, runAdminServicesBulkAction, restoreAdminServiceBackup } from "./service-admin.generated"; + import { uploadAdminArtifactTransferContent,downloadAdminArtifactTransferContent,getDownloadAdminArtifactTransferContentTarget, streamAdminServiceLogs, getAdminService, getAdminComposePreview, stopAdminCompose, setAdminServiceSelection, runAdminServicesBulkAction, restoreAdminServiceBackup } from "./service-admin.generated"; getAdminService("sample").then(response => { if (response.status === 200) { const image: string = response.data.manifest.container.image; @@ -107,6 +107,17 @@ function generatedConsumerDiagnostics(files: { filename: string; content: string response.data.manifest.provides?.forEach(entry => { const capability: string = typeof entry === "string" ? entry : entry.capability; }); } }); + uploadAdminArtifactTransferContent("stage",new Blob([new Uint8Array([0,255])]),{client:undefined}); + // @ts-expect-error Uploads accept opaque Blob bodies, not JSON. + uploadAdminArtifactTransferContent("stage",{body:"invalid"}); + downloadAdminArtifactTransferContent("stage",{onByteChunk:async(chunk,signal)=>{const byte:number=chunk[0];signal.throwIfAborted();}}).then(response=>{if(response.status===200){const data:undefined=response.data;}}); + // @ts-expect-error Byte downloads require an explicit owned consumer. + downloadAdminArtifactTransferContent("stage"); + // @ts-expect-error Signal alone cannot consume a byte response. + downloadAdminArtifactTransferContent("stage",{signal:new AbortController().signal}); + const native:Promise<{url:string;ownership:"browser"}>=getDownloadAdminArtifactTransferContentTarget("stage",{headers:{}}); + // @ts-expect-error Native navigation cannot send custom headers. + getDownloadAdminArtifactTransferContentTarget("stage",{headers:{Authorization:"x"}}); const logHeaders = {"idempotency-key": "synthetic-intent-0001"}; streamAdminServiceLogs("id", logHeaders, undefined, {onTextChunk: async (chunk) => { const text: string = chunk; }, timeoutMs: 100}).then(response => { if (response.status === 200) { const data: undefined = response.data; } @@ -853,6 +864,21 @@ describe("service administration custom-client generation", () => { operationId: id, tags: ["service-admin"], parameters, + ...(Object.values(operation.responses).some((c) => c.kind === "bytes" && c.nativeDownload) + ? { "x-openmapx-native-download": true } + : {}), + ...(operation.requestBody + ? { + requestBody: { + required: true, + content: { + [operation.requestBody.mediaType]: { + schema: { type: "string", format: "binary" }, + }, + }, + }, + } + : {}), ...(request?.body ? { requestBody: { @@ -875,7 +901,9 @@ describe("service administration custom-client generation", () => { schema: "schema" in response ? schema(response.schema, `${id}Response${status}`) - : { type: "string" }, + : response.kind === "bytes" + ? { type: "string", format: "binary" } + : { type: "string" }, }, }, }, diff --git a/scripts/api-sdk/config.mjs b/scripts/api-sdk/config.mjs index 9e909c77e..de8e9e533 100644 --- a/scripts/api-sdk/config.mjs +++ b/scripts/api-sdk/config.mjs @@ -127,29 +127,38 @@ export function serviceClient(clients) { return { ...clients.fetch, async client(verb, options, ...rest) { - if ( - !Object.entries(verb.originalOperation.responses ?? {}).some( - ([status, response]) => /^2\d\d$/.test(status) && response.content?.["text/plain"], - ) - ) + const byteResponse = isByteResponse(verb.originalOperation); + const byteRequest = + !!verb.originalOperation.requestBody?.content?.["application/octet-stream"]; + const textResponse = Object.entries(verb.originalOperation.responses ?? {}).some( + ([status, response]) => /^2\d\d$/.test(status) && response.content?.["text/plain"], + ); + if (!byteResponse && !byteRequest && !textResponse) return clients.fetch.client(verb, options, ...rest); + const optionsType = byteResponse + ? "AdminByteResponseRequestOptions" + : byteRequest + ? "AdminBinaryUploadRequestOptions" + : "AdminTextStreamRequestOptions"; const generated = await clients.fetch.client( { ...verb, - response: { - ...verb.response, - definition: { ...verb.response.definition, success: "undefined" }, - types: { - ...verb.response.types, - success: verb.response.types.success.map((response) => ({ - ...response, - value: "undefined", - isRef: false, - imports: [], - schemas: [], - })), - }, - }, + response: byteRequest + ? verb.response + : { + ...verb.response, + definition: { ...verb.response.definition, success: "undefined" }, + types: { + ...verb.response.types, + success: verb.response.types.success.map((response) => ({ + ...response, + value: "undefined", + isRef: false, + imports: [], + schemas: [], + })), + }, + }, }, { ...options, @@ -182,16 +191,37 @@ return { ${name}, ${urlName} }; })(); export const ${urlName} = ${scope}.${urlName}; type ${args} = Parameters; -export const ${name}: (${verb.props.map((prop, index) => `${prop.name}: ${args}[${index}]`).join(", ")}, options: AdminTextStreamRequestOptions) => ReturnType = ${scope}.${name}; +export const ${name}: (${verb.props.map((prop, index) => `${prop.name}: ${args}[${index}]`).join(", ")}, options${byteRequest ? "?" : ""}: ${optionsType}) => ReturnType = ${scope}.${name}; +${verb.originalOperation["x-openmapx-native-download"] ? `export const get${name[0].toUpperCase()}${name.slice(1)}Target = (${verb.props.map((prop, index) => `${prop.name}: ${args}[${index}]`).join(", ")}, options?: AdminNativeDownloadTargetOptions): Promise => bindAdminDownloadTarget(${JSON.stringify(verb.operationName)})(${urlName}(${verb.props.map((prop) => prop.name).join(", ")}), options);` : ""} `, imports: [ ...generated.imports, - { name: "AdminTextStreamRequestOptions", importPath: "../../admin-sdk-transport" }, + { name: optionsType, importPath: "../../admin-sdk-transport" }, + ...(verb.originalOperation["x-openmapx-native-download"] + ? [ + { + name: "bindAdminDownloadTarget", + importPath: "../../admin-sdk-transport", + values: true, + }, + { + name: "AdminNativeDownloadTargetOptions", + importPath: "../../admin-sdk-transport", + }, + { name: "AdminNativeDownloadTarget", importPath: "../../admin-sdk-transport" }, + ] + : []), ], }; }, }; } +function isByteResponse(operation) { + return Object.entries(operation.responses ?? {}).some( + ([status, response]) => + /^2\d\d$/.test(status) && response.content?.["application/octet-stream"], + ); +} function isEventStream(operation) { return Object.entries(operation.responses ?? {}).some( ([status, response]) => /^2\d\d$/.test(status) && response.content?.["text/event-stream"], @@ -209,7 +239,16 @@ export function adminOperationBindings(inputDocument) { const optionsTypes = {}; for (const item of Object.values(combined.paths)) for (const operation of Object.values(item)) { - if (operation?.operationId && isEventStream(operation)) { + if (operation?.operationId && isByteResponse(operation)) { + bindFunctions[operation.operationId] = "bindAdminByteOperation"; + optionsTypes[operation.operationId] = "AdminSdkRequestOptions"; + } else if ( + operation?.operationId && + operation.requestBody?.content?.["application/octet-stream"] + ) { + bindFunctions[operation.operationId] = "bindAdminBinaryUploadOperation"; + optionsTypes[operation.operationId] = "AdminSdkRequestOptions"; + } else if (operation?.operationId && isEventStream(operation)) { bindFunctions[operation.operationId] = "bindAdminStreamOperation"; optionsTypes[operation.operationId] = "AdminEventStreamTargetOptions"; } diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 4d012a9c1..656d28528 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -99,7 +99,14 @@ "system.inspect", "system.update", "transitCredentials.inspect", - "vault.apply" + "vault.apply", + "artifactTransfer.create", + "artifactTransfer.list", + "artifactTransfer.inspect", + "artifactTransfer.delete", + "artifactTransfer.claim", + "artifactTransfer.inspectClaim", + "artifactTransfer.settle" ], "data-manager": [ "feedProxy.validateAndReload", @@ -147,7 +154,10 @@ "overture-sync", "overture-conflate", "search-index-build", - "notable-places-build" + "notable-places-build", + "overture-pull", + "overture-ingest", + "overture-extract" ], "apps/api/src/services/admin-job-handlers.ts#BackupOperation": ["create", "restore", "delete"], "apps/api/src/services/admin-job-handlers.ts#BulkServiceAction": [ @@ -378,6 +388,20 @@ "apps/ops-agent/src/administrative-runtime.ts#createAdministrativeRuntime" ], "typedKinds": ["data.generateApiKeys"] + }, + { + "id": "api.artifact-transfers", + "callSiteIds": [], + "semanticSources": ["apps/api/src/services/artifact-transfers.ts#AdminArtifactTransfers"], + "typedKinds": [ + "artifactTransfer.create", + "artifactTransfer.list", + "artifactTransfer.inspect", + "artifactTransfer.delete", + "artifactTransfer.claim", + "artifactTransfer.inspectClaim", + "artifactTransfer.settle" + ] } ], "mounts": [ diff --git a/services/ops-agent/service.json b/services/ops-agent/service.json index fd74295de..4dbf93b9f 100644 --- a/services/ops-agent/service.json +++ b/services/ops-agent/service.json @@ -38,10 +38,11 @@ "OPS_PRIVACY_BACKUP_COLLECTOR_ENABLED": "${OPS_PRIVACY_BACKUP_COLLECTOR_ENABLED:-false}", "OPS_PRIVACY_BACKUP_COLLECTOR_IMAGE": "${OPS_PRIVACY_BACKUP_COLLECTOR_IMAGE:-}", "OPS_PRIVACY_BACKUP_SCRATCH_ROOT": "/var/lib/openmapx/privacy-extraction", - "BACKUP_RETENTION_DAYS": "${BACKUP_RETENTION_DAYS:-30}" + "BACKUP_RETENTION_DAYS": "${BACKUP_RETENTION_DAYS:-30}", + "OPENMAPX_ARTIFACT_TRANSFER_POLICY_FILE": "/run/openmapx/artifact-transfer-policy.json" }, "capDrop": ["ALL"], - "memory": "256m", + "memory": "1g", "restart": "unless-stopped", "healthcheck": { "type": "http", @@ -85,7 +86,7 @@ "readOnly": false }, { - "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set — ops-agent renders and applies trusted local stack configuration}", + "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set \u2014 ops-agent renders and applies trusted local stack configuration}", "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}", "readOnly": false }, @@ -128,6 +129,12 @@ "source": "@infra:control-private/empty", "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/control-private/host-cli", "readOnly": true + }, + { + "source": "@infra:artifact-transfer-policy.json", + "target": "/run/openmapx/artifact-transfer-policy.json", + "readOnly": true, + "optional": true } ], "ui": { diff --git a/services/traefik/config/traefik.yml b/services/traefik/config/traefik.yml index a8ae4236b..5881ae7e2 100644 --- a/services/traefik/config/traefik.yml +++ b/services/traefik/config/traefik.yml @@ -17,6 +17,13 @@ entryPoints: websecure: address: ":443" + # Static entrypoint limits cover streamed artifact transfers. Ordinary API + # parsers/body limits remain independently bounded in the application. + transport: + respondingTimeouts: + readTimeout: 72h + writeTimeout: 72h + idleTimeout: 5m # HTTP/3 (QUIC) over UDP/443. `advertisedPort` is the port announced to # clients in the Alt-Svc header; it must match the published UDP host port # (see services/traefik/service.json `exposure.hostPorts`). Clients still From 930767bffa7b5c4a8a486c477213377f6ccbba8f Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 10:26:36 +0100 Subject: [PATCH 057/135] fix(admin): align artifact claim settlement contract --- packages/core/src/ops/artifact-transfer.test.ts | 9 +++++++++ packages/core/src/ops/artifact-transfer.ts | 12 ++++++------ packages/core/src/ops/contract.test.ts | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/core/src/ops/artifact-transfer.test.ts b/packages/core/src/ops/artifact-transfer.test.ts index c530acd87..2f2822b93 100644 --- a/packages/core/src/ops/artifact-transfer.test.ts +++ b/packages/core/src/ops/artifact-transfer.test.ts @@ -5,3 +5,12 @@ it("installs only the finite API stage operations, including owner-scoped listin for (const action of ["create", "list", "inspect", "delete", "claim", "inspectClaim", "settle"]) expect(OPS_OPERATION_KINDS).toContain(`artifactTransfer.${action}`); }); +it("uses the shared outcome discriminator for durable claim settlement", async () => { + const { stageClaimSettlementSchema } = await import("./artifact-transfer"); + expect( + stageClaimSettlementSchema.safeParse({ outcome: "unknown", receiptId: "retained" }).success, + ).toBe(true); + expect( + stageClaimSettlementSchema.safeParse({ kind: "unknown", receiptId: "retained" }).success, + ).toBe(false); +}); diff --git a/packages/core/src/ops/artifact-transfer.ts b/packages/core/src/ops/artifact-transfer.ts index 658cef5e7..83a5ba7ea 100644 --- a/packages/core/src/ops/artifact-transfer.ts +++ b/packages/core/src/ops/artifact-transfer.ts @@ -63,12 +63,12 @@ export const artifactStageClaimSchema = z.strictObject({ sha256: digest, bytes: count, }); -export const stageClaimSettlementSchema = z.discriminatedUnion("kind", [ - z.strictObject({ kind: z.literal("validated"), inspection: stageInspectionBindingSchema }), - z.strictObject({ kind: z.literal("released"), evidence: z.enum(["no_effect", "restored"]) }), - z.strictObject({ kind: z.literal("consumed"), receiptId: z.string().min(1).max(256) }), - z.strictObject({ kind: z.literal("failed"), receiptId: z.string().min(1).max(256) }), - z.strictObject({ kind: z.literal("unknown"), receiptId: z.string().min(1).max(256) }), +export const stageClaimSettlementSchema = z.discriminatedUnion("outcome", [ + z.strictObject({ outcome: z.literal("validated"), inspection: stageInspectionBindingSchema }), + z.strictObject({ outcome: z.literal("released"), evidence: z.enum(["no_effect", "restored"]) }), + z.strictObject({ outcome: z.literal("consumed"), receiptId: z.string().min(1).max(256) }), + z.strictObject({ outcome: z.literal("failed"), receiptId: z.string().min(1).max(256) }), + z.strictObject({ outcome: z.literal("unknown"), receiptId: z.string().min(1).max(256) }), ]); export const createTransferStageSchema = z.strictObject({ purpose: transferPurposeSchema, diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 23ab22f0b..35fe9cb1b 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -109,7 +109,7 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { "artifactTransfer.settle": { kind: "artifactTransfer.settle", claim: transferClaim, - settlement: { kind: "unknown", receiptId: "retained" }, + settlement: { outcome: "unknown", receiptId: "retained" }, }, "controlRetention.acknowledge": { kind: "controlRetention.acknowledge", From 1487b5d98adf485dd66aaf0a3fc9cabe30541b44 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 10:34:06 +0100 Subject: [PATCH 058/135] fix(admin): isolate transfer state from configuration queue --- .../src/artifact-transfer/store.test.ts | 33 +++++++++++++++++++ apps/ops-agent/src/artifact-transfer/store.ts | 7 ++++ apps/ops-agent/src/index.ts | 9 +++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/apps/ops-agent/src/artifact-transfer/store.test.ts b/apps/ops-agent/src/artifact-transfer/store.test.ts index 26237b222..d0debfc20 100644 --- a/apps/ops-agent/src/artifact-transfer/store.test.ts +++ b/apps/ops-agent/src/artifact-transfer/store.test.ts @@ -311,3 +311,36 @@ it("closes new admissions during the existing drain without dropping the active expect(await pending).toBeInstanceOf(Error); expect(store.activeStreams).toBe(0); }); +it("keeps production transfer metadata out of the strict configuration startup queue", async () => { + const { root } = await fixture(); + const infraDir = join(root, "infra"); + const queue = join(infraDir, "data", "ops-agent", "trusted-config"); + await mkdir(queue, { recursive: true, mode: 0o700 }); + const { artifactTransferPrivateRoot } = await import("./store"); + const { initializeTerminalClaimCleanup } = await import("../trusted-config-source"); + await initializeTerminalClaimCleanup(queue, { + allowedUids: [process.getuid!()], + expectedGid: process.getgid!(), + token: "t".repeat(43), + journalRecords: [], + }); + const store = new ArtifactTransferStore({ + dataRoot: join(infraDir, "data", ".openmapx-transfers"), + privateRoot: artifactTransferPrivateRoot(infraDir), + token: "t".repeat(43), + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 10000 }), + }); + stores.push(store); + await store.createStage({ ...input(), declaredBytes: 4 }); + await expect( + initializeTerminalClaimCleanup(queue, { + allowedUids: [process.getuid!()], + expectedGid: process.getgid!(), + token: "t".repeat(43), + journalRecords: [], + }), + ).resolves.toBeDefined(); + expect(artifactTransferPrivateRoot(infraDir, true)).toBe( + "/var/lib/openmapx/control-private/artifact-transfers", + ); +}); diff --git a/apps/ops-agent/src/artifact-transfer/store.ts b/apps/ops-agent/src/artifact-transfer/store.ts index bfd3a803c..7676f3a5d 100644 --- a/apps/ops-agent/src/artifact-transfer/store.ts +++ b/apps/ops-agent/src/artifact-transfer/store.ts @@ -685,3 +685,10 @@ export class ArtifactTransferStore { failed(409, "Artifact purpose consumer is not installed"); } } + +/** Fixed agent-private location; never a browser-selected path. */ +export function artifactTransferPrivateRoot(infraDir: string, container = false): string { + return container + ? "/var/lib/openmapx/control-private/artifact-transfers" + : join(infraDir, "control-private", "ops-agent", "artifact-transfers"); +} diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index dc813a0e1..d60beb5bb 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -16,7 +16,7 @@ import { import { isTerminalHandoff, readAgentHandoff } from "./agent-handoff"; import { loadArtifactTransferPolicy } from "./artifact-transfer/policy"; import { installArtifactTransferRuntime } from "./artifact-transfer/runtime"; -import { ArtifactTransferStore } from "./artifact-transfer/store"; +import { ArtifactTransferStore, artifactTransferPrivateRoot } from "./artifact-transfer/store"; import { startOpsAgentServer } from "./bootstrap"; import { CommunityRepositoryStore } from "./community-repositories"; import { loadOpsAgentConfig } from "./config"; @@ -208,10 +208,13 @@ async function main(): Promise { }; const transferPolicy = loadArtifactTransferPolicy(); const artifactTransfers = - "policy" in transferPolicy + "policy" in transferPolicy && controlStartup.installation ? new ArtifactTransferStore({ dataRoot: join(paths.infraDir, "data", ".openmapx-transfers"), - privateRoot: join(config.trustedConfigDirectory, "artifact-transfers"), + privateRoot: artifactTransferPrivateRoot( + paths.infraDir, + existsSync("/run/secrets/openmapx-control-verifiers"), + ), token: apiToken, policy: transferPolicy.policy, admissionsOpen: () => !replacementInProgress(), From d53de19a5c88fff6790b7b6e0a437e7e6330ef96 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 29 Oct 2026 13:00:30 +0100 Subject: [PATCH 059/135] feat(admin): install standalone declarative integrations --- apps/api/openapi.json | 1247 +++ apps/api/src/db/installed-extension-schema.ts | 20 +- .../src/db/installed-integration-schema.ts | 2 +- .../0049_admin_integration_ownership.sql | 6 + .../src/db/migrations/meta/0049_snapshot.json | 9121 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + .../routes/admin-standalone-integrations.ts | 43 + apps/api/src/routes/index.ts | 2 + apps/api/src/server.ts | 6 + .../extension-install-journal.test.ts | 93 +- .../__tests__/extension-installer.test.ts | 35 +- apps/api/src/services/activity-jobs.ts | 5 +- apps/api/src/services/admin-job-checkpoint.ts | 10 +- apps/api/src/services/admin-job-failure.ts | 11 + apps/api/src/services/admin-job-ops.ts | 11 + apps/api/src/services/admin-job-parent.ts | 6 + apps/api/src/services/admin-job-state.ts | 4 +- .../src/services/extension-install-journal.ts | 126 +- apps/api/src/services/extension-installer.ts | 37 +- apps/api/src/services/job-runner.ts | 90 +- .../standalone-integrations/install.test.ts | 413 + .../standalone-integrations/install.ts | 444 + .../settlement-checkpoint.ts | 164 + .../services/standalone-integrations/state.ts | 54 + apps/ops-agent/package.json | 1 + .../integration-consumer.test.ts | 225 + apps/ops-agent/src/artifact-transfer/store.ts | 640 +- apps/ops-agent/src/index.ts | 6 + .../src/standalone-integration-records.ts | 39 + .../standalone-integration-runtime.test.ts | 301 + .../src/standalone-integration-runtime.ts | 220 + .../admin/extensions/ExtensionStorePage.tsx | 5 + .../StandaloneInstallDialog.test.tsx | 222 + .../extensions/StandaloneInstallDialog.tsx | 395 + .../administration/community-extensions.md | 21 + .../cli/__tests__/cli-integrations.test.ts | 23 +- .../api/contracts/integration-admin.test.ts | 28 +- .../api/contracts/integration-admin/index.ts | 2 + .../contracts/integration-admin/operations.ts | 3 + .../standalone-integrations.ts | 51 + .../admin/integration-admin.generated.ts | 248 + .../admin/integration-models/index.ts | 25 + .../installAdminStandaloneIntegration202.ts | 9 + .../installAdminStandaloneIntegration400.ts | 20 + .../installAdminStandaloneIntegration401.ts | 20 + .../installAdminStandaloneIntegration403.ts | 20 + .../installAdminStandaloneIntegration409.ts | 20 + .../installAdminStandaloneIntegration413.ts | 20 + .../installAdminStandaloneIntegration415.ts | 20 + .../installAdminStandaloneIntegration429.ts | 11 + .../installAdminStandaloneIntegration500.ts | 20 + .../installAdminStandaloneIntegration503.ts | 20 + .../installAdminStandaloneIntegrationBody.ts | 9 + ...nstallAdminStandaloneIntegrationHeaders.ts | 9 + .../previewAdminStandaloneIntegration202.ts | 9 + .../previewAdminStandaloneIntegration400.ts | 20 + .../previewAdminStandaloneIntegration401.ts | 20 + .../previewAdminStandaloneIntegration403.ts | 20 + .../previewAdminStandaloneIntegration409.ts | 20 + .../previewAdminStandaloneIntegration413.ts | 20 + .../previewAdminStandaloneIntegration415.ts | 20 + .../previewAdminStandaloneIntegration429.ts | 11 + .../previewAdminStandaloneIntegration500.ts | 20 + .../previewAdminStandaloneIntegration503.ts | 20 + .../previewAdminStandaloneIntegrationBody.ts | 7 + ...iewAdminStandaloneIntegrationBodySource.ts | 25 + ...reviewAdminStandaloneIntegrationHeaders.ts | 9 + .../admin/operation-bindings.generated.ts | 18 + .../src/build-artifacts/control-authority.ts | 7 +- packages/core/src/ops/contract.test.ts | 22 + packages/core/src/ops/contract.ts | 12 + packages/core/src/ops/index.ts | 1 + .../src/ops/standalone-integrations.test.ts | 26 + .../core/src/ops/standalone-integrations.ts | 118 + .../src/__tests__/installer-source.test.ts | 103 + .../src/__tests__/installer-streaming.test.ts | 444 + .../src/artifact-stream.ts | 378 + .../integration-framework/src/installer.ts | 1721 +++- pnpm-lock.yaml | 3 + scripts/database-test-groups.json | 3 +- scripts/ops-authority-inventory.json | 34 +- 81 files changed, 17211 insertions(+), 510 deletions(-) create mode 100644 apps/api/src/db/migrations/0049_admin_integration_ownership.sql create mode 100644 apps/api/src/db/migrations/meta/0049_snapshot.json create mode 100644 apps/api/src/routes/admin-standalone-integrations.ts create mode 100644 apps/api/src/services/standalone-integrations/install.test.ts create mode 100644 apps/api/src/services/standalone-integrations/install.ts create mode 100644 apps/api/src/services/standalone-integrations/settlement-checkpoint.ts create mode 100644 apps/api/src/services/standalone-integrations/state.ts create mode 100644 apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts create mode 100644 apps/ops-agent/src/standalone-integration-records.ts create mode 100644 apps/ops-agent/src/standalone-integration-runtime.test.ts create mode 100644 apps/ops-agent/src/standalone-integration-runtime.ts create mode 100644 apps/web/src/components/admin/extensions/StandaloneInstallDialog.test.tsx create mode 100644 apps/web/src/components/admin/extensions/StandaloneInstallDialog.tsx create mode 100644 packages/core/src/api/contracts/integration-admin/standalone-integrations.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration202.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration400.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration401.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration403.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration413.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration415.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration429.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration500.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration503.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationBody.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationHeaders.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration202.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration400.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration401.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration403.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration413.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration415.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration429.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration500.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration503.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBody.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBodySource.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationHeaders.ts create mode 100644 packages/core/src/ops/standalone-integrations.test.ts create mode 100644 packages/core/src/ops/standalone-integrations.ts create mode 100644 packages/integration-framework/src/__tests__/installer-source.test.ts create mode 100644 packages/integration-framework/src/__tests__/installer-streaming.test.ts create mode 100644 packages/integration-framework/src/artifact-stream.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 8fcdb3a2f..b6e8c1d99 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -65352,6 +65352,1253 @@ "x-openmapx-auth": "admin" } }, + "/api/admin/standalone-integrations/install": { + "post": { + "operationId": "installAdminStandaloneIntegration", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "inspectionId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedContentDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "inspectionId", + "expectedContentDigest" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "jobId", + "operationId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "integration-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/standalone-integrations/preview": { + "post": { + "operationId": "previewAdminStandaloneIntegration", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "artifact" + ] + }, + "url": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "url", + "sha256" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "source" + ] + }, + "repository": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "ref": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$" + } + }, + "required": [ + "kind", + "repository" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "uploaded-artifact" + ] + }, + "stageId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedSha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "stageId", + "expectedSha256" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "jobId", + "operationId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "integration-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/admin/status": { "get": { "operationId": "getApiAdminStatus", diff --git a/apps/api/src/db/installed-extension-schema.ts b/apps/api/src/db/installed-extension-schema.ts index 4d4327283..8e2c8bca1 100644 --- a/apps/api/src/db/installed-extension-schema.ts +++ b/apps/api/src/db/installed-extension-schema.ts @@ -1,4 +1,7 @@ +import type { StandaloneIntegrationProvenance } from "@openmapx/core/ops"; +import { sql } from "drizzle-orm"; import { + check, index, jsonb, pgTable, @@ -22,6 +25,11 @@ export const installedExtension = pgTable( sourceUrl: text("source_url"), // Effective trust tier resolved at install: built-in | verified | community. sourceTrust: text("source_trust").notNull().default("community"), + installationKind: text("installation_kind", { enum: ["catalog", "standalone"] }) + .notNull() + .default("catalog"), + provenance: jsonb("provenance").$type(), + lastOperationId: text("last_operation_id"), installedVersion: text("installed_version").notNull(), // Snapshot of the resolved extension.json we installed (what to remove/update). manifest: jsonb("manifest"), @@ -29,7 +37,17 @@ export const installedExtension = pgTable( updatedAt: timestamp("updated_at").defaultNow().notNull(), installedBy: text("installed_by").references(() => user.id, { onDelete: "set null" }), }, - (table) => [index("installedExtension_sourceTrust_idx").on(table.sourceTrust)], + (table) => [ + index("installedExtension_sourceTrust_idx").on(table.sourceTrust), + check( + "installed_extension_installation_kind_check", + sql`${table.installationKind} in ('catalog','standalone')`, + ), + check( + "installed_extension_standalone_provenance_check", + sql`${table.installationKind} <> 'standalone' or coalesce((${table.sourceTrust} = 'community' and ${table.id} ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof(${table.provenance}) = 'object' and ${table.provenance}->>'kind' in ('artifact','source','uploaded-artifact') and ${table.provenance}->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)`, + ), + ], ); // One row per component an extension installed, linking to the per-component diff --git a/apps/api/src/db/installed-integration-schema.ts b/apps/api/src/db/installed-integration-schema.ts index 6d78c49c5..e4b8f7833 100644 --- a/apps/api/src/db/installed-integration-schema.ts +++ b/apps/api/src/db/installed-integration-schema.ts @@ -5,7 +5,7 @@ export const installedIntegration = pgTable( "installed_integration", { id: text("id").primaryKey(), - repository: text("repository").notNull(), + repository: text("repository"), installedVersion: text("installed_version").notNull(), sourceType: text("source_type").notNull().default("registry"), // registry | artifact installedAt: timestamp("installed_at").defaultNow().notNull(), diff --git a/apps/api/src/db/migrations/0049_admin_integration_ownership.sql b/apps/api/src/db/migrations/0049_admin_integration_ownership.sql new file mode 100644 index 000000000..136799cc4 --- /dev/null +++ b/apps/api/src/db/migrations/0049_admin_integration_ownership.sql @@ -0,0 +1,6 @@ +ALTER TABLE "installed_integration" ALTER COLUMN "repository" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "installed_extension" ADD COLUMN "installation_kind" text DEFAULT 'catalog' NOT NULL;--> statement-breakpoint +ALTER TABLE "installed_extension" ADD COLUMN "provenance" jsonb;--> statement-breakpoint +ALTER TABLE "installed_extension" ADD COLUMN "last_operation_id" text;--> statement-breakpoint +ALTER TABLE "installed_extension" ADD CONSTRAINT "installed_extension_installation_kind_check" CHECK ("installed_extension"."installation_kind" in ('catalog','standalone'));--> statement-breakpoint +ALTER TABLE "installed_extension" ADD CONSTRAINT "installed_extension_standalone_provenance_check" CHECK ("installed_extension"."installation_kind" <> 'standalone' or coalesce(("installed_extension"."source_trust" = 'community' and "installed_extension"."id" ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof("installed_extension"."provenance") = 'object' and "installed_extension"."provenance"->>'kind' in ('artifact','source','uploaded-artifact') and "installed_extension"."provenance"->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)); \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0049_snapshot.json b/apps/api/src/db/migrations/meta/0049_snapshot.json new file mode 100644 index 000000000..48ef2597c --- /dev/null +++ b/apps/api/src/db/migrations/meta/0049_snapshot.json @@ -0,0 +1,9121 @@ +{ + "id": "0aa5791a-a19b-489f-9999-00d604524afe", + "prevId": "8ee8e5bb-6745-4202-ba28-f7579b4c192f", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_action_receipt": { + "name": "account_action_receipt", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mail_requested": { + "name": "mail_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_action_actor_time_idx": { + "name": "account_action_actor_time_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_action_target_time_idx": { + "name": "account_action_target_time_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_action_receipt_actor_id_user_id_fk": { + "name": "account_action_receipt_actor_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "account_action_receipt_target_id_user_id_fk": { + "name": "account_action_receipt_target_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_admin_guard": { + "name": "account_admin_guard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "active_count": { + "name": "active_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_admin_guard_singleton": { + "name": "account_admin_guard_singleton", + "value": "\"account_admin_guard\".\"id\"=1" + }, + "account_admin_guard_nonnegative": { + "name": "account_admin_guard_nonnegative", + "value": "\"account_admin_guard\".\"active_count\">=0" + } + }, + "isRLSEnabled": false + }, + "public.account_mail_outbox": { + "name": "account_mail_outbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_mail_action_kind_uq": { + "name": "account_mail_action_kind_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_mail_pending_idx": { + "name": "account_mail_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_mail_outbox_request_key_account_action_receipt_key_fk": { + "name": "account_mail_outbox_request_key_account_action_receipt_key_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "account_action_receipt", + "columnsFrom": [ + "request_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "account_mail_outbox_user_id_user_id_fk": { + "name": "account_mail_outbox_user_id_user_id_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_mail_state_check": { + "name": "account_mail_state_check", + "value": "\"account_mail_outbox\".\"state\" in ('queued','sending','sent','failed','unknown')" + }, + "account_mail_kind_check": { + "name": "account_mail_kind_check", + "value": "\"account_mail_outbox\".\"kind\" in ('verification','password_setup')" + } + }, + "isRLSEnabled": false + }, + "public.admin_action_reauthentication": { + "name": "admin_action_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admin_reauth_actor_expiry_idx": { + "name": "admin_reauth_actor_expiry_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admin_reauth_request_uq": { + "name": "admin_reauth_request_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_action_reauthentication_actor_id_user_id_fk": { + "name": "admin_action_reauthentication_actor_id_user_id_fk", + "tableFrom": "admin_action_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "admin_reauth_state_check": { + "name": "admin_reauth_state_check", + "value": "\"admin_action_reauthentication\".\"state\" in ('pending','completed','consumed','superseded','expired')" + }, + "admin_reauth_purpose_check": { + "name": "admin_reauth_purpose_check", + "value": "\"admin_action_reauthentication\".\"purpose\" in ('account.verify','account.password.set','account.reset.send','account.invite','credentials.export','credentials.import','transit.credentials','platform.domain.apply')" + } + }, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_application_actor_id_user_id_fk": { + "name": "auth_policy_application_actor_id_user_id_fk", + "tableFrom": "auth_policy_application", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_state_activated_by_user_id_fk": { + "name": "auth_policy_state_activated_by_user_id_fk", + "tableFrom": "auth_policy_state", + "tableTo": "user", + "columnsFrom": [ + "activated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installation_kind": { + "name": "installation_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'catalog'" + }, + "provenance": { + "name": "provenance", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_operation_id": { + "name": "last_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "installed_extension_installation_kind_check": { + "name": "installed_extension_installation_kind_check", + "value": "\"installed_extension\".\"installation_kind\" in ('catalog','standalone')" + }, + "installed_extension_standalone_provenance_check": { + "name": "installed_extension_standalone_provenance_check", + "value": "\"installed_extension\".\"installation_kind\" <> 'standalone' or coalesce((\"installed_extension\".\"source_trust\" = 'community' and \"installed_extension\".\"id\" ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof(\"installed_extension\".\"provenance\") = 'object' and \"installed_extension\".\"provenance\"->>'kind' in ('artifact','source','uploaded-artifact') and \"installed_extension\".\"provenance\"->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)" + } + }, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_independent_password_login": { + "name": "origin_independent_password_login", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transit_credential_preview": { + "name": "transit_credential_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_ciphertext": { + "name": "authorization_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_iv": { + "name": "authorization_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_tag": { + "name": "authorization_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "credentials_revision": { + "name": "credentials_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "transit_credential_preview_actor_key_uq": { + "name": "transit_credential_preview_actor_key_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transit_credential_preview_expiry_idx": { + "name": "transit_credential_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "transit_credential_preview_actor_id_user_id_fk": { + "name": "transit_credential_preview_actor_id_user_id_fk", + "tableFrom": "transit_credential_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "transit_credential_preview_bounds": { + "name": "transit_credential_preview_bounds", + "value": "octet_length(\"transit_credential_preview\".\"ciphertext\") <= 1500000 and length(\"transit_credential_preview\".\"intent_fingerprint\") = 64 and length(\"transit_credential_preview\".\"iv\") = 16 and length(\"transit_credential_preview\".\"tag\") = 24" + } + }, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_application_receipt": { + "name": "integration_application_receipt", + "schema": "", + "columns": { + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "saved_binding_revision": { + "name": "saved_binding_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_application_target_time_idx": { + "name": "integration_application_target_time_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_application_actor_idx": { + "name": "integration_application_actor_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_application_receipt_actor_id_user_id_fk": { + "name": "integration_application_receipt_actor_id_user_id_fk", + "tableFrom": "integration_application_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_binding_revision": { + "name": "integration_binding_revision", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 44b1eff07..1eb7555aa 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -344,6 +344,13 @@ "when": 1793227919397, "tag": "0048_admin_transit_credentials", "breakpoints": true + }, + { + "idx": 49, + "version": "7", + "when": 1793271038219, + "tag": "0049_admin_integration_ownership", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/routes/admin-standalone-integrations.ts b/apps/api/src/routes/admin-standalone-integrations.ts new file mode 100644 index 000000000..e74fb346a --- /dev/null +++ b/apps/api/src/routes/admin-standalone-integrations.ts @@ -0,0 +1,43 @@ +import type { FastifyInstance } from "fastify"; +import { jobRunner, RetainedJobConflict } from "../services/job-runner"; +import { writeAuditLog } from "../utils/audit-log"; +import { getAdminSession, requireAdmin } from "../utils/require-admin"; +import { declareRouteAuth } from "../utils/route-auth"; +import { registerIntegrationAdminRoute } from "./admin-integration-contract"; +export async function adminStandaloneIntegrationsRoute(app: FastifyInstance): Promise { + declareRouteAuth(app, "admin"); + app.addHook("preHandler", async (request) => { + request.adminSession = await requireAdmin(request); + }); + for (const id of [ + "previewAdminStandaloneIntegration", + "installAdminStandaloneIntegration", + ] as const) { + registerIntegrationAdminRoute(app, id, async (request, reply) => { + try { + const actorId = getAdminSession(request).user.id; + const type = + id === "previewAdminStandaloneIntegration" ? "standalone.preview" : "standalone.install"; + const accepted = await jobRunner.enqueueStandalone( + type, + request.body, + actorId, + String(request.headers["idempotency-key"]), + ); + await writeAuditLog({ + actorId, + action: type, + targetType: "integration", + targetId: accepted.operationId, + details: { jobId: accepted.jobId }, + request, + }); + return reply.code(202).send(accepted); + } catch (error) { + if (error instanceof RetainedJobConflict) + return reply.code(409).send({ error: error.message }); + throw error; + } + }); + } +} diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index 3785f2479..289aa4c00 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -14,6 +14,7 @@ import { adminOperationsRoute } from "./admin-operations"; import { adminReauthenticationRoute } from "./admin-reauthentication"; import { adminServicesRoute } from "./admin-services"; import { adminSettingsRoute } from "./admin-settings"; +import { adminStandaloneIntegrationsRoute } from "./admin-standalone-integrations"; import { adminSystemRoute } from "./admin-system"; import { adminTransitCredentialsRoute } from "./admin-transit-credentials"; import { adminTransitPinsRoute } from "./admin-transit-pins"; @@ -131,6 +132,7 @@ export async function registerCoreRoutes( await server.register(dataManagerRoute, { prefix: "/api" }); await server.register(adminSettingsRoute, { prefix: "/api" }); await server.register(adminExtensionsRoute, { prefix: "/api" }); + await server.register(adminStandaloneIntegrationsRoute, { prefix: "/api" }); await server.register(adminCacheRoute, { prefix: "/api" }); await server.register(adminSystemRoute, { prefix: "/api" }); await server.register(attributionRoute, { prefix: "/api" }); diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 25b1e7863..506a2d91d 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -710,6 +710,12 @@ jobRunner.register("integration.reload", async (ctx) => { return result as Record; }); +jobRunner.register("standalone.preview", async (ctx) => + (await import("./services/standalone-integrations/install")).handleStandalonePreview(ctx), +); +jobRunner.register("standalone.install", async (ctx) => + (await import("./services/standalone-integrations/install")).handleStandaloneInstall(ctx), +); jobRunner.register("extension.install", handleExtensionInstallJob); jobRunner.register("extension.remove", handleExtensionRemoveJob); diff --git a/apps/api/src/services/__tests__/extension-install-journal.test.ts b/apps/api/src/services/__tests__/extension-install-journal.test.ts index 9de98b487..20b7c6da9 100644 --- a/apps/api/src/services/__tests__/extension-install-journal.test.ts +++ b/apps/api/src/services/__tests__/extension-install-journal.test.ts @@ -30,7 +30,7 @@ function fixture() { } describe("extension integration install journal", () => { - it("restores upgraded artifacts when bookkeeping did not commit", async () => { + it("preserves a historical upgraded artifact without an exact publication ownership receipt", async () => { const root = fixture(); const target = join(root, "custom_integrations", "probe"); const backup = join(root, "custom_integrations", ".rollback-integration-probe-aaaaaaaaaaaa"); @@ -43,14 +43,15 @@ describe("extension integration install journal", () => { appendIntegrationJournalEntry(journal, { id: "probe", backupDirectory: backup }); markIntegrationJournalEntryInstalled(journal, "probe"); - await reconcileExtensionInstallJournal(root, journal.path, async () => false); - - expect(readFileSync(join(target, "version"), "utf8")).toBe("old"); - expect(existsSync(backup)).toBe(false); - expect(existsSync(journal.path)).toBe(false); + await expect( + reconcileExtensionInstallJournal(root, journal.path, async () => false), + ).rejects.toThrow(/ownership receipt/); + expect(readFileSync(join(target, "version"), "utf8")).toBe("new"); + expect(existsSync(backup)).toBe(true); + expect(existsSync(journal.path)).toBe(true); }); - it("removes a fresh artifact when bookkeeping did not commit", async () => { + it("preserves an unprovable historical fresh artifact instead of removing a later writer", async () => { const root = fixture(); const target = join(root, "custom_integrations", "probe"); mkdirSync(target); @@ -58,10 +59,11 @@ describe("extension integration install journal", () => { appendIntegrationJournalEntry(journal, { id: "probe", backupDirectory: null }); markIntegrationJournalEntryInstalled(journal, "probe"); - await reconcileExtensionInstallJournal(root, journal.path, async () => false); - - expect(existsSync(target)).toBe(false); - expect(existsSync(journal.path)).toBe(false); + await expect( + reconcileExtensionInstallJournal(root, journal.path, async () => false), + ).rejects.toThrow(/ownership receipt/); + expect(existsSync(target)).toBe(true); + expect(existsSync(journal.path)).toBe(true); }); it("keeps installed artifacts and discards backups after committed bookkeeping", async () => { @@ -111,3 +113,72 @@ describe("extension integration install journal", () => { expect(existsSync(journal.path)).toBe(true); }); }); + +it.each([false, true])( + "settles an actual shared publication from retained bundle journal (bookkeeping=%s)", + async (committed) => { + const root = fixture(); + mkdirSync(join(root, "services")); + const { createDeterministicTarGz } = await import( + "../../../../../packages/integration-framework/src/deterministic-tar" + ); + const { installIntegration, removeIntegration } = await import( + "@openmapx/integration-framework/installer" + ); + const { recordIntegrationPublicationPrepared } = await import("../extension-install-journal"); + const archive = join(root, "fixture.tar.gz"); + writeFileSync( + archive, + createDeterministicTarGz([ + { + path: "manifest.json", + contents: Buffer.from( + JSON.stringify({ + id: "bundle-fixture", + name: "Bundle", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ), + }, + ]), + ); + const journal = createExtensionInstallJournal(root, "bundle", { version: "1.0.0" }); + appendIntegrationJournalEntry(journal, { + id: "bundle-fixture", + backupDirectory: null, + transactional: true, + }); + await installIntegration({ + rootDir: root, + source: archive, + sourceKind: "artifact", + expectedId: "bundle-fixture", + retainPublication: true, + beforePublish: (receipt) => + recordIntegrationPublicationPrepared(journal, "bundle-fixture", receipt), + }); + expect(() => removeIntegration({ rootDir: root, id: "bundle-fixture" })).toThrow(/busy/); + await reconcileExtensionInstallJournal(root, journal.path, async () => committed); + expect(existsSync(join(root, "custom_integrations", "bundle-fixture"))).toBe(committed); + expect(existsSync(journal.path)).toBe(false); + if (committed) + expect(() => removeIntegration({ rootDir: root, id: "bundle-fixture" })).not.toThrow(); + }, +); +it("does not delete a current integration when a transactional install never reached publication", async () => { + const root = fixture(), + target = join(root, "custom_integrations", "unmodified"); + mkdirSync(target); + writeFileSync(join(target, "keep"), "original"); + const journal = createExtensionInstallJournal(root, "bundle", { version: "1.0.0" }); + appendIntegrationJournalEntry(journal, { + id: "unmodified", + backupDirectory: null, + transactional: true, + }); + await reconcileExtensionInstallJournal(root, journal.path, async () => false); + expect(readFileSync(join(target, "keep"), "utf8")).toBe("original"); +}); diff --git a/apps/api/src/services/__tests__/extension-installer.test.ts b/apps/api/src/services/__tests__/extension-installer.test.ts index c6bf8c852..3f4635c61 100644 --- a/apps/api/src/services/__tests__/extension-installer.test.ts +++ b/apps/api/src/services/__tests__/extension-installer.test.ts @@ -24,11 +24,13 @@ vi.mock("@openmapx/integration-framework/installer", () => ({ backupDirectory: "/safe/mock-integration-backup", })), discardInstalledIntegrationBackup: vi.fn(), + settleIntegrationPublication: vi.fn(async () => ({ outcome: "restored", contentDigest: null })), installIntegration: vi.fn(async (opts: { source: string }) => ({ // The installer asserts the artifact reports the declared component id. id: /intg-[a-z0-9-]+/.exec(opts.source)?.[0] ?? "intg-one", directory: "/safe/mock-integration", replaced: false, + publication: Object.freeze({ fixture: "mocked filesystem transaction port" }), })), removeIntegration: vi.fn(), restoreInstalledIntegration: vi.fn(), @@ -74,6 +76,7 @@ vi.mock("../extension-install-journal", () => ({ data: { schemaVersion: 1, extensionId, targetManifest, integrations: [] }, })), appendIntegrationJournalEntry: vi.fn(), + recordIntegrationPublicationPrepared: vi.fn(), markIntegrationJournalEntryInstalled: vi.fn(), markExtensionInstallJournalRestoring: vi.fn(), reconcileExtensionInstallJournal: vi.fn().mockResolvedValue(undefined), @@ -111,6 +114,7 @@ import { installIntegration, removeIntegration, restoreInstalledIntegration, + settleIntegrationPublication, } from "@openmapx/integration-framework/installer"; import { reloadIntegrations } from "../../integration-host"; import { dockerComposeAction } from "../../utils/docker-compose"; @@ -835,10 +839,13 @@ describe("installExtension", () => { installExtension(makeCtx(), { manifest: INTEGRATION_ONLY, sourceTrust: "community" }), ).rejects.toThrow(/reload-fail/); - expect(dbRemoveMock).toHaveBeenCalledWith( - expect.objectContaining({ rootDir: expect.any(String), id: "intg-one" }), + expect(settleIntegrationPublication).toHaveBeenCalledWith( + expect.anything(), + "restore", + expect.any(AbortSignal), ); - expect(dbMock.db.delete).toHaveBeenCalled(); // installedIntegration cleanup + expect(dbRemoveMock).not.toHaveBeenCalled(); + expect(dbMock.db.delete).not.toHaveBeenCalled(); // no bookkeeping rows existed to delete // Pipeline reload (threw) + rollback reload. expect(reloadMock).toHaveBeenCalledTimes(2); // The reload threw before the bookkeeping transaction, so no rows to leak. @@ -853,11 +860,13 @@ describe("installExtension", () => { installExtension(makeCtx(), { manifest: INTEGRATION_ONLY, sourceTrust: "community" }), ).rejects.toThrow(/reload-fail/); - expect(integrationBackupMock).toHaveBeenCalledWith(expect.any(String), "intg-one"); - expect(integrationRestoreMock).toHaveBeenCalledWith(expect.any(String), { - id: "intg-one", - backupDirectory: "/safe/mock-integration-backup", - }); + expect(integrationBackupMock).not.toHaveBeenCalled(); + expect(integrationRestoreMock).not.toHaveBeenCalled(); + expect(settleIntegrationPublication).toHaveBeenCalledWith( + expect.anything(), + "restore", + expect.any(AbortSignal), + ); expect(dbRemoveMock).not.toHaveBeenCalled(); expect(integrationBackupDiscardMock).not.toHaveBeenCalled(); expect(reloadMock).toHaveBeenCalledTimes(2); @@ -872,10 +881,12 @@ describe("installExtension", () => { }); expect(result).toEqual({ id: "bundle-x", components: 1 }); - expect(integrationBackupDiscardMock).toHaveBeenCalledWith(expect.any(String), { - id: "intg-one", - backupDirectory: "/safe/mock-integration-backup", - }); + expect(integrationBackupDiscardMock).not.toHaveBeenCalled(); + expect(reconcileInstallJournalMock).toHaveBeenCalledWith( + expect.any(String), + expect.any(String), + expect.any(Function), + ); expect(dbMock.db.delete).toHaveBeenCalledTimes(1); // Inserts: [0]=installedIntegration, [1]=installedExtension, [2]=component. diff --git a/apps/api/src/services/activity-jobs.ts b/apps/api/src/services/activity-jobs.ts index 57d9f04bc..59be064f0 100644 --- a/apps/api/src/services/activity-jobs.ts +++ b/apps/api/src/services/activity-jobs.ts @@ -12,7 +12,7 @@ import { unionAll } from "drizzle-orm/pg-core"; import { db } from "../db"; import { adminJob, adminJobLog, dataManagerJobStages, dataManagerJobs } from "../db/schema"; import { type ActorSummary, resolveActors } from "../utils/resolve-actor"; -import { publicAdminJobResult } from "./admin-job-failure"; +import { publicAdminJobPayload, publicAdminJobResult } from "./admin-job-failure"; export type ActivityJobSource = "application" | "data-manager"; export type ActivityJobFilter = "active" | "completed" | "failed"; @@ -220,7 +220,7 @@ export async function listActivityJobs(options: { id: row.id, type: row.type, status: row.status, - payload: row.payload, + payload: publicAdminJobPayload(row.type, row.payload), error: row.error, progress: row.progress, createdBy, @@ -248,6 +248,7 @@ export async function getActivityJob(id: string, source: ActivityJobSource) { return { ...job, result: publicAdminJobResult(job.result), + payload: publicAdminJobPayload(job.type, job.payload), source, actorLabel: job.createdBy ? "Administrator" : "System", cancelable: job.status === "queued" || job.status === "running", diff --git a/apps/api/src/services/admin-job-checkpoint.ts b/apps/api/src/services/admin-job-checkpoint.ts index 04392b22a..f95503d30 100644 --- a/apps/api/src/services/admin-job-checkpoint.ts +++ b/apps/api/src/services/admin-job-checkpoint.ts @@ -16,6 +16,8 @@ import { } from "./admin-job-state"; import { assertConfigurationMaintenanceProjectedStep } from "./configuration-maintenance-step-records"; +import { assertStandaloneSettlementCheckpoint } from "./standalone-integrations/settlement-checkpoint"; + const equal = (a: unknown, b: unknown) => canonicalJson(a ?? null) === canonicalJson(b ?? null); /** Called only for v3. The database row lock owns uniqueness, capacity and merges. */ export async function checkpointOwnedAdminJob( @@ -69,8 +71,12 @@ export async function checkpointOwnedAdminJob( return { conflict: true as const, result: current }; if (Object.keys(old.steps).length >= 64 || Object.keys(old.operations).length >= 64) return { capacity: true as const, result: current }; - if (row.status !== "running") throw new JobOwnershipLost(); - if (current._cancelRequestedAt !== undefined) { + const standaloneSettlement = + row.type === "standalone.install" && operation.operation?.kind === "integration.settle"; + if (standaloneSettlement) + await assertStandaloneSettlementCheckpoint(row, operation.operation!, tx); + if (row.status !== "running" && !standaloneSettlement) throw new JobOwnershipLost(); + if (current._cancelRequestedAt !== undefined && !standaloneSettlement) { // Cancellation never authorizes a forward phase. Only the original // no-effect Abandon/Settle controls may be checkpointed against the // actual already-delivered preparation in THIS owned job row. diff --git a/apps/api/src/services/admin-job-failure.ts b/apps/api/src/services/admin-job-failure.ts index 1af83f97f..c5090e6c1 100644 --- a/apps/api/src/services/admin-job-failure.ts +++ b/apps/api/src/services/admin-job-failure.ts @@ -42,3 +42,14 @@ export class JobObservationPending extends Error { super("Administrative job observation is pending"); } } + +/** Standalone acquisition inputs are public metadata; ownership and request MACs are not. */ +export function publicAdminJobPayload(type: string, payload: unknown): unknown { + if (type !== "standalone.preview" && type !== "standalone.install") return payload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; + const fields = + type === "standalone.preview" + ? ["source"] + : ["inspectionId", "expectedContentDigest", "operationId"]; + return Object.fromEntries(Object.entries(payload).filter(([key]) => fields.includes(key))); +} diff --git a/apps/api/src/services/admin-job-ops.ts b/apps/api/src/services/admin-job-ops.ts index 7353196d4..c33cb714a 100644 --- a/apps/api/src/services/admin-job-ops.ts +++ b/apps/api/src/services/admin-job-ops.ts @@ -1023,6 +1023,17 @@ export async function cancelAdminJobOperations( else if (backend === "already_terminal" || latestBackend === "already_terminal") outcomes.push("already_terminal"); if (hasUnsettledHardlinks(result)) return "pending"; + const standaloneResult = options.context?.checkpointResult ?? result; + if ( + standaloneResult && + typeof standaloneResult === "object" && + "_standalone" in standaloneResult + ) { + const { hasVerifiedStandaloneRestoration } = await import( + "./standalone-integrations/settlement-checkpoint" + ); + if (await hasVerifiedStandaloneRestoration(standaloneResult, options.jobId)) return "canceled"; + } if (outcomes.includes("contained")) return "canceled"; return outcomes.every((outcome) => outcome === "succeeded") ? "completed" : "already_terminal"; } diff --git a/apps/api/src/services/admin-job-parent.ts b/apps/api/src/services/admin-job-parent.ts index 495f76053..22428ac34 100644 --- a/apps/api/src/services/admin-job-parent.ts +++ b/apps/api/src/services/admin-job-parent.ts @@ -17,6 +17,12 @@ export { type ApiJobChildAttempt, apiJobChildAttemptSchema } from "@openmapx/cor import type { ApiJobChildAttempt } from "@openmapx/core/ops"; function parentJobBudget(ctx: JobContext, kind: OpsOperationKind): number { + if (ctx.jobType === "standalone.install") + return ( + opsExecutionBudget("integration.install").processMs + + opsExecutionBudget("integration.settle").processMs + + 60000 + ); if (ctx.jobType === "system.update") return opsExecutionBudget("system.update").processMs; if (ctx.jobType === "service.start" || ctx.jobType === "service.apply") { // Fixed current workflow: render, inspect, start the coordinator, await its diff --git a/apps/api/src/services/admin-job-state.ts b/apps/api/src/services/admin-job-state.ts index e02802acb..9f1b232e5 100644 --- a/apps/api/src/services/admin-job-state.ts +++ b/apps/api/src/services/admin-job-state.ts @@ -44,7 +44,9 @@ export function hasUnsettledManagedOperation(result: unknown): boolean { } | null )?.opsProjection; const data = (result as { _dataExecutions?: unknown } | null)?._dataExecutions; + const standalone = result as { _standalone?: unknown; _standaloneSettled?: unknown } | null; return ( + (standalone?._standalone !== undefined && standalone._standaloneSettled !== true) || (p?.version === 3 && Object.values(p.operations ?? {}).some((o) => o.terminal !== true)) || (data !== undefined && (!Array.isArray(data) || @@ -61,7 +63,7 @@ export function hasUnsettledManagedOperation(result: unknown): boolean { ); } export function noUnsettledManagedOperation() { - return sql`NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS(SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS active_entry WHERE coalesce(active_entry.value->>'terminal','false') <> 'true')) AND CASE WHEN ${adminJob.result}->'_dataExecutions' IS NULL THEN true WHEN jsonb_typeof(${adminJob.result}->'_dataExecutions') <> 'array' THEN false ELSE NOT EXISTS(SELECT 1 FROM jsonb_array_elements(${adminJob.result}->'_dataExecutions') AS backend_entry WHERE backend_entry->>'state' IS DISTINCT FROM 'terminal' OR coalesce(backend_entry->>'publication','unknown') NOT IN ('not_started','committed') OR coalesce(backend_entry->>'status','') NOT IN ('ok','partial','error','canceled','interrupted') OR coalesce(backend_entry->>'id','') !~ '^[a-f0-9-]{36}$' OR coalesce(backend_entry->>'operationKey','') !~ '^opk1_[A-Za-z0-9_-]{16,64}$' OR coalesce(backend_entry->>'fingerprint','') !~ '^[a-f0-9]{64}$') END`; + return sql`(${adminJob.result}->'_standalone' IS NULL OR ${adminJob.result}->'_standaloneSettled' = 'true'::jsonb) AND NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS(SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS active_entry WHERE coalesce(active_entry.value->>'terminal','false') <> 'true')) AND CASE WHEN ${adminJob.result}->'_dataExecutions' IS NULL THEN true WHEN jsonb_typeof(${adminJob.result}->'_dataExecutions') <> 'array' THEN false ELSE NOT EXISTS(SELECT 1 FROM jsonb_array_elements(${adminJob.result}->'_dataExecutions') AS backend_entry WHERE backend_entry->>'state' IS DISTINCT FROM 'terminal' OR coalesce(backend_entry->>'publication','unknown') NOT IN ('not_started','committed') OR coalesce(backend_entry->>'status','') NOT IN ('ok','partial','error','canceled','interrupted') OR coalesce(backend_entry->>'id','') !~ '^[a-f0-9-]{36}$' OR coalesce(backend_entry->>'operationKey','') !~ '^opk1_[A-Za-z0-9_-]{16,64}$' OR coalesce(backend_entry->>'fingerprint','') !~ '^[a-f0-9]{64}$') END`; } export function noPendingReceiptDelivery() { return sql`NOT (coalesce(${adminJob.result}->'opsProjection'->>'version','') = '3' AND EXISTS (SELECT 1 FROM jsonb_each(coalesce(${adminJob.result}->'opsProjection'->'operations','{}'::jsonb)) AS receipt_entry WHERE receipt_entry.value->'receiptDelivery'->>'state' = 'pending'))`; diff --git a/apps/api/src/services/extension-install-journal.ts b/apps/api/src/services/extension-install-journal.ts index ab5d3eb74..3532c5dcf 100644 --- a/apps/api/src/services/extension-install-journal.ts +++ b/apps/api/src/services/extension-install-journal.ts @@ -8,13 +8,20 @@ import { rmSync, writeFileSync, } from "node:fs"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; +import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; import { discardInstalledIntegrationBackup, + type IntegrationPublicationReceipt, type IntegrationRollbackBackup, + inspectIntegrationPublication, + integrationPublicationReceiptSchema, removeIntegration, restoreInstalledIntegration, + restoreIntegrationPublication, + settleIntegrationPublication, } from "@openmapx/integration-framework/installer"; +import z from "zod/v4"; const JOURNAL_PREFIX = ".extension-install-journal-"; const JOURNAL_RE = /^\.extension-install-journal-[a-f0-9]{24}\.json$/; @@ -25,6 +32,8 @@ interface JournalEntry { id: string; backupDirectory: string | null; phase: EntryPhase; + transactional?: true; + publication?: IntegrationPublicationReceipt; } export interface ExtensionInstallJournalData { @@ -44,12 +53,16 @@ function customDir(rootDir: string): string { } function persist(journal: ExtensionInstallJournal): void { - const temporary = `${journal.path}.tmp-${process.pid}`; - writeFileSync(temporary, `${JSON.stringify(journal.data, null, 2)}\n`, { - encoding: "utf8", - mode: 0o600, - }); - renameSync(temporary, journal.path); + if (!JOURNAL_RE.test(basename(journal.path))) throw Error("Invalid extension journal path"); + const bytes = JSON.stringify(journal.data); + if (Buffer.byteLength(bytes) > 2 * 1024 ** 2) + throw Error("Extension integration journal exceeds its metadata limit"); + const directory = new AnchoredDirectory(dirname(journal.path)); + try { + directory.write(basename(journal.path), bytes); + } finally { + directory.close(); + } } export function createExtensionInstallJournal( @@ -69,7 +82,7 @@ export function createExtensionInstallJournal( export function appendIntegrationJournalEntry( journal: ExtensionInstallJournal, - entry: { id: string; backupDirectory: string | null }, + entry: { id: string; backupDirectory: string | null; transactional?: true }, ): void { if (journal.data.integrations.some((candidate) => candidate.id === entry.id)) { throw new Error(`Integration ${entry.id} is already present in the extension install journal`); @@ -78,6 +91,46 @@ export function appendIntegrationJournalEntry( persist(journal); } +/** Called synchronously by the actual publisher before its first directory effect. */ +export function recordIntegrationPublicationPrepared( + journal: ExtensionInstallJournal, + id: string, + receipt: IntegrationPublicationReceipt, +): void { + const entry = journal.data.integrations.find((candidate) => candidate.id === id); + if ( + !entry?.transactional || + entry.publication || + receipt.integrationId !== id || + receipt.outcome !== "prepared" + ) + throw Error("Integration publication journal mismatch"); + entry.publication = integrationPublicationReceiptSchema.parse(receipt); + persist(journal); +} +async function settleTransactionalEntry( + rootDir: string, + entry: JournalEntry, + decision: "commit" | "restore", +) { + if (!entry.publication) { + if (decision === "commit") throw Error("Integration publication receipt missing"); + return; + } + const receipt = inspectIntegrationPublication({ + rootDir, + preparation: entry.publication.preparation, + transactionId: entry.publication.transactionId, + }); + if (receipt.integrationId !== entry.id) throw Error("Integration journal component changed"); + const publication = await restoreIntegrationPublication({ + rootDir, + receipt, + signal: new AbortController().signal, + }); + await settleIntegrationPublication(publication, decision, new AbortController().signal); +} + export function markIntegrationJournalEntryInstalled( journal: ExtensionInstallJournal, id: string, @@ -95,19 +148,34 @@ export function markExtensionInstallJournalRestoring(journal: ExtensionInstallJo persist(journal); } -function readJournal(path: string): ExtensionInstallJournal { - if (!JOURNAL_RE.test(basename(path))) throw new Error("Invalid extension install journal path"); - const data = JSON.parse(readFileSync(path, "utf8")) as ExtensionInstallJournalData; - if ( - data.schemaVersion !== 1 || - typeof data.extensionId !== "string" || - !data.targetManifest || - typeof data.targetManifest !== "object" || - !Array.isArray(data.integrations) - ) { - throw new Error(`Invalid extension install journal ${basename(path)}`); +const journalSchema = z.strictObject({ + schemaVersion: z.literal(1), + extensionId: z.string().min(1).max(128), + targetManifest: z.record(z.string(), z.unknown()), + integrations: z + .array( + z.strictObject({ + id: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + backupDirectory: z.string().max(4096).nullable(), + phase: z.enum(["prepared", "installed", "restoring", "restored"]), + transactional: z.literal(true).optional(), + publication: integrationPublicationReceiptSchema.optional(), + }), + ) + .max(256), +}); +function readJournal(rootDir: string, path: string): ExtensionInstallJournal { + if (!JOURNAL_RE.test(basename(path)) || resolve(dirname(path)) !== resolve(customDir(rootDir))) + throw Error("Invalid extension install journal path"); + const directory = new AnchoredDirectory(customDir(rootDir)); + try { + const data = journalSchema.parse( + JSON.parse(directory.read(basename(path), 2 * 1024 ** 2).toString("utf8")), + ); + return { path, data }; + } finally { + directory.close(); } - return { path, data }; } export async function reconcileExtensionInstallJournal( @@ -115,9 +183,13 @@ export async function reconcileExtensionInstallJournal( path: string, isBookkeepingCommitted: (journal: ExtensionInstallJournalData) => Promise, ): Promise { - const journal = readJournal(path); + const journal = readJournal(rootDir, path); if (await isBookkeepingCommitted(journal.data)) { for (const entry of journal.data.integrations) { + if (entry.transactional) { + await settleTransactionalEntry(rootDir, entry, "commit"); + continue; + } if (entry.backupDirectory && existsSync(entry.backupDirectory)) { discardInstalledIntegrationBackup(rootDir, { id: entry.id, @@ -132,6 +204,12 @@ export async function reconcileExtensionInstallJournal( for (const entry of [...journal.data.integrations].reverse()) { const target = join(customDir(rootDir), entry.id); if (entry.phase === "restored") continue; + if (entry.transactional) { + await settleTransactionalEntry(rootDir, entry, "restore"); + entry.phase = "restored"; + persist(journal); + continue; + } const previousPhase = entry.phase; if ( entry.backupDirectory && @@ -149,7 +227,9 @@ export async function reconcileExtensionInstallJournal( }; if (existsSync(entry.backupDirectory)) restoreInstalledIntegration(rootDir, backup); } else if (existsSync(target)) { - removeIntegration({ rootDir, id: entry.id }); + throw Error( + "Historical integration removal lacks a publication ownership receipt; preserve the current integration and reconcile the original journal", + ); } entry.phase = "restored"; persist(journal); @@ -173,7 +253,7 @@ export async function reconcileExtensionInstallJournals( } catch (error) { throw new Error( `Extension install journal ${path} could not be reconciled: ${(error as Error).message}. ` + - "If its integration backups were removed deliberately, delete this journal and restart.", + "Preserve its current files and original backups until explicit integration reconciliation completes.", { cause: error }, ); } diff --git a/apps/api/src/services/extension-installer.ts b/apps/api/src/services/extension-installer.ts index ceb49d508..0ee5144ee 100644 --- a/apps/api/src/services/extension-installer.ts +++ b/apps/api/src/services/extension-installer.ts @@ -2,12 +2,13 @@ import { createHash, randomUUID } from "node:crypto"; import { services as coreServices, findRepoRoot } from "@openmapx/core/server"; import { PLATFORM_VERSION, satisfiesPlatformVersion } from "@openmapx/integration-framework"; import { - backupInstalledIntegration, installIntegration as coreInstallIntegration, removeIntegration as coreRemoveIntegration, discardInstalledIntegrationBackup, + type IntegrationPublication, type IntegrationRollbackBackup, restoreInstalledIntegration, + settleIntegrationPublication, } from "@openmapx/integration-framework/installer"; import { and, eq } from "drizzle-orm"; import { db } from "../db"; @@ -38,6 +39,7 @@ import { markExtensionInstallJournalRestoring, markIntegrationJournalEntryInstalled, reconcileExtensionInstallJournal, + recordIntegrationPublicationPrepared, } from "./extension-install-journal"; import { recoverInterruptedExtensionChanges, withExtensionChangeLock } from "./extension-recovery"; import { getKillSwitch } from "./extension-store"; @@ -349,6 +351,7 @@ interface InstallLedger { enabledServiceIds: string[]; // services newly enabled this run installedIntegrationIds: string[]; // integrations newly installed this run previousIntegrations: IntegrationRollbackBackup[]; + publishedIntegrations: Array<{ id: string; publication: IntegrationPublication }>; touchedServiceIds: string[]; previouslyEnabledServiceIds: string[]; selectedRootsBefore: string[] | null; @@ -394,6 +397,7 @@ async function installExtensionNow( enabledServiceIds: [], installedIntegrationIds: [], previousIntegrations: [], + publishedIntegrations: [], touchedServiceIds: [], previouslyEnabledServiceIds: [], selectedRootsBefore: null, @@ -563,22 +567,18 @@ async function installExtensionNow( for (const intg of integrationComponents) { if (!integrationJournal) throw new Error("Integration install journal was not initialized"); const existed = await integrationInstalledExists(intg.id); - if (existed) { - const backup = backupInstalledIntegration(ROOT_DIR, intg.id); - if (!backup) { - throw new Error(`Installed integration "${intg.id}" is missing from disk`); - } - ledger.previousIntegrations.push(backup); - } appendIntegrationJournalEntry(integrationJournal, { id: intg.id, - backupDirectory: - ledger.previousIntegrations.find((backup) => backup.id === intg.id)?.backupDirectory ?? - null, + backupDirectory: null, + transactional: true, }); await ctx.log(`Installing integration ${intg.id} from ${intg.artifact}...`); const installed = await coreInstallIntegration({ rootDir: ROOT_DIR, + expectedId: intg.id, + retainPublication: true, + beforePublish: (receipt) => + recordIntegrationPublicationPrepared(integrationJournal!, intg.id, receipt), source: intg.artifact, sourceKind: "artifact", artifactSha256: intg.sha256, @@ -596,6 +596,9 @@ async function installExtensionNow( `Integration artifact installed "${installed.id}" but the manifest declared "${intg.id}"`, ); } + if (!installed.publication) + throw Error("Integration publisher did not retain its original transaction"); + ledger.publishedIntegrations.push({ id: intg.id, publication: installed.publication }); markIntegrationJournalEntryInstalled(integrationJournal, intg.id); if (!existed) ledger.installedIntegrationIds.push(intg.id); @@ -798,8 +801,12 @@ async function installExtensionNow( } async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise { - // Integrations first (cheap), then services. + // Only the exact held publication may restore its previous directory. + for (const entry of [...ledger.publishedIntegrations].reverse()) + await settleIntegrationPublication(entry.publication, "restore", new AbortController().signal); + // Historical entries retain their existing recovery path. for (const id of ledger.installedIntegrationIds) { + if (ledger.publishedIntegrations.some((entry) => entry.id === id)) continue; try { coreRemoveIntegration({ rootDir: ROOT_DIR, id }); await db.delete(installedIntegration).where(eq(installedIntegration.id, id)); @@ -821,7 +828,11 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< .catch(() => {}); } } - if (ledger.installedIntegrationIds.length > 0 || ledger.previousIntegrations.length > 0) { + if ( + ledger.publishedIntegrations.length > 0 || + ledger.installedIntegrationIds.length > 0 || + ledger.previousIntegrations.length > 0 + ) { await reloadIntegrations().catch(() => {}); } diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index b1fdf8402..a7526bb8b 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -107,6 +107,8 @@ function cancellationReceiptContext(job: typeof adminJob.$inferSelect): JobConte type JobHandler = (ctx: JobContext) => Promise | undefined>; export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ + "standalone.preview", + "standalone.install", "backup.operation", "data.operation", "service.bulk", @@ -180,6 +182,20 @@ function hasRetainedQueuedCoordinator(job: { ordinal !== -1 && candidate.operationKey === coordinatorLifecycleOperationKey(job.id, ordinal) ); } +export class RetainedJobConflict extends Error {} +/** Routing only: the fixed handler authenticates the original request and sealed checkpoint before continuing. */ +function hasRetainedStandalone(job: { type: string; result: unknown }) { + const value = job.result as { + _standalone?: { body?: unknown; mac?: unknown }; + _standaloneSettled?: unknown; + } | null; + return ( + job.type === "standalone.install" && + value?._standaloneSettled !== true && + typeof value?._standalone?.body === "string" && + typeof value._standalone.mac === "string" + ); +} class JobPersistenceUnavailable extends Error {} class AdminJobRunner { @@ -210,6 +226,68 @@ class AdminJobRunner { return id; } + /** Two fixed standalone producers share actor/key serialization and immutable request identity. */ + async enqueueStandalone( + type: "standalone.preview" | "standalone.install", + payload: Record, + actorId: string, + key: string, + ): Promise<{ jobId: string; operationId: string }> { + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/.test(key)) + throw new RetainedJobConflict("Invalid mutation key"); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const digest = createHmac("sha256", token) + .update("openmapx-standalone-request-v1\0") + .update(canonicalJson({ type, payload, actorId })) + .digest("hex"); + const accepted = await db.transaction(async (tx) => { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${"standalone:" + actorId + ":" + key},0))`, + ); + const rows = await tx + .select() + .from(adminJob) + .where( + and( + sql`${adminJob.payload}->>'_standaloneActor' = ${actorId}`, + sql`${adminJob.payload}->>'_standaloneKey' = ${key}`, + ), + ) + .limit(2); + if (rows.length) { + const old = rows[0]!, + retained = old.payload as Record; + if ( + rows.length !== 1 || + old.type !== type || + retained._requestDigest !== digest || + typeof retained.operationId !== "string" + ) + throw new RetainedJobConflict("Mutation key belongs to a different standalone request"); + return { jobId: old.id, operationId: retained.operationId }; + } + const jobId = randomUUID(), + operationId = randomUUID(); + await tx.insert(adminJob).values({ + id: jobId, + type, + status: "queued", + createdBy: dbActorId(actorId), + payload: { + ...payload, + ownerId: actorId, + operationId, + _standaloneActor: actorId, + _standaloneKey: key, + _requestDigest: digest, + }, + }); + return { jobId, operationId }; + }); + setTimeout(() => void this.pump(), 0); + return accepted; + } + private async immediate( options: { type: @@ -912,7 +990,10 @@ class AdminJobRunner { ): Promise { const controller = new AbortController(), retainedConfiguration = - observe && (hasRetainedImmediateConfiguration(job) || hasRetainedQueuedCoordinator(job)); + observe && + (hasRetainedImmediateConfiguration(job) || + hasRetainedQueuedCoordinator(job) || + hasRetainedStandalone(job)); this.active.set(job.id, controller); this.logSeq.set(job.id, 0); @@ -986,10 +1067,11 @@ class AdminJobRunner { retainedQueuedCoordinator = observe && hasRetainedQueuedCoordinator({ id: jobId, type, payload, result: checkpointResult }), - retainedConfiguration = retainedImmediate || retainedQueuedCoordinator, + retainedStandalone = observe && hasRetainedStandalone({ type, result: checkpointResult }), + retainedConfiguration = retainedImmediate || retainedQueuedCoordinator || retainedStandalone, handler = retainedImmediate ? (ctx: JobContext) => this.resumeImmediateConfiguration(ctx) - : retainedQueuedCoordinator + : retainedQueuedCoordinator || retainedStandalone ? this.handlers.get(type) : observe ? resumeAdminJobObservation @@ -1231,7 +1313,7 @@ class AdminJobRunner { .where(jobOwnership(jobId, execution, ["running", "cancel_pending"])) .limit(1); if (!current) return; - if (current.status === "cancel_pending") { + if (current.status === "cancel_pending" && !retainedStandalone) { const outcome = await cancelAdminJobOperations(current.result, undefined, { context: ctx, jobId, diff --git a/apps/api/src/services/standalone-integrations/install.test.ts b/apps/api/src/services/standalone-integrations/install.test.ts new file mode 100644 index 000000000..2d544d94e --- /dev/null +++ b/apps/api/src/services/standalone-integrations/install.test.ts @@ -0,0 +1,413 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { eq } from "drizzle-orm"; +import Fastify from "fastify"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { createDeterministicTarGz } from "../../../../../packages/integration-framework/src/deterministic-tar"; +import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; +import { installArtifactTransferRuntime } from "../../../../ops-agent/src/artifact-transfer/runtime"; +import { ArtifactTransferStore } from "../../../../ops-agent/src/artifact-transfer/store"; +import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; +import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; +import { installStandaloneIntegrationRuntime } from "../../../../ops-agent/src/standalone-integration-runtime"; +import * as schema from "../../db/schema"; +import { createMigratedPostgresFixture } from "../../test/postgres-fixture"; + +const state = vi.hoisted(() => ({ + fixture: null as unknown as Awaited>, + host: null as unknown as Awaited>, +})); +vi.mock("../../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); +vi.mock("../../redis", () => ({ redis: null })); +vi.mock("../service-registry", async (actual) => ({ + ...(await actual()), + getServiceRegistry: () => ({ + list: () => state.host.loaded, + get: (id: string) => state.host.loaded.find((service) => service.manifest.id === id), + }), +})); +describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "standalone install actual API/job/agent/host/PostgreSQL", + () => { + const cleanups: (() => Promise)[] = []; + let apiUrl: string, agent: ReturnType, store: ArtifactTransferStore; + let dropInstallResponse = false, + installPosts = 0, + cancelIntegrationId: string | null = null; + let jobRunner: typeof import("../job-runner").jobRunner; + beforeAll(async () => { + state.fixture = await createMigratedPostgresFixture(); + cleanups.push(() => state.fixture.stop()); + state.host = await ownedServiceHost(); + cleanups.push(async () => rmSync(state.host.cleanupRoot, { recursive: true, force: true })); + vi.stubEnv("BETTER_AUTH_SECRET", "standalone-fixture-auth-secret-not-for-production"); + vi.stubEnv("NODE_ENV", "test"); + vi.stubEnv("OPENMAPX_LOCAL_ADMIN_TOKEN", "standalone-local-fixture"); + const tokenFile = join(state.host.cleanupRoot, "api-token"); + writeFileSync(tokenFile, state.host.token, { mode: 0o600 }); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", tokenFile); + const installation = openControlInstallation({ + infraDir: state.host.infraDir, + role: "ops-agent", + serviceIds: state.host.loaded.map((service) => service.manifest.id), + }); + cleanups.push(async () => closeControlInstallation(installation)); + store = new ArtifactTransferStore({ + dataRoot: join(state.host.infraDir, "data/.openmapx-transfers"), + privateRoot: join(state.host.cleanupRoot, "transfer-metadata"), + token: state.host.token, + capacity: () => ({ bytes: 4 * 1024 ** 3, inodes: 100000 }), + }); + installArtifactTransferRuntime(state.host.runtime, store); + installStandaloneIntegrationRuntime(state.host.runtime, { rootDir: state.host.root, store }); + agent = buildOpsAgentServer({ + tokens: { api: state.host.token, "data-manager": "b".repeat(43) }, + runtime: state.host.runtime, + resourceClaimer: state.host.claimer, + journal: await openOpsJobJournal(join(state.host.infraDir, "data/ops-agent/jobs-v1.json")), + controlInstallation: installation, + artifactTransfers: store, + }); + agent.addHook("onSend", async (request, reply, payload) => { + if ( + request.method === "POST" && + (request.body as { operation?: { kind?: string } })?.operation?.kind === + "integration.install" + ) { + installPosts++; + if (dropInstallResponse) { + dropInstallResponse = false; + reply.raw.destroy(); + } + } + if (cancelIntegrationId && request.method === "GET" && typeof payload === "string") { + const status = JSON.parse(payload).result; + if ( + status?.state === "succeeded" && + status.kind === "integration.install" && + status.result?.integrationId === cancelIntegrationId + ) { + const target = cancelIntegrationId; + cancelIntegrationId = null; + const [job] = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.type, "standalone.install")) + .then((rows) => + rows.filter((row) => { + const sealed = (row.result as { _standalone?: { body?: string } } | null) + ?._standalone?.body; + return ( + row.status === "running" && + sealed && + JSON.parse(sealed).inspection.integrationId === target + ); + }), + ); + if (!job) throw Error("Missing exact published standalone job"); + await jobRunner.cancel(job.id); + } + } + return payload; + }); + cleanups.push(async () => { + await agent.close(); + store.close(); + }); + vi.stubEnv("OPS_AGENT_URL", await agent.listen({ host: "127.0.0.1", port: 0 })); + const app = Fastify({ logger: false }); + cleanups.push(() => app.close()); + const { initIntegrations, shutdownIntegrations } = await import("../../integration-host"); + await initIntegrations(app, [ + { directory: join(state.host.root, "custom_integrations"), isBuiltIn: false }, + ]); + cleanups.push(() => shutdownIntegrations()); + const { adminStandaloneIntegrationsRoute } = await import( + "../../routes/admin-standalone-integrations" + ); + await app.register(adminStandaloneIntegrationsRoute, { prefix: "/api" }); + const handlers = await import("./install"); + ({ jobRunner } = await import("../job-runner")); + jobRunner.register("standalone.preview", handlers.handleStandalonePreview); + jobRunner.register("standalone.install", handlers.handleStandaloneInstall); + apiUrl = await app.listen({ host: "127.0.0.1", port: 0 }); + }, 120000); + afterAll(async () => { + for (const cleanup of cleanups.reverse()) await cleanup(); + vi.unstubAllEnvs(); + }); + const post = async (path: string, body: unknown, key: string) => { + const response = await fetch(`${apiUrl}/api/admin/standalone-integrations/${path}`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": key, + "x-openmapx-local-admin": "standalone-local-fixture", + }, + body: JSON.stringify(body), + }); + return { status: response.status, body: (await response.json()) as Record }; + }; + async function completed(jobId: unknown, observe = false) { + const deadline = Date.now() + 60000; + for (;;) { + if (observe) await jobRunner.reconcile(); + const [row] = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, String(jobId))); + if (row && ["success", "failed", "canceled"].includes(row.status)) return row; + if (Date.now() > deadline) + throw Error( + `Standalone job did not finish: ${row?.status} ${row?.error ?? ""} ${JSON.stringify(row?.result)}`, + ); + await delay(50); + } + } + async function upload(id: string) { + const bytes = createDeterministicTarGz([ + { + path: "manifest.json", + contents: Buffer.from( + JSON.stringify({ + id, + name: "Uploaded fixture", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ), + }, + ]); + const digest = createHash("sha256").update(bytes).digest("hex"); + const stage = await store.createStage({ + purpose: "integration-artifact", + direction: "upload", + ownerId: "loopback", + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + declaredBytes: bytes.length, + expectedSha256: digest, + }); + await store.upload({ + stageId: stage.stageId, + ownerId: "loopback", + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 60000, + body: new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }), + }); + return { + stage, + source: { kind: "uploaded-artifact", stageId: stage.stageId, expectedSha256: digest }, + }; + } + it("publishes, actually reloads and atomically records uploaded provenance without an invented URL", async () => { + const { stage, source } = await upload("standalone-http-fixture"); + const previewKey = randomUUID(), + preview = await post("preview", { source }, previewKey); + expect(preview.status, JSON.stringify(preview.body)).toBe(202); + expect(await post("preview", { source }, previewKey)).toEqual(preview); + expect( + ( + await post( + "preview", + { source: { ...source, expectedSha256: "f".repeat(64) } }, + previewKey, + ) + ).status, + ).toBe(409); + const previewJob = await completed(preview.body.jobId); + expect(previewJob.status, previewJob.error ?? JSON.stringify(previewJob.result)).toBe( + "success", + ); + const inspection = previewJob.result as { inspectionId: string; contentDigest: string }; + const key = randomUUID(), + request = { + inspectionId: inspection.inspectionId, + expectedContentDigest: inspection.contentDigest, + }; + const accepted = await post("install", request, key); + expect(accepted.status, JSON.stringify(accepted.body)).toBe(202); + const done = await completed(accepted.body.jobId); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("success"); + expect(await post("install", request, key)).toEqual(accepted); + const [parent] = await state.fixture.db + .select() + .from(schema.installedExtension) + .where(eq(schema.installedExtension.id, `local-${accepted.body.operationId}`)); + expect(parent).toMatchObject({ + installationKind: "standalone", + sourceTrust: "community", + sourceUrl: null, + lastOperationId: accepted.body.operationId, + provenance: { + kind: "uploaded-artifact", + stageId: stage.stageId, + artifactSha256: source.expectedSha256, + contentDigest: inspection.contentDigest, + }, + }); + const [integration] = await state.fixture.db + .select() + .from(schema.installedIntegration) + .where(eq(schema.installedIntegration.id, "standalone-http-fixture")); + expect(integration).toMatchObject({ + repository: null, + managedByExtension: parent!.id, + sourceType: "uploaded-artifact", + }); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "loopback", + purpose: stage.purpose, + }) + ).state, + ).toBe("consumed"); + const { getIntegration } = await import("../../integration-host"); + expect(getIntegration("standalone-http-fixture")?.manifest.version).toBe("1.0.0"); + }, 90000); + it("adopts a lost admission response without publishing a second installation", async () => { + const { stage, source } = await upload("standalone-lost-response"); + const preview = await post("preview", { source }, randomUUID()), + previewJob = await completed(preview.body.jobId); + expect(previewJob.status).toBe("success"); + const inspection = previewJob.result as { inspectionId: string; contentDigest: string }; + dropInstallResponse = true; + const before = installPosts; + const accepted = await post( + "install", + { inspectionId: inspection.inspectionId, expectedContentDigest: inspection.contentDigest }, + randomUUID(), + ); + const done = await completed(accepted.body.jobId); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("success"); + expect(installPosts - before).toBe(1); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "loopback", + purpose: stage.purpose, + }) + ).state, + ).toBe("consumed"); + }, 90000); + it("restores a conclusively rolled-back database install and permits a new inspection of the same stage", async () => { + const { stage, source } = await upload("standalone-rollback"); + const preview = await post("preview", { source }, randomUUID()), + previewJob = await completed(preview.body.jobId); + expect(previewJob.status).toBe("success"); + const inspection = previewJob.result as { inspectionId: string; contentDigest: string }; + await state.fixture + .sql`alter table installed_integration add constraint fixture_reject_standalone check (id <> 'standalone-rollback')`; + const accepted = await post( + "install", + { inspectionId: inspection.inspectionId, expectedContentDigest: inspection.contentDigest }, + randomUUID(), + ); + const done = await completed(accepted.body.jobId); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("failed"); + expect(done.result).toMatchObject({ outcome: "restored" }); + expect( + await state.fixture.db + .select() + .from(schema.installedExtension) + .where(eq(schema.installedExtension.id, `local-${accepted.body.operationId}`)), + ).toEqual([]); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "loopback", + purpose: stage.purpose, + }) + ).state, + ).toBe("ready"); + const { getIntegration } = await import("../../integration-host"); + expect(getIntegration("standalone-rollback")).toBeUndefined(); + await state.fixture + .sql`alter table installed_integration drop constraint fixture_reject_standalone`; + const retry = await post("preview", { source }, randomUUID()), + retryJob = await completed(retry.body.jobId); + expect(retryJob.status, retryJob.error ?? JSON.stringify(retryJob.result)).toBe("success"); + }, 90000); + + it("keeps a published cancellation pending until exact restoration and stage release", async () => { + const { stage, source } = await upload("standalone-cancel"); + const preview = await post("preview", { source }, randomUUID()), + previewJob = await completed(preview.body.jobId); + expect(previewJob.status).toBe("success"); + const inspection = previewJob.result as { inspectionId: string; contentDigest: string }; + cancelIntegrationId = "standalone-cancel"; + const accepted = await post( + "install", + { inspectionId: inspection.inspectionId, expectedContentDigest: inspection.contentDigest }, + randomUUID(), + ); + for (let i = 0; i < 200 && cancelIntegrationId; i++) await delay(25); + expect(cancelIntegrationId).toBeNull(); + await delay(1000); + const [pending] = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, String(accepted.body.jobId))); + expect(pending).toMatchObject({ status: "cancel_pending", finishedAt: null }); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "loopback", + purpose: stage.purpose, + }) + ).state, + ).toBe("claimed"); + // Real ownership lease expires; no fake clock or forced owner replacement. + await delay(61000); + await jobRunner.reconcile(); + const [restored] = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, String(accepted.body.jobId))); + const { hasVerifiedStandaloneRestoration } = await import("./settlement-checkpoint"); + expect(await hasVerifiedStandaloneRestoration(restored!.result, restored!.id)).toBe(true); + const forged = structuredClone(restored!.result) as { _standalone: { mac: string } }; + forged._standalone.mac = "0".repeat(64); + expect(await hasVerifiedStandaloneRestoration(forged, restored!.id)).toBe(false); + const done = await completed(accepted.body.jobId, true); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("canceled"); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "loopback", + purpose: stage.purpose, + }) + ).state, + ).toBe("ready"); + expect( + await state.fixture.db + .select() + .from(schema.installedIntegration) + .where(eq(schema.installedIntegration.id, "standalone-cancel")), + ).toEqual([]); + }, 150000); + }, +); diff --git a/apps/api/src/services/standalone-integrations/install.ts b/apps/api/src/services/standalone-integrations/install.ts new file mode 100644 index 000000000..660a93341 --- /dev/null +++ b/apps/api/src/services/standalone-integrations/install.ts @@ -0,0 +1,444 @@ +import { createHmac } from "node:crypto"; +import { + standaloneInstallRequestSchema, + standalonePreviewRequestSchema, +} from "@openmapx/core/api-contracts"; +import { + canonicalJson, + readOpsTokenFile, + type StandaloneIntegrationProvenance, + standaloneInstallInspectionSchema, + standaloneInstallSourceSchema, + standaloneIntegrationProvenanceSchema, +} from "@openmapx/core/ops"; +import { and, eq, sql } from "drizzle-orm"; +import z from "zod/v4"; +import { db } from "../../db"; +import { + adminJob, + installedExtension, + installedExtensionComponent, + installedIntegration, +} from "../../db/schema"; +import { reloadIntegrations } from "../../integration-host"; +import { dbActorId } from "../../utils/actor"; +import { AdminJobFailure, JobObservationPending } from "../admin-job-failure"; +import { executeAdminJobOperation } from "../admin-job-ops"; +import { readProjection } from "../admin-job-projection"; +import { withExtensionChangeLock } from "../extension-recovery"; +import type { JobContext } from "../job-runner"; + +type State = import("./state").StandaloneState; + +import { readStandaloneState, sealStandaloneState } from "./state"; + +async function token() { + return readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); +} +async function authenticatePayload(ctx: JobContext) { + const payload = ctx.payload; + if ( + typeof payload.ownerId !== "string" || + payload.ownerId !== payload._standaloneActor || + typeof payload.operationId !== "string" || + !z.uuid().safeParse(payload.operationId).success + ) + throw new JobObservationPending("recovery_required"); + const type = ctx.jobType; + const body = + type === "standalone.preview" + ? standalonePreviewRequestSchema.parse({ source: payload.source }) + : standaloneInstallRequestSchema.parse({ + inspectionId: payload.inspectionId, + expectedContentDigest: payload.expectedContentDigest, + }); + const expected = createHmac("sha256", await token()) + .update("openmapx-standalone-request-v1\0") + .update(canonicalJson({ type, payload: body, actorId: payload.ownerId })) + .digest("hex"); + if (payload._requestDigest !== expected) throw new JobObservationPending("recovery_required"); + return { ownerId: payload.ownerId, operationId: payload.operationId, body }; +} +async function save(ctx: JobContext, state: State) { + const sealed = await sealStandaloneState(state); + await ctx.checkpoint({ + ...ctx.checkpointResult, + _standalone: sealed, + _standaloneSettled: state.phase === "complete", + }); +} +async function restore(ctx: JobContext): Promise { + const value = ctx.checkpointResult?._standalone; + if (value === undefined) return null; + const state = await readStandaloneState(value); + const payload = await authenticatePayload(ctx); + if ( + state.jobId !== ctx.jobId || + state.operationId !== payload.operationId || + state.ownerId !== payload.ownerId || + canonicalJson(state.request) !== canonicalJson(payload.body) + ) + throw new JobObservationPending("recovery_required"); + return state; +} +async function cancellationRequested(ctx: JobContext) { + const [row] = await db + .select({ status: adminJob.status }) + .from(adminJob) + .where(eq(adminJob.id, ctx.jobId)); + if (!row) throw new JobObservationPending("recovery_required"); + return row.status === "cancel_pending" || row.status === "canceled"; +} +function provenance(state: State): StandaloneIntegrationProvenance { + const inspection = state.inspection, + source = state.source; + return standaloneIntegrationProvenanceSchema.parse( + source.kind === "source" + ? { + kind: source.kind, + url: source.repository, + commit: inspection.commit, + contentDigest: inspection.contentDigest, + } + : source.kind === "artifact" + ? { + kind: source.kind, + url: source.url, + artifactSha256: inspection.archiveSha256, + contentDigest: inspection.contentDigest, + } + : { + kind: source.kind, + stageId: source.stageId, + artifactSha256: inspection.archiveSha256, + contentDigest: inspection.contentDigest, + }, + ); +} +async function previewForInstall(ctx: JobContext, ownerId: string, inspectionId: string) { + const rows = await db + .select() + .from(adminJob) + .where( + and( + eq(adminJob.type, "standalone.preview"), + eq(adminJob.status, "success"), + sql`${adminJob.payload}->>'ownerId' = ${ownerId}`, + sql`${adminJob.result}->>'inspectionId' = ${inspectionId}`, + ), + ) + .limit(2); + if (rows.length !== 1) + throw new AdminJobFailure("Standalone preview is missing or belongs to another administrator", { + operationId: ctx.payload.operationId, + }); + const row = rows[0]!; + await authenticatePayload({ + ...ctx, + jobId: row.id, + jobType: row.type, + payload: row.payload as Record, + }); + return { + source: standaloneInstallSourceSchema.parse((row.payload as { source: unknown }).source), + inspection: standaloneInstallInspectionSchema.parse( + Object.fromEntries( + Object.entries(row.result as Record).filter(([key]) => + Object.hasOwn(standaloneInstallInspectionSchema.shape, key), + ), + ), + ), + }; +} +async function ownership(integrationId: string) { + const [row] = await db + .select({ parent: installedExtension }) + .from(installedExtensionComponent) + .innerJoin( + installedExtension, + eq(installedExtension.id, installedExtensionComponent.extensionId), + ) + .where( + and( + eq(installedExtensionComponent.kind, "integration"), + eq(installedExtensionComponent.componentId, integrationId), + ), + ) + .limit(1); + const [integration] = await db + .select() + .from(installedIntegration) + .where(eq(installedIntegration.id, integrationId)) + .limit(1); + if (row && row.parent.installationKind !== "standalone") + throw new AdminJobFailure("Integration belongs to a catalog extension", { integrationId }); + if (integration?.managedByExtension && integration.managedByExtension !== row?.parent.id) + throw new JobObservationPending("recovery_required"); + return row?.parent.id ?? null; +} +async function bookkeeping( + ctx: JobContext, + state: State, + write: boolean, +): Promise<"committed" | "absent"> { + return db.transaction(async (tx) => { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${"standalone-bookkeeping:" + state.inspection.integrationId},0))`, + ); + const [parent] = await tx + .select() + .from(installedExtension) + .where(eq(installedExtension.id, state.parentId)) + .for("update"); + const [integration] = await tx + .select() + .from(installedIntegration) + .where(eq(installedIntegration.id, state.inspection.integrationId)) + .for("update"); + const [component] = await tx + .select() + .from(installedExtensionComponent) + .where( + and( + eq(installedExtensionComponent.kind, "integration"), + eq(installedExtensionComponent.componentId, state.inspection.integrationId), + ), + ) + .for("update"); + const source = provenance(state); + if (parent?.lastOperationId === state.operationId) { + if ( + parent.installationKind !== "standalone" || + parent.sourceTrust !== "community" || + canonicalJson(parent.provenance) !== canonicalJson(source) || + integration?.managedByExtension !== state.parentId || + component?.extensionId !== state.parentId || + integration.installedVersion !== state.inspection.version + ) + throw new JobObservationPending("recovery_required"); + return "committed"; + } + if (!write) return "absent"; + if (component && component.extensionId !== state.parentId) + throw new AdminJobFailure("Integration ownership changed during installation", { + operationId: state.operationId, + }); + if (parent && parent.installationKind !== "standalone") + throw new AdminJobFailure("Catalog extensions cannot be replaced by a standalone install", { + operationId: state.operationId, + }); + const now = new Date(), + repository = source.kind === "uploaded-artifact" ? null : source.url; + await tx + .insert(installedExtension) + .values({ + id: state.parentId, + name: state.inspection.integrationId, + sourceUrl: repository, + sourceTrust: "community", + installationKind: "standalone", + provenance: source, + lastOperationId: state.operationId, + installedVersion: state.inspection.version, + manifest: null, + installedBy: dbActorId(state.ownerId), + updatedAt: now, + }) + .onConflictDoUpdate({ + target: installedExtension.id, + set: { + sourceUrl: repository, + sourceTrust: "community", + installationKind: "standalone", + provenance: source, + lastOperationId: state.operationId, + installedVersion: state.inspection.version, + updatedAt: now, + }, + }); + await tx + .insert(installedIntegration) + .values({ + id: state.inspection.integrationId, + repository, + sourceType: source.kind, + installedVersion: state.inspection.version, + installedBy: dbActorId(state.ownerId), + managedByExtension: state.parentId, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: installedIntegration.id, + set: { + repository, + sourceType: source.kind, + installedVersion: state.inspection.version, + managedByExtension: state.parentId, + updatedAt: now, + }, + }); + if (!component) + await tx.insert(installedExtensionComponent).values({ + extensionId: state.parentId, + kind: "integration", + componentId: state.inspection.integrationId, + }); + return "committed"; + }); +} +export async function handleStandalonePreview(ctx: JobContext): Promise> { + const { ownerId, body } = await authenticatePayload(ctx), + input = standalonePreviewRequestSchema.parse(body); + return executeAdminJobOperation( + ctx, + { kind: "integration.inspectInstall", source: input.source, ownerId }, + "standalone.preview", + ); +} +export function handleStandaloneInstall(ctx: JobContext): Promise> { + return withExtensionChangeLock(async () => { + const payload = await authenticatePayload(ctx), + request = standaloneInstallRequestSchema.parse(payload.body); + let state = await restore(ctx); + if (!state) { + if (ctx.observationOnly) throw new JobObservationPending("recovery_required"); + ctx.signal.throwIfAborted(); + const preview = await previewForInstall(ctx, payload.ownerId, request.inspectionId); + if ( + preview.inspection.contentDigest !== request.expectedContentDigest || + Date.parse(preview.inspection.expiresAt) <= Date.now() + ) + throw new AdminJobFailure("Preview expired or changed; inspect the source again", { + operationId: payload.operationId, + }); + state = { + version: 1, + jobId: ctx.jobId, + operationId: payload.operationId, + ownerId: payload.ownerId, + request, + ...preview, + parentId: + (await ownership(preview.inspection.integrationId)) ?? `local-${payload.operationId}`, + phase: "prepared", + outcome: null, + }; + await save(ctx, state); + } + if (state.phase === "complete") + return { + operationId: state.operationId, + integrationId: state.inspection.integrationId, + outcome: state.outcome, + }; + if ( + state.phase === "prepared" && + (await cancellationRequested(ctx)) && + !Object.values(readProjection(ctx.checkpointResult).operations).some( + (operation) => operation.kind === "integration.install", + ) + ) { + state.phase = "complete"; + state.outcome = "restored"; + await save(ctx, state); + throw new AdminJobFailure("Standalone installation canceled before publication", { + operationId: state.operationId, + outcome: "not_started", + }); + } + const install = { + kind: "integration.install" as const, + inspectionId: state.request.inspectionId, + expectedContentDigest: state.request.expectedContentDigest, + operationId: state.operationId, + ownerId: state.ownerId, + }; + // The retained operation helper adopts exact attempted addresses; it never replays an uncertain POST. + await executeAdminJobOperation(ctx, install, "standalone.publish"); + if (state.phase === "prepared") { + state.phase = "published"; + await save(ctx, state); + } + if (state.phase === "published") { + if (await cancellationRequested(ctx)) { + state.phase = "restore"; + await save(ctx, state); + } else { + state.phase = "reload_attempted"; + await save(ctx, state); + try { + await reloadIntegrations(); + } catch { + state.phase = "restore"; + await save(ctx, state); + } + if (state.phase === "reload_attempted") { + state.phase = "reloaded"; + await save(ctx, state); + } + } + } + if (state.phase === "reload_attempted") throw new JobObservationPending("recovery_required"); + if (state.phase === "reloaded") { + if (await cancellationRequested(ctx)) { + state.phase = "restore"; + await save(ctx, state); + } else { + state.phase = "persist_attempted"; + await save(ctx, state); + } + } + if (state.phase === "persist_attempted") { + try { + const outcome = await bookkeeping(ctx, state, !(await cancellationRequested(ctx))); + state.phase = outcome === "committed" ? "persisted" : "restore"; + await save(ctx, state); + } catch { + // Serialize behind the original SQL transaction before interpreting its + // acknowledgement loss. A missing row while that transaction is live is + // never rollback evidence. + try { + state.phase = + (await bookkeeping(ctx, state, false)) === "committed" ? "persisted" : "restore"; + await save(ctx, state); + } catch { + throw new JobObservationPending("recovery_required"); + } + } + } + const decision = state.phase === "persisted" ? "commit" : "restore"; + if (decision === "commit" && (await bookkeeping(ctx, state, false)) !== "committed") + throw new JobObservationPending("recovery_required"); + const settled = await executeAdminJobOperation( + ctx, + { + kind: "integration.settle", + transactionId: state.operationId, + decision, + expectedContentDigest: state.inspection.contentDigest, + }, + `standalone.${decision}`, + ); + if (decision === "restore") { + try { + await reloadIntegrations(); + } catch { + throw new JobObservationPending("recovery_required"); + } + } + state.phase = "complete"; + state.outcome = settled.outcome; + await save(ctx, state); + const result = { + operationId: state.operationId, + integrationId: state.inspection.integrationId, + outcome: state.outcome, + }; + if (state.outcome === "restored") + throw new AdminJobFailure( + "Standalone integration was restored; installation did not commit", + result, + ); + return result; + }); +} diff --git a/apps/api/src/services/standalone-integrations/settlement-checkpoint.ts b/apps/api/src/services/standalone-integrations/settlement-checkpoint.ts new file mode 100644 index 000000000..2c949cb8e --- /dev/null +++ b/apps/api/src/services/standalone-integrations/settlement-checkpoint.ts @@ -0,0 +1,164 @@ +import { type OpsOperation, standaloneInstallReceiptSchema } from "@openmapx/core/ops"; +import { and, eq, sql } from "drizzle-orm"; +import type { db } from "../../db"; +import { + type adminJob, + installedExtension, + installedExtensionComponent, + installedIntegration, +} from "../../db/schema"; +import { readProjection } from "../admin-job-projection"; +import { JobOwnershipLost } from "../admin-job-state"; +import { readStandaloneState } from "./state"; +/** A cancellation can finalize only its own published component. No forward install admission. */ +export async function assertStandaloneSettlementCheckpoint( + row: typeof adminJob.$inferSelect, + operation: OpsOperation, + connection: Pick, +): Promise { + if (row.type !== "standalone.install" || operation.kind !== "integration.settle") + throw new JobOwnershipLost(); + const current = row.result as Record | null, + payload = row.payload as Record | null; + const state = await readStandaloneState(current?._standalone); + if ( + state.jobId !== row.id || + state.operationId !== payload?.operationId || + state.ownerId !== payload.ownerId || + state.ownerId !== payload._standaloneActor || + state.request.inspectionId !== payload.inspectionId || + state.request.expectedContentDigest !== payload.expectedContentDigest || + operation.transactionId !== state.operationId || + operation.expectedContentDigest !== state.inspection.contentDigest || + state.request.expectedContentDigest !== state.inspection.contentDigest || + state.request.inspectionId !== state.inspection.inspectionId + ) + throw new JobOwnershipLost(); + const original = Object.values(readProjection(current).operations).filter( + (entry) => + entry.operation?.kind === "integration.install" && + entry.operation.operationId === state.operationId, + ); + if (original.length !== 1) throw new JobOwnershipLost(); + const source = original[0]!, + body = source.operation, + delivery = source.receiptDelivery; + if ( + source.attempt?.apiJobId !== row.id || + body?.kind !== "integration.install" || + body.ownerId !== state.ownerId || + body.inspectionId !== state.request.inspectionId || + body.expectedContentDigest !== state.request.expectedContentDigest || + !source.terminal || + delivery?.state !== "confirmed" || + delivery.terminalReceipt.state !== "succeeded" + ) + throw new JobOwnershipLost(); + if ( + delivery.terminalReceipt.operationId !== source.operationId || + delivery.terminalReceipt.operationKey !== source.operationKey || + delivery.terminalReceipt.fingerprint !== source.fingerprint + ) + throw new JobOwnershipLost(); + const published = standaloneInstallReceiptSchema.parse(delivery.terminalReceipt.result); + if ( + published.transactionId !== state.operationId || + published.integrationId !== state.inspection.integrationId || + published.contentDigest !== state.inspection.contentDigest + ) + throw new JobOwnershipLost(); + await connection.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${"standalone-bookkeeping:" + state.inspection.integrationId},0))`, + ); + const [parent] = await connection + .select() + .from(installedExtension) + .where(eq(installedExtension.id, state.parentId)) + .for("update"); + const [integration] = await connection + .select() + .from(installedIntegration) + .where(eq(installedIntegration.id, state.inspection.integrationId)) + .for("update"); + const [component] = await connection + .select() + .from(installedExtensionComponent) + .where( + and( + eq(installedExtensionComponent.kind, "integration"), + eq(installedExtensionComponent.componentId, state.inspection.integrationId), + ), + ) + .for("update"); + if (operation.decision === "commit") { + if ( + state.phase !== "persisted" || + parent?.lastOperationId !== state.operationId || + parent.installationKind !== "standalone" || + parent.sourceTrust !== "community" || + parent.provenance?.contentDigest !== state.inspection.contentDigest || + integration?.managedByExtension !== state.parentId || + integration.installedVersion !== state.inspection.version || + component?.extensionId !== state.parentId + ) + throw new JobOwnershipLost(); + } else if ( + state.phase !== "restore" || + parent?.lastOperationId === state.operationId || + (component && component.extensionId !== state.parentId) || + (integration?.managedByExtension && integration.managedByExtension !== state.parentId) + ) + throw new JobOwnershipLost(); + // The unchanged parent/body/address checks in the caller and actual signed + // agent transaction independently enforce the original lifetime and ownership. +} + +/** Whole-job cancellation is supported by the original successful restore leaf, + * not by the generic fact that every child returned successfully. */ +export async function hasVerifiedStandaloneRestoration( + result: unknown, + jobId: string | undefined, +): Promise { + try { + if (!jobId || !result || typeof result !== "object") return false; + const value = result as Record; + if ( + value._standaloneSettled !== true || + typeof value._cancelRequestedAt !== "string" || + !Number.isFinite(Date.parse(value._cancelRequestedAt)) + ) + return false; + const state = await readStandaloneState(value._standalone); + if (state.jobId !== jobId || state.phase !== "complete" || state.outcome !== "restored") + return false; + const operations = Object.values(readProjection(value).operations); + if ( + operations.some( + (operation) => !operation.terminal || operation.receiptDelivery?.state !== "confirmed", + ) + ) + return false; + const restore = operations.filter( + (operation) => + operation.operation?.kind === "integration.settle" && + operation.operation.transactionId === state.operationId, + ); + if (restore.length !== 1) return false; + const operation = restore[0]!, + body = operation.operation, + receipt = operation.receiptDelivery!.terminalReceipt; + return ( + body?.kind === "integration.settle" && + body.decision === "restore" && + body.expectedContentDigest === state.inspection.contentDigest && + receipt.state === "succeeded" && + receipt.kind === "integration.settle" && + receipt.operationKey === operation.operationKey && + receipt.fingerprint === operation.fingerprint && + (receipt.result as { outcome?: unknown; transactionId?: unknown })?.outcome === "restored" && + (receipt.result as { transactionId?: unknown }).transactionId === state.operationId + ); + } catch { + return false; + } +} diff --git a/apps/api/src/services/standalone-integrations/state.ts b/apps/api/src/services/standalone-integrations/state.ts new file mode 100644 index 000000000..6c303d492 --- /dev/null +++ b/apps/api/src/services/standalone-integrations/state.ts @@ -0,0 +1,54 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { standaloneInstallRequestSchema } from "@openmapx/core/api-contracts"; +import { + canonicalJson, + readOpsTokenFile, + standaloneInstallInspectionSchema, + standaloneInstallSourceSchema, +} from "@openmapx/core/ops"; +import z from "zod/v4"; +export const standaloneStateSchema = z.strictObject({ + version: z.literal(1), + jobId: z.uuid(), + operationId: z.uuid(), + ownerId: z.string().min(1).max(128), + request: standaloneInstallRequestSchema, + source: standaloneInstallSourceSchema, + inspection: standaloneInstallInspectionSchema, + parentId: z.string().regex(/^local-[a-f0-9-]{36}$/), + phase: z.enum([ + "prepared", + "published", + "reload_attempted", + "reloaded", + "persist_attempted", + "persisted", + "restore", + "complete", + ]), + outcome: z.enum(["committed", "restored"]).nullable(), +}); +export type StandaloneState = z.infer; + +export async function sealStandaloneState(state: StandaloneState) { + const body = canonicalJson(standaloneStateSchema.parse(state)); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const mac = createHmac("sha256", token) + .update("openmapx-standalone-install-state-v1\0") + .update(body) + .digest("hex"); + return { body, mac }; +} +export async function readStandaloneState(value: unknown): Promise { + const sealed = z + .strictObject({ body: z.string().max(16384), mac: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse(value); + const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const expected = createHmac("sha256", token) + .update("openmapx-standalone-install-state-v1\0") + .update(sealed.body) + .digest(); + if (!timingSafeEqual(expected, Buffer.from(sealed.mac, "hex"))) + throw Error("Standalone checkpoint authentication failed"); + return standaloneStateSchema.parse(JSON.parse(sealed.body)); +} diff --git a/apps/ops-agent/package.json b/apps/ops-agent/package.json index 2ca7e1bbc..06d9aa885 100644 --- a/apps/ops-agent/package.json +++ b/apps/ops-agent/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@openmapx/core": "workspace:^", + "@openmapx/integration-framework": "workspace:^", "@openmapx/transitous-core": "workspace:^", "fastify": "^5.12.7", "zod": "^4.6.5" diff --git a/apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts b/apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts new file mode 100644 index 000000000..ece43b821 --- /dev/null +++ b/apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts @@ -0,0 +1,225 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + createControlRootOperationKey, + normalizeExecutionAdmission, + type OpsOperation, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { + createIntegrationArtifactPreparation, + describePreparedIntegration, + publishPreparedIntegration, + settleIntegrationPublication, +} from "@openmapx/integration-framework/installer"; +import { afterEach, expect, it } from "vitest"; +import { createDeterministicTarGz } from "../../../../packages/integration-framework/src/deterministic-tar"; +import type { OpsExecutionContext } from "../runtime"; +import { ArtifactTransferStore } from "./store"; + +const cleanup: (() => void)[] = []; +afterEach(() => { + cleanup + .reverse() + .splice(0) + .forEach((fn) => { + fn(); + }); +}); +it("validates real uploaded bytes only under the exact retained integration execution", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-stage-consumer-")); + cleanup.push(() => rmSync(root, { recursive: true, force: true })); + const infra = join(root, "infra"); + mkdirSync(join(infra, "data"), { recursive: true }); + mkdirSync(join(root, "services")); + await control.provisionControlInstallation(infra); + const installation = control.openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["app-api"], + }); + cleanup.push(() => control.closeControlInstallation(installation)); + const store = new ArtifactTransferStore({ + dataRoot: join(root, "stages"), + privateRoot: join(root, "stage-meta"), + token: "fixture-test-token", + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), + }); + cleanup.push(() => store.close()); + const bytes = createDeterministicTarGz([ + { + path: "manifest.json", + contents: Buffer.from( + JSON.stringify({ + id: "upload-fixture", + name: "Uploaded", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ), + }, + ]); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const stage = await store.createStage({ + purpose: "integration-artifact", + direction: "upload", + ownerId: "fixture-admin", + declaredBytes: bytes.length, + expectedSha256: sha256, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + }); + await store.upload({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 60000, + body: new ReadableStream({ + start(c) { + c.enqueue(bytes); + c.close(); + }, + }), + }); + const operation: OpsOperation = { + kind: "integration.inspectInstall", + ownerId: "fixture-admin", + source: { kind: "uploaded-artifact", stageId: stage.stageId, expectedSha256: sha256 }, + }; + const makeContext = (operation: OpsOperation): OpsExecutionContext => { + const admittedAt = Date.now(), + operationFingerprint = opsOperationFingerprint(operation), + operationKey = createControlRootOperationKey({ + notAfter: admittedAt + 600000, + nonce: randomBytes(16), + }), + admission = normalizeExecutionAdmission({ + operationFingerprint, + operationKey, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: admittedAt + 600000, + }); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey, + operationFingerprint, + deadlineAt: admission.deadlineAt, + fingerprint: admission.fingerprint, + }; + const authority = control.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: admittedAt + 600000, + serviceIds: ["app-api"], + }); + return { + execution, + controlAuthority: authority, + signal: new AbortController().signal, + emitLog: () => {}, + claim: { + fingerprint: operationFingerprint, + operation, + source: "registry", + capability: { revisionId: "fixture", values: {} }, + }, + }; + }; + const context = makeContext(operation); + const preparedContext = createIntegrationArtifactPreparation({ rootDir: root }); + const result = await store.validateIntegrationArtifact(context, preparedContext); + expect(result.inspection).toMatchObject({ stageId: stage.stageId, sha256, bytes: bytes.length }); + expect(describePreparedIntegration(result.prepared)).toMatchObject({ + id: "upload-fixture", + archiveSha256: sha256, + }); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + }) + ).state, + ).toBe("ready"); + expect( + await store.inspectClaim({ + stageId: stage.stageId, + ownerId: "fixture-admin", + identity: result.claim.identity, + }), + ).toMatchObject({ settlement: { outcome: "validated", inspection: result.inspection } }); + await expect(store.claimStage({ ...result.claim, expectedSha256: sha256 })).rejects.toMatchObject( + { statusCode: 409 }, + ); + await expect( + store.validateIntegrationArtifact( + { ...context, controlAuthority: undefined }, + createIntegrationArtifactPreparation({ rootDir: root }), + ), + ).rejects.toThrow(/authority|execution/i); + const transactionId = randomUUID(); + const installing = makeContext({ + kind: "integration.install", + inspectionId: result.inspection.inspectionId, + ownerId: "fixture-admin", + expectedContentDigest: result.prepared.contentDigest, + operationId: transactionId, + }); + const installClaim = await store.claimIntegrationArtifactInstall(installing, result.inspection); + await expect( + store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), + ).rejects.toMatchObject({ statusCode: 409 }); + const publication = await publishPreparedIntegration(result.prepared, { + journal: store.standalonePublicationJournal(installing), + transactionId, + expectedPreviousContentDigest: null, + signal: installing.signal, + }); + const settling = makeContext({ + kind: "integration.settle", + transactionId, + decision: "commit", + expectedContentDigest: result.prepared.contentDigest, + }); + await expect( + store.settleIntegrationArtifact(settling, installClaim, publication), + ).rejects.toThrow(/settled|terminal|committed/); + await settleIntegrationPublication(publication, "commit", settling.signal); + expect((await store.settleIntegrationArtifact(settling, installClaim, publication)).state).toBe( + "consumed", + ); + expect((await store.settleIntegrationArtifact(settling, installClaim, publication)).state).toBe( + "consumed", + ); + expect( + await store.inspectClaim({ + stageId: stage.stageId, + ownerId: "fixture-admin", + identity: installClaim.identity, + }), + ).toMatchObject({ settlement: { outcome: "consumed", receiptId: transactionId } }); + await expect( + store.claimIntegrationArtifactInstall( + makeContext({ + ...installing.claim.operation, + kind: "integration.install", + inspectionId: result.inspection.inspectionId, + ownerId: "fixture-admin", + expectedContentDigest: result.prepared.contentDigest, + operationId: randomUUID(), + }), + result.inspection, + ), + ).rejects.toMatchObject({ statusCode: 409 }); +}); diff --git a/apps/ops-agent/src/artifact-transfer/store.ts b/apps/ops-agent/src/artifact-transfer/store.ts index 7676f3a5d..3da82cc0d 100644 --- a/apps/ops-agent/src/artifact-transfer/store.ts +++ b/apps/ops-agent/src/artifact-transfer/store.ts @@ -13,20 +13,46 @@ import { import { type FileHandle, open } from "node:fs/promises"; import { join } from "node:path"; import { Readable } from "node:stream"; -import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; +import { AnchoredDirectory, executionAuthorityState } from "@openmapx/core/build-artifacts"; import { type ArtifactStageClaim, + artifactStageClaimSchema, type ClaimTransferStage, type CreateTransferStage, canonicalJson, createTransferStageSchema, + opsOperationFingerprint, type StageClaimIdentity, type StageClaimSettlement, + type StageInspectionBinding, + stageClaimSettlementSchema, + stageInspectionBindingSchema, type TransferPurpose, type TransferStage, transferStageSchema, } from "@openmapx/core/ops"; +import { + assertIntegrationPublicationJournal, + closeIntegrationPublicationJournal, + createIntegrationPublicationJournal, + describeIntegrationPreparationRoot, + describeIntegrationPublication, + describePreparedIntegration, + discardIntegrationPreparation, + type IntegrationArtifactPreparationContext, + type IntegrationPublication, + type IntegrationPublicationJournal, + prepareIntegrationArtifact, + restorePreparedIntegration, +} from "@openmapx/integration-framework/installer"; import z from "zod/v4"; +import type { OpsExecutionContext } from "../runtime"; +import { + type StandaloneInspectionRecord, + type StandaloneTransactionRecord, + standaloneInspectionRecordSchema, + standaloneTransactionRecordSchema, +} from "../standalone-integration-records"; import { ArtifactTransferError } from "./errors"; import { type ArtifactTransferPolicy, @@ -36,6 +62,12 @@ import { } from "./policy"; const id = z.uuid(); +const integrationClaimRecordSchema = z.strictObject({ + claim: artifactStageClaimSchema, + settlement: stageClaimSettlementSchema.nullable(), + preparation: z.record(z.string(), z.unknown()).nullable(), + transactionId: z.uuid().nullable(), +}); const metaSchema = z.strictObject({ stage: transferStageSchema, ownerId: z.string().min(1).max(128), @@ -58,6 +90,8 @@ const metaSchema = z.strictObject({ ctimeNs: z.string(), }) .nullable(), + integrationInspection: stageInspectionBindingSchema.nullable().default(null), + claims: z.array(integrationClaimRecordSchema).max(8).default([]), attempts: z .array( z.strictObject({ @@ -97,6 +131,8 @@ export class ArtifactTransferStore { private readonly key: Buffer; private readonly data: AnchoredDirectory; private readonly records: AnchoredDirectory; + private readonly integrationRecords: AnchoredDirectory; + private readonly integrationJournals = new Map(); private readonly streams = new Map(); private readonly capacity: () => { bytes: number; inodes: number }; private readonly now: () => number; @@ -120,6 +156,7 @@ export class ArtifactTransferStore { mkdirSync(options.privateRoot, { recursive: true, mode: 0o700 }); this.data = new AnchoredDirectory(options.dataRoot, undefined, true); this.records = new AnchoredDirectory(options.privateRoot, undefined, true); + this.integrationRecords = this.records.directory("integrations", true, true); this.capacity = options.capacity ?? (() => { @@ -137,6 +174,10 @@ export class ArtifactTransferStore { } close() { if (this.streams.size) throw Error("Transfer streams still owned"); + for (const journal of this.integrationJournals.values()) + closeIntegrationPublicationJournal(journal); + this.integrationJournals.clear(); + this.integrationRecords.close(); this.records.close(); this.data.close(); this.key.fill(0); @@ -294,6 +335,8 @@ export class ArtifactTransferStore { directory: { dev: String(directory.stat.dev), ino: String(directory.stat.ino) }, content: null, attempts: [], + integrationInspection: null, + claims: [], }); } finally { directory.close(); @@ -645,21 +688,507 @@ export class ArtifactTransferStore { } } } - async claimStage(input: ClaimTransferStage): Promise { - this.admit(); + private integrationContext(context: OpsExecutionContext) { + if (!context.controlAuthority) throw Error("Integration execution authority is required"); + const authority = executionAuthorityState(context.controlAuthority); + const operation = context.claim.operation; + if ( + authority.installation.installation.role !== "ops-agent" || + authority.execution.principal !== "api" || + canonicalJson(authority.execution) !== canonicalJson(context.execution) || + authority.kind !== operation.kind || + opsOperationFingerprint(operation) !== authority.execution.operationFingerprint || + !["integration.inspectInstall", "integration.install", "integration.settle"].includes( + operation.kind, + ) + ) + throw Error("Integration execution authority does not match the retained operation"); + return operation; + } + standaloneInstallationId(context: OpsExecutionContext): string { + this.integrationContext(context); + return executionAuthorityState(context.controlAuthority!).installation.installation + .installationId; + } + private readIntegrationRecord(name: string, schema: z.ZodType): T { + const outer = z + .strictObject({ body: z.string().max(65536), mac: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse(JSON.parse(this.integrationRecords.read(name, 131072).toString("utf8"))); + const expected = createHmac("sha256", this.key) + .update("openmapx-standalone-record-v1\0") + .update(name) + .update("\0") + .update(outer.body) + .digest(); + if (!timingSafeEqual(expected, Buffer.from(outer.mac, "hex"))) + throw Error("Standalone integration record authentication failed"); + return schema.parse(JSON.parse(outer.body)); + } + private writeIntegrationRecord(name: string, value: unknown, exclusive: boolean): void { + if (exclusive && this.integrationRecords.listBounded(2049).length >= 2048) + failed(507, "Integration purpose journal capacity reached"); + const body = canonicalJson(value); + if (Buffer.byteLength(body) > 65536) failed(413, "Standalone integration record is too large"); + this.integrationRecords.write( + name, + JSON.stringify({ + body, + mac: createHmac("sha256", this.key) + .update("openmapx-standalone-record-v1\0") + .update(name) + .update("\0") + .update(body) + .digest("hex"), + }), + exclusive, + ); + } + readStandaloneInspection( + context: OpsExecutionContext, + inspectionId: string, + ): StandaloneInspectionRecord { + const installationId = this.standaloneInstallationId(context), + operation = this.integrationContext(context); + if (operation.kind !== "integration.install" || operation.inspectionId !== inspectionId) + throw Error("Standalone inspection read execution mismatch"); + return this.lock(() => { + const record = this.readIntegrationRecord( + `inspection-${id.parse(inspectionId)}.json`, + standaloneInspectionRecordSchema, + ); + if (record.installationId !== installationId || record.ownerId !== operation.ownerId) + throw Error("Standalone inspection owner mismatch"); + return record; + }); + } + retainStandaloneInspection( + context: OpsExecutionContext, + input: StandaloneInspectionRecord, + ): void { + const installationId = this.standaloneInstallationId(context), + operation = this.integrationContext(context), + record = standaloneInspectionRecordSchema.parse(input); + if ( + operation.kind !== "integration.inspectInstall" || + record.installationId !== installationId || + record.ownerId !== operation.ownerId || + canonicalJson(record.source) !== canonicalJson(operation.source) + ) + throw Error("Standalone inspection producer mismatch"); + this.lock(() => + this.writeIntegrationRecord( + `inspection-${record.inspection.inspectionId}.json`, + record, + true, + ), + ); + } + readStandaloneTransaction( + context: OpsExecutionContext, + transactionId: string, + ): StandaloneTransactionRecord { + const installationId = this.standaloneInstallationId(context), + operation = this.integrationContext(context); + if ( + (operation.kind !== "integration.install" || operation.operationId !== transactionId) && + (operation.kind !== "integration.settle" || operation.transactionId !== transactionId) + ) + throw Error("Standalone transaction read execution mismatch"); + return this.lock(() => { + const record = this.readIntegrationRecord( + `transaction-${id.parse(transactionId)}.json`, + standaloneTransactionRecordSchema, + ); + if ( + record.installationId !== installationId || + record.inspection.installationId !== installationId || + (operation.kind === "integration.install" && record.ownerId !== operation.ownerId) + ) + throw Error("Standalone transaction ownership mismatch"); + return record; + }); + } + retainStandaloneTransaction( + context: OpsExecutionContext, + input: StandaloneTransactionRecord, + publication?: IntegrationPublication, + ): void { + const installationId = this.standaloneInstallationId(context), + operation = this.integrationContext(context), + record = standaloneTransactionRecordSchema.parse(input); + if ( + record.installationId !== installationId || + record.inspection.installationId !== installationId + ) + throw Error("Standalone transaction installation mismatch"); + if (operation.kind === "integration.install") { + if ( + operation.operationId !== record.transactionId || + operation.ownerId !== record.ownerId || + operation.inspectionId !== record.inspection.inspection.inspectionId || + operation.expectedContentDigest !== record.inspection.inspection.contentDigest || + canonicalJson(context.execution) !== canonicalJson(record.installExecution) + ) + throw Error("Standalone transaction producer mismatch"); + } else if ( + operation.kind !== "integration.settle" || + operation.transactionId !== record.transactionId || + operation.expectedContentDigest !== record.inspection.inspection.contentDigest + ) + throw Error("Standalone transaction settlement mismatch"); + if (publication) { + assertIntegrationPublicationJournal(publication, this.standalonePublicationJournal(context)); + record.publication = describeIntegrationPublication(publication); + } else if (operation.kind !== "integration.install" || record.publication !== null) + throw Error("Actual standalone publication producer required"); this.lock(() => { + const name = `transaction-${record.transactionId}.json`; + let previous: StandaloneTransactionRecord | null = null; + try { + previous = this.readIntegrationRecord(name, standaloneTransactionRecordSchema); + } catch (error) { + if (!absent(error)) throw error; + } + if (previous) { + const { publication: _old, ...old } = previous, + { publication: _new, ...next } = record; + if (canonicalJson(old) !== canonicalJson(next)) + throw Error("Immutable standalone transaction changed"); + if (!publication) throw Error("Standalone transaction already exists"); + } + this.writeIntegrationRecord(name, record, previous === null); + }); + } + standalonePublicationJournal(context: OpsExecutionContext): IntegrationPublicationJournal { + this.standaloneInstallationId(context); + const operation = this.integrationContext(context); + const transactionId = + operation.kind === "integration.install" + ? operation.operationId + : operation.kind === "integration.settle" + ? operation.transactionId + : null; + if (!transactionId) throw Error("Expected standalone publication execution"); + let journal = this.integrationJournals.get(transactionId); + if (!journal) { + journal = createIntegrationPublicationJournal({ + directory: this.integrationRecords, + coordination: this.records, + key: this.key, + transactionId, + }); + this.integrationJournals.set(transactionId, journal); + } + return journal; + } + async claimStage( + input: ClaimTransferStage, + context?: OpsExecutionContext, + ): Promise { + if (!context) failed(409, "Artifact purpose consumer is not installed for this operation"); + const operation = this.integrationContext(context); + this.admit(); + return this.lock(() => { const meta = this.owned(input.stageId, input.ownerId, input.purpose); + const identity = { + executionId: context.execution.executionId, + operationKey: context.execution.operationKey, + fingerprint: context.execution.fingerprint, + executionDeadlineAt: new Date(context.execution.deadlineAt).toISOString(), + }; + if ( + input.purpose !== "integration-artifact" || + canonicalJson(input.identity) !== canonicalJson(identity) + ) + failed(409, "Artifact claim execution mismatch"); + const prior = meta.claims.find( + (entry) => canonicalJson(entry.claim.identity) === canonicalJson(identity), + ); + if (prior) { + if (prior.claim.use !== input.use || prior.claim.sha256 !== input.expectedSha256) + failed(409, "Artifact claim replay mismatch"); + return prior.claim; + } this.alive(meta); + context.signal.throwIfAborted(); + if (context.execution.deadlineAt <= this.now()) + failed(410, "Artifact claim execution expired"); + if ( + meta.stage.direction !== "upload" || + !meta.content || + !meta.stage.sha256 || + meta.stage.sha256 !== input.expectedSha256 || + meta.claims.length >= 8 + ) + failed(409, "Artifact content is not available for this claim"); + if (operation.kind === "integration.inspectInstall") { + if ( + operation.ownerId !== input.ownerId || + operation.source.kind !== "uploaded-artifact" || + operation.source.stageId !== input.stageId || + operation.source.expectedSha256 !== input.expectedSha256 || + input.use !== "inspect-integration" || + !["uploaded", "ready"].includes(meta.stage.state) || + meta.claims.some((record) => record.settlement === null) + ) + failed(409, "Integration inspection claim mismatch"); + } else if (operation.kind === "integration.install") { + if ( + operation.ownerId !== input.ownerId || + input.use !== "install-integration" || + meta.stage.state !== "ready" || + !meta.integrationInspection || + !input.inspection || + canonicalJson(meta.integrationInspection) !== canonicalJson(input.inspection) || + operation.inspectionId !== meta.integrationInspection.inspectionId || + operation.expectedContentDigest !== meta.integrationInspection.validationFingerprint || + Date.parse(meta.integrationInspection.expiresAt) <= this.now() + ) + failed(409, "Integration installation inspection mismatch"); + } else failed(409, "This operation cannot claim uploaded bytes"); + // A prepared copy does not excuse mutation of its pinned upload. Check + // the original descriptor tuple again before granting installation. + const directory = this.directory(meta); + let fd: number | undefined; + try { + directory.fileIdentity("content"); + fd = openSync(directory.name("content"), C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + const stat = fstatSync(fd, { bigint: true }); + if ( + !meta.content || + !stat.isFile() || + stat.nlink !== 1n || + !same(meta.content, stat) || + String(stat.size) !== meta.content.size || + String(stat.mtimeNs) !== meta.content.mtimeNs || + String(stat.ctimeNs) !== meta.content.ctimeNs + ) + failed(409, "Artifact content changed since upload"); + directory.check(); + } finally { + if (fd !== undefined) closeSync(fd); + directory.close(); + } + const claim: ArtifactStageClaim = { + claimId: randomUUID(), + stageId: input.stageId, + ownerId: input.ownerId, + purpose: input.purpose, + use: input.use, + identity, + sha256: meta.stage.sha256, + bytes: meta.stage.bytes, + }; + meta.claims.push({ + claim, + settlement: null, + preparation: + operation.kind === "integration.install" + ? (meta.claims.find( + (record) => + record.settlement?.outcome === "validated" && + record.settlement.inspection.inspectionId === operation.inspectionId, + )?.preparation ?? null) + : null, + transactionId: operation.kind === "integration.install" ? operation.operationId : null, + }); + meta.stage.state = input.use === "inspect-integration" ? "validating" : "claimed"; + this.save(meta); + return claim; }); - failed(409, "Artifact purpose consumer is not installed"); } async inspectClaim(input: { stageId: string; ownerId: string; identity: StageClaimIdentity; }): Promise<{ claim: ArtifactStageClaim; settlement: StageClaimSettlement | null } | null> { - this.lock(() => this.owned(input.stageId, input.ownerId)); - return null; + return this.lock(() => { + const meta = this.owned(input.stageId, input.ownerId); + const entry = meta.claims.find( + (record) => canonicalJson(record.claim.identity) === canonicalJson(input.identity), + ); + return entry ? { claim: entry.claim, settlement: entry.settlement } : null; + }); + } + async validateIntegrationArtifact( + context: OpsExecutionContext, + preparation: IntegrationArtifactPreparationContext, + ) { + const operation = this.integrationContext(context); + if ( + operation.kind !== "integration.inspectInstall" || + operation.source.kind !== "uploaded-artifact" + ) + throw Error("Expected an uploaded integration inspection execution"); + const identity = { + executionId: context.execution.executionId, + operationKey: context.execution.operationKey, + fingerprint: context.execution.fingerprint, + executionDeadlineAt: new Date(context.execution.deadlineAt).toISOString(), + }; + const claim = await this.claimStage( + { + stageId: operation.source.stageId, + ownerId: operation.ownerId, + purpose: "integration-artifact", + use: "inspect-integration", + identity, + expectedSha256: operation.source.expectedSha256, + }, + context, + ); + const prior = this.lock( + () => + this.owned(claim.stageId, claim.ownerId).claims.find( + (record) => record.claim.claimId === claim.claimId, + )!, + ); + if (prior.settlement) { + if (prior.settlement.outcome !== "validated" || !prior.preparation) + failed(409, "Integration validation has already settled"); + const root = describeIntegrationPreparationRoot(preparation); + discardIntegrationPreparation(preparation); + return { + claim, + inspection: prior.settlement.inspection, + prepared: await restorePreparedIntegration({ + rootDir: root, + receipt: prior.preparation, + signal: context.signal, + }), + }; + } + try { + const prepared = await this.withClaimedIntegrationArtifact( + { claim, signal: context.signal }, + async (input) => + prepareIntegrationArtifact( + { + source: input.source, + expectedSha256: input.sha256, + maxArtifactBytes: 200 * 1024 ** 2, + maxExpandedBytes: 200 * 1024 ** 2, + signal: input.signal, + }, + preparation, + ), + ); + const receipt = describePreparedIntegration(prepared); + const inspection: StageInspectionBinding = { + inspectionId: randomUUID(), + stageId: claim.stageId, + sha256: claim.sha256, + bytes: claim.bytes, + policyRevision: this.revision, + validationFingerprint: prepared.contentDigest, + expiresAt: new Date( + Math.min( + this.now() + 600000, + Date.parse(this.lock(() => this.load(claim.stageId).stage.expiresAt)), + ), + ).toISOString(), + }; + this.lock(() => { + const meta = this.owned(claim.stageId, claim.ownerId); + const record = meta.claims.find((entry) => entry.claim.claimId === claim.claimId)!; + if (record.settlement || meta.stage.state !== "validating") + failed(409, "Integration validation ownership changed"); + record.preparation = receipt; + record.settlement = { outcome: "validated", inspection }; + meta.integrationInspection = inspection; + meta.stage.state = "ready"; + this.save(meta); + }); + return { claim, inspection, prepared }; + } catch (error) { + this.lock(() => { + const meta = this.owned(claim.stageId, claim.ownerId); + const record = meta.claims.find((entry) => entry.claim.claimId === claim.claimId)!; + if (!record.settlement) { + record.settlement = { outcome: "failed", receiptId: claim.claimId }; + meta.stage.state = "failed"; + this.save(meta); + } + }); + throw error; + } + } + async claimIntegrationArtifactInstall( + context: OpsExecutionContext, + inspection: StageInspectionBinding, + ) { + const operation = this.integrationContext(context); + if (operation.kind !== "integration.install") + throw Error("Expected integration install execution"); + return this.claimStage( + { + stageId: inspection.stageId, + ownerId: operation.ownerId, + purpose: "integration-artifact", + use: "install-integration", + identity: { + executionId: context.execution.executionId, + operationKey: context.execution.operationKey, + fingerprint: context.execution.fingerprint, + executionDeadlineAt: new Date(context.execution.deadlineAt).toISOString(), + }, + expectedSha256: inspection.sha256, + inspection, + }, + context, + ); + } + async settleIntegrationArtifact( + context: OpsExecutionContext, + claim: ArtifactStageClaim, + publication: IntegrationPublication, + ): Promise { + const operation = this.integrationContext(context); + assertIntegrationPublicationJournal(publication, this.standalonePublicationJournal(context)); + const receipt = describeIntegrationPublication(publication); + if ( + operation.kind !== "integration.settle" || + operation.transactionId !== receipt.transactionId || + operation.expectedContentDigest !== receipt.preparation.contentDigest || + (operation.decision === "commit" + ? receipt.outcome !== "committed" + : receipt.outcome !== "restored") + ) + throw Error("Integration publication has not settled under this original execution"); + return this.lock(() => { + const meta = this.owned(claim.stageId, claim.ownerId, "integration-artifact"); + const record = meta.claims.find( + (entry) => canonicalJson(entry.claim) === canonicalJson(claim), + ); + if ( + !record || + record.claim.use !== "install-integration" || + record.transactionId !== receipt.transactionId || + !record.preparation || + canonicalJson(record.preparation) !== canonicalJson(receipt.preparation) + ) + failed(409, "Integration stage publication ownership mismatch"); + const settlement: StageClaimSettlement = + operation.decision === "commit" + ? { outcome: "consumed", receiptId: receipt.transactionId } + : { outcome: "released", evidence: "restored" }; + if (record.settlement) { + if (canonicalJson(record.settlement) !== canonicalJson(settlement)) + failed(409, "Integration stage already settled differently"); + return meta.stage; + } + if (meta.stage.state !== "claimed") + failed(409, "Integration stage is not held by its original installation"); + record.settlement = settlement; + meta.stage.state = + operation.decision === "commit" + ? "consumed" + : Date.parse(meta.stage.expiresAt) > this.now() + ? "ready" + : "expired"; + this.save(meta); + return meta.stage; + }); } async settleStageClaim(_input: { claim: ArtifactStageClaim; @@ -668,21 +1197,108 @@ export class ArtifactTransferStore { failed(409, "Artifact purpose consumer is not installed"); } async withStageFile( - _input: { claim: ArtifactStageClaim; signal: AbortSignal }, - _run: (file: FileHandle, signal: AbortSignal) => Promise, + input: { claim: ArtifactStageClaim; signal: AbortSignal }, + run: (file: FileHandle, signal: AbortSignal) => Promise, ): Promise { - failed(409, "Artifact purpose consumer is not installed"); + const claim = artifactStageClaimSchema.parse(input.claim); + const meta = this.lock(() => { + const meta = this.owned(claim.stageId, claim.ownerId, "integration-artifact"); + const record = meta.claims.find( + (entry) => canonicalJson(entry.claim) === canonicalJson(claim), + ); + if ( + !record || + record.settlement || + !["validating", "claimed"].includes(meta.stage.state) || + this.streams.has(claim.stageId) + ) + failed(409, "Artifact claim is not an active purpose consumer"); + if ( + this.streams.size >= this.policy.maxConcurrentTransfers || + [...this.streams.values()].filter((entry) => entry.ownerId === claim.ownerId).length >= + this.policy.maxConcurrentTransfersPerPrincipal + ) + failed(429, "Artifact consumer concurrency limit exceeded"); + return meta; + }); + const controller = new AbortController(); + const abort = () => controller.abort(input.signal.reason); + input.signal.addEventListener("abort", abort, { once: true }); + if (input.signal.aborted) abort(); + const timer = setTimeout( + () => controller.abort(new ArtifactTransferError(410, "Artifact claim deadline expired")), + Math.max(1, Date.parse(claim.identity.executionDeadlineAt) - this.now()), + ); + timer.unref?.(); + this.streams.set(claim.stageId, { ownerId: claim.ownerId, controller }); + let directory: AnchoredDirectory | undefined, file: FileHandle | undefined; + try { + controller.signal.throwIfAborted(); + directory = this.directory(meta); + file = await open(directory.name("content"), C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + const verify = async () => { + const stat = await file!.stat({ bigint: true }); + if ( + !meta.content || + !stat.isFile() || + stat.nlink !== 1n || + !same(meta.content, stat) || + String(stat.size) !== meta.content.size || + String(stat.mtimeNs) !== meta.content.mtimeNs || + String(stat.ctimeNs) !== meta.content.ctimeNs || + canonicalJson({ dev: meta.content.dev, ino: meta.content.ino }) !== + canonicalJson(directory!.fileIdentity("content")) + ) + failed(409, "Artifact content changed"); + directory!.check(); + }; + await verify(); + const result = await run(file, controller.signal); + await verify(); + return result; + } finally { + try { + await file?.close(); + directory?.close(); + } finally { + clearTimeout(timer); + input.signal.removeEventListener("abort", abort); + this.streams.delete(claim.stageId); + } + } } async withClaimedIntegrationArtifact( - _input: { claim: ArtifactStageClaim; signal: AbortSignal }, - _run: (input: { + input: { claim: ArtifactStageClaim; signal: AbortSignal }, + run: (input: { source: AsyncIterable; signal: AbortSignal; bytes: number; sha256: string; }) => Promise, ): Promise { - failed(409, "Artifact purpose consumer is not installed"); + if (input.claim.purpose !== "integration-artifact" || input.claim.use !== "inspect-integration") + failed(409, "Expected integration inspection claim"); + return this.withStageFile(input, async (file, signal) => { + const source = (async function* () { + let position = 0; + for (;;) { + signal.throwIfAborted(); + const buffer = Buffer.alloc(64 * 1024); + const { bytesRead } = await file.read(buffer, 0, buffer.length, position); + if (!bytesRead) break; + position += bytesRead; + if (position > input.claim.bytes) + failed(409, "Artifact consumer exceeded its immutable byte count"); + yield buffer.subarray(0, bytesRead); + } + if (position !== input.claim.bytes) failed(409, "Artifact consumer byte count changed"); + })(); + try { + return await run({ source, signal, bytes: input.claim.bytes, sha256: input.claim.sha256 }); + } finally { + await source.return(undefined); + } + }); } } diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index d60beb5bb..573230d91 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -38,6 +38,7 @@ import { productionOwnedServices, productionReleaseEffects, } from "./production-runtime"; +import { installStandaloneIntegrationRuntime } from "./standalone-integration-runtime"; import { afterValidatedServiceAuthority, resolveBootstrapEnabledServiceIds, @@ -222,6 +223,11 @@ async function main(): Promise { : undefined; if (artifactTransfers) installArtifactTransferRuntime(runtime, artifactTransfers); else if ("reason" in transferPolicy) console.error(transferPolicy.reason); + if (controlStartup.installation) + installStandaloneIntegrationRuntime(runtime, { + rootDir: config.rootDir, + store: artifactTransfers, + }); const releaseEffects = productionReleaseEffects(runtime); const retentionDays = Number(process.env.BACKUP_RETENTION_DAYS?.trim() || "30"); if (!Number.isSafeInteger(retentionDays) || retentionDays <= 0 || retentionDays > 36_500) { diff --git a/apps/ops-agent/src/standalone-integration-records.ts b/apps/ops-agent/src/standalone-integration-records.ts new file mode 100644 index 000000000..0e6f806ff --- /dev/null +++ b/apps/ops-agent/src/standalone-integration-records.ts @@ -0,0 +1,39 @@ +import { + artifactStageClaimSchema, + executionRefSchema, + stageInspectionBindingSchema, + standaloneInstallInspectionSchema, + standaloneInstallSourceSchema, +} from "@openmapx/core/ops"; +import { + integrationPreparationReceiptSchema, + integrationPublicationReceiptSchema, +} from "@openmapx/integration-framework/installer"; +import z from "zod/v4"; + +const inode = z.strictObject({ dev: z.string(), ino: z.string() }); +export const standaloneInspectionRecordSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + ownerId: z.string().min(1).max(128), + source: standaloneInstallSourceSchema, + inspection: standaloneInstallInspectionSchema, + preparation: integrationPreparationReceiptSchema, + previous: z + .strictObject({ contentDigest: z.string().regex(/^[a-f0-9]{64}$/), identity: inode }) + .nullable(), + stage: stageInspectionBindingSchema.nullable(), +}); +export const standaloneTransactionRecordSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + transactionId: z.uuid(), + ownerId: z.string().min(1).max(128), + installExecution: executionRefSchema, + inspection: standaloneInspectionRecordSchema, + claim: artifactStageClaimSchema.nullable(), + publication: integrationPublicationReceiptSchema.nullable(), +}); + +export type StandaloneInspectionRecord = z.infer; +export type StandaloneTransactionRecord = z.infer; diff --git a/apps/ops-agent/src/standalone-integration-runtime.test.ts b/apps/ops-agent/src/standalone-integration-runtime.test.ts new file mode 100644 index 000000000..8f9c46511 --- /dev/null +++ b/apps/ops-agent/src/standalone-integration-runtime.test.ts @@ -0,0 +1,301 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { + createControlRootOperationKey, + normalizeExecutionAdmission, + type OpsOperation, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { + createIntegrationArtifactPreparation, + describePreparedIntegration, + publishPreparedIntegration, + settleIntegrationPublication, +} from "@openmapx/integration-framework/installer"; +import { afterEach, expect, it } from "vitest"; +import { createDeterministicTarGz } from "../../../packages/integration-framework/src/deterministic-tar"; +import { ArtifactTransferStore } from "./artifact-transfer/store"; +import type { OpsExecutionContext } from "./runtime"; +import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; +import { installStandaloneIntegrationRuntime } from "./standalone-integration-runtime"; + +const cleanup: (() => void)[] = []; +afterEach(() => { + cleanup + .reverse() + .splice(0) + .forEach((fn) => { + fn(); + }); +}); +it.each(["normal", "upload-mutated", "forged-completion"] as const)( + "dispatches uploaded standalone installation and restart settlement under original signed receipts (scenario=%s)", + async (scenario) => { + const root = mkdtempSync(join(tmpdir(), "openmapx-stage-consumer-")); + cleanup.push(() => rmSync(root, { recursive: true, force: true })); + const infra = join(root, "infra"); + mkdirSync(join(infra, "data"), { recursive: true }); + mkdirSync(join(root, "services")); + await control.provisionControlInstallation(infra); + const installation = control.openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["app-api"], + }); + cleanup.push(() => control.closeControlInstallation(installation)); + let store = new ArtifactTransferStore({ + dataRoot: join(root, "stages"), + privateRoot: join(root, "stage-meta"), + token: "fixture-test-token", + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), + }); + cleanup.push(() => store.close()); + const bytes = createDeterministicTarGz([ + { + path: "manifest.json", + contents: Buffer.from( + JSON.stringify({ + id: "upload-fixture", + name: "Uploaded", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ), + }, + ]); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const stage = await store.createStage({ + purpose: "integration-artifact", + direction: "upload", + ownerId: "fixture-admin", + declaredBytes: bytes.length, + expectedSha256: sha256, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + }); + await store.upload({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 60000, + body: new ReadableStream({ + start(c) { + c.enqueue(bytes); + c.close(); + }, + }), + }); + const operation: OpsOperation = { + kind: "integration.inspectInstall", + ownerId: "fixture-admin", + source: { kind: "uploaded-artifact", stageId: stage.stageId, expectedSha256: sha256 }, + }; + const makeContext = (operation: OpsOperation): OpsExecutionContext => { + const admittedAt = Date.now(), + operationFingerprint = opsOperationFingerprint(operation), + operationKey = createControlRootOperationKey({ + notAfter: admittedAt + 600000, + nonce: randomBytes(16), + }), + admission = normalizeExecutionAdmission({ + operationFingerprint, + operationKey, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: admittedAt + 600000, + }); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey, + operationFingerprint, + deadlineAt: admission.deadlineAt, + fingerprint: admission.fingerprint, + }; + const authority = control.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: admittedAt + 600000, + serviceIds: ["app-api"], + }); + return { + execution, + controlAuthority: authority, + signal: new AbortController().signal, + emitLog: () => {}, + claim: { + fingerprint: operationFingerprint, + operation, + source: "registry", + capability: { revisionId: "fixture", values: {} }, + }, + }; + }; + const context = makeContext(operation); + const runtime = createUnavailableRuntime(); + installStandaloneIntegrationRuntime(runtime, { rootDir: root, store }); + const inspection = await dispatchOpsOperation(runtime, operation, context); + expect(inspection).toMatchObject({ + sourceKind: "uploaded-artifact", + integrationId: "upload-fixture", + archiveSha256: sha256, + denials: [], + }); + const transactionId = randomUUID(); + const installing: OpsOperation = { + kind: "integration.install", + inspectionId: inspection.inspectionId, + ownerId: "fixture-admin", + expectedContentDigest: inspection.contentDigest, + operationId: transactionId, + }; + await expect( + dispatchOpsOperation( + runtime, + { ...installing, ownerId: "foreign" }, + makeContext({ ...installing, ownerId: "foreign" }), + ), + ).rejects.toThrow(/owner/); + const inspectionPath = join( + root, + "stage-meta/integrations", + `inspection-${inspection.inspectionId}.json`, + ); + const originalBytes = readFileSync(inspectionPath), + envelope = JSON.parse(originalBytes.toString("utf8")), + forgedRecord = JSON.parse(envelope.body); + forgedRecord.ownerId = "foreign"; + writeFileSync( + inspectionPath, + JSON.stringify({ ...envelope, body: JSON.stringify(forgedRecord) }), + ); + try { + await expect( + dispatchOpsOperation(runtime, installing, makeContext(installing)), + ).rejects.toThrow(/authentication/); + } finally { + writeFileSync(inspectionPath, originalBytes); + } + if (scenario === "upload-mutated") { + writeFileSync(join(root, "stages", stage.stageId, "content"), Buffer.from("changed")); + await expect( + dispatchOpsOperation(runtime, installing, makeContext(installing)), + ).rejects.toThrow(/content changed/); + expect(existsSync(join(root, "custom_integrations", "upload-fixture"))).toBe(false); + return; + } + const installContext = makeContext(installing); + const result = await dispatchOpsOperation(runtime, installing, installContext); + expect(result).toMatchObject({ + outcome: "published", + integrationId: "upload-fixture", + contentDigest: inspection.contentDigest, + }); + await expect( + store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), + ).rejects.toMatchObject({ statusCode: 409 }); + if (scenario === "forged-completion") { + const staging = join(root, "custom_integrations/.staging"); + const workspace = readdirSync(staging).find( + (name) => + name.startsWith("preparation-") && existsSync(join(staging, name, "publication.json")), + ); + if (!workspace) throw Error("Missing actual shared publication journal"); + const path = join(staging, workspace, "publication.json"), + record = JSON.parse(readFileSync(path, "utf8")); + writeFileSync(path, JSON.stringify({ ...record, outcome: "committed" })); + rmSync(join(root, "custom_integrations", "upload-fixture"), { recursive: true }); + await expect(dispatchOpsOperation(runtime, installing, installContext)).rejects.toThrow( + /ENOENT|publication|changed|Published/i, + ); + const settle: OpsOperation = { + kind: "integration.settle", + transactionId, + decision: "commit", + expectedContentDigest: inspection.contentDigest, + }; + await expect(dispatchOpsOperation(runtime, settle, makeContext(settle))).rejects.toThrow( + /journal|publication|changed|auth/i, + ); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + }) + ).state, + ).toBe("claimed"); + return; + } + store.close(); + store = new ArtifactTransferStore({ + dataRoot: join(root, "stages"), + privateRoot: join(root, "stage-meta"), + token: "fixture-test-token", + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), + }); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + }) + ).state, + ).toBe("claimed"); + await expect( + store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), + ).rejects.toMatchObject({ statusCode: 409 }); + const replacement = createUnavailableRuntime(); + installStandaloneIntegrationRuntime(replacement, { rootDir: root, store }); + const settle: OpsOperation = { + kind: "integration.settle", + transactionId, + decision: "commit", + expectedContentDigest: inspection.contentDigest, + }; + const settleContext = makeContext(settle); + expect(await dispatchOpsOperation(replacement, settle, settleContext)).toMatchObject({ + outcome: "committed", + transactionId, + }); + expect(await dispatchOpsOperation(replacement, settle, settleContext)).toMatchObject({ + outcome: "committed", + transactionId, + }); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + }) + ).state, + ).toBe("consumed"); + await expect( + dispatchOpsOperation( + replacement, + { ...settle, decision: "restore" }, + makeContext({ ...settle, decision: "restore" }), + ), + ).rejects.toThrow(/different decision/); + }, +); diff --git a/apps/ops-agent/src/standalone-integration-runtime.ts b/apps/ops-agent/src/standalone-integration-runtime.ts new file mode 100644 index 000000000..55240d430 --- /dev/null +++ b/apps/ops-agent/src/standalone-integration-runtime.ts @@ -0,0 +1,220 @@ +import { randomUUID } from "node:crypto"; +import { + canonicalJson, + type stageInspectionBindingSchema, + standaloneInstallInspectionSchema, +} from "@openmapx/core/ops"; +import { + createIntegrationArtifactPreparation, + describeIntegrationPublication, + describePreparedIntegration, + discardIntegrationPreparation, + inspectInstalledIntegrationFingerprint, + inspectIntegrationPublication, + prepareIntegrationSource, + prepareRemoteIntegrationArtifact, + publishPreparedIntegration, + restoreIntegrationPublication, + restorePreparedIntegration, + settleIntegrationPublication, +} from "@openmapx/integration-framework/installer"; +import type z from "zod/v4"; +import type { ArtifactTransferStore } from "./artifact-transfer/store"; +import type { OpsRuntime } from "./runtime"; + +import { + standaloneInspectionRecordSchema as inspectionRecordSchema, + standaloneTransactionRecordSchema as transactionRecordSchema, +} from "./standalone-integration-records"; + +function missing(error: unknown) { + return (error as NodeJS.ErrnoException).code === "ENOENT"; +} +export function installStandaloneIntegrationRuntime( + runtime: OpsRuntime, + options: { rootDir: string; store?: ArtifactTransferStore; now?: () => number }, +): void { + const now = options.now ?? Date.now; + const store = () => { + if (!options.store) throw Error("Standalone integration purpose storage is unavailable"); + return options.store; + }; + runtime["integration.inspectInstall"] = async (operation, context) => { + const installationId = store().standaloneInstallationId(context), + preparationContext = createIntegrationArtifactPreparation({ rootDir: options.rootDir }); + let published = false; + try { + let commit: string | null = null, + stage: z.infer | null = null; + let prepared: Awaited>; + if (operation.source.kind === "source") { + const result = await prepareIntegrationSource( + { + repository: operation.source.repository, + ref: operation.source.ref, + signal: context.signal, + }, + preparationContext, + ); + prepared = result.prepared; + commit = result.commit; + } else if (operation.source.kind === "artifact") { + prepared = await prepareRemoteIntegrationArtifact( + { url: operation.source.url, sha256: operation.source.sha256, signal: context.signal }, + preparationContext, + ); + } else { + if (!options.store) throw Error("Integration artifact transfer is unavailable"); + const result = await options.store.validateIntegrationArtifact(context, preparationContext); + prepared = result.prepared; + stage = result.inspection; + } + const inspection = standaloneInstallInspectionSchema.parse({ + inspectionId: stage?.inspectionId ?? randomUUID(), + sourceKind: operation.source.kind, + integrationId: prepared.id, + version: prepared.version, + contentDigest: prepared.contentDigest, + commit, + stageId: stage?.stageId ?? null, + archiveSha256: prepared.archiveSha256, + expiresAt: stage?.expiresAt ?? new Date(now() + 600000).toISOString(), + denials: [], + }); + const record = inspectionRecordSchema.parse({ + version: 1, + installationId, + ownerId: operation.ownerId, + source: operation.source, + inspection, + preparation: describePreparedIntegration(prepared), + previous: inspectInstalledIntegrationFingerprint({ + rootDir: options.rootDir, + id: prepared.id, + signal: context.signal, + }), + stage, + }); + store().retainStandaloneInspection(context, record); + published = true; + return inspection; + } finally { + if (!published) discardIntegrationPreparation(preparationContext); + } + }; + runtime["integration.install"] = async (operation, context) => { + const installationId = store().standaloneInstallationId(context); + const inspection = store().readStandaloneInspection(context, operation.inspectionId); + if ( + inspection.ownerId !== operation.ownerId || + inspection.inspection.contentDigest !== operation.expectedContentDigest + ) + throw Error("Standalone inspection owner or content changed"); + const journal = store().standalonePublicationJournal(context); + let existing: z.infer | null = null; + try { + existing = store().readStandaloneTransaction(context, operation.operationId); + } catch (error) { + if (!missing(error)) throw error; + } + if (existing) { + if ( + existing.ownerId !== operation.ownerId || + existing.inspection.inspection.inspectionId !== operation.inspectionId || + canonicalJson(existing.installExecution) !== canonicalJson(context.execution) + ) + throw Error("Standalone transaction replay mismatch"); + const publication = inspectIntegrationPublication({ + rootDir: options.rootDir, + preparation: existing.inspection.preparation, + transactionId: existing.transactionId, + journal, + }); + if (publication.outcome !== "published") + throw Error("Standalone installation requires original transaction reconciliation"); + await restoreIntegrationPublication({ + rootDir: options.rootDir, + receipt: publication, + signal: context.signal, + journal, + }); + return { + transactionId: existing.transactionId, + integrationId: publication.integrationId, + previousContentDigest: publication.previousContentDigest, + contentDigest: publication.preparation.contentDigest, + outcome: "published", + }; + } + context.signal.throwIfAborted(); + if (Date.parse(inspection.inspection.expiresAt) <= now()) + throw Error("Standalone inspection expired; preview again"); + const prepared = await restorePreparedIntegration({ + rootDir: options.rootDir, + receipt: inspection.preparation, + signal: context.signal, + }); + const claim = inspection.stage + ? await options.store?.claimIntegrationArtifactInstall(context, inspection.stage) + : null; + if (inspection.stage && !claim) throw Error("Integration artifact claim is unavailable"); + const record = transactionRecordSchema.parse({ + version: 1, + installationId, + transactionId: operation.operationId, + ownerId: operation.ownerId, + installExecution: context.execution, + inspection, + claim: claim ?? null, + publication: null, + }); + store().retainStandaloneTransaction(context, record); + const publication = await publishPreparedIntegration(prepared, { + transactionId: operation.operationId, + expectedPreviousContentDigest: inspection.previous?.contentDigest ?? null, + expectedPreviousIdentity: inspection.previous?.identity ?? null, + journal, + signal: context.signal, + }); + record.publication = describeIntegrationPublication(publication); + store().retainStandaloneTransaction(context, record, publication); + return { + transactionId: record.transactionId, + integrationId: prepared.id, + previousContentDigest: record.publication.previousContentDigest, + contentDigest: prepared.contentDigest, + outcome: "published", + }; + }; + runtime["integration.settle"] = async (operation, context) => { + store().standaloneInstallationId(context); + const record = store().readStandaloneTransaction(context, operation.transactionId); + const journal = store().standalonePublicationJournal(context); + if (record.inspection.inspection.contentDigest !== operation.expectedContentDigest) + throw Error("Standalone settlement content mismatch"); + const publicationReceipt = inspectIntegrationPublication({ + rootDir: options.rootDir, + preparation: record.inspection.preparation, + transactionId: record.transactionId, + journal, + }); + const publication = await restoreIntegrationPublication({ + rootDir: options.rootDir, + receipt: publicationReceipt, + signal: context.signal, + journal, + }); + const result = await settleIntegrationPublication( + publication, + operation.decision, + context.signal, + ); + if (record.claim) { + if (!options.store) throw Error("Integration artifact settlement is unavailable"); + await options.store.settleIntegrationArtifact(context, record.claim, publication); + } + record.publication = describeIntegrationPublication(publication); + store().retainStandaloneTransaction(context, record, publication); + return { transactionId: record.transactionId, ...result }; + }; +} diff --git a/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx b/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx index 8f28a0699..c16f5cf91 100644 --- a/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx +++ b/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx @@ -67,6 +67,8 @@ import { useAdminToast } from "../shared/AdminToast"; import { TableSearchField, TableToolbar } from "../shared/TableToolbar"; import { useClientPagination } from "../shared/tableHooks"; +import { StandaloneInstallDialog } from "./StandaloneInstallDialog"; + type Trust = AdminExtensionInstallPreview["sourceTrust"]; function TrustChip({ trust }: { trust?: Trust }) { @@ -923,6 +925,9 @@ function ExtensionStoreContent() { title="Extensions" subtitle="Browse, install, and manage community extensions — integrations and services." /> + + + setTab(v)} aria-label="Extension management"> diff --git a/apps/web/src/components/admin/extensions/StandaloneInstallDialog.test.tsx b/apps/web/src/components/admin/extensions/StandaloneInstallDialog.test.tsx new file mode 100644 index 000000000..587f63cbc --- /dev/null +++ b/apps/web/src/components/admin/extensions/StandaloneInstallDialog.test.tsx @@ -0,0 +1,222 @@ +import { QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { AdminJobEffectsProvider } from "@/lib/admin/adminJobEffects"; +import { act, cleanup, createTestQueryClient, fireEvent, render, screen, waitFor } from "@/test"; +import { deferred } from "@/test/extensionAdmin"; +import { job } from "../activity/__tests__/fixtures"; +import { StandaloneInstallDialog } from "./StandaloneInstallDialog"; + +let apiUrl = "https://admin.example"; +vi.mock("@/integration-api/runtime/EnvProvider", () => ({ useEnv: () => ({ apiUrl }) })); +const queries: ReturnType[] = []; +let calls: { url: URL; init: RequestInit }[] = []; +let intercept: (url: URL, init: RequestInit) => Response | Promise | undefined = () => + undefined; +const previewJob = "ddc83a2c-cd99-47c5-8eaf-ff20a2291d37", + installJob = "b864063e-a930-4239-85f6-5d0672c368da"; +const inspection = { + inspectionId: "4093ebdd-04d2-4cda-970e-2bcc7407c8cf", + sourceKind: "source", + integrationId: "map-example", + version: "1.0.0", + contentDigest: "c".repeat(64), + commit: "a".repeat(40), + stageId: null, + archiveSha256: null, + expiresAt: new Date(Date.now() + 600000).toISOString(), + denials: [], +}; +beforeEach(() => { + apiUrl = "https://admin.example"; + calls = []; + intercept = () => undefined; + vi.stubGlobal("fetch", async (raw: string, init: RequestInit) => { + const url = new URL(raw, window.location.origin); + calls.push({ url, init }); + const handled = intercept(url, init); + if (handled) return handled; + if (url.pathname.endsWith("/preview")) + return Response.json( + { jobId: previewJob, operationId: "d0865fbe-1458-4cae-90a0-bb395fdb14b5" }, + { status: 202 }, + ); + if (url.pathname.endsWith("/install")) + return Response.json( + { jobId: installJob, operationId: "c67d1e20-566d-4cd1-8d13-899dcfc57880" }, + { status: 202 }, + ); + const id = url.pathname.endsWith(previewJob) ? previewJob : installJob; + return Response.json( + job({ + id, + status: "success", + cancelable: false, + result: id === previewJob ? inspection : { outcome: "committed" }, + }), + ); + }); +}); +afterEach(() => { + cleanup(); + queries.splice(0).forEach((query) => { + query.clear(); + }); + vi.unstubAllGlobals(); +}); +function mount() { + const qc = createTestQueryClient(); + queries.push(qc); + return render( + + + + + , + ); +} +it("reviews a fixed source commit before sending the retained inspection to install", async () => { + mount(); + fireEvent.click(screen.getByRole("button", { name: "Install standalone integration" })); + fireEvent.change(screen.getByLabelText("Repository HTTPS URL"), { + target: { value: "https://github.com/public/example.git" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Review source" })); + expect(await screen.findByText(`Resolved commit: ${inspection.commit}`)).toBeVisible(); + expect(screen.getByRole("button", { name: "Install integration" })).not.toBeDisabled(); + const preview = calls.find((call) => call.url.pathname.endsWith("/preview"))!; + expect(JSON.parse(String(preview.init.body))).toEqual({ + source: { kind: "source", repository: "https://github.com/public/example.git" }, + }); + expect(new Headers(preview.init.headers).get("idempotency-key")).toMatch(/^[a-f0-9-]{36}$/); + fireEvent.click(screen.getByRole("button", { name: "Install integration" })); + expect(await screen.findByText("Standalone integration installed.")).toBeVisible(); + const install = calls.find((call) => call.url.pathname.endsWith("/install"))!; + expect(JSON.parse(String(install.init.body))).toEqual({ + inspectionId: inspection.inspectionId, + expectedContentDigest: inspection.contentDigest, + }); + expect(calls.filter((call) => call.url.pathname.endsWith("/install"))).toHaveLength(1); +}); +it("retires a prior inspection when source changes and never installs it", async () => { + mount(); + fireEvent.click(screen.getByRole("button", { name: "Install standalone integration" })); + fireEvent.change(screen.getByLabelText("Repository HTTPS URL"), { + target: { value: "https://github.com/public/example.git" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Review source" })); + await screen.findByText(`Resolved commit: ${inspection.commit}`); + fireEvent.change(screen.getByLabelText("Repository HTTPS URL"), { + target: { value: "https://github.com/public/other.git" }, + }); + await waitFor(() => + expect(screen.queryByRole("button", { name: "Install integration" })).toBeNull(), + ); + expect(calls.some((call) => call.url.pathname.endsWith("/install"))).toBe(false); +}); + +it("ignores a late accepted preview after closing and reopening the dialog", async () => { + const held = deferred(); + intercept = (url) => (url.pathname.endsWith("/preview") ? held.promise : undefined); + mount(); + fireEvent.click(screen.getByRole("button", { name: "Install standalone integration" })); + fireEvent.change(screen.getByLabelText("Repository HTTPS URL"), { + target: { value: "https://github.com/public/example.git" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Review source" })); + await waitFor(() => + expect(calls.some((call) => call.url.pathname.endsWith("/preview"))).toBe(true), + ); + fireEvent.click(screen.getByRole("button", { name: "Close" })); + fireEvent.click(await screen.findByRole("button", { name: "Install standalone integration" })); + await act(async () => + held.resolve( + Response.json( + { jobId: previewJob, operationId: "d0865fbe-1458-4cae-90a0-bb395fdb14b5" }, + { status: 202 }, + ), + ), + ); + await waitFor(() => + expect(screen.getByRole("button", { name: "Review source" })).not.toBeDisabled(), + ); + expect(screen.queryByRole("button", { name: "Install integration" })).toBeNull(); +}); +it("allocates a fresh inspection request when explicitly reviewing an existing preview again", async () => { + mount(); + fireEvent.click(screen.getByRole("button", { name: "Install standalone integration" })); + fireEvent.change(screen.getByLabelText("Repository HTTPS URL"), { + target: { value: "https://github.com/public/example.git" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Review source" })); + await screen.findByText(`Resolved commit: ${inspection.commit}`); + const first = calls.find((call) => call.url.pathname.endsWith("/preview"))!; + fireEvent.click(screen.getByRole("button", { name: "Review source" })); + await waitFor(() => + expect(calls.filter((call) => call.url.pathname.endsWith("/preview"))).toHaveLength(2), + ); + const second = calls.filter((call) => call.url.pathname.endsWith("/preview"))[1]!; + expect(new Headers(second.init.headers).get("idempotency-key")).not.toBe( + new Headers(first.init.headers).get("idempotency-key"), + ); +}); +it("retries a lost preview acknowledgement using the already-uploaded immutable file", async () => { + const stageId = "3524c227-2876-42ec-a25d-3bc9e2b87872", + sha256 = "d".repeat(64); + let previewAttempts = 0; + const stage = { + stageId, + purpose: "integration-artifact", + direction: "upload", + state: "created", + bytes: 0, + sha256: null, + expiresAt: new Date(Date.now() + 600000).toISOString(), + cleanup: "pending", + }; + intercept = (url, init) => { + if (url.pathname.endsWith("/artifact-transfers")) return Response.json(stage); + if (url.pathname.endsWith("/content") && init.method === "PUT") + return Response.json({ ...stage, state: "uploaded", bytes: 3, sha256 }); + if (url.pathname.endsWith("/preview") && previewAttempts++ === 0) + return Promise.reject(new TypeError("preview-transport-lost")); + if (url.pathname.endsWith(previewJob)) + return Response.json( + job({ + id: previewJob, + status: "success", + cancelable: false, + result: { + ...inspection, + sourceKind: "uploaded-artifact", + stageId, + archiveSha256: sha256, + commit: null, + }, + }), + ); + return undefined; + }; + mount(); + fireEvent.click(screen.getByRole("button", { name: "Install standalone integration" })); + fireEvent.mouseDown(screen.getByRole("combobox", { name: "Install source" })); + fireEvent.click(await screen.findByRole("option", { name: "Upload archive" })); + const file = new File([new Uint8Array([0, 1, 255])], "fixture.tar.gz"); + fireEvent.change(screen.getByLabelText("Integration archive"), { target: { files: [file] } }); + fireEvent.click(screen.getByRole("button", { name: "Review source" })); + await waitFor(() => + expect(calls.filter((call) => call.url.pathname.endsWith("/preview"))).toHaveLength(1), + ); + await waitFor(() => + expect(screen.getByRole("button", { name: "Review source" })).not.toBeDisabled(), + ); + fireEvent.click(screen.getByRole("button", { name: "Review source" })); + expect(await screen.findByRole("button", { name: "Install integration" })).not.toBeDisabled(); + const uploads = calls.filter((call) => call.init.method === "PUT"); + expect(uploads).toHaveLength(1); + expect(uploads[0]!.init.body).toBe(file); + const previews = calls.filter((call) => call.url.pathname.endsWith("/preview")); + expect(previews).toHaveLength(2); + expect(new Headers(previews[0]!.init.headers).get("idempotency-key")).toBe( + new Headers(previews[1]!.init.headers).get("idempotency-key"), + ); +}); diff --git a/apps/web/src/components/admin/extensions/StandaloneInstallDialog.tsx b/apps/web/src/components/admin/extensions/StandaloneInstallDialog.tsx new file mode 100644 index 000000000..39b2f2f22 --- /dev/null +++ b/apps/web/src/components/admin/extensions/StandaloneInstallDialog.tsx @@ -0,0 +1,395 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Button from "@mui/material/Button"; +import Dialog from "@mui/material/Dialog"; +import DialogActions from "@mui/material/DialogActions"; +import DialogContent from "@mui/material/DialogContent"; +import DialogTitle from "@mui/material/DialogTitle"; +import MenuItem from "@mui/material/MenuItem"; +import Stack from "@mui/material/Stack"; +import TextField from "@mui/material/TextField"; +import Typography from "@mui/material/Typography"; +import * as sdk from "@openmapx/core/api-admin-sdk"; +import { + type StandaloneInstallInspection, + type StandaloneInstallSource, + standaloneInstallInspectionSchema, + standaloneInstallSourceSchema, +} from "@openmapx/core/api-contracts"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useEffect, useRef, useState } from "react"; +import { useAdminJobRegistration } from "@/lib/admin/adminJobEffects"; +import { invalidateExtensionAdministration } from "@/lib/admin/extensionAdmin"; +import { useAdminMutationOwner } from "@/lib/admin/useAdminMutationOwner"; +import { + activityJobQueryOptions, + hasUnresolvedJobObservation, + useActivityClient, +} from "../activity/activityQueries"; + +const terminal = new Set(["success", "failed", "canceled"]); +export function StandaloneInstallDialog() { + const { apiUrl } = useActivityClient(); + return ; +} +function StandaloneInstallDialogContent() { + const { client, apiUrl } = useActivityClient(), + owner = useAdminMutationOwner(client, apiUrl), + register = useAdminJobRegistration(), + qc = useQueryClient(); + const [open, setOpen] = useState(false), + [kind, setKind] = useState("source"), + [url, setUrl] = useState(""), + [ref, setRef] = useState(""), + [sha, setSha] = useState(""), + [file, setFile] = useState(null); + const [busy, setBusy] = useState(false), + [error, setError] = useState(null), + [inspection, setInspection] = useState(null), + [done, setDone] = useState(false), + [expired, setExpired] = useState(false); + const generation = useRef(0), + running = useRef(false), + upload = useRef(null), + keys = useRef({ preview: crypto.randomUUID(), install: crypto.randomUUID() }); + const uploadedSource = useRef<{ + file: File; + source: Extract; + } | null>(null); + const [observed, setObserved] = useState<{ + jobId: string; + phase: "preview" | "install"; + generation: number; + owner: typeof owner; + source?: StandaloneInstallSource; + } | null>(null); + const active = + observed?.owner === owner && observed.generation === generation.current ? observed : null; + const query = useQuery({ + ...activityJobQueryOptions(client, apiUrl, active?.jobId, "application"), + refetchInterval: (query) => + query.state.data && + terminal.has(query.state.data.status) && + !hasUnresolvedJobObservation(query.state.data.result) + ? false + : 1000, + }); + const waiting = Boolean( + active && + (!query.data || + !terminal.has(query.data.status) || + hasUnresolvedJobObservation(query.data.result)), + ); + const reset = (preserveUpload = false) => { + if (!preserveUpload) uploadedSource.current = null; + generation.current++; + setObserved(null); + setInspection(null); + setError(null); + setDone(false); + setExpired(false); + keys.current = { preview: crypto.randomUUID(), install: crypto.randomUUID() }; + }; + const close = () => { + upload.current?.abort(); + reset(); + setOpen(false); + }; + useEffect(() => () => upload.current?.abort(), []); + useEffect(() => { + if (!inspection) return; + const timer = setTimeout( + () => setExpired(true), + Math.max(0, Date.parse(inspection.expiresAt) - Date.now()), + ); + return () => clearTimeout(timer); + }, [inspection]); + useEffect(() => { + const job = query.data; + if ( + !active || + job?.id !== active.jobId || + !owner.active || + !terminal.has(job.status) || + hasUnresolvedJobObservation(job.result) + ) + return; + if (job.status !== "success") { + setError( + job.error ?? "Operation did not complete successfully. Inspect Activity before retrying.", + ); + return; + } + if (active.phase === "preview") { + const parsed = standaloneInstallInspectionSchema.safeParse(job.result); + const expected = active.source; + if ( + !parsed.success || + !expected || + parsed.data.sourceKind !== expected.kind || + (expected.kind === "uploaded-artifact" && + (parsed.data.stageId !== expected.stageId || + parsed.data.archiveSha256 !== expected.expectedSha256)) || + (expected.kind === "artifact" && parsed.data.archiveSha256 !== expected.sha256) || + (expected.kind === "source" && + /^[a-f0-9]{40}$/.test(expected.ref ?? "") && + parsed.data.commit !== expected.ref) + ) { + setError("The inspection response could not be verified."); + return; + } + setInspection(parsed.data); + } else setDone(true); + }, [query.data, active, owner]); + const capture = () => { + const current = generation.current; + return () => owner.active && generation.current === current; + }; + const accept = ( + jobId: string, + phase: "preview" | "install", + current: () => boolean, + source?: StandaloneInstallSource, + ) => { + register({ + client, + apiUrl, + jobId, + onSettled: () => invalidateExtensionAdministration(qc, apiUrl, "components"), + }); + if (current()) setObserved({ jobId, phase, generation: generation.current, owner, source }); + }; + const review = async () => { + if (running.current) return; + if (inspection) reset(true); + running.current = true; + setBusy(true); + setError(null); + const current = capture(); + try { + let source: StandaloneInstallSource; + if (kind === "source") + source = standaloneInstallSourceSchema.parse({ + kind, + repository: url, + ...(ref ? { ref } : {}), + }); + else if (kind === "artifact") + source = standaloneInstallSourceSchema.parse({ kind, url, sha256: sha }); + else { + if (!file || file.size === 0 || file.size > 200 * 1024 ** 2) + throw Error("Select a nonempty integration archive of at most 200 MiB."); + if (uploadedSource.current?.file === file) source = uploadedSource.current.source; + else { + const controller = new AbortController(); + upload.current = controller; + const stage = sdk.unwrapAdminResponse( + await sdk.createAdminArtifactTransfer( + { purpose: "integration-artifact", direction: "upload", declaredBytes: file.size }, + { "idempotency-key": `${keys.current.preview}.upload` }, + { client, signal: controller.signal }, + ), + ); + const uploaded = sdk.unwrapAdminResponse( + await sdk.uploadAdminArtifactTransferContent(stage.stageId, file, { + client, + signal: controller.signal, + }), + ); + upload.current = null; + if (!current()) return; + if (!uploaded.sha256) throw Error("Uploaded archive digest is unavailable."); + source = { kind, stageId: stage.stageId, expectedSha256: uploaded.sha256 }; + uploadedSource.current = { file, source }; + } + } + const result = sdk.unwrapAdminResponse( + await sdk.previewAdminStandaloneIntegration( + { source }, + { "idempotency-key": keys.current.preview }, + { client }, + ), + ); + accept(result.jobId, "preview", current, source); + } catch (error) { + if (current()) setError(error instanceof Error ? error.message : "Inspection failed"); + } finally { + upload.current = null; + running.current = false; + if (owner.active) setBusy(false); + } + }; + const install = async () => { + if (running.current || !inspection || expired || inspection.denials.length) return; + running.current = true; + setBusy(true); + setError(null); + const current = capture(); + try { + const result = sdk.unwrapAdminResponse( + await sdk.installAdminStandaloneIntegration( + { + inspectionId: inspection.inspectionId, + expectedContentDigest: inspection.contentDigest, + }, + { "idempotency-key": keys.current.install }, + { client }, + ), + ); + accept(result.jobId, "install", current); + } catch (error) { + if (current()) setError(error instanceof Error ? error.message : "Installation failed"); + } finally { + running.current = false; + if (owner.active) setBusy(false); + } + }; + return ( + <> + + + Install standalone integration + + + + Community integrations are declarative. Executable code and package scripts are + rejected. + + { + reset(); + setKind(event.target.value as StandaloneInstallSource["kind"]); + }} + > + Source repository + Artifact URL + Upload archive + + {kind === "uploaded-artifact" ? ( + + ) : ( + { + reset(); + setUrl(event.target.value); + }} + /> + )} + {kind === "source" && ( + { + reset(); + setRef(event.target.value); + }} + /> + )} + {kind === "artifact" && ( + { + reset(); + setSha(event.target.value); + }} + /> + )} + {kind === "uploaded-artifact" && ( + + Maximum archive and expanded content: 200 MiB each. + + )} + {inspection && ( + + + {inspection.integrationId} · {inspection.version} + + {inspection.commit && ( + + Resolved commit: {inspection.commit} + + )} + + Content SHA-256: {inspection.contentDigest} + + + This installs or replaces the standalone integration with this ID. Catalog + ownership is preserved. + + {inspection.denials.map((denial) => ( + + {denial} + + ))} + + )} + {expired && ( + Inspection expired. Review the source again. + )} + {(busy || waiting) && ( + + {busy + ? "Submitting…" + : "Operation is running. You can close this dialog and follow it in Activity."} + + )} + {query.error && ( + + Unable to observe the operation. Check Activity before retrying. + + )} + {error && {error}} + {done && Standalone integration installed.} + + + + + {!done && ( + + )} + {inspection && !done && ( + + )} + + + + ); +} diff --git a/docs/docs/administration/community-extensions.md b/docs/docs/administration/community-extensions.md index efb4b627a..843801d19 100644 --- a/docs/docs/administration/community-extensions.md +++ b/docs/docs/administration/community-extensions.md @@ -76,6 +76,27 @@ Installing from the catalog is one click. To install something not in any catalog, use **Install from URL** and paste an `extension.json` URL — that always lands as **community**. +## Standalone integrations + +Use **Extensions → Install integration** for a declarative integration that has +no `extension.json` bundle. Choose an HTTPS repository and optional ref, an HTTPS +artifact URL with its SHA-256, or a local `.tar.gz` archive. Archives are limited +to 200 MiB compressed and expanded. Symbolic links, executable community modules, +incompatible protocols and unsafe archive paths are rejected. + +**Review source** prepares the exact bytes without replacing the installed +integration. Check the integration ID, version, content digest and resolved Git +commit before installing. Reviews expire after ten minutes; review again if one +expires. Uploads remain private to the administrator who created them. + +Installation runs as an Activity job and records community trust plus the actual +source provenance. The agent holds the component and uploaded artifact while the +API reloads the integration and records ownership. A confirmed failure restores +the original directory; an uncertain result keeps the original transaction held +for reconciliation. Closing the dialog only stops observation. Cancellation can +remain pending until restoration is confirmed. Retrying the same request observes +its existing job instead of installing twice. + The same operations are available on the CLI: ```bash diff --git a/packages/cli/__tests__/cli-integrations.test.ts b/packages/cli/__tests__/cli-integrations.test.ts index ac6b84744..ef5874d7a 100644 --- a/packages/cli/__tests__/cli-integrations.test.ts +++ b/packages/cli/__tests__/cli-integrations.test.ts @@ -708,7 +708,7 @@ describe("installIntegration (artifact source)", () => { ).toBe(true); }); - it("extracts safe internal symlinks from artifacts", async () => { + it("rejects internal symlinks under the shared regular-file-only archive policy", async () => { const src = join(tmp, "symlink-artifact-src"); const targetDir = join(src, "assets", "real"); mkdirSync(targetDir, { recursive: true }); @@ -722,15 +722,11 @@ describe("installIntegration (artifact source)", () => { const artifact = join(tmp, "symlink-artifact.tar.gz"); writeTarGz(src, artifact); - await installIntegration({ - source: artifact, - sourceKind: "artifact", - rootDir: tmp, - }); - - const installedLink = join(tmp, "custom_integrations", "symlink-artifact", "assets", "linked"); - expect(lstatSync(installedLink).isSymbolicLink()).toBe(true); - expect(existsSync(join(installedLink, "data.json"))).toBe(true); + await expectArtifactRejection( + artifact, + "symlink-artifact", + /Unsupported artifact entry type '2'/, + ); }); it("rejects artifacts that ship a node_modules directory", async () => { @@ -924,7 +920,7 @@ describe("packageIntegration", () => { }); describe("integration rollback backups", () => { - it("restores the exact pre-update files", () => { + it("preserves a current target without a transaction receipt and restores the backup only after explicit removal", () => { writeIntegration("custom_integrations", "weather-radar", { ...baseManifest, id: "weather-radar", @@ -936,6 +932,11 @@ describe("integration rollback backups", () => { writeFileSync(join(target, "version.txt"), "new", "utf8"); writeFileSync(join(target, "new-only.txt"), "new", "utf8"); + expect(() => restoreInstalledIntegration(tmp, backup as NonNullable)).toThrow( + /ownership receipt/, + ); + expect(readFileSync(join(target, "version.txt"), "utf8")).toBe("new"); + removeIntegration({ rootDir: tmp, id: "weather-radar" }); restoreInstalledIntegration(tmp, backup as NonNullable); expect(readFileSync(join(target, "version.txt"), "utf8")).toBe("old"); diff --git a/packages/core/src/api/contracts/integration-admin.test.ts b/packages/core/src/api/contracts/integration-admin.test.ts index 537956c64..495aaf822 100644 --- a/packages/core/src/api/contracts/integration-admin.test.ts +++ b/packages/core/src/api/contracts/integration-admin.test.ts @@ -11,6 +11,8 @@ beforeEach(() => afterEach(() => vi.unstubAllGlobals()); const at = "2026-10-20T12:00:00.000Z"; const inventory = [ + ["previewAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/preview"], + ["installAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/install"], ["listAdminIntegrations", "GET", "/api/admin/integrations"], ["getAdminIntegration", "GET", "/api/admin/integrations/:id"], ["listAdminIntegrationEnvVars", "GET", "/api/admin/integrations/env-vars"], @@ -171,14 +173,20 @@ const detail = { secretsConfigured: true, }; -function success(id: keyof typeof contracts.integrationAdminOperations, value: unknown) { +function success( + id: Exclude< + keyof typeof contracts.integrationAdminOperations, + "previewAdminStandaloneIntegration" | "installAdminStandaloneIntegration" + >, + value: unknown, +) { const contract = contracts.integrationAdminOperations[id].responses[200]; expect(contract.kind).toBe("json"); if (contract.kind !== "json") throw new Error("Expected JSON"); return contract.schema.parse(value); } describe("integration administration canonical producer contracts", () => { - it("binds exactly the 22 approved operation IDs, methods, paths and admin auth", () => { + it("binds exactly the 24 approved operation IDs, methods, paths and admin auth", () => { expect(contracts).toHaveProperty("integrationAdminOperations"); expect( Object.entries(contracts.integrationAdminOperations).map(([id, op]) => [ @@ -618,3 +626,19 @@ describe("integration administration canonical producer contracts", () => { ).toBe(false); }); }); + +it.each(["previewAdminStandaloneIntegration", "installAdminStandaloneIntegration"] as const)( + "retains standalone durable acceptance for %s", + (id) => { + const op = contracts.integrationAdminOperations[id]; + expect( + op.responses[202].schema.parse({ + jobId: "568ce225-448a-4a8f-86c3-3994d4c6d8a9", + operationId: "99b22375-7d2c-486c-a2f5-d92391ceda2a", + }), + ).toHaveProperty("operationId"); + expect(op.request.headers.safeParse({ "idempotency-key": "fixture-operation" }).success).toBe( + true, + ); + }, +); diff --git a/packages/core/src/api/contracts/integration-admin/index.ts b/packages/core/src/api/contracts/integration-admin/index.ts index c5dfcb7c8..a89b0ddcd 100644 --- a/packages/core/src/api/contracts/integration-admin/index.ts +++ b/packages/core/src/api/contracts/integration-admin/index.ts @@ -4,3 +4,5 @@ export * from "./credentials"; export * from "./health"; export * from "./operations"; export * from "./overview"; + +export * from "./standalone-integrations"; diff --git a/packages/core/src/api/contracts/integration-admin/operations.ts b/packages/core/src/api/contracts/integration-admin/operations.ts index d1281edf1..5aa92bd13 100644 --- a/packages/core/src/api/contracts/integration-admin/operations.ts +++ b/packages/core/src/api/contracts/integration-admin/operations.ts @@ -45,6 +45,8 @@ import { } from "./health"; import { adminOverviewSchema } from "./overview"; +import { standaloneIntegrationAdminOperations } from "./standalone-integrations"; + const common = adminCommonResponses; const missing = { ...common, 404: adminJson(httpErrorSchema) }; const configErrors = adminJson(z.union([httpErrorSchema, adminIntegrationValidationErrorSchema])); @@ -63,6 +65,7 @@ const _ok = adminJson(z.strictObject({ ok: z.literal(true) })); /** Complete integration/credential/overview family. Authentication metadata never enforces access. */ export const integrationAdminOperations = { + ...standaloneIntegrationAdminOperations, listAdminIntegrations: { method: "GET", path: "/api/admin/integrations", diff --git a/packages/core/src/api/contracts/integration-admin/standalone-integrations.ts b/packages/core/src/api/contracts/integration-admin/standalone-integrations.ts new file mode 100644 index 000000000..35845c036 --- /dev/null +++ b/packages/core/src/api/contracts/integration-admin/standalone-integrations.ts @@ -0,0 +1,51 @@ +import z from "zod/v4"; +import { standaloneInstallSourceSchema } from "../../../ops/standalone-integrations"; +import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; +import { adminIntegrationIdempotencyHeadersSchema } from "./common"; +export const standaloneInstallRequestSchema = z.strictObject({ + inspectionId: z.uuid(), + expectedContentDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +export const standalonePreviewRequestSchema = z.strictObject({ + source: standaloneInstallSourceSchema, +}); +export const standaloneJobAcceptedSchema = z.strictObject({ + jobId: z.uuid(), + operationId: z.uuid(), +}); +const responses = { + ...adminCommonResponses, + 202: adminJson(standaloneJobAcceptedSchema), + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), +}; +export const standaloneIntegrationAdminOperations = { + previewAdminStandaloneIntegration: { + method: "POST", + path: "/api/admin/standalone-integrations/preview", + auth: "admin", + request: { + body: standalonePreviewRequestSchema, + headers: adminIntegrationIdempotencyHeadersSchema, + }, + responses, + }, + installAdminStandaloneIntegration: { + method: "POST", + path: "/api/admin/standalone-integrations/install", + auth: "admin", + request: { + body: standaloneInstallRequestSchema, + headers: adminIntegrationIdempotencyHeadersSchema, + }, + responses, + }, +} as const satisfies Record; + +// Browser-safe schemas; clients must not import the server operations barrel. +export { + type StandaloneInstallInspection, + type StandaloneInstallSource, + standaloneInstallInspectionSchema, + standaloneInstallSourceSchema, +} from "../../../ops/standalone-integrations"; diff --git a/packages/core/src/api/generated/admin/integration-admin.generated.ts b/packages/core/src/api/generated/admin/integration-admin.generated.ts index 2df913cbf..004825ce5 100644 --- a/packages/core/src/api/generated/admin/integration-admin.generated.ts +++ b/packages/core/src/api/generated/admin/integration-admin.generated.ts @@ -114,6 +114,18 @@ import type { ImportAdminIntegrationConfigs500, ImportAdminIntegrationConfigsBody, ImportAdminIntegrationConfigsHeaders, + InstallAdminStandaloneIntegration202, + InstallAdminStandaloneIntegration400, + InstallAdminStandaloneIntegration401, + InstallAdminStandaloneIntegration403, + InstallAdminStandaloneIntegration409, + InstallAdminStandaloneIntegration413, + InstallAdminStandaloneIntegration415, + InstallAdminStandaloneIntegration429, + InstallAdminStandaloneIntegration500, + InstallAdminStandaloneIntegration503, + InstallAdminStandaloneIntegrationBody, + InstallAdminStandaloneIntegrationHeaders, ListAdminCredentials200, ListAdminCredentials400, ListAdminCredentials401, @@ -158,6 +170,18 @@ import type { PatchAdminIntegrationConfig500, PatchAdminIntegrationConfigBody, PatchAdminIntegrationConfigHeaders, + PreviewAdminStandaloneIntegration202, + PreviewAdminStandaloneIntegration400, + PreviewAdminStandaloneIntegration401, + PreviewAdminStandaloneIntegration403, + PreviewAdminStandaloneIntegration409, + PreviewAdminStandaloneIntegration413, + PreviewAdminStandaloneIntegration415, + PreviewAdminStandaloneIntegration429, + PreviewAdminStandaloneIntegration500, + PreviewAdminStandaloneIntegration503, + PreviewAdminStandaloneIntegrationBody, + PreviewAdminStandaloneIntegrationHeaders, ReloadAdminIntegration200, ReloadAdminIntegration400, ReloadAdminIntegration401, @@ -234,11 +258,13 @@ import { getAdminIntegrationRequest, getAdminOverviewRequest, importAdminIntegrationConfigsRequest, + installAdminStandaloneIntegrationRequest, listAdminCredentialsRequest, listAdminIntegrationBindingsRequest, listAdminIntegrationEnvVarsRequest, listAdminIntegrationsRequest, patchAdminIntegrationConfigRequest, + previewAdminStandaloneIntegrationRequest, reloadAdminIntegrationRequest, reloadAdminIntegrationsRequest, runAdminIntegrationHealthChecksRequest, @@ -2340,3 +2366,225 @@ export const getAdminOverview = async ( method: "GET", }); }; + +export type installAdminStandaloneIntegrationResponse202 = { + data: InstallAdminStandaloneIntegration202; + status: 202; +}; + +export type installAdminStandaloneIntegrationResponse400 = { + data: InstallAdminStandaloneIntegration400; + status: 400; +}; + +export type installAdminStandaloneIntegrationResponse401 = { + data: InstallAdminStandaloneIntegration401; + status: 401; +}; + +export type installAdminStandaloneIntegrationResponse403 = { + data: InstallAdminStandaloneIntegration403; + status: 403; +}; + +export type installAdminStandaloneIntegrationResponse409 = { + data: InstallAdminStandaloneIntegration409; + status: 409; +}; + +export type installAdminStandaloneIntegrationResponse413 = { + data: InstallAdminStandaloneIntegration413; + status: 413; +}; + +export type installAdminStandaloneIntegrationResponse415 = { + data: InstallAdminStandaloneIntegration415; + status: 415; +}; + +export type installAdminStandaloneIntegrationResponse429 = { + data: InstallAdminStandaloneIntegration429; + status: 429; +}; + +export type installAdminStandaloneIntegrationResponse500 = { + data: InstallAdminStandaloneIntegration500; + status: 500; +}; + +export type installAdminStandaloneIntegrationResponse503 = { + data: InstallAdminStandaloneIntegration503; + status: 503; +}; + +export type installAdminStandaloneIntegrationResponseSuccess = + installAdminStandaloneIntegrationResponse202 & { + headers: Headers; + }; +export type installAdminStandaloneIntegrationResponseError = ( + | installAdminStandaloneIntegrationResponse400 + | installAdminStandaloneIntegrationResponse401 + | installAdminStandaloneIntegrationResponse403 + | installAdminStandaloneIntegrationResponse409 + | installAdminStandaloneIntegrationResponse413 + | installAdminStandaloneIntegrationResponse415 + | installAdminStandaloneIntegrationResponse429 + | installAdminStandaloneIntegrationResponse500 + | installAdminStandaloneIntegrationResponse503 +) & { + headers: Headers; +}; + +export type installAdminStandaloneIntegrationResponse = + | installAdminStandaloneIntegrationResponseSuccess + | installAdminStandaloneIntegrationResponseError; + +export const getInstallAdminStandaloneIntegrationUrl = () => { + return `/api/admin/standalone-integrations/install`; +}; + +export const installAdminStandaloneIntegration = async ( + installAdminStandaloneIntegrationBody: InstallAdminStandaloneIntegrationBody, + headers: InstallAdminStandaloneIntegrationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return installAdminStandaloneIntegrationRequest( + getInstallAdminStandaloneIntegrationUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(installAdminStandaloneIntegrationBody), + }, + ); +}; + +export type previewAdminStandaloneIntegrationResponse202 = { + data: PreviewAdminStandaloneIntegration202; + status: 202; +}; + +export type previewAdminStandaloneIntegrationResponse400 = { + data: PreviewAdminStandaloneIntegration400; + status: 400; +}; + +export type previewAdminStandaloneIntegrationResponse401 = { + data: PreviewAdminStandaloneIntegration401; + status: 401; +}; + +export type previewAdminStandaloneIntegrationResponse403 = { + data: PreviewAdminStandaloneIntegration403; + status: 403; +}; + +export type previewAdminStandaloneIntegrationResponse409 = { + data: PreviewAdminStandaloneIntegration409; + status: 409; +}; + +export type previewAdminStandaloneIntegrationResponse413 = { + data: PreviewAdminStandaloneIntegration413; + status: 413; +}; + +export type previewAdminStandaloneIntegrationResponse415 = { + data: PreviewAdminStandaloneIntegration415; + status: 415; +}; + +export type previewAdminStandaloneIntegrationResponse429 = { + data: PreviewAdminStandaloneIntegration429; + status: 429; +}; + +export type previewAdminStandaloneIntegrationResponse500 = { + data: PreviewAdminStandaloneIntegration500; + status: 500; +}; + +export type previewAdminStandaloneIntegrationResponse503 = { + data: PreviewAdminStandaloneIntegration503; + status: 503; +}; + +export type previewAdminStandaloneIntegrationResponseSuccess = + previewAdminStandaloneIntegrationResponse202 & { + headers: Headers; + }; +export type previewAdminStandaloneIntegrationResponseError = ( + | previewAdminStandaloneIntegrationResponse400 + | previewAdminStandaloneIntegrationResponse401 + | previewAdminStandaloneIntegrationResponse403 + | previewAdminStandaloneIntegrationResponse409 + | previewAdminStandaloneIntegrationResponse413 + | previewAdminStandaloneIntegrationResponse415 + | previewAdminStandaloneIntegrationResponse429 + | previewAdminStandaloneIntegrationResponse500 + | previewAdminStandaloneIntegrationResponse503 +) & { + headers: Headers; +}; + +export type previewAdminStandaloneIntegrationResponse = + | previewAdminStandaloneIntegrationResponseSuccess + | previewAdminStandaloneIntegrationResponseError; + +export const getPreviewAdminStandaloneIntegrationUrl = () => { + return `/api/admin/standalone-integrations/preview`; +}; + +export const previewAdminStandaloneIntegration = async ( + previewAdminStandaloneIntegrationBody: PreviewAdminStandaloneIntegrationBody, + headers: PreviewAdminStandaloneIntegrationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return previewAdminStandaloneIntegrationRequest( + getPreviewAdminStandaloneIntegrationUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(previewAdminStandaloneIntegrationBody), + }, + ); +}; diff --git a/packages/core/src/api/generated/admin/integration-models/index.ts b/packages/core/src/api/generated/admin/integration-models/index.ts index 41021ba30..1566cb4d6 100644 --- a/packages/core/src/api/generated/admin/integration-models/index.ts +++ b/packages/core/src/api/generated/admin/integration-models/index.ts @@ -225,6 +225,18 @@ export * from "./importAdminIntegrationConfigsBody"; export * from "./importAdminIntegrationConfigsBodyIntegrationsItem"; export * from "./importAdminIntegrationConfigsBodyIntegrationsItemConfig"; export * from "./importAdminIntegrationConfigsHeaders"; +export * from "./installAdminStandaloneIntegration202"; +export * from "./installAdminStandaloneIntegration400"; +export * from "./installAdminStandaloneIntegration401"; +export * from "./installAdminStandaloneIntegration403"; +export * from "./installAdminStandaloneIntegration409"; +export * from "./installAdminStandaloneIntegration413"; +export * from "./installAdminStandaloneIntegration415"; +export * from "./installAdminStandaloneIntegration429"; +export * from "./installAdminStandaloneIntegration500"; +export * from "./installAdminStandaloneIntegration503"; +export * from "./installAdminStandaloneIntegrationBody"; +export * from "./installAdminStandaloneIntegrationHeaders"; export * from "./listAdminCredentials200"; export * from "./listAdminCredentials200CredentialsItem"; export * from "./listAdminCredentials400"; @@ -285,6 +297,19 @@ export * from "./patchAdminIntegrationConfig429"; export * from "./patchAdminIntegrationConfig500"; export * from "./patchAdminIntegrationConfigBody"; export * from "./patchAdminIntegrationConfigHeaders"; +export * from "./previewAdminStandaloneIntegration202"; +export * from "./previewAdminStandaloneIntegration400"; +export * from "./previewAdminStandaloneIntegration401"; +export * from "./previewAdminStandaloneIntegration403"; +export * from "./previewAdminStandaloneIntegration409"; +export * from "./previewAdminStandaloneIntegration413"; +export * from "./previewAdminStandaloneIntegration415"; +export * from "./previewAdminStandaloneIntegration429"; +export * from "./previewAdminStandaloneIntegration500"; +export * from "./previewAdminStandaloneIntegration503"; +export * from "./previewAdminStandaloneIntegrationBody"; +export * from "./previewAdminStandaloneIntegrationBodySource"; +export * from "./previewAdminStandaloneIntegrationHeaders"; export * from "./reloadAdminIntegration200"; export * from "./reloadAdminIntegration400"; export * from "./reloadAdminIntegration401"; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration202.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration202.ts new file mode 100644 index 000000000..e9c04c0fa --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration202.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + operationId: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration400.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration400.ts new file mode 100644 index 000000000..df7559e21 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration401.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration401.ts new file mode 100644 index 000000000..50ee5f48f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration403.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration403.ts new file mode 100644 index 000000000..f46c5f407 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration409.ts new file mode 100644 index 000000000..b42d0e0ff --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration413.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration413.ts new file mode 100644 index 000000000..a07abe5aa --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration415.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration415.ts new file mode 100644 index 000000000..0f37e6672 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration429.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration429.ts new file mode 100644 index 000000000..8e9da64a2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration500.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration500.ts new file mode 100644 index 000000000..0459bec2a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration503.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration503.ts new file mode 100644 index 000000000..38ea4da56 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationBody.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationBody.ts new file mode 100644 index 000000000..f0d33f241 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationBody.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegrationBody = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + inspectionId: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedContentDigest: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationHeaders.ts new file mode 100644 index 000000000..19bb6e9d9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/installAdminStandaloneIntegrationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type InstallAdminStandaloneIntegrationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration202.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration202.ts new file mode 100644 index 000000000..45552f96c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration202.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + operationId: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration400.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration400.ts new file mode 100644 index 000000000..744b5828d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration401.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration401.ts new file mode 100644 index 000000000..ba9f95f92 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration403.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration403.ts new file mode 100644 index 000000000..d60fbac37 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration409.ts new file mode 100644 index 000000000..f9bf79615 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration413.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration413.ts new file mode 100644 index 000000000..41857788f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration415.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration415.ts new file mode 100644 index 000000000..f90d21c12 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration429.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration429.ts new file mode 100644 index 000000000..a1c9e23e8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration500.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration500.ts new file mode 100644 index 000000000..9ae6e4bf2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration503.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration503.ts new file mode 100644 index 000000000..dfc569fc9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBody.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBody.ts new file mode 100644 index 000000000..0d51ded0c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBody.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminStandaloneIntegrationBodySource } from "./previewAdminStandaloneIntegrationBodySource"; + +export type PreviewAdminStandaloneIntegrationBody = { + source: PreviewAdminStandaloneIntegrationBodySource; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBodySource.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBodySource.ts new file mode 100644 index 000000000..a97d8d36f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationBodySource.ts @@ -0,0 +1,25 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegrationBodySource = + | { + kind: "artifact"; + /** @maxLength 2048 */ + url: string; + /** @pattern ^[a-f0-9]{64}$ */ + sha256: string; + } + | { + kind: "source"; + /** @maxLength 2048 */ + repository: string; + /** @pattern ^[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$ */ + ref?: string; + } + | { + kind: "uploaded-artifact"; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + stageId: string; + /** @pattern ^[a-f0-9]{64}$ */ + expectedSha256: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationHeaders.ts new file mode 100644 index 000000000..59b49e921 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminStandaloneIntegrationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminStandaloneIntegrationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index eb7d2983a..465719a82 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -461,6 +461,15 @@ export async function installAdminExtensionRequest( ): Promise { return installAdminExtensionBound(url, options); } +const installAdminStandaloneIntegrationBound = bindAdminOperation( + "installAdminStandaloneIntegration", +); +export async function installAdminStandaloneIntegrationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return installAdminStandaloneIntegrationBound(url, options); +} const inviteAdminAccountBound = bindAdminOperation("inviteAdminAccount"); export async function inviteAdminAccountRequest( url: string, @@ -684,6 +693,15 @@ export async function previewAdminOperationRequest( ): Promise { return previewAdminOperationBound(url, options); } +const previewAdminStandaloneIntegrationBound = bindAdminOperation( + "previewAdminStandaloneIntegration", +); +export async function previewAdminStandaloneIntegrationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return previewAdminStandaloneIntegrationBound(url, options); +} const previewAdminTransitCredentialsBound = bindAdminOperation("previewAdminTransitCredentials"); export async function previewAdminTransitCredentialsRequest( url: string, diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 57ca3d8f6..a7ce14ed5 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -2059,7 +2059,12 @@ export function retainControlEffectReceipt( if ( !result.outcome || ["applied", "unchanged", "committed"].includes(result.outcome) || - (s.kind === "transitousLock.approve" && result.outcome === "activated") + (s.kind === "transitousLock.approve" && result.outcome === "activated") || + // These closed results settle their own awaited filesystem leaf. A + // published install still holds its component lock until API bookkeeping + // selects the separate commit/restore operation. + (s.kind === "integration.install" && result.outcome === "published") || + (s.kind === "integration.settle" && result.outcome === "restored") ) { outcome = "committed"; publication = "committed"; diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 35fe9cb1b..d08e75c7e 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -66,6 +66,28 @@ const transferClaim = { bytes: 4, }; const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { + "integration.inspectInstall": { + kind: "integration.inspectInstall", + ownerId: "admin", + source: { + kind: "artifact", + url: "https://example.com/integration.tar.gz", + sha256: "a".repeat(64), + }, + }, + "integration.install": { + kind: "integration.install", + ownerId: "admin", + inspectionId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + expectedContentDigest: "a".repeat(64), + operationId: "81a78b3e-208b-4ba4-9ddf-870fbdf1da22", + }, + "integration.settle": { + kind: "integration.settle", + transactionId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + decision: "commit", + expectedContentDigest: "a".repeat(64), + }, "artifactTransfer.create": { kind: "artifactTransfer.create", ownerId: "admin", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 403b750ee..4089248fd 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -24,6 +24,10 @@ import { vaultApplyOperationSchema, } from "./configuration-operation-records"; import { dataWorkflowReceiptSchema } from "./job-evidence"; +import { + standaloneIntegrationOperationSchemas, + standaloneIntegrationResultSchemas, +} from "./standalone-integrations"; export { configurationGenerationRevisionSchema } from "./configuration-operation-records"; @@ -65,6 +69,9 @@ export const OPS_MAX_BACKUP_ID_LENGTH = 128; export const OPS_MAX_BACKUP_INVENTORY_ENTRIES = 128; export const OPS_OPERATION_KINDS = [ + "integration.inspectInstall", + "integration.install", + "integration.settle", "artifactTransfer.create", "artifactTransfer.list", "artifactTransfer.inspect", @@ -203,6 +210,9 @@ const asyncEffect = ( const LIFECYCLE_TIMEOUT_MS = 10 * 60_000 + 30_000; export const OPS_KIND_POLICIES = { + "integration.inspectInstall": asyncEffect("api", 10 * 60_000), + "integration.install": asyncEffect("api", 10 * 60_000), + "integration.settle": asyncEffect("api", 60_000), "artifactTransfer.create": sync("api", 30_000), "artifactTransfer.list": sync("api", 30_000), "artifactTransfer.inspect": sync("api", 30_000), @@ -380,6 +390,7 @@ const serviceOperation = (kind: T) => z.strictObject({ kind: z.literal(kind), serviceId: serviceIdSchema }); export const opsOperationSchema = z.discriminatedUnion("kind", [ + ...standaloneIntegrationOperationSchemas, ...artifactTransferOperationSchemas, z.strictObject({ kind: z.literal("controlRetention.acknowledge"), @@ -1027,6 +1038,7 @@ const pinnedImageReference = z .regex(/^[a-z0-9][a-z0-9._:/-]*@sha256:[a-f0-9]{64}$/); export const OPS_RESULT_SCHEMAS = { + ...standaloneIntegrationResultSchemas, ...artifactTransferResultSchemas, "controlRetention.acknowledge": z.strictObject({ acknowledged: z.literal(true) }), "docker.status": z.strictObject({ diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index 635c46836..addd6264e 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -21,5 +21,6 @@ export * from "./job-evidence"; export * from "./privacy-backup"; export * from "./service-operation-records"; export * from "./service-results"; +export * from "./standalone-integrations"; export * from "./trusted-config"; export * from "./trusted-config-queue-lock"; diff --git a/packages/core/src/ops/standalone-integrations.test.ts b/packages/core/src/ops/standalone-integrations.test.ts new file mode 100644 index 000000000..07374de3f --- /dev/null +++ b/packages/core/src/ops/standalone-integrations.test.ts @@ -0,0 +1,26 @@ +import { expect, it } from "vitest"; +import { OPS_OPERATION_KINDS, parseOpsRequest } from "./contract"; + +it("admits only the finite API-owned standalone inspection/install/settlement operations", () => { + for (const kind of ["integration.inspectInstall", "integration.install", "integration.settle"]) + expect(OPS_OPERATION_KINDS).toContain(kind); + const operation = { + kind: "integration.inspectInstall", + ownerId: "admin", + source: { + kind: "uploaded-artifact", + stageId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + expectedSha256: "a".repeat(64), + }, + }; + const request = { + version: 1, + requestId: "ops1_abcdefghijklmnop", + operationKey: "opk1_abcdefghijklmnop", + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 30000).toISOString(), + operation, + }; + expect(() => parseOpsRequest(request, { role: "api" })).not.toThrow(); + expect(() => parseOpsRequest(request, { role: "data-manager" })).toThrow(); +}); diff --git a/packages/core/src/ops/standalone-integrations.ts b/packages/core/src/ops/standalone-integrations.ts new file mode 100644 index 000000000..93b88f42f --- /dev/null +++ b/packages/core/src/ops/standalone-integrations.ts @@ -0,0 +1,118 @@ +import z from "zod/v4"; +import { assertAllowedGitUrl } from "../git-url"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const owner = z.string().min(1).max(128); +const component = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const remote = z + .url() + .max(2048) + .refine((value) => { + try { + const url = new URL(value); + return ( + url.protocol === "https:" && !url.username && !url.password && !url.hash && !url.search + ); + } catch { + return false; + } + }); +const repository = remote.refine((value) => { + try { + assertAllowedGitUrl(value); + return true; + } catch { + return false; + } +}); +export const standaloneInstallSourceSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("artifact"), url: remote, sha256: digest }), + z.strictObject({ + kind: z.literal("source"), + repository, + ref: z + .string() + .regex(/^[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$/) + .optional(), + }), + z.strictObject({ + kind: z.literal("uploaded-artifact"), + stageId: z.uuid(), + expectedSha256: digest, + }), +]); +export const standaloneInstallInspectionSchema = z.strictObject({ + inspectionId: z.uuid(), + sourceKind: z.enum(["artifact", "source", "uploaded-artifact"]), + integrationId: component, + version: z.string().min(1).max(128), + contentDigest: digest, + commit: z + .string() + .regex(/^[a-f0-9]{40}$/) + .nullable(), + stageId: z.uuid().nullable(), + archiveSha256: digest.nullable(), + expiresAt: z.iso.datetime(), + denials: z.array(z.string().max(512)).max(64), +}); +export const standaloneIntegrationProvenanceSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("artifact"), + url: remote, + artifactSha256: digest, + contentDigest: digest, + }), + z.strictObject({ + kind: z.literal("source"), + url: repository, + commit: z.string().regex(/^[a-f0-9]{40}$/), + contentDigest: digest, + }), + z.strictObject({ + kind: z.literal("uploaded-artifact"), + stageId: z.uuid(), + artifactSha256: digest, + contentDigest: digest, + }), +]); +export type StandaloneIntegrationProvenance = z.infer; +export const standaloneInstallReceiptSchema = z.strictObject({ + transactionId: z.uuid(), + integrationId: component, + previousContentDigest: digest.nullable(), + contentDigest: digest, + outcome: z.literal("published"), +}); +export const standaloneIntegrationOperationSchemas = [ + z.strictObject({ + kind: z.literal("integration.inspectInstall"), + source: standaloneInstallSourceSchema, + ownerId: owner, + }), + z.strictObject({ + kind: z.literal("integration.install"), + inspectionId: z.uuid(), + ownerId: owner, + expectedContentDigest: digest, + operationId: z.uuid(), + }), + z.strictObject({ + kind: z.literal("integration.settle"), + transactionId: z.uuid(), + decision: z.enum(["commit", "restore"]), + expectedContentDigest: digest, + }), +] as const; +export const standaloneIntegrationResultSchemas = { + "integration.inspectInstall": standaloneInstallInspectionSchema, + "integration.install": standaloneInstallReceiptSchema, + "integration.settle": z.strictObject({ + transactionId: z.uuid(), + outcome: z.enum(["committed", "restored"]), + contentDigest: digest.nullable(), + }), +} as const; +export type StandaloneInstallSource = z.infer; +export type StandaloneInstallInspection = z.infer; +export type StandaloneInstallReceipt = z.infer; diff --git a/packages/integration-framework/src/__tests__/installer-source.test.ts b/packages/integration-framework/src/__tests__/installer-source.test.ts new file mode 100644 index 000000000..678ce039c --- /dev/null +++ b/packages/integration-framework/src/__tests__/installer-source.test.ts @@ -0,0 +1,103 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; + +const clone = vi.hoisted(() => vi.fn()); +vi.mock("@openmapx/core/server", async (actual) => ({ + ...(await actual()), + gitShallowCloneSnapshot: clone, +})); + +import { + createIntegrationArtifactPreparation, + prepareIntegrationSource, + publishPreparedIntegration, + settleIntegrationPublication, +} from "../installer"; + +const roots: string[] = []; +afterEach(() => { + roots.splice(0).forEach((root) => { + rmSync(root, { recursive: true, force: true }); + }); + clone.mockReset(); +}); +it("retains the once-resolved source commit through actual declarative publication", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-source-preparation-")); + roots.push(root); + mkdirSync(join(root, "services")); + clone.mockImplementation(async ({ targetDir }: { targetDir: string }) => { + mkdirSync(targetDir, { mode: 0o755 }); + writeFileSync( + join(targetDir, "manifest.json"), + JSON.stringify({ + id: "source-fixture", + name: "Source", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + { mode: 0o644 }, + ); + return { + directory: targetDir, + canonicalUrl: "https://github.com/public/fixture.git", + commit: "a".repeat(40), + }; + }); + const signal = new AbortController().signal; + const result = await prepareIntegrationSource( + { repository: "https://github.com/public/fixture.git", ref: "main", signal }, + createIntegrationArtifactPreparation({ rootDir: root }), + ); + expect(result).toMatchObject({ + commit: "a".repeat(40), + prepared: { id: "source-fixture", archiveSha256: null }, + }); + clone.mockRejectedValue(new Error("Mutable branch now points elsewhere")); + const publication = await publishPreparedIntegration(result.prepared, { + transactionId: "9a0826b5-4478-4b9d-88b8-a3bc1b9755f6", + expectedPreviousContentDigest: null, + signal, + }); + await settleIntegrationPublication(publication, "commit", signal); + expect(clone).toHaveBeenCalledTimes(1); + expect( + JSON.parse( + readFileSync(join(root, "custom_integrations", "source-fixture", "manifest.json"), "utf8"), + ).id, + ).toBe("source-fixture"); +}); +it("rejects executable source without publishing it", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-source-rejection-")); + roots.push(root); + mkdirSync(join(root, "services")); + clone.mockImplementation(async ({ targetDir }: { targetDir: string }) => { + mkdirSync(targetDir); + writeFileSync( + join(targetDir, "manifest.json"), + JSON.stringify({ + id: "source-fixture", + name: "Source", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + writeFileSync(join(targetDir, "index.js"), "throw Error('never execute')"); + return { + directory: targetDir, + canonicalUrl: "https://github.com/public/fixture.git", + commit: "a".repeat(40), + }; + }); + await expect( + prepareIntegrationSource( + { repository: "https://github.com/public/fixture.git", signal: new AbortController().signal }, + createIntegrationArtifactPreparation({ rootDir: root }), + ), + ).rejects.toThrow(/code|declarative|executable/i); +}); diff --git a/packages/integration-framework/src/__tests__/installer-streaming.test.ts b/packages/integration-framework/src/__tests__/installer-streaming.test.ts new file mode 100644 index 000000000..183b8c9b2 --- /dev/null +++ b/packages/integration-framework/src/__tests__/installer-streaming.test.ts @@ -0,0 +1,444 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { createDeterministicTarGz } from "../deterministic-tar"; +import { installIntegration } from "../installer"; + +const calls = vi.hoisted(() => ({ + watched: "", + wholeReads: 0, + removeFault: "", + unlinkFault: false, +})); +vi.mock("node:fs", async (actual) => { + const fs = await actual(); + return { + ...fs, + rmSync: (...args: Parameters) => { + if (calls.removeFault && String(args[0]).endsWith(calls.removeFault)) { + calls.removeFault = ""; + throw new Error("named private cleanup failure"); + } + return Reflect.apply(fs.rmSync, fs, args); + }, + unlinkSync: (...args: Parameters) => { + const result = Reflect.apply(fs.unlinkSync, fs, args); + if (calls.unlinkFault && String(args[0]).endsWith("/owner.json")) { + calls.unlinkFault = false; + throw new Error("named owner unlink acknowledgement loss"); + } + return result; + }, + readFileSync: (...args: Parameters) => { + if (String(args[0]) === calls.watched) calls.wholeReads++; + return Reflect.apply(fs.readFileSync, fs, args); + }, + }; +}); +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); + calls.watched = ""; + calls.wholeReads = 0; + calls.removeFault = ""; + calls.unlinkFault = false; +}); +it("installs a valid existing artifact without whole-file archive reads or gzip allocation", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-integration-stream-")); + roots.push(root); + mkdirSync(join(root, "services")); + const archive = join(root, "public.tar.gz"); + const manifest = { + id: "stream-fixture", + name: "Stream fixture", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }; + const bytes = createDeterministicTarGz([ + { path: "manifest.json", contents: Buffer.from(JSON.stringify(manifest)) }, + ]); + writeFileSync(archive, bytes); + calls.watched = archive; + const result = await installIntegration({ + rootDir: root, + source: archive, + sourceKind: "artifact", + artifactSha256: createHash("sha256").update(bytes).digest("hex"), + }); + expect(result.id).toBe(manifest.id); + expect(JSON.parse(readFileSync(join(result.directory, "manifest.json"), "utf8"))).toEqual( + manifest, + ); + expect(calls.wholeReads).toBe(0); +}); +it("returns ownership of an unopened byte source when cancellation precedes extraction", async () => { + const { extractIntegrationArtifactStream } = await import("../artifact-stream"); + const root = mkdtempSync(join(tmpdir(), "openmapx-integration-abort-")); + roots.push(root); + let returned = false, + pulled = 0; + const source: AsyncIterable = { + [Symbol.asyncIterator]: () => ({ + next: async () => { + pulled++; + return { done: true, value: undefined }; + }, + return: async () => { + returned = true; + return { done: true, value: undefined }; + }, + }), + }; + const abort = new AbortController(); + abort.abort(); + await expect( + extractIntegrationArtifactStream({ + source, + destination: root, + maxArtifactBytes: 100, + maxExpandedBytes: 100, + signal: abort.signal, + }), + ).rejects.toBeDefined(); + expect(pulled).toBe(0); + expect(returned).toBe(true); +}); +it("prepares immutable bytes without publishing and refuses a changed retained preparation", async () => { + const installer = await import("../installer"); + const root = mkdtempSync(join(tmpdir(), "openmapx-integration-prepare-")); + roots.push(root); + mkdirSync(join(root, "services")); + const bytes = createDeterministicTarGz([ + { + path: "manifest.json", + contents: Buffer.from( + JSON.stringify({ + id: "prepared-fixture", + name: "Prepared", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ), + }, + ]); + const context = installer.createIntegrationArtifactPreparation({ rootDir: root }); + const prepared = await installer.prepareIntegrationArtifact( + { + source: (async function* () { + for (let offset = 0; offset < bytes.length; offset += 17) + yield bytes.subarray(offset, offset + 17); + })(), + expectedSha256: createHash("sha256").update(bytes).digest("hex"), + maxArtifactBytes: 200 * 1024 ** 2, + maxExpandedBytes: 200 * 1024 ** 2, + signal: new AbortController().signal, + }, + context, + ); + expect(prepared.id).toBe("prepared-fixture"); + const { existsSync } = await import("node:fs"); + expect(existsSync(join(root, "custom_integrations", "prepared-fixture"))).toBe(false); + const receipt = installer.describePreparedIntegration(prepared); + const restored = await installer.restorePreparedIntegration({ + rootDir: root, + receipt, + signal: new AbortController().signal, + }); + expect(restored.contentDigest).toBe(prepared.contentDigest); + writeFileSync( + join( + root, + "custom_integrations", + ".staging", + `preparation-${receipt.preparationId}`, + "content", + "manifest.json", + ), + "{}", + ); + await expect( + installer.restorePreparedIntegration({ + rootDir: root, + receipt, + signal: new AbortController().signal, + }), + ).rejects.toThrow(); + installer.discardIntegrationPreparation(context); +}); +it("holds a shared component lock across publication and permits only exact-owner restoration", async () => { + const installer = await import("../installer"); + const root = mkdtempSync(join(tmpdir(), "openmapx-integration-publish-")); + roots.push(root); + mkdirSync(join(root, "services")); + const manifest = { + id: "transaction-fixture", + name: "Transaction", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }; + const bytes = createDeterministicTarGz([ + { path: "manifest.json", contents: Buffer.from(JSON.stringify(manifest)) }, + ]); + const context = installer.createIntegrationArtifactPreparation({ rootDir: root }); + const prepared = await installer.prepareIntegrationArtifact( + { + source: (async function* () { + yield bytes; + })(), + expectedSha256: createHash("sha256").update(bytes).digest("hex"), + maxArtifactBytes: 200 * 1024 ** 2, + maxExpandedBytes: 200 * 1024 ** 2, + signal: new AbortController().signal, + }, + context, + ); + const transaction = await installer.publishPreparedIntegration(prepared, { + transactionId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + expectedPreviousContentDigest: null, + signal: new AbortController().signal, + }); + expect(installer.describeIntegrationPublication(transaction)).toMatchObject({ + integrationId: manifest.id, + outcome: "published", + preparation: { contentDigest: prepared.contentDigest }, + }); + expect(() => installer.removeIntegration({ rootDir: root, id: manifest.id })).toThrow(/busy/); + const saved = installer.describeIntegrationPublication(transaction); + const recovered = await installer.restoreIntegrationPublication({ + rootDir: root, + receipt: saved, + signal: new AbortController().signal, + }); + await installer.settleIntegrationPublication(recovered, "restore", new AbortController().signal); + const { existsSync } = await import("node:fs"); + expect(existsSync(join(root, "custom_integrations", manifest.id))).toBe(false); + expect(() => installer.removeIntegration({ rootDir: root, id: manifest.id })).toThrow( + /not installed/, + ); +}); + +async function publicationFixture(version = "1.0.0", rootDir?: string) { + const installer = await import("../installer"); + const root = rootDir ?? mkdtempSync(join(tmpdir(), "openmapx-integration-replacement-")); + if (!rootDir) { + roots.push(root); + mkdirSync(join(root, "services")); + } + const manifest = { + id: "replacement-fixture", + name: "Replacement", + version, + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }; + const bytes = createDeterministicTarGz([ + { path: "manifest.json", contents: Buffer.from(JSON.stringify(manifest)) }, + ]); + const context = installer.createIntegrationArtifactPreparation({ rootDir: root }); + const prepared = await installer.prepareIntegrationArtifact( + { + source: (async function* () { + yield bytes; + })(), + expectedSha256: createHash("sha256").update(bytes).digest("hex"), + maxArtifactBytes: 200 * 1024 ** 2, + maxExpandedBytes: 200 * 1024 ** 2, + signal: new AbortController().signal, + }, + context, + ); + const previous = installer.inspectInstalledIntegrationFingerprint({ + rootDir: root, + id: manifest.id, + signal: new AbortController().signal, + }); + return { installer, root, prepared, previous, signal: new AbortController().signal }; +} +it("replays a committed publication after restart without releasing a later writer's lock", async () => { + const f = await publicationFixture(); + const publication = await f.installer.publishPreparedIntegration(f.prepared, { + transactionId: "944c81db-75da-418a-aef8-f839174cf87f", + expectedPreviousContentDigest: null, + signal: f.signal, + }); + const receipt = f.installer.describeIntegrationPublication(publication); + await f.installer.settleIntegrationPublication(publication, "commit", f.signal); + const later = f.installer.acquireIntegrationComponentLock({ rootDir: f.root, id: f.prepared.id }); + try { + const recovered = await f.installer.restoreIntegrationPublication({ + rootDir: f.root, + receipt, + signal: f.signal, + }); + expect( + await f.installer.settleIntegrationPublication(recovered, "commit", f.signal), + ).toMatchObject({ outcome: "committed", contentDigest: f.prepared.contentDigest }); + expect(() => f.installer.removeIntegration({ rootDir: f.root, id: f.prepared.id })).toThrow( + /busy/, + ); + await expect( + f.installer.settleIntegrationPublication(recovered, "restore", f.signal), + ).rejects.toThrow(/different decision/); + } finally { + f.installer.releaseIntegrationComponentLock(later); + } +}); +it("restores the exact prior directory on replacement and refuses changed published bytes", async () => { + const first = await publicationFixture(); + const old = await first.installer.publishPreparedIntegration(first.prepared, { + transactionId: "26ffcae8-e6d3-414d-bf09-a37162591a80", + expectedPreviousContentDigest: null, + signal: first.signal, + }); + await first.installer.settleIntegrationPublication(old, "commit", first.signal); + const next = await publicationFixture("2.0.0", first.root); + const published = await next.installer.publishPreparedIntegration(next.prepared, { + transactionId: "ff42f4ee-d8c9-4b4d-a94f-e5ced70e5356", + expectedPreviousContentDigest: first.prepared.contentDigest, + signal: next.signal, + }); + const manifestPath = join(first.root, "custom_integrations", first.prepared.id, "manifest.json"); + const original = readFileSync(manifestPath); + writeFileSync(manifestPath, "{}"); + await expect( + next.installer.settleIntegrationPublication(published, "restore", next.signal), + ).rejects.toThrow(/changed/); + expect(() => + next.installer.removeIntegration({ rootDir: first.root, id: first.prepared.id }), + ).toThrow(/busy/); + writeFileSync(manifestPath, original); + await next.installer.settleIntegrationPublication(published, "restore", next.signal); + expect( + next.installer.inspectInstalledIntegrationFingerprint({ + rootDir: first.root, + id: first.prepared.id, + signal: next.signal, + }), + ).toEqual(next.previous); + const replay = await next.installer.restoreIntegrationPublication({ + rootDir: first.root, + receipt: next.installer.describeIntegrationPublication(published), + signal: next.signal, + }); + expect( + await next.installer.settleIntegrationPublication(replay, "restore", next.signal), + ).toMatchObject({ outcome: "restored", contentDigest: first.prepared.contentDigest }); +}); + +it("closes an unchanged pre-publication transaction when its retained callback fails", async () => { + const f = await publicationFixture(); + const preparation = f.installer.describePreparedIntegration(f.prepared); + await expect( + f.installer.publishPreparedIntegration(f.prepared, { + transactionId: "91b603d6-37d0-46c5-b3ed-5c1f72976fb8", + expectedPreviousContentDigest: null, + signal: f.signal, + beforePublish: () => { + throw Error("named retained checkpoint failure"); + }, + }), + ).rejects.toThrow("named retained checkpoint failure"); + const receipt = f.installer.inspectIntegrationPublication({ + rootDir: f.root, + preparation, + transactionId: "91b603d6-37d0-46c5-b3ed-5c1f72976fb8", + }); + expect(receipt.outcome).toBe("restored"); + const restored = await f.installer.restoreIntegrationPublication({ + rootDir: f.root, + receipt, + signal: f.signal, + }); + expect( + await f.installer.settleIntegrationPublication(restored, "restore", f.signal), + ).toMatchObject({ outcome: "restored" }); +}); +it("retries only original private cleanup after a restored terminal was durably written", async () => { + const f = await publicationFixture(); + const published = await f.installer.publishPreparedIntegration(f.prepared, { + transactionId: "7cc570a9-17b6-4905-b0a2-2da4cbd5c2cf", + expectedPreviousContentDigest: null, + signal: f.signal, + }); + calls.removeFault = "/rejected"; + await expect( + f.installer.settleIntegrationPublication(published, "restore", f.signal), + ).rejects.toThrow("named private cleanup failure"); + const receipt = f.installer.describeIntegrationPublication(published); + expect(receipt.outcome).toBe("restored"); + const recovered = await f.installer.restoreIntegrationPublication({ + rootDir: f.root, + receipt, + signal: f.signal, + }); + await f.installer.settleIntegrationPublication(recovered, "restore", f.signal); + const { existsSync } = await import("node:fs"); + expect( + existsSync( + join( + f.root, + "custom_integrations/.staging", + `preparation-${receipt.preparation.preparationId}`, + "rejected", + ), + ), + ).toBe(false); +}); +it("finishes original empty lock removal after owner unlink acknowledgement loss", async () => { + const f = await publicationFixture(); + const published = await f.installer.publishPreparedIntegration(f.prepared, { + transactionId: "ac23034b-553c-4c01-97c4-a0b1c67bb029", + expectedPreviousContentDigest: null, + signal: f.signal, + }); + calls.unlinkFault = true; + await expect( + f.installer.settleIntegrationPublication(published, "commit", f.signal), + ).rejects.toThrow("named owner unlink acknowledgement loss"); + const receipt = f.installer.describeIntegrationPublication(published); + expect(receipt.outcome).toBe("committed"); + const recovered = await f.installer.restoreIntegrationPublication({ + rootDir: f.root, + receipt, + signal: f.signal, + }); + await f.installer.settleIntegrationPublication(recovered, "commit", f.signal); + const later = f.installer.acquireIntegrationComponentLock({ rootDir: f.root, id: f.prepared.id }); + f.installer.releaseIntegrationComponentLock(later); +}); +it("rejects oversized JSON metadata before parsing or whole-file reads", async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-large-manifest-")); + roots.push(root); + mkdirSync(join(root, "services")); + const manifest = + JSON.stringify({ + id: "oversized-json", + name: "Oversized", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }) + " ".repeat(1024 * 1024); + const bytes = createDeterministicTarGz([ + { path: "manifest.json", contents: Buffer.from(manifest) }, + ]); + const archive = join(root, "oversized.tar.gz"); + writeFileSync(archive, bytes); + await expect( + installIntegration({ + rootDir: root, + source: archive, + sourceKind: "artifact", + artifactSha256: createHash("sha256").update(bytes).digest("hex"), + }), + ).rejects.toThrow(/manifest|large|limit/i); +}); diff --git a/packages/integration-framework/src/artifact-stream.ts b/packages/integration-framework/src/artifact-stream.ts new file mode 100644 index 000000000..b041419c4 --- /dev/null +++ b/packages/integration-framework/src/artifact-stream.ts @@ -0,0 +1,378 @@ +/** Bounded declarative archive extraction. All paths are server-local private staging. */ +import { createHash } from "node:crypto"; +import { constants as C, closeSync, fstatSync, fsyncSync, openSync, writeSync } from "node:fs"; +import { Readable } from "node:stream"; +import { createGunzip } from "node:zlib"; +import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; + +const BLOCK = 512; +const UTF8 = new TextDecoder("utf-8", { fatal: true }); +class ByteSource { + private readonly iterator: AsyncIterator; + private current: Uint8Array = new Uint8Array(0); + private offset = 0; + constructor( + source: AsyncIterable, + private readonly signal: AbortSignal, + ) { + this.iterator = source[Symbol.asyncIterator](); + } + async part(maximum: number): Promise { + this.signal.throwIfAborted(); + while (this.offset === this.current.byteLength) { + const next = await this.iterator.next(); + this.signal.throwIfAborted(); + if (next.done) return null; + this.current = next.value; + this.offset = 0; + if (this.current.byteLength > 1024 ** 2) + throw Error("Artifact source chunk exceeds bounded decoder limit"); + if (!this.current.byteLength) continue; + } + const value = this.current.subarray(this.offset, this.offset + maximum); + this.offset += value.byteLength; + return value; + } + async exact( + length: number, + allowEof = false, + label = "Artifact tar payload is truncated", + ): Promise { + const result = Buffer.alloc(length); + let count = 0; + while (count < length) { + const value = await this.part(length - count); + if (!value) { + if (count === 0 && allowEof) return null; + throw Error(label); + } + result.set(value, count); + count += value.byteLength; + } + return result; + } + async *remaining(): AsyncGenerator { + for (;;) { + const part = await this.part(64 * 1024); + if (!part) return; + yield part; + } + } + async close() { + await this.iterator.return?.(); + } +} +function text(header: Buffer, start: number, length: number): string { + const bytes = header.subarray(start, start + length), + nul = bytes.indexOf(0); + return UTF8.decode(bytes.subarray(0, nul < 0 ? undefined : nul)); +} +function octal(header: Buffer, start: number, length: number): number { + const value = text(header, start, length).trim(); + if (value && !/^[0-7]+$/.test(value)) throw Error("Invalid tar numeric field"); + const parsed = value ? Number.parseInt(value, 8) : 0; + if (!Number.isSafeInteger(parsed) || parsed < 0) throw Error("Invalid tar numeric field"); + return parsed; +} +function pathStructure(value: string, label = "Artifact path"): void { + if (Buffer.byteLength(value) > 1024) throw Error(`${label} exceeds 1,024 UTF-8 bytes`); + const parts = value.split("/").filter((p) => p !== "" && p !== "."); + if (parts.length > 32) throw Error(`${label} exceeds 32 components`); + if (parts.some((p) => Buffer.byteLength(p) > 255)) + throw Error(`${label} component exceeds 255 UTF-8 bytes`); +} +function safePath(value: string): string[] { + pathStructure(value); + if ( + !value || + value.startsWith("/") || + /^[A-Za-z]:/.test(value) || + value.includes("\\") || + Array.from(value).some( + (character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127, + ) + ) + throw Error("Unsafe artifact path"); + const parts = value.split("/").filter((p) => p !== "" && p !== "."); + if (parts.some((p) => p === "..")) throw Error("Unsafe artifact path"); + return parts; +} +function pax(bytes: Buffer): Record { + const values: Record = {}; + let offset = 0, + count = 0; + while (offset < bytes.length) { + const space = bytes.indexOf(32, offset); + if (space < 0 || space - offset > 20) + throw Error("Invalid PAX payload: unterminated trailing data"); + const raw = bytes.subarray(offset, space).toString("ascii"); + if (!/^\d+$/.test(raw)) throw Error("Invalid PAX record length"); + const length = Number(raw), + end = offset + length; + if ( + !Number.isSafeInteger(length) || + length <= space - offset + 1 || + end > bytes.length || + bytes[end - 1] !== 10 + ) + throw Error("Invalid PAX record length or terminating newline"); + if (++count > 256) throw Error("PAX payload exceeds 256 records"); + const record = bytes.subarray(space + 1, end - 1), + equals = record.indexOf(61); + if (equals <= 0) throw Error("Invalid PAX record: missing key/value separator"); + const key = UTF8.decode(record.subarray(0, equals)); + if (key.includes("sparse") || key === "SCHILY.realsize") + throw Error("Sparse artifact metadata is not supported"); + if (["path", "linkpath", "size"].includes(key)) { + const value = UTF8.decode(record.subarray(equals + 1)); + if (key === "path") pathStructure(value); + if (key === "linkpath") pathStructure(value, "Artifact link target"); + if (Object.hasOwn(values, key)) throw Error("Duplicate PAX override"); + values[key] = value; + } + offset = end; + } + return values; +} +function openParents(root: AnchoredDirectory, parts: string[]) { + const opened: AnchoredDirectory[] = []; + let current = root; + try { + for (const part of parts) { + current = current.directory(part, true, true); + opened.push(current); + } + return { + parent: current, + close: () => { + for (const dir of opened.reverse()) dir.close(); + }, + }; + } catch (error) { + for (const dir of opened.reverse()) dir.close(); + throw error; + } +} +export async function extractIntegrationArtifactStream(options: { + source: AsyncIterable; + destination: string; + expectedSha256?: string; + maxArtifactBytes: number; + maxExpandedBytes: number; + signal: AbortSignal; +}): Promise<{ archiveSha256: string; expandedBytes: number }> { + const original = options.source[Symbol.asyncIterator](); + if (options.signal.aborted) { + await original.return?.(); + options.signal.throwIfAborted(); + } + const ownedSource: AsyncIterable = { [Symbol.asyncIterator]: () => original }; + const hash = createHash("sha256"); + let wireBytes = 0, + expandedBytes = 0; + const hashed = async function* () { + for await (const part of ownedSource) { + options.signal.throwIfAborted(); + wireBytes += part.byteLength; + if (!Number.isSafeInteger(wireBytes) || wireBytes > options.maxArtifactBytes) + throw Error(`Artifact size exceeds max ${options.maxArtifactBytes} bytes`); + hash.update(part); + yield part; + } + }; + const wire = new ByteSource(hashed(), options.signal); + let magic: Buffer; + try { + const prefix = await wire.exact(2); + if (!prefix) throw Error("Artifact is empty"); + magic = prefix; + } catch (error) { + try { + await wire.close(); + } finally { + await original.return?.(); + } + throw error; + } + const supplied = Readable.from( + (async function* () { + yield magic; + yield* wire.remaining(); + })(), + { objectMode: false, highWaterMark: 64 * 1024 }, + ); + const gzip = + magic[0] === 0x1f && magic[1] === 0x8b ? createGunzip({ chunkSize: 64 * 1024 }) : undefined; + if (gzip) { + supplied.on("error", (error) => { + gzip.destroy(error); + }); + supplied.pipe(gzip); + } + const expanded = gzip ?? supplied; + const bounded = async function* () { + for await (const part of expanded) { + const bytes = part as Buffer; + expandedBytes += bytes.byteLength; + if (!Number.isSafeInteger(expandedBytes) || expandedBytes > options.maxExpandedBytes) + throw Error(`Artifact extracted size exceeds max ${options.maxExpandedBytes} bytes`); + yield bytes; + } + }; + const reader = new ByteSource(bounded(), options.signal); + let root: AnchoredDirectory; + try { + root = new AnchoredDirectory(options.destination, undefined, true); + } catch (error) { + supplied.destroy(); + gzip?.destroy(); + try { + await reader.close(); + await wire.close(); + } finally { + await original.return?.(); + } + throw error; + } + const seen = new Set(); + let headers = 0, + pending: Record = {}, + sawEnd = false; + const abort = () => { + supplied.destroy( + options.signal.reason instanceof Error + ? options.signal.reason + : Error("Artifact extraction aborted"), + ); + gzip?.destroy(); + }; + options.signal.addEventListener("abort", abort, { once: true }); + try { + for (;;) { + const header = await reader.exact(BLOCK, true, "Artifact contains a partial tar header"); + if (!header) break; + if (header.every((b) => b === 0)) { + const second = await reader.exact(BLOCK); + if (!second?.every((b) => b === 0)) throw Error("Invalid artifact tar end blocks"); + sawEnd = true; + break; + } + if (++headers > 4096) throw Error("Artifact tar header limit exceeded: maximum 4,096"); + const expected = octal(header, 148, 8); + let checksum = 0; + for (let i = 0; i < BLOCK; i++) checksum += i >= 148 && i < 156 ? 32 : header[i]; + if (checksum !== expected) throw Error("Invalid artifact tar header checksum"); + const name = text(header, 0, 100), + prefix = text(header, 345, 155), + type = text(header, 156, 1) || "0"; + const size = octal(header, 124, 12); + const rawPath = pending.path ?? (prefix ? `${prefix}/${name}` : name), + link = pending.linkpath ?? text(header, 157, 100); + if (["x", "g", "L", "K"].includes(type)) { + if (size > 16384) throw Error("Artifact extended header exceeds 16 KiB"); + const bytes = await reader.exact(size); + if (!bytes) throw Error("Truncated artifact metadata"); + if (type === "x" || type === "g") { + const values = pax(bytes); + if (type === "x") pending = { ...pending, ...values }; + } else { + const value = text(bytes, 0, bytes.length); + pathStructure(value, type === "K" ? "Artifact link target" : "Artifact path"); + pending = { ...pending, [type === "L" ? "path" : "linkpath"]: value }; + } + const padding = (BLOCK - (size % BLOCK)) % BLOCK; + if (padding) + await reader.exact(padding, false, "Artifact tar payload padding is truncated"); + continue; + } + if ( + pending.size !== undefined && + (!/^\d+$/.test(pending.size) || Number(pending.size) !== size) + ) + throw Error("Artifact PAX size does not match header"); + pending = {}; + const parts = safePath(rawPath); + if (link) pathStructure(link, "Artifact link target"); + if (type !== "0" && type !== "5") throw Error(`Unsupported artifact entry type '${type}'`); + if (!parts.length) { + if (type !== "5" || size !== 0) throw Error("Unsafe artifact root member"); + continue; + } + const path = parts.join("/"); + if (seen.has(path)) throw Error("Duplicate normalized artifact entry"); + seen.add(path); + if (type === "5") { + if (size !== 0) throw Error("Artifact directory has a payload"); + const opened = openParents(root, parts); + opened.close(); + continue; + } + const opened = openParents(root, parts.slice(0, -1)); + let fd: number | undefined; + try { + fd = openSync( + opened.parent.name(parts.at(-1)!), + C.O_WRONLY | C.O_CREAT | C.O_EXCL | C.O_NOFOLLOW, + 0o600, + ); + let remaining = size; + while (remaining) { + const chunk = await reader.part(Math.min(remaining, 64 * 1024)); + if (!chunk) throw Error("Artifact tar payload is truncated"); + opened.parent.check(); + let count = 0; + while (count < chunk.byteLength) + count += writeSync(fd, chunk, count, chunk.byteLength - count); + remaining -= chunk.byteLength; + } + fsyncSync(fd); + const value = fstatSync(fd, { bigint: true }), + current = opened.parent.fileIdentity(parts.at(-1)!); + if ( + value.size !== BigInt(size) || + value.nlink !== 1n || + String(value.dev) !== current.dev || + String(value.ino) !== current.ino + ) + throw Error("Artifact member changed during extraction"); + } finally { + if (fd !== undefined) closeSync(fd); + opened.close(); + } + const padding = (BLOCK - (size % BLOCK)) % BLOCK; + if (padding) { + const bytes = await reader.exact( + padding, + false, + "Artifact tar payload padding is truncated", + ); + if (!bytes?.every((b) => b === 0)) throw Error("Invalid artifact tar padding"); + } + } + if (!sawEnd || Object.keys(pending).length) + throw Error("Artifact tar is truncated or missing end blocks"); + for await (const trailing of reader.remaining()) + if (trailing.some((byte) => byte !== 0)) + throw Error("Artifact has unexplained trailing data"); + options.signal.throwIfAborted(); + const archiveSha256 = hash.digest("hex"); + if (options.expectedSha256 && archiveSha256 !== options.expectedSha256) + throw Error("Artifact sha256 mismatch"); + root.check(); + return { archiveSha256, expandedBytes }; + } finally { + options.signal.removeEventListener("abort", abort); + supplied.destroy(); + gzip?.destroy(); + try { + await reader.close(); + await wire.close(); + } finally { + try { + await original.return?.(); + } finally { + root.close(); + } + } + } +} diff --git a/packages/integration-framework/src/installer.ts b/packages/integration-framework/src/installer.ts index a2273c9bf..2305a4eb6 100644 --- a/packages/integration-framework/src/installer.ts +++ b/packages/integration-framework/src/installer.ts @@ -1,3 +1,4 @@ +import { extractIntegrationArtifactStream } from "./artifact-stream"; // Pure functions for installing, removing, validating, and bundling community // integrations under `custom_integrations/`. The CLI (`pnpm openmapx // integrations …`) and the admin Store (`apps/api/src/services/store.ts`) both @@ -6,32 +7,43 @@ // Community install/package entry points enforce declarative-only artifacts; // app-api never builds or executes community bundles at runtime. -import { createHash, randomBytes } from "node:crypto"; +import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; import { + constants as C, chmodSync, + closeSync, cpSync, + createReadStream, type Dirent, existsSync, - linkSync, + fstatSync, + fsyncSync, lstatSync, mkdirSync, + openSync, readdirSync, readFileSync, + readSync, realpathSync, renameSync, rmSync, statSync, - symlinkSync, writeFileSync, } from "node:fs"; -import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep } from "node:path"; -import { gunzipSync } from "node:zlib"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; +import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; import { scanLicenses } from "@openmapx/core/licenses"; -import { assertAllowedGitUrl, gitShallowClone, safeDownload } from "@openmapx/core/server"; +import { + assertAllowedGitUrl, + gitShallowClone, + gitShallowCloneSnapshot, + safeDownload, +} from "@openmapx/core/server"; import { type ArtifactMetadata, validateArtifactMetadata as validatePortableArtifactMetadata, } from "@openmapx/extension-contracts/artifact"; +import z from "zod/v4"; import { createDeterministicTarGz } from "./deterministic-tar"; import { INTEGRATION_ID_REGEX, validateManifest } from "./manifest"; import { PLATFORM_VERSION, satisfiesPlatformVersion } from "./platform"; @@ -51,13 +63,6 @@ function customDir(rootDir: string): string { const STAGING_DIR_NAME = ".staging"; const DEFAULT_MAX_ARTIFACT_BYTES = 200 * 1024 * 1024; const MAX_LAYER_SELECTOR_PREVIEW_BYTES = 64 * 1024; -const TAR_BLOCK_BYTES = 512; -// Header count includes PAX and GNU metadata headers. -const MAX_TAR_HEADERS = 4096; -const MAX_PAX_RECORDS = 256; -const MAX_ARCHIVE_PATH_BYTES = 1024; -const MAX_ARCHIVE_PATH_COMPONENTS = 32; -const MAX_ARCHIVE_PATH_COMPONENT_BYTES = 255; function stagingDir(rootDir: string): string { return join(customDir(rootDir), STAGING_DIR_NAME); @@ -100,11 +105,41 @@ function resolveInstallTarget(rootDir: string, id: string): string { return target; } +function readBoundedJson(path: string, limit: number): unknown { + const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + try { + const before = fstatSync(fd, { bigint: true }); + if (!before.isFile() || before.size > BigInt(limit)) + throw Error("Integration JSON metadata exceeds its size limit"); + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(fd, bytes, offset, bytes.length - offset, offset); + if (!count) throw Error("Integration JSON metadata changed"); + offset += count; + } + const after = fstatSync(fd, { bigint: true }), + named = lstatSync(path, { bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + named.dev !== before.dev || + named.ino !== before.ino + ) + throw Error("Integration JSON metadata changed"); + return JSON.parse(bytes.toString("utf8")); + } finally { + closeSync(fd); + } +} function readManifest(directory: string): Record | null { const manifestPath = join(directory, "manifest.json"); if (!existsSync(manifestPath)) return null; try { - return JSON.parse(readFileSync(manifestPath, "utf-8")) as Record; + return readBoundedJson(manifestPath, 1024 * 1024) as Record; } catch { return null; } @@ -272,9 +307,14 @@ export interface InstallOptions { /** Stream-style line callback for log output (stdout + stderr). */ onLog?: (line: string, stream: "stdout" | "stderr") => void; signal?: AbortSignal; + /** Server-local bundle orchestration retains publication through reload/bookkeeping. */ + retainPublication?: boolean; + expectedId?: string; + beforePublish?: (receipt: IntegrationPublicationReceipt) => void; } export interface InstallResult { + publication?: IntegrationPublication; id: string; directory: string; replaced: boolean; @@ -331,9 +371,12 @@ export async function installIntegration(opts: InstallOptions): Promise { +async function stageArtifactSource( + opts: InstallOptions, +): Promise<{ directory: string; archiveSha256: string }> { const token = randomBytes(4).toString("hex"); const parent = stagingDir(opts.rootDir); const maxArtifactBytes = opts.maxArtifactBytes ?? DEFAULT_MAX_ARTIFACT_BYTES; mkdirSync(parent, { recursive: true }); const extractRoot = join(parent, `artifact-${token}`); const archivePath = join(parent, `artifact-${token}.tar.gz`); - mkdirSync(extractRoot, { recursive: true }); + mkdirSync(extractRoot, { recursive: true, mode: 0o700 }); try { let sourcePath = opts.source; @@ -420,12 +494,18 @@ async function stageArtifactSource(opts: InstallOptions): Promise { } assertArtifactFileSize(sourcePath, maxArtifactBytes); - verifyArtifactSha256(sourcePath, opts.artifactSha256); - extractTarGz(sourcePath, extractRoot, maxArtifactBytes); + const extracted = await extractIntegrationArtifactStream({ + source: createReadStream(sourcePath, { highWaterMark: 64 * 1024 }), + destination: extractRoot, + expectedSha256: opts.artifactSha256, + maxArtifactBytes, + maxExpandedBytes: maxArtifactBytes, + signal: opts.signal ?? new AbortController().signal, + }); const stage = normalizeExtractedArtifactRoot(extractRoot, join(parent, `source-${token}`)); if (sourcePath === archivePath) rmSync(archivePath, { force: true }); - return stage; + return { directory: stage, archiveSha256: extracted.archiveSha256 }; } catch (err) { rmSync(extractRoot, { recursive: true, force: true }); rmSync(archivePath, { force: true }); @@ -433,18 +513,6 @@ async function stageArtifactSource(opts: InstallOptions): Promise { } } -function verifyArtifactSha256(path: string, expected: string | undefined): void { - if (!expected) return; - const actual = sha256File(path); - if (actual.toLowerCase() !== expected.toLowerCase()) { - throw new Error(`Artifact sha256 mismatch: expected ${expected}, got ${actual}`); - } -} - -function sha256File(path: string): string { - return createHash("sha256").update(readFileSync(path)).digest("hex"); -} - function assertArtifactFileSize(path: string, maxBytes: number): void { const size = statSync(path).size; if (size > maxBytes) { @@ -471,400 +539,1215 @@ function normalizeExtractedArtifactRoot(extractRoot: string, promoted: string): return extractRoot; } -type TarDescriptorType = "directory" | "file" | "hardlink" | "symlink"; - -interface TarDescriptor { - type: TarDescriptorType; - path: string; - linkTarget?: string; - payloadOffset: number; - payloadLength: number; - mode: number; +declare const preparationBrand: unique symbol; +declare const preparedBrand: unique symbol; +export type IntegrationArtifactPreparationContext = { readonly [preparationBrand]: true }; +export type PreparedIntegrationArtifact = Readonly<{ + preparationId: string; + id: string; + version: string; + contentDigest: string; + archiveSha256: string | null; + bytes: number; + [preparedBrand]: true; +}>; +const inodeSchema = z.strictObject({ + dev: z.string().regex(/^[0-9]{1,32}$/), + ino: z.string().regex(/^[0-9]{1,32}$/), +}); +const preparationReceiptSchema = z.strictObject({ + version: z.literal(1), + preparationId: z.uuid(), + id: z.string().regex(INTEGRATION_ID_REGEX), + integrationVersion: z.string().min(1).max(128), + contentDigest: z.string().regex(/^[a-f0-9]{64}$/), + archiveSha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + bytes: z.number().int().safe().nonnegative().max(DEFAULT_MAX_ARTIFACT_BYTES), + workspace: inodeSchema, + content: inodeSchema, +}); +export type IntegrationPreparationReceipt = z.infer; +interface PreparationState { + rootDir: string; + preparationId: string; + workspace: { dev: string; ino: string }; + phase: "empty" | "preparing" | "prepared" | "published" | "discarded"; + receipt?: IntegrationPreparationReceipt; + journal?: IntegrationPublicationJournal; + manifest?: Record; +} +const preparations = new WeakMap(); +const preparedHandles = new WeakMap< + PreparedIntegrationArtifact, + IntegrationArtifactPreparationContext +>(); +function preparationState(context: IntegrationArtifactPreparationContext): PreparationState { + const state = preparations.get(context); + if (!state) throw Error("Unverified integration preparation context"); + return state; +} +/** Server-local descriptor ownership; this path never crosses an HTTP contract. */ +export function describeIntegrationPreparationRoot( + context: IntegrationArtifactPreparationContext, +): string { + const state = preparationState(context); + const workspace = checkedWorkspace(state); + workspace.close(); + return state.rootDir; +} +function workspacePath(state: PreparationState): string { + return join(stagingDir(state.rootDir), `preparation-${state.preparationId}`); +} +function checkedWorkspace(state: PreparationState): AnchoredDirectory { + const directory = new AnchoredDirectory(workspacePath(state), undefined, true); + if ( + String(directory.stat.dev) !== state.workspace.dev || + String(directory.stat.ino) !== state.workspace.ino + ) { + directory.close(); + throw Error("Integration preparation directory changed"); + } + return directory; } - -function extractTarGz(archivePath: string, destDir: string, maxBytes: number): void { - let data: Buffer; +export function createIntegrationArtifactPreparation(options: { + rootDir: string; +}): IntegrationArtifactPreparationContext { + const rootDir = resolve(options.rootDir); + mkdirSync(customDir(rootDir), { recursive: true }); + const root = new AnchoredDirectory(customDir(rootDir)); + let staging: AnchoredDirectory | undefined, workspace: AnchoredDirectory | undefined; try { - data = gunzipSync(readFileSync(archivePath), { maxOutputLength: maxBytes }); - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") { - throw new Error(`Artifact extracted size exceeds max ${maxBytes} bytes`); - } - throw err; - } - - const descriptors = validateTarDescriptors(data, destDir); - materializeTarDescriptors(data, destDir, descriptors); -} - -function validateTarDescriptors(data: Buffer, destDir: string): TarDescriptor[] { - const descriptors: TarDescriptor[] = []; - let pendingPath: string | undefined; - let pendingLinkPath: string | undefined; - const base = resolve(destDir); - let headerCount = 0; - let offset = 0; - let reachedEndBlock = false; - for (; offset + TAR_BLOCK_BYTES <= data.length; ) { - const header = data.subarray(offset, offset + TAR_BLOCK_BYTES); - offset += TAR_BLOCK_BYTES; - if (isZeroBlock(header)) { - reachedEndBlock = true; - break; - } - headerCount += 1; - if (headerCount > MAX_TAR_HEADERS) { - throw new Error( - `Artifact tar header limit exceeded: maximum ${MAX_TAR_HEADERS.toLocaleString("en-US")}`, - ); - } - - const name = readTarString(header, 0, 100); - const prefix = readTarString(header, 345, 155); - const headerPath = prefix ? `${prefix}/${name}` : name; - const headerLinkPath = readTarString(header, 157, 100); - const size = parseTarOctal(header, 124, 12); - const mode = parseTarOctal(header, 100, 8); - const typeflag = readTarString(header, 156, 1) || "0"; - const payloadOffset = offset; - if (size > data.length - payloadOffset) { - throw new Error(`Artifact tar payload is truncated for ${headerPath || "unnamed entry"}`); - } - const payloadEnd = payloadOffset + size; - const paddingLength = (TAR_BLOCK_BYTES - (size % TAR_BLOCK_BYTES)) % TAR_BLOCK_BYTES; - if (paddingLength > data.length - payloadEnd) { - throw new Error( - `Artifact tar payload padding is truncated for ${headerPath || "unnamed entry"}`, - ); - } - const fileData = data.subarray(payloadOffset, payloadEnd); - offset = payloadEnd + paddingLength; - - if (typeflag === "x") { - const records = parsePaxRecords(fileData); - pendingPath = records.path ?? pendingPath; - pendingLinkPath = records.linkpath ?? pendingLinkPath; - continue; - } - if (typeflag === "g") { - parsePaxRecords(fileData, false); - continue; // Global PAX metadata does not carry per-entry paths. - } - if (typeflag === "L") { - assertMetadataPathPayloadSize(fileData, "Artifact path"); - pendingPath = readTarPayloadString(fileData); - continue; - } - if (typeflag === "K") { - assertMetadataPathPayloadSize(fileData, "Artifact link target"); - pendingLinkPath = readTarPayloadString(fileData); - continue; - } - - const rawPath = pendingPath ?? headerPath; - const rawLinkPath = pendingLinkPath ?? headerLinkPath; - pendingPath = undefined; - pendingLinkPath = undefined; - const safePath = safeArchivePath(rawPath); - assertArchivePathStructure(safePath, "Artifact path"); - const target = resolve(destDir, safePath); - if (target !== base && !target.startsWith(`${base}${sep}`)) { - throw new Error(`Artifact entry escapes extraction directory: ${rawPath}`); - } - - if (typeflag === "5") { - descriptors.push({ - type: "directory", - path: safePath, - payloadOffset, - payloadLength: size, - mode, - }); - continue; - } - if (typeflag === "2") { - const linkTarget = resolveSymlinkTarget(base, target, rawLinkPath); - descriptors.push({ - type: "symlink", - path: safePath, - linkTarget: linkTarget.raw, - payloadOffset, - payloadLength: size, - mode, - }); - continue; - } - if (typeflag === "1") { - const linkTarget = validateHardlinkTarget(base, rawLinkPath); - descriptors.push({ - type: "hardlink", - path: safePath, - linkTarget, - payloadOffset, - payloadLength: size, - mode, - }); - continue; - } - if (typeflag !== "0") { - throw new Error(`Unsupported artifact entry type '${typeflag}' for ${rawPath}`); - } - - descriptors.push({ - type: "file", - path: safePath, - payloadOffset, - payloadLength: size, - mode, + staging = root.directory(STAGING_DIR_NAME, true, false); + const preparationId = randomUUID(); + workspace = staging.directory(`preparation-${preparationId}`, true, true); + const context = Object.freeze({}) as IntegrationArtifactPreparationContext; + preparations.set(context, { + rootDir, + preparationId, + workspace: { dev: String(workspace.stat.dev), ino: String(workspace.stat.ino) }, + phase: "empty", }); + return context; + } finally { + workspace?.close(); + staging?.close(); + root.close(); } - if (!reachedEndBlock && offset !== data.length) { - throw new Error("Artifact contains a partial tar header"); - } - return descriptors; } - -function materializeTarDescriptors( - data: Buffer, - destDir: string, - descriptors: TarDescriptor[], -): void { - const base = resolve(destDir); - for (const descriptor of descriptors) { - const target = resolve(destDir, descriptor.path); - if (descriptor.type === "directory") { - assertPathNotThroughSymlink(base, target); - mkdirSync(target, { recursive: true }); - continue; - } - if (descriptor.type === "symlink") { - if (!descriptor.linkTarget) throw new Error("Validated artifact symlink has no target"); - resolveSymlinkTarget(base, target, descriptor.linkTarget); - assertPathNotThroughSymlink(base, dirname(target)); - mkdirSync(dirname(target), { recursive: true }); - if (existsSync(target)) { - throw new Error(`Artifact link target already exists: ${descriptor.path}`); +function validatePreparedIntegration(directory: string, rootDir: string): Record { + const manifest = readManifest(directory); + if (!manifest || typeof manifest.id !== "string") + throw Error("Source has no manifest.json with a string id"); + const validation = validateManifest(manifest); + if (!validation.valid) throw Error(`Manifest validation failed: ${validation.errors.join("; ")}`); + assertCompatibleManifestProtocol(manifest); + assertSafeId(manifest.id); + assertDoesNotShadowBuiltIn(rootDir, manifest.id); + resolveLayerSelectorPreview(directory, manifest); + assertNoExecutableCommunityCode(directory, manifest); + validateDeclarativeArtifact(directory); + validateArtifactMetadata(directory, manifest); + return manifest; +} +function fingerprintPreparedDirectory( + directory: string, + signal: AbortSignal, + strict = true, +): { contentDigest: string; bytes: number } { + const root = new AnchoredDirectory(directory, undefined, strict); + const hash = createHash("sha256"); + let bytes = 0, + count = 0; + const walk = (parent: AnchoredDirectory, prefix: string) => { + for (const name of parent.listBounded(4096).sort()) { + signal.throwIfAborted(); + if (++count > 4096) throw Error("Integration preparation entry limit exceeded"); + const path = prefix ? `${prefix}/${name}` : name; + const stat = lstatSync(parent.name(name), { bigint: true }); + if (stat.isDirectory()) { + const child = parent.directory(name, false, strict); + try { + hash.update(JSON.stringify(["directory", path])); + walk(child, path); + } finally { + child.close(); + } + } else { + if ( + !stat.isFile() || + stat.nlink !== 1n || + stat.uid !== BigInt(process.getuid?.() ?? -1) || + (stat.mode & BigInt(strict ? 0o077 : 0o022)) !== 0n + ) + throw Error("Unsafe integration preparation member"); + bytes += Number(stat.size); + if (!Number.isSafeInteger(bytes) || bytes > DEFAULT_MAX_ARTIFACT_BYTES) + throw Error("Integration preparation exceeds 200 MiB"); + const fd = openSync(parent.name(name), C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + try { + const before = fstatSync(fd, { bigint: true }); + const fileHash = createHash("sha256"), + buffer = Buffer.alloc(64 * 1024); + for (;;) { + signal.throwIfAborted(); + const n = readSync(fd, buffer); + if (!n) break; + fileHash.update(buffer.subarray(0, n)); + } + const after = fstatSync(fd, { bigint: true }), + current = parent.fileIdentity(name, strict); + if ( + before.dev !== stat.dev || + before.ino !== stat.ino || + before.size !== stat.size || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + String(after.dev) !== current.dev || + String(after.ino) !== current.ino + ) + throw Error("Integration preparation changed during fingerprinting"); + hash.update(JSON.stringify(["file", path, Number(before.size), fileHash.digest("hex")])); + } finally { + closeSync(fd); + } } - symlinkSync(descriptor.linkTarget, target); - continue; - } - if (descriptor.type === "hardlink") { - if (!descriptor.linkTarget) throw new Error("Validated artifact hardlink has no target"); - const linkTarget = resolveHardlinkTarget(base, descriptor.linkTarget); - assertPathNotThroughSymlink(base, dirname(target)); - mkdirSync(dirname(target), { recursive: true }); - assertPathNotThroughSymlink(base, target); - linkSync(linkTarget, target); - continue; } - - assertPathNotThroughSymlink(base, dirname(target)); - mkdirSync(dirname(target), { recursive: true }); - assertPathNotThroughSymlink(base, target); - writeFileSync( - target, - data.subarray(descriptor.payloadOffset, descriptor.payloadOffset + descriptor.payloadLength), + parent.check(); + }; + try { + walk(root, ""); + return { contentDigest: hash.digest("hex"), bytes }; + } finally { + root.close(); + } +} +function preparedHandle( + context: IntegrationArtifactPreparationContext, + receipt: IntegrationPreparationReceipt, +): PreparedIntegrationArtifact { + const prepared = Object.freeze({ + preparationId: receipt.preparationId, + id: receipt.id, + version: receipt.integrationVersion, + contentDigest: receipt.contentDigest, + archiveSha256: receipt.archiveSha256, + bytes: receipt.bytes, + }) as PreparedIntegrationArtifact; + preparedHandles.set(prepared, context); + return prepared; +} +export async function prepareIntegrationArtifact( + input: { + source: AsyncIterable; + expectedSha256: string; + maxArtifactBytes: number; + maxExpandedBytes: number; + signal: AbortSignal; + }, + context: IntegrationArtifactPreparationContext, +): Promise { + const state = preparationState(context); + if (state.phase !== "empty") throw Error("Integration preparation already used"); + if ( + !/^[a-f0-9]{64}$/.test(input.expectedSha256) || + !Number.isSafeInteger(input.maxArtifactBytes) || + input.maxArtifactBytes <= 0 || + input.maxArtifactBytes > DEFAULT_MAX_ARTIFACT_BYTES || + !Number.isSafeInteger(input.maxExpandedBytes) || + input.maxExpandedBytes <= 0 || + input.maxExpandedBytes > DEFAULT_MAX_ARTIFACT_BYTES + ) + throw Error("Invalid integration preparation limits"); + const workspace = checkedWorkspace(state); + state.phase = "preparing"; + try { + const payload = workspace.directory("payload", true, true); + payload.close(); + const output = await extractIntegrationArtifactStream({ + source: input.source, + destination: join(workspacePath(state), "payload"), + expectedSha256: input.expectedSha256, + maxArtifactBytes: input.maxArtifactBytes, + maxExpandedBytes: input.maxExpandedBytes, + signal: input.signal, + }); + workspace.check(); + const content = join(workspacePath(state), "content"); + const normalized = normalizeExtractedArtifactRoot( + join(workspacePath(state), "payload"), + content, ); - if (descriptor.mode > 0) { - try { - chmodSync(target, descriptor.mode & 0o777); - } catch { - // Non-fatal on filesystems that do not support chmod. - } + if (normalized !== content) renameSync(normalized, content); + return finishIntegrationPreparation(context, output.archiveSha256, input.signal); + } catch (error) { + discardIntegrationPreparation(context); + throw error; + } finally { + workspace.close(); + } +} +/** HTTPS/redirect/public-network checks and byte cap are enforced by the shared downloader. */ +export async function prepareRemoteIntegrationArtifact( + input: { url: string; sha256: string; signal: AbortSignal }, + context: IntegrationArtifactPreparationContext, +): Promise { + const state = preparationState(context); + if (state.phase !== "empty") throw Error("Integration preparation already used"); + const workspace = checkedWorkspace(state); + try { + const file = workspace.name("download.tar.gz"); + await safeDownload({ + url: new URL(input.url), + destination: file, + maxBytes: DEFAULT_MAX_ARTIFACT_BYTES, + timeoutMs: 5 * 60000, + allowedContentTypes: [], + credentialPolicy: "none", + signal: input.signal, + }); + const source = createReadStream(file, { highWaterMark: 64 * 1024, signal: input.signal }); + try { + return await prepareIntegrationArtifact( + { + source, + expectedSha256: input.sha256, + maxArtifactBytes: DEFAULT_MAX_ARTIFACT_BYTES, + maxExpandedBytes: DEFAULT_MAX_ARTIFACT_BYTES, + signal: input.signal, + }, + context, + ); + } finally { + source.destroy(); } + } finally { + workspace.close(); } } - -function isZeroBlock(block: Buffer): boolean { - for (const byte of block) { - if (byte !== 0) return false; +function finishIntegrationPreparation( + context: IntegrationArtifactPreparationContext, + archiveSha256: string | null, + signal: AbortSignal, +): PreparedIntegrationArtifact { + const state = preparationState(context); + const workspace = checkedWorkspace(state); + try { + const content = join(workspacePath(state), "content"); + const manifest = validatePreparedIntegration(content, state.rootDir); + const proof = fingerprintPreparedDirectory(content, signal); + const directory = workspace.directory("content", false, true); + try { + const receipt = preparationReceiptSchema.parse({ + version: 1, + preparationId: state.preparationId, + id: manifest.id, + integrationVersion: manifest.version, + contentDigest: proof.contentDigest, + archiveSha256, + bytes: proof.bytes, + workspace: state.workspace, + content: { dev: String(directory.stat.dev), ino: String(directory.stat.ino) }, + }); + state.receipt = receipt; + state.manifest = structuredClone(manifest); + state.phase = "prepared"; + return preparedHandle(context, receipt); + } finally { + directory.close(); + } + } finally { + workspace.close(); + } +} +function privatizeIntegrationDirectory(root: string, signal: AbortSignal): void { + let entries = 0, + bytes = 0; + const walk = (directory: string) => { + signal.throwIfAborted(); + const stat = lstatSync(directory, { bigint: true }); + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.uid !== BigInt(process.getuid?.() ?? -1) + ) + throw Error("Unsafe integration preparation directory"); + chmodSync(directory, 0o700); + for (const name of readdirSync(directory)) { + if (++entries > 4096) throw Error("Integration preparation entry limit exceeded"); + const path = join(directory, name), + member = lstatSync(path, { bigint: true }); + if (member.isDirectory()) walk(path); + else if ( + member.isFile() && + member.nlink === 1n && + member.uid === BigInt(process.getuid?.() ?? -1) + ) { + bytes += Number(member.size); + if (!Number.isSafeInteger(bytes) || bytes > DEFAULT_MAX_ARTIFACT_BYTES) + throw Error("Integration source exceeds 200 MiB"); + chmodSync(path, 0o600); + } else throw Error("Integration source contains a link or unsupported entry"); + } + }; + walk(root); +} + +/** Resolve a bounded allowlisted repository exactly once, retaining its immutable commit. */ +export async function prepareIntegrationSource( + input: { repository: string; ref?: string; signal: AbortSignal }, + context: IntegrationArtifactPreparationContext, +): Promise<{ prepared: PreparedIntegrationArtifact; repository: string; commit: string }> { + const state = preparationState(context); + if (state.phase !== "empty") throw Error("Integration preparation already used"); + state.phase = "preparing"; + const workspace = checkedWorkspace(state); + try { + const snapshot = await gitShallowCloneSnapshot({ + url: input.repository, + ref: input.ref, + targetDir: workspace.name("content"), + signal: input.signal, + }); + if (!/^[a-f0-9]{40}$/.test(snapshot.commit)) throw Error("Invalid resolved integration commit"); + privatizeIntegrationDirectory(workspace.name("content"), input.signal); + const prepared = finishIntegrationPreparation(context, null, input.signal); + return { prepared, repository: snapshot.canonicalUrl, commit: snapshot.commit }; + } catch (error) { + discardIntegrationPreparation(context); + throw error; + } finally { + workspace.close(); } - return true; } -function readTarString(block: Buffer, start: number, length: number): string { - const slice = block.subarray(start, start + length); - const nul = slice.indexOf(0); - return slice - .subarray(0, nul >= 0 ? nul : undefined) - .toString("utf-8") - .trim(); +export function describePreparedIntegration( + prepared: PreparedIntegrationArtifact, +): IntegrationPreparationReceipt { + const context = preparedHandles.get(prepared); + if (!context) throw Error("Unverified prepared integration"); + const state = preparationState(context); + if (state.phase !== "prepared" || !state.receipt) + throw Error("Integration preparation is not available"); + return structuredClone(state.receipt); } - -function parseTarOctal(block: Buffer, start: number, length: number): number { - const raw = readTarString(block, start, length).replace(/\0/g, "").trim(); - if (!raw) return 0; - const value = Number.parseInt(raw, 8); - if (!Number.isSafeInteger(value) || value < 0) { - throw new Error(`Invalid tar numeric field: ${raw}`); +export async function restorePreparedIntegration(options: { + rootDir: string; + receipt: unknown; + signal: AbortSignal; +}): Promise { + const receipt = preparationReceiptSchema.parse(options.receipt); + const state: PreparationState = { + rootDir: resolve(options.rootDir), + preparationId: receipt.preparationId, + workspace: receipt.workspace, + phase: "prepared", + receipt, + }; + const directory = checkedWorkspace(state); + try { + const content = directory.directory("content", false, true); + try { + if ( + String(content.stat.dev) !== receipt.content.dev || + String(content.stat.ino) !== receipt.content.ino + ) + throw Error("Prepared integration content directory changed"); + } finally { + content.close(); + } + const path = join(workspacePath(state), "content"); + const proof = fingerprintPreparedDirectory(path, options.signal); + if (proof.contentDigest !== receipt.contentDigest || proof.bytes !== receipt.bytes) + throw Error("Prepared integration content changed"); + const manifest = validatePreparedIntegration(path, state.rootDir); + if (manifest.id !== receipt.id || manifest.version !== receipt.integrationVersion) + throw Error("Prepared integration manifest changed"); + state.manifest = structuredClone(manifest); + const context = Object.freeze({}) as IntegrationArtifactPreparationContext; + preparations.set(context, state); + return preparedHandle(context, receipt); + } finally { + directory.close(); + } +} +export function discardIntegrationPreparation( + context: IntegrationArtifactPreparationContext, +): void { + const state = preparationState(context); + if (state.phase === "discarded") return; + if (state.phase === "published") + throw Error("Published integration requires transaction settlement"); + const directory = checkedWorkspace(state); + try { + directory.check(); + rmSync(workspacePath(state), { recursive: true, force: false }); + state.phase = "discarded"; + } finally { + directory.close(); } - return value; -} - -function readTarPayloadString(data: Buffer): string { - const nul = data.indexOf(0); - return data - .subarray(0, nul >= 0 ? nul : undefined) - .toString("utf-8") - .trimEnd(); } -function parsePaxRecords(data: Buffer, capturePaths = true): Record { - const records: Record = {}; - let offset = 0; - let count = 0; - while (offset < data.length) { - const space = data.indexOf(0x20, offset); - if (space < 0) throw new Error("Invalid PAX payload: unterminated trailing data"); - if (space - offset > 20) throw new Error("Invalid PAX record length"); - const lengthRaw = data.subarray(offset, space).toString("utf-8"); - if (!/^\d+$/.test(lengthRaw)) throw new Error(`Invalid PAX record length: ${lengthRaw}`); - const length = Number.parseInt(lengthRaw, 10); - if (!Number.isInteger(length) || length <= 0 || offset + length > data.length) { - throw new Error(`Invalid PAX record length: ${lengthRaw}`); - } - count += 1; - if (count > MAX_PAX_RECORDS) { - throw new Error(`PAX payload exceeds ${MAX_PAX_RECORDS} records`); - } - - const recordEnd = offset + length; - if (recordEnd <= space + 1 || data[recordEnd - 1] !== 10) { - throw new Error("Invalid PAX record: declared length must include a terminating newline"); - } - const record = data.subarray(space + 1, recordEnd - 1); - const equals = record.indexOf(0x3d); - if (equals <= 0) throw new Error("Invalid PAX record: missing key/value separator"); - if (capturePaths) { - const keyBytes = record.subarray(0, equals); - const valueBytes = record.subarray(equals + 1); - if (keyBytes.equals(Buffer.from("path"))) { - assertArchiveTextBytes(valueBytes, "Artifact path"); - records.path = valueBytes.toString("utf-8"); - } else if (keyBytes.equals(Buffer.from("linkpath"))) { - assertArchiveTextBytes(valueBytes, "Artifact link target"); - records.linkpath = valueBytes.toString("utf-8"); - } - } - offset += length; +declare const componentLockBrand: unique symbol; +export type IntegrationComponentLock = { readonly [componentLockBrand]: true }; +interface ComponentLockState { + rootDir: string; + id: string; + transactionId: string; + identity: { dev: string; ino: string }; +} +const componentLocks = new WeakMap(); +function componentLockDirectory(state: ComponentLockState): { + root: AnchoredDirectory; + locks: AnchoredDirectory; + directory: AnchoredDirectory; +} { + const root = new AnchoredDirectory(customDir(state.rootDir)); + let locks: AnchoredDirectory | undefined, directory: AnchoredDirectory | undefined; + try { + locks = root.directory(".installation-locks", false, true); + directory = locks.directory(state.id, false, true); + const owner = z + .strictObject({ version: z.literal(1), transactionId: z.uuid() }) + .parse(JSON.parse(directory.read("owner.json", 2048).toString("utf8"))); + if ( + owner.transactionId !== state.transactionId || + String(directory.stat.dev) !== state.identity.dev || + String(directory.stat.ino) !== state.identity.ino || + directory.listBounded(2).length !== 1 + ) + throw Error("Integration component lock ownership changed"); + return { root, locks, directory }; + } catch (error) { + directory?.close(); + locks?.close(); + root.close(); + throw error; } - return records; } - -function assertMetadataPathPayloadSize(data: Buffer, label: string): void { - const trailingNul = data.length > 0 && data[data.length - 1] === 0 ? 1 : 0; - if (data.length - trailingNul > MAX_ARCHIVE_PATH_BYTES) { - throw new Error( - `${label} exceeds ${MAX_ARCHIVE_PATH_BYTES.toLocaleString("en-US")} UTF-8 bytes`, - ); +export function acquireIntegrationComponentLock(options: { + rootDir: string; + id: string; + transactionId?: string; +}): IntegrationComponentLock { + assertSafeId(options.id); + const transactionId = z.uuid().parse(options.transactionId ?? randomUUID()); + mkdirSync(customDir(options.rootDir), { recursive: true }); + const root = new AnchoredDirectory(customDir(options.rootDir)); + let locks: AnchoredDirectory | undefined, directory: AnchoredDirectory | undefined; + try { + locks = root.directory(".installation-locks", true, true); + try { + mkdirSync(locks.name(options.id), { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") + throw Error("Integration component is busy; reconcile its original transaction"); + throw error; + } + fsyncSync(locks.fd); + directory = locks.directory(options.id, false, true); + directory.write("owner.json", JSON.stringify({ version: 1, transactionId }), true); + const lock = Object.freeze({}) as IntegrationComponentLock; + componentLocks.set(lock, { + rootDir: options.rootDir, + id: options.id, + transactionId, + identity: { dev: String(directory.stat.dev), ino: String(directory.stat.ino) }, + }); + return lock; + } finally { + directory?.close(); + locks?.close(); + root.close(); } } - -function assertArchiveTextBytes(data: Buffer, label: string): void { - if (data.length > MAX_ARCHIVE_PATH_BYTES) { - throw new Error( - `${label} exceeds ${MAX_ARCHIVE_PATH_BYTES.toLocaleString("en-US")} UTF-8 bytes`, - ); +export function releaseIntegrationComponentLock(lock: IntegrationComponentLock): void { + const state = componentLocks.get(lock); + if (!state) throw Error("Unverified integration component lock"); + const opened = componentLockDirectory(state); + try { + opened.directory.removeFile("owner.json"); + opened.directory.close(); + opened.locks.removeDirectory(state.id, state.identity); + componentLocks.delete(lock); + } finally { + opened.directory.close(); + opened.locks.close(); + opened.root.close(); + } +} +function restoreComponentLock(state: ComponentLockState): IntegrationComponentLock { + const opened = componentLockDirectory(state); + opened.directory.close(); + opened.locks.close(); + opened.root.close(); + const lock = Object.freeze({}) as IntegrationComponentLock; + componentLocks.set(lock, state); + return lock; +} +function currentIntegrationFingerprint( + rootDir: string, + id: string, + signal: AbortSignal, +): { contentDigest: string; identity: { dev: string; ino: string } } | null { + const target = resolveInstallTarget(rootDir, id); + let stat: import("node:fs").BigIntStats; + try { + stat = lstatSync(target, { bigint: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; } + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw Error("Integration destination is not a regular owned directory"); + const fingerprint = fingerprintPreparedDirectory(target, signal, false); + const after = lstatSync(target, { bigint: true }); + if (after.dev !== stat.dev || after.ino !== stat.ino) + throw Error("Integration destination changed during inspection"); + return { + contentDigest: fingerprint.contentDigest, + identity: { dev: String(stat.dev), ino: String(stat.ino) }, + }; } - -function assertArchivePathStructure(path: string, label: string): void { - if (Buffer.byteLength(path, "utf-8") > MAX_ARCHIVE_PATH_BYTES) { - throw new Error( - `${label} exceeds ${MAX_ARCHIVE_PATH_BYTES.toLocaleString("en-US")} UTF-8 bytes`, +export function inspectInstalledIntegrationFingerprint(options: { + rootDir: string; + id: string; + signal: AbortSignal; +}): { contentDigest: string; identity: { dev: string; ino: string } } | null { + const lock = acquireIntegrationComponentLock(options); + try { + return currentIntegrationFingerprint(options.rootDir, options.id, options.signal); + } finally { + releaseIntegrationComponentLock(lock); + } +} +const publicationSchema = z.strictObject({ + version: z.literal(1), + transactionId: z.uuid(), + integrationId: z.string().regex(INTEGRATION_ID_REGEX), + preparation: preparationReceiptSchema, + previousContentDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + previousIdentity: inodeSchema.nullable(), + lockIdentity: inodeSchema, + outcome: z.enum([ + "prepared", + "previous-moved", + "published", + "commit-requested", + "committed", + "restore-requested", + "restored", + ]), +}); +export type IntegrationPublicationReceipt = z.infer; +export const integrationPublicationReceiptSchema = publicationSchema; +export const integrationPreparationReceiptSchema = preparationReceiptSchema; +declare const publicationJournalBrand: unique symbol; +export type IntegrationPublicationJournal = Readonly<{ [publicationJournalBrand]: true }>; +const publicationJournals = new WeakMap< + IntegrationPublicationJournal, + { + directory: AnchoredDirectory; + coordination: AnchoredDirectory; + key: Buffer; + transactionId: string; + closed: boolean; + } +>(); +/** Server-local purpose store descriptor. No filesystem path or key is accepted over HTTP. */ +export function createIntegrationPublicationJournal(input: { + directory: AnchoredDirectory; + coordination: AnchoredDirectory; + key: Uint8Array; + transactionId: string; +}): IntegrationPublicationJournal { + z.uuid().parse(input.transactionId); + input.directory.check(); + if (input.key.byteLength !== 32) throw Error("Invalid integration journal authentication key"); + const handle = Object.freeze({}) as IntegrationPublicationJournal; + publicationJournals.set(handle, { + directory: input.directory, + coordination: input.coordination, + key: Buffer.from(input.key), + transactionId: input.transactionId, + closed: false, + }); + return handle; +} +export function closeIntegrationPublicationJournal(handle: IntegrationPublicationJournal): void { + const journal = publicationJournals.get(handle); + if (!journal) throw Error("Unverified integration publication journal"); + journal.closed = true; + journal.key.fill(0); +} +function publicationJournal(handle: IntegrationPublicationJournal) { + const journal = publicationJournals.get(handle); + if (!journal || journal.closed) throw Error("Integration publication journal is unavailable"); + journal.directory.check(); + return journal; +} +function readPublicationRecord(state: PreparationState): IntegrationPublicationReceipt { + if (state.journal) { + const journal = publicationJournal(state.journal); + const saved = z + .strictObject({ body: z.string().max(16384), mac: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse( + JSON.parse( + journal.directory + .read(`publication-${journal.transactionId}.json`, 32768) + .toString("utf8"), + ), + ); + const expected = createHmac("sha256", journal.key) + .update("openmapx-integration-publication-v1\0") + .update(saved.body) + .digest(); + if (!timingSafeEqual(expected, Buffer.from(saved.mac, "hex"))) + throw Error("Integration publication journal authentication failed"); + const receipt = publicationSchema.parse(JSON.parse(saved.body)); + if (receipt.transactionId !== journal.transactionId) + throw Error("Integration publication journal identity changed"); + return receipt; + } + const directory = checkedWorkspace(state); + try { + return publicationSchema.parse( + JSON.parse(directory.read("publication.json", 16384).toString("utf8")), ); - } - const components = path.split(sep).filter((component) => component && component !== "."); - if (components.length > MAX_ARCHIVE_PATH_COMPONENTS) { - throw new Error(`${label} exceeds ${MAX_ARCHIVE_PATH_COMPONENTS} components`); - } - for (const component of components) { - if (Buffer.byteLength(component, "utf-8") > MAX_ARCHIVE_PATH_COMPONENT_BYTES) { - throw new Error(`${label} component exceeds ${MAX_ARCHIVE_PATH_COMPONENT_BYTES} UTF-8 bytes`); + } finally { + directory.close(); + } +} +declare const publicationBrand: unique symbol; +export type IntegrationPublication = { readonly [publicationBrand]: true }; +const publications = new WeakMap< + IntegrationPublication, + { + context: IntegrationArtifactPreparationContext; + lock: IntegrationComponentLock | null; + receipt: IntegrationPublicationReceipt; + } +>(); +function publicationState(handle: IntegrationPublication) { + const state = publications.get(handle); + if (!state) throw Error("Unverified integration publication"); + return state; +} +export function assertIntegrationPublicationJournal( + publication: IntegrationPublication, + journal: IntegrationPublicationJournal, +): void { + if (preparationState(publicationState(publication).context).journal !== journal) + throw Error("Integration publication is not owned by its authenticated purpose journal"); + publicationJournal(journal); +} +function writePublication( + context: IntegrationArtifactPreparationContext, + receipt: IntegrationPublicationReceipt, + exclusive = false, +) { + const state = preparationState(context), + directory = checkedWorkspace(state); + try { + const body = JSON.stringify(publicationSchema.parse(receipt)); + if (state.journal) { + const journal = publicationJournal(state.journal); + if (journal.transactionId !== receipt.transactionId) + throw Error("Integration publication journal transaction mismatch"); + let guard: AnchoredDirectory | undefined; + try { + mkdirSync(journal.coordination.name("writer"), { mode: 0o700 }); + guard = journal.coordination.directory("writer", false, true); + const name = `publication-${receipt.transactionId}.json`; + if (exclusive && journal.directory.listBounded(2049).length >= 2048) + throw Error("Integration purpose journal capacity reached"); + journal.directory.write( + name, + JSON.stringify({ + body, + mac: createHmac("sha256", journal.key) + .update("openmapx-integration-publication-v1\0") + .update(body) + .digest("hex"), + }), + exclusive, + ); + } finally { + if (guard) { + const identity = { dev: String(guard.stat.dev), ino: String(guard.stat.ino) }; + guard.close(); + journal.coordination.removeDirectory("writer", identity); + } + } } + directory.write("publication.json", body); + } finally { + directory.close(); } } - -function safeArchivePath(path: string): string { - const normalized = normalize(path.replace(/\\/g, "/")); - if ( - !normalized || - normalized === "." || - isAbsolute(normalized) || - /^[A-Za-z]:/.test(normalized) || - normalized === ".." || - normalized.startsWith(`..${sep}`) || - normalized.includes(`${sep}..${sep}`) - ) { - throw new Error(`Unsafe artifact path: ${path}`); - } - return normalized; +function sameDirectory(a: { dev: string; ino: string }, b: { dev: string; ino: string }) { + return a.dev === b.dev && a.ino === b.ino; } - -function assertPathNotThroughSymlink(base: string, target: string): void { - const resolvedBase = resolve(base); - const resolvedTarget = resolve(target); - if (resolvedTarget !== resolvedBase && !resolvedTarget.startsWith(`${resolvedBase}${sep}`)) { - throw new Error(`Artifact path escapes extraction directory: ${target}`); - } - - const rel = relative(resolvedBase, resolvedTarget); - if (!rel) return; - - let current = resolvedBase; - for (const part of rel.split(sep)) { - if (!part) continue; - current = join(current, part); - if (existsSync(current) && lstatSync(current).isSymbolicLink()) { - throw new Error(`Artifact path traverses a symlink: ${relative(resolvedBase, current)}`); +function requirePublishedTarget(state: ReturnType, signal: AbortSignal) { + const value = currentIntegrationFingerprint( + preparationState(state.context).rootDir, + state.receipt.integrationId, + signal, + ); + if ( + !value || + value.contentDigest !== state.receipt.preparation.contentDigest || + !sameDirectory(value.identity, state.receipt.preparation.content) + ) + throw Error("Published integration changed; preserve original transaction"); +} +function loadPublication(state: ReturnType) { + const saved = readPublicationRecord(preparationState(state.context)); + const { outcome: _old, ...old } = state.receipt, + { outcome: _new, ...current } = saved; + if (JSON.stringify(old) !== JSON.stringify(current)) + throw Error("Integration publication journal changed"); + state.receipt = saved; + return saved; +} + +export async function publishPreparedIntegration( + prepared: PreparedIntegrationArtifact, + input: { + transactionId: string; + expectedPreviousContentDigest: string | null; + expectedPreviousIdentity?: { dev: string; ino: string } | null; + beforePublish?: (receipt: IntegrationPublicationReceipt) => void; + journal?: IntegrationPublicationJournal; + signal: AbortSignal; + }, +): Promise { + z.uuid().parse(input.transactionId); + const context = preparedHandles.get(prepared); + if (!context) throw Error("Unverified prepared integration"); + const state = preparationState(context); + if (state.phase !== "prepared" || !state.receipt) + throw Error("Integration preparation unavailable"); + await restorePreparedIntegration({ + rootDir: state.rootDir, + receipt: state.receipt, + signal: input.signal, + }); + const lock = acquireIntegrationComponentLock({ + rootDir: state.rootDir, + id: prepared.id, + transactionId: input.transactionId, + }); + state.journal = input.journal; + let effectStarted = false; + let receipt: IntegrationPublicationReceipt | undefined; + try { + const previous = currentIntegrationFingerprint(state.rootDir, prepared.id, input.signal); + if ((previous?.contentDigest ?? null) !== input.expectedPreviousContentDigest) + throw Error("Integration destination changed since inspection"); + if ( + input.expectedPreviousIdentity !== undefined && + (previous + ? !input.expectedPreviousIdentity || + !sameDirectory(previous.identity, input.expectedPreviousIdentity) + : input.expectedPreviousIdentity !== null) + ) + throw Error("Integration destination identity changed since inspection"); + const lockState = componentLocks.get(lock)!; + receipt = { + version: 1, + transactionId: input.transactionId, + integrationId: prepared.id, + preparation: state.receipt, + previousContentDigest: previous?.contentDigest ?? null, + previousIdentity: previous?.identity ?? null, + lockIdentity: lockState.identity, + outcome: "prepared", + }; + writePublication(context, receipt, true); + input.beforePublish?.(structuredClone(receipt)); + input.signal.throwIfAborted(); + const workspace = checkedWorkspace(state); + try { + const target = resolveInstallTarget(state.rootDir, prepared.id); + state.phase = "published"; + effectStarted = true; + if (previous) { + renameSync(target, workspace.name("previous")); + fsyncSync(workspace.fd); + const root = new AnchoredDirectory(customDir(state.rootDir)); + try { + fsyncSync(root.fd); + } finally { + root.close(); + } + receipt.outcome = "previous-moved"; + writePublication(context, receipt); + } + renameSync(workspace.name("content"), target); + fsyncSync(workspace.fd); + const root = new AnchoredDirectory(customDir(state.rootDir)); + try { + fsyncSync(root.fd); + } finally { + root.close(); + } + receipt.outcome = "published"; + writePublication(context, receipt); + const handle = Object.freeze({}) as IntegrationPublication; + publications.set(handle, { context, lock, receipt }); + requirePublishedTarget(publicationState(handle), new AbortController().signal); + return handle; + } finally { + workspace.close(); } + } catch (error) { + if (!effectStarted) { + if (receipt) { + const previous = currentIntegrationFingerprint( + state.rootDir, + prepared.id, + new AbortController().signal, + ); + if ( + (previous?.contentDigest ?? null) !== receipt.previousContentDigest || + (previous && + (!receipt.previousIdentity || + !sameDirectory(previous.identity, receipt.previousIdentity))) + ) + throw Error("Integration pre-publication failure requires reconciliation", { + cause: error, + }); + receipt.outcome = "restored"; + writePublication(context, receipt); + state.phase = "published"; + } + releaseIntegrationComponentLock(lock); + } + throw error; } } - -function assertSafeLinkText(rawLinkPath: string): string { - const linkPath = rawLinkPath.replace(/\\/g, "/"); - if (!linkPath || linkPath.includes("\0")) { - throw new Error("Artifact link has an empty or invalid target"); - } - if (isAbsolute(linkPath) || /^[A-Za-z]:/.test(linkPath)) { - throw new Error(`Artifact link target is not relative: ${rawLinkPath}`); - } - return linkPath; -} - -function resolveSymlinkTarget( - base: string, - linkPath: string, - rawLinkPath: string, -): { raw: string; resolved: string } { - const raw = assertSafeLinkText(rawLinkPath); - assertArchivePathStructure(normalize(raw), "Artifact link target"); - const resolved = resolve(dirname(linkPath), raw); - if (resolved !== base && !resolved.startsWith(`${base}${sep}`)) { - throw new Error(`Artifact symlink target escapes extraction directory: ${rawLinkPath}`); - } - return { raw, resolved }; +export function describeIntegrationPublication( + handle: IntegrationPublication, +): IntegrationPublicationReceipt { + return structuredClone(loadPublication(publicationState(handle))); } - -function validateHardlinkTarget(base: string, rawLinkPath: string): string { - const linkPath = assertSafeLinkText(rawLinkPath); - const safePath = safeArchivePath(linkPath); - assertArchivePathStructure(safePath, "Artifact link target"); - const resolved = resolve(base, safePath); - if (resolved !== base && !resolved.startsWith(`${base}${sep}`)) { - throw new Error(`Artifact hardlink target escapes extraction directory: ${rawLinkPath}`); +/** Read only the fixed journal belonging to an already retained preparation. */ +export function inspectIntegrationPublication(options: { + rootDir: string; + preparation: IntegrationPreparationReceipt; + transactionId: string; + journal?: IntegrationPublicationJournal; +}): IntegrationPublicationReceipt { + const preparation = preparationReceiptSchema.parse(options.preparation); + const state: PreparationState = { + rootDir: resolve(options.rootDir), + preparationId: preparation.preparationId, + workspace: preparation.workspace, + phase: "published", + receipt: preparation, + journal: options.journal, + }; + const directory = checkedWorkspace(state); + try { + const receipt = readPublicationRecord(state); + if ( + receipt.transactionId !== options.transactionId || + JSON.stringify(receipt.preparation) !== JSON.stringify(preparation) + ) + throw Error("Integration publication does not belong to its original preparation"); + return receipt; + } finally { + directory.close(); + } +} +export async function restoreIntegrationPublication(options: { + rootDir: string; + receipt: IntegrationPublicationReceipt; + signal: AbortSignal; + journal?: IntegrationPublicationJournal; +}): Promise { + const receipt = publicationSchema.parse(options.receipt); + const context = Object.freeze({}) as IntegrationArtifactPreparationContext; + preparations.set(context, { + rootDir: resolve(options.rootDir), + preparationId: receipt.preparation.preparationId, + workspace: receipt.preparation.workspace, + phase: "published", + receipt: receipt.preparation, + journal: options.journal, + }); + const handle = Object.freeze({}) as IntegrationPublication; + publications.set(handle, { context, lock: null, receipt }); + const actual = loadPublication(publicationState(handle)); + const terminal = actual.outcome === "committed" || actual.outcome === "restored"; + const lockState = { + rootDir: options.rootDir, + id: receipt.integrationId, + transactionId: receipt.transactionId, + identity: receipt.lockIdentity, + }; + if (!terminal) publicationState(handle).lock = restoreComponentLock(lockState); + else { + const path = join(customDir(options.rootDir), ".installation-locks", receipt.integrationId); + let identity: import("node:fs").BigIntStats | null = null; + try { + identity = lstatSync(path, { bigint: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if ( + identity && + sameDirectory({ dev: String(identity.dev), ino: String(identity.ino) }, receipt.lockIdentity) + ) { + const root = new AnchoredDirectory(customDir(options.rootDir)); + let locks: AnchoredDirectory | undefined, directory: AnchoredDirectory | undefined; + try { + locks = root.directory(".installation-locks", false, true); + directory = locks.directory(receipt.integrationId, false, true); + if ( + !sameDirectory( + { dev: String(directory.stat.dev), ino: String(directory.stat.ino) }, + receipt.lockIdentity, + ) + ) + throw Error("Original integration lock changed"); + if (directory.listBounded(2).length === 0) { + directory.close(); + locks.removeDirectory(receipt.integrationId, receipt.lockIdentity); + } else publicationState(handle).lock = restoreComponentLock(lockState); + } finally { + directory?.close(); + locks?.close(); + root.close(); + } + } } - return safePath; -} - -function resolveHardlinkTarget(base: string, rawLinkPath: string): string { - const safePath = validateHardlinkTarget(base, rawLinkPath); - const resolved = resolve(base, safePath); - assertPathNotThroughSymlink(base, resolved); - if (!existsSync(resolved)) { - throw new Error(`Artifact hardlink target is missing: ${rawLinkPath}`); + if (["published", "commit-requested"].includes(actual.outcome)) + requirePublishedTarget(publicationState(handle), options.signal); + return handle; +} +export async function settleIntegrationPublication( + handle: IntegrationPublication, + decision: "commit" | "restore", + signal: AbortSignal, +): Promise<{ outcome: "committed" | "restored"; contentDigest: string | null }> { + const state = publicationState(handle), + receipt = loadPublication(state), + preparation = preparationState(state.context); + const terminal = receipt.outcome === "committed" || receipt.outcome === "restored"; + if (terminal && (decision === "commit") !== (receipt.outcome === "committed")) + throw Error("Integration transaction already settled with a different decision"); + if (!terminal && !state.lock) + throw Error("Integration transaction has no original component lock"); + const workspace = checkedWorkspace(preparation); + let guard: AnchoredDirectory | undefined; + try { + try { + mkdirSync(workspace.name("settlement-writer"), { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") + throw Error("Integration settlement is busy"); + throw error; + } + guard = workspace.directory("settlement-writer"); + const target = resolveInstallTarget(preparation.rootDir, receipt.integrationId); + const removeOwned = ( + name: string, + expected: { dev: string; ino: string }, + expectedDigest: string | null, + verifyDigest: boolean, + ) => { + let current: import("node:fs").BigIntStats; + try { + current = lstatSync(workspace.name(name), { bigint: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + if ( + !current.isDirectory() || + String(current.dev) !== expected.dev || + String(current.ino) !== expected.ino + ) + throw Error("Integration transaction directory changed"); + if ( + verifyDigest && + fingerprintPreparedDirectory(workspace.name(name), signal, false).contentDigest !== + expectedDigest + ) + throw Error("Integration transaction bytes changed"); + rmSync(workspace.name(name), { recursive: true, force: false }); + fsyncSync(workspace.fd); + }; + if (terminal) { + if (receipt.outcome === "restored") { + removeOwned( + "rejected", + receipt.preparation.content, + receipt.preparation.contentDigest, + false, + ); + removeOwned( + "content", + receipt.preparation.content, + receipt.preparation.contentDigest, + false, + ); + } + if (state.lock) { + releaseIntegrationComponentLock(state.lock!); + state.lock = null; + } + return { + outcome: receipt.outcome as "committed" | "restored", + contentDigest: + receipt.outcome === "committed" + ? receipt.preparation.contentDigest + : receipt.previousContentDigest, + }; + } + if (decision === "commit") { + if (receipt.outcome === "restore-requested") + throw Error("Integration transaction was restored"); + requirePublishedTarget(state, signal); + const wasRequested = receipt.outcome === "commit-requested"; + receipt.outcome = "commit-requested"; + writePublication(state.context, receipt); + if (receipt.previousIdentity) + removeOwned( + "previous", + receipt.previousIdentity, + receipt.previousContentDigest, + !wasRequested, + ); + receipt.outcome = "committed"; + writePublication(state.context, receipt); + releaseIntegrationComponentLock(state.lock!); + state.lock = null; + return { outcome: "committed", contentDigest: receipt.preparation.contentDigest }; + } + if (receipt.outcome === "commit-requested") + throw Error("Integration transaction was committed"); + { + if (receipt.outcome !== "restore-requested") { + if (receipt.outcome === "prepared" || receipt.outcome === "previous-moved") { + const current = currentIntegrationFingerprint( + preparation.rootDir, + receipt.integrationId, + signal, + ); + const newTarget = + current && + sameDirectory(current.identity, receipt.preparation.content) && + current.contentDigest === receipt.preparation.contentDigest; + if (!newTarget) { + if ( + current && + (!receipt.previousIdentity || + !sameDirectory(current.identity, receipt.previousIdentity) || + current.contentDigest !== receipt.previousContentDigest) + ) + throw Error("Integration destination changed before restoration"); + const original = workspace.directory("content", false, true); + try { + if ( + !sameDirectory( + { dev: String(original.stat.dev), ino: String(original.stat.ino) }, + receipt.preparation.content, + ) || + fingerprintPreparedDirectory(workspace.name("content"), signal).contentDigest !== + receipt.preparation.contentDigest + ) + throw Error("Prepared integration changed before restoration"); + } finally { + original.close(); + } + } + } else requirePublishedTarget(state, signal); + receipt.outcome = "restore-requested"; + writePublication(state.context, receipt); + } + const current = currentIntegrationFingerprint( + preparation.rootDir, + receipt.integrationId, + signal, + ); + if ( + current && + sameDirectory(current.identity, receipt.preparation.content) && + current.contentDigest === receipt.preparation.contentDigest + ) { + renameSync(target, workspace.name("rejected")); + fsyncSync(workspace.fd); + } else if ( + current && + (!receipt.previousIdentity || + !sameDirectory(current.identity, receipt.previousIdentity) || + current.contentDigest !== receipt.previousContentDigest) + ) + throw Error("Integration destination changed during restore"); + const after = currentIntegrationFingerprint( + preparation.rootDir, + receipt.integrationId, + signal, + ); + if (!after && receipt.previousIdentity) { + const previous = workspace.directory("previous", false, false); + try { + if ( + String(previous.stat.dev) !== receipt.previousIdentity.dev || + String(previous.stat.ino) !== receipt.previousIdentity.ino || + fingerprintPreparedDirectory(workspace.name("previous"), signal, false) + .contentDigest !== receipt.previousContentDigest + ) + throw Error("Original integration backup changed"); + } finally { + previous.close(); + } + renameSync(workspace.name("previous"), target); + fsyncSync(workspace.fd); + } + const restored = currentIntegrationFingerprint( + preparation.rootDir, + receipt.integrationId, + signal, + ); + if ( + (restored?.contentDigest ?? null) !== receipt.previousContentDigest || + (restored && + receipt.previousIdentity && + !sameDirectory(restored.identity, receipt.previousIdentity)) + ) + throw Error("Integration restoration is not proven"); + const root = new AnchoredDirectory(customDir(preparation.rootDir)); + try { + fsyncSync(root.fd); + } finally { + root.close(); + } + receipt.outcome = "restored"; + writePublication(state.context, receipt); + } + removeOwned("rejected", receipt.preparation.content, receipt.preparation.contentDigest, false); + removeOwned("content", receipt.preparation.content, receipt.preparation.contentDigest, false); + releaseIntegrationComponentLock(state.lock!); + state.lock = null; + return { outcome: "restored", contentDigest: receipt.previousContentDigest }; + } finally { + if (guard) { + const identity = { dev: String(guard.stat.dev), ino: String(guard.stat.ino) }; + guard.close(); + workspace.removeDirectory("settlement-writer", identity); + } + workspace.close(); } - return resolved; } export interface RemoveOptions { @@ -895,10 +1778,32 @@ function validateIntegrationRollbackBackup( return { target, backupDirectory }; } -/** Copy an installed artifact aside while leaving the active files available. */ export function backupInstalledIntegration( rootDir: string, id: string, +): IntegrationRollbackBackup | null { + const lock = acquireIntegrationComponentLock({ rootDir, id }); + try { + return backupInstalledIntegrationUnlocked(rootDir, id); + } finally { + releaseIntegrationComponentLock(lock); + } +} +export function restoreInstalledIntegration( + rootDir: string, + backup: IntegrationRollbackBackup, +): void { + const lock = acquireIntegrationComponentLock({ rootDir, id: backup.id }); + try { + restoreInstalledIntegrationUnlocked(rootDir, backup); + } finally { + releaseIntegrationComponentLock(lock); + } +} +/** Copy an installed artifact aside while leaving the active files available. */ +function backupInstalledIntegrationUnlocked( + rootDir: string, + id: string, ): IntegrationRollbackBackup | null { const target = resolveInstallTarget(rootDir, id); if (!existsSync(target)) return null; @@ -915,7 +1820,7 @@ export function backupInstalledIntegration( } /** Restore the exact artifact snapshot after a failed multi-component update. */ -export function restoreInstalledIntegration( +function restoreInstalledIntegrationUnlocked( rootDir: string, backup: IntegrationRollbackBackup, ): void { @@ -924,7 +1829,10 @@ export function restoreInstalledIntegration( const restoreStage = createStagePath(rootDir, `restore-${backup.id}`); try { cpSync(backupDirectory, restoreStage, { recursive: true, errorOnExist: true, force: false }); - if (existsSync(target)) rmSync(target, { recursive: true, force: true }); + if (existsSync(target)) + throw Error( + "Historical integration rollback lacks a publication ownership receipt; preserve the current integration and reconcile the original journal", + ); renameSync(restoreStage, target); } catch (error) { rmSync(restoreStage, { recursive: true, force: true }); @@ -941,7 +1849,7 @@ export function discardInstalledIntegrationBackup( rmSync(backupDirectory, { recursive: true, force: true }); } -export function removeIntegration(opts: RemoveOptions): { directory: string } { +function removeIntegrationUnlocked(opts: RemoveOptions): { directory: string } { const target = resolveInstallTarget(opts.rootDir, opts.id); if (!existsSync(target)) { throw new Error( @@ -1201,7 +2109,7 @@ function validateArtifactMetadata(directory: string, manifest: Record Date: Thu, 29 Oct 2026 13:42:11 +0100 Subject: [PATCH 060/135] feat(admin): adopt and remove local standalone integrations --- apps/api/openapi.json | 1877 +++- apps/api/src/db/installed-extension-schema.ts | 2 +- ...0050_admin_legacy_integration_adoption.sql | 2 + .../src/db/migrations/meta/0050_snapshot.json | 9121 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + .../routes/admin-standalone-integrations.ts | 31 + apps/api/src/server.ts | 6 + apps/api/src/services/admin-job-checkpoint.ts | 3 +- apps/api/src/services/admin-job-failure.ts | 11 +- apps/api/src/services/admin-job-parent.ts | 7 + apps/api/src/services/extension-store.ts | 6 +- apps/api/src/services/job-runner.ts | 6 +- .../standalone-integrations/install.test.ts | 349 +- .../standalone-integrations/install.ts | 2 +- .../standalone-integrations/inventory.ts | 75 + .../ownership-checkpoint.ts | 70 + .../ownership-database.ts | 59 + .../ownership-state.ts | 80 + .../standalone-integrations/ownership.test.ts | 74 + .../standalone-integrations/ownership.ts | 390 + .../settlement-checkpoint.ts | 30 + apps/ops-agent/src/artifact-transfer/store.ts | 133 +- .../src/standalone-integration-records.ts | 30 + .../standalone-integration-runtime.test.ts | 110 +- .../src/standalone-integration-runtime.ts | 4 +- .../src/standalone-ownership-runtime.ts | 396 + .../admin/extensions/ExtensionStorePage.tsx | 4 +- .../StandaloneIntegrationsPanel.test.tsx | 139 + .../StandaloneIntegrationsPanel.tsx | 283 + .../administration/community-extensions.md | 16 + .../api/contracts/integration-admin.test.ts | 11 +- .../standalone-integrations.ts | 49 +- .../admin/integration-admin.generated.ts | 335 + .../adoptAdminStandaloneIntegration202.ts | 9 + .../adoptAdminStandaloneIntegration400.ts | 20 + .../adoptAdminStandaloneIntegration401.ts | 20 + .../adoptAdminStandaloneIntegration403.ts | 20 + .../adoptAdminStandaloneIntegration409.ts | 20 + .../adoptAdminStandaloneIntegration413.ts | 20 + .../adoptAdminStandaloneIntegration415.ts | 20 + .../adoptAdminStandaloneIntegration429.ts | 11 + .../adoptAdminStandaloneIntegration500.ts | 20 + .../adoptAdminStandaloneIntegration503.ts | 20 + .../adoptAdminStandaloneIntegrationBody.ts | 7 + .../adoptAdminStandaloneIntegrationHeaders.ts | 9 + .../admin/integration-models/index.ts | 35 + .../listAdminStandaloneIntegrations200.ts | 8 + ...minStandaloneIntegrations200EntriesItem.ts | 19 + ...loneIntegrations200EntriesItemOwnership.ts | 12 + ...oneIntegrations200EntriesItemProvenance.ts | 12 + .../listAdminStandaloneIntegrations400.ts | 20 + .../listAdminStandaloneIntegrations401.ts | 20 + .../listAdminStandaloneIntegrations403.ts | 20 + .../listAdminStandaloneIntegrations413.ts | 20 + .../listAdminStandaloneIntegrations415.ts | 20 + .../listAdminStandaloneIntegrations429.ts | 11 + .../listAdminStandaloneIntegrations500.ts | 20 + .../removeAdminStandaloneIntegration202.ts | 9 + .../removeAdminStandaloneIntegration400.ts | 20 + .../removeAdminStandaloneIntegration401.ts | 20 + .../removeAdminStandaloneIntegration403.ts | 20 + .../removeAdminStandaloneIntegration409.ts | 20 + .../removeAdminStandaloneIntegration413.ts | 20 + .../removeAdminStandaloneIntegration415.ts | 20 + .../removeAdminStandaloneIntegration429.ts | 11 + .../removeAdminStandaloneIntegration500.ts | 20 + .../removeAdminStandaloneIntegration503.ts | 20 + .../removeAdminStandaloneIntegrationBody.ts | 6 + ...removeAdminStandaloneIntegrationHeaders.ts | 9 + .../admin/operation-bindings.generated.ts | 23 + .../src/build-artifacts/control-authority.ts | 1 + packages/core/src/ops/contract.test.ts | 13 + packages/core/src/ops/contract.ts | 6 + .../src/ops/standalone-integrations.test.ts | 8 + .../core/src/ops/standalone-integrations.ts | 61 +- .../integration-framework/src/installer.ts | 93 + scripts/ops-authority-inventory.json | 12 + 77 files changed, 14438 insertions(+), 105 deletions(-) create mode 100644 apps/api/src/db/migrations/0050_admin_legacy_integration_adoption.sql create mode 100644 apps/api/src/db/migrations/meta/0050_snapshot.json create mode 100644 apps/api/src/services/standalone-integrations/inventory.ts create mode 100644 apps/api/src/services/standalone-integrations/ownership-checkpoint.ts create mode 100644 apps/api/src/services/standalone-integrations/ownership-database.ts create mode 100644 apps/api/src/services/standalone-integrations/ownership-state.ts create mode 100644 apps/api/src/services/standalone-integrations/ownership.test.ts create mode 100644 apps/api/src/services/standalone-integrations/ownership.ts create mode 100644 apps/ops-agent/src/standalone-ownership-runtime.ts create mode 100644 apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.test.tsx create mode 100644 apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.tsx create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration202.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration400.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration401.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration403.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration413.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration415.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration429.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration500.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration503.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationBody.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationHeaders.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItem.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemOwnership.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemProvenance.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations400.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations401.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations403.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations413.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations415.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations429.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations500.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration202.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration400.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration401.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration403.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration409.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration413.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration415.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration429.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration500.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration503.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationBody.ts create mode 100644 packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationHeaders.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index b6e8c1d99..d690bc403 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -65352,10 +65352,1784 @@ "x-openmapx-auth": "admin" } }, + "/api/admin/standalone-integrations": { + "get": { + "operationId": "listAdminStandaloneIntegrations", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "entries": { + "maxItems": 2048, + "type": "array", + "items": { + "type": "object", + "properties": { + "integrationId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + }, + "version": { + "anyOf": [ + { + "type": "string", + "maxLength": 128 + }, + { + "type": "null" + } + ] + }, + "contentDigest": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "denialReason": { + "anyOf": [ + { + "type": "string", + "maxLength": 256 + }, + { + "type": "null" + } + ] + }, + "pendingTransactionId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "ownership": { + "type": "string", + "enum": [ + "extension", + "standalone", + "unmanaged", + "conflict" + ] + }, + "extensionId": { + "anyOf": [ + { + "type": "string", + "maxLength": 128 + }, + { + "type": "null" + } + ] + }, + "provenance": { + "type": "string", + "enum": [ + "artifact", + "source", + "uploaded-artifact", + "legacy_unknown" + ] + }, + "adoptable": { + "type": "boolean" + }, + "removable": { + "type": "boolean" + } + }, + "required": [ + "integrationId", + "version", + "contentDigest", + "denialReason", + "ownership", + "extensionId", + "provenance", + "adoptable", + "removable" + ], + "additionalProperties": false + } + } + }, + "required": [ + "entries" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "integration-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/admin/standalone-integrations/install": { "post": { - "operationId": "installAdminStandaloneIntegration", + "operationId": "installAdminStandaloneIntegration", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "inspectionId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedContentDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "inspectionId", + "expectedContentDigest" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "jobId", + "operationId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "integration-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/standalone-integrations/preview": { + "post": { + "operationId": "previewAdminStandaloneIntegration", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "artifact" + ] + }, + "url": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "url", + "sha256" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "source" + ] + }, + "repository": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "ref": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$" + } + }, + "required": [ + "kind", + "repository" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "uploaded-artifact" + ] + }, + "stageId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedSha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "stageId", + "expectedSha256" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "operationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "jobId", + "operationId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "integration-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/standalone-integrations/{id}/adopt": { + "post": { + "operationId": "adoptAdminStandaloneIntegration", "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + }, + "in": "path", + "name": "id", + "required": true + }, { "schema": { "type": "string", @@ -65373,18 +67147,12 @@ "schema": { "type": "object", "properties": { - "inspectionId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, "expectedContentDigest": { "type": "string", "pattern": "^[a-f0-9]{64}$" } }, "required": [ - "inspectionId", "expectedContentDigest" ], "additionalProperties": false @@ -65940,10 +67708,19 @@ "x-openmapx-auth": "admin" } }, - "/api/admin/standalone-integrations/preview": { + "/api/admin/standalone-integrations/{id}/remove": { "post": { - "operationId": "previewAdminStandaloneIntegration", + "operationId": "removeAdminStandaloneIntegration", "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + }, + "in": "path", + "name": "id", + "required": true + }, { "schema": { "type": "string", @@ -65961,90 +67738,20 @@ "schema": { "type": "object", "properties": { - "source": { - "oneOf": [ - { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": [ - "artifact" - ] - }, - "url": { - "type": "string", - "maxLength": 2048, - "format": "uri" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - } - }, - "required": [ - "kind", - "url", - "sha256" - ], - "additionalProperties": false - }, + "expectedContentDigest": { + "anyOf": [ { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": [ - "source" - ] - }, - "repository": { - "type": "string", - "maxLength": 2048, - "format": "uri" - }, - "ref": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,254}$" - } - }, - "required": [ - "kind", - "repository" - ], - "additionalProperties": false + "type": "string", + "pattern": "^[a-f0-9]{64}$" }, { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": [ - "uploaded-artifact" - ] - }, - "stageId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "expectedSha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - } - }, - "required": [ - "kind", - "stageId", - "expectedSha256" - ], - "additionalProperties": false + "type": "null" } ] } }, "required": [ - "source" + "expectedContentDigest" ], "additionalProperties": false } diff --git a/apps/api/src/db/installed-extension-schema.ts b/apps/api/src/db/installed-extension-schema.ts index 8e2c8bca1..3e3693fca 100644 --- a/apps/api/src/db/installed-extension-schema.ts +++ b/apps/api/src/db/installed-extension-schema.ts @@ -45,7 +45,7 @@ export const installedExtension = pgTable( ), check( "installed_extension_standalone_provenance_check", - sql`${table.installationKind} <> 'standalone' or coalesce((${table.sourceTrust} = 'community' and ${table.id} ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof(${table.provenance}) = 'object' and ${table.provenance}->>'kind' in ('artifact','source','uploaded-artifact') and ${table.provenance}->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)`, + sql`${table.installationKind} <> 'standalone' or coalesce((${table.sourceTrust} = 'community' and ${table.id} ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof(${table.provenance}) = 'object' and ${table.provenance}->>'kind' in ('artifact','source','uploaded-artifact','legacy_unknown') and ${table.provenance}->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)`, ), ], ); diff --git a/apps/api/src/db/migrations/0050_admin_legacy_integration_adoption.sql b/apps/api/src/db/migrations/0050_admin_legacy_integration_adoption.sql new file mode 100644 index 000000000..d73434c85 --- /dev/null +++ b/apps/api/src/db/migrations/0050_admin_legacy_integration_adoption.sql @@ -0,0 +1,2 @@ +ALTER TABLE "installed_extension" DROP CONSTRAINT "installed_extension_standalone_provenance_check";--> statement-breakpoint +ALTER TABLE "installed_extension" ADD CONSTRAINT "installed_extension_standalone_provenance_check" CHECK ("installed_extension"."installation_kind" <> 'standalone' or coalesce(("installed_extension"."source_trust" = 'community' and "installed_extension"."id" ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof("installed_extension"."provenance") = 'object' and "installed_extension"."provenance"->>'kind' in ('artifact','source','uploaded-artifact','legacy_unknown') and "installed_extension"."provenance"->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)); \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0050_snapshot.json b/apps/api/src/db/migrations/meta/0050_snapshot.json new file mode 100644 index 000000000..3380a242d --- /dev/null +++ b/apps/api/src/db/migrations/meta/0050_snapshot.json @@ -0,0 +1,9121 @@ +{ + "id": "4f0db974-6157-45b7-9478-54bd16a30cc7", + "prevId": "0aa5791a-a19b-489f-9999-00d604524afe", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_action_receipt": { + "name": "account_action_receipt", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mail_requested": { + "name": "mail_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_action_actor_time_idx": { + "name": "account_action_actor_time_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_action_target_time_idx": { + "name": "account_action_target_time_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_action_receipt_actor_id_user_id_fk": { + "name": "account_action_receipt_actor_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "account_action_receipt_target_id_user_id_fk": { + "name": "account_action_receipt_target_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_admin_guard": { + "name": "account_admin_guard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "active_count": { + "name": "active_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_admin_guard_singleton": { + "name": "account_admin_guard_singleton", + "value": "\"account_admin_guard\".\"id\"=1" + }, + "account_admin_guard_nonnegative": { + "name": "account_admin_guard_nonnegative", + "value": "\"account_admin_guard\".\"active_count\">=0" + } + }, + "isRLSEnabled": false + }, + "public.account_mail_outbox": { + "name": "account_mail_outbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_mail_action_kind_uq": { + "name": "account_mail_action_kind_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_mail_pending_idx": { + "name": "account_mail_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_mail_outbox_request_key_account_action_receipt_key_fk": { + "name": "account_mail_outbox_request_key_account_action_receipt_key_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "account_action_receipt", + "columnsFrom": [ + "request_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "account_mail_outbox_user_id_user_id_fk": { + "name": "account_mail_outbox_user_id_user_id_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_mail_state_check": { + "name": "account_mail_state_check", + "value": "\"account_mail_outbox\".\"state\" in ('queued','sending','sent','failed','unknown')" + }, + "account_mail_kind_check": { + "name": "account_mail_kind_check", + "value": "\"account_mail_outbox\".\"kind\" in ('verification','password_setup')" + } + }, + "isRLSEnabled": false + }, + "public.admin_action_reauthentication": { + "name": "admin_action_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admin_reauth_actor_expiry_idx": { + "name": "admin_reauth_actor_expiry_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admin_reauth_request_uq": { + "name": "admin_reauth_request_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_action_reauthentication_actor_id_user_id_fk": { + "name": "admin_action_reauthentication_actor_id_user_id_fk", + "tableFrom": "admin_action_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "admin_reauth_state_check": { + "name": "admin_reauth_state_check", + "value": "\"admin_action_reauthentication\".\"state\" in ('pending','completed','consumed','superseded','expired')" + }, + "admin_reauth_purpose_check": { + "name": "admin_reauth_purpose_check", + "value": "\"admin_action_reauthentication\".\"purpose\" in ('account.verify','account.password.set','account.reset.send','account.invite','credentials.export','credentials.import','transit.credentials','platform.domain.apply')" + } + }, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_application_actor_id_user_id_fk": { + "name": "auth_policy_application_actor_id_user_id_fk", + "tableFrom": "auth_policy_application", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_state_activated_by_user_id_fk": { + "name": "auth_policy_state_activated_by_user_id_fk", + "tableFrom": "auth_policy_state", + "tableTo": "user", + "columnsFrom": [ + "activated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installation_kind": { + "name": "installation_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'catalog'" + }, + "provenance": { + "name": "provenance", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_operation_id": { + "name": "last_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "installed_extension_installation_kind_check": { + "name": "installed_extension_installation_kind_check", + "value": "\"installed_extension\".\"installation_kind\" in ('catalog','standalone')" + }, + "installed_extension_standalone_provenance_check": { + "name": "installed_extension_standalone_provenance_check", + "value": "\"installed_extension\".\"installation_kind\" <> 'standalone' or coalesce((\"installed_extension\".\"source_trust\" = 'community' and \"installed_extension\".\"id\" ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof(\"installed_extension\".\"provenance\") = 'object' and \"installed_extension\".\"provenance\"->>'kind' in ('artifact','source','uploaded-artifact','legacy_unknown') and \"installed_extension\".\"provenance\"->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)" + } + }, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_independent_password_login": { + "name": "origin_independent_password_login", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transit_credential_preview": { + "name": "transit_credential_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_ciphertext": { + "name": "authorization_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_iv": { + "name": "authorization_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_tag": { + "name": "authorization_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "credentials_revision": { + "name": "credentials_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "transit_credential_preview_actor_key_uq": { + "name": "transit_credential_preview_actor_key_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transit_credential_preview_expiry_idx": { + "name": "transit_credential_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "transit_credential_preview_actor_id_user_id_fk": { + "name": "transit_credential_preview_actor_id_user_id_fk", + "tableFrom": "transit_credential_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "transit_credential_preview_bounds": { + "name": "transit_credential_preview_bounds", + "value": "octet_length(\"transit_credential_preview\".\"ciphertext\") <= 1500000 and length(\"transit_credential_preview\".\"intent_fingerprint\") = 64 and length(\"transit_credential_preview\".\"iv\") = 16 and length(\"transit_credential_preview\".\"tag\") = 24" + } + }, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_application_receipt": { + "name": "integration_application_receipt", + "schema": "", + "columns": { + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "saved_binding_revision": { + "name": "saved_binding_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_application_target_time_idx": { + "name": "integration_application_target_time_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_application_actor_idx": { + "name": "integration_application_actor_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_application_receipt_actor_id_user_id_fk": { + "name": "integration_application_receipt_actor_id_user_id_fk", + "tableFrom": "integration_application_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_binding_revision": { + "name": "integration_binding_revision", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 1eb7555aa..82b45af26 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -351,6 +351,13 @@ "when": 1793271038219, "tag": "0049_admin_integration_ownership", "breakpoints": true + }, + { + "idx": 50, + "version": "7", + "when": 1793275863684, + "tag": "0050_admin_legacy_integration_adoption", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/routes/admin-standalone-integrations.ts b/apps/api/src/routes/admin-standalone-integrations.ts index e74fb346a..81c8958de 100644 --- a/apps/api/src/routes/admin-standalone-integrations.ts +++ b/apps/api/src/routes/admin-standalone-integrations.ts @@ -9,6 +9,37 @@ export async function adminStandaloneIntegrationsRoute(app: FastifyInstance): Pr app.addHook("preHandler", async (request) => { request.adminSession = await requireAdmin(request); }); + registerIntegrationAdminRoute(app, "listAdminStandaloneIntegrations", async () => + (await import("../services/standalone-integrations/ownership")).standaloneInventory(), + ); + for (const action of ["adopt", "remove"] as const) { + const id = + action === "adopt" ? "adoptAdminStandaloneIntegration" : "removeAdminStandaloneIntegration"; + registerIntegrationAdminRoute(app, id, async (request, reply) => { + try { + const actorId = getAdminSession(request).user.id; + const accepted = await jobRunner.enqueueStandalone( + `standalone.${action}`, + { integrationId: request.params.id, ...request.body }, + actorId, + String(request.headers["idempotency-key"]), + ); + await writeAuditLog({ + actorId, + action: `standalone.${action}`, + targetType: "integration", + targetId: request.params.id, + details: { jobId: accepted.jobId }, + request, + }); + return reply.code(202).send(accepted); + } catch (error) { + if (error instanceof RetainedJobConflict) + return reply.code(409).send({ error: error.message }); + throw error; + } + }); + } for (const id of [ "previewAdminStandaloneIntegration", "installAdminStandaloneIntegration", diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 506a2d91d..fc08c35a0 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -710,6 +710,12 @@ jobRunner.register("integration.reload", async (ctx) => { return result as Record; }); +jobRunner.register("standalone.adopt", async (ctx) => + (await import("./services/standalone-integrations/ownership")).handleStandaloneOwnership(ctx), +); +jobRunner.register("standalone.remove", async (ctx) => + (await import("./services/standalone-integrations/ownership")).handleStandaloneOwnership(ctx), +); jobRunner.register("standalone.preview", async (ctx) => (await import("./services/standalone-integrations/install")).handleStandalonePreview(ctx), ); diff --git a/apps/api/src/services/admin-job-checkpoint.ts b/apps/api/src/services/admin-job-checkpoint.ts index f95503d30..2497c6cbf 100644 --- a/apps/api/src/services/admin-job-checkpoint.ts +++ b/apps/api/src/services/admin-job-checkpoint.ts @@ -72,7 +72,8 @@ export async function checkpointOwnedAdminJob( if (Object.keys(old.steps).length >= 64 || Object.keys(old.operations).length >= 64) return { capacity: true as const, result: current }; const standaloneSettlement = - row.type === "standalone.install" && operation.operation?.kind === "integration.settle"; + ["standalone.install", "standalone.adopt", "standalone.remove"].includes(row.type) && + operation.operation?.kind === "integration.settle"; if (standaloneSettlement) await assertStandaloneSettlementCheckpoint(row, operation.operation!, tx); if (row.status !== "running" && !standaloneSettlement) throw new JobOwnershipLost(); diff --git a/apps/api/src/services/admin-job-failure.ts b/apps/api/src/services/admin-job-failure.ts index c5090e6c1..68cc7e106 100644 --- a/apps/api/src/services/admin-job-failure.ts +++ b/apps/api/src/services/admin-job-failure.ts @@ -45,11 +45,18 @@ export class JobObservationPending extends Error { /** Standalone acquisition inputs are public metadata; ownership and request MACs are not. */ export function publicAdminJobPayload(type: string, payload: unknown): unknown { - if (type !== "standalone.preview" && type !== "standalone.install") return payload; + if ( + !["standalone.preview", "standalone.install", "standalone.adopt", "standalone.remove"].includes( + type, + ) + ) + return payload; if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; const fields = type === "standalone.preview" ? ["source"] - : ["inspectionId", "expectedContentDigest", "operationId"]; + : type === "standalone.install" + ? ["inspectionId", "expectedContentDigest", "operationId"] + : ["integrationId", "expectedContentDigest", "operationId"]; return Object.fromEntries(Object.entries(payload).filter(([key]) => fields.includes(key))); } diff --git a/apps/api/src/services/admin-job-parent.ts b/apps/api/src/services/admin-job-parent.ts index 22428ac34..8cec00d4f 100644 --- a/apps/api/src/services/admin-job-parent.ts +++ b/apps/api/src/services/admin-job-parent.ts @@ -17,6 +17,13 @@ export { type ApiJobChildAttempt, apiJobChildAttemptSchema } from "@openmapx/cor import type { ApiJobChildAttempt } from "@openmapx/core/ops"; function parentJobBudget(ctx: JobContext, kind: OpsOperationKind): number { + if (ctx.jobType === "standalone.adopt" || ctx.jobType === "standalone.remove") + return ( + opsExecutionBudget("integration.inspectOwnership").processMs + + opsExecutionBudget("integration.remove").processMs + + opsExecutionBudget("integration.settle").processMs + + 60000 + ); if (ctx.jobType === "standalone.install") return ( opsExecutionBudget("integration.install").processMs + diff --git a/apps/api/src/services/extension-store.ts b/apps/api/src/services/extension-store.ts index 13a72d541..b2dfac5f0 100644 --- a/apps/api/src/services/extension-store.ts +++ b/apps/api/src/services/extension-store.ts @@ -645,7 +645,11 @@ export async function listInstalledExtensions(): Promise ({ ...r, components: byExt.get(r.id) ?? [] })); + // Standalone parents have no distributable extension manifest. Their exact + // ownership/removal controls live in the independent Local integrations view. + return rows + .filter((r) => r.installationKind !== "standalone") + .map((r) => ({ ...r, components: byExt.get(r.id) ?? [] })); } export { PLATFORM_VERSION }; diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index a7526bb8b..4f1d811c1 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -107,6 +107,8 @@ function cancellationReceiptContext(job: typeof adminJob.$inferSelect): JobConte type JobHandler = (ctx: JobContext) => Promise | undefined>; export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ + "standalone.adopt", + "standalone.remove", "standalone.preview", "standalone.install", "backup.operation", @@ -190,7 +192,7 @@ function hasRetainedStandalone(job: { type: string; result: unknown }) { _standaloneSettled?: unknown; } | null; return ( - job.type === "standalone.install" && + ["standalone.install", "standalone.adopt", "standalone.remove"].includes(job.type) && value?._standaloneSettled !== true && typeof value?._standalone?.body === "string" && typeof value._standalone.mac === "string" @@ -228,7 +230,7 @@ class AdminJobRunner { /** Two fixed standalone producers share actor/key serialization and immutable request identity. */ async enqueueStandalone( - type: "standalone.preview" | "standalone.install", + type: "standalone.preview" | "standalone.install" | "standalone.adopt" | "standalone.remove", payload: Record, actorId: string, key: string, diff --git a/apps/api/src/services/standalone-integrations/install.test.ts b/apps/api/src/services/standalone-integrations/install.test.ts index 2d544d94e..2c2b5c0d0 100644 --- a/apps/api/src/services/standalone-integrations/install.test.ts +++ b/apps/api/src/services/standalone-integrations/install.test.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; @@ -43,7 +43,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( let apiUrl: string, agent: ReturnType, store: ArtifactTransferStore; let dropInstallResponse = false, installPosts = 0, - cancelIntegrationId: string | null = null; + cancelIntegrationId: string | null = null, + cancelOwnershipId: string | null = null, + raceOwnershipId: string | null = null; let jobRunner: typeof import("../job-runner").jobRunner; beforeAll(async () => { state.fixture = await createMigratedPostgresFixture(); @@ -118,6 +120,53 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await jobRunner.cancel(job.id); } } + if (cancelOwnershipId && request.method === "GET" && typeof payload === "string") { + const status = JSON.parse(payload).result; + if ( + status?.state === "succeeded" && + status.kind === "integration.remove" && + status.result?.integrationId === cancelOwnershipId + ) { + const id = cancelOwnershipId; + cancelOwnershipId = null; + const rows = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.type, "standalone.remove")); + const job = rows.find((row) => { + const body = (row.result as { _standalone?: { body?: string } } | null)?._standalone + ?.body; + return ( + row.status === "running" && body && JSON.parse(body).request?.integrationId === id + ); + }); + if (!job) throw Error("Missing original quarantined removal job"); + await jobRunner.cancel(job.id); + } + } + if (raceOwnershipId && request.method === "GET" && typeof payload === "string") { + const status = JSON.parse(payload).result; + if ( + status?.state === "succeeded" && + status.kind === "integration.inspectOwnership" && + status.result?.integrationId === raceOwnershipId + ) { + const id = raceOwnershipId; + raceOwnershipId = null; + await state.fixture.db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(schema.installedIntegration) + .where(eq(schema.installedIntegration.id, id)) + .for("update"); + if (!row) throw Error("Missing exact metadata race source"); + await tx + .delete(schema.installedIntegration) + .where(eq(schema.installedIntegration.id, id)); + await tx.insert(schema.installedIntegration).values(row); + }); + } + } return payload; }); cleanups.push(async () => { @@ -140,6 +189,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ({ jobRunner } = await import("../job-runner")); jobRunner.register("standalone.preview", handlers.handleStandalonePreview); jobRunner.register("standalone.install", handlers.handleStandaloneInstall); + const ownership = await import("./ownership"); + jobRunner.register("standalone.adopt", ownership.handleStandaloneOwnership); + jobRunner.register("standalone.remove", ownership.handleStandaloneOwnership); apiUrl = await app.listen({ host: "127.0.0.1", port: 0 }); }, 120000); afterAll(async () => { @@ -409,5 +461,298 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .where(eq(schema.installedIntegration.id, "standalone-cancel")), ).toEqual([]); }, 150000); + it("inventories and explicitly adopts local CLI bytes, then removes through quarantine/reload/bookkeeping", async () => { + const id = "legacy-cli-managed", + directory = join(state.host.root, "custom_integrations", id); + const staging = join(state.host.root, "custom_integrations/.staging"); + mkdirSync(staging, { recursive: true }); + chmodSync(staging, 0o755); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id, + name: "Legacy CLI", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + const response = await fetch(`${apiUrl}/api/admin/standalone-integrations`, { + headers: { "x-openmapx-local-admin": "standalone-local-fixture" }, + }); + expect(response.status).toBe(200); + const entry = (await response.json()).entries.find( + (row: { integrationId: string }) => row.integrationId === id, + ); + expect(entry).toMatchObject({ + ownership: "unmanaged", + adoptable: true, + provenance: "legacy_unknown", + }); + const body = { expectedContentDigest: entry.contentDigest }, + key = randomUUID(); + const accepted = await post(`${id}/adopt`, body, key); + expect(accepted.status).toBe(202); + const adopted = await completed(accepted.body.jobId); + expect(adopted.status, adopted.error ?? JSON.stringify(adopted.result)).toBe("success"); + expect(await post(`${id}/adopt`, body, key)).toEqual(accepted); + const [parent] = await state.fixture.db + .select() + .from(schema.installedExtension) + .where(eq(schema.installedExtension.id, `local-${accepted.body.operationId}`)); + expect(parent).toMatchObject({ + manifest: null, + sourceUrl: null, + installationKind: "standalone", + sourceTrust: "community", + provenance: { kind: "legacy_unknown", contentDigest: entry.contentDigest }, + }); + const remove = await post(`${id}/remove`, body, randomUUID()); + expect(remove.status).toBe(202); + const done = await completed(remove.body.jobId); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("success"); + expect(existsSync(directory)).toBe(false); + expect( + await state.fixture.db + .select() + .from(schema.installedIntegration) + .where(eq(schema.installedIntegration.id, id)), + ).toHaveLength(0); + expect( + await state.fixture.db + .select() + .from(schema.installedExtension) + .where(eq(schema.installedExtension.id, parent!.id)), + ).toHaveLength(0); + const { getIntegration } = await import("../../integration-host"); + expect(getIntegration(id)).toBeUndefined(); + }, 120000); + + it("restores quarantined bytes and exact ownership when a real database delete rolls back", async () => { + const id = "legacy-remove-rollback", + directory = join(state.host.root, "custom_integrations", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id, + name: "Legacy rollback", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + const inventory = await ( + await fetch(`${apiUrl}/api/admin/standalone-integrations`, { + headers: { "x-openmapx-local-admin": "standalone-local-fixture" }, + }) + ).json(); + const body = { + expectedContentDigest: inventory.entries.find( + (row: { integrationId: string }) => row.integrationId === id, + ).contentDigest, + }; + const adopt = await post(`${id}/adopt`, body, randomUUID()); + expect((await completed(adopt.body.jobId)).status).toBe("success"); + const before = await state.fixture + .sql`select xmin::text as xmin from installed_integration where id=${id}`; + await state.fixture + .sql`create table fixture_legacy_remove_hold (id text primary key references installed_integration(id))`; + await state.fixture.sql`insert into fixture_legacy_remove_hold(id) values(${id})`; + try { + const rejected = await post(`${id}/remove`, body, randomUUID()), + done = await completed(rejected.body.jobId); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("failed"); + expect(done.result).toMatchObject({ outcome: "restored" }); + expect(existsSync(directory)).toBe(true); + expect( + await state.fixture + .sql`select xmin::text as xmin from installed_integration where id=${id}`, + ).toEqual(before); + const { getIntegration } = await import("../../integration-host"); + expect(getIntegration(id)?.manifest.version).toBe("1.0.0"); + } finally { + await state.fixture.sql`drop table fixture_legacy_remove_hold`; + } + const retry = await post(`${id}/remove`, body, randomUUID()), + done = await completed(retry.body.jobId); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("success"); + expect(existsSync(directory)).toBe(false); + }, 120000); + + it("cleans standalone bookkeeping only after the agent verifies files removed by CLI are absent", async () => { + const id = "legacy-cli-removed", + directory = join(state.host.root, "custom_integrations", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id, + name: "CLI removed", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + const inventory = async () => + await ( + await fetch(`${apiUrl}/api/admin/standalone-integrations`, { + headers: { "x-openmapx-local-admin": "standalone-local-fixture" }, + }) + ).json(); + const first = (await inventory()).entries.find( + (row: { integrationId: string }) => row.integrationId === id, + ); + const adopt = await post( + `${id}/adopt`, + { expectedContentDigest: first.contentDigest }, + randomUUID(), + ); + expect((await completed(adopt.body.jobId)).status).toBe("success"); + const { removeIntegration } = await import("@openmapx/integration-framework/installer"); + removeIntegration({ rootDir: state.host.root, id }); + const missing = (await inventory()).entries.find( + (row: { integrationId: string }) => row.integrationId === id, + ); + expect(missing).toMatchObject({ + ownership: "standalone", + contentDigest: null, + adoptable: false, + removable: true, + }); + const accepted = await post(`${id}/remove`, { expectedContentDigest: null }, randomUUID()); + expect(accepted.status).toBe(202); + const done = await completed(accepted.body.jobId); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("success"); + expect( + (await inventory()).entries.some( + (row: { integrationId: string }) => row.integrationId === id, + ), + ).toBe(false); + }, 120000); + it("keeps canceled removal pending until its exact quarantined component is restored", async () => { + const id = "legacy-remove-cancel", + directory = join(state.host.root, "custom_integrations", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id, + name: "Cancel removal", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + const inventory = await ( + await fetch(`${apiUrl}/api/admin/standalone-integrations`, { + headers: { "x-openmapx-local-admin": "standalone-local-fixture" }, + }) + ).json(); + const body = { + expectedContentDigest: inventory.entries.find( + (row: { integrationId: string }) => row.integrationId === id, + ).contentDigest, + }; + cancelOwnershipId = id; + const accepted = await post(`${id}/remove`, body, randomUUID()); + for (let i = 0; i < 200 && cancelOwnershipId; i++) await delay(25); + expect(cancelOwnershipId).toBeNull(); + await delay(1000); + const [pending] = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, String(accepted.body.jobId))); + if (pending!.status === "cancel_pending") { + expect(pending!.finishedAt).toBeNull(); + expect(existsSync(directory)).toBe(false); + await delay(61000); + await jobRunner.reconcile(); + } + const done = await completed(accepted.body.jobId, true); + expect(done.status, done.error ?? JSON.stringify(done.result)).toBe("canceled"); + expect(existsSync(directory)).toBe(true); + const { hasVerifiedStandaloneRestoration } = await import("./settlement-checkpoint"); + expect(await hasVerifiedStandaloneRestoration(done.result, done.id)).toBe(true); + const forged = structuredClone(done.result) as { _standalone: { mac: string } }; + forged._standalone.mac = "0".repeat(64); + expect(await hasVerifiedStandaloneRestoration(forged, done.id)).toBe(false); + }, 180000); + it("preserves a delete/reinsert ownership race instead of deleting the replacement row", async () => { + const id = "legacy-row-reinsert", + directory = join(state.host.root, "custom_integrations", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id, + name: "Ownership race", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + const inventory = await ( + await fetch(`${apiUrl}/api/admin/standalone-integrations`, { + headers: { "x-openmapx-local-admin": "standalone-local-fixture" }, + }) + ).json(), + body = { + expectedContentDigest: inventory.entries.find( + (row: { integrationId: string }) => row.integrationId === id, + ).contentDigest, + }; + const adopted = await post(`${id}/adopt`, body, randomUUID()); + expect((await completed(adopted.body.jobId)).status).toBe("success"); + const before = await state.fixture + .sql`select xmin::text as xmin from installed_integration where id=${id}`; + raceOwnershipId = id; + const accepted = await post(`${id}/remove`, body, randomUUID()); + let held: typeof schema.adminJob.$inferSelect | undefined; + for (let i = 0; i < 2400; i++) { + const [row] = await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, String(accepted.body.jobId))); + const result = row?.result as { + _standalone?: { body: string }; + observation?: { reason: string }; + } | null; + if ( + result?._standalone && + JSON.parse(result._standalone.body).phase === "persist_attempted" && + result.observation?.reason === "recovery_required" + ) { + held = row; + break; + } + await delay(25); + } + expect(raceOwnershipId).toBeNull(); + expect(held?.finishedAt).toBeNull(); + expect(held?.status).toBe("running"); + const after = await state.fixture + .sql`select xmin::text as xmin from installed_integration where id=${id}`; + expect(after).toHaveLength(1); + expect(after[0]!.xmin).not.toBe(before[0]!.xmin); + const heldInventory = await ( + await fetch(`${apiUrl}/api/admin/standalone-integrations`, { + headers: { "x-openmapx-local-admin": "standalone-local-fixture" }, + }) + ).json(); + expect( + heldInventory.entries.find((row: { integrationId: string }) => row.integrationId === id), + ).toMatchObject({ + pendingTransactionId: accepted.body.operationId, + adoptable: false, + removable: false, + }); + }, 120000); }, ); diff --git a/apps/api/src/services/standalone-integrations/install.ts b/apps/api/src/services/standalone-integrations/install.ts index 660a93341..19c5b8103 100644 --- a/apps/api/src/services/standalone-integrations/install.ts +++ b/apps/api/src/services/standalone-integrations/install.ts @@ -228,7 +228,7 @@ async function bookkeeping( operationId: state.operationId, }); const now = new Date(), - repository = source.kind === "uploaded-artifact" ? null : source.url; + repository = "url" in source ? source.url : null; await tx .insert(installedExtension) .values({ diff --git a/apps/api/src/services/standalone-integrations/inventory.ts b/apps/api/src/services/standalone-integrations/inventory.ts new file mode 100644 index 000000000..222f3f63e --- /dev/null +++ b/apps/api/src/services/standalone-integrations/inventory.ts @@ -0,0 +1,75 @@ +import type { StandaloneFilesystemEntry, StandaloneInventoryEntry } from "@openmapx/core/ops"; + +interface Input { + files: StandaloneFilesystemEntry[]; + integrations: Array<{ id: string; installedVersion: string; managedByExtension: string | null }>; + parents: Array<{ + id: string; + installationKind: "catalog" | "standalone"; + provenance: { kind: string; contentDigest?: string } | null; + }>; + components: Array<{ kind: string; componentId: string; extensionId: string }>; +} +/** Local ownership comes from installed rows and actual bytes, never catalog enrichment. */ +export function mergeStandaloneInventory(input: Input): StandaloneInventoryEntry[] { + const ids = new Set([ + ...input.files.map((x) => x.integrationId), + ...input.integrations.map((x) => x.id), + ...input.components.filter((x) => x.kind === "integration").map((x) => x.componentId), + ]); + return [...ids].sort().map((integrationId) => { + const file = input.files.find((x) => x.integrationId === integrationId), + row = input.integrations.find((x) => x.id === integrationId), + components = input.components.filter( + (x) => x.kind === "integration" && x.componentId === integrationId, + ), + owner = components[0]?.extensionId ?? row?.managedByExtension ?? null, + parent = input.parents.find((x) => x.id === owner); + let ownership: StandaloneInventoryEntry["ownership"] = "unmanaged", + denialReason = file?.denialReason ?? null; + if ( + (owner && (!row || components.length !== 1 || row.managedByExtension !== owner)) || + components.length > 1 || + (row?.managedByExtension && row.managedByExtension !== owner) || + (owner && !parent) + ) + ownership = "conflict"; + else if (parent) + ownership = parent.installationKind === "standalone" ? "standalone" : "extension"; + if (!file) { + if (ownership === "standalone" && row && components.length === 1) + denialReason = + "Local files are missing. Removal verifies absence before cleaning up ownership records."; + else { + ownership = "conflict"; + denialReason = + "Recorded installation is missing from local storage; reconcile its original Activity job."; + } + } else if (!file.valid) + denialReason ??= "Legacy content cannot be safely managed; inspect it on the host."; + if (ownership === "conflict") + denialReason ??= + "Installation ownership is inconsistent; resolve the original owner before changing it."; + if (ownership === "extension") + denialReason ??= "Managed by an extension; use that extension's controls."; + const usable = !!file?.valid && file.contentDigest !== null; + return { + ...(file?.pendingTransactionId ? { pendingTransactionId: file.pendingTransactionId } : {}), + integrationId, + version: file?.version ?? row?.installedVersion ?? null, + ownership, + extensionId: owner, + contentDigest: file?.contentDigest ?? null, + provenance: (["artifact", "source", "uploaded-artifact"].includes( + parent?.provenance?.kind ?? "", + ) + ? parent!.provenance!.kind + : "legacy_unknown") as StandaloneInventoryEntry["provenance"], + adoptable: usable && ownership === "unmanaged", + removable: + (usable && (ownership === "standalone" || ownership === "unmanaged")) || + (!file && ownership === "standalone"), + denialReason, + }; + }); +} diff --git a/apps/api/src/services/standalone-integrations/ownership-checkpoint.ts b/apps/api/src/services/standalone-integrations/ownership-checkpoint.ts new file mode 100644 index 000000000..4cadbea7e --- /dev/null +++ b/apps/api/src/services/standalone-integrations/ownership-checkpoint.ts @@ -0,0 +1,70 @@ +import { canonicalJson, type OpsOperation } from "@openmapx/core/ops"; +import type { db } from "../../db"; +import type { adminJob } from "../../db/schema"; +import { readProjection } from "../admin-job-projection"; +import { JobOwnershipLost } from "../admin-job-state"; +import { lockOwnership, ownershipSnapshot } from "./ownership-database"; +import { readOwnershipState } from "./ownership-state"; +export async function assertOwnershipSettlementCheckpoint( + row: typeof adminJob.$inferSelect, + operation: OpsOperation, + connection: Pick, +) { + if ( + !["standalone.adopt", "standalone.remove"].includes(row.type) || + operation.kind !== "integration.settle" + ) + throw new JobOwnershipLost(); + const result = row.result as Record, + payload = row.payload as Record, + state = await readOwnershipState(result._standalone); + if ( + state.jobId !== row.id || + state.ownerId !== payload.ownerId || + state.ownerId !== payload._standaloneActor || + state.operationId !== payload.operationId || + state.request.integrationId !== payload.integrationId || + state.request.expectedContentDigest !== payload.expectedContentDigest || + operation.transactionId !== state.operationId || + operation.expectedContentDigest !== state.request.expectedContentDigest || + !state.inspection?.allowed + ) + throw new JobOwnershipLost(); + const sources = Object.values(readProjection(result).operations).filter( + (x) => + x.operation?.kind === "integration.inspectOwnership" && + x.operation.operationId === state.operationId, + ); + const source = sources[0], + receipt = source?.receiptDelivery; + if ( + sources.length !== 1 || + !source?.terminal || + source.attempt?.apiJobId !== row.id || + receipt?.state !== "confirmed" || + receipt.terminalReceipt.state !== "succeeded" || + receipt.terminalReceipt.operationKey !== source.operationKey || + receipt.terminalReceipt.fingerprint !== source.fingerprint || + canonicalJson(receipt.terminalReceipt.result) !== canonicalJson(state.inspection) + ) + throw new JobOwnershipLost(); + await lockOwnership(connection, state.request.integrationId); + const current = await ownershipSnapshot(connection, state.request.integrationId); + if (operation.decision === "commit") { + if (state.phase !== "persisted") throw new JobOwnershipLost(); + if (state.action === "adopt") { + if ( + current.parent?.id !== state.parentId || + current.parent.lastOperationId !== state.operationId || + current.parent.installationKind !== "standalone" || + current.integration?.managedByExtension !== state.parentId || + current.integration.sourceType !== "legacy_unknown" || + current.components.length !== 1 || + current.components[0]?.extensionId !== state.parentId + ) + throw new JobOwnershipLost(); + } else if (current.integration || current.components.length || current.parent) + throw new JobOwnershipLost(); + } else if (state.phase !== "restore" || canonicalJson(current) !== state.before) + throw new JobOwnershipLost(); +} diff --git a/apps/api/src/services/standalone-integrations/ownership-database.ts b/apps/api/src/services/standalone-integrations/ownership-database.ts new file mode 100644 index 000000000..ad0598898 --- /dev/null +++ b/apps/api/src/services/standalone-integrations/ownership-database.ts @@ -0,0 +1,59 @@ +import { and, eq, sql } from "drizzle-orm"; +import type { db } from "../../db"; +import { + installedExtension, + installedExtensionComponent, + installedIntegration, +} from "../../db/schema"; + +type Connection = Pick; +export async function ownershipSnapshot(tx: Connection, id: string) { + const [integration] = await tx + .select({ + id: installedIntegration.id, + installedVersion: installedIntegration.installedVersion, + managedByExtension: installedIntegration.managedByExtension, + repository: installedIntegration.repository, + sourceType: installedIntegration.sourceType, + xmin: sql`xmin::text`, + }) + .from(installedIntegration) + .where(eq(installedIntegration.id, id)) + .for("update"); + const components = await tx + .select({ + extensionId: installedExtensionComponent.extensionId, + kind: installedExtensionComponent.kind, + componentId: installedExtensionComponent.componentId, + xmin: sql`xmin::text`, + }) + .from(installedExtensionComponent) + .where( + and( + eq(installedExtensionComponent.kind, "integration"), + eq(installedExtensionComponent.componentId, id), + ), + ) + .for("update"); + const parentId = components[0]?.extensionId ?? integration?.managedByExtension ?? null; + const parents = parentId + ? await tx + .select({ + id: installedExtension.id, + installationKind: installedExtension.installationKind, + sourceTrust: installedExtension.sourceTrust, + provenance: installedExtension.provenance, + lastOperationId: installedExtension.lastOperationId, + xmin: sql`xmin::text`, + }) + .from(installedExtension) + .where(eq(installedExtension.id, parentId)) + .for("update") + : []; + return { integration: integration ?? null, components, parent: parents[0] ?? null }; +} +export async function lockOwnership(tx: Connection, id: string) { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${`standalone-bookkeeping:${id}`},0))`, + ); +} diff --git a/apps/api/src/services/standalone-integrations/ownership-state.ts b/apps/api/src/services/standalone-integrations/ownership-state.ts new file mode 100644 index 000000000..4ca1a2f3c --- /dev/null +++ b/apps/api/src/services/standalone-integrations/ownership-state.ts @@ -0,0 +1,80 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { + canonicalJson, + readOpsTokenFile, + standaloneOwnershipInspectionSchema, +} from "@openmapx/core/ops"; +import z from "zod/v4"; +export const ownershipRequestSchema = z.strictObject({ + integrationId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + expectedContentDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), +}); +export const ownershipStateSchema = z.strictObject({ + version: z.literal(1), + kind: z.literal("ownership"), + jobId: z.uuid(), + operationId: z.uuid(), + ownerId: z.string().max(128), + action: z.enum(["adopt", "remove"]), + request: ownershipRequestSchema, + parentId: z.string().max(128).nullable(), + before: z.string().max(65536), + inspection: standaloneOwnershipInspectionSchema.nullable(), + phase: z.enum([ + "prepared", + "inspected", + "removed", + "reload_attempted", + "reloaded", + "persist_attempted", + "persisted", + "restore", + "complete", + ]), + outcome: z.enum(["committed", "restored"]).nullable(), +}); +export type OwnershipState = z.infer; +const token = () => readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); +export async function sealOwnershipState(state: OwnershipState) { + const body = canonicalJson(ownershipStateSchema.parse(state)); + return { + body, + mac: createHmac("sha256", await token()) + .update("openmapx-standalone-ownership-v1\0") + .update(body) + .digest("hex"), + }; +} +export async function readOwnershipState(input: unknown) { + const saved = z + .strictObject({ body: z.string().max(131072), mac: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse(input), + expected = createHmac("sha256", await token()) + .update("openmapx-standalone-ownership-v1\0") + .update(saved.body) + .digest(); + if (!timingSafeEqual(expected, Buffer.from(saved.mac, "hex"))) + throw Error("Ownership checkpoint authentication failed"); + return ownershipStateSchema.parse(JSON.parse(saved.body)); +} +export async function authenticateOwnershipRequest(type: string, payload: Record) { + const request = ownershipRequestSchema.parse({ + integrationId: payload.integrationId, + expectedContentDigest: payload.expectedContentDigest, + }); + const expected = createHmac("sha256", await token()) + .update("openmapx-standalone-request-v1\0") + .update(canonicalJson({ type, payload: request, actorId: payload.ownerId })) + .digest("hex"); + if ( + payload.ownerId !== payload._standaloneActor || + payload._requestDigest !== expected || + !z.uuid().safeParse(payload.operationId).success || + typeof payload.ownerId !== "string" + ) + throw Error("Ownership request authentication failed"); + return { request, ownerId: payload.ownerId, operationId: String(payload.operationId) }; +} diff --git a/apps/api/src/services/standalone-integrations/ownership.test.ts b/apps/api/src/services/standalone-integrations/ownership.test.ts new file mode 100644 index 000000000..7eaa5f2ee --- /dev/null +++ b/apps/api/src/services/standalone-integrations/ownership.test.ts @@ -0,0 +1,74 @@ +import { expect, it } from "vitest"; +import { mergeStandaloneInventory } from "./inventory"; + +it("merges filesystem and database ownership without catalog availability or invented provenance", () => { + const files = [ + { + integrationId: "cli-installed", + version: "1.0.0", + contentDigest: "a".repeat(64), + valid: true, + denialReason: null, + }, + { + integrationId: "invalid-code", + version: null, + contentDigest: null, + valid: false, + denialReason: "Unsupported executable integration", + }, + ]; + const rows = [ + { + id: "database-only", + installedVersion: "2.0", + managedByExtension: "local-11111111-1111-4111-8111-111111111111", + }, + ]; + const entries = mergeStandaloneInventory({ + files, + integrations: rows, + parents: [], + components: [], + }); + expect(entries.find((x) => x.integrationId === "cli-installed")).toMatchObject({ + ownership: "unmanaged", + provenance: "legacy_unknown", + adoptable: true, + removable: true, + }); + expect(entries.find((x) => x.integrationId === "invalid-code")).toMatchObject({ + adoptable: false, + removable: false, + }); + expect(entries.find((x) => x.integrationId === "database-only")).toMatchObject({ + ownership: "conflict", + version: "2.0", + contentDigest: null, + adoptable: false, + removable: false, + }); +}); +it("refuses foreign or inconsistent owners and preserves catalog-owned rows", () => { + const f = { + integrationId: "fixture", + version: "1.0", + contentDigest: "b".repeat(64), + valid: true, + denialReason: null, + }; + const catalog = mergeStandaloneInventory({ + files: [f], + integrations: [{ id: "fixture", installedVersion: "1.0", managedByExtension: "catalog" }], + parents: [{ id: "catalog", installationKind: "catalog", provenance: null }], + components: [{ kind: "integration", componentId: "fixture", extensionId: "catalog" }], + }); + expect(catalog[0]).toMatchObject({ ownership: "extension", adoptable: false, removable: false }); + const conflict = mergeStandaloneInventory({ + files: [f], + integrations: [{ id: "fixture", installedVersion: "1.0", managedByExtension: "other" }], + parents: [], + components: [{ kind: "integration", componentId: "fixture", extensionId: "catalog" }], + }); + expect(conflict[0]).toMatchObject({ ownership: "conflict", adoptable: false, removable: false }); +}); diff --git a/apps/api/src/services/standalone-integrations/ownership.ts b/apps/api/src/services/standalone-integrations/ownership.ts new file mode 100644 index 000000000..dd65e5997 --- /dev/null +++ b/apps/api/src/services/standalone-integrations/ownership.ts @@ -0,0 +1,390 @@ +import { randomUUID } from "node:crypto"; +import { canonicalJson } from "@openmapx/core/ops"; +import { and, eq, sql } from "drizzle-orm"; +import { db } from "../../db"; +import { + adminJob, + installedExtension, + installedExtensionComponent, + installedIntegration, +} from "../../db/schema"; +import { getIntegration, reloadIntegrations } from "../../integration-host"; +import { dbActorId } from "../../utils/actor"; +import { AdminJobFailure, JobObservationPending } from "../admin-job-failure"; +import { executeAdminJobOperation } from "../admin-job-ops"; +import { jobOwnership } from "../admin-job-state"; +import { withExtensionChangeLock } from "../extension-recovery"; +import type { JobContext } from "../job-runner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../ops-client"; +import { mergeStandaloneInventory } from "./inventory"; +import { lockOwnership, ownershipSnapshot } from "./ownership-database"; +import { + authenticateOwnershipRequest, + type OwnershipState, + readOwnershipState, + sealOwnershipState, +} from "./ownership-state"; + +type Connection = Parameters[0]>[0]; +export async function standaloneInventory() { + const [files, integrations, parents, components] = await Promise.all([ + executeAndWait( + createApiOpsClient(), + { kind: "integration.inventory" }, + createDurableOpsKey("standalone.inventory", randomUUID()), + ), + db.select().from(installedIntegration), + db.select().from(installedExtension), + db.select().from(installedExtensionComponent), + ]); + return { + entries: mergeStandaloneInventory({ files: files.entries, integrations, parents, components }), + }; +} +function permission( + before: Awaited>, + action: "adopt" | "remove", +) { + const { integration, components, parent } = before, + owner = components[0]?.extensionId ?? integration?.managedByExtension ?? null; + if ( + (owner && + (!integration || components.length !== 1 || integration.managedByExtension !== owner)) || + components.length > 1 || + (integration?.managedByExtension && integration.managedByExtension !== owner) || + (owner && (!parent || parent.installationKind !== "standalone")) + ) + throw new AdminJobFailure( + "Integration has a conflicting or catalog owner; use that owner's controls", + {}, + ); + if (action === "adopt" && owner) throw new AdminJobFailure("Integration is already managed", {}); + return owner; +} +async function save(ctx: JobContext, state: OwnershipState) { + await ctx.checkpoint({ + ...ctx.checkpointResult, + _standalone: await sealOwnershipState(state), + _standaloneSettled: state.phase === "complete", + }); +} +async function readState( + ctx: JobContext, + payload: Awaited>, +) { + if (!ctx.checkpointResult?._standalone) return null; + const state = await readOwnershipState(ctx.checkpointResult._standalone); + if ( + state.jobId !== ctx.jobId || + state.operationId !== payload.operationId || + state.ownerId !== payload.ownerId || + canonicalJson(state.request) !== canonicalJson(payload.request) || + state.action !== (ctx.jobType === "standalone.adopt" ? "adopt" : "remove") + ) + throw new JobObservationPending("recovery_required"); + return state; +} +async function canceled(ctx: JobContext) { + const [row] = await db + .select({ status: adminJob.status }) + .from(adminJob) + .where(eq(adminJob.id, ctx.jobId)); + if (!row) throw new JobObservationPending("recovery_required"); + return row.status === "cancel_pending" || row.status === "canceled"; +} +/** Desired ownership and its authenticated completion checkpoint commit in the SAME transaction. + * A lost COMMIT response is resolved under the original advisory and owned-job locks. */ +async function persist(ctx: JobContext, state: OwnershipState, write: boolean): Promise { + return db.transaction(async (tx) => { + const [job] = await tx + .select() + .from(adminJob) + .where(jobOwnership(ctx.jobId, ctx.execution, ["running", "cancel_pending"])) + .for("update"); + if (!job) throw new JobObservationPending("recovery_required"); + await lockOwnership(tx, state.request.integrationId); + const value = job.result as Record, + retained = await readOwnershipState(value._standalone); + if ( + retained.operationId !== state.operationId || + retained.jobId !== ctx.jobId || + canonicalJson(retained.request) !== canonicalJson(state.request) + ) + throw new JobObservationPending("recovery_required"); + if ( + retained.phase === "persisted" || + (retained.phase === "complete" && retained.outcome === "committed") + ) { + state.phase = retained.phase; + state.outcome = retained.outcome; + ctx.checkpointResult = value; + return true; + } + const current = await ownershipSnapshot(tx, state.request.integrationId); + if (canonicalJson(current) !== state.before) + throw new JobObservationPending("recovery_required"); + if (!write) return false; + permission(current, state.action); + if (!state.inspection?.allowed) throw new JobObservationPending("recovery_required"); + const id = state.request.integrationId; + if (state.action === "adopt") { + const contentDigest = state.request.expectedContentDigest, + version = state.inspection.version; + if (!contentDigest || !version) throw new JobObservationPending("recovery_required"); + const parentId = state.parentId; + if (!parentId) throw new JobObservationPending("recovery_required"); + await tx.insert(installedExtension).values({ + id: parentId, + name: id, + sourceUrl: null, + sourceTrust: "community", + installationKind: "standalone", + provenance: { kind: "legacy_unknown", contentDigest }, + lastOperationId: state.operationId, + installedVersion: version, + manifest: null, + installedBy: dbActorId(state.ownerId), + }); + await tx + .insert(installedIntegration) + .values({ + id, + repository: null, + sourceType: "legacy_unknown", + installedVersion: version, + managedByExtension: parentId, + installedBy: dbActorId(state.ownerId), + }) + .onConflictDoUpdate({ + target: installedIntegration.id, + set: { + managedByExtension: parentId, + repository: null, + sourceType: "legacy_unknown", + installedVersion: version, + updatedAt: new Date(), + }, + }); + await tx + .insert(installedExtensionComponent) + .values({ extensionId: parentId, kind: "integration", componentId: id }); + } else { + if (state.parentId) { + const owned = await tx + .select() + .from(installedExtensionComponent) + .where(eq(installedExtensionComponent.extensionId, state.parentId)) + .for("update"); + if (owned.length !== 1 || owned[0]?.componentId !== id || owned[0].kind !== "integration") + throw new AdminJobFailure( + "Installation contains other components; use its extension controls", + {}, + ); + await tx + .delete(installedExtensionComponent) + .where( + and( + eq(installedExtensionComponent.extensionId, state.parentId), + eq(installedExtensionComponent.kind, "integration"), + eq(installedExtensionComponent.componentId, id), + ), + ); + } + await tx.delete(installedIntegration).where(eq(installedIntegration.id, id)); + if (state.parentId) + await tx.delete(installedExtension).where(eq(installedExtension.id, state.parentId)); + } + const committed = { ...state, phase: "persisted" as const }; + const result = { + ...value, + _standalone: await sealOwnershipState(committed), + _standaloneSettled: false, + }; + const updated = await tx + .update(adminJob) + .set({ result }) + .where(jobOwnership(ctx.jobId, ctx.execution, ["running", "cancel_pending"])) + .returning({ id: adminJob.id }); + if (updated.length !== 1) throw new JobObservationPending("recovery_required"); + // Updating this local view is safe only after the transaction resolves. The + // caller performs an authenticated read after ambiguous COMMIT instead. + return true; + }); +} +async function observedCommit(ctx: JobContext, state: OwnershipState) { + const committed = await persist(ctx, state, false); + if (committed) { + state.phase = "persisted"; + await save(ctx, state); + } + return committed; +} +export function handleStandaloneOwnership(ctx: JobContext): Promise> { + return withExtensionChangeLock(async () => { + const payload = await authenticateOwnershipRequest(ctx.jobType ?? "", ctx.payload); + let state = await readState(ctx, payload); + if (!state) { + if (ctx.observationOnly) throw new JobObservationPending("recovery_required"); + ctx.signal.throwIfAborted(); + const action = ctx.jobType === "standalone.adopt" ? "adopt" : "remove"; + if (action === "adopt" && !payload.request.expectedContentDigest) + throw new AdminJobFailure("Missing integrations cannot be adopted", {}); + const before = await db.transaction(async (tx) => { + await lockOwnership(tx, payload.request.integrationId); + const value = await ownershipSnapshot(tx, payload.request.integrationId); + permission(value, action); + return value; + }); + state = { + version: 1, + kind: "ownership", + jobId: ctx.jobId, + operationId: payload.operationId, + ownerId: payload.ownerId, + action, + request: payload.request, + parentId: action === "adopt" ? `local-${payload.operationId}` : (before.parent?.id ?? null), + before: canonicalJson(before), + inspection: null, + phase: "prepared", + outcome: null, + }; + await save(ctx, state); + } + if (state.phase === "prepared") { + const inspection = await executeAdminJobOperation( + ctx, + { + kind: "integration.inspectOwnership", + integrationId: state.request.integrationId, + expectedContentDigest: state.request.expectedContentDigest, + ownerId: state.ownerId, + operationId: state.operationId, + }, + "standalone.ownership.inspect", + ); + if (!inspection.allowed) { + state.phase = "complete"; + state.outcome = "restored"; + await save(ctx, state); + throw new AdminJobFailure(inspection.reason, { + integrationId: state.request.integrationId, + }); + } + state.inspection = inspection; + state.phase = "inspected"; + await save(ctx, state); + } + if ( + ["inspected", "removed", "reloaded"].includes(state.phase) && + state.inspection?.allowed && + Date.parse(state.inspection.expiresAt) <= Date.now() + ) { + state.phase = "restore"; + await save(ctx, state); + } + if (state.phase === "inspected" && (await canceled(ctx))) { + state.phase = "restore"; + await save(ctx, state); + } + if (state.phase === "inspected") { + if (state.action === "remove") + await executeAdminJobOperation( + ctx, + { + kind: "integration.remove", + transactionId: state.operationId, + expectedContentDigest: state.request.expectedContentDigest, + }, + "standalone.ownership.remove", + ); + state.phase = "removed"; + await save(ctx, state); + } + if (state.phase === "removed") { + if (await canceled(ctx)) { + state.phase = "restore"; + await save(ctx, state); + } else { + state.phase = "reload_attempted"; + await save(ctx, state); + await reloadIntegrations(); + if ( + (state.action === "remove" && getIntegration(state.request.integrationId)) || + (state.action === "adopt" && + getIntegration(state.request.integrationId)?.manifest.version !== + (state.inspection?.allowed ? state.inspection.version : null)) + ) + throw new JobObservationPending("recovery_required"); + state.phase = "reloaded"; + await save(ctx, state); + } + } + if (state.phase === "reload_attempted") throw new JobObservationPending("recovery_required"); + if (state.phase === "reloaded") { + state.phase = (await canceled(ctx)) ? "restore" : "persist_attempted"; + await save(ctx, state); + if (state.phase === "persist_attempted") { + try { + await persist(ctx, state, true); + state.phase = "persisted"; + } catch (error) { + if (!(await observedCommit(ctx, state))) { + state.phase = "restore"; + } + } + await save(ctx, state); + } + } + if (state.phase === "persist_attempted") { + if (await observedCommit(ctx, state)) state.phase = "persisted"; + else { + state.phase = "restore"; + await save(ctx, state); + } + } + if (state.phase === "restore") { + await executeAdminJobOperation( + ctx, + { + kind: "integration.settle", + transactionId: state.operationId, + decision: "restore", + expectedContentDigest: state.request.expectedContentDigest, + }, + "standalone.ownership.restore", + ); + // Reload ambiguity remains held in Activity; never infer rollback completion. + await reloadIntegrations(); + state.phase = "complete"; + state.outcome = "restored"; + await save(ctx, state); + } + if (state.phase === "persisted") { + await executeAdminJobOperation( + ctx, + { + kind: "integration.settle", + transactionId: state.operationId, + decision: "commit", + expectedContentDigest: state.request.expectedContentDigest, + }, + "standalone.ownership.commit", + ); + state.phase = "complete"; + state.outcome = "committed"; + await save(ctx, state); + } + if (state.outcome !== "committed") + throw new AdminJobFailure("Installation ownership change was restored", { + integrationId: state.request.integrationId, + outcome: "restored", + }); + return { + operationId: state.operationId, + integrationId: state.request.integrationId, + action: state.action, + outcome: "committed", + }; + }); +} diff --git a/apps/api/src/services/standalone-integrations/settlement-checkpoint.ts b/apps/api/src/services/standalone-integrations/settlement-checkpoint.ts index 2c949cb8e..bc05c69fa 100644 --- a/apps/api/src/services/standalone-integrations/settlement-checkpoint.ts +++ b/apps/api/src/services/standalone-integrations/settlement-checkpoint.ts @@ -9,6 +9,8 @@ import { } from "../../db/schema"; import { readProjection } from "../admin-job-projection"; import { JobOwnershipLost } from "../admin-job-state"; +import { assertOwnershipSettlementCheckpoint } from "./ownership-checkpoint"; +import { readOwnershipState } from "./ownership-state"; import { readStandaloneState } from "./state"; /** A cancellation can finalize only its own published component. No forward install admission. */ export async function assertStandaloneSettlementCheckpoint( @@ -16,6 +18,8 @@ export async function assertStandaloneSettlementCheckpoint( operation: OpsOperation, connection: Pick, ): Promise { + if (row.type === "standalone.adopt" || row.type === "standalone.remove") + return assertOwnershipSettlementCheckpoint(row, operation, connection); if (row.type !== "standalone.install" || operation.kind !== "integration.settle") throw new JobOwnershipLost(); const current = row.result as Record | null, @@ -128,6 +132,32 @@ export async function hasVerifiedStandaloneRestoration( !Number.isFinite(Date.parse(value._cancelRequestedAt)) ) return false; + let ownership: Awaited> | undefined; + try { + ownership = await readOwnershipState(value._standalone); + } catch {} + if (ownership) { + if ( + ownership.jobId !== jobId || + ownership.phase !== "complete" || + ownership.outcome !== "restored" + ) + return false; + const children = Object.values(readProjection(value).operations), + restores = children.filter( + (x) => + x.operation?.kind === "integration.settle" && + x.operation.transactionId === ownership.operationId && + x.operation.decision === "restore", + ); + return ( + children.every((x) => x.terminal && x.receiptDelivery?.state === "confirmed") && + restores.length === 1 && + restores[0]?.receiptDelivery?.terminalReceipt.state === "succeeded" && + (restores[0].receiptDelivery.terminalReceipt.result as { outcome?: string }).outcome === + "restored" + ); + } const state = await readStandaloneState(value._standalone); if (state.jobId !== jobId || state.phase !== "complete" || state.outcome !== "restored") return false; diff --git a/apps/ops-agent/src/artifact-transfer/store.ts b/apps/ops-agent/src/artifact-transfer/store.ts index 3da82cc0d..a57dde949 100644 --- a/apps/ops-agent/src/artifact-transfer/store.ts +++ b/apps/ops-agent/src/artifact-transfer/store.ts @@ -49,8 +49,10 @@ import z from "zod/v4"; import type { OpsExecutionContext } from "../runtime"; import { type StandaloneInspectionRecord, + type StandaloneOwnershipRecord, type StandaloneTransactionRecord, standaloneInspectionRecordSchema, + standaloneOwnershipRecordSchema, standaloneTransactionRecordSchema, } from "../standalone-integration-records"; import { ArtifactTransferError } from "./errors"; @@ -698,9 +700,14 @@ export class ArtifactTransferStore { canonicalJson(authority.execution) !== canonicalJson(context.execution) || authority.kind !== operation.kind || opsOperationFingerprint(operation) !== authority.execution.operationFingerprint || - !["integration.inspectInstall", "integration.install", "integration.settle"].includes( - operation.kind, - ) + ![ + "integration.inspectInstall", + "integration.install", + "integration.settle", + "integration.inventory", + "integration.inspectOwnership", + "integration.remove", + ].includes(operation.kind) ) throw Error("Integration execution authority does not match the retained operation"); return operation; @@ -859,6 +866,126 @@ export class ArtifactTransferStore { this.writeIntegrationRecord(name, record, previous === null); }); } + standalonePendingInventory() { + return this.lock(() => { + const entries: Array<{ + integrationId: string; + version: string | null; + contentDigest: string | null; + valid: boolean; + denialReason: string; + pendingTransactionId: string; + }> = []; + for (const name of this.integrationRecords.listBounded(2048)) { + if (/^ownership-[a-f0-9-]{36}\.json$/.test(name)) { + const record = this.readIntegrationRecord(name, standaloneOwnershipRecordSchema); + if (!["committed", "restored"].includes(record.phase)) + entries.push({ + integrationId: record.integrationId, + version: record.componentVersion, + contentDigest: record.contentDigest, + valid: false, + pendingTransactionId: record.transactionId, + denialReason: `Transaction ${record.transactionId} holds this component. Reconcile its original Activity job.`, + }); + } else if (/^transaction-[a-f0-9-]{36}\.json$/.test(name)) { + const record = this.readIntegrationRecord(name, standaloneTransactionRecordSchema); + if ( + !record.publication || + !["committed", "restored"].includes(record.publication.outcome) + ) + entries.push({ + integrationId: record.inspection.inspection.integrationId, + version: record.inspection.inspection.version, + contentDigest: record.inspection.inspection.contentDigest, + valid: false, + pendingTransactionId: record.transactionId, + denialReason: `Transaction ${record.transactionId} holds this component. Reconcile its original Activity job.`, + }); + } + } + return entries; + }); + } + readStandaloneOwnership( + context: OpsExecutionContext, + transactionId: string, + ): StandaloneOwnershipRecord { + const installationId = this.standaloneInstallationId(context), + op = this.integrationContext(context); + if ( + (op.kind !== "integration.inspectOwnership" || op.operationId !== transactionId) && + (op.kind !== "integration.remove" || op.transactionId !== transactionId) && + (op.kind !== "integration.settle" || op.transactionId !== transactionId) + ) + throw Error("Ownership execution mismatch"); + return this.lock(() => { + const record = this.readIntegrationRecord( + `ownership-${id.parse(transactionId)}.json`, + standaloneOwnershipRecordSchema, + ); + if ( + record.installationId !== installationId || + (op.kind === "integration.inspectOwnership" && record.ownerId !== op.ownerId) + ) + throw Error("Ownership installation or actor mismatch"); + return record; + }); + } + retainStandaloneOwnership(context: OpsExecutionContext, input: StandaloneOwnershipRecord): void { + const record = standaloneOwnershipRecordSchema.parse(input), + installationId = this.standaloneInstallationId(context), + op = this.integrationContext(context); + if (record.installationId !== installationId) throw Error("Ownership installation mismatch"); + if (op.kind === "integration.inspectOwnership") { + if ( + op.operationId !== record.transactionId || + op.integrationId !== record.integrationId || + op.ownerId !== record.ownerId || + op.expectedContentDigest !== record.contentDigest || + canonicalJson(context.execution) !== canonicalJson(record.inspectionExecution) || + record.phase !== "inspected" + ) + throw Error("Ownership inspection producer mismatch"); + } else if (op.kind === "integration.remove") { + if ( + op.transactionId !== record.transactionId || + op.expectedContentDigest !== record.contentDigest || + canonicalJson(context.execution) !== canonicalJson(record.removeExecution) || + !["remove_requested", "removed"].includes(record.phase) + ) + throw Error("Removal producer mismatch"); + } else if ( + op.kind !== "integration.settle" || + op.transactionId !== record.transactionId || + op.expectedContentDigest !== record.contentDigest || + op.decision !== record.decision + ) + throw Error("Ownership settlement producer mismatch"); + this.lock(() => { + const name = `ownership-${record.transactionId}.json`; + let previous: StandaloneOwnershipRecord | null = null; + try { + previous = this.readIntegrationRecord(name, standaloneOwnershipRecordSchema); + } catch (error) { + if (!absent(error)) throw error; + } + if (previous) { + const { phase: _p, decision: _d, removeExecution: _e, ...before } = previous, + { phase: _n, decision: _nd, removeExecution: _ne, ...after } = record; + if ( + canonicalJson(before) !== canonicalJson(after) || + (previous.removeExecution && + canonicalJson(previous.removeExecution) !== canonicalJson(record.removeExecution)) || + (previous.decision && previous.decision !== record.decision) || + (["committed", "restored"].includes(previous.phase) && + canonicalJson(previous) !== canonicalJson(record)) + ) + throw Error("Immutable ownership transaction changed"); + } + this.writeIntegrationRecord(name, record, previous === null); + }); + } standalonePublicationJournal(context: OpsExecutionContext): IntegrationPublicationJournal { this.standaloneInstallationId(context); const operation = this.integrationContext(context); diff --git a/apps/ops-agent/src/standalone-integration-records.ts b/apps/ops-agent/src/standalone-integration-records.ts index 0e6f806ff..f2e29175e 100644 --- a/apps/ops-agent/src/standalone-integration-records.ts +++ b/apps/ops-agent/src/standalone-integration-records.ts @@ -37,3 +37,33 @@ export const standaloneTransactionRecordSchema = z.strictObject({ export type StandaloneInspectionRecord = z.infer; export type StandaloneTransactionRecord = z.infer; + +export const standaloneOwnershipRecordSchema = z.strictObject({ + version: z.literal(1), + installationId: z.uuid(), + transactionId: z.uuid(), + integrationId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + ownerId: z.string().min(1).max(128), + inspectionExecution: executionRefSchema, + removeExecution: executionRefSchema.nullable(), + contentDigest: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + componentVersion: z.string().max(128).nullable(), + expiresAt: z.iso.datetime(), + targetIdentity: inode.nullable(), + lockIdentity: inode, + workspaceIdentity: inode, + phase: z.enum([ + "inspected", + "remove_requested", + "removed", + "commit_requested", + "committed", + "restore_requested", + "restored", + ]), + decision: z.enum(["commit", "restore"]).nullable(), +}); +export type StandaloneOwnershipRecord = z.infer; diff --git a/apps/ops-agent/src/standalone-integration-runtime.test.ts b/apps/ops-agent/src/standalone-integration-runtime.test.ts index 8f9c46511..7bf69712b 100644 --- a/apps/ops-agent/src/standalone-integration-runtime.test.ts +++ b/apps/ops-agent/src/standalone-integration-runtime.test.ts @@ -264,7 +264,7 @@ it.each(["normal", "upload-mutated", "forged-completion"] as const)( await expect( store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), ).rejects.toMatchObject({ statusCode: 409 }); - const replacement = createUnavailableRuntime(); + let replacement = createUnavailableRuntime(); installStandaloneIntegrationRuntime(replacement, { rootDir: root, store }); const settle: OpsOperation = { kind: "integration.settle", @@ -297,5 +297,113 @@ it.each(["normal", "upload-mutated", "forged-completion"] as const)( makeContext({ ...settle, decision: "restore" }), ), ).rejects.toThrow(/different decision/); + const inventoryOperation: OpsOperation = { kind: "integration.inventory" }; + const inventory = await dispatchOpsOperation( + replacement, + inventoryOperation, + makeContext(inventoryOperation), + ); + expect(inventory.entries).toContainEqual( + expect.objectContaining({ + integrationId: "upload-fixture", + valid: true, + contentDigest: inspection.contentDigest, + }), + ); + for (const decision of ["restore", "commit"] as const) { + const transactionId = randomUUID(), + own: OpsOperation = { + kind: "integration.inspectOwnership", + ownerId: "fixture-admin", + integrationId: "upload-fixture", + operationId: transactionId, + expectedContentDigest: inspection.contentDigest, + }; + const checked = await dispatchOpsOperation(replacement, own, makeContext(own)); + expect(checked).toMatchObject({ allowed: true, transactionId, version: "1.0.0" }); + const remove: OpsOperation = { + kind: "integration.remove", + transactionId, + expectedContentDigest: inspection.contentDigest, + }; + expect(await dispatchOpsOperation(replacement, remove, makeContext(remove))).toMatchObject({ + outcome: "removed", + }); + expect(existsSync(join(root, "custom_integrations/upload-fixture"))).toBe(false); + store.close(); + store = new ArtifactTransferStore({ + dataRoot: join(root, "stages"), + privateRoot: join(root, "stage-meta"), + token: "fixture-test-token", + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), + }); + replacement = createUnavailableRuntime(); + installStandaloneIntegrationRuntime(replacement, { rootDir: root, store }); + + const settling: OpsOperation = { + kind: "integration.settle", + transactionId, + decision, + expectedContentDigest: inspection.contentDigest, + }, + ctx = makeContext(settling); + expect(await dispatchOpsOperation(replacement, settling, ctx)).toMatchObject({ + outcome: decision === "commit" ? "committed" : "restored", + }); + expect(await dispatchOpsOperation(replacement, settling, ctx)).toMatchObject({ + outcome: decision === "commit" ? "committed" : "restored", + }); + expect(existsSync(join(root, "custom_integrations/upload-fixture"))).toBe( + decision === "restore", + ); + } + + { + const directory = join(root, "custom_integrations/upload-fixture"); + mkdirSync(directory); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id: "upload-fixture", + name: "Uploaded", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + const checkedInventory = await dispatchOpsOperation( + replacement, + inventoryOperation, + makeContext(inventoryOperation), + ), + entry = checkedInventory.entries.find((x) => x.integrationId === "upload-fixture"); + if (!entry?.contentDigest) throw Error("Missing real inventory digest"); + const transactionId = randomUUID(), + own: OpsOperation = { + kind: "integration.inspectOwnership", + integrationId: "upload-fixture", + operationId: transactionId, + ownerId: "fixture-admin", + expectedContentDigest: entry.contentDigest, + }; + expect(await dispatchOpsOperation(replacement, own, makeContext(own))).toMatchObject({ + allowed: true, + }); + renameSync(directory, join(root, "original-directory")); + mkdirSync(directory); + writeFileSync(join(directory, "foreign.txt"), "preserve this foreign replacement"); + const remove: OpsOperation = { + kind: "integration.remove", + transactionId, + expectedContentDigest: entry.contentDigest, + }; + await expect(dispatchOpsOperation(replacement, remove, makeContext(remove))).rejects.toThrow( + /manifest|changed/, + ); + expect(readFileSync(join(directory, "foreign.txt"), "utf8")).toBe( + "preserve this foreign replacement", + ); + } }, ); diff --git a/apps/ops-agent/src/standalone-integration-runtime.ts b/apps/ops-agent/src/standalone-integration-runtime.ts index 55240d430..031fdaaa6 100644 --- a/apps/ops-agent/src/standalone-integration-runtime.ts +++ b/apps/ops-agent/src/standalone-integration-runtime.ts @@ -21,11 +21,11 @@ import { import type z from "zod/v4"; import type { ArtifactTransferStore } from "./artifact-transfer/store"; import type { OpsRuntime } from "./runtime"; - import { standaloneInspectionRecordSchema as inspectionRecordSchema, standaloneTransactionRecordSchema as transactionRecordSchema, } from "./standalone-integration-records"; +import { installStandaloneOwnershipRuntime } from "./standalone-ownership-runtime"; function missing(error: unknown) { return (error as NodeJS.ErrnoException).code === "ENOENT"; @@ -217,4 +217,6 @@ export function installStandaloneIntegrationRuntime( store().retainStandaloneTransaction(context, record, publication); return { transactionId: record.transactionId, ...result }; }; + if (options.store) + installStandaloneOwnershipRuntime(runtime, { ...options, store: options.store }); } diff --git a/apps/ops-agent/src/standalone-ownership-runtime.ts b/apps/ops-agent/src/standalone-ownership-runtime.ts new file mode 100644 index 000000000..d519f00be --- /dev/null +++ b/apps/ops-agent/src/standalone-ownership-runtime.ts @@ -0,0 +1,396 @@ +import { createHash } from "node:crypto"; +import { fsyncSync, lstatSync, mkdirSync, renameSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; +import { canonicalJson } from "@openmapx/core/ops"; +import { + acquireIntegrationComponentLock, + describeIntegrationComponentLock, + finishIntegrationComponentLock, + inspectDeclarativeIntegrationDirectory, + inspectLockedIntegrationComponent, + inspectOptionalLockedIntegrationComponent, + releaseIntegrationComponentLock, + restoreIntegrationComponentLock, +} from "@openmapx/integration-framework/installer"; +import type { ArtifactTransferStore } from "./artifact-transfer/store"; +import type { OpsExecutionContext, OpsRuntime } from "./runtime"; +import type { StandaloneOwnershipRecord } from "./standalone-integration-records"; + +const absent = (error: unknown) => (error as NodeJS.ErrnoException).code === "ENOENT"; +const same = (a: { dev: string; ino: string }, b: { dev: string; ino: string }) => + a.dev === b.dev && a.ino === b.ino; +export function installStandaloneOwnershipRuntime( + runtime: OpsRuntime, + options: { rootDir: string; store: ArtifactTransferStore; now?: () => number }, +) { + const { store, rootDir } = options, + now = options.now ?? Date.now, + custom = join(rootDir, "custom_integrations"); + const lockState = (record: StandaloneOwnershipRecord) => ({ + rootDir, + id: record.integrationId, + transactionId: record.transactionId, + identity: record.lockIdentity, + }); + const workspace = (record: StandaloneOwnershipRecord) => { + const dir = new AnchoredDirectory( + join(custom, ".staging", `ownership-${record.transactionId}`), + undefined, + true, + ); + if (!same({ dev: String(dir.stat.dev), ino: String(dir.stat.ino) }, record.workspaceIdentity)) { + dir.close(); + throw Error("Ownership quarantine changed"); + } + return dir; + }; + const target = (record: StandaloneOwnershipRecord, signal: AbortSignal) => { + try { + lstatSync(join(custom, record.integrationId)); + } catch (error) { + if (absent(error)) return null; + throw error; + } + const value = inspectDeclarativeIntegrationDirectory( + join(custom, record.integrationId), + rootDir, + signal, + ); + if ( + value.id !== record.integrationId || + value.contentDigest !== record.contentDigest || + !record.targetIdentity || + !same(value.identity, record.targetIdentity) + ) + throw Error("Integration bytes or identity changed since inspection"); + return value; + }; + runtime["integration.inventory"] = async (_operation, context) => { + mkdirSync(custom, { recursive: true }); + const root = new AnchoredDirectory(custom); + try { + const entries = []; + for (const name of root.listBounded(1026).sort()) { + if (name === ".staging" || name === ".installation-locks") continue; + const safe = /^[a-z0-9][a-z0-9-]{0,63}$/.test(name), + integrationId = safe + ? name + : `invalid-${createHash("sha256").update(name).digest("hex").slice(0, 24)}`; + if (!safe) { + entries.push({ + integrationId, + version: null, + contentDigest: null, + valid: false, + denialReason: "Invalid legacy directory name; inspect it on the host.", + }); + continue; + } + let lock: ReturnType | undefined; + try { + lock = acquireIntegrationComponentLock({ rootDir, id: name }); + const value = inspectLockedIntegrationComponent(lock, context.signal); + entries.push({ + integrationId, + version: value.version, + contentDigest: value.contentDigest, + valid: true, + denialReason: null, + }); + } catch { + entries.push({ + integrationId, + version: null, + contentDigest: null, + valid: false, + denialReason: + "Unsupported, busy or inconsistent legacy content; inspect its original Activity job or the host.", + }); + } finally { + if (lock) releaseIntegrationComponentLock(lock); + } + } + for (const pending of store.standalonePendingInventory()) { + const index = entries.findIndex((entry) => entry.integrationId === pending.integrationId); + if (index === -1) entries.push(pending); + else entries[index] = pending; + } + return { entries }; + } finally { + root.close(); + } + }; + runtime["integration.inspectOwnership"] = async (operation, context) => { + const installationId = store.standaloneInstallationId(context); + try { + const old = store.readStandaloneOwnership(context, operation.operationId); + if (canonicalJson(old.inspectionExecution) !== canonicalJson(context.execution)) + throw Error("Ownership inspection replay mismatch"); + const lock = restoreIntegrationComponentLock(lockState(old)); + const current = inspectOptionalLockedIntegrationComponent(lock, context.signal); + if ( + (current?.contentDigest ?? null) !== old.contentDigest || + (current && (!old.targetIdentity || !same(current.identity, old.targetIdentity))) + ) + throw Error("Ownership changed"); + return { + allowed: true, + transactionId: old.transactionId, + integrationId: old.integrationId, + contentDigest: old.contentDigest, + version: old.componentVersion, + expiresAt: old.expiresAt, + }; + } catch (error) { + if (!absent(error)) throw error; + } + context.signal.throwIfAborted(); + let lock: ReturnType; + try { + lock = acquireIntegrationComponentLock({ + rootDir, + id: operation.integrationId, + transactionId: operation.operationId, + }); + } catch { + return { + allowed: false, + transactionId: operation.operationId, + integrationId: operation.integrationId, + reason: "Component is busy; reconcile its original Activity job.", + }; + } + let retained = false; + try { + const current = inspectOptionalLockedIntegrationComponent(lock, context.signal); + if ((current?.contentDigest ?? null) !== operation.expectedContentDigest) + throw Error("Integration changed since inventory; refresh before continuing"); + const root = new AnchoredDirectory(custom), + staging = root.directory(".staging", true, false); + let dir: AnchoredDirectory | undefined; + try { + mkdirSync(staging.name(`ownership-${operation.operationId}`), { mode: 0o700 }); + fsyncSync(staging.fd); + dir = staging.directory(`ownership-${operation.operationId}`, false, true); + const record: StandaloneOwnershipRecord = { + version: 1, + installationId, + transactionId: operation.operationId, + integrationId: operation.integrationId, + ownerId: operation.ownerId, + inspectionExecution: context.execution, + removeExecution: null, + contentDigest: current?.contentDigest ?? null, + componentVersion: current?.version ?? null, + expiresAt: new Date(Math.min(now() + 600000, context.execution.deadlineAt)).toISOString(), + targetIdentity: current?.identity ?? null, + lockIdentity: describeIntegrationComponentLock(lock).identity, + workspaceIdentity: { dev: String(dir.stat.dev), ino: String(dir.stat.ino) }, + phase: "inspected", + decision: null, + }; + // A publication error is ambiguous: keep the exact lock for reconciliation. + retained = true; + store.retainStandaloneOwnership(context, record); + return { + allowed: true, + transactionId: record.transactionId, + integrationId: record.integrationId, + contentDigest: record.contentDigest, + version: record.componentVersion, + expiresAt: record.expiresAt, + }; + } finally { + dir?.close(); + staging.close(); + root.close(); + } + } catch (error) { + if (retained) throw error; + return { + allowed: false, + transactionId: operation.operationId, + integrationId: operation.integrationId, + reason: "Integration content changed or cannot be safely adopted; refresh its inventory.", + }; + } finally { + if (!retained) releaseIntegrationComponentLock(lock); + } + }; + runtime["integration.remove"] = async (operation, context) => { + const record = store.readStandaloneOwnership(context, operation.transactionId); + if (record.contentDigest !== operation.expectedContentDigest || record.decision) + throw Error("Removal inspection changed or settled"); + const lock = restoreIntegrationComponentLock(lockState(record)); + describeIntegrationComponentLock(lock); + if ( + record.removeExecution && + canonicalJson(record.removeExecution) !== canonicalJson(context.execution) + ) + throw Error("Removal already belongs to another execution"); + if (!["inspected", "remove_requested", "removed"].includes(record.phase)) + throw Error("Removal transaction is unavailable"); + const dir = workspace(record); + try { + if (record.phase !== "removed") { + if (record.phase === "inspected" && Date.parse(record.expiresAt) <= now()) + throw Error("Ownership inspection expired"); + const current = target(record, context.signal); + if (current) { + context.signal.throwIfAborted(); + record.removeExecution = context.execution; + record.phase = "remove_requested"; + store.retainStandaloneOwnership(context, record); + renameSync(join(custom, record.integrationId), dir.name("component")); + fsyncSync(dir.fd); + const root = new AnchoredDirectory(custom); + try { + fsyncSync(root.fd); + } finally { + root.close(); + } + } + if (record.targetIdentity) { + const old = inspectDeclarativeIntegrationDirectory( + dir.name("component"), + rootDir, + context.signal, + ); + if ( + old.contentDigest !== record.contentDigest || + !same(old.identity, record.targetIdentity) || + target(record, context.signal) + ) + throw Error("Removal quarantine is not the original component"); + } else { + if (target(record, context.signal)) + throw Error("Missing integration appeared during removal"); + record.removeExecution = context.execution; + } + record.phase = "removed"; + store.retainStandaloneOwnership(context, record); + } + return { + transactionId: record.transactionId, + integrationId: record.integrationId, + contentDigest: record.contentDigest, + outcome: "removed", + }; + } finally { + dir.close(); + } + }; + const installSettle = runtime["integration.settle"]; + runtime["integration.settle"] = async (operation, context) => { + let record: StandaloneOwnershipRecord; + try { + record = store.readStandaloneOwnership(context, operation.transactionId); + } catch (error) { + if (absent(error)) return installSettle(operation, context); + throw error; + } + if ( + record.contentDigest !== operation.expectedContentDigest || + (record.decision && record.decision !== operation.decision) + ) + throw Error("Ownership settlement mismatch"); + if (record.phase === "committed" || record.phase === "restored") { + if ((record.phase === "committed") !== (operation.decision === "commit")) + throw Error("Ownership already settled differently"); + finishIntegrationComponentLock(lockState(record)); + return { + transactionId: record.transactionId, + outcome: record.phase, + contentDigest: record.contentDigest, + }; + } + const lock = restoreIntegrationComponentLock(lockState(record)); + describeIntegrationComponentLock(lock); + const dir = workspace(record); + try { + record.decision = operation.decision; + if (!record.removeExecution) { + if (record.targetIdentity && !target(record, context.signal)) + throw Error("Adoption target disappeared"); + if (!record.targetIdentity && target(record, context.signal)) + throw Error("Previously absent integration appeared"); + } else if (!record.targetIdentity) { + if (target(record, context.signal)) throw Error("Previously absent integration appeared"); + } else if (operation.decision === "restore") { + record.phase = "restore_requested"; + store.retainStandaloneOwnership(context, record); + const current = target(record, context.signal); + if (!current) { + const old = inspectDeclarativeIntegrationDirectory( + dir.name("component"), + rootDir, + context.signal, + ); + if ( + !same(old.identity, record.targetIdentity) || + old.contentDigest !== record.contentDigest + ) + throw Error("Original removal backup changed"); + renameSync(dir.name("component"), join(custom, record.integrationId)); + fsyncSync(dir.fd); + const root = new AnchoredDirectory(custom); + try { + fsyncSync(root.fd); + } finally { + root.close(); + } + } + } else { + if (target(record, context.signal)) + throw Error("Removal destination was replaced; preserve quarantine"); + if (record.phase !== "commit_requested") { + const original = inspectDeclarativeIntegrationDirectory( + dir.name("component"), + rootDir, + context.signal, + ); + if ( + original.contentDigest !== record.contentDigest || + !same(original.identity, record.targetIdentity) + ) + throw Error("Removal backup bytes changed"); + } + record.phase = "commit_requested"; + store.retainStandaloneOwnership(context, record); + let child: AnchoredDirectory | null; + try { + child = dir.directory("component", false, false); + } catch (error) { + if (!absent(error)) throw error; + child = null; + } + if (child) { + try { + if ( + !same( + { dev: String(child.stat.dev), ino: String(child.stat.ino) }, + record.targetIdentity, + ) + ) + throw Error("Removal quarantine changed"); + } finally { + child.close(); + } + rmSync(dir.name("component"), { recursive: true }); + fsyncSync(dir.fd); + } + } + record.phase = operation.decision === "commit" ? "committed" : "restored"; + store.retainStandaloneOwnership(context, record); + finishIntegrationComponentLock(lockState(record)); + return { + transactionId: record.transactionId, + outcome: record.phase, + contentDigest: record.contentDigest, + }; + } finally { + dir.close(); + } + }; +} diff --git a/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx b/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx index c16f5cf91..6910c42a6 100644 --- a/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx +++ b/apps/web/src/components/admin/extensions/ExtensionStorePage.tsx @@ -66,8 +66,8 @@ import { AdminTableSurface } from "../shared/AdminTableSurface"; import { useAdminToast } from "../shared/AdminToast"; import { TableSearchField, TableToolbar } from "../shared/TableToolbar"; import { useClientPagination } from "../shared/tableHooks"; - import { StandaloneInstallDialog } from "./StandaloneInstallDialog"; +import { StandaloneIntegrationsPanel } from "./StandaloneIntegrationsPanel"; type Trust = AdminExtensionInstallPreview["sourceTrust"]; @@ -932,10 +932,12 @@ function ExtensionStoreContent() { + {tab === 0 && } {tab === 1 && } {tab === 2 && } + {tab === 3 && } ); } diff --git a/apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.test.tsx b/apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.test.tsx new file mode 100644 index 000000000..825181771 --- /dev/null +++ b/apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.test.tsx @@ -0,0 +1,139 @@ +import { QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { AdminJobEffectsProvider } from "@/lib/admin/adminJobEffects"; +import { act, cleanup, createTestQueryClient, fireEvent, render, screen, waitFor } from "@/test"; +import { deferred } from "@/test/extensionAdmin"; +import { job } from "../activity/__tests__/fixtures"; +import { StandaloneIntegrationsPanel } from "./StandaloneIntegrationsPanel"; + +let apiUrl = "https://admin.example"; +vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useEnv: () => ({ apiUrl }), +})); +const requests: Array<{ url: URL; init: RequestInit }> = [], + queries: ReturnType[] = []; +const entry = { + integrationId: "legacy-map", + version: "1.0", + ownership: "unmanaged", + extensionId: null, + contentDigest: "a".repeat(64), + provenance: "legacy_unknown", + adoptable: true, + removable: true, + denialReason: null, +}; +const jobId = "a56bdfc1-f66e-4b9d-926a-5b442fa74834", + operationId = "07439c52-4ebd-4d77-a5c2-4a15a7eb2661"; +beforeEach(() => { + apiUrl = "https://admin.example"; + requests.length = 0; + vi.stubGlobal("fetch", async (raw: string, init: RequestInit) => { + const url = new URL(raw); + requests.push({ url, init }); + if (url.pathname.includes("/catalog")) throw Error("offline catalog"); + if (url.pathname.endsWith("/standalone-integrations")) + return Response.json({ + entries: [ + entry, + { + ...entry, + integrationId: "catalog-map", + ownership: "extension", + adoptable: false, + removable: false, + denialReason: "Managed by a catalog extension", + }, + ], + }); + if (url.pathname.endsWith("/adopt") || url.pathname.endsWith("/remove")) + return Response.json({ jobId, operationId }, { status: 202 }); + if (url.pathname.endsWith(jobId)) + return Response.json( + job({ + id: jobId, + status: "success", + cancelable: false, + result: { + operationId, + outcome: "committed", + action: "adopt", + integrationId: entry.integrationId, + }, + }), + ); + throw Error(`Unexpected ${url.pathname}`); + }); +}); +afterEach(() => { + cleanup(); + for (const query of queries.splice(0)) query.clear(); + vi.unstubAllGlobals(); +}); +function mount() { + const query = createTestQueryClient(); + queries.push(query); + render( + + + + + , + ); +} +it("shows local legacy inventory independently of the catalog and adopts only reviewed bytes", async () => { + mount(); + await screen.findByText("legacy-map"); + expect(screen.getAllByText(/Source unknown/)[0]).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Adopt legacy-map" })); + fireEvent.click(screen.getByRole("button", { name: "Confirm adoption" })); + await screen.findByText(/Ownership change completed/); + const mutation = requests.find((x) => x.url.pathname.endsWith("/adopt")); + expect(JSON.parse(String(mutation?.init.body))).toEqual({ + expectedContentDigest: entry.contentDigest, + }); + expect(new Headers(mutation?.init.headers).get("idempotency-key")).toMatch(/^[a-f0-9-]{36}$/); + expect(requests.some((x) => x.url.pathname.includes("/catalog"))).toBe(false); + expect(screen.queryByRole("button", { name: "Remove catalog-map" })).not.toBeInTheDocument(); +}); +it("requires an explicit removal confirmation and submits only the exact current digest", async () => { + mount(); + await screen.findByText("legacy-map"); + fireEvent.click(screen.getByRole("button", { name: "Remove legacy-map" })); + expect(requests.some((x) => x.init.method === "POST")).toBe(false); + fireEvent.click(screen.getByRole("button", { name: "Confirm removal" })); + await waitFor(() => expect(requests.some((x) => x.url.pathname.endsWith("/remove"))).toBe(true)); + expect( + JSON.parse(String(requests.find((x) => x.url.pathname.endsWith("/remove"))?.init.body)), + ).toEqual({ expectedContentDigest: entry.contentDigest }); +}); + +it("retires a late mutation acknowledgement across A to B to A navigation", async () => { + const query = createTestQueryClient(); + queries.push(query); + const ui = () => ( + + + + + + ); + const original = globalThis.fetch, + held = deferred(); + vi.stubGlobal("fetch", async (raw: string, init: RequestInit) => + new URL(raw).pathname.endsWith("/adopt") ? held.promise : original(raw, init), + ); + const view = render(ui()); + await screen.findByText("legacy-map"); + fireEvent.click(screen.getByRole("button", { name: "Adopt legacy-map" })); + fireEvent.click(screen.getByRole("button", { name: "Confirm adoption" })); + apiUrl = "https://other.example"; + view.rerender(ui()); + await screen.findByText("legacy-map"); + apiUrl = "https://admin.example"; + view.rerender(ui()); + await screen.findByText("legacy-map"); + await act(async () => held.resolve(Response.json({ jobId, operationId }, { status: 202 }))); + expect(screen.queryByText(/Ownership change completed/)).not.toBeInTheDocument(); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); +}); diff --git a/apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.tsx b/apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.tsx new file mode 100644 index 000000000..f9f8d0dca --- /dev/null +++ b/apps/web/src/components/admin/extensions/StandaloneIntegrationsPanel.tsx @@ -0,0 +1,283 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Button from "@mui/material/Button"; +import Card from "@mui/material/Card"; +import CardContent from "@mui/material/CardContent"; +import CircularProgress from "@mui/material/CircularProgress"; +import Dialog from "@mui/material/Dialog"; +import DialogActions from "@mui/material/DialogActions"; +import DialogContent from "@mui/material/DialogContent"; +import DialogTitle from "@mui/material/DialogTitle"; +import Stack from "@mui/material/Stack"; +import Typography from "@mui/material/Typography"; +import * as sdk from "@openmapx/core/api-admin-sdk"; +import { + type StandaloneInventoryEntry, + standaloneInventoryEntrySchema, +} from "@openmapx/core/api-contracts"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import Link from "next/link"; +import { useEffect, useMemo, useState } from "react"; +import { useAdminJobRegistration } from "@/lib/admin/adminJobEffects"; +import { invalidateExtensionAdministration } from "@/lib/admin/extensionAdmin"; +import { useAdminMutationOwner } from "@/lib/admin/useAdminMutationOwner"; +import { + activityJobQueryOptions, + hasUnresolvedJobObservation, + useActivityClient, +} from "../activity/activityQueries"; + +export function StandaloneIntegrationsPanel() { + const { apiUrl } = useActivityClient(); + return ; +} +function LocalInstallations() { + const { client, apiUrl } = useActivityClient(), + owner = useAdminMutationOwner(client, apiUrl), + register = useAdminJobRegistration(), + qc = useQueryClient(), + key = useMemo(() => ["admin", "standalone-integrations", apiUrl] as const, [apiUrl]); + const [selected, setSelected] = useState<{ + entry: StandaloneInventoryEntry; + action: "adopt" | "remove"; + key: string; + } | null>(null), + [busy, setBusy] = useState(false), + [error, setError] = useState(null), + [accepted, setAccepted] = useState<{ jobId: string; operationId: string } | null>(null), + [success, setSuccess] = useState(false); + const inventory = useQuery({ + queryKey: key, + queryFn: async ({ signal }) => { + const data = sdk.unwrapAdminResponse( + await sdk.listAdminStandaloneIntegrations({ client, signal, timeoutMs: 60000 }), + ); + return data.entries.map((entry) => standaloneInventoryEntrySchema.parse(entry)); + }, + }); + const operation = useQuery({ + ...activityJobQueryOptions(client, apiUrl, accepted?.jobId, "application"), + refetchInterval: (q) => + q.state.data && + ["success", "failed", "canceled"].includes(q.state.data.status) && + !hasUnresolvedJobObservation(q.state.data.result) + ? false + : 1000, + }); + const pending = + !!accepted && + (!operation.data || + !["success", "failed", "canceled"].includes(operation.data.status) || + hasUnresolvedJobObservation(operation.data.result)); + useEffect(() => { + const job = operation.data; + if ( + !owner.active || + !accepted || + job?.id !== accepted.jobId || + !["success", "failed", "canceled"].includes(job.status) || + hasUnresolvedJobObservation(job.result) + ) + return; + if (job.status === "success") { + const result = job.result as { + operationId?: unknown; + integrationId?: unknown; + action?: unknown; + outcome?: unknown; + } | null; + if ( + !selected || + result?.operationId !== accepted.operationId || + result.integrationId !== selected.entry.integrationId || + result.action !== selected.action || + result.outcome !== "committed" + ) { + setError("The ownership response could not be verified. Inspect Activity."); + return; + } + setSuccess(true); + void qc.invalidateQueries({ queryKey: key }); + void invalidateExtensionAdministration(qc, apiUrl, "components"); + } else setError(job.error ?? "Operation did not complete. Inspect Activity before retrying."); + }, [operation.data, accepted, owner, selected, qc, apiUrl, key]); + const open = (entry: StandaloneInventoryEntry, action: "adopt" | "remove") => { + setSelected({ entry, action, key: crypto.randomUUID() }); + setAccepted(null); + setError(null); + setSuccess(false); + }; + const submit = async () => { + if ( + !selected || + (selected.action === "adopt" && !selected.entry.contentDigest) || + busy || + accepted + ) + return; + setBusy(true); + setError(null); + try { + const result = sdk.unwrapAdminResponse( + await (selected.action === "adopt" + ? sdk.adoptAdminStandaloneIntegration( + selected.entry.integrationId, + { expectedContentDigest: selected.entry.contentDigest! }, + { "idempotency-key": selected.key }, + { client, timeoutMs: 30000 }, + ) + : sdk.removeAdminStandaloneIntegration( + selected.entry.integrationId, + { expectedContentDigest: selected.entry.contentDigest }, + { "idempotency-key": selected.key }, + { client, timeoutMs: 30000 }, + )), + ); + register({ + client, + jobId: result.jobId, + apiUrl, + onSettled: async () => { + await qc.invalidateQueries({ queryKey: key }); + await invalidateExtensionAdministration(qc, apiUrl, "components"); + }, + }); + if (owner.active) setAccepted(result); + } catch (error) { + if (owner.active) + setError( + sdk.adminErrorMessage( + error, + "Submission was not confirmed. Retry uses the same operation key; inspect Activity if uncertain.", + ), + ); + } finally { + if (owner.active) setBusy(false); + } + }; + return ( + + + Local integrations + + + + Manage standalone and CLI-installed integrations independently of catalog availability. + Adoption records community trust and does not download or execute code. + + {inventory.isPending && } + {inventory.error && ( + + Local inventory is unavailable. Existing work may still be running; inspect Activity + before retrying. + + )} + {inventory.data?.length === 0 && No local integrations found.} + {inventory.data?.map((entry) => ( + + + + + {entry.integrationId} + + + Version: {entry.version ?? "unknown"} · Ownership: {entry.ownership} + + + {entry.provenance === "legacy_unknown" + ? "Source unknown; review a new source to update." + : `Source: ${entry.provenance}`} + + {entry.contentDigest && ( + + Content SHA-256: {entry.contentDigest} + + )} + {entry.denialReason && {entry.denialReason}} + {entry.pendingTransactionId && ( + + )} + + {entry.adoptable && ( + + )} + {entry.removable && ( + + )} + + + + + ))} + { + if (!busy) setSelected(null); + }} + fullWidth + maxWidth="sm" + > + + {selected?.action === "adopt" ? "Adopt local integration" : "Remove local integration"} + + + + + {selected?.entry.integrationId} + + + {selected?.action === "adopt" + ? "Record the reviewed local bytes under a standalone community owner. Their original source remains unknown." + : "Quarantine the reviewed directory, reload the instance, then remove its ownership records. An uncertain result remains held for reconciliation."} + + + Content SHA-256: {selected?.entry.contentDigest} + + {(busy || pending) && ( + + {busy + ? "Submitting…" + : "Operation is running or awaiting reconciliation. Follow its original job in Activity."} + + )} + {operation.error && accepted && ( + + Unable to observe the original operation. Check Activity; this does not cancel it. + + )} + {error && {error}} + {success && Ownership change completed.} + + + + + {!accepted && ( + + )} + + + + ); +} diff --git a/docs/docs/administration/community-extensions.md b/docs/docs/administration/community-extensions.md index 843801d19..8773a9c5f 100644 --- a/docs/docs/administration/community-extensions.md +++ b/docs/docs/administration/community-extensions.md @@ -97,6 +97,22 @@ for reconciliation. Closing the dialog only stops observation. Cancellation can remain pending until restoration is confirmed. Retrying the same request observes its existing job instead of installing twice. +In **Local integrations**, the inventory includes CLI-installed directories, +standalone owners, invalid legacy content and missing local files, independently +of catalog availability. **Adopt** records the reviewed content under a community +owner. Without authenticated provenance, its source remains `legacy_unknown`; +adoption does not invent a repository URL or grant verified trust. + +**Remove** requires confirmation of the reviewed content digest. The agent keeps +the original directory in quarantine until reload and ownership bookkeeping +complete. If CLI already removed the files, removal verifies their absence under +the same component lock before cleaning up the remaining standalone records. +A changed directory or foreign/catalog owner blocks the action. Invalid executable +legacy entries stay inert and require host inspection rather than privileged +execution or automatic deletion. Pending transactions show their original +Activity reference; an unknown result keeps the component held for reconciliation. +Standalone parents use this view rather than bundle update/uninstall controls. + The same operations are available on the CLI: ```bash diff --git a/packages/core/src/api/contracts/integration-admin.test.ts b/packages/core/src/api/contracts/integration-admin.test.ts index 495aaf822..c87ac6294 100644 --- a/packages/core/src/api/contracts/integration-admin.test.ts +++ b/packages/core/src/api/contracts/integration-admin.test.ts @@ -11,6 +11,9 @@ beforeEach(() => afterEach(() => vi.unstubAllGlobals()); const at = "2026-10-20T12:00:00.000Z"; const inventory = [ + ["listAdminStandaloneIntegrations", "GET", "/api/admin/standalone-integrations"], + ["adoptAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/:id/adopt"], + ["removeAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/:id/remove"], ["previewAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/preview"], ["installAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/install"], ["listAdminIntegrations", "GET", "/api/admin/integrations"], @@ -176,7 +179,11 @@ const detail = { function success( id: Exclude< keyof typeof contracts.integrationAdminOperations, - "previewAdminStandaloneIntegration" | "installAdminStandaloneIntegration" + | "previewAdminStandaloneIntegration" + | "installAdminStandaloneIntegration" + | "listAdminStandaloneIntegrations" + | "adoptAdminStandaloneIntegration" + | "removeAdminStandaloneIntegration" >, value: unknown, ) { @@ -186,7 +193,7 @@ function success( return contract.schema.parse(value); } describe("integration administration canonical producer contracts", () => { - it("binds exactly the 24 approved operation IDs, methods, paths and admin auth", () => { + it("binds exactly the 27 approved operation IDs, methods, paths and admin auth", () => { expect(contracts).toHaveProperty("integrationAdminOperations"); expect( Object.entries(contracts.integrationAdminOperations).map(([id, op]) => [ diff --git a/packages/core/src/api/contracts/integration-admin/standalone-integrations.ts b/packages/core/src/api/contracts/integration-admin/standalone-integrations.ts index 35845c036..686dec42f 100644 --- a/packages/core/src/api/contracts/integration-admin/standalone-integrations.ts +++ b/packages/core/src/api/contracts/integration-admin/standalone-integrations.ts @@ -1,5 +1,8 @@ import z from "zod/v4"; -import { standaloneInstallSourceSchema } from "../../../ops/standalone-integrations"; +import { + standaloneInstallSourceSchema, + standaloneInventoryEntrySchema, +} from "../../../ops/standalone-integrations"; import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; import { adminIntegrationIdempotencyHeadersSchema } from "./common"; export const standaloneInstallRequestSchema = z.strictObject({ @@ -19,7 +22,49 @@ const responses = { 409: adminJson(httpErrorSchema), 503: adminJson(httpErrorSchema), }; +export const standaloneOwnershipParamsSchema = z.strictObject({ + id: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), +}); +export const standaloneOwnershipRequestSchema = z.strictObject({ + expectedContentDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +const standaloneRemovalRequestSchema = standaloneOwnershipRequestSchema.extend({ + expectedContentDigest: standaloneOwnershipRequestSchema.shape.expectedContentDigest.nullable(), +}); export const standaloneIntegrationAdminOperations = { + listAdminStandaloneIntegrations: { + method: "GET", + path: "/api/admin/standalone-integrations", + auth: "admin", + responses: { + ...adminCommonResponses, + 200: adminJson( + z.strictObject({ entries: z.array(standaloneInventoryEntrySchema).max(2048) }), + ), + }, + }, + adoptAdminStandaloneIntegration: { + method: "POST", + path: "/api/admin/standalone-integrations/:id/adopt", + auth: "admin", + request: { + params: standaloneOwnershipParamsSchema, + body: standaloneOwnershipRequestSchema, + headers: adminIntegrationIdempotencyHeadersSchema, + }, + responses, + }, + removeAdminStandaloneIntegration: { + method: "POST", + path: "/api/admin/standalone-integrations/:id/remove", + auth: "admin", + request: { + params: standaloneOwnershipParamsSchema, + body: standaloneRemovalRequestSchema, + headers: adminIntegrationIdempotencyHeadersSchema, + }, + responses, + }, previewAdminStandaloneIntegration: { method: "POST", path: "/api/admin/standalone-integrations/preview", @@ -46,6 +91,8 @@ export const standaloneIntegrationAdminOperations = { export { type StandaloneInstallInspection, type StandaloneInstallSource, + type StandaloneInventoryEntry, standaloneInstallInspectionSchema, standaloneInstallSourceSchema, + standaloneInventoryEntrySchema, } from "../../../ops/standalone-integrations"; diff --git a/packages/core/src/api/generated/admin/integration-admin.generated.ts b/packages/core/src/api/generated/admin/integration-admin.generated.ts index 004825ce5..7de525e22 100644 --- a/packages/core/src/api/generated/admin/integration-admin.generated.ts +++ b/packages/core/src/api/generated/admin/integration-admin.generated.ts @@ -1,6 +1,18 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. import type { + AdoptAdminStandaloneIntegration202, + AdoptAdminStandaloneIntegration400, + AdoptAdminStandaloneIntegration401, + AdoptAdminStandaloneIntegration403, + AdoptAdminStandaloneIntegration409, + AdoptAdminStandaloneIntegration413, + AdoptAdminStandaloneIntegration415, + AdoptAdminStandaloneIntegration429, + AdoptAdminStandaloneIntegration500, + AdoptAdminStandaloneIntegration503, + AdoptAdminStandaloneIntegrationBody, + AdoptAdminStandaloneIntegrationHeaders, DeleteAdminIntegrationBinding200, DeleteAdminIntegrationBinding400, DeleteAdminIntegrationBinding401, @@ -158,6 +170,14 @@ import type { ListAdminIntegrations415, ListAdminIntegrations429, ListAdminIntegrations500, + ListAdminStandaloneIntegrations200, + ListAdminStandaloneIntegrations400, + ListAdminStandaloneIntegrations401, + ListAdminStandaloneIntegrations403, + ListAdminStandaloneIntegrations413, + ListAdminStandaloneIntegrations415, + ListAdminStandaloneIntegrations429, + ListAdminStandaloneIntegrations500, PatchAdminIntegrationConfig200, PatchAdminIntegrationConfig400, PatchAdminIntegrationConfig401, @@ -201,6 +221,18 @@ import type { ReloadAdminIntegrations429, ReloadAdminIntegrations500, ReloadAdminIntegrationsBody, + RemoveAdminStandaloneIntegration202, + RemoveAdminStandaloneIntegration400, + RemoveAdminStandaloneIntegration401, + RemoveAdminStandaloneIntegration403, + RemoveAdminStandaloneIntegration409, + RemoveAdminStandaloneIntegration413, + RemoveAdminStandaloneIntegration415, + RemoveAdminStandaloneIntegration429, + RemoveAdminStandaloneIntegration500, + RemoveAdminStandaloneIntegration503, + RemoveAdminStandaloneIntegrationBody, + RemoveAdminStandaloneIntegrationHeaders, RunAdminIntegrationHealthChecks200, RunAdminIntegrationHealthChecks400, RunAdminIntegrationHealthChecks401, @@ -247,6 +279,7 @@ import type { TestAdminIntegrationCredentialsBody, } from "./integration-models"; import { + adoptAdminStandaloneIntegrationRequest, deleteAdminIntegrationBindingRequest, deleteAdminIntegrationCredentialRequest, disableAdminIntegrationRequest, @@ -263,10 +296,12 @@ import { listAdminIntegrationBindingsRequest, listAdminIntegrationEnvVarsRequest, listAdminIntegrationsRequest, + listAdminStandaloneIntegrationsRequest, patchAdminIntegrationConfigRequest, previewAdminStandaloneIntegrationRequest, reloadAdminIntegrationRequest, reloadAdminIntegrationsRequest, + removeAdminStandaloneIntegrationRequest, runAdminIntegrationHealthChecksRequest, setAdminIntegrationBindingRequest, setAdminIntegrationCredentialRequest, @@ -2367,6 +2402,82 @@ export const getAdminOverview = async ( }); }; +export type listAdminStandaloneIntegrationsResponse200 = { + data: ListAdminStandaloneIntegrations200; + status: 200; +}; + +export type listAdminStandaloneIntegrationsResponse400 = { + data: ListAdminStandaloneIntegrations400; + status: 400; +}; + +export type listAdminStandaloneIntegrationsResponse401 = { + data: ListAdminStandaloneIntegrations401; + status: 401; +}; + +export type listAdminStandaloneIntegrationsResponse403 = { + data: ListAdminStandaloneIntegrations403; + status: 403; +}; + +export type listAdminStandaloneIntegrationsResponse413 = { + data: ListAdminStandaloneIntegrations413; + status: 413; +}; + +export type listAdminStandaloneIntegrationsResponse415 = { + data: ListAdminStandaloneIntegrations415; + status: 415; +}; + +export type listAdminStandaloneIntegrationsResponse429 = { + data: ListAdminStandaloneIntegrations429; + status: 429; +}; + +export type listAdminStandaloneIntegrationsResponse500 = { + data: ListAdminStandaloneIntegrations500; + status: 500; +}; + +export type listAdminStandaloneIntegrationsResponseSuccess = + listAdminStandaloneIntegrationsResponse200 & { + headers: Headers; + }; +export type listAdminStandaloneIntegrationsResponseError = ( + | listAdminStandaloneIntegrationsResponse400 + | listAdminStandaloneIntegrationsResponse401 + | listAdminStandaloneIntegrationsResponse403 + | listAdminStandaloneIntegrationsResponse413 + | listAdminStandaloneIntegrationsResponse415 + | listAdminStandaloneIntegrationsResponse429 + | listAdminStandaloneIntegrationsResponse500 +) & { + headers: Headers; +}; + +export type listAdminStandaloneIntegrationsResponse = + | listAdminStandaloneIntegrationsResponseSuccess + | listAdminStandaloneIntegrationsResponseError; + +export const getListAdminStandaloneIntegrationsUrl = () => { + return `/api/admin/standalone-integrations`; +}; + +export const listAdminStandaloneIntegrations = async ( + options?: Parameters[1], +): Promise => { + return listAdminStandaloneIntegrationsRequest( + getListAdminStandaloneIntegrationsUrl(), + { + ...options, + method: "GET", + }, + ); +}; + export type installAdminStandaloneIntegrationResponse202 = { data: InstallAdminStandaloneIntegration202; status: 202; @@ -2588,3 +2699,227 @@ export const previewAdminStandaloneIntegration = async ( }, ); }; + +export type adoptAdminStandaloneIntegrationResponse202 = { + data: AdoptAdminStandaloneIntegration202; + status: 202; +}; + +export type adoptAdminStandaloneIntegrationResponse400 = { + data: AdoptAdminStandaloneIntegration400; + status: 400; +}; + +export type adoptAdminStandaloneIntegrationResponse401 = { + data: AdoptAdminStandaloneIntegration401; + status: 401; +}; + +export type adoptAdminStandaloneIntegrationResponse403 = { + data: AdoptAdminStandaloneIntegration403; + status: 403; +}; + +export type adoptAdminStandaloneIntegrationResponse409 = { + data: AdoptAdminStandaloneIntegration409; + status: 409; +}; + +export type adoptAdminStandaloneIntegrationResponse413 = { + data: AdoptAdminStandaloneIntegration413; + status: 413; +}; + +export type adoptAdminStandaloneIntegrationResponse415 = { + data: AdoptAdminStandaloneIntegration415; + status: 415; +}; + +export type adoptAdminStandaloneIntegrationResponse429 = { + data: AdoptAdminStandaloneIntegration429; + status: 429; +}; + +export type adoptAdminStandaloneIntegrationResponse500 = { + data: AdoptAdminStandaloneIntegration500; + status: 500; +}; + +export type adoptAdminStandaloneIntegrationResponse503 = { + data: AdoptAdminStandaloneIntegration503; + status: 503; +}; + +export type adoptAdminStandaloneIntegrationResponseSuccess = + adoptAdminStandaloneIntegrationResponse202 & { + headers: Headers; + }; +export type adoptAdminStandaloneIntegrationResponseError = ( + | adoptAdminStandaloneIntegrationResponse400 + | adoptAdminStandaloneIntegrationResponse401 + | adoptAdminStandaloneIntegrationResponse403 + | adoptAdminStandaloneIntegrationResponse409 + | adoptAdminStandaloneIntegrationResponse413 + | adoptAdminStandaloneIntegrationResponse415 + | adoptAdminStandaloneIntegrationResponse429 + | adoptAdminStandaloneIntegrationResponse500 + | adoptAdminStandaloneIntegrationResponse503 +) & { + headers: Headers; +}; + +export type adoptAdminStandaloneIntegrationResponse = + | adoptAdminStandaloneIntegrationResponseSuccess + | adoptAdminStandaloneIntegrationResponseError; + +export const getAdoptAdminStandaloneIntegrationUrl = (id: string) => { + return `/api/admin/standalone-integrations/${encodeURIComponent(String(id))}/adopt`; +}; + +export const adoptAdminStandaloneIntegration = async ( + id: string, + adoptAdminStandaloneIntegrationBody: AdoptAdminStandaloneIntegrationBody, + headers: AdoptAdminStandaloneIntegrationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return adoptAdminStandaloneIntegrationRequest( + getAdoptAdminStandaloneIntegrationUrl(id), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(adoptAdminStandaloneIntegrationBody), + }, + ); +}; + +export type removeAdminStandaloneIntegrationResponse202 = { + data: RemoveAdminStandaloneIntegration202; + status: 202; +}; + +export type removeAdminStandaloneIntegrationResponse400 = { + data: RemoveAdminStandaloneIntegration400; + status: 400; +}; + +export type removeAdminStandaloneIntegrationResponse401 = { + data: RemoveAdminStandaloneIntegration401; + status: 401; +}; + +export type removeAdminStandaloneIntegrationResponse403 = { + data: RemoveAdminStandaloneIntegration403; + status: 403; +}; + +export type removeAdminStandaloneIntegrationResponse409 = { + data: RemoveAdminStandaloneIntegration409; + status: 409; +}; + +export type removeAdminStandaloneIntegrationResponse413 = { + data: RemoveAdminStandaloneIntegration413; + status: 413; +}; + +export type removeAdminStandaloneIntegrationResponse415 = { + data: RemoveAdminStandaloneIntegration415; + status: 415; +}; + +export type removeAdminStandaloneIntegrationResponse429 = { + data: RemoveAdminStandaloneIntegration429; + status: 429; +}; + +export type removeAdminStandaloneIntegrationResponse500 = { + data: RemoveAdminStandaloneIntegration500; + status: 500; +}; + +export type removeAdminStandaloneIntegrationResponse503 = { + data: RemoveAdminStandaloneIntegration503; + status: 503; +}; + +export type removeAdminStandaloneIntegrationResponseSuccess = + removeAdminStandaloneIntegrationResponse202 & { + headers: Headers; + }; +export type removeAdminStandaloneIntegrationResponseError = ( + | removeAdminStandaloneIntegrationResponse400 + | removeAdminStandaloneIntegrationResponse401 + | removeAdminStandaloneIntegrationResponse403 + | removeAdminStandaloneIntegrationResponse409 + | removeAdminStandaloneIntegrationResponse413 + | removeAdminStandaloneIntegrationResponse415 + | removeAdminStandaloneIntegrationResponse429 + | removeAdminStandaloneIntegrationResponse500 + | removeAdminStandaloneIntegrationResponse503 +) & { + headers: Headers; +}; + +export type removeAdminStandaloneIntegrationResponse = + | removeAdminStandaloneIntegrationResponseSuccess + | removeAdminStandaloneIntegrationResponseError; + +export const getRemoveAdminStandaloneIntegrationUrl = (id: string) => { + return `/api/admin/standalone-integrations/${encodeURIComponent(String(id))}/remove`; +}; + +export const removeAdminStandaloneIntegration = async ( + id: string, + removeAdminStandaloneIntegrationBody: RemoveAdminStandaloneIntegrationBody, + headers: RemoveAdminStandaloneIntegrationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return removeAdminStandaloneIntegrationRequest( + getRemoveAdminStandaloneIntegrationUrl(id), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(removeAdminStandaloneIntegrationBody), + }, + ); +}; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration202.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration202.ts new file mode 100644 index 000000000..7fcfb2343 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration202.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + operationId: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration400.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration400.ts new file mode 100644 index 000000000..1ce9737e8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration401.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration401.ts new file mode 100644 index 000000000..7555b5b54 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration403.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration403.ts new file mode 100644 index 000000000..f18b2bb61 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration409.ts new file mode 100644 index 000000000..e0313d0f1 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration413.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration413.ts new file mode 100644 index 000000000..f95abfd41 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration415.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration415.ts new file mode 100644 index 000000000..7117cbfab --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration429.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration429.ts new file mode 100644 index 000000000..5cf844a12 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration500.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration500.ts new file mode 100644 index 000000000..67591285a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration503.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration503.ts new file mode 100644 index 000000000..04482c64c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationBody.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationBody.ts new file mode 100644 index 000000000..eba5c229e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationBody.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegrationBody = { + /** @pattern ^[a-f0-9]{64}$ */ + expectedContentDigest: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationHeaders.ts new file mode 100644 index 000000000..2afbcb4f6 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adoptAdminStandaloneIntegrationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdoptAdminStandaloneIntegrationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/index.ts b/packages/core/src/api/generated/admin/integration-models/index.ts index 1566cb4d6..41bc19bc7 100644 --- a/packages/core/src/api/generated/admin/integration-models/index.ts +++ b/packages/core/src/api/generated/admin/integration-models/index.ts @@ -7,6 +7,18 @@ export * from "./adminGetAdminOverview200Schema0"; export * from "./adminImportAdminIntegrationConfigsBodySchema0"; export * from "./adminListAdminIntegrationEnvVars200Schema0"; export * from "./adminPatchAdminIntegrationConfigBodySchema0"; +export * from "./adoptAdminStandaloneIntegration202"; +export * from "./adoptAdminStandaloneIntegration400"; +export * from "./adoptAdminStandaloneIntegration401"; +export * from "./adoptAdminStandaloneIntegration403"; +export * from "./adoptAdminStandaloneIntegration409"; +export * from "./adoptAdminStandaloneIntegration413"; +export * from "./adoptAdminStandaloneIntegration415"; +export * from "./adoptAdminStandaloneIntegration429"; +export * from "./adoptAdminStandaloneIntegration500"; +export * from "./adoptAdminStandaloneIntegration503"; +export * from "./adoptAdminStandaloneIntegrationBody"; +export * from "./adoptAdminStandaloneIntegrationHeaders"; export * from "./deleteAdminIntegrationBinding200"; export * from "./deleteAdminIntegrationBinding200Database"; export * from "./deleteAdminIntegrationBinding200HostApply"; @@ -285,6 +297,17 @@ export * from "./listAdminIntegrations413"; export * from "./listAdminIntegrations415"; export * from "./listAdminIntegrations429"; export * from "./listAdminIntegrations500"; +export * from "./listAdminStandaloneIntegrations200"; +export * from "./listAdminStandaloneIntegrations200EntriesItem"; +export * from "./listAdminStandaloneIntegrations200EntriesItemOwnership"; +export * from "./listAdminStandaloneIntegrations200EntriesItemProvenance"; +export * from "./listAdminStandaloneIntegrations400"; +export * from "./listAdminStandaloneIntegrations401"; +export * from "./listAdminStandaloneIntegrations403"; +export * from "./listAdminStandaloneIntegrations413"; +export * from "./listAdminStandaloneIntegrations415"; +export * from "./listAdminStandaloneIntegrations429"; +export * from "./listAdminStandaloneIntegrations500"; export * from "./patchAdminIntegrationConfig200"; export * from "./patchAdminIntegrationConfig400"; export * from "./patchAdminIntegrationConfig401"; @@ -329,6 +352,18 @@ export * from "./reloadAdminIntegrations415"; export * from "./reloadAdminIntegrations429"; export * from "./reloadAdminIntegrations500"; export * from "./reloadAdminIntegrationsBody"; +export * from "./removeAdminStandaloneIntegration202"; +export * from "./removeAdminStandaloneIntegration400"; +export * from "./removeAdminStandaloneIntegration401"; +export * from "./removeAdminStandaloneIntegration403"; +export * from "./removeAdminStandaloneIntegration409"; +export * from "./removeAdminStandaloneIntegration413"; +export * from "./removeAdminStandaloneIntegration415"; +export * from "./removeAdminStandaloneIntegration429"; +export * from "./removeAdminStandaloneIntegration500"; +export * from "./removeAdminStandaloneIntegration503"; +export * from "./removeAdminStandaloneIntegrationBody"; +export * from "./removeAdminStandaloneIntegrationHeaders"; export * from "./runAdminIntegrationHealthChecks200"; export * from "./runAdminIntegrationHealthChecks200ResultsItem"; export * from "./runAdminIntegrationHealthChecks200ResultsItemStatus"; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200.ts new file mode 100644 index 000000000..d473b1392 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminStandaloneIntegrations200EntriesItem } from "./listAdminStandaloneIntegrations200EntriesItem"; + +export type ListAdminStandaloneIntegrations200 = { + /** @maxItems 2048 */ + entries: ListAdminStandaloneIntegrations200EntriesItem[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItem.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItem.ts new file mode 100644 index 000000000..f8e658c93 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItem.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ListAdminStandaloneIntegrations200EntriesItemOwnership } from "./listAdminStandaloneIntegrations200EntriesItemOwnership"; +import type { ListAdminStandaloneIntegrations200EntriesItemProvenance } from "./listAdminStandaloneIntegrations200EntriesItemProvenance"; + +export type ListAdminStandaloneIntegrations200EntriesItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + version: string | null; + contentDigest: string | null; + denialReason: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + pendingTransactionId?: string; + ownership: ListAdminStandaloneIntegrations200EntriesItemOwnership; + extensionId: string | null; + provenance: ListAdminStandaloneIntegrations200EntriesItemProvenance; + adoptable: boolean; + removable: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemOwnership.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemOwnership.ts new file mode 100644 index 000000000..c321e75ee --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemOwnership.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations200EntriesItemOwnership = + (typeof ListAdminStandaloneIntegrations200EntriesItemOwnership)[keyof typeof ListAdminStandaloneIntegrations200EntriesItemOwnership]; + +export const ListAdminStandaloneIntegrations200EntriesItemOwnership = { + extension: "extension", + standalone: "standalone", + unmanaged: "unmanaged", + conflict: "conflict", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemProvenance.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemProvenance.ts new file mode 100644 index 000000000..29ce17ccf --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations200EntriesItemProvenance.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations200EntriesItemProvenance = + (typeof ListAdminStandaloneIntegrations200EntriesItemProvenance)[keyof typeof ListAdminStandaloneIntegrations200EntriesItemProvenance]; + +export const ListAdminStandaloneIntegrations200EntriesItemProvenance = { + artifact: "artifact", + source: "source", + "uploaded-artifact": "uploaded-artifact", + legacy_unknown: "legacy_unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations400.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations400.ts new file mode 100644 index 000000000..47dd72b39 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations401.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations401.ts new file mode 100644 index 000000000..031dbf1cc --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations403.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations403.ts new file mode 100644 index 000000000..bf3b1688e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations413.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations413.ts new file mode 100644 index 000000000..62776a2ba --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations415.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations415.ts new file mode 100644 index 000000000..23aa7494e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations429.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations429.ts new file mode 100644 index 000000000..ff5736973 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations500.ts b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations500.ts new file mode 100644 index 000000000..022dddee8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/listAdminStandaloneIntegrations500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ListAdminStandaloneIntegrations500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration202.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration202.ts new file mode 100644 index 000000000..8096e5311 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration202.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration202 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + operationId: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration400.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration400.ts new file mode 100644 index 000000000..13b9cebf4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration401.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration401.ts new file mode 100644 index 000000000..21af2734a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration403.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration403.ts new file mode 100644 index 000000000..836251678 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration409.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration409.ts new file mode 100644 index 000000000..593a77f60 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration413.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration413.ts new file mode 100644 index 000000000..bf2fd11eb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration415.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration415.ts new file mode 100644 index 000000000..0ffbf959c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration429.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration429.ts new file mode 100644 index 000000000..c8a591025 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration500.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration500.ts new file mode 100644 index 000000000..9bcd6ca6e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration503.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration503.ts new file mode 100644 index 000000000..3a5ab6614 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationBody.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationBody.ts new file mode 100644 index 000000000..a0db47171 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationBody.ts @@ -0,0 +1,6 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegrationBody = { + expectedContentDigest: string | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationHeaders.ts new file mode 100644 index 000000000..84f26e585 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/removeAdminStandaloneIntegrationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type RemoveAdminStandaloneIntegrationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index 465719a82..a7388fc80 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -23,6 +23,13 @@ export async function addAdminExtensionSourceRequest( ): Promise { return addAdminExtensionSourceBound(url, options); } +const adoptAdminStandaloneIntegrationBound = bindAdminOperation("adoptAdminStandaloneIntegration"); +export async function adoptAdminStandaloneIntegrationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return adoptAdminStandaloneIntegrationBound(url, options); +} const applyAdminAuthPolicyBound = bindAdminOperation("applyAdminAuthPolicy"); export async function applyAdminAuthPolicyRequest( url: string, @@ -561,6 +568,13 @@ export async function listAdminServicesRequest( ): Promise { return listAdminServicesBound(url, options); } +const listAdminStandaloneIntegrationsBound = bindAdminOperation("listAdminStandaloneIntegrations"); +export async function listAdminStandaloneIntegrationsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return listAdminStandaloneIntegrationsBound(url, options); +} const patchAdminIntegrationConfigBound = bindAdminOperation("patchAdminIntegrationConfig"); export async function patchAdminIntegrationConfigRequest( url: string, @@ -758,6 +772,15 @@ export async function removeAdminExtensionSourceRequest( ): Promise { return removeAdminExtensionSourceBound(url, options); } +const removeAdminStandaloneIntegrationBound = bindAdminOperation( + "removeAdminStandaloneIntegration", +); +export async function removeAdminStandaloneIntegrationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return removeAdminStandaloneIntegrationBound(url, options); +} const restoreAdminServiceBackupBound = bindAdminOperation("restoreAdminServiceBackup"); export async function restoreAdminServiceBackupRequest( url: string, diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index a7ce14ed5..fab32911b 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -2064,6 +2064,7 @@ export function retainControlEffectReceipt( // published install still holds its component lock until API bookkeeping // selects the separate commit/restore operation. (s.kind === "integration.install" && result.outcome === "published") || + (s.kind === "integration.remove" && result.outcome === "removed") || (s.kind === "integration.settle" && result.outcome === "restored") ) { outcome = "committed"; diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index d08e75c7e..fd3f4e08f 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -66,6 +66,19 @@ const transferClaim = { bytes: 4, }; const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { + "integration.inventory": { kind: "integration.inventory" }, + "integration.inspectOwnership": { + kind: "integration.inspectOwnership", + integrationId: "fixture", + expectedContentDigest: "a".repeat(64), + ownerId: "admin", + operationId: "81a78b3e-208b-4ba4-9ddf-870fbdf1da22", + }, + "integration.remove": { + kind: "integration.remove", + transactionId: "81a78b3e-208b-4ba4-9ddf-870fbdf1da22", + expectedContentDigest: "a".repeat(64), + }, "integration.inspectInstall": { kind: "integration.inspectInstall", ownerId: "admin", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 4089248fd..f18fde48d 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -69,6 +69,9 @@ export const OPS_MAX_BACKUP_ID_LENGTH = 128; export const OPS_MAX_BACKUP_INVENTORY_ENTRIES = 128; export const OPS_OPERATION_KINDS = [ + "integration.inventory", + "integration.inspectOwnership", + "integration.remove", "integration.inspectInstall", "integration.install", "integration.settle", @@ -210,6 +213,9 @@ const asyncEffect = ( const LIFECYCLE_TIMEOUT_MS = 10 * 60_000 + 30_000; export const OPS_KIND_POLICIES = { + "integration.inventory": sync("api", 60_000, 1024 * 1024), + "integration.inspectOwnership": asyncEffect("api", 10 * 60_000), + "integration.remove": asyncEffect("api", 10 * 60_000), "integration.inspectInstall": asyncEffect("api", 10 * 60_000), "integration.install": asyncEffect("api", 10 * 60_000), "integration.settle": asyncEffect("api", 60_000), diff --git a/packages/core/src/ops/standalone-integrations.test.ts b/packages/core/src/ops/standalone-integrations.test.ts index 07374de3f..9d73fa754 100644 --- a/packages/core/src/ops/standalone-integrations.test.ts +++ b/packages/core/src/ops/standalone-integrations.test.ts @@ -24,3 +24,11 @@ it("admits only the finite API-owned standalone inspection/install/settlement op expect(() => parseOpsRequest(request, { role: "api" })).not.toThrow(); expect(() => parseOpsRequest(request, { role: "data-manager" })).toThrow(); }); +it("exposes finite legacy inventory/ownership/removal operations", () => { + for (const kind of [ + "integration.inventory", + "integration.inspectOwnership", + "integration.remove", + ]) + expect(OPS_OPERATION_KINDS).toContain(kind); +}); diff --git a/packages/core/src/ops/standalone-integrations.ts b/packages/core/src/ops/standalone-integrations.ts index 93b88f42f..63a104070 100644 --- a/packages/core/src/ops/standalone-integrations.ts +++ b/packages/core/src/ops/standalone-integrations.ts @@ -57,6 +57,7 @@ export const standaloneInstallInspectionSchema = z.strictObject({ denials: z.array(z.string().max(512)).max(64), }); export const standaloneIntegrationProvenanceSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("legacy_unknown"), contentDigest: digest }), z.strictObject({ kind: z.literal("artifact"), url: remote, @@ -84,7 +85,55 @@ export const standaloneInstallReceiptSchema = z.strictObject({ contentDigest: digest, outcome: z.literal("published"), }); +export const standaloneFilesystemEntrySchema = z.strictObject({ + integrationId: component, + version: z.string().max(128).nullable(), + contentDigest: digest.nullable(), + valid: z.boolean(), + denialReason: z.string().max(256).nullable(), + pendingTransactionId: z.uuid().optional(), +}); +export type StandaloneFilesystemEntry = z.infer; +export const standaloneInventoryEntrySchema = standaloneFilesystemEntrySchema + .omit({ valid: true }) + .extend({ + ownership: z.enum(["extension", "standalone", "unmanaged", "conflict"]), + extensionId: z.string().max(128).nullable(), + provenance: z.enum(["artifact", "source", "uploaded-artifact", "legacy_unknown"]), + adoptable: z.boolean(), + removable: z.boolean(), + }); +export type StandaloneInventoryEntry = z.infer; +export const standaloneOwnershipInspectionSchema = z.discriminatedUnion("allowed", [ + z.strictObject({ + allowed: z.literal(true), + transactionId: z.uuid(), + integrationId: component, + contentDigest: digest.nullable(), + version: z.string().max(128).nullable(), + expiresAt: z.iso.datetime(), + }), + z.strictObject({ + allowed: z.literal(false), + transactionId: z.uuid(), + integrationId: component, + reason: z.string().max(256), + }), +]); export const standaloneIntegrationOperationSchemas = [ + z.strictObject({ kind: z.literal("integration.inventory") }), + z.strictObject({ + kind: z.literal("integration.inspectOwnership"), + integrationId: component, + expectedContentDigest: digest.nullable(), + ownerId: owner, + operationId: z.uuid(), + }), + z.strictObject({ + kind: z.literal("integration.remove"), + transactionId: z.uuid(), + expectedContentDigest: digest.nullable(), + }), z.strictObject({ kind: z.literal("integration.inspectInstall"), source: standaloneInstallSourceSchema, @@ -101,10 +150,20 @@ export const standaloneIntegrationOperationSchemas = [ kind: z.literal("integration.settle"), transactionId: z.uuid(), decision: z.enum(["commit", "restore"]), - expectedContentDigest: digest, + expectedContentDigest: digest.nullable(), }), ] as const; export const standaloneIntegrationResultSchemas = { + "integration.inventory": z.strictObject({ + entries: z.array(standaloneFilesystemEntrySchema).max(1024), + }), + "integration.inspectOwnership": standaloneOwnershipInspectionSchema, + "integration.remove": z.strictObject({ + transactionId: z.uuid(), + integrationId: component, + contentDigest: digest.nullable(), + outcome: z.literal("removed"), + }), "integration.inspectInstall": standaloneInstallInspectionSchema, "integration.install": standaloneInstallReceiptSchema, "integration.settle": z.strictObject({ diff --git a/packages/integration-framework/src/installer.ts b/packages/integration-framework/src/installer.ts index 2305a4eb6..3443e4fd5 100644 --- a/packages/integration-framework/src/installer.ts +++ b/packages/integration-framework/src/installer.ts @@ -1088,6 +1088,99 @@ function restoreComponentLock(state: ComponentLockState): IntegrationComponentLo componentLocks.set(lock, state); return lock; } +/** Server-local receipt; HTTP callers cannot choose lock identities or paths. */ +export function describeIntegrationComponentLock( + lock: IntegrationComponentLock, +): ComponentLockState { + const state = componentLocks.get(lock); + if (!state) throw Error("Unverified integration component lock"); + const opened = componentLockDirectory(state); + opened.directory.close(); + opened.locks.close(); + opened.root.close(); + return structuredClone(state); +} +export function restoreIntegrationComponentLock( + state: ComponentLockState, +): IntegrationComponentLock { + return restoreComponentLock(state); +} +export function inspectOptionalLockedIntegrationComponent( + lock: IntegrationComponentLock, + signal: AbortSignal, +) { + const state = describeIntegrationComponentLock(lock); + try { + lstatSync(resolveInstallTarget(state.rootDir, state.id)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + return inspectLockedIntegrationComponent(lock, signal); +} +export function inspectLockedIntegrationComponent( + lock: IntegrationComponentLock, + signal: AbortSignal, +): { contentDigest: string; version: string; identity: { dev: string; ino: string } } { + const state = describeIntegrationComponentLock(lock); + const fingerprint = currentIntegrationFingerprint(state.rootDir, state.id, signal); + if (!fingerprint) throw Error("Integration directory is missing"); + const manifest = validatePreparedIntegration( + resolveInstallTarget(state.rootDir, state.id), + state.rootDir, + ); + if (manifest.id !== state.id) throw Error("Integration directory and manifest identity differ"); + describeIntegrationComponentLock(lock); + return { ...fingerprint, version: String(manifest.version) }; +} +export function inspectDeclarativeIntegrationDirectory( + directory: string, + rootDir: string, + signal: AbortSignal, +) { + const stat = lstatSync(directory, { bigint: true }); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw Error("Unsafe integration directory"); + const manifest = validatePreparedIntegration(directory, rootDir), + value = fingerprintPreparedDirectory(directory, signal, false), + after = lstatSync(directory, { bigint: true }); + if (stat.dev !== after.dev || stat.ino !== after.ino) + throw Error("Integration directory changed"); + return { + ...value, + id: String(manifest.id), + version: String(manifest.version), + identity: { dev: String(stat.dev), ino: String(stat.ino) }, + }; +} +/** Finish only the original terminal lock; a later writer is never released. */ +export function finishIntegrationComponentLock(state: ComponentLockState): void { + const root = new AnchoredDirectory(customDir(state.rootDir)); + let locks: AnchoredDirectory | undefined, directory: AnchoredDirectory | undefined; + try { + locks = root.directory(".installation-locks", false, true); + try { + directory = locks.directory(state.id, false, true); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + if ( + String(directory.stat.dev) !== state.identity.dev || + String(directory.stat.ino) !== state.identity.ino + ) + return; + if (directory.listBounded(2).length === 0) { + directory.close(); + locks.removeDirectory(state.id, state.identity); + return; + } + } finally { + directory?.close(); + locks?.close(); + root.close(); + } + releaseIntegrationComponentLock(restoreComponentLock(state)); +} function currentIntegrationFingerprint( rootDir: string, id: string, diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 17029eb27..e1e7b7c46 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -74,7 +74,10 @@ "dawarich.provisioning.inspect", "docker.status", "integration.inspectInstall", + "integration.inspectOwnership", "integration.install", + "integration.inventory", + "integration.remove", "integration.settle", "integrationConfig.apply", "release.apply", @@ -414,6 +417,15 @@ "apps/ops-agent/src/standalone-integration-runtime.ts#installStandaloneIntegrationRuntime" ], "typedKinds": ["integration.inspectInstall", "integration.install", "integration.settle"] + }, + { + "id": "api.standalone-integration-ownership", + "callSiteIds": [], + "semanticSources": [ + "apps/api/src/services/standalone-integrations/ownership.ts#handleStandaloneOwnership", + "apps/ops-agent/src/standalone-ownership-runtime.ts#installStandaloneOwnershipRuntime" + ], + "typedKinds": ["integration.inventory", "integration.inspectOwnership", "integration.remove"] } ], "mounts": [ From 86e993cb752727a7859b0851e6592cfa3ff8841a Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 00:59:32 +0100 Subject: [PATCH 061/135] feat(admin): configure built-in services and runtime resources --- .../routes/__tests__/admin-services.test.ts | 45 ++++- apps/api/src/routes/admin-services.ts | 27 ++- .../__tests__/service-config-resolver.test.ts | 39 ++++ apps/api/src/services/configuration-input.ts | 1 + .../services/configuration-ownership-view.ts | 1 + .../src/services/dawarich/managed-resolver.ts | 1 + .../api/src/services/dawarich/provisioning.ts | 1 + .../src/services/prospective-configuration.ts | 1 + .../src/services/service-config-resolver.ts | 23 ++- apps/api/src/utils/validate-config-body.ts | 7 + .../ServiceConfigForm.resources.test.tsx | 34 ++++ .../admin/services/ServiceConfigForm.tsx | 14 +- .../admin/shared/SchemaConfigForm.tsx | 20 ++ .../overlay/overlayZoomGate.ts | 5 +- .../integration-api/runtime/EnvProvider.tsx | 6 +- .../src/integration-api/runtime/env.test.ts | 46 +++++ apps/web/src/integration-api/runtime/env.ts | 55 +++--- .../administration/services-administration.md | 57 ++++++ .../overlay-traffic-tomtom/map-layer.test.tsx | 68 +++++++ packages/cli/src/commands/compose.ts | 20 +- packages/core/package.json | 4 + .../services-config-resolver.test.ts | 3 +- .../configuration-source-witness.ts | 4 + .../core/src/configuration-values.test.ts | 47 +++++ packages/core/src/configuration-values.ts | 63 +++++++ .../__tests__/runtime-resources.test.ts | 173 ++++++++++++++++++ .../core/src/services/compose-renderer.ts | 63 ++++++- packages/core/src/services/config-resolver.ts | 34 +++- .../core/src/services/configuration-schema.ts | 13 +- packages/core/src/services/index.ts | 6 + .../core/src/services/runtime-resources.ts | 50 +++++ .../extension-contracts/src/service-policy.ts | 9 + services/app-api/service.json | 76 +++++++- services/app-web/service.json | 88 ++++++++- services/brouter/service.json | 52 +++++- services/data-manager/service.json | 39 +++- services/dawarich-app/service.json | 94 ++++++++-- services/dawarich-postgis/service.json | 18 +- services/dawarich-redis/service.json | 23 ++- services/dawarich-sidekiq/service.json | 94 ++++++++-- services/elasticsearch/service.json | 37 +++- services/local-ai/service.json | 51 +++++- services/martin/service.json | 44 ++++- services/motis-feed-proxy/service.json | 23 ++- services/motis-staging/service.json | 53 +++++- services/motis/service.json | 56 +++++- services/nominatim/service.json | 75 +++++++- services/ops-agent/service.json | 21 ++- services/osrm/service.json | 47 ++++- services/otp/service.json | 60 +++++- services/overpass/service.json | 77 +++++++- services/pelias-pip/service.json | 42 ++++- services/pelias-placeholder/service.json | 42 ++++- services/pelias/service.json | 67 ++++++- services/photon/service.json | 60 +++++- services/postgis/service.json | 39 +++- services/redis/service.json | 38 +++- services/tileserver/service.json | 59 +++++- services/traefik/service.json | 51 +++++- services/transitous-runner/service.json | 23 ++- services/valhalla/service.json | 26 +++ services/well-known/service.json | 33 +++- 62 files changed, 2268 insertions(+), 180 deletions(-) create mode 100644 apps/web/src/components/admin/services/ServiceConfigForm.resources.test.tsx create mode 100644 integrations/overlay-traffic-tomtom/map-layer.test.tsx create mode 100644 packages/core/src/configuration-values.test.ts create mode 100644 packages/core/src/configuration-values.ts create mode 100644 packages/core/src/services/__tests__/runtime-resources.test.ts create mode 100644 packages/core/src/services/runtime-resources.ts diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index a54d02e67..e4fca374e 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -98,13 +98,21 @@ vi.mock("../../utils/rate-limit.js", () => ({ tilePublicApiLimit: { preHandler: () => vi.fn().mockResolvedValue(undefined) }, })); -// Resolve service config (service config GET uses this) +// Route-only fixture: the installed coherent ownership reader owns acquisition. +// Its source/DB invariants are tested in prospective-configuration tests. const mockResolveServiceConfigWithSources = vi.fn().mockResolvedValue({}); vi.mock("../../services/service-config-resolver.js", () => ({ resolveServiceConfigWithSources: (...args: unknown[]) => mockResolveServiceConfigWithSources(...args), })); +vi.mock("../../services/configuration-ownership-view.js", () => ({ + inspectServiceConfiguration: async (...args: unknown[]) => ({ + resolved: await mockResolveServiceConfigWithSources(...args), + ownership: {}, + }), +})); + const mockRestoreServiceConfig = vi.fn().mockResolvedValue(true); const mockMergeServiceConfig = vi.fn().mockResolvedValue(undefined); vi.mock("../../services/service-config-writer.js", () => ({ @@ -786,6 +794,9 @@ describe("service credentials", () => { }); it("GET lists declared secret fields with vault/missing status + setup guide", async () => { + mockResolveServiceConfigWithSources.mockResolvedValueOnce({ + NY_511_API_KEY: { value: "synthetic-configured-sentinel", source: "vault" }, + }); mockListServiceSecrets.mockResolvedValue([ { key: "NY_511_API_KEY", updatedAt: new Date("2026-06-26T10:00:00Z"), updatedBy: "u1" }, ]); @@ -1075,3 +1086,35 @@ describe("explicit destructive selection intent", () => { expect(mockWriteAuditLog).not.toHaveBeenCalled(); }); }); + +describe("resource reset transport", () => { + it("uses the original sealed producer's unset for an explicitly cleared built-in resource", async () => { + mockRegistryGet.mockReturnValue({ + isBuiltIn: true, + enabled: true, + manifest: { + id: "redis", + configSchema: { + type: "object", + properties: { OPENMAPX_MEMORY_MIB: { type: "integer", minimum: 16, maximum: 1048576 } }, + }, + }, + }); + const response = await app.inject({ + method: "POST", + url: "/admin/services/redis/config", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, + payload: { config: { OPENMAPX_MEMORY_MIB: null } }, + }); + expect(response.statusCode).toBe(200); + expect(applyTrustedConfiguration).toHaveBeenLastCalledWith( + expect.objectContaining({ + overlay: expect.objectContaining({ + changes: [ + { kind: "service-config", serviceId: "redis", set: {}, unset: ["OPENMAPX_MEMORY_MIB"] }, + ], + }), + }), + ); + }); +}); diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index 79a52caf0..9d10831b6 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -1,4 +1,5 @@ import { existsSync } from "node:fs"; +import { SERVICE_RESOURCE_FIELDS } from "@openmapx/core/configuration-values"; import { type OpsResultFor, trustedConfigurationPayloadSchema } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; import { isValidSecretKey } from "@openmapx/core/services/secret-key"; @@ -380,7 +381,27 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { reply.status(404); return { error: "Service not found" }; } - const body = req.body?.config; + const body = { ...req.body.config }; + const unset: string[] = []; + const properties = svc.manifest.configSchema?.properties as + | Record + | undefined; + for (const key of Object.keys(body)) { + if ( + body[key] === null && + svc.isBuiltIn && + (SERVICE_RESOURCE_FIELDS as readonly string[]).includes(key) && + properties?.[key] && + properties[key].readOnly !== true && + !( + Array.isArray(svc.manifest.configSchema?.required) && + svc.manifest.configSchema.required.includes(key) + ) + ) { + unset.push(key); + delete body[key]; + } + } // Validate against the manifest's configSchema so operators can't store // values the service won't accept at startup. Same validator used by the // integration config endpoint. @@ -414,7 +435,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { set: trustedConfigurationPayloadSchema.shape.serviceConfigs.element.shape.values.parse( config, ), - unset: [], + unset, }, ], }, @@ -424,7 +445,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { targetId: req.params.id, targetType: "service", action: "service.config.update", - details: { keys: Object.keys(config) }, + details: { keys: [...Object.keys(config), ...unset] }, request: req, }); return { ok: true }; diff --git a/apps/api/src/services/__tests__/service-config-resolver.test.ts b/apps/api/src/services/__tests__/service-config-resolver.test.ts index 0e4c6dde7..f0ad5bb7c 100644 --- a/apps/api/src/services/__tests__/service-config-resolver.test.ts +++ b/apps/api/src/services/__tests__/service-config-resolver.test.ts @@ -158,3 +158,42 @@ describe("resolveAllServiceConfigs", () => { expect(envKeys.get("test") ?? []).toEqual([]); }); }); + +describe("typed render settings and host ownership", () => { + it("materializes public numeric resources while retaining legacy host expressions", async () => { + process.env.SERVICE_TEST_OPENMAPX_MEMORY_MIB = "4096"; + selectLimitMock.mockResolvedValueOnce([{ config: { OPENMAPX_MEMORY_MIB: 1024 } }]); + const input = { + id: "test", + configSchema: { + properties: { OPENMAPX_MEMORY_MIB: { type: "integer", minimum: 16, maximum: 1048576 } }, + }, + containerMemory: "${LOCAL_AI_MEMORY:-8g}", + }; + const resolved = await resolveAllServiceConfigs([input]); + expect(resolved.values.get("test")).toEqual({ OPENMAPX_MEMORY_MIB: 4096 }); + expect(resolved.envKeys.has("test")).toBe(false); + delete process.env.SERVICE_TEST_OPENMAPX_MEMORY_MIB; + process.env.SERVICE_TEST_HOST_MEMORY = "16g"; + selectLimitMock.mockResolvedValueOnce([{ config: { OPENMAPX_MEMORY_MIB: 1024 } }]); + const legacy = await resolveAllServiceConfigs([ + { ...input, containerMemory: "${SERVICE_TEST_HOST_MEMORY:-8g}" }, + ]); + expect(legacy.values.get("test")).toEqual({}); + expect(legacy.envKeys.has("test")).toBe(false); + }); + it("locks direct host knobs and retains their actual manifest interpolation", async () => { + process.env.SERVICE_TEST_ORIGINAL_THREADS = "8"; + selectLimitMock.mockResolvedValueOnce([{ config: { THREADS: 4 } }]); + const input = { + id: "test", + configSchema: { properties: { THREADS: { type: "integer" } } }, + containerEnv: { THREADS: "${SERVICE_TEST_ORIGINAL_THREADS:-2}" }, + }; + expect(await resolveServiceConfigWithSources(input)).toMatchObject({ + THREADS: { value: "8", source: "env", manifestExpression: true }, + }); + selectLimitMock.mockResolvedValueOnce([{ config: { THREADS: 4 } }]); + expect((await resolveAllServiceConfigs([input])).values.get("test")).toEqual({}); + }); +}); diff --git a/apps/api/src/services/configuration-input.ts b/apps/api/src/services/configuration-input.ts index 291bec10d..fd40658c7 100644 --- a/apps/api/src/services/configuration-input.ts +++ b/apps/api/src/services/configuration-input.ts @@ -39,6 +39,7 @@ export async function buildConfigurationInput( id: service.manifest.id, configSchema: service.manifest.configSchema, containerEnv: service.manifest.container.environment, + containerMemory: service.manifest.container.memory, proxyHostConfigKey: service.manifest.exposure?.proxy?.host?.configKey, })), context, diff --git a/apps/api/src/services/configuration-ownership-view.ts b/apps/api/src/services/configuration-ownership-view.ts index 957d44abd..ea0841943 100644 --- a/apps/api/src/services/configuration-ownership-view.ts +++ b/apps/api/src/services/configuration-ownership-view.ts @@ -119,6 +119,7 @@ export async function serviceOwnershipSnapshot( id: serviceId, configSchema: service.manifest.configSchema, containerEnv: service.manifest.container.environment, + containerMemory: service.manifest.container.memory, }, context, ), diff --git a/apps/api/src/services/dawarich/managed-resolver.ts b/apps/api/src/services/dawarich/managed-resolver.ts index ebe8ed7bf..57cdf0171 100644 --- a/apps/api/src/services/dawarich/managed-resolver.ts +++ b/apps/api/src/services/dawarich/managed-resolver.ts @@ -294,6 +294,7 @@ async function readConfig(serviceId: string): Promise> { id: service.manifest.id, configSchema: service.manifest.configSchema, containerEnv: service.manifest.container.environment, + containerMemory: service.manifest.container.memory, }); } diff --git a/apps/api/src/services/dawarich/provisioning.ts b/apps/api/src/services/dawarich/provisioning.ts index d69542c64..f6b238084 100644 --- a/apps/api/src/services/dawarich/provisioning.ts +++ b/apps/api/src/services/dawarich/provisioning.ts @@ -690,6 +690,7 @@ async function readEffectiveServiceConfig(serviceId: string): Promise; + containerMemory?: string; /** * The config key naming the service's proxy host (`exposure.proxy.host.configKey`). * The host is routed by Traefik labels, which a Compose env reference @@ -85,7 +86,7 @@ export async function resolveServiceConfigWithSources( { id: manifest.id, configSchema: manifest.configSchema, - container: manifest.containerEnv ? { environment: manifest.containerEnv } : undefined, + container: { environment: manifest.containerEnv, memory: manifest.containerMemory }, }, process.env, ); @@ -164,7 +165,13 @@ export async function resolveAllServiceConfigs( const flat: Record = {}; const fromEnv: string[] = []; for (const [key, entry] of Object.entries(resolved)) { - if (entry.source === "env" && key !== m.proxyHostConfigKey) fromEnv.push(key); + if (entry.manifestExpression) continue; + if ( + entry.source === "env" && + key !== m.proxyHostConfigKey && + !materializeServiceConfigKey(m.id, key) + ) + fromEnv.push(key); else if (entry.source !== "default") flat[key] = entry.value; } return { id: m.id, flat, fromEnv }; @@ -199,7 +206,7 @@ export async function resolveServiceConfigWithSourcesStrict( { id: manifest.id, configSchema: manifest.configSchema, - container: manifest.containerEnv ? { environment: manifest.containerEnv } : undefined, + container: { environment: manifest.containerEnv, memory: manifest.containerMemory }, }, context.env, ); @@ -227,7 +234,13 @@ export async function resolveAllServiceConfigsStrict( const flat: Record = {}; const fromEnv: string[] = []; for (const [key, entry] of Object.entries(resolved)) { - if (entry.source === "env" && key !== manifest.proxyHostConfigKey) fromEnv.push(key); + if (entry.manifestExpression) continue; + if ( + entry.source === "env" && + key !== manifest.proxyHostConfigKey && + !materializeServiceConfigKey(manifest.id, key) + ) + fromEnv.push(key); else if (entry.source !== "default") flat[key] = entry.value; } values.set(manifest.id, flat); diff --git a/apps/api/src/utils/validate-config-body.ts b/apps/api/src/utils/validate-config-body.ts index facd253b5..a57b20a88 100644 --- a/apps/api/src/utils/validate-config-body.ts +++ b/apps/api/src/utils/validate-config-body.ts @@ -1,3 +1,4 @@ +import { OPERATOR_STRING_FORMATS, validOperatorString } from "@openmapx/core/configuration-values"; /** * Lightweight per-property validator + introspection helpers for the * JSON-Schema-shaped manifest `configSchema` block used by both integrations @@ -139,6 +140,12 @@ function validateSchemaValue( if (typeof schema.pattern === "string" && !new RegExp(schema.pattern).test(value)) { return [`${path} has an invalid format`]; } + if ( + typeof schema.format === "string" && + (OPERATOR_STRING_FORMATS as readonly string[]).includes(schema.format) && + !validOperatorString(value, schema.format) + ) + return [`${path} has an invalid format`]; if (schema.format === "url" && value !== "") { try { const parsed = new URL(value); diff --git a/apps/web/src/components/admin/services/ServiceConfigForm.resources.test.tsx b/apps/web/src/components/admin/services/ServiceConfigForm.resources.test.tsx new file mode 100644 index 000000000..691a4523c --- /dev/null +++ b/apps/web/src/components/admin/services/ServiceConfigForm.resources.test.tsx @@ -0,0 +1,34 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen, waitFor } from "@/test"; +import { ServiceConfigForm } from "./ServiceConfigForm"; + +vi.mock("../activity/activityQueries", () => ({ + useActivityClient: () => ({ apiUrl: "http://synthetic.invalid" }), +})); +afterEach(cleanup); +it("clears a saved resource through explicit reset while leaving unrelated values untouched", async () => { + const save = vi.fn(async () => {}); + render( + { + save(values); + }} + />, + ); + const inputs = screen.getAllByRole("spinbutton"); + fireEvent.change(inputs[0]!, { target: { value: "" } }); + fireEvent.click(screen.getByRole("button", { name: /^Save/ })); + await waitFor(() => expect(save).toHaveBeenCalledWith({ OPENMAPX_MEMORY_MIB: null })); +}); diff --git a/apps/web/src/components/admin/services/ServiceConfigForm.tsx b/apps/web/src/components/admin/services/ServiceConfigForm.tsx index 7aa93edd3..a6fe0b291 100644 --- a/apps/web/src/components/admin/services/ServiceConfigForm.tsx +++ b/apps/web/src/components/admin/services/ServiceConfigForm.tsx @@ -4,6 +4,7 @@ import type { AdminServiceResolvedConfig, ConfigurationOwnership, } from "@openmapx/core/api-contracts"; +import { SERVICE_RESOURCE_FIELDS } from "@openmapx/core/configuration-values"; import { useActivityClient } from "../activity/activityQueries"; import { SchemaConfigForm } from "../shared/SchemaConfigForm"; @@ -39,6 +40,13 @@ export function ServiceConfigForm({ onSaveAndApply, }: ServiceConfigFormProps) { const { apiUrl } = useActivityClient(); + const withResourceResets = (values: Record) => + Object.fromEntries( + Object.entries(values).map(([key, value]) => [ + key, + value === "" && (SERVICE_RESOURCE_FIELDS as readonly string[]).includes(key) ? null : value, + ]), + ); return ( onSave(withResourceResets(values))} + onSaveAndApply={ + onSaveAndApply ? (values) => onSaveAndApply(withResourceResets(values)) : undefined + } applyLabel="Save & apply" emptyMessage="No editable configuration fields declared for this service." /> diff --git a/apps/web/src/components/admin/shared/SchemaConfigForm.tsx b/apps/web/src/components/admin/shared/SchemaConfigForm.tsx index dd33079eb..5fef47303 100644 --- a/apps/web/src/components/admin/shared/SchemaConfigForm.tsx +++ b/apps/web/src/components/admin/shared/SchemaConfigForm.tsx @@ -16,6 +16,7 @@ import TextField from "@mui/material/TextField"; import Tooltip from "@mui/material/Tooltip"; import Typography from "@mui/material/Typography"; import type { ConfigurationOwnership } from "@openmapx/core/api-contracts"; +import { OPERATOR_STRING_FORMATS, validOperatorString } from "@openmapx/core/configuration-values"; import { type CredentialSetup, readCredentialSetup } from "@openmapx/integration-framework"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { CredentialSetupGuide } from "../integrations/CredentialSetupGuide"; @@ -54,6 +55,8 @@ interface SchemaProperty { default?: unknown; enum?: string[]; format?: string; + minimum?: number; + maximum?: number; "x-openmapx-secret"?: boolean; readOnly?: boolean; "x-openmapx-setup"?: CredentialSetup; @@ -66,6 +69,8 @@ export interface SchemaConfigField { type: string; enum?: string[]; format?: string; + minimum?: number; + maximum?: number; default?: unknown; setup?: CredentialSetup; } @@ -93,6 +98,12 @@ export function buildConfigDiff( for (const key of changedKeys) { const field = fields.find((candidate) => candidate.key === key); const value = values[key]; + if ( + field?.format && + (OPERATOR_STRING_FORMATS as readonly string[]).includes(field.format) && + (typeof value !== "string" || !validOperatorString(value, field.format)) + ) + throw Error(`${field.title} has an invalid format`); if (field && isStructuredField(field)) { try { diff[key] = JSON.parse(String(value)); @@ -135,6 +146,8 @@ export function extractConfigFields( type: def?.type ?? "string", enum: def?.enum, format: def?.format, + minimum: def?.minimum, + maximum: def?.maximum, default: def?.default, setup: readCredentialSetup(def), })); @@ -425,6 +438,13 @@ export function SchemaConfigForm({ ? "url" : "text" } + slotProps={{ + htmlInput: { + min: field.minimum, + max: field.maximum, + step: field.type === "integer" ? 1 : "any", + }, + }} placeholder={field.default !== undefined ? String(field.default) : undefined} sx={{ maxWidth: 480 }} /> diff --git a/apps/web/src/integration-api/overlay/overlayZoomGate.ts b/apps/web/src/integration-api/overlay/overlayZoomGate.ts index 6e14f7413..21d4e51c8 100644 --- a/apps/web/src/integration-api/overlay/overlayZoomGate.ts +++ b/apps/web/src/integration-api/overlay/overlayZoomGate.ts @@ -3,6 +3,7 @@ import { integrationIdToOverlayId, useMapStore } from "@openmapx/core"; import { useIntegrationRegistry } from "@openmapx/integration-framework/react"; import { useMemo } from "react"; +import { useOptionalEnv } from "../runtime/EnvProvider"; /** * Zoom gating for overlays, declared per integration as @@ -22,12 +23,14 @@ import { useMemo } from "react"; /** Manifest `minZoom` for an overlay; 0 when it isn't zoom-gated. */ export function useOverlayMinZoom(overlayId: string): number { const registry = useIntegrationRegistry(); + const env = useOptionalEnv(); return useMemo(() => { + if (overlayId === "traffic" && env?.trafficMinZoom !== undefined) return env.trafficMinZoom; const match = registry .getAll() .find((integration) => integrationIdToOverlayId(integration.id) === overlayId); return match?.frontend?.overlay?.minZoom ?? 0; - }, [registry, overlayId]); + }, [registry, overlayId, env?.trafficMinZoom]); } export interface OverlayZoomGate { diff --git a/apps/web/src/integration-api/runtime/EnvProvider.tsx b/apps/web/src/integration-api/runtime/EnvProvider.tsx index 1eda123c7..865b9b746 100644 --- a/apps/web/src/integration-api/runtime/EnvProvider.tsx +++ b/apps/web/src/integration-api/runtime/EnvProvider.tsx @@ -9,8 +9,12 @@ export function EnvProvider({ config, children }: { config: ClientEnv; children: return {children}; } +export function useOptionalEnv(): ClientEnv | null { + return useContext(EnvContext); +} + export function useEnv(): ClientEnv { - const ctx = useContext(EnvContext); + const ctx = useOptionalEnv(); if (!ctx) throw new Error("useEnv must be used within EnvProvider"); return ctx; } diff --git a/apps/web/src/integration-api/runtime/env.test.ts b/apps/web/src/integration-api/runtime/env.test.ts index e2e73a61c..dacc37be3 100644 --- a/apps/web/src/integration-api/runtime/env.test.ts +++ b/apps/web/src/integration-api/runtime/env.test.ts @@ -131,3 +131,49 @@ describe("vector basemap selection", () => { expect(env.mapStyleUrl).toBe("/tiles"); }); }); + +describe("managed runtime values", () => { + it("reads managed values from each server invocation rather than a build-time NEXT_PUBLIC literal", () => { + expect( + buildClientEnv(undefined, { + NEXT_PUBLIC_TRAFFIC_MIN_ZOOM: "8", + NEXT_PUBLIC_TILES_URL: "/runtime-a/tiles.json", + }), + ).toMatchObject({ trafficMinZoom: 8, tilesUrl: "/runtime-a/tiles.json" }); + expect( + buildClientEnv(undefined, { + NEXT_PUBLIC_TRAFFIC_MIN_ZOOM: "12", + NEXT_PUBLIC_TILES_URL: "/runtime-b/tiles.json", + }), + ).toMatchObject({ trafficMinZoom: 12, tilesUrl: "/runtime-b/tiles.json" }); + }); + it.each([undefined, "", "1.5", "25", "-1", "NaN", "Infinity", "8junk"])( + "omits invalid raster minimum zoom %s", + (value) => { + expect(buildClientEnv(undefined, { NEXT_PUBLIC_TRAFFIC_MIN_ZOOM: value })).not.toHaveProperty( + "trafficMinZoom", + ); + }, + ); + it("preserves same-origin URL defaults and rejects unsafe runtime URL overrides", () => { + expect( + buildClientEnv(undefined, { + NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL: "/local-dem/tiles.json", + NEXT_PUBLIC_MAP_STYLE_URL: "/tiles", + NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE: "/traffic/{z}/{x}/{y}.png", + }), + ).toMatchObject({ + terrainDemTilejsonUrl: "/local-dem/tiles.json", + mapStyleUrl: "/tiles", + trafficTileUrlTemplate: "/traffic/{z}/{x}/{y}.png", + }); + const env = buildClientEnv(undefined, { + NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL: "//evil.test/tiles.json", + NEXT_PUBLIC_MAP_STYLE_URL: "https://u:p@evil.test/tiles", + NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE: "/x/{arbitrary}", + }); + expect(env.mapStyleUrl).toBe(""); + expect(env.terrainDemTilejsonUrl).toContain("/api/mapterhorn/tiles.json"); + expect(env.trafficTileUrlTemplate).toBe("/api/traffic/flow/{z}/{x}/{y}.png"); + }); +}); diff --git a/apps/web/src/integration-api/runtime/env.ts b/apps/web/src/integration-api/runtime/env.ts index ad6c0bc4a..0e455aa3a 100644 --- a/apps/web/src/integration-api/runtime/env.ts +++ b/apps/web/src/integration-api/runtime/env.ts @@ -11,6 +11,7 @@ * read from the context via the useEnv() hook. */ +import { parseOperatorUrl } from "@openmapx/core/configuration-values"; import { hostedBasemapProviderSchema, type MapConfig } from "@openmapx/core/map-config"; export interface ClientEnv { @@ -20,6 +21,7 @@ export interface ClientEnv { tilesUrl: string; styleProvider: "maptiler" | "openmapx"; trafficTileUrlTemplate: string; + trafficMinZoom?: number; cyclOsmTileUrlTemplate: string; terrainDemTilejsonUrl: string; terrainDemTileUrlTemplate: string; @@ -33,21 +35,28 @@ export interface ClientEnv { } /** - * Build the client environment config from process.env. + * Build the client environment config from runtime. * Must only be called from server components (where process.env is real). */ -export function buildClientEnv(mapConfig?: MapConfig): ClientEnv { - const apiBase = (process.env.NEXT_PUBLIC_API_URL ?? "").replace(/\/$/, ""); - const tilesUrl = - process.env.NEXT_PUBLIC_TILES_URL?.trim() || (mapConfig?.selfHostedTilesUrl ?? ""); - const mapStyleUrl = - process.env.NEXT_PUBLIC_MAP_STYLE_URL?.trim() || (mapConfig?.selfHostedGlyphsUrl ?? ""); +export function buildClientEnv( + mapConfig?: MapConfig, + runtime: Readonly> = process.env, +): ClientEnv { + // Reading through the object prevents Next's NEXT_PUBLIC build-time substitution. + const url = (key: string, template = false) => + parseOperatorUrl(runtime[key] ?? "", template) || ""; + const rawZoom = runtime.NEXT_PUBLIC_TRAFFIC_MIN_ZOOM; + const trafficMinZoom = + rawZoom && /^\d+$/.test(rawZoom) && Number(rawZoom) <= 24 ? Number(rawZoom) : undefined; + const apiBase = (runtime.NEXT_PUBLIC_API_URL ?? "").replace(/\/$/, ""); + const tilesUrl = url("NEXT_PUBLIC_TILES_URL") || (mapConfig?.selfHostedTilesUrl ?? ""); + const mapStyleUrl = url("NEXT_PUBLIC_MAP_STYLE_URL") || (mapConfig?.selfHostedGlyphsUrl ?? ""); const preference = mapConfig?.hostedBasemapProvider ?? - hostedBasemapProviderSchema.catch("auto").parse(process.env.BASEMAP_PROVIDER); + hostedBasemapProviderSchema.catch("auto").parse(runtime.BASEMAP_PROVIDER); const keyConfigured = mapConfig?.maptilerConfigured ?? - Boolean(process.env.MAPTILER_KEY?.trim() || process.env.NEXT_PUBLIC_MAPTILER_KEY?.trim()); + Boolean(runtime.MAPTILER_KEY?.trim() || runtime.NEXT_PUBLIC_MAPTILER_KEY?.trim()); const basemapProvider = tilesUrl ? "selfhosted" : preference === "auto" @@ -56,56 +65,56 @@ export function buildClientEnv(mapConfig?: MapConfig): ClientEnv { : "openfreemap" : preference; const styleProvider = - basemapProvider === "maptiler" && process.env.NEXT_PUBLIC_STYLE_PROVIDER === "maptiler" + basemapProvider === "maptiler" && runtime.NEXT_PUBLIC_STYLE_PROVIDER === "maptiler" ? "maptiler" : "openmapx"; const keylessTerrain = basemapProvider !== "maptiler"; - const customDem = Boolean(process.env.NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL); + const customDem = Boolean(url("NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL")); const useMapterhorn = keylessTerrain && !customDem; - const customContour = Boolean(process.env.NEXT_PUBLIC_TERRAIN_CONTOUR_TILEJSON_URL); + const customContour = Boolean(url("NEXT_PUBLIC_TERRAIN_CONTOUR_TILEJSON_URL")); const generatedContours = (useMapterhorn || customDem) && !customContour; return { - apiUrl: process.env.NEXT_PUBLIC_API_URL ?? "", + ...(trafficMinZoom === undefined ? {} : { trafficMinZoom }), + apiUrl: runtime.NEXT_PUBLIC_API_URL ?? "", mapStyleUrl, tilesUrl, basemapProvider, styleProvider, trafficTileUrlTemplate: - process.env.NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE || + url("NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE", true) || (apiBase ? `${apiBase}/api/traffic/flow/{z}/{x}/{y}.png` : "/api/traffic/flow/{z}/{x}/{y}.png"), cyclOsmTileUrlTemplate: - process.env.NEXT_PUBLIC_CYCLOSM_TILE_URL_TEMPLATE || + url("NEXT_PUBLIC_CYCLOSM_TILE_URL_TEMPLATE", true) || (apiBase ? `${apiBase}/api/tiles/cyclosm/{z}/{x}/{y}.png` : "/api/tiles/cyclosm/{z}/{x}/{y}.png"), terrainDemTilejsonUrl: - process.env.NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL || + url("NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL") || (useMapterhorn ? `${apiBase}/api/mapterhorn/tiles.json` : `${apiBase}/api/maptiler/tiles/terrain-rgb-v2/tiles.json`), terrainDemTileUrlTemplate: - process.env.NEXT_PUBLIC_TERRAIN_DEM_TILE_URL_TEMPLATE || + url("NEXT_PUBLIC_TERRAIN_DEM_TILE_URL_TEMPLATE", true) || (useMapterhorn ? `${apiBase}/api/mapterhorn/{z}/{x}/{y}.webp` : ""), terrainDemEncoding: - process.env.NEXT_PUBLIC_TERRAIN_DEM_ENCODING === "terrarium" || useMapterhorn + runtime.NEXT_PUBLIC_TERRAIN_DEM_ENCODING === "terrarium" || useMapterhorn ? "terrarium" : "mapbox", terrainContourMode: generatedContours ? "generated" : "vector", terrainContourTilejsonUrl: - process.env.NEXT_PUBLIC_TERRAIN_CONTOUR_TILEJSON_URL || + url("NEXT_PUBLIC_TERRAIN_CONTOUR_TILEJSON_URL") || `${apiBase}/api/maptiler/tiles/contours-v2/tiles.json`, terrainAttributionName: - process.env.NEXT_PUBLIC_TERRAIN_ATTRIBUTION_NAME || + runtime.NEXT_PUBLIC_TERRAIN_ATTRIBUTION_NAME || (useMapterhorn ? "© Mapterhorn" : "© MapTiler"), terrainAttributionUrl: - process.env.NEXT_PUBLIC_TERRAIN_ATTRIBUTION_URL || + url("NEXT_PUBLIC_TERRAIN_ATTRIBUTION_URL") || (useMapterhorn ? "https://mapterhorn.com/attribution/" : "https://www.maptiler.com/copyright/"), - martinBaseUrl: - process.env.NEXT_PUBLIC_MARTIN_URL || (apiBase ? `${apiBase}/martin` : "/martin"), + martinBaseUrl: url("NEXT_PUBLIC_MARTIN_URL") || (apiBase ? `${apiBase}/martin` : "/martin"), }; } diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index f0fc2d451..27fe3014b 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -354,3 +354,60 @@ ownership and does not silently adopt their values. The host-local `openmapx data generate-api-keys` command creates or updates a template from the active pins while preserving all existing values and orphaned keys. The previous browser generator operation is no longer available. + +## Operator settings and resource limits + +Every built-in service has a Config tab with optional memory (16–1048576 MiB) +and CPU (0.125–256 cores) limits. These are Docker `deploy.resources.limits`, +not environment variables. Leaving a field unset preserves the manifest limit, +including host interpolation. Clear a saved resource field to restore that limit. Host-owned fields are locked; both `SERVICE_*` +overrides and the original variable referenced by a manifest take precedence. +Saving uses the same validated, sealed prospective configuration as applying: +failed database reads, invalid values or an oversized snapshot refuse the entire +change. Saving configuration does not rebuild prepared data. + +MOTIS resource settings belong to **primary MOTIS**. The same generation gives +both primary and staging slots identical limits and updates the data-manager's +capacity projection. The staging fields are read-only. Apply the coordinated +stack, then rerun transit preflight before another build: profile minimums, +candidate/rollback headroom, CPU, disk and timeout checks still apply. Reserving +resources for only the active slot is insufficient. + +| Service | Editable settings | Actual consumer | +| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| All 30 built-ins | `OPENMAPX_MEMORY_MIB`, `OPENMAPX_CPU_LIMIT` (staging MOTIS follows primary) | Docker Compose resource limits; never container environment | +| app-api | `LOG_LEVEL`; `RATE_LIMIT_IMAGE_PROXY_MAX_BYTES`, `RATE_LIMIT_IMAGE_PROXY_WINDOW_MS`, `RATE_LIMIT_IMAGE_PROXY_MAX_CONCURRENT_PER_CLIENT`, `RATE_LIMIT_IMAGE_PROXY_MAX_CONCURRENT_GLOBAL` | Pino logger and image-proxy request/byte/concurrency budget | +| app-web | `NEXT_PUBLIC_STYLE_PROVIDER`, `NEXT_PUBLIC_MAP_STYLE_URL`, `NEXT_PUBLIC_TILES_URL`, `NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE`, `NEXT_PUBLIC_CYCLOSM_TILE_URL_TEMPLATE`, `NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL`, `NEXT_PUBLIC_TERRAIN_CONTOUR_TILEJSON_URL`, `NEXT_PUBLIC_TERRAIN_ATTRIBUTION_NAME`, `NEXT_PUBLIC_TERRAIN_ATTRIBUTION_URL`, `NEXT_PUBLIC_TRAFFIC_MIN_ZOOM` | Server runtime projection into the browser; map glyphs/basemap, raster tiles, terrain, attribution and raster traffic zoom gate | +| data-manager | `LOG_LEVEL`, `OVERTURE_DUCKDB_MEMORY_MB`, `OVERTURE_DUCKDB_THREADS`, `OVERTURE_DUCKDB_TEMP_MB` | Logger and bounded DuckDB memory/thread/spill settings | +| local-ai | `OLLAMA_KEEP_ALIVE` (0 or duration up to 24h), `OLLAMA_MAX_LOADED_MODELS` | Ollama's model lifetime and loaded-model count | +| elasticsearch | `OPENMAPX_JAVA_HEAP_MIB` | Only `-Xmsm -Xmxm` in `ES_JAVA_OPTS`; choose heap below the container limit | +| nominatim | `THREADS`, `REPLICATION_UPDATE_INTERVAL`, `IMPORT_STYLE`, `REVERSE_ONLY`, `IMPORT_WIKIPEDIA`, `IMPORT_US_POSTCODES`, `IMPORT_GB_POSTCODES` | Image import and replication environment; import settings require a database rebuild | +| photon | `REGION`, `UPDATE_STRATEGY` (PARALLEL/SEQUENTIAL), `UPDATE_INTERVAL` (1h–8760h), `LOG_LEVEL` (DEBUG/INFO/WARN/ERROR) | Photon image index acquisition, update scheduling and Python logger; TRACE is unsupported | +| overpass | `OVERPASS_SPACE`, `OVERPASS_FASTCGI_PROCESSES`, `OVERPASS_UPDATE_SLEEP`, `OVERPASS_RULES_LOAD` | Image query memory, workers, refresh and rules budget | +| postgis | `POSTGIS_DIAGNOSTICS`, `POSTGIS_TRACK_IO_TIMING` | OpenMapX's PostgreSQL entrypoint diagnostics | +| valhalla | `serve_tiles`, `build_time_zones`, `use_tiles_ignore_pbf`, plus existing `build_elevation` and `build_admins` | Scripted image environment, serialized as `True`/`False` | +| Existing BRouter and Dawarich schemas | Existing routing/import/thread and application/database/credential fields | Existing service consumers and typed credential workflows, preserved alongside resource fields | + +The legacy `NEXT_PUBLIC_MAP_STYLE_URL` name denotes the **glyph server base**, +not a style JSON document. URL fields accept an absolute HTTP(S) URL without +credentials or a same-origin path beginning with one `/`. Empty means default. +Protocol-relative URLs, backslashes, controls and unsupported placeholders are +refused. Only tile-template fields accept `{z}`, `{x}`, `{y}` and `{s}`. +Local paths such as `/local-dem/tiles.json`, `/local-contours/tiles.json` and +`/tiles` remain supported. Runtime values are read by the server for each page +request; they are not frozen into the Docker image during `next build`. + +Raster traffic's optional minimum zoom is an integer from 0 through 24. Unset +or malformed host values retain its manifest default of 10. Both the raster +layer and the layer selector use the same threshold; the separate vector +traffic-flow integration keeps its default of 6. + +Free-form JVM flags, Node flags and shell preprocessing commands are not editable +through these forms. Community manifests cannot declare the platform's reserved +resource fields to expand their resource authority. + +Consumer references: [Ollama environment parser](https://github.com/ollama/ollama/blob/v0.40.3/envconfig/config.go), +[Photon 2.4.0 settings](https://github.com/rtuszik/photon-docker/blob/2.4.0/src/utils/config.py) +and [logger](https://github.com/rtuszik/photon-docker/blob/2.4.0/src/utils/logger.py), +[Nominatim import styles](https://nominatim.org/release-docs/latest/customize/Import-Styles/), +and [Valhalla scripted image](https://github.com/valhalla/valhalla/blob/master/docker/README.md). diff --git a/integrations/overlay-traffic-tomtom/map-layer.test.tsx b/integrations/overlay-traffic-tomtom/map-layer.test.tsx new file mode 100644 index 000000000..dc7be9e52 --- /dev/null +++ b/integrations/overlay-traffic-tomtom/map-layer.test.tsx @@ -0,0 +1,68 @@ +// @vitest-environment jsdom +import { useMapStore } from "@openmapx/core"; +import { IntegrationRegistry } from "@openmapx/integration-framework"; +import { IntegrationRegistryContext } from "@openmapx/integration-framework/react"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; +import { useOverlayZoomGate } from "@/integration-api/overlay/overlayZoomGate"; +import { EnvProvider } from "@/integration-api/runtime/EnvProvider"; +import { buildClientEnv } from "@/integration-api/runtime/env"; +import { render, renderHook } from "@/test"; +import manifest from "./manifest.json"; +import { TrafficLayer } from "./map-layer"; + +const state = vi.hoisted(() => ({ layer: undefined as { minzoom: number } | undefined })); +vi.mock("@/integration-api/map/MapContext", () => ({ + useMap: () => ({ mapRef: { current: null }, mapReady: true, styleVersion: 0 }), +})); +vi.mock("@/integration-api/map/useStyleSyncedLayer", () => ({ + useStyleSyncedLayer: (options: { addLayer: (map: unknown) => void }) => { + options.addLayer({}); + }, +})); +vi.mock("@/integration-api/map/layerStack", () => ({ + addLayerInSlot: (_map: unknown, layer: { minzoom: number }) => { + state.layer = layer; + }, +})); +vi.mock("@/integration-api/overlay/useContributorResolver", () => ({ + useContributorResolver: () => () => [], +})); +vi.mock("@/integration-api/overlay/useRasterMapAttributions", () => ({ + useRasterMapAttributions: () => {}, +})); +const registry = new IntegrationRegistry([ + { ...manifest, isBuiltIn: true, enabled: true }, + { + id: "overlay-traffic-flow", + name: "Vector flow", + enabled: true, + isBuiltIn: true, + domains: ["map-overlay"], + frontend: { overlay: { minZoom: 6 } }, + }, +]); +describe("raster layer and selector share the managed zoom gate", () => { + it.each([ + ["8", 8, false], + ["garbage", 10, true], + [undefined, 10, true], + ] as const)("uses %s consistently", (value, expected, blocked) => { + useMapStore.getState().setZoom(8); + const config = buildClientEnv(undefined, { NEXT_PUBLIC_TRAFFIC_MIN_ZOOM: value }); + const wrapper = ({ children }: { children: ReactNode }) => ( + + {children} + + ); + render(, { wrapper }); + expect(state.layer?.minzoom).toBe(expected); + expect(renderHook(() => useOverlayZoomGate("traffic"), { wrapper }).result.current).toEqual({ + minZoom: expected, + belowMinZoom: blocked, + }); + expect( + renderHook(() => useOverlayZoomGate("traffic-flow"), { wrapper }).result.current.minZoom, + ).toBe(6); + }); +}); diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index 78ca5be4d..52615d291 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -284,14 +284,28 @@ export async function renderComposeWithWriter( ).filter(([, entry]) => entry.source === "env"); // An env value is a reference in the YAML, except the proxy host, which // the routes need as a value. - const envKeys = fromEnv.map(([key]) => key).filter((key) => key !== proxyHostKey); + const envKeys = fromEnv + .map(([key]) => key) + .filter( + (key) => + key !== proxyHostKey && + !coreServices.materializeServiceConfigKey(id, key) && + !fromEnv.find(([name]) => name === key)?.[1].manifestExpression, + ); const values: coreServices.ConfigurationInput["serviceConfigs"][number]["values"] = Object.fromEntries( Object.entries(savedConfigs.get(id)?.values ?? {}).filter( - ([key]) => !envKeys.includes(key), + ([key]) => + !envKeys.includes(key) && + !fromEnv.some(([name, entry]) => name === key && entry.manifestExpression), ), ); - for (const [key, entry] of fromEnv) if (key === proxyHostKey) values[key] = String(entry.value); + for (const [key, entry] of fromEnv) + if ( + !entry.manifestExpression && + (key === proxyHostKey || coreServices.materializeServiceConfigKey(id, key)) + ) + values[key] = entry.value as string | number | boolean; if (Object.keys(values).length === 0 && envKeys.length === 0) return []; return [{ serviceId: id, values, ...(envKeys.length > 0 ? { envKeys } : {}) }]; }); diff --git a/packages/core/package.json b/packages/core/package.json index da8e73b5c..86ac1a00e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -209,6 +209,10 @@ "./catalog-pins/types": { "types": "./src/catalog-pins/types.ts", "default": "./src/catalog-pins/types.ts" + }, + "./configuration-values": { + "types": "./src/configuration-values.ts", + "default": "./src/configuration-values.ts" } }, "scripts": { diff --git a/packages/core/src/__tests__/services-config-resolver.test.ts b/packages/core/src/__tests__/services-config-resolver.test.ts index ebae454ef..2d01a9f95 100644 --- a/packages/core/src/__tests__/services-config-resolver.test.ts +++ b/packages/core/src/__tests__/services-config-resolver.test.ts @@ -180,7 +180,8 @@ describe("resolveServiceConfigFromEnv", () => { }, { SERVICE_VALHALLA_BUILD_ELEVATION: "False" }, ); - expect(r.build_elevation).toEqual({ value: "False", source: "env" }); + // Valhalla overrides are typed before sealing; its renderer emits capitalized False. + expect(r.build_elevation).toEqual({ value: false, source: "env" }); }); }); diff --git a/packages/core/src/build-artifacts/configuration-source-witness.ts b/packages/core/src/build-artifacts/configuration-source-witness.ts index df5084c0c..848be580c 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.ts @@ -184,6 +184,10 @@ export function configurationHostFields( const keys = new Set([...declared, "OPENMAPX_MEMORY_MIB", "OPENMAPX_CPU_LIMIT"]); const fields = [...keys].sort().map((key) => { const names = [`${serviceConfigEnvPrefix(manifest.id)}${key.toUpperCase()}`]; + const hostExpression = manifest.container.environment?.[key]?.match( + /^\$\{([A-Z][A-Z0-9_]*)(?::[-?][^}]*)?\}$/, + )?.[1]; + if (hostExpression) names.push(hostExpression); const variable = memory[manifest.id]; if ( key === "OPENMAPX_MEMORY_MIB" && diff --git a/packages/core/src/configuration-values.test.ts b/packages/core/src/configuration-values.test.ts new file mode 100644 index 000000000..811382c62 --- /dev/null +++ b/packages/core/src/configuration-values.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { parseOperatorUrl, validOperatorString } from "./configuration-values"; + +describe("shared operator string grammar", () => { + it.each([ + "", + "/local-dem/tiles.json", + "/local-contours/tiles.json", + "/tiles", + "https://tiles.example/a?x=1", + "http://localhost:3000/a", + ])("accepts ordinary URL %s", (value) => expect(parseOperatorUrl(value)).toBe(value)); + it.each([ + "//other.test/a", + "https://u:p@other.test/a", + "javascript:alert(1)", + "file:///etc/passwd", + "/a\\b", + "/a\nb", + "/tiles/{z}", + "http:/example.com", + " https://example.com", + "/" + "x".repeat(2048), + ])("rejects unsafe ordinary URL %s", (value) => expect(parseOperatorUrl(value)).toBeUndefined()); + it("limits placeholders to actual tile consumer tokens", () => { + expect(parseOperatorUrl("https://{s}.example.test/{z}/{x}/{y}", true)).toBe( + "https://{s}.example.test/{z}/{x}/{y}", + ); + expect(parseOperatorUrl("/{other}", true)).toBeUndefined(); + }); + it("bounds durations and matches pinned consumer units", () => { + for (const value of ["0", "30m", "0.5h", "24h"]) + expect(validOperatorString(value, "ollama-duration"), value).toBe(true); + for (const value of ["-1", "25h", "${SHELL}", "1e4h"]) + expect(validOperatorString(value, "ollama-duration"), value).toBe(false); + for (const value of ["1h", "60m", "8760h"]) + expect(validOperatorString(value, "photon-duration"), value).toBe(true); + for (const value of ["0h", "8761h", "0.5h"]) + expect(validOperatorString(value, "photon-duration"), value).toBe(false); + }); + it("uses the existing region path grammar", () => { + for (const value of ["planet", "europe/germany", "north-america/us"]) + expect(validOperatorString(value, "resource-region")).toBe(true); + for (const value of ["../etc", "/europe", "Europe", "eu//de", "x/../y"]) + expect(validOperatorString(value, "resource-region")).toBe(false); + }); +}); diff --git a/packages/core/src/configuration-values.ts b/packages/core/src/configuration-values.ts new file mode 100644 index 000000000..3ee62b342 --- /dev/null +++ b/packages/core/src/configuration-values.ts @@ -0,0 +1,63 @@ +/** Browser-safe validation shared by forms, the API and sealed configuration imports. */ +export const OPERATOR_STRING_FORMATS = [ + "map-url", + "map-tile-url", + "ollama-duration", + "photon-duration", + "resource-region", +] as const; + +export function parseOperatorUrl(value: unknown, template = false): string | undefined { + if ( + typeof value !== "string" || + value.length > 2048 || + /[\s\\]/u.test(value) || + [...value].some((character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127) + ) + return undefined; + if (value === "") return ""; + let address = value; + if (template) address = address.replace(/\{[zxys]\}/g, "tile"); + if (/[{}]/.test(address)) return undefined; + if (address.startsWith("/") && !address.startsWith("//")) return value; + try { + const url = new URL(address); + return (url.protocol === "http:" || url.protocol === "https:") && + !url.username && + !url.password && + /^https?:\/\//i.test(address) + ? value + : undefined; + } catch { + return undefined; + } +} + +export function validOperatorString(value: string, format: string): boolean { + if (format === "map-url" || format === "map-tile-url") + return parseOperatorUrl(value, format === "map-tile-url") !== undefined; + if (format === "resource-region") + return ( + value.length <= 512 && + value !== "." && + !value.includes("..") && + /^[a-z][a-z0-9_-]*(?:\/[a-z][a-z0-9_-]*)*$/.test(value) + ); + if (format === "ollama-duration" && value === "0") return true; + const match = /^(\d+(?:\.\d+)?)(s|m|h)$/.exec(value); + if (!match) return false; + const seconds = Number(match[1]) * { s: 1, m: 60, h: 3600 }[match[2] as "s" | "m" | "h"]; + if (!Number.isFinite(seconds)) return false; + if (format === "ollama-duration") return seconds > 0 && seconds <= 86400; + // The pinned Photon parser accepts integer units, not fractional durations. + if (format === "photon-duration") + return !match[1].includes(".") && seconds >= 3600 && seconds <= 8760 * 3600; + return false; +} + +/** Only these built-in numeric controls support explicit reset to the manifest. */ +export const SERVICE_RESOURCE_FIELDS = [ + "OPENMAPX_MEMORY_MIB", + "OPENMAPX_CPU_LIMIT", + "OPENMAPX_JAVA_HEAP_MIB", +] as const; diff --git a/packages/core/src/services/__tests__/runtime-resources.test.ts b/packages/core/src/services/__tests__/runtime-resources.test.ts new file mode 100644 index 000000000..f3c760f1f --- /dev/null +++ b/packages/core/src/services/__tests__/runtime-resources.test.ts @@ -0,0 +1,173 @@ +import { readdirSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { renderServiceSnippet } from "../compose-renderer"; +import { resolveServiceConfigFromEnv } from "../config-resolver"; +import { + assertTrustedConfigurationSchema, + validateTrustedConfigurationValues, +} from "../configuration-schema"; +import type { LoadedService, ServiceManifest } from "../types"; + +const servicesDir = resolve(import.meta.dirname, "../../../../../services"); +const expectedIds = + "app-api app-web brouter data-manager dawarich-app dawarich-postgis dawarich-redis dawarich-sidekiq elasticsearch local-ai martin motis motis-feed-proxy motis-staging nominatim ops-agent osrm otp overpass pelias pelias-pip pelias-placeholder photon postgis redis tileserver traefik transitous-runner valhalla well-known" + .split(" ") + .sort(); +function manifest(id: string): ServiceManifest { + return JSON.parse(readFileSync(resolve(servicesDir, id, "service.json"), "utf8")); +} +function render(id: string, values: Record, isBuiltIn = true) { + const service: LoadedService = { + manifest: manifest(id), + directory: resolve(servicesDir, id), + isBuiltIn, + enabled: true, + }; + return renderServiceSnippet(service, { + existsSync: () => true, + resolvedServiceConfigs: new Map([[id, values]]), + }); +} +describe("operator runtime resources", () => { + it("covers the exact built-in inventory with bounded optional resource fields", () => { + expect( + readdirSync(servicesDir) + .filter((id) => { + try { + manifest(id); + return true; + } catch { + return false; + } + }) + .sort(), + ).toEqual(expectedIds); + for (const id of expectedIds) { + expect(() => assertTrustedConfigurationSchema(manifest(id).configSchema), id).not.toThrow(); + const props = manifest(id).configSchema?.properties as Record; + expect(props?.OPENMAPX_MEMORY_MIB, id).toMatchObject({ + type: "integer", + minimum: 16, + maximum: 1048576, + }); + expect(props?.OPENMAPX_CPU_LIMIT, id).toMatchObject({ + type: "number", + minimum: 0.125, + maximum: 256, + }); + } + expect(manifest("valhalla").configSchema?.properties).toHaveProperty("build_elevation"); + expect(manifest("dawarich-app").configSchema?.properties).toHaveProperty("DATABASE_PASSWORD"); + }); + it("renders resource limits instead of environment, preserving unoverridden defaults and GPU", () => { + expect( + render("local-ai", { OPENMAPX_MEMORY_MIB: 4096, OPENMAPX_CPU_LIMIT: 1.5 }).deploy?.resources, + ).toMatchObject({ limits: { memory: "4096m", cpus: "1.5" } }); + expect(render("local-ai", {}).deploy?.resources?.limits?.memory).toBe("${LOCAL_AI_MEMORY:-8g}"); + expect(render("local-ai", { OPENMAPX_MEMORY_MIB: 4096 }).environment).not.toHaveProperty( + "OPENMAPX_MEMORY_MIB", + ); + expect(render("redis", {}).environment).toEqual(manifest("redis").container.environment); + }); + it.each([0, 15, 1048577, 16.5, "32", NaN, Infinity])( + "refuses invalid memory %s at the render boundary", + (value) => { + expect(() => render("redis", { OPENMAPX_MEMORY_MIB: value })).toThrow(); + }, + ); + it.each([0, 0.124, 257, "2", NaN])("refuses invalid CPU %s", (value) => { + expect(() => render("redis", { OPENMAPX_CPU_LIMIT: value })).toThrow(); + }); + it("refuses community resource escalation regardless of its claimed built-in id", () => { + expect(() => render("redis", { OPENMAPX_MEMORY_MIB: 4096 }, false)).toThrow(); + }); + it("coordinates both MOTIS slots and the data-manager capacity projection", () => { + const values = new Map([["motis", { OPENMAPX_MEMORY_MIB: 32768, OPENMAPX_CPU_LIMIT: 8 }]]); + for (const id of ["motis", "motis-staging"]) + expect( + renderServiceSnippet( + { manifest: manifest(id), directory: "/fake", isBuiltIn: true, enabled: true }, + { existsSync: () => true, resolvedServiceConfigs: values }, + ).deploy?.resources?.limits, + ).toEqual({ memory: "32768m", cpus: "8" }); + expect( + renderServiceSnippet( + { manifest: manifest("data-manager"), directory: "/fake", isBuiltIn: true, enabled: true }, + { + existsSync: () => true, + allServices: expectedIds.map((id) => ({ + manifest: manifest(id), + directory: resolve(servicesDir, id), + isBuiltIn: true, + enabled: true, + })), + resolvedServiceConfigs: values, + }, + ).environment, + ).toMatchObject({ MOTIS_SLOT_MEMORY_GB: "32", MOTIS_SLOT_CPU: "8" }); + expect(() => render("motis-staging", { OPENMAPX_MEMORY_MIB: 4096 })).toThrow(/primary/); + }); + it("synthesizes only finite Elasticsearch heap flags and capitalizes Valhalla booleans", () => { + expect( + render("elasticsearch", { OPENMAPX_JAVA_HEAP_MIB: 1024 }).environment?.ES_JAVA_OPTS, + ).toBe("-Xms1024m -Xmx1024m"); + expect( + render("elasticsearch", { OPENMAPX_JAVA_HEAP_MIB: 1024 }).environment, + ).not.toHaveProperty("OPENMAPX_JAVA_HEAP_MIB"); + expect( + render("valhalla", { serve_tiles: false, build_elevation: true }).environment, + ).toMatchObject({ serve_tiles: "False", build_elevation: "True" }); + expect(() => + render("elasticsearch", { OPENMAPX_JAVA_HEAP_MIB: "1024 -javaagent:x" }), + ).toThrow(); + }); + it("owns host-backed resource expressions and parses SERVICE numeric intent strictly", () => { + expect( + resolveServiceConfigFromEnv(manifest("local-ai"), { LOCAL_AI_MEMORY: "16g" }) + .OPENMAPX_MEMORY_MIB, + ).toMatchObject({ source: "env" }); + expect( + resolveServiceConfigFromEnv(manifest("redis"), { SERVICE_REDIS_OPENMAPX_MEMORY_MIB: "4096" }) + .OPENMAPX_MEMORY_MIB, + ).toMatchObject({ source: "env", value: 4096 }); + expect(() => + resolveServiceConfigFromEnv(manifest("redis"), { + SERVICE_REDIS_OPENMAPX_MEMORY_MIB: "garbage", + }), + ).toThrow(); + }); + it("validates supported settings and URL forms with the same trusted snapshot grammar", () => { + const schema = manifest("app-web").configSchema; + expect( + validateTrustedConfigurationValues( + { + NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL: "/local-dem/tiles.json", + NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE: "/traffic/{z}/{x}/{y}.png", + NEXT_PUBLIC_TRAFFIC_MIN_ZOOM: 8, + }, + schema, + ), + ).toBe(true); + for (const value of [ + "//evil.test/x", + "https://u:p@evil.test/x", + "/x\\y", + "/x/{unsupported}", + "/x\n", + ]) + expect( + validateTrustedConfigurationValues( + { NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE: value }, + schema, + ), + value, + ).toBe(false); + expect(validateTrustedConfigurationValues({ NEXT_PUBLIC_TILES_URL: "/x/{z}" }, schema)).toBe( + false, + ); + expect(validateTrustedConfigurationValues({ NEXT_PUBLIC_TRAFFIC_MIN_ZOOM: 25 }, schema)).toBe( + false, + ); + }); +}); diff --git a/packages/core/src/services/compose-renderer.ts b/packages/core/src/services/compose-renderer.ts index 9b1a550c8..64603f8b1 100644 --- a/packages/core/src/services/compose-renderer.ts +++ b/packages/core/src/services/compose-renderer.ts @@ -11,6 +11,11 @@ import { readExternalProxyNetwork, } from "./external-proxy"; import { detectConsumesCycle } from "./resolver"; +import { + isRuntimeResourceKey, + resolveServiceRuntimeResources, + runtimeResourceNumber, +} from "./runtime-resources"; import { assertRenderSandbox } from "./sandbox-policy"; import type { HardlinkEntry, @@ -400,7 +405,7 @@ export interface ComposeServiceSnippet { logging?: { driver: string; options?: Record }; deploy?: { resources?: { - limits?: { memory?: string }; + limits?: { memory?: string; cpus?: string }; reservations?: { devices?: Array<{ driver: string; count: number | "all"; capabilities: string[] }>; }; @@ -458,6 +463,7 @@ function writeServiceConfigEnvReferences( const prefix = serviceConfigEnvPrefix(m.id); const env = snippet.environment ?? {}; for (const key of envKeys) { + if (isRuntimeResourceKey(key)) throw Error("Resource settings require validated scalar values"); if (!fields.has(key)) { throw new Error( `Service "${m.id}": env-sourced config key "${key}" is not a non-secret configSchema field`, @@ -561,15 +567,61 @@ export function renderServiceSnippet( // Resolved values win over manifest defaults (that's the whole point of // admin/env config). Keys with `undefined`/`null` values are skipped so a // partial config map doesn't blank out manifest defaults. - const resolved = ctx.resolvedServiceConfigs?.get(m.id); + const ownResolved = ctx.resolvedServiceConfigs?.get(m.id); + if ( + service.isBuiltIn && + m.id === "motis-staging" && + ["OPENMAPX_MEMORY_MIB", "OPENMAPX_CPU_LIMIT"].some((key) => ownResolved?.[key] !== undefined) + ) + throw Error("Configure the primary MOTIS slot to coordinate both resource limits"); + const motis = ctx.resolvedServiceConfigs?.get("motis"); + const resolved = + service.isBuiltIn && m.id === "motis-staging" + ? { + ...ownResolved, + ...Object.fromEntries( + Object.entries(motis ?? {}).filter( + ([key]) => key === "OPENMAPX_MEMORY_MIB" || key === "OPENMAPX_CPU_LIMIT", + ), + ), + } + : ownResolved; if (resolved) { const env = snippet.environment ?? {}; for (const [key, value] of Object.entries(resolved)) { if (value === undefined || value === null) continue; - env[key] = String(value); + if (isRuntimeResourceKey(key)) { + if (!service.isBuiltIn) throw Error("Community resource override forbidden"); + if (key === "OPENMAPX_JAVA_HEAP_MIB") { + if (m.id !== "elasticsearch") throw Error("Unsupported heap override"); + const heap = runtimeResourceNumber(key, value); + env.ES_JAVA_OPTS = `-Xms${heap}m -Xmx${heap}m`; + } + continue; + } + env[key] = + m.id === "valhalla" && service.isBuiltIn && typeof value === "boolean" + ? value + ? "True" + : "False" + : String(value); } if (Object.keys(env).length > 0) snippet.environment = env; } + // Keep the transit preflight's declared envelope consistent with both slots. + // Its profile minimums, headroom and runtime checks remain authoritative. + if (service.isBuiltIn && m.id === "data-manager" && motis) { + const env = snippet.environment ?? {}; + if (motis.OPENMAPX_MEMORY_MIB !== undefined) + env.MOTIS_SLOT_MEMORY_GB = String( + runtimeResourceNumber("OPENMAPX_MEMORY_MIB", motis.OPENMAPX_MEMORY_MIB) / 1024, + ); + if (motis.OPENMAPX_CPU_LIMIT !== undefined) + env.MOTIS_SLOT_CPU = String( + runtimeResourceNumber("OPENMAPX_CPU_LIMIT", motis.OPENMAPX_CPU_LIMIT), + ); + snippet.environment = env; + } // Vault secrets (the container track): mount each as a Docker secret at // `/run/secrets/` and point `_FILE` at it. The value never enters // the environment (which is exposed via `docker inspect`/`/proc/environ`); @@ -730,9 +782,10 @@ export function renderServiceSnippet( } if (c.logging) snippet.logging = c.logging; - if (c.memory || c.gpu) { + const limits = resolveServiceRuntimeResources(m, resolved ?? {}); + if (limits.memory || limits.cpus || c.gpu) { const resources: NonNullable["resources"]> = {}; - if (c.memory) resources.limits = { memory: c.memory }; + if (limits.memory || limits.cpus) resources.limits = limits; if (c.gpu) { resources.reservations = { devices: [{ driver: c.gpu.driver, count: c.gpu.count, capabilities: c.gpu.capabilities }], diff --git a/packages/core/src/services/config-resolver.ts b/packages/core/src/services/config-resolver.ts index 34b5e1d3b..5bbc1374e 100644 --- a/packages/core/src/services/config-resolver.ts +++ b/packages/core/src/services/config-resolver.ts @@ -13,6 +13,11 @@ // `SERVICE_VALHALLA_MEMORYLIMIT=2g` — or, for more readable env var names, // the config key should already be upper_snake. +import { + isRuntimeResourceKey, + memoryHostVariable, + runtimeResourceNumber, +} from "./runtime-resources"; import type { ServiceManifest } from "./types"; export type ConfigSource = "default" | "database" | "vault" | "env"; @@ -20,6 +25,8 @@ export type ConfigSource = "default" | "database" | "vault" | "env"; export interface ConfigValueWithSource { value: unknown; source: ConfigSource; + /** Legacy host resource ownership is displayed, but its original Compose expression is retained. */ + manifestExpression?: true; } /** @@ -72,7 +79,7 @@ export function configSchemaKeys( */ export function resolveServiceConfigFromEnv( manifest: Pick & { - container?: Pick; + container?: Pick; }, env: NodeJS.ProcessEnv = process.env, ): Record { @@ -88,6 +95,12 @@ export function resolveServiceConfigFromEnv( result[key] = { value: defVal, source: "default" }; } + // A manifest-owned host interpolation remains authoritative over database values. + for (const { key } of keys) { + const variable = memoryHostVariable(manifest.container?.environment?.[key]); + if (variable && env[variable] !== undefined && env[variable] !== "") + result[key] = { value: env[variable], source: "env", manifestExpression: true }; + } const prefix = serviceConfigEnvPrefix(manifest.id); const known = new Set(keys.map((k) => k.key)); const upperToKey = new Map(); @@ -101,9 +114,24 @@ export function resolveServiceConfigFromEnv( const rest = envKey.slice(prefix.length); const canonical = upperToKey.get(rest); if (!canonical) continue; - result[canonical] = { value: envVal, source: "env" }; + const value = isRuntimeResourceKey(canonical) + ? runtimeResourceNumber( + canonical, + /^[0-9]+(?:\.[0-9]+)?$/.test(envVal) ? Number(envVal) : NaN, + ) + : manifest.id === "valhalla" && ["True", "False", "true", "false"].includes(envVal) + ? envVal.toLowerCase() === "true" + : envVal; + result[canonical] = { value, source: "env" }; } + const hostMemory = memoryHostVariable(manifest.container?.memory); + if (!result.OPENMAPX_MEMORY_MIB && hostMemory && env[hostMemory]) + result.OPENMAPX_MEMORY_MIB = { + value: env[hostMemory], + source: "env", + manifestExpression: true, + }; return result; } @@ -117,7 +145,7 @@ export function flattenResolvedConfig( ): Record { const out: Record = {}; for (const [key, entry] of Object.entries(resolved)) { - out[key] = entry.value; + if (!entry.manifestExpression) out[key] = entry.value; } return out; } diff --git a/packages/core/src/services/configuration-schema.ts b/packages/core/src/services/configuration-schema.ts index e25d259b8..a6e9235cf 100644 --- a/packages/core/src/services/configuration-schema.ts +++ b/packages/core/src/services/configuration-schema.ts @@ -1,3 +1,5 @@ +import { OPERATOR_STRING_FORMATS, validOperatorString } from "../configuration-values"; + const FAILED = "Trusted configuration schema rejected"; const MAX_SCHEMA_DEPTH = 16; const MAX_SCHEMA_NODES = 4_096; @@ -156,7 +158,10 @@ function assertSchemaNode( rejected(); if (schema.pattern !== undefined && typeof schema.pattern !== "string") rejected(); if (typeof schema.pattern === "string") safePattern(schema.pattern); - if (schema.format !== undefined && !["url", "password"].includes(schema.format as string)) + if ( + schema.format !== undefined && + !["url", "password", ...OPERATOR_STRING_FORMATS].includes(schema.format as string) + ) rejected(); if ( schema.additionalProperties !== undefined && @@ -268,6 +273,12 @@ function matchesSchema( if (typeof schema.maxLength === "number" && value.length > schema.maxLength) return false; if (typeof schema.pattern === "string" && !safePattern(schema.pattern).test(value)) return false; + if ( + typeof schema.format === "string" && + (OPERATOR_STRING_FORMATS as readonly string[]).includes(schema.format) && + !validOperatorString(value, schema.format) + ) + return false; if (schema.format === "url" && value !== "") { try { const parsed = new URL(value); diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index eb1c824d6..77e9635d3 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -173,6 +173,12 @@ export { type ResolverContext, resolveRequirement, } from "./resolver"; +export { + isRuntimeResourceKey, + materializeServiceConfigKey, + memoryHostVariable, + resolveServiceRuntimeResources, +} from "./runtime-resources"; export { assertRenderSandbox, COMMUNITY_SAFE_CAPS, diff --git a/packages/core/src/services/runtime-resources.ts b/packages/core/src/services/runtime-resources.ts new file mode 100644 index 000000000..ec2300544 --- /dev/null +++ b/packages/core/src/services/runtime-resources.ts @@ -0,0 +1,50 @@ +import type { ServiceManifest } from "./types"; + +export { SERVICE_RESOURCE_FIELDS as RUNTIME_RESOURCE_KEYS } from "../configuration-values"; + +import { SERVICE_RESOURCE_FIELDS } from "../configuration-values"; +export function isRuntimeResourceKey(key: string): boolean { + return (SERVICE_RESOURCE_FIELDS as readonly string[]).includes(key); +} +export function runtimeResourceNumber(key: string, value: unknown): number { + const [minimum, maximum, integer] = + key === "OPENMAPX_CPU_LIMIT" + ? [0.125, 256, false] + : key === "OPENMAPX_JAVA_HEAP_MIB" + ? [128, 262144, true] + : [16, 1048576, true]; + if ( + typeof value !== "number" || + !Number.isFinite(value) || + value < minimum || + value > maximum || + (integer && !Number.isInteger(value)) + ) + throw Error(`Invalid ${key}`); + return value; +} +export function resolveServiceRuntimeResources( + manifest: ServiceManifest, + values: Record, +): { memory?: string; cpus?: string } { + const memory = values.OPENMAPX_MEMORY_MIB; + const cpu = values.OPENMAPX_CPU_LIMIT; + return { + ...(memory === undefined + ? manifest.container.memory + ? { memory: manifest.container.memory } + : {} + : { memory: `${runtimeResourceNumber("OPENMAPX_MEMORY_MIB", memory)}m` }), + ...(cpu === undefined + ? {} + : { cpus: String(runtimeResourceNumber("OPENMAPX_CPU_LIMIT", cpu)) }), + }; +} +/** These public, typed scalars need rendering adapters rather than raw Compose references. */ +export function materializeServiceConfigKey(serviceId: string, key: string): boolean { + return isRuntimeResourceKey(key) || serviceId === "valhalla"; +} +/** Only a manifest-declared resource expression can establish legacy host ownership. */ +export function memoryHostVariable(memory: string | undefined): string | undefined { + return memory?.match(/^\$\{([A-Z][A-Z0-9_]*)(?::[-?][^}]*)?\}$/)?.[1]; +} diff --git a/packages/extension-contracts/src/service-policy.ts b/packages/extension-contracts/src/service-policy.ts index 3d0c59352..904539ed0 100644 --- a/packages/extension-contracts/src/service-policy.ts +++ b/packages/extension-contracts/src/service-policy.ts @@ -61,6 +61,15 @@ export function checkManifestSandbox(m: ServiceManifest, provenance: ManifestPro errors.push("container.privileged is not allowed for community services"); } if (!hostPrivilegesAllowed) { + const properties = m.configSchema?.properties; + if ( + properties && + typeof properties === "object" && + ["OPENMAPX_MEMORY_MIB", "OPENMAPX_CPU_LIMIT", "OPENMAPX_JAVA_HEAP_MIB"].some( + (key) => key in properties, + ) + ) + errors.push("Reserved platform resource settings are not allowed for community services"); for (const cap of m.container.capAdd ?? []) { if (!COMMUNITY_SAFE_CAPS.has(cap)) { errors.push(`container.capAdd: '${cap}' is not allowed for community services`); diff --git a/services/app-api/service.json b/services/app-api/service.json index 83688e734..8af9b7574 100644 --- a/services/app-api/service.json +++ b/services/app-api/service.json @@ -125,9 +125,18 @@ "memory": "2g", "restart": "unless-stopped", "dependsOn": [ - { "service": "postgis", "condition": "service_healthy" }, - { "service": "redis", "condition": "service_healthy" }, - { "service": "ops-agent", "condition": "service_healthy" } + { + "service": "postgis", + "condition": "service_healthy" + }, + { + "service": "redis", + "condition": "service_healthy" + }, + { + "service": "ops-agent", + "condition": "service_healthy" + } ], "healthcheck": { "type": "http", @@ -210,10 +219,65 @@ "enabled": true, "pathPrefix": "/api", "additionalRoutes": [ - { "path": "/health" }, - { "pathPrefix": "/api/internal", "middleware": ["internal-deny@file"] } + { + "path": "/health" + }, + { + "pathPrefix": "/api/internal", + "middleware": ["internal-deny@file"] + } ] } }, - "ui": { "category": "application" } + "ui": { + "category": "application" + }, + "configSchema": { + "type": "object", + "properties": { + "LOG_LEVEL": { + "type": "string", + "title": "Log level", + "enum": ["fatal", "error", "warn", "info", "debug", "trace", "silent"] + }, + "RATE_LIMIT_IMAGE_PROXY_MAX_BYTES": { + "type": "integer", + "title": "Image proxy bytes per window", + "minimum": 1048576, + "maximum": 10737418240 + }, + "RATE_LIMIT_IMAGE_PROXY_WINDOW_MS": { + "type": "integer", + "title": "Image proxy window (ms)", + "minimum": 1000, + "maximum": 3600000 + }, + "RATE_LIMIT_IMAGE_PROXY_MAX_CONCURRENT_PER_CLIENT": { + "type": "integer", + "title": "Concurrent images per client", + "minimum": 1, + "maximum": 256 + }, + "RATE_LIMIT_IMAGE_PROXY_MAX_CONCURRENT_GLOBAL": { + "type": "integer", + "title": "Concurrent images for instance", + "minimum": 1, + "maximum": 4096 + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/app-web/service.json b/services/app-web/service.json index 81cd6de02..2ce658614 100644 --- a/services/app-web/service.json +++ b/services/app-web/service.json @@ -40,7 +40,12 @@ }, "memory": "512m", "restart": "unless-stopped", - "dependsOn": [{ "service": "app-api", "condition": "service_healthy" }] + "dependsOn": [ + { + "service": "app-api", + "condition": "service_healthy" + } + ] }, "exposure": { "proxy": { @@ -59,5 +64,84 @@ ] } }, - "ui": { "category": "application" } + "ui": { + "category": "application" + }, + "configSchema": { + "type": "object", + "properties": { + "NEXT_PUBLIC_STYLE_PROVIDER": { + "type": "string", + "title": "Map style provider", + "enum": ["openmapx", "maptiler"] + }, + "NEXT_PUBLIC_MAP_STYLE_URL": { + "type": "string", + "title": "Glyph server base URL", + "format": "map-url", + "maxLength": 2048 + }, + "NEXT_PUBLIC_TILES_URL": { + "type": "string", + "title": "Basemap TileJSON URL", + "format": "map-url", + "maxLength": 2048 + }, + "NEXT_PUBLIC_TRAFFIC_TILE_URL_TEMPLATE": { + "type": "string", + "title": "Raster traffic tile URL template", + "format": "map-tile-url", + "maxLength": 2048 + }, + "NEXT_PUBLIC_CYCLOSM_TILE_URL_TEMPLATE": { + "type": "string", + "title": "CyclOSM tile URL template", + "format": "map-tile-url", + "maxLength": 2048 + }, + "NEXT_PUBLIC_TERRAIN_DEM_TILEJSON_URL": { + "type": "string", + "title": "Terrain DEM TileJSON URL", + "format": "map-url", + "maxLength": 2048 + }, + "NEXT_PUBLIC_TERRAIN_CONTOUR_TILEJSON_URL": { + "type": "string", + "title": "Contour TileJSON URL", + "format": "map-url", + "maxLength": 2048 + }, + "NEXT_PUBLIC_TERRAIN_ATTRIBUTION_NAME": { + "type": "string", + "title": "Terrain attribution", + "maxLength": 200 + }, + "NEXT_PUBLIC_TERRAIN_ATTRIBUTION_URL": { + "type": "string", + "title": "Terrain attribution URL", + "format": "map-url", + "maxLength": 2048 + }, + "NEXT_PUBLIC_TRAFFIC_MIN_ZOOM": { + "type": "integer", + "title": "Raster traffic minimum zoom", + "minimum": 0, + "maximum": 24 + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/brouter/service.json b/services/brouter/service.json index 1c52d5a7d..fbe4ca79c 100644 --- a/services/brouter/service.json +++ b/services/brouter/service.json @@ -30,11 +30,29 @@ "retries": 3, "startPeriod": "30m" }, - "logging": { "driver": "json-file", "options": { "max-size": "10m", "max-file": "3" } } + "logging": { + "driver": "json-file", + "options": { + "max-size": "10m", + "max-file": "3" + } + } }, "provides": ["routing-engine"], - "volumes": [{ "name": "openmapx-brouter", "mountAt": "/data/brouter", "backup": false }], - "bindMounts": [{ "source": "scripts", "target": "/opt/openmapx/brouter", "readOnly": true }], + "volumes": [ + { + "name": "openmapx-brouter", + "mountAt": "/data/brouter", + "backup": false + } + ], + "bindMounts": [ + { + "source": "scripts", + "target": "/opt/openmapx/brouter", + "readOnly": true + } + ], "configSchema": { "type": "object", "properties": { @@ -71,9 +89,33 @@ "minimum": 1, "maximum": 300, "default": 30 + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." } } }, - "exposure": { "hostPorts": [{ "container": 17777, "host": 17777, "bindAddress": "127.0.0.1" }] }, - "ui": { "category": "routing" } + "exposure": { + "hostPorts": [ + { + "container": 17777, + "host": 17777, + "bindAddress": "127.0.0.1" + } + ] + }, + "ui": { + "category": "routing" + } } diff --git a/services/data-manager/service.json b/services/data-manager/service.json index fbcb86f9e..c4d8a33df 100644 --- a/services/data-manager/service.json +++ b/services/data-manager/service.json @@ -23,7 +23,7 @@ "PORT": "4000", "HOST": "0.0.0.0", "LOG_LEVEL": "${LOG_LEVEL:-}", - "DATA_MANAGER_AUTH_TOKEN": "${DATA_MANAGER_AUTH_TOKEN:?DATA_MANAGER_AUTH_TOKEN must be set \u2014 generate with: openssl rand -hex 32}", + "DATA_MANAGER_AUTH_TOKEN": "${DATA_MANAGER_AUTH_TOKEN:?DATA_MANAGER_AUTH_TOKEN must be set — generate with: openssl rand -hex 32}", "DATABASE_URL": "postgresql://postgres:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set}@postgis:5432/openmapx", "OPS_AGENT_URL": "${OPS_AGENT_URL:-http://ops-agent:4300}", "OPS_AGENT_TOKEN_FILE": "/run/secrets/ops-agent-data-manager-token", @@ -236,6 +236,43 @@ "title": "Transit feed credentials", "description": "Managed by the dedicated Transit credentials page.", "x-openmapx-secret": true + }, + "LOG_LEVEL": { + "type": "string", + "title": "Log level", + "enum": ["fatal", "error", "warn", "info", "debug", "trace", "silent"] + }, + "OVERTURE_DUCKDB_MEMORY_MB": { + "type": "integer", + "title": "Overture DuckDB memory (MiB)", + "minimum": 256, + "maximum": 1048576 + }, + "OVERTURE_DUCKDB_THREADS": { + "type": "integer", + "title": "Overture DuckDB threads", + "minimum": 1, + "maximum": 64 + }, + "OVERTURE_DUCKDB_TEMP_MB": { + "type": "integer", + "title": "Overture DuckDB temporary disk (MiB)", + "minimum": 1024, + "maximum": 10485760 + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." } } } diff --git a/services/dawarich-app/service.json b/services/dawarich-app/service.json index 5a55be8d2..9836baca0 100644 --- a/services/dawarich-app/service.json +++ b/services/dawarich-app/service.json @@ -60,12 +60,21 @@ "startPeriod": "30s" }, "dependsOn": [ - { "service": "dawarich-postgis", "condition": "service_healthy" }, - { "service": "dawarich-redis", "condition": "service_healthy" } + { + "service": "dawarich-postgis", + "condition": "service_healthy" + }, + { + "service": "dawarich-redis", + "condition": "service_healthy" + } ], "logging": { "driver": "json-file", - "options": { "max-size": "100m", "max-file": "5" } + "options": { + "max-size": "100m", + "max-file": "5" + } } }, "provides": ["timeline"], @@ -83,19 +92,44 @@ "title": "Public Dawarich URL", "default": "https://timeline.{domain}" }, - "DOMAIN": { "type": "string", "title": "Dawarich public hostname" }, + "DOMAIN": { + "type": "string", + "title": "Dawarich public hostname" + }, "APPLICATION_PROTOCOL": { "type": "string", "title": "Dawarich public protocol", "default": "https" }, - "TIME_ZONE": { "type": "string", "title": "Default time zone", "default": "UTC" }, - "REDIS_URL": { "type": "string", "title": "Dawarich Redis URL" }, - "DATABASE_HOST": { "type": "string", "title": "Dawarich database host" }, - "DATABASE_PORT": { "type": "string", "title": "Dawarich database port" }, - "DATABASE_USERNAME": { "type": "string", "title": "Dawarich database user" }, - "DATABASE_NAME": { "type": "string", "title": "Dawarich database name" }, - "OIDC_CLIENT_ID": { "type": "string", "title": "OpenMapX OIDC client ID" }, + "TIME_ZONE": { + "type": "string", + "title": "Default time zone", + "default": "UTC" + }, + "REDIS_URL": { + "type": "string", + "title": "Dawarich Redis URL" + }, + "DATABASE_HOST": { + "type": "string", + "title": "Dawarich database host" + }, + "DATABASE_PORT": { + "type": "string", + "title": "Dawarich database port" + }, + "DATABASE_USERNAME": { + "type": "string", + "title": "Dawarich database user" + }, + "DATABASE_NAME": { + "type": "string", + "title": "Dawarich database name" + }, + "OIDC_CLIENT_ID": { + "type": "string", + "title": "OpenMapX OIDC client ID" + }, "OIDC_ISSUER": { "type": "string", "title": "OpenMapX OIDC issuer", @@ -106,9 +140,18 @@ "title": "Dawarich OIDC callback URL", "default": "https://timeline.{domain}/users/auth/openid_connect/callback" }, - "OIDC_PROVIDER_NAME": { "type": "string", "title": "OIDC provider name" }, - "OIDC_AUTO_REGISTER": { "type": "string", "title": "OIDC automatic registration" }, - "OIDC_PKCE_ENABLED": { "type": "string", "title": "OIDC PKCE requirement" }, + "OIDC_PROVIDER_NAME": { + "type": "string", + "title": "OIDC provider name" + }, + "OIDC_AUTO_REGISTER": { + "type": "string", + "title": "OIDC automatic registration" + }, + "OIDC_PKCE_ENABLED": { + "type": "string", + "title": "OIDC PKCE requirement" + }, "OPENMAPX_DAWARICH_COMMIT": { "type": "string", "title": "Pinned Dawarich collector revision", @@ -155,6 +198,20 @@ "type": "string", "title": "OpenMapX OIDC client secret", "x-openmapx-secret": true + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." } } }, @@ -198,8 +255,13 @@ "exposure": { "proxy": { "enabled": true, - "host": { "default": "timeline.{domain}", "configKey": "APPLICATION_HOSTS" } + "host": { + "default": "timeline.{domain}", + "configKey": "APPLICATION_HOSTS" + } } }, - "ui": { "category": "timeline" } + "ui": { + "category": "timeline" + } } diff --git a/services/dawarich-postgis/service.json b/services/dawarich-postgis/service.json index 906d062f6..15419d6f5 100644 --- a/services/dawarich-postgis/service.json +++ b/services/dawarich-postgis/service.json @@ -51,6 +51,20 @@ "type": "string", "title": "Dawarich database password", "x-openmapx-secret": true + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." } } }, @@ -62,5 +76,7 @@ "backupMode": "pg_dump" } ], - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + } } diff --git a/services/dawarich-redis/service.json b/services/dawarich-redis/service.json index 48230bb9b..901aaa08f 100644 --- a/services/dawarich-redis/service.json +++ b/services/dawarich-redis/service.json @@ -37,5 +37,26 @@ "backup": false } ], - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/dawarich-sidekiq/service.json b/services/dawarich-sidekiq/service.json index e2b709c53..c34f0987b 100644 --- a/services/dawarich-sidekiq/service.json +++ b/services/dawarich-sidekiq/service.json @@ -59,13 +59,25 @@ "startPeriod": "30s" }, "dependsOn": [ - { "service": "dawarich-postgis", "condition": "service_healthy" }, - { "service": "dawarich-redis", "condition": "service_healthy" }, - { "service": "dawarich-app", "condition": "service_healthy" } + { + "service": "dawarich-postgis", + "condition": "service_healthy" + }, + { + "service": "dawarich-redis", + "condition": "service_healthy" + }, + { + "service": "dawarich-app", + "condition": "service_healthy" + } ], "logging": { "driver": "json-file", - "options": { "max-size": "100m", "max-file": "5" } + "options": { + "max-size": "100m", + "max-file": "5" + } } }, "configSchema": { @@ -81,19 +93,44 @@ "title": "Public Dawarich URL", "default": "https://timeline.{domain}" }, - "DOMAIN": { "type": "string", "title": "Dawarich public hostname" }, + "DOMAIN": { + "type": "string", + "title": "Dawarich public hostname" + }, "APPLICATION_PROTOCOL": { "type": "string", "title": "Dawarich public protocol", "default": "https" }, - "TIME_ZONE": { "type": "string", "title": "Default time zone", "default": "UTC" }, - "REDIS_URL": { "type": "string", "title": "Dawarich Redis URL" }, - "DATABASE_HOST": { "type": "string", "title": "Dawarich database host" }, - "DATABASE_PORT": { "type": "string", "title": "Dawarich database port" }, - "DATABASE_USERNAME": { "type": "string", "title": "Dawarich database user" }, - "DATABASE_NAME": { "type": "string", "title": "Dawarich database name" }, - "OIDC_CLIENT_ID": { "type": "string", "title": "OpenMapX OIDC client ID" }, + "TIME_ZONE": { + "type": "string", + "title": "Default time zone", + "default": "UTC" + }, + "REDIS_URL": { + "type": "string", + "title": "Dawarich Redis URL" + }, + "DATABASE_HOST": { + "type": "string", + "title": "Dawarich database host" + }, + "DATABASE_PORT": { + "type": "string", + "title": "Dawarich database port" + }, + "DATABASE_USERNAME": { + "type": "string", + "title": "Dawarich database user" + }, + "DATABASE_NAME": { + "type": "string", + "title": "Dawarich database name" + }, + "OIDC_CLIENT_ID": { + "type": "string", + "title": "OpenMapX OIDC client ID" + }, "OIDC_ISSUER": { "type": "string", "title": "OpenMapX OIDC issuer", @@ -104,9 +141,18 @@ "title": "Dawarich OIDC callback URL", "default": "https://timeline.{domain}/users/auth/openid_connect/callback" }, - "OIDC_PROVIDER_NAME": { "type": "string", "title": "OIDC provider name" }, - "OIDC_AUTO_REGISTER": { "type": "string", "title": "OIDC automatic registration" }, - "OIDC_PKCE_ENABLED": { "type": "string", "title": "OIDC PKCE requirement" }, + "OIDC_PROVIDER_NAME": { + "type": "string", + "title": "OIDC provider name" + }, + "OIDC_AUTO_REGISTER": { + "type": "string", + "title": "OIDC automatic registration" + }, + "OIDC_PKCE_ENABLED": { + "type": "string", + "title": "OIDC PKCE requirement" + }, "OPENMAPX_DAWARICH_COMMIT": { "type": "string", "title": "Pinned Dawarich collector revision", @@ -158,6 +204,20 @@ "type": "string", "title": "OpenMapX OIDC client secret", "x-openmapx-secret": true + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." } } }, @@ -185,5 +245,7 @@ "readOnly": true } ], - "ui": { "category": "timeline" } + "ui": { + "category": "timeline" + } } diff --git a/services/elasticsearch/service.json b/services/elasticsearch/service.json index dee7415ab..c7651bcbe 100644 --- a/services/elasticsearch/service.json +++ b/services/elasticsearch/service.json @@ -41,7 +41,40 @@ } ], "exposure": { - "hostPorts": [{ "container": 9200, "host": 9200, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 9200, + "host": 9200, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "geocoding" } + "ui": { + "category": "geocoding" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_JAVA_HEAP_MIB": { + "type": "integer", + "title": "Java heap (MiB)", + "minimum": 128, + "maximum": 262144 + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/local-ai/service.json b/services/local-ai/service.json index 07f14cd97..2d26a939b 100644 --- a/services/local-ai/service.json +++ b/services/local-ai/service.json @@ -33,9 +33,54 @@ } }, "provides": ["llm-inference"], - "volumes": [{ "name": "openmapx-ollama", "mountAt": "/root/.ollama", "backup": false }], + "volumes": [ + { + "name": "openmapx-ollama", + "mountAt": "/root/.ollama", + "backup": false + } + ], "exposure": { - "hostPorts": [{ "container": 11434, "host": 11434, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 11434, + "host": 11434, + "bindAddress": "127.0.0.1" + } + ] + }, + "ui": { + "category": "infrastructure" }, - "ui": { "category": "infrastructure" } + "configSchema": { + "type": "object", + "properties": { + "OLLAMA_KEEP_ALIVE": { + "type": "string", + "title": "Model keep-alive (0 or duration, at most 24h)", + "format": "ollama-duration", + "maxLength": 32 + }, + "OLLAMA_MAX_LOADED_MODELS": { + "type": "integer", + "title": "Maximum loaded models", + "minimum": 1, + "maximum": 32 + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/martin/service.json b/services/martin/service.json index 1efdb5069..ab100a836 100644 --- a/services/martin/service.json +++ b/services/martin/service.json @@ -22,14 +22,29 @@ }, "command": ["--config", "/etc/martin/martin.yaml"], "restart": "unless-stopped", - "dependsOn": [{ "service": "postgis", "condition": "service_healthy" }] + "dependsOn": [ + { + "service": "postgis", + "condition": "service_healthy" + } + ] }, "provides": ["tile-server"], "bindMounts": [ - { "source": "config/martin.yaml", "target": "/etc/martin/martin.yaml", "readOnly": true } + { + "source": "config/martin.yaml", + "target": "/etc/martin/martin.yaml", + "readOnly": true + } ], "exposure": { - "hostPorts": [{ "container": 3000, "host": 3002, "bindAddress": "127.0.0.1" }], + "hostPorts": [ + { + "container": 3000, + "host": 3002, + "bindAddress": "127.0.0.1" + } + ], "proxy": { "enabled": true, "pathPrefix": "/martin", @@ -43,5 +58,26 @@ ] } }, - "ui": { "category": "tiles" } + "ui": { + "category": "tiles" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/motis-feed-proxy/service.json b/services/motis-feed-proxy/service.json index 2fb9a102b..7d40269fa 100644 --- a/services/motis-feed-proxy/service.json +++ b/services/motis-feed-proxy/service.json @@ -39,5 +39,26 @@ "required": true } ], - "ui": { "category": "transit" } + "ui": { + "category": "transit" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/motis-staging/service.json b/services/motis-staging/service.json index e44b98060..debc1e09b 100644 --- a/services/motis-staging/service.json +++ b/services/motis-staging/service.json @@ -2,7 +2,7 @@ "id": "motis-staging", "name": "MOTIS (staging)", "version": "1.0.0", - "description": "Staging MOTIS instance — rebuild target for the data-manager's Transitous atomic-swap pipeline. Not query-facing; only the primary `motis` service receives application traffic. Opt-in: start with `--services motis-staging` (or add to the persisted selection) when running the data-manager promotion flow.", + "description": "Staging MOTIS instance \u2014 rebuild target for the data-manager's Transitous atomic-swap pipeline. Not query-facing; only the primary `motis` service receives application traffic. Opt-in: start with `--services motis-staging` (or add to the persisted selection) when running the data-manager promotion flow.", "license": "MIT", "quality": "built-in", "homepage": "https://github.com/motis-project/motis", @@ -24,7 +24,12 @@ "command": [ "until [ -f /motis-data/config.yml ]; do echo 'motis-staging: waiting for /motis-data/config.yml'; sleep 5; done; /motis import && /motis server || { echo 'motis-staging: import/serve failed; holding (a pipeline restart will retry)'; while true; do sleep 3600; done; }" ], - "dependsOn": [{ "service": "motis-feed-proxy", "condition": "service_started" }], + "dependsOn": [ + { + "service": "motis-feed-proxy", + "condition": "service_started" + } + ], "memory": "${MOTIS_SLOT_MEMORY:-16g}", "restart": "unless-stopped", "healthcheck": { @@ -38,15 +43,51 @@ }, "logging": { "driver": "json-file", - "options": { "max-size": "50m", "max-file": "3" } + "options": { + "max-size": "50m", + "max-file": "3" + } } }, "provides": ["transit-engine-staging"], "bindMounts": [ - { "source": "@infra:data/motis/staging", "target": "/motis-data", "readOnly": false } + { + "source": "@infra:data/motis/staging", + "target": "/motis-data", + "readOnly": false + } ], "exposure": { - "hostPorts": [{ "container": 8080, "host": 8083, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 8080, + "host": 8083, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "transit" } + "ui": { + "category": "transit" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Configure this resource on primary MOTIS. One generation applies the same limit to both slots and the transit preflight capacity envelope.", + "readOnly": true + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Configure this resource on primary MOTIS. One generation applies the same limit to both slots and the transit preflight capacity envelope.", + "readOnly": true + } + } + } } diff --git a/services/motis/service.json b/services/motis/service.json index 0dd284b95..73f4b8bc3 100644 --- a/services/motis/service.json +++ b/services/motis/service.json @@ -24,7 +24,12 @@ "command": [ "until [ -f /motis-data/config.yml ]; do echo 'motis: waiting for /motis-data/config.yml'; sleep 5; done; /motis import && /motis server || { echo 'motis: import/serve failed; holding (a pipeline restart will retry)'; while true; do sleep 3600; done; }" ], - "dependsOn": [{ "service": "motis-feed-proxy", "condition": "service_started" }], + "dependsOn": [ + { + "service": "motis-feed-proxy", + "condition": "service_started" + } + ], "memory": "${MOTIS_SLOT_MEMORY:-16g}", "restart": "unless-stopped", "healthcheck": { @@ -38,17 +43,56 @@ }, "logging": { "driver": "json-file", - "options": { "max-size": "50m", "max-file": "3" } + "options": { + "max-size": "50m", + "max-file": "3" + } } }, "buildCommand": "openmapx services build motis", "provides": ["transit-engine", "routing-engine"], - "produces": [{ "type": "motis-feed-proxy-config", "sourceDir": "data/motis-feed-proxy/conf" }], + "produces": [ + { + "type": "motis-feed-proxy-config", + "sourceDir": "data/motis-feed-proxy/conf" + } + ], "bindMounts": [ - { "source": "@infra:data/motis/live", "target": "/motis-data", "readOnly": false } + { + "source": "@infra:data/motis/live", + "target": "/motis-data", + "readOnly": false + } ], "exposure": { - "hostPorts": [{ "container": 8080, "host": 8081, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 8080, + "host": 8081, + "bindAddress": "127.0.0.1" + } + ] + }, + "ui": { + "category": "transit" }, - "ui": { "category": "transit" } + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Shared by the primary and staging MOTIS slots; apply the coordinated stack and rerun transit preflight. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Shared by the primary and staging MOTIS slots; apply the coordinated stack and rerun transit preflight. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/nominatim/service.json b/services/nominatim/service.json index 9f7a677a6..ed82127e4 100644 --- a/services/nominatim/service.json +++ b/services/nominatim/service.json @@ -57,10 +57,79 @@ "mountAt": "/var/lib/postgresql/16/main", "backup": false }, - { "name": "openmapx-nominatim-flatnode", "mountAt": "/nominatim/flatnode", "backup": false } + { + "name": "openmapx-nominatim-flatnode", + "mountAt": "/nominatim/flatnode", + "backup": false + } ], "exposure": { - "hostPorts": [{ "container": 8080, "host": 8088, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 8080, + "host": 8088, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "geocoding" } + "ui": { + "category": "geocoding" + }, + "configSchema": { + "type": "object", + "properties": { + "THREADS": { + "type": "integer", + "title": "Import threads", + "minimum": 1, + "maximum": 256 + }, + "REPLICATION_UPDATE_INTERVAL": { + "type": "integer", + "title": "Replication interval (seconds)", + "minimum": 60, + "maximum": 604800 + }, + "IMPORT_STYLE": { + "type": "string", + "title": "Import style", + "enum": ["full", "admin", "street", "address"], + "description": "Import-time setting. Requires rebuilding the Nominatim database; applying configuration alone does not rebuild data." + }, + "REVERSE_ONLY": { + "type": "boolean", + "title": "Reverse only", + "description": "Import-time setting. Requires rebuilding the Nominatim database; applying configuration alone does not rebuild data." + }, + "IMPORT_WIKIPEDIA": { + "type": "boolean", + "title": "Import wikipedia", + "description": "Import-time setting. Requires rebuilding the Nominatim database; applying configuration alone does not rebuild data." + }, + "IMPORT_US_POSTCODES": { + "type": "boolean", + "title": "Import us postcodes", + "description": "Import-time setting. Requires rebuilding the Nominatim database; applying configuration alone does not rebuild data." + }, + "IMPORT_GB_POSTCODES": { + "type": "boolean", + "title": "Import gb postcodes", + "description": "Import-time setting. Requires rebuilding the Nominatim database; applying configuration alone does not rebuild data." + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/ops-agent/service.json b/services/ops-agent/service.json index 4dbf93b9f..e14d87241 100644 --- a/services/ops-agent/service.json +++ b/services/ops-agent/service.json @@ -86,7 +86,7 @@ "readOnly": false }, { - "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set \u2014 ops-agent renders and applies trusted local stack configuration}", + "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set — ops-agent renders and applies trusted local stack configuration}", "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}", "readOnly": false }, @@ -139,5 +139,24 @@ ], "ui": { "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } } } diff --git a/services/osrm/service.json b/services/osrm/service.json index 492aac029..fbf4236be 100644 --- a/services/osrm/service.json +++ b/services/osrm/service.json @@ -23,10 +23,49 @@ }, "buildCommand": "openmapx services build osrm", "provides": ["routing-engine"], - "produces": [{ "type": "osrm-graph", "sourceDir": "data/osrm-graph" }], - "consumes": [{ "type": "osrm-graph", "mountAt": "/data", "readOnly": true, "required": true }], + "produces": [ + { + "type": "osrm-graph", + "sourceDir": "data/osrm-graph" + } + ], + "consumes": [ + { + "type": "osrm-graph", + "mountAt": "/data", + "readOnly": true, + "required": true + } + ], "exposure": { - "hostPorts": [{ "container": 5000, "host": 5000, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 5000, + "host": 5000, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "routing" } + "ui": { + "category": "routing" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/otp/service.json b/services/otp/service.json index c3cc40320..fd00fd384 100644 --- a/services/otp/service.json +++ b/services/otp/service.json @@ -18,20 +18,66 @@ "digest": "sha256:8d54e5c589186707ee365417f2202dc878c451fa3001b8edff07019531100933", "expose": [8080], "command": ["--load", "--serve"], - "environment": { "JAVA_TOOL_OPTIONS": "-Xmx12g" }, + "environment": { + "JAVA_TOOL_OPTIONS": "-Xmx12g" + }, "memory": "16g", "restart": "unless-stopped" }, "buildCommand": "openmapx services build otp", "provides": ["transit-engine"], - "produces": [{ "type": "otp-graph", "sourceDir": "data/otp-graph" }], - "consumes": [{ "type": "otp-graph", "mountAt": "/var/opentripplanner", "required": true }], + "produces": [ + { + "type": "otp-graph", + "sourceDir": "data/otp-graph" + } + ], + "consumes": [ + { + "type": "otp-graph", + "mountAt": "/var/opentripplanner", + "required": true + } + ], "bindMounts": [ - { "source": "config/build-config.json", "target": "/var/opentripplanner/build-config.json" }, - { "source": "config/router-config.json", "target": "/var/opentripplanner/router-config.json" } + { + "source": "config/build-config.json", + "target": "/var/opentripplanner/build-config.json" + }, + { + "source": "config/router-config.json", + "target": "/var/opentripplanner/router-config.json" + } ], "exposure": { - "hostPorts": [{ "container": 8080, "host": 8090, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 8080, + "host": 8090, + "bindAddress": "127.0.0.1" + } + ] + }, + "ui": { + "category": "transit" }, - "ui": { "category": "transit" } + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/overpass/service.json b/services/overpass/service.json index 7be4f8348..30900beb0 100644 --- a/services/overpass/service.json +++ b/services/overpass/service.json @@ -48,14 +48,81 @@ }, "logging": { "driver": "json-file", - "options": { "max-size": "50m", "max-file": "3" } + "options": { + "max-size": "50m", + "max-file": "3" + } } }, "provides": ["osm-query"], - "consumes": [{ "type": "osm-pbf-bz2", "mountAt": "/osm", "readOnly": true, "required": true }], - "bindMounts": [{ "source": "@infra:data/overpass/db", "target": "/db", "readOnly": false }], + "consumes": [ + { + "type": "osm-pbf-bz2", + "mountAt": "/osm", + "readOnly": true, + "required": true + } + ], + "bindMounts": [ + { + "source": "@infra:data/overpass/db", + "target": "/db", + "readOnly": false + } + ], "exposure": { - "hostPorts": [{ "container": 80, "host": 8082, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 80, + "host": 8082, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "osm-query" } + "ui": { + "category": "osm-query" + }, + "configSchema": { + "type": "object", + "properties": { + "OVERPASS_SPACE": { + "type": "integer", + "title": "Maximum query memory (bytes)", + "minimum": 1073741824, + "maximum": 10995116277760 + }, + "OVERPASS_FASTCGI_PROCESSES": { + "type": "integer", + "title": "FastCGI processes", + "minimum": 1, + "maximum": 128 + }, + "OVERPASS_UPDATE_SLEEP": { + "type": "integer", + "title": "Update interval (seconds)", + "minimum": 60, + "maximum": 604800 + }, + "OVERPASS_RULES_LOAD": { + "type": "integer", + "title": "Rules load", + "minimum": 1, + "maximum": 100 + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/pelias-pip/service.json b/services/pelias-pip/service.json index 07d7f726d..263088a31 100644 --- a/services/pelias-pip/service.json +++ b/services/pelias-pip/service.json @@ -16,7 +16,9 @@ "tag": "v5.0.0", "digest": "sha256:6bde49c2a5bd5d6ab80b7ec521500b09ad36c05335db0fa720a96d3fc6e6595a", "expose": [4200], - "environment": { "PELIAS_CONFIG": "/code/pelias.json" }, + "environment": { + "PELIAS_CONFIG": "/code/pelias.json" + }, "restart": "unless-stopped" }, "consumes": [ @@ -27,9 +29,41 @@ "required": true } ], - "bindMounts": [{ "source": "@service:pelias:config/pelias.json", "target": "/code/pelias.json" }], + "bindMounts": [ + { + "source": "@service:pelias:config/pelias.json", + "target": "/code/pelias.json" + } + ], "exposure": { - "hostPorts": [{ "container": 4200, "host": 4200, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 4200, + "host": 4200, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "geocoding" } + "ui": { + "category": "geocoding" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/pelias-placeholder/service.json b/services/pelias-placeholder/service.json index 43af22c64..327502b2c 100644 --- a/services/pelias-placeholder/service.json +++ b/services/pelias-placeholder/service.json @@ -16,7 +16,9 @@ "tag": "v5.11.0", "digest": "sha256:79a8e55b63d3e0b257f4e72a86250c97d0e33d6880d6f83f4d68bb06a9636ebe", "expose": [4100], - "environment": { "PELIAS_CONFIG": "/code/pelias.json" }, + "environment": { + "PELIAS_CONFIG": "/code/pelias.json" + }, "restart": "unless-stopped" }, "consumes": [ @@ -27,9 +29,41 @@ "required": true } ], - "bindMounts": [{ "source": "@service:pelias:config/pelias.json", "target": "/code/pelias.json" }], + "bindMounts": [ + { + "source": "@service:pelias:config/pelias.json", + "target": "/code/pelias.json" + } + ], "exposure": { - "hostPorts": [{ "container": 4100, "host": 4100, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 4100, + "host": 4100, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "geocoding" } + "ui": { + "category": "geocoding" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/pelias/service.json b/services/pelias/service.json index 33e26029d..d4d3d23c2 100644 --- a/services/pelias/service.json +++ b/services/pelias/service.json @@ -17,12 +17,23 @@ "tag": "v7.11.0", "digest": "sha256:7174871a9ecd1343fc0dcb0a0d11848a0b634677de40f9b80ed2ce6b64308080", "expose": [4000], - "environment": { "PELIAS_CONFIG": "/code/pelias.json" }, + "environment": { + "PELIAS_CONFIG": "/code/pelias.json" + }, "restart": "unless-stopped", "dependsOn": [ - { "service": "elasticsearch", "condition": "service_healthy" }, - { "service": "pelias-placeholder", "condition": "service_started" }, - { "service": "pelias-pip", "condition": "service_started" } + { + "service": "elasticsearch", + "condition": "service_healthy" + }, + { + "service": "pelias-placeholder", + "condition": "service_started" + }, + { + "service": "pelias-pip", + "condition": "service_started" + } ] }, "buildCommand": "openmapx services build pelias", @@ -45,12 +56,50 @@ }, "provides": ["geocoder"], "produces": [ - { "type": "pelias-placeholder-data", "sourceDir": "data/pelias/placeholder" }, - { "type": "pelias-whosonfirst-data", "sourceDir": "data/pelias/whosonfirst" } + { + "type": "pelias-placeholder-data", + "sourceDir": "data/pelias/placeholder" + }, + { + "type": "pelias-whosonfirst-data", + "sourceDir": "data/pelias/whosonfirst" + } + ], + "bindMounts": [ + { + "source": "config/pelias.json", + "target": "/code/pelias.json" + } ], - "bindMounts": [{ "source": "config/pelias.json", "target": "/code/pelias.json" }], "exposure": { - "hostPorts": [{ "container": 4000, "host": 4300, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 4000, + "host": 4300, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "geocoding" } + "ui": { + "category": "geocoding" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/photon/service.json b/services/photon/service.json index 929693a51..fcf957754 100644 --- a/services/photon/service.json +++ b/services/photon/service.json @@ -36,9 +36,63 @@ } }, "provides": ["geocoder"], - "bindMounts": [{ "source": "@infra:data/photon", "target": "/photon/data", "readOnly": false }], + "bindMounts": [ + { + "source": "@infra:data/photon", + "target": "/photon/data", + "readOnly": false + } + ], "exposure": { - "hostPorts": [{ "container": 2322, "host": 2322, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 2322, + "host": 2322, + "bindAddress": "127.0.0.1" + } + ] + }, + "ui": { + "category": "geocoding" }, - "ui": { "category": "geocoding" } + "configSchema": { + "type": "object", + "properties": { + "REGION": { + "type": "string", + "title": "Index region", + "format": "resource-region" + }, + "UPDATE_STRATEGY": { + "type": "string", + "title": "Update strategy", + "enum": ["PARALLEL", "SEQUENTIAL"] + }, + "UPDATE_INTERVAL": { + "type": "string", + "title": "Update interval (1h–8760h)", + "format": "photon-duration", + "maxLength": 32 + }, + "LOG_LEVEL": { + "type": "string", + "title": "Log level", + "enum": ["DEBUG", "INFO", "WARN", "ERROR"] + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/postgis/service.json b/services/postgis/service.json index 5fc0d0d5f..53dc30f45 100644 --- a/services/postgis/service.json +++ b/services/postgis/service.json @@ -71,7 +71,42 @@ } ], "exposure": { - "hostPorts": [{ "container": 5432, "host": 5432, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 5432, + "host": 5432, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "POSTGIS_DIAGNOSTICS": { + "type": "boolean", + "title": "Enable diagnostics" + }, + "POSTGIS_TRACK_IO_TIMING": { + "type": "boolean", + "title": "Track I/O timing" + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/redis/service.json b/services/redis/service.json index 19fddfc8c..f5c7afd29 100644 --- a/services/redis/service.json +++ b/services/redis/service.json @@ -42,10 +42,42 @@ } ], "volumes": [ - { "name": "openmapx-redisdata", "mountAt": "/data", "backup": true, "backupMode": "tar" } + { + "name": "openmapx-redisdata", + "mountAt": "/data", + "backup": true, + "backupMode": "tar" + } ], "exposure": { - "hostPorts": [{ "container": 6379, "host": 6379, "bindAddress": "127.0.0.1" }] + "hostPorts": [ + { + "container": 6379, + "host": 6379, + "bindAddress": "127.0.0.1" + } + ] }, - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/tileserver/service.json b/services/tileserver/service.json index 2f41e3852..dc1b96028 100644 --- a/services/tileserver/service.json +++ b/services/tileserver/service.json @@ -30,14 +30,40 @@ } }, "provides": ["tile-server"], - "produces": [{ "type": "tile-mbtiles", "sourceDir": "data/tile-mbtiles" }], + "produces": [ + { + "type": "tile-mbtiles", + "sourceDir": "data/tile-mbtiles" + } + ], "consumes": [ - { "type": "tile-mbtiles", "mountAt": "/data/mbtiles", "readOnly": true, "required": true }, - { "type": "tile-fonts", "mountAt": "/data/fonts", "readOnly": true, "required": true } + { + "type": "tile-mbtiles", + "mountAt": "/data/mbtiles", + "readOnly": true, + "required": true + }, + { + "type": "tile-fonts", + "mountAt": "/data/fonts", + "readOnly": true, + "required": true + } + ], + "bindMounts": [ + { + "source": "config/config.json", + "target": "/data/config.json" + } ], - "bindMounts": [{ "source": "config/config.json", "target": "/data/config.json" }], "exposure": { - "hostPorts": [{ "container": 8080, "host": 8080, "bindAddress": "127.0.0.1" }], + "hostPorts": [ + { + "container": 8080, + "host": 8080, + "bindAddress": "127.0.0.1" + } + ], "proxy": { "enabled": true, "pathPrefix": "/tiles", @@ -45,5 +71,26 @@ "middleware": ["tiles-cache@file"] } }, - "ui": { "category": "tiles" } + "ui": { + "category": "tiles" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/traefik/service.json b/services/traefik/service.json index 24ecfb681..ed62d0e76 100644 --- a/services/traefik/service.json +++ b/services/traefik/service.json @@ -33,15 +33,54 @@ } ], "bindMounts": [ - { "source": "config/traefik.yml", "target": "/etc/traefik/traefik.yml", "readOnly": true }, - { "source": "config/dynamic", "target": "/etc/traefik/dynamic", "readOnly": true } + { + "source": "config/traefik.yml", + "target": "/etc/traefik/traefik.yml", + "readOnly": true + }, + { + "source": "config/dynamic", + "target": "/etc/traefik/dynamic", + "readOnly": true + } ], "exposure": { "hostPorts": [ - { "container": 80, "host": 80 }, - { "container": 443, "host": 443 }, - { "container": 443, "host": 443, "protocol": "udp" } + { + "container": 80, + "host": 80 + }, + { + "container": 443, + "host": 443 + }, + { + "container": 443, + "host": 443, + "protocol": "udp" + } ] }, - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/transitous-runner/service.json b/services/transitous-runner/service.json index d438b5dda..574b1b2c6 100644 --- a/services/transitous-runner/service.json +++ b/services/transitous-runner/service.json @@ -70,5 +70,26 @@ "optional": true } ], - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } diff --git a/services/valhalla/service.json b/services/valhalla/service.json index 0a1d8d4aa..d9feae6f6 100644 --- a/services/valhalla/service.json +++ b/services/valhalla/service.json @@ -75,6 +75,32 @@ "type": "boolean", "default": true, "title": "Build admin boundaries" + }, + "serve_tiles": { + "type": "boolean", + "title": "Serve tiles" + }, + "build_time_zones": { + "type": "boolean", + "title": "Build time zones" + }, + "use_tiles_ignore_pbf": { + "type": "boolean", + "title": "Use tiles ignore pbf" + }, + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." } } }, diff --git a/services/well-known/service.json b/services/well-known/service.json index ed45cd68a..b621640b0 100644 --- a/services/well-known/service.json +++ b/services/well-known/service.json @@ -20,8 +20,14 @@ "restart": "unless-stopped" }, "bindMounts": [ - { "source": "config/html", "target": "/usr/share/nginx/html/.well-known" }, - { "source": "config/nginx.conf", "target": "/etc/nginx/conf.d/default.conf" } + { + "source": "config/html", + "target": "/usr/share/nginx/html/.well-known" + }, + { + "source": "config/nginx.conf", + "target": "/etc/nginx/conf.d/default.conf" + } ], "exposure": { "proxy": { @@ -29,5 +35,26 @@ "pathPrefix": "/.well-known" } }, - "ui": { "category": "infrastructure" } + "ui": { + "category": "infrastructure" + }, + "configSchema": { + "type": "object", + "properties": { + "OPENMAPX_MEMORY_MIB": { + "type": "integer", + "title": "Container memory limit (MiB)", + "minimum": 16, + "maximum": 1048576, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + }, + "OPENMAPX_CPU_LIMIT": { + "type": "number", + "title": "Container CPU limit", + "minimum": 0.125, + "maximum": 256, + "description": "Optional Docker resource limit. Unset preserves the image manifest limit. Clear the field to restore the manifest limit." + } + } + } } From 6b1160ec9f489974f04afc60f72f1256132f68d6 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 01:26:35 +0100 Subject: [PATCH 062/135] fix(config): require registry provenance for host interpolation --- .../__tests__/service-config-resolver.test.ts | 2 ++ apps/api/src/services/configuration-input.ts | 1 + .../services/configuration-ownership-view.ts | 1 + .../src/services/dawarich/managed-resolver.ts | 1 + .../api/src/services/dawarich/provisioning.ts | 1 + .../src/services/prospective-configuration.ts | 1 + .../src/services/service-config-resolver.ts | 8 ++++-- packages/cli/src/commands/compose.ts | 4 +-- .../configuration-source-witness.test.ts | 27 +++++++++++++++++++ .../configuration-source-witness.ts | 5 ++-- .../__tests__/runtime-resources.test.ts | 23 ++++++++++++++-- packages/core/src/services/config-resolver.ts | 15 +++++++++-- .../src/services/configuration-generation.ts | 4 ++- 13 files changed, 82 insertions(+), 11 deletions(-) diff --git a/apps/api/src/services/__tests__/service-config-resolver.test.ts b/apps/api/src/services/__tests__/service-config-resolver.test.ts index f0ad5bb7c..7fda9be10 100644 --- a/apps/api/src/services/__tests__/service-config-resolver.test.ts +++ b/apps/api/src/services/__tests__/service-config-resolver.test.ts @@ -165,6 +165,7 @@ describe("typed render settings and host ownership", () => { selectLimitMock.mockResolvedValueOnce([{ config: { OPENMAPX_MEMORY_MIB: 1024 } }]); const input = { id: "test", + isBuiltIn: true, configSchema: { properties: { OPENMAPX_MEMORY_MIB: { type: "integer", minimum: 16, maximum: 1048576 } }, }, @@ -187,6 +188,7 @@ describe("typed render settings and host ownership", () => { selectLimitMock.mockResolvedValueOnce([{ config: { THREADS: 4 } }]); const input = { id: "test", + isBuiltIn: true, configSchema: { properties: { THREADS: { type: "integer" } } }, containerEnv: { THREADS: "${SERVICE_TEST_ORIGINAL_THREADS:-2}" }, }; diff --git a/apps/api/src/services/configuration-input.ts b/apps/api/src/services/configuration-input.ts index fd40658c7..66d58ce50 100644 --- a/apps/api/src/services/configuration-input.ts +++ b/apps/api/src/services/configuration-input.ts @@ -40,6 +40,7 @@ export async function buildConfigurationInput( configSchema: service.manifest.configSchema, containerEnv: service.manifest.container.environment, containerMemory: service.manifest.container.memory, + isBuiltIn: service.isBuiltIn, proxyHostConfigKey: service.manifest.exposure?.proxy?.host?.configKey, })), context, diff --git a/apps/api/src/services/configuration-ownership-view.ts b/apps/api/src/services/configuration-ownership-view.ts index ea0841943..cce49c9d4 100644 --- a/apps/api/src/services/configuration-ownership-view.ts +++ b/apps/api/src/services/configuration-ownership-view.ts @@ -120,6 +120,7 @@ export async function serviceOwnershipSnapshot( configSchema: service.manifest.configSchema, containerEnv: service.manifest.container.environment, containerMemory: service.manifest.container.memory, + isBuiltIn: service.isBuiltIn, }, context, ), diff --git a/apps/api/src/services/dawarich/managed-resolver.ts b/apps/api/src/services/dawarich/managed-resolver.ts index 57cdf0171..98ab71b84 100644 --- a/apps/api/src/services/dawarich/managed-resolver.ts +++ b/apps/api/src/services/dawarich/managed-resolver.ts @@ -295,6 +295,7 @@ async function readConfig(serviceId: string): Promise> { configSchema: service.manifest.configSchema, containerEnv: service.manifest.container.environment, containerMemory: service.manifest.container.memory, + isBuiltIn: service.isBuiltIn, }); } diff --git a/apps/api/src/services/dawarich/provisioning.ts b/apps/api/src/services/dawarich/provisioning.ts index f6b238084..4db33af0d 100644 --- a/apps/api/src/services/dawarich/provisioning.ts +++ b/apps/api/src/services/dawarich/provisioning.ts @@ -691,6 +691,7 @@ async function readEffectiveServiceConfig(serviceId: string): Promise; containerMemory?: string; + /** Registry provenance, never the manifest quality label. */ + isBuiltIn?: boolean; /** * The config key naming the service's proxy host (`exposure.proxy.host.configKey`). * The host is routed by Traefik labels, which a Compose env reference @@ -89,6 +91,7 @@ export async function resolveServiceConfigWithSources( container: { environment: manifest.containerEnv, memory: manifest.containerMemory }, }, process.env, + { firstParty: manifest.isBuiltIn === true }, ); const result: Record = {}; @@ -165,7 +168,7 @@ export async function resolveAllServiceConfigs( const flat: Record = {}; const fromEnv: string[] = []; for (const [key, entry] of Object.entries(resolved)) { - if (entry.manifestExpression) continue; + if (entry.manifestExpression && key !== m.proxyHostConfigKey) continue; if ( entry.source === "env" && key !== m.proxyHostConfigKey && @@ -209,6 +212,7 @@ export async function resolveServiceConfigWithSourcesStrict( container: { environment: manifest.containerEnv, memory: manifest.containerMemory }, }, context.env, + { firstParty: manifest.isBuiltIn === true }, ); const result: Record = {}; for (const [key, entry] of Object.entries(envLayer)) @@ -234,7 +238,7 @@ export async function resolveAllServiceConfigsStrict( const flat: Record = {}; const fromEnv: string[] = []; for (const [key, entry] of Object.entries(resolved)) { - if (entry.manifestExpression) continue; + if (entry.manifestExpression && key !== manifest.proxyHostConfigKey) continue; if ( entry.source === "env" && key !== manifest.proxyHostConfigKey && diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index 52615d291..724f73b21 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -280,7 +280,7 @@ export async function renderComposeWithWriter( const { id } = service.manifest; const proxyHostKey = service.manifest.exposure?.proxy?.host?.configKey; const fromEnv = Object.entries( - resolveServiceConfigFromEnv(service.manifest, process.env), + resolveServiceConfigFromEnv(service.manifest, process.env, { firstParty: service.isBuiltIn }), ).filter(([, entry]) => entry.source === "env"); // An env value is a reference in the YAML, except the proxy host, which // the routes need as a value. @@ -302,7 +302,7 @@ export async function renderComposeWithWriter( ); for (const [key, entry] of fromEnv) if ( - !entry.manifestExpression && + (!entry.manifestExpression || key === proxyHostKey) && (key === proxyHostKey || coreServices.materializeServiceConfigKey(id, key)) ) values[key] = entry.value as string | number | boolean; diff --git a/packages/core/src/build-artifacts/configuration-source-witness.test.ts b/packages/core/src/build-artifacts/configuration-source-witness.test.ts index 577edc13c..3594c237e 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.test.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.test.ts @@ -11,6 +11,7 @@ import { } from "../services/release-manifest"; import type { LoadedService } from "../services/types"; import { + configurationHostFields, observeConfigurationSources, verifyConfigurationSourceWitness, } from "./configuration-source-witness"; @@ -261,3 +262,29 @@ it("projects only validated non-secret platform ownership under the original rev closeControlInstallation(installation); } }); + +it("does not assign foreign host ownership from community interpolation", () => { + const service: LoadedService = { + manifest: { + id: "community-example", + name: "Example", + version: "1", + quality: "community", + configSchema: { type: "object", properties: { PUBLIC_VALUE: { type: "string" } } }, + container: { + image: "example/community", + tag: "1", + environment: { PUBLIC_VALUE: "${DATABASE_URL}" }, + }, + }, + directory: "/synthetic", + isBuiltIn: false, + enabled: true, + }; + const result = configurationHostFields([service], { DATABASE_URL: "synthetic-platform-secret" }); + expect(result[0]?.fields.find((field) => field.key === "PUBLIC_VALUE")).toEqual({ + key: "PUBLIC_VALUE", + source: "none", + }); + expect(JSON.stringify(result)).not.toContain("synthetic-platform-secret"); +}); diff --git a/packages/core/src/build-artifacts/configuration-source-witness.ts b/packages/core/src/build-artifacts/configuration-source-witness.ts index 848be580c..fa923a443 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.ts @@ -166,7 +166,7 @@ export function configurationHostFields( "motis-staging": "MOTIS_SLOT_MEMORY", postgis: "POSTGIS_MEMORY", }; - return services.map(({ manifest }) => { + return services.map(({ manifest, isBuiltIn }) => { const schema = (manifest.configSchema?.properties ?? manifest.configSchema ?? {}) as Record< string, unknown @@ -187,9 +187,10 @@ export function configurationHostFields( const hostExpression = manifest.container.environment?.[key]?.match( /^\$\{([A-Z][A-Z0-9_]*)(?::[-?][^}]*)?\}$/, )?.[1]; - if (hostExpression) names.push(hostExpression); + if (isBuiltIn && hostExpression) names.push(hostExpression); const variable = memory[manifest.id]; if ( + isBuiltIn && key === "OPENMAPX_MEMORY_MIB" && variable && manifest.container.memory?.startsWith(`\${${variable}`) diff --git a/packages/core/src/services/__tests__/runtime-resources.test.ts b/packages/core/src/services/__tests__/runtime-resources.test.ts index f3c760f1f..10abccb07 100644 --- a/packages/core/src/services/__tests__/runtime-resources.test.ts +++ b/packages/core/src/services/__tests__/runtime-resources.test.ts @@ -122,10 +122,29 @@ describe("operator runtime resources", () => { render("elasticsearch", { OPENMAPX_JAVA_HEAP_MIB: "1024 -javaagent:x" }), ).toThrow(); }); + it("does not interpret a community manifest's environment as host-variable authority", () => { + const untrusted = { + id: "community-example", + quality: "built-in", + firstParty: true, + configSchema: { + type: "object", + properties: { PUBLIC_VALUE: { type: "string", default: "safe-default" } }, + }, + container: { environment: { PUBLIC_VALUE: "${DATABASE_URL}" } }, + }; + const resolved = resolveServiceConfigFromEnv(untrusted, { + DATABASE_URL: "synthetic-platform-secret", + }); + expect(JSON.stringify(resolved)).not.toContain("synthetic-platform-secret"); + }); it("owns host-backed resource expressions and parses SERVICE numeric intent strictly", () => { expect( - resolveServiceConfigFromEnv(manifest("local-ai"), { LOCAL_AI_MEMORY: "16g" }) - .OPENMAPX_MEMORY_MIB, + resolveServiceConfigFromEnv( + manifest("local-ai"), + { LOCAL_AI_MEMORY: "16g" }, + { firstParty: true }, + ).OPENMAPX_MEMORY_MIB, ).toMatchObject({ source: "env" }); expect( resolveServiceConfigFromEnv(manifest("redis"), { SERVICE_REDIS_OPENMAPX_MEMORY_MIB: "4096" }) diff --git a/packages/core/src/services/config-resolver.ts b/packages/core/src/services/config-resolver.ts index 5bbc1374e..52d14cab8 100644 --- a/packages/core/src/services/config-resolver.ts +++ b/packages/core/src/services/config-resolver.ts @@ -82,6 +82,7 @@ export function resolveServiceConfigFromEnv( container?: Pick; }, env: NodeJS.ProcessEnv = process.env, + provenance: { firstParty?: boolean } = {}, ): Record { const keys = configSchemaKeys(manifest.configSchema); if (keys.length === 0) return {}; @@ -98,7 +99,12 @@ export function resolveServiceConfigFromEnv( // A manifest-owned host interpolation remains authoritative over database values. for (const { key } of keys) { const variable = memoryHostVariable(manifest.container?.environment?.[key]); - if (variable && env[variable] !== undefined && env[variable] !== "") + if ( + provenance.firstParty === true && + variable && + env[variable] !== undefined && + env[variable] !== "" + ) result[key] = { value: env[variable], source: "env", manifestExpression: true }; } const prefix = serviceConfigEnvPrefix(manifest.id); @@ -126,7 +132,12 @@ export function resolveServiceConfigFromEnv( } const hostMemory = memoryHostVariable(manifest.container?.memory); - if (!result.OPENMAPX_MEMORY_MIB && hostMemory && env[hostMemory]) + if ( + provenance.firstParty === true && + !result.OPENMAPX_MEMORY_MIB && + hostMemory && + env[hostMemory] + ) result.OPENMAPX_MEMORY_MIB = { value: env[hostMemory], source: "env", diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index d0d8b9253..e249fff31 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -383,7 +383,9 @@ function resolveInput( const entry = saved.get(service.manifest.id); const envKeys = new Set(entry?.envKeys ?? []); const defaults = Object.entries( - flattenResolvedConfig(resolveServiceConfigFromEnv(service.manifest, {})), + flattenResolvedConfig( + resolveServiceConfigFromEnv(service.manifest, {}, { firstParty: service.isBuiltIn }), + ), ).filter(([key]) => !envKeys.has(key)); const values = { ...Object.fromEntries(defaults), ...entry?.values }; if (Object.keys(values).length > 0) serviceConfigs.set(service.manifest.id, values); From d21836c9760e84ad71d747806772d16e1221adf8 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 01:37:47 +0100 Subject: [PATCH 063/135] feat(platform): retain actor-bound configuration previews --- .../0051_platform_configuration_preview.sql | 22 + .../src/db/migrations/meta/0051_snapshot.json | 9290 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + .../platform-configuration-preview-schema.ts | 43 + apps/api/src/db/schema.ts | 1 + .../platform-preview-postgres.test.ts | 172 + .../src/services/platform-configuration.ts | 50 + .../src/services/platform-preview-store.ts | 130 + .../admin/shared/SchemaConfigForm.tsx | 8 +- .../SchemaConfigForm.ownership.test.tsx | 21 + ...-30-admin-protected-release-mode-design.md | 37 + .../src/api/contracts/service-admin/index.ts | 2 + .../api/contracts/service-admin/platform.ts | 66 + .../__tests__/platform-preview.test.ts | 159 + .../core/src/services/compose-renderer.ts | 17 + packages/core/src/services/index.ts | 4 + .../core/src/services/platform-preview.ts | 137 + scripts/database-test-groups.json | 3 +- 18 files changed, 10166 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/db/migrations/0051_platform_configuration_preview.sql create mode 100644 apps/api/src/db/migrations/meta/0051_snapshot.json create mode 100644 apps/api/src/db/platform-configuration-preview-schema.ts create mode 100644 apps/api/src/services/__tests__/platform-preview-postgres.test.ts create mode 100644 apps/api/src/services/platform-configuration.ts create mode 100644 apps/api/src/services/platform-preview-store.ts create mode 100644 docs/superpowers/specs/2026-10-30-admin-protected-release-mode-design.md create mode 100644 packages/core/src/api/contracts/service-admin/platform.ts create mode 100644 packages/core/src/services/__tests__/platform-preview.test.ts create mode 100644 packages/core/src/services/platform-preview.ts diff --git a/apps/api/src/db/migrations/0051_platform_configuration_preview.sql b/apps/api/src/db/migrations/0051_platform_configuration_preview.sql new file mode 100644 index 000000000..02a3fcb16 --- /dev/null +++ b/apps/api/src/db/migrations/0051_platform_configuration_preview.sql @@ -0,0 +1,22 @@ +CREATE TABLE "platform_configuration_preview" ( + "id" uuid PRIMARY KEY NOT NULL, + "actor_id" text NOT NULL, + "operation_key" text NOT NULL, + "request_fingerprint" text NOT NULL, + "preview_fingerprint" text NOT NULL, + "auth_policy_revision" text NOT NULL, + "source_revision" uuid NOT NULL, + "expected_current_revision_id" text, + "preview" jsonb NOT NULL, + "consumed_by_job_id" uuid, + "consumed_at" timestamp with time zone, + "retired_at" timestamp with time zone, + "expires_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "platform_preview_bounds" CHECK (length("platform_configuration_preview"."operation_key") between 1 and 128 and "platform_configuration_preview"."request_fingerprint" ~ '^[a-f0-9]{64}$' and "platform_configuration_preview"."preview_fingerprint" ~ '^[a-f0-9]{64}$' and "platform_configuration_preview"."auth_policy_revision" ~ '^[a-f0-9]{64}$' and octet_length("platform_configuration_preview"."preview"::text) <= 131072 and jsonb_typeof("platform_configuration_preview"."preview") = 'object' and "platform_configuration_preview"."expires_at" > "platform_configuration_preview"."created_at" and "platform_configuration_preview"."expires_at" <= "platform_configuration_preview"."created_at" + interval '10 minutes' and (("platform_configuration_preview"."consumed_at" is null) = ("platform_configuration_preview"."consumed_by_job_id" is null)) and ("platform_configuration_preview"."retired_at" is null or ("platform_configuration_preview"."consumed_at" is not null and "platform_configuration_preview"."retired_at" >= "platform_configuration_preview"."consumed_at"))) +); +--> statement-breakpoint +ALTER TABLE "platform_configuration_preview" ADD CONSTRAINT "platform_configuration_preview_actor_id_user_id_fk" FOREIGN KEY ("actor_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "platform_preview_actor_key_uq" ON "platform_configuration_preview" USING btree ("actor_id","operation_key");--> statement-breakpoint +CREATE UNIQUE INDEX "platform_preview_fingerprint_uq" ON "platform_configuration_preview" USING btree ("preview_fingerprint");--> statement-breakpoint +CREATE INDEX "platform_preview_expiry_idx" ON "platform_configuration_preview" USING btree ("expires_at"); \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0051_snapshot.json b/apps/api/src/db/migrations/meta/0051_snapshot.json new file mode 100644 index 000000000..be47b88a3 --- /dev/null +++ b/apps/api/src/db/migrations/meta/0051_snapshot.json @@ -0,0 +1,9290 @@ +{ + "id": "9a25f282-f0dc-428e-a3a0-9a79d9d3d9e9", + "prevId": "4f0db974-6157-45b7-9478-54bd16a30cc7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_action_receipt": { + "name": "account_action_receipt", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mail_requested": { + "name": "mail_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_action_actor_time_idx": { + "name": "account_action_actor_time_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_action_target_time_idx": { + "name": "account_action_target_time_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_action_receipt_actor_id_user_id_fk": { + "name": "account_action_receipt_actor_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "account_action_receipt_target_id_user_id_fk": { + "name": "account_action_receipt_target_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_admin_guard": { + "name": "account_admin_guard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "active_count": { + "name": "active_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_admin_guard_singleton": { + "name": "account_admin_guard_singleton", + "value": "\"account_admin_guard\".\"id\"=1" + }, + "account_admin_guard_nonnegative": { + "name": "account_admin_guard_nonnegative", + "value": "\"account_admin_guard\".\"active_count\">=0" + } + }, + "isRLSEnabled": false + }, + "public.account_mail_outbox": { + "name": "account_mail_outbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_mail_action_kind_uq": { + "name": "account_mail_action_kind_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_mail_pending_idx": { + "name": "account_mail_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_mail_outbox_request_key_account_action_receipt_key_fk": { + "name": "account_mail_outbox_request_key_account_action_receipt_key_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "account_action_receipt", + "columnsFrom": [ + "request_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "account_mail_outbox_user_id_user_id_fk": { + "name": "account_mail_outbox_user_id_user_id_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_mail_state_check": { + "name": "account_mail_state_check", + "value": "\"account_mail_outbox\".\"state\" in ('queued','sending','sent','failed','unknown')" + }, + "account_mail_kind_check": { + "name": "account_mail_kind_check", + "value": "\"account_mail_outbox\".\"kind\" in ('verification','password_setup')" + } + }, + "isRLSEnabled": false + }, + "public.admin_action_reauthentication": { + "name": "admin_action_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admin_reauth_actor_expiry_idx": { + "name": "admin_reauth_actor_expiry_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admin_reauth_request_uq": { + "name": "admin_reauth_request_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_action_reauthentication_actor_id_user_id_fk": { + "name": "admin_action_reauthentication_actor_id_user_id_fk", + "tableFrom": "admin_action_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "admin_reauth_state_check": { + "name": "admin_reauth_state_check", + "value": "\"admin_action_reauthentication\".\"state\" in ('pending','completed','consumed','superseded','expired')" + }, + "admin_reauth_purpose_check": { + "name": "admin_reauth_purpose_check", + "value": "\"admin_action_reauthentication\".\"purpose\" in ('account.verify','account.password.set','account.reset.send','account.invite','credentials.export','credentials.import','transit.credentials','platform.domain.apply')" + } + }, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_application_actor_id_user_id_fk": { + "name": "auth_policy_application_actor_id_user_id_fk", + "tableFrom": "auth_policy_application", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_state_activated_by_user_id_fk": { + "name": "auth_policy_state_activated_by_user_id_fk", + "tableFrom": "auth_policy_state", + "tableTo": "user", + "columnsFrom": [ + "activated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "public.data_export_artifact": { + "name": "data_export_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation_id": { + "name": "generation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'assembling'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'application/zip'" + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "download_count": { + "name": "download_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_export_artifact_request_state_idx": { + "name": "data_export_artifact_request_state_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_artifact_expiry_idx": { + "name": "data_export_artifact_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_artifact_request_id_data_subject_request_id_fk": { + "name": "data_export_artifact_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_artifact", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_export_artifact_storage_key_unique": { + "name": "data_export_artifact_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_export_artifact_download_count_check": { + "name": "data_export_artifact_download_count_check", + "value": "\"data_export_artifact\".\"download_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_backup_review": { + "name": "data_subject_request_backup_review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "platform_version": { + "name": "platform_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason_code": { + "name": "reason_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewed_by": { + "name": "reviewed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "data_subject_request_backup_review_request_backup_idx": { + "name": "data_subject_request_backup_review_request_backup_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_backup_review_manifest_idx": { + "name": "data_subject_request_backup_review_manifest_idx", + "columns": [ + { + "expression": "manifest_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_backup_review_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_backup_review_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_backup_review_reviewed_by_user_id_fk": { + "name": "data_subject_request_backup_review_reviewed_by_user_id_fk", + "tableFrom": "data_subject_request_backup_review", + "tableTo": "user", + "columnsFrom": [ + "reviewed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_backup_review_decision_check": { + "name": "data_subject_request_backup_review_decision_check", + "value": "\"data_subject_request_backup_review\".\"decision\" in ('not_applicable', 'no_material_difference', 'extract', 'unavailable')" + } + }, + "isRLSEnabled": false + }, + "public.data_export_reauthentication": { + "name": "data_export_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_user_id": { + "name": "initiating_admin_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "initiating_admin_session_id": { + "name": "initiating_admin_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_channel": { + "name": "delivery_channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'self_service'" + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_export_reauthentication_user_expiry_idx": { + "name": "data_export_reauthentication_user_expiry_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_artifact_state_idx": { + "name": "data_export_reauthentication_artifact_state_idx", + "columns": [ + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_export_reauthentication_active_idx": { + "name": "data_export_reauthentication_active_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_export_reauthentication_request_id_data_subject_request_id_fk": { + "name": "data_export_reauthentication_request_id_data_subject_request_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_artifact_id_data_export_artifact_id_fk": { + "name": "data_export_reauthentication_artifact_id_data_export_artifact_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "data_export_artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_export_reauthentication_user_id_user_id_fk": { + "name": "data_export_reauthentication_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_user_id_user_id_fk": { + "name": "data_export_reauthentication_initiating_admin_user_id_user_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "initiating_admin_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_export_reauthentication_initiating_admin_session_id_session_id_fk": { + "name": "data_export_reauthentication_initiating_admin_session_id_session_id_fk", + "tableFrom": "data_export_reauthentication", + "tableTo": "session", + "columnsFrom": [ + "initiating_admin_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_export_reauthentication_state_check": { + "name": "data_export_reauthentication_state_check", + "value": "\"data_export_reauthentication\".\"state\" in ('pending', 'completed', 'consumed', 'superseded', 'expired')" + }, + "data_export_reauthentication_channel_check": { + "name": "data_export_reauthentication_channel_check", + "value": "\"data_export_reauthentication\".\"delivery_channel\" in ('self_service', 'assisted', 'representative')" + }, + "data_export_reauthentication_method_check": { + "name": "data_export_reauthentication_method_check", + "value": "\"data_export_reauthentication\".\"completed_method\" is null or \"data_export_reauthentication\".\"completed_method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_maintenance_uq": { + "name": "data_manager_jobs_configuration_maintenance_uq", + "columns": [ + { + "expression": "(\"metadata\"->'configurationMaintenance'->'receipt'->>'maintenanceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_manager_jobs_configuration_guard_uq": { + "name": "data_manager_jobs_configuration_guard_uq", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"kind\" = 'configuration-maintenance' AND coalesce(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState','recovery_required') <> 'settled'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + }, + "data_manager_jobs_configuration_guard_shape": { + "name": "data_manager_jobs_configuration_guard_shape", + "value": "\"data_manager\".\"jobs\".\"kind\" <> 'configuration-maintenance' OR coalesce((jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance') = 'object' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'version' = '1' AND \"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->>'guardState' IN ('preparing','prepared','settling','settled','recovery_required') AND jsonb_typeof(\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance'->'receipt') = 'object' AND octet_length((\"data_manager\".\"jobs\".\"metadata\"->'configurationMaintenance')::text) <= 65536), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_approval": { + "name": "data_subject_request_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approver_user_id": { + "name": "approver_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approver_role": { + "name": "approver_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "findings_digest": { + "name": "findings_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_approval_scope_idx": { + "name": "data_subject_request_approval_scope_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_approval_idempotency_idx": { + "name": "data_subject_request_approval_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_approval\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_approval_approver_user_id_user_id_fk": { + "name": "data_subject_request_approval_approver_user_id_user_id_fk", + "tableFrom": "data_subject_request_approval", + "tableTo": "user", + "columnsFrom": [ + "approver_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_approval_scope_check": { + "name": "data_subject_request_approval_scope_check", + "value": "\"data_subject_request_approval\".\"scope\" in ('legal-content', 'dsar-process', 'security-review')" + }, + "data_subject_request_approval_decision_check": { + "name": "data_subject_request_approval_decision_check", + "value": "\"data_subject_request_approval\".\"decision\" in ('approved', 'rejected')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_state": { + "name": "identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refusal_reason": { + "name": "refusal_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_state": { + "name": "delivery_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_delivered'" + }, + "withdrawal_at": { + "name": "withdrawal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "protected_notes_envelope": { + "name": "protected_notes_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_locator_digest_idx": { + "name": "data_subject_request_locator_digest_idx", + "columns": [ + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" not in ('withdrawn', 'refused', 'closed', 'delivered', 'artifact_expired')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('received', 'identity_pending', 'preserving', 'collecting', 'pending_processor', 'operator_review', 'assembling', 'ready', 'delivered', 'artifact_expired', 'clarification_needed', 'withdrawn', 'refused', 'closed')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_attachment": { + "name": "data_subject_request_attachment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rights_review_state": { + "name": "rights_review_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_attachment_request_idx": { + "name": "data_subject_request_attachment_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_attachment_expiry_idx": { + "name": "data_subject_request_attachment_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_attachment_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_attachment_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_attachment_owner_id_user_id_fk": { + "name": "data_subject_request_attachment_owner_id_user_id_fk", + "tableFrom": "data_subject_request_attachment", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_attachment_storage_key_unique": { + "name": "data_subject_request_attachment_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_attachment_purpose_check": { + "name": "data_subject_request_attachment_purpose_check", + "value": "\"data_subject_request_attachment\".\"purpose\" in ('identity_evidence', 'representative_authority', 'processor_response', 'operator_supplement')" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_email_challenge": { + "name": "data_subject_request_email_challenge", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "identity_id": { + "name": "identity_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'identity_verification'" + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_source": { + "name": "recipient_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "recipient_envelope": { + "name": "recipient_envelope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_digest": { + "name": "recipient_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_digest": { + "name": "code_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_key_version": { + "name": "code_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_attempts": { + "name": "delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_delivery_attempts": { + "name": "max_delivery_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_delivery_attempt_at": { + "name": "next_delivery_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error_code": { + "name": "last_delivery_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_email_challenge_request_idx": { + "name": "data_subject_request_email_challenge_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_identity_idx": { + "name": "data_subject_request_email_challenge_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_delivery_idx": { + "name": "data_subject_request_email_challenge_delivery_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_delivery_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_expiry_idx": { + "name": "data_subject_request_email_challenge_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_email_challenge_active_identity_idx": { + "name": "data_subject_request_email_challenge_active_identity_idx", + "columns": [ + { + "expression": "identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_email_challenge_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_email_challenge_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk": { + "name": "data_subject_request_email_challenge_identity_id_data_subject_request_identity_id_fk", + "tableFrom": "data_subject_request_email_challenge", + "tableTo": "data_subject_request_identity", + "columnsFrom": [ + "identity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_email_challenge_purpose_check": { + "name": "data_subject_request_email_challenge_purpose_check", + "value": "\"data_subject_request_email_challenge\".\"purpose\" = 'identity_verification'" + }, + "data_subject_request_email_challenge_party_check": { + "name": "data_subject_request_email_challenge_party_check", + "value": "\"data_subject_request_email_challenge\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_email_challenge_recipient_source_check": { + "name": "data_subject_request_email_challenge_recipient_source_check", + "value": "\"data_subject_request_email_challenge\".\"recipient_source\" in ('live_account', 'request_locator', 'representative_contact')" + }, + "data_subject_request_email_challenge_state_check": { + "name": "data_subject_request_email_challenge_state_check", + "value": "\"data_subject_request_email_challenge\".\"state\" in ('queued', 'sending', 'issued', 'retryable', 'failed', 'revoked', 'consumed', 'expired')" + }, + "data_subject_request_email_challenge_attempts_check": { + "name": "data_subject_request_email_challenge_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_attempts\" > 0 and \"data_subject_request_email_challenge\".\"attempts\" <= \"data_subject_request_email_challenge\".\"max_attempts\"" + }, + "data_subject_request_email_challenge_delivery_attempts_check": { + "name": "data_subject_request_email_challenge_delivery_attempts_check", + "value": "\"data_subject_request_email_challenge\".\"delivery_attempts\" >= 0 and \"data_subject_request_email_challenge\".\"max_delivery_attempts\" > 0 and \"data_subject_request_email_challenge\".\"delivery_attempts\" <= \"data_subject_request_email_challenge\".\"max_delivery_attempts\"" + }, + "data_subject_request_email_challenge_code_check": { + "name": "data_subject_request_email_challenge_code_check", + "value": "(\"data_subject_request_email_challenge\".\"code_digest\" is null and \"data_subject_request_email_challenge\".\"code_key_version\" is null) or (\"data_subject_request_email_challenge\".\"code_digest\" is not null and \"data_subject_request_email_challenge\".\"code_key_version\" > 0)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_identity": { + "name": "data_subject_request_identity", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contact_envelope": { + "name": "contact_envelope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_digest": { + "name": "contact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reasonable_doubt_code": { + "name": "reasonable_doubt_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_attachment_id": { + "name": "evidence_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "authority_state": { + "name": "authority_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'not_applicable'" + }, + "authority_attachment_id": { + "name": "authority_attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "delivery_authorized": { + "name": "delivery_authorized", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "verified_by": { + "name": "verified_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_identity_party_idx": { + "name": "data_subject_request_identity_party_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "party", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_identity_state_idx": { + "name": "data_subject_request_identity_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authority_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_identity_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_identity_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_evidence_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "evidence_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk": { + "name": "data_subject_request_identity_authority_attachment_id_data_subject_request_attachment_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "data_subject_request_attachment", + "columnsFrom": [ + "authority_attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_identity_verified_by_user_id_fk": { + "name": "data_subject_request_identity_verified_by_user_id_fk", + "tableFrom": "data_subject_request_identity", + "tableTo": "user", + "columnsFrom": [ + "verified_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_identity_party_check": { + "name": "data_subject_request_identity_party_check", + "value": "\"data_subject_request_identity\".\"party\" in ('subject', 'representative')" + }, + "data_subject_request_identity_state_check": { + "name": "data_subject_request_identity_state_check", + "value": "\"data_subject_request_identity\".\"state\" in ('pending', 'verified', 'failed', 'clarification')" + }, + "data_subject_request_identity_method_check": { + "name": "data_subject_request_identity_method_check", + "value": "\"data_subject_request_identity\".\"method\" is null or \"data_subject_request_identity\".\"method\" in ('account_login', 'verified_email_challenge', 'exceptional_evidence')" + }, + "data_subject_request_identity_authority_check": { + "name": "data_subject_request_identity_authority_check", + "value": "\"data_subject_request_identity\".\"authority_state\" in ('not_applicable', 'pending', 'approved', 'rejected')" + }, + "data_subject_request_identity_delivery_check": { + "name": "data_subject_request_identity_delivery_check", + "value": "\"data_subject_request_identity\".\"delivery_authorized\" in (0, 1) and (\"data_subject_request_identity\".\"delivery_authorized\" = 0 or (\"data_subject_request_identity\".\"party\" = 'representative' and \"data_subject_request_identity\".\"authority_state\" = 'approved' and \"data_subject_request_identity\".\"state\" = 'verified'))" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'clarification', 'extension', 'ready', 'delivered', 'closed', 'refused', 'escalation_due_soon', 'escalation_overdue', 'escalation_identity', 'escalation_extension', 'escalation_stuck')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_digest": { + "name": "locator_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_locator_idx": { + "name": "data_subject_request_preservation_locator_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_source_snapshot": { + "name": "data_subject_request_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'captured'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'subject-record-jsonl-v1'" + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_bytes": { + "name": "plaintext_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "encrypted_bytes": { + "name": "encrypted_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "plaintext_sha256": { + "name": "plaintext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext_sha256": { + "name": "ciphertext_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cipher_version": { + "name": "cipher_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "aad_version": { + "name": "aad_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "master_key_version": { + "name": "master_key_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delete_attempts": { + "name": "delete_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_delete_attempt_at": { + "name": "last_delete_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_source_snapshot_request_registration_idx": { + "name": "data_subject_request_source_snapshot_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_source_snapshot_expiry_idx": { + "name": "data_subject_request_source_snapshot_expiry_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_source_snapshot_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_source_snapshot", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "data_subject_request_source_snapshot_storage_key_unique": { + "name": "data_subject_request_source_snapshot_storage_key_unique", + "nullsNotDistinct": false, + "columns": [ + "storage_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "data_subject_request_source_snapshot_state_check": { + "name": "data_subject_request_source_snapshot_state_check", + "value": "\"data_subject_request_source_snapshot\".\"state\" in ('captured', 'delete_failed', 'deleted')" + }, + "data_subject_request_source_snapshot_format_check": { + "name": "data_subject_request_source_snapshot_format_check", + "value": "\"data_subject_request_source_snapshot\".\"format\" = 'subject-record-jsonl-v1'" + }, + "data_subject_request_source_snapshot_counts_check": { + "name": "data_subject_request_source_snapshot_counts_check", + "value": "\"data_subject_request_source_snapshot\".\"record_count\" >= 0 and \"data_subject_request_source_snapshot\".\"plaintext_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"encrypted_bytes\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" >= 0 and \"data_subject_request_source_snapshot\".\"delete_attempts\" <= 5" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_key": { + "name": "task_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "collector_id": { + "name": "collector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "collector_version": { + "name": "collector_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "assigned_to": { + "name": "assigned_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cutoff_at": { + "name": "cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collected_at": { + "name": "collected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_detail": { + "name": "encrypted_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exception_code": { + "name": "exception_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redaction_code": { + "name": "redaction_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_key_idx": { + "name": "data_subject_request_task_request_key_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "task_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_registration_idx": { + "name": "data_subject_request_task_registration_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_task_assigned_to_user_id_fk": { + "name": "data_subject_request_task_assigned_to_user_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "user", + "columnsFrom": [ + "assigned_to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_attempts_check": { + "name": "data_subject_request_task_attempts_check", + "value": "\"data_subject_request_task\".\"attempts\" >= 0" + }, + "data_subject_request_task_required_check": { + "name": "data_subject_request_task_required_check", + "value": "\"data_subject_request_task\".\"required\" in (0, 1)" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installation_kind": { + "name": "installation_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'catalog'" + }, + "provenance": { + "name": "provenance", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_operation_id": { + "name": "last_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "installed_extension_installation_kind_check": { + "name": "installed_extension_installation_kind_check", + "value": "\"installed_extension\".\"installation_kind\" in ('catalog','standalone')" + }, + "installed_extension_standalone_provenance_check": { + "name": "installed_extension_standalone_provenance_check", + "value": "\"installed_extension\".\"installation_kind\" <> 'standalone' or coalesce((\"installed_extension\".\"source_trust\" = 'community' and \"installed_extension\".\"id\" ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof(\"installed_extension\".\"provenance\") = 'object' and \"installed_extension\".\"provenance\"->>'kind' in ('artifact','source','uploaded-artifact','legacy_unknown') and \"installed_extension\".\"provenance\"->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)" + } + }, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "encryption_mode": { + "name": "encryption_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private_jwk": { + "name": "private_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.platform_configuration_preview": { + "name": "platform_configuration_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "preview_fingerprint": { + "name": "preview_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_revision": { + "name": "source_revision", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expected_current_revision_id": { + "name": "expected_current_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "consumed_by_job_id": { + "name": "consumed_by_job_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "platform_preview_actor_key_uq": { + "name": "platform_preview_actor_key_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "platform_preview_fingerprint_uq": { + "name": "platform_preview_fingerprint_uq", + "columns": [ + { + "expression": "preview_fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "platform_preview_expiry_idx": { + "name": "platform_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "platform_configuration_preview_actor_id_user_id_fk": { + "name": "platform_configuration_preview_actor_id_user_id_fk", + "tableFrom": "platform_configuration_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "platform_preview_bounds": { + "name": "platform_preview_bounds", + "value": "length(\"platform_configuration_preview\".\"operation_key\") between 1 and 128 and \"platform_configuration_preview\".\"request_fingerprint\" ~ '^[a-f0-9]{64}$' and \"platform_configuration_preview\".\"preview_fingerprint\" ~ '^[a-f0-9]{64}$' and \"platform_configuration_preview\".\"auth_policy_revision\" ~ '^[a-f0-9]{64}$' and octet_length(\"platform_configuration_preview\".\"preview\"::text) <= 131072 and jsonb_typeof(\"platform_configuration_preview\".\"preview\") = 'object' and \"platform_configuration_preview\".\"expires_at\" > \"platform_configuration_preview\".\"created_at\" and \"platform_configuration_preview\".\"expires_at\" <= \"platform_configuration_preview\".\"created_at\" + interval '10 minutes' and ((\"platform_configuration_preview\".\"consumed_at\" is null) = (\"platform_configuration_preview\".\"consumed_by_job_id\" is null)) and (\"platform_configuration_preview\".\"retired_at\" is null or (\"platform_configuration_preview\".\"consumed_at\" is not null and \"platform_configuration_preview\".\"retired_at\" >= \"platform_configuration_preview\".\"consumed_at\"))" + } + }, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_independent_password_login": { + "name": "origin_independent_password_login", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transit_credential_preview": { + "name": "transit_credential_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_ciphertext": { + "name": "authorization_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_iv": { + "name": "authorization_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_tag": { + "name": "authorization_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "credentials_revision": { + "name": "credentials_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "transit_credential_preview_actor_key_uq": { + "name": "transit_credential_preview_actor_key_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transit_credential_preview_expiry_idx": { + "name": "transit_credential_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "transit_credential_preview_actor_id_user_id_fk": { + "name": "transit_credential_preview_actor_id_user_id_fk", + "tableFrom": "transit_credential_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "transit_credential_preview_bounds": { + "name": "transit_credential_preview_bounds", + "value": "octet_length(\"transit_credential_preview\".\"ciphertext\") <= 1500000 and length(\"transit_credential_preview\".\"intent_fingerprint\") = 64 and length(\"transit_credential_preview\".\"iv\") = 16 and length(\"transit_credential_preview\".\"tag\") = 24" + } + }, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_application_receipt": { + "name": "integration_application_receipt", + "schema": "", + "columns": { + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "saved_binding_revision": { + "name": "saved_binding_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_application_target_time_idx": { + "name": "integration_application_target_time_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_application_actor_idx": { + "name": "integration_application_actor_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_application_receipt_actor_id_user_id_fk": { + "name": "integration_application_receipt_actor_id_user_id_fk", + "tableFrom": "integration_application_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_binding_revision": { + "name": "integration_binding_revision", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 82b45af26..8c5791ce9 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -358,6 +358,13 @@ "when": 1793275863684, "tag": "0050_admin_legacy_integration_adoption", "breakpoints": true + }, + { + "idx": 51, + "version": "7", + "when": 1793319100728, + "tag": "0051_platform_configuration_preview", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/platform-configuration-preview-schema.ts b/apps/api/src/db/platform-configuration-preview-schema.ts new file mode 100644 index 000000000..4a7fb8151 --- /dev/null +++ b/apps/api/src/db/platform-configuration-preview-schema.ts @@ -0,0 +1,43 @@ +/** Non-secret exact platform intent retained for actor-bound confirmation. */ +import { sql } from "drizzle-orm"; +import { + check, + index, + jsonb, + pgTable, + text, + timestamp, + uniqueIndex, + uuid, +} from "drizzle-orm/pg-core"; +import { user } from "./auth-schema"; +export const platformConfigurationPreview = pgTable( + "platform_configuration_preview", + { + id: uuid("id").primaryKey(), + actorId: text("actor_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + operationKey: text("operation_key").notNull(), + requestFingerprint: text("request_fingerprint").notNull(), + previewFingerprint: text("preview_fingerprint").notNull(), + authPolicyRevision: text("auth_policy_revision").notNull(), + sourceRevision: uuid("source_revision").notNull(), + expectedCurrentRevisionId: text("expected_current_revision_id"), + preview: jsonb("preview").notNull(), + consumedByJobId: uuid("consumed_by_job_id"), + consumedAt: timestamp("consumed_at", { withTimezone: true }), + retiredAt: timestamp("retired_at", { withTimezone: true }), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [ + uniqueIndex("platform_preview_actor_key_uq").on(t.actorId, t.operationKey), + uniqueIndex("platform_preview_fingerprint_uq").on(t.previewFingerprint), + index("platform_preview_expiry_idx").on(t.expiresAt), + check( + "platform_preview_bounds", + sql`length(${t.operationKey}) between 1 and 128 and ${t.requestFingerprint} ~ '^[a-f0-9]{64}$' and ${t.previewFingerprint} ~ '^[a-f0-9]{64}$' and ${t.authPolicyRevision} ~ '^[a-f0-9]{64}$' and octet_length(${t.preview}::text) <= 131072 and jsonb_typeof(${t.preview}) = 'object' and ${t.expiresAt} > ${t.createdAt} and ${t.expiresAt} <= ${t.createdAt} + interval '10 minutes' and ((${t.consumedAt} is null) = (${t.consumedByJobId} is null)) and (${t.retiredAt} is null or (${t.consumedAt} is not null and ${t.retiredAt} >= ${t.consumedAt}))`, + ), + ], +); diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index c8ee464f1..126e41e1f 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -74,6 +74,7 @@ export { integrationSecret } from "./integration-secret-schema"; export { mangroveKeypair, mangroveKeypairWrap } from "./mangrove-keypair-schema"; // Short-lived system-browser → WebView session handoff export { mobileAuthHandoff } from "./mobile-auth-handoff-schema"; +export { platformConfigurationPreview } from "./platform-configuration-preview-schema"; // Export-download authentication assurance and purpose-bound challenges export { dataExportReauthentication, sessionAuthAssurance } from "./privacy-auth-schema"; // GDPR subject-access request lifecycle and encrypted artifact metadata diff --git a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts new file mode 100644 index 000000000..1d4a5e0d6 --- /dev/null +++ b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts @@ -0,0 +1,172 @@ +import { randomUUID } from "node:crypto"; +import { services as coreServices } from "@openmapx/core/server"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { user } from "../../db/schema"; +import { createMigratedPostgresFixture } from "../../test/postgres-fixture"; + +const { buildPlatformConfigurationPreview } = coreServices; + +import { readPlatformPreview, retainPlatformPreview } from "../platform-preview-store"; + +const dbDescribe = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1" ? describe : describe.skip; +dbDescribe("durable exact platform preview", () => { + let fixture: Awaited>; + beforeAll(async () => { + fixture = await createMigratedPostgresFixture(); + }, 90000); + afterAll(async () => { + await fixture?.stop(); + }); + async function actor() { + const id = randomUUID(); + await fixture.db.insert(user).values({ + id, + name: "Synthetic administrator", + email: `${id}@example.test`, + role: "admin", + banned: false, + }); + return id; + } + function preview(actorId: string, previewId = randomUUID()) { + return buildPlatformConfigurationPreview({ + actorId, + previewId, + authPolicyRevision: "a".repeat(64), + sourceRevision: randomUUID(), + currentRevisionId: `cfg1_${"a".repeat(43)}`, + expectedCurrentRevisionId: `cfg1_${"a".repeat(43)}`, + env: { DOMAIN: "before.example.test" }, + changes: { version: 1, domain: { source: "database", value: "after.example.test" } }, + services: [], + }); + } + it("retains one original request and rejects changed intent, actor substitution and expiry", async () => { + const actorId = await actor(), + operationKey = randomUUID(), + original = preview(actorId), + requestFingerprint = "b".repeat(64), + now = new Date(); + const row = await retainPlatformPreview( + { actorId, operationKey, requestFingerprint, preview: original, now }, + fixture.db, + ); + expect(row.preview.previewFingerprint).toBe(original.previewFingerprint); + expect( + await readPlatformPreview( + { actorId, previewFingerprint: original.previewFingerprint, now }, + fixture.db, + ), + ).toEqual(row); + const replay = await retainPlatformPreview( + { actorId, operationKey, requestFingerprint, preview: preview(actorId), now }, + fixture.db, + ); + expect(replay).toEqual(row); + await expect( + retainPlatformPreview( + { actorId, operationKey, requestFingerprint: "c".repeat(64), preview: original, now }, + fixture.db, + ), + ).rejects.toThrow(/changed/i); + await expect( + readPlatformPreview( + { actorId: await actor(), previewFingerprint: original.previewFingerprint, now }, + fixture.db, + ), + ).rejects.toThrow(); + await expect( + readPlatformPreview( + { + actorId, + previewFingerprint: original.previewFingerprint, + now: new Date(now.getTime() + 600001), + }, + fixture.db, + ), + ).rejects.toThrow(/expired/i); + }); + it("concurrent duplicate requests converge on the original row and invalid policy/source copies refuse", async () => { + const actorId = await actor(), + operationKey = randomUUID(), + one = preview(actorId), + two = preview(actorId), + now = new Date(); + const results = await Promise.all( + [one, two].map((p) => + retainPlatformPreview( + { actorId, operationKey, requestFingerprint: "d".repeat(64), preview: p, now }, + fixture.db, + ), + ), + ); + expect(results[0]).toEqual(results[1]); + const { platformConfigurationPreview } = await import("../../db/schema"); + await fixture.db + .update(platformConfigurationPreview) + .set({ preview: { ...results[0]!.preview, authPolicyRevision: "e".repeat(64) } }) + .where(eq(platformConfigurationPreview.id, results[0]!.preview.previewId)); + await expect( + readPlatformPreview( + { actorId, previewFingerprint: results[0]!.preview.previewFingerprint, now }, + fixture.db, + ), + ).rejects.toThrow(); + }); + it("requires the same actor/revision/live policy in the desired-row transaction", async () => { + const { assertDomainRecoveryPreview } = await import("../platform-configuration"); + const { loadAuthPolicy, applyAuthPolicy } = await import("../../account/auth-policy"); + const actorId = await actor(), + policy = await loadAuthPolicy(fixture.db), + value = preview(actorId); + const bound = buildPlatformConfigurationPreview({ + actorId, + previewId: value.previewId, + authPolicyRevision: policy.revision, + sourceRevision: value.sourceRevision, + currentRevisionId: value.expectedCurrentRevisionId, + expectedCurrentRevisionId: value.expectedCurrentRevisionId, + env: { DOMAIN: "before.example.test" }, + changes: value.configuration, + services: [], + }); + await retainPlatformPreview( + { actorId, operationKey: randomUUID(), requestFingerprint: "f".repeat(64), preview: bound }, + fixture.db, + ); + const request = { + actorId, + previewFingerprint: bound.previewFingerprint, + expectedCurrentRevisionId: bound.expectedCurrentRevisionId, + expectedAuthPolicyRevision: policy.revision, + }; + await fixture.db.transaction((tx) => assertDomainRecoveryPreview(request, tx)); + for (const altered of [ + { actorId: await actor() }, + { expectedCurrentRevisionId: null }, + { expectedAuthPolicyRevision: "b".repeat(64) }, + ]) + await expect( + fixture.db.transaction((tx) => assertDomainRecoveryPreview({ ...request, ...altered }, tx)), + ).rejects.toThrow(); + await applyAuthPolicy( + { + confirmed: true, + expectedRevision: policy.revision, + idempotencyKey: randomUUID(), + settings: { + emailVerificationRequired: true, + twoFaEnabled: true, + passwordMinLength: 12, + sessionDurationHours: 168, + }, + }, + actorId, + fixture.db, + ); + await expect( + fixture.db.transaction((tx) => assertDomainRecoveryPreview(request, tx)), + ).rejects.toThrow(/policy/i); + }); +}); diff --git a/apps/api/src/services/platform-configuration.ts b/apps/api/src/services/platform-configuration.ts new file mode 100644 index 000000000..b7ed5f3e4 --- /dev/null +++ b/apps/api/src/services/platform-configuration.ts @@ -0,0 +1,50 @@ +/** Platform target authorization. Host effects remain behind the nominal + * configuration participant and typed maintenance/handoff operations. */ +import { and, eq, sql } from "drizzle-orm"; +import type { AuthDatabase } from "../account/auth-database"; +import { loadAuthPolicy } from "../account/auth-policy"; +import { platformConfigurationPreview } from "../db/schema"; +import { PlatformPreviewConflict, readPlatformPreview } from "./platform-preview-store"; + +/** Called in the SAME desired-row transaction as purpose-proof consumption + * and its saved continue checkpoint; it does not itself consume a proof. */ +export async function assertDomainRecoveryPreview( + input: { + actorId: string; + previewFingerprint: string; + expectedCurrentRevisionId: string | null; + expectedAuthPolicyRevision: string; + }, + tx: AuthDatabase, +): Promise { + // Also protects the not-yet-activated policy singleton from an INSERT. + await tx.execute(sql`LOCK TABLE auth_policy_state IN SHARE MODE`); + await tx + .select({ id: platformConfigurationPreview.id }) + .from(platformConfigurationPreview) + .where( + and( + eq(platformConfigurationPreview.actorId, input.actorId), + eq(platformConfigurationPreview.previewFingerprint, input.previewFingerprint), + ), + ) + .for("share"); + const retained = await readPlatformPreview( + { actorId: input.actorId, previewFingerprint: input.previewFingerprint }, + tx, + ); + if ( + retained.consumedByJobId || + !retained.preview.requiredConfirmation.domain || + retained.preview.expectedCurrentRevisionId !== input.expectedCurrentRevisionId + ) + throw new PlatformPreviewConflict(); + const policy = await loadAuthPolicy(tx); + if ( + policy.revision !== input.expectedAuthPolicyRevision || + retained.preview.authPolicyRevision !== policy.revision + ) + throw new PlatformPreviewConflict( + "Platform authentication policy changed; create a new preview", + ); +} diff --git a/apps/api/src/services/platform-preview-store.ts b/apps/api/src/services/platform-preview-store.ts new file mode 100644 index 000000000..0bb41cc60 --- /dev/null +++ b/apps/api/src/services/platform-preview-store.ts @@ -0,0 +1,130 @@ +/** Original preview rows are immutable; a request retry cannot extend its TTL. */ +import { createHash } from "node:crypto"; +import { adminPlatformPreviewSchema } from "@openmapx/core/api-contracts"; +import { canonicalJson } from "@openmapx/core/ops"; +import { and, eq, isNull, lt, sql } from "drizzle-orm"; +import { z } from "zod"; +import type { AuthDatabase } from "../account/auth-database"; +import { db } from "../db"; +import { platformConfigurationPreview as table } from "../db/schema"; +import { preservationAllowsDeletion, withPreservationRetentionLock } from "../privacy/preservation"; + +const TTL = 600_000; +export class PlatformPreviewConflict extends Error { + constructor(message = "Platform preview unavailable or changed") { + super(message); + } +} +function decode(row: typeof table.$inferSelect) { + try { + const preview = adminPlatformPreviewSchema.parse(row.preview); + const { previewFingerprint, ...intent } = preview; + const digest = createHash("sha256") + .update("openmapx-platform-preview-v1\0") + .update(canonicalJson(intent)) + .digest("hex"); + if ( + digest !== previewFingerprint || + previewFingerprint !== row.previewFingerprint || + preview.previewId !== row.id || + preview.actorId !== row.actorId || + preview.authPolicyRevision !== row.authPolicyRevision || + preview.sourceRevision !== row.sourceRevision || + preview.expectedCurrentRevisionId !== row.expectedCurrentRevisionId || + row.expiresAt.getTime() !== row.createdAt.getTime() + TTL + ) + throw Error(); + return { + preview, + expiresAt: row.expiresAt.toISOString(), + consumedByJobId: row.consumedByJobId, + }; + } catch { + throw new PlatformPreviewConflict(); + } +} +export async function retainPlatformPreview( + input: { + actorId: string; + operationKey: string; + requestFingerprint: string; + preview: unknown; + now?: Date; + }, + database: typeof db = db, +) { + const preview = adminPlatformPreviewSchema.parse(input.preview), + now = input.now ?? new Date(); + z.string().min(1).max(128).parse(input.operationKey); + z.string() + .regex(/^[a-f0-9]{64}$/) + .parse(input.requestFingerprint); + if (preview.actorId !== input.actorId) throw new PlatformPreviewConflict(); + return withPreservationRetentionLock(database, async (tx) => { + // One capacity/expiry boundary also serializes all duplicate request keys. + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended('openmapx.platform.preview',71956))`, + ); + const [existing] = await tx + .select() + .from(table) + .where(and(eq(table.actorId, input.actorId), eq(table.operationKey, input.operationKey))) + .for("update"); + if (existing) { + if (existing.requestFingerprint !== input.requestFingerprint) + throw new PlatformPreviewConflict("Platform preview request changed"); + if (existing.expiresAt <= now) + throw new PlatformPreviewConflict("Platform preview expired; create a new preview"); + return decode(existing); + } + // Unconsumed previews never authorized effects; consumed/unknown work stays pinned. + await tx.delete(table).where( + and( + isNull(table.consumedByJobId), + lt(table.expiresAt, now), + preservationAllowsDeletion({ + registrationId: "admin-jobs-attribution", + candidateTimestamp: table.createdAt, + subjectIds: [table.actorId], + }), + ), + ); + const capacity = await tx.select({ id: table.id }).from(table).limit(4096); + if (capacity.length >= 4096) + throw new PlatformPreviewConflict("Platform preview capacity unavailable"); + const [row] = await tx + .insert(table) + .values({ + id: preview.previewId, + actorId: input.actorId, + operationKey: input.operationKey, + requestFingerprint: input.requestFingerprint, + previewFingerprint: preview.previewFingerprint, + authPolicyRevision: preview.authPolicyRevision, + sourceRevision: preview.sourceRevision, + expectedCurrentRevisionId: preview.expectedCurrentRevisionId, + preview, + createdAt: now, + expiresAt: new Date(now.getTime() + TTL), + }) + .returning(); + if (!row) throw new PlatformPreviewConflict(); + return decode(row); + }); +} +export async function readPlatformPreview( + input: { actorId: string; previewFingerprint: string; now?: Date }, + database: AuthDatabase = db, +) { + const [row] = await database + .select() + .from(table) + .where( + and(eq(table.actorId, input.actorId), eq(table.previewFingerprint, input.previewFingerprint)), + ) + .limit(1); + if (!row) throw new PlatformPreviewConflict(); + if (row.expiresAt <= (input.now ?? new Date())) + throw new PlatformPreviewConflict("Platform preview expired; create a new preview"); + return decode(row); +} diff --git a/apps/web/src/components/admin/shared/SchemaConfigForm.tsx b/apps/web/src/components/admin/shared/SchemaConfigForm.tsx index 5fef47303..e05cdb3e2 100644 --- a/apps/web/src/components/admin/shared/SchemaConfigForm.tsx +++ b/apps/web/src/components/admin/shared/SchemaConfigForm.tsx @@ -371,7 +371,11 @@ export function SchemaConfigForm({ control={ setValues((prev) => ({ ...prev, [field.key]: e.target.checked })) } @@ -432,7 +436,7 @@ export function SchemaConfigForm({ }} disabled={isEnvOverridden || saving} type={ - field.type === "number" || field.type === "integer" + !isEnvOverridden && (field.type === "number" || field.type === "integer") ? "number" : field.format === "url" ? "url" diff --git a/apps/web/src/components/admin/shared/__tests__/SchemaConfigForm.ownership.test.tsx b/apps/web/src/components/admin/shared/__tests__/SchemaConfigForm.ownership.test.tsx index a5255a79a..f1287ceaf 100644 --- a/apps/web/src/components/admin/shared/__tests__/SchemaConfigForm.ownership.test.tsx +++ b/apps/web/src/components/admin/shared/__tests__/SchemaConfigForm.ownership.test.tsx @@ -25,3 +25,24 @@ it("locks config.json winners and never sends them in a save alongside an editab expect(save).toHaveBeenCalledWith({ region: "new" }); expect(await screen.findByText(/Controlled by the host config file/i)).toBeInTheDocument(); }); + +it("shows legacy host memory units and false booleans honestly in locked controls", () => { + render( + {}} + />, + ); + expect(screen.getByDisplayValue("16g")).toBeDisabled(); + expect(screen.getByRole("switch")).not.toBeChecked(); + expect(screen.getByRole("switch")).toBeDisabled(); +}); diff --git a/docs/superpowers/specs/2026-10-30-admin-protected-release-mode-design.md b/docs/superpowers/specs/2026-10-30-admin-protected-release-mode-design.md new file mode 100644 index 000000000..b6bd1c90c --- /dev/null +++ b/docs/superpowers/specs/2026-10-30-admin-protected-release-mode-design.md @@ -0,0 +1,37 @@ +# Protected configuration and release handoff modes + +Status: APPROVED after bounded independent Codex R1 READY (0 Critical / 0 Important). Design only; product/runtime/CI verification remains required. Extends only Task08.3, same50 tasks/one PR. No source clearance or runtime acceptance. + +## Decision + +Use the existing maintenance root -> maintenance-handoff -> configuration-helper finite graph for both modes, with a discriminated, frozen handoff intent. Do not stage release images under an unrelated root and then try to recover old containers from the changed cache. The maintenance root captures the ordinary owned containers BEFORE pulling a release. The detached helper performs the fixed release pull only AFTER terminal handoff preparation, under the original delegated full target closure. Desired configuration can be committed before this follow-up; failure retains its truthful desired-commit evidence. + +Same-image mode retains the exact proposed generation, current trusted image references and old container/image identities. It performs no image pull. Resource/env/mount verification must use the proposed generation even when image identity is unchanged. + +Release mode additionally retains the exact checked release ID, canonical manifest digest and closed per-service pinned image-reference map. The agent loads this map from its existing verified release manifest; the API cannot provide image strings, overlay bytes or paths. A pinned reference identifies the requested image BEFORE download; a Docker image ID is an observed result AFTER download. They are distinct fields and must never be compared as if they were the same hash. Root/step fingerprints bind the immutable pinned reference map, not later Docker-assigned execution IDs or observations. + +## Fixed producer and scope + +Add strict configurationHandoff.prepare/inspect operations as allocated by C3. A prepare's mode is either configuration or release. Both bind handoffId, maintenanceId, candidateId, planDigest, exact old/new configuration revisions, preview fingerprint, bounded sorted target subset and retained old-container image identities. Configuration mode uses the proposed generation's existing image references. Release mode names only releaseId+canonical manifest digest; the actual agent derives/compares its pinned map during whole-candidate preflight, includes its digest in the private frozen plan and refuses missing/unpinned/mismatching entries BEFORE any desired save. No runtime callback, path, caller boolean or arbitrary image reference is an issuer. + +The frozen maintenance effect list includes the existing configuration commit, optional sole DM hardlinks leaf, ordinary recreates and one protected handoff. The handoff's exact operation fingerprint/target closure is reserved at root admission; its sole helper child's fixed body binds the handoff ID plus immutable intent digest. Only identity/deadline normalization uses the accepted late-child protocol. No late operation-body field or grant retargeting is introduced. Protected configuration target order is postgis, redis, app-web, app-api, traefik, ops-agent. A release's ordinary enabled service targets are also frozen and processed before protected agent-last replacement. Any included DM recreation uses the original durable guard/startup mirror and settled hardlinks prerequisite. + +All target/image expectations, download count/bytes/deadline and journal space are preflighted before ownership admission. The existing bounded receipt cells and role-private fixed journals remain the storage roots; new purpose records must be indexed/charged and included in existing retention families, never a second journal key or signing root. + +## Helper execution and publication + +Persist immutable launch intent and deterministic helper identity before Docker create. Inspect an attempted launch rather than create another helper. The helper adopts only its original configuration-helper grant and verifies terminal prepared handoff before draining mutating operations/byte streams. Read-only logs/inspection and terminal logical preparation records do not block drain. No configuration/release writer lock crosses Docker process replacement. Each write takes original target ownership -> configuration writer -> release-store and revalidates actual source, original grant, generation and manifest at that effect boundary. + +In release mode the helper pulls ONLY frozen pinned references. After each awaited pull, inspect that exact reference, require the expected RepoDigest/architecture and retain actual Docker image ID before any recreate. Existing old containers remain captured by their original immutable image IDs; there is no general owned-services resolver weakening. The new generation/release overlay is constructed by the existing release renderer from the verified manifest, not API-supplied YAML. Every subsequent container must have its intended image ID and exact desired config/hash/environment/resources/mount/readiness. Secret comparisons stay private/keyed. Self replacement is last; the helper uses the known old agent image and survives its parent's exit. + +Publication intent precedes each overlay/current-state/container effect. Lost results inspect original receipts/Docker identities; no attempted pull/recreate/publication is blindly replayed. A new agent reconstructs the same helper/owner from authenticated retained sources and never relaunches it. The original root and DM guard cannot release while a launch, child, container, source or SQL outcome is unknown. + +## Failure, rollback and retirement + +A conclusive pre-recreation pull failure permits a contained follow-up result only after proving zero recreation and restoring the exact old host generation/overlay under the original root. Desired database rows remain committed and reported unapplied. Restoration is a conditional comparison against the exact original proposed pointer/overlay and retained original bytes/inodes, never an arbitrary path or source. Once recreation begins, automatic rollback is permitted only if all changed targets can be restored to the exact original image/config/mount/resource/readiness and all original attempts are contained. Restoring overlay bytes alone is never rollback proof. Conflicting ownership, changed source, missing proof or failed restoration stays recovery_required, with original evidence and fences retained. + +Only actual applied or fully contained original helper evidence enters the existing maintenance final/SQL/ACK/root-closure flow. Late recovery is observation/settlement-only; it does not renew a forward budget. Retention uses the existing later control/ACK floor and whole-family verification; unknown handoffs remain pinned. No legacy helper/startup mutation is re-enabled until this installed workflow and original source validators are used by its actual production entrypoints. + +## Required verification + +Keep task C3's exact API/domain/transaction/rollback/restart/capacity and same-image tests. Add actual production API -> agent -> detached helper process tests for a new pinned image, unchanged-image resource changes, original process exit, old/new helper death/restart, lost launch and terminal responses, delayed helper write versus staged DM, unknown/failed restore, release-store/configuration-writer ordering and exact SQL/ACK/root release. No old cached-image workaround, synthetic signed authority, public-status containment, callback-selected verifier, or permanent unsupported release control qualifies the final PR. diff --git a/packages/core/src/api/contracts/service-admin/index.ts b/packages/core/src/api/contracts/service-admin/index.ts index 9f667fef6..dee60cd25 100644 --- a/packages/core/src/api/contracts/service-admin/index.ts +++ b/packages/core/src/api/contracts/service-admin/index.ts @@ -416,3 +416,5 @@ export const serviceAdminOperations = { }, } as const satisfies Record; export type ServiceAdminOperationId = keyof typeof serviceAdminOperations; + +export * from "./platform"; diff --git a/packages/core/src/api/contracts/service-admin/platform.ts b/packages/core/src/api/contracts/service-admin/platform.ts new file mode 100644 index 000000000..a07e810da --- /dev/null +++ b/packages/core/src/api/contracts/service-admin/platform.ts @@ -0,0 +1,66 @@ +/** Portable platform preview data; these schemas confer no host authority. */ +import z from "zod/v4"; +import { platformConfigurationSchema } from "../../../services/platform-configuration"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const revision = z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .nullable(); +const source = z.enum(["host", "database"]); +const url = z.url().max(2048); +export const adminPlatformPreviewInputSchema = z.strictObject({ + expectedCurrentRevisionId: revision, + changes: platformConfigurationSchema, +}); +export const adminPlatformConfirmationSchema = z.strictObject({ + domain: z.boolean(), + dnsTls: z.boolean(), + oauth: z.boolean(), + passkeys: z.boolean(), + alternateLogin: z.boolean(), +}); +export const adminPlatformPreviewSchema = z.strictObject({ + version: z.literal(1), + actorId: z.string().min(1).max(128), + previewId: z.uuid(), + authPolicyRevision: digest, + sourceRevision: z.uuid(), + expectedCurrentRevisionId: revision, + configuration: platformConfigurationSchema, + values: z.strictObject({ + domain: z.string().min(1).max(253), + acmeEmail: z.email().max(320), + releaseSelector: z.string().min(1).max(128), + defaultRegion: z.string().max(128).nullable(), + backupRetentionDays: z.number().int().min(1).max(36500), + }), + sources: z.strictObject({ + domain: source, + acmeEmail: source, + releaseSelector: source, + defaultRegion: source, + backupRetentionDays: source, + }), + affectedServiceIds: z.array(z.string().min(1).max(64)).max(256), + oldOrigin: url, + newOrigin: url, + originConsumers: z.strictObject({ + routingHost: z.string().min(1).max(253), + corsOrigin: url, + authUrl: url, + passkeyRpId: z.string().min(1).max(253), + passkeyOrigin: z.string().max(2048), + webApiUrl: url, + tilesPublicUrl: url, + }), + requiredConfirmation: adminPlatformConfirmationSchema, + warnings: z.array(z.string().max(512)).max(16), + previewFingerprint: digest, +}); +export const adminPlatformApplyInputSchema = z.strictObject({ + expectedCurrentRevisionId: revision, + previewFingerprint: digest, + confirmations: adminPlatformConfirmationSchema, +}); +export type AdminPlatformPreview = z.infer; diff --git a/packages/core/src/services/__tests__/platform-preview.test.ts b/packages/core/src/services/__tests__/platform-preview.test.ts new file mode 100644 index 000000000..78757e596 --- /dev/null +++ b/packages/core/src/services/__tests__/platform-preview.test.ts @@ -0,0 +1,159 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { renderServiceSnippet } from "../compose-renderer"; +import { buildPlatformConfigurationPreview } from "../platform-preview"; +import type { LoadedService } from "../types"; + +const services: LoadedService[] = [ + "app-api", + "app-web", + "traefik", + "tileserver", + "ops-agent", + "data-manager", + "redis", +].map((id) => ({ + manifest: JSON.parse( + readFileSync( + resolve(import.meta.dirname, "../../../../../services", id, "service.json"), + "utf8", + ), + ), + directory: "/synthetic/" + id, + isBuiltIn: true, + enabled: true, +})); +const input = () => ({ + actorId: "admin-a", + previewId: "08be2da1-c0b5-46a8-a85e-aaedb03ff7bf", + authPolicyRevision: "a".repeat(64), + sourceRevision: "ef854296-a818-4cd6-a4be-c1e28be376ac", + currentRevisionId: `cfg1_${"a".repeat(43)}`, + expectedCurrentRevisionId: `cfg1_${"a".repeat(43)}`, + env: { DOMAIN: "old.example.test", ACME_EMAIL: "host@example.test" }, + changes: { + version: 1 as const, + domain: { source: "database" as const, value: "new.example.test" }, + }, + services, +}); +describe("platform configuration preview", () => { + it("shows exact public origin consumers and requires domain recovery confirmation", () => { + const source = input(); + const preview = buildPlatformConfigurationPreview(source); + expect(preview).toMatchObject({ + oldOrigin: "https://old.example.test", + newOrigin: "https://new.example.test", + requiredConfirmation: { + domain: true, + dnsTls: true, + oauth: true, + passkeys: true, + alternateLogin: true, + }, + originConsumers: { + routingHost: "new.example.test", + corsOrigin: "https://new.example.test", + authUrl: "https://new.example.test", + passkeyRpId: "new.example.test", + passkeyOrigin: "https://new.example.test,openmapx://", + webApiUrl: "https://new.example.test", + tilesPublicUrl: "https://new.example.test/tiles/", + }, + }); + expect(preview.affectedServiceIds).toEqual(["app-api", "app-web", "tileserver", "traefik"]); + expect(source.env.DOMAIN).toBe("old.example.test"); + }); + it("binds preview identity, actor, policy, source and exact intent without including secrets", () => { + const initial = buildPlatformConfigurationPreview(input()); + expect(buildPlatformConfigurationPreview(input()).previewFingerprint).toBe( + initial.previewFingerprint, + ); + for (const change of [ + { actorId: "admin-b" }, + { authPolicyRevision: "b".repeat(64) }, + { sourceRevision: "27142d4e-1db8-43b8-a4a4-becdfddbb368" }, + { previewId: "4ad04498-79fc-4fe5-b059-78c3440440da" }, + ]) + expect( + buildPlatformConfigurationPreview({ ...input(), ...change }).previewFingerprint, + ).not.toBe(initial.previewFingerprint); + expect( + JSON.stringify( + buildPlatformConfigurationPreview({ + ...input(), + env: { ...input().env, DATABASE_URL: "never-public" }, + }), + ), + ).not.toContain("never-public"); + }); + it("rejects stale revisions, unknown fields and invalid origins before a desired change", () => { + expect(() => + buildPlatformConfigurationPreview({ ...input(), expectedCurrentRevisionId: null }), + ).toThrow(/revision/i); + expect(() => + buildPlatformConfigurationPreview({ + ...input(), + changes: { ...input().changes, unknown: true }, + } as never), + ).toThrow(); + expect(() => + buildPlatformConfigurationPreview({ + ...input(), + changes: { version: 1, domain: { source: "database", value: "https://bad.example/path" } }, + }), + ).toThrow(); + }); + it("preserves unrelated adopted settings and derives actual release/retention consumers", () => { + const preview = buildPlatformConfigurationPreview({ + ...input(), + currentPlatform: { + version: 1, + acmeEmail: { source: "database", value: "saved@example.test" }, + }, + changes: { version: 1, backupRetentionDays: { source: "database", value: 60 } }, + }); + expect(preview.configuration.acmeEmail).toEqual({ + source: "database", + value: "saved@example.test", + }); + expect(preview.values.backupRetentionDays).toBe(60); + expect(preview.requiredConfirmation.domain).toBe(false); + expect(preview.affectedServiceIds).toEqual(["app-api", "ops-agent"]); + }); + it("can explicitly return a field to host ownership without changing the host", () => { + const preview = buildPlatformConfigurationPreview({ + ...input(), + currentPlatform: { + version: 1, + domain: { source: "database", value: "previous.example.test" }, + }, + changes: { version: 1, domain: { source: "host" } }, + }); + expect(preview.newOrigin).toBe("https://old.example.test"); + expect(preview.sources.domain).toBe("host"); + }); +}); + +it("materializes the adopted origin in every actual built-in origin consumer", () => { + const preview = buildPlatformConfigurationPreview(input()); + const context = { + domain: preview.values.domain, + platform: preview.configuration, + platformValues: preview.values, + existsSync: () => true, + allServices: services, + }; + const byId = (id: string) => + renderServiceSnippet(services.find((s) => s.manifest.id === id)!, context); + expect(byId("app-api").environment).toMatchObject({ + DOMAIN: "new.example.test", + CORS_ORIGIN: "https://new.example.test", + BETTER_AUTH_URL: "https://new.example.test", + PASSKEY_RP_ID: "new.example.test", + PASSKEY_ORIGIN: "https://new.example.test,openmapx://", + }); + expect(byId("app-web").environment?.NEXT_PUBLIC_API_URL).toBe("https://new.example.test"); + expect(byId("tileserver").command).toEqual(["--public_url", "https://new.example.test/tiles/"]); +}); diff --git a/packages/core/src/services/compose-renderer.ts b/packages/core/src/services/compose-renderer.ts index 64603f8b1..9b025a61b 100644 --- a/packages/core/src/services/compose-renderer.ts +++ b/packages/core/src/services/compose-renderer.ts @@ -532,6 +532,23 @@ export function renderServiceSnippet( if (c.command !== undefined) snippet.command = c.command; if (c.entrypoint !== undefined) snippet.entrypoint = c.entrypoint; + // This one audited command slot publishes TileJSON URLs. Do not leave a + // stale host DOMAIN reference after platform domain adoption, and do not + // evaluate arbitrary command templates. + if ( + service.isBuiltIn && + m.id === "tileserver" && + ctx.platform?.domain?.source === "database" && + ctx.platformValues && + Array.isArray(snippet.command) + ) { + const command = [...snippet.command]; + const position = command.indexOf("--public_url"); + if (position < 0 || command[position + 1] !== "https://${DOMAIN:-localhost}/tiles/") + throw Error("Tileserver public origin slot changed; review platform rendering"); + command[position + 1] = `https://${ctx.platformValues.domain}/tiles/`; + snippet.command = command; + } // Env files are deployment files under the infra root (built-in services // only), so a generation directory resolves them back to it. if (c.envFile?.length) { diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 77e9635d3..6d7ce791f 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -114,6 +114,10 @@ export { platformConfigurationSchema, resolvePlatformConfiguration, } from "./platform-configuration"; +export { + buildPlatformConfigurationPreview, + type PlatformConfigurationPreview, +} from "./platform-preview"; export { ServiceRegistry, type ServiceRegistryOptions } from "./registry"; export { captureReleaseServiceAuthority, diff --git a/packages/core/src/services/platform-preview.ts b/packages/core/src/services/platform-preview.ts new file mode 100644 index 000000000..f4d4deeaa --- /dev/null +++ b/packages/core/src/services/platform-preview.ts @@ -0,0 +1,137 @@ +/** Pure preview data. A fingerprint is an intent checksum, never host authority. */ +import { createHash } from "node:crypto"; +import z from "zod/v4"; +import { canonicalJson } from "../ops/canonical-json"; +import { resolveProxyHost } from "./compose-renderer"; +import { + type PlatformConfiguration, + platformConfigurationSchema, + resolvePlatformConfiguration, +} from "./platform-configuration"; +import type { LoadedService } from "./types"; + +const revision = z + .string() + .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) + .nullable(); +const variables = { + domain: "DOMAIN", + acmeEmail: "ACME_EMAIL", + releaseSelector: "OPENMAPX_RELEASE", + defaultRegion: "OPENMAPX_REGION", + backupRetentionDays: "BACKUP_RETENTION_DAYS", +} as const; +export function buildPlatformConfigurationPreview(input: { + actorId: string; + previewId: string; + authPolicyRevision: string; + sourceRevision: string; + currentRevisionId: string | null; + expectedCurrentRevisionId: string | null; + currentPlatform?: PlatformConfiguration; + changes: PlatformConfiguration; + env: Readonly>; + services: readonly LoadedService[]; + resolvedServiceConfigs?: Map>; +}) { + const currentRevisionId = revision.parse(input.currentRevisionId); + if (revision.parse(input.expectedCurrentRevisionId) !== currentRevisionId) + throw Error("Platform configuration revision changed"); + const actorId = z.string().min(1).max(128).parse(input.actorId); + const previewId = z.uuid().parse(input.previewId); + const authPolicyRevision = z + .string() + .regex(/^[a-f0-9]{64}$/) + .parse(input.authPolicyRevision); + const sourceRevision = z.uuid().parse(input.sourceRevision); + const changes = platformConfigurationSchema.parse(input.changes); + const configuration = platformConfigurationSchema.parse({ + ...input.currentPlatform, + ...changes, + version: 1, + }); + const before = resolvePlatformConfiguration({ saved: input.currentPlatform, env: input.env }); + const after = resolvePlatformConfiguration({ saved: configuration, env: input.env }); + const changedFields = (Object.keys(variables) as (keyof typeof variables)[]).filter( + (key) => before.values[key] !== after.values[key], + ); + const domain = before.values.domain !== after.values.domain; + const affectedServiceIds = input.services + .filter((service) => { + if (!service.isBuiltIn || !service.enabled) return false; + const m = service.manifest; + const consumer = JSON.stringify({ + environment: m.container.environment, + command: m.container.command, + entrypoint: m.container.entrypoint, + }); + if (changedFields.some((key) => consumer.includes(`\${${variables[key]}`))) return true; + if (domain && m.id === "traefik") return true; + if (!domain || !m.exposure?.proxy?.enabled) return false; + const proxy = (hostname: string) => + resolveProxyHost(m, { + domain: hostname, + resolvedServiceConfigs: input.resolvedServiceConfigs, + }) ?? hostname; + return proxy(before.values.domain) !== proxy(after.values.domain); + }) + .map((service) => service.manifest.id) + .sort(); + const oldOrigin = `https://${before.values.domain}`; + const newOrigin = `https://${after.values.domain}`; + const requiredConfirmation = { + domain, + dnsTls: domain, + oauth: domain, + passkeys: domain, + alternateLogin: domain, + }; + const originConsumers = { + routingHost: after.values.domain, + corsOrigin: newOrigin, + authUrl: newOrigin, + passkeyRpId: after.values.domain, + passkeyOrigin: `${newOrigin},openmapx://`, + webApiUrl: newOrigin, + tilesPublicUrl: `${newOrigin}/tiles/`, + }; + const warnings = [ + ...(domain + ? [ + "Configure DNS and TLS for the new origin before applying.", + "Update OAuth redirect registrations and retain a proven password login; origin-bound passkeys may require registration again.", + ] + : []), + ...(changedFields.includes("defaultRegion") + ? ["Changing the default region does not rebuild prepared datasets."] + : []), + ...(changedFields.includes("releaseSelector") + ? [ + "Changing the release selector does not install that release. Check and apply it separately.", + ] + : []), + ]; + const intent = { + version: 1 as const, + actorId, + previewId, + authPolicyRevision, + sourceRevision, + expectedCurrentRevisionId: currentRevisionId, + configuration, + values: after.values, + sources: after.sources, + affectedServiceIds, + oldOrigin, + newOrigin, + originConsumers, + requiredConfirmation, + warnings, + }; + const previewFingerprint = createHash("sha256") + .update("openmapx-platform-preview-v1\0") + .update(canonicalJson(intent)) + .digest("hex"); + return { ...intent, previewFingerprint }; +} +export type PlatformConfigurationPreview = ReturnType; diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index a5510c870..d01d9b214 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -31,7 +31,8 @@ "apps/api/src/account/account-provisioning-postgres.test.ts", "apps/api/src/services/integration-application-postgres.test.ts", "apps/api/src/services/integration-application.test.ts", - "apps/api/src/services/standalone-integrations/install.test.ts" + "apps/api/src/services/standalone-integrations/install.test.ts", + "apps/api/src/services/__tests__/platform-preview-postgres.test.ts" ], "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", From 126517e2d4f33df25ce03cd34c9330670cb887aa Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 02:24:22 +0100 Subject: [PATCH 064/135] feat(platform): bind desired saves to retained confirmations --- .../platform-preview-postgres.test.ts | 295 +++++++++++++++- ...ministrative-configuration-writer-owner.ts | 3 +- .../configuration-commit-participant.ts | 6 +- .../configuration-maintenance-decision.ts | 21 +- .../platform-configuration-participant.ts | 330 ++++++++++++++++++ .../src/services/platform-configuration.ts | 173 ++++++++- .../src/services/prospective-configuration.ts | 123 +++++-- ...nfiguration-maintenance-forward-records.ts | 1 + .../src/build-artifacts/control-authority.ts | 2 + .../fixed-configuration-maintenance.test.ts | 94 +++++ .../fixed-configuration-maintenance.ts | 58 ++- .../src/ops/configuration-handoff-records.ts | 109 ++++++ .../src/ops/configuration-handoff.test.ts | 100 ++++++ .../core/src/ops/configuration-maintenance.ts | 15 +- .../ops/configuration-operation-records.ts | 2 + .../ops/configuration-preparation-records.ts | 38 ++ packages/core/src/ops/contract.test.ts | 16 + packages/core/src/ops/contract.ts | 14 + packages/core/src/ops/execution-identity.ts | 1 + packages/core/src/ops/index.ts | 6 + 20 files changed, 1357 insertions(+), 50 deletions(-) create mode 100644 apps/api/src/services/platform-configuration-participant.ts create mode 100644 packages/core/src/ops/configuration-handoff-records.ts create mode 100644 packages/core/src/ops/configuration-handoff.test.ts diff --git a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts index 1d4a5e0d6..54b39f05d 100644 --- a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts @@ -1,10 +1,31 @@ import { randomUUID } from "node:crypto"; import { services as coreServices } from "@openmapx/core/server"; -import { eq } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { user } from "../../db/schema"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + adminActionReauthentication, + adminJob, + platformConfigurationPreview, + session, + sessionAuthAssurance, + systemSettings, + user, +} from "../../db/schema"; import { createMigratedPostgresFixture } from "../../test/postgres-fixture"; +const connection = vi.hoisted(() => ({ + fixture: null as unknown as Awaited>, +})); +vi.mock("../../db", () => ({ + get db() { + return connection.fixture.db; + }, + get sql() { + return connection.fixture.sql; + }, +})); +vi.mock("../../integration-host", () => ({ getAllIntegrations: () => [] })); +vi.mock("../service-registry", () => ({ getServiceRegistry: () => ({ list: () => [] }) })); const { buildPlatformConfigurationPreview } = coreServices; import { readPlatformPreview, retainPlatformPreview } from "../platform-preview-store"; @@ -14,9 +35,12 @@ dbDescribe("durable exact platform preview", () => { let fixture: Awaited>; beforeAll(async () => { fixture = await createMigratedPostgresFixture(); + connection.fixture = fixture; + vi.stubEnv("OPENMAPX_SECRETS_KEY", "41".repeat(32)); }, 90000); afterAll(async () => { await fixture?.stop(); + vi.unstubAllEnvs(); }); async function actor() { const id = randomUUID(); @@ -169,4 +193,269 @@ dbDescribe("durable exact platform preview", () => { fixture.db.transaction((tx) => assertDomainRecoveryPreview(request, tx)), ).rejects.toThrow(/policy/i); }); + async function commitIntent() { + const { loadAuthPolicy } = await import("../../account/auth-policy"); + const { AdminActionReauthentication } = await import("../../account/admin-reauthentication"); + const actorId = await actor(), + policy = await loadAuthPolicy(fixture.db); + const original = preview(actorId); + const bound = buildPlatformConfigurationPreview({ + actorId, + previewId: original.previewId, + authPolicyRevision: policy.revision, + sourceRevision: original.sourceRevision, + currentRevisionId: original.expectedCurrentRevisionId, + expectedCurrentRevisionId: original.expectedCurrentRevisionId, + env: { DOMAIN: "before.example.test" }, + changes: original.configuration, + services: [], + }); + await retainPlatformPreview( + { actorId, operationKey: randomUUID(), requestFingerprint: "a".repeat(64), preview: bound }, + fixture.db, + ); + const clock = new Date(), + reauth = new AdminActionReauthentication(fixture.db, () => clock); + async function login(at: Date) { + const id = randomUUID(); + await fixture.db.insert(session).values({ + id, + userId: actorId, + token: randomUUID(), + expiresAt: new Date(Date.now() + 3600000), + createdAt: at, + updatedAt: at, + }); + await fixture.db.insert(sessionAuthAssurance).values({ + sessionId: id, + userId: actorId, + method: "password", + authenticatedAt: at, + originIndependentPasswordLogin: true, + }); + return { + id, + userId: actorId, + method: "password" as const, + authenticatedAt: at, + mfaConfigured: false, + }; + } + const old = await login(new Date(clock.getTime() - 10000)); + const proof = await reauth.start({ + actorId, + startingSessionId: old.id, + purpose: "platform.domain.apply", + targetId: bound.previewFingerprint, + }); + clock.setTime(clock.getTime() + 10); + const fresh = await login(new Date(clock.getTime() - 1)); + await reauth.complete({ challengeId: proof.challengeId, nonce: proof.nonce, session: fresh }); + const jobId = randomUUID(), + execution = { owner: randomUUID(), version: 1 }; + await fixture.db.insert(adminJob).values({ + id: jobId, + type: "platform.apply", + status: "running", + createdBy: actorId, + result: { _execution: execution }, + }); + const [desired] = await fixture.db + .select({ version: sql`xmin::text` }) + .from(systemSettings) + .where(eq(systemSettings.key, "platformConfiguration")); + return { + actorId, + previewFingerprint: bound.previewFingerprint, + expectedCurrentRevisionId: bound.expectedCurrentRevisionId, + expectedAuthPolicyRevision: policy.revision, + sourceRevision: bound.sourceRevision, + expectedDesiredRowVersion: desired?.version ?? null, + jobId, + execution, + candidateId: randomUUID(), + revisionId: `cfg1_${"b".repeat(43)}`, + confirmations: { + domain: true, + dnsTls: true, + oauth: true, + passkeys: true, + alternateLogin: true, + }, + proof: { challengeId: proof.challengeId, sessionId: fresh.id, nonce: proof.nonce }, + }; + } + it("atomically commits the desired platform, consumed proof, preview and job checkpoint", async () => { + const { commitPlatformConfigurationIntent } = await import("../platform-configuration"); + const input = await commitIntent(); + await expect( + fixture.db.transaction(async (tx) => { + await commitPlatformConfigurationIntent(input, tx); + await tx + .update(adminJob) + .set({ result: { _execution: input.execution, saved: true } }) + .where(eq(adminJob.id, input.jobId)); + throw Error("synthetic checkpoint rollback"); + }), + ).rejects.toThrow("synthetic checkpoint rollback"); + const [proof] = await fixture.db + .select() + .from(adminActionReauthentication) + .where(eq(adminActionReauthentication.id, input.proof.challengeId)); + expect(proof?.state).toBe("completed"); + const [stored] = await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.previewFingerprint, input.previewFingerprint)); + expect(stored?.consumedByJobId).toBeNull(); + await fixture.db.transaction(async (tx) => { + const receipt = await commitPlatformConfigurationIntent(input, tx); + const { assertConfigurationSavedRows } = await import( + "../configuration-maintenance-decision" + ); + await assertConfigurationSavedRows(tx, [receipt]); + await tx + .update(adminJob) + .set({ result: { _execution: input.execution, saved: true } }) + .where(eq(adminJob.id, input.jobId)); + }); + const [desired] = await fixture.db + .select() + .from(systemSettings) + .where(eq(systemSettings.key, "platformConfiguration")); + expect(desired?.value).toMatchObject({ + version: 1, + jobId: input.jobId, + candidateId: input.candidateId, + revisionId: input.revisionId, + configuration: { domain: { source: "database", value: "after.example.test" } }, + }); + expect(JSON.stringify(desired)).not.toContain(input.proof.nonce.toString("base64")); + await expect( + fixture.db.transaction((tx) => commitPlatformConfigurationIntent(input, tx)), + ).rejects.toThrow(); + }); + it("rejects omitted acknowledgments, wrong proof, source or job owner before desired writes", async () => { + const { commitPlatformConfigurationIntent } = await import("../platform-configuration"); + const input = await commitIntent(); + const [before] = await fixture.db + .select() + .from(systemSettings) + .where(eq(systemSettings.key, "platformConfiguration")); + for (const change of [ + { confirmations: { ...input.confirmations, alternateLogin: false } }, + { proof: { ...input.proof, nonce: Buffer.alloc(32) } }, + { sourceRevision: randomUUID() }, + { execution: { ...input.execution, owner: randomUUID() } }, + { expectedDesiredRowVersion: "0" }, + ]) + await expect( + fixture.db.transaction((tx) => + commitPlatformConfigurationIntent({ ...input, ...change }, tx), + ), + ).rejects.toThrow(); + const [after] = await fixture.db + .select() + .from(systemSettings) + .where(eq(systemSettings.key, "platformConfiguration")); + expect(after).toEqual(before); + const [proof] = await fixture.db + .select() + .from(adminActionReauthentication) + .where(eq(adminActionReauthentication.id, input.proof.challengeId)); + expect(proof?.state).toBe("completed"); + }); + + it("issues only the fixed actor/job/preview participant and rejects copied overlay or issuer", async () => { + const { + preparePlatformConfigurationChange, + platformConfigurationParticipation, + assertPlatformConfigurationOverlay, + isPlatformConfigurationParticipant, + assertPlatformConfigurationSource, + retainedPlatformConfigurationIntent, + } = await import("../platform-configuration-participant"); + const { withAdministrativeConfigurationJob, withAdministrativeConfigurationWriter } = + await import("../administrative-configuration-writer-owner"); + const { withConfigurationCommitParticipant } = await import( + "../configuration-commit-participant" + ); + const input = await commitIntent(); + const ctx = { + jobId: input.jobId, + jobType: "platform.apply", + execution: input.execution, + payload: {}, + signal: new AbortController().signal, + checkpointResult: { _execution: input.execution }, + log: async () => {}, + checkpoint: async () => {}, + setProgress: async () => {}, + }; + await withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { + actorId: input.actorId, + operationKey: `opk1_${input.jobId.replaceAll("-", "")}`, + integrationIds: [], + }, + async (writer) => { + const intent = await preparePlatformConfigurationChange(input, writer); + const { overlay, participant } = platformConfigurationParticipation(intent, writer); + expect(() => { + const change = overlay.changes[0]; + if (change?.kind !== "platform") throw Error("Missing platform overlay"); + change.patch.domain = { source: "database", value: "substituted.example.test" }; + }).toThrow(); + expect(isPlatformConfigurationParticipant(participant, writer)).toBe(true); + expect(isPlatformConfigurationParticipant({ ...participant }, writer)).toBe(false); + await withConfigurationCommitParticipant(writer, participant, async () => { + expect(() => assertPlatformConfigurationOverlay(writer, overlay)).not.toThrow(); + const source = { + revisionId: input.expectedCurrentRevisionId, + sourceRevision: input.sourceRevision, + services: [], + }; + const sourceInput = { + env: { DOMAIN: "before.example.test" }, + services: [], + resolvedServiceConfigs: new Map(), + }; + expect(() => + assertPlatformConfigurationSource(writer, source, sourceInput), + ).not.toThrow(); + expect(() => + assertPlatformConfigurationSource( + writer, + { ...source, sourceRevision: randomUUID() }, + sourceInput, + ), + ).toThrow(); + const metadata = retainedPlatformConfigurationIntent(writer); + expect(JSON.stringify(metadata)).not.toContain(input.proof.nonce.toString("base64")); + expect(JSON.stringify(metadata)).not.toContain(input.proof.sessionId); + expect(() => + assertPlatformConfigurationOverlay(writer, { ...overlay, changes: [] }), + ).toThrow(); + }); + expect(() => platformConfigurationParticipation({ ...intent }, writer)).toThrow(); + }, + ), + ); + }); + it("serializes competing commits without consuming or saving twice", async () => { + const { commitPlatformConfigurationIntent } = await import("../platform-configuration"); + const input = await commitIntent(); + const results = await Promise.allSettled( + [1, 2].map(() => + fixture.db.transaction((tx) => commitPlatformConfigurationIntent(input, tx)), + ), + ); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + const [row] = await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.previewFingerprint, input.previewFingerprint)); + expect(row?.consumedByJobId).toBe(input.jobId); + }); }); diff --git a/apps/api/src/services/administrative-configuration-writer-owner.ts b/apps/api/src/services/administrative-configuration-writer-owner.ts index 544dafa9f..638f229c6 100644 --- a/apps/api/src/services/administrative-configuration-writer-owner.ts +++ b/apps/api/src/services/administrative-configuration-writer-owner.ts @@ -53,7 +53,8 @@ export interface ConfigurationCommitParticipant { | "integration-binding" | "configuration-migration" | "credential-import" - | "transit-credentials"; + | "transit-credentials" + | "platform"; readonly mode: "preview" | "apply"; readonly actorId: string; readonly operationKey: string; diff --git a/apps/api/src/services/configuration-commit-participant.ts b/apps/api/src/services/configuration-commit-participant.ts index c92ae6925..d9e54284d 100644 --- a/apps/api/src/services/configuration-commit-participant.ts +++ b/apps/api/src/services/configuration-commit-participant.ts @@ -18,11 +18,15 @@ export async function withConfigurationCommitParticipant( ); const { isIntegrationBindingParticipant } = await import("./integration-binding-participant"); const { isTransitCredentialParticipant } = await import("./transit-feed-credential-participant"); + const { isPlatformConfigurationParticipant } = await import( + "./platform-configuration-participant" + ); // Exact installed built-in issuer, never a registration or caller callback. if ( (!isLegacyIntegrationImportParticipant(participant, writer) && !isIntegrationBindingParticipant(participant, writer) && - !isTransitCredentialParticipant(participant, writer)) || + !isTransitCredentialParticipant(participant, writer) && + !isPlatformConfigurationParticipant(participant, writer)) || participant.actorId !== writer.actorId || participant.operationKey !== writer.operationKey || (state.participant && state.participant !== participant) diff --git a/apps/api/src/services/configuration-maintenance-decision.ts b/apps/api/src/services/configuration-maintenance-decision.ts index 26eadbd42..f20ce2a83 100644 --- a/apps/api/src/services/configuration-maintenance-decision.ts +++ b/apps/api/src/services/configuration-maintenance-decision.ts @@ -15,7 +15,13 @@ const version = z.string().regex(/^\d{1,10}$/), id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); export const configurationSavedRowSchema = z .strictObject({ - kind: z.enum(["service-config", "integration-config", "service-secret", "integration-secret"]), + kind: z.enum([ + "service-config", + "integration-config", + "service-secret", + "integration-secret", + "platform", + ]), ownerId: id, key: z.string().min(1).max(128).optional(), rowId: z.string().min(1).max(128), @@ -25,6 +31,10 @@ export const configurationSavedRowSchema = z }) .refine( (row) => + (row.kind !== "platform" || + (row.ownerId === "platform" && + row.rowId === "platformConfiguration" && + row.state === "present")) && row.kind.endsWith("secret") === (row.key !== undefined) && (row.state === "present" ? row.rowVersion !== null : row.rowVersion === null), ); @@ -90,6 +100,15 @@ export async function assertConfigurationSavedRows( const key = `${row.kind}:${row.ownerId}:${row.key ?? ""}`; if (seen.has(key)) throw Error("Configuration saved row ownership changed"); seen.add(key); + if (row.kind === "platform") { + const [actual] = await tx + .select({ version: sql`xmin::text` }) + .from(schema.systemSettings) + .where(eq(schema.systemSettings.key, "platformConfiguration")); + if (actual?.version !== row.rowVersion) + throw Error("Configuration saved row ownership changed"); + continue; + } const table = row.kind === "service-config" ? schema.serviceConfig diff --git a/apps/api/src/services/platform-configuration-participant.ts b/apps/api/src/services/platform-configuration-participant.ts new file mode 100644 index 000000000..fe993c98f --- /dev/null +++ b/apps/api/src/services/platform-configuration-participant.ts @@ -0,0 +1,330 @@ +/** Sole built-in platform issuer. Parsed previews/overlays and copied objects + * are data; only this writer-bound instance participates in a desired save. */ +import { randomUUID } from "node:crypto"; +import type { AdminPlatformPreview } from "@openmapx/core/api-contracts"; +import { type ConfigurationSourcesInspection, canonicalJson } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { eq, sql } from "drizzle-orm"; +import { AdminActionReauthentication } from "../account/admin-reauthentication"; +import { loadAuthPolicy } from "../account/auth-policy"; +import { platformConfigurationPreview, systemSettings } from "../db/schema"; +import { + type AdministrativeConfigurationWriter, + administrativeConfigurationState, + type ConfigurationCommitParticipant, + type ConfigurationWriteTransaction, +} from "./administrative-configuration-writer-owner"; +import { + commitPlatformConfigurationIntent, + PLATFORM_CONFIGURATION_SETTING, + type PlatformConfigurationCommitInput, +} from "./platform-configuration"; +import { PlatformPreviewConflict, readPlatformPreview } from "./platform-preview-store"; +import type { + ProspectiveConfigurationOverlay, + ValidatedConfigurationCandidate, +} from "./prospective-configuration"; +import { encrypt } from "./secrets"; + +declare const intentBrand: unique symbol; +export interface PlatformConfigurationIntent { + readonly [intentBrand]: true; +} +interface State { + writer: AdministrativeConfigurationWriter; + input: Pick< + PlatformConfigurationCommitInput, + "previewFingerprint" | "expectedCurrentRevisionId" | "confirmations" | "proof" + >; + preview: AdminPlatformPreview; + expectedDesiredRowVersion: string | null; + handoffId: string; + authorization?: ReturnType; + overlay: ProspectiveConfigurationOverlay; + participant: ConfigurationCommitParticipant; + committed?: { candidateId: string; revisionId: string; rowVersion: string }; +} +function immutable(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) immutable(child); + Object.freeze(value); + } + return value; +} +const intents = new WeakMap(), + participants = new WeakMap(); +function active(writer: AdministrativeConfigurationWriter) { + const owner = administrativeConfigurationState(writer); + if (owner.job.jobType !== "platform.apply" || !owner.job.execution) + throw new PlatformPreviewConflict(); + return owner; +} +export function isPlatformConfigurationParticipant( + participant: ConfigurationCommitParticipant, + writer: AdministrativeConfigurationWriter, +) { + return participants.get(participant)?.writer === writer; +} +function stateFor(writer: AdministrativeConfigurationWriter) { + const owner = active(writer), + state = owner.participant && participants.get(owner.participant); + if (!state || state.writer !== writer) throw new PlatformPreviewConflict(); + return state; +} +export function assertPlatformConfigurationOverlay( + writer: AdministrativeConfigurationWriter, + overlay: unknown, +) { + const state = stateFor(writer); + if (canonicalJson(overlay) !== canonicalJson(state.overlay)) throw new PlatformPreviewConflict(); +} +export function platformConfigurationParticipation( + intent: PlatformConfigurationIntent, + writer: AdministrativeConfigurationWriter, +) { + const state = intents.get(intent); + if (!state || state.writer !== writer) throw new PlatformPreviewConflict(); + active(writer); + return { overlay: state.overlay, participant: state.participant }; +} +/** Private, encrypted restart context is counted by the existing candidate's + * size/queue budget. It never enters the public job payload or preview DTO. */ +export function retainedPlatformConfigurationIntent(writer: AdministrativeConfigurationWriter) { + const state = stateFor(writer), + { preview } = state; + return { + version: 1 as const, + handoffId: state.handoffId, + previewId: preview.previewId, + previewFingerprint: preview.previewFingerprint, + authPolicyRevision: preview.authPolicyRevision, + sourceRevision: preview.sourceRevision, + expectedCurrentRevisionId: preview.expectedCurrentRevisionId, + configuration: structuredClone(preview.configuration), + affectedServiceIds: [...preview.affectedServiceIds], + expectedDesiredRowVersion: state.expectedDesiredRowVersion, + confirmations: { ...state.input.confirmations }, + ...(state.authorization ? { authorization: { ...state.authorization } } : {}), + }; +} +export function assertPlatformConfigurationSource( + writer: AdministrativeConfigurationWriter, + source: ConfigurationSourcesInspection, + input: { + env: Readonly>; + services: Parameters[0]["services"]; + resolvedServiceConfigs: Map>; + }, +) { + const state = stateFor(writer), + p = state.preview; + const rebuilt = services.buildPlatformConfigurationPreview({ + actorId: writer.actorId, + previewId: p.previewId, + authPolicyRevision: p.authPolicyRevision, + sourceRevision: source.sourceRevision, + currentRevisionId: source.revisionId, + expectedCurrentRevisionId: p.expectedCurrentRevisionId, + currentPlatform: source.platform, + changes: p.configuration, + ...input, + }); + if (rebuilt.previewFingerprint !== p.previewFingerprint) throw new PlatformPreviewConflict(); +} +export async function preparePlatformConfigurationChange( + input: Pick< + PlatformConfigurationCommitInput, + "previewFingerprint" | "expectedCurrentRevisionId" | "confirmations" | "proof" + >, + writer: AdministrativeConfigurationWriter, +): Promise { + const owner = active(writer), + retained = await readPlatformPreview( + { actorId: writer.actorId, previewFingerprint: input.previewFingerprint }, + owner.connection, + ), + preview = immutable(retained.preview); + if ( + retained.consumedByJobId || + preview.expectedCurrentRevisionId !== input.expectedCurrentRevisionId || + preview.authPolicyRevision !== (await loadAuthPolicy(owner.connection)).revision || + Object.entries(preview.requiredConfirmation).some( + ([key, required]) => + required && input.confirmations[key as keyof typeof input.confirmations] !== true, + ) + ) + throw new PlatformPreviewConflict(); + const proof = input.proof ? { ...input.proof, nonce: Buffer.from(input.proof.nonce) } : undefined; + if (preview.requiredConfirmation.domain) { + if (!proof) throw new PlatformPreviewConflict("Domain recovery proof required"); + await new AdminActionReauthentication(owner.connection).check( + { + ...proof, + purpose: "platform.domain.apply", + targetId: preview.previewFingerprint, + intentFingerprint: preview.previewFingerprint, + }, + owner.connection, + ); + } + const [desired] = await owner.connection + .select({ version: sql`xmin::text` }) + .from(systemSettings) + .where(eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING)); + const { version: _version, ...patch } = preview.configuration; + const overlay: ProspectiveConfigurationOverlay = immutable({ + purpose: "platform", + coordinatorMode: "require-running", + changes: [{ kind: "platform", patch }], + }); + let state: State; + const verify = async ( + actual: AdministrativeConfigurationWriter, + candidate?: ValidatedConfigurationCandidate, + ) => { + if (actual !== writer || participants.get(state.participant) !== state) + throw new PlatformPreviewConflict(); + const current = active(writer); + if (state.committed) { + const [saved] = await current.connection + .select({ value: systemSettings.value, version: sql`xmin::text` }) + .from(systemSettings) + .where(eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING)); + const [consumed] = await current.connection + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.id, preview.previewId)); + if ( + !candidate || + candidate.candidateId !== state.committed.candidateId || + candidate.revisionId !== state.committed.revisionId || + consumed?.consumedByJobId !== current.job.jobId || + consumed.previewFingerprint !== preview.previewFingerprint || + saved?.version !== state.committed.rowVersion || + canonicalJson(saved.value) !== + canonicalJson({ + version: 1, + jobId: current.job.jobId, + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + previewFingerprint: preview.previewFingerprint, + configuration: preview.configuration, + }) + ) + throw new PlatformPreviewConflict(); + return; + } + const stored = await readPlatformPreview( + { actorId: writer.actorId, previewFingerprint: preview.previewFingerprint }, + current.connection, + ); + if ( + stored.consumedByJobId || + canonicalJson(stored.preview) !== canonicalJson(preview) || + (await loadAuthPolicy(current.connection)).revision !== preview.authPolicyRevision + ) + throw new PlatformPreviewConflict(); + const [desired] = await current.connection + .select({ version: sql`xmin::text` }) + .from(systemSettings) + .where(eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING)); + if ((desired?.version ?? null) !== state.expectedDesiredRowVersion) + throw new PlatformPreviewConflict(); + if (proof && preview.requiredConfirmation.domain) + await new AdminActionReauthentication(current.connection).check( + { + ...proof, + purpose: "platform.domain.apply", + targetId: preview.previewFingerprint, + intentFingerprint: preview.previewFingerprint, + }, + current.connection, + ); + if ( + candidate && + (candidate.operationKey !== writer.operationKey || + candidate.expectedCurrentRevisionId !== preview.expectedCurrentRevisionId) + ) + throw new PlatformPreviewConflict(); + }; + const participant = Object.freeze({ + kind: "platform", + mode: "apply", + actorId: writer.actorId, + operationKey: writer.operationKey, + managedServiceSecretKeys: Object.freeze([]), + authorize: (actual: AdministrativeConfigurationWriter) => verify(actual), + verify: ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + ) => verify(actual, candidate), + commit: async ( + candidate: ValidatedConfigurationCandidate, + actual: AdministrativeConfigurationWriter, + tx: ConfigurationWriteTransaction, + ) => { + const current = active(actual); + if ( + actual !== writer || + current.connection !== tx || + !current.transactionActive || + current.transactionMode !== "write" + ) + throw new PlatformPreviewConflict(); + await verify(actual, candidate); + const saved = await commitPlatformConfigurationIntent( + { + actorId: writer.actorId, + jobId: current.job.jobId, + execution: current.job.execution!, + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + expectedAuthPolicyRevision: preview.authPolicyRevision, + sourceRevision: preview.sourceRevision, + expectedDesiredRowVersion: state.expectedDesiredRowVersion, + ...state.input, + }, + current.connection, + ); + state.committed = { + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + rowVersion: saved.rowVersion, + }; + }, + }) as unknown as ConfigurationCommitParticipant; + state = { + writer, + input: { + previewFingerprint: input.previewFingerprint, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + confirmations: { ...input.confirmations }, + ...(proof ? { proof } : {}), + }, + preview, + expectedDesiredRowVersion: desired?.version ?? null, + handoffId: randomUUID(), + overlay, + participant, + ...(proof + ? { + authorization: encrypt( + canonicalJson({ + version: 1, + actorId: writer.actorId, + jobId: owner.job.jobId, + operationKey: writer.operationKey, + previewFingerprint: preview.previewFingerprint, + challengeId: proof.challengeId, + sessionId: proof.sessionId, + nonce: proof.nonce.toString("base64"), + }), + ), + } + : {}), + }; + const intent = Object.freeze({}) as PlatformConfigurationIntent; + intents.set(intent, state); + participants.set(participant, state); + return intent; +} diff --git a/apps/api/src/services/platform-configuration.ts b/apps/api/src/services/platform-configuration.ts index b7ed5f3e4..a8fe56901 100644 --- a/apps/api/src/services/platform-configuration.ts +++ b/apps/api/src/services/platform-configuration.ts @@ -1,9 +1,24 @@ /** Platform target authorization. Host effects remain behind the nominal * configuration participant and typed maintenance/handoff operations. */ +import { randomUUID } from "node:crypto"; +import { + type AdminPlatformPreview, + adminPlatformConfirmationSchema, +} from "@openmapx/core/api-contracts"; +import { configurationGenerationRevisionSchema } from "@openmapx/core/ops"; import { and, eq, sql } from "drizzle-orm"; +import { z } from "zod"; +import { AdminActionReauthentication } from "../account/admin-reauthentication"; import type { AuthDatabase } from "../account/auth-database"; import { loadAuthPolicy } from "../account/auth-policy"; -import { platformConfigurationPreview } from "../db/schema"; +import { + adminAuditLog, + adminJob, + platformConfigurationPreview, + systemSettings, + user, +} from "../db/schema"; +import { jobOwnership } from "./admin-job-state"; import { PlatformPreviewConflict, readPlatformPreview } from "./platform-preview-store"; /** Called in the SAME desired-row transaction as purpose-proof consumption @@ -48,3 +63,159 @@ export async function assertDomainRecoveryPreview( "Platform authentication policy changed; create a new preview", ); } + +export interface PlatformConfigurationCommitInput { + actorId: string; + jobId: string; + execution: { owner: string; version: number }; + candidateId: string; + revisionId: string; + previewFingerprint: string; + expectedCurrentRevisionId: string | null; + expectedAuthPolicyRevision: string; + sourceRevision: string; + expectedDesiredRowVersion: string | null; + confirmations: AdminPlatformPreview["requiredConfirmation"]; + proof?: { challengeId: string; sessionId: string; nonce: Buffer | string }; +} +/** Desired state is deliberately separate from the applied generation. An + * unrelated renderer must never adopt an uncompleted domain intent. */ +export const PLATFORM_CONFIGURATION_SETTING = "platformConfiguration"; + +/** SQL-only part of the closed platform participant. The caller supplies its + * ORIGINAL writer transaction after actual remote preparation. This routine + * grants no host authority and never commits the transaction or dispatches a + * continuation; its preview/proof/desired writes roll back with the caller's + * saved continue checkpoint. Recovery observes that checkpoint, never calls + * this routine a second time or consumes another proof. */ +export async function commitPlatformConfigurationIntent( + input: PlatformConfigurationCommitInput, + tx: AuthDatabase, +) { + z.uuid().parse(input.candidateId); + configurationGenerationRevisionSchema.parse(input.revisionId); + const confirmations = adminPlatformConfirmationSchema.parse(input.confirmations); + const [job] = await tx + .select() + .from(adminJob) + .where(jobOwnership(input.jobId, input.execution)) + .for("update"); + const [actor] = await tx.select().from(user).where(eq(user.id, input.actorId)).for("share"); + if ( + !job || + job.type !== "platform.apply" || + job.createdBy !== input.actorId || + actor?.role !== "admin" || + actor.banned === true + ) + throw new PlatformPreviewConflict(); + // Fixed order also covers the not-yet-activated policy singleton. + await tx.execute(sql`LOCK TABLE auth_policy_state IN SHARE MODE`); + await tx + .select({ id: platformConfigurationPreview.id }) + .from(platformConfigurationPreview) + .where( + and( + eq(platformConfigurationPreview.actorId, input.actorId), + eq(platformConfigurationPreview.previewFingerprint, input.previewFingerprint), + ), + ) + .for("update"); + const retained = await readPlatformPreview(input, tx), + preview = retained.preview; + if ( + retained.consumedByJobId || + preview.sourceRevision !== input.sourceRevision || + preview.expectedCurrentRevisionId !== input.expectedCurrentRevisionId || + preview.authPolicyRevision !== input.expectedAuthPolicyRevision || + (await loadAuthPolicy(tx)).revision !== preview.authPolicyRevision || + Object.entries(preview.requiredConfirmation).some( + ([key, required]) => required && confirmations[key as keyof typeof confirmations] !== true, + ) + ) + throw new PlatformPreviewConflict(); + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended('openmapx.platform.desired',71956))`, + ); + const [prior] = await tx + .select({ value: systemSettings.value, version: sql`xmin::text` }) + .from(systemSettings) + .where(eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING)) + .for("update"); + if ((prior?.version ?? null) !== input.expectedDesiredRowVersion) + throw new PlatformPreviewConflict("Platform desired state changed; create a new preview"); + if (preview.requiredConfirmation.domain) { + await assertDomainRecoveryPreview(input, tx); + if (!input.proof) throw new PlatformPreviewConflict("Domain recovery proof required"); + await new AdminActionReauthentication(tx).consume( + { + ...input.proof, + purpose: "platform.domain.apply", + targetId: preview.previewFingerprint, + intentFingerprint: preview.previewFingerprint, + }, + tx, + ); + } + const value = { + version: 1, + jobId: input.jobId, + candidateId: input.candidateId, + revisionId: input.revisionId, + previewFingerprint: preview.previewFingerprint, + configuration: preview.configuration, + }; + const written = prior + ? await tx + .update(systemSettings) + .set({ value, updatedAt: new Date(), updatedBy: input.actorId }) + .where( + and( + eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING), + sql`xmin::text=${prior.version}`, + ), + ) + .returning({ rowVersion: sql`xmin::text` }) + : await tx + .insert(systemSettings) + .values({ key: PLATFORM_CONFIGURATION_SETTING, value, updatedBy: input.actorId }) + .onConflictDoNothing() + .returning({ rowVersion: sql`xmin::text` }); + if (written.length !== 1) throw new PlatformPreviewConflict(); + const consumed = await tx + .update(platformConfigurationPreview) + .set({ consumedByJobId: input.jobId, consumedAt: new Date() }) + .where( + and( + eq(platformConfigurationPreview.id, preview.previewId), + sql`${platformConfigurationPreview.consumedAt} IS NULL`, + sql`${platformConfigurationPreview.expiresAt}>now()`, + ), + ) + .returning({ id: platformConfigurationPreview.id }); + if (consumed.length !== 1) throw new PlatformPreviewConflict(); + await tx.insert(adminAuditLog).values({ + id: randomUUID(), + actorId: input.actorId, + targetType: "platform", + targetId: preview.previewId, + action: "platform.configuration.save", + details: { + previewFingerprint: preview.previewFingerprint, + candidateId: input.candidateId, + revisionId: input.revisionId, + oldOrigin: preview.oldOrigin, + newOrigin: preview.newOrigin, + }, + }); + // Only non-secret row identity/version enters the retained decision checksum. + const result = { + kind: "platform" as const, + ownerId: "platform", + rowId: PLATFORM_CONFIGURATION_SETTING, + previousRowVersion: prior?.version ?? null, + rowVersion: written[0]!.rowVersion, + state: "present" as const, + }; + return result; +} diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 72b563c99..870988491 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -122,6 +122,11 @@ import { capturedBindingState } from "./integration-binding-state"; import { integrationConfigRevision } from "./integration-runtime-evidence"; import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { + assertPlatformConfigurationOverlay, + assertPlatformConfigurationSource, + retainedPlatformConfigurationIntent, +} from "./platform-configuration-participant"; import { encrypt } from "./secrets"; import { resolveServiceConfigWithSourcesStrict } from "./service-config-resolver"; import { getServiceRegistry, resolveRequiresForIntegration } from "./service-registry"; @@ -338,6 +343,19 @@ async function verifyConfigurationCommitParticipant( assertLegacyIntegrationImportOverlay(writer, state.overlay); if (state.overlay.purpose === "integration-binding") assertIntegrationBindingOverlay(writer, state.overlay); + if (state.overlay.purpose === "platform") { + assertPlatformConfigurationOverlay(writer, state.overlay); + const intent = retainedPlatformConfigurationIntent(writer); + if ( + state.writer !== writer || + !state.record.maintenanceId || + state.record.apiFollowUp !== "platform-apply" || + state.source?.sourceRevision !== intent.sourceRevision || + canonicalJson(state.payload.platform) !== canonicalJson(intent.configuration) || + canonicalJson(state.record.privatePlatformIntent) !== canonicalJson(intent) + ) + throw Error("Original platform candidate required"); + } await owner.participant?.verify(candidate, writer); } export function assertCommittedConfigurationCandidate( @@ -2792,14 +2810,15 @@ function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWr if (overlay.purpose === "integration-binding") assertIntegrationBindingOverlay(writer, overlay); if (overlay.purpose === "migration" && state.participant?.kind !== "configuration-migration") throw Error("Configuration commit participant unavailable"); + if (overlay.purpose === "platform") assertPlatformConfigurationOverlay(writer, overlay); if ( - overlay.purpose === "platform" || - overlay.changes.some((change) => change.kind === "platform") + overlay.changes.some((change) => change.kind === "platform") && + overlay.purpose !== "platform" ) - throw Error("Platform configuration adapter unavailable"); + throw Error("Platform configuration participant required"); const fields = new Set(); for (const change of overlay.changes) { - if (change.kind === "platform") throw Error("Platform configuration adapter unavailable"); + if (change.kind === "platform") continue; const target = "serviceId" in change ? change.serviceId @@ -2865,8 +2884,11 @@ function validateOverlay(overlay: Overlay, writer: AdministrativeConfigurationWr "integration-binding": "integration-binding", "transit-credentials": "service-secret", selection: "selection", + platform: "platform", } as const; const kind = permitted[overlay.purpose as keyof typeof permitted]; + if (overlay.purpose === "platform" && overlay.changes.length !== 1) + throw Error("Single fixed platform intent required"); if (kind && overlay.changes.some((change) => change.kind !== kind)) throw Error("Configuration purpose conflicts with overlay"); } @@ -3056,6 +3078,8 @@ export async function prepareProspectiveConfiguration(input: { : null; input.signal?.throwIfAborted(); const overlay = overlaySchema.parse(input.overlay); + const platformIntent = + overlay.purpose === "platform" ? retainedPlatformConfigurationIntent(writer) : undefined; if (overlay.purpose === "transit-credentials") { const { assertTransitCredentialOverlay } = await import( "./transit-feed-credential-participant" @@ -3093,10 +3117,18 @@ export async function prepareProspectiveConfiguration(input: { const selection = overlay.changes.find((change) => change.kind === "selection"); const payload = trustedConfigurationPayloadSchema.parse({ ...(await withAdministrativeConfigurationReadScope(read, () => - buildConfigurationInput(read, source.platform), + buildConfigurationInput(read, platformIntent?.configuration ?? source.platform), )), ...(selection?.kind === "selection" ? { selectedRoots: selection.selectedRoots } : {}), }); + if (platformIntent) + assertPlatformConfigurationSource(writer, source, { + services: context.services, + env: context.env, + resolvedServiceConfigs: new Map( + payload.serviceConfigs.map((entry) => [entry.serviceId, entry.values]), + ), + }); const integrationReloadSnapshot = await withAdministrativeConfigurationReadScope( read, async () => @@ -3127,17 +3159,21 @@ export async function prepareProspectiveConfiguration(input: { throw Error("Configuration schema rejected"); } const candidateId = coordinator?.candidateId ?? randomUUID(); - const affectedServiceIds = coordinator - ? ["data-manager"] - : [ - ...new Set( - overlay.changes.flatMap((change) => - change.kind !== "integration-binding" && "serviceId" in change && change.serviceId - ? [change.serviceId] - : [], + const affectedServiceIds = platformIntent + ? [...platformIntent.affectedServiceIds] + : coordinator + ? ["data-manager"] + : [ + ...new Set( + overlay.changes.flatMap((change) => + change.kind !== "integration-binding" && + "serviceId" in change && + change.serviceId + ? [change.serviceId] + : [], + ), ), - ), - ].sort(); + ].sort(); const kind = overlay.purpose === "selection" ? "serviceSelection.apply" @@ -3151,11 +3187,17 @@ export async function prepareProspectiveConfiguration(input: { if (input.followUp && (kind !== "vault.apply" || affectedServiceIds.length !== 1)) throw Error("Invalid credential continuation"); const maintenanceId = - coordinator || (input.followUp && affectedServiceIds[0] === "data-manager") + platformIntent || + coordinator || + (input.followUp && affectedServiceIds[0] === "data-manager") ? randomUUID() : null; const apiFollowUp = - (coordinator ? (`coordinator-${coordinator.action}` as const) : input.followUp) ?? + (platformIntent + ? "platform-apply" + : coordinator + ? (`coordinator-${coordinator.action}` as const) + : input.followUp) ?? (kind === "integrationConfig.apply" || (overlay.purpose === "migration" && owner.participant?.kind === "configuration-migration") ? "integration-reload" @@ -3264,6 +3306,7 @@ export async function prepareProspectiveConfiguration(input: { parentDeadlineAt: root.parent.deadlineAt, affectedServiceIds, apiFollowUp, + ...(platformIntent ? { privatePlatformIntent: platformIntent } : {}), ...(coordinatorLifecycle ? { privateCoordinatorLifecycle: coordinatorLifecycle, @@ -3388,6 +3431,7 @@ export async function prepareProspectiveConfiguration(input: { overlay.coordinatorMode === "require-running" && (state.preview.entries || coordinator || + platformIntent || (maintenanceId && input.followUp === "service-apply")) && state.preview.coordinatorState !== "running" ) @@ -3481,6 +3525,16 @@ function assertMaintenanceIntentCapacity(state: CandidateState, result: unknown) function savedRowIntents(state: CandidateState) { const { rows, sourceRows } = state; + if (state.record.privatePlatformIntent) + return [ + { + kind: "platform" as const, + ownerId: "platform", + rowId: "platformConfiguration", + previousRowVersion: state.record.privatePlatformIntent.expectedDesiredRowVersion, + state: "present" as const, + }, + ]; if (!rows || !sourceRows) throw Error("Original prospective desired state required"); if (state.record.privateCoordinatorLifecycle) { const rowVersion = (row: object) => { @@ -3624,18 +3678,31 @@ async function readSavedRowVersions( rowVersion: sql`xmin::text`, }) .from(schema.integrationSecret); - const receipts = savedRowIntents(state).map((intent) => { + const intents = savedRowIntents(state); + const platformRows = intents.some((row) => row.kind === "platform") + ? await tx + .select({ + id: schema.systemSettings.key, + ownerId: sql`'platform'`, + rowVersion: sql`xmin::text`, + }) + .from(schema.systemSettings) + .where(eq(schema.systemSettings.key, "platformConfiguration")) + : []; + const receipts = intents.map((intent) => { const rows = - intent.kind === "service-config" - ? serviceRows - : intent.kind === "integration-config" - ? integrationRows - : intent.kind === "service-secret" - ? serviceSecrets - : integrationSecrets; + intent.kind === "platform" + ? platformRows + : intent.kind === "service-config" + ? serviceRows + : intent.kind === "integration-config" + ? integrationRows + : intent.kind === "service-secret" + ? serviceSecrets + : integrationSecrets; + const key = "key" in intent ? intent.key : undefined; const matches = rows.filter( - (row) => - row.ownerId === intent.ownerId && (!intent.key || ("key" in row && row.key === intent.key)), + (row) => row.ownerId === intent.ownerId && (!key || ("key" in row && row.key === key)), ); if (intent.state === "absent") { if (matches.length) throw Error("Configuration deletion receipt conflict"); @@ -4269,7 +4336,7 @@ export async function persistAndApplyProspectiveConfiguration( ) throw Error("Configuration candidate persistence refused"); if ( - ["migration", "transit-credentials"].includes(state.overlay.purpose) && + ["migration", "transit-credentials", "platform"].includes(state.overlay.purpose) && !owner.participant && !( state.overlay.purpose === "transit-credentials" && diff --git a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts index 321fb9af1..079fd2588 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts @@ -23,6 +23,7 @@ export const configurationMaintenanceForwardGraphSchema = z.strictObject({ parentDeadlineAt: z.number().int().positive().safe(), planDigest: digest, coordinatorImage: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.coordinatorImage), + handoff: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.handoff), serviceIds: z.lazy(() => configurationMaintenanceReceiptSchema.shape.serviceIds), steps: z.lazy(() => configurationMaintenanceReceiptSchema.shape.steps), }); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index fab32911b..591dc4204 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -2970,6 +2970,7 @@ export function publishConfigurationMaintenanceForwardWitness( parentDeadlineAt: receipt.parentDeadlineAt, planDigest: receipt.planDigest, ...(owner.plan.coordinatorImage ? { coordinatorImage: owner.plan.coordinatorImage } : {}), + ...(owner.plan.handoff ? { handoff: owner.plan.handoff } : {}), serviceIds: receipt.serviceIds, steps: receipt.steps, }); @@ -3168,6 +3169,7 @@ export function verifyConfigurationMaintenanceGenerationWitness( sourceAuthorityRevision: "not-shared", previewDigest: "0".repeat(64), ...(graph.coordinatorImage ? { coordinatorImage: graph.coordinatorImage } : {}), + ...(graph.handoff ? { handoff: graph.handoff } : {}), hardlinkPlan: request.plan, serviceIds: root.serviceIds, recreateServiceIds: graph.steps.flatMap((step) => diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts index f2ecc1aa7..6474fcb19 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -178,3 +178,97 @@ it("retains the frozen coordinator image in the cross-role graph used to reconst expect(rebuilt.planDigest).toBe(plan.planDigest); expect(rebuilt.steps).toEqual(plan.steps); }); + +it("freezes protected handoff after the original generation and ordinary children", async () => { + const source = input(); + const { serviceId: _serviceId, ...ordinary } = source.apply as Extract< + typeof source.apply, + { kind: "vault.apply" } + >; + const apply = { ...ordinary, kind: "stack.render" as const }; + const operation = { + ...source.operation, + followUp: "platform-apply" as const, + applyOperationFingerprint: opsOperationFingerprint(apply), + }; + const operationFingerprint = opsOperationFingerprint(operation); + const prepare = { + ...source.prepare, + operationFingerprint, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt: source.prepare.deadlineAt, + parent: source.apiParent, + }), + }; + const handoff = { + handoffId: "77777777-7777-4777-8777-777777777777", + previewFingerprint: "e".repeat(64), + serviceIds: ["app-api", "ops-agent"], + expectedImageIdentities: { + "app-api": `sha256:${"1".repeat(64)}`, + "ops-agent": `sha256:${"2".repeat(64)}`, + }, + mode: { kind: "configuration" as const }, + }; + const args = { + ...source, + apply, + operation, + prepare, + serviceIds: ["app-api", "data-manager", "motis", "ops-agent"], + handoff, + }; + const plan = buildFixedConfigurationMaintenancePlan(args); + expect(plan.steps.map((s) => s.phase)).toEqual([ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ]); + const step = plan.steps.at(-1)!; + expect(step.operation).toMatchObject({ + kind: "configurationHandoff.prepare", + handoffId: handoff.handoffId, + expectedCurrentRevisionId: operation.revisionId, + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + planDigest: plan.planDigest, + mode: { kind: "configuration" }, + }); + expect(step.prerequisites).toEqual([2]); + expect(plan.nestedHandoff).toMatchObject({ + phase: "configuration-handoff", + ordinal: 0, + kind: "configuration-helper", + recipient: "configuration-helper", + serviceIds: handoff.serviceIds, + }); + const { configurationMaintenanceForwardGraphSchema } = await import( + "./configuration-maintenance-forward-records" + ); + const { kind: _kind, ...binding } = operation; + if (source.apiParent.issuer !== "api") throw Error("API plan fixture required"); + const graph = configurationMaintenanceForwardGraphSchema.parse({ + version: 1, + binding, + prepare, + apiJobId: source.apiParent.apiJobId, + parentOperationKey: source.apiParent.operationKey, + parentFingerprint: source.apiParent.fingerprint, + parentDeadlineAt: source.apiParent.deadlineAt, + planDigest: plan.planDigest, + serviceIds: plan.serviceIds, + steps: plan.steps, + handoff: plan.handoff, + }); + expect(graph).toHaveProperty("handoff", handoff); + expect(plan.nestedHandoff?.execution.operationKey).toBe( + controlChildOperationKey(step.operationKey, "configuration-handoff", 0), + ); + expect(() => assertFixedConfigurationMaintenancePlan(plan, source.installationId)).not.toThrow(); + const changed = structuredClone(plan); + changed.handoff!.expectedImageIdentities["app-api"] = `sha256:${"3".repeat(64)}`; + expect(() => assertFixedConfigurationMaintenancePlan(changed, source.installationId)).toThrow(); +}); diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts index 3069b8b6c..5bd4cb5cc 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts @@ -2,6 +2,10 @@ * production admission/store may turn one into original owner authority. */ import { createHash } from "node:crypto"; import { z } from "zod"; +import { + type ConfigurationHandoffIntent, + configurationHandoffIntentSchema, +} from "../ops/configuration-handoff-records"; import { type ConfigurationMaintenancePrepare, type ConfigurationMaintenanceStep, @@ -56,6 +60,8 @@ export const fixedConfigurationMaintenancePlanSchema = z.strictObject({ sourceAuthorityRevision: z.string().min(1).max(128), previewDigest: digest, coordinatorImage: coordinatorImageSchema.optional(), + handoff: configurationHandoffIntentSchema.optional(), + nestedHandoff: z.lazy(() => controlChildReservationSchema).optional(), serviceIds: ids, planDigest: digest, hardlinkPlan: generationHardlinksPlanSchema, @@ -86,7 +92,7 @@ function address( root: string, phase: string, ordinal: number, - executor: "ops-agent" | "data-manager", + executor: "ops-agent" | "data-manager" | "host-cli", ) { const bytes = createHash("sha256") .update(`openmapx-fixed-configuration-maintenance-v1\0${root}\0${phase}\0${ordinal}`) @@ -103,7 +109,7 @@ function address( })(); return { executor, - principal: "api" as const, + principal: executor === "host-cli" ? ("host-cli" as const) : ("api" as const), executionId, operationKey: controlChildOperationKey( root, @@ -132,6 +138,7 @@ export function buildFixedConfigurationMaintenancePlan(input: { serviceIds: readonly string[]; recreateServiceIds: readonly string[]; coordinatorImage?: z.infer; + handoff?: ConfigurationHandoffIntent; }): FixedConfigurationMaintenancePlan { const prepare = executionRefSchema.parse(input.prepare), operation = configurationMaintenancePrepareSchema.parse(input.operation), @@ -144,7 +151,6 @@ export function buildFixedConfigurationMaintenancePlan(input: { prepare.executor !== "ops-agent" || prepare.principal !== "api" || prepare.operationFingerprint !== opsOperationFingerprint(operation) || - operation.followUp === "platform-apply" || ![ "stack.render", "serviceSelection.apply", @@ -176,6 +182,13 @@ export function buildFixedConfigurationMaintenancePlan(input: { ) throw Error("Fixed maintenance candidate binding conflict"); const recreates = [...new Set(input.recreateServiceIds)].sort(); + const handoff = input.handoff ? configurationHandoffIntentSchema.parse(input.handoff) : undefined; + if ( + handoff && + (operation.followUp !== "platform-apply" || + handoff.serviceIds.some((id) => !serviceIds.includes(id) || recreates.includes(id))) + ) + throw Error("Fixed protected handoff scope conflict"); if ( input.coordinatorImage && (canonicalJson(recreates) !== '["data-manager"]' || @@ -197,7 +210,9 @@ export function buildFixedConfigurationMaintenancePlan(input: { throw Error("Fixed maintenance continuation conflict"); const intents: { phase: ConfigurationMaintenanceStep["phase"]; - operation: OpsOperation; + operation: + | OpsOperation + | Omit, "planDigest">; prerequisites: number[]; }[] = [{ phase: "configuration-commit", operation: apply, prerequisites: [] }]; if (hardlinkPlan.plan.length) @@ -223,6 +238,18 @@ export function buildFixedConfigurationMaintenancePlan(input: { }, prerequisites: [intents.length - 1], }); + if (handoff) + intents.push({ + phase: "maintenance-handoff", + operation: { + ...handoff, + kind: "configurationHandoff.prepare", + maintenanceId: operation.maintenanceId, + candidateId: operation.candidateId, + expectedCurrentRevisionId: operation.revisionId, + }, + prerequisites: [intents.length - 1], + }); if (intents.length > 32) throw Error("Fixed maintenance effect capacity exceeded"); const { kind: _kind, ...binding } = operation; const planDigest = createHash("sha256") @@ -233,6 +260,7 @@ export function buildFixedConfigurationMaintenancePlan(input: { binding, serviceIds, ...(input.coordinatorImage ? { coordinatorImage: input.coordinatorImage } : {}), + ...(handoff ? { handoff } : {}), steps: intents.map((step, ordinal) => ({ ...step, ordinal, serviceIds })), }), ) @@ -244,7 +272,7 @@ export function buildFixedConfigurationMaintenancePlan(input: { ? intent.operation : { ...intent.operation, planDigest }; const execution = address(prepare.operationKey, intent.phase, ordinal, "ops-agent"), - operationFingerprint = opsOperationFingerprint(body as OpsOperation); + operationFingerprint = opsOperationFingerprint(opsOperationSchema.parse(body)); reservations.push({ version: 1, phase: intent.phase, @@ -317,6 +345,24 @@ export function buildFixedConfigurationMaintenancePlan(input: { operationFingerprint: controlDigest({ kind: "configuration-maintenance.settle", ...dmSettle }), }); const middle = reservations.find((value) => value.phase === "maintenance-hardlinks"); + const handoffStep = steps.find((step) => step.phase === "maintenance-handoff"); + const nestedHandoff = + handoff && handoffStep + ? { + version: 1 as const, + phase: "configuration-handoff" as const, + ordinal: 0, + kind: "configuration-helper", + recipient: "configuration-helper" as const, + serviceIds: handoff.serviceIds, + execution: address(handoffStep.operationKey, "configuration-handoff", 0, "host-cli"), + operationFingerprint: controlDigest({ + kind: "configuration-helper", + handoffId: handoff.handoffId, + intentDigest: controlDigest(handoffStep.operation), + }), + } + : undefined; return freeze( fixedConfigurationMaintenancePlanSchema.parse({ version: 1, @@ -327,6 +373,7 @@ export function buildFixedConfigurationMaintenancePlan(input: { sourceAuthorityRevision: input.sourceAuthorityRevision, previewDigest: input.previewDigest, ...(input.coordinatorImage ? { coordinatorImage: input.coordinatorImage } : {}), + ...(handoff ? { handoff, nestedHandoff } : {}), serviceIds, planDigest, hardlinkPlan, @@ -371,6 +418,7 @@ export function assertFixedConfigurationMaintenancePlan( sourceAuthorityRevision: plan.sourceAuthorityRevision, previewDigest: plan.previewDigest, ...(plan.coordinatorImage ? { coordinatorImage: plan.coordinatorImage } : {}), + ...(plan.handoff ? { handoff: plan.handoff } : {}), hardlinkPlan: plan.hardlinkPlan, serviceIds: plan.serviceIds, recreateServiceIds, diff --git a/packages/core/src/ops/configuration-handoff-records.ts b/packages/core/src/ops/configuration-handoff-records.ts new file mode 100644 index 000000000..9f0369c22 --- /dev/null +++ b/packages/core/src/ops/configuration-handoff-records.ts @@ -0,0 +1,109 @@ +/** Fixed handoff data. A matching DTO never supplies target authority. */ +import { z } from "zod"; + +const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +export const protectedConfigurationServiceIdSchema = z.enum([ + "postgis", + "redis", + "app-web", + "app-api", + "traefik", + "ops-agent", +]); +export type ProtectedConfigurationServiceId = z.infer; +export const configurationHandoffModeSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("configuration") }), + z.strictObject({ + kind: z.literal("release"), + releaseId: z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/), + manifestDigest: digest, + }), +]); +export const configurationHandoffIntentSchema = z + .strictObject({ + handoffId: z.uuid(), + previewFingerprint: digest, + serviceIds: z + .array(id) + .min(1) + .max(64) + .refine((ids) => ids.every((v, i) => i === 0 || ids[i - 1]! < v)), + expectedImageIdentities: z.record(id, z.string().regex(/^sha256:[a-f0-9]{64}$/)), + mode: configurationHandoffModeSchema, + }) + .superRefine((value, ctx) => { + if ( + JSON.stringify(value).length > 24576 || + JSON.stringify(Object.keys(value.expectedImageIdentities).sort()) !== + JSON.stringify(value.serviceIds) || + (value.mode.kind === "configuration" && + value.serviceIds.some((id) => !protectedConfigurationServiceIdSchema.safeParse(id).success)) + ) + ctx.addIssue({ code: "custom", message: "Invalid frozen handoff target/image set" }); + }); +export type ConfigurationHandoffIntent = z.infer; +export const configurationHandoffPrepareOperationSchema = + configurationHandoffIntentSchema.safeExtend({ + kind: z.literal("configurationHandoff.prepare"), + maintenanceId: z.uuid(), + candidateId: z.uuid(), + planDigest: digest, + expectedCurrentRevisionId: revision, + }); +export const configurationHandoffInspectOperationSchema = z.strictObject({ + kind: z.literal("configurationHandoff.inspect"), + handoffId: z.uuid(), +}); +export const configurationHandoffPreparedSchema = z.strictObject({ + handoffId: z.uuid(), + phase: z.literal("prepared"), +}); +export const configurationHandoffInspectionSchema = z + .strictObject({ + handoffId: z.uuid(), + phase: z.enum([ + "prepared", + "draining", + "applying", + "verifying", + "applied", + "rolled_back", + "failed", + "recovery_required", + ]), + expectedRevisionId: revision, + appliedRevisionId: revision.nullable(), + completedServiceIds: z.array(id).max(64), + failedServiceIds: z.array(id).max(64), + recoveryCodes: z + .array( + z.enum([ + "helper_unavailable", + "launch_unknown", + "source_changed", + "deadline_expired", + "cancel_pending", + "container_unknown", + "readiness_failed", + "restore_failed", + "receipt_pending", + ]), + ) + .max(16), + }) + .superRefine((value, ctx) => { + if ( + new Set(value.completedServiceIds).size !== value.completedServiceIds.length || + new Set(value.failedServiceIds).size !== value.failedServiceIds.length || + value.failedServiceIds.some((id) => value.completedServiceIds.includes(id)) || + (value.phase === "applied" && + (value.appliedRevisionId !== value.expectedRevisionId || value.failedServiceIds.length > 0)) + ) + ctx.addIssue({ code: "custom", message: "Inconsistent handoff observation" }); + }); diff --git a/packages/core/src/ops/configuration-handoff.test.ts b/packages/core/src/ops/configuration-handoff.test.ts new file mode 100644 index 000000000..64d84ad3f --- /dev/null +++ b/packages/core/src/ops/configuration-handoff.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { authorizeOpsOperation, opsOperationSchema, parseOpsResult } from "./contract"; +import { opsExecutionBudget } from "./execution-identity"; + +const operation = () => ({ + kind: "configurationHandoff.prepare", + handoffId: "11111111-1111-4111-8111-111111111111", + maintenanceId: "22222222-2222-4222-8222-222222222222", + candidateId: "33333333-3333-4333-8333-333333333333", + planDigest: "a".repeat(64), + previewFingerprint: "b".repeat(64), + expectedCurrentRevisionId: `cfg1_${"c".repeat(43)}`, + serviceIds: ["app-api"], + expectedImageIdentities: { "app-api": `sha256:${"1".repeat(64)}` }, + mode: { kind: "configuration" }, +}); +describe("fixed protected handoff wire data", () => { + it("has API-only mutation authority and a terminal prepared result, not an applied claim", () => { + const body = opsOperationSchema.parse(operation()); + if (body.kind !== "configurationHandoff.prepare") throw Error("Unexpected operation"); + expect(authorizeOpsOperation("api", body.kind)).toBe(true); + expect(authorizeOpsOperation("data-manager", body.kind)).toBe(false); + expect( + parseOpsResult("configurationHandoff.prepare", { + handoffId: body.handoffId, + phase: "prepared", + }), + ).toEqual({ handoffId: body.handoffId, phase: "prepared" }); + expect(() => + parseOpsResult("configurationHandoff.prepare", { + handoffId: body.handoffId, + phase: "applied", + }), + ).toThrow(); + expect(opsExecutionBudget("configuration-helper").processMs).toBe(3600000); + }); + it("rejects extra paths, arbitrary images, unordered/duplicate targets and unbound target maps", () => { + for (const value of [ + { ...operation(), path: "/other" }, + { ...operation(), image: "arbitrary/image" }, + { ...operation(), serviceIds: ["app-api", "app-api"] }, + { ...operation(), serviceIds: ["ops-agent", "app-api"] }, + { ...operation(), expectedImageIdentities: { "ops-agent": `sha256:${"2".repeat(64)}` } }, + { + ...operation(), + serviceIds: ["motis"], + expectedImageIdentities: { motis: `sha256:${"2".repeat(64)}` }, + }, + ]) + expect(opsOperationSchema.safeParse(value).success).toBe(false); + }); + it("keeps release identity distinct from observed old image IDs without accepting caller pins", () => { + const body = { + ...operation(), + serviceIds: ["motis"], + expectedImageIdentities: { motis: `sha256:${"2".repeat(64)}` }, + mode: { kind: "release", releaseId: "v0.1.0-beta.1", manifestDigest: "d".repeat(64) }, + }; + expect(opsOperationSchema.safeParse(body).success).toBe(true); + expect( + opsOperationSchema.safeParse({ + ...body, + mode: { ...body.mode, imageReferences: { motis: "arbitrary/image" } }, + }).success, + ).toBe(false); + }); +}); + +it("bounds private platform intent without admitting plaintext recovery credentials", async () => { + const { platformConfigurationIntentSchema } = await import("./configuration-preparation-records"); + const value = { + version: 1, + handoffId: operation().handoffId, + previewId: operation().candidateId, + previewFingerprint: "a".repeat(64), + authPolicyRevision: "b".repeat(64), + sourceRevision: operation().maintenanceId, + expectedCurrentRevisionId: operation().expectedCurrentRevisionId, + expectedDesiredRowVersion: null, + configuration: { version: 1, domain: { source: "host" } }, + affectedServiceIds: ["app-api"], + confirmations: { + domain: false, + dnsTls: false, + oauth: false, + passkeys: false, + alternateLogin: false, + }, + }; + expect(platformConfigurationIntentSchema.parse(value)).toEqual(value); + expect( + platformConfigurationIntentSchema.safeParse({ ...value, proof: { nonce: "raw" } }).success, + ).toBe(false); + expect( + platformConfigurationIntentSchema.safeParse({ + ...value, + authorization: { ciphertext: "a".repeat(8193), iv: "x", tag: "x" }, + }).success, + ).toBe(false); +}); diff --git a/packages/core/src/ops/configuration-maintenance.ts b/packages/core/src/ops/configuration-maintenance.ts index 0f540a105..2d80feed0 100644 --- a/packages/core/src/ops/configuration-maintenance.ts +++ b/packages/core/src/ops/configuration-maintenance.ts @@ -15,16 +15,11 @@ const ids = z .max(256) .refine((ids) => ids.every((id, index) => index === 0 || ids[index - 1]! < id)); const sequence = z.number().int().nonnegative().safe(); -/** Reserved08.3 continuation targets; this task never admits that adapter. */ -export const protectedConfigurationServiceIdSchema = z.enum([ - "postgis", - "redis", - "app-web", - "app-api", - "traefik", - "ops-agent", -]); -export type ProtectedConfigurationServiceId = z.infer; + +export { + type ProtectedConfigurationServiceId, + protectedConfigurationServiceIdSchema, +} from "./configuration-handoff-records"; export const configurationMaintenanceBindingSchema = z.strictObject({ maintenanceId: z.uuid(), candidateId: z.uuid(), diff --git a/packages/core/src/ops/configuration-operation-records.ts b/packages/core/src/ops/configuration-operation-records.ts index a1c17957f..de5fa0899 100644 --- a/packages/core/src/ops/configuration-operation-records.ts +++ b/packages/core/src/ops/configuration-operation-records.ts @@ -1,6 +1,7 @@ /** Canonical existing configuration effect inputs, shared by request and * maintenance receipt schemas. Contains no result schema/runtime policy. */ import { z } from "zod"; +import { configurationHandoffPrepareOperationSchema } from "./configuration-handoff-records"; import { coordinatorModeSchema } from "./configuration-preparation-records"; export const serviceIdSchema = z @@ -94,4 +95,5 @@ export const configurationMaintenanceEffectOperationSchema = z.discriminatedUnio serviceConfigApplyOperationSchema, integrationConfigApplyOperationSchema, vaultApplyOperationSchema, + configurationHandoffPrepareOperationSchema, ]); diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts index 3238236bb..5b9625ae2 100644 --- a/packages/core/src/ops/configuration-preparation-records.ts +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -42,6 +42,42 @@ export const configurationHardlinksPreviewSchema = z.strictObject({ }); export type ConfigurationHardlinksPreview = z.infer; +export const platformConfigurationIntentSchema = z + .strictObject({ + version: z.literal(1), + handoffId: z.uuid(), + previewId: z.uuid(), + previewFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + authPolicyRevision: z.string().regex(/^[a-f0-9]{64}$/), + sourceRevision: z.uuid(), + expectedCurrentRevisionId: revision.nullable(), + configuration: platformConfigurationSchema, + affectedServiceIds: z + .array(id) + .max(256) + .refine((ids) => ids.every((v, i) => i === 0 || ids[i - 1]! < v)), + expectedDesiredRowVersion: z + .string() + .regex(/^\d{1,10}$/) + .nullable(), + confirmations: z.strictObject({ + domain: z.boolean(), + dnsTls: z.boolean(), + oauth: z.boolean(), + passkeys: z.boolean(), + alternateLogin: z.boolean(), + }), + authorization: z + .strictObject({ + ciphertext: z.string().min(1).max(8192), + iv: z.string().min(1).max(64), + tag: z.string().min(1).max(64), + }) + .optional(), + }) + .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 32768); +export type PlatformConfigurationIntentRecord = z.infer; + export const preparedConfigurationRecordSchema = z.strictObject({ version: z.literal(1), candidateId: z.uuid(), @@ -64,8 +100,10 @@ export const preparedConfigurationRecordSchema = z.strictObject({ "integration-reload", "coordinator-apply", "coordinator-update", + "platform-apply", ]) .optional(), + privatePlatformIntent: platformConfigurationIntentSchema.optional(), privateCoordinatorLifecycle: coordinatorLifecycleMetadataSchema.optional(), privateCoordinatorSource: configurationSourcesInspectionSchema.optional(), // Exact current DM credential persistence inputs retained before its helper diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index fd3f4e08f..10c1fbdc2 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -66,6 +66,22 @@ const transferClaim = { bytes: 4, }; const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { + "configurationHandoff.prepare": { + kind: "configurationHandoff.prepare", + handoffId: "77777777-7777-4777-8777-777777777777", + maintenanceId: maintenanceBinding.maintenanceId, + candidateId: maintenanceBinding.candidateId, + planDigest: "d".repeat(64), + previewFingerprint: "e".repeat(64), + expectedCurrentRevisionId: maintenanceBinding.revisionId, + serviceIds: ["app-api"], + expectedImageIdentities: { "app-api": `sha256:${"1".repeat(64)}` }, + mode: { kind: "configuration" }, + }, + "configurationHandoff.inspect": { + kind: "configurationHandoff.inspect", + handoffId: "77777777-7777-4777-8777-777777777777", + }, "integration.inventory": { kind: "integration.inventory" }, "integration.inspectOwnership": { kind: "integration.inspectOwnership", diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index f18fde48d..9e2cffae2 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -10,6 +10,12 @@ import { artifactTransferOperationSchemas, artifactTransferResultSchemas, } from "./artifact-transfer"; +import { + configurationHandoffInspectionSchema, + configurationHandoffInspectOperationSchema, + configurationHandoffPreparedSchema, + configurationHandoffPrepareOperationSchema, +} from "./configuration-handoff-records"; import { configurationGenerationRevisionSchema, configurationHardlinksApplyOperationSchema, @@ -96,6 +102,8 @@ export const OPS_OPERATION_KINDS = [ "configurationMaintenance.inspect", "configurationMaintenance.decide", "configurationMaintenance.settle", + "configurationHandoff.prepare", + "configurationHandoff.inspect", "stack.start", "stack.stop", "service.start", @@ -240,6 +248,8 @@ export const OPS_KIND_POLICIES = { "configurationMaintenance.inspect": sync("api", 15_000, 65536), "configurationMaintenance.decide": asyncEffect("api", 60_000, 65536), "configurationMaintenance.settle": asyncEffect("api", 60_000, 65536), + "configurationHandoff.prepare": asyncEffect("api", 3600_000, 65536), + "configurationHandoff.inspect": sync("api", 15_000, 65536), "stack.render": asyncEffect("api", 120_000), "stack.start": asyncEffect("api", LIFECYCLE_TIMEOUT_MS), "stack.stop": asyncEffect("api", 5 * 60_000), @@ -412,6 +422,8 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ kindOnly("configuration.inspect"), kindOnly("configuration.sources.inspect"), configurationMaintenancePrepareSchema, + configurationHandoffPrepareOperationSchema, + configurationHandoffInspectOperationSchema, z.strictObject({ kind: z.literal("configurationMaintenance.inspect"), maintenanceId: z.uuid(), @@ -1092,6 +1104,8 @@ export const OPS_RESULT_SCHEMAS = { "configuration.hardlinks.preview": configurationHardlinksPreviewSchema, "configuration.hardlinks.apply": hardlinkApplySummarySchema, "configurationMaintenance.prepare": configurationMaintenanceReceiptSchema, + "configurationHandoff.prepare": configurationHandoffPreparedSchema, + "configurationHandoff.inspect": configurationHandoffInspectionSchema, "configurationMaintenance.inspect": configurationMaintenanceReceiptSchema, "configurationMaintenance.decide": configurationMaintenanceReceiptSchema, "configurationMaintenance.settle": configurationMaintenanceReceiptSchema, diff --git a/packages/core/src/ops/execution-identity.ts b/packages/core/src/ops/execution-identity.ts index 349e4951f..9776c3a83 100644 --- a/packages/core/src/ops/execution-identity.ts +++ b/packages/core/src/ops/execution-identity.ts @@ -54,6 +54,7 @@ const futureBudgets: Readonly> = { "data.overtureIngest": 24 * 3600_000, "data.overtureExtract": 24 * 3600_000, "platform.config.apply": 3600_000, + "configuration-helper": 3600_000, "download-osm": 24 * 3600_000, "download-fonts": 24 * 3600_000, "convert-overpass": 24 * 3600_000, diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index addd6264e..d5884773c 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -4,6 +4,12 @@ export * from "./artifact-transfer-capability"; export { ArtifactStageTransportError } from "./artifact-transfer-client"; export * from "./catalog-pin-approval"; export * from "./client"; +export { + type ConfigurationHandoffIntent, + configurationHandoffInspectionSchema, + configurationHandoffIntentSchema, + configurationHandoffPrepareOperationSchema, +} from "./configuration-handoff-records"; export * from "./configuration-maintenance"; export * from "./configuration-maintenance-control-validation"; export * from "./configuration-preparation"; From bf95abb010336644be84d3b9d2f34ad16d596df9 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 03:51:46 +0100 Subject: [PATCH 065/135] feat(platform): prepare protected targets under original maintenance ownership --- ...orm-configuration-runtime-postgres.test.ts | 324 ++++++++++++++++++ .../platform-preview-postgres.test.ts | 17 +- .../platform-configuration-participant.ts | 10 + .../src/services/platform-configuration.ts | 2 + .../src/services/platform-preview-store.ts | 51 ++- .../src/services/prospective-configuration.ts | 43 ++- .../src/__fixtures__/owned-service-host.ts | 22 +- .../src/configuration-handoff-store.ts | 267 +++++++++++++++ .../src/configuration-handoff.test.ts | 194 +++++++++++ apps/ops-agent/src/configuration-handoff.ts | 134 ++++++++ .../src/configuration-maintenance-store.ts | 123 ++++++- apps/ops-agent/src/production-runtime.ts | 3 + apps/ops-agent/src/server.ts | 5 +- apps/ops-agent/src/trusted-config-runtime.ts | 38 +- .../configuration-handoff-records.test.ts | 66 ++++ .../configuration-handoff-records.ts | 115 +++++++ .../configuration-maintenance-budget.ts | 45 ++- .../src/ops/configuration-handoff.test.ts | 5 + .../ops/configuration-preparation-records.ts | 3 +- packages/core/src/ops/index.ts | 1 + scripts/database-test-groups.json | 3 +- 21 files changed, 1431 insertions(+), 40 deletions(-) create mode 100644 apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts create mode 100644 apps/ops-agent/src/configuration-handoff-store.ts create mode 100644 apps/ops-agent/src/configuration-handoff.test.ts create mode 100644 apps/ops-agent/src/configuration-handoff.ts create mode 100644 packages/core/src/build-artifacts/configuration-handoff-records.test.ts create mode 100644 packages/core/src/build-artifacts/configuration-handoff-records.ts diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts new file mode 100644 index 000000000..27d91adc6 --- /dev/null +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -0,0 +1,324 @@ +/** Actual API candidate -> production registry/agent -> real DM HTTP/SQL guard. + * Docker inspection is the only substituted I/O; this qualifies preparation, + * not yet helper launch/application or the public platform HTTP routes. */ +import { randomUUID } from "node:crypto"; +import { rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + restoreAdmittedControlExecution, +} from "@openmapx/core/build-artifacts"; +import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; +import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; +import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; +import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; +import * as schema from "../../db/schema"; +import { createMigratedPostgresFixture } from "../../test/postgres-fixture"; +import type { JobContext } from "../job-runner"; + +const state = vi.hoisted(() => ({ + fixture: null as unknown as Awaited>, + services: [] as services.LoadedService[], +})); +vi.mock("../../db", () => ({ + get db() { + return state.fixture.db; + }, + get sql() { + return state.fixture.sql; + }, +})); +vi.mock("../../integration-host", () => ({ getAllIntegrations: () => [] })); +vi.mock("../service-registry", () => ({ + getServiceRegistry: () => ({ list: () => state.services }), +})); +const writer = await import("../administrative-configuration-writer-owner"); +const { withConfigurationCommitParticipant } = await import("../configuration-commit-participant"); +const { preparePlatformConfigurationChange, platformConfigurationParticipation } = await import( + "../platform-configuration-participant" +); +const prospective = await import("../prospective-configuration"); +const { createApiOpsClient, createDurableOpsKey, executeAndWait } = await import("../ops-client"); +const { retainPlatformPreview } = await import("../platform-preview-store"); +const { loadAuthPolicy } = await import("../../account/auth-policy"); +const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); + +describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( + "owned platform preparation", + () => { + beforeAll(async () => { + state.fixture = await createMigratedPostgresFixture(); + }, 90000); + afterAll(async () => { + await state.fixture?.stop(); + }); + it.each(["old", "1"])( + "preflights actual helper image version %s before desired changes", + async (version) => { + const ids = [ + "motis", + "ops-agent", + "app-api", + "traefik", + "postgis", + "redis", + "data-manager", + ]; + const host = await ownedServiceHost({ + withCoordinator: true, + selectedRoots: ids, + environment: { + "ops-agent": { BACKUP_RETENTION_DAYS: "${BACKUP_RETENTION_DAYS:-30}" }, + "app-api": { BACKUP_RETENTION_DAYS: "${BACKUP_RETENTION_DAYS:-30}" }, + }, + configurationHelperVersion: version, + actualComposeConfig: true, + }); + state.services = host.loaded; + host.setRows( + ids.map((id, index) => ({ + ...host.row, + Id: String(index + 1).repeat(64), + ConfigImage: `example/${id}:1`, + Labels: { ...host.row.Labels, "com.docker.compose.service": id }, + })), + ); + const installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: ids, + }); + const dm = await ownedDataManager({ + sql: state.fixture.sql, + infraDir: host.infraDir, + serviceIds: ids, + }); + const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const app = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal, + controlInstallation: installation, + }); + const url = await app.listen({ host: "127.0.0.1", port: 0 }); + const tokenFile = join(host.root, "platform-api-token"); + writeFileSync(tokenFile, host.token, { mode: 0o600 }); + for (const [key, value] of Object.entries({ + OPS_AGENT_URL: url, + OPS_AGENT_TOKEN_FILE: tokenFile, + OPS_TRUSTED_CONFIG_DIR: host.queue, + OPS_TRUSTED_CONFIG_UID: String(statSync(host.queue).uid), + OPS_TRUSTED_CONFIG_GID: String(statSync(host.queue).gid), + OPENMAPX_SECRETS_KEY: "31".repeat(32), + DOMAIN: "example.test", + DATA_MANAGER_URL: dm.baseUrl, + DATA_MANAGER_AUTH_TOKEN: dm.token, + })) + vi.stubEnv(key, value); + try { + const actorId = randomUUID(), + jobId = randomUUID(), + execution = { owner: randomUUID(), version: 1 }, + operationKey = `opk1_${randomUUID().replaceAll("-", "")}`; + await state.fixture.db.insert(schema.user).values({ + id: actorId, + name: "Synthetic admin", + email: `${actorId}@example.test`, + role: "admin", + }); + await state.fixture.db.insert(schema.adminJob).values({ + id: jobId, + type: "platform.apply", + status: "running", + createdBy: actorId, + result: { _execution: execution }, + }); + const ctx: JobContext = { + jobId, + jobType: "platform.apply", + execution, + payload: {}, + signal: new AbortController().signal, + checkpointResult: { _execution: execution }, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (result, progress, key, address) => { + if ((result.opsProjection as { version?: number })?.version === 3) + ctx.checkpointResult = await checkpointOwnedAdminJob( + jobId, + execution, + result, + progress, + key, + address, + ctx.checkpointResult ?? {}, + ); + else { + await state.fixture.db + .update(schema.adminJob) + .set({ result }) + .where(eq(schema.adminJob.id, jobId)); + ctx.checkpointResult = result; + } + }, + }; + await writer.withAdministrativeConfigurationJob(ctx, () => + writer.withAdministrativeConfigurationWriter( + { actorId, operationKey, integrationIds: [] }, + async (owned) => { + const source = await executeAndWait( + createApiOpsClient(), + { kind: "configuration.sources.inspect" }, + createDurableOpsKey("platform.sources", jobId), + ); + const policy = await loadAuthPolicy(state.fixture.db); + const preview = services.buildPlatformConfigurationPreview({ + actorId, + previewId: randomUUID(), + authPolicyRevision: policy.revision, + sourceRevision: source.sourceRevision, + currentRevisionId: source.revisionId, + expectedCurrentRevisionId: source.revisionId, + currentPlatform: source.platform, + changes: { version: 1, backupRetentionDays: { source: "database", value: 14 } }, + env: process.env, + services: state.services, + }); + await retainPlatformPreview( + { + actorId, + operationKey, + requestFingerprint: "a".repeat(64), + preview, + sourceSnapshot: source, + desiredRowVersion: null, + }, + state.fixture.db, + ); + const intent = await preparePlatformConfigurationChange( + { + previewFingerprint: preview.previewFingerprint, + expectedCurrentRevisionId: source.revisionId, + confirmations: preview.requiredConfirmation, + }, + owned, + ); + const participation = platformConfigurationParticipation(intent, owned); + const run = () => + withConfigurationCommitParticipant(owned, participation.participant, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId, + operationKey, + expectedCurrentRevisionId: source.revisionId, + overlay: participation.overlay, + }); + await prospective.stageProspectiveConfigurationMaintenance(candidate); + await prospective.prepareConfigurationMaintenance(candidate); + return candidate; + }); + if (version === "old") + await expect(run()).rejects.toMatchObject({ + result: { configurationPreflight: { errorClass: "host_update_required" } }, + }); + else { + let candidate: Awaited< + ReturnType + >; + try { + candidate = await run(); + } catch (error) { + const value = error as { + result?: { errorCode?: unknown; configurationPreflight?: unknown }; + }; + console.info( + "platform-preflight-diagnostic", + JSON.stringify({ + errorCode: value.result?.errorCode, + preflight: value.result?.configurationPreflight, + }), + ); + throw error; + } + const retained = ctx.checkpointResult?._configurationMaintenance as + | { phase?: string; receipt?: { state?: string; steps?: { phase: string }[] } } + | undefined; + expect(retained?.receipt?.state).toBe("prepared"); + expect(retained?.receipt?.steps?.at(-1)?.phase).toBe("maintenance-handoff"); + expect(candidate.affectedServiceIds).toEqual(["app-api", "ops-agent"]); + const receipt = await prospective.inspectConfigurationMaintenance({ + maintenanceId: + prospective.validatedConfigurationCandidateState(candidate).record + .maintenanceId!, + candidateId: candidate.candidateId, + }); + const { readOwnedConfigurationMaintenance } = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ); + const owner = readOwnedConfigurationMaintenance( + restoreAdmittedControlExecution(installation, receipt.prepare), + ); + expect(owner.plan.handoff?.expectedImageIdentities).toEqual({ + "app-api": host.row.Image, + "ops-agent": host.row.Image, + }); + expect(owner.plan.nestedHandoff?.recipient).toBe("configuration-helper"); + expect(receipt.binding.expectedSourceRevision).toBe(source.sourceRevision); + const abandon = + await prospective.prepareConfigurationMaintenanceAbandon(candidate); + await prospective.decideConfigurationMaintenance(abandon); + const unused = await prospective.inspectConfigurationMaintenance({ + maintenanceId: receipt.binding.maintenanceId, + candidateId: candidate.candidateId, + }); + const settled = await prospective.settleConfigurationMaintenance({ + kind: "configurationMaintenance.settle", + maintenanceId: receipt.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: unused.prepare.operationKey, + prepareFingerprint: unused.prepare.operationFingerprint, + planDigest: unused.planDigest, + decisionId: abandon.decisionId, + expectedSequence: unused.sequence, + evidenceDigest: configurationMaintenanceSettleRequestDigest(unused), + }); + expect(settled.state).toBe("settled"); + expect(dm.settlePosts).toBe(1); + await dm.assertDataAdmission(); + await prospective.discardUnusedPreparedProspectiveConfiguration(candidate); + } + expect(dm.preparePosts).toBe(version === "old" ? 0 : 1); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + const [desired] = await state.fixture.db + .select() + .from(schema.systemSettings) + .where(eq(schema.systemSettings.key, "platformConfiguration")); + expect(desired).toBeUndefined(); + const [retained] = await state.fixture.db + .select() + .from(schema.platformConfigurationPreview) + .where(eq(schema.platformConfigurationPreview.id, preview.previewId)); + expect(retained?.consumedAt).toBeNull(); + }, + ), + ); + } finally { + await dm.close(); + await app.close(); + closeControlInstallation(installation); + state.services = []; + vi.unstubAllEnvs(); + rmSync(host.cleanupRoot, { recursive: true, force: true }); + } + }, + 120000, + ); + }, +); diff --git a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts index 54b39f05d..dcdee8ba6 100644 --- a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts @@ -210,8 +210,23 @@ dbDescribe("durable exact platform preview", () => { changes: original.configuration, services: [], }); + const [priorDesired] = await fixture.db + .select({ version: sql`xmin::text` }) + .from(systemSettings) + .where(eq(systemSettings.key, "platformConfiguration")); await retainPlatformPreview( - { actorId, operationKey: randomUUID(), requestFingerprint: "a".repeat(64), preview: bound }, + { + actorId, + operationKey: randomUUID(), + requestFingerprint: "a".repeat(64), + preview: bound, + sourceSnapshot: { + revisionId: bound.expectedCurrentRevisionId, + sourceRevision: bound.sourceRevision, + services: [], + }, + desiredRowVersion: priorDesired?.version ?? null, + }, fixture.db, ); const clock = new Date(), diff --git a/apps/api/src/services/platform-configuration-participant.ts b/apps/api/src/services/platform-configuration-participant.ts index fe993c98f..392076ea9 100644 --- a/apps/api/src/services/platform-configuration-participant.ts +++ b/apps/api/src/services/platform-configuration-participant.ts @@ -37,6 +37,7 @@ interface State { "previewFingerprint" | "expectedCurrentRevisionId" | "confirmations" | "proof" >; preview: AdminPlatformPreview; + sourceSnapshot: ConfigurationSourcesInspection; expectedDesiredRowVersion: string | null; handoffId: string; authorization?: ReturnType; @@ -94,6 +95,7 @@ export function retainedPlatformConfigurationIntent(writer: AdministrativeConfig { preview } = state; return { version: 1 as const, + sourceSnapshot: structuredClone(state.sourceSnapshot), handoffId: state.handoffId, previewId: preview.previewId, previewFingerprint: preview.previewFingerprint, @@ -107,6 +109,9 @@ export function retainedPlatformConfigurationIntent(writer: AdministrativeConfig ...(state.authorization ? { authorization: { ...state.authorization } } : {}), }; } +export function retainedPlatformConfigurationSource(writer: AdministrativeConfigurationWriter) { + return stateFor(writer).sourceSnapshot; +} export function assertPlatformConfigurationSource( writer: AdministrativeConfigurationWriter, source: ConfigurationSourcesInspection, @@ -146,6 +151,8 @@ export async function preparePlatformConfigurationChange( preview = immutable(retained.preview); if ( retained.consumedByJobId || + !retained.sourceSnapshot || + retained.desiredRowVersion === undefined || preview.expectedCurrentRevisionId !== input.expectedCurrentRevisionId || preview.authPolicyRevision !== (await loadAuthPolicy(owner.connection)).revision || Object.entries(preview.requiredConfirmation).some( @@ -171,6 +178,8 @@ export async function preparePlatformConfigurationChange( .select({ version: sql`xmin::text` }) .from(systemSettings) .where(eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING)); + if ((desired?.version ?? null) !== retained.desiredRowVersion) + throw new PlatformPreviewConflict("Platform desired state changed; create a new preview"); const { version: _version, ...patch } = preview.configuration; const overlay: ProspectiveConfigurationOverlay = immutable({ purpose: "platform", @@ -302,6 +311,7 @@ export async function preparePlatformConfigurationChange( ...(proof ? { proof } : {}), }, preview, + sourceSnapshot: immutable(retained.sourceSnapshot!), expectedDesiredRowVersion: desired?.version ?? null, handoffId: randomUUID(), overlay, diff --git a/apps/api/src/services/platform-configuration.ts b/apps/api/src/services/platform-configuration.ts index a8fe56901..748f2c9bb 100644 --- a/apps/api/src/services/platform-configuration.ts +++ b/apps/api/src/services/platform-configuration.ts @@ -125,6 +125,8 @@ export async function commitPlatformConfigurationIntent( preview = retained.preview; if ( retained.consumedByJobId || + (retained.desiredRowVersion !== undefined && + retained.desiredRowVersion !== input.expectedDesiredRowVersion) || preview.sourceRevision !== input.sourceRevision || preview.expectedCurrentRevisionId !== input.expectedCurrentRevisionId || preview.authPolicyRevision !== input.expectedAuthPolicyRevision || diff --git a/apps/api/src/services/platform-preview-store.ts b/apps/api/src/services/platform-preview-store.ts index 0bb41cc60..8c760bb3f 100644 --- a/apps/api/src/services/platform-preview-store.ts +++ b/apps/api/src/services/platform-preview-store.ts @@ -1,7 +1,11 @@ /** Original preview rows are immutable; a request retry cannot extend its TTL. */ import { createHash } from "node:crypto"; import { adminPlatformPreviewSchema } from "@openmapx/core/api-contracts"; -import { canonicalJson } from "@openmapx/core/ops"; +import { + type ConfigurationSourcesInspection, + canonicalJson, + configurationSourcesInspectionSchema, +} from "@openmapx/core/ops"; import { and, eq, isNull, lt, sql } from "drizzle-orm"; import { z } from "zod"; import type { AuthDatabase } from "../account/auth-database"; @@ -10,6 +14,15 @@ import { platformConfigurationPreview as table } from "../db/schema"; import { preservationAllowsDeletion, withPreservationRetentionLock } from "../privacy/preservation"; const TTL = 600_000; +const storedPreviewSchema = z.strictObject({ + version: z.literal(1), + preview: adminPlatformPreviewSchema, + sourceSnapshot: configurationSourcesInspectionSchema, + desiredRowVersion: z + .string() + .regex(/^\d{1,10}$/) + .nullable(), +}); export class PlatformPreviewConflict extends Error { constructor(message = "Platform preview unavailable or changed") { super(message); @@ -17,7 +30,16 @@ export class PlatformPreviewConflict extends Error { } function decode(row: typeof table.$inferSelect) { try { - const preview = adminPlatformPreviewSchema.parse(row.preview); + const envelope = storedPreviewSchema.safeParse(row.preview); + const preview = envelope.success + ? envelope.data.preview + : adminPlatformPreviewSchema.parse(row.preview); + if ( + envelope.success && + (envelope.data.sourceSnapshot.sourceRevision !== preview.sourceRevision || + envelope.data.sourceSnapshot.revisionId !== preview.expectedCurrentRevisionId) + ) + throw Error(); const { previewFingerprint, ...intent } = preview; const digest = createHash("sha256") .update("openmapx-platform-preview-v1\0") @@ -36,6 +58,8 @@ function decode(row: typeof table.$inferSelect) { throw Error(); return { preview, + sourceSnapshot: envelope.success ? envelope.data.sourceSnapshot : undefined, + desiredRowVersion: envelope.success ? envelope.data.desiredRowVersion : undefined, expiresAt: row.expiresAt.toISOString(), consumedByJobId: row.consumedByJobId, }; @@ -49,6 +73,8 @@ export async function retainPlatformPreview( operationKey: string; requestFingerprint: string; preview: unknown; + sourceSnapshot?: ConfigurationSourcesInspection; + desiredRowVersion?: string | null; now?: Date; }, database: typeof db = db, @@ -92,6 +118,25 @@ export async function retainPlatformPreview( const capacity = await tx.select({ id: table.id }).from(table).limit(4096); if (capacity.length >= 4096) throw new PlatformPreviewConflict("Platform preview capacity unavailable"); + const stored = input.sourceSnapshot + ? storedPreviewSchema.parse({ + version: 1, + preview, + sourceSnapshot: input.sourceSnapshot, + desiredRowVersion: input.desiredRowVersion, + }) + : preview; + if ( + input.sourceSnapshot && + (input.sourceSnapshot.sourceRevision !== preview.sourceRevision || + input.sourceSnapshot.revisionId !== preview.expectedCurrentRevisionId) + ) + throw new PlatformPreviewConflict(); + const [size] = await tx.execute<{ bytes: number }>( + sql`select octet_length(${JSON.stringify(stored)}::jsonb::text) as bytes`, + ); + if (!size || size.bytes > 131072) + throw new PlatformPreviewConflict("Platform preview capacity unavailable"); const [row] = await tx .insert(table) .values({ @@ -103,7 +148,7 @@ export async function retainPlatformPreview( authPolicyRevision: preview.authPolicyRevision, sourceRevision: preview.sourceRevision, expectedCurrentRevisionId: preview.expectedCurrentRevisionId, - preview, + preview: stored, createdAt: now, expiresAt: new Date(now.getTime() + TTL), }) diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 870988491..3ea459f34 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -34,6 +34,7 @@ import { opsOperationSchema, type PreparedConfigurationRecord, parseControlRootOperationKey, + protectedConfigurationServiceIdSchema, readOpsTokenFile, sealTrustedConfigurationSnapshot, type TrustedConfigurationPayload, @@ -126,6 +127,7 @@ import { assertPlatformConfigurationOverlay, assertPlatformConfigurationSource, retainedPlatformConfigurationIntent, + retainedPlatformConfigurationSource, } from "./platform-configuration-participant"; import { encrypt } from "./secrets"; import { resolveServiceConfigWithSourcesStrict } from "./service-config-resolver"; @@ -3097,12 +3099,14 @@ export async function prepareProspectiveConfiguration(input: { throw Error("Fixed unchanged coordinator render required"); const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); const publisher = publisherOptions(token); - const source = await executeAndWait( - createApiOpsClient(), - { kind: "configuration.sources.inspect" }, - createDurableOpsKey("configuration.sources.inspect", input.operationKey), - { signal: input.signal }, - ); + const source = platformIntent + ? retainedPlatformConfigurationSource(writer) + : await executeAndWait( + createApiOpsClient(), + { kind: "configuration.sources.inspect" }, + createDurableOpsKey("configuration.sources.inspect", input.operationKey), + { signal: input.signal }, + ); if (source.revisionId !== input.expectedCurrentRevisionId) throw Error("Configuration revision conflict"); let cleanupOperationKey: string | undefined; @@ -3492,11 +3496,20 @@ function assertMaintenanceIntentCapacity(state: CandidateState, result: unknown) if (!record.maintenanceId || !record.prepareOperationKey) return; const projection = readProjection(result), keys = new Set([record.prepareOperationKey]); - const recreate = - record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + const platform = record.privatePlatformIntent; + const recreate = platform + ? record.affectedServiceIds.filter( + (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle ? record.affectedServiceIds : []; - const count = 1 + (state.preview?.entries ? 1 : 0) + recreate.length; + const handoff = + !!platform && + record.affectedServiceIds.some( + (id) => protectedConfigurationServiceIdSchema.safeParse(id).success, + ); + const count = 1 + (state.preview?.entries ? 1 : 0) + recreate.length + (handoff ? 1 : 0); if (count > 32) throw Error("Prospective maintenance effect capacity exceeded"); keys.add(controlChildOperationKey(record.prepareOperationKey, "configuration-commit", 0)); let ordinal = 1; @@ -3508,6 +3521,10 @@ function assertMaintenanceIntentCapacity(state: CandidateState, result: unknown) keys.add( controlChildOperationKey(record.prepareOperationKey, "maintenance-recreate", ordinal++), ); + if (handoff) + keys.add( + controlChildOperationKey(record.prepareOperationKey, "maintenance-handoff", ordinal++), + ); if ( record.coordinatorPrerequisite && (state.preview?.entries || record.privateCoordinatorLifecycle) @@ -3898,7 +3915,8 @@ async function verifyCurrentPreview( preview.previewDigest !== state.preview?.previewDigest || (state.overlay.coordinatorMode === "require-running" && (preview.entries || - (state.record.maintenanceId && state.record.apiFollowUp === "service-apply")) && + (state.record.maintenanceId && + (state.record.apiFollowUp === "service-apply" || state.record.privatePlatformIntent))) && preview.coordinatorState !== "running") ) rejectUnissuedCandidate(candidate, "Configuration preview changed"); @@ -5435,8 +5453,9 @@ export async function prepareConfigurationMaintenance( revisionId: candidate.revisionId, expectedSourceRevision: record.expectedSourceRevision, coordinatorMode: source.overlay.coordinatorMode, - followUp: - record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + followUp: record.privatePlatformIntent + ? "platform-apply" + : record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle ? "service-apply" : "render-only", }); diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index 53ecb2c8f..4d12e33d8 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -89,14 +89,14 @@ export async function ownedServiceHost(options: { environment?:Record = {}; const calls: string[][] = []; const actualComposeConfig = options.actualComposeConfig; + const configurationHelperVersion=options.configurationHelperVersion; let creatingProcess: | ((args: readonly string[], signal: AbortSignal) => Promise) | undefined; @@ -133,7 +134,20 @@ export async function ownedServiceHost(options: { environment?:Record = async (_file, args, options) => { calls.push([...args]); let stdout = ""; - if (args[0] === "container" && args[1] === "ls") { + if(configurationHelperVersion!==undefined && args[0]==="container" && args[1]==="inspect" && args[2]==="--format" && (["{{.Image}}","{{.Config.User}}","{{json .HostConfig.GroupAdd}}","{{json .Mounts}}"].includes(args[3]??"") || args[3]?.includes("range .Config.Env"))) { + const selected=rows.find(row=>row.Id===args[4]); + if(!selected)throw Error("Named helper prerequisite container unavailable"); + const format=args[3]; + if(format==="{{.Image}}")stdout=selected.Image; + else if(format==="{{.Config.User}}")stdout=`${process.getuid?.()??0}:${process.getgid?.()??0}`; + else if(format==="{{json .HostConfig.GroupAdd}}")stdout="[]"; + else if(format?.includes("range .Config.Env"))stdout="HOME\nPATH\n"; + else if(format==="{{json .Mounts}}")stdout=JSON.stringify([{Destination:"/var/run/docker.sock",Source:"/var/run/docker.sock"},{Destination:root,Source:root}]); + else throw Error("Unexpected fixed helper prerequisite format"); + } else if(configurationHelperVersion!==undefined && args[0]==="image" && args[1]==="inspect" && args[3]?.includes("openmapx.configuration-handoff.version")) { + if(args.at(-1)!==imageId)throw Error("Unexpected old helper image"); + stdout=configurationHelperVersion; + } else if (args[0] === "container" && args[1] === "ls") { stdout = rawListing ?? rows.map((r) => r.Id).join("\n"); const pending = inventoryProcess; inventoryProcess = undefined; diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts new file mode 100644 index 000000000..880dd1945 --- /dev/null +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -0,0 +1,267 @@ +import { existsSync } from "node:fs"; +/** Fixed original-owner helper journal. All cells stay in the existing indexed + * role-private observations namespace and original retention family. */ +import { + bindControlChildPhase, + closeControlInstallation, + controlDigest, + controlRolePublicKey, + executionAuthorityState, + inspectArtifactDelegation, + issueControlChildPhase, + lookupControlChildPhaseAttempt, + lookupControlChildPhaseBinding, + openFixedChildInstallation, + readPrivateRecord, + signControl, + type VerifiedExecutionAuthority, + verifyControl, + verifyFamilyOwnedRecord, + writePrivateRecord, +} from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + configurationHandoffPrepareOperationSchema, + type OpsOperation, +} from "@openmapx/core/ops"; +import { + type ConfigurationHandoffIntentRecord, + type ConfigurationHandoffProgress, + configurationHandoffIntentRecordSchema, + configurationHandoffProgressSchema, + configurationHandoffRecordDomains as domains, + configurationHandoffRecordName as recordName, +} from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; +import { inspectConfigurationHandoffPrerequisite } from "./configuration-handoff"; +import { assertOwnedMaintenanceHandoffRuntime } from "./configuration-maintenance-store"; +import type { OpsExecutionContext, OpsRuntime } from "./runtime"; + +declare const preparedBrand: unique symbol; +export type PreparedConfigurationHandoff = Readonly<{ readonly [preparedBrand]: true }>; +const prepared = new WeakMap< + PreparedConfigurationHandoff, + { + authority: VerifiedExecutionAuthority; + intent: ConfigurationHandoffIntentRecord; + facts: Awaited>; + dispatchTicket?: Awaited>["dispatchTicket"]; + } +>(); +const bytes = (value: unknown) => canonicalJson(value); +function immutable(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) immutable(child); + Object.freeze(value); + } + return value; +} +function indexed(authority: VerifiedExecutionAuthority, name: string, raw: unknown) { + const local = executionAuthorityState(authority); + if (!local.origin) throw Error("Original handoff family required"); + verifyFamilyOwnedRecord( + local.installation, + local.origin, + { scope: "private", namespace: "observations", container: "", name }, + bytes(raw), + ); +} +function readIntent(authority: VerifiedExecutionAuthority, handoffId: string) { + const local = executionAuthorityState(authority), + name = recordName(handoffId, "intent"), + raw = readPrivateRecord(local.installation, "observations", name), + value = verifyControl( + controlRolePublicKey(local.installation.installation, "ops-agent"), + domains.intent, + raw, + configurationHandoffIntentRecordSchema, + ); + indexed(authority, name, raw); + if ( + value.preparerAdmissionDigest !== local.receiptDigest || + bytes(value.preparer) !== bytes(local.execution) || + bytes(value.origin) !== bytes(local.origin) + ) + throw Error("Original handoff preparer changed"); + return value; +} +function writeProgress( + authority: VerifiedExecutionAuthority, + intent: ConfigurationHandoffIntentRecord, + value: ConfigurationHandoffProgress, + exclusive = false, +) { + const local = executionAuthorityState(authority); + const progress = configurationHandoffProgressSchema.parse(value); + if (progress.intentDigest !== controlDigest(intent)) + throw Error("Original handoff intent required"); + writePrivateRecord( + local.installation, + "observations", + recordName(intent.operation.handoffId, "progress"), + signControl(local.installation.privateKey, domains.progress, progress), + exclusive, + intent.origin, + ); +} +/** Only the real borrowed typed phase can produce this nominal launch. The + * old image prerequisite and original generation/guard source are checked + * before key/grant publication; no new target root is acquired. */ +export async function prepareConfigurationHandoffLaunch( + runtime: OpsRuntime, + inputOperation: Extract, + context: OpsExecutionContext, +): Promise { + const operation = immutable(configurationHandoffPrepareOperationSchema.parse(inputOperation)), + authority = context.controlAuthority, + lease = context.artifactLease; + if (!authority || !lease) throw Error("Actual original borrowed maintenance handoff required"); + const owned = await assertOwnedMaintenanceHandoffRuntime(runtime, operation, context), + local = executionAuthorityState(authority), + agent = owned.owner.originalContainers.find((value) => value.serviceId === "ops-agent"); + if (!agent || agent.state !== "running" || !owned.source.rootDir) + throw Error("Original running agent required"); + const facts = await inspectConfigurationHandoffPrerequisite({ + runDocker: async (args, command) => + ( + await owned.source.execFile("docker", args, { + signal: command.signal, + timeout: 15000, + maxBuffer: 65536, + }) + ).stdout, + containerId: agent.containerId, + imageId: agent.provenance.imageId, + rootDir: owned.source.rootDir, + signal: context.signal, + }); + const marker = Object.freeze({}) as PreparedConfigurationHandoff; + try { + const original = readIntent(authority, operation.handoffId); + if (bytes(original.operation) !== bytes(operation)) + throw Error("Original handoff body changed"); + // A retained grant/launch is observation-only. No fresh one-use ticket or + // another container create can be reconstructed from its identifiers. + prepared.set( + marker, + Object.freeze({ authority, intent: immutable(original), facts: immutable(facts) }), + ); + return marker; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const targets = operation.serviceIds.map((serviceId) => { + const old = owned.owner.originalContainers.find((value) => value.serviceId === serviceId); + if (!old || old.provenance.imageId !== operation.expectedImageIdentities[serviceId]) + throw Error("Original handoff container/image required"); + return { + serviceId, + phase: "pending" as const, + oldContainerId: old.containerId, + newContainerId: null, + expectedImageId: old.provenance.imageId, + }; + }); + context.signal.throwIfAborted(); + const reservation = owned.owner.plan.nestedHandoff!; + if (lookupControlChildPhaseAttempt(authority, reservation)) + throw Error("Attempted helper without original launch intent requires recovery"); + const bound = await bindControlChildPhase(authority, reservation), + issued = await issueControlChildPhase(authority, { + phase: "configuration-handoff", + ordinal: 0, + childPublicKey: "", + lease, + }), + attempt = lookupControlChildPhaseAttempt(authority, reservation), + binding = lookupControlChildPhaseBinding(authority, reservation); + if ( + !attempt || + !binding || + attempt.channel !== "delegation" || + attempt.assertion !== issued.assertion + ) + throw Error("Original retained helper grant required"); + const observed = inspectArtifactDelegation({ + dataRoot: local.installation.dataRoot, + installation: local.installation.installation, + assertion: attempt.assertion, + }); + if ( + bytes(observed.execution) !== bytes(bound.execution) || + observed.recipient !== "configuration-helper" || + observed.ownerCertificateDigest !== owned.owner.rootCertificateDigest || + bytes(observed.owner) !== bytes(owned.owner.plan.prepare) + ) + throw Error("Original helper target ownership changed"); + const child = openFixedChildInstallation({ + infraDir: owned.source.stack.infraDir, + assertion: attempt.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + let helperPublicKey: string; + try { + helperPublicKey = child.publicKey; + } finally { + closeControlInstallation(child); + } + const intent = configurationHandoffIntentRecordSchema.parse({ + version: 1, + installationId: local.installation.installation.installationId, + origin: local.origin, + ownerPrepare: owned.owner.plan.prepare, + preparer: local.execution, + helper: bound.execution, + admittedAt: binding.boundAt, + requestedDeadlineAt: binding.requestedDeadlineAt, + operation, + helperPublicKey, + preparerAdmissionDigest: local.receiptDigest, + ownerCertificateDigest: owned.owner.rootCertificateDigest, + leaseId: observed.leaseId, + delegationId: observed.delegationId, + assertion: attempt.assertion, + helperContainerName: `openmapx-configuration-handoff-${operation.handoffId}`, + preparedAt: Date.now(), + }); + writePrivateRecord( + local.installation, + "observations", + recordName(operation.handoffId, "intent"), + signControl(local.installation.privateKey, domains.intent, intent), + true, + owned.owner.origin, + ); + + writeProgress( + authority, + intent, + { + version: 1, + intentDigest: controlDigest(intent), + sequence: 0, + phase: "prepared", + launch: { state: "not_attempted", containerId: null }, + targets, + updatedAt: Date.now(), + completedAt: null, + }, + true, + ); + prepared.set( + marker, + Object.freeze({ + authority, + intent: immutable(intent), + facts: immutable(facts), + dispatchTicket: issued.dispatchTicket, + }), + ); + return marker; +} +export function preparedConfigurationHandoffState(value: PreparedConfigurationHandoff) { + const state = prepared.get(value); + if (!state) throw Error("Original nominal handoff required"); + const observed = readIntent(state.authority, state.intent.operation.handoffId); + if (bytes(observed) !== bytes(state.intent)) throw Error("Original handoff launch changed"); + return state; +} diff --git a/apps/ops-agent/src/configuration-handoff.test.ts b/apps/ops-agent/src/configuration-handoff.test.ts new file mode 100644 index 000000000..7178611a6 --- /dev/null +++ b/apps/ops-agent/src/configuration-handoff.test.ts @@ -0,0 +1,194 @@ +import { expect, it } from "vitest"; +import { + configurationHandoffRunArgs, + configurationHandoffServiceOrder, + inspectConfigurationHandoffPrerequisite, +} from "./configuration-handoff"; + +const handoffId = "11111111-1111-4111-8111-111111111111", + image = `sha256:${"a".repeat(64)}`; +const facts = { + imageId: image, + user: "1000:1000", + groups: ["10"], + envNames: [ + "HOME", + "OPS_AGENT_HANDOFF_ID", + "OPS_CONFIGURATION_HANDOFF_ID", + "OPENMAPX_CONTROL_EXECUTION", + "OPENMAPX_CONTROL_DELEGATION", + "DOCKER_CONFIG", + ], + dockerSocketSource: "/var/run/docker.sock", + rootSource: "/synthetic/host", +}; +it("launches the existing confined old-image helper with only original control references", () => { + const args = configurationHandoffRunArgs(facts, "/synthetic/host", { + handoffId, + assertion: "original-signed-assertion", + execution: { + executor: "host-cli", + principal: "api", + executionId: "22222222-2222-4222-8222-222222222222", + operationKey: "opk1_helperoriginaladdress", + operationFingerprint: "b".repeat(64), + fingerprint: "c".repeat(64), + deadlineAt: Date.now() + 60000, + }, + }); + expect(args[args.indexOf("--name") + 1]).toBe(`openmapx-configuration-handoff-${handoffId}`); + expect(args).toContain("dist/configuration-handoff-main.mjs"); + expect(args).toContain(image); + expect(args).toContain("--read-only"); + expect(args.join(" ")).toContain("control-private/hidden-key"); + expect(args.join(" ")).toContain("control-private/empty"); + expect(args.filter((a) => a.startsWith("OPENMAPX_CONTROL_DELEGATION="))).toHaveLength(1); + expect(args.filter((a) => a.startsWith("OPS_CONFIGURATION_HANDOFF_ID="))).toEqual([ + `OPS_CONFIGURATION_HANDOFF_ID=${handoffId}`, + ]); + expect(args).not.toContain("OPS_AGENT_HANDOFF_ID"); + expect(args).not.toContain("OPS_CONFIGURATION_HANDOFF_ID"); +}); +it("orders dependencies and the agent last without silently expanding frozen targets", () => { + expect( + configurationHandoffServiceOrder([ + "ops-agent", + "traefik", + "app-api", + "redis", + "postgis", + "app-web", + ]), + ).toEqual(["postgis", "redis", "app-web", "app-api", "traefik", "ops-agent"]); + expect( + configurationHandoffServiceOrder(["motis", "app-api", "data-manager", "ops-agent"]), + ).toEqual(["data-manager", "motis", "app-api", "ops-agent"]); + expect(() => configurationHandoffServiceOrder(["app-api", "app-api"])).toThrow(); +}); +it("requires the actual old image's helper version before any launch", async () => { + const calls: string[][] = []; + const run = async (args: readonly string[]) => { + calls.push([...args]); + const format = args[args.indexOf("--format") + 1]; + if (args[0] === "image") return ""; + if (format === "{{.Image}}") return image; + if (format === "{{.Config.User}}") return "1000:1000"; + if (format === "{{json .HostConfig.GroupAdd}}") return "[]"; + if (format?.includes("range .Config.Env")) return "HOME\n"; + if (format === "{{json .Mounts}}") + return JSON.stringify([ + { Destination: "/var/run/docker.sock", Source: "/var/run/docker.sock" }, + { Destination: "/synthetic/host", Source: "/synthetic/host" }, + ]); + throw Error("Unexpected Docker read"); + }; + await expect( + inspectConfigurationHandoffPrerequisite({ + runDocker: run, + containerId: "c".repeat(64), + imageId: image, + rootDir: "/synthetic/host", + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ name: "OpsHostUpdateRequiredError" }); + expect(calls.every((args) => args.includes("inspect"))).toBe(true); +}); + +it("does not let observations or terminal preparations indefinitely block replacement", async () => { + const { configurationHandoffBlocksReplacement } = await import("./configuration-handoff"); + expect(configurationHandoffBlocksReplacement("service.logs.follow", "running")).toBe(false); + expect(configurationHandoffBlocksReplacement("system.diagnostics", "running")).toBe(false); + expect(configurationHandoffBlocksReplacement("service.start", "running")).toBe(true); + expect(configurationHandoffBlocksReplacement("service.start", "termination_pending")).toBe(true); + expect(configurationHandoffBlocksReplacement("service.start", "succeeded")).toBe(false); +}); + +it("uses actual server job state when excluding a live log observer from drain", async () => { + const { buildOpsAgentServer } = await import("./server"); + const api = Buffer.alloc(32, 31).toString("base64url"), + dm = Buffer.alloc(32, 32).toString("base64url"); + const logging = Promise.withResolvers(), + release = Promise.withResolvers(), + checked = Promise.withResolvers(); + const app = buildOpsAgentServer({ + tokens: { api, "data-manager": dm }, + resourcePolicy: { allowGlobal: () => true, allowService: () => true }, + dispatch: async (operation, _runtime, context) => { + if (operation.kind === "service.logs.follow") { + logging.resolve(); + await release.promise; + return { lines: 0, truncated: false }; + } + checked.resolve(context.otherActiveOperations?.() ?? -1); + return { ok: true, checks: [] }; + }, + }); + const baseUrl = await app.listen({ host: "127.0.0.1", port: 0 }); + const send = async (operation: Record, suffix: string) => { + const response = await fetch(`${baseUrl}/v1/operations`, { + method: "POST", + headers: { + authorization: `Bearer ${api}`, + "x-ops-protocol-version": "2", + "content-type": "application/json", + }, + body: JSON.stringify({ + version: 2, + requestId: `ops1_observation${suffix}`, + operationKey: `opk1_observation${suffix}`, + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 15000).toISOString(), + execution: { requestedDeadlineAt: Date.now() + 60000 }, + operation, + }), + }); + return { statusCode: response.status, body: await response.json() }; + }; + try { + const logResponse = await send( + { kind: "service.logs.follow", serviceId: "motis", tail: 1, maxDurationSeconds: 60 }, + "logs0000", + ); + expect(logResponse.statusCode, JSON.stringify(logResponse.body)).toBe(202); + await logging.promise; + const diagnosticResponse = await send({ kind: "system.diagnostics" }, "diag0000"); + expect(diagnosticResponse.statusCode, JSON.stringify(diagnosticResponse.body)).toBe(202); + expect(await checked.promise).toBe(0); + } finally { + release.resolve(); + await app.close(); + } +}); + +it("rejects a handoff DTO without the actual borrowed original owner", async () => { + const { assertOwnedMaintenanceHandoffRuntime } = await import( + "./configuration-maintenance-store" + ); + const { createUnavailableRuntime } = await import("./runtime"); + const { testExecution } = await import("./__fixtures__/execution"); + const operation = { + kind: "configurationHandoff.prepare" as const, + handoffId, + maintenanceId: "22222222-2222-4222-8222-222222222222", + candidateId: "33333333-3333-4333-8333-333333333333", + planDigest: "a".repeat(64), + previewFingerprint: "b".repeat(64), + serviceIds: ["ops-agent"], + expectedCurrentRevisionId: `cfg1_${"c".repeat(43)}`, + expectedImageIdentities: { "ops-agent": image }, + mode: { kind: "configuration" as const }, + }; + await expect( + assertOwnedMaintenanceHandoffRuntime(createUnavailableRuntime(), operation, { + execution: testExecution(), + signal: new AbortController().signal, + emitLog: () => {}, + claim: { + operation, + fingerprint: "a".repeat(64), + source: "registry", + capability: { revisionId: "registry1", values: {} }, + }, + }), + ).rejects.toThrow("Actual original borrowed maintenance handoff required"); +}); diff --git a/apps/ops-agent/src/configuration-handoff.ts b/apps/ops-agent/src/configuration-handoff.ts new file mode 100644 index 000000000..0ed8f6a3d --- /dev/null +++ b/apps/ops-agent/src/configuration-handoff.ts @@ -0,0 +1,134 @@ +/** Fixed configuration helper launch mechanics. These data builders confer no + * authority: the installed owner/runtime must first join the original frozen + * step, actual borrowed lease, signed source and one-use helper dispatch. */ +import { + type ExecutionRef, + executionRefSchema, + type OpsJobState, + type OpsOperation, +} from "@openmapx/core/ops"; +import { z } from "zod"; +import { + AGENT_HANDOFF_ENTRYPOINT, + type AgentContainerFacts, + agentHandoffRunArgs, + inspectAgentContainer, +} from "./agent-handoff"; +import type { DockerCommandRunner } from "./release-images"; +import { OpsHostUpdateRequiredError } from "./runtime"; + +export const CONFIGURATION_HANDOFF_ENTRYPOINT = "dist/configuration-handoff-main.mjs"; +export const CONFIGURATION_HANDOFF_LABEL = "openmapx.configuration-handoff"; +export const CONFIGURATION_HANDOFF_VERSION = "1"; +export const CONFIGURATION_HANDOFF_IMAGE_LABEL = "openmapx.configuration-handoff.version"; +const protectedOrder = ["postgis", "redis", "app-web", "app-api", "traefik", "ops-agent"] as const; +const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +export function configurationHandoffServiceOrder(serviceIds: readonly string[]): string[] { + const values = z.array(id).min(1).max(64).parse(serviceIds); + if (new Set(values).size !== values.length) throw Error("Duplicate frozen handoff target"); + const protectedIds = new Set(protectedOrder); + return [ + ...values.filter((value) => !protectedIds.has(value)).sort(), + ...protectedOrder.filter((value) => values.includes(value)), + ]; +} +export function configurationHandoffRunArgs( + facts: AgentContainerFacts, + rootDir: string, + input: { handoffId: string; assertion: string; execution: ExecutionRef }, +): string[] { + const handoffId = z.uuid().parse(input.handoffId), + execution = executionRefSchema.parse(input.execution); + z.string() + .min(1) + .max(4096) + .regex(/^[A-Za-z0-9_-]+$/) + .parse(input.assertion); + if (execution.executor !== "host-cli") + throw Error("Original configuration helper execution required"); + const args = agentHandoffRunArgs( + { + ...facts, + envNames: facts.envNames.filter((name) => name !== "OPS_CONFIGURATION_HANDOFF_ID"), + }, + rootDir, + handoffId, + ), + name = args.indexOf("--name"), + label = args.indexOf("--label"), + workdir = args.indexOf("--workdir"); + if (name < 0 || label < 0 || workdir < 0 || args.at(-1) !== AGENT_HANDOFF_ENTRYPOINT) + throw Error("Fixed helper launch template changed"); + args[name + 1] = `openmapx-configuration-handoff-${handoffId}`; + args[label + 1] = `${CONFIGURATION_HANDOFF_LABEL}=${handoffId}`; + args[args.length - 1] = CONFIGURATION_HANDOFF_ENTRYPOINT; + // Insert before the image/command boundary, preserving mounts, key masking, + // nonroot identity, no-network and resource confinement from the old helper. + args.splice( + workdir, + 0, + "--env", + `OPS_CONFIGURATION_HANDOFF_ID=${handoffId}`, + "--env", + `OPENMAPX_CONTROL_DELEGATION=${input.assertion}`, + "--env", + `OPENMAPX_CONTROL_EXECUTION=${JSON.stringify(execution)}`, + ); + const legacy = args.indexOf(`OPS_AGENT_HANDOFF_ID=${handoffId}`); + if (legacy < 1 || args[legacy - 1] !== "--env") throw Error("Fixed helper environment changed"); + args.splice(legacy - 1, 2); + return args; +} +/** The running old image must contain the fixed helper bundle. Docker/Compose + * reachability alone cannot advertise a supported protected follow-up. Only + * bounded inspect commands run here; no ownership, pull or launch occurs. */ +export async function inspectConfigurationHandoffPrerequisite(input: { + runDocker: DockerCommandRunner; + containerId: string; + imageId: string; + rootDir: string; + signal: AbortSignal; +}): Promise { + if (!/^[a-f0-9]{64}$/.test(input.containerId) || !/^sha256:[a-f0-9]{64}$/.test(input.imageId)) + throw new OpsHostUpdateRequiredError(); + try { + const facts = await inspectAgentContainer( + input.runDocker, + input.containerId, + input.rootDir, + input.signal, + ); + if (facts.imageId !== input.imageId) throw new OpsHostUpdateRequiredError(); + const version = ( + await input.runDocker( + [ + "image", + "inspect", + "--format", + `{{index .Config.Labels "${CONFIGURATION_HANDOFF_IMAGE_LABEL}"}}`, + facts.imageId, + ], + { signal: input.signal, emitLog: () => {} }, + ) + ).trim(); + if (version !== CONFIGURATION_HANDOFF_VERSION) throw new OpsHostUpdateRequiredError(); + return facts; + } catch (error) { + input.signal.throwIfAborted(); + if (error instanceof OpsHostUpdateRequiredError) throw error; + throw new OpsHostUpdateRequiredError(); + } +} + +/** Live server jobs are the source; caller-supplied counts are never host + * authority. Active byte streams are added separately by the actual store. */ +export function configurationHandoffBlocksReplacement( + kind: OpsOperation["kind"], + state: OpsJobState, +): boolean { + return ( + ["queued", "running", "termination_pending"].includes(state) && + kind !== "service.logs.follow" && + kind !== "system.diagnostics" + ); +} diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index 5ab30be77..80ced0ae2 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -53,6 +53,7 @@ import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; import { type ArtifactTargetLease, adoptArtifactTargetLease, + assertArtifactTargetLeaseExecution, assertConfigurationMaintenanceRootClosure, assertFixedConfigurationMaintenancePlan, bindControlChildPhase, @@ -127,6 +128,7 @@ import { type OpsOperation, opsOperationFingerprint, opsResourceId, + protectedConfigurationServiceIdSchema, type RetainedParentExecution, type TrustedConfigurationQueueLock, type VerifiedParentExecution, @@ -871,6 +873,57 @@ export async function assertOwnedMaintenanceRecreateRuntime( if (!source) throw Error("Actual original source required"); return inspectOriginalOwnerServices(source, root, context.signal); } +/** Exact fixed handoff source from its actual borrowed child and signed + * maintenance owner. No ordinary lifecycle operation can call this adapter. */ +export async function assertOwnedMaintenanceHandoffRuntime( + runtime: OpsRuntime, + operation: Extract, + context: import("./runtime").OpsExecutionContext, +) { + if (!context.controlAuthority || !context.artifactLease) + throw Error("Actual original borrowed maintenance handoff required"); + assertArtifactTargetLeaseExecution(context.artifactLease, context.execution); + const child = executionAuthorityState(context.controlAuthority), + observed = verifyControl( + child.installation.installation.publicKey, + domain, + readPrivateRecord( + child.installation, + "configuration-maintenance", + `${operation.maintenanceId}.json`, + ), + ownerSchema, + ), + root = restoreAdmittedControlExecution(child.installation.installation, observed.plan.prepare), + owner = readOwnedConfigurationMaintenance(root), + step = owner.plan.steps.find( + (value) => + value.operationKey === child.execution.operationKey && + value.operationFingerprint === child.execution.operationFingerprint, + ); + if ( + !step || + step.phase !== "maintenance-handoff" || + !owner.plan.handoff || + !owner.plan.nestedHandoff || + !child.origin || + canonicalJson(child.origin) !== canonicalJson(owner.origin) || + canonicalJson(step.operation) !== canonicalJson(operation) || + child.kind !== operation.kind || + canonicalJson(child.reservations) !== canonicalJson([owner.plan.nestedHandoff]) + ) + throw Error("Original frozen handoff source changed"); + await verifyFrozenMaintenanceFollowUpSource( + runtime, + root, + step.ordinal, + context.signal, + "maintenance-handoff", + ); + const source = sources.get(runtime); + if (!source) throw Error("Actual original source required"); + return { root, owner, source }; +} /** Retention capture for the exact already-authorized borrowed generation child. * No caller snapshot or maintenance UUID alone can select OLD ownership. */ export async function inspectOwnedMaintenanceGenerationRuntime( @@ -3571,6 +3624,7 @@ export async function closeOwnedConfigurationMaintenanceWithoutGuard( // The actual admission/inspection producer and its private consumer share ONE nominal owner. interface Source { + rootDir?: string; infraDir: string; candidateSource?: ConfigurationCandidateSource; stack: services.StackPaths; @@ -3983,6 +4037,21 @@ export async function verifyFrozenMaintenanceRecreateSource( authority: VerifiedExecutionAuthority, ordinal: number, signal: AbortSignal, +): Promise { + return verifyFrozenMaintenanceFollowUpSource( + runtime, + authority, + ordinal, + signal, + "maintenance-recreate", + ); +} +async function verifyFrozenMaintenanceFollowUpSource( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, + ordinal: number, + signal: AbortSignal, + expectedPhase: "maintenance-recreate" | "maintenance-handoff", ): Promise { const source = sources.get(runtime), owner = readOwnedConfigurationMaintenance(authority), @@ -3991,9 +4060,12 @@ export async function verifyFrozenMaintenanceRecreateSource( if ( !source || !step || - step.phase !== "maintenance-recreate" || - (step.operation.kind !== "service.recreate" && + step.phase !== expectedPhase || + (expectedPhase === "maintenance-recreate" && + step.operation.kind !== "service.recreate" && step.operation.kind !== "service.recreateIsolated") || + (expectedPhase === "maintenance-handoff" && + step.operation.kind !== "configurationHandoff.prepare") || owner.receipt.state !== "applying" || owner.receipt.decision?.decision !== "continue" || !owner.apiControls.decide?.receiptDelivery || @@ -4907,13 +4979,29 @@ export async function prepareConfigurationMaintenanceAdmission( ].includes(original.kind) ) throw Error("Original candidate/parent maintenance binding conflict"); - const recreateServiceIds = - candidate.record.apiFollowUp === "service-apply" || candidate.record.privateCoordinatorLifecycle + const platform = candidate.record.privatePlatformIntent; + if ( + platform && + (candidate.record.apiFollowUp !== "platform-apply" || + original.kind !== "stack.render" || + platform.sourceRevision !== input.operation.expectedSourceRevision || + platform.expectedCurrentRevisionId !== input.operation.expectedCurrentRevisionId || + canonicalJson(platform.configuration) !== canonicalJson(candidate.payload.platform) || + canonicalJson(platform.affectedServiceIds) !== + canonicalJson(candidate.record.affectedServiceIds)) + ) + throw Error("Original platform candidate binding conflict"); + const recreateServiceIds = platform + ? candidate.record.affectedServiceIds.filter( + (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : candidate.record.apiFollowUp === "service-apply" || + candidate.record.privateCoordinatorLifecycle ? candidate.record.affectedServiceIds : []; if ( - input.operation.followUp === "platform-apply" || - (recreateServiceIds.length ? "service-apply" : "render-only") !== input.operation.followUp + (platform ? "platform-apply" : recreateServiceIds.length ? "service-apply" : "render-only") !== + input.operation.followUp ) throw Error("Configuration maintenance continuation unavailable"); const authority = await source.loadAuthority(); @@ -5139,6 +5227,28 @@ export async function prepareConfigurationMaintenanceAdmission( }), ) .digest("hex"); + const protectedIds = platform + ? platform.affectedServiceIds.filter( + (id) => protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : []; + const handoff = + platform && protectedIds.length + ? { + handoffId: platform.handoffId, + previewFingerprint: platform.previewFingerprint, + serviceIds: protectedIds, + expectedImageIdentities: Object.fromEntries( + protectedIds.map((id) => { + const old = originalContainers.find((value) => value.serviceId === id); + if (!old || old.state !== "running") + throw Error("Original protected target unavailable"); + return [id, old.provenance.imageId]; + }), + ), + mode: { kind: "configuration" as const }, + } + : undefined; const frozen = buildFixedConfigurationMaintenancePlan({ installationId: input.installation.installationId, prepare: input.execution, @@ -5153,6 +5263,7 @@ export async function prepareConfigurationMaintenanceAdmission( hardlinkPlan: { plan, prune: true }, serviceIds, recreateServiceIds, + ...(handoff ? { handoff } : {}), ...(coordinatorImage ? { coordinatorImage } : {}), }); assertProspectiveConfigurationMaintenanceBudget({ diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 34b7b01fe..6a122ad3d 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -192,12 +192,15 @@ export async function createProductionOpsRuntime( integrationSchemas: initial.integrationSchemas, loadAuthority: options.loadAuthority, infraDir: options.stack.infraDir, + rootDir: options.rootDir, + execFile: options.docker?.execFile ?? runContainedProcess, ownedServices: owned, candidateSource: options.candidateSource, }); installCommunityRepositoryRuntime(runtime, options.communityRepositories, owned, lifecycle); installConfigurationHardlinksRuntime(runtime, { infraDir: options.stack.infraDir }); registerConfigurationMaintenanceSource(runtime, { + rootDir: options.rootDir, infraDir: options.stack.infraDir, candidateSource: options.candidateSource, stack: options.stack, diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 171289c46..543e4378f 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -99,6 +99,7 @@ import { ArtifactTransferError } from "./artifact-transfer/errors"; import { registerArtifactTransferRoutes } from "./artifact-transfer/routes"; import type { ArtifactTransferStore } from "./artifact-transfer/store"; import { readCommittedCatalogApproval } from "./catalog-approval-runtime"; +import { configurationHandoffBlocksReplacement } from "./configuration-handoff"; import { prepareConfigurationHardlinksAdmission } from "./configuration-hardlinks-runtime"; import { configurationMaintenanceAdmissionState, @@ -1057,9 +1058,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif [...jobsById.values()].filter( (other) => other.operationId !== job.operationId && - (other.state === "queued" || - other.state === "running" || - other.state === "termination_pending"), + configurationHandoffBlocksReplacement(other.operation.kind, other.state), ).length + (options.artifactTransfers?.activeStreams ?? 0), ...(job.operation.kind === "services.buildAll" ? { diff --git a/apps/ops-agent/src/trusted-config-runtime.ts b/apps/ops-agent/src/trusted-config-runtime.ts index 63021b7d9..d7d897a07 100644 --- a/apps/ops-agent/src/trusted-config-runtime.ts +++ b/apps/ops-agent/src/trusted-config-runtime.ts @@ -27,14 +27,17 @@ import { type ConfigurationCandidateSource, readConfigurationCandidate, } from "./configuration-candidate-source"; +import { inspectConfigurationHandoffPrerequisite } from "./configuration-handoff"; import { assertConfigurationCoordinatorReady } from "./configuration-hardlinks-runtime"; import { assertStagedCoordinatorConfigurationWrite, inspectOwnedMaintenanceGenerationRuntime, } from "./configuration-maintenance-store"; import { configurationPolicyRevision } from "./configured-resource-authority"; +import type { DockerRuntimeOptions } from "./docker-runtime"; import type { OwnedServiceResolver } from "./owned-services"; import type { OpsExecutionContext, OpsRuntime, OpsTrustedClaim } from "./runtime"; +import { OpsHostUpdateRequiredError } from "./runtime"; import { renderCoordinatorCandidate } from "./service-pull-runtime"; const FAILED = "Trusted configuration apply failed"; @@ -57,6 +60,8 @@ export interface TrustedConfigurationRuntimeOptions { integrationSchemas: ReadonlyMap>; loadAuthority?: () => Promise; infraDir: string; + rootDir?: string; + execFile?: NonNullable; ownedServices?: OwnedServiceResolver; candidateSource?: ConfigurationCandidateSource; /** The process environment, read for the operator's selection override. */ @@ -383,6 +388,31 @@ export function installTrustedConfigurationRuntime( : undefined; if (candidate.record.maintenanceId) { if (!options.ownedServices) throw Error("Actual maintenance ownership inventory unavailable"); + const originalContainers = await options.ownedServices.inspectOwnedServices(context.signal); + if ( + candidate.record.privatePlatformIntent && + candidate.record.affectedServiceIds.some((id) => + ["postgis", "redis", "app-web", "app-api", "traefik", "ops-agent"].includes(id), + ) + ) { + const agent = originalContainers.find((value) => value.serviceId === "ops-agent"); + if (!options.rootDir || !options.execFile || !agent || agent.state !== "running") + throw new OpsHostUpdateRequiredError(); + await inspectConfigurationHandoffPrerequisite({ + rootDir: options.rootDir, + containerId: agent.containerId, + imageId: agent.provenance.imageId, + signal: context.signal, + runDocker: async (args, command) => + ( + await options.execFile!("docker", args, { + signal: command.signal, + timeout: 15000, + maxBuffer: 65536, + }) + ).stdout, + }); + } assertProspectiveConfigurationMaintenancePreludeBudget({ installationId: installation.installationId, record: candidate.record, @@ -391,14 +421,18 @@ export function installTrustedConfigurationRuntime( currentServiceIds: [ ...(coreServices.readAppliedServiceIds(options.infraDir, services) ?? []), ], - originalContainers: await options.ownedServices.inspectOwnedServices(context.signal), + originalContainers, rendered, ...(prospectiveCoordinatorImage ? { coordinatorImage: prospectiveCoordinatorImage } : {}), }); } if (plan.length > 256) throw Error("Configuration preview budget exceeded"); let coordinatorState: ConfigurationHardlinksPreview["coordinatorState"] = "not_required"; - if (plan.length || candidate.record.privateCoordinatorLifecycle) { + if ( + plan.length || + candidate.record.privateCoordinatorLifecycle || + candidate.record.privatePlatformIntent + ) { const dm = rendered.resolved.enabled.find( (service) => service.manifest.id === "data-manager", ); diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.test.ts b/packages/core/src/build-artifacts/configuration-handoff-records.test.ts new file mode 100644 index 000000000..7db3f9c17 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-handoff-records.test.ts @@ -0,0 +1,66 @@ +import { expect, it } from "vitest"; +import { + configurationHandoffHelperBodySchema, + configurationHandoffProgressSchema, +} from "./configuration-handoff-records"; + +const image = `sha256:${"a".repeat(64)}`; +const progress = () => ({ + version: 1, + intentDigest: "b".repeat(64), + sequence: 0, + phase: "prepared", + launch: { state: "not_attempted", containerId: null }, + targets: [ + { + serviceId: "app-api", + phase: "pending", + oldContainerId: "c".repeat(64), + newContainerId: null, + expectedImageId: image, + }, + ], + updatedAt: 1, + completedAt: null, +}); +it("retains a reference-only helper body and truthful bounded launch/target progress", () => { + expect( + configurationHandoffHelperBodySchema.parse({ + kind: "configuration-helper", + handoffId: "11111111-1111-4111-8111-111111111111", + intentDigest: "a".repeat(64), + }).kind, + ).toBe("configuration-helper"); + expect( + configurationHandoffHelperBodySchema.safeParse({ + kind: "configuration-helper", + handoffId: "11111111-1111-4111-8111-111111111111", + intentDigest: "a".repeat(64), + path: "/tmp/selected", + }).success, + ).toBe(false); + expect(configurationHandoffProgressSchema.parse(progress()).phase).toBe("prepared"); + expect( + configurationHandoffProgressSchema.safeParse({ + ...progress(), + phase: "applied", + completedAt: 2, + }).success, + ).toBe(false); + expect( + configurationHandoffProgressSchema.parse({ + ...progress(), + phase: "applied", + completedAt: 2, + updatedAt: 2, + launch: { state: "confirmed", containerId: "d".repeat(64) }, + targets: [{ ...progress().targets[0]!, phase: "verified", newContainerId: "e".repeat(64) }], + }).phase, + ).toBe("applied"); + expect( + configurationHandoffProgressSchema.safeParse({ + ...progress(), + targets: [...progress().targets, ...progress().targets], + }).success, + ).toBe(false); +}); diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.ts b/packages/core/src/build-artifacts/configuration-handoff-records.ts new file mode 100644 index 000000000..b8f3e6664 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-handoff-records.ts @@ -0,0 +1,115 @@ +/** Versioned helper journal data. Decoding confers neither a lease nor forward + * authority; producers and observers join these to the original signed grant. */ +import { z } from "zod"; +import { configurationHandoffPrepareOperationSchema } from "../ops/configuration-handoff-records"; +import { executionRefSchema } from "../ops/execution-ref"; +import { controlOriginSchema } from "./control-records"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/), + containerId = z.string().regex(/^[a-f0-9]{64}$/), + time = z.number().int().nonnegative().safe(); +export const configurationHandoffHelperBodySchema = z.strictObject({ + kind: z.literal("configuration-helper"), + handoffId: z.uuid(), + intentDigest: digest, +}); +export const configurationHandoffIntentRecordSchema = z + .strictObject({ + version: z.literal(1), + installationId: z.uuid(), + origin: controlOriginSchema, + ownerPrepare: executionRefSchema, + preparer: executionRefSchema, + helper: executionRefSchema, + operation: configurationHandoffPrepareOperationSchema, + admittedAt: time, + requestedDeadlineAt: time, + helperPublicKey: z + .string() + .min(32) + .max(128) + .regex(/^[A-Za-z0-9_-]+$/), + preparerAdmissionDigest: digest, + ownerCertificateDigest: digest, + leaseId: z.uuid(), + delegationId: z.uuid(), + assertion: z.string().min(1).max(4096), + helperContainerName: z.string().regex(/^openmapx-configuration-handoff-[a-f0-9-]{36}$/), + preparedAt: time, + }) + .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 65536); +export const configurationHandoffProgressSchema = z + .strictObject({ + version: z.literal(1), + intentDigest: digest, + sequence: z.number().int().min(0).max(2048), + phase: z.enum([ + "prepared", + "launch_attempted", + "launched", + "draining", + "applying", + "verifying", + "applied", + "rolled_back", + "failed", + "recovery_required", + ]), + launch: z.strictObject({ + state: z.enum(["not_attempted", "attempted", "confirmed"]), + containerId: containerId.nullable(), + }), + targets: z + .array( + z.strictObject({ + serviceId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + phase: z.enum([ + "pending", + "pull_attempted", + "pulled", + "recreate_attempted", + "verified", + "restore_attempted", + "restored", + "unknown", + ]), + oldContainerId: containerId, + newContainerId: containerId.nullable(), + expectedImageId: z.string().regex(/^sha256:[a-f0-9]{64}$/), + }), + ) + .min(1) + .max(64), + updatedAt: time, + completedAt: time.nullable(), + }) + .superRefine((value, ctx) => { + if ( + Buffer.byteLength(JSON.stringify(value)) > 65536 || + new Set(value.targets.map((target) => target.serviceId)).size !== value.targets.length || + (value.launch.state === "confirmed") !== (value.launch.containerId !== null) || + (value.completedAt !== null && value.completedAt > value.updatedAt) || + (["applied", "rolled_back"].includes(value.phase) && + (value.completedAt === null || + value.launch.state !== "confirmed" || + value.targets.some( + (target) => + target.phase !== (value.phase === "applied" ? "verified" : "restored") || + !target.newContainerId, + ))) + ) + ctx.addIssue({ code: "custom", message: "Inconsistent configuration helper progress" }); + }); +export type ConfigurationHandoffIntentRecord = z.infer< + typeof configurationHandoffIntentRecordSchema +>; +export type ConfigurationHandoffProgress = z.infer; +export const configurationHandoffRecordDomains = { + intent: "openmapx-configuration-handoff-intent-v1", + progress: "openmapx-configuration-handoff-progress-v1", + completion: "openmapx-configuration-handoff-completion-v1", +} as const; +export const configurationHandoffRecordName = ( + id: string, + kind: "intent" | "progress" | "completion", +) => `configuration-handoff-${z.uuid().parse(id)}.${kind}.json`; diff --git a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts index 857c2c401..97cb212e7 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts @@ -2,6 +2,7 @@ * expressions never become a signed witness, authority handle or publication. */ import { dirname, join } from "node:path"; import { canonicalJson } from "../ops/canonical-json"; +import { protectedConfigurationServiceIdSchema } from "../ops/configuration-handoff-records"; import { configurationMaintenancePrepareSchema } from "../ops/configuration-maintenance"; import type { PreparedConfigurationRecord } from "../ops/configuration-preparation-records"; import type { OpsOperation } from "../ops/contract"; @@ -81,6 +82,7 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { (value) => !["configuration-fence", "configuration-fence-settle"].includes(value.phase), ), ...(plan.nestedHardlinks ? [plan.nestedHardlinks] : []), + ...(plan.nestedHandoff ? [plan.nestedHandoff] : []), ], children = reservations.map((value) => ({ planIndex: value.ordinal > 31 ? 31 : value.ordinal, @@ -95,14 +97,15 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { effectSettlementDigest: hash, publication: "not_started", })), - recreates = new Set( - plan.steps.flatMap((value) => + recreates = new Set([ + ...(plan.handoff?.serviceIds ?? []), + ...plan.steps.flatMap((value) => value.operation.kind === "service.recreate" || value.operation.kind === "service.recreateIsolated" ? [value.operation.serviceId] : [], ), - ), + ]), futureBase = join( rendered.context.infraDir!, ".trusted-config-generations", @@ -285,7 +288,9 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { ownerBytes = ownerBaseBytes + inspectionBytes + finalBytes + 2 * 8192 + stepBytes + 8192, noGuardOwnerBytes = ownerBaseBytes + 32768, receiptBytes = bytes(receipt), - phaseBytes = (plan.reservations.length + (plan.nestedHardlinks ? 1 : 0)) * 8192; + phaseBytes = + (plan.reservations.length + (plan.nestedHardlinks ? 1 : 0) + (plan.nestedHandoff ? 1 : 0)) * + 8192; if ( inspectionBytes > 65536 || Math.max(ownerBytes, noGuardOwnerBytes) > 131072 || @@ -322,10 +327,31 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { const apply = record.operation as Parameters< typeof buildFixedConfigurationMaintenancePlan >[0]["apply"]; - const recreateServiceIds = - record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + const platform = record.privatePlatformIntent; + const protectedIds = platform + ? record.affectedServiceIds.filter( + (id) => protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : []; + const recreateServiceIds = platform + ? record.affectedServiceIds.filter( + (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle ? record.affectedServiceIds : []; + const handoff = + platform && protectedIds.length + ? { + handoffId: platform.handoffId, + previewFingerprint: platform.previewFingerprint, + serviceIds: protectedIds, + expectedImageIdentities: Object.fromEntries( + protectedIds.map((id) => [id, `sha256:${hash}`]), + ), + mode: { kind: "configuration" as const }, + } + : undefined; const operation = configurationMaintenancePrepareSchema.parse({ kind: "configurationMaintenance.prepare", maintenanceId: record.maintenanceId, @@ -336,7 +362,11 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { revisionId: apply.revisionId, expectedSourceRevision: record.expectedSourceRevision, coordinatorMode: apply.coordinatorMode, - followUp: recreateServiceIds.length ? "service-apply" : "render-only", + followUp: platform + ? "platform-apply" + : recreateServiceIds.length + ? "service-apply" + : "render-only", }); const parent: RetainedParentExecution = { issuer: "api", @@ -387,6 +417,7 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { hardlinkPlan: { plan: rendered.rendered.hardlinkPlan, prune: true }, serviceIds, recreateServiceIds, + ...(handoff ? { handoff } : {}), ...(input.coordinatorImage ? { coordinatorImage: { ...input.coordinatorImage, imageId: `sha256:${hash}` } } : {}), diff --git a/packages/core/src/ops/configuration-handoff.test.ts b/packages/core/src/ops/configuration-handoff.test.ts index 64d84ad3f..4c45ea78f 100644 --- a/packages/core/src/ops/configuration-handoff.test.ts +++ b/packages/core/src/ops/configuration-handoff.test.ts @@ -77,6 +77,11 @@ it("bounds private platform intent without admitting plaintext recovery credenti sourceRevision: operation().maintenanceId, expectedCurrentRevisionId: operation().expectedCurrentRevisionId, expectedDesiredRowVersion: null, + sourceSnapshot: { + revisionId: operation().expectedCurrentRevisionId, + sourceRevision: operation().maintenanceId, + services: [], + }, configuration: { version: 1, domain: { source: "host" } }, affectedServiceIds: ["app-api"], confirmations: { diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts index 5b9625ae2..7e00b75a2 100644 --- a/packages/core/src/ops/configuration-preparation-records.ts +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -45,6 +45,7 @@ export type ConfigurationHardlinksPreview = z.infer Buffer.byteLength(JSON.stringify(value)) <= 32768); + .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 196608); export type PlatformConfigurationIntentRecord = z.infer; export const preparedConfigurationRecordSchema = z.strictObject({ diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index d5884773c..573a201c2 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -9,6 +9,7 @@ export { configurationHandoffInspectionSchema, configurationHandoffIntentSchema, configurationHandoffPrepareOperationSchema, + protectedConfigurationServiceIdSchema, } from "./configuration-handoff-records"; export * from "./configuration-maintenance"; export * from "./configuration-maintenance-control-validation"; diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index d01d9b214..bec232edf 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -32,7 +32,8 @@ "apps/api/src/services/integration-application-postgres.test.ts", "apps/api/src/services/integration-application.test.ts", "apps/api/src/services/standalone-integrations/install.test.ts", - "apps/api/src/services/__tests__/platform-preview-postgres.test.ts" + "apps/api/src/services/__tests__/platform-preview-postgres.test.ts", + "apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts" ], "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", From 11683cbe50626cebb24c57fe9ce33dacba023fc2 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 04:27:26 +0100 Subject: [PATCH 066/135] feat(platform): retain guarded helper launch and prepared acknowledgements --- ...orm-configuration-runtime-postgres.test.ts | 127 +++++++++++++- .../src/__fixtures__/owned-service-host.ts | 16 +- .../src/configuration-handoff-control.ts | 160 +++++++++++++++++ .../src/configuration-handoff-store.ts | 162 +++++++++++++++++- .../src/configuration-maintenance-store.ts | 61 +++++-- apps/ops-agent/src/index.ts | 3 + .../src/platform-configuration-runtime.ts | 9 + apps/ops-agent/src/production-runtime.ts | 2 + apps/ops-agent/src/server.ts | 41 ++++- .../src/build-artifacts/control-authority.ts | 47 +++++ packages/core/src/build-artifacts/index.ts | 1 + 11 files changed, 601 insertions(+), 28 deletions(-) create mode 100644 apps/ops-agent/src/configuration-handoff-control.ts create mode 100644 apps/ops-agent/src/platform-configuration-runtime.ts diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 27d91adc6..934a4b93c 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -1,6 +1,6 @@ /** Actual API candidate -> production registry/agent -> real DM HTTP/SQL guard. - * Docker inspection is the only substituted I/O; this qualifies preparation, - * not yet helper launch/application or the public platform HTTP routes. */ + * Docker I/O is substituted. This qualifies preparation, launch reconciliation + * and real child adoption; not native helper application/public platform HTTP. */ import { randomUUID } from "node:crypto"; import { rmSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -12,7 +12,7 @@ import { import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { eq } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; @@ -51,14 +51,14 @@ const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "owned platform preparation", () => { - beforeAll(async () => { + beforeEach(async () => { state.fixture = await createMigratedPostgresFixture(); }, 90000); - afterAll(async () => { + afterEach(async () => { await state.fixture?.stop(); }); - it.each(["old", "1"])( - "preflights actual helper image version %s before desired changes", + it.each(["old", "1", "launch", "launch-lost"])( + "guards and retains helper mode %s through original ownership", async (version) => { const ids = [ "motis", @@ -76,7 +76,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "ops-agent": { BACKUP_RETENTION_DAYS: "${BACKUP_RETENTION_DAYS:-30}" }, "app-api": { BACKUP_RETENTION_DAYS: "${BACKUP_RETENTION_DAYS:-30}" }, }, - configurationHelperVersion: version, + configurationHelperVersion: version.startsWith("launch") ? "1" : version, + configurationHelperLaunchResponseLoss: version === "launch-lost", actualComposeConfig: true, }); state.services = host.loaded; @@ -220,6 +221,30 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); await prospective.stageProspectiveConfigurationMaintenance(candidate); await prospective.prepareConfigurationMaintenance(candidate); + if (version.startsWith("launch")) { + const decision = + await prospective.commitProspectiveConfigurationMaintenance(candidate); + await prospective.decideConfigurationMaintenance(decision); + const prepared = await prospective.inspectConfigurationMaintenance({ + maintenanceId: + prospective.validatedConfigurationCandidateState(candidate).record + .maintenanceId!, + candidateId: candidate.candidateId, + }); + for (const step of prepared.steps) + await prospective.executeConfigurationMaintenanceStep( + candidate, + step.ordinal, + ); + expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); + // Reattachment observes the same terminal preparation; + // it cannot consume another dispatch ticket/create again. + await prospective.executeConfigurationMaintenanceStep( + candidate, + prepared.steps.at(-1)!.ordinal, + ); + expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); + } return candidate; }); if (version === "old") @@ -233,6 +258,14 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( try { candidate = await run(); } catch (error) { + console.info( + "platform-step-status", + journal.records().map((job) => ({ + kind: job.operation.kind, + state: job.state, + errorClass: job.errorClass, + })), + ); const value = error as { result?: { errorCode?: unknown; configurationPreflight?: unknown }; }; @@ -248,7 +281,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const retained = ctx.checkpointResult?._configurationMaintenance as | { phase?: string; receipt?: { state?: string; steps?: { phase: string }[] } } | undefined; - expect(retained?.receipt?.state).toBe("prepared"); + expect(retained?.receipt?.state).toBe( + version.startsWith("launch") ? "authorized" : "prepared", + ); expect(retained?.receipt?.steps?.at(-1)?.phase).toBe("maintenance-handoff"); expect(candidate.affectedServiceIds).toEqual(["app-api", "ops-agent"]); const receipt = await prospective.inspectConfigurationMaintenance({ @@ -269,6 +304,80 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); expect(owner.plan.nestedHandoff?.recipient).toBe("configuration-helper"); expect(receipt.binding.expectedSourceRevision).toBe(source.sourceRevision); + if (version.startsWith("launch")) { + expect(receipt).toMatchObject({ + state: "applying", + followUp: "unknown", + dmFence: { state: "prepared" }, + }); + await expect(dm.assertDataAdmission()).rejects.toThrow(); + const [saved] = await state.fixture.db + .select() + .from(schema.systemSettings) + .where(eq(schema.systemSettings.key, "platformConfiguration")); + expect(saved?.value).toMatchObject({ + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }); + expect(dm.settlePosts).toBe(0); + const blocked = await fetch(`${url}/v1/operations`, { + method: "POST", + headers: { + authorization: `Bearer ${host.token}`, + "x-ops-protocol-version": "2", + "content-type": "application/json", + }, + body: JSON.stringify({ + version: 2, + requestId: `ops1_${randomUUID().replaceAll("-", "")}`, + operationKey: createDurableOpsKey("platform.blocked", jobId), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 15000).toISOString(), + execution: { requestedDeadlineAt: Date.now() + 60000 }, + operation: { kind: "service.restart", serviceId: "motis" }, + }), + }); + expect(blocked.status).toBe(409); + expect(await blocked.json()).toMatchObject({ error: { class: "busy" } }); + expect( + (await createApiOpsClient().execute({ kind: "docker.status" })).execution, + ).toBe("sync"); + const childControl = await import( + "../../../../ops-agent/src/configuration-handoff-control" + ); + const args = host.calls.find((args) => args[0] === "run")!; + const env = (name: string) => + args.find((value) => value.startsWith(`${name}=`))!.slice(name.length + 1); + await expect( + childControl.openConfigurationHandoffChild({ + rootDir: host.root, + handoffId: owner.plan.handoff!.handoffId, + assertion: env("OPENMAPX_CONTROL_DELEGATION"), + execution: { + ...JSON.parse(env("OPENMAPX_CONTROL_EXECUTION")), + executionId: randomUUID(), + }, + }), + ).rejects.toThrow("Original configuration helper spawn required"); + const child = await childControl.openConfigurationHandoffChild({ + rootDir: host.root, + handoffId: owner.plan.handoff!.handoffId, + assertion: env("OPENMAPX_CONTROL_DELEGATION"), + execution: JSON.parse(env("OPENMAPX_CONTROL_EXECUTION")), + }); + try { + await childControl.waitForConfigurationHandoffPreparation( + child, + AbortSignal.timeout(2000), + ); + expect( + host.calls.filter((args) => args[0] === "compose" && args.includes("up")), + ).toHaveLength(0); + } finally { + childControl.closeConfigurationHandoffChild(child); + } + return; + } const abandon = await prospective.prepareConfigurationMaintenanceAbandon(candidate); await prospective.decideConfigurationMaintenance(abandon); diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index 4d12e33d8..a4096aa26 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { configurationHelperLaunchResponseLoss?:boolean; selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -131,10 +131,22 @@ export async function ownedServiceHost(options: { selectedRoots?:string[]; confi let inventoryProcess: (() => Promise) | undefined; let imagePullProcess: ((reference:string,signal:AbortSignal)=>Promise) | undefined; const imageIdentities = new Map(); + const optionsForHelperLaunchResponseLoss = options.configurationHelperLaunchResponseLoss; + const helperContainers = new Map(); const execFile: NonNullable = async (_file, args, options) => { calls.push([...args]); let stdout = ""; - if(configurationHelperVersion!==undefined && args[0]==="container" && args[1]==="inspect" && args[2]==="--format" && (["{{.Image}}","{{.Config.User}}","{{json .HostConfig.GroupAdd}}","{{json .Mounts}}"].includes(args[3]??"") || args[3]?.includes("range .Config.Env"))) { + if (configurationHelperVersion !== undefined && args[0] === "run") { + const name = args[args.indexOf("--name") + 1]!, label = args[args.indexOf("--label") + 1]!; + if (!name.startsWith("openmapx-configuration-handoff-") || !label.startsWith("openmapx.configuration-handoff=")) throw Error("Unexpected helper launch"); + if (helperContainers.has(name)) throw Error("Duplicate helper launch"); + const Id = "e".repeat(64); + helperContainers.set(name, { Id, Image: imageId, Config: { Labels: { "openmapx.configuration-handoff": label.split("=")[1] } } }); + stdout = Id; + if (optionsForHelperLaunchResponseLoss) throw Error("Named lost Docker launch response"); + } else if (args[0] === "container" && args[1] === "inspect" && helperContainers.has(args.at(-1)!)) { + stdout = JSON.stringify(helperContainers.get(args.at(-1)!)); + } else if(configurationHelperVersion!==undefined && args[0]==="container" && args[1]==="inspect" && args[2]==="--format" && (["{{.Image}}","{{.Config.User}}","{{json .HostConfig.GroupAdd}}","{{json .Mounts}}"].includes(args[3]??"") || args[3]?.includes("range .Config.Env"))) { const selected=rows.find(row=>row.Id===args[4]); if(!selected)throw Error("Named helper prerequisite container unavailable"); const format=args[3]; diff --git a/apps/ops-agent/src/configuration-handoff-control.ts b/apps/ops-agent/src/configuration-handoff-control.ts new file mode 100644 index 000000000..de2f229e7 --- /dev/null +++ b/apps/ops-agent/src/configuration-handoff-control.ts @@ -0,0 +1,160 @@ +/** The detached helper adopts only its original fixed child grant. It has an + * ephemeral signing key, no installation role key and no root acquisition. */ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { + adoptArtifactTargetLease, + assertArtifactTargetLeaseExecution, + type BorrowedArtifactTargetLease, + beginFixedCliExecution, + bindBorrowedControlExecution, + type ControlInstallation, + closeArtifactTargetLeaseHandle, + closeControlInstallation, + controlDigest, + executionAuthorityState, + openFixedChildInstallation, + readConfigurationHandoffSpawnIntent, + retainAdmittedControlExecution, + type VerifiedExecutionAuthority, + verifiedParentForArtifactDelegation, + verifyConfigurationHelperReady, + verifyFixedCliControlPeer, +} from "@openmapx/core/build-artifacts"; +import { canonicalJson, type ExecutionRef, executionRefSchema } from "@openmapx/core/ops"; +import type { ConfigurationHandoffIntentRecord } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; + +declare const childBrand: unique symbol; +export type ConfigurationHandoffChild = Readonly<{ readonly [childBrand]: true }>; +const children = new WeakMap< + ConfigurationHandoffChild, + { + installation: ControlInstallation; + authority: VerifiedExecutionAuthority; + lease: BorrowedArtifactTargetLease; + intent: ConfigurationHandoffIntentRecord; + } +>(); +function immutable(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) immutable(child); + Object.freeze(value); + } + return value; +} + +export async function openConfigurationHandoffChild(input: { + rootDir: string; + handoffId: string; + assertion: string; + execution: ExecutionRef; +}): Promise { + const execution = executionRefSchema.parse(input.execution), + infraDir = join(input.rootDir, "infra/docker"), + installation = openFixedChildInstallation({ + infraDir, + assertion: input.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + let lease: BorrowedArtifactTargetLease | undefined; + try { + const intent = immutable( + readConfigurationHandoffSpawnIntent(installation, execution, input.handoffId), + ); + if ( + intent.installationId !== installation.installationId || + intent.operation.handoffId !== input.handoffId || + intent.assertion !== input.assertion || + intent.helperPublicKey !== installation.publicKey || + canonicalJson(intent.helper) !== canonicalJson(execution) + ) + throw Error("Original fixed helper launch required"); + const body = { + kind: "configuration-helper", + handoffId: input.handoffId, + intentDigest: controlDigest(intent.operation), + }, + peer = verifyFixedCliControlPeer(installation, { + assertion: input.assertion, + execution, + kind: body.kind, + recipient: "configuration-helper", + }), + parent = verifiedParentForArtifactDelegation(installation, input.assertion, execution); + if ( + canonicalJson(parent.operationKey) !== canonicalJson(intent.preparer.operationKey) || + parent.fingerprint !== intent.preparer.fingerprint + ) + throw Error("Original helper preparer required"); + lease = await adoptArtifactTargetLease({ + dataRoot: join(infraDir, "data"), + assertion: input.assertion, + peer, + execution, + serviceIds: intent.operation.serviceIds, + }); + const authority = retainAdmittedControlExecution(installation, { + execution, + kind: body.kind, + operation: body, + admittedAt: intent.admittedAt, + requestedDeadlineAt: intent.requestedDeadlineAt, + parent, + serviceIds: intent.operation.serviceIds, + }); + bindBorrowedControlExecution(lease, authority); + assertArtifactTargetLeaseExecution(lease, execution); + beginFixedCliExecution(installation, execution); + const child = Object.freeze({}) as ConfigurationHandoffChild; + children.set(child, { installation, authority, lease, intent }); + return child; + } catch (error) { + if (lease) closeArtifactTargetLeaseHandle(lease); + closeControlInstallation(installation); + throw error; + } +} +export function configurationHandoffChildState(child: ConfigurationHandoffChild) { + const state = children.get(child); + if (!state) throw Error("Original nominal helper required"); + executionAuthorityState(state.authority); + return state; +} +export async function waitForConfigurationHandoffPreparation( + child: ConfigurationHandoffChild, + signal: AbortSignal, +): Promise { + const { lease, intent } = configurationHandoffChildState(child); + for (;;) { + signal.throwIfAborted(); + if (Date.now() >= intent.helper.deadlineAt) throw Error("Original helper deadline expired"); + try { + verifyConfigurationHelperReady(lease); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await new Promise((resolve, reject) => { + const done = () => { + clearTimeout(timer); + signal.removeEventListener("abort", aborted); + }; + const aborted = () => { + done(); + reject(signal.reason); + }; + const timer = setTimeout(() => { + done(); + resolve(); + }, 100); + signal.addEventListener("abort", aborted, { once: true }); + if (signal.aborted) aborted(); + }); + } +} +export function closeConfigurationHandoffChild(child: ConfigurationHandoffChild): void { + const state = configurationHandoffChildState(child); + children.delete(child); + closeArtifactTargetLeaseHandle(state.lease); + closeControlInstallation(state.installation); +} diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index 880dd1945..bc3811756 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -1,9 +1,12 @@ +import { randomUUID } from "node:crypto"; import { existsSync } from "node:fs"; /** Fixed original-owner helper journal. All cells stay in the existing indexed * role-private observations namespace and original retention family. */ import { + assertArtifactTargetLeaseForward, bindControlChildPhase, closeControlInstallation, + consumeControlChildDispatch, controlDigest, controlRolePublicKey, executionAuthorityState, @@ -12,6 +15,7 @@ import { lookupControlChildPhaseAttempt, lookupControlChildPhaseBinding, openFixedChildInstallation, + prepareFixedControlChild, readPrivateRecord, signControl, type VerifiedExecutionAuthority, @@ -32,9 +36,13 @@ import { configurationHandoffRecordDomains as domains, configurationHandoffRecordName as recordName, } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; -import { inspectConfigurationHandoffPrerequisite } from "./configuration-handoff"; +import { + CONFIGURATION_HANDOFF_LABEL, + configurationHandoffRunArgs, + inspectConfigurationHandoffPrerequisite, +} from "./configuration-handoff"; import { assertOwnedMaintenanceHandoffRuntime } from "./configuration-maintenance-store"; -import type { OpsExecutionContext, OpsRuntime } from "./runtime"; +import { type OpsExecutionContext, OpsExecutionFailure, type OpsRuntime } from "./runtime"; declare const preparedBrand: unique symbol; export type PreparedConfigurationHandoff = Readonly<{ readonly [preparedBrand]: true }>; @@ -166,10 +174,11 @@ export async function prepareConfigurationHandoffLaunch( if (lookupControlChildPhaseAttempt(authority, reservation)) throw Error("Attempted helper without original launch intent requires recovery"); const bound = await bindControlChildPhase(authority, reservation), + publicKey = prepareFixedControlChild(authority, randomUUID()), issued = await issueControlChildPhase(authority, { phase: "configuration-handoff", ordinal: 0, - childPublicKey: "", + childPublicKey: publicKey, lease, }), attempt = lookupControlChildPhaseAttempt(authority, reservation), @@ -265,3 +274,150 @@ export function preparedConfigurationHandoffState(value: PreparedConfigurationHa if (bytes(observed) !== bytes(state.intent)) throw Error("Original handoff launch changed"); return state; } + +export function assertPreparedConfigurationHandoff( + authority: VerifiedExecutionAuthority, + result: unknown, +): void { + const local = executionAuthorityState(authority); + if (local.kind !== "configurationHandoff.prepare") + throw Error("Original handoff preparation required"); + const body = result as { handoffId?: string; phase?: string } | null; + if (!body || body.phase !== "prepared" || !body.handoffId) + throw Error("Original prepared result required"); + const intent = readIntent(authority, body.handoffId), + progress = readProgress(authority, intent); + if (progress.phase !== "launched" || progress.launch.state !== "confirmed") + throw Error("Original confirmed helper launch required"); +} + +function readProgress( + authority: VerifiedExecutionAuthority, + intent: ConfigurationHandoffIntentRecord, +) { + const local = executionAuthorityState(authority), + name = recordName(intent.operation.handoffId, "progress"), + raw = readPrivateRecord(local.installation, "observations", name), + value = verifyControl( + local.installation.installation.publicKey, + domains.progress, + raw, + configurationHandoffProgressSchema, + ); + indexed(authority, name, raw); + if ( + value.intentDigest !== controlDigest(intent) || + bytes(value.targets.map((target) => target.serviceId).sort()) !== + bytes(intent.operation.serviceIds) + ) + throw Error("Original handoff progress changed"); + return value; +} + +/** First dispatch alone consumes the original helper ticket. Once launch intent + * is durable, every recovery only inspects the deterministic container. */ +export async function launchPreparedConfigurationHandoff( + runtime: OpsRuntime, + operation: Extract, + context: OpsExecutionContext, +) { + const marker = await prepareConfigurationHandoffLaunch(runtime, operation, context), + state = preparedConfigurationHandoffState(marker), + { authority, intent } = state, + source = (await assertOwnedMaintenanceHandoffRuntime(runtime, operation, context)).source; + let progress = readProgress(authority, intent); + const inspect = async () => { + const result = await source.execFile( + "docker", + ["container", "inspect", "--format", "{{json .}}", intent.helperContainerName], + { + signal: context.signal, + timeout: 15000, + maxBuffer: 65536, + }, + ); + const value = JSON.parse(result.stdout) as { + Id?: string; + Image?: string; + Config?: { Labels?: Record }; + }; + if ( + !value || + !/^[a-f0-9]{64}$/.test(value.Id ?? "") || + value.Image !== state.facts.imageId || + value.Config?.Labels?.[CONFIGURATION_HANDOFF_LABEL] !== operation.handoffId + ) + throw Error("Original helper container required"); + return value.Id!; + }; + if (progress.launch.state === "not_attempted") { + if (!state.dispatchTicket) throw new OpsExecutionFailure("recovery_required", {}); + context.signal.throwIfAborted(); + assertArtifactTargetLeaseForward(context.artifactLease!); + // Source and target verification precede this durable one-use launch marker. + progress = { + ...progress, + sequence: progress.sequence + 1, + phase: "launch_attempted", + launch: { state: "attempted", containerId: null }, + updatedAt: Date.now(), + }; + writeProgress(authority, intent, progress); + consumeControlChildDispatch(state.dispatchTicket); + prepared.set(marker, Object.freeze({ authority, intent, facts: state.facts })); + try { + await source.execFile( + "docker", + configurationHandoffRunArgs(state.facts, source.rootDir!, { + handoffId: operation.handoffId, + assertion: intent.assertion, + execution: intent.helper, + }), + { signal: context.signal, timeout: 30000, maxBuffer: 65536 }, + ); + } catch { + // A lost create response is resolved by the exact name/label/image below; + // no second run is permitted, including after a process replacement. + } + } + let containerId: string; + try { + containerId = await inspect(); + } catch { + throw new OpsExecutionFailure("recovery_required", {}); + } + if (progress.launch.containerId && progress.launch.containerId !== containerId) + throw new OpsExecutionFailure("recovery_required", {}); + if (progress.launch.state !== "confirmed") { + progress = { + ...progress, + sequence: progress.sequence + 1, + phase: "launched", + launch: { state: "confirmed", containerId }, + updatedAt: Date.now(), + }; + writeProgress(authority, intent, progress); + } + // The helper cannot start effects until the real server publishes this + // original terminal preparation. There is no writer held while draining. + while ((context.otherActiveOperations?.() ?? 0) > 0) { + context.signal.throwIfAborted(); + await new Promise((resolve, reject) => { + const done = () => { + clearTimeout(timer); + context.signal.removeEventListener("abort", aborted); + }; + const aborted = () => { + done(); + reject(context.signal.reason); + }; + const timer = setTimeout(() => { + done(); + resolve(); + }, 100); + context.signal.addEventListener("abort", aborted, { once: true }); + if (context.signal.aborted) aborted(); + }); + } + return { handoffId: operation.handoffId, phase: "prepared" as const }; +} diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index 80ced0ae2..33d273db6 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -205,6 +205,7 @@ import { installationState, publishConfigurationMaintenanceForwardWitness, verifyConfigurationMaintenanceChildTerminal, + verifyControlPreparedTerminal, withPrivateEffectWriter, } from "../../../packages/core/src/build-artifacts/control-authority"; import { CONTROL_RETENTION_LIMITS } from "../../../packages/core/src/build-artifacts/control-retention"; @@ -614,9 +615,12 @@ export async function prepareOwnedConfigurationMaintenanceStepAdmission( ); if ( !step || - !["configuration-commit", "maintenance-hardlinks", "maintenance-recreate"].includes( - step.phase, - ) || + ![ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(step.phase) || canonicalJson(step.operation) !== canonicalJson(operation) || actor.apiJobId !== original.apiJobId || actor.parentOperationKey !== original.operationKey || @@ -641,6 +645,14 @@ export async function prepareOwnedConfigurationMaintenanceStepAdmission( await verifyFrozenMaintenanceHardlinksSource(runtime, authority, request.signal); else if (step.phase === "maintenance-recreate") await verifyFrozenMaintenanceRecreateSource(runtime, authority, step.ordinal, request.signal); + else if (step.phase === "maintenance-handoff") + await verifyFrozenMaintenanceFollowUpSource( + runtime, + authority, + step.ordinal, + request.signal, + "maintenance-handoff", + ); else throw new ConfigurationMaintenanceControlUnauthorized( "Uninstalled frozen maintenance continuation", @@ -758,7 +770,9 @@ export async function prepareOwnedConfigurationMaintenanceStepAdmission( reservations: step.phase === "maintenance-hardlinks" && owner.plan.nestedHardlinks ? [owner.plan.nestedHardlinks] - : [], + : step.phase === "maintenance-handoff" && owner.plan.nestedHandoff + ? [owner.plan.nestedHandoff] + : [], async adopt(execution: ExecutionRef) { if (!freshDispatch || canonicalJson(execution) !== canonicalJson(binding.child.execution)) throw new ConfigurationMaintenanceControlConflict("Attempted frozen phase is lookup-only"); @@ -1600,7 +1614,9 @@ export function recordOwnedConfigurationMaintenanceStepTerminal( lease = restoreArtifactTargetLeaseForSettlement(authority); try { const physical = inspectArtifactTargetLeaseChildSettlement(lease, execution); - if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) + const handoff = owner.plan.steps[source.ordinal]?.phase === "maintenance-handoff"; + if (handoff) verifyControlPreparedTerminal(local, execution); + else if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) throw Error("Original phase physical settlement unavailable"); withConfigurationMaintenanceOwnerObservation(lease, authority, () => { const current = read(authority), @@ -1622,9 +1638,11 @@ export function recordOwnedConfigurationMaintenanceStepTerminal( value.ordinal === source.ordinal ? { ...value, canonical } : value, ), sequence = current.sequence + 1, - complete = current.plan.steps.every((step) => - steps.some((value) => value.ordinal === step.ordinal && value.canonical !== null), - ), + complete = + !current.plan.handoff && + current.plan.steps.every((step) => + steps.some((value) => value.ordinal === step.ordinal && value.canonical !== null), + ), generation = current.plan.steps[source.ordinal]!.phase === "configuration-commit", notCommitted = generation && @@ -1645,7 +1663,13 @@ export function recordOwnedConfigurationMaintenanceStepTerminal( appliedRevisionId: notCommitted ? null : current.plan.operation.revisionId, } : {}), - ...(notCommitted ? { followUp: "contained" } : complete ? { followUp: "complete" } : {}), + ...(handoff + ? { followUp: "unknown" } + : notCommitted + ? { followUp: "contained" } + : complete + ? { followUp: "complete" } + : {}), }, }); }); @@ -1699,7 +1723,9 @@ export function acknowledgeConfigurationMaintenanceStep( lease = restoreArtifactTargetLeaseForSettlement(authority); try { const physical = inspectArtifactTargetLeaseChildSettlement(lease, source.execution); - if (!physical || canonicalJson(physical) !== canonicalJson(actual.settlement)) + if (owner.plan.steps[source.ordinal]?.phase === "maintenance-handoff") + verifyControlPreparedTerminal(local, source.execution); + else if (!physical || canonicalJson(physical) !== canonicalJson(actual.settlement)) throw Error("Original copied phase settlement unavailable"); return withConfigurationMaintenanceOwnerObservation(lease, authority, () => { const current = read(authority), @@ -3771,6 +3797,21 @@ export async function assertStagedCoordinatorConfigurationWrite( throw new OpsExecutionFailure("configuration_prerequisite", {}); } const journals = new WeakMap(); +/** Conservative drain gate from the actual durable journal. This closes new + * work only; a matching ID never supplies continuation or release authority. */ +export function configurationHandoffPendingMaintenanceIds( + runtime: OpsRuntime, +): ReadonlySet { + return new Set( + (journals.get(runtime)?.records() ?? []) + .filter((job) => job.operation.kind === "configurationHandoff.prepare") + .map( + (job) => + (job.operation as Extract) + .maintenanceId, + ), + ); +} /** Bound to the server's actual durable journal. Records alone cannot qualify * a pull: the fixed reader authenticates canonical/effect/private admissions. */ export function registerConfigurationMaintenanceJournal( diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 573230d91..4865f8651 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -20,6 +20,7 @@ import { ArtifactTransferStore, artifactTransferPrivateRoot } from "./artifact-t import { startOpsAgentServer } from "./bootstrap"; import { CommunityRepositoryStore } from "./community-repositories"; import { loadOpsAgentConfig } from "./config"; +import { configurationHandoffPendingMaintenanceIds } from "./configuration-maintenance-store"; import { createDataTransitTrustedOpsDataSource } from "./data-transit-resource-source"; import { createDockerDawarichCollector, @@ -200,6 +201,7 @@ async function main(): Promise { }, }); const replacementInProgress = () => { + if (configurationHandoffPendingMaintenanceIds(runtime).size) return true; try { const handoff = readAgentHandoff(config.rootDir); return handoff !== null && !isTerminalHandoff(handoff.phase); @@ -308,6 +310,7 @@ async function main(): Promise { runtime, artifactTransfers, replacementInProgress: () => { + if (configurationHandoffPendingMaintenanceIds(runtime).size) return true; try { const handoff = readAgentHandoff(config.rootDir); return handoff !== null && !isTerminalHandoff(handoff.phase); diff --git a/apps/ops-agent/src/platform-configuration-runtime.ts b/apps/ops-agent/src/platform-configuration-runtime.ts new file mode 100644 index 000000000..01e6f26b8 --- /dev/null +++ b/apps/ops-agent/src/platform-configuration-runtime.ts @@ -0,0 +1,9 @@ +/** Fixed original-owner protected handoff. Old images fail the explicit bundle + * prerequisite before ownership; the installer does not advertise a helper. */ +import { launchPreparedConfigurationHandoff } from "./configuration-handoff-store"; +import type { OpsRuntime } from "./runtime"; + +export function installPlatformConfigurationRuntime(runtime: OpsRuntime): void { + runtime["configurationHandoff.prepare"] = (operation, context) => + launchPreparedConfigurationHandoff(runtime, operation, context); +} diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 6a122ad3d..6958a77ed 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -31,6 +31,7 @@ import { } from "./docker-runtime"; import { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; import { createOwnedServiceResolver, type OwnedServiceResolver } from "./owned-services"; +import { installPlatformConfigurationRuntime } from "./platform-configuration-runtime"; import { serviceActionForOperation } from "./policy"; import { type OpsExecutionContext, @@ -249,6 +250,7 @@ export async function createProductionOpsRuntime( execFile: options.docker?.execFile, }); installCatalogApprovalRuntime(runtime, { rootDir: options.rootDir }); + installPlatformConfigurationRuntime(runtime); createAdministrativeRuntime(runtime, { rootDir: options.rootDir, runFixedCli, diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 543e4378f..ce516db48 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -100,6 +100,7 @@ import { registerArtifactTransferRoutes } from "./artifact-transfer/routes"; import type { ArtifactTransferStore } from "./artifact-transfer/store"; import { readCommittedCatalogApproval } from "./catalog-approval-runtime"; import { configurationHandoffBlocksReplacement } from "./configuration-handoff"; +import { assertPreparedConfigurationHandoff } from "./configuration-handoff-store"; import { prepareConfigurationHardlinksAdmission } from "./configuration-hardlinks-runtime"; import { configurationMaintenanceAdmissionState, @@ -113,6 +114,7 @@ import { assertManagedCliConfigurationPreflight, ConfigurationMaintenanceControlConflict, ConfigurationMaintenanceControlUnauthorized, + configurationHandoffPendingMaintenanceIds, configurationMaintenanceControlOwnerState, inspectConfigurationMaintenanceDecisionControl, inspectConfigurationMaintenanceSettleControl, @@ -188,6 +190,9 @@ export const OPS_AGENT_RETENTION_LIMITS = { } as const; // Conservative read-only surface while a retained release transaction needs reconciliation. const RELEASE_RECOVERY_READ_KINDS = new Set([ + "configurationHandoff.inspect", + "configurationMaintenance.inspect", + "system.diagnostics", "docker.status", "stack.status", "services.inspect", @@ -462,7 +467,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif registerArtifactTransferRoutes(app, { store: options.artifactTransfers, apiToken: options.tokens.api, - admissionsOpen: () => !options.replacementInProgress?.(), + admissionsOpen: () => + !options.replacementInProgress?.() && + configurationHandoffPendingMaintenanceIds(runtime).size === 0, }); const runtime = options.runtime ?? createUnavailableRuntime(); const dispatch = @@ -817,6 +824,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const committed = state === "succeeded" && updates.result !== undefined; const preparedMaintenance = committed && job.operation.kind === "configurationMaintenance.prepare"; + const preparedHandoff = committed && job.operation.kind === "configurationHandoff.prepare"; + if (preparedHandoff) assertPreparedConfigurationHandoff(authority, updates.result); if ( preparedMaintenance && canonicalJson(readOwnedConfigurationMaintenance(authority).receipt) !== @@ -885,7 +894,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } catch { /* Original retained owner/recovery path remains mandatory. */ } - } else if (lease && job.operation.kind !== "configurationMaintenance.prepare") { + } else if ( + lease && + job.operation.kind !== "configurationMaintenance.prepare" && + !preparedHandoff + ) { await settleArtifactTargetLeaseChild( lease, verifyControlEffectSettlement(options.controlInstallation, authority.execution), @@ -925,7 +938,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if ( authority && state === "succeeded" && - job.operation.kind === "configurationMaintenance.prepare" + (job.operation.kind === "configurationMaintenance.prepare" || + job.operation.kind === "configurationHandoff.prepare") ) retainControlPreparedTerminal(authority); Object.assign(job, updates); @@ -1530,6 +1544,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif job.state === "succeeded" && (isTrustedConfigurationApplyOperation(job.operation) || job.operation.kind === "configuration.hardlinks.apply" || + job.operation.kind === "configurationHandoff.prepare" || job.operation.kind === "service.recreate" || job.operation.kind === "service.recreateIsolated") && job.operation.maintenanceId @@ -1998,6 +2013,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif existing && (isTrustedConfigurationApplyOperation(existing.operation) || existing.operation.kind === "configuration.hardlinks.apply" || + existing.operation.kind === "configurationHandoff.prepare" || existing.operation.kind === "service.recreate" || existing.operation.kind === "service.recreateIsolated") && existing.operation.maintenanceId; @@ -2134,6 +2150,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if ( (isTrustedConfigurationApplyOperation(operation) || operation.kind === "configuration.hardlinks.apply" || + operation.kind === "configurationHandoff.prepare" || operation.kind === "service.recreate" || operation.kind === "service.recreateIsolated") && operation.maintenanceId @@ -2254,10 +2271,26 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("busy", 429); } // A resumed agent update re-attaches above; nothing new may start. - if (operation.kind !== "agent.update" && options.replacementInProgress?.()) { + if ( + operation.kind !== "agent.update" && + options.replacementInProgress?.() && + configurationHandoffPendingMaintenanceIds(runtime).size === 0 + ) { throw new RequestGuardError("busy", 409); } } + const pendingHandoffs = configurationHandoffPendingMaintenanceIds(runtime); + if ( + pendingHandoffs.size && + !RELEASE_RECOVERY_READ_KINDS.has(kind) && + !( + configurationMaintenanceControl && + "maintenanceId" in operation && + operation.maintenanceId && + pendingHandoffs.has(operation.maintenanceId) + ) + ) + throw new RequestGuardError("busy", 409); if (options.releaseRecovery?.ready === false && !RELEASE_RECOVERY_READ_KINDS.has(kind)) throw new RequestGuardError("recovery_required", 409); if ( diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 591dc4204..d4d292922 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1,4 +1,9 @@ import { dataWorkflowReceiptSchema } from "../ops/job-evidence"; +import { + configurationHandoffIntentRecordSchema, + configurationHandoffRecordDomains, + configurationHandoffRecordName, +} from "./configuration-handoff-records"; import { type ControlChildPhase, type ControlRecipient, @@ -4922,6 +4927,48 @@ export function readFixedCliSpawnIntent( s.registeredTargets = new Set(saved.serviceIds); return saved; } +/** Fixed helper enrollment from the original indexed role-signed launch. A + * caller target list cannot enroll a child, and the role key remains masked. */ +export function readConfigurationHandoffSpawnIntent( + installation: ControlInstallation, + execution: ExecutionRef, + handoffId: string, +) { + const s = installationState(installation), + name = configurationHandoffRecordName(handoffId, "intent"), + raw = readPrivateRecord(s, "observations", name), + saved = verifyControl( + s.publicKeys["ops-agent"], + configurationHandoffRecordDomains.intent, + raw, + configurationHandoffIntentRecordSchema, + ); + if ( + !s.fixedChildId || + installation.role !== "ops-agent" || + s.fixedChildId !== saved.delegationId || + installation.installationId !== saved.installationId || + installation.publicKey !== saved.helperPublicKey || + saved.operation.handoffId !== handoffId || + canonicalJson(saved.helper) !== canonicalJson(execution) || + execution.executor !== "host-cli" || + execution.operationFingerprint !== + controlDigest({ + kind: "configuration-helper", + handoffId, + intentDigest: controlDigest(saved.operation), + }) + ) + throw Error("Original configuration helper spawn required"); + verifyFamilyOwnedRecord( + s, + saved.origin, + { scope: "private", namespace: "observations", container: "", name }, + canonicalJson(raw), + ); + s.registeredTargets = new Set(saved.operation.serviceIds); + return saved; +} export function beginFixedCliExecution( installation: ControlInstallation, execution: ExecutionRef, diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index e70dfb6d9..b746fddb9 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -79,6 +79,7 @@ export { ownedStagedCoordinatorBaseline, prepareFixedControlChild, provisionControlInstallation, + readConfigurationHandoffSpawnIntent, readFixedCliSpawnIntent, readPrivateRecord, readVerifiedStagedCoordinatorOwnership, From 5dd40b333c3b4b9dc521ebce54ed7165650563eb Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 04:59:49 +0100 Subject: [PATCH 067/135] feat(platform): verify helper sources and guard first publication --- ...orm-configuration-runtime-postgres.test.ts | 62 +++++++- .../platform-configuration-participant.ts | 23 +++ .../src/services/prospective-configuration.ts | 2 + .../src/configuration-handoff-control.ts | 6 + .../src/configuration-handoff-store.ts | 69 ++++++++- apps/ops-agent/src/configuration-handoff.ts | 25 ++++ .../src/platform-configuration-runtime.ts | 12 +- apps/ops-agent/src/production-runtime.ts | 2 +- .../configuration-handoff-records.ts | 12 ++ .../configuration-source-witness.ts | 140 +++++++++++++++++- packages/core/src/build-artifacts/index.ts | 2 + 11 files changed, 345 insertions(+), 10 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 934a4b93c..a3c1944db 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -2,7 +2,7 @@ * Docker I/O is substituted. This qualifies preparation, launch reconciliation * and real child adoption; not native helper application/public platform HTTP. */ import { randomUUID } from "node:crypto"; -import { rmSync, statSync, writeFileSync } from "node:fs"; +import { existsSync, rmSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { closeControlInstallation, @@ -57,7 +57,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( afterEach(async () => { await state.fixture?.stop(); }); - it.each(["old", "1", "launch", "launch-lost"])( + it.each(["old", "1", "launch", "launch-lost", "policy-change"])( "guards and retains helper mode %s through original ownership", async (version) => { const ids = [ @@ -76,7 +76,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "ops-agent": { BACKUP_RETENTION_DAYS: "${BACKUP_RETENTION_DAYS:-30}" }, "app-api": { BACKUP_RETENTION_DAYS: "${BACKUP_RETENTION_DAYS:-30}" }, }, - configurationHelperVersion: version.startsWith("launch") ? "1" : version, + configurationHelperVersion: version === "old" ? "old" : "1", configurationHelperLaunchResponseLoss: version === "launch-lost", actualComposeConfig: true, }); @@ -221,7 +221,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); await prospective.stageProspectiveConfigurationMaintenance(candidate); await prospective.prepareConfigurationMaintenance(candidate); - if (version.startsWith("launch")) { + if (version.startsWith("launch") || version === "policy-change") { const decision = await prospective.commitProspectiveConfigurationMaintenance(candidate); await prospective.decideConfigurationMaintenance(decision); @@ -231,6 +231,41 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .maintenanceId!, candidateId: candidate.candidateId, }); + if (version === "policy-change") { + const { applyAuthPolicy, defaultAuthPolicySettings } = await import( + "../../account/auth-policy" + ); + await applyAuthPolicy( + { + confirmed: true, + expectedRevision: policy.revision, + idempotencyKey: randomUUID(), + settings: { ...defaultAuthPolicySettings, passwordMinLength: 14 }, + }, + actorId, + state.fixture.db, + ); + await expect( + prospective.executeConfigurationMaintenanceStep(candidate, 0), + ).rejects.toThrow(/policy|preview/i); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(0); + expect(existsSync(join(host.queue, `${candidate.revisionId}.json`))).toBe( + false, + ); + const [desired] = await state.fixture.db + .select() + .from(schema.systemSettings) + .where(eq(schema.systemSettings.key, "platformConfiguration")); + expect(desired?.value).toMatchObject({ + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + }); + await expect(dm.assertDataAdmission()).rejects.toThrow(); + return candidate; + } for (const step of prepared.steps) await prospective.executeConfigurationMaintenanceStep( candidate, @@ -278,6 +313,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ); throw error; } + if (version === "policy-change") return; const retained = ctx.checkpointResult?._configurationMaintenance as | { phase?: string; receipt?: { state?: string; steps?: { phase: string }[] } } | undefined; @@ -305,6 +341,19 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(owner.plan.nestedHandoff?.recipient).toBe("configuration-helper"); expect(receipt.binding.expectedSourceRevision).toBe(source.sourceRevision); if (version.startsWith("launch")) { + const handoff = await createApiOpsClient().execute({ + kind: "configurationHandoff.inspect", + handoffId: owner.plan.handoff!.handoffId, + }); + expect(handoff).toMatchObject({ + execution: "sync", + value: { + phase: "prepared", + appliedRevisionId: null, + completedServiceIds: [], + failedServiceIds: [], + }, + }); expect(receipt).toMatchObject({ state: "applying", followUp: "unknown", @@ -370,6 +419,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( child, AbortSignal.timeout(2000), ); + childControl.assertConfigurationHandoffHostSources(child); + writeFileSync(join(host.infraDir, ".env"), "OPENMAPX_SYNTHETIC_DRIFT=1\n"); + expect(() => + childControl.assertConfigurationHandoffHostSources(child), + ).toThrow("Helper host sources changed"); expect( host.calls.filter((args) => args[0] === "compose" && args.includes("up")), ).toHaveLength(0); diff --git a/apps/api/src/services/platform-configuration-participant.ts b/apps/api/src/services/platform-configuration-participant.ts index 392076ea9..c705b68e6 100644 --- a/apps/api/src/services/platform-configuration-participant.ts +++ b/apps/api/src/services/platform-configuration-participant.ts @@ -8,6 +8,7 @@ import { eq, sql } from "drizzle-orm"; import { AdminActionReauthentication } from "../account/admin-reauthentication"; import { loadAuthPolicy } from "../account/auth-policy"; import { platformConfigurationPreview, systemSettings } from "../db/schema"; +import { readProjection } from "./admin-job-projection"; import { type AdministrativeConfigurationWriter, administrativeConfigurationState, @@ -221,6 +222,28 @@ export async function preparePlatformConfigurationChange( }) ) throw new PlatformPreviewConflict(); + const generation = Object.values( + readProjection(current.job.checkpointResult).operations, + ).find( + (entry) => + entry.operation?.kind === "stack.render" && + entry.operation.candidateId === candidate.candidateId && + entry.operation.revisionId === candidate.revisionId, + ); + if (!generation?.attempt) { + // A consumed proof authorizes this saved intent, not an indefinitely + // delayed first effect under a changed policy or expired preview. + // Attempted work remains lookup-only and never consumes another proof. + const original = await readPlatformPreview( + { actorId: writer.actorId, previewFingerprint: preview.previewFingerprint }, + current.connection, + ); + if ( + canonicalJson(original.preview) !== canonicalJson(preview) || + (await loadAuthPolicy(current.connection)).revision !== preview.authPolicyRevision + ) + throw new PlatformPreviewConflict("Platform policy changed; create a new preview"); + } return; } const stored = await readPlatformPreview( diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 3ea459f34..a7b84da13 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -4754,6 +4754,8 @@ export async function publishPreparedTrustedConfigurationSnapshot( ) { assertCommittedConfigurationCandidate(candidate); const state = validatedConfigurationCandidateState(candidate); + if (state.record.privatePlatformIntent) + await verifyConfigurationCommitParticipant(candidate, state.writer); const inspected = inspectPreparedConfigurationRecord( encodePreparedConfigurationRecord(state.record, state.publisher.token), state.publisher.token, diff --git a/apps/ops-agent/src/configuration-handoff-control.ts b/apps/ops-agent/src/configuration-handoff-control.ts index de2f229e7..125157f6c 100644 --- a/apps/ops-agent/src/configuration-handoff-control.ts +++ b/apps/ops-agent/src/configuration-handoff-control.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { adoptArtifactTargetLease, assertArtifactTargetLeaseExecution, + assertConfigurationHandoffHostSources as assertHostSources, type BorrowedArtifactTargetLease, beginFixedCliExecution, bindBorrowedControlExecution, @@ -120,6 +121,11 @@ export function configurationHandoffChildState(child: ConfigurationHandoffChild) executionAuthorityState(state.authority); return state; } +export function assertConfigurationHandoffHostSources(child: ConfigurationHandoffChild): void { + const state = configurationHandoffChildState(child); + verifyConfigurationHelperReady(state.lease); + assertHostSources(state.authority); +} export async function waitForConfigurationHandoffPreparation( child: ConfigurationHandoffChild, signal: AbortSignal, diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index bc3811756..f60619ea0 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -5,6 +5,7 @@ import { existsSync } from "node:fs"; import { assertArtifactTargetLeaseForward, bindControlChildPhase, + captureConfigurationHandoffHostSources, closeControlInstallation, consumeControlChildDispatch, controlDigest, @@ -14,9 +15,11 @@ import { issueControlChildPhase, lookupControlChildPhaseAttempt, lookupControlChildPhaseBinding, + openControlInstallation, openFixedChildInstallation, prepareFixedControlChild, readPrivateRecord, + restoreAdmittedControlExecution, signControl, type VerifiedExecutionAuthority, verifyControl, @@ -27,6 +30,7 @@ import { canonicalJson, configurationHandoffPrepareOperationSchema, type OpsOperation, + type OpsResultFor, } from "@openmapx/core/ops"; import { type ConfigurationHandoffIntentRecord, @@ -36,8 +40,10 @@ import { configurationHandoffRecordDomains as domains, configurationHandoffRecordName as recordName, } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; +import { installationState } from "../../../packages/core/src/build-artifacts/control-authority"; import { CONFIGURATION_HANDOFF_LABEL, + configurationHandoffEnvironmentNames, configurationHandoffRunArgs, inspectConfigurationHandoffPrerequisite, } from "./configuration-handoff"; @@ -207,9 +213,11 @@ export async function prepareConfigurationHandoffLaunch( assertion: attempt.assertion, container: existsSync("/run/secrets/openmapx-control-verifiers"), }); - let helperPublicKey: string; + let helperPublicKey: string, sourceDigest: string; + const environmentNames = configurationHandoffEnvironmentNames(facts); try { helperPublicKey = child.publicKey; + sourceDigest = captureConfigurationHandoffHostSources(authority, child, environmentNames); } finally { closeControlInstallation(child); } @@ -224,6 +232,8 @@ export async function prepareConfigurationHandoffLaunch( requestedDeadlineAt: binding.requestedDeadlineAt, operation, helperPublicKey, + sourceDigest, + environmentNames, preparerAdmissionDigest: local.receiptDigest, ownerCertificateDigest: owned.owner.rootCertificateDigest, leaseId: observed.leaseId, @@ -314,6 +324,63 @@ function readProgress( return value; } +/** Safe observation only. Terminal progress is not application or containment; + * those require the completed original helper/physical receipt chain. */ +export function inspectConfigurationHandoff( + infraDir: string, + handoffId: string, +): OpsResultFor<"configurationHandoff.inspect"> { + const installation = openControlInstallation({ + infraDir, + role: "ops-agent", + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const local = installationState(installation), + raw = readPrivateRecord(local, "observations", recordName(handoffId, "intent")), + observed = verifyControl( + installation.publicKey, + domains.intent, + raw, + configurationHandoffIntentRecordSchema, + ), + authority = restoreAdmittedControlExecution(installation, observed.preparer), + intent = readIntent(authority, handoffId), + progress = readProgress(authority, intent), + unresolved = ["applied", "rolled_back", "failed", "recovery_required"].includes( + progress.phase, + ), + phase: OpsResultFor<"configurationHandoff.inspect">["phase"] = + unresolved || progress.launch.state === "attempted" + ? "recovery_required" + : progress.phase === "draining" + ? "draining" + : progress.phase === "applying" + ? "applying" + : progress.phase === "verifying" + ? "verifying" + : "prepared"; + return { + handoffId, + phase, + expectedRevisionId: intent.operation.expectedCurrentRevisionId, + appliedRevisionId: null, + completedServiceIds: [], + failedServiceIds: progress.targets + .filter((target) => target.phase === "unknown") + .map((target) => target.serviceId), + recoveryCodes: + progress.launch.state === "attempted" ? ["launch_unknown"] : ["receipt_pending"], + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + throw new OpsExecutionFailure("not_found", {}); + throw error; + } finally { + closeControlInstallation(installation); + } +} + /** First dispatch alone consumes the original helper ticket. Once launch intent * is durable, every recovery only inspects the deterministic container. */ export async function launchPreparedConfigurationHandoff( diff --git a/apps/ops-agent/src/configuration-handoff.ts b/apps/ops-agent/src/configuration-handoff.ts index 0ed8f6a3d..6c880d393 100644 --- a/apps/ops-agent/src/configuration-handoff.ts +++ b/apps/ops-agent/src/configuration-handoff.ts @@ -23,6 +23,30 @@ export const CONFIGURATION_HANDOFF_VERSION = "1"; export const CONFIGURATION_HANDOFF_IMAGE_LABEL = "openmapx.configuration-handoff.version"; const protectedOrder = ["postgis", "redis", "app-web", "app-api", "traefik", "ops-agent"] as const; const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +/** Only original inherited environment values participate in the private source + * copy. Container identity and fixed helper transport necessarily differ. */ +export function configurationHandoffEnvironmentNames(facts: AgentContainerFacts): string[] { + const excluded = new Set([ + "HOSTNAME", + "DOCKER_CONFIG", + "OPS_AGENT_HANDOFF_ID", + "OPS_CONFIGURATION_HANDOFF_ID", + "OPENMAPX_CONTROL_EXECUTION", + "OPENMAPX_CONTROL_DELEGATION", + "OPENMAPX_PARENT_EXECUTION", + ]); + return z + .array( + z + .string() + .min(1) + .max(256) + .regex(/^[A-Za-z_][A-Za-z0-9_]*$/), + ) + .max(256) + .refine((names) => Buffer.byteLength(JSON.stringify(names)) <= 8192) + .parse([...new Set(facts.envNames.filter((name) => !excluded.has(name)))].sort()); +} export function configurationHandoffServiceOrder(serviceIds: readonly string[]): string[] { const values = z.array(id).min(1).max(64).parse(serviceIds); if (new Set(values).size !== values.length) throw Error("Duplicate frozen handoff target"); @@ -99,6 +123,7 @@ export async function inspectConfigurationHandoffPrerequisite(input: { input.signal, ); if (facts.imageId !== input.imageId) throw new OpsHostUpdateRequiredError(); + configurationHandoffEnvironmentNames(facts); const version = ( await input.runDocker( [ diff --git a/apps/ops-agent/src/platform-configuration-runtime.ts b/apps/ops-agent/src/platform-configuration-runtime.ts index 01e6f26b8..5ef53f068 100644 --- a/apps/ops-agent/src/platform-configuration-runtime.ts +++ b/apps/ops-agent/src/platform-configuration-runtime.ts @@ -1,9 +1,17 @@ /** Fixed original-owner protected handoff. Old images fail the explicit bundle * prerequisite before ownership; the installer does not advertise a helper. */ -import { launchPreparedConfigurationHandoff } from "./configuration-handoff-store"; +import { + inspectConfigurationHandoff, + launchPreparedConfigurationHandoff, +} from "./configuration-handoff-store"; import type { OpsRuntime } from "./runtime"; -export function installPlatformConfigurationRuntime(runtime: OpsRuntime): void { +export function installPlatformConfigurationRuntime( + runtime: OpsRuntime, + options: { infraDir: string }, +): void { runtime["configurationHandoff.prepare"] = (operation, context) => launchPreparedConfigurationHandoff(runtime, operation, context); + runtime["configurationHandoff.inspect"] = async (operation) => + inspectConfigurationHandoff(options.infraDir, operation.handoffId); } diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 6958a77ed..c32f018b7 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -250,7 +250,7 @@ export async function createProductionOpsRuntime( execFile: options.docker?.execFile, }); installCatalogApprovalRuntime(runtime, { rootDir: options.rootDir }); - installPlatformConfigurationRuntime(runtime); + installPlatformConfigurationRuntime(runtime, { infraDir: options.stack.infraDir }); createAdministrativeRuntime(runtime, { rootDir: options.rootDir, runFixedCli, diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.ts b/packages/core/src/build-artifacts/configuration-handoff-records.ts index b8f3e6664..448d9b714 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.ts @@ -29,6 +29,18 @@ export const configurationHandoffIntentRecordSchema = z .min(32) .max(128) .regex(/^[A-Za-z0-9_-]+$/), + sourceDigest: digest, + environmentNames: z + .array( + z + .string() + .min(1) + .max(256) + .regex(/^[A-Za-z_][A-Za-z0-9_]*$/), + ) + .max(256) + .refine((names) => Buffer.byteLength(JSON.stringify(names)) <= 8192) + .refine((names) => names.every((name, index) => index === 0 || names[index - 1]! < name)), preparerAdmissionDigest: digest, ownerCertificateDigest: digest, leaseId: z.uuid(), diff --git a/packages/core/src/build-artifacts/configuration-source-witness.ts b/packages/core/src/build-artifacts/configuration-source-witness.ts index fa923a443..c31c52e74 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.ts @@ -1,9 +1,10 @@ -import { createHmac, randomUUID } from "node:crypto"; +import { createHmac, randomUUID, timingSafeEqual } from "node:crypto"; import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync } from "node:fs"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { parseEnv } from "node:util"; import { z } from "zod"; import { canonicalJson } from "../ops/canonical-json"; +import { configurationHandoffPrepareOperationSchema } from "../ops/configuration-handoff-records"; import { type ConfigurationSourcesInspection, configurationSourcesInspectionSchema, @@ -15,13 +16,21 @@ import { } from "../services/configuration-generation"; import { SERVICE_SELECTION_FILE } from "../services/deployment"; import type { LoadedService } from "../services/types"; +import { configurationMaintenanceOwnerRecordSchema } from "./configuration-maintenance-owner-records"; import { + assertControlExecutionForward, type ControlInstallation, + controlDigest, + executionAuthorityState, + fixedControlChildId, installationState, + readConfigurationHandoffSpawnIntent, readPrivateRecord, + type VerifiedExecutionAuthority, writePrivateRecord, } from "./control-authority"; import { signControl, verifyControl } from "./control-records"; +import { verifyFamilyOwnedRecord } from "./retention-family"; const domain = "openmapx-configuration-source-witness-v1"; const witnessSchema = z.strictObject({ @@ -33,6 +42,133 @@ const witnessSchema = z.strictObject({ pins: z.array(z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/)).max(128), }); const name = (id: string) => `configuration-source-${z.uuid().parse(id)}.json`; + +function handoffSourceBytes(infraDir: string, environmentNames: readonly string[]) { + // File values stay private/keyed, including credentials. Process-only helper + // transport variables are absent from the original inherited-name projection. + return canonicalJson({ + files: currentSources(infraDir, {}), + runtimeEnvironment: Object.fromEntries( + environmentNames.map((name) => [name, process.env[name] ?? null]), + ), + }); +} +const helperSourceDomain = "openmapx-configuration-helper-host-sources-v1\0"; +/** The installed parent copies continuity of the already committed generation, + * not fresh mutation authority. Its original role HMAC must still match after + * substituting only this owner's exact intended platform-generation change. */ +export function captureConfigurationHandoffHostSources( + authority: VerifiedExecutionAuthority, + helper: ControlInstallation, + environmentNames: readonly string[], +): string { + const state = executionAuthorityState(authority), + local = state.installation, + child = installationState(helper); + if ( + state.kind !== "configurationHandoff.prepare" || + local.fixedChildId || + !state.origin || + !child.fixedChildId || + helper.installationId !== local.installation.installationId || + fixedControlChildId(local, helper.publicKey, state.execution) !== child.fixedChildId + ) + throw Error("Original handoff source producer required"); + assertControlExecutionForward(authority); + const admission = readPrivateRecord( + local, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: unknown } }; + if (controlDigest(admission) !== state.receiptDigest) + throw Error("Original handoff admission changed"); + const operation = configurationHandoffPrepareOperationSchema.parse(admission.payload?.operation), + ownerName = `${operation.maintenanceId}.json`, + raw = readPrivateRecord(local, "configuration-maintenance", ownerName), + owner = verifyControl( + local.installation.publicKey, + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ), + infraDir = dirname(local.dataRoot), + witness = verifyControl( + local.installation.publicKey, + domain, + readPrivateRecord(local, "observations", name(owner.plan.operation.expectedSourceRevision)), + witnessSchema, + ); + verifyFamilyOwnedRecord( + local, + state.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name: ownerName }, + canonicalJson(raw), + ); + if ( + canonicalJson(owner.origin) !== canonicalJson(state.origin) || + owner.receipt.generation !== "committed" || + owner.receipt.appliedRevisionId !== operation.expectedCurrentRevisionId || + owner.receipt.decision?.decision !== "continue" || + !owner.apiControls.decide?.receiptDelivery || + !owner.plan.steps.some( + (step) => + step.phase === "maintenance-handoff" && + canonicalJson(step.operation) === canonicalJson(operation), + ) || + currentConfigurationGeneration(infraDir) !== operation.expectedCurrentRevisionId || + witness.authorityRevision !== owner.plan.sourceAuthorityRevision || + witness.inspection.sourceRevision !== owner.plan.operation.expectedSourceRevision || + witness.inspection.revisionId !== owner.plan.operation.expectedCurrentRevisionId + ) + throw Error("Original owned handoff generation required"); + const { platform: _current, ...observed } = currentSources(infraDir, process.env), + original = { + ...observed, + ...(witness.inspection.platform ? { platform: witness.inspection.platform } : {}), + }, + originalDigest = createHmac("sha256", local.privateKey.export({ type: "pkcs8", format: "der" })) + .update(`${domain}\0`) + .update(canonicalJson(original)) + .digest("hex"); + if (!timingSafeEqual(Buffer.from(originalDigest, "hex"), Buffer.from(witness.observation, "hex"))) + throw Error("Original host sources changed before handoff"); + return createHmac("sha256", child.privateKey.export({ type: "pkcs8", format: "der" })) + .update(helperSourceDomain) + .update(handoffSourceBytes(infraDir, environmentNames)) + .digest("hex"); +} +/** Only the actual fixed child can check its own private keyed source copy. + * No supplied hash, environment values, path or boolean is accepted. */ +export function assertConfigurationHandoffHostSources(authority: VerifiedExecutionAuthority): void { + const state = executionAuthorityState(authority), + local = state.installation; + if (state.kind !== "configuration-helper" || !local.fixedChildId) + throw Error("Original configuration helper required"); + const raw = readPrivateRecord( + local, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ) as { payload?: { operation?: { handoffId?: string } } }; + if (controlDigest(raw) !== state.receiptDigest || !raw.payload?.operation?.handoffId) + throw Error("Original helper admission changed"); + const intent = readConfigurationHandoffSpawnIntent( + local.installation, + state.execution, + raw.payload.operation.handoffId, + ), + observed = createHmac("sha256", local.privateKey.export({ type: "pkcs8", format: "der" })) + .update(helperSourceDomain) + .update(handoffSourceBytes(dirname(local.dataRoot), intent.environmentNames)) + .digest("hex"); + if ( + currentConfigurationGeneration(dirname(local.dataRoot)) !== + intent.operation.expectedCurrentRevisionId || + !timingSafeEqual(Buffer.from(observed, "hex"), Buffer.from(intent.sourceDigest, "hex")) + ) + throw Error("Helper host sources changed"); +} function hostEnv(infraDir: string, env: NodeJS.ProcessEnv) { const path = join(infraDir, ".env"); let file: NodeJS.ProcessEnv = {}; diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index b746fddb9..62110ceb5 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -34,6 +34,8 @@ export { verifyPublishedUnusedConfigurationCapacity, } from "./configuration-no-guard"; export { + assertConfigurationHandoffHostSources, + captureConfigurationHandoffHostSources, configurationHostFields, observeConfigurationSources, verifyConfigurationReleaseSourceWitness, From d5c05108937c511245b9ad5c4b99ae9995f58bbd Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 05:30:52 +0100 Subject: [PATCH 068/135] fix(admin): verify actual Docker memory and CPU limits --- ...configuration-maintenance-postgres.test.ts | 3 ++ .../src/__fixtures__/owned-service-host.ts | 7 +++- ...iguration-maintenance-docker-inspection.ts | 3 ++ .../configuration-resource-inspection.test.ts | 38 ++++++++++++++++++ .../src/configuration-resource-inspection.ts | 40 +++++++++++++++++++ .../configuration-maintenance-budget.ts | 1 + ...onfiguration-maintenance-docker-records.ts | 34 ++++++++++++++++ 7 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 apps/ops-agent/src/configuration-resource-inspection.test.ts create mode 100644 apps/ops-agent/src/configuration-resource-inspection.ts diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts index 2a03bfbff..1be570d10 100644 --- a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -2105,6 +2105,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); }, 60000); it("uses one ownership-only maintenance decision before a durable nonempty coordinator credential continuation", async () => { + // Synthetic example/* images intentionally have no published release overlay. + vi.stubEnv("OPENMAPX_RELEASE", "local"); + vi.stubEnv("DOMAIN", "example.test"); dm = await ownedDataManager({ sql: fixture.sql, infraDir: host.infraDir, diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index a4096aa26..f79493cd2 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -369,7 +369,12 @@ export async function ownedServiceHost(options: { configurationHelperLaunchRespo if(!service)throw Error("Named Docker fixture service unavailable"); service.State=state; }, - setRuntimeInspections(value: typeof runtimeInspections) { runtimeInspections = value; }, + setRuntimeInspections(value: typeof runtimeInspections) { + // Real Docker inspect includes these fields. Historical fixture services + // declare no limits; explicitly supplied resource observations win. + for(const row of Object.values(value)) if(row && typeof row==="object" && !Object.hasOwn(row,"HostConfig")) Object.assign(row,{HostConfig:{Memory:0,NanoCpus:0}}); + runtimeInspections = value; + }, setRuntimeSecretBytes(value: typeof runtimeSecretBytes) { runtimeSecretBytes = value; }, setRawListing(value: string) { rawListing = value; diff --git a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts index b0e93766c..2263d31e7 100644 --- a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts +++ b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts @@ -10,6 +10,7 @@ import { maintenanceDockerContainerSchema as container, } from "../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records"; import type { ConfigurationMaintenanceOwnerRecord } from "./configuration-maintenance-store"; +import { verifyConfigurationRuntimeResources } from "./configuration-resource-inspection"; import type { DockerRuntimeOptions } from "./docker-runtime"; import type { OwnedServiceSnapshot } from "./owned-services"; @@ -109,6 +110,7 @@ export async function inspectMaintenanceDockerRuntime(options: { ); if (Object.keys(expectedEnv).some((key) => actualEnv[key] !== expectedEnv[key])) throw Error("Original applied runtime environment differs"); + const resources = verifyConfigurationRuntimeResources(spec, observed); checks.push({ serviceId: id, ordinal: null, @@ -120,6 +122,7 @@ export async function inspectMaintenanceDockerRuntime(options: { containerId: owned.containerId, expectedConfigHash: hash, actualConfigHash: observed.Config.Labels["com.docker.compose.config-hash"], + resources, expectedEnvironmentDigest: keyed(canonicalJson(expectedEnv)), actualEnvironmentDigest: keyed( canonicalJson( diff --git a/apps/ops-agent/src/configuration-resource-inspection.test.ts b/apps/ops-agent/src/configuration-resource-inspection.test.ts new file mode 100644 index 000000000..1abbd4d68 --- /dev/null +++ b/apps/ops-agent/src/configuration-resource-inspection.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { verifyConfigurationRuntimeResources } from "./configuration-resource-inspection"; + +describe("actual Docker runtime limits", () => { + const expected = { deploy: { resources: { limits: { memory: "268435456", cpus: 0.5 } } } }; + it("compares normalized Compose limits with actual cgroup settings for unchanged images", () => { + expect( + verifyConfigurationRuntimeResources(expected, { + HostConfig: { Memory: 268435456, NanoCpus: 500000000 }, + }), + ).toEqual({ memoryBytes: 268435456, nanoCpus: 500000000 }); + expect( + verifyConfigurationRuntimeResources(expected, { + HostConfig: { Memory: 268435456, NanoCpus: 0, CpuQuota: 50000, CpuPeriod: 100000 }, + }), + ).toEqual({ memoryBytes: 268435456, nanoCpus: 500000000 }); + }); + it.each([ + { HostConfig: { Memory: 536870912, NanoCpus: 500000000 } }, + { HostConfig: { Memory: 268435456, NanoCpus: 1000000000 } }, + { HostConfig: { Memory: 268435456, NanoCpus: 0 } }, + { HostConfig: { Memory: 268435456, NanoCpus: 500000000, CpuQuota: 100000, CpuPeriod: 100000 } }, + {}, + ])("rejects missing or mismatched runtime memory/CPU %#", (actual) => { + expect(() => verifyConfigurationRuntimeResources(expected, actual)).toThrow(); + }); + it("verifies explicit reset to no limit rather than keeping previous resource values", () => { + expect( + verifyConfigurationRuntimeResources({}, { HostConfig: { Memory: 0, NanoCpus: 0 } }), + ).toEqual({ memoryBytes: 0, nanoCpus: 0 }); + expect(() => + verifyConfigurationRuntimeResources( + {}, + { HostConfig: { Memory: 268435456, NanoCpus: 500000000 } }, + ), + ).toThrow(); + }); +}); diff --git a/apps/ops-agent/src/configuration-resource-inspection.ts b/apps/ops-agent/src/configuration-resource-inspection.ts new file mode 100644 index 000000000..51946144b --- /dev/null +++ b/apps/ops-agent/src/configuration-resource-inspection.ts @@ -0,0 +1,40 @@ +/** Data-only comparison of the actual normalized Compose/Docker observations. + * No parsed resource object supplies a target, source or settlement authority. */ +import { z } from "zod"; +import { + maintenanceDockerHostResourcesSchema, + maintenanceDockerResourceSpecSchema, +} from "../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records"; + +export function verifyConfigurationRuntimeResources( + spec: unknown, + container: unknown, +): { memoryBytes: number; nanoCpus: number } { + const expected = maintenanceDockerResourceSpecSchema.parse(spec), + actual = z + .object({ HostConfig: maintenanceDockerHostResourcesSchema }) + .parse(container).HostConfig, + limits = expected.deploy?.resources?.limits, + memoryBytes = Number(limits?.memory ?? expected.mem_limit ?? 0), + nanoCpus = Math.round(Number(limits?.cpus ?? expected.cpus ?? 0) * 1e9), + quotaNanoCpus = + actual.CpuQuota > 0 && actual.CpuPeriod > 0 + ? Math.round((actual.CpuQuota / actual.CpuPeriod) * 1e9) + : 0; + if ( + !Number.isSafeInteger(memoryBytes) || + !Number.isSafeInteger(nanoCpus) || + (limits?.memory !== undefined && + expected.mem_limit !== undefined && + Number(limits.memory) !== Number(expected.mem_limit)) || + (limits?.cpus !== undefined && + expected.cpus !== undefined && + Number(limits.cpus) !== Number(expected.cpus)) || + (actual.CpuQuota > 0 && actual.CpuPeriod === 0) || + (actual.NanoCpus > 0 && quotaNanoCpus > 0 && actual.NanoCpus !== quotaNanoCpus) || + actual.Memory !== memoryBytes || + (actual.NanoCpus || quotaNanoCpus) !== nanoCpus + ) + throw Error("Original applied runtime resources differ"); + return { memoryBytes, nanoCpus }; +} diff --git a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts index 97cb212e7..2b7cf8380 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts @@ -172,6 +172,7 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { containerId: hash, expectedConfigHash: hash, actualConfigHash: hash, + resources: { memoryBytes: Number.MAX_SAFE_INTEGER, nanoCpus: Number.MAX_SAFE_INTEGER }, expectedEnvironmentDigest: hash, actualEnvironmentDigest: hash, }, diff --git a/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts index 5b181073d..4924f3851 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts @@ -9,9 +9,42 @@ const string = z.string().max(65536), Destination: string, RW: z.boolean(), }); +const normalizedMemory = z.union([ + z.number().int().nonnegative().safe(), + z.string().max(16).regex(/^\d+$/), + ]), + normalizedCpu = z.union([ + z.number().finite().nonnegative().max(4096), + z + .string() + .max(32) + .regex(/^\d+(?:\.\d+)?$/), + ]); +export const maintenanceDockerResourceSpecSchema = z.object({ + mem_limit: normalizedMemory.optional(), + cpus: normalizedCpu.optional(), + deploy: z + .object({ + resources: z + .object({ + limits: z + .object({ memory: normalizedMemory.optional(), cpus: normalizedCpu.optional() }) + .optional(), + }) + .optional(), + }) + .optional(), +}); +export const maintenanceDockerHostResourcesSchema = z.object({ + Memory: z.number().int().nonnegative().safe(), + NanoCpus: z.number().int().nonnegative().safe(), + CpuQuota: z.number().int().min(-1).safe().default(0), + CpuPeriod: z.number().int().nonnegative().safe().default(0), +}); export const maintenanceDockerContainerSchema = z.object({ Id: z.string().regex(/^[a-f0-9]{64}$/), Image: z.string().regex(/^sha256:[a-f0-9]{64}$/), + HostConfig: maintenanceDockerHostResourcesSchema, State: z.object({ Status: z.string(), Running: z.boolean(), @@ -25,6 +58,7 @@ export const maintenanceDockerContainerSchema = z.object({ Mounts: z.array(mount).max(512), }); const service = z.object({ + ...maintenanceDockerResourceSpecSchema.shape, image: string, environment: z .record(z.string(), z.union([string, z.number(), z.boolean(), z.null()])) From 32ff2b9276d335095e3dd0f17fd186cae4212a8d Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 06:15:01 +0100 Subject: [PATCH 069/135] feat(platform): settle verified helper application through original maintenance root --- ...orm-configuration-runtime-postgres.test.ts | 191 +++++++++- .../src/__fixtures__/owned-service-host.ts | 8 +- .../src/configuration-handoff-application.ts | 327 ++++++++++++++++++ .../src/configuration-handoff-store.ts | 90 ++++- .../src/configuration-handoff.test.ts | 3 + apps/ops-agent/src/configuration-handoff.ts | 7 +- ...iguration-maintenance-docker-inspection.ts | 16 +- .../src/configuration-maintenance-store.ts | 101 +++++- apps/ops-agent/src/server.ts | 6 +- .../configuration-handoff-records.ts | 30 ++ ...configuration-maintenance-final-records.ts | 7 +- .../src/build-artifacts/control-authority.ts | 173 +++++++++ .../core/src/build-artifacts/target-lease.ts | 118 +++++++ 13 files changed, 1048 insertions(+), 29 deletions(-) create mode 100644 apps/ops-agent/src/configuration-handoff-application.ts diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index a3c1944db..4ad15739c 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -2,12 +2,13 @@ * Docker I/O is substituted. This qualifies preparation, launch reconciliation * and real child adoption; not native helper application/public platform HTTP. */ import { randomUUID } from "node:crypto"; -import { existsSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { closeControlInstallation, openControlInstallation, restoreAdmittedControlExecution, + verifyRetainedArtifactTargetSettlement, } from "@openmapx/core/build-artifacts"; import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; @@ -57,7 +58,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( afterEach(async () => { await state.fixture?.stop(); }); - it.each(["old", "1", "launch", "launch-lost", "policy-change"])( + it.each(["old", "1", "launch", "launch-lost", "policy-change", "launch-apply"])( "guards and retains helper mode %s through original ownership", async (version) => { const ids = [ @@ -78,17 +79,90 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, configurationHelperVersion: version === "old" ? "old" : "1", configurationHelperLaunchResponseLoss: version === "launch-lost", + configurationHelperReleaseInventory: version === "launch-apply", + ...(version === "launch-apply" + ? { + schemas: { + "ops-agent": JSON.parse( + readFileSync( + new URL("../../../../../services/ops-agent/service.json", import.meta.url), + "utf8", + ), + ).configSchema, + }, + } + : {}), actualComposeConfig: true, }); state.services = host.loaded; - host.setRows( - ids.map((id, index) => ({ - ...host.row, - Id: String(index + 1).repeat(64), - ConfigImage: `example/${id}:1`, - Labels: { ...host.row.Labels, "com.docker.compose.service": id }, - })), - ); + let rows = ids.map((id, index) => ({ + ...host.row, + Id: String(index + 1).repeat(64), + ConfigImage: `example/${id}:1`, + Labels: { ...host.row.Labels, "com.docker.compose.service": id }, + })); + host.setRows(rows); + if (version === "launch-apply") { + const inspections: Parameters[0] = {}; + host.setCreatingProcess(async (args) => { + const id = args.at(-1)!, + old = rows.find((row) => row.Labels["com.docker.compose.service"] === id); + if (!old || !["app-api", "ops-agent"].includes(id)) + throw Error("Unexpected protected target"); + const prefix = args.slice(0, args.indexOf("up")), + options = { + signal: new AbortController().signal, + timeout: 15000, + maxBuffer: 1024 * 1024, + }, + config = JSON.parse( + (await host.execFile("docker", [...prefix, "config", "--format", "json"], options)) + .stdout, + ), + hash = ( + await host.execFile("docker", [...prefix, "config", "--hash", id], options) + ).stdout + .trim() + .split(/\s+/) + .at(-1)!, + compose = services.configurationGenerationComposeFile( + host.infraDir, + services.currentConfigurationGeneration(host.infraDir)!, + ), + row = { + ...old, + Id: (id === "app-api" ? "a" : "b").repeat(64), + Labels: { + ...old.Labels, + "com.docker.compose.project.working_dir": join(compose, ".."), + "com.docker.compose.project.config_files": compose, + "com.docker.compose.config-hash": hash, + }, + }, + spec = config.services[id], + limits = spec.deploy?.resources?.limits; + rows = rows.map((value) => (value === old ? row : value)); + host.setRows(rows); + inspections[row.Id] = { + Id: row.Id, + Image: row.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { + Image: row.ConfigImage, + Labels: row.Labels, + Env: Object.entries(spec.environment ?? {}).map( + ([key, value]) => `${key}=${value}`, + ), + }, + HostConfig: { + Memory: Number(limits?.memory ?? 0), + NanoCpus: Math.round(Number(limits?.cpus ?? 0) * 1e9), + }, + Mounts: [], + }; + host.setRuntimeInspections(inspections); + }); + } const installation = openControlInstallation({ infraDir: host.infraDir, role: "ops-agent", @@ -133,6 +207,12 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( email: `${actorId}@example.test`, role: "admin", }); + if (version === "launch-apply") + await state.fixture.db.insert(schema.serviceConfig).values({ + id: randomUUID(), + serviceId: "ops-agent", + config: { OPENMAPX_MEMORY_MIB: 256, OPENMAPX_CPU_LIMIT: 0.5 }, + }); await state.fixture.db.insert(schema.adminJob).values({ id: jobId, type: "platform.apply", @@ -419,6 +499,97 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( child, AbortSignal.timeout(2000), ); + if (version === "launch-apply") { + const { runConfigurationHandoffApplication } = await import( + "../../../../ops-agent/src/configuration-handoff-application" + ); + expect( + await runConfigurationHandoffApplication({ + child, + rootDir: host.root, + execFile: host.execFile, + }), + ).toMatchObject({ + phase: "applied", + completedServiceIds: ["app-api", "ops-agent"], + }); + const actual = childControl.configurationHandoffChildState(child); + verifyRetainedArtifactTargetSettlement( + actual.installation, + { + leaseId: actual.intent.leaseId, + delegationId: actual.intent.delegationId, + phase: "ready", + }, + actual.intent.helper, + ); + const { inspectConfigurationHandoff } = await import( + "../../../../ops-agent/src/configuration-handoff-store" + ); + const completionFile = join( + host.infraDir, + "control-private/ops-agent/observations", + `configuration-handoff-${actual.intent.operation.handoffId}.completion.json`, + ), + retainedFile = `${completionFile}.retained`; + renameSync(completionFile, retainedFile); + try { + writeFileSync(completionFile, readFileSync(retainedFile), { + mode: 0o600, + }); + expect(() => + inspectConfigurationHandoff( + host.infraDir, + actual.intent.operation.handoffId, + ), + ).toThrow(); + } finally { + rmSync(completionFile); + renameSync(retainedFile, completionFile); + } + expect( + await inspectConfigurationHandoff( + host.infraDir, + actual.intent.operation.handoffId, + ), + ).toMatchObject({ + phase: "applied", + appliedRevisionId: candidate.revisionId, + completedServiceIds: ["app-api", "ops-agent"], + recoveryCodes: [], + }); + const applied = await prospective.inspectConfigurationMaintenance({ + maintenanceId: receipt.binding.maintenanceId, + candidateId: candidate.candidateId, + }); + expect(applied.followUp).toBe("complete"); + const settled = await prospective.settleConfigurationMaintenance({ + kind: "configurationMaintenance.settle", + maintenanceId: applied.binding.maintenanceId, + candidateId: candidate.candidateId, + prepareOperationKey: applied.prepare.operationKey, + prepareFingerprint: applied.prepare.operationFingerprint, + planDigest: applied.planDigest, + decisionId: applied.decision!.decisionId, + expectedSequence: applied.sequence, + evidenceDigest: configurationMaintenanceSettleRequestDigest(applied), + }); + expect(settled.state).toBe("settled"); + expect(dm.settlePosts).toBe(1); + await dm.assertDataAdmission(); + const { configurationHandoffPendingMaintenanceIds } = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ); + expect(configurationHandoffPendingMaintenanceIds(host.runtime).size).toBe( + 0, + ); + expect( + host.calls + .filter((args) => args[0] === "compose" && args.includes("up")) + .map((args) => args.at(-1)), + ).toEqual(["app-api", "ops-agent"]); + return; + } childControl.assertConfigurationHandoffHostSources(child); writeFileSync(join(host.infraDir, ".env"), "OPENMAPX_SYNTHETIC_DRIFT=1\n"); expect(() => diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index f79493cd2..e7c89311c 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { configurationHelperLaunchResponseLoss?:boolean; selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { configurationHelperReleaseInventory?:boolean; configurationHelperLaunchResponseLoss?:boolean; selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -44,7 +44,7 @@ export async function ownedServiceHost(options: { configurationHelperLaunchRespo mkdirSync(join(root, "services")); mkdirSync(join(root, "custom_integrations")); writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); - const loaded: services.LoadedService[] = [ + const loaded: services.LoadedService[] = (options.configurationHelperReleaseInventory ? [...services.RELEASE_BUILT_IN_SERVICE_IDS] : [ "motis", "ops-agent", "app-api", @@ -53,7 +53,7 @@ export async function ownedServiceHost(options: { configurationHelperLaunchRespo "redis", ...(withCoordinator ? ["data-manager"] : []), ...(options.extraServiceIds ?? []), - ].map((id) => ({ + ]).map((id) => ({ manifest: { id, name: id, @@ -80,9 +80,11 @@ export async function ownedServiceHost(options: { configurationHelperLaunchRespo mkdirSync(service.directory, { recursive: true }); writeFileSync(join(service.directory, "service.json"), JSON.stringify(service.manifest)); } + const releaseInventory = options.configurationHelperReleaseInventory ? await services.captureReleaseServiceAuthority(root) : undefined; const loadAuthority = createTrustedConfigurationAuthorityLoader({ rootDir: root, builtInServices: loaded, + ...(releaseInventory ? {builtInAuthorityDigest:releaseInventory.digest} : {}), }); const initial = await services.commitConfigurationGeneration({ infraDir, diff --git a/apps/ops-agent/src/configuration-handoff-application.ts b/apps/ops-agent/src/configuration-handoff-application.ts new file mode 100644 index 000000000..706f74aa0 --- /dev/null +++ b/apps/ops-agent/src/configuration-handoff-application.ts @@ -0,0 +1,327 @@ +/** Fixed same-image helper effects. Only Docker process I/O is replaceable in + * tests; ownership, source checks, ordering and verification are installed. */ +import { join } from "node:path"; +import { + assertArtifactTargetLeaseForward, + beginControlEffect, + controlDigest, + controlRolePublicKey, + readPrivateRecord, + retainControlEffectReceipt, + settleArtifactTargetLeaseChild, + signControl, + verifyControl, + verifyControlEffectSettlement, + verifyFamilyOwnedRecord, + writePrivateRecord, +} from "@openmapx/core/build-artifacts"; +import { canonicalJson } from "@openmapx/core/ops"; +import { repoPaths, services } from "@openmapx/core/server"; +import { + type ConfigurationHandoffWorkerRecord, + configurationHandoffProgressSchema, + configurationHandoffWorkerRecordSchema, + configurationHandoffRecordDomains as domains, + configurationHandoffRecordName as name, +} from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; +import { configurationMaintenanceOwnerRecordSchema } from "../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records"; +import { installationState } from "../../../packages/core/src/build-artifacts/control-authority"; +import { configurationHandoffServiceOrder } from "./configuration-handoff"; +import { + assertConfigurationHandoffHostSources, + type ConfigurationHandoffChild, + configurationHandoffChildState, + waitForConfigurationHandoffPreparation, +} from "./configuration-handoff-control"; +import { inspectMaintenanceDockerRuntime } from "./configuration-maintenance-docker-inspection"; +import { type DockerRuntimeOptions, runContainedProcess } from "./docker-runtime"; +import { createOwnedServiceResolver } from "./owned-services"; +import { createTrustedConfigurationAuthorityLoader } from "./trusted-configuration-authority"; + +const running = new WeakSet(); +function ownerFor(child: ConfigurationHandoffChild) { + const state = configurationHandoffChildState(child), + local = installationState(state.installation), + intent = state.intent, + file = `${intent.operation.maintenanceId}.json`, + raw = readPrivateRecord(local, "configuration-maintenance", file), + owner = verifyControl( + controlRolePublicKey(state.installation, "ops-agent"), + "openmapx-configuration-maintenance-owner-v2", + raw, + configurationMaintenanceOwnerRecordSchema, + ), + step = owner.plan.steps.find((value) => value.phase === "maintenance-handoff"), + source = step && owner.apiControls.steps.find((value) => value.ordinal === step.ordinal); + verifyFamilyOwnedRecord( + local, + intent.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name: file }, + canonicalJson(raw), + ); + if ( + canonicalJson(owner.origin) !== canonicalJson(intent.origin) || + canonicalJson(owner.plan.prepare) !== canonicalJson(intent.ownerPrepare) || + owner.rootCertificateDigest !== intent.ownerCertificateDigest || + !step || + canonicalJson(step.operation) !== canonicalJson(intent.operation) || + !source?.grant || + canonicalJson(source.execution) !== canonicalJson(intent.preparer) || + source.grant.leaseId !== intent.leaseId || + owner.issuance !== "open" || + owner.noGuard || + owner.final || + owner.inspection || + !owner.guardSource || + owner.receipt.dmFence?.state !== "prepared" || + owner.receipt.state !== "applying" || + owner.receipt.decision?.decision !== "continue" || + !owner.apiControls.decide?.receiptDelivery || + owner.receipt.generation !== "committed" || + owner.receipt.appliedRevisionId !== intent.operation.expectedCurrentRevisionId || + intent.operation.mode.kind !== "configuration" + ) + throw Error("Original active configuration handoff owner required"); + return owner; +} + +/** This result describes observed helper runtime only. The parent/root remain + * held until the independent effect/physical/final/SQL/ACK chain settles. */ +export async function runConfigurationHandoffApplication(input: { + child: ConfigurationHandoffChild; + rootDir: string; + execFile?: DockerRuntimeOptions["execFile"]; + signal?: AbortSignal; +}) { + const { child } = input; + const state = configurationHandoffChildState(child); + if (running.has(child)) throw Error("Original helper already running"); + running.add(child); + const local = installationState(state.installation), + intent = state.intent, + signal = AbortSignal.any([ + input.signal ?? new AbortController().signal, + AbortSignal.timeout(Math.max(1, intent.helper.deadlineAt - Date.now())), + ]), + execFile = input.execFile ?? runContainedProcess, + stack = repoPaths(input.rootDir), + file = name(intent.operation.handoffId, "completion"); + let worker: ConfigurationHandoffWorkerRecord | undefined; + const save = (next: ConfigurationHandoffWorkerRecord, exclusive = false) => { + const value = configurationHandoffWorkerRecordSchema.parse(next); + writePrivateRecord( + local, + "observations", + file, + signControl(local.privateKey, domains.completion, value), + exclusive, + intent.origin, + ); + worker = value; + }; + try { + await waitForConfigurationHandoffPreparation(child, signal); + assertConfigurationHandoffHostSources(child); + if (stack.infraDir !== join(input.rootDir, "infra/docker")) + throw Error("Original helper checkout required"); + const capture = await services.captureReleaseServiceAuthority(input.rootDir), + loadAuthority = createTrustedConfigurationAuthorityLoader({ + rootDir: input.rootDir, + builtInServices: capture.services, + builtInAuthorityDigest: capture.digest, + }), + owned = createOwnedServiceResolver({ stack, execFile, loadAuthority }), + parentName = name(intent.operation.handoffId, "progress"), + parentRaw = readPrivateRecord(local, "observations", parentName), + parent = verifyControl( + controlRolePublicKey(state.installation, "ops-agent"), + domains.progress, + parentRaw, + configurationHandoffProgressSchema, + ); + verifyFamilyOwnedRecord( + local, + intent.origin, + { scope: "private", namespace: "observations", container: "", name: parentName }, + canonicalJson(parentRaw), + ); + if ( + parent.intentDigest !== controlDigest(intent) || + parent.phase !== "launched" || + parent.launch.state !== "confirmed" + ) + throw Error("Original confirmed launch required"); + // Existing worker publication is observation-only, never another dispatch. + try { + readPrivateRecord(local, "observations", file); + throw Error("Original attempted helper requires observation"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + beginControlEffect(state.authority); + save( + { + ...parent, + execution: intent.helper, + preparationDigest: controlDigest(parent), + observations: [], + sequence: 0, + phase: "draining", + updatedAt: Date.now(), + }, + true, + ); + for (const id of configurationHandoffServiceOrder(intent.operation.serviceIds)) { + signal.throwIfAborted(); + const owner = ownerFor(child), + authority = await loadAuthority(); + if (authority.revisionId !== owner.plan.sourceAuthorityRevision) + throw Error("Original helper authority changed"); + const target = (await owned.inspectOwnedServices(signal)).find( + (value) => value.serviceId === id, + ), + original = owner.originalContainers.find((value) => value.serviceId === id); + if ( + !target || + !original || + target.containerId !== original.containerId || + target.state !== "running" || + target.provenance.imageId !== intent.operation.expectedImageIdentities[id] + ) + throw Error("Original helper target changed"); + const args = await services.withConfigurationGenerationWriter( + stack.infraDir, + async () => { + const release = await services.acquireReleaseStoreLock( + join(stack.infraDir, ".ops-agent-releases"), + ); + try { + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + ownerFor(child); + if ( + (await loadAuthority()).revisionId !== owner.plan.sourceAuthorityRevision || + services.currentConfigurationGeneration(stack.infraDir) !== + intent.operation.expectedCurrentRevisionId + ) + throw Error("Original helper generation changed"); + const fresh = (await owned.inspectOwnedServices(signal)).find( + (value) => value.serviceId === id, + ); + if (!fresh || canonicalJson(fresh) !== canonicalJson(target)) + throw Error("Original helper target raced"); + const current = worker!; + save({ + ...current, + sequence: current.sequence + 1, + phase: "applying", + targets: current.targets.map((value) => + value.serviceId === id ? { ...value, phase: "recreate_attempted" } : value, + ), + updatedAt: Date.now(), + }); + return [ + ...services.stackComposeArgs(stack), + "up", + "-d", + "--force-recreate", + "--no-deps", + id, + ]; + } finally { + release.release(); + } + }, + signal, + ); + // No configuration/release writer survives container replacement. + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + await execFile("docker", args, { + signal, + timeout: Math.max(1, intent.helper.deadlineAt - Date.now()), + maxBuffer: 1024 * 1024, + }); + const containers = await owned.inspectOwnedServices(signal), + inspection = await inspectMaintenanceDockerRuntime({ + stack, + execFile, + owner: ownerFor(child), + containers, + privateKey: local.privateKey, + signal, + handoffTargetId: id, + }), + observed = containers.find((value) => value.serviceId === id); + if (!observed) throw Error("Applied helper container unavailable"); + const current = worker!; + save({ + ...current, + sequence: current.sequence + 1, + phase: "verifying", + targets: current.targets.map((value) => + value.serviceId === id + ? { ...value, phase: "verified", newContainerId: observed.containerId } + : value, + ), + observations: [ + ...current.observations, + { + serviceId: id, + checksDigest: controlDigest(inspection.checks), + detailsDigest: controlDigest(inspection.details), + }, + ], + updatedAt: Date.now(), + }); + } + const owner = ownerFor(child); + assertConfigurationHandoffHostSources(child); + await inspectMaintenanceDockerRuntime({ + stack, + execFile, + owner, + containers: await owned.inspectOwnedServices(signal), + privateKey: local.privateKey, + signal, + }); + const completedAt = Date.now(), + current = worker!; + save({ + ...current, + sequence: current.sequence + 1, + phase: "applied", + updatedAt: completedAt, + completedAt, + }); + retainControlEffectReceipt(state.authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "configuration-helper-completion", completionDigest: controlDigest(worker) }, + }); + await settleArtifactTargetLeaseChild( + state.lease, + verifyControlEffectSettlement(state.installation, intent.helper), + ); + return { + phase: "applied" as const, + completedServiceIds: configurationHandoffServiceOrder(intent.operation.serviceIds), + }; + } catch (error) { + if (worker && worker.completedAt === null) { + try { + save({ + ...worker, + sequence: worker.sequence + 1, + phase: "recovery_required", + updatedAt: Date.now(), + }); + } catch { + /* Unknown publication retains the original root. */ + } + } + throw error; + } finally { + running.delete(child); + } +} diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index f60619ea0..f44a0449d 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -23,7 +23,9 @@ import { signControl, type VerifiedExecutionAuthority, verifyControl, + verifyControlEffectSettlement, verifyFamilyOwnedRecord, + verifyRetainedArtifactTargetSettlement, writePrivateRecord, } from "@openmapx/core/build-artifacts"; import { @@ -37,10 +39,14 @@ import { type ConfigurationHandoffProgress, configurationHandoffIntentRecordSchema, configurationHandoffProgressSchema, + configurationHandoffWorkerRecordSchema, configurationHandoffRecordDomains as domains, configurationHandoffRecordName as recordName, } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; -import { installationState } from "../../../packages/core/src/build-artifacts/control-authority"; +import { + childSettlementState, + installationState, +} from "../../../packages/core/src/build-artifacts/control-authority"; import { CONFIGURATION_HANDOFF_LABEL, configurationHandoffEnvironmentNames, @@ -360,6 +366,88 @@ export function inspectConfigurationHandoff( : progress.phase === "verifying" ? "verifying" : "prepared"; + if (progress.launch.state === "confirmed") { + let completionRaw: unknown; + const completionName = recordName(handoffId, "completion"); + try { + completionRaw = readPrivateRecord(local, "observations", completionName); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (completionRaw !== undefined) { + const worker = verifyControl( + intent.helperPublicKey, + domains.completion, + completionRaw, + configurationHandoffWorkerRecordSchema, + ); + indexed(authority, completionName, completionRaw); + if ( + worker.intentDigest !== controlDigest(intent) || + worker.preparationDigest !== controlDigest(progress) || + bytes(worker.execution) !== bytes(intent.helper) + ) + throw Error("Original helper completion changed"); + if (worker.phase === "applied") { + const child = openFixedChildInstallation({ + infraDir, + assertion: intent.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const effect = childSettlementState( + verifyControlEffectSettlement(child, intent.helper), + ).evidence; + if ( + effect.outcome !== "committed" || + effect.publication !== "committed" || + effect.terminalReceiptDigest !== + controlDigest({ + kind: "configuration-helper-completion", + completionDigest: controlDigest(worker), + }) + ) + throw Error("Original helper completion remains uncontained"); + verifyRetainedArtifactTargetSettlement( + child, + { leaseId: intent.leaseId, delegationId: intent.delegationId, phase: "ready" }, + intent.helper, + ); + } finally { + closeControlInstallation(child); + } + return { + handoffId, + phase: "applied", + expectedRevisionId: intent.operation.expectedCurrentRevisionId, + appliedRevisionId: intent.operation.expectedCurrentRevisionId, + completedServiceIds: worker.targets + .filter((target) => target.phase === "verified") + .map((target) => target.serviceId), + failedServiceIds: [], + recoveryCodes: [], + }; + } + return { + handoffId, + phase: + worker.phase === "draining" || + worker.phase === "applying" || + worker.phase === "verifying" + ? worker.phase + : "recovery_required", + expectedRevisionId: intent.operation.expectedCurrentRevisionId, + appliedRevisionId: null, + completedServiceIds: worker.targets + .filter((target) => target.phase === "verified") + .map((target) => target.serviceId), + failedServiceIds: worker.targets + .filter((target) => target.phase === "unknown") + .map((target) => target.serviceId), + recoveryCodes: ["receipt_pending"], + }; + } + } return { handoffId, phase, diff --git a/apps/ops-agent/src/configuration-handoff.test.ts b/apps/ops-agent/src/configuration-handoff.test.ts index 7178611a6..9b83717c2 100644 --- a/apps/ops-agent/src/configuration-handoff.test.ts +++ b/apps/ops-agent/src/configuration-handoff.test.ts @@ -48,6 +48,9 @@ it("launches the existing confined old-image helper with only original control r ]); expect(args).not.toContain("OPS_AGENT_HANDOFF_ID"); expect(args).not.toContain("OPS_CONFIGURATION_HANDOFF_ID"); + expect(args.filter((value) => value.startsWith("OPENMAPX_ROOT_DIR="))).toEqual([ + "OPENMAPX_ROOT_DIR=/synthetic/host", + ]); }); it("orders dependencies and the agent last without silently expanding frozen targets", () => { expect( diff --git a/apps/ops-agent/src/configuration-handoff.ts b/apps/ops-agent/src/configuration-handoff.ts index 6c880d393..5d45d0279 100644 --- a/apps/ops-agent/src/configuration-handoff.ts +++ b/apps/ops-agent/src/configuration-handoff.ts @@ -34,6 +34,7 @@ export function configurationHandoffEnvironmentNames(facts: AgentContainerFacts) "OPENMAPX_CONTROL_EXECUTION", "OPENMAPX_CONTROL_DELEGATION", "OPENMAPX_PARENT_EXECUTION", + "OPENMAPX_ROOT_DIR", ]); return z .array( @@ -73,7 +74,9 @@ export function configurationHandoffRunArgs( const args = agentHandoffRunArgs( { ...facts, - envNames: facts.envNames.filter((name) => name !== "OPS_CONFIGURATION_HANDOFF_ID"), + envNames: facts.envNames.filter( + (name) => !["OPS_CONFIGURATION_HANDOFF_ID", "OPENMAPX_ROOT_DIR"].includes(name), + ), }, rootDir, handoffId, @@ -92,6 +95,8 @@ export function configurationHandoffRunArgs( workdir, 0, "--env", + `OPENMAPX_ROOT_DIR=${rootDir}`, + "--env", `OPS_CONFIGURATION_HANDOFF_ID=${handoffId}`, "--env", `OPENMAPX_CONTROL_DELEGATION=${input.assertion}`, diff --git a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts index 2263d31e7..3ac8229f1 100644 --- a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts +++ b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts @@ -21,6 +21,8 @@ export async function inspectMaintenanceDockerRuntime(options: { containers: readonly OwnedServiceSnapshot[]; privateKey: KeyObject; signal: AbortSignal; + /** Interim fixed-helper check only; the final producer always checks all targets. */ + handoffTargetId?: string; }) { const { owner, containers, signal } = options, checks = [], @@ -29,13 +31,17 @@ export async function inspectMaintenanceDockerRuntime(options: { owner.receipt.generation === "not_committed" ? owner.plan.operation.expectedCurrentRevisionId : owner.plan.operation.revisionId, - ids = [ - ...new Set( - owner.plan.steps + allIds = [ + ...new Set([ + ...owner.plan.steps .filter((value) => value.phase === "maintenance-recreate") .map((value) => ("serviceId" in value.operation ? value.operation.serviceId : "")), - ), - ].sort(); + ...(owner.plan.handoff?.serviceIds ?? []), + ]), + ].sort(), + ids = options.handoffTargetId ? [options.handoffTargetId] : allIds; + if (options.handoffTargetId && !owner.plan.handoff?.serviceIds.includes(options.handoffTargetId)) + throw Error("Original handoff target required"); const run = async (args: string[]) => { signal.throwIfAborted(); const response = await options.execFile("docker", args, { diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index 33d273db6..a495e161c 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -80,6 +80,7 @@ import { restoreArtifactTargetLeaseForSettlement, restoreConfigurationMaintenancePhaseIssuer, retainControlEffectReceipt, + settlePreparedConfigurationHandoff, signControl, type VerifiedExecutionAuthority, type VerifiedPreparedDataConfigurationFence, @@ -467,6 +468,50 @@ export function readOwnedConfigurationMaintenance( ): ConfigurationMaintenanceOwnerRecord { return read(authority); } +/** Observation-only completion of the original prepared handoff. All sources + * and physical descendants must already be settled; this runs no Docker I/O. */ +async function observeOwnedConfigurationHandoff( + authority: VerifiedExecutionAuthority, +): Promise { + const owner = read(authority); + if ( + !owner.plan.handoff || + owner.receipt.followUp === "complete" || + owner.final || + owner.receipt.generation !== "committed" + ) + return; + const step = owner.plan.steps.find((value) => value.phase === "maintenance-handoff"), + source = owner.apiControls.steps.find((value) => value.ordinal === step?.ordinal); + if (!source?.canonical || !source.receiptDelivery) return; + await settlePreparedConfigurationHandoff( + restoreAdmittedControlExecution( + executionAuthorityState(authority).installation.installation, + source.execution, + ), + ); + const lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + readOriginalMaintenanceFinalChildren( + authority, + observeConfigurationMaintenanceRuntimeTree(authority), + ); + withConfigurationMaintenanceOwnerObservation(lease, authority, () => { + const current = read(authority); + if (current.receipt.followUp === "complete") return; + if (canonicalJson(current) !== canonicalJson(owner)) + throw Error("Original handoff owner changed during completion"); + const sequence = current.sequence + 1; + write(authority, { + ...current, + sequence, + receipt: { ...current.receipt, sequence, followUp: "complete" }, + }); + }); + } finally { + closeArtifactTargetLeaseHandle(lease); + } +} declare const controlOwnerBrand: unique symbol; export type ConfigurationMaintenanceControlOwner = Readonly<{ [controlOwnerBrand]: true }>; const controlOwners = new WeakMap< @@ -3797,6 +3842,7 @@ export async function assertStagedCoordinatorConfigurationWrite( throw new OpsExecutionFailure("configuration_prerequisite", {}); } const journals = new WeakMap(); +const journalInstallations = new WeakMap(); /** Conservative drain gate from the actual durable journal. This closes new * work only; a matching ID never supplies continuation or release authority. */ export function configurationHandoffPendingMaintenanceIds( @@ -3804,7 +3850,36 @@ export function configurationHandoffPendingMaintenanceIds( ): ReadonlySet { return new Set( (journals.get(runtime)?.records() ?? []) - .filter((job) => job.operation.kind === "configurationHandoff.prepare") + .filter((job) => { + if (job.operation.kind !== "configurationHandoff.prepare") return false; + const installation = journalInstallations.get(runtime); + if (!installation) return true; + try { + const local = installationState(installation), + file = `${job.operation.maintenanceId}.json`, + raw = readPrivateRecord(local, "configuration-maintenance", file), + owner = verifyControl(installation.publicKey, domain, raw, ownerSchema); + verifyFamilyOwnedRecord( + local, + owner.origin, + { scope: "private", namespace: "configuration-maintenance", container: "", name: file }, + canonicalJson(raw), + ); + if ( + owner.receipt.state !== "settled" || + !owner.plan.handoff || + owner.plan.handoff.handoffId !== job.operation.handoffId + ) + return true; + const authority = restoreAdmittedControlExecution(installation, owner.plan.prepare); + restoreOwnedMaintenanceFinal(authority); + assertConfigurationMaintenanceRootClosure(authority); + verifyRetainedArtifactTargetRootRelease(authority); + return false; + } catch { + return true; + } + }) .map( (job) => (job.operation as Extract) @@ -3817,11 +3892,16 @@ export function configurationHandoffPendingMaintenanceIds( export function registerConfigurationMaintenanceJournal( runtime: OpsRuntime, journal: OpsJobJournal, + installation?: ControlInstallation, ) { if (journals.has(runtime)) throw Error("Maintenance journal source already bound"); journals.set(runtime, journal); + if (installation) journalInstallations.set(runtime, installation); return () => { - if (journals.get(runtime) === journal) journals.delete(runtime); + if (journals.get(runtime) === journal) { + journals.delete(runtime); + journalInstallations.delete(runtime); + } }; } declare const admissionBrand: unique symbol; @@ -4257,9 +4337,12 @@ export async function inspectFrozenMaintenanceRuntime( ? inspectOriginalOwnerServices(source, authority, signal) : source.ownedServices.inspectOwnedServices(signal); const containers = await inspectContainers(), - recreateIds = owner.plan.steps - .filter((step) => step.phase === "maintenance-recreate") - .map((step) => ("serviceId" in step.operation ? step.operation.serviceId : "")), + recreateIds = [ + ...owner.plan.steps + .filter((step) => step.phase === "maintenance-recreate") + .map((step) => ("serviceId" in step.operation ? step.operation.serviceId : "")), + ...(owner.plan.handoff?.serviceIds ?? []), + ], unchanged = containers.filter((value) => !recreateIds.includes(value.serviceId)); if ( canonicalJson(unchanged) !== @@ -4818,6 +4901,14 @@ export function registerConfigurationMaintenanceSource(runtime: OpsRuntime, sour state = executionAuthorityState(authority); let owner = readOwnedConfigurationMaintenance(authority); context.signal.throwIfAborted(); + if (owner.plan.handoff && owner.receipt.followUp !== "complete") { + try { + await observeOwnedConfigurationHandoff(authority); + } catch { + /* Unfinished/unknown helpers retain all fences. */ + } + owner = readOwnedConfigurationMaintenance(authority); + } if (owner.final && owner.apiControls.settle?.result === null) { try { if (owner.receipt.state !== "settled") diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index ce516db48..12f0946b2 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -487,7 +487,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const journal: OpsJobJournal = options.journal ?? ({ records: () => [], replace: async () => undefined } satisfies OpsJobJournal); - const unregisterMaintenanceJournal = registerConfigurationMaintenanceJournal(runtime, journal); + const unregisterMaintenanceJournal = registerConfigurationMaintenanceJournal( + runtime, + journal, + options.controlInstallation, + ); app.addHook("onClose", async () => unregisterMaintenanceJournal()); let literalProductionRuntime = false; try { diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.ts b/packages/core/src/build-artifacts/configuration-handoff-records.ts index 448d9b714..d2f9fae6b 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.ts @@ -116,6 +116,36 @@ export type ConfigurationHandoffIntentRecord = z.infer< typeof configurationHandoffIntentRecordSchema >; export type ConfigurationHandoffProgress = z.infer; +export const configurationHandoffWorkerRecordSchema = configurationHandoffProgressSchema + .safeExtend({ + execution: executionRefSchema, + preparationDigest: digest, + observations: z + .array( + z.strictObject({ + serviceId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), + checksDigest: digest, + detailsDigest: digest, + }), + ) + .max(64), + }) + .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 65536) + .refine( + (value) => + new Set(value.observations.map((item) => item.serviceId)).size === value.observations.length, + ) + .refine( + (value) => + value.phase !== "applied" || + (value.observations.length === value.targets.length && + value.targets.every((target) => + value.observations.some((item) => item.serviceId === target.serviceId), + )), + ); +export type ConfigurationHandoffWorkerRecord = z.infer< + typeof configurationHandoffWorkerRecordSchema +>; export const configurationHandoffRecordDomains = { intent: "openmapx-configuration-handoff-intent-v1", progress: "openmapx-configuration-handoff-progress-v1", diff --git a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts index 519b0e4a8..9a5991283 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts @@ -276,11 +276,12 @@ function assertExecutedConfigurationMaintenanceFinalState( ? owner.plan.operation.expectedCurrentRevisionId : owner.plan.operation.revisionId, ids = [ - ...new Set( - owner.plan.steps + ...new Set([ + ...owner.plan.steps .filter((value) => value.phase === "maintenance-recreate") .map((value) => ("serviceId" in value.operation ? value.operation.serviceId : "")), - ), + ...(owner.plan.handoff?.serviceIds ?? []), + ]), ].sort(), requiredChecks = configurationMaintenanceRequiredChecksSchema.parse([ ...["generation", "owner-containers", "dm-guard"].map((check) => ({ diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index d4d292922..d4b835f81 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1,8 +1,10 @@ import { dataWorkflowReceiptSchema } from "../ops/job-evidence"; import { configurationHandoffIntentRecordSchema, + configurationHandoffProgressSchema, configurationHandoffRecordDomains, configurationHandoffRecordName, + configurationHandoffWorkerRecordSchema, } from "./configuration-handoff-records"; import { type ControlChildPhase, @@ -1595,6 +1597,7 @@ const effectObservationSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("data-command"), status: dataExecutionStatusSchema }), z.strictObject({ kind: z.literal("ops-result"), result: z.unknown() }), z.strictObject({ kind: z.literal("feed-proxy-validation-failure"), commitmentDigest: digest }), + z.strictObject({ kind: z.literal("configuration-helper-completion"), completionDigest: digest }), z.strictObject({ kind: z.literal("fixed-cli-exit"), args: z.array(z.string().max(256)).max(64), @@ -2105,6 +2108,92 @@ export function retainControlEffectReceipt( outcome = "committed"; publication = "committed"; } + } else if (receipt.kind === "configuration-helper-completion") { + if (s.kind !== "configuration-helper" || !s.installation.fixedChildId || !s.origin) + throw Error("Original fixed configuration helper required"); + const admitted = readPrivateRecord( + s.installation, + "executions", + `${s.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: unknown } }, + body = z + .strictObject({ + kind: z.literal("configuration-helper"), + handoffId: uuid, + intentDigest: digest, + }) + .parse(admitted.payload.operation), + intent = readConfigurationHandoffSpawnIntent( + s.installation.installation, + s.execution, + body.handoffId, + ), + progressName = configurationHandoffRecordName(body.handoffId, "progress"), + progressRaw = readPrivateRecord(s.installation, "observations", progressName), + progress = verifyControl( + s.installation.publicKeys["ops-agent"], + configurationHandoffRecordDomains.progress, + progressRaw, + configurationHandoffProgressSchema, + ), + completionName = configurationHandoffRecordName(body.handoffId, "completion"), + completionRaw = readPrivateRecord(s.installation, "observations", completionName), + completion = verifyControl( + s.installation.installation.publicKey, + configurationHandoffRecordDomains.completion, + completionRaw, + configurationHandoffWorkerRecordSchema, + ), + spawnName = `${s.installation.fixedChildId}.json`, + spawn = readPrivateRecord(s.installation, "spawn-started", spawnName) as { + execution: unknown; + }, + startedName = `${s.execution.operationKey}.json`, + started = readPrivateRecord(s.installation, "effect-started", startedName) as { + execution: unknown; + receiptDigest: unknown; + }; + for (const [namespace, name, value] of [ + ["executions", startedName, admitted], + ["observations", progressName, progressRaw], + ["observations", completionName, completionRaw], + ["spawn-started", spawnName, spawn], + ["effect-started", startedName, started], + ] as const) + verifyFamilyOwnedRecord( + s.installation, + s.origin, + { scope: "private", namespace, container: "", name }, + canonicalJson(value), + ); + if ( + body.intentDigest !== controlDigest(intent.operation) || + controlDigest(spawn.execution) !== controlDigest(s.execution) || + controlDigest(started.execution) !== controlDigest(s.execution) || + started.receiptDigest !== s.receiptDigest || + progress.intentDigest !== controlDigest(intent) || + progress.phase !== "launched" || + progress.launch.state !== "confirmed" || + completion.intentDigest !== controlDigest(intent) || + completion.preparationDigest !== controlDigest(progress) || + controlDigest(completion.execution) !== controlDigest(s.execution) || + completion.phase !== "applied" || + completion.launch.containerId !== progress.launch.containerId || + receipt.completionDigest !== controlDigest(completion) || + completion.targets.length !== intent.operation.serviceIds.length || + completion.targets.some( + (target) => + !intent.operation.serviceIds.includes(target.serviceId) || + target.expectedImageId !== intent.operation.expectedImageIdentities[target.serviceId] || + target.oldContainerId !== + progress.targets.find((original) => original.serviceId === target.serviceId) + ?.oldContainerId, + ) + ) + throw Error("Actual original helper completion required"); + outcome = "committed"; + publication = "committed"; } else if (receipt.kind === "fixed-cli-exit") { if (!s.installation.fixedChildId) throw Error("Fixed command receipt requires certified child"); @@ -4992,6 +5081,90 @@ export function beginFixedCliExecution( else begin(); } +/** Original agent-owned helper publication and its fixed-child committed + * effect. Data only: no forward/adoption/lease authority is constructed. */ +export function readCompletedConfigurationHandoff( + installation: ControlInstallation, + handoffId: string, +) { + const state = installationState(installation); + if (installation.role !== "ops-agent") throw Error("Original helper observer required"); + const intentName = configurationHandoffRecordName(handoffId, "intent"), + intentRaw = readPrivateRecord(state, "observations", intentName), + intent = verifyControl( + state.publicKeys["ops-agent"], + configurationHandoffRecordDomains.intent, + intentRaw, + configurationHandoffIntentRecordSchema, + ), + progressName = configurationHandoffRecordName(handoffId, "progress"), + progressRaw = readPrivateRecord(state, "observations", progressName), + progress = verifyControl( + state.publicKeys["ops-agent"], + configurationHandoffRecordDomains.progress, + progressRaw, + configurationHandoffProgressSchema, + ), + completionName = configurationHandoffRecordName(handoffId, "completion"), + completionRaw = readPrivateRecord(state, "observations", completionName), + completion = verifyControl( + intent.helperPublicKey, + configurationHandoffRecordDomains.completion, + completionRaw, + configurationHandoffWorkerRecordSchema, + ), + effectName = `${intent.helper.operationKey}.json`, + effectRaw = readPrivateRecord(state, "effects", effectName, 65536), + effect = verifyControl( + intent.helperPublicKey, + "openmapx-control-settlement-v1", + effectRaw, + settlementSchema.extend({ receipt: effectObservationSchema }), + ); + for (const [namespace, name, raw] of [ + ["observations", intentName, intentRaw], + ["observations", progressName, progressRaw], + ["observations", completionName, completionRaw], + ["effects", effectName, effectRaw], + ] as const) + verifyFamilyOwnedRecord( + state, + intent.origin, + { scope: "private", namespace, container: "", name }, + canonicalJson(raw), + ); + if ( + intent.installationId !== installation.installationId || + intent.operation.handoffId !== handoffId || + progress.intentDigest !== controlDigest(intent) || + progress.phase !== "launched" || + progress.launch.state !== "confirmed" || + completion.intentDigest !== controlDigest(intent) || + completion.preparationDigest !== controlDigest(progress) || + completion.phase !== "applied" || + completion.launch.containerId !== progress.launch.containerId || + canonicalJson(completion.execution) !== canonicalJson(intent.helper) || + canonicalJson(effect.identity) !== canonicalJson(intent.helper) || + effect.outcome !== "committed" || + effect.publication !== "committed" || + effect.receipt.kind !== "configuration-helper-completion" || + effect.receipt.completionDigest !== controlDigest(completion) || + effect.terminalReceiptDigest !== controlDigest(effect.receipt) || + completion.targets.length !== intent.operation.serviceIds.length || + completion.targets.some( + (target) => + !intent.operation.serviceIds.includes(target.serviceId) || + target.expectedImageId !== intent.operation.expectedImageIdentities[target.serviceId] || + target.oldContainerId !== + progress.targets.find((original) => original.serviceId === target.serviceId) + ?.oldContainerId, + ) + ) + throw Error("Original committed configuration helper required"); + const { receipt: _private, ...settlement } = effect; + return { intent, completion, settlement }; +} + const executionWitnessSchema = z.strictObject({ version: z.union([z.literal(1), z.literal(2)]), origin: controlOriginSchema.optional(), diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index acff8cab7..e97584158 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -83,6 +83,7 @@ import { installationState, newControlKey, parseControlAssertion, + readCompletedConfigurationHandoff, readControlExecutionWitness, readPrivateRecord, recordLeaseControlAttempt, @@ -2016,6 +2017,95 @@ export async function settleArtifactTargetLeaseChild( tree(s); }); } +/** Complete only the original terminal handoff preparer after its exact fixed + * helper has committed and physically settled. No target/root is acquired. */ +export async function settlePreparedConfigurationHandoff( + authority: VerifiedExecutionAuthority, +): Promise { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "configurationHandoff.prepare" || + owner.installation.fixedChildId || + owner.installation.installation.role !== "ops-agent" || + !owner.origin + ) + throw Error("Original handoff preparer required"); + const admitted = readPrivateRecord( + owner.installation, + "executions", + `${owner.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: unknown } }, + operation = opsOperationSchema.parse(admitted.payload.operation); + if (operation.kind !== "configurationHandoff.prepare") + throw Error("Original handoff operation required"); + const source = readCompletedConfigurationHandoff( + owner.installation.installation, + operation.handoffId, + ), + s = openState(owner.installation.dataRoot, source.intent.leaseId, owner.installation); + try { + const current = tree(s), + parent = [...current.grants.values()].filter( + (grant) => canonicalJson(grant.child) === canonicalJson(owner.execution), + ), + helper = current.grants.get(source.intent.delegationId), + child = current.children.get(source.intent.delegationId); + if ( + parent.length !== 1 || + canonicalJson(source.intent.preparer) !== canonicalJson(owner.execution) || + canonicalJson(source.intent.operation) !== canonicalJson(operation) || + source.intent.preparerAdmissionDigest !== owner.receiptDigest || + source.intent.ownerCertificateDigest !== controlDigest(s.certificateEnvelope) || + canonicalJson(source.intent.ownerPrepare) !== canonicalJson(s.execution) || + helper?.parentNode !== parent[0]!.delegationId || + helper.recipient !== "configuration-helper" || + helper.phase !== "configuration-handoff" || + helper.childKind !== "configuration-helper" || + helper.childPublicKey !== source.intent.helperPublicKey || + canonicalJson(helper.child) !== canonicalJson(source.intent.helper) || + child?.phase !== "settled" || + canonicalJson(child.settlement) !== canonicalJson(source.settlement) || + parent[0]!.childPublicKey !== owner.installation.installation.publicKey + ) + throw Error("Original prepared helper tree changed"); + verifyControlPreparedTerminal(owner.installation, owner.execution); + const evidence = verifyControlEffectSettlement( + owner.installation.installation, + owner.execution, + ), + proof = childSettlementState(evidence).evidence; + if (proof.outcome !== "committed" || proof.publication !== "committed") + throw Error("Original prepared handoff effect required"); + const prior = current.children.get(parent[0]!.delegationId); + if (prior?.phase === "settled") { + if (canonicalJson(prior.settlement) !== canonicalJson(proof)) + throw Error("Original handoff settlement changed"); + return; + } + s.node = parent[0]!.delegationId; + s.execution = owner.execution; + s.key = owner.installation.privateKey; + s.authority = owner; + s.verifiedAuthority = authority; + s.settlementOnly = true; + const lease = Object.freeze({ + leaseId: source.intent.leaseId, + owner: s.certificate.owner, + execution: owner.execution, + serviceIds: parent[0]!.serviceIds, + mode: "borrowed" as const, + }); + handles.set(lease, s); + try { + await settleArtifactTargetLeaseChild(lease, evidence); + } finally { + handles.delete(lease); + } + } finally { + closeState(s); + } +} /** Verify the exact agent-signed shared child node; no sibling private mount or * caller status can stand in for actual target containment. */ export function inspectArtifactTargetLeaseChildSettlement( @@ -3213,6 +3303,34 @@ export function readOriginalMaintenanceFinalChildren( }), ); } + if (step.phase === "maintenance-handoff" && owner.plan.handoff && owner.plan.nestedHandoff) { + const helper = readCompletedConfigurationHandoff( + state.installation.installation, + owner.plan.handoff.handoffId, + ), + physical = tree.children.find( + (value) => canonicalJson(value.execution) === canonicalJson(helper.intent.helper), + ); + if ( + canonicalJson(helper.intent.operation) !== canonicalJson(step.operation) || + canonicalJson(helper.intent.preparer) !== canonicalJson(actual.execution) || + helper.intent.ownerCertificateDigest !== owner.rootCertificateDigest || + !physical || + canonicalJson(physical.settlement) !== canonicalJson(helper.settlement) + ) + throw Error("Original fixed helper settlement unavailable"); + allowed.add(controlDigest(helper.intent.helper)); + children.push( + configurationChildEvidenceSchema.parse({ + planIndex: step.ordinal, + execution: helper.intent.helper, + terminalDigest: controlDigest(helper.completion), + terminalAt: helper.completion.completedAt, + effectSettlementDigest: controlDigest(helper.settlement), + publication: "committed", + }), + ); + } } if ( tree.children.some((value) => !allowed.has(controlDigest(value.execution))) || From de0d7f4392a935e9b54d3d5d208398ae513975e1 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 07:12:37 +0100 Subject: [PATCH 070/135] feat(platform): run detached helper and recover original completion receipts --- ...orm-configuration-runtime-postgres.test.ts | 420 ++++++++++++++++-- .../src/configuration-handoff-control.ts | 66 ++- .../src/configuration-handoff-main.ts | 55 +++ .../src/configuration-handoff-store.ts | 52 ++- .../configuration-helper-docker-preload.mjs | 46 ++ .../configuration-helper-docker.mjs | 22 + .../src/build-artifacts/control-authority.ts | 23 +- .../core/src/build-artifacts/target-lease.ts | 104 +++++ scripts/build-runtime-bundles.mjs | 1 + scripts/ops-authority-inventory.json | 10 +- 10 files changed, 743 insertions(+), 56 deletions(-) create mode 100644 apps/ops-agent/src/configuration-handoff-main.ts create mode 100644 apps/ops-agent/src/test-fixtures/configuration-helper-docker-preload.mjs create mode 100644 apps/ops-agent/src/test-fixtures/configuration-helper-docker.mjs diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 4ad15739c..e2e549ca3 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -1,21 +1,31 @@ /** Actual API candidate -> production registry/agent -> real DM HTTP/SQL guard. * Docker I/O is substituted. This qualifies preparation, launch reconciliation * and real child adoption; not native helper application/public platform HTTP. */ +import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { closeControlInstallation, + controlDigest, openControlInstallation, + openFixedChildInstallation, + readConfigurationHandoffSpawnIntent, + reconcileControlRetention, restoreAdmittedControlExecution, + verifyRetainedArtifactTargetRootRelease, verifyRetainedArtifactTargetSettlement, } from "@openmapx/core/build-artifacts"; import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { eq } from "drizzle-orm"; +import Fastify from "fastify"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; +import { runContainedProcess } from "../../../../ops-agent/src/docker-runtime"; import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; import * as schema from "../../db/schema"; @@ -49,6 +59,152 @@ const { retainPlatformPreview } = await import("../platform-preview-store"); const { loadAuthPolicy } = await import("../../account/auth-policy"); const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); +/** Only the Docker executable is replaced. The separate Node process runs the + * actual production entrypoint, fixed grant/adoption, source and effect code. */ +async function nativeHelper( + host: Awaited>, + environment: Record, + options: { + bundled?: boolean; + fault?: "before-effect" | "after-effect"; + failureStage?: "adoption"; + kill?: { when: Promise; release(): void }; + } = {}, +) { + const { bundled = false, fault } = options; + if (bundled) + await runContainedProcess( + process.execPath, + [ + fileURLToPath(new URL("../../../../../scripts/build-runtime-bundles.mjs", import.meta.url)), + "ops-agent", + ], + { signal: AbortSignal.timeout(60000), timeout: 60000, maxBuffer: 1024 * 1024 }, + ); + const bridge = Fastify({ logger: false, bodyLimit: 64 * 1024 }); + bridge.post("/docker", async (request, reply) => { + const args = (request.body as { args: unknown }).args; + if ( + !Array.isArray(args) || + args.length > 128 || + args.some((value) => typeof value !== "string") + ) + return reply.code(400).send({ error: "Invalid Docker IO" }); + try { + return await host.execFile("docker", args, { + signal: AbortSignal.timeout(30000), + timeout: 30000, + maxBuffer: 1024 * 1024, + }); + } catch { + return reply.code(502).send({ error: "Named Docker IO failed" }); + } + }); + const url = await bridge.listen({ host: "127.0.0.1", port: 0 }); + try { + const child = spawn( + process.execPath, + [ + ...(bundled ? [] : ["--import", createRequire(import.meta.url).resolve("tsx/esm")]), + "--import", + fileURLToPath( + new URL( + "../../../../ops-agent/src/test-fixtures/configuration-helper-docker-preload.mjs", + import.meta.url, + ), + ), + fileURLToPath( + new URL( + bundled + ? "../../../../ops-agent/dist/configuration-handoff-main.mjs" + : "../../../../ops-agent/src/configuration-handoff-main.ts", + import.meta.url, + ), + ), + ], + { + cwd: process.cwd(), + env: { + ...process.env, + ...environment, + OPENMAPX_TEST_DOCKER_BRIDGE: url, + ...(fault ? { OPENMAPX_TEST_HELPER_FAULT: fault } : {}), + ...(options.kill + ? { OPENMAPX_TEST_HELPER_PROCESS_FILE: join(host.root, "native-helper-processes") } + : {}), + }, + signal: AbortSignal.timeout(60000), + stdio: ["ignore", "pipe", "pipe"], + }, + ); + let output = "", + errorOutput = "", + killed = false, + closedSignal: NodeJS.Signals | null = null; + child.stdout.on("data", (value) => { + output += String(value); + }); + child.stderr.on("data", (value) => { + errorOutput += String(value); + }); + const closed = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => { + closedSignal = signal; + resolve(code); + }); + }); + if (options.kill) + void options.kill.when.then(() => { + killed = child.kill("SIGKILL"); + }); + const code = await closed; + options.kill?.release(); + if (options.kill) { + expect({ code, closedSignal, killed, output, errorOutput }).toEqual({ + code: null, + closedSignal: "SIGKILL", + killed: true, + output: "", + errorOutput: "", + }); + } else if (fault || options.failureStage) { + expect({ code, output, errorOutput }).toEqual({ + code: 1, + output: "", + errorOutput: `configuration_handoff_recovery_required:${options.failureStage ?? "application"}\n`, + }); + if (fault) expect(existsSync(join(host.root, "native-helper-fault-observed"))).toBe(true); + } else + expect({ code, output, errorOutput }).toEqual({ + code: 0, + output: "configuration_handoff_applied\n", + errorOutput: "", + }); + } finally { + options.kill?.release(); + await bridge.close(); + if (options.kill && existsSync(join(host.root, "native-helper-processes"))) { + const ids = readFileSync(join(host.root, "native-helper-processes"), "utf8") + .trim() + .split("\n") + .map(Number), + alive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }, + until = Date.now() + 5000; + while (ids.some(alive) && Date.now() < until) + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(ids.filter(alive)).toEqual([]); + } + } +} + describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "owned platform preparation", () => { @@ -58,9 +214,26 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( afterEach(async () => { await state.fixture?.stop(); }); - it.each(["old", "1", "launch", "launch-lost", "policy-change", "launch-apply"])( + it.each([ + "old", + "1", + "launch", + "launch-lost", + "policy-change", + "launch-apply", + "launch-native", + "launch-bundle", + "launch-native-before-effect", + "launch-native-after-effect", + "launch-native-killed", + ])( "guards and retains helper mode %s through original ownership", async (version) => { + const helperEffectEntered = Promise.withResolvers(), + continueHelperEffect = Promise.withResolvers(); + let writersWereReleased = false; + const native = version.startsWith("launch-native") || version === "launch-bundle", + application = version === "launch-apply" || native; const ids = [ "motis", "ops-agent", @@ -79,8 +252,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, configurationHelperVersion: version === "old" ? "old" : "1", configurationHelperLaunchResponseLoss: version === "launch-lost", - configurationHelperReleaseInventory: version === "launch-apply", - ...(version === "launch-apply" + configurationHelperReleaseInventory: application, + ...(application ? { schemas: { "ops-agent": JSON.parse( @@ -102,13 +275,27 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( Labels: { ...host.row.Labels, "com.docker.compose.service": id }, })); host.setRows(rows); - if (version === "launch-apply") { + if (application) { const inspections: Parameters[0] = {}; host.setCreatingProcess(async (args) => { const id = args.at(-1)!, old = rows.find((row) => row.Labels["com.docker.compose.service"] === id); if (!old || !["app-api", "ops-agent"].includes(id)) throw Error("Unexpected protected target"); + if (version === "launch-native-killed") { + await services.withConfigurationGenerationWriter(host.infraDir, async () => { + const lock = await services.acquireReleaseStoreLock( + join(host.infraDir, ".ops-agent-releases"), + {}, + { failIfBusy: true }, + ); + lock.release(); + writersWereReleased = true; + }); + helperEffectEntered.resolve(); + await continueHelperEffect.promise; + return; + } const prefix = args.slice(0, args.indexOf("up")), options = { signal: new AbortController().signal, @@ -207,7 +394,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( email: `${actorId}@example.test`, role: "admin", }); - if (version === "launch-apply") + if (application) await state.fixture.db.insert(schema.serviceConfig).values({ id: randomUUID(), serviceId: "ops-agent", @@ -488,32 +675,152 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, }), ).rejects.toThrow("Original configuration helper spawn required"); - const child = await childControl.openConfigurationHandoffChild({ - rootDir: host.root, - handoffId: owner.plan.handoff!.handoffId, - assertion: env("OPENMAPX_CONTROL_DELEGATION"), - execution: JSON.parse(env("OPENMAPX_CONTROL_EXECUTION")), - }); + let nativeInstallation: + | ReturnType + | undefined; + const child = native + ? undefined + : await childControl.openConfigurationHandoffChild({ + rootDir: host.root, + handoffId: owner.plan.handoff!.handoffId, + assertion: env("OPENMAPX_CONTROL_DELEGATION"), + execution: JSON.parse(env("OPENMAPX_CONTROL_EXECUTION")), + }); try { - await childControl.waitForConfigurationHandoffPreparation( - child, - AbortSignal.timeout(2000), - ); - if (version === "launch-apply") { - const { runConfigurationHandoffApplication } = await import( - "../../../../ops-agent/src/configuration-handoff-application" + if (child) + await childControl.waitForConfigurationHandoffPreparation( + child, + AbortSignal.timeout(2000), ); - expect( - await runConfigurationHandoffApplication({ - child, - rootDir: host.root, - execFile: host.execFile, - }), - ).toMatchObject({ - phase: "applied", - completedServiceIds: ["app-api", "ops-agent"], - }); - const actual = childControl.configurationHandoffChildState(child); + if (application) { + if (native) { + const environment = { + OPENMAPX_ROOT_DIR: host.root, + OPS_CONFIGURATION_HANDOFF_ID: owner.plan.handoff!.handoffId, + OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), + OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), + }, + fault = + version === "launch-native-before-effect" + ? "before-effect" + : version === "launch-native-after-effect" + ? "after-effect" + : undefined; + await nativeHelper(host, environment, { + bundled: version === "launch-bundle", + fault, + ...(version === "launch-native-killed" + ? { + kill: { + when: helperEffectEntered.promise, + release: () => continueHelperEffect.resolve(), + }, + } + : {}), + }); + if (version === "launch-native-killed") { + const file = join( + host.infraDir, + "control-private/ops-agent/observations", + `configuration-handoff-${owner.plan.handoff!.handoffId}.completion.json`, + ), + original = readFileSync(file); + expect(writersWereReleased).toBe(true); + expect(JSON.parse(String(original)).payload).toMatchObject({ + phase: "applying", + completedAt: null, + }); + await nativeHelper(host, environment, { failureStage: "adoption" }); + expect(readFileSync(file)).toEqual(original); + expect( + host.calls.filter( + (args) => args[0] === "compose" && args.includes("up"), + ), + ).toHaveLength(1); + expect(dm.settlePosts).toBe(0); + await expect(dm.assertDataAdmission()).rejects.toThrow(); + expect(() => + verifyRetainedArtifactTargetRootRelease( + restoreAdmittedControlExecution(installation, owner.plan.prepare), + ), + ).toThrow(); + const { inspectConfigurationHandoff } = await import( + "../../../../ops-agent/src/configuration-handoff-store" + ), + execution = JSON.parse(environment.OPENMAPX_CONTROL_EXECUTION), + clock = vi + .spyOn(Date, "now") + .mockReturnValue(execution.deadlineAt + 1); + try { + expect( + inspectConfigurationHandoff( + host.infraDir, + owner.plan.handoff!.handoffId, + ), + ).toMatchObject({ + phase: "recovery_required", + appliedRevisionId: null, + }); + } finally { + clock.mockRestore(); + } + return; + } + if (fault) { + const file = join( + host.infraDir, + "control-private/ops-agent/observations", + `configuration-handoff-${owner.plan.handoff!.handoffId}.completion.json`, + ), + before = readFileSync(file); + await expect(dm.assertDataAdmission()).rejects.toThrow(); + const { inspectConfigurationHandoff } = await import( + "../../../../ops-agent/src/configuration-handoff-store" + ); + expect( + inspectConfigurationHandoff( + host.infraDir, + owner.plan.handoff!.handoffId, + ), + ).toMatchObject({ + phase: "recovery_required", + appliedRevisionId: null, + }); + await nativeHelper(host, environment); + expect(readFileSync(file)).toEqual(before); + await nativeHelper(host, environment); + expect(readFileSync(file)).toEqual(before); + } + nativeInstallation = openFixedChildInstallation({ + infraDir: host.infraDir, + assertion: env("OPENMAPX_CONTROL_DELEGATION"), + }); + } else { + if (!child) throw Error("Original inline helper required"); + const { runConfigurationHandoffApplication } = await import( + "../../../../ops-agent/src/configuration-handoff-application" + ); + expect( + await runConfigurationHandoffApplication({ + child, + rootDir: host.root, + execFile: host.execFile, + }), + ).toMatchObject({ + phase: "applied", + completedServiceIds: ["app-api", "ops-agent"], + }); + } + const actual = child + ? childControl.configurationHandoffChildState(child) + : { + installation: nativeInstallation!, + intent: readConfigurationHandoffSpawnIntent( + nativeInstallation!, + JSON.parse(env("OPENMAPX_CONTROL_EXECUTION")), + owner.plan.handoff!.handoffId, + ), + }; verifyRetainedArtifactTargetSettlement( actual.installation, { @@ -588,8 +895,60 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .filter((args) => args[0] === "compose" && args.includes("up")) .map((args) => args.at(-1)), ).toEqual(["app-api", "ops-agent"]); + if (version === "launch-native-after-effect") { + const location = (role: string) => + join( + host.infraDir, + "control-private", + role, + "retention-locations", + `${controlDigest(owner.origin)}.json`, + ), + clock = vi + .spyOn(Date, "now") + .mockReturnValue(Date.now() + 96 * 3600000); + try { + for ( + let round = 0; + round < 16 && + (existsSync(location("ops-agent")) || + existsSync(location("data-manager"))); + round++ + ) { + await reconcileControlRetention(installation, { maxFamilies: 64 }); + await dm.reconcileRetention(); + } + if (existsSync(location("ops-agent"))) { + const family = JSON.parse( + readFileSync(location("ops-agent"), "utf8"), + ), + progress = JSON.parse( + readFileSync( + join( + host.infraDir, + "control-private/ops-agent/retention-families", + family.name, + "progress.json", + ), + "utf8", + ), + ), + fact = progress.executions[controlDigest(actual.intent.helper)]; + console.info("original helper retention fact", { + known: fact?.known, + terminalAt: fact?.terminalAt, + requiresJournal: fact?.requiresJournal, + }); + } + expect(existsSync(location("ops-agent"))).toBe(false); + expect(existsSync(location("data-manager"))).toBe(false); + } finally { + clock.mockRestore(); + } + } return; } + if (!child) throw Error("Original inline helper required"); childControl.assertConfigurationHandoffHostSources(child); writeFileSync(join(host.infraDir, ".env"), "OPENMAPX_SYNTHETIC_DRIFT=1\n"); expect(() => @@ -599,7 +958,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( host.calls.filter((args) => args[0] === "compose" && args.includes("up")), ).toHaveLength(0); } finally { - childControl.closeConfigurationHandoffChild(child); + if (child) childControl.closeConfigurationHandoffChild(child); + if (nativeInstallation) closeControlInstallation(nativeInstallation); } return; } diff --git a/apps/ops-agent/src/configuration-handoff-control.ts b/apps/ops-agent/src/configuration-handoff-control.ts index 125157f6c..aa491d05d 100644 --- a/apps/ops-agent/src/configuration-handoff-control.ts +++ b/apps/ops-agent/src/configuration-handoff-control.ts @@ -16,14 +16,26 @@ import { executionAuthorityState, openFixedChildInstallation, readConfigurationHandoffSpawnIntent, + readPrivateRecord, + restoreAdmittedControlExecution, retainAdmittedControlExecution, + retainControlEffectReceipt, + settleRetainedConfigurationHandoffHelper, type VerifiedExecutionAuthority, verifiedParentForArtifactDelegation, verifyConfigurationHelperReady, + verifyControl, + verifyFamilyOwnedRecord, verifyFixedCliControlPeer, } from "@openmapx/core/build-artifacts"; import { canonicalJson, type ExecutionRef, executionRefSchema } from "@openmapx/core/ops"; -import type { ConfigurationHandoffIntentRecord } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; +import { + type ConfigurationHandoffIntentRecord, + configurationHandoffRecordDomains, + configurationHandoffRecordName, + configurationHandoffWorkerRecordSchema, +} from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; +import { installationState } from "../../../packages/core/src/build-artifacts/control-authority"; declare const childBrand: unique symbol; export type ConfigurationHandoffChild = Readonly<{ readonly [childBrand]: true }>; @@ -115,6 +127,58 @@ export async function openConfigurationHandoffChild(input: { throw error; } } +/** A completed original worker can repair only its receipt/physical leaf. An + * unfinished attempted worker must never become another forward execution. */ +export async function reconcileCompletedConfigurationHandoff(input: { + rootDir: string; + handoffId: string; + assertion: string; + execution: ExecutionRef; +}): Promise<"applied" | null> { + const execution = executionRefSchema.parse(input.execution), + installation = openFixedChildInstallation({ + infraDir: join(input.rootDir, "infra/docker"), + assertion: input.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const intent = readConfigurationHandoffSpawnIntent(installation, execution, input.handoffId), + state = installationState(installation), + file = configurationHandoffRecordName(input.handoffId, "completion"); + if (intent.assertion !== input.assertion) throw Error("Original helper assertion required"); + let raw: unknown; + try { + raw = readPrivateRecord(state, "observations", file); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + const worker = verifyControl( + installation.publicKey, + configurationHandoffRecordDomains.completion, + raw, + configurationHandoffWorkerRecordSchema, + ); + verifyFamilyOwnedRecord( + state, + intent.origin, + { scope: "private", namespace: "observations", container: "", name: file }, + canonicalJson(raw), + ); + if (worker.phase !== "applied" || canonicalJson(worker.execution) !== canonicalJson(execution)) + throw Error("Unfinished original helper remains fenced"); + const authority = restoreAdmittedControlExecution(installation, execution); + retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { kind: "configuration-helper-completion", completionDigest: controlDigest(worker) }, + }); + await settleRetainedConfigurationHandoffHelper(authority); + return "applied"; + } finally { + closeControlInstallation(installation); + } +} export function configurationHandoffChildState(child: ConfigurationHandoffChild) { const state = children.get(child); if (!state) throw Error("Original nominal helper required"); diff --git a/apps/ops-agent/src/configuration-handoff-main.ts b/apps/ops-agent/src/configuration-handoff-main.ts new file mode 100644 index 000000000..6ff73ab66 --- /dev/null +++ b/apps/ops-agent/src/configuration-handoff-main.ts @@ -0,0 +1,55 @@ +/** Detached fixed helper entrypoint. Only the original signed child can open + * the retained target tree. Failures keep the original fence for observation. */ +import { isAbsolute } from "node:path"; +import { executionRefSchema } from "@openmapx/core/ops"; +import { z } from "zod"; +import { runConfigurationHandoffApplication } from "./configuration-handoff-application"; +import { + closeConfigurationHandoffChild, + openConfigurationHandoffChild, + reconcileCompletedConfigurationHandoff, +} from "./configuration-handoff-control"; + +let stage: "input" | "adoption" | "application" = "input"; +async function main(): Promise { + const rootDir = process.env.OPENMAPX_ROOT_DIR ?? ""; + if (!isAbsolute(rootDir)) return 2; + const handoffId = z.uuid().parse(process.env.OPS_CONFIGURATION_HANDOFF_ID), + assertion = z.string().min(1).max(4096).parse(process.env.OPENMAPX_CONTROL_DELEGATION), + execution = executionRefSchema.parse( + JSON.parse(process.env.OPENMAPX_CONTROL_EXECUTION ?? "null"), + ), + controller = new AbortController(), + stop = () => controller.abort(Error("Configuration helper stopping")); + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + stage = "adoption"; + if (await reconcileCompletedConfigurationHandoff({ rootDir, handoffId, assertion, execution })) { + process.stdout.write("configuration_handoff_applied\n"); + process.removeListener("SIGINT", stop); + process.removeListener("SIGTERM", stop); + return 0; + } + const child = await openConfigurationHandoffChild({ rootDir, handoffId, assertion, execution }); + try { + stage = "application"; + await runConfigurationHandoffApplication({ child, rootDir, signal: controller.signal }); + process.stdout.write("configuration_handoff_applied\n"); + return 0; + } finally { + closeConfigurationHandoffChild(child); + process.removeListener("SIGINT", stop); + process.removeListener("SIGTERM", stop); + } +} +main().then( + (code) => { + process.exitCode = code; + }, + () => { + // Paths, Docker diagnostics, environment values and control assertions stay + // private. The authenticated journal supplies the safe operator status. + process.stderr.write(`configuration_handoff_recovery_required:${stage}\n`); + process.exitCode = 1; + }, +); diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index f44a0449d..70567778e 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -389,12 +389,14 @@ export function inspectConfigurationHandoff( ) throw Error("Original helper completion changed"); if (worker.phase === "applied") { - const child = openFixedChildInstallation({ - infraDir, - assertion: intent.assertion, - container: existsSync("/run/secrets/openmapx-control-verifiers"), - }); + let child: ReturnType | undefined, + contained = false; try { + child = openFixedChildInstallation({ + infraDir, + assertion: intent.assertion, + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); const effect = childSettlementState( verifyControlEffectSettlement(child, intent.helper), ).evidence; @@ -413,27 +415,33 @@ export function inspectConfigurationHandoff( { leaseId: intent.leaseId, delegationId: intent.delegationId, phase: "ready" }, intent.helper, ); + contained = true; + } catch { + // A worker publication without the exact effect and physical leaf + // remains recovery-required. It cannot satisfy parent/root closure. } finally { - closeControlInstallation(child); + if (child) closeControlInstallation(child); } - return { - handoffId, - phase: "applied", - expectedRevisionId: intent.operation.expectedCurrentRevisionId, - appliedRevisionId: intent.operation.expectedCurrentRevisionId, - completedServiceIds: worker.targets - .filter((target) => target.phase === "verified") - .map((target) => target.serviceId), - failedServiceIds: [], - recoveryCodes: [], - }; + if (contained) + return { + handoffId, + phase: "applied", + expectedRevisionId: intent.operation.expectedCurrentRevisionId, + appliedRevisionId: intent.operation.expectedCurrentRevisionId, + completedServiceIds: worker.targets + .filter((target) => target.phase === "verified") + .map((target) => target.serviceId), + failedServiceIds: [], + recoveryCodes: [], + }; } return { handoffId, phase: - worker.phase === "draining" || - worker.phase === "applying" || - worker.phase === "verifying" + Date.now() < intent.helper.deadlineAt && + (worker.phase === "draining" || + worker.phase === "applying" || + worker.phase === "verifying") ? worker.phase : "recovery_required", expectedRevisionId: intent.operation.expectedCurrentRevisionId, @@ -444,7 +452,9 @@ export function inspectConfigurationHandoff( failedServiceIds: worker.targets .filter((target) => target.phase === "unknown") .map((target) => target.serviceId), - recoveryCodes: ["receipt_pending"], + recoveryCodes: [ + Date.now() >= intent.helper.deadlineAt ? "deadline_expired" : "receipt_pending", + ], }; } } diff --git a/apps/ops-agent/src/test-fixtures/configuration-helper-docker-preload.mjs b/apps/ops-agent/src/test-fixtures/configuration-helper-docker-preload.mjs new file mode 100644 index 000000000..4100481f0 --- /dev/null +++ b/apps/ops-agent/src/test-fixtures/configuration-helper-docker-preload.mjs @@ -0,0 +1,46 @@ +/** Replace only the external Docker executable. Inherited PATH and all source + * values stay identical to the actual signed old-agent snapshot. */ +import childProcess from "node:child_process"; +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { basename, dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const spawn = childProcess.spawn, + fixture = fileURLToPath(new URL("./configuration-helper-docker.mjs", import.meta.url)); +childProcess.spawn = (file, args, options) => { + const child = + file === "docker" + ? spawn(process.execPath, [fixture, ...args], options) + : spawn(file, args, options); + if (file === "docker" && process.env.OPENMAPX_TEST_HELPER_PROCESS_FILE && child.pid) + fs.appendFileSync(process.env.OPENMAPX_TEST_HELPER_PROCESS_FILE, `${child.pid}\n`, { + mode: 0o600, + }); + return child; +}; +const fault = process.env.OPENMAPX_TEST_HELPER_FAULT; +if (fault === "before-effect" || fault === "after-effect") { + const rename = fs.renameSync, + root = process.env.OPENMAPX_ROOT_DIR, + key = JSON.parse(process.env.OPENMAPX_CONTROL_EXECUTION).operationKey; + let injected = false; + fs.renameSync = (from, to) => { + let match = false; + if (!injected && basename(String(to)) === `${key}.json`) { + const actual = fs.statSync(dirname(String(to))), + expected = fs.statSync(join(root, "infra/docker/control-private/ops-agent/effects")); + match = + actual.dev === expected.dev && + actual.ino === expected.ino && + JSON.parse(fs.readFileSync(from, "utf8"))?.payload?.receipt?.kind === + "configuration-helper-completion"; + } + if (!match) return rename(from, to); + injected = true; + if (fault === "after-effect") rename(from, to); + fs.writeFileSync(join(root, "native-helper-fault-observed"), fault, { mode: 0o600 }); + throw Error("Named helper effect publication interruption"); + }; +} +syncBuiltinESMExports(); diff --git a/apps/ops-agent/src/test-fixtures/configuration-helper-docker.mjs b/apps/ops-agent/src/test-fixtures/configuration-helper-docker.mjs new file mode 100644 index 000000000..80bb81dc6 --- /dev/null +++ b/apps/ops-agent/src/test-fixtures/configuration-helper-docker.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +/** Named external Docker I/O only for the owned native helper fixture. */ +async function main() { + const url = new URL(process.env.OPENMAPX_TEST_DOCKER_BRIDGE); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1") + throw Error("Invalid fixture bridge"); + const response = await fetch(new URL("/docker", url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ args: process.argv.slice(2) }), + }); + if (!response.ok) throw Error("Named Docker IO failed"); + const result = await response.json(); + if (typeof result.stdout !== "string" || typeof result.stderr !== "string") + throw Error("Invalid fixture response"); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); +} +main().catch(() => { + process.stderr.write("Named Docker IO failed\n"); + process.exitCode = 1; +}); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index d4b835f81..21b0d602b 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1974,7 +1974,8 @@ export function retainControlEffectReceipt( if (!parsed.success) throw Error("Unsupported effect receipt evidence"); const receipt = parsed.data; let outcome: ChildSettlement["outcome"] = "unknown", - publication: ChildSettlement["publication"] = "unknown"; + publication: ChildSettlement["publication"] = "unknown", + helperCompletedAt: number | undefined; if (receipt.kind === "catalog-pins-initialized") { if ( s.kind !== "catalog-pins.initialize" || @@ -2179,6 +2180,7 @@ export function retainControlEffectReceipt( completion.preparationDigest !== controlDigest(progress) || controlDigest(completion.execution) !== controlDigest(s.execution) || completion.phase !== "applied" || + completion.completedAt === null || completion.launch.containerId !== progress.launch.containerId || receipt.completionDigest !== controlDigest(completion) || completion.targets.length !== intent.operation.serviceIds.length || @@ -2192,6 +2194,7 @@ export function retainControlEffectReceipt( ) ) throw Error("Actual original helper completion required"); + helperCompletedAt = completion.completedAt; outcome = "committed"; publication = "committed"; } else if (receipt.kind === "fixed-cli-exit") { @@ -2337,6 +2340,7 @@ export function retainControlEffectReceipt( s.execution, outcome !== "unknown", controlDigest(payload), + helperCompletedAt, ); return; } @@ -2346,7 +2350,21 @@ export function retainControlEffectReceipt( raw, settlementSchema.extend({ receipt: effectObservationSchema }), ); - if (controlDigest(prior) === controlDigest(payload)) return; + if (controlDigest(prior) === controlDigest(payload)) { + // A completed helper may have lost the write acknowledgement after + // the signed effect landed but before its family terminal fact. Repair + // only this exact source using its original signed completion time. + if (s.origin && helperCompletedAt !== undefined) + markFamilyTerminal( + s.installation, + s.origin, + s.execution, + true, + controlDigest(payload), + helperCompletedAt, + ); + return; + } // A later transport/journal failure must never erase proved committed effects. if (prior.outcome !== "unknown") return; writePrivateRecord(s.installation, "effects", name, signed, false, s.origin); @@ -2361,6 +2379,7 @@ export function retainControlEffectReceipt( s.execution, outcome !== "unknown", controlDigest(payload), + helperCompletedAt, ); }); } diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index e97584158..e28893cf0 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -2106,6 +2106,110 @@ export async function settlePreparedConfigurationHandoff( closeState(s); } } +/** Receipt-only reopening of the original completed fixed helper. The signed + * completion/effect and original ready adoption must already exist. No target + * acquire/adopt, forward capability or clock renewal is available here. */ +export async function settleRetainedConfigurationHandoffHelper( + authority: VerifiedExecutionAuthority, +): Promise { + const owner = executionAuthorityState(authority); + if (owner.kind !== "configuration-helper" || !owner.installation.fixedChildId || !owner.origin) + throw Error("Original completed fixed helper required"); + const raw = readPrivateRecord( + owner.installation, + "executions", + `${owner.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: { handoffId?: unknown } } }, + source = readCompletedConfigurationHandoff( + owner.installation.installation, + uuid.parse(raw.payload.operation.handoffId), + ), + adopted = readPrivateRecord( + owner.installation, + "adoptions", + `${source.intent.delegationId}.json`, + 8192, + ), + adoption = z + .object({ + state: z.literal("ready"), + execution: executionRefSchema, + assertion: z.string(), + grantDigest: digest, + }) + .parse(adopted), + assertion = parseControlAssertion(adoption.assertion); + verifyFamilyOwnedRecord( + owner.installation, + owner.origin, + { + scope: "private", + namespace: "adoptions", + container: "", + name: `${source.intent.delegationId}.json`, + }, + canonicalJson(adopted), + ); + if ( + canonicalJson(source.intent.helper) !== canonicalJson(owner.execution) || + source.intent.helperPublicKey !== owner.installation.installation.publicKey || + source.intent.delegationId !== owner.installation.fixedChildId || + adoption.assertion !== source.intent.assertion || + canonicalJson(adoption.execution) !== canonicalJson(owner.execution) || + assertion.leaseId !== source.intent.leaseId || + assertion.delegationId !== source.intent.delegationId || + assertion.grantDigest !== adoption.grantDigest + ) + throw Error("Original helper ready adoption changed"); + const s = openState(owner.installation.dataRoot, source.intent.leaseId, owner.installation); + try { + const current = tree(s), + grant = current.grants.get(source.intent.delegationId), + child = current.children.get(source.intent.delegationId); + if ( + controlDigest(s.certificateEnvelope) !== source.intent.ownerCertificateDigest || + !sameControlOrigin(s.certificate.origin, owner.origin) || + grant?.recipient !== "configuration-helper" || + grant.childKind !== "configuration-helper" || + grant.childPublicKey !== owner.installation.installation.publicKey || + child?.grantDigest !== adoption.grantDigest || + canonicalJson(grant.child) !== canonicalJson(owner.execution) || + !child + ) + throw Error("Original retained helper tree changed"); + if (child.phase === "settled") { + if (canonicalJson(child.settlement) !== canonicalJson(source.settlement)) + throw Error("Original helper settlement changed"); + return; + } + verifyControlPreparedTerminal(owner.installation, source.intent.preparer); + s.node = source.intent.delegationId; + s.execution = owner.execution; + s.key = owner.installation.privateKey; + s.authority = owner; + s.verifiedAuthority = authority; + s.settlementOnly = true; + const lease = Object.freeze({ + leaseId: source.intent.leaseId, + owner: s.certificate.owner, + execution: owner.execution, + serviceIds: grant.serviceIds, + mode: "borrowed" as const, + }); + handles.set(lease, s); + try { + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(owner.installation.installation, owner.execution), + ); + } finally { + handles.delete(lease); + } + } finally { + closeState(s); + } +} /** Verify the exact agent-signed shared child node; no sibling private mount or * caller status can stand in for actual target containment. */ export function inspectArtifactTargetLeaseChildSettlement( diff --git a/scripts/build-runtime-bundles.mjs b/scripts/build-runtime-bundles.mjs index 3c0b42933..71b81f37f 100644 --- a/scripts/build-runtime-bundles.mjs +++ b/scripts/build-runtime-bundles.mjs @@ -45,6 +45,7 @@ export const RUNTIME_BUNDLES = { entries: { agent: "apps/ops-agent/src/index.ts", "agent-handoff-main": "apps/ops-agent/src/agent-handoff-main.ts", + "configuration-handoff-main": "apps/ops-agent/src/configuration-handoff-main.ts", cli: "packages/cli/src/index.ts", }, }, diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index e1e7b7c46..e6ea54a0e 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -50,6 +50,8 @@ "configuration.hardlinks.preview", "configuration.inspect", "configuration.sources.inspect", + "configurationHandoff.inspect", + "configurationHandoff.prepare", "configurationMaintenance.decide", "configurationMaintenance.inspect", "configurationMaintenance.prepare", @@ -368,13 +370,17 @@ "apps/api/src/services/prospective-configuration.ts#inspectConfigurationMaintenance", "apps/api/src/services/prospective-configuration.ts#decideConfigurationMaintenance", "apps/api/src/services/prospective-configuration.ts#settleConfigurationMaintenance", - "apps/ops-agent/src/configuration-maintenance-store.ts#prepareConfigurationMaintenanceAdmission" + "apps/ops-agent/src/configuration-maintenance-store.ts#prepareConfigurationMaintenanceAdmission", + "apps/ops-agent/src/configuration-handoff-store.ts#launchPreparedConfigurationHandoff", + "apps/ops-agent/src/configuration-handoff-store.ts#inspectConfigurationHandoff" ], "typedKinds": [ "configurationMaintenance.prepare", "configurationMaintenance.inspect", "configurationMaintenance.decide", - "configurationMaintenance.settle" + "configurationMaintenance.settle", + "configurationHandoff.prepare", + "configurationHandoff.inspect" ] }, { From 19d6b7899b583d9b7500125aaebaef58416028e6 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 07:40:17 +0100 Subject: [PATCH 071/135] feat(platform): retire completed previews and preserve recovery accountability --- apps/api/src/privacy/catalogue.ts | 22 ++ apps/api/src/privacy/openmapx-collectors.ts | 25 ++ .../__tests__/activity-retention.test.ts | 19 +- ...orm-configuration-runtime-postgres.test.ts | 70 ++++- .../platform-preview-postgres.test.ts | 279 ++++++++++++++++++ apps/api/src/services/activity-retention.ts | 11 +- .../src/services/platform-preview-store.ts | 54 +++- .../src/services/prospective-configuration.ts | 61 ++++ packages/i18n/locales/de.json | 9 + packages/i18n/locales/en.json | 9 + scripts/check-subject-data.ts | 1 + 11 files changed, 531 insertions(+), 29 deletions(-) diff --git a/apps/api/src/privacy/catalogue.ts b/apps/api/src/privacy/catalogue.ts index 5e40134e6..6ad96ceb1 100644 --- a/apps/api/src/privacy/catalogue.ts +++ b/apps/api/src/privacy/catalogue.ts @@ -417,6 +417,28 @@ const registrationDefinitions: SubjectDataRegistration[] = [ secretPolicy: { mode: "metadata", code: "attribution-safe-metadata" }, rightsOfOthers: { mode: "none" }, }, + { + id: "platform-configuration-previews", + version: 1, + category: "administrative-accountability", + source: "openmapx-db", + strategy: "collector", + collectorId: "openmapx-administration", + locators: [{ type: "actor_id", field: "platform_configuration_preview.actor_id" }], + purposes: ["service-provisioning", "security-and-abuse-prevention"], + legalBases: ["contract", "legitimate-interest"], + origin: { kind: "controller", descriptionCode: "administrative-record" }, + retention: { + kind: "account-lifetime", + daysAfterClosure: 0, + reasonCode: "configuration-recovery", + }, + recipients: [{ id: "openmapx-controller", roleCode: "controller" }], + article15: "include", + portability: { decision: "exclude", reasonCode: "operator-configuration" }, + secretPolicy: { mode: "metadata", code: "configuration-intent-redacted" }, + rightsOfOthers: { mode: "operator_review", code: "rights-of-others" }, + }, { id: "secret-update-attribution", version: 1, diff --git a/apps/api/src/privacy/openmapx-collectors.ts b/apps/api/src/privacy/openmapx-collectors.ts index 2b904ee41..e13ec342a 100644 --- a/apps/api/src/privacy/openmapx-collectors.ts +++ b/apps/api/src/privacy/openmapx-collectors.ts @@ -37,6 +37,7 @@ import { passkey, personalTimelineConnection, personalVehicle, + platformConfigurationPreview, savedList, savedPlace, serviceSecret, @@ -914,6 +915,30 @@ export async function* streamOpenMapxRegistrationRecords( }, ); return; + case "platform-configuration-previews": + yield* emit( + database + .select({ + id: platformConfigurationPreview.id, + createdAt: platformConfigurationPreview.createdAt, + expiresAt: platformConfigurationPreview.expiresAt, + consumedAt: platformConfigurationPreview.consumedAt, + retiredAt: platformConfigurationPreview.retiredAt, + }) + .from(platformConfigurationPreview) + .where( + and( + eq(platformConfigurationPreview.actorId, context.userId), + lte(platformConfigurationPreview.createdAt, cutoff), + ), + ) + .orderBy(asc(platformConfigurationPreview.id)), + { + policyCodes: ["configuration-intent-redacted", "rights-of-others"], + map: ({ id, ...row }) => ({ ...row, previewDigest: pseudonymousIdentifier(id) }), + }, + ); + return; case "admin-jobs-attribution": yield* emit( database diff --git a/apps/api/src/services/__tests__/activity-retention.test.ts b/apps/api/src/services/__tests__/activity-retention.test.ts index 6fbfcfc1c..b986c336c 100644 --- a/apps/api/src/services/__tests__/activity-retention.test.ts +++ b/apps/api/src/services/__tests__/activity-retention.test.ts @@ -1,19 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { adminJob } from "../../db/schema"; import { createDbMock, type DbMock } from "../../test/db.js"; const dbMock: DbMock = createDbMock(); vi.mock("../../db", () => ({ db: dbMock.db })); -vi.mock("../../db/schema", () => ({ - adminAuditLog: { - id: "auditId", - actorId: "auditActorId", - targetId: "auditTargetId", - createdAt: "auditCreatedAt", - }, - adminJob: { id: "jobId", createdBy: "jobCreatedBy", finishedAt: "jobFinishedAt" }, - appLog: { id: "appLogId", metadata: "appLogMetadata", createdAt: "appLogCreatedAt" }, - verification: { id: "verificationId", expiresAt: "verificationExpiresAt" }, -})); vi.mock("drizzle-orm", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, lt: vi.fn((column: unknown, value: unknown) => ({ column, value })) }; @@ -74,10 +64,15 @@ describe("pruneAuditLog / pruneCompletedJobs retention guard", () => { }); it("deletes rows and computes the cutoff for a positive day count", async () => { + if (fn === pruneCompletedJobs) { + dbMock.queueDelete([]); + dbMock.queueDelete([]); + } dbMock.queueDelete([{ id: "a" }, { id: "b" }]); const result = await fn(happyDays); expect(result).toBe(2); - expect(dbMock.db.delete).toHaveBeenCalledTimes(1); + if (fn === pruneCompletedJobs) expect(dbMock.db.delete).toHaveBeenLastCalledWith(adminJob); + else expect(dbMock.db.delete).toHaveBeenCalledTimes(1); const expectedCutoff = new Date(NOW.getTime() - happyDays * MS_PER_DAY); expect(lt).toHaveBeenCalledWith(expect.anything(), expectedCutoff); }); diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index e2e549ca3..4c42c750e 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -35,6 +35,7 @@ import type { JobContext } from "../job-runner"; const state = vi.hoisted(() => ({ fixture: null as unknown as Awaited>, services: [] as services.LoadedService[], + registry: null as InstanceType | null, })); vi.mock("../../db", () => ({ get db() { @@ -46,7 +47,7 @@ vi.mock("../../db", () => ({ })); vi.mock("../../integration-host", () => ({ getAllIntegrations: () => [] })); vi.mock("../service-registry", () => ({ - getServiceRegistry: () => ({ list: () => state.services }), + getServiceRegistry: () => state.registry ?? { list: () => state.services }, })); const writer = await import("../administrative-configuration-writer-owner"); const { withConfigurationCommitParticipant } = await import("../configuration-commit-participant"); @@ -226,6 +227,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-before-effect", "launch-native-after-effect", "launch-native-killed", + "launch-native-producer", ])( "guards and retains helper mode %s through original ownership", async (version) => { @@ -268,6 +270,14 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( actualComposeConfig: true, }); state.services = host.loaded; + if (version === "launch-native-producer") { + state.registry = new services.ServiceRegistry({ + rootDir: host.root, + enabledIds: new Set(ids), + }); + await state.registry.load(); + state.services = state.registry.list(); + } let rows = ids.map((id, index) => ({ ...host.row, Id: String(index + 1).repeat(64), @@ -539,6 +549,61 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( step.ordinal, ); expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); + if (version === "launch-native-producer") { + const args = host.calls.find((args) => args[0] === "run")!, + env = (name: string) => + args + .find((value) => value.startsWith(`${name}=`))! + .slice(name.length + 1); + await nativeHelper(host, { + OPENMAPX_ROOT_DIR: host.root, + OPS_CONFIGURATION_HANDOFF_ID: env("OPS_CONFIGURATION_HANDOFF_ID"), + OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), + OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), + }); + const result = await prospective.persistAndApplyProspectiveConfiguration( + candidate, + { coordinatorMode: "require-running" }, + ); + expect(result).toMatchObject({ + saved: "committed", + application: { + outcome: "committed", + followUp: "complete", + publisherCleanupFailed: false, + registryUpdateFailed: false, + }, + }); + const [retired] = await state.fixture.db + .select() + .from(schema.platformConfigurationPreview) + .where(eq(schema.platformConfigurationPreview.id, preview.previewId)); + expect(retired?.retiredAt).toBeInstanceOf(Date); + const original = + prospective.validatedConfigurationCandidateState(candidate); + expect( + existsSync( + join(host.queue, ".candidates", `${original.record.operationKey}.json`), + ), + ).toBe(false); + expect(ctx.checkpointResult?._completion).toMatchObject({ + result: { + platformConfiguration: { + previewFingerprint: preview.previewFingerprint, + revisionId: candidate.revisionId, + followUp: "complete", + }, + }, + }); + expect(dm.settlePosts).toBe(1); + expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); + expect( + host.calls + .filter((args) => args[0] === "compose" && args.includes("up")) + .map((args) => args.at(-1)), + ).toEqual(["app-api", "ops-agent"]); + return candidate; + } // Reattachment observes the same terminal preparation; // it cannot consume another dispatch ticket/create again. await prospective.executeConfigurationMaintenanceStep( @@ -580,7 +645,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ); throw error; } - if (version === "policy-change") return; + if (version === "policy-change" || version === "launch-native-producer") return; const retained = ctx.checkpointResult?._configurationMaintenance as | { phase?: string; receipt?: { state?: string; steps?: { phase: string }[] } } | undefined; @@ -1008,6 +1073,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await app.close(); closeControlInstallation(installation); state.services = []; + state.registry = null; vi.unstubAllEnvs(); rmSync(host.cleanupRoot, { recursive: true, force: true }); } diff --git a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts index dcdee8ba6..749247b1e 100644 --- a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts @@ -66,6 +66,285 @@ dbDescribe("durable exact platform preview", () => { services: [], }); } + it("reclaims only closed previews after both retirement and job completion floors", async () => { + const actorId = await actor(), + created = new Date("2025-02-01T00:00:00Z"), + rows = [] as { id: string; kind: string }[]; + for (const kind of ["closed", "uncertain", "running", "late-job", "recent-retirement"]) { + const value = preview(actorId), + jobId = randomUUID(); + await retainPlatformPreview( + { + actorId, + operationKey: randomUUID(), + requestFingerprint: "e".repeat(64), + preview: value, + now: created, + }, + fixture.db, + ); + await fixture.db.insert(adminJob).values({ + id: jobId, + type: "platform.apply", + createdBy: actorId, + payload: {}, + result: {}, + status: kind === "uncertain" || kind === "running" ? "running" : "success", + createdAt: created, + finishedAt: + kind === "uncertain" || kind === "running" + ? null + : new Date(created.getTime() + (kind === "late-job" ? 95 * 3600000 : 180000)), + }); + await fixture.db + .update(platformConfigurationPreview) + .set({ + consumedByJobId: jobId, + consumedAt: new Date(created.getTime() + 60000), + retiredAt: + kind === "uncertain" + ? null + : new Date( + created.getTime() + (kind === "recent-retirement" ? 95 * 3600000 : 120000), + ), + }) + .where(eq(platformConfigurationPreview.id, value.previewId)); + rows.push({ id: value.previewId, kind }); + } + const trigger = async (now: Date) => + retainPlatformPreview( + { + actorId, + operationKey: randomUUID(), + requestFingerprint: "f".repeat(64), + preview: preview(actorId), + now, + }, + fixture.db, + ); + await trigger(new Date(created.getTime() + 72 * 3600000)); + for (const row of rows) + expect( + await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.id, row.id)), + ).toHaveLength(1); + await trigger(new Date(created.getTime() + 96 * 3600000)); + for (const row of rows) + expect( + await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.id, row.id)), + row.kind, + ).toHaveLength(row.kind === "closed" ? 0 : 1); + }); + it("honors an actor-bound privacy preservation hold when unused previews expire", async () => { + const actorId = await actor(), + value = preview(actorId), + created = new Date(Date.now() - 20 * 60000); + await retainPlatformPreview( + { + actorId, + operationKey: randomUUID(), + requestFingerprint: "d".repeat(64), + preview: value, + now: created, + }, + fixture.db, + ); + const { PrivacyRequestService } = await import("../../privacy/request-service"), + { loadMasterKeyRing } = await import("../../privacy/crypto"), + { dataSubjectRequestPreservation } = await import("../../db/schema"), + { releasePreservation } = await import("../../privacy/preservation"); + const service = new PrivacyRequestService({ + database: fixture.db, + keyRing: loadMasterKeyRing({ + env: { + NODE_ENV: "development", + OPENMAPX_EXPORTS_KEY: Buffer.alloc(32, 9).toString("base64url"), + }, + }), + }), + request = await service.create({ + userId: actorId, + kind: "access", + channel: "self_service", + locale: "en", + timeZone: "UTC", + }); + const [hold] = await fixture.db + .select() + .from(dataSubjectRequestPreservation) + .where( + sql`${dataSubjectRequestPreservation.requestId}=${request.id} AND ${dataSubjectRequestPreservation.registrationId}='platform-configuration-previews'`, + ); + expect(hold?.status).toBe("held"); + const trigger = () => + retainPlatformPreview( + { + actorId, + operationKey: randomUUID(), + requestFingerprint: "c".repeat(64), + preview: preview(actorId), + }, + fixture.db, + ); + await trigger(); + expect( + await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.id, value.previewId)), + ).toHaveLength(1); + await releasePreservation({ id: hold!.id, database: fixture.db }); + await trigger(); + expect( + await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.id, value.previewId)), + ).toHaveLength(0); + }); + it("keeps the original job while a closed platform preview is still preserved", async () => { + const actorId = await actor(), + value = preview(actorId), + jobId = randomUUID(), + created = new Date(Date.now() - 5 * 86400000), + closed = new Date(Date.now() - 4 * 86400000); + await retainPlatformPreview( + { + actorId, + operationKey: randomUUID(), + requestFingerprint: "b".repeat(64), + preview: value, + now: created, + }, + fixture.db, + ); + await fixture.db.insert(adminJob).values({ + id: jobId, + type: "platform.apply", + createdBy: actorId, + payload: {}, + result: {}, + status: "success", + createdAt: created, + finishedAt: closed, + }); + await fixture.db + .update(platformConfigurationPreview) + .set({ + consumedByJobId: jobId, + consumedAt: new Date(created.getTime() + 60000), + retiredAt: closed, + }) + .where(eq(platformConfigurationPreview.id, value.previewId)); + const { PrivacyRequestService } = await import("../../privacy/request-service"), + { loadMasterKeyRing } = await import("../../privacy/crypto"), + { dataSubjectRequestPreservation } = await import("../../db/schema"), + { releasePreservation } = await import("../../privacy/preservation"), + { pruneCompletedJobs } = await import("../activity-retention"); + const service = new PrivacyRequestService({ + database: fixture.db, + keyRing: loadMasterKeyRing({ + env: { + NODE_ENV: "development", + OPENMAPX_EXPORTS_KEY: Buffer.alloc(32, 9).toString("base64url"), + }, + }), + }), + request = await service.create({ + userId: actorId, + kind: "access", + channel: "self_service", + locale: "en", + timeZone: "UTC", + }); + const holds = await fixture.db + .select() + .from(dataSubjectRequestPreservation) + .where(eq(dataSubjectRequestPreservation.requestId, request.id)); + const jobHold = holds.find((row) => row.registrationId === "admin-jobs-attribution")!, + previewHold = holds.find((row) => row.registrationId === "platform-configuration-previews")!; + await releasePreservation({ id: jobHold.id, database: fixture.db }); + await pruneCompletedJobs(1); + expect(await fixture.db.select().from(adminJob).where(eq(adminJob.id, jobId))).toHaveLength(1); + expect( + await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.id, value.previewId)), + ).toHaveLength(1); + await releasePreservation({ id: previewHold.id, database: fixture.db }); + await pruneCompletedJobs(1); + expect( + await fixture.db + .select() + .from(platformConfigurationPreview) + .where(eq(platformConfigurationPreview.id, value.previewId)), + ).toHaveLength(0); + expect(await fixture.db.select().from(adminJob).where(eq(adminJob.id, jobId))).toHaveLength(0); + }); + it("exports only the subject's cutoff-bound platform preview metadata, never private intent", async () => { + const actorId = await actor(), + otherId = await actor(), + now = new Date("2025-01-01T00:00:00Z"), + own = preview(actorId), + foreign = preview(otherId), + future = preview(actorId), + privateSentinel = "private-purpose-proof-must-never-export"; + for (const [who, value, time] of [ + [actorId, own, now], + [otherId, foreign, now], + [actorId, future, new Date(now.getTime() + 60000)], + ] as const) + await retainPlatformPreview( + { + actorId: who, + operationKey: randomUUID(), + requestFingerprint: "f".repeat(64), + preview: value, + now: time, + }, + fixture.db, + ); + await fixture.db + .update(platformConfigurationPreview) + .set({ preview: { ...own, privatePurposeProof: privateSentinel } }) + .where(eq(platformConfigurationPreview.id, own.previewId)); + const { collectOpenMapxRegistration } = await import("../../privacy/openmapx-collectors"), + part = await collectOpenMapxRegistration("platform-configuration-previews", { + userId: actorId, + cutoffAt: new Date(now.getTime() + 30000), + database: fixture.db, + }); + expect(part.records).toHaveLength(1); + expect(part.records[0]?.data).toMatchObject({ + createdAt: now.toISOString(), + expiresAt: new Date(now.getTime() + 600000).toISOString(), + consumedAt: null, + retiredAt: null, + }); + expect(JSON.stringify(part)).not.toContain(privateSentinel); + for (const value of [ + own.previewId, + foreign.previewId, + future.previewId, + own.previewFingerprint, + own.authPolicyRevision, + own.newOrigin, + ]) + expect(JSON.stringify(part)).not.toContain(value); + expect(Object.keys(part.records[0]!.data).sort()).toEqual([ + "consumedAt", + "createdAt", + "expiresAt", + "previewDigest", + "retiredAt", + ]); + }); it("retains one original request and rejects changed intent, actor substitution and expiry", async () => { const actorId = await actor(), operationKey = randomUUID(), diff --git a/apps/api/src/services/activity-retention.ts b/apps/api/src/services/activity-retention.ts index 0c0f8f283..ee0cc61fc 100644 --- a/apps/api/src/services/activity-retention.ts +++ b/apps/api/src/services/activity-retention.ts @@ -1,11 +1,18 @@ import { and, lt, sql } from "drizzle-orm"; import { db } from "../db"; -import { adminAuditLog, adminJob, appLog, verification } from "../db/schema"; +import { + adminAuditLog, + adminJob, + appLog, + platformConfigurationPreview, + verification, +} from "../db/schema"; import { preservationAllowsDeletion, withPreservationRetentionLock, } from "../privacy/preservation.js"; import { noPendingReceiptDelivery, noUnsettledManagedOperation } from "./admin-job-state"; +import { prunePlatformConfigurationPreviews } from "./platform-preview-store"; import { pruneTransitCredentialPreviews } from "./transit-credential-retention"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -70,6 +77,7 @@ export async function pruneCompletedJobs(days: number): Promise { // adminJobLog has ON DELETE CASCADE so deleting parent rows clears logs. const result = await withPreservationRetentionLock(db, async (tx) => { await pruneTransitCredentialPreviews(tx); + await prunePlatformConfigurationPreviews(tx); return tx .delete(adminJob) .where( @@ -77,6 +85,7 @@ export async function pruneCompletedJobs(days: number): Promise { lt(adminJob.finishedAt, cutoff), noPendingReceiptDelivery(), noUnsettledManagedOperation(), + sql`NOT EXISTS (SELECT 1 FROM ${platformConfigurationPreview} WHERE ${platformConfigurationPreview.consumedByJobId}::text = ${adminJob.id})`, sql`(coalesce(${adminJob.result}->'opsProjection'->>'version','') <> '3' OR ${adminJob.finishedAt} < ${new Date(Date.now() - 72 * 60 * 60 * 1000).toISOString()}::timestamp)`, preservationAllowsDeletion({ registrationId: "admin-jobs-attribution", diff --git a/apps/api/src/services/platform-preview-store.ts b/apps/api/src/services/platform-preview-store.ts index 8c760bb3f..26bad72d0 100644 --- a/apps/api/src/services/platform-preview-store.ts +++ b/apps/api/src/services/platform-preview-store.ts @@ -6,12 +6,13 @@ import { canonicalJson, configurationSourcesInspectionSchema, } from "@openmapx/core/ops"; -import { and, eq, isNull, lt, sql } from "drizzle-orm"; +import { and, eq, isNotNull, isNull, lt, or, sql } from "drizzle-orm"; import { z } from "zod"; import type { AuthDatabase } from "../account/auth-database"; import { db } from "../db"; -import { platformConfigurationPreview as table } from "../db/schema"; +import { adminJob, platformConfigurationPreview as table } from "../db/schema"; import { preservationAllowsDeletion, withPreservationRetentionLock } from "../privacy/preservation"; +import { noPendingReceiptDelivery, noUnsettledManagedOperation } from "./admin-job-state"; const TTL = 600_000; const storedPreviewSchema = z.strictObject({ @@ -67,6 +68,42 @@ function decode(row: typeof table.$inferSelect) { throw new PlatformPreviewConflict(); } } +/** Caller holds the shared preservation-retention lock. Unknown/running work + * is pinned; a completed preview and its exact job share the later 72h floor. */ +export async function prunePlatformConfigurationPreviews( + connection: Pick, + now = new Date(), +): Promise { + const rows = connection + .delete(table) + .where( + and( + or( + and(isNull(table.consumedByJobId), lt(table.expiresAt, now)), + and( + isNotNull(table.retiredAt), + sql`exists ( + select 1 from ${adminJob} + where ${adminJob.id} = ${table.consumedByJobId}::text + and ${adminJob.status} in ('success','failed','canceled') + and ${adminJob.finishedAt} is not null + and greatest(${table.retiredAt},${adminJob.finishedAt}) < ${new Date(now.getTime() - 72 * 3600000).toISOString()}::timestamptz + and ${noPendingReceiptDelivery()} + and ${noUnsettledManagedOperation()} + )`, + ), + ), + preservationAllowsDeletion({ + registrationId: "platform-configuration-previews", + candidateTimestamp: table.createdAt, + subjectIds: [table.actorId], + }), + ), + ) + .returning({ id: table.id }); + return (await rows).length; +} + export async function retainPlatformPreview( input: { actorId: string; @@ -103,18 +140,7 @@ export async function retainPlatformPreview( throw new PlatformPreviewConflict("Platform preview expired; create a new preview"); return decode(existing); } - // Unconsumed previews never authorized effects; consumed/unknown work stays pinned. - await tx.delete(table).where( - and( - isNull(table.consumedByJobId), - lt(table.expiresAt, now), - preservationAllowsDeletion({ - registrationId: "admin-jobs-attribution", - candidateTimestamp: table.createdAt, - subjectIds: [table.actorId], - }), - ), - ); + await prunePlatformConfigurationPreviews(tx, now); const capacity = await tx.select({ id: table.id }).from(table).limit(4096); if (capacity.length >= 4096) throw new PlatformPreviewConflict("Platform preview capacity unavailable"); diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index a7b84da13..7453c4341 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -1091,6 +1091,17 @@ export async function assertSettledConfigurationCandidate( reload.revisionId !== candidate.revisionId ) throw Error("Configuration reload unavailable"); + } else if (state.record.privatePlatformIntent) { + const followUp = result.platformConfiguration as Record | undefined, + fixed = state.record.privatePlatformIntent; + if ( + state.record.apiFollowUp !== "platform-apply" || + followUp?.previewFingerprint !== fixed.previewFingerprint || + followUp.handoffId !== fixed.handoffId || + followUp.revisionId !== candidate.revisionId || + followUp.followUp !== (contained ? "contained" : "complete") + ) + throw Error("Original platform completion required"); } else if (state.record.privateCoordinatorLifecycle) { const followUp = result.coordinatorLifecycle as Record | undefined, fixed = state.record.privateCoordinatorLifecycle; @@ -1110,12 +1121,49 @@ export async function retireSettledProspectiveConfiguration( await assertSettledConfigurationCandidate(candidate); const state = validatedConfigurationCandidateState(candidate), { job } = administrativeConfigurationState(state.writer); + if (state.record.privatePlatformIntent) { + const fixed = state.record.privatePlatformIntent; + // This function has already verified the exact remote final/SQL/ACK/root + // closure and API durable copy. A status flag cannot retire this preview. + await withConfigurationWriteTransaction(state.writer, async (tx) => { + const [current] = await tx + .select({ result: schema.adminJob.result }) + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + proof = (current?.result as Record | undefined)?._configurationCandidate as + | Record + | undefined; + if ( + proof?.candidateId !== candidate.candidateId || + proof.revisionId !== candidate.revisionId || + proof.phase !== "settled" + ) + throw new JobObservationPending("recovery_required"); + const [retired] = await tx + .update(schema.platformConfigurationPreview) + .set({ + retiredAt: sql`coalesce(${schema.platformConfigurationPreview.retiredAt},greatest(${schema.platformConfigurationPreview.consumedAt},${new Date().toISOString()}::timestamptz))`, + }) + .where( + and( + eq(schema.platformConfigurationPreview.id, fixed.previewId), + eq(schema.platformConfigurationPreview.actorId, state.writer.actorId), + eq(schema.platformConfigurationPreview.previewFingerprint, fixed.previewFingerprint), + eq(schema.platformConfigurationPreview.consumedByJobId, job.jobId), + ), + ) + .returning({ id: schema.platformConfigurationPreview.id }); + if (!retired) throw new JobObservationPending("recovery_required"); + }); + } const queuedCoordinator = !!state.record.privateCoordinatorLifecycle && ((job.jobType === "service.apply" && job.payload.service === "data-manager") || (job.jobType === "service.bulk" && job.payload.action === "update")); if ( queuedCoordinator || + state.record.privatePlatformIntent || (job.checkpointResult?._immediateJobId === job.jobId && [ "service.apply", @@ -1134,6 +1182,9 @@ export async function retireSettledProspectiveConfiguration( ...(original.coordinatorLifecycle ? { coordinatorLifecycle: original.coordinatorLifecycle } : {}), + ...(original.platformConfiguration + ? { platformConfiguration: original.platformConfiguration } + : {}), ...(original.integrationReload ? { integrationReload: original.integrationReload } : {}), ...(original.integrationBinding ? { integrationBinding: original.integrationBinding } : {}), ...(original.integrationApplication @@ -4553,6 +4604,16 @@ export async function persistAndApplyProspectiveConfiguration( }, } : {}), + ...(state.record.privatePlatformIntent + ? { + platformConfiguration: { + previewFingerprint: state.record.privatePlatformIntent.previewFingerprint, + handoffId: state.record.privatePlatformIntent.handoffId, + revisionId: candidate.revisionId, + followUp: contained ? "contained" : "complete", + }, + } + : {}), }); if (!registryUpdateFailed) await retireSettledProspectiveConfiguration(candidate); return { diff --git a/packages/i18n/locales/de.json b/packages/i18n/locales/de.json index 9ee4d1dd6..5bf26f7f3 100644 --- a/packages/i18n/locales/de.json +++ b/packages/i18n/locales/de.json @@ -3773,6 +3773,15 @@ }, "catalogue": { "entries": { + "platform-configuration-previews": { + "category": "Metadaten zur Vorschau der Plattformkonfiguration", + "purpose": "Eine Konfigurationsänderung einem Administratorkonto zuordnen und sicher wiederherstellen", + "source": "Von diesem Konto erstellte Plattformvorschauen", + "retention": "Unbenutzte Vorschauen laufen nach zehn Minuten ab. Aktive oder unklare Änderungen bleiben für die Wiederherstellung erhalten. Abgeschlossene Datensätze können nach Abschluss des Auftrags und seiner Belege sowie nach Ablauf der Aufbewahrungsfrist gelöscht werden. Eine Kontolöschung entfernt freigegebene Zuordnungsdaten.", + "safeRepresentation": "Nur Zeitangaben und eine pseudonyme Vorschaureferenz. Konfigurationswerte, Nachweise, Host-Fingerabdrücke und Auftragsreferenzen werden zurückgehalten.", + "portabilityExclusion": "Betriebliche Konfigurationsmetadaten sind keine übertragbaren Kontoinhalte", + "rightsOfOthers": "Konfigurationswerte und Angaben über andere Personen erfordern eine Prüfung durch den Betreiber" + }, "account-profile": { "category": "Kontoprofil", "purpose": "Ihr OpenMapX-Konto bereitstellen und schützen", diff --git a/packages/i18n/locales/en.json b/packages/i18n/locales/en.json index 1aaebe038..9e6b7cabd 100644 --- a/packages/i18n/locales/en.json +++ b/packages/i18n/locales/en.json @@ -3773,6 +3773,15 @@ }, "catalogue": { "entries": { + "platform-configuration-previews": { + "category": "Platform configuration preview metadata", + "purpose": "Attribute and safely recover an administrator's configuration change", + "source": "Platform previews created by this account", + "retention": "Unused previews expire after ten minutes. Active or uncertain changes remain held for recovery; completed records are eligible for deletion after the job and its receipts close and the retention period passes. Account deletion removes eligible attribution records.", + "safeRepresentation": "Dates and a pseudonymous preview reference only. Configuration values, proof material, host-source fingerprints and job references are withheld.", + "portabilityExclusion": "Operational configuration metadata is not designated as portable account content", + "rightsOfOthers": "Configuration values and information about other people require operator review" + }, "account-profile": { "category": "Account profile", "purpose": "Provide and secure your OpenMapX account", diff --git a/scripts/check-subject-data.ts b/scripts/check-subject-data.ts index 3162ace23..65ff7a153 100644 --- a/scripts/check-subject-data.ts +++ b/scripts/check-subject-data.ts @@ -32,6 +32,7 @@ export const DIRECT_USER_FK_CLASSIFICATIONS: Record = { "oauth_consent.user_id": "auth-oauth-resources", "oauth_refresh_token.user_id": "auth-oauth-resources", "parked_location.user_id": "parked-locations", + "platform_configuration_preview.actor_id": "platform-configuration-previews", "passkey.user_id": "auth-passkeys", "personal_timeline_connection.user_id": "timeline-connections", "personal_vehicle.user_id": "personal-vehicles", From 147a38a4103d40a22b5e0d781bc04635e73907d9 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 07:48:57 +0100 Subject: [PATCH 072/135] feat(admin): expose actor-bound platform configuration previews --- apps/api/openapi.json | 2078 +++++++++++++++++ .../routes/__tests__/admin-platform.test.ts | 138 ++ apps/api/src/routes/admin-platform.ts | 73 + apps/api/src/routes/index.ts | 2 + .../platform-preview-postgres.test.ts | 55 + .../services/platform-configuration-view.ts | 88 + .../src/api/contracts/service-admin.test.ts | 10 +- .../src/api/contracts/service-admin/index.ts | 3 + .../api/contracts/service-admin/platform.ts | 39 + .../admin/operation-bindings.generated.ts | 16 + .../admin/service-admin.generated.ts | 216 ++ .../getAdminPlatformConfiguration200.ts | 12 + ...inPlatformConfiguration200Configuration.ts | 17 + ...mConfiguration200ConfigurationAcmeEmail.ts | 15 + ...tion200ConfigurationBackupRetentionDays.ts | 15 + ...figuration200ConfigurationDefaultRegion.ts | 16 + ...formConfiguration200ConfigurationDomain.ts | 12 + ...guration200ConfigurationReleaseSelector.ts | 12 + ...ormConfiguration200ConfigurationVersion.ts | 9 + ...getAdminPlatformConfiguration200Sources.ts | 15 + ...latformConfiguration200SourcesAcmeEmail.ts | 10 + ...figuration200SourcesBackupRetentionDays.ts | 10 + ...ormConfiguration200SourcesDefaultRegion.ts | 10 + ...inPlatformConfiguration200SourcesDomain.ts | 10 + ...mConfiguration200SourcesReleaseSelector.ts | 10 + .../getAdminPlatformConfiguration200Values.ts | 26 + .../getAdminPlatformConfiguration400.ts | 20 + .../getAdminPlatformConfiguration401.ts | 20 + .../getAdminPlatformConfiguration403.ts | 20 + .../getAdminPlatformConfiguration413.ts | 20 + .../getAdminPlatformConfiguration415.ts | 20 + .../getAdminPlatformConfiguration429.ts | 11 + .../getAdminPlatformConfiguration500.ts | 20 + .../getAdminPlatformConfiguration503.ts | 20 + .../generated/admin/service-models/index.ts | 60 + .../previewAdminPlatformConfiguration200.ts | 9 + ...iewAdminPlatformConfiguration200Preview.ts | 46 + ...ormConfiguration200PreviewConfiguration.ts | 17 + ...uration200PreviewConfigurationAcmeEmail.ts | 15 + ...PreviewConfigurationBackupRetentionDays.ts | 15 + ...ion200PreviewConfigurationDefaultRegion.ts | 16 + ...figuration200PreviewConfigurationDomain.ts | 12 + ...n200PreviewConfigurationReleaseSelector.ts | 12 + ...iguration200PreviewConfigurationVersion.ts | 9 + ...mConfiguration200PreviewOriginConsumers.ts | 25 + ...iguration200PreviewRequiredConfirmation.ts | 10 + ...nPlatformConfiguration200PreviewSources.ts | 15 + ...Configuration200PreviewSourcesAcmeEmail.ts | 10 + ...ion200PreviewSourcesBackupRetentionDays.ts | 10 + ...iguration200PreviewSourcesDefaultRegion.ts | 10 + ...ormConfiguration200PreviewSourcesDomain.ts | 10 + ...uration200PreviewSourcesReleaseSelector.ts | 10 + ...inPlatformConfiguration200PreviewValues.ts | 26 + ...nPlatformConfiguration200PreviewVersion.ts | 9 + .../previewAdminPlatformConfiguration400.ts | 20 + .../previewAdminPlatformConfiguration401.ts | 20 + .../previewAdminPlatformConfiguration403.ts | 20 + .../previewAdminPlatformConfiguration409.ts | 20 + .../previewAdminPlatformConfiguration413.ts | 20 + .../previewAdminPlatformConfiguration415.ts | 20 + .../previewAdminPlatformConfiguration429.ts | 11 + .../previewAdminPlatformConfiguration500.ts | 20 + .../previewAdminPlatformConfiguration503.ts | 20 + .../previewAdminPlatformConfigurationBody.ts | 8 + ...ewAdminPlatformConfigurationBodyChanges.ts | 17 + ...atformConfigurationBodyChangesAcmeEmail.ts | 15 + ...igurationBodyChangesBackupRetentionDays.ts | 15 + ...rmConfigurationBodyChangesDefaultRegion.ts | 16 + ...nPlatformConfigurationBodyChangesDomain.ts | 12 + ...ConfigurationBodyChangesReleaseSelector.ts | 12 + ...PlatformConfigurationBodyChangesVersion.ts | 9 + ...reviewAdminPlatformConfigurationHeaders.ts | 9 + 72 files changed, 3697 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/routes/__tests__/admin-platform.test.ts create mode 100644 apps/api/src/routes/admin-platform.ts create mode 100644 apps/api/src/services/platform-configuration-view.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Configuration.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationAcmeEmail.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationBackupRetentionDays.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDefaultRegion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDomain.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationReleaseSelector.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationVersion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Sources.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesAcmeEmail.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesBackupRetentionDays.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDefaultRegion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDomain.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesReleaseSelector.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Values.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfiguration.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDomain.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationVersion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewOriginConsumers.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRequiredConfirmation.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSources.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDefaultRegion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDomain.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesReleaseSelector.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewValues.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewVersion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChanges.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesAcmeEmail.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesBackupRetentionDays.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDefaultRegion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDomain.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesReleaseSelector.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesVersion.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationHeaders.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index d690bc403..242957a61 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -49548,6 +49548,2084 @@ "x-openmapx-auth": "admin" } }, + "/api/admin/platform/configuration": { + "get": { + "operationId": "getAdminPlatformConfiguration", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "configuration": { + "type": "object", + "properties": { + "version": { + "type": "number", + "enum": [ + 1 + ] + }, + "domain": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 253 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "acmeEmail": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 320, + "format": "email", + "pattern": "^(?:[A-Za-z0-9_'+\\-]+\\.)*[A-Za-z0-9_'+\\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$" + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "releaseSelector": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 128 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "defaultRegion": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9_-]*(?:\\/[a-z0-9][a-z0-9_-]*)*$" + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "backupRetentionDays": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "integer", + "minimum": 1, + "maximum": 36500 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "version" + ], + "additionalProperties": false + }, + "values": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "minLength": 1, + "maxLength": 253 + }, + "acmeEmail": { + "type": "string", + "maxLength": 320, + "format": "email", + "pattern": "^(?:[A-Za-z0-9_'+\\-]+\\.)*[A-Za-z0-9_'+\\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$" + }, + "releaseSelector": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "defaultRegion": { + "anyOf": [ + { + "type": "string", + "maxLength": 128 + }, + { + "type": "null" + } + ] + }, + "backupRetentionDays": { + "type": "integer", + "minimum": 1, + "maximum": 36500 + } + }, + "required": [ + "domain", + "acmeEmail", + "releaseSelector", + "defaultRegion", + "backupRetentionDays" + ], + "additionalProperties": false + }, + "sources": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "acmeEmail": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "releaseSelector": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "defaultRegion": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "backupRetentionDays": { + "type": "string", + "enum": [ + "host", + "database" + ] + } + }, + "required": [ + "domain", + "acmeEmail", + "releaseSelector", + "defaultRegion", + "backupRetentionDays" + ], + "additionalProperties": false + } + }, + "required": [ + "expectedCurrentRevisionId", + "configuration", + "values", + "sources" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, + "/api/admin/platform/configuration/preview": { + "post": { + "operationId": "previewAdminPlatformConfiguration", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "changes": { + "type": "object", + "properties": { + "version": { + "type": "number", + "enum": [ + 1 + ] + }, + "domain": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 253 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "acmeEmail": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 320, + "format": "email", + "pattern": "^(?:[A-Za-z0-9_'+\\-]+\\.)*[A-Za-z0-9_'+\\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$" + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "releaseSelector": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 128 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "defaultRegion": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9_-]*(?:\\/[a-z0-9][a-z0-9_-]*)*$" + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "backupRetentionDays": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "integer", + "minimum": 1, + "maximum": 36500 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "version" + ], + "additionalProperties": false + } + }, + "required": [ + "expectedCurrentRevisionId", + "changes" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "preview": { + "type": "object", + "properties": { + "version": { + "type": "number", + "enum": [ + 1 + ] + }, + "actorId": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "previewId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "authPolicyRevision": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceRevision": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "configuration": { + "type": "object", + "properties": { + "version": { + "type": "number", + "enum": [ + 1 + ] + }, + "domain": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 253 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "acmeEmail": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 320, + "format": "email", + "pattern": "^(?:[A-Za-z0-9_'+\\-]+\\.)*[A-Za-z0-9_'+\\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$" + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "releaseSelector": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "maxLength": 128 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "defaultRegion": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9_-]*(?:\\/[a-z0-9][a-z0-9_-]*)*$" + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + }, + "backupRetentionDays": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "host" + ] + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "database" + ] + }, + "value": { + "type": "integer", + "minimum": 1, + "maximum": 36500 + } + }, + "required": [ + "source", + "value" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "version" + ], + "additionalProperties": false + }, + "values": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "minLength": 1, + "maxLength": 253 + }, + "acmeEmail": { + "type": "string", + "maxLength": 320, + "format": "email", + "pattern": "^(?:[A-Za-z0-9_'+\\-]+\\.)*[A-Za-z0-9_'+\\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$" + }, + "releaseSelector": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "defaultRegion": { + "anyOf": [ + { + "type": "string", + "maxLength": 128 + }, + { + "type": "null" + } + ] + }, + "backupRetentionDays": { + "type": "integer", + "minimum": 1, + "maximum": 36500 + } + }, + "required": [ + "domain", + "acmeEmail", + "releaseSelector", + "defaultRegion", + "backupRetentionDays" + ], + "additionalProperties": false + }, + "sources": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "acmeEmail": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "releaseSelector": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "defaultRegion": { + "type": "string", + "enum": [ + "host", + "database" + ] + }, + "backupRetentionDays": { + "type": "string", + "enum": [ + "host", + "database" + ] + } + }, + "required": [ + "domain", + "acmeEmail", + "releaseSelector", + "defaultRegion", + "backupRetentionDays" + ], + "additionalProperties": false + }, + "affectedServiceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "oldOrigin": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "newOrigin": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "originConsumers": { + "type": "object", + "properties": { + "routingHost": { + "type": "string", + "minLength": 1, + "maxLength": 253 + }, + "corsOrigin": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "authUrl": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "passkeyRpId": { + "type": "string", + "minLength": 1, + "maxLength": 253 + }, + "passkeyOrigin": { + "type": "string", + "maxLength": 2048 + }, + "webApiUrl": { + "type": "string", + "maxLength": 2048, + "format": "uri" + }, + "tilesPublicUrl": { + "type": "string", + "maxLength": 2048, + "format": "uri" + } + }, + "required": [ + "routingHost", + "corsOrigin", + "authUrl", + "passkeyRpId", + "passkeyOrigin", + "webApiUrl", + "tilesPublicUrl" + ], + "additionalProperties": false + }, + "requiredConfirmation": { + "type": "object", + "properties": { + "domain": { + "type": "boolean" + }, + "dnsTls": { + "type": "boolean" + }, + "oauth": { + "type": "boolean" + }, + "passkeys": { + "type": "boolean" + }, + "alternateLogin": { + "type": "boolean" + } + }, + "required": [ + "domain", + "dnsTls", + "oauth", + "passkeys", + "alternateLogin" + ], + "additionalProperties": false + }, + "warnings": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "maxLength": 512 + } + }, + "previewFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "version", + "actorId", + "previewId", + "authPolicyRevision", + "sourceRevision", + "expectedCurrentRevisionId", + "configuration", + "values", + "sources", + "affectedServiceIds", + "oldOrigin", + "newOrigin", + "originConsumers", + "requiredConfirmation", + "warnings", + "previewFingerprint" + ], + "additionalProperties": false + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "preview", + "expiresAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/admin/reauth/complete": { "post": { "operationId": "completeAdminReauthentication", diff --git a/apps/api/src/routes/__tests__/admin-platform.test.ts b/apps/api/src/routes/__tests__/admin-platform.test.ts new file mode 100644 index 000000000..09681d1e1 --- /dev/null +++ b/apps/api/src/routes/__tests__/admin-platform.test.ts @@ -0,0 +1,138 @@ +import { randomUUID } from "node:crypto"; +import { createApiClient } from "@openmapx/core"; +import { + getAdminPlatformConfiguration, + previewAdminPlatformConfiguration, + unwrapAdminResponse, +} from "@openmapx/core/api-admin-sdk"; +import { services } from "@openmapx/core/server"; +import Fastify, { type FastifyInstance } from "fastify"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { mockAdminSession } from "../../test/auth"; + +const mocks = vi.hoisted(() => ({ + session: null as unknown, + inspect: vi.fn(), + preview: vi.fn(), + apply: vi.fn(), +})); +vi.mock("../../auth", () => ({ auth: { api: { getSession: async () => mocks.session } } })); +vi.mock("../../services/platform-configuration-view", () => ({ + inspectPlatformConfiguration: mocks.inspect, + previewPlatformConfiguration: mocks.preview, +})); +vi.mock("../../services/job-runner", () => ({ + jobRunner: { startPlatformConfiguration: mocks.apply }, +})); + +import { adminPlatformRoute } from "../admin-platform"; + +let app: FastifyInstance; +let client: ReturnType; +const request = { expectedCurrentRevisionId: null, changes: { version: 1 as const } }; +let preview: ReturnType; +beforeEach(async () => { + vi.stubEnv("OPENMAPX_DISABLE_LOCALHOST_AUTH", "1"); + vi.clearAllMocks(); + mocks.session = mockAdminSession(); + preview = services.buildPlatformConfigurationPreview({ + actorId: "test-admin-id", + previewId: randomUUID(), + authPolicyRevision: "a".repeat(64), + sourceRevision: randomUUID(), + currentRevisionId: null, + expectedCurrentRevisionId: null, + changes: { version: 1 }, + env: {}, + services: [], + }); + mocks.inspect.mockResolvedValue({ + expectedCurrentRevisionId: null, + configuration: { version: 1 }, + values: preview.values, + sources: preview.sources, + }); + mocks.preview.mockResolvedValue({ + preview, + expiresAt: new Date(Date.now() + 600000).toISOString(), + }); + mocks.apply.mockResolvedValue({ ok: true, jobId: randomUUID() }); + app = Fastify(); + await app.register(adminPlatformRoute, { prefix: "/api" }); + client = createApiClient({ baseUrl: await app.listen({ host: "127.0.0.1", port: 0 }) }); +}); +afterEach(async () => { + await app.close(); + vi.unstubAllEnvs(); +}); +it("serves an administrator's effective configuration and retains a reviewed preview", async () => { + expect(unwrapAdminResponse(await getAdminPlatformConfiguration({ client }))).toMatchObject({ + values: preview.values, + }); + expect( + unwrapAdminResponse( + await previewAdminPlatformConfiguration( + request, + { "idempotency-key": "platform-preview-sdk-01" }, + { client }, + ), + ), + ).toMatchObject({ preview }); + const current = await app.inject({ url: "/api/admin/platform/configuration" }); + expect(current.statusCode).toBe(200); + expect(current.json()).toMatchObject({ configuration: { version: 1 }, values: preview.values }); + expect(current.headers["cache-control"]).toBe("private, no-store"); + const result = await app.inject({ + method: "POST", + url: "/api/admin/platform/configuration/preview", + headers: { "idempotency-key": "platform-preview-key-01" }, + payload: request, + }); + expect(result.statusCode).toBe(200); + expect(result.json().preview.previewFingerprint).toBe(preview.previewFingerprint); + expect(mocks.preview).toHaveBeenCalledWith( + expect.objectContaining({ actorId: "test-admin-id", ...request }), + ); +}); +it.each([ + null, + { ...mockAdminSession(), user: { ...mockAdminSession().user, role: "support_admin" } }, +])("requires the actual full-administrator guard", async (session) => { + mocks.session = session; + const result = await app.inject({ url: "/api/admin/platform/configuration" }); + expect(result.statusCode).toBe(session ? 403 : 401); + expect(mocks.inspect).not.toHaveBeenCalled(); +}); +it.each(["loopback", "impersonated"])("refuses %s preview authority", async (kind) => { + const actor = mockAdminSession(); + if (kind === "loopback") actor.user.id = "loopback"; + else actor.session.impersonatedBy = "other"; + mocks.session = actor; + const result = await app.inject({ + method: "POST", + url: "/api/admin/platform/configuration/preview", + headers: { "idempotency-key": "platform-preview-key-01" }, + payload: request, + }); + expect(result.statusCode).toBe(403); + expect(mocks.preview).not.toHaveBeenCalled(); +}); +it("rejects unknown input and missing mutation identity before invoking a producer", async () => { + for (const payload of [request, { ...request, changes: { version: 1, arbitrary: true } }]) { + const result = await app.inject({ + method: "POST", + url: "/api/admin/platform/configuration/preview", + ...(payload === request ? {} : { headers: { "idempotency-key": "platform-preview-key-01" } }), + payload, + }); + expect(result.statusCode).toBe(400); + } + expect(mocks.preview).not.toHaveBeenCalled(); +}); +it("keeps host failure details out of public responses", async () => { + mocks.inspect.mockRejectedValue(Error("/private/host/path secret-value")); + const result = await app.inject({ url: "/api/admin/platform/configuration" }); + expect(result.statusCode).toBe(503); + expect(result.body).not.toContain("secret-value"); + expect(result.body).not.toContain("/private"); +}); diff --git a/apps/api/src/routes/admin-platform.ts b/apps/api/src/routes/admin-platform.ts new file mode 100644 index 000000000..2f3468a40 --- /dev/null +++ b/apps/api/src/routes/admin-platform.ts @@ -0,0 +1,73 @@ +import type { FastifyInstance } from "fastify"; +import { AdminReauthenticationError } from "../account/admin-reauthentication"; +import { + createDirectAdminOpsKey, + parseDirectOpsIdempotency, +} from "../services/direct-ops-idempotency"; +import { + inspectPlatformConfiguration, + previewPlatformConfiguration, +} from "../services/platform-configuration-view"; +import { PlatformPreviewConflict } from "../services/platform-preview-store"; +import { RateLimiter } from "../utils/rate-limit"; +import { getAdminSession, requireAdmin } from "../utils/require-admin"; +import { declareRouteAuth } from "../utils/route-auth"; +import { registerServiceAdminRoute } from "./admin-service-contract"; + +export async function adminPlatformRoute(app: FastifyInstance) { + declareRouteAuth(app, "admin"); + app.addHook("preHandler", async (request) => { + request.adminSession = await requireAdmin(request); + }); + app.addHook("onRequest", async (_request, reply) => { + reply.header("Cache-Control", "private, no-store"); + }); + const limits = new RateLimiter({ + max: 12, + windowMs: 60000, + keyFn: (req) => getAdminSession(req).user.id, + }); + app.addHook("onClose", async () => limits.destroy()); + const failure = (error: unknown) => + error instanceof PlatformPreviewConflict || error instanceof AdminReauthenticationError + ? { + status: 409, + error: "Platform preview, source or authorization changed; create a new preview", + } + : { + status: 503, + error: + "Platform configuration is unavailable; inspect Activity and host recovery guidance", + }; + registerServiceAdminRoute(app, "getAdminPlatformConfiguration", async (_request, reply) => { + try { + return await inspectPlatformConfiguration(); + } catch (error) { + const value = failure(error); + return reply.code(value.status).send({ error: value.error }); + } + }); + registerServiceAdminRoute( + app, + "previewAdminPlatformConfiguration", + async (request, reply) => { + const actor = getAdminSession(request); + if (actor.user.id === "loopback" || actor.session.impersonatedBy) + return reply.code(403).send({ error: "A real full administrator session is required" }); + try { + return await previewPlatformConfiguration({ + ...request.body, + actorId: actor.user.id, + operationKey: createDirectAdminOpsKey( + actor.user.id, + parseDirectOpsIdempotency(request.headers["idempotency-key"]), + ), + }); + } catch (error) { + const value = failure(error); + return reply.code(value.status).send({ error: value.error }); + } + }, + { preHandler: limits.preHandler() }, + ); +} diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index 289aa4c00..4612c46a7 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -11,6 +11,7 @@ import { adminDawarichRoute } from "./admin-dawarich"; import { adminExtensionsRoute } from "./admin-extensions"; import { adminJobEventsRoute } from "./admin-job-events"; import { adminOperationsRoute } from "./admin-operations"; +import { adminPlatformRoute } from "./admin-platform"; import { adminReauthenticationRoute } from "./admin-reauthentication"; import { adminServicesRoute } from "./admin-services"; import { adminSettingsRoute } from "./admin-settings"; @@ -121,6 +122,7 @@ export async function registerCoreRoutes( await server.register(statusRoute, { prefix: "/api" }); await server.register(adminRoute, { prefix: "/api" }); await server.register(adminServicesRoute, { prefix: "/api" }); + await server.register(adminPlatformRoute, { prefix: "/api" }); await server.register(adminArtifactTransfersRoute, { prefix: "/api" }); await server.register(adminTransitCredentialsRoute, { prefix: "/api" }); await server.register(adminTransitPinsRoute, { prefix: "/api" }); diff --git a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts index 749247b1e..06ecb0590 100644 --- a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts @@ -15,6 +15,15 @@ import { createMigratedPostgresFixture } from "../../test/postgres-fixture"; const connection = vi.hoisted(() => ({ fixture: null as unknown as Awaited>, + source: { revisionId: null as string | null, sourceRevision: "", services: [] }, +})); +vi.mock("../ops-client", async (original) => ({ + ...(await original()), + createApiOpsClient: () => ({}), + executeAndWait: async (_client: unknown, operation: { kind: string }) => { + if (operation.kind !== "configuration.sources.inspect") throw Error("Unexpected effect"); + return connection.source; + }, })); vi.mock("../../db", () => ({ get db() { @@ -36,6 +45,7 @@ dbDescribe("durable exact platform preview", () => { beforeAll(async () => { fixture = await createMigratedPostgresFixture(); connection.fixture = fixture; + connection.source.sourceRevision = randomUUID(); vi.stubEnv("OPENMAPX_SECRETS_KEY", "41".repeat(32)); }, 90000); afterAll(async () => { @@ -66,6 +76,51 @@ dbDescribe("durable exact platform preview", () => { services: [], }); } + it("constructs and retains the public preview from current typed host and database sources", async () => { + const actorId = await actor(), + operationKey = randomUUID(); + const { inspectPlatformConfiguration, previewPlatformConfiguration } = await import( + "../platform-configuration-view" + ); + const current = await inspectPlatformConfiguration(); + expect(current).toMatchObject({ + expectedCurrentRevisionId: null, + configuration: { version: 1 }, + }); + const input = { + actorId, + operationKey, + expectedCurrentRevisionId: null, + changes: { + version: 1 as const, + backupRetentionDays: { source: "database" as const, value: 14 }, + }, + }; + const first = await previewPlatformConfiguration(input), + again = await previewPlatformConfiguration(input); + expect(first.preview.values.backupRetentionDays).toBe(14); + expect(again).toEqual(first); + const stored = await readPlatformPreview({ + actorId, + previewFingerprint: first.preview.previewFingerprint, + }); + expect(stored.sourceSnapshot).toEqual(connection.source); + expect(stored.desiredRowVersion).toBeNull(); + await expect( + previewPlatformConfiguration({ + ...input, + changes: { version: 1, backupRetentionDays: { source: "database", value: 15 } }, + }), + ).rejects.toThrow(); + expect(await fixture.db.select().from(systemSettings)).toHaveLength(0); + await expect( + previewPlatformConfiguration({ + ...input, + operationKey: randomUUID(), + expectedCurrentRevisionId: `cfg1_${"x".repeat(43)}`, + }), + ).rejects.toThrow("Platform preview unavailable or changed"); + }); it("reclaims only closed previews after both retirement and job completion floors", async () => { const actorId = await actor(), created = new Date("2025-02-01T00:00:00Z"), diff --git a/apps/api/src/services/platform-configuration-view.ts b/apps/api/src/services/platform-configuration-view.ts new file mode 100644 index 000000000..93bbd1c31 --- /dev/null +++ b/apps/api/src/services/platform-configuration-view.ts @@ -0,0 +1,88 @@ +/** Read-only acquisition plus immutable, actor-bound reviewed previews. */ +import { createHash, randomUUID } from "node:crypto"; +import { adminPlatformPreviewInputSchema } from "@openmapx/core/api-contracts"; +import { canonicalJson } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { eq, sql } from "drizzle-orm"; +import { loadAuthPolicy } from "../account/auth-policy"; +import { db } from "../db"; +import { systemSettings } from "../db/schema"; +import { withAdministrativeConfigurationRead } from "./administrative-configuration-writer-owner"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { PLATFORM_CONFIGURATION_SETTING } from "./platform-configuration"; +import { PlatformPreviewConflict, retainPlatformPreview } from "./platform-preview-store"; +import { resolveAllServiceConfigsStrict } from "./service-config-resolver"; + +async function source() { + return executeAndWait( + createApiOpsClient(), + { kind: "configuration.sources.inspect" }, + createDurableOpsKey("platform.sources", randomUUID()), + ); +} +export async function inspectPlatformConfiguration() { + const current = await source(); + const effective = services.resolvePlatformConfiguration({ + saved: current.platform, + env: process.env, + }); + return { + expectedCurrentRevisionId: current.revisionId, + configuration: current.platform ?? { version: 1 as const }, + values: effective.values, + sources: effective.sources, + }; +} +export async function previewPlatformConfiguration(input: { + actorId: string; + operationKey: string; + expectedCurrentRevisionId: string | null; + changes: services.PlatformConfiguration; +}) { + const parsed = adminPlatformPreviewInputSchema.parse({ + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + changes: input.changes, + }), + current = await source(); + if (parsed.expectedCurrentRevisionId !== current.revisionId) throw new PlatformPreviewConflict(); + const policy = await loadAuthPolicy(db); + const [desired] = await db + .select({ version: sql`xmin::text` }) + .from(systemSettings) + .where(eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING)); + const preview = await withAdministrativeConfigurationRead({}, async (read) => { + const resolved = await resolveAllServiceConfigsStrict( + read.services.map((service) => ({ + id: service.manifest.id, + configSchema: service.manifest.configSchema, + containerEnv: service.manifest.container.environment, + containerMemory: service.manifest.container.memory, + isBuiltIn: service.isBuiltIn, + proxyHostConfigKey: service.manifest.exposure?.proxy?.host?.configKey, + })), + read, + ); + return services.buildPlatformConfigurationPreview({ + actorId: input.actorId, + previewId: randomUUID(), + authPolicyRevision: policy.revision, + sourceRevision: current.sourceRevision, + currentRevisionId: current.revisionId, + expectedCurrentRevisionId: parsed.expectedCurrentRevisionId, + currentPlatform: current.platform, + changes: parsed.changes, + env: read.env, + services: read.services, + resolvedServiceConfigs: resolved.values, + }); + }); + const retained = await retainPlatformPreview({ + actorId: input.actorId, + operationKey: input.operationKey, + requestFingerprint: createHash("sha256").update(canonicalJson(parsed)).digest("hex"), + preview, + sourceSnapshot: current, + desiredRowVersion: desired?.version ?? null, + }); + return { preview: retained.preview, expiresAt: retained.expiresAt }; +} diff --git a/packages/core/src/api/contracts/service-admin.test.ts b/packages/core/src/api/contracts/service-admin.test.ts index 409ac3849..6d12cc17d 100644 --- a/packages/core/src/api/contracts/service-admin.test.ts +++ b/packages/core/src/api/contracts/service-admin.test.ts @@ -16,6 +16,14 @@ const actions = Object.fromEntries( ); const at = "2026-10-21T10:00:00.000Z"; const inventory = [ + ["listAdminArtifactTransfers", "GET", "/artifact-transfers"], + ["createAdminArtifactTransfer", "POST", "/artifact-transfers"], + ["getAdminArtifactTransfer", "GET", "/artifact-transfers/:stageId"], + ["deleteAdminArtifactTransfer", "DELETE", "/artifact-transfers/:stageId"], + ["uploadAdminArtifactTransferContent", "PUT", "/artifact-transfers/:stageId/content"], + ["downloadAdminArtifactTransferContent", "GET", "/artifact-transfers/:stageId/content"], + ["getAdminPlatformConfiguration", "GET", "/platform/configuration"], + ["previewAdminPlatformConfiguration", "POST", "/platform/configuration/preview"], ["listAdminServices", "GET", "/services"], ["getAdminService", "GET", "/services/:id"], ["getAdminServiceConfig", "GET", "/services/:id/config"], @@ -213,7 +221,7 @@ function response( } describe("canonical complete service administration contracts", () => { - it("binds the exact30 approved identities, methods, paths and admin auth", () => { + it("binds the complete approved identities, methods, paths and admin auth", () => { expect(contracts).toHaveProperty("serviceAdminOperations"); expect( Object.entries(contracts.serviceAdminOperations).map(([id, op]) => [ diff --git a/packages/core/src/api/contracts/service-admin/index.ts b/packages/core/src/api/contracts/service-admin/index.ts index dee60cd25..44aacb934 100644 --- a/packages/core/src/api/contracts/service-admin/index.ts +++ b/packages/core/src/api/contracts/service-admin/index.ts @@ -1,6 +1,8 @@ import { artifactTransferAdminOperations } from "./artifact-transfers"; +import { platformAdminOperations } from "./platform"; export * from "./artifact-transfers"; +export * from "./platform"; import z from "zod/v4"; import { @@ -79,6 +81,7 @@ export * from "./workflows"; /** Complete service/workflow family. Authentication metadata never grants access. */ export const serviceAdminOperations = { ...artifactTransferAdminOperations, + ...platformAdminOperations, listAdminServices: { method: "GET", path: "/api/admin/services", diff --git a/packages/core/src/api/contracts/service-admin/platform.ts b/packages/core/src/api/contracts/service-admin/platform.ts index a07e810da..dd95cfc3f 100644 --- a/packages/core/src/api/contracts/service-admin/platform.ts +++ b/packages/core/src/api/contracts/service-admin/platform.ts @@ -1,6 +1,8 @@ /** Portable platform preview data; these schemas confer no host authority. */ import z from "zod/v4"; import { platformConfigurationSchema } from "../../../services/platform-configuration"; +import { adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; +import { adminServiceIdempotencyHeadersSchema } from "./common"; const digest = z.string().regex(/^[a-f0-9]{64}$/); const revision = z @@ -64,3 +66,40 @@ export const adminPlatformApplyInputSchema = z.strictObject({ confirmations: adminPlatformConfirmationSchema, }); export type AdminPlatformPreview = z.infer; +export const adminPlatformConfigurationSchema = adminPlatformPreviewSchema.pick({ + expectedCurrentRevisionId: true, + configuration: true, + values: true, + sources: true, +}); +export const adminPlatformPreviewRetainedSchema = z.strictObject({ + preview: adminPlatformPreviewSchema, + expiresAt: z.iso.datetime({ offset: true }), +}); +export const platformAdminOperations = { + getAdminPlatformConfiguration: { + method: "GET", + path: "/api/admin/platform/configuration", + auth: "admin", + responses: { + ...adminCommonResponses, + 200: adminJson(adminPlatformConfigurationSchema), + 503: adminJson(httpErrorSchema), + }, + }, + previewAdminPlatformConfiguration: { + method: "POST", + path: "/api/admin/platform/configuration/preview", + auth: "admin", + request: { + headers: adminServiceIdempotencyHeadersSchema, + body: adminPlatformPreviewInputSchema, + }, + responses: { + ...adminCommonResponses, + 200: adminJson(adminPlatformPreviewRetainedSchema), + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), + }, + }, +} as const; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index a7388fc80..5cbe7c43f 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -278,6 +278,13 @@ export async function getAdminOverviewRequest( ): Promise { return getAdminOverviewBound(url, options); } +const getAdminPlatformConfigurationBound = bindAdminOperation("getAdminPlatformConfiguration"); +export async function getAdminPlatformConfigurationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminPlatformConfigurationBound(url, options); +} const getAdminServiceBound = bindAdminOperation("getAdminService"); export async function getAdminServiceRequest( url: string, @@ -707,6 +714,15 @@ export async function previewAdminOperationRequest( ): Promise { return previewAdminOperationBound(url, options); } +const previewAdminPlatformConfigurationBound = bindAdminOperation( + "previewAdminPlatformConfiguration", +); +export async function previewAdminPlatformConfigurationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return previewAdminPlatformConfigurationBound(url, options); +} const previewAdminStandaloneIntegrationBound = bindAdminOperation( "previewAdminStandaloneIntegration", ); diff --git a/packages/core/src/api/generated/admin/service-admin.generated.ts b/packages/core/src/api/generated/admin/service-admin.generated.ts index d24f6354b..de9901d5f 100644 --- a/packages/core/src/api/generated/admin/service-admin.generated.ts +++ b/packages/core/src/api/generated/admin/service-admin.generated.ts @@ -21,6 +21,7 @@ import { getAdminComposePreviewRequest, getAdminDawarichRequest, getAdminDeploymentRequest, + getAdminPlatformConfigurationRequest, getAdminServiceConfigRequest, getAdminServiceCredentialsRequest, getAdminServiceDataRequest, @@ -32,6 +33,7 @@ import { listAdminServiceBackupsRequest, listAdminServicesRequest, previewAdminOperationRequest, + previewAdminPlatformConfigurationRequest, provisionAdminDawarichRequest, restoreAdminServiceBackupRequest, rollbackAdminSharedMobilityRequest, @@ -172,6 +174,15 @@ import type { GetAdminDeployment415, GetAdminDeployment429, GetAdminDeployment500, + GetAdminPlatformConfiguration200, + GetAdminPlatformConfiguration400, + GetAdminPlatformConfiguration401, + GetAdminPlatformConfiguration403, + GetAdminPlatformConfiguration413, + GetAdminPlatformConfiguration415, + GetAdminPlatformConfiguration429, + GetAdminPlatformConfiguration500, + GetAdminPlatformConfiguration503, GetAdminService200, GetAdminService400, GetAdminService401, @@ -275,6 +286,18 @@ import type { PreviewAdminOperation429, PreviewAdminOperation500, PreviewAdminOperationBody, + PreviewAdminPlatformConfiguration200, + PreviewAdminPlatformConfiguration400, + PreviewAdminPlatformConfiguration401, + PreviewAdminPlatformConfiguration403, + PreviewAdminPlatformConfiguration409, + PreviewAdminPlatformConfiguration413, + PreviewAdminPlatformConfiguration415, + PreviewAdminPlatformConfiguration429, + PreviewAdminPlatformConfiguration500, + PreviewAdminPlatformConfiguration503, + PreviewAdminPlatformConfigurationBody, + PreviewAdminPlatformConfigurationHeaders, ProvisionAdminDawarich200, ProvisionAdminDawarich400, ProvisionAdminDawarich401, @@ -2142,6 +2165,199 @@ export const runAdminOperation = async ( }); }; +export type getAdminPlatformConfigurationResponse200 = { + data: GetAdminPlatformConfiguration200; + status: 200; +}; + +export type getAdminPlatformConfigurationResponse400 = { + data: GetAdminPlatformConfiguration400; + status: 400; +}; + +export type getAdminPlatformConfigurationResponse401 = { + data: GetAdminPlatformConfiguration401; + status: 401; +}; + +export type getAdminPlatformConfigurationResponse403 = { + data: GetAdminPlatformConfiguration403; + status: 403; +}; + +export type getAdminPlatformConfigurationResponse413 = { + data: GetAdminPlatformConfiguration413; + status: 413; +}; + +export type getAdminPlatformConfigurationResponse415 = { + data: GetAdminPlatformConfiguration415; + status: 415; +}; + +export type getAdminPlatformConfigurationResponse429 = { + data: GetAdminPlatformConfiguration429; + status: 429; +}; + +export type getAdminPlatformConfigurationResponse500 = { + data: GetAdminPlatformConfiguration500; + status: 500; +}; + +export type getAdminPlatformConfigurationResponse503 = { + data: GetAdminPlatformConfiguration503; + status: 503; +}; + +export type getAdminPlatformConfigurationResponseSuccess = + getAdminPlatformConfigurationResponse200 & { + headers: Headers; + }; +export type getAdminPlatformConfigurationResponseError = ( + | getAdminPlatformConfigurationResponse400 + | getAdminPlatformConfigurationResponse401 + | getAdminPlatformConfigurationResponse403 + | getAdminPlatformConfigurationResponse413 + | getAdminPlatformConfigurationResponse415 + | getAdminPlatformConfigurationResponse429 + | getAdminPlatformConfigurationResponse500 + | getAdminPlatformConfigurationResponse503 +) & { + headers: Headers; +}; + +export type getAdminPlatformConfigurationResponse = + | getAdminPlatformConfigurationResponseSuccess + | getAdminPlatformConfigurationResponseError; + +export const getGetAdminPlatformConfigurationUrl = () => { + return `/api/admin/platform/configuration`; +}; + +export const getAdminPlatformConfiguration = async ( + options?: Parameters[1], +): Promise => { + return getAdminPlatformConfigurationRequest( + getGetAdminPlatformConfigurationUrl(), + { + ...options, + method: "GET", + }, + ); +}; + +export type previewAdminPlatformConfigurationResponse200 = { + data: PreviewAdminPlatformConfiguration200; + status: 200; +}; + +export type previewAdminPlatformConfigurationResponse400 = { + data: PreviewAdminPlatformConfiguration400; + status: 400; +}; + +export type previewAdminPlatformConfigurationResponse401 = { + data: PreviewAdminPlatformConfiguration401; + status: 401; +}; + +export type previewAdminPlatformConfigurationResponse403 = { + data: PreviewAdminPlatformConfiguration403; + status: 403; +}; + +export type previewAdminPlatformConfigurationResponse409 = { + data: PreviewAdminPlatformConfiguration409; + status: 409; +}; + +export type previewAdminPlatformConfigurationResponse413 = { + data: PreviewAdminPlatformConfiguration413; + status: 413; +}; + +export type previewAdminPlatformConfigurationResponse415 = { + data: PreviewAdminPlatformConfiguration415; + status: 415; +}; + +export type previewAdminPlatformConfigurationResponse429 = { + data: PreviewAdminPlatformConfiguration429; + status: 429; +}; + +export type previewAdminPlatformConfigurationResponse500 = { + data: PreviewAdminPlatformConfiguration500; + status: 500; +}; + +export type previewAdminPlatformConfigurationResponse503 = { + data: PreviewAdminPlatformConfiguration503; + status: 503; +}; + +export type previewAdminPlatformConfigurationResponseSuccess = + previewAdminPlatformConfigurationResponse200 & { + headers: Headers; + }; +export type previewAdminPlatformConfigurationResponseError = ( + | previewAdminPlatformConfigurationResponse400 + | previewAdminPlatformConfigurationResponse401 + | previewAdminPlatformConfigurationResponse403 + | previewAdminPlatformConfigurationResponse409 + | previewAdminPlatformConfigurationResponse413 + | previewAdminPlatformConfigurationResponse415 + | previewAdminPlatformConfigurationResponse429 + | previewAdminPlatformConfigurationResponse500 + | previewAdminPlatformConfigurationResponse503 +) & { + headers: Headers; +}; + +export type previewAdminPlatformConfigurationResponse = + | previewAdminPlatformConfigurationResponseSuccess + | previewAdminPlatformConfigurationResponseError; + +export const getPreviewAdminPlatformConfigurationUrl = () => { + return `/api/admin/platform/configuration/preview`; +}; + +export const previewAdminPlatformConfiguration = async ( + previewAdminPlatformConfigurationBody: PreviewAdminPlatformConfigurationBody, + headers: PreviewAdminPlatformConfigurationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return previewAdminPlatformConfigurationRequest( + getPreviewAdminPlatformConfigurationUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(previewAdminPlatformConfigurationBody), + }, + ); +}; + export type listAdminServicesResponse200 = { data: ListAdminServices200; status: 200; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts new file mode 100644 index 000000000..ad6f87b51 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminPlatformConfiguration200Configuration } from "./getAdminPlatformConfiguration200Configuration"; +import type { GetAdminPlatformConfiguration200Sources } from "./getAdminPlatformConfiguration200Sources"; +import type { GetAdminPlatformConfiguration200Values } from "./getAdminPlatformConfiguration200Values"; + +export type GetAdminPlatformConfiguration200 = { + expectedCurrentRevisionId: string | null; + configuration: GetAdminPlatformConfiguration200Configuration; + values: GetAdminPlatformConfiguration200Values; + sources: GetAdminPlatformConfiguration200Sources; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Configuration.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Configuration.ts new file mode 100644 index 000000000..de49ed01f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Configuration.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminPlatformConfiguration200ConfigurationAcmeEmail } from "./getAdminPlatformConfiguration200ConfigurationAcmeEmail"; +import type { GetAdminPlatformConfiguration200ConfigurationBackupRetentionDays } from "./getAdminPlatformConfiguration200ConfigurationBackupRetentionDays"; +import type { GetAdminPlatformConfiguration200ConfigurationDefaultRegion } from "./getAdminPlatformConfiguration200ConfigurationDefaultRegion"; +import type { GetAdminPlatformConfiguration200ConfigurationDomain } from "./getAdminPlatformConfiguration200ConfigurationDomain"; +import type { GetAdminPlatformConfiguration200ConfigurationReleaseSelector } from "./getAdminPlatformConfiguration200ConfigurationReleaseSelector"; +import type { GetAdminPlatformConfiguration200ConfigurationVersion } from "./getAdminPlatformConfiguration200ConfigurationVersion"; + +export type GetAdminPlatformConfiguration200Configuration = { + version: GetAdminPlatformConfiguration200ConfigurationVersion; + domain?: GetAdminPlatformConfiguration200ConfigurationDomain; + acmeEmail?: GetAdminPlatformConfiguration200ConfigurationAcmeEmail; + releaseSelector?: GetAdminPlatformConfiguration200ConfigurationReleaseSelector; + defaultRegion?: GetAdminPlatformConfiguration200ConfigurationDefaultRegion; + backupRetentionDays?: GetAdminPlatformConfiguration200ConfigurationBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationAcmeEmail.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationAcmeEmail.ts new file mode 100644 index 000000000..8cd2cec45 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationAcmeEmail.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200ConfigurationAcmeEmail = + | { + source: "host"; + } + | { + source: "database"; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationBackupRetentionDays.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationBackupRetentionDays.ts new file mode 100644 index 000000000..60d6e46a6 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationBackupRetentionDays.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200ConfigurationBackupRetentionDays = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minimum 1 + * @maximum 36500 + */ + value: number; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDefaultRegion.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDefaultRegion.ts new file mode 100644 index 000000000..978c35112 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDefaultRegion.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200ConfigurationDefaultRegion = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDomain.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDomain.ts new file mode 100644 index 000000000..515bd08ae --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationDomain.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200ConfigurationDomain = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 253 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationReleaseSelector.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationReleaseSelector.ts new file mode 100644 index 000000000..fb93bf311 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationReleaseSelector.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200ConfigurationReleaseSelector = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 128 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationVersion.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationVersion.ts new file mode 100644 index 000000000..162a5f3d1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200ConfigurationVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200ConfigurationVersion = + (typeof GetAdminPlatformConfiguration200ConfigurationVersion)[keyof typeof GetAdminPlatformConfiguration200ConfigurationVersion]; + +export const GetAdminPlatformConfiguration200ConfigurationVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Sources.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Sources.ts new file mode 100644 index 000000000..0b0c48c59 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Sources.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminPlatformConfiguration200SourcesAcmeEmail } from "./getAdminPlatformConfiguration200SourcesAcmeEmail"; +import type { GetAdminPlatformConfiguration200SourcesBackupRetentionDays } from "./getAdminPlatformConfiguration200SourcesBackupRetentionDays"; +import type { GetAdminPlatformConfiguration200SourcesDefaultRegion } from "./getAdminPlatformConfiguration200SourcesDefaultRegion"; +import type { GetAdminPlatformConfiguration200SourcesDomain } from "./getAdminPlatformConfiguration200SourcesDomain"; +import type { GetAdminPlatformConfiguration200SourcesReleaseSelector } from "./getAdminPlatformConfiguration200SourcesReleaseSelector"; + +export type GetAdminPlatformConfiguration200Sources = { + domain: GetAdminPlatformConfiguration200SourcesDomain; + acmeEmail: GetAdminPlatformConfiguration200SourcesAcmeEmail; + releaseSelector: GetAdminPlatformConfiguration200SourcesReleaseSelector; + defaultRegion: GetAdminPlatformConfiguration200SourcesDefaultRegion; + backupRetentionDays: GetAdminPlatformConfiguration200SourcesBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesAcmeEmail.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesAcmeEmail.ts new file mode 100644 index 000000000..9705ec2da --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesAcmeEmail.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200SourcesAcmeEmail = + (typeof GetAdminPlatformConfiguration200SourcesAcmeEmail)[keyof typeof GetAdminPlatformConfiguration200SourcesAcmeEmail]; + +export const GetAdminPlatformConfiguration200SourcesAcmeEmail = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesBackupRetentionDays.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesBackupRetentionDays.ts new file mode 100644 index 000000000..51a3de7a7 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesBackupRetentionDays.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200SourcesBackupRetentionDays = + (typeof GetAdminPlatformConfiguration200SourcesBackupRetentionDays)[keyof typeof GetAdminPlatformConfiguration200SourcesBackupRetentionDays]; + +export const GetAdminPlatformConfiguration200SourcesBackupRetentionDays = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDefaultRegion.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDefaultRegion.ts new file mode 100644 index 000000000..d65dcc321 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDefaultRegion.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200SourcesDefaultRegion = + (typeof GetAdminPlatformConfiguration200SourcesDefaultRegion)[keyof typeof GetAdminPlatformConfiguration200SourcesDefaultRegion]; + +export const GetAdminPlatformConfiguration200SourcesDefaultRegion = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDomain.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDomain.ts new file mode 100644 index 000000000..b5bce544a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesDomain.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200SourcesDomain = + (typeof GetAdminPlatformConfiguration200SourcesDomain)[keyof typeof GetAdminPlatformConfiguration200SourcesDomain]; + +export const GetAdminPlatformConfiguration200SourcesDomain = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesReleaseSelector.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesReleaseSelector.ts new file mode 100644 index 000000000..1e5ebcd8c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200SourcesReleaseSelector.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200SourcesReleaseSelector = + (typeof GetAdminPlatformConfiguration200SourcesReleaseSelector)[keyof typeof GetAdminPlatformConfiguration200SourcesReleaseSelector]; + +export const GetAdminPlatformConfiguration200SourcesReleaseSelector = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Values.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Values.ts new file mode 100644 index 000000000..8fd9b8ad1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200Values.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200Values = { + /** + * @minLength 1 + * @maxLength 253 + */ + domain: string; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + acmeEmail: string; + /** + * @minLength 1 + * @maxLength 128 + */ + releaseSelector: string; + defaultRegion: string | null; + /** + * @minimum 1 + * @maximum 36500 + */ + backupRetentionDays: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration400.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration400.ts new file mode 100644 index 000000000..5f4455d9f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration401.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration401.ts new file mode 100644 index 000000000..51458adfb --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration403.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration403.ts new file mode 100644 index 000000000..8fc2542eb --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration413.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration413.ts new file mode 100644 index 000000000..3f24791a1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration415.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration415.ts new file mode 100644 index 000000000..19c27d97b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration429.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration429.ts new file mode 100644 index 000000000..74799d70f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration500.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration500.ts new file mode 100644 index 000000000..2cfeeb5ac --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration503.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration503.ts new file mode 100644 index 000000000..a63cb6e43 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index d50c14ec0..d9bec3d22 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -152,6 +152,29 @@ export * from "./getAdminDeployment413"; export * from "./getAdminDeployment415"; export * from "./getAdminDeployment429"; export * from "./getAdminDeployment500"; +export * from "./getAdminPlatformConfiguration200"; +export * from "./getAdminPlatformConfiguration200Configuration"; +export * from "./getAdminPlatformConfiguration200ConfigurationAcmeEmail"; +export * from "./getAdminPlatformConfiguration200ConfigurationBackupRetentionDays"; +export * from "./getAdminPlatformConfiguration200ConfigurationDefaultRegion"; +export * from "./getAdminPlatformConfiguration200ConfigurationDomain"; +export * from "./getAdminPlatformConfiguration200ConfigurationReleaseSelector"; +export * from "./getAdminPlatformConfiguration200ConfigurationVersion"; +export * from "./getAdminPlatformConfiguration200Sources"; +export * from "./getAdminPlatformConfiguration200SourcesAcmeEmail"; +export * from "./getAdminPlatformConfiguration200SourcesBackupRetentionDays"; +export * from "./getAdminPlatformConfiguration200SourcesDefaultRegion"; +export * from "./getAdminPlatformConfiguration200SourcesDomain"; +export * from "./getAdminPlatformConfiguration200SourcesReleaseSelector"; +export * from "./getAdminPlatformConfiguration200Values"; +export * from "./getAdminPlatformConfiguration400"; +export * from "./getAdminPlatformConfiguration401"; +export * from "./getAdminPlatformConfiguration403"; +export * from "./getAdminPlatformConfiguration413"; +export * from "./getAdminPlatformConfiguration415"; +export * from "./getAdminPlatformConfiguration429"; +export * from "./getAdminPlatformConfiguration500"; +export * from "./getAdminPlatformConfiguration503"; export * from "./getAdminService200"; export * from "./getAdminService200Acquisition"; export * from "./getAdminService200Actions"; @@ -436,6 +459,43 @@ export * from "./previewAdminOperation415"; export * from "./previewAdminOperation429"; export * from "./previewAdminOperation500"; export * from "./previewAdminOperationBody"; +export * from "./previewAdminPlatformConfiguration200"; +export * from "./previewAdminPlatformConfiguration200Preview"; +export * from "./previewAdminPlatformConfiguration200PreviewConfiguration"; +export * from "./previewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail"; +export * from "./previewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays"; +export * from "./previewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion"; +export * from "./previewAdminPlatformConfiguration200PreviewConfigurationDomain"; +export * from "./previewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector"; +export * from "./previewAdminPlatformConfiguration200PreviewConfigurationVersion"; +export * from "./previewAdminPlatformConfiguration200PreviewOriginConsumers"; +export * from "./previewAdminPlatformConfiguration200PreviewRequiredConfirmation"; +export * from "./previewAdminPlatformConfiguration200PreviewSources"; +export * from "./previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail"; +export * from "./previewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays"; +export * from "./previewAdminPlatformConfiguration200PreviewSourcesDefaultRegion"; +export * from "./previewAdminPlatformConfiguration200PreviewSourcesDomain"; +export * from "./previewAdminPlatformConfiguration200PreviewSourcesReleaseSelector"; +export * from "./previewAdminPlatformConfiguration200PreviewValues"; +export * from "./previewAdminPlatformConfiguration200PreviewVersion"; +export * from "./previewAdminPlatformConfiguration400"; +export * from "./previewAdminPlatformConfiguration401"; +export * from "./previewAdminPlatformConfiguration403"; +export * from "./previewAdminPlatformConfiguration409"; +export * from "./previewAdminPlatformConfiguration413"; +export * from "./previewAdminPlatformConfiguration415"; +export * from "./previewAdminPlatformConfiguration429"; +export * from "./previewAdminPlatformConfiguration500"; +export * from "./previewAdminPlatformConfiguration503"; +export * from "./previewAdminPlatformConfigurationBody"; +export * from "./previewAdminPlatformConfigurationBodyChanges"; +export * from "./previewAdminPlatformConfigurationBodyChangesAcmeEmail"; +export * from "./previewAdminPlatformConfigurationBodyChangesBackupRetentionDays"; +export * from "./previewAdminPlatformConfigurationBodyChangesDefaultRegion"; +export * from "./previewAdminPlatformConfigurationBodyChangesDomain"; +export * from "./previewAdminPlatformConfigurationBodyChangesReleaseSelector"; +export * from "./previewAdminPlatformConfigurationBodyChangesVersion"; +export * from "./previewAdminPlatformConfigurationHeaders"; export * from "./provisionAdminDawarich200"; export * from "./provisionAdminDawarich200OauthClient"; export * from "./provisionAdminDawarich200Secrets"; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200.ts new file mode 100644 index 000000000..dc2ad4adc --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminPlatformConfiguration200Preview } from "./previewAdminPlatformConfiguration200Preview"; + +export type PreviewAdminPlatformConfiguration200 = { + preview: PreviewAdminPlatformConfiguration200Preview; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + expiresAt: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts new file mode 100644 index 000000000..ea72c0092 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts @@ -0,0 +1,46 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminPlatformConfiguration200PreviewConfiguration } from "./previewAdminPlatformConfiguration200PreviewConfiguration"; +import type { PreviewAdminPlatformConfiguration200PreviewOriginConsumers } from "./previewAdminPlatformConfiguration200PreviewOriginConsumers"; +import type { PreviewAdminPlatformConfiguration200PreviewRequiredConfirmation } from "./previewAdminPlatformConfiguration200PreviewRequiredConfirmation"; +import type { PreviewAdminPlatformConfiguration200PreviewSources } from "./previewAdminPlatformConfiguration200PreviewSources"; +import type { PreviewAdminPlatformConfiguration200PreviewValues } from "./previewAdminPlatformConfiguration200PreviewValues"; +import type { PreviewAdminPlatformConfiguration200PreviewVersion } from "./previewAdminPlatformConfiguration200PreviewVersion"; + +export type PreviewAdminPlatformConfiguration200Preview = { + version: PreviewAdminPlatformConfiguration200PreviewVersion; + /** + * @minLength 1 + * @maxLength 128 + */ + actorId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** @pattern ^[a-f0-9]{64}$ */ + authPolicyRevision: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + sourceRevision: string; + expectedCurrentRevisionId: string | null; + configuration: PreviewAdminPlatformConfiguration200PreviewConfiguration; + values: PreviewAdminPlatformConfiguration200PreviewValues; + sources: PreviewAdminPlatformConfiguration200PreviewSources; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + */ + affectedServiceIds: string[]; + /** @maxLength 2048 */ + oldOrigin: string; + /** @maxLength 2048 */ + newOrigin: string; + originConsumers: PreviewAdminPlatformConfiguration200PreviewOriginConsumers; + requiredConfirmation: PreviewAdminPlatformConfiguration200PreviewRequiredConfirmation; + /** + * @maxItems 16 + * @items.maxLength 512 + */ + warnings: string[]; + /** @pattern ^[a-f0-9]{64}$ */ + previewFingerprint: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfiguration.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfiguration.ts new file mode 100644 index 000000000..c0c474432 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfiguration.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail } from "./previewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail"; +import type { PreviewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays } from "./previewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays"; +import type { PreviewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion } from "./previewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion"; +import type { PreviewAdminPlatformConfiguration200PreviewConfigurationDomain } from "./previewAdminPlatformConfiguration200PreviewConfigurationDomain"; +import type { PreviewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector } from "./previewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector"; +import type { PreviewAdminPlatformConfiguration200PreviewConfigurationVersion } from "./previewAdminPlatformConfiguration200PreviewConfigurationVersion"; + +export type PreviewAdminPlatformConfiguration200PreviewConfiguration = { + version: PreviewAdminPlatformConfiguration200PreviewConfigurationVersion; + domain?: PreviewAdminPlatformConfiguration200PreviewConfigurationDomain; + acmeEmail?: PreviewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail; + releaseSelector?: PreviewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector; + defaultRegion?: PreviewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion; + backupRetentionDays?: PreviewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail.ts new file mode 100644 index 000000000..d28b9bfda --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewConfigurationAcmeEmail = + | { + source: "host"; + } + | { + source: "database"; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays.ts new file mode 100644 index 000000000..aa34720c5 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewConfigurationBackupRetentionDays = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minimum 1 + * @maximum 36500 + */ + value: number; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion.ts new file mode 100644 index 000000000..04f4c97a0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewConfigurationDefaultRegion = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDomain.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDomain.ts new file mode 100644 index 000000000..03ae94bd1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationDomain.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewConfigurationDomain = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 253 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector.ts new file mode 100644 index 000000000..d2c85696d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 128 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationVersion.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationVersion.ts new file mode 100644 index 000000000..379ed889f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewConfigurationVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewConfigurationVersion = + (typeof PreviewAdminPlatformConfiguration200PreviewConfigurationVersion)[keyof typeof PreviewAdminPlatformConfiguration200PreviewConfigurationVersion]; + +export const PreviewAdminPlatformConfiguration200PreviewConfigurationVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewOriginConsumers.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewOriginConsumers.ts new file mode 100644 index 000000000..5ac2ecd28 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewOriginConsumers.ts @@ -0,0 +1,25 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewOriginConsumers = { + /** + * @minLength 1 + * @maxLength 253 + */ + routingHost: string; + /** @maxLength 2048 */ + corsOrigin: string; + /** @maxLength 2048 */ + authUrl: string; + /** + * @minLength 1 + * @maxLength 253 + */ + passkeyRpId: string; + /** @maxLength 2048 */ + passkeyOrigin: string; + /** @maxLength 2048 */ + webApiUrl: string; + /** @maxLength 2048 */ + tilesPublicUrl: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRequiredConfirmation.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRequiredConfirmation.ts new file mode 100644 index 000000000..332d78b06 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRequiredConfirmation.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewRequiredConfirmation = { + domain: boolean; + dnsTls: boolean; + oauth: boolean; + passkeys: boolean; + alternateLogin: boolean; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSources.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSources.ts new file mode 100644 index 000000000..43b890294 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSources.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminPlatformConfiguration200PreviewSourcesAcmeEmail } from "./previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail"; +import type { PreviewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays } from "./previewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays"; +import type { PreviewAdminPlatformConfiguration200PreviewSourcesDefaultRegion } from "./previewAdminPlatformConfiguration200PreviewSourcesDefaultRegion"; +import type { PreviewAdminPlatformConfiguration200PreviewSourcesDomain } from "./previewAdminPlatformConfiguration200PreviewSourcesDomain"; +import type { PreviewAdminPlatformConfiguration200PreviewSourcesReleaseSelector } from "./previewAdminPlatformConfiguration200PreviewSourcesReleaseSelector"; + +export type PreviewAdminPlatformConfiguration200PreviewSources = { + domain: PreviewAdminPlatformConfiguration200PreviewSourcesDomain; + acmeEmail: PreviewAdminPlatformConfiguration200PreviewSourcesAcmeEmail; + releaseSelector: PreviewAdminPlatformConfiguration200PreviewSourcesReleaseSelector; + defaultRegion: PreviewAdminPlatformConfiguration200PreviewSourcesDefaultRegion; + backupRetentionDays: PreviewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail.ts new file mode 100644 index 000000000..572633bc3 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewSourcesAcmeEmail = + (typeof PreviewAdminPlatformConfiguration200PreviewSourcesAcmeEmail)[keyof typeof PreviewAdminPlatformConfiguration200PreviewSourcesAcmeEmail]; + +export const PreviewAdminPlatformConfiguration200PreviewSourcesAcmeEmail = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays.ts new file mode 100644 index 000000000..c12a17e1f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays = + (typeof PreviewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays)[keyof typeof PreviewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays]; + +export const PreviewAdminPlatformConfiguration200PreviewSourcesBackupRetentionDays = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDefaultRegion.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDefaultRegion.ts new file mode 100644 index 000000000..42864577e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDefaultRegion.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewSourcesDefaultRegion = + (typeof PreviewAdminPlatformConfiguration200PreviewSourcesDefaultRegion)[keyof typeof PreviewAdminPlatformConfiguration200PreviewSourcesDefaultRegion]; + +export const PreviewAdminPlatformConfiguration200PreviewSourcesDefaultRegion = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDomain.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDomain.ts new file mode 100644 index 000000000..d954b3969 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesDomain.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewSourcesDomain = + (typeof PreviewAdminPlatformConfiguration200PreviewSourcesDomain)[keyof typeof PreviewAdminPlatformConfiguration200PreviewSourcesDomain]; + +export const PreviewAdminPlatformConfiguration200PreviewSourcesDomain = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesReleaseSelector.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesReleaseSelector.ts new file mode 100644 index 000000000..dfad634f6 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewSourcesReleaseSelector.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewSourcesReleaseSelector = + (typeof PreviewAdminPlatformConfiguration200PreviewSourcesReleaseSelector)[keyof typeof PreviewAdminPlatformConfiguration200PreviewSourcesReleaseSelector]; + +export const PreviewAdminPlatformConfiguration200PreviewSourcesReleaseSelector = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewValues.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewValues.ts new file mode 100644 index 000000000..0c89d0f9c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewValues.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewValues = { + /** + * @minLength 1 + * @maxLength 253 + */ + domain: string; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + acmeEmail: string; + /** + * @minLength 1 + * @maxLength 128 + */ + releaseSelector: string; + defaultRegion: string | null; + /** + * @minimum 1 + * @maximum 36500 + */ + backupRetentionDays: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewVersion.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewVersion.ts new file mode 100644 index 000000000..4e9e592e4 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewVersion = + (typeof PreviewAdminPlatformConfiguration200PreviewVersion)[keyof typeof PreviewAdminPlatformConfiguration200PreviewVersion]; + +export const PreviewAdminPlatformConfiguration200PreviewVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration400.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration400.ts new file mode 100644 index 000000000..1d7f56eaa --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration401.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration401.ts new file mode 100644 index 000000000..3a986fa46 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration403.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration403.ts new file mode 100644 index 000000000..8d95b41ed --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration409.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration409.ts new file mode 100644 index 000000000..3037a8e50 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration413.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration413.ts new file mode 100644 index 000000000..da5b9318b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration415.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration415.ts new file mode 100644 index 000000000..607625607 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration429.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration429.ts new file mode 100644 index 000000000..b8215790a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration500.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration500.ts new file mode 100644 index 000000000..c5f185a08 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration503.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration503.ts new file mode 100644 index 000000000..6292f7111 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts new file mode 100644 index 000000000..aecbc9745 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminPlatformConfigurationBodyChanges } from "./previewAdminPlatformConfigurationBodyChanges"; + +export type PreviewAdminPlatformConfigurationBody = { + expectedCurrentRevisionId: string | null; + changes: PreviewAdminPlatformConfigurationBodyChanges; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChanges.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChanges.ts new file mode 100644 index 000000000..102cdf60b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChanges.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminPlatformConfigurationBodyChangesAcmeEmail } from "./previewAdminPlatformConfigurationBodyChangesAcmeEmail"; +import type { PreviewAdminPlatformConfigurationBodyChangesBackupRetentionDays } from "./previewAdminPlatformConfigurationBodyChangesBackupRetentionDays"; +import type { PreviewAdminPlatformConfigurationBodyChangesDefaultRegion } from "./previewAdminPlatformConfigurationBodyChangesDefaultRegion"; +import type { PreviewAdminPlatformConfigurationBodyChangesDomain } from "./previewAdminPlatformConfigurationBodyChangesDomain"; +import type { PreviewAdminPlatformConfigurationBodyChangesReleaseSelector } from "./previewAdminPlatformConfigurationBodyChangesReleaseSelector"; +import type { PreviewAdminPlatformConfigurationBodyChangesVersion } from "./previewAdminPlatformConfigurationBodyChangesVersion"; + +export type PreviewAdminPlatformConfigurationBodyChanges = { + version: PreviewAdminPlatformConfigurationBodyChangesVersion; + domain?: PreviewAdminPlatformConfigurationBodyChangesDomain; + acmeEmail?: PreviewAdminPlatformConfigurationBodyChangesAcmeEmail; + releaseSelector?: PreviewAdminPlatformConfigurationBodyChangesReleaseSelector; + defaultRegion?: PreviewAdminPlatformConfigurationBodyChangesDefaultRegion; + backupRetentionDays?: PreviewAdminPlatformConfigurationBodyChangesBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesAcmeEmail.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesAcmeEmail.ts new file mode 100644 index 000000000..b230d4bb4 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesAcmeEmail.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyChangesAcmeEmail = + | { + source: "host"; + } + | { + source: "database"; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesBackupRetentionDays.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesBackupRetentionDays.ts new file mode 100644 index 000000000..85a5d6dda --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesBackupRetentionDays.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyChangesBackupRetentionDays = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minimum 1 + * @maximum 36500 + */ + value: number; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDefaultRegion.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDefaultRegion.ts new file mode 100644 index 000000000..8ef8a7269 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDefaultRegion.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyChangesDefaultRegion = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDomain.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDomain.ts new file mode 100644 index 000000000..5015ac94d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesDomain.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyChangesDomain = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 253 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesReleaseSelector.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesReleaseSelector.ts new file mode 100644 index 000000000..00313e32d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesReleaseSelector.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyChangesReleaseSelector = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 128 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesVersion.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesVersion.ts new file mode 100644 index 000000000..4cb0ac98c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyChangesVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyChangesVersion = + (typeof PreviewAdminPlatformConfigurationBodyChangesVersion)[keyof typeof PreviewAdminPlatformConfigurationBodyChangesVersion]; + +export const PreviewAdminPlatformConfigurationBodyChangesVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationHeaders.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationHeaders.ts new file mode 100644 index 000000000..e28e9e900 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; From a64d5dc76ec6b6ebc28c06b7c087e71af87df622 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 08:36:45 +0100 Subject: [PATCH 073/135] feat(admin): apply platform previews through durable protected handoffs --- apps/api/openapi.json | 661 ++++++++++++++++++ apps/api/src/account/auth-policy.ts | 2 +- .../routes/__tests__/admin-platform.test.ts | 78 +++ apps/api/src/routes/admin-platform.ts | 38 +- .../__tests__/job-runner-initialize.test.ts | 6 + ...orm-configuration-runtime-postgres.test.ts | 162 ++++- apps/api/src/services/job-runner.ts | 41 ++ .../platform-configuration-application.ts | 62 ++ .../platform-configuration-participant.ts | 4 + .../platform-configuration-recovery.ts | 74 ++ .../services/platform-configuration-view.ts | 3 + .../src/services/platform-preview-store.ts | 16 +- .../src/services/prospective-configuration.ts | 60 +- .../src/configuration-handoff-control.ts | 61 +- .../src/api/contracts/service-admin.test.ts | 1 + .../api/contracts/service-admin/platform.ts | 16 + .../admin/operation-bindings.generated.ts | 7 + .../admin/service-admin.generated.ts | 124 ++++ .../applyAdminPlatformConfiguration202.ts | 8 + .../applyAdminPlatformConfiguration400.ts | 20 + .../applyAdminPlatformConfiguration401.ts | 20 + .../applyAdminPlatformConfiguration403.ts | 20 + .../applyAdminPlatformConfiguration409.ts | 20 + .../applyAdminPlatformConfiguration413.ts | 20 + .../applyAdminPlatformConfiguration415.ts | 20 + .../applyAdminPlatformConfiguration429.ts | 11 + .../applyAdminPlatformConfiguration500.ts | 20 + .../applyAdminPlatformConfiguration503.ts | 20 + .../applyAdminPlatformConfigurationBody.ts | 12 + ...nPlatformConfigurationBodyConfirmations.ts | 10 + .../applyAdminPlatformConfigurationHeaders.ts | 9 + .../generated/admin/service-models/index.ts | 15 + ...iewAdminPlatformConfiguration200Preview.ts | 6 + ...ration200PreviewReconcileServiceIdsItem.ts | 14 + .../previewAdminPlatformConfigurationBody.ts | 6 + ...onfigurationBodyReconcileServiceIdsItem.ts | 14 + .../src/ops/configuration-handoff-records.ts | 6 + .../ops/configuration-preparation-records.ts | 2 + .../__tests__/platform-preview.test.ts | 24 + .../core/src/services/platform-preview.ts | 17 + 40 files changed, 1699 insertions(+), 31 deletions(-) create mode 100644 apps/api/src/services/platform-configuration-application.ts create mode 100644 apps/api/src/services/platform-configuration-recovery.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration202.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration409.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBody.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBodyConfirmations.ts create mode 100644 packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationHeaders.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyReconcileServiceIdsItem.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 242957a61..efb58800f 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -50350,6 +50350,635 @@ "x-openmapx-auth": "admin" } }, + "/api/admin/platform/configuration/apply": { + "post": { + "operationId": "applyAdminPlatformConfiguration", + "parameters": [ + { + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$" + }, + "in": "header", + "name": "idempotency-key", + "required": true + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "expectedCurrentRevisionId": { + "anyOf": [ + { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + { + "type": "null" + } + ] + }, + "previewFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "confirmations": { + "type": "object", + "properties": { + "domain": { + "type": "boolean" + }, + "dnsTls": { + "type": "boolean" + }, + "oauth": { + "type": "boolean" + }, + "passkeys": { + "type": "boolean" + }, + "alternateLogin": { + "type": "boolean" + } + }, + "required": [ + "domain", + "dnsTls", + "oauth", + "passkeys", + "alternateLogin" + ], + "additionalProperties": false + }, + "domainProofToken": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "expectedCurrentRevisionId", + "previewFingerprint", + "confirmations" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [ + true + ] + }, + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + } + }, + "required": [ + "ok", + "jobId" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "409": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/admin/platform/configuration/preview": { "post": { "operationId": "previewAdminPlatformConfiguration", @@ -50596,6 +51225,22 @@ "version" ], "additionalProperties": false + }, + "reconcileServiceIds": { + "minItems": 1, + "maxItems": 6, + "type": "array", + "items": { + "type": "string", + "enum": [ + "postgis", + "redis", + "app-web", + "app-api", + "traefik", + "ops-agent" + ] + } } }, "required": [ @@ -50971,6 +51616,22 @@ "maxLength": 64 } }, + "reconcileServiceIds": { + "minItems": 1, + "maxItems": 6, + "type": "array", + "items": { + "type": "string", + "enum": [ + "postgis", + "redis", + "app-web", + "app-api", + "traefik", + "ops-agent" + ] + } + }, "oldOrigin": { "type": "string", "maxLength": 2048, diff --git a/apps/api/src/account/auth-policy.ts b/apps/api/src/account/auth-policy.ts index 390de085a..c2cdfbb9f 100644 --- a/apps/api/src/account/auth-policy.ts +++ b/apps/api/src/account/auth-policy.ts @@ -55,7 +55,7 @@ function snapshot(settings: AuthPolicySettings, activationVersion: 0 | 1): AuthP } export const unactivatedAuthPolicy = snapshot(defaultAuthPolicySettings, 0); export async function loadAuthPolicy( - database: AuthDatabase = currentAuthDatabase(), + database: Pick = currentAuthDatabase(), ): Promise { try { const rows = await database.select().from(authPolicyState).where(eq(authPolicyState.id, 1)); diff --git a/apps/api/src/routes/__tests__/admin-platform.test.ts b/apps/api/src/routes/__tests__/admin-platform.test.ts index 09681d1e1..0ef776b76 100644 --- a/apps/api/src/routes/__tests__/admin-platform.test.ts +++ b/apps/api/src/routes/__tests__/admin-platform.test.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import { createApiClient } from "@openmapx/core"; import { + applyAdminPlatformConfiguration, getAdminPlatformConfiguration, previewAdminPlatformConfiguration, unwrapAdminResponse, @@ -8,6 +9,8 @@ import { import { services } from "@openmapx/core/server"; import Fastify, { type FastifyInstance } from "fastify"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { ADMIN_REAUTH_COOKIE_NAME } from "../../account/admin-reauthentication"; +import { AdminJobFailure } from "../../services/admin-job-failure"; import { mockAdminSession } from "../../test/auth"; const mocks = vi.hoisted(() => ({ @@ -136,3 +139,78 @@ it("keeps host failure details out of public responses", async () => { expect(result.body).not.toContain("secret-value"); expect(result.body).not.toContain("/private"); }); +it("returns the durable apply identity without putting session proof material into the public body", async () => { + const body = { + expectedCurrentRevisionId: null, + previewFingerprint: preview.previewFingerprint, + confirmations: preview.requiredConfirmation, + }; + const result = await applyAdminPlatformConfiguration( + body, + { "idempotency-key": "platform-apply-key-001" }, + { client }, + ); + expect(result.status).toBe(202); + expect(unwrapAdminResponse(result)).toEqual(await mocks.apply.mock.results[0].value); + expect(mocks.apply).toHaveBeenCalledWith( + expect.objectContaining({ ...body, actorId: "test-admin-id" }), + ); + expect(JSON.stringify(result.data)).not.toContain("session"); +}); +it("requires the bound HttpOnly nonce for a domain proof and forwards it privately", async () => { + const token = randomUUID(), + nonce = Buffer.alloc(32, 7); + const body = { + expectedCurrentRevisionId: null, + previewFingerprint: preview.previewFingerprint, + confirmations: preview.requiredConfirmation, + domainProofToken: token, + }; + const headers = { "idempotency-key": "platform-proof-key-001" }; + expect( + ( + await app.inject({ + method: "POST", + url: "/api/admin/platform/configuration/apply", + headers, + payload: body, + }) + ).statusCode, + ).toBe(409); + expect(mocks.apply).not.toHaveBeenCalled(); + const response = await app.inject({ + method: "POST", + url: "/api/admin/platform/configuration/apply", + headers: { ...headers, cookie: `${ADMIN_REAUTH_COOKIE_NAME}=${nonce.toString("base64url")}` }, + payload: body, + }); + expect(response.statusCode).toBe(202); + expect(mocks.apply).toHaveBeenCalledWith( + expect.objectContaining({ + proof: { + challengeId: token, + sessionId: "test-session-id", + nonce, + }, + }), + ); + expect(response.body).not.toContain(token); + expect(response.body).not.toContain(nonce.toString("base64url")); +}); +it("reports a changed retained mutation as a conflict without exposing internal failure details", async () => { + mocks.apply.mockRejectedValue( + new AdminJobFailure("private internal context", { errorCode: "conflict" }), + ); + const result = await app.inject({ + method: "POST", + url: "/api/admin/platform/configuration/apply", + headers: { "idempotency-key": "platform-conflict-001" }, + payload: { + expectedCurrentRevisionId: null, + previewFingerprint: preview.previewFingerprint, + confirmations: preview.requiredConfirmation, + }, + }); + expect(result.statusCode).toBe(409); + expect(result.body).not.toContain("private internal"); +}); diff --git a/apps/api/src/routes/admin-platform.ts b/apps/api/src/routes/admin-platform.ts index 2f3468a40..c1c20e073 100644 --- a/apps/api/src/routes/admin-platform.ts +++ b/apps/api/src/routes/admin-platform.ts @@ -1,9 +1,11 @@ import type { FastifyInstance } from "fastify"; import { AdminReauthenticationError } from "../account/admin-reauthentication"; +import { AdminJobFailure } from "../services/admin-job-failure"; import { createDirectAdminOpsKey, parseDirectOpsIdempotency, } from "../services/direct-ops-idempotency"; +import { jobRunner } from "../services/job-runner"; import { inspectPlatformConfiguration, previewPlatformConfiguration, @@ -12,6 +14,7 @@ import { PlatformPreviewConflict } from "../services/platform-preview-store"; import { RateLimiter } from "../utils/rate-limit"; import { getAdminSession, requireAdmin } from "../utils/require-admin"; import { declareRouteAuth } from "../utils/route-auth"; +import { adminReauthNonce } from "./admin-reauthentication"; import { registerServiceAdminRoute } from "./admin-service-contract"; export async function adminPlatformRoute(app: FastifyInstance) { @@ -29,7 +32,9 @@ export async function adminPlatformRoute(app: FastifyInstance) { }); app.addHook("onClose", async () => limits.destroy()); const failure = (error: unknown) => - error instanceof PlatformPreviewConflict || error instanceof AdminReauthenticationError + error instanceof PlatformPreviewConflict || + error instanceof AdminReauthenticationError || + (error instanceof AdminJobFailure && error.result.errorCode === "conflict") ? { status: 409, error: "Platform preview, source or authorization changed; create a new preview", @@ -70,4 +75,35 @@ export async function adminPlatformRoute(app: FastifyInstance) { }, { preHandler: limits.preHandler() }, ); + registerServiceAdminRoute( + app, + "applyAdminPlatformConfiguration", + async (request, reply) => { + const actor = getAdminSession(request); + if (actor.user.id === "loopback" || actor.session.impersonatedBy) + return reply.code(403).send({ error: "A real full administrator session is required" }); + const { domainProofToken, ...intent } = request.body; + const nonce = adminReauthNonce(request.headers.cookie); + if (domainProofToken && !nonce) + return reply.code(409).send({ error: "Domain recovery proof required" }); + try { + const accepted = await jobRunner.startPlatformConfiguration({ + ...intent, + actorId: actor.user.id, + operationKey: createDirectAdminOpsKey( + actor.user.id, + parseDirectOpsIdempotency(request.headers["idempotency-key"]), + ), + ...(domainProofToken && nonce + ? { proof: { challengeId: domainProofToken, sessionId: actor.session.id, nonce } } + : {}), + }); + return reply.code(202).send(accepted); + } catch (error) { + const value = failure(error); + return reply.code(value.status).send({ error: value.error }); + } + }, + { preHandler: limits.preHandler() }, + ); } diff --git a/apps/api/src/services/__tests__/job-runner-initialize.test.ts b/apps/api/src/services/__tests__/job-runner-initialize.test.ts index f09b6cedc..6e441f9f7 100644 --- a/apps/api/src/services/__tests__/job-runner-initialize.test.ts +++ b/apps/api/src/services/__tests__/job-runner-initialize.test.ts @@ -60,6 +60,10 @@ afterEach(() => { describe("job runner restart initialization", () => { it("keeps typed agent-backed jobs for observation without handler replay", async () => { expect(AGENT_RECOVERABLE_ADMIN_JOB_TYPES).toEqual([ + "standalone.adopt", + "standalone.remove", + "standalone.preview", + "standalone.install", "backup.operation", "data.operation", "service.bulk", @@ -70,8 +74,10 @@ describe("job runner restart initialization", () => { "service.apply", "compose.render", "configuration.apply", + "platform.apply", "compose.start", "integration.config.import", + "integration.binding.apply", "system.diagnostics", "system.update", ]); diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 4c42c750e..c3a38c351 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -7,6 +7,12 @@ import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } import { createRequire } from "node:module"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; +import { createApiClient } from "@openmapx/core"; +import { + applyAdminPlatformConfiguration, + previewAdminPlatformConfiguration, + unwrapAdminResponse, +} from "@openmapx/core/api-admin-sdk"; import { closeControlInstallation, controlDigest, @@ -21,7 +27,7 @@ import { import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { eq } from "drizzle-orm"; -import Fastify from "fastify"; +import Fastify, { type FastifyInstance } from "fastify"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; @@ -29,6 +35,7 @@ import { runContainedProcess } from "../../../../ops-agent/src/docker-runtime"; import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; import { buildOpsAgentServer } from "../../../../ops-agent/src/server"; import * as schema from "../../db/schema"; +import { mockAdminSession } from "../../test/auth"; import { createMigratedPostgresFixture } from "../../test/postgres-fixture"; import type { JobContext } from "../job-runner"; @@ -36,7 +43,9 @@ const state = vi.hoisted(() => ({ fixture: null as unknown as Awaited>, services: [] as services.LoadedService[], registry: null as InstanceType | null, + session: null as ReturnType | null, })); +vi.mock("../../auth", () => ({ auth: { api: { getSession: async () => state.session } } })); vi.mock("../../db", () => ({ get db() { return state.fixture.db; @@ -228,6 +237,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-after-effect", "launch-native-killed", "launch-native-producer", + "launch-native-recovery", + "launch-native-job", + "launch-native-resource", ])( "guards and retains helper mode %s through original ownership", async (version) => { @@ -270,7 +282,14 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( actualComposeConfig: true, }); state.services = host.loaded; - if (version === "launch-native-producer") { + if ( + [ + "launch-native-producer", + "launch-native-recovery", + "launch-native-job", + "launch-native-resource", + ].includes(version) + ) { state.registry = new services.ServiceRegistry({ rootDir: host.root, enabledIds: new Set(ids), @@ -391,8 +410,10 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( DOMAIN: "example.test", DATA_MANAGER_URL: dm.baseUrl, DATA_MANAGER_AUTH_TOKEN: dm.token, + OPENMAPX_DISABLE_LOCALHOST_AUTH: "1", })) vi.stubEnv(key, value); + let publicApi: FastifyInstance | undefined; try { const actorId = randomUUID(), jobId = randomUUID(), @@ -410,6 +431,113 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( serviceId: "ops-agent", config: { OPENMAPX_MEMORY_MIB: 256, OPENMAPX_CPU_LIMIT: 0.5 }, }); + if (["launch-native-job", "launch-native-resource"].includes(version)) { + const { adminPlatformRoute } = await import("../../routes/admin-platform"), + { jobRunner } = await import("../job-runner"), + { createDirectAdminOpsKey } = await import("../direct-ops-idempotency"); + state.session = mockAdminSession(); + state.session.user.id = actorId; + state.session.session.userId = actorId; + publicApi = Fastify(); + await publicApi.register(adminPlatformRoute, { prefix: "/api" }); + const client = createApiClient({ + baseUrl: await publicApi.listen({ host: "127.0.0.1", port: 0 }), + }), + headers = { "idempotency-key": operationKey }, + reviewed = unwrapAdminResponse( + await previewAdminPlatformConfiguration( + { + expectedCurrentRevisionId: host.initial.revisionId, + changes: + version === "launch-native-resource" + ? { version: 1 as const } + : { + version: 1 as const, + backupRetentionDays: { source: "database", value: 14 }, + }, + ...(version === "launch-native-resource" + ? { reconcileServiceIds: ["ops-agent" as const] } + : {}), + }, + headers, + { client }, + ), + ), + input = { + expectedCurrentRevisionId: reviewed.preview.expectedCurrentRevisionId, + previewFingerprint: reviewed.preview.previewFingerprint, + confirmations: reviewed.preview.requiredConfirmation, + }; + const apply = async () => + unwrapAdminResponse( + await applyAdminPlatformConfiguration(input, headers, { client }), + ); + const accepted = await apply(); + if (version === "launch-native-resource") { + expect(reviewed.preview.configuration).toEqual({ version: 1 }); + expect(reviewed.preview.affectedServiceIds).toEqual(["ops-agent"]); + expect(reviewed.preview.oldOrigin).toBe(reviewed.preview.newOrigin); + } + expect(await apply()).toEqual(accepted); + const readJob = async () => + ( + await state.fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, accepted.jobId)) + )[0]!; + const launchDeadline = Date.now() + 60000; + while (!host.calls.some((args) => args[0] === "run") && Date.now() < launchDeadline) { + expect((await readJob()).status).not.toBe("failed"); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + const args = host.calls.find((args) => args[0] === "run"); + expect(args).toBeDefined(); + const env = (name: string) => + args!.find((value) => value.startsWith(`${name}=`))!.slice(name.length + 1); + await nativeHelper(host, { + OPENMAPX_ROOT_DIR: host.root, + OPS_CONFIGURATION_HANDOFF_ID: env("OPS_CONFIGURATION_HANDOFF_ID"), + OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), + OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), + }); + const settleDeadline = Date.now() + 90000; + let job = await readJob(); + while (job.status === "running" && Date.now() < settleDeadline) { + await jobRunner.reconcile(); + await new Promise((resolve) => setTimeout(resolve, 250)); + job = await readJob(); + } + expect( + job.status, + JSON.stringify({ + error: job.error?.split("\n")[0].slice(0, 200), + configuration: (job.result as Record | null)?.configuration, + states: journal.records().map((value) => ({ + kind: value.operation.kind, + state: value.state, + errorClass: value.errorClass, + })), + }), + ).toBe("success"); + expect(job.payload).toEqual({ + operationKey: createDirectAdminOpsKey(actorId, operationKey), + }); + expect(job.result).toMatchObject({ + configuration: { outcome: "committed", followUp: "complete" }, + }); + expect(await apply()).toEqual(accepted); + expect(dm.settlePosts).toBe(1); + expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); + expect( + host.calls + .filter((args) => args[0] === "compose" && args.includes("up")) + .map((args) => args.at(-1)), + ).toEqual( + version === "launch-native-resource" ? ["ops-agent"] : ["app-api", "ops-agent"], + ); + return; + } await state.fixture.db.insert(schema.adminJob).values({ id: jobId, type: "platform.apply", @@ -549,7 +677,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( step.ordinal, ); expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); - if (version === "launch-native-producer") { + if (["launch-native-producer", "launch-native-recovery"].includes(version)) { const args = host.calls.find((args) => args[0] === "run")!, env = (name: string) => args @@ -561,6 +689,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), }); + if (version === "launch-native-recovery") return candidate; const result = await prospective.persistAndApplyProspectiveConfiguration( candidate, { coordinatorMode: "require-running" }, @@ -645,6 +774,29 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ); throw error; } + if (version === "launch-native-recovery") { + execution.version = 2; + ctx.checkpointResult = { + ...ctx.checkpointResult, + _execution: { ...execution }, + }; + await state.fixture.db + .update(schema.adminJob) + .set({ result: ctx.checkpointResult }) + .where(eq(schema.adminJob.id, jobId)); + const recovered = await prospective.resumeRetainedConfigurationCandidate(); + expect(recovered).toMatchObject({ + configuration: { outcome: "committed", followUp: "complete" }, + }); + expect(dm.settlePosts).toBe(1); + expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); + expect( + host.calls + .filter((args) => args[0] === "compose" && args.includes("up")) + .map((args) => args.at(-1)), + ).toEqual(["app-api", "ops-agent"]); + return; + } if (version === "policy-change" || version === "launch-native-producer") return; const retained = ctx.checkpointResult?._configurationMaintenance as | { phase?: string; receipt?: { state?: string; steps?: { phase: string }[] } } @@ -1069,16 +1221,18 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ); } finally { + await publicApi?.close(); await dm.close(); await app.close(); closeControlInstallation(installation); state.services = []; state.registry = null; + state.session = null; vi.unstubAllEnvs(); rmSync(host.cleanupRoot, { recursive: true, force: true }); } }, - 120000, + 180000, ); }, ); diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 4f1d811c1..88402aa24 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -121,6 +121,7 @@ export const AGENT_RECOVERABLE_ADMIN_JOB_TYPES = [ "service.apply", "compose.render", "configuration.apply", + "platform.apply", "compose.start", "integration.config.import", "integration.binding.apply", @@ -143,6 +144,7 @@ function hasRetainedImmediateConfiguration(job: { [ "service.apply", "configuration.apply", + "platform.apply", "integration.config.import", "integration.binding.apply", ].includes(job.type) && @@ -294,6 +296,7 @@ class AdminJobRunner { options: { type: | "configuration.apply" + | "platform.apply" | "service.apply" | "compose.start" | "integration.config.import" @@ -302,6 +305,7 @@ class AdminJobRunner { operationKey: string; intent: unknown; signal?: AbortSignal; + accepted?: (jobId: string) => void; }, run: ( ctx: JobContext, @@ -367,6 +371,7 @@ class AdminJobRunner { throw new AdminJobFailure("Retained configuration intent conflicts", { errorCode: "conflict", }); + options.accepted?.(job.id); if (!(job as { _fresh?: boolean })._fresh) { if (["success", "failed"].includes(job.status)) { if (checkpoint._configurationRejection) @@ -379,6 +384,7 @@ class AdminJobRunner { ![ "service.apply", "configuration.apply", + "platform.apply", "integration.config.import", "integration.binding.apply", ].includes(options.type) @@ -624,6 +630,41 @@ class AdminJobRunner { ); } + /** The response observes durable admission only; disconnecting HTTP cannot + * cancel the original mutation or expose its purpose-proof material. */ + startPlatformConfiguration( + options: import("./platform-configuration-application").PlatformConfigurationApplicationInput, + ) { + const accepted = Promise.withResolvers<{ ok: true; jobId: string }>(); + void this.immediate( + { + type: "platform.apply", + actorId: options.actorId, + operationKey: options.operationKey, + intent: { + purpose: "platform", + previewFingerprint: options.previewFingerprint, + expectedCurrentRevisionId: options.expectedCurrentRevisionId, + confirmations: options.confirmations, + }, + accepted: (jobId) => accepted.resolve({ ok: true, jobId }), + }, + async (ctx) => { + const { runPlatformConfigurationChange } = await import( + "./platform-configuration-application" + ); + const value = await runPlatformConfigurationChange(ctx, options); + return { + value, + result: { configuration: value.application }, + failed: !completeTrustedConfigurationApply(value.application), + }; + }, + (result) => ({ application: parseTrustedConfigurationApplyOutcome(result.configuration) }), + ).catch(accepted.reject); + return accepted.promise; + } + importImmediateIntegrationConfiguration(options: { body: unknown; actorId: string; diff --git a/apps/api/src/services/platform-configuration-application.ts b/apps/api/src/services/platform-configuration-application.ts new file mode 100644 index 000000000..40453e725 --- /dev/null +++ b/apps/api/src/services/platform-configuration-application.ts @@ -0,0 +1,62 @@ +/** Fixed platform producer, sharing the original administrative job and writer. + * Purpose proofs are transient until encrypted inside the bounded candidate. */ + +import { getAllIntegrations } from "../integration-host"; +import { + currentAdministrativeConfigurationWriter, + withAdministrativeConfigurationJob, + withAdministrativeConfigurationWriter, +} from "./administrative-configuration-writer-owner"; +import { withConfigurationCommitParticipant } from "./configuration-commit-participant"; +import type { JobContext } from "./job-runner"; +import type { PlatformConfigurationCommitInput } from "./platform-configuration"; +import { + platformConfigurationParticipation, + preparePlatformConfigurationChange, +} from "./platform-configuration-participant"; +import { + persistAndApplyProspectiveConfiguration, + prepareProspectiveConfiguration, +} from "./prospective-configuration"; + +export type PlatformConfigurationApplicationInput = Pick< + PlatformConfigurationCommitInput, + "actorId" | "previewFingerprint" | "expectedCurrentRevisionId" | "confirmations" | "proof" +> & { operationKey: string }; + +export async function applyPlatformConfigurationIntent( + input: PlatformConfigurationApplicationInput, +) { + const writer = currentAdministrativeConfigurationWriter(); + if (writer.actorId !== input.actorId || writer.operationKey !== input.operationKey) + throw Error("Platform configuration owner changed"); + const intent = await preparePlatformConfigurationChange(input, writer), + scope = platformConfigurationParticipation(intent, writer); + return withConfigurationCommitParticipant(writer, scope.participant, async () => { + const candidate = await prepareProspectiveConfiguration({ + actorId: input.actorId, + operationKey: input.operationKey, + expectedCurrentRevisionId: input.expectedCurrentRevisionId, + overlay: scope.overlay, + }); + return persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "require-running", + }); + }); +} +export async function runPlatformConfigurationChange( + ctx: JobContext, + input: PlatformConfigurationApplicationInput, +) { + return withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { + actorId: input.actorId, + operationKey: input.operationKey, + signal: ctx.signal, + integrationIds: getAllIntegrations().map((integration) => integration.id), + }, + () => applyPlatformConfigurationIntent(input), + ), + ); +} diff --git a/apps/api/src/services/platform-configuration-participant.ts b/apps/api/src/services/platform-configuration-participant.ts index c705b68e6..d9a54069e 100644 --- a/apps/api/src/services/platform-configuration-participant.ts +++ b/apps/api/src/services/platform-configuration-participant.ts @@ -105,6 +105,9 @@ export function retainedPlatformConfigurationIntent(writer: AdministrativeConfig expectedCurrentRevisionId: preview.expectedCurrentRevisionId, configuration: structuredClone(preview.configuration), affectedServiceIds: [...preview.affectedServiceIds], + ...(preview.reconcileServiceIds + ? { reconcileServiceIds: [...preview.reconcileServiceIds] } + : {}), expectedDesiredRowVersion: state.expectedDesiredRowVersion, confirmations: { ...state.input.confirmations }, ...(state.authorization ? { authorization: { ...state.authorization } } : {}), @@ -133,6 +136,7 @@ export function assertPlatformConfigurationSource( expectedCurrentRevisionId: p.expectedCurrentRevisionId, currentPlatform: source.platform, changes: p.configuration, + ...(p.reconcileServiceIds ? { reconcileServiceIds: p.reconcileServiceIds } : {}), ...input, }); if (rebuilt.previewFingerprint !== p.previewFingerprint) throw new PlatformPreviewConflict(); diff --git a/apps/api/src/services/platform-configuration-recovery.ts b/apps/api/src/services/platform-configuration-recovery.ts new file mode 100644 index 000000000..cebe8de70 --- /dev/null +++ b/apps/api/src/services/platform-configuration-recovery.ts @@ -0,0 +1,74 @@ +/** Read-only join for an already MAC-authenticated nominal saved candidate. + * This issues no participant/proof, saves nothing and cannot authorize a root. */ +import { canonicalJson, type PlatformConfigurationIntentRecord } from "@openmapx/core/ops"; +import { eq } from "drizzle-orm"; +import type { AuthDatabase } from "../account/auth-database"; +import { loadAuthPolicy } from "../account/auth-policy"; +import { systemSettings } from "../db/schema"; +import { JobObservationPending } from "./admin-job-failure"; +import { readProjection } from "./admin-job-projection"; +import { PLATFORM_CONFIGURATION_SETTING } from "./platform-configuration"; +import { readRetainedPlatformPreview } from "./platform-preview-store"; + +export async function assertSavedPlatformConfiguration( + input: { + actorId: string; + jobId: string; + candidateId: string; + revisionId: string; + intent: PlatformConfigurationIntentRecord; + checkpoint: Record | null | undefined; + }, + connection: Pick, +) { + const { intent } = input, + retained = await readRetainedPlatformPreview( + { actorId: input.actorId, previewFingerprint: intent.previewFingerprint }, + connection, + ), + p = retained.preview; + const [saved] = await connection + .select({ value: systemSettings.value }) + .from(systemSettings) + .where(eq(systemSettings.key, PLATFORM_CONFIGURATION_SETTING)); + if ( + retained.consumedByJobId !== input.jobId || + p.previewId !== intent.previewId || + p.authPolicyRevision !== intent.authPolicyRevision || + p.sourceRevision !== intent.sourceRevision || + p.expectedCurrentRevisionId !== intent.expectedCurrentRevisionId || + canonicalJson(p.configuration) !== canonicalJson(intent.configuration) || + canonicalJson(p.affectedServiceIds) !== canonicalJson(intent.affectedServiceIds) || + canonicalJson(p.reconcileServiceIds ?? null) !== + canonicalJson(intent.reconcileServiceIds ?? null) || + canonicalJson(retained.sourceSnapshot) !== canonicalJson(intent.sourceSnapshot) || + retained.desiredRowVersion !== intent.expectedDesiredRowVersion || + Object.entries(p.requiredConfirmation).some( + ([key, required]) => + required && !intent.confirmations[key as keyof typeof intent.confirmations], + ) || + (p.requiredConfirmation.domain && !intent.authorization) || + canonicalJson(saved?.value) !== + canonicalJson({ + version: 1, + jobId: input.jobId, + candidateId: input.candidateId, + revisionId: input.revisionId, + previewFingerprint: p.previewFingerprint, + configuration: p.configuration, + }) + ) + throw new JobObservationPending("invalid_identity"); + const generation = Object.values(readProjection(input.checkpoint).operations).find( + (entry) => + entry.operation?.kind === "stack.render" && + entry.operation.candidateId === input.candidateId && + entry.operation.revisionId === input.revisionId, + ); + if ( + !generation?.attempt && + (new Date(retained.expiresAt) <= new Date() || + (await loadAuthPolicy(connection)).revision !== p.authPolicyRevision) + ) + throw new JobObservationPending("recovery_required"); +} diff --git a/apps/api/src/services/platform-configuration-view.ts b/apps/api/src/services/platform-configuration-view.ts index 93bbd1c31..c721eb2c7 100644 --- a/apps/api/src/services/platform-configuration-view.ts +++ b/apps/api/src/services/platform-configuration-view.ts @@ -38,10 +38,12 @@ export async function previewPlatformConfiguration(input: { operationKey: string; expectedCurrentRevisionId: string | null; changes: services.PlatformConfiguration; + reconcileServiceIds?: readonly string[]; }) { const parsed = adminPlatformPreviewInputSchema.parse({ expectedCurrentRevisionId: input.expectedCurrentRevisionId, changes: input.changes, + ...(input.reconcileServiceIds ? { reconcileServiceIds: input.reconcileServiceIds } : {}), }), current = await source(); if (parsed.expectedCurrentRevisionId !== current.revisionId) throw new PlatformPreviewConflict(); @@ -71,6 +73,7 @@ export async function previewPlatformConfiguration(input: { expectedCurrentRevisionId: parsed.expectedCurrentRevisionId, currentPlatform: current.platform, changes: parsed.changes, + ...(parsed.reconcileServiceIds ? { reconcileServiceIds: parsed.reconcileServiceIds } : {}), env: read.env, services: read.services, resolvedServiceConfigs: resolved.values, diff --git a/apps/api/src/services/platform-preview-store.ts b/apps/api/src/services/platform-preview-store.ts index 26bad72d0..35a93305e 100644 --- a/apps/api/src/services/platform-preview-store.ts +++ b/apps/api/src/services/platform-preview-store.ts @@ -183,9 +183,10 @@ export async function retainPlatformPreview( return decode(row); }); } -export async function readPlatformPreview( +/** Historical reads confer no apply authority or renewed TTL. */ +export async function readRetainedPlatformPreview( input: { actorId: string; previewFingerprint: string; now?: Date }, - database: AuthDatabase = db, + database: Pick = db, ) { const [row] = await database .select() @@ -195,7 +196,14 @@ export async function readPlatformPreview( ) .limit(1); if (!row) throw new PlatformPreviewConflict(); - if (row.expiresAt <= (input.now ?? new Date())) - throw new PlatformPreviewConflict("Platform preview expired; create a new preview"); return decode(row); } +export async function readPlatformPreview( + input: { actorId: string; previewFingerprint: string; now?: Date }, + database: AuthDatabase = db, +) { + const retained = await readRetainedPlatformPreview(input, database); + if (new Date(retained.expiresAt) <= (input.now ?? new Date())) + throw new PlatformPreviewConflict("Platform preview expired; create a new preview"); + return retained; +} diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 7453c4341..9e27ab7d8 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -346,6 +346,24 @@ async function verifyConfigurationCommitParticipant( if (state.overlay.purpose === "integration-binding") assertIntegrationBindingOverlay(writer, state.overlay); if (state.overlay.purpose === "platform") { + if (recoveredCommittedCandidates.has(candidate)) { + const { assertSavedPlatformConfiguration } = await import( + "./platform-configuration-recovery" + ); + if (!state.record.privatePlatformIntent) throw Error("Original platform candidate required"); + await assertSavedPlatformConfiguration( + { + actorId: writer.actorId, + jobId: owner.job.jobId, + candidateId: candidate.candidateId, + revisionId: candidate.revisionId, + intent: state.record.privatePlatformIntent, + checkpoint: owner.job.checkpointResult, + }, + owner.connection, + ); + return; + } assertPlatformConfigurationOverlay(writer, state.overlay); const intent = retainedPlatformConfigurationIntent(writer); if ( @@ -1779,6 +1797,7 @@ async function restoreRetainedMaintenanceCandidate(): Promise change.kind !== "service-secret" || change.serviceId !== "data-manager", ) - : !coordinator || - overlay.purpose !== "render" || - overlay.changes.length !== 0 || - overlay.coordinatorMode !== "ensure-running") || + : coordinator + ? overlay.purpose !== "render" || + overlay.changes.length !== 0 || + overlay.coordinatorMode !== "ensure-running" + : !platform || + row?.type !== "platform.apply" || + overlay.purpose !== "platform" || + overlay.coordinatorMode !== "require-running" || + record.apiFollowUp !== "platform-apply" || + overlay.changes.length !== 1 || + overlay.changes[0]?.kind !== "platform" || + canonicalJson({ version: 1, ...overlay.changes[0].patch }) !== + canonicalJson(platform.configuration) || + platform.sourceRevision !== record.expectedSourceRevision || + platform.expectedCurrentRevisionId !== operation.expectedCurrentRevisionId) || !record.maintenanceId || !record.prepareOperationKey || record.apiJobId !== job.jobId || @@ -2083,6 +2113,24 @@ async function restoreRetainedMaintenanceCandidate(): Promise(); +/** Parent terminal/copy publication can briefly own the authenticated source + * index. Wait only around source reads, before adoption/forward writes. The + * original deadline remains authoritative; no lock is removed or stolen. */ +async function waitForHandoffSource(execution: ExecutionRef, read: () => T): Promise { + const until = Math.min(execution.deadlineAt, Date.now() + 5000); + for (;;) { + try { + return read(); + } catch (error) { + const io = error as NodeJS.ErrnoException; + if ( + io.code !== "EEXIST" || + io.syscall !== "mkdir" || + !io.path || + basename(io.path) !== ".family-writer" || + Date.now() >= until + ) + throw error; + await delay(Math.min(50, Math.max(1, until - Date.now()))); + } + } +} function immutable(value: T): T { if (value && typeof value === "object") { for (const child of Object.values(value)) immutable(child); @@ -72,7 +95,9 @@ export async function openConfigurationHandoffChild(input: { let lease: BorrowedArtifactTargetLease | undefined; try { const intent = immutable( - readConfigurationHandoffSpawnIntent(installation, execution, input.handoffId), + await waitForHandoffSource(execution, () => + readConfigurationHandoffSpawnIntent(installation, execution, input.handoffId), + ), ); if ( intent.installationId !== installation.installationId || @@ -87,13 +112,15 @@ export async function openConfigurationHandoffChild(input: { handoffId: input.handoffId, intentDigest: controlDigest(intent.operation), }, - peer = verifyFixedCliControlPeer(installation, { - assertion: input.assertion, - execution, - kind: body.kind, - recipient: "configuration-helper", - }), - parent = verifiedParentForArtifactDelegation(installation, input.assertion, execution); + { peer, parent } = await waitForHandoffSource(execution, () => ({ + peer: verifyFixedCliControlPeer(installation, { + assertion: input.assertion, + execution, + kind: body.kind, + recipient: "configuration-helper", + }), + parent: verifiedParentForArtifactDelegation(installation, input.assertion, execution), + })); if ( canonicalJson(parent.operationKey) !== canonicalJson(intent.preparer.operationKey) || parent.fingerprint !== intent.preparer.fingerprint @@ -142,7 +169,9 @@ export async function reconcileCompletedConfigurationHandoff(input: { container: existsSync("/run/secrets/openmapx-control-verifiers"), }); try { - const intent = readConfigurationHandoffSpawnIntent(installation, execution, input.handoffId), + const intent = await waitForHandoffSource(execution, () => + readConfigurationHandoffSpawnIntent(installation, execution, input.handoffId), + ), state = installationState(installation), file = configurationHandoffRecordName(input.handoffId, "completion"); if (intent.assertion !== input.assertion) throw Error("Original helper assertion required"); @@ -159,11 +188,13 @@ export async function reconcileCompletedConfigurationHandoff(input: { raw, configurationHandoffWorkerRecordSchema, ); - verifyFamilyOwnedRecord( - state, - intent.origin, - { scope: "private", namespace: "observations", container: "", name: file }, - canonicalJson(raw), + await waitForHandoffSource(execution, () => + verifyFamilyOwnedRecord( + state, + intent.origin, + { scope: "private", namespace: "observations", container: "", name: file }, + canonicalJson(raw), + ), ); if (worker.phase !== "applied" || canonicalJson(worker.execution) !== canonicalJson(execution)) throw Error("Unfinished original helper remains fenced"); diff --git a/packages/core/src/api/contracts/service-admin.test.ts b/packages/core/src/api/contracts/service-admin.test.ts index 6d12cc17d..034d75395 100644 --- a/packages/core/src/api/contracts/service-admin.test.ts +++ b/packages/core/src/api/contracts/service-admin.test.ts @@ -24,6 +24,7 @@ const inventory = [ ["downloadAdminArtifactTransferContent", "GET", "/artifact-transfers/:stageId/content"], ["getAdminPlatformConfiguration", "GET", "/platform/configuration"], ["previewAdminPlatformConfiguration", "POST", "/platform/configuration/preview"], + ["applyAdminPlatformConfiguration", "POST", "/platform/configuration/apply"], ["listAdminServices", "GET", "/services"], ["getAdminService", "GET", "/services/:id"], ["getAdminServiceConfig", "GET", "/services/:id/config"], diff --git a/packages/core/src/api/contracts/service-admin/platform.ts b/packages/core/src/api/contracts/service-admin/platform.ts index dd95cfc3f..da81aef90 100644 --- a/packages/core/src/api/contracts/service-admin/platform.ts +++ b/packages/core/src/api/contracts/service-admin/platform.ts @@ -1,5 +1,6 @@ /** Portable platform preview data; these schemas confer no host authority. */ import z from "zod/v4"; +import { protectedConfigurationReconcileIdsSchema } from "../../../ops/configuration-handoff-records"; import { platformConfigurationSchema } from "../../../services/platform-configuration"; import { adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; import { adminServiceIdempotencyHeadersSchema } from "./common"; @@ -14,6 +15,7 @@ const url = z.url().max(2048); export const adminPlatformPreviewInputSchema = z.strictObject({ expectedCurrentRevisionId: revision, changes: platformConfigurationSchema, + reconcileServiceIds: protectedConfigurationReconcileIdsSchema.optional(), }); export const adminPlatformConfirmationSchema = z.strictObject({ domain: z.boolean(), @@ -45,6 +47,7 @@ export const adminPlatformPreviewSchema = z.strictObject({ backupRetentionDays: source, }), affectedServiceIds: z.array(z.string().min(1).max(64)).max(256), + reconcileServiceIds: protectedConfigurationReconcileIdsSchema.optional(), oldOrigin: url, newOrigin: url, originConsumers: z.strictObject({ @@ -64,6 +67,7 @@ export const adminPlatformApplyInputSchema = z.strictObject({ expectedCurrentRevisionId: revision, previewFingerprint: digest, confirmations: adminPlatformConfirmationSchema, + domainProofToken: z.uuid().optional(), }); export type AdminPlatformPreview = z.infer; export const adminPlatformConfigurationSchema = adminPlatformPreviewSchema.pick({ @@ -102,4 +106,16 @@ export const platformAdminOperations = { 503: adminJson(httpErrorSchema), }, }, + applyAdminPlatformConfiguration: { + method: "POST", + path: "/api/admin/platform/configuration/apply", + auth: "admin", + request: { headers: adminServiceIdempotencyHeadersSchema, body: adminPlatformApplyInputSchema }, + responses: { + ...adminCommonResponses, + 202: adminJson(z.strictObject({ ok: z.literal(true), jobId: z.uuid() })), + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), + }, + }, } as const; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index 5cbe7c43f..fc6e85aa8 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -37,6 +37,13 @@ export async function applyAdminAuthPolicyRequest( ): Promise { return applyAdminAuthPolicyBound(url, options); } +const applyAdminPlatformConfigurationBound = bindAdminOperation("applyAdminPlatformConfiguration"); +export async function applyAdminPlatformConfigurationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return applyAdminPlatformConfigurationBound(url, options); +} const applyAdminTransitCredentialsBound = bindAdminOperation("applyAdminTransitCredentials"); export async function applyAdminTransitCredentialsRequest( url: string, diff --git a/packages/core/src/api/generated/admin/service-admin.generated.ts b/packages/core/src/api/generated/admin/service-admin.generated.ts index de9901d5f..fc2187f3a 100644 --- a/packages/core/src/api/generated/admin/service-admin.generated.ts +++ b/packages/core/src/api/generated/admin/service-admin.generated.ts @@ -10,6 +10,7 @@ import type { } from "../../admin-sdk-transport"; import { bindAdminDownloadTarget } from "../../admin-sdk-transport"; import { + applyAdminPlatformConfigurationRequest, checkAdminServicesRequest, createAdminArtifactTransferRequest, createAdminServiceBackupRequest, @@ -50,6 +51,18 @@ import { uploadAdminArtifactTransferContentRequest, } from "./operation-bindings.generated"; import type { + ApplyAdminPlatformConfiguration202, + ApplyAdminPlatformConfiguration400, + ApplyAdminPlatformConfiguration401, + ApplyAdminPlatformConfiguration403, + ApplyAdminPlatformConfiguration409, + ApplyAdminPlatformConfiguration413, + ApplyAdminPlatformConfiguration415, + ApplyAdminPlatformConfiguration429, + ApplyAdminPlatformConfiguration500, + ApplyAdminPlatformConfiguration503, + ApplyAdminPlatformConfigurationBody, + ApplyAdminPlatformConfigurationHeaders, CheckAdminServices200, CheckAdminServices400, CheckAdminServices401, @@ -2247,6 +2260,117 @@ export const getAdminPlatformConfiguration = async ( ); }; +export type applyAdminPlatformConfigurationResponse202 = { + data: ApplyAdminPlatformConfiguration202; + status: 202; +}; + +export type applyAdminPlatformConfigurationResponse400 = { + data: ApplyAdminPlatformConfiguration400; + status: 400; +}; + +export type applyAdminPlatformConfigurationResponse401 = { + data: ApplyAdminPlatformConfiguration401; + status: 401; +}; + +export type applyAdminPlatformConfigurationResponse403 = { + data: ApplyAdminPlatformConfiguration403; + status: 403; +}; + +export type applyAdminPlatformConfigurationResponse409 = { + data: ApplyAdminPlatformConfiguration409; + status: 409; +}; + +export type applyAdminPlatformConfigurationResponse413 = { + data: ApplyAdminPlatformConfiguration413; + status: 413; +}; + +export type applyAdminPlatformConfigurationResponse415 = { + data: ApplyAdminPlatformConfiguration415; + status: 415; +}; + +export type applyAdminPlatformConfigurationResponse429 = { + data: ApplyAdminPlatformConfiguration429; + status: 429; +}; + +export type applyAdminPlatformConfigurationResponse500 = { + data: ApplyAdminPlatformConfiguration500; + status: 500; +}; + +export type applyAdminPlatformConfigurationResponse503 = { + data: ApplyAdminPlatformConfiguration503; + status: 503; +}; + +export type applyAdminPlatformConfigurationResponseSuccess = + applyAdminPlatformConfigurationResponse202 & { + headers: Headers; + }; +export type applyAdminPlatformConfigurationResponseError = ( + | applyAdminPlatformConfigurationResponse400 + | applyAdminPlatformConfigurationResponse401 + | applyAdminPlatformConfigurationResponse403 + | applyAdminPlatformConfigurationResponse409 + | applyAdminPlatformConfigurationResponse413 + | applyAdminPlatformConfigurationResponse415 + | applyAdminPlatformConfigurationResponse429 + | applyAdminPlatformConfigurationResponse500 + | applyAdminPlatformConfigurationResponse503 +) & { + headers: Headers; +}; + +export type applyAdminPlatformConfigurationResponse = + | applyAdminPlatformConfigurationResponseSuccess + | applyAdminPlatformConfigurationResponseError; + +export const getApplyAdminPlatformConfigurationUrl = () => { + return `/api/admin/platform/configuration/apply`; +}; + +export const applyAdminPlatformConfiguration = async ( + applyAdminPlatformConfigurationBody: ApplyAdminPlatformConfigurationBody, + headers: ApplyAdminPlatformConfigurationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return applyAdminPlatformConfigurationRequest( + getApplyAdminPlatformConfigurationUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(applyAdminPlatformConfigurationBody), + }, + ); +}; + export type previewAdminPlatformConfigurationResponse200 = { data: PreviewAdminPlatformConfiguration200; status: 200; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration202.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration202.ts new file mode 100644 index 000000000..14a6f6db6 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration202.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration202 = { + ok: true; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration400.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration400.ts new file mode 100644 index 000000000..a1a1abbac --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration401.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration401.ts new file mode 100644 index 000000000..a109adea0 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration403.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration403.ts new file mode 100644 index 000000000..c6f4a3a87 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration409.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration409.ts new file mode 100644 index 000000000..737510f06 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration413.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration413.ts new file mode 100644 index 000000000..b02e12988 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration415.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration415.ts new file mode 100644 index 000000000..12a914d4d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration429.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration429.ts new file mode 100644 index 000000000..455f1610f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration500.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration500.ts new file mode 100644 index 000000000..38a495b5b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration503.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration503.ts new file mode 100644 index 000000000..7a4cd6871 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfiguration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfiguration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBody.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBody.ts new file mode 100644 index 000000000..8882029ef --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBody.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminPlatformConfigurationBodyConfirmations } from "./applyAdminPlatformConfigurationBodyConfirmations"; + +export type ApplyAdminPlatformConfigurationBody = { + expectedCurrentRevisionId: string | null; + /** @pattern ^[a-f0-9]{64}$ */ + previewFingerprint: string; + confirmations: ApplyAdminPlatformConfigurationBodyConfirmations; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + domainProofToken?: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBodyConfirmations.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBodyConfirmations.ts new file mode 100644 index 000000000..52e5556a4 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationBodyConfirmations.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfigurationBodyConfirmations = { + domain: boolean; + dnsTls: boolean; + oauth: boolean; + passkeys: boolean; + alternateLogin: boolean; +}; diff --git a/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationHeaders.ts b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationHeaders.ts new file mode 100644 index 000000000..4c71bbcda --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/applyAdminPlatformConfigurationHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminPlatformConfigurationHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index d9bec3d22..1903cf33a 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -8,6 +8,19 @@ export * from "./adminPreviewAdminOperation200Schema0"; export * from "./adminPreviewAdminOperationBodySchema0"; export * from "./adminRunAdminOperationBodySchema0"; export * from "./adminUpdateAdminServiceConfigBodySchema0"; +export * from "./applyAdminPlatformConfiguration202"; +export * from "./applyAdminPlatformConfiguration400"; +export * from "./applyAdminPlatformConfiguration401"; +export * from "./applyAdminPlatformConfiguration403"; +export * from "./applyAdminPlatformConfiguration409"; +export * from "./applyAdminPlatformConfiguration413"; +export * from "./applyAdminPlatformConfiguration415"; +export * from "./applyAdminPlatformConfiguration429"; +export * from "./applyAdminPlatformConfiguration500"; +export * from "./applyAdminPlatformConfiguration503"; +export * from "./applyAdminPlatformConfigurationBody"; +export * from "./applyAdminPlatformConfigurationBodyConfirmations"; +export * from "./applyAdminPlatformConfigurationHeaders"; export * from "./checkAdminServices200"; export * from "./checkAdminServices200Acquisition"; export * from "./checkAdminServices200StatusesItem"; @@ -469,6 +482,7 @@ export * from "./previewAdminPlatformConfiguration200PreviewConfigurationDomain" export * from "./previewAdminPlatformConfiguration200PreviewConfigurationReleaseSelector"; export * from "./previewAdminPlatformConfiguration200PreviewConfigurationVersion"; export * from "./previewAdminPlatformConfiguration200PreviewOriginConsumers"; +export * from "./previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem"; export * from "./previewAdminPlatformConfiguration200PreviewRequiredConfirmation"; export * from "./previewAdminPlatformConfiguration200PreviewSources"; export * from "./previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail"; @@ -495,6 +509,7 @@ export * from "./previewAdminPlatformConfigurationBodyChangesDefaultRegion"; export * from "./previewAdminPlatformConfigurationBodyChangesDomain"; export * from "./previewAdminPlatformConfigurationBodyChangesReleaseSelector"; export * from "./previewAdminPlatformConfigurationBodyChangesVersion"; +export * from "./previewAdminPlatformConfigurationBodyReconcileServiceIdsItem"; export * from "./previewAdminPlatformConfigurationHeaders"; export * from "./provisionAdminDawarich200"; export * from "./provisionAdminDawarich200OauthClient"; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts index ea72c0092..58c398045 100644 --- a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts @@ -2,6 +2,7 @@ // Generated by pnpm sdk:generate. Do not edit. import type { PreviewAdminPlatformConfiguration200PreviewConfiguration } from "./previewAdminPlatformConfiguration200PreviewConfiguration"; import type { PreviewAdminPlatformConfiguration200PreviewOriginConsumers } from "./previewAdminPlatformConfiguration200PreviewOriginConsumers"; +import type { PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem } from "./previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem"; import type { PreviewAdminPlatformConfiguration200PreviewRequiredConfirmation } from "./previewAdminPlatformConfiguration200PreviewRequiredConfirmation"; import type { PreviewAdminPlatformConfiguration200PreviewSources } from "./previewAdminPlatformConfiguration200PreviewSources"; import type { PreviewAdminPlatformConfiguration200PreviewValues } from "./previewAdminPlatformConfiguration200PreviewValues"; @@ -30,6 +31,11 @@ export type PreviewAdminPlatformConfiguration200Preview = { * @items.maxLength 64 */ affectedServiceIds: string[]; + /** + * @minItems 1 + * @maxItems 6 + */ + reconcileServiceIds?: PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem[]; /** @maxLength 2048 */ oldOrigin: string; /** @maxLength 2048 */ diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem.ts new file mode 100644 index 000000000..4642fba2f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem = + (typeof PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem)[keyof typeof PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem]; + +export const PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem = { + postgis: "postgis", + redis: "redis", + "app-web": "app-web", + "app-api": "app-api", + traefik: "traefik", + "ops-agent": "ops-agent", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts index aecbc9745..501b7b137 100644 --- a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts @@ -1,8 +1,14 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. import type { PreviewAdminPlatformConfigurationBodyChanges } from "./previewAdminPlatformConfigurationBodyChanges"; +import type { PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem } from "./previewAdminPlatformConfigurationBodyReconcileServiceIdsItem"; export type PreviewAdminPlatformConfigurationBody = { expectedCurrentRevisionId: string | null; changes: PreviewAdminPlatformConfigurationBodyChanges; + /** + * @minItems 1 + * @maxItems 6 + */ + reconcileServiceIds?: PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem[]; }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyReconcileServiceIdsItem.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyReconcileServiceIdsItem.ts new file mode 100644 index 000000000..7eb2cd538 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyReconcileServiceIdsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem = + (typeof PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem)[keyof typeof PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem]; + +export const PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem = { + postgis: "postgis", + redis: "redis", + "app-web": "app-web", + "app-api": "app-api", + traefik: "traefik", + "ops-agent": "ops-agent", +} as const; diff --git a/packages/core/src/ops/configuration-handoff-records.ts b/packages/core/src/ops/configuration-handoff-records.ts index 9f0369c22..da48750b7 100644 --- a/packages/core/src/ops/configuration-handoff-records.ts +++ b/packages/core/src/ops/configuration-handoff-records.ts @@ -13,6 +13,12 @@ export const protectedConfigurationServiceIdSchema = z.enum([ "ops-agent", ]); export type ProtectedConfigurationServiceId = z.infer; +/** Explicit reconciliation of already-saved protected settings, never a new image selection. */ +export const protectedConfigurationReconcileIdsSchema = z + .array(protectedConfigurationServiceIdSchema) + .min(1) + .max(6) + .refine((ids) => ids.every((id, index) => index === 0 || ids[index - 1]! < id)); export const configurationHandoffModeSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("configuration") }), z.strictObject({ diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts index 7e00b75a2..9afdc900c 100644 --- a/packages/core/src/ops/configuration-preparation-records.ts +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -1,6 +1,7 @@ /** Portable bounded candidate records; parsing creates no authority. */ import { z } from "zod"; import { platformConfigurationSchema } from "../services/platform-configuration"; +import { protectedConfigurationReconcileIdsSchema } from "./configuration-handoff-records"; import { coordinatorLifecycleMetadataSchema } from "./coordinator-lifecycle-records"; const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); @@ -57,6 +58,7 @@ export const platformConfigurationIntentSchema = z .array(id) .max(256) .refine((ids) => ids.every((v, i) => i === 0 || ids[i - 1]! < v)), + reconcileServiceIds: protectedConfigurationReconcileIdsSchema.optional(), expectedDesiredRowVersion: z .string() .regex(/^\d{1,10}$/) diff --git a/packages/core/src/services/__tests__/platform-preview.test.ts b/packages/core/src/services/__tests__/platform-preview.test.ts index 78757e596..8872d39d5 100644 --- a/packages/core/src/services/__tests__/platform-preview.test.ts +++ b/packages/core/src/services/__tests__/platform-preview.test.ts @@ -39,6 +39,30 @@ const input = () => ({ services, }); describe("platform configuration preview", () => { + it("previews an explicit protected resource reconciliation without inventing a global setting change", () => { + const base = { ...input(), changes: { version: 1 as const } }; + const preview = buildPlatformConfigurationPreview({ + ...base, + reconcileServiceIds: ["ops-agent"], + }); + expect(preview.affectedServiceIds).toEqual(["ops-agent"]); + expect(preview.configuration).toEqual({ version: 1 }); + expect(preview.oldOrigin).toBe(preview.newOrigin); + expect(preview.requiredConfirmation.domain).toBe(false); + expect(preview.previewFingerprint).not.toBe( + buildPlatformConfigurationPreview(base).previewFingerprint, + ); + expect(() => + buildPlatformConfigurationPreview({ ...base, reconcileServiceIds: ["data-manager"] }), + ).toThrow(); + expect(() => + buildPlatformConfigurationPreview({ + ...base, + services: services.map((service) => ({ ...service, enabled: false })), + reconcileServiceIds: ["ops-agent"], + }), + ).toThrow(); + }); it("shows exact public origin consumers and requires domain recovery confirmation", () => { const source = input(); const preview = buildPlatformConfigurationPreview(source); diff --git a/packages/core/src/services/platform-preview.ts b/packages/core/src/services/platform-preview.ts index f4d4deeaa..889eb56dd 100644 --- a/packages/core/src/services/platform-preview.ts +++ b/packages/core/src/services/platform-preview.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import z from "zod/v4"; import { canonicalJson } from "../ops/canonical-json"; +import { protectedConfigurationReconcileIdsSchema } from "../ops/configuration-handoff-records"; import { resolveProxyHost } from "./compose-renderer"; import { type PlatformConfiguration, @@ -33,6 +34,7 @@ export function buildPlatformConfigurationPreview(input: { env: Readonly>; services: readonly LoadedService[]; resolvedServiceConfigs?: Map>; + reconcileServiceIds?: readonly string[]; }) { const currentRevisionId = revision.parse(input.currentRevisionId); if (revision.parse(input.expectedCurrentRevisionId) !== currentRevisionId) @@ -45,6 +47,19 @@ export function buildPlatformConfigurationPreview(input: { .parse(input.authPolicyRevision); const sourceRevision = z.uuid().parse(input.sourceRevision); const changes = platformConfigurationSchema.parse(input.changes); + const reconcileServiceIds = + input.reconcileServiceIds === undefined + ? undefined + : protectedConfigurationReconcileIdsSchema.parse(input.reconcileServiceIds); + if ( + reconcileServiceIds?.some( + (id) => + !input.services.some( + (service) => service.isBuiltIn && service.enabled && service.manifest.id === id, + ), + ) + ) + throw Error("Protected reconciliation requires an enabled built-in service"); const configuration = platformConfigurationSchema.parse({ ...input.currentPlatform, ...changes, @@ -60,6 +75,7 @@ export function buildPlatformConfigurationPreview(input: { .filter((service) => { if (!service.isBuiltIn || !service.enabled) return false; const m = service.manifest; + if (reconcileServiceIds?.includes(m.id as (typeof reconcileServiceIds)[number])) return true; const consumer = JSON.stringify({ environment: m.container.environment, command: m.container.command, @@ -122,6 +138,7 @@ export function buildPlatformConfigurationPreview(input: { values: after.values, sources: after.sources, affectedServiceIds, + ...(reconcileServiceIds ? { reconcileServiceIds } : {}), oldOrigin, newOrigin, originConsumers, From 777a4df7d3bc264fa6a3a5fb225290a7ee501ad0 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 09:09:32 +0100 Subject: [PATCH 074/135] fix(platform): serialize helper source reads and release inspection --- ...orm-configuration-runtime-postgres.test.ts | 46 +++++++++++++++++-- .../src/configuration-handoff-control.ts | 18 +++++--- ...iguration-maintenance-docker-inspection.ts | 7 ++- .../src/build-artifacts/target-lease.test.ts | 34 +++++++++++++- .../core/src/build-artifacts/target-lease.ts | 11 ++++- 5 files changed, 101 insertions(+), 15 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index c3a38c351..f6bffa036 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -1,6 +1,6 @@ -/** Actual API candidate -> production registry/agent -> real DM HTTP/SQL guard. - * Docker I/O is substituted. This qualifies preparation, launch reconciliation - * and real child adoption; not native helper application/public platform HTTP. */ +/** Actual API/SDK -> production registry/agent -> real DM HTTP/SQL guard and + * separate native helper. Only the session provider and Docker executable I/O + * are substituted; image/container replacement acceptance remains separate. */ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; @@ -25,7 +25,7 @@ import { verifyRetainedArtifactTargetSettlement, } from "@openmapx/core/build-artifacts"; import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; -import { services } from "@openmapx/core/server"; +import { repoPaths, services } from "@openmapx/core/server"; import { eq } from "drizzle-orm"; import Fastify, { type FastifyInstance } from "fastify"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -775,6 +775,44 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( throw error; } if (version === "launch-native-recovery") { + const { readOwnedConfigurationMaintenance } = await import( + "../../../../ops-agent/src/configuration-maintenance-store" + ), + { inspectMaintenanceDockerRuntime } = await import( + "../../../../ops-agent/src/configuration-maintenance-docker-inspection" + ), + { installationState } = await import( + "../../../../../packages/core/src/build-artifacts/control-authority" + ), + record = prospective.validatedConfigurationCandidateState(candidate).record, + receipt = await prospective.inspectConfigurationMaintenance({ + maintenanceId: record.maintenanceId!, + candidateId: candidate.candidateId, + }), + authority = restoreAdmittedControlExecution(installation, receipt.prepare), + owner = readOwnedConfigurationMaintenance(authority), + containers = await host.owned.inspectOwnedServices( + new AbortController().signal, + ), + lock = await services.acquireReleaseStoreLock( + join(host.infraDir, ".ops-agent-releases"), + ); + const callsBeforeInspection = host.calls.length; + try { + await expect( + inspectMaintenanceDockerRuntime({ + stack: repoPaths(host.root), + execFile: host.execFile, + owner, + containers, + privateKey: installationState(installation).privateKey, + signal: new AbortController().signal, + }), + ).rejects.toThrow("Release store is busy"); + expect(host.calls).toHaveLength(callsBeforeInspection); + } finally { + lock.release(); + } execution.version = 2; ctx.checkpointResult = { ...ctx.checkpointResult, diff --git a/apps/ops-agent/src/configuration-handoff-control.ts b/apps/ops-agent/src/configuration-handoff-control.ts index 9871e95cb..a186a44e0 100644 --- a/apps/ops-agent/src/configuration-handoff-control.ts +++ b/apps/ops-agent/src/configuration-handoff-control.ts @@ -4,6 +4,7 @@ import { existsSync } from "node:fs"; import { basename, join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { + ArtifactTargetJournalBusyError, adoptArtifactTargetLease, assertArtifactTargetLeaseExecution, assertConfigurationHandoffHostSources as assertHostSources, @@ -53,21 +54,24 @@ const children = new WeakMap< * index. Wait only around source reads, before adoption/forward writes. The * original deadline remains authoritative; no lock is removed or stolen. */ async function waitForHandoffSource(execution: ExecutionRef, read: () => T): Promise { - const until = Math.min(execution.deadlineAt, Date.now() + 5000); + const started = performance.now(); for (;;) { try { return read(); } catch (error) { const io = error as NodeJS.ErrnoException; + const familyBusy = + io.code === "EEXIST" && + io.syscall === "mkdir" && + !!io.path && + basename(io.path) === ".family-writer"; if ( - io.code !== "EEXIST" || - io.syscall !== "mkdir" || - !io.path || - basename(io.path) !== ".family-writer" || - Date.now() >= until + (!familyBusy && !(error instanceof ArtifactTargetJournalBusyError)) || + Date.now() >= execution.deadlineAt || + performance.now() - started >= 5000 ) throw error; - await delay(Math.min(50, Math.max(1, until - Date.now()))); + await delay(50); } } } diff --git a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts index 3ac8229f1..cf13d3216 100644 --- a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts +++ b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts @@ -3,6 +3,7 @@ import { createHmac, type KeyObject, timingSafeEqual } from "node:crypto"; import { constants } from "node:fs"; import { lstat, open, realpath } from "node:fs/promises"; +import { join } from "node:path"; import { canonicalJson } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { @@ -59,7 +60,11 @@ export async function inspectMaintenanceDockerRuntime(options: { .update("openmapx-private-maintenance-runtime-v1\0") .update(value) .digest("hex"), - lock = await services.acquireReleaseStoreLock(options.stack.infraDir, {}, { failIfBusy: true }); + lock = await services.acquireReleaseStoreLock( + join(options.stack.infraDir, ".ops-agent-releases"), + {}, + { failIfBusy: true }, + ); try { if ( !inspectedRevision || diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index 078678bed..8aa0225d1 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -1,5 +1,13 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmdirSync, + rmSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createControlRootOperationKey } from "@openmapx/core/ops"; @@ -237,6 +245,30 @@ async function api() { ).toBe(true); return import("./target-lease"); } +it("distinguishes an occupied original journal guard before running its source check", async () => { + const a = await api(), + authority = owner(), + lease = await a.acquireArtifactTargetLease({ + dataRoot: join(root, "data"), + serviceIds: ["osrm"], + owner: authority, + }); + const directory = join(root, "data/.admin-target-control/leases", lease.leaseId), + guard = join(directory, ".journal-writer"), + before = readFileSync(join(directory, "owner.json")); + mkdirSync(guard, { mode: 0o700 }); + try { + expect(() => a.assertArtifactTargetLeaseForward(lease)).toThrow( + a.ArtifactTargetJournalBusyError, + ); + expect(readFileSync(join(directory, "owner.json"))).toEqual(before); + expect(existsSync(guard)).toBe(true); + } finally { + rmdirSync(guard); + } + a.assertArtifactTargetLeaseForward(lease); + a.closeArtifactTargetLeaseHandle(lease); +}); describe("certified target leases", () => { it("requires nominal durable ownership, then fences unknown own effects", async () => { const a = await api(); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index e28893cf0..ef6154523 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -718,6 +718,13 @@ function writeBounded( if (dir !== s.ledger) dir.close(); } } +/** Only failure to acquire the existing journal guard, before its body runs. + * This is coordination evidence, never permission to reclaim a held lock. */ +export class ArtifactTargetJournalBusyError extends Error { + constructor(cause: unknown) { + super("Lease journal writer is busy or requires supervised recovery", { cause }); + } +} function guarded(s: State, run: () => T, checkTargets = true): T { if (s.journalGuardHeld) throw Error("Nested lease journal writer"); if (checkTargets) verifyAnchors(s); @@ -729,8 +736,8 @@ function guarded(s: State, run: () => T, checkTargets = true): T { try { mkdirSync(path, { mode: 0o700 }); fsyncSync(s.ledger.fd); - } catch { - throw Error("Lease journal writer is busy or requires supervised recovery"); + } catch (cause) { + throw new ArtifactTargetJournalBusyError(cause); } const guard = s.ledger.directory(".journal-writer"); const expected = identity(guard.stat); From 00fc4737f27fc95a3e4c02ee58e642ecefa831f5 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 09:10:52 +0100 Subject: [PATCH 075/135] feat(platform): validate checked release sources for guarded handoffs --- .../src/configuration-release.test.ts | 98 +++++++++++++++++++ apps/ops-agent/src/configuration-release.ts | 61 ++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 apps/ops-agent/src/configuration-release.test.ts create mode 100644 apps/ops-agent/src/configuration-release.ts diff --git a/apps/ops-agent/src/configuration-release.test.ts b/apps/ops-agent/src/configuration-release.test.ts new file mode 100644 index 000000000..1571fd8ad --- /dev/null +++ b/apps/ops-agent/src/configuration-release.test.ts @@ -0,0 +1,98 @@ +import { createHash } from "node:crypto"; +import { linkSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { readCheckedConfigurationRelease } from "./configuration-release"; + +let root: string, directory: string, raw: string, manifestDigest: string; +const releaseId = "checked-release-01"; +beforeEach(() => { + vi.stubEnv("OPENMAPX_RELEASE", "main"); + vi.stubEnv("OPENMAPX_RELEASE_MANIFEST_REPOSITORY", "ghcr.io/openmapx/release-manifest"); + root = mkdtempSync(join(tmpdir(), "openmapx-configuration-release-")); + directory = join(root, ".ops-agent-releases"); + mkdirSync(directory, { mode: 0o700 }); + raw = services.canonicalReleaseManifest( + services.parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: releaseId, + images: Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`, + ]), + ), + privacyReleaseValidation: { + version: 1, + sourceBuildFingerprint: "b".repeat(64), + validatedAt: "2026-10-30T00:00:00.000Z", + checks: { translationsConsistent: true, openApiConsistent: true, policyConsistent: true }, + }, + }), + ), + ); + manifestDigest = createHash("sha256").update(raw).digest("hex"); + writeFileSync(join(directory, `${releaseId}.json`), raw, { mode: 0o600 }); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); + vi.unstubAllEnvs(); +}); +it("loads only the exact checked canonical manifest and derives the finite pinned map", () => { + const release = readCheckedConfigurationRelease(root, { releaseId, manifestDigest }); + expect(release).toMatchObject({ + releaseId, + manifestDigest, + serviceImages: { + "app-api": `ghcr.io/openmapx/api@sha256:${"a".repeat(64)}`, + "app-web": `ghcr.io/openmapx/web@sha256:${"a".repeat(64)}`, + "ops-agent": `ghcr.io/openmapx/ops-agent@sha256:${"a".repeat(64)}`, + "data-manager": `ghcr.io/openmapx/data-manager@sha256:${"a".repeat(64)}`, + "transitous-runner": `ghcr.io/openmapx/transitous-runner@sha256:${"a".repeat(64)}`, + }, + }); + expect(Object.isFrozen(release)).toBe(true); +}); +it.each(["../outside", "/absolute", "checked/child"])( + "rejects non-identity input %s", + (invalid) => { + expect(() => + readCheckedConfigurationRelease(root, { releaseId: invalid, manifestDigest }), + ).toThrow(); + }, +); +it("rejects digest drift and noncanonical cached bytes", () => { + expect(() => + readCheckedConfigurationRelease(root, { releaseId, manifestDigest: "0".repeat(64) }), + ).toThrow(); + writeFileSync(join(directory, `${releaseId}.json`), `${raw}\n`); + expect(() => readCheckedConfigurationRelease(root, { releaseId, manifestDigest })).toThrow(); +}); +it.each(["symlink", "hardlink"])("rejects a %s cached manifest", (kind) => { + const path = join(directory, `${releaseId}.json`), + original = join(root, "original.json"); + writeFileSync(original, raw, { mode: 0o600 }); + rmSync(path); + if (kind === "symlink") symlinkSync(original, path); + else linkSync(original, path); + expect(() => readCheckedConfigurationRelease(root, { releaseId, manifestDigest })).toThrow(); +}); +it.each(["foreign", "tag", "oversized", "privacy", "local"])( + "rejects ineligible checked data: %s", + (kind) => { + const manifest = JSON.parse(raw); + if (kind === "foreign") manifest.images.api = `ghcr.io/foreign/api@sha256:${"a".repeat(64)}`; + if (kind === "tag") manifest.images.api = "ghcr.io/openmapx/api:latest"; + if (kind === "privacy") manifest.privacyReleaseValidation.checks.policyConsistent = false; + if (kind === "local") vi.stubEnv("OPENMAPX_RELEASE", "local"); + const modified = JSON.stringify(manifest) + (kind === "oversized" ? " ".repeat(32768) : ""); + writeFileSync(join(directory, `${releaseId}.json`), modified); + const digest = createHash("sha256").update(modified).digest("hex"); + expect(() => + readCheckedConfigurationRelease(root, { releaseId, manifestDigest: digest }), + ).toThrow(); + }, +); diff --git a/apps/ops-agent/src/configuration-release.ts b/apps/ops-agent/src/configuration-release.ts new file mode 100644 index 000000000..329e3192c --- /dev/null +++ b/apps/ops-agent/src/configuration-release.ts @@ -0,0 +1,61 @@ +/** Checked release data only. Reading this does not acquire targets or grant effects. */ +import { createHash } from "node:crypto"; +import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import { z } from "zod"; + +const selectionSchema = z.strictObject({ + releaseId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/), + manifestDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +function immutable(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) immutable(child); + Object.freeze(value); + } + return value; +} +/** Snapshot the existing private checked store, under its existing32KiB limit. + * Effect producers must revalidate this same identity under their real locks; + * neither the parsed data nor a matching digest is mutation authority. */ +export function readCheckedConfigurationRelease( + infraDir: string, + input: { releaseId: string; manifestDigest: string }, +) { + const selected = selectionSchema.parse(input), + channel = services.releaseSelection(); + if (channel.kind === "local") throw new services.ReleaseChannelDisabledError(); + const infra = new AnchoredDirectory(infraDir); + try { + const directory = infra.directory(".ops-agent-releases", false, true); + try { + const raw = directory.read(`${selected.releaseId}.json`, 32 * 1024).toString("utf8"), + manifest = services.parseReleaseManifest(raw, channel.imagePrefix), + canonical = services.canonicalReleaseManifest(manifest), + digest = createHash("sha256").update(canonical).digest("hex"); + if ( + manifest.release !== selected.releaseId || + canonical !== raw || + digest !== selected.manifestDigest + ) + throw Error("Checked configuration release changed"); + return immutable({ + releaseId: selected.releaseId, + manifestDigest: digest, + manifest, + serviceImages: { + "app-api": manifest.images.api, + "app-web": manifest.images.web, + "data-manager": manifest.images["data-manager"], + "ops-agent": manifest.images["ops-agent"], + "transitous-runner": manifest.images["transitous-runner"], + }, + }); + } finally { + directory.close(); + } + } finally { + infra.close(); + } +} +export type CheckedConfigurationRelease = ReturnType; From bbe350e9b13dadff0442218fde48533ffadaea83 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 09:30:02 +0100 Subject: [PATCH 076/135] feat(platform): bound checked release download metadata before pulls --- .../src/configuration-release-downloads.ts | 121 ++++++++++++++++++ .../src/configuration-release.test.ts | 106 +++++++++++++++ .../ops/configuration-release-downloads.ts | 43 +++++++ packages/core/src/ops/index.ts | 1 + 4 files changed, 271 insertions(+) create mode 100644 apps/ops-agent/src/configuration-release-downloads.ts create mode 100644 packages/core/src/ops/configuration-release-downloads.ts diff --git a/apps/ops-agent/src/configuration-release-downloads.ts b/apps/ops-agent/src/configuration-release-downloads.ts new file mode 100644 index 000000000..99a39894a --- /dev/null +++ b/apps/ops-agent/src/configuration-release-downloads.ts @@ -0,0 +1,121 @@ +/** Read-only Docker/OCI metadata preflight; no image pull or target acquisition. + * Shapes follow Docker manifest inspect and OCI image index/manifest descriptors: + * https://docs.docker.com/reference/cli/docker/manifest/ + * https://github.com/opencontainers/image-spec/blob/main/manifest.md */ +import { + CONFIGURATION_RELEASE_IMAGE_BYTES, + CONFIGURATION_RELEASE_IMAGE_NAMES, + type ConfigurationReleaseDownloads, + configurationReleaseDownloadsSchema, + configurationReleasePlatformSchema, +} from "@openmapx/core/ops"; +import { z } from "zod"; +import type { CheckedConfigurationRelease } from "./configuration-release"; +import type { DockerRuntimeOptions } from "./docker-runtime"; + +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/), + descriptor = z.object({ + digest, + size: z.number().int().nonnegative().safe().max(CONFIGURATION_RELEASE_IMAGE_BYTES), + }), + manifestSchema = z.object({ + schemaVersion: z.literal(2), + config: descriptor.extend({ + size: z + .number() + .int() + .positive() + .max(1024 * 1024), + }), + layers: z.array(descriptor).max(2048), + manifests: z.never().optional(), + }), + indexSchema = z.object({ + schemaVersion: z.literal(2), + manifests: z + .array( + descriptor.extend({ + platform: z + .object({ + os: z.string().max(32), + architecture: z.string().max(32), + variant: z.string().max(32).optional(), + }) + .optional(), + }), + ) + .min(1) + .max(256), + config: z.never().optional(), + layers: z.never().optional(), + }); + +export async function inspectConfigurationReleaseDownloads(input: { + release: CheckedConfigurationRelease; + execFile: NonNullable; + signal: AbortSignal; +}): Promise { + const run = async (args: readonly string[]) => { + input.signal.throwIfAborted(); + const result = await input.execFile("docker", args, { + signal: AbortSignal.any([input.signal, AbortSignal.timeout(15000)]), + timeout: 15000, + maxBuffer: 1024 * 1024, + }); + input.signal.throwIfAborted(); + if (Buffer.byteLength(result.stdout) > 1024 * 1024) + throw Error("Release metadata capacity exceeded"); + return result.stdout; + }; + const observed = (await run(["info", "--format", "{{.OSType}}/{{.Architecture}}"])).trim(), + platform = configurationReleasePlatformSchema.parse( + observed === "linux/x86_64" + ? "linux/amd64" + : observed === "linux/aarch64" + ? "linux/arm64" + : observed, + ), + [os, architecture] = platform.split("/"), + images = []; + for (const image of CONFIGURATION_RELEASE_IMAGE_NAMES) { + const reference = input.release.manifest.images[image]; + let body: unknown = JSON.parse(await run(["manifest", "inspect", reference])), + platformManifestDigest = digest.parse(reference.slice(reference.lastIndexOf("@") + 1)); + if (body && typeof body === "object" && Object.hasOwn(body, "manifests")) { + const index = indexSchema.parse(body), + matches = index.manifests.filter( + (item) => + item.platform?.os === os && + item.platform.architecture === architecture && + (item.platform.variant === undefined || + (architecture === "arm64" && item.platform.variant === "v8")), + ); + if (matches.length !== 1) throw Error("Release platform is unavailable or ambiguous"); + platformManifestDigest = matches[0]!.digest; + const selected = `${reference.slice(0, reference.lastIndexOf("@"))}@${platformManifestDigest}`; + body = JSON.parse(await run(["manifest", "inspect", selected])); + } + const manifest = manifestSchema.parse(body), + declaredBytes = + manifest.config.size + manifest.layers.reduce((sum, layer) => sum + layer.size, 0); + if (manifest.config.size < 1) throw Error("Release image configuration is empty"); + images.push({ + image, + reference, + platformManifestDigest, + configDigest: manifest.config.digest, + declaredBytes, + }); + } + // Declared config/layer blob bytes, not an assertion about Docker's + // unpacked filesystem size. Actual pulled images still require verification. + const snapshot = configurationReleaseDownloadsSchema.parse({ + version: 1, + platform, + images, + declaredBytes: images.reduce((sum, image) => sum + image.declaredBytes, 0), + }); + snapshot.images.forEach(Object.freeze); + Object.freeze(snapshot.images); + return Object.freeze(snapshot); +} diff --git a/apps/ops-agent/src/configuration-release.test.ts b/apps/ops-agent/src/configuration-release.test.ts index 1571fd8ad..62cd8281b 100644 --- a/apps/ops-agent/src/configuration-release.test.ts +++ b/apps/ops-agent/src/configuration-release.test.ts @@ -2,9 +2,11 @@ import { createHash } from "node:crypto"; import { linkSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { CONFIGURATION_RELEASE_IMAGE_NAMES } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { readCheckedConfigurationRelease } from "./configuration-release"; +import { inspectConfigurationReleaseDownloads } from "./configuration-release-downloads"; let root: string, directory: string, raw: string, manifestDigest: string; const releaseId = "checked-release-01"; @@ -96,3 +98,107 @@ it.each(["foreign", "tag", "oversized", "privacy", "local"])( ).toThrow(); }, ); +it.each(["single", "indexed"])( + "preflights declared OCI bytes and platform without pulling images: %s", + async (kind) => { + expect(CONFIGURATION_RELEASE_IMAGE_NAMES).toEqual(services.RELEASE_IMAGE_NAMES); + const release = readCheckedConfigurationRelease(root, { releaseId, manifestDigest }); + const execFile = vi.fn(async (_file: string, args: readonly string[]) => { + let stdout: string; + if (args[0] === "info") stdout = "linux/x86_64"; + else if (args[0] === "manifest" && args[1] === "inspect") { + const reference = args[2]!; + stdout = JSON.stringify( + kind === "indexed" && reference.endsWith(`@sha256:${"a".repeat(64)}`) + ? { + schemaVersion: 2, + manifests: [ + { + digest: `sha256:${"b".repeat(64)}`, + size: 512, + platform: { os: "linux", architecture: "amd64" }, + }, + ], + } + : { + schemaVersion: 2, + config: { size: 256, digest: `sha256:${"c".repeat(64)}` }, + layers: [{ size: 1024, digest: `sha256:${"d".repeat(64)}` }], + }, + ); + } else throw Error("Unexpected Docker mutation during preflight"); + return { stdout, stderr: "" }; + }); + const result = await inspectConfigurationReleaseDownloads({ + release, + execFile, + signal: new AbortController().signal, + }); + expect(result).toMatchObject({ version: 1, platform: "linux/amd64", declaredBytes: 7 * 1280 }); + expect(result.images).toHaveLength(7); + expect(result.images[0]).toMatchObject({ + image: "api", + reference: release.manifest.images.api, + platformManifestDigest: `sha256:${(kind === "indexed" ? "b" : "a").repeat(64)}`, + configDigest: `sha256:${"c".repeat(64)}`, + }); + expect( + execFile.mock.calls.every( + ([file, args]) => file === "docker" && ["info", "manifest"].includes(args[0]!), + ), + ).toBe(true); + }, +); +it.each([ + "platform", + "unavailable-index", + "ambiguous", + "oversized-image", + "oversized-total", + "malformed", + "aborted", + "too-much-output", +])("refuses unsafe download preflight without a pull: %s", async (kind) => { + const release = readCheckedConfigurationRelease(root, { releaseId, manifestDigest }), + controller = new AbortController(); + if (kind === "aborted") controller.abort(Error("Synthetic observation canceled")); + const execFile = vi.fn(async (_file: string, args: readonly string[]) => { + if (args[0] === "info") + return { stdout: kind === "platform" ? "windows/amd64" : "linux/amd64", stderr: "" }; + if (args[0] !== "manifest" || args[1] !== "inspect") throw Error("Unexpected mutation"); + let value: unknown = { + schemaVersion: 2, + config: { size: 256, digest: `sha256:${"c".repeat(64)}` }, + layers: [ + { + size: + kind === "oversized-image" + ? 8 * 1024 ** 3 + 1 + : kind === "oversized-total" + ? 5 * 1024 ** 3 + : 1024, + digest: `sha256:${"d".repeat(64)}`, + }, + ], + ...(kind === "too-much-output" ? { annotations: { value: "x".repeat(1024 * 1024) } } : {}), + }; + if (kind === "malformed") value = { schemaVersion: 2, layers: [] }; + if (kind === "ambiguous" || kind === "unavailable-index") + value = { + schemaVersion: 2, + manifests: Array.from({ length: kind === "ambiguous" ? 2 : 1 }, () => ({ + size: 512, + digest: `sha256:${"b".repeat(64)}`, + platform: { os: "linux", architecture: kind === "unavailable-index" ? "arm64" : "amd64" }, + })), + }; + return { stdout: JSON.stringify(value), stderr: "" }; + }); + await expect( + inspectConfigurationReleaseDownloads({ release, execFile, signal: controller.signal }), + ).rejects.toThrow(); + expect(execFile.mock.calls.every(([, args]) => ["info", "manifest"].includes(args[0]!))).toBe( + true, + ); + if (kind === "aborted") expect(execFile).not.toHaveBeenCalled(); +}); diff --git a/packages/core/src/ops/configuration-release-downloads.ts b/packages/core/src/ops/configuration-release-downloads.ts new file mode 100644 index 000000000..51bdf1ba3 --- /dev/null +++ b/packages/core/src/ops/configuration-release-downloads.ts @@ -0,0 +1,43 @@ +/** Bounded public registry metadata. These records never supply host authority. */ +import { z } from "zod"; +export const CONFIGURATION_RELEASE_IMAGE_NAMES = [ + "api", + "web", + "data-manager", + "ops-agent", + "privacy-backup", + "transitous-runner", + "transitous-tools", +] as const; +export const CONFIGURATION_RELEASE_IMAGE_BYTES = 8 * 1024 ** 3; +export const CONFIGURATION_RELEASE_TOTAL_BYTES = 32 * 1024 ** 3; +export const configurationReleasePlatformSchema = z.enum(["linux/amd64", "linux/arm64"]); +export const configurationReleaseDownloadSchema = z.strictObject({ + image: z.enum(CONFIGURATION_RELEASE_IMAGE_NAMES), + reference: z + .string() + .max(512) + .regex(/^[a-z0-9][a-z0-9._:/-]*@sha256:[a-f0-9]{64}$/), + platformManifestDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + configDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + declaredBytes: z.number().int().nonnegative().safe().max(CONFIGURATION_RELEASE_IMAGE_BYTES), +}); +export const configurationReleaseDownloadsSchema = z + .strictObject({ + version: z.literal(1), + platform: configurationReleasePlatformSchema, + images: z + .array(configurationReleaseDownloadSchema) + .length(CONFIGURATION_RELEASE_IMAGE_NAMES.length), + declaredBytes: z.number().int().nonnegative().safe().max(CONFIGURATION_RELEASE_TOTAL_BYTES), + }) + .superRefine((value, context) => { + if ( + value.images.some( + (image, index) => image.image !== CONFIGURATION_RELEASE_IMAGE_NAMES[index], + ) || + value.images.reduce((sum, image) => sum + image.declaredBytes, 0) !== value.declaredBytes + ) + context.addIssue({ code: "custom", message: "Inconsistent release download metadata" }); + }); +export type ConfigurationReleaseDownloads = z.infer; diff --git a/packages/core/src/ops/index.ts b/packages/core/src/ops/index.ts index 573a201c2..ccfb68e81 100644 --- a/packages/core/src/ops/index.ts +++ b/packages/core/src/ops/index.ts @@ -14,6 +14,7 @@ export { export * from "./configuration-maintenance"; export * from "./configuration-maintenance-control-validation"; export * from "./configuration-preparation"; +export * from "./configuration-release-downloads"; export * from "./contract"; export * from "./control-retention-receipt"; export * from "./control-root-key"; From bc875ceb5ed465e4ceaaedd667957483adeeff31 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 09:50:29 +0100 Subject: [PATCH 077/135] feat(platform): freeze checked release inputs in maintenance plans --- ...nfiguration-maintenance-forward-records.ts | 1 + .../configuration-release-snapshot.ts | 50 +++++++ .../src/build-artifacts/control-authority.ts | 2 + .../fixed-configuration-maintenance.test.ts | 137 ++++++++++++++++++ .../fixed-configuration-maintenance.ts | 24 +++ .../ops/configuration-release-downloads.ts | 16 ++ 6 files changed, 230 insertions(+) create mode 100644 packages/core/src/build-artifacts/configuration-release-snapshot.ts diff --git a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts index 079fd2588..aba655844 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts @@ -24,6 +24,7 @@ export const configurationMaintenanceForwardGraphSchema = z.strictObject({ planDigest: digest, coordinatorImage: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.coordinatorImage), handoff: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.handoff), + release: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.release), serviceIds: z.lazy(() => configurationMaintenanceReceiptSchema.shape.serviceIds), steps: z.lazy(() => configurationMaintenanceReceiptSchema.shape.steps), }); diff --git a/packages/core/src/build-artifacts/configuration-release-snapshot.ts b/packages/core/src/build-artifacts/configuration-release-snapshot.ts new file mode 100644 index 000000000..6a684a404 --- /dev/null +++ b/packages/core/src/build-artifacts/configuration-release-snapshot.ts @@ -0,0 +1,50 @@ +/** Frozen checked release inputs. This schema is data, never an effect issuer. */ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import { configurationReleaseForwardSchema } from "../ops/configuration-release-downloads"; +import { + canonicalReleaseManifest, + normalizeReleaseSelection, + parseReleaseManifest, +} from "../services/release-manifest"; + +export const configurationReleaseSnapshotSchema = configurationReleaseForwardSchema + .extend({ + imagePrefix: z.string().min(1).max(255), + canonicalManifest: z.string().min(1).max(32768), + }) + .superRefine((value, context) => { + try { + const selection = normalizeReleaseSelection("main", `${value.imagePrefix}/release-manifest`); + if ( + selection.kind === "local" || + selection.imagePrefix !== value.imagePrefix || + Buffer.byteLength(value.canonicalManifest) > 32768 + ) + throw Error(); + const manifest = parseReleaseManifest(value.canonicalManifest, value.imagePrefix), + canonical = canonicalReleaseManifest(manifest), + images = { + "app-api": manifest.images.api, + "app-web": manifest.images.web, + "data-manager": manifest.images["data-manager"], + "ops-agent": manifest.images["ops-agent"], + "transitous-runner": manifest.images["transitous-runner"], + }; + if ( + manifest.release !== value.releaseId || + canonical !== value.canonicalManifest || + createHash("sha256").update(canonical).digest("hex") !== value.manifestDigest || + canonicalJson(images) !== canonicalJson(value.serviceImages) || + value.downloads.images.some((image) => image.reference !== manifest.images[image.image]) + ) + throw Error(); + } catch { + context.addIssue({ + code: "custom", + message: "Checked release snapshot changed or exceeds capacity", + }); + } + }); +export type ConfigurationReleaseSnapshot = z.infer; diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 21b0d602b..4760a33ba 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -3084,6 +3084,7 @@ export function publishConfigurationMaintenanceForwardWitness( planDigest: receipt.planDigest, ...(owner.plan.coordinatorImage ? { coordinatorImage: owner.plan.coordinatorImage } : {}), ...(owner.plan.handoff ? { handoff: owner.plan.handoff } : {}), + ...(owner.plan.release ? { release: owner.plan.release } : {}), serviceIds: receipt.serviceIds, steps: receipt.steps, }); @@ -3282,6 +3283,7 @@ export function verifyConfigurationMaintenanceGenerationWitness( sourceAuthorityRevision: "not-shared", previewDigest: "0".repeat(64), ...(graph.coordinatorImage ? { coordinatorImage: graph.coordinatorImage } : {}), + ...(graph.release ? { release: graph.release } : {}), ...(graph.handoff ? { handoff: graph.handoff } : {}), hardlinkPlan: request.plan, serviceIds: root.serviceIds, diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts index 6474fcb19..d70f666cd 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -179,6 +179,136 @@ it("retains the frozen coordinator image in the cross-role graph used to reconst expect(rebuilt.steps).toEqual(plan.steps); }); +it("freezes a checked release snapshot in the original helper graph without a pre-pull recreate", async () => { + const source = input(), + { serviceId: _serviceId, ...ordinary } = source.apply as Extract< + typeof source.apply, + { kind: "vault.apply" } + >, + apply = { ...ordinary, kind: "stack.render" as const }, + operation = { + ...source.operation, + followUp: "platform-apply" as const, + applyOperationFingerprint: opsOperationFingerprint(apply), + }, + operationFingerprint = opsOperationFingerprint(operation), + prepare = { + ...source.prepare, + operationFingerprint, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt: source.prepare.deadlineAt, + parent: source.apiParent, + }), + }; + const manifest = parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: "checked-release-01", + images: Object.fromEntries( + RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${"3".repeat(64)}`, + ]), + ), + privacyReleaseValidation: { + version: 1, + sourceBuildFingerprint: "a".repeat(64), + validatedAt: "2026-10-30T00:00:00Z", + checks: { translationsConsistent: true, openApiConsistent: true, policyConsistent: true }, + }, + }), + ); + const canonicalManifest = canonicalReleaseManifest(manifest), + manifestDigest = createHash("sha256").update(canonicalManifest).digest("hex"), + release = { + version: 1 as const, + releaseId: manifest.release, + manifestDigest, + canonicalManifest, + imagePrefix: "ghcr.io/openmapx", + serviceImages: { + "app-api": manifest.images.api, + "app-web": manifest.images.web, + "data-manager": manifest.images["data-manager"], + "ops-agent": manifest.images["ops-agent"], + "transitous-runner": manifest.images["transitous-runner"], + }, + downloads: { + version: 1 as const, + platform: "linux/amd64" as const, + declaredBytes: 7 * 512, + images: RELEASE_IMAGE_NAMES.map((image) => ({ + image, + reference: manifest.images[image], + platformManifestDigest: `sha256:${"4".repeat(64)}`, + configDigest: `sha256:${"5".repeat(64)}`, + declaredBytes: 512, + })), + }, + }, + handoff = { + handoffId: "77777777-7777-4777-8777-777777777777", + previewFingerprint: "e".repeat(64), + serviceIds: ["app-api", "ops-agent"], + expectedImageIdentities: { + "app-api": `sha256:${"1".repeat(64)}`, + "ops-agent": `sha256:${"2".repeat(64)}`, + }, + mode: { kind: "release" as const, releaseId: manifest.release, manifestDigest }, + }, + args = { + ...source, + operation, + prepare, + apply, + serviceIds: ["app-api", "data-manager", "motis", "ops-agent"], + recreateServiceIds: [], + handoff, + release, + }; + const plan = buildFixedConfigurationMaintenancePlan(args); + expect(plan).toHaveProperty("release", release); + expect(plan.steps.map((step) => step.phase)).toEqual([ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-handoff", + ]); + expect(() => assertFixedConfigurationMaintenancePlan(plan, source.installationId)).not.toThrow(); + const { configurationMaintenanceForwardGraphSchema } = await import( + "./configuration-maintenance-forward-records" + ); + const { kind: _kind, ...binding } = operation; + if (source.apiParent.issuer !== "api") throw Error("API graph required"); + const graph = configurationMaintenanceForwardGraphSchema.parse({ + version: 1, + binding, + prepare, + apiJobId: source.apiParent.apiJobId, + parentOperationKey: source.apiParent.operationKey, + parentFingerprint: source.apiParent.fingerprint, + parentDeadlineAt: source.apiParent.deadlineAt, + planDigest: plan.planDigest, + serviceIds: plan.serviceIds, + steps: plan.steps, + handoff: plan.handoff, + release: plan.release, + }); + expect(graph.release).toEqual(release); + const { release: _release, ...missing } = args; + expect(() => buildFixedConfigurationMaintenancePlan(missing)).toThrow(); + expect(() => + buildFixedConfigurationMaintenancePlan({ ...args, recreateServiceIds: ["data-manager"] }), + ).toThrow(); + expect(() => + buildFixedConfigurationMaintenancePlan({ + ...args, + handoff: { ...handoff, mode: { kind: "configuration" } }, + }), + ).toThrow(); +}); + it("freezes protected handoff after the original generation and ordinary children", async () => { const source = input(); const { serviceId: _serviceId, ...ordinary } = source.apply as Extract< @@ -272,3 +402,10 @@ it("freezes protected handoff after the original generation and ordinary childre changed.handoff!.expectedImageIdentities["app-api"] = `sha256:${"3".repeat(64)}`; expect(() => assertFixedConfigurationMaintenancePlan(changed, source.installationId)).toThrow(); }); + +import { createHash } from "node:crypto"; +import { + canonicalReleaseManifest, + parseReleaseManifest, + RELEASE_IMAGE_NAMES, +} from "../services/release-manifest"; diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts index 5bd4cb5cc..68d0a5083 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.ts @@ -29,6 +29,10 @@ import { } from "../services/data-manager/contracts/configuration-maintenance"; import { generationHardlinksPlanSchema } from "../services/data-manager/contracts/executions"; import { deriveConfigurationFenceSettlementId } from "./configuration-fence-settlement-records"; +import { + type ConfigurationReleaseSnapshot, + configurationReleaseSnapshotSchema, +} from "./configuration-release-snapshot"; import { type ControlChildReservation, controlChildOperationKey, @@ -61,6 +65,7 @@ export const fixedConfigurationMaintenancePlanSchema = z.strictObject({ previewDigest: digest, coordinatorImage: coordinatorImageSchema.optional(), handoff: configurationHandoffIntentSchema.optional(), + release: configurationReleaseSnapshotSchema.optional(), nestedHandoff: z.lazy(() => controlChildReservationSchema).optional(), serviceIds: ids, planDigest: digest, @@ -139,6 +144,7 @@ export function buildFixedConfigurationMaintenancePlan(input: { recreateServiceIds: readonly string[]; coordinatorImage?: z.infer; handoff?: ConfigurationHandoffIntent; + release?: ConfigurationReleaseSnapshot; }): FixedConfigurationMaintenancePlan { const prepare = executionRefSchema.parse(input.prepare), operation = configurationMaintenancePrepareSchema.parse(input.operation), @@ -183,6 +189,21 @@ export function buildFixedConfigurationMaintenancePlan(input: { throw Error("Fixed maintenance candidate binding conflict"); const recreates = [...new Set(input.recreateServiceIds)].sort(); const handoff = input.handoff ? configurationHandoffIntentSchema.parse(input.handoff) : undefined; + const release = input.release + ? configurationReleaseSnapshotSchema.parse(input.release) + : undefined; + if ( + (handoff?.mode.kind === "release") !== (release !== undefined) || + (release && + (!handoff || + handoff.mode.kind !== "release" || + input.coordinatorImage || + recreates.length !== 0 || + handoff.mode.releaseId !== release.releaseId || + handoff.mode.manifestDigest !== release.manifestDigest || + handoff.serviceIds.some((id) => !Object.hasOwn(release.serviceImages, id)))) + ) + throw Error("Fixed checked release handoff conflict"); if ( handoff && (operation.followUp !== "platform-apply" || @@ -261,6 +282,7 @@ export function buildFixedConfigurationMaintenancePlan(input: { serviceIds, ...(input.coordinatorImage ? { coordinatorImage: input.coordinatorImage } : {}), ...(handoff ? { handoff } : {}), + ...(release ? { release } : {}), steps: intents.map((step, ordinal) => ({ ...step, ordinal, serviceIds })), }), ) @@ -374,6 +396,7 @@ export function buildFixedConfigurationMaintenancePlan(input: { previewDigest: input.previewDigest, ...(input.coordinatorImage ? { coordinatorImage: input.coordinatorImage } : {}), ...(handoff ? { handoff, nestedHandoff } : {}), + ...(release ? { release } : {}), serviceIds, planDigest, hardlinkPlan, @@ -419,6 +442,7 @@ export function assertFixedConfigurationMaintenancePlan( previewDigest: plan.previewDigest, ...(plan.coordinatorImage ? { coordinatorImage: plan.coordinatorImage } : {}), ...(plan.handoff ? { handoff: plan.handoff } : {}), + ...(plan.release ? { release: plan.release } : {}), hardlinkPlan: plan.hardlinkPlan, serviceIds: plan.serviceIds, recreateServiceIds, diff --git a/packages/core/src/ops/configuration-release-downloads.ts b/packages/core/src/ops/configuration-release-downloads.ts index 51bdf1ba3..15dc9bdf0 100644 --- a/packages/core/src/ops/configuration-release-downloads.ts +++ b/packages/core/src/ops/configuration-release-downloads.ts @@ -1,5 +1,6 @@ /** Bounded public registry metadata. These records never supply host authority. */ import { z } from "zod"; +import { configurationHandoffModeSchema } from "./configuration-handoff-records"; export const CONFIGURATION_RELEASE_IMAGE_NAMES = [ "api", "web", @@ -41,3 +42,18 @@ export const configurationReleaseDownloadsSchema = z context.addIssue({ code: "custom", message: "Inconsistent release download metadata" }); }); export type ConfigurationReleaseDownloads = z.infer; +export const configurationReleaseSelectionSchema = configurationHandoffModeSchema.options[1].omit({ + kind: true, +}); +export const configurationReleaseServiceImagesSchema = z.strictObject({ + "app-api": configurationReleaseDownloadSchema.shape.reference, + "app-web": configurationReleaseDownloadSchema.shape.reference, + "data-manager": configurationReleaseDownloadSchema.shape.reference, + "ops-agent": configurationReleaseDownloadSchema.shape.reference, + "transitous-runner": configurationReleaseDownloadSchema.shape.reference, +}); +export const configurationReleaseForwardSchema = configurationReleaseSelectionSchema.extend({ + version: z.literal(1), + serviceImages: configurationReleaseServiceImagesSchema, + downloads: configurationReleaseDownloadsSchema, +}); From 35ff26ff9a5567c19d003224f4c9e677a240fdd5 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 10:02:10 +0100 Subject: [PATCH 078/135] feat(platform): bind checked release preflight to original handoff plans --- .../src/configuration-handoff-application.ts | 2 + .../src/configuration-handoff-store.ts | 1 + .../src/configuration-maintenance-store.ts | 56 ++++++-- .../src/configuration-release-images.test.ts | 135 ++++++++++++++++++ .../src/configuration-release-images.ts | 57 ++++++++ .../src/configuration-release-preflight.ts | 45 ++++++ .../src/configuration-release.test.ts | 48 +++++++ apps/ops-agent/src/trusted-config-runtime.ts | 13 +- .../configuration-handoff-records.ts | 21 ++- .../configuration-maintenance-budget.ts | 40 ++++-- .../fixed-configuration-maintenance.test.ts | 50 +++++++ .../ops/configuration-preparation-records.ts | 2 + 12 files changed, 442 insertions(+), 28 deletions(-) create mode 100644 apps/ops-agent/src/configuration-release-images.test.ts create mode 100644 apps/ops-agent/src/configuration-release-images.ts create mode 100644 apps/ops-agent/src/configuration-release-preflight.ts diff --git a/apps/ops-agent/src/configuration-handoff-application.ts b/apps/ops-agent/src/configuration-handoff-application.ts index 706f74aa0..0552feeb5 100644 --- a/apps/ops-agent/src/configuration-handoff-application.ts +++ b/apps/ops-agent/src/configuration-handoff-application.ts @@ -73,6 +73,8 @@ function ownerFor(child: ConfigurationHandoffChild) { owner.final || owner.inspection || !owner.guardSource || + (intent.release !== undefined) !== (owner.plan.release !== undefined) || + (intent.release && canonicalJson(intent.release) !== canonicalJson(owner.plan.release)) || owner.receipt.dmFence?.state !== "prepared" || owner.receipt.state !== "applying" || owner.receipt.decision?.decision !== "continue" || diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index 70567778e..81e08edde 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -246,6 +246,7 @@ export async function prepareConfigurationHandoffLaunch( delegationId: observed.delegationId, assertion: attempt.assertion, helperContainerName: `openmapx-configuration-handoff-${operation.handoffId}`, + ...(owned.owner.plan.release ? { release: owned.owner.plan.release } : {}), preparedAt: Date.now(), }); writePrivateRecord( diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index a495e161c..ee46137da 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -33,6 +33,7 @@ import { configurationDataManagerTransport, } from "./configuration-hardlinks-runtime"; import { inspectMaintenanceDockerRuntime } from "./configuration-maintenance-docker-inspection"; +import { preflightConfigurationRelease } from "./configuration-release-preflight"; import type { DockerRuntimeOptions } from "./docker-runtime"; import type { OpsJobJournal } from "./journal"; import { @@ -5123,14 +5124,24 @@ export async function prepareConfigurationMaintenanceAdmission( canonicalJson(candidate.record.affectedServiceIds)) ) throw Error("Original platform candidate binding conflict"); - const recreateServiceIds = platform - ? candidate.record.affectedServiceIds.filter( - (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, - ) - : candidate.record.apiFollowUp === "service-apply" || - candidate.record.privateCoordinatorLifecycle - ? candidate.record.affectedServiceIds - : []; + const release = platform?.release + ? await preflightConfigurationRelease({ + infraDir: source.infraDir, + selection: platform.release, + execFile: source.execFile, + signal: input.signal, + }) + : undefined; + const recreateServiceIds = release + ? [] + : platform + ? candidate.record.affectedServiceIds.filter( + (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : candidate.record.apiFollowUp === "service-apply" || + candidate.record.privateCoordinatorLifecycle + ? candidate.record.affectedServiceIds + : []; if ( (platform ? "platform-apply" : recreateServiceIds.length ? "service-apply" : "render-only") !== input.operation.followUp @@ -5359,11 +5370,21 @@ export async function prepareConfigurationMaintenanceAdmission( }), ) .digest("hex"); - const protectedIds = platform - ? platform.affectedServiceIds.filter( - (id) => protectedConfigurationServiceIdSchema.safeParse(id).success, - ) - : []; + if (release) { + const targets = rendered.resolved.enabled + .map((entry) => entry.manifest.id) + .filter((id) => Object.hasOwn(release.serviceImages, id)) + .sort(); + if (canonicalJson(targets) !== canonicalJson(platform!.affectedServiceIds)) + throw Error("Exact enabled checked release targets required"); + } + const protectedIds = release + ? platform!.affectedServiceIds + : platform + ? platform.affectedServiceIds.filter( + (id) => protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : []; const handoff = platform && protectedIds.length ? { @@ -5378,7 +5399,13 @@ export async function prepareConfigurationMaintenanceAdmission( return [id, old.provenance.imageId]; }), ), - mode: { kind: "configuration" as const }, + mode: release + ? { + kind: "release" as const, + releaseId: release.releaseId, + manifestDigest: release.manifestDigest, + } + : { kind: "configuration" as const }, } : undefined; const frozen = buildFixedConfigurationMaintenancePlan({ @@ -5396,6 +5423,7 @@ export async function prepareConfigurationMaintenanceAdmission( serviceIds, recreateServiceIds, ...(handoff ? { handoff } : {}), + ...(release ? { release } : {}), ...(coordinatorImage ? { coordinatorImage } : {}), }); assertProspectiveConfigurationMaintenanceBudget({ diff --git a/apps/ops-agent/src/configuration-release-images.test.ts b/apps/ops-agent/src/configuration-release-images.test.ts new file mode 100644 index 000000000..e1761c531 --- /dev/null +++ b/apps/ops-agent/src/configuration-release-images.test.ts @@ -0,0 +1,135 @@ +import { + CONFIGURATION_RELEASE_IMAGE_NAMES, + type ConfigurationReleaseDownloads, +} from "@openmapx/core/ops"; +import { expect, it, vi } from "vitest"; +import { inspectConfigurationReleaseImages } from "./configuration-release-images"; + +function downloads(): ConfigurationReleaseDownloads { + return { + version: 1, + platform: "linux/amd64", + declaredBytes: 7, + images: CONFIGURATION_RELEASE_IMAGE_NAMES.map((image) => ({ + image, + reference: `ghcr.io/openmapx/${image}@sha256:${"a".repeat(64)}`, + platformManifestDigest: `sha256:${"b".repeat(64)}`, + configDigest: `sha256:${"c".repeat(64)}`, + declaredBytes: 1, + })), + }; +} +function io(change: Record = {}) { + return vi.fn(async (_file: string, args: readonly string[]) => { + expect(args.slice(0, 4)).toEqual(["image", "inspect", "--format", "{{json .}}"]); + return { + stdout: JSON.stringify({ + Id: `sha256:${"d".repeat(64)}`, + RepoDigests: [args[4]], + Os: "linux", + Architecture: "amd64", + ...change, + }), + stderr: "", + }; + }); +} +it("observes each exact immutable reference and retains the actual Docker ID separately", async () => { + const execFile = io(), + snapshot = downloads(); + const result = await inspectConfigurationReleaseImages({ + downloads: snapshot, + execFile, + signal: new AbortController().signal, + }); + expect(result).toEqual( + snapshot.images.map(({ image, reference }) => ({ + image, + reference, + imageId: `sha256:${"d".repeat(64)}`, + })), + ); + expect(execFile).toHaveBeenCalledTimes(7); + expect(execFile.mock.calls.map(([, args]) => args[4])).toEqual( + snapshot.images.map((v) => v.reference), + ); + expect(Object.isFrozen(result)).toBe(true); + expect(result.every(Object.isFrozen)).toBe(true); +}); +it("accepts the exact selected platform digest for a multi-platform repository pin", async () => { + const snapshot = downloads(); + const execFile = io(); + execFile.mockImplementation(async (_file, args) => ({ + stdout: JSON.stringify({ + Id: `sha256:${"d".repeat(64)}`, + RepoDigests: [`${args[4]!.slice(0, args[4]!.indexOf("@"))}@sha256:${"b".repeat(64)}`], + Os: "linux", + Architecture: "amd64", + }), + stderr: "", + })); + expect( + await inspectConfigurationReleaseImages({ + downloads: snapshot, + execFile, + signal: new AbortController().signal, + }), + ).toHaveLength(7); +}); +it.each([ + { RepoDigests: [] }, + { RepoDigests: [`ghcr.io/foreign/api@sha256:${"a".repeat(64)}`] }, + { RepoDigests: [`ghcr.io/openmapx/api@sha256:${"e".repeat(64)}`] }, + { Id: "image:latest" }, + { Os: "windows" }, + { Architecture: "arm64" }, +])("refuses incompatible local image data before returning an accepted set: %j", async (change) => { + await expect( + inspectConfigurationReleaseImages({ + downloads: downloads(), + execFile: io(change), + signal: new AbortController().signal, + }), + ).rejects.toThrow(); +}); +it("propagates missing-image failure and stops before any further inspection", async () => { + const execFile = io(); + execFile.mockRejectedValueOnce(Error("No such image")); + await expect( + inspectConfigurationReleaseImages({ + downloads: downloads(), + execFile, + signal: new AbortController().signal, + }), + ).rejects.toThrow(); + expect(execFile).toHaveBeenCalledTimes(1); +}); +it("checks cancellation before and after external IO and bounds stdout", async () => { + const controller = new AbortController(), + execFile = io(); + controller.abort(); + await expect( + inspectConfigurationReleaseImages({ + downloads: downloads(), + execFile, + signal: controller.signal, + }), + ).rejects.toThrow(); + expect(execFile).not.toHaveBeenCalled(); + const after = new AbortController(); + execFile.mockImplementationOnce(async () => { + after.abort(); + return { stdout: "{}", stderr: "" }; + }); + await expect( + inspectConfigurationReleaseImages({ downloads: downloads(), execFile, signal: after.signal }), + ).rejects.toThrow(); + execFile.mockResolvedValueOnce({ stdout: " ".repeat(65537), stderr: "" }); + await expect( + inspectConfigurationReleaseImages({ + downloads: downloads(), + execFile, + signal: new AbortController().signal, + }), + ).rejects.toThrow(); +}); diff --git a/apps/ops-agent/src/configuration-release-images.ts b/apps/ops-agent/src/configuration-release-images.ts new file mode 100644 index 000000000..b1c5d5c84 --- /dev/null +++ b/apps/ops-agent/src/configuration-release-images.ts @@ -0,0 +1,57 @@ +/** Read-only local image verification. These observations are data, not a pull or lease. */ +import { + type ConfigurationReleaseDownloads, + configurationReleaseDownloadSchema, + configurationReleaseDownloadsSchema, +} from "@openmapx/core/ops"; +import { z } from "zod"; +import type { DockerRuntimeOptions } from "./docker-runtime"; + +const imageSchema = z.object({ + Id: z.string().regex(/^sha256:[a-f0-9]{64}$/), + RepoDigests: z.array(configurationReleaseDownloadSchema.shape.reference).max(256), + Os: z.string().max(32), + Architecture: z.string().max(32), + Variant: z.string().max(32).optional(), +}); +export async function inspectConfigurationReleaseImages(input: { + downloads: ConfigurationReleaseDownloads; + execFile: NonNullable; + signal: AbortSignal; +}) { + const downloads = configurationReleaseDownloadsSchema.parse(input.downloads), + [os, architecture] = downloads.platform.split("/"), + observations = []; + for (const image of downloads.images) { + input.signal.throwIfAborted(); + const response = await input.execFile( + "docker", + ["image", "inspect", "--format", "{{json .}}", image.reference], + { + signal: AbortSignal.any([input.signal, AbortSignal.timeout(15000)]), + timeout: 15000, + maxBuffer: 65536, + }, + ); + input.signal.throwIfAborted(); + if (Buffer.byteLength(response.stdout) > 65536) + throw Error("Release image observation capacity exceeded"); + const observed = imageSchema.parse(JSON.parse(response.stdout)), + platformReference = `${image.reference.slice(0, image.reference.lastIndexOf("@"))}@${image.platformManifestDigest}`; + if ( + observed.Os !== os || + observed.Architecture !== architecture || + (observed.Variant && !(architecture === "arm64" && observed.Variant === "v8")) || + !observed.RepoDigests.some( + (reference) => reference === image.reference || reference === platformReference, + ) + ) + throw Error("Checked release local image does not match its pinned platform"); + // Docker's local image ID is observed independently. An OCI index digest + // or repository reference must never be substituted for that runtime ID. + observations.push( + Object.freeze({ image: image.image, reference: image.reference, imageId: observed.Id }), + ); + } + return Object.freeze(observations); +} diff --git a/apps/ops-agent/src/configuration-release-preflight.ts b/apps/ops-agent/src/configuration-release-preflight.ts new file mode 100644 index 000000000..1caf09a18 --- /dev/null +++ b/apps/ops-agent/src/configuration-release-preflight.ts @@ -0,0 +1,45 @@ +/** Resolve public release inputs under the actual private release-store lock. + * This is a read-only preflight, not target ownership or permission to pull. */ +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; +import { configurationReleaseSnapshotSchema } from "../../../packages/core/src/build-artifacts/configuration-release-snapshot"; +import { readCheckedConfigurationRelease } from "./configuration-release"; +import { inspectConfigurationReleaseDownloads } from "./configuration-release-downloads"; +import type { DockerRuntimeOptions } from "./docker-runtime"; + +export async function preflightConfigurationRelease(input: { + infraDir: string; + selection: { releaseId: string; manifestDigest: string }; + execFile: NonNullable; + signal: AbortSignal; +}) { + const lock = await services.acquireReleaseStoreLock(join(input.infraDir, ".ops-agent-releases")); + try { + input.signal.throwIfAborted(); + const release = readCheckedConfigurationRelease(input.infraDir, input.selection), + downloads = await inspectConfigurationReleaseDownloads({ + release, + execFile: input.execFile, + signal: input.signal, + }), + current = readCheckedConfigurationRelease(input.infraDir, input.selection), + channel = services.releaseSelection(); + if ( + channel.kind === "local" || + services.canonicalReleaseManifest(current.manifest) !== + services.canonicalReleaseManifest(release.manifest) + ) + throw Error("Checked release changed during preflight"); + return configurationReleaseSnapshotSchema.parse({ + version: 1, + releaseId: release.releaseId, + manifestDigest: release.manifestDigest, + imagePrefix: channel.imagePrefix, + canonicalManifest: services.canonicalReleaseManifest(release.manifest), + serviceImages: release.serviceImages, + downloads, + }); + } finally { + lock.release(); + } +} diff --git a/apps/ops-agent/src/configuration-release.test.ts b/apps/ops-agent/src/configuration-release.test.ts index 62cd8281b..878c9702d 100644 --- a/apps/ops-agent/src/configuration-release.test.ts +++ b/apps/ops-agent/src/configuration-release.test.ts @@ -202,3 +202,51 @@ it.each([ ); if (kind === "aborted") expect(execFile).not.toHaveBeenCalled(); }); + +// The checked preflight joins the real private cache/lock to registry reads; +// this is not a target grant, pull or release runtime acceptance test. +it.each([false, true])( + "binds one checked preflight snapshot and rejects metadata-time source drift: %s", + async (drift) => { + const { preflightConfigurationRelease } = await import("./configuration-release-preflight"); + let changed = false; + const execFile = vi.fn(async (_file: string, args: readonly string[]) => { + if (args[0] === "info") return { stdout: "linux/amd64", stderr: "" }; + expect(args.slice(0, 2)).toEqual(["manifest", "inspect"]); + if (drift && !changed) { + changed = true; + writeFileSync(join(directory, `${releaseId}.json`), `${raw}\n`, { mode: 0o600 }); + } + return { + stdout: JSON.stringify({ + schemaVersion: 2, + config: { digest: `sha256:${"c".repeat(64)}`, size: 32 }, + layers: [{ digest: `sha256:${"d".repeat(64)}`, size: 64 }], + }), + stderr: "", + }; + }); + const run = () => + preflightConfigurationRelease({ + infraDir: root, + selection: { releaseId, manifestDigest }, + execFile, + signal: new AbortController().signal, + }); + if (drift) await expect(run()).rejects.toThrow("Checked configuration release changed"); + else + expect(await run()).toMatchObject({ + releaseId, + manifestDigest, + canonicalManifest: raw, + imagePrefix: "ghcr.io/openmapx", + downloads: { declaredBytes: 7 * 96 }, + }); + expect( + execFile.mock.calls.every(([, args]) => args[0] === "info" || args[0] === "manifest"), + ).toBe(true); + writeFileSync(join(directory, `${releaseId}.json`), raw, { mode: 0o600 }); + // Original lock is released on either return path; no cleanup/admission is needed. + expect(await run()).toMatchObject({ releaseId, manifestDigest }); + }, +); diff --git a/apps/ops-agent/src/trusted-config-runtime.ts b/apps/ops-agent/src/trusted-config-runtime.ts index d7d897a07..f6caac672 100644 --- a/apps/ops-agent/src/trusted-config-runtime.ts +++ b/apps/ops-agent/src/trusted-config-runtime.ts @@ -33,8 +33,9 @@ import { assertStagedCoordinatorConfigurationWrite, inspectOwnedMaintenanceGenerationRuntime, } from "./configuration-maintenance-store"; +import { preflightConfigurationRelease } from "./configuration-release-preflight"; import { configurationPolicyRevision } from "./configured-resource-authority"; -import type { DockerRuntimeOptions } from "./docker-runtime"; +import { type DockerRuntimeOptions, runContainedProcess } from "./docker-runtime"; import type { OwnedServiceResolver } from "./owned-services"; import type { OpsExecutionContext, OpsRuntime, OpsTrustedClaim } from "./runtime"; import { OpsHostUpdateRequiredError } from "./runtime"; @@ -413,7 +414,17 @@ export function installTrustedConfigurationRuntime( ).stdout, }); } + const releaseSelection = candidate.record.privatePlatformIntent?.release; + const release = releaseSelection + ? await preflightConfigurationRelease({ + infraDir: options.infraDir, + selection: releaseSelection, + execFile: options.execFile ?? runContainedProcess, + signal: context.signal, + }) + : undefined; assertProspectiveConfigurationMaintenancePreludeBudget({ + ...(release ? { release } : {}), installationId: installation.installationId, record: candidate.record, sourceAuthorityRevision: authority?.revisionId ?? "registry-v1", diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.ts b/packages/core/src/build-artifacts/configuration-handoff-records.ts index d2f9fae6b..1496cf9f6 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { configurationHandoffPrepareOperationSchema } from "../ops/configuration-handoff-records"; import { executionRefSchema } from "../ops/execution-ref"; +import { configurationReleaseSnapshotSchema } from "./configuration-release-snapshot"; import { controlOriginSchema } from "./control-records"; const digest = z.string().regex(/^[a-f0-9]{64}$/), @@ -48,8 +49,26 @@ export const configurationHandoffIntentRecordSchema = z assertion: z.string().min(1).max(4096), helperContainerName: z.string().regex(/^openmapx-configuration-handoff-[a-f0-9-]{36}$/), preparedAt: time, + release: configurationReleaseSnapshotSchema.optional(), }) - .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 65536); + .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 65536) + .superRefine((value, context) => { + const { mode } = value.operation; + if ( + (mode.kind === "release") !== (value.release !== undefined) || + (mode.kind === "release" && + value.release && + (mode.releaseId !== value.release.releaseId || + mode.manifestDigest !== value.release.manifestDigest || + value.operation.serviceIds.some( + (id) => !Object.hasOwn(value.release!.serviceImages, id), + ))) + ) + context.addIssue({ + code: "custom", + message: "Original checked release handoff snapshot required", + }); + }); export const configurationHandoffProgressSchema = z .strictObject({ version: z.literal(1), diff --git a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts index 2b7cf8380..da12f6d9b 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts @@ -318,6 +318,7 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { currentServiceIds: readonly string[]; originalContainers: ConfigurationMaintenanceOwnerRecord["originalContainers"]; rendered: ReturnType; + release?: FixedConfigurationMaintenancePlan["release"]; coordinatorImage?: Omit< NonNullable, "imageId" @@ -329,18 +330,30 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { typeof buildFixedConfigurationMaintenancePlan >[0]["apply"]; const platform = record.privatePlatformIntent; - const protectedIds = platform - ? record.affectedServiceIds.filter( - (id) => protectedConfigurationServiceIdSchema.safeParse(id).success, - ) - : []; - const recreateServiceIds = platform - ? record.affectedServiceIds.filter( - (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, - ) - : record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle - ? record.affectedServiceIds + if ( + !!platform?.release !== !!input.release || + (platform?.release && + input.release && + (platform.release.releaseId !== input.release.releaseId || + platform.release.manifestDigest !== input.release.manifestDigest)) + ) + throw Error("Original checked release preflight required"); + const protectedIds = platform?.release + ? record.affectedServiceIds + : platform + ? record.affectedServiceIds.filter( + (id) => protectedConfigurationServiceIdSchema.safeParse(id).success, + ) : []; + const recreateServiceIds = platform?.release + ? [] + : platform + ? record.affectedServiceIds.filter( + (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, + ) + : record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + ? record.affectedServiceIds + : []; const handoff = platform && protectedIds.length ? { @@ -350,7 +363,9 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { expectedImageIdentities: Object.fromEntries( protectedIds.map((id) => [id, `sha256:${hash}`]), ), - mode: { kind: "configuration" as const }, + mode: platform.release + ? { kind: "release" as const, ...platform.release } + : { kind: "configuration" as const }, } : undefined; const operation = configurationMaintenancePrepareSchema.parse({ @@ -419,6 +434,7 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { serviceIds, recreateServiceIds, ...(handoff ? { handoff } : {}), + ...(input.release ? { release: input.release } : {}), ...(input.coordinatorImage ? { coordinatorImage: { ...input.coordinatorImage, imageId: `sha256:${hash}` } } : {}), diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts index d70f666cd..a1932a4ee 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -276,6 +276,56 @@ it("freezes a checked release snapshot in the original helper graph without a pr "maintenance-handoff", ]); expect(() => assertFixedConfigurationMaintenancePlan(plan, source.installationId)).not.toThrow(); + const { configurationHandoffIntentRecordSchema } = await import( + "./configuration-handoff-records" + ); + const intent = { + version: 1, + installationId: source.installationId, + origin: { + version: 1, + rootKey: prepare.operationKey, + issuer: "ops-agent", + notAfter: prepare.deadlineAt, + }, + ownerPrepare: prepare, + preparer: prepare, + helper: { + ...prepare, + executor: "host-cli", + executionId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + }, + operation: plan.steps.at(-1)!.operation, + admittedAt: Date.now(), + requestedDeadlineAt: prepare.deadlineAt, + helperPublicKey: "a".repeat(43), + sourceDigest: "b".repeat(64), + environmentNames: [], + preparerAdmissionDigest: "c".repeat(64), + ownerCertificateDigest: "d".repeat(64), + leaseId: "88888888-8888-4888-8888-888888888888", + delegationId: "99999999-9999-4999-8999-999999999999", + assertion: "original-fixed-assertion", + helperContainerName: `openmapx-configuration-handoff-${handoff.handoffId}`, + preparedAt: Date.now(), + release, + }; + expect(configurationHandoffIntentRecordSchema.parse(intent).release).toEqual(release); + expect( + configurationHandoffIntentRecordSchema.safeParse({ ...intent, release: undefined }).success, + ).toBe(false); + expect( + configurationHandoffIntentRecordSchema.safeParse({ + ...intent, + release: { ...release, manifestDigest: "f".repeat(64) }, + }).success, + ).toBe(false); + expect( + configurationHandoffIntentRecordSchema.safeParse({ + ...intent, + operation: { ...intent.operation, mode: { kind: "configuration" } }, + }).success, + ).toBe(false); const { configurationMaintenanceForwardGraphSchema } = await import( "./configuration-maintenance-forward-records" ); diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts index 9afdc900c..ffcbca811 100644 --- a/packages/core/src/ops/configuration-preparation-records.ts +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { platformConfigurationSchema } from "../services/platform-configuration"; import { protectedConfigurationReconcileIdsSchema } from "./configuration-handoff-records"; +import { configurationReleaseSelectionSchema } from "./configuration-release-downloads"; import { coordinatorLifecycleMetadataSchema } from "./coordinator-lifecycle-records"; const revision = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); @@ -59,6 +60,7 @@ export const platformConfigurationIntentSchema = z .max(256) .refine((ids) => ids.every((v, i) => i === 0 || ids[i - 1]! < v)), reconcileServiceIds: protectedConfigurationReconcileIdsSchema.optional(), + release: configurationReleaseSelectionSchema.optional(), expectedDesiredRowVersion: z .string() .regex(/^\d{1,10}$/) From c1e1c89bb79de25c5716554b36641031c824a280 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 10:07:28 +0100 Subject: [PATCH 079/135] feat(admin): retain checked release selections in platform previews --- apps/api/openapi.json | 40 +++++++++++++++++++ .../routes/__tests__/admin-platform.test.ts | 31 ++++++++++++++ .../platform-preview-postgres.test.ts | 19 +++++++++ .../platform-configuration-participant.ts | 3 ++ .../platform-configuration-recovery.ts | 1 + .../services/platform-configuration-view.ts | 3 ++ .../api/contracts/service-admin/platform.ts | 3 ++ .../generated/admin/service-models/index.ts | 2 + ...iewAdminPlatformConfiguration200Preview.ts | 2 + ...nPlatformConfiguration200PreviewRelease.ts | 13 ++++++ .../previewAdminPlatformConfigurationBody.ts | 2 + ...ewAdminPlatformConfigurationBodyRelease.ts | 13 ++++++ .../__tests__/platform-preview.test.ts | 22 ++++++++++ .../core/src/services/platform-preview.ts | 17 ++++++++ 14 files changed, 171 insertions(+) create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRelease.ts create mode 100644 packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyRelease.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index efb58800f..8ecceab50 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -51241,6 +51241,26 @@ "ops-agent" ] } + }, + "release": { + "type": "object", + "properties": { + "releaseId": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "manifestDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "releaseId", + "manifestDigest" + ], + "additionalProperties": false } }, "required": [ @@ -51632,6 +51652,26 @@ ] } }, + "release": { + "type": "object", + "properties": { + "releaseId": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "manifestDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "releaseId", + "manifestDigest" + ], + "additionalProperties": false + }, "oldOrigin": { "type": "string", "maxLength": 2048, diff --git a/apps/api/src/routes/__tests__/admin-platform.test.ts b/apps/api/src/routes/__tests__/admin-platform.test.ts index 0ef776b76..259f8a903 100644 --- a/apps/api/src/routes/__tests__/admin-platform.test.ts +++ b/apps/api/src/routes/__tests__/admin-platform.test.ts @@ -214,3 +214,34 @@ it("reports a changed retained mutation as a conflict without exposing internal expect(result.statusCode).toBe(409); expect(result.body).not.toContain("private internal"); }); + +it("transports a checked-release preview identity without accepting caller image or path authority", async () => { + const release = { releaseId: "checked-release-01", manifestDigest: "b".repeat(64) }; + mocks.preview.mockResolvedValueOnce({ + preview: { ...preview, release }, + expiresAt: new Date(Date.now() + 600000).toISOString(), + }); + const result = unwrapAdminResponse( + await previewAdminPlatformConfiguration( + { ...request, release }, + { "idempotency-key": "platform-release-preview-01" }, + { client }, + ), + ); + expect(result.preview).toHaveProperty("release", release); + expect(mocks.preview).toHaveBeenCalledWith( + expect.objectContaining({ actorId: "test-admin-id", release }), + ); + mocks.preview.mockClear(); + const invalid = await app.inject({ + method: "POST", + url: "/api/admin/platform/configuration/preview", + headers: { "idempotency-key": "platform-release-preview-02" }, + payload: { + ...request, + release: { ...release, images: { "ops-agent": "foreign:latest" }, path: "/host/file" }, + }, + }); + expect(invalid.statusCode).toBe(400); + expect(mocks.preview).not.toHaveBeenCalled(); +}); diff --git a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts index 06ecb0590..411fca958 100644 --- a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts @@ -76,6 +76,25 @@ dbDescribe("durable exact platform preview", () => { services: [], }); } + it("retains the exact checked-release preview without writing desired configuration", async () => { + const actorId = await actor(); + const { previewPlatformConfiguration } = await import("../platform-configuration-view"); + const input = { + actorId, + operationKey: randomUUID(), + expectedCurrentRevisionId: null, + changes: { version: 1 as const }, + release: { releaseId: "checked-release-01", manifestDigest: "b".repeat(64) }, + }; + const result = await previewPlatformConfiguration(input); + expect(result.preview).toHaveProperty("release", input.release); + const stored = await readPlatformPreview({ + actorId, + previewFingerprint: result.preview.previewFingerprint, + }); + expect(stored.preview).toHaveProperty("release", input.release); + expect(await fixture.db.select().from(systemSettings)).toHaveLength(0); + }); it("constructs and retains the public preview from current typed host and database sources", async () => { const actorId = await actor(), operationKey = randomUUID(); diff --git a/apps/api/src/services/platform-configuration-participant.ts b/apps/api/src/services/platform-configuration-participant.ts index d9a54069e..390d3f083 100644 --- a/apps/api/src/services/platform-configuration-participant.ts +++ b/apps/api/src/services/platform-configuration-participant.ts @@ -108,6 +108,7 @@ export function retainedPlatformConfigurationIntent(writer: AdministrativeConfig ...(preview.reconcileServiceIds ? { reconcileServiceIds: [...preview.reconcileServiceIds] } : {}), + ...(preview.release ? { release: { ...preview.release } } : {}), expectedDesiredRowVersion: state.expectedDesiredRowVersion, confirmations: { ...state.input.confirmations }, ...(state.authorization ? { authorization: { ...state.authorization } } : {}), @@ -137,6 +138,7 @@ export function assertPlatformConfigurationSource( currentPlatform: source.platform, changes: p.configuration, ...(p.reconcileServiceIds ? { reconcileServiceIds: p.reconcileServiceIds } : {}), + ...(p.release ? { release: p.release } : {}), ...input, }); if (rebuilt.previewFingerprint !== p.previewFingerprint) throw new PlatformPreviewConflict(); @@ -317,6 +319,7 @@ export async function preparePlatformConfigurationChange( revisionId: candidate.revisionId, expectedAuthPolicyRevision: preview.authPolicyRevision, sourceRevision: preview.sourceRevision, + ...(preview.release ? { release: { ...preview.release } } : {}), expectedDesiredRowVersion: state.expectedDesiredRowVersion, ...state.input, }, diff --git a/apps/api/src/services/platform-configuration-recovery.ts b/apps/api/src/services/platform-configuration-recovery.ts index cebe8de70..a321f3f65 100644 --- a/apps/api/src/services/platform-configuration-recovery.ts +++ b/apps/api/src/services/platform-configuration-recovery.ts @@ -41,6 +41,7 @@ export async function assertSavedPlatformConfiguration( canonicalJson(p.affectedServiceIds) !== canonicalJson(intent.affectedServiceIds) || canonicalJson(p.reconcileServiceIds ?? null) !== canonicalJson(intent.reconcileServiceIds ?? null) || + canonicalJson(p.release ?? null) !== canonicalJson(intent.release ?? null) || canonicalJson(retained.sourceSnapshot) !== canonicalJson(intent.sourceSnapshot) || retained.desiredRowVersion !== intent.expectedDesiredRowVersion || Object.entries(p.requiredConfirmation).some( diff --git a/apps/api/src/services/platform-configuration-view.ts b/apps/api/src/services/platform-configuration-view.ts index c721eb2c7..794d1be1d 100644 --- a/apps/api/src/services/platform-configuration-view.ts +++ b/apps/api/src/services/platform-configuration-view.ts @@ -39,11 +39,13 @@ export async function previewPlatformConfiguration(input: { expectedCurrentRevisionId: string | null; changes: services.PlatformConfiguration; reconcileServiceIds?: readonly string[]; + release?: { releaseId: string; manifestDigest: string }; }) { const parsed = adminPlatformPreviewInputSchema.parse({ expectedCurrentRevisionId: input.expectedCurrentRevisionId, changes: input.changes, ...(input.reconcileServiceIds ? { reconcileServiceIds: input.reconcileServiceIds } : {}), + ...(input.release ? { release: input.release } : {}), }), current = await source(); if (parsed.expectedCurrentRevisionId !== current.revisionId) throw new PlatformPreviewConflict(); @@ -74,6 +76,7 @@ export async function previewPlatformConfiguration(input: { currentPlatform: current.platform, changes: parsed.changes, ...(parsed.reconcileServiceIds ? { reconcileServiceIds: parsed.reconcileServiceIds } : {}), + ...(parsed.release ? { release: parsed.release } : {}), env: read.env, services: read.services, resolvedServiceConfigs: resolved.values, diff --git a/packages/core/src/api/contracts/service-admin/platform.ts b/packages/core/src/api/contracts/service-admin/platform.ts index da81aef90..733b6e2aa 100644 --- a/packages/core/src/api/contracts/service-admin/platform.ts +++ b/packages/core/src/api/contracts/service-admin/platform.ts @@ -1,6 +1,7 @@ /** Portable platform preview data; these schemas confer no host authority. */ import z from "zod/v4"; import { protectedConfigurationReconcileIdsSchema } from "../../../ops/configuration-handoff-records"; +import { configurationReleaseSelectionSchema } from "../../../ops/configuration-release-downloads"; import { platformConfigurationSchema } from "../../../services/platform-configuration"; import { adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; import { adminServiceIdempotencyHeadersSchema } from "./common"; @@ -16,6 +17,7 @@ export const adminPlatformPreviewInputSchema = z.strictObject({ expectedCurrentRevisionId: revision, changes: platformConfigurationSchema, reconcileServiceIds: protectedConfigurationReconcileIdsSchema.optional(), + release: configurationReleaseSelectionSchema.optional(), }); export const adminPlatformConfirmationSchema = z.strictObject({ domain: z.boolean(), @@ -48,6 +50,7 @@ export const adminPlatformPreviewSchema = z.strictObject({ }), affectedServiceIds: z.array(z.string().min(1).max(64)).max(256), reconcileServiceIds: protectedConfigurationReconcileIdsSchema.optional(), + release: configurationReleaseSelectionSchema.optional(), oldOrigin: url, newOrigin: url, originConsumers: z.strictObject({ diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index 1903cf33a..99d70ffc3 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -483,6 +483,7 @@ export * from "./previewAdminPlatformConfiguration200PreviewConfigurationRelease export * from "./previewAdminPlatformConfiguration200PreviewConfigurationVersion"; export * from "./previewAdminPlatformConfiguration200PreviewOriginConsumers"; export * from "./previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem"; +export * from "./previewAdminPlatformConfiguration200PreviewRelease"; export * from "./previewAdminPlatformConfiguration200PreviewRequiredConfirmation"; export * from "./previewAdminPlatformConfiguration200PreviewSources"; export * from "./previewAdminPlatformConfiguration200PreviewSourcesAcmeEmail"; @@ -510,6 +511,7 @@ export * from "./previewAdminPlatformConfigurationBodyChangesDomain"; export * from "./previewAdminPlatformConfigurationBodyChangesReleaseSelector"; export * from "./previewAdminPlatformConfigurationBodyChangesVersion"; export * from "./previewAdminPlatformConfigurationBodyReconcileServiceIdsItem"; +export * from "./previewAdminPlatformConfigurationBodyRelease"; export * from "./previewAdminPlatformConfigurationHeaders"; export * from "./provisionAdminDawarich200"; export * from "./provisionAdminDawarich200OauthClient"; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts index 58c398045..b3a3b5fe3 100644 --- a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200Preview.ts @@ -3,6 +3,7 @@ import type { PreviewAdminPlatformConfiguration200PreviewConfiguration } from "./previewAdminPlatformConfiguration200PreviewConfiguration"; import type { PreviewAdminPlatformConfiguration200PreviewOriginConsumers } from "./previewAdminPlatformConfiguration200PreviewOriginConsumers"; import type { PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem } from "./previewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem"; +import type { PreviewAdminPlatformConfiguration200PreviewRelease } from "./previewAdminPlatformConfiguration200PreviewRelease"; import type { PreviewAdminPlatformConfiguration200PreviewRequiredConfirmation } from "./previewAdminPlatformConfiguration200PreviewRequiredConfirmation"; import type { PreviewAdminPlatformConfiguration200PreviewSources } from "./previewAdminPlatformConfiguration200PreviewSources"; import type { PreviewAdminPlatformConfiguration200PreviewValues } from "./previewAdminPlatformConfiguration200PreviewValues"; @@ -36,6 +37,7 @@ export type PreviewAdminPlatformConfiguration200Preview = { * @maxItems 6 */ reconcileServiceIds?: PreviewAdminPlatformConfiguration200PreviewReconcileServiceIdsItem[]; + release?: PreviewAdminPlatformConfiguration200PreviewRelease; /** @maxLength 2048 */ oldOrigin: string; /** @maxLength 2048 */ diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRelease.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRelease.ts new file mode 100644 index 000000000..e989bcfb9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfiguration200PreviewRelease.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfiguration200PreviewRelease = { + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + releaseId: string; + /** @pattern ^[a-f0-9]{64}$ */ + manifestDigest: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts index 501b7b137..5409869e9 100644 --- a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBody.ts @@ -2,6 +2,7 @@ // Generated by pnpm sdk:generate. Do not edit. import type { PreviewAdminPlatformConfigurationBodyChanges } from "./previewAdminPlatformConfigurationBodyChanges"; import type { PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem } from "./previewAdminPlatformConfigurationBodyReconcileServiceIdsItem"; +import type { PreviewAdminPlatformConfigurationBodyRelease } from "./previewAdminPlatformConfigurationBodyRelease"; export type PreviewAdminPlatformConfigurationBody = { expectedCurrentRevisionId: string | null; @@ -11,4 +12,5 @@ export type PreviewAdminPlatformConfigurationBody = { * @maxItems 6 */ reconcileServiceIds?: PreviewAdminPlatformConfigurationBodyReconcileServiceIdsItem[]; + release?: PreviewAdminPlatformConfigurationBodyRelease; }; diff --git a/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyRelease.ts b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyRelease.ts new file mode 100644 index 000000000..a993d12dd --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/previewAdminPlatformConfigurationBodyRelease.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminPlatformConfigurationBodyRelease = { + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + releaseId: string; + /** @pattern ^[a-f0-9]{64}$ */ + manifestDigest: string; +}; diff --git a/packages/core/src/services/__tests__/platform-preview.test.ts b/packages/core/src/services/__tests__/platform-preview.test.ts index 8872d39d5..4bc3494e8 100644 --- a/packages/core/src/services/__tests__/platform-preview.test.ts +++ b/packages/core/src/services/__tests__/platform-preview.test.ts @@ -63,6 +63,28 @@ describe("platform configuration preview", () => { }), ).toThrow(); }); + it("freezes a separate checked-release selection and only enabled release targets", () => { + const base = { ...input(), changes: { version: 1 as const } }; + const release = { releaseId: "checked-release-01", manifestDigest: "b".repeat(64) }; + const request = { ...base, release }; + const preview = buildPlatformConfigurationPreview(request); + expect(preview).toHaveProperty("release", release); + expect(preview.affectedServiceIds).toEqual(["app-api", "app-web", "data-manager", "ops-agent"]); + expect(preview.requiredConfirmation.domain).toBe(false); + expect(preview.previewFingerprint).not.toBe( + buildPlatformConfigurationPreview(base).previewFingerprint, + ); + expect( + buildPlatformConfigurationPreview({ + ...request, + release: { ...release, releaseId: "checked-release-02" }, + }).previewFingerprint, + ).not.toBe(preview.previewFingerprint); + expect(() => buildPlatformConfigurationPreview({ ...input(), release })).toThrow(); + expect(() => + buildPlatformConfigurationPreview({ ...request, reconcileServiceIds: ["ops-agent"] }), + ).toThrow(); + }); it("shows exact public origin consumers and requires domain recovery confirmation", () => { const source = input(); const preview = buildPlatformConfigurationPreview(source); diff --git a/packages/core/src/services/platform-preview.ts b/packages/core/src/services/platform-preview.ts index 889eb56dd..795cfa3b6 100644 --- a/packages/core/src/services/platform-preview.ts +++ b/packages/core/src/services/platform-preview.ts @@ -3,6 +3,10 @@ import { createHash } from "node:crypto"; import z from "zod/v4"; import { canonicalJson } from "../ops/canonical-json"; import { protectedConfigurationReconcileIdsSchema } from "../ops/configuration-handoff-records"; +import { + configurationReleaseSelectionSchema, + configurationReleaseServiceImagesSchema, +} from "../ops/configuration-release-downloads"; import { resolveProxyHost } from "./compose-renderer"; import { type PlatformConfiguration, @@ -35,6 +39,7 @@ export function buildPlatformConfigurationPreview(input: { services: readonly LoadedService[]; resolvedServiceConfigs?: Map>; reconcileServiceIds?: readonly string[]; + release?: { releaseId: string; manifestDigest: string }; }) { const currentRevisionId = revision.parse(input.currentRevisionId); if (revision.parse(input.expectedCurrentRevisionId) !== currentRevisionId) @@ -65,6 +70,16 @@ export function buildPlatformConfigurationPreview(input: { ...changes, version: 1, }); + const release = + input.release === undefined + ? undefined + : configurationReleaseSelectionSchema.parse(input.release); + if ( + release && + (reconcileServiceIds || + canonicalJson(configuration) !== canonicalJson({ version: 1, ...input.currentPlatform })) + ) + throw Error("Apply platform settings separately from a checked release"); const before = resolvePlatformConfiguration({ saved: input.currentPlatform, env: input.env }); const after = resolvePlatformConfiguration({ saved: configuration, env: input.env }); const changedFields = (Object.keys(variables) as (keyof typeof variables)[]).filter( @@ -75,6 +90,7 @@ export function buildPlatformConfigurationPreview(input: { .filter((service) => { if (!service.isBuiltIn || !service.enabled) return false; const m = service.manifest; + if (release) return Object.hasOwn(configurationReleaseServiceImagesSchema.shape, m.id); if (reconcileServiceIds?.includes(m.id as (typeof reconcileServiceIds)[number])) return true; const consumer = JSON.stringify({ environment: m.container.environment, @@ -139,6 +155,7 @@ export function buildPlatformConfigurationPreview(input: { sources: after.sources, affectedServiceIds, ...(reconcileServiceIds ? { reconcileServiceIds } : {}), + ...(release ? { release } : {}), oldOrigin, newOrigin, originConsumers, From a14e938cf1f4556830fa604dee6aa7a9338c0a5b Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 11:21:35 +0100 Subject: [PATCH 080/135] feat(platform): apply checked releases through the owned detached helper --- ...orm-configuration-runtime-postgres.test.ts | 205 ++++++++++++-- .../src/__fixtures__/owned-service-host.ts | 8 +- .../src/configuration-handoff-application.ts | 265 +++++++++++++++++- .../src/configuration-handoff-store.ts | 14 +- ...iguration-maintenance-docker-inspection.ts | 55 +++- .../src/configuration-maintenance-store.ts | 67 ++++- .../src/configuration-release-images.test.ts | 19 ++ .../src/configuration-release-images.ts | 10 +- apps/ops-agent/src/trusted-config-runtime.ts | 16 +- .../configuration-handoff-records.test.ts | 38 +++ .../configuration-handoff-records.ts | 69 +++++ .../configuration-maintenance-budget.ts | 66 ++++- ...onfiguration-maintenance-docker-records.ts | 5 + ...configuration-maintenance-final-records.ts | 54 +++- .../configuration-source-witness.ts | 80 +++++- .../src/build-artifacts/control-authority.ts | 11 + .../fixed-configuration-maintenance.test.ts | 2 + 17 files changed, 913 insertions(+), 71 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index f6bffa036..fb5c05b11 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -2,8 +2,16 @@ * separate native helper. Only the session provider and Docker executable I/O * are substituted; image/container replacement acceptance remains separate. */ import { spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { createRequire } from "node:module"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -77,7 +85,7 @@ async function nativeHelper( options: { bundled?: boolean; fault?: "before-effect" | "after-effect"; - failureStage?: "adoption"; + failureStage?: "adoption" | "application"; kill?: { when: Promise; release(): void }; } = {}, ) { @@ -240,6 +248,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-recovery", "launch-native-job", "launch-native-resource", + "launch-native-release", + "launch-native-release-wrong-config", ])( "guards and retains helper mode %s through original ownership", async (version) => { @@ -267,6 +277,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( configurationHelperVersion: version === "old" ? "old" : "1", configurationHelperLaunchResponseLoss: version === "launch-lost", configurationHelperReleaseInventory: application, + configurationReleaseMetadata: version.startsWith("launch-native-release"), ...(application ? { schemas: { @@ -288,6 +299,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-recovery", "launch-native-job", "launch-native-resource", + "launch-native-release", + "launch-native-release-wrong-config", ].includes(version) ) { state.registry = new services.ServiceRegistry({ @@ -304,12 +317,70 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( Labels: { ...host.row.Labels, "com.docker.compose.service": id }, })); host.setRows(rows); + let checkedRelease: { releaseId: string; manifestDigest: string } | undefined; + if (version.startsWith("launch-native-release")) { + vi.stubEnv("OPENMAPX_RELEASE", "main"); + vi.stubEnv("OPENMAPX_RELEASE_MANIFEST_REPOSITORY", "ghcr.io/openmapx/release-manifest"); + const manifest = services.parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: "checked-runtime-release-01", + images: Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((image) => [ + image, + `ghcr.io/openmapx/${image}@sha256:${"9".repeat(64)}`, + ]), + ), + privacyReleaseValidation: { + version: 1, + sourceBuildFingerprint: "f".repeat(64), + validatedAt: "2026-10-30T00:00:00.000Z", + checks: { + translationsConsistent: true, + openApiConsistent: true, + policyConsistent: true, + }, + }, + }), + ); + const raw = services.canonicalReleaseManifest(manifest); + checkedRelease = { + releaseId: manifest.release, + manifestDigest: createHash("sha256").update(raw).digest("hex"), + }; + mkdirSync(join(host.infraDir, ".ops-agent-releases"), { recursive: true, mode: 0o700 }); + writeFileSync( + join(host.infraDir, ".ops-agent-releases", `${manifest.release}.json`), + raw, + { mode: 0o600 }, + ); + host.setImagePullProcess(async (reference) => { + host.setImageIdentity(reference, `sha256:${"e".repeat(64)}`); + }); + } if (application) { const inspections: Parameters[0] = {}; + if (checkedRelease) { + for (const old of rows) + inspections[old.Id] = { + Id: old.Id, + Image: old.Image, + State: { Status: "running", Running: true, Health: { Status: "healthy" } }, + Config: { Image: old.ConfigImage, Labels: old.Labels, Env: [] }, + HostConfig: { Memory: 0, NanoCpus: 0 }, + Mounts: [], + }; + host.setRuntimeInspections(inspections); + } host.setCreatingProcess(async (args) => { const id = args.at(-1)!, old = rows.find((row) => row.Labels["com.docker.compose.service"] === id); - if (!old || !["app-api", "ops-agent"].includes(id)) + if ( + !old || + !( + checkedRelease ? ["data-manager", "app-api", "ops-agent"] : ["app-api", "ops-agent"] + ).includes(id) + ) throw Error("Unexpected protected target"); if (version === "launch-native-killed") { await services.withConfigurationGenerationWriter(host.infraDir, async () => { @@ -347,11 +418,16 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), row = { ...old, - Id: (id === "app-api" ? "a" : "b").repeat(64), + Id: (id === "app-api" ? "a" : id === "data-manager" ? "c" : "b").repeat(64), + ...(checkedRelease + ? { Image: `sha256:${"e".repeat(64)}`, ConfigImage: config.services[id].image } + : {}), Labels: { ...old.Labels, "com.docker.compose.project.working_dir": join(compose, ".."), - "com.docker.compose.project.config_files": compose, + "com.docker.compose.project.config_files": checkedRelease + ? `${compose},${host.stack.composeReleasePath}` + : compose, "com.docker.compose.config-hash": hash, }, }, @@ -374,7 +450,15 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( Memory: Number(limits?.memory ?? 0), NanoCpus: Math.round(Number(limits?.cpus ?? 0) * 1e9), }, - Mounts: [], + Mounts: + checkedRelease && version !== "launch-native-release-wrong-config" + ? (spec.configs ?? []).map((mount: { source: string; target?: string }) => ({ + Type: "bind", + Source: config.configs[mount.source].file, + Destination: mount.target ?? `/${mount.source}`, + RW: false, + })) + : [], }; host.setRuntimeInspections(inspections); }); @@ -431,7 +515,14 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( serviceId: "ops-agent", config: { OPENMAPX_MEMORY_MIB: 256, OPENMAPX_CPU_LIMIT: 0.5 }, }); - if (["launch-native-job", "launch-native-resource"].includes(version)) { + if ( + [ + "launch-native-job", + "launch-native-resource", + "launch-native-release", + "launch-native-release-wrong-config", + ].includes(version) + ) { const { adminPlatformRoute } = await import("../../routes/admin-platform"), { jobRunner } = await import("../job-runner"), { createDirectAdminOpsKey } = await import("../direct-ops-idempotency"); @@ -448,16 +539,20 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await previewAdminPlatformConfiguration( { expectedCurrentRevisionId: host.initial.revisionId, - changes: - version === "launch-native-resource" - ? { version: 1 as const } - : { - version: 1 as const, - backupRetentionDays: { source: "database", value: 14 }, - }, + changes: [ + "launch-native-resource", + "launch-native-release", + "launch-native-release-wrong-config", + ].includes(version) + ? { version: 1 as const } + : { + version: 1 as const, + backupRetentionDays: { source: "database", value: 14 }, + }, ...(version === "launch-native-resource" ? { reconcileServiceIds: ["ops-agent" as const] } : {}), + ...(checkedRelease ? { release: checkedRelease } : {}), }, headers, { client }, @@ -488,19 +583,53 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( )[0]!; const launchDeadline = Date.now() + 60000; while (!host.calls.some((args) => args[0] === "run") && Date.now() < launchDeadline) { - expect((await readJob()).status).not.toBe("failed"); + const current = await readJob(); + expect( + current.status, + JSON.stringify({ + error: current.error?.split("\n")[0].slice(0, 200), + preflight: (current.result as Record | null) + ?.configurationPreflight, + configuration: (current.result as Record | null)?.configuration, + states: journal.records().map((record) => ({ + kind: record.operation.kind, + state: record.state, + errorClass: record.errorClass, + })), + }), + ).not.toBe("failed"); await new Promise((resolve) => setTimeout(resolve, 100)); } const args = host.calls.find((args) => args[0] === "run"); expect(args).toBeDefined(); const env = (name: string) => args!.find((value) => value.startsWith(`${name}=`))!.slice(name.length + 1); - await nativeHelper(host, { - OPENMAPX_ROOT_DIR: host.root, - OPS_CONFIGURATION_HANDOFF_ID: env("OPS_CONFIGURATION_HANDOFF_ID"), - OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), - OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), - }); + await nativeHelper( + host, + { + OPENMAPX_ROOT_DIR: host.root, + OPS_CONFIGURATION_HANDOFF_ID: env("OPS_CONFIGURATION_HANDOFF_ID"), + OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), + OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), + }, + version === "launch-native-release-wrong-config" + ? { failureStage: "application" } + : {}, + ); + if (version === "launch-native-release-wrong-config") { + await jobRunner.reconcile(); + expect((await readJob()).status).toBe("running"); + expect(dm.settlePosts).toBe(0); + expect(existsSync(join(host.infraDir, ".ops-agent-releases", "current.json"))).toBe( + false, + ); + expect( + host.calls + .filter((args) => args[0] === "compose" && args.includes("up")) + .map((args) => args.at(-1)), + ).toEqual(["data-manager", "app-api"]); + return; + } const settleDeadline = Date.now() + 90000; let job = await readJob(); while (job.status === "running" && Date.now() < settleDeadline) { @@ -527,6 +656,32 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( configuration: { outcome: "committed", followUp: "complete" }, }); expect(await apply()).toEqual(accepted); + if (checkedRelease) { + const pulls = host.calls.filter((args) => args[0] === "image" && args[1] === "pull"); + expect(pulls).toHaveLength(services.RELEASE_IMAGE_NAMES.length); + expect(pulls.map((args) => args.at(-1))).toEqual( + services.RELEASE_IMAGE_NAMES.map( + (image) => `ghcr.io/openmapx/${image}@sha256:${"9".repeat(64)}`, + ), + ); + expect( + rows + .filter((row) => + ["data-manager", "app-api", "ops-agent"].includes( + row.Labels["com.docker.compose.service"], + ), + ) + .every((row) => row.Image === `sha256:${"e".repeat(64)}`), + ).toBe(true); + expect( + JSON.parse( + readFileSync(join(host.infraDir, ".ops-agent-releases", "current.json"), "utf8"), + ), + ).toMatchObject({ + releaseId: checkedRelease.releaseId, + digest: checkedRelease.manifestDigest, + }); + } expect(dm.settlePosts).toBe(1); expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); expect( @@ -534,7 +689,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .filter((args) => args[0] === "compose" && args.includes("up")) .map((args) => args.at(-1)), ).toEqual( - version === "launch-native-resource" ? ["ops-agent"] : ["app-api", "ops-agent"], + version === "launch-native-resource" + ? ["ops-agent"] + : checkedRelease + ? ["data-manager", "app-api", "ops-agent"] + : ["app-api", "ops-agent"], ); return; } diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index e7c89311c..8f01e7c94 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { configurationHelperReleaseInventory?:boolean; configurationHelperLaunchResponseLoss?:boolean; selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { configurationReleaseMetadata?:boolean; configurationHelperReleaseInventory?:boolean; configurationHelperLaunchResponseLoss?:boolean; selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -135,6 +135,7 @@ export async function ownedServiceHost(options: { configurationHelperReleaseInve const imageIdentities = new Map(); const optionsForHelperLaunchResponseLoss = options.configurationHelperLaunchResponseLoss; const helperContainers = new Map(); + const releaseMetadata = options.configurationReleaseMetadata; const execFile: NonNullable = async (_file, args, options) => { calls.push([...args]); let stdout = ""; @@ -185,8 +186,11 @@ export async function ownedServiceHost(options: { configurationHelperReleaseInve ]), ), }); + else if (releaseMetadata && args[0] === "info") stdout="linux/amd64"; + else if (releaseMetadata && args[0] === "manifest" && args[1] === "inspect") stdout=JSON.stringify({schemaVersion:2,config:{digest:`sha256:${"c".repeat(64)}`,size:128},layers:[{digest:`sha256:${"d".repeat(64)}`,size:512}]}); + else if (releaseMetadata && args[0] === "image" && args[1] === "inspect" && args[3] === "{{json .}}") stdout=JSON.stringify({Id:imageIdentities.get(args.at(-1)!)??imageId,RepoDigests:[args.at(-1)],Os:"linux",Architecture:"amd64"}); else if (args[0] === "image" && args[1] === "inspect") stdout = imageIdentities.get(args.at(-1)!) ?? imageId; - else if (args[0] === "image" && args[1] === "pull") await imagePullProcess?.(args[2]!,options.signal); + else if (args[0] === "image" && args[1] === "pull") await imagePullProcess?.(args.at(-1)!,options.signal); else if (args[0] === "compose" && args.includes("up")) await creatingProcess?.(args, options.signal); else if (args[0] === "stop") { diff --git a/apps/ops-agent/src/configuration-handoff-application.ts b/apps/ops-agent/src/configuration-handoff-application.ts index 0552feeb5..743cfe46f 100644 --- a/apps/ops-agent/src/configuration-handoff-application.ts +++ b/apps/ops-agent/src/configuration-handoff-application.ts @@ -1,5 +1,6 @@ /** Fixed same-image helper effects. Only Docker process I/O is replaceable in * tests; ownership, source checks, ordering and verification are installed. */ +import { realpathSync } from "node:fs"; import { join } from "node:path"; import { assertArtifactTargetLeaseForward, @@ -16,6 +17,7 @@ import { writePrivateRecord, } from "@openmapx/core/build-artifacts"; import { canonicalJson } from "@openmapx/core/ops"; +import { atomicWrite } from "@openmapx/core/release-files"; import { repoPaths, services } from "@openmapx/core/server"; import { type ConfigurationHandoffWorkerRecord, @@ -24,7 +26,12 @@ import { configurationHandoffRecordDomains as domains, configurationHandoffRecordName as name, } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; +import { maintenanceDockerContainerSchema } from "../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records"; import { configurationMaintenanceOwnerRecordSchema } from "../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records"; +import { + readConfigurationHandoffReleaseOverlay, + readConfigurationHandoffReleaseState, +} from "../../../packages/core/src/build-artifacts/configuration-source-witness"; import { installationState } from "../../../packages/core/src/build-artifacts/control-authority"; import { configurationHandoffServiceOrder } from "./configuration-handoff"; import { @@ -34,6 +41,8 @@ import { waitForConfigurationHandoffPreparation, } from "./configuration-handoff-control"; import { inspectMaintenanceDockerRuntime } from "./configuration-maintenance-docker-inspection"; +import { readCheckedConfigurationRelease } from "./configuration-release"; +import { inspectConfigurationReleaseImages } from "./configuration-release-images"; import { type DockerRuntimeOptions, runContainedProcess } from "./docker-runtime"; import { createOwnedServiceResolver } from "./owned-services"; import { createTrustedConfigurationAuthorityLoader } from "./trusted-configuration-authority"; @@ -81,7 +90,7 @@ function ownerFor(child: ConfigurationHandoffChild) { !owner.apiControls.decide?.receiptDelivery || owner.receipt.generation !== "committed" || owner.receipt.appliedRevisionId !== intent.operation.expectedCurrentRevisionId || - intent.operation.mode.kind !== "configuration" + (intent.operation.mode.kind === "release" && !intent.release) ) throw Error("Original active configuration handoff owner required"); return owner; @@ -167,27 +176,204 @@ export async function runConfigurationHandoffApplication(input: { execution: intent.helper, preparationDigest: controlDigest(parent), observations: [], + ...(intent.release + ? { + release: { + images: intent.release.downloads.images.map((image) => ({ + image: image.image, + reference: image.reference, + phase: "pending" as const, + imageId: null, + })), + publication: "not_attempted" as const, + overlay: null, + statePublication: "not_attempted" as const, + state: null, + }, + } + : {}), sequence: 0, phase: "draining", updatedAt: Date.now(), }, true, ); + if (intent.release) { + const release = intent.release; + for (const image of release.downloads.images) { + await services.withConfigurationGenerationWriter( + stack.infraDir, + async () => { + const lock = await services.acquireReleaseStoreLock( + join(stack.infraDir, ".ops-agent-releases"), + ); + try { + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + ownerFor(child); + readCheckedConfigurationRelease(stack.infraDir, { + releaseId: release.releaseId, + manifestDigest: release.manifestDigest, + }); + const current = worker!; + if ( + current.release!.images.find((value) => value.image === image.image)?.phase !== + "pending" + ) + throw Error("Original attempted release pull requires observation"); + save({ + ...current, + sequence: current.sequence + 1, + release: { + ...current.release!, + images: current.release!.images.map((value) => + value.image === image.image ? { ...value, phase: "attempted" } : value, + ), + }, + updatedAt: Date.now(), + }); + } finally { + lock.release(); + } + }, + signal, + ); + // No writer lock spans external pull/recreation. The original target + // closure and deadline survive this fixed awaited process. + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + await execFile( + "docker", + ["image", "pull", "--platform", release.downloads.platform, image.reference], + { + signal, + timeout: Math.max(1, intent.helper.deadlineAt - Date.now()), + maxBuffer: 1024 * 1024, + }, + ); + const [observed] = await inspectConfigurationReleaseImages({ + downloads: release.downloads, + image: image.image, + execFile, + signal, + }); + if (!observed) throw Error("Original completed release pull unavailable"); + const current = worker!; + save({ + ...current, + sequence: current.sequence + 1, + release: { + ...current.release!, + images: current.release!.images.map((value) => + value.image === image.image + ? { ...value, phase: "verified", imageId: observed.imageId } + : value, + ), + }, + updatedAt: Date.now(), + }); + } + await services.withConfigurationGenerationWriter( + stack.infraDir, + async () => { + const lock = await services.acquireReleaseStoreLock( + join(stack.infraDir, ".ops-agent-releases"), + ); + try { + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + ownerFor(child); + const checked = readCheckedConfigurationRelease(stack.infraDir, { + releaseId: release.releaseId, + manifestDigest: release.manifestDigest, + }), + current = worker!; + if ( + current.release!.images.some((image) => image.phase !== "verified") || + canonicalJson(readConfigurationHandoffReleaseOverlay(stack.infraDir)) !== + canonicalJson(intent.previousReleaseOverlay) + ) + throw Error("Original release publication source changed"); + save({ + ...current, + sequence: current.sequence + 1, + phase: "applying", + release: { ...current.release!, publication: "attempted" }, + updatedAt: Date.now(), + }); + services.writeReleaseComposeArtifacts(checked.manifest, stack.composeReleasePath); + const published = readConfigurationHandoffReleaseOverlay(stack.infraDir); + if ( + !published || + Buffer.from(published.bytes, "base64").toString("utf8") !== + services.renderReleaseCompose(checked.manifest) + ) + throw Error("Original release overlay publication incomplete"); + save({ + ...worker!, + sequence: worker!.sequence + 1, + release: { ...worker!.release!, publication: "published", overlay: published }, + updatedAt: Date.now(), + }); + } finally { + lock.release(); + } + }, + signal, + ); + assertConfigurationHandoffHostSources(child); + } + const inspectTarget = async (id: string, owner: ReturnType) => { + if (!intent.release) + return (await owned.inspectOwnedServices(signal)).find((value) => value.serviceId === id); + // A changed overlay must not weaken the general owned-service resolver. + // Only this original owner may retain its exact old immutable container. + const original = owner.originalContainers.find((value) => value.serviceId === id); + if (!original) throw Error("Original release container required"); + const result = await execFile( + "docker", + ["container", "inspect", "--format", "{{json .}}", original.containerId], + { signal, timeout: 15000, maxBuffer: 1024 * 1024 }, + ); + if (Buffer.byteLength(result.stdout) > 1024 * 1024) + throw Error("Original container observation capacity"); + const observed = maintenanceDockerContainerSchema.parse(JSON.parse(result.stdout)), + labels = observed.Config.Labels; + if ( + observed.Id !== original.containerId || + observed.Image !== original.provenance.imageId || + labels["com.docker.compose.project"] !== original.provenance.project || + labels["com.docker.compose.service"] !== id || + realpathSync(labels["com.docker.compose.project.working_dir"]!) !== + original.provenance.composeWorkingDirectory || + canonicalJson( + labels["com.docker.compose.project.config_files"] + ?.split(",") + .map((path) => realpathSync(path.trim())), + ) !== canonicalJson(original.provenance.composeConfigFiles) || + (original.state === "running" + ? !observed.State.Running || observed.State.Status !== "running" + : observed.State.Running || + (original.state === "created" + ? observed.State.Status !== "created" + : !["exited", "dead"].includes(observed.State.Status))) + ) + throw Error("Original release container changed"); + return original; + }; for (const id of configurationHandoffServiceOrder(intent.operation.serviceIds)) { signal.throwIfAborted(); const owner = ownerFor(child), authority = await loadAuthority(); if (authority.revisionId !== owner.plan.sourceAuthorityRevision) throw Error("Original helper authority changed"); - const target = (await owned.inspectOwnedServices(signal)).find( - (value) => value.serviceId === id, - ), + const target = await inspectTarget(id, owner), original = owner.originalContainers.find((value) => value.serviceId === id); if ( !target || !original || target.containerId !== original.containerId || - target.state !== "running" || + (!intent.release && target.state !== "running") || target.provenance.imageId !== intent.operation.expectedImageIdentities[id] ) throw Error("Original helper target changed"); @@ -207,11 +393,24 @@ export async function runConfigurationHandoffApplication(input: { intent.operation.expectedCurrentRevisionId ) throw Error("Original helper generation changed"); - const fresh = (await owned.inspectOwnedServices(signal)).find( - (value) => value.serviceId === id, - ); + const fresh = await inspectTarget(id, ownerFor(child)); if (!fresh || canonicalJson(fresh) !== canonicalJson(target)) throw Error("Original helper target raced"); + if (intent.release) { + const reference = + intent.release.serviceImages[id as keyof typeof intent.release.serviceImages], + pulled = worker!.release!.images.find((value) => value.reference === reference); + if (!pulled || pulled.phase !== "verified") + throw Error("Original verified release image required"); + const [observed] = await inspectConfigurationReleaseImages({ + downloads: intent.release.downloads, + image: pulled.image, + execFile, + signal, + }); + if (observed?.imageId !== pulled.imageId) + throw Error("Original pulled release image changed"); + } const current = worker!; save({ ...current, @@ -287,6 +486,56 @@ export async function runConfigurationHandoffApplication(input: { privateKey: local.privateKey, signal, }); + if (intent.release) { + await services.withConfigurationGenerationWriter( + stack.infraDir, + async () => { + const lock = await services.acquireReleaseStoreLock( + join(stack.infraDir, ".ops-agent-releases"), + ); + try { + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + const owner = ownerFor(child); + readCheckedConfigurationRelease(stack.infraDir, { + releaseId: intent.release!.releaseId, + manifestDigest: intent.release!.manifestDigest, + }); + if ( + owner.plan.apiParent.issuer !== "api" || + canonicalJson(readConfigurationHandoffReleaseState(stack.infraDir)) !== + canonicalJson(intent.previousReleaseState) + ) + throw Error("Original release state publication source changed"); + const current = worker!, + bytes = JSON.stringify({ + releaseId: intent.release!.releaseId, + digest: intent.release!.manifestDigest, + updateJobId: owner.plan.apiParent.apiJobId, + }); + save({ + ...current, + sequence: current.sequence + 1, + release: { ...current.release!, statePublication: "attempted" }, + updatedAt: Date.now(), + }); + atomicWrite(join(stack.infraDir, ".ops-agent-releases", "current.json"), bytes); + const published = readConfigurationHandoffReleaseState(stack.infraDir); + if (!published || Buffer.from(published.bytes, "base64").toString("utf8") !== bytes) + throw Error("Original release state publication incomplete"); + save({ + ...worker!, + sequence: worker!.sequence + 1, + release: { ...worker!.release!, statePublication: "published", state: published }, + updatedAt: Date.now(), + }); + } finally { + lock.release(); + } + }, + signal, + ); + } const completedAt = Date.now(), current = worker!; save({ diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index 81e08edde..4d333a342 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -43,6 +43,10 @@ import { configurationHandoffRecordDomains as domains, configurationHandoffRecordName as recordName, } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; +import { + readConfigurationHandoffReleaseOverlay, + readConfigurationHandoffReleaseState, +} from "../../../packages/core/src/build-artifacts/configuration-source-witness"; import { childSettlementState, installationState, @@ -246,7 +250,15 @@ export async function prepareConfigurationHandoffLaunch( delegationId: observed.delegationId, assertion: attempt.assertion, helperContainerName: `openmapx-configuration-handoff-${operation.handoffId}`, - ...(owned.owner.plan.release ? { release: owned.owner.plan.release } : {}), + ...(owned.owner.plan.release + ? { + release: owned.owner.plan.release, + previousReleaseOverlay: readConfigurationHandoffReleaseOverlay( + owned.source.stack.infraDir, + ), + previousReleaseState: readConfigurationHandoffReleaseState(owned.source.stack.infraDir), + } + : {}), preparedAt: Date.now(), }); writePrivateRecord( diff --git a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts index cf13d3216..ab01684c4 100644 --- a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts +++ b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts @@ -11,6 +11,7 @@ import { maintenanceDockerContainerSchema as container, } from "../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records"; import type { ConfigurationMaintenanceOwnerRecord } from "./configuration-maintenance-store"; +import { inspectConfigurationReleaseImages } from "./configuration-release-images"; import { verifyConfigurationRuntimeResources } from "./configuration-resource-inspection"; import type { DockerRuntimeOptions } from "./docker-runtime"; import type { OwnedServiceSnapshot } from "./owned-services"; @@ -83,9 +84,27 @@ export async function inspectMaintenanceDockerRuntime(options: { id === "data-manager" && owner.receipt.generation === "committed" ? owner.plan.coordinatorImage : undefined, - expectedImageId = frozenImage?.imageId ?? original[0]!.provenance.imageId; - if (frozenImage && spec.image !== frozenImage.imageReference) - throw Error("Frozen coordinator image reference changed"); + release = owner.receipt.generation === "committed" ? owner.plan.release : undefined, + releaseReference = release?.serviceImages[id as keyof typeof release.serviceImages], + download = release?.downloads.images.find((value) => value.reference === releaseReference), + releaseImage = + release && download + ? ( + await inspectConfigurationReleaseImages({ + downloads: release.downloads, + image: download.image, + execFile: options.execFile, + signal, + }) + )[0] + : undefined, + expectedImageId = + releaseImage?.imageId ?? frozenImage?.imageId ?? original[0]!.provenance.imageId; + if ( + (frozenImage && spec.image !== frozenImage.imageReference) || + (release && (!download || !releaseImage || spec.image !== releaseReference)) + ) + throw Error("Frozen release/coordinator image reference changed"); const owned = current[0]!, observed = container.parse( JSON.parse( @@ -154,6 +173,12 @@ export async function inspectMaintenanceDockerRuntime(options: { containerId: owned.containerId, imageId: observed.Image, expectedImageId, + ...(releaseImage && download + ? { + releaseReference: releaseImage.reference, + releasePlatformManifestDigest: download.platformManifestDigest, + } + : {}), }); const mounts = []; for (const volume of spec.volumes) { @@ -184,10 +209,20 @@ export async function inspectMaintenanceDockerRuntime(options: { readOnly: !actual.RW, }); } - for (const secret of spec.secrets) { - const path = expected.secrets[secret.source]?.file, - target = `/run/secrets/${secret.target ?? secret.source}`, - matches = observed.Mounts.filter((value) => value.Destination === target), + const mountedFiles = [ + ...spec.secrets.map((secret) => ({ + path: expected.secrets[secret.source]?.file, + target: `/run/secrets/${secret.target ?? secret.source}`, + })), + ...spec.configs.map((config) => ({ + path: expected.configs[config.source]?.file, + target: config.target?.startsWith("/") + ? config.target + : `/${config.target ?? config.source}`, + })), + ]; + for (const { path, target } of mountedFiles) { + const matches = observed.Mounts.filter((value) => value.Destination === target), actual = matches[0]; if ( !path || @@ -198,7 +233,7 @@ export async function inspectMaintenanceDockerRuntime(options: { !actual.Source || (await realpath(path)) !== (await realpath(actual.Source)) ) - throw Error("Original applied secret mount differs"); + throw Error("Original applied configuration/secret mount differs"); const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); try { const before = await file.stat(), @@ -210,7 +245,7 @@ export async function inspectMaintenanceDockerRuntime(options: { before.dev !== named.dev || before.ino !== named.ino ) - throw Error("Original secret source unavailable"); + throw Error("Original configuration/secret source unavailable"); const expectedBytes = await file.readFile(), encoded = (await run(["container", "exec", observed.Id, "base64", target])).replace( /[\r\n]/g, @@ -230,7 +265,7 @@ export async function inspectMaintenanceDockerRuntime(options: { encoded !== actualBytes.toString("base64") || !timingSafeEqual(Buffer.from(expectedDigest, "hex"), Buffer.from(actualDigest, "hex")) ) - throw Error("Original runtime secret content differs"); + throw Error("Original runtime configuration/secret content differs"); mounts.push({ targetDigest: keyed(target), type: actual.Type, diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index ee46137da..1f38a0484 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -13,6 +13,10 @@ import { coordinatorPullSubmissionDigest, parseOpsJobStatus } from "@openmapx/co import { services } from "@openmapx/core/server"; import { assertProspectiveConfigurationSettlementPacketBudget } from "../../../packages/core/src/build-artifacts/configuration-fence-settlement-budget"; import { assertProspectiveConfigurationMaintenanceBudget } from "../../../packages/core/src/build-artifacts/configuration-maintenance-budget"; +import { + readConfigurationHandoffReleaseOverlay, + readConfigurationHandoffReleaseState, +} from "../../../packages/core/src/build-artifacts/configuration-source-witness"; import { type CoordinatorPullWitness, coordinatorPullObservationDomain, @@ -206,6 +210,7 @@ import { configurationMaintenanceChildState, installationState, publishConfigurationMaintenanceForwardWitness, + readCompletedConfigurationHandoff, verifyConfigurationMaintenanceChildTerminal, verifyControlPreparedTerminal, withPrivateEffectWriter, @@ -4354,6 +4359,44 @@ export async function inspectFrozenMaintenanceRuntime( throw Error("Unchanged original owned runtime differs"); if (contained && canonicalJson(containers) !== canonicalJson(owner.originalContainers)) throw Error("Original noncommitted runtime changed"); + let currentRelease: { releaseId: string; digest: string; updateJobId: string } | undefined; + if (!contained && owner.plan.release) { + if (!owner.plan.handoff || owner.plan.apiParent.issuer !== "api") + throw Error("Original release helper/current-state owner required"); + const original = readCompletedConfigurationHandoff( + state.installation.installation, + owner.plan.handoff.handoffId, + ), + published = original.completion.release, + overlay = readConfigurationHandoffReleaseOverlay(source.infraDir), + pointer = readConfigurationHandoffReleaseState(source.infraDir); + currentRelease = { + releaseId: owner.plan.release.releaseId, + digest: owner.plan.release.manifestDigest, + updateJobId: owner.plan.apiParent.apiJobId, + }; + if ( + canonicalJson(original.intent.ownerPrepare) !== canonicalJson(owner.plan.prepare) || + original.intent.ownerCertificateDigest !== owner.rootCertificateDigest || + original.intent.operation.planDigest !== owner.plan.planDigest || + canonicalJson(original.intent.release) !== canonicalJson(owner.plan.release) || + published?.publication !== "published" || + published.statePublication !== "published" || + canonicalJson(overlay) !== canonicalJson(published.overlay) || + canonicalJson(pointer) !== canonicalJson(published.state) || + !overlay || + !pointer || + Buffer.from(pointer.bytes, "base64").toString("utf8") !== JSON.stringify(currentRelease) || + Buffer.from(overlay.bytes, "base64").toString("utf8") !== + services.renderReleaseCompose( + services.parseReleaseManifest( + owner.plan.release.canonicalManifest, + owner.plan.release.imagePrefix, + ), + ) + ) + throw Error("Original helper release publication changed before final inspection"); + } const docker = await inspectMaintenanceDockerRuntime({ stack: source.stack, execFile: source.execFile, @@ -4426,7 +4469,10 @@ export async function inspectFrozenMaintenanceRuntime( ...docker.checks, ]), details = [ - { currentRevisionId: services.currentConfigurationGeneration(source.infraDir) }, + { + currentRevisionId: services.currentConfigurationGeneration(source.infraDir), + ...(currentRelease ? { currentRelease } : {}), + }, { containers }, { guard }, ...owner.plan.steps.map((step) => ({ @@ -5394,8 +5440,13 @@ export async function prepareConfigurationMaintenanceAdmission( expectedImageIdentities: Object.fromEntries( protectedIds.map((id) => { const old = originalContainers.find((value) => value.serviceId === id); - if (!old || old.state !== "running") - throw Error("Original protected target unavailable"); + if ( + !old || + (release && !["running", "stopped", "created"].includes(old.state)) || + (!release && old.state !== "running") || + (release && ["data-manager", "ops-agent"].includes(id) && old.state !== "running") + ) + throw Error("Original handoff target unavailable"); return [id, old.provenance.imageId]; }), ), @@ -5427,6 +5478,16 @@ export async function prepareConfigurationMaintenanceAdmission( ...(coordinatorImage ? { coordinatorImage } : {}), }); assertProspectiveConfigurationMaintenanceBudget({ + ...(release + ? { + releaseSourceBytes: Buffer.byteLength( + canonicalJson({ + overlay: readConfigurationHandoffReleaseOverlay(source.infraDir), + state: readConfigurationHandoffReleaseState(source.infraDir), + }), + ), + } + : {}), installationId: input.installation.installationId, plan: frozen, originalContainers: structuredClone(originalContainers), diff --git a/apps/ops-agent/src/configuration-release-images.test.ts b/apps/ops-agent/src/configuration-release-images.test.ts index e1761c531..d3e3c7fab 100644 --- a/apps/ops-agent/src/configuration-release-images.test.ts +++ b/apps/ops-agent/src/configuration-release-images.test.ts @@ -133,3 +133,22 @@ it("checks cancellation before and after external IO and bounds stdout", async ( }), ).rejects.toThrow(); }); + +it("checks the exact completed pull independently before moving to another image", async () => { + const execFile = io(); + const input = { + downloads: downloads(), + execFile, + signal: new AbortController().signal, + image: "ops-agent" as const, + }; + const result = await inspectConfigurationReleaseImages(input); + expect(result).toEqual([ + { + image: "ops-agent", + reference: input.downloads.images[3]!.reference, + imageId: `sha256:${"d".repeat(64)}`, + }, + ]); + expect(execFile).toHaveBeenCalledTimes(1); +}); diff --git a/apps/ops-agent/src/configuration-release-images.ts b/apps/ops-agent/src/configuration-release-images.ts index b1c5d5c84..fde54bafa 100644 --- a/apps/ops-agent/src/configuration-release-images.ts +++ b/apps/ops-agent/src/configuration-release-images.ts @@ -18,11 +18,17 @@ export async function inspectConfigurationReleaseImages(input: { downloads: ConfigurationReleaseDownloads; execFile: NonNullable; signal: AbortSignal; + image?: ConfigurationReleaseDownloads["images"][number]["image"]; }) { const downloads = configurationReleaseDownloadsSchema.parse(input.downloads), [os, architecture] = downloads.platform.split("/"), observations = []; - for (const image of downloads.images) { + const selected = + input.image === undefined + ? downloads.images + : downloads.images.filter((image) => image.image === input.image); + if (selected.length === 0) throw Error("Fixed checked release image required"); + for (const image of selected) { input.signal.throwIfAborted(); const response = await input.execFile( "docker", @@ -49,6 +55,8 @@ export async function inspectConfigurationReleaseImages(input: { throw Error("Checked release local image does not match its pinned platform"); // Docker's local image ID is observed independently. An OCI index digest // or repository reference must never be substituted for that runtime ID. + // OCI no longer specifies runtime ImageID construction: + // https://github.com/opencontainers/image-spec/blob/main/config.md#imageid observations.push( Object.freeze({ image: image.image, reference: image.reference, imageId: observed.Id }), ); diff --git a/apps/ops-agent/src/trusted-config-runtime.ts b/apps/ops-agent/src/trusted-config-runtime.ts index f6caac672..fa59168f4 100644 --- a/apps/ops-agent/src/trusted-config-runtime.ts +++ b/apps/ops-agent/src/trusted-config-runtime.ts @@ -18,6 +18,10 @@ import { } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; import { assertProspectiveConfigurationMaintenancePreludeBudget } from "../../../packages/core/src/build-artifacts/configuration-maintenance-budget"; +import { + readConfigurationHandoffReleaseOverlay, + readConfigurationHandoffReleaseState, +} from "../../../packages/core/src/build-artifacts/configuration-source-witness"; import { executionAuthorityState, retainConfigurationGenerationNonCommit, @@ -424,7 +428,17 @@ export function installTrustedConfigurationRuntime( }) : undefined; assertProspectiveConfigurationMaintenancePreludeBudget({ - ...(release ? { release } : {}), + ...(release + ? { + release, + releaseSourceBytes: Buffer.byteLength( + canonicalJson({ + overlay: readConfigurationHandoffReleaseOverlay(options.infraDir), + state: readConfigurationHandoffReleaseState(options.infraDir), + }), + ), + } + : {}), installationId: installation.installationId, record: candidate.record, sourceAuthorityRevision: authority?.revisionId ?? "registry-v1", diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.test.ts b/packages/core/src/build-artifacts/configuration-handoff-records.test.ts index 7db3f9c17..f0257dfee 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.test.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "vitest"; import { configurationHandoffHelperBodySchema, configurationHandoffProgressSchema, + configurationHandoffReleaseProgressSchema, } from "./configuration-handoff-records"; const image = `sha256:${"a".repeat(64)}`; @@ -64,3 +65,40 @@ it("retains a reference-only helper body and truthful bounded launch/target prog }).success, ).toBe(false); }); + +it("does not confuse an attempted release pull/publication with verified local images and overlay", () => { + const image = { + image: "api", + reference: `ghcr.io/openmapx/api@sha256:${"a".repeat(64)}`, + phase: "pending", + imageId: null, + }; + const initial = { + images: [image], + publication: "not_attempted", + overlay: null, + statePublication: "not_attempted", + state: null, + }; + expect(configurationHandoffReleaseProgressSchema.parse(initial)).toEqual(initial); + expect( + configurationHandoffReleaseProgressSchema.safeParse({ + ...initial, + images: [{ ...image, phase: "verified" }], + }).success, + ).toBe(false); + expect( + configurationHandoffReleaseProgressSchema.safeParse({ + ...initial, + images: [{ ...image, imageId: `sha256:${"b".repeat(64)}` }], + }).success, + ).toBe(false); + expect( + configurationHandoffReleaseProgressSchema.safeParse({ ...initial, publication: "published" }) + .success, + ).toBe(false); + expect( + configurationHandoffReleaseProgressSchema.safeParse({ ...initial, images: [image, image] }) + .success, + ).toBe(false); +}); diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.ts b/packages/core/src/build-artifacts/configuration-handoff-records.ts index 1496cf9f6..921916918 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.ts @@ -2,6 +2,10 @@ * authority; producers and observers join these to the original signed grant. */ import { z } from "zod"; import { configurationHandoffPrepareOperationSchema } from "../ops/configuration-handoff-records"; +import { + CONFIGURATION_RELEASE_IMAGE_NAMES, + configurationReleaseDownloadSchema, +} from "../ops/configuration-release-downloads"; import { executionRefSchema } from "../ops/execution-ref"; import { configurationReleaseSnapshotSchema } from "./configuration-release-snapshot"; import { controlOriginSchema } from "./control-records"; @@ -9,6 +13,57 @@ import { controlOriginSchema } from "./control-records"; const digest = z.string().regex(/^[a-f0-9]{64}$/), containerId = z.string().regex(/^[a-f0-9]{64}$/), time = z.number().int().nonnegative().safe(); +/** Private original overlay bytes; never part of the cross-role public graph. */ +export const configurationHandoffOverlaySourceSchema = z + .strictObject({ + identity: z.strictObject({ + dev: z.number().int().nonnegative().safe(), + ino: z.number().int().nonnegative().safe(), + mtime: z.number().finite(), + ctime: z.number().finite(), + size: z + .number() + .int() + .nonnegative() + .max(128 * 1024), + }), + bytes: z.string().max(174764), + }) + .refine( + (value) => + Buffer.from(value.bytes, "base64").toString("base64") === value.bytes && + Buffer.from(value.bytes, "base64").length === value.identity.size, + ); +export const configurationHandoffReleaseProgressSchema = z + .strictObject({ + images: z + .array( + z.strictObject({ + image: z.enum(CONFIGURATION_RELEASE_IMAGE_NAMES), + reference: configurationReleaseDownloadSchema.shape.reference, + phase: z.enum(["pending", "attempted", "verified"]), + imageId: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .nullable(), + }), + ) + .max(CONFIGURATION_RELEASE_IMAGE_NAMES.length), + publication: z.enum(["not_attempted", "attempted", "published"]), + overlay: configurationHandoffOverlaySourceSchema.nullable(), + statePublication: z.enum(["not_attempted", "attempted", "published"]), + state: configurationHandoffOverlaySourceSchema.nullable(), + }) + .superRefine((value, context) => { + if ( + new Set(value.images.map((image) => image.image)).size !== value.images.length || + value.images.some((image) => (image.phase === "verified") !== (image.imageId !== null)) || + (value.publication === "published") !== (value.overlay !== null) || + (value.statePublication === "published") !== (value.state !== null) || + (value.statePublication !== "not_attempted" && value.publication !== "published") + ) + context.addIssue({ code: "custom", message: "Original verified release progress required" }); + }); export const configurationHandoffHelperBodySchema = z.strictObject({ kind: z.literal("configuration-helper"), handoffId: z.uuid(), @@ -50,12 +105,16 @@ export const configurationHandoffIntentRecordSchema = z helperContainerName: z.string().regex(/^openmapx-configuration-handoff-[a-f0-9-]{36}$/), preparedAt: time, release: configurationReleaseSnapshotSchema.optional(), + previousReleaseOverlay: configurationHandoffOverlaySourceSchema.nullable().optional(), + previousReleaseState: configurationHandoffOverlaySourceSchema.nullable().optional(), }) .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 65536) .superRefine((value, context) => { const { mode } = value.operation; if ( (mode.kind === "release") !== (value.release !== undefined) || + (mode.kind === "release") !== (value.previousReleaseOverlay !== undefined) || + (mode.kind === "release") !== (value.previousReleaseState !== undefined) || (mode.kind === "release" && value.release && (mode.releaseId !== value.release.releaseId || @@ -139,6 +198,7 @@ export const configurationHandoffWorkerRecordSchema = configurationHandoffProgre .safeExtend({ execution: executionRefSchema, preparationDigest: digest, + release: configurationHandoffReleaseProgressSchema.optional(), observations: z .array( z.strictObject({ @@ -150,6 +210,15 @@ export const configurationHandoffWorkerRecordSchema = configurationHandoffProgre .max(64), }) .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 65536) + .refine( + (value) => + !value.release || + value.phase !== "applied" || + (value.release.publication === "published" && + value.release.statePublication === "published" && + value.release.images.length === CONFIGURATION_RELEASE_IMAGE_NAMES.length && + value.release.images.every((image) => image.phase === "verified")), + ) .refine( (value) => new Set(value.observations.map((item) => item.serviceId)).size === value.observations.length, diff --git a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts index da12f6d9b..5dc6c7091 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts @@ -16,6 +16,7 @@ import { GENERATED_COMPOSE_FILE, STACK_PROJECT, } from "../services/deployment"; +import { parseReleaseManifest, renderReleaseCompose } from "../services/release-manifest"; import { assertProspectiveConfigurationSettlementPacketBudget } from "./configuration-fence-settlement-budget"; import type { ConfigurationMaintenanceOwnerRecord } from "./configuration-maintenance-owner-records"; import { @@ -32,6 +33,7 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { plan: FixedConfigurationMaintenancePlan; originalContainers: ConfigurationMaintenanceOwnerRecord["originalContainers"]; rendered: ReturnType; + releaseSourceBytes?: number; }) { const { plan, originalContainers, rendered } = input, { kind: _kind, ...binding } = plan.operation, @@ -147,7 +149,18 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { expectedImageId: null as string | null, })), details: unknown[] = [ - { currentRevisionId: plan.operation.revisionId }, + { + currentRevisionId: plan.operation.revisionId, + ...(plan.release + ? { + currentRelease: { + releaseId: plan.release.releaseId, + digest: plan.release.manifestDigest, + updateJobId: uuid, + }, + } + : {}), + }, { containers: futureContainers }, { guard }, ...plan.steps.map((step) => ({ @@ -158,7 +171,11 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { const service = rendered.resolved.enabled.find((value) => value.manifest.id === id); if (!service) throw Error("Prospective maintenance runtime target unavailable"); const snippet = renderServiceSnippet(service, rendered.context), - mountCount = (snippet.volumes?.length ?? 0) + (snippet.secrets?.length ?? 0); + // The fixed release renderer mounts its privacy validation config in API. + mountCount = + (snippet.volumes?.length ?? 0) + + (snippet.secrets?.length ?? 0) + + (plan.release && id === "app-api" ? 1 : 0); for (const check of ["configuration", "image", "mount", "readiness"]) checks.push({ serviceId: id, @@ -176,7 +193,18 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { expectedEnvironmentDigest: hash, actualEnvironmentDigest: hash, }, - { containerId: hash, imageId: `sha256:${hash}`, expectedImageId: `sha256:${hash}` }, + { + containerId: hash, + imageId: `sha256:${hash}`, + expectedImageId: `sha256:${hash}`, + ...(plan.release + ? { + releaseReference: + plan.release.serviceImages[id as keyof typeof plan.release.serviceImages], + releasePlatformManifestDigest: `sha256:${hash}`, + } + : {}), + }, { containerId: hash, mounts: Array.from({ length: mountCount }, () => ({ @@ -292,11 +320,39 @@ export function assertProspectiveConfigurationMaintenanceBudget(input: { phaseBytes = (plan.reservations.length + (plan.nestedHardlinks ? 1 : 0) + (plan.nestedHandoff ? 1 : 0)) * 8192; + const releaseWorkerBytes = plan.release + ? bytes({ + images: plan.release.downloads.images.map((image) => ({ + image: image.image, + reference: image.reference, + phase: "verified", + imageId: `sha256:${hash}`, + })), + overlay: Buffer.from( + renderReleaseCompose( + parseReleaseManifest(plan.release.canonicalManifest, plan.release.imagePrefix), + ), + ).toString("base64"), + state: Buffer.from( + JSON.stringify({ + releaseId: plan.release.releaseId, + digest: plan.release.manifestDigest, + updateJobId: uuid, + }), + ).toString("base64"), + }) + 24576 + : 0; if ( inspectionBytes > 65536 || + releaseWorkerBytes > 65536 || Math.max(ownerBytes, noGuardOwnerBytes) > 131072 || receiptBytes > 65536 || - phaseBytes > 131072 + phaseBytes > 131072 || + (plan.release && + (input.releaseSourceBytes === undefined || + !Number.isSafeInteger(input.releaseSourceBytes) || + input.releaseSourceBytes < 4 || + bytes(plan.release) + input.releaseSourceBytes + 24576 > 65536)) ) throw Error("Prospective maintenance owner/inspection/receipt/phase capacity exceeded"); return Object.freeze({ @@ -319,6 +375,7 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { originalContainers: ConfigurationMaintenanceOwnerRecord["originalContainers"]; rendered: ReturnType; release?: FixedConfigurationMaintenancePlan["release"]; + releaseSourceBytes?: number; coordinatorImage?: Omit< NonNullable, "imageId" @@ -472,6 +529,7 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { }; }); assertProspectiveConfigurationMaintenanceBudget({ + ...(input.release ? { releaseSourceBytes: input.releaseSourceBytes } : {}), installationId: input.installationId, plan, originalContainers: containers, diff --git a/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts index 4924f3851..2a732f680 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-docker-records.ts @@ -78,10 +78,15 @@ const service = z.object({ .array(z.object({ source: string, target: string.optional() })) .max(128) .default([]), + configs: z + .array(z.object({ source: string, target: string.optional() })) + .max(128) + .default([]), healthcheck: z.object({ disable: z.boolean().optional() }).optional(), }); export const maintenanceDockerComposeSchema = z.object({ services: z.record(z.string(), service), secrets: z.record(z.string(), z.object({ file: string.optional() })).default({}), + configs: z.record(z.string(), z.object({ file: string.optional() })).default({}), volumes: z.record(z.string(), z.object({ name: string.optional() })).default({}), }); diff --git a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts index 9a5991283..48794d02e 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts @@ -1,6 +1,7 @@ /** Private fixed final producer records. Parsing is never final authority. */ import { z } from "zod"; import { canonicalJson } from "../ops/canonical-json"; +import { ownedServiceSnapshotSchema } from "../ops/owned-service-records"; import { configurationFinalEvidenceProjectionSchema, configurationFinalEvidenceSchema, @@ -283,6 +284,40 @@ function assertExecutedConfigurationMaintenanceFinalState( ...(owner.plan.handoff?.serviceIds ?? []), ]), ].sort(), + releaseImage = (serviceId: string) => { + const release = owner.plan.release; + if (contained || !release) return undefined; + const index = 3 + owner.plan.steps.length + ids.indexOf(serviceId) * 4 + 1, + observation = inspection.observations[index]; + if (!observation || observation.checkIndex !== index) + throw Error("Original release image observation required"); + const image = z + .strictObject({ + containerId: z.string().regex(/^[a-f0-9]{64}$/), + imageId: z.string().regex(/^sha256:[a-f0-9]{64}$/), + expectedImageId: z.string().regex(/^sha256:[a-f0-9]{64}$/), + releaseReference: z.string(), + releasePlatformManifestDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + }) + .parse(JSON.parse(observation.detail)), + reference = release.serviceImages[serviceId as keyof typeof release.serviceImages], + download = release.downloads.images.find((value) => value.reference === reference), + containers = z + .strictObject({ containers: z.array(ownedServiceSnapshotSchema).max(256) }) + .parse(JSON.parse(inspection.observations[1]!.detail)) + .containers.filter((value) => value.serviceId === serviceId); + if ( + containers.length !== 1 || + containers[0]!.containerId !== image.containerId || + containers[0]!.provenance.imageId !== image.imageId || + !download || + image.releaseReference !== reference || + image.releasePlatformManifestDigest !== download.platformManifestDigest || + image.imageId !== image.expectedImageId + ) + throw Error("Original checked release image observation changed"); + return image.expectedImageId; + }, requiredChecks = configurationMaintenanceRequiredChecksSchema.parse([ ...["generation", "owner-containers", "dm-guard"].map((check) => ({ serviceId: null, @@ -305,10 +340,11 @@ function assertExecutedConfigurationMaintenanceFinalState( check, expectedRevisionId: inspectedRevision, expectedImageId: - !contained && serviceId === "data-manager" && owner.plan.coordinatorImage + releaseImage(serviceId) ?? + (!contained && serviceId === "data-manager" && owner.plan.coordinatorImage ? owner.plan.coordinatorImage.imageId : owner.originalContainers.find((value) => value.serviceId === serviceId)?.provenance - .imageId, + .imageId), })), ), ]), @@ -379,7 +415,19 @@ function assertExecutedConfigurationMaintenanceFinalState( !same(inspection.requiredChecks, requiredChecks) || inspection.observations.length !== requiredChecks.length || !inspection.observations.every((value, index) => value.checkIndex === index && value.passed) || - !same(details[0], { currentRevisionId: inspectedRevision }) || + !same(details[0], { + currentRevisionId: inspectedRevision, + ...(!contained && owner.plan.release + ? { + currentRelease: { + releaseId: owner.plan.release.releaseId, + digest: owner.plan.release.manifestDigest, + updateJobId: + owner.plan.apiParent.issuer === "api" ? owner.plan.apiParent.apiJobId : null, + }, + } + : {}), + }) || !same(details[2], { guard: owner.guardSource.preparedReceipt }) || !owner.plan.steps.every((step, index) => same(details[index + 3], { diff --git a/packages/core/src/build-artifacts/configuration-source-witness.ts b/packages/core/src/build-artifacts/configuration-source-witness.ts index c31c52e74..9dedd3bb5 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.ts @@ -15,7 +15,13 @@ import { readAppliedPlatformConfiguration, } from "../services/configuration-generation"; import { SERVICE_SELECTION_FILE } from "../services/deployment"; +import { parseReleaseManifest, renderReleaseCompose } from "../services/release-manifest"; import type { LoadedService } from "../services/types"; +import { + configurationHandoffRecordDomains, + configurationHandoffRecordName, + configurationHandoffWorkerRecordSchema, +} from "./configuration-handoff-records"; import { configurationMaintenanceOwnerRecordSchema } from "./configuration-maintenance-owner-records"; import { assertControlExecutionForward, @@ -43,11 +49,15 @@ const witnessSchema = z.strictObject({ }); const name = (id: string) => `configuration-source-${z.uuid().parse(id)}.json`; -function handoffSourceBytes(infraDir: string, environmentNames: readonly string[]) { +function handoffSourceBytes( + infraDir: string, + environmentNames: readonly string[], + files = currentSources(infraDir, {}), +) { // File values stay private/keyed, including credentials. Process-only helper // transport variables are absent from the original inherited-name projection. return canonicalJson({ - files: currentSources(infraDir, {}), + files, runtimeEnvironment: Object.fromEntries( environmentNames.map((name) => [name, process.env[name] ?? null]), ), @@ -158,16 +168,50 @@ export function assertConfigurationHandoffHostSources(authority: VerifiedExecuti state.execution, raw.payload.operation.handoffId, ), - observed = createHmac("sha256", local.privateKey.export({ type: "pkcs8", format: "der" })) - .update(helperSourceDomain) - .update(handoffSourceBytes(dirname(local.dataRoot), intent.environmentNames)) - .digest("hex"); + infraDir = dirname(local.dataRoot), + keyed = (files?: ReturnType) => + createHmac("sha256", local.privateKey.export({ type: "pkcs8", format: "der" })) + .update(helperSourceDomain) + .update(handoffSourceBytes(infraDir, intent.environmentNames, files)) + .digest("hex"), + matches = (value: string) => + timingSafeEqual(Buffer.from(value, "hex"), Buffer.from(intent.sourceDigest, "hex")); + if (currentConfigurationGeneration(infraDir) !== intent.operation.expectedCurrentRevisionId) + throw Error("Helper host generation changed"); + if (matches(keyed())) return; + if (!intent.release || intent.previousReleaseOverlay === undefined) + throw Error("Helper host sources changed"); + // The only allowed source transition is this original helper's indexed, + // signed publication of its exact frozen renderer output. All other files, + // environment values and the original generation remain under the old HMAC. + const file = configurationHandoffRecordName(intent.operation.handoffId, "completion"), + publication = readPrivateRecord(local, "observations", file), + worker = verifyControl( + intent.helperPublicKey, + configurationHandoffRecordDomains.completion, + publication, + configurationHandoffWorkerRecordSchema, + ), + files = currentSources(infraDir, {}); + verifyFamilyOwnedRecord( + local, + intent.origin, + { scope: "private", namespace: "observations", container: "", name: file }, + canonicalJson(publication), + ); if ( - currentConfigurationGeneration(dirname(local.dataRoot)) !== - intent.operation.expectedCurrentRevisionId || - !timingSafeEqual(Buffer.from(observed, "hex"), Buffer.from(intent.sourceDigest, "hex")) + worker.intentDigest !== controlDigest(intent) || + canonicalJson(worker.execution) !== canonicalJson(intent.helper) || + worker.release?.publication !== "published" || + canonicalJson(worker.release.overlay) !== canonicalJson(files.release) || + !files.release || + Buffer.from(files.release.bytes, "base64").toString("utf8") !== + renderReleaseCompose( + parseReleaseManifest(intent.release.canonicalManifest, intent.release.imagePrefix), + ) || + !matches(keyed({ ...files, release: intent.previousReleaseOverlay })) ) - throw Error("Helper host sources changed"); + throw Error("Original helper release publication changed"); } function hostEnv(infraDir: string, env: NodeJS.ProcessEnv) { const path = join(infraDir, ".env"); @@ -241,14 +285,13 @@ function hostEnv(infraDir: string, env: NodeJS.ProcessEnv) { return { env: { ...file, ...env }, identity, selection }; } -/** Private, bounded source capture. Raw overlay bytes never enter the public inspection. */ -function releaseSource(infraDir: string) { - const path = join(infraDir, "docker-compose.release.yml"); +/** Bounded private data only. Callers own their existing writer/target checks. */ +function readHandoffSourceFile(path: string, maximumBytes: number) { let fd: number | undefined; try { fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); const before = fstatSync(fd); - if (!before.isFile() || before.nlink !== 1 || before.size > 128 * 1024) + if (!before.isFile() || before.nlink !== 1 || before.size > maximumBytes) throw Error("Host sources unavailable"); const bytes = readFileSync(fd), after = fstatSync(fd), @@ -281,11 +324,18 @@ function releaseSource(infraDir: string) { if (fd !== undefined) closeSync(fd); } } +/** Raw values stay exclusively in role-private helper records. */ +export function readConfigurationHandoffReleaseOverlay(infraDir: string) { + return readHandoffSourceFile(join(infraDir, "docker-compose.release.yml"), 128 * 1024); +} +export function readConfigurationHandoffReleaseState(infraDir: string) { + return readHandoffSourceFile(join(infraDir, ".ops-agent-releases", "current.json"), 4096); +} function currentSources(infraDir: string, env: NodeJS.ProcessEnv) { const platform = readAppliedPlatformConfiguration(infraDir); return { ...hostEnv(infraDir, env), - release: releaseSource(infraDir), + release: readConfigurationHandoffReleaseOverlay(infraDir), ...(platform ? { platform } : {}), }; } diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 4760a33ba..416ff517e 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -5163,6 +5163,17 @@ export function readCompletedConfigurationHandoff( completion.intentDigest !== controlDigest(intent) || completion.preparationDigest !== controlDigest(progress) || completion.phase !== "applied" || + !!intent.release !== !!completion.release || + (intent.release && + completion.release && + (completion.release.images.length !== intent.release.downloads.images.length || + completion.release.images.some( + (image, index) => + image.image !== intent.release!.downloads.images[index]?.image || + image.reference !== intent.release!.downloads.images[index]?.reference || + image.phase !== "verified" || + !image.imageId, + ))) || completion.launch.containerId !== progress.launch.containerId || canonicalJson(completion.execution) !== canonicalJson(intent.helper) || canonicalJson(effect.identity) !== canonicalJson(intent.helper) || diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts index a1932a4ee..db399e2fd 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -309,6 +309,8 @@ it("freezes a checked release snapshot in the original helper graph without a pr helperContainerName: `openmapx-configuration-handoff-${handoff.handoffId}`, preparedAt: Date.now(), release, + previousReleaseOverlay: null, + previousReleaseState: null, }; expect(configurationHandoffIntentRecordSchema.parse(intent).release).toEqual(release); expect( From e661935f2d10da60d85a5f49f4712fb529a4525c Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 11:39:52 +0100 Subject: [PATCH 081/135] fix(platform): fence guarded updates from unfinished legacy handoffs --- apps/ops-agent/src/agent-handoff.ts | 9 ++- .../src/configuration-release.test.ts | 68 ++++++++++++++++++- apps/ops-agent/src/configuration-release.ts | 16 +++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/apps/ops-agent/src/agent-handoff.ts b/apps/ops-agent/src/agent-handoff.ts index 25c152f12..1fb0486ab 100644 --- a/apps/ops-agent/src/agent-handoff.ts +++ b/apps/ops-agent/src/agent-handoff.ts @@ -85,9 +85,16 @@ function parseRecord(raw: string): AgentHandoffRecord { } export function readAgentHandoff(rootDir: string): AgentHandoffRecord | null { + return readAgentHandoffFile(agentHandoffPath(rootDir)); +} +/** Existing legacy journal data, read without startup mutation or repair. */ +export function readAgentHandoffFromInfra(infraDir: string): AgentHandoffRecord | null { + return readAgentHandoffFile(join(infraDir, "data", "ops-agent", "agent-handoff.json")); +} +function readAgentHandoffFile(path: string): AgentHandoffRecord | null { let raw: string; try { - raw = strictReadFile(agentHandoffPath(rootDir), MAX_HANDOFF_BYTES); + raw = strictReadFile(path, MAX_HANDOFF_BYTES); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw new Error("Agent handoff record rejected"); diff --git a/apps/ops-agent/src/configuration-release.test.ts b/apps/ops-agent/src/configuration-release.test.ts index 878c9702d..5bece7421 100644 --- a/apps/ops-agent/src/configuration-release.test.ts +++ b/apps/ops-agent/src/configuration-release.test.ts @@ -1,5 +1,13 @@ import { createHash } from "node:crypto"; -import { linkSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { CONFIGURATION_RELEASE_IMAGE_NAMES } from "@openmapx/core/ops"; @@ -250,3 +258,61 @@ it.each([false, true])( expect(await run()).toMatchObject({ releaseId, manifestDigest }); }, ); + +it.each(["transaction", "active-handoff", "consumed-handoff"])( + "preserves legacy release recovery sources during guarded preflight: %s", + async (kind) => { + const { preflightConfigurationRelease } = await import("./configuration-release-preflight"); + const { writeAgentHandoff, agentHandoffPath } = await import("./agent-handoff"); + const checkout = join(root, "legacy-checkout"), + infraDir = join(checkout, "infra/docker"), + store = join(infraDir, ".ops-agent-releases"); + mkdirSync(store, { recursive: true, mode: 0o700 }); + writeFileSync(join(store, `${releaseId}.json`), raw, { mode: 0o600 }); + let legacy: string; + if (kind === "transaction") { + legacy = join(store, "transaction.json"); + writeFileSync(legacy, JSON.stringify({ version: 1, phase: "prepared" }), { mode: 0o600 }); + } else { + mkdirSync(join(infraDir, "data/ops-agent"), { recursive: true, mode: 0o700 }); + writeAgentHandoff(checkout, { + version: 1, + handoffId: "legacy-handoff-01", + releaseId, + digest: manifestDigest, + phase: kind === "consumed-handoff" ? "applied" : "launched", + consumed: kind === "consumed-handoff", + previousOverlay: null, + createdAt: "2026-10-30T00:00:00Z", + updatedAt: "2026-10-30T00:00:00Z", + }); + legacy = agentHandoffPath(checkout); + } + const before = readFileSync(legacy); + const execFile = vi.fn(async (_file: string, args: readonly string[]) => ({ + stdout: + args[0] === "info" + ? "linux/amd64" + : JSON.stringify({ + schemaVersion: 2, + config: { digest: `sha256:${"c".repeat(64)}`, size: 32 }, + layers: [{ digest: `sha256:${"d".repeat(64)}`, size: 64 }], + }), + stderr: "", + })); + const run = () => + preflightConfigurationRelease({ + infraDir, + selection: { releaseId, manifestDigest }, + execFile, + signal: new AbortController().signal, + }); + if (kind === "consumed-handoff") + expect(await run()).toMatchObject({ releaseId, manifestDigest }); + else { + await expect(run()).rejects.toThrow(/legacy/i); + expect(execFile).not.toHaveBeenCalled(); + } + expect(readFileSync(legacy)).toEqual(before); + }, +); diff --git a/apps/ops-agent/src/configuration-release.ts b/apps/ops-agent/src/configuration-release.ts index 329e3192c..7e6b90b9f 100644 --- a/apps/ops-agent/src/configuration-release.ts +++ b/apps/ops-agent/src/configuration-release.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { AnchoredDirectory } from "@openmapx/core/build-artifacts"; import { services } from "@openmapx/core/server"; import { z } from "zod"; +import { readAgentHandoffFromInfra } from "./agent-handoff"; const selectionSchema = z.strictObject({ releaseId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/), @@ -29,6 +30,21 @@ export function readCheckedConfigurationRelease( try { const directory = infra.directory(".ops-agent-releases", false, true); try { + // The legacy entrypoint has no owned handoff source for this new root. + // Preserve its journal; never reinterpret startup cleanup as authority. + try { + directory.read("transaction.json", 128 * 1024); + throw Error("Legacy release transaction requires recovery"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") + throw Error("Legacy release transaction requires recovery"); + } + const legacy = readAgentHandoffFromInfra(infraDir); + if ( + legacy && + !(legacy.consumed === true && ["applied", "rolled_back"].includes(legacy.phase)) + ) + throw Error("Legacy release handoff requires recovery"); const raw = directory.read(`${selected.releaseId}.json`, 32 * 1024).toString("utf8"), manifest = services.parseReleaseManifest(raw, channel.imagePrefix), canonical = services.canonicalReleaseManifest(manifest), From 5512cb51144f0b16c26aea0e8c8b8e4af3222975 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 12:13:34 +0100 Subject: [PATCH 082/135] feat(admin): apply checked maintenance releases through guarded configuration jobs --- .../src/routes/__tests__/admin-system.test.ts | 27 +++ apps/api/src/routes/admin-system.ts | 4 + .../platform-configuration-job.test.ts | 41 ++++ ...orm-configuration-runtime-postgres.test.ts | 162 ++++++++++---- .../__tests__/system-maintenance.test.ts | 204 +++--------------- apps/api/src/services/job-runner.ts | 14 +- .../services/platform-configuration-job.ts | 27 +++ .../platform-configuration-participant.ts | 7 +- .../src/services/platform-configuration.ts | 4 +- .../src/services/prospective-configuration.ts | 19 +- apps/api/src/services/system-maintenance.ts | 156 +------------- .../services/system-release-application.ts | 128 +++++++++++ 12 files changed, 426 insertions(+), 367 deletions(-) create mode 100644 apps/api/src/services/__tests__/platform-configuration-job.test.ts create mode 100644 apps/api/src/services/platform-configuration-job.ts create mode 100644 apps/api/src/services/system-release-application.ts diff --git a/apps/api/src/routes/__tests__/admin-system.test.ts b/apps/api/src/routes/__tests__/admin-system.test.ts index 0d9cbec03..69296d34d 100644 --- a/apps/api/src/routes/__tests__/admin-system.test.ts +++ b/apps/api/src/routes/__tests__/admin-system.test.ts @@ -174,6 +174,7 @@ describe("admin system routes", () => { "system.update", { operation: "apply", + workflow: "guarded-release", createBackup: true, releaseId: "release-123", expectedDigest: "a".repeat(64), @@ -183,6 +184,31 @@ describe("admin system routes", () => { expect(audit).toHaveBeenCalledWith(expect.objectContaining({ action: "system.update.apply" })); }); + it.each(["loopback", "impersonated"])( + "refuses %s release application before enqueue", + async (kind) => { + const original = session.user.id; + if (kind === "loopback") session.user.id = "loopback"; + else session.session.impersonatedBy = "another-admin"; + try { + const response = await app.inject({ + method: "POST", + url: "/admin/system/updates/apply", + payload: { + confirmation: "UPDATE OPENMAPX", + releaseId: "release-123", + expectedDigest: "a".repeat(64), + }, + }); + expect(response.statusCode).toBe(403); + expect(enqueue).not.toHaveBeenCalled(); + } finally { + session.user.id = original; + session.session.impersonatedBy = null; + } + }, + ); + it("allows an operator to skip the optional backup", async () => { const response = await app.inject({ method: "POST", @@ -199,6 +225,7 @@ describe("admin system routes", () => { "system.update", { operation: "apply", + workflow: "guarded-release", createBackup: false, releaseId: "release-123", expectedDigest: "a".repeat(64), diff --git a/apps/api/src/routes/admin-system.ts b/apps/api/src/routes/admin-system.ts index 56d2be91d..402d7a74e 100644 --- a/apps/api/src/routes/admin-system.ts +++ b/apps/api/src/routes/admin-system.ts @@ -40,6 +40,7 @@ function isExactSystemUpdateBody(body: unknown): boolean { const HOST_MUTATING_JOB_TYPES = [ "system.update", + "platform.apply", "system.diagnostics", "service.start", "service.stop", @@ -277,11 +278,14 @@ export async function adminSystemRoute(app: FastifyInstance): Promise { return { error: "A maintenance operation is already active", jobId: active.id }; } const admin = getAdminSession(request); + if (admin.user.id === "loopback" || admin.session.impersonatedBy) + return reply.code(403).send({ error: "A real full administrator session is required" }); const createBackup = request.body?.createBackup !== false; const jobId = await jobRunner.enqueue( "system.update", { operation: "apply", + workflow: "guarded-release", createBackup, releaseId: request.body.releaseId, expectedDigest: request.body.expectedDigest, diff --git a/apps/api/src/services/__tests__/platform-configuration-job.test.ts b/apps/api/src/services/__tests__/platform-configuration-job.test.ts new file mode 100644 index 000000000..ae4c29066 --- /dev/null +++ b/apps/api/src/services/__tests__/platform-configuration-job.test.ts @@ -0,0 +1,41 @@ +import { expect, it } from "vitest"; +import { platformJobMatchesPreview } from "../platform-configuration-job"; + +const release = { releaseId: "release-checked", manifestDigest: "a".repeat(64) }; +const job = { + type: "system.update", + payload: { + operation: "apply", + workflow: "guarded-release", + releaseId: release.releaseId, + expectedDigest: release.manifestDigest, + }, +}; +it("binds the queued update to its exact reviewed release", () => { + expect(platformJobMatchesPreview(job, { release })).toBe(true); + expect(platformJobMatchesPreview(job, {})).toBe(false); + expect( + platformJobMatchesPreview(job, { release: { ...release, manifestDigest: "b".repeat(64) } }), + ).toBe(false); + expect(platformJobMatchesPreview(job, { release: { ...release, releaseId: "another" } })).toBe( + false, + ); +}); +it("refuses legacy, extra-field and unrelated job selectors", () => { + expect( + platformJobMatchesPreview( + { ...job, payload: { ...job.payload, workflow: undefined } }, + { release }, + ), + ).toBe(false); + expect( + platformJobMatchesPreview( + { ...job, payload: { ...job.payload, image: "untrusted" } }, + { release }, + ), + ).toBe(false); + expect(platformJobMatchesPreview({ ...job, type: "service.apply" }, { release })).toBe(false); +}); +it("preserves the existing platform job path", () => { + expect(platformJobMatchesPreview({ type: "platform.apply", payload: {} }, {})).toBe(true); +}); diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index fb5c05b11..dc72456bd 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -250,6 +250,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-resource", "launch-native-release", "launch-native-release-wrong-config", + "launch-native-system-release", + "launch-native-system-release-ack-loss", ])( "guards and retains helper mode %s through original ownership", async (version) => { @@ -257,7 +259,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( continueHelperEffect = Promise.withResolvers(); let writersWereReleased = false; const native = version.startsWith("launch-native") || version === "launch-bundle", - application = version === "launch-apply" || native; + application = version === "launch-apply" || native, + systemRelease = version.startsWith("launch-native-system-release"); const ids = [ "motis", "ops-agent", @@ -277,7 +280,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( configurationHelperVersion: version === "old" ? "old" : "1", configurationHelperLaunchResponseLoss: version === "launch-lost", configurationHelperReleaseInventory: application, - configurationReleaseMetadata: version.startsWith("launch-native-release"), + configurationReleaseMetadata: + version.startsWith("launch-native-release") || systemRelease, ...(application ? { schemas: { @@ -301,6 +305,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-resource", "launch-native-release", "launch-native-release-wrong-config", + "launch-native-system-release", + "launch-native-system-release-ack-loss", ].includes(version) ) { state.registry = new services.ServiceRegistry({ @@ -318,7 +324,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( })); host.setRows(rows); let checkedRelease: { releaseId: string; manifestDigest: string } | undefined; - if (version.startsWith("launch-native-release")) { + if (version.startsWith("launch-native-release") || systemRelease) { vi.stubEnv("OPENMAPX_RELEASE", "main"); vi.stubEnv("OPENMAPX_RELEASE_MANIFEST_REPOSITORY", "ghcr.io/openmapx/release-manifest"); const manifest = services.parseReleaseManifest( @@ -481,6 +487,29 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( journal, controlInstallation: installation, }); + let acknowledgementLost = false; + app.addHook("onSend", async (request, reply, payload) => { + const operation = ( + request.body as { operation?: { kind?: string; operationKey?: string } } | undefined + )?.operation; + if ( + version === "launch-native-system-release-ack-loss" && + !acknowledgementLost && + reply.statusCode === 200 && + operation?.kind === "controlRetention.acknowledge" && + journal + .records() + .some( + (row) => + row.operationKey === operation.operationKey && + row.operation.kind === "configurationHandoff.prepare", + ) + ) { + acknowledgementLost = true; + reply.raw.destroy(); + } + return payload; + }); const url = await app.listen({ host: "127.0.0.1", port: 0 }); const tokenFile = join(host.root, "platform-api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); @@ -521,6 +550,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-resource", "launch-native-release", "launch-native-release-wrong-config", + "launch-native-system-release", + "launch-native-system-release-ack-loss", ].includes(version) ) { const { adminPlatformRoute } = await import("../../routes/admin-platform"), @@ -535,45 +566,70 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( baseUrl: await publicApi.listen({ host: "127.0.0.1", port: 0 }), }), headers = { "idempotency-key": operationKey }, - reviewed = unwrapAdminResponse( - await previewAdminPlatformConfiguration( - { - expectedCurrentRevisionId: host.initial.revisionId, - changes: [ - "launch-native-resource", - "launch-native-release", - "launch-native-release-wrong-config", - ].includes(version) - ? { version: 1 as const } - : { - version: 1 as const, - backupRetentionDays: { source: "database", value: 14 }, - }, - ...(version === "launch-native-resource" - ? { reconcileServiceIds: ["ops-agent" as const] } - : {}), - ...(checkedRelease ? { release: checkedRelease } : {}), - }, - headers, - { client }, - ), - ), - input = { - expectedCurrentRevisionId: reviewed.preview.expectedCurrentRevisionId, - previewFingerprint: reviewed.preview.previewFingerprint, - confirmations: reviewed.preview.requiredConfirmation, - }; + reviewed = systemRelease + ? undefined + : unwrapAdminResponse( + await previewAdminPlatformConfiguration( + { + expectedCurrentRevisionId: host.initial.revisionId, + changes: [ + "launch-native-resource", + "launch-native-release", + "launch-native-release-wrong-config", + "launch-native-system-release", + "launch-native-system-release-ack-loss", + ].includes(version) + ? { version: 1 as const } + : { + version: 1 as const, + backupRetentionDays: { source: "database", value: 14 }, + }, + ...(version === "launch-native-resource" + ? { reconcileServiceIds: ["ops-agent" as const] } + : {}), + ...(checkedRelease ? { release: checkedRelease } : {}), + }, + headers, + { client }, + ), + ), + input = reviewed + ? { + expectedCurrentRevisionId: reviewed.preview.expectedCurrentRevisionId, + previewFingerprint: reviewed.preview.previewFingerprint, + confirmations: reviewed.preview.requiredConfirmation, + } + : undefined; + if (systemRelease) { + const { handleSystemUpdateJob } = await import("../system-maintenance"); + jobRunner.register("system.update", handleSystemUpdateJob); + } const apply = async () => - unwrapAdminResponse( - await applyAdminPlatformConfiguration(input, headers, { client }), - ); + systemRelease + ? { + ok: true as const, + jobId: await jobRunner.enqueue( + "system.update", + { + operation: "apply", + workflow: "guarded-release", + createBackup: false, + releaseId: checkedRelease!.releaseId, + expectedDigest: checkedRelease!.manifestDigest, + }, + actorId, + ), + } + : unwrapAdminResponse( + await applyAdminPlatformConfiguration(input!, headers, { client }), + ); const accepted = await apply(); if (version === "launch-native-resource") { - expect(reviewed.preview.configuration).toEqual({ version: 1 }); - expect(reviewed.preview.affectedServiceIds).toEqual(["ops-agent"]); - expect(reviewed.preview.oldOrigin).toBe(reviewed.preview.newOrigin); + expect(reviewed!.preview.configuration).toEqual({ version: 1 }); + expect(reviewed!.preview.affectedServiceIds).toEqual(["ops-agent"]); + expect(reviewed!.preview.oldOrigin).toBe(reviewed!.preview.newOrigin); } - expect(await apply()).toEqual(accepted); + if (!systemRelease) expect(await apply()).toEqual(accepted); const readJob = async () => ( await state.fixture.db @@ -641,6 +697,16 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( job.status, JSON.stringify({ error: job.error?.split("\n")[0].slice(0, 200), + observation: (job.result as Record | null)?.observation, + settled: !!(job.result as Record | null)?._settled, + candidate: (() => { + const value = ( + job.result as { + _configurationCandidate?: { phase?: string; retirement?: unknown }; + } | null + )?._configurationCandidate; + return { phase: value?.phase, retired: !!value?.retirement }; + })(), configuration: (job.result as Record | null)?.configuration, states: journal.records().map((value) => ({ kind: value.operation.kind, @@ -649,13 +715,23 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( })), }), ).toBe("success"); - expect(job.payload).toEqual({ - operationKey: createDirectAdminOpsKey(actorId, operationKey), - }); + expect(job.payload).toEqual( + systemRelease + ? { + operation: "apply", + workflow: "guarded-release", + createBackup: false, + releaseId: checkedRelease!.releaseId, + expectedDigest: checkedRelease!.manifestDigest, + } + : { + operationKey: createDirectAdminOpsKey(actorId, operationKey), + }, + ); expect(job.result).toMatchObject({ configuration: { outcome: "committed", followUp: "complete" }, }); - expect(await apply()).toEqual(accepted); + if (!systemRelease) expect(await apply()).toEqual(accepted); if (checkedRelease) { const pulls = host.calls.filter((args) => args[0] === "image" && args[1] === "pull"); expect(pulls).toHaveLength(services.RELEASE_IMAGE_NAMES.length); @@ -682,6 +758,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( digest: checkedRelease.manifestDigest, }); } + if (version === "launch-native-system-release-ack-loss") + expect(acknowledgementLost).toBe(true); expect(dm.settlePosts).toBe(1); expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); expect( diff --git a/apps/api/src/services/__tests__/system-maintenance.test.ts b/apps/api/src/services/__tests__/system-maintenance.test.ts index ec8983afa..9ae5a1fd5 100644 --- a/apps/api/src/services/__tests__/system-maintenance.test.ts +++ b/apps/api/src/services/__tests__/system-maintenance.test.ts @@ -1,24 +1,17 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { JobContext } from "../job-runner"; -import { type ApiOpsClient, ApiOpsError } from "../ops-client"; -let agentReplaced: boolean | Error = false; const executeAdminOperation = vi.fn(async (_ctx: JobContext, operation: { kind: string }) => { if (operation.kind === "system.diagnostics") return { ok: true, checks: [] }; - if (operation.kind === "release.inspect") { + if (operation.kind === "release.inspect") return { currentReleaseId: "release-old", availableReleaseId: "release-123" }; - } - if (operation.kind === "agent.update") { - if (agentReplaced instanceof Error) throw agentReplaced; - return { handoffId: "handoff", replaced: agentReplaced }; - } return { releaseId: "release-123" }; }); - +const guardedRelease = vi.fn(async (_ctx: JobContext) => ({ phase: "complete" })); vi.mock("../admin-job-ops", () => ({ executeAdminJobOperation: executeAdminOperation })); - +vi.mock("../system-release-application", () => ({ runGuardedSystemRelease: guardedRelease })); const { handleSystemDiagnosticsJob, handleSystemUpdateJob } = await import("../system-maintenance"); - +const { JobObservationPending } = await import("../admin-job-failure"); function context(payload: Record): JobContext { return { jobId: "job-1", @@ -29,187 +22,56 @@ function context(payload: Record): JobContext { checkpoint: vi.fn().mockResolvedValue(undefined), }; } - -beforeEach(() => { - vi.clearAllMocks(); - agentReplaced = false; -}); - -/** A fake agent answering handoff inspections in order; Errors model a restarting agent. */ -function handoffClient(answers: Array) { - let index = 0; - const execute = vi.fn(async () => { - const answer = answers[Math.min(index, answers.length - 1)]; - index += 1; - if (answer instanceof Error) throw answer; - return { - execution: "sync" as const, - value: { - handoffId: "handoff", - phase: answer, - ...(answer === "rolled_back" ? { reason: "The new agent stopped" } : {}), - }, - }; - }); - let clock = 0; - return { - execute, - dependencies: { - client: { execute } as unknown as ApiOpsClient, - sleep: async (ms: number) => { - clock += ms; - }, - now: () => clock, - }, - }; -} - +beforeEach(() => vi.clearAllMocks()); const applyPayload = { operation: "apply", createBackup: true, releaseId: "release-checked", expectedDigest: "a".repeat(64), }; - -describe("agent-owned system maintenance", () => { - it("checks a release from its manifest without downloading images", async () => { - const ctx = context({ operation: "check" }); - await expect(handleSystemUpdateJob(ctx)).resolves.toEqual({ +describe("guarded system maintenance routing", () => { + it("checks only the release manifest and inspection", async () => { + await expect(handleSystemUpdateJob(context({ operation: "check" }))).resolves.toEqual({ operation: "check", release: "release-123", currentReleaseId: "release-old", availableReleaseId: "release-123", }); - expect(executeAdminOperation.mock.calls.map((call) => call.slice(1))).toEqual([ - [{ kind: "release.resolve" }, "admin-job.release.resolve"], - [{ kind: "release.inspect" }, "admin-job.release.inspect"], - ]); - }); - - it("downloads the checked release before delegating backup and application", async () => { - const ctx = context({ - operation: "apply", - createBackup: true, - releaseId: "release-checked", - expectedDigest: "a".repeat(64), - }); - await expect(handleSystemUpdateJob(ctx)).resolves.toEqual({ - operation: "apply", - release: "release-123", - phase: "complete", - }); expect(executeAdminOperation.mock.calls.map((call) => call[1].kind)).toEqual([ - "release.pull", - "agent.update", - "system.update", + "release.resolve", + "release.inspect", ]); - expect(executeAdminOperation).toHaveBeenNthCalledWith( - 1, - ctx, - { kind: "release.pull", releaseId: "release-checked" }, - "admin-job.release.pull", - ); - expect(executeAdminOperation).toHaveBeenLastCalledWith( - ctx, - { - kind: "system.update", - releaseId: "release-checked", - expectedDigest: "a".repeat(64), - createBackup: true, - backupId: "pre-update-job-1", - }, - "admin-job.system.update", - ); - expect(ctx.checkpoint).toHaveBeenCalledWith( - expect.objectContaining({ - _completion: expect.objectContaining({ operationKey: expect.stringMatching(/^opk1_/) }), - }), - ); - expect( - executeAdminOperation.mock.calls.some((call) => call[1].kind === "release.resolve"), - ).toBe(false); + expect(guardedRelease).not.toHaveBeenCalled(); }); - - it("replaces an outdated agent and waits for it before updating the application", async () => { - agentReplaced = true; - const agent = handoffClient([new Error("connection refused"), "recreated", "applied"]); - const ctx = context(applyPayload); - await expect(handleSystemUpdateJob(ctx, agent.dependencies)).resolves.toMatchObject({ - phase: "complete", - }); - expect(agent.execute).toHaveBeenCalledTimes(3); - expect(executeAdminOperation.mock.calls.map((call) => call[1].kind)).toEqual([ - "release.pull", - "agent.update", - "system.update", - ]); - const update = executeAdminOperation.mock.calls[1]?.[1] as { handoffId?: string }; - expect(agent.execute).toHaveBeenCalledWith( - { kind: "agent.handoff.inspect", handoffId: update.handoffId }, - expect.anything(), - ); + it("delegates a checked update to the fixed configuration producer", async () => { + const ctx = context({ ...applyPayload, workflow: "guarded-release" }); + await expect(handleSystemUpdateJob(ctx)).resolves.toEqual({ phase: "complete" }); + expect(guardedRelease).toHaveBeenCalledExactlyOnceWith(ctx); + expect(executeAdminOperation).not.toHaveBeenCalled(); }); - - it("stops before the application when the agent replacement was rolled back", async () => { - agentReplaced = true; - const agent = handoffClient(["overlay_written", "rolled_back"]); - const ctx = context(applyPayload); - await expect(handleSystemUpdateJob(ctx, agent.dependencies)).rejects.toThrow( - "previous agent was restored: The new agent stopped", - ); - expect(ctx.checkpoint).toHaveBeenCalledWith( - expect.objectContaining({ errorCode: "host_update_required" }), - ); - expect(executeAdminOperation.mock.calls.some((call) => call[1].kind === "system.update")).toBe( - false, + it("keeps historical unmarked updates observation-only", async () => { + await expect(handleSystemUpdateJob(context(applyPayload))).rejects.toBeInstanceOf( + JobObservationPending, ); + expect(guardedRelease).not.toHaveBeenCalled(); + expect(executeAdminOperation).not.toHaveBeenCalled(); }); - - it("follows the handoff when the old agent stopped before returning its result", async () => { - agentReplaced = new ApiOpsError("runtime"); - const agent = handoffClient([new Error("connection refused"), "applied"]); + it.each([ + { ...applyPayload, expectedDigest: "bad" }, + { ...applyPayload, releaseId: undefined }, + ])("refuses an unchecked release", async (payload) => { await expect( - handleSystemUpdateJob(context(applyPayload), agent.dependencies), - ).resolves.toMatchObject({ phase: "complete" }); - }); - - it("fails a lost update result when the agent never recorded the handoff", async () => { - agentReplaced = new ApiOpsError("runtime"); - const agent = handoffClient(["none"]); - await expect(handleSystemUpdateJob(context(applyPayload), agent.dependencies)).rejects.toThrow( - "Operations request failed", - ); - expect(executeAdminOperation.mock.calls.some((call) => call[1].kind === "system.update")).toBe( - false, - ); - }); - - it("explains that the agent is busy with other operations", async () => { - agentReplaced = new ApiOpsError("busy"); - const agent = handoffClient(["applied"]); - await expect(handleSystemUpdateJob(context(applyPayload), agent.dependencies)).rejects.toThrow( - "Other operations are running on the operations agent", - ); - expect(agent.execute).not.toHaveBeenCalled(); + handleSystemUpdateJob(context({ ...payload, workflow: "guarded-release" })), + ).rejects.toThrow("Check a release"); + expect(guardedRelease).not.toHaveBeenCalled(); }); - - it("does not guess about agent update errors that prove it did not run", async () => { - agentReplaced = new ApiOpsError("validation"); - const agent = handoffClient(["applied"]); + it("does not fall back to legacy updates after a guarded failure", async () => { + guardedRelease.mockRejectedValueOnce(new JobObservationPending("recovery_required")); await expect( - handleSystemUpdateJob(context(applyPayload), agent.dependencies), - ).rejects.toThrow(); - expect(agent.execute).not.toHaveBeenCalled(); + handleSystemUpdateJob(context({ ...applyPayload, workflow: "guarded-release" })), + ).rejects.toBeInstanceOf(JobObservationPending); + expect(executeAdminOperation).not.toHaveBeenCalled(); }); - - it("gives up when the agent replacement never finishes", async () => { - agentReplaced = true; - const agent = handoffClient(["recreated"]); - await expect(handleSystemUpdateJob(context(applyPayload), agent.dependencies)).rejects.toThrow( - "did not finish in time", - ); - }); - it("uses the kind-only diagnostic operation", async () => { const ctx = context({ argv: ["--verbose"] }); await expect(handleSystemDiagnosticsJob(ctx)).resolves.toEqual({ diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 88402aa24..b183ea73f 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -41,7 +41,8 @@ import { import { coordinatorLifecycleOperationKey } from "./coordinator-lifecycle"; import { IntegrationBindingConflict as RetainedBindingConflict } from "./integration-binding-participant"; import { type JobStatusEvent, jobEventBus, TERMINAL_JOB_STATUSES } from "./job-events"; -import { ApiOpsError, OpsObservationPending } from "./ops-client"; +import { ApiOpsError, createDurableOpsKey, OpsObservationPending } from "./ops-client"; +import { guardedSystemReleasePayloadSchema } from "./platform-configuration-job"; import { appApiRestartCheckpoint } from "./system-update-state"; import { completeTrustedConfigurationApply, @@ -172,6 +173,17 @@ function hasRetainedQueuedCoordinator(job: { const batch = (job.result as Record | null)?._serviceUpdateBatch; if (job.type === "service.bulk" && payload.action === "update" && batch) return true; // Routing only; the fixed handler validates the actual owned batch/ACK chain. if (!candidate) return false; + if (job.type === "system.update") { + const fixed = (job.result as Record | null)?._systemRelease; + return ( + guardedSystemReleasePayloadSchema.safeParse(payload).success && + !!fixed && + typeof fixed === "object" && + (fixed as { operationKey?: unknown }).operationKey === + createDurableOpsKey("admin-job.guarded-release", job.id) && + candidate.operationKey === createDurableOpsKey("admin-job.guarded-release", job.id) + ); + } const ordinal = job.type === "service.apply" && payload.service === "data-manager" && diff --git a/apps/api/src/services/platform-configuration-job.ts b/apps/api/src/services/platform-configuration-job.ts new file mode 100644 index 000000000..dbebe9a84 --- /dev/null +++ b/apps/api/src/services/platform-configuration-job.ts @@ -0,0 +1,27 @@ +/** Job selectors are data, never host or configuration-participant authority. */ +import type { AdminPlatformPreview } from "@openmapx/core/api-contracts"; +import { configurationReleaseSelectionSchema } from "@openmapx/core/ops"; +import { z } from "zod"; + +export const guardedSystemReleasePayloadSchema = z.strictObject({ + operation: z.literal("apply"), + workflow: z.literal("guarded-release"), + createBackup: z.boolean().optional(), + releaseId: configurationReleaseSelectionSchema.shape.releaseId, + expectedDigest: configurationReleaseSelectionSchema.shape.manifestDigest, +}); +/** Additional rejection predicate. The actual actor, owned row, original MAC + * candidate, preview and frozen controls must still pass their own checks. */ +export function platformJobMatchesPreview( + job: { type: string; payload: unknown }, + preview: Pick, +) { + if (job.type === "platform.apply") return true; + if (job.type !== "system.update" || !preview.release) return false; + const payload = guardedSystemReleasePayloadSchema.safeParse(job.payload); + return ( + payload.success && + payload.data.releaseId === preview.release.releaseId && + payload.data.expectedDigest === preview.release.manifestDigest + ); +} diff --git a/apps/api/src/services/platform-configuration-participant.ts b/apps/api/src/services/platform-configuration-participant.ts index 390d3f083..4fa136229 100644 --- a/apps/api/src/services/platform-configuration-participant.ts +++ b/apps/api/src/services/platform-configuration-participant.ts @@ -20,6 +20,7 @@ import { PLATFORM_CONFIGURATION_SETTING, type PlatformConfigurationCommitInput, } from "./platform-configuration"; +import { platformJobMatchesPreview } from "./platform-configuration-job"; import { PlatformPreviewConflict, readPlatformPreview } from "./platform-preview-store"; import type { ProspectiveConfigurationOverlay, @@ -57,7 +58,10 @@ const intents = new WeakMap(), participants = new WeakMap(); function active(writer: AdministrativeConfigurationWriter) { const owner = administrativeConfigurationState(writer); - if (owner.job.jobType !== "platform.apply" || !owner.job.execution) + if ( + !["platform.apply", "system.update"].includes(owner.job.jobType ?? "") || + !owner.job.execution + ) throw new PlatformPreviewConflict(); return owner; } @@ -157,6 +161,7 @@ export async function preparePlatformConfigurationChange( ), preview = immutable(retained.preview); if ( + !platformJobMatchesPreview({ type: owner.job.jobType!, payload: owner.job.payload }, preview) || retained.consumedByJobId || !retained.sourceSnapshot || retained.desiredRowVersion === undefined || diff --git a/apps/api/src/services/platform-configuration.ts b/apps/api/src/services/platform-configuration.ts index 748f2c9bb..480db138f 100644 --- a/apps/api/src/services/platform-configuration.ts +++ b/apps/api/src/services/platform-configuration.ts @@ -19,6 +19,7 @@ import { user, } from "../db/schema"; import { jobOwnership } from "./admin-job-state"; +import { platformJobMatchesPreview } from "./platform-configuration-job"; import { PlatformPreviewConflict, readPlatformPreview } from "./platform-preview-store"; /** Called in the SAME desired-row transaction as purpose-proof consumption @@ -103,7 +104,7 @@ export async function commitPlatformConfigurationIntent( const [actor] = await tx.select().from(user).where(eq(user.id, input.actorId)).for("share"); if ( !job || - job.type !== "platform.apply" || + !["platform.apply", "system.update"].includes(job.type) || job.createdBy !== input.actorId || actor?.role !== "admin" || actor.banned === true @@ -124,6 +125,7 @@ export async function commitPlatformConfigurationIntent( const retained = await readPlatformPreview(input, tx), preview = retained.preview; if ( + !platformJobMatchesPreview(job, preview) || retained.consumedByJobId || (retained.desiredRowVersion !== undefined && retained.desiredRowVersion !== input.expectedDesiredRowVersion) || diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 9e27ab7d8..04bf9f4c5 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -123,6 +123,10 @@ import { capturedBindingState } from "./integration-binding-state"; import { integrationConfigRevision } from "./integration-runtime-evidence"; import type { JobContext } from "./job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +import { + guardedSystemReleasePayloadSchema, + platformJobMatchesPreview, +} from "./platform-configuration-job"; import { assertPlatformConfigurationOverlay, assertPlatformConfigurationSource, @@ -1354,9 +1358,21 @@ async function resumeCompletedConfigurationRetirement(): Promise Promise; - now?: () => number; -} - -export async function handleSystemUpdateJob( - ctx: JobContext, - dependencies: SystemUpdateJobDependencies = {}, -): Promise> { +export async function handleSystemUpdateJob(ctx: JobContext): Promise> { const payload = ctx.payload as { operation?: "check" | "apply"; createBackup?: boolean; @@ -59,130 +33,12 @@ export async function handleSystemUpdateJob( !/^[a-f0-9]{64}$/.test(payload.expectedDigest) ) throw new Error("Check a release before applying it"); - await ctx.setProgress(5); - await executeAdminJobOperation( - ctx, - { kind: "release.pull", releaseId: payload.releaseId }, - "admin-job.release.pull", - ); - await ctx.setProgress(45); - await replaceAgentIfNeeded(ctx, payload.releaseId, payload.expectedDigest, dependencies); - await ctx.setProgress(60); - const createBackup = payload.createBackup !== false; - await ctx.checkpoint({ - ...ctx.checkpointResult, - _completion: { - operationKey: createDurableOpsKey("admin-job.system.update", ctx.jobId), - result: { operation: "apply", release: payload.releaseId, phase: "complete" }, - }, - }); - const result = await executeAdminJobOperation( - ctx, - { - kind: "system.update", - releaseId: payload.releaseId, - expectedDigest: payload.expectedDigest, - createBackup, - ...(createBackup ? { backupId: `pre-update-${ctx.jobId}` } : {}), - }, - "admin-job.system.update", - ); - await ctx.setProgress(100); - return { operation: "apply", release: result.releaseId, phase: "complete" }; -} - -/** - * Updates the operations agent before the application services, as its own - * step: the agent starts a helper that replaces its container, verifies the - * new agent and restores the previous one if it does not start. - */ -async function replaceAgentIfNeeded( - ctx: JobContext, - releaseId: string, - expectedDigest: string, - dependencies: SystemUpdateJobDependencies, -): Promise { - // Derived from the job, so a resumed job and a lost result both find it. - const handoffId = createDurableOpsKey("admin-job.agent.handoff", ctx.jobId); - let resultLost = false; - try { - const result = await executeAdminJobOperation( - ctx, - { kind: "agent.update", releaseId, expectedDigest, handoffId }, - "admin-job.agent.update", - ); - if (!result.replaced) return; - } catch (error) { - if (error instanceof ApiOpsError && error.errorClass === "busy") { - throw new Error( - "Other operations are running on the operations agent. Update again when they finish.", - ); - } - if (!(error instanceof ApiOpsError) || !AMBIGUOUS_AGENT_UPDATE_ERRORS.has(error.errorClass)) { - throw error; - } - resultLost = true; - } - await ctx.log("Replacing the operations agent before the application services"); - - const client = dependencies.client ?? createApiOpsClient(); - const sleep = - dependencies.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); - const now = dependencies.now ?? Date.now; - const deadline = now() + AGENT_HANDOFF_TIMEOUT_MS; - let absentSince: number | undefined; - for (let attempt = 0; ; attempt += 1) { - const handoff = await inspectHandoff(client, ctx, handoffId, attempt); - if (handoff?.phase === "applied") { - await ctx.log("The new operations agent is running"); - return; - } - if (handoff?.phase === "rolled_back" || handoff?.phase === "failed") { - const checkpoint = { ...ctx.checkpointResult, errorCode: "host_update_required" }; - await ctx.checkpoint(checkpoint); - ctx.checkpointResult = checkpoint; - throw new Error( - `The operations agent could not be replaced${ - handoff.phase === "rolled_back" ? " and the previous agent was restored" : "" - }${handoff.reason ? `: ${handoff.reason}` : ""}`, - ); - } - if (handoff?.phase === "none") { - absentSince ??= now(); - if (resultLost && now() - absentSince >= AGENT_HANDOFF_ABSENT_MS) { - throw new ApiOpsError("runtime"); - } - } - if (now() >= deadline) { - throw new Error("The operations agent replacement did not finish in time"); - } - await sleep(AGENT_HANDOFF_POLL_MS); - } -} - -/** Undefined while the agent is unreachable, which is expected during its restart. */ -async function inspectHandoff( - client: ApiOpsClient, - ctx: JobContext, - handoffId: string, - attempt: number, -): Promise | undefined> { - try { - const result = await client.execute( - { kind: "agent.handoff.inspect", handoffId }, - { - operationKey: createDurableOpsKey( - "admin-job.agent.handoff.inspect", - `${ctx.jobId}:${attempt}`, - ), - signal: ctx.signal, - }, - ); - return result.execution === "sync" ? result.value : undefined; - } catch { - if (ctx.signal.aborted) throw new Error("System update canceled"); - return undefined; + if (ctx.payload.workflow !== "guarded-release") { + const { JobObservationPending } = await import("./admin-job-failure"); + throw new JobObservationPending("recovery_required"); } + const { runGuardedSystemRelease } = await import("./system-release-application"); + return runGuardedSystemRelease(ctx); } export async function handleSystemDiagnosticsJob( diff --git a/apps/api/src/services/system-release-application.ts b/apps/api/src/services/system-release-application.ts new file mode 100644 index 000000000..8ecd52d22 --- /dev/null +++ b/apps/api/src/services/system-release-application.ts @@ -0,0 +1,128 @@ +/** The ordinary backup finishes before the existing owned configuration root. + * Release application itself uses the same nominal platform participant/helper. */ +import { adminPlatformConfirmationSchema } from "@openmapx/core/api-contracts"; +import { configurationGenerationRevisionSchema } from "@openmapx/core/ops"; +import { eq } from "drizzle-orm"; +import { z } from "zod"; +import { db } from "../db"; +import { adminJob, user } from "../db/schema"; +import { getAllIntegrations } from "../integration-host"; +import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; +import { executeAdminJobOperation } from "./admin-job-ops"; +import { jobOwnership } from "./admin-job-state"; +import { + withAdministrativeConfigurationJob, + withAdministrativeConfigurationWriter, +} from "./administrative-configuration-writer-owner"; +import type { JobContext } from "./job-runner"; +import { createDurableOpsKey } from "./ops-client"; +import { applyPlatformConfigurationIntent } from "./platform-configuration-application"; +import { + guardedSystemReleasePayloadSchema, + platformJobMatchesPreview, +} from "./platform-configuration-job"; +import { + inspectPlatformConfiguration, + previewPlatformConfiguration, +} from "./platform-configuration-view"; +import { readPlatformPreview } from "./platform-preview-store"; +import { + resumeRetainedConfigurationCandidate, + retainedConfigurationIntegrationIds, +} from "./prospective-configuration"; +import { + completeTrustedConfigurationApply, + parseTrustedConfigurationApplyOutcome, +} from "./trusted-config-outcome"; + +const checkpointSchema = z.strictObject({ + version: z.literal(1), + actorId: z.string().min(1).max(128), + operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), + previewFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + expectedCurrentRevisionId: configurationGenerationRevisionSchema.nullable(), + confirmations: adminPlatformConfirmationSchema, +}); +export async function runGuardedSystemRelease(ctx: JobContext): Promise> { + const payload = guardedSystemReleasePayloadSchema.parse(ctx.payload); + if (ctx.jobType !== "system.update" || !ctx.execution) + throw new JobObservationPending("invalid_identity"); + const [job] = await db + .select() + .from(adminJob) + .where(jobOwnership(ctx.jobId, ctx.execution, ["running", "cancel_pending"])); + if (!job?.createdBy || job.type !== "system.update") + throw new JobObservationPending("invalid_identity"); + const actorId = job.createdBy, + operationKey = createDurableOpsKey("admin-job.guarded-release", ctx.jobId), + retained = ctx.checkpointResult?._configurationCandidate; + let fixed = + ctx.checkpointResult?._systemRelease === undefined + ? undefined + : checkpointSchema.parse(ctx.checkpointResult._systemRelease); + if (!retained) { + const [actor] = await db.select().from(user).where(eq(user.id, actorId)); + if (actor?.role !== "admin" || actor.banned === true) + throw new AdminJobFailure("Release administrator is unavailable", { errorCode: "forbidden" }); + if (ctx.observationOnly) throw new JobObservationPending("recovery_required"); + if (payload.createBackup !== false) { + await ctx.log("Creating a backup before applying the checked release"); + await executeAdminJobOperation( + ctx, + { kind: "backup.create", backupId: `pre-update-${ctx.jobId}` }, + "admin-job.guarded-release.backup", + ); + } + if (!fixed) { + const current = await inspectPlatformConfiguration(), + reviewed = await previewPlatformConfiguration({ + actorId, + operationKey: createDurableOpsKey("admin-job.release.preview", ctx.jobId), + expectedCurrentRevisionId: current.expectedCurrentRevisionId, + changes: { version: 1 }, + release: { releaseId: payload.releaseId, manifestDigest: payload.expectedDigest }, + }); + fixed = checkpointSchema.parse({ + version: 1, + actorId, + operationKey, + previewFingerprint: reviewed.preview.previewFingerprint, + expectedCurrentRevisionId: reviewed.preview.expectedCurrentRevisionId, + confirmations: reviewed.preview.requiredConfirmation, + }); + const result = { ...ctx.checkpointResult, _systemRelease: fixed }; + await ctx.checkpoint(result, 15); + ctx.checkpointResult = result; + } + } + if (!fixed || fixed.actorId !== actorId || fixed.operationKey !== operationKey) + throw new JobObservationPending("invalid_identity"); + if (!retained) { + const reviewed = await readPlatformPreview({ + actorId, + previewFingerprint: fixed.previewFingerprint, + }); + if (!platformJobMatchesPreview(job, reviewed.preview)) + throw new JobObservationPending("invalid_identity"); + } + const integrationIds = retained + ? await retainedConfigurationIntegrationIds(ctx, actorId, operationKey) + : getAllIntegrations().map((value) => value.id); + const input = fixed; + const result = await withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { actorId, operationKey, integrationIds, signal: ctx.signal }, + async () => { + if (retained) return resumeRetainedConfigurationCandidate(); + const applied = await applyPlatformConfigurationIntent(input); + return { configuration: applied.application }; + }, + ), + ); + const configuration = parseTrustedConfigurationApplyOutcome(result.configuration), + output = { operation: "apply", release: payload.releaseId, phase: "complete", configuration }; + if (!completeTrustedConfigurationApply(configuration)) + throw new AdminJobFailure("Checked release did not apply completely", output); + await ctx.setProgress(100); + return output; +} From 5e502e2d3731170ee48c343e8f38a867be5dc7b6 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 12:47:24 +0100 Subject: [PATCH 083/135] fix(platform): preserve original prepared handoffs across native agent replacement --- ...orm-configuration-runtime-postgres.test.ts | 183 +++++++++++++++++- .../src/configuration-handoff-store.ts | 13 ++ apps/ops-agent/src/server.ts | 21 +- .../staged-coordinator-request-process.mjs | 5 + 4 files changed, 217 insertions(+), 5 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index dc72456bd..6ba5dc790 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -223,6 +223,104 @@ async function nativeHelper( } } +/** Existing native production-agent driver; only external Docker I/O is bridged. */ +async function nativeReleaseAgent( + host: Awaited>, + ids: string[], + port: number, +) { + const bridge = Fastify(); + bridge.post("/docker", async (request, reply) => { + const { args } = request.body as { args: string[] }; + try { + return await host.execFile("docker", args, { + signal: AbortSignal.timeout(30000), + timeout: 30000, + maxBuffer: 1024 * 1024, + }); + } catch { + return reply.code(502).send({ error: "Named Docker IO failed" }); + } + }); + const dockerBridge = await bridge.listen({ host: "127.0.0.1", port: 0 }); + const child = spawn( + process.execPath, + [ + "--import", + createRequire(import.meta.url).resolve("tsx/esm"), + fileURLToPath( + new URL( + "../../../../ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs", + import.meta.url, + ), + ), + ], + { stdio: ["pipe", "pipe", "pipe"], env: process.env }, + ); + const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal })); + }, + ); + const ready = Promise.withResolvers(); + let pending = ""; + child.stdout.on("data", (data) => { + pending += String(data); + while (pending.includes("\n")) { + const end = pending.indexOf("\n"), + line = pending.slice(0, end); + pending = pending.slice(end + 1); + try { + if (JSON.parse(line).event === "ready") ready.resolve(); + } catch { + /* Only the fixed ready event is consumed. */ + } + } + }); + child.stderr.resume(); + void closed.then( + ({ code }) => ready.reject(Error(`Native agent exited before readiness (${code})`)), + (error) => ready.reject(error), + ); + child.stdin.end( + JSON.stringify({ + root: host.root, + infraDir: host.infraDir, + stack: host.stack, + serviceIds: ids, + queue: host.queue, + token: host.token, + dockerBridge: `${dockerBridge}/docker`, + listenPort: port, + captureReleaseAuthority: true, + }), + ); + const timer = setTimeout(() => { + child.kill("SIGKILL"); + ready.reject(Error("Native agent readiness timed out")); + }, 30000); + try { + await ready.promise; + } catch (error) { + await bridge.close(); + throw error; + } finally { + clearTimeout(timer); + } + const kill = async () => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + return await closed; + }; + return { + kill, + close: async () => { + await kill(); + await bridge.close(); + }, + }; +} + describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "owned platform preparation", () => { @@ -252,9 +350,13 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-release-wrong-config", "launch-native-system-release", "launch-native-system-release-ack-loss", + "launch-native-system-release-killed", + "launch-native-system-release-agent-restart", ])( "guards and retains helper mode %s through original ownership", async (version) => { + const nativeAgents: Array>> = []; + let nativeAgentExited = false; const helperEffectEntered = Promise.withResolvers(), continueHelperEffect = Promise.withResolvers(); let writersWereReleased = false; @@ -307,6 +409,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-release-wrong-config", "launch-native-system-release", "launch-native-system-release-ack-loss", + "launch-native-system-release-killed", + "launch-native-system-release-agent-restart", ].includes(version) ) { state.registry = new services.ServiceRegistry({ @@ -388,7 +492,10 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ).includes(id) ) throw Error("Unexpected protected target"); - if (version === "launch-native-killed") { + if ( + version === "launch-native-killed" || + version === "launch-native-system-release-killed" + ) { await services.withConfigurationGenerationWriter(host.infraDir, async () => { const lock = await services.acquireReleaseStoreLock( join(host.infraDir, ".ops-agent-releases"), @@ -402,6 +509,10 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await continueHelperEffect.promise; return; } + if (version === "launch-native-system-release-agent-restart" && id === "ops-agent") { + expect(await nativeAgents[0]!.kill()).toEqual({ code: null, signal: "SIGKILL" }); + nativeAgentExited = true; + } const prefix = args.slice(0, args.indexOf("up")), options = { signal: new AbortController().signal, @@ -511,6 +622,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( return payload; }); const url = await app.listen({ host: "127.0.0.1", port: 0 }); + if (version === "launch-native-system-release-agent-restart") await app.close(); const tokenFile = join(host.root, "platform-api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); for (const [key, value] of Object.entries({ @@ -526,6 +638,14 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( OPENMAPX_DISABLE_LOCALHOST_AUTH: "1", })) vi.stubEnv(key, value); + if (version === "launch-native-system-release-agent-restart") + nativeAgents.push( + await nativeReleaseAgent( + host, + host.loaded.map((service) => service.manifest.id), + Number(new URL(url).port), + ), + ); let publicApi: FastifyInstance | undefined; try { const actorId = randomUUID(), @@ -552,6 +672,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-release-wrong-config", "launch-native-system-release", "launch-native-system-release-ack-loss", + "launch-native-system-release-killed", + "launch-native-system-release-agent-restart", ].includes(version) ) { const { adminPlatformRoute } = await import("../../routes/admin-platform"), @@ -578,6 +700,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-release-wrong-config", "launch-native-system-release", "launch-native-system-release-ack-loss", + "launch-native-system-release-killed", + "launch-native-system-release-agent-restart", ].includes(version) ? { version: 1 as const } : { @@ -670,8 +794,62 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, version === "launch-native-release-wrong-config" ? { failureStage: "application" } - : {}, + : version === "launch-native-system-release-killed" + ? { + kill: { + when: helperEffectEntered.promise, + release: () => continueHelperEffect.resolve(), + }, + } + : {}, ); + if (version === "launch-native-system-release-agent-restart") { + expect(nativeAgentExited).toBe(true); + nativeAgents.push( + await nativeReleaseAgent( + host, + host.loaded.map((service) => service.manifest.id), + Number(new URL(url).port), + ), + ); + } + if (version === "launch-native-system-release-killed") { + const file = join( + host.infraDir, + "control-private/ops-agent/observations", + `configuration-handoff-${env("OPS_CONFIGURATION_HANDOFF_ID")}.completion.json`, + ), + original = readFileSync(file), + environment = { + OPENMAPX_ROOT_DIR: host.root, + OPS_CONFIGURATION_HANDOFF_ID: env("OPS_CONFIGURATION_HANDOFF_ID"), + OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), + OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), + }; + expect(writersWereReleased).toBe(true); + expect(JSON.parse(String(original)).payload).toMatchObject({ + phase: "applying", + completedAt: null, + }); + await nativeHelper(host, environment, { failureStage: "adoption" }); + expect(readFileSync(file)).toEqual(original); + await jobRunner.reconcile(); + expect((await readJob()).status).toBe("running"); + expect(dm.settlePosts).toBe(0); + await expect(dm.assertDataAdmission()).rejects.toThrow(); + expect( + host.calls.filter((args) => args[0] === "image" && args[1] === "pull"), + ).toHaveLength(services.RELEASE_IMAGE_NAMES.length); + expect( + host.calls + .filter((args) => args[0] === "compose" && args.includes("up")) + .map((args) => args.at(-1)), + ).toEqual(["data-manager"]); + expect(existsSync(join(host.infraDir, ".ops-agent-releases/current.json"))).toBe( + false, + ); + return; + } if (version === "launch-native-release-wrong-config") { await jobRunner.reconcile(); expect((await readJob()).status).toBe("running"); @@ -1497,6 +1675,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ); } finally { await publicApi?.close(); + for (const agent of nativeAgents) await agent.close(); await dm.close(); await app.close(); closeControlInstallation(installation); diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index 4d333a342..fa754608e 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -50,6 +50,7 @@ import { import { childSettlementState, installationState, + verifyControlPreparedTerminal, } from "../../../packages/core/src/build-artifacts/control-authority"; import { CONFIGURATION_HANDOFF_LABEL, @@ -320,6 +321,18 @@ export function assertPreparedConfigurationHandoff( throw Error("Original confirmed helper launch required"); } +/** Restart observes a terminal preparation, not an already settled target. + * The original signed capsule, indexed intent and confirmed launch remain + * mandatory. This returns no lease, dispatch ticket or replacement authority. */ +export function assertRetainedConfigurationHandoffPreparation( + authority: VerifiedExecutionAuthority, + result: unknown, +): void { + assertPreparedConfigurationHandoff(authority, result); + const local = executionAuthorityState(authority); + verifyControlPreparedTerminal(local.installation, local.execution); +} + function readProgress( authority: VerifiedExecutionAuthority, intent: ConfigurationHandoffIntentRecord, diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 12f0946b2..80775ba64 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -100,7 +100,10 @@ import { registerArtifactTransferRoutes } from "./artifact-transfer/routes"; import type { ArtifactTransferStore } from "./artifact-transfer/store"; import { readCommittedCatalogApproval } from "./catalog-approval-runtime"; import { configurationHandoffBlocksReplacement } from "./configuration-handoff"; -import { assertPreparedConfigurationHandoff } from "./configuration-handoff-store"; +import { + assertPreparedConfigurationHandoff, + assertRetainedConfigurationHandoffPreparation, +} from "./configuration-handoff-store"; import { prepareConfigurationHardlinksAdmission } from "./configuration-hardlinks-runtime"; import { configurationMaintenanceAdmissionState, @@ -437,7 +440,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const streaming = (request.routeOptions.config as { artifactTransferBytes?: boolean }).artifactTransferBytes === true; - request.raw.socket.setTimeout(streaming ? 5 * 60_000 : DEFAULT_ADMISSION_TIMEOUT_MS); + // Fastify's injected transport has a minimal socket. Real connections keep + // their idle timeout; admission also retains its independent destroy timer. + request.raw.socket.setTimeout?.(streaming ? 5 * 60_000 : DEFAULT_ADMISSION_TIMEOUT_MS); if (streaming) return; const timer = setTimeout(() => request.raw.destroy(), DEFAULT_ADMISSION_TIMEOUT_MS); timer.unref?.(); @@ -1790,7 +1795,17 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif try { if (!options.controlInstallation) throw Error("Control trust unavailable"); const execution = executionFor(job); - if (job.state === "succeeded") { + if (job.state === "succeeded" && job.operation.kind === "configurationHandoff.prepare") { + retainedControlAdoptionAssertion( + options.controlInstallation, + job.controlAdoption, + execution, + ); + assertRetainedConfigurationHandoffPreparation( + restoreAdmittedControlExecution(options.controlInstallation, execution), + job.result, + ); + } else if (job.state === "succeeded") { verifyRetainedArtifactTargetSettlement( options.controlInstallation, job.controlAdoption, diff --git a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs index c33319a86..407eeb5a7 100644 --- a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs +++ b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs @@ -104,6 +104,11 @@ const builtInServices = input.serviceIds.map((id) => ({ const loadAuthority = createTrustedConfigurationAuthorityLoader({ rootDir: input.root, builtInServices, + // The protected helper uses the production release-inventory authority. + // Read the actual bounded checkout inventory; never accept a supplied digest. + ...(input.captureReleaseAuthority + ? { builtInAuthorityDigest: (await services.captureReleaseServiceAuthority(input.root)).digest } + : {}), }); const repositories = new CommunityRepositoryStore({ rootDir: input.root }); await repositories.recover(); From c72493bd03ba4d1cc7ef5964947ed2349694dee8 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 13:00:09 +0100 Subject: [PATCH 084/135] feat(admin): enable checked maintenance updates with installed guarded helpers --- ...orm-configuration-runtime-postgres.test.ts | 47 +++++++------ apps/ops-agent/Dockerfile | 2 + apps/ops-agent/src/production-runtime.test.ts | 42 +++++++++++ apps/ops-agent/src/production-runtime.ts | 70 ++++++++++++++----- .../admin/system/SystemMaintenance.tsx | 6 +- 5 files changed, 125 insertions(+), 42 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 6ba5dc790..ac26d6529 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -677,6 +677,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ].includes(version) ) { const { adminPlatformRoute } = await import("../../routes/admin-platform"), + { adminSystemRoute } = await import("../../routes/admin-system"), { jobRunner } = await import("../job-runner"), { createDirectAdminOpsKey } = await import("../direct-ops-idempotency"); state.session = mockAdminSession(); @@ -684,9 +685,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( state.session.session.userId = actorId; publicApi = Fastify(); await publicApi.register(adminPlatformRoute, { prefix: "/api" }); - const client = createApiClient({ - baseUrl: await publicApi.listen({ host: "127.0.0.1", port: 0 }), - }), + await publicApi.register(adminSystemRoute, { prefix: "/api" }); + const publicUrl = await publicApi.listen({ host: "127.0.0.1", port: 0 }), + client = createApiClient({ baseUrl: publicUrl }), headers = { "idempotency-key": operationKey }, reviewed = systemRelease ? undefined @@ -728,25 +729,27 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const { handleSystemUpdateJob } = await import("../system-maintenance"); jobRunner.register("system.update", handleSystemUpdateJob); } - const apply = async () => - systemRelease - ? { - ok: true as const, - jobId: await jobRunner.enqueue( - "system.update", - { - operation: "apply", - workflow: "guarded-release", - createBackup: false, - releaseId: checkedRelease!.releaseId, - expectedDigest: checkedRelease!.manifestDigest, - }, - actorId, - ), - } - : unwrapAdminResponse( - await applyAdminPlatformConfiguration(input!, headers, { client }), - ); + const apply = async () => { + if (systemRelease) { + const response = await fetch(`${publicUrl}/api/admin/system/updates/apply`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + confirmation: "UPDATE OPENMAPX", + createBackup: false, + releaseId: checkedRelease!.releaseId, + expectedDigest: checkedRelease!.manifestDigest, + }), + }); + expect(response.status).toBe(200); + const body = await response.json(); + expect(body).toEqual({ ok: true, jobId: expect.any(String) }); + return body as { ok: true; jobId: string }; + } + return unwrapAdminResponse( + await applyAdminPlatformConfiguration(input!, headers, { client }), + ); + }; const accepted = await apply(); if (version === "launch-native-resource") { expect(reviewed!.preview.configuration).toEqual({ version: 1 }); diff --git a/apps/ops-agent/Dockerfile b/apps/ops-agent/Dockerfile index 04f74ec11..61240d68a 100644 --- a/apps/ops-agent/Dockerfile +++ b/apps/ops-agent/Dockerfile @@ -54,6 +54,7 @@ COPY apps/ops-agent/ apps/ops-agent/ COPY --parents \ services/data-manager/src/transitous-lock.ts \ services/data-manager/src/gbfs-catalog-lock.ts \ + services/data-manager/src/jobs/transitous-feeds-overlay.ts \ services/data-manager/src/utils/atomic-write.ts \ integrations/transit-dynamic-registry/ \ ./ @@ -71,6 +72,7 @@ COPY --from=bundle /app/apps/ops-agent/dist/runtime-packages.json /tmp/runtime-p RUN node scripts/copy-runtime-packages.mjs /tmp/runtime-packages.json /app /runtime FROM base AS runner +LABEL openmapx.configuration-handoff.version="1" WORKDIR /app # The pinned base predates Alpine's OpenSSL fix for CVE-2026-14456. # git fetches community service repositories into services/.community. diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index e1525868a..c8088bc8f 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -46,6 +46,48 @@ afterEach(() => { }); describe("assembled production service authority", () => { + it.each(["1", "old", undefined])( + "reports guarded maintenance readiness from the actual helper image (%s)", + async (version) => { + const ids = ["motis", "data-manager", "ops-agent"]; + const host = await fixture({ + withCoordinator: true, + configurationHelperVersion: version, + selectedRoots: ids, + }); + host.setRows( + ids.map((id, index) => ({ + ...host.row, + Id: String(index + 1).repeat(64), + ConfigImage: services.serviceContainerImageReference( + host.loaded.find((service) => service.manifest.id === id)!.manifest.container, + ), + Labels: { ...host.row.Labels, "com.docker.compose.service": id }, + })), + ); + const effects = productionReleaseEffects(host.runtime); + const io = { signal: new AbortController().signal, emitLog() {} }; + const before = services.currentConfigurationGeneration(host.infraDir); + await expect(effects.inspectSystem(io)).resolves.toMatchObject({ + maintenanceReady: version === "1", + }); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); + expect( + host.calls.some((args) => args[0] === "run" || args.includes("up") || args[0] === "pull"), + ).toBe(false); + if (version === "1") { + host.setServiceState("data-manager", "exited"); + await expect(effects.inspectSystem(io)).resolves.toMatchObject({ maintenanceReady: false }); + host.setServiceState("data-manager", "running"); + const path = join(host.infraDir, ".ops-agent-releases", "transaction.json"); + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + writeFileSync(path, "{}", { mode: 0o600 }); + await expect(effects.inspectSystem(io)).resolves.toMatchObject({ maintenanceReady: false }); + expect(readFileSync(path, "utf8")).toBe("{}"); + } + }, + ); + it("refuses unsupported legacy release mutation without staging and keeps inspection available", async () => { const host = await fixture(); const effects = productionReleaseEffects(host.runtime); diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index c32f018b7..ddc6f2b1f 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -9,7 +9,7 @@ import { createDefaultReleaseEffects, type FixedCliRunner, } from "./administrative-runtime"; -import { isTerminalHandoff, readAgentHandoff } from "./agent-handoff"; +import { readAgentHandoff } from "./agent-handoff"; import { installCatalogApprovalRuntime } from "./catalog-approval-runtime"; import { installCatalogValidationInputsRuntime } from "./catalog-validation-inputs"; import { @@ -17,6 +17,7 @@ import { installCommunityRepositoryRuntime, } from "./community-repositories"; import type { ConfigurationCandidateSource } from "./configuration-candidate-source"; +import { inspectConfigurationHandoffPrerequisite } from "./configuration-handoff"; import { installConfigurationHardlinksRuntime } from "./configuration-hardlinks-runtime"; import { registerConfigurationMaintenanceSource } from "./configuration-maintenance-runtime"; import { @@ -73,8 +74,24 @@ function installReleaseCapabilityBoundary( rootDir: string, infraDir: string, legacy: AdministrativeReleaseEffects, + owned: OwnedServiceResolver, + execFile: NonNullable, ): AdministrativeReleaseEffects { const directory = join(infraDir, ".ops-agent-releases"); + const assertLegacyRecoveryComplete = () => { + try { + lstatSync(join(directory, "transaction.json")); + throw new OpsExecutionFailure("recovery_required", {}); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const handoff = readAgentHandoff(rootDir); + if ( + handoff && + !(handoff.consumed === true && ["applied", "rolled_back"].includes(handoff.phase)) + ) + throw new OpsExecutionFailure("recovery_required", {}); + }; const withSource = async (signal: AbortSignal, action: () => Promise): Promise => { try { return await services.withConfigurationGenerationWriter( @@ -112,24 +129,41 @@ function installReleaseCapabilityBoundary( // Inspection bypasses the legacy cached initializer, which may replay // fixed CLI under the release lock or mutate handoff evidence. await legacy.inspect(); - let transaction = false; - try { - lstatSync(join(directory, "transaction.json")); - transaction = true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - const handoff = readAgentHandoff(rootDir); - if (transaction || (handoff && !isTerminalHandoff(handoff.phase))) - throw new OpsExecutionFailure("recovery_required", {}); + assertLegacyRecoveryComplete(); }, resolve: (context) => withSource(context.signal, () => legacy.resolve(context)), - inspectSystem: async (context) => ({ - ...(await legacy.inspectSystem(context)), - // Docker/Compose readiness alone cannot advertise the absent guarded - // release installer as runnable maintenance. - maintenanceReady: false, - }), + inspectSystem: async (context) => { + const inspection = await legacy.inspectSystem(context); + let maintenanceReady = false; + if (inspection.dockerReachable && inspection.composeReady) + try { + assertLegacyRecoveryComplete(); + const containers = await owned.inspectOwnedServices(context.signal), + agent = containers.find((row) => row.serviceId === "ops-agent"), + coordinator = containers.find((row) => row.serviceId === "data-manager"); + if (agent?.state === "running" && coordinator?.state === "running") { + await inspectConfigurationHandoffPrerequisite({ + rootDir, + containerId: agent.containerId, + imageId: agent.provenance.imageId, + signal: context.signal, + runDocker: async (args, io) => + ( + await execFile("docker", args, { + signal: io.signal, + timeout: 15_000, + maxBuffer: 1024 * 1024, + }) + ).stdout, + }); + maintenanceReady = true; + } + } catch { + // Availability is a read-only observation. The actual candidate + // repeats every prerequisite before acquiring or applying ownership. + } + return { ...inspection, maintenanceReady }; + }, pull: (_releaseId, context) => unavailable(context.signal), apply: (_releaseId, _serviceIds, context) => unavailable(context.signal), updateAgent: (_request, context) => unavailable(context.signal), @@ -234,6 +268,8 @@ export async function createProductionOpsRuntime( options.rootDir, options.stack.infraDir, createDefaultReleaseEffects(options.rootDir, runFixedCli), + owned, + options.docker?.execFile ?? runContainedProcess, ); createTransitousLockRuntime(runtime, { rootDir: options.rootDir }); installTransitCredentialsRuntime(runtime, { diff --git a/apps/web/src/components/admin/system/SystemMaintenance.tsx b/apps/web/src/components/admin/system/SystemMaintenance.tsx index 08b95513c..0c20726dc 100644 --- a/apps/web/src/components/admin/system/SystemMaintenance.tsx +++ b/apps/web/src/components/admin/system/SystemMaintenance.tsx @@ -472,9 +472,9 @@ export function SystemMaintenance() { {candidateToApply?.expectedDigest} - Images are downloaded before any container is replaced. The operations agent updates - first when the release changes it, then Data Manager and Web; API updates last and - briefly interrupts this page. + Images are downloaded before containers are replaced. Services update in dependency + order, with the operations agent last. API or proxy updates may briefly interrupt this + page; reconnect to follow the original job. Date: Fri, 30 Oct 2026 13:21:59 +0100 Subject: [PATCH 085/135] fix(admin): retain checked release recovery across API worker death --- .../fixtures/native-coordinator-api.ts | 54 +++-- ...orm-configuration-runtime-postgres.test.ts | 219 +++++++++++++++++- .../services/system-release-application.ts | 5 +- .../src/__fixtures__/owned-data-manager.ts | 5 + 4 files changed, 265 insertions(+), 18 deletions(-) diff --git a/apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts b/apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts index 22ea47d60..a5720946b 100644 --- a/apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts +++ b/apps/api/src/services/__tests__/fixtures/native-coordinator-api.ts @@ -9,11 +9,15 @@ import { handleServiceBulkJob } from "../../admin-job-handlers"; import { serviceApply } from "../../admin-ops"; import { jobRunner } from "../../job-runner"; import { initServiceRegistry } from "../../service-registry"; +import { handleSystemUpdateJob } from "../../system-maintenance"; let body = ""; for await (const chunk of process.stdin) body += chunk; const input = JSON.parse(body) as { - action: "apply" | "update" | "reconcile"; + action: "apply" | "update" | "release" | "reconcile"; + actorId?: string; + releaseId?: string; + manifestDigest?: string; jobId?: string; serviceIds?: string[]; noDeps?: boolean; @@ -93,6 +97,7 @@ try { ...(await serviceApply(ctx.payload.service as string, ctx)), })); jobRunner.register("service.bulk", handleServiceBulkJob); + jobRunner.register("system.update", handleSystemUpdateJob); let id: string; if (input.action === "reconcile") { if (typeof input.jobId !== "string" || !/^[a-f0-9-]{36}$/.test(input.jobId)) @@ -100,18 +105,41 @@ try { id = input.jobId; await jobRunner.reconcile({ limit: 1 }); } else { - if (input.action !== "apply" && input.action !== "update") - throw Error("Invalid fixture action"); - id = await jobRunner.enqueue( - input.action === "apply" ? "service.apply" : "service.bulk", - input.action === "apply" - ? { service: "data-manager" } - : { - action: "update", - serviceIds: input.serviceIds ?? ["data-manager"], - ...(input.noDeps ? { noDeps: true } : {}), - }, - ); + if (input.action === "release") { + if ( + typeof input.actorId !== "string" || + !/^[a-f0-9-]{36}$/.test(input.actorId) || + typeof input.releaseId !== "string" || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(input.releaseId) || + typeof input.manifestDigest !== "string" || + !/^[a-f0-9]{64}$/.test(input.manifestDigest) + ) + throw Error("Invalid fixture release"); + id = await jobRunner.enqueue( + "system.update", + { + operation: "apply", + workflow: "guarded-release", + createBackup: false, + releaseId: input.releaseId, + expectedDigest: input.manifestDigest, + }, + input.actorId, + ); + } else { + if (input.action !== "apply" && input.action !== "update") + throw Error("Invalid fixture action"); + id = await jobRunner.enqueue( + input.action === "apply" ? "service.apply" : "service.bulk", + input.action === "apply" + ? { service: "data-manager" } + : { + action: "update", + serviceIds: input.serviceIds ?? ["data-manager"], + ...(input.noDeps ? { noDeps: true } : {}), + }, + ); + } process.stdout.write(JSON.stringify({ event: "queued", jobId: id }) + "\n"); const until = Date.now() + 240000; for (;;) { diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index ac26d6529..df8f1ed60 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -34,7 +34,7 @@ import { } from "@openmapx/core/build-artifacts"; import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; import { repoPaths, services } from "@openmapx/core/server"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import Fastify, { type FastifyInstance } from "fastify"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; @@ -321,6 +321,86 @@ async function nativeReleaseAgent( }; } +/** The API's actual queued producer/reconciler runs in a killable process. */ +async function nativeReleaseWorker( + host: Awaited>, + databaseUrl: string, + input: + | { action: "release"; actorId: string; releaseId: string; manifestDigest: string } + | { action: "reconcile"; jobId: string }, +) { + const child = spawn( + process.execPath, + [ + "--import", + createRequire(import.meta.url).resolve("tsx/esm"), + fileURLToPath(new URL("./fixtures/native-coordinator-api.ts", import.meta.url)), + ], + { + cwd: host.root, + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + DATABASE_URL: databaseUrl, + OPENMAPX_ROOT_DIR: host.root, + NODE_ENV: "test", + BETTER_AUTH_SECRET: "synthetic-platform-worker-auth-fixture", + }, + }, + ); + const queued = Promise.withResolvers(); + let pending = "", + result: { status?: string; errorClass?: string } | undefined; + child.stdout.on("data", (data) => { + pending += String(data); + if (pending.length > 16384) pending = ""; + while (pending.includes("\n")) { + const end = pending.indexOf("\n"), + line = pending.slice(0, end); + pending = pending.slice(end + 1); + try { + const event = JSON.parse(line); + if (event.event === "queued" && typeof event.jobId === "string") + queued.resolve(event.jobId); + if (event.event === "result") + result = { status: event.status, errorClass: event.errorClass }; + } catch { + /* Only fixed, non-secret fixture events are observed. */ + } + } + }); + child.stderr.resume(); + const done = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + result: typeof result; + }>((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal, result })); + }); + void done.then( + (value) => queued.reject(Error(`Native API exited before queueing (${value.code})`)), + (error) => queued.reject(error), + ); + if (input.action === "reconcile") queued.resolve(input.jobId); + child.stdin.end(JSON.stringify(input)); + const timer = setTimeout(() => { + child.kill("SIGKILL"); + queued.reject(Error("Native API queueing timed out")); + }, 30000); + let jobId: string; + try { + jobId = await queued.promise; + } finally { + clearTimeout(timer); + } + const kill = async () => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + return await done; + }; + return { jobId, done, kill }; +} + describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "owned platform preparation", () => { @@ -352,17 +432,21 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-ack-loss", "launch-native-system-release-killed", "launch-native-system-release-agent-restart", + "launch-native-system-release-api-before-save", + "launch-native-system-release-api-after-save", ])( "guards and retains helper mode %s through original ownership", async (version) => { const nativeAgents: Array>> = []; + const nativeWorkers: Array>> = []; let nativeAgentExited = false; const helperEffectEntered = Promise.withResolvers(), continueHelperEffect = Promise.withResolvers(); let writersWereReleased = false; const native = version.startsWith("launch-native") || version === "launch-bundle", application = version === "launch-apply" || native, - systemRelease = version.startsWith("launch-native-system-release"); + systemRelease = version.startsWith("launch-native-system-release"), + apiRestart = version.startsWith("launch-native-system-release-api-"); const ids = [ "motis", "ops-agent", @@ -411,6 +495,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-ack-loss", "launch-native-system-release-killed", "launch-native-system-release-agent-restart", + "launch-native-system-release-api-before-save", + "launch-native-system-release-api-after-save", ].includes(version) ) { state.registry = new services.ServiceRegistry({ @@ -589,6 +675,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( sql: state.fixture.sql, infraDir: host.infraDir, serviceIds: ids, + pausePrepareResponse: version === "launch-native-system-release-api-before-save", }); const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); const app = buildOpsAgentServer({ @@ -674,6 +761,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-ack-loss", "launch-native-system-release-killed", "launch-native-system-release-agent-restart", + "launch-native-system-release-api-before-save", + "launch-native-system-release-api-after-save", ].includes(version) ) { const { adminPlatformRoute } = await import("../../routes/admin-platform"), @@ -703,6 +792,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-ack-loss", "launch-native-system-release-killed", "launch-native-system-release-agent-restart", + "launch-native-system-release-api-before-save", + "launch-native-system-release-api-after-save", ].includes(version) ? { version: 1 as const } : { @@ -731,6 +822,16 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( } const apply = async () => { if (systemRelease) { + if (apiRestart) { + const worker = await nativeReleaseWorker(host, state.fixture.connectionUri, { + action: "release", + actorId, + releaseId: checkedRelease!.releaseId, + manifestDigest: checkedRelease!.manifestDigest, + }); + nativeWorkers.push(worker); + return { ok: true as const, jobId: worker.jobId }; + } const response = await fetch(`${publicUrl}/api/admin/system/updates/apply`, { method: "POST", headers: { "content-type": "application/json" }, @@ -764,6 +865,74 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .from(schema.adminJob) .where(eq(schema.adminJob.id, accepted.jobId)) )[0]!; + const waitForOriginalLease = async () => { + const until = Date.now() + 65000; + for (;;) { + const row = await readJob(), + expiry = (row.result as { _observationLeaseUntil?: string } | null) + ?._observationLeaseUntil; + expect(typeof expiry).toBe("string"); + if (Date.parse(expiry!) <= Date.now()) break; + expect(Date.now()).toBeLessThan(until); + await new Promise((resolve) => setTimeout(resolve, 250)); + } + }; + const readDesiredPlatform = async () => { + const { PLATFORM_CONFIGURATION_SETTING } = await import("../platform-configuration"); + return state.fixture.db + .select({ key: schema.systemSettings.key, rowVersion: sql`xmin::text` }) + .from(schema.systemSettings) + .where(eq(schema.systemSettings.key, PLATFORM_CONFIGURATION_SETTING)); + }; + if (version === "launch-native-system-release-api-before-save") { + await Promise.race([ + dm.prepareReplyEntered, + nativeWorkers[0]!.done.then(() => { + throw Error("API exited before prepared response"); + }), + ]); + expect(await nativeWorkers[0]!.kill()).toMatchObject({ + code: null, + signal: "SIGKILL", + }); + dm.releasePrepareResponse(); + await waitForOriginalLease(); + const replacement = await nativeReleaseWorker(host, state.fixture.connectionUri, { + action: "reconcile", + jobId: accepted.jobId, + }); + nativeWorkers.push(replacement); + expect(await replacement.done).toMatchObject({ + code: 2, + signal: null, + result: { status: "failed" }, + }); + const recovered = await readJob(), + result = recovered.result as Record; + expect((result._execution as { version: number }).version).toBe(2); + expect(result.configuration).toMatchObject({ + outcome: "not_committed", + cleanupFailed: false, + }); + expect(result.phase).toBe("not_applied"); + expect(await readDesiredPlatform()).toEqual([]); + expect(dm.preparePosts).toBe(1); + expect(dm.settlePosts).toBe(1); + await dm.assertDataAdmission(); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + expect( + host.calls.some( + (args) => + args[0] === "run" || + args.includes("up") || + args[0] === "pull" || + (args[0] === "image" && args[1] === "pull"), + ), + ).toBe(false); + return; + } const launchDeadline = Date.now() + 60000; while (!host.calls.some((args) => args[0] === "run") && Date.now() < launchDeadline) { const current = await readJob(); @@ -785,6 +954,27 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( } const args = host.calls.find((args) => args[0] === "run"); expect(args).toBeDefined(); + let originalRows: unknown, originalPlatformRows: unknown; + if (version === "launch-native-system-release-api-after-save") { + const original = (await readJob()).result as { + _configurationCandidate?: { sealedOperationKey?: string; rowVersions?: unknown }; + }; + const packet = join( + host.queue, + ".candidates", + `${original._configurationCandidate!.sealedOperationKey}.json`, + ), + bytes = readFileSync(packet); + originalRows = original._configurationCandidate!.rowVersions; + originalPlatformRows = await readDesiredPlatform(); + expect(originalPlatformRows).toHaveLength(1); + expect(originalRows).toBeDefined(); + expect(await nativeWorkers[0]!.kill()).toMatchObject({ + code: null, + signal: "SIGKILL", + }); + expect(readFileSync(packet)).toEqual(bytes); + } const env = (name: string) => args!.find((value) => value.startsWith(`${name}=`))!.slice(name.length + 1); await nativeHelper( @@ -816,6 +1006,26 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ); } + if (version === "launch-native-system-release-api-after-save") { + await waitForOriginalLease(); + const replacement = await nativeReleaseWorker(host, state.fixture.connectionUri, { + action: "reconcile", + jobId: accepted.jobId, + }); + nativeWorkers.push(replacement); + expect(await replacement.done).toMatchObject({ + code: 0, + signal: null, + result: { status: "success" }, + }); + const recovered = (await readJob()).result as { + _execution?: { version: number }; + _configurationCandidate?: { rowVersions?: unknown }; + }; + expect(recovered._execution?.version).toBe(2); + expect(recovered._configurationCandidate?.rowVersions).toEqual(originalRows); + expect(await readDesiredPlatform()).toEqual(originalPlatformRows); + } if (version === "launch-native-system-release-killed") { const file = join( host.infraDir, @@ -870,7 +1080,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const settleDeadline = Date.now() + 90000; let job = await readJob(); while (job.status === "running" && Date.now() < settleDeadline) { - await jobRunner.reconcile(); + if (!apiRestart) await jobRunner.reconcile(); await new Promise((resolve) => setTimeout(resolve, 250)); job = await readJob(); } @@ -1678,6 +1888,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ); } finally { await publicApi?.close(); + for (const worker of nativeWorkers) await worker.kill(); for (const agent of nativeAgents) await agent.close(); await dm.close(); await app.close(); @@ -1689,7 +1900,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( rmSync(host.cleanupRoot, { recursive: true, force: true }); } }, - 180000, + 240000, ); }, ); diff --git a/apps/api/src/services/system-release-application.ts b/apps/api/src/services/system-release-application.ts index 8ecd52d22..13048e24c 100644 --- a/apps/api/src/services/system-release-application.ts +++ b/apps/api/src/services/system-release-application.ts @@ -122,7 +122,10 @@ export async function runGuardedSystemRelease(ctx: JobContext): Promise(), continueCommit = Promise.withResolvers(); + const prepareReplyEntered = Promise.withResolvers(), continuePrepareReply = Promise.withResolvers(); const uncertainConnections = new Set>>(); // Named external SQL transport fault. The installed production adapter and // original reserved PostgreSQL connection still perform the actual UPDATE @@ -92,6 +94,7 @@ export async function ownedDataManager(options: { if (id) maintenanceIds.add(id); preparePosts++; maintenanceReplies.push({ status: reply.statusCode, error: typeof payload === "string" ? JSON.parse(payload).error : null }); + if (options.pausePrepareResponse && reply.statusCode === 200) { prepareReplyEntered.resolve(); await continuePrepareReply.promise; } if (options.losePrepareResponse) reply.raw.destroy(); } if (request.method === "POST" && request.url === "/executions/generation-hardlinks") { @@ -121,11 +124,13 @@ export async function ownedDataManager(options: { let stopped = false; const stopForRestart = async () => { if (stopped) return; + continuePrepareReply.resolve(); await runtime.close(); app.server.closeAllConnections(); await app.close(); for (const connection of uncertainConnections) connection.release(); uncertainConnections.clear(); journal.close(); closeControlInstallation(installation); stopped = true; }; return { baseUrl, token, store, receiver, pauseHttp, resumeHttp, get posts() { return posts; }, get preparePosts() { return preparePosts; }, get settlePosts() { return settlePosts; }, + prepareReplyEntered: prepareReplyEntered.promise, releasePrepareResponse: () => continuePrepareReply.resolve(), get settlementSqlIo() { return { commits: settlementCommits, updates: settlementUpdates }; }, get settlementContainmentIo() { return { rollbacks: settlementRollbacks, terminated: terminatedSettlementSession }; }, get fenceInspections() { return structuredClone(fenceInspectionReplies); }, stopForRestart, From 2cfc17f43dad368b8a45bc4de2d32e2a32ad6733 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 13:40:51 +0100 Subject: [PATCH 086/135] test(admin): cover guarded release cancellation and pull failure --- ...orm-configuration-runtime-postgres.test.ts | 91 ++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index df8f1ed60..b035a7072 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -434,6 +434,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-agent-restart", "launch-native-system-release-api-before-save", "launch-native-system-release-api-after-save", + "launch-native-system-release-late-cancel", + "launch-native-system-release-pull-failure", ])( "guards and retains helper mode %s through original ownership", async (version) => { @@ -497,6 +499,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-agent-restart", "launch-native-system-release-api-before-save", "launch-native-system-release-api-after-save", + "launch-native-system-release-late-cancel", + "launch-native-system-release-pull-failure", ].includes(version) ) { state.registry = new services.ServiceRegistry({ @@ -551,6 +555,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( { mode: 0o600 }, ); host.setImagePullProcess(async (reference) => { + if (version === "launch-native-system-release-pull-failure") + throw Error("Named release pull process failure"); host.setImageIdentity(reference, `sha256:${"e".repeat(64)}`); }); } @@ -763,6 +769,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-agent-restart", "launch-native-system-release-api-before-save", "launch-native-system-release-api-after-save", + "launch-native-system-release-late-cancel", + "launch-native-system-release-pull-failure", ].includes(version) ) { const { adminPlatformRoute } = await import("../../routes/admin-platform"), @@ -794,6 +802,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "launch-native-system-release-agent-restart", "launch-native-system-release-api-before-save", "launch-native-system-release-api-after-save", + "launch-native-system-release-late-cancel", + "launch-native-system-release-pull-failure", ].includes(version) ? { version: 1 as const } : { @@ -954,6 +964,36 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( } const args = host.calls.find((args) => args[0] === "run"); expect(args).toBeDefined(); + if (version === "launch-native-system-release-late-cancel") { + const until = Date.now() + 15000; + for (;;) { + const row = await readJob(), + projection = ( + row.result as { + opsProjection?: { + operations?: Record< + string, + { kind: string; receiptDelivery?: { state: string } } + >; + }; + } | null + )?.opsProjection; + if ( + Object.values(projection?.operations ?? {}).some( + (operation) => + operation.kind === "configurationHandoff.prepare" && + operation.receiptDelivery?.state === "confirmed", + ) + ) + break; + expect(Date.now()).toBeLessThan(until); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + // The return value means confirmed containment, not stored intent. + expect(await jobRunner.cancel(accepted.jobId)).toBe(false); + expect((await readJob()).finishedAt).toBeNull(); + expect(dm.settlePosts).toBe(0); + } let originalRows: unknown, originalPlatformRows: unknown; if (version === "launch-native-system-release-api-after-save") { const original = (await readJob()).result as { @@ -985,7 +1025,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), }, - version === "launch-native-release-wrong-config" + version === "launch-native-release-wrong-config" || + version === "launch-native-system-release-pull-failure" ? { failureStage: "application" } : version === "launch-native-system-release-killed" ? { @@ -996,6 +1037,45 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( } : {}, ); + if (version === "launch-native-system-release-pull-failure") { + const file = join( + host.infraDir, + "control-private/ops-agent/observations", + `configuration-handoff-${env("OPS_CONFIGURATION_HANDOFF_ID")}.completion.json`, + ), + original = readFileSync(file); + expect(JSON.parse(String(original)).payload).toMatchObject({ + phase: "recovery_required", + completedAt: null, + }); + await nativeHelper( + host, + { + OPENMAPX_ROOT_DIR: host.root, + OPS_CONFIGURATION_HANDOFF_ID: env("OPS_CONFIGURATION_HANDOFF_ID"), + OPENMAPX_CONTROL_DELEGATION: env("OPENMAPX_CONTROL_DELEGATION"), + OPENMAPX_CONTROL_EXECUTION: env("OPENMAPX_CONTROL_EXECUTION"), + }, + { failureStage: "adoption" }, + ); + expect(readFileSync(file)).toEqual(original); + await jobRunner.reconcile(); + const pending = await readJob(); + expect(pending.status).toBe("running"); + expect(pending.finishedAt).toBeNull(); + expect(dm.settlePosts).toBe(0); + await expect(dm.assertDataAdmission()).rejects.toThrow(); + expect( + host.calls.filter((args) => args[0] === "image" && args[1] === "pull"), + ).toHaveLength(1); + expect(host.calls.some((args) => args[0] === "compose" && args.includes("up"))).toBe( + false, + ); + expect(existsSync(join(host.infraDir, ".ops-agent-releases/current.json"))).toBe( + false, + ); + return; + } if (version === "launch-native-system-release-agent-restart") { expect(nativeAgentExited).toBe(true); nativeAgents.push( @@ -1079,7 +1159,10 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( } const settleDeadline = Date.now() + 90000; let job = await readJob(); - while (job.status === "running" && Date.now() < settleDeadline) { + while ( + ["running", "cancel_pending"].includes(job.status) && + Date.now() < settleDeadline + ) { if (!apiRestart) await jobRunner.reconcile(); await new Promise((resolve) => setTimeout(resolve, 250)); job = await readJob(); @@ -1106,6 +1189,10 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( })), }), ).toBe("success"); + if (version === "launch-native-system-release-late-cancel") + expect( + typeof (job.result as { _cancelRequestedAt?: unknown })._cancelRequestedAt, + ).toBe("string"); expect(job.payload).toEqual( systemRelease ? { From 66a7ab0d0b4fa35f3f50e95e553ac58f070564ab Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 14:01:10 +0100 Subject: [PATCH 087/135] feat(admin): manage platform ownership and reviewed configuration in settings --- apps/api/openapi.json | 27 +- .../routes/__tests__/admin-platform.test.ts | 1 + .../platform-preview-postgres.test.ts | 31 ++ .../services/platform-configuration-view.ts | 20 +- .../src/administrative-runtime.test.ts | 36 +- apps/ops-agent/src/administrative-runtime.ts | 11 +- apps/ops-agent/src/configuration-release.ts | 2 +- apps/ops-agent/src/index.ts | 20 +- .../app/admin/(full-admin)/settings/page.tsx | 9 +- .../settings/PlatformConfiguration.test.tsx | 213 ++++++++++ .../admin/settings/PlatformConfiguration.tsx | 375 ++++++++++++++++++ .../administration/platform-configuration.md | 65 +++ .../administration/services-administration.md | 7 +- packages/cli/__tests__/cli-backup.test.ts | 29 ++ packages/cli/__tests__/cli-release.test.ts | 42 ++ packages/cli/src/commands/backup.ts | 13 +- packages/cli/src/commands/compose.ts | 2 +- packages/cli/src/lib/motis-data.ts | 12 +- packages/cli/src/lib/release.ts | 14 +- .../integration-admin/application.ts | 2 +- .../api/contracts/integration-admin/common.ts | 2 +- .../src/api/contracts/service-admin/data.ts | 4 +- .../api/contracts/service-admin/platform.ts | 18 +- .../getAdminPlatformConfiguration200.ts | 3 + ...dminPlatformConfiguration200PendingItem.ts | 12 + .../generated/admin/service-models/index.ts | 1 + .../core/src/ops/configuration-identifiers.ts | 13 + .../ops/configuration-operation-records.ts | 22 +- .../src/ops/configuration-result-records.ts | 22 + packages/core/src/ops/contract.ts | 148 +------ .../core/src/ops/data-inventory-records.ts | 128 ++++++ .../configuration-generation.test.ts | 37 ++ .../src/services/configuration-generation.ts | 20 + packages/core/src/services/index.ts | 2 + 34 files changed, 1157 insertions(+), 206 deletions(-) create mode 100644 apps/web/src/components/admin/settings/PlatformConfiguration.test.tsx create mode 100644 apps/web/src/components/admin/settings/PlatformConfiguration.tsx create mode 100644 docs/docs/administration/platform-configuration.md create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200PendingItem.ts create mode 100644 packages/core/src/ops/configuration-identifiers.ts create mode 100644 packages/core/src/ops/configuration-result-records.ts create mode 100644 packages/core/src/ops/data-inventory-records.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 8ecceab50..097f2e5cb 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -49877,13 +49877,38 @@ "backupRetentionDays" ], "additionalProperties": false + }, + "pending": { + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "status": { + "type": "string", + "minLength": 1, + "maxLength": 32 + } + }, + "required": [ + "jobId", + "status" + ], + "additionalProperties": false + } } }, "required": [ "expectedCurrentRevisionId", "configuration", "values", - "sources" + "sources", + "pending" ], "additionalProperties": false } diff --git a/apps/api/src/routes/__tests__/admin-platform.test.ts b/apps/api/src/routes/__tests__/admin-platform.test.ts index 259f8a903..6a672cde2 100644 --- a/apps/api/src/routes/__tests__/admin-platform.test.ts +++ b/apps/api/src/routes/__tests__/admin-platform.test.ts @@ -54,6 +54,7 @@ beforeEach(async () => { configuration: { version: 1 }, values: preview.values, sources: preview.sources, + pending: [], }); mocks.preview.mockResolvedValue({ preview, diff --git a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts index 411fca958..c3b52df49 100644 --- a/apps/api/src/services/__tests__/platform-preview-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-preview-postgres.test.ts @@ -214,6 +214,37 @@ dbDescribe("durable exact platform preview", () => { row.kind, ).toHaveLength(row.kind === "closed" ? 0 : 1); }); + it("reconnects to original unretired platform jobs without returning private preview bodies", async () => { + const actorId = await actor(), + value = preview(actorId), + jobId = randomUUID(); + await retainPlatformPreview( + { actorId, operationKey: randomUUID(), requestFingerprint: "a".repeat(64), preview: value }, + fixture.db, + ); + await fixture.db.insert(adminJob).values({ + id: jobId, + type: "platform.apply", + createdBy: actorId, + status: "running", + payload: { private: "must-not-leak" }, + }); + await fixture.db + .update(platformConfigurationPreview) + .set({ consumedByJobId: jobId, consumedAt: new Date() }) + .where(eq(platformConfigurationPreview.id, value.previewId)); + const { inspectPlatformConfiguration } = await import("../platform-configuration-view"); + const current = await inspectPlatformConfiguration(); + expect(current.pending).toContainEqual({ jobId, status: "running" }); + expect(JSON.stringify(current)).not.toContain("must-not-leak"); + await fixture.db + .update(platformConfigurationPreview) + .set({ retiredAt: new Date() }) + .where(eq(platformConfigurationPreview.id, value.previewId)); + expect((await inspectPlatformConfiguration()).pending.map((job) => job.jobId)).not.toContain( + jobId, + ); + }); it("honors an actor-bound privacy preservation hold when unused previews expire", async () => { const actorId = await actor(), value = preview(actorId), diff --git a/apps/api/src/services/platform-configuration-view.ts b/apps/api/src/services/platform-configuration-view.ts index 794d1be1d..9a190f2ba 100644 --- a/apps/api/src/services/platform-configuration-view.ts +++ b/apps/api/src/services/platform-configuration-view.ts @@ -3,10 +3,10 @@ import { createHash, randomUUID } from "node:crypto"; import { adminPlatformPreviewInputSchema } from "@openmapx/core/api-contracts"; import { canonicalJson } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; -import { eq, sql } from "drizzle-orm"; +import { and, desc, eq, isNotNull, isNull, sql } from "drizzle-orm"; import { loadAuthPolicy } from "../account/auth-policy"; import { db } from "../db"; -import { systemSettings } from "../db/schema"; +import { adminJob, platformConfigurationPreview, systemSettings } from "../db/schema"; import { withAdministrativeConfigurationRead } from "./administrative-configuration-writer-owner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; import { PLATFORM_CONFIGURATION_SETTING } from "./platform-configuration"; @@ -26,11 +26,27 @@ export async function inspectPlatformConfiguration() { saved: current.platform, env: process.env, }); + const pending = await db + .select({ jobId: adminJob.id, status: adminJob.status }) + .from(platformConfigurationPreview) + .innerJoin( + adminJob, + eq(adminJob.id, sql`${platformConfigurationPreview.consumedByJobId}::text`), + ) + .where( + and( + isNotNull(platformConfigurationPreview.consumedByJobId), + isNull(platformConfigurationPreview.retiredAt), + ), + ) + .orderBy(desc(platformConfigurationPreview.createdAt)) + .limit(16); return { expectedCurrentRevisionId: current.revisionId, configuration: current.platform ?? { version: 1 as const }, values: effective.values, sources: effective.sources, + pending, }; } export async function previewPlatformConfiguration(input: { diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index 550c792f6..b06f73a21 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -5,11 +5,13 @@ import { mkdtempSync, readdirSync, readFileSync, + rmSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { OpsOperation, ServiceBatchResult } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; import { afterEach, describe, expect, it, vi } from "vitest"; import { testExecution } from "./__fixtures__/execution"; import { @@ -733,6 +735,21 @@ describe("administrative diagnostics runtime", () => { }); }); +async function appliedInspectionFixture(rootDir: string) { + const infraDir = join(rootDir, "infra", "docker"); + await coreServices.commitConfigurationGeneration({ + infraDir, + services: [], + input: { + domain: "localhost", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + }); +} + describe("administrative release runtime", () => { it("stores immutable canonical release IDs with exact-content idempotency and bounded pre-reads", async () => { const rootDir = temporaryRoot(); @@ -1255,7 +1272,7 @@ describe("administrative release runtime", () => { const rootDir = temporaryRoot(); const releaseDirectory = join(rootDir, "infra", "docker", ".ops-agent-releases"); mkdirSync(releaseDirectory, { mode: 0o700 }); - mkdirSync(join(rootDir, "infra", "docker", ".trusted-config-current"), { recursive: true }); + await appliedInspectionFixture(rootDir); writeFileSync( join(rootDir, "infra", "docker", ".trusted-config-current", "docker-compose.generated.yml"), "services:{}\n", @@ -1329,14 +1346,14 @@ describe("administrative release runtime", () => { }); describe("system inspection fails closed on unusable observations", () => { - function systemFixture( + async function systemFixture( compose: "file" | "directory" | "missing", runDocker: (args: readonly string[]) => Promise, ) { const rootDir = temporaryRoot(); const releaseDirectory = join(rootDir, "infra", "docker", ".ops-agent-releases"); mkdirSync(releaseDirectory, { mode: 0o700 }); - mkdirSync(join(rootDir, "infra", "docker", ".trusted-config-current"), { recursive: true }); + await appliedInspectionFixture(rootDir); const composePath = join( rootDir, "infra", @@ -1344,8 +1361,9 @@ describe("administrative release runtime", () => { ".trusted-config-current", "docker-compose.generated.yml", ); - if (compose === "file") writeFileSync(composePath, "services:{}\n"); - if (compose === "directory") mkdirSync(composePath); + rmSync(composePath); + if (compose === "file") writeFileSync(composePath, "services:{}\n", { mode: 0o600 }); + if (compose === "directory") mkdirSync(composePath, { mode: 0o700 }); const releaseId = "release-123"; writeFileSync( join(releaseDirectory, `${releaseId}.json`), @@ -1380,7 +1398,7 @@ describe("administrative release runtime", () => { }); it("reports unknown rather than stopped when a per-service observation throws", async () => { - const effects = systemFixture("file", async (args) => { + const effects = await systemFixture("file", async (args) => { const joined = args.join(" "); if (joined.startsWith("info ")) return "28.0.0\n"; if (joined.endsWith(" config -q")) return ""; @@ -1418,7 +1436,7 @@ describe("administrative release runtime", () => { "transitous-runner": "f", }; const calls: string[][] = []; - const effects = systemFixture("file", async (args) => { + const effects = await systemFixture("file", async (args) => { calls.push([...args]); const joined = args.join(" "); if (joined.startsWith("info ")) return "28.0.0\n"; @@ -1449,7 +1467,7 @@ describe("administrative release runtime", () => { }); it("rejects a non-file Compose path instead of reporting it ready", async () => { - const effects = systemFixture("directory", async (args) => { + const effects = await systemFixture("directory", async (args) => { const joined = args.join(" "); if (joined.startsWith("info ")) return "28.0.0\n"; return ""; @@ -1461,7 +1479,7 @@ describe("administrative release runtime", () => { }); it("rejects a Compose document the Compose boundary cannot parse", async () => { - const effects = systemFixture("file", async (args) => { + const effects = await systemFixture("file", async (args) => { const joined = args.join(" "); if (joined.startsWith("info ")) return "28.0.0\n"; if (joined.endsWith(" config -q")) throw new Error("invalid compose project"); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 901523e2b..36a0e6369 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -964,7 +964,9 @@ export function createDefaultReleaseEffects( }; const readState = () => readReleasePointer(paths.state, true); const configurationIdentity = () => - releaseDigest(JSON.stringify(coreServices.releaseSelection())); + releaseDigest( + JSON.stringify(coreServices.readAppliedReleaseSelection(join(rootDir, "infra", "docker"))), + ); const selectionReceipts = (): Record< string, { configuration: string; ociDigest: string; digest: string } @@ -1003,7 +1005,7 @@ export function createDefaultReleaseEffects( "Checked release is unavailable, changed or belongs to another configuration; check again", ); coreServices.validateResolvedSelection( - coreServices.releaseSelection(), + coreServices.readAppliedReleaseSelection(join(rootDir, "infra", "docker")), manifest, receipt.ociDigest, ); @@ -1018,7 +1020,8 @@ export function createDefaultReleaseEffects( const available = availableReleaseId ? loadManifest(availableReleaseId) : undefined; const receipt = availableReleaseId ? selectionReceipts()[availableReleaseId] : undefined; return { - configuredSelector: process.env.OPENMAPX_RELEASE ?? "main", + configuredSelector: coreServices.readAppliedPlatformValues(join(rootDir, "infra", "docker")) + .releaseSelector, ...(state.releaseId ? { currentReleaseId: state.releaseId } : {}), ...(current?.product ? { currentVersion: current.product.version } : {}), ...(availableReleaseId ? { availableReleaseId } : {}), @@ -1311,7 +1314,7 @@ export function createDefaultReleaseEffects( validateCandidate, resolve: async (context) => { manifestNames(); - const selection = coreServices.releaseSelection(); + const selection = coreServices.readAppliedReleaseSelection(join(rootDir, "infra", "docker")); if (selection.kind === "local") throw new coreServices.ReleaseChannelDisabledError(); const image = selection.manifestImage; const pulled = await runDocker(["pull", image], context); diff --git a/apps/ops-agent/src/configuration-release.ts b/apps/ops-agent/src/configuration-release.ts index 7e6b90b9f..8161ea2c9 100644 --- a/apps/ops-agent/src/configuration-release.ts +++ b/apps/ops-agent/src/configuration-release.ts @@ -24,7 +24,7 @@ export function readCheckedConfigurationRelease( input: { releaseId: string; manifestDigest: string }, ) { const selected = selectionSchema.parse(input), - channel = services.releaseSelection(); + channel = services.readAppliedReleaseSelection(infraDir); if (channel.kind === "local") throw new services.ReleaseChannelDisabledError(); const infra = new AnchoredDirectory(infraDir); try { diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 4865f8651..eacf70e3b 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -231,14 +231,18 @@ async function main(): Promise { store: artifactTransfers, }); const releaseEffects = productionReleaseEffects(runtime); - const retentionDays = Number(process.env.BACKUP_RETENTION_DAYS?.trim() || "30"); - if (!Number.isSafeInteger(retentionDays) || retentionDays <= 0 || retentionDays > 36_500) { - throw new Error("Invalid BACKUP_RETENTION_DAYS"); - } - const runBackupRetention = () => - void pruneBackupRetention(config.rootDir, retentionDays, administrativeCli).catch(() => { - process.stderr.write("Scheduled backup retention failed\n"); - }); + const runBackupRetention = () => { + void Promise.resolve() + .then(() => { + const retentionDays = services.readAppliedPlatformValues( + paths.infraDir, + ).backupRetentionDays; + return pruneBackupRetention(config.rootDir, retentionDays, administrativeCli); + }) + .catch(() => { + process.stderr.write("Scheduled backup retention failed\n"); + }); + }; const policyAuthority = async () => { const authority = await authorityLoader(); const resources = loadConfiguredResourceAuthority(); diff --git a/apps/web/src/app/admin/(full-admin)/settings/page.tsx b/apps/web/src/app/admin/(full-admin)/settings/page.tsx index 9cf00bca6..d21e54944 100644 --- a/apps/web/src/app/admin/(full-admin)/settings/page.tsx +++ b/apps/web/src/app/admin/(full-admin)/settings/page.tsx @@ -1,8 +1,15 @@ +import Stack from "@mui/material/Stack"; import type { Metadata } from "next"; +import { PlatformConfiguration } from "@/components/admin/settings/PlatformConfiguration"; import { SystemSettings } from "@/components/admin/settings/SystemSettings"; export const metadata: Metadata = { title: "Settings — Admin — OpenMapX" }; export default function AdminSettingsPage() { - return ; + return ( + + + + + ); } diff --git a/apps/web/src/components/admin/settings/PlatformConfiguration.test.tsx b/apps/web/src/components/admin/settings/PlatformConfiguration.test.tsx new file mode 100644 index 000000000..55b96930c --- /dev/null +++ b/apps/web/src/components/admin/settings/PlatformConfiguration.test.tsx @@ -0,0 +1,213 @@ +import { QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { cleanup, createTestQueryClient, fireEvent, render, screen, waitFor } from "@/test"; +import { PlatformConfiguration } from "./PlatformConfiguration"; + +const mocks = vi.hoisted(() => ({ get: vi.fn(), preview: vi.fn(), apply: vi.fn(), job: vi.fn() })); +vi.mock("@openmapx/core", async () => ({ + ...(await vi.importActual("@openmapx/core")), + authClient: { + useSession: () => ({ data: { user: { id: "admin-a", role: "admin" } } }), + admin: {}, + }, +})); +vi.mock("@openmapx/core/api-admin-sdk", async () => ({ + ...(await vi.importActual( + "@openmapx/core/api-admin-sdk", + )), + getAdminPlatformConfiguration: mocks.get, + previewAdminPlatformConfiguration: mocks.preview, + applyAdminPlatformConfiguration: mocks.apply, + getAdminActivityJob: mocks.job, +})); +vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useEnv: () => ({ apiUrl: "http://admin.test" }), +})); +vi.mock("../activity/JobDetail", () => ({ + JobDetail: ({ jobId }: { jobId: string }) =>
Job {jobId}
, +})); +const values = { + domain: "maps.example.test", + acmeEmail: "admin@example.test", + releaseSelector: "main", + defaultRegion: null, + backupRetentionDays: 30, +}; +const sources = { + domain: "host", + acmeEmail: "host", + releaseSelector: "host", + defaultRegion: "host", + backupRetentionDays: "host", +}; +const initial = { + expectedCurrentRevisionId: null, + configuration: { version: 1 }, + values, + sources, +}; +const confirmations = { + domain: false, + dnsTls: false, + oauth: false, + passkeys: false, + alternateLogin: false, +}; +const preview = { + ...initial, + version: 1, + actorId: "admin-a", + previewId: "17aa76e8-0359-43d1-8741-307cebb13b90", + authPolicyRevision: "a".repeat(64), + sourceRevision: "17aa76e8-0359-43d1-8741-307cebb13b90", + affectedServiceIds: ["ops-agent"], + oldOrigin: "https://maps.example.test", + newOrigin: "https://maps.example.test", + originConsumers: { + routingHost: values.domain, + corsOrigin: "https://maps.example.test", + authUrl: "https://maps.example.test", + passkeyRpId: values.domain, + passkeyOrigin: "https://maps.example.test", + webApiUrl: "https://maps.example.test", + tilesPublicUrl: "https://maps.example.test", + }, + requiredConfirmation: confirmations, + warnings: [], + previewFingerprint: "b".repeat(64), +}; +const caches: ReturnType[] = []; +function mount() { + const cache = createTestQueryClient(); + caches.push(cache); + return render( + + + , + ); +} +beforeEach(() => { + vi.clearAllMocks(); + sessionStorage.clear(); + mocks.get.mockResolvedValue({ status: 200, data: { ...initial, pending: [] } }); + mocks.job.mockResolvedValue({ + status: 200, + data: { id: "original-job", source: "application", status: "running", finishedAt: null }, + }); + mocks.preview.mockResolvedValue({ + status: 200, + data: { preview, expiresAt: new Date(Date.now() + 600000).toISOString() }, + }); + mocks.apply.mockResolvedValue({ status: 202, data: { ok: true, jobId: "original-job" } }); +}); +afterEach(() => { + cleanup(); + caches.splice(0).forEach((c) => { + c.clear(); + }); +}); +it("locks host fields until explicit adoption, previews without saving and binds apply to that preview", async () => { + mount(); + const retention = await screen.findByLabelText("Backup retention (days)"); + expect(retention).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Manage Backup retention (days) in UI" })); + fireEvent.change(retention, { target: { value: "60" } }); + expect(mocks.apply).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Preview changes" })); + await screen.findByText("Review platform changes"); + expect(mocks.preview.mock.calls[0][0]).toMatchObject({ + expectedCurrentRevisionId: null, + changes: { version: 1, backupRetentionDays: { source: "database", value: 60 } }, + }); + expect(mocks.apply).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Apply reviewed changes" })); + await waitFor(() => expect(mocks.apply).toHaveBeenCalledTimes(1)); + expect(mocks.apply.mock.calls[0][0]).toEqual({ + expectedCurrentRevisionId: null, + previewFingerprint: preview.previewFingerprint, + confirmations, + }); + expect(await screen.findByText("Job original-job")).toBeVisible(); +}); +it("invalidates a preview on edit and preserves the draft on conflict", async () => { + mocks.preview.mockResolvedValue({ status: 409, data: { error: "changed" } }); + mount(); + await screen.findByLabelText("Domain"); + fireEvent.click(screen.getByRole("button", { name: "Manage Domain in UI" })); + fireEvent.change(screen.getByLabelText("Domain"), { target: { value: "new.example.test" } }); + fireEvent.click(screen.getByRole("button", { name: "Preview changes" })); + expect(await screen.findByText(/Preview unavailable/)).toBeVisible(); + expect(screen.getByLabelText("Domain")).toHaveValue("new.example.test"); + mocks.preview.mockResolvedValue({ + status: 200, + data: { preview, expiresAt: new Date(Date.now() + 600000).toISOString() }, + }); + fireEvent.click(screen.getByRole("button", { name: "Preview changes" })); + await screen.findByText("Review platform changes"); + fireEvent.change(screen.getByLabelText("Domain"), { target: { value: "other.example.test" } }); + expect(screen.queryByRole("button", { name: "Apply reviewed changes" })).toBeNull(); + expect(mocks.apply).not.toHaveBeenCalled(); +}); +it("reconnects to pending work and does not offer another apply", async () => { + mocks.get.mockResolvedValue({ + status: 200, + data: { ...initial, pending: [{ jobId: "retained-job", status: "running" }] }, + }); + mount(); + expect(await screen.findByText("Job retained-job")).toBeVisible(); + expect(screen.getByRole("button", { name: "Preview changes" })).toBeDisabled(); +}); + +it("requires all origin acknowledgments and the exact preview sign-in proof", async () => { + mocks.preview.mockResolvedValue({ + status: 200, + data: { + preview: { + ...preview, + requiredConfirmation: { + domain: true, + dnsTls: true, + oauth: true, + passkeys: true, + alternateLogin: true, + }, + newOrigin: "https://new.example.test", + }, + expiresAt: new Date(Date.now() + 600000).toISOString(), + }, + }); + mount(); + await screen.findByLabelText("Domain"); + fireEvent.click(screen.getByRole("button", { name: "Preview changes" })); + const apply = await screen.findByRole("button", { name: "Apply reviewed changes" }); + expect(apply).toBeDisabled(); + for (const checkbox of screen.getAllByRole("checkbox")) fireEvent.click(checkbox); + expect(apply).toBeDisabled(); + expect(screen.getByRole("link", { name: "Open reviewed admin address" })).toHaveAttribute( + "href", + "https://new.example.test/admin", + ); + expect(mocks.apply).not.toHaveBeenCalled(); +}); +it("cannot apply an expired preview", async () => { + mocks.preview.mockResolvedValue({ + status: 200, + data: { preview, expiresAt: new Date(Date.now() - 1000).toISOString() }, + }); + mount(); + await screen.findByLabelText("Domain"); + fireEvent.click(screen.getByRole("button", { name: "Preview changes" })); + expect(await screen.findByText(/This preview expired/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Apply reviewed changes" })).toBeDisabled(); +}); + +it("retains the exact unexpired actor-bound preview across the alternate-login return", async () => { + mount(); + await screen.findByLabelText("Domain"); + fireEvent.click(screen.getByRole("button", { name: "Preview changes" })); + await screen.findByText("Review platform changes"); + cleanup(); + mount(); + expect(await screen.findByText("Review platform changes")).toBeVisible(); + expect(mocks.preview).toHaveBeenCalledTimes(1); +}); diff --git a/apps/web/src/components/admin/settings/PlatformConfiguration.tsx b/apps/web/src/components/admin/settings/PlatformConfiguration.tsx new file mode 100644 index 000000000..c22391478 --- /dev/null +++ b/apps/web/src/components/admin/settings/PlatformConfiguration.tsx @@ -0,0 +1,375 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Button from "@mui/material/Button"; +import Checkbox from "@mui/material/Checkbox"; +import FormControlLabel from "@mui/material/FormControlLabel"; +import Paper from "@mui/material/Paper"; +import Stack from "@mui/material/Stack"; +import TextField from "@mui/material/TextField"; +import Typography from "@mui/material/Typography"; +import { + applyAdminPlatformConfiguration, + getAdminPlatformConfiguration, + previewAdminPlatformConfiguration, + unwrapAdminResponse, +} from "@openmapx/core/api-admin-sdk"; +import { + type AdminPlatformPreview, + adminPlatformPreviewRetainedSchema, +} from "@openmapx/core/api-contracts"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useEffect, useRef, useState } from "react"; +import { useAccountAdmin } from "@/lib/admin/accountAdmin"; +import { activityJobQueryOptions, useActivityClient } from "../activity/activityQueries"; +import { JobDetail } from "../activity/JobDetail"; +import { AdminReauthenticationDialog } from "../users/AdminReauthenticationDialog"; + +type View = Awaited> & { status: 200 }; +type Configuration = AdminPlatformPreview["configuration"]; +type Field = keyof AdminPlatformPreview["values"]; +const fields: { key: Field; label: string; help: string }[] = [ + { + key: "domain", + label: "Domain", + help: "Hostname only. Requires DNS/TLS, login and passkey recovery review.", + }, + { key: "acmeEmail", label: "ACME contact email", help: "Certificate renewal contact." }, + { + key: "releaseSelector", + label: "Release selector", + help: "Channel, exact version or local. Existing images stay pinned until an explicit checked update.", + }, + { + key: "defaultRegion", + label: "Default region", + help: "Optional resource region. The deployment allowlist still applies.", + }, + { + key: "backupRetentionDays", + label: "Backup retention (days)", + help: "1–36500 days. Applies to creation, restore and scheduled/manual pruning.", + }, +]; +const confirmationLabels = { + domain: "I have reviewed the new public origin", + dnsTls: "DNS and TLS are ready for the new domain", + oauth: "OAuth provider callback URLs are ready", + passkeys: "I understand that existing passkeys may require recovery", + alternateLogin: "I have proved an alternate supported password login and second factor", +}; +const emptyConfirmations = { + domain: false, + dnsTls: false, + oauth: false, + passkeys: false, + alternateLogin: false, +}; + +/** Effective ownership and retained job links stay available after a browser reconnect. */ +export function PlatformConfiguration() { + const { actorId } = useAccountAdmin(), + { client, apiUrl } = useActivityClient(); + const query = useQuery({ + queryKey: ["admin", "platform", apiUrl, actorId], + enabled: !!actorId, + retry: false, + refetchInterval: 10000, + queryFn: async () => unwrapAdminResponse(await getAdminPlatformConfiguration({ client })), + }); + return ( + + Platform configuration + {query.isError && ( + void query.refetch()}>Reload} + > + Platform state is unavailable. Inspect Activity and host recovery guidance before changing + configuration. + + )} + {query.data ? ( + + ) : ( + !query.isError && Loading platform configuration… + )} + + ); +} +function PlatformForm({ view, actorId }: { view: View["data"]; actorId: string }) { + const { client, apiUrl } = useActivityClient(), + storageKey = `openmapx-platform-preview:${apiUrl}:${actorId}`; + const [retained, setRetained] = useState | null>(() => { + try { + const raw = sessionStorage.getItem(storageKey); + if (!raw) return null; + const parsed = adminPlatformPreviewRetainedSchema.parse(JSON.parse(raw)); + return parsed.preview.actorId === actorId && + parsed.preview.expectedCurrentRevisionId === view.expectedCurrentRevisionId && + Date.parse(parsed.expiresAt) > Date.now() + ? parsed + : null; + } catch { + return null; + } + }); + const [draft, setDraft] = useState( + retained?.preview.configuration ?? view.configuration, + ), + [confirmations, setConfirmations] = useState(emptyConfirmations), + [proof, setProof] = useState(), + [reauth, setReauth] = useState(false), + [accepted, setAccepted] = useState(), + [, tick] = useState(0); + const previewKey = useRef(null), + applyKey = useRef(null); + const acceptedJob = useQuery({ + ...activityJobQueryOptions(client, apiUrl, accepted, "application"), + refetchInterval: 5000, + }); + const observedTerminal = + !!acceptedJob.data?.finishedAt && + ["success", "failed", "canceled"].includes(acceptedJob.data.status); + const pending = view.pending ?? [], + busy = pending.length > 0 || (!!accepted && !observedTerminal); + const clearPreview = () => { + setRetained(null); + setConfirmations(emptyConfirmations); + setProof(undefined); + setReauth(false); + sessionStorage.removeItem(storageKey); + previewKey.current = null; + applyKey.current = null; + }; + const previewMutation = useMutation({ + retry: false, + mutationFn: async () => { + previewKey.current ??= crypto.randomUUID(); + return unwrapAdminResponse( + await previewAdminPlatformConfiguration( + { expectedCurrentRevisionId: view.expectedCurrentRevisionId, changes: draft }, + { "idempotency-key": previewKey.current }, + { client }, + ), + ); + }, + onError: () => { + previewKey.current = null; + }, + onSuccess: (value) => { + setRetained(value); + sessionStorage.setItem(storageKey, JSON.stringify(value)); + applyKey.current = null; + }, + }); + const applyMutation = useMutation({ + retry: false, + mutationFn: async () => { + if (!retained || Date.parse(retained.expiresAt) <= Date.now()) throw Error("Preview expired"); + applyKey.current ??= crypto.randomUUID(); + return unwrapAdminResponse( + await applyAdminPlatformConfiguration( + { + expectedCurrentRevisionId: retained.preview.expectedCurrentRevisionId, + previewFingerprint: retained.preview.previewFingerprint, + confirmations, + ...(proof ? { domainProofToken: proof } : {}), + }, + { "idempotency-key": applyKey.current }, + { client }, + ), + ); + }, + onSuccess: (value) => { + setAccepted(value.jobId); + clearPreview(); + }, + }); + useEffect(() => { + if (!retained) return; + const timeout = setTimeout( + () => tick((x) => x + 1), + Math.max(1, Date.parse(retained.expiresAt) - Date.now() + 1), + ); + return () => clearTimeout(timeout); + }, [retained]); + const working = previewMutation.isPending || applyMutation.isPending; + const edit = (key: Field, value: Configuration[Field]) => { + clearPreview(); + previewMutation.reset(); + applyMutation.reset(); + setDraft((old) => ({ ...old, [key]: value })); + }; + const preview = retained?.preview, + expired = !!retained && Date.parse(retained.expiresAt) <= Date.now(); + return ( + + + Host-owned values are locked until you explicitly adopt them. Returning a field to the host + uses its current environment value. Previewing does not save or restart services. + + {fields.map(({ key, label, help }) => { + const field = draft[key], + managed = field?.source === "database"; + const value = managed ? field.value : view.values[key]; + return ( + + + edit(key, { + source: "database", + value: key === "backupRetentionDays" ? Number(e.target.value) : e.target.value, + } as Configuration[Field]) + } + /> + + + ); + })} + {previewMutation.isError && ( + + Preview unavailable. The revision, input or source may have changed. Reload before + retrying; your draft is retained. + + )} + {applyMutation.isError && ( + + Apply was not confirmed. Inspect Activity before retrying with the same request. If the + preview or policy changed, reload and review a new preview. + + )} + + {preview && ( + + Review platform changes + + Public origin: {preview.oldOrigin} → {preview.newOrigin} + + + Affected services: {preview.affectedServiceIds.join(", ") || "None"} + + + Reconnect at {preview.newOrigin}/admin after a domain change. Keep host access available + if DNS, TLS, login or the control plane cannot recover. + + + {preview.warnings.map((warning) => ( + + {warning} + + ))} + {Object.entries(preview.requiredConfirmation) + .filter(([, required]) => required) + .map(([key]) => ( + + setConfirmations((old) => ({ ...old, [key]: e.target.checked })) + } + /> + } + label={confirmationLabels[key as keyof typeof confirmationLabels]} + /> + ))} + {preview.requiredConfirmation.alternateLogin && ( + <> + + A fresh supported password login and effective second factor are required. + Passkey-only or OAuth sessions cannot authorize a domain change. Establish a + supported alternate login first. + + + + )} + {expired && This preview expired. Create a new preview.} + + + )} + {reauth && preview && ( + setReauth(false)} + onCompleted={(proofs) => { + setProof(proofs[0]?.challengeId); + setReauth(false); + }} + /> + )} + {busy && ( + + An original platform operation is retained. Follow its outcome in Activity; disconnecting + does not cancel it. Unknown outcomes require recovery before another apply. + + )} + {[...new Set([...pending.map((job) => job.jobId), ...(accepted ? [accepted] : [])])].map( + (jobId) => ( + + ), + )} + + + Host recovery: inspect the retained operation in Activity and the host ops-agent journal. Do + not delete locks or replay an uncertain update. Bootstrap secrets, Docker installation and + offline agent recovery require host access. + + + ); +} diff --git a/docs/docs/administration/platform-configuration.md b/docs/docs/administration/platform-configuration.md new file mode 100644 index 000000000..3d0cf38bd --- /dev/null +++ b/docs/docs/administration/platform-configuration.md @@ -0,0 +1,65 @@ +--- +title: Platform configuration +description: Review and apply domain, TLS contact, release and retention settings with explicit ownership and recoverable jobs. +sidebar_position: 3 +--- + +# Platform configuration + +Open **Admin → Settings → Platform configuration**. The form shows effective values +and their current source. Host-owned fields are locked. Choose **Manage … in UI** +to adopt a supported field, or **Return … to host** to use the current host value +again. Editing and previewing do not save or restart anything. + +| Field | Host default | When an applied change takes effect | +| ------------------ | ------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| Domain | `DOMAIN`, otherwise `localhost` | Coordinated routing, API/auth origin and web runtime configuration | +| ACME contact email | `ACME_EMAIL`, otherwise `admin@example.com` | Applied certificate configuration | +| Release selector | `OPENMAPX_RELEASE`, otherwise `main` | The next explicit release check/selection; existing image pins stay selected | +| Default region | `OPENMAPX_REGION`, otherwise unset | Subsequent data/build operations without an explicit region; deployment allowlists still apply | +| Backup retention | `BACKUP_RETENTION_DAYS`, otherwise 30 days | Subsequent creation, restore validation and scheduled/manual pruning | + +Use **Preview changes** to review the exact origin, affected services and warnings. +The preview expires after ten minutes. Changing the draft invalidates the preview. +Apply uses that exact preview and current configuration revision. A source, policy +or revision conflict requires a new preview; a failed request preserves the draft. +Service resource fields remain in the individual service configuration form. See +[Operator settings and resource limits](./services-administration.md#operator-settings-and-resource-limits). + +## Changing the domain + +Prepare DNS and TLS, update OAuth callback URLs, and arrange recovery for passkeys +bound to the old domain. The preview lists the new address. Keep host access and +an alternate login available before applying. + +Confirm the listed acknowledgments and verify a fresh supported password login +with the account's effective second factor. A passkey-only or OAuth session cannot +authorize this change. Establish and prove the supported alternate login first. +The sign-in confirmation is bound to the exact preview and cannot be reused for +another domain or policy. Return to Settings after sign-in to complete the review. + +After applying, reconnect at the reviewed admin address. Browser disconnection is +an observation failure, not cancellation or evidence that the change failed. + +## Operation status and recovery + +Accepted changes are durable jobs. Settings reconnects to retained platform jobs; +Activity shows the original job, outcome and logs. The application coordinates the +configuration generation, Data Manager guard, protected service helper and image +verification before reporting completion. The agent is replaced last during a +checked release update. Images must provide the supported configuration helper; +older installations receive a host-upgrade prerequisite before application. + +A cancel request does not undo an already committed update. If work has committed, +the original verified completion wins. A failed pull, interrupted helper, missing +mount or uncertain publication can leave the operation fenced as requiring recovery. +The system does not claim a rollback from an error message or an old overlay alone. +It releases ownership only after proving the original operation settled. + +Inspect the original job and host ops-agent journal. Do not delete locks, fabricate +completion, or submit repeated fresh updates while an outcome is unknown. Restore +connectivity and observe the original operation; where exact recovery is unavailable, +use host access to diagnose and recover the deployment. Initial Docker installation, +bootstrap credentials, offline agent recovery and destructive full-stack shutdown +remain host operations. The UI does not expose arbitrary environment variables, +mount paths, executable options or Docker socket access. diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index 27fe3014b..7cfb7485e 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -6,8 +6,9 @@ sidebar_position: 2 # Services administration -Everything you can do to a backend service from the command line, you can also -do from the admin panel — in a browser, behind the same login. The admin pages +Manage supported backend service operations from the admin panel in a browser, +behind the same login. Operation availability follows the current instance and +service ownership; initial installation and offline recovery require host access. The admin pages under `/admin/services` are a window onto the service registry: they list what's installed, show each service's manifest and live container status, let you edit config and stream logs, preview the compose file the renderer would write, and @@ -411,3 +412,5 @@ Consumer references: [Ollama environment parser](https://github.com/ollama/ollam and [logger](https://github.com/rtuszik/photon-docker/blob/2.4.0/src/utils/logger.py), [Nominatim import styles](https://nominatim.org/release-docs/latest/customize/Import-Styles/), and [Valhalla scripted image](https://github.com/valhalla/valhalla/blob/master/docker/README.md). + +Platform-wide settings use the reviewed [Platform configuration](./platform-configuration.md) workflow. diff --git a/packages/cli/__tests__/cli-backup.test.ts b/packages/cli/__tests__/cli-backup.test.ts index b0bc6fa1e..d2970a5c0 100644 --- a/packages/cli/__tests__/cli-backup.test.ts +++ b/packages/cli/__tests__/cli-backup.test.ts @@ -1508,6 +1508,35 @@ describe("restore data-protection preflight", () => { }; } + it("uses applied retention for restore and observes return to host on the next call", async () => { + const { services } = await import("@openmapx/core/server"), + infraDir = join(tmp, "infra", "docker"); + const input = { + domain: "maps.example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }; + process.env.BACKUP_RETENTION_DAYS = "30"; + await services.commitConfigurationGeneration({ + infraDir, + services: [], + input: { + ...input, + platform: { version: 1, backupRetentionDays: { source: "database", value: 60 } }, + }, + }); + const manifest = protectedManifest("2026-07-01T00:00:00.000Z"), + opts = { rootDir: tmp, now: new Date("2026-08-01T00:00:00.000Z") }; + expect(() => validateRestoreDataProtection(manifest, opts)).not.toThrow(); + await services.commitConfigurationGeneration({ + infraDir, + services: [], + input: { ...input, platform: { version: 1, backupRetentionDays: { source: "host" } } }, + }); + expect(() => validateRestoreDataProtection(manifest, opts)).toThrow(/retention policy/); + }); it("accepts an in-policy OpenMapX backup covered by the external journal", () => { const journalPath = join(tmp, "infra", "docker", "data", "erasure", "journal.jsonl"); mkdirSync(join(journalPath, ".."), { recursive: true }); diff --git a/packages/cli/__tests__/cli-release.test.ts b/packages/cli/__tests__/cli-release.test.ts index a5e6a9049..35a7f73f1 100644 --- a/packages/cli/__tests__/cli-release.test.ts +++ b/packages/cli/__tests__/cli-release.test.ts @@ -427,3 +427,45 @@ it("rejects a malformed existing overlay without replacing it or reaching the re expect(readFileSync(path, "utf8")).toBe("services: {}\n"); expect(docker).not.toHaveBeenCalled(); }); + +it("uses the applied UI selector on each invocation without changing process environment", async () => { + const { services } = await import("@openmapx/core/server"); + temp = mkdtempSync(join(tmpdir(), "omx-applied-release-")); + const infraDir = join(temp, "infra", "docker"); + mkdirSync(infraDir, { recursive: true }); + const input = { + domain: "maps.example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }; + vi.stubEnv("OPENMAPX_ROOT_DIR", temp); + vi.stubEnv("OPENMAPX_RELEASE", "local"); + try { + await services.commitConfigurationGeneration({ + infraDir, + services: [], + input: { + ...input, + platform: { version: 1, releaseSelector: { source: "database", value: "local" } }, + }, + }); + const docker = fakeDocker(); + await expect(ensureReleaseOverlay({ docker })).resolves.toEqual({ status: "disabled" }); + expect(docker).not.toHaveBeenCalled(); + await services.commitConfigurationGeneration({ + infraDir, + services: [], + input: { + ...input, + platform: { version: 1, releaseSelector: { source: "database", value: "main" } }, + }, + }); + await resolveReleaseManifest(docker); + expect(docker).toHaveBeenCalledWith(["pull", "ghcr.io/openmapx/release-manifest:main"]); + expect(process.env.OPENMAPX_RELEASE).toBe("local"); + } finally { + vi.unstubAllEnvs(); + } +}); diff --git a/packages/cli/src/commands/backup.ts b/packages/cli/src/commands/backup.ts index cc90c20b4..64323fa59 100644 --- a/packages/cli/src/commands/backup.ts +++ b/packages/cli/src/commands/backup.ts @@ -581,7 +581,7 @@ export async function createBackup(opts: CreateBackupOptions = {}): Promise 36_500) { - throw new Error("BACKUP_RETENTION_DAYS must be an integer between 1 and 36500"); - } - return days; +function configuredBackupRetentionDays(rootDir?: string): number { + return coreServices.readAppliedPlatformValues(repoPaths(rootDir).infraDir).backupRetentionDays; } function containsOpenMapXDatabase(manifest: BackupManifest): boolean { @@ -1048,7 +1043,7 @@ export function validateRestoreDataProtection( opts: { rootDir?: string; retentionDays?: number; now?: Date } = {}, ): void { if (!containsOpenMapXDatabase(manifest)) return; - const retentionDays = opts.retentionDays ?? configuredBackupRetentionDays(); + const retentionDays = opts.retentionDays ?? configuredBackupRetentionDays(opts.rootDir); if (!Number.isSafeInteger(retentionDays) || retentionDays <= 0 || retentionDays > 36_500) { throw new Error("Backup retention days must be an integer between 1 and 36500"); } diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index 724f73b21..9cf9e44ce 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -196,8 +196,8 @@ export function rotateRedisPasswordForRepo( export async function renderComposeForRepo(opts: RenderRepoOptions): Promise { assertCliExecutionAvailable(); - coreServices.releaseSelection(); const paths = repoPaths(opts.rootDir); + coreServices.readAppliedReleaseSelection(paths.infraDir); assertCliDeploymentSecrets(); if (!existsSync("/run/secrets/openmapx-control-verifiers")) await provisionControlInstallation(paths.infraDir); diff --git a/packages/cli/src/lib/motis-data.ts b/packages/cli/src/lib/motis-data.ts index d43982f51..3be4e7ef9 100644 --- a/packages/cli/src/lib/motis-data.ts +++ b/packages/cli/src/lib/motis-data.ts @@ -68,7 +68,7 @@ export const DEFAULT_TRANSITOUS_TOOLS_IMAGE = */ export function resolveTransitousToolsImage(rootDir?: string): string { const fromEnv = process.env.OPENMAPX_TRANSITOUS_TOOLS_IMAGE; - const selection = coreServices.releaseSelection(); + const selection = coreServices.readAppliedReleaseSelection(repoPaths(rootDir).infraDir); const overlayPath = repoPaths(rootDir).composeReleasePath; if (existsSync(overlayPath)) { const selected = coreServices.parseReleaseComposeSelection(readFileSync(overlayPath, "utf8")); @@ -249,11 +249,17 @@ async function ensureTransitousToolsImage( await runner("docker", ["pull", image], { cwd: rootDir, stdio: "inherit" }); return; } catch (error) { - if (image.includes("@sha256:") || coreServices.releaseSelection().kind !== "local") + if ( + image.includes("@sha256:") || + coreServices.readAppliedReleaseSelection(repoPaths(rootDir).infraDir).kind !== "local" + ) throw error; } } - if (image.includes("@sha256:") || coreServices.releaseSelection().kind !== "local") + if ( + image.includes("@sha256:") || + coreServices.readAppliedReleaseSelection(repoPaths(rootDir).infraDir).kind !== "local" + ) throw new Error("Refusing to build a replacement for a release-selected helper image"); const contextDir = join(rootDir, "services", "motis", "tools", "transitous"); await runner("docker", ["build", "-t", image, contextDir], { cwd: rootDir, stdio: "inherit" }); diff --git a/packages/cli/src/lib/release.ts b/packages/cli/src/lib/release.ts index 304562956..f66fb2599 100644 --- a/packages/cli/src/lib/release.ts +++ b/packages/cli/src/lib/release.ts @@ -24,12 +24,16 @@ const { RELEASE_SELECTOR_ENV, RELEASE_PINNED_SERVICE_IDS, parseReleaseManifest, - releaseChannel, writeReleaseComposeArtifacts, } = coreServices; export type ReleaseManifest = coreServices.ReleaseManifest; +function appliedReleaseChannel() { + const selector = coreServices.readAppliedPlatformValues(repoPaths().infraDir).releaseSelector; + return coreServices.releaseChannel(selector, process.env.OPENMAPX_RELEASE_MANIFEST_REPOSITORY); +} + export type ReleaseDockerRunner = ( args: string[], ) => Promise<{ stdout: string; exitCode: number; stderr: string }>; @@ -53,7 +57,7 @@ function expectOk(result: { exitCode: number; stderr: string }, what: string): v export async function resolveReleaseManifest( docker: ReleaseDockerRunner = defaultDocker, ): Promise { - const channel = releaseChannel(); + const channel = appliedReleaseChannel(); if (channel.kind === "disabled") { throw new Error(`release resolution is disabled (${RELEASE_SELECTOR_ENV}=local)`); } @@ -133,7 +137,7 @@ export async function ensureReleaseOverlay( } // An operator running local images opts out once instead of paying a // failed registry pull (and a warning) on every start. - if (releaseChannel().kind === "disabled") return { status: "disabled" }; + if (appliedReleaseChannel().kind === "disabled") return { status: "disabled" }; try { const manifest = await resolveReleaseManifest(opts.docker); writeReleaseOverlay(manifest, path); @@ -161,7 +165,7 @@ function readSelection(path: string): coreServices.ReleaseComposeSelection | nul /** Read-only local status; an existing overlay still applies when resolution is disabled. */ export function releaseStatusLines(path = repoPaths().composeReleasePath): string[] { const selected = readSelection(path); - const disabled = releaseChannel().kind === "disabled"; + const disabled = appliedReleaseChannel().kind === "disabled"; if (!selected) return [ disabled @@ -169,7 +173,7 @@ export function releaseStatusLines(path = repoPaths().composeReleasePath): strin : "No release lockfile selected locally. Run `pnpm openmapx compose release`.", ]; return [ - `Configured selection: ${process.env.OPENMAPX_RELEASE ?? "main"}; existing pins remain selected until explicit reselection.`, + `Configured selection: ${coreServices.readAppliedPlatformValues(repoPaths().infraDir).releaseSelector}; existing pins remain selected until explicit reselection.`, ...(disabled ? ["Release resolution disabled; existing overlay still applies."] : []), `Selected release: ${selected.release ?? "unknown (legacy or modified overlay)"}`, ...coreServices.RELEASE_IMAGE_NAMES.map( diff --git a/packages/core/src/api/contracts/integration-admin/application.ts b/packages/core/src/api/contracts/integration-admin/application.ts index 4d79b1612..75c19aeec 100644 --- a/packages/core/src/api/contracts/integration-admin/application.ts +++ b/packages/core/src/api/contracts/integration-admin/application.ts @@ -1,5 +1,5 @@ import z from "zod/v4"; -import { configurationApplyResultSchema } from "../../../ops/contract"; +import { configurationApplyResultSchema } from "../../../ops/configuration-result-records"; /** Live host evidence, independent of the desired database binding. */ export const integrationRuntimeEvidenceSchema = z.strictObject({ generationId: z.uuid(), diff --git a/packages/core/src/api/contracts/integration-admin/common.ts b/packages/core/src/api/contracts/integration-admin/common.ts index 63e836028..bbcbef915 100644 --- a/packages/core/src/api/contracts/integration-admin/common.ts +++ b/packages/core/src/api/contracts/integration-admin/common.ts @@ -3,7 +3,7 @@ import { integrationManifestSchema, } from "@openmapx/extension-contracts/integration"; import z from "zod/v4"; -import { configurationApplyResultSchema } from "../../../ops/contract"; +import { configurationApplyResultSchema } from "../../../ops/configuration-result-records"; import { adminCountSchema, adminJsonValueSchema, adminTimestampSchema } from "../admin"; import { configurationOwnershipSchema } from "../configuration-ownership"; import { adminJsonWire, preserveManifestJson } from "../manifest-json"; diff --git a/packages/core/src/api/contracts/service-admin/data.ts b/packages/core/src/api/contracts/service-admin/data.ts index fd00060d7..35d56f727 100644 --- a/packages/core/src/api/contracts/service-admin/data.ts +++ b/packages/core/src/api/contracts/service-admin/data.ts @@ -1,5 +1,5 @@ import z from "zod/v4"; -import { OPS_RESULT_SCHEMAS } from "../../../ops/contract"; +import { dataInspectionResultSchema } from "../../../ops/data-inventory-records"; import { adminCountSchema, adminTimestampSchema } from "../admin"; import { adminServiceIdListSchema } from "./common"; @@ -7,7 +7,7 @@ export const adminServiceDataQuerySchema = z.strictObject({ osmLimit: z.coerce.number().int().min(1).max(500).optional(), osmOffset: z.coerce.number().int().min(0).max(1000000).optional(), }); -export const adminServiceDataSchema = OPS_RESULT_SCHEMAS["data.inspect"].extend({ +export const adminServiceDataSchema = dataInspectionResultSchema.extend({ fetchedAt: adminTimestampSchema, }); export const adminServiceSelectionSchema = z.strictObject({ diff --git a/packages/core/src/api/contracts/service-admin/platform.ts b/packages/core/src/api/contracts/service-admin/platform.ts index 733b6e2aa..24745b7cc 100644 --- a/packages/core/src/api/contracts/service-admin/platform.ts +++ b/packages/core/src/api/contracts/service-admin/platform.ts @@ -73,12 +73,18 @@ export const adminPlatformApplyInputSchema = z.strictObject({ domainProofToken: z.uuid().optional(), }); export type AdminPlatformPreview = z.infer; -export const adminPlatformConfigurationSchema = adminPlatformPreviewSchema.pick({ - expectedCurrentRevisionId: true, - configuration: true, - values: true, - sources: true, -}); +export const adminPlatformConfigurationSchema = adminPlatformPreviewSchema + .pick({ + expectedCurrentRevisionId: true, + configuration: true, + values: true, + sources: true, + }) + .extend({ + pending: z + .array(z.strictObject({ jobId: z.uuid(), status: z.string().min(1).max(32) })) + .max(16), + }); export const adminPlatformPreviewRetainedSchema = z.strictObject({ preview: adminPlatformPreviewSchema, expiresAt: z.iso.datetime({ offset: true }), diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts index ad6f87b51..671ea297f 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200.ts @@ -1,6 +1,7 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. import type { GetAdminPlatformConfiguration200Configuration } from "./getAdminPlatformConfiguration200Configuration"; +import type { GetAdminPlatformConfiguration200PendingItem } from "./getAdminPlatformConfiguration200PendingItem"; import type { GetAdminPlatformConfiguration200Sources } from "./getAdminPlatformConfiguration200Sources"; import type { GetAdminPlatformConfiguration200Values } from "./getAdminPlatformConfiguration200Values"; @@ -9,4 +10,6 @@ export type GetAdminPlatformConfiguration200 = { configuration: GetAdminPlatformConfiguration200Configuration; values: GetAdminPlatformConfiguration200Values; sources: GetAdminPlatformConfiguration200Sources; + /** @maxItems 16 */ + pending: GetAdminPlatformConfiguration200PendingItem[]; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200PendingItem.ts b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200PendingItem.ts new file mode 100644 index 000000000..34d17d44d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminPlatformConfiguration200PendingItem.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminPlatformConfiguration200PendingItem = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; + /** + * @minLength 1 + * @maxLength 32 + */ + status: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index 99d70ffc3..c1e89bba2 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -173,6 +173,7 @@ export * from "./getAdminPlatformConfiguration200ConfigurationDefaultRegion"; export * from "./getAdminPlatformConfiguration200ConfigurationDomain"; export * from "./getAdminPlatformConfiguration200ConfigurationReleaseSelector"; export * from "./getAdminPlatformConfiguration200ConfigurationVersion"; +export * from "./getAdminPlatformConfiguration200PendingItem"; export * from "./getAdminPlatformConfiguration200Sources"; export * from "./getAdminPlatformConfiguration200SourcesAcmeEmail"; export * from "./getAdminPlatformConfiguration200SourcesBackupRetentionDays"; diff --git a/packages/core/src/ops/configuration-identifiers.ts b/packages/core/src/ops/configuration-identifiers.ts new file mode 100644 index 000000000..30645c004 --- /dev/null +++ b/packages/core/src/ops/configuration-identifiers.ts @@ -0,0 +1,13 @@ +/** Portable identifier schemas shared by inputs and browser-visible results. */ +import { z } from "zod"; +export const serviceIdSchema = z + .string() + .min(1) + .max(64) + .regex(/^[a-z0-9][a-z0-9-]*$/); +export const stableIdSchema = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/); +export const configurationGenerationRevisionSchema = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); diff --git a/packages/core/src/ops/configuration-operation-records.ts b/packages/core/src/ops/configuration-operation-records.ts index de5fa0899..227de06c3 100644 --- a/packages/core/src/ops/configuration-operation-records.ts +++ b/packages/core/src/ops/configuration-operation-records.ts @@ -2,19 +2,19 @@ * maintenance receipt schemas. Contains no result schema/runtime policy. */ import { z } from "zod"; import { configurationHandoffPrepareOperationSchema } from "./configuration-handoff-records"; +import { + configurationGenerationRevisionSchema, + serviceIdSchema, + stableIdSchema, +} from "./configuration-identifiers"; import { coordinatorModeSchema } from "./configuration-preparation-records"; -export const serviceIdSchema = z - .string() - .min(1) - .max(64) - .regex(/^[a-z0-9][a-z0-9-]*$/); -export const stableIdSchema = z - .string() - .min(1) - .max(128) - .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/); -export const configurationGenerationRevisionSchema = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); +export { + configurationGenerationRevisionSchema, + serviceIdSchema, + stableIdSchema, +} from "./configuration-identifiers"; + const configurationCondition = { expectedSourceRevision: z.uuid(), coordinatorMode: coordinatorModeSchema, diff --git a/packages/core/src/ops/configuration-result-records.ts b/packages/core/src/ops/configuration-result-records.ts new file mode 100644 index 000000000..4c8f14a47 --- /dev/null +++ b/packages/core/src/ops/configuration-result-records.ts @@ -0,0 +1,22 @@ +/** Portable results; no host runtime or executable operation imports. */ +import { z } from "zod"; +import { + configurationGenerationRevisionSchema, + serviceIdSchema, +} from "./configuration-identifiers"; +export const configurationApplyResultSchema = z.discriminatedUnion("outcome", [ + z.strictObject({ + outcome: z.literal("committed"), + revisionId: configurationGenerationRevisionSchema, + enabledServiceIds: z.array(serviceIdSchema).max(256), + followUp: z.enum(["complete", "failed"]), + cleanupFailed: z.boolean(), + }), + z.strictObject({ + outcome: z.literal("not_committed"), + reason: z.enum(["conflict", "busy", "failed"]), + currentRevisionId: configurationGenerationRevisionSchema.nullable().optional(), + cleanupFailed: z.boolean(), + }), +]); +export type ConfigurationApplyResult = z.infer; diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 9e2cffae2..e9416a47a 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -29,6 +29,8 @@ import { stackRenderOperationSchema, vaultApplyOperationSchema, } from "./configuration-operation-records"; +import { configurationApplyResultSchema } from "./configuration-result-records"; +import { dataInspectionResultSchema } from "./data-inventory-records"; import { dataWorkflowReceiptSchema } from "./job-evidence"; import { standaloneIntegrationOperationSchemas, @@ -961,87 +963,17 @@ const repositoryPending = z.strictObject({ previousCommit: gitCommitSchema.nullable(), recovery: repositoryRecoverySchema, }); -export const configurationApplyResultSchema = z.discriminatedUnion("outcome", [ - z.strictObject({ - outcome: z.literal("committed"), - revisionId: configurationGenerationRevisionSchema, - enabledServiceIds: z.array(serviceIdSchema).max(256), - followUp: z.enum(["complete", "failed"]), - cleanupFailed: z.boolean(), - }), - z.strictObject({ - outcome: z.literal("not_committed"), - reason: z.enum(["conflict", "busy", "failed"]), - currentRevisionId: configurationGenerationRevisionSchema.nullable().optional(), - cleanupFailed: z.boolean(), - }), -]); -export type ConfigurationApplyResult = z.infer; + +export { + type ConfigurationApplyResult, + configurationApplyResultSchema, +} from "./configuration-result-records"; + const revision = configurationApplyResultSchema; const data = z.strictObject({ completed: z.literal(true), resourceId: resourceIdSchema.optional(), }); -const nonnegativeSafeInteger = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER); -const boundedInventoryString = z.string().max(2_048); -const nullableInventoryString = boundedInventoryString.nullable(); -const motisGbfsSource = z.strictObject({ - sourceId: boundedInventoryString, - country: boundedInventoryString, - status: z.enum(["configured", "excluded"]), - observation: z.enum(["validated", "unknown"]), - errorClass: boundedInventoryString.optional(), - lastObservedSuccess: boundedInventoryString.optional(), - lastErrorAt: boundedInventoryString.optional(), - dataAge: z.literal("unknown"), -}); -const motisTransitousInventory = z.strictObject({ - configFound: z.boolean(), - datasetCount: nonnegativeSafeInteger, - realtimeFeedCount: nonnegativeSafeInteger, - gbfsFeedCount: nonnegativeSafeInteger, - feedProxyUrlCount: nonnegativeSafeInteger, - gbfsProxyUrl: nullableInventoryString, - feedProxyMode: z.enum(["none", "self-hosted", "transitous-cloud", "mixed"]), - feedProxyConfigFound: z.boolean(), - feedProxyVarsFound: z.boolean(), - feedProxyFeedCount: nonnegativeSafeInteger, - capabilityState: z.enum(["healthy", "stale", "missing", "error"]), - capabilityError: boundedInventoryString.optional(), - activeEpoch: nullableInventoryString, - candidateEpoch: nullableInventoryString, - testedAt: nullableInventoryString, - configHash: z - .string() - .regex(/^[0-9a-f]{64}$/) - .nullable(), - licenseHash: z - .string() - .regex(/^[0-9a-f]{64}$/) - .nullable(), - rentalProviderCount: nonnegativeSafeInteger, - rentalProviderGroupCount: nonnegativeSafeInteger, - rollbackAvailable: z.boolean(), - operationsProfile: z.enum(["regional-assisted", "regional-sovereign", "planet", "unknown"]), - activeSlot: z.enum(["A", "B"]).nullable(), - previousHealthySlot: z.enum(["A", "B"]).nullable(), - preflightState: z.enum(["passed", "blocked", "missing"]), - preflightRequiredDiskBytes: nonnegativeSafeInteger.nullable(), - preflightFreeDiskBytes: nonnegativeSafeInteger.nullable(), - pinProposalPending: z.boolean(), - crowdsourceState: z.literal("disabled-pending-review"), - gbfsCatalog: z.strictObject({ - state: z.enum(["active", "missing", "error"]), - commit: nullableInventoryString, - lockedAt: nullableInventoryString, - registryRows: nonnegativeSafeInteger, - registryAdded: nonnegativeSafeInteger, - transitousPreferred: nonnegativeSafeInteger, - quarantined: nonnegativeSafeInteger, - validationFailed: nonnegativeSafeInteger, - sources: z.array(motisGbfsSource).max(500), - }), -}); const systemCoreServiceId = z.enum([ "app-api", "app-web", @@ -1300,69 +1232,7 @@ export const OPS_RESULT_SCHEMAS = { "serviceConfig.apply": revision, "integrationConfig.apply": revision, "vault.apply": revision, - "data.inspect": z.strictObject({ - osm: z.strictObject({ - items: z - .array( - z.strictObject({ - artifactId: z.string().regex(/^[a-f0-9]{64}$/), - filename: z - .string() - .min(1) - .max(255) - .regex(/^[^/\\]+$/), - region: z.string().max(512).nullable(), - sizeBytes: nonnegativeSafeInteger, - modifiedAt: timestampSchema, - sourceFingerprint: z - .string() - .regex(/^[a-f0-9]{64}$/) - .nullable(), - fingerprintState: z.enum(["verified", "missing", "stale"]), - }), - ) - .max(500), - total: nonnegativeSafeInteger, - limit: z.number().int().min(1).max(500), - offset: nonnegativeSafeInteger, - unavailable: z.boolean(), - }), - builds: z - .array( - z.strictObject({ - target: z.enum([ - "valhalla", - "osrm", - "otp", - "motis", - "motisFeedProxy", - "tiles", - "pelias", - "nominatim", - "photon", - "overpass", - ]), - built: z.boolean(), - builtAt: timestampSchema.optional(), - readiness: z.enum(["ready", "missing", "incomplete", "stale", "unknown"]), - reason: boundedInventoryString.nullable(), - sizeBytes: nonnegativeSafeInteger.nullable(), - sourceFingerprints: z - .array( - z.strictObject({ - id: boundedInventoryString, - sha256: z.string().regex(/^[a-f0-9]{64}$/), - }), - ) - .max(10000), - buildRecordVersion: z.union([z.literal(1), z.literal(2)]).nullable(), - region: boundedInventoryString.nullable(), - runtimeImages: z.record(z.string().max(64), boundedInventoryString), - }), - ) - .max(10), - motisTransitous: motisTransitousInventory, - }), + "data.inspect": dataInspectionResultSchema, "data.downloadOsm": data, "data.downloadFonts": data, "data.update": data.extend({ dataWorkflow: dataWorkflowReceiptSchema.optional() }), diff --git a/packages/core/src/ops/data-inventory-records.ts b/packages/core/src/ops/data-inventory-records.ts new file mode 100644 index 000000000..206921c5b --- /dev/null +++ b/packages/core/src/ops/data-inventory-records.ts @@ -0,0 +1,128 @@ +/** Portable bounded inventory. Data only, never host authority. */ +import { z } from "zod"; + +const timestampSchema = z.string().datetime({ offset: true }); +const nonnegativeSafeInteger = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER); +const boundedInventoryString = z.string().max(2_048); +const nullableInventoryString = boundedInventoryString.nullable(); +const motisGbfsSource = z.strictObject({ + sourceId: boundedInventoryString, + country: boundedInventoryString, + status: z.enum(["configured", "excluded"]), + observation: z.enum(["validated", "unknown"]), + errorClass: boundedInventoryString.optional(), + lastObservedSuccess: boundedInventoryString.optional(), + lastErrorAt: boundedInventoryString.optional(), + dataAge: z.literal("unknown"), +}); +const motisTransitousInventory = z.strictObject({ + configFound: z.boolean(), + datasetCount: nonnegativeSafeInteger, + realtimeFeedCount: nonnegativeSafeInteger, + gbfsFeedCount: nonnegativeSafeInteger, + feedProxyUrlCount: nonnegativeSafeInteger, + gbfsProxyUrl: nullableInventoryString, + feedProxyMode: z.enum(["none", "self-hosted", "transitous-cloud", "mixed"]), + feedProxyConfigFound: z.boolean(), + feedProxyVarsFound: z.boolean(), + feedProxyFeedCount: nonnegativeSafeInteger, + capabilityState: z.enum(["healthy", "stale", "missing", "error"]), + capabilityError: boundedInventoryString.optional(), + activeEpoch: nullableInventoryString, + candidateEpoch: nullableInventoryString, + testedAt: nullableInventoryString, + configHash: z + .string() + .regex(/^[0-9a-f]{64}$/) + .nullable(), + licenseHash: z + .string() + .regex(/^[0-9a-f]{64}$/) + .nullable(), + rentalProviderCount: nonnegativeSafeInteger, + rentalProviderGroupCount: nonnegativeSafeInteger, + rollbackAvailable: z.boolean(), + operationsProfile: z.enum(["regional-assisted", "regional-sovereign", "planet", "unknown"]), + activeSlot: z.enum(["A", "B"]).nullable(), + previousHealthySlot: z.enum(["A", "B"]).nullable(), + preflightState: z.enum(["passed", "blocked", "missing"]), + preflightRequiredDiskBytes: nonnegativeSafeInteger.nullable(), + preflightFreeDiskBytes: nonnegativeSafeInteger.nullable(), + pinProposalPending: z.boolean(), + crowdsourceState: z.literal("disabled-pending-review"), + gbfsCatalog: z.strictObject({ + state: z.enum(["active", "missing", "error"]), + commit: nullableInventoryString, + lockedAt: nullableInventoryString, + registryRows: nonnegativeSafeInteger, + registryAdded: nonnegativeSafeInteger, + transitousPreferred: nonnegativeSafeInteger, + quarantined: nonnegativeSafeInteger, + validationFailed: nonnegativeSafeInteger, + sources: z.array(motisGbfsSource).max(500), + }), +}); + +export const dataInspectionResultSchema = z.strictObject({ + osm: z.strictObject({ + items: z + .array( + z.strictObject({ + artifactId: z.string().regex(/^[a-f0-9]{64}$/), + filename: z + .string() + .min(1) + .max(255) + .regex(/^[^/\\]+$/), + region: z.string().max(512).nullable(), + sizeBytes: nonnegativeSafeInteger, + modifiedAt: timestampSchema, + sourceFingerprint: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + fingerprintState: z.enum(["verified", "missing", "stale"]), + }), + ) + .max(500), + total: nonnegativeSafeInteger, + limit: z.number().int().min(1).max(500), + offset: nonnegativeSafeInteger, + unavailable: z.boolean(), + }), + builds: z + .array( + z.strictObject({ + target: z.enum([ + "valhalla", + "osrm", + "otp", + "motis", + "motisFeedProxy", + "tiles", + "pelias", + "nominatim", + "photon", + "overpass", + ]), + built: z.boolean(), + builtAt: timestampSchema.optional(), + readiness: z.enum(["ready", "missing", "incomplete", "stale", "unknown"]), + reason: boundedInventoryString.nullable(), + sizeBytes: nonnegativeSafeInteger.nullable(), + sourceFingerprints: z + .array( + z.strictObject({ + id: boundedInventoryString, + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }), + ) + .max(10000), + buildRecordVersion: z.union([z.literal(1), z.literal(2)]).nullable(), + region: boundedInventoryString.nullable(), + runtimeImages: z.record(z.string().max(64), boundedInventoryString), + }), + ) + .max(10), + motisTransitous: motisTransitousInventory, +}); diff --git a/packages/core/src/services/__tests__/configuration-generation.test.ts b/packages/core/src/services/__tests__/configuration-generation.test.ts index cfc112178..fba582859 100644 --- a/packages/core/src/services/__tests__/configuration-generation.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation.test.ts @@ -386,3 +386,40 @@ it("keeps host-owned Compose interpolation equivalent to legacy generations", () }); expect(after.rendered.composeYaml).toBe(before.rendered.composeYaml); }); + +it("applied selector and retention follow the current generation and explicit return to host", async () => { + const { infraDir } = checkout(); + const { readAppliedPlatformValues, readAppliedReleaseSelection } = await import( + "../configuration-generation" + ); + const env = { OPENMAPX_RELEASE: "main", BACKUP_RETENTION_DAYS: "30" }; + await commitConfigurationGeneration({ + infraDir, + services, + input: input({ + platform: { + version: 1, + releaseSelector: { source: "database", value: "preview" }, + backupRetentionDays: { source: "database", value: 60 }, + }, + }), + }); + expect(readAppliedReleaseSelection(infraDir, env)).toMatchObject({ kind: "preview" }); + expect(readAppliedPlatformValues(infraDir, env).backupRetentionDays).toBe(60); + await commitConfigurationGeneration({ + infraDir, + services, + input: input({ + platform: { + version: 1, + releaseSelector: { source: "host" }, + backupRetentionDays: { source: "host" }, + }, + }), + }); + expect(readAppliedReleaseSelection(infraDir, env)).toMatchObject({ kind: "main" }); + expect( + readAppliedPlatformValues(infraDir, { ...env, BACKUP_RETENTION_DAYS: "45" }) + .backupRetentionDays, + ).toBe(45); +}); diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index e249fff31..4a787884a 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -57,6 +57,7 @@ import { platformConfigurationSchema, resolvePlatformConfiguration, } from "./platform-configuration"; +import { normalizeReleaseSelection } from "./release-selection"; import { buildAppApiServiceEnv, expandServiceSelection } from "./selection"; import { renderTraefikDynamicConfiguration, renderTraefikDynamicYaml } from "./traefik-renderer"; import type { LoadedService } from "./types"; @@ -756,6 +757,25 @@ export function readAppliedPlatformConfiguration( : platformConfigurationSchema.parse(resolved.platform); } +/** Resolve each invocation against the currently applied non-secret projection. */ +export function readAppliedPlatformValues( + infraDir: string, + env: Readonly> = process.env, +) { + return resolvePlatformConfiguration({ saved: readAppliedPlatformConfiguration(infraDir), env }) + .values; +} + +export function readAppliedReleaseSelection( + infraDir: string, + env: Readonly> = process.env, +) { + return normalizeReleaseSelection( + readAppliedPlatformValues(infraDir, env).releaseSelector, + env.OPENMAPX_RELEASE_MANIFEST_REPOSITORY, + ); +} + /** Call under writer ownership so the pointer and its projections are one observation. * Only the resolved integration file is read: service secrets never enter this result. */ diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 6d7ce791f..4cc6d7d41 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -42,6 +42,8 @@ export { type RenderConfigurationOptions, readAppliedConfiguration, readAppliedPlatformConfiguration, + readAppliedPlatformValues, + readAppliedReleaseSelection, readAppliedServiceIds, readAppliedTransitCredentialsRevision, renderConfiguration, From f2264698ab4b37d84d7d6f8ac619eb5f493fa535 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 14:38:07 +0100 Subject: [PATCH 088/135] feat(admin): add service presets discovery filters and complete build options --- apps/api/openapi.json | 610 +++++++++++++++++- .../__tests__/admin-services-sdk.test.ts | 42 +- .../routes/__tests__/admin-services.test.ts | 67 +- apps/api/src/routes/admin-services.ts | 65 +- .../__tests__/admin-job-handlers.test.ts | 20 + .../__tests__/admin-operation-catalog.test.ts | 7 + .../__tests__/service-inventory.test.ts | 15 + apps/api/src/services/admin-job-handlers.ts | 5 + apps/api/src/services/service-inventory.ts | 13 + .../src/administrative-runtime.test.ts | 29 + apps/ops-agent/src/administrative-runtime.ts | 23 +- apps/ops-agent/src/docker-runtime.test.ts | 32 + apps/ops-agent/src/docker-runtime.ts | 14 +- .../services/ServiceAdministration.test.tsx | 268 +++++++- .../admin/services/ServiceCatalog.tsx | 218 ++++++- .../administration/services-administration.md | 21 +- packages/cli/__tests__/cli-compose.test.ts | 17 + packages/cli/src/commands/compose.ts | 8 +- packages/cli/src/lib/presets.ts | 63 +- packages/cli/src/lib/release.ts | 18 +- .../src/api/contracts/service-admin/index.ts | 11 + .../api/contracts/service-admin/operations.ts | 3 + .../api/contracts/service-admin/services.ts | 78 ++- .../admin/operation-bindings.generated.ts | 7 + .../admin/service-admin.generated.ts | 91 +++ .../generated/admin/service-models/index.ts | 12 + .../listAdminOperations200OperationsItemId.ts | 3 + .../listAdminServicePresets200.ts | 8 + .../listAdminServicePresets200PresetsItem.ts | 22 + .../listAdminServicePresets400.ts | 20 + .../listAdminServicePresets401.ts | 20 + .../listAdminServicePresets403.ts | 20 + .../listAdminServicePresets413.ts | 20 + .../listAdminServicePresets415.ts | 20 + .../listAdminServicePresets429.ts | 11 + .../listAdminServicePresets500.ts | 20 + .../listAdminServicePresets503.ts | 20 + .../service-models/listAdminServices200.ts | 2 + ...listAdminServices200CapabilityInventory.ts | 13 + ...s200CapabilityInventoryUnrecognizedItem.ts | 12 + .../runAdminServicesBulkActionBody.ts | 21 +- packages/core/src/ops/contract.ts | 2 + .../configuration-generation.test.ts | 18 + .../src/services/__tests__/presets.test.ts | 31 + .../src/services/configuration-generation.ts | 26 +- packages/core/src/services/index.ts | 2 + packages/core/src/services/presets.ts | 59 ++ 47 files changed, 1993 insertions(+), 134 deletions(-) create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets200.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets200PresetsItem.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets400.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets401.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets403.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets413.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets415.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets429.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets500.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServicePresets503.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200CapabilityInventory.ts create mode 100644 packages/core/src/api/generated/admin/service-models/listAdminServices200CapabilityInventoryUnrecognizedItem.ts create mode 100644 packages/core/src/services/__tests__/presets.test.ts create mode 100644 packages/core/src/services/presets.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 097f2e5cb..e93504803 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -46982,6 +46982,9 @@ "convert-overpass", "link", "clean", + "overture-pull", + "overture-ingest", + "overture-extract", "overture-sync", "overture-conflate", "search-index-build", @@ -54334,6 +54337,50 @@ "additionalProperties": false } }, + "capabilityInventory": { + "type": "object", + "properties": { + "recognized": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "maxLength": 256 + } + }, + "unrecognized": { + "maxItems": 4096, + "type": "array", + "items": { + "type": "object", + "properties": { + "capability": { + "type": "string", + "maxLength": 256 + }, + "serviceIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "maxLength": 64 + } + } + }, + "required": [ + "capability", + "serviceIds" + ], + "additionalProperties": false + } + } + }, + "required": [ + "recognized", + "unrecognized" + ], + "additionalProperties": false + }, "summary": { "type": "object", "properties": { @@ -54370,6 +54417,7 @@ "required": [ "acquisition", "services", + "capabilityInventory", "summary" ], "additionalProperties": false @@ -56935,20 +56983,27 @@ ] }, "serviceIds": { - "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } }, + "presetIds": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, "continueOnError": { "type": "boolean" } }, "required": [ - "action", - "serviceIds" + "action" ], "additionalProperties": false }, @@ -56962,13 +57017,21 @@ ] }, "serviceIds": { - "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } }, + "presetIds": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, "noDeps": { "type": "boolean" }, @@ -56977,8 +57040,7 @@ } }, "required": [ - "action", - "serviceIds" + "action" ], "additionalProperties": false }, @@ -56998,6 +57060,15 @@ "minLength": 1 } }, + "presetIds": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, "all": { "type": "boolean" }, @@ -57006,6 +57077,9 @@ }, "continueOnError": { "type": "boolean" + }, + "motisImport": { + "type": "boolean" } }, "required": [ @@ -59030,6 +59104,530 @@ "x-openmapx-auth": "admin" } }, + "/api/admin/services/presets": { + "get": { + "operationId": "listAdminServicePresets", + "responses": { + "200": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "presets": { + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "maxLength": 64 + }, + "rootIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "maxLength": 64 + } + }, + "effectiveIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "maxLength": 64 + } + }, + "missingIds": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "maxLength": 64 + } + } + }, + "required": [ + "id", + "rootIds", + "effectiveIds", + "missingIds" + ], + "additionalProperties": false + } + } + }, + "required": [ + "presets" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "401": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "403": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "413": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "415": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "429": { + "description": "Response", + "headers": { + "retry-after": { + "schema": { + "type": "string", + "pattern": "^\\d+$" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "retryAfter": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "error", + "retryAfter" + ], + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "503": { + "description": "Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "ok": { + "type": "boolean", + "enum": [ + false + ] + }, + "message": { + "type": "string" + }, + "jobId": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "statusCode": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "statusCode", + "error", + "message" + ], + "additionalProperties": false + } + ] + } + } + } + } + }, + "tags": [ + "service-admin" + ], + "x-openmapx-auth": "admin" + } + }, "/api/admin/services/selection": { "get": { "operationId": "getAdminServiceSelection", diff --git a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts index 0133f678f..deeea35d2 100644 --- a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts @@ -1,5 +1,9 @@ import { createApiClient } from "@openmapx/core"; -import { serviceAdminOperations } from "@openmapx/core/api-contracts"; +import { + artifactTransferAdminOperations, + platformAdminOperations, + serviceAdminOperations, +} from "@openmapx/core/api-contracts"; import { type OpsOperation, type OpsOperationKind, @@ -122,6 +126,16 @@ vi.mock("../../services/service-config-resolver", () => ({ API_KEY: { value: "synthetic-secret", source: "env" }, }), })); +vi.mock("../../services/configuration-ownership-view", async (original) => ({ + ...(await original()), + inspectServiceConfiguration: async () => ({ + resolved: { + count: { value: 2, source: "default" }, + API_KEY: { value: "synthetic-secret", source: "env" }, + }, + ownership: {}, + }), +})); const setSecret = vi.fn(); const deleteSecret = vi.fn(); vi.mock("../../services/service-secrets", () => ({ @@ -249,6 +263,12 @@ const execute = vi.fn( kind: operation.kind, value: { services: [{ serviceId: "motis", state: "running" }] }, }; + if (operation.kind === "configuration.inspect") + return { + execution: "sync", + kind: operation.kind, + value: { revisionId: null, integrationConfigDigests: [] }, + }; if (operation.kind === "data.inspect") return { execution: "sync", kind: operation.kind, value: serviceDataFixture }; if (operation.kind === "backup.list") @@ -331,6 +351,7 @@ afterEach(() => { }); async function wire(max = 1000) { const app = Fastify({ bodyLimit: 8192 }); + const limiter = new RateLimiter({ max, windowMs: 60000 }); const unexpected = async () => { throw Error("Wrong tier"); @@ -536,13 +557,21 @@ describe("actual service producers through final exported SDK", () => { } }); - it("exports all 30 actual registered methods", async () => { + it("exports every registered service-family method", async () => { const { registered, close } = await wire(); try { - expect([...registered.keys()].sort()).toEqual(Object.keys(serviceAdminOperations).sort()); + const dedicated = new Set([ + ...Object.keys(platformAdminOperations), + ...Object.keys(artifactTransferAdminOperations), + ]); + expect([...registered.keys()].sort()).toEqual( + Object.keys(serviceAdminOperations) + .filter((id) => !dedicated.has(id)) + .sort(), + ); for (const [id, contract] of Object.entries(serviceAdminOperations)) { expect(sdk).toHaveProperty(id, expect.any(Function)); - expect(registered.get(id)).toBe(contract.path); + if (!dedicated.has(id)) expect(registered.get(id)).toBe(contract.path); } } finally { await close(); @@ -586,7 +615,7 @@ describe("actual service producers through final exported SDK", () => { credentials: [ { key: "API_KEY", - source: "vault", + source: "env", updatedAt: "2026-10-21T00:00:00.000Z", updatedBy: null, setup: { url: "https://example.test/setup" }, @@ -678,7 +707,8 @@ describe("actual service producers through final exported SDK", () => { () => sdk.rollbackAdminSharedMobility("bike", { enabled: false }, options), () => sdk.getAdminDawarich(options), ]; - for (const call of calls) expect((await call()).status).toBe(200); + for (const [index, call] of calls.entries()) + expect((await call()).status, `call ${index}`).toBe(200); serviceActionLimit.reset(); expect((await sdk.provisionAdminDawarich(undefined, options)).status).toBe(200); serviceActionLimit.reset(); diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index e4fca374e..4276248a7 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -126,8 +126,10 @@ vi.mock("../../services/trusted-config-operations.js", () => ({ })); // @openmapx/core/server — getProvidedCapabilityNames + serviceConfigEnvPrefix -vi.mock("@openmapx/core/server", () => ({ +vi.mock("@openmapx/core/server", async (original) => ({ + ...(await original()), services: { + ...(await original()).services, getProvidedCapabilityNames: vi.fn().mockReturnValue([]), serviceConfigEnvPrefix: vi.fn().mockReturnValue("SERVICE_ID"), }, @@ -1118,3 +1120,66 @@ describe("resource reset transport", () => { ); }); }); + +describe("service presets and option boundaries", () => { + it("reports exact roots, effective dependencies and missing preset members", async () => { + mockRegistryList.mockReturnValueOnce([ + { + ...MOCK_SERVICE_RUNNING, + manifest: { + ...MOCK_SERVICE_RUNNING.manifest, + container: { + ...MOCK_SERVICE_RUNNING.manifest.container, + dependsOn: [{ service: "postgis", condition: "service_started" }], + }, + }, + }, + { ...MOCK_SERVICE_STOPPED, manifest: { ...MOCK_SERVICE_STOPPED.manifest, id: "postgis" } }, + ]); + const result = await app.inject({ url: "/admin/services/presets" }); + expect(result.statusCode).toBe(200); + expect(result.json().presets.find((preset: { id: string }) => preset.id === "transit")).toEqual( + { + id: "transit", + rootIds: ["motis", "motis-feed-proxy", "otp"], + effectiveIds: ["motis", "postgis"], + missingIds: ["motis-feed-proxy", "otp"], + }, + ); + }); + it("expands and deduplicates preset roots server-side while preserving update options", async () => { + mockRegistryGet.mockImplementation((id: string) => + ["osrm", "valhalla"].includes(id) ? MOCK_SERVICE_STOPPED : null, + ); + const result = await app.inject({ + method: "POST", + url: "/admin/services/bulk-action", + payload: { action: "update", serviceIds: ["osrm"], presetIds: ["routing"], noDeps: true }, + }); + expect(result.statusCode).toBe(200); + expect(mockJobRunnerEnqueue).toHaveBeenCalledWith( + "service.bulk", + expect.objectContaining({ action: "update", serviceIds: ["osrm", "valhalla"], noDeps: true }), + expect.any(String), + ); + }); + it.each(["unknown", "__proto__"])("rejects unknown preset %s before enqueue", async (id) => { + const result = await app.inject({ + method: "POST", + url: "/admin/services/bulk-action", + payload: { action: "start", presetIds: [id] }, + }); + expect(result.statusCode).toBe(400); + expect(mockJobRunnerEnqueue).not.toHaveBeenCalled(); + }); + it("rejects MOTIS import without a MOTIS build before enqueue", async () => { + mockRegistryGet.mockReturnValue(MOCK_SERVICE_STOPPED); + const result = await app.inject({ + method: "POST", + url: "/admin/services/bulk-action", + payload: { action: "build", serviceIds: ["valhalla"], motisImport: true }, + }); + expect(result.statusCode).toBe(400); + expect(mockJobRunnerEnqueue).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index 9d10831b6..6768abd6c 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -128,6 +128,25 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { return readServiceInventory(); }); + registerServiceAdminRoute(app, "listAdminServicePresets", async (_req, reply) => { + try { + const loaded = getServiceRegistry().list(); + return { + presets: Object.entries(coreServices.PRESETS).map(([id, rootIds]) => { + const selection = coreServices.expandServiceSelection(loaded, rootIds); + return { + id, + rootIds: [...rootIds], + effectiveIds: [...selection.enabledIds], + missingIds: selection.missingIds, + }; + }), + }; + } catch { + return reply.code(503).send({ error: "Service registry not available" }); + } + }); + // GET /admin/services/:id — single service detail registerServiceAdminRoute(app, "getAdminService", async (req, reply) => { let svc: ReturnType["get"]>; @@ -297,16 +316,53 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { } const allowedKeys = action === "build" - ? new Set(["action", "serviceIds", "all", "region", "continueOnError"]) + ? new Set([ + "action", + "serviceIds", + "all", + "region", + "continueOnError", + "presetIds", + "motisImport", + ]) : action === "update" - ? new Set(["action", "serviceIds", "continueOnError", "noDeps"]) - : new Set(["action", "serviceIds", "continueOnError"]); + ? new Set(["action", "serviceIds", "continueOnError", "noDeps", "presetIds"]) + : new Set(["action", "serviceIds", "continueOnError", "presetIds"]); if (!hasOnlyBodyKeys(req.body, allowedKeys)) { reply.status(400); return { error: "Request contains unsupported fields" }; } - const serviceIds = [...new Set(req.body?.serviceIds ?? [])]; + let serviceIds: string[]; + try { + serviceIds = [ + ...new Set([ + ...(req.body.serviceIds ?? []), + ...coreServices.expandPresets(req.body.presetIds ?? []), + ]), + ]; + const registry = getServiceRegistry(); + if (serviceIds.some((id) => !registry.get(id))) + return reply + .code(400) + .send({ error: "Selected preset or service contains an uninstalled service" }); + if (action === "build" && "motisImport" in req.body && req.body.motisImport) { + const motis = registry.get("motis"); + if ( + !motis?.enabled || + !motis.isBuiltIn || + !motis.manifest.buildCommand || + !(serviceIds.includes("motis") || ("all" in req.body && req.body.all === true)) + ) + return reply + .code(400) + .send({ error: "MOTIS import requires a build including enabled built-in MOTIS" }); + } + } catch (error) { + if (error instanceof coreServices.UnknownPresetError) + return reply.code(400).send({ error: "Unknown service preset" }); + return reply.code(503).send({ error: "Service registry not available" }); + } if (action !== "build" && serviceIds.length === 0) { reply.status(400); return { error: `Action "${action}" requires one or more serviceIds` }; @@ -331,6 +387,7 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { region, continueOnError: req.body?.continueOnError, ...("noDeps" in req.body ? { noDeps: req.body.noDeps } : {}), + ...("motisImport" in req.body ? { motisImport: req.body.motisImport } : {}), }, adminSession.user.id, ); diff --git a/apps/api/src/services/__tests__/admin-job-handlers.test.ts b/apps/api/src/services/__tests__/admin-job-handlers.test.ts index 9ea55daa4..96fa2e934 100644 --- a/apps/api/src/services/__tests__/admin-job-handlers.test.ts +++ b/apps/api/src/services/__tests__/admin-job-handlers.test.ts @@ -41,6 +41,7 @@ vi.mock("../service-registry", () => ({ list: () => [ { manifest: { id: "valhalla" } }, { manifest: { id: "osrm" } }, + { manifest: { id: "motis" } }, { manifest: { id: "app-api" } }, ], }), @@ -565,6 +566,25 @@ describe("admin job handlers", () => { expect(runCliMock).not.toHaveBeenCalled(); }); + it("carries MOTIS import only into the fixed MOTIS child", async () => { + runAdminOperationMock.mockReset().mockResolvedValue({ completed: true }); + const ctx = { + jobId: "job", + payload: { action: "build", serviceIds: ["motis"], motisImport: true }, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn(), + }; + await handleServiceBulkJob(ctx); + expect(runAdminOperationMock).toHaveBeenCalledWith( + ctx, + { kind: "service.build", serviceId: "motis", motisImport: true }, + "admin-job.service.build", + { durableIdentity: "motis" }, + ); + }); + it("uses the bounded build-all variant and typed region options", async () => { runAdminOperationMock.mockClear(); runAdminOperationMock.mockResolvedValueOnce({ diff --git a/apps/api/src/services/__tests__/admin-operation-catalog.test.ts b/apps/api/src/services/__tests__/admin-operation-catalog.test.ts index 64606082c..62c1968e3 100644 --- a/apps/api/src/services/__tests__/admin-operation-catalog.test.ts +++ b/apps/api/src/services/__tests__/admin-operation-catalog.test.ts @@ -241,3 +241,10 @@ describe("guarded Overture recovery catalogue", () => { it("keeps remote credential-template generation retired", () => { expect(getAdminOperation("generate-api-keys")).toBeUndefined(); }); + +it("serializes every installed operation through the actual public catalogue schema", async () => { + const { adminOperationsSchema } = await import("@openmapx/core/api-contracts"); + expect(() => + adminOperationsSchema.parse({ operations: describeAdminOperationCatalog() }), + ).not.toThrow(); +}); diff --git a/apps/api/src/services/__tests__/service-inventory.test.ts b/apps/api/src/services/__tests__/service-inventory.test.ts index 0c47d20f4..db547cfea 100644 --- a/apps/api/src/services/__tests__/service-inventory.test.ts +++ b/apps/api/src/services/__tests__/service-inventory.test.ts @@ -50,3 +50,18 @@ it("reports configured deployment intent independently of live agent acquisition expect(hostControlConfigured()).toBe(true); expect(isSelfHostedDeployment()).toBe(true); }); + +it("separates known vocabulary from every custom capability's owning service", async () => { + const a = serviceFixture("motis"), + b = serviceFixture("valhalla"); + a.manifest.provides = ["routing-engine", "vendor/custom"]; + b.manifest.provides = ["vendor/custom", "misspelled"]; + list.mockReturnValue([a, b]); + inspect.mockResolvedValue({ services: [] }); + const inventory = (await readServiceInventory()).capabilityInventory; + expect(inventory.recognized).toContain("routing-engine"); + expect(inventory.unrecognized).toEqual([ + { capability: "misspelled", serviceIds: ["valhalla"] }, + { capability: "vendor/custom", serviceIds: ["motis", "valhalla"] }, + ]); +}); diff --git a/apps/api/src/services/admin-job-handlers.ts b/apps/api/src/services/admin-job-handlers.ts index 9fc8c21be..ab64b0fb1 100644 --- a/apps/api/src/services/admin-job-handlers.ts +++ b/apps/api/src/services/admin-job-handlers.ts @@ -133,6 +133,7 @@ export async function handleServiceBulkJob(ctx: JobContext): Promise { ? { state: "available", reason: null } : { state: "unavailable", reason: "ops_unavailable" }, services, + capabilityInventory: { + recognized: [...coreServices.WELL_KNOWN_CAPABILITIES].sort(), + unrecognized: [...new Set(services.flatMap((service) => service.provides))] + .filter((capability) => !coreServices.WELL_KNOWN_CAPABILITIES.has(capability)) + .sort() + .map((capability) => ({ + capability, + serviceIds: services + .filter((service) => service.provides.includes(capability)) + .map((service) => service.id) + .sort(), + })), + }, summary: { running: services.filter((s) => s.status === "running").length, stopped: services.filter((s) => s.status !== "running" && s.status !== "unknown").length, diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index b06f73a21..c4292f764 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -528,6 +528,35 @@ describe("administrative build runtime", () => { expect(calls).toEqual([["services", "build", "valhalla", "--region", "europe/germany"]]); }); + it("passes MOTIS import only to the actual MOTIS build and refuses unsupported selections before effects", async () => { + const calls: string[][] = [], + runtime = createUnavailableRuntime(); + createAdministrativeRuntime(runtime, { + rootDir: temporaryRoot(), + loadBuildAuthority: async () => [ + ...buildAuthority, + { serviceId: "motis", enabled: true, isBuiltIn: true, buildCommand: "motis" }, + ], + runFixedCli: async (args) => { + calls.push([...args]); + }, + }); + await dispatchOpsOperation( + runtime, + { kind: "service.build", serviceId: "motis", motisImport: true }, + context("service.build"), + ); + expect(calls).toEqual([["services", "build", "motis", "--import"]]); + await expect( + dispatchOpsOperation( + runtime, + { kind: "service.build", serviceId: "valhalla", motisImport: true }, + context("service.build"), + ), + ).rejects.toThrow(/MOTIS/); + expect(calls).toHaveLength(1); + }); + it("derives build-all IDs inside the agent and preserves continue semantics", async () => { const calls: string[][] = []; const runtime = createUnavailableRuntime(); diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 36a0e6369..9f7792d98 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -1011,6 +1011,7 @@ export function createDefaultReleaseEffects( ); }; const inspectRelease = async (): Promise> => { + const configured = coreServices.readAppliedReleaseSelection(join(rootDir, "infra", "docker")); const state = readState(); const latest = readReleasePointer(paths.latest, true); const candidates = manifestNames(); @@ -1020,8 +1021,7 @@ export function createDefaultReleaseEffects( const available = availableReleaseId ? loadManifest(availableReleaseId) : undefined; const receipt = availableReleaseId ? selectionReceipts()[availableReleaseId] : undefined; return { - configuredSelector: coreServices.readAppliedPlatformValues(join(rootDir, "infra", "docker")) - .releaseSelector, + configuredSelector: configured.kind === "local" ? "local" : configured.selector, ...(state.releaseId ? { currentReleaseId: state.releaseId } : {}), ...(current?.product ? { currentVersion: current.product.version } : {}), ...(availableReleaseId ? { availableReleaseId } : {}), @@ -1744,11 +1744,12 @@ export function createAdministrativeRuntime( const releaseEffects = options.releaseEffects ?? createDefaultReleaseEffects(options.rootDir, runFixedCli); const loadBuildAuthority = options.loadBuildAuthority ?? (async () => []); - const buildArgs = (serviceId: string, regionId?: string): string[] => [ + const buildArgs = (serviceId: string, regionId?: string, motisImport = false): string[] => [ "services", "build", serviceId, ...(regionId ? ["--region", regionId] : []), + ...(motisImport && serviceId === "motis" ? ["--import"] : []), ]; const loadBuildable = async (): Promise => { const authority = [...(await loadBuildAuthority())]; @@ -1771,18 +1772,26 @@ export function createAdministrativeRuntime( .sort((left, right) => compareServiceBuildOrder(left.serviceId, right.serviceId)); }; runtime["service.build"] = async (operation, context) => { + if (operation.motisImport && operation.serviceId !== "motis") + throw Error("MOTIS import requires a MOTIS build"); const buildable = await loadBuildable(); if (!buildable.some((service) => service.serviceId === operation.serviceId)) { throw new Error("Build authority rejected"); } await runFixedCli( - buildArgs(operation.serviceId, operation.regionId ?? resolveDataDefaultRegion(runtime)), + buildArgs( + operation.serviceId, + operation.regionId ?? resolveDataDefaultRegion(runtime), + operation.motisImport, + ), context, ); return { completed: true }; }; runtime["services.buildAll"] = async (operation, context) => { const plan = await loadBuildable(); + if (operation.motisImport && !plan.some((service) => service.serviceId === "motis")) + throw Error("MOTIS import requires enabled MOTIS"); const serviceIds = plan.map((service) => service.serviceId); const completedServiceIds: string[] = []; const failedServiceIds: string[] = []; @@ -1796,7 +1805,11 @@ export function createAdministrativeRuntime( context.signal.throwIfAborted(); attempted = true; await runFixedCli( - buildArgs(service.serviceId, operation.regionId ?? resolveDataDefaultRegion(runtime)), + buildArgs( + service.serviceId, + operation.regionId ?? resolveDataDefaultRegion(runtime), + operation.motisImport, + ), context, ); completedServiceIds.push(service.serviceId); diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index 1857865e6..fe545f8d5 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -861,6 +861,38 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done 240_000, ); + it("names only the frozen generated closure when a release overlay declares disabled services", async () => { + const prepared = await preparedStack(), + calls: string[][] = []; + writeFileSync( + prepared.stack.composeReleasePath, + "services:\n app-api:\n image: example/api:1\n data-manager:\n image: example/data:1\n", + { mode: 0o600 }, + ); + const runtime = createDockerRuntime({ + stack: prepared.stack, + fileExists: existsSync, + ownedServices, + execFile: async (file, args, options) => { + calls.push([...args]); + if (args.includes("config")) return runContainedProcess(file, args, options); + return { stdout: "", stderr: "" }; + }, + }); + await dispatchOpsOperation( + runtime, + { kind: "stack.start", expectedConfigurationRevisionId: prepared.revisionId }, + context(), + ); + expect(calls.find((args) => args.includes("up"))?.slice(-5)).toEqual([ + "up", + "-d", + "motis", + "postgis", + "redis", + ]); + }); + it.each([ { kind: "stack.start", expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}` }, { diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 184df1ef8..5735512c4 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -591,7 +591,10 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { ), targets = operation.serviceId ? [operation.serviceId] - : Object.keys(configuration.services ?? {}); + : services.readAppliedComposeServiceIds( + options.stack.infraDir, + operation.expectedConfigurationRevisionId, + ); if ( services .composeCreationDependencyIds(configuration, targets) @@ -623,7 +626,14 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { } }; runtime["stack.start"] = (operation, context) => - preparedCreation(operation, context, () => [...composePrefix(), "up", "-d"]); + preparedCreation(operation, context, () => { + const ids = services.readAppliedComposeServiceIds( + options.stack.infraDir, + operation.expectedConfigurationRevisionId, + ); + if (ids.length === 0) throw Error("No enabled services to start"); + return [...composePrefix(), "up", "-d", ...ids]; + }); runtime["service.start"] = (operation, context) => preparedCreation(operation, context, () => [ ...composePrefix(), diff --git a/apps/web/src/components/admin/services/ServiceAdministration.test.tsx b/apps/web/src/components/admin/services/ServiceAdministration.test.tsx index a4247e636..b02de4f53 100644 --- a/apps/web/src/components/admin/services/ServiceAdministration.test.tsx +++ b/apps/web/src/components/admin/services/ServiceAdministration.test.tsx @@ -69,6 +69,17 @@ function setup( const response = handler?.(url, init); if (response) return response; const path = url.pathname; + if (path === "/api/admin/services/presets") + return Response.json({ + presets: [ + { + id: "routing", + rootIds: ["demo"], + effectiveIds: ["demo", "dependency"], + missingIds: [], + }, + ], + }); if (path.startsWith("/api/admin/jobs/")) return Response.json(job({ id: path.split("/").at(-1), status: "running" })); if (path.endsWith("/selection")) return Response.json(selection); @@ -96,6 +107,7 @@ function setup( if (path === "/api/admin/services") return Response.json({ acquisition, + capabilityInventory: { recognized: ["routing"], unrecognized: [] }, services: [ { id: "demo", @@ -490,13 +502,30 @@ describe("service administration visible behavior with the generated SDK", () => if (url.pathname === "/api/admin/integrations/demo/bindings") return Response.json({ bindings: bound ? [{ capability: "routing", serviceId: "demo" }] : [], + revision: "a".repeat(64), + desiredRevision: "a".repeat(64), + runtime: bound + ? { + generationId: "11111111-1111-4111-8111-111111111111", + bindings: { routing: "demo" }, + configRevision: "a".repeat(64), + } + : null, + lastApplication: null, }); if ( url.pathname === "/api/admin/integrations/demo/bindings/routing" && init.method === "POST" ) { bound = true; - return Response.json({ ok: true }); + return Response.json({ + operationId: "binding-test", + integrationId: "demo", + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "applied", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", + }); } return undefined; }); @@ -509,9 +538,11 @@ describe("service administration visible behavior with the generated SDK", () => fireEvent.mouseDown(screen.getByRole("combobox")); fireEvent.click(await screen.findByRole("option", { name: "Demo (demo)" })); await waitFor(() => expect(bound).toBe(true)); - expect(requests.find((r) => r.init.method === "POST")?.init.body).toBe( - JSON.stringify({ serviceId: "demo" }), - ); + expect(JSON.parse(String(requests.find((r) => r.init.method === "POST")?.init.body))).toEqual({ + serviceId: "demo", + expectedServiceId: null, + expectedRevision: "a".repeat(64), + }); expect(await screen.findByText("Demo (demo)")).toBeInTheDocument(); }); it("queues selected lifecycle and explicit all-build jobs, then refreshes catalog state independently", async () => { @@ -537,6 +568,7 @@ describe("service administration visible behavior with the generated SDK", () => if (url.pathname === "/api/admin/services" && init.method === "GET") return Response.json({ acquisition, + capabilityInventory: { recognized: ["routing"], unrecognized: [] }, services: [ { id: "demo", @@ -562,7 +594,7 @@ describe("service administration visible behavior with the generated SDK", () => }); const { qc } = mount(); await screen.findByText("Demo"); - fireEvent.click(screen.getAllByRole("checkbox")[0]); + fireEvent.click(screen.getByRole("checkbox", { name: "Select visible services" })); fireEvent.click(screen.getByRole("button", { name: "Start Selected" })); await waitFor(() => expect(toast).toHaveBeenCalledTimes(1)); fireEvent.click(screen.getByRole("button", { name: "Build All" })); @@ -678,6 +710,7 @@ it("disables a mixed bulk action and lists ineligible IDs", async () => { url.pathname === "/api/admin/services" ? Response.json({ acquisition, + capabilityInventory: { recognized: ["routing"], unrecognized: [] }, services: [ { id: "demo", @@ -710,8 +743,231 @@ it("disables a mixed bulk action and lists ineligible IDs", async () => { ); mount(); await screen.findByRole("link", { name: "Demo" }); - fireEvent.click(screen.getAllByRole("checkbox")[0]); + fireEvent.click(screen.getByRole("checkbox", { name: "Select visible services" })); const button = screen.getByRole("button", { name: "Restart Selected" }); expect(button).toBeDisabled(); expect(button).toHaveAttribute("title", expect.stringContaining("app-api")); }); + +it("selects server-defined preset roots for actions without persisting selection", async () => { + const requests = setup(); + mount(); + await screen.findByRole("link", { name: "Demo" }); + fireEvent.mouseDown(screen.getByRole("combobox", { name: "Preset (actions only)" })); + fireEvent.click(await screen.findByRole("option", { name: /routing \(demo\)/ })); + expect(screen.getByRole("checkbox", { name: "Select demo" })).toBeChecked(); + expect( + requests.some( + (request) => request.url.pathname.endsWith("/selection") && request.init.method === "POST", + ), + ).toBe(false); +}); +it("filters enabled state and exact capabilities while exposing custom vocabulary", async () => { + setup((url) => + url.pathname === "/api/admin/services" + ? Response.json({ + acquisition, + capabilityInventory: { + recognized: ["routing", "routing-engine"], + unrecognized: [{ capability: "vendor/custom", serviceIds: ["other"] }], + }, + services: [ + { + id: "demo", + name: "Demo", + version: "1", + quality: "built-in", + provides: ["routing-engine"], + consumes: [], + enabled: true, + isBuiltIn: true, + actions: availableActions, + status: "running", + }, + { + id: "other", + name: "Other", + version: "1", + quality: "built-in", + provides: ["routing", "vendor/custom"], + consumes: [], + enabled: false, + isBuiltIn: true, + actions: availableActions, + status: "not-running", + }, + ], + summary: { running: 1, stopped: 1, unknown: 0, total: 2 }, + }) + : undefined, + ); + mount(); + await screen.findByRole("link", { name: "Demo" }); + fireEvent.mouseDown(screen.getByRole("combobox", { name: "Exact capability" })); + fireEvent.click(await screen.findByRole("option", { name: "routing" })); + expect(screen.queryByRole("link", { name: "Demo" })).toBeNull(); + expect(screen.getByRole("link", { name: "Other" })).toBeInTheDocument(); + fireEvent.mouseDown(screen.getByRole("combobox", { name: "Enabled state" })); + fireEvent.click(await screen.findByRole("option", { name: "Enabled" })); + expect(screen.queryByRole("link", { name: "Other" })).toBeNull(); + expect(screen.getByText("Unrecognized capability vocabulary (1)")).toBeInTheDocument(); +}); +it("removes only an explicit root through the existing selection producer", async () => { + const requests = setup((url, init) => + url.pathname.endsWith("/selection") && init.method === "PUT" + ? Response.json({ + ok: true, + revisionId: "synthetic-revision", + summary: { ...selection, selectedRoots: [], requestedIds: [], effectiveIds: [] }, + }) + : undefined, + ); + mount(); + await screen.findByRole("link", { name: "Demo" }); + fireEvent.click(screen.getByRole("button", { name: "Remove demo root" })); + await waitFor(() => + expect( + requests.some( + (request) => request.url.pathname.endsWith("/selection") && request.init.method === "PUT", + ), + ).toBe(true), + ); + expect( + JSON.parse( + String( + requests.find( + (request) => request.url.pathname.endsWith("/selection") && request.init.method === "PUT", + )?.init.body, + ), + ), + ).toEqual({ selectedRoots: [] }); + await waitFor(() => expect(toast).toHaveBeenCalledWith("Service selection saved")); +}); + +it("sends update dependency options and MOTIS import only on their corresponding action", async () => { + const requests = setup((url, init) => + url.pathname === "/api/admin/services" + ? Response.json({ + acquisition, + capabilityInventory: { recognized: [], unrecognized: [] }, + services: [ + { + id: "motis", + name: "MOTIS", + version: "1", + quality: "built-in", + provides: [], + consumes: [], + enabled: true, + isBuiltIn: true, + actions: availableActions, + status: "not-running", + }, + ], + summary: { running: 0, stopped: 1, unknown: 0, total: 1 }, + }) + : url.pathname.endsWith("/bulk-action") && init.method === "POST" + ? Response.json({ ok: true, jobId: "job-1" }) + : undefined, + ); + mount(); + await screen.findByRole("link", { name: "MOTIS" }); + fireEvent.click(screen.getByRole("checkbox", { name: "Select motis" })); + fireEvent.click(screen.getByRole("checkbox", { name: "Update without dependency recreation" })); + fireEvent.click(screen.getByRole("button", { name: "Update selected" })); + await waitFor(() => + expect( + requests.filter((request) => request.url.pathname.endsWith("/bulk-action")), + ).toHaveLength(1), + ); + fireEvent.click(screen.getByRole("checkbox", { name: "Import MOTIS after build" })); + fireEvent.click(screen.getByRole("button", { name: "Build Selected" })); + await waitFor(() => + expect( + requests.filter((request) => request.url.pathname.endsWith("/bulk-action")), + ).toHaveLength(2), + ); + expect( + requests + .filter((request) => request.url.pathname.endsWith("/bulk-action")) + .map((request) => JSON.parse(String(request.init.body))), + ).toEqual([ + { action: "update", serviceIds: ["motis"], noDeps: true }, + { action: "build", serviceIds: ["motis"], continueOnError: true, motisImport: true }, + ]); +}); + +it.each(["file", "env"])( + "preserves required dependencies and %s-owned root selection", + async (source) => { + const requests = setup((url, init) => { + if (url.pathname.endsWith("/selection")) + return Response.json( + init.method === "PUT" + ? { ok: true, revisionId: "synthetic", summary: selection } + : { ...selection, source }, + ); + if (url.pathname === "/api/admin/services") + return Response.json({ + acquisition, + capabilityInventory: { recognized: [], unrecognized: [] }, + services: [ + { + id: "demo", + name: "Demo", + version: "1", + quality: "built-in", + provides: [], + consumes: [], + enabled: true, + isBuiltIn: true, + actions: availableActions, + status: "running", + }, + { + id: "dependency", + name: "Dependency", + version: "1", + quality: "built-in", + provides: [], + consumes: [], + enabled: true, + isBuiltIn: true, + actions: availableActions, + status: "running", + }, + { + id: "valhalla", + name: "Valhalla", + version: "1", + quality: "built-in", + provides: [], + consumes: [], + enabled: false, + isBuiltIn: true, + actions: availableActions, + status: "not-running", + }, + ], + summary: { running: 2, stopped: 1, unknown: 0, total: 3 }, + }); + return undefined; + }); + mount(); + await screen.findByRole("link", { name: "Dependency" }); + expect(screen.getByRole("button", { name: "Required dependency" })).toBeDisabled(); + const enable = screen.getByRole("button", { name: "Enable valhalla" }); + if (source === "env") { + expect(enable).toBeDisabled(); + expect(screen.getByRole("button", { name: "Remove demo root" })).toBeDisabled(); + } else { + fireEvent.click(enable); + await waitFor(() => + expect(requests.some((request) => request.init.method === "PUT")).toBe(true), + ); + expect( + JSON.parse(String(requests.find((request) => request.init.method === "PUT")?.init.body)), + ).toEqual({ selectedRoots: ["demo", "valhalla"] }); + } + }, +); diff --git a/apps/web/src/components/admin/services/ServiceCatalog.tsx b/apps/web/src/components/admin/services/ServiceCatalog.tsx index 474b54e47..50fc3ee9e 100644 --- a/apps/web/src/components/admin/services/ServiceCatalog.tsx +++ b/apps/web/src/components/admin/services/ServiceCatalog.tsx @@ -25,7 +25,9 @@ import TableRow from "@mui/material/TableRow"; import TextField from "@mui/material/TextField"; import Tooltip from "@mui/material/Tooltip"; import Typography from "@mui/material/Typography"; +import { listAdminServicePresets, unwrapAdminResponse } from "@openmapx/core/api-admin-sdk"; import type { AdminServiceSummary, AdminServicesBulkAction } from "@openmapx/core/api-contracts"; +import { useQuery } from "@tanstack/react-query"; import Link from "next/link"; import { useEffect, useMemo, useState } from "react"; import { @@ -138,10 +140,14 @@ function filterServices( services: AdminServiceSummary[], search: string, quality: QualityFilter, + enabled: "all" | "enabled" | "disabled", + capability: string, ): AdminServiceSummary[] { const q = search.trim().toLowerCase(); return services.filter((s) => { if (quality !== "all" && s.quality !== quality) return false; + if (enabled !== "all" && s.enabled !== (enabled === "enabled")) return false; + if (capability && !s.provides.includes(capability)) return false; if (!q) return true; return ( s.id.toLowerCase().includes(q) || @@ -165,7 +171,7 @@ export function ServiceCatalog() { } function ServiceCatalogView() { - const { client } = useActivityClient(); + const { client, apiUrl } = useActivityClient(); const owner = useAdminMutationOwner(client); const showToast = useAdminToast(); const { data, isLoading, isError, refetch } = useServicesList(); @@ -175,6 +181,16 @@ function ServiceCatalogView() { const [selectedIds, setSelectedIds] = useState>(new Set()); const [buildRegion, setBuildRegion] = useState(""); const [buildContinueOnError, setBuildContinueOnError] = useState(true); + const [enabledFilter, setEnabledFilter] = useState<"all" | "enabled" | "disabled">("all"); + const [capability, setCapability] = useState(""); + const [preset, setPreset] = useState(""); + const [noDeps, setNoDeps] = useState(false); + const [motisImport, setMotisImport] = useState(false); + const presets = useQuery({ + queryKey: ["admin", "service-presets", apiUrl], + queryFn: async () => unwrapAdminResponse(await listAdminServicePresets({ client })), + retry: false, + }); const selectionQuery = useServiceSelection(); @@ -188,8 +204,8 @@ function ServiceCatalogView() { const services = data?.services ?? emptyServices; const summary = data?.summary; const filtered = useMemo( - () => filterServices(services, search, quality), - [services, search, quality], + () => filterServices(services, search, quality, enabledFilter, capability), + [services, search, quality, enabledFilter, capability], ); const { paged, paginationProps } = useClientPagination(filtered); @@ -207,6 +223,7 @@ function ServiceCatalogView() { const allVisibleSelected = paged.length > 0 && paged.every((svc) => selectedIds.has(svc.id)); const toggleSelected = (id: string) => { + setPreset(""); setSelectedIds((prev) => { const next = new Set(prev); if (next.has(id)) next.delete(id); @@ -216,6 +233,7 @@ function ServiceCatalogView() { }; const toggleSelectVisible = () => { + setPreset(""); setSelectedIds((prev) => { const next = new Set(prev); if (allVisibleSelected) { @@ -252,16 +270,33 @@ function ServiceCatalogView() { showToast(error instanceof Error ? error.message : "Check failed", "error"); } } - async function saveSelection() { + async function persistRoots(roots: string[]) { const execution = owner; try { - await saveSelectionMutation.mutateAsync(parseRootInput(selectionDraft)); - if (execution.active) showToast("Service selection saved"); + const result = await saveSelectionMutation.mutateAsync(roots); + if (execution.active) + showToast( + "jobId" in result + ? `Queued service selection (${result.jobId})` + : "Service selection saved", + ); } catch (error) { if (execution.active) showToast(error instanceof Error ? error.message : "Save failed", "error"); } } + const saveSelection = () => persistRoots(parseRootInput(selectionDraft)); + const selectPreset = (id: string) => { + setPreset(id); + const choice = presets.data?.presets.find((preset) => preset.id === id); + if (!choice) return; + if (choice.missingIds.length) { + setPreset(""); + showToast(`Preset is missing: ${choice.missingIds.join(", ")}`, "warning"); + return; + } + setSelectedIds(new Set(choice.rootIds)); + }; async function queueBulk(payload: AdminServicesBulkAction) { const execution = owner; try { @@ -285,16 +320,23 @@ function ServiceCatalogView() { } if (action === "build") { + if (motisImport && !ids.includes("motis")) { + showToast("MOTIS import requires selecting MOTIS", "warning"); + return; + } void queueBulk({ action, serviceIds: ids, region: buildRegion.trim() || undefined, continueOnError: buildContinueOnError, + ...(motisImport ? { motisImport: true } : {}), }); return; } - void queueBulk({ action, serviceIds: ids }); + void queueBulk( + action === "update" ? { action, serviceIds: ids, noDeps } : { action, serviceIds: ids }, + ); }; const queueBuildAll = () => { @@ -303,6 +345,7 @@ function ServiceCatalogView() { all: true, region: buildRegion.trim() || undefined, continueOnError: buildContinueOnError, + ...(motisImport ? { motisImport: true } : {}), }); }; @@ -320,6 +363,34 @@ function ServiceCatalogView() { )} + + Action presets + + Choose root services for the action checkboxes. This does not save service selection; + dependency expansion and action availability still apply. + + selectPreset(e.target.value)} + size="small" + sx={{ mt: 1, minWidth: 240 }} + > + Custom selection + {(presets.data?.presets ?? []).map((item) => ( + + {item.id} ({item.rootIds.join(", ")}) + {item.missingIds.length ? " — missing services" : ""} + + ))} + + {presets.isError && ( + + Presets are unavailable; service selection remains unchanged. + + )} + Manage persisted service roots from service-selection.json. Effective - services include dependencies of selected roots. + services include dependencies of selected roots. Removing a root can keep it enabled as + another root’s dependency. {selectionQuery.data?.source === "env" && selectionQuery.data.envVarValue && ( @@ -496,6 +568,26 @@ function ServiceCatalogView() { flexWrap: "wrap", }} > + setNoDeps(e.target.checked)} + disabled={bulkMutation.isPending} + /> + } + label="Update without dependency recreation" + /> + setMotisImport(e.target.checked)} + disabled={bulkMutation.isPending} + /> + } + label="Import MOTIS after build" + /> @@ -595,6 +699,8 @@ function ServiceCatalogView() { direction="row" sx={{ alignItems: "center", + flexWrap: "wrap", + minWidth: 0, gap: 1, }} > @@ -612,6 +718,7 @@ function ServiceCatalogView() { direction="row" sx={{ gap: 0.5, + flexWrap: "wrap", }} > {isLoading && } + {!!data?.capabilityInventory?.unrecognized.length && ( + +
+ + Unrecognized capability vocabulary ({data.capabilityInventory.unrecognized.length}) + + + Custom vendor names are allowed. Check non-namespaced entries for spelling errors. + + {data.capabilityInventory.unrecognized.map((item) => ( + + {item.capability}: {item.serviceIds.join(", ")} + + ))} +
+ + )} + setEnabledFilter(e.target.value as typeof enabledFilter)} + sx={{ minWidth: 160 }} + > + All services + Enabled + Disabled + + setCapability(e.target.value)} + sx={{ minWidth: 200 }} + > + All capabilities + {[ + ...new Set([ + ...(data?.capabilityInventory?.recognized ?? []), + ...(data?.capabilityInventory?.unrecognized ?? []).map((item) => item.capability), + ]), + ] + .sort() + .map((item) => ( + + {item} + + ))} + { + setFile(event.target.files?.[0] ?? null); + setProgress(null); + }} + /> + + + {busy && ( + + )} + + + {file && ( + + {file.name} · {formatBytes(file.size)} + + )} + {progress && ( + <> + {progress.state === "uploading" && } + + {formatBytes(progress.receivedBytes)} received · {progress.state} + + + )} + + setSelected(event.target.value)} + sx={{ minWidth: 180 }} + > + {(capabilities.data?.targets ?? []).map((item) => ( + + {item.serviceId} + + ))} + + + + {target?.reason && {target.reason}} + {!selected && + capabilities.data?.targets + .filter((item) => !item.available) + .map((item) => ( + + {item.reason} + + ))} + {(stages.data?.items ?? []).map((stage) => ( + + + {stage.direction} · {stage.state} · {formatBytes(stage.bytes)} · {stage.stageId} + + {stage.sha256 && ( + + SHA-256 {stage.sha256} + + )} + + Expires {new Date(stage.expiresAt).toLocaleString()} · cleanup {stage.cleanup} + + + {stage.direction === "upload" && ["uploaded", "ready"].includes(stage.state) && ( + + )} + {stage.direction === "upload" && stage.state === "ready" && ( + + )} + {stage.direction === "download" && stage.state === "ready" && ( + + )} + + + + ))} + + + { + if (!busy) setPreview(null); + }} + fullWidth + maxWidth="sm" + > + Confirm prepared artifact import + + + + Target: {preview?.serviceId} · region {preview?.region ?? "unspecified"} + + + Archive {formatBytes(preview?.bytes ?? 0)} · additional space{" "} + {formatBytes(preview?.requiredFreeBytes ?? 0)} + + + Credentials required: {preview?.requiredCredentials.join(", ") || "none"} + + {preview?.blockers.map((blocker) => ( + + {blocker} + + ))} + {preview?.runtimeImageMismatches.map((item) => ( + + {item.service}: bundle {item.bundle}; local {item.local} + + ))} + + Consumers stay stopped. A queued job is not a completed import; check Activity for the + publication and hardlink result. + + + + + + + + + + ); +} diff --git a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx index c37935e07..88385e4e5 100644 --- a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx +++ b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx @@ -36,6 +36,7 @@ import type { AdminOperationContract } from "../operations/adminOperationsApi"; import { AdminPageHeader } from "../shared/AdminPageHeader"; import { useAdminToast } from "../shared/AdminToast"; import { AmbientPlacesMaintenance } from "./AmbientPlacesMaintenance"; +import { ArtifactTransfersSection } from "./ArtifactTransfersSection"; import { DataInventorySection } from "./DataInventorySection"; import { NotablePlacesMaintenance, notablePlacesStatusQueryKey } from "./NotablePlacesMaintenance"; import { OvertureMaintenance, overtureStatusQueryKey } from "./OvertureMaintenance"; @@ -429,6 +430,7 @@ function DataWorkflowsView() { + diff --git a/apps/web/src/components/admin/settings/ConfigurationMigrationPanel.test.tsx b/apps/web/src/components/admin/settings/ConfigurationMigrationPanel.test.tsx new file mode 100644 index 000000000..7b82d7681 --- /dev/null +++ b/apps/web/src/components/admin/settings/ConfigurationMigrationPanel.test.tsx @@ -0,0 +1,46 @@ +import { QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, expect, it, vi } from "vitest"; +import { cleanup, createTestQueryClient, fireEvent, render, screen } from "@/test"; +import { ConfigurationMigrationPanel } from "./ConfigurationMigrationPanel"; + +const mocks = vi.hoisted(() => ({ inventory: vi.fn(), preview: vi.fn(), apply: vi.fn() })); +vi.mock("@/lib/admin/accountAdmin", () => ({ + useAccountAdmin: () => ({ actorId: "synthetic-admin" }), +})); +vi.mock("../activity/activityQueries", () => ({ + useActivityClient: () => ({ apiUrl: "http://synthetic.test/api", client: {} }), +})); +vi.mock("@openmapx/core/api-admin-sdk", async () => ({ + ...(await vi.importActual( + "@openmapx/core/api-admin-sdk", + )), + getAdminConfigurationMigrationInventory: mocks.inventory, + previewAdminConfigurationMigration: mocks.preview, + applyAdminConfigurationMigration: mocks.apply, +})); +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); +it("checks file size before reading and never admits an oversized migration", async () => { + mocks.inventory.mockResolvedValue({ + status: 200, + data: { document: { prerequisites: [] }, fields: [] }, + }); + const cache = createTestQueryClient(); + render( + + + , + ); + const input = screen.getByText("Review configuration file").querySelector("input"); + if (!input) throw Error("File input missing"); + const text = vi.fn(), + file = { size: 1048577, text }; + fireEvent.change(input, { target: { files: [file] } }); + expect(await screen.findByText("Configuration file must be at most1MiB.")).toBeVisible(); + expect(text).not.toHaveBeenCalled(); + expect(mocks.preview).not.toHaveBeenCalled(); + expect(mocks.apply).not.toHaveBeenCalled(); + cache.clear(); +}); diff --git a/apps/web/src/components/admin/settings/ConfigurationMigrationPanel.tsx b/apps/web/src/components/admin/settings/ConfigurationMigrationPanel.tsx new file mode 100644 index 000000000..2b35b9084 --- /dev/null +++ b/apps/web/src/components/admin/settings/ConfigurationMigrationPanel.tsx @@ -0,0 +1,325 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Button from "@mui/material/Button"; +import Checkbox from "@mui/material/Checkbox"; +import FormControlLabel from "@mui/material/FormControlLabel"; +import Paper from "@mui/material/Paper"; +import Stack from "@mui/material/Stack"; +import Typography from "@mui/material/Typography"; +import { + applyAdminConfigurationMigration, + exportAdminConfigurationMigration, + getAdminConfigurationMigrationInventory, + previewAdminConfigurationMigration, + unwrapAdminResponse, +} from "@openmapx/core/api-admin-sdk"; +import { + type ConfigurationMigrationPreview, + configurationMigrationDocumentSchema, + configurationMigrationPreviewSchema, +} from "@openmapx/core/api-contracts"; +import { useQuery } from "@tanstack/react-query"; +import { useEffect, useRef, useState } from "react"; +import { useAccountAdmin } from "@/lib/admin/accountAdmin"; +import { useActivityClient } from "../activity/activityQueries"; +import { AdminReauthenticationDialog } from "../users/AdminReauthenticationDialog"; +import { CredentialTransferDialog } from "./CredentialTransferDialog"; + +import { downloadMigrationJson } from "./migration-json-download"; +export function ConfigurationMigrationPanel() { + const { actorId } = useAccountAdmin(); + const { client, apiUrl } = useActivityClient(), + [preview, setPreview] = useState(null), + [error, setError] = useState(""), + [result, setResult] = useState<{ + outcome: string; + steps: Array<{ scope: string; id: string; status: string }>; + } | null>(null), + [busy, setBusy] = useState(false), + [credentials, setCredentials] = useState(false), + [platformConfirmations, setPlatformConfirmations] = useState({ + domain: false, + dnsTls: false, + oauth: false, + passkeys: false, + alternateLogin: false, + }), + [platformReauth, setPlatformReauth] = useState(false), + controller = useRef(null), + epoch = useRef(0); + const storageKey = `openmapx-configuration-migration-preview:${JSON.stringify([apiUrl, actorId])}`; + const remember = (value: ConfigurationMigrationPreview) => { + sessionStorage.setItem( + storageKey, + JSON.stringify(configurationMigrationPreviewSchema.parse(value)), + ); + setPreview(value); + }; + useEffect(() => { + epoch.current++; + controller.current?.abort(); + setPreview(null); + try { + const raw = sessionStorage.getItem(storageKey); + if (raw) { + const value = configurationMigrationPreviewSchema.parse(JSON.parse(raw)); + if (Date.parse(value.expiresAt) > Date.now()) setPreview(value); + else sessionStorage.removeItem(storageKey); + } + } catch { + sessionStorage.removeItem(storageKey); + } + setResult(null); + if (!apiUrl || !actorId) setError("Administrator session unavailable"); + return () => { + epoch.current++; + controller.current?.abort(); + }; + }, [apiUrl, actorId, storageKey]); + const inventory = useQuery({ + queryKey: ["admin", "migration-inventory", apiUrl, actorId], + retry: false, + queryFn: async ({ signal }) => + unwrapAdminResponse( + await getAdminConfigurationMigrationInventory({ + client, + signal, + maxJsonBytes: 1048576, + maxErrorBytes: 65536, + }), + ), + }); + const run = async (action: (signal: AbortSignal) => Promise) => { + controller.current?.abort(); + const current = new AbortController(), + owner = epoch.current; + controller.current = current; + setBusy(true); + setError(""); + try { + await action(current.signal); + } catch { + if (owner === epoch.current && !current.signal.aborted) + setError("Migration did not settle. Inspect Activity before submitting another operation."); + } finally { + if (owner === epoch.current) setBusy(false); + } + }; + + const apply = async (proofToken?: string) => + run(async (signal) => { + if (!preview) return; + const body = { + previewId: preview.previewId, + fingerprint: preview.fingerprint, + ...(preview.platform + ? { + platformAuthorization: { + confirmations: platformConfirmations, + ...(proofToken ? { proofToken } : {}), + }, + } + : {}), + }; + const value = unwrapAdminResponse( + await applyAdminConfigurationMigration( + body, + { "idempotency-key": crypto.randomUUID() }, + { client, signal, maxJsonBytes: 1048576, maxErrorBytes: 65536 }, + ), + ); + if (!signal.aborted) { + setResult(value); + setPreview(null); + sessionStorage.removeItem(storageKey); + void inventory.refetch(); + } + }); + return ( + + + Configuration migration + + Export portable database configuration. Host overrides, credentials and bootstrap secrets + require separate preparation. Authentication settings are staged for explicit policy + Apply. A file may contain up to1,000 fields,512 change groups and128 keys per group. The + full resulting configuration, including retained values, must fit512KiB. + + {error && {error}} + {inventory.isError && ( + Configuration inventory is unavailable. + )} + {inventory.data && ( + + {inventory.data.fields.filter((f) => f.included).length} portable fields;{" "} + {inventory.data.document.prerequisites.length} prerequisites. Review every omission + before applying. + + )} + + + + + + {preview && ( + <> + + {preview.changes.length} selected changes. Preview expires{" "} + {new Date(preview.expiresAt).toLocaleTimeString()}. + + {preview.prerequisites.map((p) => ( + + {p.scope}/{p.id} + {p.key ? `/${p.key}` : ""}: {p.reason} ( + {p.source === "config.json" ? "file" : p.source}) + + ))} + {preview.denials.map((d) => ( + + {d} + + ))} + {preview.platform && ( + <> + + Platform origin: {preview.platform.oldOrigin} → {preview.platform.newOrigin}. + Services and login consumers activate under the same guarded migration. + + {Object.entries(preview.platform.requiredConfirmation) + .filter(([, required]) => required) + .map(([key]) => ( + + setPlatformConfirmations((v) => ({ ...v, [key]: checked })) + } + /> + } + /> + ))} + + )} + + + )} + {result && ( + + Migration outcome: {result.outcome}.{" "} + {result.steps.map((s) => `${s.scope}/${s.id}: ${s.status}`).join("; ")} + + )} + {platformReauth && preview?.platform && ( + { + setPlatformReauth(false); + void apply(proofs[0]?.challengeId); + }} + onClose={() => setPlatformReauth(false)} + /> + )} + {credentials && ( + setCredentials(false)} + inventory={ + inventory.data?.fields + .filter( + (f) => + f.ownership.secret && + f.ownership.source === "vault" && + (f.scope === "integration" || f.scope === "service"), + ) + .map((f) => ({ + scope: f.scope as "integration" | "service", + id: f.id, + key: f.key, + })) ?? [] + } + /> + )} + + + ); +} diff --git a/apps/web/src/components/admin/settings/CredentialTransferDialog.test.tsx b/apps/web/src/components/admin/settings/CredentialTransferDialog.test.tsx new file mode 100644 index 000000000..2a9120c2f --- /dev/null +++ b/apps/web/src/components/admin/settings/CredentialTransferDialog.test.tsx @@ -0,0 +1,55 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen } from "@/test"; +import { CredentialTransferDialog } from "./CredentialTransferDialog"; +import { credentialPreviewStorageKey, retainCredentialPreview } from "./credential-preview-storage"; + +vi.mock("@/lib/admin/accountAdmin", () => ({ + useAccountAdmin: () => ({ actorId: "synthetic-admin" }), +})); +vi.mock("../activity/activityQueries", () => ({ + useActivityClient: () => ({ apiUrl: "http://synthetic.test/api", client: {} }), +})); +afterEach(() => { + cleanup(); + sessionStorage.clear(); +}); +it("restores only reviewed public preview metadata after fresh sign-in navigation", async () => { + retainCredentialPreview( + credentialPreviewStorageKey("http://synthetic.test/api", "synthetic-admin"), + { + previewId: "00000000-0000-4000-8000-000000000001", + intentFingerprint: "a".repeat(64), + expectedCurrentRevisionId: null, + expiresAt: new Date(Date.now() + 600000).toISOString(), + entries: [{ scope: "integration", id: "example", key: "TOKEN", status: "selected" }], + applyable: true, + }, + ); + render( {}} />); + expect( + await screen.findByRole("button", { name: "Confirm selected credential import" }), + ).not.toBeDisabled(); + expect(screen.queryByLabelText("Transfer passphrase")).toBeNull(); + expect(JSON.stringify(sessionStorage)).not.toContain("passphrase"); +}); +it("rejects a large encrypted file before reading and requires explicit selection and passphrase", async () => { + render( + {}} + inventory={[{ scope: "integration", id: "example", key: "TOKEN" }]} + />, + ); + expect(screen.getByRole("button", { name: "Confirm encrypted export" })).toBeDisabled(); + const label = screen.getByText("Choose encrypted credential file"), + input = label.querySelector("input"); + if (!input) throw Error("File input missing"); + const text = vi.fn(); + fireEvent.change(input, { target: { files: [{ size: 1048577, text }] } }); + expect(await screen.findByText("Encrypted file must be at most1MiB.")).toBeVisible(); + expect(text).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("checkbox", { name: "integration/example/TOKEN" })); + fireEvent.change(screen.getByLabelText("Transfer passphrase"), { + target: { value: "synthetic passphrase" }, + }); + expect(screen.getByRole("button", { name: "Confirm encrypted export" })).not.toBeDisabled(); +}); diff --git a/apps/web/src/components/admin/settings/CredentialTransferDialog.tsx b/apps/web/src/components/admin/settings/CredentialTransferDialog.tsx new file mode 100644 index 000000000..df23b6e4a --- /dev/null +++ b/apps/web/src/components/admin/settings/CredentialTransferDialog.tsx @@ -0,0 +1,487 @@ +"use client"; +import Alert from "@mui/material/Alert"; +import Button from "@mui/material/Button"; +import Checkbox from "@mui/material/Checkbox"; +import Dialog from "@mui/material/Dialog"; +import DialogActions from "@mui/material/DialogActions"; +import DialogContent from "@mui/material/DialogContent"; +import DialogTitle from "@mui/material/DialogTitle"; +import FormControlLabel from "@mui/material/FormControlLabel"; +import Stack from "@mui/material/Stack"; +import TextField from "@mui/material/TextField"; +import Typography from "@mui/material/Typography"; +import { + applyAdminCredentialTransfer, + chooseAdminCredentialTransferFields, + exportAdminCredentialTransfer, + getAdminTransitCredentials, + previewAdminCredentialTransfer, + unwrapAdminResponse, +} from "@openmapx/core/api-admin-sdk"; +import { + CREDENTIAL_TRANSFER_PREVIEW_INTENT, + type CredentialImportChoices, + type CredentialImportPreview, + type CredentialTransferEnvelope, + credentialImportPreviewSchema, + credentialTransferEnvelopeSchema, +} from "@openmapx/core/api-contracts"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { useAccountAdmin } from "@/lib/admin/accountAdmin"; +import { useActivityClient } from "../activity/activityQueries"; +import { AdminReauthenticationDialog } from "../users/AdminReauthenticationDialog"; +import { + credentialPreviewStorageKey, + readCredentialPreview, + retainCredentialPreview, +} from "./credential-preview-storage"; +import { downloadMigrationJson } from "./migration-json-download"; + +type Field = { scope: "integration" | "service"; id: string; key: string }; +const identity = (field: Field) => `${field.scope}/${field.id}/${field.key}`; +export function CredentialTransferDialog({ + onClose, + inventory = [], +}: { + onClose: () => void; + inventory?: Field[]; +}) { + const { client, apiUrl } = useActivityClient(), + { actorId } = useAccountAdmin(), + [passphrase, setPassphrase] = useState(""), + [envelope, setEnvelope] = useState(null), + [selected, setSelected] = useState([]), + [preview, setPreview] = useState(null), + [error, setError] = useState(""), + [busy, setBusy] = useState(false), + [choicesDirty, setChoicesDirty] = useState(false), + [ceremony, setCeremony] = useState<"export" | "preview" | "apply" | null>(null), + [outcome, setOutcome] = useState(""), + [catalog, setCatalog] = useState<{ + catalogRevisionId: string; + credentialsRevision: string; + entries: Array<{ key: string }>; + } | null>(null), + [setKeys, setSetKeys] = useState(""), + [deleteKeys, setDeleteKeys] = useState(""), + controller = useRef(null), + epoch = useRef(0); + const storageKey = credentialPreviewStorageKey(apiUrl, actorId); + const remember = (value: CredentialImportPreview) => { + retainCredentialPreview(storageKey, value); + setPreview(value); + }; + const clear = useCallback(() => { + epoch.current++; + controller.current?.abort(); + setPassphrase(""); + setEnvelope(null); + setPreview(null); + setSelected([]); + setCeremony(null); + }, []); + useEffect(() => { + clear(); + if (!apiUrl || !actorId) setError("Administrator session unavailable"); + const restored = readCredentialPreview(storageKey); + setPreview(restored); + setSelected(restored?.entries.filter((e) => e.status === "selected").map(identity) ?? []); + setChoicesDirty(false); + setSetKeys( + restored?.feedMap?.changes + .filter((e) => e.action === "set") + .map((e) => e.key) + .join("\n") ?? "", + ); + setDeleteKeys( + restored?.feedMap?.changes + .filter((e) => e.action === "delete") + .map((e) => e.key) + .join("\n") ?? "", + ); + return () => { + epoch.current++; + controller.current?.abort(); + }; + }, [apiUrl, actorId, clear, storageKey]); + const close = () => { + sessionStorage.removeItem(storageKey); + clear(); + onClose(); + }; + const run = async (action: (signal: AbortSignal) => Promise) => { + controller.current?.abort(); + const next = new AbortController(), + owner = epoch.current; + controller.current = next; + setBusy(true); + setError(""); + try { + await action(next.signal); + } catch { + if (epoch.current === owner && !next.signal.aborted) + setError( + "Credential transfer did not settle. Inspect Activity and use fresh confirmation before another mutation.", + ); + } finally { + if (epoch.current === owner) { + setPassphrase(""); + setBusy(false); + } + } + }; + const options = (signal: AbortSignal) => ({ + client, + signal, + maxJsonBytes: 1048576, + maxErrorBytes: 65536, + }); + const choices = (): CredentialImportChoices => ({ + entries: (preview?.entries ?? []).map((field) => ({ + scope: field.scope, + id: field.id, + key: field.key, + action: selected.includes(identity(field)) ? "import" : "skip", + })), + ...(catalog && + preview?.entries.some( + (e) => + e.scope === "service" && + e.id === "data-manager" && + e.key === "TRANSITOUS_API_KEYS" && + selected.includes(identity(e)), + ) + ? { + feedMap: { + catalogRevisionId: catalog.catalogRevisionId, + expectedCredentialsRevision: catalog.credentialsRevision, + setKeys: setKeys + .split("\n") + .map((s) => s.trim()) + .filter(Boolean), + deleteKeys: deleteKeys + .split("\n") + .map((s) => s.trim()) + .filter(Boolean), + }, + } + : {}), + }); + const authorize = async (proofs: Array<{ challengeId: string }>) => { + const action = ceremony; + setCeremony(null); + await run(async (signal) => { + if (action === "export") { + const value = unwrapAdminResponse( + await exportAdminCredentialTransfer( + { + selection: inventory.filter((f) => selected.includes(identity(f))), + passphrase, + confirmed: true, + }, + { ...options(signal), headers: { "x-admin-reauth-proof": proofs[0]!.challengeId } }, + ), + ); + if (!signal.aborted) + downloadMigrationJson( + credentialTransferEnvelopeSchema.parse(value), + "openmapx-vault-transfer.json.enc", + ); + } else if (action === "preview" && envelope) { + const value = unwrapAdminResponse( + await previewAdminCredentialTransfer( + { envelope, passphrase }, + { + "idempotency-key": crypto.randomUUID(), + "x-admin-reauth-proof": proofs[0]!.challengeId, + }, + options(signal), + ), + ); + if (!signal.aborted) { + remember(credentialImportPreviewSchema.parse(value)); + setEnvelope(null); + setSelected([]); + } + } else if (action === "apply" && preview) { + const value = unwrapAdminResponse( + await applyAdminCredentialTransfer( + { + previewId: preview.previewId, + intentFingerprint: preview.intentFingerprint, + expectedCurrentRevisionId: preview.expectedCurrentRevisionId, + importProofToken: proofs[0]!.challengeId, + ...(preview.feedMap ? { transitProofToken: proofs[1]!.challengeId } : {}), + }, + { "idempotency-key": crypto.randomUUID() }, + options(signal), + ), + ); + if (!signal.aborted) { + setOutcome( + `${value.outcome}: ${value.steps.map((s) => `${s.scope}/${s.id} ${s.status}`).join("; ")}`, + ); + setPreview(null); + sessionStorage.removeItem(storageKey); + } + } + }); + }; + const validPass = + new TextEncoder().encode(passphrase).length >= 12 && + new TextEncoder().encode(passphrase).length <= 1024; + return ( + <> + + Protected credential transfer + + + + Only declared effective vault credentials can be transferred. File, environment, + authentication and bootstrap secrets require manual setup. Passphrases stay in this + dialog and are cleared after each request. + + {error && {error}} + {outcome && ( + + {outcome} + + )} + {!preview && ( + <> + {inventory.map((field) => ( + { + setChoicesDirty(true); + setCeremony(null); + setSelected((v) => + checked + ? [...v, identity(field)] + : v.filter((k) => k !== identity(field)), + ); + }} + /> + } + /> + ))} + setPassphrase(e.target.value)} + helperText="12–1024 UTF-8 bytes. Enter again after returning from sign-in." + /> + + + + + )} + {preview && ( + <> + + Choose each field explicitly. Existing ordinary credentials are overwritten only + when selected. + + {preview.entries.map((field) => ( + { + setChoicesDirty(true); + setCeremony(null); + setSelected((v) => + checked + ? [...v, identity(field)] + : v.filter((k) => k !== identity(field)), + ); + }} + /> + } + /> + ))} + {preview.entries.some( + (e) => + e.scope === "service" && + e.id === "data-manager" && + e.key === "TRANSITOUS_API_KEYS", + ) && ( + <> + + {catalog && ( + + {catalog.entries.length} current feed credential IDs; catalog{" "} + {catalog.catalogRevisionId}. + + )} + { + setChoicesDirty(true); + setCeremony(null); + setSetKeys(e.target.value); + }} + /> + { + setChoicesDirty(true); + setCeremony(null); + setDeleteKeys(e.target.value); + }} + /> + + )} + + + {preview.entries.filter((e) => e.status === "selected").length} selected;{" "} + {preview.entries.filter((e) => e.status === "skipped").length} skipped;{" "} + {preview.entries.filter((e) => e.status === "denied").length} denied. + + + + )} + + + + + + + {ceremony && + (ceremony === "apply" && preview?.feedMap ? ( + void authorize(proofs)} + onClose={() => setCeremony(null)} + /> + ) : ( + void authorize(proofs)} + onClose={() => setCeremony(null)} + /> + ))} + + ); +} diff --git a/apps/web/src/components/admin/settings/SystemSettings.tsx b/apps/web/src/components/admin/settings/SystemSettings.tsx index c3a89d240..389d15b22 100644 --- a/apps/web/src/components/admin/settings/SystemSettings.tsx +++ b/apps/web/src/components/admin/settings/SystemSettings.tsx @@ -35,6 +35,7 @@ import { useEnv } from "@/integration-api/runtime/EnvProvider"; import { useIntegrationAdmin } from "@/lib/admin/integrationAdmin"; import { AdminPageHeader } from "../shared/AdminPageHeader"; import { AuthPolicySettings } from "./AuthPolicySettings"; +import { ConfigurationMigrationPanel } from "./ConfigurationMigrationPanel"; export interface ResolvedSetting { group: string; @@ -981,6 +982,7 @@ export function SystemSettings({ /> ))} {showTransfer && } + {showTransfer && } + `openmapx-credential-preview:${JSON.stringify([apiUrl, actorId])}`; +export function readCredentialPreview(key: string): CredentialImportPreview | null { + try { + const raw = sessionStorage.getItem(key); + if (!raw) return null; + const value = credentialImportPreviewSchema.parse(JSON.parse(raw)); + if (Date.parse(value.expiresAt) <= Date.now()) { + sessionStorage.removeItem(key); + return null; + } + return value; + } catch { + sessionStorage.removeItem(key); + return null; + } +} +export function retainCredentialPreview(key: string, value: CredentialImportPreview) { + sessionStorage.setItem(key, JSON.stringify(credentialImportPreviewSchema.parse(value))); +} diff --git a/apps/web/src/components/admin/settings/migration-json-download.ts b/apps/web/src/components/admin/settings/migration-json-download.ts new file mode 100644 index 000000000..01ddbea25 --- /dev/null +++ b/apps/web/src/components/admin/settings/migration-json-download.ts @@ -0,0 +1,13 @@ +export function downloadMigrationJson(value: unknown, filename: string) { + const json = JSON.stringify(value, null, 2); + if (new TextEncoder().encode(json).length > 1048576) throw Error("Transfer exceeds1MiB"); + const url = URL.createObjectURL(new Blob([json], { type: "application/json" })), + a = document.createElement("a"); + try { + a.href = url; + a.download = filename; + a.click(); + } finally { + URL.revokeObjectURL(url); + } +} diff --git a/apps/web/src/lib/admin/artifactTransferClient.test.ts b/apps/web/src/lib/admin/artifactTransferClient.test.ts new file mode 100644 index 000000000..4f3084143 --- /dev/null +++ b/apps/web/src/lib/admin/artifactTransferClient.test.ts @@ -0,0 +1,110 @@ +import { createApiClient } from "@openmapx/core"; +import { afterEach, expect, it, vi } from "vitest"; +import { downloadArtifactToSink, uploadArtifactFile } from "./artifactTransferClient"; + +const stageId = "9b0f4718-062d-4b5f-8048-55381380aeac"; +const stage = { + stageId, + purpose: "prepared-artifact", + direction: "upload", + state: "uploaded", + bytes: 4, + sha256: "a".repeat(64), + expiresAt: "2026-12-01T00:00:00.000Z", + cleanup: "pending", +}; +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); +it("uploads the original File through the supplied client without buffering and reports server progress", async () => { + const file = new File([new Uint8Array([0, 255, 128, 7])], "graph.tar"), + progress = vi.fn(); + Object.defineProperty(file, "arrayBuffer", { + value: () => { + throw Error("Whole-file read forbidden"); + }, + }); + const requests = vi.fn(async (_url, raw) => { + const init = raw as RequestInit; + if (init.method === "PUT") expect(init.body).toBe(file); + return Response.json(stage); + }); + vi.stubGlobal("fetch", requests); + expect( + await uploadArtifactFile({ + client: createApiClient({ baseUrl: "http://transfer.test" }), + stageId, + file, + signal: new AbortController().signal, + onProgress: progress, + }), + ).toMatchObject(stage); + expect( + requests.mock.calls.filter(([, init]) => (init as RequestInit).method === "PUT"), + ).toHaveLength(1); + expect(String(requests.mock.calls[0]?.[0])).toContain( + "http://transfer.test/api/admin/artifact-transfers/", + ); + expect(progress).toHaveBeenLastCalledWith({ receivedBytes: 4, state: "uploaded" }); +}); +it("reconciles a lost upload response by inspection rather than sending the body twice", async () => { + const requests = vi.fn(async (_url, raw) => { + const init = raw as RequestInit; + if (init.method === "PUT") throw Error("Named lost response"); + return Response.json(stage); + }); + vi.stubGlobal("fetch", requests); + expect( + await uploadArtifactFile({ + client: createApiClient({ baseUrl: "http://transfer.test" }), + stageId, + file: new File(["data"], "graph.tar"), + signal: new AbortController().signal, + }), + ).toMatchObject({ state: "uploaded" }); + expect( + requests.mock.calls.filter(([, init]) => (init as RequestInit).method === "PUT"), + ).toHaveLength(1); +}); +it("streams non-UTF8 download bytes into the selected sink without Blob materialization", async () => { + const blob = vi.spyOn(Response.prototype, "blob").mockRejectedValue(Error("No memory Blob")); + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(new Uint8Array([0, 255, 128, 7]), { + headers: { + "content-type": "application/octet-stream", + "content-length": "4", + "content-disposition": 'attachment; filename="openmapx-artifact.tar"', + "x-artifact-sha256": "a".repeat(64), + "cache-control": "no-store", + "x-content-type-options": "nosniff", + }, + }), + ), + ); + const chunks: Uint8Array[] = []; + let closed = 0, + aborted = 0; + const write = async (chunk: Uint8Array) => { + chunks.push(chunk); + }, + close = async () => { + closed++; + }, + abort = async () => { + aborted++; + }; + await downloadArtifactToSink({ + client: createApiClient({ baseUrl: "http://transfer.test" }), + stageId, + sink: { write, close, abort }, + signal: new AbortController().signal, + }); + expect(Array.from(chunks[0] ?? [])).toEqual([0, 255, 128, 7]); + expect(closed).toBe(1); + expect(aborted).toBe(0); + expect(blob).not.toHaveBeenCalled(); +}); diff --git a/apps/web/src/lib/admin/artifactTransferClient.ts b/apps/web/src/lib/admin/artifactTransferClient.ts new file mode 100644 index 000000000..4099f06b4 --- /dev/null +++ b/apps/web/src/lib/admin/artifactTransferClient.ts @@ -0,0 +1,199 @@ +import { type ApiClient, ApiNativeDownloadUnavailableError } from "@openmapx/core"; +import * as sdk from "@openmapx/core/api-admin-sdk"; +import type { TransferStage } from "@openmapx/core/ops"; + +const STREAM_TIMEOUT = 72 * 3600_000; +function wait(signal: AbortSignal) { + return new Promise((resolve, reject) => { + const done = () => { + signal.removeEventListener("abort", aborted); + resolve(); + }; + const timer = setTimeout(done, 1000); + const aborted = () => { + clearTimeout(timer); + reject(signal.reason); + }; + signal.addEventListener("abort", aborted, { once: true }); + if (signal.aborted) aborted(); + }); +} +/** The File remains a native request body. Inspection is observation, never an + * automatic replay of an upload whose response was lost. */ +export async function uploadArtifactFile(input: { + client: ApiClient; + stageId: string; + file: File; + signal: AbortSignal; + onProgress?: (progress: { receivedBytes: number; state: TransferStage["state"] }) => void; +}): Promise { + input.signal.throwIfAborted(); + const observer = new AbortController(), + signal = AbortSignal.any([input.signal, observer.signal]); + const progress = (stage: TransferStage) => + input.onProgress?.({ receivedBytes: stage.bytes, state: stage.state }); + const observing = (async () => { + while (!signal.aborted) { + try { + const stage = sdk.unwrapAdminResponse( + await sdk.getAdminArtifactTransfer(input.stageId, { client: input.client, signal }), + ); + if (!signal.aborted) progress(stage); + } catch { + if (signal.aborted) return; + } + try { + await wait(signal); + } catch { + return; + } + } + })(); + try { + let stage: TransferStage; + try { + stage = sdk.unwrapAdminResponse( + await sdk.uploadAdminArtifactTransferContent(input.stageId, input.file, { + client: input.client, + signal: input.signal, + timeoutMs: STREAM_TIMEOUT, + }), + ); + } catch (error) { + input.signal.throwIfAborted(); + const retained = sdk.unwrapAdminResponse( + await sdk.getAdminArtifactTransfer(input.stageId, { + client: input.client, + signal: input.signal, + }), + ); + if ( + !["uploaded", "ready", "claimed", "consumed"].includes(retained.state) || + retained.bytes !== input.file.size || + !retained.sha256 + ) + throw error; + stage = retained; + } + input.signal.throwIfAborted(); + progress(stage); + return stage; + } finally { + observer.abort(); + await observing; + } +} + +export interface ArtifactDownloadSink { + write(chunk: Uint8Array, signal: AbortSignal): Promise; + close(): Promise; + abort(reason?: unknown): Promise; +} +export async function downloadArtifactToSink(input: { + client: ApiClient; + stageId: string; + sink: ArtifactDownloadSink; + signal: AbortSignal; +}): Promise { + let aborting: Promise | undefined; + const abort = (reason: unknown) => + (aborting ??= Promise.resolve().then(() => input.sink.abort(reason))); + const onAbort = () => { + void abort(input.signal.reason).catch(() => {}); + }; + input.signal.addEventListener("abort", onAbort, { once: true }); + try { + input.signal.throwIfAborted(); + sdk.unwrapAdminResponse( + await sdk.downloadAdminArtifactTransferContent(input.stageId, { + client: input.client, + signal: input.signal, + timeoutMs: STREAM_TIMEOUT, + onByteChunk: async (chunk, signal) => { + const cancel = () => { + void abort(signal.reason).catch(() => {}); + }; + signal.addEventListener("abort", cancel, { once: true }); + try { + signal.throwIfAborted(); + await input.sink.write(chunk, signal); + signal.throwIfAborted(); + } finally { + signal.removeEventListener("abort", cancel); + if (aborting) await aborting.catch(() => {}); + } + }, + }), + ); + input.signal.throwIfAborted(); + await input.sink.close(); + } catch (error) { + await abort(error).catch(() => {}); + throw error; + } finally { + input.signal.removeEventListener("abort", onAbort); + if (aborting) await aborting.catch(() => {}); + } +} + +type PickerWindow = Window & { + showSaveFilePicker?: (options: { suggestedName: string }) => Promise<{ + createWritable(): Promise<{ + write(chunk: Uint8Array): Promise; + close(): Promise; + abort(reason?: unknown): Promise; + }>; + }>; +}; +export function artifactFileSinkSupported(): boolean { + return ( + typeof window !== "undefined" && + typeof (window as PickerWindow).showSaveFilePicker === "function" + ); +} +/** Invoke directly in the user gesture, before awaiting a network request. */ +export async function createArtifactFileSink(): Promise { + const picker = (window as PickerWindow).showSaveFilePicker; + if (!picker) throw new ApiNativeDownloadUnavailableError("runtime"); + const handle = await picker.call(window, { suggestedName: "openmapx-artifact.tar" }), + writable = await handle.createWritable(); + let closed = false, + pending: Promise | undefined, + aborting: Promise | undefined; + const abort = async (reason?: unknown) => { + if (closed) { + if (aborting) await aborting; + return; + } + closed = true; + aborting = Promise.resolve().then(() => writable.abort(reason)); + const results = await Promise.allSettled([pending, aborting]); + const failed = results.find((result) => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; + }; + return { + async write(chunk, signal) { + if (closed) throw Error("Download sink is closed"); + signal.throwIfAborted(); + const cancel = () => { + void abort(signal.reason).catch(() => {}); + }; + signal.addEventListener("abort", cancel, { once: true }); + try { + pending = writable.write(chunk); + await pending; + signal.throwIfAborted(); + } finally { + signal.removeEventListener("abort", cancel); + if (aborting) await aborting.catch(() => {}); + } + }, + async close() { + if (closed) throw Error("Download sink is closed"); + await pending; + await writable.close(); + closed = true; + }, + abort, + }; +} diff --git a/docs/docs/administration/integrations-administration.md b/docs/docs/administration/integrations-administration.md index ab7851695..066f0d090 100644 --- a/docs/docs/administration/integrations-administration.md +++ b/docs/docs/administration/integrations-administration.md @@ -459,3 +459,40 @@ was restored, or that reconciliation is required. Restoration never authorizes overwriting a newer administrator/host edit. If reconciliation is required, inspect saved configuration, the applied host generation and runtime state before making another deliberate change. Do not automatically retry the import. + +## Portable configuration and protected credentials + +System Settings includes configuration migration with an ownership inventory. +Export contains declared database-owned non-secret settings, integration and +service configuration, bindings, and installation prerequisites. Environment +and file overrides, vault values, bootstrap keys, and unsupported metadata are +listed as omissions. Install missing extensions and resolve required credentials +before applying dependent configuration. + +Import a configuration JSON file of at most 1 MiB, review the selected scopes and +prerequisites, and apply its ten-minute preview. The preview is bound to the +administrator and original configuration sources. A changed configuration or +source requires a new preview. Selected desired changes share one guarded +transaction and generation; later provider or host effects can remain partial or +unknown. Activity retains those outcomes. Imported authentication settings are +pending drafts until the separate authentication policy Apply action. + +Platform fields use the existing origin, DNS/TLS, OAuth, passkey and alternate +login confirmations. Domain changes require a fresh purpose-bound sign-in and +use the same guarded generation as the other selected configuration. + +Protected credential transfer exports only explicitly selected, effective vault +credentials to `openmapx-vault-transfer.json.enc`. Supply a passphrase of 12–1024 +UTF-8 bytes and confirm a fresh administrator sign-in. The authenticated encrypted +file is at most 1 MiB. A lost export response requires another explicit sign-in; +the server does not retain the passphrase or a plaintext export for replay. + +On import, confirm sign-in to decrypt and review the file, then explicitly choose +which fields to import or skip. Existing ordinary credentials are overwritten +only when selected. Managed Transitous credentials also require the current +catalog and credential revisions: selected feed IDs are merged, unselected keys +are preserved, and only explicitly listed deletion IDs are removed. Applying a +managed transfer requires both import and transit-credential purposes from one +fresh sign-in. The encrypted preview expires after ten minutes; changing choices +invalidates its prior confirmation. Passphrases are cleared after each request +and when the dialog closes. diff --git a/docs/docs/developer/api-surface.md b/docs/docs/developer/api-surface.md index 876147a9a..a31289f9d 100644 --- a/docs/docs/developer/api-surface.md +++ b/docs/docs/developer/api-surface.md @@ -583,3 +583,35 @@ header interceptor. It performs no download and cannot provide custom headers, progress or cancellation. Other configured clients keep using the callback-based byte client with a supported file sink; do not silently change their credentials or buffer a large response into a Blob. + +### Prepared-artifact jobs and purpose observations + +Artifact capabilities and export/import controls use the service-admin registry. +Exports, archive validation and imports are retained administrative jobs; acceptance +is not their terminal outcome. Stage ownership is scoped to the authenticated +administrator and purpose. The API forwards byte streams to typed agent transport +without gaining Docker or arbitrary filesystem authority. + +The agent's private publication journal distinguishes file commit from stage +transfer and frozen-generation hardlink completion. A reconstructed agent may +restore an interrupted transaction or observe its original committed result, but +cannot redispatch an attempted child. A historical canonical failure remains +immutable. The fixed `artifactTransfer.inspectClaim` observer can additionally +return `publication` only after the original signed journal, child containment, +known effect and physically released whole target tree are verified. The API +durably copies that separate purpose observation into its owned SQL checkpoint +before acknowledging the original canonical tuple. Stage metadata cannot retire +that receipt before delivery. An HTTP status, expired deadline, consumed stage or absent +PID alone cannot authorize that transition. + +MOTIS's generated credential-bearing files are a private projection from a nominal +offline destination materialization handle. The public archive schema continues to +reject those files. Capacity is reserved before materialization, and publication +binds both the immutable portable fingerprint and generated local fingerprint. +The CLI and agent share that same publisher and fixed destination adapter. + +Linux byte uploads, downloads and private archive validation hold an inherited +descriptor lock. Shutdown drains byte streams; it does not cancel host jobs. +After actual process death, startup can reclaim authenticated private partial +bytes when the kernel lock is free. Unknown managed-file publication remains +fenced and is never inferred from a timeout or process identifier. diff --git a/docs/docs/install/managing-services.md b/docs/docs/install/managing-services.md index 69fb485dd..8d2d57096 100644 --- a/docs/docs/install/managing-services.md +++ b/docs/docs/install/managing-services.md @@ -584,3 +584,62 @@ profile cannot avoid ferries or provide a stronger general hill-avoidance level, so gravel requests with ferry avoidance or strong hill avoidance use Valhalla. Other travel modes keep their existing engines. Operators can configure an explicit `routing-brouter` endpoint instead of the managed service; OpenMapX never selects a public BRouter endpoint by default. + +## Move prepared data between instances + +The admin panel's **Data Workflows → Prepared data** section uploads, validates, +previews and imports prepared tileserver, OSRM, OTP, Pelias and MOTIS bundles. It +also exports verified prepared data into an authenticated, temporary download. +The CLI's `data export-build` and `data import-build` commands use the same archive +validator and journaled publisher. + +Use a bundle from a complete build with matching destination runtime images. +Validation checks the archive's framing, declared membership, sizes and SHA-256 +hashes before publication. Archive paths, links, devices and undeclared files +cannot select host destinations. A successful upload or validation is not an +import: review the compatibility preview and explicitly confirm publication. +Imports currently support at most 10,000 payload files and directories. + +Stop all consumers listed in the preview using the normal service controls. +Import never stops or starts containers, pulls images, or applies pending desired +configuration. It applies the already-current generation's hardlink plan through +the original import job; a nonempty plan requires Data Manager already running. +If files commit but the hardlink follow-up fails, keep consumers stopped and +reconcile that generation before starting them. Partial or unknown outcomes stay +visible in Activity and retain their original ownership; retrying observation does +not repeat publication. + +MOTIS export requires a retained portable recipe and compiled candidate. Source +credentials and generated proxy configuration are excluded from the bundle. The +destination must have the same approved catalog pins and all required managed feed +credentials; the preview lists credential identifiers without their values. Host +credential-file overrides require explicit host preparation. Import rebuilds the +private candidate and proxy files offline from those pins and destination secrets. +Existing `live` and `staging` slot links are preserved; replaced data does not +inherit an old successful health check. Start and validate it through the normal +MOTIS workflow before treating it as ready. + +Uploads stream directly, and downloads use browser-native navigation where the +client's origin and authentication permit it. Other clients require a streaming +file sink. Unsupported browsers receive guidance instead of buffering a large +archive in browser memory. Leaving the page aborts only an active byte transfer; +queued validation, import and export jobs continue in Activity. Cancel durable +work there. Deleting a stage owned by unfinished work returns a conflict. + +Default agent limits are a 1 TiB upload, 2 TiB expanded archive, 4 TiB combined +stage quota, a five-minute idle timeout and a 72-hour transfer deadline. Upload +stages expire after 96 hours; prepared downloads expire after 24 hours. The +private deployment policy at `/run/openmapx/artifact-transfer-policy.json` can +adjust bounded limits and memory allocation. The UI reports the effective limits, +reservations and currently observable free space; unavailable capacity refuses +admission. Country-scale throughput depends on disk, memory, ingress and timeout +configuration and is not established by a small successful transfer. + +The agent reconciles stages at startup and every 15 minutes, processing at most +1,000 per sweep. Linux descriptor locks preserve live byte owners and permit +partial-upload cleanup after actual process death. Non-Linux development hosts +retain uncertain orphan bytes for operator recovery. Expiry never releases an +unknown publication or cancels its job. Completed stage tombstones retain request +replay protection for seven days after expiry, with at most 10,000 retained +records. Failed cleanup keeps its quota charged and is retried; unknown purpose +claims remain pinned until their original evidence resolves. diff --git a/packages/cli/__tests__/cli-artifact-container-state.test.ts b/packages/cli/__tests__/cli-artifact-container-state.test.ts new file mode 100644 index 000000000..178e82d9c --- /dev/null +++ b/packages/cli/__tests__/cli-artifact-container-state.test.ts @@ -0,0 +1,13 @@ +import { expect, it, vi } from "vitest"; + +const native = vi.hoisted(() => vi.fn()); +vi.mock("execa", () => ({ execa: native })); + +import { runningComposeServicesStrict } from "../src/lib/docker"; + +it("refuses artifact movement when Docker container state cannot be acquired", async () => { + native.mockResolvedValue({ exitCode: 1, stdout: "", stderr: "private diagnostic" }); + await expect(runningComposeServicesStrict(["osrm"])).rejects.toThrow( + "Container state is unavailable", + ); +}); diff --git a/packages/cli/__tests__/cli-build-artifacts.test.ts b/packages/cli/__tests__/cli-build-artifacts.test.ts index 8895bbbf3..67f9c54d6 100644 --- a/packages/cli/__tests__/cli-build-artifacts.test.ts +++ b/packages/cli/__tests__/cli-build-artifacts.test.ts @@ -1,4 +1,5 @@ import { execFileSync } from "node:child_process"; +import { randomBytes, randomUUID } from "node:crypto"; import { existsSync, lstatSync, @@ -78,6 +79,8 @@ function seedOsrmBuild(root: string): void { const graph = join(dataDir(root), "osrm-graph"); mkdirSync(join(graph, "region.osrm.cells.d"), { recursive: true }); writeFileSync(join(graph, "region.osrm"), "GRAPH"); + for (const name of ["region.osrm.partition", "region.osrm.cells", "region.osrm.mldgr"]) + writeFileSync(join(graph, name), name); writeFileSync(join(graph, "region.osrm.cells.d", "part"), "CELLS"); writeFileSync(join(graph, "segment-speeds.csv"), ""); writeFileSync(join(graph, "region.osm.pbf"), "PBF"); @@ -97,10 +100,87 @@ afterEach(() => { }); describe("build bundles", () => { + it.each(["import", "export"] as const)( + "refuses a live target owner before CLI %s publication", + async (action) => { + seedOsrmBuild(buildHost); + const bundle = join(tmp, "owned.tar"); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle, + rootDir: buildHost, + }); + const control = await import("@openmapx/core/build-artifacts"), + ops = await import("@openmapx/core/ops"); + const targetHost = action === "import" ? serveHost : buildHost; + const infraDir = join(targetHost, "infra/docker"); + await control.provisionControlInstallation(infraDir); + const installation = control.openControlInstallation({ + infraDir, + role: "host-cli", + serviceIds: ["osrm", "motis"], + }); + let lease: import("@openmapx/core/build-artifacts").TargetLeaseAuthority | undefined; + try { + const admittedAt = Date.now(), + deadlineAt = admittedAt + 60000, + operation = { kind: "service.build" as const, serviceId: "osrm" }, + operationFingerprint = ops.opsOperationFingerprint(operation), + execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: ops.createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: ops.executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }, + authority = control.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ["osrm"], + }); + lease = await control.acquireArtifactTargetLease({ + dataRoot: dataDir(targetHost), + serviceIds: ["osrm"], + owner: authority, + }); + await expect( + action === "import" + ? importBuildBundle({ bundle, rootDir: serveHost, runningServices: noneRunning }) + : exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle: join(tmp, "busy-export.tar"), + rootDir: buildHost, + }), + ).rejects.toThrow(/busy|lease|owned/i); + if (action === "import") + expect(existsSync(join(dataDir(serveHost), "osrm-graph/region.osrm"))).toBe(false); + else expect(existsSync(join(tmp, "busy-export.tar"))).toBe(false); + } finally { + if (lease) control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(installation); + } + }, + 60000, + ); it("retains imported source provenance with fresh local output identities", async () => { seedOsrmBuild(buildHost); const { artifactFingerprint } = await import("@openmapx/core/build-artifacts"); const initial = await exportBuildBundle({ + runningServices: noneRunning, serviceId: "osrm", bundle: join(tmp, "first.tar"), rootDir: buildHost, @@ -116,7 +196,12 @@ describe("build bundles", () => { buildHost, ); const path = join(tmp, "v2.tar"); - await exportBuildBundle({ serviceId: "osrm", bundle: path, rootDir: buildHost }); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle: path, + rootDir: buildHost, + }); await importBuildBundle({ bundle: path, rootDir: serveHost, runningServices: noneRunning }); expect(readBuildRecord("osrm", serveHost)).toMatchObject({ schemaVersion: 2, @@ -132,16 +217,24 @@ describe("build bundles", () => { sizeBytes: 5, inode: String(lstatSync(join(dataDir(serveHost), "osrm-graph/region.osrm")).ino), }); - }); + }, 60000); it("round-trips a build to another host, dropping staged inputs and recording provenance", async () => { seedOsrmBuild(buildHost); const bundle = join(tmp, "osrm.tar"); - const exported = await exportBuildBundle({ serviceId: "osrm", bundle, rootDir: buildHost }); + const exported = await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle, + rootDir: buildHost, + }); expect(exported.files.map((file) => file.path)).toEqual([ "osrm-graph/region.osrm", + "osrm-graph/region.osrm.cells", "osrm-graph/region.osrm.cells.d/part", + "osrm-graph/region.osrm.mldgr", + "osrm-graph/region.osrm.partition", "osrm-graph/segment-speeds.csv", ]); @@ -160,14 +253,19 @@ describe("build bundles", () => { expect(readFileSync(join(liveGraph, "region.osrm.cells.d", "part"), "utf-8")).toBe("CELLS"); expect(existsSync(join(liveGraph, "region.osm.pbf"))).toBe(false); expect(existsSync(join(dataDir(serveHost), ".openmapx-import"))).toBe(false); - expect(readBuildRecord("osrm", serveHost)).toEqual(osrmRecord()); + expect(readBuildRecord("osrm", serveHost)).toMatchObject({ ...osrmRecord(), schemaVersion: 2 }); }); it("refuses a bundle built for a different runtime image", async () => { seedOsrmBuild(buildHost); const otherServe = makeHost("serve-other", DIGEST_B); const bundle = join(tmp, "osrm.tar"); - await exportBuildBundle({ serviceId: "osrm", bundle, rootDir: buildHost }); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle, + rootDir: buildHost, + }); await expect( importBuildBundle({ bundle, rootDir: otherServe, runningServices: noneRunning }), @@ -178,7 +276,12 @@ describe("build bundles", () => { it("refuses to swap while a consumer is running", async () => { seedOsrmBuild(buildHost); const bundle = join(tmp, "osrm.tar"); - await exportBuildBundle({ serviceId: "osrm", bundle, rootDir: buildHost }); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle, + rootDir: buildHost, + }); await expect( importBuildBundle({ @@ -192,7 +295,12 @@ describe("build bundles", () => { it("rejects a bundle whose contents no longer match its manifest", async () => { seedOsrmBuild(buildHost); const bundle = join(tmp, "osrm.tar"); - await exportBuildBundle({ serviceId: "osrm", bundle, rootDir: buildHost }); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle, + rootDir: buildHost, + }); const unpacked = join(tmp, "unpacked"); mkdirSync(unpacked); @@ -213,7 +321,12 @@ describe("build bundles", () => { it("rejects extra entries the manifest does not list", async () => { seedOsrmBuild(buildHost); const bundle = join(tmp, "osrm.tar"); - await exportBuildBundle({ serviceId: "osrm", bundle, rootDir: buildHost }); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle, + rootDir: buildHost, + }); const unpacked = join(tmp, "unpacked"); mkdirSync(unpacked); @@ -271,7 +384,12 @@ describe("build bundles", () => { const previous = process.env.INIT_CWD; process.env.INIT_CWD = tmp; try { - await exportBuildBundle({ serviceId: "osrm", bundle: "relative.tar", rootDir: buildHost }); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle: "relative.tar", + rootDir: buildHost, + }); expect(existsSync(join(tmp, "relative.tar"))).toBe(true); await importBuildBundle({ bundle: "relative.tar", @@ -292,14 +410,24 @@ describe("build bundles", () => { writeBuildRecord({ ...osrmRecord(), service: "motis", runtimeImages: {} }, buildHost); await expect( - exportBuildBundle({ serviceId: "motis", bundle: join(tmp, "m.tar"), rootDir: buildHost }), + exportBuildBundle({ + runningServices: noneRunning, + serviceId: "motis", + bundle: join(tmp, "m.tar"), + rootDir: buildHost, + }), ).rejects.toThrow(/without --import/); }); it("requires a build record before exporting", async () => { mkdirSync(join(dataDir(buildHost), "osrm-graph"), { recursive: true }); await expect( - exportBuildBundle({ serviceId: "osrm", bundle: join(tmp, "x.tar"), rootDir: buildHost }), + exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle: join(tmp, "x.tar"), + rootDir: buildHost, + }), ).rejects.toThrow(/No build record for "osrm"/); }); @@ -307,7 +435,12 @@ describe("build bundles", () => { seedOsrmBuild(buildHost); symlinkSync("/etc/passwd", join(dataDir(buildHost), "osrm-graph", "link")); await expect( - exportBuildBundle({ serviceId: "osrm", bundle: join(tmp, "x.tar"), rootDir: buildHost }), + exportBuildBundle({ + runningServices: noneRunning, + serviceId: "osrm", + bundle: join(tmp, "x.tar"), + rootDir: buildHost, + }), ).rejects.toThrow(/Refusing symlink in build artifact: osrm-graph\/link/); }); @@ -319,7 +452,12 @@ describe("build bundles", () => { writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "secret-api-key"); writeBuildRecord(motisRecord(), buildHost); await expect( - exportBuildBundle({ serviceId: "motis", bundle: join(tmp, "motis.tar"), rootDir: buildHost }), + exportBuildBundle({ + runningServices: noneRunning, + serviceId: "motis", + bundle: join(tmp, "motis.tar"), + rootDir: buildHost, + }), ).rejects.toThrow(/retained portable source metadata/); }); @@ -334,7 +472,12 @@ describe("build bundles", () => { ); writeBuildRecord(motisRecord(), buildHost); const bundle = join(tmp, "motis.tar"); - await exportBuildBundle({ serviceId: "motis", bundle, rootDir: buildHost }); + await exportBuildBundle({ + runningServices: noneRunning, + serviceId: "motis", + bundle, + rootDir: buildHost, + }); const target = dataDir(serveHost); const slotA = join(target, "motis/slots/A"); mkdirSync(join(slotA, "data"), { recursive: true }); @@ -342,7 +485,7 @@ describe("build bundles", () => { symlinkSync(slotA, join(target, "motis/live")); await expect( importBuildBundle({ bundle, rootDir: serveHost, runningServices: noneRunning }), - ).rejects.toThrow(/target credential materialization/); + ).rejects.toThrow(/generated destination configuration/); expect(lstatSync(join(target, "motis/live")).isSymbolicLink()).toBe(true); expect(readFileSync(join(slotA, "data/tt.bin"), "utf8")).toBe("OLD"); }); diff --git a/packages/cli/src/commands/data.ts b/packages/cli/src/commands/data.ts index 52b6e9279..c18afbcb4 100644 --- a/packages/cli/src/commands/data.ts +++ b/packages/cli/src/commands/data.ts @@ -3,6 +3,7 @@ import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts"; import * as dm from "@openmapx/core/data-manager-sdk"; import type { Command } from "commander"; import { + ArtifactImportCommittedError, BUILD_ARTIFACTS, buildArtifactSpec, exportBuildBundle, @@ -18,7 +19,7 @@ import { } from "../lib/data-local"; import { dataManagerUrl } from "../lib/data-manager-url"; import { runCliDataUpdate } from "../lib/data-update-command"; -import { runningComposeServices } from "../lib/docker"; +import { runningComposeServices, runningComposeServicesStrict } from "../lib/docker"; import { OPENMAPX_REGION_ENV, resolveOsmRegion, @@ -264,14 +265,18 @@ export function registerDataCommands(program: Command): void { .action(async (serviceId: string, bundle: string) => { if (bundle === "-") routeLogsToStderr(); try { - const running = await runningComposeServices(buildArtifactSpec(serviceId).consumers); + const running = await runningComposeServicesStrict(buildArtifactSpec(serviceId).consumers); if (running.length > 0) { throw new Error( `${running.join(", ")} ${running.length === 1 ? "is" : "are"} running and may be writing the artifact; stop first with \`openmapx services stop ${running.join(" ")}\``, ); } log.dim(`Hashing ${serviceId} artifact…`); - const manifest = await exportBuildBundle({ serviceId, bundle }); + const manifest = await exportBuildBundle({ + serviceId, + bundle, + runningServices: runningComposeServicesStrict, + }); const bytes = manifest.files.reduce((sum, file) => sum + file.sizeBytes, 0); log.ok( `Exported ${serviceId} (${manifest.files.length} files, ${formatBytes(bytes)}, region ${manifest.region ?? "unknown"}) → ${bundle === "-" ? "stdout" : bundle}`, @@ -285,27 +290,26 @@ export function registerDataCommands(program: Command): void { data .command("import ") .description( - "Verify a bundle written by `data export` (`-` reads stdin), check it matches this host's runtime images, swap it in, then render and link", + "Verify and journal-publish prepared data under its original target lease, then apply the current configuration's hardlinks without starting consumers (`-` reads stdin)", ) .action(async (bundle: string) => { let manifest: Awaited>; try { - manifest = await importBuildBundle({ bundle, runningServices: runningComposeServices }); + manifest = await importBuildBundle({ + bundle, + runningServices: runningComposeServicesStrict, + }); } catch (err) { - log.err(`import failed: ${(err as Error).message}`); + log.err( + err instanceof ArtifactImportCommittedError + ? err.message + : `import failed: ${(err as Error).message}`, + ); completeCliCommand(1); } log.ok( `Imported ${manifest.service} (${manifest.files.length} files, built ${manifest.builtAt}${manifest.revision ? ` at ${manifest.revision.slice(0, 12)}` : ""})`, ); - try { - await renderAndApplyHardlinks(); - } catch (err) { - log.err( - `The artifact is in place, but rendering and linking failed: ${(err as Error).message}. Fix that, then run \`openmapx data link\`.`, - ); - completeCliCommand(1); - } log.info( `Start the consumers with \`openmapx services start ${buildArtifactSpec(manifest.service).consumers.join(" ")}\``, ); diff --git a/packages/cli/src/lib/artifact-ownership.ts b/packages/cli/src/lib/artifact-ownership.ts new file mode 100644 index 000000000..3cee840a6 --- /dev/null +++ b/packages/cli/src/lib/artifact-ownership.ts @@ -0,0 +1,136 @@ +/** Standalone host-CLI roots for the two fixed artifact operations. */ +import { randomBytes, randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import type { ArtifactPublicationResult, OpsOperation, OpsResultFor } from "@openmapx/core/ops"; +import { + createControlRootOperationKey, + executionFingerprint, + opsExecutionBudget, + opsOperationFingerprint, +} from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { cliControlContext } from "./control-context"; +import { cliAcquisitionOptions, cliProcessOptions } from "./execution-deadline"; +import { repoPaths } from "./paths"; + +type ArtifactOperation = Extract< + OpsOperation, + { kind: "artifactTransfer.import" | "artifactTransfer.export" } +>; +export async function openCliArtifactOwner(operation: ArtifactOperation, rootDir?: string) { + if (cliControlContext()) + throw Error("Artifact transfers require their standalone original host owner"); + const paths = repoPaths(rootDir), + registry = new services.ServiceRegistry({ rootDir: paths.root }); + await registry.load(); + const targets = registry + .list() + .map((service) => service.manifest.id) + .sort(); + if (!existsSync("/run/secrets/openmapx-control-verifiers")) + await control.provisionControlInstallation(paths.infraDir); + const installation = control.openControlInstallation({ + infraDir: paths.infraDir, + role: "host-cli", + serviceIds: targets, + }); + let lease: control.TargetLeaseAuthority | undefined; + try { + const admittedAt = Date.now(), + budget = + cliProcessOptions(opsExecutionBudget(operation.kind).processMs).timeout ?? + opsExecutionBudget(operation.kind).processMs, + deadlineAt = admittedAt + budget, + operationFingerprint = opsOperationFingerprint(operation), + execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }; + const signal = AbortSignal.any([ + AbortSignal.timeout(budget), + ...(cliAcquisitionOptions().signal ? [cliAcquisitionOptions().signal!] : []), + ]); + const hardlinks = + operation.kind === "artifactTransfer.import" && operation.expectedCurrentRevisionId + ? control.planGenerationHardlinksOwner({ + infraDir: paths.infraDir, + parent: execution, + revisionId: operation.expectedCurrentRevisionId, + coordinatorMode: "require-running", + serviceIds: targets, + }) + : undefined; + const authority = control.retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: targets, + ...(hardlinks ? { reservations: hardlinks.reservation ? [hardlinks.reservation] : [] } : {}), + }); + if (hardlinks) control.retainGenerationHardlinksOwnerPlan(authority, hardlinks); + lease = await control.acquireArtifactTargetLease({ + dataRoot: join(paths.infraDir, "data"), + serviceIds: targets, + owner: authority, + signal, + }); + return { paths, authority, installation, lease, signal, hardlinks }; + } catch (error) { + if (lease) control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(installation); + throw error; + } +} +/** Export reads managed targets; it publishes no managed files. The fixed + * caller invokes this only after its external output stream has completed. */ +export async function settleCliReadOnlyExport(owner: CliArtifactOwner) { + if (control.executionAuthorityState(owner.authority).kind !== "artifactTransfer.export") + throw Error("Original read-only export required"); + control.retainControlEffectReceipt(owner.authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await control.settleArtifactTargetLeaseChild( + owner.lease, + control.verifyControlEffectSettlement(owner.installation, owner.authority.execution), + ); + await control.releaseArtifactTargetLease(owner.lease); +} +export type CliArtifactOwner = Awaited>; +export async function settleCliArtifactOwner( + owner: CliArtifactOwner, + result: ArtifactPublicationResult | OpsResultFor<"artifactTransfer.export">, +) { + const notCommitted = "outcome" in result && result.outcome === "not_committed"; + control.retainControlEffectReceipt(owner.authority, { + outcome: notCommitted ? "contained" : "committed", + publication: notCommitted ? "not_started" : "committed", + receipt: { kind: "ops-result", result }, + }); + await control.settleArtifactTargetLeaseChild( + owner.lease, + control.verifyControlEffectSettlement(owner.installation, owner.authority.execution), + ); + await control.releaseArtifactTargetLease(owner.lease); +} +export function closeCliArtifactOwner(owner: CliArtifactOwner) { + control.closeArtifactTargetLeaseHandle(owner.lease); + control.closeControlInstallation(owner.installation); +} diff --git a/packages/cli/src/lib/build-artifacts.ts b/packages/cli/src/lib/build-artifacts.ts index 977971add..b706fbcef 100644 --- a/packages/cli/src/lib/build-artifacts.ts +++ b/packages/cli/src/lib/build-artifacts.ts @@ -1,16 +1,16 @@ +import { randomUUID } from "node:crypto"; import { createReadStream, createWriteStream, existsSync, - lstatSync, mkdirSync, - mkdtempSync, readFileSync, - realpathSync, rmSync, + statfsSync, } from "node:fs"; -import { dirname, join, relative } from "node:path"; +import { join } from "node:path"; import { Writable } from "node:stream"; +import * as control from "@openmapx/core/build-artifacts"; import { BUILD_RECORD_DIR, type BuildRecord, @@ -18,16 +18,33 @@ import { type BundleManifest, buildArtifactSpec, buildRecordSchema, - capturePreparedBuildEvidence, DEFAULT_ARTIFACT_ARCHIVE_LIMITS, - safeArtifactPath, runtimeImageMismatches as sharedRuntimeImageMismatches, validateBuildBundle, withDataDirectory, writeBuildBundle, } from "@openmapx/core/build-artifacts"; +import { + inspectMotisPortableDestination, + materializeMotisPortableDestination, +} from "@openmapx/core/build-artifacts/motis-destination"; +import { + artifactPublicationFingerprint, + publishValidatedBuild, +} from "@openmapx/core/build-artifacts/publication"; +import * as dm from "@openmapx/core/data-manager-sdk"; +import { artifactExportOperationSchema } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; import { execa } from "execa"; -import { promoteStagedBuildDirs } from "./build-staging"; +import { + type CliArtifactOwner, + closeCliArtifactOwner, + openCliArtifactOwner, + settleCliArtifactOwner, + settleCliReadOnlyExport, +} from "./artifact-ownership"; +import { dataManagerUrl } from "./data-manager-url"; +import { dockerCompose, runningComposeServicesStrict } from "./docker"; import { assertCliExecutionAvailable, cliAcquisitionOptions, @@ -96,6 +113,7 @@ export type BundleCommandRunner = ( opts: { cwd?: string; stdin?: "inherit"; stdout?: "inherit" }, ) => Promise; export interface ExportBuildBundleOptions { + runningServices?: (ids: string[]) => Promise; serviceId: string; bundle: string; rootDir?: string; @@ -116,31 +134,88 @@ export async function exportBuildBundle(opts: ExportBuildBundleOptions): Promise throw Error( "MOTIS build ran without --import; rebuild with openmapx services build motis --import", ); - const target = - opts.bundle === "-" - ? new Writable({ - write(chunk, encoding, done) { - process.stdout.write(chunk, encoding, done); - }, - }) - : createWriteStream(resolveInvocationPath(opts.bundle), { flags: "wx", mode: 0o600 }); - target.on("error", () => undefined); + const runningServices = + opts.runningServices ?? ((ids: string[]) => runningComposeServicesStrict(ids, opts.rootDir)); + const compatible = async () => { + if ((await runningServices(buildArtifactSpec(opts.serviceId).consumers)).length) + throw Error("Prepared data consumers must be stopped before exporting"); + if ((await runtimeImageMismatches(record.runtimeImages, opts.rootDir)).length) + throw Error("Prepared data runtime images differ from this host"); + }; + await compatible(); + let expectedArtifactFingerprint: string; + if (record.schemaVersion === 2 && opts.serviceId !== "motis") + expectedArtifactFingerprint = record.artifactFingerprint; + else { + // Legacy records have no retained file digest. An inert first pass derives + // the exact immutable request; the leased output pass must still match it. + const inspected = await writeBuildBundle({ + dataRoot, + serviceId: opts.serviceId, + record, + target: new Writable({ + write(_chunk, _encoding, done) { + done(); + }, + }), + limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: cliAcquisitionOptions().signal ?? new AbortController().signal, + }); + expectedArtifactFingerprint = inspected.artifactFingerprint; + } + const owner = await openCliArtifactOwner( + artifactExportOperationSchema.parse({ + kind: "artifactTransfer.export", + serviceId: opts.serviceId, + ownerId: "host-cli", + expectedArtifactFingerprint, + }), + opts.rootDir, + ); + let target: Writable | undefined; try { - return ( - await writeBuildBundle({ - dataRoot, - serviceId: opts.serviceId, - record, - target, - limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, - signal: cliAcquisitionOptions().signal ?? new AbortController().signal, - }) - ).manifest; - } catch (e) { - if (target !== process.stdout) target.destroy(); - throw e; + await compatible(); + target = + opts.bundle === "-" + ? new Writable({ + write(chunk, encoding, done) { + process.stdout.write(chunk, encoding, done); + }, + }) + : createWriteStream(resolveInvocationPath(opts.bundle), { flags: "wx", mode: 0o600 }); + target.on("error", () => undefined); + const written = await services.withConfigurationGenerationWriter( + owner.paths.infraDir, + async () => { + const revision = services.currentConfigurationGeneration(owner.paths.infraDir); + const result = await writeBuildBundle({ + dataRoot, + serviceId: opts.serviceId, + record, + target: target!, + limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: owner.signal, + }); + await compatible(); + if ( + result.artifactFingerprint !== expectedArtifactFingerprint || + revision !== services.currentConfigurationGeneration(owner.paths.infraDir) + ) + throw Error("Prepared export source changed"); + return result; + }, + owner.signal, + ); + await settleCliReadOnlyExport(owner); + return written.manifest; + } catch (error) { + target?.destroy(); + throw error; + } finally { + closeCliArtifactOwner(owner); } } + export async function runtimeImageMismatches(images: Record, rootDir?: string) { return sharedRuntimeImageMismatches(images, repoPaths(rootDir).root); } @@ -150,60 +225,205 @@ export interface ImportBuildBundleOptions { tar?: BundleCommandRunner; runningServices: (ids: string[]) => Promise; } +export class ArtifactImportCommittedError extends Error { + constructor() { + super( + "Artifacts were committed, but hardlinks were not applied. Keep consumers stopped; reconcile the current generation before Start.", + ); + } +} export async function importBuildBundle(opts: ImportBuildBundleOptions): Promise { - const dataRoot = dataRootFor(opts.rootDir); + const paths = repoPaths(opts.rootDir), + dataRoot = dataRootFor(opts.rootDir), + stageId = randomUUID(); mkdirSync(dataRoot, { recursive: true }); - const stagingRoot = mkdtempSync(join(dataRoot, ".openmapx-import-")); + const stagingRoot = join(dataRoot, `.openmapx-import-${stageId}`); + mkdirSync(stagingRoot, { mode: 0o700 }); + let owner: CliArtifactOwner | undefined, + discard = true, + settled = false; try { - const { manifest } = await validateBuildBundle({ + const bundle = await validateBuildBundle({ source: opts.bundle === "-" ? process.stdin : createReadStream(resolveInvocationPath(opts.bundle)), stagingRoot, limits: DEFAULT_ARTIFACT_ARCHIVE_LIMITS, signal: cliAcquisitionOptions().signal ?? new AbortController().signal, }); - const spec = buildArtifactSpec(manifest.service); - const mismatches = await runtimeImageMismatches(manifest.runtimeImages, opts.rootDir); - if (mismatches.length) - throw Error( - `Bundle was built for different runtime images (${mismatches.map((m) => `${m.service}: bundle ${m.bundle}, this host ${m.local}`).join("; ")}). Build and serve from the same OpenMapX revision.`, + const { manifest } = bundle, + spec = buildArtifactSpec(manifest.service); + const compatible = async () => { + const mismatches = await runtimeImageMismatches(manifest.runtimeImages, opts.rootDir); + if (mismatches.length) + throw Error( + `Bundle was built for different runtime images (${mismatches.map((m) => `${m.service}: bundle ${m.bundle}, this host ${m.local}`).join("; ")}). Build and serve from the same OpenMapX revision.`, + ); + const running = await opts.runningServices(spec.consumers); + if (running.length) + throw Error( + `Refusing to import: ${running.join(", ")} ${running.length === 1 ? "is" : "are"} running. Stop first with openmapx services stop ${running.join(" ")}.`, + ); + }; + await compatible(); + const motisDestination = async (signal?: AbortSignal) => { + const revision = services.currentConfigurationGeneration(paths.infraDir); + if (!revision) throw Error("MOTIS import requires a generated destination configuration"); + const config = await dockerCompose(["config", "--format", "json"], opts.rootDir); + if (config.exitCode !== 0) throw Error("Destination Compose configuration is unavailable"); + const parsed: unknown = JSON.parse(config.stdout); + if (!parsed || typeof parsed !== "object") + throw Error("Invalid destination Compose configuration"); + const environment = (parsed as { services?: Record }) + .services?.["data-manager"]?.environment; + if ( + environment !== undefined && + (!environment || typeof environment !== "object" || Array.isArray(environment)) + ) + throw Error("Invalid data manager environment"); + const env = Object.fromEntries( + Object.entries(environment ?? {}).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), ); - if (manifest.schemaVersion === 2 && manifest.runtimeMaterialization) - throw Error( - "MOTIS portable bundle requires the target credential materialization workflow before readiness; rebuild locally until that workflow is available", - ); - const running = await opts.runningServices(spec.consumers); - if (running.length) - throw Error( - `Refusing to import: ${running.join(", ")} ${running.length === 1 ? "is" : "are"} running. Stop first with openmapx services stop ${running.join(" ")}.`, - ); - const swaps = spec.dirs.map((dir) => { - const live = join(dataRoot, dir); - mkdirSync(dirname(live), { recursive: true }); - const liveDir = - existsSync(live) && lstatSync(live).isSymbolicLink() ? realpathSync(live) : live; - const physical = relative( - realpathSync(dataRoot), - existsSync(liveDir) - ? realpathSync(liveDir) - : join(realpathSync(dirname(liveDir)), dir.split("/").at(-1)!), - ); - if (!safeArtifactPath(physical)) throw Error("Artifact target alias leaves data root"); - return { liveDir, nextDir: join(stagingRoot, dir) }; + return { + rootDir: paths.root, + infraDir: paths.infraDir, + stagingDir: join(stagingRoot, "motis/live"), + configurationRevisionId: revision, + dataManagerEnvironment: env, + signal, + }; + }; + const motisPreview = + manifest.service === "motis" + ? await inspectMotisPortableDestination( + await motisDestination(cliAcquisitionOptions().signal), + ) + : null; + const expectedFingerprint = await artifactPublicationFingerprint(dataRoot, manifest.service), + expectedCurrentRevisionId = services.currentConfigurationGeneration(paths.infraDir); + owner = await openCliArtifactOwner( + { + kind: "artifactTransfer.import", + stageId, + ownerId: "host-cli", + expectedSha256: bundle.archiveSha256, + previewFingerprint: control.controlDigest({ + expectedFingerprint, + expectedCurrentRevisionId, + archive: bundle.archiveSha256, + }), + expectedCurrentRevisionId, + }, + opts.rootDir, + ); + const transport = dm.createDataManagerTransport({ + baseUrl: dataManagerUrl(), + authToken: process.env.DATA_MANAGER_AUTH_TOKEN?.trim(), + requestTimeoutMs: 15000, + maxJsonBytes: 65536, }); + const requireCoordinator = async () => { + if ( + owner?.hardlinks?.plan.plan.length && + !dm.unwrapDataManagerResponse(await dm.getStatus({ transport, signal: owner.signal })).ok + ) + throw Error("Data manager must already be running to apply imported hardlinks"); + }; + await requireCoordinator(); assertCliExecutionAvailable(); - promoteStagedBuildDirs(swaps); - const { dirs: _dirs, directories: _directories, files: _files, ...record } = manifest; - if (record.schemaVersion === 2) - await capturePreparedBuildEvidence({ - dataRoot, - record, - verifiedFiles: manifest.files, - signal: cliAcquisitionOptions().signal, - }); - writeBuildRecord(record, opts.rootDir); + discard = false; + const materializeMotis = motisPreview + ? async () => { + const capacity = statfsSync(dataRoot, { bigint: true }); + if ( + capacity.bavail * capacity.bsize < BigInt(64 * 1024 * 1024 + 1024 ** 3) || + capacity.ffree < 64n + ) + throw Error("Insufficient destination capacity for MOTIS materialization"); + return ( + await materializeMotisPortableDestination({ + ...(await motisDestination(owner!.signal)), + expectedFingerprint: motisPreview.fingerprint, + archiveFingerprint: bundle.artifactFingerprint, + }) + ).materialization; + } + : undefined; + const result = await publishValidatedBuild({ + bundle, + materializeMotis, + stagingRoot, + stageId, + dataRoot, + authority: owner!.authority, + lease: owner!.lease, + expectedFingerprint, + signal: owner!.signal, + beforePublication: async () => { + await compatible(); + await requireCoordinator(); + if ( + motisPreview && + (await inspectMotisPortableDestination(await motisDestination(owner!.signal))) + .fingerprint !== motisPreview.fingerprint + ) + throw Error("MOTIS destination changed before publication"); + return () => {}; + }, + }); + if (result.outcome !== "committed") { + await settleCliArtifactOwner(owner, result); + settled = true; + discard = !result.cleanupFailed; + throw Error( + `Prepared artifacts were not committed (${result.reason}); existing data was preserved.`, + ); + } + let followUp: "complete" | "failed" = "complete"; + if (owner.hardlinks) { + try { + await services.withConfigurationGenerationWriter( + paths.infraDir, + () => + control.executeOwnedGenerationHardlinks({ + authority: owner!.authority, + lease: owner!.lease, + infraDir: paths.infraDir, + transport, + signal: owner!.signal, + }), + owner.signal, + ); + } catch (error) { + if ( + owner.hardlinks.reservation && + control.lookupControlChildPhaseBinding(owner.authority, owner.hardlinks.reservation) + ) + throw Error( + "Artifacts committed; original hardlink execution is unresolved. Retain the import workspace and reconcile its original owner.", + { cause: error }, + ); + followUp = "failed"; + } + } + await settleCliArtifactOwner(owner, { ...result, followUp }); + settled = true; + discard = !result.cleanupFailed; + if (followUp === "failed") throw new ArtifactImportCommittedError(); return manifest; + } catch (error) { + if (owner && discard && !settled) { + await settleCliArtifactOwner(owner, { + outcome: "not_committed", + reason: "failed", + cleanupFailed: false, + }); + settled = true; + } + throw error; } finally { - rmSync(stagingRoot, { recursive: true, force: true }); + if (owner) closeCliArtifactOwner(owner); + if (discard) rmSync(stagingRoot, { recursive: true, force: true }); } } diff --git a/packages/cli/src/lib/docker.ts b/packages/cli/src/lib/docker.ts index a2768f886..6dd0801e6 100644 --- a/packages/cli/src/lib/docker.ts +++ b/packages/cli/src/lib/docker.ts @@ -5,8 +5,9 @@ import { repoPaths } from "./paths"; export async function dockerCompose( args: string[], + rootDir?: string, ): Promise<{ stdout: string; stderr: string; exitCode: number }> { - const paths = repoPaths(); + const paths = repoPaths(rootDir); const result = await execa("docker", [...coreServices.stackComposeArgs(paths), ...args], { ...cliProcessOptions(), cwd: paths.infraDir, @@ -65,3 +66,20 @@ export async function runningComposeServices(serviceIds: string[]): Promise line.trim()) .filter((line) => line.length > 0 && serviceIds.includes(line)); } +/** Artifact movement must prove consumers stopped; acquisition failure is unknown. */ +export async function runningComposeServicesStrict( + serviceIds: string[], + rootDir?: string, +): Promise { + if (!serviceIds.length) return []; + const result = await dockerCompose( + ["ps", "--status=running", "--services", ...serviceIds], + rootDir, + ); + if (result.exitCode !== 0) + throw Error("Container state is unavailable; no prepared artifacts were moved"); + return result.stdout + .split("\n") + .map((value) => value.trim()) + .filter((value) => serviceIds.includes(value)); +} diff --git a/packages/core/package.json b/packages/core/package.json index 5783ef2ee..18587d89b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -217,7 +217,12 @@ "./build-artifacts/publication": { "types": "./src/build-artifacts/publication.ts", "default": "./src/build-artifacts/publication.ts" - } + }, + "./build-artifacts/motis-portable": { + "types": "./src/build-artifacts/motis-portable.ts", + "default": "./src/build-artifacts/motis-portable.ts" + }, + "./build-artifacts/motis-destination": "./src/build-artifacts/motis-destination.ts" }, "scripts": { "check-types": "tsc --noEmit" @@ -273,7 +278,8 @@ "tz-lookup": "^6.1.25", "undici": "8.11.2", "zod": "^4.6.5", - "zustand": "^5.0.15" + "zustand": "^5.0.15", + "@openmapx/motis-feed-proxy-config": "workspace:^" }, "private": true } diff --git a/packages/core/src/api/contracts/integration-admin/configuration-migration.ts b/packages/core/src/api/contracts/integration-admin/configuration-migration.ts new file mode 100644 index 000000000..0588c6754 --- /dev/null +++ b/packages/core/src/api/contracts/integration-admin/configuration-migration.ts @@ -0,0 +1,138 @@ +import z from "zod/v4"; +import { standaloneInventoryEntrySchema } from "../../../ops/standalone-integrations"; +import { platformConfigurationSchema } from "../../../services/platform-configuration"; +import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; +import { configurationOwnershipSchema } from "../configuration-ownership"; +import { + adminPlatformConfirmationSchema, + adminPlatformPreviewSchema, +} from "../service-admin/platform"; +import { migrationOperationResultSchema } from "./credential-transfer"; + +const value = z.json(); +const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const values = z + .record(z.string().min(1).max(128), value) + .refine( + (v) => + Object.keys(v).length <= 128 && + Object.keys(v).every((k) => !["__proto__", "prototype", "constructor"].includes(k)), + ); +export const migrationPrerequisiteSchema = z.strictObject({ + scope: z.enum(["integration", "service", "platform", "account"]), + id: z.string().min(1).max(128), + key: z.string().max(128).optional(), + source: z.enum(["default", "database", "vault", "config.json", "env", "missing"]), + reason: z.enum([ + "host_owned", + "credential_required", + "installation_required", + "unsupported", + "protected_bootstrap", + ]), +}); +export const configurationMigrationDocumentSchema = z + .strictObject({ + format: z.literal("openmapx-configuration"), + version: z.literal(1), + exportedAt: z.iso.datetime(), + settings: values, + platform: platformConfigurationSchema.optional(), + integrations: z.array(z.strictObject({ integrationId: id, config: values })).max(256), + serviceConfigs: z.array(z.strictObject({ serviceId: id, config: values })).max(256), + bindings: z + .array( + z.strictObject({ + integrationId: id, + capability: z.string().min(1).max(128), + serviceId: id, + }), + ) + .max(512), + installations: z.array(standaloneInventoryEntrySchema).max(2048), + prerequisites: z.array(migrationPrerequisiteSchema).max(1000), + }) + .refine( + (v) => + new TextEncoder().encode(JSON.stringify(v)).length <= 1048576 && + Object.keys(v.settings).length + + v.integrations.reduce((n, e) => n + Object.keys(e.config).length, 0) + + v.serviceConfigs.reduce((n, e) => n + Object.keys(e.config).length, 0) + + v.bindings.length <= + 1000, + ); +export type ConfigurationMigrationDocument = z.infer; +export type MigrationPrerequisite = z.infer; +export const configurationMigrationInventorySchema = z.strictObject({ + document: configurationMigrationDocumentSchema, + fields: z + .array( + z.strictObject({ + scope: z.enum(["integration", "service", "platform", "account"]), + id: z.string(), + key: z.string(), + ownership: configurationOwnershipSchema, + included: z.boolean(), + }), + ) + .max(1000), +}); +export const configurationMigrationPreviewSchema = z.strictObject({ + previewId: z.uuid(), + fingerprint: z.string(), + expectedCurrentRevisionId: z.string().nullable(), + changes: z + .array(z.strictObject({ scope: z.string(), id: z.string(), keys: z.array(z.string()) })) + .max(512), + prerequisites: z.array(migrationPrerequisiteSchema).max(1000), + denials: z.array(z.string()).max(1000), + expiresAt: z.iso.datetime(), + platform: adminPlatformPreviewSchema.optional(), +}); +export type ConfigurationMigrationPreview = z.infer; +const headers = z.object({ "idempotency-key": z.string().min(1).max(128) }); +const transferResponses = { + ...adminCommonResponses, + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), +}; +export const configurationMigrationOperations = { + getAdminConfigurationMigrationInventory: { + method: "GET", + path: "/api/admin/configuration-migration/inventory", + auth: "admin", + responses: { ...transferResponses, 200: adminJson(configurationMigrationInventorySchema) }, + }, + exportAdminConfigurationMigration: { + method: "GET", + path: "/api/admin/configuration-migration/export", + auth: "admin", + responses: { ...transferResponses, 200: adminJson(configurationMigrationDocumentSchema) }, + }, + previewAdminConfigurationMigration: { + method: "POST", + path: "/api/admin/configuration-migration/preview", + auth: "admin", + request: { body: configurationMigrationDocumentSchema, headers }, + responses: { ...transferResponses, 200: adminJson(configurationMigrationPreviewSchema) }, + }, + applyAdminConfigurationMigration: { + method: "POST", + path: "/api/admin/configuration-migration/apply", + auth: "admin", + request: { + body: z.strictObject({ + previewId: z.uuid(), + fingerprint: z.string().min(1).max(128), + platformAuthorization: z + .strictObject({ + confirmations: adminPlatformConfirmationSchema, + proofToken: z.uuid().optional(), + }) + .optional(), + }), + headers, + }, + responses: { ...transferResponses, 200: adminJson(migrationOperationResultSchema) }, + }, +} as const satisfies Record; diff --git a/packages/core/src/api/contracts/integration-admin/credential-transfer.ts b/packages/core/src/api/contracts/integration-admin/credential-transfer.ts new file mode 100644 index 000000000..4f8181679 --- /dev/null +++ b/packages/core/src/api/contracts/integration-admin/credential-transfer.ts @@ -0,0 +1,171 @@ +import { isWellFormedFeedCredentialText } from "@openmapx/transitous-core/feed-credentials"; +import z from "zod/v4"; +import { type AdminOperation, adminCommonResponses, adminJson, httpErrorSchema } from "../admin"; + +/** Public checked reference for decrypt-only review, never apply authority. */ +export const CREDENTIAL_TRANSFER_PREVIEW_INTENT = + "ee5994a7b7b38c0a7039f3f8ac7f3d16d74f6b8ffa45210e7d94190d93748bd0"; +const encoded = (bytes: number) => + z + .string() + .regex(/^[A-Za-z0-9_-]+$/) + .length(Math.ceil((bytes * 4) / 3)); +const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); +const key = z + .string() + .min(1) + .max(128) + .refine((v) => !["__proto__", "constructor", "prototype"].includes(v)); +export const credentialTransferFieldSchema = z.strictObject({ + scope: z.enum(["integration", "service"]), + id, + key, +}); +export const credentialTransferEnvelopeSchema = z.strictObject({ + format: z.literal("openmapx-vault-transfer"), + version: z.literal(1), + kdf: z.strictObject({ + name: z.literal("scrypt"), + N: z.literal(32768), + r: z.literal(8), + p: z.literal(1), + salt: encoded(16), + }), + cipher: z.strictObject({ name: z.literal("aes-256-gcm"), iv: encoded(12), tag: encoded(16) }), + ciphertext: z + .string() + .min(1) + .max(1048576) + .regex(/^[A-Za-z0-9_-]+$/), +}); +export type CredentialTransferEnvelope = z.infer; +export const credentialImportChoicesSchema = z.strictObject({ + entries: z + .array(credentialTransferFieldSchema.extend({ action: z.enum(["import", "skip"]) })) + .max(1000), + feedMap: z + .strictObject({ + catalogRevisionId: z.string().min(1).max(256), + expectedCredentialsRevision: z.string().min(1).max(256), + setKeys: z.array(z.string().min(1).max(1024)).max(1000), + deleteKeys: z.array(z.string().min(1).max(1024)).max(1000), + }) + .optional(), +}); +export type CredentialImportChoices = z.infer; +const passphrase = z + .string() + .refine( + (v) => + isWellFormedFeedCredentialText(v) && + new TextEncoder().encode(v).length >= 12 && + new TextEncoder().encode(v).length <= 1024, + ); +export const credentialExportInputSchema = z.strictObject({ + selection: z.array(credentialTransferFieldSchema).min(1).max(1000), + passphrase, + confirmed: z.literal(true), +}); +export type CredentialExportInput = z.infer; +export const credentialImportPreviewSchema = z.strictObject({ + previewId: z.uuid(), + intentFingerprint: z.string().min(1).max(128), + expectedCurrentRevisionId: z.string().nullable(), + expiresAt: z.iso.datetime(), + entries: z + .array( + credentialTransferFieldSchema.extend({ + status: z.enum(["selected", "skipped", "denied"]), + reason: z.string().optional(), + }), + ) + .max(1000), + feedMap: z + .strictObject({ + previewId: z.uuid(), + intentFingerprint: z.string(), + catalogRevisionId: z.string(), + credentialsRevision: z.string(), + expectedCurrentRevisionId: z.string().nullable(), + expiresAt: z.iso.datetime(), + changes: z + .array(z.strictObject({ key: z.string(), action: z.enum(["set", "delete"]) })) + .max(1000), + preservedCount: z.number().int().nonnegative(), + mapBytes: z.number().int().nonnegative(), + }) + .optional(), + applyable: z.boolean(), +}); +export type CredentialImportPreview = z.infer; +export const credentialTransferApplySchema = z.strictObject({ + previewId: z.uuid(), + intentFingerprint: z.string().min(1).max(128), + expectedCurrentRevisionId: z.string().nullable(), + importProofToken: z.uuid(), + transitProofToken: z.uuid().optional(), +}); +const headers = z.object({ + "idempotency-key": z.string().min(1).max(128), + "x-admin-reauth-proof": z.uuid().optional(), +}); +export const migrationOperationResultSchema = z.strictObject({ + operationId: z.string(), + outcome: z.enum(["applied", "partial", "failed", "unknown"]), + steps: z.array( + z.strictObject({ + scope: z.string(), + id: z.string(), + status: z.enum(["applied", "failed", "pending", "unknown"]), + }), + ), +}); +const transferResponses = { + ...adminCommonResponses, + 409: adminJson(httpErrorSchema), + 503: adminJson(httpErrorSchema), +}; +export const credentialTransferOperations = { + exportAdminCredentialTransfer: { + method: "POST", + path: "/api/admin/credential-transfer/export", + auth: "admin", + request: { body: credentialExportInputSchema }, + responses: { ...transferResponses, 200: adminJson(credentialTransferEnvelopeSchema) }, + }, + previewAdminCredentialTransfer: { + method: "POST", + path: "/api/admin/credential-transfer/preview", + auth: "admin", + request: { + body: z.strictObject({ + envelope: credentialTransferEnvelopeSchema, + passphrase, + choices: credentialImportChoicesSchema.optional(), + }), + headers, + }, + responses: { ...transferResponses, 200: adminJson(credentialImportPreviewSchema) }, + }, + chooseAdminCredentialTransferFields: { + method: "POST", + path: "/api/admin/credential-transfer/preview/:previewId/choices", + auth: "admin", + request: { + params: z.strictObject({ previewId: z.uuid() }), + body: z.strictObject({ + choices: credentialImportChoicesSchema, + expectedIntentFingerprint: z.string().min(1).max(128), + }), + headers, + }, + responses: { ...transferResponses, 200: adminJson(credentialImportPreviewSchema) }, + }, + applyAdminCredentialTransfer: { + method: "POST", + path: "/api/admin/credential-transfer/apply", + auth: "admin", + request: { body: credentialTransferApplySchema, headers }, + responses: { ...transferResponses, 200: adminJson(migrationOperationResultSchema) }, + }, +} as const satisfies Record; diff --git a/packages/core/src/api/contracts/integration-admin/index.ts b/packages/core/src/api/contracts/integration-admin/index.ts index a89b0ddcd..acfac9cf3 100644 --- a/packages/core/src/api/contracts/integration-admin/index.ts +++ b/packages/core/src/api/contracts/integration-admin/index.ts @@ -1,8 +1,9 @@ export * from "./application"; export * from "./common"; +export * from "./configuration-migration"; +export * from "./credential-transfer"; export * from "./credentials"; export * from "./health"; export * from "./operations"; export * from "./overview"; - export * from "./standalone-integrations"; diff --git a/packages/core/src/api/contracts/integration-admin/operations.ts b/packages/core/src/api/contracts/integration-admin/operations.ts index 5aa92bd13..04fc3c775 100644 --- a/packages/core/src/api/contracts/integration-admin/operations.ts +++ b/packages/core/src/api/contracts/integration-admin/operations.ts @@ -29,6 +29,8 @@ import { adminIntegrationValidationErrorSchema, adminIntegrationVaultUnavailableSchema, } from "./common"; +import { configurationMigrationOperations } from "./configuration-migration"; +import { credentialTransferOperations } from "./credential-transfer"; import { adminCredentialsSchema, adminIntegrationCredentialDeleteSchema, @@ -44,7 +46,6 @@ import { adminIntegrationHealthSweepSchema, } from "./health"; import { adminOverviewSchema } from "./overview"; - import { standaloneIntegrationAdminOperations } from "./standalone-integrations"; const common = adminCommonResponses; @@ -65,6 +66,8 @@ const _ok = adminJson(z.strictObject({ ok: z.literal(true) })); /** Complete integration/credential/overview family. Authentication metadata never enforces access. */ export const integrationAdminOperations = { + ...configurationMigrationOperations, + ...credentialTransferOperations, ...standaloneIntegrationAdminOperations, listAdminIntegrations: { method: "GET", diff --git a/packages/core/src/api/contracts/service-admin/artifact-transfers.ts b/packages/core/src/api/contracts/service-admin/artifact-transfers.ts index da09bf41b..a9807f66b 100644 --- a/packages/core/src/api/contracts/service-admin/artifact-transfers.ts +++ b/packages/core/src/api/contracts/service-admin/artifact-transfers.ts @@ -1,7 +1,9 @@ import z from "zod/v4"; import { + artifactExportOperationSchema, artifactImportOperationSchema, artifactImportPreviewSchema, + artifactTransferResultSchemas, transferPurposeSchema, transferStageSchema, } from "../../../ops/artifact-transfer"; @@ -49,7 +51,35 @@ export const adminArtifactImportInputSchema = artifactImportOperationSchema.omit stageId: true, ownerId: true, }); +export const adminArtifactExportInputSchema = artifactExportOperationSchema.omit({ + kind: true, + ownerId: true, +}); export const artifactTransferAdminOperations = { + getAdminArtifactTransferCapabilities: { + method: "GET", + path: "/api/admin/artifact-transfers/capabilities", + auth: "admin", + responses: { + 200: adminJson(artifactTransferResultSchemas["artifactTransfer.capabilities"]), + ...errors, + }, + }, + exportAdminArtifactTransfer: { + method: "POST", + path: "/api/admin/artifact-transfers/exports", + auth: "admin", + request: { + body: adminArtifactExportInputSchema, + headers: z.object({ + "idempotency-key": z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/), + }), + }, + responses: { + 202: adminJson(z.strictObject({ ok: z.literal(true), jobId: z.uuid() })), + ...errors, + }, + }, previewAdminArtifactImport: { method: "POST", path: "/api/admin/artifact-transfers/:stageId/preview-import", diff --git a/packages/core/src/api/generated/admin/integration-admin.generated.ts b/packages/core/src/api/generated/admin/integration-admin.generated.ts index 7de525e22..42b4dd868 100644 --- a/packages/core/src/api/generated/admin/integration-admin.generated.ts +++ b/packages/core/src/api/generated/admin/integration-admin.generated.ts @@ -13,6 +13,42 @@ import type { AdoptAdminStandaloneIntegration503, AdoptAdminStandaloneIntegrationBody, AdoptAdminStandaloneIntegrationHeaders, + ApplyAdminConfigurationMigration200, + ApplyAdminConfigurationMigration400, + ApplyAdminConfigurationMigration401, + ApplyAdminConfigurationMigration403, + ApplyAdminConfigurationMigration409, + ApplyAdminConfigurationMigration413, + ApplyAdminConfigurationMigration415, + ApplyAdminConfigurationMigration429, + ApplyAdminConfigurationMigration500, + ApplyAdminConfigurationMigration503, + ApplyAdminConfigurationMigrationBody, + ApplyAdminConfigurationMigrationHeaders, + ApplyAdminCredentialTransfer200, + ApplyAdminCredentialTransfer400, + ApplyAdminCredentialTransfer401, + ApplyAdminCredentialTransfer403, + ApplyAdminCredentialTransfer409, + ApplyAdminCredentialTransfer413, + ApplyAdminCredentialTransfer415, + ApplyAdminCredentialTransfer429, + ApplyAdminCredentialTransfer500, + ApplyAdminCredentialTransfer503, + ApplyAdminCredentialTransferBody, + ApplyAdminCredentialTransferHeaders, + ChooseAdminCredentialTransferFields200, + ChooseAdminCredentialTransferFields400, + ChooseAdminCredentialTransferFields401, + ChooseAdminCredentialTransferFields403, + ChooseAdminCredentialTransferFields409, + ChooseAdminCredentialTransferFields413, + ChooseAdminCredentialTransferFields415, + ChooseAdminCredentialTransferFields429, + ChooseAdminCredentialTransferFields500, + ChooseAdminCredentialTransferFields503, + ChooseAdminCredentialTransferFieldsBody, + ChooseAdminCredentialTransferFieldsHeaders, DeleteAdminIntegrationBinding200, DeleteAdminIntegrationBinding400, DeleteAdminIntegrationBinding401, @@ -62,6 +98,27 @@ import type { EnableAdminIntegration500, EnableAdminIntegrationBody, EnableAdminIntegrationHeaders, + ExportAdminConfigurationMigration200, + ExportAdminConfigurationMigration400, + ExportAdminConfigurationMigration401, + ExportAdminConfigurationMigration403, + ExportAdminConfigurationMigration409, + ExportAdminConfigurationMigration413, + ExportAdminConfigurationMigration415, + ExportAdminConfigurationMigration429, + ExportAdminConfigurationMigration500, + ExportAdminConfigurationMigration503, + ExportAdminCredentialTransfer200, + ExportAdminCredentialTransfer400, + ExportAdminCredentialTransfer401, + ExportAdminCredentialTransfer403, + ExportAdminCredentialTransfer409, + ExportAdminCredentialTransfer413, + ExportAdminCredentialTransfer415, + ExportAdminCredentialTransfer429, + ExportAdminCredentialTransfer500, + ExportAdminCredentialTransfer503, + ExportAdminCredentialTransferBody, ExportAdminIntegrationConfigs200, ExportAdminIntegrationConfigs400, ExportAdminIntegrationConfigs401, @@ -70,6 +127,16 @@ import type { ExportAdminIntegrationConfigs415, ExportAdminIntegrationConfigs429, ExportAdminIntegrationConfigs500, + GetAdminConfigurationMigrationInventory200, + GetAdminConfigurationMigrationInventory400, + GetAdminConfigurationMigrationInventory401, + GetAdminConfigurationMigrationInventory403, + GetAdminConfigurationMigrationInventory409, + GetAdminConfigurationMigrationInventory413, + GetAdminConfigurationMigrationInventory415, + GetAdminConfigurationMigrationInventory429, + GetAdminConfigurationMigrationInventory500, + GetAdminConfigurationMigrationInventory503, GetAdminIntegration200, GetAdminIntegration400, GetAdminIntegration401, @@ -190,6 +257,30 @@ import type { PatchAdminIntegrationConfig500, PatchAdminIntegrationConfigBody, PatchAdminIntegrationConfigHeaders, + PreviewAdminConfigurationMigration200, + PreviewAdminConfigurationMigration400, + PreviewAdminConfigurationMigration401, + PreviewAdminConfigurationMigration403, + PreviewAdminConfigurationMigration409, + PreviewAdminConfigurationMigration413, + PreviewAdminConfigurationMigration415, + PreviewAdminConfigurationMigration429, + PreviewAdminConfigurationMigration500, + PreviewAdminConfigurationMigration503, + PreviewAdminConfigurationMigrationBody, + PreviewAdminConfigurationMigrationHeaders, + PreviewAdminCredentialTransfer200, + PreviewAdminCredentialTransfer400, + PreviewAdminCredentialTransfer401, + PreviewAdminCredentialTransfer403, + PreviewAdminCredentialTransfer409, + PreviewAdminCredentialTransfer413, + PreviewAdminCredentialTransfer415, + PreviewAdminCredentialTransfer429, + PreviewAdminCredentialTransfer500, + PreviewAdminCredentialTransfer503, + PreviewAdminCredentialTransferBody, + PreviewAdminCredentialTransferHeaders, PreviewAdminStandaloneIntegration202, PreviewAdminStandaloneIntegration400, PreviewAdminStandaloneIntegration401, @@ -280,11 +371,17 @@ import type { } from "./integration-models"; import { adoptAdminStandaloneIntegrationRequest, + applyAdminConfigurationMigrationRequest, + applyAdminCredentialTransferRequest, + chooseAdminCredentialTransferFieldsRequest, deleteAdminIntegrationBindingRequest, deleteAdminIntegrationCredentialRequest, disableAdminIntegrationRequest, enableAdminIntegrationRequest, + exportAdminConfigurationMigrationRequest, + exportAdminCredentialTransferRequest, exportAdminIntegrationConfigsRequest, + getAdminConfigurationMigrationInventoryRequest, getAdminIntegrationCredentialsRequest, getAdminIntegrationHealthHistoryRequest, getAdminIntegrationHealthRequest, @@ -298,6 +395,8 @@ import { listAdminIntegrationsRequest, listAdminStandaloneIntegrationsRequest, patchAdminIntegrationConfigRequest, + previewAdminConfigurationMigrationRequest, + previewAdminCredentialTransferRequest, previewAdminStandaloneIntegrationRequest, reloadAdminIntegrationRequest, reloadAdminIntegrationsRequest, @@ -307,6 +406,848 @@ import { setAdminIntegrationCredentialRequest, testAdminIntegrationCredentialsRequest, } from "./operation-bindings.generated"; +export type applyAdminConfigurationMigrationResponse200 = { + data: ApplyAdminConfigurationMigration200; + status: 200; +}; + +export type applyAdminConfigurationMigrationResponse400 = { + data: ApplyAdminConfigurationMigration400; + status: 400; +}; + +export type applyAdminConfigurationMigrationResponse401 = { + data: ApplyAdminConfigurationMigration401; + status: 401; +}; + +export type applyAdminConfigurationMigrationResponse403 = { + data: ApplyAdminConfigurationMigration403; + status: 403; +}; + +export type applyAdminConfigurationMigrationResponse409 = { + data: ApplyAdminConfigurationMigration409; + status: 409; +}; + +export type applyAdminConfigurationMigrationResponse413 = { + data: ApplyAdminConfigurationMigration413; + status: 413; +}; + +export type applyAdminConfigurationMigrationResponse415 = { + data: ApplyAdminConfigurationMigration415; + status: 415; +}; + +export type applyAdminConfigurationMigrationResponse429 = { + data: ApplyAdminConfigurationMigration429; + status: 429; +}; + +export type applyAdminConfigurationMigrationResponse500 = { + data: ApplyAdminConfigurationMigration500; + status: 500; +}; + +export type applyAdminConfigurationMigrationResponse503 = { + data: ApplyAdminConfigurationMigration503; + status: 503; +}; + +export type applyAdminConfigurationMigrationResponseSuccess = + applyAdminConfigurationMigrationResponse200 & { + headers: Headers; + }; +export type applyAdminConfigurationMigrationResponseError = ( + | applyAdminConfigurationMigrationResponse400 + | applyAdminConfigurationMigrationResponse401 + | applyAdminConfigurationMigrationResponse403 + | applyAdminConfigurationMigrationResponse409 + | applyAdminConfigurationMigrationResponse413 + | applyAdminConfigurationMigrationResponse415 + | applyAdminConfigurationMigrationResponse429 + | applyAdminConfigurationMigrationResponse500 + | applyAdminConfigurationMigrationResponse503 +) & { + headers: Headers; +}; + +export type applyAdminConfigurationMigrationResponse = + | applyAdminConfigurationMigrationResponseSuccess + | applyAdminConfigurationMigrationResponseError; + +export const getApplyAdminConfigurationMigrationUrl = () => { + return `/api/admin/configuration-migration/apply`; +}; + +export const applyAdminConfigurationMigration = async ( + applyAdminConfigurationMigrationBody: ApplyAdminConfigurationMigrationBody, + headers: ApplyAdminConfigurationMigrationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return applyAdminConfigurationMigrationRequest( + getApplyAdminConfigurationMigrationUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(applyAdminConfigurationMigrationBody), + }, + ); +}; + +export type exportAdminConfigurationMigrationResponse200 = { + data: ExportAdminConfigurationMigration200; + status: 200; +}; + +export type exportAdminConfigurationMigrationResponse400 = { + data: ExportAdminConfigurationMigration400; + status: 400; +}; + +export type exportAdminConfigurationMigrationResponse401 = { + data: ExportAdminConfigurationMigration401; + status: 401; +}; + +export type exportAdminConfigurationMigrationResponse403 = { + data: ExportAdminConfigurationMigration403; + status: 403; +}; + +export type exportAdminConfigurationMigrationResponse409 = { + data: ExportAdminConfigurationMigration409; + status: 409; +}; + +export type exportAdminConfigurationMigrationResponse413 = { + data: ExportAdminConfigurationMigration413; + status: 413; +}; + +export type exportAdminConfigurationMigrationResponse415 = { + data: ExportAdminConfigurationMigration415; + status: 415; +}; + +export type exportAdminConfigurationMigrationResponse429 = { + data: ExportAdminConfigurationMigration429; + status: 429; +}; + +export type exportAdminConfigurationMigrationResponse500 = { + data: ExportAdminConfigurationMigration500; + status: 500; +}; + +export type exportAdminConfigurationMigrationResponse503 = { + data: ExportAdminConfigurationMigration503; + status: 503; +}; + +export type exportAdminConfigurationMigrationResponseSuccess = + exportAdminConfigurationMigrationResponse200 & { + headers: Headers; + }; +export type exportAdminConfigurationMigrationResponseError = ( + | exportAdminConfigurationMigrationResponse400 + | exportAdminConfigurationMigrationResponse401 + | exportAdminConfigurationMigrationResponse403 + | exportAdminConfigurationMigrationResponse409 + | exportAdminConfigurationMigrationResponse413 + | exportAdminConfigurationMigrationResponse415 + | exportAdminConfigurationMigrationResponse429 + | exportAdminConfigurationMigrationResponse500 + | exportAdminConfigurationMigrationResponse503 +) & { + headers: Headers; +}; + +export type exportAdminConfigurationMigrationResponse = + | exportAdminConfigurationMigrationResponseSuccess + | exportAdminConfigurationMigrationResponseError; + +export const getExportAdminConfigurationMigrationUrl = () => { + return `/api/admin/configuration-migration/export`; +}; + +export const exportAdminConfigurationMigration = async ( + options?: Parameters[1], +): Promise => { + return exportAdminConfigurationMigrationRequest( + getExportAdminConfigurationMigrationUrl(), + { + ...options, + method: "GET", + }, + ); +}; + +export type getAdminConfigurationMigrationInventoryResponse200 = { + data: GetAdminConfigurationMigrationInventory200; + status: 200; +}; + +export type getAdminConfigurationMigrationInventoryResponse400 = { + data: GetAdminConfigurationMigrationInventory400; + status: 400; +}; + +export type getAdminConfigurationMigrationInventoryResponse401 = { + data: GetAdminConfigurationMigrationInventory401; + status: 401; +}; + +export type getAdminConfigurationMigrationInventoryResponse403 = { + data: GetAdminConfigurationMigrationInventory403; + status: 403; +}; + +export type getAdminConfigurationMigrationInventoryResponse409 = { + data: GetAdminConfigurationMigrationInventory409; + status: 409; +}; + +export type getAdminConfigurationMigrationInventoryResponse413 = { + data: GetAdminConfigurationMigrationInventory413; + status: 413; +}; + +export type getAdminConfigurationMigrationInventoryResponse415 = { + data: GetAdminConfigurationMigrationInventory415; + status: 415; +}; + +export type getAdminConfigurationMigrationInventoryResponse429 = { + data: GetAdminConfigurationMigrationInventory429; + status: 429; +}; + +export type getAdminConfigurationMigrationInventoryResponse500 = { + data: GetAdminConfigurationMigrationInventory500; + status: 500; +}; + +export type getAdminConfigurationMigrationInventoryResponse503 = { + data: GetAdminConfigurationMigrationInventory503; + status: 503; +}; + +export type getAdminConfigurationMigrationInventoryResponseSuccess = + getAdminConfigurationMigrationInventoryResponse200 & { + headers: Headers; + }; +export type getAdminConfigurationMigrationInventoryResponseError = ( + | getAdminConfigurationMigrationInventoryResponse400 + | getAdminConfigurationMigrationInventoryResponse401 + | getAdminConfigurationMigrationInventoryResponse403 + | getAdminConfigurationMigrationInventoryResponse409 + | getAdminConfigurationMigrationInventoryResponse413 + | getAdminConfigurationMigrationInventoryResponse415 + | getAdminConfigurationMigrationInventoryResponse429 + | getAdminConfigurationMigrationInventoryResponse500 + | getAdminConfigurationMigrationInventoryResponse503 +) & { + headers: Headers; +}; + +export type getAdminConfigurationMigrationInventoryResponse = + | getAdminConfigurationMigrationInventoryResponseSuccess + | getAdminConfigurationMigrationInventoryResponseError; + +export const getGetAdminConfigurationMigrationInventoryUrl = () => { + return `/api/admin/configuration-migration/inventory`; +}; + +export const getAdminConfigurationMigrationInventory = async ( + options?: Parameters[1], +): Promise => { + return getAdminConfigurationMigrationInventoryRequest( + getGetAdminConfigurationMigrationInventoryUrl(), + { + ...options, + method: "GET", + }, + ); +}; + +export type previewAdminConfigurationMigrationResponse200 = { + data: PreviewAdminConfigurationMigration200; + status: 200; +}; + +export type previewAdminConfigurationMigrationResponse400 = { + data: PreviewAdminConfigurationMigration400; + status: 400; +}; + +export type previewAdminConfigurationMigrationResponse401 = { + data: PreviewAdminConfigurationMigration401; + status: 401; +}; + +export type previewAdminConfigurationMigrationResponse403 = { + data: PreviewAdminConfigurationMigration403; + status: 403; +}; + +export type previewAdminConfigurationMigrationResponse409 = { + data: PreviewAdminConfigurationMigration409; + status: 409; +}; + +export type previewAdminConfigurationMigrationResponse413 = { + data: PreviewAdminConfigurationMigration413; + status: 413; +}; + +export type previewAdminConfigurationMigrationResponse415 = { + data: PreviewAdminConfigurationMigration415; + status: 415; +}; + +export type previewAdminConfigurationMigrationResponse429 = { + data: PreviewAdminConfigurationMigration429; + status: 429; +}; + +export type previewAdminConfigurationMigrationResponse500 = { + data: PreviewAdminConfigurationMigration500; + status: 500; +}; + +export type previewAdminConfigurationMigrationResponse503 = { + data: PreviewAdminConfigurationMigration503; + status: 503; +}; + +export type previewAdminConfigurationMigrationResponseSuccess = + previewAdminConfigurationMigrationResponse200 & { + headers: Headers; + }; +export type previewAdminConfigurationMigrationResponseError = ( + | previewAdminConfigurationMigrationResponse400 + | previewAdminConfigurationMigrationResponse401 + | previewAdminConfigurationMigrationResponse403 + | previewAdminConfigurationMigrationResponse409 + | previewAdminConfigurationMigrationResponse413 + | previewAdminConfigurationMigrationResponse415 + | previewAdminConfigurationMigrationResponse429 + | previewAdminConfigurationMigrationResponse500 + | previewAdminConfigurationMigrationResponse503 +) & { + headers: Headers; +}; + +export type previewAdminConfigurationMigrationResponse = + | previewAdminConfigurationMigrationResponseSuccess + | previewAdminConfigurationMigrationResponseError; + +export const getPreviewAdminConfigurationMigrationUrl = () => { + return `/api/admin/configuration-migration/preview`; +}; + +export const previewAdminConfigurationMigration = async ( + previewAdminConfigurationMigrationBody: PreviewAdminConfigurationMigrationBody, + headers: PreviewAdminConfigurationMigrationHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return previewAdminConfigurationMigrationRequest( + getPreviewAdminConfigurationMigrationUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(previewAdminConfigurationMigrationBody), + }, + ); +}; + +export type applyAdminCredentialTransferResponse200 = { + data: ApplyAdminCredentialTransfer200; + status: 200; +}; + +export type applyAdminCredentialTransferResponse400 = { + data: ApplyAdminCredentialTransfer400; + status: 400; +}; + +export type applyAdminCredentialTransferResponse401 = { + data: ApplyAdminCredentialTransfer401; + status: 401; +}; + +export type applyAdminCredentialTransferResponse403 = { + data: ApplyAdminCredentialTransfer403; + status: 403; +}; + +export type applyAdminCredentialTransferResponse409 = { + data: ApplyAdminCredentialTransfer409; + status: 409; +}; + +export type applyAdminCredentialTransferResponse413 = { + data: ApplyAdminCredentialTransfer413; + status: 413; +}; + +export type applyAdminCredentialTransferResponse415 = { + data: ApplyAdminCredentialTransfer415; + status: 415; +}; + +export type applyAdminCredentialTransferResponse429 = { + data: ApplyAdminCredentialTransfer429; + status: 429; +}; + +export type applyAdminCredentialTransferResponse500 = { + data: ApplyAdminCredentialTransfer500; + status: 500; +}; + +export type applyAdminCredentialTransferResponse503 = { + data: ApplyAdminCredentialTransfer503; + status: 503; +}; + +export type applyAdminCredentialTransferResponseSuccess = + applyAdminCredentialTransferResponse200 & { + headers: Headers; + }; +export type applyAdminCredentialTransferResponseError = ( + | applyAdminCredentialTransferResponse400 + | applyAdminCredentialTransferResponse401 + | applyAdminCredentialTransferResponse403 + | applyAdminCredentialTransferResponse409 + | applyAdminCredentialTransferResponse413 + | applyAdminCredentialTransferResponse415 + | applyAdminCredentialTransferResponse429 + | applyAdminCredentialTransferResponse500 + | applyAdminCredentialTransferResponse503 +) & { + headers: Headers; +}; + +export type applyAdminCredentialTransferResponse = + | applyAdminCredentialTransferResponseSuccess + | applyAdminCredentialTransferResponseError; + +export const getApplyAdminCredentialTransferUrl = () => { + return `/api/admin/credential-transfer/apply`; +}; + +export const applyAdminCredentialTransfer = async ( + applyAdminCredentialTransferBody: ApplyAdminCredentialTransferBody, + headers: ApplyAdminCredentialTransferHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return applyAdminCredentialTransferRequest( + getApplyAdminCredentialTransferUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(applyAdminCredentialTransferBody), + }, + ); +}; + +export type exportAdminCredentialTransferResponse200 = { + data: ExportAdminCredentialTransfer200; + status: 200; +}; + +export type exportAdminCredentialTransferResponse400 = { + data: ExportAdminCredentialTransfer400; + status: 400; +}; + +export type exportAdminCredentialTransferResponse401 = { + data: ExportAdminCredentialTransfer401; + status: 401; +}; + +export type exportAdminCredentialTransferResponse403 = { + data: ExportAdminCredentialTransfer403; + status: 403; +}; + +export type exportAdminCredentialTransferResponse409 = { + data: ExportAdminCredentialTransfer409; + status: 409; +}; + +export type exportAdminCredentialTransferResponse413 = { + data: ExportAdminCredentialTransfer413; + status: 413; +}; + +export type exportAdminCredentialTransferResponse415 = { + data: ExportAdminCredentialTransfer415; + status: 415; +}; + +export type exportAdminCredentialTransferResponse429 = { + data: ExportAdminCredentialTransfer429; + status: 429; +}; + +export type exportAdminCredentialTransferResponse500 = { + data: ExportAdminCredentialTransfer500; + status: 500; +}; + +export type exportAdminCredentialTransferResponse503 = { + data: ExportAdminCredentialTransfer503; + status: 503; +}; + +export type exportAdminCredentialTransferResponseSuccess = + exportAdminCredentialTransferResponse200 & { + headers: Headers; + }; +export type exportAdminCredentialTransferResponseError = ( + | exportAdminCredentialTransferResponse400 + | exportAdminCredentialTransferResponse401 + | exportAdminCredentialTransferResponse403 + | exportAdminCredentialTransferResponse409 + | exportAdminCredentialTransferResponse413 + | exportAdminCredentialTransferResponse415 + | exportAdminCredentialTransferResponse429 + | exportAdminCredentialTransferResponse500 + | exportAdminCredentialTransferResponse503 +) & { + headers: Headers; +}; + +export type exportAdminCredentialTransferResponse = + | exportAdminCredentialTransferResponseSuccess + | exportAdminCredentialTransferResponseError; + +export const getExportAdminCredentialTransferUrl = () => { + return `/api/admin/credential-transfer/export`; +}; + +export const exportAdminCredentialTransfer = async ( + exportAdminCredentialTransferBody: ExportAdminCredentialTransferBody, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return exportAdminCredentialTransferRequest( + getExportAdminCredentialTransferUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, + body: JSON.stringify(exportAdminCredentialTransferBody), + }, + ); +}; + +export type previewAdminCredentialTransferResponse200 = { + data: PreviewAdminCredentialTransfer200; + status: 200; +}; + +export type previewAdminCredentialTransferResponse400 = { + data: PreviewAdminCredentialTransfer400; + status: 400; +}; + +export type previewAdminCredentialTransferResponse401 = { + data: PreviewAdminCredentialTransfer401; + status: 401; +}; + +export type previewAdminCredentialTransferResponse403 = { + data: PreviewAdminCredentialTransfer403; + status: 403; +}; + +export type previewAdminCredentialTransferResponse409 = { + data: PreviewAdminCredentialTransfer409; + status: 409; +}; + +export type previewAdminCredentialTransferResponse413 = { + data: PreviewAdminCredentialTransfer413; + status: 413; +}; + +export type previewAdminCredentialTransferResponse415 = { + data: PreviewAdminCredentialTransfer415; + status: 415; +}; + +export type previewAdminCredentialTransferResponse429 = { + data: PreviewAdminCredentialTransfer429; + status: 429; +}; + +export type previewAdminCredentialTransferResponse500 = { + data: PreviewAdminCredentialTransfer500; + status: 500; +}; + +export type previewAdminCredentialTransferResponse503 = { + data: PreviewAdminCredentialTransfer503; + status: 503; +}; + +export type previewAdminCredentialTransferResponseSuccess = + previewAdminCredentialTransferResponse200 & { + headers: Headers; + }; +export type previewAdminCredentialTransferResponseError = ( + | previewAdminCredentialTransferResponse400 + | previewAdminCredentialTransferResponse401 + | previewAdminCredentialTransferResponse403 + | previewAdminCredentialTransferResponse409 + | previewAdminCredentialTransferResponse413 + | previewAdminCredentialTransferResponse415 + | previewAdminCredentialTransferResponse429 + | previewAdminCredentialTransferResponse500 + | previewAdminCredentialTransferResponse503 +) & { + headers: Headers; +}; + +export type previewAdminCredentialTransferResponse = + | previewAdminCredentialTransferResponseSuccess + | previewAdminCredentialTransferResponseError; + +export const getPreviewAdminCredentialTransferUrl = () => { + return `/api/admin/credential-transfer/preview`; +}; + +export const previewAdminCredentialTransfer = async ( + previewAdminCredentialTransferBody: PreviewAdminCredentialTransferBody, + headers: PreviewAdminCredentialTransferHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return previewAdminCredentialTransferRequest( + getPreviewAdminCredentialTransferUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(previewAdminCredentialTransferBody), + }, + ); +}; + +export type chooseAdminCredentialTransferFieldsResponse200 = { + data: ChooseAdminCredentialTransferFields200; + status: 200; +}; + +export type chooseAdminCredentialTransferFieldsResponse400 = { + data: ChooseAdminCredentialTransferFields400; + status: 400; +}; + +export type chooseAdminCredentialTransferFieldsResponse401 = { + data: ChooseAdminCredentialTransferFields401; + status: 401; +}; + +export type chooseAdminCredentialTransferFieldsResponse403 = { + data: ChooseAdminCredentialTransferFields403; + status: 403; +}; + +export type chooseAdminCredentialTransferFieldsResponse409 = { + data: ChooseAdminCredentialTransferFields409; + status: 409; +}; + +export type chooseAdminCredentialTransferFieldsResponse413 = { + data: ChooseAdminCredentialTransferFields413; + status: 413; +}; + +export type chooseAdminCredentialTransferFieldsResponse415 = { + data: ChooseAdminCredentialTransferFields415; + status: 415; +}; + +export type chooseAdminCredentialTransferFieldsResponse429 = { + data: ChooseAdminCredentialTransferFields429; + status: 429; +}; + +export type chooseAdminCredentialTransferFieldsResponse500 = { + data: ChooseAdminCredentialTransferFields500; + status: 500; +}; + +export type chooseAdminCredentialTransferFieldsResponse503 = { + data: ChooseAdminCredentialTransferFields503; + status: 503; +}; + +export type chooseAdminCredentialTransferFieldsResponseSuccess = + chooseAdminCredentialTransferFieldsResponse200 & { + headers: Headers; + }; +export type chooseAdminCredentialTransferFieldsResponseError = ( + | chooseAdminCredentialTransferFieldsResponse400 + | chooseAdminCredentialTransferFieldsResponse401 + | chooseAdminCredentialTransferFieldsResponse403 + | chooseAdminCredentialTransferFieldsResponse409 + | chooseAdminCredentialTransferFieldsResponse413 + | chooseAdminCredentialTransferFieldsResponse415 + | chooseAdminCredentialTransferFieldsResponse429 + | chooseAdminCredentialTransferFieldsResponse500 + | chooseAdminCredentialTransferFieldsResponse503 +) & { + headers: Headers; +}; + +export type chooseAdminCredentialTransferFieldsResponse = + | chooseAdminCredentialTransferFieldsResponseSuccess + | chooseAdminCredentialTransferFieldsResponseError; + +export const getChooseAdminCredentialTransferFieldsUrl = (previewId: string) => { + return `/api/admin/credential-transfer/preview/${encodeURIComponent(String(previewId))}/choices`; +}; + +export const chooseAdminCredentialTransferFields = async ( + previewId: string, + chooseAdminCredentialTransferFieldsBody: ChooseAdminCredentialTransferFieldsBody, + headers: ChooseAdminCredentialTransferFieldsHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return chooseAdminCredentialTransferFieldsRequest( + getChooseAdminCredentialTransferFieldsUrl(previewId), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(chooseAdminCredentialTransferFieldsBody), + }, + ); +}; + export type listAdminCredentialsResponse200 = { data: ListAdminCredentials200; status: 200; diff --git a/packages/core/src/api/generated/admin/integration-models/adminExportAdminConfigurationMigration200Schema0.ts b/packages/core/src/api/generated/admin/integration-models/adminExportAdminConfigurationMigration200Schema0.ts new file mode 100644 index 000000000..56fe91cee --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adminExportAdminConfigurationMigration200Schema0.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdminExportAdminConfigurationMigration200Schema0 = + | string + | number + | boolean + | null + | (AdminExportAdminConfigurationMigration200Schema0 | null)[] + | { [key: string]: AdminExportAdminConfigurationMigration200Schema0 | null }; diff --git a/packages/core/src/api/generated/admin/integration-models/adminGetAdminConfigurationMigrationInventory200Schema0.ts b/packages/core/src/api/generated/admin/integration-models/adminGetAdminConfigurationMigrationInventory200Schema0.ts new file mode 100644 index 000000000..61d2e9f8b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adminGetAdminConfigurationMigrationInventory200Schema0.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdminGetAdminConfigurationMigrationInventory200Schema0 = + | string + | number + | boolean + | null + | (AdminGetAdminConfigurationMigrationInventory200Schema0 | null)[] + | { [key: string]: AdminGetAdminConfigurationMigrationInventory200Schema0 | null }; diff --git a/packages/core/src/api/generated/admin/integration-models/adminPreviewAdminConfigurationMigrationBodySchema0.ts b/packages/core/src/api/generated/admin/integration-models/adminPreviewAdminConfigurationMigrationBodySchema0.ts new file mode 100644 index 000000000..4ab6ad588 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/adminPreviewAdminConfigurationMigrationBodySchema0.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type AdminPreviewAdminConfigurationMigrationBodySchema0 = + | string + | number + | boolean + | null + | (AdminPreviewAdminConfigurationMigrationBodySchema0 | null)[] + | { [key: string]: AdminPreviewAdminConfigurationMigrationBodySchema0 | null }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200.ts new file mode 100644 index 000000000..58f73403b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminConfigurationMigration200Outcome } from "./applyAdminConfigurationMigration200Outcome"; +import type { ApplyAdminConfigurationMigration200StepsItem } from "./applyAdminConfigurationMigration200StepsItem"; + +export type ApplyAdminConfigurationMigration200 = { + operationId: string; + outcome: ApplyAdminConfigurationMigration200Outcome; + steps: ApplyAdminConfigurationMigration200StepsItem[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200Outcome.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200Outcome.ts new file mode 100644 index 000000000..65b67a698 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200Outcome.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration200Outcome = + (typeof ApplyAdminConfigurationMigration200Outcome)[keyof typeof ApplyAdminConfigurationMigration200Outcome]; + +export const ApplyAdminConfigurationMigration200Outcome = { + applied: "applied", + partial: "partial", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200StepsItem.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200StepsItem.ts new file mode 100644 index 000000000..40bd39ac9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200StepsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminConfigurationMigration200StepsItemStatus } from "./applyAdminConfigurationMigration200StepsItemStatus"; + +export type ApplyAdminConfigurationMigration200StepsItem = { + scope: string; + id: string; + status: ApplyAdminConfigurationMigration200StepsItemStatus; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200StepsItemStatus.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200StepsItemStatus.ts new file mode 100644 index 000000000..73bc1ec88 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration200StepsItemStatus.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration200StepsItemStatus = + (typeof ApplyAdminConfigurationMigration200StepsItemStatus)[keyof typeof ApplyAdminConfigurationMigration200StepsItemStatus]; + +export const ApplyAdminConfigurationMigration200StepsItemStatus = { + applied: "applied", + failed: "failed", + pending: "pending", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration400.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration400.ts new file mode 100644 index 000000000..7b39adf65 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration401.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration401.ts new file mode 100644 index 000000000..8c3a8c2a4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration403.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration403.ts new file mode 100644 index 000000000..9f4361a84 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration409.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration409.ts new file mode 100644 index 000000000..cb18faa0f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration413.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration413.ts new file mode 100644 index 000000000..738a5a167 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration415.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration415.ts new file mode 100644 index 000000000..bc58f8074 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration429.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration429.ts new file mode 100644 index 000000000..9bd96d3f9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration500.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration500.ts new file mode 100644 index 000000000..edf6d093b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration503.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration503.ts new file mode 100644 index 000000000..2caec604a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBody.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBody.ts new file mode 100644 index 000000000..8f1b5b983 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBody.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminConfigurationMigrationBodyPlatformAuthorization } from "./applyAdminConfigurationMigrationBodyPlatformAuthorization"; + +export type ApplyAdminConfigurationMigrationBody = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** + * @minLength 1 + * @maxLength 128 + */ + fingerprint: string; + platformAuthorization?: ApplyAdminConfigurationMigrationBodyPlatformAuthorization; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBodyPlatformAuthorization.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBodyPlatformAuthorization.ts new file mode 100644 index 000000000..220edb9d2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBodyPlatformAuthorization.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations } from "./applyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations"; + +export type ApplyAdminConfigurationMigrationBodyPlatformAuthorization = { + confirmations: ApplyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + proofToken?: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations.ts new file mode 100644 index 000000000..37f6b3685 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations = { + domain: boolean; + dnsTls: boolean; + oauth: boolean; + passkeys: boolean; + alternateLogin: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationHeaders.ts new file mode 100644 index 000000000..d4ef8bcf9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminConfigurationMigrationHeaders.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminConfigurationMigrationHeaders = { + /** + * @minLength 1 + * @maxLength 128 + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200.ts new file mode 100644 index 000000000..15e1f49fc --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminCredentialTransfer200Outcome } from "./applyAdminCredentialTransfer200Outcome"; +import type { ApplyAdminCredentialTransfer200StepsItem } from "./applyAdminCredentialTransfer200StepsItem"; + +export type ApplyAdminCredentialTransfer200 = { + operationId: string; + outcome: ApplyAdminCredentialTransfer200Outcome; + steps: ApplyAdminCredentialTransfer200StepsItem[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200Outcome.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200Outcome.ts new file mode 100644 index 000000000..00234e1b0 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200Outcome.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer200Outcome = + (typeof ApplyAdminCredentialTransfer200Outcome)[keyof typeof ApplyAdminCredentialTransfer200Outcome]; + +export const ApplyAdminCredentialTransfer200Outcome = { + applied: "applied", + partial: "partial", + failed: "failed", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200StepsItem.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200StepsItem.ts new file mode 100644 index 000000000..d9c71bdb8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200StepsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ApplyAdminCredentialTransfer200StepsItemStatus } from "./applyAdminCredentialTransfer200StepsItemStatus"; + +export type ApplyAdminCredentialTransfer200StepsItem = { + scope: string; + id: string; + status: ApplyAdminCredentialTransfer200StepsItemStatus; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200StepsItemStatus.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200StepsItemStatus.ts new file mode 100644 index 000000000..9194d6b01 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer200StepsItemStatus.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer200StepsItemStatus = + (typeof ApplyAdminCredentialTransfer200StepsItemStatus)[keyof typeof ApplyAdminCredentialTransfer200StepsItemStatus]; + +export const ApplyAdminCredentialTransfer200StepsItemStatus = { + applied: "applied", + failed: "failed", + pending: "pending", + unknown: "unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer400.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer400.ts new file mode 100644 index 000000000..da58c0335 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer401.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer401.ts new file mode 100644 index 000000000..ffbd21bdd --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer403.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer403.ts new file mode 100644 index 000000000..b5087042d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer409.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer409.ts new file mode 100644 index 000000000..2fc7f7732 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer413.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer413.ts new file mode 100644 index 000000000..097b9a062 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer415.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer415.ts new file mode 100644 index 000000000..62b8af527 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer429.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer429.ts new file mode 100644 index 000000000..1322cb86b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer500.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer500.ts new file mode 100644 index 000000000..da6d1f09f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer503.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer503.ts new file mode 100644 index 000000000..0880e8983 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransfer503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransfer503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransferBody.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransferBody.ts new file mode 100644 index 000000000..6c0ab9347 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransferBody.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransferBody = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** + * @minLength 1 + * @maxLength 128 + */ + intentFingerprint: string; + /** @nullable */ + expectedCurrentRevisionId: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + importProofToken: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + transitProofToken?: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransferHeaders.ts b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransferHeaders.ts new file mode 100644 index 000000000..79aadcc8e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/applyAdminCredentialTransferHeaders.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ApplyAdminCredentialTransferHeaders = { + /** + * @minLength 1 + * @maxLength 128 + */ + "idempotency-key": string; + /** + * @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + */ + "x-admin-reauth-proof"?: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200.ts new file mode 100644 index 000000000..4c208762e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ChooseAdminCredentialTransferFields200EntriesItem } from "./chooseAdminCredentialTransferFields200EntriesItem"; +import type { ChooseAdminCredentialTransferFields200FeedMap } from "./chooseAdminCredentialTransferFields200FeedMap"; + +export type ChooseAdminCredentialTransferFields200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** + * @minLength 1 + * @maxLength 128 + */ + intentFingerprint: string; + /** @nullable */ + expectedCurrentRevisionId: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + /** @maxItems 1000 */ + entries: ChooseAdminCredentialTransferFields200EntriesItem[]; + feedMap?: ChooseAdminCredentialTransferFields200FeedMap; + applyable: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItem.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItem.ts new file mode 100644 index 000000000..e4d160d5a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItem.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ChooseAdminCredentialTransferFields200EntriesItemScope } from "./chooseAdminCredentialTransferFields200EntriesItemScope"; +import type { ChooseAdminCredentialTransferFields200EntriesItemStatus } from "./chooseAdminCredentialTransferFields200EntriesItemStatus"; + +export type ChooseAdminCredentialTransferFields200EntriesItem = { + scope: ChooseAdminCredentialTransferFields200EntriesItemScope; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + id: string; + /** + * @minLength 1 + * @maxLength 128 + */ + key: string; + status: ChooseAdminCredentialTransferFields200EntriesItemStatus; + reason?: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItemScope.ts new file mode 100644 index 000000000..fcb8e8656 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItemScope.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields200EntriesItemScope = + (typeof ChooseAdminCredentialTransferFields200EntriesItemScope)[keyof typeof ChooseAdminCredentialTransferFields200EntriesItemScope]; + +export const ChooseAdminCredentialTransferFields200EntriesItemScope = { + integration: "integration", + service: "service", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItemStatus.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItemStatus.ts new file mode 100644 index 000000000..57b5eb9c6 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200EntriesItemStatus.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields200EntriesItemStatus = + (typeof ChooseAdminCredentialTransferFields200EntriesItemStatus)[keyof typeof ChooseAdminCredentialTransferFields200EntriesItemStatus]; + +export const ChooseAdminCredentialTransferFields200EntriesItemStatus = { + selected: "selected", + skipped: "skipped", + denied: "denied", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMap.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMap.ts new file mode 100644 index 000000000..9a356f537 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMap.ts @@ -0,0 +1,27 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ChooseAdminCredentialTransferFields200FeedMapChangesItem } from "./chooseAdminCredentialTransferFields200FeedMapChangesItem"; + +export type ChooseAdminCredentialTransferFields200FeedMap = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + intentFingerprint: string; + catalogRevisionId: string; + credentialsRevision: string; + /** @nullable */ + expectedCurrentRevisionId: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + /** @maxItems 1000 */ + changes: ChooseAdminCredentialTransferFields200FeedMapChangesItem[]; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + preservedCount: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + mapBytes: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMapChangesItem.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMapChangesItem.ts new file mode 100644 index 000000000..991e9f413 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMapChangesItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ChooseAdminCredentialTransferFields200FeedMapChangesItemAction } from "./chooseAdminCredentialTransferFields200FeedMapChangesItemAction"; + +export type ChooseAdminCredentialTransferFields200FeedMapChangesItem = { + key: string; + action: ChooseAdminCredentialTransferFields200FeedMapChangesItemAction; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMapChangesItemAction.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMapChangesItemAction.ts new file mode 100644 index 000000000..83af68081 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields200FeedMapChangesItemAction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields200FeedMapChangesItemAction = + (typeof ChooseAdminCredentialTransferFields200FeedMapChangesItemAction)[keyof typeof ChooseAdminCredentialTransferFields200FeedMapChangesItemAction]; + +export const ChooseAdminCredentialTransferFields200FeedMapChangesItemAction = { + set: "set", + delete: "delete", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields400.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields400.ts new file mode 100644 index 000000000..dd9d072d4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields401.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields401.ts new file mode 100644 index 000000000..36e4ff3ff --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields403.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields403.ts new file mode 100644 index 000000000..3ab59a033 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields409.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields409.ts new file mode 100644 index 000000000..7328824c0 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields413.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields413.ts new file mode 100644 index 000000000..8cb8d2497 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields415.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields415.ts new file mode 100644 index 000000000..f43e1c479 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields429.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields429.ts new file mode 100644 index 000000000..f3c2e1396 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields500.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields500.ts new file mode 100644 index 000000000..aa4f5f51b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields503.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields503.ts new file mode 100644 index 000000000..06a9856b0 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFields503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFields503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBody.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBody.ts new file mode 100644 index 000000000..baf3e6185 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBody.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ChooseAdminCredentialTransferFieldsBodyChoices } from "./chooseAdminCredentialTransferFieldsBodyChoices"; + +export type ChooseAdminCredentialTransferFieldsBody = { + choices: ChooseAdminCredentialTransferFieldsBodyChoices; + /** + * @minLength 1 + * @maxLength 128 + */ + expectedIntentFingerprint: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoices.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoices.ts new file mode 100644 index 000000000..a2cd99913 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoices.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItem } from "./chooseAdminCredentialTransferFieldsBodyChoicesEntriesItem"; +import type { ChooseAdminCredentialTransferFieldsBodyChoicesFeedMap } from "./chooseAdminCredentialTransferFieldsBodyChoicesFeedMap"; + +export type ChooseAdminCredentialTransferFieldsBodyChoices = { + /** @maxItems 1000 */ + entries: ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItem[]; + feedMap?: ChooseAdminCredentialTransferFieldsBodyChoicesFeedMap; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItem.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItem.ts new file mode 100644 index 000000000..dfc0c772b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItem.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction } from "./chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction"; +import type { ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope } from "./chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope"; + +export type ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItem = { + scope: ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + id: string; + /** + * @minLength 1 + * @maxLength 128 + */ + key: string; + action: ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction.ts new file mode 100644 index 000000000..505863573 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction = + (typeof ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction)[keyof typeof ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction]; + +export const ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction = { + import: "import", + skip: "skip", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope.ts new file mode 100644 index 000000000..291fbc29c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope = + (typeof ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope)[keyof typeof ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope]; + +export const ChooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope = { + integration: "integration", + service: "service", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesFeedMap.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesFeedMap.ts new file mode 100644 index 000000000..75d9cae52 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsBodyChoicesFeedMap.ts @@ -0,0 +1,27 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFieldsBodyChoicesFeedMap = { + /** + * @minLength 1 + * @maxLength 256 + */ + catalogRevisionId: string; + /** + * @minLength 1 + * @maxLength 256 + */ + expectedCredentialsRevision: string; + /** + * @maxItems 1000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + setKeys: string[]; + /** + * @maxItems 1000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + deleteKeys: string[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsHeaders.ts b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsHeaders.ts new file mode 100644 index 000000000..ec175240c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/chooseAdminCredentialTransferFieldsHeaders.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ChooseAdminCredentialTransferFieldsHeaders = { + /** + * @minLength 1 + * @maxLength 128 + */ + "idempotency-key": string; + /** + * @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + */ + "x-admin-reauth-proof"?: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200.ts new file mode 100644 index 000000000..486140ba1 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200.ts @@ -0,0 +1,30 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminConfigurationMigration200BindingsItem } from "./exportAdminConfigurationMigration200BindingsItem"; +import type { ExportAdminConfigurationMigration200Format } from "./exportAdminConfigurationMigration200Format"; +import type { ExportAdminConfigurationMigration200InstallationsItem } from "./exportAdminConfigurationMigration200InstallationsItem"; +import type { ExportAdminConfigurationMigration200IntegrationsItem } from "./exportAdminConfigurationMigration200IntegrationsItem"; +import type { ExportAdminConfigurationMigration200Platform } from "./exportAdminConfigurationMigration200Platform"; +import type { ExportAdminConfigurationMigration200PrerequisitesItem } from "./exportAdminConfigurationMigration200PrerequisitesItem"; +import type { ExportAdminConfigurationMigration200ServiceConfigsItem } from "./exportAdminConfigurationMigration200ServiceConfigsItem"; +import type { ExportAdminConfigurationMigration200Settings } from "./exportAdminConfigurationMigration200Settings"; +import type { ExportAdminConfigurationMigration200Version } from "./exportAdminConfigurationMigration200Version"; + +export type ExportAdminConfigurationMigration200 = { + format: ExportAdminConfigurationMigration200Format; + version: ExportAdminConfigurationMigration200Version; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + exportedAt: string; + settings: ExportAdminConfigurationMigration200Settings; + platform?: ExportAdminConfigurationMigration200Platform; + /** @maxItems 256 */ + integrations: ExportAdminConfigurationMigration200IntegrationsItem[]; + /** @maxItems 256 */ + serviceConfigs: ExportAdminConfigurationMigration200ServiceConfigsItem[]; + /** @maxItems 512 */ + bindings: ExportAdminConfigurationMigration200BindingsItem[]; + /** @maxItems 2048 */ + installations: ExportAdminConfigurationMigration200InstallationsItem[]; + /** @maxItems 1000 */ + prerequisites: ExportAdminConfigurationMigration200PrerequisitesItem[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200BindingsItem.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200BindingsItem.ts new file mode 100644 index 000000000..dc3a22e2b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200BindingsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200BindingsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + /** + * @minLength 1 + * @maxLength 128 + */ + capability: string; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + serviceId: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Format.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Format.ts new file mode 100644 index 000000000..f33e0fee7 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Format.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200Format = + (typeof ExportAdminConfigurationMigration200Format)[keyof typeof ExportAdminConfigurationMigration200Format]; + +export const ExportAdminConfigurationMigration200Format = { + "openmapx-configuration": "openmapx-configuration", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItem.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItem.ts new file mode 100644 index 000000000..1b49711d5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItem.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminConfigurationMigration200InstallationsItemOwnership } from "./exportAdminConfigurationMigration200InstallationsItemOwnership"; +import type { ExportAdminConfigurationMigration200InstallationsItemProvenance } from "./exportAdminConfigurationMigration200InstallationsItemProvenance"; + +export type ExportAdminConfigurationMigration200InstallationsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + version: string | null; + contentDigest: string | null; + denialReason: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + pendingTransactionId?: string; + ownership: ExportAdminConfigurationMigration200InstallationsItemOwnership; + extensionId: string | null; + provenance: ExportAdminConfigurationMigration200InstallationsItemProvenance; + adoptable: boolean; + removable: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItemOwnership.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItemOwnership.ts new file mode 100644 index 000000000..e0454214e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItemOwnership.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200InstallationsItemOwnership = + (typeof ExportAdminConfigurationMigration200InstallationsItemOwnership)[keyof typeof ExportAdminConfigurationMigration200InstallationsItemOwnership]; + +export const ExportAdminConfigurationMigration200InstallationsItemOwnership = { + extension: "extension", + standalone: "standalone", + unmanaged: "unmanaged", + conflict: "conflict", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItemProvenance.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItemProvenance.ts new file mode 100644 index 000000000..fde7babc9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200InstallationsItemProvenance.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200InstallationsItemProvenance = + (typeof ExportAdminConfigurationMigration200InstallationsItemProvenance)[keyof typeof ExportAdminConfigurationMigration200InstallationsItemProvenance]; + +export const ExportAdminConfigurationMigration200InstallationsItemProvenance = { + artifact: "artifact", + source: "source", + "uploaded-artifact": "uploaded-artifact", + legacy_unknown: "legacy_unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200IntegrationsItem.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200IntegrationsItem.ts new file mode 100644 index 000000000..8de29c034 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200IntegrationsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminConfigurationMigration200IntegrationsItemConfig } from "./exportAdminConfigurationMigration200IntegrationsItemConfig"; + +export type ExportAdminConfigurationMigration200IntegrationsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + config: ExportAdminConfigurationMigration200IntegrationsItemConfig; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200IntegrationsItemConfig.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200IntegrationsItemConfig.ts new file mode 100644 index 000000000..bd73ed7cb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200IntegrationsItemConfig.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminExportAdminConfigurationMigration200Schema0 } from "./adminExportAdminConfigurationMigration200Schema0"; + +export type ExportAdminConfigurationMigration200IntegrationsItemConfig = { + [key: string]: AdminExportAdminConfigurationMigration200Schema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Platform.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Platform.ts new file mode 100644 index 000000000..0212fceab --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Platform.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminConfigurationMigration200PlatformAcmeEmail } from "./exportAdminConfigurationMigration200PlatformAcmeEmail"; +import type { ExportAdminConfigurationMigration200PlatformBackupRetentionDays } from "./exportAdminConfigurationMigration200PlatformBackupRetentionDays"; +import type { ExportAdminConfigurationMigration200PlatformDefaultRegion } from "./exportAdminConfigurationMigration200PlatformDefaultRegion"; +import type { ExportAdminConfigurationMigration200PlatformDomain } from "./exportAdminConfigurationMigration200PlatformDomain"; +import type { ExportAdminConfigurationMigration200PlatformReleaseSelector } from "./exportAdminConfigurationMigration200PlatformReleaseSelector"; +import type { ExportAdminConfigurationMigration200PlatformVersion } from "./exportAdminConfigurationMigration200PlatformVersion"; + +export type ExportAdminConfigurationMigration200Platform = { + version: ExportAdminConfigurationMigration200PlatformVersion; + domain?: ExportAdminConfigurationMigration200PlatformDomain; + acmeEmail?: ExportAdminConfigurationMigration200PlatformAcmeEmail; + releaseSelector?: ExportAdminConfigurationMigration200PlatformReleaseSelector; + defaultRegion?: ExportAdminConfigurationMigration200PlatformDefaultRegion; + backupRetentionDays?: ExportAdminConfigurationMigration200PlatformBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformAcmeEmail.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformAcmeEmail.ts new file mode 100644 index 000000000..b8116d7c2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformAcmeEmail.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PlatformAcmeEmail = + | { + source: "host"; + } + | { + source: "database"; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformBackupRetentionDays.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformBackupRetentionDays.ts new file mode 100644 index 000000000..559bc0ce2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformBackupRetentionDays.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PlatformBackupRetentionDays = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minimum 1 + * @maximum 36500 + */ + value: number; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformDefaultRegion.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformDefaultRegion.ts new file mode 100644 index 000000000..17dd4ca23 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformDefaultRegion.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PlatformDefaultRegion = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformDomain.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformDomain.ts new file mode 100644 index 000000000..ce2a1f4c5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformDomain.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PlatformDomain = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 253 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformReleaseSelector.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformReleaseSelector.ts new file mode 100644 index 000000000..1a571ff22 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformReleaseSelector.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PlatformReleaseSelector = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 128 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformVersion.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformVersion.ts new file mode 100644 index 000000000..c1257247d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PlatformVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PlatformVersion = + (typeof ExportAdminConfigurationMigration200PlatformVersion)[keyof typeof ExportAdminConfigurationMigration200PlatformVersion]; + +export const ExportAdminConfigurationMigration200PlatformVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItem.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItem.ts new file mode 100644 index 000000000..16a8d9992 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItem.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminConfigurationMigration200PrerequisitesItemReason } from "./exportAdminConfigurationMigration200PrerequisitesItemReason"; +import type { ExportAdminConfigurationMigration200PrerequisitesItemScope } from "./exportAdminConfigurationMigration200PrerequisitesItemScope"; +import type { ExportAdminConfigurationMigration200PrerequisitesItemSource } from "./exportAdminConfigurationMigration200PrerequisitesItemSource"; + +export type ExportAdminConfigurationMigration200PrerequisitesItem = { + scope: ExportAdminConfigurationMigration200PrerequisitesItemScope; + /** + * @minLength 1 + * @maxLength 128 + */ + id: string; + /** @maxLength 128 */ + key?: string; + source: ExportAdminConfigurationMigration200PrerequisitesItemSource; + reason: ExportAdminConfigurationMigration200PrerequisitesItemReason; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemReason.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemReason.ts new file mode 100644 index 000000000..429f10951 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PrerequisitesItemReason = + (typeof ExportAdminConfigurationMigration200PrerequisitesItemReason)[keyof typeof ExportAdminConfigurationMigration200PrerequisitesItemReason]; + +export const ExportAdminConfigurationMigration200PrerequisitesItemReason = { + host_owned: "host_owned", + credential_required: "credential_required", + installation_required: "installation_required", + unsupported: "unsupported", + protected_bootstrap: "protected_bootstrap", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemScope.ts new file mode 100644 index 000000000..f1f6693c7 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemScope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PrerequisitesItemScope = + (typeof ExportAdminConfigurationMigration200PrerequisitesItemScope)[keyof typeof ExportAdminConfigurationMigration200PrerequisitesItemScope]; + +export const ExportAdminConfigurationMigration200PrerequisitesItemScope = { + integration: "integration", + service: "service", + platform: "platform", + account: "account", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemSource.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemSource.ts new file mode 100644 index 000000000..7476aa840 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200PrerequisitesItemSource.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200PrerequisitesItemSource = + (typeof ExportAdminConfigurationMigration200PrerequisitesItemSource)[keyof typeof ExportAdminConfigurationMigration200PrerequisitesItemSource]; + +export const ExportAdminConfigurationMigration200PrerequisitesItemSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", + missing: "missing", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200ServiceConfigsItem.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200ServiceConfigsItem.ts new file mode 100644 index 000000000..21c4ac233 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200ServiceConfigsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminConfigurationMigration200ServiceConfigsItemConfig } from "./exportAdminConfigurationMigration200ServiceConfigsItemConfig"; + +export type ExportAdminConfigurationMigration200ServiceConfigsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + serviceId: string; + config: ExportAdminConfigurationMigration200ServiceConfigsItemConfig; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200ServiceConfigsItemConfig.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200ServiceConfigsItemConfig.ts new file mode 100644 index 000000000..a941dae55 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200ServiceConfigsItemConfig.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminExportAdminConfigurationMigration200Schema0 } from "./adminExportAdminConfigurationMigration200Schema0"; + +export type ExportAdminConfigurationMigration200ServiceConfigsItemConfig = { + [key: string]: AdminExportAdminConfigurationMigration200Schema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Settings.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Settings.ts new file mode 100644 index 000000000..e895e934c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Settings.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminExportAdminConfigurationMigration200Schema0 } from "./adminExportAdminConfigurationMigration200Schema0"; + +export type ExportAdminConfigurationMigration200Settings = { + [key: string]: AdminExportAdminConfigurationMigration200Schema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Version.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Version.ts new file mode 100644 index 000000000..a887aebbb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration200Version.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration200Version = + (typeof ExportAdminConfigurationMigration200Version)[keyof typeof ExportAdminConfigurationMigration200Version]; + +export const ExportAdminConfigurationMigration200Version = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration400.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration400.ts new file mode 100644 index 000000000..72418589a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration401.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration401.ts new file mode 100644 index 000000000..800bb7f09 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration403.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration403.ts new file mode 100644 index 000000000..eae9a9123 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration409.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration409.ts new file mode 100644 index 000000000..c098b88c2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration413.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration413.ts new file mode 100644 index 000000000..8b4fdbcac --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration415.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration415.ts new file mode 100644 index 000000000..da0192c34 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration429.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration429.ts new file mode 100644 index 000000000..d587f8436 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration500.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration500.ts new file mode 100644 index 000000000..47099263a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration503.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration503.ts new file mode 100644 index 000000000..52c6e2cf8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminConfigurationMigration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminConfigurationMigration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200.ts new file mode 100644 index 000000000..970bb971b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminCredentialTransfer200Cipher } from "./exportAdminCredentialTransfer200Cipher"; +import type { ExportAdminCredentialTransfer200Format } from "./exportAdminCredentialTransfer200Format"; +import type { ExportAdminCredentialTransfer200Kdf } from "./exportAdminCredentialTransfer200Kdf"; +import type { ExportAdminCredentialTransfer200Version } from "./exportAdminCredentialTransfer200Version"; + +export type ExportAdminCredentialTransfer200 = { + format: ExportAdminCredentialTransfer200Format; + version: ExportAdminCredentialTransfer200Version; + kdf: ExportAdminCredentialTransfer200Kdf; + cipher: ExportAdminCredentialTransfer200Cipher; + /** + * @minLength 1 + * @maxLength 1048576 + * @pattern ^[A-Za-z0-9_-]+$ + */ + ciphertext: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Cipher.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Cipher.ts new file mode 100644 index 000000000..f4fec531b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Cipher.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminCredentialTransfer200CipherName } from "./exportAdminCredentialTransfer200CipherName"; + +export type ExportAdminCredentialTransfer200Cipher = { + name: ExportAdminCredentialTransfer200CipherName; + /** + * @minLength 16 + * @maxLength 16 + * @pattern ^[A-Za-z0-9_-]+$ + */ + iv: string; + /** + * @minLength 22 + * @maxLength 22 + * @pattern ^[A-Za-z0-9_-]+$ + */ + tag: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200CipherName.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200CipherName.ts new file mode 100644 index 000000000..22230e789 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200CipherName.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer200CipherName = + (typeof ExportAdminCredentialTransfer200CipherName)[keyof typeof ExportAdminCredentialTransfer200CipherName]; + +export const ExportAdminCredentialTransfer200CipherName = { + "aes-256-gcm": "aes-256-gcm", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Format.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Format.ts new file mode 100644 index 000000000..fa99a03bc --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Format.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer200Format = + (typeof ExportAdminCredentialTransfer200Format)[keyof typeof ExportAdminCredentialTransfer200Format]; + +export const ExportAdminCredentialTransfer200Format = { + "openmapx-vault-transfer": "openmapx-vault-transfer", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Kdf.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Kdf.ts new file mode 100644 index 000000000..b15a3ef1b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Kdf.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminCredentialTransfer200KdfN } from "./exportAdminCredentialTransfer200KdfN"; +import type { ExportAdminCredentialTransfer200KdfName } from "./exportAdminCredentialTransfer200KdfName"; +import type { ExportAdminCredentialTransfer200KdfP } from "./exportAdminCredentialTransfer200KdfP"; +import type { ExportAdminCredentialTransfer200KdfR } from "./exportAdminCredentialTransfer200KdfR"; + +export type ExportAdminCredentialTransfer200Kdf = { + name: ExportAdminCredentialTransfer200KdfName; + N: ExportAdminCredentialTransfer200KdfN; + r: ExportAdminCredentialTransfer200KdfR; + p: ExportAdminCredentialTransfer200KdfP; + /** + * @minLength 22 + * @maxLength 22 + * @pattern ^[A-Za-z0-9_-]+$ + */ + salt: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfN.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfN.ts new file mode 100644 index 000000000..3fa25b789 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfN.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer200KdfN = + (typeof ExportAdminCredentialTransfer200KdfN)[keyof typeof ExportAdminCredentialTransfer200KdfN]; + +export const ExportAdminCredentialTransfer200KdfN = { + NUMBER_32768: 32768, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfName.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfName.ts new file mode 100644 index 000000000..934caf881 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfName.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer200KdfName = + (typeof ExportAdminCredentialTransfer200KdfName)[keyof typeof ExportAdminCredentialTransfer200KdfName]; + +export const ExportAdminCredentialTransfer200KdfName = { + scrypt: "scrypt", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfP.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfP.ts new file mode 100644 index 000000000..38a8e6c7e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfP.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer200KdfP = + (typeof ExportAdminCredentialTransfer200KdfP)[keyof typeof ExportAdminCredentialTransfer200KdfP]; + +export const ExportAdminCredentialTransfer200KdfP = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfR.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfR.ts new file mode 100644 index 000000000..9064d74b4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200KdfR.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer200KdfR = + (typeof ExportAdminCredentialTransfer200KdfR)[keyof typeof ExportAdminCredentialTransfer200KdfR]; + +export const ExportAdminCredentialTransfer200KdfR = { + NUMBER_8: 8, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Version.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Version.ts new file mode 100644 index 000000000..17d7216dc --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer200Version.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer200Version = + (typeof ExportAdminCredentialTransfer200Version)[keyof typeof ExportAdminCredentialTransfer200Version]; + +export const ExportAdminCredentialTransfer200Version = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer400.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer400.ts new file mode 100644 index 000000000..49c70f2cd --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer401.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer401.ts new file mode 100644 index 000000000..9827bcb47 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer403.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer403.ts new file mode 100644 index 000000000..e5b15f28e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer409.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer409.ts new file mode 100644 index 000000000..64fb18be5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer413.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer413.ts new file mode 100644 index 000000000..91fcdc497 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer415.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer415.ts new file mode 100644 index 000000000..51c390164 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer429.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer429.ts new file mode 100644 index 000000000..0fd1e5502 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer500.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer500.ts new file mode 100644 index 000000000..c49def214 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer503.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer503.ts new file mode 100644 index 000000000..aa0492cf6 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransfer503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransfer503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBody.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBody.ts new file mode 100644 index 000000000..8f54567cf --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBody.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminCredentialTransferBodySelectionItem } from "./exportAdminCredentialTransferBodySelectionItem"; + +export type ExportAdminCredentialTransferBody = { + /** + * @minItems 1 + * @maxItems 1000 + */ + selection: ExportAdminCredentialTransferBodySelectionItem[]; + passphrase: string; + confirmed: true; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBodySelectionItem.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBodySelectionItem.ts new file mode 100644 index 000000000..f64259025 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBodySelectionItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminCredentialTransferBodySelectionItemScope } from "./exportAdminCredentialTransferBodySelectionItemScope"; + +export type ExportAdminCredentialTransferBodySelectionItem = { + scope: ExportAdminCredentialTransferBodySelectionItemScope; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + id: string; + /** + * @minLength 1 + * @maxLength 128 + */ + key: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBodySelectionItemScope.ts b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBodySelectionItemScope.ts new file mode 100644 index 000000000..1011b60c8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/exportAdminCredentialTransferBodySelectionItemScope.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminCredentialTransferBodySelectionItemScope = + (typeof ExportAdminCredentialTransferBodySelectionItemScope)[keyof typeof ExportAdminCredentialTransferBodySelectionItemScope]; + +export const ExportAdminCredentialTransferBodySelectionItemScope = { + integration: "integration", + service: "service", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200.ts new file mode 100644 index 000000000..872f82914 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200Document } from "./getAdminConfigurationMigrationInventory200Document"; +import type { GetAdminConfigurationMigrationInventory200FieldsItem } from "./getAdminConfigurationMigrationInventory200FieldsItem"; + +export type GetAdminConfigurationMigrationInventory200 = { + document: GetAdminConfigurationMigrationInventory200Document; + /** @maxItems 1000 */ + fields: GetAdminConfigurationMigrationInventory200FieldsItem[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200Document.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200Document.ts new file mode 100644 index 000000000..eade09a47 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200Document.ts @@ -0,0 +1,30 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200DocumentBindingsItem } from "./getAdminConfigurationMigrationInventory200DocumentBindingsItem"; +import type { GetAdminConfigurationMigrationInventory200DocumentFormat } from "./getAdminConfigurationMigrationInventory200DocumentFormat"; +import type { GetAdminConfigurationMigrationInventory200DocumentInstallationsItem } from "./getAdminConfigurationMigrationInventory200DocumentInstallationsItem"; +import type { GetAdminConfigurationMigrationInventory200DocumentIntegrationsItem } from "./getAdminConfigurationMigrationInventory200DocumentIntegrationsItem"; +import type { GetAdminConfigurationMigrationInventory200DocumentPlatform } from "./getAdminConfigurationMigrationInventory200DocumentPlatform"; +import type { GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItem } from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItem"; +import type { GetAdminConfigurationMigrationInventory200DocumentServiceConfigsItem } from "./getAdminConfigurationMigrationInventory200DocumentServiceConfigsItem"; +import type { GetAdminConfigurationMigrationInventory200DocumentSettings } from "./getAdminConfigurationMigrationInventory200DocumentSettings"; +import type { GetAdminConfigurationMigrationInventory200DocumentVersion } from "./getAdminConfigurationMigrationInventory200DocumentVersion"; + +export type GetAdminConfigurationMigrationInventory200Document = { + format: GetAdminConfigurationMigrationInventory200DocumentFormat; + version: GetAdminConfigurationMigrationInventory200DocumentVersion; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + exportedAt: string; + settings: GetAdminConfigurationMigrationInventory200DocumentSettings; + platform?: GetAdminConfigurationMigrationInventory200DocumentPlatform; + /** @maxItems 256 */ + integrations: GetAdminConfigurationMigrationInventory200DocumentIntegrationsItem[]; + /** @maxItems 256 */ + serviceConfigs: GetAdminConfigurationMigrationInventory200DocumentServiceConfigsItem[]; + /** @maxItems 512 */ + bindings: GetAdminConfigurationMigrationInventory200DocumentBindingsItem[]; + /** @maxItems 2048 */ + installations: GetAdminConfigurationMigrationInventory200DocumentInstallationsItem[]; + /** @maxItems 1000 */ + prerequisites: GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItem[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentBindingsItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentBindingsItem.ts new file mode 100644 index 000000000..5076fc3a1 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentBindingsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentBindingsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + /** + * @minLength 1 + * @maxLength 128 + */ + capability: string; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + serviceId: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentFormat.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentFormat.ts new file mode 100644 index 000000000..076e8d3be --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentFormat.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentFormat = + (typeof GetAdminConfigurationMigrationInventory200DocumentFormat)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentFormat]; + +export const GetAdminConfigurationMigrationInventory200DocumentFormat = { + "openmapx-configuration": "openmapx-configuration", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItem.ts new file mode 100644 index 000000000..a0f4efd03 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItem.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership } from "./getAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership"; +import type { GetAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance } from "./getAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance"; + +export type GetAdminConfigurationMigrationInventory200DocumentInstallationsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + version: string | null; + contentDigest: string | null; + denialReason: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + pendingTransactionId?: string; + ownership: GetAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership; + extensionId: string | null; + provenance: GetAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance; + adoptable: boolean; + removable: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership.ts new file mode 100644 index 000000000..1e0cd5dc5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership = + (typeof GetAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership]; + +export const GetAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership = { + extension: "extension", + standalone: "standalone", + unmanaged: "unmanaged", + conflict: "conflict", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance.ts new file mode 100644 index 000000000..aa8177558 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance = + (typeof GetAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance]; + +export const GetAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance = { + artifact: "artifact", + source: "source", + "uploaded-artifact": "uploaded-artifact", + legacy_unknown: "legacy_unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentIntegrationsItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentIntegrationsItem.ts new file mode 100644 index 000000000..a5abcf5ae --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentIntegrationsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig } from "./getAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig"; + +export type GetAdminConfigurationMigrationInventory200DocumentIntegrationsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + config: GetAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig.ts new file mode 100644 index 000000000..73c9b7486 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminGetAdminConfigurationMigrationInventory200Schema0 } from "./adminGetAdminConfigurationMigrationInventory200Schema0"; + +export type GetAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig = { + [key: string]: AdminGetAdminConfigurationMigrationInventory200Schema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatform.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatform.ts new file mode 100644 index 000000000..fff079722 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatform.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail } from "./getAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail"; +import type { GetAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays } from "./getAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays"; +import type { GetAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion } from "./getAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion"; +import type { GetAdminConfigurationMigrationInventory200DocumentPlatformDomain } from "./getAdminConfigurationMigrationInventory200DocumentPlatformDomain"; +import type { GetAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector } from "./getAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector"; +import type { GetAdminConfigurationMigrationInventory200DocumentPlatformVersion } from "./getAdminConfigurationMigrationInventory200DocumentPlatformVersion"; + +export type GetAdminConfigurationMigrationInventory200DocumentPlatform = { + version: GetAdminConfigurationMigrationInventory200DocumentPlatformVersion; + domain?: GetAdminConfigurationMigrationInventory200DocumentPlatformDomain; + acmeEmail?: GetAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail; + releaseSelector?: GetAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector; + defaultRegion?: GetAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion; + backupRetentionDays?: GetAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail.ts new file mode 100644 index 000000000..05c3cc377 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail = + | { + source: "host"; + } + | { + source: "database"; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays.ts new file mode 100644 index 000000000..961c70eb2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minimum 1 + * @maximum 36500 + */ + value: number; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion.ts new file mode 100644 index 000000000..e50f1647c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformDomain.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformDomain.ts new file mode 100644 index 000000000..deddd12ec --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformDomain.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPlatformDomain = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 253 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector.ts new file mode 100644 index 000000000..762e1e844 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 128 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformVersion.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformVersion.ts new file mode 100644 index 000000000..9b3925ecb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPlatformVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPlatformVersion = + (typeof GetAdminConfigurationMigrationInventory200DocumentPlatformVersion)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentPlatformVersion]; + +export const GetAdminConfigurationMigrationInventory200DocumentPlatformVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItem.ts new file mode 100644 index 000000000..724b274dd --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItem.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason } from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason"; +import type { GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope } from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope"; +import type { GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource } from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource"; + +export type GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItem = { + scope: GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope; + /** + * @minLength 1 + * @maxLength 128 + */ + id: string; + /** @maxLength 128 */ + key?: string; + source: GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource; + reason: GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason.ts new file mode 100644 index 000000000..c2d3208f4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason = + (typeof GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason]; + +export const GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason = { + host_owned: "host_owned", + credential_required: "credential_required", + installation_required: "installation_required", + unsupported: "unsupported", + protected_bootstrap: "protected_bootstrap", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope.ts new file mode 100644 index 000000000..689a51084 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope = + (typeof GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope]; + +export const GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope = { + integration: "integration", + service: "service", + platform: "platform", + account: "account", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource.ts new file mode 100644 index 000000000..db8a4ccd4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource = + (typeof GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource]; + +export const GetAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", + missing: "missing", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentServiceConfigsItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentServiceConfigsItem.ts new file mode 100644 index 000000000..724b0296e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentServiceConfigsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig } from "./getAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig"; + +export type GetAdminConfigurationMigrationInventory200DocumentServiceConfigsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + serviceId: string; + config: GetAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig.ts new file mode 100644 index 000000000..fdf13de4c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminGetAdminConfigurationMigrationInventory200Schema0 } from "./adminGetAdminConfigurationMigrationInventory200Schema0"; + +export type GetAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig = { + [key: string]: AdminGetAdminConfigurationMigrationInventory200Schema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentSettings.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentSettings.ts new file mode 100644 index 000000000..ba63c1966 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentSettings.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminGetAdminConfigurationMigrationInventory200Schema0 } from "./adminGetAdminConfigurationMigrationInventory200Schema0"; + +export type GetAdminConfigurationMigrationInventory200DocumentSettings = { + [key: string]: AdminGetAdminConfigurationMigrationInventory200Schema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentVersion.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentVersion.ts new file mode 100644 index 000000000..8fec5f95b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200DocumentVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200DocumentVersion = + (typeof GetAdminConfigurationMigrationInventory200DocumentVersion)[keyof typeof GetAdminConfigurationMigrationInventory200DocumentVersion]; + +export const GetAdminConfigurationMigrationInventory200DocumentVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItem.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItem.ts new file mode 100644 index 000000000..4b88dafd8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItem.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200FieldsItemOwnership } from "./getAdminConfigurationMigrationInventory200FieldsItemOwnership"; +import type { GetAdminConfigurationMigrationInventory200FieldsItemScope } from "./getAdminConfigurationMigrationInventory200FieldsItemScope"; + +export type GetAdminConfigurationMigrationInventory200FieldsItem = { + scope: GetAdminConfigurationMigrationInventory200FieldsItemScope; + id: string; + key: string; + ownership: GetAdminConfigurationMigrationInventory200FieldsItemOwnership; + included: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnership.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnership.ts new file mode 100644 index 000000000..516b938ac --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnership.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption } from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption"; +import type { GetAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy } from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy"; +import type { GetAdminConfigurationMigrationInventory200FieldsItemOwnershipReason } from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipReason"; +import type { GetAdminConfigurationMigrationInventory200FieldsItemOwnershipSource } from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipSource"; + +export type GetAdminConfigurationMigrationInventory200FieldsItemOwnership = { + source: GetAdminConfigurationMigrationInventory200FieldsItemOwnershipSource; + writable: boolean; + secret: boolean; + adoption: GetAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption; + reason: GetAdminConfigurationMigrationInventory200FieldsItemOwnershipReason; + /** @pattern ^[A-Z_][A-Z0-9_]{0,127}$ */ + envVar?: string; + managedBy?: GetAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption.ts new file mode 100644 index 000000000..6047095bb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption = + (typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption)[keyof typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption]; + +export const GetAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption = { + direct: "direct", + host_action_required: "host_action_required", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy.ts new file mode 100644 index 000000000..25976f32b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy = + (typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy)[keyof typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy]; + +export const GetAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy = { + transitcredentials: "transit.credentials", + "dawarich-provisioning": "dawarich-provisioning", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipReason.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipReason.ts new file mode 100644 index 000000000..d34ebe0db --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200FieldsItemOwnershipReason = + | (typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipReason)[keyof typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipReason] + | null; + +export const GetAdminConfigurationMigrationInventory200FieldsItemOwnershipReason = { + host_environment: "host_environment", + host_file: "host_file", + vault_only: "vault_only", + managed_workflow: "managed_workflow", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipSource.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipSource.ts new file mode 100644 index 000000000..2048a178e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemOwnershipSource.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200FieldsItemOwnershipSource = + (typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipSource)[keyof typeof GetAdminConfigurationMigrationInventory200FieldsItemOwnershipSource]; + +export const GetAdminConfigurationMigrationInventory200FieldsItemOwnershipSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemScope.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemScope.ts new file mode 100644 index 000000000..c9c581473 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory200FieldsItemScope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory200FieldsItemScope = + (typeof GetAdminConfigurationMigrationInventory200FieldsItemScope)[keyof typeof GetAdminConfigurationMigrationInventory200FieldsItemScope]; + +export const GetAdminConfigurationMigrationInventory200FieldsItemScope = { + integration: "integration", + service: "service", + platform: "platform", + account: "account", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory400.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory400.ts new file mode 100644 index 000000000..8072d0652 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory401.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory401.ts new file mode 100644 index 000000000..56710d1f0 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory403.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory403.ts new file mode 100644 index 000000000..9c0b10194 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory409.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory409.ts new file mode 100644 index 000000000..a44e6b92a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory413.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory413.ts new file mode 100644 index 000000000..64b0358eb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory415.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory415.ts new file mode 100644 index 000000000..0616d0208 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory429.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory429.ts new file mode 100644 index 000000000..983023529 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory500.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory500.ts new file mode 100644 index 000000000..4bb13a02a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory503.ts b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory503.ts new file mode 100644 index 000000000..d0634865d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/getAdminConfigurationMigrationInventory503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminConfigurationMigrationInventory503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/index.ts b/packages/core/src/api/generated/admin/integration-models/index.ts index 41bc19bc7..b64f65b67 100644 --- a/packages/core/src/api/generated/admin/integration-models/index.ts +++ b/packages/core/src/api/generated/admin/integration-models/index.ts @@ -1,12 +1,15 @@ // biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. // Generated by pnpm sdk:generate. Do not edit. +export * from "./adminExportAdminConfigurationMigration200Schema0"; export * from "./adminExportAdminIntegrationConfigs200Schema0"; +export * from "./adminGetAdminConfigurationMigrationInventory200Schema0"; export * from "./adminGetAdminIntegration200Schema0"; export * from "./adminGetAdminOverview200Schema0"; export * from "./adminImportAdminIntegrationConfigsBodySchema0"; export * from "./adminListAdminIntegrationEnvVars200Schema0"; export * from "./adminPatchAdminIntegrationConfigBodySchema0"; +export * from "./adminPreviewAdminConfigurationMigrationBodySchema0"; export * from "./adoptAdminStandaloneIntegration202"; export * from "./adoptAdminStandaloneIntegration400"; export * from "./adoptAdminStandaloneIntegration401"; @@ -19,6 +22,61 @@ export * from "./adoptAdminStandaloneIntegration500"; export * from "./adoptAdminStandaloneIntegration503"; export * from "./adoptAdminStandaloneIntegrationBody"; export * from "./adoptAdminStandaloneIntegrationHeaders"; +export * from "./applyAdminConfigurationMigration200"; +export * from "./applyAdminConfigurationMigration200Outcome"; +export * from "./applyAdminConfigurationMigration200StepsItem"; +export * from "./applyAdminConfigurationMigration200StepsItemStatus"; +export * from "./applyAdminConfigurationMigration400"; +export * from "./applyAdminConfigurationMigration401"; +export * from "./applyAdminConfigurationMigration403"; +export * from "./applyAdminConfigurationMigration409"; +export * from "./applyAdminConfigurationMigration413"; +export * from "./applyAdminConfigurationMigration415"; +export * from "./applyAdminConfigurationMigration429"; +export * from "./applyAdminConfigurationMigration500"; +export * from "./applyAdminConfigurationMigration503"; +export * from "./applyAdminConfigurationMigrationBody"; +export * from "./applyAdminConfigurationMigrationBodyPlatformAuthorization"; +export * from "./applyAdminConfigurationMigrationBodyPlatformAuthorizationConfirmations"; +export * from "./applyAdminConfigurationMigrationHeaders"; +export * from "./applyAdminCredentialTransfer200"; +export * from "./applyAdminCredentialTransfer200Outcome"; +export * from "./applyAdminCredentialTransfer200StepsItem"; +export * from "./applyAdminCredentialTransfer200StepsItemStatus"; +export * from "./applyAdminCredentialTransfer400"; +export * from "./applyAdminCredentialTransfer401"; +export * from "./applyAdminCredentialTransfer403"; +export * from "./applyAdminCredentialTransfer409"; +export * from "./applyAdminCredentialTransfer413"; +export * from "./applyAdminCredentialTransfer415"; +export * from "./applyAdminCredentialTransfer429"; +export * from "./applyAdminCredentialTransfer500"; +export * from "./applyAdminCredentialTransfer503"; +export * from "./applyAdminCredentialTransferBody"; +export * from "./applyAdminCredentialTransferHeaders"; +export * from "./chooseAdminCredentialTransferFields200"; +export * from "./chooseAdminCredentialTransferFields200EntriesItem"; +export * from "./chooseAdminCredentialTransferFields200EntriesItemScope"; +export * from "./chooseAdminCredentialTransferFields200EntriesItemStatus"; +export * from "./chooseAdminCredentialTransferFields200FeedMap"; +export * from "./chooseAdminCredentialTransferFields200FeedMapChangesItem"; +export * from "./chooseAdminCredentialTransferFields200FeedMapChangesItemAction"; +export * from "./chooseAdminCredentialTransferFields400"; +export * from "./chooseAdminCredentialTransferFields401"; +export * from "./chooseAdminCredentialTransferFields403"; +export * from "./chooseAdminCredentialTransferFields409"; +export * from "./chooseAdminCredentialTransferFields413"; +export * from "./chooseAdminCredentialTransferFields415"; +export * from "./chooseAdminCredentialTransferFields429"; +export * from "./chooseAdminCredentialTransferFields500"; +export * from "./chooseAdminCredentialTransferFields503"; +export * from "./chooseAdminCredentialTransferFieldsBody"; +export * from "./chooseAdminCredentialTransferFieldsBodyChoices"; +export * from "./chooseAdminCredentialTransferFieldsBodyChoicesEntriesItem"; +export * from "./chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemAction"; +export * from "./chooseAdminCredentialTransferFieldsBodyChoicesEntriesItemScope"; +export * from "./chooseAdminCredentialTransferFieldsBodyChoicesFeedMap"; +export * from "./chooseAdminCredentialTransferFieldsHeaders"; export * from "./deleteAdminIntegrationBinding200"; export * from "./deleteAdminIntegrationBinding200Database"; export * from "./deleteAdminIntegrationBinding200HostApply"; @@ -76,6 +134,60 @@ export * from "./enableAdminIntegration429"; export * from "./enableAdminIntegration500"; export * from "./enableAdminIntegrationBody"; export * from "./enableAdminIntegrationHeaders"; +export * from "./exportAdminConfigurationMigration200"; +export * from "./exportAdminConfigurationMigration200BindingsItem"; +export * from "./exportAdminConfigurationMigration200Format"; +export * from "./exportAdminConfigurationMigration200InstallationsItem"; +export * from "./exportAdminConfigurationMigration200InstallationsItemOwnership"; +export * from "./exportAdminConfigurationMigration200InstallationsItemProvenance"; +export * from "./exportAdminConfigurationMigration200IntegrationsItem"; +export * from "./exportAdminConfigurationMigration200IntegrationsItemConfig"; +export * from "./exportAdminConfigurationMigration200Platform"; +export * from "./exportAdminConfigurationMigration200PlatformAcmeEmail"; +export * from "./exportAdminConfigurationMigration200PlatformBackupRetentionDays"; +export * from "./exportAdminConfigurationMigration200PlatformDefaultRegion"; +export * from "./exportAdminConfigurationMigration200PlatformDomain"; +export * from "./exportAdminConfigurationMigration200PlatformReleaseSelector"; +export * from "./exportAdminConfigurationMigration200PlatformVersion"; +export * from "./exportAdminConfigurationMigration200PrerequisitesItem"; +export * from "./exportAdminConfigurationMigration200PrerequisitesItemReason"; +export * from "./exportAdminConfigurationMigration200PrerequisitesItemScope"; +export * from "./exportAdminConfigurationMigration200PrerequisitesItemSource"; +export * from "./exportAdminConfigurationMigration200ServiceConfigsItem"; +export * from "./exportAdminConfigurationMigration200ServiceConfigsItemConfig"; +export * from "./exportAdminConfigurationMigration200Settings"; +export * from "./exportAdminConfigurationMigration200Version"; +export * from "./exportAdminConfigurationMigration400"; +export * from "./exportAdminConfigurationMigration401"; +export * from "./exportAdminConfigurationMigration403"; +export * from "./exportAdminConfigurationMigration409"; +export * from "./exportAdminConfigurationMigration413"; +export * from "./exportAdminConfigurationMigration415"; +export * from "./exportAdminConfigurationMigration429"; +export * from "./exportAdminConfigurationMigration500"; +export * from "./exportAdminConfigurationMigration503"; +export * from "./exportAdminCredentialTransfer200"; +export * from "./exportAdminCredentialTransfer200Cipher"; +export * from "./exportAdminCredentialTransfer200CipherName"; +export * from "./exportAdminCredentialTransfer200Format"; +export * from "./exportAdminCredentialTransfer200Kdf"; +export * from "./exportAdminCredentialTransfer200KdfN"; +export * from "./exportAdminCredentialTransfer200KdfName"; +export * from "./exportAdminCredentialTransfer200KdfP"; +export * from "./exportAdminCredentialTransfer200KdfR"; +export * from "./exportAdminCredentialTransfer200Version"; +export * from "./exportAdminCredentialTransfer400"; +export * from "./exportAdminCredentialTransfer401"; +export * from "./exportAdminCredentialTransfer403"; +export * from "./exportAdminCredentialTransfer409"; +export * from "./exportAdminCredentialTransfer413"; +export * from "./exportAdminCredentialTransfer415"; +export * from "./exportAdminCredentialTransfer429"; +export * from "./exportAdminCredentialTransfer500"; +export * from "./exportAdminCredentialTransfer503"; +export * from "./exportAdminCredentialTransferBody"; +export * from "./exportAdminCredentialTransferBodySelectionItem"; +export * from "./exportAdminCredentialTransferBodySelectionItemScope"; export * from "./exportAdminIntegrationConfigs200"; export * from "./exportAdminIntegrationConfigs200IntegrationsItem"; export * from "./exportAdminIntegrationConfigs200IntegrationsItemConfig"; @@ -86,6 +198,46 @@ export * from "./exportAdminIntegrationConfigs413"; export * from "./exportAdminIntegrationConfigs415"; export * from "./exportAdminIntegrationConfigs429"; export * from "./exportAdminIntegrationConfigs500"; +export * from "./getAdminConfigurationMigrationInventory200"; +export * from "./getAdminConfigurationMigrationInventory200Document"; +export * from "./getAdminConfigurationMigrationInventory200DocumentBindingsItem"; +export * from "./getAdminConfigurationMigrationInventory200DocumentFormat"; +export * from "./getAdminConfigurationMigrationInventory200DocumentInstallationsItem"; +export * from "./getAdminConfigurationMigrationInventory200DocumentInstallationsItemOwnership"; +export * from "./getAdminConfigurationMigrationInventory200DocumentInstallationsItemProvenance"; +export * from "./getAdminConfigurationMigrationInventory200DocumentIntegrationsItem"; +export * from "./getAdminConfigurationMigrationInventory200DocumentIntegrationsItemConfig"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPlatform"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPlatformAcmeEmail"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPlatformBackupRetentionDays"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPlatformDefaultRegion"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPlatformDomain"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPlatformReleaseSelector"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPlatformVersion"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItem"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemReason"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemScope"; +export * from "./getAdminConfigurationMigrationInventory200DocumentPrerequisitesItemSource"; +export * from "./getAdminConfigurationMigrationInventory200DocumentServiceConfigsItem"; +export * from "./getAdminConfigurationMigrationInventory200DocumentServiceConfigsItemConfig"; +export * from "./getAdminConfigurationMigrationInventory200DocumentSettings"; +export * from "./getAdminConfigurationMigrationInventory200DocumentVersion"; +export * from "./getAdminConfigurationMigrationInventory200FieldsItem"; +export * from "./getAdminConfigurationMigrationInventory200FieldsItemOwnership"; +export * from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipAdoption"; +export * from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipManagedBy"; +export * from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipReason"; +export * from "./getAdminConfigurationMigrationInventory200FieldsItemOwnershipSource"; +export * from "./getAdminConfigurationMigrationInventory200FieldsItemScope"; +export * from "./getAdminConfigurationMigrationInventory400"; +export * from "./getAdminConfigurationMigrationInventory401"; +export * from "./getAdminConfigurationMigrationInventory403"; +export * from "./getAdminConfigurationMigrationInventory409"; +export * from "./getAdminConfigurationMigrationInventory413"; +export * from "./getAdminConfigurationMigrationInventory415"; +export * from "./getAdminConfigurationMigrationInventory429"; +export * from "./getAdminConfigurationMigrationInventory500"; +export * from "./getAdminConfigurationMigrationInventory503"; export * from "./getAdminIntegration200"; export * from "./getAdminIntegration200Application"; export * from "./getAdminIntegration200ApplicationDatabase"; @@ -320,6 +472,98 @@ export * from "./patchAdminIntegrationConfig429"; export * from "./patchAdminIntegrationConfig500"; export * from "./patchAdminIntegrationConfigBody"; export * from "./patchAdminIntegrationConfigHeaders"; +export * from "./previewAdminConfigurationMigration200"; +export * from "./previewAdminConfigurationMigration200ChangesItem"; +export * from "./previewAdminConfigurationMigration200Platform"; +export * from "./previewAdminConfigurationMigration200PlatformConfiguration"; +export * from "./previewAdminConfigurationMigration200PlatformConfigurationAcmeEmail"; +export * from "./previewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays"; +export * from "./previewAdminConfigurationMigration200PlatformConfigurationDefaultRegion"; +export * from "./previewAdminConfigurationMigration200PlatformConfigurationDomain"; +export * from "./previewAdminConfigurationMigration200PlatformConfigurationReleaseSelector"; +export * from "./previewAdminConfigurationMigration200PlatformConfigurationVersion"; +export * from "./previewAdminConfigurationMigration200PlatformOriginConsumers"; +export * from "./previewAdminConfigurationMigration200PlatformReconcileServiceIdsItem"; +export * from "./previewAdminConfigurationMigration200PlatformRelease"; +export * from "./previewAdminConfigurationMigration200PlatformRequiredConfirmation"; +export * from "./previewAdminConfigurationMigration200PlatformSources"; +export * from "./previewAdminConfigurationMigration200PlatformSourcesAcmeEmail"; +export * from "./previewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays"; +export * from "./previewAdminConfigurationMigration200PlatformSourcesDefaultRegion"; +export * from "./previewAdminConfigurationMigration200PlatformSourcesDomain"; +export * from "./previewAdminConfigurationMigration200PlatformSourcesReleaseSelector"; +export * from "./previewAdminConfigurationMigration200PlatformValues"; +export * from "./previewAdminConfigurationMigration200PlatformVersion"; +export * from "./previewAdminConfigurationMigration200PrerequisitesItem"; +export * from "./previewAdminConfigurationMigration200PrerequisitesItemReason"; +export * from "./previewAdminConfigurationMigration200PrerequisitesItemScope"; +export * from "./previewAdminConfigurationMigration200PrerequisitesItemSource"; +export * from "./previewAdminConfigurationMigration400"; +export * from "./previewAdminConfigurationMigration401"; +export * from "./previewAdminConfigurationMigration403"; +export * from "./previewAdminConfigurationMigration409"; +export * from "./previewAdminConfigurationMigration413"; +export * from "./previewAdminConfigurationMigration415"; +export * from "./previewAdminConfigurationMigration429"; +export * from "./previewAdminConfigurationMigration500"; +export * from "./previewAdminConfigurationMigration503"; +export * from "./previewAdminConfigurationMigrationBody"; +export * from "./previewAdminConfigurationMigrationBodyBindingsItem"; +export * from "./previewAdminConfigurationMigrationBodyFormat"; +export * from "./previewAdminConfigurationMigrationBodyInstallationsItem"; +export * from "./previewAdminConfigurationMigrationBodyInstallationsItemOwnership"; +export * from "./previewAdminConfigurationMigrationBodyInstallationsItemProvenance"; +export * from "./previewAdminConfigurationMigrationBodyIntegrationsItem"; +export * from "./previewAdminConfigurationMigrationBodyIntegrationsItemConfig"; +export * from "./previewAdminConfigurationMigrationBodyPlatform"; +export * from "./previewAdminConfigurationMigrationBodyPlatformAcmeEmail"; +export * from "./previewAdminConfigurationMigrationBodyPlatformBackupRetentionDays"; +export * from "./previewAdminConfigurationMigrationBodyPlatformDefaultRegion"; +export * from "./previewAdminConfigurationMigrationBodyPlatformDomain"; +export * from "./previewAdminConfigurationMigrationBodyPlatformReleaseSelector"; +export * from "./previewAdminConfigurationMigrationBodyPlatformVersion"; +export * from "./previewAdminConfigurationMigrationBodyPrerequisitesItem"; +export * from "./previewAdminConfigurationMigrationBodyPrerequisitesItemReason"; +export * from "./previewAdminConfigurationMigrationBodyPrerequisitesItemScope"; +export * from "./previewAdminConfigurationMigrationBodyPrerequisitesItemSource"; +export * from "./previewAdminConfigurationMigrationBodyServiceConfigsItem"; +export * from "./previewAdminConfigurationMigrationBodyServiceConfigsItemConfig"; +export * from "./previewAdminConfigurationMigrationBodySettings"; +export * from "./previewAdminConfigurationMigrationBodyVersion"; +export * from "./previewAdminConfigurationMigrationHeaders"; +export * from "./previewAdminCredentialTransfer200"; +export * from "./previewAdminCredentialTransfer200EntriesItem"; +export * from "./previewAdminCredentialTransfer200EntriesItemScope"; +export * from "./previewAdminCredentialTransfer200EntriesItemStatus"; +export * from "./previewAdminCredentialTransfer200FeedMap"; +export * from "./previewAdminCredentialTransfer200FeedMapChangesItem"; +export * from "./previewAdminCredentialTransfer200FeedMapChangesItemAction"; +export * from "./previewAdminCredentialTransfer400"; +export * from "./previewAdminCredentialTransfer401"; +export * from "./previewAdminCredentialTransfer403"; +export * from "./previewAdminCredentialTransfer409"; +export * from "./previewAdminCredentialTransfer413"; +export * from "./previewAdminCredentialTransfer415"; +export * from "./previewAdminCredentialTransfer429"; +export * from "./previewAdminCredentialTransfer500"; +export * from "./previewAdminCredentialTransfer503"; +export * from "./previewAdminCredentialTransferBody"; +export * from "./previewAdminCredentialTransferBodyChoices"; +export * from "./previewAdminCredentialTransferBodyChoicesEntriesItem"; +export * from "./previewAdminCredentialTransferBodyChoicesEntriesItemAction"; +export * from "./previewAdminCredentialTransferBodyChoicesEntriesItemScope"; +export * from "./previewAdminCredentialTransferBodyChoicesFeedMap"; +export * from "./previewAdminCredentialTransferBodyEnvelope"; +export * from "./previewAdminCredentialTransferBodyEnvelopeCipher"; +export * from "./previewAdminCredentialTransferBodyEnvelopeCipherName"; +export * from "./previewAdminCredentialTransferBodyEnvelopeFormat"; +export * from "./previewAdminCredentialTransferBodyEnvelopeKdf"; +export * from "./previewAdminCredentialTransferBodyEnvelopeKdfN"; +export * from "./previewAdminCredentialTransferBodyEnvelopeKdfName"; +export * from "./previewAdminCredentialTransferBodyEnvelopeKdfP"; +export * from "./previewAdminCredentialTransferBodyEnvelopeKdfR"; +export * from "./previewAdminCredentialTransferBodyEnvelopeVersion"; +export * from "./previewAdminCredentialTransferHeaders"; export * from "./previewAdminStandaloneIntegration202"; export * from "./previewAdminStandaloneIntegration400"; export * from "./previewAdminStandaloneIntegration401"; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200.ts new file mode 100644 index 000000000..a61f91d5d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigration200ChangesItem } from "./previewAdminConfigurationMigration200ChangesItem"; +import type { PreviewAdminConfigurationMigration200Platform } from "./previewAdminConfigurationMigration200Platform"; +import type { PreviewAdminConfigurationMigration200PrerequisitesItem } from "./previewAdminConfigurationMigration200PrerequisitesItem"; + +export type PreviewAdminConfigurationMigration200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + fingerprint: string; + /** @nullable */ + expectedCurrentRevisionId: string | null; + /** @maxItems 512 */ + changes: PreviewAdminConfigurationMigration200ChangesItem[]; + /** @maxItems 1000 */ + prerequisites: PreviewAdminConfigurationMigration200PrerequisitesItem[]; + /** @maxItems 1000 */ + denials: string[]; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + platform?: PreviewAdminConfigurationMigration200Platform; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200ChangesItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200ChangesItem.ts new file mode 100644 index 000000000..6fd04c952 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200ChangesItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200ChangesItem = { + scope: string; + id: string; + keys: string[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200Platform.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200Platform.ts new file mode 100644 index 000000000..b96d7849d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200Platform.ts @@ -0,0 +1,54 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigration200PlatformConfiguration } from "./previewAdminConfigurationMigration200PlatformConfiguration"; +import type { PreviewAdminConfigurationMigration200PlatformOriginConsumers } from "./previewAdminConfigurationMigration200PlatformOriginConsumers"; +import type { PreviewAdminConfigurationMigration200PlatformReconcileServiceIdsItem } from "./previewAdminConfigurationMigration200PlatformReconcileServiceIdsItem"; +import type { PreviewAdminConfigurationMigration200PlatformRelease } from "./previewAdminConfigurationMigration200PlatformRelease"; +import type { PreviewAdminConfigurationMigration200PlatformRequiredConfirmation } from "./previewAdminConfigurationMigration200PlatformRequiredConfirmation"; +import type { PreviewAdminConfigurationMigration200PlatformSources } from "./previewAdminConfigurationMigration200PlatformSources"; +import type { PreviewAdminConfigurationMigration200PlatformValues } from "./previewAdminConfigurationMigration200PlatformValues"; +import type { PreviewAdminConfigurationMigration200PlatformVersion } from "./previewAdminConfigurationMigration200PlatformVersion"; + +export type PreviewAdminConfigurationMigration200Platform = { + version: PreviewAdminConfigurationMigration200PlatformVersion; + /** + * @minLength 1 + * @maxLength 128 + */ + actorId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** @pattern ^[a-f0-9]{64}$ */ + authPolicyRevision: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + sourceRevision: string; + expectedCurrentRevisionId: string | null; + configuration: PreviewAdminConfigurationMigration200PlatformConfiguration; + values: PreviewAdminConfigurationMigration200PlatformValues; + sources: PreviewAdminConfigurationMigration200PlatformSources; + /** + * @maxItems 256 + * @items.minLength 1 + * @items.maxLength 64 + */ + affectedServiceIds: string[]; + /** + * @minItems 1 + * @maxItems 6 + */ + reconcileServiceIds?: PreviewAdminConfigurationMigration200PlatformReconcileServiceIdsItem[]; + release?: PreviewAdminConfigurationMigration200PlatformRelease; + /** @maxLength 2048 */ + oldOrigin: string; + /** @maxLength 2048 */ + newOrigin: string; + originConsumers: PreviewAdminConfigurationMigration200PlatformOriginConsumers; + requiredConfirmation: PreviewAdminConfigurationMigration200PlatformRequiredConfirmation; + /** + * @maxItems 16 + * @items.maxLength 512 + */ + warnings: string[]; + /** @pattern ^[a-f0-9]{64}$ */ + previewFingerprint: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfiguration.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfiguration.ts new file mode 100644 index 000000000..dddaf84c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfiguration.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigration200PlatformConfigurationAcmeEmail } from "./previewAdminConfigurationMigration200PlatformConfigurationAcmeEmail"; +import type { PreviewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays } from "./previewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays"; +import type { PreviewAdminConfigurationMigration200PlatformConfigurationDefaultRegion } from "./previewAdminConfigurationMigration200PlatformConfigurationDefaultRegion"; +import type { PreviewAdminConfigurationMigration200PlatformConfigurationDomain } from "./previewAdminConfigurationMigration200PlatformConfigurationDomain"; +import type { PreviewAdminConfigurationMigration200PlatformConfigurationReleaseSelector } from "./previewAdminConfigurationMigration200PlatformConfigurationReleaseSelector"; +import type { PreviewAdminConfigurationMigration200PlatformConfigurationVersion } from "./previewAdminConfigurationMigration200PlatformConfigurationVersion"; + +export type PreviewAdminConfigurationMigration200PlatformConfiguration = { + version: PreviewAdminConfigurationMigration200PlatformConfigurationVersion; + domain?: PreviewAdminConfigurationMigration200PlatformConfigurationDomain; + acmeEmail?: PreviewAdminConfigurationMigration200PlatformConfigurationAcmeEmail; + releaseSelector?: PreviewAdminConfigurationMigration200PlatformConfigurationReleaseSelector; + defaultRegion?: PreviewAdminConfigurationMigration200PlatformConfigurationDefaultRegion; + backupRetentionDays?: PreviewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationAcmeEmail.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationAcmeEmail.ts new file mode 100644 index 000000000..eae233f6b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationAcmeEmail.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformConfigurationAcmeEmail = + | { + source: "host"; + } + | { + source: "database"; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays.ts new file mode 100644 index 000000000..fdcff444d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformConfigurationBackupRetentionDays = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minimum 1 + * @maximum 36500 + */ + value: number; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationDefaultRegion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationDefaultRegion.ts new file mode 100644 index 000000000..5b7389e24 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationDefaultRegion.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformConfigurationDefaultRegion = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationDomain.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationDomain.ts new file mode 100644 index 000000000..8ba2d75c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationDomain.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformConfigurationDomain = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 253 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationReleaseSelector.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationReleaseSelector.ts new file mode 100644 index 000000000..4c410d9b2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationReleaseSelector.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformConfigurationReleaseSelector = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 128 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationVersion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationVersion.ts new file mode 100644 index 000000000..380816f0d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformConfigurationVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformConfigurationVersion = + (typeof PreviewAdminConfigurationMigration200PlatformConfigurationVersion)[keyof typeof PreviewAdminConfigurationMigration200PlatformConfigurationVersion]; + +export const PreviewAdminConfigurationMigration200PlatformConfigurationVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformOriginConsumers.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformOriginConsumers.ts new file mode 100644 index 000000000..39ad84d26 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformOriginConsumers.ts @@ -0,0 +1,25 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformOriginConsumers = { + /** + * @minLength 1 + * @maxLength 253 + */ + routingHost: string; + /** @maxLength 2048 */ + corsOrigin: string; + /** @maxLength 2048 */ + authUrl: string; + /** + * @minLength 1 + * @maxLength 253 + */ + passkeyRpId: string; + /** @maxLength 2048 */ + passkeyOrigin: string; + /** @maxLength 2048 */ + webApiUrl: string; + /** @maxLength 2048 */ + tilesPublicUrl: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformReconcileServiceIdsItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformReconcileServiceIdsItem.ts new file mode 100644 index 000000000..3f52b226a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformReconcileServiceIdsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformReconcileServiceIdsItem = + (typeof PreviewAdminConfigurationMigration200PlatformReconcileServiceIdsItem)[keyof typeof PreviewAdminConfigurationMigration200PlatformReconcileServiceIdsItem]; + +export const PreviewAdminConfigurationMigration200PlatformReconcileServiceIdsItem = { + postgis: "postgis", + redis: "redis", + "app-web": "app-web", + "app-api": "app-api", + traefik: "traefik", + "ops-agent": "ops-agent", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformRelease.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformRelease.ts new file mode 100644 index 000000000..2ddeb89df --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformRelease.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformRelease = { + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ + */ + releaseId: string; + /** @pattern ^[a-f0-9]{64}$ */ + manifestDigest: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformRequiredConfirmation.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformRequiredConfirmation.ts new file mode 100644 index 000000000..46fd53701 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformRequiredConfirmation.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformRequiredConfirmation = { + domain: boolean; + dnsTls: boolean; + oauth: boolean; + passkeys: boolean; + alternateLogin: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSources.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSources.ts new file mode 100644 index 000000000..e5d8f85ba --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSources.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigration200PlatformSourcesAcmeEmail } from "./previewAdminConfigurationMigration200PlatformSourcesAcmeEmail"; +import type { PreviewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays } from "./previewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays"; +import type { PreviewAdminConfigurationMigration200PlatformSourcesDefaultRegion } from "./previewAdminConfigurationMigration200PlatformSourcesDefaultRegion"; +import type { PreviewAdminConfigurationMigration200PlatformSourcesDomain } from "./previewAdminConfigurationMigration200PlatformSourcesDomain"; +import type { PreviewAdminConfigurationMigration200PlatformSourcesReleaseSelector } from "./previewAdminConfigurationMigration200PlatformSourcesReleaseSelector"; + +export type PreviewAdminConfigurationMigration200PlatformSources = { + domain: PreviewAdminConfigurationMigration200PlatformSourcesDomain; + acmeEmail: PreviewAdminConfigurationMigration200PlatformSourcesAcmeEmail; + releaseSelector: PreviewAdminConfigurationMigration200PlatformSourcesReleaseSelector; + defaultRegion: PreviewAdminConfigurationMigration200PlatformSourcesDefaultRegion; + backupRetentionDays: PreviewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesAcmeEmail.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesAcmeEmail.ts new file mode 100644 index 000000000..2437d1d02 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesAcmeEmail.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformSourcesAcmeEmail = + (typeof PreviewAdminConfigurationMigration200PlatformSourcesAcmeEmail)[keyof typeof PreviewAdminConfigurationMigration200PlatformSourcesAcmeEmail]; + +export const PreviewAdminConfigurationMigration200PlatformSourcesAcmeEmail = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays.ts new file mode 100644 index 000000000..92b28944a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays = + (typeof PreviewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays)[keyof typeof PreviewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays]; + +export const PreviewAdminConfigurationMigration200PlatformSourcesBackupRetentionDays = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesDefaultRegion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesDefaultRegion.ts new file mode 100644 index 000000000..0cc8fe9e8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesDefaultRegion.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformSourcesDefaultRegion = + (typeof PreviewAdminConfigurationMigration200PlatformSourcesDefaultRegion)[keyof typeof PreviewAdminConfigurationMigration200PlatformSourcesDefaultRegion]; + +export const PreviewAdminConfigurationMigration200PlatformSourcesDefaultRegion = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesDomain.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesDomain.ts new file mode 100644 index 000000000..dfecd8f0d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesDomain.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformSourcesDomain = + (typeof PreviewAdminConfigurationMigration200PlatformSourcesDomain)[keyof typeof PreviewAdminConfigurationMigration200PlatformSourcesDomain]; + +export const PreviewAdminConfigurationMigration200PlatformSourcesDomain = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesReleaseSelector.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesReleaseSelector.ts new file mode 100644 index 000000000..c3f4aae6d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformSourcesReleaseSelector.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformSourcesReleaseSelector = + (typeof PreviewAdminConfigurationMigration200PlatformSourcesReleaseSelector)[keyof typeof PreviewAdminConfigurationMigration200PlatformSourcesReleaseSelector]; + +export const PreviewAdminConfigurationMigration200PlatformSourcesReleaseSelector = { + host: "host", + database: "database", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformValues.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformValues.ts new file mode 100644 index 000000000..c27ea2aca --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformValues.ts @@ -0,0 +1,26 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformValues = { + /** + * @minLength 1 + * @maxLength 253 + */ + domain: string; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + acmeEmail: string; + /** + * @minLength 1 + * @maxLength 128 + */ + releaseSelector: string; + defaultRegion: string | null; + /** + * @minimum 1 + * @maximum 36500 + */ + backupRetentionDays: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformVersion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformVersion.ts new file mode 100644 index 000000000..5059b3dc7 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PlatformVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PlatformVersion = + (typeof PreviewAdminConfigurationMigration200PlatformVersion)[keyof typeof PreviewAdminConfigurationMigration200PlatformVersion]; + +export const PreviewAdminConfigurationMigration200PlatformVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItem.ts new file mode 100644 index 000000000..b7df1ed8e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItem.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigration200PrerequisitesItemReason } from "./previewAdminConfigurationMigration200PrerequisitesItemReason"; +import type { PreviewAdminConfigurationMigration200PrerequisitesItemScope } from "./previewAdminConfigurationMigration200PrerequisitesItemScope"; +import type { PreviewAdminConfigurationMigration200PrerequisitesItemSource } from "./previewAdminConfigurationMigration200PrerequisitesItemSource"; + +export type PreviewAdminConfigurationMigration200PrerequisitesItem = { + scope: PreviewAdminConfigurationMigration200PrerequisitesItemScope; + /** + * @minLength 1 + * @maxLength 128 + */ + id: string; + /** @maxLength 128 */ + key?: string; + source: PreviewAdminConfigurationMigration200PrerequisitesItemSource; + reason: PreviewAdminConfigurationMigration200PrerequisitesItemReason; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemReason.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemReason.ts new file mode 100644 index 000000000..fad8963b5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PrerequisitesItemReason = + (typeof PreviewAdminConfigurationMigration200PrerequisitesItemReason)[keyof typeof PreviewAdminConfigurationMigration200PrerequisitesItemReason]; + +export const PreviewAdminConfigurationMigration200PrerequisitesItemReason = { + host_owned: "host_owned", + credential_required: "credential_required", + installation_required: "installation_required", + unsupported: "unsupported", + protected_bootstrap: "protected_bootstrap", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemScope.ts new file mode 100644 index 000000000..d50e65961 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemScope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PrerequisitesItemScope = + (typeof PreviewAdminConfigurationMigration200PrerequisitesItemScope)[keyof typeof PreviewAdminConfigurationMigration200PrerequisitesItemScope]; + +export const PreviewAdminConfigurationMigration200PrerequisitesItemScope = { + integration: "integration", + service: "service", + platform: "platform", + account: "account", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemSource.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemSource.ts new file mode 100644 index 000000000..99a510094 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration200PrerequisitesItemSource.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration200PrerequisitesItemSource = + (typeof PreviewAdminConfigurationMigration200PrerequisitesItemSource)[keyof typeof PreviewAdminConfigurationMigration200PrerequisitesItemSource]; + +export const PreviewAdminConfigurationMigration200PrerequisitesItemSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", + missing: "missing", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration400.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration400.ts new file mode 100644 index 000000000..605f2dd28 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration401.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration401.ts new file mode 100644 index 000000000..5cb937082 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration403.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration403.ts new file mode 100644 index 000000000..66b9aa8e4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration409.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration409.ts new file mode 100644 index 000000000..78bed6d00 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration413.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration413.ts new file mode 100644 index 000000000..a3c0fdc03 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration415.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration415.ts new file mode 100644 index 000000000..c567a2a89 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration429.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration429.ts new file mode 100644 index 000000000..fb0cbd423 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration500.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration500.ts new file mode 100644 index 000000000..6c38a127d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration503.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration503.ts new file mode 100644 index 000000000..b264207d3 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigration503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigration503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBody.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBody.ts new file mode 100644 index 000000000..3869d542c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBody.ts @@ -0,0 +1,30 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigrationBodyBindingsItem } from "./previewAdminConfigurationMigrationBodyBindingsItem"; +import type { PreviewAdminConfigurationMigrationBodyFormat } from "./previewAdminConfigurationMigrationBodyFormat"; +import type { PreviewAdminConfigurationMigrationBodyInstallationsItem } from "./previewAdminConfigurationMigrationBodyInstallationsItem"; +import type { PreviewAdminConfigurationMigrationBodyIntegrationsItem } from "./previewAdminConfigurationMigrationBodyIntegrationsItem"; +import type { PreviewAdminConfigurationMigrationBodyPlatform } from "./previewAdminConfigurationMigrationBodyPlatform"; +import type { PreviewAdminConfigurationMigrationBodyPrerequisitesItem } from "./previewAdminConfigurationMigrationBodyPrerequisitesItem"; +import type { PreviewAdminConfigurationMigrationBodyServiceConfigsItem } from "./previewAdminConfigurationMigrationBodyServiceConfigsItem"; +import type { PreviewAdminConfigurationMigrationBodySettings } from "./previewAdminConfigurationMigrationBodySettings"; +import type { PreviewAdminConfigurationMigrationBodyVersion } from "./previewAdminConfigurationMigrationBodyVersion"; + +export type PreviewAdminConfigurationMigrationBody = { + format: PreviewAdminConfigurationMigrationBodyFormat; + version: PreviewAdminConfigurationMigrationBodyVersion; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + exportedAt: string; + settings: PreviewAdminConfigurationMigrationBodySettings; + platform?: PreviewAdminConfigurationMigrationBodyPlatform; + /** @maxItems 256 */ + integrations: PreviewAdminConfigurationMigrationBodyIntegrationsItem[]; + /** @maxItems 256 */ + serviceConfigs: PreviewAdminConfigurationMigrationBodyServiceConfigsItem[]; + /** @maxItems 512 */ + bindings: PreviewAdminConfigurationMigrationBodyBindingsItem[]; + /** @maxItems 2048 */ + installations: PreviewAdminConfigurationMigrationBodyInstallationsItem[]; + /** @maxItems 1000 */ + prerequisites: PreviewAdminConfigurationMigrationBodyPrerequisitesItem[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyBindingsItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyBindingsItem.ts new file mode 100644 index 000000000..ad0879524 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyBindingsItem.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyBindingsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + /** + * @minLength 1 + * @maxLength 128 + */ + capability: string; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + serviceId: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyFormat.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyFormat.ts new file mode 100644 index 000000000..854c131b5 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyFormat.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyFormat = + (typeof PreviewAdminConfigurationMigrationBodyFormat)[keyof typeof PreviewAdminConfigurationMigrationBodyFormat]; + +export const PreviewAdminConfigurationMigrationBodyFormat = { + "openmapx-configuration": "openmapx-configuration", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItem.ts new file mode 100644 index 000000000..21cde9acd --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItem.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigrationBodyInstallationsItemOwnership } from "./previewAdminConfigurationMigrationBodyInstallationsItemOwnership"; +import type { PreviewAdminConfigurationMigrationBodyInstallationsItemProvenance } from "./previewAdminConfigurationMigrationBodyInstallationsItemProvenance"; + +export type PreviewAdminConfigurationMigrationBodyInstallationsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + version: string | null; + contentDigest: string | null; + denialReason: string | null; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + pendingTransactionId?: string; + ownership: PreviewAdminConfigurationMigrationBodyInstallationsItemOwnership; + extensionId: string | null; + provenance: PreviewAdminConfigurationMigrationBodyInstallationsItemProvenance; + adoptable: boolean; + removable: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItemOwnership.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItemOwnership.ts new file mode 100644 index 000000000..59057753c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItemOwnership.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyInstallationsItemOwnership = + (typeof PreviewAdminConfigurationMigrationBodyInstallationsItemOwnership)[keyof typeof PreviewAdminConfigurationMigrationBodyInstallationsItemOwnership]; + +export const PreviewAdminConfigurationMigrationBodyInstallationsItemOwnership = { + extension: "extension", + standalone: "standalone", + unmanaged: "unmanaged", + conflict: "conflict", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItemProvenance.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItemProvenance.ts new file mode 100644 index 000000000..7e19891c9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyInstallationsItemProvenance.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyInstallationsItemProvenance = + (typeof PreviewAdminConfigurationMigrationBodyInstallationsItemProvenance)[keyof typeof PreviewAdminConfigurationMigrationBodyInstallationsItemProvenance]; + +export const PreviewAdminConfigurationMigrationBodyInstallationsItemProvenance = { + artifact: "artifact", + source: "source", + "uploaded-artifact": "uploaded-artifact", + legacy_unknown: "legacy_unknown", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyIntegrationsItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyIntegrationsItem.ts new file mode 100644 index 000000000..76bfe973f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyIntegrationsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigrationBodyIntegrationsItemConfig } from "./previewAdminConfigurationMigrationBodyIntegrationsItemConfig"; + +export type PreviewAdminConfigurationMigrationBodyIntegrationsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + integrationId: string; + config: PreviewAdminConfigurationMigrationBodyIntegrationsItemConfig; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyIntegrationsItemConfig.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyIntegrationsItemConfig.ts new file mode 100644 index 000000000..af7d4cbbe --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyIntegrationsItemConfig.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminPreviewAdminConfigurationMigrationBodySchema0 } from "./adminPreviewAdminConfigurationMigrationBodySchema0"; + +export type PreviewAdminConfigurationMigrationBodyIntegrationsItemConfig = { + [key: string]: AdminPreviewAdminConfigurationMigrationBodySchema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatform.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatform.ts new file mode 100644 index 000000000..f0d0a9f58 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatform.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigrationBodyPlatformAcmeEmail } from "./previewAdminConfigurationMigrationBodyPlatformAcmeEmail"; +import type { PreviewAdminConfigurationMigrationBodyPlatformBackupRetentionDays } from "./previewAdminConfigurationMigrationBodyPlatformBackupRetentionDays"; +import type { PreviewAdminConfigurationMigrationBodyPlatformDefaultRegion } from "./previewAdminConfigurationMigrationBodyPlatformDefaultRegion"; +import type { PreviewAdminConfigurationMigrationBodyPlatformDomain } from "./previewAdminConfigurationMigrationBodyPlatformDomain"; +import type { PreviewAdminConfigurationMigrationBodyPlatformReleaseSelector } from "./previewAdminConfigurationMigrationBodyPlatformReleaseSelector"; +import type { PreviewAdminConfigurationMigrationBodyPlatformVersion } from "./previewAdminConfigurationMigrationBodyPlatformVersion"; + +export type PreviewAdminConfigurationMigrationBodyPlatform = { + version: PreviewAdminConfigurationMigrationBodyPlatformVersion; + domain?: PreviewAdminConfigurationMigrationBodyPlatformDomain; + acmeEmail?: PreviewAdminConfigurationMigrationBodyPlatformAcmeEmail; + releaseSelector?: PreviewAdminConfigurationMigrationBodyPlatformReleaseSelector; + defaultRegion?: PreviewAdminConfigurationMigrationBodyPlatformDefaultRegion; + backupRetentionDays?: PreviewAdminConfigurationMigrationBodyPlatformBackupRetentionDays; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformAcmeEmail.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformAcmeEmail.ts new file mode 100644 index 000000000..70dbd63b9 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformAcmeEmail.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPlatformAcmeEmail = + | { + source: "host"; + } + | { + source: "database"; + /** + * @maxLength 320 + * @pattern ^(?:[A-Za-z0-9_'+\-]+\.)*[A-Za-z0-9_'+\-]*[A-Za-z0-9_+-]@(?:[A-Za-z0-9][A-Za-z0-9\-]*\.)+[A-Za-z]{2,}$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformBackupRetentionDays.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformBackupRetentionDays.ts new file mode 100644 index 000000000..a9eb1f34d --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformBackupRetentionDays.ts @@ -0,0 +1,15 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPlatformBackupRetentionDays = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minimum 1 + * @maximum 36500 + */ + value: number; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformDefaultRegion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformDefaultRegion.ts new file mode 100644 index 000000000..1c2561abd --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformDefaultRegion.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPlatformDefaultRegion = + | { + source: "host"; + } + | { + source: "database"; + /** + * @minLength 1 + * @maxLength 128 + * @pattern ^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$ + */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformDomain.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformDomain.ts new file mode 100644 index 000000000..4183efb09 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformDomain.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPlatformDomain = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 253 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformReleaseSelector.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformReleaseSelector.ts new file mode 100644 index 000000000..5fdb408ce --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformReleaseSelector.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPlatformReleaseSelector = + | { + source: "host"; + } + | { + source: "database"; + /** @maxLength 128 */ + value: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformVersion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformVersion.ts new file mode 100644 index 000000000..f071b0f34 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPlatformVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPlatformVersion = + (typeof PreviewAdminConfigurationMigrationBodyPlatformVersion)[keyof typeof PreviewAdminConfigurationMigrationBodyPlatformVersion]; + +export const PreviewAdminConfigurationMigrationBodyPlatformVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItem.ts new file mode 100644 index 000000000..faeb7fb87 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItem.ts @@ -0,0 +1,18 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigrationBodyPrerequisitesItemReason } from "./previewAdminConfigurationMigrationBodyPrerequisitesItemReason"; +import type { PreviewAdminConfigurationMigrationBodyPrerequisitesItemScope } from "./previewAdminConfigurationMigrationBodyPrerequisitesItemScope"; +import type { PreviewAdminConfigurationMigrationBodyPrerequisitesItemSource } from "./previewAdminConfigurationMigrationBodyPrerequisitesItemSource"; + +export type PreviewAdminConfigurationMigrationBodyPrerequisitesItem = { + scope: PreviewAdminConfigurationMigrationBodyPrerequisitesItemScope; + /** + * @minLength 1 + * @maxLength 128 + */ + id: string; + /** @maxLength 128 */ + key?: string; + source: PreviewAdminConfigurationMigrationBodyPrerequisitesItemSource; + reason: PreviewAdminConfigurationMigrationBodyPrerequisitesItemReason; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemReason.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemReason.ts new file mode 100644 index 000000000..7ab783d61 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemReason.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPrerequisitesItemReason = + (typeof PreviewAdminConfigurationMigrationBodyPrerequisitesItemReason)[keyof typeof PreviewAdminConfigurationMigrationBodyPrerequisitesItemReason]; + +export const PreviewAdminConfigurationMigrationBodyPrerequisitesItemReason = { + host_owned: "host_owned", + credential_required: "credential_required", + installation_required: "installation_required", + unsupported: "unsupported", + protected_bootstrap: "protected_bootstrap", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemScope.ts new file mode 100644 index 000000000..32dddf782 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemScope.ts @@ -0,0 +1,12 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPrerequisitesItemScope = + (typeof PreviewAdminConfigurationMigrationBodyPrerequisitesItemScope)[keyof typeof PreviewAdminConfigurationMigrationBodyPrerequisitesItemScope]; + +export const PreviewAdminConfigurationMigrationBodyPrerequisitesItemScope = { + integration: "integration", + service: "service", + platform: "platform", + account: "account", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemSource.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemSource.ts new file mode 100644 index 000000000..63b3c8143 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyPrerequisitesItemSource.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyPrerequisitesItemSource = + (typeof PreviewAdminConfigurationMigrationBodyPrerequisitesItemSource)[keyof typeof PreviewAdminConfigurationMigrationBodyPrerequisitesItemSource]; + +export const PreviewAdminConfigurationMigrationBodyPrerequisitesItemSource = { + default: "default", + database: "database", + vault: "vault", + configjson: "config.json", + env: "env", + missing: "missing", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyServiceConfigsItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyServiceConfigsItem.ts new file mode 100644 index 000000000..bbff0b05c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyServiceConfigsItem.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminConfigurationMigrationBodyServiceConfigsItemConfig } from "./previewAdminConfigurationMigrationBodyServiceConfigsItemConfig"; + +export type PreviewAdminConfigurationMigrationBodyServiceConfigsItem = { + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + serviceId: string; + config: PreviewAdminConfigurationMigrationBodyServiceConfigsItemConfig; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyServiceConfigsItemConfig.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyServiceConfigsItemConfig.ts new file mode 100644 index 000000000..e10bb985a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyServiceConfigsItemConfig.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminPreviewAdminConfigurationMigrationBodySchema0 } from "./adminPreviewAdminConfigurationMigrationBodySchema0"; + +export type PreviewAdminConfigurationMigrationBodyServiceConfigsItemConfig = { + [key: string]: AdminPreviewAdminConfigurationMigrationBodySchema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodySettings.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodySettings.ts new file mode 100644 index 000000000..4cf8198cb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodySettings.ts @@ -0,0 +1,7 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { AdminPreviewAdminConfigurationMigrationBodySchema0 } from "./adminPreviewAdminConfigurationMigrationBodySchema0"; + +export type PreviewAdminConfigurationMigrationBodySettings = { + [key: string]: AdminPreviewAdminConfigurationMigrationBodySchema0 | null; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyVersion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyVersion.ts new file mode 100644 index 000000000..e20ffac7e --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationBodyVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationBodyVersion = + (typeof PreviewAdminConfigurationMigrationBodyVersion)[keyof typeof PreviewAdminConfigurationMigrationBodyVersion]; + +export const PreviewAdminConfigurationMigrationBodyVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationHeaders.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationHeaders.ts new file mode 100644 index 000000000..a2d866df2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminConfigurationMigrationHeaders.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminConfigurationMigrationHeaders = { + /** + * @minLength 1 + * @maxLength 128 + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200.ts new file mode 100644 index 000000000..006d6beb2 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200.ts @@ -0,0 +1,22 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransfer200EntriesItem } from "./previewAdminCredentialTransfer200EntriesItem"; +import type { PreviewAdminCredentialTransfer200FeedMap } from "./previewAdminCredentialTransfer200FeedMap"; + +export type PreviewAdminCredentialTransfer200 = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + /** + * @minLength 1 + * @maxLength 128 + */ + intentFingerprint: string; + /** @nullable */ + expectedCurrentRevisionId: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + /** @maxItems 1000 */ + entries: PreviewAdminCredentialTransfer200EntriesItem[]; + feedMap?: PreviewAdminCredentialTransfer200FeedMap; + applyable: boolean; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItem.ts new file mode 100644 index 000000000..212ee66aa --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItem.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransfer200EntriesItemScope } from "./previewAdminCredentialTransfer200EntriesItemScope"; +import type { PreviewAdminCredentialTransfer200EntriesItemStatus } from "./previewAdminCredentialTransfer200EntriesItemStatus"; + +export type PreviewAdminCredentialTransfer200EntriesItem = { + scope: PreviewAdminCredentialTransfer200EntriesItemScope; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + id: string; + /** + * @minLength 1 + * @maxLength 128 + */ + key: string; + status: PreviewAdminCredentialTransfer200EntriesItemStatus; + reason?: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItemScope.ts new file mode 100644 index 000000000..2aa90ff0c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItemScope.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer200EntriesItemScope = + (typeof PreviewAdminCredentialTransfer200EntriesItemScope)[keyof typeof PreviewAdminCredentialTransfer200EntriesItemScope]; + +export const PreviewAdminCredentialTransfer200EntriesItemScope = { + integration: "integration", + service: "service", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItemStatus.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItemStatus.ts new file mode 100644 index 000000000..be67b4026 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200EntriesItemStatus.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer200EntriesItemStatus = + (typeof PreviewAdminCredentialTransfer200EntriesItemStatus)[keyof typeof PreviewAdminCredentialTransfer200EntriesItemStatus]; + +export const PreviewAdminCredentialTransfer200EntriesItemStatus = { + selected: "selected", + skipped: "skipped", + denied: "denied", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMap.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMap.ts new file mode 100644 index 000000000..cfabb5480 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMap.ts @@ -0,0 +1,27 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransfer200FeedMapChangesItem } from "./previewAdminCredentialTransfer200FeedMapChangesItem"; + +export type PreviewAdminCredentialTransfer200FeedMap = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + previewId: string; + intentFingerprint: string; + catalogRevisionId: string; + credentialsRevision: string; + /** @nullable */ + expectedCurrentRevisionId: string | null; + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z))$ */ + expiresAt: string; + /** @maxItems 1000 */ + changes: PreviewAdminCredentialTransfer200FeedMapChangesItem[]; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + preservedCount: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + mapBytes: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMapChangesItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMapChangesItem.ts new file mode 100644 index 000000000..7162d843b --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMapChangesItem.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransfer200FeedMapChangesItemAction } from "./previewAdminCredentialTransfer200FeedMapChangesItemAction"; + +export type PreviewAdminCredentialTransfer200FeedMapChangesItem = { + key: string; + action: PreviewAdminCredentialTransfer200FeedMapChangesItemAction; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMapChangesItemAction.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMapChangesItemAction.ts new file mode 100644 index 000000000..ed2ddb4ca --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer200FeedMapChangesItemAction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer200FeedMapChangesItemAction = + (typeof PreviewAdminCredentialTransfer200FeedMapChangesItemAction)[keyof typeof PreviewAdminCredentialTransfer200FeedMapChangesItemAction]; + +export const PreviewAdminCredentialTransfer200FeedMapChangesItemAction = { + set: "set", + delete: "delete", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer400.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer400.ts new file mode 100644 index 000000000..c56740b63 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer401.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer401.ts new file mode 100644 index 000000000..7707c8ed8 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer403.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer403.ts new file mode 100644 index 000000000..8c83375fe --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer409.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer409.ts new file mode 100644 index 000000000..f44f2dde0 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer413.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer413.ts new file mode 100644 index 000000000..547354e83 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer415.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer415.ts new file mode 100644 index 000000000..64037a4fa --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer429.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer429.ts new file mode 100644 index 000000000..c3cd7a652 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer429.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer429 = { + error: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + retryAfter: number; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer500.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer500.ts new file mode 100644 index 000000000..92b1661e7 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer503.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer503.ts new file mode 100644 index 000000000..0fcd71b04 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransfer503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransfer503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBody.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBody.ts new file mode 100644 index 000000000..0b31a0924 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBody.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransferBodyChoices } from "./previewAdminCredentialTransferBodyChoices"; +import type { PreviewAdminCredentialTransferBodyEnvelope } from "./previewAdminCredentialTransferBodyEnvelope"; + +export type PreviewAdminCredentialTransferBody = { + envelope: PreviewAdminCredentialTransferBodyEnvelope; + passphrase: string; + choices?: PreviewAdminCredentialTransferBodyChoices; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoices.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoices.ts new file mode 100644 index 000000000..75de04fc7 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoices.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransferBodyChoicesEntriesItem } from "./previewAdminCredentialTransferBodyChoicesEntriesItem"; +import type { PreviewAdminCredentialTransferBodyChoicesFeedMap } from "./previewAdminCredentialTransferBodyChoicesFeedMap"; + +export type PreviewAdminCredentialTransferBodyChoices = { + /** @maxItems 1000 */ + entries: PreviewAdminCredentialTransferBodyChoicesEntriesItem[]; + feedMap?: PreviewAdminCredentialTransferBodyChoicesFeedMap; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItem.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItem.ts new file mode 100644 index 000000000..bb254ba71 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItem.ts @@ -0,0 +1,16 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransferBodyChoicesEntriesItemAction } from "./previewAdminCredentialTransferBodyChoicesEntriesItemAction"; +import type { PreviewAdminCredentialTransferBodyChoicesEntriesItemScope } from "./previewAdminCredentialTransferBodyChoicesEntriesItemScope"; + +export type PreviewAdminCredentialTransferBodyChoicesEntriesItem = { + scope: PreviewAdminCredentialTransferBodyChoicesEntriesItemScope; + /** @pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ + id: string; + /** + * @minLength 1 + * @maxLength 128 + */ + key: string; + action: PreviewAdminCredentialTransferBodyChoicesEntriesItemAction; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItemAction.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItemAction.ts new file mode 100644 index 000000000..910c0c189 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItemAction.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyChoicesEntriesItemAction = + (typeof PreviewAdminCredentialTransferBodyChoicesEntriesItemAction)[keyof typeof PreviewAdminCredentialTransferBodyChoicesEntriesItemAction]; + +export const PreviewAdminCredentialTransferBodyChoicesEntriesItemAction = { + import: "import", + skip: "skip", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItemScope.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItemScope.ts new file mode 100644 index 000000000..19d7290cb --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesEntriesItemScope.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyChoicesEntriesItemScope = + (typeof PreviewAdminCredentialTransferBodyChoicesEntriesItemScope)[keyof typeof PreviewAdminCredentialTransferBodyChoicesEntriesItemScope]; + +export const PreviewAdminCredentialTransferBodyChoicesEntriesItemScope = { + integration: "integration", + service: "service", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesFeedMap.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesFeedMap.ts new file mode 100644 index 000000000..852276124 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyChoicesFeedMap.ts @@ -0,0 +1,27 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyChoicesFeedMap = { + /** + * @minLength 1 + * @maxLength 256 + */ + catalogRevisionId: string; + /** + * @minLength 1 + * @maxLength 256 + */ + expectedCredentialsRevision: string; + /** + * @maxItems 1000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + setKeys: string[]; + /** + * @maxItems 1000 + * @items.minLength 1 + * @items.maxLength 1024 + */ + deleteKeys: string[]; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelope.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelope.ts new file mode 100644 index 000000000..6365df91f --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelope.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransferBodyEnvelopeCipher } from "./previewAdminCredentialTransferBodyEnvelopeCipher"; +import type { PreviewAdminCredentialTransferBodyEnvelopeFormat } from "./previewAdminCredentialTransferBodyEnvelopeFormat"; +import type { PreviewAdminCredentialTransferBodyEnvelopeKdf } from "./previewAdminCredentialTransferBodyEnvelopeKdf"; +import type { PreviewAdminCredentialTransferBodyEnvelopeVersion } from "./previewAdminCredentialTransferBodyEnvelopeVersion"; + +export type PreviewAdminCredentialTransferBodyEnvelope = { + format: PreviewAdminCredentialTransferBodyEnvelopeFormat; + version: PreviewAdminCredentialTransferBodyEnvelopeVersion; + kdf: PreviewAdminCredentialTransferBodyEnvelopeKdf; + cipher: PreviewAdminCredentialTransferBodyEnvelopeCipher; + /** + * @minLength 1 + * @maxLength 1048576 + * @pattern ^[A-Za-z0-9_-]+$ + */ + ciphertext: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeCipher.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeCipher.ts new file mode 100644 index 000000000..99c27c009 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeCipher.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransferBodyEnvelopeCipherName } from "./previewAdminCredentialTransferBodyEnvelopeCipherName"; + +export type PreviewAdminCredentialTransferBodyEnvelopeCipher = { + name: PreviewAdminCredentialTransferBodyEnvelopeCipherName; + /** + * @minLength 16 + * @maxLength 16 + * @pattern ^[A-Za-z0-9_-]+$ + */ + iv: string; + /** + * @minLength 22 + * @maxLength 22 + * @pattern ^[A-Za-z0-9_-]+$ + */ + tag: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeCipherName.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeCipherName.ts new file mode 100644 index 000000000..44b9e0d91 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeCipherName.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyEnvelopeCipherName = + (typeof PreviewAdminCredentialTransferBodyEnvelopeCipherName)[keyof typeof PreviewAdminCredentialTransferBodyEnvelopeCipherName]; + +export const PreviewAdminCredentialTransferBodyEnvelopeCipherName = { + "aes-256-gcm": "aes-256-gcm", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeFormat.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeFormat.ts new file mode 100644 index 000000000..e0527dfd4 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeFormat.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyEnvelopeFormat = + (typeof PreviewAdminCredentialTransferBodyEnvelopeFormat)[keyof typeof PreviewAdminCredentialTransferBodyEnvelopeFormat]; + +export const PreviewAdminCredentialTransferBodyEnvelopeFormat = { + "openmapx-vault-transfer": "openmapx-vault-transfer", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdf.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdf.ts new file mode 100644 index 000000000..b691dc94a --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdf.ts @@ -0,0 +1,19 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { PreviewAdminCredentialTransferBodyEnvelopeKdfN } from "./previewAdminCredentialTransferBodyEnvelopeKdfN"; +import type { PreviewAdminCredentialTransferBodyEnvelopeKdfName } from "./previewAdminCredentialTransferBodyEnvelopeKdfName"; +import type { PreviewAdminCredentialTransferBodyEnvelopeKdfP } from "./previewAdminCredentialTransferBodyEnvelopeKdfP"; +import type { PreviewAdminCredentialTransferBodyEnvelopeKdfR } from "./previewAdminCredentialTransferBodyEnvelopeKdfR"; + +export type PreviewAdminCredentialTransferBodyEnvelopeKdf = { + name: PreviewAdminCredentialTransferBodyEnvelopeKdfName; + N: PreviewAdminCredentialTransferBodyEnvelopeKdfN; + r: PreviewAdminCredentialTransferBodyEnvelopeKdfR; + p: PreviewAdminCredentialTransferBodyEnvelopeKdfP; + /** + * @minLength 22 + * @maxLength 22 + * @pattern ^[A-Za-z0-9_-]+$ + */ + salt: string; +}; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfN.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfN.ts new file mode 100644 index 000000000..1f01f4d52 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfN.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyEnvelopeKdfN = + (typeof PreviewAdminCredentialTransferBodyEnvelopeKdfN)[keyof typeof PreviewAdminCredentialTransferBodyEnvelopeKdfN]; + +export const PreviewAdminCredentialTransferBodyEnvelopeKdfN = { + NUMBER_32768: 32768, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfName.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfName.ts new file mode 100644 index 000000000..d15a6d34c --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfName.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyEnvelopeKdfName = + (typeof PreviewAdminCredentialTransferBodyEnvelopeKdfName)[keyof typeof PreviewAdminCredentialTransferBodyEnvelopeKdfName]; + +export const PreviewAdminCredentialTransferBodyEnvelopeKdfName = { + scrypt: "scrypt", +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfP.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfP.ts new file mode 100644 index 000000000..e3d964e06 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfP.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyEnvelopeKdfP = + (typeof PreviewAdminCredentialTransferBodyEnvelopeKdfP)[keyof typeof PreviewAdminCredentialTransferBodyEnvelopeKdfP]; + +export const PreviewAdminCredentialTransferBodyEnvelopeKdfP = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfR.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfR.ts new file mode 100644 index 000000000..e0a002049 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeKdfR.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyEnvelopeKdfR = + (typeof PreviewAdminCredentialTransferBodyEnvelopeKdfR)[keyof typeof PreviewAdminCredentialTransferBodyEnvelopeKdfR]; + +export const PreviewAdminCredentialTransferBodyEnvelopeKdfR = { + NUMBER_8: 8, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeVersion.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeVersion.ts new file mode 100644 index 000000000..5739dc5ee --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferBodyEnvelopeVersion.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferBodyEnvelopeVersion = + (typeof PreviewAdminCredentialTransferBodyEnvelopeVersion)[keyof typeof PreviewAdminCredentialTransferBodyEnvelopeVersion]; + +export const PreviewAdminCredentialTransferBodyEnvelopeVersion = { + NUMBER_1: 1, +} as const; diff --git a/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferHeaders.ts b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferHeaders.ts new file mode 100644 index 000000000..5134aeb28 --- /dev/null +++ b/packages/core/src/api/generated/admin/integration-models/previewAdminCredentialTransferHeaders.ts @@ -0,0 +1,14 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type PreviewAdminCredentialTransferHeaders = { + /** + * @minLength 1 + * @maxLength 128 + */ + "idempotency-key": string; + /** + * @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + */ + "x-admin-reauth-proof"?: string; +}; diff --git a/packages/core/src/api/generated/admin/operation-bindings.generated.ts b/packages/core/src/api/generated/admin/operation-bindings.generated.ts index fea0086de..1567a16a3 100644 --- a/packages/core/src/api/generated/admin/operation-bindings.generated.ts +++ b/packages/core/src/api/generated/admin/operation-bindings.generated.ts @@ -46,6 +46,22 @@ export async function applyAdminAuthPolicyRequest( ): Promise { return applyAdminAuthPolicyBound(url, options); } +const applyAdminConfigurationMigrationBound = bindAdminOperation( + "applyAdminConfigurationMigration", +); +export async function applyAdminConfigurationMigrationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return applyAdminConfigurationMigrationBound(url, options); +} +const applyAdminCredentialTransferBound = bindAdminOperation("applyAdminCredentialTransfer"); +export async function applyAdminCredentialTransferRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return applyAdminCredentialTransferBound(url, options); +} const applyAdminPlatformConfigurationBound = bindAdminOperation("applyAdminPlatformConfiguration"); export async function applyAdminPlatformConfigurationRequest( url: string, @@ -95,6 +111,15 @@ export async function checkAdminServicesRequest( ): Promise { return checkAdminServicesBound(url, options); } +const chooseAdminCredentialTransferFieldsBound = bindAdminOperation( + "chooseAdminCredentialTransferFields", +); +export async function chooseAdminCredentialTransferFieldsRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return chooseAdminCredentialTransferFieldsBound(url, options); +} const clearAdminCacheBound = bindAdminOperation("clearAdminCache"); export async function clearAdminCacheRequest( url: string, @@ -275,6 +300,29 @@ export async function enableAdminIntegrationRequest( ): Promise { return enableAdminIntegrationBound(url, options); } +const exportAdminArtifactTransferBound = bindAdminOperation("exportAdminArtifactTransfer"); +export async function exportAdminArtifactTransferRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return exportAdminArtifactTransferBound(url, options); +} +const exportAdminConfigurationMigrationBound = bindAdminOperation( + "exportAdminConfigurationMigration", +); +export async function exportAdminConfigurationMigrationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return exportAdminConfigurationMigrationBound(url, options); +} +const exportAdminCredentialTransferBound = bindAdminOperation("exportAdminCredentialTransfer"); +export async function exportAdminCredentialTransferRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return exportAdminCredentialTransferBound(url, options); +} const exportAdminIntegrationConfigsBound = bindAdminOperation("exportAdminIntegrationConfigs"); export async function exportAdminIntegrationConfigsRequest( url: string, @@ -326,6 +374,15 @@ export async function getAdminArtifactTransferRequest( ): Promise { return getAdminArtifactTransferBound(url, options); } +const getAdminArtifactTransferCapabilitiesBound = bindAdminOperation( + "getAdminArtifactTransferCapabilities", +); +export async function getAdminArtifactTransferCapabilitiesRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminArtifactTransferCapabilitiesBound(url, options); +} const getAdminAuthPolicyBound = bindAdminOperation("getAdminAuthPolicy"); export async function getAdminAuthPolicyRequest( url: string, @@ -340,6 +397,15 @@ export async function getAdminComposePreviewRequest( ): Promise { return getAdminComposePreviewBound(url, options); } +const getAdminConfigurationMigrationInventoryBound = bindAdminOperation( + "getAdminConfigurationMigrationInventory", +); +export async function getAdminConfigurationMigrationInventoryRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return getAdminConfigurationMigrationInventoryBound(url, options); +} const getAdminDawarichBound = bindAdminOperation("getAdminDawarich"); export async function getAdminDawarichRequest( url: string, @@ -911,6 +977,22 @@ export async function previewAdminBackupPruneRequest( ): Promise { return previewAdminBackupPruneBound(url, options); } +const previewAdminConfigurationMigrationBound = bindAdminOperation( + "previewAdminConfigurationMigration", +); +export async function previewAdminConfigurationMigrationRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return previewAdminConfigurationMigrationBound(url, options); +} +const previewAdminCredentialTransferBound = bindAdminOperation("previewAdminCredentialTransfer"); +export async function previewAdminCredentialTransferRequest( + url: string, + options?: AdminSdkRequestOptions, +): Promise { + return previewAdminCredentialTransferBound(url, options); +} const previewAdminExtensionInstallBound = bindAdminOperation("previewAdminExtensionInstall"); export async function previewAdminExtensionInstallRequest( url: string, diff --git a/packages/core/src/api/generated/admin/service-admin.generated.ts b/packages/core/src/api/generated/admin/service-admin.generated.ts index e386ee1af..ba425afdf 100644 --- a/packages/core/src/api/generated/admin/service-admin.generated.ts +++ b/packages/core/src/api/generated/admin/service-admin.generated.ts @@ -20,6 +20,8 @@ import { deleteAdminServiceBackupRequest, deleteAdminServiceCredentialRequest, downloadAdminArtifactTransferContentRequest, + exportAdminArtifactTransferRequest, + getAdminArtifactTransferCapabilitiesRequest, getAdminArtifactTransferRequest, getAdminComposePreviewRequest, getAdminDawarichRequest, @@ -178,6 +180,22 @@ import type { DownloadAdminArtifactTransferContent500, DownloadAdminArtifactTransferContent503, DownloadAdminArtifactTransferContent507, + ExportAdminArtifactTransfer202, + ExportAdminArtifactTransfer400, + ExportAdminArtifactTransfer401, + ExportAdminArtifactTransfer403, + ExportAdminArtifactTransfer404, + ExportAdminArtifactTransfer409, + ExportAdminArtifactTransfer410, + ExportAdminArtifactTransfer413, + ExportAdminArtifactTransfer415, + ExportAdminArtifactTransfer422, + ExportAdminArtifactTransfer429, + ExportAdminArtifactTransfer500, + ExportAdminArtifactTransfer503, + ExportAdminArtifactTransfer507, + ExportAdminArtifactTransferBody, + ExportAdminArtifactTransferHeaders, GetAdminArtifactTransfer200, GetAdminArtifactTransfer400, GetAdminArtifactTransfer401, @@ -192,6 +210,20 @@ import type { GetAdminArtifactTransfer500, GetAdminArtifactTransfer503, GetAdminArtifactTransfer507, + GetAdminArtifactTransferCapabilities200, + GetAdminArtifactTransferCapabilities400, + GetAdminArtifactTransferCapabilities401, + GetAdminArtifactTransferCapabilities403, + GetAdminArtifactTransferCapabilities404, + GetAdminArtifactTransferCapabilities409, + GetAdminArtifactTransferCapabilities410, + GetAdminArtifactTransferCapabilities413, + GetAdminArtifactTransferCapabilities415, + GetAdminArtifactTransferCapabilities422, + GetAdminArtifactTransferCapabilities429, + GetAdminArtifactTransferCapabilities500, + GetAdminArtifactTransferCapabilities503, + GetAdminArtifactTransferCapabilities507, GetAdminComposePreview400, GetAdminComposePreview401, GetAdminComposePreview403, @@ -865,6 +897,252 @@ export const createAdminArtifactTransfer = async ( ); }; +export type getAdminArtifactTransferCapabilitiesResponse200 = { + data: GetAdminArtifactTransferCapabilities200; + status: 200; +}; + +export type getAdminArtifactTransferCapabilitiesResponse400 = { + data: GetAdminArtifactTransferCapabilities400; + status: 400; +}; + +export type getAdminArtifactTransferCapabilitiesResponse401 = { + data: GetAdminArtifactTransferCapabilities401; + status: 401; +}; + +export type getAdminArtifactTransferCapabilitiesResponse403 = { + data: GetAdminArtifactTransferCapabilities403; + status: 403; +}; + +export type getAdminArtifactTransferCapabilitiesResponse404 = { + data: GetAdminArtifactTransferCapabilities404; + status: 404; +}; + +export type getAdminArtifactTransferCapabilitiesResponse409 = { + data: GetAdminArtifactTransferCapabilities409; + status: 409; +}; + +export type getAdminArtifactTransferCapabilitiesResponse410 = { + data: GetAdminArtifactTransferCapabilities410; + status: 410; +}; + +export type getAdminArtifactTransferCapabilitiesResponse413 = { + data: GetAdminArtifactTransferCapabilities413; + status: 413; +}; + +export type getAdminArtifactTransferCapabilitiesResponse415 = { + data: GetAdminArtifactTransferCapabilities415; + status: 415; +}; + +export type getAdminArtifactTransferCapabilitiesResponse422 = { + data: GetAdminArtifactTransferCapabilities422; + status: 422; +}; + +export type getAdminArtifactTransferCapabilitiesResponse429 = { + data: GetAdminArtifactTransferCapabilities429; + status: 429; +}; + +export type getAdminArtifactTransferCapabilitiesResponse500 = { + data: GetAdminArtifactTransferCapabilities500; + status: 500; +}; + +export type getAdminArtifactTransferCapabilitiesResponse503 = { + data: GetAdminArtifactTransferCapabilities503; + status: 503; +}; + +export type getAdminArtifactTransferCapabilitiesResponse507 = { + data: GetAdminArtifactTransferCapabilities507; + status: 507; +}; + +export type getAdminArtifactTransferCapabilitiesResponseSuccess = + getAdminArtifactTransferCapabilitiesResponse200 & { + headers: Headers; + }; +export type getAdminArtifactTransferCapabilitiesResponseError = ( + | getAdminArtifactTransferCapabilitiesResponse400 + | getAdminArtifactTransferCapabilitiesResponse401 + | getAdminArtifactTransferCapabilitiesResponse403 + | getAdminArtifactTransferCapabilitiesResponse404 + | getAdminArtifactTransferCapabilitiesResponse409 + | getAdminArtifactTransferCapabilitiesResponse410 + | getAdminArtifactTransferCapabilitiesResponse413 + | getAdminArtifactTransferCapabilitiesResponse415 + | getAdminArtifactTransferCapabilitiesResponse422 + | getAdminArtifactTransferCapabilitiesResponse429 + | getAdminArtifactTransferCapabilitiesResponse500 + | getAdminArtifactTransferCapabilitiesResponse503 + | getAdminArtifactTransferCapabilitiesResponse507 +) & { + headers: Headers; +}; + +export type getAdminArtifactTransferCapabilitiesResponse = + | getAdminArtifactTransferCapabilitiesResponseSuccess + | getAdminArtifactTransferCapabilitiesResponseError; + +export const getGetAdminArtifactTransferCapabilitiesUrl = () => { + return `/api/admin/artifact-transfers/capabilities`; +}; + +export const getAdminArtifactTransferCapabilities = async ( + options?: Parameters[1], +): Promise => { + return getAdminArtifactTransferCapabilitiesRequest( + getGetAdminArtifactTransferCapabilitiesUrl(), + { + ...options, + method: "GET", + }, + ); +}; + +export type exportAdminArtifactTransferResponse202 = { + data: ExportAdminArtifactTransfer202; + status: 202; +}; + +export type exportAdminArtifactTransferResponse400 = { + data: ExportAdminArtifactTransfer400; + status: 400; +}; + +export type exportAdminArtifactTransferResponse401 = { + data: ExportAdminArtifactTransfer401; + status: 401; +}; + +export type exportAdminArtifactTransferResponse403 = { + data: ExportAdminArtifactTransfer403; + status: 403; +}; + +export type exportAdminArtifactTransferResponse404 = { + data: ExportAdminArtifactTransfer404; + status: 404; +}; + +export type exportAdminArtifactTransferResponse409 = { + data: ExportAdminArtifactTransfer409; + status: 409; +}; + +export type exportAdminArtifactTransferResponse410 = { + data: ExportAdminArtifactTransfer410; + status: 410; +}; + +export type exportAdminArtifactTransferResponse413 = { + data: ExportAdminArtifactTransfer413; + status: 413; +}; + +export type exportAdminArtifactTransferResponse415 = { + data: ExportAdminArtifactTransfer415; + status: 415; +}; + +export type exportAdminArtifactTransferResponse422 = { + data: ExportAdminArtifactTransfer422; + status: 422; +}; + +export type exportAdminArtifactTransferResponse429 = { + data: ExportAdminArtifactTransfer429; + status: 429; +}; + +export type exportAdminArtifactTransferResponse500 = { + data: ExportAdminArtifactTransfer500; + status: 500; +}; + +export type exportAdminArtifactTransferResponse503 = { + data: ExportAdminArtifactTransfer503; + status: 503; +}; + +export type exportAdminArtifactTransferResponse507 = { + data: ExportAdminArtifactTransfer507; + status: 507; +}; + +export type exportAdminArtifactTransferResponseSuccess = exportAdminArtifactTransferResponse202 & { + headers: Headers; +}; +export type exportAdminArtifactTransferResponseError = ( + | exportAdminArtifactTransferResponse400 + | exportAdminArtifactTransferResponse401 + | exportAdminArtifactTransferResponse403 + | exportAdminArtifactTransferResponse404 + | exportAdminArtifactTransferResponse409 + | exportAdminArtifactTransferResponse410 + | exportAdminArtifactTransferResponse413 + | exportAdminArtifactTransferResponse415 + | exportAdminArtifactTransferResponse422 + | exportAdminArtifactTransferResponse429 + | exportAdminArtifactTransferResponse500 + | exportAdminArtifactTransferResponse503 + | exportAdminArtifactTransferResponse507 +) & { + headers: Headers; +}; + +export type exportAdminArtifactTransferResponse = + | exportAdminArtifactTransferResponseSuccess + | exportAdminArtifactTransferResponseError; + +export const getExportAdminArtifactTransferUrl = () => { + return `/api/admin/artifact-transfers/exports`; +}; + +export const exportAdminArtifactTransfer = async ( + exportAdminArtifactTransferBody: ExportAdminArtifactTransferBody, + headers: ExportAdminArtifactTransferHeaders, + options?: Parameters[1], +): Promise => { + const getHeaders = ( + h?: NonNullable, + ): Record => { + if (!h) return {}; + if (h instanceof Headers) return Object.fromEntries(h.entries()); + if (Symbol.iterator in h) { + return Object.fromEntries( + Array.from( + h as Iterable>, + (entry) => Array.from(entry) as [string, string], + ), + ); + } + const headers: Record = {}; + for (const [name, value] of Object.entries(h)) { + if (value !== undefined) headers[name] = value; + } + return headers; + }; + return exportAdminArtifactTransferRequest( + getExportAdminArtifactTransferUrl(), + { + ...options, + method: "POST", + headers: { "Content-Type": "application/json", ...headers, ...getHeaders(options?.headers) }, + body: JSON.stringify(exportAdminArtifactTransferBody), + }, + ); +}; + export type getAdminArtifactTransferResponse200 = { data: GetAdminArtifactTransfer200; status: 200; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer202.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer202.ts new file mode 100644 index 000000000..58983e34c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer202.ts @@ -0,0 +1,8 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer202 = { + ok: true; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + jobId: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer400.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer400.ts new file mode 100644 index 000000000..c2872a1f9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer401.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer401.ts new file mode 100644 index 000000000..937e115ec --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer403.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer403.ts new file mode 100644 index 000000000..ab054cd6c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer404.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer404.ts new file mode 100644 index 000000000..727120c03 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer409.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer409.ts new file mode 100644 index 000000000..887d57f03 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer410.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer410.ts new file mode 100644 index 000000000..bc9143627 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer413.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer413.ts new file mode 100644 index 000000000..85b273220 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer415.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer415.ts new file mode 100644 index 000000000..7b9fbab96 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer422.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer422.ts new file mode 100644 index 000000000..7ee1c281b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer429.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer429.ts new file mode 100644 index 000000000..d5443c346 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer500.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer500.ts new file mode 100644 index 000000000..a5c42dd64 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer503.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer503.ts new file mode 100644 index 000000000..2f33314cb --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer507.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer507.ts new file mode 100644 index 000000000..cf8a98201 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransfer507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransfer507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferBody.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferBody.ts new file mode 100644 index 000000000..96d33d2da --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferBody.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { ExportAdminArtifactTransferBodyServiceId } from "./exportAdminArtifactTransferBodyServiceId"; + +export type ExportAdminArtifactTransferBody = { + serviceId: ExportAdminArtifactTransferBodyServiceId; + /** @pattern ^[a-f0-9]{64}$ */ + expectedArtifactFingerprint: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferBodyServiceId.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferBodyServiceId.ts new file mode 100644 index 000000000..e6412863b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferBodyServiceId.ts @@ -0,0 +1,13 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransferBodyServiceId = + (typeof ExportAdminArtifactTransferBodyServiceId)[keyof typeof ExportAdminArtifactTransferBodyServiceId]; + +export const ExportAdminArtifactTransferBodyServiceId = { + tileserver: "tileserver", + osrm: "osrm", + otp: "otp", + pelias: "pelias", + motis: "motis", +} as const; diff --git a/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferHeaders.ts b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferHeaders.ts new file mode 100644 index 000000000..834b5de5b --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/exportAdminArtifactTransferHeaders.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type ExportAdminArtifactTransferHeaders = { + /** + * @pattern ^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$ + */ + "idempotency-key": string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200.ts new file mode 100644 index 000000000..b7d840d4d --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200.ts @@ -0,0 +1,17 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. +import type { GetAdminArtifactTransferCapabilities200Limits } from "./getAdminArtifactTransferCapabilities200Limits"; +import type { GetAdminArtifactTransferCapabilities200TargetsItem } from "./getAdminArtifactTransferCapabilities200TargetsItem"; + +export type GetAdminArtifactTransferCapabilities200 = { + limits: GetAdminArtifactTransferCapabilities200Limits; + freeBytes: number | null; + freeInodes: number | null; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + reservedBytes: number; + /** @maxItems 5 */ + targets: GetAdminArtifactTransferCapabilities200TargetsItem[]; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200Limits.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200Limits.ts new file mode 100644 index 000000000..b659efa0a --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200Limits.ts @@ -0,0 +1,30 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities200Limits = { + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + maxUploadBytes: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + maxExpandedBytes: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + maxStageBytes: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + stageMs: number; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + downloadRetentionMs: number; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200TargetsItem.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200TargetsItem.ts new file mode 100644 index 000000000..579505b45 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities200TargetsItem.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities200TargetsItem = { + /** @maxLength 64 */ + serviceId: string; + available: boolean; + reason: string | null; + artifactFingerprint: string | null; +}; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities400.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities400.ts new file mode 100644 index 000000000..fe3af1620 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities400.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities400 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities401.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities401.ts new file mode 100644 index 000000000..f895f2e6f --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities401.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities401 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities403.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities403.ts new file mode 100644 index 000000000..781c3c9c1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities403.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities403 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities404.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities404.ts new file mode 100644 index 000000000..5fcbaadde --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities404.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities404 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities409.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities409.ts new file mode 100644 index 000000000..fd4aff036 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities409.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities409 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities410.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities410.ts new file mode 100644 index 000000000..7625a98d1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities410.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities410 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities413.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities413.ts new file mode 100644 index 000000000..1240c57a1 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities413.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities413 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities415.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities415.ts new file mode 100644 index 000000000..31fe46625 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities415.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities415 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities422.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities422.ts new file mode 100644 index 000000000..3fccb39f6 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities422.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities422 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities429.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities429.ts new file mode 100644 index 000000000..f6b5dbe19 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities429.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities429 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities500.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities500.ts new file mode 100644 index 000000000..9dfb6348c --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities500.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities500 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities503.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities503.ts new file mode 100644 index 000000000..3c5323b7e --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities503.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities503 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities507.ts b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities507.ts new file mode 100644 index 000000000..b25e418d9 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminArtifactTransferCapabilities507.ts @@ -0,0 +1,20 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminArtifactTransferCapabilities507 = + | { + error: string; + ok?: false; + message?: string; + jobId?: string; + } + | { + /** + * @minimum 400 + * @maximum 599 + */ + statusCode: number; + error: string; + message: string; + code?: string; + }; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index e7b104af4..437360bd1 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -135,6 +135,23 @@ export * from "./downloadAdminArtifactTransferContent429"; export * from "./downloadAdminArtifactTransferContent500"; export * from "./downloadAdminArtifactTransferContent503"; export * from "./downloadAdminArtifactTransferContent507"; +export * from "./exportAdminArtifactTransfer202"; +export * from "./exportAdminArtifactTransfer400"; +export * from "./exportAdminArtifactTransfer401"; +export * from "./exportAdminArtifactTransfer403"; +export * from "./exportAdminArtifactTransfer404"; +export * from "./exportAdminArtifactTransfer409"; +export * from "./exportAdminArtifactTransfer410"; +export * from "./exportAdminArtifactTransfer413"; +export * from "./exportAdminArtifactTransfer415"; +export * from "./exportAdminArtifactTransfer422"; +export * from "./exportAdminArtifactTransfer429"; +export * from "./exportAdminArtifactTransfer500"; +export * from "./exportAdminArtifactTransfer503"; +export * from "./exportAdminArtifactTransfer507"; +export * from "./exportAdminArtifactTransferBody"; +export * from "./exportAdminArtifactTransferBodyServiceId"; +export * from "./exportAdminArtifactTransferHeaders"; export * from "./getAdminArtifactTransfer200"; export * from "./getAdminArtifactTransfer200Cleanup"; export * from "./getAdminArtifactTransfer200Direction"; @@ -153,6 +170,22 @@ export * from "./getAdminArtifactTransfer429"; export * from "./getAdminArtifactTransfer500"; export * from "./getAdminArtifactTransfer503"; export * from "./getAdminArtifactTransfer507"; +export * from "./getAdminArtifactTransferCapabilities200"; +export * from "./getAdminArtifactTransferCapabilities200Limits"; +export * from "./getAdminArtifactTransferCapabilities200TargetsItem"; +export * from "./getAdminArtifactTransferCapabilities400"; +export * from "./getAdminArtifactTransferCapabilities401"; +export * from "./getAdminArtifactTransferCapabilities403"; +export * from "./getAdminArtifactTransferCapabilities404"; +export * from "./getAdminArtifactTransferCapabilities409"; +export * from "./getAdminArtifactTransferCapabilities410"; +export * from "./getAdminArtifactTransferCapabilities413"; +export * from "./getAdminArtifactTransferCapabilities415"; +export * from "./getAdminArtifactTransferCapabilities422"; +export * from "./getAdminArtifactTransferCapabilities429"; +export * from "./getAdminArtifactTransferCapabilities500"; +export * from "./getAdminArtifactTransferCapabilities503"; +export * from "./getAdminArtifactTransferCapabilities507"; export * from "./getAdminComposePreview400"; export * from "./getAdminComposePreview401"; export * from "./getAdminComposePreview403"; diff --git a/packages/core/src/build-artifacts/archive.test.ts b/packages/core/src/build-artifacts/archive.test.ts index 128618ef9..0bfe84443 100644 --- a/packages/core/src/build-artifacts/archive.test.ts +++ b/packages/core/src/build-artifacts/archive.test.ts @@ -1,5 +1,13 @@ import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Readable } from "node:stream"; @@ -193,6 +201,9 @@ it("exports a sanitized MOTIS candidate with credential IDs and no generated run mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); symlinkSync("slots/A", join(data, "motis/live")); writeFileSync(join(data, "motis/slots/A/data/tt.bin"), "TT"); + writeFileSync(join(data, "motis/slots/A/license.json"), '{"license":"synthetic-public"}'); + writeFileSync(join(root, "acquired.gtfs.zip"), "synthetic acquired timetable"); + linkSync(join(root, "acquired.gtfs.zip"), join(data, "motis/slots/A/de_feed.gtfs.zip")); writeFileSync(join(data, "motis/slots/A/config.yml"), "secret-api-key"); writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "secret-api-key"); writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "secret-api-key"); @@ -231,6 +242,8 @@ it("exports a sanitized MOTIS candidate with credential IDs and no generated run runtimeMaterialization: "motis-target-credentials", }); expect(Buffer.concat(chunks).includes(Buffer.from("secret-api-key"))).toBe(false); + expect(result.manifest.files.map((file) => file.path)).toContain("motis/live/license.json"); + expect(result.manifest.files.map((file) => file.path)).toContain("motis/live/de_feed.gtfs.zip"); const validated = await validate(Buffer.concat(chunks)); expect(validated.artifactFingerprint).toBe(result.artifactFingerprint); }); diff --git a/packages/core/src/build-artifacts/archive.ts b/packages/core/src/build-artifacts/archive.ts index 218488064..37750ae90 100644 --- a/packages/core/src/build-artifacts/archive.ts +++ b/packages/core/src/build-artifacts/archive.ts @@ -267,7 +267,7 @@ function pax(data: Buffer): Record { } export async function validateBuildBundle(options: { source: Readable; - stagingRoot: string; + stagingRoot: string | AnchoredDirectory; limits: ArtifactArchiveLimits; signal: AbortSignal; reservePayload?: (reservation: ArtifactPayloadReservation) => void | Promise; @@ -275,9 +275,13 @@ export async function validateBuildBundle(options: { const { source, limits, signal } = options; checkLimits(limits); signal.throwIfAborted(); - const root = new AnchoredDirectory(options.stagingRoot); + const borrowed = options.stagingRoot instanceof AnchoredDirectory; + const root = borrowed + ? (options.stagingRoot as AnchoredDirectory) + : new AnchoredDirectory(options.stagingRoot as string); + root.check(); if (root.list().length) { - root.close(); + if (!borrowed) root.close(); throw Error("Archive staging root must be empty"); } const archiveHash = createHash("sha256"); @@ -487,7 +491,7 @@ export async function validateBuildBundle(options: { signal.removeEventListener("abort", abort); source.destroy(); await decoded.return(undefined); - root.close(); + if (!borrowed) root.close(); } } function excluded(path: string, dir: boolean, patterns: readonly string[]) { @@ -534,6 +538,11 @@ export async function writeBuildBundle(options: { const files: BundleFile[] = []; const directories: string[] = []; const sources = new Map(); + // Real DM candidates hardlink acquired bulk inputs. No generated metadata, + // script, proxy configuration or other service can use this exception. + const linkedMotisInput = (path: string) => + options.serviceId === "motis" && + /^motis\/(?:live|slots\/[AB])\/[^/]+\.(?:gtfs\.zip|netex\.zip|osm\.pbf)$/.test(path); const fileRead = async (path: string, read: (fd: number, stat: Stats) => Promise) => { const parts = path.split("/"); const leaf = parts.pop()!; @@ -548,7 +557,7 @@ export async function writeBuildBundle(options: { const name = parent.name(leaf); fd = openSync(name, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); const before = fstatSync(fd); - if (!before.isFile() || before.nlink !== 1) + if (!before.isFile() || (before.nlink !== 1 && !linkedMotisInput(path))) throw Error("Refusing non-regular or hardlinked build artifact"); await read(fd, before); const after = fstatSync(fd); @@ -560,7 +569,7 @@ export async function writeBuildBundle(options: { before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || - after.nlink !== 1 + after.nlink !== before.nlink ) throw Error("Build artifact changed during export"); } finally { @@ -578,6 +587,7 @@ export async function writeBuildBundle(options: { (rel.startsWith("motis/live/") && !(rel === "motis/live/data" || rel.startsWith("motis/live/data/")) && !/\.(osm\.pbf|gtfs\.zip|netex\.zip)$/.test(rel) && + rel !== "motis/live/license.json" && rel !== "motis/live/portable-sources.json")) ) return; @@ -589,7 +599,7 @@ export async function writeBuildBundle(options: { const entries = withDirectories(root, physical.split("/"), false, (d) => d.list().sort()); for (const name of entries) await visit(`${rel}/${name}`, `${physical}/${name}`); } else { - if (!stat.isFile() || stat.nlink !== 1) + if (!stat.isFile() || (stat.nlink !== 1 && !linkedMotisInput(physical))) throw Error(`Refusing non-regular or hardlinked build artifact: ${rel}`); if (excluded(rel, false, spec.exclude)) return; let sha256 = ""; diff --git a/packages/core/src/build-artifacts/artifact-path.ts b/packages/core/src/build-artifacts/artifact-path.ts new file mode 100644 index 000000000..74d0660a6 --- /dev/null +++ b/packages/core/src/build-artifacts/artifact-path.ts @@ -0,0 +1,25 @@ +import { createHash } from "node:crypto"; +export function safeArtifactPath(path: string, maxBytes = 4096, maxDepth = 64): boolean { + return ( + !!path && + Buffer.byteLength(path) <= maxBytes && + !/^[A-Za-z]:/.test(path) && + !path.includes("\\") && + ![...path].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127) && + path.split("/").length <= maxDepth && + path.split("/").every((p) => !!p && p !== "." && p !== ".." && Buffer.byteLength(p) <= 255) + ); +} +export function artifactFingerprint( + files: readonly { path: string; sizeBytes: number; sha256: string }[], +): string { + return createHash("sha256") + .update( + JSON.stringify( + [...files] + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + .map((f) => [f.path, f.sizeBytes, f.sha256]), + ), + ) + .digest("hex"); +} diff --git a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts index 5dc6c7091..57283caaa 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-budget.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-budget.ts @@ -408,7 +408,9 @@ export function assertProspectiveConfigurationMaintenancePreludeBudget(input: { ? record.affectedServiceIds.filter( (id) => !protectedConfigurationServiceIdSchema.safeParse(id).success, ) - : record.apiFollowUp === "service-apply" || record.privateCoordinatorLifecycle + : record.apiFollowUp === "service-apply" || + record.privateCoordinatorLifecycle || + record.privateIntegrationReload?.stagedMigration ? record.affectedServiceIds : []; const handoff = diff --git a/packages/core/src/build-artifacts/copy-tree.ts b/packages/core/src/build-artifacts/copy-tree.ts index 379e530f7..aefd50bd1 100644 --- a/packages/core/src/build-artifacts/copy-tree.ts +++ b/packages/core/src/build-artifacts/copy-tree.ts @@ -14,8 +14,8 @@ import { import { open } from "node:fs/promises"; import { Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; +import { artifactFingerprint, safeArtifactPath } from "./artifact-path"; import { type AnchoredDirectory, identity, sameInode } from "./filesystem"; -import { artifactFingerprint, safeArtifactPath } from "./manifest"; export interface ArtifactTreeLimits { maximumEntries: number; maximumBytes: number; diff --git a/packages/core/src/build-artifacts/generation-hardlinks-owner.ts b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts index fba976107..d894cc719 100644 --- a/packages/core/src/build-artifacts/generation-hardlinks-owner.ts +++ b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts @@ -49,6 +49,7 @@ import { assertArtifactTargetLeaseExecution, inspectArtifactTargetLeaseChildSettlement, type TargetLeaseAuthority, + verifyRetainedArtifactTargetRootRelease, } from "./target-lease"; const schema = z.strictObject({ @@ -270,3 +271,48 @@ export async function executeOwnedGenerationHardlinks(options: { throw new DataExecutionObservationPending(identity, "transport"); return hardlinkApplySummarySchema.parse(status.result.hardlinks); } + +/** Settlement-only observation. Never bind, issue, dispatch, cancel or replay a + * hardlink leaf after its original owner has stopped forwarding. */ +export function inspectOwnedGenerationHardlinks( + authority: VerifiedExecutionAuthority, + lease?: TargetLeaseAuthority, +) { + if (!lease) verifyRetainedArtifactTargetRootRelease(authority); + const owner = executionAuthorityState(authority), + plan = readGenerationHardlinksOwnerPlan(authority); + if (!plan.reservation) return "complete" as const; + const binding = lookupControlChildPhaseBinding(authority, plan.reservation); + if (!binding) return "not_attempted" as const; + const proof = verifyDataControlTerminalWitness( + owner.installation.installation, + binding.child.execution, + ), + settled = lease + ? inspectArtifactTargetLeaseChildSettlement(lease, binding.child.execution) + : null; + if ( + lease && + (!settled || + settled.outcome === "unknown" || + settled.publication === "unknown" || + settled.publication !== proof.status.publication) + ) + throw Error("Original hardlink leaf remains unresolved"); + if ( + proof.status.status === "ok" && + proof.status.publication === "committed" && + proof.status.result?.kind === "generation-hardlinks" + ) { + if (lease && settled?.outcome !== "committed") + throw Error("Original hardlink commit remains unresolved"); + hardlinkApplySummarySchema.parse(proof.status.result.hardlinks); + return "complete" as const; + } + if ( + proof.status.publication !== "unknown" && + ["partial", "error", "canceled", "interrupted"].includes(proof.status.status) + ) + return "failed" as const; + throw Error("Original hardlink leaf remains unresolved"); +} diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index c55abd1e7..dc98f0ef6 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -199,6 +199,7 @@ export { export { executeOwnedGenerationHardlinks, type GenerationHardlinksOwnerPlan, + inspectOwnedGenerationHardlinks, planGenerationHardlinksOwner, readGenerationHardlinksOwnerPlan, readOwnedGenerationHardlinkPlan, diff --git a/packages/core/src/build-artifacts/manifest.ts b/packages/core/src/build-artifacts/manifest.ts index 68f79f4af..938ee0fe5 100644 --- a/packages/core/src/build-artifacts/manifest.ts +++ b/packages/core/src/build-artifacts/manifest.ts @@ -1,7 +1,8 @@ -import { createHash } from "node:crypto"; import { z } from "zod"; import { ServiceRegistry } from "../services/registry"; import { serviceContainerImageReference } from "../services/types"; +import { artifactFingerprint, safeArtifactPath } from "./artifact-path"; +import { portableMotisRecipeSchema } from "./motis-portable"; export interface BuildArtifactSpec { dirs: string[]; exclude: string[]; @@ -19,7 +20,7 @@ export const BUILD_ARTIFACTS: Readonly> = { motis: { dirs: ["motis/live", "motis-feed-proxy"], exclude: [], - consumers: ["motis", "motis-staging"], + consumers: ["motis", "motis-staging", "motis-feed-proxy"], }, }; export function buildArtifactSpec(serviceId: string): BuildArtifactSpec { @@ -92,28 +93,8 @@ const schema = z.discriminatedUnion("schemaVersion", [ z.strictObject({ schemaVersion: z.literal(1), ...common, ...layout }), z.strictObject({ schemaVersion: z.literal(2), ...common, ...v2, ...layout }), ]); -export function safeArtifactPath(path: string, maxBytes = 4096, maxDepth = 64): boolean { - return ( - !!path && - Buffer.byteLength(path) <= maxBytes && - !/^[A-Za-z]:/.test(path) && - !path.includes("\\") && - ![...path].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127) && - path.split("/").length <= maxDepth && - path.split("/").every((p) => !!p && p !== "." && p !== ".." && Buffer.byteLength(p) <= 255) - ); -} -export function artifactFingerprint(files: readonly BundleFile[]): string { - return createHash("sha256") - .update( - JSON.stringify( - [...files] - .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) - .map((f) => [f.path, f.sizeBytes, f.sha256]), - ), - ) - .digest("hex"); -} + +export { artifactFingerprint, safeArtifactPath } from "./artifact-path"; export function parseBundleManifest(text: string): BundleManifest { const result = schema.safeParse(JSON.parse(text)); if (!result.success) throw Error("Bundle manifest is malformed"); @@ -181,6 +162,7 @@ export async function runtimeImageMismatches( } export const portableMotisSourcesSchema = z.strictObject({ + recipe: portableMotisRecipeSchema.optional(), version: z.literal(1), sourceFingerprints: z .array( diff --git a/packages/core/src/build-artifacts/motis-candidate.ts b/packages/core/src/build-artifacts/motis-candidate.ts index 1596e8a71..81d8cb78b 100644 --- a/packages/core/src/build-artifacts/motis-candidate.ts +++ b/packages/core/src/build-artifacts/motis-candidate.ts @@ -2,9 +2,9 @@ import { realpathSync } from "node:fs"; import type { MotisConfigExpectations, MotisOperationsPolicy } from "@openmapx/transitous-core"; import { z } from "zod"; +import { safeArtifactPath } from "./artifact-path"; import { hashArtifactFile } from "./copy-tree"; import { AnchoredDirectory } from "./filesystem"; -import { safeArtifactPath } from "./manifest"; export const CANDIDATE_MANIFEST_FILENAME = "motis-candidate-manifest.json"; export const CAPABILITY_SNAPSHOT_FILENAME = "mobility-capabilities.json"; export const CANDIDATE_PROXY_DIRNAME = ".openmapx-feed-proxy"; diff --git a/packages/core/src/build-artifacts/motis-destination.ts b/packages/core/src/build-artifacts/motis-destination.ts new file mode 100644 index 000000000..32d823598 --- /dev/null +++ b/packages/core/src/build-artifacts/motis-destination.ts @@ -0,0 +1,169 @@ +/** Destination-only offline MOTIS reconstruction; no Docker, HTTP or lifecycle calls. */ +import { createHash } from "node:crypto"; +import { lstatSync, realpathSync } from "node:fs"; +import { join } from "node:path"; +import { TRANSITOUS_CATALOG_DIR } from "@openmapx/transitous-core"; +import { parseFeedCredentialMapJson } from "@openmapx/transitous-core/feed-credentials"; +import { readCatalogPinSet } from "../catalog-pins"; +import { GENERATED_SECRETS_DIRNAME } from "../services/compose-renderer"; +import { currentConfigurationGeneration } from "../services/configuration-generation"; +import { CONFIGURATION_GENERATIONS } from "../services/deployment"; +import { AnchoredDirectory } from "./filesystem"; +import { portableMotisSourcesSchema } from "./manifest"; +import { + createMotisPinnedBlobReader, + inspectPortableMotisRecipe, + MAX_MOTIS_MATERIALIZATION_BYTES, + materializePortableMotisCandidate, + PORTABLE_MOTIS_FILENAME, +} from "./motis-portable"; +export interface MotisPortableDestinationInput { + /** Host-selected checkout and infra roots, never operation/body fields. */ + rootDir: string; + infraDir: string; + /** Store-selected extracted motis/live directory, never a caller path. */ + stagingDir: string; + configurationRevisionId: string; + /** The agent's current fixed Compose config output for data-manager. */ + dataManagerEnvironment: Readonly>; + signal?: AbortSignal; +} +const digest = (raw: unknown) => createHash("sha256").update(JSON.stringify(raw)).digest("hex"); +async function destination(input: MotisPortableDestinationInput) { + input.signal?.throwIfAborted(); + if (currentConfigurationGeneration(input.infraDir) !== input.configurationRevisionId) + throw Error("MOTIS destination configuration changed; preview again"); + const pair = readCatalogPinSet(input.rootDir); + if (!pair) throw Error("MOTIS destination requires an approved retained catalog pin set"); + const pins = { + transitousCommit: pair.transitous.ref.split("@").at(-1) ?? "", + atlasCommit: pair.transitous.submodules["transitland-atlas"] ?? "", + }; + const stage = new AnchoredDirectory(realpathSync(input.stagingDir)); + let metadata: ReturnType; + try { + metadata = portableMotisSourcesSchema.parse( + JSON.parse(stage.read(PORTABLE_MOTIS_FILENAME, 1024 * 1024, false).toString("utf8")), + ); + } finally { + stage.close(); + } + if (!metadata.recipe) + throw Error("MOTIS bundle has no offline recipe; rebuild the source candidate"); + const readBlob = createMotisPinnedBlobReader( + join(input.infraDir, "data", TRANSITOUS_CATALOG_DIR), + pins, + input.signal, + ); + const inspected = await inspectPortableMotisRecipe({ recipe: metadata.recipe, pins, readBlob }); + if ( + JSON.stringify(inspected.credentialRequirements) !== + JSON.stringify([...metadata.credentialRequirements].sort()) || + JSON.stringify(metadata.sourceFingerprints) !== + JSON.stringify( + inspected.recipe.candidate.artifacts.datasets.map((f) => ({ + id: f.path, + sha256: f.sha256, + sizeBytes: f.sizeBytes, + })), + ) + ) + throw Error("MOTIS portable source or credential inventory does not match its candidate"); + let credentials: Readonly> = {}, + credentialIdentity: unknown = null; + const env = input.dataManagerEnvironment; + if (inspected.credentialRequirements.length) { + if ( + env.TRANSITOUS_API_KEYS_PATH || + (env.TRANSITOUS_API_KEYS_FILE && + env.TRANSITOUS_API_KEYS_FILE !== "/run/secrets/TRANSITOUS_API_KEYS") + ) + throw Error( + "MOTIS credentials are host-owned; prepare a supported managed credential generation before importing", + ); + if (env.TRANSITOUS_API_KEYS_FILE !== "/run/secrets/TRANSITOUS_API_KEYS") + throw Error("MOTIS destination requires the selected managed feed credentials"); + const opened: AnchoredDirectory[] = []; + try { + let directory = new AnchoredDirectory(realpathSync(input.infraDir)); + opened.push(directory); + for (const part of [ + CONFIGURATION_GENERATIONS, + input.configurationRevisionId, + GENERATED_SECRETS_DIRNAME, + "data-manager", + ]) { + directory = directory.directory(part); + opened.push(directory); + } + const path = directory.name("TRANSITOUS_API_KEYS"), + before = lstatSync(path, { bigint: true }), + bytes = directory.read("TRANSITOUS_API_KEYS", 65536, false), + after = lstatSync(path, { bigint: true }); + const fields = ["dev", "ino", "size", "mtimeNs", "ctimeNs"] as const; + if (fields.some((key) => before[key] !== after[key])) + throw Error("MOTIS destination credentials changed; preview again"); + credentials = parseFeedCredentialMapJson( + new TextDecoder("utf8", { fatal: true }).decode(bytes), + ); + if (inspected.credentialRequirements.some((key) => !credentials[key])) + throw Error("MOTIS destination has missing feed credentials"); + credentialIdentity = fields.map((key) => String(after[key])); + } finally { + for (const directory of opened.reverse()) directory.close(); + } + } + if ( + currentConfigurationGeneration(input.infraDir) !== input.configurationRevisionId || + readCatalogPinSet(input.rootDir)?.generationId !== pair.generationId + ) + throw Error("MOTIS destination catalog or configuration changed; preview again"); + const fingerprint = digest({ + recipe: metadata.recipe, + pins: pair.generationId, + configuration: input.configurationRevisionId, + credentialIdentity, + }); + return { + recipe: inspected.recipe, + pins, + readBlob, + credentials, + fingerprint, + requiredCredentials: inspected.credentialRequirements, + }; +} +/** A masked prerequisite snapshot; no raw credential values leave this function. */ +export async function inspectMotisPortableDestination(input: MotisPortableDestinationInput) { + const value = await destination(input); + return { + fingerprint: value.fingerprint, + requiredCredentials: value.requiredCredentials, + materializationMaxBytes: MAX_MOTIS_MATERIALIZATION_BYTES, + }; +} +/** Caller holds the original target/configuration writer leases and stopped-consumer fence. */ +export async function materializeMotisPortableDestination( + input: MotisPortableDestinationInput & { + expectedFingerprint: string; + archiveFingerprint: string; + }, +) { + const value = await destination(input); + if (value.fingerprint !== input.expectedFingerprint) + throw Error("MOTIS destination prerequisites changed; preview again"); + const manifest = await materializePortableMotisCandidate({ + archiveFingerprint: input.archiveFingerprint, + stagingDir: input.stagingDir, + recipe: value.recipe, + pins: value.pins, + readBlob: value.readBlob, + credentials: value.credentials, + signal: input.signal, + }); + // Detect changed pins/generation/credential inode after the bounded write. The + // private candidate remains unpublished if this fence fails. + if ((await destination(input)).fingerprint !== input.expectedFingerprint) + throw Error("MOTIS destination prerequisites changed during materialization"); + return manifest; +} diff --git a/packages/core/src/build-artifacts/motis-portable.test.ts b/packages/core/src/build-artifacts/motis-portable.test.ts new file mode 100644 index 000000000..63a267e92 --- /dev/null +++ b/packages/core/src/build-artifacts/motis-portable.test.ts @@ -0,0 +1,533 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import * as fs from "node:fs"; +import { + cpSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readlinkSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { Readable, Writable } from "node:stream"; +import { renderFeedProxyNginxConfig } from "@openmapx/motis-feed-proxy-config"; +import { afterEach, expect, it, vi } from "vitest"; +import { createCandidateManifest } from "../../../../services/data-manager/src/jobs/transitous/candidate"; +import { + inspectPortableMotisRecipe, + type MotisPinnedBlobReader, + materializePortableMotisCandidate, + producePortableMotisRecipe, + readVerifiedPortableMotisMaterialization, +} from "./motis-portable"; + +vi.mock("node:fs", async (original) => { + const native = await original(); + return { ...native, renameSync: vi.fn(native.renameSync) }; +}); +const roots: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +const root = () => { + const value = mkdtempSync(join(tmpdir(), "omx-portable-motis-")); + roots.push(value); + return value; +}; +const pins = { transitousCommit: "a".repeat(40), atlasCommit: "b".repeat(40) }; +const credentials = { "de/demo": "SOURCE_PRIVATE_KEY" }; +const destinationCredentials = { "de/demo": "DESTINATION_PRIVATE_KEY" }; +const config = { + server: { port: 8080 }, + timetable: { + datasets: { + "de-demo": { + path: "de_demo.gtfs.zip", + rt: [{ url: "http://motis-feed-proxy/feed/de-demo-0", protocol: "gtfsrt" }], + }, + }, + }, +}; +const readBlob: MotisPinnedBlobReader = vi.fn(async (repo, _commit, path) => { + if (repo === "catalog" && path === "configs/motis/config.yml") + return JSON.stringify({ server: { port: 8080 }, timetable: { datasets: {} } }); + if (repo === "catalog" && path === "feeds/de.json") + return JSON.stringify({ + sources: [{ name: "demo", type: "transitland-atlas", "transitland-atlas-id": "f-demo" }], + }); + if (repo === "atlas" && path === "feeds/.openmapx-atlas-id/f-demo") + return JSON.stringify({ + feeds: [ + { + id: "f-demo", + urls: { + static_current: "https://operator.example/static.zip", + realtime_trip_updates: "https://operator.example/realtime.pb", + }, + authorization: { type: "header", param_name: "X-Api-Key" }, + }, + ], + }); + throw Error("No pinned blob"); +}); +function fixture() { + const dir = root(), + vars = { + "de-demo-0": { + url: "https://operator.example/realtime.pb", + headers: { "X-Api-Key": credentials["de/demo"] }, + gbfs: false, + }, + }; + const files: Record = { + "config.yml": JSON.stringify(config), + "license.json": "[]", + "de_demo.gtfs.zip": "synthetic timetable", + "transit-source-manifest.json": JSON.stringify({ + version: 1, + generatedAt: "2026-10-30T00:00:00.000Z", + sources: [ + { + sourceId: "catalog:de:demo", + region: "de", + name: "demo", + format: "gtfs", + origin: "catalog", + originUrl: "https://origin.example/static?key=[redacted]", + license: {}, + transformations: [], + artifact: { + relativePath: "de_demo.gtfs.zip", + sha256: createHash("sha256").update("synthetic timetable").digest("hex"), + sizeBytes: 19, + modifiedAt: "2026-10-30T00:00:00.000Z", + }, + }, + ], + }), + ".openmapx-feed-proxy/feed-proxy-vars.json": JSON.stringify(vars), + ".openmapx-feed-proxy/conf/default.conf": renderFeedProxyNginxConfig(vars), + }; + for (const [name, bytes] of Object.entries(files)) { + mkdirSync(dirname(join(dir, name)), { recursive: true }); + writeFileSync(join(dir, name), bytes); + } + createCandidateManifest(dir, "synthetic-epoch", "2026-10-30T00:00:00.000Z"); + return dir; +} +async function produce(dir = fixture()) { + await producePortableMotisRecipe({ stagingDir: dir, pins, readBlob, credentials }); + return { dir, metadata: JSON.parse(readFileSync(join(dir, "portable-sources.json"), "utf8")) }; +} +it.each([false, true])( + "journal-publishes or restores a materialized bundle without changing MOTIS slot aliases (fault=%s)", + async (fault) => { + const control = await import("./index"), + ops = await import("../ops"), + publication = await import("./publication"); + const original = await produce(), + exportRoot = root(), + infraDir = root(), + dataRoot = join(infraDir, "data"); + cpSync(original.dir, join(exportRoot, "motis/live"), { recursive: true }); + mkdirSync(join(exportRoot, "motis/live/data")); + writeFileSync(join(exportRoot, "motis/live/data/tt.bin"), "compiled timetable"); + mkdirSync(join(exportRoot, "motis-feed-proxy")); + const chunks: Buffer[] = []; + await control.writeBuildBundle({ + dataRoot: exportRoot, + serviceId: "motis", + record: { + schemaVersion: 1, + service: "motis", + builtAt: "2026-10-30T00:00:00.000Z", + runtimeImages: { motis: "image:1" }, + toolImages: {}, + }, + target: new Writable({ + write(chunk, _encoding, done) { + chunks.push(Buffer.from(chunk)); + done(); + }, + }), + limits: control.DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: new AbortController().signal, + }); + const stageId = randomUUID(), + stagingRoot = join(dataRoot, `.openmapx-import-${stageId}`); + mkdirSync(stagingRoot, { recursive: true, mode: 0o700 }); + const bundle = await control.validateBuildBundle({ + source: Readable.from([Buffer.concat(chunks)]), + stagingRoot, + limits: control.DEFAULT_ARTIFACT_ARCHIVE_LIMITS, + signal: new AbortController().signal, + }); + const { materialization } = await materializePortableMotisCandidate({ + archiveFingerprint: bundle.artifactFingerprint, + stagingDir: join(stagingRoot, "motis/live"), + recipe: original.metadata.recipe, + pins, + readBlob, + credentials: destinationCredentials, + }); + for (const slot of ["A", "B"]) { + mkdirSync(join(dataRoot, "motis/slots", slot), { recursive: true }); + writeFileSync(join(dataRoot, "motis/slots", slot, "old"), slot); + } + symlinkSync("slots/A", join(dataRoot, "motis/live")); + symlinkSync("slots/B", join(dataRoot, "motis/staging")); + writeFileSync( + join(dataRoot, "motis/slot-state.json"), + JSON.stringify({ + schemaVersion: 1, + activeSlot: "A", + previousHealthySlot: "B", + datasetEpoch: "old", + activatedAt: "2026-01-01T00:00:00.000Z", + }), + ); + mkdirSync(join(dataRoot, "motis-feed-proxy/conf"), { recursive: true }); + writeFileSync(join(dataRoot, "motis-feed-proxy/conf/default.conf"), "old proxy"); + const aliases = ["live", "staging"].map((name) => ({ + name, + inode: lstatSync(join(dataRoot, "motis", name)).ino, + target: readlinkSync(join(dataRoot, "motis", name)), + })); + await control.provisionControlInstallation(infraDir); + const ids = ["motis", "motis-feed-proxy", "motis-staging"], + installation = control.openControlInstallation({ + infraDir, + role: "host-cli", + serviceIds: ids, + }); + let lease: import("./target-lease").TargetLeaseAuthority | undefined; + try { + const admittedAt = Date.now(), + deadlineAt = admittedAt + 300000, + expectedFingerprint = await publication.artifactPublicationFingerprint(dataRoot, "motis"), + operation = { + kind: "artifactTransfer.import" as const, + stageId, + ownerId: "host-cli", + expectedSha256: bundle.archiveSha256, + previewFingerprint: "a".repeat(64), + expectedCurrentRevisionId: null, + }, + operationFingerprint = ops.opsOperationFingerprint(operation), + execution = { + executor: "host-cli" as const, + principal: "host-cli" as const, + executionId: randomUUID(), + operationKey: ops.createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: ops.executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }, + authority = control.retainAdmittedControlExecution(installation, { + execution, + kind: operation.kind, + operation, + admittedAt, + requestedDeadlineAt: deadlineAt, + serviceIds: ids, + }); + lease = await control.acquireArtifactTargetLease({ + dataRoot, + serviceIds: ids, + owner: authority, + }); + const nativeRename = (await vi.importActual("node:fs")).renameSync; + let injected = false; + if (fault) + vi.mocked(fs.renameSync).mockImplementation((from, to) => { + if ( + !injected && + String(to).endsWith("/motis-feed-proxy") && + String(from).endsWith("/motis-feed-proxy") + ) { + injected = true; + throw Error("Named proxy publication rename failure"); + } + return nativeRename(from, to); + }); + const result = await publication.publishValidatedBuild({ + dataRoot, + stagingRoot, + stageId, + bundle, + authority, + lease, + expectedFingerprint, + materialization, + }); + expect(result.outcome).toBe(fault ? "not_committed" : "committed"); + if (fault) { + expect(injected).toBe(true); + expect(readFileSync(join(dataRoot, "motis/live/old"), "utf8")).toBe("A"); + expect(readFileSync(join(dataRoot, "motis-feed-proxy/conf/default.conf"), "utf8")).toBe( + "old proxy", + ); + expect( + JSON.parse(readFileSync(join(dataRoot, "motis/slot-state.json"), "utf8")), + ).toMatchObject({ datasetEpoch: "old", activatedAt: "2026-01-01T00:00:00.000Z" }); + } + for (const alias of aliases) { + expect(lstatSync(join(dataRoot, "motis", alias.name)).ino).toBe(alias.inode); + expect(readlinkSync(join(dataRoot, "motis", alias.name))).toBe(alias.target); + } + if (fault) return; + expect(readFileSync(join(dataRoot, "motis/live/data/tt.bin"), "utf8")).toBe( + "compiled timetable", + ); + expect(readFileSync(join(dataRoot, "motis-feed-proxy/conf/default.conf"), "utf8")).toContain( + "DESTINATION_PRIVATE_KEY", + ); + expect( + JSON.parse(readFileSync(join(dataRoot, "motis/slot-state.json"), "utf8")), + ).toMatchObject({ + activeSlot: "A", + previousHealthySlot: "B", + datasetEpoch: "synthetic-epoch", + }); + expect( + JSON.parse(readFileSync(join(dataRoot, "motis/slot-state.json"), "utf8")), + ).not.toHaveProperty("activatedAt"); + } finally { + if (lease) control.closeArtifactTargetLeaseHandle(lease); + control.closeControlInstallation(installation); + } + }, + 60000, +); +function extract(data: Awaited>) { + const dest = root(); + for (const path of ["license.json", "de_demo.gtfs.zip", "portable-sources.json"]) + writeFileSync(join(dest, path), readFileSync(join(data.dir, path))); + return dest; +} +it("produces the actual candidate's nonsecret recipe and materializes different destination credentials offline", async () => { + const source = await produce(), + serialized = JSON.stringify(source.metadata); + expect(serialized).not.toContain("SOURCE_PRIVATE_KEY"); + expect(serialized).not.toContain("operator.example/realtime"); + expect(serialized).not.toContain("originUrl"); + expect(source.metadata.credentialRequirements).toEqual(["de/demo"]); + const destination = extract(source), + { manifest } = await materializePortableMotisCandidate({ + archiveFingerprint: "d".repeat(64), + stagingDir: destination, + recipe: source.metadata.recipe, + pins, + readBlob, + credentials: destinationCredentials, + }); + expect( + readFileSync(join(destination, ".openmapx-feed-proxy/feed-proxy-vars.json"), "utf8"), + ).toContain("DESTINATION_PRIVATE_KEY"); + expect( + readFileSync(join(destination, ".openmapx-feed-proxy/conf/default.conf"), "utf8"), + ).toContain("https://operator.example/realtime.pb"); + expect(manifest.epoch).toBe("synthetic-epoch"); + expect(manifest.expectations.realtimeFeeds).toBe(1); + expect(readFileSync(join(destination, "config.yml"), "utf8")).not.toContain("PRIVATE_KEY"); +}); +it("rejects a missing destination credential without publishing candidate config", async () => { + const source = await produce(), + destination = extract(source); + await expect( + materializePortableMotisCandidate({ + archiveFingerprint: "d".repeat(64), + stagingDir: destination, + recipe: source.metadata.recipe, + pins, + readBlob, + credentials: {}, + }), + ).rejects.toThrow("credential is missing"); + expect(() => readFileSync(join(destination, "config.yml"))).toThrow(); +}); +it("binds selectors to the destination's approved catalog pair", async () => { + const source = await produce(); + await expect( + inspectPortableMotisRecipe({ + recipe: source.metadata.recipe, + pins: { ...pins, atlasCommit: "c".repeat(40) }, + readBlob, + }), + ).rejects.toThrow("pins"); +}); +it("refuses attacker-selected endpoint metadata and config exfiltration before inserting target keys", async () => { + const source = await produce(); + source.metadata.recipe.routes[0].url = "https://attacker.example"; + await expect( + inspectPortableMotisRecipe({ recipe: source.metadata.recipe, pins, readBlob }), + ).rejects.toThrow(); + delete source.metadata.recipe.routes[0].url; + const altered = JSON.stringify({ ...config, elevators: { url: "https://attacker.example" } }); + source.metadata.recipe.config = altered; + source.metadata.recipe.configSha256 = createHash("sha256").update(altered).digest("hex"); + await expect( + inspectPortableMotisRecipe({ recipe: source.metadata.recipe, pins, readBlob }), + ).rejects.toThrow(); +}); +it("refuses a changed compiled source before writing a destination config", async () => { + const source = await produce(), + destination = extract(source); + writeFileSync(join(destination, "de_demo.gtfs.zip"), "changed"); + await expect( + materializePortableMotisCandidate({ + archiveFingerprint: "d".repeat(64), + stagingDir: destination, + recipe: source.metadata.recipe, + pins, + readBlob, + credentials: destinationCredentials, + }), + ).rejects.toThrow("integrity"); + expect(() => readFileSync(join(destination, "config.yml"))).toThrow(); +}); +it.each(["headers", "secret URL", "unbound endpoint", "oversized"])( + "does not retain an unsafe %s recipe from the producer", + async (kind) => { + const dir = fixture(); + if (kind === "headers") + writeFileSync( + join(dir, "config.yml"), + JSON.stringify({ ...config, elevators: { headers: { Authorization: "HOST_SECRET" } } }), + ); + if (kind === "secret URL") + writeFileSync( + join(dir, "config.yml"), + JSON.stringify({ ...config, elevators: { url: "https://host.example/?key=HOST_SECRET" } }), + ); + if (kind === "unbound endpoint") + writeFileSync( + join(dir, ".openmapx-feed-proxy/feed-proxy-vars.json"), + JSON.stringify({ + "de-demo-0": { + url: "https://attacker.example/rt", + headers: { "X-Api-Key": credentials["de/demo"] }, + }, + }), + ); + if (kind === "oversized") + writeFileSync( + join(dir, "config.yml"), + JSON.stringify({ ...config, note: " ".repeat(1024 * 1024 + 1) }), + ); + createCandidateManifest(dir, "synthetic-epoch", "2026-10-30T00:00:00.000Z"); + await expect( + producePortableMotisRecipe({ stagingDir: dir, pins, readBlob, credentials }), + ).rejects.toThrow(); + expect(() => readFileSync(join(dir, "portable-sources.json"))).toThrow(); + }, +); +it("honors cancellation before offline destination writes", async () => { + const source = await produce(), + destination = extract(source), + controller = new AbortController(); + controller.abort(); + await expect( + materializePortableMotisCandidate({ + archiveFingerprint: "d".repeat(64), + stagingDir: destination, + recipe: source.metadata.recipe, + pins, + readBlob, + credentials: destinationCredentials, + signal: controller.signal, + }), + ).rejects.toThrow(); + expect(() => readFileSync(join(destination, "config.yml"))).toThrow(); +}); + +it("binds the nominal proof to the original archive, metadata and exact generated bytes", async () => { + const source = await produce(), + destination = extract(source); + const result = await materializePortableMotisCandidate({ + archiveFingerprint: "d".repeat(64), + stagingDir: destination, + recipe: source.metadata.recipe, + pins, + readBlob, + credentials: destinationCredentials, + }); + const metadataSha = createHash("sha256") + .update(readFileSync(join(destination, "portable-sources.json"))) + .digest("hex"); + const args = { + stagingDir: destination, + archiveFingerprint: "d".repeat(64), + portableMetadataSha256: metadataSha, + }; + const proof = await readVerifiedPortableMotisMaterialization(result.materialization, args); + expect(proof.generatedArtifacts.map((f) => f.path)).toContain("config.yml"); + expect(proof.generatedArtifacts.map((f) => f.path)).toContain( + ".openmapx-feed-proxy/feed-proxy-vars.json", + ); + await expect( + readVerifiedPortableMotisMaterialization(result.materialization, { + ...args, + archiveFingerprint: "e".repeat(64), + }), + ).rejects.toThrow("archive"); + writeFileSync(join(destination, ".openmapx-feed-proxy/feed-proxy-vars.json"), "{}"); + await expect( + readVerifiedPortableMotisMaterialization(result.materialization, args), + ).rejects.toThrow(); +}); +it("rejects a bundle-selected web directory even after the attacker updates its config digest", async () => { + const source = await produce(), + recipe = source.metadata.recipe; + const changed = JSON.stringify({ + ...config, + server: { port: 8080, web_folder: ".openmapx-feed-proxy" }, + }); + recipe.config = changed; + recipe.configSha256 = createHash("sha256").update(changed).digest("hex"); + await expect(inspectPortableMotisRecipe({ recipe, pins, readBlob })).rejects.toThrow( + "approved pinned template", + ); +}); +it.each(["userinfo", "query", "header", "source inventory", "template digest", "config digest"])( + "rejects unsafe %s before target secret materialization", + async (kind) => { + const source = await produce(), + recipe = source.metadata.recipe; + if (kind === "header") { + const cfg = JSON.parse(recipe.config); + cfg.timetable.datasets["de-demo"].rt[0].headers = { Authorization: "BUNDLE_SECRET" }; + recipe.config = JSON.stringify(cfg); + recipe.configSha256 = createHash("sha256").update(recipe.config).digest("hex"); + } + if (kind === "source inventory") + recipe.sourceManifest.sources[0].artifact.sha256 = "f".repeat(64); + if (kind === "template digest") recipe.templateSha256 = "e".repeat(64); + if (kind === "config digest") recipe.configSha256 = "e".repeat(64); + const selected: MotisPinnedBlobReader = async (repo, commit, path) => { + const text = await readBlob(repo, commit, path); + if (repo !== "atlas" || !["userinfo", "query"].includes(kind)) return text; + const value = JSON.parse(text); + value.feeds[0].urls.realtime_trip_updates = + kind === "userinfo" + ? "https://secret@operator.example/realtime.pb" + : "https://operator.example/realtime.pb?token=secret"; + return JSON.stringify(value); + }; + await expect( + inspectPortableMotisRecipe({ recipe, pins, readBlob: selected }), + ).rejects.toThrow(); + }, +); diff --git a/packages/core/src/build-artifacts/motis-portable.ts b/packages/core/src/build-artifacts/motis-portable.ts new file mode 100644 index 000000000..70b4b4e28 --- /dev/null +++ b/packages/core/src/build-artifacts/motis-portable.ts @@ -0,0 +1,957 @@ +/** Offline MOTIS recipe. Bundle bytes never select a credential destination. */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstatSync, realpathSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { + type FeedProxyVars, + normalizeFeedProxyVars, + renderFeedProxyNginxConfig, +} from "@openmapx/motis-feed-proxy-config"; +import { parseMotisConfigExpectations } from "@openmapx/transitous-core"; +import { canonicalFeedCredentialMap } from "@openmapx/transitous-core/feed-credentials"; +import { load as loadYaml } from "js-yaml"; +import { z } from "zod"; +import { hashArtifactFile } from "./copy-tree"; +import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; +import { + CANDIDATE_MANIFEST_FILENAME, + CANDIDATE_PROXY_DIRNAME, + type MotisCandidateManifest, + parseMotisCandidateManifest, + verifyCandidateManifest, +} from "./motis-candidate"; + +export const PORTABLE_MOTIS_FILENAME = "portable-sources.json"; +export const MAX_PORTABLE_MOTIS_BYTES = 1024 * 1024; +export const MAX_MOTIS_MATERIALIZATION_BYTES = 32 * 1024 * 1024; +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const token = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,255}$/); +const routeSchema = z.strictObject({ + key: token, + region: token, + sourceName: token, + endpoint: z.enum([ + "url", + "realtime_trip_updates", + "realtime_vehicle_positions", + "realtime_alerts", + "gbfs_auto_discovery", + ]), +}); +const portableSourceManifestSchema = z.strictObject({ + version: z.literal(1), + generatedAt: z.iso.datetime(), + sources: z + .array( + z.strictObject({ + sourceId: z.string().min(1).max(512), + region: token, + name: token, + format: z.enum(["gtfs", "netex"]), + origin: z.literal("catalog"), + license: z.record(z.string(), z.json()), + transformations: z.array(z.string().max(512)).max(100), + artifact: z.strictObject({ + relativePath: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_.-]*\.(gtfs|netex)\.zip$/), + sha256: digest, + sizeBytes: z.number().int().nonnegative().safe(), + modifiedAt: z.iso.datetime(), + }), + }), + ) + .min(1) + .max(10000), + profileEvidence: z + .strictObject({ + profile: z.string().max(256), + credentialRefs: z.array(z.string().max(512)).max(1000).optional(), + }) + .optional(), +}); +export const portableMotisRecipeSchema = z.strictObject({ + version: z.literal(1), + pins: z.strictObject({ + transitousCommit: z.string().regex(/^[a-f0-9]{40}$/), + atlasCommit: z.string().regex(/^[a-f0-9]{40}$/), + }), + config: z.string().min(1).max(MAX_PORTABLE_MOTIS_BYTES), + configSha256: digest, + templateSha256: digest, + candidate: z.unknown().transform((raw) => parseMotisCandidateManifest(JSON.stringify(raw))), + sourceManifest: portableSourceManifestSchema, + sourceIndex: z.json().optional(), + routes: z.array(routeSchema).max(10000), +}); +export type PortableMotisRecipe = z.infer; +export interface MotisPortablePins { + transitousCommit: string; + atlasCommit: string; +} +/** Implementations read ONLY these exact commits from the fixed local cached repositories. */ +export type MotisPinnedBlobReader = ( + repository: "catalog" | "atlas", + commit: string, + path: string, +) => Promise; +const sha = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); +const invalid = () => + Error("MOTIS portability requires a supported pinned, nonsecret source recipe"); +function object(raw: unknown): Record { + if ( + !raw || + typeof raw !== "object" || + Array.isArray(raw) || + ![null, Object.prototype].includes(Object.getPrototypeOf(raw)) + ) + throw invalid(); + return raw as Record; +} +function document(text: string): Record { + if (Buffer.byteLength(text) > MAX_PORTABLE_MOTIS_BYTES) throw invalid(); + return object(JSON.parse(text)); +} +/** Reject accidental credentials in attribution/config. Comments never survive YAML decoding. */ +function publicDocument(raw: unknown, depth = 0, seen = new Set()): void { + if (depth > 32) throw invalid(); + if (typeof raw === "string") { + if ( + raw.length > 16384 || + /(?:ENC\[|-----BEGIN |[\r\n])/.test(raw) || + raw.includes(String.fromCharCode(0)) + ) + throw invalid(); + for (const match of raw.matchAll(/https?:\/\/[^\s"'<>]+/g)) { + const url = new URL(match[0]); + if (url.username || url.password || url.search || url.hash) throw invalid(); + } + } else if (raw && typeof raw === "object") { + if (seen.has(raw)) throw invalid(); + seen.add(raw); + if (Array.isArray(raw)) for (const value of raw) publicDocument(value, depth + 1, seen); + else + for (const [key, value] of Object.entries(object(raw))) { + if (/(?:auth|header|token|password|secret|api.?key)/i.test(key)) throw invalid(); + publicDocument(value, depth + 1, seen); + } + seen.delete(raw); + } else if (raw !== null && typeof raw !== "boolean" && typeof raw !== "number") throw invalid(); +} +function configRoutes(config: Record, vars?: FeedProxyVars): Set { + const used = new Set(); + const rewrite = (raw: unknown) => { + const feed = object(raw); + if (typeof feed.url !== "string") throw invalid(); + let key: string | undefined; + if (vars) key = Object.entries(vars).find(([, value]) => value.url === feed.url)?.[0]; + if (!key) { + const match = /^http:\/\/motis-feed-proxy\/feed\/([a-zA-Z0-9_.%-]+)$/.exec(feed.url); + if (!match) throw invalid(); + key = decodeURIComponent(match[1]); + } + token.parse(key); + if (!vars && feed.headers !== undefined) throw invalid(); + delete feed.headers; + feed.url = `http://motis-feed-proxy/feed/${encodeURIComponent(key)}`; + used.add(key); + }; + const timetable = object(config.timetable), + datasets = object(timetable.datasets); + if (!Object.keys(datasets).length) throw invalid(); + for (const raw of Object.values(datasets)) { + const dataset = object(raw); + if ( + Object.keys(dataset).some( + (key) => !["path", "extend_calendar", "default_timezone", "script", "rt"].includes(key), + ) + ) + throw invalid(); + if ( + dataset.script !== undefined && + (typeof dataset.script !== "string" || + !/^scripts\/[a-zA-Z0-9][a-zA-Z0-9_.-]*\.lua$/.test(dataset.script)) + ) + throw invalid(); + if ( + typeof dataset.path !== "string" || + !/^[a-zA-Z0-9][a-zA-Z0-9_.-]*\.(gtfs|netex)\.zip$/.test(dataset.path) + ) + throw invalid(); + if (dataset.rt !== undefined) { + if (!Array.isArray(dataset.rt)) throw invalid(); + for (const rt of dataset.rt) rewrite(rt); + } + } + if (config.gbfs !== undefined && config.gbfs !== false) { + const gbfs = object(config.gbfs); + if (!vars && gbfs.proxy !== undefined && gbfs.proxy !== "http://motis-feed-proxy") + throw invalid(); + if (gbfs.proxy !== undefined) gbfs.proxy = "http://motis-feed-proxy"; + for (const feed of Object.values(object(gbfs.feeds))) rewrite(feed); + } + if ( + config.osm !== undefined && + (typeof config.osm !== "string" || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]*\.osm\.pbf$/.test(config.osm)) + ) + throw invalid(); + const walk = (raw: unknown, key = ""): void => { + if (key === "data_attribution_link") return; + if ( + typeof raw === "string" && + /https?:\/\//.test(raw) && + raw !== "http://motis-feed-proxy" && + !/^http:\/\/motis-feed-proxy\/feed\/[a-zA-Z0-9_.%-]+$/.test(raw) + ) + throw invalid(); + if (raw && typeof raw === "object") + for (const [key, value] of Object.entries(raw)) walk(value, key); + }; + // Recursion/cycle limits before the simpler URL walk. + publicDocument(config); + walk(config); + return used; +} +function samePins(a: MotisPortablePins, b: MotisPortablePins) { + if (a.transitousCommit !== b.transitousCommit || a.atlasCommit !== b.atlasCommit) + throw Error("MOTIS destination catalog pins do not match the recipe"); +} +interface ResolvedRoute { + key: string; + url: string; + gbfs: boolean; + credentialKey?: string; + header?: string; + basic?: boolean; + headers?: Record; +} +/** Selectors have no URLs; only approved destination Git blobs choose endpoints and auth. */ +async function resolveRoutes( + routes: PortableMotisRecipe["routes"], + pins: MotisPortablePins, + readBlob: MotisPinnedBlobReader, +): Promise { + const cache = new Map>>(); + const read = (repo: "catalog" | "atlas", path: string) => { + const id = `${repo}:${path}`; + let value = cache.get(id); + if (!value) { + value = readBlob( + repo, + repo === "catalog" ? pins.transitousCommit : pins.atlasCommit, + path, + ).then(document); + cache.set(id, value); + } + return value; + }; + const result: ResolvedRoute[] = []; + for (const route of routes) { + const feed = await read("catalog", `feeds/${route.region}.json`); + if (!Array.isArray(feed.sources)) throw invalid(); + const matches = feed.sources.map(object).filter((s) => s.name === route.sourceName); + if (matches.length !== 1) throw invalid(); + const source = matches[0]; + if (source.skip === true || source["url-override"] || source["api-key"]) throw invalid(); + const prefix = `${route.region}-${route.sourceName}`; + let url: unknown, + authorization: Record | undefined, + headers: Record | undefined, + gbfs = false; + if (source.type === "url" && route.endpoint === "url") { + url = source.url; + gbfs = source.spec === "gbfs"; + if (!gbfs && !["gtfs-rt", "siri-json"].includes(String(source.spec))) throw invalid(); + if (source.headers !== undefined) { + const raw = object(source.headers); + headers = {}; + for (const [key, value] of Object.entries(raw)) { + if ( + typeof value !== "string" || + value.length > 8192 || + /ENC\[|[\r\n]/.test(value) || + value.includes(String.fromCharCode(0)) || + !/^[a-zA-Z0-9-]{1,128}$/.test(key) + ) + throw invalid(); + headers[key] = value; + } + } + if (route.key !== (gbfs ? prefix : `${prefix}-0`)) throw invalid(); + } else if ( + source.type === "transitland-atlas" && + typeof source["transitland-atlas-id"] === "string" && + route.endpoint !== "url" + ) { + // Atlas stores one DMFR per id. Resolve only the exact pinned file, never mutable working-tree content. + const atlasId = source["transitland-atlas-id"]; + token.parse(atlasId); + const dmfr = await read("atlas", `feeds/.openmapx-atlas-id/${atlasId}`); + if (!Array.isArray(dmfr.feeds)) throw invalid(); + const entries = dmfr.feeds.map(object).filter((f) => f.id === atlasId); + if (entries.length !== 1) throw invalid(); + const entry = entries[0], + urls = object(entry.urls); + url = urls[route.endpoint]; + gbfs = route.endpoint === "gbfs_auto_discovery"; + const rt = ["realtime_trip_updates", "realtime_vehicle_positions", "realtime_alerts"].filter( + (k) => typeof urls[k] === "string", + ); + if (route.key !== (gbfs ? prefix : `${prefix}-${rt.indexOf(route.endpoint)}`)) + throw invalid(); + if (entry.authorization !== undefined && entry.authorization !== null) + authorization = object(entry.authorization); + } else throw invalid(); + if (typeof url !== "string") throw invalid(); + const endpoint = new URL(url); + if ( + !["http:", "https:"].includes(endpoint.protocol) || + endpoint.username || + endpoint.password || + endpoint.search || + endpoint.hash || + /[\r\n]|ENC\[/.test(url) || + url.includes(String.fromCharCode(0)) + ) + throw invalid(); + const resolved: ResolvedRoute = { key: route.key, url, gbfs, ...(headers ? { headers } : {}) }; + if (authorization) { + if (authorization.type !== "header" && authorization.type !== "basic_auth") throw invalid(); + const header = + authorization.type === "basic_auth" ? "Authorization" : authorization.param_name; + if (typeof header !== "string" || !/^[a-zA-Z0-9-]{1,128}$/.test(header)) throw invalid(); + resolved.credentialKey = `${route.region}/${route.sourceName}`; + resolved.header = header; + resolved.basic = authorization.type === "basic_auth"; + } + result.push(resolved); + } + return result; +} +function renderVars( + routes: ResolvedRoute[], + credentials: Readonly>, +): FeedProxyVars { + canonicalFeedCredentialMap(credentials); + const vars: FeedProxyVars = {}; + for (const route of routes) { + const headers = { ...route.headers }; + if (route.credentialKey) { + const value = credentials[route.credentialKey]; + if (!value) throw Error(`MOTIS destination credential is missing: ${route.credentialKey}`); + if (/[\r\n]/.test(value) || value.includes(String.fromCharCode(0))) + throw Error("MOTIS destination credential is unsafe for a proxy header"); + if (!route.header) throw invalid(); + headers[route.header] = `${route.basic ? "Basic " : ""}${value}`; + } + vars[route.key] = { + url: route.url, + gbfs: route.gbfs, + ...(Object.keys(headers).length ? { headers } : {}), + }; + } + return normalizeFeedProxyVars(vars); +} +function equalVars(a: FeedProxyVars, b: FeedProxyVars) { + const canonical = (vars: FeedProxyVars) => + JSON.stringify( + Object.entries(vars) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => [ + k, + v.url, + v.gbfs === true, + Object.entries(v.headers ?? {}).sort(([a], [b]) => a.localeCompare(b)), + ]), + ); + return canonical(a) === canonical(b); +} +/** Static runtime fields come from the exact approved pinned template. Only + * generator-owned dataset/GBFS nodes and documented scalar overrides vary. */ +async function validatePinnedTemplate( + config: Record, + pins: MotisPortablePins, + readBlob: MotisPinnedBlobReader, +) { + const template = await readBlob("catalog", pins.transitousCommit, "configs/motis/config.yml"); + if (Buffer.byteLength(template) > MAX_PORTABLE_MOTIS_BYTES) throw invalid(); + const base = object(loadYaml(template)); + const normalize = (raw: Record) => { + const value = structuredClone(raw); + delete value.osm; + delete value.tiles; + if (typeof value.osr_footpath === "boolean") delete value.osr_footpath; + if (value.elevators === false) delete value.elevators; + const timetable = object(value.timetable); + delete timetable.datasets; + if (typeof timetable.incremental_rt_update === "boolean") + delete timetable.incremental_rt_update; + if (value.gbfs !== undefined && value.gbfs !== false) { + const gbfs = object(value.gbfs); + delete gbfs.feeds; + delete gbfs.proxy; + } + return value; + }; + const canonical = (value: unknown): string => + Array.isArray(value) + ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" + ? `{${Object.entries(value) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`) + .join(",")}}` + : JSON.stringify(value); + if (canonical(normalize(config)) !== canonical(normalize(base))) + throw Error("MOTIS configuration does not match the approved pinned template"); + if (config.tiles !== undefined && canonical(config.tiles) !== canonical(base.tiles)) + throw invalid(); + return sha(template); +} +export async function inspectPortableMotisRecipe(input: { + recipe: unknown; + pins: MotisPortablePins; + readBlob: MotisPinnedBlobReader; +}) { + const recipe = portableMotisRecipeSchema.parse(input.recipe); + if (Buffer.byteLength(JSON.stringify(recipe)) > MAX_PORTABLE_MOTIS_BYTES) throw invalid(); + samePins(recipe.pins, input.pins); + if (sha(recipe.config) !== recipe.configSha256) + throw Error("MOTIS portable configuration digest mismatch"); + const config = document(recipe.config); + const used = configRoutes(config); + if ((await validatePinnedTemplate(config, input.pins, input.readBlob)) !== recipe.templateSha256) + throw Error("MOTIS pinned template digest mismatch"); + if ( + recipe.candidate.artifacts.config.sha256 !== recipe.configSha256 || + recipe.candidate.artifacts.config.sizeBytes !== Buffer.byteLength(recipe.config) || + JSON.stringify(recipe.candidate.expectations) !== + JSON.stringify(parseMotisConfigExpectations(recipe.config)) + ) + throw Error("MOTIS portable config readiness evidence does not match its template"); + const names = recipe.routes.map((r) => r.key); + if ( + new Set(names).size !== names.length || + names.length !== used.size || + names.some((key) => !used.has(key)) + ) + throw invalid(); + publicDocument(recipe.sourceManifest); + if (recipe.sourceIndex !== undefined) publicDocument(recipe.sourceIndex); + const sourceManifest = object(recipe.sourceManifest); + if (sourceManifest.version !== 1 || !Array.isArray(sourceManifest.sources)) throw invalid(); + const sourceInventory = sourceManifest.sources.map((source) => object(object(source).artifact)); + const expected = recipe.candidate.artifacts.datasets; + if ( + sourceInventory.length !== expected.length || + new Set(sourceInventory.map((a) => a.relativePath)).size !== sourceInventory.length || + expected.some( + (file) => + !sourceInventory.some( + (a) => + a.relativePath === file.path && + a.sha256 === file.sha256 && + a.sizeBytes === file.sizeBytes, + ), + ) + ) + throw Error("MOTIS source manifest does not match the candidate inventory"); + for (const source of recipe.sourceManifest.sources) { + const feed = document( + await input.readBlob("catalog", input.pins.transitousCommit, `feeds/${source.region}.json`), + ); + if (!Array.isArray(feed.sources)) throw invalid(); + const matching = feed.sources.map(object).filter((entry) => entry.name === source.name); + if ( + matching.length !== 1 || + matching[0].skip === true || + source.sourceId !== + (matching[0]["openmapx-source-id"] ?? `catalog:${source.region}:${source.name}`) || + source.artifact.relativePath !== `${source.region}_${source.name}.${source.format}.zip` + ) + throw Error("MOTIS source provenance does not match the approved pinned catalog"); + } + const routes = await resolveRoutes(recipe.routes, input.pins, input.readBlob); + return { + recipe, + routes, + credentialRequirements: [ + ...new Set(routes.flatMap((r) => (r.credentialKey ? [r.credentialKey] : []))), + ].sort(), + }; +} +/** Run only after the actual successful candidate manifest has been built. */ +export async function producePortableMotisRecipe(input: { + stagingDir: string; + pins: MotisPortablePins; + readBlob: MotisPinnedBlobReader; + credentials: Readonly>; + signal?: AbortSignal; +}): Promise { + const candidate = await verifyCandidateManifest(input.stagingDir, input.signal); + const root = new AnchoredDirectory(realpathSync(input.stagingDir)); + try { + const read = (path: string) => { + const parts = path.split("/"), + file = parts.pop(), + opened: AnchoredDirectory[] = []; + let dir = root; + if (!file) throw invalid(); + try { + for (const part of parts) { + dir = dir.directory(part); + opened.push(dir); + } + return dir.read(file, MAX_PORTABLE_MOTIS_BYTES, false).toString("utf8"); + } finally { + for (const dir of opened.reverse()) dir.close(); + } + }; + const actualVars = normalizeFeedProxyVars( + JSON.parse(read(`${CANDIDATE_PROXY_DIRNAME}/feed-proxy-vars.json`)), + ); + const config = object(loadYaml(read("config.yml"))); + const used = configRoutes(config, actualVars); + const routes: PortableMotisRecipe["routes"] = []; + // Locate each actual key using the pinned catalog source names, not bundle supplied endpoints. + for (const key of [...used].sort()) { + const found = new Map(); + for (let split = key.indexOf("-"); split > 0; split = key.indexOf("-", split + 1)) { + const region = key.slice(0, split), + suffix = key.slice(split + 1); + for (const sourceName of new Set([suffix, suffix.replace(/-\d+$/, "")])) { + if (!token.safeParse(region).success || !token.safeParse(sourceName).success) continue; + for (const endpoint of [ + "url", + "realtime_trip_updates", + "realtime_vehicle_positions", + "realtime_alerts", + "gbfs_auto_discovery", + ] as const) { + const proposed = { key, region, sourceName, endpoint }; + try { + const resolved = await resolveRoutes([proposed], input.pins, input.readBlob); + const expected = renderVars(resolved, input.credentials); + if (equalVars(expected, { [key]: actualVars[key] })) { + found.set(JSON.stringify(proposed), proposed); + } + } catch { + /* Unsupported selector: try the next exact pinned source identity. */ + } + } + } + } + if (found.size !== 1) throw invalid(); + routes.push([...found.values()][0]); + } + const sourceManifest = document(read("transit-source-manifest.json")); + if (Array.isArray(sourceManifest.sources)) + for (const raw of sourceManifest.sources) delete object(raw).originUrl; + publicDocument(sourceManifest); + publicDocument(JSON.parse(read("license.json"))); + const configText = JSON.stringify(config); + // Expectations can themselves contain source URLs. Derive them again from + // the sanitized template, never retain the source candidate's raw URLs. + candidate.expectations = parseMotisConfigExpectations(configText); + const emptyHash = { sha256: sha(""), sizeBytes: 0 }; + candidate.artifacts.config = { + ...candidate.artifacts.config, + sha256: sha(configText), + sizeBytes: Buffer.byteLength(configText), + }; + candidate.artifacts.proxyConfig = { ...candidate.artifacts.proxyConfig, ...emptyHash }; + candidate.artifacts.proxyVars = { ...candidate.artifacts.proxyVars, ...emptyHash }; + publicDocument(candidate); + for (const dataset of Object.values(object(object(config.timetable).datasets))) { + const script = object(dataset).script; + if ( + typeof script === "string" && + read(script) !== (await input.readBlob("catalog", input.pins.transitousCommit, script)) + ) + throw Error("MOTIS candidate script does not match its approved pin"); + } + const recipe = portableMotisRecipeSchema.parse({ + version: 1, + pins: input.pins, + config: configText, + configSha256: sha(configText), + templateSha256: await validatePinnedTemplate(config, input.pins, input.readBlob), + candidate, + sourceManifest, + ...(candidate.artifacts.sourceIndex + ? { sourceIndex: JSON.parse(read("gbfs-source-index.json")) } + : {}), + routes, + }); + const inspected = await inspectPortableMotisRecipe({ + recipe, + pins: input.pins, + readBlob: input.readBlob, + }); + if ( + !equalVars( + renderVars(inspected.routes, input.credentials), + Object.fromEntries([...used].map((key) => [key, actualVars[key]])), + ) + ) + throw invalid(); + const sourceFingerprints = candidate.artifacts.datasets.map((f) => ({ + id: f.path, + sha256: f.sha256, + sizeBytes: f.sizeBytes, + })); + const metadata = { + version: 1, + sourceFingerprints, + credentialRequirements: inspected.credentialRequirements, + recipe, + }; + const serialized = JSON.stringify(metadata); + if (Buffer.byteLength(serialized) > MAX_PORTABLE_MOTIS_BYTES) throw invalid(); + for (const value of Object.values(input.credentials)) + if (value && serialized.includes(value)) throw invalid(); + input.signal?.throwIfAborted(); + root.write(PORTABLE_MOTIS_FILENAME, `${serialized}\n`); + } finally { + root.close(); + } +} +/** Stage is a private owned extracted candidate. This has no network or lifecycle effects. */ +export async function materializePortableMotisCandidate(input: { + archiveFingerprint: string; + stagingDir: string; + recipe: unknown; + pins: MotisPortablePins; + readBlob: MotisPinnedBlobReader; + credentials: Readonly>; + signal?: AbortSignal; +}): Promise<{ + manifest: MotisCandidateManifest; + materialization: VerifiedPortableMotisMaterialization; +}> { + digest.parse(input.archiveFingerprint); + input.signal?.throwIfAborted(); + const { recipe, routes } = await inspectPortableMotisRecipe(input); + const vars = renderVars(routes, input.credentials), + root = new AnchoredDirectory(realpathSync(input.stagingDir)); + const inode = identity(root.stat); + let metadataSha256: string; + const generated = new Map(); + let manifest: MotisCandidateManifest; + try { + const metadataBytes = root.read(PORTABLE_MOTIS_FILENAME, MAX_PORTABLE_MOTIS_BYTES, false), + retained = object(JSON.parse(metadataBytes.toString("utf8"))); + if (JSON.stringify(portableMotisRecipeSchema.parse(retained.recipe)) !== JSON.stringify(recipe)) + throw Error("MOTIS materialization recipe does not match its retained portable metadata"); + metadataSha256 = sha(metadataBytes); + const a = recipe.candidate.artifacts; + for (const expected of [a.license, ...a.datasets]) { + const actual = await hashArtifactFile(root, expected.path, input.signal); + if (actual.sha256 !== expected.sha256 || actual.sizeBytes !== expected.sizeBytes) + throw Error(`MOTIS portable candidate integrity mismatch: ${expected.path}`); + } + const config = document(recipe.config), + datasets = Object.values(object(object(config.timetable).datasets)).map( + (d) => object(d).path, + ); + if ( + JSON.stringify([...datasets].sort()) !== JSON.stringify(a.datasets.map((d) => d.path).sort()) + ) + throw Error("MOTIS portable configuration does not match the compiled source inventory"); + if (typeof config.osm === "string") await hashArtifactFile(root, config.osm, input.signal); + const varsText = JSON.stringify(vars); + // Conservatively fence render expansion before allocating the nginx output. + if (Buffer.byteLength(varsText) * 3 + routes.length * 2048 > MAX_MOTIS_MATERIALIZATION_BYTES) + throw Error("MOTIS materialization exceeds its reserved byte budget"); + generated.set("config.yml", recipe.config); + generated.set("transit-source-manifest.json", JSON.stringify(recipe.sourceManifest)); + if (recipe.sourceIndex !== undefined) + generated.set("gbfs-source-index.json", JSON.stringify(recipe.sourceIndex)); + generated.set(`${CANDIDATE_PROXY_DIRNAME}/feed-proxy-vars.json`, varsText); + generated.set(`${CANDIDATE_PROXY_DIRNAME}/conf/default.conf`, renderFeedProxyNginxConfig(vars)); + for (const dataset of Object.values(object(object(config.timetable).datasets))) { + const script = object(dataset).script; + if (typeof script === "string") { + const bytes = await input.readBlob("catalog", input.pins.transitousCommit, script); + if (Buffer.byteLength(bytes) > MAX_PORTABLE_MOTIS_BYTES) throw invalid(); + generated.set(script, bytes); + } + } + const hash = (path: string) => { + const bytes = generated.get(path); + if (bytes === undefined) throw invalid(); + return { path, sha256: sha(bytes), sizeBytes: Buffer.byteLength(bytes) }; + }; + manifest = { + ...recipe.candidate, + expectations: parseMotisConfigExpectations(recipe.config), + artifacts: { + ...a, + config: hash("config.yml"), + proxyConfig: hash(`${CANDIDATE_PROXY_DIRNAME}/conf/default.conf`), + proxyVars: hash(`${CANDIDATE_PROXY_DIRNAME}/feed-proxy-vars.json`), + sourceManifest: hash("transit-source-manifest.json"), + ...(recipe.sourceIndex === undefined + ? {} + : { sourceIndex: hash("gbfs-source-index.json") }), + }, + }; + generated.set(CANDIDATE_MANIFEST_FILENAME, JSON.stringify(manifest)); + if ( + [...generated.values()].reduce((total, bytes) => total + Buffer.byteLength(bytes), 0) > + MAX_MOTIS_MATERIALIZATION_BYTES + ) + throw Error("MOTIS materialization exceeds its reserved byte budget"); + // No writes occur until every credential, pinned script, source integrity + // check and the complete fixed reservation has passed. + for (const [path, bytes] of generated) { + input.signal?.throwIfAborted(); + const parts = path.split("/"), + file = parts.pop(); + if (!file) throw invalid(); + const opened: AnchoredDirectory[] = []; + let directory = root; + try { + for (const part of parts) { + directory = directory.directory(part, true); + opened.push(directory); + } + directory.write(file, bytes); + } finally { + for (const directory of opened.reverse()) directory.close(); + } + } + } finally { + root.close(); + } + manifest = await verifyCandidateManifest(input.stagingDir, input.signal); + const proofRoot = new AnchoredDirectory(realpathSync(input.stagingDir)), + generatedArtifacts = []; + try { + if (!sameInode(inode, identity(proofRoot.stat))) throw invalid(); + for (const [path, bytes] of generated) { + const file = await hashArtifactFile(proofRoot, path, input.signal); + if (file.sha256 !== sha(bytes) || file.sizeBytes !== Buffer.byteLength(bytes)) + throw Error("MOTIS generated artifact changed before materialization proof"); + generatedArtifacts.push(file); + } + } finally { + proofRoot.close(); + } + const materialization = Object.freeze({}) as VerifiedPortableMotisMaterialization; + materializations.set(materialization, { + inode, + metadataSha256, + archiveFingerprint: input.archiveFingerprint, + generatedArtifacts, + manifestDigest: sha(JSON.stringify(manifest)), + sourceFingerprints: recipe.candidate.artifacts.datasets.map((file) => ({ + id: file.path, + sha256: file.sha256, + sizeBytes: file.sizeBytes, + })), + }); + return { manifest, materialization }; +} + +declare const materializationBrand: unique symbol; +export type VerifiedPortableMotisMaterialization = Readonly<{ [materializationBrand]: true }>; +const materializations = new WeakMap< + object, + { + inode: FileIdentity; + metadataSha256: string; + archiveFingerprint: string; + generatedArtifacts: Array<{ path: string; sha256: string; sizeBytes: number }>; + manifestDigest: string; + sourceFingerprints: Array<{ id: string; sha256: string; sizeBytes: number }>; + } +>(); +/** Only a successful materializer may authorize its exact generated files. */ +export async function readVerifiedPortableMotisMaterialization( + handle: VerifiedPortableMotisMaterialization, + input: { + stagingDir: string; + archiveFingerprint: string; + portableMetadataSha256: string; + signal?: AbortSignal; + }, +) { + const state = materializations.get(handle); + if ( + !state || + state.archiveFingerprint !== input.archiveFingerprint || + state.metadataSha256 !== input.portableMetadataSha256 + ) + throw Error("Verified MOTIS materialization does not match its archive"); + const root = new AnchoredDirectory(realpathSync(input.stagingDir)); + try { + if ( + !sameInode(state.inode, identity(root.stat)) || + sha(root.read(PORTABLE_MOTIS_FILENAME, MAX_PORTABLE_MOTIS_BYTES, false)) !== + state.metadataSha256 + ) + throw Error("Verified MOTIS materialization directory or source metadata changed"); + const manifest = await verifyCandidateManifest(input.stagingDir, input.signal); + if (sha(JSON.stringify(manifest)) !== state.manifestDigest) + throw Error("Verified MOTIS materialization manifest changed"); + const generatedArtifacts = []; + for (const expected of state.generatedArtifacts) { + const actual = await hashArtifactFile(root, expected.path, input.signal); + if (actual.sha256 !== expected.sha256 || actual.sizeBytes !== expected.sizeBytes) + throw Error("Verified MOTIS generated artifact changed"); + generatedArtifacts.push(actual); + } + return { + manifest, + generatedArtifacts, + sourceFingerprints: structuredClone(state.sourceFingerprints), + archiveFingerprint: state.archiveFingerprint, + }; + } finally { + root.close(); + } +} + +/** Native Git is used only as a bounded object decoder; this never fetches or checks out. */ +export function createMotisPinnedBlobReader( + catalogDir: string, + pins: MotisPortablePins, + signal?: AbortSignal, +): MotisPinnedBlobReader { + const root = realpathSync(catalogDir), + atlas = join(root, "transitland-atlas"); + for (const path of [catalogDir, atlas]) { + const stat = lstatSync(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw invalid(); + } + const execute = promisify(execFile), + cache = new Map>(); + let bytes = 0, + files = 0; + const git = async (repository: string, args: string[]) => { + signal?.throwIfAborted(); + try { + const result = await execute( + "git", + [ + "--no-replace-objects", + "--no-optional-locks", + "-c", + "core.hooksPath=/dev/null", + "-c", + "protocol.allow=never", + "-C", + repository, + ...args, + ], + { + signal, + timeout: 30000, + maxBuffer: MAX_PORTABLE_MOTIS_BYTES, + encoding: "utf8", + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + LANG: "C", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_NO_LAZY_FETCH: "1", + GIT_TERMINAL_PROMPT: "0", + }, + }, + ); + return result.stdout; + } catch { + signal?.throwIfAborted(); + throw invalid(); + } + }; + let verified: Promise | undefined; + const verify = () => + (verified ??= git(root, ["ls-tree", pins.transitousCommit, "--", "transitland-atlas"]).then( + (line) => { + if (line.trim() !== `160000 commit ${pins.atlasCommit}\ttransitland-atlas`) throw invalid(); + }, + )); + return async (repository, commit, path) => { + const expected = repository === "catalog" ? pins.transitousCommit : pins.atlasCommit; + const selector = /^feeds\/\.openmapx-atlas-id\/([a-zA-Z0-9][a-zA-Z0-9_.-]*)$/.exec(path); + if (repository === "atlas" && selector) { + if (commit !== expected || !/^[a-f0-9]{40}$/.test(commit)) throw invalid(); + await verify(); + const key = `atlas-id:${selector[1]}`; + let pending = cache.get(key); + if (!pending) { + pending = (async () => { + const names = ( + await git(atlas, [ + "grep", + "--no-textconv", + "-l", + "-F", + "-e", + JSON.stringify(selector[1]), + commit, + "--", + "feeds", + ]) + ) + .trim() + .split("\n"); + if (names.length > 128) throw invalid(); + const matches: Record[] = []; + for (const name of names) { + if (!name.startsWith(`${commit}:`)) throw invalid(); + const path = name.slice(commit.length + 1); + if (!/^feeds\/[a-zA-Z0-9][a-zA-Z0-9_.-]*\.dmfr\.json$/.test(path)) throw invalid(); + if (++files > 10000) throw invalid(); + const text = await git(atlas, [ + "show", + "--no-ext-diff", + "--no-textconv", + `${commit}:${path}`, + ]); + bytes += Buffer.byteLength(text); + if (bytes > 32 * MAX_PORTABLE_MOTIS_BYTES) throw invalid(); + const parsed = document(text); + if (!Array.isArray(parsed.feeds)) throw invalid(); + matches.push(...parsed.feeds.map(object).filter((feed) => feed.id === selector[1])); + } + if (matches.length !== 1) throw invalid(); + return JSON.stringify({ feeds: matches }); + })(); + cache.set(key, pending); + } + return pending; + } + if ( + commit !== expected || + !/^[a-f0-9]{40}$/.test(commit) || + !( + repository === "catalog" + ? /^(?:feeds\/[a-zA-Z0-9][a-zA-Z0-9_.-]*\.json|configs\/motis\/config\.yml|scripts\/[a-zA-Z0-9][a-zA-Z0-9_.-]*\.lua)$/ + : /^feeds\/[a-zA-Z0-9][a-zA-Z0-9_.-]*\.dmfr\.json$/ + ).test(path) + ) + throw invalid(); + await verify(); + const key = `${repository}:${path}`; + let pending = cache.get(key); + if (!pending) { + if (++files > 10000) throw invalid(); + pending = git(repository === "catalog" ? root : atlas, [ + "show", + "--no-ext-diff", + "--no-textconv", + `${commit}:${path}`, + ]).then((text) => { + bytes += Buffer.byteLength(text); + if (bytes > 32 * MAX_PORTABLE_MOTIS_BYTES) throw invalid(); + return text; + }); + cache.set(key, pending); + } + return pending; + }; +} diff --git a/packages/core/src/build-artifacts/motis-publication.ts b/packages/core/src/build-artifacts/motis-publication.ts new file mode 100644 index 000000000..3e652dd33 --- /dev/null +++ b/packages/core/src/build-artifacts/motis-publication.ts @@ -0,0 +1,228 @@ +/** Fixed offline MOTIS paths and private materialization projection. No authority minting. */ +import { createHash } from "node:crypto"; +import { lstatSync, readlinkSync, realpathSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import type { ValidatedBuildBundle } from "./archive"; +import { artifactFingerprint } from "./artifact-path"; +import { AnchoredDirectory, identity, sameInode, withDirectories } from "./filesystem"; +import type { BundleFile } from "./manifest"; +import { parseBundleManifest } from "./manifest"; +import type { MotisCandidateManifest } from "./motis-candidate"; +import { + MAX_MOTIS_MATERIALIZATION_BYTES, + readVerifiedPortableMotisMaterialization, + type VerifiedPortableMotisMaterialization, +} from "./motis-portable"; + +const inode = z.strictObject({ dev: z.string(), ino: z.string() }); +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const slotStateSchema = z.strictObject({ + schemaVersion: z.literal(1), + activeSlot: z.enum(["A", "B"]), + previousHealthySlot: z.enum(["A", "B"]).optional(), + datasetEpoch: z.string().max(512).optional(), + manifestHash: z.string().max(128).optional(), + imageDigest: z.string().max(4096).optional(), + activatedAt: z.string().max(128).optional(), + retainedRollback: z + .strictObject({ + proposalId: z.uuid(), + validationId: z.uuid(), + slot: z.enum(["A", "B"]), + epoch: z.string().max(512), + manifestHash: digest, + filesDigest: digest, + }) + .optional(), +}); +export const motisPublicationLayoutSchema = z.strictObject({ + physicalLive: z.enum(["motis/live", "motis/slots/A", "motis/slots/B"]), + aliases: z + .array( + z.strictObject({ + name: z.enum(["live", "staging"]), + target: z.enum(["slots/A", "slots/B"]), + inode, + }), + ) + .max(2), + slotState: z.strictObject({ inode, digest, record: slotStateSchema }).nullable(), +}); +export type MotisPublicationLayout = z.infer; +export function readMotisPublicationLayout(root: AnchoredDirectory): MotisPublicationLayout { + let motis: AnchoredDirectory; + try { + motis = root.directory("motis", false, false); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return { physicalLive: "motis/live", aliases: [], slotState: null }; + throw error; + } + try { + const aliases: MotisPublicationLayout["aliases"] = []; + for (const name of ["live", "staging"] as const) { + let stat: import("node:fs").BigIntStats; + try { + stat = lstatSync(motis.name(name), { bigint: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + if (!stat.isSymbolicLink()) { + if (!stat.isDirectory()) throw Error("MOTIS alias type changed"); + continue; + } + const target = readlinkSync(motis.name(name)); + if ( + !["slots/A", "slots/B"].includes(target) || + !sameInode(identity(stat), identity(lstatSync(motis.name(name), { bigint: true }))) + ) + throw Error("MOTIS alias changed"); + aliases.push({ name, target: target as "slots/A" | "slots/B", inode: identity(stat) }); + } + let slotState: MotisPublicationLayout["slotState"] = null; + try { + const stat = motis.fileIdentity("slot-state.json", false), + bytes = motis.read("slot-state.json", 65536, false); + slotState = { + inode: stat, + digest: createHash("sha256").update(bytes).digest("hex"), + record: slotStateSchema.parse(JSON.parse(bytes.toString("utf8"))), + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (!aliases.length) { + if (slotState) throw Error("MOTIS slot state has no matching aliases"); + return { physicalLive: "motis/live", aliases, slotState }; + } + const live = aliases.find((alias) => alias.name === "live"), + staging = aliases.find((alias) => alias.name === "staging"); + if ( + aliases.length !== 2 || + !live || + !staging || + live.target === staging.target || + !slotState || + live.target !== `slots/${slotState.record.activeSlot}` + ) + throw Error("MOTIS slot aliases and state disagree"); + return { + physicalLive: live.target === "slots/A" ? "motis/slots/A" : "motis/slots/B", + aliases, + slotState, + }; + } finally { + motis.close(); + } +} +export function assertMotisPublicationAliases( + root: AnchoredDirectory, + expected: MotisPublicationLayout, +): void { + if (!expected.aliases.length) { + const current = readMotisPublicationLayout(root); + if (current.aliases.length || current.slotState) throw Error("Unexpected MOTIS slot layout"); + return; + } + const motis = root.directory("motis", false, false); + try { + for (const alias of expected.aliases) { + const stat = lstatSync(motis.name(alias.name), { bigint: true }); + if ( + !stat.isSymbolicLink() || + !sameInode(alias.inode, identity(stat)) || + readlinkSync(motis.name(alias.name)) !== alias.target + ) + throw Error("Original MOTIS alias changed"); + } + } finally { + motis.close(); + } +} +export async function materializedMotisBundle( + bundle: ValidatedBuildBundle, + stagingRoot: string, + proof: VerifiedPortableMotisMaterialization, + signal?: AbortSignal, +): Promise { + const portable = parseBundleManifest(JSON.stringify(bundle.manifest)); + if (portable.service !== "motis" || portable.schemaVersion !== 2) + throw Error("Portable MOTIS source required"); + const metadata = portable.files.find((file) => file.path === "motis/live/portable-sources.json"); + if (!metadata) throw Error("Portable MOTIS metadata missing"); + const generated = await readVerifiedPortableMotisMaterialization(proof, { + stagingDir: join(stagingRoot, "motis/live"), + archiveFingerprint: bundle.artifactFingerprint, + portableMetadataSha256: metadata.sha256, + signal, + }); + const files = new Map(portable.files.map((file) => [file.path, file])); + for (const file of generated.generatedArtifacts) + files.set(`motis/live/${file.path}`, { ...file, path: `motis/live/${file.path}` }); + const root = new AnchoredDirectory(realpathSync(stagingRoot)); + try { + // The consumer root is a separate copy. Candidate verification continues to + // own its own .openmapx-feed-proxy files inside the live candidate. + for (const name of ["feed-proxy-vars.json", "conf/default.conf"]) { + const sourcePath = `.openmapx-feed-proxy/${name}`, + expected = generated.generatedArtifacts.find((file) => file.path === sourcePath); + if (!expected) throw Error("Materialized MOTIS proxy proof missing"); + const bytes = withDirectories( + root, + ["motis", "live", ".openmapx-feed-proxy", ...(name.startsWith("conf/") ? ["conf"] : [])], + false, + (dir) => dir.read(name.split("/").at(-1)!, MAX_MOTIS_MATERIALIZATION_BYTES, false), + ); + if ( + bytes.length !== expected.sizeBytes || + createHash("sha256").update(bytes).digest("hex") !== expected.sha256 + ) + throw Error("Materialized MOTIS proxy changed"); + withDirectories( + root, + ["motis-feed-proxy", ...(name.startsWith("conf/") ? ["conf"] : [])], + true, + (dir) => dir.write(name.split("/").at(-1)!, bytes), + ); + files.set(`motis-feed-proxy/${name}`, { + path: `motis-feed-proxy/${name}`, + sha256: expected.sha256, + sizeBytes: expected.sizeBytes, + }); + } + } finally { + root.close(); + } + const directories = new Set(portable.directories); + for (const file of files.values()) { + const parts = file.path.split("/"); + parts.pop(); + while (parts.length) { + const path = parts.join("/"); + if (portable.dirs.some((dir) => path === dir || path.startsWith(`${dir}/`))) + directories.add(path); + parts.pop(); + } + } + const inventory = [...files.values()].sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0, + ), + fingerprint = artifactFingerprint(inventory); + // This projection is deliberately private: public tar parsing still rejects + // credential-bearing config/proxy files. Only the nominal producer supplied + // this exact generated inventory and its verified source bytes. + return { + ...bundle, + motisCandidate: generated.manifest, + manifest: { + ...portable, + files: inventory, + directories: [...directories].sort(), + artifactFingerprint: fingerprint, + }, + artifactFingerprint: fingerprint, + expandedBytes: inventory.reduce((sum, file) => sum + file.sizeBytes, 0), + }; +} diff --git a/packages/core/src/build-artifacts/publication.ts b/packages/core/src/build-artifacts/publication.ts index 5e7a88574..1ba5db6f7 100644 --- a/packages/core/src/build-artifacts/publication.ts +++ b/packages/core/src/build-artifacts/publication.ts @@ -4,23 +4,35 @@ import { createHash } from "node:crypto"; import { fsyncSync, lstatSync, mkdirSync, renameSync, rmdirSync, unlinkSync } from "node:fs"; import { dirname, relative, resolve } from "node:path"; import { z } from "zod"; +import { transferStageSchema } from "../ops/artifact-transfer"; import { canonicalJson } from "../ops/canonical-json"; +import { type OpsJobStatus, opsOperationSchema } from "../ops/contract"; +import { controlTerminalReceiptDigest } from "../ops/control-retention-receipt"; +import { opsOperationFingerprint } from "../ops/fingerprint"; import { currentConfigurationGeneration, withConfigurationGenerationWriter, } from "../services/configuration-generation"; import type { ValidatedBuildBundle } from "./archive"; +import { inspectControlChildPublication } from "./child-phase"; import { beginControlEffect, + cancelControlExecution, + childSettlementState, controlDigest, executionAuthorityState, readPrivateRecord, + retainControlEffectReceipt, + retainControlTerminalCommitment, + retainControlTerminalStatus, type VerifiedExecutionAuthority, + verifyControlEffectSettlement, writePrivateRecord, } from "./control-authority"; import { signControl, verifyControl } from "./control-records"; import { hashArtifactFile, inspectArtifactTree } from "./copy-tree"; import { AnchoredDirectory, type FileIdentity, identity, sameInode } from "./filesystem"; +import { inspectOwnedGenerationHardlinks } from "./generation-hardlinks-owner"; import { BUILD_RECORD_DIR, buildArtifactSpec, @@ -28,8 +40,16 @@ import { parseBundleManifest, safeArtifactPath, } from "./manifest"; +import type { MotisCandidateManifest } from "./motis-candidate"; +import type { VerifiedPortableMotisMaterialization } from "./motis-portable"; +import { + assertMotisPublicationAliases, + materializedMotisBundle, + motisPublicationLayoutSchema, + readMotisPublicationLayout, +} from "./motis-publication"; import { capturePreparedBuildEvidence, incompletePreparedLayout } from "./prepared-evidence"; -import { verifyFamilyOwnedRecord } from "./retention-family"; +import { publishFamilyCanonicalTerminal, verifyFamilyOwnedRecord } from "./retention-family"; import { assertArtifactTargetLeaseExecution, assertArtifactTargetLeaseReceiptOwner, @@ -48,6 +68,10 @@ export interface ArtifactPublicationInput { authority: VerifiedExecutionAuthority; lease: TargetLeaseAuthority; expectedFingerprint: string; + materialization?: VerifiedPortableMotisMaterialization; + /** Fixed destination producer runs inside this one configuration writer. Its + * nominal output is reverified; a callback cannot supply generated-file authority. */ + materializeMotis?: () => Promise; signal?: AbortSignal; /** Fixed local consumer's denial-only compatibility fence, acquired after * payload hashing while this publisher owns the configuration writer. @@ -59,6 +83,7 @@ const inode = z.strictObject({ dev: z.string(), ino: z.string() }), digest = z.string().regex(/^[a-f0-9]{64}$/); const directoryEntry = z.strictObject({ path: z.string(), + targetPath: z.string().optional(), old: inode.nullable(), next: inode, oldMetadata: digest.nullable(), @@ -67,6 +92,7 @@ const directoryEntry = z.strictObject({ }); const recordEntry = z.strictObject({ name: z.string(), + scope: z.enum(["build", "motis"]).optional(), old: inode.nullable(), next: inode.nullable(), digest: digest.nullable(), @@ -81,6 +107,8 @@ const journalSchema = z.strictObject({ serviceId: z.string(), archiveSha256: digest, artifactFingerprint: digest, + portableArtifactFingerprint: digest.optional(), + motisLayout: motisPublicationLayoutSchema.optional(), expectedFingerprint: digest, stagingRelative: z.string(), stagingIdentity: inode, @@ -88,7 +116,7 @@ const journalSchema = z.strictObject({ recordsIdentity: inode, phase: z.enum(["prepared", "committed", "restored"]), directories: z.array(directoryEntry).max(8), - records: z.array(recordEntry).length(2), + records: z.array(recordEntry).min(2).max(3), cleanupFailed: z.boolean(), }); type Journal = z.infer; @@ -129,8 +157,9 @@ function parent(root: AnchoredDirectory, path: string, create = false) { return { ...directoryAt(root, parts.join("/"), create), name }; } function dirIdentity(root: AnchoredDirectory, path: string): FileIdentity | null { - const p = parent(root, path); + let p: ReturnType | undefined; try { + p = parent(root, path); const stat = lstatSync(p.dir.name(p.name), { bigint: true }); if (!stat.isDirectory()) throw Error("Artifact directory alias refused"); const d = p.dir.directory(p.name, false, false); @@ -143,7 +172,7 @@ function dirIdentity(root: AnchoredDirectory, path: string): FileIdentity | null if (missing(error)) return null; throw error; } finally { - p.close(); + p?.close(); } } function directoryMetadata(root: AnchoredDirectory, path: string) { @@ -167,13 +196,15 @@ function equal(a: FileIdentity | null, b: FileIdentity | null) { } function snapshot(root: AnchoredDirectory, serviceId: string) { const spec = buildArtifactSpec(serviceId); - if (serviceId === "motis") throw Error("MOTIS requires the destination materialization adapter"); + const motisLayout = serviceId === "motis" ? readMotisPublicationLayout(root) : undefined; const directories = spec.dirs.map((path) => { - const id = dirIdentity(root, path); - if (!id) return { path, id: null, metadata: null }; - const d = directoryAt(root, path); + const targetPath = path === "motis/live" && motisLayout ? motisLayout.physicalLive : path; + const id = dirIdentity(root, targetPath); + const entry = { path, ...(targetPath === path ? {} : { targetPath }) }; + if (!id) return { ...entry, id: null, metadata: null }; + const d = directoryAt(root, targetPath); try { - return { path, id, metadata: inspectArtifactTree(d.dir, TREE_LIMITS).metadataDigest }; + return { ...entry, id, metadata: inspectArtifactTree(d.dir, TREE_LIMITS).metadataDigest }; } finally { d.close(); } @@ -185,18 +216,28 @@ function snapshot(root: AnchoredDirectory, serviceId: string) { if (!missing(error)) throw error; } try { - return { - directories, - records: [`${serviceId}.json`, `${serviceId}.evidence.json`].map((name) => { - const id = records ? fileIdentity(records, name) : null; - return { - name, - id, - digest: - id && records ? hash(records.read(name, MAX_METADATA, false).toString("utf8")) : null, - }; - }), - }; + const snapshots: Array<{ + name: string; + id: FileIdentity | null; + digest: string | null; + scope?: "motis"; + }> = [`${serviceId}.json`, `${serviceId}.evidence.json`].map((name) => { + const id = records ? fileIdentity(records, name) : null; + return { + name, + id, + digest: + id && records ? hash(records.read(name, MAX_METADATA, false).toString("utf8")) : null, + }; + }); + if (motisLayout?.slotState) + snapshots.push({ + name: "slot-state.json", + scope: "motis", + id: motisLayout.slotState.inode, + digest: motisLayout.slotState.digest, + }); + return { directories, records: snapshots, ...(motisLayout ? { motisLayout } : {}) }; } finally { records?.close(); } @@ -360,13 +401,49 @@ function validateJournal( journal.stageId !== input.stageId || journal.serviceId !== input.bundle.manifest.service || journal.archiveSha256 !== input.bundle.archiveSha256 || - journal.artifactFingerprint !== input.bundle.artifactFingerprint || + (journal.portableArtifactFingerprint ?? journal.artifactFingerprint) !== + input.bundle.artifactFingerprint || canonicalJson(journal.directories.map((d) => d.path)) !== canonicalJson(spec.dirs) || canonicalJson(journal.records.map((r) => r.name)) !== - canonicalJson([`${journal.serviceId}.json`, `${journal.serviceId}.evidence.json`]) + canonicalJson([ + `${journal.serviceId}.json`, + `${journal.serviceId}.evidence.json`, + ...(journal.motisLayout?.slotState ? ["slot-state.json"] : []), + ]) || + journal.records.some((entry, index) => + index < 2 ? entry.scope === "motis" : entry.scope !== "motis", + ) || + journal.directories.some( + (entry) => + (entry.targetPath ?? entry.path) !== + (entry.path === "motis/live" && journal.motisLayout + ? journal.motisLayout.physicalLive + : entry.path), + ) || + (journal.serviceId === "motis" + ? !journal.motisLayout || !journal.portableArtifactFingerprint + : journal.motisLayout !== undefined || journal.portableArtifactFingerprint !== undefined) ) throw Error("Artifact publication journal identity changed"); } +function targetPath(entry: Journal["directories"][number]) { + return entry.targetPath ?? entry.path; +} +function withRecord( + root: AnchoredDirectory, + records: AnchoredDirectory, + entry: Journal["records"][number], + run: (directory: AnchoredDirectory) => T, +): T { + if (entry.scope !== "motis") return run(records); + if (entry.name !== "slot-state.json") throw Error("Invalid MOTIS metadata record"); + const motis = root.directory("motis", false, false); + try { + return run(motis); + } finally { + motis.close(); + } +} function restore( root: AnchoredDirectory, source: AnchoredDirectory, @@ -374,9 +451,10 @@ function restore( records: AnchoredDirectory, journal: Journal, ) { + if (journal.motisLayout) assertMotisPublicationAliases(root, journal.motisLayout); // Validate the WHOLE recorded set before restoring any member. for (const entry of journal.directories) { - const live = dirIdentity(root, entry.path), + const live = dirIdentity(root, targetPath(entry)), next = dirIdentity(source, entry.path), old = dirIdentity(backups, entry.backup); if ( @@ -389,7 +467,7 @@ function restore( throw Error("Artifact restoration identity is uncertain"); } for (const entry of journal.records) { - const live = fileIdentity(records, entry.name), + const live = withRecord(root, records, entry, (home) => fileIdentity(home, entry.name)), old = fileIdentity(backups, entry.backup); if ( (live && !equal(live, entry.old) && !equal(live, entry.next)) || @@ -400,8 +478,8 @@ function restore( } for (const entry of journal.directories) { const old = entry.old - ? equal(dirIdentity(root, entry.path), entry.old) - ? { root, path: entry.path } + ? equal(dirIdentity(root, targetPath(entry)), entry.old) + ? { root, path: targetPath(entry) } : { root: backups, path: entry.backup } : null; if (old && directoryMetadata(old.root, old.path) !== entry.oldMetadata) @@ -410,26 +488,39 @@ function restore( for (const entry of journal.records) { const old = entry.old && - (equal(fileIdentity(records, entry.name), entry.old) - ? { root: records, name: entry.name } + (equal( + withRecord(root, records, entry, (home) => fileIdentity(home, entry.name)), + entry.old, + ) + ? { + root: entry.scope === "motis" ? root : records, + name: entry.scope === "motis" ? "motis/slot-state.json" : entry.name, + } : { root: backups, name: entry.backup }); if ( old && - hash(old.root.read(old.name, MAX_METADATA, false).toString("utf8")) !== entry.oldDigest + hash( + (old.root === root && entry.scope === "motis" + ? withRecord(root, records, entry, (home) => home.read(entry.name, MAX_METADATA, false)) + : old.root.read(old.name, MAX_METADATA, false) + ).toString("utf8"), + ) !== entry.oldDigest ) throw Error("Artifact record restoration content is uncertain"); } for (const entry of [...journal.directories].reverse()) { - if (equal(dirIdentity(root, entry.path), entry.next)) - moveDirectory(root, entry.path, source, entry.path, entry.next); + if (equal(dirIdentity(root, targetPath(entry)), entry.next)) + moveDirectory(root, targetPath(entry), source, entry.path, entry.next); if (entry.old && equal(dirIdentity(backups, entry.backup), entry.old)) - moveDirectory(backups, entry.backup, root, entry.path, entry.old); + moveDirectory(backups, entry.backup, root, targetPath(entry), entry.old); } for (const entry of journal.records) { - if (entry.next && equal(fileIdentity(records, entry.name), entry.next)) - records.removeFile(entry.name); - if (entry.old && equal(fileIdentity(backups, entry.backup), entry.old)) - moved(backups, entry.backup, records, entry.name, entry.old, false); + withRecord(root, records, entry, (home) => { + if (entry.next && equal(fileIdentity(home, entry.name), entry.next)) + home.removeFile(entry.name); + if (entry.old && equal(fileIdentity(backups, entry.backup), entry.old)) + moved(backups, entry.backup, home, entry.name, entry.old, false); + }); } } function cleanup(backups: AnchoredDirectory, journal: Journal) { @@ -520,7 +611,24 @@ export async function publishValidatedBuild( assertArtifactTargetLeaseExecution(input.lease, owner.execution); if (controlDigest(snapshot(root, serviceId)) !== input.expectedFingerprint) return { outcome: "not_committed", reason: "conflict", cleanupFailed: false }; - const manifest = parseBundleManifest(JSON.stringify(input.bundle.manifest)); + let bundle = input.bundle, + motisCandidate: MotisCandidateManifest | undefined; + if (serviceId === "motis") { + const materialization = input.materialization ?? (await input.materializeMotis?.()); + if (!materialization) throw Error("Original destination MOTIS materialization required"); + const materialized = await materializedMotisBundle( + input.bundle, + input.stagingRoot, + materialization, + input.signal, + ); + bundle = materialized; + motisCandidate = materialized.motisCandidate; + } + const manifest = + serviceId === "motis" + ? bundle.manifest + : parseBundleManifest(JSON.stringify(bundle.manifest)); if ( manifest.files.length + manifest.directories.length > 10000 || incompletePreparedLayout(serviceId, manifest.files) @@ -589,7 +697,13 @@ export async function publishValidatedBuild( stageId: input.stageId, serviceId, archiveSha256: input.bundle.archiveSha256, - artifactFingerprint: input.bundle.artifactFingerprint, + artifactFingerprint: bundle.artifactFingerprint, + ...(initial.motisLayout + ? { + motisLayout: initial.motisLayout, + portableArtifactFingerprint: input.bundle.artifactFingerprint, + } + : {}), expectedFingerprint: input.expectedFingerprint, stagingRelative, stagingIdentity: identity(source.dir.stat), @@ -598,6 +712,7 @@ export async function publishValidatedBuild( phase: "prepared", directories: initial.directories.map((dir, index) => ({ path: dir.path, + ...(dir.targetPath ? { targetPath: dir.targetPath } : {}), old: dir.id, oldMetadata: dir.metadata, nextMetadata: sourceMetadata.get(dir.path)!, @@ -606,6 +721,7 @@ export async function publishValidatedBuild( })), records: initial.records.map((record, index) => ({ name: record.name, + ...(record.scope ? { scope: record.scope } : {}), old: record.id, oldDigest: record.digest, next: null, @@ -625,11 +741,16 @@ export async function publishValidatedBuild( for (const entry of journal.directories) { input.signal?.throwIfAborted(); assertArtifactTargetLeaseExecution(input.lease, owner.execution); - if (entry.old) moveDirectory(root, entry.path, backups, entry.backup, entry.old); - moveDirectory(source.dir, entry.path, root, entry.path, entry.next); + if (journal.motisLayout) assertMotisPublicationAliases(root, journal.motisLayout); + if (entry.old) moveDirectory(root, targetPath(entry), backups, entry.backup, entry.old); + moveDirectory(source.dir, entry.path, root, targetPath(entry), entry.next); } + const originalRecords = backups; for (const entry of journal.records) - if (entry.old) moved(records, entry.name, backups, entry.backup, entry.old, false); + if (entry.old) + withRecord(root, records, entry, (home) => + moved(home, entry.name, originalRecords, entry.backup, entry.old!, false), + ); const { dirs: _dirs, directories: _directories, files: _files, ...record } = manifest; const pin = (name: string, inode: FileIdentity, bytes: string) => { const entry = journal!.records.find((entry) => entry.name === name); @@ -645,11 +766,11 @@ export async function publishValidatedBuild( signal: input.signal, beforePublish: (inode, bytes) => pin(`${serviceId}.evidence.json`, inode, bytes), }); - if (next.artifactFingerprint !== input.bundle.artifactFingerprint) + if (next.artifactFingerprint !== bundle.artifactFingerprint) throw Error("Published artifact content differs from validation"); if ( journal.directories.some( - (entry) => directoryMetadata(root, entry.path) !== entry.nextMetadata, + (entry) => directoryMetadata(root, targetPath(entry)) !== entry.nextMetadata, ) ) throw Error("Imported content changed before commit"); @@ -657,6 +778,36 @@ export async function publishValidatedBuild( records.write(`${serviceId}.json`, bytes, false, (inode) => pin(`${serviceId}.json`, inode, bytes), ); + if (journal.motisLayout?.slotState && motisCandidate) { + const original = journal.motisLayout.slotState.record; + const manifestHash = manifest.files.find( + (file) => file.path === "motis/live/motis-candidate-manifest.json", + )?.sha256; + if (!manifestHash) throw Error("Materialized MOTIS manifest proof missing"); + const imageDigest = manifest.runtimeImages.motis?.match(/@(sha256:[a-f0-9]{64})$/)?.[1]; + const nextState = { + schemaVersion: 1, + activeSlot: original.activeSlot, + ...(original.previousHealthySlot && original.previousHealthySlot !== original.activeSlot + ? { previousHealthySlot: original.previousHealthySlot } + : {}), + ...(original.retainedRollback && original.retainedRollback.slot !== original.activeSlot + ? { retainedRollback: original.retainedRollback } + : {}), + datasetEpoch: motisCandidate.epoch, + manifestHash, + ...(imageDigest ? { imageDigest } : {}), + }; + const entry = journal.records.find((record) => record.scope === "motis"); + if (!entry) throw Error("Original MOTIS slot metadata reservation missing"); + const stateBytes = JSON.stringify(nextState); + withRecord(root, records, entry, (home) => + home.write("slot-state.json", stateBytes, false, (inode) => + pin("slot-state.json", inode, stateBytes), + ), + ); + } + if (journal.motisLayout) assertMotisPublicationAliases(root, journal.motisLayout); journal.phase = "committed"; writeJournal(owner, journal); } catch (error) { @@ -707,6 +858,7 @@ export function inspectArtifactPublication(authority: VerifiedExecutionAuthority archiveSha256: journal.archiveSha256, serviceId: journal.serviceId, artifactFingerprint: journal.artifactFingerprint, + cleanupFailed: journal.cleanupFailed, }; } @@ -751,3 +903,251 @@ export function cleanupRestoredArtifactPublication(authority: VerifiedExecutionA root.close(); } } + +/** Close only this purpose's proved historical publication. The original + * canonical status/digest/time remain unchanged even if delivery failed after + * files committed; containment is a separate, source-verified family fact. */ +export function inspectSettledArtifactImport( + authority: VerifiedExecutionAuthority, + status: OpsJobStatus, + terminalAt: number, +) { + const owner = executionAuthorityState(authority); + if (owner.kind !== "artifactTransfer.import" || !owner.origin) + throw Error("Original import required"); + const journal = readJournal({ ...owner, origin: owner.origin }); + if (!journal || !["committed", "restored"].includes(journal.phase)) + throw Error("Original settled import required"); + verifyRetainedArtifactTargetRootRelease(authority); + const effect = childSettlementState( + verifyControlEffectSettlement(owner.installation.installation, owner.execution), + ).evidence; + if ( + journal.phase === "committed" + ? effect.outcome !== "committed" || effect.publication !== "committed" + : effect.outcome === "unknown" || effect.publication !== "not_started" + ) + throw Error("Original settled effect required"); + const followUp = + inspectOwnedGenerationHardlinks(authority) === "complete" + ? ("complete" as const) + : ("failed" as const); + retainControlTerminalCommitment(authority, status, terminalAt); + retainControlTerminalStatus(owner.installation.installation, owner.execution, status, terminalAt); + publishFamilyCanonicalTerminal( + owner.installation, + owner.origin, + owner.execution, + true, + controlTerminalReceiptDigest(status), + terminalAt, + ); + if (journal.phase === "restored") + return { + outcome: "not_committed" as const, + reason: "failed" as const, + cleanupFailed: journal.cleanupFailed, + }; + return { + outcome: "committed" as const, + serviceId: journal.serviceId, + artifactFingerprint: journal.artifactFingerprint, + cleanupFailed: journal.cleanupFailed, + followUp, + }; +} + +/** A transfer claim may be released only after this exact import never began + * publication and the original whole target tree is physically released. */ +export function inspectUnstartedArtifactImport(authority: VerifiedExecutionAuthority) { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "artifactTransfer.import" || + !owner.origin || + readJournal({ ...owner, origin: owner.origin }) + ) + throw Error("Unstarted original import required"); + const started = readPrivateRecord( + owner.installation, + "effect-started", + `${owner.execution.operationKey}.json`, + ); + if (started !== null) throw Error("Import publication began"); + verifyFamilyOwnedRecord( + owner.installation, + owner.origin, + { + scope: "private", + namespace: "effect-started", + container: "", + name: `${owner.execution.operationKey}.json`, + }, + canonicalJson(started), + ); + const effect = childSettlementState( + verifyControlEffectSettlement(owner.installation.installation, owner.execution), + ).evidence; + if ( + !["failed_without_effect", "contained"].includes(effect.outcome) || + effect.publication !== "not_started" + ) + throw Error("Unstarted import containment required"); + verifyRetainedArtifactTargetRootRelease(authority); + const admitted = readPrivateRecord( + owner.installation, + "executions", + `${owner.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: unknown } }; + if (controlDigest(admitted) !== owner.receiptDigest) + throw Error("Original import admission changed"); + const operation = opsOperationSchema.parse(admitted.payload.operation); + if ( + operation.kind !== "artifactTransfer.import" || + opsOperationFingerprint(operation) !== owner.execution.operationFingerprint + ) + throw Error("Original import body required"); + return operation; +} + +/** Actual fixed runtime has finished without starting publication. Freeze the + * original issuer, prove every finite child unused, then retain nonissuance. */ +export function retainUnstartedArtifactImport(authority: VerifiedExecutionAuthority) { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "artifactTransfer.import" || + !owner.origin || + readJournal({ ...owner, origin: owner.origin }) + ) + throw Error("Unstarted import required"); + const started = readPrivateRecord( + owner.installation, + "effect-started", + `${owner.execution.operationKey}.json`, + ); + if (started !== null) throw Error("Import publication began"); + verifyFamilyOwnedRecord( + owner.installation, + owner.origin, + { + scope: "private", + namespace: "effect-started", + container: "", + name: `${owner.execution.operationKey}.json`, + }, + canonicalJson(started), + ); + if ( + cancelControlExecution(authority).length || + owner.reservations.some((slot) => { + const source = inspectControlChildPublication(authority, slot); + return source.binding !== null || source.publication.state !== "unissued"; + }) + ) + throw Error("Original import child remains attempted"); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); +} + +/** Fixed original body projection; callers cannot substitute stage/owner IDs. */ +export function originalArtifactImportOperation(authority: VerifiedExecutionAuthority) { + const owner = executionAuthorityState(authority); + if (owner.kind !== "artifactTransfer.import" || !owner.origin) + throw Error("Original import required"); + const admitted = readPrivateRecord( + owner.installation, + "executions", + `${owner.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: unknown } }; + if (controlDigest(admitted) !== owner.receiptDigest) + throw Error("Original import admission changed"); + const operation = opsOperationSchema.parse(admitted.payload.operation); + if ( + operation.kind !== "artifactTransfer.import" || + opsOperationFingerprint(operation) !== owner.execution.operationFingerprint + ) + throw Error("Original import body required"); + return operation; +} + +export function originalArtifactValidationOperation(authority: VerifiedExecutionAuthority) { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "artifactTransfer.validate" || + !owner.origin || + owner.installation.installation.role !== "ops-agent" + ) + throw Error("Original validation required"); + const admitted = readPrivateRecord( + owner.installation, + "executions", + `${owner.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: unknown } }; + if (controlDigest(admitted) !== owner.receiptDigest) + throw Error("Original validation admission changed"); + const operation = opsOperationSchema.parse(admitted.payload.operation); + if ( + operation.kind !== "artifactTransfer.validate" || + opsOperationFingerprint(operation) !== owner.execution.operationFingerprint || + owner.plan.length || + owner.reservations.length + ) + throw Error("Original private-only validation body required"); + return operation; +} +/** This fixed private-only operation has no managed target or child. The live + * store producer first proves its extraction is contained and cleaned. Match + * its actual signed effect result before upgrading only the family knowledge; + * the original public status tuple, deadline and timestamp remain unchanged. */ +export function retainArtifactValidationObservation( + authority: VerifiedExecutionAuthority, + status: OpsJobStatus, + terminalAt: number, + result: unknown, +) { + const operation = originalArtifactValidationOperation(authority), + owner = executionAuthorityState(authority), + stage = transferStageSchema.parse(result); + if ( + stage.stageId !== operation.stageId || + stage.purpose !== "prepared-artifact" || + !["ready", "failed"].includes(stage.state) || + (stage.state === "failed" && stage.cleanup !== "complete") + ) + throw Error("Original contained validation required"); + const effect = childSettlementState( + verifyControlEffectSettlement(owner.installation.installation, owner.execution), + ).evidence; + if ( + effect.outcome !== "committed" || + effect.publication !== "committed" || + effect.terminalReceiptDigest !== controlDigest({ kind: "ops-result", result: stage }) + ) + throw Error("Original validation producer result required"); + retainControlTerminalStatus(owner.installation.installation, owner.execution, status, terminalAt); + publishFamilyCanonicalTerminal( + owner.installation, + owner.origin!, + owner.execution, + true, + controlTerminalReceiptDigest(status), + terminalAt, + ); +} + +export function assertKnownArtifactStageReceipt(authority: VerifiedExecutionAuthority) { + const owner = executionAuthorityState(authority); + if (!["artifactTransfer.import", "artifactTransfer.validate"].includes(owner.kind)) + throw Error("Original prepared purpose required"); + const effect = childSettlementState( + verifyControlEffectSettlement(owner.installation.installation, owner.execution), + ).evidence; + if (effect.outcome === "unknown" || effect.publication === "unknown") + throw Error("Prepared purpose remains unknown"); + if (owner.kind === "artifactTransfer.import") verifyRetainedArtifactTargetRootRelease(authority); +} diff --git a/packages/core/src/ops/artifact-transfer.test.ts b/packages/core/src/ops/artifact-transfer.test.ts index 22e8f543a..8bee61112 100644 --- a/packages/core/src/ops/artifact-transfer.test.ts +++ b/packages/core/src/ops/artifact-transfer.test.ts @@ -40,3 +40,21 @@ it("defines bounded prepared preview and import operations without caller paths }).success, ).toBe(false); }); + +it("exports only named prepared targets under an exact artifact fingerprint", async () => { + const { opsOperationSchema, OPS_KIND_POLICIES } = await import("./contract"); + const valid = { + kind: "artifactTransfer.export", + serviceId: "osrm", + ownerId: "admin", + expectedArtifactFingerprint: "a".repeat(64), + }; + expect(opsOperationSchema.safeParse(valid).success).toBe(true); + expect(opsOperationSchema.safeParse({ ...valid, serviceId: "redis" }).success).toBe(false); + expect(opsOperationSchema.safeParse({ ...valid, path: "/arbitrary" }).success).toBe(false); + expect(OPS_KIND_POLICIES["artifactTransfer.export"]).toMatchObject({ + role: "api", + execution: "async", + timeoutMs: 72 * 3600_000, + }); +}); diff --git a/packages/core/src/ops/artifact-transfer.ts b/packages/core/src/ops/artifact-transfer.ts index 79a3c76ca..60e9d9393 100644 --- a/packages/core/src/ops/artifact-transfer.ts +++ b/packages/core/src/ops/artifact-transfer.ts @@ -140,7 +140,15 @@ export const artifactImportOperationSchema = z.strictObject({ .regex(/^cfg1_[A-Za-z0-9_-]{43}$/) .nullable(), }); +export const artifactExportOperationSchema = z.strictObject({ + kind: z.literal("artifactTransfer.export"), + serviceId: z.enum(["tileserver", "osrm", "otp", "pelias", "motis"]), + ownerId: owner, + expectedArtifactFingerprint: digest, +}); export const artifactTransferOperationSchemas = [ + z.strictObject({ kind: z.literal("artifactTransfer.capabilities"), ownerId: owner }), + artifactExportOperationSchema, z.strictObject({ kind: z.literal("artifactTransfer.previewImport"), stageId: z.uuid(), @@ -174,6 +182,34 @@ export const artifactTransferOperationSchemas = [ }), ] as const; export const artifactTransferResultSchemas = { + "artifactTransfer.capabilities": z.strictObject({ + limits: z.strictObject({ + maxUploadBytes: count, + maxExpandedBytes: count, + maxStageBytes: count, + stageMs: count, + downloadRetentionMs: count, + }), + freeBytes: count.nullable(), + freeInodes: count.nullable(), + reservedBytes: count, + targets: z + .array( + z.strictObject({ + serviceId: z.string().max(64), + available: z.boolean(), + reason: z.string().max(512).nullable(), + artifactFingerprint: digest.nullable(), + }), + ) + .max(5), + }), + "artifactTransfer.export": z.strictObject({ + stageId: z.uuid(), + sha256: digest, + bytes: count, + expiresAt: z.iso.datetime(), + }), "artifactTransfer.previewImport": artifactImportPreviewSchema, "artifactTransfer.import": artifactPublicationResultSchema, "artifactTransfer.validate": transferStageSchema, @@ -191,6 +227,9 @@ export const artifactTransferResultSchemas = { .strictObject({ claim: artifactStageClaimSchema, settlement: stageClaimSettlementSchema.nullable(), + /** Original signed publication and released-root observation, never a transfer-state inference. */ + publication: artifactPublicationResultSchema.optional(), + validation: transferStageSchema.optional(), }) .nullable(), "artifactTransfer.settle": transferStageSchema, diff --git a/packages/core/src/ops/configuration-preparation-records.ts b/packages/core/src/ops/configuration-preparation-records.ts index ffcbca811..b96aa8f96 100644 --- a/packages/core/src/ops/configuration-preparation-records.ts +++ b/packages/core/src/ops/configuration-preparation-records.ts @@ -161,6 +161,14 @@ export const preparedConfigurationRecordSchema = z.strictObject({ ), ), bindings: z.record(id, z.record(z.string().min(1).max(128), id)).optional(), + stagedMigration: z + .strictObject({ + version: z.literal(1), + kind: z.enum(["migration", "credentials"]), + previewId: z.uuid(), + integrationIds: z.array(id).max(256), + }) + .optional(), legacyImport: z .strictObject({ version: z.literal(1), diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index d429cd1fa..c326682c1 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -66,6 +66,13 @@ const transferClaim = { bytes: 4, }; const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { + "artifactTransfer.capabilities": { kind: "artifactTransfer.capabilities", ownerId: "admin" }, + "artifactTransfer.export": { + kind: "artifactTransfer.export", + serviceId: "osrm", + ownerId: "admin", + expectedArtifactFingerprint: "a".repeat(64), + }, "artifactTransfer.previewImport": { kind: "artifactTransfer.previewImport", stageId: transferClaim.stageId, diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 679f38cc0..acaa6498e 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -92,6 +92,8 @@ export const OPS_OPERATION_KINDS = [ "artifactTransfer.create", "artifactTransfer.validate", "artifactTransfer.previewImport", + "artifactTransfer.export", + "artifactTransfer.capabilities", "artifactTransfer.import", "artifactTransfer.list", "artifactTransfer.inspect", @@ -246,6 +248,8 @@ export const OPS_KIND_POLICIES = { "artifactTransfer.validate": asyncEffect("api", 72 * 3600_000), "artifactTransfer.previewImport": sync("api", 120_000, 768 * 1024), "artifactTransfer.import": asyncEffect("api", 72 * 3600_000), + "artifactTransfer.export": asyncEffect("api", 72 * 3600_000), + "artifactTransfer.capabilities": sync("api", 120_000), "artifactTransfer.list": sync("api", 30_000), "artifactTransfer.inspect": sync("api", 30_000), "artifactTransfer.delete": sync("api", 30_000), diff --git a/packages/core/src/services/__tests__/platform-preview.test.ts b/packages/core/src/services/__tests__/platform-preview.test.ts index 4bc3494e8..6727beecf 100644 --- a/packages/core/src/services/__tests__/platform-preview.test.ts +++ b/packages/core/src/services/__tests__/platform-preview.test.ts @@ -39,6 +39,29 @@ const input = () => ({ services, }); describe("platform configuration preview", () => { + it("joins exact migration targets into one unchanged-platform handoff intent", () => { + const base = { ...input(), changes: { version: 1 as const } }, + preview = buildPlatformConfigurationPreview({ + ...base, + additionalServiceIds: ["redis", "data-manager"], + }); + expect(preview.affectedServiceIds).toEqual(["data-manager", "redis"]); + expect(preview.requiredConfirmation.domain).toBe(false); + expect(preview.oldOrigin).toBe(preview.newOrigin); + expect(preview.previewFingerprint).not.toBe( + buildPlatformConfigurationPreview(base).previewFingerprint, + ); + expect(() => + buildPlatformConfigurationPreview({ ...base, additionalServiceIds: ["unknown-service"] }), + ).toThrow(); + expect(() => + buildPlatformConfigurationPreview({ + ...base, + services: services.map((s) => ({ ...s, enabled: false })), + additionalServiceIds: ["redis"], + }), + ).toThrow(); + }); it("previews an explicit protected resource reconciliation without inventing a global setting change", () => { const base = { ...input(), changes: { version: 1 as const } }; const preview = buildPlatformConfigurationPreview({ diff --git a/packages/core/src/services/platform-preview.ts b/packages/core/src/services/platform-preview.ts index 795cfa3b6..4c35d0f49 100644 --- a/packages/core/src/services/platform-preview.ts +++ b/packages/core/src/services/platform-preview.ts @@ -39,6 +39,8 @@ export function buildPlatformConfigurationPreview(input: { services: readonly LoadedService[]; resolvedServiceConfigs?: Map>; reconcileServiceIds?: readonly string[]; + /** Finite staged migration selected services, joined into this same generation. */ + additionalServiceIds?: readonly string[]; release?: { releaseId: string; manifestDigest: string }; }) { const currentRevisionId = revision.parse(input.currentRevisionId); @@ -86,9 +88,17 @@ export function buildPlatformConfigurationPreview(input: { (key) => before.values[key] !== after.values[key], ); const domain = before.values.domain !== after.values.domain; + const additional = z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .max(256) + .parse(input.additionalServiceIds ?? []); + if (additional.some((id) => !input.services.some((s) => s.enabled && s.manifest.id === id))) + throw Error("Migration service unavailable"); const affectedServiceIds = input.services .filter((service) => { - if (!service.isBuiltIn || !service.enabled) return false; + if (!service.enabled) return false; + if (additional.includes(service.manifest.id)) return true; + if (!service.isBuiltIn) return false; const m = service.manifest; if (release) return Object.hasOwn(configurationReleaseServiceImagesSchema.shape, m.id); if (reconcileServiceIds?.includes(m.id as (typeof reconcileServiceIds)[number])) return true; diff --git a/packages/transitous-core/src/feed-credentials.ts b/packages/transitous-core/src/feed-credentials.ts index d8ba0488e..ee499dd19 100644 --- a/packages/transitous-core/src/feed-credentials.ts +++ b/packages/transitous-core/src/feed-credentials.ts @@ -128,9 +128,10 @@ export function parseFeedCredentialMapJson(json: string): Readonly>, changes: readonly FeedCredentialChange[], + maxChanges: 100 | 1000 = 100, ): ValidatedFeedCredentialMap { const values = Object.fromEntries(entriesOf(existing)); - if (!Array.isArray(changes) || changes.length > 100) throw invalid(); + if (!Array.isArray(changes) || changes.length > maxChanges) throw invalid(); const seen = new Set(); for (const change of changes) { if ( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 021f5a0bd..ced21f25b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2542,6 +2542,9 @@ importers: '@openmapx/mobility-core': specifier: workspace:^ version: link:../mobility-core + '@openmapx/motis-feed-proxy-config': + specifier: workspace:^ + version: link:../motis-feed-proxy-config '@openmapx/transitous-core': specifier: workspace:^ version: link:../transitous-core diff --git a/scripts/check-subject-data.test.ts b/scripts/check-subject-data.test.ts index 16f67f546..ad7d0208a 100644 --- a/scripts/check-subject-data.test.ts +++ b/scripts/check-subject-data.test.ts @@ -6,6 +6,13 @@ describe("subject-data policy gate", () => { const result = validateSubjectDataInventory(); expect(result.errors).toEqual([]); }); + it("classifies encrypted migration preview actor attribution", () => { + expect( + validateSubjectDataInventory({ + directUserForeignKeys: ["configuration_migration_preview.actor_id"], + }).errors, + ).toEqual([]); + }); it("reports unknown manifest registration ids", () => { const result = validateSubjectDataInventory({ diff --git a/scripts/check-subject-data.ts b/scripts/check-subject-data.ts index 65ff7a153..24a70a586 100644 --- a/scripts/check-subject-data.ts +++ b/scripts/check-subject-data.ts @@ -39,6 +39,7 @@ export const DIRECT_USER_FK_CLASSIFICATIONS: Record = { "saved_list.user_id": "saved-lists", "service_secret.updated_by": "secret-update-attribution", "transit_credential_preview.actor_id": "secret-update-attribution", + "configuration_migration_preview.actor_id": "secret-update-attribution", "session.user_id": "auth-sessions", "admin_action_reauthentication.actor_id": "auth-sessions", "share_link.user_id": "share-links", diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 0326b510f..c0b6bc6a1 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -35,9 +35,11 @@ "agent.update", "appApi.replace", "appApi.runtime.inspect", + "artifactTransfer.capabilities", "artifactTransfer.claim", "artifactTransfer.create", "artifactTransfer.delete", + "artifactTransfer.export", "artifactTransfer.import", "artifactTransfer.inspect", "artifactTransfer.inspectClaim", @@ -431,7 +433,8 @@ "artifactTransfer.claim", "artifactTransfer.inspectClaim", "artifactTransfer.settle", - "artifactTransfer.previewImport" + "artifactTransfer.previewImport", + "artifactTransfer.capabilities" ] }, { @@ -483,6 +486,15 @@ "apps/api/src/services/admin-artifact-validation.ts#handlePreparedArtifactImportJob" ], "typedKinds": ["artifactTransfer.import"] + }, + { + "id": "api.artifact-export", + "callSiteIds": [], + "semanticSources": [ + "apps/api/src/services/admin-artifact-validation.ts#handlePreparedArtifactExportJob", + "apps/ops-agent/src/artifact-transfer/export-runtime.ts#installPreparedExportRuntime" + ], + "typedKinds": ["artifactTransfer.export"] } ], "mounts": [ diff --git a/services/data-manager/__tests__/transitous/fixtures/portable-pinned-motis-config.yml b/services/data-manager/__tests__/transitous/fixtures/portable-pinned-motis-config.yml new file mode 100644 index 000000000..2df0c394d --- /dev/null +++ b/services/data-manager/__tests__/transitous/fixtures/portable-pinned-motis-config.yml @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: 2024 Jonah Brüchert +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +server: + host: 0.0.0.0 + port: 8080 + web_folder: "/opt/motis/ui/" # Just for testing, not used on server + data_attribution_link: https://transitous.org/sources/ + +user_agent: transitous.org +street_routing: + elevation_data_dir: ./srtm/ +osr_footpath: false +elevators: false +geocoding: true +reverse_geocoding: true + +osm: planet-latest.osm.pbf + +tiles: + profile: /opt/motis/tiles-profiles/full.lua + coastline: land-polygons-complete-4326.zip + +timetable: + first_day: -30 + num_days: 395 + railviz: true + with_shapes: true + adjust_footpaths: true + merge_dupes_intra_src: true + merge_dupes_inter_src: true + link_stop_distance: 100 + update_interval: 60 + http_timeout: 30 + incremental_rt_update: false + preprocess_max_matching_distance: 0 + max_footpath_length: 12 + extend_missing_footpaths: true + # The datasets are filled in by ./src/generate-motis-config.py + datasets: + +gbfs: + ttl: + # static information - 6h + overwrite: + gbfs: 21600 + manifest: 21600 + system_information: 21600 + vehicle_types: 21600 + station_information: 21600 + geofencing_zones: 21600 + + # dynamic information - 3min + default: + station_status: 180 + vehicle_status: 180 + free_bike_status: 180 + feeds: + +limits: + stoptimes_max_results: 16384 + plan_max_results: 256 + plan_max_search_window_minutes: 5760 + stops_max_results: 16384 + onetoall_max_results: 524288 + onetoall_max_travel_minutes: 2880 + routing_max_timeout_seconds: 90 + street_routing_max_prepost_transit_seconds: 7200 + street_routing_max_direct_seconds: 21600 + geocode_max_suggestions: 512 + reverse_geocode_max_results: 512 diff --git a/services/data-manager/src/jobs/transitous/assemble-staging.ts b/services/data-manager/src/jobs/transitous/assemble-staging.ts index ea65ede2b..9a89f6dbf 100644 --- a/services/data-manager/src/jobs/transitous/assemble-staging.ts +++ b/services/data-manager/src/jobs/transitous/assemble-staging.ts @@ -9,8 +9,14 @@ import { rmSync, } from "node:fs"; import { join } from "node:path"; +import { + createMotisPinnedBlobReader, + PORTABLE_MOTIS_FILENAME, + producePortableMotisRecipe, +} from "@openmapx/core/build-artifacts/motis-portable"; +import { parseFeedCredentialMapJson } from "@openmapx/transitous-core/feed-credentials"; import { CANDIDATE_PROXY_DIRNAME, createCandidateManifestAsync } from "./candidate.js"; -import { GTFS_ARCHIVE_RE } from "./internal.js"; +import { GTFS_ARCHIVE_RE, resolveTransitousApiKeysPath } from "./internal.js"; import { readTransitSourceManifest, TRANSIT_SOURCE_MANIFEST_FILENAME } from "./source-manifest.js"; import type { StageFn, StageResult } from "./types.js"; @@ -219,6 +225,40 @@ export const run: StageFn = async (ctx) => { ctx.abortSignal, ); + // Only the successful real assembly may retain a portable recipe. Unsupported + // host/operator sources remain usable locally, with an explicit export prerequisite. + rmSync(join(stagingDir, PORTABLE_MOTIS_FILENAME), { force: true }); + let portableRecipeWritten = false; + if (ctx.state.catalogPins && missingFeeds.length === 0 && osmSource !== "missing") { + try { + const pins = { + transitousCommit: ctx.state.catalogPins.transitous.ref.split("@").at(-1) ?? "", + atlasCommit: ctx.state.catalogPins.transitous.submodules["transitland-atlas"] ?? "", + }; + const keysPath = resolveTransitousApiKeysPath(ctx.apiKeysPath); + const credentials = existsSync(keysPath) + ? parseFeedCredentialMapJson(readFileSync(keysPath, "utf8")) + : {}; + await producePortableMotisRecipe({ + stagingDir, + pins, + credentials, + readBlob: createMotisPinnedBlobReader( + ctx.state.catalogDir ?? ctx.catalogDir, + pins, + ctx.abortSignal, + ), + signal: ctx.abortSignal, + }); + portableRecipeWritten = true; + } catch { + ctx.abortSignal.throwIfAborted(); + ctx.logger.warn( + "assemble-staging: portable export unavailable; supported pinned nonsecret source metadata and destination credential bindings are required", + ); + } + } + const status = missingFeeds.length > 0 || osmSource === "missing" ? "partial" : "ok"; return finish( status, @@ -234,6 +274,13 @@ export const run: StageFn = async (ctx) => { osm: osm ?? null, osmSource, candidateEpoch: manifest.epoch, + portableRecipeWritten, + ...(!portableRecipeWritten + ? { + portableExportPrerequisite: + "Rebuild with supported pinned, nonsecret source metadata and credential bindings", + } + : {}), configHash: manifest.artifacts.config.sha256, licenseHash: manifest.artifacts.license.sha256, }, From 982ebf0a8ed865a0b5dbb0b76309f8876944e88b Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 31 Oct 2026 02:00:51 +0100 Subject: [PATCH 104/135] fix(admin): retain extension workflows and reconcile final integration checks --- .../data-manager-admin-contracts.test.ts | 4 +- .../extension-admin-contracts.test.ts | 4 +- .../integration-admin-contracts.test.ts | 4 +- .../__tests__/service-admin-contracts.test.ts | 78 +- .../privacy/release-approval-route.test.ts | 1 + .../admin-activity-cancel-pending.test.ts | 18 +- .../__tests__/admin-integrations-sdk.test.ts | 8 +- .../__tests__/admin-services-sdk.test.ts | 13 + .../routes/__tests__/admin-services.test.ts | 9 +- apps/api/src/routes/__tests__/admin.test.ts | 178 ++++- .../routes/__tests__/auth-discovery.test.ts | 8 + .../__tests__/data-manager-bump.test.ts | 2 + apps/api/src/routes/admin-byte-response.ts | 4 +- .../admin-job-parent-extension.test.ts | 34 + .../__tests__/admin-ops-cancellation.test.ts | 362 ++++++++-- .../__tests__/community-repositories.test.ts | 128 +++- ...configuration-maintenance-postgres.test.ts | 24 +- .../coordinator-lifecycle-postgres.test.ts | 4 +- .../__tests__/extension-installer.test.ts | 126 +++- .../fixtures/extension-workflow-native.ts | 113 +++ .../job-runner-extension-sanitization.test.ts | 36 +- .../__tests__/job-runner-initialize.test.ts | 15 +- ...orm-configuration-runtime-postgres.test.ts | 4 +- .../retained-maintenance-job-postgres.test.ts | 12 +- .../service-registry-community.test.ts | 4 +- apps/api/src/services/admin-job-ops.ts | 3 +- apps/api/src/services/admin-job-parent.ts | 30 +- apps/api/src/services/admin-ops.ts | 15 +- .../src/services/community-repositories.ts | 117 ++- .../services/community-service-inventory.ts | 55 ++ .../legacy-integration-import.test.ts | 2 +- .../migration-postgres.test.ts | 2 +- .../api/src/services/coordinator-lifecycle.ts | 5 +- apps/api/src/services/extension-installer.ts | 510 ++++++++----- apps/api/src/services/extension-recovery.ts | 1 + .../extension-workflow-postgres.test.ts | 458 ++++++++++++ .../src/services/extension-workflow.test.ts | 60 ++ apps/api/src/services/extension-workflow.ts | 201 ++++++ .../services/integration-application.test.ts | 2 +- apps/api/src/services/job-runner.ts | 22 +- .../src/services/prospective-configuration.ts | 102 ++- .../retained-maintenance-authorization.ts | 3 +- .../src/services/retained-maintenance-job.ts | 5 +- apps/api/src/services/service-registry.ts | 2 +- .../standalone-integrations/install.test.ts | 2 +- .../standalone-integrations/install.ts | 3 +- .../ownership-state.ts | 3 +- .../services/standalone-integrations/state.ts | 5 +- .../services/transit-credential-recovery.ts | 3 +- .../transit-feed-credentials-postgres.test.ts | 2 +- .../trusted-configuration-producer.ts | 26 + apps/api/src/utils/docker-compose.ts | 9 + .../src/__fixtures__/configuration-runtime.ts | 44 ++ .../src/__fixtures__/configuration-source.ts | 8 +- .../src/__fixtures__/configuration-writer.mjs | 2 +- .../src/__fixtures__/owned-service-host.ts | 7 +- apps/ops-agent/src/__fixtures__/server.ts | 38 + .../src/administrative-runtime.test.ts | 17 +- apps/ops-agent/src/agent-handoff.test.ts | 10 + .../artifact-transfer/import-runtime.test.ts | 6 +- .../src/artifact-transfer/motis.test.ts | 14 +- .../src/artifact-transfer/store.test.ts | 2 +- .../src/backup-prune-runtime.test.ts | 4 +- apps/ops-agent/src/bootstrap.test.ts | 4 +- .../src/configuration-release.test.ts | 6 +- apps/ops-agent/src/control-admission.test.ts | 5 +- apps/ops-agent/src/control-renewal.test.ts | 3 +- .../src/control-terminal-restart.test.ts | 15 +- .../src/core-service-runtime.test.ts | 12 +- .../src/data-transit-resource-source.test.ts | 3 +- .../src/data-workflow-runtime.test.ts | 2 +- apps/ops-agent/src/journal.ts | 6 +- apps/ops-agent/src/production-runtime.test.ts | 23 +- apps/ops-agent/src/production-runtime.ts | 11 + .../src/release-service-coverage.test.ts | 23 +- apps/ops-agent/src/server-foundation.test.ts | 93 ++- apps/ops-agent/src/server-review-2.test.ts | 232 +++--- apps/ops-agent/src/server.test.ts | 182 +++-- apps/ops-agent/src/server.ts | 8 +- .../standalone-integration-runtime.test.ts | 680 +++++++++--------- .../test-fixtures/data-owned-http-process.ts | 3 +- .../staged-coordinator-request-process.mjs | 2 +- .../staged-coordinator-retention-process.mjs | 2 +- .../terminal-retention-process.ts | 3 +- .../src/traffic-target-ownership.test.ts | 3 +- .../src/transit-credentials-runtime.test.ts | 36 +- .../src/transit-credentials-runtime.ts | 2 +- .../src/trusted-config-ownership.test.ts | 6 +- .../src/trusted-config-runtime.test.ts | 3 +- apps/transitous-runner/Dockerfile | 2 + .../transit/__tests__/page.test.tsx | 33 +- apps/web/src/hooks/useServices.test.tsx | 7 +- .../src/lib/admin/serviceAdminJobs.test.tsx | 7 + docs/docs/install/getting-started.md | 2 + docs/docs/install/requirements.md | 5 + .../__tests__/map-layer.test.tsx | 2 + .../__tests__/map-layer-lifecycle.test.tsx | 2 + .../__tests__/map-layer.test.tsx | 2 + integrations/overlay-readiness.test.tsx | 2 + .../overlay-weather/supplier-layers.test.tsx | 5 + .../__tests__/map-layer.test.tsx | 2 + .../cli/__tests__/cli-build-artifacts.test.ts | 3 +- .../cli-data-manager-workflow.test.ts | 48 +- .../cli/__tests__/cli-integrations.test.ts | 14 +- .../cli/__tests__/cli-notable-places.test.ts | 46 +- .../cli/__tests__/cli-pelias-data.test.ts | 6 +- .../cli/__tests__/cli-search-index.test.ts | 50 +- .../cli/__tests__/cli-services-update.test.ts | 39 +- .../__tests__/cli-transitous-api-keys.test.ts | 3 +- .../__tests__/helpers/data-command-result.ts | 21 + .../__tests__/server-import-boundary.test.ts | 10 +- .../core/src/api/admin-sdk-transport.test.ts | 4 +- packages/core/src/api/admin-sdk.test.ts | 4 +- .../data-manager-admin/contracts.test.ts | 43 +- .../api/contracts/integration-admin.test.ts | 19 +- .../src/api/contracts/service-admin.test.ts | 48 +- .../src/build-artifacts/control-authority.ts | 26 +- packages/core/src/build-artifacts/index.ts | 1 + .../src/build-artifacts/motis-destination.ts | 2 +- .../src/build-artifacts/peer-process.test.ts | 4 +- .../publication-native.test.ts | 19 +- .../repository-settlement.test.ts | 83 +++ .../src/build-artifacts/target-lease.test.ts | 4 +- .../core/src/ops/foundation-review.test.ts | 12 +- .../__fixtures__/compose-path-host.ts | 11 +- .../__tests__/service-file-store.test.ts | 7 +- .../src/services/configuration-generation.ts | 13 +- .../integration-framework/src/installer.ts | 105 ++- scripts/database-test-groups.json | 5 +- scripts/ops-agent-catalog-pins.test.ts | 56 +- scripts/ops-authority-inventory.json | 2 +- .../__tests__/ambient-places/api.test.ts | 12 + .../__tests__/openapi-freshness.test.ts | 4 +- .../__tests__/service-env-declared.test.ts | 10 +- .../__tests__/transit-sources.test.ts | 22 +- .../__tests__/transitous-pipeline.test.ts | 15 +- .../transitous/pipeline-mirror.test.ts | 15 +- services/data-manager/service.json | 1 + .../__tests__/convert-overpass-route.test.ts | 11 +- .../src/__tests__/openconditions-auth.test.ts | 12 + .../__tests__/predicted-bake-route.test.ts | 12 + .../__tests__/traffic-live-conditions.test.ts | 12 + services/data-manager/src/cron.ts | 2 +- services/service-file-history.json | 2 + 144 files changed, 4060 insertions(+), 1210 deletions(-) create mode 100644 apps/api/src/services/__tests__/admin-job-parent-extension.test.ts create mode 100644 apps/api/src/services/__tests__/fixtures/extension-workflow-native.ts create mode 100644 apps/api/src/services/community-service-inventory.ts create mode 100644 apps/api/src/services/extension-workflow-postgres.test.ts create mode 100644 apps/api/src/services/extension-workflow.test.ts create mode 100644 apps/api/src/services/extension-workflow.ts create mode 100644 apps/ops-agent/src/__fixtures__/configuration-runtime.ts create mode 100644 apps/ops-agent/src/__fixtures__/server.ts create mode 100644 packages/cli/__tests__/helpers/data-command-result.ts create mode 100644 packages/core/src/build-artifacts/repository-settlement.test.ts diff --git a/apps/api/scripts/openapi/__tests__/data-manager-admin-contracts.test.ts b/apps/api/scripts/openapi/__tests__/data-manager-admin-contracts.test.ts index df3d7c404..4e729a0e9 100644 --- a/apps/api/scripts/openapi/__tests__/data-manager-admin-contracts.test.ts +++ b/apps/api/scripts/openapi/__tests__/data-manager-admin-contracts.test.ts @@ -2,12 +2,12 @@ import { dataManagerAdminOperations } from "@openmapx/core/api-contracts"; import { expect, it } from "vitest"; import { generateDocument } from "../document"; -it("documents all 25 actual administration registrations with complete status contracts and auth", async () => { +it("documents all 38 actual administration registrations with complete status contracts and auth", async () => { const document = JSON.parse(await generateDocument()); const tagged = Object.values(document.paths) .flatMap((item) => Object.values(item as object)) .filter((value) => (value as { tags?: string[] }).tags?.includes("data-manager-admin")); - expect(tagged).toHaveLength(25); + expect(tagged).toHaveLength(38); for (const [id, contract] of Object.entries(dataManagerAdminOperations)) { const path = contract.path.replace(/:([A-Za-z_$][\w$]*)/g, "{$1}"); const actual = document.paths[path]?.[contract.method.toLowerCase()]; diff --git a/apps/api/scripts/openapi/__tests__/extension-admin-contracts.test.ts b/apps/api/scripts/openapi/__tests__/extension-admin-contracts.test.ts index 926f773a3..992ff706a 100644 --- a/apps/api/scripts/openapi/__tests__/extension-admin-contracts.test.ts +++ b/apps/api/scripts/openapi/__tests__/extension-admin-contracts.test.ts @@ -10,13 +10,13 @@ it("collects all eleven actual extension producers without changing the baseline readFileSync(new URL("../../../openapi.json", import.meta.url), "utf8"), ); expect(Object.keys(document.paths).sort()).toEqual(Object.keys(committed.paths).sort()); - expect(Object.keys(document.paths)).toHaveLength(369); + expect(Object.keys(document.paths)).toHaveLength(424); expect( Object.values(document.paths).reduce( (count: number, item) => count + Object.keys(item as object).length, 0, ), - ).toBe(395); + ).toBe(456); const tagged = Object.entries(document.paths).flatMap(([path, methods]) => Object.entries(methods as object) .filter(([, operation]) => operation.tags?.includes("extension-admin")) diff --git a/apps/api/scripts/openapi/__tests__/integration-admin-contracts.test.ts b/apps/api/scripts/openapi/__tests__/integration-admin-contracts.test.ts index 98bf494df..e3032357e 100644 --- a/apps/api/scripts/openapi/__tests__/integration-admin-contracts.test.ts +++ b/apps/api/scripts/openapi/__tests__/integration-admin-contracts.test.ts @@ -2,7 +2,7 @@ import { integrationAdminOperations } from "@openmapx/core/api-contracts"; import { expect, it, vi } from "vitest"; import { generateDocument } from "../document"; -it("collects exactly all 22 actual integration producers, complete statuses, headers and admin auth", async () => { +it("collects exactly all 35 actual integration producers, complete statuses, headers and admin auth", async () => { const network = vi.spyOn(globalThis, "fetch").mockImplementation(() => { throw Error("Network denied"); }); @@ -11,7 +11,7 @@ it("collects exactly all 22 actual integration producers, complete statuses, hea const tagged = Object.values(document.paths) .flatMap((item) => Object.values(item as object)) .filter((value) => (value as { tags?: string[] }).tags?.includes("integration-admin")); - expect(tagged).toHaveLength(22); + expect(tagged).toHaveLength(35); for (const [id, contract] of Object.entries(integrationAdminOperations)) { const actual = document.paths[contract.path.replace(/:([^/]+)/g, "{$1}")][contract.method.toLowerCase()]; diff --git a/apps/api/scripts/openapi/__tests__/service-admin-contracts.test.ts b/apps/api/scripts/openapi/__tests__/service-admin-contracts.test.ts index d874d77d1..24bbe5557 100644 --- a/apps/api/scripts/openapi/__tests__/service-admin-contracts.test.ts +++ b/apps/api/scripts/openapi/__tests__/service-admin-contracts.test.ts @@ -4,6 +4,29 @@ import { generateDocument } from "../document"; // Independent approved inventory: dropping a registry entry must fail too. const inventory = [ + ["pullAdminServiceImages", "POST", "/services/images/pull"], + ["clearAdminReleaseSelection", "POST", "/system/release/clear"], + ["previewAdminBackupPrune", "POST", "/services/backups/prune/preview"], + ["pruneAdminServiceBackups", "POST", "/services/backups/prune"], + ["listAdminCacheNamespaces", "GET", "/cache"], + ["clearAdminCache", "POST", "/cache/clear"], + ["getAdminArtifactTransferCapabilities", "GET", "/artifact-transfers/capabilities"], + ["exportAdminArtifactTransfer", "POST", "/artifact-transfers/exports"], + ["previewAdminArtifactImport", "POST", "/artifact-transfers/{stageId}/preview-import"], + ["importAdminArtifactTransfer", "POST", "/artifact-transfers/{stageId}/import"], + ["validateAdminArtifactTransfer", "POST", "/artifact-transfers/{stageId}/validate"], + ["listAdminServicePresets", "GET", "/services/presets"], + ["renderAdminComposeConfiguration", "POST", "/compose/render"], + ["listAdminArtifactTransfers", "GET", "/artifact-transfers"], + ["createAdminArtifactTransfer", "POST", "/artifact-transfers"], + ["getAdminArtifactTransfer", "GET", "/artifact-transfers/{stageId}"], + ["deleteAdminArtifactTransfer", "DELETE", "/artifact-transfers/{stageId}"], + ["uploadAdminArtifactTransferContent", "PUT", "/artifact-transfers/{stageId}/content"], + ["downloadAdminArtifactTransferContent", "GET", "/artifact-transfers/{stageId}/content"], + ["getAdminPlatformConfiguration", "GET", "/platform/configuration"], + ["previewAdminPlatformConfiguration", "POST", "/platform/configuration/preview"], + ["applyAdminPlatformConfiguration", "POST", "/platform/configuration/apply"], + ["listAdminServices", "GET", "/services"], ["getAdminService", "GET", "/services/{id}"], ["getAdminServiceConfig", "GET", "/services/{id}/config"], @@ -36,14 +59,14 @@ const inventory = [ ["rotateAdminDawarichOidcSecret", "POST", "/dawarich/rotate-oidc-secret"], ] as const; -it("collects exactly 30 actual service producers with complete statuses, media, headers and auth", async () => { +it("collects exactly 52 actual service producers with complete statuses, media, headers and auth", async () => { const network = vi.spyOn(globalThis, "fetch").mockRejectedValue(Error("Network denied")); try { const document = JSON.parse(await generateDocument()); const tagged = Object.values(document.paths) .flatMap((item) => Object.values(item as object)) .filter((value) => (value as { tags?: string[] }).tags?.includes("service-admin")); - expect(tagged).toHaveLength(30); + expect(tagged).toHaveLength(52); expect(Object.keys(serviceAdminOperations).sort()).toEqual(inventory.map(([id]) => id).sort()); for (const [id, method, path] of inventory) { const contract = serviceAdminOperations[id]; @@ -61,7 +84,38 @@ it("collects exactly 30 actual service producers with complete statuses, media, : undefined; expect(Object.keys(actual.responses[status].content ?? {})).toEqual(media ? [media] : []); } - expect(actual.responses["429"].headers).toHaveProperty("retry-after"); + if (path.startsWith("/artifact-transfers")) { + // Retained quota/attempt/concurrency denials have no truthful retry timer. + expect(actual.responses["429"].headers).toBeUndefined(); + expect(actual.responses["429"].content["application/json"].schema).toEqual({ + anyOf: [ + { + type: "object", + properties: { + error: { type: "string" }, + ok: { type: "boolean", enum: [false] }, + message: { type: "string" }, + jobId: { type: "string" }, + }, + required: ["error"], + additionalProperties: false, + }, + { + type: "object", + properties: { + statusCode: { type: "integer", minimum: 400, maximum: 599 }, + error: { type: "string" }, + message: { type: "string" }, + code: { type: "string" }, + }, + required: ["statusCode", "error", "message"], + additionalProperties: false, + }, + ], + }); + } else { + expect(actual.responses["429"].headers).toHaveProperty("retry-after"); + } } for (const path of ["/services/{id}/logs", "/compose/up"]) { const actual = document.paths[`/api/admin${path}`][path.endsWith("logs") ? "get" : "post"]; @@ -69,6 +123,24 @@ it("collects exactly 30 actual service producers with complete statuses, media, expect.objectContaining({ in: "header", name: "idempotency-key", required: true }), ); } + const bytes = document.paths["/api/admin/artifact-transfers/{stageId}/content"]; + expect(bytes.put.requestBody).toEqual({ + required: true, + content: { "application/octet-stream": { schema: { type: "string", format: "binary" } } }, + }); + expect(bytes.get.responses[200].content).toEqual({ + "application/octet-stream": { schema: { type: "string", format: "binary" } }, + }); + expect(bytes.get.responses[200].headers).toEqual({ + "content-type": { schema: { type: "string", const: "application/octet-stream" } }, + "content-length": { schema: { type: "string", pattern: "^(0|[1-9][0-9]*)$" } }, + "content-disposition": { + schema: { type: "string", const: 'attachment; filename="openmapx-artifact.tar"' }, + }, + "x-artifact-sha256": { schema: { type: "string", pattern: "^[a-f0-9]{64}$" } }, + "cache-control": { schema: { type: "string", const: "no-store" } }, + "x-content-type-options": { schema: { type: "string", const: "nosniff" } }, + }); expect(document.paths["/api/admin/compose/down"].post.responses).not.toHaveProperty("200"); expect(document.paths["/api/admin/services/backups"].post.requestBody.required).toBe(false); expect(document.paths["/api/admin/services/selection"].put.requestBody.required).toBe(true); diff --git a/apps/api/src/privacy/release-approval-route.test.ts b/apps/api/src/privacy/release-approval-route.test.ts index f70f05bf1..411b1bd7a 100644 --- a/apps/api/src/privacy/release-approval-route.test.ts +++ b/apps/api/src/privacy/release-approval-route.test.ts @@ -134,6 +134,7 @@ describe("privacy release approval routes", () => { method: "POST", url: `/privacy/admin/requests/${randomUUID()}/generate`, headers: { "idempotency-key": randomUUID() }, + payload: {}, }); expect(response.statusCode).toBe(503); expect(response.json().code).toBe("PRIVACY_RELEASE_NOT_READY"); diff --git a/apps/api/src/routes/__tests__/admin-activity-cancel-pending.test.ts b/apps/api/src/routes/__tests__/admin-activity-cancel-pending.test.ts index d1fd790fa..97e00ddb1 100644 --- a/apps/api/src/routes/__tests__/admin-activity-cancel-pending.test.ts +++ b/apps/api/src/routes/__tests__/admin-activity-cancel-pending.test.ts @@ -4,11 +4,20 @@ import { adminJobStreamEventSchemas } from "@openmapx/core/api-contracts"; import { getTableColumns } from "drizzle-orm"; import { drizzle } from "drizzle-orm/pg-proxy"; import Fastify from "fastify"; -import { afterEach, expect, it, vi } from "vitest"; +import { afterAll, afterEach, expect, it, vi } from "vitest"; import { adminJob } from "../../db/schema"; import { installAdminRouteMocks } from "./admin-test-helpers"; import { administrationSdkFetch } from "./helpers/administration-sdk-fetch"; +const authSecret = vi.hoisted(() => { + const previous = process.env.BETTER_AUTH_SECRET; + process.env.BETTER_AUTH_SECRET ??= "cancel-pending-unit-only-secret"; + return { previous }; +}); +afterAll(() => { + if (authSecret.previous === undefined) delete process.env.BETTER_AUTH_SECRET; + else process.env.BETTER_AUTH_SECRET = authSecret.previous; +}); installAdminRouteMocks(); const cancelOperations = vi.fn(); vi.mock("../../services/admin-job-ops", () => ({ @@ -38,10 +47,12 @@ it("keeps the real runner's durable pending outcome observable through Fastify a // runner, event bus, routes, serializers and generated SDK all execute. const database = drizzle(async (sql, params) => { if (sql.startsWith('update "admin_job"')) { - if (sql.includes("returning")) return { rows: [] }; // The row is running, not queued. + if (params[0] === "canceled") return { rows: [] }; // The row is running, not queued. if (params[0] !== "cancel_pending") throw new Error(`Unexpected update: ${sql}`); row.status = "cancel_pending"; - return { rows: [] }; + return { + rows: [Object.keys(getTableColumns(adminJob)).map((key) => row[key as keyof typeof row])], + }; } if (sql.startsWith('select "status", "result"')) return { rows: [[row.status, row.result]] }; if (sql.includes("union all")) { @@ -110,6 +121,7 @@ it("keeps the real runner's durable pending outcome observable through Fastify a status: 200, data: { ...row, + result: {}, // Internal operation/authority state is never public Activity data. createdAt: "2026-10-19T10:00:00.000Z", startedAt: "2026-10-19T10:00:00.000Z", source: "application", diff --git a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts index 0ca65f3a2..1f466c9e8 100644 --- a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts @@ -351,6 +351,12 @@ async function wire() { }); await app.register(adminRoute, { prefix: "/api" }); await registerCapabilityBindingRoutes(app); + for (const route of [ + (await import("../admin-standalone-integrations")).adminStandaloneIntegrationsRoute, + (await import("../admin-configuration-migration")).adminConfigurationMigrationRoute, + (await import("../admin-credential-transfer")).adminCredentialTransferRoute, + ]) + await app.register(route, { prefix: "/api" }); await app.ready(); const transport = administrationSdkFetch(app); vi.stubGlobal("fetch", (input: string | Request | URL, init?: RequestInit) => { @@ -361,7 +367,7 @@ async function wire() { return { app, registered, client }; } describe("actual integration producers through generated SDK", () => { - it("exports and executes exactly all 22 registered operations with full serialized fields", async () => { + it("exports all 35 registered operations and executes integration producers with full serialized fields", async () => { const sdk = await import("@openmapx/core/api-admin-sdk"); for (const id of Object.keys(integrationAdminOperations)) expect(sdk).toHaveProperty(id, expect.any(Function)); diff --git a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts index e80b55242..2e6a0f205 100644 --- a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts @@ -28,6 +28,18 @@ import { serviceSelectionFixture, } from "./helpers/service-admin-fixtures"; +// Existing admin session guards are supplied by installAdminRouteMocks below. +// Any accidental fallback to the external Better Auth provider fails closed. +vi.mock("../../auth", () => ({ + auth: { + api: { + getSession: async () => { + throw Error("Unexpected external session lookup in admin SDK fixture"); + }, + }, + }, +})); + const { requireAdmin, session, writeAuditLog } = installAdminRouteMocks(); const service = { ...serviceFixture(), @@ -377,6 +389,7 @@ async function wire(max = 1000) { }); for (const route of [ adminServicesRoute, + (await import("../admin-backup-maintenance")).adminBackupMaintenanceRoute, adminOperationsRoute, adminDawarichRoute, registerAdminComposeRoutes, diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index a1cdaae51..942300a33 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -106,7 +106,8 @@ vi.mock("../../services/service-config-resolver.js", () => ({ mockResolveServiceConfigWithSources(...args), })); -vi.mock("../../services/configuration-ownership-view.js", () => ({ +vi.mock("../../services/configuration-ownership-view.js", async (importActual) => ({ + ...(await importActual()), inspectServiceConfiguration: async (...args: unknown[]) => ({ resolved: await mockResolveServiceConfigWithSources(...args), ownership: {}, @@ -752,7 +753,7 @@ describe("service credentials", () => { ); }, ); - it("answers an apply the operator can fix with its reason, after restoring what was saved", async () => { + it("answers an apply the operator can fix with its reason without route-owned writes", async () => { const problem = "infra/docker/overrides/services/martin/config/martin.yaml can't be used: it is a symbolic link. Render refuses it rather than use the release's file in its place: make it a regular file of yours that only you may write, or delete it."; vi.mocked(applyTrustedConfiguration).mockRejectedValueOnce( @@ -768,11 +769,13 @@ describe("service credentials", () => { const res = await app.inject({ method: "POST", url: "/admin/services/openconditions-ingest/config", + headers: { "idempotency-key": "018f7b8a-3c7a-7b91-a9b0-9d6dd0f51ab1" }, payload: { config: {} }, }); expect(res.statusCode).toBe(409); expect(res.json()).toEqual({ error: problem }); - expect(mockRestoreServiceConfig).toHaveBeenCalledTimes(1); + expect(mockRestoreServiceConfig).not.toHaveBeenCalled(); + expect(mockMergeServiceConfig).not.toHaveBeenCalled(); }); it("persists only the route-validated update through the atomic writer", async () => { mockValidateConfigBody.mockReturnValueOnce({ diff --git a/apps/api/src/routes/__tests__/admin.test.ts b/apps/api/src/routes/__tests__/admin.test.ts index 169724ec5..8275c41d0 100644 --- a/apps/api/src/routes/__tests__/admin.test.ts +++ b/apps/api/src/routes/__tests__/admin.test.ts @@ -12,7 +12,8 @@ const { } = installAdminRouteMocks(); // Rate limiters — no-op -vi.mock("../../utils/rate-limit.js", () => ({ +vi.mock("../../utils/rate-limit.js", async (importActual) => ({ + ...(await importActual()), healthCheckSweepLimit: { preHandler: () => vi.fn().mockResolvedValue(undefined) }, serviceActionLimit: { preHandler: () => vi.fn().mockResolvedValue(undefined) }, storeInstallLimit: { preHandler: () => vi.fn().mockResolvedValue(undefined) }, @@ -26,10 +27,30 @@ vi.mock("../../utils/rate-limit.js", () => ({ // Job runner const mockJobRunnerEnqueue = vi.fn().mockResolvedValue("job-main-123"); const mockJobRunnerCancel = vi.fn().mockResolvedValue(true); +// Controlled application-broker boundary, never host or durable completion proof. +const mockImmediateCredential = vi.fn(async (intent: { integrationId: string }) => ({ + operationId: "credential-route-fixture", + integrationId: intent.integrationId, + desiredRevision: "a".repeat(64), + database: "retained", + runtime: "applied", + runtimeGenerationId: "11111111-1111-4111-8111-111111111111", +})); +const mockImmediateImport = vi.fn().mockResolvedValue({ + ok: true, + imported: 1, + skipped: [], + message: "Integrations reloaded", + reloaded: 1, + enabled: 1, +}); vi.mock("../../services/job-runner.js", () => ({ jobRunner: { enqueue: (...args: unknown[]) => mockJobRunnerEnqueue(...args), cancel: (...args: unknown[]) => mockJobRunnerCancel(...args), + applyImmediateIntegrationCredential: (intent: { integrationId: string }) => + mockImmediateCredential(intent), + importImmediateIntegrationConfiguration: (...args: unknown[]) => mockImmediateImport(...args), }, })); @@ -59,7 +80,10 @@ vi.mock("../../services/trusted-config-operations.js", () => ({ registryUpdateFailed: false, }, }), - applyTrustedConfiguration: vi.fn().mockResolvedValue({ revisionId: "revision_1" }), + applyTrustedConfiguration: vi.fn().mockResolvedValue({ + revisionId: "revision_1", + integrationReload: { message: "Integrations reloaded", reloaded: 1, enabled: 1 }, + }), })); const mockListActivityJobs = vi.fn().mockResolvedValue({ jobs: [], total: 0 }); @@ -67,6 +91,7 @@ const mockGetActivityJob = vi.fn().mockResolvedValue(null); vi.mock("../../services/activity-jobs.js", () => ({ listActivityJobs: (...args: unknown[]) => mockListActivityJobs(...args), getActivityJob: (...args: unknown[]) => mockGetActivityJob(...args), + cancelDataManagerActivityJob: vi.fn().mockResolvedValue(false), })); // DB — multiple tables used across admin.ts @@ -124,6 +149,7 @@ vi.mock("../../db/schema.js", () => ({ finishedAt: "finishedAt", createdBy: "createdBy", error: "error", + result: "result", }, adminJobLog: { jobId: "jobId", seq: "seq" }, integrationConfig: { id: "id", integrationId: "integrationId", config: "config" }, @@ -166,6 +192,7 @@ vi.mock("../../integration-host.js", () => ({ getIntegration: (...args: unknown[]) => mockGetIntegration(...args), reloadIntegrations: (...args: unknown[]) => mockReloadIntegrations(...args), resolveConfigWithSources: (...args: unknown[]) => mockResolveConfigWithSources(...args), + inspectIntegrationRuntime: () => null, })); // Health services @@ -200,6 +227,54 @@ vi.mock("../../services/secrets.js", () => ({ deleteSecret: (...args: unknown[]) => mockDeleteSecret(...args), })); +// Route-only captured-read collaborator. Real credential projection and scope +// validation execute; PostgreSQL capture and file reads belong to their owned suites. +vi.mock("../../services/administrative-configuration-writer-owner", async () => { + const { + openAdministrativeConfigurationRead, + closeAdministrativeConfigurationRead, + withAdministrativeConfigurationReadScope, + } = await import("../../services/administrative-configuration-context"); + return { + withAdministrativeConfigurationRead: async ( + _options: unknown, + run: (read: unknown) => Promise, + ) => { + const integrations = mockGetAllIntegrations(); + const resolved = await mockResolveConfigWithSources(); + const secrets = await mockListSecrets(); + const read = openAdministrativeConfigurationRead({ + services: [], + integrations: integrations as never, + env: {}, + serviceRows: [], + integrationRows: [], + serviceSecretRows: [], + integrationSecretRows: secrets.map((row: Record) => ({ + ...row, + integrationId: "geocoding-maptiler", + })) as never, + integrationReloadSnapshot: Object.fromEntries( + integrations.map((i: typeof MOCK_INTEGRATION) => [i.id, resolved]), + ), + }); + try { + return await withAdministrativeConfigurationReadScope(read, () => run(read)); + } finally { + closeAdministrativeConfigurationRead(read); + } + }, + }; +}); +vi.mock("../../services/service-inventory", () => ({ + readServiceInventory: async () => ({ + acquisition: { state: "available", reason: null }, + services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, + summary: { running: 0, stopped: 0, unknown: 0, total: 0 }, + }), +})); + // Docker compose const mockDockerComposePs = vi.fn().mockResolvedValue([]); vi.mock("../../utils/docker-compose.js", () => ({ @@ -235,7 +310,10 @@ beforeAll(async () => { }); afterAll(() => app?.close()); -afterEach(() => vi.clearAllMocks()); +afterEach(() => { + vi.clearAllMocks(); + mockDbSelect.mockReset().mockImplementation(() => makeSelectChain()); +}); describe("GET /admin/overview", () => { it("returns system health overview with integration stats", async () => { @@ -397,7 +475,7 @@ describe("POST /admin/integrations/health/run", () => { describe("POST /admin/integrations/:id/enable", () => { it.each(["unknown", "committed", "not_committed"] as const)( - "restores integration overrides only with proven non-commit (%s)", + "delegates %s apply evidence without a route-owned restore", async (outcome) => { const error = new TrustedConfigurationApplyError({ ...(outcome === "unknown" @@ -418,9 +496,25 @@ describe("POST /admin/integrations/:id/enable", () => { const res = await app.inject({ method: "POST", url: "/admin/integrations/geocoding-maptiler/enable", + headers: { "idempotency-key": "admin-route-enable-fixture-0001" }, }); expect(res.statusCode).toBe(500); - expect(mockRestoreOwned).toHaveBeenCalledTimes(outcome === "not_committed" ? 1 : 0); + expect(mockRestoreOwned).not.toHaveBeenCalled(); + expect(applyTrustedConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "integrationConfig.apply", + overlay: expect.objectContaining({ + changes: [ + { + kind: "integration-config", + integrationId: "geocoding-maptiler", + set: { enabled: true }, + unset: [], + }, + ], + }), + }), + ); expect(mockReloadIntegrations).not.toHaveBeenCalled(); }, ); @@ -428,6 +522,7 @@ describe("POST /admin/integrations/:id/enable", () => { const res = await app.inject({ method: "POST", url: "/admin/integrations/geocoding-maptiler/enable", + headers: { "idempotency-key": "admin-route-enable-fixture-0001" }, }); expect(res.statusCode).toBe(200); @@ -436,7 +531,8 @@ describe("POST /admin/integrations/:id/enable", () => { expect(mockWriteAuditLog).toHaveBeenCalledWith( expect.objectContaining({ action: "integration.enabled", targetId: "geocoding-maptiler" }), ); - expect(mockReloadIntegrations).toHaveBeenCalled(); + expect(res.json()).toMatchObject({ reloaded: 1, enabled: 1 }); + expect(mockReloadIntegrations).not.toHaveBeenCalled(); }); it("returns 404 for unknown integration", async () => { @@ -445,6 +541,7 @@ describe("POST /admin/integrations/:id/enable", () => { const res = await app.inject({ method: "POST", url: "/admin/integrations/no-such-integration/enable", + headers: { "idempotency-key": "admin-route-missing-fixture-0001" }, }); expect(res.statusCode).toBe(404); @@ -689,6 +786,7 @@ describe("POST /admin/integrations/import", () => { const res = await app.inject({ method: "POST", url: "/admin/integrations/import", + headers: { "idempotency-key": "admin-route-import-fixture-0001" }, payload: { integrations: [{ integrationId: "geocoding-maptiler", config: { enabled: false } }], }, @@ -696,41 +794,76 @@ describe("POST /admin/integrations/import", () => { expect(res.statusCode).toBe(200); expect(res.json()).toMatchObject({ ok: true, imported: 1, reloaded: 1 }); - expect(mockReloadIntegrations).toHaveBeenCalledOnce(); + expect(mockImmediateImport).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: fakeSession.user.id, + body: { + integrations: [{ integrationId: "geocoding-maptiler", config: { enabled: false } }], + }, + }), + ); + expect(mockReloadIntegrations).not.toHaveBeenCalled(); }); }); describe("PUT /admin/credentials/:integrationId/:key", () => { - it("stores the secret AND reloads so the running integration picks it up", async () => { + it("delegates the exact secret intent and exposes its captured runtime receipt", async () => { mockGetSecretFields.mockReturnValueOnce([{ key: "apiKey", title: "API Key" }]); const res = await app.inject({ method: "PUT", url: "/admin/credentials/fuel/apiKey", - payload: { value: "secret-value" }, + headers: { "idempotency-key": "admin-route-credential-set-0001" }, + payload: { value: "secret-value", expectedRevision: "a".repeat(64) }, }); expect(res.statusCode).toBe(200); - expect(res.json().ok).toBe(true); - expect(mockSetSecret).toHaveBeenCalledWith( - "fuel", - "apiKey", - "secret-value", - fakeSession.user.id, + expect(res.json()).toMatchObject({ + integrationId: "fuel", + database: "retained", + runtime: "applied", + }); + expect(mockImmediateCredential).toHaveBeenCalledWith( + expect.objectContaining({ + integrationId: "fuel", + key: "apiKey", + action: "set", + value: "secret-value", + expectedRevision: "a".repeat(64), + actorId: fakeSession.user.id, + }), ); - // The crux: without this reload the freshly-vaulted key never reaches the - // provider (config is captured once at setup() load time). - expect(mockReloadIntegrations).toHaveBeenCalled(); + expect(mockSetSecret).not.toHaveBeenCalled(); + expect(mockReloadIntegrations).not.toHaveBeenCalled(); }); }); describe("DELETE /admin/credentials/:integrationId/:key", () => { - it("deletes the secret AND reloads", async () => { - const res = await app.inject({ method: "DELETE", url: "/admin/credentials/fuel/apiKey" }); + it("delegates removal with the expected revision and exposes its captured runtime receipt", async () => { + const res = await app.inject({ + method: "DELETE", + url: "/admin/credentials/fuel/apiKey", + headers: { "idempotency-key": "admin-route-credential-delete-0001" }, + payload: { expectedRevision: "a".repeat(64) }, + }); expect(res.statusCode).toBe(200); - expect(mockDeleteSecret).toHaveBeenCalledWith("fuel", "apiKey"); - expect(mockReloadIntegrations).toHaveBeenCalled(); + expect(res.json()).toMatchObject({ + integrationId: "fuel", + database: "retained", + runtime: "applied", + }); + expect(mockImmediateCredential).toHaveBeenCalledWith( + expect.objectContaining({ + integrationId: "fuel", + key: "apiKey", + action: "delete", + expectedRevision: "a".repeat(64), + actorId: fakeSession.user.id, + }), + ); + expect(mockDeleteSecret).not.toHaveBeenCalled(); + expect(mockReloadIntegrations).not.toHaveBeenCalled(); }); }); @@ -777,6 +910,7 @@ describe("integration contract tightening", () => { key: { value: "synthetic", source: "config.json" }, }); const response = await app.inject("/admin/credentials/geocoding-maptiler"); + expect(response.statusCode, response.body).toBe(200); expect(response.json().credentials[0]).toMatchObject({ source: "config.json", updatedBy: "actor", diff --git a/apps/api/src/routes/__tests__/auth-discovery.test.ts b/apps/api/src/routes/__tests__/auth-discovery.test.ts index 4f53dc23a..ec72932d9 100644 --- a/apps/api/src/routes/__tests__/auth-discovery.test.ts +++ b/apps/api/src/routes/__tests__/auth-discovery.test.ts @@ -3,6 +3,14 @@ import Fastify from "fastify"; import pino from "pino"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +// UNIT persistence collaborator only: the actual policy loader, Better Auth +// runtime and production Fastify bridge still serve the discovery document. +vi.mock("../../db", async (importActual) => { + const actual = await importActual(); + const { createDbMock } = await import("../../test/db"); + return { ...actual, db: { ...createDbMock().db, _: actual.db._ } }; +}); + beforeEach(() => { vi.stubEnv("BETTER_AUTH_SECRET", "discovery-test-secret-at-least-thirty-two-chars"); vi.stubEnv("BETTER_AUTH_URL", "http://localhost:3001"); diff --git a/apps/api/src/routes/__tests__/data-manager-bump.test.ts b/apps/api/src/routes/__tests__/data-manager-bump.test.ts index f06638c9f..042d2c1f0 100644 --- a/apps/api/src/routes/__tests__/data-manager-bump.test.ts +++ b/apps/api/src/routes/__tests__/data-manager-bump.test.ts @@ -70,6 +70,8 @@ describe("POST /data-manager/transit/bump-transitous-ref", () => { ok: true, unchanged: false, proposed: true, + proposalId: "1d2b29cd-23de-4b19-8c32-86c196833b79", + catalogRevisionId: "a".repeat(64), submoduleSha: "deadbeef", lockedAt: "2026-10-19T00:00:00.000Z", lockedBy: "admin-1", diff --git a/apps/api/src/routes/admin-byte-response.ts b/apps/api/src/routes/admin-byte-response.ts index 9637dfea6..46a9b9bab 100644 --- a/apps/api/src/routes/admin-byte-response.ts +++ b/apps/api/src/routes/admin-byte-response.ts @@ -43,9 +43,9 @@ export async function sendAdminByteResponse( { signal: input.signal }, ); const completion = finished(reply.raw, { signal: input.signal, cleanup: true }); - reply.send(source); + const response = reply.send(source); try { - await completion; + await Promise.all([completion, response]); } finally { source.destroy(); sending.delete(reply); diff --git a/apps/api/src/services/__tests__/admin-job-parent-extension.test.ts b/apps/api/src/services/__tests__/admin-job-parent-extension.test.ts new file mode 100644 index 000000000..b4a4a83f4 --- /dev/null +++ b/apps/api/src/services/__tests__/admin-job-parent-extension.test.ts @@ -0,0 +1,34 @@ +import { randomUUID } from "node:crypto"; +import { opsExecutionBudget } from "@openmapx/core/ops"; +import { expect, it } from "vitest"; +import { prepareApiJobRoot } from "../admin-job-parent"; +import type { JobContext } from "../job-runner"; + +it("freezes a bounded whole-extension parent lifetime while retaining the shorter individual child deadline", () => { + const ctx: JobContext = { + jobId: randomUUID(), + jobType: "extension.install", + execution: { owner: randomUUID(), version: 1 }, + payload: { + manifest: { + services: [{ service: "one", repo: "https://github.com/example/one" }], + integrations: [], + }, + }, + signal: new AbortController().signal, + checkpoint: async () => {}, + log: async () => {}, + setProgress: async () => {}, + }; + const before = Date.now(); + const root = prepareApiJobRoot(ctx, "serviceRepository.publish", "fixed-step"); + expect(root.parent.deadlineAt - before).toBeGreaterThan( + opsExecutionBudget("serviceRepository.publish").processMs + + opsExecutionBudget("service.pull").processMs, + ); + expect(root.requestedDeadlineAt - before).toBeLessThanOrEqual( + opsExecutionBudget("serviceRepository.publish").processMs + 100, + ); + ctx.checkpointResult = { _parentExecution: root.parent }; + expect(prepareApiJobRoot(ctx, "service.start", "next-step").parent).toEqual(root.parent); +}); diff --git a/apps/api/src/services/__tests__/admin-ops-cancellation.test.ts b/apps/api/src/services/__tests__/admin-ops-cancellation.test.ts index 7125f6b4c..aa6e84eea 100644 --- a/apps/api/src/services/__tests__/admin-ops-cancellation.test.ts +++ b/apps/api/src/services/__tests__/admin-ops-cancellation.test.ts @@ -1,12 +1,28 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as realSetTimeout } from "node:timers"; +import { + controlTerminalReceiptDigest, + normalizeExecutionAdmission, + type OpsOperation, + opsOperationFingerprint, + parseOpsJobStatus, + verifyApiJobParentExecution, + verifyApiRetentionProof, +} from "@openmapx/core/ops"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { JobContext } from "../job-runner.js"; -// The module reaches `integration-routes` and therefore Better Auth, which -// asserts on this at import time. -process.env.BETTER_AUTH_SECRET ||= "admin-ops-cancellation-test-stub-secret"; +// Integration-route imports require a synthetic application secret, scoped to this file. +const authSecret = vi.hoisted(() => { + const previous = process.env.BETTER_AUTH_SECRET; + process.env.BETTER_AUTH_SECRET ??= "admin-ops-cancellation-unit-only-secret"; + return { previous }; +}); -// Controlled boundaries only: the operations agent, the service registry and a -// Drizzle-shaped persistence double. The admin-ops lifecycle helpers, the +// UNIT boundary test, not real host authority or PostgreSQL proof. Replaced +// boundaries: operations agent, service registry and Drizzle-shaped persistence. The admin-ops lifecycle helpers, the // administrative job broker, the ops-client wait/recovery helpers and the job // runner's cancellation entry point are all the real modules. const agent = vi.hoisted(() => { @@ -20,14 +36,23 @@ const agent = vi.hoisted(() => { operation: { kind: string; serviceId?: string }; operationKey: string; operationId: string; + admittedAt: number; + deadlineAt: number; + fingerprint: string; }>, cancelCalls: [] as string[], + acknowledgements: [] as string[], events: [] as Array<{ cursor: number; type: "log"; stream: "stdout"; message: string }>, }; return value; }); const database = vi.hoisted(() => ({ - row: { result: null as Record | null }, + row: { + id: "1d2b29cd-23de-4b19-8c32-86c196833b79", + type: "service.stop", + status: "running", + result: null as Record | null, + }, updates: [] as Array>, })); const registryGet = vi.hoisted(() => vi.fn()); @@ -39,15 +64,18 @@ function isTerminal(): boolean { function agentStatus() { const admission = agent.admissions[0]; if (!admission) throw new Error("status requested before admission"); - return { - version: 1, + return parseOpsJobStatus({ + version: 2, + admittedAt: admission.admittedAt, + deadlineAt: admission.deadlineAt, + fingerprint: admission.fingerprint, kind: admission.operation.kind, operationId: admission.operationId, operationKey: admission.operationKey, resourceId: admission.operation.serviceId, state: agent.state, - submittedAt: "2026-10-25T00:00:00.000Z", - updatedAt: "2026-10-25T00:00:01.000Z", + submittedAt: new Date(admission.admittedAt).toISOString(), + updatedAt: new Date().toISOString(), ...(agent.state === "succeeded" ? { result: { changed: true } } : {}), ...(agent.state === "failed" ? { @@ -57,21 +85,78 @@ function agentStatus() { : { terminationRequestedAt: agent.terminationRequestedAt }), } : {}), - }; + }); } // One stable client: the docker helper caches the first client it is given. const agentClient = vi.hoisted(() => ({ execute: vi.fn( - async (operation: { kind: string; serviceId?: string }, options: { operationKey: string }) => { + async ( + operation: OpsOperation, + options: { + operationKey: string; + execution?: { requestedDeadlineAt: number }; + parentAssertion?: string; + }, + ) => { + if (operation.kind === "controlRetention.acknowledge") { + const proof = verifyApiRetentionProof(token, operation.proof); + const retained = projectedOperations()?.[operation.operationKey] as + | { + attempt?: { attemptId: string }; + receiptDelivery?: { state: string; terminalDigest: string }; + } + | undefined; + expect(retained?.receiptDelivery?.state).toBe("pending"); + expect(proof).toMatchObject({ + apiJobId: JOB_ID, + originalApiOwnerVersion: 1, + checkpointOwnerVersion: 1, + attemptId: retained?.attempt?.attemptId, + operationKey: operation.operationKey, + fingerprint: operation.fingerprint, + terminalDigest: controlTerminalReceiptDigest(agentStatus()), + }); + expect(operation.terminalDigest).toBe(proof.terminalDigest); + agent.acknowledgements.push(operation.operationKey); + return { execution: "sync", kind: operation.kind, value: { acknowledged: true } }; + } + if (!options.parentAssertion) throw Error("Original parent assertion required"); + const parent = verifyApiJobParentExecution(options.parentAssertion, token, { + operationKey: options.operationKey, + operationFingerprint: opsOperationFingerprint(operation), + requestedDeadlineAt: options.execution?.requestedDeadlineAt, + }); + if (parent.issuer !== "api") throw Error("Original API parent required"); + expect(parent).toMatchObject({ issuer: "api", apiJobId: JOB_ID, apiOwnerVersion: 1 }); + const retained = projectedOperations()?.[options.operationKey]; + expect(retained).toMatchObject({ + operation, + attempt: { apiJobId: JOB_ID, apiOwnerVersion: 1, attemptId: parent.attemptId }, + }); + const identity = normalizeExecutionAdmission({ + kind: operation.kind, + operationKey: options.operationKey, + operationFingerprint: opsOperationFingerprint(operation), + admittedAt: Date.now(), + requestedDeadlineAt: options.execution?.requestedDeadlineAt, + parent, + }); const operationId = `job1_${operation.kind.replace(/\W/g, "_")}_0123456789`; - agent.admissions.push({ operation, operationKey: options.operationKey, operationId }); + agent.admissions.push({ + operation, + operationKey: options.operationKey, + operationId, + ...identity, + }); return { execution: "async", kind: operation.kind, operationId, operationKey: options.operationKey, state: "queued", + fingerprint: identity.fingerprint, + deadlineAt: identity.deadlineAt, }; }, ), @@ -88,7 +173,7 @@ const agentClient = vi.hoisted(() => ({ agent.cancelCalls.push(operationId); if (agent.cancelBehavior === "hold") { agent.state = "termination_pending"; - agent.terminationRequestedAt = "2026-10-25T00:00:02.000Z"; + agent.terminationRequestedAt = new Date().toISOString(); } else if (agent.cancelBehavior === "already-succeeded") { agent.state = "succeeded"; } else { @@ -108,31 +193,56 @@ vi.mock("../ops-client.js", async (importActual) => ({ vi.mock("../service-registry.js", () => ({ getServiceRegistry: () => ({ get: registryGet }), })); -vi.mock("../../db", () => ({ - db: { +// The in-memory row replaces PostgreSQL; real checkpoint merge/identity guards +// still run. This double does not establish locking, CAS or host authority proof. +vi.mock("../../db", () => { + function update(value: Record) { + const apply = () => { + const p = database.row.result?.opsProjection as + | { operations: Record } + | undefined; + if ( + value.status === "canceled" && + (database.row.status !== "cancel_pending" || + !p || + Object.values(p.operations).some( + (o) => !o.terminal || o.receiptDelivery?.state !== "confirmed", + )) + ) + return []; + database.updates.push(value); + if (typeof value.status === "string") database.row.status = value.status; + if (value.result && typeof value.result === "object" && !("queryChunks" in value.result)) + database.row.result = structuredClone(value.result as Record); + if (value.status === "cancel_pending") + database.row.result = { + ...database.row.result, + _cancelRequestedAt: new Date().toISOString(), + }; + return [structuredClone(database.row)]; + }; + return { + returning: async () => apply(), + // biome-ignore lint/suspicious/noThenProperty: Drizzle updates are intentionally thenable. + then: (resolve: (v: unknown[]) => unknown) => Promise.resolve(apply()).then(resolve), + }; + } + const db = { update: vi.fn(() => ({ - set: vi.fn((value: Record) => ({ - where: vi.fn(() => ({ - // The queued-job compare-and-set matches nothing: the job is already - // running, so that update is not recorded as applied. - returning: vi.fn().mockResolvedValue([]), - // biome-ignore lint/suspicious/noThenProperty: the Drizzle query builder is thenable. - then: (resolve: (value: unknown[]) => unknown) => { - database.updates.push(value); - return Promise.resolve([]).then(resolve); - }, - })), - })), + set: (value: Record) => ({ where: () => update(value) }), })), select: vi.fn(() => ({ - from: vi.fn(() => ({ - where: vi.fn(() => ({ - limit: vi.fn(async () => [{ status: "running", result: database.row.result }]), - })), - })), + from: () => ({ + where: () => ({ + limit: async () => [structuredClone(database.row)], + for: async () => [structuredClone(database.row)], + }), + }), })), - }, -})); + transaction: async (run: (tx: unknown) => Promise) => run(db), + }; + return { db }; +}); const JOB_ID = "1d2b29cd-23de-4b19-8c32-86c196833b79"; @@ -141,6 +251,24 @@ let serviceRestart: typeof import("../admin-ops.js").serviceRestart; let createDurableOpsKey: typeof import("../ops-client.js").createDurableOpsKey; let jobRunner: typeof import("../job-runner.js").jobRunner; +const token = Buffer.alloc(32, 19).toString("base64url"); +let tokenDirectory: string; +let previousTokenFile: string | undefined; +beforeAll(async () => { + tokenDirectory = await mkdtemp(join(tmpdir(), "omx-cancel-unit-")); + const file = join(tokenDirectory, "api-token"); + await writeFile(file, token, { mode: 0o600 }); + previousTokenFile = process.env.OPS_AGENT_TOKEN_FILE; + process.env.OPS_AGENT_TOKEN_FILE = file; +}); +afterAll(async () => { + if (authSecret.previous === undefined) delete process.env.BETTER_AUTH_SECRET; + else process.env.BETTER_AUTH_SECRET = authSecret.previous; + if (previousTokenFile === undefined) delete process.env.OPS_AGENT_TOKEN_FILE; + else process.env.OPS_AGENT_TOKEN_FILE = previousTokenFile; + await rm(tokenDirectory, { recursive: true, force: true }); +}); + const cleanups: Array<() => Promise> = []; beforeEach(async () => { @@ -150,8 +278,12 @@ beforeEach(async () => { agent.terminationRequestedAt = undefined; agent.admissions.length = 0; agent.cancelCalls.length = 0; + agent.acknowledgements.length = 0; agent.events = []; - database.row.result = null; + database.row.status = "running"; + database.row.result = { + _execution: { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 1 }, + }; database.updates.length = 0; vi.clearAllMocks(); registryGet.mockReturnValue({ manifest: { id: "redis" }, enabled: true, isBuiltIn: true }); @@ -179,17 +311,30 @@ function startJob(action: (service: string, ctx: JobContext) => Promise): const log = vi.fn().mockResolvedValue(undefined); const ctx: JobContext = { jobId: JOB_ID, + execution: { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 1 }, payload: {}, signal: controller.signal, log, setProgress: vi.fn().mockResolvedValue(undefined), - checkpoint: async (result) => { - // A JSON round trip models the persisted row, as the runner's own - // checkpoint stores the result and mirrors it onto the context. - database.row.result = JSON.parse(JSON.stringify(result)); - ctx.checkpointResult = result; + checkpoint: async (result, _progress, admissionKey, stepAddress) => { + // Use the production merge/identity checks for managed checkpoints. + if ((result.opsProjection as { version?: number } | undefined)?.version === 3) { + const { checkpointOwnedAdminJob } = await import("../admin-job-checkpoint"); + ctx.checkpointResult = await checkpointOwnedAdminJob( + JOB_ID, + ctx.execution, + result, + undefined, + admissionKey, + stepAddress, + ctx.checkpointResult ?? {}, + ); + } else { + database.row.result = { ...database.row.result, ...structuredClone(result) }; + ctx.checkpointResult = structuredClone(database.row.result); + } }, - checkpointResult: null, + checkpointResult: structuredClone(database.row.result), }; const active = (jobRunner as unknown as { active: Map }).active; active.set(JOB_ID, controller); @@ -219,7 +364,10 @@ function startJob(action: (service: string, ctx: JobContext) => Promise): async function until(predicate: () => boolean): Promise { for (let attempt = 0; attempt < 40 && !predicate(); attempt += 1) { await vi.advanceTimersByTimeAsync(1); + // Let native token-file reads settle while fake timers drive bounded polling. + await new Promise((resolve) => realSetTimeout(resolve, 1)); } + expect(predicate(), "bounded condition did not settle").toBe(true); } function observe(promise: Promise) { @@ -245,32 +393,58 @@ function projectedOperations(): Record> | undefi } const ACTIONS = [ - { name: "stop", kind: "service.stop", run: () => serviceStop, failure: "stop exited" }, + { name: "stop", kind: "service.stop", run: () => serviceStop }, { name: "restart", kind: "service.restart", run: () => serviceRestart, - failure: "restart exited", }, ] as const; describe.each(ACTIONS)("single-service $name cancellation", (scenario) => { - const expectedKey = () => + const stepAddress = () => createDurableOpsKey(`admin-job.service.${scenario.name}`, `${JOB_ID}:redis`); + const expectedKey = () => { + const admission = agent.admissions[0]; + if (!admission) throw Error("No admitted operation"); + return admission.operationKey; + }; - it("persists the admitted operation before waiting, with the legacy deterministic key", async () => { + it("persists the admitted operation before waiting, under its stable step address and original attempt", async () => { const job = startJob(scenario.run()); - await until(() => projectedOperations() !== undefined); + await until( + () => + agent.admissions.length === 1 && + Object.values(projectedOperations() ?? {}).some((o) => o.operationId !== undefined), + ); expect(agent.admissions).toHaveLength(1); expect(agent.admissions[0]?.operation).toEqual({ kind: scenario.kind, serviceId: "redis" }); - expect(agent.admissions[0]?.operationKey).toBe(expectedKey()); + expect(expectedKey()).toMatch(/^opk1_t1_/); + expect(database.row.result?.opsProjection).toMatchObject({ + version: 3, + steps: { + [stepAddress()]: { + operationKey: expectedKey(), + requestedDeadlineAt: agent.admissions[0]?.deadlineAt, + }, + }, + }); expect(projectedOperations()).toEqual({ [expectedKey()]: expect.objectContaining({ kind: scenario.kind, operationKey: expectedKey(), operationId: agent.admissions[0]?.operationId, terminal: false, + attempt: expect.objectContaining({ + apiJobId: JOB_ID, + apiOwnerVersion: 1, + attemptId: expect.any(String), + childOperationFingerprint: opsOperationFingerprint({ + kind: scenario.kind, + serviceId: "redis", + }), + }), }), }); expect(job.outcome.state).toBe("pending"); @@ -278,7 +452,11 @@ describe.each(ACTIONS)("single-service $name cancellation", (scenario) => { it("reports cancellation only after the exact agent operation is contained", async () => { const job = startJob(scenario.run()); - await until(() => projectedOperations() !== undefined); + await until( + () => + agent.admissions.length === 1 && + Object.values(projectedOperations() ?? {}).some((o) => o.operationId !== undefined), + ); const operationId = agent.admissions[0]?.operationId; expect(operationId).toBeDefined(); expect(projectedOperations()?.[expectedKey()]).toMatchObject({ operationId, terminal: false }); @@ -290,14 +468,15 @@ describe.each(ACTIONS)("single-service $name cancellation", (scenario) => { await vi.advanceTimersByTimeAsync(300); expect(agent.state).toBe("termination_pending"); expect(cancellation.settled).toBe(false); - expect(job.controller.signal.aborted).toBe(false); - expect(database.updates).toContainEqual({ status: "cancel_pending" }); + expect(job.controller.signal.aborted).toBe(true); + expect(database.updates).toContainEqual(expect.objectContaining({ status: "cancel_pending" })); expect(database.updates).not.toContainEqual(expect.objectContaining({ status: "canceled" })); // The agent's authoritative termination is the containment evidence. agent.state = "failed"; await vi.advanceTimersByTimeAsync(600); + await until(() => cancellation.settled); expect(cancellation).toMatchObject({ settled: true, value: true }); expect(agent.cancelCalls).toEqual([operationId]); expect(job.controller.signal.aborted).toBe(true); @@ -306,52 +485,97 @@ describe.each(ACTIONS)("single-service $name cancellation", (scenario) => { ); await vi.advanceTimersByTimeAsync(1_000); expect(job.outcome.state).toBe("rejected"); - expect(job.outcome.error).toMatchObject({ message: expect.stringContaining(scenario.failure) }); + expect(job.outcome.error).toMatchObject({ name: "OpsObservationPending" }); + expect(agent.acknowledgements).toEqual([expectedKey()]); + expect(projectedOperations()?.[expectedKey()]).toMatchObject({ + terminal: true, + receiptDelivery: { state: "confirmed" }, + }); expect(job.log).not.toHaveBeenCalledWith(expect.stringMatching(/ (stopped|restarted)\.$/)); }); - it("stays pending while the agent has not terminated and never aborts the local work", async () => { + it("retains pending containment while retiring only the local observer", async () => { const job = startJob(scenario.run()); - await until(() => projectedOperations() !== undefined); + await until( + () => + agent.admissions.length === 1 && + Object.values(projectedOperations() ?? {}).some((o) => o.operationId !== undefined), + ); const cancellation = observe(jobRunner.cancel(JOB_ID)); await vi.advanceTimersByTimeAsync(5_000); + await until(() => cancellation.settled); expect(cancellation).toMatchObject({ settled: true, value: false }); expect(agent.cancelCalls).toEqual([agent.admissions[0]?.operationId]); - expect(job.controller.signal.aborted).toBe(false); - expect(job.outcome.state).toBe("pending"); - expect(database.updates.at(-1)).toEqual({ status: "cancel_pending" }); + expect(job.controller.signal.aborted).toBe(true); + expect(job.outcome.state).toBe("rejected"); + expect(database.row.status).toBe("cancel_pending"); + expect(agent.acknowledgements).toEqual([]); expect(database.updates).not.toContainEqual(expect.objectContaining({ status: "canceled" })); }); it("does not turn an operation that already succeeded into a cancellation", async () => { agent.cancelBehavior = "already-succeeded"; const job = startJob(scenario.run()); - await until(() => projectedOperations() !== undefined); + await until( + () => + agent.admissions.length === 1 && + Object.values(projectedOperations() ?? {}).some((o) => o.operationId !== undefined), + ); const cancellation = observe(jobRunner.cancel(JOB_ID)); await vi.advanceTimersByTimeAsync(2_000); + await until(() => cancellation.settled); expect(cancellation).toMatchObject({ settled: true, value: false }); - expect(job.controller.signal.aborted).toBe(false); + expect(job.controller.signal.aborted).toBe(true); expect(database.updates).not.toContainEqual(expect.objectContaining({ status: "canceled" })); - expect(database.updates.at(-1)).toEqual({ status: "running" }); - expect(job.outcome.state).toBe("resolved"); + expect(database.row.status).toBe("cancel_pending"); + expect(projectedOperations()?.[expectedKey()]).toMatchObject({ + terminal: true, + result: { changed: true }, + receiptDelivery: { state: "confirmed", terminalReceipt: { state: "succeeded" } }, + }); + const { resumeAdminJobObservation } = await import("../admin-job-ops"); + const recovered: JobContext = { + jobId: JOB_ID, + payload: {}, + execution: { owner: "4c0c9623-252c-41b6-8f99-41915ad38b12", version: 1 }, + signal: new AbortController().signal, + checkpointResult: structuredClone(database.row.result), + checkpoint: async () => {}, + log: async () => {}, + setProgress: async () => {}, + }; + await expect(resumeAdminJobObservation(recovered)).resolves.toMatchObject({ + service: "redis", + action: scenario.name, + }); + expect(agent.admissions).toHaveLength(1); }); it("does not claim containment for a failure with no termination marker", async () => { agent.cancelBehavior = "already-failed"; const job = startJob(scenario.run()); - await until(() => projectedOperations() !== undefined); + await until( + () => + agent.admissions.length === 1 && + Object.values(projectedOperations() ?? {}).some((o) => o.operationId !== undefined), + ); const cancellation = observe(jobRunner.cancel(JOB_ID)); await vi.advanceTimersByTimeAsync(2_000); + await until(() => cancellation.settled); expect(cancellation).toMatchObject({ settled: true, value: false }); - expect(job.controller.signal.aborted).toBe(false); + expect(job.controller.signal.aborted).toBe(true); expect(database.updates).not.toContainEqual(expect.objectContaining({ status: "canceled" })); - expect(database.updates.at(-1)).toEqual({ status: "running" }); + expect(database.row.status).toBe("cancel_pending"); + expect(projectedOperations()?.[expectedKey()]).toMatchObject({ + terminal: true, + receiptDelivery: { state: "confirmed", terminalReceipt: { state: "failed" } }, + }); expect(job.outcome.state).toBe("rejected"); }); }); @@ -363,12 +587,18 @@ describe("single-service operation projection", () => { job.log.mockImplementation(async (_line: string, _stream?: string, sourceEventId?: string) => { if (sourceEventId) throw new Error("database unavailable"); }); - await until(() => projectedOperations() !== undefined); + await until( + () => + agent.admissions.length === 1 && + Object.values(projectedOperations() ?? {}).some((o) => o.operationId !== undefined), + ); await vi.advanceTimersByTimeAsync(500); agent.state = "succeeded"; await vi.advanceTimersByTimeAsync(1_000); expect(agent.cancelCalls).toEqual([]); + await until(() => job.outcome.state !== "pending"); + expect(job.outcome.error).toBeUndefined(); expect(job.outcome.state).toBe("resolved"); expect(Object.values(projectedOperations() ?? {})).toEqual([ expect.objectContaining({ kind: "service.stop", terminal: true }), diff --git a/apps/api/src/services/__tests__/community-repositories.test.ts b/apps/api/src/services/__tests__/community-repositories.test.ts index 4dc7f1556..5da989b7e 100644 --- a/apps/api/src/services/__tests__/community-repositories.test.ts +++ b/apps/api/src/services/__tests__/community-repositories.test.ts @@ -12,8 +12,18 @@ vi.mock("../ops-client", () => ({ ), executeAndWait: vi.fn(), })); +vi.mock("../admin-job-ops", () => ({ + executeAdminJobOperation: vi.fn((_ctx, operation, _namespace, options) => + executeAndWait(createApiOpsClient(), operation, options.operationKey, { signal: undefined }), + ), +})); +vi.mock("../admin-job-projection", () => ({ + readProjection: (result: { opsProjection: unknown }) => result.opsProjection, +})); -import { executeAndWait } from "../ops-client"; +import { executeAdminJobOperation } from "../admin-job-ops"; +import type { JobContext } from "../job-runner"; +import { createApiOpsClient, executeAndWait } from "../ops-client"; let repositories: typeof import("../community-repositories.js"); beforeAll(async () => { @@ -152,8 +162,42 @@ describe("inspectServiceRepository", () => { }); describe("publishServiceRepository", () => { + it("uses the original extension job broker before publishing a repository", async () => { + const ctx = { + jobId: "original-extension-job", + payload: {}, + signal: new AbortController().signal, + checkpoint: vi.fn(), + log: vi.fn(), + setProgress: vi.fn(), + } satisfies JobContext; + const result = { + repository: repositories.hashUrl(URL), + commit: COMMIT, + transactionId: TRANSACTION, + serviceIds: ["svc-one"], + }; + vi.mocked(executeAdminJobOperation).mockResolvedValueOnce(result as never); + const request = { + url: URL, + commit: COMMIT, + extensionId: "bundle-x", + recovery: { selectionBefore: null, touchedServiceIds: [], previouslyEnabledServiceIds: [] }, + }; + await repositories.publishServiceRepository(request, { + operationKey: "opk1_original_extension_publish", + ctx, + }); + expect(executeAdminJobOperation).toHaveBeenCalledWith( + ctx, + { kind: "serviceRepository.publish", ...request }, + "extension.repository.publish", + { operationKey: "opk1_original_extension_publish" }, + ); + expect(executeMock).not.toHaveBeenCalled(); + }); it("refuses an answer that installed a different commit", async () => { - executeMock.mockResolvedValueOnce({ + vi.mocked(executeAdminJobOperation).mockResolvedValueOnce({ repository: repositories.hashUrl(URL), commit: "e".repeat(40), transactionId: TRANSACTION, @@ -172,7 +216,7 @@ describe("publishServiceRepository", () => { previouslyEnabledServiceIds: [], }, }, - { operationKey: "opk1_key" }, + { operationKey: "opk1_key", ctx: context() }, ), ).rejects.toThrow("installed a different repository"); }); @@ -202,6 +246,29 @@ describe("readCommunityServices", () => { }); }); +function context(): JobContext { + return { + jobId: "original-extension-job", + payload: {}, + signal: new AbortController().signal, + log: vi.fn(), + setProgress: vi.fn(), + checkpoint: vi.fn(), + checkpointResult: { + opsProjection: { + steps: { original: { operationKey: "original" } }, + operations: { + original: { + operationKey: "original", + kind: "serviceRepository.publish", + attempt: { apiJobId: "original-extension-job" }, + operation: { kind: "serviceRepository.publish", url: URL, commit: COMMIT }, + }, + }, + }, + }, + }; +} describe("reconcileCommunityRepositoryTransactions", () => { /** The settle requests sent to the agent, in order. */ function settled() { @@ -211,10 +278,38 @@ describe("reconcileCommunityRepositoryTransactions", () => { .map((operation) => `${operation.outcome}:${operation.repository}`); } - function agentAnswers(repositoriesList: unknown[]) { + let recoveryContext: JobContext; + function agentAnswers(repositoriesList: ReturnType[]) { + recoveryContext = context(); + const steps: Record = {}, + operations: Record = {}; + for (const entry of repositoriesList) { + const url = entry.repository === repositories.hashUrl(URL) ? URL : `${URL}-two`; + steps[entry.repository] = { operationKey: entry.repository }; + operations[entry.repository] = { + operationKey: entry.repository, + kind: "serviceRepository.publish", + attempt: { apiJobId: recoveryContext.jobId }, + operation: { kind: "serviceRepository.publish", url, commit: COMMIT }, + }; + } + recoveryContext.checkpointResult = { opsProjection: { steps, operations } }; executeMock.mockImplementation(async (_client, operation) => { - const request = operation as { kind: string; repository?: string; outcome?: string }; + const request = operation as { + kind: string; + repository?: string; + outcome?: string; + url?: string; + }; if (request.kind === "communityServices.inspect") return snapshot(repositoriesList) as never; + if (request.kind === "serviceRepository.publish") { + const id = repositories.hashUrl(request.url!); + return { + repository: id, + transactionId: repositoriesList.find((entry) => entry.repository === id)!.pending + .transactionId, + } as never; + } return { repository: request.repository, outcome: request.outcome, changed: true } as never; }); } @@ -228,7 +323,10 @@ describe("reconcileCommunityRepositoryTransactions", () => { const restoreSelection = vi.fn(); await expect( - repositories.reconcileCommunityRepositoryTransactions({ restoreSelection }), + repositories.reconcileCommunityRepositoryTransactions({ + ctx: recoveryContext, + restoreSelection, + }), ).resolves.toEqual({ runtimeRecoveryNeeded: false, orphanedServiceIds: [], @@ -239,13 +337,14 @@ describe("reconcileCommunityRepositoryTransactions", () => { }); it("commits a transaction the database recorded", async () => { - const repository = "1".repeat(16); + const repository = repositories.hashUrl(URL); agentAnswers([pendingRepository(repository, TRANSACTION)]); dbMock.queueSelect([{ lastTransactionId: TRANSACTION }]); const restoreSelection = vi.fn(); const persistRuntimeRecovery = vi.fn(); const result = await repositories.reconcileCommunityRepositoryTransactions({ + ctx: recoveryContext, restoreSelection, persistRuntimeRecovery, }); @@ -257,8 +356,8 @@ describe("reconcileCommunityRepositoryTransactions", () => { }); it("records runtime repair, rolls back, restores the selection, then releases", async () => { - const one = "1".repeat(16); - const two = "2".repeat(16); + const one = repositories.hashUrl(URL); + const two = repositories.hashUrl(`${URL}-two`); const otherTransaction = `txn1_${"c".repeat(24)}`; agentAnswers([pendingRepository(one, TRANSACTION), pendingRepository(two, otherTransaction)]); // The database still records an earlier install of the first repository. @@ -273,6 +372,7 @@ describe("reconcileCommunityRepositoryTransactions", () => { }); const result = await repositories.reconcileCommunityRepositoryTransactions({ + ctx: recoveryContext, restoreSelection, persistRuntimeRecovery, }); @@ -297,11 +397,14 @@ describe("reconcileCommunityRepositoryTransactions", () => { }); it("finishes a rollback whose selection restore failed earlier", async () => { - const repository = "1".repeat(16); + const repository = repositories.hashUrl(URL); agentAnswers([pendingRepository(repository, TRANSACTION, "rolledBack")]); const restoreSelection = vi.fn(); - await repositories.reconcileCommunityRepositoryTransactions({ restoreSelection }); + await repositories.reconcileCommunityRepositoryTransactions({ + ctx: recoveryContext, + restoreSelection, + }); // A rolled-back transaction is never committed, whatever the database says. expect(dbMock.db.select).not.toHaveBeenCalled(); @@ -310,12 +413,13 @@ describe("reconcileCommunityRepositoryTransactions", () => { }); it("keeps the recovery note when restoring the selection fails, so recovery can retry", async () => { - const repository = "1".repeat(16); + const repository = repositories.hashUrl(URL); agentAnswers([pendingRepository(repository, TRANSACTION)]); dbMock.queueSelect([]); await expect( repositories.reconcileCommunityRepositoryTransactions({ + ctx: recoveryContext, restoreSelection: vi.fn().mockRejectedValue(new Error("selection unavailable")), }), ).rejects.toThrow("selection unavailable"); diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts index 80e898179..8b6fe68bc 100644 --- a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -137,7 +137,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( role: "ops-agent", serviceIds: host.loaded.map((service) => service.manifest.id), }); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); app = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, @@ -3500,7 +3500,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( slot = actual.plan.reservations[0]!, bound = lookupControlChildPhaseBinding(authority, slot), attempt = lookupControlChildPhaseAttempt(authority, slot), - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), record = journal .records() .find((value) => value.operationKey === prepared.steps[0]!.operationKey); @@ -3621,7 +3621,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( // fixture teardown instead of abandoning a timed-out API/SQL scope. const observer = setTimeout(() => { const journal = JSON.parse( - readFileSync(join(host.infraDir, "data/ops-agent/jobs-v1.json"), "utf8"), + readFileSync(join(host.infraDir, "state/ops-agent/jobs-v1.json"), "utf8"), ); console.info( JSON.stringify({ @@ -3947,7 +3947,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await app.close(); closeControlInstallation(installation); const journal = await openOpsJobJournal( - join(host.infraDir, "data/ops-agent/jobs-v1.json"), + join(host.infraDir, "state/ops-agent/jobs-v1.json"), ); installation = openControlInstallation({ infraDir: host.infraDir, @@ -4530,7 +4530,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await app.close(); closeControlInstallation(installation); const journal = await openOpsJobJournal( - join(host.infraDir, "data/ops-agent/jobs-v1.json"), + join(host.infraDir, "state/ops-agent/jobs-v1.json"), ); installation = openControlInstallation({ infraDir: host.infraDir, @@ -5546,7 +5546,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( original = readFileSync(path); await app.close(); closeControlInstallation(installation); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal( + join(host.infraDir, "state/ops-agent/jobs-v1.json"), + ); installation = openControlInstallation({ infraDir: host.infraDir, role: "ops-agent", @@ -5855,7 +5857,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(owner.apiControls.settle!.result).toBeNull(); await app.close(); closeControlInstallation(installation); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal( + join(host.infraDir, "state/ops-agent/jobs-v1.json"), + ); installation = openControlInstallation({ infraDir: host.infraDir, role: "ops-agent", @@ -6011,7 +6015,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await app.close(); closeControlInstallation(installation); const journal = await openOpsJobJournal( - join(host.infraDir, "data/ops-agent/jobs-v1.json"), + join(host.infraDir, "state/ops-agent/jobs-v1.json"), ), original = journal .records() @@ -6816,7 +6820,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( // another request/root or infer success from the API's cached result. await app.close(); closeControlInstallation(installation); - const journalPath = join(host.infraDir, "data/ops-agent/jobs-v1.json"), + const journalPath = join(host.infraDir, "state/ops-agent/jobs-v1.json"), journal = await openOpsJobJournal(journalPath), originalEntry = journal .records() @@ -7762,7 +7766,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( } finally { core.closeArtifactTargetLeaseHandle(release); } - const journalPath = join(host.infraDir, "data/ops-agent/jobs-v1.json"), + const journalPath = join(host.infraDir, "state/ops-agent/jobs-v1.json"), originalJournal = readFileSync(journalPath), maintenance = await import("../configuration-maintenance"); const inspectionAt = Date.now() + 3600000; diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts index c7ef18b1c..a8792800e 100644 --- a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -513,7 +513,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( : raw; }, }, - journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), controlInstallation: installation, }); posts.clear(); @@ -1394,7 +1394,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(existsSync(pullPath)).toBe(false); expect(retired).toBeGreaterThan(0); const jobs = JSON.parse( - readFileSync(join(host.infraDir, "data/ops-agent/jobs-v1.json"), "utf8"), + readFileSync(join(host.infraDir, "state/ops-agent/jobs-v1.json"), "utf8"), ).jobs as { operationKey: string }[]; expect(jobs.some((job) => pullPath.endsWith(`/${job.operationKey}.json`))).toBe(false); expect( diff --git a/apps/api/src/services/__tests__/extension-installer.test.ts b/apps/api/src/services/__tests__/extension-installer.test.ts index 8ae6fc6bd..3567d960f 100644 --- a/apps/api/src/services/__tests__/extension-installer.test.ts +++ b/apps/api/src/services/__tests__/extension-installer.test.ts @@ -48,7 +48,7 @@ vi.mock("../trusted-config-operations", () => ({ })); vi.mock("../admin-ops", () => ({ - renderAndPersistCompose: vi.fn().mockResolvedValue(undefined), + renderAndPersistCompose: vi.fn().mockResolvedValue({ revisionId: `cfg1_${"a".repeat(43)}` }), serviceStart: vi.fn().mockResolvedValue(undefined), })); @@ -81,6 +81,47 @@ vi.mock("../community-repositories", () => ({ removeServiceRepository: vi.fn(), })); +// Business orchestration fixture only. Authenticated cursor/SQL ownership are +// exercised independently by extension-workflow tests, not weakened here. +vi.mock("../extension-workflow", () => { + const cursors = new WeakMap>(); + return { + openExtensionWorkflow: vi.fn(async (ctx: object) => { + if (!cursors.has(ctx)) cursors.set(ctx, new Map()); + return ctx; + }), + hasExtensionPublications: vi.fn((ctx: object) => + [...(cursors.get(ctx)?.keys() ?? [])].some( + (name) => name.startsWith("repository.publish:") || name.startsWith("integration.publish:"), + ), + ), + completedExtensionStep: vi.fn((ctx: object, name: string) => cursors.get(ctx)?.get(name)), + extensionStep: vi.fn( + async (ctx: object, name: string, _input: unknown, work: () => Promise) => { + const cursor = cursors.get(ctx)!; + if (cursor.has(name)) return cursor.get(name); + const result = await work(); + cursor.set(name, result ?? null); + return result; + }, + ), + extensionDatabaseStep: vi.fn( + async ( + ctx: object, + name: string, + _input: unknown, + work: (tx: unknown) => Promise, + ) => { + const cursor = cursors.get(ctx)!; + if (cursor.has(name)) return cursor.get(name); + const result = await (dbMock.db.transaction as (fn: typeof work) => Promise)(work); + cursor.set(name, result ?? null); + return result; + }, + ), + }; +}); + vi.mock("../extension-recovery", () => ({ withExtensionChangeLock: vi.fn((work: () => Promise) => work()), recoverInterruptedExtensionChanges: vi.fn().mockResolvedValue(undefined), @@ -692,7 +733,7 @@ describe("installExtension", () => { await expect( installExtension(ctx, { manifest: INTEGRATION_ONLY, sourceTrust: "community" }), - ).resolves.toEqual({ id: "bundle-x", components: 1 }); + ).rejects.toMatchObject({ reason: "recovery_required" }); // Recovery commits the pending transaction the row records. expect(settleIntegrationMock).toHaveBeenCalledTimes(1); @@ -700,8 +741,8 @@ describe("installExtension", () => { expect.objectContaining({ outcome: "rollback" }), expect.anything(), ); - expect(ctx.log).toHaveBeenCalledWith( - expect.stringContaining("could not settle integration intg-one"), + expect(ctx.log).not.toHaveBeenCalledWith( + expect.stringContaining("agent unavailable"), "stderr", ); }); @@ -711,7 +752,9 @@ describe("installExtension", () => { settleIntegrationMock.mockRejectedValueOnce(new Error(secret)); const ctx = makeCtx(); - await installExtension(ctx, { manifest: INTEGRATION_ONLY, sourceTrust: "community" }); + await expect( + installExtension(ctx, { manifest: INTEGRATION_ONLY, sourceTrust: "community" }), + ).rejects.toMatchObject({ reason: "recovery_required" }); expect(JSON.stringify(vi.mocked(ctx.log).mock.calls)).not.toMatch( /sentinel-|example\.test|\/private/, @@ -743,10 +786,10 @@ describe("installExtension", () => { await expect( installExtension(makeCtx(), { manifest: TWO_REPO_MANIFEST, sourceTrust: "community" }), - ).resolves.toEqual({ id: "bundle-x", components: 2 }); + ).rejects.toMatchObject({ reason: "recovery_required" }); // Recovery commits a pending transaction the database records. - expect(settleMock).toHaveBeenCalledTimes(2); + expect(settleMock).toHaveBeenCalledTimes(1); expect(settleMock).not.toHaveBeenCalledWith( expect.objectContaining({ outcome: "rollback" }), expect.anything(), @@ -882,17 +925,17 @@ describe("installExtension", () => { expect(renderMock).toHaveBeenCalled(); }); - it("continues the rollback when the agent cannot undo a repository yet", async () => { + it("retains the original rollback when the agent cannot undo a repository yet", async () => { serviceStartMock.mockRejectedValueOnce(new Error("start failed")); settleMock.mockRejectedValueOnce(new Error("agent unavailable")); await expect( installExtension(makeCtx(), { manifest: MANIFEST, sourceTrust: "community" }), - ).rejects.toThrow("start failed"); + ).rejects.toMatchObject({ reason: "recovery_required" }); expect(settleMock).toHaveBeenCalledTimes(1); - expect(writeSelectionMock).toHaveBeenCalledTimes(2); - expect(renderMock).toHaveBeenCalledTimes(2); + expect(writeSelectionMock).toHaveBeenCalledTimes(1); + expect(renderMock).toHaveBeenCalledTimes(1); }); it("rolls back repositories newest first", async () => { @@ -925,7 +968,7 @@ describe("installExtension", () => { await expect( installExtension(makeCtx(), { manifest: MANIFEST, sourceTrust: "community" }), - ).rejects.toThrow("start failed"); + ).rejects.toMatchObject({ reason: "recovery_required" }); expect(settleMock.mock.calls.map(([request]) => request.outcome)).toEqual(["rollback"]); }); @@ -1063,19 +1106,15 @@ describe("installExtension", () => { ); }); - it("keeps rolling back when the agent cannot undo an integration yet", async () => { + it("keeps the original rollback pending when the agent cannot undo an integration yet", async () => { reloadMock.mockRejectedValueOnce(new Error("reload-fail")); settleIntegrationMock.mockRejectedValueOnce(new Error("agent unavailable")); await expect( installExtension(makeCtx(), { manifest: MANIFEST, sourceTrust: "community" }), - ).rejects.toThrow(/reload-fail/); + ).rejects.toMatchObject({ reason: "recovery_required" }); - // Recovery rolls the integration back later; the services are still undone. - expect(settleMock).toHaveBeenCalledWith( - expect.objectContaining({ repository: REPO_ONE, outcome: "rollback" }), - expect.anything(), - ); + expect(settleMock).not.toHaveBeenCalled(); }); it("update/re-install re-writes the component links (delete + insert)", async () => { @@ -1253,3 +1292,52 @@ describe("removeExtension", () => { expect(dbMock.db.delete).not.toHaveBeenCalled(); }); }); + +it("rejoins a completed original catalog workflow without republishing or rewriting bookkeeping", async () => { + const ctx = makeCtx(); + publishIntegrationMock.mockResolvedValue({ + integrationId: "intg-one", + transactionId: INTEGRATION_TRANSACTION, + manifestSha256: "f".repeat(64), + }); + readInstalledMock.mockResolvedValue(installedSnapshot([])); + settleIntegrationMock.mockResolvedValue({ + integrationId: "intg-one", + outcome: "commit", + changed: true, + }); + await installExtension(ctx, { manifest: INTEGRATION_ONLY, sourceTrust: "community" }); + await installExtension(ctx, { manifest: INTEGRATION_ONLY, sourceTrust: "community" }); + expect(publishIntegrationMock).toHaveBeenCalledTimes(1); + expect(dbMock.db.transaction).toHaveBeenCalledTimes(1); +}); + +it("finishes original committed bookkeeping despite a later revocation without rollback or rewriting rows", async () => { + const ctx = makeCtx(); + publishIntegrationMock.mockResolvedValue({ + integrationId: "intg-one", + transactionId: INTEGRATION_TRANSACTION, + manifestSha256: "f".repeat(64), + }); + readInstalledMock.mockResolvedValue(installedSnapshot([])); + settleIntegrationMock.mockResolvedValue({ + integrationId: "intg-one", + outcome: "commit", + changed: true, + }); + await installExtension(ctx, { manifest: INTEGRATION_ONLY, sourceTrust: "community" }); + vi.mocked(getKillSwitch).mockResolvedValue({ + removed: new Map(), + critical: new Map([["bundle-x", { reason: "later advisory" }]]), + stale: false, + }); + await expect( + installExtension(ctx, { manifest: INTEGRATION_ONLY, sourceTrust: "community" }), + ).resolves.toEqual({ id: "bundle-x", components: 1 }); + expect(publishIntegrationMock).toHaveBeenCalledTimes(1); + expect(dbMock.db.transaction).toHaveBeenCalledTimes(1); + expect(settleIntegrationMock).not.toHaveBeenCalledWith( + expect.objectContaining({ outcome: "rollback" }), + expect.anything(), + ); +}); diff --git a/apps/api/src/services/__tests__/fixtures/extension-workflow-native.ts b/apps/api/src/services/__tests__/fixtures/extension-workflow-native.ts new file mode 100644 index 000000000..367f21731 --- /dev/null +++ b/apps/api/src/services/__tests__/fixtures/extension-workflow-native.ts @@ -0,0 +1,113 @@ +/** Native API-role process fixture. Only the test's fixed owned job is read. */ +import { eq } from "drizzle-orm"; +import { db, sql } from "../../../db"; +import { adminJob, installedExtension, installedIntegration } from "../../../db/schema"; +import { checkpointOwnedAdminJob } from "../../admin-job-checkpoint"; +import { jobOwnership } from "../../admin-job-state"; +import { publishIntegration, settleIntegration } from "../../agent-integrations"; +import { + extensionDatabaseStep, + extensionStep, + openExtensionWorkflow, +} from "../../extension-workflow"; +import type { JobContext } from "../../job-runner"; +import { createDurableOpsKey } from "../../ops-client"; + +const [id, stopAt] = process.argv.slice(2); +if (!id) throw Error("Owned job required"); +const [row] = await db.select().from(adminJob).where(eq(adminJob.id, id)); +if (!row || row.type !== "extension.install" || row.status !== "running") + throw Error("Owned job missing"); +const payload = row.payload as { + source: { kind: "artifact"; url: string; sha256: string }; + bundle: string; + }, + execution = (row.result as { _execution: { owner: string; version: number } })._execution; +const ctx: JobContext = { + jobId: id, + jobType: row.type, + execution, + payload: row.payload as Record, + signal: new AbortController().signal, + checkpointResult: row.result as Record, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (value, progress, key, address) => { + if (value.opsProjection) + ctx.checkpointResult = await checkpointOwnedAdminJob( + id, + execution, + value, + progress, + key, + address, + ctx.checkpointResult ?? {}, + ); + else { + const [saved] = await db + .update(adminJob) + .set({ result: value }) + .where(jobOwnership(id, execution)) + .returning(); + if (!saved) throw Error("Owner changed"); + ctx.checkpointResult = saved.result as Record; + } + }, +}; +const cursor = await openExtensionWorkflow(ctx, { native: true }); +const publication = await extensionStep(cursor, "publish", payload.source, () => + publishIntegration( + { integrationId: "workflow-map", source: payload.source, extensionId: payload.bundle }, + { + ctx, + operationKey: createDurableOpsKey("extension.integration.publish", `${id}:workflow-map`), + }, + ), +); +async function boundary(name: string) { + if (stopAt !== name) return; + process.send?.({ boundary: name }); + await new Promise(() => {}); +} +await boundary("published"); +await extensionDatabaseStep(cursor, "bookkeeping", { bundle: payload.bundle }, async (tx) => { + await tx + .insert(installedExtension) + .values({ + id: payload.bundle, + name: payload.bundle, + sourceTrust: "community", + installedVersion: "1.0.0", + manifest: { id: payload.bundle, name: payload.bundle, version: "1.0.0" }, + }) + .onConflictDoUpdate({ target: installedExtension.id, set: { installedVersion: "1.0.0" } }); + await tx + .insert(installedIntegration) + .values({ + id: "workflow-map", + repository: payload.source.url, + sourceType: "registry", + installedVersion: "1.0.0", + managedByExtension: payload.bundle, + lastTransactionId: publication.transactionId, + }) + .onConflictDoUpdate({ + target: installedIntegration.id, + set: { managedByExtension: payload.bundle, lastTransactionId: publication.transactionId }, + }); +}); +await boundary("bookkept"); +await extensionStep(cursor, "settle", { transactionId: publication.transactionId }, () => + settleIntegration( + { integrationId: "workflow-map", transactionId: publication.transactionId, outcome: "commit" }, + { + ctx, + operationKey: createDurableOpsKey( + "extension.integration.commit", + `${id}:${publication.transactionId}`, + ), + }, + ), +); +await sql.end({ timeout: 2 }); +process.send?.({ complete: true }); diff --git a/apps/api/src/services/__tests__/job-runner-extension-sanitization.test.ts b/apps/api/src/services/__tests__/job-runner-extension-sanitization.test.ts index 1f7a8539a..b15b9665e 100644 --- a/apps/api/src/services/__tests__/job-runner-extension-sanitization.test.ts +++ b/apps/api/src/services/__tests__/job-runner-extension-sanitization.test.ts @@ -1,3 +1,4 @@ +import { PgDialect } from "drizzle-orm/pg-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const database = vi.hoisted(() => { @@ -17,8 +18,27 @@ const database = vi.hoisted(() => { })), update: vi.fn(() => ({ set: vi.fn((value: Record) => { - updates.push(value); - return { where: vi.fn().mockResolvedValue([]) }; + const captured = { ...value }; + if ( + captured.result && + typeof captured.result === "object" && + "queryChunks" in captured.result + ) { + const query = new PgDialect().sqlToQuery( + captured.result as Parameters[0], + ); + const patch = query.params.find( + (param) => typeof param === "string" && param.startsWith("{"), + ); + if (typeof patch === "string") captured.result = JSON.parse(patch); + } + updates.push(captured); + const rows = [{ id: "unit-owned-job", status: "running", result: captured.result }]; + return { + where: vi.fn(() => + Object.assign(Promise.resolve(rows), { returning: vi.fn().mockResolvedValue(rows) }), + ), + }; }), })), }, @@ -26,6 +46,15 @@ const database = vi.hoisted(() => { }); vi.mock("../../db", () => ({ db: database.db })); +// This suite tests output encoding, not SQL ownership. Real owner-CAS/native +// recovery are covered by the PostgreSQL suites. +vi.mock("../admin-job-state", async (actual) => ({ + ...(await actual()), + insertOwnedJobLog: vi.fn(async (_owner, entry: Record) => { + database.inserts.push(entry); + return true; + }), +})); vi.mock("../admin-job-ops", async (importOriginal) => ({ ...(await importOriginal()), cancelAdminJobOperations: vi.fn(), @@ -170,7 +199,8 @@ describe("extension job durable boundaries", () => { update.result !== null && "opsProjection" in update.result, )?.result as typeof checkpoint | undefined; - expect(persisted?.opsProjection.operations).toEqual(operations); + expect(Object.keys(persisted?.opsProjection.operations ?? {})).toEqual(Object.keys(operations)); + expect(persisted?.opsProjection.operations).toMatchObject(operations); }); it("redacts schemes containing URL-parser whitespace as a complete value", async () => { diff --git a/apps/api/src/services/__tests__/job-runner-initialize.test.ts b/apps/api/src/services/__tests__/job-runner-initialize.test.ts index babb9d7c3..544daa5c9 100644 --- a/apps/api/src/services/__tests__/job-runner-initialize.test.ts +++ b/apps/api/src/services/__tests__/job-runner-initialize.test.ts @@ -1,4 +1,14 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; + +const authSecret = vi.hoisted(() => { + const previous = process.env.BETTER_AUTH_SECRET; + process.env.BETTER_AUTH_SECRET ??= "job-runner-initialize-unit-only-secret"; + return { previous }; +}); +afterAll(() => { + if (authSecret.previous === undefined) delete process.env.BETTER_AUTH_SECRET; + else process.env.BETTER_AUTH_SECRET = authSecret.previous; +}); const dbMock = vi.hoisted(() => { const rows: Array<{ id: string; result: unknown }> = []; @@ -60,6 +70,8 @@ afterEach(() => { describe("job runner restart initialization", () => { it("keeps typed agent-backed jobs for observation without handler replay", async () => { expect(AGENT_RECOVERABLE_ADMIN_JOB_TYPES).toEqual([ + "extension.install", + "extension.remove", "standalone.adopt", "standalone.remove", "standalone.preview", @@ -85,6 +97,7 @@ describe("job runner restart initialization", () => { "backup.prune", "artifact.validate", "artifact.import", + "artifact.export", ]); await jobRunner.initialize({ completeRestartedUpdates: true }); expect(dbMock.updates).not.toContainEqual(expect.objectContaining({ status: "queued" })); diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index b035a7072..ec72711b6 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -683,7 +683,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( serviceIds: ids, pausePrepareResponse: version === "launch-native-system-release-api-before-save", }); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal( + join(host.infraDir, "state/ops-agent/jobs-v1.json"), + ); const app = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, diff --git a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts index dfbc087ee..d002315fb 100644 --- a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts +++ b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts @@ -107,7 +107,7 @@ dbDescribe("retained finite maintenance requests", () => { role: "ops-agent", serviceIds: host.loaded.map((service) => service.manifest.id), }); - let journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + let journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); const { ArtifactTransferStore } = await import( "../../../../ops-agent/src/artifact-transfer/store" ), @@ -346,7 +346,7 @@ dbDescribe("retained finite maintenance requests", () => { ownedServices: productionOwnedServices(runtime), execFile: host.execFile, }); - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); agent = buildOpsAgentServer({ runtime, journal, @@ -517,7 +517,7 @@ dbDescribe("retained finite maintenance requests", () => { role: "ops-agent", serviceIds: host.loaded.map((service) => service.manifest.id), }), - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), agent = buildOpsAgentServer({ runtime: host.runtime, resourceClaimer: host.claimer, @@ -626,7 +626,7 @@ dbDescribe("retained finite maintenance requests", () => { role: "ops-agent", serviceIds: host.loaded.map((service) => service.manifest.id), }), - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); const agent = buildOpsAgentServer({ runtime: host.runtime, resourceClaimer: host.claimer, @@ -772,7 +772,7 @@ dbDescribe("retained finite maintenance requests", () => { role: "ops-agent", serviceIds: host.loaded.map((s) => s.manifest.id), }); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); const agent = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, @@ -997,7 +997,7 @@ dbDescribe("retained finite maintenance requests", () => { role: "ops-agent", serviceIds: host.loaded.map((service) => service.manifest.id), }), - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); const agent = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime, diff --git a/apps/api/src/services/__tests__/service-registry-community.test.ts b/apps/api/src/services/__tests__/service-registry-community.test.ts index 36f1f73df..d0c32731b 100644 --- a/apps/api/src/services/__tests__/service-registry-community.test.ts +++ b/apps/api/src/services/__tests__/service-registry-community.test.ts @@ -3,9 +3,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("../community-repositories", () => ({ readCommunityServices: vi.fn() })); +vi.mock("../community-service-inventory", () => ({ readCommunityServices: vi.fn() })); -import { readCommunityServices } from "../community-repositories"; +import { readCommunityServices } from "../community-service-inventory"; const readMock = vi.mocked(readCommunityServices); diff --git a/apps/api/src/services/admin-job-ops.ts b/apps/api/src/services/admin-job-ops.ts index 83463b5ba..8c4452213 100644 --- a/apps/api/src/services/admin-job-ops.ts +++ b/apps/api/src/services/admin-job-ops.ts @@ -1,3 +1,4 @@ +import { envString } from "@openmapx/core/server-env"; import { type OpsProjection, type ProjectedOperation, @@ -166,7 +167,7 @@ export async function ensureAdminJobReceiptDelivery( if (delivery.state === "confirmed") return; const fingerprint = projected.fingerprint; if (!fingerprint) throw new JobObservationPending("recovery_required"); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); const proof = signApiRetentionProof(token, { version: 1, apiJobId: projected.attempt.apiJobId, diff --git a/apps/api/src/services/admin-job-parent.ts b/apps/api/src/services/admin-job-parent.ts index 2398736fb..6e2b9196e 100644 --- a/apps/api/src/services/admin-job-parent.ts +++ b/apps/api/src/services/admin-job-parent.ts @@ -9,6 +9,7 @@ import { opsOperationFingerprint, readOpsTokenFile, } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; import type { JobContext } from "./job-runner"; import { createDurableOpsKey } from "./ops-client"; @@ -30,6 +31,33 @@ function parentJobBudget(ctx: JobContext, kind: OpsOperationKind): number { opsExecutionBudget("integration.settle").processMs + 60000 ); + if (ctx.jobType === "extension.remove") return 24 * 60 * 60_000; + if (ctx.jobType === "extension.install") { + const manifest = ctx.payload.manifest as + | { services?: unknown[]; integrations?: unknown[] } + | undefined; + const services = manifest?.services?.length ?? 0, + integrations = manifest?.integrations?.length ?? 0; + if (services > 256 || integrations > 256) + throw Error("Extension exceeds the bounded job capacity"); + const serviceBudget = [ + "serviceRepository.publish", + "serviceRepository.settle", + "service.pull", + "stack.render", + "generation-hardlinks", + "service.start", + ].reduce((sum, kind) => sum + opsExecutionBudget(kind).processMs, 60_000); + const integrationBudget = + opsExecutionBudget("catalogIntegration.publish").processMs + + opsExecutionBudget("catalogIntegration.settle").processMs; + // Freeze once at the first admission. Child normalization and all recovered + // attempts still use their original shorter absolute deadlines. + return Math.min( + 24 * 60 * 60_000, + 120_000 + Math.max(1, services) * serviceBudget + integrations * integrationBudget, + ); + } if (ctx.jobType === "system.update") return opsExecutionBudget("system.update").processMs; if (ctx.jobType === "service.start" || ctx.jobType === "service.apply") { // Fixed current workflow: render, inspect, start the coordinator, await its @@ -110,7 +138,7 @@ export function prepareApiJobChildAttempt( }); } export async function issueApiJobChildAssertion(attempt: ApiJobChildAttempt): Promise { - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); // Exact schema order and NUL-terminated domain used by verifyApiJobParentExecution. const payload = Buffer.from(JSON.stringify(apiJobChildAttemptSchema.parse(attempt))).toString( "base64url", diff --git a/apps/api/src/services/admin-ops.ts b/apps/api/src/services/admin-ops.ts index 89d2a1fca..1fe76e265 100644 --- a/apps/api/src/services/admin-ops.ts +++ b/apps/api/src/services/admin-ops.ts @@ -120,15 +120,22 @@ async function serviceComposeAction( return { exitCode: 0 }; } -export async function serviceStart(service: string, ctx: JobContext): Promise { +export async function serviceStart( + service: string, + ctx: JobContext, + scope: "start" | "extension-restore" = "start", +): Promise { assertKnownService(service); - const prepared = await prepareServiceGeneration(ctx, `start:${service}`, { + const prepared = await prepareServiceGeneration(ctx, `${scope}:${service}`, { coordinatorMode: "ensure-running", }); await ctx.checkpoint({ ...ctx.checkpointResult, _completion: { - operationKey: createDurableOpsKey("admin-job.service.start", `${ctx.jobId}:${service}`), + operationKey: createDurableOpsKey( + scope === "start" ? "admin-job.service.start" : "admin-job.extension.restore", + `${ctx.jobId}:${service}`, + ), result: { service, action: "start" }, }, }); @@ -139,7 +146,7 @@ export async function serviceStart(service: string, ctx: JobContext): Promise["repositories"]; - services: Array<{ - repository: string; - directory: string; - manifest: coreServices.ServiceManifest; - }>; - /** Reported services the API's own manifest rules reject. */ - problems: string[]; - /** Ids of every service the agent reported, including rejected ones. */ - reportedServiceIds: string[]; -} +export { + type CommunityServicesSnapshot, + readCommunityServices, +} from "./community-service-inventory"; + +import { readCommunityServices } from "./community-service-inventory"; interface OperationOptions { + ctx?: JobContext; operationKey: string; signal?: AbortSignal; } +function originalOwner(options: OperationOptions): JobContext { + if (!options.ctx) throw new JobObservationPending("recovery_required"); + return options.ctx; +} + /** Fetch and validate a repository at `ref` without installing it. */ export async function inspectServiceRepository( url: string, @@ -113,11 +116,11 @@ export async function publishServiceRepository( }, options: OperationOptions, ): Promise> { - const result = await executeAndWait( - createApiOpsClient(), + const result = await executeAdminJobOperation( + originalOwner(options), { kind: "serviceRepository.publish", ...request }, - options.operationKey, - { signal: options.signal }, + "extension.repository.publish", + { operationKey: options.operationKey }, ); if (result.repository !== hashUrl(request.url) || result.commit !== request.commit) { throw new Error("The operations agent installed a different repository"); @@ -133,11 +136,11 @@ export function settleServiceRepository( }, options: OperationOptions, ): Promise> { - return executeAndWait( - createApiOpsClient(), + return executeAdminJobOperation( + originalOwner(options), { kind: "serviceRepository.settle", ...request }, - options.operationKey, - { signal: options.signal }, + "extension.repository.settle", + { operationKey: options.operationKey }, ); } @@ -145,51 +148,14 @@ export function removeServiceRepository( request: { repository: string; extensionId: string }, options: OperationOptions, ): Promise> { - return executeAndWait( - createApiOpsClient(), + return executeAdminJobOperation( + originalOwner(options), { kind: "serviceRepository.remove", ...request }, - options.operationKey, - { signal: options.signal }, + "extension.repository.remove", + { operationKey: options.operationKey }, ); } -/** Installed community services and repositories, validated again with the community manifest rules. */ -export async function readCommunityServices( - options: { signal?: AbortSignal } = {}, -): Promise { - const result = await executeAndWait( - createApiOpsClient(), - { kind: "communityServices.inspect" }, - createDurableOpsKey("community.services.inspect", randomUUID()), - { signal: options.signal }, - ); - const problems: string[] = []; - const services: CommunityServicesSnapshot["services"] = []; - for (const entry of result.services) { - const manifestProblems = coreServices.communityServiceManifestProblems(entry.manifest); - if (manifestProblems.length > 0) { - problems.push( - `Community service in repository ${entry.repository} was rejected: ${manifestProblems.join("; ")}`, - ); - continue; - } - services.push({ - repository: entry.repository, - directory: entry.directory, - manifest: entry.manifest as unknown as coreServices.ServiceManifest, - }); - } - return { - revisionId: result.revisionId, - repositories: result.repositories, - services, - problems, - reportedServiceIds: result.services.flatMap((entry) => - typeof entry.manifest.id === "string" ? [entry.manifest.id] : [], - ), - }; -} - export interface ServiceRuntimeRecovery { runtimeRecoveryNeeded: boolean; orphanedServiceIds: string[]; @@ -210,6 +176,7 @@ export interface ServiceRuntimeRecovery { */ export async function reconcileCommunityRepositoryTransactions( options: { + ctx?: JobContext; restoreSelection?: (roots: string[], recoveryId: string) => Promise; persistRuntimeRecovery?: (recovery: ServiceRuntimeRecovery) => Promise; signal?: AbortSignal; @@ -222,6 +189,31 @@ export async function reconcileCommunityRepositoryTransactions( if (pending.length === 0) { return { runtimeRecoveryNeeded: false, orphanedServiceIds: [], restartServiceIds: [] }; } + if (!options.ctx) throw new JobObservationPending("recovery_required"); + const projection = readProjection(options.ctx.checkpointResult); + for (const entry of pending) { + const original = Object.values(projection.operations).find( + (operation) => + operation.kind === "serviceRepository.publish" && + operation.attempt?.apiJobId === options.ctx!.jobId && + operation.operation?.kind === "serviceRepository.publish" && + hashUrl(operation.operation.url) === entry.repository, + ); + if (!original?.operation || original.operation.kind !== "serviceRepository.publish") + throw new JobObservationPending("recovery_required"); + const step = Object.entries(projection.steps).find( + ([, value]) => value.operationKey === original.operationKey, + )?.[0]; + if (!step) throw new JobObservationPending("recovery_required"); + const publication = await executeAdminJobOperation( + options.ctx, + original.operation, + "extension.repository.publish", + { operationKey: step }, + ); + if (publication.transactionId !== entry.pending.transactionId) + throw new JobObservationPending("recovery_required"); + } const incidentId = `recovery_${createHash("sha256") .update( pending @@ -238,6 +230,7 @@ export async function reconcileCommunityRepositoryTransactions( settleServiceRepository( { repository, transactionId, outcome }, { + ctx: options.ctx, operationKey: createDurableOpsKey( `community.repository.${outcome}`, `${incidentId}:${repository}:${transactionId}`, diff --git a/apps/api/src/services/community-service-inventory.ts b/apps/api/src/services/community-service-inventory.ts new file mode 100644 index 000000000..c7524f22c --- /dev/null +++ b/apps/api/src/services/community-service-inventory.ts @@ -0,0 +1,55 @@ +/** Read-only community inventory; service discovery never imports mutation orchestration. */ +import { randomUUID } from "node:crypto"; +import type { OpsResultFor } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; +import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "./ops-client"; +export interface CommunityServicesSnapshot { + revisionId: string; + repositories: OpsResultFor<"communityServices.inspect">["repositories"]; + services: Array<{ + repository: string; + directory: string; + manifest: coreServices.ServiceManifest; + }>; + /** Reported services the API's own manifest rules reject. */ + problems: string[]; + /** Ids of every service the agent reported, including rejected ones. */ + reportedServiceIds: string[]; +} + +/** Installed community services and repositories, validated again with the community manifest rules. */ +export async function readCommunityServices( + options: { signal?: AbortSignal } = {}, +): Promise { + const result = await executeAndWait( + createApiOpsClient(), + { kind: "communityServices.inspect" }, + createDurableOpsKey("community.services.inspect", randomUUID()), + { signal: options.signal }, + ); + const problems: string[] = []; + const services: CommunityServicesSnapshot["services"] = []; + for (const entry of result.services) { + const manifestProblems = coreServices.communityServiceManifestProblems(entry.manifest); + if (manifestProblems.length > 0) { + problems.push( + `Community service in repository ${entry.repository} was rejected: ${manifestProblems.join("; ")}`, + ); + continue; + } + services.push({ + repository: entry.repository, + directory: entry.directory, + manifest: entry.manifest as unknown as coreServices.ServiceManifest, + }); + } + return { + revisionId: result.revisionId, + repositories: result.repositories, + services, + problems, + reportedServiceIds: result.services.flatMap((entry) => + typeof entry.manifest.id === "string" ? [entry.manifest.id] : [], + ), + }; +} diff --git a/apps/api/src/services/configuration-migration/legacy-integration-import.test.ts b/apps/api/src/services/configuration-migration/legacy-integration-import.test.ts index 7e86a5cb2..4b3513e31 100644 --- a/apps/api/src/services/configuration-migration/legacy-integration-import.test.ts +++ b/apps/api/src/services/configuration-migration/legacy-integration-import.test.ts @@ -97,7 +97,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, resourceClaimer: host.claimer, - journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), controlInstallation: installation, }); driftRegistryAfterApply = false; diff --git a/apps/api/src/services/configuration-migration/migration-postgres.test.ts b/apps/api/src/services/configuration-migration/migration-postgres.test.ts index 0bda1fc41..7b3062253 100644 --- a/apps/api/src/services/configuration-migration/migration-postgres.test.ts +++ b/apps/api/src/services/configuration-migration/migration-postgres.test.ts @@ -102,7 +102,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, resourceClaimer: host.claimer, - journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), controlInstallation: installation, }); const baseUrl = await agent.listen({ host: "127.0.0.1", port: 0 }), diff --git a/apps/api/src/services/coordinator-lifecycle.ts b/apps/api/src/services/coordinator-lifecycle.ts index fa536b393..d0e79a9ad 100644 --- a/apps/api/src/services/coordinator-lifecycle.ts +++ b/apps/api/src/services/coordinator-lifecycle.ts @@ -8,6 +8,7 @@ import { type PreparedConfigurationRecord, readOpsTokenFile, } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; import { db } from "../db"; import { adminJob } from "../db/schema"; import { readProjection } from "./admin-job-projection"; @@ -183,7 +184,7 @@ export function assertRetainedCoordinatorLifecycleProvenance( * prepared pull into an attempted one. Old-owner attempt bytes are preserved. */ export async function claimCoordinatorPullDelivery(context: JobContext, candidateId: string) { if (!context.execution || context.observationOnly) throw refuse(); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); let updated: Record | undefined; const delivery = await db.transaction(async (tx) => { const [row] = await tx @@ -264,7 +265,7 @@ export async function closeUnissuedCoordinatorPull( packetOperationKey?: string, ): Promise { if (!context.execution || context.observationOnly) throw refuse(); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); const updated = await db.transaction(async (tx) => { const [row] = await tx .select() diff --git a/apps/api/src/services/extension-installer.ts b/apps/api/src/services/extension-installer.ts index 0ba80fde5..5f699438a 100644 --- a/apps/api/src/services/extension-installer.ts +++ b/apps/api/src/services/extension-installer.ts @@ -15,6 +15,7 @@ import { dockerComposeAction } from "../utils/docker-compose"; import { sanitizeExtensionOutputString } from "../utils/safe-log-fields"; import { getServiceSelectionSummary } from "./admin-cli"; import { JobObservationPending } from "./admin-job-failure"; +import { hasRetainedAdminJobOperation } from "./admin-job-ops"; import { JobOwnershipLost } from "./admin-job-state"; import { renderAndPersistCompose, serviceStart } from "./admin-ops"; import { @@ -36,6 +37,14 @@ import { assertComponentOwnership } from "./extension-component-ownership"; import { normalizeExtensionInstallPayload } from "./extension-install-payload"; import { recoverInterruptedExtensionChanges, withExtensionChangeLock } from "./extension-recovery"; import { getKillSwitch } from "./extension-store"; +import { + completedExtensionStep, + type ExtensionWorkflow, + extensionDatabaseStep, + extensionStep, + hasExtensionPublications, + openExtensionWorkflow, +} from "./extension-workflow"; import type { JobContext } from "./job-runner"; import { createDurableOpsKey, OpsObservationPending } from "./ops-client"; import { @@ -306,6 +315,7 @@ async function integrationManagedBy(id: string): Promise { * running API follows the applied generation without a restart. */ async function applySelectionRoots( + ctx: JobContext, roots: string[], allowMissing: boolean, operationKey: string, @@ -320,6 +330,7 @@ async function applySelectionRoots( throw new Error(`Selected service(s) are not installed: ${expanded.missingIds.join(", ")}`); } await applyTrustedConfiguration({ + ctx, kind: "serviceSelection.apply", selectedRoots: normalized, operationKey, @@ -327,30 +338,79 @@ async function applySelectionRoots( }); } -/** Add service ids to the selection. Returns the ids newly added. */ -async function enableServicesInSelection( - serviceIds: string[], - operationKey: string, - signal?: AbortSignal, -): Promise { - const registry = getServiceRegistry(); - const roots = [...(await getServiceSelectionSummary(registry)).selectedRoots]; - const added = serviceIds.filter((id) => !roots.includes(id)); - if (added.length === 0) return []; - await applySelectionRoots([...roots, ...added], false, operationKey, signal); - return added; -} - -async function disableServicesInSelection( - serviceIds: string[], - operationKey: string, - signal?: AbortSignal, -): Promise { - const registry = getServiceRegistry(); - const roots = (await getServiceSelectionSummary(registry)).selectedRoots.filter( - (r) => !serviceIds.includes(r), - ); - await applySelectionRoots(roots, true, operationKey, signal); +function extensionEffects(workflow: ExtensionWorkflow, ctx: JobContext) { + return { + publishServiceRepository: ( + request: Parameters[0], + options: Parameters[1], + ) => + extensionStep(workflow, `repository.publish:${options.operationKey}`, request, async () => { + if (!hasRetainedAdminJobOperation(ctx, options.operationKey)) + await assertNotRevoked(request.extensionId); + return publishServiceRepository(request, options); + }), + settleServiceRepository: ( + request: Parameters[0], + options: Parameters[1], + ) => + extensionStep(workflow, `repository.settle:${options.operationKey}`, request, () => + settleServiceRepository(request, options), + ), + removeServiceRepository: ( + request: Parameters[0], + options: Parameters[1], + ) => + extensionStep(workflow, `repository.remove:${options.operationKey}`, request, () => + removeServiceRepository(request, options), + ), + publishIntegration: ( + request: Parameters[0], + options: Parameters[1], + ) => + extensionStep(workflow, `integration.publish:${options.operationKey}`, request, async () => { + if (!hasRetainedAdminJobOperation(ctx, options.operationKey)) + await assertNotRevoked(request.extensionId!); + return publishIntegration(request, options); + }), + adoptIntegration: ( + request: Parameters[0], + options: Parameters[1], + ) => + extensionStep(workflow, `integration.adopt:${options.operationKey}`, request, () => + adoptIntegration(request, options), + ), + settleIntegration: ( + request: Parameters[0], + options: Parameters[1], + ) => + extensionStep(workflow, `integration.settle:${options.operationKey}`, request, () => + settleIntegration(request, options), + ), + removeIntegration: ( + request: Parameters[0], + options: Parameters[1], + ) => + extensionStep(workflow, `integration.remove:${options.operationKey}`, request, () => + removeIntegration(request, options), + ), + dockerComposeAction: ( + service: string, + action: Parameters[1], + options: Parameters[2], + ) => + extensionStep( + workflow, + `lifecycle:${options?.operationKey}`, + { service, action, revision: options?.expectedConfigurationRevisionId ?? null }, + () => dockerComposeAction(service, action, options), + ), + renderAndPersistCompose: (options: Parameters[0]) => + extensionStep(workflow, `render:${options?.operationKey}`, {}, () => + renderAndPersistCompose(options), + ), + serviceStart: (service: string, _ctx: JobContext) => + extensionStep(workflow, `start:${service}`, { service }, () => serviceStart(service, ctx)), + }; } interface InstallLedger { @@ -381,7 +441,8 @@ export function installExtension( ): Promise<{ id: string; components: number }> { const normalized = normalizedInstallOptions(opts); return withExtensionChangeLock(async () => { - await recoverInterruptedExtensionChanges(ctx.signal, { ctx }); + if (!ctx.checkpointResult?._extensionWorkflow) + await recoverInterruptedExtensionChanges(ctx.signal, { ctx }); return installExtensionNow(ctx, normalized); }); } @@ -390,16 +451,42 @@ async function installExtensionNow( ctx: JobContext, opts: InstallExtensionOptions, ): Promise<{ id: string; components: number }> { + const workflow = await openExtensionWorkflow(ctx, opts); + const { + publishServiceRepository, + settleServiceRepository, + publishIntegration, + adoptIntegration, + settleIntegration, + dockerComposeAction, + renderAndPersistCompose, + serviceStart, + } = extensionEffects(workflow, ctx); + const rollback = completedExtensionStep<{ ledger: InstallLedger; message: string }>( + workflow, + "rollback.intent", + ); + if (rollback) { + await extensionStep(workflow, "rollback", rollback.ledger, () => + rollbackInstall(ctx, rollback.ledger, workflow), + ); + throw new Error(rollback.message); + } const { manifest } = opts; // Synthetic (loopback) actors have no user row → attribute to null. const installedBy = dbActorId(opts.actorId); // Everything that can be decided without touching the deployment is decided // here, before the first mutation. - await preflightExtensionInstall(opts); + await extensionStep(workflow, "preflight", opts, async () => { + await preflightExtensionInstall(opts); + return null; + }); // Revocation can land between queueing and execution, so it is rechecked // immediately before the first mutation as well as at the end. - await assertNotRevoked(manifest.id); + let bookkeepingCommitted = completedExtensionStep(workflow, "bookkeeping") !== undefined; + if (!bookkeepingCommitted && !hasExtensionPublications(workflow)) + await assertNotRevoked(manifest.id); const ledger: InstallLedger = { publishedRepositories: [], @@ -409,7 +496,6 @@ async function installExtensionNow( previouslyEnabledServiceIds: [], selectedRootsBefore: null, }; - let bookkeepingCommitted = false; let rollbackRequired = false; try { @@ -425,19 +511,29 @@ async function installExtensionNow( for (const component of serviceComponents) serviceRepos.set(component.repo, component); if (serviceComponents.length > 0) { - const registry = getServiceRegistry(); - ledger.selectedRootsBefore = [...(await getServiceSelectionSummary(registry)).selectedRoots]; + const original = await extensionStep(workflow, "selection.before", {}, async () => { + const registry = getServiceRegistry(); + return { + roots: [...(await getServiceSelectionSummary(registry)).selectedRoots], + enabled: serviceComponents + .map((c) => c.service) + .filter((id) => registry.get(id)?.enabled === true), + }; + }); + ledger.selectedRootsBefore = original.roots; ledger.touchedServiceIds = serviceComponents.map((component) => component.service); - ledger.previouslyEnabledServiceIds = ledger.touchedServiceIds.filter( - (serviceId) => registry.get(serviceId)?.enabled === true, - ); + ledger.previouslyEnabledServiceIds = original.enabled; } // Pin every repository to the exact commit the security preview describes. - const inspections = await inspectExtensionServiceRepos( - opts, - { namespace: "extension.install.inspect", identity: `${ctx.jobId}:${manifest.id}` }, - ctx.signal, + const inspections = new Map( + await extensionStep(workflow, "inspections", opts, async () => [ + ...(await inspectExtensionServiceRepos( + opts, + { namespace: "extension.install.inspect", identity: `${ctx.jobId}:${manifest.id}` }, + ctx.signal, + )), + ]), ); const installPreview = buildExtensionInstallPreview(opts, inspections); if ( @@ -470,6 +566,7 @@ async function installExtensionNow( }, }, { + ctx, operationKey: createDurableOpsKey( "extension.repository.publish", `${ctx.jobId}:${manifest.id}:${inspection.repository}`, @@ -529,15 +626,29 @@ async function installExtensionNow( ); } } - const newlyEnabled = await enableServicesInSelection( + const newlyEnabled = await extensionStep( + workflow, + "selection.enable", serviceComponents.map((s) => s.service), - createDurableOpsKey("extension.selection.enable", `${ctx.jobId}:${manifest.id}`), - ctx.signal, + async () => { + const roots = ledger.selectedRootsBefore!; + const added = serviceComponents.map((s) => s.service).filter((id) => !roots.includes(id)); + if (added.length) + await applySelectionRoots( + ctx, + [...roots, ...added], + false, + createDurableOpsKey("extension.selection.enable", `${ctx.jobId}:${manifest.id}`), + ctx.signal, + ); + return added; + }, ); ledger.enabledServiceIds.push(...newlyEnabled); await ctx.log("Rendering compose..."); - await renderAndPersistCompose({ + const rendered = await renderAndPersistCompose({ + ctx, operationKey: createDurableOpsKey("extension.config.render", `${ctx.jobId}:${manifest.id}`), signal: ctx.signal, }); @@ -546,6 +657,8 @@ async function installExtensionNow( // fatal: starting a cached image would make the recorded extension // version claim an artifact that was never acquired. const pulled = await dockerComposeAction(svc.service, "pull", { + expectedConfigurationRevisionId: rendered.revisionId, + ctx, operationKey: createDurableOpsKey( "extension.service.pull", `${ctx.jobId}:${manifest.id}:${svc.service}`, @@ -571,7 +684,9 @@ async function installExtensionNow( }> = []; const agentIntegrations = integrationComponents.length > 0 - ? await readInstalledIntegrations({ signal: ctx.signal }) + ? await extensionStep(workflow, "integrations.before", {}, () => + readInstalledIntegrations({ signal: ctx.signal }), + ) : null; for (const intg of integrationComponents) { await ctx.log(`Installing integration ${intg.id}...`); @@ -645,88 +760,95 @@ async function installExtensionNow( const now = new Date(); const components = coreServices.extensionComponentSummary(manifest); // Last gate before the install becomes authoritative. - await assertNotRevoked(manifest.id); - await assertComponentOwnership(manifest.id, components); - await db.transaction(async (tx) => { - for (const row of pendingServiceRepoRows) { - await tx - .insert(serviceRepository) - .values(row) - .onConflictDoUpdate({ - target: serviceRepository.hash, - set: { - lastSha: row.lastSha, - pinnedRef: row.pinnedRef, - managedByExtension: row.managedByExtension, - lastTransactionId: row.lastTransactionId, - }, - }); - } - for (const row of pendingIntegrationRows) { - await tx - .insert(installedIntegration) - .values({ - id: row.id, - repository: row.repository, - installedVersion: manifest.version, - sourceType: "registry", - installedAt: now, - updatedAt: now, - installedBy, - managedByExtension: manifest.id, - lastTransactionId: row.transactionId, - }) - .onConflictDoUpdate({ - target: installedIntegration.id, - set: { + if (!bookkeepingCommitted) { + await assertNotRevoked(manifest.id); + await assertComponentOwnership(manifest.id, components); + } + await extensionDatabaseStep( + workflow, + "bookkeeping", + { manifest, pendingServiceRepoRows, pendingIntegrationRows }, + async (tx) => { + for (const row of pendingServiceRepoRows) { + await tx + .insert(serviceRepository) + .values(row) + .onConflictDoUpdate({ + target: serviceRepository.hash, + set: { + lastSha: row.lastSha, + pinnedRef: row.pinnedRef, + managedByExtension: row.managedByExtension, + lastTransactionId: row.lastTransactionId, + }, + }); + } + for (const row of pendingIntegrationRows) { + await tx + .insert(installedIntegration) + .values({ + id: row.id, repository: row.repository, installedVersion: manifest.version, sourceType: "registry", + installedAt: now, updatedAt: now, + installedBy, managedByExtension: manifest.id, lastTransactionId: row.transactionId, - }, - }); - } + }) + .onConflictDoUpdate({ + target: installedIntegration.id, + set: { + repository: row.repository, + installedVersion: manifest.version, + sourceType: "registry", + updatedAt: now, + managedByExtension: manifest.id, + lastTransactionId: row.transactionId, + }, + }); + } - await tx - .insert(installedExtension) - .values({ - id: manifest.id, - name: manifest.name, - sourceUrl: opts.sourceUrl ?? null, - sourceTrust: opts.sourceTrust, - installedVersion: manifest.version, - manifest: manifest as unknown as Record, - installedAt: now, - updatedAt: now, - installedBy, - }) - .onConflictDoUpdate({ - target: installedExtension.id, - set: { + await tx + .insert(installedExtension) + .values({ + id: manifest.id, name: manifest.name, sourceUrl: opts.sourceUrl ?? null, sourceTrust: opts.sourceTrust, installedVersion: manifest.version, manifest: manifest as unknown as Record, + installedAt: now, updatedAt: now, - }, - }); + installedBy, + }) + .onConflictDoUpdate({ + target: installedExtension.id, + set: { + name: manifest.name, + sourceUrl: opts.sourceUrl ?? null, + sourceTrust: opts.sourceTrust, + installedVersion: manifest.version, + manifest: manifest as unknown as Record, + updatedAt: now, + }, + }); - await tx - .delete(installedExtensionComponent) - .where(eq(installedExtensionComponent.extensionId, manifest.id)); - if (components.length > 0) { - await tx.insert(installedExtensionComponent).values( - components.map((c) => ({ - extensionId: manifest.id, - kind: c.kind, - componentId: c.id, - })), - ); - } - }); + await tx + .delete(installedExtensionComponent) + .where(eq(installedExtensionComponent.extensionId, manifest.id)); + if (components.length > 0) { + await tx.insert(installedExtensionComponent).values( + components.map((c) => ({ + extensionId: manifest.id, + kind: c.kind, + componentId: c.id, + })), + ); + } + }, + ); bookkeepingCommitted = true; // Once the bookkeeping transaction commits, the install is authoritative. @@ -764,12 +886,7 @@ async function installExtensionNow( error instanceof JobOwnershipLost ) throw error; - await ctx - .log( - `Cleanup: the operations agent could not settle integration ${integrationId}: ${safeDiagnostic(error)}`, - "stderr", - ) - .catch(() => {}); + throw new JobObservationPending("recovery_required"); } } for (const { repository, transactionId } of ledger.publishedRepositories) { @@ -777,6 +894,7 @@ async function installExtensionNow( const settled = await settleServiceRepository( { repository, transactionId, outcome: "commit" }, { + ctx, operationKey: createDurableOpsKey( "extension.repository.commit", `${ctx.jobId}:${repository}:${transactionId}`, @@ -795,12 +913,13 @@ async function installExtensionNow( .catch(() => {}); } } catch (error) { - await ctx - .log( - `Cleanup: the operations agent could not settle repository ${repository}: ${safeDiagnostic(error)}`, - "stderr", - ) - .catch(() => {}); + if ( + error instanceof JobObservationPending || + error instanceof OpsObservationPending || + error instanceof JobOwnershipLost + ) + throw error; + throw new JobObservationPending("recovery_required"); } } await ctx @@ -822,7 +941,13 @@ async function installExtensionNow( await ctx .log(`Install failed: ${safeDiagnostic(err)}. Rolling back...`, "stderr") .catch(() => {}); - await rollbackInstall(ctx, ledger).catch((rbErr) => { + await extensionStep(workflow, "rollback.intent", {}, async () => ({ + ledger, + message: safeDiagnostic(err), + })); + await extensionStep(workflow, "rollback", ledger, () => + rollbackInstall(ctx, ledger, workflow), + ).catch((rbErr) => { if ( rbErr instanceof JobObservationPending || rbErr instanceof OpsObservationPending || @@ -835,7 +960,17 @@ async function installExtensionNow( } } -async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise { +async function rollbackInstall( + ctx: JobContext, + ledger: InstallLedger, + workflow: ExtensionWorkflow, +): Promise { + const { + settleIntegration, + settleServiceRepository, + dockerComposeAction, + renderAndPersistCompose, + } = extensionEffects(workflow, ctx); // Integrations first (cheap), then services. Newest first: the agent // restores each previous copy, or removes an integration this run added. One // it cannot undo yet stays pending, and recovery rolls it back, since no @@ -865,6 +1000,7 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< "stderr", ) .catch(() => {}); + throw new JobObservationPending("recovery_required"); } } if (ledger.publishedIntegrations.length > 0) { @@ -874,6 +1010,7 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< if (ledger.enabledServiceIds.length > 0) { for (const svc of ledger.enabledServiceIds) { const removed = await dockerComposeAction(svc, "remove", { + ctx, operationKey: createDurableOpsKey("extension.service.rollback", `${ctx.jobId}:${svc}`), }); if (removed.exitCode !== 0) { @@ -889,6 +1026,7 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< await settleServiceRepository( { ...transaction, outcome: "rollback" }, { + ctx, operationKey: createDurableOpsKey( "extension.repository.rollback", `${ctx.jobId}:${transaction.repository}:${transaction.transactionId}`, @@ -903,6 +1041,7 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< "stderr", ) .catch(() => {}); + throw new JobObservationPending("recovery_required"); } } // With the previous checkouts back, restore the selection and runtimes. The @@ -912,23 +1051,25 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< try { await initServiceRegistry({ strict: true }); await applySelectionRoots( + ctx, ledger.selectedRootsBefore, false, createDurableOpsKey("extension.selection.rollback", ctx.jobId), ctx.signal, ); await renderAndPersistCompose({ + ctx, operationKey: createDurableOpsKey("extension.rollback.render", ctx.jobId), signal: ctx.signal, }); for (const serviceId of ledger.previouslyEnabledServiceIds) { - await serviceStart(serviceId, ctx); + await serviceStart(serviceId, ctx, "extension-restore"); } } catch (error) { await ctx .log(`Rollback: failed to restore service state: ${safeDiagnostic(error)}`, "stderr") .catch(() => {}); - return; + throw new JobObservationPending("recovery_required"); } } for (const transaction of rolledBack) { @@ -936,6 +1077,7 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< await settleServiceRepository( { ...transaction, outcome: "release" }, { + ctx, operationKey: createDurableOpsKey( "extension.repository.release", `${ctx.jobId}:${transaction.repository}:${transaction.transactionId}`, @@ -949,6 +1091,7 @@ async function rollbackInstall(ctx: JobContext, ledger: InstallLedger): Promise< "stderr", ) .catch(() => {}); + throw new JobObservationPending("recovery_required"); } } } @@ -962,7 +1105,8 @@ export function removeExtension( id: string, ): Promise<{ id: string; removed: number }> { return withExtensionChangeLock(async () => { - await recoverInterruptedExtensionChanges(ctx.signal, { ctx }); + if (!ctx.checkpointResult?._extensionWorkflow) + await recoverInterruptedExtensionChanges(ctx.signal, { ctx }); return removeExtensionNow(ctx, id); }); } @@ -971,17 +1115,26 @@ async function removeExtensionNow( ctx: JobContext, id: string, ): Promise<{ id: string; removed: number }> { - const [ext] = await db - .select() - .from(installedExtension) - .where(eq(installedExtension.id, id)) - .limit(1); - if (!ext) throw new Error(`Extension "${id}" is not installed`); - - const components = await db - .select() - .from(installedExtensionComponent) - .where(eq(installedExtensionComponent.extensionId, id)); + const workflow = await openExtensionWorkflow(ctx, { id }); + const { + removeServiceRepository, + removeIntegration, + adoptIntegration, + dockerComposeAction, + renderAndPersistCompose, + } = extensionEffects(workflow, ctx); + const components = await extensionStep(workflow, "remove.snapshot", { id }, async () => { + const [ext] = await db + .select() + .from(installedExtension) + .where(eq(installedExtension.id, id)) + .limit(1); + if (!ext) throw new Error(`Extension "${id}" is not installed`); + return db + .select() + .from(installedExtensionComponent) + .where(eq(installedExtensionComponent.extensionId, id)); + }); const serviceIds = components.filter((c) => c.kind === "service").map((c) => c.componentId); const integrationIds = components @@ -995,16 +1148,23 @@ async function removeExtensionNow( // agent, which no longer reports them at all, refuses to act on them. const reported = serviceIds.length > 0 - ? new Set((await readCommunityServices({ signal: ctx.signal })).reportedServiceIds) + ? new Set( + ( + await extensionStep(workflow, "remove.services.before", {}, () => + readCommunityServices({ signal: ctx.signal }), + ) + ).reportedServiceIds, + ) : new Set(); // Tear down service containers while they're still in the compose file. for (const svc of serviceIds.filter((id) => reported.has(id))) { await ctx.log(`Removing service container ${svc}...`); const removed = await dockerComposeAction(svc, "remove", { + ctx, operationKey: createDurableOpsKey("extension.service.remove", `${ctx.jobId}:${svc}`), signal: ctx.signal, - }).catch(() => ({ exitCode: 1 })); + }); if (removed.exitCode !== 0) { throw new Error(`Failed to remove community service runtime ${svc}`); } @@ -1012,19 +1172,25 @@ async function removeExtensionNow( // Remove the repos this extension owns + drop the services from selection. if (serviceIds.length > 0) { try { - await disableServicesInSelection( - serviceIds, - createDurableOpsKey("extension.selection.disable", `${ctx.jobId}:${id}`), - ctx.signal, - ); + await extensionStep(workflow, "selection.disable", serviceIds, async () => { + const roots = await extensionStep(workflow, "remove.selection.before", {}, async () => [ + ...(await getServiceSelectionSummary(getServiceRegistry())).selectedRoots, + ]); + await applySelectionRoots( + ctx, + roots.filter((root) => !serviceIds.includes(root)), + true, + createDurableOpsKey("extension.selection.disable", `${ctx.jobId}:${id}`), + ctx.signal, + ); + }); } catch (e) { await ctx.log(`Failed to disable services: ${(e as Error).message}`, "stderr"); throw e; } - const repos = await db - .select() - .from(serviceRepository) - .where(eq(serviceRepository.managedByExtension, id)); + const repos = await extensionStep(workflow, "remove.repositories.before", {}, () => + db.select().from(serviceRepository).where(eq(serviceRepository.managedByExtension, id)), + ); // A repository the agent cannot remove fails the job before the extension // record is deleted, so the removal can be retried. for (const repo of repos) { @@ -1032,6 +1198,7 @@ async function removeExtensionNow( await removeServiceRepository( { repository: repo.hash, extensionId: id }, { + ctx, operationKey: createDurableOpsKey( "extension.repository.remove", `${ctx.jobId}:${repo.hash}`, @@ -1039,12 +1206,21 @@ async function removeExtensionNow( signal: ctx.signal, }, ); - await db.delete(serviceRepository).where(eq(serviceRepository.hash, repo.hash)); + await extensionDatabaseStep( + workflow, + `remove.repository.row:${repo.hash}`, + { hash: repo.hash }, + async (tx) => { + await tx.delete(serviceRepository).where(eq(serviceRepository.hash, repo.hash)); + }, + ); } - // disableServicesInSelection already applied the reduced enabled set in - // memory; re-render to drop the service from compose (no registry re-init, - // which would re-read the stale baked env). + // Repository removal changes the authoritative manifest inventory. Re-read + // it before sealing the reduced generation; the original selection phase + // already committed its desired roots and remains a separate retained step. + await initServiceRegistry({ strict: true }); await renderAndPersistCompose({ + ctx, operationKey: createDurableOpsKey("extension.remove.render", `${ctx.jobId}:${id}`), signal: ctx.signal, }); @@ -1054,7 +1230,9 @@ async function removeExtensionNow( // Have the agent remove the integrations. One it cannot remove fails the // job before the extension record is deleted, so the removal can be retried. if (integrationIds.length > 0) { - const installed = await readInstalledIntegrations({ signal: ctx.signal }); + const installed = await extensionStep(workflow, "remove.integrations.before", {}, () => + readInstalledIntegrations({ signal: ctx.signal }), + ); const records = new Map(installed.integrations.map((entry) => [entry.integrationId, entry])); const unrecorded = new Set(installed.unrecorded); for (const intgId of integrationIds) { @@ -1095,17 +1273,29 @@ async function removeExtensionNow( }, ); } - await db - .delete(installedIntegration) - .where( - and(eq(installedIntegration.id, intgId), eq(installedIntegration.managedByExtension, id)), - ); + await extensionDatabaseStep( + workflow, + `remove.integration.row:${intgId}`, + { id: intgId }, + async (tx) => { + await tx + .delete(installedIntegration) + .where( + and( + eq(installedIntegration.id, intgId), + eq(installedIntegration.managedByExtension, id), + ), + ); + }, + ); } await reloadIntegrations().catch(() => {}); } await ctx.setProgress(90); - await db.delete(installedExtension).where(eq(installedExtension.id, id)); + await extensionDatabaseStep(workflow, "remove.parent", { id }, async (tx) => { + await tx.delete(installedExtension).where(eq(installedExtension.id, id)); + }); await ctx.setProgress(100); await ctx.log(`Extension "${id}" removed (${components.length} component(s)).`); return { id, removed: components.length }; diff --git a/apps/api/src/services/extension-recovery.ts b/apps/api/src/services/extension-recovery.ts index fe2b69b47..c478c8a9f 100644 --- a/apps/api/src/services/extension-recovery.ts +++ b/apps/api/src/services/extension-recovery.ts @@ -72,6 +72,7 @@ export async function recoverInterruptedExtensionChanges( } const runtimeRecoveryJournal = await openRuntimeRecoveryAuthority(findRepoRoot()); const serviceRecovery: ServiceRuntimeRecovery = await reconcileCommunityRepositoryTransactions({ + ctx: options.ctx, signal, restoreSelection: async (roots, recoveryId) => { await initServiceRegistry({ strict: true }); diff --git a/apps/api/src/services/extension-workflow-postgres.test.ts b/apps/api/src/services/extension-workflow-postgres.test.ts new file mode 100644 index 000000000..9191ba00c --- /dev/null +++ b/apps/api/src/services/extension-workflow-postgres.test.ts @@ -0,0 +1,458 @@ +import { spawn } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { packageIntegration } from "@openmapx/integration-framework/installer"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { ownedServiceHost } from "../../../ops-agent/src/__fixtures__/owned-service-host"; +import { + IntegrationStore, + installIntegrationRuntime, +} from "../../../ops-agent/src/integration-store"; +import { openOpsJobJournal } from "../../../ops-agent/src/journal"; +import { buildOpsAgentServer } from "../../../ops-agent/src/server"; +import * as schema from "../db/schema"; +import { createMigratedPostgresFixture } from "../test/postgres-fixture"; +import type { JobContext } from "./job-runner"; + +const state = vi.hoisted(() => ({ + fixture: null as Awaited> | null, +})); +vi.mock("../db", () => ({ + get db() { + return state.fixture!.db; + }, + get sql() { + return state.fixture!.sql; + }, +})); +vi.mock("./extension-store", () => ({ + getKillSwitch: async () => ({ removed: new Map(), critical: new Map(), stale: false }), +})); +vi.mock("../integration-host", () => ({ + getAllIntegrations: () => [], + reloadIntegrations: async () => ({ message: "ok", reloaded: 1, enabled: 1 }), +})); +const enabled = process.env.OPENMAPX_RUN_DATABASE_TESTS === "1"; +let fixture: Awaited>; +let host: Awaited>; +let agent: ReturnType; +let installation: ReturnType; +let publications = 0; +beforeAll(async () => { + if (!enabled) return; + fixture = await createMigratedPostgresFixture(); + state.fixture = fixture; + host = await ownedServiceHost({ actualComposeConfig: true }); + writeFileSync( + join(host.infraDir, ".env"), + `UID=${process.getuid?.() ?? 0}\nGID=${process.getgid?.() ?? 0}\n`, + { mode: 0o600 }, + ); + const source = join(host.root, "artifact-source"); + mkdirSync(source); + writeFileSync( + join(source, "manifest.json"), + JSON.stringify({ + id: "workflow-map", + name: "Workflow", + version: "1.0.0", + domains: ["map-overlay"], + quality: "community", + platform: "1.0", + }), + ); + const artifact = join(host.root, "workflow.tar.gz"); + await packageIntegration({ rootDir: host.root, source, outFile: artifact }); + const store = new IntegrationStore({ + rootDir: host.root, + uid: process.geteuid?.() ?? 0, + builtInIds: () => new Set(), + download: async ({ destination }) => { + copyFileSync(artifact, destination); + }, + }); + installIntegrationRuntime(host.runtime, store); + installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); + agent = buildOpsAgentServer({ + tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + runtime: host.runtime, + resourceClaimer: host.claimer, + journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), + controlInstallation: installation, + }); + agent.addHook("preHandler", async (request) => { + const kind = (request.body as { operation?: { kind?: string } } | undefined)?.operation?.kind; + if ( + request.method === "POST" && + ["catalogIntegration.publish", "catalogIntegration.settle"].includes(kind ?? "") + ) + publications++; + }); + + const url = await agent.listen({ host: "127.0.0.1", port: 0 }); + const token = join(host.root, "api-token"); + writeFileSync(token, host.token, { mode: 0o600 }); + for (const [key, value] of Object.entries({ + OPS_AGENT_URL: url, + OPS_AGENT_TOKEN_FILE: token, + OPENMAPX_ROOT_DIR: host.root, + OPS_TRUSTED_CONFIG_DIR: host.queue, + OPS_TRUSTED_CONFIG_UID: String(statSync(host.queue).uid), + OPS_TRUSTED_CONFIG_GID: String(statSync(host.queue).gid), + DOMAIN: "example.test", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + DATA_MANAGER_AUTH_TOKEN: "d".repeat(32), + })) + vi.stubEnv(key, value); +}, 90000); +afterAll(async () => { + await agent?.close(); + if (installation) closeControlInstallation(installation); + if (host) rmSync(host.root, { recursive: true, force: true }); + await fixture?.stop(); + vi.restoreAllMocks(); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); +it.skipIf(!enabled)( + "uses the original real SQL job and agent broker for publication, bookkeeping and settlement", + async () => { + const { checkpointOwnedAdminJob } = await import("./admin-job-checkpoint"); + const { openExtensionWorkflow } = await import("./extension-workflow"); + const { handleExtensionInstallJob } = await import("./extension-installer"); + const artifact = readFileSync(join(host.root, "workflow.tar.gz")); + const payload = { + manifest: { + id: "workflow-bundle", + name: "Workflow bundle", + version: "1.0.0", + integrations: [ + { + id: "workflow-map", + artifact: "https://example.test/workflow.tar.gz", + sha256: createHash("sha256").update(artifact).digest("hex"), + }, + ], + }, + sourceTrust: "community", + }; + const execution = { owner: randomUUID(), version: 1 }; + const id = randomUUID(); + await fixture.db.insert(schema.adminJob).values({ + id, + type: "extension.install", + payload, + status: "running", + result: { _execution: execution }, + }); + let baseline: Record = { _execution: execution }; + const ctx: JobContext = { + jobId: id, + jobType: "extension.install", + execution, + payload, + signal: new AbortController().signal, + checkpointResult: { _execution: execution }, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (value, progress, key, address) => { + if (value.opsProjection) + ctx.checkpointResult = await checkpointOwnedAdminJob( + id, + ctx.execution, + value, + progress, + key, + address, + baseline, + ); + else { + const rows = await fixture.db + .update(schema.adminJob) + .set({ result: value }) + .where(eq(schema.adminJob.id, id)) + .returning(); + ctx.checkpointResult = rows[0]!.result as Record; + } + baseline = structuredClone(ctx.checkpointResult ?? {}); + }, + }; + await openExtensionWorkflow(ctx, payload); + expect(await handleExtensionInstallJob(ctx)).toEqual({ id: "workflow-bundle", components: 1 }); + const count = publications; + const [installed] = await fixture.db + .select() + .from(schema.installedIntegration) + .where(eq(schema.installedIntegration.id, "workflow-map")); + expect(installed?.lastTransactionId).toBeTruthy(); + const [before] = + await fixture.sql`select xmin::text as xmin from installed_extension where id='workflow-bundle'`; + expect(await handleExtensionInstallJob(ctx)).toEqual({ id: "workflow-bundle", components: 1 }); + expect(publications).toBe(count); + const [after] = + await fixture.sql`select xmin::text as xmin from installed_extension where id='workflow-bundle'`; + expect(after?.xmin).toBe(before?.xmin); + }, + 90000, +); + +it.skipIf(!enabled)( + "reclaims native API death before and after atomic bookkeeping without a second publication", + async () => { + const { claimAdminJob } = await import("./admin-job-state"); + for (const stopAt of ["published", "bookkept"]) { + const id = randomUUID(), + bundle = "workflow-bundle"; + const source = { + kind: "artifact", + url: "https://example.test/workflow.tar.gz", + sha256: createHash("sha256") + .update(readFileSync(join(host.root, "workflow.tar.gz"))) + .digest("hex"), + }; + await fixture.db.insert(schema.adminJob).values({ + id, + type: "extension.install", + status: "running", + payload: { source, bundle }, + result: { _execution: { owner: randomUUID(), version: 1 } }, + }); + const count = publications; + const start = (boundary: string) => { + const child = spawn( + process.execPath, + [ + "--import", + createRequire(import.meta.url).resolve("tsx/esm"), + fileURLToPath( + new URL("./__tests__/fixtures/extension-workflow-native.ts", import.meta.url), + ), + id, + boundary, + ], + { + env: { ...process.env, DATABASE_URL: fixture.connectionUri }, + stdio: ["ignore", "ignore", "pipe", "ipc"], + }, + ); + let errors = ""; + child.stderr!.on("data", (data) => { + errors += String(data).slice(0, 4000); + }); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve(code)); + }); + const reached = new Promise((resolve, reject) => { + child.on("message", (message) => { + if ((message as { boundary?: string }).boundary === boundary) resolve(); + }); + child.once("exit", () => reject(new Error(`Native boundary absent: ${errors}`))); + }); + void reached.catch(() => {}); + return { child, exited, reached, errors: () => errors }; + }; + const first = start(stopAt); + try { + await first.reached; + } catch (error) { + first.child.kill("SIGKILL"); + throw error; + } + const before = + await fixture.sql`select xmin::text as xmin from installed_extension where id=${bundle}`; + first.child.kill("SIGKILL"); + await first.exited; + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, id)); + const reclaimed = await claimAdminJob(row!); + expect(reclaimed).toBeTruthy(); + const replacement = start("none"); + expect(await replacement.exited, replacement.errors()).toBe(0); + expect(publications - count).toBe(2); // one publish, one original settlement + const after = + await fixture.sql`select xmin::text as xmin from installed_extension where id=${bundle}`; + expect(after).toHaveLength(1); + if (stopAt === "bookkept") expect(after[0]!.xmin).toBe(before[0]!.xmin); + const { readInstalledIntegrations } = await import("./agent-integrations"); + expect( + (await readInstalledIntegrations()).integrations.find( + (entry) => entry.integrationId === "workflow-map", + )?.pending, + ).toBeNull(); + } + }, + 90000, +); + +it.skipIf(!enabled)( + "installs and removes a service-bearing extension through original-job repository, configuration and Docker brokers", + async () => { + const { checkpointOwnedAdminJob } = await import("./admin-job-checkpoint"); + const { openExtensionWorkflow } = await import("./extension-workflow"); + const { handleExtensionInstallJob, handleExtensionRemoveJob } = await import( + "./extension-installer" + ); + const { initServiceRegistry } = await import("./service-registry"); + await initServiceRegistry({ strict: true }); + host.setCreatingProcess(async (args) => { + if (!args.includes("community-live")) return; + const { services } = await import("@openmapx/core/server"); + const compose = services.configurationGenerationComposeFile( + host.infraDir, + services.currentConfigurationGeneration(host.infraDir)!, + ); + host.setRows([ + host.row, + { + ...host.row, + Id: "c".repeat(64), + ConfigImage: `example/community-live@sha256:${"b".repeat(64)}`, + Labels: { + ...host.row.Labels, + "com.docker.compose.service": "community-live", + "com.docker.compose.project.config_files": compose, + "com.docker.compose.project.working_dir": (await import("node:path")).dirname(compose), + }, + }, + ]); + }); + host.setRemovingProcess(async (args) => { + if (args.includes("community-live")) host.setRows([host.row]); + }); + + async function context(type: string, payload: Record) { + const id = randomUUID(), + execution = { owner: randomUUID(), version: 1 }; + await fixture.db + .insert(schema.adminJob) + .values({ id, type, status: "running", payload, result: { _execution: execution } }); + let baseline: Record = { _execution: execution }; + const ctx: JobContext = { + jobId: id, + jobType: type, + payload, + execution, + signal: new AbortController().signal, + checkpointResult: { _execution: execution }, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (value, progress, key, address) => { + if (value.opsProjection) + ctx.checkpointResult = await checkpointOwnedAdminJob( + id, + ctx.execution, + value, + progress, + key, + address, + baseline, + ); + else { + const [saved] = await fixture.db + .update(schema.adminJob) + .set({ result: value }) + .where(eq(schema.adminJob.id, id)) + .returning(); + ctx.checkpointResult = saved!.result as Record; + } + baseline = structuredClone(ctx.checkpointResult ?? {}); + }, + }; + return ctx; + } + const payload = { + manifest: { + id: "service-workflow", + name: "Service workflow", + version: "1.0.0", + services: [ + { repo: "https://github.com/example/community-services", service: "community-live" }, + ], + }, + sourceTrust: "community", + }; + const ctx = await context("extension.install", payload); + await openExtensionWorkflow(ctx, payload); + async function complete(owner: JobContext, run: () => Promise): Promise { + const { claimAdminJob, readJobExecution } = await import("./admin-job-state"); + for (let n = 0; n < 8; n++) { + try { + return await run(); + } catch (error) { + if ((error as { reason?: string }).reason !== "receipt_delivery" || n === 7) throw error; + const [row] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, owner.jobId)); + const claimed = await claimAdminJob(row!); + if (!claimed) throw error; + owner.execution = readJobExecution(claimed.result); + owner.checkpointResult = claimed.result as Record; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + throw Error("Original observation unavailable"); + } + try { + expect(await complete(ctx, () => handleExtensionInstallJob(ctx))).toEqual({ + id: "service-workflow", + components: 1, + }); + } catch (error) { + console.error( + "extension-phase", + JSON.stringify({ + candidate: ctx.checkpointResult?._configurationCandidate, + configuration: ctx.checkpointResult?.configuration, + }), + ); + throw error; + } + const operations = ( + ctx.checkpointResult!.opsProjection as { + operations: Record< + string, + { + kind: string; + operation?: { expectedConfigurationRevisionId?: string }; + attempt?: { apiJobId: string }; + } + >; + } + ).operations; + expect(Object.values(operations).some((op) => op.kind === "serviceRepository.publish")).toBe( + true, + ); + expect( + Object.values(operations).find((op) => op.kind === "service.pull")?.operation + ?.expectedConfigurationRevisionId, + ).toMatch(/^cfg1_/); + expect(Object.values(operations).every((op) => op.attempt?.apiJobId === ctx.jobId)).toBe(true); + const remove = await context("extension.remove", { id: "service-workflow" }); + await openExtensionWorkflow(remove, { id: "service-workflow" }); + expect(await complete(remove, () => handleExtensionRemoveJob(remove))).toEqual({ + id: "service-workflow", + removed: 1, + }); + expect( + await fixture.db + .select() + .from(schema.installedExtension) + .where(eq(schema.installedExtension.id, "service-workflow")), + ).toEqual([]); + }, + 180000, +); diff --git a/apps/api/src/services/extension-workflow.test.ts b/apps/api/src/services/extension-workflow.test.ts new file mode 100644 index 000000000..bf5177bd4 --- /dev/null +++ b/apps/api/src/services/extension-workflow.test.ts @@ -0,0 +1,60 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import type { JobContext } from "./job-runner"; + +const database = vi.hoisted(() => ({ transaction: vi.fn() })); +vi.mock("../db", () => ({ db: database })); + +import { extensionStep, openExtensionWorkflow } from "./extension-workflow"; + +let directory: string | undefined; +afterEach(async () => { + vi.unstubAllEnvs(); + if (directory) await rm(directory, { recursive: true, force: true }); +}); +async function context() { + directory = await mkdtemp(join(tmpdir(), "extension-cursor-")); + const file = join(directory, "token"); + await writeFile(file, "a".repeat(43), { mode: 0o600 }); + vi.stubEnv("OPS_AGENT_TOKEN_FILE", file); + const ctx: JobContext = { + jobId: randomUUID(), + jobType: "extension.install", + execution: { owner: randomUUID(), version: 1 }, + payload: { original: true }, + signal: new AbortController().signal, + log: async () => {}, + setProgress: async () => {}, + checkpoint: async (value) => { + ctx.checkpointResult = value; + }, + }; + return ctx; +} +it("resumes the original completed publication without redispatch and rejects changed intent or forged cursor", async () => { + const ctx = await context(), + publish = vi.fn(async () => ({ transactionId: "original" })); + const first = await openExtensionWorkflow(ctx, { id: "original" }); + expect(await extensionStep(first, "publish.integration", { id: "one" }, publish)).toEqual({ + transactionId: "original", + }); + const replacement = { ...ctx, execution: { owner: randomUUID(), version: 2 } }; + replacement.checkpoint = async (value) => { + replacement.checkpointResult = value; + }; + const second = await openExtensionWorkflow(replacement, { id: "original" }); + expect(await extensionStep(second, "publish.integration", { id: "one" }, publish)).toEqual({ + transactionId: "original", + }); + expect(publish).toHaveBeenCalledTimes(1); + await expect(openExtensionWorkflow(replacement, { id: "changed" })).rejects.toThrow(); + const saved = replacement.checkpointResult!._extensionWorkflow as { body: string; mac: string }; + replacement.checkpointResult = { + ...replacement.checkpointResult, + _extensionWorkflow: { ...saved, body: saved.body.replace("original", "forged") }, + }; + await expect(openExtensionWorkflow(replacement, { id: "original" })).rejects.toThrow(); +}); diff --git a/apps/api/src/services/extension-workflow.ts b/apps/api/src/services/extension-workflow.ts new file mode 100644 index 000000000..f51174df8 --- /dev/null +++ b/apps/api/src/services/extension-workflow.ts @@ -0,0 +1,201 @@ +/** Private, bounded original-job cursor. This correlates installer phases; every + * host effect still requires the normal typed broker and original deadline. */ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { canonicalJson, readOpsTokenFile } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; +import { z } from "zod"; +import { db } from "../db"; +import { adminJob } from "../db/schema"; +import { JobObservationPending } from "./admin-job-failure"; +import { jobOwnership } from "./admin-job-state"; +import type { JobContext } from "./job-runner"; + +const stepSchema = z.strictObject({ + input: z.string().regex(/^[a-f0-9]{64}$/), + state: z.enum(["attempted", "complete"]), + result: z.unknown().optional(), +}); +const stateSchema = z.strictObject({ + version: z.literal(1), + jobId: z.uuid(), + type: z.enum(["extension.install", "extension.remove"]), + intent: z.string(), + steps: z + .record(z.string().max(200), stepSchema) + .refine((value) => Object.keys(value).length <= 256), +}); +type State = z.infer; +export interface ExtensionWorkflow { + readonly jobId: string; +} +const owners = new WeakMap(); +const digest = (value: unknown) => + createHash("sha256") + .update(canonicalJson(value ?? null)) + .digest("hex"); +const fail = (): never => { + throw new JobObservationPending("recovery_required"); +}; +const token = () => readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); +async function seal(state: State) { + const body = canonicalJson(stateSchema.parse(state)); + if (Buffer.byteLength(body) > 1024 * 1024) return fail(); + return { + body, + mac: createHmac("sha256", await token()) + .update("openmapx-extension-workflow-v1\0") + .update(body) + .digest("hex"), + }; +} +async function read(value: unknown) { + const saved = z + .strictObject({ body: z.string().max(1024 * 1024), mac: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse(value); + const expected = createHmac("sha256", await token()) + .update("openmapx-extension-workflow-v1\0") + .update(saved.body) + .digest(); + if (!timingSafeEqual(expected, Buffer.from(saved.mac, "hex"))) return fail(); + return stateSchema.parse(JSON.parse(saved.body)); +} +export async function openExtensionWorkflow( + ctx: JobContext, + intent: unknown, +): Promise { + if (!ctx.execution || !["extension.install", "extension.remove"].includes(ctx.jobType ?? "")) + return fail(); + const original = digest({ payload: ctx.payload, intent }); + const state: State = + ctx.checkpointResult?._extensionWorkflow === undefined + ? { + version: 1, + jobId: ctx.jobId, + type: ctx.jobType as State["type"], + intent: original, + steps: {}, + } + : await read(ctx.checkpointResult._extensionWorkflow); + if (state.jobId !== ctx.jobId || state.type !== ctx.jobType || state.intent !== original) + return fail(); + const handle = Object.freeze({ jobId: ctx.jobId }); + owners.set(handle, { ctx, state }); + if (ctx.checkpointResult?._extensionWorkflow === undefined) { + if (ctx.observationOnly) return fail(); + await ctx.checkpoint({ ...ctx.checkpointResult, _extensionWorkflow: await seal(state) }); + } + return handle; +} +function owner(handle: ExtensionWorkflow) { + const value = owners.get(handle); + if (!value) return fail(); + return value; +} +export async function extensionStep( + handle: ExtensionWorkflow, + name: string, + input: unknown, + effect: () => Promise, +): Promise { + const { ctx, state } = owner(handle), + fingerprint = digest(input), + prior = state.steps[name]; + if (prior && prior.input !== fingerprint) return fail(); + if (prior?.state === "complete") return structuredClone(prior.result) as T; + ctx.signal.throwIfAborted(); + if (!prior) { + state.steps[name] = { input: fingerprint, state: "attempted" }; + await ctx.checkpoint({ ...ctx.checkpointResult, _extensionWorkflow: await seal(state) }); + } + const result = await effect(); + const next = { + ...state, + steps: { + ...state.steps, + [name]: { input: fingerprint, state: "complete" as const, result: result ?? null }, + }, + }; + await ctx.checkpoint({ ...ctx.checkpointResult, _extensionWorkflow: await seal(next) }); + state.steps = next.steps; + return result; +} +/** Lock the actual administrative owner before bookkeeping and publish its + * completion cursor in that SAME transaction. A reclaimed owner cannot commit. */ +export async function extensionDatabaseStep( + handle: ExtensionWorkflow, + name: string, + input: unknown, + write: (tx: Parameters[0]>[0]) => Promise, +): Promise { + const { ctx, state } = owner(handle), + fingerprint = digest(input), + prior = state.steps[name]; + if (prior && prior.input !== fingerprint) return fail(); + if (prior?.state === "complete") return structuredClone(prior.result) as T; + let committed: Record | undefined; + let next: State | undefined; + const result = await db + .transaction(async (tx) => { + const [row] = await tx + .select() + .from(adminJob) + .where(jobOwnership(ctx.jobId, ctx.execution)) + .for("update"); + if ( + !row || + row.type !== ctx.jobType || + canonicalJson(row.payload) !== canonicalJson(ctx.payload) + ) + return fail(); + const current = (row.result ?? {}) as Record, + retained = await read(current._extensionWorkflow); + if (canonicalJson(retained) !== canonicalJson(state)) return fail(); + const output = await write(tx); + next = { + ...state, + steps: { + ...state.steps, + [name]: { input: fingerprint, state: "complete", result: output ?? null }, + }, + }; + committed = { ...current, _extensionWorkflow: await seal(next) }; + const rows = await tx + .update(adminJob) + .set({ result: committed }) + .where(jobOwnership(ctx.jobId, ctx.execution)) + .returning({ id: adminJob.id }); + if (rows.length !== 1) return fail(); + return output; + }) + .catch((error) => { + // These server transaction-abort codes prove no bookkeeping commit. + // Lost COMMIT acknowledgement and all unclassified I/O remain unknown. + const source = error as { code?: unknown; cause?: { code?: unknown } }; + const code = source?.code ?? source?.cause?.code; + if (typeof code === "string" && ["23503", "23505", "23514", "40001", "40P01"].includes(code)) + throw error; + return fail(); + }); + if (!committed || !next) return fail(); + ctx.checkpointResult = committed; + state.steps = next.steps; + return result; +} +/** Routing only; openExtensionWorkflow authenticates this before any work. */ +export function hasRetainedExtensionWorkflow(job: { type: string; result: unknown }) { + const state = (job.result as Record | null)?._extensionWorkflow; + return ["extension.install", "extension.remove"].includes(job.type) && state !== undefined; +} + +export function completedExtensionStep(handle: ExtensionWorkflow, name: string): T | undefined { + const step = owner(handle).state.steps[name]; + return step?.state === "complete" ? (structuredClone(step.result) as T) : undefined; +} + +export const readExtensionWorkflow = read; + +export function hasExtensionPublications(handle: ExtensionWorkflow): boolean { + return Object.keys(owner(handle).state.steps).some( + (name) => name.startsWith("repository.publish:") || name.startsWith("integration.publish:"), + ); +} diff --git a/apps/api/src/services/integration-application.test.ts b/apps/api/src/services/integration-application.test.ts index c5292a8f9..64c65682c 100644 --- a/apps/api/src/services/integration-application.test.ts +++ b/apps/api/src/services/integration-application.test.ts @@ -131,7 +131,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, resourceClaimer: host.claimer, - journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), controlInstallation: installation, }); agent.addHook("onSend", async (request, _reply, payload) => { diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index e4fc4d271..6504cc0a1 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -5,6 +5,7 @@ import { integrationApplyReceiptSchema, } from "@openmapx/core/api-contracts"; import { canonicalJson, hardlinkApplySummarySchema, readOpsTokenFile } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; import { and, asc, eq, inArray, sql } from "drizzle-orm"; import { db } from "../db"; import { adminJob } from "../db/schema"; @@ -41,6 +42,7 @@ import { } from "./configuration-ownership-view"; import { coordinatorLifecycleOperationKey } from "./coordinator-lifecycle"; import { normalizeExtensionInstallPayload } from "./extension-install-payload"; +import { hasRetainedExtensionWorkflow } from "./extension-workflow"; import { IntegrationBindingConflict as RetainedBindingConflict } from "./integration-binding-participant"; import { type JobStatusEvent, jobEventBus, TERMINAL_JOB_STATUSES } from "./job-events"; import { ApiOpsError, createDurableOpsKey, OpsObservationPending } from "./ops-client"; @@ -318,7 +320,7 @@ class AdminJobRunner { ): Promise<{ jobId: string; operationId: string }> { if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/.test(key)) throw new RetainedJobConflict("Invalid mutation key"); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); const digest = createHmac("sha256", token) .update("openmapx-standalone-request-v1\0") .update(canonicalJson({ type, payload, actorId })) @@ -394,7 +396,7 @@ class AdminJobRunner { ) => Promise<{ value: T; result: Record; failed?: boolean }>, restore: (result: Record) => T, ): Promise { - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); const intentDigest = createHmac("sha256", token) .update("openmapx-immediate-configuration-intent-v1\0") .update(canonicalJson(options.intent)) @@ -1188,7 +1190,8 @@ class AdminJobRunner { (hasRetainedImmediateConfiguration(job) || hasRetainedQueuedCoordinator(job) || retainedMaintenanceJob(job) || - hasRetainedStandalone(job)); + hasRetainedStandalone(job) || + hasRetainedExtensionWorkflow(job)); this.active.set(job.id, controller); this.logSeq.set(job.id, 0); @@ -1263,12 +1266,21 @@ class AdminJobRunner { observe && hasRetainedQueuedCoordinator({ id: jobId, type, payload, result: checkpointResult }), retainedStandalone = observe && hasRetainedStandalone({ type, result: checkpointResult }), + retainedExtension = + observe && hasRetainedExtensionWorkflow({ type, result: checkpointResult }), retainedMaintenance = observe && retainedMaintenanceJob({ type, payload }), retainedConfiguration = - retainedImmediate || retainedQueuedCoordinator || retainedStandalone || retainedMaintenance, + retainedImmediate || + retainedQueuedCoordinator || + retainedStandalone || + retainedMaintenance || + retainedExtension, handler = retainedImmediate ? (ctx: JobContext) => this.resumeImmediateConfiguration(ctx) - : retainedQueuedCoordinator || retainedStandalone || retainedMaintenance + : retainedQueuedCoordinator || + retainedStandalone || + retainedMaintenance || + retainedExtension ? this.handlers.get(type) : observe ? ["extension.install", "extension.remove"].includes(type) diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index 35bc9fee3..db4b418d3 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -41,6 +41,7 @@ import { trustedConfigurationPayloadSchema, } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; +import { envString } from "@openmapx/core/server-env"; import { and, eq, sql } from "drizzle-orm"; import { z } from "zod"; import * as schema from "../db/schema"; @@ -123,6 +124,7 @@ import { closeUnissuedCoordinatorPull, coordinatorLifecycleOperationKey, } from "./coordinator-lifecycle"; +import { readExtensionWorkflow } from "./extension-workflow"; import { assertIntegrationBindingOverlay, assertSavedIntegrationBinding, @@ -1448,7 +1450,7 @@ async function resumeCompletedConfigurationRetirement(): Promise | undefined; const candidate = await withConfigurationWriteTransaction(writer, async (tx, read) => { @@ -2492,7 +2494,7 @@ async function restoreRetainedOrdinaryCandidate(): Promise | undefined; const candidate = await withConfigurationWriteTransaction(writer, async (tx) => { @@ -2732,9 +2734,7 @@ export async function reconcileUnstagedCoordinatorPull(): Promise { const operation = entry.operation; if (operation?.kind !== "service.pull" || !operation.candidateId) throw new JobObservationPending("invalid_identity"); - const publisher = publisherOptions( - await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""), - ), + const publisher = publisherOptions(await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", ""))), retained = await findPreparedTrustedConfigurationCandidate(publisher, operation.candidateId); if (retained) { const record = retained.record, @@ -3299,6 +3299,9 @@ export async function prepareProspectiveConfiguration(input: { const { advanceCompletedBulkCandidate } = await import("./service-update-batch"); await advanceCompletedBulkCandidate(writer); } + if (["extension.install", "extension.remove"].includes(owner.job.jobType ?? "")) { + await advanceCompletedExtensionCandidate(writer); + } const coordinator = input.coordinatorIntent ? await assertCoordinatorLifecycleIntent(input.coordinatorIntent, writer) : null; @@ -3327,7 +3330,7 @@ export async function prepareProspectiveConfiguration(input: { canonicalJson({ purpose: "render", coordinatorMode: "ensure-running", changes: [] })) ) throw Error("Fixed unchanged coordinator render required"); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); const publisher = publisherOptions(token); const source = platformIntent ? retainedPlatformConfigurationSource(writer) @@ -6248,3 +6251,86 @@ export async function persistPreparedFeedCredentialPatch(input: { ); return persistPreparedFeedCredentialRow({ ...input, candidateState: state }); } + +/** Successive installer configuration phases retain all original ops/ACKs. Only + * a completed ordinary candidate in the authenticated same-job cursor advances. */ +async function advanceCompletedExtensionCandidate( + writer: import("./administrative-configuration-writer-owner").AdministrativeConfigurationWriter, +) { + const { job } = administrativeConfigurationState(writer); + const current = job.checkpointResult?._configurationCandidate as + | Record + | undefined; + if (!current || current.operationKey === writer.operationKey) return; + const frame = await readExtensionWorkflow(job.checkpointResult?._extensionWorkflow); + if ( + frame.jobId !== job.jobId || + frame.type !== job.jobType || + job.observationOnly || + current.maintenanceId + ) + throw new JobObservationPending("recovery_required"); + const payload = job.payload as { + manifest?: { id?: string; services?: Array<{ service: string }> }; + id?: string; + }; + const id = payload.manifest?.id ?? payload.id; + if (!id) throw new JobObservationPending("recovery_required"); + const allowed = new Set([ + createDurableOpsKey("extension.selection.enable", `${job.jobId}:${id}`), + createDurableOpsKey("extension.config.render", `${job.jobId}:${id}`), + createDurableOpsKey("extension.selection.disable", `${job.jobId}:${id}`), + createDurableOpsKey("extension.remove.render", `${job.jobId}:${id}`), + createDurableOpsKey("extension.selection.rollback", job.jobId), + createDurableOpsKey("extension.rollback.render", job.jobId), + ...(payload.manifest?.services ?? []).flatMap((s) => + ["start", "extension-restore"].map((scope) => + createDurableOpsKey("admin-job.config.prepare", `${job.jobId}:${scope}:${s.service}`), + ), + ), + ]); + if (!allowed.has(String(current.operationKey)) || !allowed.has(writer.operationKey)) + throw new JobObservationPending("recovery_required"); + let committed: Record | undefined; + await withConfigurationWriteTransaction(writer, async (tx) => { + const [row] = await tx + .select() + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"); + if ( + !row || + row.type !== job.jobType || + canonicalJson(row.payload) !== canonicalJson(job.payload) + ) + throw new JobObservationPending("recovery_required"); + const result = (row.result ?? {}) as Record; + if ( + canonicalJson(await readExtensionWorkflow(result._extensionWorkflow)) !== + canonicalJson(frame) || + canonicalJson(result._configurationCandidate) !== canonicalJson(current) || + current.phase !== "settled" || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) + ) + throw new JobObservationPending("recovery_required"); + const outcome = parseTrustedConfigurationApplyOutcome( + (result._configurationOutcomes as Record)?.[String(current.operationKey)], + ); + if ( + !completeTrustedConfigurationApply(outcome) || + outcome.outcome !== "committed" || + outcome.revisionId !== current.revisionId + ) + throw new JobObservationPending("recovery_required"); + committed = { ...result }; + for (const field of ["_configurationCandidate", "_completion"]) delete committed[field]; + const rows = await tx + .update(schema.adminJob) + .set({ result: committed }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: schema.adminJob.id }); + if (rows.length !== 1) throw new JobObservationPending("recovery_required"); + }); + if (committed) job.checkpointResult = committed; +} diff --git a/apps/api/src/services/retained-maintenance-authorization.ts b/apps/api/src/services/retained-maintenance-authorization.ts index 11c75fddd..e8dd6982a 100644 --- a/apps/api/src/services/retained-maintenance-authorization.ts +++ b/apps/api/src/services/retained-maintenance-authorization.ts @@ -1,6 +1,7 @@ /** Existing API token authenticates the exact retained row, request and actor. */ import { createHmac, timingSafeEqual } from "node:crypto"; import { canonicalJson, readOpsTokenFile } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; import { JobObservationPending } from "./admin-job-failure"; import type { JobContext } from "./job-runner"; export const MAINTENANCE_JOB_TYPES = [ @@ -48,7 +49,7 @@ export async function assertRetainedMaintenanceJob(ctx: JobContext): Promise { if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/.test(key)) throw new MaintenanceRequestConflict("Invalid mutation key"); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); const digest = createHmac("sha256", token) .update("openmapx-maintenance-request-v1\0") .update(canonicalJson({ type, payload, actorId })) @@ -86,7 +87,7 @@ export async function lookupMaintenanceJob( ): Promise { if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/.test(key)) throw new MaintenanceRequestConflict("Invalid mutation key"); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""), + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")), digest = createHmac("sha256", token) .update("openmapx-maintenance-request-v1\0") .update(canonicalJson({ type, payload, actorId })) diff --git a/apps/api/src/services/service-registry.ts b/apps/api/src/services/service-registry.ts index 6d121ff00..9d8892a63 100644 --- a/apps/api/src/services/service-registry.ts +++ b/apps/api/src/services/service-registry.ts @@ -1,6 +1,6 @@ import { join } from "node:path"; import { findRepoRoot, services } from "@openmapx/core/server"; -import { readCommunityServices } from "./community-repositories"; +import { readCommunityServices } from "./community-service-inventory"; const { APPLIED_SERVICES_ENV, diff --git a/apps/api/src/services/standalone-integrations/install.test.ts b/apps/api/src/services/standalone-integrations/install.test.ts index 2c2b5c0d0..b27f772c9 100644 --- a/apps/api/src/services/standalone-integrations/install.test.ts +++ b/apps/api/src/services/standalone-integrations/install.test.ts @@ -76,7 +76,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: state.host.token, "data-manager": "b".repeat(43) }, runtime: state.host.runtime, resourceClaimer: state.host.claimer, - journal: await openOpsJobJournal(join(state.host.infraDir, "data/ops-agent/jobs-v1.json")), + journal: await openOpsJobJournal(join(state.host.infraDir, "state/ops-agent/jobs-v1.json")), controlInstallation: installation, artifactTransfers: store, }); diff --git a/apps/api/src/services/standalone-integrations/install.ts b/apps/api/src/services/standalone-integrations/install.ts index 19c5b8103..b899b8752 100644 --- a/apps/api/src/services/standalone-integrations/install.ts +++ b/apps/api/src/services/standalone-integrations/install.ts @@ -11,6 +11,7 @@ import { standaloneInstallSourceSchema, standaloneIntegrationProvenanceSchema, } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; import { and, eq, sql } from "drizzle-orm"; import z from "zod/v4"; import { db } from "../../db"; @@ -33,7 +34,7 @@ type State = import("./state").StandaloneState; import { readStandaloneState, sealStandaloneState } from "./state"; async function token() { - return readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + return readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); } async function authenticatePayload(ctx: JobContext) { const payload = ctx.payload; diff --git a/apps/api/src/services/standalone-integrations/ownership-state.ts b/apps/api/src/services/standalone-integrations/ownership-state.ts index 4ca1a2f3c..da114e2bd 100644 --- a/apps/api/src/services/standalone-integrations/ownership-state.ts +++ b/apps/api/src/services/standalone-integrations/ownership-state.ts @@ -4,6 +4,7 @@ import { readOpsTokenFile, standaloneOwnershipInspectionSchema, } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; import z from "zod/v4"; export const ownershipRequestSchema = z.strictObject({ integrationId: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/), @@ -37,7 +38,7 @@ export const ownershipStateSchema = z.strictObject({ outcome: z.enum(["committed", "restored"]).nullable(), }); export type OwnershipState = z.infer; -const token = () => readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); +const token = () => readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); export async function sealOwnershipState(state: OwnershipState) { const body = canonicalJson(ownershipStateSchema.parse(state)); return { diff --git a/apps/api/src/services/standalone-integrations/state.ts b/apps/api/src/services/standalone-integrations/state.ts index 6c303d492..2597a8980 100644 --- a/apps/api/src/services/standalone-integrations/state.ts +++ b/apps/api/src/services/standalone-integrations/state.ts @@ -6,6 +6,7 @@ import { standaloneInstallInspectionSchema, standaloneInstallSourceSchema, } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; import z from "zod/v4"; export const standaloneStateSchema = z.strictObject({ version: z.literal(1), @@ -32,7 +33,7 @@ export type StandaloneState = z.infer; export async function sealStandaloneState(state: StandaloneState) { const body = canonicalJson(standaloneStateSchema.parse(state)); - const token = await readOpsTokenFile(process.env.OPS_AGENT_TOKEN_FILE ?? ""); + const token = await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", "")); const mac = createHmac("sha256", token) .update("openmapx-standalone-install-state-v1\0") .update(body) @@ -43,7 +44,7 @@ export async function readStandaloneState(value: unknown): Promise service.manifest.id), }); await installPinnedCredentialCatalog(host.root, installation); - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); agent = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, diff --git a/apps/api/src/services/trusted-configuration-producer.ts b/apps/api/src/services/trusted-configuration-producer.ts index 96e6434e1..e07b2d2fa 100644 --- a/apps/api/src/services/trusted-configuration-producer.ts +++ b/apps/api/src/services/trusted-configuration-producer.ts @@ -95,6 +95,32 @@ export async function applyTrustedConfigurationWithEvidence( integrationReload?: CapturedIntegrationReloadResult; integrationApplication?: IntegrationApplyReceipt; }> { + if ( + options.ctx && + ["extension.install", "extension.remove"].includes(options.ctx.jobType ?? "") && + options.ctx.checkpointResult?._extensionWorkflow + ) { + const ctx = options.ctx, + retained = ctx.checkpointResult?._configurationCandidate as + | Record + | undefined; + if (retained?.operationKey === options.operationKey && retained.phase !== "settled") { + const { retainedConfigurationIntegrationIds, resumeRetainedConfigurationCandidate } = + await import("./prospective-configuration"); + const actorId = options.actorId ?? ctx.jobId; + const integrationIds = await retainedConfigurationIntegrationIds( + ctx, + actorId, + options.operationKey, + ); + await withAdministrativeConfigurationJob(ctx, () => + withAdministrativeConfigurationWriter( + { actorId, operationKey: options.operationKey, integrationIds, signal: ctx.signal }, + () => resumeRetainedConfigurationCandidate(), + ), + ); + } + } if (options.ctx && hasRetainedAdminJobOperation(options.ctx, options.operationKey)) { const outcome = configurationApplyResultSchema.parse( await observeRetainedAdminJobOperation(options.ctx, options.operationKey), diff --git a/apps/api/src/utils/docker-compose.ts b/apps/api/src/utils/docker-compose.ts index 6ba9b915c..cff277e52 100644 --- a/apps/api/src/utils/docker-compose.ts +++ b/apps/api/src/utils/docker-compose.ts @@ -1,5 +1,7 @@ import type { ServerResponse } from "node:http"; import type { OpsOperation, OpsResultFor } from "@openmapx/core/ops"; +import { executeAdminJobOperation } from "../services/admin-job-ops"; +import type { JobContext } from "../services/job-runner"; import { type ApiOpsClient, ApiOpsError, @@ -21,6 +23,7 @@ interface OpsDependencies { } interface AsyncOpsDependencies extends OpsDependencies { + ctx?: JobContext; operationKey?: string; signal?: AbortSignal; expectedConfigurationRevisionId?: string; @@ -135,6 +138,12 @@ export async function dockerComposeAction( stderr: "A durable operation identity is required for this request.", }; } + if (options.ctx) { + await executeAdminJobOperation(options.ctx, operation, "extension.lifecycle", { + operationKey: options.operationKey, + }); + return { exitCode: 0, stdout: "", stderr: "" }; + } try { await executeAndWait(client(options), operation, options.operationKey, { signal: options.signal, diff --git a/apps/ops-agent/src/__fixtures__/configuration-runtime.ts b/apps/ops-agent/src/__fixtures__/configuration-runtime.ts new file mode 100644 index 000000000..060de98f4 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/configuration-runtime.ts @@ -0,0 +1,44 @@ +import { dirname } from "node:path"; +import { services } from "@openmapx/core/server"; +import { registerConfigurationMaintenanceSource } from "../configuration-maintenance-store"; +import { createOwnedServiceResolver } from "../owned-services"; +import type { OpsRuntime } from "../runtime"; +import { + installTrustedConfigurationRuntime as install, + type TrustedConfigurationRuntimeOptions, +} from "../trusted-config-runtime"; + +/** Direct adapter tests still run the literal production inventory qualification. + * The Docker seam describes an empty host, so it cannot fabricate a coordinator + * owner, a staged proof, or permission to publish over a live staged container. */ +export function installConfigurationRuntimeFixture( + runtime: OpsRuntime, + options: TrustedConfigurationRuntimeOptions, +): void { + install(runtime, options); + const stack = { + infraDir: options.infraDir, + composePath: services.currentConfigurationFile(options.infraDir, services.GENERATED_COMPOSE_FILE), + composeReleasePath: services.releaseStatePaths(options.infraDir).overlay, + }; + const execFile = async (_file: string, args: readonly string[]) => { + if (args[0] !== "container" || args[1] !== "ls") + throw Error(`Unexpected configuration fixture Docker request: ${args.join(" ")}`); + return { stdout: "", stderr: "" }; + }; + const loadAuthority = options.loadAuthority ?? (async () => ({ + revisionId: "registry-v1", + services: options.services, + integrationSchemas: options.integrationSchemas, + })); + const ownedServices = createOwnedServiceResolver({ stack, execFile, loadAuthority }); + registerConfigurationMaintenanceSource(runtime, { + rootDir: options.rootDir ?? dirname(dirname(options.infraDir)), + infraDir: options.infraDir, + candidateSource: options.candidateSource, + stack, + execFile, + loadAuthority, + ownedServices, + }); +} diff --git a/apps/ops-agent/src/__fixtures__/configuration-source.ts b/apps/ops-agent/src/__fixtures__/configuration-source.ts index 206279d30..9b454f7a1 100644 --- a/apps/ops-agent/src/__fixtures__/configuration-source.ts +++ b/apps/ops-agent/src/__fixtures__/configuration-source.ts @@ -5,11 +5,15 @@ import { services } from "@openmapx/core/server"; import { opsOperationFingerprint } from "@openmapx/core/ops"; import { dispatchOpsOperation, type OpsRuntime } from "../runtime"; import { testExecution } from "./execution"; -import { mkdirSync } from "node:fs"; +import { mkdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; const sources = new WeakMap }>(); export async function configurationSource(runtime: OpsRuntime, infraDir: string) { - const revision = services.currentConfigurationGeneration(infraDir); + let environment: string | null = null; + try { environment = readFileSync(join(infraDir, ".env"), "utf8"); } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const revision = JSON.stringify([services.currentConfigurationGeneration(infraDir), environment]); let retained = sources.get(runtime); if (!retained || retained.revision !== revision) { const value = (async () => { diff --git a/apps/ops-agent/src/__fixtures__/configuration-writer.mjs b/apps/ops-agent/src/__fixtures__/configuration-writer.mjs index d23c09a8a..f5f4d6e48 100644 --- a/apps/ops-agent/src/__fixtures__/configuration-writer.mjs +++ b/apps/ops-agent/src/__fixtures__/configuration-writer.mjs @@ -7,7 +7,7 @@ import { closeControlInstallation, observeConfigurationSources, openControlInsta import { renderComposeForRepo } from "../../../../packages/cli/src/commands/compose"; import { createUnavailableRuntime, dispatchOpsOperation } from "../runtime"; import { testExecution } from "./execution"; -import { installTrustedConfigurationRuntime } from "../trusted-config-runtime"; +import { installConfigurationRuntimeFixture as installTrustedConfigurationRuntime } from "./configuration-runtime"; const [root, mode, letter, expected, hold] = process.argv.slice(2); const infraDir = join(root, "infra/docker"); diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index 1f24bde29..9bfeb7b13 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -41,6 +41,7 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: chmodSync(root, 0o755); const infraDir = join(root, "infra/docker"); mkdirSync(infraDir, { recursive: true }); + writeFileSync(join(infraDir, ".env"), `UID=${process.geteuid?.() ?? 0}\nGID=${process.getegid?.() ?? 0}\n`, { mode: 0o600 }); mkdirSync(join(root, "services")); mkdirSync(join(root, "custom_integrations")); writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); @@ -131,6 +132,7 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: | ((args: readonly string[], signal: AbortSignal) => Promise) | undefined; let inventoryProcess: (() => Promise) | undefined; + let removingProcess: ((args: readonly string[]) => Promise) | undefined; let imagePullProcess: ((reference:string,signal:AbortSignal)=>Promise) | undefined; const imageIdentities = new Map(); const optionsForHelperLaunchResponseLoss = options.configurationHelperLaunchResponseLoss; @@ -157,7 +159,7 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: else if(format==="{{.Config.User}}")stdout=`${process.getuid?.()??0}:${process.getgid?.()??0}`; else if(format==="{{json .HostConfig.GroupAdd}}")stdout="[]"; else if(format?.includes("range .Config.Env"))stdout="HOME\nPATH\n"; - else if(format==="{{json .Mounts}}")stdout=JSON.stringify([{Destination:"/var/run/docker.sock",Source:"/var/run/docker.sock"},{Destination:root,Source:root}]); + else if(format==="{{json .Mounts}}")stdout=JSON.stringify([{Type:"bind",RW:true,Destination:"/var/run/docker.sock",Source:"/var/run/docker.sock"},{Type:"bind",RW:true,Destination:root,Source:root}]); else throw Error("Unexpected fixed helper prerequisite format"); } else if(configurationHelperVersion!==undefined && args[0]==="image" && args[1]==="inspect" && args[3]?.includes("openmapx.configuration-handoff.version")) { if(args.at(-1)!==imageId)throw Error("Unexpected old helper image"); @@ -193,6 +195,8 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: else if (args[0] === "image" && args[1] === "pull") await imagePullProcess?.(args.at(-1)!,options.signal); else if (args[0] === "compose" && args.includes("up")) await creatingProcess?.(args, options.signal); + else if (args[0] === "compose" && args.includes("rm")) + await removingProcess?.(args); else if (args[0] === "stop") { const stopped = rows.find((candidate) => candidate.Id === args[1]); if (stopped) stopped.State = "exited"; @@ -364,6 +368,7 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: setCreatingProcess(value: typeof creatingProcess) { creatingProcess = value; }, + setRemovingProcess(value: typeof removingProcess) { removingProcess = value; }, setInventoryProcess(value: typeof inventoryProcess) { inventoryProcess = value; }, diff --git a/apps/ops-agent/src/__fixtures__/server.ts b/apps/ops-agent/src/__fixtures__/server.ts new file mode 100644 index 000000000..7e22e7369 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/server.ts @@ -0,0 +1,38 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { buildOpsAgentServer, type BuildOpsAgentServerOptions } from "../server"; + +/** Stable retry identity with an actual protocol-v2 root expiry. */ +export function controlTestKey(label: string, notAfter: number): string { + return createControlRootOperationKey({ + notAfter, + nonce: createHash("sha256").update(label).digest().subarray(0, 16), + }); +} + +/** Exercise the server's real certified admission and retained control files. */ +export async function buildControlTestServer(options: BuildOpsAgentServerOptions) { + if (options.controlInstallation) return buildOpsAgentServer(options); + const root = mkdtempSync(join(tmpdir(), "openmapx-server-control-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["redis", "motis", "osrm", "valhalla", "postgis", "app-api", "traefik", "data-manager", "alpha"], + }); + const app = buildOpsAgentServer({ ...options, controlInstallation: installation }); + app.addHook("onClose", async () => { + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + }); + return app; +} diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index 95160852d..28323843d 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -436,7 +436,7 @@ describe("the fixed CLI", () => { } finally { vi.unstubAllEnvs(); } - expect(lines).toEqual(["stdout: data link http://data-manager:4000 inherited"]); + expect(lines).toEqual(["stdout: data link http://data-manager:4000/ inherited"]); }); }); @@ -1040,6 +1040,7 @@ describe("administrative release runtime", () => { }); // Constructing the effects reads nothing: the new store is still empty. expect(readdirSync(releaseState(rootDir).store)).toEqual([]); + await effects.initialize?.(); await expect(effects.runtimeInspect?.()).resolves.toEqual({ releaseId: "release-legacy" }); expect(migrations).toEqual([ expect.objectContaining({ outcome: "migrated", overlay: "regenerated", problems: [] }), @@ -1103,7 +1104,7 @@ describe("administrative release runtime", () => { expect(readFileSync(overlay, "utf8")).toBe("services: {}\n"); }); - it("gives the self-update helper the release state's store, overlay and evidence", async () => { + it("lets the legacy self-update helper read checked pins but refuses unguarded overlay mutation", async () => { const rootDir = temporaryRoot(); const { store, overlay, evidence } = releaseState(rootDir); mkdirSync(store, { recursive: true, mode: 0o700 }); @@ -1129,11 +1130,13 @@ describe("administrative release runtime", () => { expect(() => bindings.releasePins("release-helper", "0".repeat(64))).toThrow( "Release authority rejected", ); - await bindings.withStoreLock(() => bindings.writeOverlay("release-helper", digest)); - expect(readFileSync(overlay, "utf8")).toContain("file: ../../releases/evidence/"); - expect(readdirSync(evidence)).toHaveLength(1); - expect(bindings.stackArgs()).toEqual(expect.arrayContaining(["-f", overlay])); - await bindings.withStoreLock(() => bindings.restoreOverlay(null)); + await expect( + bindings.withStoreLock(() => bindings.writeOverlay("release-helper", digest)), + ).rejects.toThrow("Typed adapter is not wired"); + await expect(bindings.withStoreLock(() => bindings.restoreOverlay(null))).rejects.toThrow( + "Typed adapter is not wired", + ); + expect(existsSync(evidence)).toBe(false); expect(existsSync(overlay)).toBe(false); expect(bindings.stackArgs()).not.toContain(overlay); }); diff --git a/apps/ops-agent/src/agent-handoff.test.ts b/apps/ops-agent/src/agent-handoff.test.ts index 42368adb2..769a18da3 100644 --- a/apps/ops-agent/src/agent-handoff.test.ts +++ b/apps/ops-agent/src/agent-handoff.test.ts @@ -432,6 +432,16 @@ describe("agent handoff helper container", () => { "type=bind,source=/srv/openmapx/infra/docker/data,target=/srv/openmapx/infra/docker/data", "type=bind,source=/srv/openmapx/infra/docker/secrets,target=/srv/openmapx/infra/docker/secrets", "type=bind,source=/srv/openmapx/infra/docker/backups,target=/srv/openmapx/infra/docker/backups", + ...["ops-agent", "data-manager", "host-cli"].map( + (role) => + `type=bind,source=/srv/openmapx/infra/docker/control-private/hidden-key,target=/srv/openmapx/infra/docker/secrets/control-${role}.key,readonly`, + ), + ...["data-manager", "host-cli"].map( + (role) => + `type=bind,source=/srv/openmapx/infra/docker/control-private/empty,target=/srv/openmapx/infra/docker/control-private/${role},readonly`, + ), + "type=bind,source=/srv/openmapx/infra/docker/control-private/ops-agent,target=/var/lib/openmapx/control-private", + "type=bind,source=/srv/openmapx/infra/docker/secrets/control-verifiers.json,target=/run/secrets/openmapx-control-verifiers,readonly", ]); expect(mounts.join("\n")).not.toContain("target=/srv/openmapx,"); }); diff --git a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts index 1815e7c2b..9b0bf3db3 100644 --- a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts +++ b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts @@ -64,7 +64,7 @@ async function fixture() { ownedServices: host.owned, execFile: host.execFile, }); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal), app = buildOpsAgentServer({ runtime: host.runtime, journal, @@ -466,7 +466,7 @@ it("reconciles the original prepared publication after exact foreign-inode repai ownedServices: productionOwnedServices(runtime), execFile: f.execFile, }); - const journal = await openOpsJobJournal(join(f.infraDir, "data/ops-agent/jobs-v1.json")), + const journal = await openOpsJobJournal(services.agentStatePaths(f.infraDir).journal), app = buildOpsAgentServer({ runtime, journal, @@ -669,7 +669,7 @@ async function reassembleCommittedFixture(f: Awaited> ownedServices: productionOwnedServices(runtime), execFile: f.execFile, }); - const journal = await openOpsJobJournal(join(f.infraDir, "data/ops-agent/jobs-v1.json")), + const journal = await openOpsJobJournal(services.agentStatePaths(f.infraDir).journal), app = buildOpsAgentServer({ runtime, journal, diff --git a/apps/ops-agent/src/artifact-transfer/motis.test.ts b/apps/ops-agent/src/artifact-transfer/motis.test.ts index a426b4114..45666a477 100644 --- a/apps/ops-agent/src/artifact-transfer/motis.test.ts +++ b/apps/ops-agent/src/artifact-transfer/motis.test.ts @@ -13,7 +13,11 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join, relative } from "node:path"; +import { services as coreServices } from "@openmapx/core/server"; + +const { CONFIGURATION_GENERATIONS, CURRENT_CONFIGURATION } = coreServices; + import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; import { createControlRootOperationKey } from "@openmapx/core/ops"; import { afterEach, expect, it, vi } from "vitest"; @@ -217,7 +221,7 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { ]) copyFileSync(join(ctx.motisStagingDataDir, name), join(stagingDir, name)); const configurationRevisionId = `cfg1_${"a".repeat(43)}`, - generation = join(infraDir, ".trusted-config-generations", configurationRevisionId), + generation = join(infraDir, CONFIGURATION_GENERATIONS, configurationRevisionId), secretDir = join(generation, ".generated-secrets/data-manager"); mkdirSync(secretDir, { recursive: true, mode: 0o700 }); for (const path of [generation, join(generation, ".generated-secrets"), secretDir]) @@ -225,8 +229,8 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { const secret = join(secretDir, "TRANSITOUS_API_KEYS"); writeFileSync(secret, JSON.stringify({ "de/demo": destinationKey }), { mode: 0o444 }); symlinkSync( - `.trusted-config-generations/${configurationRevisionId}`, - join(infraDir, ".trusted-config-current"), + relative(dirname(join(infraDir, CURRENT_CONFIGURATION)), generation), + join(infraDir, CURRENT_CONFIGURATION), ); return { rootDir: root, @@ -440,7 +444,7 @@ it("imports an actual portable candidate through the production agent with desti }; const app = buildOpsAgentServer({ runtime: host.runtime, - journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal: await openOpsJobJournal(coreServices.agentStatePaths(host.infraDir).journal), resourceClaimer: host.claimer, controlInstallation: installation, artifactTransfers: store, diff --git a/apps/ops-agent/src/artifact-transfer/store.test.ts b/apps/ops-agent/src/artifact-transfer/store.test.ts index ff61e1e43..e91d6d204 100644 --- a/apps/ops-agent/src/artifact-transfer/store.test.ts +++ b/apps/ops-agent/src/artifact-transfer/store.test.ts @@ -350,7 +350,7 @@ it("admits a declared 600 GiB stage using its actual reservation", async () => { it("streams generated 2 GiB with bounded pull-ahead and no materialized body", async () => { const { store } = await fixture(5 * 1024 ** 3); const length = 2 * 1024 ** 3, - chunk = new Uint8Array(1024 ** 2); + chunk = new Uint8Array(4 * 1024 ** 2); let emitted = 0, maximumAhead = 0; const stage = await store.createStage({ ...input(), declaredBytes: length }); diff --git a/apps/ops-agent/src/backup-prune-runtime.test.ts b/apps/ops-agent/src/backup-prune-runtime.test.ts index d510963ce..34cd21e73 100644 --- a/apps/ops-agent/src/backup-prune-runtime.test.ts +++ b/apps/ops-agent/src/backup-prune-runtime.test.ts @@ -47,7 +47,7 @@ it("awaits the actual held filesystem deletion before settling cancellation and role: "ops-agent", serviceIds: ["motis"], }), - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal), app = buildOpsAgentServer({ runtime: host.runtime, journal, @@ -142,7 +142,7 @@ it("previews and prunes only the unchanged original eligible set through the pro role: "ops-agent", serviceIds: ["motis"], }), - journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal), app = buildOpsAgentServer({ runtime: host.runtime, resourceClaimer: host.claimer, diff --git a/apps/ops-agent/src/bootstrap.test.ts b/apps/ops-agent/src/bootstrap.test.ts index 1ac3ee921..c633e6acf 100644 --- a/apps/ops-agent/src/bootstrap.test.ts +++ b/apps/ops-agent/src/bootstrap.test.ts @@ -12,7 +12,7 @@ it.each(["legacy", "expired"])( "fix1 production bootstrap retains health and observation with %s release recovery", async (mode) => { const root = mkdtempSync(join(tmpdir(), "openmapx-bootstrap-")); - const directory = join(root, "infra/docker/.ops-agent-releases"); + const directory = services.releaseStatePaths(join(root, "infra/docker")).store; mkdirSync(directory, { recursive: true, mode: 0o700 }); const manifest = { schemaVersion: 1, @@ -60,7 +60,7 @@ it.each(["legacy", "expired"])( }), { mode: 0o600 }, ); - const overlay = join(root, "infra/docker/docker-compose.release.yml"); + const overlay = services.releaseStatePaths(join(root, "infra/docker")).overlay; writeFileSync(overlay, "operator-state\n"); const run = vi.fn(async () => undefined); const dispatch = vi.fn(async () => ({ reachable: true, version: "test" })); diff --git a/apps/ops-agent/src/configuration-release.test.ts b/apps/ops-agent/src/configuration-release.test.ts index 5bece7421..e6a291a1c 100644 --- a/apps/ops-agent/src/configuration-release.test.ts +++ b/apps/ops-agent/src/configuration-release.test.ts @@ -22,8 +22,8 @@ beforeEach(() => { vi.stubEnv("OPENMAPX_RELEASE", "main"); vi.stubEnv("OPENMAPX_RELEASE_MANIFEST_REPOSITORY", "ghcr.io/openmapx/release-manifest"); root = mkdtempSync(join(tmpdir(), "openmapx-configuration-release-")); - directory = join(root, ".ops-agent-releases"); - mkdirSync(directory, { mode: 0o700 }); + directory = services.releaseStatePaths(root).store; + mkdirSync(directory, { recursive: true, mode: 0o700 }); raw = services.canonicalReleaseManifest( services.parseReleaseManifest( JSON.stringify({ @@ -266,7 +266,7 @@ it.each(["transaction", "active-handoff", "consumed-handoff"])( const { writeAgentHandoff, agentHandoffPath } = await import("./agent-handoff"); const checkout = join(root, "legacy-checkout"), infraDir = join(checkout, "infra/docker"), - store = join(infraDir, ".ops-agent-releases"); + store = services.releaseStatePaths(infraDir).store; mkdirSync(store, { recursive: true, mode: 0o700 }); writeFileSync(join(store, `${releaseId}.json`), raw, { mode: 0o600 }); let legacy: string; diff --git a/apps/ops-agent/src/control-admission.test.ts b/apps/ops-agent/src/control-admission.test.ts index 3177cf85c..1810b3332 100644 --- a/apps/ops-agent/src/control-admission.test.ts +++ b/apps/ops-agent/src/control-admission.test.ts @@ -9,6 +9,7 @@ import { executionFingerprint, opsOperationFingerprint, } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; import { expect, it } from "vitest"; import { type OpsJobJournal, openOpsJobJournal } from "./journal"; import type { OpsExecutionContext } from "./runtime"; @@ -23,7 +24,7 @@ it("refuses paired approval as a bearer-only root before reserving a worker", as role: "ops-agent", serviceIds: ["motis"], }); - const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal); const app = buildOpsAgentServer({ controlInstallation: installation, journal, @@ -149,7 +150,7 @@ it.each(["normal", "ready-failure", "wrong-target", "canceled", "cancel-journal- }); let workerContext: OpsExecutionContext | undefined; const token = "b".repeat(32); - const journalFile = join(root, "data/ops-agent/jobs-v1.json"); + const journalFile = coreServices.agentStatePaths(root).journal; const options = { tokens: { api: "a".repeat(32), "data-manager": token }, controlInstallation: agent, diff --git a/apps/ops-agent/src/control-renewal.test.ts b/apps/ops-agent/src/control-renewal.test.ts index d9c82602e..d800cce1c 100644 --- a/apps/ops-agent/src/control-renewal.test.ts +++ b/apps/ops-agent/src/control-renewal.test.ts @@ -11,6 +11,7 @@ import { parseOpsJobStatus, signApiRetentionProof, } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; import { expect, it, vi } from "vitest"; import { openOpsJobJournal } from "./journal"; import { buildOpsAgentServer } from "./server"; @@ -26,7 +27,7 @@ it("retains the exact API terminal dependency until ack, retires it, and dischar }); const token = "a".repeat(32), dmToken = "b".repeat(32), - journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")), + journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal), dispatch = vi.fn(async () => ({ backupId: "daily" })); const app = buildOpsAgentServer({ tokens: { api: token, "data-manager": dmToken }, diff --git a/apps/ops-agent/src/control-terminal-restart.test.ts b/apps/ops-agent/src/control-terminal-restart.test.ts index 615a91de1..7ad6244be 100644 --- a/apps/ops-agent/src/control-terminal-restart.test.ts +++ b/apps/ops-agent/src/control-terminal-restart.test.ts @@ -14,6 +14,7 @@ import { parseOpsJobStatus, signApiRetentionProof, } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; import { build } from "esbuild"; import { afterAll, beforeAll, expect, it, vi } from "vitest"; import { openOpsJobJournal, type PersistedOpsJob } from "./journal"; @@ -169,7 +170,7 @@ it.each( else if (change.endsWith("cancel")) altered.terminationRequestedAt = new Date(originalTime).toISOString(); else altered.terminalAt = new Date(originalTime + 1000).toISOString(); - const file = join(root, "data/ops-agent/jobs-v1.json"), + const file = coreServices.agentStatePaths(root).journal, changedJournal = await openOpsJobJournal(file); await changedJournal.replace([altered]); // Actual parser accepts each schema-valid change. const clock = vi.spyOn(Date, "now").mockReturnValue(originalTime + 71 * 3600_000); @@ -322,12 +323,14 @@ it.each( original.admittedAt += 1; original.submittedAt = new Date(original.admittedAt).toISOString(); writeFileSync( - join(root, "data/ops-agent/jobs-v1.json"), + coreServices.agentStatePaths(root).journal, JSON.stringify({ version: 2, jobs: [original] }), ); } else { original.result = { backupId: "different-result" }; - const changedJournal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const changedJournal = await openOpsJobJournal( + coreServices.agentStatePaths(root).journal, + ); await changedJournal.replace([original]); } } @@ -352,7 +355,7 @@ it.each( role: "ops-agent", serviceIds: ["motis"], }); - const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal); const terminalClaimCleanup = "revisionId" in original.operation ? await initializeTerminalClaimCleanup(join(root, "trusted-config"), { @@ -384,7 +387,7 @@ it.each( role: "ops-agent", serviceIds: ["motis"], }); - const unproved = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const unproved = await openOpsJobJournal(coreServices.agentStatePaths(root).journal); const claimed = join(root, `trusted-config/.claimed/${original.operation.revisionId}.json`); expect(existsSync(claimed)).toBe(true); app = buildOpsAgentServer({ @@ -618,7 +621,7 @@ it("repairs a borrowed terminal worker after the actual canonical-digest restart role: "ops-agent", serviceIds: ["motis"], }); - const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal); app = buildOpsAgentServer({ tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32) }, controlInstallation: installation, diff --git a/apps/ops-agent/src/core-service-runtime.test.ts b/apps/ops-agent/src/core-service-runtime.test.ts index a819db4d1..e8854457d 100644 --- a/apps/ops-agent/src/core-service-runtime.test.ts +++ b/apps/ops-agent/src/core-service-runtime.test.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { realpathSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey } from "@openmapx/core/ops"; @@ -33,7 +33,13 @@ describe("production core restart workflow", () => { async (serviceId, fail) => { const host = await ownedServiceHost(); roots.push(host.cleanupRoot); - await host.select(["app-api", "traefik"]); + mkdirSync(join(host.root, "services/traefik/config"), { recursive: true }); + cpSync( + join(import.meta.dirname, "../../../services/traefik/config"), + join(host.root, "services/traefik/config"), + { recursive: true }, + ); + expect(await host.select(["app-api", "traefik"])).toMatchObject({ outcome: "committed" }); const compose = realpathSync(host.stack.composePath); const service = host.loaded.find((s) => s.manifest.id === serviceId)!; host.setRows([ @@ -75,7 +81,7 @@ describe("production core restart workflow", () => { role: "ops-agent", serviceIds: host.loaded.map((s) => s.manifest.id), }); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal); const app = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": "d".repeat(32) }, runtime, diff --git a/apps/ops-agent/src/data-transit-resource-source.test.ts b/apps/ops-agent/src/data-transit-resource-source.test.ts index 9cbf09b8a..6f2a1be40 100644 --- a/apps/ops-agent/src/data-transit-resource-source.test.ts +++ b/apps/ops-agent/src/data-transit-resource-source.test.ts @@ -19,6 +19,7 @@ import { OpsClient, opsOperationFingerprint, } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; import { expect, it } from "vitest"; import { applyHardlinkPlan } from "../../../packages/hardlinks/src/index"; import { createDataTransitTrustedOpsDataSource } from "./data-transit-resource-source"; @@ -164,7 +165,7 @@ async function fixture( return { stdout: "", stderr: "" }; }, }), - journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")), + journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal), token = "b".repeat(43), tokenFile = join(root, "token"); writeFileSync(tokenFile, token, { mode: 0o600 }); diff --git a/apps/ops-agent/src/data-workflow-runtime.test.ts b/apps/ops-agent/src/data-workflow-runtime.test.ts index 701a0428a..e5641370f 100644 --- a/apps/ops-agent/src/data-workflow-runtime.test.ts +++ b/apps/ops-agent/src/data-workflow-runtime.test.ts @@ -155,7 +155,7 @@ it("uses adopted default regions without expanding the deployment region authori input: { ...services.readAppliedConfiguration(host.infraDir), domain: "fixture.example.test", - selectedRoots: host.loaded.map((s) => s.manifest.id), + selectedRoots: services.readDesiredSelection(host.infraDir).roots, platform: { version: 1, defaultRegion: { source: "database", value: "europe/france" } }, }, }); diff --git a/apps/ops-agent/src/journal.ts b/apps/ops-agent/src/journal.ts index af48f12f7..00604c680 100644 --- a/apps/ops-agent/src/journal.ts +++ b/apps/ops-agent/src/journal.ts @@ -26,6 +26,7 @@ import { validateRetainedConfigurationMaintenanceRecoveryControl, validateRetainedExecutionAdmission, } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; const OPS_JOB_JOURNAL_MAX_ENTRIES = 256; export const OPS_JOB_JOURNAL_MAX_BYTES = 24 * 1024 * 1024; @@ -92,7 +93,10 @@ const productionJournalIndexes = new WeakMap< /** A bounded locator view of this actual journal, never source authority. */ export function readProductionCoordinatorJournalIndex(journal: OpsJobJournal, dataRoot: string) { const owned = productionJournalIndexes.get(journal); - if (!owned || owned.path !== join(realpathSync(dataRoot), "ops-agent", "jobs-v1.json")) + if ( + !owned || + owned.path !== coreServices.agentStatePaths(dirname(realpathSync(dataRoot))).journal + ) throw Error("Actual fixed ops-agent journal required"); const records = owned.read(); if (records.length > OPS_JOB_JOURNAL_MAX_ENTRIES) throw Error("Ops journal index limit"); diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index 5f3d90926..aaa240035 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -79,7 +79,7 @@ describe("assembled production service authority", () => { host.setServiceState("data-manager", "exited"); await expect(effects.inspectSystem(io)).resolves.toMatchObject({ maintenanceReady: false }); host.setServiceState("data-manager", "running"); - const path = join(host.infraDir, ".ops-agent-releases", "transaction.json"); + const path = join(services.releaseStatePaths(host.infraDir).store, "transaction.json"); mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); writeFileSync(path, "{}", { mode: 0o600 }); await expect(effects.inspectSystem(io)).resolves.toMatchObject({ maintenanceReady: false }); @@ -90,7 +90,7 @@ describe("assembled production service authority", () => { it("clears only the exact selected pin through the production runtime without Docker mutation", async () => { const host = await fixture(), - path = join(host.infraDir, "docker-compose.release.yml"); + path = services.releaseStatePaths(host.infraDir).overlay; const manifest = { schemaVersion: 1 as const, release: "checked-pin", @@ -111,7 +111,7 @@ describe("assembled production service authority", () => { }, }, }; - services.writeReleaseComposeArtifacts(manifest as services.ReleaseManifest, path); + services.writeReleaseComposeArtifacts(manifest as services.ReleaseManifest, host.infraDir); expect(await productionReleaseEffects(host.runtime).inspect()).toMatchObject({ selectedReleaseId: "checked-pin", }); @@ -179,7 +179,9 @@ describe("assembled production service authority", () => { (args[0] === "compose" && args.includes("config")), ), ).toBe(true); - expect(existsSync(join(host.infraDir, ".ops-agent-releases", "transaction.json"))).toBe(false); + expect( + existsSync(join(services.releaseStatePaths(host.infraDir).store, "transaction.json")), + ).toBe(false); expect(existsSync(agentHandoffPath(host.root))).toBe(false); expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); await expect(effects.inspect()).resolves.toBeDefined(); @@ -233,7 +235,7 @@ describe("assembled production service authority", () => { it("production startup preserves actual interrupted legacy transaction and handoff while reads remain available", async () => { const host = await fixture(); - const directory = join(host.infraDir, ".ops-agent-releases"); + const directory = services.releaseStatePaths(host.infraDir).store; const manifest = services.parseReleaseManifest( JSON.stringify({ schemaVersion: 1, @@ -362,7 +364,7 @@ describe("assembled production service authority", () => { role: "ops-agent", serviceIds: host.loaded.map((s) => s.manifest.id), }); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal); const app = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, @@ -486,7 +488,7 @@ describe("assembled production service authority", () => { tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, resourceClaimer: host.claimer, - journal: await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")), + journal: await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal), controlInstallation: installation, }), deadline = Date.now() + 60000, @@ -594,7 +596,7 @@ describe("assembled production service authority", () => { role: "ops-agent", serviceIds: host.loaded.map((s) => s.manifest.id), }); - const journal = await openOpsJobJournal(join(host.infraDir, "data/ops-agent/jobs-v1.json")); + const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal); const app = buildOpsAgentServer({ tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, runtime: host.runtime, @@ -895,10 +897,9 @@ async function installedCommunity() { serviceId: "community-live", expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, } as const; - const checkout = join(host.root, "services/.community", publication.repository); + const checkout = join(services.agentStatePaths(host.infraDir).community, publication.repository); const record = join( - host.root, - "services/.community", + services.agentStatePaths(host.infraDir).community, `.repository-${publication.repository}.json`, ); const create = () => { diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 5220dc66d..30b24bc29 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -70,6 +70,16 @@ export interface ProductionOpsRuntimeOptions { runFixedCli?: FixedCliRunner; candidateSource?: ConfigurationCandidateSource; } +const registeredServiceSources = new WeakMap< + OpsRuntime, + ProductionOpsRuntimeOptions["loadAuthority"] +>(); +/** Literal factory-owned registry source; requests cannot supply this list. */ +export async function productionRegisteredServiceIds(runtime: OpsRuntime): Promise { + const load = registeredServiceSources.get(runtime); + if (!load) throw Error("Production service registry unavailable"); + return (await load()).services.map((service) => service.manifest.id).sort(); +} const resolvers = new WeakMap(); const releaseAdapters = new WeakMap(); @@ -506,6 +516,7 @@ export async function createProductionOpsRuntime( }); }; resolvers.set(runtime, owned); + registeredServiceSources.set(runtime, options.loadAuthority); releaseAdapters.set(runtime, releaseEffects); return runtime; } diff --git a/apps/ops-agent/src/release-service-coverage.test.ts b/apps/ops-agent/src/release-service-coverage.test.ts index 9c3c236e5..175da4a72 100644 --- a/apps/ops-agent/src/release-service-coverage.test.ts +++ b/apps/ops-agent/src/release-service-coverage.test.ts @@ -3,8 +3,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { services } from "@openmapx/core/server"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { testExecution } from "./__fixtures__/execution"; +import { + buildControlTestServer as buildOpsAgentServer, + controlTestKey, +} from "./__fixtures__/server"; import { createDefaultReleaseEffects } from "./administrative-runtime"; -import { buildOpsAgentServer } from "./server"; const roots: string[] = []; afterEach(() => { @@ -41,7 +45,11 @@ function fixture(mode = "matching", crash = false) { "services: {}\n", ); const calls: string[][] = []; - const context = { signal: new AbortController().signal, emitLog: vi.fn() }; + const context = { + execution: testExecution(), + signal: new AbortController().signal, + emitLog: vi.fn(), + }; const docker = vi.fn(async (args: readonly string[]) => { if (mode === "unreachable") throw new Error("daemon unavailable"); if (args.includes("ps")) return mode === "missing" ? "" : "b".repeat(64); @@ -110,7 +118,7 @@ describe("complete administrative release coverage", () => { const f = fixture("old-agent"); const now = new Date(); const token = Buffer.alloc(32, 1).toString("base64url"); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: token, "data-manager": Buffer.alloc(32, 2).toString("base64url") }, resourcePolicy: { allowRelease: () => true }, dispatch: async () => { @@ -121,11 +129,12 @@ describe("complete administrative release coverage", () => { const admitted = await app.inject({ method: "POST", url: "/v1/operations", - headers: { authorization: `Bearer ${token}` }, + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }, payload: { - version: 1, + version: 2, requestId: "ops1_hostUpdateRequest", - operationKey: "opk1_hostUpdateRequest", + operationKey: controlTestKey("opk1_hostUpdateRequest", now.getTime() + 60000), + execution: { requestedDeadlineAt: now.getTime() + 60000 }, issuedAt: now.toISOString(), expiresAt: new Date(now.getTime() + 20_000).toISOString(), operation: { kind: "release.apply", releaseId: "test-release" }, @@ -138,7 +147,7 @@ describe("complete administrative release coverage", () => { const response = await app.inject({ method: "GET", url: `/v1/operations/${id}`, - headers: { authorization: `Bearer ${token}` }, + headers: { authorization: `Bearer ${token}`, "x-ops-protocol-version": "2" }, }); return response.json().result; }) diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index 3e9126ffc..87df9fc18 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -4,13 +4,26 @@ import { opsOperationFingerprint, } from "@openmapx/core/ops"; import { describe, expect, it, vi } from "vitest"; +import { + buildControlTestServer as buildOpsAgentServer, + controlTestKey, +} from "./__fixtures__/server"; import type { OpsJobJournal, PersistedOpsJob } from "./journal"; import type { OpsExecutionContext } from "./runtime"; -import { buildOpsAgentServer, OPS_AGENT_RETENTION_LIMITS, type OpsAuditEvent } from "./server"; +import { OPS_AGENT_RETENTION_LIMITS, type OpsAuditEvent } from "./server"; const apiToken = Buffer.alloc(32, 21).toString("base64url"); const dataManagerToken = Buffer.alloc(32, 22).toString("base64url"); -const baseTime = Date.parse("2026-08-23T18:00:00.000Z"); +const baseTime = Date.now(); +const deadlineAt = baseTime + 3_600_000; +const key = (label: string) => controlTestKey(label, deadlineAt); +const pullResult = { + changed: true as const, + revisionId: `cfg1_${"a".repeat(43)}`, + imageReference: "example/redis:1", + imageId: `sha256:${"b".repeat(64)}`, + releaseSelectionDigest: null, +}; const allowAllResources: OpsResourcePolicy = { allowGlobal: () => true, @@ -34,6 +47,7 @@ function request(operation: OpsOperation, requestId: string, operationKey: strin version: 2, requestId, operationKey, + execution: { requestedDeadlineAt: deadlineAt }, issuedAt: new Date(baseTime).toISOString(), expiresAt: new Date(baseTime + 20_000).toISOString(), operation, @@ -63,16 +77,16 @@ describe("ops-agent async foundation", () => { }); const dispatch = vi.fn(async (operation: OpsOperation) => { await blocked; - return operation.kind === "service.pull" ? { changed: true } : { reachable: true }; + return operation.kind === "service.pull" ? pullResult : { reachable: true }; }); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 23), dispatch, }); - const operationKey = "opk1_stableTransportRetry"; + const operationKey = key("opk1_stableTransportRetry"); const operation = { kind: "service.pull", serviceId: "redis", @@ -182,7 +196,7 @@ describe("ops-agent async foundation", () => { }), (response) => response.json().result.state === "succeeded", ); - expect(succeeded.json().result.result).toEqual({ changed: true }); + expect(succeeded.json().result.result).toEqual(pullResult); const terminalLookup = await app.inject({ method: "GET", @@ -206,7 +220,7 @@ describe("ops-agent async foundation", () => { }); it("rejects resource and option mismatches for an exact retained terminal key", async () => { - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), @@ -219,7 +233,7 @@ describe("ops-agent async foundation", () => { tail: 20, maxDurationSeconds: 900, } as const; - const operationKey = "opk1_terminalExactLookup00"; + const operationKey = key("opk1_terminalExactLookup00"); const admitted = await app.inject({ method: "POST", url: "/v1/operations", @@ -289,7 +303,7 @@ describe("ops-agent async foundation", () => { }); }, ); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), @@ -303,7 +317,7 @@ describe("ops-agent async foundation", () => { tail: 20, maxDurationSeconds: 300, } as const; - const operationKey = "opk1_cancelFollow00000"; + const operationKey = key("opk1_cancelFollow00000"); const admitted = await app.inject({ method: "POST", url: "/v1/operations", @@ -358,7 +372,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "service.restart", serviceId: "redis" }, `ops1_restartAttempt${String(restartAttempt).padStart(4, "0")}`, - "opk1_restartAdmission00", + key("opk1_restartAdmission00"), ), }); if (lastRestartResponse.statusCode === 202) { @@ -429,7 +443,7 @@ describe("ops-agent async foundation", () => { return { lines: 0, truncated: false }; }, ); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, journal, @@ -450,7 +464,7 @@ describe("ops-agent async foundation", () => { maxDurationSeconds: 300, }, `ops1_cancelPersist${behavior === "abort-respecting" ? "Respect" : "Ignore"}00`, - `opk1_cancelPersist${behavior === "abort-respecting" ? "Respect" : "Ignore"}00`, + key(`opk1_cancelPersist${behavior === "abort-respecting" ? "Respect" : "Ignore"}00`), ), }); const operationId = admitted.json().result.operationId as string; @@ -470,7 +484,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "service.restart", serviceId: "redis" }, "ops1_cancelPersistBusy00", - "opk1_cancelPersistBusy00", + key("opk1_cancelPersistBusy00"), ), }); expect(busy.statusCode).toBe(429); @@ -495,7 +509,7 @@ describe("ops-agent async foundation", () => { }); let calls = 0; const audits: OpsAuditEvent[] = []; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), @@ -515,7 +529,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "docker.status" }, "ops1_timedOutRequest000", - "opk1_timedOutOperation0", + key("opk1_timedOutOperation0"), ), }); expect(first.statusCode).toBe(504); @@ -527,7 +541,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "docker.status" }, "ops1_busyAfterTimeout0", - "opk1_busyAfterTimeout00", + key("opk1_busyAfterTimeout00"), ), }); expect(stillBusy.statusCode).toBe(429); @@ -544,7 +558,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "docker.status" }, "ops1_afterSettlement0", - "opk1_afterSettlement00", + key("opk1_afterSettlement00"), ), }); expect(afterSettlement.statusCode).toBe(200); @@ -552,7 +566,7 @@ describe("ops-agent async foundation", () => { }); it("accepts only bounded UTF-8 log events from the typed execution context", async () => { - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), @@ -572,7 +586,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "service.logs.follow", serviceId: "redis", tail: 10, maxDurationSeconds: 1 }, "ops1_followRequest00000", - "opk1_followOperation000", + key("opk1_followOperation000"), ), }); const operationId = admitted.json().result.operationId as string; @@ -610,7 +624,7 @@ describe("ops-agent async foundation", () => { }); it("rejects oversized sync output and retains the maximum build-all receipt", async () => { - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), @@ -635,7 +649,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "service.logs", serviceId: "redis", tail: 300 }, "ops1_syncOverflow00000", - "opk1_syncOverflow00000", + key("opk1_syncOverflow00000"), ), }); expect(sync.statusCode).toBe(500); @@ -648,7 +662,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "services.buildAll" }, "ops1_asyncOverflow0000", - "opk1_asyncOverflow0000", + key("opk1_asyncOverflow0000"), ), }); const operationId = admitted.json().result.operationId as string; @@ -667,23 +681,27 @@ describe("ops-agent async foundation", () => { it("retains terminal idempotency through bounded result retrieval, then prunes by TTL", async () => { let nowMs = baseTime; let randomValue = 30; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(nowMs), randomBytes: () => Buffer.alloc(18, randomValue++), maxJobEntries: 1, jobRetentionMs: 1_000, - dispatch: async () => ({ changed: true }), + dispatch: async (operation) => + operation.kind === "service.logs.follow" + ? { lines: 0, truncated: false } + : { changed: true }, }); const firstPayload = request( { - kind: "service.pull", + kind: "service.logs.follow", serviceId: "redis", - expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, + tail: 10, + maxDurationSeconds: 300, }, "ops1_retainedFirst0000", - "opk1_retainedFirst0000", + key("opk1_retainedFirst0000"), ); const first = await app.inject({ method: "POST", @@ -703,7 +721,7 @@ describe("ops-agent async foundation", () => { payload: request( { kind: "service.restart", serviceId: "redis" }, "ops1_retainedFull00000", - "opk1_retainedSecond000", + key("opk1_retainedSecond000"), ), }); expect(full.statusCode).toBe(429); @@ -717,16 +735,21 @@ describe("ops-agent async foundation", () => { expect(retry.statusCode).toBe(202); expect(retry.json().result.operationId).toBe(firstId); - nowMs += 1_001; + nowMs += 72 * 3600_000 + 1; const afterTtl = await app.inject({ method: "POST", url: "/v1/operations", headers: auth(), - payload: request( - { kind: "service.restart", serviceId: "redis" }, - "ops1_afterRetention000", - "opk1_retainedSecond000", - ), + payload: { + ...request( + { kind: "service.restart", serviceId: "redis" }, + "ops1_afterRetention000", + controlTestKey("after-retention", nowMs + 3600_000), + ), + issuedAt: new Date(nowMs).toISOString(), + expiresAt: new Date(nowMs + 20_000).toISOString(), + execution: { requestedDeadlineAt: nowMs + 3600_000 }, + }, }); expect(afterTtl.statusCode).toBe(202); expect(afterTtl.json().result.operationId).not.toBe(firstId); diff --git a/apps/ops-agent/src/server-review-2.test.ts b/apps/ops-agent/src/server-review-2.test.ts index 33fc8eca1..046a34fbd 100644 --- a/apps/ops-agent/src/server-review-2.test.ts +++ b/apps/ops-agent/src/server-review-2.test.ts @@ -8,13 +8,26 @@ import { opsOperationFingerprint, } from "@openmapx/core/ops"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + buildControlTestServer as buildOpsAgentServer, + controlTestKey, +} from "./__fixtures__/server"; import { type OpsJobJournal, openOpsJobJournal, type PersistedOpsJob } from "./journal"; import type { OpsResourceClaimer } from "./policy"; -import { buildOpsAgentServer } from "./server"; + +const pullResult = { + changed: true as const, + revisionId: `cfg1_${"a".repeat(43)}`, + imageReference: "example/redis:1", + imageId: `sha256:${"b".repeat(64)}`, + releaseSelectionDigest: null, +}; const apiToken = Buffer.alloc(32, 31).toString("base64url"); const dataManagerToken = Buffer.alloc(32, 32).toString("base64url"); -const baseTime = Date.parse("2026-08-23T18:00:00.000Z"); +const baseTime = Date.now(); +const deadlineAt = baseTime + 3_600_000; +const key = (label: string) => controlTestKey(label, deadlineAt); const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); @@ -22,17 +35,23 @@ afterEach(() => { function request(operation: OpsOperation, requestId: string, operationKey: string) { return { - version: 1, + version: 2, requestId, operationKey, + execution: { requestedDeadlineAt: deadlineAt }, issuedAt: new Date(baseTime).toISOString(), expiresAt: new Date(baseTime + 20_000).toISOString(), operation, }; } +function legacyRequest(...args: Parameters) { + const { execution: _execution, ...body } = request(...args); + return { ...body, version: 1 }; +} + function auth() { - return { authorization: `Bearer ${apiToken}` }; + return { authorization: `Bearer ${apiToken}`, "x-ops-protocol-version": "2" }; } const allow: OpsResourceClaimer = { @@ -52,7 +71,7 @@ function persisted( role: "api", operation, operationId: "job1_restoredOperation0", - operationKey: "opk1_restoredOperation0", + operationKey: key("opk1_restoredOperation0"), fingerprint: opsOperationFingerprint(operation), resourceId: "redis", state: "running", @@ -73,7 +92,7 @@ describe("reviewed durable admission boundary", () => { kind: "stack.render" as const, revisionId: `cfg1_${"t".repeat(43)}`, }; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, claimTimeoutMs: 5, resourceClaimer: { @@ -103,7 +122,7 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_lateClaimRollback0", "opk1_lateClaimRollback0"), + payload: request(operation, "ops1_lateClaimRollback0", key("opk1_lateClaimRollback0")), }); expect(response.statusCode).toBe(504); finish(); @@ -126,7 +145,7 @@ describe("reviewed durable admission boundary", () => { kind: "stack.render" as const, revisionId: `cfg1_${"r".repeat(43)}`, }; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: { claim: async (_operation, fingerprint) => ({ @@ -166,7 +185,7 @@ describe("reviewed durable admission boundary", () => { payload: request( operation, `ops1_terminalRelease${outcome}0`, - `opk1_terminalRelease${outcome}0`, + key(`opk1_terminalRelease${outcome}0`), ), }); expect(admitted.statusCode).toBe(202); @@ -199,13 +218,13 @@ describe("reviewed durable admission boundary", () => { } as const; const tombstone = persisted(operation, { operationId: "job1_recoveryTombstone0", - operationKey: "opk1_recoveryTombstone0", + operationKey: key("opk1_recoveryTombstone0"), state: "failed", errorClass: "recovery_required", terminalAt: new Date(baseTime).toISOString(), }); - const dispatch = vi.fn(async () => ({ changed: true })); - const app = buildOpsAgentServer({ + const dispatch = vi.fn(async () => pullResult); + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal: { records: () => [tombstone], replace: async () => undefined }, @@ -218,7 +237,7 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_tombstoneRetry000", "opk1_recoveryTombstone0"), + payload: legacyRequest(operation, "ops1_tombstoneRetry000", key("opk1_recoveryTombstone0")), }); expect(retry.statusCode).toBe(202); expect(retry.json().result).toMatchObject({ @@ -230,7 +249,7 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_tombstoneCapacity0", "opk1_tombstoneCapacity0"), + payload: legacyRequest(operation, "ops1_tombstoneCapacity0", key("opk1_tombstoneCapacity0")), }); expect(capacity.statusCode).toBe(429); expect(dispatch).not.toHaveBeenCalled(); @@ -257,7 +276,7 @@ describe("reviewed durable admission boundary", () => { }, }; const dispatched: string[][] = []; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: { claim: async (operation, fingerprint) => { @@ -292,7 +311,7 @@ describe("reviewed durable admission boundary", () => { stopRunning: true, }, "ops1_ownedSnapshot000", - "opk1_ownedSnapshot000", + key("opk1_ownedSnapshot000"), ), }); expect(admitted.statusCode).toBe(202); @@ -313,8 +332,8 @@ describe("reviewed durable admission boundary", () => { if (writes === 2) throw new Error("disk unavailable"); }, }; - const dispatch = vi.fn(async () => ({ changed: true })); - const app = buildOpsAgentServer({ + const dispatch = vi.fn(async () => pullResult); + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, @@ -330,14 +349,14 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_startPersistFail00", "opk1_startPersistFail00"), + payload: request(operation, "ops1_startPersistFail00", key("opk1_startPersistFail00")), }); expect(first.statusCode).toBe(500); const retry = await app.inject({ method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_startPersistRetry0", "opk1_startPersistFail00"), + payload: request(operation, "ops1_startPersistRetry0", key("opk1_startPersistFail00")), }); expect(retry.statusCode).toBe(202); expect(retry.json().result.state).toBe("failed"); @@ -354,12 +373,12 @@ describe("reviewed durable admission boundary", () => { if (writes >= 3) throw new Error("disk unavailable"); }, }; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, now: () => new Date(baseTime), - dispatch: async () => ({ changed: true }), + dispatch: async () => pullResult, }); const operation = { kind: "service.pull", @@ -370,7 +389,7 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_terminalFail0000", "opk1_terminalFail0000"), + payload: request(operation, "ops1_terminalFail0000", key("opk1_terminalFail0000")), }); const operationId = admitted.json().result.operationId as string; let status = await app.inject({ @@ -378,7 +397,11 @@ describe("reviewed durable admission boundary", () => { url: `/v1/operations/${operationId}`, headers: auth(), }); - for (let attempt = 0; attempt < 100 && status.json().result.state !== "failed"; attempt += 1) { + for ( + let attempt = 0; + attempt < 100 && status.json().result.state !== "termination_pending"; + attempt += 1 + ) { await new Promise((resolve) => setTimeout(resolve, 1)); status = await app.inject({ method: "GET", @@ -387,17 +410,16 @@ describe("reviewed durable admission boundary", () => { }); } expect(status.json().result).toMatchObject({ - state: "failed", - errorClass: "recovery_required", + state: "termination_pending", }); expect(status.json().result).not.toHaveProperty("result"); const retry = await app.inject({ method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_terminalRetry000", "opk1_terminalFail0000"), + payload: request(operation, "ops1_terminalRetry000", key("opk1_terminalFail0000")), }); - expect(retry.json().result.state).toBe("failed"); + expect(retry.json().result.state).toBe("termination_pending"); await app.close(); }); @@ -406,8 +428,6 @@ describe("reviewed durable admission boundary", () => { roots.push(root); const path = join(root, "journal", "jobs-v1.json"); const durable = await openOpsJobJournal(path); - let resolveTerminalSnapshot!: () => void; - const terminalSnapshot = new Promise((resolve) => (resolveTerminalSnapshot = resolve)); let blockAdmission = false; let entered!: () => void; let unblock!: () => void; @@ -421,21 +441,23 @@ describe("reviewed durable admission boundary", () => { entered(); await admissionBlocked; } + console.error( + "INTERLEAVE write", + records.map((r) => [r.operation.kind, r.state]), + ); await durable.replace(records); - if (records.length === 2 && records.every((record) => record.state === "succeeded")) - resolveTerminalSnapshot(); }, }; let releaseFirst!: () => void; const firstBlocked = new Promise((resolve) => (releaseFirst = resolve)); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, now: () => new Date(baseTime), dispatch: async (operation) => { if (operation.kind === "service.pull") await firstBlocked; - return { changed: true }; + return operation.kind === "service.pull" ? pullResult : { changed: true }; }, }); const first = await app.inject({ @@ -449,31 +471,51 @@ describe("reviewed durable admission boundary", () => { expectedConfigurationRevisionId: `cfg1_${"a".repeat(43)}`, }, "ops1_interleaveFirst00", - "opk1_interleaveFirst00", + key("opk1_interleaveFirst00"), ), }); expect(first.statusCode).toBe(202); blockAdmission = true; - const secondPromise = app.inject({ - method: "POST", - url: "/v1/operations", - headers: auth(), - payload: request( - { kind: "service.stop", serviceId: "motis" }, - "ops1_interleaveSecond0", - "opk1_interleaveSecond0", - ), - }); - await admissionEntered; + const secondPromise = app + .inject({ + method: "POST", + url: "/v1/operations", + headers: auth(), + payload: request( + { kind: "service.stop", serviceId: "motis" }, + "ops1_interleaveSecond0", + key("opk1_interleaveSecond0"), + ), + }) + .then((response) => response); + await Promise.race([ + admissionEntered, + secondPromise.then((response) => { + throw Error(`Second admission completed before journal hold: ${response.body}`); + }), + ]); releaseFirst(); unblock(); expect((await secondPromise).statusCode).toBe(202); - await terminalSnapshot; + await expect + .poll( + () => + durable.records().map(({ operation, state, errorClass }) => ({ + kind: operation.kind, + state, + errorClass, + })), + { timeout: 3000 }, + ) + .toEqual([ + { kind: "service.pull", state: "succeeded", errorClass: undefined }, + { kind: "service.stop", state: "succeeded", errorClass: undefined }, + ]); await app.close(); const reopened = await openOpsJobJournal(path); expect(reopened.records()).toHaveLength(2); expect(new Set(reopened.records().map((record) => record.operationKey))).toEqual( - new Set(["opk1_interleaveFirst00", "opk1_interleaveSecond0"]), + new Set([key("opk1_interleaveFirst00"), key("opk1_interleaveSecond0")]), ); expect(reopened.records().every((record) => record.state === "succeeded")).toBe(true); }); @@ -485,7 +527,7 @@ describe("reviewed durable admission boundary", () => { await blocked; return { changed: true }; }); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), @@ -501,13 +543,13 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_concurrentFirst00", "opk1_concurrentShared0"), + payload: request(operation, "ops1_concurrentFirst00", key("opk1_concurrentShared0")), }), app.inject({ method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_concurrentSecond0", "opk1_concurrentShared0"), + payload: request(operation, "ops1_concurrentSecond0", key("opk1_concurrentShared0")), }), ]); expect(first.statusCode).toBe(202); @@ -524,7 +566,7 @@ describe("reviewed durable admission boundary", () => { const baseBytes = Buffer.byteLength(JSON.stringify(result), "utf8"); result.lines[255] = "x".repeat(budget - baseBytes); expect(Buffer.byteLength(JSON.stringify(result), "utf8")).toBe(budget); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), @@ -537,7 +579,7 @@ describe("reviewed durable admission boundary", () => { payload: request( { kind: "service.logs", serviceId: "redis", tail: 256 }, "ops1_exactBudget00000", - "opk1_exactBudget00000", + key("opk1_exactBudget00000"), ), }); expect(response.statusCode).toBe(200); @@ -564,7 +606,7 @@ describe("reviewed durable admission boundary", () => { const records = states.map((state, index) => persisted(operation, { operationId: `job1_retainedState${index}0000`, - operationKey: `opk1_retainedState${index}0000`, + operationKey: key(`opk1_retainedState${index}0000`), state, ...(state === "succeeded" ? { result: { changed: true }, terminalAt: new Date(baseTime).toISOString() } @@ -576,7 +618,7 @@ describe("reviewed durable admission boundary", () => { : {}), }), ); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal: { records: () => records, replace: async () => undefined }, @@ -588,10 +630,10 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request( + payload: legacyRequest( operation, `ops1_retainedState${index}0000`, - `opk1_retainedState${index}0000`, + key(`opk1_retainedState${index}0000`), ), }); expect(response.statusCode).toBe(202); @@ -614,8 +656,8 @@ describe("reviewed durable admission boundary", () => { const journal = await openOpsJobJournal(path, { now: () => new Date(baseTime + 1_000), }); - const dispatch = vi.fn(async () => ({ changed: true })); - const app = buildOpsAgentServer({ + const dispatch = vi.fn(async () => pullResult); + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, @@ -626,7 +668,7 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_restartRetry00000", "opk1_restoredOperation0"), + payload: legacyRequest(operation, "ops1_restartRetry00000", key("opk1_restoredOperation0")), }); expect(retry.statusCode).toBe(202); expect(retry.json().result).toMatchObject({ @@ -639,7 +681,7 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_explicitRetry000", "opk1_explicitRetry000"), + payload: request(operation, "ops1_explicitRetry000", key("opk1_explicitRetry000")), }); expect(explicitRetry.statusCode).toBe(202); const explicitOperationId = explicitRetry.json().result.operationId as string; @@ -672,7 +714,7 @@ describe("reviewed durable admission boundary", () => { errorClass: "recovery_required", terminalAt: new Date(baseTime).toISOString(), }); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal: { records: () => [existing], replace: async () => undefined }, @@ -684,7 +726,7 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request(operation, "ops1_collisionRequest0", "opk1_collisionRequest0"), + payload: request(operation, "ops1_collisionRequest0", key("opk1_collisionRequest0")), }); expect(response.statusCode).toBe(429); expect(response.json().error.class).toBe("busy"); @@ -708,7 +750,7 @@ describe("reviewed durable admission boundary", () => { }, }; const seen: string[] = []; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: claimer, claimTimeoutMs: 5, @@ -726,7 +768,7 @@ describe("reviewed durable admission boundary", () => { payload: request( { kind: "docker.status" }, "ops1_claimTimeout00000", - "opk1_claimTimeout00000", + key("opk1_claimTimeout00000"), ), }); expect(first.statusCode).toBe(504); @@ -734,7 +776,11 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request({ kind: "docker.status" }, "ops1_claimBusy0000000", "opk1_claimBusy0000000"), + payload: request( + { kind: "docker.status" }, + "ops1_claimBusy0000000", + key("opk1_claimBusy0000000"), + ), }); expect(busy.statusCode).toBe(429); release(); @@ -743,7 +789,11 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request({ kind: "docker.status" }, "ops1_claimAccepted000", "opk1_claimAccepted000"), + payload: request( + { kind: "docker.status" }, + "ops1_claimAccepted000", + key("opk1_claimAccepted000"), + ), }); mutable.capability.values.target = "mutated"; expect(accepted.statusCode).toBe(200); @@ -773,7 +823,7 @@ describe("reviewed durable admission boundary", () => { }; }, }; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: claimer, maxConcurrency: 1, @@ -787,16 +837,18 @@ describe("reviewed durable admission boundary", () => { headers: auth(), payload: request({ kind: "docker.status" }, requestId, operationKey), }); - const denied = await send("ops1_hostileClaim0000", "opk1_hostileClaim0000"); + const denied = await send("ops1_hostileClaim0000", key("opk1_hostileClaim0000")); expect(denied.statusCode).toBe(403); expect(denied.json().error.class).toBe("authorization"); - expect((await send("ops1_afterHostile0000", "opk1_afterHostile0000")).statusCode).toBe(200); + expect((await send("ops1_afterHostile0000", key("opk1_afterHostile0000"))).statusCode).toBe( + 200, + ); await app.close(); }); it("contains a throwing audit sink on success and failure", async () => { let fail = false; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), @@ -812,7 +864,11 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request({ kind: "docker.status" }, "ops1_auditSuccess0000", "opk1_auditSuccess0000"), + payload: request( + { kind: "docker.status" }, + "ops1_auditSuccess0000", + key("opk1_auditSuccess0000"), + ), }); expect(success.statusCode).toBe(200); fail = true; @@ -820,7 +876,11 @@ describe("reviewed durable admission boundary", () => { method: "POST", url: "/v1/operations", headers: auth(), - payload: request({ kind: "docker.status" }, "ops1_auditFailure0000", "opk1_auditFailure0000"), + payload: request( + { kind: "docker.status" }, + "ops1_auditFailure0000", + key("opk1_auditFailure0000"), + ), }); expect(failure.statusCode).toBe(500); await app.close(); @@ -830,7 +890,7 @@ describe("reviewed durable admission boundary", () => { let release!: () => void; const blocked = new Promise((resolve) => (release = resolve)); let calls = 0; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), @@ -852,11 +912,17 @@ describe("reviewed durable admission boundary", () => { headers: auth(), payload: request({ kind: "docker.status" }, requestId, operationKey), }); - expect((await send("ops1_auditTimeout0000", "opk1_auditTimeout0000")).statusCode).toBe(504); - expect((await send("ops1_auditStillBusy00", "opk1_auditStillBusy00")).statusCode).toBe(429); + expect((await send("ops1_auditTimeout0000", key("opk1_auditTimeout0000"))).statusCode).toBe( + 504, + ); + expect((await send("ops1_auditStillBusy00", key("opk1_auditStillBusy00"))).statusCode).toBe( + 429, + ); release(); await new Promise((resolve) => setTimeout(resolve, 2)); - expect((await send("ops1_auditAfterLate00", "opk1_auditAfterLate00")).statusCode).toBe(200); + expect((await send("ops1_auditAfterLate00", key("opk1_auditAfterLate00"))).statusCode).toBe( + 200, + ); await app.close(); }); }); @@ -869,7 +935,7 @@ describe("operation context for agent replacement", () => { }); const counts: number[] = []; const journal: OpsJobJournal = { records: () => [], replace: async () => {} }; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, @@ -891,7 +957,7 @@ describe("operation context for agent replacement", () => { payload: request( { kind: "service.restart", serviceId: "redis" }, "ops1_activeRestart0000", - "opk1_activeRestart0000", + key("opk1_activeRestart0000"), ), }); expect(restart.statusCode).toBe(202); @@ -907,7 +973,7 @@ describe("operation context for agent replacement", () => { handoffId: "handoff-1", }, "ops1_activeUpdate00000", - "opk1_activeUpdate00000", + key("opk1_activeUpdate00000"), ), }); expect(update.statusCode).toBe(202); @@ -927,7 +993,7 @@ describe("admission during an agent replacement", () => { const dispatch = vi.fn(async () => ({ handoffId: "handoff-1", replaced: true })); const journal: OpsJobJournal = { records: () => [], replace: async () => {} }; let replacing = false; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, @@ -943,7 +1009,7 @@ describe("admission during an agent replacement", () => { handoffId: "handoff-1", }, "ops1_gateUpdate000000", - "opk1_gateUpdate000000", + key("opk1_gateUpdate000000"), ); try { const first = await app.inject({ @@ -961,7 +1027,7 @@ describe("admission during an agent replacement", () => { payload: request( { kind: "service.restart", serviceId: "redis" }, "ops1_gateRestart00000", - "opk1_gateRestart00000", + key("opk1_gateRestart00000"), ), }); expect(restart.statusCode).toBe(409); diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index 5f836967e..01aa4bb5d 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -1,11 +1,36 @@ -import type { OpsOperation, OpsResourcePolicy } from "@openmapx/core/ops"; +import { existsSync, rmSync } from "node:fs"; +import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; +import { + type OpsOperation, + type OpsResourcePolicy, + parseControlRootOperationKey, +} from "@openmapx/core/ops"; import { describe, expect, it, vi } from "vitest"; +import { ownedServiceHost } from "./__fixtures__/owned-service-host"; +import { + buildControlTestServer as buildOpsAgentServer, + controlTestKey, +} from "./__fixtures__/server"; import { createDockerRuntime } from "./docker-runtime"; -import { type BuildOpsAgentServerOptions, buildOpsAgentServer, type OpsAuditEvent } from "./server"; +import { + type BuildOpsAgentServerOptions, + buildOpsAgentServer as buildActualOpsAgentServer, + type OpsAuditEvent, +} from "./server"; const apiToken = Buffer.alloc(32, 1).toString("base64url"); const dataManagerToken = Buffer.alloc(32, 2).toString("base64url"); -const nowIso = "2026-08-23T18:00:00.000Z"; +const baseTime = Date.now(); +const nowIso = new Date(baseTime).toISOString(); +const testDeadlineAt = baseTime + 4 * 3600_000; +const key = (label: string, deadlineAt = testDeadlineAt) => { + try { + parseControlRootOperationKey(label); + return label; + } catch { + return controlTestKey(label, deadlineAt); + } +}; const allowAll: OpsResourcePolicy = { allowGlobal: () => true, allowService: () => true, @@ -22,11 +47,26 @@ const allowAll: OpsResourcePolicy = { function envelope( operation: OpsOperation | Record, requestId = "ops1_0123456789abcdef", - operationKey = "opk1_0123456789abcdef", + operationKey = key("opk1_0123456789abcdef"), issuedAt = nowIso, - expiresAt = "2026-08-23T18:00:20.000Z", + expiresAt = new Date(baseTime + 20_000).toISOString(), ) { - return { version: 2, requestId, operationKey, issuedAt, expiresAt, operation }; + let rootKey: string; + try { + parseControlRootOperationKey(operationKey); + rootKey = operationKey; + } catch { + rootKey = key(operationKey); + } + return { + version: 2, + requestId, + operationKey: rootKey, + issuedAt, + expiresAt, + operation, + execution: { requestedDeadlineAt: testDeadlineAt }, + }; } function auth(token: string) { @@ -36,7 +76,7 @@ function auth(token: string) { describe("ops-agent request boundary", () => { it("publishes exact live privacy backup evidence and fails closed", async () => { const collectorImage = `ghcr.io/openmapx/privacy-backup@sha256:${"a".repeat(64)}`; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, privacyBackupHealth: async () => ({ ready: true, @@ -50,7 +90,7 @@ describe("ops-agent request boundary", () => { }); await app.close(); - const failedApp = buildOpsAgentServer({ + const failedApp = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, privacyBackupHealth: async () => { throw new Error("docker socket detail"); @@ -65,7 +105,7 @@ describe("ops-agent request boundary", () => { it("authenticates and resolves role before operation validation or dispatch", async () => { const dispatch = vi.fn(); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -92,7 +132,7 @@ describe("ops-agent request boundary", () => { }); it("rejects unknown fields, stale/future/duplicate requests, and excessive bodies", async () => { - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -105,9 +145,9 @@ describe("ops-agent request boundary", () => { envelope( { kind: "docker.status" }, "ops1_stale00000000000", - "opk1_stale00000000000", - "2026-08-23T17:59:30.000Z", - "2026-08-23T17:59:59.999Z", + key("opk1_stale00000000000"), + new Date(baseTime - 30_000).toISOString(), + new Date(baseTime - 1).toISOString(), ), "stale", ], @@ -115,8 +155,8 @@ describe("ops-agent request boundary", () => { envelope( { kind: "docker.status" }, "ops1_future0000000000", - "opk1_future0000000000", - "2026-08-23T18:00:06.000Z", + key("opk1_future0000000000"), + new Date(baseTime + 6_000).toISOString(), ), "future", ], @@ -165,7 +205,7 @@ describe("ops-agent request boundary", () => { it("emits only structured audit fields and redacts runtime failures", async () => { const events: OpsAuditEvent[] = []; const secret = `${apiToken} docker argv http://ops-agent:4300`; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -200,11 +240,11 @@ describe("ops-agent request boundary", () => { it("requires distinct role credentials and fails closed on resource authorization", async () => { expect(() => - buildOpsAgentServer({ + buildActualOpsAgentServer({ tokens: { api: apiToken, "data-manager": apiToken }, }), ).toThrow("distinct"); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: {}, now: () => new Date(nowIso), @@ -228,19 +268,26 @@ describe("ops-agent lifecycle deadlines", () => { ["service.stop", [299_999]], ["service.recreate", [240_000, 220_000]], ] as const)("allows bounded shutdown and dependency work for %s", async (kind, durations) => { + const host = await ownedServiceHost(); + const installation = openControlInstallation({ + infraDir: host.infraDir, + role: "ops-agent", + serviceIds: host.loaded.map((s) => s.manifest.id), + }); vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); vi.setSystemTime(new Date(nowIso)); let aborted = false; + let started = false; const completed: number[] = []; const runtime = createDockerRuntime({ - stack: { - infraDir: "/trusted", - composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", - }, - fileExists: () => false, - execFile: (_file, _args, options) => - new Promise((resolve, reject) => { + stack: host.stack, + fileExists: existsSync, + ownedServices: host.owned, + execFile: async (file, args, options) => { + if (args[0] !== "stop" && !args.includes("restart") && !args.includes("up")) + return host.execFile(file, args, options); + started = true; + return new Promise((resolve, reject) => { const timers: ReturnType[] = []; const cleanup = () => { for (const timer of timers) clearTimeout(timer); @@ -266,11 +313,13 @@ describe("ops-agent lifecycle deadlines", () => { }, elapsed), ); } - }), + }); + }, }); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, - resourcePolicy: allowAll, + resourceClaimer: host.claimer, + controlInstallation: installation, now: () => new Date(nowIso), runtime, }); @@ -279,25 +328,36 @@ describe("ops-agent lifecycle deadlines", () => { method: "POST", url: "/v1/operations", headers: auth(apiToken), - payload: envelope({ kind, serviceId: "postgis" }), + payload: envelope({ + kind, + serviceId: "motis", + ...(kind === "service.recreate" + ? { expectedConfigurationRevisionId: host.initial.revisionId } + : {}), + }), }); - expect(response.statusCode).toBe(202); + expect(response.statusCode, response.body).toBe(202); const { operationId } = response.json().result; + await vi.waitFor(() => expect(started).toBe(true)); await vi.advanceTimersByTimeAsync(durations.reduce((sum, duration) => sum + duration, 0)); - const status = ( - await app.inject({ - method: "GET", - url: `/v1/operations/${operationId}`, - headers: auth(apiToken), - }) - ).json().result; - expect(status).toMatchObject({ state: "succeeded", result: { changed: true } }); + await vi.waitFor(async () => { + const terminal = ( + await app.inject({ + method: "GET", + url: `/v1/operations/${operationId}`, + headers: auth(apiToken), + }) + ).json().result; + expect(terminal).toMatchObject({ state: "succeeded", result: { changed: true } }); + }); expect(aborted).toBe(false); expect(completed).toEqual([...durations]); } finally { await app.close(); vi.clearAllTimers(); vi.useRealTimers(); + closeControlInstallation(installation); + rmSync(host.cleanupRoot, { recursive: true, force: true }); } }); @@ -318,7 +378,7 @@ describe("ops-agent lifecycle deadlines", () => { ); }), ); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -331,7 +391,7 @@ describe("ops-agent lifecycle deadlines", () => { headers: auth(apiToken), payload: envelope({ kind: "service.stop", serviceId: "postgis" }), }); - expect(response.statusCode).toBe(202); + expect(response.statusCode, response.body).toBe(202); const { operationId, fingerprint } = response.json().result; await vi.advanceTimersByTimeAsync(mode === "deadline" ? 629_999 : 1_000); expect(signal?.aborted).toBe(false); @@ -382,7 +442,7 @@ describe("retained execution deadlines", () => { let finish!: (x: unknown) => void; let calls = 0; const records: import("./journal").PersistedOpsJob[][] = []; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, journal: { @@ -468,7 +528,7 @@ describe("retained execution deadlines", () => { vi.setSystemTime(new Date(nowIso)); let signal: AbortSignal | undefined; let finish!: (x: unknown) => void; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, dispatch: async (_o, _r, c) => { @@ -485,6 +545,7 @@ describe("retained execution deadlines", () => { headers: auth(apiToken), payload: { ...envelope({ kind: "service.build", serviceId: "valhalla" }), + operationKey: key("expiry", Date.now() + 100), execution: { requestedDeadlineAt: Date.now() + 100 }, }, }); @@ -508,7 +569,7 @@ describe("retained execution deadlines", () => { }); it("rejects unsigned wire parent and delegation headers before effects", async () => { const dispatch = vi.fn(); - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -541,12 +602,12 @@ describe("execution identity at the agent boundary", () => { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); vi.setSystemTime(new Date(nowIso)); const operation = { kind: "service.build" as const, serviceId: "valhalla" }; - const operationKey = "opk1_parentBoundChild"; + const operationKey = controlTestKey("opk1_parentBoundChild", Date.now() + 12 * 3600_000); const fields = { version: 1, apiJobId: "123e4567-e89b-42d3-a456-426614174000", apiOwnerVersion: 1, - parentOperationKey: "opk1_parentabcdefghij", + parentOperationKey: key("opk1_parentabcdefghij"), parentFingerprint: "a".repeat(64), parentDeadlineAt: Date.now() + 1000, childOperationKey: operationKey, @@ -564,7 +625,7 @@ describe("execution identity at the agent boundary", () => { .digest("base64url"); let finish!: (x: unknown) => void; let execution: import("@openmapx/core/ops").ExecutionRef | undefined; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, dispatch: async (_o, _r, c) => { @@ -618,7 +679,7 @@ describe("execution identity at the agent boundary", () => { const { OpsExecutionFailure } = await import("./runtime"); let fail!: (x: unknown) => void; const snapshots: import("./journal").PersistedOpsJob[][] = []; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -677,7 +738,7 @@ it("observer abort leaves the admitted backend running until its real completion const observer = new AbortController(); let backendSignal: AbortSignal | undefined; let finish!: (x: unknown) => void; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -796,7 +857,7 @@ it.each([0, 1])( ], }); const retained: import("./journal").PersistedOpsJob[][] = []; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, runtime, @@ -873,17 +934,22 @@ it.each([0, 1])( ); describe("fix1 baseline protocol compatibility", () => { - const legacyEnvelope = (...args: Parameters) => ({ - ...envelope(...args), - version: 1, - }); + const legacyEnvelope = (...args: Parameters) => { + const { execution: _execution, ...body } = envelope(...args); + return { ...body, version: 1 }; + }; it("projects the exact historical admission/status/lookup/cancel shapes for a v1 caller", async () => { const { opsOperationFingerprint } = await import("@openmapx/core/ops"); let finish!: (x: unknown) => void; let signal: AbortSignal | undefined; let dispatches = 0; - const operation = { kind: "service.build" as const, serviceId: "osrm" }; - const app = buildOpsAgentServer({ + const operation = { + kind: "service.logs.follow" as const, + serviceId: "osrm", + tail: 10, + maxDurationSeconds: 300, + }; + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), @@ -950,7 +1016,7 @@ describe("fix1 baseline protocol compatibility", () => { headers: { authorization: `Bearer ${apiToken}` }, }); await vi.waitFor(() => expect(signal?.aborted).toBe(true)); - finish({ completed: true }); + finish({ lines: 0, truncated: false }); expect((await cancel).statusCode).toBe(200); const modern = ( await app.inject({ @@ -970,7 +1036,7 @@ describe("fix1 baseline protocol compatibility", () => { it("fix1 cannot downgrade modern-origin cancellation to the legacy fence", async () => { let finish!: (x: unknown) => void; - const app = buildOpsAgentServer({ + const app = await buildOpsAgentServer({ tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 3d0d9e4fd..7dbbc7294 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -29,6 +29,7 @@ import { registerControlRetentionJournal, releaseArtifactTargetLease, repairControlJournalTerminal, + replaceControlRegisteredServiceIds, requireControlJournalRetention, reserveFeedProxyValidation, restoreAdmittedControlExecution, @@ -168,7 +169,7 @@ import { registerPrivacyBackupExtractionRoute, } from "./privacy-backup-extraction"; import { safeProcessDiagnostic } from "./process-output"; -import { productionOwnedServices } from "./production-runtime"; +import { productionOwnedServices, productionRegisteredServiceIds } from "./production-runtime"; import { createUnavailableRuntime, dispatchOpsOperation, @@ -2930,6 +2931,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const maintenancePlan = maintenanceAdmission && configurationMaintenanceAdmissionState(maintenanceAdmission).plan; + if (literalProductionRuntime) + replaceControlRegisteredServiceIds( + options.controlInstallation, + await productionRegisteredServiceIds(runtime), + ); const targets = maintenancePlan?.serviceIds ?? (operation.kind === "artifactTransfer.export" diff --git a/apps/ops-agent/src/standalone-integration-runtime.test.ts b/apps/ops-agent/src/standalone-integration-runtime.test.ts index 7bf69712b..4898ac35a 100644 --- a/apps/ops-agent/src/standalone-integration-runtime.test.ts +++ b/apps/ops-agent/src/standalone-integration-runtime.test.ts @@ -18,12 +18,6 @@ import { type OpsOperation, opsOperationFingerprint, } from "@openmapx/core/ops"; -import { - createIntegrationArtifactPreparation, - describePreparedIntegration, - publishPreparedIntegration, - settleIntegrationPublication, -} from "@openmapx/integration-framework/installer"; import { afterEach, expect, it } from "vitest"; import { createDeterministicTarGz } from "../../../packages/integration-framework/src/deterministic-tar"; import { ArtifactTransferStore } from "./artifact-transfer/store"; @@ -40,200 +34,253 @@ afterEach(() => { fn(); }); }); -it.each(["normal", "upload-mutated", "forged-completion"] as const)( - "dispatches uploaded standalone installation and restart settlement under original signed receipts (scenario=%s)", - async (scenario) => { - const root = mkdtempSync(join(tmpdir(), "openmapx-stage-consumer-")); - cleanup.push(() => rmSync(root, { recursive: true, force: true })); - const infra = join(root, "infra"); - mkdirSync(join(infra, "data"), { recursive: true }); - mkdirSync(join(root, "services")); - await control.provisionControlInstallation(infra); - const installation = control.openControlInstallation({ - infraDir: infra, - role: "ops-agent", - serviceIds: ["app-api"], - }); - cleanup.push(() => control.closeControlInstallation(installation)); - let store = new ArtifactTransferStore({ - dataRoot: join(root, "stages"), - privateRoot: join(root, "stage-meta"), - token: "fixture-test-token", - capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), - }); - cleanup.push(() => store.close()); - const bytes = createDeterministicTarGz([ - { - path: "manifest.json", - contents: Buffer.from( - JSON.stringify({ - id: "upload-fixture", - name: "Uploaded", - version: "1.0.0", - platform: "1.0", - quality: "community", - domains: ["map-overlay"], - }), - ), - }, - ]); - const sha256 = createHash("sha256").update(bytes).digest("hex"); - const stage = await store.createStage({ - purpose: "integration-artifact", - direction: "upload", - ownerId: "fixture-admin", - declaredBytes: bytes.length, - expectedSha256: sha256, - operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, - }); - await store.upload({ - stageId: stage.stageId, - ownerId: "fixture-admin", - purpose: stage.purpose, - attemptId: randomUUID(), - deadlineAt: Date.now() + 60000, - body: new ReadableStream({ - start(c) { - c.enqueue(bytes); - c.close(); +for (const scenario of ["normal", "upload-mutated", "forged-completion"] as const) { + it( + `dispatches uploaded standalone installation and restart settlement under original signed receipts (scenario=${scenario})`, + async () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-stage-consumer-")); + cleanup.push(() => rmSync(root, { recursive: true, force: true })); + const infra = join(root, "infra"); + mkdirSync(join(infra, "data"), { recursive: true }); + mkdirSync(join(root, "services")); + await control.provisionControlInstallation(infra); + const installation = control.openControlInstallation({ + infraDir: infra, + role: "ops-agent", + serviceIds: ["app-api"], + }); + cleanup.push(() => control.closeControlInstallation(installation)); + let store = new ArtifactTransferStore({ + dataRoot: join(root, "stages"), + privateRoot: join(root, "stage-meta"), + token: "fixture-test-token", + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), + }); + cleanup.push(() => store.close()); + const bytes = createDeterministicTarGz([ + { + path: "manifest.json", + contents: Buffer.from( + JSON.stringify({ + id: "upload-fixture", + name: "Uploaded", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ), }, - }), - }); - const operation: OpsOperation = { - kind: "integration.inspectInstall", - ownerId: "fixture-admin", - source: { kind: "uploaded-artifact", stageId: stage.stageId, expectedSha256: sha256 }, - }; - const makeContext = (operation: OpsOperation): OpsExecutionContext => { - const admittedAt = Date.now(), - operationFingerprint = opsOperationFingerprint(operation), - operationKey = createControlRootOperationKey({ - notAfter: admittedAt + 600000, - nonce: randomBytes(16), + ]); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const stage = await store.createStage({ + purpose: "integration-artifact", + direction: "upload", + ownerId: "fixture-admin", + declaredBytes: bytes.length, + expectedSha256: sha256, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + }); + await store.upload({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + attemptId: randomUUID(), + deadlineAt: Date.now() + 60000, + body: new ReadableStream({ + start(c) { + c.enqueue(bytes); + c.close(); + }, }), - admission = normalizeExecutionAdmission({ - operationFingerprint, + }); + const operation: OpsOperation = { + kind: "integration.inspectInstall", + ownerId: "fixture-admin", + source: { kind: "uploaded-artifact", stageId: stage.stageId, expectedSha256: sha256 }, + }; + const retainedContexts = new Map(); + const makeContext = (operation: OpsOperation): OpsExecutionContext => { + const operationFingerprint = opsOperationFingerprint(operation); + const retained = retainedContexts.get(operationFingerprint); + if (retained) return retained; + const admittedAt = Date.now(), + operationKey = createControlRootOperationKey({ + notAfter: admittedAt + 600000, + nonce: randomBytes(16), + }), + admission = normalizeExecutionAdmission({ + operationFingerprint, + operationKey, + kind: operation.kind, + admittedAt, + requestedDeadlineAt: admittedAt + 600000, + }); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, operationKey, + operationFingerprint, + deadlineAt: admission.deadlineAt, + fingerprint: admission.fingerprint, + }; + const authority = control.retainAdmittedControlExecution(installation, { + execution, kind: operation.kind, + operation, admittedAt, requestedDeadlineAt: admittedAt + 600000, + serviceIds: ["app-api"], }); - const execution = { - executor: "ops-agent" as const, - principal: "api" as const, - executionId: `job1_${randomUUID().replaceAll("-", "")}`, - operationKey, - operationFingerprint, - deadlineAt: admission.deadlineAt, - fingerprint: admission.fingerprint, + const context = { + execution, + controlAuthority: authority, + signal: new AbortController().signal, + emitLog: () => {}, + claim: { + fingerprint: operationFingerprint, + operation, + source: "registry", + capability: { revisionId: "fixture", values: {} }, + }, + } satisfies OpsExecutionContext; + retainedContexts.set(operationFingerprint, context); + return context; }; - const authority = control.retainAdmittedControlExecution(installation, { - execution, - kind: operation.kind, - operation, - admittedAt, - requestedDeadlineAt: admittedAt + 600000, - serviceIds: ["app-api"], + const context = makeContext(operation); + const runtime = createUnavailableRuntime(); + installStandaloneIntegrationRuntime(runtime, { rootDir: root, store }); + const inspection = await dispatchOpsOperation(runtime, operation, context); + expect(inspection).toMatchObject({ + sourceKind: "uploaded-artifact", + integrationId: "upload-fixture", + archiveSha256: sha256, + denials: [], }); - return { - execution, - controlAuthority: authority, - signal: new AbortController().signal, - emitLog: () => {}, - claim: { - fingerprint: operationFingerprint, - operation, - source: "registry", - capability: { revisionId: "fixture", values: {} }, - }, + const transactionId = randomUUID(); + const installing: OpsOperation = { + kind: "integration.install", + inspectionId: inspection.inspectionId, + ownerId: "fixture-admin", + expectedContentDigest: inspection.contentDigest, + operationId: transactionId, }; - }; - const context = makeContext(operation); - const runtime = createUnavailableRuntime(); - installStandaloneIntegrationRuntime(runtime, { rootDir: root, store }); - const inspection = await dispatchOpsOperation(runtime, operation, context); - expect(inspection).toMatchObject({ - sourceKind: "uploaded-artifact", - integrationId: "upload-fixture", - archiveSha256: sha256, - denials: [], - }); - const transactionId = randomUUID(); - const installing: OpsOperation = { - kind: "integration.install", - inspectionId: inspection.inspectionId, - ownerId: "fixture-admin", - expectedContentDigest: inspection.contentDigest, - operationId: transactionId, - }; - await expect( - dispatchOpsOperation( - runtime, - { ...installing, ownerId: "foreign" }, - makeContext({ ...installing, ownerId: "foreign" }), - ), - ).rejects.toThrow(/owner/); - const inspectionPath = join( - root, - "stage-meta/integrations", - `inspection-${inspection.inspectionId}.json`, - ); - const originalBytes = readFileSync(inspectionPath), - envelope = JSON.parse(originalBytes.toString("utf8")), - forgedRecord = JSON.parse(envelope.body); - forgedRecord.ownerId = "foreign"; - writeFileSync( - inspectionPath, - JSON.stringify({ ...envelope, body: JSON.stringify(forgedRecord) }), - ); - try { await expect( - dispatchOpsOperation(runtime, installing, makeContext(installing)), - ).rejects.toThrow(/authentication/); - } finally { - writeFileSync(inspectionPath, originalBytes); - } - if (scenario === "upload-mutated") { - writeFileSync(join(root, "stages", stage.stageId, "content"), Buffer.from("changed")); - await expect( - dispatchOpsOperation(runtime, installing, makeContext(installing)), - ).rejects.toThrow(/content changed/); - expect(existsSync(join(root, "custom_integrations", "upload-fixture"))).toBe(false); - return; - } - const installContext = makeContext(installing); - const result = await dispatchOpsOperation(runtime, installing, installContext); - expect(result).toMatchObject({ - outcome: "published", - integrationId: "upload-fixture", - contentDigest: inspection.contentDigest, - }); - await expect( - store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), - ).rejects.toMatchObject({ statusCode: 409 }); - if (scenario === "forged-completion") { - const staging = join(root, "custom_integrations/.staging"); - const workspace = readdirSync(staging).find( - (name) => - name.startsWith("preparation-") && existsSync(join(staging, name, "publication.json")), + dispatchOpsOperation( + runtime, + { ...installing, ownerId: "foreign" }, + makeContext({ ...installing, ownerId: "foreign" }), + ), + ).rejects.toThrow(/owner/); + const inspectionPath = join( + root, + "stage-meta/integrations", + `inspection-${inspection.inspectionId}.json`, ); - if (!workspace) throw Error("Missing actual shared publication journal"); - const path = join(staging, workspace, "publication.json"), - record = JSON.parse(readFileSync(path, "utf8")); - writeFileSync(path, JSON.stringify({ ...record, outcome: "committed" })); - rmSync(join(root, "custom_integrations", "upload-fixture"), { recursive: true }); - await expect(dispatchOpsOperation(runtime, installing, installContext)).rejects.toThrow( - /ENOENT|publication|changed|Published/i, + const originalBytes = readFileSync(inspectionPath), + envelope = JSON.parse(originalBytes.toString("utf8")), + forgedRecord = JSON.parse(envelope.body); + forgedRecord.ownerId = "foreign"; + writeFileSync( + inspectionPath, + JSON.stringify({ ...envelope, body: JSON.stringify(forgedRecord) }), ); + try { + await expect( + dispatchOpsOperation(runtime, installing, makeContext(installing)), + ).rejects.toThrow(/authentication/); + } finally { + writeFileSync(inspectionPath, originalBytes); + } + if (scenario === "upload-mutated") { + writeFileSync(join(root, "stages", stage.stageId, "content"), Buffer.from("changed")); + await expect( + dispatchOpsOperation(runtime, installing, makeContext(installing)), + ).rejects.toThrow(/content changed/); + expect(existsSync(join(root, "custom_integrations", "upload-fixture"))).toBe(false); + return; + } + const installContext = makeContext(installing); + const result = await dispatchOpsOperation(runtime, installing, installContext); + expect(result).toMatchObject({ + outcome: "published", + integrationId: "upload-fixture", + contentDigest: inspection.contentDigest, + }); + await expect( + store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), + ).rejects.toMatchObject({ statusCode: 409 }); + if (scenario === "forged-completion") { + const staging = join(root, "custom_integrations/.staging"); + const workspace = readdirSync(staging).find( + (name) => + name.startsWith("preparation-") && existsSync(join(staging, name, "publication.json")), + ); + if (!workspace) throw Error("Missing actual shared publication journal"); + const path = join(staging, workspace, "publication.json"), + record = JSON.parse(readFileSync(path, "utf8")); + writeFileSync(path, JSON.stringify({ ...record, outcome: "committed" })); + rmSync(join(root, "custom_integrations", "upload-fixture"), { recursive: true }); + await expect(dispatchOpsOperation(runtime, installing, installContext)).rejects.toThrow( + /ENOENT|publication|changed|Published/i, + ); + const settle: OpsOperation = { + kind: "integration.settle", + transactionId, + decision: "commit", + expectedContentDigest: inspection.contentDigest, + }; + await expect(dispatchOpsOperation(runtime, settle, makeContext(settle))).rejects.toThrow( + /journal|publication|changed|auth/i, + ); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + }) + ).state, + ).toBe("claimed"); + return; + } + store.close(); + store = new ArtifactTransferStore({ + dataRoot: join(root, "stages"), + privateRoot: join(root, "stage-meta"), + token: "fixture-test-token", + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), + }); + expect( + ( + await store.getStage({ + stageId: stage.stageId, + ownerId: "fixture-admin", + purpose: stage.purpose, + }) + ).state, + ).toBe("claimed"); + await expect( + store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), + ).rejects.toMatchObject({ statusCode: 409 }); + let replacement = createUnavailableRuntime(); + installStandaloneIntegrationRuntime(replacement, { rootDir: root, store }); const settle: OpsOperation = { kind: "integration.settle", transactionId, decision: "commit", expectedContentDigest: inspection.contentDigest, }; - await expect(dispatchOpsOperation(runtime, settle, makeContext(settle))).rejects.toThrow( - /journal|publication|changed|auth/i, - ); + const settleContext = makeContext(settle); + expect(await dispatchOpsOperation(replacement, settle, settleContext)).toMatchObject({ + outcome: "committed", + transactionId, + }); + expect(await dispatchOpsOperation(replacement, settle, settleContext)).toMatchObject({ + outcome: "committed", + transactionId, + }); expect( ( await store.getStage({ @@ -242,168 +289,125 @@ it.each(["normal", "upload-mutated", "forged-completion"] as const)( purpose: stage.purpose, }) ).state, - ).toBe("claimed"); - return; - } - store.close(); - store = new ArtifactTransferStore({ - dataRoot: join(root, "stages"), - privateRoot: join(root, "stage-meta"), - token: "fixture-test-token", - capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), - }); - expect( - ( - await store.getStage({ - stageId: stage.stageId, - ownerId: "fixture-admin", - purpose: stage.purpose, - }) - ).state, - ).toBe("claimed"); - await expect( - store.deleteStage({ stageId: stage.stageId, ownerId: "fixture-admin" }), - ).rejects.toMatchObject({ statusCode: 409 }); - let replacement = createUnavailableRuntime(); - installStandaloneIntegrationRuntime(replacement, { rootDir: root, store }); - const settle: OpsOperation = { - kind: "integration.settle", - transactionId, - decision: "commit", - expectedContentDigest: inspection.contentDigest, - }; - const settleContext = makeContext(settle); - expect(await dispatchOpsOperation(replacement, settle, settleContext)).toMatchObject({ - outcome: "committed", - transactionId, - }); - expect(await dispatchOpsOperation(replacement, settle, settleContext)).toMatchObject({ - outcome: "committed", - transactionId, - }); - expect( - ( - await store.getStage({ - stageId: stage.stageId, - ownerId: "fixture-admin", - purpose: stage.purpose, - }) - ).state, - ).toBe("consumed"); - await expect( - dispatchOpsOperation( + ).toBe("consumed"); + await expect( + dispatchOpsOperation( + replacement, + { ...settle, decision: "restore" }, + makeContext({ ...settle, decision: "restore" }), + ), + ).rejects.toThrow(/different decision/); + const inventoryOperation: OpsOperation = { kind: "integration.inventory" }; + const inventory = await dispatchOpsOperation( replacement, - { ...settle, decision: "restore" }, - makeContext({ ...settle, decision: "restore" }), - ), - ).rejects.toThrow(/different decision/); - const inventoryOperation: OpsOperation = { kind: "integration.inventory" }; - const inventory = await dispatchOpsOperation( - replacement, - inventoryOperation, - makeContext(inventoryOperation), - ); - expect(inventory.entries).toContainEqual( - expect.objectContaining({ - integrationId: "upload-fixture", - valid: true, - contentDigest: inspection.contentDigest, - }), - ); - for (const decision of ["restore", "commit"] as const) { - const transactionId = randomUUID(), - own: OpsOperation = { - kind: "integration.inspectOwnership", - ownerId: "fixture-admin", + inventoryOperation, + makeContext(inventoryOperation), + ); + expect(inventory.entries).toContainEqual( + expect.objectContaining({ integrationId: "upload-fixture", - operationId: transactionId, + valid: true, + contentDigest: inspection.contentDigest, + }), + ); + for (const decision of ["restore", "commit"] as const) { + const transactionId = randomUUID(), + own: OpsOperation = { + kind: "integration.inspectOwnership", + ownerId: "fixture-admin", + integrationId: "upload-fixture", + operationId: transactionId, + expectedContentDigest: inspection.contentDigest, + }; + const checked = await dispatchOpsOperation(replacement, own, makeContext(own)); + expect(checked).toMatchObject({ allowed: true, transactionId, version: "1.0.0" }); + const remove: OpsOperation = { + kind: "integration.remove", + transactionId, expectedContentDigest: inspection.contentDigest, }; - const checked = await dispatchOpsOperation(replacement, own, makeContext(own)); - expect(checked).toMatchObject({ allowed: true, transactionId, version: "1.0.0" }); - const remove: OpsOperation = { - kind: "integration.remove", - transactionId, - expectedContentDigest: inspection.contentDigest, - }; - expect(await dispatchOpsOperation(replacement, remove, makeContext(remove))).toMatchObject({ - outcome: "removed", - }); - expect(existsSync(join(root, "custom_integrations/upload-fixture"))).toBe(false); - store.close(); - store = new ArtifactTransferStore({ - dataRoot: join(root, "stages"), - privateRoot: join(root, "stage-meta"), - token: "fixture-test-token", - capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), - }); - replacement = createUnavailableRuntime(); - installStandaloneIntegrationRuntime(replacement, { rootDir: root, store }); + expect(await dispatchOpsOperation(replacement, remove, makeContext(remove))).toMatchObject({ + outcome: "removed", + }); + expect(existsSync(join(root, "custom_integrations/upload-fixture"))).toBe(false); + store.close(); + store = new ArtifactTransferStore({ + dataRoot: join(root, "stages"), + privateRoot: join(root, "stage-meta"), + token: "fixture-test-token", + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 100000 }), + }); + replacement = createUnavailableRuntime(); + installStandaloneIntegrationRuntime(replacement, { rootDir: root, store }); - const settling: OpsOperation = { - kind: "integration.settle", - transactionId, - decision, - expectedContentDigest: inspection.contentDigest, - }, - ctx = makeContext(settling); - expect(await dispatchOpsOperation(replacement, settling, ctx)).toMatchObject({ - outcome: decision === "commit" ? "committed" : "restored", - }); - expect(await dispatchOpsOperation(replacement, settling, ctx)).toMatchObject({ - outcome: decision === "commit" ? "committed" : "restored", - }); - expect(existsSync(join(root, "custom_integrations/upload-fixture"))).toBe( - decision === "restore", - ); - } + const settling: OpsOperation = { + kind: "integration.settle", + transactionId, + decision, + expectedContentDigest: inspection.contentDigest, + }, + ctx = makeContext(settling); + expect(await dispatchOpsOperation(replacement, settling, ctx)).toMatchObject({ + outcome: decision === "commit" ? "committed" : "restored", + }); + expect(await dispatchOpsOperation(replacement, settling, ctx)).toMatchObject({ + outcome: decision === "commit" ? "committed" : "restored", + }); + expect(existsSync(join(root, "custom_integrations/upload-fixture"))).toBe( + decision === "restore", + ); + } - { - const directory = join(root, "custom_integrations/upload-fixture"); - mkdirSync(directory); - writeFileSync( - join(directory, "manifest.json"), - JSON.stringify({ - id: "upload-fixture", - name: "Uploaded", - version: "1.0.0", - platform: "1.0", - quality: "community", - domains: ["map-overlay"], - }), - ); - const checkedInventory = await dispatchOpsOperation( - replacement, - inventoryOperation, - makeContext(inventoryOperation), - ), - entry = checkedInventory.entries.find((x) => x.integrationId === "upload-fixture"); - if (!entry?.contentDigest) throw Error("Missing real inventory digest"); - const transactionId = randomUUID(), - own: OpsOperation = { - kind: "integration.inspectOwnership", - integrationId: "upload-fixture", - operationId: transactionId, - ownerId: "fixture-admin", + { + const directory = join(root, "custom_integrations/upload-fixture"); + mkdirSync(directory); + writeFileSync( + join(directory, "manifest.json"), + JSON.stringify({ + id: "upload-fixture", + name: "Uploaded", + version: "1.0.0", + platform: "1.0", + quality: "community", + domains: ["map-overlay"], + }), + ); + const checkedInventory = await dispatchOpsOperation( + replacement, + inventoryOperation, + makeContext(inventoryOperation), + ), + entry = checkedInventory.entries.find((x) => x.integrationId === "upload-fixture"); + if (!entry?.contentDigest) throw Error("Missing real inventory digest"); + const transactionId = randomUUID(), + own: OpsOperation = { + kind: "integration.inspectOwnership", + integrationId: "upload-fixture", + operationId: transactionId, + ownerId: "fixture-admin", + expectedContentDigest: entry.contentDigest, + }; + expect(await dispatchOpsOperation(replacement, own, makeContext(own))).toMatchObject({ + allowed: true, + }); + renameSync(directory, join(root, "original-directory")); + mkdirSync(directory); + writeFileSync(join(directory, "foreign.txt"), "preserve this foreign replacement"); + const remove: OpsOperation = { + kind: "integration.remove", + transactionId, expectedContentDigest: entry.contentDigest, }; - expect(await dispatchOpsOperation(replacement, own, makeContext(own))).toMatchObject({ - allowed: true, - }); - renameSync(directory, join(root, "original-directory")); - mkdirSync(directory); - writeFileSync(join(directory, "foreign.txt"), "preserve this foreign replacement"); - const remove: OpsOperation = { - kind: "integration.remove", - transactionId, - expectedContentDigest: entry.contentDigest, - }; - await expect(dispatchOpsOperation(replacement, remove, makeContext(remove))).rejects.toThrow( - /manifest|changed/, - ); - expect(readFileSync(join(directory, "foreign.txt"), "utf8")).toBe( - "preserve this foreign replacement", - ); - } - }, -); + await expect( + dispatchOpsOperation(replacement, remove, makeContext(remove)), + ).rejects.toThrow(/manifest|changed/); + expect(readFileSync(join(directory, "foreign.txt"), "utf8")).toBe( + "preserve this foreign replacement", + ); + } + // Normal covers install, two ownership/removal settlement lifecycles and a + // foreign-directory replacement with actual signed, fsynced host receipts. + }, + scenario === "normal" ? 60_000 : 15_000, + ); +} diff --git a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts index 3315f3e13..25ea6b4eb 100644 --- a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts +++ b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts @@ -7,6 +7,7 @@ import { openControlInstallation, restoreAdmittedControlExecution, } from "@openmapx/core/build-artifacts"; +import { services as coreServices } from "@openmapx/core/server"; import { createDataTransitTrustedOpsDataSource } from "../data-transit-resource-source"; import { createDockerRuntime, runContainedProcess } from "../docker-runtime"; import { openOpsJobJournal } from "../journal"; @@ -31,7 +32,7 @@ const installation = openControlInstallation({ role: "ops-agent", serviceIds: ["motis", "motis-feed-proxy"], }); -const journal = await openOpsJobJournal(join(root, "data/ops-agent/jobs-v1.json")); +const journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal); const replace = journal.replace.bind(journal); journal.replace = async (records) => { const terminal = records.some( diff --git a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs index 407eeb5a7..3820e2086 100644 --- a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs +++ b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs @@ -174,7 +174,7 @@ const installation = openControlInstallation({ role: "ops-agent", serviceIds: input.serviceIds, }); -const journal = await openOpsJobJournal(join(input.infraDir, "data/ops-agent/jobs-v1.json")); +const journal = await openOpsJobJournal(services.agentStatePaths(input.infraDir).journal); if (["pointer", "cancel-pointer", "before-pointer"].includes(input.catalogFault)) { const rename = fs.renameSync; let stopped = false; diff --git a/apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs b/apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs index 78bcbec30..82a6dc4f8 100644 --- a/apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs +++ b/apps/ops-agent/src/test-fixtures/staged-coordinator-retention-process.mjs @@ -105,7 +105,7 @@ const installation = core.openControlInstallation({ role: "ops-agent", serviceIds: input.serviceIds, }); -const journal = await openOpsJobJournal(join(input.infraDir, "data/ops-agent/jobs-v1.json")); +const journal = await openOpsJobJournal(services.agentStatePaths(input.infraDir).journal); const app = buildOpsAgentServer({ runtime, journal, diff --git a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts index 6789aef29..700e738ba 100644 --- a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts +++ b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts @@ -18,6 +18,7 @@ import { opsOperationFingerprint, sealTrustedConfigurationSnapshot, } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; import { AnchoredDirectory } from "../../../../packages/core/src/build-artifacts/filesystem"; import { configurationSource } from "../__fixtures__/configuration-source"; import { openOpsJobJournal, type PersistedOpsJob } from "../journal"; @@ -112,7 +113,7 @@ const installation = c.openControlInstallation({ role: "ops-agent", serviceIds: ["motis"], }); -const journal = await openOpsJobJournal(join(input.root, "data/ops-agent/jobs-v1.json")); +const journal = await openOpsJobJournal(coreServices.agentStatePaths(input.root).journal); let terminal: PersistedOpsJob | undefined; function wire(job: PersistedOpsJob): OpsJobStatus { return { diff --git a/apps/ops-agent/src/traffic-target-ownership.test.ts b/apps/ops-agent/src/traffic-target-ownership.test.ts index c804664f1..43ee6400a 100644 --- a/apps/ops-agent/src/traffic-target-ownership.test.ts +++ b/apps/ops-agent/src/traffic-target-ownership.test.ts @@ -8,6 +8,7 @@ import { OpsClient, opsOperationFingerprint, } from "@openmapx/core/ops"; +import { services as coreServices } from "@openmapx/core/server"; import { expect, it } from "vitest"; import { createDockerRuntime } from "./docker-runtime"; import { openOpsJobJournal } from "./journal"; @@ -26,7 +27,7 @@ it.each(["succeeded", "canceled"] as const)( role: "ops-agent", serviceIds: ["valhalla"], }), - journal = await openOpsJobJournal(join(data, "ops-agent/jobs-v1.json")); + journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal); let ready = () => {}, release = () => {}, aborted = () => {}, diff --git a/apps/ops-agent/src/transit-credentials-runtime.test.ts b/apps/ops-agent/src/transit-credentials-runtime.test.ts index c598dcef3..be5fd84e2 100644 --- a/apps/ops-agent/src/transit-credentials-runtime.test.ts +++ b/apps/ops-agent/src/transit-credentials-runtime.test.ts @@ -1,7 +1,20 @@ import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join, relative } from "node:path"; +import { services as coreServices } from "@openmapx/core/server"; + +const { CONFIGURATION_GENERATIONS, CURRENT_CONFIGURATION, GENERATED_SECRETS_DIRNAME } = + coreServices; + import { closeControlInstallation, openControlInstallation, @@ -189,6 +202,25 @@ it("discovers pinned keys and masks host-file values without adopting or modifyi configuredKeys: [], }); }, 60_000); +it("reads and masks managed credentials from the protected current generation", async () => { + const f = await fixture(), + infra = join(f.root, "infra/docker"), + revision = `cfg1_${"a".repeat(43)}`, + generation = join(infra, CONFIGURATION_GENERATIONS, revision), + secretDir = join(generation, GENERATED_SECRETS_DIRNAME, "data-manager"), + current = join(infra, CURRENT_CONFIGURATION); + mkdirSync(secretDir, { recursive: true, mode: 0o700 }); + writeFileSync( + join(secretDir, "TRANSITOUS_API_KEYS"), + JSON.stringify({ "de/A": "synthetic-managed-private" }), + { mode: 0o444 }, + ); + symlinkSync(relative(dirname(current), generation), current); + f.setEnv({ TRANSITOUS_API_KEYS_FILE: "/run/secrets/TRANSITOUS_API_KEYS" }); + const value = await f.inspect(); + expect(value).toMatchObject({ owner: "vault", editable: true, configuredKeys: ["de/A"] }); + expect(JSON.stringify(value)).not.toContain("synthetic-managed-private"); +}, 60000); it("refuses malformed existing files rather than treating the vault as unset", async () => { const f = await fixture(); writeFileSync(join(f.local, "api-keys.json"), '{"key":"first","key":"last"}', { mode: 0o600 }); diff --git a/apps/ops-agent/src/transit-credentials-runtime.ts b/apps/ops-agent/src/transit-credentials-runtime.ts index acfe54b8a..7d2ae7d25 100644 --- a/apps/ops-agent/src/transit-credentials-runtime.ts +++ b/apps/ops-agent/src/transit-credentials-runtime.ts @@ -167,7 +167,7 @@ export function installTransitCredentialsRuntime( const components = [ "infra", "docker", - services.CONFIGURATION_GENERATIONS, + ...services.CONFIGURATION_GENERATIONS.split("/"), revision, services.GENERATED_SECRETS_DIRNAME, "data-manager", diff --git a/apps/ops-agent/src/trusted-config-ownership.test.ts b/apps/ops-agent/src/trusted-config-ownership.test.ts index 0a86314aa..620a4d7e8 100644 --- a/apps/ops-agent/src/trusted-config-ownership.test.ts +++ b/apps/ops-agent/src/trusted-config-ownership.test.ts @@ -22,16 +22,14 @@ import { prepareApiJobRoot, } from "../../api/src/services/admin-job-parent"; import type { JobContext } from "../../api/src/services/job-runner"; +import { installConfigurationRuntimeFixture as installTrustedConfigurationRuntime } from "./__fixtures__/configuration-runtime"; import { configurationSource } from "./__fixtures__/configuration-source"; import { testExecution } from "./__fixtures__/execution"; import type { PersistedOpsJob } from "./journal"; import { createRegistryResourceClaimer } from "./policy"; import { createUnavailableRuntime, dispatchOpsOperation } from "./runtime"; import { buildOpsAgentServer } from "./server"; -import { - installTrustedConfigurationRuntime, - type TrustedConfigurationRuntimeOptions, -} from "./trusted-config-runtime"; +import type { TrustedConfigurationRuntimeOptions } from "./trusted-config-runtime"; import { createFileTrustedOpsDataSource } from "./trusted-config-source"; const roots: string[] = []; diff --git a/apps/ops-agent/src/trusted-config-runtime.test.ts b/apps/ops-agent/src/trusted-config-runtime.test.ts index 586e80b67..890e3f531 100644 --- a/apps/ops-agent/src/trusted-config-runtime.test.ts +++ b/apps/ops-agent/src/trusted-config-runtime.test.ts @@ -25,13 +25,13 @@ import { } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { installConfigurationRuntimeFixture as installTrustedConfigurationRuntime } from "./__fixtures__/configuration-runtime"; import { configurationSource } from "./__fixtures__/configuration-source"; import { testExecution } from "./__fixtures__/execution"; import { configurationPolicyRevision } from "./configured-resource-authority"; import { createUnavailableRuntime, dispatchOpsOperation, type OpsTrustedClaim } from "./runtime"; import { initializeTrustedConfigurationRuntime, - installTrustedConfigurationRuntime, readTrustedEnabledServiceIds, } from "./trusted-config-runtime"; @@ -149,6 +149,7 @@ async function installSentinelGeneration(paths: ReturnType): Pro "docker-compose.generated.hardlinks.json", ]) writeFileSync(join(oldGeneration, name), "sentinel", { mode: 0o600 }); + writeFileSync(join(oldGeneration, "resolved-config.generated.json"), "{}", { mode: 0o600 }); pointCurrentAt(paths, oldRevision); } diff --git a/apps/transitous-runner/Dockerfile b/apps/transitous-runner/Dockerfile index 7f13eba4a..f8e47da10 100644 --- a/apps/transitous-runner/Dockerfile +++ b/apps/transitous-runner/Dockerfile @@ -42,6 +42,7 @@ COPY --parents \ packages/hey-api-client-fetch/package.json \ packages/mobility-core/package.json \ packages/mobility-formats/package.json \ + packages/motis-feed-proxy-config/package.json \ packages/transitous-core/package.json \ ./ COPY apps/transitous-runner/package.json apps/transitous-runner/ @@ -59,6 +60,7 @@ COPY --parents \ packages/hey-api-client-fetch/ \ packages/mobility-core/ \ packages/mobility-formats/ \ + packages/motis-feed-proxy-config/ \ packages/transitous-core/ \ ./ COPY apps/transitous-runner/ apps/transitous-runner/ diff --git a/apps/web/src/app/admin/(full-admin)/transit/__tests__/page.test.tsx b/apps/web/src/app/admin/(full-admin)/transit/__tests__/page.test.tsx index 5d9292426..79a6f61e2 100644 --- a/apps/web/src/app/admin/(full-admin)/transit/__tests__/page.test.tsx +++ b/apps/web/src/app/admin/(full-admin)/transit/__tests__/page.test.tsx @@ -30,6 +30,31 @@ vi.mock("@tanstack/react-query", () => ({ useQuery: () => ({ data: undefined, isLoading: false, isError: false }), })); +vi.mock("@/lib/admin/transitPinHooks", () => ({ + transitPinExecutionPending: () => false, + useTransitPins: () => ({ + data: { + allowedCountries: ["de"], + active: { + transitous: { + ref: "abc123def4567890", + submodules: { "transitland-atlas": "atlas-active" }, + }, + gbfs: { commit: "gbfs-active" }, + }, + page: { proposals: [], total: 0, nextOffset: null }, + }, + isLoading: false, + isError: false, + refetch: () => {}, + }), + useProposeTransitPins: () => ({ isPending: false }), + useValidateTransitPins: () => ({ isPending: false }), + useActivateTransitPins: () => ({ isPending: false }), + useCancelTransitPinExecution: () => ({ isPending: false }), + useTransitPinExecution: () => ({ data: undefined }), +})); + const sampleState: TransitStateSummary = { transitousRef: "abc123def4567890", transitousLockedAt: "2026-05-21T12:00:00.000Z", @@ -139,7 +164,7 @@ describe("TransitPipelinePage", () => { const { TransitPipelinePage } = await import("../TransitPipelinePage"); const markup = renderToStaticMarkup(); expect(markup).toContain("Transit pipeline"); - expect(markup).toContain("Transitous lock"); + expect(markup).toContain("Catalog pins"); expect(markup).toContain("Pipeline idle"); expect(markup).toContain("Feeds breakdown"); expect(markup).toContain("Recent jobs"); @@ -147,11 +172,13 @@ describe("TransitPipelinePage", () => { expect(markup).toContain("Provider health"); }); - it("renders the locked ref + last-sync status from state", async () => { + it("renders the active paired catalogs and last-sync status", async () => { const { TransitPipelinePage } = await import("../TransitPipelinePage"); const markup = renderToStaticMarkup(); expect(markup).toContain("abc123def4567890"); - expect(markup).toContain("admin@example.com"); + expect(markup).toContain("atlas-active"); + expect(markup).toContain("gbfs-active"); + expect(markup).toContain("Active pair"); // The success chip's label is what surfaces "last sync status". expect(markup).toContain("success"); }); diff --git a/apps/web/src/hooks/useServices.test.tsx b/apps/web/src/hooks/useServices.test.tsx index 4a5d0f15e..c7a8a6926 100644 --- a/apps/web/src/hooks/useServices.test.tsx +++ b/apps/web/src/hooks/useServices.test.tsx @@ -48,7 +48,12 @@ afterEach(() => { describe("service feature hooks use the generated SDK", () => { it("acquires list data and surfaces declared errors", async () => { const fetch = vi.fn(() => - Response.json({ services: [], summary: { running: 2, stopped: 1, total: 3 } }), + Response.json({ + acquisition: { state: "available", reason: null }, + services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, + summary: { running: 2, stopped: 1, unknown: 0, total: 3 }, + }), ); vi.stubGlobal("fetch", fetch); const { result } = mount(useServicesList); diff --git a/apps/web/src/lib/admin/serviceAdminJobs.test.tsx b/apps/web/src/lib/admin/serviceAdminJobs.test.tsx index 62e24dff4..2e7437a3b 100644 --- a/apps/web/src/lib/admin/serviceAdminJobs.test.tsx +++ b/apps/web/src/lib/admin/serviceAdminJobs.test.tsx @@ -89,6 +89,7 @@ describe("admin-owned accepted service jobs", () => { const oldResponse = Response.json({ acquisition, services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, summary: { unknown: 0, running: 0, stopped: 2, total: 2 }, }); let completed = false; @@ -105,6 +106,7 @@ describe("admin-owned accepted service jobs", () => { return Response.json({ acquisition, services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, summary: { unknown: 0, running: 2, stopped: 0, total: 2 }, }); }); @@ -161,6 +163,7 @@ describe("admin-owned accepted service jobs", () => { return Response.json({ acquisition, services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, summary: { unknown: 0, running: completed ? 2 : 0, stopped: 0, total: 2 }, }); }); @@ -198,6 +201,7 @@ describe("admin-owned accepted service jobs", () => { return Response.json({ acquisition, services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, summary: { unknown: 0, running: 0, stopped: 0, total: 1 }, }); }); @@ -226,6 +230,7 @@ describe("admin-owned accepted service jobs", () => { return Response.json({ acquisition, services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, summary: { unknown: 0, running: 0, stopped: 0, total: 0 }, }); }); @@ -248,6 +253,7 @@ describe("admin-owned accepted service jobs", () => { return Response.json({ acquisition, services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, summary: { unknown: 0, running: 0, stopped: 0, total: 0 }, }); }); @@ -285,6 +291,7 @@ describe("admin-owned accepted service jobs", () => { return Response.json({ acquisition, services: [], + capabilityInventory: { recognized: [], unrecognized: [] }, summary: { running: 0, stopped: 0, unknown: 0, total: 0 }, }); }); diff --git a/docs/docs/install/getting-started.md b/docs/docs/install/getting-started.md index 6061b47c8..4291165e0 100644 --- a/docs/docs/install/getting-started.md +++ b/docs/docs/install/getting-started.md @@ -19,6 +19,8 @@ can be added after you have signed in and checked the application. ```bash git clone https://github.com/OpenMapX/openmapx.git cd openmapx +# Protect the ancestors of the durable host-control keys and journal. +chmod go-w . infra infra/docker infra/docker/secrets pnpm install --frozen-lockfile ``` diff --git a/docs/docs/install/requirements.md b/docs/docs/install/requirements.md index 23caaa70e..6ba91eeb1 100644 --- a/docs/docs/install/requirements.md +++ b/docs/docs/install/requirements.md @@ -53,6 +53,11 @@ configuration files from it, `services build` uses its build contexts, and user, the Debian and Ubuntu default, meets this even with their umask of `002`. Where the account's group has other members, clone and pull with umask `022`. +The durable host-control keys require non-group-writable authority ancestors, +even when the account has a private group. After cloning, run +`chmod go-w . infra infra/docker infra/docker/secrets` before the first render. +Existing unsafe control directories are refused rather than silently repaired. + ## Hardware Size for the services that run together, including temporary build containers. diff --git a/integrations/overlay-environment/__tests__/map-layer.test.tsx b/integrations/overlay-environment/__tests__/map-layer.test.tsx index 557d7019e..a6a5906e6 100644 --- a/integrations/overlay-environment/__tests__/map-layer.test.tsx +++ b/integrations/overlay-environment/__tests__/map-layer.test.tsx @@ -21,6 +21,8 @@ vi.mock("@/integration-api/map/MapContext", () => ({ vi.mock("next-intl", async () => (await import("@/test/intl")).mockNextIntl()); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useOptionalEnv: () => ({ apiUrl: "https://api.test" }), + useEnv: () => ({ apiUrl: "https://api.test" }), })); diff --git a/integrations/overlay-live-transit/__tests__/map-layer-lifecycle.test.tsx b/integrations/overlay-live-transit/__tests__/map-layer-lifecycle.test.tsx index 792836708..1b3135f89 100644 --- a/integrations/overlay-live-transit/__tests__/map-layer-lifecycle.test.tsx +++ b/integrations/overlay-live-transit/__tests__/map-layer-lifecycle.test.tsx @@ -17,6 +17,8 @@ vi.mock("@/integration-api/map/MapContext", () => ({ useMap: () => ({ mapRef, mapReady: true, styleVersion }), })); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useOptionalEnv: () => ({ apiUrl: "https://api.test" }), + useEnv: () => ({ apiUrl: "https://api.test" }), })); vi.mock("next-intl", async () => (await import("@/test/intl")).mockNextIntl()); diff --git a/integrations/overlay-ourairports/__tests__/map-layer.test.tsx b/integrations/overlay-ourairports/__tests__/map-layer.test.tsx index 5b9543cd4..cb97f38e1 100644 --- a/integrations/overlay-ourairports/__tests__/map-layer.test.tsx +++ b/integrations/overlay-ourairports/__tests__/map-layer.test.tsx @@ -23,6 +23,8 @@ vi.mock("@/integration-api/map/MapContext", () => ({ })); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useOptionalEnv: () => ({ apiUrl: "https://api.test" }), + useEnv: () => ({ apiUrl: "https://api.test" }), })); diff --git a/integrations/overlay-readiness.test.tsx b/integrations/overlay-readiness.test.tsx index 3c3fc6bd0..6f5884b72 100644 --- a/integrations/overlay-readiness.test.tsx +++ b/integrations/overlay-readiness.test.tsx @@ -12,6 +12,8 @@ vi.mock("@/integration-api/map/MapContext", () => ({ useMap: () => ({ mapRef, mapReady: true, styleVersion: 0 }), })); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useOptionalEnv: () => ({ apiUrl: "https://api.test" }), + useEnv: () => ({ apiUrl: "https://api.test" }), })); vi.mock("next-intl", async () => (await import("@/test/intl")).mockNextIntl()); diff --git a/integrations/overlay-weather/supplier-layers.test.tsx b/integrations/overlay-weather/supplier-layers.test.tsx index 2ff7353fa..c5747400e 100644 --- a/integrations/overlay-weather/supplier-layers.test.tsx +++ b/integrations/overlay-weather/supplier-layers.test.tsx @@ -23,6 +23,11 @@ vi.mock("@/integration-api/map/MapContext", () => ({ useMap: () => ({ mapRef: MAP_REF, mapReady: true, styleVersion: 0 }), })); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useOptionalEnv: () => ({ + apiUrl: "https://api.test", + trafficTileUrlTemplate: "https://api.test/traffic/{z}/{x}/{y}.png", + }), + useEnv: () => ({ apiUrl: "https://api.test", trafficTileUrlTemplate: "https://api.test/traffic/{z}/{x}/{y}.png", diff --git a/integrations/road-conditions/__tests__/map-layer.test.tsx b/integrations/road-conditions/__tests__/map-layer.test.tsx index cbbd851a4..9c2e57557 100644 --- a/integrations/road-conditions/__tests__/map-layer.test.tsx +++ b/integrations/road-conditions/__tests__/map-layer.test.tsx @@ -22,6 +22,8 @@ vi.mock("@/integration-api/map/MapContext", () => ({ })); vi.mock("@/integration-api/runtime/EnvProvider", () => ({ + useOptionalEnv: () => ({ apiUrl: "https://api.test" }), + useEnv: () => ({ apiUrl: "https://api.test" }), })); diff --git a/packages/cli/__tests__/cli-build-artifacts.test.ts b/packages/cli/__tests__/cli-build-artifacts.test.ts index 00da93493..664bb37c0 100644 --- a/packages/cli/__tests__/cli-build-artifacts.test.ts +++ b/packages/cli/__tests__/cli-build-artifacts.test.ts @@ -320,7 +320,8 @@ describe("build bundles", () => { expect(existsSync(join(liveGraph, "region.osm.pbf"))).toBe(false); expect(existsSync(join(dataDir(serveHost), ".openmapx-import"))).toBe(false); expect(readBuildRecord("osrm", serveHost)).toMatchObject({ ...osrmRecord(), schemaVersion: 2 }); - }); + // Real private-journal export and import each fsync their retained ownership. + }, 60000); it("refuses a bundle built for a different runtime image", async () => { seedOsrmBuild(buildHost); diff --git a/packages/cli/__tests__/cli-data-manager-workflow.test.ts b/packages/cli/__tests__/cli-data-manager-workflow.test.ts index 8e8581ac2..9068bd5a6 100644 --- a/packages/cli/__tests__/cli-data-manager-workflow.test.ts +++ b/packages/cli/__tests__/cli-data-manager-workflow.test.ts @@ -1,6 +1,11 @@ import { Command } from "commander"; import { afterEach, describe, expect, it, vi } from "vitest"; import { registerDataCommands } from "../src/commands/data"; +import { runCliDataStep } from "../src/lib/data-execution"; + +// Command presentation/result-policy unit boundary. Durable POST/GET identity, +// signed containment and SQL ownership belong to the real execution suites. +vi.mock("../src/lib/data-execution", () => ({ runCliDataStep: vi.fn() })); function program() { const command = new Command(); @@ -8,28 +13,15 @@ function program() { registerDataCommands(command); return command; } -afterEach(() => vi.restoreAllMocks()); +afterEach(() => { + vi.restoreAllMocks(); + vi.mocked(runCliDataStep).mockReset(); +}); describe("data-manager CLI result policy", () => { - it("exits unsuccessfully for a valid unsuccessful conflation completion", async () => { - vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( - new Response( - JSON.stringify({ - event: "done", - ok: false, - status: "waiting_for_osm", - linked: 0, - pbfPath: "/data/osm/region.osm.pbf", - message: "OSM input unavailable", - }), - { - headers: { - "content-type": "application/x-ndjson", - "cache-control": "no-cache", - "x-accel-buffering": "no", - }, - }, - ), + it("exits unsuccessfully for an unsuccessful durable conflation completion", async () => { + vi.mocked(runCliDataStep).mockRejectedValueOnce( + new Error("Data execution original-id finished error"), ); const output = vi.spyOn(console, "log").mockImplementation(() => undefined); const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); @@ -39,11 +31,20 @@ describe("data-manager CLI result policy", () => { await expect( program().parseAsync(["data", "overture-conflate", "europe/germany"], { from: "user" }), ).rejects.toThrow("exit:1"); - expect(errors.mock.calls.flat().join(" ")).toContain("OSM input unavailable"); + expect(vi.mocked(runCliDataStep).mock.calls[0]?.[0]).toEqual({ + kind: "overture-conflate", + region: "europe/germany", + restart: false, + }); + expect(errors.mock.calls.flat().join(" ")).toContain( + "Data execution original-id finished error", + ); expect(output.mock.calls.flat().join(" ")).not.toContain("conflation complete"); }); - it("does not announce downloaded fonts for a malformed successful HTTP response", async () => { - vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(Response.json({})); + it("does not announce downloaded fonts when the durable command rejects a malformed response", async () => { + vi.mocked(runCliDataStep).mockRejectedValueOnce( + new Error("data-manager response contract failed"), + ); const output = vi.spyOn(console, "log").mockImplementation(() => undefined); const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(process, "exit").mockImplementation((code) => { @@ -52,6 +53,7 @@ describe("data-manager CLI result policy", () => { await expect( program().parseAsync(["data", "download", "fonts"], { from: "user" }), ).rejects.toThrow("exit:1"); + expect(vi.mocked(runCliDataStep).mock.calls[0]?.[0]).toEqual({ kind: "download-fonts" }); expect(errors.mock.calls.flat().join(" ")).toContain("response contract failed"); expect(output.mock.calls.flat().join(" ")).not.toContain("Downloaded map glyphs"); }); diff --git a/packages/cli/__tests__/cli-integrations.test.ts b/packages/cli/__tests__/cli-integrations.test.ts index 26c93f88a..4b5453456 100644 --- a/packages/cli/__tests__/cli-integrations.test.ts +++ b/packages/cli/__tests__/cli-integrations.test.ts @@ -1012,13 +1012,13 @@ describe("installIntegration (what the operations agent loads)", () => { { name: "copy.json", data: schemaManifest("linked-to") }, { name: "manifest.json", type: "1", linkName: "copy.json" }, ]); - await expectArtifactRejection(linkedTo, "linked-to", /must not be hard-linked/); + await expectArtifactRejection(linkedTo, "linked-to", /Unsupported artifact entry type '1'/); const linkedFrom = join(tmp, "manifest-linked-from.tar.gz"); writeTarFixture(linkedFrom, [ { name: "manifest.json", data: schemaManifest("linked-from") }, { name: "alias.json", type: "1", linkName: "manifest.json" }, ]); - await expectArtifactRejection(linkedFrom, "linked-from", /must not be hard-linked/); + await expectArtifactRejection(linkedFrom, "linked-from", /Unsupported artifact entry type '1'/); expect(agentLoads()).toEqual(["kept"]); }); @@ -1029,7 +1029,11 @@ describe("installIntegration (what the operations agent loads)", () => { { name: "real.json", data: schemaManifest("manifest-symlink") }, { name: "manifest.json", type: "2", linkName: "real.json" }, ]); - await expectArtifactRejection(artifact, "manifest-symlink", /not a symbolic link/); + await expectArtifactRejection( + artifact, + "manifest-symlink", + /Unsupported artifact entry type '2'/, + ); const source = join(tmp, "local-symlink"); mkdirSync(source); @@ -1095,6 +1099,10 @@ describe("installIntegration (what the operations agent loads)", () => { >; chmodSync(join(backup.backupDirectory, "manifest.json"), 0o666); chmodSync(backup.backupDirectory, 0o777); + // Historical backups do not grant ownership of the current publication. + expect(() => restoreInstalledIntegration(tmp, backup)).toThrow(/ownership receipt/); + expect(agentLoads()).toEqual(["kept"]); + removeIntegration({ rootDir: tmp, id: "kept" }); restoreInstalledIntegration(tmp, backup); expect(mode(target)).toBe(0o755); expect(mode(join(target, "manifest.json"))).toBe(0o644); diff --git a/packages/cli/__tests__/cli-notable-places.test.ts b/packages/cli/__tests__/cli-notable-places.test.ts index 8d901c4d7..4dbfd6242 100644 --- a/packages/cli/__tests__/cli-notable-places.test.ts +++ b/packages/cli/__tests__/cli-notable-places.test.ts @@ -1,6 +1,12 @@ import { Command } from "commander"; import { afterEach, describe, expect, it, vi } from "vitest"; import { registerDataCommands } from "../src/commands/data"; +import { runCliDataStep } from "../src/lib/data-execution"; +import { unitDataCommandResult } from "./helpers/data-command-result"; + +// Command presentation/result-policy unit boundary. Durable POST/GET identity, +// signed containment and SQL ownership belong to the real execution suites. +vi.mock("../src/lib/data-execution", () => ({ runCliDataStep: vi.fn() })); function makeProgram(): Command { const program = new Command(); @@ -8,41 +14,21 @@ function makeProgram(): Command { registerDataCommands(program); return program; } -afterEach(() => vi.restoreAllMocks()); +afterEach(() => { + vi.restoreAllMocks(); + vi.mocked(runCliDataStep).mockReset(); +}); describe("data notable-places CLI", () => { - it("builds the index and forwards validated progress messages", async () => { - vi.spyOn(globalThis, "fetch").mockImplementationOnce(async (url) => { - expect(String(url)).toMatch(/\/notable-places\/build$/); - return new Response( - [ - { event: "progress", stage: "names", message: "Fetching labels in en" }, - { - event: "done", - ok: true, - epoch: "e1", - source: "https://qlever.dev/api/wikidata", - minSitelinks: 8, - placeCount: 150_000, - nameCount: 790_000, - }, - ] - .map((event) => JSON.stringify(event)) - .join("\n"), - { - headers: { - "content-type": "application/x-ndjson", - "cache-control": "no-cache", - "x-accel-buffering": "no", - }, - }, - ); - }); + it("builds the index through the durable command and prints its execution identity", async () => { + vi.mocked(runCliDataStep).mockResolvedValueOnce(unitDataCommandResult("notable-places-build")); const output = vi.spyOn(console, "log").mockImplementation(() => undefined); await makeProgram().parseAsync(["data", "notable-places", "build"], { from: "user" }); const printed = output.mock.calls.flat().join(" "); - expect(printed).toContain("Fetching labels in en"); - expect(printed).toContain("150000 places"); + expect(vi.mocked(runCliDataStep).mock.calls[0]?.[0]).toEqual({ kind: "notable-places-build" }); + expect(printed).toContain( + "Notable places published; execution 00000000-0000-4000-8000-000000000001", + ); }); it("prints the published snapshot", async () => { vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( diff --git a/packages/cli/__tests__/cli-pelias-data.test.ts b/packages/cli/__tests__/cli-pelias-data.test.ts index 602309191..307bde388 100644 --- a/packages/cli/__tests__/cli-pelias-data.test.ts +++ b/packages/cli/__tests__/cli-pelias-data.test.ts @@ -114,7 +114,11 @@ describe("buildPeliasData", () => { expect(composeFile.startsWith(tmp)).toBe(false); expect(existsSync(dirname(composeFile))).toBe(false); // Nothing beside the data and what the generation applied before the build. - expect(readdirSync(join(tmp, "infra", "docker")).sort()).toEqual(["data", "state"]); + expect(readdirSync(join(tmp, "infra", "docker")).sort()).toEqual([ + ".owned-service-generations.json", + "data", + "state", + ]); // The copy the applied generation mounts, not the checkout's. expect(compose.services["pelias-schema"]?.volumes).toEqual([ `${peliasConfig}:/code/pelias.json:ro`, diff --git a/packages/cli/__tests__/cli-search-index.test.ts b/packages/cli/__tests__/cli-search-index.test.ts index 1e3d6d3fc..e9bb42388 100644 --- a/packages/cli/__tests__/cli-search-index.test.ts +++ b/packages/cli/__tests__/cli-search-index.test.ts @@ -1,6 +1,12 @@ import { Command } from "commander"; import { afterEach, describe, expect, it, vi } from "vitest"; import { registerDataCommands } from "../src/commands/data"; +import { runCliDataStep } from "../src/lib/data-execution"; +import { unitDataCommandResult } from "./helpers/data-command-result"; + +// Command presentation/result-policy unit boundary. Durable POST/GET identity, +// signed containment and SQL ownership belong to the real execution suites. +vi.mock("../src/lib/data-execution", () => ({ runCliDataStep: vi.fn() })); function makeProgram(): Command { const program = new Command(); @@ -8,7 +14,10 @@ function makeProgram(): Command { registerDataCommands(program); return program; } -afterEach(() => vi.restoreAllMocks()); +afterEach(() => { + vi.restoreAllMocks(); + vi.mocked(runCliDataStep).mockReset(); +}); describe("data search-index CLI", () => { it("exposes nested build and status commands", () => { @@ -16,40 +25,19 @@ describe("data search-index CLI", () => { const searchIndex = data?.commands.find((command) => command.name() === "search-index"); expect(searchIndex?.commands.map((command) => command.name())).toEqual(["build", "status"]); }); - it("builds the selected region and forwards validated progress messages", async () => { - vi.spyOn(globalThis, "fetch").mockImplementationOnce(async (url, options) => { - expect(String(url)).toMatch(/\/search-index\/build$/); - expect(JSON.parse(options?.body as string)).toEqual({ region: "europe/germany" }); - return new Response( - [ - { event: "progress", stage: "extract", message: "Extracted 42 places" }, - { - event: "done", - ok: true, - region: "europe/germany", - epoch: "epoch-2", - sourceFingerprint: "fingerprint", - placeCount: 42, - termCount: 84, - }, - ] - .map((event) => JSON.stringify(event)) - .join("\n"), - { - headers: { - "content-type": "application/x-ndjson", - "cache-control": "no-cache", - "x-accel-buffering": "no", - }, - }, - ); - }); + it("builds the selected region through the durable command and prints its execution identity", async () => { + vi.mocked(runCliDataStep).mockResolvedValueOnce(unitDataCommandResult("search-index-build")); const output = vi.spyOn(console, "log").mockImplementation(() => undefined); await makeProgram().parseAsync(["data", "search-index", "build", "europe/germany"], { from: "user", }); - expect(output.mock.calls.flat().join(" ")).toContain("Extracted 42 places"); - expect(output.mock.calls.flat().join(" ")).toContain("42 places"); + expect(vi.mocked(runCliDataStep).mock.calls[0]?.[0]).toEqual({ + kind: "search-index-build", + region: "europe/germany", + }); + expect(output.mock.calls.flat().join(" ")).toContain( + "Search index published for europe/germany; execution 00000000-0000-4000-8000-000000000001", + ); }); it("prints the operational status fields", async () => { vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( diff --git a/packages/cli/__tests__/cli-services-update.test.ts b/packages/cli/__tests__/cli-services-update.test.ts index a71b35959..5f1bb99dc 100644 --- a/packages/cli/__tests__/cli-services-update.test.ts +++ b/packages/cli/__tests__/cli-services-update.test.ts @@ -4,7 +4,8 @@ import { join } from "node:path"; import { services } from "@openmapx/core/server"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { renderComposeForRepo } from "../src/commands/compose"; +import { parse as parseYaml } from "yaml"; +import { renderComposeForRepo, renderComposeWithWriter } from "../src/commands/compose"; import { registerServicesCommands } from "../src/commands/services"; import { refuseAgentSwitchOrExit } from "../src/lib/agent-layout"; import { dockerComposeStream } from "../src/lib/docker"; @@ -15,11 +16,30 @@ vi.setConfig({ testTimeout: 60000 }); // Named external Docker/registry edges. Commands, files, renderer, locks and receipts are real. // Whether recreating the agent would switch it to its narrow mounts unready // is agent-layout.test.ts's; here it lets the update go on. -vi.mock("../src/lib/agent-layout", () => ({ refuseAgentSwitchOrExit: vi.fn(async () => {}) })); -vi.mock("../src/lib/docker", async (original) => ({ - ...(await original()), - dockerComposeStream: vi.fn(), +vi.mock("../src/lib/agent-layout", async (original) => ({ + ...(await original()), + refuseAgentSwitchOrExit: vi.fn(async () => {}), })); +vi.mock("../src/commands/compose", async (original) => { + const actual = await original(); + return { ...actual, renderComposeWithWriter: vi.fn(actual.renderComposeWithWriter) }; +}); +vi.mock("../src/lib/docker", async (original) => { + const actual = await original(); + return { + ...actual, + dockerComposeStream: vi.fn(), + // Native Compose's JSON inspection edge, over the actual generated model. + dockerCompose: vi.fn(async (args: string[], rootDir?: string) => { + if (args.join(" ") !== "config --format json") return actual.dockerCompose(args, rootDir); + const yaml = readFileSync( + join(root, "infra/docker", services.CURRENT_CONFIGURATION, "docker-compose.generated.yml"), + "utf8", + ); + return { exitCode: 0, stderr: "", stdout: JSON.stringify(parseYaml(yaml)) }; + }), + }; +}); vi.mock("../src/lib/release", async (importOriginal) => ({ ...(await importOriginal()), @@ -28,6 +48,7 @@ vi.mock("../src/lib/release", async (importOriginal) => ({ let root: string, seed: string; beforeEach(async () => { + vi.mocked(renderComposeWithWriter).mockReset(); root = mkdtempSync(join(tmpdir(), "omx-cli-update-")); writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); mkdirSync(join(root, "infra/docker"), { recursive: true }); @@ -172,14 +193,16 @@ describe("services update pull failures", () => { }); describe("services update and the operations agent's mounts", () => { - it("recreates nothing when render can't take the deployment's user from a root-owned checkout", async () => { - vi.mocked(renderComposeForRepo).mockRejectedValueOnce( + it("does not recreate when rendering after the pull cannot resolve the deployment user", async () => { + vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "present", path: "release.yml" }); + vi.mocked(renderComposeWithWriter).mockRejectedValueOnce( new Error( "The checkout /srv/openmapx is owned by root, so the deployment's user cannot be taken from it: set UID and GID in /srv/openmapx/infra/docker/.env to the account that runs the stack", ), ); await expect(update("ops-agent")).rejects.toThrow("exit:1"); - expect(dockerComposeStream).not.toHaveBeenCalled(); + expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([[["pull", "ops-agent"]]]); + expect(renderComposeWithWriter).toHaveBeenCalled(); expect(refuseAgentSwitchOrExit).not.toHaveBeenCalled(); }); diff --git a/packages/cli/__tests__/cli-transitous-api-keys.test.ts b/packages/cli/__tests__/cli-transitous-api-keys.test.ts index 9c1d1d1f6..a4f0ce83e 100644 --- a/packages/cli/__tests__/cli-transitous-api-keys.test.ts +++ b/packages/cli/__tests__/cli-transitous-api-keys.test.ts @@ -483,4 +483,5 @@ it("uses actual active pin A while upstream is B and preserves unrelated credent expect(proposedValues["retained/orphan"]).toBe(" exact value\n"); expect(readCatalogPinSet(tmp)?.generationId).toBe(active.generationId); expect(git(cache, ["rev-parse", "HEAD"])).toBe(sha); -}, 45000); + // Two real Git repositories/submodules and paired-pin journals exceed 45s on this filesystem. +}, 90000); diff --git a/packages/cli/__tests__/helpers/data-command-result.ts b/packages/cli/__tests__/helpers/data-command-result.ts new file mode 100644 index 000000000..464b7c53e --- /dev/null +++ b/packages/cli/__tests__/helpers/data-command-result.ts @@ -0,0 +1,21 @@ +import { + type DataExecutionInput, + dataExecutionStatusSchema, +} from "@openmapx/core/data-manager-contracts"; + +/** Presentation collaborator only; this value is never a signed execution proof. */ +export function unitDataCommandResult(kind: DataExecutionInput["kind"]) { + return dataExecutionStatusSchema.parse({ + id: "00000000-0000-4000-8000-000000000001", + operationKey: "opk1_unit_command_policy", + fingerprint: "a".repeat(64), + admittedAt: 1, + deadlineAt: 4102444800000, + kind, + status: "ok", + publication: "committed", + cancellationRequestedAt: null, + progress: 1, + diagnostics: { tail: [], truncated: false }, + }); +} diff --git a/packages/core/src/__tests__/server-import-boundary.test.ts b/packages/core/src/__tests__/server-import-boundary.test.ts index 3b2a1655b..bb2ab6f8a 100644 --- a/packages/core/src/__tests__/server-import-boundary.test.ts +++ b/packages/core/src/__tests__/server-import-boundary.test.ts @@ -1,3 +1,4 @@ +import { isBuiltin } from "node:module"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; import { describe, expect, it } from "vitest"; @@ -24,10 +25,17 @@ describe("server HTTP import boundaries", () => { expect(result.errors).toEqual([]); const inputs = Object.keys(result.metafile.inputs); + // The two pure platform/release value schemas live beside host services. + // Their precise exceptions below must never introduce a Node builtin. + expect( + Object.values(result.metafile.outputs) + .flatMap((output) => output.imports) + .filter((entry) => entry.external && isBuiltin(entry.path)), + ).toEqual([]); expect(inputs.some((path) => path.endsWith("/api/client.ts"))).toBe(true); expect( inputs.filter((path) => - /(?:^|\/)(?:react|react-dom|@tanstack)(?:\/|@)|packages\/core\/src\/(?:index\.ts|hooks\/|stores\/|server(?:\.ts|\/)|services\/(?!data-manager\/contracts\/))|\/admin\/serviceAdmin\.ts/.test( + /(?:^|\/)(?:react|react-dom|@tanstack)(?:\/|@)|packages\/core\/src\/(?:index\.ts|hooks\/|stores\/|server(?:\.ts|\/)|services\/(?!data-manager\/contracts\/|platform-configuration\.ts$|release-selection\.ts$))|\/admin\/serviceAdmin\.ts/.test( path, ), ), diff --git a/packages/core/src/api/admin-sdk-transport.test.ts b/packages/core/src/api/admin-sdk-transport.test.ts index d35521e6b..16ad16a86 100644 --- a/packages/core/src/api/admin-sdk-transport.test.ts +++ b/packages/core/src/api/admin-sdk-transport.test.ts @@ -215,7 +215,7 @@ describe("canonical administration SDK acquisition", () => { }); describe("pending integration family operation binding", () => { - it("validates all22canonical operation closures before final producer generation", async () => { + it("validates all35canonical operation closures before final producer generation", async () => { const { integrationAdminOperations } = await import("./contracts/integration-admin"); let calls = 0; for (const [id, operation] of Object.entries(integrationAdminOperations)) { @@ -233,7 +233,7 @@ describe("pending integration family operation binding", () => { }); expect(result).toMatchObject({ status: 400, data: { error: "Synthetic rejection" } }); } - expect(calls).toBe(22); + expect(calls).toBe(35); }); it("retains domain errors and vault hints and requires Retry-After without exposing malformed bodies", async () => { const cases = [ diff --git a/packages/core/src/api/admin-sdk.test.ts b/packages/core/src/api/admin-sdk.test.ts index f5437354d..946f0b585 100644 --- a/packages/core/src/api/admin-sdk.test.ts +++ b/packages/core/src/api/admin-sdk.test.ts @@ -6,8 +6,8 @@ import { dataManagerAdminOperations } from "./contracts/data-manager-admin"; const client = createApiClient({ baseUrl: "https://app.example", credentials: "include" }); afterEach(() => vi.unstubAllGlobals()); describe("complete generated administration operations", () => { - it("exposes all28canonical operation keys", () => { - expect(Object.keys(dataManagerAdminOperations)).toHaveLength(28); + it("exposes all38canonical operation keys", () => { + expect(Object.keys(dataManagerAdminOperations)).toHaveLength(38); for (const key of Object.keys(dataManagerAdminOperations)) expect(sdk).toHaveProperty(key, expect.any(Function)); }); diff --git a/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts b/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts index cbfc22963..b537cd888 100644 --- a/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts +++ b/packages/core/src/api/contracts/data-manager-admin/contracts.test.ts @@ -9,14 +9,49 @@ import { } from "./index"; describe("application data-manager administration contracts", () => { - it("describes all 28 actual BFF operations with the real session-only distinctions", () => { - expect(Object.keys(dataManagerAdminOperations)).toHaveLength(28); + it("describes all 38 actual BFF operations with the real session-only distinctions", () => { + expect(Object.keys(dataManagerAdminOperations)).toHaveLength(38); const operations = Object.values(dataManagerAdminOperations); - expect(new Set(operations.map(({ method, path }) => `${method} ${path}`)).size).toBe(28); - expect(operations.filter(({ auth }) => auth === "admin")).toHaveLength(11); + expect(new Set(operations.map(({ method, path }) => `${method} ${path}`)).size).toBe(38); + expect(operations.filter(({ auth }) => auth === "admin")).toHaveLength(21); expect(operations.every(({ path }) => path.startsWith("/api/"))).toBe(true); expect(dataManagerAdminOperations.getApiDataManagerTransitState.auth).toBe("service"); expect(dataManagerAdminOperations.getApiDataManagerOvertureStatus.auth).toBe("admin"); + // Approved parity additions: literal identities remain independent of registry order. + const additions = [ + ["getAdminTransitCredentials", "GET", "/api/admin/transit/credentials", "admin"], + ["previewAdminTransitCredentials", "POST", "/api/admin/transit/credentials/preview", "admin"], + ["applyAdminTransitCredentials", "PATCH", "/api/admin/transit/credentials", "admin"], + ["getAdminTransitPins", "GET", "/api/admin/transit/pins", "admin"], + ["proposeAdminTransitPins", "POST", "/api/admin/transit/pins/proposals", "admin"], + [ + "validateAdminTransitPins", + "POST", + "/api/admin/transit/pins/proposals/:proposalId/validate", + "admin", + ], + [ + "activateAdminTransitPins", + "POST", + "/api/admin/transit/pins/proposals/:proposalId/activate", + "admin", + ], + [ + "cancelAdminTransitPinExecution", + "DELETE", + "/api/admin/transit/pins/executions/:id", + "admin", + ], + ["getAdminTransitPinExecution", "GET", "/api/admin/transit/pins/executions/:id", "admin"], + [ + "getApiDataManagerRoadConditionsPolicy", + "GET", + "/api/data-manager/road-conditions/policy", + "service", + ], + ] as const; + for (const [id, method, path, auth] of additions) + expect(dataManagerAdminOperations[id]).toMatchObject({ method, path, auth }); }); it("preserves actual provider state and its not-yet-observed timestamp absences", () => { const provider = { diff --git a/packages/core/src/api/contracts/integration-admin.test.ts b/packages/core/src/api/contracts/integration-admin.test.ts index c87ac6294..ac2b41541 100644 --- a/packages/core/src/api/contracts/integration-admin.test.ts +++ b/packages/core/src/api/contracts/integration-admin.test.ts @@ -11,6 +11,23 @@ beforeEach(() => afterEach(() => vi.unstubAllGlobals()); const at = "2026-10-20T12:00:00.000Z"; const inventory = [ + [ + "getAdminConfigurationMigrationInventory", + "GET", + "/api/admin/configuration-migration/inventory", + ], + ["exportAdminConfigurationMigration", "GET", "/api/admin/configuration-migration/export"], + ["previewAdminConfigurationMigration", "POST", "/api/admin/configuration-migration/preview"], + ["applyAdminConfigurationMigration", "POST", "/api/admin/configuration-migration/apply"], + ["exportAdminCredentialTransfer", "POST", "/api/admin/credential-transfer/export"], + ["previewAdminCredentialTransfer", "POST", "/api/admin/credential-transfer/preview"], + [ + "chooseAdminCredentialTransferFields", + "POST", + "/api/admin/credential-transfer/preview/:previewId/choices", + ], + ["applyAdminCredentialTransfer", "POST", "/api/admin/credential-transfer/apply"], + ["listAdminStandaloneIntegrations", "GET", "/api/admin/standalone-integrations"], ["adoptAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/:id/adopt"], ["removeAdminStandaloneIntegration", "POST", "/api/admin/standalone-integrations/:id/remove"], @@ -193,7 +210,7 @@ function success( return contract.schema.parse(value); } describe("integration administration canonical producer contracts", () => { - it("binds exactly the 27 approved operation IDs, methods, paths and admin auth", () => { + it("binds exactly the 35 approved operation IDs, methods, paths and admin auth", () => { expect(contracts).toHaveProperty("integrationAdminOperations"); expect( Object.entries(contracts.integrationAdminOperations).map(([id, op]) => [ diff --git a/packages/core/src/api/contracts/service-admin.test.ts b/packages/core/src/api/contracts/service-admin.test.ts index 034d75395..c3b09ab49 100644 --- a/packages/core/src/api/contracts/service-admin.test.ts +++ b/packages/core/src/api/contracts/service-admin.test.ts @@ -16,6 +16,20 @@ const actions = Object.fromEntries( ); const at = "2026-10-21T10:00:00.000Z"; const inventory = [ + ["pullAdminServiceImages", "POST", "/services/images/pull"], + ["clearAdminReleaseSelection", "POST", "/system/release/clear"], + ["previewAdminBackupPrune", "POST", "/services/backups/prune/preview"], + ["pruneAdminServiceBackups", "POST", "/services/backups/prune"], + ["listAdminCacheNamespaces", "GET", "/cache"], + ["clearAdminCache", "POST", "/cache/clear"], + ["getAdminArtifactTransferCapabilities", "GET", "/artifact-transfers/capabilities"], + ["exportAdminArtifactTransfer", "POST", "/artifact-transfers/exports"], + ["previewAdminArtifactImport", "POST", "/artifact-transfers/:stageId/preview-import"], + ["importAdminArtifactTransfer", "POST", "/artifact-transfers/:stageId/import"], + ["validateAdminArtifactTransfer", "POST", "/artifact-transfers/:stageId/validate"], + ["listAdminServicePresets", "GET", "/services/presets"], + ["renderAdminComposeConfiguration", "POST", "/compose/render"], + ["listAdminArtifactTransfers", "GET", "/artifact-transfers"], ["createAdminArtifactTransfer", "POST", "/artifact-transfers"], ["getAdminArtifactTransfer", "GET", "/artifact-transfers/:stageId"], @@ -225,13 +239,12 @@ describe("canonical complete service administration contracts", () => { it("binds the complete approved identities, methods, paths and admin auth", () => { expect(contracts).toHaveProperty("serviceAdminOperations"); expect( - Object.entries(contracts.serviceAdminOperations).map(([id, op]) => [ - id, - op.method, - op.path, - op.auth, - ]), - ).toEqual(inventory.map(([id, method, path]) => [id, method, `/api/admin${path}`, "admin"])); + Object.entries(contracts.serviceAdminOperations) + .map(([id, op]) => [id, op.method, op.path, op.auth]) + .sort(), + ).toEqual( + inventory.map(([id, method, path]) => [id, method, `/api/admin${path}`, "admin"]).sort(), + ); }); it("retains full raw manifests including unknown nested JSON, structured capabilities and omitted defaults", () => { expect(serviceManifestSchema.parse(raw).container.gpu).toEqual({ @@ -324,6 +337,10 @@ describe("canonical complete service administration contracts", () => { const list = { acquisition, services: [summary], + capabilityInventory: { + recognized: ["routing-engine"], + unrecognized: [{ capability: "custom/routing", serviceIds: [raw.id] }], + }, summary: { running: 0, stopped: 1, unknown: 0, total: 1 }, }; expect(response("listAdminServices", 200, list)).toEqual(list); @@ -334,6 +351,23 @@ describe("canonical complete service administration contracts", () => { secret: { value: "***", source: "env" }, }, envPrefix: "SERVICE_SAMPLE", + ownership: { + field: { + source: "database", + secret: false, + writable: true, + adoption: "direct", + reason: null, + }, + secret: { + source: "env", + secret: true, + writable: false, + adoption: "host_action_required", + reason: "host_environment", + envVar: "SERVICE_SAMPLE_SECRET", + }, + }, }; expect(response("getAdminServiceConfig", 200, config)).toEqual(config); expect(response("getAdminServiceConfig", 200, { ...config, schema: null })).toEqual({ diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index afd92de9b..efb664519 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -969,6 +969,18 @@ export function assertFreshControlRoot( export function controlInstallationDataRoot(installation: ControlInstallation): string { return installationState(installation).dataRoot; } +/** Fixed agent registry refresh after a repository publication/removal. This + * local admission cache is populated from the literal trusted service loader, + * never an operation's requested IDs. Retained roots keep their signed targets. */ +export function replaceControlRegisteredServiceIds( + installation: ControlInstallation, + serviceIds: readonly string[], +): void { + const state = installationState(installation); + if (installation.role !== "ops-agent" || state.fixedChildId) + throw Error("Agent registry refresh required"); + state.registeredTargets = new Set(targetSchema.parse([...new Set(serviceIds)].sort())); +} export function controlRegisteredServiceIds(installation: ControlInstallation): string[] { return [...installationState(installation).registeredTargets].sort(); } @@ -2094,7 +2106,10 @@ export function retainControlEffectReceipt( (s.kind === "integration.install" && result.outcome === "published") || (s.kind === "integration.remove" && result.outcome === "removed") || (s.kind === "integration.settle" && result.outcome === "restored") || - (s.kind === "catalogIntegration.settle" && ["commit", "rollback"].includes(result.outcome)) + (s.kind === "catalogIntegration.settle" && + ["commit", "rollback"].includes(result.outcome)) || + (s.kind === "serviceRepository.settle" && + ["commit", "rollback", "release"].includes(result.outcome)) ) { outcome = "committed"; publication = "committed"; @@ -5388,10 +5403,12 @@ export function retainControlPreparedTerminal(authority: VerifiedExecutionAuthor !["configurationMaintenance.prepare", "configurationHandoff.prepare"].includes(s.kind) ) throw Error("Unsupported terminal preparation"); - const data = new AnchoredDirectory(s.installation.dataRoot); + const infra = new AnchoredDirectory(dirname(s.installation.dataRoot)); + let state: AnchoredDirectory | undefined; let jobs: AnchoredDirectory | undefined; try { - jobs = data.directory("ops-agent"); + state = infra.directory("state"); + jobs = state.directory("ops-agent"); const journal = z .object({ version: z.literal(2), jobs: z.array(z.unknown()).max(256) }) .parse(JSON.parse(jobs.read("jobs-v1.json", 24 * 1024 * 1024).toString("utf8"))); @@ -5464,7 +5481,8 @@ export function retainControlPreparedTerminal(authority: VerifiedExecutionAuthor } } finally { jobs?.close(); - data.close(); + state?.close(); + infra.close(); } } export function verifyControlPreparedTerminal( diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index dc98f0ef6..f5ddf7125 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -88,6 +88,7 @@ export { reconcileControlRetention, registerControlRetentionJournal, repairControlJournalTerminal, + replaceControlRegisteredServiceIds, requireControlJournalRetention, restoreAdmittedControlExecution, restoreConfigurationPreEffectTerminal, diff --git a/packages/core/src/build-artifacts/motis-destination.ts b/packages/core/src/build-artifacts/motis-destination.ts index 32d823598..3ceb18fa7 100644 --- a/packages/core/src/build-artifacts/motis-destination.ts +++ b/packages/core/src/build-artifacts/motis-destination.ts @@ -88,7 +88,7 @@ async function destination(input: MotisPortableDestinationInput) { let directory = new AnchoredDirectory(realpathSync(input.infraDir)); opened.push(directory); for (const part of [ - CONFIGURATION_GENERATIONS, + ...CONFIGURATION_GENERATIONS.split("/"), input.configurationRevisionId, GENERATED_SECRETS_DIRNAME, "data-manager", diff --git a/packages/core/src/build-artifacts/peer-process.test.ts b/packages/core/src/build-artifacts/peer-process.test.ts index 85a91a49b..c22502c9c 100644 --- a/packages/core/src/build-artifacts/peer-process.test.ts +++ b/packages/core/src/build-artifacts/peer-process.test.ts @@ -523,9 +523,9 @@ it.each([false, true])( plan: [plan], }; const { writeFileSync } = await import("node:fs"); - mkdirSync(join(root, "data/ops-agent"), { mode: 0o700 }); + mkdirSync(join(root, "state/ops-agent"), { recursive: true, mode: 0o700 }); writeFileSync( - join(root, "data/ops-agent/jobs-v1.json"), + join(root, "state/ops-agent/jobs-v1.json"), JSON.stringify({ version: 2, jobs: [ diff --git a/packages/core/src/build-artifacts/publication-native.test.ts b/packages/core/src/build-artifacts/publication-native.test.ts index a631bb9fa..6b82a2f9e 100644 --- a/packages/core/src/build-artifacts/publication-native.test.ts +++ b/packages/core/src/build-artifacts/publication-native.test.ts @@ -60,8 +60,11 @@ it.each(["old-directory", "build-record"])( }, { timeout: 30000, interval: 50 }, ); - const writerPath = join(root, ".trusted-config-writer"), - writerBefore = lstatSync(writerPath); + const writerPaths = [ + join(root, ".trusted-config-writer"), + join(root, "state/ops-agent/configuration/.writer"), + ]; + const writersBefore = writerPaths.map((path) => ({ path, identity: lstatSync(path) })); child.kill("SIGKILL"); const [code, signal] = await exited; expect(code).toBeNull(); @@ -69,10 +72,14 @@ it.each(["old-directory", "build-record"])( // This isolated fixture owns its ONLY writer and has just killed it. Model // the documented explicit host operation; production never steals this lock // using elapsed time, a PID, or a missing process as execution authority. - const writerAfter = lstatSync(writerPath); - expect([writerAfter.dev, writerAfter.ino]).toEqual([writerBefore.dev, writerBefore.ino]); - expect(writerAfter.isDirectory() && !writerAfter.isSymbolicLink()).toBe(true); - rmdirSync(writerPath); // Empty, exact owned lock only; no recursive deletion. + // The current writer owns its state lock and the unretired legacy lock. + // Explicit recovery removes only those two exact empty fixture-owned inodes. + for (const { path, identity: before } of writersBefore.reverse()) { + const after = lstatSync(path); + expect([after.dev, after.ino]).toEqual([before.dev, before.ino]); + expect(after.isDirectory() && !after.isSymbolicLink()).toBe(true); + rmdirSync(path); // Empty, exact owned lock only; no recursive deletion. + } const state = JSON.parse(readFileSync(join(root, "fault.json"), "utf8")) as { execution: unknown; diff --git a/packages/core/src/build-artifacts/repository-settlement.test.ts b/packages/core/src/build-artifacts/repository-settlement.test.ts new file mode 100644 index 000000000..e4df7b2f1 --- /dev/null +++ b/packages/core/src/build-artifacts/repository-settlement.test.ts @@ -0,0 +1,83 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { + createControlRootOperationKey, + executionFingerprint, + opsOperationFingerprint, +} from "../ops"; +import { + beginControlEffect, + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, + retainAdmittedControlExecution, + retainControlEffectReceipt, + verifyControlEffectSettlement, +} from "./control-authority"; + +it.each(["commit", "rollback", "release"] as const)( + "records actual awaited repository %s as a known finite filesystem settlement", + async (outcome) => { + const root = mkdtempSync(join(tmpdir(), "repository-settle-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + const installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["motis"], + }); + try { + const operation = { + kind: "serviceRepository.settle" as const, + repository: "a".repeat(16), + transactionId: `txn1_${"b".repeat(24)}`, + outcome, + }; + const deadlineAt = Date.now() + 60000, + operationFingerprint = opsOperationFingerprint(operation); + const execution = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: createControlRootOperationKey({ + notAfter: deadlineAt, + nonce: randomBytes(16), + }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ + kind: operation.kind, + operationFingerprint, + deadlineAt, + }), + }; + const authority = retainAdmittedControlExecution(installation, { + execution, + operation, + kind: operation.kind, + admittedAt: Date.now(), + requestedDeadlineAt: deadlineAt, + serviceIds: ["motis"], + }); + beginControlEffect(authority); + expect(() => + retainControlEffectReceipt(authority, { + outcome: "committed", + publication: "committed", + receipt: { + kind: "ops-result", + result: { repository: operation.repository, outcome, changed: true }, + }, + }), + ).not.toThrow(); + expect(verifyControlEffectSettlement(installation, execution)).toBeTruthy(); + } finally { + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }, + 60000, +); diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index 8aa0225d1..b97a7d89c 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -536,9 +536,9 @@ it("keeps a bare terminal maintenance summary fenced without the fixed owner sou await expect( a.delegateArtifactTargetLease(lease, { ...plan, childPublicKey: agent.publicKey }), ).rejects.toThrow(); - mkdirSync(join(root, "data/ops-agent"), { mode: 0o700 }); + mkdirSync(join(root, "state/ops-agent"), { recursive: true, mode: 0o700 }); writeFileSync( - join(root, "data/ops-agent/jobs-v1.json"), + join(root, "state/ops-agent/jobs-v1.json"), JSON.stringify({ version: 2, jobs: [ diff --git a/packages/core/src/ops/foundation-review.test.ts b/packages/core/src/ops/foundation-review.test.ts index 63a5df56c..e02ca9138 100644 --- a/packages/core/src/ops/foundation-review.test.ts +++ b/packages/core/src/ops/foundation-review.test.ts @@ -50,9 +50,14 @@ describe("reviewed ops foundation contract", () => { for (const kind of requiredKinds) expect(OPS_OPERATION_KINDS).toContain(kind); expect(Object.keys(OPS_KIND_POLICIES).sort()).toEqual([...OPS_OPERATION_KINDS].sort()); // Exact existing long-running owners, plus the approved ownership-only - // maintenance hour. Every other kind retains the original 30 minute cap. + // maintenance hour and bounded artifact transfer days. Every other kind + // retains the original 30 minute cap. const extendedBudgets: Record = { "configurationMaintenance.prepare": 3600_000, + "configurationHandoff.prepare": 3600_000, + "artifactTransfer.validate": 72 * 3600_000, + "artifactTransfer.import": 72 * 3600_000, + "artifactTransfer.export": 72 * 3600_000, "service.build": 12 * 3600_000, "services.buildAll": 24 * 3600_000, "system.update": 48 * 3600_000, @@ -61,6 +66,9 @@ describe("reviewed ops foundation contract", () => { "data.update": 24 * 3600_000, "data.convertOverpass": 24 * 3600_000, "data.overtureSync": 24 * 3600_000, + "data.overturePull": 24 * 3600_000, + "data.overtureIngest": 24 * 3600_000, + "data.overtureExtract": 24 * 3600_000, "data.overtureConflate": 24 * 3600_000, "data.searchIndexBuild": 24 * 3600_000, "data.notablePlacesBuild": 24 * 3600_000, @@ -71,7 +79,7 @@ describe("reviewed ops foundation contract", () => { expect(policy.role === "api" || policy.role === "data-manager").toBe(true); expect(policy.execution === "sync" || policy.execution === "async").toBe(true); expect(policy.timeoutMs).toBeGreaterThan(0); - expect(policy.timeoutMs).toBeLessThanOrEqual(extendedBudgets[kind] ?? 30 * 60_000); + expect(policy.timeoutMs, kind).toBeLessThanOrEqual(extendedBudgets[kind] ?? 30 * 60_000); expect(policy.maxResultBytes).toBeGreaterThan(0); expect(policy.maxResultBytes).toBeLessThanOrEqual(1024 * 1024); } diff --git a/packages/core/src/services/__fixtures__/compose-path-host.ts b/packages/core/src/services/__fixtures__/compose-path-host.ts index dff32d849..b51fd2546 100644 --- a/packages/core/src/services/__fixtures__/compose-path-host.ts +++ b/packages/core/src/services/__fixtures__/compose-path-host.ts @@ -1,3 +1,4 @@ +import { readGenerationServiceFiles, serviceFileTreePath } from "../service-file-store"; import { ensureReleaseStateDirectories } from "../release-state"; import { spawnSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; @@ -165,10 +166,12 @@ export async function composePathHost() { entry?.volumes?.find((v) => v.target === "/infra-public")?.source ?? "", join(infraDir, "public-data.txt"), ); - checks[`${id}.serviceBind`] = sameFile( - entry?.volumes?.find((v) => v.target === "/service-public")?.source ?? "", - join(service.directory, "local-public.txt"), - ); + const source = entry?.volumes?.find((v) => v.target === "/service-public")?.source ?? ""; + const tree = readGenerationServiceFiles(dirname(realpathSync(compose)))?.get(id); + const copy = tree ? join(serviceFileTreePath(infraDir, id, tree), "local-public.txt") : ""; + const original = join(service.directory, "local-public.txt"); + checks[`${id}.serviceBind`] = !!tree && sameFile(source, copy) && !sameFile(source, original) && + readFileSync(source).equals(readFileSync(original)) && (statSync(source).mode & 0o777) === 0o444; const secret = Object.values(model.secrets).find((s) => s.file.endsWith(`/${id}/PRIVATE_SETTING`), ); diff --git a/packages/core/src/services/__tests__/service-file-store.test.ts b/packages/core/src/services/__tests__/service-file-store.test.ts index 3e63c7cc0..c7e317172 100644 --- a/packages/core/src/services/__tests__/service-file-store.test.ts +++ b/packages/core/src/services/__tests__/service-file-store.test.ts @@ -400,9 +400,10 @@ describe("the runtime bundle's service files", () => { it("knows every file's current bytes as a version the release shipped", () => { for (const file of files) { - expect(history.files[`${file.serviceId}/${file.path}`]).toContain( - sha256(readFileSync(fileURLToPath(file.url))), - ); + expect( + history.files[`${file.serviceId}/${file.path}`], + `${file.serviceId}/${file.path}`, + ).toContain(sha256(readFileSync(fileURLToPath(file.url)))); } }); diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index bcee8155c..b94948950 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -1486,9 +1486,16 @@ async function commitOwnedConfigurationGeneration( releaseServiceFiles: release, agentMountLayout: options.agentMountLayout?.(), }); - const oldDays = readAppliedPlatformValues(options.infraDir).backupRetentionDays; - const newDays = resolvePlatformConfiguration({ saved: options.input.platform }).values - .backupRetentionDays; + // This store lock concerns retention only. An invocation-level domain + // override has already been validated by renderConfiguration above; parsing + // unrelated host fields again would incorrectly discard that override. + const retentionEnv = { BACKUP_RETENTION_DAYS: process.env.BACKUP_RETENTION_DAYS }; + const oldDays = resolvePlatformConfiguration({ + saved: readAppliedPlatformConfiguration(options.infraDir), + env: retentionEnv, + }).values.backupRetentionDays; + const newDays = resolvePlatformConfiguration({ saved: options.input.platform, env: retentionEnv }) + .values.backupRetentionDays; if ( oldDays !== newDays && options.expectedCurrentRevisionId !== undefined && diff --git a/packages/integration-framework/src/installer.ts b/packages/integration-framework/src/installer.ts index a33e9f419..9ddf872ed 100644 --- a/packages/integration-framework/src/installer.ts +++ b/packages/integration-framework/src/installer.ts @@ -16,6 +16,7 @@ import { createReadStream, type Dirent, existsSync, + fchmodSync, fstatSync, fsyncSync, linkSync, @@ -173,8 +174,9 @@ function stagingDir(rootDir: string): string { } function createStagePath(rootDir: string, prefix: string): string { + prepareCustomDir(rootDir); const parent = stagingDir(rootDir); - mkdirSync(parent, { recursive: true }); + mkdirSync(parent, { recursive: true, mode: 0o700 }); return join(parent, `${prefix}-${randomBytes(4).toString("hex")}`); } @@ -542,6 +544,10 @@ export async function installIntegration(opts: InstallOptions): Promise { + parent.check(); + fchmodSync(parent.fd, INSTALLED_DIRECTORY_MODE); + for (const name of parent.listBounded(4096)) { + if (++count > 4096) throw Error("Installed integration entry limit exceeded"); + const path = parent.name(name), + before = lstatSync(path, { bigint: true }); + if (before.isDirectory()) { + const child = parent.directory(name); + try { + walk(child); + } finally { + child.close(); + } + } else { + const fd = openSync(path, C.O_RDONLY | C.O_NOFOLLOW | C.O_NONBLOCK); + try { + const opened = fstatSync(fd, { bigint: true }); + if ( + !opened.isFile() || + opened.nlink !== 1n || + opened.uid !== BigInt(process.getuid?.() ?? -1) || + opened.dev !== before.dev || + opened.ino !== before.ino || + (opened.mode & 0o022n) !== 0n + ) + throw Error("Installed integration file changed"); + fchmodSync(fd, INSTALLED_FILE_MODE); + const actual = parent.fileIdentity(name, false); + if (actual.dev !== String(opened.dev) || actual.ino !== String(opened.ino)) + throw Error("Installed integration file replaced"); + } finally { + closeSync(fd); + } + } + } + parent.check(); + }; + try { + if ( + !sameDirectory( + { dev: String(directory.stat.dev), ino: String(directory.stat.ino) }, + receipt.preparation.content, + ) + ) + throw Error("Installed CLI directory changed"); + walk(directory); + requirePublishedTarget(publication, new AbortController().signal); + } finally { + directory.close(); + } +} + +function retireCompletedCliPreparation(context: IntegrationArtifactPreparationContext): void { + const state = preparationState(context); + if ( + state.journal || + state.phase !== "published" || + readPublicationRecord(state).outcome !== "committed" + ) + throw Error("Committed local CLI preparation required"); + const workspace = checkedWorkspace(state); + const root = new AnchoredDirectory(customDir(state.rootDir)); + let parent: AnchoredDirectory | undefined; + try { + if (JSON.stringify(workspace.listBounded(2)) !== '["publication.json"]') + throw Error("Unexpected committed CLI preparation contents"); + workspace.removeFile("publication.json"); + parent = root.directory(STAGING_DIR_NAME); + parent.removeDirectory(`preparation-${state.preparationId}`, state.workspace); + state.phase = "discarded"; + } finally { + parent?.close(); + root.close(); + workspace.close(); + } +} + export function discardIntegrationPreparation( context: IntegrationArtifactPreparationContext, ): void { @@ -2265,6 +2363,7 @@ function restoreInstalledIntegrationUnlocked( throw Error( "Historical integration rollback lacks a publication ownership receipt; preserve the current integration and reconcile the original journal", ); + applyInstalledModes(restoreStage); renameSync(restoreStage, target); } catch (error) { rmSync(restoreStage, { recursive: true, force: true }); diff --git a/scripts/database-test-groups.json b/scripts/database-test-groups.json index 0aaa69da6..8abaebc45 100644 --- a/scripts/database-test-groups.json +++ b/scripts/database-test-groups.json @@ -36,7 +36,10 @@ "apps/api/src/services/__tests__/platform-preview-postgres.test.ts", "apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts", "apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts", - "apps/api/src/privacy/admin-workflows-postgres.test.ts" + "apps/api/src/privacy/admin-workflows-postgres.test.ts", + "apps/api/src/services/extension-workflow-postgres.test.ts", + "apps/api/src/services/configuration-migration/credential-transfer-postgres.test.ts", + "apps/api/src/services/configuration-migration/migration-postgres.test.ts" ], "ingestion": [ "apps/api/src/services/__tests__/job-runner-postgres.test.ts", diff --git a/scripts/ops-agent-catalog-pins.test.ts b/scripts/ops-agent-catalog-pins.test.ts index 11fc2e1db..f89c30b10 100644 --- a/scripts/ops-agent-catalog-pins.test.ts +++ b/scripts/ops-agent-catalog-pins.test.ts @@ -13,31 +13,37 @@ beforeAll(async () => { const text = (name: string) => Buffer.from(files.get(name) ?? []).toString("utf8"); describe("the ops-agent bundle's catalog pins", () => { - it.each(["gbfs-catalog.lock.json", "transitous.lock.json"])( - "carries the release's %s and reads nothing else by that name", - (name) => { - const pattern = new RegExp(`^assets/[a-f0-9]{12}-${name.replaceAll(".", "\\.")}$`); - const assets = [...files.keys()].filter((file) => pattern.test(file)); - expect(assets).toHaveLength(1); - const [asset] = assets as [string]; - expect(Buffer.from(files.get(asset) ?? []).toString("utf8")).toBe( - readFileSync(join(ROOT, "infra", "docker", name), "utf8"), - ); - // Every mention of the file in the agent's and the CLI's code is the asset. - const mention = new RegExp(`[^"'\`\\s]*${name.replaceAll(".", "\\.")}`, "g"); - for (const entry of ["agent.mjs", "cli.mjs"]) { - const mentions = text(entry).match(mention) ?? []; - expect( - mentions.filter((found) => found !== `./${asset}`), - entry, - ).toEqual([]); - } - }, - ); + it.each(["transitous.lock.json"])("embeds the release's immutable %s baseline", (name) => { + const pattern = new RegExp(`^assets/[a-f0-9]{12}-${name.replaceAll(".", "\\.")}$`); + const assets = [...files.keys()].filter((file) => pattern.test(file)); + expect(assets).toHaveLength(1); + const [asset] = assets as [string]; + expect(Buffer.from(files.get(asset) ?? []).toString("utf8")).toBe( + readFileSync(join(ROOT, "infra", "docker", name), "utf8"), + ); - it("reads no Transitous proposal from the checkout", () => { - for (const entry of ["agent.mjs", "cli.mjs"]) { - expect(text(entry), entry).not.toContain("transitous.lock.proposed.json"); - } + // Paired-pin management intentionally retains fixed compatibility import + // paths. Runtime consumers still receive this immutable release baseline; + // activation and proposal isolation are covered by the actual catalog store. + for (const entry of ["agent.mjs"]) expect(text(entry), entry).toContain(`./${asset}`); + }); + + it("omits unused GBFS release files when consumers read the managed pair", () => { + expect([...files.keys()].filter((name) => name.endsWith("-gbfs-catalog.lock.json"))).toEqual( + [], + ); + expect(text("cli.mjs")).toContain("catalogPins.inspect"); + }); + + it("packages no mutable checkout proposal as a release asset", () => { + expect([...files.keys()].filter((name) => name.includes("transitous.lock.proposed"))).toEqual( + [], + ); + const runtime = readFileSync( + join(ROOT, "apps/ops-agent/src/transitous-lock-runtime.ts"), + "utf8", + ); + expect(runtime).toContain("readCatalogPinSet"); + expect(runtime).toContain("retainCatalogProposal"); }); }); diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index 37b0593c7..cd2902f75 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -322,7 +322,7 @@ "apps/api/src/services/community-repositories.ts#publishServiceRepository", "apps/api/src/services/community-repositories.ts#settleServiceRepository", "apps/api/src/services/community-repositories.ts#removeServiceRepository", - "apps/api/src/services/community-repositories.ts#readCommunityServices", + "apps/api/src/services/community-service-inventory.ts#readCommunityServices", "apps/api/src/services/extension-installer.ts#installExtension", "apps/api/src/services/extension-installer.ts#removeExtension" ], diff --git a/services/data-manager/__tests__/ambient-places/api.test.ts b/services/data-manager/__tests__/ambient-places/api.test.ts index f02d55769..4f1baf210 100644 --- a/services/data-manager/__tests__/ambient-places/api.test.ts +++ b/services/data-manager/__tests__/ambient-places/api.test.ts @@ -1,3 +1,15 @@ +// Unit boundary: these tests exercise traffic/auth/async response behavior. +// SQL admission and retained ownership are qualified by the owned PostgreSQL +// legacy-admission/configuration-maintenance suites, never by this collaborator. +vi.mock("../../src/jobs/execution/legacy-mutation.js", () => ({ + withLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + independentLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + startLegacyDataMutation: async (_kind: string, run: () => Promise) => ({ + id: "unit-writer", + completion: run(), + }), +})); + import Fastify from "fastify"; import type postgres from "postgres"; import { describe, expect, it, vi } from "vitest"; diff --git a/services/data-manager/__tests__/openapi-freshness.test.ts b/services/data-manager/__tests__/openapi-freshness.test.ts index da68f94b3..67b8efa6a 100644 --- a/services/data-manager/__tests__/openapi-freshness.test.ts +++ b/services/data-manager/__tests__/openapi-freshness.test.ts @@ -8,7 +8,9 @@ it("exports every actual data-manager route with stable IDs, auth and payload co expect(generated).toBe(await readFile(DOCUMENT_PATH, "utf8")); const document = JSON.parse(generated); const operations = Object.values(document.paths).flatMap((path) => Object.values(path as object)); - expect(operations).toHaveLength(58); + expect(operations.map((value: { operationId: string }) => value.operationId).sort()).toEqual( + Object.keys(dataManagerOperations).sort(), + ); for (const value of operations) { const operation = value as { diff --git a/services/data-manager/__tests__/service-env-declared.test.ts b/services/data-manager/__tests__/service-env-declared.test.ts index 93ea72a5b..6ee4d0f54 100644 --- a/services/data-manager/__tests__/service-env-declared.test.ts +++ b/services/data-manager/__tests__/service-env-declared.test.ts @@ -23,7 +23,13 @@ function scannedEnvironmentNames(): Set { /["'`]([A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)["'`]/g, ]; - for (const file of sourceFiles(sourceRoot)) { + // The daemon resolves transit acquisition and operations policy in this shared module. + const files = [ + ...sourceFiles(sourceRoot), + join(process.cwd(), "packages/transitous-core/src/constants.ts"), + join(process.cwd(), "packages/transitous-core/src/operations-profile.ts"), + ]; + for (const file of files) { const source = readFileSync(file, "utf-8"); for (const regex of regexes) { for (const match of source.matchAll(regex)) names.add(match[1]); @@ -45,6 +51,8 @@ const INTENTIONALLY_UNDECLARED = new Set([ // Set by services/data-manager/Dockerfile. Declaring it here would overwrite // it with an empty string and disable the production hard-fail in src/auth.ts. "NODE_ENV", + // Compatibility-only path fallback; new deployments expose API_KEYS_PATH. + "TRANSITOUS_API_KEYS_FILE", ]); /** Runtime knobs consumed by Node rather than by our own code. */ diff --git a/services/data-manager/__tests__/transit-sources.test.ts b/services/data-manager/__tests__/transit-sources.test.ts index f8b8ee1f2..88b87b898 100644 --- a/services/data-manager/__tests__/transit-sources.test.ts +++ b/services/data-manager/__tests__/transit-sources.test.ts @@ -1,7 +1,9 @@ +import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { createCandidateManifest } from "../src/jobs/transitous/candidate.js"; import { resolveTransitOverlayPath } from "../src/jobs/transitous/internal.js"; import { safeOriginUrl } from "../src/jobs/transitous/source-manifest.js"; import { @@ -63,22 +65,30 @@ function fixture() { transformations: ["transitous-cleaning"], artifact: { relativePath: "de_active-feed.gtfs.zip", - sha256: "a".repeat(64), - sizeBytes: 42, + sha256: createHash("sha256").update("synthetic timetable").digest("hex"), + sizeBytes: Buffer.byteLength("synthetic timetable"), modifiedAt: "2026-07-29T12:00:00.000Z", }, }, ], }), ); + const live = join(tmp, "motis", "live"); + mkdirSync(join(live, ".openmapx-feed-proxy", "conf"), { recursive: true }); writeFileSync( - join(tmp, "motis", "live", "motis-candidate-manifest.json"), + join(live, "config.yml"), JSON.stringify({ - schemaVersion: 1, - epoch: "epoch-live", - artifacts: { sourceManifest: { path: "transit-source-manifest.json" } }, + timetable: { datasets: { "de-active-feed": { path: "de_active-feed.gtfs.zip" } } }, }), ); + writeFileSync(join(live, "license.json"), "[]"); + writeFileSync(join(live, "de_active-feed.gtfs.zip"), "synthetic timetable"); + writeFileSync( + join(live, ".openmapx-feed-proxy", "conf", "default.conf"), + "# no realtime feeds\n", + ); + writeFileSync(join(live, ".openmapx-feed-proxy", "feed-proxy-vars.json"), "{}"); + createCandidateManifest(live, "epoch-live", "2026-07-30T00:00:00.000Z"); return { dataDir: tmp, catalogDir, overlayPath }; } diff --git a/services/data-manager/__tests__/transitous-pipeline.test.ts b/services/data-manager/__tests__/transitous-pipeline.test.ts index 096486f7e..dc3c0dba6 100644 --- a/services/data-manager/__tests__/transitous-pipeline.test.ts +++ b/services/data-manager/__tests__/transitous-pipeline.test.ts @@ -1,6 +1,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; import type { TransitousRunnerScript } from "@openmapx/core/transitous-runner"; import { afterEach, describe, expect, it, vi } from "vitest"; import { buildJobContext, runTransitousPipeline } from "../src/jobs/transitous/index.js"; @@ -17,17 +18,17 @@ const PINNED_LOCK = { }; vi.mock("../src/ops-client.js", () => ({ runOpsOperation: vi.fn(async (operation: { kind: string }) => { - if (operation.kind === "transitousLock.inspect") { - return { active: PINNED_LOCK, proposed: null }; - } - if (operation.kind === "gbfsCatalogLock.inspect") { - return { + if (operation.kind === "catalogPins.inspect") { + const active = createCatalogPinSet(PINNED_LOCK, { + schemaVersion: 1, + source: "mobilitydata-gbfs", commit: "b".repeat(40), - url: "https://example.test/catalog.csv", + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"b".repeat(40)}/systems.csv`, sha256: "c".repeat(64), lockedAt: "2026-04-20T12:00:00.000Z", lockedBy: "test", - }; + }); + return { active, catalogRevisionId: active.generationId }; } return { changed: true }; }), diff --git a/services/data-manager/__tests__/transitous/pipeline-mirror.test.ts b/services/data-manager/__tests__/transitous/pipeline-mirror.test.ts index 522618649..235f4d43c 100644 --- a/services/data-manager/__tests__/transitous/pipeline-mirror.test.ts +++ b/services/data-manager/__tests__/transitous/pipeline-mirror.test.ts @@ -1,6 +1,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; import { transitousRunnerArgv } from "@openmapx/core/transitous-runner"; import type { SafeDownloadOptions } from "@openmapx/core/utils/safe-download"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -23,17 +24,17 @@ const PINNED_LOCK = { }; vi.mock("../../src/ops-client.js", () => ({ runOpsOperation: vi.fn(async (operation: { kind: string }) => { - if (operation.kind === "transitousLock.inspect") { - return { active: PINNED_LOCK, proposed: null }; - } - if (operation.kind === "gbfsCatalogLock.inspect") { - return { + if (operation.kind === "catalogPins.inspect") { + const active = createCatalogPinSet(PINNED_LOCK, { + schemaVersion: 1, + source: "mobilitydata-gbfs", commit: "b".repeat(40), - url: "https://example.test/catalog.csv", + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"b".repeat(40)}/systems.csv`, sha256: "c".repeat(64), lockedAt: "2026-04-20T12:00:00.000Z", lockedBy: "test", - }; + }); + return { active, catalogRevisionId: active.generationId }; } return { changed: true }; }), diff --git a/services/data-manager/service.json b/services/data-manager/service.json index bb61ba0c3..f3d139eb0 100644 --- a/services/data-manager/service.json +++ b/services/data-manager/service.json @@ -62,6 +62,7 @@ "TRANSITOUS_STALENESS_CHECK_CRON": "${TRANSITOUS_STALENESS_CHECK_CRON:-}", "TRANSITOUS_FEED_PROXY_RELOAD_CRON": "${TRANSITOUS_FEED_PROXY_RELOAD_CRON:-}", "TRANSITOUS_AUTO_BUMP_CRON": "${TRANSITOUS_AUTO_BUMP_CRON:-}", + "TRANSITOUS_AUTO_BUMP_BRANCH": "${TRANSITOUS_AUTO_BUMP_BRANCH:-}", "TRANSITOUS_ALERT_GH_TOKEN": "${TRANSITOUS_ALERT_GH_TOKEN:-}", "TRANSITOUS_ALERT_GH_REPO": "${TRANSITOUS_ALERT_GH_REPO:-}", "MOTIS_ELEVATORS_URL": "${MOTIS_ELEVATORS_URL:-}", diff --git a/services/data-manager/src/__tests__/convert-overpass-route.test.ts b/services/data-manager/src/__tests__/convert-overpass-route.test.ts index aa22e4d78..fede43d94 100644 --- a/services/data-manager/src/__tests__/convert-overpass-route.test.ts +++ b/services/data-manager/src/__tests__/convert-overpass-route.test.ts @@ -57,7 +57,7 @@ describe("POST /convert/overpass", () => { }); it.each([{}, { region: "europe/germany" }])( - "keeps the trusted region-based form available for %j", + "requires the assembled durable receiver for the region-based form %j", async (payload) => { const dataDir = makeDataDir(); const app = Fastify(); @@ -70,9 +70,12 @@ describe("POST /convert/overpass", () => { payload, }); - expect(response.statusCode).toBe(200); - expect(response.headers["content-type"]).toContain("application/x-ndjson"); - expect(response.json()).toMatchObject({ event: "error" }); + expect(response.statusCode).toBe(503); + expect(response.json()).toMatchObject({ + error: "Service Unavailable", + message: "Durable data receiver is unavailable", + }); + expect(existsSync(join(dataDir, "overpass"))).toBe(false); } finally { await app.close(); rmSync(dataDir, { recursive: true, force: true }); diff --git a/services/data-manager/src/__tests__/openconditions-auth.test.ts b/services/data-manager/src/__tests__/openconditions-auth.test.ts index 252121bdd..a3e46d629 100644 --- a/services/data-manager/src/__tests__/openconditions-auth.test.ts +++ b/services/data-manager/src/__tests__/openconditions-auth.test.ts @@ -1,3 +1,15 @@ +// Unit boundary: these tests exercise traffic/auth/async response behavior. +// SQL admission and retained ownership are qualified by the owned PostgreSQL +// legacy-admission/configuration-maintenance suites, never by this collaborator. +vi.mock("../jobs/execution/legacy-mutation.js", () => ({ + withLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + independentLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + startLegacyDataMutation: async (_kind: string, run: () => Promise) => ({ + id: "unit-writer", + completion: run(), + }), +})); + import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; diff --git a/services/data-manager/src/__tests__/predicted-bake-route.test.ts b/services/data-manager/src/__tests__/predicted-bake-route.test.ts index 7f8db2714..590649180 100644 --- a/services/data-manager/src/__tests__/predicted-bake-route.test.ts +++ b/services/data-manager/src/__tests__/predicted-bake-route.test.ts @@ -1,3 +1,15 @@ +// Unit boundary: these tests exercise traffic/auth/async response behavior. +// SQL admission and retained ownership are qualified by the owned PostgreSQL +// legacy-admission/configuration-maintenance suites, never by this collaborator. +vi.mock("../jobs/execution/legacy-mutation.js", () => ({ + withLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + independentLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + startLegacyDataMutation: async (_kind: string, run: () => Promise) => ({ + id: "unit-writer", + completion: run(), + }), +})); + import Fastify from "fastify"; import { describe, expect, it, vi } from "vitest"; import { registerApi } from "../api.js"; diff --git a/services/data-manager/src/__tests__/traffic-live-conditions.test.ts b/services/data-manager/src/__tests__/traffic-live-conditions.test.ts index 58b788cdf..e9dfbd907 100644 --- a/services/data-manager/src/__tests__/traffic-live-conditions.test.ts +++ b/services/data-manager/src/__tests__/traffic-live-conditions.test.ts @@ -1,3 +1,15 @@ +// Unit boundary: these tests exercise traffic/auth/async response behavior. +// SQL admission and retained ownership are qualified by the owned PostgreSQL +// legacy-admission/configuration-maintenance suites, never by this collaborator. +vi.mock("../jobs/execution/legacy-mutation.js", () => ({ + withLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + independentLegacyDataMutation: async (_kind: string, run: () => Promise) => run(), + startLegacyDataMutation: async (_kind: string, run: () => Promise) => ({ + id: "unit-writer", + completion: run(), + }), +})); + import { mkdtemp } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; diff --git a/services/data-manager/src/cron.ts b/services/data-manager/src/cron.ts index 2d5b79079..9c362c1f3 100644 --- a/services/data-manager/src/cron.ts +++ b/services/data-manager/src/cron.ts @@ -559,7 +559,7 @@ export function setupCron(options: CronSetupOptions): CronHandles { const runAutoBump = async (): Promise => { if (!options.catalogReview) throw Error("Installed paired catalog resolver is unavailable"); const proposal = await options.catalogReview.resolveCatalogProposal({ - branch: process.env.TRANSITOUS_AUTO_BUMP_BRANCH ?? "master", + branch: envString("TRANSITOUS_AUTO_BUMP_BRANCH", "master"), }); log.info("transitous-auto-bump: paired proposal retained for administrator review", { proposalId: proposal.proposalId, diff --git a/services/service-file-history.json b/services/service-file-history.json index 997c1d4dd..bfb7cb5e9 100644 --- a/services/service-file-history.json +++ b/services/service-file-history.json @@ -66,9 +66,11 @@ "e5e9d00d389d8dfec1318a8ea0bc18df134ddc9e5b4b3a51f45c0781a05f97cb" ], "traefik/config/traefik.yml": [ + "0d6eec76c6dbb6a1e3bebc2cf573c0086255061839a440456754cd7956cbc024", "0dc3a987816309eb1c0c17190eec2ef50d2d1bb4e05f1b766f84cdaa97a0f32c", "1f72249934d0a1522c335413b13678815abb9adadb993581db29784e03c6849d", "419fa4f55029c577532252c9f00ad5c27ea9eeafcb54be6c75f831f679019798", + "467992adf5a77a7d4c7a17e091ed4f52209926ad62cd3a0c64f2f7f758df5f12", "7b2271fdd8ca1f3c5e6e414e745c545be4f158bd3f06ebdc7f699aabb812cb8a", "7b4a6b1780088e72e74400c66ad25b6d4f884b067b033f5ff86fac73ac1908d0", "8638158a25b1fdfebfe1c8031ec4eabe9f4d98a8bb148ae44670e8ff00a182c5", From ef818958150bfaf4832011426d80810fe4f1ced3 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 31 Oct 2026 02:03:20 +0100 Subject: [PATCH 105/135] docs(admin): describe upgrade requirements and extension contract release --- .../admin-instance-management-contracts.md | 8 ++++ release-notes/admin-instance-management.md | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 .changeset/admin-instance-management-contracts.md create mode 100644 release-notes/admin-instance-management.md diff --git a/.changeset/admin-instance-management-contracts.md b/.changeset/admin-instance-management-contracts.md new file mode 100644 index 000000000..4daec1001 --- /dev/null +++ b/.changeset/admin-instance-management-contracts.md @@ -0,0 +1,8 @@ +--- +"@openmapx/extension-contracts": minor +"@openmapx/extension-cli": patch +--- + +Export bounded service image-reference validation and reserve platform resource +configuration names for trusted built-in services. Rebuild the extension CLI +against the updated shared installation and service contracts. diff --git a/release-notes/admin-instance-management.md b/release-notes/admin-instance-management.md new file mode 100644 index 000000000..2e010b797 --- /dev/null +++ b/release-notes/admin-instance-management.md @@ -0,0 +1,37 @@ +--- +impacts: [migration, operator-action] +--- + +## What changes + +Administrators can manage services, supported platform settings, transit data, +prepared artifacts, integrations, account policy, backups and privacy workflows +through the panel. Mutations retain their original jobs and recovery inputs +across lost responses and process restarts. Uncertain operations keep their +writer fences until their original outcomes can be verified. + +The API migrations add account-policy, integration ownership, artifact-transfer +and privacy-workflow storage alongside durable execution records. Existing auth +settings are activated explicitly through the account-policy workflow. + +## Who is affected + +Self-hosted installations upgrading the API, web application, CLI, Data Manager +and operations agent, and authors of community service manifests. + +## Required action + +Apply the normal API database migrations and upgrade the control services +together. Preserve existing installation keys, pending jobs and recovery +records. Inspect Activity before retrying an uncertain mutation; deleting its +records does not prove that its host effect stopped. + +Before the first render, make the host-control authority ancestors +non-group-writable with `chmod go-w . infra infra/docker infra/docker/secrets` +from the checkout root. Existing unsafe directories are refused rather than +silently repaired. See the installation and upgrade guides for host-owned +overrides and recovery boundaries. + +Community service configuration schemas must not declare the reserved platform +resource names `OPENMAPX_MEMORY_MIB`, `OPENMAPX_CPU_LIMIT` or +`OPENMAPX_JAVA_HEAP_MIB`. From 14201a64e808ae2b745eef322f8f7942aa88ccc7 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 31 Oct 2026 02:09:49 +0100 Subject: [PATCH 106/135] test(agent): exercise current native control and protected bootstrap --- scripts/ops-agent-checkout-trust.test.ts | 94 ++++++++++++++++-------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/scripts/ops-agent-checkout-trust.test.ts b/scripts/ops-agent-checkout-trust.test.ts index 00409ae27..dbe676547 100644 --- a/scripts/ops-agent-checkout-trust.test.ts +++ b/scripts/ops-agent-checkout-trust.test.ts @@ -16,14 +16,17 @@ import { tmpdir } from "node:os"; import { join, relative, resolve } from "node:path"; import { promisify } from "node:util"; import { gzipSync } from "node:zlib"; -import { OpsClient } from "@openmapx/core/ops"; +import { + createControlRootOperationKey, + OpsClient, + type OpsOperation, + type OpsOperationKind, + type OpsResultFor, + opsExecutionBudget, +} from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { - createDurableOpsKey, - executeAndWait, - executeTrustedConfigurationAndWait, -} from "../apps/api/src/services/ops-client"; +import { waitForOpsResult } from "../apps/api/src/services/ops-client"; import { consumePublishedTrustedConfiguration, publishTrustedConfigurationSnapshot, @@ -58,7 +61,7 @@ afterAll(() => { for (const agent of agents.splice(0)) agent.kill("SIGKILL"); if (outdir) rmSync(outdir, { recursive: true, force: true }); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); -}); +}, 120_000); const secret = () => randomBytes(24).toString("base64url"); @@ -84,6 +87,18 @@ async function umask002Checkout(): Promise { ].join("\n"), { mode: 0o600 }, ); + // Durable control keys require non-group-writable authority ancestors. + // This is an explicit operator bootstrap step, not application chmod of a + // shared checkout. Service/integration source trees retain umask-002 modes. + const unsafe = await cli(checkout, "compose", "render"); + expect(unsafe.stderr).toContain("Directory authority changed or unsafe"); + for (const directory of [ + checkout, + join(checkout, "infra"), + join(checkout, "infra", "docker"), + join(checkout, "infra", "docker", "secrets"), + ]) + chmodSync(directory, 0o700); return checkout; } @@ -175,10 +190,32 @@ async function startAgent(checkout: string): Promise { return { port, output: () => output, stop: () => agent.kill("SIGTERM") }; } -/** - * Publishes a trusted-configuration request as app-api does, asks the agent - * to render it, and expects a committed generation. - */ +/** The fixed local CLI's truthful bearer-root identity; no API job is invented. */ +function localRootIdentity(kind: OpsOperationKind) { + const requestedDeadlineAt = Date.now() + opsExecutionBudget(kind).processMs; + return { + operationKey: createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }), + execution: { requestedDeadlineAt }, + }; +} + +async function executeLocalRoot( + client: OpsClient, + operation: Extract, + identity = localRootIdentity(operation.kind), +): Promise> { + const submitted = await client.execute(operation, identity); + if (submitted.execution === "sync") return submitted.value; + return waitForOpsResult(client, operation.kind, submitted, { + execution: identity.execution, + observerDeadlineAt: identity.execution.requestedDeadlineAt, + }); +} + +/** Publishes the real sealed payload and current source witness over agent HTTP. */ async function expectTrustedConfigurationCommitted( checkout: string, agent: RunningAgent, @@ -190,7 +227,11 @@ async function expectTrustedConfigurationCommitted( const tokenFile = join(infra, "secrets", "ops-agent-api-token"); const token = readFileSync(tokenFile, "utf8").trim(); const directory = statSync(trustedConfig); - const operationKey = createDurableOpsKey("stack.render", name); + const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agent.port}`, tokenFile }); + const inspected = await client.execute({ kind: "configuration.sources.inspect" }); + if (inspected.execution !== "sync") throw Error("Expected a synchronous source inspection"); + const identity = localRootIdentity("stack.render"); + const { operationKey } = identity; const before = services.currentConfigurationGeneration(infra); const sealed = await publishTrustedConfigurationSnapshot({ directory: trustedConfig, @@ -202,6 +243,8 @@ async function expectTrustedConfigurationCommitted( kind: "stack.render", revisionId, expectedCurrentRevisionId: before, + expectedSourceRevision: inspected.value.sourceRevision, + coordinatorMode: "require-running", }), payload: { domain: `${name}.example.test`, @@ -211,13 +254,8 @@ async function expectTrustedConfigurationCommitted( ...payload, }, }); - const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agent.port}`, tokenFile }); const consumed = await consumePublishedTrustedConfiguration(sealed, () => - executeTrustedConfigurationAndWait( - client, - sealed.operation as Parameters[1], - operationKey, - ), + executeLocalRoot(client, sealed.operation, identity), ); expect(consumed.submission.status, agent.output()).toBe("fulfilled"); const result = consumed.submission.status === "fulfilled" ? consumed.submission.value : undefined; @@ -237,12 +275,10 @@ describe("a checkout written under umask 002", () => { it("gets the bundled agent through startup and a trusted-configuration request", async () => { const checkout = await umask002Checkout(); for (const path of [ - checkout, join(checkout, "services"), join(checkout, "services", "valhalla", "service.json"), join(checkout, "integrations"), join(checkout, "integrations", "air-quality", "manifest.json"), - join(checkout, "infra", "docker"), ]) { expect(statSync(path).mode & 0o020, path).toBe(0o020); } @@ -253,8 +289,8 @@ describe("a checkout written under umask 002", () => { const agent = await startAgent(checkout); await expectTrustedConfigurationCommitted(checkout, agent, "umask-002-checkout", {}); agent.stop(); - // The agent's state is its own directory; a group-writable infra/docker is - // fine as it is, and nothing changes its mode. + // The explicitly hardened authority ancestors retain their operator modes; + // source directories keep their original umask-002 permissions. expect(statSync(join(checkout, "infra", "docker")).mode).toBe(infraMode); }, 120_000); }); @@ -412,15 +448,11 @@ describe("integrations installed through the operations agent", () => { // The earlier install is recorded only after its files match its archive. const tokenFile = join(infra, "secrets", "ops-agent-api-token"); const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agent.port}`, tokenFile }); - const adopted = await executeAndWait( - client as unknown as Parameters[0], - { - kind: "integration.adopt", - integrationId: "legacy", - source: { kind: "upload", sha256: legacy }, - }, - createDurableOpsKey("integration.adopt", "legacy"), - ); + const adopted = await executeLocalRoot(client, { + kind: "catalogIntegration.adopt", + integrationId: "legacy", + source: { kind: "upload", sha256: legacy }, + }); expect(adopted).toEqual({ integrationId: "legacy", status: "recorded", reason: null }); await expectTrustedConfigurationCommitted(checkout, agent, "installed-integrations", { From 39bc7e64bff66cbb7df759ec50389f616f14665d Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 31 Oct 2026 02:43:38 +0100 Subject: [PATCH 107/135] fix(admin): preserve safe Linux output modes and isolated recovery checks --- .../admin-instance-management-contracts.md | 3 + .../src/__fixtures__/configuration-writer.mjs | 2 +- .../src/__fixtures__/owned-service-host.ts | 15 +-- .../src/administrative-runtime.test.ts | 27 ++--- .../artifact-transfer/import-runtime.test.ts | 10 +- .../integration-consumer.test.ts | 4 +- .../src/artifact-transfer/lifecycle.test.ts | 1 + .../src/artifact-transfer/motis.test.ts | 40 +++++--- .../prepared-consumer.test.ts | 6 +- .../test-fixtures/validation-recovery.mjs | 4 +- apps/ops-agent/src/catalog-pins.test.ts | 4 +- .../configuration-writer-concurrency.test.ts | 7 +- .../src/core-service-runtime.test.ts | 2 +- apps/ops-agent/src/data-inventory.test.ts | 52 +++++----- .../src/data-transit-resource-source.test.ts | 34 ++++--- apps/ops-agent/src/docker-runtime.test.ts | 69 +++++++++---- apps/ops-agent/src/production-runtime.test.ts | 5 +- apps/ops-agent/src/server.test.ts | 1 + .../standalone-integration-runtime.test.ts | 22 +++-- .../src/transit-credentials-runtime.test.ts | 10 +- .../src/transitous-lock-runtime.test.ts | 6 +- .../src/trusted-config-ownership.test.ts | 6 +- .../src/trusted-config-runtime.test.ts | 10 +- packages/cli/__tests__/cli-compose.test.ts | 1 + ...nsitous-api-keys-mount.integration.test.ts | 87 ++++++++++++++-- packages/cli/src/commands/compose.ts | 1 + .../core/src/build-artifacts/archive.test.ts | 28 ++++-- .../configuration-source-witness.test.ts | 2 +- .../src/build-artifacts/copy-tree.test.ts | 20 ++-- .../local-credential-template.test.ts | 2 +- .../build-artifacts/motis-candidate.test.ts | 24 +++-- .../build-artifacts/motis-portable.test.ts | 36 ++++--- .../src/build-artifacts/publication.test.ts | 20 ++-- packages/core/src/catalog-pins/store.test.ts | 26 +++-- .../services/__tests__/agent-mounts.test.ts | 8 +- .../__tests__/backup-prune-plan.test.ts | 13 +-- .../configuration-generation.test.ts | 3 +- .../service-lifecycle-receipt.test.ts | 4 +- packages/hardlinks/__tests__/apply.test.ts | 32 ++++++ packages/hardlinks/src/index.ts | 10 +- packages/motis-feed-proxy-config/src/build.ts | 11 ++- release-notes/admin-instance-management.md | 5 + scripts/ops-agent-deployment-user.test.ts | 99 +++++++++++++++---- scripts/ops-agent-mount-layouts.test.ts | 86 ++++++++++------ .../__tests__/catalog-pair-read.test.ts | 3 +- .../__tests__/download-osm.test.ts | 5 +- .../__tests__/gbfs-catalog-lock.test.ts | 2 +- .../__tests__/helpers/gtfs-fixture.ts | 2 +- .../__tests__/osm-fingerprints.test.ts | 14 +-- .../__tests__/overture/local-inputs.test.ts | 22 +++-- .../__tests__/transit-sources.test.ts | 20 ++-- .../__tests__/transitous-lock.test.ts | 8 +- .../transitous/assemble-staging.test.ts | 69 +++++++------ .../__tests__/transitous/candidate.test.ts | 21 ++-- .../__tests__/transitous/motis-health.test.ts | 16 +-- .../__tests__/transitous/motis-import.test.ts | 21 ++-- .../__tests__/transitous/pipeline.test.ts | 74 +++++++++----- .../__tests__/transitous/promote.test.ts | 57 ++++++----- .../transitous/proxy-transaction.test.ts | 24 +++-- .../data-manager/src/jobs/download-osm.ts | 2 +- .../src/jobs/execution/ops-children.test.ts | 32 +++--- .../src/jobs/transitous/assemble-staging.ts | 2 +- .../src/jobs/transitous/candidate.ts | 2 +- .../src/jobs/transitous/compile-gbfs.ts | 4 +- .../src/jobs/transitous/internal.ts | 4 +- .../ops-agent/privacy-backup/test-fixture.mjs | 4 + services/ops-agent/service.json | 7 +- 67 files changed, 845 insertions(+), 428 deletions(-) diff --git a/.changeset/admin-instance-management-contracts.md b/.changeset/admin-instance-management-contracts.md index 4daec1001..09ca8790e 100644 --- a/.changeset/admin-instance-management-contracts.md +++ b/.changeset/admin-instance-management-contracts.md @@ -6,3 +6,6 @@ Export bounded service image-reference validation and reserve platform resource configuration names for trusted built-in services. Rebuild the extension CLI against the updated shared installation and service contracts. + +Keep newly created artifact and hardlink-consumer paths non-group-writable +on hosts using umask `002`. diff --git a/apps/ops-agent/src/__fixtures__/configuration-writer.mjs b/apps/ops-agent/src/__fixtures__/configuration-writer.mjs index f5f4d6e48..8b2e60b65 100644 --- a/apps/ops-agent/src/__fixtures__/configuration-writer.mjs +++ b/apps/ops-agent/src/__fixtures__/configuration-writer.mjs @@ -61,7 +61,7 @@ try { integrationConfigs: [], serviceSecrets: [], }; - mkdirSync(join(infraDir, "data"), { recursive: true }); + mkdirSync(join(infraDir, "data"), { recursive: true, mode: 0o700 }); await provisionControlInstallation(infraDir); const installation = openControlInstallation({ infraDir, role: "ops-agent", serviceIds: ["alpha"] }); let source; diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index 9bfeb7b13..7d367d7f1 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -37,14 +37,14 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); - mkdirSync(root,{recursive:true}); + mkdirSync(root,{recursive:true,mode:0o755}); chmodSync(root, 0o755); const infraDir = join(root, "infra/docker"); - mkdirSync(infraDir, { recursive: true }); + mkdirSync(infraDir, { recursive: true, mode: 0o700 }); writeFileSync(join(infraDir, ".env"), `UID=${process.geteuid?.() ?? 0}\nGID=${process.getegid?.() ?? 0}\n`, { mode: 0o600 }); - mkdirSync(join(root, "services")); - mkdirSync(join(root, "custom_integrations")); - writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(join(root, "services"), { mode: 0o755 }); + mkdirSync(join(root, "custom_integrations"), { mode: 0o755 }); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n", { mode: 0o644 }); const loaded: services.LoadedService[] = (options.configurationHelperReleaseInventory ? [...services.RELEASE_BUILT_IN_SERVICE_IDS] : [ "motis", "ops-agent", @@ -78,8 +78,8 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: isBuiltIn: true, })); for (const service of loaded) { - mkdirSync(service.directory, { recursive: true }); - writeFileSync(join(service.directory, "service.json"), JSON.stringify(service.manifest)); + mkdirSync(service.directory, { recursive: true, mode: 0o755 }); + writeFileSync(join(service.directory, "service.json"), JSON.stringify(service.manifest), { mode: 0o644 }); } const releaseInventory = options.configurationHelperReleaseInventory ? await services.captureReleaseServiceAuthority(root) : undefined; const loadAuthority = createTrustedConfigurationAuthorityLoader({ @@ -222,6 +222,7 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: digest: `sha256:${"b".repeat(64)}`, }, }), + { mode: 0o644 }, ); return { directory: targetDir, commit: "1".repeat(40) }; }, diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index 28323843d..ebbaa2e56 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -95,6 +95,7 @@ describe("default fixed CLI live output", () => { } console.log('last-diagnostic'); `, + { mode: 0o644 }, ); const seen: string[] = []; const completion = createDefaultFixedCli(root, join(root, "agent.mjs"))([], { @@ -109,11 +110,11 @@ describe("default fixed CLI live output", () => { if (check === "visibility") { await vi.waitFor(() => expect(seen).toContain("phase-one"), { timeout: 2000 }); } - writeFileSync(release, "go"); + writeFileSync(release, "go", { mode: 0o644 }); await expect(completion).resolves.toBeUndefined(); expect(seen).toContain("last-diagnostic"); } finally { - writeFileSync(release, "go"); + writeFileSync(release, "go", { mode: 0o644 }); await completion.catch(() => undefined); } }); @@ -424,6 +425,7 @@ describe("the fixed CLI", () => { writeFileSync( join(entryDirectory, "cli.mjs"), 'const { argv, env } = process;\nconsole.log([...argv.slice(2), env.DATA_MANAGER_URL, env.OPENMAPX_FIXED_CLI_PROBE].join(" "));\n', + { mode: 0o644 }, ); vi.stubEnv("DATA_MANAGER_URL", "http://localhost:4000"); vi.stubEnv("OPENMAPX_FIXED_CLI_PROBE", "inherited"); @@ -546,8 +548,8 @@ describe("administrative data runtime", () => { it("returns bounded data inventory from the agent-owned root without a host path", async () => { const rootDir = temporaryRoot(); const dataRoot = join(rootDir, "infra", "docker", "data"); - mkdirSync(join(dataRoot, "osm"), { recursive: true }); - mkdirSync(join(dataRoot, "valhalla"), { recursive: true }); + mkdirSync(join(dataRoot, "osm"), { recursive: true, mode: 0o700 }); + mkdirSync(join(dataRoot, "valhalla"), { recursive: true, mode: 0o700 }); writeFileSync(join(dataRoot, "osm", "germany-latest.osm.pbf"), "pbf", { mode: 0o600 }); const runtime = createUnavailableRuntime(); createAdministrativeRuntime(runtime, { @@ -975,7 +977,8 @@ describe("administrative release runtime", () => { if (args[0] === "pull") return `Digest: sha256:${String(attempt).repeat(64)}`; if (args[0] === "image") return `sha256:${"e".repeat(64)}`; if (args[0] === "create") return "a".repeat(64); - if (args[0] === "cp") writeFileSync(args.at(-1) as string, JSON.stringify(manifest())); + if (args[0] === "cp") + writeFileSync(args.at(-1) as string, JSON.stringify(manifest()), { mode: 0o644 }); return ""; }, }); @@ -1241,7 +1244,7 @@ describe("administrative release runtime", () => { if (joined.startsWith("create")) return "c".repeat(64); if (joined.startsWith("cp")) { const destination = args[2] as string; - writeFileSync(destination, stored(release)); + writeFileSync(destination, stored(release), { mode: 0o644 }); copied = true; // Yield so the sibling resolution interleaves here. await new Promise((resolve) => setTimeout(resolve, 5)); @@ -1533,6 +1536,7 @@ describe("administrative release runtime", () => { "docker-compose.generated.yml", ), "services:{}\n", + { mode: 0o644 }, ); const releaseId = "release-123"; const manifest = { @@ -1979,6 +1983,7 @@ describe("fixed CLI execution context", () => { writeFileSync( join(root, "cli.mjs"), `import{writeFileSync,existsSync}from'node:fs';import{setTimeout}from'node:timers/promises';if(process.env.OPENMAPX_CONTROL_DELEGATION||process.env.OPENMAPX_PARENT_EXECUTION)process.exit(2);const execution=JSON.parse(process.env.OPENMAPX_CONTROL_EXECUTION);writeFileSync(${JSON.stringify(ready)},JSON.stringify(execution));while(!existsSync(${JSON.stringify(release)}))await setTimeout(5);console.log('completed');`, + { mode: 0o644 }, ); vi.stubEnv("OPENMAPX_CONTROL_DELEGATION", "untrusted-inherited"); vi.stubEnv("OPENMAPX_PARENT_EXECUTION", "untrusted-parent"); @@ -1999,7 +2004,7 @@ describe("fixed CLI execution context", () => { expect(JSON.parse(readFileSync(ready, "utf8"))).toEqual(execution); await vi.advanceTimersByTimeAsync(31 * 60_000); expect(settled).toBe(false); - writeFileSync(release, "go"); + writeFileSync(release, "go", { mode: 0o644 }); await expect(completion).resolves.toBeUndefined(); await expect( createDefaultFixedCli(root, join(root, "agent.mjs"))([], { @@ -2009,7 +2014,7 @@ describe("fixed CLI execution context", () => { }), ).rejects.toMatchObject({ errorClass: "timeout" }); } finally { - writeFileSync(release, "go"); + writeFileSync(release, "go", { mode: 0o644 }); vi.useRealTimers(); vi.unstubAllEnvs(); await completion.catch(() => undefined); @@ -2113,9 +2118,9 @@ describe("fix1 legacy release recovery", () => { const transaction = join(directory, "transaction.json"); const record = JSON.parse(readFileSync(transaction, "utf8")); record.phase = phase; - writeFileSync(transaction, JSON.stringify(record)); + writeFileSync(transaction, JSON.stringify(record), { mode: 0o644 }); const overlay = releaseState(rootDir).overlay; - writeFileSync(overlay, "operator-state\n"); + writeFileSync(overlay, "operator-state\n", { mode: 0o644 }); const run = vi.fn(async () => undefined); const recovered = createDefaultReleaseEffects(rootDir, run, { verifyAppliedRelease: async () => true, @@ -2174,7 +2179,7 @@ it("fix1 recovery cannot publish after verification consumes the remaining deadl const path = join(directory, "transaction.json"); const record = JSON.parse(readFileSync(path, "utf8")); record.phase = "services_applied"; - writeFileSync(path, JSON.stringify(record)); + writeFileSync(path, JSON.stringify(record), { mode: 0o644 }); const recovered = createDefaultReleaseEffects(rootDir, async () => undefined, { verifyAppliedRelease: async () => { vi.setSystemTime(Date.now() + 2000); diff --git a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts index 9b0bf3db3..08f8c9d6e 100644 --- a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts +++ b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts @@ -95,14 +95,14 @@ async function fixture() { }); cleanups.push(() => app.close()); const source = join(host.root, "portable-source"); - mkdirSync(join(source, "osrm-graph"), { recursive: true }); + mkdirSync(join(source, "osrm-graph"), { mode: 0o755, recursive: true }); for (const name of [ "region.osrm", "region.osrm.partition", "region.osrm.cells", "region.osrm.mldgr", ]) - writeFileSync(join(source, "osrm-graph", name), `synthetic ${name}`); + writeFileSync(join(source, "osrm-graph", name), `synthetic ${name}`, { mode: 0o644 }); const chunks: Buffer[] = []; await writeBuildBundle({ dataRoot: source, @@ -376,8 +376,8 @@ it("blocks a live consumer and refuses a changed configuration after preview wit it("reconciles the original prepared publication after exact foreign-inode repair and server reassembly", async () => { const f = await fixture(), old = join(f.infraDir, "data/osrm-graph"); - mkdirSync(old); - writeFileSync(join(old, "old"), "old graph"); + mkdirSync(old, { mode: 0o755 }); + writeFileSync(join(old, "old"), "old graph", { mode: 0o644 }); const oldInode = fs.statSync(old).ino; const preview = await f.preview(), operation = { @@ -401,7 +401,7 @@ it("reconciles the original prepared publication after exact foreign-inode repai native.statSync(from).ino === newInode ) { replaced = true; - native.mkdirSync(to); + native.mkdirSync(to, { mode: 0o755 }); throw Error("Named foreign live-directory replacement"); } return native.renameSync(from, to); diff --git a/apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts b/apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts index ece43b821..22feec10e 100644 --- a/apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts +++ b/apps/ops-agent/src/artifact-transfer/integration-consumer.test.ts @@ -33,8 +33,8 @@ it("validates real uploaded bytes only under the exact retained integration exec const root = mkdtempSync(join(tmpdir(), "openmapx-stage-consumer-")); cleanup.push(() => rmSync(root, { recursive: true, force: true })); const infra = join(root, "infra"); - mkdirSync(join(infra, "data"), { recursive: true }); - mkdirSync(join(root, "services")); + mkdirSync(join(infra, "data"), { recursive: true, mode: 0o700 }); + mkdirSync(join(root, "services"), { mode: 0o755 }); await control.provisionControlInstallation(infra); const installation = control.openControlInstallation({ infraDir: infra, diff --git a/apps/ops-agent/src/artifact-transfer/lifecycle.test.ts b/apps/ops-agent/src/artifact-transfer/lifecycle.test.ts index 5fecf309b..ea3826875 100644 --- a/apps/ops-agent/src/artifact-transfer/lifecycle.test.ts +++ b/apps/ops-agent/src/artifact-transfer/lifecycle.test.ts @@ -39,6 +39,7 @@ it("runs expiry cleanup at startup and periodically, then stops observation on c const second = await create(); now = Date.parse(second.expiresAt) + 1; await vi.advanceTimersByTimeAsync(15 * 60000); + await observer.mock.results[1]?.value; expect(await store.reservedBytes()).toBe(0); expect(observer).toHaveBeenCalledTimes(2); await app.close(); diff --git a/apps/ops-agent/src/artifact-transfer/motis.test.ts b/apps/ops-agent/src/artifact-transfer/motis.test.ts index 45666a477..1bec4e92d 100644 --- a/apps/ops-agent/src/artifact-transfer/motis.test.ts +++ b/apps/ops-agent/src/artifact-transfer/motis.test.ts @@ -55,10 +55,10 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { data = join(infraDir, "data"), catalog = join(data, ".transitous-catalog"), atlas = join(catalog, "transitland-atlas"); - mkdirSync(join(catalog, "feeds"), { recursive: true }); - mkdirSync(join(catalog, "src")); - mkdirSync(join(catalog, "configs/motis"), { recursive: true }); - mkdirSync(join(atlas, "feeds"), { recursive: true }); + mkdirSync(join(catalog, "feeds"), { mode: 0o755, recursive: true }); + mkdirSync(join(catalog, "src"), { mode: 0o755 }); + mkdirSync(join(catalog, "configs/motis"), { mode: 0o755, recursive: true }); + mkdirSync(join(atlas, "feeds"), { mode: 0o755, recursive: true }); git(atlas, ["init"]); // Deliberately different filename and id: identity is the parsed pinned DMFR id. writeFileSync( @@ -75,6 +75,7 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { }, ], }), + { mode: 0o644 }, ); git(atlas, ["add", "."]); git(atlas, ["commit", "-m", "Synthetic Atlas"]); @@ -85,6 +86,7 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { JSON.stringify({ sources: [{ name: "demo", type: "transitland-atlas", "transitland-atlas-id": "f-demo" }], }), + { mode: 0o644 }, ); const approvedTemplate = actualPinnedTemplate ? readFileSync( @@ -95,8 +97,10 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { "utf8", ) : "server:\n port: 8080\ntimetable:\n datasets: {}\n"; - writeFileSync(join(catalog, "configs/motis/config.yml"), approvedTemplate); - writeFileSync(join(catalog, "src/generate-motis-config.py"), "# synthetic generator marker\n"); + writeFileSync(join(catalog, "configs/motis/config.yml"), approvedTemplate, { mode: 0o644 }); + writeFileSync(join(catalog, "src/generate-motis-config.py"), "# synthetic generator marker\n", { + mode: 0o644, + }); git(catalog, ["add", "feeds", "configs", "src"]); git(catalog, ["update-index", "--add", "--cacheinfo", `160000,${atlasCommit},transitland-atlas`]); git(catalog, ["commit", "-m", "Synthetic catalog"]); @@ -140,11 +144,12 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { { mode: 0o600 }, ); const out = join(data, "gtfs"); - mkdirSync(out); + mkdirSync(out, { mode: 0o755 }); symlinkSync(out, join(catalog, "out")); writeFileSync( join(out, "config.yml"), "server:\n port: 8080\ntimetable:\n datasets:\n de-demo:\n path: de_demo.gtfs.zip\n rt:\n - url: https://rt.triptix.tech/feed/de-demo-0\n protocol: gtfsrt\n", + { mode: 0o644 }, ); if (actualPinnedTemplate) { const generated = approvedTemplate @@ -154,12 +159,12 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { ) .replace(/^ {2}feeds:.*$/m, " feeds: {}") .replace(/^gbfs:$/m, "gbfs:\n proxy: https://rt.triptix.tech"); - writeFileSync(join(out, "config.yml"), generated); - writeFileSync(join(out, "planet-latest.osm.pbf"), "synthetic OSM input"); + writeFileSync(join(out, "config.yml"), generated, { mode: 0o644 }); + writeFileSync(join(out, "planet-latest.osm.pbf"), "synthetic OSM input", { mode: 0o644 }); } const archive = "synthetic timetable"; - writeFileSync(join(out, "de_demo.gtfs.zip"), archive); - writeFileSync(join(out, "license.json"), "[]"); + writeFileSync(join(out, "de_demo.gtfs.zip"), archive, { mode: 0o644 }); + writeFileSync(join(out, "license.json"), "[]", { mode: 0o644 }); writeFileSync( join(out, "transit-source-manifest.json"), JSON.stringify({ @@ -183,9 +188,10 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { }, ], }), + { mode: 0o644 }, ); const sourceKeys = join(root, "source-keys.json"); - writeFileSync(sourceKeys, JSON.stringify({ "de/demo": sourceKey })); + writeFileSync(sourceKeys, JSON.stringify({ "de/demo": sourceKey }), { mode: 0o600 }); const ctx = buildJobContext({ dataDir: data, store: new StateStore(data), @@ -202,6 +208,7 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { ...(!publicSource ? { headers: { "X-Key": sourceKey } } : {}), }, }), + { mode: 0o644 }, ); }, }); @@ -212,7 +219,7 @@ async function fixture(publicSource = false, actualPinnedTemplate = false) { expect(assembled.status).toBe("ok"); expect(assembled.artifacts?.portableRecipeWritten).toBe(true); const stagingDir = join(root, "prepared-motis-live"); - mkdirSync(stagingDir); + mkdirSync(stagingDir, { mode: 0o755 }); for (const name of [ "license.json", "de_demo.gtfs.zip", @@ -280,6 +287,7 @@ it("uses pinned Git objects even when the catalog working-tree endpoint is repla JSON.stringify({ sources: [{ name: "demo", type: "url", spec: "gtfs-rt", url: "https://attacker.example" }], }), + { mode: 0o644 }, ); const preview = await inspectMotisPortableDestination(f); await materializeMotisPortableDestination({ @@ -315,7 +323,7 @@ it("reports host-owned and missing target credentials as explicit prerequisites" }), ).rejects.toThrow("host-owned"); chmodSync(f.secret, 0o600); - writeFileSync(f.secret, "{}"); + writeFileSync(f.secret, "{}", { mode: 0o644 }); chmodSync(f.secret, 0o444); await expect(inspectMotisPortableDestination(f)).rejects.toThrow("missing feed credentials"); }, 30000); @@ -385,9 +393,9 @@ it("imports an actual portable candidate through the production agent with desti }); const portable = join(source.rootDir, "export"), live = join(portable, "motis/live"); - mkdirSync(join(live, "data"), { recursive: true }); + mkdirSync(join(live, "data"), { mode: 0o755, recursive: true }); cpSync(source.sourceStaging, live, { recursive: true }); - writeFileSync(join(live, "data/tt.bin"), "synthetic compiled timetable"); + writeFileSync(join(live, "data/tt.bin"), "synthetic compiled timetable", { mode: 0o644 }); cpSync(join(source.sourceStaging, ".openmapx-feed-proxy"), join(portable, "motis-feed-proxy"), { recursive: true, }); diff --git a/apps/ops-agent/src/artifact-transfer/prepared-consumer.test.ts b/apps/ops-agent/src/artifact-transfer/prepared-consumer.test.ts index d68d43b00..6824bea7c 100644 --- a/apps/ops-agent/src/artifact-transfer/prepared-consumer.test.ts +++ b/apps/ops-agent/src/artifact-transfer/prepared-consumer.test.ts @@ -23,7 +23,7 @@ async function fixture() { const root = mkdtempSync(join(tmpdir(), "omx-prepared-consumer-")); cleanups.push(() => rmSync(root, { recursive: true, force: true })); const infra = join(root, "infra"); - mkdirSync(join(infra, "data"), { recursive: true }); + mkdirSync(join(infra, "data"), { recursive: true, mode: 0o700 }); await control.provisionControlInstallation(infra); const installation = control.openControlInstallation({ infraDir: infra, @@ -57,9 +57,9 @@ async function fixture() { ], }; const source = join(root, "source"); - mkdirSync(join(source, "osrm-graph"), { recursive: true }); + mkdirSync(join(source, "osrm-graph"), { mode: 0o755, recursive: true }); const fs = await import("node:fs"); - fs.writeFileSync(join(source, "osrm-graph/region.osrm"), payload); + fs.writeFileSync(join(source, "osrm-graph/region.osrm"), payload, { mode: 0o644 }); const chunks: Buffer[] = []; await control.writeBuildBundle({ dataRoot: source, diff --git a/apps/ops-agent/src/artifact-transfer/test-fixtures/validation-recovery.mjs b/apps/ops-agent/src/artifact-transfer/test-fixtures/validation-recovery.mjs index 125dbf504..c81b05f29 100644 --- a/apps/ops-agent/src/artifact-transfer/test-fixtures/validation-recovery.mjs +++ b/apps/ops-agent/src/artifact-transfer/test-fixtures/validation-recovery.mjs @@ -26,8 +26,8 @@ try { const infra=${JSON.stringify(infra)},root=${JSON.stringify(root)},options=${JSON.stringify(options)}; await control.provisionControlInstallation(infra); const installation=control.openControlInstallation({infraDir:infra,role:'ops-agent',serviceIds:['osrm']}), store=new ArtifactTransferStore(options); - fs.mkdirSync(root+'/source/osrm-graph',{recursive:true}); - for(const name of ['region.osrm','region.osrm.partition','region.osrm.cells','region.osrm.mldgr'])fs.writeFileSync(root+'/source/osrm-graph/'+name,'synthetic '+name); + fs.mkdirSync(root+'/source/osrm-graph',{recursive:true,mode:0o755}); + for(const name of ['region.osrm','region.osrm.partition','region.osrm.cells','region.osrm.mldgr'])fs.writeFileSync(root+'/source/osrm-graph/'+name,'synthetic '+name,{mode:0o644}); const chunks=[];await control.writeBuildBundle({dataRoot:root+'/source',serviceId:'osrm',record:{schemaVersion:1,service:'osrm',builtAt:new Date().toISOString(),runtimeImages:{osrm:'example/osrm:1'},toolImages:{}},target:new Writable({write(chunk,e,done){chunks.push(Buffer.from(chunk));done();}}),limits:control.DEFAULT_ARTIFACT_ARCHIVE_LIMITS,signal:new AbortController().signal}); const bytes=Buffer.concat(chunks),stage=await store.createStage({ownerId:'native-admin',purpose:'prepared-artifact',direction:'upload',declaredBytes:bytes.length,operationKey:'opk1_'+randomUUID().replaceAll('-','')}); await store.upload({stageId:stage.stageId,ownerId:'native-admin',purpose:stage.purpose,attemptId:randomUUID(),deadlineAt:Date.now()+60000,body:new ReadableStream({start(c){c.enqueue(bytes);c.close();}})}); diff --git a/apps/ops-agent/src/catalog-pins.test.ts b/apps/ops-agent/src/catalog-pins.test.ts index c9234fa37..d7acbdb8b 100644 --- a/apps/ops-agent/src/catalog-pins.test.ts +++ b/apps/ops-agent/src/catalog-pins.test.ts @@ -42,8 +42,8 @@ it("serializes the largest legal paired pins through the actual HTTP server/clie lockedAt: t.lockedAt, lockedBy: t.lockedBy, }; - writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(t)); - writeFileSync(join(infra, "gbfs-catalog.lock.json"), JSON.stringify(g)); + writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(t), { mode: 0o600 }); + writeFileSync(join(infra, "gbfs-catalog.lock.json"), JSON.stringify(g), { mode: 0o600 }); await provisionControlInstallation(infra); const installation = openControlInstallation({ infraDir: infra, diff --git a/apps/ops-agent/src/configuration-writer-concurrency.test.ts b/apps/ops-agent/src/configuration-writer-concurrency.test.ts index c0e8bd822..658ae305d 100644 --- a/apps/ops-agent/src/configuration-writer-concurrency.test.ts +++ b/apps/ops-agent/src/configuration-writer-concurrency.test.ts @@ -21,15 +21,15 @@ afterEach(async () => { function fixture() { const root = mkdtempSync(join(tmpdir(), "openmapx-writers-")); roots.push(root); - mkdirSync(join(root, "infra/docker"), { recursive: true }); + mkdirSync(join(root, "infra/docker"), { recursive: true, mode: 0o700 }); // The deployment's user, which only a host render writes into .env. writeFileSync( join(root, "infra/docker/.env"), `UID=${process.geteuid?.() ?? 0}\nGID=${process.getegid?.() ?? 0}\n`, { mode: 0o600 }, ); - mkdirSync(join(root, "services/alpha"), { recursive: true }); - writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(join(root, "services/alpha"), { mode: 0o755, recursive: true }); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n", { mode: 0o644 }); writeFileSync( join(root, "services/alpha/service.json"), JSON.stringify({ @@ -40,6 +40,7 @@ function fixture() { container: { image: "example/alpha", tag: "1" }, configSchema: { type: "object", properties: { label: { type: "string" } } }, }), + { mode: 0o644 }, ); const selection = coreServices.serviceSelectionPath(join(root, "infra/docker")); mkdirSync(dirname(selection), { recursive: true, mode: 0o700 }); diff --git a/apps/ops-agent/src/core-service-runtime.test.ts b/apps/ops-agent/src/core-service-runtime.test.ts index e8854457d..f58369511 100644 --- a/apps/ops-agent/src/core-service-runtime.test.ts +++ b/apps/ops-agent/src/core-service-runtime.test.ts @@ -33,7 +33,7 @@ describe("production core restart workflow", () => { async (serviceId, fail) => { const host = await ownedServiceHost(); roots.push(host.cleanupRoot); - mkdirSync(join(host.root, "services/traefik/config"), { recursive: true }); + mkdirSync(join(host.root, "services/traefik/config"), { mode: 0o755, recursive: true }); cpSync( join(import.meta.dirname, "../../../services/traefik/config"), join(host.root, "services/traefik/config"), diff --git a/apps/ops-agent/src/data-inventory.test.ts b/apps/ops-agent/src/data-inventory.test.ts index 93c46be1d..01bec04f6 100644 --- a/apps/ops-agent/src/data-inventory.test.ts +++ b/apps/ops-agent/src/data-inventory.test.ts @@ -13,13 +13,13 @@ function fixture() { const root = mkdtempSync(join(tmpdir(), "omx-data-inventory-")); roots.push(root); const data = join(root, "infra/docker/data"); - mkdirSync(join(data, "osm"), { recursive: true }); + mkdirSync(join(data, "osm"), { mode: 0o755, recursive: true }); return { root, data }; } it("pages every extract with stable IDs and does not call unrecorded bytes verified", async () => { const { root, data } = fixture(); - writeFileSync(join(data, "osm/z.osm.pbf"), "z"); - writeFileSync(join(data, "osm/a.osm.pbf"), "aa"); + writeFileSync(join(data, "osm/z.osm.pbf"), "z", { mode: 0o644 }); + writeFileSync(join(data, "osm/a.osm.pbf"), "aa", { mode: 0o644 }); const first = await inspectDataInventory(root, { osmLimit: 1, osmOffset: 0 }), second = await inspectDataInventory(root, { osmLimit: 1, osmOffset: 1 }); expect(first.osm).toMatchObject({ @@ -43,7 +43,7 @@ it("reports empty prepared directories incomplete and database-backed stores unk "photon", "overpass", ]) - mkdirSync(join(data, p), { recursive: true }); + mkdirSync(join(data, p), { mode: 0o755, recursive: true }); const inventory = await inspectDataInventory(root); for (const target of ["osrm", "otp", "tiles", "motis"]) expect(inventory.builds.find((x) => x.target === target)).toMatchObject({ @@ -58,9 +58,9 @@ it("reports empty prepared directories incomplete and database-backed stores unk }); it("reports legacy build records as unknown rather than verified readiness", async () => { const { root, data } = fixture(); - mkdirSync(join(data, "otp-graph")); - writeFileSync(join(data, "otp-graph/graph.obj"), "graph"); - mkdirSync(join(data, ".openmapx-builds")); + mkdirSync(join(data, "otp-graph"), { mode: 0o755 }); + writeFileSync(join(data, "otp-graph/graph.obj"), "graph", { mode: 0o644 }); + mkdirSync(join(data, ".openmapx-builds"), { mode: 0o755 }); writeFileSync( join(data, ".openmapx-builds/otp.json"), JSON.stringify({ @@ -70,6 +70,7 @@ it("reports legacy build records as unknown rather than verified readiness", asy runtimeImages: { otp: "example.test/otp@sha256:" + "a".repeat(64) }, toolImages: {}, }), + { mode: 0o644 }, ); expect((await inspectDataInventory(root)).builds.find((x) => x.target === "otp")).toMatchObject({ readiness: "unknown", @@ -80,7 +81,7 @@ it("reports legacy build records as unknown rather than verified readiness", asy it("surfaces unsafe OSM directories as unavailable, without exposing host paths", async () => { const { root, data } = fixture(); renameSync(join(data, "osm"), join(data, "saved")); - writeFileSync(join(data, "osm"), "not a directory"); + writeFileSync(join(data, "osm"), "not a directory", { mode: 0o644 }); const result = await inspectDataInventory(root); expect(result.osm).toMatchObject({ unavailable: true, items: [] }); expect(JSON.stringify(result)).not.toContain(root); @@ -92,16 +93,16 @@ it.each(["output", "source"] as const)( const { capturePreparedBuildEvidence, hashDataFile, osmArtifactId } = await import( "@openmapx/core/build-artifacts" ); - mkdirSync(join(data, "osrm-graph")); + mkdirSync(join(data, "osrm-graph"), { mode: 0o755 }); for (const file of [ "region.osrm", "region.osrm.partition", "region.osrm.cells", "region.osrm.mldgr", ]) - writeFileSync(join(data, "osrm-graph", file), "graph bytes"); + writeFileSync(join(data, "osrm-graph", file), "graph bytes", { mode: 0o644 }); const path = "osm/europe-germany.osm.pbf"; - writeFileSync(join(data, path), "pbf bytes"); + writeFileSync(join(data, path), "pbf bytes", { mode: 0o644 }); const source = await hashDataFile(join(data, path)); const record = await capturePreparedBuildEvidence({ dataRoot: data, @@ -134,7 +135,9 @@ it.each(["output", "source"] as const)( }, ], }); - writeFileSync(join(data, ".openmapx-builds/osrm.json"), JSON.stringify(record)); + writeFileSync(join(data, ".openmapx-builds/osrm.json"), JSON.stringify(record), { + mode: 0o644, + }); expect( (await inspectDataInventory(root)).builds.find((x) => x.target === "osrm"), ).toMatchObject({ @@ -145,7 +148,7 @@ it.each(["output", "source"] as const)( }); const replaced = join(data, changed === "output" ? "osrm-graph/region.osrm" : path), bytes = changed === "output" ? "other bytes" : "new bytes"; - writeFileSync(replaced + ".replacement", bytes); + writeFileSync(replaced + ".replacement", bytes, { mode: 0o644 }); renameSync(replaced + ".replacement", replaced); expect( (await inspectDataInventory(root)).builds.find((x) => x.target === "osrm"), @@ -158,7 +161,7 @@ it("invalidates a download fingerprint after same-size inode replacement", async const { lstatSync } = await import("node:fs"); const name = "europe-germany.osm.pbf", path = join(data, "osm", name); - writeFileSync(path, "before"); + writeFileSync(path, "before", { mode: 0o644 }); writeFileSync( path + ".openmapx.json", JSON.stringify({ @@ -168,12 +171,13 @@ it("invalidates a download fingerprint after same-size inode replacement", async sha256: "c".repeat(64), ...fileIdentity(lstatSync(path, { bigint: true })), }), + { mode: 0o644 }, ); expect((await inspectDataInventory(root)).osm.items[0]).toMatchObject({ fingerprintState: "verified", sourceFingerprint: "c".repeat(64), }); - writeFileSync(path + ".replacement", "after!"); + writeFileSync(path + ".replacement", "after!", { mode: 0o644 }); renameSync(path + ".replacement", path); expect((await inspectDataInventory(root)).osm.items[0]).toMatchObject({ fingerprintState: "stale", @@ -183,7 +187,7 @@ it("invalidates a download fingerprint after same-size inode replacement", async it("does not silently truncate extracts at the previous first page", async () => { const { root, data } = fixture(); for (let i = 0; i < 501; i++) - writeFileSync(join(data, "osm", `${String(i).padStart(4, "0")}.osm.pbf`), "x"); + writeFileSync(join(data, "osm", `${String(i).padStart(4, "0")}.osm.pbf`), "x", { mode: 0o644 }); const page = await inspectDataInventory(root, { osmOffset: 500, osmLimit: 50 }); expect(page.osm).toMatchObject({ total: 501, offset: 500, unavailable: false }); expect(page.osm.items.map((x) => x.filename)).toEqual(["0500.osm.pbf"]); @@ -194,12 +198,13 @@ it("checks changed source evidence even outside the displayed OSM page", async ( "@openmapx/core/build-artifacts" ); const { lstatSync } = await import("node:fs"); - mkdirSync(join(data, "otp-graph")); - writeFileSync(join(data, "otp-graph/graph.obj"), "graph"); - for (let i = 0; i < 50; i++) writeFileSync(join(data, "osm", `a${i}.osm.pbf`), "x"); + mkdirSync(join(data, "otp-graph"), { mode: 0o755 }); + writeFileSync(join(data, "otp-graph/graph.obj"), "graph", { mode: 0o644 }); + for (let i = 0; i < 50; i++) + writeFileSync(join(data, "osm", `a${i}.osm.pbf`), "x", { mode: 0o644 }); const name = "zz.osm.pbf", path = join(data, "osm", name); - writeFileSync(path, "new source"); + writeFileSync(path, "new source", { mode: 0o644 }); writeFileSync( path + ".openmapx.json", JSON.stringify({ @@ -209,6 +214,7 @@ it("checks changed source evidence even outside the displayed OSM page", async ( sha256: "d".repeat(64), ...fileIdentity(lstatSync(path, { bigint: true })), }), + { mode: 0o644 }, ); const record = await capturePreparedBuildEvidence({ dataRoot: data, @@ -223,7 +229,7 @@ it("checks changed source evidence even outside the displayed OSM page", async ( artifactFingerprint: "0".repeat(64), }, }); - writeFileSync(join(data, ".openmapx-builds/otp.json"), JSON.stringify(record)); + writeFileSync(join(data, ".openmapx-builds/otp.json"), JSON.stringify(record), { mode: 0o644 }); const inventory = await inspectDataInventory(root); expect(inventory.osm.items).toHaveLength(50); expect(inventory.osm.items.some((item) => item.filename === name)).toBe(false); @@ -235,7 +241,7 @@ it("checks changed source evidence even outside the displayed OSM page", async ( it("reports a disappeared listed extract as unavailable, not an empty successful scan", async () => { const { root, data } = fixture(), file = join(data, "osm/vanishing.osm.pbf"); - writeFileSync(file, "bytes"); + writeFileSync(file, "bytes", { mode: 0o644 }); const { AnchoredDirectory } = await import( "../../../packages/core/src/build-artifacts/filesystem" ); @@ -259,7 +265,7 @@ it("reports a disappeared listed extract as unavailable, not an empty successful it("preserves the legacy top-level extract fallback when managed OSM storage is absent", async () => { const { root, data } = fixture(); rmSync(join(data, "osm"), { recursive: true }); - writeFileSync(join(data, "legacy-latest.osm.pbf"), "legacy"); + writeFileSync(join(data, "legacy-latest.osm.pbf"), "legacy", { mode: 0o644 }); expect((await inspectDataInventory(root)).osm).toMatchObject({ unavailable: false, total: 1, diff --git a/apps/ops-agent/src/data-transit-resource-source.test.ts b/apps/ops-agent/src/data-transit-resource-source.test.ts index 6f2a1be40..50d80433a 100644 --- a/apps/ops-agent/src/data-transit-resource-source.test.ts +++ b/apps/ops-agent/src/data-transit-resource-source.test.ts @@ -36,9 +36,9 @@ async function fixture( const root = mkdtempSync(join(tmpdir(), "omx-real-candidate-policy-")), data = join(root, "data"), proxy = join(data, "motis-feed-proxy"); - mkdirSync(join(proxy, "conf"), { recursive: true }); - writeFileSync(join(proxy, "conf/default.conf"), "server { listen 80; }\n"); - writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(proxy, "conf"), { recursive: true, mode: 0o700 }); + writeFileSync(join(proxy, "conf/default.conf"), "server { listen 80; }\n", { mode: 0o600 }); + writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n", { mode: 0o600 }); applyHardlinkPlan( [ { @@ -139,7 +139,7 @@ async function fixture( calls++; if (args[2] === "/bin/sh") { const bin = join(root, "fixture-bin"); - mkdirSync(bin, { recursive: true }); + mkdirSync(bin, { mode: 0o755, recursive: true }); writeFileSync(join(bin, "nginx"), `#!/bin/sh\nexit ${fault === "validation" ? 1 : 0}\n`, { mode: 0o700, }); @@ -155,7 +155,9 @@ async function fixture( options, ); if (replaceAfterValidation) - writeFileSync(join(proxy, "conf/default.conf"), "replaced after the resource claim\n"); + writeFileSync(join(proxy, "conf/default.conf"), "replaced after the resource claim\n", { + mode: 0o644, + }); if (fault === "transport") throw Error("Lost Docker transport after validation"); if (fault === "marker") return { ...completed, stdout: completed.stdout.replace(/:[0-9]+\n$/, ":256\n") }; @@ -266,7 +268,9 @@ it.each(["validation", "validation-replaced", "transport", "reload", "marker"] a }; // An original failed invocation remains known after a later legitimate // restore replaces the active bytes. Observation grants no new effect. - writeFileSync(join(f.proxy, "conf/default.conf"), "restored previous proxy bytes\n"); + writeFileSync(join(f.proxy, "conf/default.conf"), "restored previous proxy bytes\n", { + mode: 0o644, + }); expect(() => control.commitFeedProxyValidationFailure(authority, input)).not.toThrow(); expect(() => control.commitFeedProxyValidationFailure(authority, { ...input, validationExitCode: 2 }), @@ -412,21 +416,23 @@ it.each([ const path = join(f.proxy, "conf/default.conf"), bytes = readFileSync(path); renameSync(path, `${path}.original`); - writeFileSync(path, bytes); + writeFileSync(path, bytes, { mode: 0o644 }); } if (mode === "bytes") - writeFileSync(join(f.proxy, "feed-proxy-vars.json"), '{"foreign":true}\n'); + writeFileSync(join(f.proxy, "feed-proxy-vars.json"), '{"foreign":true}\n', { mode: 0o644 }); if (mode === "directory") { renameSync(f.proxy, `${f.proxy}.original`); - mkdirSync(join(f.proxy, "conf"), { recursive: true }); - writeFileSync(join(f.proxy, "conf/default.conf"), "server { listen 80; }\n"); - writeFileSync(join(f.proxy, "feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(f.proxy, "conf"), { mode: 0o755, recursive: true }); + writeFileSync(join(f.proxy, "conf/default.conf"), "server { listen 80; }\n", { + mode: 0o644, + }); + writeFileSync(join(f.proxy, "feed-proxy-vars.json"), "{}\n", { mode: 0o644 }); } if (mode === "consumer-inode" || mode === "consumer-symlink") { const path = join(f.proxy, "motis-feed-proxy-config/default.conf"), bytes = readFileSync(path); rmSync(path); - if (mode === "consumer-inode") writeFileSync(path, bytes); + if (mode === "consumer-inode") writeFileSync(path, bytes, { mode: 0o644 }); else (await import("node:fs")).symlinkSync(join(f.proxy, "conf/default.conf"), path); } if (mode === "canceled") { @@ -472,7 +478,9 @@ it("rechecks the retained snapshot after nginx validation and before reload", as }); if (accepted.execution !== "async") throw Error("Expected actual asynchronous agent admission"); await expect - .poll(async () => (await f.client.status(f.operation.kind, accepted.operationId)).state) + .poll(async () => (await f.client.status(f.operation.kind, accepted.operationId)).state, { + timeout: 15000, + }) .toBe("failed"); expect(f.calls()).toBe(1); } finally { diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index e2f63f80f..a0149200b 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -7,6 +7,7 @@ import { mkdtempSync, readdirSync, readFileSync, + realpathSync, rmSync, symlinkSync, writeFileSync, @@ -16,6 +17,7 @@ import { delimiter, join } from "node:path"; import type { OpsOperation } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { parse as parseYaml } from "yaml"; import { testExecution } from "./__fixtures__/execution"; import { createDockerRuntime, @@ -33,7 +35,7 @@ async function preparedStack(ids = ["motis", "postgis", "redis"]) { const root = mkdtempSync(join(tmpdir(), "omx-docker-prepared-")), infraDir = join(root, "infra/docker"); preparedRoots.push(root); - mkdirSync(infraDir, { recursive: true }); + mkdirSync(infraDir, { mode: 0o755, recursive: true }); const loaded: services.LoadedService[] = ids.map((id) => ({ manifest: { id, @@ -67,6 +69,32 @@ async function preparedStack(ids = ["motis", "postgis", "redis"]) { }; } +/** The unit adapter boundary returns Docker's config read from the real generated inputs. */ +function readPreparedDockerConfig( + stack: { composePath: string; composeReleasePath: string }, + file: string, + args: readonly string[], +) { + expect(file).toBe("docker"); + expect(args.slice(-3)).toEqual(["config", "--format", "json"]); + const serviceDefinitions: Record = {}; + for (let index = 0; index < args.length; index++) { + if (args[index] !== "-f") continue; + const path = args[++index]; + expect( + [stack.composePath, stack.composeReleasePath] + .filter(existsSync) + .map((path) => realpathSync(path)), + ).toContain(realpathSync(path!)); + const configuration = parseYaml(readFileSync(path!, "utf8")) as { + services: Record; + }; + Object.assign(serviceDefinitions, configuration.services); + } + expect(Object.keys(serviceDefinitions).length).toBeGreaterThan(0); + return { stdout: JSON.stringify({ services: serviceDefinitions }), stderr: "" }; +} + const ownedSnapshot = (serviceId = "redis") => ({ serviceId, containerId: "a".repeat(64), @@ -577,11 +605,11 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done const previous = await point(live); const candidate = await point(staging); const infra = join(root, `${layout}-${scenario}-infra`); - mkdirSync(infra); + mkdirSync(infra, { mode: 0o755 }); const compose = join(infra, "compose.yml"); const release = join(infra, "release.yml"); - writeFileSync(join(infra, ".env"), ""); - writeFileSync(release, "services: {}\n"); + writeFileSync(join(infra, ".env"), "", { mode: 0o644 }); + writeFileSync(release, "services: {}\n", { mode: 0o644 }); const service = (name: string, path: string) => ` ${name}: image: ${image} container_name: ${name} @@ -603,6 +631,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done writeFileSync( compose, `name: omx-canary-${randomUUID()}\nservices:\n${service("motis", live)}${service("motis-staging", staging)}`, + { mode: 0o644 }, ); const prefix = [ "compose", @@ -920,7 +949,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done ownedServices, execFile: async (file, args, options) => { calls.push([...args]); - if (args.includes("config")) return runContainedProcess(file, args, options); + if (args.includes("config")) return readPreparedDockerConfig(prepared.stack, file, args); return { stdout: "", stderr: "" }; }, }); @@ -978,7 +1007,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done fileExists: () => false, ownedServices, execFile: async (file, args, options) => { - if (args.includes("config")) return runContainedProcess(file, args, options); + if (args.includes("config")) return readPreparedDockerConfig(prepared.stack, file, args); expect(options.timeout).toBe(600_000); expect(options.signal.aborted).toBe(false); return { stdout: "", stderr: "" }; @@ -1035,7 +1064,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done fileExists: () => false, ownedServices, execFile: async (file, args, options) => { - if (args.includes("config")) return runContainedProcess(file, args, options); + if (args.includes("config")) return readPreparedDockerConfig(prepared.stack, file, args); return new Promise((resolve, reject) => { entered.resolve(); const deadline = setTimeout(() => { @@ -1082,15 +1111,15 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done it("maps lifecycle effects to fixed Compose argv without accepting caller argv or paths", async () => { const prepared = await preparedStack(); - writeFileSync(prepared.stack.composeReleasePath, "services: {}\n"); - writeFileSync(join(prepared.stack.infraDir, ".env"), ""); + writeFileSync(prepared.stack.composeReleasePath, "services: {}\n", { mode: 0o644 }); + writeFileSync(join(prepared.stack.infraDir, ".env"), "", { mode: 0o644 }); const calls: Array<{ file: string; args: readonly string[] }> = []; const runtime = createDockerRuntime({ stack: prepared.stack, fileExists: () => true, execFile: async (file, args, options) => { calls.push({ file, args }); - if (args.includes("config")) return runContainedProcess(file, args, options); + if (args.includes("config")) return readPreparedDockerConfig(prepared.stack, file, args); return { stdout: "ok\n", stderr: "" }; }, }); @@ -1468,9 +1497,9 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done const root = mkdtempSync(join(tmpdir(), "valhalla-maintenance-planted-")); const dataRoot = join(root, "data"); const shared = join(dataRoot, "valhalla", "osm-pbf"); - mkdirSync(shared, { recursive: true }); + mkdirSync(shared, { mode: 0o755, recursive: true }); const sentinel = join(root, "sentinel"); - writeFileSync(sentinel, "untouched"); + writeFileSync(sentinel, "untouched", { mode: 0o644 }); // The temporary name the agent used to write, and the manifest itself. symlinkSync(sentinel, join(shared, `traffic-generations.json.${"f".repeat(64)}.tmp`)); symlinkSync(sentinel, join(shared, "traffic-generations.json")); @@ -1523,8 +1552,8 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done const root = mkdtempSync(join(tmpdir(), "valhalla-maintenance-swapped-")); const dataRoot = join(root, "data"); const outside = join(root, "outside"); - mkdirSync(join(outside, "osm-pbf"), { recursive: true }); - mkdirSync(dataRoot); + mkdirSync(join(outside, "osm-pbf"), { mode: 0o755, recursive: true }); + mkdirSync(dataRoot, { mode: 0o755 }); symlinkSync(outside, join(dataRoot, "valhalla")); try { const runtime = createDockerRuntime({ @@ -1687,7 +1716,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done it("requires a coherent maintenance generation manifest before reporting ready", async () => { const dataRoot = mkdtempSync(join(tmpdir(), "valhalla-generation-inspect-")); const sharedDir = join(dataRoot, "valhalla", "osm-pbf"); - mkdirSync(sharedDir, { recursive: true }); + mkdirSync(sharedDir, { mode: 0o755, recursive: true }); const runtime = createDockerRuntime({ stack: { infraDir: "/trusted", @@ -1722,6 +1751,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done waysToEdgesGeneration: generation, completedAt: new Date().toISOString(), }), + { mode: 0o644 }, ); await expect( dispatchOpsOperation(runtime, { kind: "valhalla.traffic.inspect" }, context()), @@ -1733,7 +1763,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done it("reports a manifest the watchdog is re-checking as inconclusive, not as a rebuild", async () => { const dataRoot = mkdtempSync(join(tmpdir(), "valhalla-generation-unverified-")); const sharedDir = join(dataRoot, "valhalla", "osm-pbf"); - mkdirSync(sharedDir, { recursive: true }); + mkdirSync(sharedDir, { mode: 0o755, recursive: true }); const calls: string[][] = []; const runtime = createDockerRuntime({ stack: { @@ -1748,7 +1778,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done return { stdout: "100\n", stderr: "" }; }, }); - writeFileSync(join(sharedDir, "traffic-generations.unverified.json"), "{}"); + writeFileSync(join(sharedDir, "traffic-generations.unverified.json"), "{}", { mode: 0o644 }); await expect( dispatchOpsOperation(runtime, { kind: "valhalla.traffic.inspect" }, context()), @@ -2348,6 +2378,7 @@ it("streams the actual default fixed Docker human output without the JSON collec writeFileSync( join(root, "docker"), `#!${process.execPath}\nconst{writeFileSync,existsSync}=require('node:fs');console.log('phase-one');writeFileSync(${JSON.stringify(ready)},'ready');const timer=setInterval(()=>{if(existsSync(${JSON.stringify(release)})){clearInterval(timer);process.stdout.write('x'.repeat(2*1024*1024)+'\\nlast-diagnostic\\n');}},5);`, + { mode: 0o644 }, ); chmodSync(join(root, "docker"), 0o700); vi.stubEnv("PATH", root + delimiter + process.env.PATH); @@ -2374,11 +2405,11 @@ it("streams the actual default fixed Docker human output without the JSON collec try { await vi.waitFor(() => expect(existsSync(ready)).toBe(true), { timeout: 2000 }); await vi.waitFor(() => expect(seen).toContain("phase-one"), { timeout: 2000 }); - writeFileSync(release, "go"); + writeFileSync(release, "go", { mode: 0o644 }); await expect(completion).resolves.toEqual({ changed: true }); expect(seen).toContain("last-diagnostic"); } finally { - writeFileSync(release, "go"); + writeFileSync(release, "go", { mode: 0o644 }); await completion.catch(() => undefined); vi.unstubAllEnvs(); rmSync(root, { recursive: true, force: true }); diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index aaa240035..4989f2593 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -339,7 +339,7 @@ describe("assembled production service authority", () => { }); const bin = join(host.root, "fake-bin"), trace = join(host.root, "docker-trace.jsonl"); - mkdirSync(bin); + mkdirSync(bin, { mode: 0o755 }); const docker = join(bin, "docker"), realDocker = execFileSync("which", ["docker"], { encoding: "utf8" }).trim(); writeFileSync( @@ -477,7 +477,7 @@ describe("assembled production service authority", () => { compose.services.motis.depends_on = { "data-manager": { condition: "service_started", required: true }, }; - writeFileSync(host.firstCompose, JSON.stringify(compose)); + writeFileSync(host.firstCompose, JSON.stringify(compose), { mode: 0o644 }); mkdirSync(join(host.infraDir, "data/ops-agent"), { recursive: true, mode: 0o700 }); const installation = openControlInstallation({ infraDir: host.infraDir, @@ -929,6 +929,7 @@ describe("assembled lifecycle/removal exclusion", () => { writeFileSync( host.stack.composeReleasePath, "services:\n motis:\n depends_on:\n community-live:\n condition: service_started\n", + { mode: 0o644 }, ); const start = { kind: "service.start", diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index 01aa4bb5d..caba80df9 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -842,6 +842,7 @@ it.each([0, 1])( writeFileSync( join(root, "cli.mjs"), `for(const frame of ${JSON.stringify(frames)})for(const byte of Buffer.from(frame))process.stdout.write(Buffer.from([byte]));process.exitCode=${exitCode};`, + { mode: 0o644 }, ); const runtime = createUnavailableRuntime(); createAdministrativeRuntime(runtime, { diff --git a/apps/ops-agent/src/standalone-integration-runtime.test.ts b/apps/ops-agent/src/standalone-integration-runtime.test.ts index 4898ac35a..1b080fc59 100644 --- a/apps/ops-agent/src/standalone-integration-runtime.test.ts +++ b/apps/ops-agent/src/standalone-integration-runtime.test.ts @@ -41,8 +41,8 @@ for (const scenario of ["normal", "upload-mutated", "forged-completion"] as cons const root = mkdtempSync(join(tmpdir(), "openmapx-stage-consumer-")); cleanup.push(() => rmSync(root, { recursive: true, force: true })); const infra = join(root, "infra"); - mkdirSync(join(infra, "data"), { recursive: true }); - mkdirSync(join(root, "services")); + mkdirSync(join(infra, "data"), { recursive: true, mode: 0o700 }); + mkdirSync(join(root, "services"), { mode: 0o755 }); await control.provisionControlInstallation(infra); const installation = control.openControlInstallation({ infraDir: infra, @@ -185,16 +185,19 @@ for (const scenario of ["normal", "upload-mutated", "forged-completion"] as cons writeFileSync( inspectionPath, JSON.stringify({ ...envelope, body: JSON.stringify(forgedRecord) }), + { mode: 0o644 }, ); try { await expect( dispatchOpsOperation(runtime, installing, makeContext(installing)), ).rejects.toThrow(/authentication/); } finally { - writeFileSync(inspectionPath, originalBytes); + writeFileSync(inspectionPath, originalBytes, { mode: 0o644 }); } if (scenario === "upload-mutated") { - writeFileSync(join(root, "stages", stage.stageId, "content"), Buffer.from("changed")); + writeFileSync(join(root, "stages", stage.stageId, "content"), Buffer.from("changed"), { + mode: 0o644, + }); await expect( dispatchOpsOperation(runtime, installing, makeContext(installing)), ).rejects.toThrow(/content changed/); @@ -220,7 +223,7 @@ for (const scenario of ["normal", "upload-mutated", "forged-completion"] as cons if (!workspace) throw Error("Missing actual shared publication journal"); const path = join(staging, workspace, "publication.json"), record = JSON.parse(readFileSync(path, "utf8")); - writeFileSync(path, JSON.stringify({ ...record, outcome: "committed" })); + writeFileSync(path, JSON.stringify({ ...record, outcome: "committed" }), { mode: 0o644 }); rmSync(join(root, "custom_integrations", "upload-fixture"), { recursive: true }); await expect(dispatchOpsOperation(runtime, installing, installContext)).rejects.toThrow( /ENOENT|publication|changed|Published/i, @@ -360,7 +363,7 @@ for (const scenario of ["normal", "upload-mutated", "forged-completion"] as cons { const directory = join(root, "custom_integrations/upload-fixture"); - mkdirSync(directory); + mkdirSync(directory, { mode: 0o755 }); writeFileSync( join(directory, "manifest.json"), JSON.stringify({ @@ -371,6 +374,7 @@ for (const scenario of ["normal", "upload-mutated", "forged-completion"] as cons quality: "community", domains: ["map-overlay"], }), + { mode: 0o644 }, ); const checkedInventory = await dispatchOpsOperation( replacement, @@ -391,8 +395,10 @@ for (const scenario of ["normal", "upload-mutated", "forged-completion"] as cons allowed: true, }); renameSync(directory, join(root, "original-directory")); - mkdirSync(directory); - writeFileSync(join(directory, "foreign.txt"), "preserve this foreign replacement"); + mkdirSync(directory, { mode: 0o755 }); + writeFileSync(join(directory, "foreign.txt"), "preserve this foreign replacement", { + mode: 0o644, + }); const remove: OpsOperation = { kind: "integration.remove", transactionId, diff --git a/apps/ops-agent/src/transit-credentials-runtime.test.ts b/apps/ops-agent/src/transit-credentials-runtime.test.ts index be5fd84e2..df27b3ac1 100644 --- a/apps/ops-agent/src/transit-credentials-runtime.test.ts +++ b/apps/ops-agent/src/transit-credentials-runtime.test.ts @@ -50,13 +50,14 @@ async function fixture() { catalog = join(infraDir, "data/.transitous-catalog"), atlas = join(catalog, "transitland-atlas"), local = join(root, "services/motis/tools/transitous"); - mkdirSync(join(catalog, "feeds"), { recursive: true }); - mkdirSync(join(atlas, "feeds"), { recursive: true }); - mkdirSync(local, { recursive: true }); + mkdirSync(join(catalog, "feeds"), { mode: 0o755, recursive: true }); + mkdirSync(join(atlas, "feeds"), { mode: 0o755, recursive: true }); + mkdirSync(local, { mode: 0o755, recursive: true }); git(atlas, ["init"]); writeFileSync( join(atlas, "feeds/a.dmfr.json"), JSON.stringify({ feeds: [{ id: "auth", authorization: { type: "api-key" } }] }), + { mode: 0o644 }, ); git(atlas, ["add", "."]); git(atlas, ["commit", "-m", "Atlas"]); @@ -65,6 +66,7 @@ async function fixture() { writeFileSync( join(catalog, "feeds/de.json"), JSON.stringify({ sources: [{ name: "A", "transitland-atlas-id": "auth" }] }), + { mode: 0o644 }, ); git(catalog, ["add", "feeds"]); git(catalog, ["update-index", "--add", "--cacheinfo", `160000,${atlasSha},transitland-atlas`]); @@ -78,6 +80,7 @@ async function fixture() { lockedAt: "2026-10-23T00:00:00.000Z", lockedBy: "fixture", }), + { mode: 0o644 }, ); const gbfs = "b".repeat(40); writeFileSync( @@ -91,6 +94,7 @@ async function fixture() { lockedAt: "2026-10-23T00:00:00.000Z", lockedBy: "fixture", }), + { mode: 0o644 }, ); await provisionControlInstallation(infraDir); const installation = openControlInstallation({ diff --git a/apps/ops-agent/src/transitous-lock-runtime.test.ts b/apps/ops-agent/src/transitous-lock-runtime.test.ts index b6762e7eb..a3837a917 100644 --- a/apps/ops-agent/src/transitous-lock-runtime.test.ts +++ b/apps/ops-agent/src/transitous-lock-runtime.test.ts @@ -19,7 +19,7 @@ const REF = `main@${"a".repeat(40)}`; function root(): string { const directory = mkdtempSync(join(tmpdir(), "openmapx-lock-runtime-")); roots.push(directory); - mkdirSync(join(directory, "infra", "docker"), { recursive: true }); + mkdirSync(join(directory, "infra", "docker"), { mode: 0o755, recursive: true }); return directory; } @@ -198,6 +198,7 @@ describe("agent-owned Transitous catalog lock", () => { lockedBy: "operator", comment: "pinned", }), + { mode: 0o644 }, ); const commit = "b".repeat(40); writeFileSync( @@ -211,6 +212,7 @@ describe("agent-owned Transitous catalog lock", () => { lockedAt: "2026-01-01T00:00:00.000Z", lockedBy: "fixture", }), + { mode: 0o644 }, ); const infra = join(rootDir, "infra/docker"); mkdirSync(join(infra, "data"), { mode: 0o700 }); @@ -235,7 +237,7 @@ describe("agent-owned Transitous catalog lock", () => { it("rejects an oversized lock rather than loading it", async () => { const rootDir = root(); - writeFileSync(lockPath(rootDir), `{"ref":"${"x".repeat(300_000)}"}`); + writeFileSync(lockPath(rootDir), `{"ref":"${"x".repeat(300_000)}"}`, { mode: 0o644 }); await expect( dispatchOpsOperation(runtimeFor(rootDir), { kind: "transitousLock.inspect" }, context()), ).rejects.toThrow(/not wired/i); diff --git a/apps/ops-agent/src/trusted-config-ownership.test.ts b/apps/ops-agent/src/trusted-config-ownership.test.ts index 620a4d7e8..76b1830eb 100644 --- a/apps/ops-agent/src/trusted-config-ownership.test.ts +++ b/apps/ops-agent/src/trusted-config-ownership.test.ts @@ -55,9 +55,9 @@ function fixture() { const root = mkdtempSync(join(tmpdir(), "openmapx-ownership-")); roots.push(root); const infraDir = join(root, "infra/docker"); - mkdirSync(infraDir, { recursive: true }); + mkdirSync(infraDir, { recursive: true, mode: 0o700 }); // The whole checkout is mounted, as this release's manifest has it. - mkdirSync(join(root, "services")); + mkdirSync(join(root, "services"), { mode: 0o755 }); // The deployment's user, which only a host render writes into .env. writeFileSync( join(infraDir, ".env"), @@ -308,7 +308,7 @@ describe("trusted configuration ownership and outcomes", () => { const foreign = join(f.infraDir, coreServices.CONFIGURATION_DIRECTORY, ".writer", "foreign"); const result = await f.apply("a", undefined, { afterCommit: async () => { - writeFileSync(foreign, "preserve"); + writeFileSync(foreign, "preserve", { mode: 0o644 }); }, }); expect(result).toMatchObject({ diff --git a/apps/ops-agent/src/trusted-config-runtime.test.ts b/apps/ops-agent/src/trusted-config-runtime.test.ts index 890e3f531..72126fb2e 100644 --- a/apps/ops-agent/src/trusted-config-runtime.test.ts +++ b/apps/ops-agent/src/trusted-config-runtime.test.ts @@ -113,9 +113,9 @@ function fixture() { const root = mkdtempSync(join(tmpdir(), "openmapx-trusted-runtime-")); roots.push(root); const infraDir = join(root, "infra", "docker"); - mkdirSync(infraDir, { recursive: true }); + mkdirSync(infraDir, { recursive: true, mode: 0o700 }); // The whole checkout is mounted, as this release's manifest has it. - mkdirSync(join(root, "services"), { recursive: true }); + mkdirSync(join(root, "services"), { mode: 0o755, recursive: true }); // The deployment's user, which only a host render writes into .env. writeFileSync(join(infraDir, ".env"), deploymentUserEnv, { mode: 0o600 }); select(infraDir, ["alpha"]); @@ -1147,8 +1147,8 @@ describe("ops-agent trusted configuration runtime", () => { const paths = fixture(); // A built-in that mounts a file of the release. const directory = join(paths.root, "services", "alpha"); - mkdirSync(join(directory, "config"), { recursive: true }); - writeFileSync(join(directory, "config", "alpha.yml"), "release: true\n"); + mkdirSync(join(directory, "config"), { mode: 0o755, recursive: true }); + writeFileSync(join(directory, "config", "alpha.yml"), "release: true\n", { mode: 0o644 }); const alpha: services.LoadedService = { ...service("alpha"), manifest: { @@ -1159,7 +1159,7 @@ describe("ops-agent trusted configuration runtime", () => { }; const override = join(paths.infraDir, "overrides", "services", "alpha", "config", "alpha.yml"); mkdirSync(dirname(override), { recursive: true, mode: 0o755 }); - writeFileSync(override, "users: ['op:$2y$05$secret-hash']\n"); + writeFileSync(override, "users: ['op:$2y$05$secret-hash']\n", { mode: 0o644 }); chmodSync(override, 0o664); const runtime = createUnavailableRuntime(); installTrustedConfigurationRuntime(runtime, { diff --git a/packages/cli/__tests__/cli-compose.test.ts b/packages/cli/__tests__/cli-compose.test.ts index 26dcbdfe2..bd547b8a8 100644 --- a/packages/cli/__tests__/cli-compose.test.ts +++ b/packages/cli/__tests__/cli-compose.test.ts @@ -170,6 +170,7 @@ describe("renderComposeForRepo", () => { writeFileSync(env, "DOMAIN=example.com\n", { mode: 0o600 }); const owner = statSync(tmp); await renderComposeForRepo({ rootDir: tmp, domain: "example.com", services: ["alpha"] }); + expect(statSync(join(tmp, "infra/docker/catalog-pins")).mode & 0o777).toBe(0o700); expect(readFileSync(env, "utf8")).toContain(`\nUID=${owner.uid}\nGID=${owner.gid}\n`); expect(permissions(env)).toBe(0o600); diff --git a/packages/cli/__tests__/transitous-api-keys-mount.integration.test.ts b/packages/cli/__tests__/transitous-api-keys-mount.integration.test.ts index 4c13af3aa..1b11be2f9 100644 --- a/packages/cli/__tests__/transitous-api-keys-mount.integration.test.ts +++ b/packages/cli/__tests__/transitous-api-keys-mount.integration.test.ts @@ -2,6 +2,12 @@ import { execFileSync } from "node:child_process"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { + closeControlInstallation, + openControlInstallation, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { ensureTransitousApiKeysFile, @@ -46,22 +52,87 @@ function startReading(mount: string): string { const readInside = (id: string) => docker("exec", id, "cat", "/config/transitous/api-keys.json"); -beforeEach(() => { +beforeEach(async () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-api-keys-mount-")); - writeFileSync(join(tmp, "pnpm-workspace.yaml"), "packages: []\n"); - mkdirSync(join(tmp, "services")); + writeFileSync(join(tmp, "pnpm-workspace.yaml"), "packages: []\n", { mode: 0o600 }); + mkdirSync(join(tmp, "services"), { mode: 0o700 }); const catalog = join(tmp, "infra", "docker", "data", ".transitous-catalog"); - mkdirSync(join(catalog, ".git"), { recursive: true }); - mkdirSync(join(catalog, "feeds"), { recursive: true }); - mkdirSync(join(catalog, "transitland-atlas", "feeds"), { recursive: true }); + mkdirSync(join(catalog, ".git"), { recursive: true, mode: 0o700 }); + mkdirSync(join(catalog, "feeds"), { recursive: true, mode: 0o700 }); + mkdirSync(join(catalog, "transitland-atlas", "feeds"), { recursive: true, mode: 0o700 }); writeFileSync( join(catalog, "feeds", "de.json"), JSON.stringify({ sources: [{ name: "BVG", "transitland-atlas-id": "f-bvg" }] }), + { mode: 0o600 }, ); writeFileSync( join(catalog, "transitland-atlas", "feeds", "de.dmfr.json"), JSON.stringify({ feeds: [{ id: "f-bvg", authorization: { type: "header" } }] }), + { mode: 0o600 }, + ); + const git = (cwd: string, args: string[]) => + execFileSync("git", ["-c", "protocol.file.allow=always", ...args], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + GIT_AUTHOR_NAME: "OpenMapX mount fixture", + GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "OpenMapX mount fixture", + GIT_COMMITTER_EMAIL: "fixture@example.test", + }, + }).trim(); + const atlas = join(catalog, "transitland-atlas"); + git(atlas, ["init"]); + git(atlas, ["add", "feeds"]); + git(atlas, ["commit", "-m", "Pinned mount atlas"]); + const atlasSha = git(atlas, ["rev-parse", "HEAD"]); + git(catalog, ["init"]); + writeFileSync( + join(catalog, ".gitmodules"), + `[submodule "transitland-atlas"]\npath = transitland-atlas\nurl = ${atlas}\n`, + { mode: 0o600 }, + ); + git(catalog, ["add", "feeds", ".gitmodules"]); + git(catalog, ["update-index", "--add", "--cacheinfo", `160000,${atlasSha},transitland-atlas`]); + git(catalog, ["commit", "-m", "Pinned mount catalog"]); + const infraDir = join(tmp, "infra/docker"); + writeFileSync( + join(infraDir, "transitous.lock.json"), + JSON.stringify({ + ref: `main@${git(catalog, ["rev-parse", "HEAD"])}`, + submodules: { "transitland-atlas": atlasSha }, + lockedAt: "2026-10-31T00:00:00.000Z", + lockedBy: "mount-fixture", + }), + { mode: 0o600 }, + ); + writeFileSync( + join(infraDir, "gbfs-catalog.lock.json"), + JSON.stringify({ + schemaVersion: 1, + source: "mobilitydata-gbfs", + commit: "a".repeat(40), + url: `https://raw.githubusercontent.com/MobilityData/gbfs/${"a".repeat(40)}/systems.csv`, + sha256: "b".repeat(64), + lockedAt: "2026-10-31T00:00:00.000Z", + lockedBy: "mount-fixture", + }), + { mode: 0o600 }, ); + await provisionControlInstallation(infraDir); + const installation = openControlInstallation({ + infraDir, + role: "host-cli", + serviceIds: ["motis"], + }); + try { + const pins = await initializeManagedCatalogPins({ rootDir: tmp, installation }); + expect(pins?.transitous.submodules["transitland-atlas"]).toBe(atlasSha); + } finally { + closeControlInstallation(installation); + } }); afterEach(() => { @@ -74,16 +145,18 @@ describe.runIf(runMountTests)("the Transitous API keys mount", () => { const keys = ensureTransitousApiKeysFile(tmp).path; const directory = join(keys, ".."); const container = startReading(`${directory}:/config/transitous:ro`); + const startedAt = docker("inspect", "--format", "{{.State.StartedAt}}", container); // Why the directory: a file mount keeps the inode it was created with. const fileMounted = startReading(`${keys}:/config/transitous/api-keys.json:ro`); expect(readInside(container)).toBe("{}"); expect(readInside(fileMounted)).toBe("{}"); - await generateTransitousApiKeys({ rootDir: tmp, runner: async () => {} }); + await generateTransitousApiKeys({ rootDir: tmp }); const replaced = readFileSync(keys, "utf-8").trim(); expect(JSON.parse(replaced)).toEqual({ "de/BVG": "" }); expect(readInside(container)).toBe(replaced); + expect(docker("inspect", "--format", "{{.State.StartedAt}}", container)).toBe(startedAt); // Linux shows the replaced file's old bytes; Docker Desktop loses the file. let throughFileMount: string; try { diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index 20c92aabc..8a95caa36 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -261,6 +261,7 @@ export async function renderComposeWithWriter( } const agentState = agentStatePaths(paths.infraDir); ensurePlatformPrivateDirectory(agentState.trustedConfig); + ensurePlatformPrivateDirectory(join(paths.infraDir, "catalog-pins")); const erasureDirectory = join(paths.infraDir, "data", "erasure"); ensurePlatformPrivateDirectory(erasureDirectory); ensurePlatformPrivateDirectory(agentState.privacyExtraction); diff --git a/packages/core/src/build-artifacts/archive.test.ts b/packages/core/src/build-artifacts/archive.test.ts index 0bfe84443..ea7b4dee6 100644 --- a/packages/core/src/build-artifacts/archive.test.ts +++ b/packages/core/src/build-artifacts/archive.test.ts @@ -197,16 +197,22 @@ it("exports a sanitized MOTIS candidate with credential IDs and no generated run const { Writable } = await import("node:stream"); const { writeFileSync, symlinkSync } = await import("node:fs"); const data = join(root, "data"); - mkdirSync(join(data, "motis/slots/A/data"), { recursive: true }); - mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); + mkdirSync(join(data, "motis/slots/A/data"), { mode: 0o755, recursive: true }); + mkdirSync(join(data, "motis-feed-proxy/conf"), { mode: 0o755, recursive: true }); symlinkSync("slots/A", join(data, "motis/live")); - writeFileSync(join(data, "motis/slots/A/data/tt.bin"), "TT"); - writeFileSync(join(data, "motis/slots/A/license.json"), '{"license":"synthetic-public"}'); - writeFileSync(join(root, "acquired.gtfs.zip"), "synthetic acquired timetable"); + writeFileSync(join(data, "motis/slots/A/data/tt.bin"), "TT", { mode: 0o644 }); + writeFileSync(join(data, "motis/slots/A/license.json"), '{"license":"synthetic-public"}', { + mode: 0o644, + }); + writeFileSync(join(root, "acquired.gtfs.zip"), "synthetic acquired timetable", { mode: 0o644 }); linkSync(join(root, "acquired.gtfs.zip"), join(data, "motis/slots/A/de_feed.gtfs.zip")); - writeFileSync(join(data, "motis/slots/A/config.yml"), "secret-api-key"); - writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "secret-api-key"); - writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "secret-api-key"); + writeFileSync(join(data, "motis/slots/A/config.yml"), "secret-api-key", { mode: 0o644 }); + writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "secret-api-key", { + mode: 0o644, + }); + writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "secret-api-key", { + mode: 0o644, + }); writeFileSync( join(data, "motis/slots/A/portable-sources.json"), JSON.stringify({ @@ -214,6 +220,7 @@ it("exports a sanitized MOTIS candidate with credential IDs and no generated run sourceFingerprints: [], credentialRequirements: ["synthetic-feed"], }), + { mode: 0o644 }, ); const chunks: Buffer[] = []; const target = new Writable({ @@ -263,12 +270,13 @@ it.each(["tileserver", "osrm", "otp", "pelias", "motis"])( const data = join(root, "data"); const serviceDirs = dirs[service]; if (!serviceDirs?.[0]) throw Error("Missing fixture artifact directory"); - for (const dir of serviceDirs) mkdirSync(join(data, dir), { recursive: true }); - writeFileSync(join(data, serviceDirs[0], "artifact.bin"), "DATA"); + for (const dir of serviceDirs) mkdirSync(join(data, dir), { mode: 0o755, recursive: true }); + writeFileSync(join(data, serviceDirs[0], "artifact.bin"), "DATA", { mode: 0o644 }); if (service === "motis") writeFileSync( join(data, "motis/live/portable-sources.json"), JSON.stringify({ version: 1, sourceFingerprints: [], credentialRequirements: [] }), + { mode: 0o644 }, ); const chunks: Buffer[] = []; const result = await writeBuildBundle({ diff --git a/packages/core/src/build-artifacts/configuration-source-witness.test.ts b/packages/core/src/build-artifacts/configuration-source-witness.test.ts index d34069f09..b16222c9c 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.test.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.test.ts @@ -25,7 +25,7 @@ import { let root: string; beforeEach(async () => { root = mkdtempSync(join(tmpdir(), "omx-source-witness-")); - mkdirSync(join(root, "data")); + mkdirSync(join(root, "data"), { mode: 0o755 }); await provisionControlInstallation(root); }); afterEach(() => rmSync(root, { recursive: true, force: true })); diff --git a/packages/core/src/build-artifacts/copy-tree.test.ts b/packages/core/src/build-artifacts/copy-tree.test.ts index 1bb753905..a3c3cfaa8 100644 --- a/packages/core/src/build-artifacts/copy-tree.test.ts +++ b/packages/core/src/build-artifacts/copy-tree.test.ts @@ -22,7 +22,7 @@ afterEach(() => { function directories() { const root = mkdtempSync(join(tmpdir(), "omx-copy-tree-")); roots.push(root); - mkdirSync(join(root, "source")); + mkdirSync(join(root, "source"), { mode: 0o755 }); mkdirSync(join(root, "target"), { mode: 0o700 }); return { root, @@ -33,9 +33,9 @@ function directories() { it("copies a complete bounded tree with actual byte digests and separate inodes", async () => { const f = directories(); try { - mkdirSync(join(f.root, "source", "empty")); - mkdirSync(join(f.root, "source", "nested")); - writeFileSync(join(f.root, "source/nested/file"), "retained"); + mkdirSync(join(f.root, "source", "empty"), { mode: 0o755 }); + mkdirSync(join(f.root, "source", "nested"), { mode: 0o755 }); + writeFileSync(join(f.root, "source/nested/file"), "retained", { mode: 0o644 }); const limits = { maximumEntries: 10, maximumBytes: 100, maximumManifestBytes: 4096 }, inventory = inspectArtifactTree(f.source, limits); const copied = await copyArtifactTree({ @@ -64,10 +64,10 @@ it("copies a complete bounded tree with actual byte digests and separate inodes" it("refuses symlinks, exhausted budgets and source drift before copying", async () => { const f = directories(); try { - writeFileSync(join(f.root, "source/file"), "before"); + writeFileSync(join(f.root, "source/file"), "before", { mode: 0o644 }); const limits = { maximumEntries: 10, maximumBytes: 100, maximumManifestBytes: 4096 }, original = inspectArtifactTree(f.source, limits); - writeFileSync(join(f.root, "source/file"), "after"); + writeFileSync(join(f.root, "source/file"), "after", { mode: 0o644 }); await expect( copyArtifactTree({ source: f.source, @@ -89,13 +89,13 @@ it("refuses symlinks, exhausted budgets and source drift before copying", async it("isolates shared producer inodes before writes while preserving active bytes and inode", async () => { const f = directories(); try { - writeFileSync(join(f.root, "source/file"), "original feed"); + writeFileSync(join(f.root, "source/file"), "original feed", { mode: 0o644 }); utimesSync( join(f.root, "source/file"), new Date("2020-01-01T00:00:00Z"), new Date("2020-01-01T00:00:00Z"), ); - writeFileSync(join(f.root, "source/unshared"), "local"); + writeFileSync(join(f.root, "source/unshared"), "local", { mode: 0o644 }); linkSync(join(f.root, "source/file"), join(f.root, "target/live")); const active = lstatSync(join(f.root, "target/live")).ino; const local = lstatSync(join(f.root, "source/unshared")).ino; @@ -113,7 +113,7 @@ it("isolates shared producer inodes before writes while preserving active bytes 1, ); expect(lstatSync(join(f.root, "source/unshared")).ino).toBe(local); - writeFileSync(join(f.root, "source/file"), "new download"); + writeFileSync(join(f.root, "source/file"), "new download", { mode: 0o644 }); expect(readFileSync(join(f.root, "target/live"), "utf8")).toBe("original feed"); expect(lstatSync(join(f.root, "target/live")).ino).toBe(active); expect( @@ -135,7 +135,7 @@ it("isolates shared producer inodes before writes while preserving active bytes it("cancels a bounded in-flight isolation without replacing the producer or leaving a temporary writer", async () => { const f = directories(); try { - writeFileSync(join(f.root, "source/file"), Buffer.alloc(16 * 1024 * 1024, 42)); + writeFileSync(join(f.root, "source/file"), Buffer.alloc(16 * 1024 * 1024, 42), { mode: 0o644 }); linkSync(join(f.root, "source/file"), join(f.root, "target/live")); const active = lstatSync(join(f.root, "target/live")).ino; const controller = new AbortController(); diff --git a/packages/core/src/build-artifacts/local-credential-template.test.ts b/packages/core/src/build-artifacts/local-credential-template.test.ts index 03a3aded7..5527a6934 100644 --- a/packages/core/src/build-artifacts/local-credential-template.test.ts +++ b/packages/core/src/build-artifacts/local-credential-template.test.ts @@ -26,7 +26,7 @@ const operation = { async function fixture(role: "host-cli" | "ops-agent" | "data-manager") { const root = mkdtempSync(join(tmpdir(), "omx-local-credential-owner-")); roots.push(root); - mkdirSync(join(root, "data")); + mkdirSync(join(root, "data"), { mode: 0o755 }); await provisionControlInstallation(root); return openControlInstallation({ infraDir: root, role, serviceIds: ["motis", "valhalla"] }); } diff --git a/packages/core/src/build-artifacts/motis-candidate.test.ts b/packages/core/src/build-artifacts/motis-candidate.test.ts index 3145631ee..b7bf2c4b7 100644 --- a/packages/core/src/build-artifacts/motis-candidate.test.ts +++ b/packages/core/src/build-artifacts/motis-candidate.test.ts @@ -30,8 +30,8 @@ function fixture() { "transit-source-manifest.json": '{"version":1,"sources":[]}', }; for (const [path, bytes] of Object.entries(files)) { - mkdirSync(dirname(join(root, path)), { recursive: true }); - writeFileSync(join(root, path), bytes); + mkdirSync(dirname(join(root, path)), { mode: 0o755, recursive: true }); + writeFileSync(join(root, path), bytes, { mode: 0o644 }); } const artifact = (path: string) => ({ path, @@ -67,7 +67,9 @@ function fixture() { sourceManifest: artifact("transit-source-manifest.json"), }, }; - writeFileSync(join(root, "motis-candidate-manifest.json"), JSON.stringify(manifest)); + writeFileSync(join(root, "motis-candidate-manifest.json"), JSON.stringify(manifest), { + mode: 0o644, + }); return manifest; } async function api() { @@ -92,7 +94,9 @@ it.each(["traversal", "duplicate", "digest", "size", "empty", "shape"])( if (kind === "size") manifest.artifacts.datasets[0]!.sizeBytes = -1; if (kind === "empty") manifest.artifacts.datasets = []; if (kind === "shape") manifest.expectations.timetableDatasets = -1; - writeFileSync(join(root, "motis-candidate-manifest.json"), JSON.stringify(manifest)); + writeFileSync(join(root, "motis-candidate-manifest.json"), JSON.stringify(manifest), { + mode: 0o644, + }); await expect((await api()).verifyCandidateManifest(root)).rejects.toThrow(); }, ); @@ -100,7 +104,7 @@ it("rejects symlinked intermediate artifact directories", async () => { fixture(); const outside = mkdtempSync(join(tmpdir(), "omx-motis-outside-")); try { - writeFileSync(join(outside, "default.conf"), "server { listen 80; }"); + writeFileSync(join(outside, "default.conf"), "server { listen 80; }", { mode: 0o644 }); rmSync(join(root, ".openmapx-feed-proxy/conf"), { recursive: true }); symlinkSync(outside, join(root, ".openmapx-feed-proxy/conf")); await expect((await api()).verifyCandidateManifest(root)).rejects.toThrow(); @@ -110,7 +114,7 @@ it("rejects symlinked intermediate artifact directories", async () => { }); it("detects changed candidate content and honors cancellation", async () => { fixture(); - writeFileSync(join(root, "license.json"), "changed"); + writeFileSync(join(root, "license.json"), "changed", { mode: 0o644 }); const mod = await api(); await expect(mod.verifyCandidateManifest(root)).rejects.toThrow(/hash mismatch/); const abort = new AbortController(); @@ -120,7 +124,9 @@ it("detects changed candidate content and honors cancellation", async () => { it("bounds manifest metadata and leaves source bytes unchanged", async () => { fixture(); const before = readFileSync(join(root, "demo.gtfs.zip")); - writeFileSync(join(root, "motis-candidate-manifest.json"), " ".repeat(32 * 1024 * 1024 + 1)); + writeFileSync(join(root, "motis-candidate-manifest.json"), " ".repeat(32 * 1024 * 1024 + 1), { + mode: 0o644, + }); await expect((await api()).verifyCandidateManifest(root)).rejects.toThrow(); expect(readFileSync(join(root, "demo.gtfs.zip"))).toEqual(before); }); @@ -132,7 +138,9 @@ it.each(["mirror", "build"] as const)( ...fixture(), operationsPolicy: resolveOperationsProfile({ countries: ["de"], source }), }; - writeFileSync(join(root, "motis-candidate-manifest.json"), JSON.stringify(manifest)); + writeFileSync(join(root, "motis-candidate-manifest.json"), JSON.stringify(manifest), { + mode: 0o644, + }); expect(await (await api()).verifyCandidateManifest(root)).toEqual(manifest); }, ); diff --git a/packages/core/src/build-artifacts/motis-portable.test.ts b/packages/core/src/build-artifacts/motis-portable.test.ts index 63a267e92..8dac33ecd 100644 --- a/packages/core/src/build-artifacts/motis-portable.test.ts +++ b/packages/core/src/build-artifacts/motis-portable.test.ts @@ -114,8 +114,8 @@ function fixture() { ".openmapx-feed-proxy/conf/default.conf": renderFeedProxyNginxConfig(vars), }; for (const [name, bytes] of Object.entries(files)) { - mkdirSync(dirname(join(dir, name)), { recursive: true }); - writeFileSync(join(dir, name), bytes); + mkdirSync(dirname(join(dir, name)), { mode: 0o755, recursive: true }); + writeFileSync(join(dir, name), bytes, { mode: 0o644 }); } createCandidateManifest(dir, "synthetic-epoch", "2026-10-30T00:00:00.000Z"); return dir; @@ -134,10 +134,13 @@ it.each([false, true])( exportRoot = root(), infraDir = root(), dataRoot = join(infraDir, "data"); + mkdirSync(join(exportRoot, "motis"), { mode: 0o755 }); cpSync(original.dir, join(exportRoot, "motis/live"), { recursive: true }); - mkdirSync(join(exportRoot, "motis/live/data")); - writeFileSync(join(exportRoot, "motis/live/data/tt.bin"), "compiled timetable"); - mkdirSync(join(exportRoot, "motis-feed-proxy")); + mkdirSync(join(exportRoot, "motis/live/data"), { mode: 0o755 }); + writeFileSync(join(exportRoot, "motis/live/data/tt.bin"), "compiled timetable", { + mode: 0o644, + }); + mkdirSync(join(exportRoot, "motis-feed-proxy"), { mode: 0o755 }); const chunks: Buffer[] = []; await control.writeBuildBundle({ dataRoot: exportRoot, @@ -176,8 +179,8 @@ it.each([false, true])( credentials: destinationCredentials, }); for (const slot of ["A", "B"]) { - mkdirSync(join(dataRoot, "motis/slots", slot), { recursive: true }); - writeFileSync(join(dataRoot, "motis/slots", slot, "old"), slot); + mkdirSync(join(dataRoot, "motis/slots", slot), { mode: 0o755, recursive: true }); + writeFileSync(join(dataRoot, "motis/slots", slot, "old"), slot, { mode: 0o644 }); } symlinkSync("slots/A", join(dataRoot, "motis/live")); symlinkSync("slots/B", join(dataRoot, "motis/staging")); @@ -190,9 +193,12 @@ it.each([false, true])( datasetEpoch: "old", activatedAt: "2026-01-01T00:00:00.000Z", }), + { mode: 0o644 }, ); - mkdirSync(join(dataRoot, "motis-feed-proxy/conf"), { recursive: true }); - writeFileSync(join(dataRoot, "motis-feed-proxy/conf/default.conf"), "old proxy"); + mkdirSync(join(dataRoot, "motis-feed-proxy/conf"), { mode: 0o755, recursive: true }); + writeFileSync(join(dataRoot, "motis-feed-proxy/conf/default.conf"), "old proxy", { + mode: 0o644, + }); const aliases = ["live", "staging"].map((name) => ({ name, inode: lstatSync(join(dataRoot, "motis", name)).ino, @@ -314,7 +320,7 @@ it.each([false, true])( function extract(data: Awaited>) { const dest = root(); for (const path of ["license.json", "de_demo.gtfs.zip", "portable-sources.json"]) - writeFileSync(join(dest, path), readFileSync(join(data.dir, path))); + writeFileSync(join(dest, path), readFileSync(join(data.dir, path)), { mode: 0o644 }); return dest; } it("produces the actual candidate's nonsecret recipe and materializes different destination credentials offline", async () => { @@ -385,7 +391,7 @@ it("refuses attacker-selected endpoint metadata and config exfiltration before i it("refuses a changed compiled source before writing a destination config", async () => { const source = await produce(), destination = extract(source); - writeFileSync(join(destination, "de_demo.gtfs.zip"), "changed"); + writeFileSync(join(destination, "de_demo.gtfs.zip"), "changed", { mode: 0o644 }); await expect( materializePortableMotisCandidate({ archiveFingerprint: "d".repeat(64), @@ -406,11 +412,13 @@ it.each(["headers", "secret URL", "unbound endpoint", "oversized"])( writeFileSync( join(dir, "config.yml"), JSON.stringify({ ...config, elevators: { headers: { Authorization: "HOST_SECRET" } } }), + { mode: 0o644 }, ); if (kind === "secret URL") writeFileSync( join(dir, "config.yml"), JSON.stringify({ ...config, elevators: { url: "https://host.example/?key=HOST_SECRET" } }), + { mode: 0o644 }, ); if (kind === "unbound endpoint") writeFileSync( @@ -421,11 +429,13 @@ it.each(["headers", "secret URL", "unbound endpoint", "oversized"])( headers: { "X-Api-Key": credentials["de/demo"] }, }, }), + { mode: 0o644 }, ); if (kind === "oversized") writeFileSync( join(dir, "config.yml"), JSON.stringify({ ...config, note: " ".repeat(1024 * 1024 + 1) }), + { mode: 0o644 }, ); createCandidateManifest(dir, "synthetic-epoch", "2026-10-30T00:00:00.000Z"); await expect( @@ -483,7 +493,9 @@ it("binds the nominal proof to the original archive, metadata and exact generate archiveFingerprint: "e".repeat(64), }), ).rejects.toThrow("archive"); - writeFileSync(join(destination, ".openmapx-feed-proxy/feed-proxy-vars.json"), "{}"); + writeFileSync(join(destination, ".openmapx-feed-proxy/feed-proxy-vars.json"), "{}", { + mode: 0o644, + }); await expect( readVerifiedPortableMotisMaterialization(result.materialization, args), ).rejects.toThrow(); diff --git a/packages/core/src/build-artifacts/publication.test.ts b/packages/core/src/build-artifacts/publication.test.ts index afd11c91c..35fbb0b8e 100644 --- a/packages/core/src/build-artifacts/publication.test.ts +++ b/packages/core/src/build-artifacts/publication.test.ts @@ -31,9 +31,9 @@ async function fixture() { const root = mkdtempSync(join(tmpdir(), "omx-artifact-publication-")); cleanup.push(() => rmSync(root, { recursive: true, force: true })); const dataRoot = join(root, "data"); - mkdirSync(join(dataRoot, "osrm-graph"), { recursive: true }); - writeFileSync(join(dataRoot, "osrm-graph/old"), "old graph"); - mkdirSync(join(dataRoot, ".openmapx-builds")); + mkdirSync(join(dataRoot, "osrm-graph"), { mode: 0o755, recursive: true }); + writeFileSync(join(dataRoot, "osrm-graph/old"), "old graph", { mode: 0o644 }); + mkdirSync(join(dataRoot, ".openmapx-builds"), { mode: 0o755 }); const oldRecord = JSON.stringify({ schemaVersion: 1, service: "osrm", @@ -43,14 +43,14 @@ async function fixture() { }); writeFileSync(join(dataRoot, ".openmapx-builds/osrm.json"), oldRecord, { mode: 0o600 }); const source = join(root, "source"); - mkdirSync(join(source, "osrm-graph"), { recursive: true }); + mkdirSync(join(source, "osrm-graph"), { mode: 0o755, recursive: true }); for (const file of [ "region.osrm", "region.osrm.partition", "region.osrm.cells", "region.osrm.mldgr", ]) - writeFileSync(join(source, "osrm-graph", file), `new ${file}`); + writeFileSync(join(source, "osrm-graph", file), `new ${file}`, { mode: 0o644 }); const chunks: Buffer[] = []; await writeBuildBundle({ dataRoot: source, @@ -143,7 +143,7 @@ it("refuses a changed target before any rename or record write", async () => { const mod = await api(), f = await fixture(), expectedFingerprint = await mod.artifactPublicationFingerprint(f.dataRoot, "osrm"); - writeFileSync(join(f.dataRoot, "osrm-graph/old"), "changed old graph"); + writeFileSync(join(f.dataRoot, "osrm-graph/old"), "changed old graph", { mode: 0o644 }); expect( await mod.publishValidatedBuild({ ...f, @@ -179,7 +179,7 @@ it("rejects undeclared staged files before beginning publication", async () => { const mod = await api(), f = await fixture(); const expectedFingerprint = await mod.artifactPublicationFingerprint(f.dataRoot, "osrm"); - writeFileSync(join(f.stagingRoot, "osrm-graph/undeclared"), "extra"); + writeFileSync(join(f.stagingRoot, "osrm-graph/undeclared"), "extra", { mode: 0o644 }); expect(await mod.publishValidatedBuild({ ...f, expectedFingerprint })).toMatchObject({ outcome: "not_committed", reason: "invalid", @@ -221,8 +221,8 @@ it("keeps uncertain replacement fenced instead of overwriting a foreign inode du native = await vi.importActual("node:fs"); vi.mocked(fs.renameSync).mockImplementation((source, target) => { if (fs.statSync(source).ino === newInode) { - mkdirSync(target); - writeFileSync(join(String(target), "foreign"), "foreign"); + mkdirSync(target, { mode: 0o755 }); + writeFileSync(join(String(target), "foreign"), "foreign", { mode: 0o644 }); throw Error("External replacement during publication"); } return native.renameSync(source, target); @@ -272,7 +272,7 @@ it("rejects same-size staged mutation after hashing instead of recording the pre beforePublication: async () => { const path = join(f.stagingRoot, "osrm-graph/region.osrm"), bytes = readFileSync(path); - writeFileSync(path, Buffer.alloc(bytes.length, 120)); + writeFileSync(path, Buffer.alloc(bytes.length, 120), { mode: 0o644 }); return () => {}; }, }), diff --git a/packages/core/src/catalog-pins/store.test.ts b/packages/core/src/catalog-pins/store.test.ts index e544a8f9e..c6600b039 100644 --- a/packages/core/src/catalog-pins/store.test.ts +++ b/packages/core/src/catalog-pins/store.test.ts @@ -64,10 +64,12 @@ function legacy() { writeFileSync( join(docker, "transitous.lock.json"), JSON.stringify({ $schema: "legacy", ...transitous }), + { mode: 0o644 }, ); writeFileSync( join(docker, "gbfs-catalog.lock.json"), JSON.stringify({ $schema: "legacy", ...gbfs }), + { mode: 0o644 }, ); } function identity(): CatalogPinCommitIdentity { @@ -133,7 +135,7 @@ it("imports both exact legacy pins once, ignoring later compatibility-file chang const first = await initializeCatalogPinSet({ rootDir: root, lease }); expect(first).toEqual(createCatalogPinSet(transitous, gbfs)); const ptr = readFileSync(join(docker, "catalog-pins/current.json")); - writeFileSync(join(docker, "gbfs-catalog.lock.json"), "corrupt"); + writeFileSync(join(docker, "gbfs-catalog.lock.json"), "corrupt", { mode: 0o644 }); expect(await initializeCatalogPinSet({ rootDir: root, lease })).toEqual(first); expect(readCatalogPinSet(root)).toEqual(first); expect(readFileSync(join(docker, "catalog-pins/current.json"))).toEqual(ptr); @@ -149,7 +151,9 @@ it("does not initialize from missing or malformed halves", async () => { { url: "https://example.test/mutable" }, { lockedAt: "yesterday" }, ]) { - writeFileSync(join(docker, "gbfs-catalog.lock.json"), JSON.stringify({ ...gbfs, ...patch })); + writeFileSync(join(docker, "gbfs-catalog.lock.json"), JSON.stringify({ ...gbfs, ...patch }), { + mode: 0o644, + }); await expect(initializeCatalogPinSet({ rootDir: root, lease })).rejects.toThrow(); } expect(existsSync(join(docker, "catalog-pins/current.json"))).toBe(false); @@ -169,7 +173,7 @@ it("refuses symlinked locks and replacements between descriptor checks", async ( if (fs.fstatSync(args[0]).ino === target.ino && seen++ === 0) { const path = join(docker, "transitous.lock.json"); renameSync(path, `${path}.old`); - writeFileSync(path, JSON.stringify(transitous)); + writeFileSync(path, JSON.stringify(transitous), { mode: 0o644 }); } return result; }) as typeof read); @@ -268,7 +272,9 @@ it("never falls back to legacy files when a selected generation is corrupt", asy legacy(); const pins = await initializeCatalogPinSet({ rootDir: root, lease }); if (!pins) throw Error("missing fixture pair"); - writeFileSync(join(docker, `catalog-pins/generations/${pins.generationId}.json`), "{}"); + writeFileSync(join(docker, `catalog-pins/generations/${pins.generationId}.json`), "{}", { + mode: 0o644, + }); expect(() => readCatalogPinSet(root)).toThrow(); }); it("bootstraps an actual local initialization root and releases its MOTIS ownership", async () => { @@ -282,8 +288,8 @@ it("bootstraps an actual local initialization root and releases its MOTIS owners serviceIds: ["motis"], }); try { - writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(transitous)); - writeFileSync(join(infra, "gbfs-catalog.lock.json"), JSON.stringify(gbfs)); + writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(transitous), { mode: 0o644 }); + writeFileSync(join(infra, "gbfs-catalog.lock.json"), JSON.stringify(gbfs), { mode: 0o644 }); const { initializeManagedCatalogPins } = await import("./bootstrap"); const pins = await initializeManagedCatalogPins({ rootDir: other, installation: local }); expect(pins).toEqual(createCatalogPinSet(transitous, gbfs)); @@ -355,8 +361,8 @@ it("does not acquire a startup target root for a malformed legacy pair", async ( serviceIds: ["motis"], }); try { - writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(transitous)); - writeFileSync(join(infra, "gbfs-catalog.lock.json"), "invalid"); + writeFileSync(join(infra, "transitous.lock.json"), JSON.stringify(transitous), { mode: 0o644 }); + writeFileSync(join(infra, "gbfs-catalog.lock.json"), "invalid", { mode: 0o644 }); const { initializeManagedCatalogPins } = await import("./bootstrap"); await expect( initializeManagedCatalogPins({ rootDir: other, installation: local }), @@ -372,8 +378,8 @@ it("refuses to initialize another checkout with this checkout's MOTIS lease", as try { const dir = join(other, "infra/docker"); mkdirSync(join(dir, "data"), { recursive: true, mode: 0o700 }); - writeFileSync(join(dir, "transitous.lock.json"), JSON.stringify(transitous)); - writeFileSync(join(dir, "gbfs-catalog.lock.json"), JSON.stringify(gbfs)); + writeFileSync(join(dir, "transitous.lock.json"), JSON.stringify(transitous), { mode: 0o644 }); + writeFileSync(join(dir, "gbfs-catalog.lock.json"), JSON.stringify(gbfs), { mode: 0o644 }); await expect(initializeCatalogPinSet({ rootDir: other, lease })).rejects.toThrow( /root|foreign|checkout/i, ); diff --git a/packages/core/src/services/__tests__/agent-mounts.test.ts b/packages/core/src/services/__tests__/agent-mounts.test.ts index 47455a4c2..c6e281846 100644 --- a/packages/core/src/services/__tests__/agent-mounts.test.ts +++ b/packages/core/src/services/__tests__/agent-mounts.test.ts @@ -18,6 +18,12 @@ const manifest = JSON.parse( ) as ServiceManifest; const roots: string[] = []; +it("keeps managed catalog metadata writable under the narrow agent layout", () => { + const mount = agentBindMounts(agent("/srv/openmapx"), "narrow").find((entry) => + entry.target.endsWith("/infra/docker/catalog-pins"), + ); + expect(mount).toMatchObject({ readOnly: false }); +}); afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); @@ -40,7 +46,7 @@ function atCheckoutPath(volumes: readonly string[]): string[] { const NARROW = [ // biome-ignore lint/suspicious/noTemplateCurlyInString: Compose substitutes this, as in the manifest "/infra/docker:${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker:ro", - ...["state", "data", "secrets", "backups"].map( + ...["state", "data", "secrets", "backups", "catalog-pins"].map( (name) => `/infra/docker/${name}:\${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/${name}`, ), diff --git a/packages/core/src/services/__tests__/backup-prune-plan.test.ts b/packages/core/src/services/__tests__/backup-prune-plan.test.ts index 395ffe057..5e0a62a6f 100644 --- a/packages/core/src/services/__tests__/backup-prune-plan.test.ts +++ b/packages/core/src/services/__tests__/backup-prune-plan.test.ts @@ -17,17 +17,18 @@ let infra: string; const at = new Date("2026-08-31T00:00:00.000Z"); function archive(name: string, createdAt: string, expiresAt?: string) { const dir = join(infra, "backups", name); - mkdirSync(dir, { recursive: true }); + mkdirSync(dir, { mode: 0o755, recursive: true }); writeFileSync( join(dir, "manifest.json"), JSON.stringify({ formatVersion: 2, name, createdAt, expiresAt, services: [] }), + { mode: 0o644 }, ); - writeFileSync(join(dir, "archive.tar.gz"), "synthetic archive"); + writeFileSync(join(dir, "archive.tar.gz"), "synthetic archive", { mode: 0o644 }); return dir; } beforeEach(() => { infra = mkdtempSync(join(tmpdir(), "openmapx-prune-plan-")); - mkdirSync(join(infra, "backups")); + mkdirSync(join(infra, "backups"), { mode: 0o755 }); }); afterEach(() => rmSync(infra, { recursive: true, force: true })); it("previews only the original eligible set and does not write archive metadata", () => { @@ -43,7 +44,7 @@ it("binds newly created archives and byte-identical inode replacements", () => { const dir = archive("old", "2026-07-01T00:00:00.000Z"), before = backupPruneInventory(infra, 30, at); const path = join(dir, "manifest.json"); - writeFileSync(`${path}.new`, readFileSync(path)); + writeFileSync(`${path}.new`, readFileSync(path), { mode: 0o644 }); renameSync(`${path}.new`, path); expect(backupPruneInventory(infra, 30, at).inventoryDigest).not.toBe(before.inventoryDigest); const replaced = backupPruneInventory(infra, 30, at); @@ -53,7 +54,7 @@ it("binds newly created archives and byte-identical inode replacements", () => { it("keeps stamped ceilings and reports unreadable metadata separately", () => { archive("stamped", "2026-08-01T00:00:00.000Z", "2026-08-15T00:00:00.000Z"); const broken = archive("broken", "2026-08-20T00:00:00.000Z"); - writeFileSync(join(broken, "manifest.json"), "broken"); + writeFileSync(join(broken, "manifest.json"), "broken", { mode: 0o644 }); utimesSync(broken, at, at); const plan = backupPruneInventory(infra, 365, at); expect(plan.candidates.map((entry) => entry.backupId)).toEqual(["stamped"]); @@ -61,7 +62,7 @@ it("keeps stamped ceilings and reports unreadable metadata separately", () => { }); it("refuses links and ignores only the existing store lock", () => { const dir = archive("old", "2026-07-01T00:00:00.000Z"); - mkdirSync(join(infra, "backups", ".backup-store.lock")); + mkdirSync(join(infra, "backups", ".backup-store.lock"), { mode: 0o755 }); expect(backupPruneInventory(infra, 30, at).candidates).toHaveLength(1); symlinkSync(join(dir, "manifest.json"), join(dir, "link")); expect(() => backupPruneInventory(infra, 30, at)).toThrow(); diff --git a/packages/core/src/services/__tests__/configuration-generation.test.ts b/packages/core/src/services/__tests__/configuration-generation.test.ts index fc80ff9cb..54a5a955b 100644 --- a/packages/core/src/services/__tests__/configuration-generation.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation.test.ts @@ -41,7 +41,7 @@ function checkout() { const root = mkdtempSync(join(tmpdir(), "openmapx-generation-")); roots.push(root); const infraDir = join(root, "infra", "docker"); - mkdirSync(infraDir, { recursive: true }); + mkdirSync(infraDir, { mode: 0o755, recursive: true }); return { root, infraDir }; } @@ -442,6 +442,7 @@ it("reads only frozen generated service IDs even when a release overlay declares writeFileSync( join(infraDir, "docker-compose.release.yml"), "services:\n app-api:\n image: example/api\n data-manager:\n image: example/dm\n", + { mode: 0o644 }, ); expect(readAppliedComposeServiceIds(infraDir, generation.revisionId)).toEqual(["alpha"]); expect(() => readAppliedComposeServiceIds(infraDir, `cfg1_${"x".repeat(43)}`)).toThrow( diff --git a/packages/core/src/services/service-lifecycle-receipt.test.ts b/packages/core/src/services/service-lifecycle-receipt.test.ts index 1666f4802..4228e925e 100644 --- a/packages/core/src/services/service-lifecycle-receipt.test.ts +++ b/packages/core/src/services/service-lifecycle-receipt.test.ts @@ -32,7 +32,7 @@ let root: string; const revision = `cfg1_${"a".repeat(43)}`; beforeEach(async () => { root = mkdtempSync(join(tmpdir(), "omx-lifecycle-receipt-")); - mkdirSync(join(root, "data")); + mkdirSync(join(root, "data"), { mode: 0o755 }); await provisionControlInstallation(root); }); afterEach(() => { @@ -123,7 +123,7 @@ it("signs exact seed and checkpoints only with the live writer and physical targ it("rejects a foreign data root before creating any shared receipt", async () => { const o = owner(), foreign = join(root, "foreign"); - mkdirSync(foreign); + mkdirSync(foreign, { mode: 0o755 }); try { await expect( withServiceLifecycleReceiptAuthority(o.authority, () => diff --git a/packages/hardlinks/__tests__/apply.test.ts b/packages/hardlinks/__tests__/apply.test.ts index cdab7fe78..ce0a406e9 100644 --- a/packages/hardlinks/__tests__/apply.test.ts +++ b/packages/hardlinks/__tests__/apply.test.ts @@ -29,6 +29,38 @@ describe("applyHardlinkPlan", () => { if (existsSync(root)) rmSync(root, { recursive: true, force: true }); }); + it.each(["tree", "single-file"] as const)( + "creates safe consumer permissions under umask 002: %s", + (kind) => { + mkdirSync(join(root, "src", "nested"), { recursive: true, mode: 0o700 }); + writeFileSync(join(root, "src", "nested", "source.conf"), "fixture", { mode: 0o600 }); + const previous = process.umask(0o002); + try { + applyHardlinkPlan( + [ + { + source: kind === "tree" ? "src" : "src/nested", + target: "consumer/config", + consumerService: "svc", + dataType: "config", + ...(kind === "single-file" ? { targetFilename: "default.conf" } : {}), + }, + ], + { rootDir: root }, + ); + for (const path of [ + "consumer", + "consumer/config", + DEFAULT_SENTINEL_DIR, + ...(kind === "tree" ? ["consumer/config/nested"] : []), + ]) + expect(statSync(join(root, path)).mode & 0o022, path).toBe(0); + } finally { + process.umask(previous); + } + }, + ); + it("links files from source into target and writes a sentinel", () => { mkdirSync(join(root, "src"), { recursive: true }); writeFileSync(join(root, "src", "a.txt"), "A"); diff --git a/packages/hardlinks/src/index.ts b/packages/hardlinks/src/index.ts index bafa60310..458195615 100644 --- a/packages/hardlinks/src/index.ts +++ b/packages/hardlinks/src/index.ts @@ -208,8 +208,8 @@ function readSentinel(sentinelPath: string): Sentinel | null { } function writeSentinel(sentinelPath: string, sentinel: Sentinel): void { - mkdirSync(dirname(sentinelPath), { recursive: true }); - writeFileSync(sentinelPath, `${JSON.stringify(sentinel, null, 2)}\n`); + mkdirSync(dirname(sentinelPath), { recursive: true, mode: 0o755 }); + writeFileSync(sentinelPath, `${JSON.stringify(sentinel, null, 2)}\n`, { mode: 0o644 }); } function removeSentinel(sentinelPath: string): void { @@ -231,7 +231,7 @@ function linkFileAt( } rmSync(tgtPath, { force: true }); } - mkdirSync(dirname(tgtPath), { recursive: true }); + mkdirSync(dirname(tgtPath), { recursive: true, mode: 0o755 }); linkSync(srcPath, tgtPath); return { linked: 1, skipped: 0 }; } @@ -292,7 +292,7 @@ function linkTreeWithSentinel( rmSync(targetDir, { force: true }); result.pruned += 1; } - mkdirSync(targetDir, { recursive: true }); + mkdirSync(targetDir, { recursive: true, mode: 0o755 }); const sourcePaths = collectSourceFiles(sourceDir); const sourceSet = new Set(sourcePaths); @@ -351,7 +351,7 @@ function linkSingleFileWithSentinel( rmSync(targetDir, { force: true }); result.pruned += 1; } - mkdirSync(targetDir, { recursive: true }); + mkdirSync(targetDir, { recursive: true, mode: 0o755 }); const tgtPath = join(targetDir, targetFilename); const r = linkFileAt(join(sourceDir, file.name), tgtPath, rootAbs); diff --git a/packages/motis-feed-proxy-config/src/build.ts b/packages/motis-feed-proxy-config/src/build.ts index daf75f16e..d3c28a11c 100644 --- a/packages/motis-feed-proxy-config/src/build.ts +++ b/packages/motis-feed-proxy-config/src/build.ts @@ -152,9 +152,12 @@ export function normalizeFeedProxyVars(raw: unknown): FeedProxyVars { export function writeFeedProxyVarsFile(outputPath: string, raw: unknown): FeedProxyVars { const normalized = normalizeFeedProxyVars(raw); const sorted = Object.fromEntries(sortedEntries(normalized)); - mkdirSync(dirname(outputPath), { recursive: true }); + mkdirSync(dirname(outputPath), { recursive: true, mode: 0o700 }); const temporaryPath = `${outputPath}.tmp-${process.pid}`; - writeFileSync(temporaryPath, `${JSON.stringify(sorted, null, 2)}\n`, "utf-8"); + writeFileSync(temporaryPath, `${JSON.stringify(sorted, null, 2)}\n`, { + encoding: "utf-8", + mode: 0o600, + }); renameSync(temporaryPath, outputPath); return sorted; } @@ -320,7 +323,7 @@ export async function buildFeedProxyConfig( includeGbfsServers: input.includeGbfsServers, headerComment: input.headerComment, }); - mkdirSync(dirname(input.outputPath), { recursive: true }); - writeFileSync(input.outputPath, rendered, "utf-8"); + mkdirSync(dirname(input.outputPath), { recursive: true, mode: 0o700 }); + writeFileSync(input.outputPath, rendered, { encoding: "utf-8", mode: 0o600 }); return { wrote: true, entries: Object.keys(vars).length }; } diff --git a/release-notes/admin-instance-management.md b/release-notes/admin-instance-management.md index 2e010b797..edb0a284e 100644 --- a/release-notes/admin-instance-management.md +++ b/release-notes/admin-instance-management.md @@ -32,6 +32,11 @@ from the checkout root. Existing unsafe directories are refused rather than silently repaired. See the installation and upgrade guides for host-owned overrides and recovery boundaries. +New prepared-data directories and metadata use explicit safe creation modes. +If an existing dataset is reported as unsafe, verify its ownership and remove +group/other write permissions from the reported paths before retrying. The +panel does not silently change ownership or repair existing dataset permissions. + Community service configuration schemas must not declare the reserved platform resource names `OPENMAPX_MEMORY_MIB`, `OPENMAPX_CPU_LIMIT` or `OPENMAPX_JAVA_HEAP_MIB`. diff --git a/scripts/ops-agent-deployment-user.test.ts b/scripts/ops-agent-deployment-user.test.ts index ff4611f37..18f634aaf 100644 --- a/scripts/ops-agent-deployment-user.test.ts +++ b/scripts/ops-agent-deployment-user.test.ts @@ -1,6 +1,7 @@ import { type ChildProcess, execFile, spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; import { + chmodSync, closeSync, constants, fstatSync, @@ -17,13 +18,18 @@ import { createServer } from "node:net"; import { tmpdir } from "node:os"; import { join, relative, resolve } from "node:path"; import { promisify } from "node:util"; -import { OpsClient } from "@openmapx/core/ops"; +import { + configurationApplyResultSchema, + createControlRootOperationKey, + OpsClient, + type OpsOperation, + type OpsOperationKind, + type OpsResultFor, + opsExecutionBudget, +} from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { - createDurableOpsKey, - executeTrustedConfigurationAndWait, -} from "../apps/api/src/services/ops-client"; +import { waitForOpsResult } from "../apps/api/src/services/ops-client"; import { consumePublishedTrustedConfiguration, publishTrustedConfigurationSnapshot, @@ -40,8 +46,8 @@ import { * trusted-configuration request: in the host update's order (the host's * render, then the agent) and in the panel self-update's (the new agent * first). Only the host's render writes UID and GID; the agent and its fixed - * CLI read .env and never change it. app-api publishes as the UID and GID - * Compose gives it from .env. + * CLI read .env and never change it. The fixture publishes with the UID/GID + * Compose gives app-api from .env and submits a truthful local bearer root. * * Runs only on Linux where `OPENMAPX_TEST_DEPLOYMENT_USER` names the uid the * test runs as, which must not be 1000 and must own the repository, e.g. in a @@ -83,16 +89,29 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( const clone = mkdtempSync(join(tmpdir(), "openmapx-deployment-user-")); roots.push(clone); const target = join(clone, "checkout"); - const previous = process.umask(0o002); - try { - await run("git", ["clone", "-q", "--no-local", ROOT, target]); - } finally { - process.umask(previous); - } + // Set only the clone child process's umask, never the test worker's. + await run(process.execPath, [ + "-e", + 'process.umask(0o002); const { status } = require("node:child_process").spawnSync(process.argv[1], process.argv.slice(2), { stdio: "inherit" }); process.exit(status ?? 1);', + "git", + "clone", + "-q", + "--no-local", + ROOT, + target, + ]); expect(statSync(target).uid).toBe(EXPECTED_UID); writeFileSync(join(target, "infra", "docker", ".env"), envWithoutUser(target), { mode: 0o600, }); + // Explicit operator bootstrap required by the private control keys. + // Harden only their authority ancestors; source trees retain umask 002. + const infra = join(target, "infra", "docker"); + mkdirSync(join(infra, "secrets"), { recursive: true, mode: 0o700 }); + for (const directory of [target, join(target, "infra"), infra, join(infra, "secrets")]) + chmodSync(directory, statSync(directory).mode & 0o7777 & ~0o022); + for (const source of [join(target, "services"), join(target, "integrations")]) + expect(statSync(source).mode & 0o020, source).toBe(0o020); return target; } @@ -205,6 +224,31 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( return { port, output: () => output }; } + /** A timed local bearer root carrying a real sealed snapshot, not a fabricated API parent. */ + function localRootIdentity(kind: OpsOperationKind) { + const requestedDeadlineAt = Date.now() + opsExecutionBudget(kind).processMs; + return { + operationKey: createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }), + execution: { requestedDeadlineAt }, + }; + } + + async function executeLocalRoot( + client: OpsClient, + operation: Extract, + identity = localRootIdentity(operation.kind), + ): Promise> { + const submitted = await client.execute(operation, identity); + if (submitted.execution === "sync") return submitted.value; + return waitForOpsResult(client, operation.kind, submitted, { + execution: identity.execution, + observerDeadlineAt: identity.execution.requestedDeadlineAt, + }); + } + /** A request app-api publishes as the UID and GID Compose passes it. */ async function submit( target: string, @@ -214,7 +258,11 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( ) { const infra = join(target, "infra", "docker"); const tokenFile = join(infra, "secrets", "ops-agent-api-token"); - const operationKey = createDurableOpsKey("stack.render", name); + const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agent.port}`, tokenFile }); + const inspected = await client.execute({ kind: "configuration.sources.inspect" }); + if (inspected.execution !== "sync") throw Error("Expected a synchronous source inspection"); + const identity = localRootIdentity("stack.render"); + const { operationKey } = identity; const before = services.currentConfigurationGeneration(infra); const sealed = await publishTrustedConfigurationSnapshot({ directory: services.agentStatePaths(infra).trustedConfig, @@ -226,6 +274,8 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( kind: "stack.render", revisionId, expectedCurrentRevisionId: before, + expectedSourceRevision: inspected.value.sourceRevision, + coordinatorMode: "require-running", }), payload: { domain: `${name}.example.test`, @@ -234,16 +284,16 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( serviceSecrets: [], }, }); - const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agent.port}`, tokenFile }); const consumed = await consumePublishedTrustedConfiguration(sealed, () => - executeTrustedConfigurationAndWait( - client, - sealed.operation as Parameters[1], - operationKey, - ), + executeLocalRoot(client, sealed.operation, identity), ); expect(consumed.submission.status, agent.output()).toBe("fulfilled"); - return consumed.submission.status === "fulfilled" ? consumed.submission.value : undefined; + const result = configurationApplyResultSchema.parse( + consumed.submission.status === "fulfilled" ? consumed.submission.value : undefined, + ); + if (result.outcome === "committed") + expect(result.revisionId).toBe(sealed.operation.revisionId); + return result; } async function expectCommitted( @@ -284,6 +334,13 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( const written = envFileState(target, true); const agent = await startAgent(target); expect(agent.output()).not.toContain("sets no UID"); + const infra = join(target, "infra", "docker"); + const before = services.currentConfigurationGeneration(infra); + await expect( + submit(target, agent, "wrong-deployment-user", { uid: 1000, gid: statSync(target).gid }), + ).rejects.toThrow("Trusted configuration snapshot publish failed"); + expect(envFileState(target)).toEqual(written); + expect(services.currentConfigurationGeneration(infra)).toBe(before); await expectCommitted(target, agent, "host-update"); // The agent's fixed CLI renders with it too, and leaves .env as it was. await agentCli(target, "compose", "render"); diff --git a/scripts/ops-agent-mount-layouts.test.ts b/scripts/ops-agent-mount-layouts.test.ts index 5ea4678eb..7ddf19239 100644 --- a/scripts/ops-agent-mount-layouts.test.ts +++ b/scripts/ops-agent-mount-layouts.test.ts @@ -13,17 +13,16 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; import { + createControlRootOperationKey, OpsClient, type OpsOperation, type OpsOperationKind, type OpsResultFor, + opsExecutionBudget, } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterAll, describe, expect, it } from "vitest"; -import { - createDurableOpsKey, - executeTrustedConfigurationAndWait, -} from "../apps/api/src/services/ops-client"; +import { waitForOpsResult } from "../apps/api/src/services/ops-client"; import { consumePublishedTrustedConfiguration, publishTrustedConfigurationSnapshot, @@ -46,9 +45,9 @@ import { * from the host too, apart from those, so what the agent would read from it * is absent, not empty. * - * Each run starts the agent, commits a configuration change through it as - * app-api would, with a service file override, runs its fixed CLI's check - * and render, reads the stack's status and a handoff record, and inspects + * Each run starts the agent, commits a sealed configuration change through + * the supported local bearer root with a service file override, runs its fixed + * CLI's check and render, reads the stack's status and a handoff record, and inspects * what its self-update helper would mount. * * Runs only on Linux where `OPENMAPX_TEST_AGENT_IMAGE` names a built image of @@ -61,7 +60,7 @@ const roots: string[] = []; const containers: string[] = []; const OVERRIDE_MARK = "# openmapx-ci: an operator's override"; -const RUNTIME_DIRECTORIES = ["state", "data", "secrets", "backups"] as const; +const RUNTIME_DIRECTORIES = ["state", "data", "secrets", "backups", "catalog-pins"] as const; /** The stack's network, which `docker compose up` of the generation creates. */ const COMPOSE_NETWORK = `${services.STACK_PROJECT}_openmapx`; @@ -81,6 +80,18 @@ function layoutMounts(layout: "checkout" | "narrow", root: string) { target: join(infra, name), readOnly: false, })), + ...["data-manager", "host-cli"].flatMap((role) => [ + { + source: join(infra, "control-private/hidden-key"), + target: join(infra, `secrets/control-${role}.key`), + readOnly: true, + }, + { + source: join(infra, "control-private/empty"), + target: join(infra, `control-private/${role}`), + readOnly: true, + }, + ]), ]; } @@ -158,7 +169,7 @@ describe.skipIf(process.platform !== "linux" || !IMAGE)( for (const name of readdirSync(join(root, "infra"))) { if (name !== "docker") rmSync(join(root, "infra", name), { recursive: true, force: true }); } - const keep = new Set([".env", "overrides", ...RUNTIME_DIRECTORIES]); + const keep = new Set([".env", "overrides", "control-private", ...RUNTIME_DIRECTORIES]); for (const name of readdirSync(join(root, "infra", "docker"))) { if (!keep.has(name)) rmSync(join(root, "infra", "docker", name), { recursive: true, force: true }); @@ -282,28 +293,38 @@ describe.skipIf(process.platform !== "linux" || !IMAGE)( return { id, baseUrl: `http://127.0.0.1:${port}` }; } + /** A truthful timed local bearer root, without inventing an API job parent. */ + function localRootIdentity(kind: OpsOperationKind) { + const requestedDeadlineAt = Date.now() + opsExecutionBudget(kind).processMs; + return { + operationKey: createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: randomBytes(16), + }), + execution: { requestedDeadlineAt }, + }; + } + async function operate( agent: Agent, operation: Extract, + identity = localRootIdentity(operation.kind), ): Promise> { - const submitted = await agent.client.execute(operation, { - operationKey: `opk1_${randomBytes(18).toString("base64url")}`, + const submitted = await agent.client.execute(operation, identity); + if (submitted.execution === "sync") return submitted.value; + return waitForOpsResult(agent.client, operation.kind, submitted, { + execution: identity.execution, + observerDeadlineAt: identity.execution.requestedDeadlineAt, }); - if (submitted.execution === "sync") return submitted.value as OpsResultFor; - for (;;) { - const status = await agent.client.status(operation.kind, submitted.operationId); - if (status.state === "succeeded") return status.result as OpsResultFor; - if (status.state !== "queued" && status.state !== "running") { - throw new Error(`${operation.kind} ended ${status.state}: ${await agent.logs()}`); - } - await new Promise((done) => setTimeout(done, 250)); - } } - /** A configuration change app-api publishes and the agent commits. */ + /** The actual sealed configuration body and current source witness over agent HTTP. */ async function commit(root: string, agent: Agent, name: string): Promise { const infra = join(root, "infra", "docker"); - const operationKey = createDurableOpsKey("stack.render", name); + const inspected = await agent.client.execute({ kind: "configuration.sources.inspect" }); + if (inspected.execution !== "sync") throw Error("Expected a synchronous source inspection"); + const identity = localRootIdentity("stack.render"); + const { operationKey } = identity; const before = services.currentConfigurationGeneration(infra); const sealed = await publishTrustedConfigurationSnapshot({ directory: services.agentStatePaths(infra).trustedConfig, @@ -315,6 +336,8 @@ describe.skipIf(process.platform !== "linux" || !IMAGE)( kind: "stack.render", revisionId, expectedCurrentRevisionId: before, + expectedSourceRevision: inspected.value.sourceRevision, + coordinatorMode: "require-running", }), payload: { domain: `${name}.example.test`, @@ -324,16 +347,14 @@ describe.skipIf(process.platform !== "linux" || !IMAGE)( }, }); const consumed = await consumePublishedTrustedConfiguration(sealed, () => - executeTrustedConfigurationAndWait( - agent.client, - sealed.operation as Parameters[1], - operationKey, - ), + operate(agent, sealed.operation, identity), ); + expect(consumed.submission.status, await agent.logs()).toBe("fulfilled"); const result = consumed.submission.status === "fulfilled" ? consumed.submission.value : undefined; expect(result, `${JSON.stringify(result)}\n${await agent.logs()}`).toMatchObject({ outcome: "committed", + revisionId: sealed.operation.revisionId, }); expect(services.currentConfigurationGeneration(infra)).not.toBe(before); } @@ -380,7 +401,16 @@ describe.skipIf(process.platform !== "linux" || !IMAGE)( expect(listing.stdout.split("\n").filter(Boolean)).toEqual([ "infra", "docker", - ...[".env", "backups", "data", "overrides", "secrets", "state"], + ...[ + ".env", + "backups", + "catalog-pins", + "control-private", + "data", + "overrides", + "secrets", + "state", + ], ]); for (const absent of ["services", "packages", "apps", "integrations", ".git"]) { const probe = await agent.exec("test", "-e", join(root, absent)); diff --git a/services/data-manager/__tests__/catalog-pair-read.test.ts b/services/data-manager/__tests__/catalog-pair-read.test.ts index 714d215ef..3a0234a5d 100644 --- a/services/data-manager/__tests__/catalog-pair-read.test.ts +++ b/services/data-manager/__tests__/catalog-pair-read.test.ts @@ -11,7 +11,7 @@ afterEach(() => { it("never treats one legacy Transitous file as an authoritative complete catalog pair", () => { const root = mkdtempSync(join(tmpdir(), "omx-pins-read-")); roots.push(root); - mkdirSync(join(root, "infra/docker"), { recursive: true }); + mkdirSync(join(root, "infra/docker"), { mode: 0o700, recursive: true }); writeFileSync( join(root, "infra/docker/transitous.lock.json"), JSON.stringify({ @@ -20,6 +20,7 @@ it("never treats one legacy Transitous file as an authoritative complete catalog lockedAt: "2026-10-23T00:00:00.000Z", lockedBy: "fixture", }), + { mode: 0o600 }, ); expect(() => readTransitousLock(root)).toThrow(/paired|initializ|catalog/i); }); diff --git a/services/data-manager/__tests__/download-osm.test.ts b/services/data-manager/__tests__/download-osm.test.ts index 3f48b6d42..7114764d4 100644 --- a/services/data-manager/__tests__/download-osm.test.ts +++ b/services/data-manager/__tests__/download-osm.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -70,7 +70,7 @@ describe("downloadOsm publication", () => { async (_url, targetPath, options) => { events.push("download"); const tempPath = `${targetPath}.test-temp`; - writeFileSync(tempPath, "validated-pbf"); + writeFileSync(tempPath, "validated-pbf", { mode: 0o600 }); await options.beforePublish?.(tempPath); events.push("validated"); await options.withPublishLock?.(() => { @@ -97,6 +97,7 @@ describe("downloadOsm publication", () => { expect(events).toEqual(["download", "validated", "lock-start", "publish", "lock-end"]); expect(result.sizeBytes).toBe(Buffer.byteLength("validated-pbf")); + expect(statSync(join(dataDir, "osm")).mode & 0o777).toBe(0o700); } finally { rmSync(dataDir, { recursive: true, force: true }); } diff --git a/services/data-manager/__tests__/gbfs-catalog-lock.test.ts b/services/data-manager/__tests__/gbfs-catalog-lock.test.ts index d2b80fcb6..616e83a97 100644 --- a/services/data-manager/__tests__/gbfs-catalog-lock.test.ts +++ b/services/data-manager/__tests__/gbfs-catalog-lock.test.ts @@ -26,7 +26,7 @@ const LOCK = { describe("GBFS catalog lock", () => { it("roundtrips an immutable commit URL and hash", () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-gbfs-lock-")); - mkdirSync(join(tmp, "infra", "docker"), { recursive: true }); + mkdirSync(join(tmp, "infra", "docker"), { mode: 0o700, recursive: true }); writeGbfsCatalogLock(tmp, LOCK); expect(() => readGbfsCatalogLock(tmp!)).toThrow(/not initialized/); expect(readFileSync(join(tmp, "infra", "docker", "gbfs-catalog.lock.json"), "utf-8")).toContain( diff --git a/services/data-manager/__tests__/helpers/gtfs-fixture.ts b/services/data-manager/__tests__/helpers/gtfs-fixture.ts index 9a0eda7e2..28e936da8 100644 --- a/services/data-manager/__tests__/helpers/gtfs-fixture.ts +++ b/services/data-manager/__tests__/helpers/gtfs-fixture.ts @@ -13,7 +13,7 @@ const FIXTURE_GTFS_ZIP = Buffer.from( ); export function writeFixtureGtfsArchive(path: string): void { - writeFileSync(path, FIXTURE_GTFS_ZIP); + writeFileSync(path, FIXTURE_GTFS_ZIP, { mode: 0o600 }); } export function isFixtureGtfsArchive(bytes: Buffer): boolean { diff --git a/services/data-manager/__tests__/osm-fingerprints.test.ts b/services/data-manager/__tests__/osm-fingerprints.test.ts index 2d74ff343..cef756a10 100644 --- a/services/data-manager/__tests__/osm-fingerprints.test.ts +++ b/services/data-manager/__tests__/osm-fingerprints.test.ts @@ -28,9 +28,9 @@ it("publishes SHA256 metadata inside the same snapshot publication lock", async publishedUnderLock = existsSync(join(root, "osm/europe-germany.osm.pbf.openmapx.json")); }, downloadImpl: async (_url, target, options) => { - mkdirSync(join(root, "osm"), { recursive: true }); + mkdirSync(join(root, "osm"), { mode: 0o700, recursive: true }); const tmp = target + ".test-part"; - writeFileSync(tmp, "fixture pbf bytes"); + writeFileSync(tmp, "fixture pbf bytes", { mode: 0o600 }); await options.beforePublish?.(tmp); await options.withPublishLock?.(() => renameSync(tmp, target)); return { published: true }; @@ -56,9 +56,9 @@ it("publishes SHA256 metadata inside the same snapshot publication lock", async }); it("does not bind a 304 checksum to a replacement inode at publication", async () => { const root = mkdtempSync(join(tmpdir(), "omx-osm-304-evidence-")); - mkdirSync(join(root, "osm")); + mkdirSync(join(root, "osm"), { mode: 0o700 }); const target = join(root, "osm/europe-germany.osm.pbf"); - writeFileSync(target, "old bytes"); + writeFileSync(target, "old bytes", { mode: 0o600 }); try { await expect( downloadOsm({ @@ -68,7 +68,7 @@ it("does not bind a 304 checksum to a replacement inode at publication", async ( verifyChecksum: false, downloadImpl: async () => ({ published: false }), withPublishLock: async (publish) => { - writeFileSync(target + ".replacement", "new bytes"); + writeFileSync(target + ".replacement", "new bytes", { mode: 0o600 }); renameSync(target + ".replacement", target); publish(); }, @@ -95,8 +95,8 @@ it("reports failed sidecar publication without leaving verified evidence", async store: new StateStore(root), verifyChecksum: false, downloadImpl: async (_url, target, options) => { - mkdirSync(join(root, "osm"), { recursive: true }); - writeFileSync(target + ".part", "new bytes"); + mkdirSync(join(root, "osm"), { mode: 0o700, recursive: true }); + writeFileSync(target + ".part", "new bytes", { mode: 0o600 }); await options.beforePublish?.(target + ".part"); await options.withPublishLock?.(() => renameSync(target + ".part", target)); return { published: true }; diff --git a/services/data-manager/__tests__/overture/local-inputs.test.ts b/services/data-manager/__tests__/overture/local-inputs.test.ts index 31163d80b..fd031d469 100644 --- a/services/data-manager/__tests__/overture/local-inputs.test.ts +++ b/services/data-manager/__tests__/overture/local-inputs.test.ts @@ -23,9 +23,9 @@ function root() { function snapshot(dataDir: string, release: string) { const region = "europe/germany", directory = join(dataDir, "overture", release); - mkdirSync(directory, { recursive: true }); + mkdirSync(directory, { mode: 0o700, recursive: true }); const file = join(directory, "europe-germany.parquet"); - writeFileSync(file, "metadata fixture, not a real parquet"); + writeFileSync(file, "metadata fixture, not a real parquet", { mode: 0o600 }); const contract = buildOverturePullContract( { release, @@ -48,17 +48,21 @@ function snapshot(dataDir: string, release: string) { undefined, captureOvertureCatalogue("2026-10-13T00:00:00.000Z"), ); - writeFileSync(pullContractPath(dataDir, release, region), JSON.stringify(contract)); + writeFileSync(pullContractPath(dataDir, release, region), JSON.stringify(contract), { + mode: 0o600, + }); return file; } it("selects the newest completed local region contract, ignoring partial pulls without creating files", () => { const path = root(); snapshot(path, "2026-10-01.0"); const newest = snapshot(path, "2026-10-21.0"); - mkdirSync(join(path, "overture/2026-10-29.0")); - writeFileSync(join(path, "overture/2026-10-29.0/europe-germany.parquet"), "incomplete"); - mkdirSync(join(path, "osm")); - writeFileSync(join(path, "osm/europe-germany.osm.pbf"), "PBF fixture"); + mkdirSync(join(path, "overture/2026-10-29.0"), { mode: 0o700 }); + writeFileSync(join(path, "overture/2026-10-29.0/europe-germany.parquet"), "incomplete", { + mode: 0o600, + }); + mkdirSync(join(path, "osm"), { mode: 0o700 }); + writeFileSync(join(path, "osm/europe-germany.osm.pbf"), "PBF fixture", { mode: 0o600 }); expect(inspectOvertureLocalInputs(path, "europe/germany")).toEqual({ downloadedRelease: "2026-10-21.0", osmAvailable: true, @@ -67,13 +71,13 @@ it("selects the newest completed local region contract, ignoring partial pulls w downloadedRelease: null, osmAvailable: false, }); - writeFileSync(newest, "changed size"); + writeFileSync(newest, "changed size", { mode: 0o600 }); expect(inspectOvertureLocalInputs(path, "europe/germany").downloadedRelease).toBe("2026-10-01.0"); }); it("does not follow a release directory symlink or accept traversal regions", () => { const path = root(), outside = root(); - mkdirSync(join(path, "overture")); + mkdirSync(join(path, "overture"), { mode: 0o700 }); symlinkSync(outside, join(path, "overture/2026-10-21.0")); expect(() => inspectOvertureLocalInputs(path, "europe/germany")).toThrow(); expect(() => inspectOvertureLocalInputs(path, "../outside")).toThrow(); diff --git a/services/data-manager/__tests__/transit-sources.test.ts b/services/data-manager/__tests__/transit-sources.test.ts index 88b87b898..105ac18f4 100644 --- a/services/data-manager/__tests__/transit-sources.test.ts +++ b/services/data-manager/__tests__/transit-sources.test.ts @@ -25,7 +25,7 @@ function fixture() { tmp = mkdtempSync(join(tmpdir(), "openmapx-transit-sources-")); const catalogDir = join(tmp, ".transitous-catalog"); const overlayPath = join(tmp, "overrides", "feeds-overlay.json"); - mkdirSync(join(catalogDir, "feeds"), { recursive: true }); + mkdirSync(join(catalogDir, "feeds"), { mode: 0o700, recursive: true }); writeFileSync( join(catalogDir, "feeds", "de.json"), JSON.stringify({ @@ -46,8 +46,9 @@ function fixture() { }, ], }), + { mode: 0o600 }, ); - mkdirSync(join(tmp, "motis", "live"), { recursive: true }); + mkdirSync(join(tmp, "motis", "live"), { mode: 0o700, recursive: true }); writeFileSync( join(tmp, "motis", "live", "transit-source-manifest.json"), JSON.stringify({ @@ -72,22 +73,25 @@ function fixture() { }, ], }), + { mode: 0o600 }, ); const live = join(tmp, "motis", "live"); - mkdirSync(join(live, ".openmapx-feed-proxy", "conf"), { recursive: true }); + mkdirSync(join(live, ".openmapx-feed-proxy", "conf"), { mode: 0o700, recursive: true }); writeFileSync( join(live, "config.yml"), JSON.stringify({ timetable: { datasets: { "de-active-feed": { path: "de_active-feed.gtfs.zip" } } }, }), + { mode: 0o600 }, ); - writeFileSync(join(live, "license.json"), "[]"); - writeFileSync(join(live, "de_active-feed.gtfs.zip"), "synthetic timetable"); + writeFileSync(join(live, "license.json"), "[]", { mode: 0o600 }); + writeFileSync(join(live, "de_active-feed.gtfs.zip"), "synthetic timetable", { mode: 0o600 }); writeFileSync( join(live, ".openmapx-feed-proxy", "conf", "default.conf"), "# no realtime feeds\n", + { mode: 0o600 }, ); - writeFileSync(join(live, ".openmapx-feed-proxy", "feed-proxy-vars.json"), "{}"); + writeFileSync(join(live, ".openmapx-feed-proxy", "feed-proxy-vars.json"), "{}", { mode: 0o600 }); createCandidateManifest(live, "epoch-live", "2026-07-30T00:00:00.000Z"); return { dataDir: tmp, catalogDir, overlayPath }; } @@ -215,6 +219,7 @@ describe("transit source lifecycle", () => { }, ], }), + { mode: 0o600 }, ); const manifestPath = join( fx.dataDir as string, @@ -226,7 +231,7 @@ describe("transit source lifecycle", () => { sources: Array<{ originUrl?: string }>; }; manifest.sources[0].originUrl = safeOriginUrl(rawUrl); - writeFileSync(manifestPath, JSON.stringify(manifest)); + writeFileSync(manifestPath, JSON.stringify(manifest), { mode: 0o600 }); expect(listTransitSources({ ...fx }).sources).toContainEqual( expect.objectContaining({ id: "catalog:de:active-feed", lifecycle: "active" }), ); @@ -241,6 +246,7 @@ describe("transit source lifecycle", () => { { name: "fr-feed", spec: "gtfs", type: "http", url: "https://catalog.example/fr.zip" }, ], }), + { mode: 0o600 }, ); const scoped = listTransitSources({ ...fx, countries: ["de"] }); expect(scoped.sources.some((source) => source.region === "fr")).toBe(false); diff --git a/services/data-manager/__tests__/transitous-lock.test.ts b/services/data-manager/__tests__/transitous-lock.test.ts index 04a375973..cf55b2552 100644 --- a/services/data-manager/__tests__/transitous-lock.test.ts +++ b/services/data-manager/__tests__/transitous-lock.test.ts @@ -24,7 +24,7 @@ afterEach(() => { function makeRepoRoot(): string { tmp = mkdtempSync(join(tmpdir(), "openmapx-tlock-")); - mkdirSync(join(tmp, "infra", "docker"), { recursive: true }); + mkdirSync(join(tmp, "infra", "docker"), { mode: 0o700, recursive: true }); return tmp; } @@ -95,13 +95,15 @@ describe("readTransitousLock / writeTransitousLock", () => { it("throws on a malformed lockfile", () => { const root = makeRepoRoot(); - writeFileSync(join(root, TRANSITOUS_LOCK_RELATIVE_PATH), "not json"); + writeFileSync(join(root, TRANSITOUS_LOCK_RELATIVE_PATH), "not json", { mode: 0o600 }); expect(() => readTransitousLock(root)).toThrow(/not initialized/); }); it("throws when required fields are missing", () => { const root = makeRepoRoot(); - writeFileSync(join(root, TRANSITOUS_LOCK_RELATIVE_PATH), JSON.stringify({ submodules: {} })); + writeFileSync(join(root, TRANSITOUS_LOCK_RELATIVE_PATH), JSON.stringify({ submodules: {} }), { + mode: 0o600, + }); expect(() => readTransitousLock(root)).toThrow(/not initialized/); }); }); diff --git a/services/data-manager/__tests__/transitous/assemble-staging.test.ts b/services/data-manager/__tests__/transitous/assemble-staging.test.ts index 1a7950f5b..65cb82f57 100644 --- a/services/data-manager/__tests__/transitous/assemble-staging.test.ts +++ b/services/data-manager/__tests__/transitous/assemble-staging.test.ts @@ -5,6 +5,7 @@ import { readdirSync, readFileSync, rmSync, + statSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -157,45 +158,47 @@ function setupFixture(opts: FixtureOptions): { const stagingDir = join(dataDir, "motis", "staging"); const liveDir = join(dataDir, "motis", "live"); - mkdirSync(outDir, { recursive: true }); - mkdirSync(liveDir, { recursive: true }); + mkdirSync(outDir, { mode: 0o700, recursive: true }); + mkdirSync(liveDir, { mode: 0o700, recursive: true }); const feeds = opts.feeds ?? []; const osm = opts.osmInLive ?? opts.osmMissing; if (opts.withConfig) { const configText = buildConfig(feeds, osm); - writeFileSync(join(outDir, "config.yml"), configText); + writeFileSync(join(outDir, "config.yml"), configText, { mode: 0o600 }); } for (const feed of feeds) { - writeFileSync(join(outDir, feed), `fake-gtfs-data-${feed}`); + writeFileSync(join(outDir, feed), `fake-gtfs-data-${feed}`, { mode: 0o600 }); } // Create OSM in live dir only when osmInLive is set (not when osmMissing) if (opts.osmInLive) { - writeFileSync(join(liveDir, opts.osmInLive), "fake-osm-data"); + writeFileSync(join(liveDir, opts.osmInLive), "fake-osm-data", { mode: 0o600 }); } // scripts/ + license.json in outDir (small text files the stage copies) const scriptsDir = join(outDir, "scripts"); - mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(scriptsDir, "coloring.lua"), "-- lua script\n"); - writeFileSync(join(outDir, "license.json"), JSON.stringify({ licenses: [] })); - writeFileSync(join(outDir, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"sources":[]}\n'); + mkdirSync(scriptsDir, { mode: 0o700, recursive: true }); + writeFileSync(join(scriptsDir, "coloring.lua"), "-- lua script\n", { mode: 0o600 }); + writeFileSync(join(outDir, "license.json"), JSON.stringify({ licenses: [] }), { mode: 0o600 }); + writeFileSync(join(outDir, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"sources":[]}\n', { + mode: 0o600, + }); const proxyDir = join(outDir, CANDIDATE_PROXY_DIRNAME); - mkdirSync(join(proxyDir, "conf"), { recursive: true }); - writeFileSync(join(proxyDir, "conf", "default.conf"), "server { listen 80; }\n"); - writeFileSync(join(proxyDir, "feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(proxyDir, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(proxyDir, "conf", "default.conf"), "server { listen 80; }\n", { mode: 0o600 }); + writeFileSync(join(proxyDir, "feed-proxy-vars.json"), "{}\n", { mode: 0o600 }); if (opts.staleArchiveInStaging) { - mkdirSync(stagingDir, { recursive: true }); - writeFileSync(join(stagingDir, opts.staleArchiveInStaging), "stale-content"); + mkdirSync(stagingDir, { mode: 0o700, recursive: true }); + writeFileSync(join(stagingDir, opts.staleArchiveInStaging), "stale-content", { mode: 0o600 }); } if (opts.dataMarkerInStaging) { - mkdirSync(join(stagingDir, "data"), { recursive: true }); - writeFileSync(join(stagingDir, "data", "tt.bin"), "compiled-timetable"); + mkdirSync(join(stagingDir, "data"), { mode: 0o700, recursive: true }); + writeFileSync(join(stagingDir, "data", "tt.bin"), "compiled-timetable", { mode: 0o600 }); } return { dataDir, outDir, stagingDir, liveDir }; @@ -222,7 +225,7 @@ describe("assemble-staging run()", () => { expect((await run(makeCtx(fx.dataDir))).status).toBe("ok"); for (const name of ["config.yml", "license.json", TRANSIT_SOURCE_MANIFEST_FILENAME]) { const original = readFileSync(join(fx.stagingDir, name)); - writeFileSync(join(fx.outDir, name), "next producer generation"); + writeFileSync(join(fx.outDir, name), "next producer generation", { mode: 0o600 }); expect(readFileSync(join(fx.stagingDir, name))).toEqual(original); } }); @@ -276,6 +279,7 @@ describe("assemble-staging run()", () => { writeFileSync( join(fx.outDir, "config.yml"), buildConfig(["present.gtfs.zip", "missing-feed.gtfs.zip"]), + { mode: 0o600 }, ); const result = await run(makeCtx(fx.dataDir)); @@ -298,7 +302,7 @@ describe("assemble-staging run()", () => { it("returns status 'error' when every referenced feed is missing from output", async () => { const fx = setupFixture({ withConfig: true, feeds: [] }); // Config references a feed that was never written to disk → 0 linked. - writeFileSync(join(fx.outDir, "config.yml"), buildConfig(["gone.gtfs.zip"])); + writeFileSync(join(fx.outDir, "config.yml"), buildConfig(["gone.gtfs.zip"]), { mode: 0o600 }); const result = await run(makeCtx(fx.dataDir)); expect(result.status).toBe("error"); expect((result.artifacts as Record).missingFeeds).toContain("gone.gtfs.zip"); @@ -369,8 +373,10 @@ describe("assemble-staging run()", () => { const feeds = ["feed.gtfs.zip"]; const fx = setupFixture({ withConfig: true, feeds }); // Write config referencing osm, and put the osm file in outDir (not live dir) - writeFileSync(join(fx.outDir, "config.yml"), buildConfig(feeds, "local.osm.pbf")); - writeFileSync(join(fx.outDir, "local.osm.pbf"), "osm-data-from-out"); + writeFileSync(join(fx.outDir, "config.yml"), buildConfig(feeds, "local.osm.pbf"), { + mode: 0o600, + }); + writeFileSync(join(fx.outDir, "local.osm.pbf"), "osm-data-from-out", { mode: 0o600 }); const result = await run(makeCtx(fx.dataDir)); expect(result.status).toBe("ok"); @@ -386,16 +392,20 @@ describe("assemble-staging run()", () => { const outDir = join(dataDir, "gtfs"); const liveDir = join(dataDir, "motis", "live"); - mkdirSync(outDir, { recursive: true }); - mkdirSync(liveDir, { recursive: true }); - writeFileSync(join(outDir, "config.yml"), buildConfig(["feed.gtfs.zip"])); - writeFileSync(join(outDir, "feed.gtfs.zip"), "feed-content"); - writeFileSync(join(outDir, "license.json"), "[]\n"); - writeFileSync(join(outDir, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"sources":[]}\n'); + mkdirSync(outDir, { mode: 0o700, recursive: true }); + mkdirSync(liveDir, { mode: 0o700, recursive: true }); + writeFileSync(join(outDir, "config.yml"), buildConfig(["feed.gtfs.zip"]), { mode: 0o600 }); + writeFileSync(join(outDir, "feed.gtfs.zip"), "feed-content", { mode: 0o600 }); + writeFileSync(join(outDir, "license.json"), "[]\n", { mode: 0o600 }); + writeFileSync(join(outDir, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"sources":[]}\n', { + mode: 0o600, + }); const proxyDir = join(outDir, CANDIDATE_PROXY_DIRNAME); - mkdirSync(join(proxyDir, "conf"), { recursive: true }); - writeFileSync(join(proxyDir, "conf", "default.conf"), "server { listen 80; }\n"); - writeFileSync(join(proxyDir, "feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(proxyDir, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(proxyDir, "conf", "default.conf"), "server { listen 80; }\n", { + mode: 0o600, + }); + writeFileSync(join(proxyDir, "feed-proxy-vars.json"), "{}\n", { mode: 0o600 }); const ctx = buildJobContext({ dataDir, @@ -424,6 +434,7 @@ describe("assemble-staging run() — directory structure", () => { const result = await run(makeCtx(fx.dataDir)); expect(result.status).toBe("ok"); expect(existsSync(fx.stagingDir)).toBe(true); + expect(statSync(fx.stagingDir).mode & 0o777).toBe(0o700); const entries = readdirSync(fx.stagingDir); expect(entries).toContain("config.yml"); diff --git a/services/data-manager/__tests__/transitous/candidate.test.ts b/services/data-manager/__tests__/transitous/candidate.test.ts index e5f456719..e50a4fe7b 100644 --- a/services/data-manager/__tests__/transitous/candidate.test.ts +++ b/services/data-manager/__tests__/transitous/candidate.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + CANDIDATE_MANIFEST_FILENAME, CANDIDATE_PROXY_DIRNAME, createCandidateManifest, parseMotisConfigExpectations, @@ -67,14 +68,17 @@ describe("candidate manifest", () => { writeFileSync( join(tmp, "config.yml"), "timetable:\n datasets:\n demo:\n path: demo.gtfs.zip\n", + { mode: 0o600 }, ); - writeFileSync(join(tmp, "license.json"), "[]\n"); - writeFileSync(join(tmp, "demo.gtfs.zip"), "GTFS"); - writeFileSync(join(tmp, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"sources":[]}\n'); + writeFileSync(join(tmp, "license.json"), "[]\n", { mode: 0o600 }); + writeFileSync(join(tmp, "demo.gtfs.zip"), "GTFS", { mode: 0o600 }); + writeFileSync(join(tmp, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"sources":[]}\n', { + mode: 0o600, + }); const proxy = join(tmp, CANDIDATE_PROXY_DIRNAME); - mkdirSync(join(proxy, "conf"), { recursive: true }); - writeFileSync(join(proxy, "conf", "default.conf"), "server { listen 80; }\n"); - writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(proxy, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(proxy, "conf", "default.conf"), "server { listen 80; }\n", { mode: 0o600 }); + writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n", { mode: 0o600 }); return tmp; } @@ -82,6 +86,7 @@ describe("candidate manifest", () => { const dir = fixture(); const manifest = createCandidateManifest(dir, "epoch-1", "2026-07-15T00:00:00.000Z"); expect(manifest.epoch).toBe("epoch-1"); + expect(fs.statSync(join(dir, CANDIDATE_MANIFEST_FILENAME)).mode & 0o777).toBe(0o600); expect(manifest.artifacts.datasets).toHaveLength(1); expect(await verifyCandidateManifest(dir)).toEqual(manifest); }); @@ -99,7 +104,7 @@ describe("candidate manifest", () => { it("detects mutation after the manifest is written", async () => { const dir = fixture(); createCandidateManifest(dir, "epoch-1", "2026-07-15T00:00:00.000Z"); - writeFileSync(join(dir, "license.json"), '[{"changed":true}]\n'); + writeFileSync(join(dir, "license.json"), '[{"changed":true}]\n', { mode: 0o600 }); await expect(verifyCandidateManifest(dir)).rejects.toThrow(/hash mismatch: license\.json/); }); }); @@ -112,7 +117,7 @@ it("hashes datasets without loading a whole file into one buffer", () => { tmp = mkdtempSync(join(tmpdir(), "omx-candidate-stream-")); const file = join(tmp, "large.gtfs.zip"), bytes = Buffer.alloc(8 * 1024 * 1024, 7); - writeFileSync(file, bytes); + writeFileSync(file, bytes, { mode: 0o600 }); vi.mocked(fs.readFileSync).mockClear(); expect(sha256File(file).sha256).toBe(createHash("sha256").update(bytes).digest("hex")); expect(fs.readFileSync).not.toHaveBeenCalledWith(file); diff --git a/services/data-manager/__tests__/transitous/motis-health.test.ts b/services/data-manager/__tests__/transitous/motis-health.test.ts index 2423de952..280323f10 100644 --- a/services/data-manager/__tests__/transitous/motis-health.test.ts +++ b/services/data-manager/__tests__/transitous/motis-health.test.ts @@ -42,21 +42,23 @@ afterEach(() => { function makeStagingDir(): string { tmp = mkdtempSync(join(tmpdir(), "openmapx-motis-health-")); const stagingDir = join(tmp, "motis", "staging"); - mkdirSync(stagingDir, { recursive: true }); + mkdirSync(stagingDir, { mode: 0o700, recursive: true }); writeFileSync( join(stagingDir, "config.yml"), "timetable:\n datasets:\n demo:\n path: demo.gtfs.zip\n", + { mode: 0o600 }, ); - writeFileSync(join(stagingDir, "demo.gtfs.zip"), "gtfs"); - writeFileSync(join(stagingDir, "license.json"), "{}\n"); + writeFileSync(join(stagingDir, "demo.gtfs.zip"), "gtfs", { mode: 0o600 }); + writeFileSync(join(stagingDir, "license.json"), "{}\n", { mode: 0o600 }); writeFileSync( join(stagingDir, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"generatedAt":"2026-05-01T00:00:00Z","sources":[]}\n', + { mode: 0o600 }, ); const proxy = join(stagingDir, CANDIDATE_PROXY_DIRNAME); - mkdirSync(join(proxy, "conf"), { recursive: true }); - writeFileSync(join(proxy, "conf", "default.conf"), "server {}\n"); - writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(proxy, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(proxy, "conf", "default.conf"), "server {}\n", { mode: 0o600 }); + writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n", { mode: 0o600 }); createCandidateManifest(stagingDir, "test-epoch", "2026-05-01T00:00:00.000Z"); return tmp; } @@ -93,7 +95,7 @@ describe("motis-health stage", () => { it("skips when the staging data dir is empty", async () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-motis-health-empty-")); - mkdirSync(join(tmp, "motis", "staging"), { recursive: true }); + mkdirSync(join(tmp, "motis", "staging"), { mode: 0o700, recursive: true }); const result = await motisHealthRun(ctxFor(tmp)); expect(result.status).toBe("skipped"); }); diff --git a/services/data-manager/__tests__/transitous/motis-import.test.ts b/services/data-manager/__tests__/transitous/motis-import.test.ts index 771606687..730c78e6f 100644 --- a/services/data-manager/__tests__/transitous/motis-import.test.ts +++ b/services/data-manager/__tests__/transitous/motis-import.test.ts @@ -43,16 +43,23 @@ function stagingDirOf(dataDir: string): string { } function primeCandidate(stagingDir: string): void { - mkdirSync(join(stagingDir, CANDIDATE_PROXY_DIRNAME, "conf"), { recursive: true }); - writeFileSync(join(stagingDir, "config.yml"), "timetable:\n datasets:\n demo: {}\n"); - writeFileSync(join(stagingDir, "demo.gtfs.zip"), "gtfs"); - writeFileSync(join(stagingDir, "license.json"), "{}\n"); + mkdirSync(join(stagingDir, CANDIDATE_PROXY_DIRNAME, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(stagingDir, "config.yml"), "timetable:\n datasets:\n demo: {}\n", { + mode: 0o600, + }); + writeFileSync(join(stagingDir, "demo.gtfs.zip"), "gtfs", { mode: 0o600 }); + writeFileSync(join(stagingDir, "license.json"), "{}\n", { mode: 0o600 }); writeFileSync( join(stagingDir, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"generatedAt":"2026-05-01T00:00:00Z","sources":[]}\n', + { mode: 0o600 }, ); - writeFileSync(join(stagingDir, CANDIDATE_PROXY_DIRNAME, "conf", "default.conf"), "server {}\n"); - writeFileSync(join(stagingDir, CANDIDATE_PROXY_DIRNAME, "feed-proxy-vars.json"), "{}\n"); + writeFileSync(join(stagingDir, CANDIDATE_PROXY_DIRNAME, "conf", "default.conf"), "server {}\n", { + mode: 0o600, + }); + writeFileSync(join(stagingDir, CANDIDATE_PROXY_DIRNAME, "feed-proxy-vars.json"), "{}\n", { + mode: 0o600, + }); createCandidateManifest(stagingDir, "test-epoch", "2026-05-01T00:00:00.000Z"); } @@ -87,7 +94,7 @@ describe("motis-import stage", () => { it("skips when the staging config has not been generated", async () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-motis-import-noconfig-")); - mkdirSync(stagingDirOf(tmp), { recursive: true }); + mkdirSync(stagingDirOf(tmp), { mode: 0o700, recursive: true }); const ctx = makeCtx({ dataDir: tmp, diff --git a/services/data-manager/__tests__/transitous/pipeline.test.ts b/services/data-manager/__tests__/transitous/pipeline.test.ts index 8fddbe1cd..9d4ddfe31 100644 --- a/services/data-manager/__tests__/transitous/pipeline.test.ts +++ b/services/data-manager/__tests__/transitous/pipeline.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; @@ -91,11 +91,12 @@ describe("runTransitousPipeline orchestrator", () => { it("stops at a failed owner checkpoint before the next stage can mutate", async () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-pipeline-owner-")); - mkdirSync(join(tmp, ".transitous-catalog", ".git"), { recursive: true }); - mkdirSync(join(tmp, ".transitous-catalog", "feeds"), { recursive: true }); + mkdirSync(join(tmp, ".transitous-catalog", ".git"), { mode: 0o700, recursive: true }); + mkdirSync(join(tmp, ".transitous-catalog", "feeds"), { mode: 0o700, recursive: true }); writeFileSync( join(tmp, ".transitous-catalog", "feeds", "de.json"), JSON.stringify({ sources: [{ name: "fixture" }] }), + { mode: 0o600 }, ); const stages: StageResult[] = []; const ctx = buildJobContext({ @@ -117,7 +118,7 @@ describe("runTransitousPipeline orchestrator", () => { it("never reports ok when cancellation follows a successful stage", async () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-pipeline-cancel-")); const catalogDir = join(tmp, ".transitous-catalog"); - mkdirSync(join(catalogDir, ".git"), { recursive: true }); + mkdirSync(join(catalogDir, ".git"), { mode: 0o700, recursive: true }); const controller = new AbortController(); const persisted: StageResult[] = []; const ctx = buildJobContext({ @@ -143,17 +144,24 @@ describe("runTransitousPipeline orchestrator", () => { const dataDir = tmp; const catalogDir = join(dataDir, ".transitous-catalog"); const gtfsDir = join(dataDir, "gtfs"); - mkdirSync(join(catalogDir, ".git"), { recursive: true }); - mkdirSync(join(catalogDir, "feeds"), { recursive: true }); - mkdirSync(join(catalogDir, "src"), { recursive: true }); + mkdirSync(join(catalogDir, ".git"), { mode: 0o700, recursive: true }); + mkdirSync(join(catalogDir, "feeds"), { mode: 0o700, recursive: true }); + mkdirSync(join(catalogDir, "src"), { mode: 0o700, recursive: true }); writeFileSync( join(catalogDir, "feeds", "de.json"), JSON.stringify({ sources: [{ name: "BVG" }] }, null, 2), + { mode: 0o600 }, ); // Stub Transitous python scripts so the gen-* stages dispatch their run. - writeFileSync(join(catalogDir, "src", "generate-motis-config.py"), "#!/usr/bin/env python3\n"); - writeFileSync(join(catalogDir, "src", "generate-attribution.py"), "#!/usr/bin/env python3\n"); - writeFileSync(join(catalogDir, "src", "garbage-collect.py"), "#!/usr/bin/env python3\n"); + writeFileSync(join(catalogDir, "src", "generate-motis-config.py"), "#!/usr/bin/env python3\n", { + mode: 0o600, + }); + writeFileSync(join(catalogDir, "src", "generate-attribution.py"), "#!/usr/bin/env python3\n", { + mode: 0o600, + }); + writeFileSync(join(catalogDir, "src", "garbage-collect.py"), "#!/usr/bin/env python3\n", { + mode: 0o600, + }); const persisted: StageResult[] = []; @@ -169,13 +177,14 @@ describe("runTransitousPipeline orchestrator", () => { writeFileSync( join(catalogDir, "out", "config.yml"), "timetable:\n datasets:\n de_bvg:\n path: de_bvg.gtfs.zip\n", + { mode: 0o600 }, ); } else if (run.script === "feed-proxy-vars-to-json") { - writeFileSync(join(catalogDir, "out", "feed-proxy-vars.json"), "{}"); + writeFileSync(join(catalogDir, "out", "feed-proxy-vars.json"), "{}", { mode: 0o600 }); } else if (run.script === "generate-attribution") { // Simulate the upstream script writing its manifest (gen-attribution // now asserts the file exists). - writeFileSync(join(catalogDir, "out", "license.json"), "[]"); + writeFileSync(join(catalogDir, "out", "license.json"), "[]", { mode: 0o600 }); } }, now: () => "2026-05-01T00:00:00.000Z", @@ -195,6 +204,11 @@ describe("runTransitousPipeline orchestrator", () => { // The finalized config and attribution are assembled before any import. const byStage = Object.fromEntries(results.map((r) => [r.stage, r])); expect(byStage["assemble-staging"]?.status).toBe("ok"); + const proxy = join(ctx.outDir, ".openmapx-feed-proxy"); + for (const directory of [proxy, join(proxy, "conf")]) + expect(statSync(directory).mode & 0o777).toBe(0o700); + for (const path of [join(proxy, "feed-proxy-vars.json"), join(proxy, "conf/default.conf")]) + expect(statSync(path).mode & 0o777).toBe(0o600); expect(byStage["motis-import"]).toBeUndefined(); expect(byStage["motis-health"]).toBeUndefined(); expect(byStage.promote).toBeUndefined(); @@ -211,11 +225,12 @@ describe("runTransitousPipeline orchestrator", () => { tmp = mkdtempSync(join(tmpdir(), "openmapx-pipeline-hardstop-")); const dataDir = tmp; const catalogDir = join(dataDir, ".transitous-catalog"); - mkdirSync(join(catalogDir, ".git"), { recursive: true }); - mkdirSync(join(catalogDir, "feeds"), { recursive: true }); + mkdirSync(join(catalogDir, ".git"), { mode: 0o700, recursive: true }); + mkdirSync(join(catalogDir, "feeds"), { mode: 0o700, recursive: true }); writeFileSync( join(catalogDir, "feeds", "de.json"), JSON.stringify({ sources: [{ name: "BVG" }] }, null, 2), + { mode: 0o600 }, ); const persisted: StageResult[] = []; @@ -253,20 +268,28 @@ describe("runTransitousPipeline orchestrator", () => { const dataDir = tmp; const catalogDir = join(dataDir, ".transitous-catalog"); const gtfsDir = join(dataDir, "gtfs"); - mkdirSync(join(catalogDir, ".git"), { recursive: true }); - mkdirSync(join(catalogDir, "feeds"), { recursive: true }); - mkdirSync(join(catalogDir, "src"), { recursive: true }); + mkdirSync(join(catalogDir, ".git"), { mode: 0o700, recursive: true }); + mkdirSync(join(catalogDir, "feeds"), { mode: 0o700, recursive: true }); + mkdirSync(join(catalogDir, "src"), { mode: 0o700, recursive: true }); writeFileSync( join(catalogDir, "feeds", "de.json"), JSON.stringify({ sources: [{ name: "BVG" }, { name: "DELFI" }] }, null, 2), + { mode: 0o600 }, ); writeFileSync( join(catalogDir, "feeds", "ch.json"), JSON.stringify({ sources: [{ name: "SBB" }] }, null, 2), + { mode: 0o600 }, ); - writeFileSync(join(catalogDir, "src", "generate-motis-config.py"), "#!/usr/bin/env python3\n"); - writeFileSync(join(catalogDir, "src", "generate-attribution.py"), "#!/usr/bin/env python3\n"); - writeFileSync(join(catalogDir, "src", "garbage-collect.py"), "#!/usr/bin/env python3\n"); + writeFileSync(join(catalogDir, "src", "generate-motis-config.py"), "#!/usr/bin/env python3\n", { + mode: 0o600, + }); + writeFileSync(join(catalogDir, "src", "generate-attribution.py"), "#!/usr/bin/env python3\n", { + mode: 0o600, + }); + writeFileSync(join(catalogDir, "src", "garbage-collect.py"), "#!/usr/bin/env python3\n", { + mode: 0o600, + }); const ctx = buildJobContext({ dataDir, @@ -275,12 +298,12 @@ describe("runTransitousPipeline orchestrator", () => { runner: async () => {}, runScript: async (run) => { if (run.script === "fetch") { - mkdirSync(gtfsDir, { recursive: true }); + mkdirSync(gtfsDir, { mode: 0o700, recursive: true }); writeFixtureGtfsArchive(join(gtfsDir, "de_bvg.gtfs.zip")); writeFixtureGtfsArchive(join(gtfsDir, "de_delfi.gtfs.zip")); writeFixtureGtfsArchive(join(gtfsDir, "ch_sbb.gtfs.zip")); } else if (run.script === "generate-attribution") { - writeFileSync(join(catalogDir, "out", "license.json"), "[]"); + writeFileSync(join(catalogDir, "out", "license.json"), "[]", { mode: 0o600 }); } }, now: () => "2026-05-01T00:00:00.000Z", @@ -306,11 +329,12 @@ describe("runTransitousPipeline orchestrator", () => { const dataDir = tmp; const catalogDir = join(dataDir, ".transitous-catalog"); const gtfsDir = join(dataDir, "gtfs"); - mkdirSync(join(catalogDir, ".git"), { recursive: true }); - mkdirSync(join(catalogDir, "feeds"), { recursive: true }); + mkdirSync(join(catalogDir, ".git"), { mode: 0o700, recursive: true }); + mkdirSync(join(catalogDir, "feeds"), { mode: 0o700, recursive: true }); writeFileSync( join(catalogDir, "feeds", "de.json"), JSON.stringify({ sources: [{ name: "A" }, { name: "B" }] }, null, 2), + { mode: 0o600 }, ); const ctx = buildJobContext({ @@ -321,7 +345,7 @@ describe("runTransitousPipeline orchestrator", () => { runScript: async (run) => { if (run.script === "fetch") { // Partial fetch: write A, then error attributable to B. - mkdirSync(gtfsDir, { recursive: true }); + mkdirSync(gtfsDir, { mode: 0o700, recursive: true }); writeFixtureGtfsArchive(join(gtfsDir, "de_a.gtfs.zip")); throw new Error("Error: Could not fetch de-B: HTTP 500"); } diff --git a/services/data-manager/__tests__/transitous/promote.test.ts b/services/data-manager/__tests__/transitous/promote.test.ts index e5ffc785e..6525df073 100644 --- a/services/data-manager/__tests__/transitous/promote.test.ts +++ b/services/data-manager/__tests__/transitous/promote.test.ts @@ -170,34 +170,36 @@ function setupFixture(opts: FixtureOptions): { const stagingDir = join(tmp, "motis", "staging"); const currentDir = join(tmp, "motis", "live"); if (opts.staging) { - mkdirSync(join(stagingDir, "data"), { recursive: true }); - writeFileSync(join(stagingDir, "config.yml"), "server:\n port: 8080\n"); - writeFileSync(join(stagingDir, "data", "tt.bin"), "tt"); + mkdirSync(join(stagingDir, "data"), { mode: 0o700, recursive: true }); + writeFileSync(join(stagingDir, "config.yml"), "server:\n port: 8080\n", { mode: 0o600 }); + writeFileSync(join(stagingDir, "data", "tt.bin"), "tt", { mode: 0o600 }); writeFileSync( join(stagingDir, IMPORT_MARKER_FILE), JSON.stringify({ finishedAt: "2026-05-01T00:00:00.000Z" }), + { mode: 0o600 }, ); } if (opts.stagingWithoutMarker) { - mkdirSync(join(stagingDir, "data"), { recursive: true }); - writeFileSync(join(stagingDir, "config.yml"), "server:\n port: 8080\n"); - writeFileSync(join(stagingDir, "data", "tt.bin"), "tt"); + mkdirSync(join(stagingDir, "data"), { mode: 0o700, recursive: true }); + writeFileSync(join(stagingDir, "config.yml"), "server:\n port: 8080\n", { mode: 0o600 }); + writeFileSync(join(stagingDir, "data", "tt.bin"), "tt", { mode: 0o600 }); } if (opts.stagingJunk) { - mkdirSync(stagingDir, { recursive: true }); - writeFileSync(join(stagingDir, "leftover.log"), "junk"); + mkdirSync(stagingDir, { mode: 0o700, recursive: true }); + writeFileSync(join(stagingDir, "leftover.log"), "junk", { mode: 0o600 }); } if (opts.current) { - mkdirSync(join(currentDir, "data"), { recursive: true }); - writeFileSync(join(currentDir, "data", "tt.bin"), "tt-old"); + mkdirSync(join(currentDir, "data"), { mode: 0o700, recursive: true }); + writeFileSync(join(currentDir, "data", "tt.bin"), "tt-old", { mode: 0o600 }); } if (opts.staging || opts.stagingWithoutMarker) { writeFileSync( join(stagingDir, "config.yml"), "timetable:\n datasets:\n demo:\n path: demo.gtfs.zip\n", + { mode: 0o600 }, ); - writeFileSync(join(stagingDir, "demo.gtfs.zip"), "gtfs"); - writeFileSync(join(stagingDir, "license.json"), "{}\n"); + writeFileSync(join(stagingDir, "demo.gtfs.zip"), "gtfs", { mode: 0o600 }); + writeFileSync(join(stagingDir, "license.json"), "{}\n", { mode: 0o600 }); writeFileSync( join(stagingDir, TRANSIT_SOURCE_MANIFEST_FILENAME), `${JSON.stringify({ @@ -221,11 +223,12 @@ function setupFixture(opts: FixtureOptions): { }, ], })}\n`, + { mode: 0o600 }, ); const proxy = join(stagingDir, CANDIDATE_PROXY_DIRNAME); - mkdirSync(join(proxy, "conf"), { recursive: true }); - writeFileSync(join(proxy, "conf", "default.conf"), "server {}\n"); - writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(proxy, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(proxy, "conf", "default.conf"), "server {}\n", { mode: 0o600 }); + writeFileSync(join(proxy, "feed-proxy-vars.json"), "{}\n", { mode: 0o600 }); const manifest = createCandidateManifest(stagingDir, "test-epoch", "2026-05-01T00:00:00.000Z"); writeFileSync( join(stagingDir, "mobility-capabilities.json"), @@ -239,6 +242,7 @@ function setupFixture(opts: FixtureOptions): { health: { rt: true }, probes: [], })}\n`, + { mode: 0o600 }, ); } return { @@ -302,8 +306,8 @@ function directorySnapshot(root: string): unknown[] { function readyFixture(layout: "rename" | "two-slot") { const fx = setupFixture({ staging: true, current: true }); - mkdirSync(fx.previousDir, { recursive: true }); - writeFileSync(join(fx.previousDir, "retained-timetable"), "older rollback data"); + mkdirSync(fx.previousDir, { mode: 0o700, recursive: true }); + writeFileSync(join(fx.previousDir, "retained-timetable"), "older rollback data", { mode: 0o600 }); globalThis.fetch = vi.fn(async (input: unknown) => { const url = typeof input === "string" ? input : (input as Request | URL).toString(); return jsonResponse(successfulBody(url)); @@ -317,7 +321,7 @@ describe("promotion directory snapshot", () => { it("closes a captured marker descriptor when reading fails", () => { tmp = mkdtempSync(join(tmpdir(), "promotion-snapshot-close-")); const marker = join(tmp, "marker"); - writeFileSync(marker, "original marker"); + writeFileSync(marker, "original marker", { mode: 0o600 }); let descriptor: number | undefined; snapshotRace.afterAccess = (path, opened) => { if (path === marker && opened !== undefined) descriptor = opened; @@ -335,13 +339,13 @@ describe("promotion directory snapshot", () => { tmp = mkdtempSync(join(tmpdir(), "promotion-snapshot-race-")); const root = tmp; const marker = join(root, "marker"); - writeFileSync(marker, "original marker"); + writeFileSync(marker, "original marker", { mode: 0o600 }); const inode = statSync(marker).ino; snapshotRace.afterAccess = (path) => { if (path !== marker) return; snapshotRace.afterAccess = undefined; renameSync(marker, join(root, "captured")); - writeFileSync(marker, "replacement marker"); + writeFileSync(marker, "replacement marker", { mode: 0o600 }); }; expect(directorySnapshot(tmp)).toContainEqual(["marker", inode, "file", "original marker"]); }); @@ -470,8 +474,8 @@ describe("promote stage", () => { opsCalls.push(operation); if (operation.kind === "motis.staging.stop") { renameSync(ctx.motisDataDir, `${ctx.motisDataDir}.externally-moved`); - mkdirSync(ctx.motisDataDir); - writeFileSync(join(ctx.motisDataDir, "external-data"), "replacement"); + mkdirSync(ctx.motisDataDir, { mode: 0o700 }); + writeFileSync(join(ctx.motisDataDir, "external-data"), "replacement", { mode: 0o600 }); externallyChanged = directorySnapshot(fx.dataDir); } return { changed: true, state: "stopped" }; @@ -493,8 +497,8 @@ describe("promote stage", () => { opsCalls.push(operation); if (operation.kind === "motis.staging.stop") { renameSync(ctx.motisDataDir, `${ctx.motisDataDir}.externally-moved`); - mkdirSync(ctx.motisDataDir); - writeFileSync(join(ctx.motisDataDir, "external-data"), "replacement"); + mkdirSync(ctx.motisDataDir, { mode: 0o700 }); + writeFileSync(join(ctx.motisDataDir, "external-data"), "replacement", { mode: 0o600 }); externallyChanged = directorySnapshot(fx.dataDir); throw new Error("staging stop timed out"); } @@ -673,6 +677,7 @@ describe("promote stage", () => { proxyVars: {}, }, })}\n`, + { mode: 0o600 }, ); globalThis.fetch = vi.fn(async (input: unknown) => { const url = typeof input === "string" ? input : (input as Request | URL).toString(); @@ -693,7 +698,9 @@ describe("promote stage", () => { it("rejects a duplicate epoch from authoritative two-slot state", async () => { const fx = setupFixture({ staging: true, current: true }); - writeFileSync(join(fx.currentDir, CANDIDATE_MANIFEST_FILENAME), "not current-contract json\n"); + writeFileSync(join(fx.currentDir, CANDIDATE_MANIFEST_FILENAME), "not current-contract json\n", { + mode: 0o600, + }); const ctx = makeCtx({ dataDir: fx.dataDir, runner: async () => { diff --git a/services/data-manager/__tests__/transitous/proxy-transaction.test.ts b/services/data-manager/__tests__/transitous/proxy-transaction.test.ts index 2ab4d87fd..5d1cf54d2 100644 --- a/services/data-manager/__tests__/transitous/proxy-transaction.test.ts +++ b/services/data-manager/__tests__/transitous/proxy-transaction.test.ts @@ -41,24 +41,32 @@ afterEach(() => { function setup(candidateVars: Record, currentVars: Record) { tmp = mkdtempSync(join(tmpdir(), "openmapx-proxy-transaction-")); const staging = join(tmp, "motis", "staging"); - mkdirSync(join(staging, CANDIDATE_PROXY_DIRNAME, "conf"), { recursive: true }); - writeFileSync(join(staging, "config.yml"), "timetable:\n datasets:\n demo: {}\n"); - writeFileSync(join(staging, "license.json"), "{}\n"); - writeFileSync(join(staging, "demo.gtfs.zip"), "gtfs"); + mkdirSync(join(staging, CANDIDATE_PROXY_DIRNAME, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(staging, "config.yml"), "timetable:\n datasets:\n demo: {}\n", { + mode: 0o600, + }); + writeFileSync(join(staging, "license.json"), "{}\n", { mode: 0o600 }); + writeFileSync(join(staging, "demo.gtfs.zip"), "gtfs", { mode: 0o600 }); writeFileSync( join(staging, TRANSIT_SOURCE_MANIFEST_FILENAME), '{"version":1,"generatedAt":"2026-01-01T00:00:00Z","sources":[]}\n', + { mode: 0o600 }, ); - writeFileSync(join(staging, CANDIDATE_PROXY_DIRNAME, "conf", "default.conf"), "candidate\n"); + writeFileSync(join(staging, CANDIDATE_PROXY_DIRNAME, "conf", "default.conf"), "candidate\n", { + mode: 0o600, + }); writeFileSync( join(staging, CANDIDATE_PROXY_DIRNAME, "feed-proxy-vars.json"), `${JSON.stringify(candidateVars)}\n`, + { mode: 0o600 }, ); createCandidateManifest(staging, "epoch-1", "2026-01-01T00:00:00Z"); const active = join(tmp, "motis-feed-proxy"); - mkdirSync(join(active, "conf"), { recursive: true }); - writeFileSync(join(active, "conf", "default.conf"), "old bytes\n"); - writeFileSync(join(active, "feed-proxy-vars.json"), `${JSON.stringify(currentVars)}\n`); + mkdirSync(join(active, "conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(active, "conf", "default.conf"), "old bytes\n", { mode: 0o600 }); + writeFileSync(join(active, "feed-proxy-vars.json"), `${JSON.stringify(currentVars)}\n`, { + mode: 0o600, + }); return { staging, active }; } diff --git a/services/data-manager/src/jobs/download-osm.ts b/services/data-manager/src/jobs/download-osm.ts index 6666c6982..9e9be09f2 100644 --- a/services/data-manager/src/jobs/download-osm.ts +++ b/services/data-manager/src/jobs/download-osm.ts @@ -102,7 +102,7 @@ export async function downloadOsm(opts: DownloadOsmOptions): Promise renameSync(join(data, "motis/staging"), join(data, "motis/live")), ); } else { - mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); + mkdirSync(join(data, "motis-feed-proxy/conf"), { mode: 0o700, recursive: true }); writeFileSync( join(data, "motis-feed-proxy/conf/default.conf"), "server { listen 80; }\n", + { mode: 0o600 }, ); - writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "{}\n"); + writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "{}\n", { + mode: 0o600, + }); linkProxyFixture(data); writePreparedFixture(job.motisStagingDataDir, context.execution.executionId); const staged = await stageProxy(job); @@ -917,8 +923,8 @@ it ); function writePreparedFixture(staging: string, executionId: string) { - mkdirSync(staging, { recursive: true }); - mkdirSync(join(staging, ".openmapx-feed-proxy/conf"), { recursive: true }); + mkdirSync(staging, { mode: 0o700, recursive: true }); + mkdirSync(join(staging, ".openmapx-feed-proxy/conf"), { mode: 0o700, recursive: true }); for (const [name, body] of [ ["config.yml", "timetable:\n datasets:\n public: test.gtfs.zip\n"], ["license.json", "{}"], @@ -930,7 +936,7 @@ function writePreparedFixture(staging: string, executionId: string) { ], ["test.gtfs.zip", "public-synthetic-dataset"], ]) - writeFileSync(join(staging, name!), body!); + writeFileSync(join(staging, name!), body!, { mode: 0o600 }); createCandidateManifest(staging, executionId, new Date().toISOString()); } function linkProxyFixture(data: string) { @@ -952,9 +958,11 @@ it.each(["before-proof", "before-terminal", "after-terminal"] as const)( async (crash) => { const root = mkdtempSync(join(tmpdir(), "omx-proxy-proof-crash-")), data = join(root, "data"); - mkdirSync(join(data, "motis-feed-proxy/conf"), { recursive: true }); - writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "server { listen 80; }\n"); - writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "{}\n"); + mkdirSync(join(data, "motis-feed-proxy/conf"), { mode: 0o700, recursive: true }); + writeFileSync(join(data, "motis-feed-proxy/conf/default.conf"), "server { listen 80; }\n", { + mode: 0o600, + }); + writeFileSync(join(data, "motis-feed-proxy/feed-proxy-vars.json"), "{}\n", { mode: 0o600 }); linkProxyFixture(data); await control.provisionControlInstallation(root); const dm = control.openControlInstallation({ @@ -1107,7 +1115,7 @@ it.each(["before-proof", "before-terminal", "after-terminal"] as const)( control.verifyFeedProxyCanonicalTerminal(authority, bound.execution), ).toThrow(); } - const jobs = JSON.parse(readFileSync(join(data, "ops-agent/jobs-v1.json"), "utf8")); + const jobs = JSON.parse(readFileSync(join(root, "state/ops-agent/jobs-v1.json"), "utf8")); expect(JSON.stringify(jobs)).toContain(bound.execution.operationKey); await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); expect(existsSync(join(data, ".admin-target-control/locks/motis-feed-proxy"))).toBe(true); diff --git a/services/data-manager/src/jobs/transitous/assemble-staging.ts b/services/data-manager/src/jobs/transitous/assemble-staging.ts index 9a89f6dbf..bda12782e 100644 --- a/services/data-manager/src/jobs/transitous/assemble-staging.ts +++ b/services/data-manager/src/jobs/transitous/assemble-staging.ts @@ -101,7 +101,7 @@ export const run: StageFn = async (ctx) => { const { gtfs, osm } = parseConfigInputs(configText); const stagingDir = ctx.motisStagingDataDir; - mkdirSync(stagingDir, { recursive: true }); + mkdirSync(stagingDir, { recursive: true, mode: 0o700 }); // config.yml — the import-time entrypoint. copyMetadata(configSrc, join(stagingDir, "config.yml")); diff --git a/services/data-manager/src/jobs/transitous/candidate.ts b/services/data-manager/src/jobs/transitous/candidate.ts index 9d47bdeff..0099e4c76 100644 --- a/services/data-manager/src/jobs/transitous/candidate.ts +++ b/services/data-manager/src/jobs/transitous/candidate.ts @@ -173,7 +173,7 @@ function createCandidateManifestWithHashes( }, }; const output = join(stagingDir, CANDIDATE_MANIFEST_FILENAME); - atomicWriteJsonSync(output, manifest, { durability: "full" }); + atomicWriteJsonSync(output, manifest, { durability: "full", mode: 0o600 }); return manifest; } diff --git a/services/data-manager/src/jobs/transitous/compile-gbfs.ts b/services/data-manager/src/jobs/transitous/compile-gbfs.ts index 6ca327ed1..37bf09658 100644 --- a/services/data-manager/src/jobs/transitous/compile-gbfs.ts +++ b/services/data-manager/src/jobs/transitous/compile-gbfs.ts @@ -410,8 +410,8 @@ export const run: StageFn = async (ctx) => { })), }; const output = join(catalogDir, "out", "gbfs-source-index.json"); - mkdirSync(join(catalogDir, "out"), { recursive: true }); - atomicWriteJsonSync(output, index, { durability: "visibility" }); + mkdirSync(join(catalogDir, "out"), { recursive: true, mode: 0o700 }); + atomicWriteJsonSync(output, index, { durability: "visibility", mode: 0o600 }); ctx.state.gbfsCompilation = { output, healthy: healthy.length, diff --git a/services/data-manager/src/jobs/transitous/internal.ts b/services/data-manager/src/jobs/transitous/internal.ts index 12cc5225c..04eaf6261 100644 --- a/services/data-manager/src/jobs/transitous/internal.ts +++ b/services/data-manager/src/jobs/transitous/internal.ts @@ -186,8 +186,8 @@ export function ensureTransitousWorkdirs( gtfsDir: string, downloadsDir: string, ): void { - mkdirSync(gtfsDir, { recursive: true }); - mkdirSync(downloadsDir, { recursive: true }); + mkdirSync(gtfsDir, { recursive: true, mode: 0o700 }); + mkdirSync(downloadsDir, { recursive: true, mode: 0o700 }); const outDir = join(catalogDir, "out"); const cacheDir = join(catalogDir, "downloads"); if (existsSync(outDir)) rmSync(outDir, { recursive: true, force: true }); diff --git a/services/ops-agent/privacy-backup/test-fixture.mjs b/services/ops-agent/privacy-backup/test-fixture.mjs index 3492ec68f..15f55e8c3 100644 --- a/services/ops-agent/privacy-backup/test-fixture.mjs +++ b/services/ops-agent/privacy-backup/test-fixture.mjs @@ -47,6 +47,10 @@ function psql(database, sql) { database, "-v", "ON_ERROR_STOP=1", + // -f makes stdin a script so -1 encloses every migration/seed in a + // transaction, including locks that must span its later statements. + "--single-transaction", + "--file=-", ], { input: sql }, ); diff --git a/services/ops-agent/service.json b/services/ops-agent/service.json index 05f7e76eb..e92dcf267 100644 --- a/services/ops-agent/service.json +++ b/services/ops-agent/service.json @@ -86,7 +86,7 @@ "readOnly": false }, { - "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set — ops-agent renders and applies trusted local stack configuration}/infra/docker", + "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set \u2014 ops-agent renders and applies trusted local stack configuration}/infra/docker", "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker", "readOnly": true }, @@ -110,6 +110,11 @@ "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/backups", "readOnly": false }, + { + "source": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/catalog-pins", + "target": "${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/catalog-pins", + "readOnly": false + }, { "source": "${OPENMAPX_DOCKER_CONFIG_DIR:-/root/.docker}", "target": "/home/node/.docker", From 7afc65b6104593fdf8f55bb088e0d08fad052886 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 31 Oct 2026 03:12:46 +0100 Subject: [PATCH 108/135] fix: resolve actionable admin parity security findings --- apps/ops-agent/package.json | 1 + .../ops-agent/src/artifact-transfer/routes.ts | 6 +- apps/ops-agent/src/ingress-rate-limit.test.ts | 128 ++++++++++++++++++ apps/ops-agent/src/ingress-rate-limit.ts | 28 ++++ apps/ops-agent/src/server.ts | 13 +- packages/cli/__tests__/cli-backup.test.ts | 55 +++++++- packages/cli/src/commands/backup.ts | 23 +++- .../__tests__/container-mounts.test.ts | 34 +++++ .../core/src/services/container-mounts.ts | 4 +- pnpm-lock.yaml | 3 + 10 files changed, 283 insertions(+), 12 deletions(-) create mode 100644 apps/ops-agent/src/ingress-rate-limit.test.ts create mode 100644 apps/ops-agent/src/ingress-rate-limit.ts create mode 100644 packages/core/src/services/__tests__/container-mounts.test.ts diff --git a/apps/ops-agent/package.json b/apps/ops-agent/package.json index 3e6ecd1df..dfd67468a 100644 --- a/apps/ops-agent/package.json +++ b/apps/ops-agent/package.json @@ -12,6 +12,7 @@ "verify:traffic-maintenance": "tsx scripts/verify-traffic-maintenance.ts" }, "dependencies": { + "@fastify/rate-limit": "11.2.1", "@openmapx/core": "workspace:^", "@openmapx/integration-framework": "workspace:^", "@openmapx/transitous-core": "workspace:^", diff --git a/apps/ops-agent/src/artifact-transfer/routes.ts b/apps/ops-agent/src/artifact-transfer/routes.ts index a0b6e726b..54c0f3db5 100644 --- a/apps/ops-agent/src/artifact-transfer/routes.ts +++ b/apps/ops-agent/src/artifact-transfer/routes.ts @@ -3,12 +3,14 @@ import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { verifyArtifactStreamCapability } from "@openmapx/core/ops"; import type { FastifyInstance } from "fastify"; +import { registerOpsIngressRateLimit } from "../ingress-rate-limit"; import { ArtifactTransferError } from "./errors"; import type { ArtifactTransferStore } from "./store"; export function registerArtifactTransferRoutes( app: FastifyInstance, options: { store: ArtifactTransferStore; apiToken: string; admissionsOpen?: () => boolean }, ): void { + registerOpsIngressRateLimit(app); app.register(async (scoped) => { scoped.addContentTypeParser("application/octet-stream", (_request, payload, done) => done(null, payload), @@ -56,8 +58,8 @@ export function registerArtifactTransferRoutes( .code( error instanceof ArtifactTransferError ? error.statusCode - : (error as { statusCode?: number }).statusCode === 415 - ? 415 + : [415, 429].includes((error as { statusCode?: number }).statusCode ?? 0) + ? (error as { statusCode: number }).statusCode : 503, ) .send({ diff --git a/apps/ops-agent/src/ingress-rate-limit.test.ts b/apps/ops-agent/src/ingress-rate-limit.test.ts new file mode 100644 index 000000000..1eb6274e2 --- /dev/null +++ b/apps/ops-agent/src/ingress-rate-limit.test.ts @@ -0,0 +1,128 @@ +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Fastify from "fastify"; +import { afterEach, expect, it, vi } from "vitest"; +import { registerArtifactTransferRoutes } from "./artifact-transfer/routes"; +import { ArtifactTransferStore } from "./artifact-transfer/store"; +import { buildOpsAgentServer } from "./server"; + +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + vi.useRealTimers(); + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); + +async function transferStore() { + const root = await mkdtemp(join(tmpdir(), "ops-ingress-rate-")); + cleanups.push(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, "data"), { mode: 0o700 }); + await mkdir(join(root, "private"), { mode: 0o700 }); + const store = new ArtifactTransferStore({ + dataRoot: join(root, "data"), + privateRoot: join(root, "private"), + token: "a".repeat(43), + capacity: () => ({ bytes: 2 * 1024 ** 3, inodes: 10000 }), + }); + cleanups.push(async () => store.close()); + return store; +} + +it("limits standalone artifact methods before bearer, capability, store and body work despite forged forwarding headers", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const store = await transferStore(); + const stageReads = vi.spyOn(store, "getStage"); + const app = Fastify({ trustProxy: true }); + cleanups.push(() => app.close()); + let tokenReads = 0; + let parsedBodies = 0; + app.addHook("preParsing", async (_request, _reply, payload) => { + parsedBodies++; + return payload; + }); + registerArtifactTransferRoutes(app, { + store, + get apiToken() { + tokenReads++; + return "a".repeat(43); + }, + }); + const path = "/v1/artifact-transfers/123e4567-e89b-42d3-a456-426614174000/content"; + for (let index = 0; index < 6000; index++) { + const response = await app.inject({ + method: (["GET", "HEAD", "PUT"] as const)[index % 3], + url: path, + headers: { "x-forwarded-for": `203.0.${Math.floor(index / 255)}.${index % 255}` }, + }); + expect(response.statusCode).toBe(401); + } + const before = tokenReads; + for (const method of ["GET", "HEAD", "PUT"] as const) { + const response = await app.inject({ + method, + url: path, + headers: { + authorization: `Bearer ${"a".repeat(43)}`, + "x-openmapx-artifact-capability": "invalid-capability", + "x-forwarded-for": "198.51.100.254", + ...(method === "PUT" ? { "content-type": "application/json" } : {}), + }, + ...(method === "PUT" ? { payload: "{" } : {}), + }); + expect(response.statusCode).toBe(429); + expect(Number(response.headers["retry-after"])).toBeGreaterThan(0); + } + expect(tokenReads).toBe(before); + expect(stageReads).not.toHaveBeenCalled(); + expect(parsedBodies).toBe(0); + expect((await app.inject({ url: path, remoteAddress: "127.0.0.2" })).statusCode).toBe(401); + vi.setSystemTime(Date.now() + 60000); + expect((await app.inject({ url: path })).statusCode).toBe(401); +}); + +it("shares the actual server ingress budget across operations and artifact routes before authentication and JSON parsing", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const store = await transferStore(); + let tokenReads = 0; + const app = buildOpsAgentServer({ + tokens: { + get api() { + tokenReads++; + return "a".repeat(43); + }, + "data-manager": "b".repeat(43), + }, + artifactTransfers: store, + }); + cleanups.push(() => app.close()); + for (let index = 0; index < 6000; index++) { + const response = await app.inject({ method: "POST", url: "/v1/operations", payload: {} }); + expect(response.statusCode).toBe(401); + } + const before = tokenReads; + const operation = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${"a".repeat(43)}`, "content-type": "application/json" }, + payload: "{", + }); + expect(operation.statusCode).toBe(429); + expect(Number(operation.headers["retry-after"])).toBeGreaterThan(0); + const authenticated = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${"a".repeat(43)}` }, + payload: {}, + }); + expect(authenticated.statusCode).toBe(429); + expect(tokenReads).toBe(before); + const stream = await app.inject({ + url: "/v1/artifact-transfers/123e4567-e89b-42d3-a456-426614174000/content", + }); + expect(stream.statusCode).toBe(429); + expect(Number(stream.headers["retry-after"])).toBeGreaterThan(0); + vi.setSystemTime(Date.now() + 60000); + expect( + (await app.inject({ method: "POST", url: "/v1/operations", payload: {} })).statusCode, + ).toBe(401); +}); diff --git a/apps/ops-agent/src/ingress-rate-limit.ts b/apps/ops-agent/src/ingress-rate-limit.ts new file mode 100644 index 000000000..a2cb323e1 --- /dev/null +++ b/apps/ops-agent/src/ingress-rate-limit.ts @@ -0,0 +1,28 @@ +import rateLimit, { normalizeIP } from "@fastify/rate-limit"; +import type { FastifyInstance } from "fastify"; + +const installed = new WeakSet(); + +/** One bounded peer budget, charged before authentication or body parsing. */ +export function registerOpsIngressRateLimit(app: FastifyInstance): void { + if (installed.has(app)) return; + installed.add(app); + let limit: ReturnType | undefined; + app.register(rateLimit, { + global: false, + max: 6000, + timeWindow: 60000, + cache: 1000, + keyGenerator: (request) => normalizeIP(request.raw.socket.remoteAddress ?? "unknown"), + }); + void app.after((error) => { + if (error) throw error; + limit = app.rateLimit(); + }); + // The plugin's automatic onRoute hook appends after existing onRequest + // authentication hooks. Install its actual handler here instead, once ready. + app.addHook("onRequest", async function (request, reply) { + if (!limit) throw Error("Ops ingress rate limiter unavailable"); + await limit.call(this, request, reply); + }); +} diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 7dbbc7294..5cc8d0e91 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -158,6 +158,7 @@ import { type DawarichSubjectExportRouteOptions, registerDawarichSubjectExportRoute, } from "./dawarich-subject-export"; +import { registerOpsIngressRateLimit } from "./ingress-rate-limit"; import type { HistoricalPullJob, OpsJobJournal, PersistedOpsJob } from "./journal"; import { type CoordinatorPullOwnershipObservation, @@ -452,6 +453,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif requestTimeout: 0, connectionTimeout: DEFAULT_ADMISSION_TIMEOUT_MS, }); + registerOpsIngressRateLimit(app); app.addHook("onRequest", async (request, reply) => { const streaming = (request.routeOptions.config as { artifactTransferBytes?: boolean }).artifactTransferBytes === @@ -1804,10 +1806,17 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }; app.setErrorHandler((error, request, reply) => { - const statusCode = (error as { statusCode?: number }).statusCode === 413 ? 413 : 400; + const limited = (error as { statusCode?: number }).statusCode === 429; + const statusCode = limited + ? 429 + : (error as { statusCode?: number }).statusCode === 413 + ? 413 + : 400; return reply .code(statusCode) - .send(redactedOpsError(requestIdFromBody(request.body), "validation", error)); + .send( + redactedOpsError(requestIdFromBody(request.body), limited ? "busy" : "validation", error), + ); }); app.get("/v1/protocol", async (request, reply) => { const requestId = validRequestId(request.headers["x-ops-request-id"]); diff --git a/packages/cli/__tests__/cli-backup.test.ts b/packages/cli/__tests__/cli-backup.test.ts index 93f51ddfc..64807a928 100644 --- a/packages/cli/__tests__/cli-backup.test.ts +++ b/packages/cli/__tests__/cli-backup.test.ts @@ -1,17 +1,19 @@ import { createHash } from "node:crypto"; -import { +import filesystem, { existsSync, linkSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, + renameSync, rmSync, statSync, symlinkSync, utimesSync, writeFileSync, } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { Readable } from "node:stream"; @@ -499,6 +501,57 @@ describe("readBackupManifest", () => { expect(() => readBackupManifest(hardlink)).toThrow(/private regular file/); }); + it.each(["symlink", "regular file"] as const)( + "does not consume a %s substituted after observing the original manifest", + (replacement) => { + const dir = writeBackup("trusted", { + name: "trusted", + createdAt: "2026-04-19T00:00:00Z", + services: [], + }); + const foreignDir = writeBackup("foreign", { + name: "foreign", + createdAt: "2026-04-19T00:00:00Z", + services: [], + }); + const manifest = join(dir, "manifest.json"), + foreignManifest = join(foreignDir, "manifest.json"), + original = statSync(manifest), + nativeLstat = filesystem.lstatSync, + nativeFstat = filesystem.fstatSync; + let replaced = false; + // Perform real replacement after the native observation returns. The legacy + // reader observed a pathname; the fixed reader observes its already-open FD. + const replaceObservedManifest = (stat: { dev: number | bigint; ino: number | bigint }) => { + if (replaced || stat.dev !== original.dev || stat.ino !== original.ino) return; + replaced = true; + renameSync(manifest, join(dir, "original-manifest.json")); + if (replacement === "symlink") symlinkSync(foreignManifest, manifest); + else renameSync(foreignManifest, manifest); + }; + const lstat = vi.spyOn(filesystem, "lstatSync").mockImplementation((path, options) => { + const stat = nativeLstat(path, options); + if (stat) replaceObservedManifest(stat); + return stat; + }); + const fstat = vi.spyOn(filesystem, "fstatSync").mockImplementation((fd, options) => { + const stat = nativeFstat(fd, options); + replaceObservedManifest(stat); + return stat; + }); + syncBuiltinESMExports(); + try { + expect(readBackupManifest(manifest).name).toBe("trusted"); + expect(replaced).toBe(true); + expect(JSON.parse(readFileSync(manifest, "utf8")).name).toBe("foreign"); + } finally { + lstat.mockRestore(); + fstat.mockRestore(); + syncBuiltinESMExports(); + } + }, + ); + it("publishes a private manifest atomically and flushes temporary names", () => { const dir = join(tmp, "infra", "docker", "backups", "atomic"); mkdirSync(dir, { recursive: true }); diff --git a/packages/cli/src/commands/backup.ts b/packages/cli/src/commands/backup.ts index 552f3512a..34cacab1c 100644 --- a/packages/cli/src/commands/backup.ts +++ b/packages/cli/src/commands/backup.ts @@ -6,6 +6,7 @@ import { createReadStream, createWriteStream, existsSync, + fstatSync, fsyncSync, lstatSync, mkdirSync, @@ -209,16 +210,26 @@ export function isCompatiblePlatformVersion( * every field is shape-checked here rather than only at the sinks. */ export function readBackupManifest(filePath: string): BackupManifest { - let manifestStat: ReturnType; + let fd: number; try { - manifestStat = lstatSync(filePath); - } catch { + fd = openSync(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ELOOP") { + throw new Error(`Backup manifest is not a private regular file: ${filePath}`); + } throw new Error(`Backup manifest not found: ${filePath}`); } - if (!manifestStat.isFile() || manifestStat.isSymbolicLink() || manifestStat.nlink !== 1) { - throw new Error(`Backup manifest is not a private regular file: ${filePath}`); + let raw: Partial; + try { + // Validate and consume the same inode even if the directory entry is replaced. + const manifestStat = fstatSync(fd); + if (!manifestStat.isFile() || manifestStat.nlink !== 1) { + throw new Error(`Backup manifest is not a private regular file: ${filePath}`); + } + raw = JSON.parse(readFileSync(fd, "utf-8")) as Partial; + } finally { + closeSync(fd); } - const raw = JSON.parse(readFileSync(filePath, "utf-8")) as Partial; if ( !raw || typeof raw.name !== "string" || diff --git a/packages/core/src/services/__tests__/container-mounts.test.ts b/packages/core/src/services/__tests__/container-mounts.test.ts new file mode 100644 index 000000000..134332f33 --- /dev/null +++ b/packages/core/src/services/__tests__/container-mounts.test.ts @@ -0,0 +1,34 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; +import { containersOnCheckoutServiceFiles } from "../container-mounts"; + +it("normalizes trailing slashes without changing checkout mount classification", () => { + expect( + containersOnCheckoutServiceFiles("/srv/openmapx///", [ + { name: "engine", sources: ["/srv/openmapx/services/engine/config"] }, + { name: "dynamic", sources: ["/srv/openmapx/services/traefik/config/dynamic/file"] }, + { name: "other", sources: ["/different/services/config"] }, + ]), + ).toEqual(["engine"]); +}); + +it("bounds work on long non-matching slash runs", async () => { + const code = ` + const { containersOnCheckoutServiceFiles } = await import(process.argv[1]); + containersOnCheckoutServiceFiles("/".repeat(200000) + "suffix", []); + `; + await expect( + promisify(execFile)( + process.execPath, + [ + "--experimental-strip-types", + "--input-type=module", + "--eval", + code, + new URL("../container-mounts.ts", import.meta.url).href, + ], + { timeout: 2000 }, + ), + ).resolves.toBeDefined(); +}); diff --git a/packages/core/src/services/container-mounts.ts b/packages/core/src/services/container-mounts.ts index 5f2a75123..341ceff0e 100644 --- a/packages/core/src/services/container-mounts.ts +++ b/packages/core/src/services/container-mounts.ts @@ -42,7 +42,9 @@ export function containersOnCheckoutServiceFiles( rootDir: string, mounts: readonly ContainerBindMounts[], ): string[] { - const services = `${rootDir.replace(/\/+$/, "")}/services/`; + let end = rootDir.length; + while (end > 0 && rootDir.charCodeAt(end - 1) === 47) end--; + const services = `${rootDir.slice(0, end)}/services/`; const traefikDynamic = `${services}traefik/config/dynamic`; return mounts .filter((mount) => diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51fde4162..65893faa5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -536,6 +536,9 @@ importers: apps/ops-agent: dependencies: + '@fastify/rate-limit': + specifier: 11.2.1 + version: 11.2.1 '@openmapx/core': specifier: workspace:^ version: link:../../packages/core From 46a3984b3ec1a21059c42cd82a64bfc7e3f979f0 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 31 Oct 2026 03:31:41 +0100 Subject: [PATCH 109/135] fix: preserve runtime health and ownership across host layouts --- .../__tests__/user-erasure-postgres.test.ts | 2 + ...orm-configuration-runtime-postgres.test.ts | 36 ++++++---- .../retained-maintenance-job-postgres.test.ts | 8 +-- .../services/integration-application.test.ts | 8 ++- .../src/__fixtures__/owned-service-host.ts | 14 +++- .../__fixtures__/production-compose-config.ts | 21 ++++++ .../src/__fixtures__/production-docker-io.mjs | 16 +++++ .../src/artifact-transfer/store.test.ts | 18 ++++- apps/ops-agent/src/docker-runtime.test.ts | 30 +++++++-- apps/ops-agent/src/docker-runtime.ts | 13 +++- .../ops-agent/src/lifecycle-exclusion.test.ts | 65 +++++++++++++++++++ apps/ops-agent/src/lifecycle-exclusion.ts | 15 ++++- apps/ops-agent/src/production-runtime.test.ts | 41 +++++++++--- ...ackup-restore-postgres.integration.test.ts | 53 ++++++++++++--- .../artifact-publication-process.ts | 10 +-- .../src/__tests__/installer-streaming.test.ts | 36 ++++++++++ .../integration-framework/src/installer.ts | 11 +++- scripts/ops-agent-deployment-user.test.ts | 27 ++++++-- scripts/ops-agent-transitous-pin.test.ts | 8 +++ scripts/ops-authority-inventory.json | 1 + vitest.setup.node.ts | 5 ++ 21 files changed, 375 insertions(+), 63 deletions(-) create mode 100644 apps/ops-agent/src/__fixtures__/production-compose-config.ts create mode 100755 apps/ops-agent/src/__fixtures__/production-docker-io.mjs create mode 100644 apps/ops-agent/src/lifecycle-exclusion.test.ts diff --git a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts index 39178246f..c570ad5c3 100644 --- a/apps/api/src/db/__tests__/user-erasure-postgres.test.ts +++ b/apps/api/src/db/__tests__/user-erasure-postgres.test.ts @@ -55,6 +55,7 @@ describe.skipIf(skipDatabase)("user erasure constraints with PostgreSQL", () => "admin_job.created_by:n", "auth_policy_application.actor_id:n", "auth_policy_state.activated_by:n", + "configuration_migration_preview.actor_id:c", "data_disclosure_event.user_id:n", "data_export_reauthentication.initiating_admin_user_id:n", "data_export_reauthentication.user_id:n", @@ -81,6 +82,7 @@ describe.skipIf(skipDatabase)("user erasure constraints with PostgreSQL", () => "passkey.user_id:c", "personal_timeline_connection.user_id:c", "personal_vehicle.user_id:c", + "platform_configuration_preview.actor_id:c", "saved_list.user_id:c", "service_secret.updated_by:n", "session.user_id:c", diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index ec72711b6..ba59abecc 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -405,6 +405,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "owned platform preparation", () => { beforeEach(async () => { + // Each scenario is an independent deployment/session, with a fresh limiter bucket. + (await import("../../utils/rate-limit")).systemMaintenanceLimit.reset(); state.fixture = await createMigratedPostgresFixture(); }, 90000); afterEach(async () => { @@ -548,9 +550,12 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( releaseId: manifest.release, manifestDigest: createHash("sha256").update(raw).digest("hex"), }; - mkdirSync(join(host.infraDir, ".ops-agent-releases"), { recursive: true, mode: 0o700 }); + mkdirSync(services.releaseStatePaths(host.infraDir).store, { + recursive: true, + mode: 0o700, + }); writeFileSync( - join(host.infraDir, ".ops-agent-releases", `${manifest.release}.json`), + join(services.releaseStatePaths(host.infraDir).store, `${manifest.release}.json`), raw, { mode: 0o600 }, ); @@ -590,7 +595,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ) { await services.withConfigurationGenerationWriter(host.infraDir, async () => { const lock = await services.acquireReleaseStoreLock( - join(host.infraDir, ".ops-agent-releases"), + services.releaseStatePaths(host.infraDir).store, {}, { failIfBusy: true }, ); @@ -1073,9 +1078,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(host.calls.some((args) => args[0] === "compose" && args.includes("up"))).toBe( false, ); - expect(existsSync(join(host.infraDir, ".ops-agent-releases/current.json"))).toBe( - false, - ); + expect( + existsSync(join(services.releaseStatePaths(host.infraDir).store, "current.json")), + ).toBe(false); return; } if (version === "launch-native-system-release-agent-restart") { @@ -1140,18 +1145,18 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .filter((args) => args[0] === "compose" && args.includes("up")) .map((args) => args.at(-1)), ).toEqual(["data-manager"]); - expect(existsSync(join(host.infraDir, ".ops-agent-releases/current.json"))).toBe( - false, - ); + expect( + existsSync(join(services.releaseStatePaths(host.infraDir).store, "current.json")), + ).toBe(false); return; } if (version === "launch-native-release-wrong-config") { await jobRunner.reconcile(); expect((await readJob()).status).toBe("running"); expect(dm.settlePosts).toBe(0); - expect(existsSync(join(host.infraDir, ".ops-agent-releases", "current.json"))).toBe( - false, - ); + expect( + existsSync(join(services.releaseStatePaths(host.infraDir).store, "current.json")), + ).toBe(false); expect( host.calls .filter((args) => args[0] === "compose" && args.includes("up")) @@ -1231,7 +1236,10 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ).toBe(true); expect( JSON.parse( - readFileSync(join(host.infraDir, ".ops-agent-releases", "current.json"), "utf8"), + readFileSync( + join(services.releaseStatePaths(host.infraDir).store, "current.json"), + "utf8", + ), ), ).toMatchObject({ releaseId: checkedRelease.releaseId, @@ -1512,7 +1520,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( new AbortController().signal, ), lock = await services.acquireReleaseStoreLock( - join(host.infraDir, ".ops-agent-releases"), + services.releaseStatePaths(host.infraDir).store, ); const callsBeforeInspection = host.calls.length; try { diff --git a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts index d002315fb..e3b3a5e87 100644 --- a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts +++ b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts @@ -798,8 +798,8 @@ dbDescribe("retained finite maintenance requests", () => { }, }), ); - const overlay = join(host.infraDir, "docker-compose.release.yml"); - services.writeReleaseComposeArtifacts(manifest, overlay); + const overlay = services.releaseStatePaths(host.infraDir).overlay; + services.writeReleaseComposeArtifacts(manifest, host.infraDir); const tokenFile = join(host.root, "clear-api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); const agentUrl = await agent.listen({ host: "127.0.0.1", port: 0 }); @@ -928,7 +928,7 @@ dbDescribe("retained finite maintenance requests", () => { })), ); host.setRows(rows); - const overlay = join(host.infraDir, "docker-compose.release.yml"); + const overlay = services.releaseStatePaths(host.infraDir).overlay; if (pinned) services.writeReleaseComposeArtifacts( services.parseReleaseManifest( @@ -953,7 +953,7 @@ dbDescribe("retained finite maintenance requests", () => { }, }), ), - overlay, + host.infraDir, ); const beforeOverlay = existsSync(overlay) ? readFileSync(overlay, "utf8") : null; const execFile: typeof host.execFile = async (file, argv, io) => { diff --git a/apps/api/src/services/integration-application.test.ts b/apps/api/src/services/integration-application.test.ts index 64c65682c..c5a62ee21 100644 --- a/apps/api/src/services/integration-application.test.ts +++ b/apps/api/src/services/integration-application.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { mkdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; import { services } from "@openmapx/core/server"; @@ -476,7 +476,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( it("locks real service host fields and resource knobs, then permits explicit adoption after removal", async () => { vi.stubEnv("SERVICE_MOTIS_REGION", "host-region"); vi.stubEnv("SERVICE_MOTIS_TOKEN", "host-private"); - writeFileSync(join(host.infraDir, ".env"), "MOTIS_SLOT_MEMORY=8g\n"); + const envPath = join(host.infraDir, ".env"), + deploymentEnv = readFileSync(envPath, "utf8"); + writeFileSync(envPath, `${deploymentEnv}MOTIS_SLOT_MEMORY=8g\n`); const view = await api.inject({ url: "/api/admin/services/motis/config", headers }); expect(view.statusCode, view.body).toBe(200); expect(view.json().ownership.REGION).toMatchObject({ source: "env", writable: false }); @@ -515,7 +517,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(await state.fixture.sql`SELECT * FROM service_secret`).toHaveLength(0); vi.stubEnv("SERVICE_MOTIS_REGION", ""); vi.stubEnv("SERVICE_MOTIS_TOKEN", ""); - rmSync(join(host.infraDir, ".env")); + writeFileSync(envPath, deploymentEnv); (await import("../utils/rate-limit")).serviceActionLimit.reset(); const adopted = await api.inject({ method: "POST", diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index 7d367d7f1..d1cb38e20 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -33,7 +33,7 @@ import { testExecution } from "./execution"; import { createDefaultFixedCli } from "../administrative-runtime"; import { runContainedProcess } from "../docker-runtime"; -export async function ownedServiceHost(options: { configurationReleaseMetadata?:boolean; configurationHelperReleaseInventory?:boolean; configurationHelperLaunchResponseLoss?:boolean; selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean } = {}) { +export async function ownedServiceHost(options: { configurationReleaseMetadata?:boolean; configurationHelperReleaseInventory?:boolean; configurationHelperLaunchResponseLoss?:boolean; selectedRoots?:string[]; configurationHelperVersion?:string; environment?:Record>; resources?:Record; capabilities?:Record; extraServiceIds?:string[]; coordinatorDependency?:boolean; oversizedMaintenancePlan?:boolean; nestedRootSegments?:number; imageReferences?:Record; schemas?: Record>; withHardlinks?: boolean; withCoordinator?: boolean; defaultCliEntry?: string; competeAfterDefaultCli?: boolean; actualComposeConfig?: boolean; composeConfigIO?: DockerRuntimeOptions["execFile"] } = {}) { const withCoordinator = options.withHardlinks || options.withCoordinator; const cleanupRoot = mkdtempSync(join(tmpdir(), "openmapx-owned-")); const root=join(cleanupRoot,...Array.from({length:options.nestedRootSegments??0},(_,index)=>`${index}${"x".repeat(79)}`)); @@ -81,6 +81,15 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: mkdirSync(service.directory, { recursive: true, mode: 0o755 }); writeFileSync(join(service.directory, "service.json"), JSON.stringify(service.manifest), { mode: 0o644 }); } + // Selecting Traefik requires its actual release-owned static configuration. + // Seed source bytes before authority capture and generation validation. + const traefikConfig = join(root, "services", "traefik", "config"); + mkdirSync(traefikConfig, { recursive: true, mode: 0o755 }); + writeFileSync( + join(traefikConfig, "traefik.yml"), + readFileSync(new URL("../../../../services/traefik/config/traefik.yml", import.meta.url)), + { mode: 0o644 }, + ); const releaseInventory = options.configurationHelperReleaseInventory ? await services.captureReleaseServiceAuthority(root) : undefined; const loadAuthority = createTrustedConfigurationAuthorityLoader({ rootDir: root, @@ -127,6 +136,7 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: let runtimeSecretBytes: Record = {}; const calls: string[][] = []; const actualComposeConfig = options.actualComposeConfig; + const composeConfigIO = options.composeConfigIO; const configurationHelperVersion=options.configurationHelperVersion; let creatingProcess: | ((args: readonly string[], signal: AbortSignal) => Promise) @@ -180,7 +190,7 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: } else if (args[0] === "compose" && args.includes("ps")) stdout = JSON.stringify(rows.map(row => ({ Service: row.Labels["com.docker.compose.service"], State: row.State, Health: "" }))); else if (args[0] === "compose" && args.includes("config")) - stdout = actualComposeConfig ? (await runContainedProcess("docker", args, options)).stdout : JSON.stringify({ + stdout = composeConfigIO ? (await composeConfigIO("docker", args, options)).stdout : actualComposeConfig ? (await runContainedProcess("docker", args, options)).stdout : JSON.stringify({ services: Object.fromEntries( loaded.map((s) => [ s.manifest.id, diff --git a/apps/ops-agent/src/__fixtures__/production-compose-config.ts b/apps/ops-agent/src/__fixtures__/production-compose-config.ts new file mode 100644 index 000000000..254967bd5 --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/production-compose-config.ts @@ -0,0 +1,21 @@ +import { readFileSync } from "node:fs"; +import { parse } from "yaml"; + +/** Named external Compose read, from the test's actual generated files and overlay. */ +export function readProductionFixtureCompose(args: readonly string[]) { + if (args[0] !== "compose" || !args.includes("config")) + throw Error("Expected a named Compose configuration read"); + const services: Record> = {}; + let files = 0; + for (let index = 0; index < args.length; index++) { + if (args[index] !== "-f") continue; + const source = parse(readFileSync(args[++index]!, "utf8")) as { + services?: Record>; + }; + for (const [id, definition] of Object.entries(source.services ?? {})) + services[id] = { ...services[id], ...definition }; + files++; + } + if (!files || !Object.keys(services).length) throw Error("Fixture Compose input unavailable"); + return { services }; +} diff --git a/apps/ops-agent/src/__fixtures__/production-docker-io.mjs b/apps/ops-agent/src/__fixtures__/production-docker-io.mjs new file mode 100755 index 000000000..a4c1a57bc --- /dev/null +++ b/apps/ops-agent/src/__fixtures__/production-docker-io.mjs @@ -0,0 +1,16 @@ +#!/usr/bin/env node +import { appendFileSync } from "node:fs"; +import { readProductionFixtureCompose } from "./production-compose-config.ts"; + +const args = process.argv.slice(2); +if (process.env.OMX_PRODUCTION_TEST_DOCKER_TRACE) + appendFileSync(process.env.OMX_PRODUCTION_TEST_DOCKER_TRACE, JSON.stringify(args) + "\n"); +if (args[0] === "compose" && args.includes("config")) + process.stdout.write(JSON.stringify(readProductionFixtureCompose(args))); +else if (process.env.OMX_PRODUCTION_TEST_DOCKER_READ_ONLY === "1") { + if (args[0] === "container" && args[1] === "ls") process.stdout.write(""); + else if (args[0] === "ps") process.stdout.write(""); + else if (args[0] === "compose" && args.includes("ps")) process.stdout.write("[]"); + else if (args[0] === "info") process.stdout.write("fixture-empty-host"); + else throw Error("Unexpected effect at the fixture's read-only Docker boundary"); +} diff --git a/apps/ops-agent/src/artifact-transfer/store.test.ts b/apps/ops-agent/src/artifact-transfer/store.test.ts index e91d6d204..1a669459a 100644 --- a/apps/ops-agent/src/artifact-transfer/store.test.ts +++ b/apps/ops-agent/src/artifact-transfer/store.test.ts @@ -401,8 +401,17 @@ it("closes new admissions during the existing drain without dropping the active stores.push(store); const stage = await store.createStage({ ...input(), declaredBytes: 1 }); const abort = new AbortController(); + let bodyPulled!: () => void; + const pulling = new Promise((resolve) => { + bodyPulled = resolve; + }); const body = new ReadableStream( - { pull: () => new Promise(() => {}) }, + { + pull() { + bodyPulled(); + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }, ); const pending = store @@ -416,7 +425,12 @@ it("closes new admissions during the existing drain without dropping the active signal: abort.signal, }) .catch((error) => error); - await new Promise((r) => setTimeout(r, 10)); + await Promise.race([ + pulling, + pending.then((error) => { + throw error; + }), + ]); open = false; await expect(store.createStage(input())).rejects.toMatchObject({ statusCode: 503 }); expect(store.activeStreams).toBe(1); diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index a0149200b..cdc14d219 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -1904,9 +1904,9 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done async (name) => { await expect(stackStatus(captured(name))).resolves.toEqual({ services: [ - { serviceId: "crashloop", state: "restarting" }, - { serviceId: "web", state: "running" }, - { serviceId: "worker", state: "paused" }, + { serviceId: "crashloop", state: "restarting", health: "none" }, + { serviceId: "web", state: "running", health: "none" }, + { serviceId: "worker", state: "paused", health: "none" }, ], }); }, @@ -1926,7 +1926,29 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done ); const result = await stackStatus(`${lines.join("\n")}\n`); expect(result.services).toHaveLength(256); - expect(result.services.at(-1)).toEqual({ serviceId: "service-255", state: "running" }); + expect(result.services.at(-1)).toEqual({ + serviceId: "service-255", + state: "running", + health: "unknown", + }); + }); + + it.each([ + ["healthy", "healthy"], + ["unhealthy", "unhealthy"], + ["starting", "starting"], + ["none", "none"], + ["unknown", "unknown"], + ["", "none"], + ["unexpected", "unknown"], + [null, "unknown"], + [undefined, "unknown"], + ])("preserves Docker health %j as %s for readiness", async (health, expected) => { + await expect( + stackStatus(JSON.stringify({ Service: "data-manager", State: "running", Health: health })), + ).resolves.toEqual({ + services: [{ serviceId: "data-manager", state: "running", health: expected }], + }); }); it.each([ diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 5f18d5a64..45979fd91 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -434,7 +434,18 @@ function parseStackStatus(stdout: string): OpsResultFor<"stack.status"> { const state = ["running", "stopped", "created", "restarting", "paused"].includes(rawState) ? (rawState as "running" | "stopped" | "created" | "restarting" | "paused") : "unknown"; - return { serviceId: String(item.Service ?? item.Name ?? "unknown"), state }; + const rawHealth = item.Health; + const health = + rawHealth === "" + ? "none" + : rawHealth === "healthy" || + rawHealth === "unhealthy" || + rawHealth === "starting" || + rawHealth === "none" || + rawHealth === "unknown" + ? rawHealth + : "unknown"; + return { serviceId: String(item.Service ?? item.Name ?? "unknown"), state, health }; }), }; } diff --git a/apps/ops-agent/src/lifecycle-exclusion.test.ts b/apps/ops-agent/src/lifecycle-exclusion.test.ts new file mode 100644 index 000000000..79617a5f0 --- /dev/null +++ b/apps/ops-agent/src/lifecycle-exclusion.test.ts @@ -0,0 +1,65 @@ +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { services } from "@openmapx/core/server"; +import { afterEach, expect, it } from "vitest"; +import { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) { + chmodSync(root, 0o700); + rmSync(root, { recursive: true, force: true }); + } +}); +function fixture() { + const infra = mkdtempSync(join(tmpdir(), "ops-lifecycle-state-")); + roots.push(infra); + const state = services.agentStatePaths(infra).directory; + mkdirSync(state, { recursive: true, mode: 0o700 }); + return { infra, state }; +} +it("retains an unresolved lifecycle fence through restart below writable state while infrastructure is read-only", async () => { + const { infra, state } = fixture(); + chmodSync(infra, 0o555); + const lifecycle = new ServiceLifecycleExclusion(infra); + await expect( + lifecycle.creating(["alpha"], new AbortController().signal, async (begin) => { + begin(); + throw Error("unknown external completion"); + }), + ).rejects.toThrow("unknown external completion"); + const pending = join(state, "service-lifecycle/pending.json"); + expect(readFileSync(pending, "utf8")).toBe('["alpha"]'); + expect(statSync(pending).mode & 0o777).toBe(0o600); + expect(existsSync(join(infra, ".service-lifecycle"))).toBe(false); + const restored = new ServiceLifecycleExclusion(infra); + await expect( + restored.run(["alpha"], new AbortController().signal, async () => "effect"), + ).rejects.toThrow("settlement is unresolved"); + await expect( + restored.run(["beta"], new AbortController().signal, async () => "unrelated"), + ).resolves.toBe("unrelated"); + chmodSync(pending, 0o644); + expect(() => new ServiceLifecycleExclusion(infra)).toThrow("settlement is unresolved"); +}); +it("refuses legacy lifecycle state intact instead of resetting an older unresolved fence", () => { + const { infra } = fixture(); + const legacy = join(infra, ".service-lifecycle"); + mkdirSync(legacy, { mode: 0o700 }); + const pending = join(legacy, "pending.json"); + writeFileSync(pending, '["alpha"]', { mode: 0o600 }); + const before = statSync(pending); + expect(() => new ServiceLifecycleExclusion(infra)).toThrow("settlement is unresolved"); + expect(readFileSync(pending, "utf8")).toBe('["alpha"]'); + expect(statSync(pending).ino).toBe(before.ino); +}); diff --git a/apps/ops-agent/src/lifecycle-exclusion.ts b/apps/ops-agent/src/lifecycle-exclusion.ts index 211ff4c4e..7b41e97d9 100644 --- a/apps/ops-agent/src/lifecycle-exclusion.ts +++ b/apps/ops-agent/src/lifecycle-exclusion.ts @@ -12,6 +12,7 @@ import { writeFileSync, } from "node:fs"; import { join } from "node:path"; +import { services } from "@openmapx/core/server"; import { isContainedProcessFailure } from "./docker-runtime"; interface Waiter { @@ -33,10 +34,20 @@ export class ServiceLifecycleExclusion { private directory: string; private file: string; constructor(infraDir: string) { - this.directory = join(infraDir, ".service-lifecycle"); + // An older agent may have retained an unresolved effect here. A new, + // empty canonical store must never silently discard that denial. + try { + lstatSync(join(infraDir, ".service-lifecycle")); + throw Error(FAILED); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const state = services.agentStatePaths(infraDir).directory; + mkdirSync(state, { recursive: true, mode: 0o700 }); + this.directory = join(state, "service-lifecycle"); try { mkdirSync(this.directory, { mode: 0o700 }); - const parent = openSync(infraDir, constants.O_RDONLY | constants.O_DIRECTORY); + const parent = openSync(state, constants.O_RDONLY | constants.O_DIRECTORY); try { fsyncSync(parent); } finally { diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index 4989f2593..ec1d2c3bd 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -1,4 +1,4 @@ -import { execFileSync, spawnSync } from "node:child_process"; +import { spawnSync } from "node:child_process"; import { randomBytes } from "node:crypto"; import { chmodSync, @@ -7,6 +7,7 @@ import { readFileSync, realpathSync, rmSync, + symlinkSync, writeFileSync, } from "node:fs"; import { createRequire } from "node:module"; @@ -24,6 +25,7 @@ import { parse as parseYaml } from "yaml"; import { installedCompose } from "../../../packages/core/src/services/__fixtures__/compose-path-host"; import { ownedComposePathRuntime } from "./__fixtures__/owned-compose-path-runtime"; import { ownedServiceHost } from "./__fixtures__/owned-service-host"; +import { readProductionFixtureCompose } from "./__fixtures__/production-compose-config"; import { createAgentHandoffBindings, createDefaultReleaseEffects } from "./administrative-runtime"; import { agentHandoffPath, readAgentHandoff, writeAgentHandoff } from "./agent-handoff"; import { startOpsAgentServer } from "./bootstrap"; @@ -36,7 +38,13 @@ import { initializeTrustedConfigurationRuntime } from "./trusted-config-runtime" const roots: string[] = []; async function fixture(options: Parameters[0] = {}) { - const host = await ownedServiceHost(options); + const host = await ownedServiceHost({ + ...options, + composeConfigIO: async (_file, args) => ({ + stdout: JSON.stringify(readProductionFixtureCompose(args)), + stderr: "", + }), + }); roots.push(host.root); return host; } @@ -65,7 +73,24 @@ describe("assembled production service authority", () => { Labels: { ...host.row.Labels, "com.docker.compose.service": id }, })), ); - const effects = productionReleaseEffects(host.runtime); + const runtime = await createProductionOpsRuntime({ + rootDir: host.root, + stack: host.stack, + loadAuthority: host.loadAuthority, + communityRepositories: host.repositories, + docker: { execFile: host.execFile, fileExists: existsSync }, + releaseRuntimeOptions: { + runDocker: async (args, context) => + ( + await host.execFile("docker", args, { + signal: context.signal, + timeout: 30000, + maxBuffer: 1024 * 1024, + }) + ).stdout, + }, + }); + const effects = productionReleaseEffects(runtime); const io = { signal: new AbortController().signal, emitLog() {} }; const before = services.currentConfigurationGeneration(host.infraDir); await expect(effects.inspectSystem(io)).resolves.toMatchObject({ @@ -340,16 +365,14 @@ describe("assembled production service authority", () => { const bin = join(host.root, "fake-bin"), trace = join(host.root, "docker-trace.jsonl"); mkdirSync(bin, { mode: 0o755 }); - const docker = join(bin, "docker"), - realDocker = execFileSync("which", ["docker"], { encoding: "utf8" }).trim(); - writeFileSync( + const docker = join(bin, "docker"); + symlinkSync( + fileURLToPath(new URL("./__fixtures__/production-docker-io.mjs", import.meta.url)), docker, - `#!${process.execPath}\nconst fs=require("node:fs"),cp=require("node:child_process"),a=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(trace)},JSON.stringify(a)+"\\n");if(a[0]==="compose"&&a.includes("config")){const r=cp.spawnSync(${JSON.stringify(realDocker)},a,{encoding:"utf8"});process.stdout.write(r.stdout||"");process.stderr.write(r.stderr||"");process.exit(r.status??1);}\n`, - { mode: 0o700 }, ); - chmodSync(docker, 0o700); for (const [key, value] of Object.entries({ PATH: `${bin}:${process.env.PATH}`, + OMX_PRODUCTION_TEST_DOCKER_TRACE: trace, OPENMAPX_ROOT_DIR: host.root, OPENMAPX_RELEASE: "local", DOMAIN: "example.test", diff --git a/packages/cli/__tests__/backup-restore-postgres.integration.test.ts b/packages/cli/__tests__/backup-restore-postgres.integration.test.ts index fae8f6719..dfa0e562e 100644 --- a/packages/cli/__tests__/backup-restore-postgres.integration.test.ts +++ b/packages/cli/__tests__/backup-restore-postgres.integration.test.ts @@ -9,10 +9,11 @@ import { readFileSync, rmSync, statSync, + symlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { delimiter, dirname, join, relative, resolve } from "node:path"; import { pipeline } from "node:stream/promises"; import { createGzip } from "node:zlib"; import { @@ -73,13 +74,14 @@ interface FixtureState { } let fixture: FixtureState; +const originalPath = process.env.PATH; let apiDatabase: typeof import("../../../apps/api/src/db/index").db; let apiSql: typeof import("../../../apps/api/src/db/index").sql; let runPrivacyDatabaseCleanup: typeof import("../../../apps/api/src/privacy/cleanup").runPrivacyDatabaseCleanup; function writeFixtureCompose(root: string, project: string, exportRoot: string): string { const infra = join(root, "infra", "docker"); - mkdirSync(join(infra, "backups"), { recursive: true }); + mkdirSync(join(infra, "backups"), { recursive: true, mode: 0o700 }); mkdirSync(exportRoot, { recursive: true, mode: 0o700 }); chmodSync(exportRoot, 0o700); const sentinelRoot = join(root, "api-sentinel"); @@ -98,11 +100,7 @@ while :; do sleep 3600; done ); writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); mkdirSync(join(root, "services")); - mkdirSync(join(infra, services.CURRENT_CONFIGURATION), { recursive: true }); - const composeFile = join(infra, services.CURRENT_CONFIGURATION, "docker-compose.generated.yml"); - writeFileSync( - composeFile, - `name: ${project} + const composeYaml = `name: ${project} services: postgis: image: ${POSTGIS_IMAGE} @@ -131,12 +129,44 @@ services: volumes: pgdata: scratch: -`, - "utf8", +`; + // This fixture runs its own frozen Compose bytes, not the release's services. + // Give them the same private generation layout the applied reader requires. + const revisionId = `cfg1_${createHash("sha256").update(composeYaml).digest("base64url")}`, + generation = join(infra, services.CONFIGURATION_GENERATIONS, revisionId), + current = join(infra, services.CURRENT_CONFIGURATION); + mkdirSync(generation, { recursive: true, mode: 0o700 }); + const composeFile = join(generation, services.GENERATED_COMPOSE_FILE); + writeFileSync(composeFile, composeYaml, { mode: 0o400 }); + writeFileSync( + join(generation, "resolved-config.generated.json"), + JSON.stringify({ serviceConfigs: [], integrationConfigs: [] }), + { mode: 0o400 }, + ); + symlinkSync(relative(dirname(current), generation), current); + symlinkSync( + `generations/${revisionId}`, + services.configurationGenerationBasePath(infra, revisionId), ); return composeFile; } +async function installPrivateDockerProjectTransport(root: string, project: string): Promise { + const docker = (await execa("which", ["docker"])).stdout.trim(); + if (!docker.startsWith("/")) throw Error("fixture Docker executable is not absolute"); + const bin = join(root, "bin"), + shim = join(bin, "docker"); + mkdirSync(bin, { mode: 0o700 }); + // Only namespace the fixed production project. Every Docker operation, byte, + // SQL statement, image and volume still reaches the real engine unchanged. + writeFileSync( + shim, + `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const args=process.argv.slice(2);if(args[0]==='compose'){const at=args.indexOf('--project-name');if(at!==-1&&args[at+1]==='docker')args[at+1]=${JSON.stringify(project)};}const result=spawnSync(${JSON.stringify(docker)},args,{stdio:'inherit'});if(result.error)throw result.error;process.exit(result.status??1);\n`, + { mode: 0o700 }, + ); + process.env.PATH = `${bin}${delimiter}${originalPath ?? ""}`; +} + async function compose(args: string[], reject = true): Promise { const result = await execa("docker", ["compose", "-f", fixture.composeFile, ...args], { cwd: fixture.infra, @@ -231,7 +261,7 @@ async function resetMigratedDatabase(): Promise { "postgis", "sh", "-ec", - `for file in /migrations/[0-9][0-9][0-9][0-9]_*.sql; do psql -X -v ON_ERROR_STOP=1 -U ${POSTGRES_USER} -d ${DATABASE} -f "$file"; done`, + `for file in /migrations/[0-9][0-9][0-9][0-9]_*.sql; do psql -X --single-transaction -v ON_ERROR_STOP=1 -U ${POSTGRES_USER} -d ${DATABASE} -f "$file"; done`, ], { cwd: fixture.infra, reject: false }, ); @@ -473,6 +503,7 @@ describe.skipIf(!runIntegration)("backup restore erasure replay with real Postgr store: undefined as never, databaseUrl: "", }; + await installPrivateDockerProjectTransport(root, project); await compose(["up", "-d", "--wait"]); await waitForPostgresInitialization(); const port = (await compose(["port", "postgis", "5432"])).split(":").at(-1)?.trim(); @@ -508,6 +539,8 @@ describe.skipIf(!runIntegration)("backup restore erasure replay with real Postgr await apiSql?.end({ timeout: 2 }).catch(() => undefined); if (fixture?.composeFile) await compose(["down", "--volumes", "--remove-orphans"], false); if (fixture?.root) rmSync(fixture.root, { recursive: true, force: true }); + if (originalPath === undefined) delete process.env.PATH; + else process.env.PATH = originalPath; }, 30_000); it("erases the restored account without touching a suffix-email account and defers key destruction until physical cleanup", async () => { diff --git a/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts b/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts index aa96e9931..4e65270d1 100644 --- a/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts +++ b/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts @@ -19,9 +19,9 @@ if (!root || !["old-directory", "build-record"].includes(point)) throw Error("Native fixture input required"); const dataRoot = join(root, "data"), source = join(root, "source"); -fs.mkdirSync(join(dataRoot, "osrm-graph"), { recursive: true }); -fs.writeFileSync(join(dataRoot, "osrm-graph/old"), "old graph"); -fs.mkdirSync(join(dataRoot, ".openmapx-builds")); +fs.mkdirSync(join(dataRoot, "osrm-graph"), { recursive: true, mode: 0o700 }); +fs.writeFileSync(join(dataRoot, "osrm-graph/old"), "old graph", { mode: 0o600 }); +fs.mkdirSync(join(dataRoot, ".openmapx-builds"), { mode: 0o700 }); const oldRecord = JSON.stringify({ schemaVersion: 1, service: "osrm", @@ -31,14 +31,14 @@ const oldRecord = JSON.stringify({ }); fs.writeFileSync(join(dataRoot, ".openmapx-builds/osrm.json"), oldRecord, { mode: 0o600 }); const oldDirectoryInode = fs.statSync(join(dataRoot, "osrm-graph")).ino; -fs.mkdirSync(join(source, "osrm-graph"), { recursive: true }); +fs.mkdirSync(join(source, "osrm-graph"), { recursive: true, mode: 0o700 }); for (const name of [ "region.osrm", "region.osrm.partition", "region.osrm.cells", "region.osrm.mldgr", ]) - fs.writeFileSync(join(source, "osrm-graph", name), `new ${name}`); + fs.writeFileSync(join(source, "osrm-graph", name), `new ${name}`, { mode: 0o600 }); const chunks: Buffer[] = []; await writeBuildBundle({ dataRoot: source, diff --git a/packages/integration-framework/src/__tests__/installer-streaming.test.ts b/packages/integration-framework/src/__tests__/installer-streaming.test.ts index 183b8c9b2..41e276a7b 100644 --- a/packages/integration-framework/src/__tests__/installer-streaming.test.ts +++ b/packages/integration-framework/src/__tests__/installer-streaming.test.ts @@ -415,6 +415,42 @@ it("finishes original empty lock removal after owner unlink acknowledgement loss const later = f.installer.acquireIntegrationComponentLock({ rootDir: f.root, id: f.prepared.id }); f.installer.releaseIntegrationComponentLock(later); }); +it("observes a committed publication without adopting a different owner on the retained inode", async () => { + const f = await publicationFixture(); + const published = await f.installer.publishPreparedIntegration(f.prepared, { + transactionId: "ac23034b-553c-4c01-97c4-a0b1c67bb029", + expectedPreviousContentDigest: null, + signal: f.signal, + }); + calls.unlinkFault = true; + await expect( + f.installer.settleIntegrationPublication(published, "commit", f.signal), + ).rejects.toThrow("named owner unlink acknowledgement loss"); + const ownerPath = join( + f.root, + "custom_integrations", + ".installation-locks", + f.prepared.id, + "owner.json", + ); + const differentOwner = JSON.stringify({ + version: 1, + transactionId: "944c81db-75da-418a-aef8-f839174cf87f", + }); + writeFileSync(ownerPath, differentOwner, { mode: 0o600 }); + const recovered = await f.installer.restoreIntegrationPublication({ + rootDir: f.root, + receipt: f.installer.describeIntegrationPublication(published), + signal: f.signal, + }); + await expect( + f.installer.settleIntegrationPublication(recovered, "commit", f.signal), + ).resolves.toMatchObject({ outcome: "committed" }); + expect(readFileSync(ownerPath, "utf8")).toBe(differentOwner); + expect(() => f.installer.removeIntegration({ rootDir: f.root, id: f.prepared.id })).toThrow( + /busy/, + ); +}); it("rejects oversized JSON metadata before parsing or whole-file reads", async () => { const root = mkdtempSync(join(tmpdir(), "openmapx-large-manifest-")); roots.push(root); diff --git a/packages/integration-framework/src/installer.ts b/packages/integration-framework/src/installer.ts index 9ddf872ed..aad172a7e 100644 --- a/packages/integration-framework/src/installer.ts +++ b/packages/integration-framework/src/installer.ts @@ -2051,7 +2051,16 @@ export async function restoreIntegrationPublication(options: { if (directory.listBounded(2).length === 0) { directory.close(); locks.removeDirectory(receipt.integrationId, receipt.lockIdentity); - } else publicationState(handle).lock = restoreComponentLock(lockState); + } else { + // Directory inode numbers can be reused after the original lock is + // removed. A completed receipt observes its result; it must never + // adopt or release a later transaction's lock on that inode. + const owner = z + .strictObject({ version: z.literal(1), transactionId: z.uuid() }) + .parse(JSON.parse(directory.read("owner.json", 2048).toString("utf8"))); + if (owner.transactionId === receipt.transactionId) + publicationState(handle).lock = restoreComponentLock(lockState); + } } finally { directory?.close(); locks?.close(); diff --git a/scripts/ops-agent-deployment-user.test.ts b/scripts/ops-agent-deployment-user.test.ts index 18f634aaf..08694a692 100644 --- a/scripts/ops-agent-deployment-user.test.ts +++ b/scripts/ops-agent-deployment-user.test.ts @@ -11,12 +11,13 @@ import { readFileSync, rmSync, statSync, + symlinkSync, utimesSync, writeFileSync, } from "node:fs"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; -import { join, relative, resolve } from "node:path"; +import { dirname, join, relative, resolve } from "node:path"; import { promisify } from "node:util"; import { configurationApplyResultSchema, @@ -101,6 +102,12 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( target, ]); expect(statSync(target).uid).toBe(EXPECTED_UID); + const bin = join(clone, "docker-bin"); + mkdirSync(bin, { mode: 0o700 }); + symlinkSync( + join(ROOT, "apps/ops-agent/src/__fixtures__/production-docker-io.mjs"), + join(bin, "docker"), + ); writeFileSync(join(target, "infra", "docker", ".env"), envWithoutUser(target), { mode: 0o600, }); @@ -129,6 +136,16 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( return { uid: Number(uid), gid: Number(gid) }; } + /** Empty fixture host: real configuration/authority code observes named Docker reads. */ + function fixtureEnvironment(target: string) { + return { + PATH: `${join(dirname(target), "docker-bin")}:${process.env.PATH}`, + HOME: process.env.HOME, + OPENMAPX_ROOT_DIR: target, + OMX_PRODUCTION_TEST_DOCKER_READ_ONLY: "1", + }; + } + /** `pnpm openmapx` on the host: the CLI from source. */ async function hostCli(target: string, ...args: string[]) { const cli = join(ROOT, "packages", "cli"); @@ -137,7 +154,7 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( ["--tsconfig", join(cli, "tsconfig.json"), join(cli, "src", "index.ts"), ...args], { cwd: target, - env: { PATH: process.env.PATH, HOME: process.env.HOME, OPENMAPX_ROOT_DIR: target }, + env: fixtureEnvironment(target), }, ); } @@ -146,7 +163,7 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( async function agentCli(target: string, ...args: string[]) { return run(process.execPath, [join(outdir, "cli.mjs"), ...args], { cwd: target, - env: { PATH: process.env.PATH, HOME: process.env.HOME, OPENMAPX_ROOT_DIR: target }, + env: fixtureEnvironment(target), }); } @@ -192,9 +209,7 @@ describe.skipIf(process.platform !== "linux" || !EXPECTED_UID)( const agent = spawn(process.execPath, [join(outdir, "agent.mjs")], { cwd: target, env: { - PATH: process.env.PATH, - HOME: process.env.HOME, - OPENMAPX_ROOT_DIR: target, + ...fixtureEnvironment(target), OPS_AGENT_API_TOKEN_FILE: join(infra, "secrets", "ops-agent-api-token"), OPS_AGENT_DATA_MANAGER_TOKEN_FILE: join(infra, "secrets", "ops-agent-data-manager-token"), OPS_TRUSTED_CONFIG_DIR: services.agentStatePaths(infra).trustedConfig, diff --git a/scripts/ops-agent-transitous-pin.test.ts b/scripts/ops-agent-transitous-pin.test.ts index d3fe140b7..f734816c5 100644 --- a/scripts/ops-agent-transitous-pin.test.ts +++ b/scripts/ops-agent-transitous-pin.test.ts @@ -1,6 +1,7 @@ import { type ChildProcess, execFile, spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; import { + chmodSync, existsSync, mkdirSync, mkdtempSync, @@ -77,6 +78,13 @@ describe.skipIf(process.platform !== "linux")("the agent's Transitous pin", () = process.umask(previous); } const infra = join(target, "infra", "docker"); + // Git-created directories inherit the deliberately permissive clone umask. + // Provision the fixture's authority roots explicitly, as a host operator + // must, rather than teaching production to accept group-writable trust. + chmodSync(target, 0o755); + chmodSync(join(target, "infra"), 0o755); + chmodSync(infra, 0o755); + chmodSync(join(infra, "secrets"), 0o700); // The deployment's user, which a host render writes from the checkout's // owner; the bundled CLI is the agent's and only reads it. const owner = statSync(target); diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index cd2902f75..b20624fe4 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -546,6 +546,7 @@ "ops-agent|@docker-socket|/var/run/docker.sock|false", "ops-agent|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set \u2014 ops-agent renders and applies trusted local stack configuration}/infra/docker|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker|true", "ops-agent|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/backups|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/backups|false", + "ops-agent|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/catalog-pins|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/catalog-pins|false", "ops-agent|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/data|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/data|false", "ops-agent|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/secrets|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/secrets|false", "ops-agent|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/state|${OPENMAPX_HOST_DIR:?OPENMAPX_HOST_DIR must be set}/infra/docker/state|false", diff --git a/vitest.setup.node.ts b/vitest.setup.node.ts index a3f06cd61..1b29b8f39 100644 --- a/vitest.setup.node.ts +++ b/vitest.setup.node.ts @@ -9,3 +9,8 @@ // still measures real disk; `??=` lets an individual test opt into its own // value (e.g. to exercise the insufficient-disk path). process.env.MOTIS_FREE_DISK_BYTES ??= String(1024 ** 4); // 1 TiB + +// Unit suites may load the real auth module through administrative consumers. +// Supply an explicit test credential here; production still requires its own +// configured secret, and secret-policy tests pass isolated environments. +process.env.BETTER_AUTH_SECRET ??= "openmapx-node-tests-only-auth-secret"; From 9defc13d97877f3fc23cb22acb38c11eb30e18ed Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 4 Nov 2026 08:15:59 +0100 Subject: [PATCH 110/135] fix(admin): address deep review and reduce generated SDK clutter --- .gitattributes | 7 + .../__tests__/admin-integrations-sdk.test.ts | 4 - ...configuration-maintenance-postgres.test.ts | 169 +- .../trusted-config-operations.test.ts | 12 - .../services/configuration-migration/apply.ts | 24 +- .../completion.test.ts | 51 + .../configuration-migration/completion.ts | 39 + .../migration-postgres.test.ts | 81 +- apps/api/src/services/job-runner.ts | 19 +- .../src/services/prospective-configuration.ts | 199 +- .../trusted-configuration-producer.ts | 74 +- .../ops-agent/src/catalog-approval-runtime.ts | 1 - .../src/configuration-maintenance-store.ts | 32 +- apps/ops-agent/src/integration-store.test.ts | 84 +- .../ArtifactTransfersSection.test.tsx | 132 +- .../services/ArtifactTransfersSection.tsx | 90 +- .../admin/services/TransitSyncDialog.test.tsx | 33 +- .../admin/services/TransitSyncDialog.tsx | 3 +- .../src/lib/admin/artifactTransferClient.ts | 112 +- apps/web/src/lib/admin/downloadSink.ts | 107 + apps/web/src/lib/admin/privacyDownload.ts | 128 +- docs/docs/developer/service-manifest.md | 18 +- .../admin/account-admin.generated.ts | 2709 ++++++++- .../account-models/applyAdminAuthPolicy200.ts | 12 - ...pplyAdminAuthPolicy200ActivationVersion.ts | 9 - .../applyAdminAuthPolicy200Active.ts | 21 - ...minAuthPolicy200ActiveActivationVersion.ts | 4 - .../applyAdminAuthPolicy200AppliesTo.ts | 9 - .../account-models/applyAdminAuthPolicy400.ts | 20 - .../account-models/applyAdminAuthPolicy401.ts | 20 - .../account-models/applyAdminAuthPolicy403.ts | 20 - .../account-models/applyAdminAuthPolicy409.ts | 20 - .../account-models/applyAdminAuthPolicy413.ts | 20 - .../account-models/applyAdminAuthPolicy415.ts | 20 - .../account-models/applyAdminAuthPolicy429.ts | 11 - .../account-models/applyAdminAuthPolicy500.ts | 20 - .../account-models/applyAdminAuthPolicy503.ts | 20 - .../applyAdminAuthPolicyBody.ts | 12 - .../applyAdminAuthPolicyBodySettings.ts | 17 - .../changeAdminAccountRole200.ts | 9 - .../changeAdminAccountRole200Role.ts | 11 - .../changeAdminAccountRole400.ts | 20 - .../changeAdminAccountRole401.ts | 20 - .../changeAdminAccountRole403.ts | 20 - .../changeAdminAccountRole404.ts | 20 - .../changeAdminAccountRole409.ts | 20 - .../changeAdminAccountRole413.ts | 20 - .../changeAdminAccountRole415.ts | 20 - .../changeAdminAccountRole429.ts | 11 - .../changeAdminAccountRole500.ts | 20 - .../changeAdminAccountRole503.ts | 20 - .../changeAdminAccountRoleBody.ts | 9 - .../changeAdminAccountRoleBodyExpectedRole.ts | 11 - .../changeAdminAccountRoleBodyRole.ts | 11 - .../changeAdminAccountRoleHeaders.ts | 9 - .../completeAdminReauthentication200.ts | 6 - .../completeAdminReauthentication400.ts | 20 - .../completeAdminReauthentication401.ts | 20 - .../completeAdminReauthentication403.ts | 20 - .../completeAdminReauthentication409.ts | 20 - .../completeAdminReauthentication413.ts | 20 - .../completeAdminReauthentication415.ts | 20 - .../completeAdminReauthentication429.ts | 11 - .../completeAdminReauthentication500.ts | 20 - .../completeAdminReauthentication503.ts | 20 - .../completeAdminReauthenticationBody.ts | 7 - .../completeAdminReauthenticationHeaders.ts | 9 - .../account-models/createAdminAccount200.ts | 11 - ...ateAdminAccount200PasswordSetupDelivery.ts | 13 - ...eateAdminAccount200VerificationDelivery.ts | 13 - .../account-models/createAdminAccount400.ts | 20 - .../account-models/createAdminAccount401.ts | 20 - .../account-models/createAdminAccount403.ts | 20 - .../account-models/createAdminAccount404.ts | 20 - .../account-models/createAdminAccount409.ts | 20 - .../account-models/createAdminAccount413.ts | 20 - .../account-models/createAdminAccount415.ts | 20 - .../account-models/createAdminAccount429.ts | 11 - .../account-models/createAdminAccount500.ts | 20 - .../account-models/createAdminAccount503.ts | 20 - .../account-models/createAdminAccountBody.ts | 23 - .../createAdminAccountBodyRole.ts | 11 - .../createAdminAccountHeaders.ts | 9 - .../getAdminAccountDelivery200.ts | 7 - .../getAdminAccountDelivery200Latest.ts | 13 - .../getAdminAccountDelivery200LatestAction.ts | 12 - ...tDelivery200LatestPasswordSetupDelivery.ts | 13 - ...ntDelivery200LatestVerificationDelivery.ts | 13 - .../getAdminAccountDelivery400.ts | 20 - .../getAdminAccountDelivery401.ts | 20 - .../getAdminAccountDelivery403.ts | 20 - .../getAdminAccountDelivery404.ts | 20 - .../getAdminAccountDelivery409.ts | 20 - .../getAdminAccountDelivery413.ts | 20 - .../getAdminAccountDelivery415.ts | 20 - .../getAdminAccountDelivery429.ts | 11 - .../getAdminAccountDelivery500.ts | 20 - .../getAdminAccountDelivery503.ts | 20 - .../account-models/getAdminAuthPolicy200.ts | 11 - .../getAdminAuthPolicy200Active.ts | 21 - ...minAuthPolicy200ActiveActivationVersion.ts | 4 - .../getAdminAuthPolicy200Ownership.ts | 13 - ...cy200OwnershipEmailVerificationRequired.ts | 11 - ...AuthPolicy200OwnershipPasswordMinLength.ts | 10 - ...hPolicy200OwnershipSessionDurationHours.ts | 10 - ...AdminAuthPolicy200OwnershipTwoFaEnabled.ts | 10 - .../getAdminAuthPolicy200Pending.ts | 4 - .../account-models/getAdminAuthPolicy400.ts | 20 - .../account-models/getAdminAuthPolicy401.ts | 20 - .../account-models/getAdminAuthPolicy403.ts | 20 - .../account-models/getAdminAuthPolicy413.ts | 20 - .../account-models/getAdminAuthPolicy415.ts | 20 - .../account-models/getAdminAuthPolicy429.ts | 11 - .../account-models/getAdminAuthPolicy500.ts | 20 - .../account-models/getAdminAuthPolicy503.ts | 20 - .../generated/admin/account-models/index.ts | 186 - .../account-models/inviteAdminAccount200.ts | 11 - ...iteAdminAccount200PasswordSetupDelivery.ts | 13 - ...viteAdminAccount200VerificationDelivery.ts | 13 - .../account-models/inviteAdminAccount400.ts | 20 - .../account-models/inviteAdminAccount401.ts | 20 - .../account-models/inviteAdminAccount403.ts | 20 - .../account-models/inviteAdminAccount404.ts | 20 - .../account-models/inviteAdminAccount409.ts | 20 - .../account-models/inviteAdminAccount413.ts | 20 - .../account-models/inviteAdminAccount415.ts | 20 - .../account-models/inviteAdminAccount429.ts | 11 - .../account-models/inviteAdminAccount500.ts | 20 - .../account-models/inviteAdminAccount503.ts | 20 - .../account-models/inviteAdminAccountBody.ts | 6 - .../inviteAdminAccountHeaders.ts | 13 - .../sendAdminAccountPasswordReset200.ts | 11 - ...ntPasswordReset200PasswordSetupDelivery.ts | 13 - ...untPasswordReset200VerificationDelivery.ts | 13 - .../sendAdminAccountPasswordReset400.ts | 20 - .../sendAdminAccountPasswordReset401.ts | 20 - .../sendAdminAccountPasswordReset403.ts | 20 - .../sendAdminAccountPasswordReset404.ts | 20 - .../sendAdminAccountPasswordReset409.ts | 20 - .../sendAdminAccountPasswordReset413.ts | 20 - .../sendAdminAccountPasswordReset415.ts | 20 - .../sendAdminAccountPasswordReset429.ts | 11 - .../sendAdminAccountPasswordReset500.ts | 20 - .../sendAdminAccountPasswordReset503.ts | 20 - .../sendAdminAccountPasswordResetBody.ts | 6 - .../sendAdminAccountPasswordResetHeaders.ts | 13 - .../sendAdminAccountVerificationEmail200.ts | 11 - ...rificationEmail200PasswordSetupDelivery.ts | 13 - ...erificationEmail200VerificationDelivery.ts | 13 - .../sendAdminAccountVerificationEmail400.ts | 20 - .../sendAdminAccountVerificationEmail401.ts | 20 - .../sendAdminAccountVerificationEmail403.ts | 20 - .../sendAdminAccountVerificationEmail404.ts | 20 - .../sendAdminAccountVerificationEmail409.ts | 20 - .../sendAdminAccountVerificationEmail413.ts | 20 - .../sendAdminAccountVerificationEmail415.ts | 20 - .../sendAdminAccountVerificationEmail429.ts | 11 - .../sendAdminAccountVerificationEmail500.ts | 20 - .../sendAdminAccountVerificationEmail503.ts | 20 - .../sendAdminAccountVerificationEmailBody.ts | 6 - ...endAdminAccountVerificationEmailHeaders.ts | 13 - .../setAdminAccountPassword200.ts | 11 - ...AccountPassword200PasswordSetupDelivery.ts | 13 - ...nAccountPassword200VerificationDelivery.ts | 13 - .../setAdminAccountPassword400.ts | 20 - .../setAdminAccountPassword401.ts | 20 - .../setAdminAccountPassword403.ts | 20 - .../setAdminAccountPassword404.ts | 20 - .../setAdminAccountPassword409.ts | 20 - .../setAdminAccountPassword413.ts | 20 - .../setAdminAccountPassword415.ts | 20 - .../setAdminAccountPassword429.ts | 11 - .../setAdminAccountPassword500.ts | 20 - .../setAdminAccountPassword503.ts | 20 - .../setAdminAccountPasswordBody.ts | 13 - .../setAdminAccountPasswordBodyReasonCode.ts | 11 - .../setAdminAccountPasswordHeaders.ts | 13 - .../startAdminReauthentication200.ts | 11 - .../startAdminReauthentication200LoginPath.ts | 9 - .../startAdminReauthentication400.ts | 20 - .../startAdminReauthentication401.ts | 20 - .../startAdminReauthentication403.ts | 20 - .../startAdminReauthentication409.ts | 20 - .../startAdminReauthentication413.ts | 20 - .../startAdminReauthentication415.ts | 20 - .../startAdminReauthentication429.ts | 11 - .../startAdminReauthentication500.ts | 20 - .../startAdminReauthentication503.ts | 20 - .../startAdminReauthenticationBody.ts | 15 - .../startAdminReauthenticationBodyPurpose.ts | 16 - .../startAdminReauthenticationHeaders.ts | 9 - .../account-models/verifyAdminAccount200.ts | 11 - ...ifyAdminAccount200PasswordSetupDelivery.ts | 13 - ...rifyAdminAccount200VerificationDelivery.ts | 13 - .../account-models/verifyAdminAccount400.ts | 20 - .../account-models/verifyAdminAccount401.ts | 20 - .../account-models/verifyAdminAccount403.ts | 20 - .../account-models/verifyAdminAccount404.ts | 20 - .../account-models/verifyAdminAccount409.ts | 20 - .../account-models/verifyAdminAccount413.ts | 20 - .../account-models/verifyAdminAccount415.ts | 20 - .../account-models/verifyAdminAccount429.ts | 11 - .../account-models/verifyAdminAccount500.ts | 20 - .../account-models/verifyAdminAccount503.ts | 20 - .../account-models/verifyAdminAccountBody.ts | 13 - .../verifyAdminAccountBodyReasonCode.ts | 11 - .../verifyAdminAccountHeaders.ts | 13 - .../admin/privacy-admin.generated.ts | 5327 ++++++++++++++++- .../acceptPrivacyAdminBackupOmissions201.ts | 7 - ...eptPrivacyAdminBackupOmissions201Review.ts | 16 - .../acceptPrivacyAdminBackupOmissions400.ts | 14 - .../acceptPrivacyAdminBackupOmissions401.ts | 14 - .../acceptPrivacyAdminBackupOmissions403.ts | 14 - .../acceptPrivacyAdminBackupOmissions404.ts | 14 - .../acceptPrivacyAdminBackupOmissions409.ts | 14 - .../acceptPrivacyAdminBackupOmissions410.ts | 14 - .../acceptPrivacyAdminBackupOmissions413.ts | 14 - .../acceptPrivacyAdminBackupOmissions416.ts | 14 - .../acceptPrivacyAdminBackupOmissions429.ts | 14 - .../acceptPrivacyAdminBackupOmissions500.ts | 14 - .../acceptPrivacyAdminBackupOmissions503.ts | 14 - .../acceptPrivacyAdminBackupOmissionsBody.ts | 14 - ...cceptPrivacyAdminBackupOmissionsHeaders.ts | 9 - ...nGeneratePrivacyAdminArtifact201Schema0.ts | 10 - ...adminGetPrivacyAdminReadiness200Schema0.ts | 10 - .../assignPrivacyAdminTask200.ts | 7 - .../assignPrivacyAdminTask200Request.ts | 40 - ...signPrivacyAdminTask200RequestExtension.ts | 10 - .../assignPrivacyAdminTask200RequestKind.ts | 11 - .../assignPrivacyAdminTask400.ts | 14 - .../assignPrivacyAdminTask401.ts | 14 - .../assignPrivacyAdminTask403.ts | 14 - .../assignPrivacyAdminTask404.ts | 14 - .../assignPrivacyAdminTask409.ts | 14 - .../assignPrivacyAdminTask410.ts | 14 - .../assignPrivacyAdminTask413.ts | 14 - .../assignPrivacyAdminTask416.ts | 14 - .../assignPrivacyAdminTask429.ts | 14 - .../assignPrivacyAdminTask500.ts | 14 - .../assignPrivacyAdminTask503.ts | 14 - .../assignPrivacyAdminTaskBody.ts | 11 - .../assignPrivacyAdminTaskHeaders.ts | 9 - .../completePrivacyAdminEmailChallenge200.ts | 7 - .../completePrivacyAdminEmailChallenge400.ts | 14 - .../completePrivacyAdminEmailChallenge401.ts | 14 - .../completePrivacyAdminEmailChallenge403.ts | 14 - .../completePrivacyAdminEmailChallenge404.ts | 14 - .../completePrivacyAdminEmailChallenge409.ts | 14 - .../completePrivacyAdminEmailChallenge410.ts | 14 - .../completePrivacyAdminEmailChallenge413.ts | 14 - .../completePrivacyAdminEmailChallenge416.ts | 14 - .../completePrivacyAdminEmailChallenge429.ts | 14 - .../completePrivacyAdminEmailChallenge500.ts | 14 - .../completePrivacyAdminEmailChallenge503.ts | 14 - .../completePrivacyAdminEmailChallengeBody.ts | 9 - ...letePrivacyAdminEmailChallengeBodyParty.ts | 10 - ...mpletePrivacyAdminEmailChallengeHeaders.ts | 9 - ...completePrivacyAdminReauthentication200.ts | 6 - ...completePrivacyAdminReauthentication400.ts | 14 - ...completePrivacyAdminReauthentication401.ts | 14 - ...completePrivacyAdminReauthentication403.ts | 14 - ...completePrivacyAdminReauthentication404.ts | 14 - ...completePrivacyAdminReauthentication409.ts | 14 - ...completePrivacyAdminReauthentication410.ts | 14 - ...completePrivacyAdminReauthentication413.ts | 14 - ...completePrivacyAdminReauthentication416.ts | 14 - ...completePrivacyAdminReauthentication429.ts | 14 - ...completePrivacyAdminReauthentication500.ts | 14 - ...completePrivacyAdminReauthentication503.ts | 14 - ...ompletePrivacyAdminReauthenticationBody.ts | 7 - ...letePrivacyAdminReauthenticationHeaders.ts | 9 - .../createPrivacyAdminBackupReview201.ts | 7 - ...createPrivacyAdminBackupReview201Review.ts | 22 - ...ivacyAdminBackupReview201ReviewDecision.ts | 12 - .../createPrivacyAdminBackupReview400.ts | 14 - .../createPrivacyAdminBackupReview401.ts | 14 - .../createPrivacyAdminBackupReview403.ts | 14 - .../createPrivacyAdminBackupReview404.ts | 14 - .../createPrivacyAdminBackupReview409.ts | 14 - .../createPrivacyAdminBackupReview410.ts | 14 - .../createPrivacyAdminBackupReview413.ts | 14 - .../createPrivacyAdminBackupReview416.ts | 14 - .../createPrivacyAdminBackupReview429.ts | 14 - .../createPrivacyAdminBackupReview500.ts | 14 - .../createPrivacyAdminBackupReview503.ts | 14 - .../createPrivacyAdminBackupReviewBody.ts | 7 - .../createPrivacyAdminBackupReviewHeaders.ts | 9 - .../createPrivacyAdminCase202.ts | 7 - .../createPrivacyAdminCase202Request.ts | 40 - ...eatePrivacyAdminCase202RequestExtension.ts | 10 - .../createPrivacyAdminCase202RequestKind.ts | 11 - .../createPrivacyAdminCase400.ts | 14 - .../createPrivacyAdminCase401.ts | 14 - .../createPrivacyAdminCase403.ts | 14 - .../createPrivacyAdminCase404.ts | 14 - .../createPrivacyAdminCase409.ts | 14 - .../createPrivacyAdminCase410.ts | 14 - .../createPrivacyAdminCase413.ts | 14 - .../createPrivacyAdminCase416.ts | 14 - .../createPrivacyAdminCase429.ts | 14 - .../createPrivacyAdminCase500.ts | 14 - .../createPrivacyAdminCase503.ts | 14 - .../createPrivacyAdminCaseBody.ts | 19 - .../createPrivacyAdminCaseBodyChannel.ts | 12 - .../createPrivacyAdminCaseBodyKind.ts | 11 - ...reatePrivacyAdminCaseBodyRepresentative.ts | 12 - ...yAdminCaseBodyRepresentativeContactType.ts | 10 - .../createPrivacyAdminCaseBodySubject.ts | 14 - ...PrivacyAdminCaseBodySubjectAccountState.ts | 12 - ...ePrivacyAdminCaseBodySubjectLocatorType.ts | 13 - .../createPrivacyAdminCaseHeaders.ts | 9 - .../decidePrivacyAdminCase200.ts | 7 - .../decidePrivacyAdminCase200Request.ts | 40 - ...cidePrivacyAdminCase200RequestExtension.ts | 10 - .../decidePrivacyAdminCase200RequestKind.ts | 11 - .../decidePrivacyAdminCase400.ts | 14 - .../decidePrivacyAdminCase401.ts | 14 - .../decidePrivacyAdminCase403.ts | 14 - .../decidePrivacyAdminCase404.ts | 14 - .../decidePrivacyAdminCase409.ts | 14 - .../decidePrivacyAdminCase410.ts | 14 - .../decidePrivacyAdminCase413.ts | 14 - .../decidePrivacyAdminCase416.ts | 14 - .../decidePrivacyAdminCase429.ts | 14 - .../decidePrivacyAdminCase500.ts | 14 - .../decidePrivacyAdminCase503.ts | 14 - .../decidePrivacyAdminCaseBody.ts | 17 - .../decidePrivacyAdminCaseBodyDecision.ts | 11 - .../decidePrivacyAdminCaseHeaders.ts | 9 - .../decidePrivacyAdminTask200.ts | 6 - .../decidePrivacyAdminTask400.ts | 14 - .../decidePrivacyAdminTask401.ts | 14 - .../decidePrivacyAdminTask403.ts | 14 - .../decidePrivacyAdminTask404.ts | 14 - .../decidePrivacyAdminTask409.ts | 14 - .../decidePrivacyAdminTask410.ts | 14 - .../decidePrivacyAdminTask413.ts | 14 - .../decidePrivacyAdminTask416.ts | 14 - .../decidePrivacyAdminTask429.ts | 14 - .../decidePrivacyAdminTask500.ts | 14 - .../decidePrivacyAdminTask503.ts | 14 - .../decidePrivacyAdminTaskBody.ts | 27 - .../decidePrivacyAdminTaskBodyStatus.ts | 13 - .../decidePrivacyAdminTaskHeaders.ts | 9 - .../deletePrivacyAdminAttachment400.ts | 14 - .../deletePrivacyAdminAttachment401.ts | 14 - .../deletePrivacyAdminAttachment403.ts | 14 - .../deletePrivacyAdminAttachment404.ts | 14 - .../deletePrivacyAdminAttachment409.ts | 14 - .../deletePrivacyAdminAttachment410.ts | 14 - .../deletePrivacyAdminAttachment413.ts | 14 - .../deletePrivacyAdminAttachment416.ts | 14 - .../deletePrivacyAdminAttachment429.ts | 14 - .../deletePrivacyAdminAttachment500.ts | 14 - .../deletePrivacyAdminAttachment503.ts | 14 - .../deletePrivacyAdminAttachmentBody.ts | 10 - .../deletePrivacyAdminAttachmentHeaders.ts | 9 - .../downloadPrivacyAdminArtifact400.ts | 14 - .../downloadPrivacyAdminArtifact401.ts | 14 - .../downloadPrivacyAdminArtifact403.ts | 14 - .../downloadPrivacyAdminArtifact404.ts | 14 - .../downloadPrivacyAdminArtifact409.ts | 14 - .../downloadPrivacyAdminArtifact410.ts | 14 - .../downloadPrivacyAdminArtifact413.ts | 14 - .../downloadPrivacyAdminArtifact416.ts | 14 - .../downloadPrivacyAdminArtifact429.ts | 14 - .../downloadPrivacyAdminArtifact500.ts | 14 - .../downloadPrivacyAdminArtifact503.ts | 14 - .../downloadPrivacyAdminAttachment400.ts | 14 - .../downloadPrivacyAdminAttachment401.ts | 14 - .../downloadPrivacyAdminAttachment403.ts | 14 - .../downloadPrivacyAdminAttachment404.ts | 14 - .../downloadPrivacyAdminAttachment409.ts | 14 - .../downloadPrivacyAdminAttachment410.ts | 14 - .../downloadPrivacyAdminAttachment413.ts | 14 - .../downloadPrivacyAdminAttachment416.ts | 14 - .../downloadPrivacyAdminAttachment429.ts | 14 - .../downloadPrivacyAdminAttachment500.ts | 14 - .../downloadPrivacyAdminAttachment503.ts | 14 - .../extendPrivacyAdminCase200.ts | 7 - .../extendPrivacyAdminCase200Request.ts | 40 - ...tendPrivacyAdminCase200RequestExtension.ts | 10 - .../extendPrivacyAdminCase200RequestKind.ts | 11 - .../extendPrivacyAdminCase400.ts | 14 - .../extendPrivacyAdminCase401.ts | 14 - .../extendPrivacyAdminCase403.ts | 14 - .../extendPrivacyAdminCase404.ts | 14 - .../extendPrivacyAdminCase409.ts | 14 - .../extendPrivacyAdminCase410.ts | 14 - .../extendPrivacyAdminCase413.ts | 14 - .../extendPrivacyAdminCase416.ts | 14 - .../extendPrivacyAdminCase429.ts | 14 - .../extendPrivacyAdminCase500.ts | 14 - .../extendPrivacyAdminCase503.ts | 14 - .../extendPrivacyAdminCaseBody.ts | 19 - .../extendPrivacyAdminCaseHeaders.ts | 9 - .../generatePrivacyAdminArtifact201.ts | 11 - ...generatePrivacyAdminArtifact201Artifact.ts | 15 - .../generatePrivacyAdminArtifact201Request.ts | 40 - ...PrivacyAdminArtifact201RequestExtension.ts | 10 - ...eratePrivacyAdminArtifact201RequestKind.ts | 11 - .../generatePrivacyAdminArtifact400.ts | 14 - .../generatePrivacyAdminArtifact401.ts | 14 - .../generatePrivacyAdminArtifact403.ts | 14 - .../generatePrivacyAdminArtifact404.ts | 14 - .../generatePrivacyAdminArtifact409.ts | 14 - .../generatePrivacyAdminArtifact410.ts | 14 - .../generatePrivacyAdminArtifact413.ts | 14 - .../generatePrivacyAdminArtifact416.ts | 14 - .../generatePrivacyAdminArtifact429.ts | 14 - .../generatePrivacyAdminArtifact500.ts | 14 - .../generatePrivacyAdminArtifact503.ts | 14 - .../generatePrivacyAdminArtifactBody.ts | 4 - .../generatePrivacyAdminArtifactHeaders.ts | 9 - .../privacy-models/getPrivacyAdminCase200.ts | 19 - .../getPrivacyAdminCase200ArtifactsItem.ts | 15 - .../getPrivacyAdminCase200AttachmentsItem.ts | 25 - ...acyAdminCase200AttachmentsItemMediaType.ts | 14 - ...vacyAdminCase200AttachmentsItemMetadata.ts | 45 - ...ivacyAdminCase200AttachmentsItemPurpose.ts | 12 - ...Case200AttachmentsItemRightsReviewState.ts | 12 - .../getPrivacyAdminCase200Capabilities.ts | 17 - ...tPrivacyAdminCase200CapabilitiesReasons.ts | 4 - .../getPrivacyAdminCase200EventsItem.ts | 11 - ...getPrivacyAdminCase200EventsItemPayload.ts | 4 - .../getPrivacyAdminCase200IdentitiesItem.ts | 14 - ...Case200IdentitiesItemDeliveryAuthorized.ts | 7 - ...tPrivacyAdminCase200IdentitiesItemParty.ts | 10 - .../getPrivacyAdminCase200Request.ts | 40 - .../getPrivacyAdminCase200RequestExtension.ts | 10 - .../getPrivacyAdminCase200RequestKind.ts | 11 - .../getPrivacyAdminCase200TasksItem.ts | 29 - ...PrivacyAdminCase200TasksItemActionsItem.ts | 13 - ...getPrivacyAdminCase200TasksItemRequired.ts | 4 - .../privacy-models/getPrivacyAdminCase400.ts | 14 - .../privacy-models/getPrivacyAdminCase401.ts | 14 - .../privacy-models/getPrivacyAdminCase403.ts | 14 - .../privacy-models/getPrivacyAdminCase404.ts | 14 - .../privacy-models/getPrivacyAdminCase409.ts | 14 - .../privacy-models/getPrivacyAdminCase410.ts | 14 - .../privacy-models/getPrivacyAdminCase413.ts | 14 - .../privacy-models/getPrivacyAdminCase416.ts | 14 - .../privacy-models/getPrivacyAdminCase429.ts | 14 - .../privacy-models/getPrivacyAdminCase500.ts | 14 - .../privacy-models/getPrivacyAdminCase503.ts | 14 - .../getPrivacyAdminReadiness200.ts | 17 - .../getPrivacyAdminReadiness200ChecksItem.ts | 9 - ...rivacyAdminReadiness200ChecksItemStatus.ts | 11 - .../getPrivacyAdminReadiness200KeyRing.ts | 7 - .../getPrivacyAdminReadiness200Version.ts | 9 - .../getPrivacyAdminReadiness400.ts | 14 - .../getPrivacyAdminReadiness401.ts | 14 - .../getPrivacyAdminReadiness403.ts | 14 - .../getPrivacyAdminReadiness404.ts | 14 - .../getPrivacyAdminReadiness409.ts | 14 - .../getPrivacyAdminReadiness410.ts | 14 - .../getPrivacyAdminReadiness413.ts | 14 - .../getPrivacyAdminReadiness416.ts | 14 - .../getPrivacyAdminReadiness429.ts | 14 - .../getPrivacyAdminReadiness500.ts | 14 - .../getPrivacyAdminReadiness503.ts | 14 - .../generated/admin/privacy-models/index.ts | 425 -- .../issuePrivacyAdminEmailChallenge202.ts | 11 - ...vacyAdminEmailChallenge202DeliveryState.ts | 10 - .../issuePrivacyAdminEmailChallenge400.ts | 14 - .../issuePrivacyAdminEmailChallenge401.ts | 14 - .../issuePrivacyAdminEmailChallenge403.ts | 14 - .../issuePrivacyAdminEmailChallenge404.ts | 14 - .../issuePrivacyAdminEmailChallenge409.ts | 14 - .../issuePrivacyAdminEmailChallenge410.ts | 14 - .../issuePrivacyAdminEmailChallenge413.ts | 14 - .../issuePrivacyAdminEmailChallenge416.ts | 14 - .../issuePrivacyAdminEmailChallenge429.ts | 14 - .../issuePrivacyAdminEmailChallenge500.ts | 14 - .../issuePrivacyAdminEmailChallenge503.ts | 14 - .../issuePrivacyAdminEmailChallengeBody.ts | 7 - ...ssuePrivacyAdminEmailChallengeBodyParty.ts | 10 - .../issuePrivacyAdminEmailChallengeHeaders.ts | 9 - .../listPrivacyAdminBackupOmissions200.ts | 7 - ...ivacyAdminBackupOmissions200ReviewsItem.ts | 16 - .../listPrivacyAdminBackupOmissions400.ts | 14 - .../listPrivacyAdminBackupOmissions401.ts | 14 - .../listPrivacyAdminBackupOmissions403.ts | 14 - .../listPrivacyAdminBackupOmissions404.ts | 14 - .../listPrivacyAdminBackupOmissions409.ts | 14 - .../listPrivacyAdminBackupOmissions410.ts | 14 - .../listPrivacyAdminBackupOmissions413.ts | 14 - .../listPrivacyAdminBackupOmissions416.ts | 14 - .../listPrivacyAdminBackupOmissions429.ts | 14 - .../listPrivacyAdminBackupOmissions500.ts | 14 - .../listPrivacyAdminBackupOmissions503.ts | 14 - .../listPrivacyAdminBackupReviews200.ts | 7 - ...PrivacyAdminBackupReviews200ReviewsItem.ts | 22 - ...dminBackupReviews200ReviewsItemDecision.ts | 12 - .../listPrivacyAdminBackupReviews400.ts | 14 - .../listPrivacyAdminBackupReviews401.ts | 14 - .../listPrivacyAdminBackupReviews403.ts | 14 - .../listPrivacyAdminBackupReviews404.ts | 14 - .../listPrivacyAdminBackupReviews409.ts | 14 - .../listPrivacyAdminBackupReviews410.ts | 14 - .../listPrivacyAdminBackupReviews413.ts | 14 - .../listPrivacyAdminBackupReviews416.ts | 14 - .../listPrivacyAdminBackupReviews429.ts | 14 - .../listPrivacyAdminBackupReviews500.ts | 14 - .../listPrivacyAdminBackupReviews503.ts | 14 - .../listPrivacyAdminBackups200.ts | 8 - .../listPrivacyAdminBackups200BackupsItem.ts | 19 - ...cyAdminBackups200BackupsItemVolumesItem.ts | 11 - ...minBackups200BackupsItemVolumesItemMode.ts | 10 - .../listPrivacyAdminBackups400.ts | 14 - .../listPrivacyAdminBackups401.ts | 14 - .../listPrivacyAdminBackups403.ts | 14 - .../listPrivacyAdminBackups404.ts | 14 - .../listPrivacyAdminBackups409.ts | 14 - .../listPrivacyAdminBackups410.ts | 14 - .../listPrivacyAdminBackups413.ts | 14 - .../listPrivacyAdminBackups416.ts | 14 - .../listPrivacyAdminBackups429.ts | 14 - .../listPrivacyAdminBackups500.ts | 14 - .../listPrivacyAdminBackups503.ts | 14 - .../listPrivacyAdminOperators200.ts | 9 - ...stPrivacyAdminOperators200OperatorsItem.ts | 7 - .../listPrivacyAdminOperators400.ts | 14 - .../listPrivacyAdminOperators401.ts | 14 - .../listPrivacyAdminOperators403.ts | 14 - .../listPrivacyAdminOperators404.ts | 14 - .../listPrivacyAdminOperators409.ts | 14 - .../listPrivacyAdminOperators410.ts | 14 - .../listPrivacyAdminOperators413.ts | 14 - .../listPrivacyAdminOperators416.ts | 14 - .../listPrivacyAdminOperators429.ts | 14 - .../listPrivacyAdminOperators500.ts | 14 - .../listPrivacyAdminOperators503.ts | 14 - .../listPrivacyAdminOperatorsParams.ts | 14 - .../listPrivacyAdminQueue200.ts | 7 - .../listPrivacyAdminQueue200RequestsItem.ts | 40 - ...ivacyAdminQueue200RequestsItemExtension.ts | 10 - ...istPrivacyAdminQueue200RequestsItemKind.ts | 11 - .../listPrivacyAdminQueue400.ts | 14 - .../listPrivacyAdminQueue401.ts | 14 - .../listPrivacyAdminQueue403.ts | 14 - .../listPrivacyAdminQueue404.ts | 14 - .../listPrivacyAdminQueue409.ts | 14 - .../listPrivacyAdminQueue410.ts | 14 - .../listPrivacyAdminQueue413.ts | 14 - .../listPrivacyAdminQueue416.ts | 14 - .../listPrivacyAdminQueue429.ts | 14 - .../listPrivacyAdminQueue500.ts | 14 - .../listPrivacyAdminQueue503.ts | 14 - .../reviewPrivacyAdminAttachment200.ts | 9 - ...viewPrivacyAdminAttachment200Attachment.ts | 25 - ...cyAdminAttachment200AttachmentMediaType.ts | 14 - ...acyAdminAttachment200AttachmentMetadata.ts | 45 - ...vacyAdminAttachment200AttachmentPurpose.ts | 12 - ...ttachment200AttachmentRightsReviewState.ts | 12 - .../reviewPrivacyAdminAttachment200Request.ts | 40 - ...ivacyAdminAttachment200RequestExtension.ts | 10 - ...iewPrivacyAdminAttachment200RequestKind.ts | 11 - .../reviewPrivacyAdminAttachment400.ts | 14 - .../reviewPrivacyAdminAttachment401.ts | 14 - .../reviewPrivacyAdminAttachment403.ts | 14 - .../reviewPrivacyAdminAttachment404.ts | 14 - .../reviewPrivacyAdminAttachment409.ts | 14 - .../reviewPrivacyAdminAttachment410.ts | 14 - .../reviewPrivacyAdminAttachment413.ts | 14 - .../reviewPrivacyAdminAttachment416.ts | 14 - .../reviewPrivacyAdminAttachment429.ts | 14 - .../reviewPrivacyAdminAttachment500.ts | 14 - .../reviewPrivacyAdminAttachment503.ts | 14 - .../reviewPrivacyAdminAttachmentBody.ts | 17 - ...eviewPrivacyAdminAttachmentBodyDecision.ts | 10 - .../reviewPrivacyAdminAttachmentHeaders.ts | 9 - .../reviewPrivacyAdminIdentity200.ts | 7 - .../reviewPrivacyAdminIdentity200Request.ts | 40 - ...PrivacyAdminIdentity200RequestExtension.ts | 10 - ...eviewPrivacyAdminIdentity200RequestKind.ts | 11 - .../reviewPrivacyAdminIdentity400.ts | 14 - .../reviewPrivacyAdminIdentity401.ts | 14 - .../reviewPrivacyAdminIdentity403.ts | 14 - .../reviewPrivacyAdminIdentity404.ts | 14 - .../reviewPrivacyAdminIdentity409.ts | 14 - .../reviewPrivacyAdminIdentity410.ts | 14 - .../reviewPrivacyAdminIdentity413.ts | 14 - .../reviewPrivacyAdminIdentity416.ts | 14 - .../reviewPrivacyAdminIdentity429.ts | 14 - .../reviewPrivacyAdminIdentity500.ts | 14 - .../reviewPrivacyAdminIdentity503.ts | 14 - .../reviewPrivacyAdminIdentityBody.ts | 28 - ...rivacyAdminIdentityBodyAuthorityOutcome.ts | 11 - .../reviewPrivacyAdminIdentityBodyMethod.ts | 11 - .../reviewPrivacyAdminIdentityBodyOutcome.ts | 11 - .../reviewPrivacyAdminIdentityBodyParty.ts | 10 - .../reviewPrivacyAdminIdentityHeaders.ts | 9 - .../startPrivacyAdminReauthentication200.ts | 11 - ...rivacyAdminReauthentication200LoginPath.ts | 9 - .../startPrivacyAdminReauthentication400.ts | 14 - .../startPrivacyAdminReauthentication401.ts | 14 - .../startPrivacyAdminReauthentication403.ts | 14 - .../startPrivacyAdminReauthentication404.ts | 14 - .../startPrivacyAdminReauthentication409.ts | 14 - .../startPrivacyAdminReauthentication410.ts | 14 - .../startPrivacyAdminReauthentication413.ts | 14 - .../startPrivacyAdminReauthentication416.ts | 14 - .../startPrivacyAdminReauthentication429.ts | 14 - .../startPrivacyAdminReauthentication500.ts | 14 - .../startPrivacyAdminReauthentication503.ts | 14 - .../startPrivacyAdminReauthenticationBody.ts | 12 - ...PrivacyAdminReauthenticationBodyChannel.ts | 10 - ...tartPrivacyAdminReauthenticationHeaders.ts | 9 - .../uploadPrivacyAdminAttachment201.ts | 7 - ...loadPrivacyAdminAttachment201Attachment.ts | 25 - ...cyAdminAttachment201AttachmentMediaType.ts | 14 - ...acyAdminAttachment201AttachmentMetadata.ts | 45 - ...vacyAdminAttachment201AttachmentPurpose.ts | 12 - ...ttachment201AttachmentRightsReviewState.ts | 12 - .../uploadPrivacyAdminAttachment400.ts | 14 - .../uploadPrivacyAdminAttachment401.ts | 14 - .../uploadPrivacyAdminAttachment403.ts | 14 - .../uploadPrivacyAdminAttachment404.ts | 14 - .../uploadPrivacyAdminAttachment409.ts | 14 - .../uploadPrivacyAdminAttachment410.ts | 14 - .../uploadPrivacyAdminAttachment413.ts | 14 - .../uploadPrivacyAdminAttachment416.ts | 14 - .../uploadPrivacyAdminAttachment429.ts | 14 - .../uploadPrivacyAdminAttachment500.ts | 14 - .../uploadPrivacyAdminAttachment503.ts | 14 - .../uploadPrivacyAdminAttachmentBody.ts | 28 - ...loadPrivacyAdminAttachmentBodyMediaType.ts | 14 - ...ploadPrivacyAdminAttachmentBodyMetadata.ts | 43 - ...uploadPrivacyAdminAttachmentBodyPurpose.ts | 12 - .../uploadPrivacyAdminAttachmentHeaders.ts | 9 - .../src/build-artifacts/control-authority.ts | 40 +- .../build-artifacts/control-retention.test.ts | 88 + .../data-configuration-fence.ts | 14 +- .../src/build-artifacts/retention-family.ts | 50 +- .../core/src/build-artifacts/target-lease.ts | 12 - packages/core/src/ops/contract.ts | 7 +- scripts/api-sdk/codegen.test.ts | 2 +- scripts/api-sdk/config.mjs | 10 +- 639 files changed, 8653 insertions(+), 10535 deletions(-) create mode 100644 .gitattributes create mode 100644 apps/api/src/services/configuration-migration/completion.test.ts create mode 100644 apps/api/src/services/configuration-migration/completion.ts create mode 100644 apps/web/src/lib/admin/downloadSink.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActivationVersion.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200Active.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200ActiveActivationVersion.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy200AppliesTo.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicy503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/applyAdminAuthPolicyBodySettings.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole200Role.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRole503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRoleBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRoleBodyExpectedRole.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRoleBodyRole.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/changeAdminAccountRoleHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthentication503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthenticationBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/completeAdminReauthenticationHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount200PasswordSetupDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount200VerificationDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccount503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccountBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccountBodyRole.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/createAdminAccountHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery200Latest.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery200LatestAction.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery200LatestPasswordSetupDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery200LatestVerificationDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAccountDelivery503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Active.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200ActiveActivationVersion.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Ownership.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipEmailVerificationRequired.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipPasswordMinLength.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipSessionDurationHours.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200OwnershipTwoFaEnabled.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy200Pending.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/getAdminAuthPolicy503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/index.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount200PasswordSetupDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount200VerificationDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccount503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccountBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/inviteAdminAccountHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset200PasswordSetupDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset200VerificationDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordReset503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordResetBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountPasswordResetHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail200PasswordSetupDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail200VerificationDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmail503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmailBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/sendAdminAccountVerificationEmailHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword200PasswordSetupDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword200VerificationDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPassword503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPasswordBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPasswordBodyReasonCode.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/setAdminAccountPasswordHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication200LoginPath.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthentication503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthenticationBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthenticationBodyPurpose.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/startAdminReauthenticationHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount200.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount200PasswordSetupDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount200VerificationDelivery.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount400.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount401.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount403.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount404.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount409.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount413.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount415.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount429.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount500.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccount503.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccountBody.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccountBodyReasonCode.ts delete mode 100644 packages/core/src/api/generated/admin/account-models/verifyAdminAccountHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions201.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions201Review.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissions503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissionsBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/acceptPrivacyAdminBackupOmissionsHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/adminGeneratePrivacyAdminArtifact201Schema0.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/adminGetPrivacyAdminReadiness200Schema0.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask200Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask200RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask200RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTask503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTaskBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/assignPrivacyAdminTaskHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallenge503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallengeBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallengeBodyParty.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminEmailChallengeHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthentication503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthenticationBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/completePrivacyAdminReauthenticationHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview201.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview201Review.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview201ReviewDecision.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReview503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReviewBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminBackupReviewHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase202.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase202Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase202RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase202RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCase503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBodyChannel.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBodyKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBodyRepresentative.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBodyRepresentativeContactType.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBodySubject.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBodySubjectAccountState.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseBodySubjectLocatorType.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/createPrivacyAdminCaseHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase200Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase200RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase200RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCase503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCaseBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCaseBodyDecision.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminCaseHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTask503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTaskBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTaskBodyStatus.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/decidePrivacyAdminTaskHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachment503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachmentBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/deletePrivacyAdminAttachmentHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminArtifact503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/downloadPrivacyAdminAttachment503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase200Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase200RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase200RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCase503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCaseBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/extendPrivacyAdminCaseHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact201.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact201Artifact.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact201Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact201RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact201RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifact503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifactBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/generatePrivacyAdminArtifactHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200ArtifactsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200AttachmentsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200AttachmentsItemMediaType.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200AttachmentsItemMetadata.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200AttachmentsItemPurpose.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200AttachmentsItemRightsReviewState.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200Capabilities.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200CapabilitiesReasons.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200EventsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200EventsItemPayload.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200IdentitiesItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200IdentitiesItemDeliveryAuthorized.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200IdentitiesItemParty.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200TasksItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200TasksItemActionsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase200TasksItemRequired.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminCase503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness200ChecksItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness200ChecksItemStatus.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness200KeyRing.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness200Version.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/getPrivacyAdminReadiness503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/index.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge202.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge202DeliveryState.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallenge503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallengeBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallengeBodyParty.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/issuePrivacyAdminEmailChallengeHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions200ReviewsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupOmissions503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews200ReviewsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews200ReviewsItemDecision.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackupReviews503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups200BackupsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups200BackupsItemVolumesItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups200BackupsItemVolumesItemMode.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminBackups503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators200OperatorsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperators503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminOperatorsParams.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue200RequestsItem.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue200RequestsItemExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue200RequestsItemKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/listPrivacyAdminQueue503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200Attachment.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200AttachmentMediaType.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200AttachmentMetadata.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200AttachmentPurpose.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200AttachmentRightsReviewState.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment200RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachment503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachmentBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachmentBodyDecision.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminAttachmentHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity200Request.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity200RequestExtension.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity200RequestKind.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentity503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentityBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentityBodyAuthorityOutcome.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentityBodyMethod.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentityBodyOutcome.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentityBodyParty.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/reviewPrivacyAdminIdentityHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication200.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication200LoginPath.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthentication503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthenticationBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthenticationBodyChannel.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/startPrivacyAdminReauthenticationHeaders.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment201.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment201Attachment.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment201AttachmentMediaType.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment201AttachmentMetadata.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment201AttachmentPurpose.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment201AttachmentRightsReviewState.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment400.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment401.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment403.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment404.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment409.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment410.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment413.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment416.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment429.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment500.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachment503.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachmentBody.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachmentBodyMediaType.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachmentBodyMetadata.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachmentBodyPurpose.ts delete mode 100644 packages/core/src/api/generated/admin/privacy-models/uploadPrivacyAdminAttachmentHeaders.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..63cf04ad2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Reproducible outputs: review their contracts/generators first, then expand +# these files when checking the generated wire surface or migration metadata. +apps/api/openapi.json linguist-generated=true +services/data-manager/openapi.json linguist-generated=true +packages/core/src/api/generated/** linguist-generated=true +packages/core/src/services/data-manager/generated/** linguist-generated=true +apps/api/src/db/migrations/meta/*_snapshot.json linguist-generated=true diff --git a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts index 1f466c9e8..fb91046d6 100644 --- a/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-integrations-sdk.test.ts @@ -38,13 +38,11 @@ vi.mock("../../services/job-runner.js", () => ({ const configDb = integrationConfigDbFixture(); const mockApplyWithEvidence = vi.fn(); -const mockCompensate = vi.fn(); const mockApply = vi.fn().mockResolvedValue({ revisionId: "revision_1" }); vi.mock("../../services/trusted-config-operations.js", () => ({ integrationSchemas: () => new Map([["geocoding-maptiler", MOCK_INTEGRATION.manifest.configSchema]]), applyTrustedConfigurationWithEvidence: () => mockApplyWithEvidence(), - applyTrustedConfigurationSnapshot: (...args: unknown[]) => mockCompensate(...args), applyTrustedConfiguration: (...args: unknown[]) => mockApply(...args), })); @@ -282,7 +280,6 @@ beforeEach(() => { registryUpdateFailed: false, }, }); - mockCompensate.mockReset().mockRejectedValue(Error("Must not compensate")); mockApply.mockReset().mockResolvedValue({ revisionId: "revision_1", integrationReload: { message: "Integrations reloaded", reloaded: 1, enabled: 1 }, @@ -869,7 +866,6 @@ describe("actual integration producers through generated SDK", () => { expect(configDb.rows[0].config).toEqual({ label: "saved-before" }); expect(configDb.writes).toBe(0); expect(mockImmediateImport).toHaveBeenCalledTimes(1); - expect(mockCompensate).not.toHaveBeenCalled(); expect(mockApplyWithEvidence).not.toHaveBeenCalled(); expect(mockReloadIntegrations).not.toHaveBeenCalled(); } finally { diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts index 8b6fe68bc..1ef80c5a9 100644 --- a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -259,6 +259,109 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ); } + it.each(["prepared", "save_attempted"])( + "recovers only the proven-unissued ordinary checkpoint (phase=%s)", + async (phase) => { + const ctx = await job(), + checkpoint = ctx.checkpoint; + ctx.checkpoint = async (...args) => { + await checkpoint(...args); + if ((args[0]._configurationCandidate as { phase?: string } | undefined)?.phase === phase) + throw Error("API stopped after durable checkpoint"); + }; + await expect( + owned(ctx, async () => { + const candidate = await prospective.prepareProspectiveConfiguration({ + actorId: ctx.jobId, + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, + expectedCurrentRevisionId: host.initial.revisionId, + overlay: { + purpose: "service-config", + coordinatorMode: "ensure-running", + changes: [ + { kind: "service-config", serviceId: "motis", set: { WORKERS: 7 }, unset: [] }, + ], + }, + }); + await prospective.persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "ensure-running", + }); + }), + ).rejects.toThrow("API stopped after durable checkpoint"); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual([]); + const [original] = await fixture.db + .select() + .from(schema.adminJob) + .where(eq(schema.adminJob.id, ctx.jobId)); + const candidate = (original!.result as Record) + ._configurationCandidate as Record, + path = join(host.queue, ".candidates", `${candidate.sealedOperationKey}.json`), + bytes = readFileSync(path), + { claimAdminJob } = await import("../admin-job-state"), + reclaimed = await claimAdminJob(original!); + ctx.execution = ( + reclaimed!.result as { _execution: NonNullable } + )._execution; + ctx.checkpointResult = reclaimed!.result as Record; + ctx.checkpoint = async (result) => { + await fixture.db + .update(schema.adminJob) + .set({ result }) + .where(eq(schema.adminJob.id, ctx.jobId)); + ctx.checkpointResult = result; + }; + if (phase === "save_attempted") { + await expect( + owned(ctx, () => prospective.resumeRetainedConfigurationCandidate()), + ).rejects.toThrow(); + expect(readFileSync(path).equals(bytes)).toBe(true); + await expect(owned(await job(), async () => undefined)).rejects.toThrow( + "Configuration writer unavailable", + ); + } else { + const complete = ctx.checkpoint; + let interrupted = false; + ctx.checkpoint = async (...args) => { + if ( + !interrupted && + (args[0]._configurationCandidate as { phase?: string } | undefined)?.phase === + "discarded" + ) { + interrupted = true; + throw Error("API stopped after original unlink"); + } + return complete(...args); + }; + await expect( + owned(ctx, () => prospective.resumeRetainedConfigurationCandidate()), + ).rejects.toThrow("API stopped after original unlink"); + expect(existsSync(path)).toBe(false); + expect( + await prospective.retainedConfigurationIntegrationIds( + ctx, + ctx.jobId, + String(candidate.operationKey), + ), + ).toEqual([]); + const result = await owned(ctx, () => prospective.resumeRetainedConfigurationCandidate()); + expect(result.configuration).toMatchObject({ + outcome: "not_committed", + cleanupFailed: false, + }); + expect(existsSync(path)).toBe(false); + expect(ctx.checkpointResult?._configurationCandidate).toMatchObject({ + phase: "discarded", + }); + await expect(owned(await job(), async () => "available")).resolves.toBe("available"); + } + expect(actualPosts.get("serviceConfig.apply") ?? 0).toBe(0); + expect(await fixture.db.select().from(schema.serviceConfig)).toEqual([]); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe( + host.initial.revisionId, + ); + }, + 30000, + ); it.each([ "saved", "applied", @@ -3352,37 +3455,31 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const ctx = await job(); await owned(ctx, async () => { const candidate = await prospective.prepareProspectiveConfiguration({ - actorId: ctx.jobId, - operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, - expectedCurrentRevisionId: host.initial.revisionId, - followUp: "service-apply", - overlay: { - purpose: "service-secret", - coordinatorMode: "require-running", - changes: [ - { - kind: "service-secret", - serviceId: "data-manager", - key: "TOKEN", - action: "set", - value: "synthetic-rollback-unused-credential", - }, - ], - }, - }), - current = await import("../trusted-config-operations"); - await fixture.sql`ALTER TABLE service_secret ADD CONSTRAINT fixture_reject_desired_token CHECK (key <> 'TOKEN')`; - const result = await current.applyTrustedConfigurationWithEvidence({ - ctx, actorId: ctx.jobId, - kind: "vault.apply", - serviceId: "data-manager", - operationKey: candidate.operationKey, - candidate, - coordinatorMode: "require-running", - followUp: "service-apply", + operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, expectedCurrentRevisionId: host.initial.revisionId, + followUp: "service-apply", + overlay: { + purpose: "service-secret", + coordinatorMode: "require-running", + changes: [ + { + kind: "service-secret", + serviceId: "data-manager", + key: "TOKEN", + action: "set", + value: "synthetic-rollback-unused-credential", + }, + ], + }, }); + await fixture.sql`ALTER TABLE service_secret ADD CONSTRAINT fixture_reject_desired_token CHECK (key <> 'TOKEN')`; + const result = await prospective + .persistAndApplyProspectiveConfiguration(candidate, { + coordinatorMode: "require-running", + signal: ctx.signal, + }) + .then(({ application }) => ({ outcome: application })); expect(result.outcome).toMatchObject({ outcome: "not_committed", reason: "conflict" }); expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( @@ -4338,18 +4435,12 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( return fetchActual(input, options); }) : undefined; - const current = await import("../trusted-config-operations"), - application = current.applyTrustedConfigurationWithEvidence({ - ctx, - actorId: ctx.jobId, - kind: "vault.apply", - serviceId: "data-manager", - operationKey: candidate.operationKey, - candidate, + const application = prospective + .persistAndApplyProspectiveConfiguration(candidate, { coordinatorMode: "require-running", - followUp: "service-apply", - expectedCurrentRevisionId: host.initial.revisionId, - }); + signal: ctx.signal, + }) + .then(({ application }) => ({ outcome: application })); void application.catch(() => {}); if (mode === "sql-control-expired") { let lateStatus: number; diff --git a/apps/api/src/services/__tests__/trusted-config-operations.test.ts b/apps/api/src/services/__tests__/trusted-config-operations.test.ts index 2f5cd8499..fea016338 100644 --- a/apps/api/src/services/__tests__/trusted-config-operations.test.ts +++ b/apps/api/src/services/__tests__/trusted-config-operations.test.ts @@ -177,18 +177,6 @@ describe("retained trusted configuration producer", () => { }, ); - it("does not restore a snapshot different from the strictly owned candidate", async () => { - await expect( - subject.applyTrustedConfigurationSnapshot({ - payload: { ...payload, domain: "changed.example.test" }, - expectedCurrentRevisionId: previousRevisionId, - operationKey: key, - ctx: owner(), - }), - ).rejects.toThrow("Strict compensation requires current owned desired rows"); - expect(ports.apply).not.toHaveBeenCalled(); - }); - it("keeps a known failure actionable in the owned checkpoint and public immediate response", async () => { const ctx = owner(); const problem = "services/martin/config/martin.yaml has more than one link"; diff --git a/apps/api/src/services/configuration-migration/apply.ts b/apps/api/src/services/configuration-migration/apply.ts index a5f7eeb61..54e54ae53 100644 --- a/apps/api/src/services/configuration-migration/apply.ts +++ b/apps/api/src/services/configuration-migration/apply.ts @@ -18,9 +18,10 @@ import { retireSettledProspectiveConfiguration, } from "../prospective-configuration"; import { completeTrustedConfigurationApply } from "../trusted-config-outcome"; +import { migrationCompletionResult } from "./completion"; import type { CredentialPlan } from "./credential-preview"; import { type MigrationProof, prepareStagedMigrationParticipant } from "./participant"; -import { type MigrationPlan, migrationPlanPreview, previewConfigurationMigration } from "./preview"; +import { type MigrationPlan, previewConfigurationMigration } from "./preview"; import { MigrationConflict, readMigrationStage } from "./staging"; export type MigrationResult = { @@ -119,6 +120,7 @@ export async function runConfigurationMigration(ctx: JobContext, input: Migratio await ctx.checkpoint({ ...ctx.checkpointResult, _migrationSteps: steps, + _migrationPendingServiceIds: plan.pendingServiceIds ?? [], _migrationStage: { id: stage.previewId, kind: stage.kind, @@ -151,10 +153,6 @@ export async function runConfigurationMigration(ctx: JobContext, input: Migratio const reloaded = await reloadProspectiveIntegrations(candidate); await ctx.checkpoint({ ...ctx.checkpointResult, - _configurationCandidate: { - ...(ctx.checkpointResult?._configurationCandidate as Record), - phase: "settled", - }, integrationReload: { state: "complete", candidateId: candidate.candidateId, @@ -162,13 +160,15 @@ export async function runConfigurationMigration(ctx: JobContext, input: Migratio result: reloaded, }, }); - for (const step of steps) - if ( - step.scope !== "pending-auth" && - !(step.scope.startsWith("service-") && plan.pendingServiceIds?.includes(step.id)) - ) - step.status = "applied"; - outcome = steps.some((s) => s.status === "pending") ? "partial" : "applied"; + const completed = migrationCompletionResult({ + operationId: ctx.jobId, + steps, + pendingServiceIds: plan.pendingServiceIds ?? [], + application: receipt.application, + }); + steps.splice(0, steps.length, ...completed.steps); + outcome = completed.outcome; + await ctx.checkpoint({ ...ctx.checkpointResult, migration: completed }); failurePhase = "retirement"; await retireSettledProspectiveConfiguration(candidate); } catch (error) { diff --git a/apps/api/src/services/configuration-migration/completion.test.ts b/apps/api/src/services/configuration-migration/completion.test.ts new file mode 100644 index 000000000..348a3382a --- /dev/null +++ b/apps/api/src/services/configuration-migration/completion.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { migrationCompletionResult } from "./completion"; + +const application = { + outcome: "committed" as const, + revisionId: `cfg1_${"r".repeat(43)}`, + enabledServiceIds: ["motis"], + followUp: "complete" as const, + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, +}; +describe("migration completion", () => { + it("preserves captured disabled services and pending auth after a recovered apply", () => { + expect( + migrationCompletionResult({ + operationId: "original-job", + application, + steps: [ + { scope: "service-config", id: "disabled", status: "pending" }, + { scope: "service-secret", id: "motis", status: "pending" }, + { scope: "integration-config", id: "alpha", status: "pending" }, + { scope: "pending-auth", id: "registrationEnabled", status: "pending" }, + ], + pendingServiceIds: ["disabled"], + }), + ).toEqual({ + operationId: "original-job", + outcome: "partial", + steps: [ + { scope: "service-config", id: "disabled", status: "pending" }, + { scope: "service-secret", id: "motis", status: "applied" }, + { scope: "integration-config", id: "alpha", status: "applied" }, + { scope: "pending-auth", id: "registrationEnabled", status: "pending" }, + ], + }); + }); + it("does not infer activation when an older checkpoint omitted the captured classification", () => { + expect( + migrationCompletionResult({ + operationId: "original-job", + application, + steps: [{ scope: "service-secret", id: "disabled", status: "pending" }], + }), + ).toEqual({ + operationId: "original-job", + outcome: "unknown", + steps: [{ scope: "service-secret", id: "disabled", status: "unknown" }], + }); + }); +}); diff --git a/apps/api/src/services/configuration-migration/completion.ts b/apps/api/src/services/configuration-migration/completion.ts new file mode 100644 index 000000000..43332c041 --- /dev/null +++ b/apps/api/src/services/configuration-migration/completion.ts @@ -0,0 +1,39 @@ +import { + completeTrustedConfigurationApply, + type TrustedConfigurationApplyOutcome, +} from "../trusted-config-outcome"; +import type { MigrationResult } from "./apply"; + +/** Result projection only. The captured disabled targets remain pending even + * when recovery runs after service selection has changed. Missing old evidence + * cannot establish activation. This helper grants no execution authority. */ +export function migrationCompletionResult(input: { + operationId: string; + steps: MigrationResult["steps"]; + pendingServiceIds?: readonly string[]; + application: TrustedConfigurationApplyOutcome; +}): MigrationResult { + const known = input.pendingServiceIds !== undefined, + complete = completeTrustedConfigurationApply(input.application), + pending = new Set(input.pendingServiceIds), + steps = input.steps.map((step) => ({ + ...step, + status: + step.scope === "pending-auth" || + (known && step.scope.startsWith("service-") && pending.has(step.id)) + ? ("pending" as const) + : complete && known + ? ("applied" as const) + : ("unknown" as const), + })); + return { + operationId: input.operationId, + outcome: + !complete || !known + ? "unknown" + : steps.some((step) => step.status === "pending") + ? "partial" + : "applied", + steps, + }; +} diff --git a/apps/api/src/services/configuration-migration/migration-postgres.test.ts b/apps/api/src/services/configuration-migration/migration-postgres.test.ts index 7b3062253..8f02b77fe 100644 --- a/apps/api/src/services/configuration-migration/migration-postgres.test.ts +++ b/apps/api/src/services/configuration-migration/migration-postgres.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { mkdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import * as fsPromises from "node:fs/promises"; import { join } from "node:path"; import { createApiClient } from "@openmapx/core"; import { @@ -13,6 +14,7 @@ import { } from "@openmapx/core/api-admin-sdk"; import { closeControlInstallation, openControlInstallation } from "@openmapx/core/build-artifacts"; import type { services } from "@openmapx/core/server"; +import { eq } from "drizzle-orm"; import Fastify from "fastify"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; @@ -26,6 +28,10 @@ const state = vi.hoisted(() => ({ fixture: null as unknown as Awaited>, loaded: [] as services.LoadedService[], })); +vi.mock("node:fs/promises", async (original) => { + const actual = await original(); + return { ...actual, unlink: vi.fn(actual.unlink) }; +}); vi.mock("../../db", () => ({ get db() { return state.fixture.db; @@ -44,7 +50,7 @@ vi.mock("../service-registry", async (actual) => ({ })); import type { ConfigurationMigrationDocument } from "../../../../../packages/core/src/api/contracts/integration-admin/configuration-migration"; -import { integrationConfig, systemSettings, user } from "../../db/schema"; +import { adminJob, integrationConfig, systemSettings, user } from "../../db/schema"; import { createDurableOpsKey } from "../ops-client"; describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( @@ -212,6 +218,79 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(JSON.stringify(portable)).not.toContain("***"); expect(JSON.stringify(portable)).not.toContain("OPENMAPX_SECRETS_KEY"); }, 90000); + it("recovers a migration after packet unlink using its original durable completion", async () => { + const { jobRunner } = await import("../job-runner"), + preview = await jobRunner.runImmediateConfigurationMigrationPreview({ + actorId: "migration-admin", + operationKey: createDurableOpsKey("migration.test.retirement-preview", randomUUID()), + document: document("retirement-recovery"), + }), + input = { + actorId: "migration-admin", + operationKey: createDurableOpsKey("migration.test.retirement-apply", randomUUID()), + previewId: preview.previewId, + fingerprint: preview.fingerprint, + kind: "migration" as const, + }, + originalUnlink = ( + await vi.importActual("node:fs/promises") + ).unlink; + let interrupted = false; + vi.mocked(fsPromises.unlink).mockImplementation(async (path) => { + if (!interrupted && String(path).includes("/.candidates/")) { + const row = (await state.fixture.db.select().from(adminJob)).find( + (value) => + (value.payload as { operationKey?: string })?.operationKey === input.operationKey, + ); + const retained = row?.result as Record | undefined; + expect(retained?._configurationCandidate).toMatchObject({ + phase: "settled", + retirement: { version: 1 }, + }); + expect((retained?._completion as { result?: unknown } | undefined)?.result).toMatchObject( + { + migration: { outcome: "applied", operationId: row!.id }, + }, + ); + await originalUnlink(path); + interrupted = true; + throw Error("API interrupted after migration packet unlink"); + } + return originalUnlink(path); + }); + try { + await expect(jobRunner.runImmediateConfigurationMigration(input)).rejects.toThrow(); + expect(interrupted).toBe(true); + const rows = await state.fixture.db.select().from(adminJob), + row = rows.find( + (value) => + (value.payload as { operationKey?: string })?.operationKey === input.operationKey, + )!; + expect(row.status).toBe("running"); + const retained = row.result as Record; + await state.fixture.db + .update(adminJob) + .set({ + result: { + ...retained, + _observationLeaseUntil: new Date(0).toISOString(), + observation: { + state: "unknown", + reason: "recovery_required", + nextCheckAt: new Date(0).toISOString(), + }, + }, + }) + .where(eq(adminJob.id, row.id)); + const result = await jobRunner.runImmediateConfigurationMigration(input); + expect(result).toEqual( + (retained._completion as { result: { migration: unknown } }).result.migration, + ); + expect(await jobRunner.runImmediateConfigurationMigration(input)).toEqual(result); + } finally { + vi.mocked(fsPromises.unlink).mockImplementation(originalUnlink); + } + }, 90000); it("denies secret injection and stale source with zero selected writes", async () => { const { jobRunner } = await import("../job-runner"); const invalid = document("not-written"); diff --git a/apps/api/src/services/job-runner.ts b/apps/api/src/services/job-runner.ts index 6504cc0a1..c07963e6b 100644 --- a/apps/api/src/services/job-runner.ts +++ b/apps/api/src/services/job-runner.ts @@ -639,23 +639,7 @@ class AdminJobRunner { (result) => { if (result.migration) return result.migration as import("./configuration-migration/apply").MigrationResult; - const steps = (result._migrationSteps ?? - []) as import("./configuration-migration/apply").MigrationResult["steps"], - applied = completeTrustedConfigurationApply( - parseTrustedConfigurationApplyOutcome(result.configuration), - ); - return { - operationId: String(result._immediateJobId), - outcome: applied - ? steps.some((s) => s.scope === "pending-auth") - ? "partial" - : "applied" - : "unknown", - steps: steps.map((s) => ({ - ...s, - status: s.scope === "pending-auth" ? "pending" : applied ? "applied" : "unknown", - })), - } as import("./configuration-migration/apply").MigrationResult; + throw new JobObservationPending("recovery_required"); }, ); } @@ -677,7 +661,6 @@ class AdminJobRunner { selectedRoots: options.selectedRoots, overlay: options.overlay, coordinatorMode: options.coordinatorMode, - snapshot: options.snapshot, followUp: options.followUp, ...(Object.hasOwn(options, "expectedCurrentRevisionId") ? { expectedCurrentRevisionId: options.expectedCurrentRevisionId } diff --git a/apps/api/src/services/prospective-configuration.ts b/apps/api/src/services/prospective-configuration.ts index db4b418d3..73ad4e1f3 100644 --- a/apps/api/src/services/prospective-configuration.ts +++ b/apps/api/src/services/prospective-configuration.ts @@ -102,6 +102,8 @@ import { completedCoordinatorMaintenance, completedCoordinatorMaintenanceSchema, } from "./configuration-maintenance-step-records"; +import type { MigrationResult } from "./configuration-migration/apply"; +import { migrationCompletionResult } from "./configuration-migration/completion"; import { assertLegacyIntegrationImportOverlay } from "./configuration-migration/legacy-integration-import-intent"; import { assertStagedMigrationOverlay, @@ -1079,7 +1081,25 @@ export async function assertSettledConfigurationCandidate( receipt.revisionId !== candidate.revisionId || receipt.operationKey !== candidate.operationKey || receipt.sealedOperationKey !== state.record.operationKey || - receipt.phase !== "settled" || + (receipt.phase !== "settled" && + !( + state.record.privateIntegrationReload?.stagedMigration && + ["saved", "follow_up_pending"].includes(String(receipt.phase)) && + canonicalJson(result.migration) === + canonicalJson( + migrationCompletionResult({ + operationId: owner.job.jobId, + steps: (result._migrationSteps ?? []) as MigrationResult["steps"], + pendingServiceIds: z + .array(id) + .max(256) + .optional() + .parse(result._migrationPendingServiceIds), + application: parseTrustedConfigurationApplyOutcome(result.configuration), + }), + ) && + (result.migration as MigrationResult | undefined)?.outcome !== "unknown" + )) || hasPendingReceiptDelivery(result) || hasUnsettledManagedOperation(result) ) @@ -1185,7 +1205,8 @@ export async function retireSettledProspectiveConfiguration( if ( proof?.candidateId !== candidate.candidateId || proof.revisionId !== candidate.revisionId || - proof.phase !== "settled" + (proof.phase !== "settled" && + !(state.record.privateIntegrationReload?.stagedMigration && proof.phase === "saved")) ) throw new JobObservationPending("recovery_required"); const [retired] = await tx @@ -1218,6 +1239,8 @@ export async function retireSettledProspectiveConfiguration( "configuration.apply", "integration.config.import", "integration.binding.apply", + "configuration.migration.apply", + "credential.import", ].includes(job.jobType ?? "")) ) { const original = job.checkpointResult; @@ -1226,6 +1249,7 @@ export async function retireSettledProspectiveConfiguration( reload = original.integrationReload as { result?: Record } | undefined, result = { configuration: original.configuration, + ...(original.migration ? { migration: original.migration } : {}), ...(original.serviceCredential ? { serviceCredential: original.serviceCredential } : {}), ...(original.coordinatorLifecycle ? { coordinatorLifecycle: original.coordinatorLifecycle } @@ -1294,6 +1318,7 @@ export async function retireSettledProspectiveConfiguration( _completion: { operationKey, result }, _configurationCandidate: { ...(original._configurationCandidate as Record), + phase: "settled", retirement: { version: 1, integrationIds: [...state.writer.integrationIds] }, }, }); @@ -1729,6 +1754,12 @@ export async function retainedConfigurationIntegrationIds( const retained = job.checkpointResult?._configurationCandidate as | Record | undefined; + if ( + retained?.maintenanceId === null && + ["discarding_unissued", "discarded"].includes(String(retained.phase)) && + retained.operationKey === operationKey + ) + return retirementSchema.parse(retained.unissuedCleanup).integrationIds; if (retained?.retirement !== undefined) return retirementSchema.parse(retained.retirement).integrationIds; if (typeof retained?.sealedOperationKey !== "string") throw Error(); @@ -2485,6 +2516,131 @@ async function restoreRetainedMaintenanceCandidate(): Promise> { + const writer = currentAdministrativeConfigurationWriter(), + { job } = administrativeConfigurationState(writer), + publisher = publisherOptions(await readOpsTokenFile(envString("OPS_AGENT_TOKEN_FILE", ""))); + let checkpoint: Record | undefined; + await withConfigurationWriteTransaction(writer, async (tx) => { + const [row] = await tx + .select() + .from(schema.adminJob) + .where(jobOwnership(job.jobId, job.execution)) + .for("update"), + result = row?.result as Record | undefined, + retained = result?._configurationCandidate as Record | undefined; + if ( + !row || + !result || + row.type !== job.jobType || + canonicalJson(row.payload ?? {}) !== canonicalJson(job.payload) || + !job.execution || + job.execution.version < 2 || + (writer.actorId !== job.jobId && result._immediateActorId !== writer.actorId) || + retained?.operationKey !== writer.operationKey || + retained.maintenanceId !== null || + retained.prepareOperationKey !== null || + retained.rowVersions !== undefined || + !["prepared", "discarding_unissued", "discarded"].includes(String(retained.phase)) || + result._configurationMaintenance !== undefined || + result._configurationMaintenanceDecision !== undefined || + result._configurationMaintenanceRollback !== undefined || + Object.keys(readProjection(result).operations).length !== 0 || + hasPendingReceiptDelivery(result) || + hasUnsettledManagedOperation(result) + ) + throw new JobObservationPending("recovery_required"); + if (retained.phase === "prepared") { + const { record, snapshot } = await readPreparedTrustedConfigurationCandidate( + publisher, + String(retained.sealedOperationKey), + ), + operation = opsOperationSchema.parse(record.operation); + if ( + record.apiJobId !== job.jobId || + record.actorId !== writer.actorId || + record.candidateId !== retained.candidateId || + record.operationKey !== retained.sealedOperationKey || + record.sourceFingerprint !== retained.sourceFingerprint || + snapshot.revisionId !== retained.revisionId || + snapshot.operationFingerprint !== opsOperationFingerprint(operation) || + record.maintenanceId !== null || + record.prepareOperationKey !== null || + canonicalJson( + retainedIntegrationIds(record, overlaySchema.parse(record.privateOverlay)), + ) !== canonicalJson(writer.integrationIds) + ) + throw new JobObservationPending("recovery_required"); + } else if ( + canonicalJson(retirementSchema.parse(retained.unissuedCleanup).integrationIds) !== + canonicalJson(writer.integrationIds) + ) + throw new JobObservationPending("recovery_required"); + checkpoint = { + ...result, + _configurationCandidate: { + ...retained, + phase: "discarding_unissued", + unissuedCleanup: { version: 1, integrationIds: [...writer.integrationIds] }, + }, + }; + const saved = await tx + .update(schema.adminJob) + .set({ result: checkpoint }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + }); + if (!checkpoint) throw new JobObservationPending("recovery_required"); + job.checkpointResult = checkpoint; + const handle = Object.freeze({}) as RetainedConfigurationRetirement; + retirements.set(handle, { + writer, + publisher, + checkpoint: checkpoint._configurationCandidate as Record, + apiJobId: job.jobId, + actorId: writer.actorId, + }); + try { + await completeRetainedConfigurationRetirement(handle); + } finally { + retirements.delete(handle); + } + const configuration: TrustedConfigurationApplyOutcome = { + outcome: "not_committed", + reason: "conflict", + cleanupFailed: false, + publisherCleanupFailed: false, + registryUpdateFailed: false, + }; + await job.checkpoint({ + ...job.checkpointResult, + _configurationCandidate: { + ...(checkpoint._configurationCandidate as Record), + phase: "discarded", + }, + configuration, + _configurationOutcomes: { + ...(checkpoint._configurationOutcomes as Record), + [writer.operationKey]: configuration, + }, + }); + if (checkpoint._cancelRequestedAt !== undefined) { + await withConfigurationWriteTransaction(writer, async (tx) => { + const saved = await tx + .update(schema.adminJob) + .set({ status: "canceled", finishedAt: new Date(), result: job.checkpointResult }) + .where(jobOwnership(job.jobId, job.execution)) + .returning({ id: schema.adminJob.id }); + if (saved.length !== 1) throw new JobObservationPending("recovery_required"); + }); + } + return { configuration }; +} + /** The existing ordinary generation producer, with no reload/credential or * participant continuation. Its saved row receipts and exact original private * packet are authoritative; current desired values never rebuild its seal. */ @@ -2808,6 +2964,11 @@ export async function reconcileUnstagedCoordinatorPull(): Promise { export async function resumeRetainedConfigurationCandidate(): Promise> { const current = administrativeConfigurationState(currentAdministrativeConfigurationWriter()).job .checkpointResult?._configurationCandidate as Record | undefined; + if ( + current?.maintenanceId === null && + ["prepared", "discarding_unissued", "discarded"].includes(String(current.phase)) + ) + return discardRetainedUnissuedOrdinaryCandidate(); if (current?.retirement !== undefined) return resumeCompletedConfigurationRetirement(); let candidate: ValidatedConfigurationCandidate; try { @@ -2921,10 +3082,6 @@ export async function resumeRetainedConfigurationCandidate(): Promise), - phase: "settled", - }, integrationReload: { state: "complete", candidateId: candidate.candidateId, @@ -2933,10 +3090,34 @@ export async function resumeRetainedConfigurationCandidate(): Promise), + phase: "settled", + }, + }); + } await retireSettledProspectiveConfiguration(candidate); const provenance = state.record.privateIntegrationReload?.legacyImport; return { configuration: receipt.application, + ...(migration ? { migration } : {}), ...(provenance ? { integrationImport: { @@ -4808,7 +4989,11 @@ export async function persistAndApplyProspectiveConfiguration( ...owner.job.checkpointResult, _configurationCandidate: { ...checkpoint, - phase: registryUpdateFailed ? "unknown" : "settled", + phase: registryUpdateFailed + ? "unknown" + : state.record.privateIntegrationReload?.stagedMigration + ? "saved" + : "settled", }, configuration: application, _configurationOutcomes: { diff --git a/apps/api/src/services/trusted-configuration-producer.ts b/apps/api/src/services/trusted-configuration-producer.ts index e07b2d2fa..c248f6683 100644 --- a/apps/api/src/services/trusted-configuration-producer.ts +++ b/apps/api/src/services/trusted-configuration-producer.ts @@ -6,7 +6,6 @@ import { configurationGenerationRevisionSchema, type OpsResultFor, type TrustedConfigurationPayload, - trustedConfigurationPayloadSchema, } from "@openmapx/core/ops"; import { getAllIntegrations } from "../integration-host"; import { AdminJobFailure, JobObservationPending } from "./admin-job-failure"; @@ -22,7 +21,6 @@ import { persistAndApplyProspectiveConfiguration, prepareProspectiveConfiguration, retireSettledProspectiveConfiguration, - type ValidatedConfigurationCandidate, } from "./prospective-configuration"; import { completeTrustedConfigurationApply, @@ -57,9 +55,6 @@ export interface ApplyTrustedConfigurationOptions { followUp?: "service-apply"; expectedCurrentRevisionId?: string | null; overlay?: ProspectiveConfigurationOverlay; - candidate?: ValidatedConfigurationCandidate; - /** Internal compensation must still equal the strictly acquired desired snapshot. */ - snapshot?: TrustedConfigurationPayload; } export { buildConfigurationInput } from "./configuration-input"; @@ -151,7 +146,7 @@ export async function applyTrustedConfigurationWithEvidence( : options.kind === "stack.render" && options.selectedRoots === undefined ? { purpose: "render" as const, coordinatorMode, changes: [] } : undefined); - if (!overlay && !options.candidate) + if (!overlay) return { outcome: { outcome: "not_committed", @@ -182,16 +177,14 @@ export async function applyTrustedConfigurationWithEvidence( const expectedCurrentRevisionId = Object.hasOwn(options, "expectedCurrentRevisionId") ? options.expectedCurrentRevisionId! : previousApplied.revisionId; - const candidate = - options.candidate ?? - (await prepareProspectiveConfiguration({ - actorId, - operationKey: options.operationKey, - expectedCurrentRevisionId, - overlay: overlay!, - signal: options.signal ?? ctx.signal, - followUp: options.followUp, - })); + const candidate = await prepareProspectiveConfiguration({ + actorId, + operationKey: options.operationKey, + expectedCurrentRevisionId, + overlay, + signal: options.signal ?? ctx.signal, + followUp: options.followUp, + }); const { validatedConfigurationCandidateState } = await import( "./prospective-configuration" ); @@ -212,8 +205,6 @@ export async function applyTrustedConfigurationWithEvidence( throw Error("Configuration candidate intent conflict"); if (options.followUp && candidateState.record.apiFollowUp !== options.followUp) throw Error("Configuration continuation intent conflict"); - if (options.snapshot && canonicalJson(options.snapshot) !== canonicalJson(payload)) - throw new Error("Strict compensation requires current owned desired rows"); const receipt = await persistAndApplyProspectiveConfiguration(candidate, { coordinatorMode, signal: options.signal ?? ctx.signal, @@ -314,38 +305,6 @@ export async function applyTrustedConfigurationWithEvidence( ); } -/** Full-snapshot compensation; compare the applied generation, NOT the manifest authority. */ -export async function applyTrustedConfigurationSnapshot(options: { - payload: TrustedConfigurationPayload; - expectedCurrentRevisionId: string | null; - operationKey: string; - signal?: AbortSignal; - ctx?: JobContext; -}): Promise { - if ( - !configurationGenerationRevisionSchema.nullable().safeParse(options.expectedCurrentRevisionId) - .success - ) { - return { - outcome: "not_committed", - reason: "failed", - cleanupFailed: false, - publisherCleanupFailed: false, - registryUpdateFailed: false, - }; - } - return applySnapshot( - { - kind: "stack.render", - operationKey: options.operationKey, - signal: options.signal, - ctx: options.ctx, - }, - options.payload, - options.expectedCurrentRevisionId, - ); -} - /** Why the agent did not commit `revisionId`, when the operator can act on it; null otherwise or unknown. */ async function readConfigurationFailure( revisionId: string, @@ -368,21 +327,6 @@ async function readConfigurationFailure( } } -async function applySnapshot( - options: ApplyTrustedConfigurationOptions, - input: TrustedConfigurationPayload, - expectedCurrentRevisionId: string | null, -): Promise { - if (!options.ctx) throw new TrustedConfigurationUpgradeRequired(); - return ( - await applyTrustedConfigurationWithEvidence({ - ...options, - expectedCurrentRevisionId, - snapshot: trustedConfigurationPayloadSchema.parse(input), - }) - ).outcome; -} - /** The config schema of every loaded integration, by id. */ export function integrationSchemas(): Map> { return new Map( diff --git a/apps/ops-agent/src/catalog-approval-runtime.ts b/apps/ops-agent/src/catalog-approval-runtime.ts index 94e855a68..36bf66385 100644 --- a/apps/ops-agent/src/catalog-approval-runtime.ts +++ b/apps/ops-agent/src/catalog-approval-runtime.ts @@ -263,7 +263,6 @@ export function installCatalogApprovalRuntime( own.installation, "effects", `${activation.promotion.execution.operationKey}.json`, - 65536, ), catalogPromotionEffectSchema, ); diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index 0b281d544..cdbadc159 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -3632,7 +3632,6 @@ export async function closeOwnedConfigurationMaintenanceWithoutGuard( state.installation, "effects", `${state.execution.operationKey}.json`, - 65536, ) as { payload: unknown }; withConfigurationMaintenanceNoGuardPublication(authority, () => { const current = read(authority), @@ -4949,36 +4948,7 @@ export function registerConfigurationMaintenanceSource(runtime: OpsRuntime, sour /* Lookup of that original immutable tuple only; never retry POST. */ } } - const state = executionAuthorityState(authority), - issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: operation.maintenanceId, - candidateId: operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: issuedAt + 15000, - }, - ); - // A lost COMMIT acknowledgement may precede DM's shared terminal write. - // Its authenticated GET repairs publication only from the original atomic - // SQL tuple before the sender tries to copy that terminal. No POST/replay. - await inspectDataConfigurationFence(operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal: context.signal, - }); - observeConfigurationFenceSettlementControl(settlement); - // This second read lets DM retain the sender's now-durable copy ACK. - await inspectDataConfigurationFence(operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal: context.signal, - }); - await releaseOwnedConfigurationMaintenanceRoot(authority); + await observeCommittedMaintenanceSettlement(authority, context.signal); return completeConfigurationMaintenanceSettleControl( context.configurationMaintenanceControl!, operation, diff --git a/apps/ops-agent/src/integration-store.test.ts b/apps/ops-agent/src/integration-store.test.ts index 012f33250..37d9fdb86 100644 --- a/apps/ops-agent/src/integration-store.test.ts +++ b/apps/ops-agent/src/integration-store.test.ts @@ -21,7 +21,7 @@ import { import { createControlRootOperationKey, type OpsResourcePolicy } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { packageIntegration } from "@openmapx/integration-framework/installer"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { IntegrationStore, installIntegrationRuntime } from "./integration-store"; import { openOpsJobJournal } from "./journal"; import { createUnavailableRuntime } from "./runtime"; @@ -97,11 +97,17 @@ function fixture(options: { downloadError?: Error } = {}): Fixture { async function artifact( f: Fixture, id: string, - options: { region?: string; strings?: Record } = {}, + options: { region?: string; strings?: Record; description?: string } = {}, ) { const source = mkdtempSync(join(tmpdir(), "openmapx-integration-source-")); roots.push(source); - writeFileSync(join(source, "manifest.json"), JSON.stringify(manifest(id, options.region))); + writeFileSync( + join(source, "manifest.json"), + JSON.stringify({ + ...manifest(id, options.region), + ...(options.description ? { description: options.description } : {}), + }), + ); if (options.strings) { mkdirSync(join(source, "strings")); for (const [locale, text] of Object.entries(options.strings)) { @@ -356,6 +362,78 @@ describe("integration store", () => { } }); + it("inspects a large integration manifest without allocating mutation authority", async () => { + const f = fixture(); + const description = "large integration description ".repeat(3000); + const source = await artifact(f, "weather-radar", { description }); + const infraDir = join(f.root, "infra/docker"); + mkdirSync(join(infraDir, "data"), { recursive: true, mode: 0o700 }); + await provisionControlInstallation(infraDir); + const installation = openControlInstallation({ + infraDir, + role: "ops-agent", + serviceIds: ["ops-agent"], + }); + const privateRoot = join(infraDir, "control-private/ops-agent"); + const before = readdirSync(privateRoot, { recursive: true }).sort(); + const journal = await openOpsJobJournal(services.agentStatePaths(infraDir).journal); + const now = new Date(); + const requestedDeadlineAt = now.getTime() + 60_000; + const runtime = createUnavailableRuntime(); + installIntegrationRuntime(runtime, f.store); + const app = buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": Buffer.alloc(32, 32).toString("base64url") }, + resourcePolicy: allowAllResources, + journal, + controlInstallation: installation, + now: () => now, + randomBytes: () => Buffer.alloc(18, 35), + runtime, + }); + try { + const admitted = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: { authorization: `Bearer ${apiToken}`, "x-ops-protocol-version": "2" }, + payload: { + version: 2, + execution: { requestedDeadlineAt }, + requestId: "ops1_largeInspection000", + operationKey: createControlRootOperationKey({ + notAfter: requestedDeadlineAt, + nonce: Buffer.alloc(16, 36), + }), + issuedAt: now.toISOString(), + expiresAt: new Date(now.getTime() + 20_000).toISOString(), + operation: { kind: "catalogIntegration.inspect", source }, + }, + }); + expect(admitted.statusCode, admitted.body).toBe(202); + const operationId = admitted.json().result.operationId as string; + await vi.waitFor(async () => { + const status = await app.inject({ + method: "GET", + url: `/v1/operations/${operationId}`, + headers: { authorization: `Bearer ${apiToken}`, "x-ops-protocol-version": "2" }, + }); + expect(status.statusCode).toBe(200); + expect(status.json().result).toMatchObject({ + state: "succeeded", + result: { integrationId: "weather-radar", valid: true, manifest: { description } }, + }); + expect(Buffer.byteLength(JSON.stringify(status.json().result.result))).toBeGreaterThan( + 65536, + ); + }); + expect(readdirSync(privateRoot, { recursive: true }).sort()).toEqual(before); + expect(existsSync(join(infraDir, "data/.admin-target-control"))).toBe(false); + expect(record(f, "weather-radar")).toBeUndefined(); + } finally { + await app.close(); + closeControlInstallation(installation); + } + }); + it("keeps the previous copy until settled and restores it on rollback", async () => { const f = fixture(); const first = await artifact(f, "weather-radar", { region: "eu" }); diff --git a/apps/web/src/components/admin/services/ArtifactTransfersSection.test.tsx b/apps/web/src/components/admin/services/ArtifactTransfersSection.test.tsx index 389f93b89..c8dc0ff38 100644 --- a/apps/web/src/components/admin/services/ArtifactTransfersSection.test.tsx +++ b/apps/web/src/components/admin/services/ArtifactTransfersSection.test.tsx @@ -2,7 +2,7 @@ import { createApiClient } from "@openmapx/core"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { useMemo } from "react"; import { afterEach, expect, it, vi } from "vitest"; -import { cleanup, fireEvent, render, screen, waitFor } from "@/test"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@/test"; import { ArtifactTransfersSection } from "./ArtifactTransfersSection"; const fixture = vi.hoisted(() => ({ url: "http://artifact.test" })); @@ -25,6 +25,136 @@ function view() { afterEach(() => { cleanup(); vi.unstubAllGlobals(); + delete (window as unknown as Record).showSaveFilePicker; +}); +const downloadable = { + stageId: "9b0f4718-062d-4b5f-8048-55381380aeac", + purpose: "prepared-artifact", + direction: "download", + state: "ready", + bytes: 4, + sha256: "a".repeat(64), + expiresAt: "2030-12-01T00:00:00.000Z", + cleanup: "pending", +}; +function transferResponses(items = [downloadable]) { + const content = vi.fn( + async () => + new Response(new Uint8Array([0, 255, 128, 7]), { + headers: { + "content-type": "application/octet-stream", + "content-length": "4", + "content-disposition": 'attachment; filename="openmapx-artifact.tar"', + "x-artifact-sha256": "a".repeat(64), + "cache-control": "no-store", + "x-content-type-options": "nosniff", + }, + }), + ); + vi.stubGlobal( + "fetch", + vi.fn(async (raw) => { + const url = String(raw); + if (url.endsWith("/content")) return content(); + return Response.json( + url.endsWith("capabilities") + ? { + limits: { + maxUploadBytes: 1024, + maxExpandedBytes: 2048, + maxStageBytes: 4096, + stageMs: 100000, + downloadRetentionMs: 10000, + }, + freeBytes: 8000, + freeInodes: 500, + reservedBytes: 4, + targets: [], + } + : { items, total: items.length, limit: 50, offset: 0 }, + ); + }), + ); + return content; +} +it("aborts a late save sink without downloading after the transfer view is disposed", async () => { + const content = transferResponses(); + let resolvePicker!: (handle: { + createWritable(): Promise<{ + write: ReturnType; + close: ReturnType; + abort: ReturnType; + }>; + }) => void; + const picker = vi.fn( + () => + new Promise((resolve) => { + resolvePicker = resolve; + }), + ); + Object.defineProperty(window, "showSaveFilePicker", { configurable: true, value: picker }); + const rendered = render(view()); + fireEvent.click(await screen.findByRole("button", { name: "Download bundle" })); + await waitFor(() => expect(picker).toHaveBeenCalledTimes(1)); + rendered.unmount(); + const writable = { write: vi.fn(), close: vi.fn(), abort: vi.fn(async () => undefined) }; + await act(async () => { + resolvePicker({ createWritable: async () => writable }); + }); + await waitFor(() => expect(writable.abort).toHaveBeenCalledTimes(1)); + expect(content).not.toHaveBeenCalled(); + expect(writable.close).not.toHaveBeenCalled(); +}); +it("enables cancellation for the sink download after its save picker resolves", async () => { + const content = transferResponses(); + let entered!: () => void, release!: () => void; + const writing = new Promise((resolve) => { + entered = resolve; + }); + const pending = new Promise((resolve) => { + release = resolve; + }); + const writable = { + write: vi.fn(async () => { + entered(); + await pending; + }), + close: vi.fn(), + abort: vi.fn(async () => { + release(); + }), + }; + Object.defineProperty(window, "showSaveFilePicker", { + configurable: true, + value: async () => ({ createWritable: async () => writable }), + }); + render(view()); + fireEvent.click(await screen.findByRole("button", { name: "Download bundle" })); + await act(async () => { + await writing; + }); + expect(screen.getByRole("button", { name: "Cancel byte transfer" })).not.toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Cancel byte transfer" })); + await waitFor(() => expect(writable.abort).toHaveBeenCalledTimes(1)); + expect(content).toHaveBeenCalledTimes(1); + expect(writable.close).not.toHaveBeenCalled(); +}); +it("keeps prepared validation and import actions off integration upload stages", async () => { + transferResponses([ + { ...downloadable, purpose: "integration-artifact", direction: "upload", state: "uploaded" }, + { + ...downloadable, + stageId: "eaa852c6-c4cf-40b4-89b3-e6f74a7b722a", + purpose: "integration-artifact", + direction: "upload", + }, + { ...downloadable, stageId: "bdc999c3-44fa-4a5f-834e-06851f0bb93e", direction: "upload" }, + ]); + render(view()); + await screen.findByText(/bdc999c3-44fa-4a5f-834e-06851f0bb93e/); + expect(screen.getAllByRole("button", { name: "Validate" })).toHaveLength(1); + expect(screen.getAllByRole("button", { name: "Preview import" })).toHaveLength(1); + expect(screen.getAllByRole("button", { name: "Delete stage" })).toHaveLength(3); }); it("shows deployment limits and unavailable export reasons without offering a doomed action", async () => { vi.stubGlobal( diff --git a/apps/web/src/components/admin/services/ArtifactTransfersSection.tsx b/apps/web/src/components/admin/services/ArtifactTransfersSection.tsx index dd16dd082..2ea04adb2 100644 --- a/apps/web/src/components/admin/services/ArtifactTransfersSection.tsx +++ b/apps/web/src/components/admin/services/ArtifactTransfersSection.tsx @@ -42,6 +42,7 @@ function ArtifactTransfersView({ apiUrl }: { apiUrl: string }) { fileInput = useRef(null); const [file, setFile] = useState(null), [busy, setBusy] = useState(false), + [canCancel, setCanCancel] = useState(false), [error, setError] = useState(null); const [progress, setProgress] = useState<{ receivedBytes: number; state: string } | null>(null), [message, setMessage] = useState(null); @@ -99,6 +100,7 @@ function ArtifactTransfersView({ apiUrl }: { apiUrl: string }) { const capturedFile = file, controller = new AbortController(); byteRequest.current = { owner, controller }; + setCanCancel(true); try { const stage = sdk.unwrapAdminResponse( await sdk.createAdminArtifactTransfer( @@ -124,6 +126,7 @@ function ArtifactTransfersView({ apiUrl }: { apiUrl: string }) { } } finally { if (byteRequest.current?.controller === controller) byteRequest.current = null; + if (owner.active) setCanCancel(false); } }); const download = (stage: TransferStage) => @@ -149,10 +152,15 @@ function ArtifactTransfersView({ apiUrl }: { apiUrl: string }) { "This connection requires a browser with a file-save sink. Reconnect through the same-origin admin panel or use the CLI.", ); if (!owner.active) return; - const sink = await createArtifactFileSink(), - controller = new AbortController(); + const controller = new AbortController(); byteRequest.current = { owner, controller }; + setCanCancel(true); try { + const sink = await createArtifactFileSink(); + if (!owner.active || controller.signal.aborted) { + await sink.abort(controller.signal.reason); + return; + } await downloadArtifactToSink({ client, stageId: stage.stageId, @@ -162,6 +170,7 @@ function ArtifactTransfersView({ apiUrl }: { apiUrl: string }) { if (owner.active) setMessage("Download saved."); } finally { if (byteRequest.current?.controller === controller) byteRequest.current = null; + if (owner.active) setCanCancel(false); } }); const target = capabilities.data?.targets.find((item) => item.serviceId === selected); @@ -224,10 +233,7 @@ function ArtifactTransfersView({ apiUrl }: { apiUrl: string }) { Upload bundle {busy && ( - )} @@ -317,40 +323,44 @@ function ArtifactTransfersView({ apiUrl }: { apiUrl: string }) { Expires {new Date(stage.expiresAt).toLocaleString()} · cleanup {stage.cleanup} - {stage.direction === "upload" && ["uploaded", "ready"].includes(stage.state) && ( - - )} - {stage.direction === "upload" && stage.state === "ready" && ( - - )} + {stage.purpose === "prepared-artifact" && + stage.direction === "upload" && + ["uploaded", "ready"].includes(stage.state) && ( + + )} + {stage.purpose === "prepared-artifact" && + stage.direction === "upload" && + stage.state === "ready" && ( + + )} {stage.direction === "download" && stage.state === "ready" && (